From 118f1fc726524c7ce728cc1474b7679fa4168177 Mon Sep 17 00:00:00 2001 From: maxiao1 Date: Sat, 13 Sep 2025 17:00:20 +0800 Subject: [PATCH] sglangv0.5.2 & support Qwen3-Next-80B-A3B-Instruct --- .clang-format-ignore | 1 + .devcontainer/Dockerfile | 35 + .devcontainer/devcontainer.json | 30 + .editorconfig | 25 + .github/CODEOWNERS | 21 + .github/ISSUE_TEMPLATE/1-bug-report.yml | 38 + .github/ISSUE_TEMPLATE/2-feature-request.yml | 23 + .github/REVIEWERS.md | 53 + .github/pull_request_template.md | 24 + .../cancel-all-pending-pr-test-runs.yml | 45 + .../workflows/cancel-pr-workflow-on-merge.yml | 22 + .github/workflows/close-inactive-issues.yml | 96 + .github/workflows/execute-notebook.yml | 60 + .github/workflows/experiment-runner.yml | 30 + .github/workflows/lint.yml | 22 + .github/workflows/nightly-test-amd.yml | 41 + .github/workflows/nightly-test.yml | 33 + .github/workflows/open-pr-copy-from-oss.yml | 28 + .github/workflows/open-pr-copy-to-oss.yml | 31 + .github/workflows/pr-benchmark-rust.yml | 306 ++ .github/workflows/pr-test-amd.yml | 377 ++ .github/workflows/pr-test-h20.yml | 81 + .github/workflows/pr-test-npu.yml | 184 + .github/workflows/pr-test-pd-router.yml | 599 ++++ .github/workflows/pr-test-rust.yml | 190 + .github/workflows/pr-test-sgl-kernel.yml | 151 + .github/workflows/pr-test-xeon.yml | 106 + .github/workflows/pr-test.yml | 437 +++ .../workflows/release-docker-amd-nightly.yml | 65 + .github/workflows/release-docker-amd.yml | 56 + .github/workflows/release-docker-dev.yml | 49 + .github/workflows/release-docker-gb200.yml | 36 + .../workflows/release-docker-npu-nightly.yml | 78 + .github/workflows/release-docker-npu.yml | 74 + .github/workflows/release-docker-router.yml | 30 + .github/workflows/release-docker-xeon.yml | 35 + .github/workflows/release-docker.yml | 97 + .github/workflows/release-docs.yml | 65 + .github/workflows/release-fake-tag.yml | 35 + .github/workflows/release-pypi-router.yml | 119 + .github/workflows/release-pypi.yml | 31 + .../workflows/release-whl-kernel-cu118.yml | 92 + .github/workflows/release-whl-kernel.yml | 283 ++ .github/workflows/vllm-dependency-test.yml | 43 + .gitignore | 240 ++ .isort.cfg | 3 + .pre-commit-config.yaml | 59 + 3rdparty/amd/profiling/PROFILING.md | 425 +++ 3rdparty/amd/profiling/client.sh | 27 + 3rdparty/amd/profiling/install_rpd.sh | 10 + 3rdparty/amd/profiling/loadTracer.sh | 43 + 3rdparty/amd/profiling/rpd.patch | 12 + .../profiling/rpd_profile_server_enable.patch | 49 + ...rofile_server_enable_wCPU_activities.patch | 126 + 3rdparty/amd/profiling/server.sh | 20 + 3rdparty/amd/profiling/torch_profiler.patch | 25 + 3rdparty/amd/tuning/TUNING.md | 118 + 3rdparty/amd/tuning/benchmark_moe_rocm.py | 380 ++ LICENSE | 201 ++ Makefile | 46 + README.md | 78 + assets/logo.png | Bin 0 -> 402692 bytes assets/logo.svg | 1 + assets/logo_square.svg | 1 + .../bench_attention_sink_triton.py | 250 ++ .../bench_in_batch_prefix.py | 130 + benchmark/benchmark_batch/benchmark_batch.py | 193 ++ .../benchmark_batch/benchmark_tokenizer.py | 126 + benchmark/benchmark_vllm_060/README.md | 89 + benchmark/blog_v0_2/405b_sglang.sh | 24 + benchmark/blog_v0_2/405b_trt.sh | 17 + benchmark/blog_v0_2/405b_vllm.sh | 24 + benchmark/blog_v0_2/README.md | 164 + benchmark/blog_v0_2/config.md | 100 + benchmark/boolq/README.md | 19 + benchmark/boolq/bench_sglang.py | 124 + benchmark/boolq/convert_parquet_to_json.py | 28 + benchmark/boolq/parquet_to_json.sh | 26 + benchmark/ceval/README.md | 15 + benchmark/ceval/bench_sglang.py | 138 + benchmark/deepseek_v3/README.md | 373 ++ benchmark/dspy/README.md | 51 + benchmark/dspy/bench_dspy_intro.py | 192 ++ benchmark/generative_agents/README.md | 38 + .../generative_agents/agent_functions.py | 300 ++ benchmark/generative_agents/bench_other.py | 80 + benchmark/generative_agents/bench_sglang.py | 74 + benchmark/gpt_oss/README.md | 163 + benchmark/gsm8k/README.md | 47 + benchmark/gsm8k/bench_other.py | 151 + benchmark/gsm8k/bench_sglang.py | 148 + benchmark/hellaswag/README.md | 47 + benchmark/hellaswag/bench_other.py | 118 + benchmark/hellaswag/bench_sglang.py | 109 + benchmark/hf3fs/bench.sh | 59 + benchmark/hf3fs/bench_client.py | 162 + benchmark/hf3fs/bench_storage.py | 258 ++ benchmark/hf3fs/bench_zerocopy.py | 140 + benchmark/hicache/README.md | 91 + benchmark/hicache/bench_long_context.py | 101 + benchmark/hicache/bench_mix.py | 567 +++ benchmark/hicache/bench_mix.sh | 42 + benchmark/hicache/bench_multiturn.py | 517 +++ benchmark/hicache/bench_serving.py | 1029 ++++++ benchmark/hicache/data_processing.py | 590 ++++ benchmark/hicache/download.sh | 66 + benchmark/hicache/nextqa.py | 159 + benchmark/json_decode_regex/README.md | 60 + benchmark/json_decode_regex/bench_other.py | 98 + benchmark/json_decode_regex/bench_sglang.py | 101 + benchmark/json_decode_regex/build_dataset.py | 58 + benchmark/json_jump_forward/README.md | 88 + benchmark/json_jump_forward/bench_other.py | 288 ++ benchmark/json_jump_forward/bench_sglang.py | 143 + benchmark/json_jump_forward/build_dataset.py | 58 + benchmark/json_jump_forward/dataset.txt | 50 + benchmark/json_schema/README.md | 15 + benchmark/json_schema/bench_sglang.py | 146 + .../kernels/all_reduce/benchmark_mscclpp.py | 224 ++ .../triton_flashinfer_cudnn.py | 403 +++ benchmark/kernels/deepep/deepep_utils.py | 218 ++ benchmark/kernels/deepep/tuning_deepep.py | 476 +++ benchmark/kernels/deepseek/README.md | 19 + .../deepseek/benchmark_deepgemm_fp8_gemm.py | 400 +++ .../benchmark_deepgemm_fp8_group_gemm.py | 486 +++ benchmark/kernels/fbgemm/README.md | 29 + .../fbgemm/benchmark_fbgemm_grouped_gemm.py | 516 +++ .../flashinfer_allreduce_fusion/README.md | 102 + .../benchmark_fused_collective.py | 1304 +++++++ benchmark/kernels/fused_moe_triton/README.md | 76 + .../benchmark_sglang_fused_moe_triton.py | 292 ++ .../fused_moe_triton/benchmark_sum_scale.py | 202 ++ .../benchmark_torch_compile_fused_moe.py | 305 ++ ...nchmark_vllm_vs_sglang_fused_moe_triton.py | 349 ++ .../tuning_fused_moe_triton.py | 599 ++++ .../benchmark_lightning_attention_decode.py | 576 ++++ .../benchmark_lightning_attention_prefill.py | 603 ++++ .../kernels/quantization/bench_fp4_quant.py | 133 + .../kernels/quantization/bench_int8_quant.py | 94 + .../quantization/tuning_block_wise_kernel.py | 474 +++ .../kernels/rmsnorm/benchmark_rmsnorm.py | 230 ++ .../benchmark_get_last_loc_triton.py | 169 + ...enchmark_write_req_to_token_pool_triton.py | 342 ++ .../bench_triton_swa_kernel.py | 283 ++ benchmark/line_retrieval/README.md | 37 + benchmark/line_retrieval/bench_sglang.py | 149 + benchmark/line_retrieval/gen_data.py | 139 + benchmark/llava_bench/README.md | 61 + benchmark/llava_bench/bench_hf_llava_bench.sh | 9 + benchmark/llava_bench/bench_hf_mme.sh | 9 + benchmark/llava_bench/bench_sglang.py | 96 + benchmark/llava_bench/bench_sglang_mme.sh | 2 + benchmark/llava_bench/download_images.py | 20 + benchmark/llm_judge/README.md | 33 + benchmark/llm_judge/bench_other.py | 151 + benchmark/llm_judge/bench_sglang.py | 97 + benchmark/long_json_decode/README.md | 33 + benchmark/long_json_decode/bench_other.py | 89 + benchmark/long_json_decode/bench_sglang.py | 81 + benchmark/long_json_decode/build_dataset.py | 27 + benchmark/lora/launch_server.py | 75 + benchmark/lora/lora_bench.py | 476 +++ benchmark/mmlu/README.md | 59 + benchmark/mmlu/bench_other.py | 173 + benchmark/mmlu/bench_sglang.py | 181 + benchmark/mmlu/download_data.sh | 2 + benchmark/mmmu/README.md | 46 + benchmark/mmmu/bench_hf.py | 164 + benchmark/mmmu/bench_sglang.py | 212 ++ benchmark/mmmu/data_utils.py | 221 ++ benchmark/mmmu/eval_utils.py | 649 ++++ benchmark/mmmu/internvl_utils.py | 94 + benchmark/mmmu/prompt_format.yaml | 15 + benchmark/mtbench/README.md | 48 + benchmark/mtbench/bench_other.py | 111 + benchmark/mtbench/bench_sglang.py | 99 + benchmark/mtbench/bench_sglang_eagle.py | 137 + benchmark/multi_chain_reasoning/README.md | 49 + .../multi_chain_reasoning/bench_other.py | 186 + .../multi_chain_reasoning/bench_sglang.py | 140 + benchmark/multi_document_qa/README.md | 47 + benchmark/multi_document_qa/bench_other.py | 114 + benchmark/multi_document_qa/bench_sglang.py | 93 + benchmark/multi_document_qa/build_dataset.py | 70 + benchmark/multi_turn_chat/README.md | 66 + benchmark/multi_turn_chat/bench_other.py | 93 + benchmark/multi_turn_chat/bench_sglang.py | 79 + benchmark/multi_turn_chat/data_gen.py | 29 + .../multi_turn_chat/long_prompt_multi_turn.py | 129 + benchmark/prefill_only/bench_embeddings.py | 148 + benchmark/prefill_only/bench_score.py | 192 ++ benchmark/prefill_only/util.py | 813 +++++ benchmark/react/README.md | 34 + benchmark/react/bench_other.py | 202 ++ benchmark/react/bench_sglang.py | 153 + benchmark/reasoning_benchmark/README.md | 77 + .../reasoning_benchmark/answer_extraction.py | 269 ++ benchmark/reasoning_benchmark/bench_sglang.py | 135 + benchmark/reasoning_benchmark/eval_utils.py | 206 ++ benchmark/tip_suggestion/.gitignore | 1 + benchmark/tip_suggestion/README.md | 33 + benchmark/tip_suggestion/bench_other.py | 133 + benchmark/tip_suggestion/bench_sglang.py | 100 + benchmark/tip_suggestion/lmql_funcs.py | 50 + benchmark/tip_suggestion/topic.jsonl | 50 + benchmark/tree_of_thought_deep/README.md | 51 + benchmark/tree_of_thought_deep/bench_other.py | 222 ++ .../tree_of_thought_deep/bench_sglang.py | 171 + benchmark/tree_of_thought_deep/lmql_funcs.py | 82 + benchmark/tree_of_thought_v0/README.md | 43 + benchmark/tree_of_thought_v0/bench_other.py | 179 + benchmark/tree_of_thought_v0/bench_sglang.py | 159 + docker/Dockerfile | 388 +++ docker/Dockerfile.gb200 | 348 ++ docker/Dockerfile.npu | 93 + docker/Dockerfile.rocm | 213 ++ docker/Dockerfile.router | 78 + docker/Dockerfile.sagemaker | 6 + docker/Dockerfile.xeon | 48 + docker/compose.yaml | 35 + docker/k8s-sglang-distributed-sts.yaml | 103 + docker/k8s-sglang-service.yaml | 117 + docker/serve | 31 + docs/Makefile | 58 + docs/README.md | 55 + docs/_static/css/custom_log.css | 29 + docs/_static/css/readthedocs.css | 9 + docs/_static/image/logo.ico | Bin 0 -> 38078 bytes docs/_static/image/logo.png | Bin 0 -> 402692 bytes docs/advanced_features/attention_backend.md | 104 + .../hyperparameter_tuning.md | 77 + docs/advanced_features/lora.ipynb | 529 +++ docs/advanced_features/observability.md | 35 + docs/advanced_features/pd_disaggregation.md | 150 + docs/advanced_features/quantization.md | 152 + docs/advanced_features/router.md | 445 +++ .../separate_reasoning.ipynb | 381 ++ docs/advanced_features/server_arguments.md | 318 ++ .../speculative_decoding.ipynb | 370 ++ .../structured_outputs.ipynb | 853 +++++ ...uctured_outputs_for_reasoning_models.ipynb | 830 +++++ docs/advanced_features/tool_parser.ipynb | 852 +++++ docs/advanced_features/vlm_query.ipynb | 325 ++ docs/basic_usage/deepseek.md | 228 ++ docs/basic_usage/gpt_oss.md | 114 + docs/basic_usage/llama4.md | 61 + docs/basic_usage/native_api.ipynb | 497 +++ docs/basic_usage/offline_engine_api.ipynb | 235 ++ docs/basic_usage/openai_api.rst | 9 + docs/basic_usage/openai_api_completions.ipynb | 389 +++ docs/basic_usage/openai_api_embeddings.ipynb | 195 ++ docs/basic_usage/openai_api_vision.ipynb | 254 ++ docs/basic_usage/qwen3.md | 27 + docs/basic_usage/sampling_params.md | 305 ++ docs/basic_usage/send_request.ipynb | 253 ++ docs/conf.py | 205 ++ docs/deploy.py | 22 + docs/developer_guide/bench_serving.md | 334 ++ .../benchmark_and_profiling.md | 182 + docs/developer_guide/contribution_guide.md | 103 + .../development_guide_using_docker.md | 108 + docs/developer_guide/release_process.md | 18 + docs/developer_guide/setup_github_runner.md | 49 + docs/get_started/install.md | 131 + docs/index.rst | 95 + docs/platforms/amd_gpu.md | 158 + docs/platforms/ascend_npu.md | 206 ++ docs/platforms/blackwell_gpu.md | 9 + docs/platforms/cpu_server.md | 204 ++ docs/platforms/nvidia_jetson.md | 80 + docs/platforms/tpu.md | 3 + docs/references/custom_chat_template.md | 42 + docs/references/environment_variables.md | 97 + docs/references/faq.md | 35 + docs/references/frontend/choices_methods.md | 77 + docs/references/frontend/frontend_index.rst | 9 + .../frontend/frontend_tutorial.ipynb | 456 +++ docs/references/learn_more.md | 7 + .../multi_node_deployment/deploy_on_k8s.md | 337 ++ .../lws_pd/lws-examples/d-svc.yaml | 12 + .../lws_pd/lws-examples/d.yaml | 290 ++ .../lws_pd/lws-examples/lb.yaml | 56 + .../lws_pd/lws-examples/p-svc.yaml | 12 + .../lws_pd/lws-examples/p.yaml | 304 ++ .../lws_pd/lws_pd_deploy.md | 783 +++++ .../multi_node_deployment/multi_node.md | 90 + .../multi_node_index.rst | 13 + docs/references/production_metrics.md | 217 ++ docs/references/torch_compile_cache.md | 13 + docs/requirements.txt | 20 + docs/serve.sh | 3 + docs/supported_models/embedding_models.md | 87 + docs/supported_models/generative_models.md | 56 + docs/supported_models/modelscope.md | 28 + .../multimodal_language_models.md | 42 + docs/supported_models/rerank_models.md | 49 + docs/supported_models/reward_models.md | 28 + docs/supported_models/support_new_models.md | 320 ++ .../supported_models/transformers_fallback.md | 58 + docs/wrap_run_llm.py | 47 + .../tool_chat_template_deepseekr1.jinja | 92 + .../tool_chat_template_deepseekv3.jinja | 91 + .../tool_chat_template_deepseekv31.jinja | 91 + .../tool_chat_template_llama4_pythonic.jinja | 112 + .../quick_start/anthropic_example_chat.py | 73 + .../quick_start/anthropic_example_complete.py | 68 + .../quick_start/azure_openai_example_chat.py | 83 + .../quick_start/gemini_example_chat.py | 73 + .../quick_start/gemini_example_complete.py | 68 + .../gemini_example_multimodal_chat.py | 30 + .../quick_start/images/cat.jpeg | Bin 0 -> 345163 bytes .../quick_start/images/dog.jpeg | Bin 0 -> 416580 bytes .../quick_start/local_example_chat.py | 75 + .../quick_start/local_example_complete.py | 70 + .../quick_start/local_example_llava_next.py | 78 + .../quick_start/openai_example_chat.py | 74 + .../quick_start/openai_example_complete.py | 68 + .../quick_start/openai_example_n.py | 71 + .../quick_start/openai_example_o1.py | 57 + .../quick_start/openrouter_example_chat.py | 81 + .../quick_start/together_example_chat.py | 81 + .../quick_start/together_example_complete.py | 76 + .../frontend_language/usage/chinese_regex.py | 53 + .../usage/choices_logprob.py | 44 + .../frontend_language/usage/cot_decoding.py | 115 + .../frontend_language/usage/json_decode.py | 83 + .../frontend_language/usage/json_logprobs.py | 103 + .../usage/llava_video/srt_example_llava_v.py | 260 ++ .../usage/llava_video/srt_example_llava_v.sh | 131 + .../usage/openai_chat_speculative.py | 155 + .../usage/openai_speculative.py | 54 + .../usage/parallel_sample.py | 40 + .../trace_and_evaluate_rag_using_parea.ipynb | 408 +++ .../usage/readme_examples.py | 109 + .../usage/sgl_gen_min_tokens.py | 35 + examples/frontend_language/usage/streaming.py | 49 + .../frontend_language/usage/triton/Dockerfile | 10 + .../frontend_language/usage/triton/README.md | 35 + .../models/character_generation/config.pbtxt | 23 + examples/monitoring/README.md | 76 + examples/monitoring/docker-compose.yaml | 28 + .../grafana/dashboards/config/dashboard.yaml | 11 + .../dashboards/json/sglang-dashboard.json | 984 ++++++ .../grafana/datasources/datasource.yaml | 8 + examples/monitoring/prometheus.yaml | 10 + .../profiler/nsys_profile_tools/README.md | 176 + .../nsys_profile_tools/gputrc2graph.py | 344 ++ .../sglang_engine_model.json | 61 + examples/runtime/README.md | 45 + examples/runtime/engine/custom_server.py | 53 + examples/runtime/engine/embedding.py | 27 + .../engine/fastapi_engine_inference.py | 189 + examples/runtime/engine/launch_engine.py | 17 + .../runtime/engine/offline_batch_inference.py | 43 + .../engine/offline_batch_inference_async.py | 65 + .../engine/offline_batch_inference_eagle.py | 38 + .../engine/offline_batch_inference_qwen_1m.py | 74 + .../engine/offline_batch_inference_vlm.py | 52 + examples/runtime/engine/readme.md | 54 + examples/runtime/engine/save_remote_state.py | 59 + examples/runtime/engine/save_sharded_state.py | 74 + .../hidden_states/hidden_states_engine.py | 66 + .../hidden_states/hidden_states_server.py | 81 + examples/runtime/lora.py | 37 + .../runtime/multimodal/llama3_llava_server.py | 111 + .../multimodal/llava_onevision_server.py | 264 ++ examples/runtime/multimodal/pixtral_server.py | 127 + .../runtime/multimodal/qwen_llava_server.py | 111 + examples/runtime/multimodal_embedding.py | 18 + .../openai_chat_with_response_prefill.py | 53 + examples/runtime/reward_model.py | 32 + .../token_in_token_out_llm_engine.py | 43 + .../token_in_token_out_llm_server.py | 68 + .../token_in_token_out_vlm_engine.py | 74 + .../token_in_token_out_vlm_server.py | 78 + examples/runtime/vertex_predict.py | 66 + package-lock.json | 6 + python/pyproject.toml | 167 + python/sglang/README.md | 16 + python/sglang/__init__.py | 83 + python/sglang/bench_offline_throughput.py | 452 +++ python/sglang/bench_one_batch.py | 665 ++++ python/sglang/bench_one_batch_server.py | 439 +++ python/sglang/bench_serving.py | 2403 +++++++++++++ python/sglang/check_env.py | 305 ++ python/sglang/compile_deep_gemm.py | 184 + python/sglang/eval/llama3_eval.py | 315 ++ python/sglang/eval/loogle_eval.py | 164 + python/sglang/global_config.py | 53 + python/sglang/lang/api.py | 286 ++ python/sglang/lang/backend/anthropic.py | 73 + python/sglang/lang/backend/base_backend.py | 82 + python/sglang/lang/backend/litellm.py | 90 + python/sglang/lang/backend/openai.py | 475 +++ .../sglang/lang/backend/runtime_endpoint.py | 527 +++ python/sglang/lang/backend/vertexai.py | 148 + python/sglang/lang/chat_template.py | 662 ++++ python/sglang/lang/choices.py | 164 + python/sglang/lang/compiler.py | 231 ++ python/sglang/lang/interpreter.py | 1060 ++++++ python/sglang/lang/ir.py | 635 ++++ python/sglang/lang/tracer.py | 279 ++ python/sglang/launch_server.py | 16 + python/sglang/profiler.py | 166 + python/sglang/srt/_custom_ops.py | 177 + python/sglang/srt/aio_rwlock.py | 100 + python/sglang/srt/bench_utils.py | 137 + python/sglang/srt/configs/__init__.py | 29 + python/sglang/srt/configs/chatglm.py | 78 + python/sglang/srt/configs/dbrx.py | 279 ++ python/sglang/srt/configs/deepseekvl2.py | 688 ++++ python/sglang/srt/configs/device_config.py | 17 + python/sglang/srt/configs/exaone.py | 195 ++ python/sglang/srt/configs/internvl.py | 706 ++++ python/sglang/srt/configs/janus_pro.py | 634 ++++ python/sglang/srt/configs/kimi_vl.py | 38 + python/sglang/srt/configs/kimi_vl_moonvit.py | 32 + python/sglang/srt/configs/load_config.py | 89 + python/sglang/srt/configs/longcat_flash.py | 104 + python/sglang/srt/configs/model_config.py | 811 +++++ python/sglang/srt/configs/qwen3_next.py | 326 ++ python/sglang/srt/configs/step3_vl.py | 172 + python/sglang/srt/configs/update_config.py | 156 + python/sglang/srt/configs/utils.py | 25 + python/sglang/srt/connector/__init__.py | 51 + python/sglang/srt/connector/base_connector.py | 111 + python/sglang/srt/connector/redis.py | 85 + python/sglang/srt/connector/s3.py | 122 + python/sglang/srt/connector/serde/__init__.py | 31 + .../sglang/srt/connector/serde/safe_serde.py | 30 + python/sglang/srt/connector/serde/serde.py | 43 + python/sglang/srt/connector/utils.py | 35 + python/sglang/srt/constants.py | 3 + .../srt/constrained/base_grammar_backend.py | 213 ++ .../srt/constrained/llguidance_backend.py | 174 + .../srt/constrained/outlines_backend.py | 191 ++ .../srt/constrained/outlines_jump_forward.py | 200 ++ .../constrained/reasoner_grammar_backend.py | 90 + .../srt/constrained/triton_ops/bitmask_ops.py | 141 + .../srt/constrained/xgrammar_backend.py | 239 ++ python/sglang/srt/custom_op.py | 102 + python/sglang/srt/debug_utils/__init__.py | 0 .../sglang/srt/debug_utils/dump_comparator.py | 168 + python/sglang/srt/debug_utils/dump_loader.py | 97 + python/sglang/srt/debug_utils/dumper.py | 116 + .../sglang/srt/debug_utils/text_comparator.py | 234 ++ .../srt/disaggregation/ascend/__init__.py | 6 + .../sglang/srt/disaggregation/ascend/conn.py | 117 + .../disaggregation/ascend/transfer_engine.py | 58 + .../srt/disaggregation/base/__init__.py | 8 + python/sglang/srt/disaggregation/base/conn.py | 134 + .../srt/disaggregation/common/__init__.py | 5 + .../sglang/srt/disaggregation/common/conn.py | 438 +++ .../sglang/srt/disaggregation/common/utils.py | 42 + python/sglang/srt/disaggregation/decode.py | 894 +++++ .../decode_schedule_batch_mixin.py | 159 + .../srt/disaggregation/fake/__init__.py | 1 + python/sglang/srt/disaggregation/fake/conn.py | 85 + python/sglang/srt/disaggregation/kv_events.py | 412 +++ python/sglang/srt/disaggregation/mini_lb.py | 6 + .../srt/disaggregation/mooncake/__init__.py | 6 + .../srt/disaggregation/mooncake/conn.py | 1704 +++++++++ .../mooncake/transfer_engine.py | 164 + .../srt/disaggregation/nixl/__init__.py | 6 + python/sglang/srt/disaggregation/nixl/conn.py | 696 ++++ python/sglang/srt/disaggregation/prefill.py | 867 +++++ python/sglang/srt/disaggregation/utils.py | 329 ++ python/sglang/srt/distributed/__init__.py | 3 + .../srt/distributed/communication_op.py | 35 + .../device_communicators/cuda_wrapper.py | 183 + .../device_communicators/custom_all_reduce.py | 421 +++ .../custom_all_reduce_utils.py | 386 +++ .../device_communicators/hpu_communicator.py | 49 + .../device_communicators/npu_communicator.py | 39 + .../device_communicators/pymscclpp.py | 315 ++ .../device_communicators/pynccl.py | 341 ++ .../device_communicators/pynccl_allocator.py | 133 + .../device_communicators/pynccl_wrapper.py | 563 +++ .../device_communicators/quick_all_reduce.py | 273 ++ .../device_communicators/shm_broadcast.py | 509 +++ .../device_communicators/xpu_communicator.py | 48 + .../srt/distributed/naive_distributed.py | 112 + .../sglang/srt/distributed/parallel_state.py | 1776 ++++++++++ python/sglang/srt/distributed/utils.py | 224 ++ python/sglang/srt/entrypoints/EngineBase.py | 72 + python/sglang/srt/entrypoints/context.py | 246 ++ python/sglang/srt/entrypoints/engine.py | 872 +++++ .../sglang/srt/entrypoints/harmony_utils.py | 372 ++ python/sglang/srt/entrypoints/http_server.py | 1398 ++++++++ .../srt/entrypoints/http_server_engine.py | 142 + .../sglang/srt/entrypoints/openai/__init__.py | 0 .../sglang/srt/entrypoints/openai/protocol.py | 929 +++++ .../srt/entrypoints/openai/serving_base.py | 156 + .../srt/entrypoints/openai/serving_chat.py | 1083 ++++++ .../entrypoints/openai/serving_completions.py | 455 +++ .../entrypoints/openai/serving_embedding.py | 174 + .../srt/entrypoints/openai/serving_rerank.py | 102 + .../entrypoints/openai/serving_responses.py | 1276 +++++++ .../srt/entrypoints/openai/serving_score.py | 61 + .../srt/entrypoints/openai/tool_server.py | 175 + .../srt/entrypoints/openai/usage_processor.py | 81 + python/sglang/srt/entrypoints/openai/utils.py | 72 + python/sglang/srt/entrypoints/tool.py | 87 + python/sglang/srt/eplb/__init__.py | 0 .../srt/eplb/eplb_algorithms/__init__.py | 63 + .../srt/eplb/eplb_algorithms/deepseek.py | 223 ++ .../srt/eplb/eplb_algorithms/deepseek_vec.py | 276 ++ python/sglang/srt/eplb/eplb_manager.py | 118 + .../srt/eplb/eplb_simulator/__init__.py | 1 + .../sglang/srt/eplb/eplb_simulator/reader.py | 51 + python/sglang/srt/eplb/expert_distribution.py | 966 ++++++ python/sglang/srt/eplb/expert_location.py | 468 +++ .../srt/eplb/expert_location_dispatch.py | 109 + .../srt/eplb/expert_location_updater.py | 575 ++++ .../srt/function_call/base_format_detector.py | 367 ++ python/sglang/srt/function_call/core_types.py | 34 + .../srt/function_call/deepseekv31_detector.py | 222 ++ .../srt/function_call/deepseekv3_detector.py | 220 ++ .../sglang/srt/function_call/ebnf_composer.py | 344 ++ .../srt/function_call/function_call_parser.py | 214 ++ .../srt/function_call/glm4_moe_detector.py | 164 + .../srt/function_call/gpt_oss_detector.py | 219 ++ .../srt/function_call/kimik2_detector.py | 245 ++ .../srt/function_call/llama32_detector.py | 96 + .../srt/function_call/mistral_detector.py | 139 + .../srt/function_call/pythonic_detector.py | 231 ++ .../srt/function_call/qwen25_detector.py | 130 + .../srt/function_call/qwen3_coder_detector.py | 362 ++ .../srt/function_call/step3_detector.py | 436 +++ python/sglang/srt/function_call/utils.py | 52 + python/sglang/srt/hf_transformers_utils.py | 442 +++ python/sglang/srt/host_shared_memory.py | 83 + python/sglang/srt/layers/activation.py | 272 ++ python/sglang/srt/layers/amx_utils.py | 86 + .../srt/layers/attention/aiter_backend.py | 1179 +++++++ .../srt/layers/attention/ascend_backend.py | 577 ++++ .../srt/layers/attention/base_attn_backend.py | 117 + .../layers/attention/cutlass_mla_backend.py | 285 ++ .../attention/double_sparsity_backend.py | 257 ++ .../dual_chunk_flashattention_backend.py | 1700 +++++++++ .../sglang/srt/layers/attention/fla/chunk.py | 242 ++ .../srt/layers/attention/fla/chunk_delta_h.py | 890 +++++ .../srt/layers/attention/fla/chunk_o.py | 178 + .../attention/fla/chunk_scaled_dot_kkt.py | 151 + .../sglang/srt/layers/attention/fla/cumsum.py | 300 ++ .../layers/attention/fla/fused_recurrent.py | 640 ++++ .../fla/fused_sigmoid_gating_recurrent.py | 232 ++ .../sglang/srt/layers/attention/fla/index.py | 37 + .../sglang/srt/layers/attention/fla/l2norm.py | 150 + .../layers/attention/fla/layernorm_gated.py | 326 ++ python/sglang/srt/layers/attention/fla/op.py | 66 + .../srt/layers/attention/fla/solve_tril.py | 465 +++ .../sglang/srt/layers/attention/fla/utils.py | 331 ++ .../srt/layers/attention/fla/wy_fast.py | 158 + .../attention/flashattention_backend.py | 2365 +++++++++++++ .../layers/attention/flashinfer_backend.py | 1441 ++++++++ .../attention/flashinfer_mla_backend.py | 1079 ++++++ .../srt/layers/attention/flashmla_backend.py | 541 +++ .../layers/attention/hybrid_attn_backend.py | 139 + .../attention/hybrid_linear_attn_backend.py | 584 ++++ .../srt/layers/attention/intel_amx_backend.py | 131 + .../layers/attention/mamba/causal_conv1d.py | 128 + .../attention/mamba/causal_conv1d_triton.py | 1052 ++++++ .../srt/layers/attention/mamba/mamba.py | 64 + .../srt/layers/attention/merge_state.py | 46 + .../srt/layers/attention/tbo_backend.py | 260 ++ .../layers/attention/torch_native_backend.py | 276 ++ .../srt/layers/attention/triton_backend.py | 1090 ++++++ .../attention/triton_ops/decode_attention.py | 776 +++++ .../triton_ops/double_sparsity_attention.py | 1106 ++++++ .../attention/triton_ops/extend_attention.py | 550 +++ .../attention/triton_ops/merge_state.py | 96 + .../attention/triton_ops/prefill_attention.py | 217 ++ .../triton_ops/rocm_mla_decode_rope.py | 439 +++ .../layers/attention/trtllm_mha_backend.py | 694 ++++ .../layers/attention/trtllm_mla_backend.py | 589 ++++ python/sglang/srt/layers/attention/utils.py | 99 + python/sglang/srt/layers/attention/vision.py | 630 ++++ .../srt/layers/attention/vision_utils.py | 65 + .../srt/layers/attention/wave_backend.py | 627 ++++ .../attention/wave_ops/decode_attention.py | 184 + .../attention/wave_ops/extend_attention.py | 147 + .../attention/wave_ops/prefill_attention.py | 79 + python/sglang/srt/layers/communicator.py | 676 ++++ python/sglang/srt/layers/dp_attention.py | 489 +++ python/sglang/srt/layers/elementwise.py | 582 ++++ .../srt/layers/flashinfer_comm_fusion.py | 230 ++ python/sglang/srt/layers/layernorm.py | 343 ++ python/sglang/srt/layers/linear.py | 1337 ++++++++ python/sglang/srt/layers/logits_processor.py | 652 ++++ python/sglang/srt/layers/model_parallel.py | 155 + python/sglang/srt/layers/moe/__init__.py | 32 + python/sglang/srt/layers/moe/cutlass_moe.py | 365 ++ .../srt/layers/moe/cutlass_moe_params.py | 169 + .../sglang/srt/layers/moe/cutlass_w4a8_moe.py | 206 ++ .../sglang/srt/layers/moe/ep_moe/__init__.py | 0 .../sglang/srt/layers/moe/ep_moe/kernels.py | 1438 ++++++++ python/sglang/srt/layers/moe/ep_moe/layer.py | 869 +++++ .../sglang/srt/layers/moe/fused_moe_native.py | 101 + .../layers/moe/fused_moe_triton/__init__.py | 42 + .../moe/fused_moe_triton/configs/README.md | 15 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 218 ++ ...072,device_name=NVIDIA_H100_80GB_HBM3.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...024,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=16,N=1024,device_name=NVIDIA_H200.json | 146 + ...344,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...344,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...344,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...336,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 218 ++ ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...688,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...688,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 218 ++ ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...VIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json | 146 + ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 130 + ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...024,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...ame=NVIDIA_H20,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...evice_name=NVIDIA_L20,dtype=int8_w8a8.json | 146 + ...vice_name=NVIDIA_L40S,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...280,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...280,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...280,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=1280,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=2560,device_name=NVIDIA_H200.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...320,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=320,device_name=NVIDIA_H200.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...640,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...640,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...640,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=64,N=640,device_name=NVIDIA_H200.json | 146 + ...14336,device_name=AMD_Instinct_MI300X.json | 200 ++ ...14336,device_name=AMD_Instinct_MI325X.json | 200 ++ ...14336,device_name=AMD_Radeon_Graphics.json | 200 ++ ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 138 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=14336,device_name=NVIDIA_H200.json | 146 + ...=1792,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=1792,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=1792,device_name=AMD_Radeon_Graphics.json | 200 ++ ...792,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...792,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...792,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=1792,device_name=NVIDIA_H200.json | 146 + ...048,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...048,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=2048,device_name=NVIDIA_H200.json | 146 + ...=3584,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=3584,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=3584,device_name=AMD_Radeon_Graphics.json | 200 ++ ...584,device_name=NVIDIA_A100-SXM4-40GB.json | 146 + ...584,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...VIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...584,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=3584,device_name=NVIDIA_H200.json | 146 + .../E=8,N=3584,device_name=NVIDIA_L40S.json | 173 + ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 178 + ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 178 + ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 178 + ...096,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...096,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=4096,device_name=NVIDIA_H200.json | 146 + ...=7168,device_name=AMD_Instinct_MI300X.json | 200 ++ ...=7168,device_name=AMD_Instinct_MI325X.json | 200 ++ ...=7168,device_name=AMD_Radeon_Graphics.json | 200 ++ ...168,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + .../E=8,N=7168,device_name=NVIDIA_H200.json | 146 + ...me=AMD_Instinct_MI300X,dtype=fp8_w8a8.json | 175 + ...me=AMD_Instinct_MI325X,dtype=fp8_w8a8.json | 175 + ...me=AMD_Radeon_Graphics,dtype=fp8_w8a8.json | 175 + ...=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...192,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...192,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=128,N=192,device_name=NVIDIA_H20.json | 146 + .../E=128,N=192,device_name=NVIDIA_H200.json | 146 + ...384,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=384,device_name=NVIDIA_H20.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=384,device_name=NVIDIA_H200.json | 146 + ...512,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...768,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...768,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H20.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H200.json | 146 + .../E=128,N=96,device_name=NVIDIA_H20.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...=64,device_name=NVIDIA_A800-SXM4-80GB.json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../E=16,N=1024,device_name=NVIDIA_B200.json | 146 + ...TX_6000_Ada_Generation,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../E=128,N=768,device_name=NVIDIA_H20.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...E=160,N=320,device_name=NVIDIA_H20-3e.json | 146 + ...evice_name=NVIDIA_H200,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...168,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 + ...-Q_Workstation_Edition,dtype=fp8_w8a8.json | 146 + ...evice_name=NVIDIA_B200,dtype=fp8_w8a8.json | 146 + ...-Q_Workstation_Edition,dtype=fp8_w8a8.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...=64,device_name=NVIDIA_A100-SXM4-80GB.json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...128,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + ...E=512,N=128,device_name=NVIDIA_H20-3e.json | 146 + ...E=512,N=256,device_name=NVIDIA_H20-3e.json | 146 + ...=64,device_name=NVIDIA_H100_80GB_HBM3.json | 146 + .../layers/moe/fused_moe_triton/fused_moe.py | 701 ++++ .../fused_moe_triton_config.py | 212 ++ .../fused_moe_triton_kernels.py | 799 +++++ .../srt/layers/moe/fused_moe_triton/layer.py | 1078 ++++++ .../fused_moe_triton/moe_align_block_size.py | 87 + .../fused_moe_triton/triton_kernels_moe.py | 335 ++ .../srt/layers/moe/moe_runner/__init__.py | 4 + .../sglang/srt/layers/moe/moe_runner/base.py | 286 ++ .../srt/layers/moe/moe_runner/runner.py | 80 + .../srt/layers/moe/moe_runner/triton.py | 448 +++ .../sglang/srt/layers/moe/rocm_moe_utils.py | 141 + python/sglang/srt/layers/moe/router.py | 392 +++ .../layers/moe/token_dispatcher/__init__.py | 41 + .../srt/layers/moe/token_dispatcher/base.py | 150 + .../srt/layers/moe/token_dispatcher/deepep.py | 732 ++++ .../layers/moe/token_dispatcher/standard.py | 61 + python/sglang/srt/layers/moe/topk.py | 876 +++++ python/sglang/srt/layers/moe/utils.py | 201 ++ python/sglang/srt/layers/multimodal.py | 189 + python/sglang/srt/layers/parameter.py | 533 +++ python/sglang/srt/layers/pooler.py | 112 + .../srt/layers/quantization/__init__.py | 224 ++ python/sglang/srt/layers/quantization/awq.py | 782 +++++ .../srt/layers/quantization/awq_triton.py | 339 ++ .../srt/layers/quantization/base_config.py | 216 ++ .../srt/layers/quantization/blockwise_int8.py | 380 ++ .../quantization/compressed_tensors/README.md | 6 + .../compressed_tensors/__init__.py | 0 .../compressed_tensors/compressed_tensors.py | 666 ++++ .../compressed_tensors_moe.py | 692 ++++ .../compressed_tensors/schemes/__init__.py | 11 + .../schemes/compressed_tensors_scheme.py | 56 + .../schemes/compressed_tensors_w8a16_fp8.py | 153 + .../schemes/compressed_tensors_w8a8_fp8.py | 172 + .../quantization/compressed_tensors/utils.py | 218 ++ ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 26 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 164 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + ...type=int8_w8a8,block_shape=[128, 128].json | 146 + ...dtype=fp8_w8a8,block_shape=[128, 128].json | 146 + .../deep_gemm_wrapper/__init__.py | 1 + .../deep_gemm_wrapper/compile_utils.py | 241 ++ .../deep_gemm_wrapper/configurer.py | 32 + .../deep_gemm_wrapper/entrypoint.py | 119 + python/sglang/srt/layers/quantization/fp8.py | 1240 +++++++ .../srt/layers/quantization/fp8_kernel.py | 1806 ++++++++++ .../srt/layers/quantization/fp8_utils.py | 829 +++++ .../srt/layers/quantization/fpgemm_fp8.py | 203 ++ python/sglang/srt/layers/quantization/gptq.py | 1097 ++++++ .../srt/layers/quantization/int8_kernel.py | 425 +++ .../srt/layers/quantization/int8_utils.py | 73 + .../srt/layers/quantization/kv_cache.py | 82 + .../srt/layers/quantization/marlin_utils.py | 808 +++++ .../layers/quantization/marlin_utils_fp8.py | 352 ++ .../srt/layers/quantization/modelopt_quant.py | 1402 ++++++++ .../srt/layers/quantization/moe_wna16.py | 501 +++ .../sglang/srt/layers/quantization/mxfp4.py | 862 +++++ .../srt/layers/quantization/mxfp4_tensor.py | 135 + .../sglang/srt/layers/quantization/petit.py | 252 ++ .../srt/layers/quantization/petit_utils.py | 104 + python/sglang/srt/layers/quantization/qoq.py | 245 ++ .../srt/layers/quantization/quark/__init__.py | 0 .../srt/layers/quantization/quark/quark.py | 390 +++ .../layers/quantization/quark/quark_moe.py | 202 ++ .../quantization/quark/schemes/__init__.py | 6 + .../quark/schemes/quark_scheme.py | 55 + .../quark/schemes/quark_w4a4_mxfp4.py | 137 + .../srt/layers/quantization/quark/utils.py | 204 ++ .../layers/quantization/rocm_mxfp4_utils.py | 13 + .../sglang/srt/layers/quantization/unquant.py | 468 +++ .../sglang/srt/layers/quantization/utils.py | 564 +++ .../sglang/srt/layers/quantization/w4afp8.py | 346 ++ .../srt/layers/quantization/w8a8_fp8.py | 305 ++ .../srt/layers/quantization/w8a8_int8.py | 1034 ++++++ python/sglang/srt/layers/radix_attention.py | 116 + python/sglang/srt/layers/rocm_linear_utils.py | 44 + python/sglang/srt/layers/rotary_embedding.py | 2017 +++++++++++ python/sglang/srt/layers/sampler.py | 294 ++ python/sglang/srt/layers/torchao_utils.py | 124 + python/sglang/srt/layers/utils.py | 36 + .../srt/layers/vocab_parallel_embedding.py | 571 +++ .../sglang/srt/lora/backend/base_backend.py | 155 + .../sglang/srt/lora/backend/triton_backend.py | 176 + python/sglang/srt/lora/layers.py | 352 ++ python/sglang/srt/lora/lora.py | 178 + python/sglang/srt/lora/lora_config.py | 47 + python/sglang/srt/lora/lora_manager.py | 440 +++ python/sglang/srt/lora/lora_registry.py | 207 ++ python/sglang/srt/lora/mem_pool.py | 309 ++ python/sglang/srt/lora/triton_ops/__init__.py | 11 + .../srt/lora/triton_ops/gate_up_lora_b.py | 187 + .../sglang/srt/lora/triton_ops/qkv_lora_b.py | 198 ++ .../srt/lora/triton_ops/sgemm_lora_a.py | 170 + .../srt/lora/triton_ops/sgemm_lora_b.py | 176 + python/sglang/srt/lora/utils.py | 143 + .../sglang/srt/managers/cache_controller.py | 886 +++++ .../sglang/srt/managers/configure_logging.py | 47 + .../srt/managers/data_parallel_controller.py | 390 +++ .../srt/managers/detokenizer_manager.py | 303 ++ python/sglang/srt/managers/disagg_service.py | 46 + python/sglang/srt/managers/io_struct.py | 1332 +++++++ python/sglang/srt/managers/mm_utils.py | 775 +++++ .../srt/managers/multi_tokenizer_mixin.py | 579 ++++ .../srt/managers/multimodal_processor.py | 49 + python/sglang/srt/managers/schedule_batch.py | 2017 +++++++++++ python/sglang/srt/managers/schedule_policy.py | 570 +++ python/sglang/srt/managers/scheduler.py | 2631 ++++++++++++++ .../srt/managers/scheduler_input_blocker.py | 106 + .../srt/managers/scheduler_metrics_mixin.py | 350 ++ .../scheduler_output_processor_mixin.py | 732 ++++ .../srt/managers/scheduler_profiler_mixin.py | 299 ++ .../srt/managers/scheduler_recv_skipper.py | 37 + .../scheduler_update_weights_mixin.py | 155 + .../sglang/srt/managers/session_controller.py | 158 + .../sglang/srt/managers/template_manager.py | 308 ++ .../managers/tokenizer_communicator_mixin.py | 491 +++ .../sglang/srt/managers/tokenizer_manager.py | 1715 +++++++++ python/sglang/srt/managers/tp_worker.py | 329 ++ .../srt/managers/tp_worker_overlap_thread.py | 294 ++ python/sglang/srt/managers/utils.py | 142 + python/sglang/srt/mem_cache/allocator.py | 581 ++++ .../sglang/srt/mem_cache/allocator_ascend.py | 147 + .../sglang/srt/mem_cache/base_prefix_cache.py | 109 + python/sglang/srt/mem_cache/chunk_cache.py | 100 + .../mem_cache/cpp_radix_tree/.clang-format | 1 + .../srt/mem_cache/cpp_radix_tree/common.h | 29 + .../mem_cache/cpp_radix_tree/radix_tree.py | 182 + .../srt/mem_cache/cpp_radix_tree/tree_v2.cpp | 143 + .../srt/mem_cache/cpp_radix_tree/tree_v2.h | 59 + .../cpp_radix_tree/tree_v2_binding.cpp | 32 + .../cpp_radix_tree/tree_v2_debug.cpp | 194 ++ .../mem_cache/cpp_radix_tree/tree_v2_impl.h | 276 ++ .../mem_cache/cpp_radix_tree/tree_v2_node.h | 257 ++ python/sglang/srt/mem_cache/flush_cache.py | 33 + .../sglang/srt/mem_cache/hicache_storage.py | 226 ++ python/sglang/srt/mem_cache/hiradix_cache.py | 849 +++++ .../sglang/srt/mem_cache/lora_radix_cache.py | 421 +++ python/sglang/srt/mem_cache/memory_pool.py | 1423 ++++++++ .../sglang/srt/mem_cache/memory_pool_host.py | 747 ++++ .../sglang/srt/mem_cache/multimodal_cache.py | 68 + python/sglang/srt/mem_cache/radix_cache.py | 572 +++ .../sglang/srt/mem_cache/radix_cache_cpp.py | 229 ++ .../mem_cache/storage/hf3fs/docs/README.md | 35 + .../hf3fs/docs/deploy_sglang_3fs_multinode.md | 65 + .../storage/hf3fs/docs/setup_usrbio_client.md | 68 + .../mem_cache/storage/hf3fs/hf3fs_client.py | 164 + .../storage/hf3fs/hf3fs_usrbio_client.py | 187 + .../mem_cache/storage/hf3fs/hf3fs_utils.cpp | 35 + .../storage/hf3fs/mini_3fs_metadata_server.py | 470 +++ .../mem_cache/storage/hf3fs/storage_hf3fs.py | 523 +++ .../storage/hf3fs/test_hf3fs_utils.py | 43 + .../srt/mem_cache/storage/lmcache/README.md | 43 + .../storage/lmcache/example_config.yaml | 7 + .../storage/lmcache/lmc_radix_cache.py | 280 ++ .../mem_cache/storage/lmcache/unit_test.py | 121 + .../storage/mooncake_store/README.md | 167 + .../storage/mooncake_store/mooncake_store.py | 324 ++ .../mooncake_store/test_mooncake_store.py | 161 + .../srt/mem_cache/storage/nixl/README.md | 172 + .../mem_cache/storage/nixl/hicache_nixl.py | 257 ++ .../srt/mem_cache/storage/nixl/nixl_utils.py | 204 ++ .../storage/nixl/test_hicache_nixl_storage.py | 255 ++ .../sglang/srt/mem_cache/swa_radix_cache.py | 1023 ++++++ python/sglang/srt/metrics/collector.py | 974 ++++++ python/sglang/srt/metrics/func_timer.py | 106 + .../srt/metrics/startup_func_log_and_timer.py | 150 + python/sglang/srt/metrics/utils.py | 48 + .../srt/model_executor/cpu_graph_runner.py | 640 ++++ .../srt/model_executor/cuda_graph_runner.py | 868 +++++ .../srt/model_executor/forward_batch_info.py | 1094 ++++++ .../sglang/srt/model_executor/model_runner.py | 2071 +++++++++++ .../srt/model_executor/npu_graph_runner.py | 94 + python/sglang/srt/model_loader/__init__.py | 40 + python/sglang/srt/model_loader/loader.py | 1584 +++++++++ python/sglang/srt/model_loader/utils.py | 119 + .../sglang/srt/model_loader/weight_utils.py | 1030 ++++++ python/sglang/srt/models/arcee.py | 532 +++ python/sglang/srt/models/baichuan.py | 441 +++ python/sglang/srt/models/bailing_moe.py | 425 +++ python/sglang/srt/models/bert.py | 498 +++ python/sglang/srt/models/chatglm.py | 427 +++ python/sglang/srt/models/clip.py | 572 +++ python/sglang/srt/models/commandr.py | 418 +++ python/sglang/srt/models/dbrx.py | 460 +++ python/sglang/srt/models/deepseek.py | 489 +++ .../sglang/srt/models/deepseek_janus_pro.py | 2065 +++++++++++ python/sglang/srt/models/deepseek_nextn.py | 170 + python/sglang/srt/models/deepseek_v2.py | 2980 ++++++++++++++++ python/sglang/srt/models/deepseek_vl2.py | 362 ++ python/sglang/srt/models/ernie4.py | 426 +++ python/sglang/srt/models/ernie4_eagle.py | 203 ++ python/sglang/srt/models/exaone.py | 377 ++ python/sglang/srt/models/gemma.py | 410 +++ python/sglang/srt/models/gemma2.py | 480 +++ python/sglang/srt/models/gemma2_reward.py | 70 + python/sglang/srt/models/gemma3_causal.py | 764 +++++ python/sglang/srt/models/gemma3_mm.py | 444 +++ python/sglang/srt/models/gemma3n_audio.py | 949 +++++ python/sglang/srt/models/gemma3n_causal.py | 1010 ++++++ python/sglang/srt/models/gemma3n_mm.py | 534 +++ python/sglang/srt/models/glm4.py | 318 ++ python/sglang/srt/models/glm4_moe.py | 1083 ++++++ python/sglang/srt/models/glm4_moe_nextn.py | 168 + python/sglang/srt/models/glm4v.py | 642 ++++ python/sglang/srt/models/glm4v_moe.py | 397 +++ python/sglang/srt/models/gpt2.py | 288 ++ python/sglang/srt/models/gpt_bigcode.py | 304 ++ python/sglang/srt/models/gpt_oss.py | 1204 +++++++ python/sglang/srt/models/granite.py | 505 +++ python/sglang/srt/models/granitemoe.py | 387 +++ python/sglang/srt/models/grok.py | 1132 ++++++ python/sglang/srt/models/hunyuan.py | 819 +++++ python/sglang/srt/models/idefics2.py | 342 ++ python/sglang/srt/models/internlm2.py | 357 ++ python/sglang/srt/models/internlm2_reward.py | 64 + python/sglang/srt/models/interns1.py | 293 ++ python/sglang/srt/models/internvl.py | 699 ++++ python/sglang/srt/models/kimi_vl.py | 313 ++ python/sglang/srt/models/kimi_vl_moonvit.py | 639 ++++ python/sglang/srt/models/llama.py | 749 ++++ python/sglang/srt/models/llama4.py | 561 +++ .../sglang/srt/models/llama_classification.py | 81 + python/sglang/srt/models/llama_eagle.py | 149 + python/sglang/srt/models/llama_eagle3.py | 266 ++ python/sglang/srt/models/llama_embedding.py | 87 + python/sglang/srt/models/llama_reward.py | 126 + python/sglang/srt/models/llava.py | 843 +++++ python/sglang/srt/models/llavavid.py | 283 ++ python/sglang/srt/models/longcat_flash.py | 1026 ++++++ .../sglang/srt/models/longcat_flash_nextn.py | 699 ++++ python/sglang/srt/models/mimo.py | 171 + python/sglang/srt/models/mimo_mtp.py | 204 ++ python/sglang/srt/models/minicpm.py | 399 +++ python/sglang/srt/models/minicpm3.py | 517 +++ python/sglang/srt/models/minicpmo.py | 1914 +++++++++++ python/sglang/srt/models/minicpmv.py | 1046 ++++++ python/sglang/srt/models/mistral.py | 93 + python/sglang/srt/models/mixtral.py | 480 +++ python/sglang/srt/models/mixtral_quant.py | 430 +++ python/sglang/srt/models/mllama.py | 1062 ++++++ python/sglang/srt/models/mllama4.py | 990 ++++++ python/sglang/srt/models/nemotron_nas.py | 435 +++ python/sglang/srt/models/olmo.py | 377 ++ python/sglang/srt/models/olmo2.py | 413 +++ python/sglang/srt/models/olmoe.py | 437 +++ python/sglang/srt/models/opt.py | 637 ++++ python/sglang/srt/models/persimmon.py | 330 ++ python/sglang/srt/models/phi.py | 321 ++ python/sglang/srt/models/phi3_small.py | 476 +++ python/sglang/srt/models/phi4mm.py | 538 +++ python/sglang/srt/models/phi4mm_audio.py | 1260 +++++++ python/sglang/srt/models/phi4mm_utils.py | 1917 +++++++++++ python/sglang/srt/models/phimoe.py | 560 +++ python/sglang/srt/models/pixtral.py | 463 +++ python/sglang/srt/models/qwen.py | 356 ++ python/sglang/srt/models/qwen2.py | 653 ++++ python/sglang/srt/models/qwen2_5_vl.py | 671 ++++ python/sglang/srt/models/qwen2_audio.py | 201 ++ .../sglang/srt/models/qwen2_classification.py | 75 + python/sglang/srt/models/qwen2_eagle.py | 146 + python/sglang/srt/models/qwen2_moe.py | 774 +++++ python/sglang/srt/models/qwen2_rm.py | 78 + python/sglang/srt/models/qwen2_vl.py | 609 ++++ python/sglang/srt/models/qwen3.py | 523 +++ .../sglang/srt/models/qwen3_classification.py | 84 + python/sglang/srt/models/qwen3_moe.py | 896 +++++ python/sglang/srt/models/qwen3_next.py | 1049 ++++++ python/sglang/srt/models/qwen3_next_mtp.py | 109 + python/sglang/srt/models/registry.py | 107 + python/sglang/srt/models/roberta.py | 286 ++ python/sglang/srt/models/siglip.py | 294 ++ python/sglang/srt/models/stablelm.py | 331 ++ python/sglang/srt/models/step3_vl.py | 1005 ++++++ .../sglang/srt/models/torch_native_llama.py | 500 +++ python/sglang/srt/models/transformers.py | 288 ++ python/sglang/srt/models/vila.py | 306 ++ python/sglang/srt/models/xverse.py | 382 +++ python/sglang/srt/models/xverse_moe.py | 478 +++ python/sglang/srt/models/yivl.py | 115 + python/sglang/srt/multimodal/mm_utils.py | 349 ++ .../multimodal/processors/base_processor.py | 653 ++++ .../sglang/srt/multimodal/processors/clip.py | 35 + .../multimodal/processors/deepseek_vl_v2.py | 65 + .../srt/multimodal/processors/gemma3.py | 54 + .../srt/multimodal/processors/gemma3n.py | 71 + .../sglang/srt/multimodal/processors/glm4v.py | 132 + .../srt/multimodal/processors/internvl.py | 267 ++ .../srt/multimodal/processors/janus_pro.py | 45 + .../srt/multimodal/processors/kimi_vl.py | 46 + .../sglang/srt/multimodal/processors/llava.py | 211 ++ .../srt/multimodal/processors/minicpm.py | 144 + .../sglang/srt/multimodal/processors/mlama.py | 37 + .../srt/multimodal/processors/mllama4.py | 57 + .../srt/multimodal/processors/phi4mm.py | 101 + .../srt/multimodal/processors/pixtral.py | 100 + .../srt/multimodal/processors/qwen_audio.py | 67 + .../srt/multimodal/processors/qwen_vl.py | 281 ++ .../srt/multimodal/processors/step3_vl.py | 517 +++ .../sglang/srt/multimodal/processors/vila.py | 69 + python/sglang/srt/offloader.py | 548 +++ python/sglang/srt/operations.py | 209 ++ python/sglang/srt/operations_strategy.py | 211 ++ .../srt/parser/code_completion_parser.py | 132 + python/sglang/srt/parser/conversation.py | 1040 ++++++ python/sglang/srt/parser/harmony_parser.py | 588 ++++ .../sglang/srt/parser/jinja_template_utils.py | 197 ++ python/sglang/srt/parser/reasoning_parser.py | 309 ++ python/sglang/srt/patch_torch.py | 82 + python/sglang/srt/poll_based_barrier.py | 31 + .../srt/sampling/custom_logit_processor.py | 53 + .../srt/sampling/penaltylib/__init__.py | 11 + .../sampling/penaltylib/frequency_penalty.py | 65 + .../srt/sampling/penaltylib/min_new_tokens.py | 94 + .../srt/sampling/penaltylib/orchestrator.py | 209 ++ .../sampling/penaltylib/presence_penalty.py | 65 + .../srt/sampling/sampling_batch_info.py | 392 +++ python/sglang/srt/sampling/sampling_params.py | 165 + python/sglang/srt/server_args.py | 2714 +++++++++++++++ .../srt/speculative/build_eagle_tree.py | 427 +++ .../eagle_draft_cuda_graph_runner.py | 355 ++ .../eagle_draft_extend_cuda_graph_runner.py | 391 +++ python/sglang/srt/speculative/eagle_utils.py | 1297 +++++++ python/sglang/srt/speculative/eagle_worker.py | 1142 ++++++ python/sglang/srt/speculative/spec_info.py | 32 + .../srt/speculative/standalone_worker.py | 109 + .../srt/tokenizer/tiktoken_tokenizer.py | 166 + .../sglang/srt/torch_memory_saver_adapter.py | 94 + python/sglang/srt/two_batch_overlap.py | 989 ++++++ python/sglang/srt/utils.py | 3055 +++++++++++++++++ python/sglang/srt/warmup.py | 56 + .../sglang/srt/weight_sync/tensor_bucket.py | 106 + python/sglang/srt/weight_sync/utils.py | 119 + python/sglang/test/__init__.py | 0 python/sglang/test/attention/__init__.py | 0 .../test/attention/test_flashattn_backend.py | 350 ++ .../attention/test_flashattn_mla_backend.py | 285 ++ .../test/attention/test_prefix_chunk_info.py | 226 ++ .../test/attention/test_trtllm_mla_backend.py | 1268 +++++++ python/sglang/test/doc_patch.py | 59 + python/sglang/test/few_shot_gsm8k.py | 149 + python/sglang/test/few_shot_gsm8k_engine.py | 144 + python/sglang/test/long_prompt.txt | 1 + python/sglang/test/run_eval.py | 129 + python/sglang/test/runners.py | 874 +++++ python/sglang/test/send_one.py | 158 + python/sglang/test/simple_eval_common.py | 472 +++ python/sglang/test/simple_eval_gpqa.py | 95 + python/sglang/test/simple_eval_humaneval.py | 133 + python/sglang/test/simple_eval_math.py | 73 + python/sglang/test/simple_eval_mgsm.py | 203 ++ python/sglang/test/simple_eval_mmlu.py | 121 + python/sglang/test/test_activation.py | 105 + python/sglang/test/test_block_fp8.py | 661 ++++ .../test_block_fp8_deep_gemm_blackwell.py | 252 ++ python/sglang/test/test_block_fp8_ep.py | 358 ++ python/sglang/test/test_custom_ops.py | 148 + python/sglang/test/test_cutlass_moe.py | 299 ++ python/sglang/test/test_cutlass_w4a8_moe.py | 295 ++ python/sglang/test/test_deepep_utils.py | 219 ++ .../sglang/test/test_disaggregation_utils.py | 66 + python/sglang/test/test_dynamic_grad_mode.py | 58 + python/sglang/test/test_fp4_moe.py | 329 ++ python/sglang/test/test_layernorm.py | 113 + python/sglang/test/test_marlin_moe.py | 286 ++ python/sglang/test/test_marlin_utils.py | 171 + python/sglang/test/test_programs.py | 575 ++++ python/sglang/test/test_utils.py | 1430 ++++++++ python/sglang/utils.py | 535 +++ python/sglang/version.py | 1 + scripts/ci/amd_ci_exec.sh | 60 + scripts/ci/amd_ci_install_dependency.sh | 47 + scripts/ci/amd_ci_start_container.sh | 132 + scripts/ci/ci_install_deepep.sh | 68 + scripts/ci/ci_install_dependency.sh | 81 + scripts/ci/ci_install_rust.sh | 24 + scripts/ci/ci_start_disaggregation_servers.sh | 94 + scripts/ci/npu_ci_install_dependency.sh | 61 + scripts/code_sync/copy_from_oss.py | 293 ++ scripts/code_sync/copy_to_oss.py | 425 +++ scripts/code_sync/guideline.md | 27 + scripts/code_sync/install_github_cli.sh | 18 + scripts/deprecated/convert_yi_vl.py | 38 + scripts/deprecated/convert_yi_vl.sh | 13 + scripts/deprecated/test_curl.sh | 9 + scripts/deprecated/test_flashinfer.py | 217 ++ .../deprecated/test_httpserver_concurrent.py | 56 + scripts/deprecated/test_httpserver_decode.py | 55 + .../test_httpserver_decode_stream.py | 68 + scripts/deprecated/test_httpserver_llava.py | 88 + scripts/deprecated/test_httpserver_reuse.py | 42 + scripts/deprecated/test_jump_forward.py | 138 + scripts/deprecated/test_robust.py | 132 + scripts/export_deepseek_nextn.py | 115 + scripts/killall_sglang.sh | 32 + scripts/playground/bench_speculative.py | 308 ++ .../playground/disaggregation/cli-logprob.py | 22 + scripts/playground/disaggregation/cli-so.py | 34 + scripts/playground/disaggregation/cli.py | 29 + scripts/playground/frontend_reasoning.ipynb | 241 ++ scripts/playground/launch_tgi.sh | 7 + scripts/playground/load_tokenizer.py | 14 + scripts/playground/long_context_example.py | 36 + scripts/playground/lora/analyzer.py | 77 + scripts/playground/lora/lora_hf_play.py | 62 + scripts/playground/lora/lora_vllm_play.py | 30 + scripts/playground/reference_hf.py | 197 ++ scripts/playground/replay_request_dump.py | 151 + scripts/playground/router/test_tree.py | 207 ++ scripts/playground/router/tree.py | 292 ++ scripts/update_kernel_whl_index.py | 33 + scripts/version_branch_to_tag.sh | 30 + sgl-kernel/.clang-format | 15 + sgl-kernel/CMakeLists.txt | 534 +++ sgl-kernel/LICENSE | 201 ++ sgl-kernel/Makefile | 71 + sgl-kernel/README.md | 263 ++ sgl-kernel/THIRDPARTYNOTICES.txt | 488 +++ sgl-kernel/benchmark/bench_activation.py | 153 + sgl-kernel/benchmark/bench_awq_dequant.py | 118 + sgl-kernel/benchmark/bench_cutlass_mla.py | 145 + .../benchmark/bench_dsv3_fused_a_gemm.py | 57 + .../benchmark/bench_dsv3_router_gemm.py | 127 + sgl-kernel/benchmark/bench_fp4_gemm.py | 210 ++ .../benchmark/bench_fp8_blockwise_gemm.py | 183 + .../bench_fp8_blockwise_group_gemm.py | 328 ++ sgl-kernel/benchmark/bench_fp8_gemm.py | 184 + sgl-kernel/benchmark/bench_int8_gemm.py | 146 + .../bench_lightning_attention_decode.py | 299 ++ .../benchmark/bench_moe_align_block_size.py | 401 +++ .../benchmark/bench_moe_ep_post_reorder.py | 75 + sgl-kernel/benchmark/bench_moe_fused_gate.py | 77 + .../benchmark/bench_moe_topk_softmax.py | 116 + .../benchmark/bench_nvfp4_scaled_gemm.py | 172 + .../benchmark/bench_per_tensor_quant_fp8.py | 98 + .../bench_per_token_group_quant_8bit.py | 98 + .../benchmark/bench_per_token_quant_fp8.py | 177 + .../benchmark/bench_qserve_w4a8_gemm.py | 198 ++ .../benchmark/bench_rotary_embedding.py | 96 + .../benchmark/bench_top_k_top_p_sampling.py | 128 + sgl-kernel/build.sh | 75 + sgl-kernel/cmake/utils.cmake | 21 + .../csrc/allreduce/custom_all_reduce.cu | 137 + .../csrc/allreduce/custom_all_reduce.cuh | 489 +++ .../csrc/allreduce/custom_all_reduce.hip | 180 + .../csrc/allreduce/custom_all_reduce_hip.cuh | 582 ++++ .../csrc/allreduce/mscclpp_allreduce.cu | 140 + .../csrc/allreduce/mscclpp_allreduce.cuh | 779 +++++ sgl-kernel/csrc/allreduce/quick_all_reduce.cu | 111 + .../csrc/allreduce/quick_all_reduce.cuh | 633 ++++ sgl-kernel/csrc/allreduce/quick_all_reduce.h | 233 ++ .../csrc/allreduce/quick_all_reduce.hip | 113 + .../csrc/allreduce/quick_all_reduce_base.h | 318 ++ .../csrc/allreduce/quick_all_reduce_hip.h | 235 ++ .../csrc/allreduce/test_mscclpp_allreduce.cu | 153 + sgl-kernel/csrc/attention/cascade.cu | 55 + .../csrc/attention/cutlass_mla_kernel.cu | 274 ++ .../cutlass_sm100_mla/device/sm100_mla.hpp | 358 ++ .../kernel/sm100_fmha_mla_reduction.hpp | 198 ++ .../sm100_fmha_mla_tma_warpspecialized.hpp | 2018 +++++++++++ .../kernel/sm100_mla_tile_scheduler.hpp | 160 + .../lightning_attention_decode_kernel.cu | 154 + .../csrc/attention/merge_attn_states.cu | 204 ++ .../csrc/attention/vertical_slash_index.cu | 462 +++ sgl-kernel/csrc/common_extension.cc | 481 +++ sgl-kernel/csrc/common_extension_rocm.cc | 176 + sgl-kernel/csrc/cpu/CMakeLists.txt | 83 + sgl-kernel/csrc/cpu/activation.cpp | 135 + sgl-kernel/csrc/cpu/bmm.cpp | 123 + sgl-kernel/csrc/cpu/common.h | 324 ++ sgl-kernel/csrc/cpu/decode.cpp | 1575 +++++++++ sgl-kernel/csrc/cpu/extend.cpp | 723 ++++ sgl-kernel/csrc/cpu/gemm.cpp | 525 +++ sgl-kernel/csrc/cpu/gemm.h | 202 ++ sgl-kernel/csrc/cpu/gemm_fp8.cpp | 551 +++ sgl-kernel/csrc/cpu/gemm_int8.cpp | 547 +++ sgl-kernel/csrc/cpu/interface.cpp | 74 + sgl-kernel/csrc/cpu/moe.cpp | 1322 +++++++ sgl-kernel/csrc/cpu/moe_fp8.cpp | 491 +++ sgl-kernel/csrc/cpu/moe_int8.cpp | 1068 ++++++ sgl-kernel/csrc/cpu/norm.cpp | 304 ++ sgl-kernel/csrc/cpu/numa_utils.cpp | 91 + sgl-kernel/csrc/cpu/qkv_proj.cpp | 701 ++++ sgl-kernel/csrc/cpu/rope.cpp | 346 ++ sgl-kernel/csrc/cpu/shm.cpp | 666 ++++ sgl-kernel/csrc/cpu/shm.h | 11 + sgl-kernel/csrc/cpu/topk.cpp | 662 ++++ sgl-kernel/csrc/cpu/torch_extension_cpu.cpp | 373 ++ sgl-kernel/csrc/cpu/vec.h | 308 ++ sgl-kernel/csrc/cutlass_extensions/common.hpp | 21 + .../detail/collective/mixed_input_utils.hpp | 482 +++ .../epilogue/epilogue_per_row_per_col_scale.h | 309 ++ .../sm90_gmma_builder_mixed_input.inl | 278 ++ .../collective_builder_mixed_input.hpp | 52 + .../collective_mma_array_mixed_input.hpp | 53 + ...a_gmma_rs_warpspecialized_mixed_input_.hpp | 1538 +++++++++ .../gemm/cutlass_gemm_caller.cuh | 62 + .../gemm/dispatch_policy.hpp | 38 + .../gemm/fp8_blockwise_gemm_sm90_dispatch.cuh | 197 ++ .../gemm/gemm_universal_base_compat.h | 356 ++ .../gemm/gemm_with_epilogue_visitor.h | 492 +++ sgl-kernel/csrc/elementwise/activation.cu | 170 + sgl-kernel/csrc/elementwise/activation.hip | 173 + sgl-kernel/csrc/elementwise/cast.cu | 171 + sgl-kernel/csrc/elementwise/concat_mla.cu | 117 + sgl-kernel/csrc/elementwise/copy.cu | 58 + .../elementwise/fused_add_rms_norm_kernel.cu | 59 + sgl-kernel/csrc/elementwise/pos_enc.cuh | 467 +++ sgl-kernel/csrc/elementwise/rope.cu | 164 + sgl-kernel/csrc/flash_extension.cc | 63 + sgl-kernel/csrc/gemm/awq_kernel.cu | 221 ++ sgl-kernel/csrc/gemm/bmm_fp8.cu | 76 + sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu | 673 ++++ .../csrc/gemm/dsv3_router_gemm_bf16_out.cu | 284 ++ .../csrc/gemm/dsv3_router_gemm_entry.cu | 161 + .../csrc/gemm/dsv3_router_gemm_float_out.cu | 283 ++ .../csrc/gemm/fp8_blockwise_gemm_kernel.cu | 465 +++ sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu | 1252 +++++++ sgl-kernel/csrc/gemm/gptq/compat.cuh | 62 + sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu | 1950 +++++++++++ sgl-kernel/csrc/gemm/gptq/matrix_view.cuh | 269 ++ sgl-kernel/csrc/gemm/gptq/qdq_2.cuh | 74 + sgl-kernel/csrc/gemm/gptq/qdq_3.cuh | 146 + sgl-kernel/csrc/gemm/gptq/qdq_4.cuh | 114 + sgl-kernel/csrc/gemm/gptq/qdq_8.cuh | 30 + sgl-kernel/csrc/gemm/gptq/qdq_util.cuh | 53 + sgl-kernel/csrc/gemm/int8_gemm_kernel.cu | 747 ++++ .../csrc/gemm/marlin/awq_marlin_repack.cu | 253 ++ sgl-kernel/csrc/gemm/marlin/dequant.h | 459 +++ sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu | 1120 ++++++ .../csrc/gemm/marlin/gptq_marlin_repack.cu | 329 ++ sgl-kernel/csrc/gemm/marlin/kernel.h | 36 + sgl-kernel/csrc/gemm/marlin/marlin.cuh | 96 + sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh | 82 + sgl-kernel/csrc/gemm/marlin/marlin_template.h | 1629 +++++++++ sgl-kernel/csrc/gemm/math.hpp | 28 + sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu | 728 ++++ sgl-kernel/csrc/gemm/nvfp4_quant.cuh | 176 + sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu | 74 + sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu | 239 ++ sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu | 39 + .../csrc/gemm/nvfp4_scaled_mm_kernels.cu | 453 +++ sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu | 123 + .../csrc/gemm/per_token_group_quant_8bit.cu | 240 ++ sgl-kernel/csrc/gemm/per_token_quant_fp8.cu | 227 ++ .../csrc/gemm/qserve_w4a8_per_chn_gemm.cu | 710 ++++ .../csrc/gemm/qserve_w4a8_per_group_gemm.cu | 795 +++++ .../apply_token_bitmask_inplace_cuda.cu | 271 ++ .../apply_token_bitmask_inplace_cuda.hip | 273 ++ sgl-kernel/csrc/kvcacheio/transfer.cu | 573 ++++ sgl-kernel/csrc/kvcacheio/transfer.hip | 576 ++++ sgl-kernel/csrc/mamba/causal_conv1d.cu | 669 ++++ sgl-kernel/csrc/mamba/causal_conv1d.h | 159 + sgl-kernel/csrc/memory/store.cu | 147 + .../moe/cutlass_moe/w4a8/scaled_mm_entry.cu | 91 + .../w4a8/w4a8_get_group_starts.cuh | 92 + .../cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu | 386 +++ .../cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh | 277 ++ .../moe/cutlass_moe/w4a8/w4a8_moe_data.cu | 79 + sgl-kernel/csrc/moe/cutlass_moe_helper.cu | 142 + .../csrc/moe/fp8_blockwise_moe_kernel.cu | 810 +++++ .../moe/marlin_moe_wna16/generate_kernels.py | 129 + sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h | 41 + .../moe/marlin_moe_wna16/kernel_bf16_ku4.cuh | 90 + .../marlin_moe_wna16/kernel_bf16_ku4b8.cuh | 110 + .../marlin_moe_wna16/kernel_bf16_ku8b128.cuh | 110 + .../moe/marlin_moe_wna16/kernel_fp16_ku4.cuh | 90 + .../marlin_moe_wna16/kernel_fp16_ku4b8.cuh | 110 + .../marlin_moe_wna16/kernel_fp16_ku8b128.cuh | 110 + .../moe/marlin_moe_wna16/kernel_marlin.cuh | 10 + .../moe/marlin_moe_wna16/marlin_template.h | 1805 ++++++++++ sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu | 1111 ++++++ sgl-kernel/csrc/moe/moe_align_kernel.cu | 364 ++ sgl-kernel/csrc/moe/moe_align_kernel.hip | 367 ++ sgl-kernel/csrc/moe/moe_fused_gate.cu | 521 +++ .../csrc/moe/moe_topk_softmax_kernels.cu | 591 ++++ .../csrc/moe/moe_topk_softmax_kernels.hip | 594 ++++ sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu | 471 +++ sgl-kernel/csrc/moe/prepare_moe_input.cu | 392 +++ sgl-kernel/csrc/spatial/cuda_utils.h | 46 + sgl-kernel/csrc/spatial/greenctx_stream.cu | 98 + sgl-kernel/csrc/spatial/greenctx_stream.h | 2 + sgl-kernel/csrc/spatial_extension.cc | 29 + sgl-kernel/csrc/speculative/eagle_utils.cu | 408 +++ sgl-kernel/csrc/speculative/eagle_utils.hip | 411 +++ sgl-kernel/csrc/speculative/packbit.cu | 51 + .../csrc/speculative/speculative_sampling.cu | 152 + .../csrc/speculative/speculative_sampling.cuh | 231 ++ sgl-kernel/include/hip/hip_act_and_mul.cuh | 87 + .../include/hip/hip_act_and_mul_hip.cuh | 90 + sgl-kernel/include/hip/hip_math_def.h | 94 + sgl-kernel/include/hip/hip_vec_dtypes.h | 101 + .../include/hip/impl/hip_vec_bf16_impl.h | 177 + .../include/hip/impl/hip_vec_fp32_impl.h | 129 + .../include/hip/impl/hip_vec_half_impl.h | 172 + .../include/pytorch_extension_utils_rocm.h | 20 + sgl-kernel/include/scalar_type.hpp | 330 ++ sgl-kernel/include/sgl_flash_kernel_ops.h | 86 + sgl-kernel/include/sgl_kernel_ops.h | 771 +++++ sgl-kernel/include/sgl_kernel_torch_shim.h | 122 + sgl-kernel/include/utils.h | 449 +++ sgl-kernel/include/utils_hip.h | 451 +++ sgl-kernel/pyproject.toml | 39 + sgl-kernel/pyproject_cpu.toml | 36 + sgl-kernel/pyproject_rocm.toml | 32 + sgl-kernel/python/sgl_kernel/__init__.py | 117 + .../python/sgl_kernel/_fa4_interface.py | 376 ++ sgl-kernel/python/sgl_kernel/allreduce.py | 173 + sgl-kernel/python/sgl_kernel/attention.py | 138 + sgl-kernel/python/sgl_kernel/cutlass_moe.py | 112 + sgl-kernel/python/sgl_kernel/elementwise.py | 381 ++ sgl-kernel/python/sgl_kernel/flash_attn.py | 459 +++ sgl-kernel/python/sgl_kernel/fused_moe.py | 225 ++ sgl-kernel/python/sgl_kernel/gemm.py | 550 +++ sgl-kernel/python/sgl_kernel/grammar.py | 15 + sgl-kernel/python/sgl_kernel/kvcacheio.py | 243 ++ sgl-kernel/python/sgl_kernel/mamba.py | 50 + sgl-kernel/python/sgl_kernel/marlin.py | 44 + sgl-kernel/python/sgl_kernel/memory.py | 18 + sgl-kernel/python/sgl_kernel/moe.py | 197 ++ sgl-kernel/python/sgl_kernel/sampling.py | 543 +++ sgl-kernel/python/sgl_kernel/scalar_type.py | 352 ++ .../python/sgl_kernel/sparse_flash_attn.py | 293 ++ sgl-kernel/python/sgl_kernel/spatial.py | 63 + sgl-kernel/python/sgl_kernel/speculative.py | 107 + .../python/sgl_kernel/testing/__init__.py | 0 .../sgl_kernel/testing/rotary_embedding.py | 217 ++ sgl-kernel/python/sgl_kernel/top_k.py | 11 + sgl-kernel/python/sgl_kernel/utils.py | 50 + sgl-kernel/python/sgl_kernel/version.py | 1 + sgl-kernel/rename_wheels.sh | 32 + sgl-kernel/setup_hip.py | 100 + sgl-kernel/setup_rocm.py | 116 + .../tests/spatial/test_greenctx_stream.py | 25 + .../tests/speculative/test_eagle_utils.py | 87 + .../speculative/test_speculative_sampling.py | 129 + sgl-kernel/tests/test_activation.py | 39 + .../tests/test_apply_token_bitmask_inplace.py | 23 + sgl-kernel/tests/test_awq_dequant.py | 115 + sgl-kernel/tests/test_bmm_fp8.py | 43 + sgl-kernel/tests/test_causal_conv1d.py | 489 +++ sgl-kernel/tests/test_custom_allreduce.py | 185 + sgl-kernel/tests/test_cutlass_mla.py | 104 + sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py | 283 ++ sgl-kernel/tests/test_dsv3_fused_a_gemm.py | 32 + sgl-kernel/tests/test_dsv3_router_gemm.py | 35 + sgl-kernel/tests/test_flash_attention.py | 1368 ++++++++ sgl-kernel/tests/test_flash_attention_4.py | 877 +++++ sgl-kernel/tests/test_fp4_gemm.py | 154 + sgl-kernel/tests/test_fp4_quantize.py | 261 ++ sgl-kernel/tests/test_fp8_blockwise_gemm.py | 93 + sgl-kernel/tests/test_fp8_blockwise_moe.py | 221 ++ sgl-kernel/tests/test_fp8_gemm.py | 49 + sgl-kernel/tests/test_gptq_kernel.py | 131 + sgl-kernel/tests/test_int8_gemm.py | 48 + sgl-kernel/tests/test_kvcacheio.py | 485 +++ .../tests/test_lightning_attention_decode.py | 84 + sgl-kernel/tests/test_marlin_gemm.py | 121 + sgl-kernel/tests/test_marlin_repack.py | 148 + sgl-kernel/tests/test_merge_state.py | 142 + sgl-kernel/tests/test_merge_state_v2.py | 400 +++ sgl-kernel/tests/test_moe_align.py | 250 ++ sgl-kernel/tests/test_moe_fused_gate.py | 103 + sgl-kernel/tests/test_moe_topk_softmax.py | 139 + sgl-kernel/tests/test_mscclpp.py | 146 + sgl-kernel/tests/test_norm.py | 133 + sgl-kernel/tests/test_per_tensor_quant_fp8.py | 67 + .../tests/test_per_token_group_quant_8bit.py | 97 + sgl-kernel/tests/test_per_token_quant_fp8.py | 57 + .../tests/test_qserve_w4a8_per_chn_gemm.py | 118 + .../tests/test_qserve_w4a8_per_group_gemm.py | 183 + sgl-kernel/tests/test_rotary_embedding.py | 138 + sgl-kernel/tests/test_sampling.py | 185 + sgl-kernel/tests/test_sparse_flash_attn.py | 492 +++ sgl-kernel/tests/utils.py | 9 + sgl-router/.cargo/config.toml | 15 + sgl-router/.coveragerc | 9 + sgl-router/Cargo.toml | 121 + sgl-router/MANIFEST.in | 5 + sgl-router/Makefile | 92 + sgl-router/README.md | 402 +++ sgl-router/benches/request_processing.rs | 692 ++++ sgl-router/benches/tokenizer_benchmark.rs | 1400 ++++++++ sgl-router/benches/tool_parser_benchmark.rs | 848 +++++ sgl-router/build.rs | 35 + sgl-router/py_src/sglang_router/__init__.py | 3 + .../py_src/sglang_router/launch_router.py | 109 + .../py_src/sglang_router/launch_server.py | 203 ++ sgl-router/py_src/sglang_router/mini_lb.py | 395 +++ sgl-router/py_src/sglang_router/router.py | 129 + .../py_src/sglang_router/router_args.py | 577 ++++ sgl-router/py_src/sglang_router/version.py | 1 + sgl-router/py_test/__init__.py | 1 + sgl-router/py_test/conftest.py | 8 + sgl-router/py_test/e2e/conftest.py | 512 +++ sgl-router/py_test/e2e/test_pd_router.py | 262 ++ sgl-router/py_test/e2e/test_regular_router.py | 169 + sgl-router/py_test/fixtures/__init__.py | 1 + sgl-router/py_test/fixtures/mock_worker.py | 252 ++ sgl-router/py_test/fixtures/ports.py | 8 + sgl-router/py_test/fixtures/router_manager.py | 158 + sgl-router/py_test/integration/__init__.py | 1 + sgl-router/py_test/integration/conftest.py | 109 + .../integration/load_balancing/__init__.py | 1 + .../load_balancing/test_cache_aware.py | 73 + .../load_balancing/test_power_of_two.py | 89 + .../integration/load_balancing/test_random.py | 33 + .../load_balancing/test_round_robin.py | 34 + .../py_test/integration/test_api_auth.py | 38 + .../integration/test_circuit_breaker.py | 191 ++ .../integration/test_fault_tolerance.py | 36 + .../py_test/integration/test_payload_size.py | 33 + .../py_test/integration/test_pd_routing.py | 127 + .../py_test/integration/test_rate_limiting.py | 91 + .../py_test/integration/test_retries.py | 65 + .../test_service_discovery_shim.py | 36 + .../integration/test_worker_management.py | 61 + sgl-router/py_test/unit/__init__.py | 7 + sgl-router/py_test/unit/test_arg_parser.py | 628 ++++ sgl-router/py_test/unit/test_router_config.py | 421 +++ .../py_test/unit/test_startup_sequence.py | 1053 ++++++ sgl-router/py_test/unit/test_validation.py | 506 +++ sgl-router/pyproject.toml | 31 + sgl-router/pytest.ini | 5 + sgl-router/scripts/run_benchmarks.py | 250 ++ sgl-router/setup.py | 21 + sgl-router/src/config/mod.rs | 28 + sgl-router/src/config/types.rs | 1241 +++++++ sgl-router/src/config/validation.rs | 776 +++++ sgl-router/src/core/circuit_breaker.rs | 555 +++ sgl-router/src/core/error.rs | 240 ++ sgl-router/src/core/mod.rs | 24 + sgl-router/src/core/retry.rs | 409 +++ sgl-router/src/core/token_bucket.rs | 195 ++ sgl-router/src/core/worker.rs | 1980 +++++++++++ sgl-router/src/grpc/client.rs | 327 ++ sgl-router/src/grpc/mod.rs | 8 + sgl-router/src/lib.rs | 508 +++ sgl-router/src/logging.rs | 163 + sgl-router/src/main.rs | 636 ++++ sgl-router/src/mcp/client_manager.rs | 535 +++ sgl-router/src/mcp/config.rs | 52 + sgl-router/src/mcp/error.rs | 42 + sgl-router/src/mcp/mod.rs | 18 + sgl-router/src/mcp/oauth.rs | 191 ++ sgl-router/src/metrics.rs | 1011 ++++++ sgl-router/src/middleware.rs | 502 +++ sgl-router/src/policies/cache_aware.rs | 423 +++ sgl-router/src/policies/factory.rs | 94 + sgl-router/src/policies/mod.rs | 148 + sgl-router/src/policies/power_of_two.rs | 201 ++ sgl-router/src/policies/random.rs | 121 + sgl-router/src/policies/round_robin.rs | 140 + sgl-router/src/proto/sglang_scheduler.proto | 541 +++ sgl-router/src/protocols/mod.rs | 5 + sgl-router/src/protocols/spec.rs | 2681 +++++++++++++++ sgl-router/src/protocols/validation.rs | 1164 +++++++ sgl-router/src/reasoning_parser/README.md | 474 +++ sgl-router/src/reasoning_parser/factory.rs | 566 +++ sgl-router/src/reasoning_parser/mod.rs | 10 + .../src/reasoning_parser/parsers/base.rs | 386 +++ .../reasoning_parser/parsers/deepseek_r1.rs | 112 + .../src/reasoning_parser/parsers/glm45.rs | 118 + .../src/reasoning_parser/parsers/kimi.rs | 137 + .../src/reasoning_parser/parsers/mod.rs | 13 + .../src/reasoning_parser/parsers/qwen3.rs | 178 + .../src/reasoning_parser/parsers/step3.rs | 123 + sgl-router/src/reasoning_parser/traits.rs | 130 + sgl-router/src/routers/factory.rs | 195 ++ sgl-router/src/routers/grpc/mod.rs | 4 + sgl-router/src/routers/grpc/pd_router.rs | 328 ++ sgl-router/src/routers/grpc/router.rs | 266 ++ sgl-router/src/routers/header_utils.rs | 53 + sgl-router/src/routers/http/mod.rs | 6 + sgl-router/src/routers/http/openai_router.rs | 379 ++ sgl-router/src/routers/http/pd_router.rs | 2500 ++++++++++++++ sgl-router/src/routers/http/pd_types.rs | 81 + sgl-router/src/routers/http/router.rs | 1387 ++++++++ sgl-router/src/routers/mod.rs | 107 + sgl-router/src/server.rs | 474 +++ sgl-router/src/service_discovery.rs | 1168 +++++++ sgl-router/src/tokenizer/README.md | 1021 ++++++ sgl-router/src/tokenizer/chat_template.rs | 182 + sgl-router/src/tokenizer/factory.rs | 318 ++ sgl-router/src/tokenizer/hub.rs | 238 ++ sgl-router/src/tokenizer/huggingface.rs | 234 ++ sgl-router/src/tokenizer/mock.rs | 112 + sgl-router/src/tokenizer/mod.rs | 123 + sgl-router/src/tokenizer/sequence.rs | 238 ++ sgl-router/src/tokenizer/stop.rs | 506 +++ sgl-router/src/tokenizer/stream.rs | 105 + sgl-router/src/tokenizer/tests.rs | 143 + sgl-router/src/tokenizer/tiktoken.rs | 276 ++ sgl-router/src/tokenizer/traits.rs | 83 + sgl-router/src/tool_parser/errors.rs | 32 + sgl-router/src/tool_parser/mod.rs | 30 + .../tool_parser/parsers/deepseek_parser.rs | 277 ++ .../tool_parser/parsers/glm4_moe_parser.rs | 292 ++ .../src/tool_parser/parsers/gpt_oss_parser.rs | 292 ++ .../src/tool_parser/parsers/json_parser.rs | 619 ++++ .../src/tool_parser/parsers/kimik2_parser.rs | 270 ++ .../src/tool_parser/parsers/llama_parser.rs | 156 + .../src/tool_parser/parsers/mistral_parser.rs | 347 ++ sgl-router/src/tool_parser/parsers/mod.rs | 27 + .../tool_parser/parsers/pythonic_parser.rs | 434 +++ .../src/tool_parser/parsers/qwen_parser.rs | 396 +++ .../src/tool_parser/parsers/step3_parser.rs | 348 ++ sgl-router/src/tool_parser/partial_json.rs | 527 +++ .../src/tool_parser/python_literal_parser.rs | 442 +++ sgl-router/src/tool_parser/registry.rs | 224 ++ sgl-router/src/tool_parser/state.rs | 181 + sgl-router/src/tool_parser/tests.rs | 886 +++++ sgl-router/src/tool_parser/traits.rs | 35 + sgl-router/src/tool_parser/types.rs | 73 + sgl-router/src/tree.rs | 1478 ++++++++ sgl-router/tests/api_endpoints_test.rs | 1669 +++++++++ sgl-router/tests/common/mock_mcp_server.rs | 177 + sgl-router/tests/common/mock_openai_server.rs | 238 ++ sgl-router/tests/common/mock_worker.rs | 614 ++++ sgl-router/tests/common/mod.rs | 103 + sgl-router/tests/common/test_app.rs | 52 + sgl-router/tests/mcp_test.rs | 457 +++ sgl-router/tests/request_formats_test.rs | 392 +++ sgl-router/tests/responses_api_test.rs | 210 ++ sgl-router/tests/streaming_tests.rs | 370 ++ sgl-router/tests/test_chat_template.rs | 150 + .../tests/test_chat_template_loading.rs | 183 + sgl-router/tests/test_openai_routing.rs | 419 +++ sgl-router/tests/test_pd_routing.rs | 918 +++++ sgl-router/tests/tokenizer_integration.rs | 320 ++ sgl-router/tests/tool_parser_deepseek.rs | 183 + sgl-router/tests/tool_parser_edge_cases.rs | 330 ++ sgl-router/tests/tool_parser_glm4_moe.rs | 194 ++ sgl-router/tests/tool_parser_gpt_oss.rs | 201 ++ sgl-router/tests/tool_parser_json.rs | 147 + sgl-router/tests/tool_parser_kimik2.rs | 160 + sgl-router/tests/tool_parser_llama.rs | 424 +++ sgl-router/tests/tool_parser_mistral.rs | 153 + .../tests/tool_parser_mixed_edge_cases.rs | 301 ++ sgl-router/tests/tool_parser_pythonic.rs | 559 +++ sgl-router/tests/tool_parser_qwen.rs | 259 ++ sgl-router/tests/tool_parser_registry.rs | 194 ++ sgl-router/tests/tool_parser_step3.rs | 245 ++ sgl-router/tests/tool_parser_streaming.rs | 341 ++ .../tests/tool_parser_wrapper_tokens.rs | 247 ++ test/README.md | 41 + test/lang/run_suite.py | 38 + test/lang/test_anthropic_backend.py | 25 + test/lang/test_bind_cache.py | 51 + test/lang/test_choices.py | 91 + test/lang/test_litellm_backend.py | 25 + test/lang/test_openai_backend.py | 92 + test/lang/test_separate_reasoning.py | 68 + .../lang/test_separate_reasoning_execution.py | 195 ++ test/lang/test_srt_backend.py | 86 + test/lang/test_tracing.py | 129 + test/lang/test_vertexai_backend.py | 53 + test/pytest.ini | 2 + test/srt/ascend/test_ascend_deepep.py | 121 + test/srt/ascend/test_ascend_graph_tp1_bf16.py | 95 + test/srt/ascend/test_ascend_graph_tp2_bf16.py | 97 + .../ascend/test_ascend_mla_fia_w8a8int8.py | 103 + test/srt/ascend/test_ascend_mla_w8a8int8.py | 101 + test/srt/ascend/test_ascend_tp1_bf16.py | 96 + test/srt/ascend/test_ascend_tp2_bf16.py | 98 + test/srt/ascend/test_ascend_tp2_fia_bf16.py | 101 + test/srt/ascend/test_ascend_tp4_bf16.py | 101 + .../ascend/test_ascend_w8a8_quantization.py | 104 + test/srt/configs/deepseek_v3.yaml | 28 + .../srt/configs/deepseek_v3_long_context.yaml | 28 + test/srt/configs/llama_405b.yaml | 28 + test/srt/configs/random_config.yaml | 25 + .../random_flashinfer_vs_triton_config.yaml | 25 + test/srt/configs/sharegpt_config.yaml | 7 + test/srt/cpu/test_activation.py | 55 + test/srt/cpu/test_binding.py | 28 + test/srt/cpu/test_decode.py | 170 + test/srt/cpu/test_extend.py | 190 + test/srt/cpu/test_gemm.py | 189 + test/srt/cpu/test_mla.py | 157 + test/srt/cpu/test_moe.py | 265 ++ test/srt/cpu/test_norm.py | 90 + test/srt/cpu/test_qkv_proj_with_rope.py | 432 +++ test/srt/cpu/test_rope.py | 178 + test/srt/cpu/test_shared_expert.py | 223 ++ test/srt/cpu/test_topk.py | 199 ++ test/srt/cpu/utils.py | 274 ++ ...sparsity-config-Llama-3.1-8B-Instruct.json | 1 + .../http_server/test_abort_request.py | 206 ++ test/srt/ep/test_deepep_internode.py | 445 +++ test/srt/ep/test_deepep_intranode.py | 379 ++ test/srt/ep/test_deepep_large.py | 149 + test/srt/ep/test_deepep_low_latency.py | 325 ++ test/srt/ep/test_deepep_small.py | 389 +++ test/srt/ep/test_eplb.py | 155 + test/srt/ep/test_hybrid_dp_ep_tp_mtp.py | 2728 +++++++++++++++ test/srt/ep/test_moe_deepep.py | 119 + .../ep/test_moe_deepep_eval_accuracy_large.py | 75 + test/srt/ep/test_moe_ep.py | 112 + test/srt/experiment_runner.py | 365 ++ test/srt/hicache/test_hicache.py | 53 + test/srt/hicache/test_hicache_mla.py | 67 + test/srt/hicache/test_hicache_page.py | 51 + test/srt/hicache/test_hicache_storage.py | 57 + .../test_hicache_storage_3fs_backend.py | 135 + .../test_hicache_storage_file_backend.py | 334 ++ test/srt/kv_cache_scales_llama3_1_8b.json | 42 + test/srt/kv_cache_scales_llama3_8b.json | 42 + test/srt/kv_cache_scales_qwen2_1_5b.json | 38 + test/srt/lora/test_lora.py | 172 + test/srt/lora/test_lora_backend.py | 76 + test/srt/lora/test_lora_cuda_graph.py | 110 + test/srt/lora/test_lora_eviction.py | 146 + test/srt/lora/test_lora_llama4.py | 61 + test/srt/lora/test_lora_qwen3.py | 212 ++ test/srt/lora/test_lora_radix_cache.py | 83 + test/srt/lora/test_lora_tp.py | 78 + test/srt/lora/test_lora_update.py | 1306 +++++++ test/srt/lora/test_multi_lora_backend.py | 90 + test/srt/lora/utils.py | 397 +++ test/srt/models/compare.py | 52 + test/srt/models/test_clip_models.py | 80 + .../models/test_compressed_tensors_models.py | 46 + test/srt/models/test_cross_encoder_models.py | 91 + test/srt/models/test_dummy_grok_models.py | 34 + test/srt/models/test_embedding_models.py | 111 + .../models/test_encoder_embedding_models.py | 162 + test/srt/models/test_generation_models.py | 181 + test/srt/models/test_gme_qwen_models.py | 85 + test/srt/models/test_grok_models.py | 53 + test/srt/models/test_llama4_models.py | 74 + test/srt/models/test_mtp_models.py | 58 + test/srt/models/test_qwen_models.py | 77 + test/srt/models/test_reward_models.py | 92 + test/srt/models/test_transformers_models.py | 181 + test/srt/models/test_unsloth_models.py | 213 ++ test/srt/models/test_vlm_models.py | 315 ++ test/srt/openai_server/__init__.py | 0 test/srt/openai_server/basic/__init__.py | 0 .../basic/test_openai_embedding.py | 97 + .../openai_server/basic/test_openai_server.py | 669 ++++ test/srt/openai_server/basic/test_protocol.py | 368 ++ .../openai_server/basic/test_serving_chat.py | 425 +++ .../basic/test_serving_completions.py | 157 + .../basic/test_serving_embedding.py | 145 + test/srt/openai_server/features/__init__.py | 0 .../features/test_cache_report.py | 212 ++ .../features/test_enable_thinking.py | 243 ++ .../features/test_json_constrained.py | 153 + .../openai_server/features/test_json_mode.py | 137 + .../features/test_openai_server_ebnf.py | 98 + .../test_openai_server_hidden_states.py | 356 ++ .../features/test_reasoning_content.py | 343 ++ .../openai_server/function_call/__init__.py | 0 .../test_openai_function_calling.py | 953 +++++ .../function_call/test_tool_choice.py | 550 +++ test/srt/openai_server/validation/__init__.py | 0 .../validation/test_large_max_new_tokens.py | 103 + .../validation/test_matched_stop.py | 140 + .../test_openai_server_ignore_eos.py | 84 + .../test_request_length_validation.py | 88 + test/srt/parse_results.py | 57 + test/srt/quant/test_awq.py | 45 + test/srt/quant/test_awq_dequant.py | 175 + test/srt/quant/test_block_int8.py | 227 ++ test/srt/quant/test_fp8_kernel.py | 126 + test/srt/quant/test_fp8_kvcache.py | 114 + test/srt/quant/test_int8_kernel.py | 169 + test/srt/quant/test_triton_scaled_mm.py | 94 + test/srt/quant/test_w4a8_deepseek_v3.py | 122 + test/srt/quant/test_w8a8_quantization.py | 75 + test/srt/rl/test_update_weights_from_disk.py | 371 ++ .../test_update_weights_from_distributed.py | 615 ++++ .../srt/rl/test_update_weights_from_tensor.py | 178 + test/srt/rl/test_verl_engine_2_gpu.py | 277 ++ test/srt/rl/test_verl_engine_4_gpu.py | 291 ++ test/srt/run_suite.py | 410 +++ test/srt/test_abort.py | 114 + test/srt/test_bench_one_batch.py | 73 + test/srt/test_bench_serving.py | 445 +++ test/srt/test_bnb.py | 314 ++ test/srt/test_chunked_prefill.py | 31 + test/srt/test_cpp_radix_cache.py | 47 + test/srt/test_cpu_graph.py | 87 + test/srt/test_create_kvindices.py | 75 + test/srt/test_custom_allreduce.py | 174 + test/srt/test_data_parallelism.py | 78 + test/srt/test_disaggregation.py | 434 +++ test/srt/test_disaggregation_different_tp.py | 184 + test/srt/test_disaggregation_pp.py | 101 + test/srt/test_double_sparsity.py | 65 + test/srt/test_dp_attention.py | 170 + test/srt/test_eagle_infer_a.py | 323 ++ test/srt/test_eagle_infer_b.py | 511 +++ test/srt/test_ebnf_constrained.py | 282 ++ test/srt/test_eval_accuracy_large.py | 94 + test/srt/test_eval_fp8_accuracy.py | 114 + test/srt/test_expert_distribution.py | 102 + test/srt/test_expert_location_updater.py | 255 ++ test/srt/test_fa3.py | 261 ++ test/srt/test_fim_completion.py | 72 + test/srt/test_flashmla.py | 153 + test/srt/test_forward_split_prefill.py | 299 ++ test/srt/test_full_deepseek_v3.py | 162 + test/srt/test_function_call_parser.py | 2194 ++++++++++++ test/srt/test_fused_moe.py | 236 ++ test/srt/test_get_weights_by_name.py | 183 + test/srt/test_gguf.py | 27 + test/srt/test_gpt_oss_1gpu.py | 31 + test/srt/test_gpt_oss_4gpu.py | 36 + test/srt/test_gpt_oss_common.py | 111 + test/srt/test_gptqmodel_dynamic.py | 204 ++ test/srt/test_harmony_parser.py | 876 +++++ test/srt/test_health_check.py | 28 + test/srt/test_hidden_states.py | 139 + test/srt/test_hybrid_attn_backend.py | 161 + test/srt/test_input_embeddings.py | 151 + test/srt/test_intel_amx_attention_backend.py | 135 + test/srt/test_io_struct.py | 577 ++++ test/srt/test_jinja_template_utils.py | 310 ++ test/srt/test_kv_events.py | 377 ++ test/srt/test_local_attn.py | 72 + test/srt/test_metrics.py | 88 + test/srt/test_metrics_utils.py | 137 + test/srt/test_mla.py | 52 + test/srt/test_mla_deepseek_v3.py | 190 + test/srt/test_mla_flashinfer.py | 124 + test/srt/test_mla_fp8.py | 49 + test/srt/test_mla_int8_deepseek_v3.py | 214 ++ test/srt/test_mla_tp.py | 66 + test/srt/test_modelopt.py | 58 + test/srt/test_modelopt_fp8kvcache.py | 30 + test/srt/test_models_from_modelscope.py | 40 + test/srt/test_moe_eval_accuracy_large.py | 94 + test/srt/test_mscclpp.py | 205 ++ ...ulti_instance_release_memory_occupation.py | 250 ++ test/srt/test_multi_tokenizer.py | 84 + test/srt/test_nightly_gsm8k_eval.py | 171 + test/srt/test_nightly_gsm8k_eval_amd.py | 221 ++ test/srt/test_no_chunked_prefill.py | 30 + test/srt/test_no_overlap_scheduler.py | 35 + test/srt/test_original_logprobs.py | 196 ++ test/srt/test_page_size.py | 47 + test/srt/test_patch_torch.py | 133 + test/srt/test_penalty.py | 95 + test/srt/test_pp_single_node.py | 322 ++ test/srt/test_pytorch_sampling_backend.py | 90 + test/srt/test_quick_allreduce.py | 212 ++ test/srt/test_radix_attention.py | 136 + test/srt/test_reasoning_parser.py | 603 ++++ test/srt/test_regex_constrained.py | 187 + test/srt/test_release_memory_occupation.py | 328 ++ test/srt/test_request_queue_validation.py | 87 + test/srt/test_retract_decode.py | 60 + test/srt/test_rope_rocm.py | 116 + test/srt/test_sagemaker_server.py | 179 + test/srt/test_schedule_policy.py | 65 + test/srt/test_score_api.py | 300 ++ test/srt/test_server_args.py | 273 ++ test/srt/test_session_control.py | 786 +++++ test/srt/test_skip_tokenizer_init.py | 246 ++ test/srt/test_srt_endpoint.py | 640 ++++ test/srt/test_srt_engine.py | 219 ++ test/srt/test_srt_engine_with_quant_args.py | 60 + .../test_standalone_speculative_decoding.py | 115 + test/srt/test_start_profile.py | 115 + test/srt/test_swa_unittest.py | 182 + test/srt/test_tokenizer_batch_encode.py | 121 + test/srt/test_torch_compile.py | 80 + test/srt/test_torch_compile_moe.py | 77 + .../test_torch_native_attention_backend.py | 47 + test/srt/test_torch_tp.py | 30 + test/srt/test_torchao.py | 74 + test/srt/test_triton_attention_backend.py | 66 + test/srt/test_triton_attention_kernels.py | 576 ++++ test/srt/test_triton_attention_rocm_mla.py | 259 ++ test/srt/test_triton_fused_moe.py | 162 + .../srt/test_triton_moe_channel_fp8_kernel.py | 181 + test/srt/test_triton_moe_wna16.py | 243 ++ test/srt/test_triton_sliding_window.py | 129 + test/srt/test_two_batch_overlap.py | 152 + test/srt/test_utils_update_weights.py | 170 + test/srt/test_vertex_endpoint.py | 64 + test/srt/test_vision_chunked_prefill.py | 188 + test/srt/test_vision_openai_server_a.py | 342 ++ test/srt/test_vision_openai_server_b.py | 250 ++ test/srt/test_vision_openai_server_common.py | 527 +++ test/srt/test_vllm_dependency.py | 164 + test/srt/test_vlm_accuracy.py | 324 ++ test/srt/test_vlm_input_format.py | 258 ++ test/srt/test_wave_attention_backend.py | 61 + test/srt/test_wave_attention_kernels.py | 322 ++ test/srt/test_weight_version.py | 227 ++ 2037 files changed, 515371 insertions(+) create mode 100644 .clang-format-ignore create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .editorconfig create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/1-bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/2-feature-request.yml create mode 100644 .github/REVIEWERS.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/cancel-all-pending-pr-test-runs.yml create mode 100644 .github/workflows/cancel-pr-workflow-on-merge.yml create mode 100644 .github/workflows/close-inactive-issues.yml create mode 100644 .github/workflows/execute-notebook.yml create mode 100644 .github/workflows/experiment-runner.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nightly-test-amd.yml create mode 100644 .github/workflows/nightly-test.yml create mode 100644 .github/workflows/open-pr-copy-from-oss.yml create mode 100644 .github/workflows/open-pr-copy-to-oss.yml create mode 100644 .github/workflows/pr-benchmark-rust.yml create mode 100644 .github/workflows/pr-test-amd.yml create mode 100644 .github/workflows/pr-test-h20.yml create mode 100644 .github/workflows/pr-test-npu.yml create mode 100644 .github/workflows/pr-test-pd-router.yml create mode 100644 .github/workflows/pr-test-rust.yml create mode 100644 .github/workflows/pr-test-sgl-kernel.yml create mode 100644 .github/workflows/pr-test-xeon.yml create mode 100644 .github/workflows/pr-test.yml create mode 100644 .github/workflows/release-docker-amd-nightly.yml create mode 100644 .github/workflows/release-docker-amd.yml create mode 100644 .github/workflows/release-docker-dev.yml create mode 100644 .github/workflows/release-docker-gb200.yml create mode 100644 .github/workflows/release-docker-npu-nightly.yml create mode 100644 .github/workflows/release-docker-npu.yml create mode 100644 .github/workflows/release-docker-router.yml create mode 100644 .github/workflows/release-docker-xeon.yml create mode 100644 .github/workflows/release-docker.yml create mode 100644 .github/workflows/release-docs.yml create mode 100644 .github/workflows/release-fake-tag.yml create mode 100644 .github/workflows/release-pypi-router.yml create mode 100644 .github/workflows/release-pypi.yml create mode 100644 .github/workflows/release-whl-kernel-cu118.yml create mode 100644 .github/workflows/release-whl-kernel.yml create mode 100644 .github/workflows/vllm-dependency-test.yml create mode 100644 .gitignore create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 3rdparty/amd/profiling/PROFILING.md create mode 100755 3rdparty/amd/profiling/client.sh create mode 100644 3rdparty/amd/profiling/install_rpd.sh create mode 100755 3rdparty/amd/profiling/loadTracer.sh create mode 100644 3rdparty/amd/profiling/rpd.patch create mode 100644 3rdparty/amd/profiling/rpd_profile_server_enable.patch create mode 100644 3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch create mode 100755 3rdparty/amd/profiling/server.sh create mode 100644 3rdparty/amd/profiling/torch_profiler.patch create mode 100644 3rdparty/amd/tuning/TUNING.md create mode 100644 3rdparty/amd/tuning/benchmark_moe_rocm.py create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 assets/logo.png create mode 100644 assets/logo.svg create mode 100644 assets/logo_square.svg create mode 100644 benchmark/bench_attention_sink/bench_attention_sink_triton.py create mode 100644 benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py create mode 100644 benchmark/benchmark_batch/benchmark_batch.py create mode 100644 benchmark/benchmark_batch/benchmark_tokenizer.py create mode 100644 benchmark/benchmark_vllm_060/README.md create mode 100644 benchmark/blog_v0_2/405b_sglang.sh create mode 100644 benchmark/blog_v0_2/405b_trt.sh create mode 100644 benchmark/blog_v0_2/405b_vllm.sh create mode 100644 benchmark/blog_v0_2/README.md create mode 100644 benchmark/blog_v0_2/config.md create mode 100644 benchmark/boolq/README.md create mode 100644 benchmark/boolq/bench_sglang.py create mode 100644 benchmark/boolq/convert_parquet_to_json.py create mode 100755 benchmark/boolq/parquet_to_json.sh create mode 100644 benchmark/ceval/README.md create mode 100644 benchmark/ceval/bench_sglang.py create mode 100644 benchmark/deepseek_v3/README.md create mode 100644 benchmark/dspy/README.md create mode 100644 benchmark/dspy/bench_dspy_intro.py create mode 100644 benchmark/generative_agents/README.md create mode 100644 benchmark/generative_agents/agent_functions.py create mode 100644 benchmark/generative_agents/bench_other.py create mode 100644 benchmark/generative_agents/bench_sglang.py create mode 100644 benchmark/gpt_oss/README.md create mode 100644 benchmark/gsm8k/README.md create mode 100644 benchmark/gsm8k/bench_other.py create mode 100644 benchmark/gsm8k/bench_sglang.py create mode 100644 benchmark/hellaswag/README.md create mode 100644 benchmark/hellaswag/bench_other.py create mode 100644 benchmark/hellaswag/bench_sglang.py create mode 100644 benchmark/hf3fs/bench.sh create mode 100644 benchmark/hf3fs/bench_client.py create mode 100644 benchmark/hf3fs/bench_storage.py create mode 100644 benchmark/hf3fs/bench_zerocopy.py create mode 100644 benchmark/hicache/README.md create mode 100644 benchmark/hicache/bench_long_context.py create mode 100644 benchmark/hicache/bench_mix.py create mode 100755 benchmark/hicache/bench_mix.sh create mode 100644 benchmark/hicache/bench_multiturn.py create mode 100644 benchmark/hicache/bench_serving.py create mode 100644 benchmark/hicache/data_processing.py create mode 100755 benchmark/hicache/download.sh create mode 100644 benchmark/hicache/nextqa.py create mode 100644 benchmark/json_decode_regex/README.md create mode 100644 benchmark/json_decode_regex/bench_other.py create mode 100644 benchmark/json_decode_regex/bench_sglang.py create mode 100644 benchmark/json_decode_regex/build_dataset.py create mode 100644 benchmark/json_jump_forward/README.md create mode 100644 benchmark/json_jump_forward/bench_other.py create mode 100644 benchmark/json_jump_forward/bench_sglang.py create mode 100644 benchmark/json_jump_forward/build_dataset.py create mode 100644 benchmark/json_jump_forward/dataset.txt create mode 100644 benchmark/json_schema/README.md create mode 100644 benchmark/json_schema/bench_sglang.py create mode 100644 benchmark/kernels/all_reduce/benchmark_mscclpp.py create mode 100644 benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py create mode 100644 benchmark/kernels/deepep/deepep_utils.py create mode 100644 benchmark/kernels/deepep/tuning_deepep.py create mode 100644 benchmark/kernels/deepseek/README.md create mode 100644 benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py create mode 100644 benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py create mode 100644 benchmark/kernels/fbgemm/README.md create mode 100644 benchmark/kernels/fbgemm/benchmark_fbgemm_grouped_gemm.py create mode 100644 benchmark/kernels/flashinfer_allreduce_fusion/README.md create mode 100644 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py create mode 100644 benchmark/kernels/fused_moe_triton/README.md create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_sum_scale.py create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py create mode 100644 benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py create mode 100644 benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py create mode 100644 benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py create mode 100644 benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py create mode 100644 benchmark/kernels/quantization/bench_fp4_quant.py create mode 100644 benchmark/kernels/quantization/bench_int8_quant.py create mode 100644 benchmark/kernels/quantization/tuning_block_wise_kernel.py create mode 100644 benchmark/kernels/rmsnorm/benchmark_rmsnorm.py create mode 100644 benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py create mode 100644 benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py create mode 100644 benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py create mode 100644 benchmark/line_retrieval/README.md create mode 100644 benchmark/line_retrieval/bench_sglang.py create mode 100644 benchmark/line_retrieval/gen_data.py create mode 100644 benchmark/llava_bench/README.md create mode 100755 benchmark/llava_bench/bench_hf_llava_bench.sh create mode 100755 benchmark/llava_bench/bench_hf_mme.sh create mode 100644 benchmark/llava_bench/bench_sglang.py create mode 100644 benchmark/llava_bench/bench_sglang_mme.sh create mode 100644 benchmark/llava_bench/download_images.py create mode 100644 benchmark/llm_judge/README.md create mode 100644 benchmark/llm_judge/bench_other.py create mode 100644 benchmark/llm_judge/bench_sglang.py create mode 100644 benchmark/long_json_decode/README.md create mode 100644 benchmark/long_json_decode/bench_other.py create mode 100644 benchmark/long_json_decode/bench_sglang.py create mode 100644 benchmark/long_json_decode/build_dataset.py create mode 100644 benchmark/lora/launch_server.py create mode 100644 benchmark/lora/lora_bench.py create mode 100644 benchmark/mmlu/README.md create mode 100644 benchmark/mmlu/bench_other.py create mode 100644 benchmark/mmlu/bench_sglang.py create mode 100644 benchmark/mmlu/download_data.sh create mode 100644 benchmark/mmmu/README.md create mode 100644 benchmark/mmmu/bench_hf.py create mode 100644 benchmark/mmmu/bench_sglang.py create mode 100644 benchmark/mmmu/data_utils.py create mode 100644 benchmark/mmmu/eval_utils.py create mode 100644 benchmark/mmmu/internvl_utils.py create mode 100644 benchmark/mmmu/prompt_format.yaml create mode 100644 benchmark/mtbench/README.md create mode 100644 benchmark/mtbench/bench_other.py create mode 100644 benchmark/mtbench/bench_sglang.py create mode 100644 benchmark/mtbench/bench_sglang_eagle.py create mode 100644 benchmark/multi_chain_reasoning/README.md create mode 100644 benchmark/multi_chain_reasoning/bench_other.py create mode 100644 benchmark/multi_chain_reasoning/bench_sglang.py create mode 100644 benchmark/multi_document_qa/README.md create mode 100644 benchmark/multi_document_qa/bench_other.py create mode 100644 benchmark/multi_document_qa/bench_sglang.py create mode 100644 benchmark/multi_document_qa/build_dataset.py create mode 100644 benchmark/multi_turn_chat/README.md create mode 100644 benchmark/multi_turn_chat/bench_other.py create mode 100644 benchmark/multi_turn_chat/bench_sglang.py create mode 100644 benchmark/multi_turn_chat/data_gen.py create mode 100644 benchmark/multi_turn_chat/long_prompt_multi_turn.py create mode 100644 benchmark/prefill_only/bench_embeddings.py create mode 100644 benchmark/prefill_only/bench_score.py create mode 100644 benchmark/prefill_only/util.py create mode 100644 benchmark/react/README.md create mode 100644 benchmark/react/bench_other.py create mode 100644 benchmark/react/bench_sglang.py create mode 100644 benchmark/reasoning_benchmark/README.md create mode 100644 benchmark/reasoning_benchmark/answer_extraction.py create mode 100644 benchmark/reasoning_benchmark/bench_sglang.py create mode 100644 benchmark/reasoning_benchmark/eval_utils.py create mode 100644 benchmark/tip_suggestion/.gitignore create mode 100644 benchmark/tip_suggestion/README.md create mode 100644 benchmark/tip_suggestion/bench_other.py create mode 100644 benchmark/tip_suggestion/bench_sglang.py create mode 100644 benchmark/tip_suggestion/lmql_funcs.py create mode 100644 benchmark/tip_suggestion/topic.jsonl create mode 100644 benchmark/tree_of_thought_deep/README.md create mode 100644 benchmark/tree_of_thought_deep/bench_other.py create mode 100644 benchmark/tree_of_thought_deep/bench_sglang.py create mode 100644 benchmark/tree_of_thought_deep/lmql_funcs.py create mode 100644 benchmark/tree_of_thought_v0/README.md create mode 100644 benchmark/tree_of_thought_v0/bench_other.py create mode 100644 benchmark/tree_of_thought_v0/bench_sglang.py create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.gb200 create mode 100644 docker/Dockerfile.npu create mode 100644 docker/Dockerfile.rocm create mode 100644 docker/Dockerfile.router create mode 100644 docker/Dockerfile.sagemaker create mode 100644 docker/Dockerfile.xeon create mode 100644 docker/compose.yaml create mode 100644 docker/k8s-sglang-distributed-sts.yaml create mode 100644 docker/k8s-sglang-service.yaml create mode 100755 docker/serve create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/_static/css/custom_log.css create mode 100644 docs/_static/css/readthedocs.css create mode 100644 docs/_static/image/logo.ico create mode 100644 docs/_static/image/logo.png create mode 100644 docs/advanced_features/attention_backend.md create mode 100644 docs/advanced_features/hyperparameter_tuning.md create mode 100644 docs/advanced_features/lora.ipynb create mode 100644 docs/advanced_features/observability.md create mode 100644 docs/advanced_features/pd_disaggregation.md create mode 100644 docs/advanced_features/quantization.md create mode 100644 docs/advanced_features/router.md create mode 100644 docs/advanced_features/separate_reasoning.ipynb create mode 100644 docs/advanced_features/server_arguments.md create mode 100644 docs/advanced_features/speculative_decoding.ipynb create mode 100644 docs/advanced_features/structured_outputs.ipynb create mode 100644 docs/advanced_features/structured_outputs_for_reasoning_models.ipynb create mode 100644 docs/advanced_features/tool_parser.ipynb create mode 100644 docs/advanced_features/vlm_query.ipynb create mode 100644 docs/basic_usage/deepseek.md create mode 100644 docs/basic_usage/gpt_oss.md create mode 100644 docs/basic_usage/llama4.md create mode 100644 docs/basic_usage/native_api.ipynb create mode 100644 docs/basic_usage/offline_engine_api.ipynb create mode 100644 docs/basic_usage/openai_api.rst create mode 100644 docs/basic_usage/openai_api_completions.ipynb create mode 100644 docs/basic_usage/openai_api_embeddings.ipynb create mode 100644 docs/basic_usage/openai_api_vision.ipynb create mode 100644 docs/basic_usage/qwen3.md create mode 100644 docs/basic_usage/sampling_params.md create mode 100644 docs/basic_usage/send_request.ipynb create mode 100644 docs/conf.py create mode 100644 docs/deploy.py create mode 100644 docs/developer_guide/bench_serving.md create mode 100644 docs/developer_guide/benchmark_and_profiling.md create mode 100644 docs/developer_guide/contribution_guide.md create mode 100644 docs/developer_guide/development_guide_using_docker.md create mode 100644 docs/developer_guide/release_process.md create mode 100644 docs/developer_guide/setup_github_runner.md create mode 100644 docs/get_started/install.md create mode 100644 docs/index.rst create mode 100644 docs/platforms/amd_gpu.md create mode 100644 docs/platforms/ascend_npu.md create mode 100644 docs/platforms/blackwell_gpu.md create mode 100644 docs/platforms/cpu_server.md create mode 100644 docs/platforms/nvidia_jetson.md create mode 100644 docs/platforms/tpu.md create mode 100644 docs/references/custom_chat_template.md create mode 100644 docs/references/environment_variables.md create mode 100644 docs/references/faq.md create mode 100644 docs/references/frontend/choices_methods.md create mode 100644 docs/references/frontend/frontend_index.rst create mode 100644 docs/references/frontend/frontend_tutorial.ipynb create mode 100644 docs/references/learn_more.md create mode 100644 docs/references/multi_node_deployment/deploy_on_k8s.md create mode 100644 docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml create mode 100644 docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml create mode 100644 docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml create mode 100644 docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml create mode 100644 docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml create mode 100644 docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md create mode 100644 docs/references/multi_node_deployment/multi_node.md create mode 100644 docs/references/multi_node_deployment/multi_node_index.rst create mode 100644 docs/references/production_metrics.md create mode 100644 docs/references/torch_compile_cache.md create mode 100644 docs/requirements.txt create mode 100644 docs/serve.sh create mode 100644 docs/supported_models/embedding_models.md create mode 100644 docs/supported_models/generative_models.md create mode 100644 docs/supported_models/modelscope.md create mode 100644 docs/supported_models/multimodal_language_models.md create mode 100644 docs/supported_models/rerank_models.md create mode 100644 docs/supported_models/reward_models.md create mode 100644 docs/supported_models/support_new_models.md create mode 100644 docs/supported_models/transformers_fallback.md create mode 100644 docs/wrap_run_llm.py create mode 100644 examples/chat_template/tool_chat_template_deepseekr1.jinja create mode 100644 examples/chat_template/tool_chat_template_deepseekv3.jinja create mode 100644 examples/chat_template/tool_chat_template_deepseekv31.jinja create mode 100644 examples/chat_template/tool_chat_template_llama4_pythonic.jinja create mode 100644 examples/frontend_language/quick_start/anthropic_example_chat.py create mode 100644 examples/frontend_language/quick_start/anthropic_example_complete.py create mode 100644 examples/frontend_language/quick_start/azure_openai_example_chat.py create mode 100644 examples/frontend_language/quick_start/gemini_example_chat.py create mode 100644 examples/frontend_language/quick_start/gemini_example_complete.py create mode 100644 examples/frontend_language/quick_start/gemini_example_multimodal_chat.py create mode 100644 examples/frontend_language/quick_start/images/cat.jpeg create mode 100644 examples/frontend_language/quick_start/images/dog.jpeg create mode 100644 examples/frontend_language/quick_start/local_example_chat.py create mode 100644 examples/frontend_language/quick_start/local_example_complete.py create mode 100644 examples/frontend_language/quick_start/local_example_llava_next.py create mode 100644 examples/frontend_language/quick_start/openai_example_chat.py create mode 100644 examples/frontend_language/quick_start/openai_example_complete.py create mode 100644 examples/frontend_language/quick_start/openai_example_n.py create mode 100644 examples/frontend_language/quick_start/openai_example_o1.py create mode 100644 examples/frontend_language/quick_start/openrouter_example_chat.py create mode 100644 examples/frontend_language/quick_start/together_example_chat.py create mode 100644 examples/frontend_language/quick_start/together_example_complete.py create mode 100644 examples/frontend_language/usage/chinese_regex.py create mode 100644 examples/frontend_language/usage/choices_logprob.py create mode 100644 examples/frontend_language/usage/cot_decoding.py create mode 100644 examples/frontend_language/usage/json_decode.py create mode 100644 examples/frontend_language/usage/json_logprobs.py create mode 100644 examples/frontend_language/usage/llava_video/srt_example_llava_v.py create mode 100755 examples/frontend_language/usage/llava_video/srt_example_llava_v.sh create mode 100644 examples/frontend_language/usage/openai_chat_speculative.py create mode 100644 examples/frontend_language/usage/openai_speculative.py create mode 100644 examples/frontend_language/usage/parallel_sample.py create mode 100644 examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb create mode 100644 examples/frontend_language/usage/readme_examples.py create mode 100644 examples/frontend_language/usage/sgl_gen_min_tokens.py create mode 100644 examples/frontend_language/usage/streaming.py create mode 100644 examples/frontend_language/usage/triton/Dockerfile create mode 100644 examples/frontend_language/usage/triton/README.md create mode 100644 examples/frontend_language/usage/triton/models/character_generation/config.pbtxt create mode 100644 examples/monitoring/README.md create mode 100644 examples/monitoring/docker-compose.yaml create mode 100644 examples/monitoring/grafana/dashboards/config/dashboard.yaml create mode 100644 examples/monitoring/grafana/dashboards/json/sglang-dashboard.json create mode 100644 examples/monitoring/grafana/datasources/datasource.yaml create mode 100644 examples/monitoring/prometheus.yaml create mode 100644 examples/profiler/nsys_profile_tools/README.md create mode 100755 examples/profiler/nsys_profile_tools/gputrc2graph.py create mode 100644 examples/profiler/nsys_profile_tools/sglang_engine_model.json create mode 100644 examples/runtime/README.md create mode 100644 examples/runtime/engine/custom_server.py create mode 100644 examples/runtime/engine/embedding.py create mode 100644 examples/runtime/engine/fastapi_engine_inference.py create mode 100644 examples/runtime/engine/launch_engine.py create mode 100644 examples/runtime/engine/offline_batch_inference.py create mode 100644 examples/runtime/engine/offline_batch_inference_async.py create mode 100644 examples/runtime/engine/offline_batch_inference_eagle.py create mode 100644 examples/runtime/engine/offline_batch_inference_qwen_1m.py create mode 100644 examples/runtime/engine/offline_batch_inference_vlm.py create mode 100644 examples/runtime/engine/readme.md create mode 100644 examples/runtime/engine/save_remote_state.py create mode 100644 examples/runtime/engine/save_sharded_state.py create mode 100644 examples/runtime/hidden_states/hidden_states_engine.py create mode 100644 examples/runtime/hidden_states/hidden_states_server.py create mode 100644 examples/runtime/lora.py create mode 100644 examples/runtime/multimodal/llama3_llava_server.py create mode 100644 examples/runtime/multimodal/llava_onevision_server.py create mode 100644 examples/runtime/multimodal/pixtral_server.py create mode 100644 examples/runtime/multimodal/qwen_llava_server.py create mode 100644 examples/runtime/multimodal_embedding.py create mode 100644 examples/runtime/openai_chat_with_response_prefill.py create mode 100644 examples/runtime/reward_model.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_llm_server.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py create mode 100644 examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py create mode 100644 examples/runtime/vertex_predict.py create mode 100644 package-lock.json create mode 100755 python/pyproject.toml create mode 100644 python/sglang/README.md create mode 100644 python/sglang/__init__.py create mode 100644 python/sglang/bench_offline_throughput.py create mode 100644 python/sglang/bench_one_batch.py create mode 100644 python/sglang/bench_one_batch_server.py create mode 100644 python/sglang/bench_serving.py create mode 100644 python/sglang/check_env.py create mode 100644 python/sglang/compile_deep_gemm.py create mode 100644 python/sglang/eval/llama3_eval.py create mode 100644 python/sglang/eval/loogle_eval.py create mode 100644 python/sglang/global_config.py create mode 100644 python/sglang/lang/api.py create mode 100644 python/sglang/lang/backend/anthropic.py create mode 100644 python/sglang/lang/backend/base_backend.py create mode 100644 python/sglang/lang/backend/litellm.py create mode 100644 python/sglang/lang/backend/openai.py create mode 100644 python/sglang/lang/backend/runtime_endpoint.py create mode 100644 python/sglang/lang/backend/vertexai.py create mode 100644 python/sglang/lang/chat_template.py create mode 100644 python/sglang/lang/choices.py create mode 100644 python/sglang/lang/compiler.py create mode 100644 python/sglang/lang/interpreter.py create mode 100644 python/sglang/lang/ir.py create mode 100644 python/sglang/lang/tracer.py create mode 100644 python/sglang/launch_server.py create mode 100644 python/sglang/profiler.py create mode 100644 python/sglang/srt/_custom_ops.py create mode 100644 python/sglang/srt/aio_rwlock.py create mode 100644 python/sglang/srt/bench_utils.py create mode 100644 python/sglang/srt/configs/__init__.py create mode 100644 python/sglang/srt/configs/chatglm.py create mode 100644 python/sglang/srt/configs/dbrx.py create mode 100644 python/sglang/srt/configs/deepseekvl2.py create mode 100644 python/sglang/srt/configs/device_config.py create mode 100644 python/sglang/srt/configs/exaone.py create mode 100644 python/sglang/srt/configs/internvl.py create mode 100644 python/sglang/srt/configs/janus_pro.py create mode 100644 python/sglang/srt/configs/kimi_vl.py create mode 100644 python/sglang/srt/configs/kimi_vl_moonvit.py create mode 100644 python/sglang/srt/configs/load_config.py create mode 100644 python/sglang/srt/configs/longcat_flash.py create mode 100644 python/sglang/srt/configs/model_config.py create mode 100644 python/sglang/srt/configs/qwen3_next.py create mode 100644 python/sglang/srt/configs/step3_vl.py create mode 100644 python/sglang/srt/configs/update_config.py create mode 100644 python/sglang/srt/configs/utils.py create mode 100644 python/sglang/srt/connector/__init__.py create mode 100644 python/sglang/srt/connector/base_connector.py create mode 100644 python/sglang/srt/connector/redis.py create mode 100644 python/sglang/srt/connector/s3.py create mode 100644 python/sglang/srt/connector/serde/__init__.py create mode 100644 python/sglang/srt/connector/serde/safe_serde.py create mode 100644 python/sglang/srt/connector/serde/serde.py create mode 100644 python/sglang/srt/connector/utils.py create mode 100644 python/sglang/srt/constants.py create mode 100644 python/sglang/srt/constrained/base_grammar_backend.py create mode 100644 python/sglang/srt/constrained/llguidance_backend.py create mode 100644 python/sglang/srt/constrained/outlines_backend.py create mode 100644 python/sglang/srt/constrained/outlines_jump_forward.py create mode 100644 python/sglang/srt/constrained/reasoner_grammar_backend.py create mode 100644 python/sglang/srt/constrained/triton_ops/bitmask_ops.py create mode 100644 python/sglang/srt/constrained/xgrammar_backend.py create mode 100644 python/sglang/srt/custom_op.py create mode 100644 python/sglang/srt/debug_utils/__init__.py create mode 100644 python/sglang/srt/debug_utils/dump_comparator.py create mode 100644 python/sglang/srt/debug_utils/dump_loader.py create mode 100644 python/sglang/srt/debug_utils/dumper.py create mode 100644 python/sglang/srt/debug_utils/text_comparator.py create mode 100644 python/sglang/srt/disaggregation/ascend/__init__.py create mode 100644 python/sglang/srt/disaggregation/ascend/conn.py create mode 100644 python/sglang/srt/disaggregation/ascend/transfer_engine.py create mode 100644 python/sglang/srt/disaggregation/base/__init__.py create mode 100644 python/sglang/srt/disaggregation/base/conn.py create mode 100644 python/sglang/srt/disaggregation/common/__init__.py create mode 100644 python/sglang/srt/disaggregation/common/conn.py create mode 100644 python/sglang/srt/disaggregation/common/utils.py create mode 100644 python/sglang/srt/disaggregation/decode.py create mode 100644 python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py create mode 100644 python/sglang/srt/disaggregation/fake/__init__.py create mode 100644 python/sglang/srt/disaggregation/fake/conn.py create mode 100644 python/sglang/srt/disaggregation/kv_events.py create mode 100644 python/sglang/srt/disaggregation/mini_lb.py create mode 100644 python/sglang/srt/disaggregation/mooncake/__init__.py create mode 100644 python/sglang/srt/disaggregation/mooncake/conn.py create mode 100644 python/sglang/srt/disaggregation/mooncake/transfer_engine.py create mode 100644 python/sglang/srt/disaggregation/nixl/__init__.py create mode 100644 python/sglang/srt/disaggregation/nixl/conn.py create mode 100644 python/sglang/srt/disaggregation/prefill.py create mode 100644 python/sglang/srt/disaggregation/utils.py create mode 100644 python/sglang/srt/distributed/__init__.py create mode 100644 python/sglang/srt/distributed/communication_op.py create mode 100644 python/sglang/srt/distributed/device_communicators/cuda_wrapper.py create mode 100644 python/sglang/srt/distributed/device_communicators/custom_all_reduce.py create mode 100644 python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py create mode 100644 python/sglang/srt/distributed/device_communicators/hpu_communicator.py create mode 100644 python/sglang/srt/distributed/device_communicators/npu_communicator.py create mode 100644 python/sglang/srt/distributed/device_communicators/pymscclpp.py create mode 100644 python/sglang/srt/distributed/device_communicators/pynccl.py create mode 100644 python/sglang/srt/distributed/device_communicators/pynccl_allocator.py create mode 100644 python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py create mode 100644 python/sglang/srt/distributed/device_communicators/quick_all_reduce.py create mode 100644 python/sglang/srt/distributed/device_communicators/shm_broadcast.py create mode 100644 python/sglang/srt/distributed/device_communicators/xpu_communicator.py create mode 100644 python/sglang/srt/distributed/naive_distributed.py create mode 100644 python/sglang/srt/distributed/parallel_state.py create mode 100644 python/sglang/srt/distributed/utils.py create mode 100644 python/sglang/srt/entrypoints/EngineBase.py create mode 100644 python/sglang/srt/entrypoints/context.py create mode 100644 python/sglang/srt/entrypoints/engine.py create mode 100644 python/sglang/srt/entrypoints/harmony_utils.py create mode 100644 python/sglang/srt/entrypoints/http_server.py create mode 100644 python/sglang/srt/entrypoints/http_server_engine.py create mode 100644 python/sglang/srt/entrypoints/openai/__init__.py create mode 100644 python/sglang/srt/entrypoints/openai/protocol.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_base.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_chat.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_completions.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_embedding.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_rerank.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_responses.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_score.py create mode 100644 python/sglang/srt/entrypoints/openai/tool_server.py create mode 100644 python/sglang/srt/entrypoints/openai/usage_processor.py create mode 100644 python/sglang/srt/entrypoints/openai/utils.py create mode 100644 python/sglang/srt/entrypoints/tool.py create mode 100644 python/sglang/srt/eplb/__init__.py create mode 100644 python/sglang/srt/eplb/eplb_algorithms/__init__.py create mode 100644 python/sglang/srt/eplb/eplb_algorithms/deepseek.py create mode 100644 python/sglang/srt/eplb/eplb_algorithms/deepseek_vec.py create mode 100644 python/sglang/srt/eplb/eplb_manager.py create mode 100644 python/sglang/srt/eplb/eplb_simulator/__init__.py create mode 100644 python/sglang/srt/eplb/eplb_simulator/reader.py create mode 100644 python/sglang/srt/eplb/expert_distribution.py create mode 100644 python/sglang/srt/eplb/expert_location.py create mode 100644 python/sglang/srt/eplb/expert_location_dispatch.py create mode 100644 python/sglang/srt/eplb/expert_location_updater.py create mode 100644 python/sglang/srt/function_call/base_format_detector.py create mode 100644 python/sglang/srt/function_call/core_types.py create mode 100644 python/sglang/srt/function_call/deepseekv31_detector.py create mode 100644 python/sglang/srt/function_call/deepseekv3_detector.py create mode 100644 python/sglang/srt/function_call/ebnf_composer.py create mode 100644 python/sglang/srt/function_call/function_call_parser.py create mode 100644 python/sglang/srt/function_call/glm4_moe_detector.py create mode 100644 python/sglang/srt/function_call/gpt_oss_detector.py create mode 100644 python/sglang/srt/function_call/kimik2_detector.py create mode 100644 python/sglang/srt/function_call/llama32_detector.py create mode 100644 python/sglang/srt/function_call/mistral_detector.py create mode 100644 python/sglang/srt/function_call/pythonic_detector.py create mode 100644 python/sglang/srt/function_call/qwen25_detector.py create mode 100644 python/sglang/srt/function_call/qwen3_coder_detector.py create mode 100644 python/sglang/srt/function_call/step3_detector.py create mode 100644 python/sglang/srt/function_call/utils.py create mode 100644 python/sglang/srt/hf_transformers_utils.py create mode 100644 python/sglang/srt/host_shared_memory.py create mode 100644 python/sglang/srt/layers/activation.py create mode 100644 python/sglang/srt/layers/amx_utils.py create mode 100644 python/sglang/srt/layers/attention/aiter_backend.py create mode 100644 python/sglang/srt/layers/attention/ascend_backend.py create mode 100644 python/sglang/srt/layers/attention/base_attn_backend.py create mode 100644 python/sglang/srt/layers/attention/cutlass_mla_backend.py create mode 100644 python/sglang/srt/layers/attention/double_sparsity_backend.py create mode 100644 python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py create mode 100644 python/sglang/srt/layers/attention/fla/chunk.py create mode 100644 python/sglang/srt/layers/attention/fla/chunk_delta_h.py create mode 100644 python/sglang/srt/layers/attention/fla/chunk_o.py create mode 100644 python/sglang/srt/layers/attention/fla/chunk_scaled_dot_kkt.py create mode 100644 python/sglang/srt/layers/attention/fla/cumsum.py create mode 100644 python/sglang/srt/layers/attention/fla/fused_recurrent.py create mode 100644 python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py create mode 100644 python/sglang/srt/layers/attention/fla/index.py create mode 100644 python/sglang/srt/layers/attention/fla/l2norm.py create mode 100644 python/sglang/srt/layers/attention/fla/layernorm_gated.py create mode 100644 python/sglang/srt/layers/attention/fla/op.py create mode 100644 python/sglang/srt/layers/attention/fla/solve_tril.py create mode 100644 python/sglang/srt/layers/attention/fla/utils.py create mode 100644 python/sglang/srt/layers/attention/fla/wy_fast.py create mode 100644 python/sglang/srt/layers/attention/flashattention_backend.py create mode 100644 python/sglang/srt/layers/attention/flashinfer_backend.py create mode 100644 python/sglang/srt/layers/attention/flashinfer_mla_backend.py create mode 100644 python/sglang/srt/layers/attention/flashmla_backend.py create mode 100644 python/sglang/srt/layers/attention/hybrid_attn_backend.py create mode 100644 python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py create mode 100644 python/sglang/srt/layers/attention/intel_amx_backend.py create mode 100644 python/sglang/srt/layers/attention/mamba/causal_conv1d.py create mode 100644 python/sglang/srt/layers/attention/mamba/causal_conv1d_triton.py create mode 100644 python/sglang/srt/layers/attention/mamba/mamba.py create mode 100644 python/sglang/srt/layers/attention/merge_state.py create mode 100644 python/sglang/srt/layers/attention/tbo_backend.py create mode 100644 python/sglang/srt/layers/attention/torch_native_backend.py create mode 100644 python/sglang/srt/layers/attention/triton_backend.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/decode_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/extend_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/merge_state.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/prefill_attention.py create mode 100644 python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py create mode 100644 python/sglang/srt/layers/attention/trtllm_mha_backend.py create mode 100755 python/sglang/srt/layers/attention/trtllm_mla_backend.py create mode 100644 python/sglang/srt/layers/attention/utils.py create mode 100644 python/sglang/srt/layers/attention/vision.py create mode 100644 python/sglang/srt/layers/attention/vision_utils.py create mode 100644 python/sglang/srt/layers/attention/wave_backend.py create mode 100644 python/sglang/srt/layers/attention/wave_ops/decode_attention.py create mode 100644 python/sglang/srt/layers/attention/wave_ops/extend_attention.py create mode 100644 python/sglang/srt/layers/attention/wave_ops/prefill_attention.py create mode 100644 python/sglang/srt/layers/communicator.py create mode 100644 python/sglang/srt/layers/dp_attention.py create mode 100644 python/sglang/srt/layers/elementwise.py create mode 100644 python/sglang/srt/layers/flashinfer_comm_fusion.py create mode 100644 python/sglang/srt/layers/layernorm.py create mode 100644 python/sglang/srt/layers/linear.py create mode 100644 python/sglang/srt/layers/logits_processor.py create mode 100644 python/sglang/srt/layers/model_parallel.py create mode 100644 python/sglang/srt/layers/moe/__init__.py create mode 100755 python/sglang/srt/layers/moe/cutlass_moe.py create mode 100644 python/sglang/srt/layers/moe/cutlass_moe_params.py create mode 100644 python/sglang/srt/layers/moe/cutlass_w4a8_moe.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/__init__.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/kernels.py create mode 100644 python/sglang/srt/layers/moe/ep_moe/layer.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_native.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/__init__.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/README.md create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=144,N=512,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=20,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=24,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_L40S.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI300X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI325X.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Radeon_Graphics.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H20.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=96,device_name=NVIDIA_H20.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=129,N=352,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=160,N=320,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=161,N=192,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=288,N=64,device_name=NVIDIA_A800-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_0/E=16,N=1024,device_name=NVIDIA_B200.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=352,device_name=NVIDIA_RTX_6000_Ada_Generation,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=320,device_name=NVIDIA_H20-3e.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_B200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=64,device_name=NVIDIA_A100-SXM4-80GB.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H20-3e.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H20-3e.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H100_80GB_HBM3.json create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_config.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/layer.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/moe_align_block_size.py create mode 100644 python/sglang/srt/layers/moe/fused_moe_triton/triton_kernels_moe.py create mode 100644 python/sglang/srt/layers/moe/moe_runner/__init__.py create mode 100644 python/sglang/srt/layers/moe/moe_runner/base.py create mode 100644 python/sglang/srt/layers/moe/moe_runner/runner.py create mode 100644 python/sglang/srt/layers/moe/moe_runner/triton.py create mode 100644 python/sglang/srt/layers/moe/rocm_moe_utils.py create mode 100644 python/sglang/srt/layers/moe/router.py create mode 100644 python/sglang/srt/layers/moe/token_dispatcher/__init__.py create mode 100644 python/sglang/srt/layers/moe/token_dispatcher/base.py create mode 100644 python/sglang/srt/layers/moe/token_dispatcher/deepep.py create mode 100644 python/sglang/srt/layers/moe/token_dispatcher/standard.py create mode 100644 python/sglang/srt/layers/moe/topk.py create mode 100644 python/sglang/srt/layers/moe/utils.py create mode 100644 python/sglang/srt/layers/multimodal.py create mode 100644 python/sglang/srt/layers/parameter.py create mode 100644 python/sglang/srt/layers/pooler.py create mode 100644 python/sglang/srt/layers/quantization/__init__.py create mode 100644 python/sglang/srt/layers/quantization/awq.py create mode 100644 python/sglang/srt/layers/quantization/awq_triton.py create mode 100644 python/sglang/srt/layers/quantization/base_config.py create mode 100644 python/sglang/srt/layers/quantization/blockwise_int8.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/README.md create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/__init__.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py create mode 100644 python/sglang/srt/layers/quantization/compressed_tensors/utils.py create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json create mode 100644 python/sglang/srt/layers/quantization/deep_gemm_wrapper/__init__.py create mode 100644 python/sglang/srt/layers/quantization/deep_gemm_wrapper/compile_utils.py create mode 100644 python/sglang/srt/layers/quantization/deep_gemm_wrapper/configurer.py create mode 100644 python/sglang/srt/layers/quantization/deep_gemm_wrapper/entrypoint.py create mode 100644 python/sglang/srt/layers/quantization/fp8.py create mode 100644 python/sglang/srt/layers/quantization/fp8_kernel.py create mode 100644 python/sglang/srt/layers/quantization/fp8_utils.py create mode 100644 python/sglang/srt/layers/quantization/fpgemm_fp8.py create mode 100644 python/sglang/srt/layers/quantization/gptq.py create mode 100644 python/sglang/srt/layers/quantization/int8_kernel.py create mode 100644 python/sglang/srt/layers/quantization/int8_utils.py create mode 100644 python/sglang/srt/layers/quantization/kv_cache.py create mode 100644 python/sglang/srt/layers/quantization/marlin_utils.py create mode 100644 python/sglang/srt/layers/quantization/marlin_utils_fp8.py create mode 100755 python/sglang/srt/layers/quantization/modelopt_quant.py create mode 100644 python/sglang/srt/layers/quantization/moe_wna16.py create mode 100644 python/sglang/srt/layers/quantization/mxfp4.py create mode 100644 python/sglang/srt/layers/quantization/mxfp4_tensor.py create mode 100644 python/sglang/srt/layers/quantization/petit.py create mode 100644 python/sglang/srt/layers/quantization/petit_utils.py create mode 100644 python/sglang/srt/layers/quantization/qoq.py create mode 100644 python/sglang/srt/layers/quantization/quark/__init__.py create mode 100644 python/sglang/srt/layers/quantization/quark/quark.py create mode 100644 python/sglang/srt/layers/quantization/quark/quark_moe.py create mode 100644 python/sglang/srt/layers/quantization/quark/schemes/__init__.py create mode 100644 python/sglang/srt/layers/quantization/quark/schemes/quark_scheme.py create mode 100644 python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py create mode 100644 python/sglang/srt/layers/quantization/quark/utils.py create mode 100644 python/sglang/srt/layers/quantization/rocm_mxfp4_utils.py create mode 100644 python/sglang/srt/layers/quantization/unquant.py create mode 100644 python/sglang/srt/layers/quantization/utils.py create mode 100644 python/sglang/srt/layers/quantization/w4afp8.py create mode 100644 python/sglang/srt/layers/quantization/w8a8_fp8.py create mode 100644 python/sglang/srt/layers/quantization/w8a8_int8.py create mode 100644 python/sglang/srt/layers/radix_attention.py create mode 100644 python/sglang/srt/layers/rocm_linear_utils.py create mode 100644 python/sglang/srt/layers/rotary_embedding.py create mode 100644 python/sglang/srt/layers/sampler.py create mode 100644 python/sglang/srt/layers/torchao_utils.py create mode 100644 python/sglang/srt/layers/utils.py create mode 100644 python/sglang/srt/layers/vocab_parallel_embedding.py create mode 100644 python/sglang/srt/lora/backend/base_backend.py create mode 100644 python/sglang/srt/lora/backend/triton_backend.py create mode 100644 python/sglang/srt/lora/layers.py create mode 100644 python/sglang/srt/lora/lora.py create mode 100644 python/sglang/srt/lora/lora_config.py create mode 100644 python/sglang/srt/lora/lora_manager.py create mode 100644 python/sglang/srt/lora/lora_registry.py create mode 100644 python/sglang/srt/lora/mem_pool.py create mode 100644 python/sglang/srt/lora/triton_ops/__init__.py create mode 100644 python/sglang/srt/lora/triton_ops/gate_up_lora_b.py create mode 100644 python/sglang/srt/lora/triton_ops/qkv_lora_b.py create mode 100644 python/sglang/srt/lora/triton_ops/sgemm_lora_a.py create mode 100644 python/sglang/srt/lora/triton_ops/sgemm_lora_b.py create mode 100644 python/sglang/srt/lora/utils.py create mode 100644 python/sglang/srt/managers/cache_controller.py create mode 100644 python/sglang/srt/managers/configure_logging.py create mode 100644 python/sglang/srt/managers/data_parallel_controller.py create mode 100644 python/sglang/srt/managers/detokenizer_manager.py create mode 100644 python/sglang/srt/managers/disagg_service.py create mode 100644 python/sglang/srt/managers/io_struct.py create mode 100644 python/sglang/srt/managers/mm_utils.py create mode 100644 python/sglang/srt/managers/multi_tokenizer_mixin.py create mode 100644 python/sglang/srt/managers/multimodal_processor.py create mode 100644 python/sglang/srt/managers/schedule_batch.py create mode 100644 python/sglang/srt/managers/schedule_policy.py create mode 100644 python/sglang/srt/managers/scheduler.py create mode 100644 python/sglang/srt/managers/scheduler_input_blocker.py create mode 100644 python/sglang/srt/managers/scheduler_metrics_mixin.py create mode 100644 python/sglang/srt/managers/scheduler_output_processor_mixin.py create mode 100644 python/sglang/srt/managers/scheduler_profiler_mixin.py create mode 100644 python/sglang/srt/managers/scheduler_recv_skipper.py create mode 100644 python/sglang/srt/managers/scheduler_update_weights_mixin.py create mode 100644 python/sglang/srt/managers/session_controller.py create mode 100644 python/sglang/srt/managers/template_manager.py create mode 100644 python/sglang/srt/managers/tokenizer_communicator_mixin.py create mode 100644 python/sglang/srt/managers/tokenizer_manager.py create mode 100644 python/sglang/srt/managers/tp_worker.py create mode 100644 python/sglang/srt/managers/tp_worker_overlap_thread.py create mode 100644 python/sglang/srt/managers/utils.py create mode 100644 python/sglang/srt/mem_cache/allocator.py create mode 100644 python/sglang/srt/mem_cache/allocator_ascend.py create mode 100644 python/sglang/srt/mem_cache/base_prefix_cache.py create mode 100644 python/sglang/srt/mem_cache/chunk_cache.py create mode 120000 python/sglang/srt/mem_cache/cpp_radix_tree/.clang-format create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/common.h create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/radix_tree.py create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.cpp create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2.h create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_binding.cpp create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_debug.cpp create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_impl.h create mode 100644 python/sglang/srt/mem_cache/cpp_radix_tree/tree_v2_node.h create mode 100644 python/sglang/srt/mem_cache/flush_cache.py create mode 100644 python/sglang/srt/mem_cache/hicache_storage.py create mode 100644 python/sglang/srt/mem_cache/hiradix_cache.py create mode 100644 python/sglang/srt/mem_cache/lora_radix_cache.py create mode 100644 python/sglang/srt/mem_cache/memory_pool.py create mode 100644 python/sglang/srt/mem_cache/memory_pool_host.py create mode 100644 python/sglang/srt/mem_cache/multimodal_cache.py create mode 100644 python/sglang/srt/mem_cache/radix_cache.py create mode 100644 python/sglang/srt/mem_cache/radix_cache_cpp.py create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/docs/README.md create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/docs/deploy_sglang_3fs_multinode.md create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/docs/setup_usrbio_client.md create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_client.py create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_usrbio_client.py create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/mini_3fs_metadata_server.py create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py create mode 100644 python/sglang/srt/mem_cache/storage/hf3fs/test_hf3fs_utils.py create mode 100644 python/sglang/srt/mem_cache/storage/lmcache/README.md create mode 100644 python/sglang/srt/mem_cache/storage/lmcache/example_config.yaml create mode 100644 python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py create mode 100644 python/sglang/srt/mem_cache/storage/lmcache/unit_test.py create mode 100644 python/sglang/srt/mem_cache/storage/mooncake_store/README.md create mode 100644 python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py create mode 100644 python/sglang/srt/mem_cache/storage/mooncake_store/test_mooncake_store.py create mode 100644 python/sglang/srt/mem_cache/storage/nixl/README.md create mode 100644 python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py create mode 100644 python/sglang/srt/mem_cache/storage/nixl/nixl_utils.py create mode 100755 python/sglang/srt/mem_cache/storage/nixl/test_hicache_nixl_storage.py create mode 100644 python/sglang/srt/mem_cache/swa_radix_cache.py create mode 100644 python/sglang/srt/metrics/collector.py create mode 100644 python/sglang/srt/metrics/func_timer.py create mode 100644 python/sglang/srt/metrics/startup_func_log_and_timer.py create mode 100644 python/sglang/srt/metrics/utils.py create mode 100644 python/sglang/srt/model_executor/cpu_graph_runner.py create mode 100644 python/sglang/srt/model_executor/cuda_graph_runner.py create mode 100644 python/sglang/srt/model_executor/forward_batch_info.py create mode 100644 python/sglang/srt/model_executor/model_runner.py create mode 100644 python/sglang/srt/model_executor/npu_graph_runner.py create mode 100644 python/sglang/srt/model_loader/__init__.py create mode 100644 python/sglang/srt/model_loader/loader.py create mode 100644 python/sglang/srt/model_loader/utils.py create mode 100644 python/sglang/srt/model_loader/weight_utils.py create mode 100644 python/sglang/srt/models/arcee.py create mode 100644 python/sglang/srt/models/baichuan.py create mode 100644 python/sglang/srt/models/bailing_moe.py create mode 100644 python/sglang/srt/models/bert.py create mode 100644 python/sglang/srt/models/chatglm.py create mode 100644 python/sglang/srt/models/clip.py create mode 100644 python/sglang/srt/models/commandr.py create mode 100644 python/sglang/srt/models/dbrx.py create mode 100644 python/sglang/srt/models/deepseek.py create mode 100644 python/sglang/srt/models/deepseek_janus_pro.py create mode 100644 python/sglang/srt/models/deepseek_nextn.py create mode 100644 python/sglang/srt/models/deepseek_v2.py create mode 100644 python/sglang/srt/models/deepseek_vl2.py create mode 100644 python/sglang/srt/models/ernie4.py create mode 100644 python/sglang/srt/models/ernie4_eagle.py create mode 100644 python/sglang/srt/models/exaone.py create mode 100644 python/sglang/srt/models/gemma.py create mode 100644 python/sglang/srt/models/gemma2.py create mode 100644 python/sglang/srt/models/gemma2_reward.py create mode 100644 python/sglang/srt/models/gemma3_causal.py create mode 100644 python/sglang/srt/models/gemma3_mm.py create mode 100644 python/sglang/srt/models/gemma3n_audio.py create mode 100644 python/sglang/srt/models/gemma3n_causal.py create mode 100644 python/sglang/srt/models/gemma3n_mm.py create mode 100644 python/sglang/srt/models/glm4.py create mode 100644 python/sglang/srt/models/glm4_moe.py create mode 100644 python/sglang/srt/models/glm4_moe_nextn.py create mode 100644 python/sglang/srt/models/glm4v.py create mode 100644 python/sglang/srt/models/glm4v_moe.py create mode 100644 python/sglang/srt/models/gpt2.py create mode 100644 python/sglang/srt/models/gpt_bigcode.py create mode 100644 python/sglang/srt/models/gpt_oss.py create mode 100644 python/sglang/srt/models/granite.py create mode 100644 python/sglang/srt/models/granitemoe.py create mode 100644 python/sglang/srt/models/grok.py create mode 100644 python/sglang/srt/models/hunyuan.py create mode 100644 python/sglang/srt/models/idefics2.py create mode 100644 python/sglang/srt/models/internlm2.py create mode 100644 python/sglang/srt/models/internlm2_reward.py create mode 100644 python/sglang/srt/models/interns1.py create mode 100644 python/sglang/srt/models/internvl.py create mode 100644 python/sglang/srt/models/kimi_vl.py create mode 100644 python/sglang/srt/models/kimi_vl_moonvit.py create mode 100644 python/sglang/srt/models/llama.py create mode 100644 python/sglang/srt/models/llama4.py create mode 100644 python/sglang/srt/models/llama_classification.py create mode 100644 python/sglang/srt/models/llama_eagle.py create mode 100644 python/sglang/srt/models/llama_eagle3.py create mode 100644 python/sglang/srt/models/llama_embedding.py create mode 100644 python/sglang/srt/models/llama_reward.py create mode 100644 python/sglang/srt/models/llava.py create mode 100644 python/sglang/srt/models/llavavid.py create mode 100644 python/sglang/srt/models/longcat_flash.py create mode 100644 python/sglang/srt/models/longcat_flash_nextn.py create mode 100644 python/sglang/srt/models/mimo.py create mode 100644 python/sglang/srt/models/mimo_mtp.py create mode 100644 python/sglang/srt/models/minicpm.py create mode 100644 python/sglang/srt/models/minicpm3.py create mode 100644 python/sglang/srt/models/minicpmo.py create mode 100644 python/sglang/srt/models/minicpmv.py create mode 100644 python/sglang/srt/models/mistral.py create mode 100644 python/sglang/srt/models/mixtral.py create mode 100644 python/sglang/srt/models/mixtral_quant.py create mode 100644 python/sglang/srt/models/mllama.py create mode 100644 python/sglang/srt/models/mllama4.py create mode 100644 python/sglang/srt/models/nemotron_nas.py create mode 100644 python/sglang/srt/models/olmo.py create mode 100644 python/sglang/srt/models/olmo2.py create mode 100644 python/sglang/srt/models/olmoe.py create mode 100644 python/sglang/srt/models/opt.py create mode 100644 python/sglang/srt/models/persimmon.py create mode 100644 python/sglang/srt/models/phi.py create mode 100644 python/sglang/srt/models/phi3_small.py create mode 100644 python/sglang/srt/models/phi4mm.py create mode 100644 python/sglang/srt/models/phi4mm_audio.py create mode 100644 python/sglang/srt/models/phi4mm_utils.py create mode 100644 python/sglang/srt/models/phimoe.py create mode 100644 python/sglang/srt/models/pixtral.py create mode 100644 python/sglang/srt/models/qwen.py create mode 100644 python/sglang/srt/models/qwen2.py create mode 100644 python/sglang/srt/models/qwen2_5_vl.py create mode 100644 python/sglang/srt/models/qwen2_audio.py create mode 100644 python/sglang/srt/models/qwen2_classification.py create mode 100644 python/sglang/srt/models/qwen2_eagle.py create mode 100644 python/sglang/srt/models/qwen2_moe.py create mode 100644 python/sglang/srt/models/qwen2_rm.py create mode 100644 python/sglang/srt/models/qwen2_vl.py create mode 100644 python/sglang/srt/models/qwen3.py create mode 100644 python/sglang/srt/models/qwen3_classification.py create mode 100644 python/sglang/srt/models/qwen3_moe.py create mode 100644 python/sglang/srt/models/qwen3_next.py create mode 100644 python/sglang/srt/models/qwen3_next_mtp.py create mode 100644 python/sglang/srt/models/registry.py create mode 100644 python/sglang/srt/models/roberta.py create mode 100644 python/sglang/srt/models/siglip.py create mode 100644 python/sglang/srt/models/stablelm.py create mode 100644 python/sglang/srt/models/step3_vl.py create mode 100644 python/sglang/srt/models/torch_native_llama.py create mode 100644 python/sglang/srt/models/transformers.py create mode 100644 python/sglang/srt/models/vila.py create mode 100644 python/sglang/srt/models/xverse.py create mode 100644 python/sglang/srt/models/xverse_moe.py create mode 100644 python/sglang/srt/models/yivl.py create mode 100644 python/sglang/srt/multimodal/mm_utils.py create mode 100644 python/sglang/srt/multimodal/processors/base_processor.py create mode 100644 python/sglang/srt/multimodal/processors/clip.py create mode 100644 python/sglang/srt/multimodal/processors/deepseek_vl_v2.py create mode 100644 python/sglang/srt/multimodal/processors/gemma3.py create mode 100644 python/sglang/srt/multimodal/processors/gemma3n.py create mode 100644 python/sglang/srt/multimodal/processors/glm4v.py create mode 100644 python/sglang/srt/multimodal/processors/internvl.py create mode 100644 python/sglang/srt/multimodal/processors/janus_pro.py create mode 100644 python/sglang/srt/multimodal/processors/kimi_vl.py create mode 100644 python/sglang/srt/multimodal/processors/llava.py create mode 100644 python/sglang/srt/multimodal/processors/minicpm.py create mode 100644 python/sglang/srt/multimodal/processors/mlama.py create mode 100644 python/sglang/srt/multimodal/processors/mllama4.py create mode 100644 python/sglang/srt/multimodal/processors/phi4mm.py create mode 100644 python/sglang/srt/multimodal/processors/pixtral.py create mode 100644 python/sglang/srt/multimodal/processors/qwen_audio.py create mode 100644 python/sglang/srt/multimodal/processors/qwen_vl.py create mode 100644 python/sglang/srt/multimodal/processors/step3_vl.py create mode 100644 python/sglang/srt/multimodal/processors/vila.py create mode 100644 python/sglang/srt/offloader.py create mode 100644 python/sglang/srt/operations.py create mode 100644 python/sglang/srt/operations_strategy.py create mode 100644 python/sglang/srt/parser/code_completion_parser.py create mode 100644 python/sglang/srt/parser/conversation.py create mode 100644 python/sglang/srt/parser/harmony_parser.py create mode 100644 python/sglang/srt/parser/jinja_template_utils.py create mode 100644 python/sglang/srt/parser/reasoning_parser.py create mode 100644 python/sglang/srt/patch_torch.py create mode 100644 python/sglang/srt/poll_based_barrier.py create mode 100644 python/sglang/srt/sampling/custom_logit_processor.py create mode 100644 python/sglang/srt/sampling/penaltylib/__init__.py create mode 100644 python/sglang/srt/sampling/penaltylib/frequency_penalty.py create mode 100644 python/sglang/srt/sampling/penaltylib/min_new_tokens.py create mode 100644 python/sglang/srt/sampling/penaltylib/orchestrator.py create mode 100644 python/sglang/srt/sampling/penaltylib/presence_penalty.py create mode 100644 python/sglang/srt/sampling/sampling_batch_info.py create mode 100644 python/sglang/srt/sampling/sampling_params.py create mode 100644 python/sglang/srt/server_args.py create mode 100644 python/sglang/srt/speculative/build_eagle_tree.py create mode 100644 python/sglang/srt/speculative/eagle_draft_cuda_graph_runner.py create mode 100644 python/sglang/srt/speculative/eagle_draft_extend_cuda_graph_runner.py create mode 100644 python/sglang/srt/speculative/eagle_utils.py create mode 100644 python/sglang/srt/speculative/eagle_worker.py create mode 100644 python/sglang/srt/speculative/spec_info.py create mode 100644 python/sglang/srt/speculative/standalone_worker.py create mode 100644 python/sglang/srt/tokenizer/tiktoken_tokenizer.py create mode 100644 python/sglang/srt/torch_memory_saver_adapter.py create mode 100644 python/sglang/srt/two_batch_overlap.py create mode 100644 python/sglang/srt/utils.py create mode 100644 python/sglang/srt/warmup.py create mode 100644 python/sglang/srt/weight_sync/tensor_bucket.py create mode 100644 python/sglang/srt/weight_sync/utils.py create mode 100644 python/sglang/test/__init__.py create mode 100644 python/sglang/test/attention/__init__.py create mode 100644 python/sglang/test/attention/test_flashattn_backend.py create mode 100644 python/sglang/test/attention/test_flashattn_mla_backend.py create mode 100644 python/sglang/test/attention/test_prefix_chunk_info.py create mode 100755 python/sglang/test/attention/test_trtllm_mla_backend.py create mode 100644 python/sglang/test/doc_patch.py create mode 100644 python/sglang/test/few_shot_gsm8k.py create mode 100644 python/sglang/test/few_shot_gsm8k_engine.py create mode 100644 python/sglang/test/long_prompt.txt create mode 100644 python/sglang/test/run_eval.py create mode 100644 python/sglang/test/runners.py create mode 100644 python/sglang/test/send_one.py create mode 100644 python/sglang/test/simple_eval_common.py create mode 100644 python/sglang/test/simple_eval_gpqa.py create mode 100644 python/sglang/test/simple_eval_humaneval.py create mode 100644 python/sglang/test/simple_eval_math.py create mode 100644 python/sglang/test/simple_eval_mgsm.py create mode 100644 python/sglang/test/simple_eval_mmlu.py create mode 100644 python/sglang/test/test_activation.py create mode 100644 python/sglang/test/test_block_fp8.py create mode 100644 python/sglang/test/test_block_fp8_deep_gemm_blackwell.py create mode 100644 python/sglang/test/test_block_fp8_ep.py create mode 100644 python/sglang/test/test_custom_ops.py create mode 100755 python/sglang/test/test_cutlass_moe.py create mode 100644 python/sglang/test/test_cutlass_w4a8_moe.py create mode 100644 python/sglang/test/test_deepep_utils.py create mode 100644 python/sglang/test/test_disaggregation_utils.py create mode 100644 python/sglang/test/test_dynamic_grad_mode.py create mode 100644 python/sglang/test/test_fp4_moe.py create mode 100644 python/sglang/test/test_layernorm.py create mode 100644 python/sglang/test/test_marlin_moe.py create mode 100644 python/sglang/test/test_marlin_utils.py create mode 100644 python/sglang/test/test_programs.py create mode 100644 python/sglang/test/test_utils.py create mode 100644 python/sglang/utils.py create mode 100644 python/sglang/version.py create mode 100755 scripts/ci/amd_ci_exec.sh create mode 100755 scripts/ci/amd_ci_install_dependency.sh create mode 100755 scripts/ci/amd_ci_start_container.sh create mode 100755 scripts/ci/ci_install_deepep.sh create mode 100755 scripts/ci/ci_install_dependency.sh create mode 100755 scripts/ci/ci_install_rust.sh create mode 100755 scripts/ci/ci_start_disaggregation_servers.sh create mode 100755 scripts/ci/npu_ci_install_dependency.sh create mode 100644 scripts/code_sync/copy_from_oss.py create mode 100644 scripts/code_sync/copy_to_oss.py create mode 100644 scripts/code_sync/guideline.md create mode 100755 scripts/code_sync/install_github_cli.sh create mode 100644 scripts/deprecated/convert_yi_vl.py create mode 100644 scripts/deprecated/convert_yi_vl.sh create mode 100644 scripts/deprecated/test_curl.sh create mode 100644 scripts/deprecated/test_flashinfer.py create mode 100644 scripts/deprecated/test_httpserver_concurrent.py create mode 100644 scripts/deprecated/test_httpserver_decode.py create mode 100644 scripts/deprecated/test_httpserver_decode_stream.py create mode 100644 scripts/deprecated/test_httpserver_llava.py create mode 100644 scripts/deprecated/test_httpserver_reuse.py create mode 100644 scripts/deprecated/test_jump_forward.py create mode 100644 scripts/deprecated/test_robust.py create mode 100644 scripts/export_deepseek_nextn.py create mode 100755 scripts/killall_sglang.sh create mode 100644 scripts/playground/bench_speculative.py create mode 100644 scripts/playground/disaggregation/cli-logprob.py create mode 100644 scripts/playground/disaggregation/cli-so.py create mode 100644 scripts/playground/disaggregation/cli.py create mode 100644 scripts/playground/frontend_reasoning.ipynb create mode 100644 scripts/playground/launch_tgi.sh create mode 100644 scripts/playground/load_tokenizer.py create mode 100644 scripts/playground/long_context_example.py create mode 100644 scripts/playground/lora/analyzer.py create mode 100644 scripts/playground/lora/lora_hf_play.py create mode 100644 scripts/playground/lora/lora_vllm_play.py create mode 100644 scripts/playground/reference_hf.py create mode 100644 scripts/playground/replay_request_dump.py create mode 100644 scripts/playground/router/test_tree.py create mode 100644 scripts/playground/router/tree.py create mode 100644 scripts/update_kernel_whl_index.py create mode 100755 scripts/version_branch_to_tag.sh create mode 100644 sgl-kernel/.clang-format create mode 100644 sgl-kernel/CMakeLists.txt create mode 100644 sgl-kernel/LICENSE create mode 100644 sgl-kernel/Makefile create mode 100644 sgl-kernel/README.md create mode 100644 sgl-kernel/THIRDPARTYNOTICES.txt create mode 100644 sgl-kernel/benchmark/bench_activation.py create mode 100644 sgl-kernel/benchmark/bench_awq_dequant.py create mode 100644 sgl-kernel/benchmark/bench_cutlass_mla.py create mode 100644 sgl-kernel/benchmark/bench_dsv3_fused_a_gemm.py create mode 100644 sgl-kernel/benchmark/bench_dsv3_router_gemm.py create mode 100755 sgl-kernel/benchmark/bench_fp4_gemm.py create mode 100644 sgl-kernel/benchmark/bench_fp8_blockwise_gemm.py create mode 100644 sgl-kernel/benchmark/bench_fp8_blockwise_group_gemm.py create mode 100644 sgl-kernel/benchmark/bench_fp8_gemm.py create mode 100644 sgl-kernel/benchmark/bench_int8_gemm.py create mode 100644 sgl-kernel/benchmark/bench_lightning_attention_decode.py create mode 100644 sgl-kernel/benchmark/bench_moe_align_block_size.py create mode 100644 sgl-kernel/benchmark/bench_moe_ep_post_reorder.py create mode 100644 sgl-kernel/benchmark/bench_moe_fused_gate.py create mode 100644 sgl-kernel/benchmark/bench_moe_topk_softmax.py create mode 100644 sgl-kernel/benchmark/bench_nvfp4_scaled_gemm.py create mode 100644 sgl-kernel/benchmark/bench_per_tensor_quant_fp8.py create mode 100644 sgl-kernel/benchmark/bench_per_token_group_quant_8bit.py create mode 100644 sgl-kernel/benchmark/bench_per_token_quant_fp8.py create mode 100644 sgl-kernel/benchmark/bench_qserve_w4a8_gemm.py create mode 100644 sgl-kernel/benchmark/bench_rotary_embedding.py create mode 100644 sgl-kernel/benchmark/bench_top_k_top_p_sampling.py create mode 100755 sgl-kernel/build.sh create mode 100644 sgl-kernel/cmake/utils.cmake create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce.cu create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce.cuh create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce.hip create mode 100644 sgl-kernel/csrc/allreduce/custom_all_reduce_hip.cuh create mode 100644 sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu create mode 100644 sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce.cu create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce.cuh create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce.h create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce.hip create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce_base.h create mode 100644 sgl-kernel/csrc/allreduce/quick_all_reduce_hip.h create mode 100644 sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu create mode 100644 sgl-kernel/csrc/attention/cascade.cu create mode 100644 sgl-kernel/csrc/attention/cutlass_mla_kernel.cu create mode 100644 sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp create mode 100644 sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp create mode 100644 sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp create mode 100644 sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp create mode 100644 sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu create mode 100644 sgl-kernel/csrc/attention/merge_attn_states.cu create mode 100644 sgl-kernel/csrc/attention/vertical_slash_index.cu create mode 100644 sgl-kernel/csrc/common_extension.cc create mode 100644 sgl-kernel/csrc/common_extension_rocm.cc create mode 100755 sgl-kernel/csrc/cpu/CMakeLists.txt create mode 100644 sgl-kernel/csrc/cpu/activation.cpp create mode 100644 sgl-kernel/csrc/cpu/bmm.cpp create mode 100644 sgl-kernel/csrc/cpu/common.h create mode 100644 sgl-kernel/csrc/cpu/decode.cpp create mode 100644 sgl-kernel/csrc/cpu/extend.cpp create mode 100644 sgl-kernel/csrc/cpu/gemm.cpp create mode 100644 sgl-kernel/csrc/cpu/gemm.h create mode 100644 sgl-kernel/csrc/cpu/gemm_fp8.cpp create mode 100644 sgl-kernel/csrc/cpu/gemm_int8.cpp create mode 100644 sgl-kernel/csrc/cpu/interface.cpp create mode 100644 sgl-kernel/csrc/cpu/moe.cpp create mode 100644 sgl-kernel/csrc/cpu/moe_fp8.cpp create mode 100644 sgl-kernel/csrc/cpu/moe_int8.cpp create mode 100644 sgl-kernel/csrc/cpu/norm.cpp create mode 100644 sgl-kernel/csrc/cpu/numa_utils.cpp create mode 100644 sgl-kernel/csrc/cpu/qkv_proj.cpp create mode 100644 sgl-kernel/csrc/cpu/rope.cpp create mode 100644 sgl-kernel/csrc/cpu/shm.cpp create mode 100644 sgl-kernel/csrc/cpu/shm.h create mode 100644 sgl-kernel/csrc/cpu/topk.cpp create mode 100644 sgl-kernel/csrc/cpu/torch_extension_cpu.cpp create mode 100644 sgl-kernel/csrc/cpu/vec.h create mode 100644 sgl-kernel/csrc/cutlass_extensions/common.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h create mode 100644 sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h create mode 100644 sgl-kernel/csrc/elementwise/activation.cu create mode 100644 sgl-kernel/csrc/elementwise/activation.hip create mode 100644 sgl-kernel/csrc/elementwise/cast.cu create mode 100644 sgl-kernel/csrc/elementwise/concat_mla.cu create mode 100644 sgl-kernel/csrc/elementwise/copy.cu create mode 100644 sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu create mode 100644 sgl-kernel/csrc/elementwise/pos_enc.cuh create mode 100644 sgl-kernel/csrc/elementwise/rope.cu create mode 100644 sgl-kernel/csrc/flash_extension.cc create mode 100644 sgl-kernel/csrc/gemm/awq_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/bmm_fp8.cu create mode 100644 sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu create mode 100644 sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu create mode 100644 sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu create mode 100644 sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu create mode 100644 sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/gptq/compat.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/gptq/matrix_view.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/qdq_2.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/qdq_3.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/qdq_4.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/qdq_8.cuh create mode 100644 sgl-kernel/csrc/gemm/gptq/qdq_util.cuh create mode 100644 sgl-kernel/csrc/gemm/int8_gemm_kernel.cu create mode 100644 sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu create mode 100644 sgl-kernel/csrc/gemm/marlin/dequant.h create mode 100644 sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu create mode 100644 sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu create mode 100644 sgl-kernel/csrc/gemm/marlin/kernel.h create mode 100644 sgl-kernel/csrc/gemm/marlin/marlin.cuh create mode 100644 sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh create mode 100644 sgl-kernel/csrc/gemm/marlin/marlin_template.h create mode 100644 sgl-kernel/csrc/gemm/math.hpp create mode 100644 sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_quant.cuh create mode 100644 sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu create mode 100644 sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu create mode 100644 sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu create mode 100644 sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu create mode 100644 sgl-kernel/csrc/gemm/per_token_quant_fp8.cu create mode 100644 sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu create mode 100644 sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu create mode 100644 sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu create mode 100644 sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.hip create mode 100644 sgl-kernel/csrc/kvcacheio/transfer.cu create mode 100644 sgl-kernel/csrc/kvcacheio/transfer.hip create mode 100644 sgl-kernel/csrc/mamba/causal_conv1d.cu create mode 100644 sgl-kernel/csrc/mamba/causal_conv1d.h create mode 100644 sgl-kernel/csrc/memory/store.cu create mode 100644 sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu create mode 100644 sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh create mode 100644 sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu create mode 100644 sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh create mode 100644 sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu create mode 100644 sgl-kernel/csrc/moe/cutlass_moe_helper.cu create mode 100644 sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h create mode 100644 sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu create mode 100644 sgl-kernel/csrc/moe/moe_align_kernel.cu create mode 100644 sgl-kernel/csrc/moe/moe_align_kernel.hip create mode 100644 sgl-kernel/csrc/moe/moe_fused_gate.cu create mode 100644 sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu create mode 100644 sgl-kernel/csrc/moe/moe_topk_softmax_kernels.hip create mode 100644 sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu create mode 100644 sgl-kernel/csrc/moe/prepare_moe_input.cu create mode 100644 sgl-kernel/csrc/spatial/cuda_utils.h create mode 100644 sgl-kernel/csrc/spatial/greenctx_stream.cu create mode 100644 sgl-kernel/csrc/spatial/greenctx_stream.h create mode 100644 sgl-kernel/csrc/spatial_extension.cc create mode 100644 sgl-kernel/csrc/speculative/eagle_utils.cu create mode 100644 sgl-kernel/csrc/speculative/eagle_utils.hip create mode 100644 sgl-kernel/csrc/speculative/packbit.cu create mode 100644 sgl-kernel/csrc/speculative/speculative_sampling.cu create mode 100644 sgl-kernel/csrc/speculative/speculative_sampling.cuh create mode 100644 sgl-kernel/include/hip/hip_act_and_mul.cuh create mode 100644 sgl-kernel/include/hip/hip_act_and_mul_hip.cuh create mode 100644 sgl-kernel/include/hip/hip_math_def.h create mode 100644 sgl-kernel/include/hip/hip_vec_dtypes.h create mode 100644 sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h create mode 100644 sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h create mode 100644 sgl-kernel/include/hip/impl/hip_vec_half_impl.h create mode 100644 sgl-kernel/include/pytorch_extension_utils_rocm.h create mode 100644 sgl-kernel/include/scalar_type.hpp create mode 100644 sgl-kernel/include/sgl_flash_kernel_ops.h create mode 100644 sgl-kernel/include/sgl_kernel_ops.h create mode 100644 sgl-kernel/include/sgl_kernel_torch_shim.h create mode 100644 sgl-kernel/include/utils.h create mode 100644 sgl-kernel/include/utils_hip.h create mode 100644 sgl-kernel/pyproject.toml create mode 100644 sgl-kernel/pyproject_cpu.toml create mode 100644 sgl-kernel/pyproject_rocm.toml create mode 100755 sgl-kernel/python/sgl_kernel/__init__.py create mode 100644 sgl-kernel/python/sgl_kernel/_fa4_interface.py create mode 100644 sgl-kernel/python/sgl_kernel/allreduce.py create mode 100644 sgl-kernel/python/sgl_kernel/attention.py create mode 100644 sgl-kernel/python/sgl_kernel/cutlass_moe.py create mode 100644 sgl-kernel/python/sgl_kernel/elementwise.py create mode 100644 sgl-kernel/python/sgl_kernel/flash_attn.py create mode 100644 sgl-kernel/python/sgl_kernel/fused_moe.py create mode 100644 sgl-kernel/python/sgl_kernel/gemm.py create mode 100644 sgl-kernel/python/sgl_kernel/grammar.py create mode 100644 sgl-kernel/python/sgl_kernel/kvcacheio.py create mode 100644 sgl-kernel/python/sgl_kernel/mamba.py create mode 100644 sgl-kernel/python/sgl_kernel/marlin.py create mode 100644 sgl-kernel/python/sgl_kernel/memory.py create mode 100755 sgl-kernel/python/sgl_kernel/moe.py create mode 100644 sgl-kernel/python/sgl_kernel/sampling.py create mode 100644 sgl-kernel/python/sgl_kernel/scalar_type.py create mode 100644 sgl-kernel/python/sgl_kernel/sparse_flash_attn.py create mode 100644 sgl-kernel/python/sgl_kernel/spatial.py create mode 100644 sgl-kernel/python/sgl_kernel/speculative.py create mode 100644 sgl-kernel/python/sgl_kernel/testing/__init__.py create mode 100644 sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py create mode 100644 sgl-kernel/python/sgl_kernel/top_k.py create mode 100644 sgl-kernel/python/sgl_kernel/utils.py create mode 100644 sgl-kernel/python/sgl_kernel/version.py create mode 100755 sgl-kernel/rename_wheels.sh create mode 100644 sgl-kernel/setup_hip.py create mode 100644 sgl-kernel/setup_rocm.py create mode 100644 sgl-kernel/tests/spatial/test_greenctx_stream.py create mode 100644 sgl-kernel/tests/speculative/test_eagle_utils.py create mode 100644 sgl-kernel/tests/speculative/test_speculative_sampling.py create mode 100644 sgl-kernel/tests/test_activation.py create mode 100644 sgl-kernel/tests/test_apply_token_bitmask_inplace.py create mode 100644 sgl-kernel/tests/test_awq_dequant.py create mode 100644 sgl-kernel/tests/test_bmm_fp8.py create mode 100644 sgl-kernel/tests/test_causal_conv1d.py create mode 100644 sgl-kernel/tests/test_custom_allreduce.py create mode 100644 sgl-kernel/tests/test_cutlass_mla.py create mode 100644 sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py create mode 100644 sgl-kernel/tests/test_dsv3_fused_a_gemm.py create mode 100644 sgl-kernel/tests/test_dsv3_router_gemm.py create mode 100644 sgl-kernel/tests/test_flash_attention.py create mode 100644 sgl-kernel/tests/test_flash_attention_4.py create mode 100644 sgl-kernel/tests/test_fp4_gemm.py create mode 100644 sgl-kernel/tests/test_fp4_quantize.py create mode 100644 sgl-kernel/tests/test_fp8_blockwise_gemm.py create mode 100755 sgl-kernel/tests/test_fp8_blockwise_moe.py create mode 100644 sgl-kernel/tests/test_fp8_gemm.py create mode 100644 sgl-kernel/tests/test_gptq_kernel.py create mode 100644 sgl-kernel/tests/test_int8_gemm.py create mode 100644 sgl-kernel/tests/test_kvcacheio.py create mode 100644 sgl-kernel/tests/test_lightning_attention_decode.py create mode 100644 sgl-kernel/tests/test_marlin_gemm.py create mode 100644 sgl-kernel/tests/test_marlin_repack.py create mode 100644 sgl-kernel/tests/test_merge_state.py create mode 100644 sgl-kernel/tests/test_merge_state_v2.py create mode 100644 sgl-kernel/tests/test_moe_align.py create mode 100644 sgl-kernel/tests/test_moe_fused_gate.py create mode 100644 sgl-kernel/tests/test_moe_topk_softmax.py create mode 100644 sgl-kernel/tests/test_mscclpp.py create mode 100644 sgl-kernel/tests/test_norm.py create mode 100644 sgl-kernel/tests/test_per_tensor_quant_fp8.py create mode 100644 sgl-kernel/tests/test_per_token_group_quant_8bit.py create mode 100644 sgl-kernel/tests/test_per_token_quant_fp8.py create mode 100644 sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py create mode 100644 sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py create mode 100644 sgl-kernel/tests/test_rotary_embedding.py create mode 100644 sgl-kernel/tests/test_sampling.py create mode 100644 sgl-kernel/tests/test_sparse_flash_attn.py create mode 100644 sgl-kernel/tests/utils.py create mode 100644 sgl-router/.cargo/config.toml create mode 100644 sgl-router/.coveragerc create mode 100644 sgl-router/Cargo.toml create mode 100644 sgl-router/MANIFEST.in create mode 100644 sgl-router/Makefile create mode 100644 sgl-router/README.md create mode 100644 sgl-router/benches/request_processing.rs create mode 100644 sgl-router/benches/tokenizer_benchmark.rs create mode 100644 sgl-router/benches/tool_parser_benchmark.rs create mode 100644 sgl-router/build.rs create mode 100644 sgl-router/py_src/sglang_router/__init__.py create mode 100644 sgl-router/py_src/sglang_router/launch_router.py create mode 100644 sgl-router/py_src/sglang_router/launch_server.py create mode 100644 sgl-router/py_src/sglang_router/mini_lb.py create mode 100644 sgl-router/py_src/sglang_router/router.py create mode 100644 sgl-router/py_src/sglang_router/router_args.py create mode 100644 sgl-router/py_src/sglang_router/version.py create mode 100644 sgl-router/py_test/__init__.py create mode 100644 sgl-router/py_test/conftest.py create mode 100644 sgl-router/py_test/e2e/conftest.py create mode 100644 sgl-router/py_test/e2e/test_pd_router.py create mode 100644 sgl-router/py_test/e2e/test_regular_router.py create mode 100644 sgl-router/py_test/fixtures/__init__.py create mode 100644 sgl-router/py_test/fixtures/mock_worker.py create mode 100644 sgl-router/py_test/fixtures/ports.py create mode 100644 sgl-router/py_test/fixtures/router_manager.py create mode 100644 sgl-router/py_test/integration/__init__.py create mode 100644 sgl-router/py_test/integration/conftest.py create mode 100644 sgl-router/py_test/integration/load_balancing/__init__.py create mode 100644 sgl-router/py_test/integration/load_balancing/test_cache_aware.py create mode 100644 sgl-router/py_test/integration/load_balancing/test_power_of_two.py create mode 100644 sgl-router/py_test/integration/load_balancing/test_random.py create mode 100644 sgl-router/py_test/integration/load_balancing/test_round_robin.py create mode 100644 sgl-router/py_test/integration/test_api_auth.py create mode 100644 sgl-router/py_test/integration/test_circuit_breaker.py create mode 100644 sgl-router/py_test/integration/test_fault_tolerance.py create mode 100644 sgl-router/py_test/integration/test_payload_size.py create mode 100644 sgl-router/py_test/integration/test_pd_routing.py create mode 100644 sgl-router/py_test/integration/test_rate_limiting.py create mode 100644 sgl-router/py_test/integration/test_retries.py create mode 100644 sgl-router/py_test/integration/test_service_discovery_shim.py create mode 100644 sgl-router/py_test/integration/test_worker_management.py create mode 100644 sgl-router/py_test/unit/__init__.py create mode 100644 sgl-router/py_test/unit/test_arg_parser.py create mode 100644 sgl-router/py_test/unit/test_router_config.py create mode 100644 sgl-router/py_test/unit/test_startup_sequence.py create mode 100644 sgl-router/py_test/unit/test_validation.py create mode 100644 sgl-router/pyproject.toml create mode 100644 sgl-router/pytest.ini create mode 100755 sgl-router/scripts/run_benchmarks.py create mode 100644 sgl-router/setup.py create mode 100644 sgl-router/src/config/mod.rs create mode 100644 sgl-router/src/config/types.rs create mode 100644 sgl-router/src/config/validation.rs create mode 100644 sgl-router/src/core/circuit_breaker.rs create mode 100644 sgl-router/src/core/error.rs create mode 100644 sgl-router/src/core/mod.rs create mode 100644 sgl-router/src/core/retry.rs create mode 100644 sgl-router/src/core/token_bucket.rs create mode 100644 sgl-router/src/core/worker.rs create mode 100644 sgl-router/src/grpc/client.rs create mode 100644 sgl-router/src/grpc/mod.rs create mode 100644 sgl-router/src/lib.rs create mode 100644 sgl-router/src/logging.rs create mode 100644 sgl-router/src/main.rs create mode 100644 sgl-router/src/mcp/client_manager.rs create mode 100644 sgl-router/src/mcp/config.rs create mode 100644 sgl-router/src/mcp/error.rs create mode 100644 sgl-router/src/mcp/mod.rs create mode 100644 sgl-router/src/mcp/oauth.rs create mode 100644 sgl-router/src/metrics.rs create mode 100644 sgl-router/src/middleware.rs create mode 100644 sgl-router/src/policies/cache_aware.rs create mode 100644 sgl-router/src/policies/factory.rs create mode 100644 sgl-router/src/policies/mod.rs create mode 100644 sgl-router/src/policies/power_of_two.rs create mode 100644 sgl-router/src/policies/random.rs create mode 100644 sgl-router/src/policies/round_robin.rs create mode 100644 sgl-router/src/proto/sglang_scheduler.proto create mode 100644 sgl-router/src/protocols/mod.rs create mode 100644 sgl-router/src/protocols/spec.rs create mode 100644 sgl-router/src/protocols/validation.rs create mode 100644 sgl-router/src/reasoning_parser/README.md create mode 100644 sgl-router/src/reasoning_parser/factory.rs create mode 100644 sgl-router/src/reasoning_parser/mod.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/base.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/glm45.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/kimi.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/mod.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/qwen3.rs create mode 100644 sgl-router/src/reasoning_parser/parsers/step3.rs create mode 100644 sgl-router/src/reasoning_parser/traits.rs create mode 100644 sgl-router/src/routers/factory.rs create mode 100644 sgl-router/src/routers/grpc/mod.rs create mode 100644 sgl-router/src/routers/grpc/pd_router.rs create mode 100644 sgl-router/src/routers/grpc/router.rs create mode 100644 sgl-router/src/routers/header_utils.rs create mode 100644 sgl-router/src/routers/http/mod.rs create mode 100644 sgl-router/src/routers/http/openai_router.rs create mode 100644 sgl-router/src/routers/http/pd_router.rs create mode 100644 sgl-router/src/routers/http/pd_types.rs create mode 100644 sgl-router/src/routers/http/router.rs create mode 100644 sgl-router/src/routers/mod.rs create mode 100644 sgl-router/src/server.rs create mode 100644 sgl-router/src/service_discovery.rs create mode 100644 sgl-router/src/tokenizer/README.md create mode 100644 sgl-router/src/tokenizer/chat_template.rs create mode 100644 sgl-router/src/tokenizer/factory.rs create mode 100644 sgl-router/src/tokenizer/hub.rs create mode 100644 sgl-router/src/tokenizer/huggingface.rs create mode 100644 sgl-router/src/tokenizer/mock.rs create mode 100644 sgl-router/src/tokenizer/mod.rs create mode 100644 sgl-router/src/tokenizer/sequence.rs create mode 100644 sgl-router/src/tokenizer/stop.rs create mode 100644 sgl-router/src/tokenizer/stream.rs create mode 100644 sgl-router/src/tokenizer/tests.rs create mode 100644 sgl-router/src/tokenizer/tiktoken.rs create mode 100644 sgl-router/src/tokenizer/traits.rs create mode 100644 sgl-router/src/tool_parser/errors.rs create mode 100644 sgl-router/src/tool_parser/mod.rs create mode 100644 sgl-router/src/tool_parser/parsers/deepseek_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/glm4_moe_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/gpt_oss_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/json_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/kimik2_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/llama_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/mistral_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/mod.rs create mode 100644 sgl-router/src/tool_parser/parsers/pythonic_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/qwen_parser.rs create mode 100644 sgl-router/src/tool_parser/parsers/step3_parser.rs create mode 100644 sgl-router/src/tool_parser/partial_json.rs create mode 100644 sgl-router/src/tool_parser/python_literal_parser.rs create mode 100644 sgl-router/src/tool_parser/registry.rs create mode 100644 sgl-router/src/tool_parser/state.rs create mode 100644 sgl-router/src/tool_parser/tests.rs create mode 100644 sgl-router/src/tool_parser/traits.rs create mode 100644 sgl-router/src/tool_parser/types.rs create mode 100644 sgl-router/src/tree.rs create mode 100644 sgl-router/tests/api_endpoints_test.rs create mode 100644 sgl-router/tests/common/mock_mcp_server.rs create mode 100644 sgl-router/tests/common/mock_openai_server.rs create mode 100755 sgl-router/tests/common/mock_worker.rs create mode 100644 sgl-router/tests/common/mod.rs create mode 100644 sgl-router/tests/common/test_app.rs create mode 100644 sgl-router/tests/mcp_test.rs create mode 100644 sgl-router/tests/request_formats_test.rs create mode 100644 sgl-router/tests/responses_api_test.rs create mode 100644 sgl-router/tests/streaming_tests.rs create mode 100644 sgl-router/tests/test_chat_template.rs create mode 100644 sgl-router/tests/test_chat_template_loading.rs create mode 100644 sgl-router/tests/test_openai_routing.rs create mode 100644 sgl-router/tests/test_pd_routing.rs create mode 100644 sgl-router/tests/tokenizer_integration.rs create mode 100644 sgl-router/tests/tool_parser_deepseek.rs create mode 100644 sgl-router/tests/tool_parser_edge_cases.rs create mode 100644 sgl-router/tests/tool_parser_glm4_moe.rs create mode 100644 sgl-router/tests/tool_parser_gpt_oss.rs create mode 100644 sgl-router/tests/tool_parser_json.rs create mode 100644 sgl-router/tests/tool_parser_kimik2.rs create mode 100644 sgl-router/tests/tool_parser_llama.rs create mode 100644 sgl-router/tests/tool_parser_mistral.rs create mode 100644 sgl-router/tests/tool_parser_mixed_edge_cases.rs create mode 100644 sgl-router/tests/tool_parser_pythonic.rs create mode 100644 sgl-router/tests/tool_parser_qwen.rs create mode 100644 sgl-router/tests/tool_parser_registry.rs create mode 100644 sgl-router/tests/tool_parser_step3.rs create mode 100644 sgl-router/tests/tool_parser_streaming.rs create mode 100644 sgl-router/tests/tool_parser_wrapper_tokens.rs create mode 100644 test/README.md create mode 100644 test/lang/run_suite.py create mode 100644 test/lang/test_anthropic_backend.py create mode 100644 test/lang/test_bind_cache.py create mode 100644 test/lang/test_choices.py create mode 100644 test/lang/test_litellm_backend.py create mode 100644 test/lang/test_openai_backend.py create mode 100644 test/lang/test_separate_reasoning.py create mode 100644 test/lang/test_separate_reasoning_execution.py create mode 100644 test/lang/test_srt_backend.py create mode 100644 test/lang/test_tracing.py create mode 100644 test/lang/test_vertexai_backend.py create mode 100644 test/pytest.ini create mode 100644 test/srt/ascend/test_ascend_deepep.py create mode 100644 test/srt/ascend/test_ascend_graph_tp1_bf16.py create mode 100644 test/srt/ascend/test_ascend_graph_tp2_bf16.py create mode 100644 test/srt/ascend/test_ascend_mla_fia_w8a8int8.py create mode 100644 test/srt/ascend/test_ascend_mla_w8a8int8.py create mode 100644 test/srt/ascend/test_ascend_tp1_bf16.py create mode 100644 test/srt/ascend/test_ascend_tp2_bf16.py create mode 100644 test/srt/ascend/test_ascend_tp2_fia_bf16.py create mode 100644 test/srt/ascend/test_ascend_tp4_bf16.py create mode 100644 test/srt/ascend/test_ascend_w8a8_quantization.py create mode 100644 test/srt/configs/deepseek_v3.yaml create mode 100644 test/srt/configs/deepseek_v3_long_context.yaml create mode 100644 test/srt/configs/llama_405b.yaml create mode 100644 test/srt/configs/random_config.yaml create mode 100644 test/srt/configs/random_flashinfer_vs_triton_config.yaml create mode 100644 test/srt/configs/sharegpt_config.yaml create mode 100644 test/srt/cpu/test_activation.py create mode 100644 test/srt/cpu/test_binding.py create mode 100644 test/srt/cpu/test_decode.py create mode 100644 test/srt/cpu/test_extend.py create mode 100644 test/srt/cpu/test_gemm.py create mode 100644 test/srt/cpu/test_mla.py create mode 100644 test/srt/cpu/test_moe.py create mode 100644 test/srt/cpu/test_norm.py create mode 100644 test/srt/cpu/test_qkv_proj_with_rope.py create mode 100644 test/srt/cpu/test_rope.py create mode 100644 test/srt/cpu/test_shared_expert.py create mode 100644 test/srt/cpu/test_topk.py create mode 100644 test/srt/cpu/utils.py create mode 100644 test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json create mode 100644 test/srt/entrypoints/http_server/test_abort_request.py create mode 100644 test/srt/ep/test_deepep_internode.py create mode 100644 test/srt/ep/test_deepep_intranode.py create mode 100644 test/srt/ep/test_deepep_large.py create mode 100644 test/srt/ep/test_deepep_low_latency.py create mode 100644 test/srt/ep/test_deepep_small.py create mode 100755 test/srt/ep/test_eplb.py create mode 100644 test/srt/ep/test_hybrid_dp_ep_tp_mtp.py create mode 100644 test/srt/ep/test_moe_deepep.py create mode 100644 test/srt/ep/test_moe_deepep_eval_accuracy_large.py create mode 100644 test/srt/ep/test_moe_ep.py create mode 100644 test/srt/experiment_runner.py create mode 100644 test/srt/hicache/test_hicache.py create mode 100644 test/srt/hicache/test_hicache_mla.py create mode 100644 test/srt/hicache/test_hicache_page.py create mode 100644 test/srt/hicache/test_hicache_storage.py create mode 100644 test/srt/hicache/test_hicache_storage_3fs_backend.py create mode 100644 test/srt/hicache/test_hicache_storage_file_backend.py create mode 100644 test/srt/kv_cache_scales_llama3_1_8b.json create mode 100644 test/srt/kv_cache_scales_llama3_8b.json create mode 100644 test/srt/kv_cache_scales_qwen2_1_5b.json create mode 100644 test/srt/lora/test_lora.py create mode 100644 test/srt/lora/test_lora_backend.py create mode 100644 test/srt/lora/test_lora_cuda_graph.py create mode 100644 test/srt/lora/test_lora_eviction.py create mode 100644 test/srt/lora/test_lora_llama4.py create mode 100644 test/srt/lora/test_lora_qwen3.py create mode 100644 test/srt/lora/test_lora_radix_cache.py create mode 100644 test/srt/lora/test_lora_tp.py create mode 100644 test/srt/lora/test_lora_update.py create mode 100644 test/srt/lora/test_multi_lora_backend.py create mode 100644 test/srt/lora/utils.py create mode 100644 test/srt/models/compare.py create mode 100644 test/srt/models/test_clip_models.py create mode 100644 test/srt/models/test_compressed_tensors_models.py create mode 100644 test/srt/models/test_cross_encoder_models.py create mode 100644 test/srt/models/test_dummy_grok_models.py create mode 100644 test/srt/models/test_embedding_models.py create mode 100644 test/srt/models/test_encoder_embedding_models.py create mode 100644 test/srt/models/test_generation_models.py create mode 100644 test/srt/models/test_gme_qwen_models.py create mode 100644 test/srt/models/test_grok_models.py create mode 100644 test/srt/models/test_llama4_models.py create mode 100644 test/srt/models/test_mtp_models.py create mode 100644 test/srt/models/test_qwen_models.py create mode 100644 test/srt/models/test_reward_models.py create mode 100644 test/srt/models/test_transformers_models.py create mode 100644 test/srt/models/test_unsloth_models.py create mode 100644 test/srt/models/test_vlm_models.py create mode 100644 test/srt/openai_server/__init__.py create mode 100644 test/srt/openai_server/basic/__init__.py create mode 100644 test/srt/openai_server/basic/test_openai_embedding.py create mode 100644 test/srt/openai_server/basic/test_openai_server.py create mode 100644 test/srt/openai_server/basic/test_protocol.py create mode 100644 test/srt/openai_server/basic/test_serving_chat.py create mode 100644 test/srt/openai_server/basic/test_serving_completions.py create mode 100644 test/srt/openai_server/basic/test_serving_embedding.py create mode 100644 test/srt/openai_server/features/__init__.py create mode 100644 test/srt/openai_server/features/test_cache_report.py create mode 100644 test/srt/openai_server/features/test_enable_thinking.py create mode 100644 test/srt/openai_server/features/test_json_constrained.py create mode 100644 test/srt/openai_server/features/test_json_mode.py create mode 100644 test/srt/openai_server/features/test_openai_server_ebnf.py create mode 100644 test/srt/openai_server/features/test_openai_server_hidden_states.py create mode 100644 test/srt/openai_server/features/test_reasoning_content.py create mode 100644 test/srt/openai_server/function_call/__init__.py create mode 100644 test/srt/openai_server/function_call/test_openai_function_calling.py create mode 100644 test/srt/openai_server/function_call/test_tool_choice.py create mode 100644 test/srt/openai_server/validation/__init__.py create mode 100644 test/srt/openai_server/validation/test_large_max_new_tokens.py create mode 100644 test/srt/openai_server/validation/test_matched_stop.py create mode 100644 test/srt/openai_server/validation/test_openai_server_ignore_eos.py create mode 100644 test/srt/openai_server/validation/test_request_length_validation.py create mode 100644 test/srt/parse_results.py create mode 100644 test/srt/quant/test_awq.py create mode 100644 test/srt/quant/test_awq_dequant.py create mode 100644 test/srt/quant/test_block_int8.py create mode 100644 test/srt/quant/test_fp8_kernel.py create mode 100644 test/srt/quant/test_fp8_kvcache.py create mode 100644 test/srt/quant/test_int8_kernel.py create mode 100644 test/srt/quant/test_triton_scaled_mm.py create mode 100644 test/srt/quant/test_w4a8_deepseek_v3.py create mode 100644 test/srt/quant/test_w8a8_quantization.py create mode 100644 test/srt/rl/test_update_weights_from_disk.py create mode 100644 test/srt/rl/test_update_weights_from_distributed.py create mode 100644 test/srt/rl/test_update_weights_from_tensor.py create mode 100644 test/srt/rl/test_verl_engine_2_gpu.py create mode 100644 test/srt/rl/test_verl_engine_4_gpu.py create mode 100644 test/srt/run_suite.py create mode 100644 test/srt/test_abort.py create mode 100644 test/srt/test_bench_one_batch.py create mode 100644 test/srt/test_bench_serving.py create mode 100644 test/srt/test_bnb.py create mode 100644 test/srt/test_chunked_prefill.py create mode 100644 test/srt/test_cpp_radix_cache.py create mode 100644 test/srt/test_cpu_graph.py create mode 100644 test/srt/test_create_kvindices.py create mode 100644 test/srt/test_custom_allreduce.py create mode 100644 test/srt/test_data_parallelism.py create mode 100644 test/srt/test_disaggregation.py create mode 100644 test/srt/test_disaggregation_different_tp.py create mode 100644 test/srt/test_disaggregation_pp.py create mode 100644 test/srt/test_double_sparsity.py create mode 100644 test/srt/test_dp_attention.py create mode 100644 test/srt/test_eagle_infer_a.py create mode 100644 test/srt/test_eagle_infer_b.py create mode 100644 test/srt/test_ebnf_constrained.py create mode 100644 test/srt/test_eval_accuracy_large.py create mode 100644 test/srt/test_eval_fp8_accuracy.py create mode 100755 test/srt/test_expert_distribution.py create mode 100644 test/srt/test_expert_location_updater.py create mode 100644 test/srt/test_fa3.py create mode 100644 test/srt/test_fim_completion.py create mode 100644 test/srt/test_flashmla.py create mode 100644 test/srt/test_forward_split_prefill.py create mode 100644 test/srt/test_full_deepseek_v3.py create mode 100644 test/srt/test_function_call_parser.py create mode 100644 test/srt/test_fused_moe.py create mode 100644 test/srt/test_get_weights_by_name.py create mode 100644 test/srt/test_gguf.py create mode 100644 test/srt/test_gpt_oss_1gpu.py create mode 100644 test/srt/test_gpt_oss_4gpu.py create mode 100644 test/srt/test_gpt_oss_common.py create mode 100644 test/srt/test_gptqmodel_dynamic.py create mode 100644 test/srt/test_harmony_parser.py create mode 100644 test/srt/test_health_check.py create mode 100644 test/srt/test_hidden_states.py create mode 100644 test/srt/test_hybrid_attn_backend.py create mode 100644 test/srt/test_input_embeddings.py create mode 100644 test/srt/test_intel_amx_attention_backend.py create mode 100644 test/srt/test_io_struct.py create mode 100644 test/srt/test_jinja_template_utils.py create mode 100644 test/srt/test_kv_events.py create mode 100644 test/srt/test_local_attn.py create mode 100644 test/srt/test_metrics.py create mode 100644 test/srt/test_metrics_utils.py create mode 100644 test/srt/test_mla.py create mode 100644 test/srt/test_mla_deepseek_v3.py create mode 100644 test/srt/test_mla_flashinfer.py create mode 100644 test/srt/test_mla_fp8.py create mode 100644 test/srt/test_mla_int8_deepseek_v3.py create mode 100644 test/srt/test_mla_tp.py create mode 100644 test/srt/test_modelopt.py create mode 100644 test/srt/test_modelopt_fp8kvcache.py create mode 100644 test/srt/test_models_from_modelscope.py create mode 100644 test/srt/test_moe_eval_accuracy_large.py create mode 100644 test/srt/test_mscclpp.py create mode 100644 test/srt/test_multi_instance_release_memory_occupation.py create mode 100644 test/srt/test_multi_tokenizer.py create mode 100644 test/srt/test_nightly_gsm8k_eval.py create mode 100644 test/srt/test_nightly_gsm8k_eval_amd.py create mode 100644 test/srt/test_no_chunked_prefill.py create mode 100644 test/srt/test_no_overlap_scheduler.py create mode 100644 test/srt/test_original_logprobs.py create mode 100644 test/srt/test_page_size.py create mode 100644 test/srt/test_patch_torch.py create mode 100644 test/srt/test_penalty.py create mode 100644 test/srt/test_pp_single_node.py create mode 100644 test/srt/test_pytorch_sampling_backend.py create mode 100644 test/srt/test_quick_allreduce.py create mode 100644 test/srt/test_radix_attention.py create mode 100644 test/srt/test_reasoning_parser.py create mode 100644 test/srt/test_regex_constrained.py create mode 100644 test/srt/test_release_memory_occupation.py create mode 100644 test/srt/test_request_queue_validation.py create mode 100644 test/srt/test_retract_decode.py create mode 100644 test/srt/test_rope_rocm.py create mode 100644 test/srt/test_sagemaker_server.py create mode 100644 test/srt/test_schedule_policy.py create mode 100644 test/srt/test_score_api.py create mode 100644 test/srt/test_server_args.py create mode 100644 test/srt/test_session_control.py create mode 100644 test/srt/test_skip_tokenizer_init.py create mode 100644 test/srt/test_srt_endpoint.py create mode 100644 test/srt/test_srt_engine.py create mode 100644 test/srt/test_srt_engine_with_quant_args.py create mode 100644 test/srt/test_standalone_speculative_decoding.py create mode 100644 test/srt/test_start_profile.py create mode 100644 test/srt/test_swa_unittest.py create mode 100644 test/srt/test_tokenizer_batch_encode.py create mode 100644 test/srt/test_torch_compile.py create mode 100644 test/srt/test_torch_compile_moe.py create mode 100644 test/srt/test_torch_native_attention_backend.py create mode 100644 test/srt/test_torch_tp.py create mode 100644 test/srt/test_torchao.py create mode 100644 test/srt/test_triton_attention_backend.py create mode 100644 test/srt/test_triton_attention_kernels.py create mode 100644 test/srt/test_triton_attention_rocm_mla.py create mode 100644 test/srt/test_triton_fused_moe.py create mode 100644 test/srt/test_triton_moe_channel_fp8_kernel.py create mode 100644 test/srt/test_triton_moe_wna16.py create mode 100644 test/srt/test_triton_sliding_window.py create mode 100644 test/srt/test_two_batch_overlap.py create mode 100644 test/srt/test_utils_update_weights.py create mode 100644 test/srt/test_vertex_endpoint.py create mode 100644 test/srt/test_vision_chunked_prefill.py create mode 100644 test/srt/test_vision_openai_server_a.py create mode 100644 test/srt/test_vision_openai_server_b.py create mode 100644 test/srt/test_vision_openai_server_common.py create mode 100644 test/srt/test_vllm_dependency.py create mode 100644 test/srt/test_vlm_accuracy.py create mode 100644 test/srt/test_vlm_input_format.py create mode 100644 test/srt/test_wave_attention_backend.py create mode 100644 test/srt/test_wave_attention_kernels.py create mode 100644 test/srt/test_weight_version.py diff --git a/.clang-format-ignore b/.clang-format-ignore new file mode 100644 index 000000000..15c76cc45 --- /dev/null +++ b/.clang-format-ignore @@ -0,0 +1 @@ +sgl-kernel/3rdparty/tensorrt_llm/* diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..3c7b67cac --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,35 @@ +FROM lmsysorg/sglang:dev + +# Create non-root user with specified UID and GID +# NOTE: Replace with your own UID and GID. This is a workaround from https://github.com/microsoft/vscode-remote-release/issues/49#issuecomment-489060908. +ARG HOST_UID=1003 +ARG HOST_GID=1003 +RUN groupadd -g $HOST_GID devuser && \ + useradd -m -u $HOST_UID -g $HOST_GID -s /bin/zsh devuser + +# Give devuser sudo access +RUN apt-get update && apt-get install -y sudo && \ + echo "devuser ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/devuser && \ + rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Set up oh-my-zsh for devuser +RUN cp -r /root/.oh-my-zsh /home/devuser/.oh-my-zsh && \ + cp /root/.zshrc /home/devuser/.zshrc && \ + cp /root/.vimrc /home/devuser/.vimrc && \ + cp /root/.tmux.conf /home/devuser/.tmux.conf && \ + sed -i 's|/root/.oh-my-zsh|/home/devuser/.oh-my-zsh|g' /home/devuser/.zshrc && \ + chown -R devuser:devuser /home/devuser/ + +# Set workspace directory and ownership +WORKDIR /sgl-workspace/sglang +RUN chown -R devuser:devuser /sgl-workspace + +# Switch to devuser +USER devuser + +# Install uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..338b10fe6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,30 @@ +{ + "name": "sglang", + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "devuser", + "customizations": { + "vscode": { + "extensions": [ + // Python development + "ms-python.python", + "charliermarsh.ruff", + // Rust development + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml" + ] + } + }, + "forwardPorts": [], + "runArgs": [ + "--gpus", + "all" + ], + // The two lines below ensures that your local changes in the sglang + // repo is automatically synced to the sglang pip package installed + // in the dev docker container. You can remove / comment out these + // two lines if you prefer to sync code changes manually. + "workspaceMount": "source=${localWorkspaceFolder},target=/sgl-workspace/sglang,type=bind", + "workspaceFolder": "/sgl-workspace/sglang" +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..030a7293d --- /dev/null +++ b/.editorconfig @@ -0,0 +1,25 @@ +# https://editorconfig.org/ + +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.{json,yaml,yml}] +indent_size = 2 + +[*.md] +indent_size = 2 +x-soft-wrap-text = true + +[*.rst] +indent_size = 4 +x-soft-wrap-text = true + +[Makefile] +indent_style = tab diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..32435d6ed --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,21 @@ +.github @merrymercy @zhyncs +/docker @zhyncs @HaiShaw @ByronHsu +/python/pyproject.toml @merrymercy @zhyncs +/python/sglang/* @merrymercy @Ying1123 @zhyncs @hnyls2002 +/python/sglang/srt/constrained @hnyls2002 +/python/sglang/srt/disaggregation @ByronHsu @hnyls2002 +/python/sglang/srt/disaggregation/mooncake @ShangmingCai +/python/sglang/srt/distributed @yizhang2077 @merrymercy +/python/sglang/srt/entrypoints @ispobock @CatherineSue @slin1237 @merrymercy +/python/sglang/srt/eplb @fzyzcjy +/python/sglang/srt/function_call @CatherineSue +/python/sglang/srt/layers @merrymercy @Ying1123 @zhyncs @ispobock @HaiShaw @ch-wan @BBuf @kushanam @Edwardf0t1 +/python/sglang/srt/lora @Ying1123 @Fridge003 @lifuhuang +/python/sglang/srt/managers @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann +/python/sglang/srt/mem_cache @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann +/python/sglang/srt/model_executor @merrymercy @Ying1123 @hnyls2002 @zhyncs @ispobock +/python/sglang/srt/multimodal @mickqian @JustinTong0323 +/python/sglang/srt/speculative @Ying1123 @merrymercy @rkooo567 @kssteven418 +/sgl-kernel @zhyncs @ispobock @HandH1998 @BBuf @yizhang2077 @merrymercy @FlamingoPg @HaiShaw +/sgl-router @slin1237 @ByronHsu +/test/srt/test_modelopt* @Edwardf0t1 diff --git a/.github/ISSUE_TEMPLATE/1-bug-report.yml b/.github/ISSUE_TEMPLATE/1-bug-report.yml new file mode 100644 index 000000000..5f6734867 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1-bug-report.yml @@ -0,0 +1,38 @@ +name: 🐞 Bug report +description: Create a report to help us reproduce and fix the bug +title: "[Bug] " +labels: ['Bug'] + +body: +- type: checkboxes + attributes: + label: Checklist + options: + - label: 1. I have searched related issues but cannot get the expected help. + - label: 2. The bug has not been fixed in the latest version. + - label: 3. Please note that if the bug-related issue you submitted lacks corresponding environment info and a minimal reproducible demo, it will be challenging for us to reproduce and resolve the issue, reducing the likelihood of receiving feedback. + - label: 4. If the issue you raised is not a bug but a question, please raise a discussion at https://github.com/sgl-project/sglang/discussions/new/choose Otherwise, it will be closed. + - label: 5. Please use English, otherwise it will be closed. +- type: textarea + attributes: + label: Describe the bug + description: A clear and concise description of what the bug is. + validations: + required: true +- type: textarea + attributes: + label: Reproduction + description: | + What command or script did you run? Which **model** are you using? + placeholder: | + A placeholder for the command. + validations: + required: true +- type: textarea + attributes: + label: Environment + description: | + Please provide necessary environment information here with `python3 -m sglang.check_env`. Otherwise the issue will be closed. + placeholder: Environment here. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/2-feature-request.yml b/.github/ISSUE_TEMPLATE/2-feature-request.yml new file mode 100644 index 000000000..31bc4a127 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-feature-request.yml @@ -0,0 +1,23 @@ +name: 🚀 Feature request +description: Suggest an idea for this project +title: "[Feature] " + +body: +- type: checkboxes + attributes: + label: Checklist + options: + - label: 1. If the issue you raised is not a feature but a question, please raise a discussion at https://github.com/sgl-project/sglang/discussions/new/choose Otherwise, it will be closed. + - label: 2. Please use English, otherwise it will be closed. +- type: textarea + attributes: + label: Motivation + description: | + A clear and concise description of the motivation of the feature. + validations: + required: true +- type: textarea + attributes: + label: Related resources + description: | + If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful. diff --git a/.github/REVIEWERS.md b/.github/REVIEWERS.md new file mode 100644 index 000000000..ac9ce6102 --- /dev/null +++ b/.github/REVIEWERS.md @@ -0,0 +1,53 @@ +# Area Reviewer + +Here are some reviewers for common areas. You can ping them to review your code if you touch related parts. + +## Hardware platforms +- general @Alcanderian +- AMD GPU @HaiShaw +- Blackwell GPU @kushanam @trevor-m @zhyncs +- CPU @mingfeima + +## Kernel +- general @zhyncs @ispobock @HandH1998 @BBuf @yizhang2077 @HaiShaw +- triton attention backend @ispobock +- aiter attention backend @HaiShaw @kkHuang-amd @valarLip +- flash attention backend @hebiao064 +- flashinfer attention backend @Fridge003 +- moe kernel @BBuf @fzyzcjy @ch-wan @Alcanderian + +## Scheduler and memory pool +- general @merrymercy @Ying1123 @hnyls2002 @xiezhq-hermann +- constrained decoding @hnyls2002 +- hierarchical cache @xiezhq-hermann @DarkSharpness +- lora @Fridge003 @Ying1123 @lifuhuang +- speculative decoding @merrymercy @Ying1123 @kssteven418 @Qiaolin-Yu +- sliding window attention @hanming-lu + +## Parallelism +- expert parallelism @fzyzcjy @ch-wan +- data parallelism attention @ch-wan +- pipeline parallelism @Ying1123 +- tensor parallelism @merrymercy + +## PD disaggregation +- general @ByronHsu @ShangmingCai @hnyls2002 +- Mooncake backend @ShangmingCai + +## Build and release +- general @zhyncs @merrymercy + +## API Server +- general @CatherineSue @slin1237 @ispobock +- function calling and reasoning parsing @CatherineSue +- OpenAI API @CatherineSue @slin1237 + +## SGL-Router +- general @slin1237 @ByronHsu + +## Model +- multimodal models @mickqian @JustinTong0323 +- other new models @zhaochenyang20 + +## Reinforcment learning +- general @zhaochenyang20 @hebiao064 @fzyzcjy @zhuzilin diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..ab51d4bf5 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ + + +## Motivation + + + +## Modifications + + + +## Accuracy Tests + + + +## Benchmarking and Profiling + + + +## Checklist + +- [ ] Format your code according to the [Format code with pre-commit](https://docs.sglang.ai/developer_guide/contribution_guide.html#format-code-with-pre-commit). +- [ ] Add unit tests according to the [Run and add unit tests](https://docs.sglang.ai/developer_guide/contribution_guide.html#run-and-add-unit-tests). +- [ ] Update documentation according to [Write documentations](https://docs.sglang.ai/developer_guide/contribution_guide.html#write-documentations). +- [ ] Provide accuracy and speed benchmark results according to [Test the accuracy](https://docs.sglang.ai/developer_guide/contribution_guide.html#test-the-accuracy) and [Benchmark the speed](https://docs.sglang.ai/developer_guide/contribution_guide.html#benchmark-the-speed). diff --git a/.github/workflows/cancel-all-pending-pr-test-runs.yml b/.github/workflows/cancel-all-pending-pr-test-runs.yml new file mode 100644 index 000000000..6217542e7 --- /dev/null +++ b/.github/workflows/cancel-all-pending-pr-test-runs.yml @@ -0,0 +1,45 @@ +name: Cancel All Pending PR Test Runs + +on: + workflow_dispatch: + inputs: + workflows: + description: 'Space-separated list of workflow filenames to cancel' + required: true + type: string + default: 'pr-test.yml pr-test-xeon.yml' + +permissions: + actions: write # Needed to cancel runs + contents: read # Needed to read repo info + +jobs: + cancel-pending: + runs-on: ubuntu-latest + steps: + - name: Install GitHub CLI + run: sudo apt-get install -y gh jq + + - name: Cancel all pending/waiting runs for specified workflows + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + run: | + # Read the space-separated string from the input into a bash array + WORKFLOW_FILES=(${{ github.event.inputs.workflows }}) + + echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${{ github.event.inputs.workflows }}" + + for workflow_file in "${WORKFLOW_FILES[@]}"; do + echo "--- Checking workflow: $workflow_file ---" + gh run list \ + --repo "$REPO" \ + --workflow "$workflow_file" \ + --json databaseId,status \ + --limit 1000 \ + | jq -r '.[] | select(.status=="queued" or .status=="in_progress") | .databaseId' \ + | while read run_id; do + echo "Cancelling run ID: $run_id for workflow: $workflow_file" + gh run cancel "$run_id" --repo "$REPO" + done + done diff --git a/.github/workflows/cancel-pr-workflow-on-merge.yml b/.github/workflows/cancel-pr-workflow-on-merge.yml new file mode 100644 index 000000000..535884ba6 --- /dev/null +++ b/.github/workflows/cancel-pr-workflow-on-merge.yml @@ -0,0 +1,22 @@ +name: Cancel PR Workflows on Merge + +on: + pull_request_target: + types: + - closed + +permissions: + actions: write + +jobs: + cancel: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.12.1 + with: + workflow_id: all + access_token: ${{ secrets.GITHUB_TOKEN }} + ignore_sha: true + pr_number: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/close-inactive-issues.yml b/.github/workflows/close-inactive-issues.yml new file mode 100644 index 000000000..048e6c443 --- /dev/null +++ b/.github/workflows/close-inactive-issues.yml @@ -0,0 +1,96 @@ +name: Close Inactive Issues + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + contents: read + +jobs: + close-inactive-issues: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + steps: + - name: Check and close inactive issues + uses: actions/github-script@v6 + with: + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const sixtyDaysAgo = new Date(Date.now() - 60 * 24 * 60 * 60 * 1000); + + const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); + console.log(`Owner: ${owner}, Repo: ${repo}`); + + async function fetchIssues(page = 1) { + console.log(`Fetching issues for ${owner}/${repo}, page ${page}`); + return await github.rest.issues.listForRepo({ + owner, + repo, + state: 'open', + sort: 'updated', + direction: 'asc', + per_page: 100, + page: page + }); + } + + async function processIssues() { + console.log('Starting to process issues'); + console.log(`Repository: ${owner}/${repo}`); + + let page = 1; + let hasMoreIssues = true; + while (hasMoreIssues) { + try { + const issues = await fetchIssues(page); + console.log(`Fetched ${issues.data.length} issues on page ${page}`); + + if (issues.data.length === 0) { + hasMoreIssues = false; + break; + } + + for (const issue of issues.data) { + // Skip if the issue has 'good first issue' label + if (issue.labels.some(label => label.name === 'good first issue')) { + console.log(`Skipping issue #${issue.number} as it's marked as 'good first issue'`); + continue; + } + if (new Date(issue.updated_at) < sixtyDaysAgo) { + try { + await github.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: 'closed', + labels: [...issue.labels.map(l => l.name), 'inactive'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: 'This issue has been automatically closed due to inactivity. Please feel free to reopen it if needed.' + }); + console.log(`Closed issue #${issue.number} due to inactivity.`); + } catch (error) { + console.error(`Failed to close issue #${issue.number}: ${error.message}`); + } + } else { + console.log(`Issue #${issue.number} is still active. Stopping processing.`); + hasMoreIssues = false; + break; + } + } + page += 1; + } catch (error) { + console.error(`Error fetching issues on page ${page}: ${error.message}`); + hasMoreIssues = false; + } + } + console.log('Finished processing issues'); + } + + await processIssues(); diff --git a/.github/workflows/execute-notebook.yml b/.github/workflows/execute-notebook.yml new file mode 100644 index 000000000..7298d80ec --- /dev/null +++ b/.github/workflows/execute-notebook.yml @@ -0,0 +1,60 @@ +name: Execute Notebooks + +on: + pull_request: + branches: [ main ] + paths: + - "python/sglang/**" + - "docs/**" + workflow_dispatch: + + +concurrency: + group: execute-notebook-${{ github.ref }} + cancel-in-progress: true + + +jobs: + run-all-notebooks: + runs-on: 1-gpu-runner + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + pip install -r docs/requirements.txt + apt-get update && apt-get install -y pandoc parallel retry + ln -sf "$(which python3)" /usr/bin/python + + - name: Setup Jupyter Kernel + run: | + python -m ipykernel install --user --name python3 --display-name "Python 3" + + - name: Execute notebooks + timeout-minutes: 40 + run: | + cd docs + make clean + make compile + + + notebook-finish: + needs: [ + run-all-notebooks + ] + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/experiment-runner.yml b/.github/workflows/experiment-runner.yml new file mode 100644 index 000000000..487ed9ba3 --- /dev/null +++ b/.github/workflows/experiment-runner.yml @@ -0,0 +1,30 @@ +name: Experiment Runner + +on: + workflow_dispatch: + inputs: + script: + description: "Experiment Runner Script" + default: "configs/sharegpt_config.yaml" + +concurrency: + group: experiment-runner-${{ github.ref }} + cancel-in-progress: true + +jobs: + experiment-runner-1-gpu: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Test experiment runner + timeout-minutes: 120 + run: | + cd test/srt + python3 experiment_runner.py --config ${{ inputs.script }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..3a281299a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,22 @@ +name: Lint + +on: [pull_request] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install pre-commit hook + run: | + python -m pip install pre-commit + pre-commit install + + - name: Linting + run: pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/nightly-test-amd.yml b/.github/workflows/nightly-test-amd.yml new file mode 100644 index 000000000..096e876de --- /dev/null +++ b/.github/workflows/nightly-test-amd.yml @@ -0,0 +1,41 @@ +name: Nightly Test (AMD) + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +concurrency: + group: nightly-test-amd-${{ github.ref }} + cancel-in-progress: true + +jobs: + nightly-test: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + strategy: + matrix: + runner: [linux-mi300-gpu-2, linux-mi325-gpu-2-nightly] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup docker + run: | + touch github_summary.md + bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Nightly Test + run: | + bash scripts/ci/amd_ci_exec.sh -e GITHUB_STEP_SUMMARY="/sglang-checkout/github_summary.md" python3 run_suite.py --suite nightly-amd --timeout-per-file 7200 + echo "$(> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/nightly-test.yml b/.github/workflows/nightly-test.yml new file mode 100644 index 000000000..a32c1dbea --- /dev/null +++ b/.github/workflows/nightly-test.yml @@ -0,0 +1,33 @@ +name: Nightly Test + +on: + schedule: + - cron: '0 0 * * *' + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +concurrency: + group: nightly-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + nightly-test: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 120 + run: | + cd test/srt + python3 run_suite.py --suite nightly --timeout-per-file 3600 diff --git a/.github/workflows/open-pr-copy-from-oss.yml b/.github/workflows/open-pr-copy-from-oss.yml new file mode 100644 index 000000000..05af6ea44 --- /dev/null +++ b/.github/workflows/open-pr-copy-from-oss.yml @@ -0,0 +1,28 @@ +name: Open A PR to Copy Code From OSS + +on: + workflow_dispatch: + # schedule: + # - cron: '0 10 * * *' + +permissions: + contents: write + +jobs: + copy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: 'main' + + - name: Install GitHub CLI (if not present) + run: | + bash scripts/code_sync/install_github_cli.sh + + - name: Copy from OSS code + env: + GH_TOKEN: ${{ secrets.PAT_FOR_CODE_SYNC_FROM_LIANMIN }} + run: | + python3 scripts/code_sync/copy_from_oss.py diff --git a/.github/workflows/open-pr-copy-to-oss.yml b/.github/workflows/open-pr-copy-to-oss.yml new file mode 100644 index 000000000..b3bb6aae4 --- /dev/null +++ b/.github/workflows/open-pr-copy-to-oss.yml @@ -0,0 +1,31 @@ +name: Open A PR to Copy Diff To OSS + +on: + workflow_dispatch: + inputs: + commit_sha: + description: 'The commit SHA to copy. Defaults to LAST to copy the latest commit.' + required: false + default: 'LAST' + +permissions: + contents: write + +jobs: + copy: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install GitHub CLI (if not present) + run: | + bash scripts/code_sync/install_github_cli.sh + + - name: Copy to OSS code + env: + GH_TOKEN: ${{ secrets.PAT_FOR_CODE_SYNC_FROM_LIANMIN }} + run: | + python3 scripts/code_sync/copy_to_oss.py --commit ${{ github.event.inputs.commit_sha }} diff --git a/.github/workflows/pr-benchmark-rust.yml b/.github/workflows/pr-benchmark-rust.yml new file mode 100644 index 000000000..e039cba23 --- /dev/null +++ b/.github/workflows/pr-benchmark-rust.yml @@ -0,0 +1,306 @@ +name: PR Benchmark (Rust Router) + +on: + push: + branches: [ main ] + paths: + - "sgl-router/**" + pull_request: + branches: [ main ] + paths: + - "sgl-router/**" + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + +concurrency: + group: pr-benchmark-rust-${{ github.ref }} + cancel-in-progress: true +permissions: + contents: read + pull-requests: write + issues: write +jobs: + # Quick check job that always runs on PRs + benchmark-compile-check: + name: Benchmark Compilation Check + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.3 + continue-on-error: true + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + # Save cache even on failure + save-if: true + + - name: Check benchmarks compile + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + cargo check --benches + + # Full benchmark jobs that only run with label or on main branch + benchmark-request-processing: + name: Request Processing Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'benchmark')) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + # Fetch enough history for baseline comparison + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.3 + continue-on-error: true + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + # Save cache even on failure + save-if: true + + - name: Run request processing benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + # Run only the summary benchmark for quick validation in PRs + cargo bench --bench request_processing -- benchmark_summary --exact + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: request-processing-results-${{ github.sha }} + path: | + sgl-router/target/criterion/benchmark_summary/ + retention-days: 30 + + benchmark-tokenizer: + name: Tokenizer Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'benchmark')) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.3 + continue-on-error: true + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + # Save cache even on failure + save-if: true + + - name: Run tokenizer benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + cargo bench --bench tokenizer_benchmark + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tokenizer-results-${{ github.sha }} + path: | + sgl-router/target/criterion/tokenizer*/ + retention-days: 30 + + benchmark-tool-parser: + name: Tool Parser Benchmark + if: | + github.repository == 'sgl-project/sglang' && + (github.event_name == 'push' || + github.event_name == 'workflow_dispatch' || + contains(github.event.pull_request.labels.*.name, 'benchmark')) + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.3 + continue-on-error: true + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + # Share cache across all benchmark jobs + shared-key: "rust-cache" + # Save cache even on failure + save-if: true + + - name: Run tool parser benchmark + timeout-minutes: 30 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Try to use sccache, but disable if it fails + if command -v sccache &> /dev/null; then + echo "Testing sccache availability..." + # Try to start sccache and check if it works + export RUSTC_WRAPPER=sccache + export SCCACHE_GHA_ENABLED="true" + if sccache --start-server 2>/dev/null && sccache --show-stats 2>/dev/null; then + echo "sccache is working, using it for compilation" + else + echo "sccache failed to start, falling back to regular cargo" + unset RUSTC_WRAPPER + unset SCCACHE_GHA_ENABLED + fi + else + echo "sccache not available, using regular cargo" + fi + cargo bench --bench tool_parser_benchmark + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: tool-parser-results-${{ github.sha }} + path: | + sgl-router/target/criterion/tool_parser*/ + retention-days: 30 + + benchmark-summary: + name: Benchmark Summary + needs: [benchmark-request-processing, benchmark-tokenizer, benchmark-tool-parser] + if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') + runs-on: ubuntu-latest + steps: + - name: Download all benchmark results + uses: actions/download-artifact@v4 + with: + pattern: '*-results-${{ github.sha }}' + path: benchmark-results + + - name: Generate summary + run: | + echo "## Benchmark Results Summary" > summary.md + echo "" >> summary.md + echo "### Request Processing" >> summary.md + if [ -d "benchmark-results/request-processing-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + echo "" >> summary.md + echo "### Tokenizer" >> summary.md + if [ -d "benchmark-results/tokenizer-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + echo "" >> summary.md + echo "### Tool Parser" >> summary.md + if [ -d "benchmark-results/tool-parser-results-${{ github.sha }}" ]; then + echo "✅ Completed" >> summary.md + else + echo "❌ Failed or skipped" >> summary.md + fi + cat summary.md + + - name: Upload summary + uses: actions/upload-artifact@v4 + with: + name: benchmark-summary-${{ github.sha }} + path: summary.md + retention-days: 30 diff --git a/.github/workflows/pr-test-amd.yml b/.github/workflows/pr-test-amd.yml new file mode 100644 index 000000000..2c7e2c652 --- /dev/null +++ b/.github/workflows/pr-test-amd.yml @@ -0,0 +1,377 @@ +name: PR Test (AMD) + +on: + push: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-amd.yml" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-amd.yml" + workflow_dispatch: + +concurrency: + group: pr-test-amd-${{ github.ref }} + cancel-in-progress: true + +jobs: + accuracy-test-1-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Evaluate Accuracy + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_eval_accuracy_large.py + bash scripts/ci/amd_ci_exec.sh python3 test_eval_fp8_accuracy.py + bash scripts/ci/amd_ci_exec.sh python3 models/test_qwen_models.py + + accuracy-test-2-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2, linux-mi325-gpu-2, linux-mi35x-gpu-2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Evaluate accuracy (TP=2) + timeout-minutes: 60 + run: | + bash scripts/ci/amd_ci_exec.sh -e SGLANG_USE_AITER=0 python3 test_moe_eval_accuracy_large.py + + mla-test-1-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: MLA TEST + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 test_mla.py + + performance-test-1-gpu-part-1-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark single latency + timeout-minutes: 20 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_small + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_default + + - name: Benchmark online latency + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default + + - name: Benchmark offline throughput + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default + + - name: Benchmark offline throughput (Non-streaming, small batch size) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_non_stream_small_batch_size + + performance-test-1-gpu-part-2-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark offline throughput (w/o RadixAttention) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache + + - name: Benchmark offline throughput (w/ Triton) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend + + - name: Benchmark offline throughput (w/ FP8) + timeout-minutes: 15 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default_fp8 + + bench-test-2-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2, linux-mi325-gpu-2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Benchmark dummy grok (TP=2) + timeout-minutes: 30 + run: | + bash scripts/ci/amd_ci_exec.sh python3 models/test_dummy_grok_models.py + + - name: Benchmark single latency (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_moe_tp2_bs1 + + - name: Benchmark single latency + torch.compile (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_torch_compile_tp2_bs1 + + - name: Benchmark offline throughput (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default + + - name: Benchmark offline throughput (w/o RadixAttention) (TP=2) + timeout-minutes: 25 + run: | + bash scripts/ci/amd_ci_exec.sh python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache + + unit-test-backend-1-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + part: [0, 1, 2, 3, 4, 5, 6, 7] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 50 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd --auto-partition-id ${{ matrix.part }} --auto-partition-size 8 + + unit-test-backend-1-gpu-amd-mi35x: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi35x-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 50 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-amd-mi35x + + unit-test-backend-2-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-2, linux-mi325-gpu-2] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 40 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-2-gpu-amd + + unit-test-backend-8-gpu-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-8] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 60 + run: | + bash scripts/ci/amd_ci_exec.sh python3 run_suite.py --suite per-commit-8-gpu-amd --timeout-per-file 3600 + + unit-test-sgl-kernel-amd: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runner: [linux-mi300-gpu-1, linux-mi325-gpu-1] + runs-on: ${{matrix.runner}} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Start CI container + run: bash scripts/ci/amd_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + - name: Install dependencies + run: | + bash scripts/ci/amd_ci_install_dependency.sh + + - name: Run test + timeout-minutes: 14 + run: | + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_align.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_moe_topk_softmax.py + docker exec -w /sglang-checkout/sgl-kernel/tests/speculative ci_sglang python3 -m pytest test_eagle_utils.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_apply_token_bitmask_inplace.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_activation.py + docker exec -w /sglang-checkout/sgl-kernel/tests ci_sglang python3 -m pytest test_kvcacheio.py + + pr-test-amd-finish: + if: always() + needs: [ + accuracy-test-1-gpu-amd, mla-test-1-gpu-amd, bench-test-2-gpu-amd, + accuracy-test-2-gpu-amd, performance-test-1-gpu-part-1-amd, performance-test-1-gpu-part-2-amd, + unit-test-backend-1-gpu-amd, unit-test-backend-1-gpu-amd-mi35x, unit-test-backend-2-gpu-amd, + unit-test-backend-8-gpu-amd, unit-test-sgl-kernel-amd + ] + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test-h20.yml b/.github/workflows/pr-test-h20.yml new file mode 100644 index 000000000..58e335289 --- /dev/null +++ b/.github/workflows/pr-test-h20.yml @@ -0,0 +1,81 @@ +name: PR Test (H20) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + inputs: + version: + required: true + type: choice + default: 'release' + options: + - 'release' + - 'nightly' + +concurrency: + group: pr-test-h20-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-changes: + runs-on: ubuntu-latest + outputs: + src: ${{ steps.filter.outputs.src }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + src: + - "python/sglang/srt/models/deepseek*" + - "python/sglang/srt/layers/moe/**" + - ".github/workflows/pr-test-h20.yml" + - "python/pyproject.toml" + + per-commit-8-gpu-h20: + needs: [check-changes] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 8-gpu-h20 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-h20 + + pr-test-h20-finish: + needs: [ + check-changes, + per-commit-8-gpu-h20, + ] + if: needs.check-changes.outputs.src == 'true' + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test-npu.yml b/.github/workflows/pr-test-npu.yml new file mode 100644 index 000000000..c0fe381e3 --- /dev/null +++ b/.github/workflows/pr-test-npu.yml @@ -0,0 +1,184 @@ +name: PR Test (Ascend NPU) + +on: + push: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - ".github/workflows/pr-test-npu.yml" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - ".github/workflows/pr-test-npu.yml" + workflow_dispatch: + +concurrency: + group: pr-test-npu-${{ github.ref }} + cancel-in-progress: true + +jobs: + per-commit-1-ascend-npu: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: linux-arm64-npu-1 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host ${CACHING_URL} + + bash scripts/ci/npu_ci_install_dependency.sh + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 60 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + run: | + cd test/srt + python3 run_suite.py --suite per-commit-1-ascend-npu + + per-commit-2-ascend-npu: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: linux-arm64-npu-2 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host ${CACHING_URL} + + bash scripts/ci/npu_ci_install_dependency.sh + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 90 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + run: | + cd test/srt + python3 run_suite.py --suite per-commit-2-ascend-npu + + per-commit-4-ascend-npu: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: linux-arm64-npu-4 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host ${CACHING_URL} + + bash scripts/ci/npu_ci_install_dependency.sh + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 120 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-ascend-npu --timeout-per-file 3600 + + per-commit-16-ascend-a3: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: linux-aarch64-a3-16 + container: + image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-a3-ubuntu22.04-py3.11 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + # speed up by using infra cache services + CACHING_URL="cache-service.nginx-pypi-cache.svc.cluster.local" + sed -Ei "s@(ports|archive).ubuntu.com@${CACHING_URL}:8081@g" /etc/apt/sources.list + pip config set global.index-url http://${CACHING_URL}/pypi/simple + pip config set global.trusted-host ${CACHING_URL} + + bash scripts/ci/npu_ci_install_dependency.sh + # copy required file from our daily cache + cp ~/.cache/modelscope/hub/datasets/otavia/ShareGPT_Vicuna_unfiltered/ShareGPT_V3_unfiltered_cleaned_split.json /tmp + # copy download through proxy + curl -o /tmp/test.jsonl -L https://gh-proxy.test.osinfra.cn/https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl + + - name: Run test + timeout-minutes: 90 + env: + SGLANG_USE_MODELSCOPE: true + SGLANG_IS_IN_CI: true + HF_ENDPOINT: https://hf-mirror.com + TORCH_EXTENSIONS_DIR: /tmp/torch_extensions + run: | + cd test/srt + python3 run_suite.py --suite per-commit-16-ascend-a3 --timeout-per-file 5400 + + pr-test-npu-finish: + if: always() + needs: + - per-commit-1-ascend-npu + - per-commit-2-ascend-npu + - per-commit-4-ascend-npu + - per-commit-16-ascend-a3 + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test-pd-router.yml b/.github/workflows/pr-test-pd-router.yml new file mode 100644 index 000000000..2a1bde1b4 --- /dev/null +++ b/.github/workflows/pr-test-pd-router.yml @@ -0,0 +1,599 @@ +name: PR Test (PD Router) + +on: + push: + branches: [ main ] + paths: + - 'python/sglang/srt/disaggregation/**' + - 'scripts/ci/ci_start_disaggregation_servers.sh' + - 'sgl-router/**' + pull_request: + branches: [ main ] + paths: + - 'python/sglang/srt/disaggregation/**' + - 'scripts/ci/ci_start_disaggregation_servers.sh' + - 'sgl-router/**' + workflow_dispatch: + +concurrency: + group: test-disaggregation-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + test-disaggregation: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: [h200] + timeout-minutes: 45 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 10 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Setup Rust + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Cache Rust dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + sgl-router/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('sgl-router/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('python/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Validate environment + run: | + echo "=== System Validation ===" + nvidia-smi + echo "GPU count: $(nvidia-smi -L | wc -l)" + if [ $(nvidia-smi -L | wc -l) -lt 8 ]; then + echo "Error: This test requires at least 8 GPUs" + exit 1 + fi + + echo "=== GPU Process Check ===" + # Fail fast if any GPU compute processes are active + if command -v nvidia-smi >/dev/null 2>&1; then + # Try to query compute apps first (preferred and concise) + gpu_procs=$(nvidia-smi --query-compute-apps=pid,process_name,gpu_uuid --format=csv,noheader 2>/dev/null | sed '/^$/d' || true) + + # Fallback to detailed PIDS report if the query returns nothing but there might still be processes + if [ -z "$gpu_procs" ]; then + gpu_procs=$(nvidia-smi -q -d PIDS 2>/dev/null | awk '/Processes/{flag=1;next}/^$/{flag=0}flag' | sed '/^\s*Processes:/d' | sed '/^\s*$/d' || true) + fi + + if [ -n "$gpu_procs" ]; then + echo "Error: Found active GPU processes using the device(s):" + echo "$gpu_procs" + exit 1 + else + echo "No active GPU compute processes detected." + fi + else + echo "Error: nvidia-smi not found; skipping GPU process check." + exit 1 + fi + + echo "=== RDMA Validation ===" + if ! command -v ibv_devices >/dev/null 2>&1; then + echo "Error: InfiniBand tools not found" + exit 1 + fi + + # Check for active IB devices + found_active_device=false + for device in mlx5_{0..11}; do + if ibv_devinfo $device >/dev/null 2>&1; then + state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}') + if [[ "$state" == "PORT_ACTIVE" ]]; then + echo "✓ Found active device: $device" + found_active_device=true + break + fi + fi + done + + if [ "$found_active_device" = false ]; then + echo "Error: No active IB devices found" + echo "Available devices:" + ibv_devices || true + exit 1 + fi + + echo "=== Model Validation ===" + if [ ! -d "/raid/models/meta-llama/Llama-3.1-8B-Instruct" ]; then + echo "Error: Model not found" + ls -la /raid/models/ || echo "No models directory" + exit 1 + fi + echo "✓ Model found" + + - name: Install SGLang dependencies + run: | + echo "Installing SGLang with all extras..." + python3 -m pip --no-cache-dir install --upgrade pip + python3 -m pip --no-cache-dir install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu126 + python3 -m pip --no-cache-dir install -e "python[all]" --break-system-packages + python3 -m pip --no-cache-dir install mooncake-transfer-engine==0.3.5 + python3 -m pip --no-cache-dir install --user --force-reinstall genai-bench==0.0.2 + python3 -m pip --no-cache-dir install sgl-kernel==0.3.9.post2 + + - name: Build and install sgl-router + run: | + source "$HOME/.cargo/env" + echo "Building sgl-router..." + cd sgl-router + cargo build && python3 -m build && pip install --force-reinstall dist/*.whl + + - name: Start disaggregation servers + id: start_servers + run: | + echo "Starting disaggregation servers..." + bash scripts/ci/ci_start_disaggregation_servers.sh & + SERVER_PID=$! + echo "server_pid=$SERVER_PID" >> $GITHUB_OUTPUT + + # Wait for all 8 servers to be healthy (script already does this) + wait_count=0 + while [ $wait_count -lt 30 ]; do + if ps -p $SERVER_PID > /dev/null; then + # Check if the startup script printed success message + sleep 2 + wait_count=$((wait_count + 1)) + else + # Script exited - check if it was successful + wait $SERVER_PID + exit_code=$? + if [ $exit_code -eq 0 ]; then + echo "✓ All disaggregation servers are healthy" + break + else + echo "Error: Server startup failed with code $exit_code" + exit 1 + fi + fi + done + + echo "✓ Servers started (PID: $SERVER_PID)" + + - name: Test all policies sequentially + timeout-minutes: 30 + run: | + POLICIES=("random" "round_robin" "cache_aware" "power_of_two") + BASE_URL="http://127.0.0.9:8000" + + # Free commonly used ports for router and metrics + echo "Freeing ports 29000 (metrics) and 8000 (API), if in use..." + fuser -k -n tcp 29000 2>/dev/null || true + fuser -k -n tcp 8000 2>/dev/null || true + sleep 1 + + for policy in "${POLICIES[@]}"; do + echo "" + echo "==================================================" + echo "Testing policy: $policy" + echo "==================================================" + + # Free ports before starting router + fuser -k -n tcp 29000 2>/dev/null || true + fuser -k -n tcp 8000 2>/dev/null || true + + # Start router with the current policy + echo "Starting router with policy: $policy..." + RUST_BACKTRACE=1 python3 -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy "$policy" \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --prefill http://127.0.0.3:30003 9003 \ + --prefill http://127.0.0.4:30004 9004 \ + --decode http://127.0.0.5:30005 \ + --decode http://127.0.0.6:30006 \ + --decode http://127.0.0.7:30007 \ + --decode http://127.0.0.8:30008 \ + --host 127.0.0.9 \ + --port 8000 & + ROUTER_PID=$! + + # Wait for router to become healthy + echo "Waiting for router to become healthy..." + TIMEOUT=60 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + if curl --connect-timeout 5 --silent http://127.0.0.9:8000 > /dev/null 2>&1; then + echo "✓ Router is reachable" + break + fi + if ! ps -p $ROUTER_PID > /dev/null; then + echo "Error: Router process died" + exit 1 + fi + sleep 5 + ELAPSED=$((ELAPSED + 5)) + done + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "Error: Router health check timeout" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Test API functionality + echo "Testing API completions for $policy..." + response=$(curl -s -X POST "$BASE_URL/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-token" \ + -d '{ + "model": "/raid/models/meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + {"role": "user", "content": "Write a Python function to calculate fibonacci numbers recursively"} + ], + "stream": false, + "max_tokens": 100 + }') + + if echo "$response" | jq -e '.choices[0].message.content' > /dev/null 2>&1; then + echo "✓ API test passed for $policy" + else + echo "✗ API test failed for $policy: $response" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Test streaming + echo "Testing streaming API for $policy..." + stream_response=$(timeout 30 curl -s -X POST "$BASE_URL/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer test-token" \ + -d '{ + "model": "/raid/models/meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + {"role": "user", "content": "Count from 1 to 5"} + ], + "stream": true, + "max_tokens": 50 + }') + + if echo "$stream_response" | grep -q "data:"; then + echo "✓ Streaming API test passed for $policy" + else + echo "✗ Streaming API test failed for $policy" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Run genai-bench benchmark + echo "Running genai-bench for $policy..." + genai-bench benchmark \ + --api-backend openai \ + --api-base "http://127.0.0.9:8000" \ + --api-key "dummy-token" \ + --api-model-name "/raid/models/meta-llama/Llama-3.1-8B-Instruct" \ + --model-tokenizer /raid/models/meta-llama/Llama-3.1-8B-Instruct \ + --task text-to-text \ + --num-concurrency 64 \ + --traffic-scenario "D(8000,2000)" \ + --max-requests-per-run 640 \ + --max-time-per-run 2 \ + --experiment-folder-name "benchmark_${policy}" \ + --experiment-base-dir "." + + # Find the actual experiment folder + actual_folder=$(find . -maxdepth 1 -name "benchmark_${policy}" -type d | head -1) + + if [ -n "$actual_folder" ]; then + # Extract metrics from the Excel summary or JSON files + summary_file="$actual_folder"/*_summary.xlsx + json_files=$(find "$actual_folder" -name "*.json" | grep -v experiment_metadata) + + echo "Genai-bench results saved in: $actual_folder" + + # Extract mean values and validate performance thresholds + echo "📊 Extracting performance metrics for $policy..." + + # Find JSON files excluding experiment metadata + json_files=$(find "$actual_folder" -name "*.json" | grep -v experiment_metadata) + + if [ -n "$json_files" ]; then + # Extract metrics using jq and validate against loose thresholds + for json_file in $json_files; do + echo "Processing: $(basename "$json_file")" + + # Extract mean values for performance validation + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean' "$json_file") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean' "$json_file") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean' "$json_file") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean' "$json_file") + + echo " TTFT mean: ${ttft_mean}s" + echo " E2E Latency mean: ${e2e_latency_mean}s" + echo " Input Throughput mean: ${input_throughput_mean} tokens/s" + echo " Output Throughput mean: ${output_throughput_mean} tokens/s" + + # Set mean thresholds (allowing for reasonable variance) + # These can be adjusted based on your performance requirements + ttft_threshold=4.7 # Max 4.7 seconds for mean TTFT + e2e_latency_threshold=35.0 # Max 35.0 seconds for mean E2E latency + input_throughput_threshold=12000 # Min 12000 tokens/s for mean input throughput + output_throughput_threshold=68 # Min 68 tokens/s for mean output throughput + + + # Validate mean thresholds + validation_passed=true + + if (( $(echo "$ttft_mean > $ttft_threshold" | bc -l) )); then + echo "❌ TTFT validation failed: $ttft_mean > $ttft_threshold" + validation_passed=false + fi + + if (( $(echo "$e2e_latency_mean > $e2e_latency_threshold" | bc -l) )); then + echo "❌ E2E Latency validation failed: $e2e_latency_mean > $e2e_latency_threshold" + validation_passed=false + fi + + if (( $(echo "$input_throughput_mean < $input_throughput_threshold" | bc -l) )); then + echo "❌ Input Throughput validation failed: $input_throughput_mean < $input_throughput_threshold" + validation_passed=false + fi + + if (( $(echo "$output_throughput_mean < $output_throughput_threshold" | bc -l) )); then + echo "❌ Output Throughput validation failed: $output_throughput_mean < $output_throughput_threshold" + validation_passed=false + fi + + if [ "$validation_passed" = true ]; then + echo "✅ Performance validation passed for $policy" + else + echo "❌ Performance validation failed for $policy" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + done + + echo "✓ Genai-bench completed successfully for $policy" + echo "📊 Detailed metrics and plots available in: $actual_folder" + else + echo "✗ Benchmark failed for $policy: No JSON results found" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + else + echo "✗ Benchmark failed for $policy: Experiment folder not found" + kill $ROUTER_PID 2>/dev/null || true + exit 1 + fi + + # Stop router before testing next policy + echo "Stopping router for $policy..." + # First try graceful shutdown + kill $ROUTER_PID 2>/dev/null || true + + # Wait up to 5 seconds for graceful shutdown + for i in {1..5}; do + if ! ps -p $ROUTER_PID > /dev/null 2>&1; then + echo "Router stopped gracefully" + break + fi + sleep 1 + done + + # Force kill if still running + if ps -p $ROUTER_PID > /dev/null 2>&1; then + echo "Force killing router..." + kill -9 $ROUTER_PID 2>/dev/null || true + fi + + # Short delay to ensure port is released + sleep 2 + + echo "✓ Completed testing for $policy" + done + + echo "" + echo "✅ All policies tested successfully!" + + + - name: Upload benchmark results + if: success() + uses: actions/upload-artifact@v4 + with: + name: genai-bench-results-all-policies + path: benchmark_**/ + + - name: Cleanup servers + if: always() + run: | + if [ -n "${{ steps.start_servers.outputs.server_pid }}" ]; then + pkill -P ${{ steps.start_servers.outputs.server_pid }} || true + kill ${{ steps.start_servers.outputs.server_pid }} || true + fi + pkill -f "sglang.launch_server" || true + sleep 5 + remaining=$(ps aux | grep -c "sglang.launch_server" || echo "0") + echo "Cleanup completed. Remaining processes: $remaining" + + summarize-benchmarks: + needs: test-disaggregation + runs-on: ubuntu-latest + if: success() + + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq bc + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: genai-bench-results-all-policies + + - name: List downloaded contents + run: | + echo "Contents after download:" + ls -la + find . -name "benchmark_*" -type d + echo "JSON files found:" + find . -name "*.json" | head -10 + + - name: Create benchmark summary + run: | + echo "=== DEBUG: Creating benchmark summary ===" + echo "Available benchmark directories:" + find . -name "benchmark_*" -type d + echo "==========================================" + + echo "## PD Router Genai-Bench Results Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "🚀 **Benchmarked with genai-bench for comprehensive LLM serving performance evaluation**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Policy | Status | TTFT (s) | E2E Latency (s) | Input Throughput (tok/s) | Output Throughput (tok/s) |" >> $GITHUB_STEP_SUMMARY + echo "|--------|--------|----------|-----------------|--------------------------|---------------------------|" >> $GITHUB_STEP_SUMMARY + + # First, complete the table with all policies + for policy in random round_robin cache_aware power_of_two; do + # Find genai-bench result folders for this policy (handle zip extraction structure) + result_folder=$(find . -maxdepth 2 -name "benchmark_${policy}" -type d | head -1) + if [ -z "$result_folder" ]; then + # Try alternative patterns in case of different extraction structure + result_folder=$(find . -maxdepth 3 -path "*benchmark_${policy}*" -type d | head -1) + fi + + echo "DEBUG: Policy ${policy} -> Found folder: ${result_folder:-'NOT FOUND'}" + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + # Find JSON file with metrics + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + # Extract performance metrics + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + + # Format numbers for display (2 decimal places) + if [ "$ttft_mean" != "N/A" ] && [ "$ttft_mean" != "null" ]; then + ttft_display=$(printf "%.2f" "$ttft_mean" 2>/dev/null || echo "$ttft_mean") + else + ttft_display="N/A" + fi + + if [ "$e2e_latency_mean" != "N/A" ] && [ "$e2e_latency_mean" != "null" ]; then + e2e_display=$(printf "%.2f" "$e2e_latency_mean" 2>/dev/null || echo "$e2e_latency_mean") + else + e2e_display="N/A" + fi + + if [ "$input_throughput_mean" != "N/A" ] && [ "$input_throughput_mean" != "null" ]; then + input_display=$(printf "%.0f" "$input_throughput_mean" 2>/dev/null || echo "$input_throughput_mean") + else + input_display="N/A" + fi + + if [ "$output_throughput_mean" != "N/A" ] && [ "$output_throughput_mean" != "null" ]; then + output_display=$(printf "%.0f" "$output_throughput_mean" 2>/dev/null || echo "$output_throughput_mean") + else + output_display="N/A" + fi + + echo "| ${policy} | ✅ Success | $ttft_display | $e2e_display | $input_display | $output_display |" >> $GITHUB_STEP_SUMMARY + else + echo "| ${policy} | ❌ No Data | N/A | N/A | N/A | N/A |" >> $GITHUB_STEP_SUMMARY + fi + else + echo "| ${policy} | ❌ Failed | N/A | N/A | N/A | N/A |" >> $GITHUB_STEP_SUMMARY + fi + done + + # Add performance validation summary + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📊 Performance Validation" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Thresholds:** TTFT ≤ 2.0s | E2E Latency ≤ 8.0s | Input Throughput ≥ 10,000 tok/s | Output Throughput ≥ 100 tok/s" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + validation_summary="" + for policy in random round_robin cache_aware power_of_two; do + # Use same robust path finding as above + result_folder=$(find . -maxdepth 2 -name "benchmark_${policy}" -type d | head -1) + if [ -z "$result_folder" ]; then + result_folder=$(find . -maxdepth 3 -path "*benchmark_${policy}*" -type d | head -1) + fi + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + # Extract metrics for validation + ttft=$(jq -r '.aggregated_metrics.stats.ttft.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + e2e_latency=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + input_throughput=$(jq -r '.aggregated_metrics.stats.input_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + output_throughput=$(jq -r '.aggregated_metrics.stats.output_throughput.mean // "N/A"' "$json_file" 2>/dev/null || echo "N/A") + + # Check thresholds (using same values as in main workflow) + validation_status="✅" + if [ "$ttft" != "N/A" ] && [ "$ttft" != "null" ]; then + if (( $(echo "$ttft > 2.0" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$e2e_latency" != "N/A" ] && [ "$e2e_latency" != "null" ]; then + if (( $(echo "$e2e_latency > 24.0" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$input_throughput" != "N/A" ] && [ "$input_throughput" != "null" ]; then + if (( $(echo "$input_throughput < 10000" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + if [ "$output_throughput" != "N/A" ] && [ "$output_throughput" != "null" ]; then + if (( $(echo "$output_throughput < 90" | bc -l 2>/dev/null || echo "0") )); then + validation_status="❌" + fi + fi + + validation_summary="${validation_summary}- **${policy}**: $validation_status\n" + else + validation_summary="${validation_summary}- **${policy}**: ❌ No data\n" + fi + else + validation_summary="${validation_summary}- **${policy}**: ❌ Failed\n" + fi + done + + echo -e "$validation_summary" >> $GITHUB_STEP_SUMMARY + + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📊 Genai-Bench Features Used" >> $GITHUB_STEP_SUMMARY + echo "- **Token-level Performance**: TTFT, TPOT, End-to-End latency" >> $GITHUB_STEP_SUMMARY + echo "- **Throughput Analysis**: Input/Output/Total token throughput" >> $GITHUB_STEP_SUMMARY + echo "- **Statistical Analysis**: Percentiles, mean, std dev for all metrics" >> $GITHUB_STEP_SUMMARY + echo "- **Visual Reports**: Automated plots and Excel summaries" >> $GITHUB_STEP_SUMMARY + echo "- **SGLang Backend**: Native integration with SGLang serving" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ All policies tested successfully with genai-bench!" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/pr-test-rust.yml b/.github/workflows/pr-test-rust.yml new file mode 100644 index 000000000..60c7ebdf2 --- /dev/null +++ b/.github/workflows/pr-test-rust.yml @@ -0,0 +1,190 @@ +name: PR Test (Rust) + +on: + push: + branches: [ main ] + paths: + - "sgl-router/**" + pull_request: + branches: [ main ] + paths: + - "sgl-router/**" + workflow_dispatch: + +concurrency: + group: pr-test-rust-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-test-rust: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: sgl-router + + - name: Run lint + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo clippy --all-targets --all-features -- -D warnings + + - name: Run fmt + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo fmt -- --check + + - name: Run Rust tests + timeout-minutes: 20 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo test + + - name: Check benchmark compilation + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + cargo check --benches + + - name: Quick benchmark sanity check + timeout-minutes: 15 + run: | + source "$HOME/.cargo/env" + cd sgl-router/ + # Run quick benchmarks to ensure they work using Python script + python3 scripts/run_benchmarks.py --quick + + pytest-rust: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: BM.A10.4 + timeout-minutes: 25 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install rust dependencies + run: | + bash scripts/ci/ci_install_rust.sh + + - name: Install SGLang dependencies + run: | + sudo bash scripts/ci/ci_install_dependency.sh + + - name: Build python binding + run: | + source "$HOME/.cargo/env" + cd sgl-router + pip install setuptools-rust wheel build + python3 -m build + pip install --force-reinstall dist/*.whl + + + - name: Run Python unit tests + run: | + cd sgl-router + source "$HOME/.cargo/env" + pip install pytest pytest-cov pytest-xdist + pytest -q py_test/unit --cov=sglang_router --cov-report=term-missing --cov-fail-under=80 + + - name: Run Python integration tests + run: | + cd sgl-router + source "$HOME/.cargo/env" + # Integration tests use FastAPI/uvicorn for mock workers + pip install fastapi uvicorn orjson + pytest -q -m integration + + - name: Run Python E2E tests + run: | + bash scripts/killall_sglang.sh "nuk_gpus" + cd sgl-router + python3 -m pip --no-cache-dir install --upgrade --ignore-installed blinker + python3 -m pip --no-cache-dir install --upgrade --break-system-packages genai-bench==0.0.2 + pytest -m e2e -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Upload benchmark results + if: success() + uses: actions/upload-artifact@v4 + with: + name: genai-bench-results-all-policies + path: sgl-router/benchmark_**/ + + finish: + needs: [unit-test-rust, pytest-rust] + runs-on: ubuntu-latest + steps: + - name: Finish + run: echo "This is an empty step to ensure that all jobs are completed." + + summarize-benchmarks: + needs: pytest-rust + runs-on: ubuntu-latest + if: success() + + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq bc + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: genai-bench-results-all-policies + + - name: List downloaded contents + run: | + echo "Contents after download:" + ls -la + find . -name "benchmark_*" -type d + echo "JSON files found:" + find . -name "*.json" | head -10 + + - name: Create benchmark summary + run: | + echo "=== DEBUG: Creating benchmark summary ===" + echo "Available benchmark directories:" + find . -name "benchmark_*" -type d || true + echo "==========================================" + + echo "## Router E2E Genai-Bench Results Summary" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Results captured from E2E tests for two scenarios: regular router (2 workers, dp=2) and PD router (2 prefill + 2 decode)." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| Scenario | Status | TTFT (s) | E2E Latency (s) | Input Throughput (tok/s) | Output Throughput (tok/s) |" >> $GITHUB_STEP_SUMMARY + echo "|----------|--------|----------|-----------------|--------------------------|---------------------------|" >> $GITHUB_STEP_SUMMARY + + scenarios=$'Regular (dp=2, round_robin)|benchmark_round_robin_regular\nPD (2 prefill + 2 decode, round_robin)|benchmark_round_robin_pd' + + echo "$scenarios" | sed 's/^\s*//' | while IFS='|' read -r label pattern; do + [ -z "$label" ] && continue + # Find the result folder (handle different extraction layouts) + result_folder=$(find . -maxdepth 3 \( -name "$pattern" -o -path "*${pattern}*" \) -type d | head -1) + + if [ -n "$result_folder" ] && [ -d "$result_folder" ]; then + json_file=$(find "$result_folder" -name "*.json" -not -name "experiment_metadata.json" | head -1) + + if [ -n "$json_file" ] && [ -f "$json_file" ]; then + ttft_mean=$(jq -r '.aggregated_metrics.stats.ttft.mean' "$json_file") + e2e_latency_mean=$(jq -r '.aggregated_metrics.stats.e2e_latency.mean' "$json_file") + input_throughput_mean=$(jq -r '.aggregated_metrics.stats.input_throughput.mean' "$json_file") + output_throughput_mean=$(jq -r '.aggregated_metrics.stats.output_throughput.mean' "$json_file") + + ttft_display=$(printf "%.2f" "$ttft_mean" 2>/dev/null || echo "$ttft_mean") + e2e_display=$(printf "%.2f" "$e2e_latency_mean" 2>/dev/null || echo "$e2e_latency_mean") + input_display=$(printf "%.0f" "$input_throughput_mean" 2>/dev/null || echo "$input_throughput_mean") + output_display=$(printf "%.0f" "$output_throughput_mean" 2>/dev/null || echo "$output_throughput_mean") + + echo "| ${label} | ✅ Success | $ttft_display | $e2e_display | $input_display | $output_display |" >> $GITHUB_STEP_SUMMARY + fi + fi + done diff --git a/.github/workflows/pr-test-sgl-kernel.yml b/.github/workflows/pr-test-sgl-kernel.yml new file mode 100644 index 000000000..832188cdd --- /dev/null +++ b/.github/workflows/pr-test-sgl-kernel.yml @@ -0,0 +1,151 @@ +name: PR Test (sgl-kernel) + +on: + push: + branches: [main] + paths: + - "sgl-kernel/**" + pull_request: + branches: [main] + paths: + - "sgl-kernel/**" + workflow_dispatch: + +concurrency: + group: pr-test-sgl-kernel-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check clang-format + uses: DoozyX/clang-format-lint-action@v0.18.1 + with: + source: sgl-kernel + extensions: h,c,cpp,hpp,cu,cuh,cc + clangFormatVersion: 18 + style: file + + build-wheels: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + runs-on: sgl-kernel-build-node + strategy: + matrix: + include: + - python-version: "3.10" + cuda-version: "12.4" + - python-version: "3.10" + cuda-version: "12.8" + - python-version: "3.10" + cuda-version: "12.9" + name: Build Wheel (CUDA ${{ matrix.cuda-version }}) + steps: + - name: Cleanup + run: | + sudo rm -rf $GITHUB_WORKSPACE/* || true + + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheel for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }} + if: github.event_name != 'push' || (matrix.cuda-version != '12.4' && matrix.cuda-version != '12.8') + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }} + path: sgl-kernel/dist/* + + unit-test: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + needs: build-wheels + runs-on: 1-gpu-runner + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install + run: | + bash scripts/ci/ci_install_dependency.sh + pip3 install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu126 && pip3 install pytest + pip3 uninstall sgl-kernel -y || true + pip3 install sgl-kernel/dist/*whl --force-reinstall --no-deps + pip3 list | grep sgl-kernel + + - name: Run test + timeout-minutes: 30 + run: | + cd sgl-kernel + pytest tests/ + + - name: Uninstall dependencies + run: | + pip3 uninstall sgl-kernel -y + + mla-test: + if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request' + needs: build-wheels + runs-on: 1-gpu-runner + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-python3.10-cuda12.9 + + - name: Install + run: | + bash scripts/ci/ci_install_dependency.sh + pip3 install torch==2.8.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/test/cu126 + pip3 uninstall sgl-kernel -y || true + pip3 install sgl-kernel/dist/*whl --force-reinstall --no-deps + pip3 list | grep sgl-kernel + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 test_mla_deepseek_v3.py + + - name: Uninstall dependencies + run: | + pip3 uninstall sgl-kernel -y + + finish: + needs: [unit-test, mla-test, lint, build-wheels] + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test-xeon.yml b/.github/workflows/pr-test-xeon.yml new file mode 100644 index 000000000..fcc70f286 --- /dev/null +++ b/.github/workflows/pr-test-xeon.yml @@ -0,0 +1,106 @@ +name: PR Test (Xeon) + +on: + push: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-xeon.yml" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + - "sgl-kernel/**" + - ".github/workflows/pr-test-xeon.yml" + workflow_dispatch: + +concurrency: + group: pr-test-xeon-${{ github.ref }} + cancel-in-progress: false + +jobs: + build-test: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: xeon-gnr + env: + HF_HOME: /home/sdp/.cache/huggingface + strategy: + matrix: + build_type: ['all'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-xeon + + docker build . -f docker/Dockerfile.xeon -t sglang_xeon --no-cache + + - name: Run container + run: | + docker run -dt \ + -v ${{ github.workspace }}:/sglang-checkout/ --ipc=host \ + -v ${HF_HOME}:/root/.cache/huggingface \ + --name ci_sglang_xeon \ + sglang_xeon + + - name: Install dependencies + timeout-minutes: 20 + run: | + docker exec ci_sglang_xeon bash -c "python3 -m pip install --upgrade pip" + docker exec ci_sglang_xeon pip uninstall sgl-kernel -y || true + docker exec -w /sglang-checkout/sgl-kernel ci_sglang_xeon bash -c "cp pyproject_cpu.toml pyproject.toml && pip install -v ." + docker exec -w /sglang-checkout/ ci_sglang_xeon bash -c "pip install -e "python[dev_cpu]"" + + - name: Check AMX support + id: check_amx + timeout-minutes: 5 + run: | + docker exec -w /sglang-checkout/ ci_sglang_xeon \ + bash -c "python3 -c 'import torch; import sgl_kernel; assert torch._C._cpu._is_amx_tile_supported(); assert hasattr(torch.ops.sgl_kernel, \"convert_weight_packed\"); '" + continue-on-error: true + + - name: Run unit tests + if: steps.check_amx.outcome == 'success' + timeout-minutes: 36 + run: | + docker exec -w /sglang-checkout/ ci_sglang_xeon \ + bash -c "cd ./test/srt && python3 run_suite.py --suite per-commit-cpu" + + - name: Change permission + timeout-minutes: 2 + run: | + docker exec -u root ci_sglang_xeon bash -c " + rm -rf /tmp/ci-home && + chown -R $(id -u):$(id -g) /sglang-checkout/ 2>/dev/null || true + " + + - name: Cleanup container + if: always() + run: | + docker rm -f ci_sglang_xeon || true + + pr-test-xeon-finish: + if: always() + needs: [build-test] + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml new file mode 100644 index 000000000..bd1053902 --- /dev/null +++ b/.github/workflows/pr-test.yml @@ -0,0 +1,437 @@ +name: PR Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + inputs: + version: + description: "FlashInfer version" + required: true + type: choice + default: 'release' + options: + - 'release' + - 'nightly' + +concurrency: + group: pr-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + check-changes: + runs-on: ubuntu-latest + outputs: + src: ${{ steps.filter.outputs.src }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Detect file changes + id: filter + uses: dorny/paths-filter@v3 + with: + filters: | + src: + - "python/**" + - "scripts/ci/**" + - "test/**" + - ".github/workflows/pr-test.yml" + + unit-test-frontend: + needs: check-changes + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 10 + run: | + cd test/lang + python3 run_suite.py --suite per-commit + + unit-test-backend-1-gpu: + needs: [check-changes, unit-test-frontend] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 1-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite per-commit --auto-partition-id ${{ matrix.part }} --auto-partition-size 10 + + unit-test-backend-2-gpu: + needs: [check-changes] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 2-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 30 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-2-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + unit-test-backend-4-gpu: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 4-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + unit-test-backend-8-gpu: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 8-gpu-runner + strategy: + fail-fast: false + matrix: + part: [0, 1] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu --auto-partition-id ${{ matrix.part }} --auto-partition-size 2 + + performance-test-1-gpu-part-1: + needs: check-changes + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark single latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_small + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_bs1_default + + - name: Benchmark online latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_default + + - name: Benchmark offline throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default + + - name: Benchmark offline throughput (Non-streaming, small batch size) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_non_stream_small_batch_size + + - name: Benchmark online latency (EAGLE) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_online_latency_eagle + + - name: Benchmark online latency (LoRA) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency + python3 -m unittest test_bench_serving.TestBenchServing.test_lora_online_latency_with_concurrent_adapter_updates + + performance-test-1-gpu-part-2: + needs: check-changes + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark offline throughput (w/o RadixAttention) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_without_radix_cache + + - name: Benchmark offline throughput (w/ Triton) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_with_triton_attention_backend + + - name: Benchmark offline throughput (w/ FP8) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_offline_throughput_default_fp8 + + - name: Benchmark VLM offline throughput + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_offline_throughput + + - name: Benchmark VLM online latency + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_vlm_online_latency + + performance-test-2-gpu: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + + - name: Benchmark single latency (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_moe_tp2_bs1 + + - name: Benchmark single latency + torch.compile (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_one_batch.TestBenchOneBatch.test_torch_compile_tp2_bs1 + + - name: Benchmark offline throughput (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_default + + - name: Benchmark offline throughput (w/o RadixAttention) (TP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_moe_offline_throughput_without_radix_cache + + - name: Benchmark offline PP decode throughput (PP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_pp_offline_throughput_default_decode + + - name: Benchmark offline PP prefill throughput (PP=2) + timeout-minutes: 10 + run: | + cd test/srt + python3 -m unittest test_bench_serving.TestBenchServing.test_pp_long_context_prefill + + accuracy-test-1-gpu: + needs: check-changes + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + git clone https://github.com/merrymercy/human-eval.git + cd human-eval + pip install -e . + + - name: Evaluate accuracy + timeout-minutes: 20 + run: | + cd test/srt + python3 test_eval_accuracy_large.py + + accuracy-test-2-gpu: + needs: [check-changes, accuracy-test-1-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 2-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + git clone https://github.com/merrymercy/human-eval.git + cd human-eval + pip install -e . + + - name: Evaluate accuracy (TP=2) + timeout-minutes: 20 + run: | + cd test/srt + python3 test_moe_eval_accuracy_large.py + + unit-test-deepep-4-gpu: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 4-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-4-gpu-deepep + + unit-test-deepep-8-gpu: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: 8-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_deepep.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-deepep + + unit-test-backend-8-gpu-b200: + needs: [check-changes, unit-test-backend-2-gpu] + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false && + needs.check-changes.outputs.src == 'true' + runs-on: b200-runner + strategy: + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + IS_BLACKWELL=1 bash scripts/ci/ci_install_dependency.sh + + - name: Run test + timeout-minutes: 20 + run: | + cd test/srt + python3 run_suite.py --suite per-commit-8-gpu-b200 --auto-partition-id 0 --auto-partition-size 1 + + + pr-test-finish: + needs: [ + check-changes, + unit-test-frontend, unit-test-backend-1-gpu, + unit-test-backend-2-gpu, unit-test-backend-4-gpu, unit-test-backend-8-gpu, + performance-test-1-gpu-part-1, performance-test-1-gpu-part-2, performance-test-2-gpu, + accuracy-test-1-gpu, accuracy-test-2-gpu, + unit-test-deepep-4-gpu, unit-test-deepep-8-gpu, + unit-test-backend-8-gpu-b200, + ] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check all dependent job statuses + run: | + results=(${{ join(needs.*.result, ' ') }}) + for result in "${results[@]}"; do + if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then + echo "Job failed with result: $result" + exit 1 + fi + done + echo "All jobs completed successfully" + exit 0 diff --git a/.github/workflows/release-docker-amd-nightly.yml b/.github/workflows/release-docker-amd-nightly.yml new file mode 100644 index 000000000..c61e200df --- /dev/null +++ b/.github/workflows/release-docker-amd-nightly.yml @@ -0,0 +1,65 @@ +name: Release Docker Images Nightly (AMD) +on: + workflow_dispatch: + schedule: + - cron: '0 13 * * *' + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: amd-docker-scale + environment: 'prod' + strategy: + matrix: + gpu_arch: ['gfx942', 'gfx942-rocm700', 'gfx950'] + build_type: ['all', 'srt'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: "Set Date" + run: | + echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_AMD_USERNAME }} + password: ${{ secrets.DOCKERHUB_AMD_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + + if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + rocm_tag="rocm630-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm700" ]; then + rocm_tag="rocm700-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then + rocm_tag="rocm700-mi35x" + else + echo "Unsupported gfx arch" + exit 1 + fi + + tag=v${version}-${rocm_tag} + + if [ "${{ matrix.build_type }}" = "all" ]; then + tag_suffix="" + elif [ "${{ matrix.build_type }}" = "srt" ]; then + tag_suffix="-srt" + else + echo "Unsupported build type" + exit 1 + fi + + docker build . -f docker/Dockerfile.rocm --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} -t rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix} --no-cache + docker push rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix} diff --git a/.github/workflows/release-docker-amd.yml b/.github/workflows/release-docker-amd.yml new file mode 100644 index 000000000..98c11e2fa --- /dev/null +++ b/.github/workflows/release-docker-amd.yml @@ -0,0 +1,56 @@ +name: Release Docker Images (AMD) +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: amd-docker-scale + environment: 'prod' + strategy: + matrix: + gpu_arch: ['gfx942', 'gfx942-rocm700', 'gfx950'] + build_type: ['all', 'srt'] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + + if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then + rocm_tag="rocm630-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx942-rocm700" ]; then + rocm_tag="rocm700-mi30x" + elif [ "${{ matrix.gpu_arch }}" = "gfx950" ]; then + rocm_tag="rocm700-mi35x" + else + echo "Unsupported gfx arch" + exit 1 + fi + + tag=v${version}-${rocm_tag} + + if [ "${{ matrix.build_type }}" = "all" ]; then + tag_suffix="" + elif [ "${{ matrix.build_type }}" = "srt" ]; then + tag_suffix="-srt" + else + echo "Unsupported build type" + exit 1 + fi + + docker build . -f docker/Dockerfile.rocm --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} -t lmsysorg/sglang:${tag}${tag_suffix} --no-cache + docker push lmsysorg/sglang:${tag}${tag_suffix} diff --git a/.github/workflows/release-docker-dev.yml b/.github/workflows/release-docker-dev.yml new file mode 100644 index 000000000..38e2e790f --- /dev/null +++ b/.github/workflows/release-docker-dev.yml @@ -0,0 +1,49 @@ +name: Build Development Docker Image + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + build-dev: + if: ${{ github.repository == 'sgl-project/sglang' }} + runs-on: ubuntu-22.04 + strategy: + matrix: + variant: + - version: 12.6.1 + type: all + tag: dev + - version: 12.8.1 + type: blackwell + tag: blackwell + - version: 12.9.1 + type: blackwell + tag: b200-cu129 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + docker-images: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: false + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push Dev Image + run: | + docker buildx build --output type=image,compression=zstd . -f docker/Dockerfile --build-arg CUDA_VERSION=${{ matrix.variant.version }} --build-arg BUILD_TYPE=${{ matrix.variant.type }} --build-arg CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) -t lmsysorg/sglang:${{ matrix.variant.tag }} --no-cache + docker push lmsysorg/sglang:${{ matrix.variant.tag }} diff --git a/.github/workflows/release-docker-gb200.yml b/.github/workflows/release-docker-gb200.yml new file mode 100644 index 000000000..fbcacb330 --- /dev/null +++ b/.github/workflows/release-docker-gb200.yml @@ -0,0 +1,36 @@ +name: Release Docker Images (GB200) +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-22.04-arm + environment: "prod" + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-cu129-gb200 + + docker buildx build --platform linux/arm64 --push --output type=image -t lmsysorg/sglang:${tag} -f docker/Dockerfile.gb200 --build-arg CUDA_VERSION=12.9.1 --build-arg BUILD_TYPE=blackwell --no-cache . diff --git a/.github/workflows/release-docker-npu-nightly.yml b/.github/workflows/release-docker-npu-nightly.yml new file mode 100644 index 000000000..527a0cdc2 --- /dev/null +++ b/.github/workflows/release-docker-npu-nightly.yml @@ -0,0 +1,78 @@ +name: Release Docker Images Nightly (Ascend NPU) +on: + pull_request: + branches: + - main + paths: + - ".github/workflows/release-docker-npu-nightly.yml" + - "docker/Dockerfile.npu" + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-22.04-arm + strategy: + matrix: + cann_version: ["8.2.rc1"] + device_type: ["910b", "a3"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + docker-images: false + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + lmsysorg/sglang + # push with schedule event + # push with workflow_dispatch event + tags: | + type=ref,event=pr + type=ref,event=branch + type=schedule,pattern=main + flavor: | + latest=false + suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }},onlatest=true + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into docker hub + uses: docker/login-action@v3 + if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: docker + file: docker/Dockerfile.npu + # TODO: need add x86 platforms support when memfabric is ready + platforms: linux/arm64 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags }} + push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + provenance: false + build-args: | + SGLANG_KERNEL_NPU_TAG=20250901 + CANN_VERSION=${{ matrix.cann_version }} + DEVICE_TYPE=${{ matrix.device_type }} diff --git a/.github/workflows/release-docker-npu.yml b/.github/workflows/release-docker-npu.yml new file mode 100644 index 000000000..f9d52eb4b --- /dev/null +++ b/.github/workflows/release-docker-npu.yml @@ -0,0 +1,74 @@ +name: Release Docker Images (Ascend NPU) +on: + push: + tags: + - "*" # Trigger on all tags and filterred by pep440 later + workflow_dispatch: + pull_request: + branches: + - main + paths: + - ".github/workflows/release-docker-npu.yml" + - "docker/Dockerfile.npu" + +jobs: + build: + runs-on: ubuntu-22.04-arm + strategy: + matrix: + cann_version: ["8.2.rc1"] + device_type: ["910b", "a3"] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + tool-cache: true + docker-images: false + + # push with tag + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + lmsysorg/sglang + tags: | + type=ref,event=pr + type=ref,event=tag,suffix=-cann${{ matrix.cann_version }}-${{ matrix.device_type }} + flavor: | + latest=false + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Login to Docker Hub + uses: docker/login-action@v2 + if: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Get version + id: get_version + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "TAG=lmsysorg/sglang:v$version-cann${{ matrix.cann_version }}-${{ matrix.device_type }}" >> $GITHUB_OUTPUT + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v6 + with: + context: docker + file: docker/Dockerfile.npu + # TODO: need add x86 platforms support when memfabric is ready + platforms: linux/arm64 + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.outputs.tags || steps.get_version.outputs.TAG }} + push: ${{ github.repository == 'sgl-project/sglang' && github.event_name != 'pull_request' }} + provenance: false + build-args: | + SGLANG_KERNEL_NPU_TAG=20250901 + CANN_VERSION=${{ matrix.cann_version }} + DEVICE_TYPE=${{ matrix.device_type }} diff --git a/.github/workflows/release-docker-router.yml b/.github/workflows/release-docker-router.yml new file mode 100644 index 000000000..f98651e8a --- /dev/null +++ b/.github/workflows/release-docker-router.yml @@ -0,0 +1,30 @@ +name: Release SGLang Router Docker Image +on: + push: + branches: + - main + paths: + - "sgl-router/py_src/sglang_router/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat sgl-router/py_src/sglang_router/version.py | cut -d'"' -f2) + tag=v${version} + + docker build . -f docker/Dockerfile.router -t lmsysorg/sglang-router:${tag} --no-cache + docker push lmsysorg/sglang-router:${tag} diff --git a/.github/workflows/release-docker-xeon.yml b/.github/workflows/release-docker-xeon.yml new file mode 100644 index 000000000..bd2a3910f --- /dev/null +++ b/.github/workflows/release-docker-xeon.yml @@ -0,0 +1,35 @@ +name: Release Docker Xeon Images +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-24.04 + environment: 'prod' + strategy: + matrix: + build_type: ['all'] + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + tag=v${version}-xeon + + docker build . -f docker/Dockerfile.xeon -t lmsysorg/sglang:${tag} --no-cache + docker push lmsysorg/sglang:${tag} diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml new file mode 100644 index 000000000..60a8df621 --- /dev/null +++ b/.github/workflows/release-docker.yml @@ -0,0 +1,97 @@ +name: Release Docker Images +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + environment: 'prod' + strategy: + matrix: + cuda_version: ['12.6.1', '12.8.1', '12.9.1'] + build_type: ['all', 'blackwell'] + exclude: + - cuda_version: '12.6.1' + build_type: 'blackwell' + - cuda_version: '12.8.1' + build_type: 'all' + - cuda_version: '12.9.1' + build_type: 'all' + steps: + - name: Delete huge unnecessary tools folder + run: rm -rf /opt/hostedtoolcache + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + docker-images: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: false + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and Push + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + + if [ "${{ matrix.cuda_version }}" = "11.8.0" ]; then + cuda_tag="cu118" + elif [ "${{ matrix.cuda_version }}" = "12.1.1" ]; then + cuda_tag="cu121" + elif [ "${{ matrix.cuda_version }}" = "12.4.1" ]; then + cuda_tag="cu124" + elif [ "${{ matrix.cuda_version }}" = "12.5.1" ]; then + cuda_tag="cu125" + elif [ "${{ matrix.cuda_version }}" = "12.6.1" ]; then + cuda_tag="cu126" + elif [ "${{ matrix.cuda_version }}" = "12.8.1" ]; then + cuda_tag="cu128" + elif [ "${{ matrix.cuda_version }}" = "12.9.1" ]; then + cuda_tag="cu129" + else + echo "Unsupported CUDA version" + exit 1 + fi + + tag=v${version}-${cuda_tag} + + if [ "${{ matrix.build_type }}" = "all" ]; then + tag_suffix="" + elif [ "${{ matrix.build_type }}" = "srt" ]; then + tag_suffix="-srt" + elif [ "${{ matrix.build_type }}" = "blackwell" ]; then + tag_suffix="-b200" + else + echo "Unsupported build type" + exit 1 + fi + + docker buildx build --output type=image,compression=zstd . -f docker/Dockerfile --build-arg CUDA_VERSION=${{ matrix.cuda_version }} --build-arg BUILD_TYPE=${{ matrix.build_type }} -t lmsysorg/sglang:${tag}${tag_suffix} --no-cache + docker push lmsysorg/sglang:${tag}${tag_suffix} + + if [ "${{ matrix.cuda_version }}" = "12.6.1" ]; then + docker tag lmsysorg/sglang:${tag}${tag_suffix} lmsysorg/sglang:latest${tag_suffix} + docker push lmsysorg/sglang:latest${tag_suffix} + fi + + if [ "${{ matrix.cuda_version }}" = "12.9.1" ]; then + docker tag lmsysorg/sglang:${tag}${tag_suffix} lmsysorg/sglang:v${version} + docker push lmsysorg/sglang:v${version} + fi diff --git a/.github/workflows/release-docs.yml b/.github/workflows/release-docs.yml new file mode 100644 index 000000000..0e09eec93 --- /dev/null +++ b/.github/workflows/release-docs.yml @@ -0,0 +1,65 @@ +name: Release Documentation + +on: + push: + branches: + - main + paths: + - "docs/**" + - "python/sglang/version.py" + - "python/sglang/**" + workflow_dispatch: + +concurrency: + group: release-docs-${{ github.ref }} + cancel-in-progress: true + +jobs: + execute-and-deploy: + runs-on: 1-gpu-runner + if: github.repository == 'sgl-project/sglang' + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + pip install -r docs/requirements.txt + apt-get update && apt-get install -y pandoc parallel retry + ln -sf "$(which python3)" /usr/bin/python + + - name: Setup Jupyter Kernel + run: | + python -m ipykernel install --user --name python3 --display-name "Python 3" + + - name: Execute notebooks + timeout-minutes: 40 + run: | + cd docs + make clean + make compile + + - name: Push HTML to sgl-project.github.io + timeout-minutes: 60 + env: + GITHUB_TOKEN: ${{ secrets.DOCUMENTATION_PAT_TOKEN }} + run: | + cd docs + make html + python3 wrap_run_llm.py + + cd _build/html + + git clone https://$GITHUB_TOKEN@github.com/sgl-project/sgl-project.github.io.git ../sgl-project.github.io --depth 1 + find ../sgl-project.github.io/ -mindepth 1 -not -path "../sgl-project.github.io/.git*" -not -name CNAME -not -name ".jekyll" -not -name ".nojekyll" -delete + cp -r * ../sgl-project.github.io + cp ../../README.md ../sgl-project.github.io/README.md + cd ../sgl-project.github.io + git config user.name "zhaochenyang20" + git config user.email "zhaochenyang20@gmail.com" + git add . + git commit -m "Update $(date +'%Y-%m-%d %H:%M:%S')" + git push https://$GITHUB_TOKEN@github.com/sgl-project/sgl-project.github.io.git main + cd .. + rm -rf sgl-project.github.io diff --git a/.github/workflows/release-fake-tag.yml b/.github/workflows/release-fake-tag.yml new file mode 100644 index 000000000..ce5999506 --- /dev/null +++ b/.github/workflows/release-fake-tag.yml @@ -0,0 +1,35 @@ +name: Release Fake Tag +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +permissions: + contents: write + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + environment: 'prod' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get version + id: get_version + run: | + version=$(cat python/sglang/version.py | cut -d'"' -f2) + echo "TAG=v$version" >> $GITHUB_OUTPUT + + - name: Create and push fake tag + env: + GITHUB_TOKEN: ${{ secrets.REPO_TOKEN }} + run: | + git config user.name zhyncs + git config user.email me@zhyncs.com + git checkout -b ${{ steps.get_version.outputs.TAG }} + git push --set-upstream origin ${{ steps.get_version.outputs.TAG }} diff --git a/.github/workflows/release-pypi-router.yml b/.github/workflows/release-pypi-router.yml new file mode 100644 index 000000000..a2128be83 --- /dev/null +++ b/.github/workflows/release-pypi-router.yml @@ -0,0 +1,119 @@ +# Reference: https://github.com/openai/tiktoken/blob/63527649963def8c759b0f91f2eb69a40934e468/.github/workflows/build_wheels.yml#L1 + +name: Release SGLang Router to PyPI + +on: + push: + branches: + - main + paths: + - sgl-router/pyproject.toml + workflow_dispatch: + +jobs: + build: + name: Build on ${{ matrix.os }} (${{ matrix.target }}) + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu + target: x86_64 + + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + rm -rf sglang-repo + ls -alt + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install build dependencies + run: | + python -m pip install -U pip + python -m pip install build twine auditwheel + + - name: Build package + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD: "cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64" + CIBW_BEFORE_ALL: | + yum update -y && yum install -y openssl-devel wget unzip && \ + # Install latest protoc (v32.0) that supports proto3 + cd /tmp && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip && \ + unzip protoc-32.0-linux-x86_64.zip -d /usr/local && \ + rm protoc-32.0-linux-x86_64.zip && \ + # Install Rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + CIBW_ENVIRONMENT: "PATH=$HOME/.cargo/bin:$PATH" + + - name: List built packages + run: ls -lh wheelhouse/ + + - name: Check packages + run: twine check --strict wheelhouse/* + + - uses: actions/upload-artifact@v4 + with: + name: packages-${{ matrix.os }}-${{ matrix.target }} + path: wheelhouse/ + + build-sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: sglang-repo + + - name: Move sgl-router folder to root, copy the license file, and delete sglang-repo + run: | + mv sglang-repo/sgl-router/* . + mv sglang-repo/LICENSE . + rm -rf sglang-repo + ls -alt + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Build SDist + run: | + pip install build + python -m pip install -U packaging + python -m build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + upload: + name: Upload to PyPI + if: github.repository == 'sgl-project/sglang' # Ensure this job only runs for the sgl-project/sglang repository + needs: [build, build-sdist] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Upload to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN_ROUTER }} + run: | + pip install twine + twine upload dist/* --verbose diff --git a/.github/workflows/release-pypi.yml b/.github/workflows/release-pypi.yml new file mode 100644 index 000000000..354ee21d1 --- /dev/null +++ b/.github/workflows/release-pypi.yml @@ -0,0 +1,31 @@ +name: Release PyPI +on: + push: + branches: + - main + paths: + - "python/sglang/version.py" + workflow_dispatch: + +jobs: + publish: + if: github.repository == 'sgl-project/sglang' + runs-on: ubuntu-latest + environment: "prod" + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Upload to pypi + run: | + cd python + cp ../README.md ../LICENSE . + pip install build + python3 -m build + pip install twine + python3 -m twine upload dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/release-whl-kernel-cu118.yml b/.github/workflows/release-whl-kernel-cu118.yml new file mode 100644 index 000000000..4757bcaa1 --- /dev/null +++ b/.github/workflows/release-whl-kernel-cu118.yml @@ -0,0 +1,92 @@ +name: Release SGLang Kernel Wheel (cu118) + +on: + workflow_dispatch: + inputs: + tag_name: + type: string + push: + branches: + - main + paths: + - sgl-kernel/python/sgl_kernel/version.py + +jobs: + build-wheels: + if: github.repository == 'sgl-project/sglang' + runs-on: sgl-kernel-release-node + strategy: + matrix: + python-version: ["3.9"] + cuda-version: ["11.8"] + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels for Python ${{ matrix.python-version }} and CUDA ${{ matrix.cuda-version }} + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }} + path: sgl-kernel/dist/* + + release: + needs: build-wheels + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.WHL_TOKEN }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.WHL_TOKEN }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "github-actions[bot]" + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "update whl index" + git push diff --git a/.github/workflows/release-whl-kernel.yml b/.github/workflows/release-whl-kernel.yml new file mode 100644 index 000000000..b12c91288 --- /dev/null +++ b/.github/workflows/release-whl-kernel.yml @@ -0,0 +1,283 @@ +name: Release SGLang Kernels + +on: + push: + branches: + - main + paths: + - sgl-kernel/python/sgl_kernel/version.py + workflow_dispatch: + inputs: + tag_name: + type: string + required: false + +concurrency: + group: release-sglang-kernels-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-cu129: + if: github.repository == 'sgl-project/sglang' + runs-on: sgl-kernel-release-node + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["12.9"] + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + + - name: Upload to PyPI + working-directory: sgl-kernel + run: | + pip install twine + python3 -m twine upload dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }} + + build-cu124: + if: github.repository == 'sgl-project/sglang' + needs: build-cu129 + runs-on: sgl-kernel-release-node + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["12.4"] + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }} + path: sgl-kernel/dist/* + + release-cu124: + needs: build-cu124 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.WHL_TOKEN }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.WHL_TOKEN }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py --cuda 124 + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "github-actions[bot]" + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "update whl index" + git push + + build-cu128: + if: github.repository == 'sgl-project/sglang' + needs: build-cu129 + runs-on: sgl-kernel-release-node + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["12.8"] + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }} + path: sgl-kernel/dist/* + + release-cu128: + needs: build-cu128 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.WHL_TOKEN }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.WHL_TOKEN }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py --cuda 128 + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "github-actions[bot]" + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "update whl index" + git push + + build-cu129-aarch64: + if: github.repository == 'sgl-project/sglang' + runs-on: sgl-kernel-release-node-arm + strategy: + matrix: + python-version: ["3.10"] + cuda-version: ["12.9"] + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Build wheels + run: | + cd sgl-kernel + chmod +x ./build.sh + ./build.sh "${{ matrix.python-version }}" "${{ matrix.cuda-version }}" aarch64 + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-python${{ matrix.python-version }}-cuda${{ matrix.cuda-version }}-aarch64 + path: sgl-kernel/dist/* + + release-cu129-aarch64: + needs: build-cu129-aarch64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: sgl-kernel/dist/ + merge-multiple: true + pattern: wheel-* + + - name: Set tag name + id: set_tag_name + run: | + if [ -z "${{ inputs.tag_name }}" ]; then + TAG_NAME="v$(cat sgl-kernel/python/sgl_kernel/version.py | cut -d'"' -f2)" + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + else + echo "tag_name=${{ inputs.tag_name }}" >> $GITHUB_OUTPUT + fi + + - name: Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.set_tag_name.outputs.tag_name }} + repository: sgl-project/whl + token: ${{ secrets.WHL_TOKEN }} + files: | + sgl-kernel/dist/* + + - name: Clone wheel index + run: git clone https://oauth2:${WHL_TOKEN}@github.com/sgl-project/whl.git sgl-whl + env: + WHL_TOKEN: ${{ secrets.WHL_TOKEN }} + + - name: Update wheel index + run: python3 scripts/update_kernel_whl_index.py --cuda 129 + + - name: Push wheel index + run: | + cd sgl-whl + git config --local user.name "github-actions[bot]" + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -A + git commit -m "update whl index" + git push diff --git a/.github/workflows/vllm-dependency-test.yml b/.github/workflows/vllm-dependency-test.yml new file mode 100644 index 000000000..442d76d50 --- /dev/null +++ b/.github/workflows/vllm-dependency-test.yml @@ -0,0 +1,43 @@ +name: VLLM Dependency Test + +on: + push: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + pull_request: + branches: [ main ] + paths: + - "python/**" + - "scripts/ci/**" + - "test/**" + +concurrency: + group: vllm-dependency-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + vllm-dependency-test: + if: (github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request') && + github.event.pull_request.draft == false + runs-on: 1-gpu-runner + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + bash scripts/ci/ci_install_dependency.sh + pip install "bitsandbytes>=0.44.0" + + pip install "sgl-kernel==0.3.7" + + - name: Run vLLM dependency tests + timeout-minutes: 60 + run: | + export SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK=1 + + cd test/srt + python3 run_suite.py --suite vllm_dependency_test --timeout-per-file 3600 diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..9725fabd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,240 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ + +# Tokenizer cache for tests +.tokenizer_cache/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ + +# MacOS +.DS_Store + +# Vim +*.swp + +# Documentation +docs/_build + +# SGL +benchmark/mmlu/data +benchmark/mmlu/data.tar +benchmark/llava_bench/images +benchmark/llava_bench/mme_pack +*.jsonl +tmp*.txt + +# Plots +*.png +*.pdf + +# personnal +work_dirs/ +*.csv + +!logo.png + +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +compile_commands.json + +*.iml + +# VSCode +.vscode + +1 + +# Autoenv +.env.leave + +# Rust lib +Cargo.lock + +lmms-eval diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 000000000..835095154 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile=black +known_first_party=sglang diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..a295f2eb4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,59 @@ +default_stages: [pre-commit, pre-push, manual] + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-symlinks + - id: destroyed-symlinks + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-toml + - id: check-ast + - id: check-added-large-files + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: detect-private-key + - id: debug-statements + - id: no-commit-to-branch + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.7 + hooks: + - id: ruff + args: [--select=F401, --fixable=F401] + files: ^(benchmark/|docs/|examples/) + exclude: \.ipynb$ + - repo: https://github.com/psf/black + rev: 24.10.0 + hooks: + - id: black-jupyter + - repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + additional_dependencies: ['tomli'] + args: ['--toml', 'python/pyproject.toml', '-L', 'cann,thi,makro,wil,rouge'] + exclude: | + (?x)^( + test/srt/test_reasoning_parser\.py| + docs/advanced_features/vlm_query\.ipynb + )$ + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.8 + hooks: + - id: clang-format + types_or: [c++, cuda] + args: [--style=file, --verbose] + - repo: https://github.com/kynan/nbstripout + rev: 0.8.1 + hooks: + - id: nbstripout + args: + - '--keep-output' + - '--extra-keys=metadata.kernelspec metadata.language_info.version' diff --git a/3rdparty/amd/profiling/PROFILING.md b/3rdparty/amd/profiling/PROFILING.md new file mode 100644 index 000000000..7e15ec844 --- /dev/null +++ b/3rdparty/amd/profiling/PROFILING.md @@ -0,0 +1,425 @@ +## Profiling SGLang Infer System with AMD GPUs +This AppNote describes the SGLang profiling technical, code augment and running steps for systems with AMD Instinct GPUs, nevertheless the same procedure may work with Nvidia GPUs too. +Examples and steps are provided in detail, to facilitate easy reproduce and use to localize performance problem towards optimizations. +Two primary methods are covered: +- [RPD](https://github.com/ROCm/rocmProfileData.git) +- [PyTorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) + +### Profiling SGLang Infer System with RPD Profiler +RPD profiler is a low-overhead cross-platform profiler. Therefore, the same RPD code augment not only works for profiling on ROCm/AMD GPUs, but also works for profiling on CUDA/Nvidia GPUs as well. To do RPD profiling on SGLang repository, please use scripts and patch files included in this directory and follow the steps below: +1. Install RPD with rpd.patch applied during installation using install_rpd.sh, both files are in this directory. + +install_rpd.sh + +```bash +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git checkout 976899e9c6dbc6dd2bccf770818e4e44125590ac +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. +``` + +rpd.patch + +```bash +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif +``` +2. Add loadTracer.sh file included in this directory to /sglang/python/sglang. + +loadTracer.sh + +```bash +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" +``` +3. Apply patch (provided in this directory) with "git apply rpd_profile_server_enable.patch" if the main profiling purpose is to get info on gpu kernels as well as limited cpu activity info. + +#### Common Notes 1 +Please note that although we are doing TP=8 in the example, we purposely only log RPD profiling on 2 ranks in the patch file (i.e.tp_rank=0/1) for profiling/visualization convenience, as even Perfetto streaming mode can only load maximal 8GB json file for visualization. With 2 ranks logged in RPD profiling, we could still check whether there are issues among ranks (e.g. load imbalance issue, nccl issue), and at the same time, we could log relatively longer time duration before the json file generated from RPD file hits 8GB size. + +rpd_profile_server_enable.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") +``` + +#### Advanced Debugging with RPD Profiler +Sometimes, we want to use rpd profiler to capture more CPU and python activities in order to debug some challenging issues (e.g. root cause of load imbalance across gpu processes, root cause of bubbles, etc). Only in such cases, we need to apply patch "git apply rpd_profile_server_enable_wCPU_activities.patch", where 3 files are modified. + +rpd_profile_server_enable_wCPU_activities.patch + +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, +``` + +4. As an example for grok1 profiling, we create a dummy_grok1 directory with config.json (see content below) inside this directory and copy this directory to the right path for "--model-path" if you want to use the example server.sh file provided. +```bash +cat ../dummy_grok1/config.json +{ + "architectures": [ + "Grok1ModelForCausalLM" + ], + "embedding_multiplier_scale": 78.38367176906169, + "output_multiplier_scale": 0.5773502691896257, + "vocab_size": 131072, + "hidden_size": 6144, + "intermediate_size": 32768, + "max_position_embeddings": 8192, + "num_experts_per_tok": 2, + "num_local_experts": 8, + "num_attention_heads": 48, + "num_hidden_layers": 64, + "num_key_value_heads": 8, + "head_dim": 128, + "rms_norm_eps": 1e-05, + "rope_theta": 10000.0, + "model_type": "mixtral", + "torch_dtype": "bfloat16" +} +``` +5. Launch server with rpd enabled script ./server.sh in one terminal inside the docker container. + +#### Common Notes 2 +- Remember to change model-path to the correct path +- loadTracer.sh is needed to conduct profiling +- SGLANG_TORCH_PROFILER_DIR is used for default torch profiler +- Do not use loadTracer.sh if you are using the torch profiler, simply use python3 -m sglang.launch_server. + + +server.sh + +```bash +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" +``` +6. Open another terminal for the same docker container, and run the rpd enabled ./client.sh after you see "The server is fired up and is ready to roll!" message from server side terminal. + +#### Common Notes 3 +- Use curl http://localhost:30000/start_profile & curl http://localhost:30000/stop_profile to control the start and end of profiling. Check sglang/python/sglang/srt/managers/scheduler.py for more details. +- Please don't use RPD profiler together with PyTorch profiler to avoid interference. +- The rocmProfileData/tools/rpd2tracing.py file is used to generate json file from RPD file. + +client.sh + +```bash +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 120 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 ./rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json +``` +7. Follow [Perfetto docs](https://perfetto.dev/docs/visualization/large-traces) to visualize large json files. Try to adjust parameters so that the trace.json file size is less than 9GB. + +### Profiling SGLang Infer System with PyTorch Profiler + +Please use the steps as follows: + +1. Apply the patch torch_profiler.patch. Note that you can modify "if self.tp_rank == 0" in the patch to allow more ranks be recorded in profiling. + +torch_profiler.patch +```bash +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") +``` + +2. Create the model path directory and copy it to the right path for "--model-path" if you want to use the server.sh file provided. + +3. Modify the included server.sh by removing "loadTracer.sh" before python command and launch script ./server.sh in one terminal inside the docker container. + +4. Similar to step 6 in RPD profiling section, but remove the last 2 lines in client.sh, which converted rpd file into csv and json files. Run modified client.sh for PyTorch profiling. +------- +- [Torch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) diff --git a/3rdparty/amd/profiling/client.sh b/3rdparty/amd/profiling/client.sh new file mode 100755 index 000000000..150ea9f19 --- /dev/null +++ b/3rdparty/amd/profiling/client.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Start profiling via API +curl http://localhost:30000/start_profile -H "Content-Type: application/json" + +# Benchmark serving using sglang with random dataset and tokenizer +# Define the log file with a timestamp +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOGFILE="sglang_client_log_$TIMESTAMP.json" + +# Run the benchmark with specified parameters and save logs +python3 -m sglang.bench_serving \ + --backend sglang \ + --tokenizer Xenova/grok-1-tokenizer \ + --dataset-name random \ + --random-input 1024\ + --random-output 1024 \ + --num-prompts 240 \ + --request-rate 8 \ + --output-file online.jsonl 2>&1 | tee "$LOGFILE" + +# Stop profiling via API +curl http://localhost:30000/stop_profile -H "Content-Type: application/json" + +# Convert tracing file to csv & json +sqlite3 trace.rpd ".mode csv" ".header on" ".output trace.csv" "select * from top;" ".output stdout" +python3 /sgl-workspace/rocmProfileData/tools/rpd2tracing.py trace.rpd trace.json diff --git a/3rdparty/amd/profiling/install_rpd.sh b/3rdparty/amd/profiling/install_rpd.sh new file mode 100644 index 000000000..d1b04b988 --- /dev/null +++ b/3rdparty/amd/profiling/install_rpd.sh @@ -0,0 +1,10 @@ +# download and install RPD +apt update && apt install -y sqlite3 libsqlite3-dev libfmt-dev + +# install rpd module +git clone https://github.com/ROCmSoftwarePlatform/rocmProfileData +cd rocmProfileData +git apply rpd.patch +make && make install +cd rocpd_python && python setup.py install && cd .. +cd rpd_tracer && make clean;make install && python setup.py install && cd .. diff --git a/3rdparty/amd/profiling/loadTracer.sh b/3rdparty/amd/profiling/loadTracer.sh new file mode 100755 index 000000000..8a95a335c --- /dev/null +++ b/3rdparty/amd/profiling/loadTracer.sh @@ -0,0 +1,43 @@ +#!/bin/bash +################################################################################ +# Copyright (c) 2021 - 2023 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +################################################################################ +OUTPUT_FILE="trace.rpd" + +if [ "$1" = "-o" ] ; then + OUTPUT_FILE=$2 + shift + shift +fi + +if [ -e ${OUTPUT_FILE} ] ; then + rm ${OUTPUT_FILE} +fi + +python3 -m rocpd.schema --create ${OUTPUT_FILE} +if [ $? != 0 ] ; then + echo "Error: Could not create rpd file. Please run 'python setup.py install' from the rocpd_python dir" + exit +fi + +export RPDT_FILENAME=${OUTPUT_FILE} +export RPDT_AUTOSTART=0 +LD_PRELOAD=librocm-smi_64:librpd_tracer.so "$@" diff --git a/3rdparty/amd/profiling/rpd.patch b/3rdparty/amd/profiling/rpd.patch new file mode 100644 index 000000000..87917654a --- /dev/null +++ b/3rdparty/amd/profiling/rpd.patch @@ -0,0 +1,12 @@ +diff --git a/rpd_tracer/Makefile b/rpd_tracer/Makefile +index e9d9feb..b2e9e1a 100644 +--- a/rpd_tracer/Makefile ++++ b/rpd_tracer/Makefile +@@ -16,7 +16,7 @@ ifneq (,$(HIP_PATH)) + $(info Building with roctracer) + RPD_LIBS += -L/opt/rocm/lib -lroctracer64 -lroctx64 -lamdhip64 -lrocm_smi64 + RPD_INCLUDES += -I/opt/rocm/include -I/opt/rocm/include/roctracer -I/opt/rocm/include/hsa +- RPD_SRCS += RoctracerDataSource.cpp RocmSmiDataSource.cpp ++ RPD_SRCS += RoctracerDataSource.cpp + RPD_INCLUDES += -D__HIP_PLATFORM_AMD__ + endif diff --git a/3rdparty/amd/profiling/rpd_profile_server_enable.patch b/3rdparty/amd/profiling/rpd_profile_server_enable.patch new file mode 100644 index 000000000..3cd391533 --- /dev/null +++ b/3rdparty/amd/profiling/rpd_profile_server_enable.patch @@ -0,0 +1,49 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..9021c01 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,24 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() #block pytorch profiler for rpd profiler enabling ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.start() ++ self.rpd.rangePush("", "rpd profile range", "") ++ logger.info("rpd is enabled") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done") + logger.info("Profiler is done") diff --git a/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch b/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch new file mode 100644 index 000000000..5416f4d57 --- /dev/null +++ b/3rdparty/amd/profiling/rpd_profile_server_enable_wCPU_activities.patch @@ -0,0 +1,126 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..2edb427 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -71,6 +71,8 @@ from sglang.srt.utils import ( + suppress_other_loggers, + ) + from sglang.utils import get_exception_traceback ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + logger = logging.getLogger(__name__) + +@@ -245,6 +247,7 @@ class Scheduler: + ], + with_stack=True, + ) ++ self.rpd = rpdTracerControl() + + @torch.inference_mode() + def event_loop(self): +@@ -1027,15 +1030,26 @@ class Scheduler: + def start_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.start() ++ #self.profiler.start() ++ logger.info("torch profiler is disabled") ++ if self.tp_rank == 0 or self.tp_rank == 1: ++ self.rpd.setPythonTrace(True) ++ self.rpd.start() ++ self.rpd.rangePush("", "scheduler", "") ++ logger.info("rpd is enabled inside scheduler profiling") + + def stop_profile(self) -> None: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") +- self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ #self.profiler.stop() ++ #self.profiler.export_chrome_trace( ++ # self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" ++ #) ++ if self.tp_rank ==0 or self.tp_rank ==1: ++ self.rpd.rangePop() ++ self.rpd.stop() ++ self.rpd.flush() ++ logger.info("rpd is done inside scheduler") + logger.info("Profiler is done") + + +diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py +index 2621ccd..181df85 100644 +--- a/python/sglang/srt/managers/tokenizer_manager.py ++++ b/python/sglang/srt/managers/tokenizer_manager.py +@@ -58,6 +58,10 @@ from sglang.srt.sampling.sampling_params import SamplingParams + from sglang.srt.server_args import PortArgs, ServerArgs + from sglang.srt.utils import is_generation_model, is_multimodal_model + ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() ++ ++ + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + + logger = logging.getLogger(__name__) +@@ -514,10 +518,20 @@ class TokenizerManager: + self.send_to_scheduler.send_pyobj(req) + + def start_profile(self): ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "tokenizer_manager", "") ++ logger.info("tokenizer_manager rpd profiling started!") + req = ProfileReq.START_PROFILE + self.send_to_scheduler.send_pyobj(req) + + def stop_profile(self): ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done inside tokenizer_manager!") + req = ProfileReq.STOP_PROFILE + self.send_to_scheduler.send_pyobj(req) + +diff --git a/python/sglang/srt/server.py b/python/sglang/srt/server.py +index 7111c93..2bd722c 100644 +--- a/python/sglang/srt/server.py ++++ b/python/sglang/srt/server.py +@@ -30,6 +30,8 @@ import threading + import time + from http import HTTPStatus + from typing import Dict, List, Optional, Union ++from rpdTracerControl import rpdTracerControl ++rpdTracerControl.skipCreate() + + # Fix a bug of Python threading + setattr(threading, "_register_atexit", lambda *args, **kwargs: None) +@@ -152,6 +154,11 @@ async def flush_cache(): + @app.post("/start_profile") + async def start_profile(): + """Start profiling.""" ++ rpd = rpdTracerControl() ++ rpd.setPythonTrace(True) ++ rpd.start() ++ rpd.rangePush("", "server rpd profile range", "") ++ logger.info("rpd profiling started in server.py!") + tokenizer_manager.start_profile() + return Response( + content="Start profiling.\n", +@@ -164,6 +171,11 @@ async def start_profile(): + async def stop_profile(): + """Stop profiling.""" + tokenizer_manager.stop_profile() ++ rpd = rpdTracerControl() ++ rpd.rangePop() ++ rpd.stop() ++ rpd.flush() ++ logger.info("rpd profiling is done in server.py!") + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, diff --git a/3rdparty/amd/profiling/server.sh b/3rdparty/amd/profiling/server.sh new file mode 100755 index 000000000..f877e6c7a --- /dev/null +++ b/3rdparty/amd/profiling/server.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# export SGLANG_TORCH_PROFILER_DIR=/data/sglang/ +export SGLANG_TORCH_PROFILER_DIR=/sgl-workspace/sglang/profile/ + +# Get the current timestamp +TIMESTAMP=$(date +"%Y%m%d_%H%M%S") + +# Define the log file with a timestamp +LOGFILE="sglang_server_log_$TIMESTAMP.json" + +# Run the Python command and save the output to the log file +loadTracer.sh python3 -m sglang.launch_server \ + --model-path /sgl-workspace/sglang/dummy_grok1 \ + --tokenizer-path Xenova/grok-1-tokenizer \ + --load-format dummy \ + --quantization fp8 \ + --tp 8 \ + --port 30000 \ + --disable-radix-cache 2>&1 | tee "$LOGFILE" diff --git a/3rdparty/amd/profiling/torch_profiler.patch b/3rdparty/amd/profiling/torch_profiler.patch new file mode 100644 index 000000000..40f557406 --- /dev/null +++ b/3rdparty/amd/profiling/torch_profiler.patch @@ -0,0 +1,25 @@ +diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py +index 62d1ff9..6ecd78c 100644 +--- a/python/sglang/srt/managers/scheduler.py ++++ b/python/sglang/srt/managers/scheduler.py +@@ -240,7 +240,6 @@ class Scheduler: + ) + self.profiler = torch.profiler.profile( + activities=[ +- torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, +@@ -1033,9 +1032,11 @@ class Scheduler: + if self.profiler is None: + raise RuntimeError("Profiler is not enabled.") + self.profiler.stop() +- self.profiler.export_chrome_trace( +- self.torch_profiler_trace_dir + "/" + str(time.time()) + ".trace.json.gz" +- ) ++ if self.tp_rank == 0: ++ with open(f"stats_repro_{int(time.time())}.txt", "w") as f: ++ print(self.profiler.key_averages(group_by_input_shape=True).table(sort_by="cuda_time_total", row_limit=-1), file=f) ++ print("Profiling stats done.") ++ + logger.info("Profiler is done") diff --git a/3rdparty/amd/tuning/TUNING.md b/3rdparty/amd/tuning/TUNING.md new file mode 100644 index 000000000..e7b9b2049 --- /dev/null +++ b/3rdparty/amd/tuning/TUNING.md @@ -0,0 +1,118 @@ +## Tuning SGLang Infer System with AMD GPUs +This AppNote describes the SGLang performance tuning technical, code harness and running steps for systems with AMD Instinct GPUs. +Harness code, examples and steps are provided in detail, to facilitate easy reproduce & use to tune performance towards workloads. +Three primary runtime areas are covered: + +## 1. Triton Kernels +To maximize Triton kernel efficiency, several strategies can be employed: + +### Key Environment Variables: +- **num_stages**: Adjusts the number of pipeline stages to optimize kernel efficiency based on the specific type of operations (e.g., General Matrix Multiplication - GEMM). +- **waves_per_eu**: Controls the usage of Vector General Purpose Registers (VGPR) to enhance occupancy, thereby improving latency or throughput. +- **BLOCK_M, BLOCK_N, BLOCK_K**: Tunable tile sizes that assist in balancing memory transfer and computational efficiency. +- **matrix_instr_nonkdim**: Optimizes the usage of Matrix-Fused Multiply-Add (MFMA) instructions for specific kernel types, such as Flash Attention. +- **OPTIMIZE_EPILOGUE**: An environment variable that can be set to `1` to enhance performance by eliminating the `convert_layout` operation in the kernel's epilogue. +```python +@triton.autotune(configs=[ + triton.Config({'waves_per_eu': 1}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 1}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 2}, num_warps=16, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=4, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=8, num_stages=1), + triton.Config({'waves_per_eu': 4}, num_warps=16, num_stages=1), + ], key=['BLOCK_N', 'NUM_TOKEN_BLKS'], use_cuda_graph=True) +@triton.jit +def _triton_kernel_funtion(): + ... +``` +## 2. Torch Tunable Operations +**TunableOp** is a feature in PyTorch that allows for the definition and optimization of custom kernels with tunable parameters. This feature is particularly useful for enhancing the performance of kernels by experimenting with different configurations. + +### Key Environment Variables: +1. **PYTORCH_TUNABLEOP_ENABLED**: + - Default: `0` + - Set to `1` to enable TunableOp. + +2. **PYTORCH_TUNABLEOP_TUNING**: + - Default: `1` + - Set to `0` to disable tuning. If a tuned entry is not found, it will run the tuning step and record the entry when PYTORCH_TUNABLEOP_ENABLED is enabled. + +3. **PYTORCH_TUNABLEOP_VERBOSE**: + - Default: `0` + - Set to `1` to enable verbose output for TunableOp. + +### Usage Example: +To enable TunableOp and tuning, and optionally enable verbose mode, you can run the following command in your terminal: + +```bash +#Tuning +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=1 your_script.sh + +#Inference with tuning op +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 your_script.sh + +#Print out the log +PYTORCH_TUNABLEOP_ENABLED=1 PYTORCH_TUNABLEOP_TUNING=0 PYTORCH_TUNABLEOP_VERBOSE=1 your_script.sh + +``` +## 3. Torch Compilation + + +The following are suggestions for optimizing matrix multiplication (GEMM) and convolution (conv) operations in PyTorch using Inductor, a part of the PyTorch compilation framework. The goal is to leverage Triton to achieve better performance. + +To tune Triton kernels with GEMM and convolution ops (conv), use the `torch.compile` function with the max-autotune mode. This benchmarks a predefined list of Triton configurations and selects the fastest one for each shape. + +### Key Configurations: +1. **Max Autotune**: + - Set `torch._inductor.config.max_autotune = True` or `TORCHINDUCTOR_MAX_AUTOTUNE=1`. + +2. **Fine-Grained Control**: + - Enable GEMM tuning: `torch._inductor.config.max_autotune_gemm = True`. + - Enable tuning for pointwise/reduction ops: `torch._inductor.config.max_autotune.pointwise = True`. + +3. **Backend Selection**: + - Use `torch._inductor.max_autotune_gemm_backends` to limit backends to TRITON for better performance. + +4. **Freezing for Inference**: + - Use `torch._inductor.config.freezing=True` to enable constant folding optimizations. + +5. **Debugging**: + - Set `TORCH_COMPILE_DEBUG=1` to extract Triton kernels generated by Inductor. + +### Example Code Block: +```bash +#Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 your_script.sh + +#Specify your backend to TRITON for Gemm Tuning +TORCHINDUCTOR_MAX_AUTOTUNE=1 TORCHINDUCTOR_COORDINATE_DESCENT_TUNING=1 TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_BACKENDS=TRITON your_script.sh + +#Inference with large improvement on AMD GPU +TORCHINDUCTOR_FREEZING=1 your_script.sh +``` +## 4. Fused MOE kernel +To maximize moe kernel efficiency, need to use below scripts to find out the best launch configuration + +### Key parameters: +- **--model**: what moe model type to do tuning, it will automatically decide the size of d_model, model_intermediate_size, num_layers +- **--tp-size**: simulate the whole model run configuration to set the dimension size using tp correctly +- **--batch**: M dimension size of moe kernel, for prefill moe kernel the value is batch*input_len, for decode moe kernel the value is batch +- **--dtype**: computation type + +```bash +#Tuning +#for example, we have one case like this "python3 -m sglang.bench_latency --model dummy_grok1/ --load-format dummy --tokenizer-path Xenova/grok-1-tokenizer --tp 8 --batch-size 32 --input 1024 --output 8 --attention-backend triton --sampling-backend pytorch --quantization fp8" to run, it defined batch-size 32 input length 1024 and output length 8, from "--batch" in moe view point, the prefill batch is 32*1024 = 32768, the decode batch is 32*1(only one output token generated in each run). +#so we can tune decode moe use below command +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32" +# and use this command to tune prefill moe +python benchmark_moe_rocm.py --model grok1 --tp-size 8 --dtype float8 --batch "32768" +``` + +## Reference + +For more detailed information on tuning SGLang performance with AMD GPUs, please refer to the following link: + +[ROCm Documentation: Triton Kernel Performance Optimization](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/workload.html#triton-kernel-performance-optimization) diff --git a/3rdparty/amd/tuning/benchmark_moe_rocm.py b/3rdparty/amd/tuning/benchmark_moe_rocm.py new file mode 100644 index 000000000..af596d218 --- /dev/null +++ b/3rdparty/amd/tuning/benchmark_moe_rocm.py @@ -0,0 +1,380 @@ +import argparse +import json +import os +import sys + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from tqdm import tqdm +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe, + get_config_file_name, +) + +padding_size = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0 + + +def main(model, tp_size, dtype: str, batches): + method = fused_moe + + for bs in batches: + run_grid(int(bs), model=model, method=method, tp_size=tp_size, dtype=dtype) + + +def prune_configs(M, N, K, configs): + pruned_configs = [] + elemBytes_a = 1 # [DV Note] Hard-coded for float16 (2 bytes) + elemBytes_b = 1 # [DV Note] Hard-coded for float16 (2 bytes) + + mfma = 16 if M < 32 or N < 32 else 32 + + # TODO (zhanglx): figure out the boundary between large and small gemms + large_gemm = False + if M >= 2048 and N >= 2048: + large_gemm = True + + for config in configs: + BLOCK_SIZE_M = config.get("BLOCK_SIZE_M") + BLOCK_SIZE_N = config.get("BLOCK_SIZE_N") + BLOCK_SIZE_K = config.get("BLOCK_SIZE_K") + num_warps = config.get("num_warps") + matrix_instr_nonkdim = config.get("matrix_instr_nonkdim") + # kpack = config.get("kpack") + if matrix_instr_nonkdim > mfma: + continue + if mfma == 4 and BLOCK_SIZE_K < 64: + continue + # some layouts could not work properly in case + # number elements per thread is less 1 + if BLOCK_SIZE_M * BLOCK_SIZE_N < 64: + continue + SPLIT_K = 1 # config.get("SPLIT_K") + GROUP_M = config.get("GROUP_SIZE_M") + if matrix_instr_nonkdim > BLOCK_SIZE_M or matrix_instr_nonkdim > BLOCK_SIZE_N: + continue + if matrix_instr_nonkdim >= M and matrix_instr_nonkdim != BLOCK_SIZE_M: + continue + if matrix_instr_nonkdim >= N and matrix_instr_nonkdim != BLOCK_SIZE_N: + continue + # Skip BLOCK_SIZE that is too large compare to M/N + # unless BLOCK_SIZE is already small enough + if M * 2 < BLOCK_SIZE_M and BLOCK_SIZE_M != 16: + continue + if N * 2 < BLOCK_SIZE_N and BLOCK_SIZE_N != 16: + continue + # skip large split_k when not necessary + if SPLIT_K != 1 and not need_split_k(M, N, K): + continue + # skip split_k that leads to EVEN_K = false + leap = SPLIT_K * BLOCK_SIZE_K + modv = K % leap + if modv != 0: + continue + # skip large GROUP_M + if GROUP_M * BLOCK_SIZE_M > M and GROUP_M != 1: + continue + # out of shared memory resource + # TODO (zhanglx): This does not consider the LDS usage in the epilogue + LDS = ( + BLOCK_SIZE_K * BLOCK_SIZE_M * elemBytes_a + + BLOCK_SIZE_K * BLOCK_SIZE_N * elemBytes_b + ) + if LDS > 65536: + continue + # Skip small block sizes and num_warps for large gemm + # For fp16 and f8, we want to only use BLOCK_SIZE >= 64 + if large_gemm: + if BLOCK_SIZE_M < 64 or BLOCK_SIZE_N < 64: + continue + if BLOCK_SIZE_K < 64: + continue + if num_warps < 4: + continue + + pruned_configs.append(config) + + return pruned_configs + + +def union_of_list_of_dicts(l1, l2): + result = [] + temp_list = l1.copy() + temp_list.extend(l2) + for myDict in temp_list: + if myDict not in result: + result.append(myDict) + + return result + + +def run_grid(bs, model, method, tp_size, dtype: str): + + config = AutoConfig.from_pretrained(model) + + top_k = config.num_experts_per_tok + d_model = config.hidden_size + model_intermediate_size = config.intermediate_size + num_layers = config.num_hidden_layers + hidden_states_dtype = config.torch_dtype + + if config.num_experts_per_tok: + if config.architectures[0] == "Grok1ModelForCausalLM": + num_total_experts = config.num_experts + else: + num_total_experts = config.num_local_experts + else: + raise ValueError(f"Unsupported Mixtral model {model}") + + # tp_size = 2 + num_warmup_calls = 10 + num_calls = 30 + + num_warmup_trials = 1 + num_trials = 1 + + full_configs = [] + + block_m_range = [16, 32, 64, 128, 256] + block_n_range = [16, 32, 64, 128, 256] + block_k_range = [32, 64, 128, 256] # MUST >= 32 + num_warps_range = [1, 2, 4, 8] + group_m_range = [1, 4, 8, 16, 32] + # For now we see better perf with num_stages=0 for all gemm configs we care + # But keep this explicit so that we do not forget we may need to set it to + # other values in the future + num_stage_range = [2] + waves_per_eu_range = [0, 1, 2, 4, 8] + # Remove 32 because of triton compiling error + matrix_instr_nonkdim_range = [16] + kpack_range = [1, 2] + + for block_size_m in block_m_range: + for block_size_n in block_n_range: + for block_size_k in block_k_range: + for group_size_m in group_m_range: + for num_warps in num_warps_range: + for num_stages in num_stage_range: + for waves_per_eu in waves_per_eu_range: + for matrix_instr_nonkdim in matrix_instr_nonkdim_range: + for kpack in kpack_range: + full_configs.append( + { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "kpack": kpack, + } + ) + + M1 = bs * 2 + N1 = model_intermediate_size * 2 // tp_size + K1 = d_model + prune_configs_1 = prune_configs(M1, N1, K1, full_configs) + + M2 = bs * 2 + N2 = d_model + K2 = model_intermediate_size // tp_size + prune_configs_2 = prune_configs(M2, N2, K2, full_configs) + + configs = union_of_list_of_dicts(prune_configs_1, prune_configs_2) + + print( + f"{bs=} || {len(full_configs)=} | {len(prune_configs_1)=} | \ + {len(prune_configs_2)=} | {len(configs)=}" + ) + + best_config = None + best_time_us = 1e20 + + print(f"{tp_size=} {bs=}") + + for config in tqdm(configs): + # warmup + try: + print(config) + for _ in range(num_warmup_trials): + run_timing( + num_calls=num_warmup_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + except triton.runtime.autotuner.OutOfResources: + continue + + # trial + for _ in range(num_trials): + kernel_dur_ms = run_timing( + num_calls=num_calls, + bs=bs, + d_model=d_model, + num_total_experts=num_total_experts, + top_k=top_k, + tp_size=tp_size, + model_intermediate_size=model_intermediate_size, + method=method, + config=config, + dtype=dtype, + hidden_states_dtype=hidden_states_dtype, + ) + + kernel_dur_us = 1000 * kernel_dur_ms + model_dur_ms = kernel_dur_ms * num_layers + + if kernel_dur_us < best_time_us: + best_config = config + best_time_us = kernel_dur_us + + tqdm.write( + f"{kernel_dur_us=:.1f} {model_dur_ms=:.1f}" + f" {bs=} {tp_size=} {top_k=} {num_total_experts=} " + f"{d_model=} {model_intermediate_size=} {num_layers=}" + ) + + print("best_time_us", best_time_us) + print("best_config", best_config) + + # holds Dict[str, Dict[str, int]] + filename = get_config_file_name( + num_total_experts, + model_intermediate_size // tp_size, + "float8" if dtype == "float8" else None, + ) + print(f"writing config to file {filename}") + existing_content = {} + if os.path.exists(filename): + with open(filename, "r") as f: + existing_content = json.load(f) + existing_content[str(bs)] = best_config + with open(filename, "w") as f: + json.dump(existing_content, f, indent=4) + f.write("\n") + + +def run_timing( + num_calls: int, + bs: int, + d_model: int, + num_total_experts: int, + top_k: int, + tp_size: int, + model_intermediate_size: int, + method, + config, + dtype: str, + hidden_states_dtype, +) -> float: + shard_intermediate_size = model_intermediate_size // tp_size + + hidden_states = torch.rand( + (bs, d_model), + device="cuda:0", + dtype=hidden_states_dtype, + ) + + w1 = torch.rand( + (num_total_experts, 2 * shard_intermediate_size, d_model + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w2 = torch.rand( + (num_total_experts, d_model, shard_intermediate_size + padding_size), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + + if dtype == "float8": + w1 = w1.to(torch.float8_e4m3fnuz) + w2 = w2.to(torch.float8_e4m3fnuz) + w1_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + w2_scale = torch.ones( + num_total_experts, device=hidden_states.device, dtype=torch.float32 + ) + a1_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + a2_scale = torch.ones(1, device=hidden_states.device, dtype=torch.float32) + + gating_output = F.softmax( + torch.rand( + (num_calls, bs, num_total_experts), + device=hidden_states.device, + dtype=torch.float32, + ), + dim=-1, + ) + + ################################## + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + start_event.record() + for i in range(num_calls): + hidden_states = method( + hidden_states=hidden_states, + w1=w1, + w2=w2, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + gating_output=gating_output[0], + topk=top_k, + renormalize=True, + inplace=True, + override_config=config, + use_fp8=dtype == "float8", + ) + + end_event.record() + end_event.synchronize() + + dur_ms = start_event.elapsed_time(end_event) / num_calls + return dur_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="benchmark_mixtral_moe", + description="Benchmark and tune the fused_moe kernel", + ) + parser.add_argument( + "--dtype", + type=str, + default="auto", + choices=["float8", "float16", "bfloat16"], + help="Data type used for fused_moe kernel computations", + ) + parser.add_argument("--model", type=str, default="hpcai-tech/grok-1") + + parser.add_argument("--tp-size", type=int, default=2, help="Tensor paralleli size") + parser.add_argument("-b", "--batches", type=str) + + args = parser.parse_args() + + batches = args.batches.split(",") + + sys.exit(main(args.model, args.tp_size, args.dtype, batches)) diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..9c422689c --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..459dfa573 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +.PHONY: check-deps install-deps format update help + +# Show help for each target +help: + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install isort black + +format: check-deps ## Format modified Python files using isort and black + @echo "Formatting modified Python files..." + git diff --name-only --diff-filter=M | grep '\.py$$' | xargs -I {} sh -c 'isort {} && black {}' + +FILES_TO_UPDATE = docker/Dockerfile.rocm \ + python/pyproject.toml \ + python/sglang/version.py \ + docs/developer_guide/setup_github_runner.md \ + docs/get_started/install.md \ + docs/platforms/amd_gpu.md \ + docs/platforms/ascend_npu.md \ + benchmark/deepseek_v3/README.md + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sglang/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/README.md b/README.md new file mode 100644 index 000000000..451a6d424 --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +
+logo + +[![PyPI](https://img.shields.io/pypi/v/sglang)](https://pypi.org/project/sglang) +![PyPI - Downloads](https://static.pepy.tech/badge/sglang?period=month) +[![license](https://img.shields.io/github/license/sgl-project/sglang.svg)](https://github.com/sgl-project/sglang/tree/main/LICENSE) +[![issue resolution](https://img.shields.io/github/issues-closed-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![open issues](https://img.shields.io/github/issues-raw/sgl-project/sglang)](https://github.com/sgl-project/sglang/issues) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/sgl-project/sglang) + +
+ +-------------------------------------------------------------------------------- + +| [**Blog**](https://lmsys.org/blog/2025-05-05-large-scale-ep/) +| [**Documentation**](https://docs.sglang.ai/) +| [**Join Slack**](https://slack.sglang.ai/) +| [**Join Bi-Weekly Development Meeting**](https://meeting.sglang.ai/) +| [**Roadmap**](https://github.com/sgl-project/sglang/issues/7736) +| [**Slides**](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#slides) | + +## News +- [2025/08] 🔔 SGLang x AMD SF Meetup on 8/22: Hands-on GPU workshop, tech talks by AMD/xAI/SGLang, and networking ([Roadmap](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_roadmap.pdf), [Large-scale EP](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_sglang_ep.pdf), [Highlights](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_highlights.pdf), [AITER/MoRI](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_aiter_mori.pdf), [Wave](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/amd_meetup_wave.pdf)). +- [2025/08] 🔥 SGLang provides day-0 support for OpenAI gpt-oss model ([instructions](https://github.com/sgl-project/sglang/issues/8833)) +- [2025/06] 🔥 SGLang, the high-performance serving infrastructure powering trillions of tokens daily, has been awarded the third batch of the Open Source AI Grant by a16z ([a16z blog](https://a16z.com/advancing-open-source-ai-through-benchmarks-and-bold-experimentation/)). +- [2025/06] 🔥 Deploying DeepSeek on GB200 NVL72 with PD and Large Scale EP (Part I): 2.7x Higher Decoding Throughput ([blog](https://lmsys.org/blog/2025-06-16-gb200-part-1/)). +- [2025/05] 🔥 Deploying DeepSeek with PD Disaggregation and Large-scale Expert Parallelism on 96 H100 GPUs ([blog](https://lmsys.org/blog/2025-05-05-large-scale-ep/)). +- [2025/03] Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html)) +- [2025/03] SGLang Joins PyTorch Ecosystem: Efficient LLM Serving Engine ([PyTorch blog](https://pytorch.org/blog/sglang-joins-pytorch/)) +- [2024/12] v0.4 Release: Zero-Overhead Batch Scheduler, Cache-Aware Load Balancer, Faster Structured Outputs ([blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/)). + +
+More + +- [2025/02] Unlock DeepSeek-R1 Inference Performance on AMD Instinct™ MI300X GPU ([AMD blog](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1_Perf/README.html)) +- [2025/01] SGLang provides day one support for DeepSeek V3/R1 models on NVIDIA and AMD GPUs with DeepSeek-specific optimizations. ([instructions](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3), [AMD blog](https://www.amd.com/en/developer/resources/technical-articles/amd-instinct-gpus-power-deepseek-v3-revolutionizing-ai-development-with-sglang.html), [10+ other companies](https://x.com/lmsysorg/status/1887262321636221412)) +- [2024/10] The First SGLang Online Meetup ([slides](https://github.com/sgl-project/sgl-learning-materials?tab=readme-ov-file#the-first-sglang-online-meetup)). +- [2024/09] v0.3 Release: 7x Faster DeepSeek MLA, 1.5x Faster torch.compile, Multi-Image/Video LLaVA-OneVision ([blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/)). +- [2024/07] v0.2 Release: Faster Llama3 Serving with SGLang Runtime (vs. TensorRT-LLM, vLLM) ([blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/)). +- [2024/02] SGLang enables **3x faster JSON decoding** with compressed finite state machine ([blog](https://lmsys.org/blog/2024-02-05-compressed-fsm/)). +- [2024/01] SGLang provides up to **5x faster inference** with RadixAttention ([blog](https://lmsys.org/blog/2024-01-17-sglang/)). +- [2024/01] SGLang powers the serving of the official **LLaVA v1.6** release demo ([usage](https://github.com/haotian-liu/LLaVA?tab=readme-ov-file#demo)). + +
+ +## About +SGLang is a fast serving framework for large language models and vision language models. +It makes your interaction with models faster and more controllable by co-designing the backend runtime and frontend language. +The core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-lora batching. +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, including chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Qwen, DeepSeek, Kimi, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models. +- **Active Community**: SGLang is open-source and backed by an active community with wide industry adoption. + +## Getting Started +- [Install SGLang](https://docs.sglang.ai/get_started/install.html) +- [Quick Start](https://docs.sglang.ai/basic_usage/send_request.html) +- [Backend Tutorial](https://docs.sglang.ai/basic_usage/openai_api_completions.html) +- [Frontend Tutorial](https://docs.sglang.ai/references/frontend/frontend_tutorial.html) +- [Contribution Guide](https://docs.sglang.ai/developer_guide/contribution_guide.html) + +## Benchmark and Performance +Learn more in the release blogs: [v0.2 blog](https://lmsys.org/blog/2024-07-25-sglang-llama3/), [v0.3 blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/), [v0.4 blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/), [Large-scale expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/). + +## Roadmap +[Development Roadmap (2025 H2)](https://github.com/sgl-project/sglang/issues/7736) + +## Adoption and Sponsorship +SGLang has been deployed at large scale, generating trillions of tokens in production each day. It is trusted and adopted by a wide range of leading enterprises and institutions, including xAI, AMD, NVIDIA, Intel, LinkedIn, Cursor, Oracle Cloud, Google Cloud, Microsoft Azure, AWS, Atlas Cloud, Voltage Park, Nebius, DataCrunch, Novita, InnoMatrix, MIT, UCLA, the University of Washington, Stanford, UC Berkeley, Tsinghua University, Jam & Tea Studios, Baseten, and other major technology organizations across North America and Asia. As an open-source LLM inference engine, SGLang has become the de facto industry standard, with deployments running on over 1,000,000 GPUs worldwide. + +logo + +## Contact Us +For enterprises interested in adopting or deploying SGLang at scale, including technical consulting, sponsorship opportunities, or partnership inquiries, please contact us at contact@sglang.ai. + +## Acknowledgment +We learned the design and reused code from the following projects: [Guidance](https://github.com/guidance-ai/guidance), [vLLM](https://github.com/vllm-project/vllm), [LightLLM](https://github.com/ModelTC/lightllm), [FlashInfer](https://github.com/flashinfer-ai/flashinfer), [Outlines](https://github.com/outlines-dev/outlines), and [LMQL](https://github.com/eth-sri/lmql). diff --git a/assets/logo.png b/assets/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_ diff --git a/assets/logo_square.svg b/assets/logo_square.svg new file mode 100644 index 000000000..a82fa0aeb --- /dev/null +++ b/assets/logo_square.svg @@ -0,0 +1 @@ + diff --git a/benchmark/bench_attention_sink/bench_attention_sink_triton.py b/benchmark/bench_attention_sink/bench_attention_sink_triton.py new file mode 100644 index 000000000..21bc2a593 --- /dev/null +++ b/benchmark/bench_attention_sink/bench_attention_sink_triton.py @@ -0,0 +1,250 @@ +import argparse + +import torch +import triton + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import extend_attention_fwd + +# gpt oss +head_num = 64 +head_dim = 64 +head_kv_num = 8 + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["S"], # sequence length on x-axis + x_vals=[128, 256, 512, 1024, 2048, 4096], + x_log=True, + line_arg="B", # batch size as different lines + line_vals=[1, 8, 32, 128], + line_names=["B=1", "B=8", "B=32", "B=128"], + styles=[ + ("blue", "-"), + ("green", "-"), + ("red", "-"), + ("cyan", "-"), + ], + ylabel="TFLOPS", + plot_name="attention-sink-triton-decode", + args={}, + ) +) +def benchmark_decode(B, S, H_Q, H_KV, D): + D_V = D + dtype = torch.bfloat16 + seq_len = S + total_tokens = B * seq_len + device = torch.device("cuda") + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len, dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + sink = torch.randn(H_Q, device=device, dtype=torch.float32) + + # warmup + for _ in range(5): + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=sink, + ) + + # benchmark + run_step = 500 + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(run_step): + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=sink, + ) + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + ms = start_event.elapsed_time(end_event) / run_step + tflops = lambda ms: (2 * B * S * H_Q * D) * 1e-9 / ms # must be causal + return tflops(ms) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["S"], # sequence length on x-axis + x_vals=[128, 256, 512, 1024, 2048, 4096], + x_log=True, + line_arg="B", # batch size as different lines + line_vals=[1, 8, 32, 128], + line_names=["B=1", "B=8", "B=32", "B=128"], + styles=[ + ("blue", "-"), + ("green", "-"), + ("red", "-"), + ("cyan", "-"), + ], + ylabel="TFLOPS", + plot_name="attention-sink-triton-extend", + args={}, + ) +) +def benchmark_extend(B, S, H_Q, H_KV, D): + # S here represents N_CTX from the test + dtype = torch.bfloat16 + device = "cuda" + + # Split S into prefix and extend lengths + prefill_len = S // 2 # Similar to test's N_CTX // 2 + extend_len = S // 4 # Make extend length smaller than prefix + + # Calculate total tokens and extend tokens + total_extend_tokens = B * extend_len + total_prefix_tokens = B * prefill_len + + # Create query, key, value tensors for extension + q_extend = torch.randn(total_extend_tokens, H_Q, D, dtype=dtype, device=device) + k_extend = torch.randn(total_extend_tokens, H_KV, D, dtype=dtype, device=device) + v_extend = torch.randn(total_extend_tokens, H_KV, D, dtype=dtype, device=device) + o_extend = torch.empty_like(q_extend) + + # Create key-value buffers for prefix + k_buffer = torch.randn(total_prefix_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_prefix_tokens, H_KV, D, dtype=dtype, device=device) + + # Create index pointers + qo_indptr = torch.arange(0, (B + 1) * extend_len, extend_len, device=device).to( + torch.int32 + ) + kv_indptr = torch.arange(0, (B + 1) * prefill_len, prefill_len, device=device).to( + torch.int32 + ) + kv_indices = torch.arange(0, total_prefix_tokens, device=device).to(torch.int32) + + sm_scale = 1.0 / (D**0.5) + # sliding_window = 128 # From GPT-OSS config, skip for now + sliding_window = -1 + + sink = torch.randn(H_Q, device=device, dtype=torch.float32) + + # warmup + for _ in range(5): + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=extend_len, + sm_scale=sm_scale, + sliding_window_size=sliding_window, + sinks=sink, + ) + + # benchmark + run_step = 500 + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(run_step): + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=extend_len, + sm_scale=sm_scale, + sliding_window_size=sliding_window, + sinks=sink, + ) + end_event.record() + end_event.synchronize() + torch.cuda.synchronize() + ms = start_event.elapsed_time(end_event) / run_step + + # FLOPS calculation: each attention operation requires 2 multiplications per element + total_flops = 2 * total_extend_tokens * H_Q * (prefill_len + extend_len / 2) * D + tflops = lambda ms: total_flops * 1e-12 / (ms * 1e-3) # convert to TFLOPS + return tflops(ms) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--bench", type=str, default="all", help="all, extend, decode") + args = parser.parse_args() + + kwargs = { + "H_Q": head_num, + "H_KV": head_kv_num, + "D": head_dim, + } + + if args.bench in ["all", "decode"]: + benchmark_decode.run(print_data=True, show_plots=False, **kwargs) + + if args.bench in ["all", "extend"]: + benchmark_extend.run(print_data=True, show_plots=False, **kwargs) + + print("Benchmark finished!") diff --git a/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py b/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py new file mode 100644 index 000000000..282097112 --- /dev/null +++ b/benchmark/bench_in_batch_prefix/bench_in_batch_prefix.py @@ -0,0 +1,130 @@ +# Benchmark with lots of common prefixes. Used to benchmark prefix caching performance. +# +# Launch a server: +# python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --log-level-http warning + +import random +import string +import time + +from tqdm import tqdm +from transformers import AutoTokenizer + +import sglang as sgl +from sglang import set_default_backend +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + +def generate_random_string(token_length: int) -> str: + random_string = "".join( + random.choices(string.ascii_letters + string.digits, k=token_length * 100) + ) + tokenized_output = tokenizer.encode(random_string, add_special_tokens=False)[ + :token_length + ] + + if len(tokenized_output) < token_length: + tokenized_output = tokenized_output + [tokenizer.pad_token_id] * ( + token_length - len(tokenized_output) + ) + + decoded_string = tokenizer.decode(tokenized_output, skip_special_tokens=False) + return decoded_string + + +def generate_unique_prefix(base_text, index): + return str(index) + base_text[len(str(index)) :] + + +@sgl.function +def text_qa(s, question, gen_len): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0, max_tokens=gen_len) + + +def prepare_prompts(num_prefix, num_samples_per_prefix, prefix_length, suffix_length): + base_prefix = generate_random_string(prefix_length) + + tot_input_len = 0 + all_prompts = [] + for i in tqdm(range(num_prefix), desc="prepare prompts"): + unique_prefix = generate_unique_prefix(base_prefix, i) + prompt_list = [] + for j in range(num_samples_per_prefix): + suffix = generate_random_string(suffix_length) + prompt = unique_prefix + suffix + prompt_list.append(prompt) + tot_input_len += len(tokenizer.encode(prompt)) + all_prompts.append(prompt_list) + return all_prompts, tot_input_len + + +def test_batch_by_batch(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.perf_counter() + text_qa.run_batch( + list(zip(all_prompts[i], [gen_len] * len(all_prompts[i]))), + ) + tot_time += time.perf_counter() - tic + + return tot_time + + +def test_batch_by_batch_with_hint(all_prompts, gen_len): + backend.flush_cache() + + tot_time = 0 + for i in range(len(all_prompts)): + tic = time.perf_counter() + # Send a hint to cache the prefix + text_qa.run_batch(list(zip(all_prompts[i][:1], [gen_len]))) + # Send the batch + text_qa.run_batch(list(zip(all_prompts[i], [gen_len] * len(all_prompts[i])))) + + tot_time += time.perf_counter() - tic + + return tot_time + + +def test_send_all(all_prompts, gen_len): + backend.flush_cache() + + all_prompts = [x for prompt_list in all_prompts for x in prompt_list] + + tic = time.perf_counter() + text_qa.run_batch( + list(zip(all_prompts, [gen_len] * len(all_prompts))), + ) + tot_time = time.perf_counter() - tic + + return tot_time + + +if __name__ == "__main__": + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + backend = RuntimeEndpoint("http://127.0.0.1:30000") + set_default_backend(backend) + + random.seed(0) + num_prefix = 10 + num_samples_per_prefix = 32 + prefix_length = 1024 + suffix_length = 128 + gen_len = 1 + all_prompts, tot_input_len = prepare_prompts( + num_prefix, num_samples_per_prefix, prefix_length, suffix_length + ) + + print(f"Total input token length: {tot_input_len}\n") + + cost = test_batch_by_batch(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch : {cost:.4f} s\n") + + cost = test_batch_by_batch_with_hint(all_prompts, gen_len) + print(f"Latency of test_batch_by_batch_with_hint: {cost:.4f} s\n") + + cost = test_send_all(all_prompts, gen_len) + print(f"Latency of test_send_all : {cost:.4f} s\n") diff --git a/benchmark/benchmark_batch/benchmark_batch.py b/benchmark/benchmark_batch/benchmark_batch.py new file mode 100644 index 000000000..a8592d48a --- /dev/null +++ b/benchmark/benchmark_batch/benchmark_batch.py @@ -0,0 +1,193 @@ +import concurrent.futures +import os +import random +import time +from concurrent.futures import ProcessPoolExecutor +from statistics import mean + +import requests +from tqdm import tqdm +from transformers import AutoTokenizer + +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + +############################################################################### +# CONFIG +############################################################################### +ENDPOINT_URL = "http://127.0.0.1:30000" +TOKENIZER_DIR = "/models/meta-llama/Llama-3.2-3B" + +# Benchmark configurations +NUM_REQUESTS = 10 # Total number of requests (each with BATCH_SIZE prompts) +NUM_TOKENS = 32000 # Tokens per prompt +BATCH_SIZE = 8 # Number of prompts per request +GEN_TOKENS = 0 # Tokens to generate per prompt + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def generate_random_prompt(index, tokenizer_dir, num_tokens): + """Generate a single random prompt with specified token count.""" + tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir) + vocab_size = tokenizer.vocab_size + + def generate_random_text(num_toks): + random_token_ids = [random.randint(0, vocab_size - 1) for _ in range(num_toks)] + return tokenizer.decode(random_token_ids, clean_up_tokenization_spaces=True) + + random_text = generate_random_text(num_tokens) + return f"Prompt {index}: {random_text}" + + +def prepare_all_prompts(num_requests, batch_size, num_tokens, tokenizer_dir): + """Generate prompts for all requests in parallel.""" + total_prompts = num_requests * batch_size + all_prompts = [None] * total_prompts + max_workers = min(os.cpu_count() or 1, total_prompts) + + with ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = [ + executor.submit(generate_random_prompt, i, tokenizer_dir, num_tokens) + for i in range(total_prompts) + ] + for future in tqdm( + concurrent.futures.as_completed(futures), + total=total_prompts, + desc="Generating prompts", + ): + index = futures.index(future) + all_prompts[index] = future.result() + + batched_prompts = [ + all_prompts[i * batch_size : (i + 1) * batch_size] for i in range(num_requests) + ] + + print( + f"Generated {total_prompts} prompts with {num_tokens} tokens each, grouped into {num_requests} requests of {batch_size} prompts.\n" + ) + return batched_prompts + + +############################################################################### +# HTTP CALLS +############################################################################### +def send_batch_request(endpoint, prompts, gen_tokens, request_id): + """Send a batch of prompts to the /generate endpoint synchronously.""" + sampling_params = { + "max_new_tokens": gen_tokens, + "temperature": 0.7, + "stop": "\n", + } + data = {"text": prompts, "sampling_params": sampling_params} + + start_time = time.perf_counter() + try: + response = requests.post( + endpoint.base_url + "/generate", json=data, timeout=3600 + ) + if response.status_code != 200: + error = response.json() + raise RuntimeError(f"Request {request_id} failed: {error}") + result = response.json() + elapsed_time = (time.perf_counter() - start_time) * 1000 # Convert to ms + avg_per_prompt = elapsed_time / len(prompts) if prompts else 0 + return request_id, elapsed_time, avg_per_prompt, True, len(prompts) + except Exception as e: + print(f"[Request] Error for request {request_id}: {e}") + return request_id, 0, 0, False, len(prompts) + + +def run_benchmark(endpoint, batched_prompts, batch_size, gen_tokens): + """Run the benchmark sequentially.""" + results = [] + num_requests = len(batched_prompts) + + # Record start time for total latency + benchmark_start_time = time.perf_counter() + + for i, batch_prompts in enumerate(batched_prompts): + request_id = i + 1 + assert ( + len(batch_prompts) == batch_size + ), f"Request {request_id} should have {batch_size} prompts, got {len(batch_prompts)}" + + print( + f"[Request] Sending request {request_id}/{num_requests} with {len(batch_prompts)} prompts at {int(time.time()*1000)}" + ) + result = send_batch_request(endpoint, batch_prompts, gen_tokens, request_id) + results.append(result) + + # Calculate total latency + total_latency = (time.perf_counter() - benchmark_start_time) * 1000 # Convert to ms + + return results, total_latency + + +############################################################################### +# RESULTS +############################################################################### +def process_results(results, total_latency, num_requests): + """Process and display benchmark results.""" + total_time = 0 + successful_requests = 0 + failed_requests = 0 + request_latencies = [] + per_prompt_latencies = [] + total_prompts = 0 + + for request_id, elapsed_time, avg_per_prompt, success, batch_size in results: + if success: + successful_requests += 1 + total_prompts += batch_size + request_latencies.append(elapsed_time) + per_prompt_latencies.append(avg_per_prompt) + total_time += elapsed_time / 1000 # Convert to seconds + else: + failed_requests += 1 + + avg_request_latency = mean(request_latencies) if request_latencies else 0 + avg_per_prompt_latency = mean(per_prompt_latencies) if per_prompt_latencies else 0 + throughput = total_prompts / total_time if total_time > 0 else 0 + + print("\nBenchmark Summary:") + print(f" Total requests sent: {len(results)}") + print(f" Total prompts sent: {total_prompts}") + print(f" Successful requests: {successful_requests}") + print(f" Failed requests: {failed_requests}") + print(f" Total latency (all requests): {total_latency:.2f} ms") + print(f" Avg per request latency: {avg_request_latency:.2f} ms") + print(f" Avg per prompt latency: {avg_per_prompt_latency:.2f} ms") + print(f" Throughput: {throughput:.2f} prompts/second\n") + + +############################################################################### +# MAIN +############################################################################### +def main(): + # Initialize endpoint + endpoint = RuntimeEndpoint(ENDPOINT_URL) + + # Generate prompts + batched_prompts = prepare_all_prompts( + NUM_REQUESTS, BATCH_SIZE, NUM_TOKENS, TOKENIZER_DIR + ) + + # Flush cache before benchmark + # endpoint.flush_cache() + + # Run benchmark + print( + f"Starting benchmark: NUM_TOKENS={NUM_TOKENS}, BATCH_SIZE={BATCH_SIZE}, NUM_REQUESTS={NUM_REQUESTS}\n" + ) + results, total_latency = run_benchmark( + endpoint, batched_prompts, BATCH_SIZE, GEN_TOKENS + ) + + # Process and display results + process_results(results, total_latency, NUM_REQUESTS) + + +if __name__ == "__main__": + random.seed(0) + main() diff --git a/benchmark/benchmark_batch/benchmark_tokenizer.py b/benchmark/benchmark_batch/benchmark_tokenizer.py new file mode 100644 index 000000000..88a5820b6 --- /dev/null +++ b/benchmark/benchmark_batch/benchmark_tokenizer.py @@ -0,0 +1,126 @@ +import random +import time +from statistics import mean + +from transformers import AutoTokenizer + +# CONFIG +TOKENIZER_DIR = ( + "/shared/public/sharing/fait360brew/training/models/meta-llama/Llama-3.2-3B" +) +NUM_TOKENS = 20000 # Each prompt should contain this many tokens +BATCH_SIZES = [1, 2, 4, 8] # Test different batch sizes +NUM_RUNS = 5 # Number of runs for each batch size to get reliable measurements + + +def generate_random_prompts(num_prompts, num_tokens, tokenizer): + """Generate random prompts with specified token count.""" + vocab_size = tokenizer.vocab_size + all_prompts = [] + + print(f"Generating {num_prompts} random prompts with {num_tokens} tokens each...") + for i in range(num_prompts): + # Generate random token IDs - this directly gives us the exact token count + random_token_ids = [ + random.randint(0, vocab_size - 1) for _ in range(num_tokens) + ] + random_text = tokenizer.decode( + random_token_ids, clean_up_tokenization_spaces=True + ) + + prompt = f"Prompt {i}: {random_text}" + tokens = tokenizer.encode(prompt) + print(f" Prompt {i}: {len(tokens)} tokens") + all_prompts.append(prompt) + + return all_prompts + + +def benchmark_sequential_vs_batch(prompts, batch_size, tokenizer): + """Compare sequential vs batch tokenization for a given batch size.""" + + # Sequential tokenization using encode() + sequential_times = [] + for run in range(NUM_RUNS): + batch_prompts = prompts[:batch_size] # Use same prompts for fair comparison + + start_time = time.perf_counter() + for prompt in batch_prompts: + tokens = tokenizer.encode(prompt) + sequential_time = (time.perf_counter() - start_time) * 1000 + sequential_times.append(sequential_time) + + # Batch tokenization using tokenizer() + batch_times = [] + for run in range(NUM_RUNS): + batch_prompts = prompts[:batch_size] # Use same prompts for fair comparison + + start_time = time.perf_counter() + tokens = tokenizer(batch_prompts) + batch_time = (time.perf_counter() - start_time) * 1000 + batch_times.append(batch_time) + + return { + "batch_size": batch_size, + "avg_sequential_ms": mean(sequential_times), + "avg_batch_ms": mean(batch_times), + "speedup_factor": ( + mean(sequential_times) / mean(batch_times) if mean(batch_times) > 0 else 0 + ), + "sequential_runs": sequential_times, + "batch_runs": batch_times, + } + + +def main(): + print("Tokenizer Benchmark: Sequential vs Batch Processing") + print("-" * 60) + print(f"Tokenizer: {TOKENIZER_DIR}") + print(f"Tokens per prompt: {NUM_TOKENS}") + print(f"Number of runs per batch size: {NUM_RUNS}") + print("-" * 60) + + # Load tokenizer once for all operations + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR) + + # The largest batch size determines how many prompts we need + max_batch_size = max(BATCH_SIZES) + all_prompts = generate_random_prompts(max_batch_size, NUM_TOKENS, tokenizer) + + results = [] + print("\nRunning benchmark...") + + for batch_size in BATCH_SIZES: + print(f"\nBenchmarking batch size: {batch_size}") + result = benchmark_sequential_vs_batch(all_prompts, batch_size, tokenizer) + results.append(result) + + print(f" Sequential tokenization (encode):") + for i, run_time in enumerate(result["sequential_runs"]): + print(f" Run {i+1}: {run_time:.2f} ms") + print(f" Average: {result['avg_sequential_ms']:.2f} ms") + + print(f" Batch tokenization (tokenizer):") + for i, run_time in enumerate(result["batch_runs"]): + print(f" Run {i+1}: {run_time:.2f} ms") + print(f" Average: {result['avg_batch_ms']:.2f} ms") + + print(f" Speedup factor: {result['speedup_factor']:.2f}x") + + print("\n" + "=" * 60) + print("SUMMARY OF RESULTS") + print("=" * 60) + print( + f"{'Batch Size':<10} {'Sequential (ms)':<18} {'Batch (ms)':<18} {'Speedup':<10}" + ) + print("-" * 60) + + for result in results: + print( + f"{result['batch_size']:<10} {result['avg_sequential_ms']:.2f} ms{' ' * 8} {result['avg_batch_ms']:.2f} ms{' ' * 8} {result['speedup_factor']:.2f}x" + ) + + +if __name__ == "__main__": + random.seed(0) + main() diff --git a/benchmark/benchmark_vllm_060/README.md b/benchmark/benchmark_vllm_060/README.md new file mode 100644 index 000000000..b480dabf2 --- /dev/null +++ b/benchmark/benchmark_vllm_060/README.md @@ -0,0 +1,89 @@ +## How to reproduce the benchmark results for SGLang v0.3.0 compared to vLLM v0.6.0 + +In short, with multi step enabled, in online scenarios that we benchmarked, the Median TTFT of vLLM is **3 times** that of SGLang, and the Median ITL is **10 times** that of SGLang. Lower Median TTFT and ITL are better. vLLM's multi-step optimization did not improve throughput while ensuring lower Median TTFT and ITL. Also, under maximum throughput benchmark, if vLLM does not set gpu util to 0.95 separately and uses the default configuration instead, its maximum throughput is **lower** than that of SGLang. + +## Online benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 1564.17 | **31.98** | 13.17 | **11.93** | +| 4 | 1200 | vLLM | 1691.97 | **100.48** | 14.14 | **129.32** | +| 8 | 2400 | SGLang | 2175.02 | **35.68** | 17.85 | **14.41** | +| 8 | 2400 | vLLM | 2137.16 | **120.39** | 17.09 | **158.63** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Median E2E Latency | Median TTFT | Median TPOT | Median ITL | +|------|-------------|--------|--------------------|-------------|-------------|------------| +| 4 | 1200 | SGLang | 3005.24 | **53.94** | 25.03 | **21.67** | +| 4 | 1200 | vLLM | 2915.60 | **179.15** | 23.58 | **231.23** | +| 8 | 2400 | SGLang | 4064.98 | **58.11** | 33.07 | **24.45** | +| 8 | 2400 | vLLM | 3752.38 | **207.12** | 29.15 | **275.32** | + +## Offline benchmark results + +### Llama 3.1 8B Instruct 1 x A100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 22.03 | **4281.51** | +| inf | 5000 | vLLM | 21.27 | **4132.37** | + +### Llama 3.1 70B Insruct 4 x H100 80G + +| RPS | Num Prompts | Engine | Request throughput | Output token throughput | +|------|-------------|--------|--------------------|-------------------------| +| inf | 5000 | SGLang | 19.84 | **3856.01** | +| inf | 5000 | vLLM | 19.04 | **3700.64** | + +## Installation + +```bash +# install sglang v0.3.0 +pip install --upgrade pip +pip install "sglang[all]"==0.3.0 +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.4/ + +# install vllm v0.6.0 +pip install vllm==0.6.0 +``` + +## Notes + +We referred to the reproduction method in https://github.com/vllm-project/vllm/issues/8176, and added the `--num-scheduler-steps 10` parameter when starting the vLLM server. The `gpu_memory_utilization` of vLLM is by default 0.9 at both TP 1 and TP 4, while SGLang's `mem_frac` is 0.88 at TP 1 and 0.85 at TP 4, so we manually set it to 0.88 at TP 4. + +## Online benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 1200 --request-rate 4 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 2400 --request-rate 8 +``` + +## Offline benchmarks + +```bash +# Llama 3.1 8B Instruct on 1 x A100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests --num-scheduler-steps 10 --max_model_len 4096 + +# Llama 3.1 70B Instruct on 4 x H100 +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 4 --mem-frac 0.88 +python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-70B-Instruct --disable-log-requests --num-scheduler-steps 10 --tensor 4 --max_model_len 4096 + +# bench serving +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 5000 +python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --num-prompts 5000 +``` diff --git a/benchmark/blog_v0_2/405b_sglang.sh b/benchmark/blog_v0_2/405b_sglang.sh new file mode 100644 index 000000000..491853782 --- /dev/null +++ b/benchmark/blog_v0_2/405b_sglang.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch sglang +# python -m sglang.launch_server --model-path ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --tp 8 --quantization fp8 --disable-radix --mem-frac 0.87 + +# offline +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > sglang_log11 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > sglang_log12 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > sglang_log13 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > sglang_log14 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > sglang_log15 +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompt 2000 > sglang_log21 + +# online +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > sglang_log31 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > sglang_log32 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > sglang_log33 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > sglang_log34 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > sglang_log35 diff --git a/benchmark/blog_v0_2/405b_trt.sh b/benchmark/blog_v0_2/405b_trt.sh new file mode 100644 index 000000000..1950bc92b --- /dev/null +++ b/benchmark/blog_v0_2/405b_trt.sh @@ -0,0 +1,17 @@ +# Launch trtllm +# https://github.com/sgl-project/tensorrt-demo + +# offline +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log11 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log12 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log13 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log14 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log15 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name sharegpt --num-prompt 2000 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log31 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log32 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log33 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log34 +python3 ../../python/sglang/bench_serving.py --backend trt --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 --model /root/Meta-Llama-3-8B-Instruct > trtllm_log35 diff --git a/benchmark/blog_v0_2/405b_vllm.sh b/benchmark/blog_v0_2/405b_vllm.sh new file mode 100644 index 000000000..d1ef048e8 --- /dev/null +++ b/benchmark/blog_v0_2/405b_vllm.sh @@ -0,0 +1,24 @@ +# Create dummy weights: +# 1. Create a folder `~/llama-3.1-405b-fp8-dummy` and create `config.json` and tokenizer under this folder. +# 2. Get `config.json`` from ./config.md +# 3. Download the tokenizer +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer.json +# wget https://huggingface.co/neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a8/resolve/main/tokenizer_config.json + +# Launch vllm +# python3 -m vllm.entrypoints.openai.api_server --model ~/llama-3.1-405b-fp8-dummy/ --load-format dummy --disable-log-requests --tensor-parallel-size 8 --max-model-len 10000 + +# offline +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3000 --random-input 1024 --random-output 1024 > vllm_log11 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 4000 --random-input 1024 --random-output 512 > vllm_log12 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 800 --random-input 4096 --random-output 2048 > vllm_log13 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1500 --random-input 4096 --random-output 1024 > vllm_log14 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 6000 --random-input 256 --random-output 512 > vllm_log15 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name sharegpt --num-prompt 2000 > vllm_log21 + +# online +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 300 --request-rate 1 --random-input 1024 --random-output 1024 > vllm_log31 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 600 --request-rate 2 --random-input 1024 --random-output 1024 > vllm_log32 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 1200 --request-rate 4 --random-input 1024 --random-output 1024 > vllm_log33 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 2400 --request-rate 8 --random-input 1024 --random-output 1024 > vllm_log34 +python3 ../../python/sglang/bench_serving.py --backend vllm --dataset-name random --num-prompt 3200 --request-rate 16 --random-input 1024 --random-output 1024 > vllm_log35 diff --git a/benchmark/blog_v0_2/README.md b/benchmark/blog_v0_2/README.md new file mode 100644 index 000000000..7448554ee --- /dev/null +++ b/benchmark/blog_v0_2/README.md @@ -0,0 +1,164 @@ +# How to reproduce the benchmark results of SGLang + +## Prerequisite + +### Install the latest SGLang + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout v0.2.7 + +pip install --upgrade pip +pip install -e "python[all]" + +pip install flashinfer -i https://flashinfer.ai/whl/cu121/torch2.3/ +``` + +### Set up ulimit and HF_TOKEN + +```bash +ulimit -n 65535 +# Change the token to a real and usable one, with access permissions for the Llama 3 models. +export HF_TOKEN=hf_token +``` + +### Launch the server + +```bash +# Meta-Llama-3.1-8B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile --disable-radix-cache + +# Meta-Llama-3.1-70B-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-70B-Instruct --disable-radix-cache --tp 8 + +# Meta-Llama-3-70B-Instruct-FP8 +python -m sglang.launch_server --model-path neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-radix-cache --tp 8 +``` + +## Benchmark + +### Hardware Requirements + +- 8B models: Single NVIDIA A100 80GB GPU +- 70B models: 8 x NVIDIA A100 80GB GPUs with Tensor Parallelism (TP) 8 +- 70B FP8 models: 8 x NVIDIA H100 GPUs with Tensor Parallelism (TP) 8 + +Please ensure you have the appropriate hardware before running the benchmarks. + +#### Offline benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --num-prompts 3000 --output-file offline.jsonl +cat offline.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +#### Online benchmark + +```bash +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online.jsonl +python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online.jsonl +cat online.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +## Other + +We tried using vLLM 0.5.3.post1, but it often crashes under high loads, and it seems to have similar or worse performance compared to vLLM 0.5.2 from our partial benchmarking, so we are using the older version, vLLM 0.5.2. + +Preparation for TensorRT LLM can refer to https://github.com/sgl-project/tensorrt-demo. Specifically, we used a batch size of 512, a max input length of 8192, and a max number of tokens of 8192. The instance count for preprocessing and postprocessing in Triton Server is 16. + +```bash +# vLLM +pip install vllm==0.5.2 +pip install jsonschema==4.21.1 + +# Meta-Llama-3-8B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-8B-Instruct --disable-log-requests + +# meta-llama/Meta-Llama-3-70B-Instruct +python -m vllm.entrypoints.openai.api_server --model meta-llama/Meta-Llama-3-70B-Instruct --disable-log-requests --tensor 8 + +# neuralmagic/Meta-Llama-3-70B-Instruct-FP8 +python -m vllm.entrypoints.openai.api_server --model neuralmagic/Meta-Llama-3-70B-Instruct-FP8 --disable-log-requests --tensor 8 +``` + +```bash +wget https://raw.githubusercontent.com/sgl-project/sglang/main/python/sglang/bench_serving.py +``` + +```bash +# vLLM Offline + +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name sharegpt --num-prompts 3000 --output-file offline_vllm.jsonl +cat offline_vllm.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# vLLM Online + +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_vllm.jsonl +python3 bench_serving.py --backend vllm --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_vllm.jsonl +cat online_vllm.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_8b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_8b.jsonl --model meta-llama/Meta-Llama-3-8B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_8b.jsonl +cat offline_trt_8b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 8B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_8b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_8b.jsonl +cat online_trt_8b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Offline 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 4000 --random-input 1024 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 5000 --random-input 1024 --random-output 512 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 1000 --random-input 4096 --random-output 2048 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --num-prompts 2000 --random-input 4096 --random-output 1024 --output-file offline_trt_70b.jsonl +python3 bench_serving.py --backend trt --dataset-name random --num-prompts 6000 --random-input 256 --random-output 512 --output-file offline_trt_70b.jsonl --model meta-llama/Meta-Llama-3-70B-Instruct +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name sharegpt --num-prompts 3000 --output-file offline_trt_70b.jsonl +cat offline_trt_70b.jsonl | cut -d':' -f12 | cut -d',' -f1 +``` + +```bash +# TensorRT LLM Online 70B + +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 300 --request-rate 1 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 600 --request-rate 2 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 1200 --request-rate 4 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 2400 --request-rate 8 --output-file online_trt_70b.jsonl +python3 bench_serving.py --backend trt --model meta-llama/Meta-Llama-3-70B-Instruct --dataset-name random --random-input 1024 --random-output 1024 --num-prompts 3200 --request-rate 16 --output-file online_trt_70b.jsonl +cat online_trt_70b.jsonl | cut -d':' -f9 | cut -d',' -f1 +``` diff --git a/benchmark/blog_v0_2/config.md b/benchmark/blog_v0_2/config.md new file mode 100644 index 000000000..3faf6009b --- /dev/null +++ b/benchmark/blog_v0_2/config.md @@ -0,0 +1,100 @@ +### used for TensorRT LLM + +``` +{ + "architecture": "LlamaForCausalLM", + "dtype": "float16", + "logits_dtype": "float32", + "vocab_size": 128256, + "max_position_embeddings": 8192, + "hidden_size": 16384, + "num_hidden_layers": 126, + "num_attention_heads": 128, + "num_key_value_heads": 16, + "head_size": 128, + "qk_layernorm": false, + "hidden_act": "silu", + "intermediate_size": 53248, + "norm_epsilon": 1e-05, + "position_embedding_type": "rope_gpt_neox", + "use_parallel_embedding": false, + "embedding_sharding_dim": 0, + "share_embedding_table": false, + "mapping": { + "world_size": 8, + "tp_size": 8, + "pp_size": 1, + "gpus_per_node": 8 + }, + "quantization": { + "quant_algo": "FP8", + "kv_cache_quant_algo": null, + "group_size": 128, + "smoothquant_val": null, + "has_zero_point": false, + "pre_quant_scale": false, + "exclude_modules": [ + "lm_head" + ] + }, + "kv_dtype": "float16", + "rotary_scaling": null, + "residual_mlp": false, + "moe_normalization_mode": null, + "rotary_base": 500000.0, + "moe_num_experts": 0, + "moe_top_k": 0, + "moe_tp_mode": 2, + "attn_bias": false, + "disable_weight_only_quant_plugin": false, + "mlp_bias": false +} +``` + +### used for vLLM and SGLang + +``` +{ + "_name_or_path": "dummy_fp8", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": 128009, + "hidden_act": "silu", + "hidden_size": 16384, + "initializer_range": 0.02, + "intermediate_size": 53248, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 128, + "num_hidden_layers": 126, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "quantization_config": { + "activation_scheme": "static", + "ignored_layers": [ + "lm_head" + ], + "quant_method": "fp8" + }, + "rope_scaling": { + "factor": 8.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "max_position_embeddings": 131072, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 500000.0, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.41.1", + "use_cache": true, + "vocab_size": 128256 +} +``` diff --git a/benchmark/boolq/README.md b/benchmark/boolq/README.md new file mode 100644 index 000000000..3704742ee --- /dev/null +++ b/benchmark/boolq/README.md @@ -0,0 +1,19 @@ +## Download data +``` +git clone https://hf-mirror.com/datasets/google/boolq +``` + +## Convert parquet to json +``` +bash parquet_to_json.sh +``` +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path ramblingpolymath/Qwen3-32B-W8A8 --port 30000 +``` + +``` +python3 bench_sglang.py +``` diff --git a/benchmark/boolq/bench_sglang.py b/benchmark/boolq/bench_sglang.py new file mode 100644 index 000000000..b3ce3c996 --- /dev/null +++ b/benchmark/boolq/bench_sglang.py @@ -0,0 +1,124 @@ +import argparse +import json +import time + +import numpy as np + +from sglang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import read_jsonl + + +def get_example(lines, i, answer): + prompt = "Question: " + lines[i]["question"] + lines[i]["passage"] + "\nAnswer:" + if answer: + prompt += str(lines[i]["answer"]) + return prompt + + +def few_shot_examples(lines, k): + prompts = "" + for i in range(k): + prompts += get_example(lines, i, True) + "\n\n" + return prompts + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + train_data_path = args.train_data_path + test_data_path = args.test_data_path + lines_train = list(read_jsonl(train_data_path)) + lines_test = list(read_jsonl(test_data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shots = few_shot_examples(lines_train, num_shots) + + questions = [] + answer = [] + for i in range(len(lines_test[:num_questions])): + questions.append(get_example(lines_test, i, False)) + answer.append(str(lines_test[i]["answer"])) + arguments = [{"question": q} for q in questions] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_boolq(s, question): + s += few_shots + question + s += sgl.gen("answer", max_tokens=5, stop=["\n"]) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + states = few_shot_boolq.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(states[i]["answer"]) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(answer)) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Results + with open(args.result_file, "a") as fout: + value = { + "task": "boolq", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument( + "--train-data-path", type=str, default="./boolq/data/train-00000-of-00001.json" + ) + parser.add_argument( + "--test-data-path", + type=str, + default="./boolq/data/validation-00000-of-00001.json", + ) + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/boolq/convert_parquet_to_json.py b/benchmark/boolq/convert_parquet_to_json.py new file mode 100644 index 000000000..e3e69cb31 --- /dev/null +++ b/benchmark/boolq/convert_parquet_to_json.py @@ -0,0 +1,28 @@ +import sys + +import pyarrow.parquet as pq + + +def convert_parquet_to_json(input_file, output_file): + # read parquet file + table = pq.read_table(input_file) + + # turn parquet data to dataframe + df = table.to_pandas() + + # turn dataframe to json form + json_data = df.to_json(orient="records", lines=True) + + # write json to file + with open(output_file, "w") as f: + f.write(json_data) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage:python convert_parquet_to_json.py ") + + input_file = sys.argv[1] + output_file = sys.argv[2] + + convert_parquet_to_json(input_file, output_file) diff --git a/benchmark/boolq/parquet_to_json.sh b/benchmark/boolq/parquet_to_json.sh new file mode 100755 index 000000000..9aaf087ff --- /dev/null +++ b/benchmark/boolq/parquet_to_json.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +#define input and output direction +input_dir="./boolq/data" +output_dir="./boolq/data" + +#define files needed to be handled +files=( + "train-00000-of-00001.parquet" + "validation-00000-of-00001.parquet" +) + +#foe files above, use python script to convert the form +for file in "${files[@]}"; do + input_file="${input_dir}/${file}" + output_file="${output_dir}/${file%.parquet}.json" + + echo "Converting ${input_file} to ${output_file} ..." + python3 convert_parquet_to_json.py "${input_file}" "${output_file}" + + if [ $? -eq 0 ]; then + echo "Conversion successful: ${output_file}" + else + echo "Conversion failed: ${input_file}" + fi +done diff --git a/benchmark/ceval/README.md b/benchmark/ceval/README.md new file mode 100644 index 000000000..b822e43c3 --- /dev/null +++ b/benchmark/ceval/README.md @@ -0,0 +1,15 @@ +## Download data +``` +git lfs clone https://huggingface.co/datasets/ceval/ceval-exam +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path ramblingpolymath/Qwen3-32B-W8A8 --port 30000 +``` + +``` +python3 bench_sglang.py +``` diff --git a/benchmark/ceval/bench_sglang.py b/benchmark/ceval/bench_sglang.py new file mode 100644 index 000000000..32ed0baf2 --- /dev/null +++ b/benchmark/ceval/bench_sglang.py @@ -0,0 +1,138 @@ +import argparse +import json +import os +import random +import re +import time + +import numpy as np +from datasets import load_dataset + +from sglang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +choices = ["A", "B", "C", "D"] + + +def get_one_example(line, include_answer): + res = line["question"] + res += f"\nA. {line['A']}" + res += f"\nB. {line['B']}" + res += f"\nC. {line['C']}" + res += f"\nD. {line['D']}" + + if include_answer: + res += f"\nAnswer: {line['answer']} \n\n" + return res + + +def get_few_shot_examples(lines): + res = "" + for line in lines: + res += get_one_example(line, True) + "\n\n" + return res + + +def get_answer_value(response): + pattern = r"(Answer:|answer:|答案是|答案是:|正确答案是:|答案:|Assistant:)\s*([A-D])(?![\w])" + match = re.search(pattern, response) + + if match: + return match.group(2) + + return random.choice(choices) + + +def main(args): + # Read data && Construct prompts + arguments = [] + labels = [] + examples = "examples:\n" + data_path = args.data_path + for subject in os.listdir(data_path): + subject_path = os.path.join(data_path, subject) + if os.path.isdir(subject_path) and subject != ".git": + dataset = load_dataset(data_path, name=subject) + dev_lines_temp = dataset["dev"] + val_lines_temp = dataset["val"] + few_shot_examples = get_few_shot_examples(dev_lines_temp, subject) + examples += f"{few_shot_examples}" + for val_line in val_lines_temp: + arguments.append( + { + "examples": few_shot_examples, + "question": get_one_example(val_line, False), + } + ) + labels.append(val_line["answer"]) + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_ceval(s, examples, question): + s += examples + question + sgl.gen("Answer") + + ##################################### + ########## SGL Program End ########## + ##################################### + + num_questions = args.num_questions if args.num_questions else len(arguments) + + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Run requests + tic = time.perf_counter() + states = few_shot_ceval.run_batch( + arguments[:num_questions], + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [get_answer_value(states[i]["Answer"]) for i in range(num_questions)] + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels[:num_questions])) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("Answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "ceval", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="ceval-exam") + parser.add_argument("--num-questions", type=int, default=None) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/deepseek_v3/README.md b/benchmark/deepseek_v3/README.md new file mode 100644 index 000000000..53efc23f5 --- /dev/null +++ b/benchmark/deepseek_v3/README.md @@ -0,0 +1,373 @@ +# DeepSeek V3.1/V3/R1 Support + +The SGLang and DeepSeek teams collaborated to get DeepSeek V3 FP8 running on NVIDIA and AMD GPUs **from day one**. SGLang also supports [MLA optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [DP attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models), making SGLang one of the best open-source LLM engines for running DeepSeek models. SGLang is the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended). + +Special thanks to Meituan's Search & Recommend Platform Team and Baseten's Model Performance Team for implementing the model, and DataCrunch for providing GPU resources. + +For optimizations made on the DeepSeek series models regarding SGLang, please refer to [DeepSeek Model Optimizations in SGLang](https://docs.sglang.ai/basic_usage/deepseek.html). + +## Installation & Launch + +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. + +### Using Docker (Recommended) + +```bash +# Pull latest image +# https://hub.docker.com/r/lmsysorg/sglang/tags +docker pull lmsysorg/sglang:latest + +# Launch +docker run --gpus all --shm-size 32g -p 30000:30000 -v ~/.cache/huggingface:/root/.cache/huggingface --ipc=host --network=host --privileged lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code --port 30000 +``` + +If you are using RDMA, please note that: + +1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. +2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +Add [performance optimization options](#performance-optimization-options) as needed. + +### Using pip + +```bash +# Installation +pip install "sglang[all]>=0.5.2" + +# Launch +python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` + +Add [performance optimization options](#performance-optimization-options) as needed. + + + +### Performance Optimization Options + +[MLA optimizations](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) are enabled by default. Here are some optional optimizations can be enabled as needed. + +- [Data Parallelism Attention](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models): For high QPS scenarios, add the `--enable-dp-attention` argument to boost throughput. +- [Torch.compile Optimization](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#torchcompile-latency-optimizations): Add `--enable-torch-compile` argument to enable it. This will take some time while server starts. The maximum batch size for torch.compile optimization can be controlled with `--torch-compile-max-bs`. It's recommended to set it between `1` and `8`. (e.g., `--torch-compile-max-bs 8`) + +### Usage: Chat with DeepSeek + +#### DeepSeek V3/R1 + +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "List 3 countries and their capitals."}, + ], + temperature=0, + max_tokens=64, +) +print(response) +``` + +#### DeepSeek V3.1 +On top of the basic usage similar to the DeepSeek V3/R1 example, DeepSeek V3.1 supports a request-level thinking/non-thinking toggle. Simply switch the `"thinking"` field in `extra_body={"chat_template_kwargs": {"thinking": True}}` to enable/disable the thinking mode. + +##### Non Thinking +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Answer the following with the second letter of the correct answer only: What is the capital of France?"}, + ], + temperature=0, + max_tokens=1024, + extra_body = {"chat_template_kwargs": {"thinking": False}} +) +print(response.choices[0].message.content) +``` +Answer: +``` +h +``` +* The correct response should be 'A', as the correct answer to the question is 'Paris'. +##### Thinking +```python3 +import openai +client = openai.Client( + base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +# Chat completion +response = client.chat.completions.create( + model="default", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Answer the following with the second letter of the correct answer only: What is the capital of France?"}, + ], + temperature=0, + max_tokens=1024, + extra_body = {"chat_template_kwargs": {"thinking": True}} +) +print(response) +``` +Answer: +``` +First, the question is: "What is the capital of France?" I know that the capital of France is Paris. + +The user says: "Answer the following with the second letter of the correct answer only." So, I need to provide only the second letter of the correct answer. + +The correct answer is "Paris". Now, I need to find the second letter of "Paris". + +Let's spell it out: P-A-R-I-S. + +- First letter: P + +- Second letter: A + +- Third letter: R + +- Fourth letter: I + +- Fifth letter: S + +So, the second letter is "A". + +I should only output the second letter, which is "A". No additional text or explanation, just the letter. + +The user emphasized "the second letter of the correct answer only", so my response should be just "A". + +Finally, I need to make sure that this is the correct answer. Yes, Paris is indeed the capital of France.A +``` +* The response contains `` thinking trace and model was able to derive the correct answer from it. + +### Example: Serving with two H20\*8 nodes + +For example, there are two H20 nodes, each with 8 GPUs. The first node's IP is `10.0.0.1`, and the second node's IP is `10.0.0.2`. Please **use the first node's IP** for both commands. + +If the command fails, try setting the `GLOO_SOCKET_IFNAME` parameter. For more information, see [Common Environment Variables](https://pytorch.org/docs/stable/distributed.html#common-environment-variables). + +If the multi nodes support NVIDIA InfiniBand and encounter hanging issues during startup, consider adding the parameter `export NCCL_IB_GID_INDEX=3`. For more information, see [this](https://github.com/sgl-project/sglang/issues/3516#issuecomment-2668493307). + +```bash +# node 1 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 0 --trust-remote-code + +# node 2 +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 10.0.0.1:5000 --nnodes 2 --node-rank 1 --trust-remote-code +``` + +If you have two H100 nodes, the usage is similar to the aforementioned H20. + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with two H200\*8 nodes and docker + +There are two H200 nodes, each with 8 GPUs. The first node's IP is `192.168.114.10`, and the second node's IP is `192.168.114.11`. Configure the endpoint to expose it to another Docker container using `--host 0.0.0.0` and `--port 40000`, and set up communications with `--dist-init-addr 192.168.114.10:20000`. +A single H200 with 8 devices can run DeepSeek V3, the dual H200 setup is just to demonstrate multi-node usage. + +```bash +# node 1 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode1 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +```bash +# node 2 +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode2 \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3 --tp 16 --dist-init-addr 192.168.114.10:20000 --nnodes 2 --node-rank 1 --trust-remote-code --host 0.0.0.0 --port 40000 +``` + +To ensure functionality, we include a test from a client Docker container. + +```bash +docker run --gpus all \ + --shm-size 32g \ + --network=host \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --name sglang_multinode_client \ + -it \ + --rm \ + --env "HF_TOKEN=$HF_TOKEN" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 1 --random-output 512 --random-range-ratio 1 --num-prompts 1 --host 0.0.0.0 --port 40000 --output-file "deepseekv3_multinode.jsonl" +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +### Example: Serving with four A100\*8 nodes + +To serve DeepSeek-V3 with A100 GPUs, we need to convert the [FP8 model checkpoints](https://huggingface.co/deepseek-ai/DeepSeek-V3) to BF16 with [script](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) mentioned [here](https://github.com/deepseek-ai/DeepSeek-V3/blob/main/inference/fp8_cast_bf16.py) first. + +Since the BF16 model is over 1.3 TB, we need to prepare four A100 nodes, each with 8 80GB GPUs. Assume the first node's IP is `10.0.0.1`, and the converted model path is `/path/to/DeepSeek-V3-BF16`, we can have following commands to launch the server. + +```bash +# node 1 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 0 --trust-remote-code --host 0.0.0.0 --port 30000 + +# node 2 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 1 --trust-remote-code + +# node 3 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 2 --trust-remote-code + +# node 4 +python3 -m sglang.launch_server --model-path /path/to/DeepSeek-V3-BF16 --tp 32 --dist-init-addr 10.0.0.1:5000 --nnodes 4 --node-rank 3 --trust-remote-code +``` + +> **Note that the launch command here does not enable Data Parallelism Attention or `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then we can benchmark the accuracy and latency by accessing the first node's exposed port with the following example commands. + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 --host http://10.0.0.1 --port 30000 + +# bench latency +python3 -m sglang.bench_one_batch_server --model None --base-url http://10.0.0.1:30000 --batch-size 1 --input-len 128 --output-len 128 +``` + + +### Example: Serving with 8 A100/A800 with AWQ Quantization + +**Recommended Usage** + +Add `--quantization moe_wna16` flag to enable moe wna16 kernel for better performance. +One example is as follows: + +```bash +python3 -m sglang.launch_server --model cognitivecomputations/DeepSeek-R1-AWQ --tp 8 --trust-remote-code --quantization moe_wna16 +``` + +Alternatively, you can use `--quantization awq_marlin` as follows: + +```bash +python3 -m sglang.launch_server --model cognitivecomputations/DeepSeek-R1-AWQ --tp 8 --trust-remote-code --quantization awq_marlin --dtype float16 +``` + +Note that `awq_marlin` only supports `float16` now, which may lead to some precision loss. + +### Example: Serving with 16 A100/A800 with int8 Quantization + +There are block-wise and per-channel quantization methods, and the quantization parameters have already been uploaded to Huggingface. One example is as follows: + +- [meituan/DeepSeek-R1-Block-INT8](https://huggingface.co/meituan/DeepSeek-R1-Block-INT8) +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-INT8` and port=5000, we can have following commands to launch the server: +```bash +#master +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 0 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +#cluster +python3 -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Block-INT8 --tp 16 --dist-init-addr \ + MASTER_IP:5000 --nnodes 2 --node-rank 1 --trust-remote-code --enable-torch-compile --torch-compile-max-bs 8 +``` + +> **Note that the launch command here enables `torch.compile` Optimization**. For optimal performance, please refer to the command options in [Performance Optimization Options](#option_args). + +Then on the **master node**, supposing the ShareGPT data is located at `/path/to/ShareGPT_V3_unfiltered_cleaned_split.json`, you can run the following commands to benchmark the launched server: + +```bash +# bench accuracy +python3 benchmark/gsm8k/bench_sglang.py --num-questions 1319 + +# bench serving +python3 -m sglang.bench_serving --dataset-path /path/to/ShareGPT_V3_unfiltered_cleaned_split.json --dataset-name random --random-input 128 --random-output 128 --num-prompts 1000 --request-rate 128 --random-range-ratio 1.0 +``` + +> **Note: using `--parallel 200` can accelerate accuracy benchmarking**. + +### Example: Serving with 32 L40S with int8 Quantization + +Running with per-channel quantization model: + +- [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) + +Assuming that master node IP is `MASTER_IP`, checkpoint path is `/path/to/DeepSeek-R1-Channel-INT8` and port=5000, we can have following commands to launch the server: + +```bash +#master +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 0 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +#cluster +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 1 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 2 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +python3 -m sglang.launch_server --model meituan/DeepSeek-R1-Channel-INT8 --tp 32 --quantization w8a8_int8 \ + --dist-init-addr MASTER_IP:5000 --nnodes 4 --node-rank 3 --trust-remote \ + --enable-torch-compile --torch-compile-max-bs 32 +``` + +The benchmarking method is the same as describted in the previous [16 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) example. + +### Example: Serving on any cloud or Kubernetes with SkyPilot + +SkyPilot helps find cheapest available GPUs across any cloud or existing Kubernetes clusters and launch distributed serving with a single command. See details [here](https://github.com/skypilot-org/skypilot/tree/master/llm/deepseek-r1). + +To serve on multiple nodes: + +```bash +git clone https://github.com/skypilot-org/skypilot.git +# Serve on 2 H100/H200x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B.yaml --retry-until-up +# Serve on 4 A100x8 nodes +sky launch -c r1 llm/deepseek-r1/deepseek-r1-671B-A100.yaml --retry-until-up +``` + +#### Troubleshooting + +If you encounter the following error with fp16/bf16 checkpoint: + +```bash +ValueError: Weight output_partition_size = 576 is not divisible by weight quantization block_n = 128. +``` + +edit your `config.json` and remove the `quantization_config` block. For example: + +```json +"quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [128, 128] +}, +``` + +Removing this block typically resolves the error. For more details, see the discussion in [sgl-project/sglang#3491](https://github.com/sgl-project/sglang/issues/3491#issuecomment-2650779851). + +## DeepSeek V3 Optimization Plan + +https://github.com/sgl-project/sglang/issues/2591 diff --git a/benchmark/dspy/README.md b/benchmark/dspy/README.md new file mode 100644 index 000000000..a2b213aa6 --- /dev/null +++ b/benchmark/dspy/README.md @@ -0,0 +1,51 @@ +## Install + +``` +pip3 install dspy-ai +``` + +Turn off cache at https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/dsp/modules/cache_utils.py#L10. +``` +cache_turn_on = False +``` + +or set the environment variable + +``` +export DSP_CACHEBOOL=false +``` + +## Benchmark SGLang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_dspy_intro.py --backend sglang +``` + + +## Benchmark TGI +``` +docker run --name tgi --rm -ti --gpus all --network host \ + -v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \ + ghcr.io/huggingface/text-generation-inference:1.3.0 \ + --model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \ + --max-input-length 2048 --max-total-tokens 4096 \ + --port 24000 +``` + +``` +python3 bench_dspy_intro.py --backend tgi +``` + + + +## Benchmark vLLM +``` +python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_dspy_intro.py --backend vllm +``` diff --git a/benchmark/dspy/bench_dspy_intro.py b/benchmark/dspy/bench_dspy_intro.py new file mode 100644 index 000000000..2b0936ed2 --- /dev/null +++ b/benchmark/dspy/bench_dspy_intro.py @@ -0,0 +1,192 @@ +""" +Adapted from +https://github.com/stanfordnlp/dspy/blob/34d8420383ec752037aa271825c1d3bf391e1277/intro.ipynb#L9 +""" + +import argparse + +import dspy +from dspy.datasets import HotPotQA + + +class BasicQA(dspy.Signature): + """Answer questions with short factoid answers.""" + + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class GenerateAnswer(dspy.Signature): + """Answer questions with short factoid answers.""" + + context = dspy.InputField(desc="may contain relevant facts") + question = dspy.InputField() + answer = dspy.OutputField(desc="often between 1 and 5 words") + + +class RAG(dspy.Module): + def __init__(self, num_passages=3): + super().__init__() + + self.retrieve = dspy.Retrieve(k=num_passages) + self.generate_answer = dspy.ChainOfThought(GenerateAnswer) + + def forward(self, question): + context = self.retrieve(question).passages + prediction = self.generate_answer(context=context, question=question) + return dspy.Prediction(context=context, answer=prediction.answer) + + +def main(args): + # lm = dspy.OpenAI(model='gpt-3.5-turbo') + if args.backend == "tgi": + lm = dspy.HFClientTGI( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "sglang": + lm = dspy.HFClientSGLang( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + elif args.backend == "vllm": + lm = dspy.HFClientVLLM( + model="meta-llama/Llama-2-7b-chat-hf", + port=args.port, + url="http://localhost", + ) + else: + raise ValueError(f"Invalid backend: {args.backend}") + + colbertv2_wiki17_abstracts = dspy.ColBERTv2( + url="http://20.102.90.50:2017/wiki17_abstracts" + ) + dspy.settings.configure(lm=lm, rm=colbertv2_wiki17_abstracts) + + # Load the dataset. + dataset = HotPotQA( + train_seed=1, train_size=20, eval_seed=2023, dev_size=args.dev_size, test_size=0 + ) + + # Tell DSPy that the 'question' field is the input. Any other fields are labels and/or metadata. + trainset = [x.with_inputs("question") for x in dataset.train] + devset = [x.with_inputs("question") for x in dataset.dev] + + print(len(trainset), len(devset)) + + train_example = trainset[0] + print(f"Question: {train_example.question}") + print(f"Answer: {train_example.answer}") + + dev_example = devset[18] + print(f"Question: {dev_example.question}") + print(f"Answer: {dev_example.answer}") + print(f"Relevant Wikipedia Titles: {dev_example.gold_titles}") + + print( + f"For this dataset, training examples have input keys {train_example.inputs().keys()} and label keys {train_example.labels().keys()}" + ) + print( + f"For this dataset, dev examples have input keys {dev_example.inputs().keys()} and label keys {dev_example.labels().keys()}" + ) + + # Define the predictor. + generate_answer = dspy.Predict(BasicQA) + + # Call the predictor on a particular input. + pred = generate_answer(question=dev_example.question) + + # Print the input and the prediction. + print(f"Question: {dev_example.question}") + print(f"Predicted Answer: {pred.answer}") + + lm.inspect_history(n=1) + + # Define the predictor. Notice we're just changing the class. The signature BasicQA is unchanged. + generate_answer_with_chain_of_thought = dspy.ChainOfThought(BasicQA) + + # Call the predictor on the same input. + pred = generate_answer_with_chain_of_thought(question=dev_example.question) + + # Print the input, the chain of thought, and the prediction. + print(f"Question: {dev_example.question}") + print(f"Thought: {pred.rationale.split('.', 1)[1].strip()}") + print(f"Predicted Answer: {pred.answer}") + + retrieve = dspy.Retrieve(k=3) + topK_passages = retrieve(dev_example.question).passages + + print( + f"Top {retrieve.k} passages for question: {dev_example.question} \n", + "-" * 30, + "\n", + ) + + for idx, passage in enumerate(topK_passages): + print(f"{idx+1}]", passage, "\n") + + retrieve("When was the first FIFA World Cup held?").passages[0] + + from dspy.teleprompt import BootstrapFewShot + + # Validation logic: check that the predicted answer is correct. + # Also check that the retrieved context does actually contain that answer. + def validate_context_and_answer(example, pred, trace=None): + answer_EM = dspy.evaluate.answer_exact_match(example, pred) + answer_PM = dspy.evaluate.answer_passage_match(example, pred) + return answer_EM and answer_PM + + # Set up a basic teleprompter, which will compile our RAG program. + teleprompter = BootstrapFewShot(metric=validate_context_and_answer) + + # Compile! + compiled_rag = teleprompter.compile(RAG(), trainset=trainset) + + # Ask any question you like to this simple RAG program. + my_question = "What castle did David Gregory inherit?" + + # Get the prediction. This contains `pred.context` and `pred.answer`. + pred = compiled_rag(my_question) + + # Print the contexts and the answer. + print(f"Question: {my_question}") + print(f"Predicted Answer: {pred.answer}") + print(f"Retrieved Contexts (truncated): {[c[:200] + '...' for c in pred.context]}") + + from dspy.evaluate.evaluate import Evaluate + + # Set up the `evaluate_on_hotpotqa` function. We'll use this many times below. + evaluate_on_hotpotqa = Evaluate( + devset=devset, + num_threads=args.num_threads, + display_progress=True, + display_table=5, + ) + + # Evaluate the `compiled_rag` program with the `answer_exact_match` metric. + metric = dspy.evaluate.answer_exact_match + evaluate_on_hotpotqa(compiled_rag, metric=metric) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int) + parser.add_argument("--num-threads", type=int, default=32) + parser.add_argument("--dev-size", type=int, default=150) + parser.add_argument( + "--backend", type=str, choices=["sglang", "tgi", "vllm"], default="sglang" + ) + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "lightllm": 22000, + "tgi": 24000, + "sglang": 30000, + } + args.port = default_port.get(args.backend, None) + + main(args) diff --git a/benchmark/generative_agents/README.md b/benchmark/generative_agents/README.md new file mode 100644 index 000000000..393a9ce83 --- /dev/null +++ b/benchmark/generative_agents/README.md @@ -0,0 +1,38 @@ +## Download the dataset + +``` +wget -O agent_calls.jsonl https://drive.google.com/uc?export=download&id=19qLpD45e9JGTKF2cUjJJegwzSUEZEKht +``` + +## Run benchmark + +Ensure that this benchmark is run in a serial manner (using --parallel 1) to preserve any potential dependencies between requests. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-events 1000 --parallel 1 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-events 1000 --backend vllm --parallel 1 +``` + +### Benchmark guidance +``` +python3 bench_other.py --num-events 1000 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-events 1000 --backend lmql --parallel 1 +``` diff --git a/benchmark/generative_agents/agent_functions.py b/benchmark/generative_agents/agent_functions.py new file mode 100644 index 000000000..9785fc1ec --- /dev/null +++ b/benchmark/generative_agents/agent_functions.py @@ -0,0 +1,300 @@ +import sglang as sgl + +# here are the top five agent functions contributing ~70% LLM calls +# reference: https://github.com/joonspk-research/generative_agents/ + + +@sgl.function +def poignancy_event(s, persona_name, persona_iss, event): + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + s += sgl.gen(name="Rate", max_tokens=2) + + +def poignancy_event_prompt(persona_name, persona_iss, event): + # return prompt and max_tokens + s = "" + s += "Here is a brief description of " + persona_name + ".\n" + s += persona_iss + "\n" + s += "On the scale of 1 to 10, where 1 is purely mundane (e.g., brushing teeth, making bed) and 10 is extremely poignant (e.g., a break up, college acceptance), rate the likely poignancy of the following event for" + s += persona_name + ".\n\n" + s += "Event: " + event + s += "Rate (return a number between 1 to 10):" + return {"prompt": s, "max_tokens": 2, "stop": None} + + +@sgl.function +def generate_event_triple(s, persona_name, action): + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + s += sgl.gen(name="Triple", max_tokens=20, stop=")") + + +def generate_event_triple_prompt(persona_name, action): + s = "" + s += """Task: Turn the input into (subject, predicate, object). +Input: Sam Johnson is eating breakfast. +Output: (Dolores Murphy, eat, breakfast) +--- +Input: Joon Park is brewing coffee. +Output: (Joon Park, brew, coffee) +--- +Input: Jane Cook is sleeping. +Output: (Jane Cook, is, sleep) +--- +Input: Michael Bernstein is writing email on a computer. +Output: (Michael Bernstein, write, email) +--- +Input: Percy Liang is teaching students in a classroom. +Output: (Percy Liang, teach, students) +--- +Input: Merrie Morris is running on a treadmill. +Output: (Merrie Morris, run, treadmill) +---""" + s += persona_name + "is" + action + ".\n" + s += "(" + persona_name + "," + return {"prompt": s, "max_tokens": 20, "stop": ")"} + + +@sgl.function +def generate_pronunciatio(s, action): + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + sgl.gen(name="Emoji", max_tokens=6) + + +def generate_pronunciatio_prompt(action): + s = "" + s += "Convert an action description to an emoji (important: use two or less emojis).\n" + s += "Action description: " + action + ".\n" + s += "Emoji:" + return {"prompt": s, "max_tokens": 6, "stop": None} + + +@sgl.function +def action_location_sector( + s, + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + s += sgl.gen(name="Location", max_tokens=10, stop="}") + + +def action_location_sector_prompt( + persona_name, + living_sector, + living_sector_areas, + current_sector, + current_sector_areas, + daily_plan, + sector_options, + current_action, + next_action, +): + s = "" + s += """Task -- choose an appropriate area from the area options for a task at hand. +Sam Kim lives in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Sam Kim is currently in {Sam Kim's house} that has Sam Kim's room, bathroom, kitchen. +Area options: {Sam Kim's house, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For taking a walk, Sam Kim should go to the following area: {Johnson Park} +--- +Jane Anderson lives in {Oak Hill College Student Dormatory} that has Jane Anderson's room. +Jane Anderson is currently in {Oak Hill College} that has a classroom, library +Area options: {Oak Hill College Student Dormatory, The Rose and Crown Pub, Hobbs Cafe, Oak Hill College, Johnson Park, Harvey Oak Supply Store, The Willows Market and Pharmacy}. +* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim. +For eating dinner, Jane Anderson should go to the following area: {Hobbs Cafe} +---""" + s += ( + persona_name + + " lives in " + + living_sector + + " that has " + + living_sector_areas + + ".\n" + ) + s += ( + persona_name + + " is currently in " + + current_sector + + " that has " + + current_sector_areas + + ".\n" + ) + s += daily_plan + ".\n" + s += "Area options: " + sector_options + ".\n" + s += """* Stay in the current area if the activity can be done there. Only go out if the activity needs to take place in another place. +* Must be one of the "Area options," verbatim.\n""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + " should go to the following area: {" + ) + return {"prompt": s, "max_tokens": 10, "stop": "}"} + + +@sgl.function +def action_location_object( + s, persona_name, target_sector, target_sector_areas, current_action, next_action +): + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + sgl.gen(name="Area", max_tokens=5, stop="}") + + +def action_location_object_prompt( + persona_name, target_sector, target_sector_areas, current_action, next_action +): + s = "" + s += """ +Jane Anderson is in kitchen in Jane Anderson's house. +Jane Anderson is going to Jane Anderson's house that has the following areas: {kitchen, bedroom, bathroom} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For cooking, Jane Anderson should go to the following area in Jane Anderson's house: +Answer: {kitchen} +--- +Tom Watson is in common room in Tom Watson's apartment. +Tom Watson is going to Hobbs Cafe that has the following areas: {cafe} +Stay in the current area if the activity can be done there. Never go into other people's rooms unless necessary. +For getting coffee, Tom Watson should go to the following area in Hobbs Cafe: +Answer: {cafe} +---""" + s += ( + persona_name + + " is going to " + + target_sector + + " that has the following areas: {" + + target_sector_areas + + "}\n" + ) + s += """* Stay in the current area if the activity can be done there. +* NEVER go into other people's rooms unless necessary.""" + s += ( + persona_name + + " is " + + current_action + + ". For " + + next_action + + ", " + + persona_name + + "should go to the following area in " + + target_sector + ) + s += " (MUST pick one of {" + target_sector_areas + "}):\n" + s += "Answer: {" + return {"prompt": s, "max_tokens": 5, "stop": "}"} diff --git a/benchmark/generative_agents/bench_other.py b/benchmark/generative_agents/bench_other.py new file mode 100644 index 000000000..c0b3a3406 --- /dev/null +++ b/benchmark/generative_agents/bench_other.py @@ -0,0 +1,80 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object_prompt, + action_location_sector_prompt, + generate_event_triple_prompt, + generate_pronunciatio_prompt, + poignancy_event_prompt, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event_prompt, + "generate_event_triple": generate_event_triple_prompt, + "generate_pronunciatio": generate_pronunciatio_prompt, + "action_location_sector": action_location_sector_prompt, + "action_location_object": action_location_object_prompt, + } + + arguments = [mapping[k](**v) for l in lines for k, v in l.items()] + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def get_one_answer(arg): + answer = call_generate(**arg, temperature=0) + states.append(answer) + + async def get_one_answer_async(arg): + answer = await call_generate(**arg, temperature=0) + states.append(answer) + + tic = time.perf_counter() + # we always sequentially execute agent calls to maintain its dependency + if args.backend != "lmql": + for arg in tqdm(arguments): + get_one_answer(arg) + else: + import asyncio + + loop = asyncio.get_event_loop() + for arg in tqdm(arguments): + loop.run_until_complete(get_one_answer_async(arg)) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/generative_agents/bench_sglang.py b/benchmark/generative_agents/bench_sglang.py new file mode 100644 index 000000000..034b16591 --- /dev/null +++ b/benchmark/generative_agents/bench_sglang.py @@ -0,0 +1,74 @@ +import argparse +import json +import time + +from agent_functions import ( + action_location_object, + action_location_sector, + generate_event_triple, + generate_pronunciatio, + poignancy_event, +) + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_events] + mapping = { + "poignancy_event": poignancy_event, + "generate_event_triple": generate_event_triple, + "generate_pronunciatio": generate_pronunciatio, + "action_location_sector": action_location_sector, + "action_location_object": action_location_object, + } + arguments = [{mapping[k]: v for k, v in l.items()} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + # Run requests + tic = time.perf_counter() + for a in arguments: + # only a single key in the dict + for func, arg in a.items(): + result = func.run(**arg) + result.sync() + states.append(result) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "Generative Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + # to pack weighted functions as a single agent + "num_requests": len(arguments) / len(mapping), + "other": { + "num_events": args.num_events, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="agent_calls.jsonl") + parser.add_argument("--num-events", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/gpt_oss/README.md b/benchmark/gpt_oss/README.md new file mode 100644 index 000000000..4d1b00e91 --- /dev/null +++ b/benchmark/gpt_oss/README.md @@ -0,0 +1,163 @@ +# How to reproduce the result of GPT-OSS with SGLang + +### Install the latest SGLang + +```bash +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout v0.5.1.post3 + +pip install --upgrade pip +pip install -e "python[all]" +``` + +### Reproduce the benchmark throughput result (Batch Size 1) + +Launch Command + +```bash +# MXFP4 120B on H100 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 8 --attention-backend triton + +# BF16 120B on H100 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 8 --attention-backend triton + +# MXFP4 120B on B200 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 4 + +# BF16 120B on B200 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 4 +``` + +Benchmark Command + +```bash + +# MXFP4 120B on H100 +python3 -m sglang.bench_one_batch_server --model openai/gpt-oss-120b --base-url http://localhost:30000 --batch-size 1 --input-len 1024 --output-len 512 --show-report +``` + +### Reproduce the benchmark throughput result (Batch Size 32) + +Launch Command + +```bash +# MXFP4 120B on H100 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 8 + +# BF16 120B on H100 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 8 + +# MXFP4 120B on B200 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --tp 4 + +# BF16 120B on B200 +python3 -m sglang.launch_server --model lmsys/gpt-oss-120b-bf16 --tp 4 +``` + +Benchmark Command + +```bash +python3 -m sglang.bench_one_batch_server --model openai/gpt-oss-120b --base-url http://localhost:30000 --batch-size 32 --input-len 1024 8192 --output-len 512 --show-report +``` + +### Reproduce the evaluation result + +Install gpt-oss + +```bash +git clone https://github.com/openai/gpt-oss.git +cd gpt-oss +pip install -e . +``` + +Evaluation Command + +```bash +DATASET=gpqa +BASE_URL=YOUR_BASE_URL +OPENAI_API_KEY=dummy python -m gpt_oss.evals \ + --base-url ${BASE_URL}/v1 \ + --model dummy \ + --reasoning-effort low,medium,high \ + --eval $DATASET \ + --n-threads 1000 +``` + +### Reproduce the benchmark result of acceptance length +> Note: On B200, if top k is 1, set `--attention-backend trtllm_mha` +```bash +git clone https://github.com/sgl-project/SpecForge.git +cd SpecForge/benchmarks +config_list=( + "1,0,0,0" + "1,3,1,4" + "1,5,4,8" +) +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list mtbench:80 gsm8k:200 humaneval:200 math500:200 \ + --output lmsys_gpt-oss-120b_Eagle3_result.jsonl + +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path nvidia/gpt-oss-120b-Eagle3 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list mtbench:80 gsm8k:200 humaneval:200 math500:200 \ + --output nv_gpt-oss-120b_Eagle3_result.jsonl +``` + +### Reproduce the result of speculative decoding speedup + +Launch Command + +```bash +# On Hopper: +# - Tree decoding (topk > 1) and chain decoding (topk = 1) are supported on both FA3 and Triton backends. +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algorithm EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --tp 4 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algorithm EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8 --tp 4 + +# On Blackwell: +# - Chain decoding (topk = 1) is supported on TRTLLM-MHA backend. Tree decoding (topk > 1) is in progress, stay tuned! +# - Both tree decoding (topk > 1) and chain decoding (topk = 1) are supported on the Triton backend. +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algo EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --tp 4 +python3 -m sglang.launch_server --model openai/gpt-oss-120b --speculative-algo EAGLE3 --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 --speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8 --attention-backend triton --tp 4 +``` + +Benchmark Command + +```bash +config_list=( + "1,0,0,0" + "1,3,1,4" + "1,5,4,8" +) +python3 bench_model_speedup.py \ + --model-path openai/gpt-oss-120b \ + --speculative-draft-model-path lmsys/EAGLE3-gpt-oss-120b-bf16 \ + --port 20001 \ + --trust-remote-code \ + --mem-fraction-static 0.8 \ + --tp-size 4 \ + --attention-backend fa3 \ + --config-list "${config_list[@]}" \ + --benchmark-list gsm8k:200 humaneval:200 math500:200 \ + --output lmsys_gpt-oss-120b_Eagle3_result.jsonl +``` + +We can gain the best speedup with the following settings: + +- **1.39x** speedup with the `--speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4` setting. +- **1.52x** speedup with the `--speculative-num-steps 5 --speculative-eagle-topk 4 --speculative-num-draft-tokens 8` setting. diff --git a/benchmark/gsm8k/README.md b/benchmark/gsm8k/README.md new file mode 100644 index 000000000..c110f533c --- /dev/null +++ b/benchmark/gsm8k/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 2 +``` diff --git a/benchmark/gsm8k/bench_other.py b/benchmark/gsm8k/bench_other.py new file mode 100644 index 000000000..6dcb9ad7c --- /dev/null +++ b/benchmark/gsm8k/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + call_generate = get_call_generate(args) + + # Read data + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = call_generate( + prompt=few_shot_examples + questions[i], + temperature=0, + max_tokens=256, + stop=["Question", "Assistant:", "<|separator|>"], + ) + states[i] = answer + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q in questions[i : i + batch_size]: + tasks.append( + call_generate( + few_shot_examples + q, + temperature=0, + max_tokens=256, + stop="Question", + ) + ) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + states[i + j] = rets[j] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/gsm8k/bench_sglang.py b/benchmark/gsm8k/bench_sglang.py new file mode 100644 index 000000000..9cdc2cf84 --- /dev/null +++ b/benchmark/gsm8k/bench_sglang.py @@ -0,0 +1,148 @@ +import argparse +import ast +import json +import os +import re +import time + +import numpy as np + +from sglang.lang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + dump_bench_raw_result, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_one_example(lines, i, include_answer): + ret = "Question: " + lines[i]["question"] + "\nAnswer:" + if include_answer: + ret += " " + lines[i]["answer"] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_gsm8k(s, question): + s += few_shot_examples + question + s += sgl.gen( + "answer", max_tokens=512, stop=["Question", "Assistant:", "<|separator|>"] + ) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + states = few_shot_gsm8k.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + + # Compute speed + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + + # Print results + print(f"Accuracy: {acc:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Latency: {latency:.3f} s") + print(f"Output throughput: {output_throughput:.3f} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + dump_bench_raw_result( + path=args.raw_result_file, + states=states, + preds=preds, + labels=labels, + ) + + with open(args.result_file, "a") as fout: + value = { + "task": "gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/hellaswag/README.md b/benchmark/hellaswag/README.md new file mode 100644 index 000000000..cb7e65366 --- /dev/null +++ b/benchmark/hellaswag/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 200 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lightllm +``` + + +### Benchmark guidance +``` +CUDA_VISIBLE_DEVICES=0,1 python3 bench_other.py --num-questions 200 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --num-questions 200 --backend lmql --port 23000 --parallel 1 +``` diff --git a/benchmark/hellaswag/bench_other.py b/benchmark/hellaswag/bench_other.py new file mode 100644 index 000000000..cde0794bb --- /dev/null +++ b/benchmark/hellaswag/bench_other.py @@ -0,0 +1,118 @@ +import argparse +import asyncio +import json +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_select +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + call_select = get_call_select(args) + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + + preds = [None] * len(labels) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + preds[i] = call_select( + context=few_shot_examples + questions[i], choices=choices[i] + ) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(questions), batch_size): + tasks = [] + for q, c in zip( + questions[i : i + batch_size], choices[i : i + batch_size] + ): + tasks.append(call_select(context=few_shot_examples + q, choices=c)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + + latency = time.perf_counter() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/hellaswag/bench_sglang.py b/benchmark/hellaswag/bench_sglang.py new file mode 100644 index 000000000..2adce99b8 --- /dev/null +++ b/benchmark/hellaswag/bench_sglang.py @@ -0,0 +1,109 @@ +import argparse +import json +import os +import time + +import numpy as np + +from sglang.lang.api import set_default_backend +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import download_and_cache_file, read_jsonl + + +def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + +def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + +def main(args): + # Select backend + set_default_backend(select_sglang_backend(args)) + + # Read data + data_path = args.data_path + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + if not os.path.isfile(data_path): + data_path = download_and_cache_file(url) + lines = list(read_jsonl(data_path)) + + # Construct prompts + num_questions = args.num_questions + num_shots = args.num_shots + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [choices[i].index(rets[i]["answer"]) for i in range(len(rets))] + latency = time.perf_counter() - tic + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + print(f"Latency: {latency:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "hellaswag", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shots", type=int, default=20) + parser.add_argument("--data-path", type=str, default="hellaswag_val.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/hf3fs/bench.sh b/benchmark/hf3fs/bench.sh new file mode 100644 index 000000000..049116b89 --- /dev/null +++ b/benchmark/hf3fs/bench.sh @@ -0,0 +1,59 @@ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +python3 benchmark/hf3fs/bench_client.py + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +SGLANG_HICACHE_HF3FS_CONFIG_PATH=/sgl-workspace/sglang/benchmark/hf3fs/hf3fs.json \ +python3 benchmark/hf3fs/bench_storage.py + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +export SGLANG_HICACHE_HF3FS_CONFIG_PATH=/sgl-workspace/sglang/benchmark/hf3fs/hf3fs.json +echo '{"file_path_prefix": "/data/hf3fs-test-0", "file_size": 1099511627776, "numjobs": 16, "entries": 8}' > \ +${SGLANG_HICACHE_HF3FS_CONFIG_PATH} +python3 benchmark/hf3fs/bench_zerocopy.py + +#################################################################################################### + +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --model-path /code/models/Qwen3-32B/ \ + --host 0.0.0.0 --port 33301 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 --hicache-size 0 \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs & + +rm -rf bench_multiturn.out && \ +nohup python3 benchmark/hicache/bench_multiturn.py \ + --model-path /code/models/Qwen3-32B \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --request-length 2048 --num-clients 512 --num-rounds 3 --max-parallel 8 \ + > bench_multiturn.out & + +#################################################################################################### + +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --model-path /code/models/DeepSeek-R1/ \ + --tp 16 --nnodes 2 --node-rank 0 \ + --dist-init-addr 10.74.249.153:5000 \ + --host 0.0.0.0 --port 33301 \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2 --hicache-size 60 \ + --hicache-write-policy write_through \ + --hicache-storage-backend hf3fs & + +rm -rf bench_multiturn.out && \ +nohup python3 benchmark/hicache/bench_multiturn.py \ + --model-path /code/models/Qwen3-32B \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --request-length 2048 --num-clients 1024 --num-rounds 3 --max-parallel 8 \ + > bench_multiturn.out & + +#################################################################################################### + +ps aux | grep "sglang.launch_server" | grep -v grep | awk '{print $2}' | xargs kill -9 +ps aux | grep "bench_multiturn.py" | grep -v grep | awk '{print $2}' | xargs kill -9 diff --git a/benchmark/hf3fs/bench_client.py b/benchmark/hf3fs/bench_client.py new file mode 100644 index 000000000..33c502575 --- /dev/null +++ b/benchmark/hf3fs/bench_client.py @@ -0,0 +1,162 @@ +import concurrent.futures +import logging +import random +import time +from typing import List + +import torch +from tqdm import tqdm + +from sglang.srt.mem_cache.storage.hf3fs.client_hf3fs import Hf3fsClient + + +def print_stats(x: List[int]): + x = sorted(x) + lenx = len(x) + print( + f"mean = {sum(x)/len(x):.2f}, " + f"min = {min(x):.2f}, " + f"p25 = {x[int(lenx*0.25)]:.2f}, " + f"p50 = {x[int(lenx*0.5)]:.2f}, " + f"p75 = {x[int(lenx*0.75)]:.2f}, " + f"max = {max(x):.2f}" + ) + + +def test(): + # /path/to/hf3fs + file_path = "/data/bench.bin" + file_size = 1 << 40 + bytes_per_page = 16 << 20 + entries = 32 + file_ops = Hf3fsClient(file_path, file_size, bytes_per_page, entries) + + print("test batch_read / batch_write") + num_pages = 128 + dtype = torch.bfloat16 + numel = bytes_per_page // dtype.itemsize + offsets = list(range(file_size // bytes_per_page)) + random.shuffle(offsets) + offsets = offsets[:num_pages] + offsets = [i * bytes_per_page for i in offsets] + tensor_writes = [ + torch.randn(numel, dtype=dtype) + for _ in tqdm(range(num_pages), desc="prepare tensor") + ] + for i in tqdm(range(0, num_pages, file_ops.entries), desc="batch_write"): + results = file_ops.batch_write( + offsets[i : i + file_ops.entries], tensor_writes[i : i + file_ops.entries] + ) + assert all([result == numel * dtype.itemsize for result in results]) + tensor_reads = [ + torch.empty(numel, dtype=dtype) + for _ in tqdm(range(num_pages), desc="prepare tensor") + ] + for i in tqdm(range(0, num_pages, file_ops.entries), desc="batch_read"): + results = file_ops.batch_read( + offsets[i : i + file_ops.entries], tensor_reads[i : i + file_ops.entries] + ) + assert all([result == numel * dtype.itemsize for result in results]) + assert all([torch.allclose(r, w) for r, w in zip(tensor_reads, tensor_writes)]) + + file_ops.close() + print("test done") + + +def bench(): + file_path = "/data/bench.bin" + file_size = 1 << 40 + bytes_per_page = 16 << 20 + entries = 8 + numjobs = 16 + + dtype = torch.bfloat16 + numel = bytes_per_page // dtype.itemsize + + file_ops = [ + Hf3fsClient(file_path, file_size, bytes_per_page, entries) + for _ in range(numjobs) + ] + + num_page = entries + + offsets = list(range(file_size // bytes_per_page)) + tensors_write = [torch.randn(numel, dtype=dtype)] * num_page + tensors_read = [torch.empty(numel, dtype=dtype)] * num_page + random.shuffle(offsets) + + warmup = 50 + iteration = 100 + + executor = concurrent.futures.ThreadPoolExecutor(max_workers=numjobs) + + w_bw = [] + w_size = num_page * numjobs * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking write (GB/s)"): + _offsets = [ + [ + offset * bytes_per_page + for offset in offsets[ + (i * numjobs + j) * num_page : (i * numjobs + j + 1) * num_page + ] + ] + for j in range(numjobs) + ] + tik = time.perf_counter() + futures = [ + executor.submit(file_ops[j].batch_write, offset, tensors_write) + for j, offset in enumerate(_offsets) + ] + results = [future.result() for future in futures] + tok = time.perf_counter() + if i < warmup: + continue + w_bw.append(w_size / (tok - tik)) + results = [ + _result == bytes_per_page for result in results for _result in result + ] + assert all(results) + print_stats(w_bw) + + r_bw = [] + r_size = w_size + for i in tqdm(range(warmup + iteration), desc="Benchmarking read (GB/s)"): + _offsets = [ + [ + offset * bytes_per_page + for offset in offsets[ + (i * numjobs + j) * num_page : (i * numjobs + j + 1) * num_page + ] + ] + for j in range(numjobs) + ] + tik = time.perf_counter() + futures = [ + executor.submit(file_ops[j].batch_read, offset, tensors_read) + for j, offset in enumerate(_offsets) + ] + results = [future.result() for future in futures] + tok = time.perf_counter() + if i < warmup: + continue + r_bw.append(r_size / (tok - tik)) + results = [ + _result == bytes_per_page for result in results for _result in result + ] + assert all(results) + print_stats(r_bw) + + executor.shutdown(wait=True) + for _file_ops in file_ops: + _file_ops.close() + print("bench done") + + +def main(): + logging.basicConfig(level=logging.INFO) + test() + bench() + + +if __name__ == "__main__": + main() diff --git a/benchmark/hf3fs/bench_storage.py b/benchmark/hf3fs/bench_storage.py new file mode 100644 index 000000000..f0ce171bf --- /dev/null +++ b/benchmark/hf3fs/bench_storage.py @@ -0,0 +1,258 @@ +import json +import logging +import os +import random +import time +from typing import List + +import torch +from tqdm import tqdm + +from sglang.srt.mem_cache.storage.hf3fs.mini_3fs_metadata_server import ( + Hf3fsLocalMetadataClient, +) +from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import HiCacheHF3FS + + +def print_stats(x: List[int]): + x = sorted(x) + lenx = len(x) + print( + f"mean = {sum(x)/len(x):.2f}, " + f"min = {min(x):.2f}, " + f"p25 = {x[int(lenx*0.25)]:.2f}, " + f"p50 = {x[int(lenx*0.5)]:.2f}, " + f"p75 = {x[int(lenx*0.75)]:.2f}, " + f"max = {max(x):.2f}" + ) + + +def test(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path_prefix = "/data/test" + file_size = 128 << 20 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 2 + dtype = store_dtype + + config_path = os.getenv(HiCacheHF3FS.default_env_var) + assert config_path + try: + with open(config_path, "w") as f: + json.dump( + { + "file_path_prefix": file_path_prefix, + "file_size": file_size, + "numjobs": numjobs, + "entries": entries, + }, + f, + ) + except Exception as e: + raise RuntimeError(f"Failed to dump config to {config_path}: {str(e)}") + hicache_hf3fs = HiCacheHF3FS.from_env_config(bytes_per_page, dtype) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_pages = 10 + tensors = {} + for i in range(num_pages): + k = f"key_{i}" + v = torch.randn((numel,)).to(dtype=dtype) + ok = hicache_hf3fs.set(k, v) + if i < (file_size // bytes_per_page): + assert ok, f"Failed to insert {k}" + else: + assert not ok + tensors[k] = v + assert hicache_hf3fs.get("key_8") is None + assert hicache_hf3fs.get("key_9") is None + + start = 0 + for i in range(start, start + hicache_hf3fs.num_pages): + k = f"key_{i}" + assert hicache_hf3fs.exists(k) + out = hicache_hf3fs.get(k) + assert out is not None + v = tensors[k] + assert torch.allclose(v, out, atol=1e-3), f"Tensor mismatch for {k}" + + assert not hicache_hf3fs.exists("not_exists") + + hicache_hf3fs.delete("key_7") + v2 = torch.randn((numel,)).to(dtype=dtype) + assert hicache_hf3fs.set("key_new", v2) + assert torch.allclose(hicache_hf3fs.get("key_new"), v2, atol=1e-3) + + hicache_hf3fs.clear() + assert ( + len(hicache_hf3fs.metadata_client.rank_metadata.free_pages) + == hicache_hf3fs.metadata_client.rank_metadata.num_pages + ) + + # batch + num_pages = 10 + tensors = {} + keys = [] + values = [] + for i in range(num_pages): + k = f"key_{i}" + keys.append(k) + v = torch.randn((numel,)).to(dtype=dtype) + values.append(v) + + ok = hicache_hf3fs.batch_set(keys, values) + assert not ok + assert hicache_hf3fs.get("key_8") is None + assert hicache_hf3fs.get("key_9") is None + + results = hicache_hf3fs.batch_get(keys[: hicache_hf3fs.num_pages]) + for result, key, value in zip( + results, keys[: hicache_hf3fs.num_pages], values[: hicache_hf3fs.num_pages] + ): + assert torch.allclose(value, result, atol=1e-3), f"Tensor mismatch for {key}" + + hicache_hf3fs.close() + os.remove(hicache_hf3fs.file_path) + + print("All test cases passed.") + + +def bench(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path = "/data/test.bin" + file_size = 1 << 40 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 8 + dtype = store_dtype + hicache_hf3fs = HiCacheHF3FS( + rank=0, + file_path=file_path, + file_size=file_size, + numjobs=numjobs, + bytes_per_page=bytes_per_page, + entries=entries, + dtype=dtype, + metadata_client=Hf3fsLocalMetadataClient(), + ) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_page = 128 + values = [torch.randn((numel,)).to(dtype=dtype) for _ in tqdm(range(num_page))] + + warmup = 50 + iteration = 100 + + w_bw = [] + w_size = num_page * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking write (GB/s)"): + keys = [f"{j}" for j in range(i * num_page, (i + 1) * num_page)] + tik = time.perf_counter() + ok = hicache_hf3fs.batch_set(keys, values) + tok = time.perf_counter() + if i < warmup: + continue + w_bw.append(w_size / (tok - tik)) + assert ok + print_stats(w_bw) + + r_bw = [] + r_size = num_page * bytes_per_page / (1 << 30) + for i in tqdm(range(warmup + iteration), desc="Benchmarking read (GB/s)"): + keys = random.sample( + list(hicache_hf3fs.metadata_client.rank_metadata.key_to_index.keys()), + num_page, + ) + tik = time.perf_counter() + results = hicache_hf3fs.batch_get(keys) + tok = time.perf_counter() + if i < warmup: + continue + r_bw.append(r_size / (tok - tik)) + assert all([r is not None for r in results]) + print_stats(r_bw) + + hicache_hf3fs.close() + + +def allclose(): + # Qwen3-32B + layer_num = 64 + head_num, head_dim = 8, 128 + kv_lora_rank, qk_rope_head_dim = 0, 0 + store_dtype = torch.bfloat16 + tokens_per_page = 64 + + file_path = "/data/test.bin" + file_size = 1 << 40 + numjobs = 16 + bytes_per_page = 16 << 20 + entries = 8 + dtype = store_dtype + hicache_hf3fs = HiCacheHF3FS( + rank=0, + file_path=file_path, + file_size=file_size, + numjobs=numjobs, + bytes_per_page=bytes_per_page, + entries=entries, + dtype=dtype, + metadata_client=Hf3fsLocalMetadataClient(), + ) + + numel = 2 * tokens_per_page * layer_num * head_num * head_dim + assert numel * dtype.itemsize == bytes_per_page + + num_page = 128 + values = [torch.randn((numel,)).to(dtype=dtype) for _ in tqdm(range(num_page))] + + iteration = 100 + + for i in tqdm(range(iteration), desc="Benchmarking write (GB/s)"): + keys = [f"{j}" for j in range(i * num_page, (i + 1) * num_page)] + ok = hicache_hf3fs.batch_set(keys, values) + assert ok + + read_keys, read_results = [], [] + for i in tqdm(range(iteration), desc="Benchmarking read (GB/s)"): + keys = random.sample( + list(hicache_hf3fs.metadata_client.rank_metadata.key_to_index.keys()), + num_page, + ) + results = hicache_hf3fs.batch_get(keys) + read_keys.extend(keys) + read_results.extend(results) + assert all([r is not None for r in results]) + + for key, result in tqdm(zip(read_keys, read_results)): + assert torch.allclose(values[int(key) % num_page], result, atol=1e-3) + + hicache_hf3fs.close() + + +def main(): + logging.basicConfig(level=logging.INFO) + test() + bench() + allclose() + + +if __name__ == "__main__": + main() diff --git a/benchmark/hf3fs/bench_zerocopy.py b/benchmark/hf3fs/bench_zerocopy.py new file mode 100644 index 000000000..bfa7bff0e --- /dev/null +++ b/benchmark/hf3fs/bench_zerocopy.py @@ -0,0 +1,140 @@ +import threading +import time + +import torch +from tqdm import tqdm + +from sglang.srt.distributed import ( + get_world_group, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.managers.cache_controller import ( + HiCacheController, + PrefetchOperation, + StorageOperation, +) +from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.srt.mem_cache.memory_pool_host import MHATokenToKVPoolHost + +init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="gloo", +) + +initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, +) + +group = get_world_group().cpu_group + +max_total_num_tokens = 524288 +page_size = 64 +kv_cache_dtype = torch.bfloat16 +layer_num = 64 +head_num, head_dim = 8, 128 +device = "cuda" +hicache_ratio = 2 +hicache_size = 0 +hicache_mem_layout = "page_first" +# hicache_mem_layout = "layer_first" +hicache_write_policy = "write_through" +hicache_io_backend = "kernel" +hicache_storage_backend = "hf3fs" +prefetch_threshold = 256 + +op_size = 1024 +op_num = 16 + +token_to_kv_pool = MHATokenToKVPool( + max_total_num_tokens, + page_size=page_size, + dtype=kv_cache_dtype, + head_num=head_num, + head_dim=head_dim, + layer_num=layer_num, + device=device, + enable_memory_saver=True, +) + +token_to_kv_pool_allocator = TokenToKVPoolAllocator( + max_total_num_tokens, + dtype=kv_cache_dtype, + device=device, + kvcache=token_to_kv_pool, + need_sort=False, +) + +kv_cache = token_to_kv_pool_allocator.get_kvcache() +token_to_kv_pool_host = MHATokenToKVPoolHost( + kv_cache, + hicache_ratio, + hicache_size, + page_size, + hicache_mem_layout, +) + +load_cache_event = threading.Event() +cache_controller = HiCacheController( + token_to_kv_pool_allocator, + token_to_kv_pool_host, + page_size, + group, + load_cache_event=load_cache_event, + write_policy=hicache_write_policy, + io_backend=hicache_io_backend, + storage_backend=hicache_storage_backend, + prefetch_threshold=prefetch_threshold, +) + +operations = [ + StorageOperation( + torch.tensor(list(range(i, i + op_size))), + list(range(i, i + op_size)), + hash_value=[f"{j}" for j in range(i, i + op_size, page_size)], + ) + for i in tqdm(range(0, op_num * op_size, op_size)) +] + +tik = time.monotonic() +if hicache_mem_layout == "page_first": + for operation in operations: + cache_controller.zerocopy_page_backup(operation, batch_size=128) +elif hicache_mem_layout == "layer_first": + for operation in operations: + cache_controller.generic_page_backup(operation, batch_size=128) +tok = time.monotonic() +print(f"{tok-tik:.6f} s") + +operations = [ + PrefetchOperation( + f"{i}", + torch.tensor(list(range(i, i + op_size))), + list(range(i, i + op_size)), + f"{i}", + ) + for i in tqdm(range(0, op_num * op_size, op_size)) +] + +for operation in operations: + operation.hash_value = [ + f"{j}" + for j in range( + int(operation.last_hash), int(operation.last_hash) + op_size, page_size + ) + ] + +tik = time.monotonic() +if hicache_mem_layout == "page_first": + for operation in operations: + cache_controller.zerocopy_page_transfer(operation, batch_size=128) +elif hicache_mem_layout == "layer_first": + for operation in operations: + cache_controller.generic_page_transfer(operation, batch_size=128) +tok = time.monotonic() +print(f"{tok-tik:.6f} s") diff --git a/benchmark/hicache/README.md b/benchmark/hicache/README.md new file mode 100644 index 000000000..f2aa6ff05 --- /dev/null +++ b/benchmark/hicache/README.md @@ -0,0 +1,91 @@ +## Run synthetic multi-turn benchmark + +``` +# SGLang server with radix cache disabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --disable-radix-cache + +# SGLang server with radix cache on and first-come-first-serve policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --schedule-policy fcfs + +# The default SGLang server with radix cache on and long-prefix-match policy +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 + +# SGLang server with hierarchical radix cache enabled +python -m sglang.launch_server --model-path Qwen/Qwen2.5-14B-Instruct --port 30000 --enable-hierarchical-cache + +``` + +``` +python bench_multiturn.py --model-path Qwen/Qwen2.5-14B-Instruct +``` + +Note: The performance gain of hierarchical caching depends on the ratio of reusable tokens to GPU memory capacity. The more tokens to be reused, the larger the model, and the more constrained the GPU memory size, the greater the benefit one can expect from hierarchical caching. + + +# Benchmark with more datasets +## Download Dataset +```bash +./download.sh {sharegpt|ultragpt|loogle|nextqa|all} +``` +This script will automatically download the required dataset to the current working directory + +## Multiturn Benchmark +### Supported Datasets +- sharegpt +- ultrachat +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-multiturn --disable-shuffle +``` +This uses `mistralai/Mistral-7B-Instruct-v0.3` model with `sglang` as backend. The dataset +is `longdep_qa.json`. We send `10 conversations` with `10 req/s` to port 8001. We enable +multiturn chat without shuffling the order of conversations (i.e. following the original +order in the dataset file). + +### Note: +The requests of multiple conversations are sent in a round robin fashion. +For example, if we have 3 conversations A, B, C whose rounds are `[2, 3, 4]` correspondingly, +multiturn chat will send the requests to the backend in the following order: `[A1, B1, C1, A2, B2, C2, B3, C3, C4]` +This has implications on the cache reuse patterns: the cache reuse distance is the largest +under this request pattern (which means a prefix-aware local scheduler in the backend can +yield the most benefit compared to a FIFO scheduler) + +## Shared Prefix Benchmark +### Supported Datasets +- loogle +### Example Usage: +```bash +python3 bench_serving.py --model mistralai/Mistral-7B-Instruct-v0.3 --backend sglang \ +--dataset-path longdep_qa.json --dataset-name loogle --request-rate 10 --num-prompts 10 \ +--port 8001 --enable-shared-prefix --disable-shuffle +``` +### Note: +Shared Prefix benchmark sends the questions for the same prompt together. For example, +if we have 3 shared prefix A, B, C, which have [2, 3, 4] questions correspondingly, +the shared prefix benchmark will send the requests to the +backend in the following order: `[A+Q1, A+Q2, B+Q1, B+Q2, B+Q3, C+Q1, C+Q2, C+Q3]`. + + +## Multi Modality Benchmark (WIP) +### Supported Datasets: +- nextqa +### Example Usage: +```bash +Server: +python3 -m sglang.launch_server --model-path lmms-lab/LLaVA-NeXT-Video-7B --tp 2 --dp 1 --port 8001 \ +--host 0.0.0.0 --mem-fraction-static 0.9 --tokenizer-path llava-hf/llava-1.5-7b-hf \ +--json-model-override-args "{\"architectures\": [\"LlavaVidForCausalLM\"], \"model_type\":\"llava\", \"mm_spatial_pool_stride\":2}" + +Client: +python3 bench_serving.py --model lmms-lab/LLaVA-NeXT-Video-7B --backend sglang --dataset-path \ +NExTVideo --dataset-name nextqa --request-rate 10 --num-prompts 1 --disable-shuffle --port 8001 \ --enable-multiturn --max-frames 16 --tokenizer llava-hf/llava-1.5-7b-hf --fixed-output-len 2048 +``` +Note: for the server args, `tokenizer-path`, overriding architecture are necessary. + +## Supported Backend +- sglang (oai) +- vllm (oai) +- lmdeploy (oai) diff --git a/benchmark/hicache/bench_long_context.py b/benchmark/hicache/bench_long_context.py new file mode 100644 index 000000000..eed0ae5dc --- /dev/null +++ b/benchmark/hicache/bench_long_context.py @@ -0,0 +1,101 @@ +import json +import queue +import time + +import requests +from bench_multiturn import ( + ReadyQueue, + WorkloadGenerator, + gen_payload, + log_to_jsonl_file, + parse_args, +) +from tqdm.asyncio import tqdm + +from sglang.bench_serving import get_tokenizer + + +class ContextWorkloadGenerator(WorkloadGenerator): + def __init__(self, args): + # Construct the base URL for requests + self.baseurl = f"http://{args.host}:{args.port}/" + self.url = self.baseurl + "generate" + + self.tokenizer = get_tokenizer(args.model_path) + self.distribution = args.distribution + self.request_rate = args.request_rate + self.start_time = None + self.finished_time = None + + self.sent_requests = 0 + self.completed_requests = 0 + + self.dataset = json.load(open(args.dataset_path)) + num_requests = min(args.num_clients, len(self.dataset["queries"])) + + init_requests = [] + for i in range(num_requests): + context_id = self.dataset["queries"][i]["context"] + init_requests.append( + ( + i, + gen_payload( + self.dataset["contexts"][context_id] + + self.dataset["queries"][i]["question"], + len( + self.tokenizer( + self.dataset["queries"][i]["reference_answer"] + )["input_ids"] + ), + ), + ) + ) + self.ready_queue = ReadyQueue(init_requests=init_requests) + + self.response_queue = queue.Queue() + self.pbar = tqdm(total=num_requests) + self.performance_metrics = { + "ttft": [], + "latency": [], + "itl": [], + "prompt_len": [], + "cached_tokens": [], + "generated_len": [], + } + + self.max_parallel = args.max_parallel + self.logfile = args.log_file + + def response_handler(self): + while True: + try: + client_id, response = self.response_queue.get( + timeout=10 + ) # Block until response is available + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["itl"].extend(response.itl) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.performance_metrics["generated_len"].append(response.generated_len) + self.completed_requests += 1 + + except queue.Empty: + if self.pbar.n == self.pbar.total: + break + + +if __name__ == "__main__": + args = parse_args() + args.num_rounds = 1 + args.max_parallel = 24 + flush_cache_url = f"http://{args.host}:{args.port}/flush_cache" + + for request_rate in [24, 16, 12, 8, 4, 2, 1]: + args.request_rate = request_rate + requests.post(flush_cache_url) + time.sleep(1) + performance_data = ContextWorkloadGenerator(args).run() + log_to_jsonl_file(performance_data, args.log_file, args.tag) diff --git a/benchmark/hicache/bench_mix.py b/benchmark/hicache/bench_mix.py new file mode 100644 index 000000000..cfd25bc40 --- /dev/null +++ b/benchmark/hicache/bench_mix.py @@ -0,0 +1,567 @@ +import argparse +import asyncio +import json +import logging +import os +import queue +import random +import threading +import time +from dataclasses import dataclass +from functools import wraps + +import aiohttp + +from sglang.bench_serving import ( + RequestFuncOutput, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +# Set up logger +logger = logging.getLogger(__name__) + +# Set up JSONL file for debug logging +debug_log_file = None +# Create a lock for thread-safe debug log writing +debug_log_lock = threading.Lock() + + +def write_debug_log(data): + global debug_log_file + + """Write debug information to a JSONL file""" + if debug_log_file is None: + return + + # Acquire lock for thread-safe writing + with debug_log_lock: + # Write as JSONL (JSON Line format) + debug_log_file.write(json.dumps(data) + "\n") + debug_log_file.flush() + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to benchmark concurrent requests to a server." + ) + parser.add_argument( + "--model-path", + type=str, + default="/data/models/Qwen3-0.6B", + help="model path compatible with Hugging Face Transformers", + ) + parser.add_argument( + "--dataset-path", + type=str, + default="/data/models/ShareGPT_V3_unfiltered_cleaned_split/ShareGPT_V3_unfiltered_cleaned_split.json", + help="local dataset to sample tokens from", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server hostname or IP (default: localhost)", + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="Server port (default: 30000)", + ) + parser.add_argument( + "--duration", + type=int, + default=600, + help="Duration to run the benchmark in seconds (default: 300 seconds)", + ) + parser.add_argument( + "--log-level", + type=str, + default="info", + choices=["debug", "info"], + help="Set the logging level (default: info)", + ) + parser.add_argument( + "--debug-log-file", + type=str, + default="debug.log.jsonl", + help="File to write debug logs in JSONL format", + ) + return parser.parse_args() + + +def load_config(): + config_path = os.getenv("CONFIG_PATH") + if not config_path: + raise ValueError("Environment variable 'CONFIG_PATH' is not set.") + + with open(config_path, "r") as f: + config = json.load(f) + + required_keys = [ + "num_rounds", + "num_clients", + "round_ratios", + "mean_new_tokens_per_round", + "mean_return_tokens_per_round", + "mean_inter_round_interval", + ] + + for key in required_keys: + if key not in config: + raise KeyError(f"Missing required configuration key: {key}") + + num_rounds = config["num_rounds"] + assert len(config["round_ratios"]) == num_rounds + assert len(config["mean_new_tokens_per_round"]) == num_rounds + assert len(config["mean_return_tokens_per_round"]) == num_rounds + assert len(config["mean_inter_round_interval"]) == num_rounds + + print(config) + + return config + + +@dataclass +class UserData: + user_id: int + current_round: int + total_rounds: int + prompt: str + return_tokens: int + start: int + + +def synchronized(): + def _decorator(func): + @wraps(func) + def wrapper(self, *args, **kwargs): + with self.lock: + return func(self, *args, **kwargs) + + return wrapper + + return _decorator + + +class UserGenerator: + def __init__(self, config, model_path, dataset_path): + self.tokenizer_path = model_path + self.tokenizer = get_tokenizer(self.tokenizer_path) + self.dataset_path = dataset_path + + self.user_id = 0 + self.lock = threading.Lock() + + self.num_rounds = config["num_rounds"] + + self.cumulative_ratios = [ + sum(config["round_ratios"][: i + 1]) + for i in range(len(config["round_ratios"])) + ] + self.mean_new_tokens_per_round = config["mean_new_tokens_per_round"] + self.mean_return_tokens_per_round = config["mean_return_tokens_per_round"] + self.mean_inter_round_interval = config["mean_inter_round_interval"] + + self.sigma = 100 + self.range_ratio = 0.8 + assert self.range_ratio <= 1 + + self.candidate_inputs = [ + [ + r + for r in sample_random_requests( + input_len=( + self.mean_new_tokens_per_round[i] * (2 - self.range_ratio) + ), + output_len=( + self.mean_return_tokens_per_round[i] * (2 - self.range_ratio) + ), + num_prompts=config["num_clients"], + range_ratio=self.range_ratio / (2 - self.range_ratio), + tokenizer=self.tokenizer, + dataset_path=self.dataset_path, + random_sample=False, + ) + ] + for i in range(self.num_rounds) + ] + + self.multiturn_queue = [] + + self.user_stats = [0 for _ in range(self.num_rounds)] + self.input_stats = [[0, 0] for _ in range(self.num_rounds)] + self.output_stats = [[0, 0] for _ in range(self.num_rounds)] + + def gen(self): + user_id = self.user_id + self.user_id += 1 + + rand_ratio = random.randint(0, self.cumulative_ratios[-1]) + i = len(self.cumulative_ratios) + for idx, cumulative_ratio in enumerate(self.cumulative_ratios): + if rand_ratio >= cumulative_ratio: + continue + else: + i = idx + 1 + break + total_rounds = i + current_round = 0 + + candidate_input = random.sample(self.candidate_inputs[current_round], 1)[0] + self.input_stats[0][0] += candidate_input.prompt_len + self.input_stats[0][1] += 1 + prompt = f"{user_id} " + candidate_input.prompt + return_tokens = int( + random.gauss(self.mean_return_tokens_per_round[current_round], self.sigma) + ) + if return_tokens <= 0: + return_tokens = self.mean_return_tokens_per_round[current_round] + start = 0 + + user_data = UserData( + user_id, current_round, total_rounds, prompt, return_tokens, start + ) + + self.user_stats[total_rounds - 1] += 1 + + return user_data + + @synchronized() + def push(self, user_data, generated_text, len_itl): + self.output_stats[user_data.current_round][0] += len_itl + 1 + self.output_stats[user_data.current_round][1] += 1 + user_data.current_round += 1 + if user_data.current_round >= user_data.total_rounds: + return + + candidate_input = random.sample( + self.candidate_inputs[user_data.current_round], 1 + )[0] + self.input_stats[user_data.current_round][0] += candidate_input.prompt_len + self.input_stats[user_data.current_round][1] += 1 + user_data.prompt += generated_text + candidate_input.prompt + user_data.return_tokens = int( + random.gauss( + self.mean_return_tokens_per_round[user_data.current_round], self.sigma + ) + ) + if user_data.return_tokens <= 0: + user_data.return_tokens = self.mean_return_tokens_per_round[ + user_data.current_round + ] + interval = random.gauss( + self.mean_inter_round_interval[user_data.current_round], self.sigma + ) + if interval <= 0: + interval = self.mean_inter_round_interval[user_data.current_round] + user_data.start = time.perf_counter() + interval + + if len(self.multiturn_queue) == 0: + self.multiturn_queue.append(user_data) + else: + i = len(self.multiturn_queue) + for idx, d in enumerate(self.multiturn_queue): + if user_data.start < d.start: + i = idx + break + self.multiturn_queue.insert(idx, user_data) + + @synchronized() + def pop(self): + if ( + len(self.multiturn_queue) + and time.perf_counter() > self.multiturn_queue[0].start + ): + return self.multiturn_queue.pop(0) + return self.gen() + + +def gen_payload(prompt, output_len): + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "stream": True, + "stream_options": {"include_usage": True}, + "lora_path": "", + "return_logprob": False, + "logprob_start_len": -1, + } + return payload + + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +async def async_request_sglang_generate( + user_data, + url, + atomic_counter, +): + """ + Sends a streaming request to the server. Gathers text token-by-token. + """ + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + headers = {} + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + output = RequestFuncOutput() + payload = gen_payload(user_data.prompt, user_data.return_tokens) + write_debug_log({"timestamp": st, "user_data": user_data.__dict__}) + + try: + async with session.post(url=url, json=payload, headers=headers) as response: + if response.status == 200: + prompt_tokens = 0 + cached_tokens = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + if data.get("text"): + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + prompt_tokens = (data.get("meta_info") or {}).get( + "prompt_tokens", 0 + ) + cached_tokens = (data.get("meta_info") or {}).get( + "cached_tokens", 0 + ) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text = data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.prompt_len = prompt_tokens + output.cached_tokens = cached_tokens + else: + output.error = response.reason or "" + output.success = False + except Exception as e: + output.success = False + output.error = str(e) + print(f"Request failed: {e}") + + atomic_counter.increment(1) + return output + + +class AtomicCounter: + def __init__(self, initial_value=0): + self._value = initial_value + self.lock = threading.Lock() + + @synchronized() + def increment(self, amount=1): + self._value += amount + + @synchronized() + def get(self): + return self._value + + +class WorkloadGenerator: + def __init__(self, args): + config = load_config() + user_generator = UserGenerator( + config, + args.model_path, + args.dataset_path, + ) + + self.url = f"http://{args.host}:{args.port}/generate" + + self.tokenizer = user_generator.tokenizer + self.start_time = None + self.finished_time = None + self.duration = args.duration + self.done = False + + self.sent_requests = 0 + self.completed_requests = 0 + + self.user_generator = user_generator + self.response_queue = queue.Queue() + self.performance_metrics = { + "ttft": [], + "latency": [], + "prompt_len": [], + "cached_tokens": [], + } + self.max_parallel = config["num_clients"] + + self.atomic_counter = AtomicCounter() + + async def handle_request(self, user_data): + try: + response = await async_request_sglang_generate( + user_data, self.url, self.atomic_counter + ) + self.response_queue.put((user_data, response)) + except Exception as e: + print(f"Request failed: {e}") + self.completed_requests += 1 + + def request_sender(self): + async def request_loop(): + while True: + if self.sent_requests - self.completed_requests < self.max_parallel: + new_request = self.user_generator.pop() + if new_request: + asyncio.create_task(self.handle_request(new_request)) + self.sent_requests += 1 + else: + await asyncio.sleep(0.05) + continue + + if time.perf_counter() - self.start_time > self.duration: + self.done = True + break + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(request_loop()) + loop.close() + + def response_handler(self): + while True: + try: + user_data, response = self.response_queue.get(timeout=10) + logger.info( + f"{((time.perf_counter()-self.start_time)/self.duration*100):.2f}%" + ) + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + + self.user_generator.push( + user_data, response.generated_text, len(response.itl) + ) + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.completed_requests += 1 + self.finished_time = time.perf_counter() + + except queue.Empty: + if self.done: + break + except ValueError as e: + print(f"Error processing response for client {user_data}: {e}") + continue + + def run(self): + request_thread = threading.Thread(target=self.request_sender, daemon=True) + response_thread = threading.Thread(target=self.response_handler, daemon=True) + + self.start_time = time.perf_counter() + request_thread.start() + response_thread.start() + + request_thread.join() + response_thread.join() + + performance_data = { + "summary": { + "total_requests": len(self.performance_metrics["ttft"]), + "average_ttft": sum(self.performance_metrics["ttft"]) + / len(self.performance_metrics["ttft"]), + "p90_ttft": sorted(self.performance_metrics["ttft"])[ + int(0.9 * len(self.performance_metrics["ttft"])) + ], + "median_ttft": sorted(self.performance_metrics["ttft"])[ + len(self.performance_metrics["ttft"]) // 2 + ], + "average_latency": sum(self.performance_metrics["latency"]) + / len(self.performance_metrics["latency"]), + "p90_latency": sorted(self.performance_metrics["latency"])[ + int(0.9 * len(self.performance_metrics["latency"])) + ], + "median_latency": sorted(self.performance_metrics["latency"])[ + len(self.performance_metrics["latency"]) // 2 + ], + "throughput": self.atomic_counter.get() + / (self.finished_time - self.start_time), + "cache_hit_rate": ( + 0 + if sum(self.performance_metrics["prompt_len"]) == 0 + else sum(self.performance_metrics["cached_tokens"]) + / sum(self.performance_metrics["prompt_len"]) + ), + }, + } + print("All requests completed") + print("Performance metrics summary:") + print(f" Total requests: {performance_data['summary']['total_requests']}") + print(f" Average TTFT: {performance_data['summary']['average_ttft']:.2f}") + print(f" P90 TTFT: {performance_data['summary']['p90_ttft']:.2f}") + print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") + print( + f" Average latency: {performance_data['summary']['average_latency']:.2f}" + ) + print(f" P90 latency: {performance_data['summary']['p90_latency']:.2f}") + print(f" Median latency: {performance_data['summary']['median_latency']:.2f}") + print( + f" Throughput: {performance_data['summary']['throughput']:.2f} requests per second" + ) + print(f" Cache Hit Rate: {performance_data['summary']['cache_hit_rate']:.6f}") + + user_stats = self.user_generator.user_stats + input_stats = self.user_generator.input_stats + output_stats = self.user_generator.output_stats + print(f"round_ratios: {user_stats}") + print( + f"mean_new_tokens_per_round: {[int(a/b) if b > 0 else 0 for a, b in input_stats]}" + ) + print( + f"mean_return_tokens_per_round: {[int(a/b) if b > 0 else 0 for a, b in output_stats]}" + ) + return performance_data + + +def main(): + global debug_log_file + + args = parse_args() + if args.log_level == "debug": + logging.basicConfig(level=logging.DEBUG) + logger.info("use log_level debug") + # Initialize debug log file + debug_log_file = open(args.debug_log_file, "w") + else: + logging.basicConfig(level=logging.INFO) + logger.info("use log_level info") + performance_data = WorkloadGenerator(args).run() + + # Close debug log file if it was opened + if debug_log_file: + debug_log_file.close() + + +if __name__ == "__main__": + main() diff --git a/benchmark/hicache/bench_mix.sh b/benchmark/hicache/bench_mix.sh new file mode 100755 index 000000000..5ff6dca94 --- /dev/null +++ b/benchmark/hicache/bench_mix.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.12/dist-packages:/usr/local/lib/python3.12/dist-packages/torch/lib +rm -rf nohup.out && \ +nohup python3 -m sglang.launch_server \ + --attention-backend triton \ + --model-path /code/models/Qwen3-32B/ \ + --log-level info \ + --tp 4 --mem-frac 0.25 \ + --host 0.0.0.0 --port 33301 \ + --enable-metrics --enable-cache-report \ + --page-size 64 \ + --enable-hierarchical-cache \ + --hicache-ratio 2.5 --hicache-size 0 \ + --hicache-io-backend kernel \ + --hicache-mem-layout layer_first \ + --hicache-write-policy write_through \ + & + +################################################## + +export CONFIG_PATH=/tmp/bench_mix_config.json + +# num_clients: Maximum number of concurrent client requests to be simulated +# round_ratios: Distribution of requests across rounds. Given sum(round_ratios) total requests, +# round_ratios[i] denotes the number of requests that will execute for (i+1) rounds +echo '{ + "num_rounds": 10, + "num_clients": 60, + "round_ratios": [50, 25, 15, 15, 10, 10, 9, 8, 7, 6], + "mean_new_tokens_per_round": [1000, 400, 350, 300, 280, 260, 240, 220, 210, 200], + "mean_return_tokens_per_round": [100, 100, 100, 100, 100, 100, 100, 100, 100, 100], + "mean_inter_round_interval": [30, 30, 30, 30, 30, 30, 30, 30, 30, 30] +}' > ${CONFIG_PATH} + +rm -rf bench_mix.out && \ +nohup python3 /sgl-workspace/sglang/benchmark/hicache/bench_mix.py \ + --model-path /code/models/Qwen3-32B/ \ + --dataset-path /code/models/ShareGPT_V3_unfiltered_cleaned_split.json \ + --port 33301 \ + --duration 600 \ +> bench_mix.out & diff --git a/benchmark/hicache/bench_multiturn.py b/benchmark/hicache/bench_multiturn.py new file mode 100644 index 000000000..a3e8b0d74 --- /dev/null +++ b/benchmark/hicache/bench_multiturn.py @@ -0,0 +1,517 @@ +import argparse +import asyncio +import json +import queue +import random +import threading +import time +from datetime import datetime +from typing import Optional + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm + +from sglang.bench_serving import ( + RequestFuncOutput, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Script to benchmark concurrent requests to a server." + ) + parser.add_argument( + "--num-clients", + type=int, + default=256, + help="Number of concurrent clients", + ) + parser.add_argument( + "--max-parallel", + type=int, + default=128, + help="Maximum number of parallel requests", + ) + parser.add_argument( + "--request-length", + type=int, + default=512, + help="Length of each new request", + ) + parser.add_argument( + "--output-length", + type=int, + default=64, + help="Length of each output", + ) + parser.add_argument( + "--num-rounds", + type=int, + default=5, + help="Number of rounds per client", + ) + parser.add_argument( + "--distribution", + type=str, + default="poisson", + choices=["poisson", "uniform"], + help="Distribution type for request intervals (poisson or uniform)", + ) + parser.add_argument( + "--request-rate", + type=float, + default=1.0, + help="Average number of requests per second", + ) + parser.add_argument( + "--host", + type=str, + default="localhost", + help="Server hostname or IP (default: localhost)", + ) + parser.add_argument( + "--port", + type=int, + default=30000, + help="Server port (default: 30000)", + ) + parser.add_argument( + "--model-path", + type=str, + default="meta-llama/Llama-3.1-8B-Instruct", + help="model path compatible with Hugging Face Transformers", + ) + parser.add_argument( + "--dataset-path", + type=str, + default="", + help="local dataset to sample tokens from", + ) + parser.add_argument( + "--log-file", + type=str, + default="performance_metrics.jsonl", + help="File to log performance metrics", + ) + parser.add_argument( + "--disable-auto-run", + action="store_true", + help="If set, disable automatically testing with a range of request rates.", + ) + + parser.add_argument( + "--disable-random-sample", + action="store_true", + help="If set, disable random sampling of requests from the ShareGPT dataset.", + ) + parser.add_argument( + "--sub-question-input-length", + type=int, + default=0, + help="Length of the sub question input for each request, if set 0 use request_length", + ) + parser.add_argument( + "--ready-queue-policy", + type=str, + default="random", + help="Policy for popping requests from the ready queue (random or fifo)", + ) + parser.add_argument( + "--tag", + type=str, + default="", + help="Tag of a certain run in the log file", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--lora-path", + type=str, + default="", + help="String of LoRA path. Currently we only support benchmarking on a single LoRA adaptor.", + ) + return parser.parse_args() + + +async def async_request_sglang_generate( + payload, + url, + pbar: Optional[tqdm] = None, +): + """ + Sends a streaming request to the server. Gathers text token-by-token. + """ + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + headers = {} + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + output = RequestFuncOutput() + + try: + async with session.post(url=url, json=payload, headers=headers) as response: + if response.status == 200: + prompt_tokens = 0 + cached_tokens = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + if data["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + prompt_tokens = (data.get("meta_info") or {}).get( + "prompt_tokens", 0 + ) + cached_tokens = (data.get("meta_info") or {}).get( + "cached_tokens", 0 + ) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text = data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.prompt_len = prompt_tokens + output.cached_tokens = cached_tokens + output.generated_len = len(output.itl) + 1 + else: + output.error = response.reason or "" + output.success = False + except Exception as e: + output.success = False + output.error = str(e) + print(f"Request failed: {e}") + + if pbar: + pbar.update(1) + return output + + +def gen_payload(prompt, output_len, lora_path=""): + payload = { + "text": prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "stream": True, + "stream_options": {"include_usage": True}, + "lora_path": lora_path, + "return_logprob": False, + "logprob_start_len": -1, + } + return payload + + +def log_to_jsonl_file(data, file_path="performance_metrics.jsonl", tag=""): + """Append the data with a timestamp and tag to the specified JSONL file.""" + timestamped_data = {"timestamp": datetime.now().isoformat(), "tag": tag, **data} + try: + with open(file_path, "a") as file: + file.write( + json.dumps(timestamped_data) + "\n" + ) # Write as a single line in JSONL format + except IOError as e: + print(f"Error writing to JSONL file: {e}") + + +class ReadyQueue: + """ + Thread-safe queue that can pop requests in different orders based on given policy. + """ + + def __init__(self, init_requests=None, policy="random"): + self.lock = threading.Lock() + self.requests = init_requests or [] + self.policy = policy + + def append(self, item): + with self.lock: + self.requests.append(item) + + def pop(self): + with self.lock: + if not self.requests: + return None + if self.policy == "random": + index = random.randrange(len(self.requests)) + return self.requests.pop(index) + elif self.policy == "fifo": + return self.requests.pop(0) + else: + # todo, varying thinking time of clients + raise ValueError(f"{self.policy} not implemented") + + +class WorkloadGenerator: + def __init__(self, args): + # Construct the base URL for requests + self.url = f"http://{args.host}:{args.port}/generate" + + self.tokenizer = get_tokenizer(args.model_path) + self.distribution = args.distribution + self.request_rate = args.request_rate + self.start_time = None + self.finished_time = None + + self.sent_requests = 0 + self.completed_requests = 0 + + self.candidate_inputs = sample_random_requests( + input_len=args.request_length, + output_len=args.output_length, + num_prompts=args.num_clients, + range_ratio=1.0, + tokenizer=self.tokenizer, + dataset_path=args.dataset_path, + random_sample=not args.disable_random_sample, + ) + self.candidate_inputs = [i.prompt for i in self.candidate_inputs] + + if args.sub_question_input_length != 0: + sub_question_input_length = args.sub_question_input_length + else: + sub_question_input_length = args.request_length + + self.sub_question_inputs = sample_random_requests( + input_len=sub_question_input_length, + output_len=args.output_length, + num_prompts=args.num_clients * max(args.num_rounds - 1, 1), + range_ratio=1.0, + tokenizer=self.tokenizer, + dataset_path=args.dataset_path, + random_sample=not args.disable_random_sample, + ) + + init_requests = [ + ( + i, + gen_payload( + self.candidate_inputs[i], args.output_length, args.lora_path + ), + ) + for i in range(args.num_clients) + ] + self.client_records = { + i: {"round": 0, "history": init_requests[i][1]["text"]} + for i in range(args.num_clients) + } + self.ready_queue = ReadyQueue( + init_requests=init_requests, policy=args.ready_queue_policy + ) + self.candidate_inputs = self.candidate_inputs[args.num_clients :] + + self.response_queue = queue.Queue() + self.pbar = tqdm(total=args.num_clients * args.num_rounds) + self.performance_metrics = { + "ttft": [], + "latency": [], + "prompt_len": [], + "cached_tokens": [], + "generated_len": [], + } + self.num_rounds = args.num_rounds + self.max_parallel = args.max_parallel + self.output_length = args.output_length + + async def handle_request(self, item): + try: + client_id, payload = item + response = await async_request_sglang_generate(payload, self.url, self.pbar) + if self.pbar.n == self.pbar.total: + self.finished_time = time.perf_counter() + self.response_queue.put((client_id, response)) + except Exception as e: + print(f"Request failed: {e}") + + def request_sender(self): + async def request_loop(): + while True: + if self.sent_requests - self.completed_requests < self.max_parallel: + new_request = self.ready_queue.pop() + if new_request: + asyncio.create_task(self.handle_request(new_request)) + self.sent_requests += 1 + else: + await asyncio.sleep(0.05) + continue + + if self.pbar.n == self.pbar.total: + break + + # Calculate Poisson-distributed wait time + if self.distribution == "poisson": + sleep_time = random.expovariate(self.request_rate) + elif self.distribution == "uniform": + avg_interval = ( + 1.0 / self.request_rate if self.request_rate > 0 else 1.0 + ) + sleep_time = random.uniform(0, 2 * avg_interval) + else: + raise ValueError("Invalid distribution type") + await asyncio.sleep(sleep_time) # Wait before sending the next request + + # Create and run the event loop for asynchronous requests + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(request_loop()) + loop.close() + + def response_handler(self): + while True: + try: + client_id, response = self.response_queue.get( + timeout=10 + ) # Block until response is available + if not response.success: + raise ValueError(f"Request failed with error: {response.error}") + self.client_records[client_id]["history"] += response.generated_text + self.client_records[client_id]["round"] += 1 + self.performance_metrics["ttft"].append(response.ttft) + self.performance_metrics["latency"].append(response.latency) + self.performance_metrics["prompt_len"].append(response.prompt_len) + self.performance_metrics["cached_tokens"].append(response.cached_tokens) + self.performance_metrics["generated_len"].append(response.generated_len) + self.completed_requests += 1 + + if self.client_records[client_id]["round"] < self.num_rounds: + # append new request to client's history + self.client_records[client_id][ + "history" + ] += self.sub_question_inputs.pop().prompt + self.ready_queue.append( + ( + client_id, + gen_payload( + self.client_records[client_id]["history"], + self.output_length, + args.lora_path, + ), + ) + ) + except queue.Empty: + if self.pbar.n == self.pbar.total: + break + except ValueError as e: + print(f"Error processing response for client {client_id}: {e}") + continue + + def run(self): + request_thread = threading.Thread(target=self.request_sender, daemon=True) + response_thread = threading.Thread(target=self.response_handler, daemon=True) + + self.start_time = time.perf_counter() + request_thread.start() + response_thread.start() + + request_thread.join() + response_thread.join() + self.pbar.close() + + duration = self.finished_time - self.start_time + performance_data = { + "summary": { + "total_requests": len(self.performance_metrics["ttft"]), + "request_rate": self.request_rate, + "average_ttft": sum(self.performance_metrics["ttft"]) + / len(self.performance_metrics["ttft"]), + "p90_ttft": sorted(self.performance_metrics["ttft"])[ + int(0.9 * len(self.performance_metrics["ttft"])) + ], + "median_ttft": sorted(self.performance_metrics["ttft"])[ + len(self.performance_metrics["ttft"]) // 2 + ], + "average_latency": sum(self.performance_metrics["latency"]) + / len(self.performance_metrics["latency"]), + "p90_latency": sorted(self.performance_metrics["latency"])[ + int(0.9 * len(self.performance_metrics["latency"])) + ], + "median_latency": sorted(self.performance_metrics["latency"])[ + len(self.performance_metrics["latency"]) // 2 + ], + "input_token_throughput": sum(self.performance_metrics["prompt_len"]) + / duration, + "output_token_throughput": sum( + self.performance_metrics["generated_len"] + ) + / duration, + "throughput": self.pbar.total / duration, + "cache_hit_rate": ( + 0 + if sum(self.performance_metrics["prompt_len"]) == 0 + else sum(self.performance_metrics["cached_tokens"]) + / sum(self.performance_metrics["prompt_len"]) + ), + }, + } + print("All requests completed") + print("Performance metrics summary:") + print( + f" Total requests: {performance_data['summary']['total_requests']} at {performance_data['summary']['request_rate']} requests per second" + ) + print(f" Average TTFT: {performance_data['summary']['average_ttft']:.2f}") + print(f" P90 TTFT: {performance_data['summary']['p90_ttft']:.2f}") + print(f" Median TTFT: {performance_data['summary']['median_ttft']:.2f}") + print( + f" Average latency: {performance_data['summary']['average_latency']:.2f}" + ) + print(f" P90 latency: {performance_data['summary']['p90_latency']:.2f}") + print(f" Median latency: {performance_data['summary']['median_latency']:.2f}") + print( + f" Input token throughput: {performance_data['summary']['input_token_throughput']:.2f} tokens per second" + ) + print( + f" Output token throughput: {performance_data['summary']['output_token_throughput']:.2f} tokens per second" + ) + print( + f" Request Throughput: {performance_data['summary']['throughput']:.2f} requests per second" + ) + print(f" Cache Hit Rate: {performance_data['summary']['cache_hit_rate']:.6f}") + return performance_data + + +if __name__ == "__main__": + args = parse_args() + flush_cache_url = f"http://{args.host}:{args.port}/flush_cache" + + random.seed(args.seed) + np.random.seed(args.seed) + + if args.disable_auto_run: + print("Running with specified request rate...") + request_rates = [args.request_rate] + else: + print("Auto-running with different request rates...") + request_rates = [16, 14, 12, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + + for rate in request_rates: + args.request_rate = rate + requests.post(flush_cache_url) + time.sleep(1) + performance_data = WorkloadGenerator(args).run() + log_to_jsonl_file(performance_data, args.log_file, tag=args.tag) diff --git a/benchmark/hicache/bench_serving.py b/benchmark/hicache/bench_serving.py new file mode 100644 index 000000000..e38d0d0ea --- /dev/null +++ b/benchmark/hicache/bench_serving.py @@ -0,0 +1,1029 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +python3 -m sglang.bench_serving --backend sglang --dataset-name random --request-rate-range 1,2,4,8,16,32 --random-input 4096 --random-output 1024 --random-range-ratio 0.125 --multi +""" + +import argparse +import asyncio +import json +import os +import random +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +import requests +from data_processing import MsgContent, SampleOutput, get_dataset +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import get_tokenizer, remove_prefix, set_ulimit + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + +global args + + +@dataclass +class RequestFuncInput: + prompts: List[Tuple[MsgContent, int, int]] + api_url: str + model: str + lora_name: str + extra_request_body: Dict[str, Any] + + # For multiturn chat, store the context + prev_messages: List = field(default_factory=list) + finished_prompts: int = 0 + + +@dataclass +class RequestFuncOutput: + generated_text: List[str] = field(default_factory=list) + prompt_len: List[int] = field(default_factory=list) + output_len: List[int] = field(default_factory=list) + latency: List[float] = field(default_factory=list) + ttft: List[float] = field(default_factory=list) + itl: List[float] = field(default_factory=list) # List of inter-token latencies + + success: bool = False + error: str = "" + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + queue: asyncio.Queue, + tokenizer: PreTrainedTokenizerBase, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + payload = { + "model": request_func_input.model, + "temperature": 0.0, + "best_of": 1, + "stream": not args.disable_stream, + "stream_options": {"include_usage": True}, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", + } + + output = RequestFuncOutput() + + prompt_idx = request_func_input.finished_prompts + messages = request_func_input.prev_messages + prompt, input_len, max_tokens = request_func_input.prompts[prompt_idx] + prompt_len = sum( + prompt[1] + prompt[2] # input_len + output_len + for prompt in request_func_input.prompts[:prompt_idx] + ) + prompt_len += input_len + + # Messages + messages.append( + { + "role": "user", + "content": prompt, + } + ) + payload["messages"] = messages + payload["max_tokens"] = max_tokens + + # output.prompt_len = request_func_input.prompt_len + # print(payload) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + actual_prompt_len = prompt_len - 1 + actual_output_len = 0 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + timestamp = time.perf_counter() + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["usage"] is not None and len(data["usage"]) > 0: + actual_prompt_len = data["usage"]["prompt_tokens"] + actual_output_len = data["usage"]["completion_tokens"] + continue + delta = data["choices"][0]["delta"] + + if delta.get("content", None): + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft.append(ttft) + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + generated_text += delta["content"] + most_recent_timestamp = timestamp + + output.prompt_len.append(actual_prompt_len) # truncate + output.output_len.append(actual_output_len) + output.generated_text.append(generated_text) + output.success = True + output.latency.append(latency) + + # Prepare for the new request + request_func_input.prompts[prompt_idx] = ( + prompt, + input_len, + actual_output_len, # changes from max_tokens to output_len + ) + prompt_idx += 1 + messages.append( + { + "role": "assistant", + "content": generated_text, + } + ) + + # Move the new request to the end of the queue + if prompt_idx < len(request_func_input.prompts): + request_func_input.finished_prompts = prompt_idx + request_func_input.prev_messages = messages + await queue.put(request_func_input) + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, + "vllm": async_request_openai_completions, + "lmdeploy": async_request_openai_completions, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p90_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p90_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p90_itl_ms: float + p99_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + + +async def get_requests( + input_requests_queue: asyncio.Queue, + request_rate: float, + num_actual_requests: int, +) -> AsyncGenerator[RequestFuncInput, None]: + for _ in range(num_actual_requests): + try: + request = await asyncio.wait_for( + input_requests_queue.get(), timeout=300 + ) # Wait for 5 minutes then abort + except Exception as e: + print(f"exception: {e}") + break + + yield request + + if request_rate == float("inf"): + continue + + interval = np.random.exponential(1.0 / request_rate) + await asyncio.sleep(interval) + + +def calculate_metrics( + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + output_success = 0 + for i in range(len(outputs)): + if outputs[i].success: + output_success += 1 + assert len(outputs[i].generated_text) == len(outputs[i].latency) + assert len(outputs[i].generated_text) == len(outputs[i].ttft) + for j in range(len(outputs[i].generated_text)): + output_len = outputs[i].output_len[j] + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode( + outputs[i].generated_text[j], add_special_tokens=False + ) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += outputs[i].prompt_len[j] + if output_len > 1: + tpots.append( + (outputs[i].latency[j] - outputs[i].ttft[j]) / (output_len - 1) + ) + + completed += 1 + itls += outputs[i].itl + ttfts += outputs[i].ttft + e2e_latencies += outputs[i].latency + + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p90_ttft_ms=np.percentile(ttfts or 0, 90) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p90_tpot_ms=np.percentile(tpots or 0, 90) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p90_itl_ms=np.percentile(itls or 0, 90) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: SampleOutput, + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_name: str, + extra_request_body: Dict[str, Any], + profile: bool, + enable_shared_prefix: bool, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, queue, tokenizer, pbar): + if semaphore is None: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + async with semaphore: + return await request_func( + request_func_input=request_func_input, + queue=queue, + tokenizer=tokenizer, + pbar=pbar, + ) + + num_actual_requests = sum(len(r) for r in input_requests) + print(f"Num of shared prefixes or conversations: {len(input_requests)}") + print(f"Num of total requests: {num_actual_requests}") + + # flatten the requests for shared prefix + if enable_shared_prefix: + input_requests = [[r] for requests in input_requests for r in requests] + inputs_requests_queue = asyncio.Queue(maxsize=len(input_requests)) + print("Starting initial single prompt test run...") + # NOTE: Just use the first request of the first conversation for warmup + test_input = RequestFuncInput( + model=model_id, + prompts=input_requests[0][:1], + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + test_output = await request_func( + request_func_input=test_input, queue=inputs_requests_queue, tokenizer=tokenizer + ) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + # Check the states + assert inputs_requests_queue.empty() + + # Flush cache + if "sglang" in backend: + requests.post(base_url + "/flush_cache") + + time.sleep(1.0) + + # Start profiler + if profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + for request in input_requests: + request_func_input = RequestFuncInput( + model=model_id, + prompts=request, + api_url=api_url, + lora_name=lora_name, + extra_request_body=extra_request_body, + ) + inputs_requests_queue.put_nowait(request_func_input) + if ( + not args.enable_multiturn + and not args.enable_shared_prefix + and not args.dataset_name == "generated-shared-prefix" + ): + assert len(input_requests) == num_actual_requests + + pbar = None if disable_tqdm else tqdm(total=num_actual_requests) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_requests( + inputs_requests_queue, request_rate, num_actual_requests + ): + tasks.append( + asyncio.create_task( + limited_request_func( + request_func_input=request, + queue=inputs_requests_queue, + tokenizer=tokenizer, + pbar=pbar, + ) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=base_url + "/stop_profile") + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P90 TTFT (ms):", metrics.p90_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P90 TPOT (ms):", metrics.p90_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P90 ITL (ms):", metrics.p90_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": request_rate, + "max_concurrency": max_concurrency, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "fixed_output_len": args.fixed_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "random": + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p90_ttft_ms": metrics.p90_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p90_tpot_ms": metrics.p90_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p90_itl_ms": metrics.p90_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "lmdeploy": 23333, + "vllm": 8000, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/chat/completions" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + # Dataset compatibility check + if args.enable_multiturn: + # TODO: Support multiturn for random + if args.dataset_name not in ["sharegpt", "ultrachat", "loogle", "nextqa"]: + print( + "Multiturn conversation is only supported for sharegpt, ultrachat, loogle, and nextqa datasets." + ) + sys.exit(1) + + if args.enable_shared_prefix: + if args.dataset_name not in ["loogle", "nextqa"]: + print("Shared prefix is only supported for loogle and nextqa datasets.") + sys.exit(1) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = get_dataset(args, tokenizer) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_name=args.lora_name, + extra_request_body=extra_request_body, + profile=args.profile, + enable_shared_prefix=args.enable_shared_prefix, + ) + ) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "random", + "generated-shared-prefix", + "ultrachat", + "loogle", + "nextqa", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--chat-template", + type=str, + help="The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--fixed-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument( + "--multi", + action="store_true", + help="Use request rate range rather than single value.", + ) + parser.add_argument( + "--request-rate-range", + type=str, + default="2,34,2", + help="Range of request rates in the format start,stop,step. Default is 2,34,2. It also supports a list of request rates, requiring the parameters to not equal three.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--enable-multiturn", + action="store_true", + help="Enable multiturn chat for online serving benchmarking. " + "This option is effective on the following datasets: " + "sharegpt, ultrachat, loogle, nextqa", + ) + parser.add_argument( + "--enable-shared-prefix", + action="store_true", + help="Enable shared prefix for online serving benchmarking. " + "This option is effective on the following datasets: " + "loogle, nextqa", + ) + + parser.add_argument( + "--disable-shuffle", + action="store_true", + help="Disable shuffling datasets. This is useful to generate stable output " + "in benchmarking", + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--lora-name", + type=str, + default=None, + help="The name of LoRA adapter", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + # videos specific + parser.add_argument( + "--max-frames", + type=int, + default=sys.maxsize, + help="The maximum number of frames to extract from each video. " + "This option is specific to the nextqa dataset (video benchmark). ", + ) + args = parser.parse_args() + + if args.enable_multiturn and args.enable_shared_prefix: + parser.error( + "--enable-multiturn and --enable-shared-prefix cannot be set at the same time." + ) + + run_benchmark(args) diff --git a/benchmark/hicache/data_processing.py b/benchmark/hicache/data_processing.py new file mode 100644 index 000000000..8f72a0d95 --- /dev/null +++ b/benchmark/hicache/data_processing.py @@ -0,0 +1,590 @@ +import json +import os +import pickle +import random +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import numpy as np +from nextqa import NExTQALoader + +# from nextqa.video import , VideoPrompt +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + +from sglang.bench_serving import ( + download_and_cache_file, + gen_prompt, + get_gen_prefix_cache_path, +) +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.srt.entrypoints.openai.protocol import ChatCompletionMessageContentPart +from sglang.utils import encode_video_base64 + +# type of content fields, can be only prompts or with images/videos +MsgContent = Union[str, List[ChatCompletionMessageContentPart]] + +# A list of all the conversations. Each conversation is a list of +# tuples. If multiturn is not enabled, the length of list is 1, +# containing only the first Q&A pair. +# For the shared prefix workload (synthetic, loogle, nextqa), it +# is a list of conversations sharing the same prefix (synthetic, +# doc, video) +SampleOutput = List[List[Tuple[MsgContent, int, int]]] + + +def common_filter_chat( + num_requests: int, + new_dataset: List, + tokenizer: PreTrainedTokenizerBase, + min_prompt_len: Optional[int], + min_output_len: Optional[int], + max_prompt_len: Optional[int], + max_output_len: Optional[int], + fixed_output_len: Optional[int], +) -> SampleOutput: + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = [] + l = 0 + input_tokens = 0 + output_tokens = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + processed = [] + for j in new_dataset[i]: + # Tokenize the prompts and completions. + prompt = j[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + completion = j[1] + completion_token_ids = tokenizer.encode(completion) + output_len = ( + len(completion_token_ids) + if fixed_output_len is None + else fixed_output_len + ) + if ( + min_prompt_len is not None + and prompt_len < min_prompt_len + or min_output_len is not None + and output_len < min_output_len + or max_prompt_len is not None + and prompt_len > max_prompt_len + or max_output_len is not None + and output_len > max_output_len + ): + # Prune too short sequences. + continue + input_tokens += prompt_len + output_tokens += output_len + processed.append((prompt, prompt_len, output_len)) + if len(processed) != 0: + filtered_dataset.append(processed) + l += 1 + + print(f"#Input tokens: {input_tokens}") + print(f"#Output tokens: {output_tokens}") + return filtered_dataset + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["conversations"]) % 2 != 0: + continue + if data["conversations"][0]["from"] != "human": + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["conversations"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append( + ( + data["conversations"][i]["value"], + data["conversations"][i + 1]["value"], + ) + ) + new_dataset.append(chat) + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_ultrachat_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["data"]) >= 2] + + # Keep one conversation in one list + new_dataset = [] + for data in dataset: + if len(data["data"]) % 2 != 0: + continue + chat = [] + total_len = 2 + if enable_multiturn: + total_len = len(data["data"]) + for i in range(0, total_len, 2): + # One user One Assistant + chat.append((data["data"][i], data["data"][i + 1])) + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, 4, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_loogle_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + disable_shuffle: bool = False, + enable_multiturn: bool = True, + enable_shared_prefix: bool = False, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Load the dataset + dataset = [] + with open(dataset_path) as f: + while True: + line = f.readline() + if not line: + break + dataset.append(json.loads(line)) + + # Keep one conversation in one list + new_dataset = [] + # TODO: Add shared prefix support for loogle + # NOTE: Now we preprocess it only for chat + for data in dataset: + chat = [] + if ( + "qa_pairs" not in data + or data["qa_pairs"] == "none" + or len(data["qa_pairs"]) == 0 + ): + # If Q is none (for summarization), + # We add a question for summarization + # And keep the summary up to 1024 words + chat.append( + ( + "Input: " + + data["input"] + + " Question: " + + "Please summarize the input", + data["input"][:1024], + ) + ) + new_dataset.append(chat) + else: + qa_pairs = eval(data["qa_pairs"]) + for i, qa in enumerate(qa_pairs): + if i == 0 or enable_shared_prefix: + # Combine input with the first Q + chat.append( + ("Input: " + data["input"] + " Question: " + qa["Q"], qa["A"]) + ) + elif enable_multiturn: + chat.append((qa["Q"], qa["A"])) + + new_dataset.append(chat) + + # Shuffle the dataset. + if not disable_shuffle: + random.shuffle(new_dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: SampleOutput = common_filter_chat( + num_requests, new_dataset, tokenizer, 4, None, None, None, fixed_output_len + ) + return filtered_dataset + + +def sample_nextqa_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + max_frames: int, # Specific for video + model_path: str, + disable_shuffle: bool = False, + enable_multiturn: bool = True, # No multiturn support for now + backend: str = "sglang-oai", + chat_template_name: Optional[str] = None, + fixed_output_len: Optional[int] = None, +) -> SampleOutput: + """ + Example of messages: + message = { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": video.prompt}, + ], + } + """ + + if fixed_output_len is None: + fixed_output_len = 4096 + + # TODO: Check for multiturn + dataset = NExTQALoader(video_dir=dataset_path, max_frames=max_frames) + new_dataset = [] + for v in dataset: + new_dataset.append(v) + + if not disable_shuffle: + random.shuffle(new_dataset) + + # TODO: prompt len can get from server side + filtered_dataset = [] + l = 0 + while l < num_requests: + for i in range(len(new_dataset)): + if l == num_requests: + break + + video = new_dataset[i] + + # text prompt + prompt = video.prompt + + # NOTE: Chat Template is a must for video benchmark because we have to + # add special image token for later expansion + if backend == "sglang" or backend == "sglang-native": + if "chat_template" in tokenizer.init_kwargs: + chat_template = get_chat_template(tokenizer.get_chat_template()) + elif chat_template_name is not None: + chat_template = get_chat_template(chat_template_name) + else: + chat_template = get_chat_template_by_model_path(model_path) + prompt = chat_template.image_token + prompt + + prompt_token_ids = tokenizer(prompt).input_ids + prompt_len = len(prompt_token_ids) + output_len = fixed_output_len # max output len, not real output len + + # video input + base64_data = encode_video_base64(video.path, video.num_frames) + + # NOTE: This will be replaced by the expanded length from the server + prompt_len += video.num_frames + + # add to content + content = [ + {"type": "image_url", "image_url": {"url": base64_data}}, + {"type": "text", "text": prompt}, + ] + + filtered_dataset.append([(content, prompt_len, output_len)]) + l += 1 + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, + disable_shuffle: bool = False, +) -> SampleOutput: + + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if True: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not os.path.isfile(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [data for data in dataset if len(data["conversations"]) >= 2] + # Only keep the first two turns of each conversation. + dataset = [ + (data["conversations"][0]["value"], data["conversations"][1]["value"]) + for data in dataset + ] + + if not disable_shuffle: + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: SampleOutput = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + prompt = tokenizer.decode(input_ids) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + prompt = tokenizer.decode( + [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + ) + input_requests.append([(prompt, int(input_lens[i]), int(output_lens[i]))]) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def gen_prompt(tokenizer, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def get_gen_prefix_cache_path(args, tokenizer): + """Create cache directory under ~/.cache/sglang/benchmark""" + cache_dir = Path.home() / ".cache" / "sglang" / "benchmark" + + # Create a unique cache filename based on the generation parameters + cache_key = ( + f"gsp_prefix_{args.gsp_num_groups}_{args.gsp_prompts_per_group}_" + f"{args.gsp_system_prompt_len}_{args.gsp_question_len}_{args.gsp_output_len}_" + f"{tokenizer.__class__.__name__}.pkl" + ) + return cache_dir / cache_key + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args, + disable_shuffle: bool = False, +) -> SampleOutput: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + input_requests.append([]) + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + input_requests[-1].append((full_prompt, prompt_len, output_len)) + total_input_tokens += prompt_len + total_output_tokens += output_len + + if not disable_shuffle: + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests) * prompts_per_group}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +def get_dataset(args, tokenizer): + if args.dataset_name == "sharegpt": + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "ultrachat": + input_requests = sample_ultrachat_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "loogle": + input_requests = sample_loogle_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + enable_shared_prefix=args.enable_shared_prefix, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "nextqa": + input_requests = sample_nextqa_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + max_frames=args.max_frames, + model_path=args.model, + disable_shuffle=args.disable_shuffle, + enable_multiturn=args.enable_multiturn, + backend=args.backend, + chat_template_name=args.chat_template, + fixed_output_len=args.fixed_output_len, + ) + elif args.dataset_name == "random": + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + ) + elif args.dataset_name == "generated-shared-prefix": + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gsp_num_groups, + prompts_per_group=args.gsp_prompts_per_group, + system_prompt_len=args.gsp_system_prompt_len, + question_len=args.gsp_question_len, + output_len=args.gsp_output_len, + args=args, + tokenizer=tokenizer, + ) + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests diff --git a/benchmark/hicache/download.sh b/benchmark/hicache/download.sh new file mode 100755 index 000000000..340bcc3bc --- /dev/null +++ b/benchmark/hicache/download.sh @@ -0,0 +1,66 @@ +#!/usr/bin/bash + +# The usage function +usage() { + echo "Usage: $0 {sharegpt|ultragpt|loogle|nextqa|all}" + exit 1 +} + +# The download function +download() { + case "$1" in + sharegpt) + echo $1 + wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json + ;; + ultragpt) + echo $1 + # Questions about the world + wget https://cloud.tsinghua.edu.cn/seafhttp/files/be1d7b87-22ca-449e-a6a7-c61d1ea7e010/ultrachat_release_230407.json + # Writing and Creation + wget https://cloud.tsinghua.edu.cn/seafhttp/files/61742d2a-25e2-4d08-b2b9-15f47ae50ace/ultrachat_material_release_230417.json + wget https://cloud.tsinghua.edu.cn/seafhttp/files/f71f6aa6-d346-4b16-85b7-8502efa3d608/ultrachat_material_release_230412.json + # External materials + wget https://cloud.tsinghua.edu.cn/seafhttp/files/42d22e28-e899-4975-a70f-5eda163e265d/ultrachat_existent_material_release_230420.json.gz + gunzip ultrachat_existent_material_release_230420.json.gz + ;; + loogle) + echo $1 + git lfs install + git clone git@hf.co:datasets/bigainlco/LooGLE + unzip LooGLE/data.zip + ;; + nextqa) + echo $1 + git lfs install + git clone https://huggingface.co/datasets/lmms-lab/NExTQA + unzip NExTQA/videos.zip + ;; + *) + usage + exit 1 + ;; + esac +} + +# Arg check +if [ "$#" -ne 1 ]; then + usage +fi + +# Invoke + +case "$1" in + sharegpt|ultragpt|loogle|nextqa) + download "$1" + ;; + all) + download sharegpt + download ultragpt + download loogle + download nextqa + ;; + *) + usage + ;; +esac diff --git a/benchmark/hicache/nextqa.py b/benchmark/hicache/nextqa.py new file mode 100644 index 000000000..4db6caa1c --- /dev/null +++ b/benchmark/hicache/nextqa.py @@ -0,0 +1,159 @@ +import os +import sys +from typing import List + +import av +from datasets import load_dataset + + +def find_video_files(video_dir) -> List[str]: + if os.path.isfile(video_dir): + return [video_dir] + + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + # if file is dir + elif os.path.isdir(file): + video_files.extend(find_video_files(file)) + return video_files + + +def video_frames(video_path, max_frames) -> int: + container = av.open(video_path) + total_frames = container.streams.video[0].frames + return min(total_frames, max_frames) + + +class Video: + def __init__(self, video_path, num_frames): + self.path = video_path + self.num_frames = num_frames + + def __str__(self): + return f"Video({self.path}, {self.num_frames})" + + def __iter__(self): + return iter((self.path, self.num_frames)) + + +class VideoPrompt(Video): + def __init__(self, video_path, num_frames, prompt): + super().__init__(video_path, num_frames) + self.prompt = prompt + + def __str__(self): + return f"VideoPrompt({self.path}, {self.num_frames}, {self.prompt})" + + def __iter__(self): + return iter((self.path, self.num_frames, self.prompt)) + + +class VideoLoader: + pass + + +class VideoFileLoader(VideoLoader): + """ + Load all the videos in a directory + """ + + def __init__(self, video_dir, batch_size=1, max_frames=sys.maxsize): + super().__init__() + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.batch_size = batch_size + self.max_frames = max_frames + print(f"batch_size: {batch_size}, max_frames: {max_frames}") + + def __iter__(self): # (file, number of frames) + if self.batch_size == 1: + for video_file in self.video_files: + yield Video(video_file, video_frames(video_file, self.max_frames)) + else: + batch = [] + for video_file in self.video_files: + video = Video(video_file, video_frames(video_file, self.max_frames)) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +class NExTQALoader(VideoLoader): + """ + Load vdideos and prompts from NExT dataset + set: train, test or validation + """ + + def __init__( + self, video_dir, batch_size=1, max_frames=sys.maxsize, dset="test", task="OE" + ): + """ + task: 'MV' or 'OE' + """ + super().__init__() + self.task = task + print(f"Loading the {dset} data of {task} from lmms-lab/NExTQA") + self.ds = load_dataset("lmms-lab/NExTQA", task) + self.ds = self.ds[dset] + + # self.n = ds.num_rows + self.video_dir = video_dir + self.video_files = find_video_files(video_dir) + self.video_to_path = dict() + for video_file in self.video_files: + video_id = video_file.split("/")[-1].split(".")[0] + self.video_to_path[video_id] = video_file + + self.batch_size = batch_size + self.max_frames = max_frames + + def get_video_prompt(self, entry, max_frames) -> VideoPrompt: + # Get video + video_id = entry["video"] + video_path = self.video_to_path[video_id] + assert os.path.exists(video_path), f"Video not found: {video_path}" + num_frames = min(entry["frame_count"], max_frames) + video = Video(video_path, num_frames) + prompt = entry["question"] + "?" + if self.task == "MC": # add choices + prompt += f' a0: {entry["a0"]}, a1: {entry["a1"]}, a2: {entry["a2"]}, a3: {entry["a3"]}' + return VideoPrompt(video_path, num_frames, prompt) + + def __iter__(self): + if self.batch_size == 1: + for entry in self.ds: + yield self.get_video_prompt(entry, self.max_frames) + else: + batch = [] + for entry in self.ds: + video = self.get_video_prompt(entry, self.max_frames) + batch.append(video) + if len(batch) == self.batch_size: + yield batch + batch = [] + + +# main +if __name__ == "__main__": + video_dir = "./videos" + # video_loader = VideoFileLoader(video_dir, batch_size=16) + # for batch in video_loader: + # print(f"Number of videos in batch: {len(batch)}") + # for video_file, num_frames in batch: + # print(f"Video: {video_file} number of frames: {num_frames}") + + video_loader = NExTQALoader(video_dir, batch_size=16, dset="test", task="OE") + for batch in video_loader: + print(f"Number of videos in batch: {len(batch)}") + for video_file, num_frames, prompt in batch: + print( + f"Video: {video_file} number of frames: {num_frames}, prompt: {prompt}" + ) + # break + # for video_file, prompt in batch: + # print(f"Video: {video_file} prompt: {prompt}") + # break diff --git a/benchmark/json_decode_regex/README.md b/benchmark/json_decode_regex/README.md new file mode 100644 index 000000000..26acabae9 --- /dev/null +++ b/benchmark/json_decode_regex/README.md @@ -0,0 +1,60 @@ +## Run benchmark + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` + +### Dependencies + +``` +llama_cpp_python 0.2.19 +guidance 0.1.10 +vllm 0.2.5 +outlines 0.0.22 +``` + +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark + +``` +python3 bench_sglang.py --num-questions 10 +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +``` +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark + +``` +python3 bench_other.py --backend outlines --num-questions 10 +``` + + +### Benchmark guidance + +Run Llama-7B and benchmark + +``` +python3 bench_other.py --backend guidance --num-questions 10 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/benchmark/json_decode_regex/bench_other.py b/benchmark/json_decode_regex/bench_other.py new file mode 100644 index 000000000..87051ea82 --- /dev/null +++ b/benchmark/json_decode_regex/bench_other.py @@ -0,0 +1,98 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + + +# fmt: off +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + s += generate(s, max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + s += generate(s, max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + s += generate(s, max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + s += generate(s, max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + + return s +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/json_decode_regex/bench_sglang.py b/benchmark/json_decode_regex/bench_sglang.py new file mode 100644 index 000000000..9aab11e43 --- /dev/null +++ b/benchmark/json_decode_regex/bench_sglang.py @@ -0,0 +1,101 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +REGEX_LIST = r"\[(" + REGEX_STR + ", )*" + REGEX_STR + r"\]" + +# fmt: off +@sgl.function +def json_warm_up(s): + s += "The information about Hogwarts is in the following JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" + print(f'The warmp up json result is:\n{s["json_output"]}') +# fmt: on + +# fmt: off +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen("name", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "country": ' + sgl.gen("country", max_tokens=8, regex=REGEX_STR + ",") + "\n" + s += ' "latitude": ' + sgl.gen("latitude", max_tokens=8, regex=REGEX_FLOAT + ",") + "\n" + s += ' "population": ' + sgl.gen("population", max_tokens=8, regex=REGEX_INT + ",") + "\n" + s += ' "top 3 landmarks": ' + sgl.gen( "landmarks", max_tokens=24, regex=REGEX_LIST) + "\n" + s += "}\n" +# fmt: on + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Warm up + json_warm_up.run().sync() + + # Run requests + tic = time.perf_counter() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(f"tmp_{args.backend}_json_results.txt", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_decode_regex", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/json_decode_regex/build_dataset.py b/benchmark/json_decode_regex/build_dataset.py new file mode 100644 index 000000000..1396e5ede --- /dev/null +++ b/benchmark/json_decode_regex/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/json_jump_forward/README.md b/benchmark/json_jump_forward/README.md new file mode 100644 index 000000000..38fb67e89 --- /dev/null +++ b/benchmark/json_jump_forward/README.md @@ -0,0 +1,88 @@ +## Run benchmark + +### Dependencies + +``` +llama_cpp_python 0.2.38 +guidance 0.1.10 +vllm 0.2.7 +outlines 0.0.25 +``` + +### Build dataset + +When benchmarking long document information retrieval, run the following command to build the dataset: + +```bash +pip install wikipedia +python3 build_dataset.py +``` + +### Benchmark sglang + +Run Llama-7B + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Benchmark Character Generation + +```bash +python3 bench_sglang.py --mode character +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_sglang.py --mode city +``` + + +### Benchmark Outlines + vLLM + +Run Llama-7B + +```bash +python3 -m outlines.serve.serve --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Benchmark Character Generation + +```bash +python3 bench_other.py --mode character --backend outlines +``` + +Benchmark City Information Retrieval + +```bash +python3 bench_other.py --mode city --backend outlines +``` + +### Benchmark guidance + +Run Llama-7B and benchmark character generation + +```bash +python3 bench_other.py --mode character --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Run Llama-7B and benchmark city information retrieval + +```bash +python3 bench_other.py --mode city --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +Run Llama-7B and benchmark character generation + +``` +python3 bench_other.py --mode character --backend lmql --parallel 1 +``` + +Run Llama-7B and benchmark city information retrieval + +``` +python3 bench_other.py --mode city --backend lmql --parallel 1 +``` diff --git a/benchmark/json_jump_forward/bench_other.py b/benchmark/json_jump_forward/bench_other.py new file mode 100644 index 000000000..a64e950d7 --- /dev/null +++ b/benchmark/json_jump_forward/bench_other.py @@ -0,0 +1,288 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +import guidance +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +def character_gen(name, generate): + s = name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += generate(s, max_tokens=256, regex=character_regex) + return s +# fmt: on + +# fmt: off +def city_gen(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += generate(s, max_tokens=256, regex=city_regex) + return s +# fmt: on + + +@guidance +def character_maker(lm, name): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + {name} is a character in Harry Potter. Please fill in the following information about this character. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "house": "{guidance.select(options=['Gryffindor', 'Slytherin', 'Ravenclaw', 'Hufflepuff'], name='house')}", + "blood status": "{guidance.select(options=['Pure-blood', 'Half-blood', 'Muggle-born'], name='blood status')}", + "occupation": "{guidance.select(options=['student', 'teacher', 'auror', 'ministry of magic', 'death eater', 'order of the phoenix'], name='occupation')}", + "wand": {{ + "wood": "{guidance.gen("wood", max_tokens=16, regex=regex_str_no_quote)}", + "core": "{guidance.gen('core', max_tokens=16, regex=regex_str_no_quote)}", + "length": {guidance.gen('length', max_tokens=10, regex=regex_float)} + }}, + "alive": "{guidance.select(options=['Alive', 'Deceased'], name='alive')}", + "patronus": "{guidance.gen('patronus', max_tokens=16, regex=regex_str_no_quote)}", + "bogart": "{guidance.gen('bogart', max_tokens=16, regex=regex_str_no_quote)}" + }} + """ + + return lm + + +async def call_generate_lmql( + prompt, temperature, max_tokens, regex, max_len=4096, model=None, **kwargs +): + assert model is not None + import lmql + + @lmql.query(model=model) + async def program(question, max_tokens, regex): + '''lmql + """{question}[ANSWER]""" where len(TOKENS(ANSWER)) < max_tokens and REGEX(ANSWER, regex) + return ANSWER + ''' + + return await program( + question=prompt, + temperature=temperature, + max_tokens=max_tokens, + max_len=max_len, + regex=regex, + **kwargs, + ) + + +@guidance +def city_maker(lm, document): + regex_str_no_quote = r"[\w\d\s]+" + regex_float = r"[0-9]+\.[0-9]+" + lm += f"""\ + Please extract the information of a city from the following wikipedia page. + Page begin. + {document} + Page end. + Here is the name, country, and symbol of the city in JSON format. + {{ + "name": "{guidance.gen("name", max_tokens=16, regex=regex_str_no_quote)}", + "country": "{guidance.gen("country", max_tokens=16, regex=regex_str_no_quote)}", + "latitude": {guidance.gen("latitude", max_tokens=10, regex=regex_float)}, + "population": {guidance.gen("population", max_tokens=10, regex=r"[0-9]+")}, + "top 3 landmarks": [ + "{guidance.gen("landmark1", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark2", max_tokens=16, regex=regex_str_no_quote)}", "{guidance.gen("landmark3", max_tokens=16, regex=regex_str_no_quote)}" + ] + }} + """ + + return lm + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = character_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + character_maker(**arguments[i]) + states[i] = lm + + elif args.backend == "lmql": + import asyncio + + import lmql + + model = lmql.model(args.model_path, endpoint=f"{args.host}:{args.port}") + call_generate = partial( + call_generate_lmql, + model=model, + max_tokens=256, + regex=character_regex, + ) + + async def get_one_answer_async(i): + states[i] = await call_generate(prompt=arguments[i]["name"], temperature=0) + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.perf_counter() + + if args.backend != "lmql": + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + for _ in rets: + pass + else: + batches = [] + for i in range(0, len(arguments), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(arguments))))) + loop = asyncio.get_event_loop() + + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.perf_counter() - tic + + return states, latency + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + states = [None] * len(arguments) + + # Select backend + if args.backend == "outlines": + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = city_gen(**arguments[i], generate=call_generate) + + elif args.backend == "guidance": + model = guidance.models.LlamaCpp( + args.model_path, + n_gpu_layers=-1, + n_ctx=args.n_ctx, + ) + + def get_one_answer(i): + lm = model + city_maker(**arguments[i]) + states[i] = lm + + else: + raise ValueError(f"Invalid backend: {args.backend}") + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = executor.map(get_one_answer, list(range(len(arguments)))) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/json_jump_forward/bench_sglang.py b/benchmark/json_jump_forward/bench_sglang.py new file mode 100644 index 000000000..29f635f75 --- /dev/null +++ b/benchmark/json_jump_forward/bench_sglang.py @@ -0,0 +1,143 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +# there are some FSM bugs with json regex converted from pydantic model +# here use a string regex instead +# regex_string = build_regex_from_object(HarryPoterRole) +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + +city_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "country": "[\w\d\s]{1,16}",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]{0,2},\n""" + + r""" "population": [-+]?[0-9]{1,9},\n""" + + r""" "top 3 landmarks": \["[\w\d\s]{1,16}", "[\w\d\s]{1,16}", "[\w\d\s]{1,16}"\]\n""" + + r"""\}""" +) + +# fmt: off +@sgl.function +def character_gen(s, name): + s += name + " is a character in Harry Potter. Please fill in the following information about this character.\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) +# fmt: on + +# fmt: off +@sgl.function +def city_gen(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += sgl.gen("json_output",max_tokens=256, regex=city_regex) +# fmt: on + + +def bench_city_doc(args): + arguments = [] + for line in read_jsonl(args.data_path): + arguments.append({"document": line["document"]}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = city_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + return states, latency + + +def bench_character(args): + arguments = [] + with open(args.data_path, "r") as f: + for line in f: + arguments.append({"name": line.strip()}) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = character_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + return states, latency + + +def main(args): + if args.mode == "character": + args.data_path = "dataset.txt" + states, latency = bench_character(args) + elif args.mode == "city": + args.data_path = "questions.jsonl" + states, latency = bench_city_doc(args) + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}_{args.mode}.txt", states) + with open(f"{args.backend}_{args.mode}.json", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_jump_forward", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "mode": args.mode, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str) + parser.add_argument("--num-jsons", type=int, default=50) + parser.add_argument( + "--mode", type=str, default="character", choices=["character", "city"] + ) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/json_jump_forward/build_dataset.py b/benchmark/json_jump_forward/build_dataset.py new file mode 100644 index 000000000..1396e5ede --- /dev/null +++ b/benchmark/json_jump_forward/build_dataset.py @@ -0,0 +1,58 @@ +import json + +import transformers +import wikipedia + +model_path = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(model_path) +city_names = [ + "los angles", + "london", + "tokyo", + "beijing", + "singapore", + "paris", + "dubai", + "sydney", + "moscow", + "rome", + "toronto", + "rio de janeiro", + "istanbul", + "berlin", + "auckland", + "buenos aires", + "mexico city", + "mumbai", + "seoul", + "bangkok", + "cairo", + "athens", + "jerusalem", +] + + +def get_content(city_name): + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + expected_tokens = 3000 + truncate_len = int((expected_tokens / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + return truncate_content + + +if __name__ == "__main__": + with open("questions.jsonl", "w") as fout: + for city_name in city_names: + truncate_content = get_content(city_name) + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/json_jump_forward/dataset.txt b/benchmark/json_jump_forward/dataset.txt new file mode 100644 index 000000000..c12421e5d --- /dev/null +++ b/benchmark/json_jump_forward/dataset.txt @@ -0,0 +1,50 @@ +Harry Potter +Hermione Granger +Ron Weasley +Albus Dumbledore +Severus Snape +Rubeus Hagrid +Draco Malfoy +Ginny Weasley +Fred Weasley +George Weasley +Percy Weasley +Sirius Black +Remus Lupin +Neville Longbottom +Luna Lovegood +Cedric Diggory +Cho Chang +Lord Voldemort +Minerva McGonagall +Filius Flitwick +Dolores Umbridge +Bellatrix Lestrange +Lucius Malfoy +Molly Weasley +Arthur Weasley +Nymphadora Tonks +Dobby +Moaning Myrtle +Peter Pettigrew +Alastor 'Mad-Eye' Moody +Horace Slughorn +Vernon Dursley +Petunia Dursley +Dudley Dursley +Argus Filch +Sybill Trelawney +Gilderoy Lockhart +Fleur Delacour +Viktor Krum +Bill Weasley +Oliver Wood +Cornelius Fudge +Barty Crouch Sr. +Barty Crouch Jr. +Kingsley Shacklebolt +Quirinus Quirrell +Nearly Headless Nick +Aunt Marge +Griphook +Ludo Bagman diff --git a/benchmark/json_schema/README.md b/benchmark/json_schema/README.md new file mode 100644 index 000000000..e6790e1fc --- /dev/null +++ b/benchmark/json_schema/README.md @@ -0,0 +1,15 @@ +## Run benchmark + +### Benchmark sglang + +Run Llama-8b + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 +``` + +Benchmark + +```bash +python3 bench_sglang.py +``` diff --git a/benchmark/json_schema/bench_sglang.py b/benchmark/json_schema/bench_sglang.py new file mode 100644 index 000000000..55365ff2e --- /dev/null +++ b/benchmark/json_schema/bench_sglang.py @@ -0,0 +1,146 @@ +import argparse +import json +import time +from typing import List, Tuple + +import jsonschema +from datasets import load_dataset + +import sglang as sgl +from sglang.global_config import global_config +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def schema_gen(s, message: Tuple[str, str], json_schema: str): + system, user = message + s += sgl.system(system) + s += sgl.user(user) + s += sgl.assistant( + sgl.gen("json_output", temperature=0, max_tokens=256, json_schema=json_schema) + ) + + +def contains_formats(schema, formats: List[str]): + if isinstance(schema, dict): + if schema.get("format", None) in formats: + return True + for value in schema.values(): + if contains_formats(value, formats): + return True + elif isinstance(schema, list): + for item in schema: + if contains_formats(item, formats): + return True + return False + + +def convert_dataset(path: str): + raw_dataset = load_dataset(path) + dataset = [] + for data in raw_dataset["train"]: + messages = data["prompt"] + schema = data["schema"] + obj = json.loads(schema) + + # skip some corrupted examples + if obj.get("type", None) is None: + continue + + # skip schema with format "email" + # which is not supported by outlines for now + if contains_formats(obj, ["email"]): + continue + + system = messages[0] + user = messages[1] + assert system["role"] == "system", "invalid role" + assert user["role"] == "user", "invalid role" + assert len(messages) == 2, "invalid message length" + message = json.dumps(system["content"]), json.dumps(user["content"]) + dataset.append( + { + "message": message, + "json_schema": schema, + } + ) + + return dataset + + +def bench_schema(args): + arguments = convert_dataset(args.data_path) + + if args.num_jsons < 0 or args.num_jsons > len(arguments): + args.num_jsons = len(arguments) + arguments = arguments[: args.num_jsons] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = schema_gen.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + # Check if the outputs are valid + indexes = [] + for i, state in enumerate(states): + try: + schema = json.loads(arguments[i]["json_schema"]) + obj = json.loads(state["json_output"]) + assert jsonschema.validate(obj, schema) is None + except Exception as e: + print(e) + indexes.append(i) + + return states, latency + + +def main(args): + states, latency = bench_schema(args) + + # Compute accuracy + tokenizer = get_tokenizer( + global_config.default_backend.get_server_info()["tokenizer_path"] + ) + output_jsons = [state["json_output"] for state in states] + num_output_tokens = sum(len(tokenizer.encode(x)) for x in output_jsons) + print(f"Latency: {latency:.3f}") + print(f"Output throughput: {num_output_tokens / latency:.3f} token/s") + print(f"#output tokens: {num_output_tokens}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + with open(f"{args.backend}.jsonl", "w") as fout: + for state in states: + fout.write(state["json_output"] + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "json_schema", + "backend": args.backend, + "latency": round(latency, 3), + "num_jsons": args.num_jsons, + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="NousResearch/json-mode-eval") + parser.add_argument("--num-jsons", type=int, default=-1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py new file mode 100644 index 000000000..eebbd00ce --- /dev/null +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -0,0 +1,224 @@ +"""For Now, MSCCL is only supported on TP16 and TP8 case + +export WORLD_SIZE=1 +export RANK=0 +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=12345 + +torchrun --nproc_per_node gpu \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py +""" + +import os +from contextlib import nullcontext +from typing import List + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator +from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, + set_mscclpp_all_reduce, +) + + +def torch_allreduce(torch_input: torch.Tensor, group: ProcessGroup) -> torch.Tensor: + dist.all_reduce(torch_input, group=group) + return torch_input + + +def msccl_allreduce( + msccl_input: torch.Tensor, msccl_comm: PyMscclppCommunicator +) -> torch.Tensor: + return msccl_comm.all_reduce(msccl_input) + + +def pynccl_allreduce( + msccl_input: torch.Tensor, pynccl_comm: PyNcclCommunicator +) -> torch.Tensor: + pynccl_comm.all_reduce(msccl_input) + return msccl_input + + +def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10): + graph_input = inp_randn.clone() + with graph_capture() as graph_capture_context: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + for _ in range(graph_loop): + graph_out = func(graph_input) + + graph.replay() + func_output = graph_out.clone() + + for _ in range(warmup_loop): + graph.replay() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for _ in range(test_loop): + torch.cuda.synchronize() + dist.barrier() + start_event.record() + graph.replay() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + func_cost_us = sum(latencies) / len(latencies) / graph_loop * 1000 + graph.reset() + return func_output, func_cost_us + + +def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10): + eager_input = inp_randn.clone() + eager_output = func(eager_input) + func_output = eager_output.clone() + + for _ in range(warmup_loop): + func(eager_input) + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start_event.record() + for _ in range(test_loop): + func(eager_input) + end_event.record() + torch.cuda.synchronize() + func_cost_us = start_event.elapsed_time(end_event) / test_loop * 1000 + + return func_output, func_cost_us + + +def get_torch_prof_ctx(do_prof: bool): + ctx = ( + torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + record_shapes=True, + with_stack=True, + ) + if do_prof + else nullcontext() + ) + return ctx + + +def human_readable_size(size, decimal_places=1): + for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: + if size < 1024.0 or unit == "PiB": + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + + +try: + from tabulate import tabulate +except ImportError: + print("tabulate not installed, skipping table printing") + tabulate = None + + +def print_markdown_table(data): + if tabulate is not None: + print(tabulate(data, headers="keys", tablefmt="github")) + return + headers = data[0].keys() + header_row = "| " + " | ".join(headers) + " |" + separator = "| " + " | ".join(["---"] * len(headers)) + " |" + rows = [] + for item in data: + row = "| " + " | ".join(str(item[key]) for key in headers) + " |" + rows.append(row) + markdown_table = "\n".join([header_row, separator] + rows) + print(markdown_table) + + +if __name__ == "__main__": + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + force=True, + ) + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + world, world_size = dist.group.WORLD, dist.get_world_size() + rank = dist.get_rank() + torch.cuda.set_device(rank % 8) + device = torch.cuda.current_device() + set_mscclpp_all_reduce(True) + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank % 8, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + cpu_group = get_tensor_model_parallel_group().cpu_group + pynccl_comm = get_tensor_model_parallel_group().pynccl_comm + pymscclpp_comm = get_tensor_model_parallel_group().pymscclpp_comm + dist.barrier() + profile = False + dtype = torch.bfloat16 + ctx = get_torch_prof_ctx(profile) + result = [] + + with ctx: + for i in range(10, 20): + sz = 2**i + if sz * dtype.itemsize > 2**20: + break + inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + + memory = torch.empty_like(inp_randn) + memory_out = torch.empty_like(memory) + torch_eager_output, torch_eager_time = _bench_eager_time( + lambda inp: torch_allreduce(inp, group), inp_randn + ) + msccl_eager_output, msccl_eager_time = _bench_eager_time( + lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + ) + msccl_graph_output, msccl_graph_time = _bench_graph_time( + lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + ) + # since pynccl is inplace op, this return result is not correct if graph loop > 1 + _, pynccl_graph_time = _bench_graph_time( + lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn + ) + torch.testing.assert_close(torch_eager_output, msccl_graph_output) + torch.testing.assert_close(torch_eager_output, msccl_eager_output) + result.append( + { + "msg_size": human_readable_size(inp_randn.nbytes), + "torch eager time": torch_eager_time, + "msccl eager time": msccl_eager_time, + "msccl graph time": msccl_graph_time, + "pynccl graph time": pynccl_graph_time, + } + ) + if rank == 0: + print(f"sz={sz}, dtype={dtype}: correctness check PASS!") + if rank == 0: + print_markdown_table(result) + if profile: + prof_dir = f"prof/msccl" + os.makedirs(prof_dir, exist_ok=True) + ctx.export_chrome_trace(f"{prof_dir}/trace_rank{dist.get_rank()}.json.gz") diff --git a/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py new file mode 100644 index 000000000..b61240b05 --- /dev/null +++ b/benchmark/kernels/decoding_attention_triton/triton_flashinfer_cudnn.py @@ -0,0 +1,403 @@ +import itertools +import math + +import cudnn +import torch +import torch.utils.benchmark as benchmark +from flashinfer import BatchDecodeWithPagedKVCacheWrapper + +from sglang.srt.layers.attention.triton_ops.decode_attention import decode_attention_fwd +from sglang.srt.utils import should_use_tensor_core + + +def benchmark_forward( + fn, + *inputs, + repeats=10, + amp=False, + amp_dtype=torch.float16, + **kwinputs, +): + def amp_wrapper(*inputs, **kwinputs): + with torch.autocast(device_type="cuda", dtype=amp_dtype, enabled=amp): + fn(*inputs, **kwinputs) + + t = benchmark.Timer( + stmt="fn_amp(*inputs, **kwinputs)", + globals={"fn_amp": amp_wrapper, "inputs": inputs, "kwinputs": kwinputs}, + num_threads=torch.get_num_threads(), + ) + m = t.timeit(repeats) + return t, m + + +def time_fwd(func, *args, **kwargs): + time_f = benchmark_forward(func, *args, **kwargs) + return time_f[1].mean * 1e6 + + +def decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits, + warmup=10, +): + + k_buffer = kv_data[0].view(-1, head_num_kv, head_dim) + v_buffer = kv_data[1].view(-1, head_num_kv, head_dim) + o = torch.empty_like(q) + total_tokens = batch_size * kv_len + req_to_token = torch.arange(0, total_tokens).to(0).int().view(batch_size, kv_len) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32, device="cuda") + max_len_in_batch = kv_len + sm_scale = 1.0 / (head_dim**0.5) + + attn_logits = torch.empty( + (batch_size, head_num_q, num_kv_splits, head_dim + 1), + dtype=torch.float32, + device="cuda", + ) + + for _ in range(warmup): + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + f = time_fwd( + decode_attention_fwd, + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_seq_len, + attn_logits, + num_kv_splits, + sm_scale, + ) + + return f, o + + +def decode_attention_flashinfer(dtype, head_num_q, head_num_kv): + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device="cuda") + use_tensor_cores = should_use_tensor_core( + kv_cache_dtype=dtype, + num_attention_heads=head_num_q, + num_kv_heads=head_num_kv, + ) + flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores + ) + + class FlashinferAttention(torch.autograd.Function): + @staticmethod + def forward( + ctx, + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + dtype, + warmup=10, + ): + total_tokens = batch_size * kv_len + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full( + (batch_size,), 1, dtype=torch.int32, device="cuda" + ) + + flashinfer_decode_wrapper.end_forward() + flashinfer_decode_wrapper.begin_forward( + kv_indptr, + kv_indices, + kv_last_page_len, + head_num_q, + head_num_kv, + head_dim, + 1, + pos_encoding_mode="NONE", + data_type=dtype, + ) + + for _ in range(warmup): + o = flashinfer_decode_wrapper.forward( + q.contiguous().view(-1, head_num_q, head_dim), kv_data + ) + + f = time_fwd( + flashinfer_decode_wrapper.forward, + q.contiguous().view(-1, head_num_q, head_dim), + kv_data, + ) + + return f, o + + return FlashinferAttention + + +def convert_to_cudnn_type(torch_type): + if torch_type == torch.float16: + return cudnn.data_type.HALF + elif torch_type == torch.bfloat16: + return cudnn.data_type.BFLOAT16 + elif torch_type == torch.float32: + return cudnn.data_type.FLOAT + elif torch_type == torch.int32: + return cudnn.data_type.INT32 + elif torch_type == torch.int64: + return cudnn.data_type.INT64 + else: + raise ValueError("Unsupported tensor data type.") + + +def decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype, warmup=10 +): + # Prepare data: continuous q,k,v + dims_q = (batch_size, head_num_q, 1, head_dim) + strides_q = (head_num_q * head_dim, head_dim, head_num_q * head_dim, 1) + q_gpu = q.as_strided(dims_q, strides_q) + o_gpu = ( + torch.empty(batch_size * head_num_q * head_dim) + .half() + .cuda() + .as_strided(dims_q, strides_q) + ) + + dims_kv = (batch_size, head_num_kv, kv_len, head_dim) + strides_kv = ( + kv_len * head_num_kv * head_dim, + head_dim, + head_num_kv * head_dim, + 1, + ) + k_gpu = kv_data[0].as_strided(dims_kv, strides_kv) + v_gpu = kv_data[1].as_strided(dims_kv, strides_kv) + + seq_len_q_gpu = torch.full((batch_size, 1, 1, 1), 1, device="cuda") + seq_len_kv_gpu = torch.full((batch_size, 1, 1, 1), kv_len, device="cuda") + attn_scale = 1.0 / (head_dim**0.5) + + # Prepare data: paged k,v + block_size = 1 + blocks_per_batch = math.ceil(kv_len / block_size) + # [num_blocks, head_num_kv, block_size, head_dim], num_blocks = batch_size * blocks_per_batch + container_k_gpu = torch.cat(k_gpu.chunk(blocks_per_batch, dim=2), dim=0) + container_v_gpu = torch.cat(v_gpu.chunk(blocks_per_batch, dim=2), dim=0) + page_table_k_gpu = ( + torch.linspace( + 0, + batch_size * blocks_per_batch - 1, + batch_size * blocks_per_batch, + device="cuda", + dtype=torch.int32, + ) + .reshape(blocks_per_batch, 1, batch_size, 1) + .transpose(0, 2) + ) + page_table_v_gpu = page_table_k_gpu.clone() + + graph = cudnn.pygraph( + io_data_type=convert_to_cudnn_type(dtype), + intermediate_data_type=cudnn.data_type.FLOAT, + compute_data_type=cudnn.data_type.FLOAT, + ) + + q = graph.tensor_like(q_gpu) + container_k = graph.tensor_like(container_k_gpu) + container_v = graph.tensor_like(container_v_gpu) + page_table_k = graph.tensor_like(page_table_k_gpu) + page_table_v = graph.tensor_like(page_table_v_gpu) + + seq_len_q = graph.tensor_like(seq_len_q_gpu) + seq_len_kv = graph.tensor_like(seq_len_kv_gpu) + + o, _ = graph.sdpa( + name="sdpa", + q=q, + k=container_k, # Container K: non contiguous container with K blocks + v=container_v, # Container V: non contiguous container with V blocks + is_inference=True, + attn_scale=attn_scale, + use_causal_mask=False, + use_padding_mask=True, + seq_len_q=seq_len_q, + seq_len_kv=seq_len_kv, + paged_attention_k_table=page_table_k, # Page Table K: Tensor containing offsets to the container with K blocks + paged_attention_v_table=page_table_v, # Page Table V: Tensor containing offsets to the container with V blocks + paged_attention_max_seq_len_kv=kv_len, # The maximum sequence length for K caches (this is optional, but recommended) + ) + + o.set_output(True).set_dim(dims_q).set_stride(strides_q) + + graph.validate() + graph.build_operation_graph() + graph.create_execution_plans([cudnn.heur_mode.A]) + graph.check_support() + graph.build_plans() + + workspace = torch.empty( + graph.get_workspace_size(), device="cuda", dtype=torch.uint8 + ) + + variant_pack = { + q: q_gpu, + container_k: container_k_gpu, + container_v: container_v_gpu, + page_table_k: page_table_k_gpu, + page_table_v: page_table_v_gpu, + seq_len_q: seq_len_q_gpu, + seq_len_kv: seq_len_kv_gpu, + o: o_gpu, + } + + for _ in range(warmup): + graph.execute(variant_pack, workspace) + + f = time_fwd( + graph.execute, + variant_pack, + workspace, + ) + + return f, o_gpu.squeeze(dim=2) + + +def calculate_diff(): + + dtype = torch.float16 + batch_size = 64 + kv_len = 4096 + head_num_q = 64 + head_num_kv = 8 + head_dim = 128 + + q = torch.randn(batch_size, head_num_q, head_dim, dtype=dtype, device="cuda") + kv_data = ( + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + torch.randn( + batch_size * kv_len, head_num_kv, head_dim, dtype=dtype, device="cuda" + ), + ) + + _, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + + attn_flashinfer = decode_attention_flashinfer(dtype, head_num_q, head_num_kv).apply + _, output_flashinfer = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + _, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + + print(f"SGLang output={output_sglang}") + print(f"FlashInfer output={output_flashinfer}") + print(f"cuDNN output={output_cudnn}") + if torch.allclose(output_sglang, output_flashinfer, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and FlashInfer match") + else: + print("❌ SGLang[Triton] and FlashInfer differ") + + if torch.allclose(output_sglang, output_cudnn, atol=1e-2, rtol=1e-2): + print("✅ SGLang[Triton] and cuDNN match") + else: + print("❌ SGLang[Triton] and cuDNN differ") + + +if __name__ == "__main__": + calculate_diff() + + head_dim = 128 + dtype = torch.float16 + batch_size_range = [2**i for i in range(0, 8, 2)] + kv_len_range = [2**i for i in range(6, 13, 1)] + configs = list(itertools.product(batch_size_range, kv_len_range)) + + for head_num_q, head_num_kv in [[32, 32], [64, 8], [40, 8]]: + attn_flashinfer = decode_attention_flashinfer( + dtype, head_num_q, head_num_kv + ).apply + for batch_size, kv_len in configs: + q = torch.randn( + batch_size, head_num_q, head_dim, dtype=dtype, device="cuda" + ) + kv_data = ( + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + torch.randn( + batch_size * kv_len, + head_num_kv, + head_dim, + dtype=dtype, + device="cuda", + ), + ) + us_cudnn, output_cudnn = decode_attention_cudnn( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + us_sglang, output_sglang = decode_attention_sglang( + q, + kv_data, + batch_size, + kv_len, + head_num_q, + head_num_kv, + head_dim, + num_kv_splits=8, + ) + us_flashinfer, _ = attn_flashinfer( + q, kv_data, batch_size, kv_len, head_num_q, head_num_kv, head_dim, dtype + ) + print( + head_num_q, + " ", + head_num_kv, + " ", + batch_size, + " ", + kv_len, + " ", + us_cudnn, + " ", + us_sglang, + " ", + us_flashinfer, + ) diff --git a/benchmark/kernels/deepep/deepep_utils.py b/benchmark/kernels/deepep/deepep_utils.py new file mode 100644 index 000000000..169529ef6 --- /dev/null +++ b/benchmark/kernels/deepep/deepep_utils.py @@ -0,0 +1,218 @@ +# ADAPTED FROM https://github.com/deepseek-ai/DeepEP/blob/main/tests/utils.py + +import os +import sys +from typing import Optional + +import numpy as np +import torch +import torch.distributed as dist + + +def init_dist(local_rank: int, num_local_ranks: int, args): + ip = args.master_addr + port = args.master_port + num_nodes = args.nnodes + node_rank = args.node_rank + assert (num_local_ranks < 8 and num_nodes == 1) or num_local_ranks == 8 + + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{ip}:{port}", + world_size=num_nodes * num_local_ranks, + rank=node_rank * num_local_ranks + local_rank, + ) + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.cuda.set_device(local_rank) + + return ( + dist.get_rank(), + dist.get_world_size(), + dist.new_group(list(range(num_local_ranks * num_nodes))), + ) + + +def calc_diff(x: torch.Tensor, y: torch.Tensor): + x, y = x.double() + 1, y.double() + 1 + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return (1 - sim).item() + + +def per_token_cast_to_fp8(x: torch.Tensor): + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_token_cast_back(x_fp8: torch.Tensor, x_scales: torch.Tensor): + x_fp32 = x_fp8.to(torch.float32).view(x_fp8.size(0), -1, 128) + x_scales = x_scales.view(x_fp8.size(0), -1, 1) + return (x_fp32 * x_scales).view(x_fp8.shape).to(torch.bfloat16) + + +def inplace_unique(x: torch.Tensor, num_slots: int): + assert x.dim() == 2 + mask = x < 0 + x_padded = x.masked_fill(mask, num_slots) + bin_count = torch.zeros((x.size(0), num_slots + 1), dtype=x.dtype, device=x.device) + bin_count.scatter_add_(1, x_padded, torch.ones_like(x_padded)) + bin_count = bin_count[:, :num_slots] + sorted_bin_count, sorted_bin_idx = torch.sort(bin_count, dim=-1, descending=True) + sorted_bin_idx.masked_fill_(sorted_bin_count == 0, -1) + sorted_bin_idx = torch.sort(sorted_bin_idx, descending=True, dim=-1).values + x[:, :].fill_(-1) + valid_len = min(num_slots, x.size(1)) + x[:, :valid_len] = sorted_bin_idx[:, :valid_len] + + +def create_grouped_scores( + scores: torch.Tensor, group_idx: torch.Tensor, num_groups: int +): + num_tokens, num_experts = scores.shape + scores = scores.view(num_tokens, num_groups, -1) + mask = torch.zeros((num_tokens, num_groups), dtype=torch.bool, device=scores.device) + mask = mask.scatter_(1, group_idx, True).unsqueeze(-1).expand_as(scores) + return (scores * mask).view(num_tokens, num_experts) + + +def bench(fn, num_warmups: int = 20, num_tests: int = 30, post_fn=None): + # Flush L2 cache with 256 MB data + torch.cuda.synchronize() + cache = torch.empty(int(256e6 // 4), dtype=torch.int, device="cuda") + + # Warmup + for _ in range(num_warmups): + fn() + + # Flush L2 + cache.zero_() + + # Testing + start_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + for i in range(num_tests): + # Record + start_events[i].record() + fn() + end_events[i].record() + if post_fn is not None: + post_fn() + torch.cuda.synchronize() + + times = np.array( + [s.elapsed_time(e) / 1e3 for s, e in zip(start_events, end_events)] + )[1:] + return np.average(times), np.min(times), np.max(times) + + +class empty_suppress: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + +class suppress_stdout_stderr: + def __enter__(self): + self.outnull_file = open(os.devnull, "w") + self.errnull_file = open(os.devnull, "w") + + self.old_stdout_fileno_undup = sys.stdout.fileno() + self.old_stderr_fileno_undup = sys.stderr.fileno() + + self.old_stdout_fileno = os.dup(sys.stdout.fileno()) + self.old_stderr_fileno = os.dup(sys.stderr.fileno()) + + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + + os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup) + os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup) + + sys.stdout = self.outnull_file + sys.stderr = self.errnull_file + return self + + def __exit__(self, *_): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + + os.close(self.old_stdout_fileno) + os.close(self.old_stderr_fileno) + + self.outnull_file.close() + self.errnull_file.close() + + +def bench_kineto( + fn, + kernel_names, + num_tests: int = 30, + suppress_kineto_output: bool = False, + trace_path: Optional[str] = None, + barrier_comm_profiling: bool = False, +): + # Profile + suppress = suppress_stdout_stderr if suppress_kineto_output else empty_suppress + with suppress(): + schedule = torch.profiler.schedule(wait=0, warmup=1, active=1, repeat=1) + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA], schedule=schedule + ) as prof: + for i in range(2): + # NOTES: use a large kernel and a barrier to eliminate the unbalanced CPU launch overhead + if barrier_comm_profiling: + lhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + rhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + lhs @ rhs + dist.all_reduce(torch.ones(1, dtype=torch.float, device="cuda")) + for _ in range(num_tests): + fn() + prof.step() + + # Parse the profiling table + assert isinstance(kernel_names, str) or isinstance(kernel_names, tuple) + is_tupled = isinstance(kernel_names, tuple) + prof_lines = ( + prof.key_averages() + .table(sort_by="cuda_time_total", max_name_column_width=100) + .split("\n") + ) + kernel_names = (kernel_names,) if isinstance(kernel_names, str) else kernel_names + assert all([isinstance(name, str) for name in kernel_names]) + for name in kernel_names: + assert ( + sum([name in line for line in prof_lines]) == 1 + ), f"Errors of the kernel {name} in the profiling table" + + # Save chrome traces + if trace_path is not None: + prof.export_chrome_trace(trace_path) + + # Return average kernel times + units = {"ms": 1e3, "us": 1e6} + kernel_times = [] + for name in kernel_names: + for line in prof_lines: + if name in line: + time_str = line.split()[-2] + for unit, scale in units.items(): + if unit in time_str: + kernel_times.append(float(time_str.replace(unit, "")) / scale) + break + break + return tuple(kernel_times) if is_tupled else kernel_times[0] + + +def hash_tensor(t: torch.Tensor): + return t.view(torch.int64).sum().item() diff --git a/benchmark/kernels/deepep/tuning_deepep.py b/benchmark/kernels/deepep/tuning_deepep.py new file mode 100644 index 000000000..bb900a875 --- /dev/null +++ b/benchmark/kernels/deepep/tuning_deepep.py @@ -0,0 +1,476 @@ +# MODIFIED FROM https://github.com/deepseek-ai/DeepEP/blob/main/tests/test_internode.py + +""" +Example usage: +python tuning_deepep.py --nnodes 4 --node-rank $MY_NODE_RANK --master-addr 1.2.3.4 +Then check `deepep_tuned.json` +""" + +import argparse +import json +import time +from copy import deepcopy +from pathlib import Path + +# noinspection PyUnresolvedReferences +import deep_ep +import torch +import torch.distributed as dist +from deepep_utils import ( + bench, + calc_diff, + create_grouped_scores, + init_dist, + inplace_unique, + per_token_cast_back, + per_token_cast_to_fp8, +) + + +def test_main( + num_sms: int, + local_rank: int, + num_local_ranks: int, + num_ranks: int, + num_nodes: int, + rank: int, + buffer: deep_ep.Buffer, + group: dist.ProcessGroup, + args, +): + # Settings + num_tokens, hidden, num_topk_groups, num_topk, num_experts = ( + 4096, + 7168, + min(num_nodes, 4), + 8, + (256 // num_ranks) * num_ranks, + ) + assert num_experts % num_ranks == 0 and num_local_ranks == 8 + if local_rank == 0: + print( + f"[config] num_tokens={num_tokens}, hidden={hidden}, num_topk_groups={num_topk_groups}, num_topk={num_topk}", + flush=True, + ) + + # Random data + x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * rank + x_pure_rand = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") + x_e4m3 = per_token_cast_to_fp8(x) + scores = ( + torch.randn((num_tokens, num_experts), dtype=torch.float32, device="cuda").abs() + + 1 + ) + group_scores = scores.view(num_tokens, num_nodes, -1).amax(dim=-1) + group_idx = torch.topk( + group_scores, k=num_topk_groups, dim=-1, sorted=False + ).indices + masked_scores = create_grouped_scores(scores, group_idx, num_nodes) + topk_idx = torch.topk(masked_scores, num_topk, dim=-1, largest=True, sorted=False)[ + 1 + ] + topk_weights = ( + torch.ones((num_tokens, num_topk), dtype=torch.float32, device="cuda") * rank + ) + topk_weights_pure_rand = torch.randn( + (num_tokens, num_topk), dtype=torch.float32, device="cuda" + ) + rank_idx = topk_idx // (num_experts // num_ranks) + rank_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rank_idx, num_ranks) + rdma_rank_idx = rank_idx // num_local_ranks + rdma_rank_idx.masked_fill_(rank_idx == -1, -1) + inplace_unique(rdma_rank_idx, num_nodes) + + # RDMA dispatch counts + rdma_idx = topk_idx // (num_experts // num_nodes) + rdma_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rdma_idx, num_nodes) + num_rdma_token_sent = rdma_idx.ne(-1).sum().item() + + # Expert meta + num_tokens_per_expert = torch.zeros((num_experts,), dtype=torch.int, device="cuda") + for i in range(num_experts): + num_tokens_per_expert[i] = (topk_idx == i).sum() + gbl_num_tokens_per_expert = num_tokens_per_expert.clone() + dist.all_reduce(gbl_num_tokens_per_expert, group=group) + + # Rank layout meta + num_tokens_per_rank = torch.empty((num_ranks,), dtype=torch.int, device="cuda") + num_tokens_per_rdma_rank = torch.empty((num_nodes,), dtype=torch.int, device="cuda") + token_idx_in_rank = torch.full( + (num_ranks, num_tokens), -1, dtype=torch.long, device="cuda" + ) + for i in range(num_ranks): + num_tokens_per_rank[i] = (rank_idx == i).sum() + token_sel = (rank_idx == i).max(dim=-1)[0] + count = token_sel.sum().item() + tokens = torch.sort(token_sel.to(torch.int), descending=True)[1] + tokens[:count] = torch.sort(tokens[:count])[0] + token_idx_in_rank[i][tokens[:count]] = torch.arange( + count, dtype=torch.long, device="cuda" + ) + for i in range(num_nodes): + num_tokens_per_rdma_rank[i] = (rdma_rank_idx == i).sum() + token_idx_in_rank = token_idx_in_rank.T.contiguous().to(torch.int) + is_token_in_rank = token_idx_in_rank >= 0 + gbl_num_tokens_per_rank = num_tokens_per_rank.clone() + dist.all_reduce(gbl_num_tokens_per_rank, group=group) + + ( + ref_num_tokens_per_rank, + ref_num_tokens_per_rdma_rank, + ref_num_tokens_per_expert, + ref_is_token_in_rank, + _, + ) = buffer.get_dispatch_layout(topk_idx, num_experts) + assert torch.allclose(ref_num_tokens_per_rank, num_tokens_per_rank) + assert torch.allclose(ref_num_tokens_per_rdma_rank, num_tokens_per_rdma_rank) + assert torch.allclose(ref_num_tokens_per_expert, num_tokens_per_expert) + assert torch.allclose(ref_is_token_in_rank, is_token_in_rank) + t = bench(lambda: buffer.get_dispatch_layout(topk_idx, num_experts))[0] + if local_rank == 0: + print(f"[layout] Kernel performance: {t * 1000:.3f} ms", flush=True) + print("", flush=True) + group.barrier() + time.sleep(1) + + # Config + rdma_buffer_size, nvl_buffer_size = 128, (720 if num_ranks in (144, 160) else 512) + config = deep_ep.Config(num_sms, 8, nvl_buffer_size, 16, rdma_buffer_size) + + # Test dispatch + # noinspection PyShadowingNames + def check_data(check_x, recv_gbl_rank_prefix_sum): + assert torch.allclose(check_x.amin(dim=1), check_x.amax(dim=1)) + check_start = 0 + for i in range(num_ranks): + check_end = recv_gbl_rank_prefix_sum[i].item() + assert (check_x[check_start:check_end, :].int() - i).sum().item() == 0 + check_start = check_end + + for previous_mode in (False, True): + for async_mode in (False, True): + for current_x in (x_pure_rand, x, x_e4m3): + for with_topk in (False, True): + if local_rank == 0: + print( + f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...', + flush=True, + end="", + ) + dispatch_args = { + "x": current_x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": config, + "async_finish": async_mode, + } + if with_topk: + dispatch_args.update( + { + "topk_idx": topk_idx, + "topk_weights": ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ), + } + ) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + ( + recv_x, + recv_topk_idx, + recv_topk_weights, + recv_num_tokens_per_expert_list, + handle, + event, + ) = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + + # Checks + recv_gbl_rank_prefix_sum = handle[-4] + assert gbl_num_tokens_per_rank[rank].item() == recv_x.size( + 0 + ), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}" + assert ( + gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist() + == recv_num_tokens_per_expert_list + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + if with_topk: + # Check `topk_idx` + assert ( + recv_topk_idx.eq(-1) + | ( + (recv_topk_idx >= 0) + & (recv_topk_idx < (num_experts // num_ranks)) + ) + ).sum().item() == recv_topk_idx.numel() + for i, count in enumerate(recv_num_tokens_per_expert_list): + assert recv_topk_idx.eq(i).sum().item() == count + + # Check `topk_weights` + if current_x is not x_pure_rand: + recv_topk_weights[recv_topk_idx.eq(-1)] = ( + recv_topk_weights.amax(dim=1, keepdim=True).expand_as( + recv_topk_weights + )[recv_topk_idx.eq(-1)] + ) + check_data(recv_topk_weights, recv_gbl_rank_prefix_sum) + + # Test cached dispatch (must without top-k staffs) + if not with_topk: + dispatch_args = { + "x": current_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + recv_x, _, _, _, _, event = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + + # Test combine + combine_args = { + "x": recv_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if with_topk: + combine_args.update({"topk_weights": recv_topk_weights}) + if previous_mode: + combine_args.update({"previous_event": buffer.capture()}) + combined_x, combined_topk_weights, event = buffer.combine( + **combine_args + ) + event.current_stream_wait() if async_mode else () + check_x = combined_x.float() / is_token_in_rank.sum( + dim=1 + ).unsqueeze(1) + ref_x = x_pure_rand if current_x is x_pure_rand else x + assert calc_diff(check_x, ref_x) < 5e-6 + if with_topk: + check_topk_weights = ( + combined_topk_weights + if (current_x is x_pure_rand) + else ( + combined_topk_weights + / is_token_in_rank.sum(dim=1).unsqueeze(1) + ) + ) + ref_topk_weights = ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ) + assert calc_diff(check_topk_weights, ref_topk_weights) < 1e-9 + + # For later tuning + dispatch_bf16_rdma_send_bytes = num_rdma_token_sent * hidden * 2 + dispatch_bf16_nvl_recv_bytes = recv_x.numel() * 2 + combine_bf16_nvl_send_bytes = dispatch_bf16_nvl_recv_bytes + combine_bf16_rdma_recv_bytes = dispatch_bf16_rdma_send_bytes + + if local_rank == 0: + print(" passed", flush=True) + if local_rank == 0: + print("", flush=True) + + output_data = {} + + # Tune dispatch performance + best_dispatch_results = None + fp8_factor = (1 + 4 / 128) / 2 + for current_x in (x_e4m3, x): + best_time, best_results = 1e10, None + rdma_send_bytes = ( + (dispatch_bf16_rdma_send_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_rdma_send_bytes + ) + nvl_recv_bytes = ( + (dispatch_bf16_nvl_recv_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_nvl_recv_bytes + ) + for nvl_chunk_size in range(4, 33, 4): + for rdma_chunk_size in range(4, 33, 4): + config_kwargs = { + "num_sms": num_sms, + "num_max_nvl_chunked_send_tokens": nvl_chunk_size, + "num_max_nvl_chunked_recv_tokens": nvl_buffer_size, + "num_max_rdma_chunked_send_tokens": rdma_chunk_size, + "num_max_rdma_chunked_recv_tokens": rdma_buffer_size, + } + config = deep_ep.Config(**config_kwargs) + tune_args = {"x": current_x, "handle": handle, "config": config} + t = bench(lambda: buffer.dispatch(**tune_args))[0] + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + config_kwargs, + ) + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {rdma_send_bytes / 1e9 / t:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if local_rank == 0: + print( + f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)', + flush=True, + ) + print("", flush=True) + is_fp8 = isinstance(current_x, tuple) + if is_fp8: + output_data["normal_dispatch"] = deepcopy(best_results[3]) + + if isinstance(current_x, tuple): + # Gather FP8 the best config from rank 0 + best_dispatch_results = torch.tensor( + [best_results[0], best_results[1], best_results[2]], + dtype=torch.int32, + device="cuda", + ) + all_best_fp8_results_list = [ + torch.zeros_like(best_dispatch_results) + for _ in range(torch.distributed.get_world_size()) + ] + dist.all_gather( + all_best_fp8_results_list, best_dispatch_results, group=group + ) + best_dispatch_results = all_best_fp8_results_list[0].tolist() + dispatch_config = deep_ep.Config( + best_dispatch_results[0], + best_dispatch_results[1], + nvl_buffer_size, + best_dispatch_results[2], + rdma_buffer_size, + ) + + dispatch_args = { + "x": x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": dispatch_config if dispatch_config is not None else config, + } + recv_x, _, _, _, handle, _ = buffer.dispatch(**dispatch_args) + + # Tune combine performance + best_time, best_results = 1e10, None + for nvl_chunk_size in range(1, 5, 1): + for rdma_chunk_size in range(8, 33, 4): + config_kwargs = { + "num_sms": num_sms, + "num_max_nvl_chunked_send_tokens": nvl_chunk_size, + "num_max_nvl_chunked_recv_tokens": nvl_buffer_size, + "num_max_rdma_chunked_send_tokens": rdma_chunk_size, + "num_max_rdma_chunked_recv_tokens": rdma_buffer_size, + } + config = deep_ep.Config(**config_kwargs) + tune_args = {"x": recv_x, "handle": handle, "config": config} + t = bench(lambda: buffer.combine(**tune_args))[0] + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {combine_bf16_rdma_recv_bytes / 1e9 / t:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + config_kwargs, + ) + + if local_rank == 0: + print( + f"[tuning] Best combine: SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {combine_bf16_rdma_recv_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / best_time:.2f} GB/s (NVL)", + flush=True, + ) + print("", flush=True) + output_data["normal_combine"] = deepcopy(best_results[3]) + + if rank == 0 and local_rank == 0: + _write_output(args, output_data) + + +def _write_output(args, output_data): + text = json.dumps(output_data, indent=4) + output_path = args.output_path + print(f"Write to {output_path} with {text}") + Path(output_path).write_text(text) + + +# noinspection PyUnboundLocalVariable +def test_loop(local_rank: int, num_local_ranks: int, args): + num_nodes = args.nnodes + rank, num_ranks, group = init_dist(local_rank, num_local_ranks, args) + + num_sms = args.num_sms + num_qps_per_rank = num_sms // 2 + + buffer = deep_ep.Buffer( + group, + int(1e9), + int(1e9), + low_latency_mode=False, + num_qps_per_rank=num_qps_per_rank, + ) + assert num_local_ranks == 8 and num_ranks > 8 + torch.manual_seed(rank) + + for i in (num_sms,): + test_main( + i, + local_rank, + num_local_ranks, + num_ranks, + num_nodes, + rank, + buffer, + group, + args, + ) + if local_rank == 0: + print("", flush=True) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-sms", type=int, default=24) + parser.add_argument("--output-path", type=str, default="deepep_tuned.json") + parser.add_argument("--nnodes", type=int, default=1) + parser.add_argument("--node-rank", type=int, default=0) + parser.add_argument("--master-addr", type=str, default="127.0.0.1") + parser.add_argument("--master-port", type=int, default=8361) + args = parser.parse_args() + print(f"Start system with {args=}") + + num_processes = 8 + torch.multiprocessing.spawn( + test_loop, args=(num_processes, args), nprocs=num_processes + ) diff --git a/benchmark/kernels/deepseek/README.md b/benchmark/kernels/deepseek/README.md new file mode 100644 index 000000000..fba775f23 --- /dev/null +++ b/benchmark/kernels/deepseek/README.md @@ -0,0 +1,19 @@ +## DeepSeek kernels benchmark + + +### Prerequisites +- You should install [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) from source before run `benchmark_deepgemm_fp8_gemm.py` and `benchmark_deepgemm_fp8_group_gemm.py`. + +### Benchmark +- `benchmark_deepgemm_fp8_gemm.py` + ```bash + python benchmark_deepgemm_fp8_gemm.py --run_correctness --tp_size 1 + ``` + +- `benchmark_deepgemm_fp8_group_gemm.py` + ```bash + python benchmark_deepgemm_fp8_group_gemm.py --run_correctness --tp_size 1 + ``` + + - You can use the `--run_correctness` parameter to verify all kernels results's correctness. + - You can use the `--tp_size` parameter to benchmark all FP8 w8a8 block-wise matrix multiplications involved in DeepSeek V3/R1 under the current tensor parallelism (TP) setting. This benchmark compares DeepSeek's open-source [DeepGemm](https://github.com/deepseek-ai/DeepGEMM) implementation with SGLang's and VLLM Triton implementation. diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py new file mode 100644 index 000000000..f93732154 --- /dev/null +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_gemm.py @@ -0,0 +1,400 @@ +from typing import Tuple + +import deep_gemm +import tilelang +import tilelang.language as T +import torch +import triton +from deep_gemm import ceil_div, get_col_major_tma_aligned_tensor +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + w8a8_block_fp8_matmul as vllm_w8a8_block_fp8_matmul, +) + +from sglang.srt.layers.quantization.fp8_kernel import ( + w8a8_block_fp8_matmul_deepgemm as w8a8_block_fp8_matmul, +) + + +# Adapted from https://github.com/tile-ai/tilelang/blob/a8cfdce92795cb861c9033573534653ee040b5ed/examples/deepseek_deepgemm/example_deepgemm_fp8_2xAcc.py#L1 +def tl_gemm( + M, + N, + K, + in_dtype, + out_dtype, + accum_dtype, +): + assert in_dtype in [ + "e4m3_float8", + ], "Currently only e4m3_float8 is supported" + assert out_dtype in [ + "bfloat16", + "float16", + ], "Currently only bfloat16 and float16 are supported" + + TILE_SIZE = (128, 128, 128) + block_M = TILE_SIZE[0] + block_N = TILE_SIZE[1] + block_K = TILE_SIZE[2] + + A_shape = (M, K) + Scales_A_shape = (M, T.ceildiv(K, block_K)) + B_shape = (N, K) + Scales_B_shape = (T.ceildiv(N, block_N), T.ceildiv(K, block_K)) + A_shared_shape = (block_M, block_K) + B_shared_shape = (block_N, block_K) + C_shared_shape = (block_M, block_N) + + @T.prim_func + def main( + A: T.Buffer(A_shape, in_dtype), + scales_a: T.Buffer(Scales_A_shape, "float32"), + B: T.Buffer(B_shape, in_dtype), + scales_b: T.Buffer(Scales_B_shape, "float32"), + C: T.Buffer((M, N), out_dtype), + ): + with T.Kernel(T.ceildiv(N, block_N), T.ceildiv(M, block_M), threads=128) as ( + bx, + by, + ): + + A_shared = T.alloc_shared(A_shared_shape, in_dtype) + B_shared = T.alloc_shared(B_shared_shape, in_dtype) + C_shared = T.alloc_shared(C_shared_shape, out_dtype) + Scale_C_shared = T.alloc_shared((block_M), "float32") + C_local = T.alloc_fragment(C_shared_shape, accum_dtype) + C_local_accum = T.alloc_fragment(C_shared_shape, accum_dtype) + + # Improve L2 Cache + T.use_swizzle(panel_size=10) + + T.clear(C_local) + T.clear(C_local_accum) + K_iters = T.ceildiv(K, block_K) + for k in T.Pipelined(K_iters, num_stages=4): + # Load A into shared memory + T.copy(A[by * block_M, k * block_K], A_shared) + # Load B into shared memory + T.copy(B[bx * block_N, k * block_K], B_shared) + # Load scale into shared memory + Scale_B = scales_b[bx, k] + for i in T.Parallel(block_M): + Scale_C_shared[i] = scales_a[by * block_M + i, k] * Scale_B + + T.gemm(A_shared, B_shared, C_local, transpose_B=True) + # Promote to enable 2xAcc + for i, j in T.Parallel(block_M, block_N): + C_local_accum[i, j] += C_local[i, j] * Scale_C_shared[i] + T.clear(C_local) + # TMA store + T.copy(C_local_accum, C_shared) + T.copy(C_shared, C[by * block_M, bx * block_N]) + + return main + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def fp8_gemm_deepgemm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """DeepGEMM implementation of FP8 GEMM""" + out = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + + # Run DeepGEMM kernel + deep_gemm.gemm_fp8_fp8_bf16_nt((x_fp8, x_scale), (y_fp8, y_scale), out) + return out + + +def fp8_gemm_sglang( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """SGLang implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run SGLang kernel + out = w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def fp8_gemm_vllm( + x_fp8: torch.Tensor, + x_scale: torch.Tensor, + y_fp8: torch.Tensor, + y_scale: torch.Tensor, + m: int, + n: int, + k: int, +): + """vLLM implementation of FP8 GEMM""" + block_size = [128, 128] # Matches the block size in per_block_cast_to_fp8 + + # Run vLLM kernel + out = vllm_w8a8_block_fp8_matmul( + x_fp8, y_fp8, x_scale, y_scale, block_size, torch.bfloat16 + ) + return out + + +def calculate_diff(m: int, n: int, k: int): + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + x_fp8, x_scale = per_token_cast_to_fp8(x.clone()) + y_fp8, y_scale = per_block_cast_to_fp8(y.clone()) + x_scale_col_major = get_col_major_tma_aligned_tensor(x_scale.clone()) + + out_deepgemm = fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ) + out_sglang = fp8_gemm_sglang( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone(), m, n, k + ) + + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + out_tilelang = tilelang_kernel( + x_fp8.clone(), x_scale.clone(), y_fp8.clone(), y_scale.clone() + ) + + diff_sglang_deepgemm = torch.abs(out_deepgemm - out_sglang).mean().item() + diff_tilelang_deepgemm = torch.abs(out_deepgemm - out_tilelang).mean().item() + diff_tilelang_sglang = torch.abs(out_tilelang - out_sglang).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"SGLang output: {out_sglang[0, 0:5]}") + print(f"TileLang output: {out_tilelang[0, 0:5]}") + print(f"Mean absolute difference (SGLang-DeepGEMM): {diff_sglang_deepgemm}") + print(f"Mean absolute difference (TileLang-DeepGEMM): {diff_tilelang_deepgemm}") + print(f"Mean absolute difference (TileLang-SGLang): {diff_tilelang_sglang}") + + sglang_deepgemm_match = torch.allclose( + out_deepgemm, out_sglang, atol=1e-2, rtol=1e-2 + ) + tilelang_deepgemm_match = torch.allclose( + out_deepgemm, out_tilelang, atol=1e-2, rtol=1e-2 + ) + tilelang_sglang_match = torch.allclose( + out_tilelang, out_sglang, atol=1e-2, rtol=1e-2 + ) + + if sglang_deepgemm_match and tilelang_deepgemm_match and tilelang_sglang_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print(f" - SGLang vs DeepGEMM: {'✅' if sglang_deepgemm_match else '❌'}") + print(f" - TileLang vs DeepGEMM: {'✅' if tilelang_deepgemm_match else '❌'}") + print(f" - TileLang vs SGLang: {'✅' if tilelang_sglang_match else '❌'}\n") + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [8, 16, 32, 64, 128, 256, 1024, 2048, 4096] + + for n, k in weight_shapes: + for m in batch_sizes: + configs.append((m, n, k, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "tp_size"], + x_vals=[list(config) for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "sglang", "tilelang"], + line_names=["DeepGEMM", "SGLang", "TileLang"], + styles=[("blue", "-"), ("red", "-"), ("green", "-")], + ylabel="ms", + plot_name=f"fp8-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, tp_size, provider): + print(f"Shape (m={m}, n={n}, k={k}, tp={tp_size}), Provider: {provider}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + + # Preprocess data before benchmarking + x_fp8, x_scale = per_token_cast_to_fp8(x) + y_fp8, y_scale = per_block_cast_to_fp8(y) + x_scale_col_major = get_col_major_tma_aligned_tensor(x_scale.clone()) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_deepgemm( + x_fp8.clone(), + x_scale_col_major.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + elif provider == "sglang": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_sglang( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + m, + n, + k, + ), + quantiles=quantiles, + ) + else: # tilelang + tilelang_func = tl_gemm(m, n, k, "e4m3_float8", "bfloat16", "float32") + tilelang_kernel = tilelang.compile(tilelang_func, out_idx=[-1]) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: tilelang_kernel( + x_fp8.clone(), + x_scale.clone(), + y_fp8.clone(), + y_scale.clone(), + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + # Print shape-specific results with TFLOPS + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_gemm/", + help="Path to save fp8 gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + default=True, + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(64, 512, 7168) # Small test + calculate_diff(64, 7168, 16384) # Medium test + calculate_diff(64, 18432, 7168) # Large test + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py new file mode 100644 index 000000000..2c3e8dfcc --- /dev/null +++ b/benchmark/kernels/deepseek/benchmark_deepgemm_fp8_group_gemm.py @@ -0,0 +1,486 @@ +from typing import Tuple + +import deep_gemm +import torch +import triton +import triton.language as tl +from deep_gemm import calc_diff, get_col_major_tma_aligned_tensor + +# Import shared functionality from the regular GEMM benchmark +from sglang.benchmark.kernels.deepseek.benchmark_deepgemm_fp8_gemm import ( + per_block_cast_to_fp8, + per_token_cast_to_fp8, +) + + +def construct_grouped_and_flat_fp8( + x: torch.Tensor, y: torch.Tensor, num_groups: int, is_masked: bool +) -> Tuple[ + Tuple[torch.Tensor, torch.Tensor], # grouped x_fp8 + Tuple[torch.Tensor, torch.Tensor], # grouped y_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat x_fp8 + Tuple[torch.Tensor, torch.Tensor], # flat y_fp8 + torch.Tensor, # output + torch.Tensor, # reference output +]: + # Verify input shapes + m, k = x.shape + n, k_y = y.shape + assert k == k_y, f"Incompatible shapes: x({m}, {k}), y({n}, {k_y})" + assert m % num_groups == 0, f"m({m}) must be divisible by num_groups({num_groups})" + assert m % 4 == 0, f"TMA alignment error: {m}" + + # Reshape inputs for grouped processing + m_per_group = m // num_groups + x_grouped = x.view(num_groups, m_per_group, k) + y_grouped = y.unsqueeze(0).expand(num_groups, n, k) + + # Initialize output tensors + out = torch.empty((num_groups, m_per_group, n), device="cuda", dtype=torch.bfloat16) + ref_out = torch.einsum("gmk,gnk->gmn", x_grouped, y_grouped) + + # Quantize grouped tensors + x_fp8_grouped = ( + torch.empty_like(x_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, m_per_group, k // 128), device="cuda", dtype=torch.float + ), + ) + y_fp8_grouped = ( + torch.empty_like(y_grouped, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, (n + 127) // 128, k // 128), device="cuda", dtype=torch.float + ), + ) + for i in range(num_groups): + x_fp8_grouped[0][i], x_fp8_grouped[1][i] = per_token_cast_to_fp8(x_grouped[i]) + y_fp8_grouped[0][i], y_fp8_grouped[1][i] = per_block_cast_to_fp8(y_grouped[i]) + + # Quantize flat tensors + x_fp8_flat = per_token_cast_to_fp8(x) + y_fp8_flat = per_block_cast_to_fp8(y) + + # For non-masked input, merge the group and M dims in output + if not is_masked: + x_fp8_grouped = ( + x_fp8_grouped[0].view(-1, k), + per_token_cast_to_fp8(x_grouped.view(-1, k))[1], + ) + out, ref_out = out.view(-1, n), ref_out.view(-1, n) + + # Transpose earlier for testing + x_fp8_grouped = ( + x_fp8_grouped[0], + get_col_major_tma_aligned_tensor(x_fp8_grouped[1]), + ) + x_fp8_flat = (x_fp8_flat[0], get_col_major_tma_aligned_tensor(x_fp8_flat[1])) + + return x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, ref_out + + +# Since we don't have a group gemm kernel in SGLang/vLLM, we implemented a +# custom kernel based on the Triton tutorial. +# https://triton-lang.org/main/getting-started/tutorials/03-matrix-multiplication.html +@triton.jit +def fp8_gemm_group_triton_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + # Pointers to scaling factors + a_scale_ptr, + b_scale_ptr, + # Matrix dimensions + M, + N, + K, + # The stride variables represent how much to increase the ptr by when moving by 1 + # element in a particular dimension. + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + # Strides for scaling factors + stride_a_scale_m, + stride_a_scale_k, + stride_b_scale_n, + stride_b_scale_k, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Kernel for computing the matmul C = A x B with FP8 inputs and scaling factors. + A has shape (M, K), B has shape (K, N) and C has shape (M, N) + + Note: Block sizes must be multiples of 32 for optimal TMA performance. + """ + # Map program ids to the block of C it should compute + pid_group = tl.program_id(axis=0) # Group ID + pid_n = tl.program_id(axis=1) # N dimension ID + + # Compute the M block ID within this group + group_size_m = min(M - pid_group * GROUP_SIZE_M, GROUP_SIZE_M) + pid_m_within_group = tl.program_id(axis=2) % group_size_m + pid_m = pid_group * GROUP_SIZE_M + pid_m_within_group + + # Create pointers for the first blocks of A and B + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + # Initialize accumulator + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + # Main loop + for k_block in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + k_offset = k_block * BLOCK_SIZE_K + + # Load the next block of A and B, with masks + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k_offset, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k_offset, other=0.0) + + # Calculate indices for scaling factors for this K block + a_scale_ptrs = a_scale_ptr + ( + offs_am * stride_a_scale_m + k_block * stride_a_scale_k + ) + b_scale_ptrs = b_scale_ptr + ( + pid_n * stride_b_scale_n + k_block * stride_b_scale_k + ) + + # Perform matrix multiplication in FP8 + res = tl.dot(a, b) + + # Load scaling factors for the current block + a_scale = tl.load(a_scale_ptrs)[:, None] # [BLOCK_SIZE_M, 1] + b_scale = tl.load(b_scale_ptrs) + + # Apply scaling factors to the accumulated result + accumulator += res * a_scale * b_scale + + # Advance pointers + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # Convert to bfloat16 for output + c = accumulator.to(tl.bfloat16) + + # Write back the result + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +def fp8_gemm_group_triton(a_tuple, b_tuple, c, num_groups): + """ + Perform matrix multiplication with FP8 inputs and proper scaling. + + Args: + a_tuple: Tuple of (quantized_tensor, scale_factors) for input A + b_tuple: Tuple of (quantized_tensor, scale_factors) for input B + c: Output tensor in BF16 format + num_groups: Number of groups for grouped GEMM + + Returns: + Result tensor in BF16 format + """ + # Unpack the tuples + a, a_scale = a_tuple + b, b_scale = b_tuple + + M, K = a.shape + _, N = b.shape + + # Configure block sizes - must be multiples of 32 for TMA alignment + BLOCK_SIZE_M = 128 + BLOCK_SIZE_N = 128 + BLOCK_SIZE_K = 128 + + # Calculate grid dimensions + num_pid_m = triton.cdiv(M, BLOCK_SIZE_M) + num_pid_n = triton.cdiv(N, BLOCK_SIZE_N) + num_groups_grid = triton.cdiv(num_pid_m, num_groups) + + # 3D grid launch - (group, n_blocks, m_blocks_per_group) + grid = (num_groups_grid, num_pid_n, min(num_groups, num_pid_m)) + + fp8_gemm_group_triton_kernel[grid]( + a, + b, + c, + a_scale, + b_scale, + M, + N, + K, + a.stride(0), + a.stride(1), + b.stride(0), + b.stride(1), + c.stride(0), + c.stride(1), + a_scale.stride(0), + 1, # Stride in the K dimension may be 1 + b_scale.stride(0), + 1 if b_scale.dim() > 1 else 0, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + GROUP_SIZE_M=num_groups, + ) + + return c + + +def fp8_gemm_group_deepgemm(x_fp8_grouped, y_fp8_grouped, out, m_indices): + deep_gemm.m_grouped_gemm_fp8_fp8_bf16_nt_contiguous( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ) + return out + + +def calculate_diff(m: int, n: int, k: int, num_groups: int): + print(f"Shape (m={m}, n={n}, k={k}") + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + out_deepgemm = out.clone() + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1).expand(num_groups, m_per_group).contiguous().view(-1) + ) + + fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out_deepgemm, + m_indices, + ) + torch.cuda.synchronize() + + # Prepare inputs for Triton + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + out_triton = fp8_gemm_group_triton((a, a_scale), (b, b_scale), c, num_groups) + torch.cuda.synchronize() + + diff_torch_deepgemm = torch.abs(out_torch - out_deepgemm).mean().item() + diff_torch_triton = torch.abs(out_torch - out_triton).mean().item() + diff_deepgemm_triton = torch.abs(out_deepgemm - out_triton).mean().item() + + print(f"Shape m={m}, n={n}, k={k}:") + print(f"Torch output: {out_torch[0, 0:5]}") + print(f"DeepGEMM output: {out_deepgemm[0, 0:5]}") + print(f"Triton output: {out_triton[0, 0:5]}") + print(f"Mean absolute difference (Torch-DeepGEMM): {diff_torch_deepgemm}") + print(f"Mean absolute difference (Torch-Triton): {diff_torch_triton}") + print(f"Mean absolute difference (DeepGEMM-Triton): {diff_deepgemm_triton}") + + deepgemm_torch_diff = calc_diff(out_deepgemm, out_torch) + triton_torch_diff = calc_diff(out_triton, out_torch) + deepgemm_triton_diff = calc_diff(out_deepgemm, out_triton) + + DIFF_THRESHOLD = 0.001 + all_match = ( + deepgemm_torch_diff < DIFF_THRESHOLD + and triton_torch_diff < DIFF_THRESHOLD + and deepgemm_triton_diff < DIFF_THRESHOLD + ) + if all_match: + print("✅ All implementations match\n") + else: + print("❌ Some implementations differ:") + print( + f" - Torch vs DeepGEMM: {'✅' if deepgemm_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - Torch vs Triton: {'✅' if triton_torch_diff < DIFF_THRESHOLD else '❌'}" + f" - DeepGEMM vs Triton: {'✅' if deepgemm_triton_diff < DIFF_THRESHOLD else '❌'}" + ) + + +def get_weight_shapes(tp_size): + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + + return weight_shapes + + +def create_benchmark_configs(tp_size): + configs = [] + weight_shapes = get_weight_shapes(tp_size) + batch_sizes = [2048, 4096] + group_sizes = [4, 8] + for n, k in weight_shapes: + for m in batch_sizes: + for num_groups in group_sizes: + configs.append((m, n, k, num_groups, tp_size)) + + return configs + + +def get_benchmark(tp_size): + all_configs = create_benchmark_configs(tp_size) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["m", "n", "k", "num_groups", "tp_size"], + x_vals=[config for config in all_configs], + line_arg="provider", + line_vals=["deepgemm", "triton"], + line_names=["DeepGEMM", "Triton"], + styles=[("blue", "-"), ("red", "-")], + ylabel="ms", + plot_name=f"fp8-group-gemm-performance-comparison-tp{tp_size}", + args={}, + ) + ) + def benchmark(m, n, k, num_groups, tp_size, provider): + print( + f"Shape (m={m}, n={n}, k={k}, tp={tp_size}, num_groups={num_groups}, Provider: {provider}" + ) + x = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + y = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + x_fp8_grouped, y_fp8_grouped, x_fp8_flat, y_fp8_flat, out, out_torch = ( + construct_grouped_and_flat_fp8(x, y, num_groups, is_masked=False) + ) + m_per_group = m // num_groups + m_indices = torch.arange(0, num_groups, device="cuda", dtype=torch.int) + m_indices = ( + m_indices.unsqueeze(-1) + .expand(num_groups, m_per_group) + .contiguous() + .view(-1) + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "deepgemm": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_deepgemm( + x_fp8_grouped, + y_fp8_grouped, + out, + m_indices, + ), + quantiles=quantiles, + ) + elif provider == "triton": + # Prepare inputs for Triton + # We did it outside of the lambda function to make it fair comparison like deepgemm + a, a_scale = x_fp8_flat + b, b_scale = y_fp8_flat + b = b.T.contiguous() + # Ensure scales are in the right format and contiguous + a_scale, b_scale = a_scale.contiguous(), b_scale.contiguous() + M, _ = a.shape + _, N = b.shape + c = torch.empty((M, N), device=a.device, dtype=torch.bfloat16) + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: fp8_gemm_group_triton( + (a, a_scale), + (b, b_scale), + c, + num_groups, + ), + quantiles=quantiles, + ) + + # Calculate TFLOPS + flops = 2 * m * n * k # multiply-adds + tflops = flops / (ms * 1e-3) / 1e12 + + print(f"Time: {ms*1000:.2f} ms, TFLOPS: {tflops:.2f}") + return ms * 1000, max_ms * 1000, min_ms * 1000 # convert to ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/fp8_group_gemm/", + help="Path to save deepgemm fp8 group gemm benchmark results", + ) + parser.add_argument( + "--run_correctness", + action="store_true", + help="Whether to run correctness test", + ) + parser.add_argument( + "--tp_size", + type=int, + default=1, + help="Tensor parallelism size to benchmark (default: 1)", + ) + args = parser.parse_args() + + # Set random seed for reproducibility + torch.manual_seed(0) + torch.cuda.manual_seed(0) + + # Enable TF32, adapted from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_core.py#L148 + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + + # Run correctness tests on a few examples + if args.run_correctness: + print("Running correctness tests...") + calculate_diff(8192, 7168, 4096, 4) + calculate_diff(8192, 2048, 7168, 4) + calculate_diff(4096, 7168, 4096, 8) + calculate_diff(4096, 2048, 7168, 8) + calculate_diff(4096, 576, 7168, 8) + + # Get the benchmark function with the specified tp_size + benchmark = get_benchmark(args.tp_size) + + print(f"Running performance benchmark for TP size = {args.tp_size}...") + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/fbgemm/README.md b/benchmark/kernels/fbgemm/README.md new file mode 100644 index 000000000..e51356d8a --- /dev/null +++ b/benchmark/kernels/fbgemm/README.md @@ -0,0 +1,29 @@ +## Benchmark FBGEMM Grouped GEMM + +Benchmark FBGEMM Grouped GEMM in both Triton and CUDA version and SGLang Triton Grouped GEMM, it will be used to compare the bandwidth of different implementations. + +### Requirements + +```shell +pip install fbgemm-gpu-genai +``` + +### Usage + +```bash +python3 benchmark/fbgemm/benchmark_fbgemm_grouped_gemm.py --model Qwen/Qwen2-57B-A14B-Instruct --tp-size 4 --use-fp8-w8a8 +``` + +For example, in H200, the Qwen2-57B-A14B-Instruct TP4 fp8w8a8 grouped gemm bandwidth result is as follows: + +```shell +grouped-gemm-performance: + batch_size FBGEMM Triton Grouped GEMM FP8 FBGEMM CUTLASS F8F8BF16 Rowwise SGLang Grouped GEMM FP8 +0 256.0 3704.841339 3042.626402 2254.725030 +1 512.0 3691.426346 3029.065684 2269.504543 +2 1024.0 3653.938629 2258.471467 2358.319020 +3 2048.0 3596.644313 2271.611904 2476.895397 +4 4096.0 3468.496435 2231.283986 2179.473910 +``` + +The theoretical peak bandwidth of H200 is 4.8 TB/s. Taking batch_size 256 as an example, the bandwidth of FBGEMM Triton Grouped GEMM FP8 is 3704.841339 GB/s, the bandwidth of FBGEMM CUTLASS F8F8BF16 Rowwise is 3042.626402 GB/s, and the bandwidth of SGLang Grouped GEMM FP8 is 2254.725030 GB/s. Therefore, FBGEMM Triton Grouped GEMM FP8 achieves 77.9% of H200's theoretical peak bandwidth, FBGEMM CUTLASS F8F8BF16 Rowwise achieves 63.4% of H200's theoretical peak bandwidth, and SGLang Grouped GEMM FP8 achieves 46.9% of H200's theoretical peak bandwidth. diff --git a/benchmark/kernels/fbgemm/benchmark_fbgemm_grouped_gemm.py b/benchmark/kernels/fbgemm/benchmark_fbgemm_grouped_gemm.py new file mode 100644 index 000000000..6e8c8dcf2 --- /dev/null +++ b/benchmark/kernels/fbgemm/benchmark_fbgemm_grouped_gemm.py @@ -0,0 +1,516 @@ +# python3 benchmark/fbgemm/benchmark_fbgemm_grouped_gemm.py --model Qwen/Qwen2-57B-A14B-Instruct --tp-size 4 --use-fp8-w8a8 +import argparse + +import torch +import triton +from fbgemm_gpu.experimental.gemm.triton_gemm.fp8_gemm import ( + quantize_fp8_row, + triton_quantize_fp8_row, +) +from fbgemm_gpu.experimental.gemm.triton_gemm.grouped_gemm import ( + grouped_gemm as fbgemm_grouped_gemm, +) +from fbgemm_gpu.experimental.gemm.triton_gemm.grouped_gemm import ( + grouped_gemm_fp8_rowwise as fbgemm_grouped_gemm_fp8_rowwise, +) +from transformers import AutoConfig + +from sglang.srt.layers.moe.ep_moe.kernels import ( + grouped_gemm_triton as sglang_grouped_gemm, +) + + +def get_model_config(model_name: str, tp_size: int): + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + num_groups = config.ffn_config.moe_num_experts + intermediate_size = config.ffn_config.ffn_hidden_size + elif config.architectures[0] == "JambaForCausalLM": + num_groups = config.num_experts + intermediate_size = config.intermediate_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + num_groups = config.num_experts + intermediate_size = config.moe_intermediate_size + elif config.architectures[0] == "Qwen3MoeForCausalLM": + num_groups = config.num_experts + intermediate_size = config.moe_intermediate_size + elif config.architectures[0] in [ + "DeepseekV2ForCausalLM", + "DeepseekV3ForCausalLM", + ]: + num_groups = config.n_routed_experts + intermediate_size = config.moe_intermediate_size + elif config.architectures[0] == "Llama4ForConditionalGeneration": + num_groups = config.text_config.num_local_experts + intermediate_size = config.text_config.intermediate_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + num_groups = config.num_local_experts + intermediate_size = config.moe_intermediate_size + else: + num_groups = config.num_local_experts + intermediate_size = config.intermediate_size + + shape_configs = { + "num_groups": num_groups, + "hidden_size": config.hidden_size, + "intermediate_size": intermediate_size, + "dtype": config.torch_dtype, + } + print(f"{shape_configs=}") + return shape_configs + + +def create_test_data(batch_size, num_groups, hidden_size, intermediate_size): + torch.manual_seed(42) + + tokens_per_group = batch_size // num_groups + m_sizes = torch.full( + (num_groups,), tokens_per_group, dtype=torch.int32, device="cuda" + ) + + x = torch.randn(batch_size, hidden_size, dtype=torch.bfloat16, device="cuda") + + base_weights = torch.randn( + num_groups, intermediate_size, hidden_size, dtype=torch.bfloat16, device="cuda" + ) + + w_fbgemm = base_weights.reshape(num_groups * intermediate_size, hidden_size) + w_sglang = base_weights + + c_fbgemm = torch.empty( + batch_size, intermediate_size, dtype=torch.bfloat16, device="cuda" + ) + c_sglang = torch.empty( + batch_size, intermediate_size, dtype=torch.bfloat16, device="cuda" + ) + + seg_indptr = torch.zeros(num_groups + 1, dtype=torch.int32, device="cuda") + for i in range(1, num_groups + 1): + seg_indptr[i] = seg_indptr[i - 1] + tokens_per_group + + weight_indices = torch.arange(num_groups, dtype=torch.int32, device="cuda") + + return ( + x, + w_fbgemm, + w_sglang, + c_fbgemm, + c_sglang, + m_sizes, + seg_indptr, + weight_indices, + ) + + +def create_fp8_test_data( + batch_size, num_groups, hidden_size, intermediate_size, backend="triton" +): + """ + Create test data for FP8 grouped GEMM operations. + + Args: + batch_size: Total batch size + num_groups: Number of groups + hidden_size: Hidden dimension size + intermediate_size: Intermediate dimension size + backend: "triton" for Triton GEMM, "cutlass" for CUTLASS GEMM + + Returns: + For triton: (x_fp8, w_fp8, m_sizes, x_scale, w_scale) + For cutlass: (x, wq, w_scale, m_sizes) + """ + torch.manual_seed(42) + + tokens_per_group = batch_size // num_groups + + # Create weight matrices for each group + w_list = [] + for _ in range(num_groups): + w = torch.randn( + intermediate_size, hidden_size, dtype=torch.float16, device="cuda" + ) + w_list.append(w) + + # Quantize weights using quantize_fp8_row for each group + wq_list, w_scale_list = zip(*[quantize_fp8_row(w) for w in w_list]) + + if backend == "triton": + # Triton format: concatenated weights + w_fp8 = torch.concat(wq_list, dim=0).contiguous() + w_scale = torch.concat(w_scale_list, dim=0).contiguous() + + # Create m_sizes as int32 for triton + m_sizes = torch.full( + (num_groups,), tokens_per_group, dtype=torch.int32, device="cuda" + ) + + # Create and quantize input + x_fp16 = torch.randn( + batch_size, hidden_size, dtype=torch.float16, device="cuda" + ) + x_fp8, x_scale = triton_quantize_fp8_row(x_fp16) + x_scale = x_scale.view(batch_size, -1) + + return x_fp8, w_fp8, m_sizes, x_scale, w_scale + + elif backend == "cutlass": + # CUTLASS format: stacked weights + wq = torch.stack(wq_list, dim=0).contiguous() + w_scale = torch.stack(w_scale_list, dim=0).contiguous() + + # Create m_sizes as int64 for cutlass + m_values = [tokens_per_group] * num_groups + m_sizes = torch.tensor(m_values).to(dtype=torch.int64, device="cuda") + + # Create input data - separate for each group then concat + x_list = [] + for _ in range(num_groups): + x = torch.randn( + tokens_per_group, hidden_size, dtype=torch.float16, device="cuda" + ) + x_list.append(x) + + # Concatenate inputs into single tensor + x = torch.concat(x_list, dim=0).contiguous() + + return x, wq, w_scale, m_sizes + + else: + raise ValueError(f"Unsupported backend: {backend}") + + +def calculate_memory_bandwidth(m_sizes, hidden_size, intermediate_size, dtype): + """ + Calculate memory bandwidth based on accessed expert weights. + + Args: + m_sizes: Tensor containing batch sizes for each group + hidden_size: Hidden dimension size + intermediate_size: Intermediate dimension size + dtype: Data type of weights + + Returns: + Memory size in bytes for accessed expert weights + """ + # Count non-zero groups (active experts) + if hasattr(m_sizes, "cpu"): + active_experts = torch.count_nonzero(m_sizes).item() + else: + active_experts = sum(1 for m in m_sizes if m > 0) + + # Calculate bytes per element based on dtype + if dtype in [torch.float16, torch.bfloat16]: + bytes_per_element = 2 + elif dtype in [torch.float8_e4m3fn, torch.float8_e5m2]: + bytes_per_element = 1 + elif dtype == torch.float32: + bytes_per_element = 4 + else: + # Default to 2 bytes for unknown dtypes + bytes_per_element = 2 + + # Memory per expert weight matrix + memory_per_expert = hidden_size * intermediate_size * bytes_per_element + + # Total memory for active experts + total_memory_bytes = active_experts * memory_per_expert + + return total_memory_bytes + + +def get_benchmark_config(use_fp8_w8a8=False): + if use_fp8_w8a8: + return { + "line_vals": [ + "fbgemm_triton_grouped_gemm_fp8", + "fbgemm_cutlass_f8f8bf16_rowwise", + "sglang_grouped_gemm", + ], + "line_names": [ + "FBGEMM Triton Grouped GEMM FP8", + "FBGEMM CUTLASS F8F8BF16 Rowwise", + "SGLang Grouped GEMM FP8", + ], + "styles": [("blue", "-"), ("orange", "-"), ("red", "-")], + } + else: + return { + "line_vals": ["fbgemm_triton_grouped_gemm", "sglang_grouped_gemm"], + "line_names": [ + "FBGEMM Triton Grouped GEMM BF16", + "SGLang Grouped GEMM BF16", + ], + "styles": [("blue", "-"), ("green", "-")], + } + + +def run_benchmark( + model_config, use_fp8_w8a8=False, save_path="./benchmark_grouped_gemm/" +): + config = get_benchmark_config(use_fp8_w8a8) + + benchmark_config = triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[256, 512, 1024, 2048, 4096], + line_arg="provider", + line_vals=config["line_vals"], + line_names=config["line_names"], + styles=config["styles"], + ylabel="Bandwidth (GB/s)", + plot_name="grouped-gemm-performance", + args={}, + ) + + @triton.testing.perf_report(benchmark_config) + def dynamic_benchmark(batch_size, provider, model_config, use_fp8_w8a8=False): + print(f"Benchmarking {provider} with batch_size={batch_size}") + torch.cuda.manual_seed_all(0) + + num_groups = model_config["num_groups"] + hidden_size = model_config["hidden_size"] + intermediate_size = model_config["intermediate_size"] + + if provider == "fbgemm_triton_grouped_gemm_fp8": + try: + test_data = create_fp8_test_data( + batch_size, + num_groups, + hidden_size, + intermediate_size, + backend="triton", + ) + x_fp8, w_fp8, m_sizes, x_scale, w_scale = test_data + + # Calculate memory bandwidth + memory_bytes = calculate_memory_bandwidth( + m_sizes, hidden_size, intermediate_size, torch.float8_e4m3fn + ) + + def run_func(): + return fbgemm_grouped_gemm_fp8_rowwise( + x_fp8, w_fp8, m_sizes, x_scale, w_scale, use_fast_accum=True + ) + + except Exception as e: + print(f"FP8 not supported, skipping: {e}") + return float("inf"), float("inf"), float("inf") + + elif provider == "fbgemm_cutlass_f8f8bf16_rowwise": + try: + test_data = create_fp8_test_data( + batch_size, + num_groups, + hidden_size, + intermediate_size, + backend="cutlass", + ) + x, wq, w_scale, m_sizes = test_data + + # Calculate memory bandwidth + memory_bytes = calculate_memory_bandwidth( + m_sizes, hidden_size, intermediate_size, torch.float8_e4m3fn + ) + + # Quantize input using triton_quantize_fp8_row + xq, x_scale = triton_quantize_fp8_row(x) + x_scale = x_scale.view(batch_size, -1) + + def run_func(): + return torch.ops.fbgemm.f8f8bf16_rowwise_grouped_stacked( + xq, wq, x_scale, w_scale, m_sizes + ) + + except Exception as e: + print( + f"CUTLASS f8f8bf16_rowwise_grouped_stacked not supported, " + f"skipping: {e}" + ) + return float("inf"), float("inf"), float("inf") + else: + test_data = create_test_data( + batch_size, num_groups, hidden_size, intermediate_size + ) + ( + x, + w_fbgemm, + w_sglang, + c_fbgemm, + c_sglang, + m_sizes, + seg_indptr, + weight_indices, + ) = test_data + + # Calculate memory bandwidth for BF16 operations + memory_bytes = calculate_memory_bandwidth( + m_sizes, hidden_size, intermediate_size, torch.bfloat16 + ) + + if provider == "fbgemm_triton_grouped_gemm": + + def run_func(): + return fbgemm_grouped_gemm( + x, w_fbgemm, m_sizes, use_fast_accum=True + ) + + else: + + def run_func(): + return sglang_grouped_gemm( + x, + w_sglang, + c_sglang, + num_groups, + weight_column_major=True, + seg_indptr=seg_indptr, + weight_indices=weight_indices, + c_dtype=c_sglang.dtype, + ) + + for _ in range(10): + try: + run_func() + except Exception as e: + print(f"Error during warmup for {provider}: {e}") + return float("inf"), float("inf"), float("inf") + + torch.cuda.synchronize() + + try: + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench(run_func, quantiles=quantiles) + + # Convert time (ms) to bandwidth (GB/s) + # Bandwidth = Memory (bytes) / Time (seconds) + # Convert ms to seconds and bytes to GB (1e9) + gb_per_s = (memory_bytes / 1e9) / (ms / 1000) + # min bandwidth = max time, max bandwidth = min time + min_gb_per_s = (memory_bytes / 1e9) / (max_ms / 1000) + max_gb_per_s = (memory_bytes / 1e9) / (min_ms / 1000) + + return gb_per_s, min_gb_per_s, max_gb_per_s + except Exception as e: + print(f"Error during benchmarking for {provider}: {e}") + return 0.0, 0.0, 0.0 + + dynamic_benchmark.run( + show_plots=True, + print_data=True, + save_path=save_path, + model_config=model_config, + use_fp8_w8a8=use_fp8_w8a8, + ) + + +def verify_correctness(model_config): + print("Verifying correctness...") + batch_size = 128 + num_groups = model_config["num_groups"] + hidden_size = model_config["hidden_size"] + intermediate_size = model_config["intermediate_size"] + + test_data = create_test_data(batch_size, num_groups, hidden_size, intermediate_size) + ( + x, + w_fbgemm, + w_sglang, + c_fbgemm, + c_sglang, + m_sizes, + seg_indptr, + weight_indices, + ) = test_data + + result_fbgemm = fbgemm_grouped_gemm(x, w_fbgemm, m_sizes, use_fast_accum=True) + + result_sglang = sglang_grouped_gemm( + x, + w_sglang, + c_sglang, + num_groups, + weight_column_major=True, + seg_indptr=seg_indptr, + weight_indices=weight_indices, + c_dtype=c_sglang.dtype, + ) + + if torch.allclose(result_fbgemm, result_sglang, rtol=1e-3, atol=1e-3): + print("✓ BF16 Correctness verification passed!") + else: + max_diff = torch.max(torch.abs(result_fbgemm - result_sglang)) + print(f"✗ BF16 Correctness verification failed! Max diff: {max_diff}") + return False + + return True + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark FBGEMM vs SGLang Grouped GEMM" + ) + parser.add_argument( + "--model", + type=str, + default="mistralai/Mixtral-8x7B-Instruct-v0.1", + help="Model name to get configuration from", + ) + parser.add_argument( + "--tp-size", type=int, default=1, help="Tensor parallelism size" + ) + parser.add_argument( + "--use-fp8-w8a8", action="store_true", help="Enable FP8 W8A8 benchmark" + ) + parser.add_argument( + "--save-path", + type=str, + default="./benchmark_grouped_gemm/", + help="Path to save benchmark results", + ) + parser.add_argument( + "--verify-correctness", + action="store_true", + help="Verify correctness before benchmarking", + ) + + args = parser.parse_args() + + try: + model_config = get_model_config(args.model, args.tp_size) + except Exception as e: + print(f"Failed to get model config: {e}") + print("Using default configuration...") + model_config = { + "num_groups": 8, + "hidden_size": 4096, + "intermediate_size": 14336, + "dtype": torch.bfloat16, + } + + print("Running benchmark with:") + print(f" num_groups: {model_config['num_groups']}") + print(f" hidden_size: {model_config['hidden_size']}") + print(f" intermediate_size: {model_config['intermediate_size']}") + print(f" use_fp8_w8a8: {args.use_fp8_w8a8}") + + if args.verify_correctness: + if not verify_correctness(model_config): + print("Correctness verification failed. Exiting...") + return + + try: + run_benchmark( + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + save_path=args.save_path, + ) + except Exception as e: + print(f"Benchmark failed: {e}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/flashinfer_allreduce_fusion/README.md b/benchmark/kernels/flashinfer_allreduce_fusion/README.md new file mode 100644 index 000000000..e651604c7 --- /dev/null +++ b/benchmark/kernels/flashinfer_allreduce_fusion/README.md @@ -0,0 +1,102 @@ +# FlashInfer Fused AllReduce + RMSNorm Benchmark + +This benchmark script is modified from the [original implementation](https://github.com/vllm-project/vllm/blob/237e1fb887c7f5a579420fa0295097f24b006594/benchmarks/kernels/benchmark_fused_collective.py) by the vLLM community. It aims to compare the performance differences between FlashInfer fused operators in SGLang (trtllm_allreduce_fusion: AllReduce + Residual Add + RMSNorm + optional quantization) and conventional implementations (standard `tensor_model_parallel_all_reduce` + separate RMSNorm/quantization). Specifically, this script tests the timing performance of two implementation paths: 1) Standard AllReduce and RMSNorm executed separately; 2) FlashInfer's fused operator combining AllReduce, Residual Add, RMSNorm, and optional quantization operations. + +This benchmark script helps us tune the ipc workspace size of the `flashinfer_allreduce_residual_rmsnorm` operator in SGLang and prepare for applications with FP8/FP4 quantized fused operators. + +Script path: `benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py` + +## Feature Overview + +- Compare average execution time (ms) and calculate speedup ratios for the following paths: + - standard_allreduce_rmsnorm (Standard AllReduce + RMSNorm) + - flashinfer_fused_allreduce_rmsnorm (Fused AllReduce + RMSNorm), including oneshot and twoshot modes + - Optionally compare FP8/FP4 quantized fused paths with standard paths +- Use CUDA Graph capture and batch replay to reduce measurement noise +- Automatically select the faster "standard baseline" (native/compiled version) as the denominator for speedup calculation +- Optionally export results in Markdown format + +## Runtime Environment and Prerequisites + +- At least 2 GPUs, and launch multi-process distributed training using `torchrun` (NCCL backend) +- Properly install/compile sglang along with sgl-kernel and custom operators + +## Quick Start (Command Examples) + +The following examples use world_size=2. You can modify `--nproc_per_node` and parameters according to your machine: + +- Regular paths only (no quantization): +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--no-quant --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- FP8 quantization paths only: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--quant-fp8 --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- FP4 quantization paths only: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--quant-fp4 --hidden-dim 1024 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +- Larger hidden dimensions: +``` +torchrun --nproc_per_node=2 \ +benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py \ +--no-quant --hidden-dim 4096 --seq-lens 512 1024 2048 4096 --trials 100 +``` + +## Parameter Description +- `--seq-lens`: List of sequence lengths to test (default: 128 512 1024 2048) +- `--hidden-dim`: Hidden dimension (default: 8192) +- `--dtypes`: Data type list, `float16|bfloat16|float32` (default: bfloat16) +- `--no-residual`: Only test "no residual" scenarios (default tests both "with/without residual") +- Mutually exclusive quantization options: + - `--no-quant`: No quantization testing + - `--quant-fp8`: Only FP8 quantization testing + - `--quant-fp4`: Only FP4 quantization testing + - `--quant-all`: Test all (default) +- FlashInfer related: + - `--disable-oneshot`: Disable oneshot mode (default enables oneshot and tests twoshot simultaneously) +- Runtime configuration: + - `--warmup`: Warmup count before graph capture and before graph replay (default 5) + - `--trials`: Benchmark iteration count (default 20; internally each `graph.replay()` will batch replay multiple times) + - `--output-file`: Save results as Markdown file (only rank0 takes effect) + +## Output Example + +Each configuration group prints a table showing average execution time and relative speedup ratios (baseline is the faster standard implementation). For example: +``` +================================================================================ +Results: seq_len=1024, hidden_dim=1024 +dtype=torch.bfloat16, residual=yes, quant_mode=none +================================================================================ +Operation Time (ms) Speedup +-------------------------------------------------------------------------------- +standard_allreduce_rmsnorm 0.024 0.98x +standard_allreduce_rmsnorm_native_compiled 0.023 baseline +flashinfer_fused_allreduce_rmsnorm_oneshot 0.011 2.19x +flashinfer_fused_allreduce_rmsnorm_twoshot 0.041 0.57x +``` + +If `--output-file` is specified, all configurations will be summarized in Markdown tables in that file. + +## Important Notes and Recommendations + +- Distributed: The script uses `torchrun` environment variables to initialize distributed training and binds tensors/communication groups to the current rank's corresponding device. +- World size: Requires `WORLD_SIZE > 1` to perform communication operator benchmarks. Otherwise, the script will error and prompt. +- FlashInfer: + - If not installed or interfaces are missing, the script will only run standard paths and provide prompts in the logs. + - The fused operator internally uses "oneshot"/"twoshot" two trigger methods; oneshot is enabled by default and twoshot is tested simultaneously. +- FP8/FP4: + - FP8 uses sglang's FP8 tools and dtype, with underlying platform selection of `e4m3`/`e4m3fnuz` etc. + - FP4 uses sgl-kernel's `scaled_fp4_quant`, requiring corresponding platform support. +- CUDA Graph: + - Uses sglang's `graph_capture()` to prepare capture-ready state for communication, then uses `torch.cuda.graph` to capture kernels, reducing measurement jitter. diff --git a/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py b/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py new file mode 100644 index 000000000..4aebf62b9 --- /dev/null +++ b/benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py @@ -0,0 +1,1304 @@ +# Modified from https://github.com/vllm-project/vllm/blob/237e1fb887c7f5a579420fa0295097f24b006594/benchmarks/kernels/benchmark_fused_collective.py + +""" +Benchmark for FlashInfer fused collective operations vs standard operations. + +This benchmark compares: +1. FlashInfer's trtllm_allreduce_fusion (fused allreduce + rmsnorm + optional quant) +2. Standard tensor_model_parallel_all_reduce + separate rmsnorm/quant operations + +Usage with torchrun: + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --no-quant --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp8 --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp4 --hidden-dim 1024 --seq-len 512 1024 2048 4096 --trials 100 + + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --no-quant --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp8 --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 + torchrun --nproc_per_node=2 benchmark/kernels/flashinfer_allreduce_fusion/benchmark_fused_collective.py --quant-fp4 --hidden-dim 4096 --seq-len 512 1024 2048 4096 --trials 100 +""" + +import argparse +import contextlib +import itertools +import logging +import os +import time +from typing import Optional + +import torch # type: ignore +import torch.distributed as dist # type: ignore + +from sglang.srt.distributed import get_tp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + graph_capture, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.layernorm import RMSNorm # noqa +from sglang.srt.layers.quantization.fp8_kernel import fp8_dtype as SGLANG_FP8_DTYPE +from sglang.srt.layers.quantization.fp8_kernel import static_quant_fp8 + +try: + from sgl_kernel import fused_add_rmsnorm as SGL_FUSED_ADD_RMS_NORM + from sgl_kernel import rmsnorm as SGL_RMS_NORM + from sgl_kernel import scaled_fp4_quant as SGL_SCALED_FP4_QUANT +except Exception: # pragma: no cover - fallback on non-supported platforms + SGL_FUSED_ADD_RMS_NORM = None + SGL_RMS_NORM = None + SGL_SCALED_FP4_QUANT = None + +FP8_DTYPE = SGLANG_FP8_DTYPE + +logger = logging.getLogger(__name__) + +# Try to import FlashInfer +try: + import flashinfer.comm as flashinfer_comm # type: ignore + + if not hasattr(flashinfer_comm, "trtllm_allreduce_fusion"): + flashinfer_comm = None + logger.warning( + "FlashInfer comm module found but missing trtllm_allreduce_fusion" + ) +except ImportError: + flashinfer_comm = None + logger.warning("FlashInfer not found, only benchmarking standard operations") + +# Constants +MiB = 1024 * 1024 + +# FlashInfer max sizes per world size +# Enable 64MB for 2, 4, 8 world sizes to verify large input sizes +# use --disable-oneshot to disable oneshot mode for very large input sizes +_FI_MAX_SIZES = { + 2: 64 * MiB, # 64MB + 4: 64 * MiB, # 64MB + 8: 64 * MiB, # 64MB +} + +# Global workspace tensor for FlashInfer +_FI_WORKSPACE_TENSOR = None + + +def setup_flashinfer_workspace( + world_size: int, + rank: int, + hidden_dim: int, + max_token_num: int, + use_fp32_lamport: bool = False, +): + """Setup FlashInfer workspace for fused allreduce operations.""" + global _FI_WORKSPACE_TENSOR + + if flashinfer_comm is None: + return None, None + + if world_size not in _FI_MAX_SIZES: + logger.warning("FlashInfer not supported for world size %s", world_size) + return None, None + + try: + # Create IPC workspace + ipc_handles, workspace_tensor = ( + flashinfer_comm.trtllm_create_ipc_workspace_for_all_reduce_fusion( + tp_rank=rank, + tp_size=world_size, + max_token_num=max_token_num, + hidden_dim=hidden_dim, + group=get_tp_group().device_group, + use_fp32_lamport=use_fp32_lamport, + ) + ) + + _FI_WORKSPACE_TENSOR = workspace_tensor + return ipc_handles, workspace_tensor + except Exception as e: + logger.error("Failed to setup FlashInfer workspace: %s", e) + return None, None + + +def cleanup_flashinfer_workspace(ipc_handles): + """Cleanup FlashInfer workspace.""" + if flashinfer_comm is None or ipc_handles is None: + return + + try: + group = get_tp_group().device_group + flashinfer_comm.trtllm_destroy_ipc_workspace_for_all_reduce(ipc_handles, group) + except Exception as e: + logger.error("Failed to cleanup FlashInfer workspace: %s", e) + + +class FlashInferFusedAllReduceParams: + """Parameters for FlashInfer fused allreduce operations.""" + + def __init__( + self, + rank: int, + world_size: int, + use_fp32_lamport: bool = False, + max_token_num: int = 1024, + ): + self.rank = rank + self.world_size = world_size + self.use_fp32_lamport = use_fp32_lamport + self.trigger_completion_at_end = True + self.launch_with_pdl = True + self.fp32_acc = True + self.max_token_num = max_token_num + + def get_trtllm_fused_allreduce_kwargs(self): + return { + "world_rank": self.rank, + "world_size": self.world_size, + "launch_with_pdl": self.launch_with_pdl, + "trigger_completion_at_end": self.trigger_completion_at_end, + "fp32_acc": self.fp32_acc, + } + + +def flashinfer_fused_allreduce_rmsnorm( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + allreduce_params: "FlashInferFusedAllReduceParams", + use_oneshot: bool, + norm_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm operation.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm, + allreduce_out=None, + quant_out=None, + scale_out=None, + layout_code=None, + scale_factor=None, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def flashinfer_fused_allreduce_rmsnorm_fp8_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + scale_factor: torch.Tensor, + allreduce_params: FlashInferFusedAllReduceParams, + use_oneshot: bool = True, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm + FP8 quantization.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP8Quant, + allreduce_out=None, + quant_out=quant_out, + scale_out=None, + layout_code=None, + scale_factor=scale_factor, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def flashinfer_fused_allreduce_rmsnorm_fp4_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + input_global_scale: torch.Tensor, + allreduce_params: FlashInferFusedAllReduceParams, + quant_out: torch.Tensor, + use_oneshot: bool, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """FlashInfer fused allreduce + rmsnorm + FP4 quantization.""" + if flashinfer_comm is None or _FI_WORKSPACE_TENSOR is None: + raise RuntimeError("FlashInfer not available or workspace not initialized") + + if norm_out is None: + norm_out = input_tensor + residual_out = residual + else: + residual_out = input_tensor + + flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + token_num=input_tensor.shape[0], + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + hidden_dim=input_tensor.shape[-1], + workspace_ptrs=_FI_WORKSPACE_TENSOR, + pattern_code=flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNormFP4Quant, + allreduce_out=None, + quant_out=quant_out, + scale_out=output_scale, + layout_code=None, + scale_factor=input_global_scale, + use_oneshot=use_oneshot, + **allreduce_params.get_trtllm_fused_allreduce_kwargs(), + ) + + +def standard_allreduce_rmsnorm( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm operations.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + # Then RMS norm + if residual is not None: + # Fused add + RMS norm (in-place on allreduce_out) + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + rms.forward_native(allreduce_out, residual) + else: + # Just RMS norm + if SGL_RMS_NORM is not None: + _ = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + _ = rms.forward_native(allreduce_out) + + +def standard_allreduce_rmsnorm_fp8_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP8 quantization.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Then RMS norm + static FP8 quantization + if residual is not None: + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + quant_out, _ = static_quant_fp8( + allreduce_out, scale_factor, repeat_scale=False + ) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + normed, _ = rms.forward_native(allreduce_out, residual) + quant_out, _ = static_quant_fp8(normed, scale_factor, repeat_scale=False) + return quant_out, residual + else: + if SGL_RMS_NORM is not None: + normed = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + normed = rms.forward_native(allreduce_out) + quant_out, _ = static_quant_fp8(normed, scale_factor, repeat_scale=False) + return quant_out + + +def standard_allreduce_rmsnorm_fp4_quant( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rms_gamma: torch.Tensor, + rms_eps: float, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP4 quantization.""" + + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Then RMS norm + if residual is not None: + if SGL_FUSED_ADD_RMS_NORM is not None: + SGL_FUSED_ADD_RMS_NORM(allreduce_out, residual, rms_gamma, rms_eps) + quant_input = allreduce_out + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + quant_input, _ = rms.forward_native(allreduce_out, residual) + residual_out = residual + else: + if SGL_RMS_NORM is not None: + quant_input = SGL_RMS_NORM(allreduce_out, rms_gamma, rms_eps) + else: + rms = RMSNorm(allreduce_out.shape[-1], eps=rms_eps) + rms.weight.data = rms_gamma + quant_input = rms.forward_native(allreduce_out) + residual_out = allreduce_out + + # Finally FP4 quantization + if SGL_SCALED_FP4_QUANT is None: + raise RuntimeError("scaled_fp4_quant is not available on this platform") + quant_res, output_scale_res = SGL_SCALED_FP4_QUANT(quant_input, input_global_scale) + if residual is not None: + return quant_res, residual_out, output_scale_res + else: + return quant_res, quant_input + + +def standard_allreduce_rmsnorm_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm operations using native RMSNorm forward.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + # Apply native RMSNorm + if residual is not None: + result = rmsnorm_layer.forward_native(allreduce_out, residual) + return result # Returns (norm_out, residual_out) + else: + result = rmsnorm_layer.forward_native(allreduce_out) + return result # Returns norm_out + + +def standard_allreduce_rmsnorm_fp8_quant_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP8 quantization using native implementations.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Apply native RMSNorm + if residual is not None: + norm_out, residual_out = rmsnorm_layer.forward_native(allreduce_out, residual) + else: + norm_out = rmsnorm_layer.forward_native(allreduce_out) + residual_out = allreduce_out + + # Apply native FP8 quantization + quant_out, _ = static_quant_fp8(norm_out, scale_factor, repeat_scale=False) + + if residual is not None: + return quant_out, residual_out + else: + return quant_out + + +def standard_allreduce_rmsnorm_fp4_quant_native( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Standard allreduce + rmsnorm + FP4 quantization using native RMSNorm.""" + # All-reduce first + allreduce_out = tensor_model_parallel_all_reduce(input_tensor) + + # Apply native RMSNorm + if residual is not None: + norm_out, residual_out = rmsnorm_layer.forward_native(allreduce_out, residual) + quant_input = norm_out + else: + norm_out = rmsnorm_layer.forward_native(allreduce_out) + quant_input = norm_out + residual_out = allreduce_out + + # Apply FP4 quantization (still using fused CUDA op as there's no native FP4) + if SGL_SCALED_FP4_QUANT is None: + raise RuntimeError("scaled_fp4_quant is not available on this platform") + quant_res, output_scale_res = SGL_SCALED_FP4_QUANT(quant_input, input_global_scale) + + if residual is not None: + return quant_res, residual_out, output_scale_res + else: + return quant_res, norm_out + + +# Compiled versions of native functions +@torch.compile +def standard_allreduce_rmsnorm_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + norm_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm.""" + return standard_allreduce_rmsnorm_native( + input_tensor, residual, rmsnorm_layer, norm_out + ) + + +@torch.compile +def standard_allreduce_rmsnorm_fp8_quant_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + scale_factor: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, + quant_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm + FP8 quantization.""" + return standard_allreduce_rmsnorm_fp8_quant_native( + input_tensor, + residual, + rmsnorm_layer, + scale_factor, + norm_out, + quant_out, + ) + + +@torch.compile +def standard_allreduce_rmsnorm_fp4_quant_native_compiled( + input_tensor: torch.Tensor, + residual: Optional[torch.Tensor], + rmsnorm_layer: RMSNorm, + input_global_scale: torch.Tensor, + quant_out: torch.Tensor, + output_scale: torch.Tensor, + norm_out: Optional[torch.Tensor] = None, +): + """Compiled version of standard allreduce + rmsnorm + FP4 quantization.""" + return standard_allreduce_rmsnorm_fp4_quant_native( + input_tensor, + residual, + rmsnorm_layer, + input_global_scale, + quant_out, + output_scale, + norm_out, + ) + + +def create_test_tensors( + seq_len: int, hidden_dim: int, dtype: torch.dtype, use_residual: bool = True +): + """Create test tensors for benchmarking.""" + input_tensor = torch.randn(seq_len, hidden_dim, dtype=dtype) + residual = ( + torch.randn_like(input_tensor) + if use_residual + else torch.zeros_like(input_tensor) + ) + rms_gamma = torch.ones(hidden_dim, dtype=dtype) + norm_out = None if use_residual else torch.empty_like(input_tensor) + + # Quantization scales + scale_fp8 = torch.tensor(1.0, dtype=torch.float32) + scale_fp4 = torch.tensor(1.0, dtype=torch.float32) + quant_out_fp8 = torch.empty_like(input_tensor, dtype=FP8_DTYPE) + # Pre-allocate FP4 output tensors (to avoid allocation overhead in benchmarks) + fp4_quant_out = torch.empty((seq_len, hidden_dim // 2), dtype=torch.uint8) + fp4_output_scale = torch.empty((128, 4), dtype=torch.int32) + + return ( + input_tensor, + norm_out, + residual, + rms_gamma, + scale_fp8, + quant_out_fp8, + scale_fp4, + fp4_quant_out, + fp4_output_scale, + ) + + +def benchmark_operation( + operation_func, *args, warmup: int = 5, trials: int = 20, **kwargs +): + """Benchmark a single operation using CUDA graphs.""" + # Warmup before graph capture + for _ in range(warmup): + operation_func(*args, **kwargs) + torch.cuda.synchronize() + + # Create CUDA graph + graph = torch.cuda.CUDAGraph() + num_op_per_cudagraph = 10 + + # Use sglang's graph_capture to make tensor_model_parallel_all_reduce graph-safe + with graph_capture() as graph_capture_context: + with torch.cuda.graph(graph, stream=graph_capture_context.stream): + for _ in range(num_op_per_cudagraph): + operation_func(*args, **kwargs) + + # Graph warmup + torch.cuda.synchronize() + for _ in range(warmup): + graph.replay() + + # Benchmark with CUDA graph + torch.cuda.synchronize() + start_time = time.perf_counter() + + for _ in range(trials // num_op_per_cudagraph): + # operation_func(*args, **kwargs) + graph.replay() + + torch.cuda.synchronize() + end_time = time.perf_counter() + + avg_time_ms = ((end_time - start_time) / trials) * 1000 + return avg_time_ms + + +def run_benchmarks( + seq_len: int, + hidden_dim: int, + dtype: torch.dtype, + use_residual: bool, + allreduce_params: Optional[FlashInferFusedAllReduceParams], + quant_mode: str = "all", + disable_oneshot: bool = False, +): + """Run all benchmarks for given configuration. + + Args: + quant_mode: "none", "fp8_only", "fp4_only", or "all" + """ + ( + input_tensor, + norm_out, + residual, + rms_gamma, + scale_fp8, + quant_out_fp8, + scale_fp4, + fp4_quant_out, + fp4_output_scale, + ) = create_test_tensors(seq_len, hidden_dim, dtype, use_residual) + + rms_eps = 1e-6 + results = {} + + # Create RMSNorm once for native benchmarks + rmsnorm_layer = RMSNorm(hidden_dim, eps=rms_eps) + rmsnorm_layer.weight.data = rms_gamma + + if quant_mode in ["all", "none"]: + # Standard AllReduce + RMSNorm + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + ) + results["standard_allreduce_rmsnorm"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm failed: %s", e) + results["standard_allreduce_rmsnorm"] = float("inf") + + # Standard AllReduce + RMSNorm Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + norm_out=norm_out, + ) + results["standard_allreduce_rmsnorm_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_native_compiled"] = float("inf") + + # FlashInfer Fused AllReduce + RMSNorm Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + allreduce_params=allreduce_params, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_oneshot"] = time_ms + except Exception as e: + logger.error("FlashInfer Fused AllReduce+RMSNorm Oneshot failed: %s", e) + results["flashinfer_fused_allreduce_rmsnorm_oneshot"] = float("inf") + + # FlashInfer Fused AllReduce + RMSNorm Two-shot + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + allreduce_params=allreduce_params, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_twoshot"] = time_ms + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm Two-shot failed: %s", e + ) + results["flashinfer_fused_allreduce_rmsnorm_twoshot"] = float("inf") + + if quant_mode in ["all", "fp8_only"]: + # Standard AllReduce + RMSNorm + FP8 Quant + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + ) + results["standard_allreduce_rmsnorm_fp8_quant"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP8 failed: %s", e) + results["standard_allreduce_rmsnorm_fp8_quant"] = float("inf") + + # Standard AllReduce + RMSNorm + FP8 Quant Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp8_quant_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + # quant_fp8_layer removed in sglang version; static_quant_fp8 is used within the function + scale_factor=scale_fp8, + norm_out=norm_out, + quant_out=quant_out_fp8, + ) + results["standard_allreduce_rmsnorm_fp8_quant_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP8 Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_fp8_quant_native_compiled"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP8 Quant Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + allreduce_params=allreduce_params, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_oneshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP8 Oneshot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_oneshot"] = float( + "inf" + ) + # FlashInfer Fused AllReduce + RMSNorm + FP8 Quant Two-shot + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp8_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + scale_factor=scale_fp8, + quant_out=quant_out_fp8, + allreduce_params=allreduce_params, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_twoshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP8 Two-shot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp8_quant_twoshot"] = float( + "inf" + ) + + if quant_mode in ["all", "fp4_only"]: + # Standard AllReduce + RMSNorm + FP4 Quant + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp4_quant, + input_tensor, + norm_out=norm_out, + residual=residual, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + ) + results["standard_allreduce_rmsnorm_fp4_quant"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP4 failed: %s", e) + results["standard_allreduce_rmsnorm_fp4_quant"] = float("inf") + + # Standard AllReduce + RMSNorm + FP4 Quant Native Compiled + try: + time_ms = benchmark_operation( + standard_allreduce_rmsnorm_fp4_quant_native_compiled, + input_tensor, + residual=residual, + rmsnorm_layer=rmsnorm_layer, + input_global_scale=scale_fp4, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + norm_out=norm_out, + ) + results["standard_allreduce_rmsnorm_fp4_quant_native_compiled"] = time_ms + except Exception as e: + logger.error("Standard AllReduce+RMSNorm+FP4 Native Compiled failed: %s", e) + results["standard_allreduce_rmsnorm_fp4_quant_native_compiled"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP4 Quant Oneshot + if flashinfer_comm is not None and allreduce_params is not None: + try: + if not disable_oneshot: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp4_quant, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + allreduce_params=allreduce_params, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + use_oneshot=True, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_oneshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP4 Oneshot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_oneshot"] = float( + "inf" + ) + + # FlashInfer Fused AllReduce + RMSNorm + FP4 Quant Two-shot + if flashinfer_comm is not None and allreduce_params is not None: + try: + time_ms = benchmark_operation( + flashinfer_fused_allreduce_rmsnorm_fp4_quant, + input_tensor, + residual=residual, + norm_out=norm_out, + rms_gamma=rms_gamma, + rms_eps=rms_eps, + input_global_scale=scale_fp4, + allreduce_params=allreduce_params, + quant_out=fp4_quant_out, + output_scale=fp4_output_scale, + use_oneshot=False, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_twoshot"] = ( + time_ms + ) + except Exception as e: + logger.error( + "FlashInfer Fused AllReduce+RMSNorm+FP4 Two-shot failed: %s", + e, + ) + results["flashinfer_fused_allreduce_rmsnorm_fp4_quant_twoshot"] = float( + "inf" + ) + + return results + + +def prepare_results_with_speedups(results_dict): + """Prepare results with speedup calculations based on dynamic baseline selection.""" + prepared_results = [] + + # Determine the fastest baseline for each operation type + def get_fastest_baseline(op_name, results_dict): + """Get the fastest baseline between standard and native_compiled versions.""" + if "fp8_quant" in op_name: + candidates = [ + "standard_allreduce_rmsnorm_fp8_quant", + "standard_allreduce_rmsnorm_fp8_quant_native_compiled", + ] + elif "fp4_quant" in op_name: + candidates = [ + "standard_allreduce_rmsnorm_fp4_quant", + "standard_allreduce_rmsnorm_fp4_quant_native_compiled", + ] + else: + candidates = [ + "standard_allreduce_rmsnorm", + "standard_allreduce_rmsnorm_native_compiled", + ] + + # Find the fastest among available candidates + fastest_time = float("inf") + fastest_baseline = None + + for candidate in candidates: + if ( + candidate in results_dict + and results_dict[candidate] != float("inf") + and results_dict[candidate] < fastest_time + ): + fastest_time = results_dict[candidate] + fastest_baseline = candidate + + return fastest_baseline + + # Create dynamic baseline mapping + dynamic_baseline_mapping = {} + for op_name in results_dict: + if ( + op_name.startswith("flashinfer_") + or op_name.startswith("standard_") + and not op_name.endswith("_native_compiled") + ): + dynamic_baseline_mapping[op_name] = get_fastest_baseline( + op_name, results_dict + ) + + for op_name, time_ms in results_dict.items(): + if time_ms == float("inf"): + speedup_str = "FAILED" + time_str = "FAILED" + else: + time_str = f"{time_ms:.3f}" + # Find the appropriate baseline for this operation + baseline_op = dynamic_baseline_mapping.get(op_name) + if baseline_op and baseline_op in results_dict: + baseline_time = results_dict[baseline_op] + if baseline_time != float("inf") and baseline_time > 0: + speedup = baseline_time / time_ms + speedup_str = f"{speedup:.2f}x" + else: + speedup_str = "N/A" + else: + # For baseline operations, determine if this is the fastest baseline + if op_name.endswith("_native_compiled") or ( + op_name.startswith("standard_") + and not op_name.endswith("_native_compiled") + ): + fastest_baseline = get_fastest_baseline(op_name, results_dict) + if fastest_baseline == op_name: + speedup_str = "baseline" + else: + if fastest_baseline and fastest_baseline in results_dict: + baseline_time = results_dict[fastest_baseline] + if baseline_time != float("inf") and baseline_time > 0: + speedup = baseline_time / time_ms + speedup_str = f"{speedup:.2f}x" + else: + speedup_str = "N/A" + else: + speedup_str = "N/A" + else: + speedup_str = "N/A" + + prepared_results.append( + { + "operation": op_name, + "time_ms": time_ms, + "time_str": time_str, + "speedup_str": speedup_str, + } + ) + + return prepared_results + + +def print_results(results_dict, seq_len, hidden_dim, dtype, use_residual, quant_mode): + """Print benchmark results in a formatted table.""" + print(f"\n{'=' * 80}") + print(f"Results: seq_len={seq_len}, hidden_dim={hidden_dim}") + print( + f"dtype={dtype}, residual={'yes' if use_residual else 'no'}, " + f"quant_mode={quant_mode}" + ) + print(f"{'=' * 80}") + print(f"{'Operation':<50} {'Time (ms)':<12} {'Speedup':<10}") + print(f"{'-' * 80}") + + # Prepare results with speedup calculations + prepared_results = prepare_results_with_speedups(results_dict) + + for result in prepared_results: + if result["time_ms"] == float("inf"): + time_display = result["time_str"] + else: + time_display = f"{result['time_ms']:.3f}" + + print( + f"{result['operation']:<50} {time_display:<12} {result['speedup_str']:<10}" + ) + + +def format_results_markdown( + all_results: list[dict], world_size: int, args: argparse.Namespace +) -> str: + """Format all benchmark results as markdown.""" + markdown = f"""# FlashInfer Fused Collective Operations Benchmark Results + +**World Size:** {world_size} +**Hidden Dimension:** {args.hidden_dim} +**Warmup Iterations:** {args.warmup} +**Benchmark Trials:** {args.trials} +**Quantization Mode:** {all_results[0]["quant_mode"] if all_results else "N/A"} + +--- + +""" + + for result in all_results: + seq_len = result["seq_len"] + dtype = result["dtype"] + use_residual = result["use_residual"] + results_dict = result["results"] + + residual_str = "with residual" if use_residual else "no residual" + + markdown += f""" +## Configuration: seq_len={seq_len}, dtype={dtype}, {residual_str} + +| Operation | Time (ms) | Speedup | +|-----------|-----------|---------| +""" + + # Prepare results with speedup calculations + prepared_results = prepare_results_with_speedups(results_dict) + + for result in prepared_results: + # Format operation name for better readability + formatted_op_name = result["operation"].replace("_", " ").title() + markdown += f"| {formatted_op_name} | {result['time_str']} |" + markdown += f"{result['speedup_str']} |\n" + + markdown += "\n" + + return markdown + + +def save_results_to_file( + all_results: list[dict], world_size: int, args: argparse.Namespace, rank: int +): + """Save benchmark results to markdown file (only on rank 0).""" + if rank != 0: + return + + if not all_results: + logger.warning("No results to save") + return + + output_path = args.output_file + + try: + markdown_content = format_results_markdown(all_results, world_size, args) + + with open(output_path, "w") as f: + f.write(markdown_content) + + except Exception as e: + logger.error("Failed to save results to file: %s", e) + + +def main(): + parser = argparse.ArgumentParser( + description="Benchmark fused collective operations" + ) + parser.add_argument( + "--seq-lens", + type=int, + nargs="+", + default=[128, 512, 1024, 2048], + help="Sequence lengths to test", + ) + parser.add_argument( + "--hidden-dim", type=int, default=8192, help="Hidden dimension size" + ) + parser.add_argument( + "--dtypes", + type=str, + nargs="+", + default=["bfloat16"], + choices=["float16", "bfloat16", "float32"], + help="Data types to test", + ) + parser.add_argument( + "--no-residual", + action="store_true", + help="Skip residual connection tests", + ) + + # Quantization mode options (mutually exclusive with --no-quant) + quant_group = parser.add_mutually_exclusive_group() + quant_group.add_argument( + "--no-quant", action="store_true", help="Skip all quantization tests" + ) + quant_group.add_argument( + "--quant-fp8", action="store_true", help="Only run FP8 quantization tests" + ) + quant_group.add_argument( + "--quant-fp4", action="store_true", help="Only run FP4 quantization tests" + ) + quant_group.add_argument( + "--quant-all", + action="store_true", + help="Run all quantization tests (default)", + ) + + parser.add_argument( + "--disable-oneshot", + action="store_true", + help="Disable oneshot mode for FlashInfer operations", + ) + parser.add_argument( + "--warmup", type=int, default=5, help="Number of warmup iterations" + ) + parser.add_argument( + "--trials", type=int, default=20, help="Number of benchmark trials" + ) + parser.add_argument( + "--output-file", + type=str, + help="""Output file path for markdown results + (default: benchmark_results_.md) + """, + ) + + args = parser.parse_args() + + # Check if running with torchrun (required for collective operations) + if "RANK" not in os.environ or "WORLD_SIZE" not in os.environ: + raise RuntimeError( + "Must run with torchrun for distributed benchmarking. " + "Example: torchrun --nproc_per_node=2 benchmark_fused_collective.py" + ) + + # Initialize distributed environment + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + torch.set_default_device(device) + + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=rank, + backend="nccl", + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + + # Validate world size (must be > 1 for collective operations) + if world_size <= 1: + raise ValueError( + "World size must be > 1 for collective operations benchmarking. " + f"Current world size: {world_size}. Use torchrun with --nproc_per_node > 1." + ) + + # Determine quantization mode + if args.no_quant: + quant_mode = "none" + elif args.quant_fp8: + quant_mode = "fp8_only" + elif args.quant_fp4: + quant_mode = "fp4_only" + else: # args.quant_all or default + quant_mode = "all" + + if rank == 0: + logger.info("Running benchmark with world_size=%s, rank=%s", world_size, rank) + logger.info("Quantization mode: %s", quant_mode) + if flashinfer_comm is not None: + oneshot_status = "enabled" if not args.disable_oneshot else "disabled" + logger.info( + "FlashInfer available - will benchmark fused operations (oneshot: %s)", + oneshot_status, + ) + else: + logger.info( + "FlashInfer not available - only benchmarking standard operations" + ) + + # Convert dtype strings to torch dtypes + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + dtypes = [dtype_map[dt] for dt in args.dtypes] + + # Test configurations + residual_options = [True] if not args.no_residual else [False] + if not args.no_residual: + residual_options.append(False) + + configs = list(itertools.product(args.seq_lens, dtypes, residual_options)) + + # Setup FlashInfer workspace if available + ipc_handles = None + allreduce_params = None + + if flashinfer_comm is not None: + # Use the largest hidden dimension for workspace setup + max_num_token = _FI_MAX_SIZES.get(world_size) // ( + args.hidden_dim * world_size * 2 + ) + + ipc_handles, workspace_tensor = setup_flashinfer_workspace( + world_size, rank, args.hidden_dim, max_num_token + ) + + if workspace_tensor is not None: + allreduce_params = FlashInferFusedAllReduceParams( + rank=rank, + world_size=world_size, + max_token_num=max_num_token, + ) + + # Collect all results for markdown export + all_results = [] + + try: + # Run benchmarks + for seq_len, dtype, use_residual in configs: + if rank == 0: + logger.info( + "\nTesting: seq_len=%s, hidden_dim=%s, dtype=%s, residual=%s", + seq_len, + args.hidden_dim, + dtype, + use_residual, + ) + + results = run_benchmarks( + seq_len, + args.hidden_dim, + dtype, + use_residual, + allreduce_params, + quant_mode=quant_mode, + disable_oneshot=args.disable_oneshot, + ) + + # Store results for markdown export + if rank == 0: + all_results.append( + { + "seq_len": seq_len, + "hidden_dim": args.hidden_dim, + "dtype": str(dtype).replace("torch.", ""), + "use_residual": use_residual, + "quant_mode": quant_mode, + "results": results, + } + ) + + print_results( + results, + seq_len, + args.hidden_dim, + dtype, + use_residual, + quant_mode, + ) + + # Save results to markdown file + if args.output_file and rank == 0: + save_results_to_file(all_results, world_size, args, rank) + + finally: + # Cleanup + if ipc_handles is not None: + cleanup_flashinfer_workspace(ipc_handles) + + with contextlib.suppress(Exception): + dist.barrier() + cleanup_dist_env_and_memory(shutdown_ray=False) + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/README.md b/benchmark/kernels/fused_moe_triton/README.md new file mode 100644 index 000000000..48598854a --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/README.md @@ -0,0 +1,76 @@ +## Tuning Triton MoE Kernels + +This directory contains benchmarking tools for MoE (Mixture of Experts) kernels. + +### Tuning Tool + +- `tuning_fused_moe_triton.py`: A tool for tuning the `fused_moe_triton` kernel. Adapted from [vllm's benchmark_moe.py](https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py), with added support for various model architectures. + +Example usage: +```bash +# Tune Mixtral-8x7B with default settings +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model mistralai/Mixtral-8x7B-Instruct-v0.1 \ + --tune + +# Tune Qwen2-57B with FP8 and TP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --tp-size 4 \ + --dtype fp8_w8a8 \ + --tune + +# Tune Qwen3-235B-A22B-FP8 and TP=4 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model Qwen/Qwen3-235B-A22B-FP8 \ + --tp-size 4 \ + --dtype fp8_w8a8 \ + --tune + +# Tune DeepSeek-V3 with FP8 and TP=8 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 \ + --dtype fp8_w8a8 \ + --tune + +# Tune DeepSeek-R1 with channel-wise INT8 and TP=16 +python benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py \ + --model meituan/DeepSeek-R1-Channel-INT8 \ + --tp-size 16 \ + --dtype int8_w8a8 \ + --tune +``` + +After tuning, a configuration file (e.g., `E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json`) will be generated in the current directory. You can move this file to `sglang/srt/layers/fused_moe_triton/configs/triton_version` dir to use it in `sglang`. + +### Performance Comparison Tool + +- `benchmark_vllm_vs_sglang_fused_moe_triton.py`: A tool for comparing the performance of fused MoE kernels between vllm and sglang implementations. Supports various model architectures and data types. + +Example usage: +```bash +# Compare with default settings (Mixtral model) +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py + +# Compare with FP8 mode for Qwen2-57B +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model Qwen/Qwen2-57B-A14B-Instruct \ + --use-fp8-w8a8 + +# Compare with custom TP size +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 + +# Compare with custom TP size +python benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py \ + --model deepseek-ai/DeepSeek-V3-0324 \ + --tp-size 8 +``` + +The benchmark results will be saved as plots and data files in the specified output directory (default: `./configs/benchmark_ops/vllm_sglang_fused_moe/`). + +- `benchmark_torch_compile_fused_moe.py`: A tool for benchmarking the performance of the fused MoE kernel with `torch.compile` and original fused MoE kernel. + +Usage is the same as `benchmark_vllm_vs_sglang_fused_moe_triton.py`, note that `torch.compile` does not support `fp8_w8a8` and `int8_w8a8` fused_moe_kernel. diff --git a/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py new file mode 100644 index 000000000..7621628c1 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_sglang_fused_moe_triton.py @@ -0,0 +1,292 @@ +# python3 benchmark/kernels/fused_moe_triton/sglang_fused_moe_triton.py --model /DeepSeek-V3/ --tp-size 8 +import argparse + +import torch +import triton +from transformers import AutoConfig + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_sglang, +) +from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_forward, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopK, TopKConfig, select_experts + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen3MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "DeepseekV2ForCausalLM", + "DeepseekV3ForCausalLM", + "Glm4MoeForCausalLM", + ]: + E = ( + config.n_routed_experts + 1 + if config.architectures[0] in ["DeepseekV3ForCausalLM"] + else config.n_routed_experts + ) + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + "block_shape": block_shape, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_moe_triton_api( + x, + w1, + w2, + input_gating, + topk, +): + topk_op = TopK( + top_k=topk, + renormalize=False, + use_grouped_topk=False, + ) + topk_op.use_triton_kernels = True + triton_topk_output = topk_op.forward_cuda( + hidden_states=x, + router_logits=input_gating, + ) + + moe_runner_config = MoeRunnerConfig( + inplace=False, + ) + return triton_kernel_moe_forward( + x, + w1, + w2, + triton_topk_output, + moe_runner_config, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + topk_output = select_experts( + hidden_states=x, + router_logits=input_gating, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + return fused_moe_sglang( + x, + w1, + w2, + topk_output, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list([128, 256, 512, 1024, 2048, 4096, 8192]), + line_arg="provider", + line_vals=[ + "sglang_fused_moe_triton_v340", + "sglang_fused_moe_triton", + ], + line_names=[ + "sglang_fused_moe_triton_v340", + "sglang_fused_moe_triton", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark( + batch_size, + provider, + model_config, + use_fp8_w8a8=False, + use_cuda_graph: bool = False, +): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + + w1_tri = w1.clone() + w2_tri = w2.clone() + w1_tri = w1_tri.transpose(-2, -1).contiguous() + w2_tri = w2_tri.transpose(-2, -1).contiguous() + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + if provider == "sglang_fused_moe_triton_v340": + api_func = fused_moe_triton_api + api_kwargs = { + "x": x, + "w1": w1_tri, + "w2": w2_tri, + "input_gating": input_gating, + "topk": topk, + } + else: + api_func = fused_moe_sglang_api + api_kwargs = { + "x": x, + "w1": w1, + "w2": w2, + "input_gating": input_gating, + "topk": topk, + "use_fp8_w8a8": use_fp8_w8a8, + "block_shape": block_shape, + } + + # Warmup + for _ in range(10): + _ = api_func(**api_kwargs) + torch.cuda.synchronize() + + if use_cuda_graph: + stream = torch.cuda.Stream() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + api_func(**api_kwargs) + torch.cuda.synchronize() + + bench_lambda = lambda: graph.replay() + else: + bench_lambda = lambda: api_func(**api_kwargs) + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench(bench_lambda, quantiles=quantiles) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--use-cuda-graph", action="store_true", help="Enable CUDA Graph capture/replay" + ) + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/sglang_fused_moe/", + ) + parser.add_argument("--trust-remote-code", action="store_true") + args = parser.parse_args() + + try: + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", + init_method="tcp://127.0.0.1:23456", + world_size=1, + rank=0, + ) + + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="nccl" if torch.cuda.is_available() else "gloo", + ) + + initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + ) + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + use_cuda_graph=args.use_cuda_graph, + ) + finally: + destroy_model_parallel() + destroy_distributed_environment() + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/benchmark_sum_scale.py b/benchmark/kernels/fused_moe_triton/benchmark_sum_scale.py new file mode 100644 index 000000000..979d2bbd1 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_sum_scale.py @@ -0,0 +1,202 @@ +import torch +import triton +import triton.language as tl +from triton.testing import do_bench + + +@triton.jit +def _moe_sum_reduce_kernel( + input_ptr, + input_stride_0, + input_stride_1, + input_stride_2, + output_ptr, + output_stride_0, + output_stride_1, + token_num: int, + topk_num: int, + hidden_dim: int, + routed_scaling_factor: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DIM: tl.constexpr, + NUM_STAGE: tl.constexpr, +): + input_stride_0 = tl.cast(input_stride_0, dtype=tl.int64) + input_stride_1 = tl.cast(input_stride_1, dtype=tl.int64) + output_stride_0 = tl.cast(output_stride_0, dtype=tl.int64) + + token_block_id = tl.program_id(0) + dim_block_id = tl.program_id(1) + + offs_token = token_block_id * BLOCK_M + tl.arange(0, BLOCK_M) + offs_dim = dim_block_id * BLOCK_DIM + tl.arange(0, BLOCK_DIM) + + mask_token = offs_token < token_num + mask_dim = offs_dim < hidden_dim + + base_ptrs = input_ptr + offs_token[:, None] * input_stride_0 + offs_dim[None, :] + + accumulator = tl.zeros((BLOCK_M, BLOCK_DIM), dtype=tl.float32) + + for i in tl.range(0, topk_num, num_stages=NUM_STAGE): + tile = tl.load( + base_ptrs + i * input_stride_1, + mask=mask_token[:, None] & mask_dim[None, :], + other=0.0, + ) + accumulator += tile.to(tl.float32) + accumulator *= routed_scaling_factor + + # -------- Write back -------- + store_ptrs = output_ptr + offs_token[:, None] * output_stride_0 + offs_dim[None, :] + tl.store( + store_ptrs, + accumulator.to(input_ptr.dtype.element_ty), + mask=mask_token[:, None] & mask_dim[None, :], + ) + + +# _moe_sum_reduce_kernel kernel modified from https://github.com/ModelTC/lightllm/blob/main/lightllm/common/fused_moe/moe_sum_reduce.py +def moe_sum_reduce( + input: torch.Tensor, output: torch.Tensor, routed_scaling_factor: float +): + assert input.is_contiguous() + assert output.is_contiguous() + + token_num, topk_num, hidden_dim = input.shape + assert output.shape[0] == token_num and output.shape[1] == hidden_dim + + BLOCK_M = 1 + BLOCK_DIM = 2048 + NUM_STAGE = 1 + num_warps = 16 + + grid = ( + triton.cdiv(token_num, BLOCK_M), + triton.cdiv(hidden_dim, BLOCK_DIM), + ) + + _moe_sum_reduce_kernel[grid]( + input, + *input.stride(), + output, + *output.stride(), + token_num=token_num, + topk_num=topk_num, + hidden_dim=hidden_dim, + routed_scaling_factor=routed_scaling_factor, + BLOCK_M=BLOCK_M, + BLOCK_DIM=BLOCK_DIM, + NUM_STAGE=NUM_STAGE, + num_warps=num_warps, + ) + return + + +def compute_sum_scaled_baseline( + x: torch.Tensor, out: torch.Tensor, routed_scaling_factor: float +) -> torch.Tensor: + torch.sum(x, dim=1, out=out) + out.mul_(routed_scaling_factor) + return out + + +@torch.compile +def compute_sum_scaled_compiled( + x: torch.Tensor, out: torch.Tensor, routed_scaling_factor: float +) -> torch.Tensor: + torch.sum(x * routed_scaling_factor, dim=1, out=out) + return out + + +def get_benchmark(): + num_tokens_range = [2**i for i in range(0, 13)] + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["num_tokens"], + x_vals=num_tokens_range, + line_arg="version", + line_vals=["baseline", "compiled", "triton"], + line_names=["Original", "TorchCompile", "TritonKernel"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="sum_scaled_performance", + args={}, + ) + ) + def benchmark(num_tokens, version): + topk = 9 + hidden_size = 4096 + dtype = torch.bfloat16 + scaling_factor = 0.3 + + x = torch.randn(num_tokens, topk, hidden_size, dtype=dtype, device="cuda") + out = torch.empty(num_tokens, hidden_size, dtype=dtype, device="cuda") + + # Warmup + for _ in range(3): + if version == "baseline": + compute_sum_scaled_baseline(x, out, scaling_factor) + elif version == "compiled": + compute_sum_scaled_compiled(x, out, scaling_factor) + else: + moe_sum_reduce(x, out, scaling_factor) + + # Benchmark + quantiles = [0.5, 0.2, 0.8] + if version == "baseline": + ms, min_ms, max_ms = do_bench( + lambda: compute_sum_scaled_baseline(x, out, scaling_factor), + quantiles=quantiles, + ) + elif version == "compiled": + ms, min_ms, max_ms = do_bench( + lambda: compute_sum_scaled_compiled(x, out, scaling_factor), + quantiles=quantiles, + ) + else: + ms, min_ms, max_ms = do_bench( + lambda: moe_sum_reduce(x, out, scaling_factor), quantiles=quantiles + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def verify_correctness(num_tokens=1024): + x = torch.randn(num_tokens, 9, 4096, device="cuda", dtype=torch.bfloat16) + scaling_factor = 0.3 + + out_baseline = torch.empty_like(x[:, 0]) + compute_sum_scaled_baseline(x, out_baseline, scaling_factor) + + out_compiled = torch.empty_like(out_baseline) + compute_sum_scaled_compiled(x, out_compiled, scaling_factor) + + out_triton = torch.empty_like(out_baseline) + moe_sum_reduce(x, out_triton, scaling_factor) + + if torch.allclose( + out_baseline, out_compiled, atol=1e-2, rtol=1e-2 + ) and torch.allclose(out_baseline, out_triton, atol=1e-2, rtol=1e-2): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + print( + f"Baseline vs Compiled: {(out_baseline - out_compiled).abs().max().item()}" + ) + print(f"Baseline vs Triton: {(out_baseline - out_triton).abs().max().item()}") + + +if __name__ == "__main__": + print("Running correctness verification...") + verify_correctness() + + print("\nRunning performance benchmark...") + benchmark = get_benchmark() + benchmark.run( + print_data=True, + # save_path="./configs/benchmark_ops/sum_scaled/" + ) diff --git a/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py b/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py new file mode 100644 index 000000000..2b4faa24b --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py @@ -0,0 +1,305 @@ +# python3 benchmark/kernels/fused_moe_triton/benchmark_torch_compile_fused_moe.py --model /DeepSeek-V3/ --tp-size 8 --use-fp8-w8a8 +import argparse + +import torch +import triton +from torch.nn import functional as F +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_triton, +) +from sglang.srt.model_executor.cuda_graph_runner import set_torch_compile_config + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen3MoeForCausalLM": + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Llama4ForConditionalGeneration": + E = config.text_config.num_local_experts + topk = config.text_config.num_experts_per_tok + intermediate_size = config.text_config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_topk_native( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + M, _ = hidden_states.shape + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + topk_weights = F.softmax(gating_output.float(), dim=-1) + topk_weights, topk_ids = torch.topk(topk_weights, topk, dim=-1) + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + return topk_weights, topk_ids + + +@torch.compile(dynamic=False) +def fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +) -> torch.Tensor: + assert not use_fp8_w8a8, "Fp8_w8a8 fused_moe is not supported for torch compile" + + topk_weights, topk_ids = fused_topk_native( + hidden_states=x, + gating_output=input_gating, + topk=topk, + renormalize=True, + ) + w13_weights = w1[topk_ids] + w1_weights, w3_weights = torch.chunk(w13_weights, 2, dim=2) + w2_weights = w2[topk_ids] + x1 = torch.einsum("ti,taoi -> tao", x, w1_weights) + x1 = F.silu(x1) + x3 = torch.einsum("ti, taoi -> tao", x, w3_weights) + expert_outs = torch.einsum("tao, taio -> tai", (x1 * x3), w2_weights) + return torch.einsum("tai,ta -> ti", expert_outs, topk_weights.to(expert_outs.dtype)) + + +def fused_moe_torch_compile( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_torch( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, +): + return fused_moe_triton( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 5)), + line_arg="provider", + line_vals=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + line_names=[ + "fused_moe_triton", + "fused_moe_torch_compile", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8_w8a8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + set_torch_compile_config() + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + + if use_fp8_w8a8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + w1_scale = w2_scale = a1_scale = a2_scale = None + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_torch_compile + if provider == "fused_moe_torch_compile" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/fused_moe_torch_compile/", + ) + args = parser.parse_args() + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py new file mode 100644 index 000000000..6afd7f354 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py @@ -0,0 +1,349 @@ +# python3 benchmark/kernels/fused_moe_triton/benchmark_vllm_vs_sglang_fused_moe_triton.py --model /DeepSeek-V3/ --tp-size 8 --use-fp8-w8a8 +import argparse + +import torch +import triton +import vllm +from transformers import AutoConfig +from vllm.model_executor.layers.fused_moe.fused_moe import fused_moe as fused_moe_vllm + +from sglang.srt.distributed.parallel_state import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe as fused_moe_sglang, +) + + +def get_model_config(model_name: str, tp_size: int): + """Get model configuration parameters""" + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) + + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen2MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Qwen3MoeForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "DeepseekV2ForCausalLM", + "DeepseekV3ForCausalLM", + "Glm4MoeForCausalLM", + ]: + E = ( + config.n_routed_experts + 1 + if config.architectures[0] in ["DeepseekV3ForCausalLM"] + else config.n_routed_experts + ) + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] == "Llama4ForConditionalGeneration": + E = config.text_config.num_local_experts + topk = config.text_config.num_experts_per_tok + intermediate_size = config.text_config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + vllm_version_num = ( + vllm.__version_tuple__[0] * 100 + + vllm.__version_tuple__[1] * 10 + + vllm.__version_tuple__[2] + ) + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + assert ( + vllm_version_num >= 66 + ), "Block-wise quantized fp8 fused_moe is only supported for VLLM>=0.6.6.post1" + + shape_configs = { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + "block_shape": block_shape, + } + print(f"{shape_configs=}") + return shape_configs + + +def fused_moe_vllm_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + if block_shape is not None: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + else: + return fused_moe_vllm( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + + +def fused_moe_sglang_api( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=False, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + block_shape=None, +): + return fused_moe_sglang( + x, + w1, + w2, + input_gating, + topk, + renormalize=True, + inplace=True, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=list(range(1, 513)), + line_arg="provider", + line_vals=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + line_names=[ + "vllm_fused_moe_triton", + "sglang_fused_moe_triton", + ], + styles=[ + ("blue", "-"), + ("green", "-"), + ], + ylabel="Time (ms)", + plot_name="fused-moe-performance", + args={}, + ) +) +def benchmark(batch_size, provider, model_config, use_fp8_w8a8=False): + print(f"benchmark {provider} with batch_size={batch_size}") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + + num_tokens = batch_size + num_experts = model_config["num_experts"] + hidden_size = model_config["hidden_size"] + shard_intermediate_size = model_config["shard_intermediate_size"] + topk = model_config["topk"] + dtype = model_config["dtype"] + block_shape = model_config["block_shape"] + + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + w1_scale = w2_scale = a1_scale = a2_scale = None + + if use_fp8_w8a8: + init_dtype = dtype + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + + if block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + else: + w1 = torch.randn(num_experts, shard_intermediate_size, hidden_size, dtype=dtype) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=dtype + ) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + + # Warmup + api_func = ( + fused_moe_vllm_api + if provider == "vllm_fused_moe_triton" + else fused_moe_sglang_api + ) + for _ in range(10): + y = api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + torch.cuda.synchronize() + + quantiles = [0.5, 0.2, 0.8] + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: api_func( + x, + w1, + w2, + input_gating, + topk, + use_fp8_w8a8=use_fp8_w8a8, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + )[0], + quantiles=quantiles, + ) + return ms, min_ms, max_ms + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", type=int, default=2) + parser.add_argument("--use-fp8-w8a8", action="store_true") + parser.add_argument( + "--save-path", + type=str, + default="./configs/benchmark_ops/vllm_sglang_fused_moe/", + ) + args = parser.parse_args() + + try: + if not torch.distributed.is_initialized(): + torch.distributed.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo", + init_method="tcp://127.0.0.1:23456", + world_size=1, + rank=0, + ) + + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method="tcp://127.0.0.1:23456", + local_rank=0, + backend="nccl" if torch.cuda.is_available() else "gloo", + ) + + initialize_model_parallel( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + ) + + model_config = get_model_config(args.model, args.tp_size) + benchmark.run( + show_plots=True, + print_data=True, + save_path=args.save_path, + model_config=model_config, + use_fp8_w8a8=args.use_fp8_w8a8, + ) + finally: + destroy_model_parallel() + destroy_distributed_environment() + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py b/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py new file mode 100644 index 000000000..7b52f02a3 --- /dev/null +++ b/benchmark/kernels/fused_moe_triton/tuning_fused_moe_triton.py @@ -0,0 +1,599 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py +import argparse +import json +import time +from contextlib import nullcontext +from datetime import datetime +from typing import Any, Dict, List, Tuple, TypedDict + +import ray +import torch +import triton +from ray.experimental.tqdm_ray import tqdm +from transformers import AutoConfig + +from sglang.srt.layers.moe.fused_moe_triton import override_config +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + fused_moe, + get_config_dtype_str, + get_config_file_name, + get_default_config, + get_moe_configs, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +class BenchmarkConfig(TypedDict): + BLOCK_SIZE_M: int + BLOCK_SIZE_N: int + BLOCK_SIZE_K: int + GROUP_SIZE_M: int + num_warps: int + num_stages: int + + +def benchmark_config( + config: BenchmarkConfig, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int] = None, + num_iters: int = 100, +) -> float: + init_dtype = torch.float16 if use_fp8_w8a8 else dtype + x = torch.randn(num_tokens, hidden_size, dtype=dtype) + if use_int8_w8a16 or use_int8_w8a8: + w1 = torch.randint( + -127, + 127, + ( + num_experts, + shard_intermediate_size, + hidden_size, + ), + dtype=torch.int8, + ) + w2 = torch.randint( + -127, + 127, + ( + num_experts, + hidden_size, + shard_intermediate_size // 2, + ), + dtype=torch.int8, + ) + else: + w1 = torch.randn( + num_experts, shard_intermediate_size, hidden_size, dtype=init_dtype + ) + w2 = torch.randn( + num_experts, hidden_size, shard_intermediate_size // 2, dtype=init_dtype + ) + gating_output = torch.randn(num_iters, num_tokens, num_experts, dtype=torch.float32) + + w1_scale = None + w2_scale = None + a1_scale = None + a2_scale = None + if use_int8_w8a16: + w1_scale = torch.randn( + (num_experts, 2 * shard_intermediate_size), dtype=torch.float32 + ) + w2_scale = torch.randn((hidden_size, num_experts), dtype=torch.float32) + if use_fp8_w8a8 or use_int8_w8a8: + if use_int8_w8a8 and block_shape is None: + w1_scale = torch.randn( + num_experts, shard_intermediate_size, dtype=torch.float32 + ) + w2_scale = torch.randn(num_experts, hidden_size, dtype=torch.float32) + elif block_shape is None: + w1_scale = torch.randn(num_experts, dtype=torch.float32) + w2_scale = torch.randn(num_experts, dtype=torch.float32) + a1_scale = torch.randn(1, dtype=torch.float32) + a2_scale = torch.randn(1, dtype=torch.float32) + else: + block_n, block_k = block_shape[0], block_shape[1] + n_tiles_w1 = (shard_intermediate_size + block_n - 1) // block_n + n_tiles_w2 = (hidden_size + block_n - 1) // block_n + k_tiles_w1 = (hidden_size + block_k - 1) // block_k + k_tiles_w2 = (shard_intermediate_size // 2 + block_k - 1) // block_k + w1_scale = torch.rand( + (num_experts, n_tiles_w1, k_tiles_w1), dtype=torch.float32 + ) + w2_scale = torch.rand( + (num_experts, n_tiles_w2, k_tiles_w2), dtype=torch.float32 + ) + + if use_fp8_w8a8: + w1 = w1.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn) + + input_gating = torch.randn(num_tokens, num_experts, dtype=torch.float32) + topk_config = TopKConfig( + top_k=topk, + renormalize=True, + ) + topk_output = select_experts(x, input_gating, topk_config) + + def prepare(i: int): + input_gating = gating_output[i] + new_topk_output = select_experts(x, input_gating, topk_config) + topk_output.topk_weights.copy_(new_topk_output.topk_weights) + topk_output.topk_ids.copy_(new_topk_output.topk_ids) + topk_output.router_logits.copy_(new_topk_output.router_logits) + + def run(): + moe_runner_config = MoeRunnerConfig( + inplace=True, + ) + + with override_config(config): + fused_moe( + x, + w1, + w2, + topk_output, + moe_runner_config=moe_runner_config, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + # JIT compilation & warmup + run() + torch.cuda.synchronize() + + # Capture 10 invocations with CUDA graph + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(10): + run() + torch.cuda.synchronize() + + # Warmup + for _ in range(5): + graph.replay() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for i in range(num_iters): + prepare(i) + torch.cuda.synchronize() + + start_event.record() + graph.replay() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + graph.reset() + return avg + + +def get_rocm_configs_compute_bound() -> List[Dict[str, int]]: + configs: List[BenchmarkConfig] = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [1, 2, 4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound() -> List[Dict[str, int]]: + # Reduced search space for faster tuning. + # TODO(woosuk): Increase the search space and use a performance model to + # prune the search space. + configs: List[BenchmarkConfig] = [] + if _is_hip: + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128, 256]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +@ray.remote(num_gpus=1) +class BenchmarkWorker: + + def __init__(self, seed: int) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + self.seed = seed + # Get the device ID to allocate tensors and kernels + # on the respective GPU. + self.device_id = int(ray.get_gpu_ids()[0]) + + def benchmark( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + ) -> Tuple[Dict[str, int], float]: + torch.cuda.manual_seed_all(0) + dtype_str = get_config_dtype_str( + dtype, use_int8_w8a16=use_int8_w8a16, use_fp8_w8a8=use_fp8_w8a8 + ) + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + block_n = block_shape[0] if block_shape else 0 + block_k = block_shape[1] if block_shape else 0 + op_config = get_moe_configs( + num_experts, shard_intermediate_size // 2, dtype_str, block_n, block_k + ) + if op_config is None: + config = get_default_config( + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype_str, + False, + block_shape, + ) + else: + config = op_config[min(op_config.keys(), key=lambda x: abs(x - num_tokens))] + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + return config, kernel_time + + def tune( + self, + num_tokens: int, + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], + search_space: List[Dict[str, int]], + ) -> Dict[str, int]: + best_config = None + best_time = float("inf") + with torch.cuda.device(self.device_id) if is_hip() else nullcontext(): + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + config, + num_tokens, + num_experts, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + num_iters=10, + ) + except (triton.runtime.autotuner.OutOfResources, RuntimeError): + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={num_tokens}") + assert best_config is not None + return best_config + + +def sort_config(config: BenchmarkConfig) -> BenchmarkConfig: + return { + "BLOCK_SIZE_M": config["BLOCK_SIZE_M"], + "BLOCK_SIZE_N": config["BLOCK_SIZE_N"], + "BLOCK_SIZE_K": config["BLOCK_SIZE_K"], + "GROUP_SIZE_M": config["GROUP_SIZE_M"], + "num_warps": config["num_warps"], + "num_stages": config["num_stages"], + **( + {"waves_per_eu": config["waves_per_eu"]} if "waves_per_eu" in config else {} + ), + } + + +def save_configs( + configs: Dict[int, BenchmarkConfig], + num_experts: int, + shard_intermediate_size: int, + hidden_size: int, + topk: int, + dtype: torch.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + block_shape: List[int], +) -> None: + dtype_str = get_config_dtype_str( + dtype, + use_int8_w8a16=use_int8_w8a16, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + ) + + # NOTE(woosuk): The current naming convention uses w2.shape[2], which + # is the intermediate size after silu_and_mul. + filename = get_config_file_name( + num_experts, + shard_intermediate_size // 2, + dtype_str, + block_shape, + ) + + print(f"Writing best config to {filename}...") + with open(filename, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def main(args: argparse.Namespace): + print(args) + + config = AutoConfig.from_pretrained(args.model, trust_remote_code=True) + if config.architectures[0] == "DbrxForCausalLM": + E = config.ffn_config.moe_num_experts + topk = config.ffn_config.moe_top_k + intermediate_size = config.ffn_config.ffn_hidden_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] == "JambaForCausalLM": + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in ["Qwen2MoeForCausalLM", "Qwen3MoeForCausalLM"]: + E = config.num_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in ["DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM"]: + E = ( + config.n_routed_experts + (0 if args.disable_shared_experts_fusion else 1) + if config.architectures[0] in ["DeepseekV3ForCausalLM"] + else config.n_routed_experts + ) + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] == "Llama4ForConditionalGeneration": + E = config.text_config.num_local_experts + ( + 0 if args.disable_shared_experts_fusion else 1 + ) + topk = config.text_config.num_experts_per_tok + intermediate_size = config.text_config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in [ + "Grok1ForCausalLM", + "Grok1ImgGen", + "Grok1AForCausalLM", + ]: + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + elif config.architectures[0] in ["Glm4MoeForCausalLM"]: + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + else: + # Default: Mixtral + E = config.num_local_experts + topk = config.num_experts_per_tok + intermediate_size = config.intermediate_size + shard_intermediate_size = 2 * intermediate_size // args.tp_size + + hidden_size = getattr(config, "hidden_size", None) or config.text_config.hidden_size + dtype = config.torch_dtype + use_fp8_w8a8 = args.dtype == "fp8_w8a8" + use_int8_w8a8 = args.dtype == "int8_w8a8" + use_int8_w8a16 = args.dtype == "int8_w8a16" + block_shape = None + if ( + hasattr(config, "quantization_config") + and "weight_block_size" in config.quantization_config + ): + block_shape = config.quantization_config["weight_block_size"] + assert len(block_shape) == 2 + + if args.batch_size is None: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] + else: + batch_sizes = [args.batch_size] + + ray.init() + num_gpus = int(ray.available_resources()["GPU"]) + workers = [BenchmarkWorker.remote(args.seed) for _ in range(num_gpus)] + + def _distribute(method: str, inputs: List[Any]) -> List[Any]: + outputs = [] + worker_idx = 0 + for input_args in inputs: + worker = workers[worker_idx] + worker_method = getattr(worker, method) + output = worker_method.remote(*input_args) + outputs.append(output) + worker_idx = (worker_idx + 1) % num_gpus + return ray.get(outputs) + + if args.tune: + search_space = get_configs_compute_bound() + if block_shape is not None: + block_n, block_k = block_shape[0], block_shape[1] + search_space = [ + config + for config in search_space + if block_k % config["BLOCK_SIZE_K"] == 0 + ] + print(f"Start tuning over {len(search_space)} configurations...") + + start = time.perf_counter() + configs = _distribute( + "tune", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + search_space, + ) + for batch_size in batch_sizes + ], + ) + best_configs = { + M: sort_config(config) for M, config in zip(batch_sizes, configs) + } + save_configs( + best_configs, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + end = time.perf_counter() + print(f"Tuning took {end - start:.2f} seconds") + else: + outputs = _distribute( + "benchmark", + [ + ( + batch_size, + E, + shard_intermediate_size, + hidden_size, + topk, + dtype, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + block_shape, + ) + for batch_size in batch_sizes + ], + ) + + for batch_size, (config, kernel_time) in zip(batch_sizes, outputs): + print(f"Batch size: {batch_size}, config: {config}") + print(f"Kernel time: {kernel_time:.2f} us") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model", type=str, default="mistralai/Mixtral-8x7B-Instruct-v0.1" + ) + parser.add_argument("--tp-size", "--tp", type=int, default=2) + parser.add_argument( + "--dtype", + type=str, + choices=["auto", "fp8_w8a8", "int8_w8a16", "int8_w8a8"], + default="auto", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument("--tune", action="store_true") + parser.add_argument("--disable-shared-experts-fusion", action="store_true") + args = parser.parse_args() + + main(args) diff --git a/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py new file mode 100644 index 000000000..78d81499e --- /dev/null +++ b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_decode.py @@ -0,0 +1,576 @@ +import itertools +import math +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import triton +import triton.language as tl +from einops import rearrange +from sgl_kernel import lightning_attention_decode as sgl_lightning_attention_decode + + +@triton.jit +def _decode_kernel( + Q, + K, + V, + KV, + Out, + S, + b: tl.constexpr, + h: tl.constexpr, + n: tl.constexpr, + d: tl.constexpr, + d_original: tl.constexpr, + e: tl.constexpr, + e_original: tl.constexpr, +): + off_bh = tl.program_id(0) + off_h = off_bh % h + + qk_offset = off_bh * n * d + v_offset = off_bh * n * e + o_offset = off_bh * n * e + kv_offset = off_bh * d * e + + s = tl.load(S + off_h) + ratio = tl.exp(-s) + + d_idx = tl.arange(0, d) + e_idx = tl.arange(0, e) + + # Create masks for original dimensions + d_mask = d_idx < d_original + e_mask = e_idx < e_original + + # Load with masking + q = tl.load(Q + qk_offset + d_idx, mask=d_mask, other=0.0) + k = tl.load(K + qk_offset + d_idx, mask=d_mask, other=0.0) + v = tl.load(V + v_offset + e_idx, mask=e_mask, other=0.0) + + # Load KV with 2D masking + kv = tl.load( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + mask=(d_mask[:, None] & e_mask[None, :]), + other=0.0, + ) + + # Compute outer product using element-wise operations + k_v_prod = k[:, None] * v[None, :] + kv = ratio * kv + k_v_prod + + # Store KV with 2D masking + tl.store( + KV + kv_offset + d_idx[:, None] * e + e_idx[None, :], + kv.to(KV.dtype.element_ty), + mask=(d_mask[:, None] & e_mask[None, :]), + ) + + # Compute matrix-vector multiplication using element-wise operations and reduction + o = tl.sum(q[:, None] * kv, axis=0) + + # Store output with masking + tl.store(Out + o_offset + e_idx, o.to(Out.dtype.element_ty), mask=e_mask) + + +def lightning_attn_decode(q, k, v, kv, s): + """Triton implementation of Lightning Attention decode operation""" + b, h, n, d = q.shape + e = v.shape[-1] + assert n == 1, "Sequence length must be 1 in decode mode" + + # Get padded dimensions (power of 2) + d_padded = next_power_of_2(d) + e_padded = next_power_of_2(e) + + # Create output tensor (padded) + o_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + + # Create padded tensors without actually padding the data + q_padded = torch.empty(b, h, n, d_padded, dtype=q.dtype, device=q.device) + k_padded = torch.empty(b, h, n, d_padded, dtype=k.dtype, device=k.device) + v_padded = torch.empty(b, h, n, e_padded, dtype=v.dtype, device=v.device) + kv_padded = torch.empty( + b, h, d_padded, e_padded, dtype=torch.float32, device=kv.device + ) + + # Copy data to padded tensors + q_padded[..., :d] = q + k_padded[..., :d] = k + v_padded[..., :e] = v + kv_padded[..., :d, :e] = kv + + # Launch kernel + grid = (b * h, 1) + _decode_kernel[grid]( + q_padded, + k_padded, + v_padded, + kv_padded, + o_padded, + s, + b=b, + h=h, + n=n, + d=d_padded, + d_original=d, + e=e_padded, + e_original=e, + ) + + # Get unpadded outputs + o = o_padded[..., :e] + kv_out = kv_padded[..., :d, :e] + + return o, kv_out + + +def next_power_of_2(n): + return 2 ** (int(math.ceil(math.log(n, 2)))) + + +class MiniMaxText01LightningAttention(nn.Module): + def __init__(self, config=None, layer_idx: Optional[int] = None, **kwargs): + super().__init__() + if config is None: + config = type("Config", (), kwargs) + + bias = False + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads) + + self.out_proj = nn.Linear( + self.head_dim * self.num_heads, self.hidden_size, bias=bias + ) + self.act = get_activation_fn(config.hidden_act) + self.norm = MiniMaxText01RMSNorm(self.head_dim * self.num_heads) + + self.qkv_proj = nn.Linear( + self.hidden_size, 3 * self.head_dim * self.num_heads, bias=bias + ) + self.output_gate = nn.Linear( + self.hidden_size, self.head_dim * self.num_heads, bias=bias + ) + + # for inference only + self.offset = 0 + self.layer_idx = layer_idx + + def forward( + self, + hidden_states, + attn_mask: Optional[torch.Tensor] = None, # (b, h, n, m) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, + **kwargs, + ): + if (not self.training) and (not do_eval): + return self.inference( + hidden_states, + attn_mask, + output_attentions, + past_key_value, + use_cache, + slope_rate, + ) + + def inference( + self, + x, + attn_mask: Optional[torch.Tensor] = None, # (b, n) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, # (h, 1, 1) + ): + # x: b n d + b, n, d = x.shape + # linear map + qkv = self.act(self.qkv_proj(x)) + new_shape = qkv.size()[:-1] + (self.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [self.head_dim] * 3, dim=3) + q = q.transpose(1, 2) # [b, n, h, d] -> [b, h, n, d] + k = k.transpose(1, 2) # [b, n, h, d] -> [b, h, n, d] + v = v.transpose(1, 2) # [b, n, h, d] -> [b, h, n, e] + + self.offset += 1 + ratio = torch.exp(-slope_rate) # [h, 1, 1] + + # decode mode + kv = past_key_value # [b, h, d, e] + output = [] + for i in range(n): + # kv: [b, h, d, e] + # ratio: [h, 1, 1] + # k: [b, h, n, d] + # v: [b, h, n, e] + # k[:, :, i : i + 1]: [b, h, 1, d] + # v[:, :, i : i + 1]: [b, h, 1, e] + # ratio * kv: [b, h, d, e] + # torch.einsum( + # "... n d, ... n e -> ... d e", + # k[:, :, i : i + 1], + # v[:, :, i : i + 1], + # ) + # [b, h, d, e] + [b, h, d, e] -> [b, h, d, e] + kv = ratio * kv + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + # q[:, :, i : i + 1]: [b, h, 1, d] + # kv.to(q.dtype): [b, h, d, e] + # torch.einsum( + # "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + # ) + # [b, h, 1, d] * [b, h, d, e] -> [b, h, 1, e] + qkv = torch.einsum( + "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + + # reshape + output = rearrange(output, "b h n d -> b n (h d)") + # normalize + output = self.norm(output) + # gate + output = F.sigmoid(self.output_gate(x)) * output + # outproj + output = self.out_proj(output) + + attn_weights = None + + return output, attn_weights, kv + + +def get_activation_fn(activation): + if activation == "gelu": + return F.gelu + elif activation == "relu": + return F.relu + elif activation == "elu": + return F.elu + elif activation == "sigmoid": + return F.sigmoid + elif activation == "exp": + + def f(x): + with torch.no_grad(): + x_max = torch.max(x, dim=-1, keepdims=True).values + y = torch.exp(x - x_max) + return y + + return f + elif activation == "leak": + return F.leaky_relu + elif activation == "1+elu": + + def f(x): + return 1 + F.elu(x) + + return f + elif activation == "2+elu": + + def f(x): + return 2 + F.elu(x) + + return f + elif activation == "silu" or activation == "swish": + return F.silu + elif activation == "sine": + return torch.sin + else: + return lambda x: x + + +class MiniMaxText01RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MiniMaxText01RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +def test_lightning_attention_implementations(model_params): + torch.manual_seed(42) + + batch_size = 64 + seq_len = 1 + dtype = torch.bfloat16 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + hidden_states = torch.randn( + batch_size, seq_len, model_params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(model_params["num_attention_heads"]).to(device) + + model_attn = MiniMaxText01LightningAttention(**model_params).to(dtype).to(device) + model_attn.eval() + + d = model_params["head_dim"] + past_kv = torch.randn( + batch_size, + model_params["num_attention_heads"], + d, + d, + device=device, + ) + with torch.no_grad(): + model_output, _, new_kv = model_attn.inference( + hidden_states, + attn_mask=attention_mask, + slope_rate=slope_rate, + past_key_value=past_kv, + ) + + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + past_kv = past_kv.contiguous() + slope_rate = slope_rate.contiguous() + + # Test Triton implementation + triton_output, triton_new_kv = lightning_attn_decode(q, k, v, past_kv, slope_rate) + triton_output = triton_output.transpose(1, 2).contiguous() + triton_output = triton_output.view(batch_size, seq_len, -1) + triton_output = model_attn.norm(triton_output) + triton_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * triton_output + triton_output = model_attn.out_proj(triton_output) + + # Test SGL implementation + sgl_output = torch.empty_like(v) + sgl_new_kv = torch.empty_like(past_kv) + sgl_lightning_attention_decode(q, k, v, past_kv, slope_rate, sgl_output, sgl_new_kv) + + sgl_output = sgl_output.transpose(1, 2).contiguous() + sgl_output = sgl_output.view(batch_size, seq_len, -1) + sgl_output = model_attn.norm(sgl_output) + sgl_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * sgl_output + sgl_output = model_attn.out_proj(sgl_output) + + # Verify Triton implementation results + torch.testing.assert_close( + model_output, + triton_output, + rtol=1e-3, + atol=1e-2, + msg="Triton lightning attention implementation produces different output results", + ) + torch.testing.assert_close( + new_kv, + triton_new_kv, + rtol=1e-3, + atol=1e-2, + msg="Triton lightning attention implementation produces different kv results", + ) + + # Verify SGL implementation results + torch.testing.assert_close( + model_output, + sgl_output, + rtol=1e-3, + atol=1e-2, + msg="SGL lightning attention implementation produces different output results", + ) + torch.testing.assert_close( + new_kv, + sgl_new_kv, + rtol=1e-3, + atol=1e-2, + msg="SGL lightning attention implementation produces different kv results", + ) + + print("✅ All implementations match") + + +def _build_slope_tensor(n_attention_heads: int): + def get_slopes(n): + def get_slopes_power_of_2(n): + start = 2 ** (-(2 ** -(math.log2(n) - 3))) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2(n) + else: + closest_power_of_2 = 2 ** math.floor(math.log2(n)) + return ( + get_slopes_power_of_2(closest_power_of_2) + + get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2] + ) + + slopes = torch.tensor(get_slopes(n_attention_heads)).reshape( + n_attention_heads, 1, 1 + ) + return slopes + + +def get_benchmark(): + batch_size_range = [i for i in range(1, 33)] # max 32 + seq_length_range = [1] # decode mode sequence length is fixed to 1 + configs = list(itertools.product(batch_size_range, seq_length_range)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["Original", "Triton", "SGL"], + line_names=[ + "Original PyTorch Implementation", + "Triton Implementation", + "SGL Implementation", + ], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="lightning-attention-decode-performance", + args={}, + ) + ) + def benchmark(batch_size, seq_len, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "gelu", + } + + hidden_states = torch.randn( + batch_size, seq_len, params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(params["num_attention_heads"]).to(device) + model_attn = MiniMaxText01LightningAttention(**params).to(dtype).to(device) + model_attn.eval() + + d = params["head_dim"] + past_kv = torch.randn( + batch_size, + params["num_attention_heads"], + d, + d, + device=device, + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "Original": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: model_attn.inference( + hidden_states, + attn_mask=attention_mask, + slope_rate=slope_rate, + past_key_value=past_kv, + ), + quantiles=quantiles, + ) + elif provider == "Triton": + + def run_triton(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + output, new_kv = lightning_attn_decode(q, k, v, past_kv, slope_rate) + output = output.transpose(1, 2).contiguous() + output = output.view(batch_size, seq_len, -1) + output = model_attn.norm(output) + output = torch.sigmoid(model_attn.output_gate(hidden_states)) * output + return model_attn.out_proj(output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_triton, + quantiles=quantiles, + ) + else: # SGL + + def run_sgl(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2).contiguous() + k = k.transpose(1, 2).contiguous() + v = v.transpose(1, 2).contiguous() + + output = torch.empty_like(v) + new_kv = torch.empty_like(past_kv) + sgl_lightning_attention_decode( + q, k, v, past_kv, slope_rate, output, new_kv + ) + + output = output.transpose(1, 2).contiguous() + output = output.view(batch_size, seq_len, -1) + output = model_attn.norm(output) + output = torch.sigmoid(model_attn.output_gate(hidden_states)) * output + return model_attn.out_proj(output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_sgl, + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/lightning_attention_decode/", + help="Path to save lightning attention decode benchmark results", + ) + args = parser.parse_args() + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "silu", + } + # Run correctness test first + # Adapted from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/config.json + test_lightning_attention_implementations(params) + + # Run performance benchmark + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py new file mode 100644 index 000000000..3bf9054bd --- /dev/null +++ b/benchmark/kernels/minmax-text-01-lightning_attention/benchmark_lightning_attention_prefill.py @@ -0,0 +1,603 @@ +import itertools +import math +import os +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import triton +import triton.language as tl +from einops import rearrange + + +# Adapted from https://github.com/OpenNLPLab/lightning-attention/blob/main/lightning_attn/ops/triton/lightning_attn2.py +@triton.jit +def _fwd_kernel( + Q, + K, + V, + Out, + S, # log lambda + b: tl.constexpr, + h: tl.constexpr, + n: tl.constexpr, + d: tl.constexpr, + e: tl.constexpr, + BLOCK: tl.constexpr, + NUM_BLOCK: tl.constexpr, + BLOCK_MODEL: tl.constexpr, +): + ##### get offset + off_bh = tl.program_id(0) + off_h = off_bh % h + off_e = tl.program_id(1) + qk_offset = off_bh * n * d + v_offset = off_bh * n * e + o_offset = off_bh * n * e + # channel offset + e_offset = off_e * BLOCK_MODEL + + ##### get block ptr + Q_block_ptr = Q + qk_offset + tl.arange(0, d)[None, :] + K_trans_block_ptr = K + qk_offset + tl.arange(0, d)[:, None] + V_block_ptr = V + v_offset + e_offset + tl.arange(0, BLOCK_MODEL)[None, :] + O_block_ptr = Out + o_offset + e_offset + tl.arange(0, BLOCK_MODEL)[None, :] + S_block_ptr = S + off_h + + ##### init diag decay(Lambda); q, k decay; kv + s = tl.load(S_block_ptr) + # q, k decay + off_block = tl.arange( + 0, BLOCK + ) # Not bug, this is a bit different from algorithm 1, but is mathematically equivalent + q_decay = tl.exp(-s.to(tl.float32) * off_block[:, None]) + k_trans_decay = tl.exp(-s.to(tl.float32) * (BLOCK - off_block[None, :])) + block_decay = tl.exp(-s.to(tl.float32) * BLOCK) + # diag decay + index = off_block[:, None] - off_block[None, :] + s_index = s * index + s_index = tl.where(index >= 0, -s_index, float("-inf")) + diag_decay = tl.exp(s_index) + kv = tl.zeros([d, BLOCK_MODEL], dtype=tl.float32) + + ##### compute + for i in range(NUM_BLOCK): + # load + q = tl.load( + Q_block_ptr + off_block[:, None] * d, mask=off_block[:, None] < n, other=0.0 + ).to(tl.float32) + k_trans = tl.load( + K_trans_block_ptr + off_block[None, :] * d, + mask=off_block[None, :] < n, + other=0.0, + ).to(tl.float32) + v = tl.load( + V_block_ptr + off_block[:, None] * e, mask=off_block[:, None] < n, other=0.0 + ).to(tl.float32) + + # compute + qk = tl.dot(q, k_trans) * diag_decay + o_intra = tl.dot(qk, v) + o_inter = tl.dot(q, kv) * q_decay + o = o_intra + o_inter + + # save and update + tl.store( + O_block_ptr + off_block[:, None] * e, + o.to(O_block_ptr.dtype.element_ty), + mask=off_block[:, None] < n, + ) + kv = block_decay * kv + tl.dot(k_trans * k_trans_decay, v) + off_block += BLOCK + + +def lightning_attn2(q, k, v, s): + q = q.contiguous() + k = k.contiguous() + v = v.contiguous() + s = s.contiguous() + + b, h, n, d = q.shape + e = v.shape[-1] + + # Pad d to next power of 2 + d_padded = next_power_of_2(d) + if d_padded != d: + q_padded = F.pad(q, (0, d_padded - d)) + k_padded = F.pad(k, (0, d_padded - d)) + else: + q_padded = q + k_padded = k + + # Pad e to next power of 2 + e_padded = next_power_of_2(e) + if e_padded != e: + v_padded = F.pad(v, (0, e_padded - e)) + else: + v_padded = v + + o_padded = torch.empty((b, h, n, e_padded), dtype=q.dtype, device=q.device) + + BLOCK = 64 + NUM_BLOCK = triton.cdiv(q.shape[2], BLOCK) + # parallel over channel + BLOCK_MODEL = min(triton.next_power_of_2(e_padded), 32) + grid = (b * h, triton.cdiv(e_padded, BLOCK_MODEL)) + + _fwd_kernel[grid]( + q_padded, + k_padded, + v_padded, + o_padded, + s, + b, + h, + n, + d_padded, + e_padded, + BLOCK=BLOCK, + NUM_BLOCK=NUM_BLOCK, + BLOCK_MODEL=BLOCK_MODEL, + ) + + # Remove padding from output + if e_padded != e: + o = o_padded[..., :e] + else: + o = o_padded + + return o + + +def is_support(dim): + return 16 % dim + + +def next_power_of_2(n): + return 2 ** (int(math.ceil(math.log(n, 2)))) + + +def lightning_attn_func(q, k, v, s): + b, h, n, d = q.shape + e = v.shape[-1] + assert is_support(d) and is_support(e) + + # pad v's feature dim to power of 2 + e_pad = next_power_of_2(e) + need_pad = e_pad != e + if need_pad: + v = F.pad(v, (0, e_pad - e)) + + if d > 128: + # split over head + if 64 % d: + m = 64 + elif 32 % d: + m = 32 + elif 16 % d: + m = 16 + arr = [m * i for i in range(d // m + 1)] + if arr[-1] != d: + arr.append(d) + n = len(arr) + o = 0 + for i in range(n - 1): + start = arr[i] + end = arr[i + 1] + q1 = q[..., start:end] + k1 = k[..., start:end] + o += lightning_attn2(q1, k1, v, s) + else: + o = lightning_attn2(q, k, v, s) + + if need_pad: + o = o[:, :, :, :e] + + return o + + +debug = eval(os.environ.get("debug", default="False")) + +BLOCK = 256 + + +# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->MiniMaxText01 +class MiniMaxText01RMSNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ + MiniMaxText01RMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + +# Copied from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/modeling_minimax_text_01.py +def get_activation_fn(activation): + if debug: + logger.info(f"activation: {activation}") + if activation == "gelu": + return F.gelu + elif activation == "relu": + return F.relu + elif activation == "elu": + return F.elu + elif activation == "sigmoid": + return F.sigmoid + elif activation == "exp": + + def f(x): + with torch.no_grad(): + x_max = torch.max(x, dim=-1, keepdims=True).values + y = torch.exp(x - x_max) + + return y + + return f + elif activation == "leak": + return F.leaky_relu + elif activation == "1+elu": + + def f(x): + return 1 + F.elu(x) + + return f + elif activation == "2+elu": + + def f(x): + return 2 + F.elu(x) + + return f + elif activation == "silu" or activation == "swish": + return F.silu + elif activation == "sine": + return torch.sin + else: + logger.info(f"activation: does not support {activation}, use Identity!!!") + return lambda x: x + + +# Copied from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/modeling_minimax_text_01.py +class MiniMaxText01LightningAttention(nn.Module): + def __init__(self, config=None, layer_idx: Optional[int] = None, **kwargs): + super().__init__() + if config is None: + config = type("Config", (), kwargs) + + bias = False + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = getattr(config, "head_dim", self.hidden_size // self.num_heads) + + self.out_proj = nn.Linear( + self.head_dim * self.num_heads, self.hidden_size, bias=bias + ) + self.act = get_activation_fn(config.hidden_act) + self.norm = MiniMaxText01RMSNorm(self.head_dim * self.num_heads) + + self.qkv_proj = nn.Linear( + self.hidden_size, 3 * self.head_dim * self.num_heads, bias=bias + ) + self.output_gate = nn.Linear( + self.hidden_size, self.head_dim * self.num_heads, bias=bias + ) + + # for inference only + self.offset = 0 + self.layer_idx = layer_idx + + def forward( + self, + hidden_states, + attn_mask: Optional[torch.Tensor] = None, # (b, h, n, m) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, + **kwargs, + ): + if (not self.training) and (not do_eval): + return self.inference( + hidden_states, + attn_mask, + output_attentions, + past_key_value, + use_cache, + slope_rate, + ) + + def inference( + self, + x, + attn_mask: Optional[torch.Tensor] = None, # (b, n) + output_attentions: bool = False, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + use_cache: bool = False, + slope_rate: Optional[torch.Tensor] = None, # (h, 1, 1) + ): + # x: b n d + b, n, d = x.shape + # linear map + qkv = self.act(self.qkv_proj(x)) + new_shape = qkv.size()[:-1] + (self.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [self.head_dim] * 3, dim=3) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + if past_key_value is None: + self.offset = q.shape[-2] + else: + self.offset += 1 + + # for align with metaseq + ratio = torch.exp(-slope_rate) + + # only use for the first time + if past_key_value is None: + slope_rate = slope_rate.to(torch.float32) + if attn_mask is not None: + v = v.masked_fill( + (1 - attn_mask).unsqueeze(1).unsqueeze(-1).to(torch.bool), 0 + ) + NUM_BLOCK = (n + BLOCK - 1) // BLOCK + b, h, n, d = q.shape + e = v.shape[-1] + # other + array = torch.arange(BLOCK).to(q) + 1 + q_decay = torch.exp(-slope_rate * array.reshape(-1, 1)) + k_decay = torch.exp(-slope_rate * (BLOCK - array.reshape(-1, 1))) + index = array[:, None] - array[None, :] + s_index = ( + slope_rate + * index[ + None, + None, + ] + ) + s_index = torch.where(index >= 0, -s_index, float("-inf")) + diag_decay = torch.exp(s_index) + + kv = torch.zeros(b, h, d, e).to(torch.float32).to(q.device) + output = torch.empty((b, h, n, e), dtype=q.dtype, device=q.device) + for i in range(NUM_BLOCK): + si = i * BLOCK + ei = min(si + BLOCK, n) + m = ei - si + qi = q[:, :, si:ei].contiguous() + ki = k[:, :, si:ei].contiguous() + vi = v[:, :, si:ei].contiguous() + qkv_none_diag = torch.matmul(qi * q_decay[:, :m], kv).to(torch.float32) + + # diag + qk = ( + torch.matmul(qi, ki.transpose(-1, -2)).to(torch.float32) + * diag_decay[:, :, :m, :m] + ) + qkv_diag = torch.matmul(qk, vi.to(torch.float32)) + block_decay = torch.exp(-slope_rate * m) + output[:, :, si:ei] = qkv_none_diag + qkv_diag + kv = block_decay * kv + torch.matmul( + (ki * k_decay[:, -m:]).transpose(-1, -2).to(vi.dtype), vi + ) + + else: + kv = past_key_value + output = [] + for i in range(n): + kv = ratio * kv + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + qkv = torch.einsum( + "... n e, ... e d -> ... n d", q[:, :, i : i + 1], kv.to(q.dtype) + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + # reshape + output = rearrange(output, "b h n d -> b n (h d)") + # normalize + output = self.norm(output) + # gate + output = F.sigmoid(self.output_gate(x)) * output + # outproj + output = self.out_proj(output) + + attn_weights = None + + return output, attn_weights, kv + + +def _build_slope_tensor(n_attention_heads: int): + def get_slopes(n): + def get_slopes_power_of_2(n): + start = 2 ** (-(2 ** -(math.log2(n) - 3))) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2( + n + ) # In the paper, we only train models that have 2^a heads for some a. This function has + else: # some good properties that only occur when the input is a power of 2. To maintain that even + closest_power_of_2 = 2 ** math.floor( + math.log2(n) + ) # when the number of heads is not a power of 2, we use this workaround. + return ( + get_slopes_power_of_2(closest_power_of_2) + + get_slopes(2 * closest_power_of_2)[0::2][: n - closest_power_of_2] + ) + + # h, 1, 1 + slopes = torch.tensor(get_slopes(n_attention_heads)).reshape( + n_attention_heads, 1, 1 + ) + + return slopes + + +def test_lightning_attention_implementations(model_params): + torch.manual_seed(42) + + batch_size = 2 + seq_len = 1024 + dtype = torch.bfloat16 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + hidden_states = torch.randn( + batch_size, seq_len, model_params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(model_params["num_attention_heads"]).to(device) + + model_attn = MiniMaxText01LightningAttention(**model_params).to(dtype).to(device) + model_attn.eval() + + with torch.no_grad(): + model_output, _, _ = model_attn.inference( + hidden_states, attn_mask=attention_mask, slope_rate=slope_rate + ) + + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + lib_output = lightning_attn_func(q, k, v, slope_rate) + lib_output = lib_output.transpose(1, 2).contiguous() + lib_output = lib_output.view(batch_size, seq_len, -1) + lib_output = model_attn.norm(lib_output) + lib_output = torch.sigmoid(model_attn.output_gate(hidden_states)) * lib_output + lib_output = model_attn.out_proj(lib_output) + + torch.testing.assert_close( + model_output, + lib_output, + rtol=1e-3, + atol=1e-2, + msg="Lightning attention implementations produce different results", + ) + + print("✅ Two implementations match") + + +def get_benchmark(): + batch_size_range = [2**i for i in range(0, 7)] # max 64 + seq_length_range = [256, 512, 1024, 2048, 4096] # max 4096 + configs = list(itertools.product(batch_size_range, seq_length_range)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["MiniMax-Text-01", "OpenNLPLab"], + line_names=[ + "MiniMax-Text-01 Model Implementation", + "OpenNLPLab Library Implementation", + ], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="lightning-attention-prefill-performance", + args={}, + ) + ) + def benchmark(batch_size, seq_len, provider): + dtype = torch.bfloat16 + device = torch.device("cuda") + + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "gelu", + } + + hidden_states = torch.randn( + batch_size, seq_len, params["hidden_size"], dtype=dtype, device=device + ) + + attention_mask = torch.ones(batch_size, seq_len, dtype=dtype, device=device) + + slope_rate = _build_slope_tensor(params["num_attention_heads"]).to(device) + model_attn = MiniMaxText01LightningAttention(**params).to(dtype).to(device) + model_attn.eval() + + quantiles = [0.5, 0.2, 0.8] + if provider == "MiniMax-Text-01": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: model_attn.inference( + hidden_states, attn_mask=attention_mask, slope_rate=slope_rate + ), + quantiles=quantiles, + ) + else: + + def run_lib(): + qkv = model_attn.act(model_attn.qkv_proj(hidden_states)) + new_shape = qkv.size()[:-1] + (model_attn.num_heads, -1) + qkv = qkv.view(*new_shape) + q, k, v = torch.split(qkv, [model_attn.head_dim] * 3, dim=-1) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + lib_output = lightning_attn_func(q, k, v, slope_rate) + lib_output = lib_output.transpose(1, 2).contiguous() + lib_output = lib_output.view(batch_size, seq_len, -1) + lib_output = model_attn.norm(lib_output) + lib_output = ( + torch.sigmoid(model_attn.output_gate(hidden_states)) * lib_output + ) + return model_attn.out_proj(lib_output) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_lib, + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/lightning_attention_prefill/", + help="Path to save lightning attention prefill benchmark results", + ) + args = parser.parse_args() + + # Run correctness test first + # Adapted from https://huggingface.co/MiniMaxAI/MiniMax-Text-01/blob/main/config.json + params = { + "hidden_size": 6144, + "num_attention_heads": 64, + "head_dim": 96, + "hidden_act": "silu", + } + test_lightning_attention_implementations(params) + + # Run performance benchmark + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/quantization/bench_fp4_quant.py b/benchmark/kernels/quantization/bench_fp4_quant.py new file mode 100644 index 000000000..318e820ad --- /dev/null +++ b/benchmark/kernels/quantization/bench_fp4_quant.py @@ -0,0 +1,133 @@ +import argparse +import itertools + +import torch +import triton +from sgl_kernel import scaled_fp4_grouped_quant, silu_and_mul_scaled_fp4_grouped_quant +from sgl_kernel.elementwise import silu_and_mul + +from sglang.srt.layers.moe.ep_moe.kernels import silu_and_mul_masked_post_quant_fwd +from sglang.srt.layers.quantization import deep_gemm_wrapper + + +def _test_accuracy_once(E, M, K, input_dtype, device): + x = torch.randn(E, M, K, device=device, dtype=input_dtype) + glb_scales = torch.ones((E,), dtype=torch.float32, device=device) + masks = torch.full((E,), M, dtype=torch.int32, device=device) + out, blk_scales = silu_and_mul_scaled_fp4_grouped_quant(x, glb_scales, masks) + out1, blk_scales1 = scaled_fp4_grouped_quant( + silu_and_mul(x), + glb_scales, + masks, + ) + + torch.testing.assert_close(out, out1) + torch.testing.assert_close(blk_scales, blk_scales1) + print(f"E: {E}, M: {M}, K: {K}, type: {input_dtype} OK") + + +NUM_RANKS = 48 +M_PER_RANKs = [128, 256, 512, 1024] +Ms = [M_PER_RANK * NUM_RANKS for M_PER_RANK in M_PER_RANKs] +Ks = [2048, 4096, 7168] + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["M", "K"], + x_vals=list(itertools.product(Ms, Ks)), + x_log=False, + line_arg="provider", + line_vals=["triton_fp8", "cuda_unfused_fp4", "cuda_fused_fp4"], + line_names=["triton_fp8", "cuda_unfused_fp4", "cuda_fused_fp4"], + styles=[("blue", "-"), ("orange", "-"), ("green", "-")], + ylabel="ms", + plot_name="fp4 quant", + args={}, + ) +) +def benchmark(M, K, provider): + E = 6 + device = "cuda" + x = torch.randn(E, M, K, device=device, dtype=torch.bfloat16) + glb_scales = torch.ones((E,), dtype=torch.float32, device=device) + masks = torch.randint(1, 4096, (E,), dtype=torch.int32, device=device) + fp8_out = torch.empty( + ( + x.shape[0], + x.shape[1], + x.shape[2] // 2, + ), + device=x.device, + dtype=torch.float8_e4m3fn, + ) + scale_block_size = 128 + fp8_scales = torch.empty( + ( + x.shape[0], + x.shape[1], + x.shape[2] // 2 // scale_block_size, + ), + device=x.device, + dtype=torch.float32, + ) + + quantiles = [0.5, 0.2, 0.8] + if provider == "triton_fp8": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: silu_and_mul_masked_post_quant_fwd( + x, + fp8_out, + fp8_scales, + scale_block_size, + masks, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ), + quantiles=quantiles, + ) + if provider == "cuda_unfused_fp4": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: scaled_fp4_grouped_quant( + silu_and_mul(x), + glb_scales, + masks, + ), + quantiles=quantiles, + ) + if provider == "cuda_fused_fp4": + ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( + lambda: silu_and_mul_scaled_fp4_grouped_quant( + x, + glb_scales, + masks, + ), + quantiles=quantiles, + ) + + return ms, min_ms, max_ms + + +def test_accuracy(): + E = 6 + N_RANKS = 48 + Ms = [128, 256, 512, 1024] + Ks = [2048, 4096, 7168] + input_dtype = torch.bfloat16 + for M in Ms: + for K in Ks: + _test_accuracy_once(E, N_RANKS * M, K, input_dtype, "cuda") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./bench_fp4_quant_res", + help="Path to save fp4 quant benchmark results", + ) + args = parser.parse_args() + + test_accuracy() + + benchmark.run(print_data=True, show_plots=True, save_path=args.save_path) diff --git a/benchmark/kernels/quantization/bench_int8_quant.py b/benchmark/kernels/quantization/bench_int8_quant.py new file mode 100644 index 000000000..94b795690 --- /dev/null +++ b/benchmark/kernels/quantization/bench_int8_quant.py @@ -0,0 +1,94 @@ +import argparse + +import torch +import triton +from vllm._custom_ops import scaled_int8_quant as vllm_scaled_int8_quant + +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 + + +@torch.compile(backend="inductor") +def torch_int8_quant(x): + int8_max = torch.iinfo(torch.int8).max + + abs_max = x.abs().max(dim=-1, keepdim=True).values + scales = abs_max.to(torch.float32) / float(int8_max) + + q_x = (x / scales).round().to(torch.int8) + + return q_x, scales + + +def _test_accuracy_once(M, K, input_dtype, device): + x = torch.randn(M, K, dtype=input_dtype, device=device) * 5000 + out, scales, _ = vllm_scaled_int8_quant(x, symmetric=True) + out1, scales1 = per_token_quant_int8(x) + out2, scales2 = torch_int8_quant(x) + torch.testing.assert_close(out, out2, atol=1, rtol=0) + torch.testing.assert_close(out, out1, atol=1, rtol=0) + torch.testing.assert_close(scales, scales2) + torch.testing.assert_close(scales1, scales2) + print(f"M: {M}, K: {K}, type: {input_dtype} OK") + + +def test_accuracy(): + Ms = [1, 13, 128, 1024, 2048, 4096] + Ks = [512, 1024, 2048, 8192] + input_dtypes = [torch.float16, torch.bfloat16] + for M in Ms: + for K in Ks: + for input_dtype in input_dtypes: + _test_accuracy_once(M, K, input_dtype, "cuda") + + +@triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=[1, 16, 32, 64, 128, 256, 512, 1024, 2048], + x_log=False, + line_arg="provider", + line_vals=["vllm op", "triton", "torch.compile"], + line_names=["vllm op", "triton", "torch.compile"], + styles=[("blue", "-"), ("orange", "-"), ("red", "-")], + ylabel="ms", + plot_name="int8 per token quant", + args={}, + ) +) +def benchmark(batch_size, provider): + M, K = batch_size, 16384 + x = torch.randn(M, K, dtype=torch.float16, device="cuda") * 1000 + + quantiles = [0.5, 0.2, 0.8] + if provider == "vllm op": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: vllm_scaled_int8_quant(x, symmetric=True), + quantiles=quantiles, + ) + if provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: per_token_quant_int8(x), + quantiles=quantiles, + ) + if provider == "torch.compile": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: torch_int8_quant(x), + quantiles=quantiles, + ) + + return ms, min_ms, max_ms + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./bench_int8_quant_res", + help="Path to save int8 quant benchmark results", + ) + args = parser.parse_args() + + test_accuracy() + + benchmark.run(print_data=True, show_plots=True, save_path=args.save_path) diff --git a/benchmark/kernels/quantization/tuning_block_wise_kernel.py b/benchmark/kernels/quantization/tuning_block_wise_kernel.py new file mode 100644 index 000000000..1b51e54b7 --- /dev/null +++ b/benchmark/kernels/quantization/tuning_block_wise_kernel.py @@ -0,0 +1,474 @@ +# Copyright 2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import json +import multiprocessing as mp +import os +import time +from datetime import datetime +from typing import Any, Dict, List + +import torch +import triton +from tqdm import tqdm + +mp.set_start_method("spawn", force=True) + +from sglang.srt.layers.quantization.fp8_kernel import ( + _w8a8_block_fp8_matmul, + _w8a8_block_fp8_matmul_unrolledx4, +) +from sglang.srt.layers.quantization.int8_kernel import _w8a8_block_int8_matmul +from sglang.srt.utils import get_device_core_count, get_device_name, is_hip + +_is_hip = is_hip() + +DTYPE_MAP = { + "float32": torch.float32, + "float16": torch.float16, + "half": torch.half, + "bfloat16": torch.bfloat16, +} + + +def w8a8_block_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + config: Dict[str, Any], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. + # Empirical testing shows the sweet spot lies when it's less than the # of + # compute units available on the device. + num_workgroups = triton.cdiv(M, config["BLOCK_SIZE_M"]) * triton.cdiv( + N, config["BLOCK_SIZE_N"] + ) + + if A.dtype == torch.float8_e4m3fnuz or A.dtype == torch.float8_e4m3fn: + kernel = ( + _w8a8_block_fp8_matmul_unrolledx4 + if (_is_hip == True and num_workgroups <= get_device_core_count()) + else _w8a8_block_fp8_matmul + ) + else: + kernel = _w8a8_block_int8_matmul + + kernel[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C + + +def get_rocm_configs_compute_bound(): + configs = [] + waves_per_eu_range = 0 + for num_stages in [2]: + for block_m in [32, 64, 128, 256]: + for block_k in [32, 64, 128, 256]: + for block_n in [16, 32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 4, 8, 16, 32]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + "waves_per_eu": waves_per_eu_range, + } + ) + return configs + + +def get_configs_compute_bound(): + configs = [] + if _is_hip: + configs = get_rocm_configs_compute_bound() + else: + for num_stages in [2, 3, 4, 5]: + for block_m in [16, 32, 64, 128, 256]: + for block_k in [64, 128]: + for block_n in [32, 64, 128, 256]: + for num_warps in [4, 8]: + for group_size in [1, 16, 32, 64]: + configs.append( + { + "BLOCK_SIZE_M": block_m, + "BLOCK_SIZE_N": block_n, + "BLOCK_SIZE_K": block_k, + "GROUP_SIZE_M": group_size, + "num_warps": num_warps, + "num_stages": num_stages, + } + ) + return configs + + +def get_weight_shapes(tp_size): + # NOTE(HandH1998): The weight shapes only works for DeepSeek-V3. Modify them, if you tune for another different model. + # cannot TP + total = [ + (512 + 64, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (7168, 16384), + (7168, 18432), + ] + # N can TP + n_tp = [ + (18432 * 2, 7168), + ((128 + 64) * 128, 7168), + (128 * (128 + 128), 512), + (24576, 1536), + (4096, 7168), + ] + # K can TP + k_tp = [(7168, 18432), (7168, 16384), (7168, 2048)] + + weight_shapes = [] + for t in total: + weight_shapes.append(t) + for n_t in n_tp: + new_t = (n_t[0] // tp_size, n_t[1]) + weight_shapes.append(new_t) + for k_t in k_tp: + new_t = (k_t[0], k_t[1] // tp_size) + weight_shapes.append(new_t) + return weight_shapes + + +def benchmark_config( + A, B, As, Bs, block_size, config, out_dtype=torch.float16, num_iters=10 +): + def run(): + w8a8_block_matmul(A, B, As, Bs, block_size, config, out_dtype) + + torch.cuda.synchronize() + # JIT complication & warmup + for _ in range(5): + run() + torch.cuda.synchronize() + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + latencies: List[float] = [] + for i in range(num_iters): + torch.cuda.synchronize() + start_event.record() + run() + end_event.record() + end_event.synchronize() + latencies.append(start_event.elapsed_time(end_event)) + avg = sum(latencies) / (num_iters * 10) * 1000 # us + return avg + + +def tune(M, N, K, block_size, out_dtype, search_space, input_type): + factor_for_scale = 1e-2 + + if input_type == "fp8": + fp8_info = torch.finfo( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + A = A_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * fp8_max + ) + B = B_fp32.clamp(min=fp8_min, max=fp8_max).to( + torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + ) + else: + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + A_fp32 = ( + (torch.rand(M, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + A = A_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + B_fp32 = ( + (torch.rand(N, K, dtype=torch.float32, device="cuda") - 0.5) * 2 * int8_max + ) + B = B_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32, device="cuda") * factor_for_scale + Bs = ( + torch.rand(n_tiles, k_tiles, dtype=torch.float32, device="cuda") + * factor_for_scale + ) + + best_config = None + best_time = float("inf") + for config in tqdm(search_space): + try: + kernel_time = benchmark_config( + A, + B, + As, + Bs, + block_size, + config, + out_dtype, + num_iters=10, + ) + except triton.runtime.autotuner.OutOfResources: + # Some configurations may be invalid and fail to compile. + continue + + if kernel_time < best_time: + best_time = kernel_time + best_config = config + now = datetime.now() + print(f"{now.ctime()}] Completed tuning for batch_size={M}") + assert best_config is not None + return best_config + + +def save_configs( + N, + K, + block_n, + block_k, + configs, + save_path, + input_type="fp8", +) -> None: + os.makedirs(save_path, exist_ok=True) + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype={input_type}_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join(save_path, json_file_name) + print(f"Writing best config to {config_file_path}...") + + with open(config_file_path, "w") as f: + json.dump(configs, f, indent=4) + f.write("\n") + + +def get_available_gpu_count(): + """Get the number of available GPUs.""" + return torch.cuda.device_count() + + +def tune_on_gpu(args_dict): + """Run tuning on a specific GPU.""" + gpu_id = args_dict["gpu_id"] + batch_sizes = args_dict["batch_sizes"] + weight_shapes = args_dict["weight_shapes"] + args = args_dict["args"] + + torch.cuda.set_device(gpu_id) + print(f"Starting tuning on GPU {gpu_id} with batch sizes {batch_sizes}") + + block_n = args.block_n + block_k = args.block_k + out_dtype = DTYPE_MAP[args.out_dtype] + save_path = args.save_path + input_type = args.input_type + + search_space = get_configs_compute_bound() + search_space = [ + config for config in search_space if block_k % config["BLOCK_SIZE_K"] == 0 + ] + + start = time.perf_counter() + results = {} + for shape in tqdm(weight_shapes, desc=f"GPU {gpu_id} - Shapes"): + N, K = shape[0], shape[1] + print(f"[GPU {gpu_id}] Tune for weight shape of `N: {N}, K: {K}`") + benchmark_results = [ + tune( + batch_size, + N, + K, + [block_n, block_k], + out_dtype, + search_space, + input_type, + ) + for batch_size in tqdm(batch_sizes, desc=f"GPU {gpu_id} - Batch sizes") + ] + best_configs = {M: config for M, config in zip(batch_sizes, benchmark_results)} + save_configs(N, K, block_n, block_k, best_configs, save_path, input_type) + + end = time.perf_counter() + print(f"Tuning on GPU {gpu_id} took {end - start:.2f} seconds") + + +def distribute_batch_sizes(batch_sizes, num_gpus): + """Distribute batch sizes across available GPUs.""" + batches_per_gpu = [] + for i in range(num_gpus): + start_idx = i * len(batch_sizes) // num_gpus + end_idx = (i + 1) * len(batch_sizes) // num_gpus + batches_per_gpu.append(batch_sizes[start_idx:end_idx]) + return batches_per_gpu + + +def main(args): + print(args) + + num_gpus = get_available_gpu_count() + if num_gpus == 0: + raise RuntimeError("No GPU available for tuning") + print(f"Found {num_gpus} GPUs for parallel tuning") + + torch.cuda.init() + + if args.batch_size is None: + batch_sizes = [ + 1, + 2, + 4, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, + 256, + 512, + 1024, + 1536, + 2048, + 3072, + 4096, + ] + else: + batch_sizes = [args.batch_size] + num_gpus = 1 # If only one batch size, use only one GPU + + weight_shapes = get_weight_shapes(args.tp_size) + + batches_per_gpu = distribute_batch_sizes(batch_sizes, num_gpus) + + process_args = [] + for gpu_id in range(num_gpus): + process_args.append( + { + "gpu_id": gpu_id, + "batch_sizes": batches_per_gpu[gpu_id], + "weight_shapes": weight_shapes, # Each GPU processes all weight shapes + "args": args, + } + ) + + ctx = mp.get_context("spawn") + with ctx.Pool(num_gpus) as pool: + pool.map(tune_on_gpu, process_args) + + print("Multi-GPU tuning completed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--tp-size", "-tp", type=int, default=8) + parser.add_argument( + "--input-type", type=str, choices=["fp8", "int8"], default="fp8" + ) + parser.add_argument( + "--out-dtype", + type=str, + choices=["float32", "float16", "bfloat16", "half"], + default="float16", + ) + parser.add_argument("--block-n", type=int, default=128) + parser.add_argument("--block-k", type=int, default=128) + parser.add_argument("--batch-size", type=int, required=False) + parser.add_argument( + "--save-path", type=str, default="python/sglang/srt/layers/quantization/configs" + ) + args = parser.parse_args() + + main(args) diff --git a/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py b/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py new file mode 100644 index 000000000..aeeea62c0 --- /dev/null +++ b/benchmark/kernels/rmsnorm/benchmark_rmsnorm.py @@ -0,0 +1,230 @@ +import itertools +from typing import Optional, Tuple, Union + +import torch +import triton +from flashinfer.norm import fused_add_rmsnorm, rmsnorm +from torch import nn +from vllm import _custom_ops as vllm_ops + + +class HuggingFaceRMSNorm(nn.Module): + def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = x.to(orig_dtype) * self.weight + if residual is None: + return x + else: + return x, residual + + +def rmsnorm_naive( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + naive_norm = HuggingFaceRMSNorm(x.shape[-1], eps=eps) + naive_norm.weight = nn.Parameter(weight) + naive_norm = naive_norm.to(x.device) + + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + output = naive_norm(x, residual) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_flashinfer( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + fused_add_rmsnorm(x, residual, weight, eps) + output = (x, residual) + else: + output = rmsnorm(x, weight, eps) + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def rmsnorm_vllm( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + vllm_ops.fused_add_rms_norm(x, residual, weight, eps) + output = (x, residual) + else: + out = torch.empty_like(x) + vllm_ops.rms_norm(out, x, weight, eps) + output = out + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + + +def calculate_diff(batch_size, seq_len, hidden_size, use_residual=True): + dtype = torch.bfloat16 + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") + weight = torch.ones(hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) if use_residual else None + + output_naive = rmsnorm_naive( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_flashinfer = rmsnorm_flashinfer( + x.clone(), weight, residual.clone() if residual is not None else None + ) + output_vllm = rmsnorm_vllm( + x.clone(), weight, residual.clone() if residual is not None else None + ) + + if use_residual: + output_naive = output_naive[0] + output_flashinfer = output_flashinfer[0] + output_vllm = output_vllm[0] + + print(f"Naive output={output_naive}") + print(f"FlashInfer output={output_flashinfer}") + print(f"VLLM output={output_vllm}") + + if torch.allclose( + output_naive, output_flashinfer, atol=1e-2, rtol=1e-2 + ) and torch.allclose(output_naive, output_vllm, atol=1e-2, rtol=1e-2): + print("✅ All implementations match") + else: + print("❌ Implementations differ") + + +batch_size_range = [2**i for i in range(0, 7, 2)] +seq_length_range = [2**i for i in range(6, 11, 1)] +head_num_range = [32, 48] +configs = list(itertools.product(head_num_range, batch_size_range, seq_length_range)) + + +def get_benchmark(use_residual): + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["head_num", "batch_size", "seq_len"], + x_vals=[list(_) for _ in configs], + line_arg="provider", + line_vals=["huggingface", "flashinfer", "vllm"], + line_names=["HuggingFace", "FlashInfer", "vLLM"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name=f"rmsnorm-performance-{'with' if use_residual else 'without'}-residual", + args={}, + ) + ) + def benchmark(head_num, batch_size, seq_len, provider): + dtype = torch.bfloat16 + hidden_size = head_num * 128 # assuming head_dim = 128 + + x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") + weight = torch.ones(hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) if use_residual else None + + quantiles = [0.5, 0.2, 0.8] + + if provider == "huggingface": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_naive( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + elif provider == "flashinfer": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_flashinfer( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + else: + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: rmsnorm_vllm( + x.clone(), + weight, + residual.clone() if residual is not None else None, + ), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--use_residual", action="store_true", help="Whether to use residual connection" + ) + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/rmsnorm/", + help="Path to save rmsnorm benchmark results", + ) + args = parser.parse_args() + + # Run correctness test + calculate_diff( + batch_size=4, seq_len=128, hidden_size=4096, use_residual=args.use_residual + ) + + # Get the benchmark function with proper use_residual setting + benchmark = get_benchmark(args.use_residual) + # Run performance benchmark + benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py b/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py new file mode 100644 index 000000000..3e17205e7 --- /dev/null +++ b/benchmark/kernels/scheduler_batch/benchmark_get_last_loc_triton.py @@ -0,0 +1,169 @@ +import os + +import torch +import triton +import triton.language as tl + + +@torch.compile(dynamic=True) +def get_last_loc_torch( + req_to_token: torch.Tensor, + req_pool_indices_tensor: torch.Tensor, + prefix_lens_tensor: torch.Tensor, +) -> torch.Tensor: + return torch.where( + prefix_lens_tensor > 0, + req_to_token[req_pool_indices_tensor, prefix_lens_tensor - 1], + torch.full_like(prefix_lens_tensor, -1), + ) + + +@triton.jit +def get_last_loc_kernel( + req_to_token, + req_pool_indices_tensor, + prefix_lens_tensor, + result, + num_tokens, + req_to_token_stride, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(0) + offset = tl.arange(0, BLOCK_SIZE) + pid * BLOCK_SIZE + mask = offset < num_tokens + + prefix_lens = tl.load(prefix_lens_tensor + offset, mask=mask, other=0) + req_pool_indices = tl.load(req_pool_indices_tensor + offset, mask=mask, other=0) + + token_mask = prefix_lens > 0 + token_index = req_pool_indices * req_to_token_stride + (prefix_lens - 1) + tokens = tl.load(req_to_token + token_index, mask=token_mask, other=-1) + + tl.store(result + offset, tokens, mask=mask) + + +def get_last_loc_triton( + req_to_token: torch.Tensor, + req_pool_indices_tensor: torch.Tensor, + prefix_lens_tensor: torch.Tensor, +) -> torch.Tensor: + BLOCK_SIZE = 256 + num_tokens = prefix_lens_tensor.shape[0] + result = torch.empty_like(prefix_lens_tensor) + grid = (triton.cdiv(num_tokens, BLOCK_SIZE),) + + get_last_loc_kernel[grid]( + req_to_token, + req_pool_indices_tensor, + prefix_lens_tensor, + result, + num_tokens, + req_to_token.stride(0), + BLOCK_SIZE, + ) + return result + + +def test_get_last_loc(): + max_batch = 4097 + max_context_len = 6148 + batch_size = 20 + + # Initialize input tensors + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int64, device="cuda") + pre_lens = torch.randint( + -max_context_len // 2, + max_context_len, + (batch_size,), + dtype=torch.int64, + device="cuda", + ) + + last_loc_res = get_last_loc_triton(req_to_token, req_pool_indices, pre_lens) + last_loc_ref = get_last_loc_torch(req_to_token, req_pool_indices, pre_lens) + + # Compare results + torch.testing.assert_close(last_loc_res, last_loc_ref) + + +def get_benchmark(): + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024] + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + line_arg="provider", + line_vals=["reference", "triton"], + line_names=["PyTorch", "Triton"], + styles=[("blue", "-"), ("green", "-")], + ylabel="us", + plot_name="get-last-loc-performance", + args={}, + ) + ) + def benchmark(batch_size, provider): + max_batch = 2048 + max_context_len = 16384 + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int64, device="cuda") + pre_lens = torch.randint( + -max_context_len // 2, + max_context_len, + (batch_size,), + dtype=torch.int64, + device="cuda", + ) + + quantiles = [0.5, 0.2, 0.8] + + if provider == "reference": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: get_last_loc_torch(req_to_token, req_pool_indices, pre_lens), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: get_last_loc_triton(req_to_token, req_pool_indices, pre_lens), + quantiles=quantiles, + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def run_benchmark(save_path: str = "./configs/benchmark_ops/get_last_loc/"): + """Run benchmark and save results""" + + # Ensure save path exists + os.makedirs(save_path, exist_ok=True) + + # Run correctness test + test_get_last_loc() + print("Correctness test passed!") + + # Run performance test + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=save_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/get_last_loc/", + help="Path to save benchmark results", + ) + args = parser.parse_args() + + run_benchmark(args.save_path) diff --git a/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py b/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py new file mode 100644 index 000000000..1ce43c8ba --- /dev/null +++ b/benchmark/kernels/scheduler_batch/benchmark_write_req_to_token_pool_triton.py @@ -0,0 +1,342 @@ +import itertools +import os + +import torch +import triton +import triton.language as tl + + +@triton.jit +def write_req_to_token_pool_triton( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, +): + BLOCK_SIZE: tl.constexpr = 512 + pid = tl.program_id(0) + + req_pool_index = tl.load(req_pool_indices + pid) + pre_len = tl.load(pre_lens + pid) + seq_len = tl.load(seq_lens + pid) + + # TODO: optimize this? + cumsum_start = 0 + for i in range(pid): + cumsum_start += tl.load(extend_lens + i) + + num_loop = tl.cdiv(seq_len - pre_len, BLOCK_SIZE) + for i in range(num_loop): + offset = tl.arange(0, BLOCK_SIZE) + i * BLOCK_SIZE + mask = offset < (seq_len - pre_len) + value = tl.load(out_cache_loc + cumsum_start + offset, mask=mask) + tl.store( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + offset + + pre_len, + value, + mask=mask, + ) + + +@triton.jit +def write_req_to_token_pool_triton_optimize( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + req_to_token_ptr_stride: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid_batch = tl.program_id(0) + pid_token = tl.program_id(1) + + req_pool_index = tl.load(req_pool_indices + pid_batch) + pre_len = tl.load(pre_lens + pid_batch) + seq_len = tl.load(seq_lens + pid_batch) + extend_len = seq_len - pre_len + + cumsum_start = 0 + for i in range(pid_batch): + cumsum_start += tl.load(extend_lens + i) + + token_start = pid_token * BLOCK_SIZE + + offset = tl.arange(0, BLOCK_SIZE) + actual_offset = token_start + offset + mask = actual_offset < extend_len + + src_ptr = out_cache_loc + cumsum_start + actual_offset + src_ptr = tl.max_contiguous(tl.multiple_of(src_ptr, BLOCK_SIZE), BLOCK_SIZE) + value = tl.load(src_ptr, mask=mask) + dst_ptr = ( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + actual_offset + + pre_len + ) + dst_ptr = tl.max_contiguous(tl.multiple_of(dst_ptr, BLOCK_SIZE), BLOCK_SIZE) + + tl.store(dst_ptr, value, mask=mask) + + +def write_req_to_token_pool_reference( + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + pre_lens: torch.Tensor, + seq_lens: torch.Tensor, + extend_lens: torch.Tensor, + out_cache_loc: torch.Tensor, +) -> None: + """Reference implementation using PyTorch""" + for i in range(len(req_pool_indices)): + req_pool_idx = req_pool_indices[i].item() + pre_len = pre_lens[i].item() + seq_len = seq_lens[i].item() + extend_len = extend_lens[i].item() + + cumsum_start = sum(extend_lens[:i].tolist()) + + # Copy values from out_cache_loc to req_to_token + req_to_token[req_pool_idx, pre_len:seq_len] = out_cache_loc[ + cumsum_start : cumsum_start + extend_len + ] + + +def test_write_req_to_token_pool(): + max_batch = 4097 + max_context_len = 6148 + batch_size = 1 + extend_len = 14 + + # Initialize input tensors + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor([extend_len], dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(extend_len, dtype=torch.int32, device="cuda") + + # Create copies for reference implementation + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + def grid(batch_size, extend_len): + num_token_blocks = triton.cdiv(extend_len, 512) + return (batch_size, num_token_blocks) + + write_req_to_token_pool_triton_optimize[grid(batch_size, extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + # Test case 2: batch size > 1 + batch_size = 3 + extend_lens_list = [14, 20, 30] + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.tensor([42, 100, 200], dtype=torch.int32, device="cuda") + pre_lens = torch.tensor([8, 10, 15], dtype=torch.int32, device="cuda") + seq_lens = torch.tensor([22, 30, 45], dtype=torch.int32, device="cuda") + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + req_to_token_ref = req_to_token.clone() + req_to_token_opt = req_to_token.clone() + + # Run original triton kernel + write_req_to_token_pool_triton[(batch_size,)]( + req_to_token, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ) + + # Run optimized triton kernel + max_extend_len = max(extend_lens_list) + write_req_to_token_pool_triton_optimize[grid(batch_size, max_extend_len)]( + req_to_token_opt, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=512, + ) + + # Run reference implementation + write_req_to_token_pool_reference( + req_to_token_ref, + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ) + + # Compare results + torch.testing.assert_close(req_to_token, req_to_token_ref) + torch.testing.assert_close(req_to_token_opt, req_to_token_ref) + + +def get_benchmark(): + batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128] + extend_lens = [32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + configs = list(itertools.product(batch_sizes, extend_lens)) + + @triton.testing.perf_report( + triton.testing.Benchmark( + x_names=["batch_size", "extend_len"], + x_vals=configs, + line_arg="provider", + line_vals=["reference", "triton", "triton_optimize"], + line_names=["PyTorch", "Triton", "Triton Optimized"], + styles=[("blue", "-"), ("green", "-"), ("red", "-")], + ylabel="us", + plot_name="write-req-to-token-pool-performance", + args={}, + ) + ) + def benchmark(batch_size, extend_len, provider): + max_batch = 256 + max_context_len = 16384 + + extend_lens_list = [extend_len] * batch_size + total_extend_len = sum(extend_lens_list) + + req_to_token = torch.zeros( + (max_batch, max_context_len), dtype=torch.int32, device="cuda" + ) + req_pool_indices = torch.arange(batch_size, dtype=torch.int32, device="cuda") + pre_lens = torch.ones(batch_size, dtype=torch.int32, device="cuda") * 8 + seq_lens = pre_lens + extend_len + extend_lens = torch.tensor(extend_lens_list, dtype=torch.int32, device="cuda") + out_cache_loc = torch.arange(total_extend_len, dtype=torch.int32, device="cuda") + + quantiles = [0.5, 0.2, 0.8] + + if provider == "reference": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_reference( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + ), + quantiles=quantiles, + ) + elif provider == "triton": + ms, min_ms, max_ms = triton.testing.do_bench( + lambda: write_req_to_token_pool_triton[(batch_size,)]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + ), + quantiles=quantiles, + ) + else: + + def run_optimized(): + block_size = 128 if extend_len <= 1024 else 512 + grid_config = (batch_size, triton.cdiv(extend_len, block_size)) + write_req_to_token_pool_triton_optimize[grid_config]( + req_to_token.clone(), + req_pool_indices, + pre_lens, + seq_lens, + extend_lens, + out_cache_loc, + max_context_len, + BLOCK_SIZE=block_size, + ) + + ms, min_ms, max_ms = triton.testing.do_bench( + run_optimized, quantiles=quantiles + ) + + return 1000 * ms, 1000 * max_ms, 1000 * min_ms + + return benchmark + + +def run_benchmark(save_path: str = "./configs/benchmark_ops/write_req_to_token_pool/"): + """Run benchmark and save results""" + + # Ensure save path exists + os.makedirs(save_path, exist_ok=True) + + # Run correctness test + test_write_req_to_token_pool() + print("Correctness test passed!") + + # Run performance test + benchmark = get_benchmark() + benchmark.run(print_data=True, save_path=save_path) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--save_path", + type=str, + default="./configs/benchmark_ops/write_req_to_token_pool/", + help="Path to save benchmark results", + ) + args = parser.parse_args() + + run_benchmark(args.save_path) diff --git a/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py b/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py new file mode 100644 index 000000000..98144d470 --- /dev/null +++ b/benchmark/kernels/sliding_window_attention_triton/bench_triton_swa_kernel.py @@ -0,0 +1,283 @@ +import itertools + +import torch +import torch.nn.functional as F +import triton.testing as tt + +from sglang.srt.layers.attention.triton_ops.extend_attention import extend_attention_fwd + + +def extend_attention_fwd_torch( + q: torch.Tensor, # [extend_tokens, H_Q, D] + k: torch.Tensor, # [extend_tokens, H_KV, D] + v: torch.Tensor, # [extend_tokens, H_KV, D] + o: torch.Tensor, # [extend_tokens, H_Q, D] + k_cache: torch.Tensor, # [total_tokens, H_KV, D] + v_cache: torch.Tensor, # [total_tokens, H_KV, D] + qo_indptr: torch.Tensor, # [B+1] + kv_indptr: torch.Tensor, # [B+1] + kv_indices: torch.Tensor, # [prefix_tokens] + sliding_window_size: int, +): + B = qo_indptr.size(0) - 1 + _, H_Q, D = q.shape + _, H_KV, _ = k.shape + + group_size = H_Q // H_KV + scale = 1.0 / D**0.5 + + for i in range(B): + q_start = int(qo_indptr[i].item()) + q_end = int(qo_indptr[i + 1].item()) + kv_start = int(kv_indptr[i].item()) + kv_end = int(kv_indptr[i + 1].item()) + + prefix_indices = kv_indices[kv_start:kv_end] + k_prefix = k_cache[prefix_indices] # [prefix_len, H_KV, D] + v_prefix = v_cache[prefix_indices] # [prefix_len, H_KV, D] + + k_extend = k[q_start:q_end] # [extend_len, H_KV, D] + v_extend = v[q_start:q_end] # [extend_len, H_KV, D] + q_extend = q[q_start:q_end] # [extend_len, H_Q, D] + + k_full = torch.cat([k_prefix, k_extend], dim=0) # [total_len, H_KV, D] + v_full = torch.cat([v_prefix, v_extend], dim=0) # [total_len, H_KV, D] + + if group_size != 1: + k_full_hq = k_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + v_full_hq = v_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + else: + k_full_hq = k_full + v_full_hq = v_full + + prefix_len = k_prefix.size(0) + extend_len = k_extend.size(0) + total_len = prefix_len + extend_len + + # causal + pos_keys = torch.arange(total_len, device=q.device) + t = prefix_len + torch.arange(extend_len, device=q.device) # [extend_len] + causal_mask = pos_keys.unsqueeze(0) <= t.unsqueeze(1) + + # sliding window + if sliding_window_size is not None and sliding_window_size > 0: + start = (t - (sliding_window_size)).clamp_min(0) # [extend_len] + else: + start = torch.zeros_like(t) + window_mask = pos_keys.unsqueeze(0) >= start.unsqueeze(1) + + final_mask = causal_mask & window_mask + + attn_scores = ( + torch.einsum("qhd,khd->qhk", q_extend, k_full_hq) * scale + ) # [extend_len, H_Q, total_len] + attn_scores = attn_scores.masked_fill(~final_mask.unsqueeze(1), float("-inf")) + + attn_weights = F.softmax(attn_scores, dim=-1) + o[q_start:q_end] = torch.einsum("qhk,khd->qhd", attn_weights, v_full_hq) + + +def _build_batch( + B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE, dtype=torch.bfloat16, device="cuda" +): + b_seq_len_prefix = torch.randint( + 1, max(2, N_CTX // 2), (B,), dtype=torch.int32, device=device + ) + b_seq_len_extend = torch.randint( + 1, max(2, N_CTX // 2), (B,), dtype=torch.int32, device=device + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + + b_start_loc = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device=device) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + + kv_indices = torch.zeros( + (int(b_seq_len_prefix.sum().item()),), dtype=torch.int32, device=device + ) + for i in range(B): + s = kv_indptr[i].item() + e = kv_indptr[i + 1].item() + kv_indices[s:e] = torch.arange( + b_start_loc[i], + b_start_loc[i] + b_seq_len_prefix[i], + dtype=torch.int32, + device=device, + ) + + total_token_num = int(torch.sum(b_seq_len).item()) + extend_token_num = int(torch.sum(b_seq_len_extend).item()) + + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device=device) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (int(b_seq_len_extend[i].item()), H_Q, D), dtype=dtype, device=device + ).normal_(mean=0.1, std=0.2) + + o_extend_triton = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device=device + ) + o_extend_torch = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device=device) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = int(torch.max(b_seq_len_extend, 0)[0].item()) + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device=device) + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + inputs = dict( + q_extend=q_extend, + k_extend=k_extend, + v_extend=v_extend, + k_buffer=k_buffer, + v_buffer=v_buffer, + o_extend_triton=o_extend_triton, + o_extend_torch=o_extend_torch, + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, + kv_indices=kv_indices, + max_len_extend=max_len_extend, + WINDOW_SIZE=WINDOW_SIZE, + ) + meta = dict( + B=B, N_CTX=N_CTX, H_Q=H_Q, H_KV=H_KV, D=D, extend_token_num=extend_token_num + ) + return inputs, meta + + +def _run_triton(inputs): + extend_attention_fwd( + inputs["q_extend"], + inputs["k_extend"], + inputs["v_extend"], + inputs["o_extend_triton"], + inputs["k_buffer"], + inputs["v_buffer"], + inputs["qo_indptr"], + inputs["kv_indptr"], + inputs["kv_indices"], + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=inputs["max_len_extend"], + sliding_window_size=inputs["WINDOW_SIZE"], + ) + + +def _run_torch_ref(inputs): + extend_attention_fwd_torch( + inputs["q_extend"], + inputs["k_extend"], + inputs["v_extend"], + inputs["o_extend_torch"], + inputs["k_buffer"], + inputs["v_buffer"], + inputs["qo_indptr"], + inputs["kv_indptr"], + inputs["kv_indices"], + inputs["WINDOW_SIZE"], + ) + + +N_CTXS = [1024, 2048, 4096, 8192] +WINDOW_SIZES = [-1, 127, 256, 512] + +CONFIGS = list(itertools.product(N_CTXS, WINDOW_SIZES)) + +PROVIDERS = ["torch", "triton"] + + +@tt.perf_report( + tt.Benchmark( + x_names=["N_CTX", "WINDOW_SIZE"], + x_vals=CONFIGS, + line_arg="provider", + line_vals=PROVIDERS, + line_names=PROVIDERS, + ylabel="Runtime (ms)", + plot_name="extend_attention_triton_vs_torch", + args={ + "B": 32, + "H_Q": 64, + "H_KV": 8, + "D": 128, + "dtype": "bf16", + "device": "cuda", + "check_correctness": False, + "warmup": 25, + "rep": 100, + }, + ) +) +def bench( + N_CTX, + provider, + B, + H_Q, + H_KV, + D, + dtype, + device, + WINDOW_SIZE, + check_correctness, + warmup, + rep, +): + torch.manual_seed(0) + torch.cuda.manual_seed(0) + dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} + dt = dtype_map[dtype] + + inputs, _ = _build_batch( + B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE, dtype=dt, device=device + ) + + if check_correctness and provider == "triton": + _run_triton(inputs) + _run_torch_ref(inputs) + torch.cuda.synchronize() + if not torch.allclose( + inputs["o_extend_triton"], inputs["o_extend_torch"], rtol=1e-3, atol=1e-3 + ): + raise AssertionError("Mismatch between triton and torch reference.") + + if provider == "triton": + ms = tt.do_bench(lambda: _run_triton(inputs), warmup=warmup, rep=rep) + elif provider == "torch": + ms = tt.do_bench(lambda: _run_torch_ref(inputs), warmup=warmup, rep=rep) + else: + raise ValueError(provider) + + return ms + + +if __name__ == "__main__": + bench.run(print_data=True, show_plots=False) diff --git a/benchmark/line_retrieval/README.md b/benchmark/line_retrieval/README.md new file mode 100644 index 000000000..3c81a63cb --- /dev/null +++ b/benchmark/line_retrieval/README.md @@ -0,0 +1,37 @@ +## Download data + +``` +wget https://raw.githubusercontent.com/merrymercy/merrymercy.github.io/master/files/random_words.json +python3 gen_data.py --number 1000 +``` + +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-hf --port 30000 +``` + +``` +python3 bench_sglang.py --src-index 600 --num-q 50 --parallel 1 +``` + + +### + +``` +# original +Accuracy: 0.940, latency: 332.83 s + +# parallel encoding (no_adjust, offset = 1000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 3000) +Accuracy: 0.760, latency: 238.46 s + +# parallel encoding (no_adjust, offset = 0) +Accuracy: 0.520, latency: 238.46 s + +# parallel encoding (adjust_cache) +Accuracy: 0.460, latency: 257.66 s +``` diff --git a/benchmark/line_retrieval/bench_sglang.py b/benchmark/line_retrieval/bench_sglang.py new file mode 100644 index 000000000..e974e7dd3 --- /dev/null +++ b/benchmark/line_retrieval/bench_sglang.py @@ -0,0 +1,149 @@ +import argparse +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def line_retrieval(s, prefix, suffix, body_0, body_1, body_2, body_3): + s += prefix + "\n" + + contexts = [body_0, body_1, body_2, body_3] + position_ids_offset = [i * 1000 for i in range(len(contexts))] + forks = s.fork(len(contexts), position_ids_offset) + forks += lambda i: contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "\n" + suffix + s += sgl.gen("answer", max_tokens=16) + + +def eval_model(args, line_obj, num_hoops, src_indices, dst_percents): + arguments = [] + labels = [] + sum_src_indices = [] + sum_dst_indices = [] + + for i in range(len(src_indices)): + for j in range(len(dst_percents)): + src_index = src_indices[i] + dst_percent = dst_percents[j] + + query_indices = line_obj["group_by_num_hoops"][str(num_hoops)] + query_indices = [ + q + for q in query_indices + if all(l <= src_index for l in line_obj["links"][q]) and q < src_index + ] + dst_index = query_indices[ + min(int(len(query_indices) * dst_percent), len(query_indices) - 1) + ] + label = line_obj["values"][dst_index] + + body = line_obj["lines"][: src_index + 1] + suffix = line_obj["suffix"].replace("???", line_obj["indices"][dst_index]) + body_part_len = len(body) // 4 + + arguments.append( + { + "prefix": line_obj["prefix"], + "body_0": "\n".join(body[:body_part_len]), + "body_1": "\n".join(body[body_part_len : 2 * body_part_len]), + "body_2": "\n".join(body[2 * body_part_len : 3 * body_part_len]), + "body_3": "\n".join(body[3 * body_part_len :]), + "suffix": suffix, + } + ) + labels.append(label) + sum_src_indices.append(src_index) + sum_dst_indices.append(dst_index) + + # Select backend + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = line_retrieval.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + corrects = [] + for i in range(len(arguments)): + output = states[i]["answer"] + prompt_len = states[i].get_meta_info("answer").get("prompt_length", -1) + label = labels[i] + + # Try all numbers + findall = re.findall("\d+", output) + if not findall: + response_number = output + else: + for response_number in findall: + if response_number == label: + break + + correct = response_number == label + corrects.append(correct) + + # Log results + summary = ( + f"Line index: {sum_src_indices[i]} -> {sum_dst_indices[i]}, " + f"Prompt len: {prompt_len}, " + f"Correct: {correct}, " + f"Label: {label}, Predicted: {response_number}, " + ) + print(summary) + + accuracy = np.mean(corrects) + print(f"Accuracy: {accuracy:.3f}, latency: {latency:.2f} s") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "line_retrieval", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": len(arguments), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +def main(args): + line_obj = json.load(open(args.data_path, "r")) + + num_hoops = args.num_hoops + for src_index in args.src_index: + src_indices = [src_index] + num_queries = args.num_queries_per_src + dst_percents = [i * (1 / (num_queries)) for i in range(num_queries)] + eval_model(args, line_obj, num_hoops, src_indices, dst_percents) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="lines_1000_0.0.json") + parser.add_argument("--src-index", type=int, nargs="+", default=[100]) + parser.add_argument("--num-queries-per-src", type=int, default=10) + parser.add_argument("--num-hoops", type=int, default=1) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/line_retrieval/gen_data.py b/benchmark/line_retrieval/gen_data.py new file mode 100644 index 000000000..a01d40a06 --- /dev/null +++ b/benchmark/line_retrieval/gen_data.py @@ -0,0 +1,139 @@ +""" +Generate line data for line retrieval task. + +Usage: +python3 gen_data.py --number 1000 +""" + +import argparse +import json +from collections import defaultdict + +import numpy as np +from tqdm import tqdm + + +def generate_lines(random_words, num_lines, redirect_ratio): + prefix = "Here is a list of lines, each with its corresponding REGISTER_CONTENT value. Please memorize them. Be prepared to provide the REGISTER_CONTENT value for a specific line index when I ask." + suffix = "The list has ended. Please give the final REGISTER_CONTENT value for a specific line after resolving the redirections and references. For example, the REGISTER_CONTENT of Line __idx0__ is __val0__. The REGISTER_CONTENT of Line __idx1__ is __val1__. The REGISTER_CONTENT of Line __idx2__ is __val2__. The REGISTER_CONTENT of Line ??? is" + + # Raw lines + visited_indices = set([None]) + visited_values = set([None]) + + lines = [] + redirects = [] + indices = [] + values = [] + for i in tqdm(range(num_lines)): + line_index = None + while line_index in visited_indices: + line_index = "-".join(np.random.choice(random_words, size=(2,))) + visited_indices.add(line_index) + + line_value = np.random.randint(low=0, high=999999) + line_value = f"{line_value:06}" + + line = f"Line {line_index}: The REGISTER_CONTENT is {line_value}." + lines.append(line) + redirects.append(None) + indices.append(line_index) + values.append(line_value) + + # Add redirect + if redirect_ratio > 0: + num_redirect_lines = int(len(lines) * redirect_ratio) + redirect_indices = np.random.choice( + np.arange(len(lines)), size=(num_redirect_lines,), replace=False + ) + for i in redirect_indices: + target_idx = np.random.choice(min(i * 2 + 100, num_lines)) + lines[i] = ( + f"Line {indices[i]}: The REGISTER_CONTENT is the same as Line {indices[target_idx]}." + ) + redirects[i] = target_idx + + # Build links and find sources + links = [[] for _ in range(num_lines)] + contains_ring = set() + for i in range(num_lines): + if redirects[i] is None: + continue + + tmp_link = [] + cur = i + visited = set() + while redirects[cur] is not None: + visited.add(cur) + tmp_link.append(redirects[cur]) + cur = redirects[cur] + + if cur in visited: + contains_ring.add(i) + tmp_link = None + break + values[i] = values[cur] + links[i] = tmp_link + + # Group by num_links + group_by_num_hoops = defaultdict(list) + for i in range(num_lines): + if i in contains_ring: + continue + group_by_num_hoops[len(links[i]) + 1].append(i) + + keys = sorted(list(group_by_num_hoops.keys())) + for num_links in keys: + print(f"#links: {num_links}, #lines: {len(group_by_num_hoops[num_links])}") + + # Append few-shot examples + hoop1_candidates = list(group_by_num_hoops[1]) + hoop1_candidate_keys = {c: max([c] + links[c]) for c in hoop1_candidates} + hoop1_candidates.sort(key=lambda c: hoop1_candidate_keys[c]) + hoop2_candidates = list(group_by_num_hoops[2]) + hoop2_candidate_keys = {c: max([c] + links[c]) for c in hoop2_candidates} + hoop2_candidates.sort(key=lambda c: hoop2_candidate_keys[c]) + + i = hoop1_candidates[5] + suffix = suffix.replace("__idx0__", indices[i]).replace("__val0__", values[i]) + if len(hoop2_candidates): + i = hoop2_candidates[0] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop2_candidates[1] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + else: + i = hoop1_candidates[1] + suffix = suffix.replace("__idx1__", indices[i]).replace("__val1__", values[i]) + i = hoop1_candidates[10] + suffix = suffix.replace("__idx2__", indices[i]).replace("__val2__", values[i]) + + obj = { + "prefix": prefix, + "suffix": suffix, + "lines": lines, + "indices": indices, + "values": values, + "links": links, + "group_by_num_hoops": group_by_num_hoops, + "contains_ring": sorted(list(contains_ring)), + } + return obj + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--number", type=int) + parser.add_argument("--redirect-ratio", type=float, default=0.0) + args = parser.parse_args() + + num_lines = args.number + + random_words_filename = "random_words.json" + random_words = json.load(open(random_words_filename, "r")) + + np.random.seed(42) + obj = generate_lines(random_words, num_lines, args.redirect_ratio) + + fout = f"lines_{num_lines}_{args.redirect_ratio:.1f}.json" + with open(fout, "w") as fout: + json.dump(obj, fout, indent=2) diff --git a/benchmark/llava_bench/README.md b/benchmark/llava_bench/README.md new file mode 100644 index 000000000..a71f339df --- /dev/null +++ b/benchmark/llava_bench/README.md @@ -0,0 +1,61 @@ +## Download benchmark images + +``` +python3 download_images.py +``` + +image benchmark source: https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild + +### Other Dependency +``` +pip3 install "sglang[all]" +pip3 install "torch>=2.1.2" "transformers>=4.36" pillow +``` + +## Run benchmark + +### Benchmark sglang +Launch a server +``` +python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.6-vicuna-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 +``` + +Run benchmark +``` +# Run with local models +python3 bench_sglang.py --num-questions 60 + +# Run with OpenAI models +python3 bench_sglang.py --num-questions 60 --backend gpt-4-vision-preview +``` + +### Bench LLaVA original code +``` +git clone git@github.com:haotian-liu/LLaVA.git +cd LLaVA +git reset --hard 9a26bd1435b4ac42c282757f2c16d34226575e96 +pip3 install -e . + +cd ~/sglang/benchmark/llava_bench +CUDA_VISIBLE_DEVICES=0 bash bench_hf_llava_bench.sh +``` + + +### Benchmark llama.cpp + +``` +# Install +CMAKE_ARGS="-DLLAMA_CUBLAS=on" pip install llama-cpp-python +pip install sse_starlette starlette_context pydantic_settings + +# Download weights +mkdir -p ~/model_weights/llava-v1.5-7b/ +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/ggml-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf +wget https://huggingface.co/mys/ggml_llava-v1.5-7b/resolve/main/mmproj-model-f16.gguf -O ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf +``` + +``` +python3 -m llama_cpp.server --model ~/model_weights/llava-v1.5-7b/ggml-model-f16.gguf --clip_model_path ~/model_weights/llava-v1.5-7b/mmproj-model-f16.gguf --chat_format llava-1-5 --port 23000 + +OPENAI_BASE_URL=http://localhost:23000/v1 python3 bench_sglang.py --backend gpt-4-vision-preview --num-q 1 +``` diff --git a/benchmark/llava_bench/bench_hf_llava_bench.sh b/benchmark/llava_bench/bench_hf_llava_bench.sh new file mode 100755 index 000000000..a51a715b6 --- /dev/null +++ b/benchmark/llava_bench/bench_hf_llava_bench.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./questions.jsonl \ + --image-folder ./images \ + --answers-file ./answers_hf.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/benchmark/llava_bench/bench_hf_mme.sh b/benchmark/llava_bench/bench_hf_mme.sh new file mode 100755 index 000000000..6ed332fe0 --- /dev/null +++ b/benchmark/llava_bench/bench_hf_mme.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +python -m llava.eval.model_vqa_loader \ + --model-path liuhaotian/llava-v1.5-7b \ + --question-file ./mme_pack/llava_mme_bench_replace.jsonl \ + --image-folder ./mme_pack/MME_Benchmark_release_version \ + --answers-file ./answers_hf_mme.jsonl \ + --temperature 0 \ + --conv-mode vicuna_v1 diff --git a/benchmark/llava_bench/bench_sglang.py b/benchmark/llava_bench/bench_sglang.py new file mode 100644 index 000000000..b9e8c1405 --- /dev/null +++ b/benchmark/llava_bench/bench_sglang.py @@ -0,0 +1,96 @@ +import argparse +import json +import os +import time + +import tqdm + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def image_qa(s, image_file, question): + s += sgl.user(sgl.image(image_file) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=args.max_tokens)) + + +def main(args): + lines = list(read_jsonl(args.question_file))[: args.num_questions] + arguments = [ + { + "image_file": os.path.abspath(args.image_folder + "/" + l["image"]), + "question": l["text"], + } + for l in lines + ] + # arguments = [ + # {"image_file": + # Image.open(os.path.abspath(args.image_folder + "/" + l["image"])), + # "question": l["text"]} for l in lines + # ] + + states = [None] * len(lines) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm.tqdm(range(len(lines))): + image_file = arguments[i]["image_file"] + question = arguments[i]["question"] + ret = image_qa.run(image_file=image_file, question=question, temperature=0) + states[i] = ret + else: + states = image_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + print(f"Write output to {args.answer_file}") + with open(args.answer_file, "w") as fout: + for i in range(len(lines)): + value = { + "question_id": lines[i]["question_id"], + "prompt": lines[i]["text"], + "text": states[i]["answer"].strip(), + "model_id": backend.model_info["model_path"], + "answer_id": i, + "metadata": {}, + } + fout.write(json.dumps(value) + "\n") + + with open(args.result_file, "a") as fout: + value = { + "task": "llava_bench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(lines), + "parallel": args.parallel, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="questions.jsonl") + parser.add_argument("--answer-file", type=str, default="answers.jsonl") + parser.add_argument("--image-folder", type=str, default="./images") + parser.add_argument("--temperature", type=float, default=0.0) + parser.add_argument("--num-questions", type=int, default=None) + parser.add_argument("--max-tokens", type=int, default=768) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/llava_bench/bench_sglang_mme.sh b/benchmark/llava_bench/bench_sglang_mme.sh new file mode 100644 index 000000000..8c8c09cc0 --- /dev/null +++ b/benchmark/llava_bench/bench_sglang_mme.sh @@ -0,0 +1,2 @@ +MME_FOLDER=./mme_pack +python3 bench_sglang.py --num-questions 5000 --question-file $MME_FOLDER/llava_mme_bench_replace.jsonl --answer-file answer_mme.jsonl --image-folder $MME_FOLDER/MME_Benchmark_release_version --max-tokens 4 diff --git a/benchmark/llava_bench/download_images.py b/benchmark/llava_bench/download_images.py new file mode 100644 index 000000000..701190a03 --- /dev/null +++ b/benchmark/llava_bench/download_images.py @@ -0,0 +1,20 @@ +import os + +# Create the 'images' directory if it doesn't exist +if not os.path.exists("images"): + os.makedirs("images") + +# Base URL +base_url = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/" + +# Loop through image numbers +for i in range(1, 25): + # Format the image number with leading zeros + image_number = str(i).zfill(3) + image_url = base_url + image_number + ".jpg" + image_path = "images/" + image_number + ".jpg" + + # Download the image using wget + os.system(f"wget -O {image_path} {image_url}") + +print("Download complete.") diff --git a/benchmark/llm_judge/README.md b/benchmark/llm_judge/README.md new file mode 100644 index 000000000..164b19a08 --- /dev/null +++ b/benchmark/llm_judge/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 25 --parallel 8 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 25 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 25 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 25 --parallel 1 +``` diff --git a/benchmark/llm_judge/bench_other.py b/benchmark/llm_judge/bench_other.py new file mode 100644 index 000000000..8e6029067 --- /dev/null +++ b/benchmark/llm_judge/bench_other.py @@ -0,0 +1,151 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +def multi_dimension_judge(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += generate(s, max_tokens=2, stop=None) + + return s + + +async def multi_dimension_judge_async(article, generate): + s = system_prompt + s += "\n```\n" + article + "\n```\n\n" + + judges = [] + for i in range(len(dimension_prompts)): + comp = await generate( + s + + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:', + max_tokens=256, + stop="END", + ) + judges.append(comp) + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += dimension_prompts[i].split(":")[0] + ": " + judges[i].strip() + "\n" + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += await generate(s, max_tokens=2, stop=None) + + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.perf_counter() + + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = multi_dimension_judge(lines[i], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + async def get_one_answer_async(i): + states[i] = await multi_dimension_judge_async(lines[i], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in bt]) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/llm_judge/bench_sglang.py b/benchmark/llm_judge/bench_sglang.py new file mode 100644 index 000000000..97e6c3979 --- /dev/null +++ b/benchmark/llm_judge/bench_sglang.py @@ -0,0 +1,97 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +system_prompt = "Please serve as an impartial judge and rigorously evaluate the quality of the following article. Apply the most stringent standards possible, showing no leniency." + +dimension_prompts = [ + "Content: This refers to the essences of the essay. The substance should be well researched, accurate, relevant to the topic and should show a thorough understanding of the subject. The essay should also reflect a clear goal or purpose.", + "Organization and Structure: An essay needs to be properly structured with a clear introduction, body, and conclusion. The essay should flow naturally, with one paragraph leading seamlessly into the next.", + "Argument and Analysis: The argument made in the essay should be logical, coherent and clearly articulated. Each point made should be backed up by solid evidence and thorough analysis.", + "Clarity and Precision: The essay should be written in a clear and concise manner. The points made should be easily understood by the reader. The language used should also be precise and unambiguous.", + "Grammar and Punctuation: Proper use of grammar and punctuation is vital in an academic essay. Errors in grammar and punctuation not only distract the reader but can also negatively impact the meaning and interpretation of the content.", + "Referencing and Citation: An essay should contain proper citations and references for all sources used. This not only prevents accusations of plagiarism but also gives credit to the authors of the works that have contributed to the essay. The citation should adhere to a specific format as required by the academic institution or specified by the professor.", +] + + +@sgl.function +def multi_dimension_judge(s, article): + s += system_prompt + s += "\n```\n" + article + "\n```\n\n" + + forks = s.fork(len(dimension_prompts)) + for i in range(len(dimension_prompts)): + forks[i] += ( + "USER: Please judge the quality based on the following metric. " + + dimension_prompts[i] + + " Please provide a single-paragraph judgement. " + + "Focus on the provided metric and do not say other things. " + 'End your judgement paragraph with the word "END"\nJUDGE:' + ) + forks[i] += sgl.gen("judgement", max_tokens=256, stop="END") + forks.join() + + s += "I will judge the quality based on the following metrics.\n" + for i in range(len(dimension_prompts)): + s += ( + dimension_prompts[i].split(":")[0] + + ": " + + forks[i]["judgement"].strip() + + "\n" + ) + + s += "In summary, on a scale of 1 to 10, I would give the article a score of" + s += sgl.gen("score", max_tokens=2) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"article": l} for l in lines] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = multi_dimension_judge.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "llm_judge", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="articles.jsonl") + parser.add_argument("--num-questions", type=int, default=20) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/README.md b/benchmark/long_json_decode/README.md new file mode 100644 index 000000000..37fceee13 --- /dev/null +++ b/benchmark/long_json_decode/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 5 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 5 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 5 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + +### Build dataset +``` +pip install wikipedia +python3 build_dataset.py +``` diff --git a/benchmark/long_json_decode/bench_other.py b/benchmark/long_json_decode/bench_other.py new file mode 100644 index 000000000..0ad38a014 --- /dev/null +++ b/benchmark/long_json_decode/bench_other.py @@ -0,0 +1,89 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def json_decode(document, generate): + s = "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "air port code": "' + s += generate(s, max_tokens=8, stop='"') + '",\n' + s += ' "top 3 landmarks": "' + s += generate(s, max_tokens=24, stop='"') + '",\n' + s += "}\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = json_decode(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(arguments))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(arguments)))), + total=len(arguments), + ) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/bench_sglang.py b/benchmark/long_json_decode/bench_sglang.py new file mode 100644 index 000000000..8394cfc2e --- /dev/null +++ b/benchmark/long_json_decode/bench_sglang.py @@ -0,0 +1,81 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def json_decode(s, document): + s += "Please extract the information of a city from the following wikipedia page.\n" + s += "Page begin.\n" + document + "Page end.\n" + s += "Here is the name, country, and symbol of the city in JSON format.\n" + s += "{\n" + s += ' "name": "' + sgl.gen("name", max_tokens=8, stop='"') + '",\n' + s += ' "country": "' + sgl.gen("country", max_tokens=8, stop='"') + '",\n' + s += ( + ' "air port code": "' + + sgl.gen("air port code", max_tokens=8, stop='"') + + '",\n' + ) + s += ( + ' "top 3 landmarks": "' + + sgl.gen("landmarks", max_tokens=24, stop='"') + + '",\n' + ) + s += "}\n" + + +def main(args): + lines = read_jsonl(args.data_path) + arguments = [] + for i in range(len(lines[: args.num_questions])): + arguments.append( + { + "document": lines[i]["document"], + } + ) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = json_decode.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "long_json_decode", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/long_json_decode/build_dataset.py b/benchmark/long_json_decode/build_dataset.py new file mode 100644 index 000000000..7b15d3951 --- /dev/null +++ b/benchmark/long_json_decode/build_dataset.py @@ -0,0 +1,27 @@ +import json + +import transformers +import wikipedia + +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +city_names = ["los angles", "london", "tokyo", "beijing", "singapore"] + + +for city_name in city_names: + content = str(wikipedia.page(city_name).content) + content = content.replace("\n\n", "\n") + + tokens = t.encode(content) + + truncate_len = int((10000 / len(tokens)) * len(content)) + truncate_content = content[:truncate_len] + truncate_tokens = t.encode(truncate_content) + + # Count token + print( + f"city_name: {city_name}, #tokens: {len(tokens)}, #truncate tokens: {len(truncate_tokens)}" + ) + + with open("questions.jsonl", "a") as fout: + fout.write(json.dumps({"document": truncate_content}) + "\n") diff --git a/benchmark/lora/launch_server.py b/benchmark/lora/launch_server.py new file mode 100644 index 000000000..b0781ca30 --- /dev/null +++ b/benchmark/lora/launch_server.py @@ -0,0 +1,75 @@ +import argparse +import os + +NUM_LORAS = 4 +LORA_PATH = { + "base": "meta-llama/Llama-2-7b-hf", + "lora": "winddude/wizardLM-LlaMA-LoRA-7B", +} + + +def launch_server(args): + base_path = LORA_PATH["base"] + lora_path = LORA_PATH["lora"] + + if args.base_only: + cmd = f"python3 -m sglang.launch_server --model {base_path} " + else: + cmd = f"python3 -m sglang.launch_server --model {base_path} --lora-paths " + for i in range(NUM_LORAS): + lora_name = f"lora{i}" + cmd += f"{lora_name}={lora_path} " + cmd += f"--disable-radix " + cmd += f"--max-loras-per-batch {args.max_loras_per_batch} " + cmd += f"--max-running-requests {args.max_running_requests} " + cmd += f"--lora-backend {args.lora_backend} " + cmd += f"--tp-size {args.tp_size} " + if args.disable_custom_all_reduce: + cmd += "--disable-custom-all-reduce" + if args.enable_mscclpp: + cmd += "--enable-mscclpp" + print(cmd) + os.system(cmd) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument( + "--max-loras-per-batch", + type=int, + default=8, + ) + parser.add_argument( + "--max-running-requests", + type=int, + default=8, + ) + parser.add_argument( + "--lora-backend", + type=str, + default="triton", + ) + parser.add_argument( + "--tp-size", + type=int, + default=1, + help="Tensor parallel size for distributed inference", + ) + # disable_custom_all_reduce + parser.add_argument( + "--disable-custom-all-reduce", + action="store_true", + help="Disable custom all reduce when device does not support p2p communication", + ) + parser.add_argument( + "--enable-mscclpp", + action="store_true", + help="Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.", + ) + args = parser.parse_args() + + launch_server(args) diff --git a/benchmark/lora/lora_bench.py b/benchmark/lora/lora_bench.py new file mode 100644 index 000000000..0a1e37a5c --- /dev/null +++ b/benchmark/lora/lora_bench.py @@ -0,0 +1,476 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import asyncio +import json +import random +import resource +import sys +import time +import traceback +from argparse import ArgumentParser +from datetime import datetime +from typing import Any, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +from launch_server import LORA_PATH, NUM_LORAS +from tqdm.asyncio import tqdm +from transformers import PreTrainedTokenizerBase + +from sglang.bench_serving import ( + AIOHTTP_TIMEOUT, + RequestFuncInput, + RequestFuncOutput, + calculate_metrics, + get_request, + get_tokenizer, + remove_prefix, + sample_random_requests, +) + +global args + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + # assert api_url.endswith( + # "completions" + # ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + # payload = { + # "model": request_func_input.model, + # "prompt": prompt, + # "temperature": 0.0, + # "best_of": 1, + # "max_tokens": request_func_input.output_len, + # "stream": not args.disable_stream, + # "ignore_eos": not args.disable_ignore_eos, + # **request_func_input.extra_request_body, + # } + # headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"} + if args.base_only: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + } + else: + payload = { + "text": prompt, + "sampling_params": {"max_new_tokens": request_func_input.output_len}, + "lora_path": f"lora{random.randint(0, NUM_LORAS - 1)}", + } + headers = {"Authorization": ""} + + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["text"]: + # if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + # generated_text += data["choices"][0]["text"] + generated_text += data["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_openai_completions, +} + + +async def benchmark( + backend: str, + api_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[Tuple[str, int, int]], + request_rate: float, + disable_tqdm: bool, + extra_request_body: Dict[str, Any], +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + print("Starting initial single prompt test run...") + test_request = input_requests[0] + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=test_request.output_len, + lora_name="dummy", # the lora_name argument will not be used + image_data=None, + extra_request_body=extra_request_body, + ) + test_output = await request_func(request_func_input=test_input) + if not test_output.success: + raise ValueError( + "Initial test run failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {test_output.error}" + ) + else: + print("Initial test run completed. Starting main benchmark run...") + + pbar = None if disable_tqdm else tqdm(total=len(input_requests)) + + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + async for request in get_request(input_requests, request_rate): + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name="dummy", + image_data=None, + extra_request_body=extra_request_body, + ) + tasks.append( + asyncio.create_task( + request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + if pbar is not None: + pbar.close() + + benchmark_duration = time.perf_counter() - benchmark_start_time + + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print("{:<40} {:<10}".format("Traffic request rate:", request_rate)) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format("Total throughput (tok/s):", metrics.total_throughput) + ) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print( + "{s:{c}^{n}}".format(s="Time per Output Token (excl. 1st token)", n=50, c="-") + ) + print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms)) + print("{:<40} {:<10.2f}".format("Median TPOT (ms):", metrics.median_tpot_ms)) + print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms)) + print("{s:{c}^{n}}".format(s="Inter-token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + "backend": args.backend, + "request_rate": request_rate, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "median_itl_ms": metrics.median_itl_ms, + "output_throughput": metrics.output_throughput, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + "duration": benchmark_duration, + "completed": metrics.completed, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + file.write(json.dumps(result) + "\n") + + result = { + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + } + return result + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + }.get(args.backend, 30000) + + # api_url = ( + # f"{args.base_url}/v1/completions" + # if args.base_url + # else f"http://{args.host}:{args.port}/v1/completions" + # ) + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model = LORA_PATH["base"] + tokenizer_id = args.model + + tokenizer = get_tokenizer(tokenizer_id) + + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path="", + ) + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + disable_tqdm=False, + extra_request_body={}, + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online lora serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=50, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=128, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--base-only", + action="store_true", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + args = parser.parse_args() + run_benchmark(args) diff --git a/benchmark/mmlu/README.md b/benchmark/mmlu/README.md new file mode 100644 index 000000000..16de20cda --- /dev/null +++ b/benchmark/mmlu/README.md @@ -0,0 +1,59 @@ +## Download data +``` +bash download_data.sh +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --nsub 10 +``` + +``` +# OpenAI models +python3 bench_sglang.py --backend gpt-3.5-turbo --parallel 8 +``` + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --nsub 10 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 + +# V100 +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 4500 --port 22000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --nsub 10 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + + +### Benchmark lmql +``` +CUDA_VISIBLE_DEVICES=0,1 lmql serve-model meta-llama/Llama-2-7b-chat-hf --cuda --port 23000 +``` + +``` +python3 bench_other.py --nsub 10 --backend lmql --parallel 2 +``` diff --git a/benchmark/mmlu/bench_other.py b/benchmark/mmlu/bench_other.py new file mode 100644 index 000000000..f1b166c2b --- /dev/null +++ b/benchmark/mmlu/bench_other.py @@ -0,0 +1,173 @@ +import argparse +import asyncio +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +import pandas as pd +import tiktoken +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def evaluate(args, subject, dev_df, test_df, call_generate): + prompts = [] + labels = [] + + # Construct prompts + k = args.ntrain + train_prompt = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(train_prompt)) > 1536: + k -= 1 + train_prompt = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + prompt = train_prompt + prompt_end + prompts.append(prompt) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + preds = [None] * len(prompts) + max_tokens = 1 + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + pred = call_generate(prompts[i], temperature=0, max_tokens=max_tokens) + preds[i] = pred.strip()[0] + + tic = time.perf_counter() + if args.parallel == 1: + for i in range(len(prompts)): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + executor.map(get_one_answer, list(range(len(prompts)))) + else: + # Use asyncio + async def batched_call(batch_size): + for i in range(0, len(prompts), batch_size): + tasks = [] + for p in prompts[i : i + batch_size]: + tasks.append(call_generate(p, temperature=0, max_tokens=max_tokens)) + rets = await asyncio.gather(*tasks) + for j in range(len(rets)): + preds[i + j] = rets[j].strip()[0] + + tic = time.perf_counter() + asyncio.run(batched_call(batch_size=args.parallel)) + latency = time.perf_counter() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + acc = np.mean(cors) + cors = np.array(cors) + + print( + "Average accuracy {:.3f}, latency {:.2f}, #q: {} - {}".format( + acc, latency, len(prompts), subject + ) + ) + + return cors, acc, latency + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + all_cors = [] + all_latencies = [] + num_requests = 0 + + # Select backend + call_generate = get_call_generate(args) + + for subject in tqdm(subjects[: args.nsub]): + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + + cors, acc, latency = evaluate(args, subject, dev_df, test_df, call_generate) + all_cors.append(cors) + all_latencies.append(latency) + num_requests += len(test_df) + + total_latency = np.sum(all_latencies) + print("Total latency: {:.3f}".format(total_latency)) + + weighted_acc = np.mean(np.concatenate(all_cors)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(total_latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": num_requests, + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", type=int, default=5) + parser.add_argument("--data_dir", type=str, default="data") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/mmlu/bench_sglang.py b/benchmark/mmlu/bench_sglang.py new file mode 100644 index 000000000..23057be4a --- /dev/null +++ b/benchmark/mmlu/bench_sglang.py @@ -0,0 +1,181 @@ +import argparse +import json +import os +import time + +import numpy as np +import pandas as pd +import tiktoken + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + dump_bench_raw_result, + select_sglang_backend, +) + +choices = ["A", "B", "C", "D"] + +tokenizer = tiktoken.encoding_for_model("gpt-3.5-turbo") + + +def format_subject(subject): + l = subject.split("_") + s = "" + for entry in l: + s += " " + entry + return s + + +def format_example(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 + for j in range(k): + prompt += "\n{}. {}".format(choices[j], df.iloc[idx, j + 1]) + prompt += "\nAnswer:" + if include_answer: + prompt += " {}\n\n".format(df.iloc[idx, k + 1]) + return prompt + + +def gen_prompt(train_df, subject, k=-1): + prompt = "The following are multiple choice questions (with answers) about{}.\n\n".format( + format_subject(subject) + ) + if k == -1: + k = train_df.shape[0] + for i in range(k): + prompt += format_example(train_df, i) + return prompt + + +def main(args): + subjects = sorted( + [ + f.split("_test.csv")[0] + for f in os.listdir(os.path.join(args.data_dir, "test")) + if "_test.csv" in f + ] + ) + + # Build prompts + arguments = [] + labels = [] + num_questions = [] + + for subject in subjects[: args.nsub]: + dev_df = pd.read_csv( + os.path.join(args.data_dir, "dev", subject + "_dev.csv"), header=None + )[: args.ntrain] + test_df = pd.read_csv( + os.path.join(args.data_dir, "test", subject + "_test.csv"), header=None + ) + num_questions.append(test_df.shape[0]) + + k = args.ntrain + few_shot_examples = gen_prompt(dev_df, subject, k) + while len(tokenizer.encode(few_shot_examples)) > 1536: + k -= 1 + few_shot_examples = gen_prompt(dev_df, subject, k) + + for i in range(test_df.shape[0]): + prompt_end = format_example(test_df, i, include_answer=False) + + arguments.append( + { + "examples": few_shot_examples, + "question": prompt_end, + } + ) + + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + if args.backend.startswith("gpt-"): + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += sgl.user(examples + question) + s += sgl.assistant(sgl.gen("answer")) + + else: + + @sgl.function + def few_shot_mmlu(s, examples, question): + s += examples + question + sgl.gen("answer") + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run + tic = time.perf_counter() + states = few_shot_mmlu.run_batch( + arguments, + temperature=0, + max_new_tokens=1, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + preds = [ + s["answer"].strip()[0] if len(s["answer"].strip()) > 0 else "" for s in states + ] + latency = time.perf_counter() - tic + + # Compute accuracy + cors = [pred == label for pred, label in zip(preds, labels)] + + pt = 0 + for subject, num_qs in zip(subjects[: args.nsub], num_questions): + print( + f"subject: {subject}, #q:{num_qs}, acc: {np.mean(cors[pt: pt + num_qs]):.3f}" + ) + pt += num_qs + assert pt == len(cors) + weighted_acc = np.mean(cors) + + dump_bench_raw_result( + path=args.raw_result_file, + states=states, + preds=preds, + labels=labels, + ) + + # Print results + print("Total latency: {:.3f}".format(latency)) + print("Average accuracy: {:.3f}".format(weighted_acc)) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "mmlu", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(weighted_acc, 3), + "num_requests": len(arguments), + "other": { + "nsub": args.nsub, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--ntrain", "-k", type=int, default=5) + parser.add_argument("--data_dir", "-d", type=str, default="data") + parser.add_argument("--save_dir", "-s", type=str, default="results") + parser.add_argument("--nsub", type=int, default=60) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/mmlu/download_data.sh b/benchmark/mmlu/download_data.sh new file mode 100644 index 000000000..fb224165e --- /dev/null +++ b/benchmark/mmlu/download_data.sh @@ -0,0 +1,2 @@ +wget https://people.eecs.berkeley.edu/~hendrycks/data.tar +tar xf data.tar diff --git a/benchmark/mmmu/README.md b/benchmark/mmmu/README.md new file mode 100644 index 000000000..80db21921 --- /dev/null +++ b/benchmark/mmmu/README.md @@ -0,0 +1,46 @@ +## Run evaluation + +### Evaluate sglang + +Host the VLM: + +``` +python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --port 30000 +``` + +It's recommended to reduce the memory usage by appending something like `--mem-fraction-static 0.6` to the command above. + +Benchmark: + +``` +python benchmark/mmmu/bench_sglang.py --port 30000 --concurrency 16 +``` + +You can adjust the `--concurrency` to control the number of concurrent OpenAI calls. + +You can use `--lora-path` to specify the LoRA adapter to apply during benchmarking. E.g., +``` +# Launch server with LoRA enabled +python -m sglang.launch_server --model-path microsoft/Phi-4-multimodal-instruct --port 30000 --trust-remote-code --disable-radix-cache --lora-paths vision= + +# Apply LoRA adapter during inferencing +python -m benchmark/mmmu/bench_sglang.py --concurrency 8 --lora-path vision +``` + +You can use `--response-answer-regex` to specify how to extract the answer from the response string. E.g., +``` +python3 -m sglang.launch_server --model-path zai-org/GLM-4.1V-9B-Thinking --reasoning-parser glm45 + +python3 bench_sglang.py --response-answer-regex "<\|begin_of_box\|>(.*)<\|end_of_box\|>" --concurrency 64 +``` + +You can use `--extra-request-body` to specify additional OpenAI request parameters. E.g., +``` +python3 bench_sglang.py --extra-request-body '{"max_new_tokens": 128, "temperature": 0.01}' +``` + +### Evaluate hf + +``` +python benchmark/mmmu/bench_hf.py --model-path Qwen/Qwen2-VL-7B-Instruct +``` diff --git a/benchmark/mmmu/bench_hf.py b/benchmark/mmmu/bench_hf.py new file mode 100644 index 000000000..949b63b80 --- /dev/null +++ b/benchmark/mmmu/bench_hf.py @@ -0,0 +1,164 @@ +import argparse + +import PIL +import torch +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm +from transformers import AutoModel, AutoProcessor, GenerationConfig + + +@torch.no_grad() +def eval_mmmu(args): + eval_args = EvalArgs.from_cli_args(args) + + sampling_params = get_sampling_params(eval_args) + generation_config = GenerationConfig( + max_new_tokens=sampling_params["max_new_tokens"], + do_sample=False, + ) + + try: + from transformers import AutoModelForImageTextToText + + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + ) + except Exception as first_exception: + try: + # check if the model is belongs to internvl + if "InternVL" in args.model_path: + from internvl_utils import load_image + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(args.model_path) + model = AutoModel.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + ) + generation_config_internvl = dict( + max_new_tokens=sampling_params["max_new_tokens"], do_sample=False + ) + + else: + model = AutoModel.from_pretrained( + args.model_path, + torch_dtype="auto", + trust_remote_code=True, + init_tts=False, + ) + except Exception as second_exception: + raise RuntimeError( + f"Failed to load model: First attempt failed with {first_exception}, " + f"second attempt failed with {second_exception}" + ) from second_exception + + model = model.eval().cuda() + + processor = AutoProcessor.from_pretrained( + args.model_path, torch_dtype="auto", device_map="auto", trust_remote_code=True + ) + + samples = prepare_samples(eval_args) + out_samples = dict() + + answer_dict = {} + for sample in tqdm(samples): + prompt = sample["final_input_prompt"] + image = sample["image"] + prefix = prompt.split("<")[0] + suffix = prompt.split(">")[1] + assert image is not None + + if "InternVL" in args.model_path: + pixel_values = load_image(sample["image_path"]).to(torch.bfloat16).cuda() + contents = "" + if prefix: + contents += prefix + contents += "\n" + if suffix: + contents += suffix + response = model.chat( + tokenizer, pixel_values, contents, generation_config_internvl + ) + print(f"response: {response}") + process_result(response, sample, answer_dict, out_samples) + continue + + contents = [] + if prefix: + contents += [{"type": "text", "text": prefix}] + contents += [ + { + "type": "image", + "image": sample["image_path"], + } + ] + if suffix: + contents += [{"type": "text", "text": suffix}] + messages = [{"role": "user", "content": contents}] + try: + model_inputs = processor.tokenizer.apply_chat_template( + messages, + tokenize=True, + return_dict=True, + add_generation_prompt=True, + return_tensors="pt", + ).to(model.device) + input_len = model_inputs["input_ids"].shape[-1] + generation = model.generate( + **model_inputs, generation_config=generation_config + ) + generation = generation[0][input_len:] + response = processor.decode(generation, skip_special_tokens=True) + except: + contents = [] + if prefix: + contents += [prefix] + image = PIL.Image.open(sample["image_path"]) + contents += [image] + if suffix: + contents += [suffix] + messages = [{"role": "user", "content": contents}] + response = model.chat( + msgs=messages, + tokenizer=processor.tokenizer, + sampling=False, + max_new_tokens=sampling_params["max_new_tokens"], + use_tts_template=False, + generate_audio=False, + temperature=0.0, + ) + print(f"response: {response}") + process_result(response, sample, answer_dict, out_samples) + + args.output_path = f"{args.model_path}_answer_hf.json" + save_json(args.output_path, out_samples) + eval_result( + model_answer_path=args.output_path, + answer_dict=answer_dict, + eval_output_path=f"{args.model_path}_val_hf.json", + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", + required=True, + ) + EvalArgs.add_cli_args(parser) + args = parser.parse_args() + + eval_mmmu(args) diff --git a/benchmark/mmmu/bench_sglang.py b/benchmark/mmmu/bench_sglang.py new file mode 100644 index 000000000..d8834ea5f --- /dev/null +++ b/benchmark/mmmu/bench_sglang.py @@ -0,0 +1,212 @@ +""" +Bench the sglang-hosted vLM with benchmark MMMU + +Usage: + Host the VLM: python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --port 30000 + + Benchmark: python benchmark/mmmu/bench_sglang.py --port 30000 --concurrency 16 + +The eval output will be logged +""" + +import argparse +import asyncio +import re +import sys +import time +import traceback +from dataclasses import dataclass, field +from typing import Any, List, Optional, Tuple + +import aiohttp +import openai +from data_utils import save_json +from eval_utils import ( + EvalArgs, + eval_result, + get_sampling_params, + prepare_samples, + process_result, +) +from tqdm import tqdm + +from sglang.test.test_utils import add_common_sglang_args_and_parse + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=20 * 60 * 60) + + +@dataclass +class RequestFuncOutput: + generated_text: List[str] = field(default_factory=list) + prompt_len: List[int] = field(default_factory=list) + output_len: List[int] = field(default_factory=list) + latency: List[float] = field(default_factory=list) + ttft: List[float] = field(default_factory=list) + itl: List[float] = field(default_factory=list) # List of inter-token latencies + + success: bool = False + error: str = "" + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def _get_prefix_suffix(prompt: str) -> Tuple[str, str]: + """Split the prompt into prefix and suffix.""" + prefix = prompt.split("<")[0] + suffix = prompt.split(">", 1)[1] + return prefix, suffix + + +async def process_sample( + client: Any, sample: dict, sampling_params: dict, lora_path: Optional[str] = None +) -> Tuple[dict, str]: + """Send a single sample to the LLM and return (sample, response).""" + prompt = sample["final_input_prompt"] + prefix, suffix = _get_prefix_suffix(prompt) + image = sample["image"] + assert image is not None + image_path = sample["image_path"] + extra_body = None if lora_path is None else {"lora_path": lora_path} + response = await client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prefix}, + {"type": "image_url", "image_url": {"url": image_path}}, + {"type": "text", "text": suffix}, + ], + } + ], + temperature=0, + max_completion_tokens=sampling_params["max_new_tokens"], + max_tokens=sampling_params["max_new_tokens"], + extra_body=extra_body, + ) + return sample, response.choices[0].message.content + + +async def process_sample_with_semaphore( + semaphore: asyncio.Semaphore, + client: Any, + sample: dict, + sampling_params: dict, + lora_path: Optional[str] = None, +) -> Tuple[dict, str]: + """Wrap process_sample with a semaphore for concurrency control.""" + async with semaphore: + return await process_sample(client, sample, sampling_params, lora_path) + + +async def eval_mmmu(args) -> None: + """Main evaluation loop with concurrency control.""" + eval_args = EvalArgs.from_cli_args(args) + sampling_params = get_sampling_params(eval_args) + samples = prepare_samples(eval_args) + lora_path = eval_args.lora_path + answer_dict = {} + out_samples = {} + client = openai.AsyncOpenAI( + api_key="sk", base_url=f"http://127.0.0.1:{args.port}/v1" + ) + start = time.perf_counter() + base_url = f"http://127.0.0.1:{args.port}" + + if args.profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=f"{base_url}/start_profile" + ) + if profile_output.success: + print("Profiler started") + + samples = samples[: args.profile_number] + + if args.concurrency == 1: + # For concurrency == 1, run in sequential mode to ensure consistent order + # this is mainly for profiling + for sample in tqdm(samples): + _, response = await process_sample( + client, sample, sampling_params, lora_path + ) + answer = ( + re.search(args.response_answer_regex, response) + if response is not None + else None + ) + process_result( + answer.group(1) if answer else response, + sample, + answer_dict, + out_samples, + ) + else: + semaphore = asyncio.Semaphore(args.concurrency) + tasks = [ + process_sample_with_semaphore( + semaphore, client, sample, sampling_params, lora_path + ) + for sample in samples + ] + + for coro in tqdm(asyncio.as_completed(tasks), total=len(tasks)): + sample, response = await coro + answer = ( + re.search(args.response_answer_regex, response) + if response is not None + else None + ) + process_result( + answer.group(1) if answer else response, + sample, + answer_dict, + out_samples, + ) + + if args.profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=f"{base_url}/stop_profile") + if profile_output.success: + print("Profiler stopped") + + print(f"Benchmark time: {time.perf_counter() - start}") + args.output_path = "./answer_sglang.json" + save_json(args.output_path, out_samples) + eval_result( + model_answer_path=args.output_path, + answer_dict=answer_dict, + eval_output_path="./val_sglang.json", + ) + + +def parse_args(): + parser = argparse.ArgumentParser() + EvalArgs.add_cli_args(parser) + args = add_common_sglang_args_and_parse(parser) + return args + + +def main(): + args = parse_args() + asyncio.run(eval_mmmu(args)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/mmmu/data_utils.py b/benchmark/mmmu/data_utils.py new file mode 100644 index 000000000..cf8916934 --- /dev/null +++ b/benchmark/mmmu/data_utils.py @@ -0,0 +1,221 @@ +"""Utils for data load, save, and process (e.g., prompt construction)""" + +import json +import os +import re + +import yaml + +DOMAIN_CAT2SUB_CAT = { + "Art and Design": ["Art", "Art_Theory", "Design", "Music"], + "Business": ["Accounting", "Economics", "Finance", "Manage", "Marketing"], + "Science": [ + "Biology", + "Chemistry", + "Geography", + "Math", + "Physics", + ], + "Health and Medicine": [ + "Basic_Medical_Science", + "Clinical_Medicine", + "Diagnostics_and_Laboratory_Medicine", + "Pharmacy", + "Public_Health", + ], + "Humanities and Social Science": [ + "History", + "Literature", + "Sociology", + "Psychology", + ], + "Tech and Engineering": [ + "Agriculture", + "Architecture_and_Engineering", + "Computer_Science", + "Electronics", + "Energy_and_Power", + "Materials", + "Mechanical_Engineering", + ], +} + + +CAT_SHORT2LONG = { + "acc": "Accounting", + "agri": "Agriculture", + "arch": "Architecture_and_Engineering", + "art": "Art", + "art_theory": "Art_Theory", + "bas_med": "Basic_Medical_Science", + "bio": "Biology", + "chem": "Chemistry", + "cli_med": "Clinical_Medicine", + "cs": "Computer_Science", + "design": "Design", + "diag_med": "Diagnostics_and_Laboratory_Medicine", + "econ": "Economics", + "elec": "Electronics", + "ep": "Energy_and_Power", + "fin": "Finance", + "geo": "Geography", + "his": "History", + "liter": "Literature", + "manage": "Manage", + "mark": "Marketing", + "mate": "Materials", + "math": "Math", + "mech": "Mechanical_Engineering", + "music": "Music", + "phar": "Pharmacy", + "phys": "Physics", + "psy": "Psychology", + "pub_health": "Public_Health", + "socio": "Sociology", +} + + +# DATA SAVING +def save_json(filename, ds): + with open(filename, "w") as f: + json.dump(ds, f, indent=4) + + +def get_multi_choice_info(options): + """ + Given the list of options for multiple choice question + Return the index2ans and all_choices + """ + + start_chr = "A" + all_choices = [] + index2ans = {} + for i, option in enumerate(options): + index2ans[chr(ord(start_chr) + i)] = option + all_choices.append(chr(ord(start_chr) + i)) + + return index2ans, all_choices + + +def load_yaml(file_path): + with open(file_path, "r") as stream: + try: + yaml_dict = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + return yaml_dict + + +def parse_img_path(text): + matches = re.findall("", text) + return matches + + +def process_single_sample(data): + question = data["question"] + o_imgs_paths = [] + for option in data["options"]: + current_o_imgs_paths = parse_img_path(option) + for img_path in current_o_imgs_paths: + o_imgs_paths.append(img_path) + + if len(o_imgs_paths) > 1: # multiple images in options, used for random selection + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": None, + "question_type": data["question_type"], + } + else: + return { + "id": data["id"], + "question": question, + "options": data["options"], + "answer": data["answer"], + "image": data["image_1"], + "question_type": data["question_type"], + } + + +# DATA SAVING +def save_json(filename, ds): + print(f"answers saved to: {filename}") + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "w") as f: + json.dump(ds, f, indent=4) + + +def save_jsonl(filename, data): + """ + Save a dictionary of data to a JSON Lines file with the filename as key and caption as value. + + Args: + filename (str): The path to the file where the data should be saved. + data (dict): The dictionary containing the data to save where key is the image path and value is the caption. + """ + with open(filename, "w", encoding="utf-8") as f: + for img_path, caption in data.items(): + # Extract the base filename without the extension + base_filename = os.path.basename(img_path) + # Create a JSON object with the filename as the key and caption as the value + json_record = json.dumps({base_filename: caption}, ensure_ascii=False) + # Write the JSON object to the file, one per line + f.write(json_record + "\n") + + +def save_args(args, path_dir): + argsDict = args.__dict__ + with open(path_dir + "setting.txt", "w") as f: + f.writelines("------------------ start ------------------" + "\n") + for eachArg, value in argsDict.items(): + f.writelines(eachArg + " : " + str(value) + "\n") + f.writelines("------------------- end -------------------") + + +# DATA PROCESSING +def construct_prompt(sample, config): + question = sample["question"] + options = eval(sample["options"]) + example = "" + if sample["question_type"] == "multiple-choice": + start_chr = "A" + prediction_range = [] + index2ans = {} + for option in options: + prediction_range.append(start_chr) + example += f"({start_chr}) {option}\n" + index2ans[start_chr] = option + start_chr = chr(ord(start_chr) + 1) + empty_prompt_sample_structure = config["multi_choice_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question, example) + res_dict = {} + res_dict["index2ans"] = index2ans + res_dict["correct_choice"] = sample["answer"] + res_dict["all_choices"] = prediction_range + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + + res_dict["gt_content"] = options[ord(sample["answer"].upper()) - ord("A")] + else: + empty_prompt_sample_structure = config["short_ans_example_format"] + empty_prompt = empty_prompt_sample_structure.format(question) + res_dict = {} + res_dict["empty_prompt"] = empty_prompt + if config["task_instructions"]: + res_dict["final_input_prompt"] = ( + config["task_instructions"].strip() + "\n\n" + empty_prompt + ) + else: + res_dict["final_input_prompt"] = empty_prompt + res_dict["gt_content"] = sample["answer"] + + res_dict.update(sample) + return res_dict diff --git a/benchmark/mmmu/eval_utils.py b/benchmark/mmmu/eval_utils.py new file mode 100644 index 000000000..ca0e87c6a --- /dev/null +++ b/benchmark/mmmu/eval_utils.py @@ -0,0 +1,649 @@ +"""Response Parsing and Evaluation for various models""" + +import argparse +import dataclasses +import json +import os +import pprint +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, Optional + +import numpy as np +import torch +from data_utils import ( + CAT_SHORT2LONG, + DOMAIN_CAT2SUB_CAT, + construct_prompt, + load_yaml, + process_single_sample, +) +from datasets import concatenate_datasets, load_dataset +from tqdm import tqdm + + +@dataclasses.dataclass +class EvalArgs: + seed: int = 42 + split: str = "validation" + image_pixels_limit: int = -1 + result_filename: str = "" + prompt_format_file: str = "prompt_format.yaml" + dataset_path: str = "MMMU/MMMU" + extra_request_body: Optional[str] = None + profile: bool = False + profile_number: int = 5 + concurrency: int = 1 + response_answer_regex: str = "(.*)" + lora_path: Optional[str] = None + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument( + "--result-filename", + type=str, + default=EvalArgs.result_filename, + help="The filename to save the evaluation results.", + ) + parser.add_argument( + "--image-pixels-limit", + type=int, + default=EvalArgs.image_pixels_limit, + help="The maximum number of pixels allowed for an image. If an image exceeds this limit, it will be skipped during evaluation.", + ) + parser.add_argument( + "--dataset-path", + type=str, + default=EvalArgs.dataset_path, + help="path to the dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--prompt-format-file", + type=str, + help="The path to the prompt format of mmmu. If not, a default format llava_config.yaml will be used", + ) + parser.add_argument( + "--split", + type=str, + default=EvalArgs.split, + help='Split of the dataset to use for evaluation. Default is "validation".', + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=EvalArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--profile", action="store_true", help="enable mmmu profile" + ) + parser.add_argument( + "--profile-number", + type=int, + default=EvalArgs.profile_number, + help="Number of samples to profile. If not set, will profile all samples.", + ) + parser.add_argument( + "--concurrency", + type=int, + default=EvalArgs.concurrency, + help="Number of concurrent requests to make during evaluation. Default is 1, which means no concurrency.", + ) + parser.add_argument( + "--response-answer-regex", + type=str, + default=EvalArgs.response_answer_regex, + help="Specific regex to capture the answer from the response, string", + ) + parser.add_argument( + "--lora-path", + type=str, + default=EvalArgs.lora_path, + help="Specify the LoRA path to use for evaluation. If specified, the value will be specified in the body of every request as `lora-path`.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def set_seed(seed_value): + """ + Set the seed for PyTorch (both CPU and CUDA), Python, and NumPy for reproducible results. + + :param seed_value: An integer value to be used as the seed. + """ + torch.manual_seed(seed_value) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed_value) + torch.cuda.manual_seed_all(seed_value) # For multi-GPU setups + random.seed(seed_value) + np.random.seed(seed_value) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def prepare_samples(eval_args: EvalArgs): + print("Preparing samples...") + # Build prompts + set_seed(eval_args.seed) + + prompt_format_file = ( + eval_args.prompt_format_file + if eval_args.prompt_format_file is not None + else os.path.join(os.path.dirname(__file__), "prompt_format.yaml") + ) + # load config and process to one value + eval_args.config = load_yaml(prompt_format_file) + for key, value in eval_args.config.items(): + if key != "eval_params" and type(value) == list: + assert len(value) == 1, "key {} has more than one value".format(key) + eval_args.config[key] = value[0] + + # run for each subject in parallel + sub_dataset_list = [] + subjects = list(CAT_SHORT2LONG.values()) # Get a fixed list of subjects + + print(f"Loading datasets for {len(subjects)} subjects...") + with ThreadPoolExecutor() as executor: + # Submit all load_dataset tasks + future_to_subject = { + executor.submit( + load_dataset, eval_args.dataset_path, subject, split=eval_args.split + ): subject + for subject in subjects + } + + # Collect results as they complete + results = {} + for future in tqdm( + as_completed(future_to_subject), + total=len(subjects), + desc="Loading datasets", + ): + subject = future_to_subject[future] + try: + results[subject] = future.result() + except Exception as exc: + print(f"{subject} generated an exception: {exc}") + + # Ensure datasets are added in the original order for consistency + for subject in subjects: + if subject in results: + sub_dataset_list.append(results[subject]) + else: + # Handle cases where a dataset failed to load (optional, depends on desired behavior) + print(f"Warning: Dataset for subject '{subject}' could not be loaded.") + + # merge all dataset + dataset = concatenate_datasets(sub_dataset_list) + + # Prepare images in parallel + images_path = os.path.expanduser("~/.cache/mmmu/images") + os.makedirs(images_path, exist_ok=True) + print(f"Saving images to: {images_path}") + + samples = [] + skip_count = 0 + + def process_sample(i, sample): + sample = process_single_sample(sample) + sample = construct_prompt(sample, eval_args.config) + image = sample["image"] + width, height = image.size + if 0 < eval_args.image_pixels_limit <= width * height: + return None, True + # Use a unique identifier for the image path to avoid potential collisions if indices reset + image_path = f"{images_path}/image_{sample['id']}.png" + if not os.path.exists(image_path): + image.save(image_path) + sample["image_path"] = image_path + return sample, False + + print("Processing samples...") + with ThreadPoolExecutor() as executor: + # Pass the sample itself to process_sample, index is less reliable now + futures = [ + executor.submit( + process_sample, i, sample + ) # Keep index i for tqdm maybe? Or remove it. Let's keep it for now. + for i, sample in enumerate(dataset) + ] + for future in tqdm( + as_completed(futures), total=len(dataset), desc="Processing samples" + ): + sample, skipped = future.result() + if skipped: + skip_count += 1 + elif sample: + samples.append(sample) + + samples.sort(key=lambda x: x["final_input_prompt"]) + + print( + f"Skipping {skip_count} samples with large images, {round((float(skip_count) / len(dataset)) * 100, 2)}% of dataset" + ) + print("Samples have been prepared") + return samples + + +def get_sampling_params(eval_args): + max_new_tokens = 30 + temperature = 0.001 + + extra_request_body = {} + if eval_args.extra_request_body: + extra_request_body = json.loads(eval_args.extra_request_body) + + return { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + **extra_request_body, + } + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct + + +# ----------- Batch Evaluation ------------- +def evaluate(samples): + """ + Batch evaluation for multiple choice and open questions. + """ + pred_correct = 0 + judge_dict = dict() + for sample in samples: + gold_i = sample["answer"] + pred_i = sample["parsed_pred"] + if sample["question_type"] == "multiple-choice": + correct = eval_multi_choice(gold_i, pred_i) + else: # open question + correct = eval_open(gold_i, pred_i) + + if correct: + judge_dict[sample["id"]] = "Correct" + pred_correct += 1 + else: + # print(f"Wrong! expected {pred_i}, answered with {gold_i}") + judge_dict[sample["id"]] = "Wrong" + + if len(samples) == 0: + return {"acc": 0} + return judge_dict, {"acc": pred_correct / len(samples)} + + +# ----------- Calculate Accuracy ------------- +def calculate_ins_level_acc(results: Dict): + """Calculate the instruction level accuracy for given Subject results""" + acc = 0 + ins_num = 0 + for cat_results in results.values(): + acc += cat_results["acc"] * cat_results["num_example"] + ins_num += cat_results["num_example"] + if ins_num == 0: + return 0 + return acc / ins_num + + +def process_result(response, sample, answer_dict, out_samples): + if response is None: + return + if sample["question_type"] == "multiple-choice": + pred_ans = parse_multi_choice_response( + response, sample["all_choices"], sample["index2ans"] + ) + else: # open question + pred_ans = response + + out_samples[sample["id"]] = pred_ans + + # set ground truth answer + answer_dict[sample["id"]] = { + "question_type": sample["question_type"], + "ground_truth": sample["answer"], + } + + +def eval_result(model_answer_path, answer_dict, eval_output_path=None): + if eval_output_path is None: + eval_output_path = model_answer_path + print("Evaluating...") + output_dict = json.load(open(model_answer_path)) + # answer_dict = json.load(open(answer_path)) + + # group by category + output_dict_w_cat = {} + for data_id, parsed_pred in output_dict.items(): + category = "_".join(data_id.split("_")[1:-1]) + if category not in output_dict_w_cat: + output_dict_w_cat.update({category: {}}) + output_dict_w_cat[category].update({data_id: parsed_pred}) + + # group by category + answer_dict_w_cat = {} + for data_id, parsed_pred in answer_dict.items(): + category = "_".join(data_id.split("_")[1:-1]) + if category not in answer_dict_w_cat: + answer_dict_w_cat.update({category: {}}) + answer_dict_w_cat[category].update({data_id: parsed_pred}) + + evaluation_result = {} + + for category in CAT_SHORT2LONG.values(): + # print("Evaluating: {}".format(category)) + # get cat_outputs and cat_answers + try: + cat_outputs = output_dict_w_cat[category] + cat_answers = answer_dict_w_cat[category] + except KeyError: + # print("Skipping {} for not found".format(category)) + continue + + exampels_to_eval = [] + for data_id, parsed_pred in cat_outputs.items(): + question_type = cat_answers[data_id]["question_type"] + if question_type != "multiple-choice": + parsed_pred = parse_open_response( + parsed_pred + ) # mainly for type consistency (make it number, etc.) + else: + parsed_pred = parsed_pred + + exampels_to_eval.append( + { + "id": data_id, + "question_type": question_type, + "answer": cat_answers[data_id]["ground_truth"], + "parsed_pred": parsed_pred, + } + ) + + judge_dict, metric_dict = evaluate(exampels_to_eval) + metric_dict.update({"num_example": len(exampels_to_eval)}) + + evaluation_result[category] = metric_dict + + printable_results = {} + # pdb.set_trace() + # add domain Subject + for domain, in_domain_cats in DOMAIN_CAT2SUB_CAT.items(): + in_domain_cat_results = {} + for cat_name in in_domain_cats: # use the order in DOMAIN_CAT2SUB_CAT + if cat_name in evaluation_result.keys(): + in_domain_cat_results[cat_name] = evaluation_result[cat_name] + else: + pass + in_domain_ins_acc = calculate_ins_level_acc(in_domain_cat_results) + in_domain_data_num = sum( + [ + cat_results["num_example"] + for cat_results in in_domain_cat_results.values() + ] + ) + printable_results["Overall-" + domain] = { + "num": int(in_domain_data_num), + "acc": round(in_domain_ins_acc, 3), + } + # add sub category + for cat_name, cat_results in in_domain_cat_results.items(): + printable_results[cat_name] = { + "num": int(cat_results["num_example"]), + "acc": round(cat_results["acc"], 3), + } + + # table.append(["-----------------------------", "-----", "----"]) + all_ins_acc = calculate_ins_level_acc(evaluation_result) + overall_acc = round(all_ins_acc, 3) + printable_results["Overall"] = { + "num": sum( + [cat_results["num_example"] for cat_results in evaluation_result.values()] + ), + "acc": overall_acc, + } + pprint.pprint(printable_results) + out = eval_output_path + with open(out, "w", encoding="utf-8") as outfile: + json.dump(printable_results, outfile) + print(f"eval out saved to {out}") + + print(f"Overall accuracy: {overall_acc}") diff --git a/benchmark/mmmu/internvl_utils.py b/benchmark/mmmu/internvl_utils.py new file mode 100644 index 000000000..44c62c99a --- /dev/null +++ b/benchmark/mmmu/internvl_utils.py @@ -0,0 +1,94 @@ +# copy from https://huggingface.co/OpenGVLab/InternVL3-1B +import torch +import torchvision.transforms as T +from PIL import Image +from torchvision.transforms.functional import InterpolationMode + +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +def build_transform(input_size): + MEAN, STD = IMAGENET_MEAN, IMAGENET_STD + transform = T.Compose( + [ + T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), + T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=MEAN, std=STD), + ] + ) + return transform + + +def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): + best_ratio_diff = float("inf") + best_ratio = (1, 1) + area = width * height + for ratio in target_ratios: + target_aspect_ratio = ratio[0] / ratio[1] + ratio_diff = abs(aspect_ratio - target_aspect_ratio) + if ratio_diff < best_ratio_diff: + best_ratio_diff = ratio_diff + best_ratio = ratio + elif ratio_diff == best_ratio_diff: + if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: + best_ratio = ratio + return best_ratio + + +def dynamic_preprocess( + image, min_num=1, max_num=12, image_size=448, use_thumbnail=False +): + orig_width, orig_height = image.size + aspect_ratio = orig_width / orig_height + + # calculate the existing image aspect ratio + target_ratios = set( + (i, j) + for n in range(min_num, max_num + 1) + for i in range(1, n + 1) + for j in range(1, n + 1) + if i * j <= max_num and i * j >= min_num + ) + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) + + # find the closest aspect ratio to the target + target_aspect_ratio = find_closest_aspect_ratio( + aspect_ratio, target_ratios, orig_width, orig_height, image_size + ) + + # calculate the target width and height + target_width = image_size * target_aspect_ratio[0] + target_height = image_size * target_aspect_ratio[1] + blocks = target_aspect_ratio[0] * target_aspect_ratio[1] + + # resize the image + resized_img = image.resize((target_width, target_height)) + processed_images = [] + for i in range(blocks): + box = ( + (i % (target_width // image_size)) * image_size, + (i // (target_width // image_size)) * image_size, + ((i % (target_width // image_size)) + 1) * image_size, + ((i // (target_width // image_size)) + 1) * image_size, + ) + # split the image + split_img = resized_img.crop(box) + processed_images.append(split_img) + assert len(processed_images) == blocks + if use_thumbnail and len(processed_images) != 1: + thumbnail_img = image.resize((image_size, image_size)) + processed_images.append(thumbnail_img) + return processed_images + + +def load_image(image_file, input_size=448, max_num=12): + image = Image.open(image_file).convert("RGB") + transform = build_transform(input_size=input_size) + images = dynamic_preprocess( + image, image_size=input_size, use_thumbnail=True, max_num=max_num + ) + pixel_values = [transform(image) for image in images] + pixel_values = torch.stack(pixel_values) + return pixel_values diff --git a/benchmark/mmmu/prompt_format.yaml b/benchmark/mmmu/prompt_format.yaml new file mode 100644 index 000000000..1a0f7211f --- /dev/null +++ b/benchmark/mmmu/prompt_format.yaml @@ -0,0 +1,15 @@ +task_instructions: +- "" +multi_choice_example_format: +- "{} + +{} + +Answer with the option's letter from the given choices directly." + +short_ans_example_format: +- "{} + +Answer the question using a single word or phrase." +temperature: +- 0 diff --git a/benchmark/mtbench/README.md b/benchmark/mtbench/README.md new file mode 100644 index 000000000..fc37caee9 --- /dev/null +++ b/benchmark/mtbench/README.md @@ -0,0 +1,48 @@ +## Download Dataset + +```sh +wget -O question.jsonl https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/llm_judge/data/mt_bench/question.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 80 +``` + +### Benchmark sglang EAGLE +``` +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algo EAGLE \ + --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \ + --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --dtype float16 --port 30000 +``` + +``` +python3 bench_sglang_eagle.py --num-questions 80 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 80 --backend lightllm +``` diff --git a/benchmark/mtbench/bench_other.py b/benchmark/mtbench/bench_other.py new file mode 100644 index 000000000..5e579e9a6 --- /dev/null +++ b/benchmark/mtbench/bench_other.py @@ -0,0 +1,111 @@ +import argparse +import json +import os +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from fastchat.model import get_conversation_template +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +def main(args): + questions = load_questions(args.question_file) + questions = (questions * 10)[: args.num_questions] + max_tokens = 256 + model_id = "llama-2-chat" + + conv_main = get_conversation_template(model_id) + + # Select backend + call_generate = get_call_generate(args) + + answers = [None] * len(questions) + + def get_answer(i): + conv = conv_main.copy() + cur_answers = [] + for j in range(2): + q = questions[i]["turns"][j] + conv.append_message(conv.roles[0], q) + conv.append_message(conv.roles[1], None) + + prompt = conv.get_prompt() + output = call_generate(prompt, temperature=0, max_tokens=max_tokens).strip() + + cur_answers.append(output) + conv.update_last_message(output) + + answers[i] = cur_answers + + # Run requests + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.perf_counter() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/mtbench/bench_sglang.py b/benchmark/mtbench/bench_sglang.py new file mode 100644 index 000000000..0d0545b3a --- /dev/null +++ b/benchmark/mtbench/bench_sglang.py @@ -0,0 +1,99 @@ +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system() + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=256, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + latency = time.perf_counter() - tic + + print(f"#questions: {len(questions)}, Latency: {latency:.2f}") + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/mtbench/bench_sglang_eagle.py b/benchmark/mtbench/bench_sglang_eagle.py new file mode 100644 index 000000000..3eb6036c7 --- /dev/null +++ b/benchmark/mtbench/bench_sglang_eagle.py @@ -0,0 +1,137 @@ +""" +Adapted from https://github.com/chromecast56/sglang/blob/6f145d2eadb93a116134f703358ce76f15381045/benchmark/mtbench/bench_sglang.py + +Benchmark SGLang EAGLE/EAGLE3 Speculative Decoding + +Usage: +python3 benchmark/mtbench/bench_sglang_eagle.py --num-questions 80 --parallel 1 +""" + +import argparse +import json +import os +import time +import uuid + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + + +def load_questions(filename): + questions = [] + with open(filename, "r") as fin: + for line in fin: + obj = json.loads(line) + questions.append(obj) + return questions + + +def write_answers(filename, model_id, questions, answers): + with open(os.path.expanduser(filename), "w") as fout: + for i in range(len(answers)): + ans_json = { + "question_id": questions[i]["question_id"], + "answer_id": uuid.uuid4().hex, + "model_id": model_id, + "choices": { + "index": 0, + "turns": [answers[i][0], answers[i][1]], + }, + "tstamp": time.time(), + } + fout.write(json.dumps(ans_json) + "\n") + + +@sgl.function +def answer_mt_bench(s, question_1, question_2): + s += sgl.system( + "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information." + ) + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def main(args): + # Construct prompts + questions = load_questions(args.question_file)[: args.num_questions] + arguments = [ + {"question_1": q["turns"][0], "question_2": q["turns"][1]} for q in questions + ] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + rets = answer_mt_bench.run_batch( + arguments, + temperature=0, + max_new_tokens=2048, + num_threads=args.parallel, + progress_bar=True, + ) + answers = [[s["answer_1"], s["answer_2"]] for s in rets] + + latency = time.perf_counter() - tic + num_output_tokens = sum( + s.get_meta_info("answer_1")["completion_tokens"] + + s.get_meta_info("answer_2")["completion_tokens"] + for s in rets + ) + + # NOTE: acceptance length is just completion_tokens / spec_verify_ct + # {'id': '3bb9c5ead109488d8ed5ee9cbecaec29', 'finish_reason': {'type': 'length', 'length': 256}, 'prompt_tokens': 37, 'spec_verify_ct': 101, 'completion_tokens': 256, 'cached_tokens': 0} + + output_throughput = num_output_tokens / latency + + has_verify = "spec_verify_ct" in rets[0].get_meta_info("answer_1") + if has_verify: + num_verify_tokens = sum( + s.get_meta_info("answer_1")["spec_verify_ct"] + + s.get_meta_info("answer_2")["spec_verify_ct"] + for s in rets + ) + + accept_length = num_output_tokens / num_verify_tokens + else: + accept_length = 1.0 + + print( + f"#questions: {len(questions)}, Throughput: {output_throughput:.2f} token/s, Acceptance length: {accept_length:.2f}" + ) + + # Write results + model_id = backend.model_info["model_path"] + answer_file = args.answer_file or f"tmp_output_{args.backend}.txt" + write_answers(answer_file, model_id, questions, answers) + + with open(args.result_file, "a") as fout: + value = { + "task": "mtbench", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "throughput": round(output_throughput, 3), + "accept_length": round(accept_length, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--question-file", type=str, default="question.jsonl") + parser.add_argument("--answer-file", type=str, default=None) + parser.add_argument("--num-questions", type=int, default=80) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_chain_reasoning/README.md b/benchmark/multi_chain_reasoning/README.md new file mode 100644 index 000000000..4c9f740f5 --- /dev/null +++ b/benchmark/multi_chain_reasoning/README.md @@ -0,0 +1,49 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --schedule-conservativeness 1.3 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 64 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 64 --backend lmql --parallel 1 +``` diff --git a/benchmark/multi_chain_reasoning/bench_other.py b/benchmark/multi_chain_reasoning/bench_other.py new file mode 100644 index 000000000..f361496ad --- /dev/null +++ b/benchmark/multi_chain_reasoning/bench_other.py @@ -0,0 +1,186 @@ +import argparse +import ast +import asyncio +import json +import re +import time +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def multi_chain_gsm8k(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +async def multi_chain_gsm8k_async(question, num_chains, call_generate): + s = "Question: " + question + "\n" + # s += call_generate(s + "Answer: " + prompt_lib[0], max_tokens=256, + # stop="Question", temperature=0) + # return s + + comps = [] + for i in range(num_chains): + comps.append( + await call_generate( + s + "Answer: " + prompt_lib[i % num_chains], + max_tokens=256, + temperature=0.3, + stop="Question", + ) + ) + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + comps[i].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += await call_generate(s, max_tokens=16, temperature=0, stop=None) + return s + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + k = args.num_shot + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + + states = [None] * len(labels) + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + if args.backend != "lmql": + # Use thread pool + def get_one_answer(i): + answer = multi_chain_gsm8k(questions[i], args.num_chains, call_generate) + states[i] = answer + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + # Use asyncio + async def get_one_answer_asyncio(i): + answer = await multi_chain_gsm8k_async( + questions[i], args.num_chains, call_generate + ) + states[i] = answer + + tic = time.perf_counter() + loop = asyncio.get_event_loop() + batches = [ + list(range(i, min(i + args.parallel, len(questions)))) + for i in range(0, len(questions), args.parallel) + ] + for bt in tqdm(batches): + tasks = [get_one_answer_asyncio(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_chain_reasoning/bench_sglang.py b/benchmark/multi_chain_reasoning/bench_sglang.py new file mode 100644 index 000000000..1d3129db2 --- /dev/null +++ b/benchmark/multi_chain_reasoning/bench_sglang.py @@ -0,0 +1,140 @@ +import argparse +import ast +import json +import re +import time + +import numpy as np + +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +prompt_lib = [ + "Let us think step by step.", + "Approach this methodically. Let's dissect the problem into smaller, more manageable parts.", + "It's important to proceed step by step, ensuring accuracy at each stage.", + "Take a deep breath and break this down.", + "A little bit of arithmetic and a logical approach will help us quickly arrive at the solution to this problem.", + "I am extremely good at math.", +] + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + # k = args.num_shot + # few_shot_examples = get_few_shot_examples(lines, k) + + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q} for q in questions] + + num_chains = args.num_chains + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def multi_chain_gsm8k(s, question): + s += "Question: " + question + "\n" + # s += "Answer: " + prompt_lib[0] + sgl.gen("answer", max_tokens=256, stop="Question", + # temperature=0) + # return + + forks = s.fork(num_chains) + for i in range(num_chains): + forks[i] += ( + "Answer: " + + prompt_lib[i % num_chains] + + sgl.gen("chain", max_tokens=256, temperature=0.3, stop="Question") + ) + forks.join() + + s += "Answer: To answer this question, here are some possible solutions. " + s += "After considering all of them, I will do a majority vote.\n\n" + for i in range(num_chains): + s += f"Solution {i+1}: " + forks[i]["chain"].strip() + "\n\n" + s += "\nBy considering the above solutions and doing a majority vote, I think the final answer (a single integer number) is " + s += sgl.gen("answer", max_tokens=16) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = multi_chain_gsm8k.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + preds = [] + for i in range(len(states)): + preds.append(get_answer_value(states[i]["answer"])) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_chain_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--num-shot", type=int, default=0) + parser.add_argument("--num-chains", type=int, default=5) + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=50) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/README.md b/benchmark/multi_document_qa/README.md new file mode 100644 index 000000000..09f49c78f --- /dev/null +++ b/benchmark/multi_document_qa/README.md @@ -0,0 +1,47 @@ +## Run benchmark + +### Benchmark sglang +``` +python3 -m sglang.launch_server --model-path codellama/CodeLlama-7b-instruct-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model codellama/CodeLlama-7b-instruct-hf --disable-log-requests --port 21000 --gpu 0.97 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 11000 --model-path path/to/code-llama/gguf +``` + + + +### Build dataset + +``` +pip install PyPDF2 +python3 build_dataset.py +``` + +```python +import PyPDF2 + +with open('llama2.pdf', 'rb') as file: + reader = PyPDF2.PdfReader(file) + text = '' + for page_num in range(len(reader.pages)): + text += reader.pages[page_num].extract_text() + with open('output.txt', 'w') as text_file: + text_file.write(text) +``` diff --git a/benchmark/multi_document_qa/bench_other.py b/benchmark/multi_document_qa/bench_other.py new file mode 100644 index 000000000..627837c5c --- /dev/null +++ b/benchmark/multi_document_qa/bench_other.py @@ -0,0 +1,114 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + + +def multi_document_qa(docs, question, generate): + s = USER_PREFIX + s += "Please answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + s += "".join(docs) + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += USER_SUFFIX + s += ASSISTANT_PREFIX + answer = generate(s, max_tokens=16, stop=None) + return answer + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + + num_docs = 10 + if args.backend == "guidance": + num_docs = 7 # due to OOM + + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:num_docs], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + states = [None] * len(arguments) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + def get_one_answer(i): + states[i] = multi_document_qa(generate=call_generate, **arguments[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(labels))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(labels)))), + total=len(labels), + ) + ) + + latency = time.perf_counter() - tic + + # Compute accuracy + print(states) + correct = 0 + for s, label in zip(states, labels): + answer = s.lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/bench_sglang.py b/benchmark/multi_document_qa/bench_sglang.py new file mode 100644 index 000000000..0b4b0dbc6 --- /dev/null +++ b/benchmark/multi_document_qa/bench_sglang.py @@ -0,0 +1,93 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def multi_document_qa(s, docs, question): + s += sgl.user_begin() + s += "Please answer a question according to given documents.\n" + s += "Question:" + question + "Documents begin.\n" + + forks = s.fork(len(docs)) + forks += lambda i: docs[i] + forks.join("concate_and_append") + + s += "\nDocuments end." + s += ( + "\n\nBased on the above documents, please answer this question:\n" + + question + + "\nAnswer in three words or fewer." + ) + s += sgl.user_end() + s += sgl.assistant(sgl.gen("answer", max_tokens=16)) + + +def main(args): + lines = read_jsonl(args.data_path) + l = lines[0] + arguments = [] + labels = [] + for i in range(len(l["questions"][: args.num_questions])): + arguments.append( + { + "docs": l["documents"][:10], + "question": l["questions"][i], + } + ) + labels.append(l["answers"][i]) + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + # Run requests + tic = time.perf_counter() + states = multi_document_qa.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print([s["answer"] for s in states]) + correct = 0 + for s, label in zip(states, labels): + answer = s["answer"].lower() + if all(x in answer for x in label.lower().split(" ")): + correct += 1 + accuracy = correct / len(labels) + print(f"Accuracy: {accuracy:.3f}") + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_document_qa", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "accuracy": accuracy, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="questions.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/multi_document_qa/build_dataset.py b/benchmark/multi_document_qa/build_dataset.py new file mode 100644 index 000000000..27df9474f --- /dev/null +++ b/benchmark/multi_document_qa/build_dataset.py @@ -0,0 +1,70 @@ +import json + +import transformers + +content = "\n".join( + open("llama2.txt", "r", encoding="utf-8", errors="ignore").readlines() +) +content = content.replace("\n\n", "\n") + +# Count token +name = "meta-llama/Llama-2-7b-chat-hf" +t = transformers.AutoTokenizer.from_pretrained(name) +print(f"num tokens: {len(t.encode(content))}") + +# Segment +SEP = "\n\n" +parts = content.split(SEP) +print(f"num segments: {len(parts)}") + +segment_len = 1100 + +segments = [] +tmp = [] +tmp_len = 0 +for i in range(len(parts)): + tmp.append(parts[i]) + tmp_len += len(t.encode(parts[i])) + + if tmp_len > segment_len: + segments.append(SEP.join(tmp)) + tmp = [] + tmp_len = 0 + +for i, s in enumerate(segments): + print(i, len(t.encode(segments[i]))) + +# Dump +with open("questions.jsonl", "w") as fout: + fout.write( + json.dumps( + { + "documents": segments[:30], + "questions": [ + "What is the name of the fine-tuned LLMs?", + "Which figure shows the helpfulness human evaluation results for Llama 2-Chat?", + "What is the number of parameters in the largest Llama 2 model?", + "What is the batch size of fine-tuning?", + "Where can we find the details of potential data contamination?", + "What is the full name of MPT?", + "What is the power consumption of RSC in Watt?", + "How many tokens of data do they train on?", + "Which model's release is delayed due to a lack of time to sufficiently red team?", + "Which activation function is used in Llama?", + ], + "answers": [ + "Llama 2 Chat", + "1", + "70 B", + "64", + "A 6", + "MosaicML", + "400", + "2 trillion", + "34 B", + "SwiGLU", + ], + } + ) + + "\n" + ) diff --git a/benchmark/multi_turn_chat/README.md b/benchmark/multi_turn_chat/README.md new file mode 100644 index 000000000..0fb5b21fa --- /dev/null +++ b/benchmark/multi_turn_chat/README.md @@ -0,0 +1,66 @@ +### Benchmark sglang + +Run Llama-7B + +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +Run Mixtral-8x7B +(When there is a CUDA out-of-memory error, try to reduce the `--mem-fraction-static`) + +``` +python3 -m sglang.launch_server --model-path mistralai/Mixtral-8x7B-Instruct-v0.1 --port 30000 --tp-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf +``` + +Benchmark(long output) + +``` +python3 bench_sglang.py --tokenizer meta-llama/Llama-2-7b-chat-hf --long +``` + +### Benchmark vLLM + +Run Llama-7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +Run Mixtral-8x7B + +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model mistralai/Mixtral-8x7B-Instruct-v0.1 --disable-log-requests --port 21000 --tensor-parallel-size 8 +``` + +Benchmark(short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm +``` + +Benchmark(long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend vllm --long +``` + +### Benchmark guidance + +Benchmark Llama-7B (short output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +Benchmark Llama-7B (long output) + +``` +python3 bench_other.py --tokenizer meta-llama/Llama-2-7b-chat-hf --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf --long +``` diff --git a/benchmark/multi_turn_chat/bench_other.py b/benchmark/multi_turn_chat/bench_other.py new file mode 100644 index 000000000..9189af5be --- /dev/null +++ b/benchmark/multi_turn_chat/bench_other.py @@ -0,0 +1,93 @@ +import json +import time +from argparse import ArgumentParser +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from data_gen import gen_arguments +from tqdm import tqdm +from vllm.transformers_utils.tokenizer import get_tokenizer + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text + + +def multi_turns(generate, qas): + s = "" + for qa in qas: + s += qa["prompt"] + s += generate(s, max_tokens=qa["new_tokens"]) + + return s + + +def main(args): + print(args) + + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + states = [None] * args.num_qa + + call_generate = partial(get_call_generate(args), temperature=0) + + def get_one_answer(i): + states[i] = multi_turns(generate=call_generate, **multi_qas[i]) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(multi_qas))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + rets = list( + tqdm( + executor.map(get_one_answer, list(range(len(multi_qas)))), + total=len(multi_qas), + ) + ) + for _ in rets: + pass + + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_other_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + main(args) diff --git a/benchmark/multi_turn_chat/bench_sglang.py b/benchmark/multi_turn_chat/bench_sglang.py new file mode 100644 index 000000000..1051bf19e --- /dev/null +++ b/benchmark/multi_turn_chat/bench_sglang.py @@ -0,0 +1,79 @@ +import json +import time +from argparse import ArgumentParser + +from data_gen import gen_arguments +from vllm.transformers_utils.tokenizer import get_tokenizer + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def multi_turns(s, qas): + for qa in qas: + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_chat", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + "output_mode": "long" if args.long else "short", + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=4) + parser.add_argument("--num-qa", type=int, default=20) + parser.add_argument("--min-len-q", type=int, default=256) + parser.add_argument("--max-len-q", type=int, default=512) + parser.add_argument("--min-len-a", type=int, default=4) + parser.add_argument("--max-len-a", type=int, default=8) + parser.add_argument("--tokenizer", type=str, required=True) + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--long", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + if args.long: + args.min_len_a = 256 + args.max_len_a = 512 + args.num_qa = 20 + + print(args) + main(args) diff --git a/benchmark/multi_turn_chat/data_gen.py b/benchmark/multi_turn_chat/data_gen.py new file mode 100644 index 000000000..043c07a76 --- /dev/null +++ b/benchmark/multi_turn_chat/data_gen.py @@ -0,0 +1,29 @@ +import random +import string + +random.seed(42) + + +def gen_prompt(tokenizer, token_num): + cha_set = string.ascii_letters + string.digits + ret = "".join(random.choices(cha_set, k=token_num)) + while len(tokenizer(ret).input_ids) < token_num: + ret += random.choice(cha_set) + return ret + + +def gen_arguments(args, tokenizer): + multi_qas = [{"qas": []} for _ in range(args.num_qa)] + for i in range(args.num_qa): + qas = multi_qas[i]["qas"] + for _ in range(args.turns): + prompt_len = random.randint(args.min_len_q, args.max_len_q) + new_tokens = random.randint(args.min_len_a, args.max_len_a) + qas.append( + { + "prompt": gen_prompt(tokenizer, prompt_len), + "new_tokens": new_tokens, + } + ) + + return multi_qas diff --git a/benchmark/multi_turn_chat/long_prompt_multi_turn.py b/benchmark/multi_turn_chat/long_prompt_multi_turn.py new file mode 100644 index 000000000..bda5bb9cc --- /dev/null +++ b/benchmark/multi_turn_chat/long_prompt_multi_turn.py @@ -0,0 +1,129 @@ +import json +import random +import time +from argparse import ArgumentParser +from pathlib import Path + +from tqdm import tqdm + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +def gen_prompt(tokenizer, token_num): + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + ret = tokenizer.decode(selected_tokens) + return ret + + +def get_cache_path(args): + # Create cache directory under ~/.cache/sglang + cache_dir = Path.home() / ".cache" / "sglang" + + # Create a unique cache filename based on the arguments that affect generation + cache_key = f"qa_{args.num_qa}_{args.turns}_{args.system_prompt_len}_{args.len_q}_{args.len_a}_{args.tokenizer.replace('/', '_')}.json" + return cache_dir / cache_key + + +def gen_arguments(args, tokenizer): + cache_path = get_cache_path(args) + + # Try to load from cache first + if cache_path.exists(): + print(f"Loading cached arguments from {cache_path}") + with open(cache_path, "r") as f: + return json.load(f) + + print("Generating new arguments...") + # First progress bar for system prompts + multi_qas = [] + for _ in tqdm(range(args.num_qa), desc="Generating system prompts"): + multi_qas.append( + {"system_prompt": gen_prompt(tokenizer, args.system_prompt_len), "qas": []} + ) + + # Nested progress bars for QA pairs + for i in tqdm(range(args.num_qa), desc="Generating QA pairs"): + qas = multi_qas[i]["qas"] + for j in range(args.turns): + qas.append( + { + "prompt": gen_prompt(tokenizer, args.len_q), + "new_tokens": args.len_a, + } + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + with open(cache_path, "w") as f: + json.dump(multi_qas, f) + print(f"Cached arguments saved to {cache_path}") + + return multi_qas + + +@sgl.function +def multi_turns(s, system_prompt, qas): + s += system_prompt + + for i, qa in enumerate(qas): + s += qa["prompt"] + s += sgl.gen(max_tokens=qa["new_tokens"], ignore_eos=True) + + +def main(args): + tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + multi_qas = gen_arguments(args, tokenizer) + + backend = select_sglang_backend(args) + + tic = time.perf_counter() + states = multi_turns.run_batch( + multi_qas, + temperature=0, + backend=backend, + num_threads="auto", + progress_bar=True, + ) + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "multi_turn_system_prompt_chat", + "backend": args.backend, + "latency": round(latency, 3), + "num_requests": args.num_qa, + "num_turns": args.turns, + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = ArgumentParser() + parser.add_argument("--turns", type=int, default=8) + parser.add_argument("--num-qa", type=int, default=128) + parser.add_argument("--system-prompt-len", type=int, default=2048) + parser.add_argument("--len-q", type=int, default=32) + parser.add_argument("--len-a", type=int, default=128) + parser.add_argument( + "--tokenizer", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + parser.add_argument("--trust-remote-code", action="store_true") + args = add_common_sglang_args_and_parse(parser) + + print(args) + main(args) diff --git a/benchmark/prefill_only/bench_embeddings.py b/benchmark/prefill_only/bench_embeddings.py new file mode 100644 index 000000000..ca66c85a3 --- /dev/null +++ b/benchmark/prefill_only/bench_embeddings.py @@ -0,0 +1,148 @@ +""" +SGLang Embeddings Benchmark Script + +This script benchmarks SGLang's /v1/embeddings API performance using HTTP requests. + +Features: +- HTTP-only implementation +- Uses /v1/embeddings API endpoint directly +- Configurable RPS, duration, and batch sizes +- Progress tracking and detailed metrics +- Poisson and constant request distributions + +Usage: +- Update configuration variables at the top of the file +- Ensure SGLang server is running on the configured HTTP_URL +- Run: python bench_embeddings.py +""" + +import asyncio +import logging + +from transformers import AutoTokenizer +from util import ( + BenchmarkConfig, + generate_text_with_token_count, + run_benchmark_main, + run_generic_benchmark, +) + +# Configure logging +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + +############################################################################### +# CONFIG +############################################################################### +# Create benchmark configuration +config = BenchmarkConfig() +config.rps_values = [500] +config.duration_secs_values = [60] +config.num_unique_requests = 100 +config.distribution = "POISSON" +config.profile = False +config.freeze_gc = True # Enable GC freeze functionality +# Profiler output directory - by default uses present working directory (pwd) +# Uncomment and customize the line below to override the default location: +# config.profiler_dir = "/sglang-oss-trace" + +# HTTP Configuration +HTTP_URL = "http://localhost:30000/v1/embeddings" + +# Embeddings API Config +EMBEDDINGS_MODEL_PATH = "/Qwen/Qwen3-Embedding-0.6B" +BATCH_SIZE = [1] # Number of items per request (batch size) + +# Configurable input token length +EMBEDDINGS_INPUT_TOKENS = 500 # Default token length + +# Load tokenizer once for embeddings text generation +print("Loading tokenizer for embeddings input generation...") +embeddings_tokenizer = AutoTokenizer.from_pretrained(EMBEDDINGS_MODEL_PATH) + +# Generate input text with the specified token length using pre-loaded tokenizer +EMBEDDINGS_INPUT_TEXT = generate_text_with_token_count( + EMBEDDINGS_MODEL_PATH, + EMBEDDINGS_INPUT_TOKENS, + config.special_replicated_token, + tokenizer=embeddings_tokenizer, +) + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def build_embeddings_request(index: int, item_count: int) -> tuple: + """Build a single embeddings request.""" + try: + # For embeddings, input can be a string or list of strings + if item_count == 1: + input_data = EMBEDDINGS_INPUT_TEXT + else: + input_data = [EMBEDDINGS_INPUT_TEXT for _ in range(item_count)] + req = { + "input": input_data, + "model": EMBEDDINGS_MODEL_PATH, + } + return (index, req) + except Exception as e: + logger.error(f"Error building request {index}: {e}") + return (index, None) + + +def validate_embeddings_response(response_data: dict) -> bool: + """Validate embeddings API response.""" + return "data" in response_data + + +def build_warmup_embeddings_request() -> dict: + """Build a warmup request for the embeddings API.""" + return { + "input": EMBEDDINGS_INPUT_TEXT, + "model": EMBEDDINGS_MODEL_PATH, + } + + +############################################################################### +# MAIN +############################################################################### +async def run_benchmark(rps, duration_secs, item_count): + """Run a single embeddings benchmark with the given RPS value.""" + return await run_generic_benchmark( + rps=rps, + duration_secs=duration_secs, + item_count=item_count, + config=config, + http_url=HTTP_URL, + build_request_func=build_embeddings_request, + response_validator=validate_embeddings_response, + api_name="EMBEDDINGS", + request_description="embeddings requests", + ) + + +async def main(): + additional_info = { + "Input text length": f"{EMBEDDINGS_INPUT_TOKENS} tokens", + "Input text preview": ( + EMBEDDINGS_INPUT_TEXT[:100] + "..." + if len(EMBEDDINGS_INPUT_TEXT) > 100 + else EMBEDDINGS_INPUT_TEXT + ), + } + + await run_benchmark_main( + config, + run_benchmark, + "EMBEDDINGS", + HTTP_URL, + BATCH_SIZE, + additional_info, + build_warmup_embeddings_request, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmark/prefill_only/bench_score.py b/benchmark/prefill_only/bench_score.py new file mode 100644 index 000000000..117335eae --- /dev/null +++ b/benchmark/prefill_only/bench_score.py @@ -0,0 +1,192 @@ +""" +SGLang Scoring Benchmark Script + +This script benchmarks SGLang's scoring API performance using HTTP requests. + +Current Features: +- HTTP-only implementation (open source compatible) +- Uses /v1/score API endpoint directly +- Single item scoring with batching support +- Configurable RPS, duration, and batch sizes +- Progress tracking and detailed metrics +- Poisson and constant request distributions + +Usage: +- Update configuration variables at the top of the file +- Ensure SGLang server is running on the configured HTTP_URL +- Run: python bench_score.py +- Each request will contain ITEM_COUNT_VALUES items for batch scoring + +""" + +import asyncio + +from transformers import AutoTokenizer +from util import ( + BenchmarkConfig, + generate_text_with_token_count, + run_benchmark_main, + run_generic_benchmark, +) + +############################################################################### +# CONFIG +############################################################################### +# Create benchmark configuration +config = BenchmarkConfig() +config.rps_values = [160] +config.duration_secs_values = [60] +config.num_unique_requests = 100 +config.distribution = "POISSON" +config.profile = False +config.freeze_gc = True # Enable GC freeze functionality +# Profiler output directory - by default uses present working directory (pwd) +# Uncomment and customize the line below to override the default location: +# config.profiler_dir = "/sglang-oss-trace" + +# HTTP Configuration +HTTP_URL = "http://localhost:30000/v1/score" # Use score API directly + +# Score API Config +# ITEM_COUNT_VALUES determines number of items per score request (batch size) +SCORE_QUERY_TOKENS = 120 +SCORE_ITEM_TOKENS = 180 +SCORE_MODEL_PATH = "Qwen/Qwen3-0.6B" +SCORE_LABEL_TOKEN_IDS = [9454, 2753] # Yes/No token IDs +ITEM_COUNT_VALUES = [10] # Number of items per request + +# Special token to replicate for precise token counting +SPECIAL_REPLICATED_TOKEN = "<|im_start|>" + + +############################################################################### +# REQUEST GENERATION (in parallel) +############################################################################### +def create_score_request_builder(): + """Create a score request builder function with shared tokenizer.""" + # Load tokenizer once here to verify special token and get precise counts + print("Loading tokenizer...") + tokenizer = AutoTokenizer.from_pretrained(SCORE_MODEL_PATH) + + # Verify that our special token produces exactly 1 token + special_token_count = len( + tokenizer.encode(config.special_replicated_token, add_special_tokens=False) + ) + print( + f"Special token '{config.special_replicated_token}' produces " + f"{special_token_count} token(s)" + ) + + def generate_text_with_token_count_local(num_toks): + """Generate text with precise token count using replicated token.""" + return generate_text_with_token_count( + SCORE_MODEL_PATH, + num_toks, + config.special_replicated_token, + tokenizer=tokenizer, + ) + + def build_score_request(index: int, item_count: int) -> tuple: + """Build a single score request.""" + try: + # Generate query and items for score API + query = generate_text_with_token_count_local(SCORE_QUERY_TOKENS) + items = [ + generate_text_with_token_count_local(SCORE_ITEM_TOKENS) + for _ in range(item_count) + ] + + # Return as dict for score API format + score_data = { + "query": query, + "items": items, + "label_token_ids": SCORE_LABEL_TOKEN_IDS, + "model": SCORE_MODEL_PATH, + } + return (index, score_data) + + except Exception as e: + print(f"Error building request {index}: {e}") + return (index, None) + + return build_score_request + + +def validate_score_response(response_data: dict) -> bool: + """Validate score API response.""" + return "scores" in response_data or "logprobs" in response_data + + +def build_warmup_score_request() -> dict: + """Build a warmup request for the score API.""" + # Load tokenizer once for warmup generation + tokenizer = AutoTokenizer.from_pretrained(SCORE_MODEL_PATH) + + warmup_query = generate_text_with_token_count( + SCORE_MODEL_PATH, + SCORE_QUERY_TOKENS, + config.special_replicated_token, + tokenizer=tokenizer, + ) + warmup_items = [ + generate_text_with_token_count( + SCORE_MODEL_PATH, + SCORE_ITEM_TOKENS, + config.special_replicated_token, + tokenizer=tokenizer, + ) + for _ in range(3) + ] + + return { + "query": warmup_query, + "items": warmup_items, + "label_token_ids": SCORE_LABEL_TOKEN_IDS, + "model": SCORE_MODEL_PATH, + # Add missing parameters for consistency with the original warmup + "apply_softmax": True, + "item_first": False, + } + + +############################################################################### +# MAIN +############################################################################### +async def run_benchmark(rps, duration_secs, item_count): + """Run a single benchmark with the given RPS value.""" + # Create the request builder function with shared tokenizer + build_request_func = create_score_request_builder() + + return await run_generic_benchmark( + rps=rps, + duration_secs=duration_secs, + item_count=item_count, + config=config, + http_url=HTTP_URL, + build_request_func=build_request_func, + response_validator=validate_score_response, + api_name="SINGLE_ITEM_SCORING", + request_description="score requests", + ) + + +async def main(): + """Main function that runs benchmarks for all RPS values.""" + additional_info = { + "Query tokens per request": SCORE_QUERY_TOKENS, + "Item tokens per item": SCORE_ITEM_TOKENS, + } + + await run_benchmark_main( + config, + run_benchmark, + "SINGLE_ITEM_SCORING", + HTTP_URL, + ITEM_COUNT_VALUES, + additional_info, + build_warmup_score_request, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/benchmark/prefill_only/util.py b/benchmark/prefill_only/util.py new file mode 100644 index 000000000..0dbc39027 --- /dev/null +++ b/benchmark/prefill_only/util.py @@ -0,0 +1,813 @@ +""" +Common utilities for SGLang benchmark scripts. + +This module contains shared code for benchmarking different SGLang APIs +including scoring, embeddings, and other endpoints. +""" + +import asyncio +import concurrent.futures +import json +import os +import random +from statistics import mean +from typing import Any, Callable, Dict, List, Optional, Tuple + +import aiohttp +import numpy as np +from tqdm import tqdm +from transformers import AutoTokenizer + + +class BenchmarkConfig: + """Configuration for benchmark parameters.""" + + def __init__(self): + # Common benchmark settings + self.server_type = "HTTP" + self.rps_values = [70] + self.duration_secs_values = [60] + self.num_unique_requests = 100 + self.distribution = "POISSON" # Options: "CONSTANT", "POISSON" + self.profile = False + + # Garbage Collection Control + self.freeze_gc = True # Enable/disable garbage collection freezing + + # Profiler configuration + self.profiler_dir = ( + os.getcwd() + ) # Default profiler output directory (current working directory) + + # Special token for text generation + self.special_replicated_token = "<|im_start|>" + + +def generate_text_with_token_count( + model_path: str, + num_tokens: int, + special_token: str = "<|im_start|>", + tokenizer: Optional[Any] = None, +) -> str: + """ + Generate text with precise token count using a replicated token. + + Args: + model_path: Path to the model for tokenizer + num_tokens: Target number of tokens + special_token: Token to replicate + tokenizer: Optional pre-loaded tokenizer to avoid repeated loading + + Returns: + Generated text with approximately the target token count + """ + if tokenizer is None: + tokenizer = AutoTokenizer.from_pretrained(model_path) + + # Verify token count + special_token_count = len(tokenizer.encode(special_token, add_special_tokens=False)) + + if special_token_count == 1: + # Simple case: token maps to exactly 1 token + return special_token * num_tokens + else: + print(f"Special token '{special_token}' produces {special_token_count} tokens") + # Handle case where special token produces multiple tokens + repetitions = (num_tokens + special_token_count - 1) // special_token_count + text = special_token * repetitions + + # Verify we got the expected token count + actual_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + if actual_tokens < num_tokens: + print(f"Warning: Generated {actual_tokens} tokens, expected {num_tokens}") + + return text + + +def setup_profiler(config: BenchmarkConfig, benchmark_name: str) -> None: + """ + Set up profiler environment if profiling is enabled. + + Args: + config: Benchmark configuration + benchmark_name: Name of the benchmark (used in directory path) + """ + if config.profile: + # Create benchmark-specific subdirectory + profiler_path = os.path.join( + config.profiler_dir, benchmark_name.lower().replace("_", "-") + ) + os.environ["SGLANG_TORCH_PROFILER_DIR"] = profiler_path + print(f"Profiler enabled. Output directory: {profiler_path}") + else: + print("Profiler disabled") + + +def prepare_all_requests_parallel( + num_requests: int, + item_count: int, + build_request_func: Callable[[int, int], Tuple[int, Any]], + config: BenchmarkConfig, + description: str = "requests", +) -> List[Any]: + """ + Generic function to generate unique requests in parallel, then reuse them. + + Args: + num_requests: Total number of requests needed + item_count: Number of items per request (batch size) + build_request_func: Function that takes (index, item_count) and returns (index, request_data) + config: Benchmark configuration + description: Description for progress bars + + Returns: + List of request data objects + """ + + def build_request_wrapper(index): + """Wrapper to call the provided build_request_func.""" + try: + return build_request_func(index, item_count) + except Exception as e: + print(f"Error building request {index}: {e}") + return (index, None) + + # Generate only the unique requests + unique_requests = [None] * config.num_unique_requests + max_workers = min(8, os.cpu_count() or 1) # Limit to 8 threads max + + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for i in tqdm( + range(config.num_unique_requests), + desc=f"Submitting {description} generation tasks", + ): + future = executor.submit(build_request_wrapper, i) + futures.append(future) + + # Collect results as they complete + for f in tqdm( + concurrent.futures.as_completed(futures), + desc=f"Building unique {description}", + total=config.num_unique_requests, + ): + try: + index, req_data = f.result() + if req_data is not None: + unique_requests[index] = req_data + else: + print(f"Failed to build request {index}") + except Exception as e: + print(f"Error processing request result: {e}") + + # Check if we have any valid requests + valid_requests = [req for req in unique_requests if req is not None] + if not valid_requests: + raise RuntimeError("Failed to generate any valid requests") + + print( + f"Successfully generated {len(valid_requests)} out of " + f"{config.num_unique_requests} unique {description}" + ) + + # Create the full request list by cycling through unique requests + print( + f"Reusing {len(valid_requests)} unique {description} to create " + f"{num_requests} total requests..." + ) + all_requests = [] + for i in tqdm(range(num_requests), desc=f"Reusing {description}"): + unique_index = i % len(valid_requests) + all_requests.append(valid_requests[unique_index]) + + print(f"All {description} prepared.\n") + return all_requests + + +async def sleep_with_distribution(distribution: str, rps: float) -> None: + """ + Sleep according to the specified distribution pattern. + + Args: + distribution: "CONSTANT" or "POISSON" + rps: Requests per second rate + """ + if distribution == "CONSTANT": + interval = 1 / rps + await asyncio.sleep(interval) + elif distribution == "POISSON": + # For Poisson process, inter-arrival times follow exponential distribution + interval = random.expovariate(rps) + await asyncio.sleep(interval) + else: + raise ValueError( + f"Unknown distribution: {distribution}. Use 'CONSTANT' or 'POISSON'." + ) + + +def build_http_request_json(request_data: Any) -> str: + """ + Generic function to build HTTP request JSON. + + Args: + request_data: The data to serialize to JSON + + Returns: + JSON string representation of the request data + """ + return json.dumps(request_data) + + +async def make_http_call( + session: aiohttp.ClientSession, + request_data: Any, + request_id: int, + results_queue: asyncio.Queue, + http_url: str, + response_validator: Callable[[Dict[str, Any]], bool], + api_name: str = "API", +) -> None: + """ + Generic HTTP call function for API requests. + + Args: + session: aiohttp client session + request_data: Data to send in the request + request_id: Unique identifier for this request + results_queue: Queue to put results + http_url: URL to send the request to + response_validator: Function to validate the response JSON + api_name: Name of the API for error messages + """ + try: + start_time = asyncio.get_event_loop().time() + + request_json = build_http_request_json(request_data) + headers = {"Content-Type": "application/json"} + + async with session.post(http_url, data=request_json, headers=headers) as resp: + resp_text = await resp.text() + + if resp.status != 200: + print( + f"[HTTP] {api_name} Request {request_id} failed with status " + f"{resp.status}: {resp_text}" + ) + completion_time = asyncio.get_event_loop().time() + await results_queue.put((request_id, 0, False, completion_time)) + return + + # Parse and validate response + try: + response_data = json.loads(resp_text) + success = response_validator(response_data) + if not success: + print( + f"[HTTP] {api_name} Request {request_id} failed response validation" + ) + except json.JSONDecodeError: + print( + f"[HTTP] {api_name} Request {request_id} failed to parse JSON response" + ) + success = False + + completion_time = asyncio.get_event_loop().time() + elapsed_time = (completion_time - start_time) * 1000 + await results_queue.put((request_id, elapsed_time, success, completion_time)) + + except Exception as e: + print(f"[HTTP] {api_name} Error for request {request_id}: {e}") + completion_time = asyncio.get_event_loop().time() + await results_queue.put((request_id, 0, False, completion_time)) + + +async def send_profile_request( + profile_text: str, http_url: str, session: Optional[aiohttp.ClientSession] = None +) -> None: + """ + Send a profile request (START_PROFILE or STOP_PROFILE) and wait for completion. + + Args: + profile_text: "START_PROFILE" or "STOP_PROFILE" + http_url: Base HTTP URL (will derive profile endpoints from this) + session: Optional aiohttp session to use + """ + try: + if session: + print(f"Sending {profile_text} request via HTTP...") + + # Determine the correct endpoint + if "/v1/" in http_url: + base_url = http_url.rsplit("/v1/", 1)[0] # Remove /v1/xxx + else: + base_url = http_url.rsplit("/", 1)[0] # Remove last path component + + if profile_text == "START_PROFILE": + endpoint_url = f"{base_url}/start_profile" + elif profile_text == "STOP_PROFILE": + endpoint_url = f"{base_url}/stop_profile" + else: + print(f"Unknown profile request: {profile_text}") + return + + headers = {"Content-Type": "application/json"} + + async with session.post(endpoint_url, headers=headers) as resp: + resp_text = await resp.text() + if resp.status == 200: + print(f"{profile_text} request completed") + else: + print( + f"{profile_text} request failed with status " + f"{resp.status}: {resp_text}" + ) + else: + print(f"Cannot send {profile_text} request - missing session") + + except Exception as e: + print(f"Error sending {profile_text} request: {e}") + + +async def call_freeze_gc_http(session: aiohttp.ClientSession, http_url: str) -> None: + """ + Call the /freeze_gc HTTP endpoint. + + Args: + session: aiohttp client session + http_url: Base HTTP URL to derive the freeze_gc endpoint from + """ + try: + # Derive freeze_gc endpoint from the API URL + if "/v1/" in http_url: + freeze_gc_url = http_url.rsplit("/v1/", 1)[0] + "/freeze_gc" + else: + freeze_gc_url = http_url.rsplit("/", 1)[0] + "/freeze_gc" + + print(f"Calling freeze_gc endpoint: {freeze_gc_url}") + + async with session.post(freeze_gc_url) as resp: + if resp.status == 200: + print("freeze_gc called successfully") + else: + resp_text = await resp.text() + print(f"freeze_gc failed with status {resp.status}: {resp_text}") + + except Exception as e: + print(f"Failed to call freeze_gc: {e}") + + +async def send_warmup_requests( + session: aiohttp.ClientSession, + http_url: str, + build_warmup_request_func: Callable[[], Any], + num_warmup: int = 3, +) -> None: + """ + Send warmup requests to HTTP server. + + Args: + session: aiohttp client session + http_url: URL to send warmup requests to + build_warmup_request_func: Function that returns a warmup request object + num_warmup: Number of warmup requests to send + """ + print(f"Sending {num_warmup} HTTP warmup requests...") + + for i in range(num_warmup): + try: + warmup_data = build_warmup_request_func() + request_json = build_http_request_json(warmup_data) + headers = {"Content-Type": "application/json"} + + async with session.post( + http_url, data=request_json, headers=headers + ) as resp: + if resp.status == 200: + print(f"Warmup request {i+1}/{num_warmup} completed successfully") + else: + print( + f"Warmup request {i+1}/{num_warmup} failed with status {resp.status}" + ) + + except Exception as e: + print(f"Warmup request {i+1}/{num_warmup} failed with error: {e}") + + print("HTTP warmup requests completed") + + +async def perform_global_warmup_and_freeze( + config: BenchmarkConfig, + http_url: str, + build_warmup_request_func: Callable[[], Any], +) -> None: + """ + Perform warmup and optionally GC freeze operations once before all benchmark runs. + + Args: + config: Benchmark configuration + http_url: URL for API requests + build_warmup_request_func: Function that returns a warmup request object + """ + print("=" * 80) + print(f"PERFORMING GLOBAL WARMUP{' AND GC FREEZE' if config.freeze_gc else ''}") + print("=" * 80) + + print(f"Performing HTTP warmup{' and GC freeze' if config.freeze_gc else ''}...") + async with aiohttp.ClientSession() as session: + await send_warmup_requests(session, http_url, build_warmup_request_func) + if config.freeze_gc: + await call_freeze_gc_http(session, http_url) + print( + f"HTTP warmup{' and GC freeze' if config.freeze_gc else ''} completed successfully." + ) + + print( + f"Global warmup{' and GC freeze' if config.freeze_gc else ''} operations completed." + ) + print("=" * 80) + + +async def process_results( + results_queue: asyncio.Queue, + num_requests: int, + send_duration: float, + total_duration: float, + rps: int, + duration_secs: int, + item_count: int, + test_start_time: float, + config: BenchmarkConfig, + http_mode: str = "UNKNOWN", +) -> List[Dict[str, Any]]: + """ + Process benchmark results and group them by minute intervals. + + Args: + results_queue: Queue containing result tuples + num_requests: Total number of requests sent + send_duration: Time taken to send all requests + total_duration: Total time for all requests to complete + rps: Target requests per second + duration_secs: Test duration in seconds + item_count: Number of items per request + test_start_time: Start time of the test + config: Benchmark configuration + http_mode: Description of the HTTP mode/API being tested + + Returns: + List of dictionaries containing minute-by-minute results + """ + all_results = [] + + # Collect all results + for _ in range(num_requests): + result = await results_queue.get() + request_id, elapsed_time, success, completion_time = result + all_results.append( + { + "request_id": request_id, + "elapsed_time": elapsed_time, + "success": success, + "completion_time": completion_time, + } + ) + + # Group results by minute intervals + minute_results = [] + num_minutes = int(duration_secs // 60) + (1 if duration_secs % 60 > 0 else 0) + + for minute in range(num_minutes): + minute_start = test_start_time + (minute * 60) + minute_end = test_start_time + ((minute + 1) * 60) + + # Filter results that completed in this minute + minute_data = [ + r for r in all_results if minute_start <= r["completion_time"] < minute_end + ] + + response_times = [r["elapsed_time"] for r in minute_data if r["success"]] + successful_requests = len([r for r in minute_data if r["success"]]) + failed_requests = len([r for r in minute_data if not r["success"]]) + + avg_response_time = mean(response_times) if response_times else 0 + + # Calculate percentiles using numpy + if response_times: + p50 = np.percentile(response_times, 50) + p90 = np.percentile(response_times, 90) + p99 = np.percentile(response_times, 99) + else: + p50 = p90 = p99 = 0 + + minute_result = { + "test_duration_secs": duration_secs, + "minute_interval": minute + 1, + "target_rps": rps, + "item_count": item_count, + "server_type": config.server_type, + "distribution": config.distribution, + "unique_requests": config.num_unique_requests, + "total_requests": len(minute_data), + "successful_requests": successful_requests, + "failed_requests": failed_requests, + "send_duration_secs": send_duration, + "total_duration_secs": total_duration, + "avg_response_time_ms": avg_response_time, + "p50_response_time_ms": p50, + "p90_response_time_ms": p90, + "p99_response_time_ms": p99, + } + + minute_results.append(minute_result) + + print( + f"\nMinute {minute + 1} Summary for RPS {rps}, " + f"Duration {duration_secs}s, Item Count {item_count}:" + ) + print(f" Requests completed in minute: {len(minute_data)}") + print(f" Successful requests: {successful_requests}") + print(f" Failed requests: {failed_requests}") + print(f" Average response time: {avg_response_time:.2f} ms") + print(f" P50 response time: {p50:.2f} ms") + print(f" P90 response time: {p90:.2f} ms") + print(f" P99 response time: {p99:.2f} ms") + + # Print overall summary + all_response_times = [r["elapsed_time"] for r in all_results if r["success"]] + total_successful = len([r for r in all_results if r["success"]]) + total_failed = len([r for r in all_results if not r["success"]]) + + overall_avg = mean(all_response_times) if all_response_times else 0 + if all_response_times: + overall_p50 = np.percentile(all_response_times, 50) + overall_p90 = np.percentile(all_response_times, 90) + overall_p99 = np.percentile(all_response_times, 99) + else: + overall_p50 = overall_p90 = overall_p99 = 0 + + print( + f"\nOverall Summary for RPS {rps}, Duration {duration_secs}s, " + f"Item Count {item_count}:" + ) + print(f" Test duration: {duration_secs} seconds") + print(f" Server type: {config.server_type}") + print(f" HTTP mode: {http_mode}") + print(f" Target RPS: {rps}") + print(f" Item count: {item_count}") + print(f" Distribution: {config.distribution}") + print(f" Unique requests generated: {config.num_unique_requests}") + print(f" Total requests sent: {num_requests}") + print(f" Successful requests: {total_successful}") + print(f" Failed requests: {total_failed}") + print(f" Time to send all requests: {send_duration:.2f} seconds") + print(f" Time for all requests to complete: {total_duration:.2f} seconds") + print(f" Average response time: {overall_avg:.2f} ms") + print(f" P50 response time: {overall_p50:.2f} ms") + print(f" P90 response time: {overall_p90:.2f} ms") + print(f" P99 response time: {overall_p99:.2f} ms\n") + + return minute_results + + +def print_csv_results(all_results: List[Dict[str, Any]]) -> None: + """ + Print benchmark results in CSV format. + + Args: + all_results: List of result dictionaries from process_results + """ + print("\n" + "=" * 80) + print("FINAL CSV RESULTS:") + print("=" * 80) + + # CSV Header + headers = [ + "test_duration_secs", + "minute_interval", + "target_rps", + "item_count", + "server_type", + "distribution", + "unique_requests", + "total_requests", + "successful_requests", + "failed_requests", + "send_duration_secs", + "total_duration_secs", + "avg_response_time_ms", + "p50_response_time_ms", + "p90_response_time_ms", + "p99_response_time_ms", + ] + print(",".join(headers)) + + # CSV Data + for result in all_results: + row = [ + result["test_duration_secs"], + result["minute_interval"], + result["target_rps"], + result["item_count"], + result["server_type"], + result["distribution"], + result["unique_requests"], + result["total_requests"], + result["successful_requests"], + result["failed_requests"], + f"{result['send_duration_secs']:.2f}", + f"{result['total_duration_secs']:.2f}", + f"{result['avg_response_time_ms']:.2f}", + f"{result['p50_response_time_ms']:.2f}", + f"{result['p90_response_time_ms']:.2f}", + f"{result['p99_response_time_ms']:.2f}", + ] + print(",".join(map(str, row))) + + +async def run_benchmark_main( + config: BenchmarkConfig, + run_single_benchmark_func, + benchmark_name: str, + http_url: str, + item_count_values: List[int], + additional_info: Optional[Dict[str, Any]] = None, + build_warmup_request_func: Optional[Callable[[], Any]] = None, +) -> None: + """ + Main benchmark orchestration function. + + Args: + config: Benchmark configuration + run_single_benchmark_func: Async function to run a single benchmark + benchmark_name: Name of the benchmark (e.g., "SCORING", "EMBEDDINGS") + http_url: URL of the API endpoint + item_count_values: List of item counts to test + additional_info: Additional information to print in the header + build_warmup_request_func: Optional function to build warmup requests + """ + total_combinations = ( + len(config.duration_secs_values) + * len(config.rps_values) + * len(item_count_values) + ) + + print( + f"Running benchmarks for {len(config.duration_secs_values)} duration " + f"values, {len(config.rps_values)} RPS values, and " + f"{len(item_count_values)} item count values = " + f"{total_combinations} total combinations" + ) + print(f"Server Type: {config.server_type}") + print(f"HTTP Mode: {benchmark_name}") + print(f"API URL: {http_url}") + + if additional_info: + for key, value in additional_info.items(): + print(f"{key}: {value}") + + print(f"Items per request (batch size): {item_count_values}") + print(f"Profiling Enabled: {config.profile}") + print(f"Duration values: {config.duration_secs_values}") + print(f"RPS values: {config.rps_values}") + print(f"Item count values: {item_count_values}") + print("=" * 80) + + # Set up profiler environment + setup_profiler(config, benchmark_name) + + # Perform global warmup and GC freeze operations if warmup function is provided + if build_warmup_request_func is not None: + await perform_global_warmup_and_freeze( + config, http_url, build_warmup_request_func + ) + + all_results = [] + + for duration_secs in config.duration_secs_values: + for rps in config.rps_values: + for item_count in item_count_values: + result = await run_single_benchmark_func(rps, duration_secs, item_count) + all_results.extend(result) # Extend with minute results + + print_csv_results(all_results) + + +async def run_generic_benchmark( + rps: int, + duration_secs: int, + item_count: int, + config: BenchmarkConfig, + http_url: str, + build_request_func: Callable[[int, int], Tuple[int, Any]], + response_validator: Callable[[Dict[str, Any]], bool], + api_name: str, + request_description: str = "requests", +) -> List[Dict[str, Any]]: + """ + Generic benchmark runner that can be used for different APIs. + + Args: + rps: Requests per second + duration_secs: Duration of the test in seconds + item_count: Number of items per request (batch size) + config: Benchmark configuration + http_url: URL of the API endpoint + build_request_func: Function to build individual requests + response_validator: Function to validate API responses + api_name: Name of the API for logging + request_description: Description for progress bars + + Returns: + List of dictionaries containing minute-by-minute results + """ + num_requests = int(rps * duration_secs) + print( + f"Starting benchmark with RPS={rps}, Duration={duration_secs}s, " + f"Item Count={item_count}, num_requests={num_requests}" + ) + print(f"Server Type: {config.server_type}") + print(f"HTTP Mode: {api_name}") + print(f"Profiling Enabled: {config.profile}") + + # Build requests in parallel (unmeasured) + all_requests = prepare_all_requests_parallel( + num_requests, item_count, build_request_func, config, request_description + ) + + results_queue = asyncio.Queue() + tasks = [] + + # Track timing for sending requests + send_start_time = asyncio.get_event_loop().time() + + # HTTP implementation + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=300) + ) as session: + + # Send START_PROFILE if profiling is enabled + if config.profile: + await send_profile_request("START_PROFILE", http_url, session=session) + + # Add progress bar for sending requests + with tqdm( + total=len(all_requests), + desc=f"Sending HTTP {request_description} at {rps} RPS", + unit="req", + ) as pbar: + for i, request_data in enumerate(all_requests): + request_id = i + 1 + tasks.append( + asyncio.create_task( + make_http_call( + session, + request_data, + request_id, + results_queue, + http_url, + response_validator, + api_name, + ) + ) + ) + + # Update progress bar + pbar.update(1) + + # Throttle based on distribution + if i < len(all_requests) - 1: + await sleep_with_distribution(config.distribution, rps) + + send_end_time = asyncio.get_event_loop().time() + send_duration = send_end_time - send_start_time + + # Wait for all requests to complete with progress tracking + print(f"Waiting for {len(tasks)} HTTP {request_description} to complete...") + with tqdm( + total=len(tasks), desc=f"Completing HTTP {request_description}", unit="req" + ) as completion_pbar: + completed_tasks = [] + for task in asyncio.as_completed(tasks): + await task + completed_tasks.append(task) + completion_pbar.update(1) + + # Send STOP_PROFILE if profiling is enabled + if config.profile: + await send_profile_request("STOP_PROFILE", http_url, session=session) + + completion_end_time = asyncio.get_event_loop().time() + total_duration = completion_end_time - send_start_time + + return await process_results( + results_queue, + num_requests, + send_duration, + total_duration, + rps, + duration_secs, + item_count, + send_start_time, + config, + api_name, + ) diff --git a/benchmark/react/README.md b/benchmark/react/README.md new file mode 100644 index 000000000..51bcaa44a --- /dev/null +++ b/benchmark/react/README.md @@ -0,0 +1,34 @@ +## Run benchmark + +NOTE: This is an implementation for replaying a given trace for throughput/latency benchmark purposes. It is not an actual ReAct agent implementation. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 100 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 100 --backend vllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 100 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 100 --backend lmql --parallel 1 +``` diff --git a/benchmark/react/bench_other.py b/benchmark/react/bench_other.py new file mode 100644 index 000000000..08666662b --- /dev/null +++ b/benchmark/react/bench_other.py @@ -0,0 +1,202 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + + +def get_prompt(question): + prompt = ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + return prompt + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + states = [] + + # Select backend + call_generate = get_call_generate(args) + + def run_single_agent(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation" + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + async def run_single_agent_async(argument): + question = argument["question"] + triplets = argument["triplets"] + prompt = get_prompt(question) + for i in range(1, len(triplets) + 2): + prompt += "Thought " + str(i) + ":" + states.append(prompt) + answer = await call_generate( + prompt, max_tokens=200, temperature=0, stop="Observation", max_len=4096 + ) + if i > len(triplets): + break + prompt += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + states.append(answer) + + tic = time.perf_counter() + + if args.backend != "lmql": + if args.parallel == 1: + for arg in tqdm(arguments): + run_single_agent(arg) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(run_single_agent, arguments), total=len(arguments) + ) + ) + + else: + import asyncio + + loop = asyncio.get_event_loop() + batches = [ + [] for _ in range((len(arguments) + args.parallel - 1) // args.parallel) + ] + for i, arg in enumerate(arguments): + batches[i // args.parallel].append(arg) + for bt in tqdm(batches): + tasks = [run_single_agent_async(arg) for arg in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/react/bench_sglang.py b/benchmark/react/bench_sglang.py new file mode 100644 index 000000000..331638e9f --- /dev/null +++ b/benchmark/react/bench_sglang.py @@ -0,0 +1,153 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + + +@sgl.function +def webthink(s, question, triplets): + s += ( + """Solve a question answering task with interleaving Thought, Action, Observation steps. Thought can reason about the current situation, and Action can be three types: +(1) Search[entity], which searches the exact entity on Wikipedia and returns the first paragraph if it exists. If not, it will return some similar entities to search. +(2) Lookup[keyword], which returns the next sentence containing keyword in the current passage. +(3) Finish[answer], which returns the answer and finishes the task. +Here are some examples. +Question: What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? +Thought 1: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado orogeny extends into, then find the elevation range of the area. +Action 1: Search[Colorado orogeny] +Observation 1: The Colorado orogeny was an episode of mountain building (an orogeny) in Colorado and surrounding areas. +Thought 2: It does not mention the eastern sector. So I need to look up eastern sector. +Action 2: Lookup[eastern sector] +Observation 2: (Result 1 / 1) The eastern sector extends into the High Plains and is called the Central Plains orogeny. +Thought 3: The eastern sector of Colorado orogeny extends into the High Plains. So I need to search High Plains and find its elevation range. +Action 3: Search[High Plains] +Observation 3: High Plains refers to one of two distinct land regions: +Thought 4: I need to instead search High Plains (United States). +Action 4: Search[High Plains (United States)] +Observation 4: The High Plains are a subregion of the Great Plains. From east to west, the High Plains rise in elevation from around 1,800 to 7,000 ft (550 to 2,130 m).[3] +Thought 5: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. +Action 5: Finish[1,800 to 7,000 ft] +Question: Musician and satirist Allie Goertz wrote a song about the "The Simpsons" character Milhouse, who Matt Groening named after who? +Thought 1: The question simplifies to "The Simpsons" character Milhouse is named after who. I only need to search Milhouse and find who it is named after. +Action 1: Search[Milhouse] +Observation 1: Milhouse Mussolini Van Houten is a recurring character in the Fox animated television series The Simpsons voiced by Pamela Hayden and created by Matt Groening. +Thought 2: The paragraph does not tell who Milhouse is named after, maybe I can look up "named after". +Action 2: Lookup[named after] +Observation 2: (Result 1 / 1) Milhouse was named after U.S. president Richard Nixon, whose middle name was Milhous. +Thought 3: Milhouse was named after U.S. president Richard Nixon, so the answer is Richard Nixon. +Action 3: Finish[Richard Nixon] +Question: Which documentary is about Finnish rock groups, Adam Clayton Powell or The Saimaa Gesture? +Thought 1: I need to search Adam Clayton Powell and The Saimaa Gesture, and find which documentary is about Finnish rock groups. +Action 1: Search[Adam Clayton Powell] +Observation 1: Could not find [Adam Clayton Powell]. Similar: ['Adam Clayton Powell III', 'Seventh Avenue (Manhattan)', 'Adam Clayton Powell Jr. State Office Building', 'Isabel Washington Powell', 'Adam Powell', 'Adam Clayton Powell (film)', 'Giancarlo Esposito']. +Thought 2: To find the documentary, I can search Adam Clayton Powell (film). +Action 2: Search[Adam Clayton Powell (film)] +Observation 2: Adam Clayton Powell is a 1989 American documentary film directed by Richard Kilberg. +The film is about the rise and fall of influential African-American politician Adam Clayton Powell Jr.[3][4] It was later aired as part of the PBS series The American Experience. +Thought 3: Adam Clayton Powell (film) is a documentary about an African-American politician, not Finnish rock groups. So the documentary about Finnish rock groups must instead be The Saimaa Gesture. +Action 3: Finish[The Saimaa Gesture] +Question: What profession does Nicholas Ray and Elia Kazan have in common? +Thought 1: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. +Action 1: Search[Nicholas Ray] +Observation 1: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 – June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. +Thought 2: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. +Action 2: Search[Elia Kazan] +Observation 2: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. +Thought 3: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. +Action 3: Finish[director, screenwriter, actor] +Question: Which magazine was started first Arthur's Magazine or First for Women? +Thought 1: I need to search Arthur's Magazine and First for Women, and find which was started first. +Action 1: Search[Arthur's Magazine] +Observation 1: Arthur's Magazine (1844-1846) was an American literary periodical published in Philadelphia in the 19th century. +Thought 2: Arthur's Magazine was started in 1844. I need to search First for Women next. +Action 2: Search[First for Women] +Observation 2: First for Women is a woman's magazine published by Bauer Media Group in the USA.[1] The magazine was started in 1989. +Thought 3: First for Women was started in 1989. 1844 (Arthur's Magazine) < 1989 (First for Women), so Arthur's Magazine was started first. +Action 3: Finish[Arthur's Magazine] +Question: Were Pavel Urysohn and Leonid Levin known for the same type of work? +Thought 1: I need to search Pavel Urysohn and Leonid Levin, find their types of work, then find if they are the same. +Action 1: Search[Pavel Urysohn] +Observation 1: Pavel Samuilovich Urysohn (February 3, 1898 â August 17, 1924) was a Soviet mathematician who is best known for his contributions in dimension theory. +Thought 2: Pavel Urysohn is a mathematician. I need to search Leonid Levin next and find its type of work. +Action 2: Search[Leonid Levin] +Observation 2: Leonid Anatolievich Levin is a Soviet-American mathematician and computer scientist. +Thought 3: Leonid Levin is a mathematician and computer scientist. So Pavel Urysohn and Leonid Levin have the same type of work. +Action 3: Finish[yes] +""" + + question + ) + for i in range(1, len(triplets) + 2): + s += "Thought " + str(i) + ":" + # NOTE: This is an implementation for replaying a given trace for benchmark purposes. It is not an actual ReAct agent implementation. + ss = s.fork(1) + ss[0] += sgl.gen(name="thought_action", max_tokens=200, stop="Observation") + ss.join() + # to verify the correctness of output, this should be collected + # print(ss[0]["thought_action"]) + if i > len(triplets): + break + s += ( + triplets[i - 1]["thought"] + + "\nAction " + + str(i) + + ":" + + triplets[i - 1]["action"] + + "\nObservation " + + str(i) + + ":" + + triplets[i - 1]["observation"] + + "\n" + ) + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"question": k, "triplets": v} for l in lines for k, v in l.items()] + + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + states = [] + tic = time.perf_counter() + states = webthink.run_batch( + arguments, + temperature=0, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "ReAct Agents", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": len(arguments), + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="hotpotqa_100.jsonl") + parser.add_argument("--num-questions", type=int, default=10) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/reasoning_benchmark/README.md b/benchmark/reasoning_benchmark/README.md new file mode 100644 index 000000000..f9d26e0c7 --- /dev/null +++ b/benchmark/reasoning_benchmark/README.md @@ -0,0 +1,77 @@ +# Run benchmark + +This benchmark is primarily intended to be used with reasoning models like `DeepSeek-R1` and its distilled models like `DeepSeek-R1-Distill-Qwen-1.5B`. Please use + +```bash +pip install antlr4-python3-runtime +``` + +for `parse_latex` which we use for symbolic equality check. + +## Benchmark sglang + +1. Launch the Server +```bash +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 +``` + +Note that depending on the GPU this benchmark will take quiet some time. To employ data parallelism please use: + +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 4 +``` + + +2. Benchmarking + +We use [suggested](https://github.com/deepseek-ai/DeepSeek-R1) parameters of `temperature=0.6`, `top_p=.95`, `max_new_tokens=32768`. The command line argument `num-tries` can be used to evaluate the model multiple times on the same question. We use the suggested `64` from the repo for AIME 2024. For LIMO, we use `8` as the number of tries due to the size of the dataset. + +By default evaluate on LIMO dataset. + +```bash +python3 bench_sglang.py --parallel 256 --num-tries 64 --port 30000 +``` + +Evaluate on AIME 2024 dataset. + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +Evaluate on [AIME 2025 I dataset](https://huggingface.co/datasets/opencompass/AIME2025). For benchmark result see [here](https://matharena.ai/). + +```bash +python3 bench_sglang.py --parallel 256 --port 30000 --data-path opencompass/AIME2025 --question-key question --answer-key answer --num-tries 64 +``` +## Results + +### Evaluation Results +| Dataset | Num Tries | Accuracy | Reference | Standard Error | +|------------|-----------|----------|-----------|-----------| +| LIMO | 8 | 47.7% | ? | ? | +| AIME 2024 | 64 | 33.2% | 28.9% | 3.4% | +| AIME 2025 I| 64 | 29.9% | 25.0% | ? | + +### Statistic Analysis Results +Set up SGLang engine for statistic analysis, for high efficiency we use `--dp-size 8` for data parallelism: +```bash +python3 -m sglang_router.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --port 30000 --dp-size 8 +``` +**Experiment 1**: +We fixed the number of attempts (num_tries) and conducted multiple runs to assess the consistency of the model's performance. The results show that all recorded accuracies lie within ± one standard error deviation from the mean. This suggests that **our metric serves as an effective upper bound for the deviation of reported accuracy**. + +To collect the accuracy, run the following command 30 times: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries 64 +``` + +![acc_hist](figure/Acc_histplot.png) + + +**Experiment 2**: We explored the relationship between the number of attempts (num_tries) and the standard error (SE) by varying num_tries across a range (e.g., 8, 16, 32, ..., 256) and performing a single run for each value. The results demonstrate that as the number of attempts increases, the standard error decreases, leading to **greater stability in answer accuracy**. + +To reveal the relationship, run the command 6 times and adjust the parameter `--num-tries` for each run: +```bash +python3 bench_sglang.py --parallel 64 --port 30000 --data-path Maxwell-Jia/AIME_2024 --question-key Problem --answer-key Answer --num-tries +``` +![SE_num_tries](figure/SE_numtries.png) diff --git a/benchmark/reasoning_benchmark/answer_extraction.py b/benchmark/reasoning_benchmark/answer_extraction.py new file mode 100644 index 000000000..45ce59afb --- /dev/null +++ b/benchmark/reasoning_benchmark/answer_extraction.py @@ -0,0 +1,269 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/data_processing/answer_extraction.py + +import re + +import regex + + +def _fix_fracs(string): + substrs = string.split("\\frac") + new_str = substrs[0] + if len(substrs) > 1: + substrs = substrs[1:] + for substr in substrs: + new_str += "\\frac" + if len(substr) > 0 and substr[0] == "{": + new_str += substr + else: + try: + assert len(substr) >= 2 + except: + return string + a = substr[0] + b = substr[1] + if b != "{": + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}{" + b + "}" + post_substr + else: + new_str += "{" + a + "}{" + b + "}" + else: + if len(substr) > 2: + post_substr = substr[2:] + new_str += "{" + a + "}" + b + post_substr + else: + new_str += "{" + a + "}" + b + string = new_str + return string + + +def _fix_a_slash_b(string): + if len(string.split("/")) != 2: + return string + a = string.split("/")[0] + b = string.split("/")[1] + try: + if "sqrt" not in a: + a = int(a) + if "sqrt" not in b: + b = int(b) + assert string == "{}/{}".format(a, b) + new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" + return new_string + except: + return string + + +def _fix_sqrt(string): + _string = re.sub(r"\\sqrt(-?[0-9.a-zA-Z]+)", r"\\sqrt{\1}", string) + _string = re.sub(r"\\sqrt\s+(\w+)$", r"\\sqrt{\1}", _string) + return _string + + +def _fix_tan(string): + _string = re.sub(r"\\tan(-?[0-9.a-zA-Z]+)", r"\\tan{\1}", string) + _string = re.sub(r"\\tan\s+(\w+)$", r"\\tan{\1}", _string) + return _string + + +def strip_string(string): + string = str(string).strip() + # linebreaks + string = string.replace("\n", "") + + # right "." + string = string.rstrip(".") + + # remove inverse spaces + string = string.replace("\\!", "") + # string = string.replace("\\ ", "") + + # replace \\ with \ + # string = string.replace("\\\\", "\\") + # string = string.replace("\\\\", "\\") + + if string.startswith("\\text{") and string.endswith("}"): + string = string.split("{", 1)[1][:-1] + + # replace tfrac and dfrac with frac + string = string.replace("tfrac", "frac") + string = string.replace("dfrac", "frac") + string = string.replace("cfrac", "frac") + + # remove \left and \right + string = string.replace("\\left", "") + string = string.replace("\\right", "") + + # Remove unit: miles, dollars if after is not none + _string = re.sub(r"\\text{.*?}$", "", string).strip() + if _string != "" and _string != string: + # print("Warning: unit not removed: '{}' -> '{}'".format(string, _string)) + string = _string + + # Remove circ (degrees) + string = string.replace("^{\\circ}", "").strip() + string = string.replace("^\\circ", "").strip() + + string = regex.sub(r"\{(c|m)?m\}(\^(2|3))?", "", string).strip() + string = regex.sub(r"p\.m\.$", "", string).strip() + string = regex.sub(r"(\d)\s*t$", r"\1", string).strip() + + # remove dollar signs + string = string.replace("\\$", "") + string = string.replace("$", "") + + # string = string.replace("\\text", "") + string = string.replace("x\\in", "") + + # remove percentage + string = string.replace("\\%", "%") + string = string.replace("\%", "%") + # string = string.replace("%", "") + + # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string + string = string.replace(" .", " 0.") + string = string.replace("{.", "{0.") + + # cdot + string = string.replace("\\cdot", "") + + # inf + string = string.replace("infinity", "\\infty") + if "\\infty" not in string: + string = string.replace("inf", "\\infty") + string = string.replace("+\\inity", "\\infty") + + # and + # string = string.replace("and", "") + string = string.replace("\\mathbf", "") + string = string.replace("\\mathrm", "") + + # use regex to remove \mbox{...} + string = re.sub(r"\\mbox{.*?}", "", string) + + # quote + string.replace("'", "") + string.replace('"', "") + + # i, j + if "j" in string and "i" not in string: + string = string.replace("j", "i") + + # replace a.000b where b is not number or b is end, with ab, use regex + string = re.sub(r"(\d+)\.0+([^\d])", r"\1\2", string) + string = re.sub(r"(\d+)\.0+$", r"\1", string) + + # if empty, return empty string + if len(string) == 0: + return string + if string[0] == ".": + string = "0" + string + + # to consider: get rid of e.g. "k = " or "q = " at beginning + # if len(string.split("=")) == 2: + # if len(string.split("=")[0]) <= 2: + # string = string.split("=")[1] + + string = _fix_sqrt(string) + string = _fix_tan(string) + string = string.replace(" ", "") + + # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} + string = _fix_fracs(string) + + # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y + string = _fix_a_slash_b(string) + + string = regex.sub(r"(\\|,|\.)+$", "", string) + + return string + + +def extract_boxed_answers(text): + answers = [] + for piece in text.split("boxed{")[1:]: + n = 0 + for i in range(len(piece)): + if piece[i] == "{": + n += 1 + elif piece[i] == "}": + n -= 1 + if n < 0: + if i + 1 < len(piece) and piece[i + 1] == "%": + answers.append(piece[: i + 1]) + else: + answers.append(piece[:i]) + break + return answers + + +def extract_program_output(pred_str): + """ + extract output between the last ```output\n...\n``` + """ + if "```output" not in pred_str: + return "" + if "```output" in pred_str: + pred_str = pred_str.split("```output")[-1] + if "```" in pred_str: + pred_str = pred_str.split("```")[0] + output = pred_str.strip() + return output + + +def extract_answer(pred_str, exhaust=False): + pred = [] + if "final answer is $" in pred_str and "$. I hope" in pred_str: + tmp = pred_str.split("final answer is $", 1)[1] + pred = [tmp.split("$. I hope", 1)[0].strip()] + elif "boxed" in pred_str: + pred = extract_boxed_answers(pred_str) + elif "he answer is" in pred_str: + pred = [pred_str.split("he answer is")[-1].strip()] + else: + program_output = extract_program_output(pred_str) + if program_output != "": + # fall back to program + pred.append(program_output) + else: # use the last number + pattern = "-?\d*\.?\d+" + ans = re.findall(pattern, pred_str.replace(",", "")) + if len(ans) >= 1: + ans = ans[-1] + else: + ans = "" + if ans: + pred.append(ans) + + # multiple line + _pred = [] + for ans in pred: + ans = ans.strip().split("\n")[0] + ans = ans.lstrip(":") + ans = ans.rstrip(".") + ans = ans.rstrip("/") + ans = strip_string(ans) + _pred.append(ans) + if exhaust: + return _pred + else: + return _pred[-1] if _pred else "" + + +def extract_math_answer(question, reasoning, task): + answer = [] + for ans in extract_answer(reasoning, exhaust=True): + if "separated by commas" in question and all(ch not in ans for ch in "()[]"): + answer.extend([a.strip() for a in ans.split(",")]) + elif regex.search(r"\\text\{\s*and\s*\}", ans): + answer.extend( + [ + a.strip() + for a in regex.sub(r"\\text\{\s*and\s*\}", "[SEP]", ans).split( + "[SEP]" + ) + ] + ) + else: + answer.append(ans.strip()) + return answer diff --git a/benchmark/reasoning_benchmark/bench_sglang.py b/benchmark/reasoning_benchmark/bench_sglang.py new file mode 100644 index 000000000..ccbff9d17 --- /dev/null +++ b/benchmark/reasoning_benchmark/bench_sglang.py @@ -0,0 +1,135 @@ +import argparse +import json +import time + +import answer_extraction +import eval_utils +import numpy as np +from datasets import load_dataset + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text + + +@sgl.function +def reasoning_gen(s, question: str): + s += sgl.user( + question + + "\nPlease reason step by step, and put your final answer within \boxed{}." + ) + s += sgl.assistant( + sgl.gen( + "answer", + ) + ) + + +def convert_dataset(path: str, question_key: str, answer_key: str, num_tries: int): + raw_dataset = load_dataset(path) + questions = [] + answers = [] + for data in raw_dataset["train"]: + question = data[question_key] + answer = data[answer_key] + for _ in range(num_tries): + questions.append({"question": question}) + answers.append({"answer": answer}) + return questions, answers + + +def main(args): + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Get dataset + questions, answers = convert_dataset( + args.data_path, args.question_key, args.answer_key, args.num_tries + ) + + # Run requests + tic = time.perf_counter() + states = reasoning_gen.run_batch( + questions, + num_threads=args.parallel, + progress_bar=True, + temperature=0.6, + max_new_tokens=32768, + top_p=0.95, + ) + latency = time.perf_counter() - tic + + # Extract results and record outcomes in a list. + outcomes = [] + for i, state in enumerate(states): + try: + pred_answer = answer_extraction.extract_math_answer( + questions[i]["question"], state["answer"], "limo" + ) + gt_answer = str(answers[i]["answer"]) + pred_answer = ( + pred_answer[-1] if isinstance(pred_answer, list) else pred_answer + ) + is_correct = 1 if eval_utils.math_equal(pred_answer, gt_answer) else 0 + except Exception as e: + print(f"Error extracting answer: {e}") + is_correct = 0 + + outcomes.append(is_correct) + + # Calculate overall accuracy using numpy + overall_accuracy = np.mean(outcomes) + print(f"Overall Accuracy: {overall_accuracy}") + + # Calculate mean standard error over questions if num_tries >= 2 + if args.num_tries > 1: + outcomes_np = np.array(outcomes).reshape(-1, args.num_tries) + # Using sample standard deviation with ddof=1 + std_per_question = np.std(outcomes_np, axis=1, ddof=1) + # Compute the standard error for each question: std / sqrt(num_tries) + se_per_question = std_per_question / np.sqrt(args.num_tries) + mean_se = se_per_question.mean() + print(f"Mean Standard Error of Accuracy across questions: {mean_se}") + else: + mean_se = None + print("Not enough samples per question to compute standard error.") + + # Calculate output throughput + num_output_tokens = sum( + s.get_meta_info("answer")["completion_tokens"] for s in states + ) + output_throughput = num_output_tokens / latency + print(f"Output throughput: {output_throughput} token/s") + + # Dump results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + # Write results + with open(args.result_file, "a") as fout: + value = { + "task": "limo", + "backend": args.backend, + "latency": round(latency, 3), + "overall_accuracy": round(overall_accuracy, 3), + "mean_se_accuracy": round(mean_se, 3) if mean_se is not None else None, + "num_requests": len(questions), + "other": { + "num_questions": len(questions), + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="GAIR/LIMO") + parser.add_argument("--question-key", type=str, default="question") + parser.add_argument("--answer-key", type=str, default="answer") + parser.add_argument("--num-tries", type=int, default=1) + add_common_sglang_args_and_parse(parser) + args = parser.parse_args() + main(args) diff --git a/benchmark/reasoning_benchmark/eval_utils.py b/benchmark/reasoning_benchmark/eval_utils.py new file mode 100644 index 000000000..ab736954f --- /dev/null +++ b/benchmark/reasoning_benchmark/eval_utils.py @@ -0,0 +1,206 @@ +# Adapted from https://github.com/deepseek-ai/DeepSeek-Math/blob/main/evaluation/eval/eval_utils.py + +from math import isclose + +import regex +from sympy import N, simplify +from sympy.parsing.latex import parse_latex +from sympy.parsing.sympy_parser import parse_expr + + +def parse_digits(num): + # format: 234.23 || 23% + num = regex.sub(",", "", str(num)) + try: + return float(num) + except: + if num.endswith("%"): + num = num[:-1] + if num.endswith("\\"): + num = num[:-1] + try: + return float(num) / 100 + except: + pass + return None + + +def is_digit(num): + # paired with parse_digits + return parse_digits(num) is not None + + +def symbolic_equal(a, b): + def _parse(s): + for f in [parse_latex, parse_expr]: + try: + return f(s) + except: + pass + return s + + a = _parse(a) + b = _parse(b) + + try: + if simplify(a - b) == 0: + return True + except: + pass + + try: + if isclose(N(a), N(b), abs_tol=1e-3): + return True + except: + pass + return False + + +def math_equal(prediction, reference, include_percentage=True, is_close=True): + """ + Exact match of math if and only if: + 1. numerical equal: both can convert to float and are equal + 2. symbolic equal: both can convert to sympy expression and are equal + """ + if str(prediction) == str(reference): + return True + + try: # 1. numerical equal + if is_digit(prediction) and is_digit(reference): + prediction = parse_digits(prediction) + reference = parse_digits(reference) + # number questions + if include_percentage: + gt_result = [reference / 100, reference, reference * 100] + else: + gt_result = [reference] + for item in gt_result: + try: + if is_close: + if isclose(item, prediction, abs_tol=1e-3): + return True + else: + if item == prediction: + return True + except Exception: + continue + return False + except: + pass + + if not prediction and prediction not in [0, False]: + return False + + # 2. symbolic equal + reference = str(reference).strip() + prediction = str(prediction).strip() + + if ( + regex.match(r"(\(|\[).+(\)|\])", prediction) is not None + and regex.match(r"(\(|\[).+(\)|\])", reference) is not None + ): + pred_parts = prediction[1:-1].split(",") + ref_parts = reference[1:-1].split(",") + if len(pred_parts) == len(ref_parts): + if all( + [ + math_equal( + pred_parts[i], ref_parts[i], include_percentage, is_close + ) + for i in range(len(pred_parts)) + ] + ): + return True + + # Add back matrix comparison + if ( + ( + prediction.startswith("\\begin{pmatrix}") + or prediction.startswith("\\begin{bmatrix}") + ) + and ( + prediction.endswith("\\end{pmatrix}") + or prediction.endswith("\\end{bmatrix}") + ) + and ( + reference.startswith("\\begin{pmatrix}") + or reference.startswith("\\begin{bmatrix}") + ) + and ( + reference.endswith("\\end{pmatrix}") or reference.endswith("\\end{bmatrix}") + ) + ): + pred_lines = [ + line.strip() + for line in prediction[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + ref_lines = [ + line.strip() + for line in reference[ + len("\\begin{pmatrix}") : -len("\\end{pmatrix}") + ].split("\\\\") + if line.strip() + ] + matched = True + if len(pred_lines) == len(ref_lines): + for pred_line, ref_line in zip(pred_lines, ref_lines): + pred_parts = pred_line.split("&") + ref_parts = ref_line.split("&") + if len(pred_parts) == len(ref_parts): + if not all( + [ + math_equal( + pred_parts[i], + ref_parts[i], + include_percentage, + is_close, + ) + for i in range(len(pred_parts)) + ] + ): + matched = False + break + else: + matched = False + if not matched: + break + else: + matched = False + if matched: + return True + + # Add back equation comparison + if prediction.count("=") == 1 and reference.count("=") == 1: + pred = prediction.split("=") + pred = f"{pred[0].strip()} - ({pred[1].strip()})" + ref = reference.split("=") + ref = f"{ref[0].strip()} - ({ref[1].strip()})" + if symbolic_equal(pred, ref) or symbolic_equal(f"-({pred})", ref): + return True + elif ( + prediction.count("=") == 1 + and len(prediction.split("=")[0].strip()) <= 2 + and "=" not in reference + ): + if math_equal( + prediction.split("=")[1], reference, include_percentage, is_close + ): + return True + elif ( + reference.count("=") == 1 + and len(reference.split("=")[0].strip()) <= 2 + and "=" not in prediction + ): + if math_equal( + prediction, reference.split("=")[1], include_percentage, is_close + ): + return True + + # symbolic equal with sympy + if symbolic_equal(prediction, reference): + return True + + return False diff --git a/benchmark/tip_suggestion/.gitignore b/benchmark/tip_suggestion/.gitignore new file mode 100644 index 000000000..3322de9b9 --- /dev/null +++ b/benchmark/tip_suggestion/.gitignore @@ -0,0 +1 @@ +!topic.jsonl diff --git a/benchmark/tip_suggestion/README.md b/benchmark/tip_suggestion/README.md new file mode 100644 index 000000000..97ebe1d82 --- /dev/null +++ b/benchmark/tip_suggestion/README.md @@ -0,0 +1,33 @@ +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 64 +python3 bench_sglang.py --num-questions 32 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --backend vllm --num-questions 64 +``` + + +### Benchmark guidance +``` +python3 bench_other.py --backend guidance --num-questions 32 --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --backend lmql --num-questions 32 --parallel 1 +``` diff --git a/benchmark/tip_suggestion/bench_other.py b/benchmark/tip_suggestion/bench_other.py new file mode 100644 index 000000000..2630081bd --- /dev/null +++ b/benchmark/tip_suggestion/bench_other.py @@ -0,0 +1,133 @@ +import argparse +import json +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +def expand_tip(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return generate(s, max_tokens=128, stop=["\n\n"]) + + +def suggest_tips(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + tip = generate(s, max_tokens=24, stop=[".", "\n"]) + s += tip + ".\n" + tips.append(tip) + + paragraphs = [expand_tip(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + states = [None] * len(lines) + + # Select backend + call_generate = partial(get_call_generate(args), temperature=0) + + # Run requests + tic = time.perf_counter() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = suggest_tips(lines[i]["topic"], call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(lines))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(lines)))), + total=len(lines), + ) + ) + + else: + import asyncio + + from lmql_funcs import suggest_tips_async + + async def get_one_answer_async(i): + states[i] = await suggest_tips_async(lines[i]["topic"], call_generate) + + batches = [] + for i in range(0, len(lines), args.parallel): + batches.append(list(range(i, min(i + args.parallel, len(lines))))) + loop = asyncio.get_event_loop() + for batch in tqdm(batches): + loop.run_until_complete( + asyncio.gather(*[get_one_answer_async(i) for i in batch]) + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tip_suggestion/bench_sglang.py b/benchmark/tip_suggestion/bench_sglang.py new file mode 100644 index 000000000..86c476f97 --- /dev/null +++ b/benchmark/tip_suggestion/bench_sglang.py @@ -0,0 +1,100 @@ +import argparse +import json +import time + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +number = 5 + + +@sgl.function +def expand_tip(s, topic, tip): + s += ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + s += sgl.gen("paragraph", max_tokens=128, stop=["\n\n"], temperature=0) + + +@sgl.function +def suggest_tips(s, topic): + s += "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + paragraphs = [] + for i in range(1, 1 + number): + s += f"{i}." + sgl.gen(f"tip_{i}", max_tokens=24, stop=[".", "\n"]) + ".\n" + paragraphs.append(expand_tip(topic=topic, tip=s[f"tip_{i}"])) + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1]["paragraph"] + "\n" + + +def main(args): + lines = read_jsonl(args.data_path)[: args.num_questions] + arguments = [{"topic": l["topic"]} for l in lines] + + # Select backend + sgl.set_default_backend(select_sglang_backend(args)) + + # Run requests + tic = time.perf_counter() + states = suggest_tips.run_batch( + arguments, temperature=0, num_threads=args.parallel, progress_bar=True + ) + latency = time.perf_counter() - tic + + # Compute accuracy + print(f"Latency: {latency:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", states) + + with open(args.result_file, "a") as fout: + value = { + "task": "tip_suggestion", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="topic.jsonl") + parser.add_argument("--num-questions", type=int, default=100) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/tip_suggestion/lmql_funcs.py b/benchmark/tip_suggestion/lmql_funcs.py new file mode 100644 index 000000000..7790bbe95 --- /dev/null +++ b/benchmark/tip_suggestion/lmql_funcs.py @@ -0,0 +1,50 @@ +number = 5 + + +async def expand_tip_async(topic, tip, generate): + s = ( + """Please expand a tip for a topic into a detailed paragraph. + +Topic: staying healthy +Tip: Regular Exercise +Paragraph: Incorporate physical activity into your daily routine. This doesn't necessarily mean intense gym workouts; it can be as simple as walking, cycling, or yoga. Regular exercise helps in maintaining a healthy weight, improves cardiovascular health, boosts mental health, and can enhance cognitive function, which is crucial for fields that require intense intellectual engagement. + +Topic: building a campfire +Tip: Choose the Right Location +Paragraph: Always build your campfire in a safe spot. This means selecting a location that's away from trees, bushes, and other flammable materials. Ideally, use a fire ring if available. If you're building a fire pit, it should be on bare soil or on a bed of stones, not on grass or near roots which can catch fire underground. Make sure the area above is clear of low-hanging branches. + +Topic: writing a blog post +Tip: structure your content effectively +Paragraph: A well-structured post is easier to read and more enjoyable. Start with an engaging introduction that hooks the reader and clearly states the purpose of your post. Use headings and subheadings to break up the text and guide readers through your content. Bullet points and numbered lists can make information more digestible. Ensure each paragraph flows logically into the next, and conclude with a summary or call-to-action that encourages reader engagement. + +Topic: """ + + topic + + "\nTip: " + + tip + + "\nParagraph:" + ) + return await generate(s, max_tokens=128, stop="\n\n") + + +async def suggest_tips_async(topic, generate): + s = "Please act as a helpful assistant. Your job is to provide users with useful tips on a specific topic.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(number) + + " concise tips, each under 8 words:\n" + ) + + tips = [] + for i in range(1, 1 + number): + s += f"{i}." + # NOTE: stop is different due to lmql does not support a list of stop tokens + tip = await generate(s, max_tokens=24, stop=".\n") + s += tip + ".\n" + tips.append(tip) + + paragraphs = [await expand_tip_async(topic, tip, generate=generate) for tip in tips] + + for i in range(1, 1 + number): + s += f"Tip {i}:" + paragraphs[i - 1] + "\n" + return s diff --git a/benchmark/tip_suggestion/topic.jsonl b/benchmark/tip_suggestion/topic.jsonl new file mode 100644 index 000000000..c8ac6d12e --- /dev/null +++ b/benchmark/tip_suggestion/topic.jsonl @@ -0,0 +1,50 @@ +{"topic": "organizing a successful charity event", "number": 6} +{"topic": "improving personal credit scores", "number": 7} +{"topic": "staying motivated during job searches", "number": 5} +{"topic": "maintaining a work-life balance", "number": 9} +{"topic": "reducing carbon footprint at home", "number": 8} +{"topic": "starting a book club", "number": 5} +{"topic": "learning to play a musical instrument", "number": 7} +{"topic": "getting into freelance writing", "number": 6} +{"topic": "beginner yoga poses", "number": 8} +{"topic": "preparing for graduate school exams", "number": 5} +{"topic": "exploring minimalist living", "number": 9} +{"topic": "effective grocery shopping", "number": 7} +{"topic": "winter camping", "number": 5} +{"topic": "starting a podcast on a budget", "number": 8} +{"topic": "creating a capsule wardrobe", "number": 6} +{"topic": "improving your writing skills", "number": 7} +{"topic": "learning a new software quickly", "number": 9} +{"topic": "reducing anxiety before public speaking", "number": 5} +{"topic": "planning a solo travel adventure", "number": 8} +{"topic": "beginner skateboarders", "number": 6} +{"topic": "studying abroad", "number": 7} +{"topic": "planting a vegetable garden", "number": 5} +{"topic": "adopting a shelter pet", "number": 9} +{"topic": "learning to cook ethnic cuisines", "number": 8} +{"topic": "effective conflict resolution", "number": 5} +{"topic": "starting a vlog", "number": 7} +{"topic": "keeping a daily journal", "number": 6} +{"topic": "improving sleep hygiene", "number": 8} +{"topic": "beginner mountain climbers", "number": 5} +{"topic": "creating a mobile app", "number": 9} +{"topic": "maintaining a saltwater aquarium", "number": 7} +{"topic": "preparing for a baby's arrival", "number": 6} +{"topic": "writing a fantasy novel", "number": 5} +{"topic": "effective team leadership", "number": 8} +{"topic": "making a documentary film", "number": 9} +{"topic": "learning about historical events", "number": 7} +{"topic": "baking gluten-free treats", "number": 6} +{"topic": "improving mental arithmetic skills", "number": 5} +{"topic": "building a treehouse", "number": 8} +{"topic": "getting started with watercolor painting", "number": 9} +{"topic": "creating a YouTube tutorial series", "number": 7} +{"topic": "landscape photography", "number": 5} +{"topic": "navigating cultural differences", "number": 6} +{"topic": "preparing for a marathon", "number": 8} +{"topic": "building an online business", "number": 9} +{"topic": "learning to dance at home", "number": 5} +{"topic": "self-publishing a book", "number": 7} +{"topic": "starting an urban farm", "number": 6} +{"topic": "improving your memory", "number": 8} +{"topic": "creating a personal brand online", "number": 9} diff --git a/benchmark/tree_of_thought_deep/README.md b/benchmark/tree_of_thought_deep/README.md new file mode 100644 index 000000000..bf5ab1638 --- /dev/null +++ b/benchmark/tree_of_thought_deep/README.md @@ -0,0 +1,51 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +NOTE: This is an implementation for throughput/latency benchmark purposes. The prompts are not tuned to achieve good accuracy on the GSM-8K tasks. + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 +python3 bench_sglang.py --num-questions 16 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 8 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` + +### Benchmark lmql + +``` +python3 bench_other.py --num-questions 8 --backend lmql --parallel 1 +``` diff --git a/benchmark/tree_of_thought_deep/bench_other.py b/benchmark/tree_of_thought_deep/bench_other.py new file mode 100644 index 000000000..0ef8c6360 --- /dev/null +++ b/benchmark/tree_of_thought_deep/bench_other.py @@ -0,0 +1,222 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def get_final_answer(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + plan_forks = propose_plan("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = get_final_answer(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + tic = time.perf_counter() + if args.backend != "lmql": + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + else: + import asyncio + + from lmql_funcs import tree_search_async + + async def get_one_answer_async(i): + states[i] = await tree_search_async( + **arguments[i], call_generate=call_generate + ) + + batches = [ + [] for _ in range((len(questions) + args.parallel - 1) // args.parallel) + ] + for i in range(len(questions)): + batches[i // args.parallel].append(i) + + loop = asyncio.get_event_loop() + for bt in tqdm(batches): + tasks = [get_one_answer_async(k) for k in bt] + loop.run_until_complete(asyncio.gather(*tasks)) + + latency = time.perf_counter() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_deep/bench_sglang.py b/benchmark/tree_of_thought_deep/bench_sglang.py new file mode 100644 index 000000000..bcdb6e54d --- /dev/null +++ b/benchmark/tree_of_thought_deep/bench_sglang.py @@ -0,0 +1,171 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.001 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +def get_final_answer(s, num_branches): + s += sgl.user( + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("final_answer", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + plan_forks = propose_plan(s, question, num_branches) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + forks = get_final_answer(sol, num_branches) + solutions.append(forks) + solutions = [[s.text() for s in forks] for forks in solutions] + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + lines = list(lines) + + # Construct prompts + num_branches = 2 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s.ret_value for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_deep/lmql_funcs.py b/benchmark/tree_of_thought_deep/lmql_funcs.py new file mode 100644 index 000000000..c783cdbe3 --- /dev/null +++ b/benchmark/tree_of_thought_deep/lmql_funcs.py @@ -0,0 +1,82 @@ +from bench_other import ( + ASSISTANT_PREFIX, + ASSISTANT_SUFFIX, + USER_PREFIX, + USER_SUFFIX, + temp, +) + + +async def propose_plan_async(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def execute_plan_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def reflect_solution_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now, evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def get_final_answer_async(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Based on your reflection, do you change your mind? Now, give me the final answer after careful consideration.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = await call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +async def tree_search_async(question, num_branches, call_generate): + plan_forks = await propose_plan_async("", question, num_branches, call_generate) + + sol_states = [] + for plan in plan_forks: + forks = await execute_plan_async(plan, num_branches, call_generate) + sol_states.extend(forks) + + ref_states = [] + for sol in sol_states: + forks = await reflect_solution_async(sol, num_branches, call_generate) + ref_states.extend(forks) + + solutions = [] + for sol in ref_states: + ans = await get_final_answer_async(sol, num_branches, call_generate) + solutions.append(ans) + + return solutions diff --git a/benchmark/tree_of_thought_v0/README.md b/benchmark/tree_of_thought_v0/README.md new file mode 100644 index 000000000..821bb20d1 --- /dev/null +++ b/benchmark/tree_of_thought_v0/README.md @@ -0,0 +1,43 @@ +## Download data +``` +wget https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl +``` + +## Run benchmark + +### Benchmark sglang +``` +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +``` + +``` +python3 bench_sglang.py --num-questions 32 --parallel 16 +python3 bench_sglang.py --num-questions 10 --parallel 1 +``` + + +### Benchmark vllm +``` +python3 -m vllm.entrypoints.api_server --tokenizer-mode auto --model meta-llama/Llama-2-7b-chat-hf --disable-log-requests --port 21000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend vllm +``` + + +### Benchmark lightllm +``` +# A10G +python -m lightllm.server.api_server --tokenizer_mode auto --model_dir ~/model_weights/llama-2-7b-chat-hf --max_total_token_num 16000 --port 22000 +``` + +``` +python3 bench_other.py --num-questions 32 --backend lightllm +``` + + +### Benchmark guidance +``` +python3 bench_other.py --num-questions 32 --backend guidance --parallel 1 --n-ctx 4096 --model-path path/to/gguf +``` diff --git a/benchmark/tree_of_thought_v0/bench_other.py b/benchmark/tree_of_thought_v0/bench_other.py new file mode 100644 index 000000000..703ecd7f4 --- /dev/null +++ b/benchmark/tree_of_thought_v0/bench_other.py @@ -0,0 +1,179 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +import numpy as np +from tqdm import tqdm + +from sglang.test.test_utils import add_common_other_args_and_parse, get_call_generate +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +USER_PREFIX = "[INST] " +USER_SUFFIX = " [/INST]" +ASSISTANT_PREFIX = "" +ASSISTANT_SUFFIX = " " + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches, call_generate): + s += ( + USER_PREFIX + + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + + USER_SUFFIX + ) + + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def execute_plan(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def reflect_solution(s, num_branches, call_generate): + s += ( + USER_PREFIX + + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + + USER_SUFFIX + ) + s += ASSISTANT_PREFIX + comps = call_generate( + s, max_tokens=256, temperature=temp, stop=None, n=num_branches + ) + return [s + comp + ASSISTANT_SUFFIX for comp in comps] + + +def tree_search(question, num_branches, call_generate): + s = "" + solutions = [] + + plan_forks = propose_plan(s, question, num_branches, call_generate) + for plan in plan_forks: + sol_forks = execute_plan(plan, num_branches, call_generate) + for sol in sol_forks: + score_forks = reflect_solution(sol, num_branches, call_generate) + solutions.append(sol_forks) + + return solutions + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + call_generate = get_call_generate(args) + + # Run requests + states = [None] * len(questions) + + def get_one_answer(i): + states[i] = tree_search(**arguments[i], call_generate=call_generate) + + tic = time.perf_counter() + if args.parallel == 1: + for i in tqdm(range(len(questions))): + get_one_answer(i) + else: + with ThreadPoolExecutor(args.parallel) as executor: + list( + tqdm( + executor.map(get_one_answer, list(range(len(questions)))), + total=len(questions), + ) + ) + + latency = time.perf_counter() - tic + + answers_text = [] + for s in states: + answers_text.append([x for xs in s for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_other_args_and_parse(parser) + main(args) diff --git a/benchmark/tree_of_thought_v0/bench_sglang.py b/benchmark/tree_of_thought_v0/bench_sglang.py new file mode 100644 index 000000000..6d7575f36 --- /dev/null +++ b/benchmark/tree_of_thought_v0/bench_sglang.py @@ -0,0 +1,159 @@ +import argparse +import ast +import json +import re +import time +from collections import Counter + +import numpy as np + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) +from sglang.utils import dump_state_text, read_jsonl + +INVALID = -9999999 + + +def get_answer_value(answer_str): + answer_str = answer_str.replace(",", "") + numbers = re.findall(r"\d+", answer_str) + if len(numbers) < 1: + return INVALID + try: + return ast.literal_eval(numbers[-1]) + except SyntaxError: + return INVALID + + +def most_frequent_number(numbers): + if not numbers: + return None + + frequency = Counter(numbers) + most_frequent = max(frequency, key=frequency.get) + return most_frequent + + +# Use a low temp to make the results more deterministic and the comparison more fair. +temp = 0.3 + + +def propose_plan(s, question, num_branches): + s += sgl.user( + """Please generate a high-level plan for solving the following question. As the first step, just say what method and idea you will use to solve the question. You can reorganize the information in the question. Do not do the actual calculation. Keep your response concise and within 80 words. Question: """ + + question + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("plan", max_tokens=256, temperature=temp)) + return forks + + +def execute_plan(s, num_branches): + s += sgl.user( + """The plan looks good! Now, use real numbers and do the calculation. Please solve the question step-by-step according to the high-level plan. Give me the final answer. Make your response short.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=temp)) + return forks + + +def reflect_solution(s, num_branches): + s += sgl.user( + """Okay. Now you evaluate your own solution and give it a score on a scale of 1 to 5. Please do rigorous check of the correctness.""" + ) + forks = s.fork(num_branches) + forks += sgl.assistant(sgl.gen("score", max_tokens=256, temperature=temp)) + return forks + + +@sgl.function +def tree_search(s, question, num_branches): + forks_to_join = [] + + plan_forks = propose_plan(s, question, num_branches) + forks_to_join.append(plan_forks) + + sol_states = [] + for plan in plan_forks: + forks = execute_plan(plan, num_branches) + forks_to_join.append(forks) + sol_states.extend(forks) + + for sol in sol_states: + forks = reflect_solution(sol, num_branches) + forks_to_join.append(forks) + + for f in reversed(forks_to_join): + f.join() + + +def main(args): + lines = read_jsonl(args.data_path) + + # Construct prompts + num_branches = 3 + questions = [] + labels = [] + for i in range(len(lines[: args.num_questions])): + questions.append(lines[i]["question"]) + labels.append(get_answer_value(lines[i]["answer"])) + assert all(l != INVALID for l in labels) + arguments = [{"question": q, "num_branches": num_branches} for q in questions] + + # Select backend + backend = select_sglang_backend(args) + + # Run requests + tic = time.perf_counter() + states = tree_search.run_batch( + arguments, + temperature=0, + backend=backend, + num_threads=args.parallel, + progress_bar=True, + ) + latency = time.perf_counter() - tic + answers_text = [] + for s in states: + answers_text.append([x for xs in s["answer"] for x in xs]) + + preds = [] + for i in range(len(states)): + answers = [get_answer_value(v) for v in answers_text[i]] + preds.append(most_frequent_number(answers)) + + # Compute accuracy + acc = np.mean(np.array(preds) == np.array(labels)) + invalid = np.mean(np.array(preds) == INVALID) + print(f"Latency: {latency:.3f}") + print(f"Invalid: {invalid:.3f}") + print(f"Accuracy: {acc:.3f}") + + # Write results + dump_state_text(f"tmp_output_{args.backend}.txt", answers_text) + + with open(args.result_file, "a") as fout: + value = { + "task": "tree_of_thought_gsm8k", + "backend": args.backend, + "num_gpus": 1, + "latency": round(latency, 3), + "accuracy": round(acc, 3), + "num_requests": args.num_questions, + "other": { + "num_questions": args.num_questions, + "parallel": args.parallel, + }, + } + fout.write(json.dumps(value) + "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--data-path", type=str, default="test.jsonl") + parser.add_argument("--num-questions", type=int, default=200) + args = add_common_sglang_args_and_parse(parser) + main(args) diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..3b9a420b3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,388 @@ +ARG CUDA_VERSION=12.9.1 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 as base + +ARG BUILD_TYPE=all +ARG BRANCH_TYPE=remote +ARG DEEPEP_COMMIT=b92d0d4860ce6866cd6d31bfbae937f9a7a3772b +ARG CMAKE_BUILD_PARALLEL_LEVEL=2 +ENV DEBIAN_FRONTEND=noninteractive \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ \ + NVSHMEM_DIR=/sgl-workspace/nvshmem/install +# Add GKE default lib and bin locations. +ENV PATH="${PATH}:/usr/local/nvidia/bin" \ + LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib:/usr/local/nvidia/lib64" + +RUN apt update && apt install wget -y && apt install software-properties-common -y \ + && add-apt-repository ppa:deadsnakes/ppa -y \ + && apt install python3.12-full python3.12-dev python3.10-venv -y \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 2 \ + && update-alternatives --set python3 /usr/bin/python3.12 \ + && wget https://bootstrap.pypa.io/get-pip.py \ + && python3 get-pip.py + +# Set timezone and install all packages +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + software-properties-common netcat-openbsd kmod unzip openssh-server \ + curl wget lsof zsh ccache tmux htop git-lfs tree \ + build-essential cmake \ + libopenmpi-dev libnuma1 libnuma-dev \ + libibverbs-dev libibverbs1 libibumad3 \ + librdmacm1 libnl-3-200 libnl-route-3-200 libnl-route-3-dev libnl-3-dev \ + ibverbs-providers infiniband-diags perftest \ + libgoogle-glog-dev libgtest-dev libjsoncpp-dev libunwind-dev \ + libboost-all-dev libssl-dev \ + libgrpc-dev libgrpc++-dev libprotobuf-dev protobuf-compiler protobuf-compiler-grpc \ + pybind11-dev \ + libhiredis-dev libcurl4-openssl-dev \ + libczmq4 libczmq-dev \ + libfabric-dev \ + patchelf \ + nvidia-dkms-550 \ + devscripts debhelper fakeroot dkms check libsubunit0 libsubunit-dev \ + && ln -sf /usr/bin/python3.12 /usr/bin/python \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# GDRCopy installation +RUN mkdir -p /tmp/gdrcopy && cd /tmp \ + && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ + && cd gdrcopy/packages \ + && CUDA=/usr/local/cuda ./build-deb-packages.sh \ + && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ + && cd / && rm -rf /tmp/gdrcopy + +# Fix DeepEP IBGDA symlink +RUN ln -sf /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so + +FROM scratch AS local_src +COPY . /src + +FROM base AS build-image +# Install SGLang +WORKDIR /sgl-workspace +ARG BRANCH_TYPE +COPY --from=local_src /src /tmp/local_src +RUN if [ "$BRANCH_TYPE" = "local" ]; then \ + cp -r /tmp/local_src /sgl-workspace/sglang; \ + else \ + git clone --depth=1 https://github.com/sgl-project/sglang.git /sgl-workspace/sglang; \ + fi \ + && rm -rf /tmp/local_src +RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel html5lib six \ + && cd sglang \ + && case "$CUDA_VERSION" in \ + 12.6.1) CUINDEX=126 ;; \ + 12.8.1) CUINDEX=128 ;; \ + 12.9.1) CUINDEX=129 ;; \ + *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ + esac \ + && python3 -m pip install --no-cache-dir -e "python[${BUILD_TYPE}]" --extra-index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m pip install --no-cache-dir nvidia-nccl-cu12==2.27.6 --force-reinstall --no-deps \ + && python3 -m flashinfer --download-cubin \ + && if [ "$CUDA_VERSION" = "12.6.1" ]; then \ + python3 -m pip install --no-cache-dir https://github.com/sgl-project/whl/releases/download/v0.3.9.post2/sgl_kernel-0.3.9.post2+cu124-cp310-abi3-manylinux2014_x86_64.whl --force-reinstall --no-deps ; \ + fi + +# Download source files +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + git clone https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout ${DEEPEP_COMMIT} && sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh && \ + cd .. && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + rm -f /sgl-workspace/nvshmem_src_cuda12-all-all-3.3.9.tar.gz + +# Build and install NVSHMEM +RUN cd /sgl-workspace/nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=${NVSHMEM_DIR} -DCMAKE_CUDA_ARCHITECTURES="90" && \ + cmake --build build --target install -j${CMAKE_BUILD_PARALLEL_LEVEL} + +# Install DeepEP +RUN cd /sgl-workspace/DeepEP && \ + case "$CUDA_VERSION" in \ + 12.6.1) \ + CHOSEN_TORCH_CUDA_ARCH_LIST='9.0' \ + ;; \ + 12.8.1|12.9.1) \ + CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0' \ + ;; \ + *) \ + echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 \ + ;; \ + esac && \ + NVSHMEM_DIR=${NVSHMEM_DIR} TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" pip install . + +# Python tools +RUN python3 -m pip install --no-cache-dir \ + datamodel_code_generator \ + mooncake-transfer-engine==0.3.5 \ + pre-commit \ + pytest \ + black \ + isort \ + icdiff \ + uv \ + wheel \ + scikit-build-core \ + nixl \ + py-spy + +# Install development tools and utilities +RUN apt-get update && apt-get install -y \ + gdb \ + ninja-build \ + vim \ + tmux \ + htop \ + wget \ + curl \ + locales \ + lsof \ + git \ + git-lfs \ + zsh \ + tree \ + silversearcher-ag \ + cloc \ + unzip \ + pkg-config \ + libssl-dev \ + bear \ + ccache \ + less \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +RUN apt update -y \ + && apt install -y --no-install-recommends gnupg \ + && echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/amd64 /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \ + && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub \ + && apt update -y \ + && apt install nsight-systems-cli -y + +# Set up locale +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +# Install minimal Python packages +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + pytest \ + black \ + isort \ + icdiff \ + scikit_build_core \ + uv \ + pre-commit \ + pandas \ + matplotlib \ + tabulate + +# Install diff-so-fancy +RUN curl -LSso /usr/local/bin/diff-so-fancy https://github.com/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \ + && chmod +x /usr/local/bin/diff-so-fancy + +# Install clang-format +RUN curl -LSso /usr/local/bin/clang-format https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \ + && chmod +x /usr/local/bin/clang-format + +# Install clangd +RUN curl -L https://github.com/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \ + && unzip clangd.zip \ + && cp -r clangd_18.1.3/bin/* /usr/local/bin/ \ + && cp -r clangd_18.1.3/lib/* /usr/local/lib/ \ + && rm -rf clangd_18.1.3 clangd.zip + +# Install CMake +RUN wget https://github.com/Kitware/CMake/releases/download/v3.31.1/cmake-3.31.1-linux-x86_64.tar.gz \ + && tar -xzf cmake-3.31.1-linux-x86_64.tar.gz \ + && cp -r cmake-3.31.1-linux-x86_64/bin/* /usr/local/bin/ \ + && cp -r cmake-3.31.1-linux-x86_64/share/* /usr/local/share/ \ + && rm -rf cmake-3.31.1-linux-x86_64 cmake-3.31.1-linux-x86_64.tar.gz + +# Install Rust toolchain for sgl-router +ENV PATH="/root/.cargo/bin:${PATH}" +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rustc --version && cargo --version + +# Build and install sgl-router +RUN python3 -m pip install --no-cache-dir setuptools-rust \ + && cd /sgl-workspace/sglang/sgl-router \ + && cargo build --release \ + && python3 -m pip install --no-cache-dir . \ + && rm -rf /root/.cache + + +# Add yank script +COPY --chown=root:root <<-"EOF" /usr/local/bin/yank +#!/bin/bash +put() { + esc=$1 + test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\" + printf "$esc" +} +put "\033]52;c;!\a" +buf=$( cat "$@" ) +len=$( printf %s "$buf" | wc -c ) max=74994 +test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2 +put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\r\n' )\a" +test -n "$TMUX" && tmux set-buffer "$buf" ||: +EOF + +RUN chmod +x /usr/local/bin/yank + +# Install oh-my-zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting + +# Configure Vim +COPY --chown=root:root <<-"EOF" /root/.vimrc +function! Yank(text) abort + let escape = system('yank', a:text) + if v:shell_error + echoerr escape + else + call writefile([escape], '/dev/tty', 'b') + endif +endfunction + +noremap y y:call Yank(@0) + +" automatically run yank(1) whenever yanking in Vim +function! CopyYank() abort + call Yank(join(v:event.regcontents, "\n")) +endfunction + +autocmd TextYankPost * call CopyYank() + +" Basic settings +set number +syntax on +set mouse=a +filetype indent on + +" Indentation +set autoindent nosmartindent +set smarttab +set expandtab +set shiftwidth=4 +set softtabstop=4 + +" Visual guides +set colorcolumn=120 +highlight ColorColumn ctermbg=5 + +" Status line +set laststatus=2 +set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P + +" Backspace behavior +set backspace=2 + +" Encoding +set encoding=utf-8 +set fileencoding=utf-8 +EOF + +# Configure tmux +COPY --chown=root:root <<-"EOF" /root/.tmux.conf +# Pane border styling +set -g pane-border-style fg='#742727',bg=black +set -g pane-active-border-style fg=red,bg=black + +# Status bar styling +set -g status-style bg='#0C8A92',fg=black + +# Change prefix key to backtick +set-option -g prefix ` +unbind C-b +bind-key ` send-prefix + +# Split panes using - and = with current path +unbind '"' +bind - splitw -v -c '#{pane_current_path}' +unbind '%' +bind = splitw -h -c '#{pane_current_path}' + +# Vi mode settings +bind-key -T copy-mode-vi Y send-keys -X copy-pipe 'yank > #{pane_tty}' +set-window-option -g mode-keys vi + +# Other settings +set-option -g escape-time 0 +set-option -g base-index 1 +set-window-option -g mouse on +set -g history-limit 100000 +EOF + +# Configure Git +RUN git config --global core.editor "vim" \ + && git config --global core.whitespace "fix,-indent-with-non-tab,trailing-space,cr-at-eol" \ + && git config --global core.pager "diff-so-fancy | less --tabs=4 -RFX" \ + && git config --global color.ui true \ + && git config --global color."diff-highlight".oldNormal "red bold" \ + && git config --global color."diff-highlight".oldHighlight "red bold 52" \ + && git config --global color."diff-highlight".newNormal "green bold" \ + && git config --global color."diff-highlight".newHighlight "green bold 22" \ + && git config --global color.diff.meta "11" \ + && git config --global color.diff.frag "magenta bold" \ + && git config --global color.diff.commit "yellow bold" \ + && git config --global color.diff.old "red bold" \ + && git config --global color.diff.new "green bold" \ + && git config --global color.diff.whitespace "red reverse" \ + && git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit --" \ + && git config --global http.sslVerify false \ + && git config --global pull.rebase true + +# Configure zsh +COPY --chown=root:root <<-"EOF" /root/.zshrc +export ZSH="/root/.oh-my-zsh" + +# Theme +ZSH_THEME="robbyrussell" + +# Plugins +plugins=( + git + z + zsh-autosuggestions + zsh-syntax-highlighting +) + +source $ZSH/oh-my-zsh.sh + +# Aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +alias vi='vim' + +# Enhanced history +HISTSIZE=10000 +SAVEHIST=10000 +setopt HIST_IGNORE_ALL_DUPS +setopt HIST_FIND_NO_DUPS +setopt INC_APPEND_HISTORY +EOF + +RUN set -euxo ; \ + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin + +# Set workspace directory +WORKDIR /sgl-workspace/sglang diff --git a/docker/Dockerfile.gb200 b/docker/Dockerfile.gb200 new file mode 100644 index 000000000..164326e23 --- /dev/null +++ b/docker/Dockerfile.gb200 @@ -0,0 +1,348 @@ +ARG CUDA_VERSION=12.9.1 +FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu22.04 + +ARG BUILD_TYPE=blackwell +ARG DEEPEP_COMMIT=1b14ad661c7640137fcfe93cccb2694ede1220b0 +ARG CMAKE_BUILD_PARALLEL_LEVEL=2 +ARG SGL_KERNEL_VERSION=0.3.9.post2 +ENV DEBIAN_FRONTEND=noninteractive \ + CUDA_HOME=/usr/local/cuda \ + GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ \ + NVSHMEM_DIR=/sgl-workspace/nvshmem/install \ + BUILD_TYPE=${BUILD_TYPE} \ + TORCH_CUDA_ARCH_LIST="10.0 12.0" + +# Set timezone and install all packages +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt-get update && apt-get install -y --no-install-recommends \ + tzdata \ + software-properties-common netcat-openbsd kmod unzip openssh-server \ + curl wget lsof zsh ccache tmux htop git-lfs tree \ + python3 python3-pip python3-dev libpython3-dev python3-venv \ + build-essential cmake \ + libopenmpi-dev libnuma1 libnuma-dev \ + libibverbs-dev libibverbs1 libibumad3 \ + librdmacm1 libnl-3-200 libnl-route-3-200 libnl-route-3-dev libnl-3-dev \ + ibverbs-providers infiniband-diags perftest \ + libgoogle-glog-dev libgtest-dev libjsoncpp-dev libunwind-dev \ + libboost-all-dev libssl-dev \ + libgrpc-dev libgrpc++-dev libprotobuf-dev protobuf-compiler-grpc \ + pybind11-dev \ + libhiredis-dev libcurl4-openssl-dev \ + libczmq4 libczmq-dev \ + libfabric-dev \ + patchelf \ + nvidia-dkms-550 \ + devscripts debhelper fakeroot dkms check libsubunit0 libsubunit-dev \ + && ln -sf /usr/bin/python3 /usr/bin/python \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +# Install SGLang missing package for blackwell build type +RUN python3 -m pip install openai httpx + +# GDRCopy installation +RUN mkdir -p /tmp/gdrcopy && cd /tmp \ + && git clone https://github.com/NVIDIA/gdrcopy.git -b v2.4.4 \ + && cd gdrcopy/packages \ + && CUDA=/usr/local/cuda ./build-deb-packages.sh \ + && dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \ + && cd / && rm -rf /tmp/gdrcopy + +# Fix DeepEP IBGDA symlink +RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so + +# Clone and install SGLang +WORKDIR /sgl-workspace +RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel html5lib six \ + && git clone --depth 1 https://github.com/sgl-project/sglang.git \ + && cd sglang \ + && case "$CUDA_VERSION" in \ + 12.9.1) CUINDEX=129 ;; \ + *) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \ + esac \ + && if [ "$CUDA_VERSION" = "12.9.1" ]; then \ + python3 -m pip install --no-cache-dir nvidia-nccl-cu12==2.27.6 --force-reinstall --no-deps ; \ + python3 -m pip install --no-cache-dir https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu129-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps ; \ + fi \ + && python3 -m pip install --no-cache-dir -e "python[${BUILD_TYPE}]" --extra-index-url https://download.pytorch.org/whl/cu${CUINDEX} \ + && python3 -m flashinfer --download-cubin + +# Download source files +RUN wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + git clone https://github.com/fzyzcjy/DeepEP.git && \ + cd DeepEP && git checkout ${DEEPEP_COMMIT} && cd .. && \ + tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz && \ + mv nvshmem_src nvshmem && \ + rm -f /sgl-workspace/nvshmem_src_cuda12-all-all-3.3.9.tar.gz + +# Build and install NVSHMEM +RUN cd /sgl-workspace/nvshmem && \ + NVSHMEM_SHMEM_SUPPORT=0 \ + NVSHMEM_UCX_SUPPORT=0 \ + NVSHMEM_USE_NCCL=0 \ + NVSHMEM_MPI_SUPPORT=0 \ + NVSHMEM_IBGDA_SUPPORT=1 \ + NVSHMEM_PMIX_SUPPORT=0 \ + NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ + NVSHMEM_USE_GDRCOPY=1 \ + cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=${NVSHMEM_DIR} -DCMAKE_CUDA_ARCHITECTURES="90;100;120" && \ + cmake --build build --target install -j${CMAKE_BUILD_PARALLEL_LEVEL} + +# Install DeepEP +RUN cd /sgl-workspace/DeepEP && \ + NVSHMEM_DIR=${NVSHMEM_DIR} pip install . + +# Python tools +RUN python3 -m pip install --no-cache-dir \ + datamodel_code_generator \ + mooncake-transfer-engine==0.3.5 \ + pre-commit \ + pytest \ + black \ + isort \ + icdiff \ + uv \ + wheel \ + scikit-build-core + +# Install nixl kv transfer backend +RUN python3 -m pip install --no-cache-dir \ + nixl + +# Install development tools and utilities +RUN apt-get update && apt-get install -y \ + gdb \ + ninja-build \ + vim \ + tmux \ + htop \ + wget \ + curl \ + locales \ + lsof \ + git \ + git-lfs \ + zsh \ + tree \ + silversearcher-ag \ + cloc \ + unzip \ + pkg-config \ + libssl-dev \ + bear \ + ccache \ + less \ + && apt install -y rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean + +RUN apt update -y \ + && apt install -y --no-install-recommends gnupg \ + && echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu2004/$(if [ "$(uname -m)" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list \ + && apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/$(if [ "$(uname -m)" = "aarch64" ]; then echo "arm64"; else echo "x86_64"; fi)/7fa2af80.pub \ + && apt update -y \ + && apt install nsight-systems-cli -y + +# Set up locale +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + +# Install minimal Python packages +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + pytest \ + black \ + isort \ + icdiff \ + scikit_build_core \ + uv \ + pre-commit \ + pandas \ + matplotlib \ + tabulate + +# Install diff-so-fancy +RUN curl -LSso /usr/local/bin/diff-so-fancy https://github.com/so-fancy/diff-so-fancy/releases/download/v1.4.4/diff-so-fancy \ + && chmod +x /usr/local/bin/diff-so-fancy + +# Install clang-format +RUN curl -LSso /usr/local/bin/clang-format https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-32d3ac78/clang-format-16_linux-amd64 \ + && chmod +x /usr/local/bin/clang-format + +# Install clangd +RUN curl -L https://github.com/clangd/clangd/releases/download/18.1.3/clangd-linux-18.1.3.zip -o clangd.zip \ + && unzip clangd.zip \ + && cp -r clangd_18.1.3/bin/* /usr/local/bin/ \ + && cp -r clangd_18.1.3/lib/* /usr/local/lib/ \ + && rm -rf clangd_18.1.3 clangd.zip + +# Install CMake +RUN CMAKE_VERSION=3.31.1 \ + && ARCH=$(uname -m) \ + && CMAKE_INSTALLER="cmake-${CMAKE_VERSION}-linux-${ARCH}" \ + && wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/${CMAKE_INSTALLER}.tar.gz" \ + && tar -xzf "${CMAKE_INSTALLER}.tar.gz" \ + && cp -r "${CMAKE_INSTALLER}/bin/"* /usr/local/bin/ \ + && cp -r "${CMAKE_INSTALLER}/share/"* /usr/local/share/ \ + && rm -rf "${CMAKE_INSTALLER}" "${CMAKE_INSTALLER}.tar.gz" + +# Add yank script +COPY --chown=root:root <<-"EOF" /usr/local/bin/yank +#!/bin/bash +put() { + esc=$1 + test -n "$TMUX" -o -z "${TERM##screen*}" && esc="\033Ptmux;\033$esc\033\\" + printf "$esc" +} +put "\033]52;c;!\a" +buf=$( cat "$@" ) +len=$( printf %s "$buf" | wc -c ) max=74994 +test $len -gt $max && echo "$0: input is $(( len - max )) bytes too long" >&2 +put "\033]52;c;$( printf %s "$buf" | head -c $max | base64 | tr -d '\r\n' )\a" +test -n "$TMUX" && tmux set-buffer "$buf" ||: +EOF + +RUN chmod +x /usr/local/bin/yank + +# Install oh-my-zsh and plugins +RUN sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" "" --unattended \ + && git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-autosuggestions \ + && git clone https://github.com/zsh-users/zsh-syntax-highlighting.git ${ZSH_CUSTOM:-~/.oh-my-zsh/custom}/plugins/zsh-syntax-highlighting + +# Configure Vim +COPY --chown=root:root <<-"EOF" /root/.vimrc +function! Yank(text) abort + let escape = system('yank', a:text) + if v:shell_error + echoerr escape + else + call writefile([escape], '/dev/tty', 'b') + endif +endfunction + +noremap y y:call Yank(@0) + +" automatically run yank(1) whenever yanking in Vim +function! CopyYank() abort + call Yank(join(v:event.regcontents, "\n")) +endfunction + +autocmd TextYankPost * call CopyYank() + +" Basic settings +set number +syntax on +set mouse=a +filetype indent on + +" Indentation +set autoindent nosmartindent +set smarttab +set expandtab +set shiftwidth=4 +set softtabstop=4 + +" Visual guides +set colorcolumn=120 +highlight ColorColumn ctermbg=5 + +" Status line +set laststatus=2 +set statusline=%<%f\ %h%m%r%=%{\"[\".(&fenc==\"\"?&enc:&fenc).((exists(\"+bomb\")\ &&\ &bomb)?\",B\":\"\").\"]\ \"}%k\ %-14.(%l,%c%V%)\ %P + +" Backspace behavior +set backspace=2 + +" Encoding +set encoding=utf-8 +set fileencoding=utf-8 +EOF + +# Configure tmux +COPY --chown=root:root <<-"EOF" /root/.tmux.conf +# Pane border styling +set -g pane-border-style fg='#742727',bg=black +set -g pane-active-border-style fg=red,bg=black + +# Status bar styling +set -g status-style bg='#0C8A92',fg=black + +# Change prefix key to backtick +set-option -g prefix ` +unbind C-b +bind-key ` send-prefix + +# Split panes using - and = with current path +unbind '"' +bind - splitw -v -c '#{pane_current_path}' +unbind '%' +bind = splitw -h -c '#{pane_current_path}' + +# Vi mode settings +bind-key -T copy-mode-vi Y send-keys -X copy-pipe 'yank > #{pane_tty}' +set-window-option -g mode-keys vi + +# Other settings +set-option -g escape-time 0 +set-option -g base-index 1 +set-window-option -g mouse on +EOF + +# Configure Git +RUN git config --global core.editor "vim" \ + && git config --global core.whitespace "fix,-indent-with-non-tab,trailing-space,cr-at-eol" \ + && git config --global core.pager "diff-so-fancy | less --tabs=4 -RFX" \ + && git config --global color.ui true \ + && git config --global color."diff-highlight".oldNormal "red bold" \ + && git config --global color."diff-highlight".oldHighlight "red bold 52" \ + && git config --global color."diff-highlight".newNormal "green bold" \ + && git config --global color."diff-highlight".newHighlight "green bold 22" \ + && git config --global color.diff.meta "11" \ + && git config --global color.diff.frag "magenta bold" \ + && git config --global color.diff.commit "yellow bold" \ + && git config --global color.diff.old "red bold" \ + && git config --global color.diff.new "green bold" \ + && git config --global color.diff.whitespace "red reverse" \ + && git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset - %s %Cgreen(%cr) %C(bold blue)<%an>%Creset%C(auto)%d%Creset' --abbrev-commit --" \ + && git config --global http.sslVerify false \ + && git config --global pull.rebase true + +# Configure zsh +COPY --chown=root:root <<-"EOF" /root/.zshrc +export ZSH="/root/.oh-my-zsh" + +# Theme +ZSH_THEME="robbyrussell" + +# Plugins +plugins=( + git + z + zsh-autosuggestions + zsh-syntax-highlighting +) + +source $ZSH/oh-my-zsh.sh + +# Aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +alias vi='vim' + +# Enhanced history +HISTSIZE=10000 +SAVEHIST=10000 +setopt HIST_IGNORE_ALL_DUPS +setopt HIST_FIND_NO_DUPS +setopt INC_APPEND_HISTORY +EOF + +RUN set -euxo ; \ + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin + +# Set workspace directory +WORKDIR /sgl-workspace/sglang diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu new file mode 100644 index 000000000..3f9b0ae42 --- /dev/null +++ b/docker/Dockerfile.npu @@ -0,0 +1,93 @@ +ARG CANN_VERSION=8.2.rc1 +ARG DEVICE_TYPE=a3 +ARG OS=ubuntu22.04 +ARG PYTHON_VERSION=py3.11 + +FROM quay.io/ascend/cann:$CANN_VERSION-$DEVICE_TYPE-$OS-$PYTHON_VERSION + +# Update pip & apt sources +ARG PIP_INDEX_URL="https://pypi.org/simple/" +ARG APTMIRROR="" +ARG MEMFABRIC_URL=https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl +ARG PYTORCH_VERSION=2.6.0 +ARG TORCHVISION_VERSION=0.21.0 +ARG PTA_URL="https://gitee.com/ascend/pytorch/releases/download/v7.1.0.1-pytorch2.6.0/torch_npu-2.6.0.post1-cp311-cp311-manylinux_2_28_aarch64.whl" +ARG VLLM_TAG=v0.8.5 +ARG TRITON_ASCEND_URL=https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/triton_ascend-3.2.0.dev20250729-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl +ARG SGLANG_TAG=main +ARG ASCEND_CANN_PATH=/usr/local/Ascend/ascend-toolkit +ARG SGLANG_KERNEL_NPU_TAG=main + +WORKDIR /workspace + +# Define environments +ENV DEBIAN_FRONTEND=noninteractive + +RUN pip config set global.index-url $PIP_INDEX_URL +RUN if [ -n "$APTMIRROR" ];then sed -i "s|.*.ubuntu.com|$APTMIRROR|g" /etc/apt/sources.list ;fi + +# Install development tools and utilities +RUN apt-get update -y && apt upgrade -y && apt-get install -y \ + build-essential \ + cmake \ + vim \ + wget \ + curl \ + net-tools \ + zlib1g-dev \ + lld \ + clang \ + locales \ + ccache \ + openssl \ + libssl-dev \ + pkg-config \ + ca-certificates \ + protobuf-compiler \ + && rm -rf /var/cache/apt/* \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates \ + && locale-gen en_US.UTF-8 + +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install dependencies +# TODO: install from pypi released memfabric +RUN pip install $MEMFABRIC_URL --no-cache-dir + +RUN pip install setuptools-rust wheel build --no-cache-dir + +# install rustup from rustup.rs +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rustc --version && cargo --version && protoc --version + +# Install vLLM +RUN git clone --depth 1 https://github.com/vllm-project/vllm.git --branch $VLLM_TAG && \ + (cd vllm && VLLM_TARGET_DEVICE="empty" pip install -v . --no-cache-dir) && rm -rf vllm + +# TODO: install from pypi released triton-ascend +RUN pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu --no-cache-dir \ + && wget ${PTA_URL} && pip install "./torch_npu-2.6.0.post1-cp311-cp311-manylinux_2_28_aarch64.whl" --no-cache-dir \ + && python3 -m pip install --no-cache-dir attrs==24.2.0 numpy==1.26.4 scipy==1.13.1 decorator==5.1.1 psutil==6.0.0 pytest==8.3.2 pytest-xdist==3.6.1 pyyaml pybind11 \ + && pip install ${TRITON_ASCEND_URL} --no-cache-dir + +# Install SGLang +RUN git clone https://github.com/sgl-project/sglang --branch $SGLANG_TAG && \ + (cd sglang/python && pip install -v .[srt_npu] --no-cache-dir) && \ + (cd sglang/sgl-router && python -m build && pip install --force-reinstall dist/*.whl) && \ + rm -rf sglang + +# Install Deep-ep +RUN git clone --branch $SGLANG_KERNEL_NPU_TAG https://github.com/sgl-project/sgl-kernel-npu.git \ + && export LD_LIBRARY_PATH=${ASCEND_CANN_PATH}/latest/runtime/lib64/stub:$LD_LIBRARY_PATH && \ + source ${ASCEND_CANN_PATH}/set_env.sh && \ + cd sgl-kernel-npu && \ + bash build.sh \ + && pip install output/deep_ep*.whl --no-cache-dir \ + && cd .. && rm -rf sgl-kernel-npu \ + && cd "$(pip show deep-ep | awk '/^Location:/ {print $2}')" && ln -s deep_ep/deep_ep_cpp*.so + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm new file mode 100644 index 000000000..2c3c9c0be --- /dev/null +++ b/docker/Dockerfile.rocm @@ -0,0 +1,213 @@ +# Usage (to build SGLang ROCm docker image): +# docker build --build-arg SGL_BRANCH=v0.5.2 --build-arg GPU_ARCH=gfx942 -t v0.5.2-rocm630-mi30x -f Dockerfile.rocm . +# docker build --build-arg SGL_BRANCH=v0.5.2 --build-arg GPU_ARCH=gfx942-rocm700 -t v0.5.2-rocm700-mi30x -f Dockerfile.rocm . +# docker build --build-arg SGL_BRANCH=v0.5.2 --build-arg GPU_ARCH=gfx950 -t v0.5.2-rocm700-mi35x -f Dockerfile.rocm . + + +# Default base images +ARG BASE_IMAGE_942="rocm/sgl-dev:vllm20250114" +ARG BASE_IMAGE_942_ROCM700="rocm/sgl-dev:rocm7-vllm-20250904" +ARG BASE_IMAGE_950="rocm/sgl-dev:rocm7-vllm-20250904" + +# This is necessary for scope purpose +ARG GPU_ARCH=gfx950 + +# =============================== +# Base image 942 with rocm630 and args +FROM $BASE_IMAGE_942 AS gfx942 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="1" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.4" +ENV NO_DEPS_FLAG="" + +# =============================== +# Base image 942 and args +FROM $BASE_IMAGE_942_ROCM700 AS gfx942-rocm700 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.5" +ENV NO_DEPS_FLAG="" + +# =============================== +# Base image 950 and args +FROM $BASE_IMAGE_950 AS gfx950 +ENV BUILD_VLLM="0" +ENV BUILD_TRITON="0" +ENV BUILD_LLVM="0" +ENV BUILD_AITER_ALL="1" +ENV BUILD_MOONCAKE="1" +ENV AITER_COMMIT="v0.1.5" +ENV NO_DEPS_FLAG="--no-deps" + +# =============================== +# Chosen arch and args +FROM ${GPU_ARCH} + +# This is necessary for scope purpose, again +ARG GPU_ARCH=gfx950 +ENV GPU_ARCH_LIST=${GPU_ARCH%-*} + +ARG SGL_REPO="https://github.com/sgl-project/sglang.git" +ARG SGL_DEFAULT="main" +ARG SGL_BRANCH=${SGL_DEFAULT} + +ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG TRITON_COMMIT="improve_fa_decode_3.0.0" + +ARG AITER_REPO="https://github.com/ROCm/aiter.git" + +ARG LLVM_REPO="https://github.com/jrbyrnes/llvm-project.git" +ARG LLVM_BRANCH="MainOpSelV2" +ARG LLVM_COMMIT="6520ace8227ffe2728148d5f3b9872a870b0a560" + +ARG MOONCAKE_REPO="https://github.com/kvcache-ai/Mooncake.git" +ARG MOONCAKE_COMMIT="dcdf1c784b40aa6975a8ed89fe26321b028e40e8" + +USER root + +# Install some basic utilities +RUN python -m pip install --upgrade pip && pip install setuptools_scm +RUN apt-get purge -y sccache; python -m pip uninstall -y sccache; rm -f "$(which sccache)" + +WORKDIR /sgl-workspace + +# ----------------------- +# llvm +RUN if [ "$BUILD_LLVM" = "1" ]; then \ + ENV HIP_CLANG_PATH="/sgl-workspace/llvm-project/build/bin/" \ + git clone --single-branch ${LLVM_REPO} -b ${LLVM_BRANCH} \ + && cd llvm-project \ + && git checkout ${LLVM_COMMIT} \ + && mkdir build \ + && cd build \ + && cmake -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=1 -DLLVM_TARGETS_TO_BUILD="AMDGPU;X86" -DLLVM_ENABLE_PROJECTS="clang;lld;" -DLLVM_ENABLE_RUNTIMES="compiler-rt" ../llvm \ + && make -j$(nproc); \ + fi + +# ----------------------- + +# ----------------------- +# AITER +RUN pip uninstall -y aiter +RUN git clone ${AITER_REPO} \ + && cd aiter \ + && git checkout ${AITER_COMMIT} \ + && git submodule update --init --recursive +RUN cd aiter \ + && if [ "$BUILD_AITER_ALL" = "1" ] && [ "$BUILD_LLVM" = "1" ]; then \ + HIP_CLANG_PATH=/sgl-workspace/llvm-project/build/bin/ PREBUILD_KERNELS=1 GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop; \ + elif [ "$BUILD_AITER_ALL" = "1" ]; then \ + PREBUILD_KERNELS=1 GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop; \ + else \ + GPU_ARCHS=$GPU_ARCH_LIST python setup.py develop; \ + fi + +# ----------------------- +# Triton +RUN if [ "$BUILD_TRITON" = "1" ]; then \ + pip uninstall -y triton \ + && git clone ${TRITON_REPO} \ + && cd triton \ + && git checkout ${TRITON_COMMIT} \ + && cd python \ + && python setup.py install; \ + fi + +# ----------------------- +# Build vLLM +ARG VLLM_REPO="https://github.com/ROCm/vllm.git" +ARG VLLM_BRANCH="9f6b92db47c3444b7a7d67451ba0c3a2d6af4c2c" +RUN if [ "$BUILD_VLLM" = "1" ]; then \ + git clone ${VLLM_REPO} \ + && cd vllm \ + && git checkout ${VLLM_BRANCH} \ + && python -m pip install -r requirements/rocm.txt \ + && python setup.py clean --all \ + && python setup.py develop; \ + fi + +# ----------------------- +# Build Mooncake +ENV PATH=$PATH:/usr/local/go/bin + +RUN if [ "$BUILD_MOONCAKE" = "1" ]; then \ + apt update && apt install -y zip unzip wget && \ + apt install -y gcc make libtool autoconf librdmacm-dev rdmacm-utils infiniband-diags ibverbs-utils perftest ethtool libibverbs-dev rdma-core && \ + apt install -y openssh-server openmpi-bin openmpi-common libopenmpi-dev && \ + git clone ${MOONCAKE_REPO} && \ + cd Mooncake && \ + git checkout ${MOONCAKE_COMMIT} && \ + git submodule update --init --recursive && \ + bash dependencies.sh -y && \ + rm -rf /usr/local/go && \ + wget https://go.dev/dl/go1.22.2.linux-amd64.tar.gz && \ + tar -C /usr/local -xzf go1.22.2.linux-amd64.tar.gz && \ + rm go1.22.2.linux-amd64.tar.gz && \ + mkdir -p build && \ + cd build && \ + cmake .. -DUSE_ETCD=ON && \ + make -j "$(nproc)" && make install; \ + fi + + +# ----------------------- +# Build SGLang +ARG BUILD_TYPE=all + +RUN pip install IPython \ + && pip install orjson \ + && pip install python-multipart \ + && pip install torchao==0.9.0 \ + && pip install pybind11 + +RUN pip uninstall -y sgl_kernel sglang +RUN git clone ${SGL_REPO} \ + && cd sglang \ + && if [ "${SGL_BRANCH}" = ${SGL_DEFAULT} ]; then \ + echo "Using ${SGL_DEFAULT}, default branch."; \ + git checkout ${SGL_DEFAULT}; \ + else \ + echo "Using ${SGL_BRANCH} branch."; \ + git checkout ${SGL_BRANCH}; \ + fi \ + && cd sgl-kernel \ + && rm -f pyproject.toml \ + && mv pyproject_rocm.toml pyproject.toml \ + && AMDGPU_TARGET=$GPU_ARCH_LIST python setup_rocm.py install \ + && cd .. \ + && if [ "$BUILD_TYPE" = "srt" ]; then \ + python -m pip --no-cache-dir install -e "python[srt_hip]" ${NO_DEPS_FLAG}; \ + else \ + python -m pip --no-cache-dir install -e "python[all_hip]" ${NO_DEPS_FLAG}; \ + fi + +RUN python -m pip cache purge + +# Copy config files to support MI300X in virtualized environments (MI300X_VF). Symlinks will not be created in image build. +RUN find /sgl-workspace/sglang/python/sglang/srt/layers/quantization/configs/ \ + /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs/ \ + -type f -name '*MI300X*' | xargs -I {} sh -c 'vf_config=$(echo "$1" | sed "s/MI300X/MI300X_VF/"); cp "$1" "$vf_config"' -- {} + +# Performance environment variable. +ENV HIP_FORCE_DEV_KERNARG=1 +ENV HSA_NO_SCRATCH_RECLAIM=1 +ENV SGLANG_SET_CPU_AFFINITY=1 +ENV SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 +ENV NCCL_MIN_NCHANNELS=112 + +ENV SGLANG_USE_AITER=1 +ENV SGLANG_MOE_PADDING=1 +ENV VLLM_FP8_PADDING=1 +ENV VLLM_FP8_ACT_PADDING=1 +ENV VLLM_FP8_WEIGHT_PADDING=1 +ENV VLLM_FP8_REDUCE_CONV=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE=1 +ENV TORCHINDUCTOR_MAX_AUTOTUNE_POINTWISE=1 + +CMD ["/bin/bash"] diff --git a/docker/Dockerfile.router b/docker/Dockerfile.router new file mode 100644 index 000000000..ded98bb8a --- /dev/null +++ b/docker/Dockerfile.router @@ -0,0 +1,78 @@ +######################## BASE IMAGE ########################## +FROM ubuntu:24.04 AS base + +ARG PYTHON_VERSION=3.12 + +# set the environment variables +ENV PATH="/root/.local/bin:${PATH}" +ENV DEBIAN_FRONTEND=noninteractive + +# uv environment variables +ENV UV_HTTP_TIMEOUT=500 +ENV VIRTUAL_ENV="/opt/venv" +ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python +ENV UV_LINK_MODE="copy" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + + +# install dependencies +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt update -y \ + && apt install -y curl \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# install uv +RUN curl -LsSf https://astral.sh/uv/install.sh | sh + +# install python +RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} + +######################### BUILD IMAGE ######################### +FROM base AS build-image + +ARG SGLANG_REPO_REF=main + +# set the environment variables +ENV PATH="/root/.cargo/bin:${PATH}" + +# install dependencies +RUN apt update -y \ + && apt install -y git build-essential libssl-dev pkg-config protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* \ + && apt clean + +# install rustup from rustup.rs +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y \ + && rustc --version && cargo --version && protoc --version + +# pull the github repository +RUN cd /opt \ + && git clone --depth=1 https://github.com/sgl-project/sglang.git \ + && cd /opt/sglang \ + && git checkout ${SGLANG_REPO_REF} + +# working directory +WORKDIR /opt/sglang/sgl-router + +# build the rust dependencies +RUN cargo build --release \ + && uv build \ + && rm -rf /root/.cache + +######################### ROUTER IMAGE ######################### +FROM base AS router-image + +# Copy the built package from the build image +COPY --from=build-image /opt/sglang/sgl-router/dist/*.whl dist/ + +# Build the package and install +RUN uv pip install --force-reinstall dist/*.whl + +# Clean up unnecessary files to reduce the image size +RUN rm -rf /root/.cache \ + && apt purge -y --auto-remove curl + +# Set the entrypoint to the main command +ENTRYPOINT ["python3", "-m", "sglang_router.launch_router"] diff --git a/docker/Dockerfile.sagemaker b/docker/Dockerfile.sagemaker new file mode 100644 index 000000000..5fbff5096 --- /dev/null +++ b/docker/Dockerfile.sagemaker @@ -0,0 +1,6 @@ +FROM lmsysorg/sglang:latest + +COPY serve /usr/bin/serve +RUN chmod 777 /usr/bin/serve + +ENTRYPOINT [ "/usr/bin/serve" ] diff --git a/docker/Dockerfile.xeon b/docker/Dockerfile.xeon new file mode 100644 index 000000000..fdc439b30 --- /dev/null +++ b/docker/Dockerfile.xeon @@ -0,0 +1,48 @@ +FROM ubuntu:24.04 +SHELL ["/bin/bash", "-c"] + +ARG VER_SGLANG=main +ARG VER_TORCH=2.7.1 +ARG VER_TORCHVISION=0.22.1 +ARG VER_TRITON=3.3.1 + +RUN apt-get update && \ + apt-get full-upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + git \ + curl \ + wget \ + vim \ + gcc \ + g++ \ + make + +WORKDIR /sgl-workspace + +RUN curl -fsSL -v -o miniforge.sh -O https://github.com/conda-forge/miniforge/releases/download/24.11.3-2/Miniforge3-24.11.3-2-Linux-x86_64.sh && \ + bash miniforge.sh -b -p ./miniforge3 && \ + rm -f miniforge.sh && \ + . miniforge3/bin/activate && \ + conda install -y libsqlite==3.48.0 gperftools tbb libnuma numactl + +ENV PATH=/sgl-workspace/miniforge3/bin:/sgl-workspace/miniforge3/condabin:${PATH} +ENV PIP_ROOT_USER_ACTION=ignore +ENV CONDA_PREFIX=/sgl-workspace/miniforge3 + +RUN pip config set global.index-url https://download.pytorch.org/whl/cpu && \ + pip config set global.extra-index-url https://pypi.org/simple + +RUN git clone https://github.com/sgl-project/sglang.git && \ + cd sglang && \ + git checkout ${VER_SGLANG} && \ + pip install -e "python[all_cpu]" && \ + pip install torch==${VER_TORCH} torchvision==${VER_TORCHVISION} triton==${VER_TRITON} --force-reinstall && \ + cd sgl-kernel && \ + cp pyproject_cpu.toml pyproject.toml && \ + pip install . + +ENV SGLANG_USE_CPU_ENGINE=1 +ENV LD_PRELOAD=/sgl-workspace/miniforge3/lib/libiomp5.so:/sgl-workspace/miniforge3/lib/libtcmalloc.so:/sgl-workspace/miniforge3/lib/libtbbmalloc.so.2 + +WORKDIR /sgl-workspace/sglang diff --git a/docker/compose.yaml b/docker/compose.yaml new file mode 100644 index 000000000..f7ff1fbd5 --- /dev/null +++ b/docker/compose.yaml @@ -0,0 +1,35 @@ +services: + sglang: + image: lmsysorg/sglang:latest + container_name: sglang + volumes: + - ${HOME}/.cache/huggingface:/root/.cache/huggingface + # If you use modelscope, you need mount this directory + # - ${HOME}/.cache/modelscope:/root/.cache/modelscope + restart: always + network_mode: host # required by RDMA + privileged: true # required by RDMA + # Or you can only publish port 30000 + # ports: + # - 30000:30000 + environment: + HF_TOKEN: + # if you use modelscope to download model, you need set this environment + # - SGLANG_USE_MODELSCOPE: true + entrypoint: python3 -m sglang.launch_server + command: --model-path meta-llama/Llama-3.1-8B-Instruct + --host 0.0.0.0 + --port 30000 + ulimits: + memlock: -1 + stack: 67108864 + ipc: host + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:30000/health || exit 1"] + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["0"] + capabilities: [gpu] diff --git a/docker/k8s-sglang-distributed-sts.yaml b/docker/k8s-sglang-distributed-sts.yaml new file mode 100644 index 000000000..4252363c7 --- /dev/null +++ b/docker/k8s-sglang-distributed-sts.yaml @@ -0,0 +1,103 @@ +# Two Nodes Sglang example + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: distributed-sglang +spec: + replicas: 2 # number of nodes/pods to run distributed sglang + selector: + matchLabels: + app: distributed-sglang + serviceName: "" + template: + metadata: + labels: + app: distributed-sglang + spec: + containers: + - name: sglang-container + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # image may be replaced by official CI versioned image + command: + - /bin/bash + - -c + # please modify the sglang serving arguments below, as necessary. + # NOTE: the --expert-parallel-size is for MoE model like DeepSeek-R1 + args: + - | + python3 -m sglang.launch_server \ + --model /llm-folder \ + --dist-init-addr sglang-master-pod:5000 \ + --tensor-parallel-size 16 \ + --nnodes 2 \ + --node-rank $POD_INDEX \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 8000 \ + --enable-metrics \ + --expert-parallel-size 16 + env: + - name: POD_INDEX # reflects the node-rank + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.labels['apps.kubernetes.io/pod-index'] + - name: NCCL_DEBUG + value: INFO + resources: + limits: + nvidia.com/gpu: "8" + requests: + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /llm-folder + name: llm + securityContext: + privileged: true # to leverage RDMA/InfiniBand device, co-work with HostNetwork=true + hostNetwork: true + volumes: + - emptyDir: + medium: Memory + sizeLimit: 10Gi + name: dshm + - hostPath: + path: /llm-folder # replace with PVC or hostPath with your model weights + type: DirectoryOrCreate + name: llm + #- persistentVolumeClaim: + # claimName: llm-pvc + # name: llm +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-master-pod +spec: + type: ClusterIP + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: dist-port + port: 5000 + targetPort: 5000 +--- +# the serving service +apiVersion: v1 +kind: Service +metadata: + name: sglang-serving-on-master +spec: + type: NodePort + selector: + app: distributed-sglang + apps.kubernetes.io/pod-index: "0" + ports: + - name: serving + port: 8000 + targetPort: 8000 + - name: metrics + port: 8080 + targetPort: 8080 diff --git a/docker/k8s-sglang-service.yaml b/docker/k8s-sglang-service.yaml new file mode 100644 index 000000000..866d50be9 --- /dev/null +++ b/docker/k8s-sglang-service.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: llama-31-8b-sglang +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 30Gi + storageClassName: default # change this to your preferred storage class + volumeMode: Filesystem +--- +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: nvidia +handler: nvidia +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: meta-llama-31-8b-instruct-sglang + template: + metadata: + labels: + app: meta-llama-31-8b-instruct-sglang + model: meta-llama-31-8b-instruct + engine: sglang + spec: + restartPolicy: Always + runtimeClassName: nvidia + containers: + - name: meta-llama-31-8b-instruct-sglang + image: docker.io/lmsysorg/sglang:latest + imagePullPolicy: Always # IfNotPresent or Never + ports: + - containerPort: 30000 + command: ["python3", "-m", "sglang.launch_server"] + args: + [ + "--model-path", + "meta-llama/Llama-3.1-8B-Instruct", + "--host", + "0.0.0.0", + "--port", + "30000", + ] + env: + - name: HF_TOKEN + value: + resources: + limits: + nvidia.com/gpu: 1 + cpu: 8 + memory: 40Gi + requests: + cpu: 2 + memory: 16Gi + nvidia.com/gpu: 1 + volumeMounts: + - name: shm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + - name: localtime + mountPath: /etc/localtime + readOnly: true + livenessProbe: + httpGet: + path: /health + port: 30000 + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health_generate + port: 30000 + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumes: + - name: shm + emptyDir: + medium: Memory + sizeLimit: 10Gi + - name: hf-cache + persistentVolumeClaim: + claimName: llama-31-8b-sglang + - name: localtime + hostPath: + path: /etc/localtime + type: File +--- +apiVersion: v1 +kind: Service +metadata: + name: meta-llama-31-8b-instruct-sglang +spec: + selector: + app: meta-llama-31-8b-instruct-sglang + ports: + - protocol: TCP + port: 80 # port on host + targetPort: 30000 # port in container + type: LoadBalancer # change to ClusterIP if needed diff --git a/docker/serve b/docker/serve new file mode 100755 index 000000000..493ecbd23 --- /dev/null +++ b/docker/serve @@ -0,0 +1,31 @@ +#!/bin/bash + +echo "Starting server" + +SERVER_ARGS="--host 0.0.0.0 --port 8080" + +if [ -n "$TENSOR_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --tp-size ${TENSOR_PARALLEL_DEGREE}" +fi + +if [ -n "$DATA_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --dp-size ${DATA_PARALLEL_DEGREE}" +fi + +if [ -n "$EXPERT_PARALLEL_DEGREE" ]; then + SERVER_ARGS="${SERVER_ARGS} --ep-size ${EXPERT_PARALLEL_DEGREE}" +fi + +if [ -n "$MEM_FRACTION_STATIC" ]; then + SERVER_ARGS="${SERVER_ARGS} --mem-fraction-static ${MEM_FRACTION_STATIC}" +fi + +if [ -n "$QUANTIZATION" ]; then + SERVER_ARGS="${SERVER_ARGS} --quantization ${QUANTIZATION}" +fi + +if [ -n "$CHUNKED_PREFILL_SIZE" ]; then + SERVER_ARGS="${SERVER_ARGS} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE}" +fi + +python3 -m sglang.launch_server --model-path /opt/ml/model $SERVER_ARGS diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..6b8792c42 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,58 @@ +# Minimal Makefile for Sphinx documentation +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD ?= sphinx-autobuild +SOURCEDIR = . +BUILDDIR = _build +PORT ?= 8003 + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + @echo "" + @echo "Additional targets:" + @echo " serve to build and serve documentation with auto-build and live reload" + +# Compile Notebook files and record execution time +compile: + @set -e; \ + echo "Starting Notebook compilation..."; \ + mkdir -p logs; \ + echo "Notebook execution timings:" > logs/timing.log; \ + START_TOTAL=$$(date +%s); \ + find $(SOURCEDIR) -path "*/_build/*" -prune -o -name "*.ipynb" -print0 | \ + parallel -0 -j3 --halt soon,fail=1 ' \ + NB_NAME=$$(basename {}); \ + START_TIME=$$(date +%s); \ + retry --delay=0 --times=2 -- \ + jupyter nbconvert --to notebook --execute --inplace "{}" \ + --ExecutePreprocessor.timeout=600 \ + --ExecutePreprocessor.kernel_name=python3; \ + RET_CODE=$$?; \ + END_TIME=$$(date +%s); \ + ELAPSED_TIME=$$((END_TIME - START_TIME)); \ + echo "$${NB_NAME}: $${ELAPSED_TIME}s" >> logs/timing.log; \ + exit $$RET_CODE' || exit 1; \ + END_TOTAL=$$(date +%s); \ + TOTAL_ELAPSED=$$((END_TOTAL - START_TOTAL)); \ + echo "---------------------------------" >> logs/timing.log; \ + echo "Total execution time: $${TOTAL_ELAPSED}s" >> logs/timing.log; \ + echo "All Notebook execution timings:" && cat logs/timing.log + +# Serve documentation with auto-build and live reload +serve: + @echo "Starting auto-build server at http://0.0.0.0:$(PORT)" + @$(SPHINXAUTOBUILD) "$(SOURCEDIR)" "$(BUILDDIR)/html" \ + --host 0.0.0.0 \ + --port $(PORT) \ + --watch $(SOURCEDIR) \ + --re-ignore ".*\.(ipynb_checkpoints|pyc|pyo|pyd|git)" + +.PHONY: help Makefile compile clean serve + +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + find . -name "*.ipynb" -exec nbstripout {} \; + rm -rf $(BUILDDIR) + rm -rf logs diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..2edb2c619 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,55 @@ +# SGLang Documentation + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. +Most documentation files are located under the `docs/` folder. + +## Docs Workflow + +### Install Dependency + +```bash +apt-get update && apt-get install -y pandoc parallel retry +pip install -r requirements.txt +``` + +### Update Documentation + +Update your Jupyter notebooks in the appropriate subdirectories under `docs/`. If you add new files, remember to update `index.rst` (or relevant `.rst` files) accordingly. + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. + +```bash +# 1) Compile all Jupyter notebooks +make compile # This step can take a long time (10+ mins). You can consider skipping this step if you can make sure your added files are correct. +make html + +# 2) Compile and Preview documentation locally with auto-build +# This will automatically rebuild docs when files change +# Open your browser at the displayed port to view the docs +bash serve.sh + +# 2a) Alternative ways to serve documentation +# Directly use make serve +make serve +# With custom port +PORT=8080 make serve + +# 3) Clean notebook outputs +# nbstripout removes notebook outputs so your PR stays clean +pip install nbstripout +find . -name '*.ipynb' -exec nbstripout {} \; + +# 4) Pre-commit checks and create a PR +# After these checks pass, push your changes and open a PR on your branch +pre-commit run --all-files +``` +--- + +## Documentation Style Guidelines + +- For common functionalities, we prefer **Jupyter Notebooks** over Markdown so that all examples can be executed and validated by our docs CI pipeline. For complex features (e.g., distributed serving), Markdown is preferred. +- Keep in mind the documentation execution time when writing interactive Jupyter notebooks. Each interactive notebook will be run and compiled against every commit to ensure they are runnable, so it is important to apply some tips to reduce the documentation compilation time: + - Use small models (e.g., `qwen/qwen2.5-0.5b-instruct`) for most cases to reduce server launch time. + - Reuse the launched server as much as possible to reduce server launch time. +- Do not use absolute links (e.g., `https://docs.sglang.ai/get_started/install.html`). Always prefer relative links (e.g., `../get_started/install.md`). +- Follow the existing examples to learn how to launch a server, send a query and other common styles. diff --git a/docs/_static/css/custom_log.css b/docs/_static/css/custom_log.css new file mode 100644 index 000000000..61f65d019 --- /dev/null +++ b/docs/_static/css/custom_log.css @@ -0,0 +1,29 @@ +.output_area { + color: #615656; +} + +table.autosummary td { + width: 50% + } + + img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.output_area.stderr { + color: #d3d3d3 !important; +} + +.output_area.stdout { + color: #d3d3d3 !important; +} + +div.output_area.stderr { + color: #d3d3d3 !important; +} + +div.output_area.stdout { + color: #d3d3d3 !important; +} diff --git a/docs/_static/css/readthedocs.css b/docs/_static/css/readthedocs.css new file mode 100644 index 000000000..aca6649b4 --- /dev/null +++ b/docs/_static/css/readthedocs.css @@ -0,0 +1,9 @@ +table.autosummary td { + width: 50% +} + +img.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} diff --git a/docs/_static/image/logo.ico b/docs/_static/image/logo.ico new file mode 100644 index 0000000000000000000000000000000000000000..7a0b5b2f00e4b626e250e5c4a03de4136d96943a GIT binary patch literal 38078 zcmeHQcX(B0o{l?zIQz`A&+L)}#lGwm-JNk~$604reROtoq=YJnVxbC{kuFFRK|rbj z1nJET1Of;eq=uB6p4^*q(|d1(^j`7Z_jkfg&MkrDdgVUP?)i9L&bjA)<(&8XeZTgd zBrY!Z@!zy*F8u!q7vC>kTn4+ixI9Vll8cUz>q^<#eYn&A09XYon0?iId+a#2#zKISlx4Sjp>X#W@TZIBdOhhSo+U z1Cm4T(wJ}>1Cmb`Uq~y?VLz)zsA3F8F23Cn*3}0JC_0Hj;_hS#=mnJaPH`u{A>I>mjUOF zuF3yh`2U`}BR>>(7{V_8Z}En@ZkdB0?^>ivb4w?R%9>zEs>IdMQk=V1h~s{FICef4 zCobgUoPQC5^%Y3VYrxIg4z#x4!ma*vt{PqH->7UsjPWME3oM4ug**A$5tsWZ^GfzS@`mJ`hc%V12eMnzStGLIJH z42XYuuE_YC_`*9K+YYCiVfXP27*cD{*=74#s%zVD_)HewUK)v)W`yE56NB;S$Ur3 z*MjiS$l#v-9~`OUm;w0B>%sWbj1YLPkHg{f1&GURwAp_`S~cEW6oySlvatO`9+BHG z?C{CMg3U&ZnG=XLUnJnv#XKaWR-vQgmikuZd_eqH*R|oVYh&>7muXnNHyvyCW#F^J zS=ewW3-dlpz`Tv|s3K3citzHsHv{nV5m9iRmUGGBFgac%n-PZ2*IH7@k&27`+uGNkBfurh;Tgg z>NU(-mw?Oqdfec8Q7uJzs*YCkx@_TZD6AHTayz9VhcKYmEVARnnz(5xp~UVFU(^ zjm6-}Y37`B4&2}9hB;qpaF@9l6@{n91z_c#Oz4stQBA#JsWNmi<@nFZ!Eha8FzegY zpSz4a;TNmZm2=7Z}3h!~b9S|6D+F{6Uhu(xNo~t0= zUDelN3i)5M$$)()GY}GU1EK>g%7FQwzrldj2eROGx(K^D54+BlVEMi*tlpoA*=u9_ zx1tAzr9i&|68ZmJvN7u0iJT*Q58^6j& zL~0Y7T6@>B$i)QCt+Z3Oai7F-#^~WbA`}mK=oGk(42L_%Hq~>RXvDw25sY15mm)T+ z*-FRl^~r$O@m$2`brKzjOl!sZgPHg%b=SPl;<0dZ0zTYk#L~|bd+?E05*BVs!0;IX znDcQY^>rGqgp{C+_i0fE#J||Ix$6w@xmJP07t8StWq$|f!TV|jcKH_Iy-y4%ueL4! zGVZZ+Ie284zoOs82E4mckLBAFvD+sLC%(&v53%E`ba;Mh!0*V3`&e^6T}MS?m}dlz zU#~(|No$YY6#e?>D4m%;6OJGA2*N9~!!e(_c(+eB4xP=$^3N0SxG}!om$P*tjndzTf2`D5~5F zI?BBG8Cm)D5E*G{wbIQ4;=h^gH||TutkrtV-4KVV%cC)UWsGt@Wmy!wyfe|*()a6h z8F}?MN)7^JDv*@Z*r$IZ>poO}1Ai7>K92h8w%(9+Gjwzy)*Z=1NK%vH8%bT6FIURB zy53_Rf$-dzgm1&EQC8i7PB9QHHT7+nuqYC)v=65GV)y55O~J*Odek(UbCi)=M>&r} zXiBrfzv!QpU!>rzMIqRJG!>V0#mFYVc9sG0FJsi!w<9I17V)Xon7>|+PxqxFj($&S zb{*;(I~@8l(JuN~!i|Y1WZGAopPCfu%Ze_syn>uWwJzcUn78gQpm@YZ|xc5W`{Zr}{mECmrh``KG zjW{1shl;vRS$Jz!sX!iY!@toI5-TtrJ1S6iF7=)c?OJJ~JgAbg_#O9;fiu}tO@Q-c4BFR=!PtNU_Pktfj>C8XRZL=>1JDVyR!6Rd%V1G1e}|cWd{iqEY3C^k=|3 z)sfgOr=T7?j-}z*w`hOInA>0B>tV)cOLt|!KfXcnfu#QLCky|*^6x$@Ske8W&&5}j zSY7mZw6PKqlvR1IUyb&M`|UPjgCAkPpB&7wX@f5X7h}n`MB3nP|5JR;CG>gbuZw}i zxxvxpC@Qt4mi_MH%Uzn=HgC3#uPfhx~H$D^J^@XV+ICSv_lJmv?0sV?*yfBSE z>ubgy`FEv{IdyfsqN60H5*s~_Tn+W`$G?oz;fc{fkU0{2Zn6PlXBK*;b=zRhnYxIr zo4Y<15%fJxF~DkaGkaA8Htna44=7T6obN4z5_KoG&#m~6OR2(pA4e=r4nh14K|IyUvR`RmT!y0vG2O$-~`%$Z$qnT zgTt`$^X~kD*c2Q2g}c#U#sA;F$-$@OU+j!&tw}t+XiFkK;2MliujBog&$Gx^40Y>l z%82)cLd7q?+rTx`za-@Gf&6y0~hq{ETHoKkCfKM~gfro9P>O<^v>O@LT#LGuBYHu9hLAsD$r(KE5rE0d4e-8PhD_$e zn(1HN&8~?Jc>5#fRaG&t_>{jI8>r-q#9mnD6>QztovtNk)+jzn7~^2k3!)#^>`lYu zg(28_i1A=h5z^=j_a!5J^linz)a#@jk@3!6DS$8cEaK^l%05Vc%Jqm+jPcY#pIT?O zhShCljy>1KD!IoVokN{Ggt_|J8;tOyZ(m;1F`#ET%{xbj!c!p`oLh-}EkEuBAJ-bEokv zzpJcn#iYfN7&Kay2V$QO(B_I?;}cMcyz=&b&w-_?6JKM&20e~mC{=tDk^i$nm3VJu zB-ZUVDn7f!jFJ}^Kye8FqECDnGkAU)i}N8>2uW_n>42MlsH)QPX~e$8u9vqP{v`=%HCd3 zR2f7L22dQre@$JxlA}69d64{+=tnuqzQr`=M{`Q7^4|29tMMO`f|$4Ijho!*Yd*@H z@v!&9v0_(>;?G_UE5lXVwv){9Zrsn@>nD1Q_6)&czhdY!o2l1(V?T+rWbaY-9M!Rv ztS!++zZeq;j}IcSc&ia#9M7cRlBM)He`6d*c!uGZD;wCoXwaQ=$R24sUB!oSGQJo={B4E2g#GTEn#k+pc2YalVZ z!Ac~=26!D!BUjABa6iyw2ZT3?pJXjOP9IU?)uDv!&$x{U>$VxM1}ga-B?oE|t4e-$ zKJFJOFZ{* z-(PEB{MT~Z26tf1o-};0CJI{*CPRF>Oy=0{<}axFbO`?q)CH0YJ>pjiW5Mk`QtC&E z5odExvbe%_ZmNp8-48d%De|hsSxTN$_dmA1S$u>1z`3byzJ*xo@B@H>q3128&#Y{L32O#h75?++bxN=FH`M z?z2}9#0EQrf4N6lo6}cBVJG*2zYea%mSZ`Xyd)Ce1eKwK_O`DgahT+=o_T|Mo3c2F zc?37E2elv5vo<&%vN!)v9?^J_IiI84vz2_Y+6I`$X`?Q>$h_Hv1>tyPWRPO(B}d&o zpB8f={mPy^_YnT^UF16@+>aMolew6-EJDde`!dEh zBvlP04t5y-;?so3RbbWjcbp z520RooH44zS(2mkTpv$;SBJuz?Kb&0&4b9SUqB)ChF*E^MbnL(8>p}L{2dhgav1=7UH6=6j6!X`z>rl z75!R?sU)}DpCUd^QCTDR6)KcC1(<5L|*tV->QCnowQOd*?Is)o)r( zos#boU+4g1O*u-;DesX*#8*J}dIzu<<8c1_!^QuFIx_2O*!fHv|8^4Rpk^BX4kFUd zZZ!VwB+fz2H2xh#q@CSp{M$*KgPLjlJBUa-yV3Z!lQ;)8)A)A~k#=^Y@oy(_4r-?H z?;s-W>_+3?PU0NYOyl1{MB3Sn#=o7!IjEV&zk`Uhvm1?nJBf2pGmU=-5ou>P8vk|@ z=b&a9{|+M3&TfQ%^Os!g>_}^77X$LLmi5a~c6FsSvV(zgw#l@koz}!L48*aUi;n3^ z>%|TReA#9dVfzI^J37>w^qT?sfz^4$ABlWIEBBUx3=a1+@ejnIdpoo?8X4F@$lv{y zzkw=$bG($$%DrJAkHb7qnEv+l&v?F_=-&@{v7%P&W+rMqMdF)Ee2`CQaWYA`OKIizImBM#CBo$6_wcXg-140Dd+MpqB}S7t`C~%zMOlRW z;xSD>b?5IThb%Tz{+9bP;tzz#t`(P-eAo3}_<2F^`Te@y^PjBkJ-@H_R6Elc&=}Ad XFlS(G@5lRez2`qE=smw*t#ketBIbCo literal 0 HcmV?d00001 diff --git a/docs/_static/image/logo.png b/docs/_static/image/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..2a8bc258f6663112f02b40f04843789585d86d68 GIT binary patch literal 402692 zcmeFZcQ~AD7e2}uj7}It9a5B0qm2@b5JX9onbB*4h~7J+1S3I;lBkgoMkjjjqKoLn z=$#;XJF>I)_uJn&-}&?WcU+h2E#`f%=ULBs*1GR|Efe`jQ<;L4i4+eHk3v;NQ5z4B zL=O)SFa{#Jx)X9pCVzF%e4?jnrJ;exdvy-N1L8B`5ni3)U;WDBGymuOK0Y@d!C&_Q zS1+Lb`~3H({`$dxaCNx4`g6QNbf|!bCy%G92-o$(->7>QdvnzLmWN0ww%WJamyLyz zh;dHI0u;fhpvVZ)=Y(gBUM%Zr6%sz-3~rM1u-PA?@yLs(8yBGrKfiaO5#BNzyDEmK zL(LKStI&9oDU8mo^)>a5EuIKo@jXCuNXRUCR?X*{TwW3XzTR3yMM-I@@wBnAzra9T zVbY;hYFCbwdxLAg^$QF&H9$cg1j6Tl07HZR|BL^RY=BmULlWPT4Z8oo7}__w@#YJ% zPrd=8Ymff57;hpQ1Dr&hi@*BOyCME-nP{Ik2E^`@gvck{_+RUWYlIB!d!CtNUbqu- z_%c(`(h)B;4Tq8bz08T)!0K?q~?=JWB$=U>7j_9Xvy0`g#ab~YImTI~OP zzdVRMvax(-9-~MRiKE0i5;RQ5|9CD=Z$Jz9Gfby3>aqJYisbE=AUgudb%jSU?6zgx z{~oH46|mZAT=+0!>3`asg%T(gVEs`2Q)f*q8H0L zv2G*&{q%XyUj@ouyOcyEwfyX6kHYJIkr5;hkWzcpS`s8zhVb4p-aNTc64E`K^Uq^h zgy<2}&s4vij5jTPy+lhE`JPsXY-I`zzPq4}AJf@S{=L7@s`t|y?{Xb$yW9Ws{C_6$ zd#DFMbo5MD&9%%`nnc4Nv;49(cTB>uTpm7mrDbO1&Ldq@TjB!EBKFCQ*9_pu4(XZ9 zIZBTN(FSPDD5Zeb=l`toW1xX)rsXBzE@aA%&g5W&ab_^ZrMPa%JfHY>i`hm@FXIP< z&VP-77!M6-kikRcsnG%c>koe}gxG)^j&sFI2JLi->s+x+qD3A%YhGcdbWV|ND z?YuPxd6rw;+44A>?RS0G{(grLC&0J4KR&z?<}6}KG;4A0#mK?_JSxt)$eDC*Gh`%y zvi;yq`t+xf|FS|B1sV{ZTs}E;{y#TOXXt(D==X0y*+dk5?qoggv}zQb(FVISV}A3; zo*jZ%pVn>b&i&=^JUNwp<-P@r+^EfoAGW^M=J_bIB9y$-YXG`7vsu0`ntdIhKJm|V zi%_7k!Y{0WuU>ZcQ}RkM$0uwOGSt9uv_mhW+~>_hoQlY9`3t78t0$Ve*gi|>sPH(8 zC}$e}uQ1w(R&Jb+B)%|lBlsUSbgHDWId5l`ymm(+wt!C_bvt?O)_a4wQ!CRK*A;KJ z;U;oNqYRQ=tb4Ctm-VZ@5}nN&GO;Z2&I=Y-8M`+0Vdg zU-j}X89?9$Xw1jOc>5!NofT25^Opd(_`kzBnuFTL|L)7PMbVpb(h*a34c_CLm5knu zY2on7F-hFj1F6UNxRDwxH&Zb4qrUDNpqnhYPU&))i+R_QT^S*}B&5cq& z$2xz_G_0kZUx(k5^oniF=w3?H4@pNK;#vd|NljGQ9p@29Mjkvk=!rs}79gnhjO@o< zCwHT#P6kwA<5eH;&0Z_x0se8C^oT~qz0zBrWqPY^{Yz4z83k+5o%#N2Lmn2&Fjx2?b$wP9U%-wd=Nc&OA|dAfpZxGOjewJXJsM#7Va?=p^= z#y$xNvb)SUm)p*9Egq7G7}gAoe;SV@BS_+v0kS62SAWa;XMyhkH&9;(Od+t@=CFg& z0T=!5W7)KiFGJGG!gNBMMNHb$&+K#^#8qqUnAw@pQ7@ZY)-naMt`kpovX)?CR0U~} zazBQq~9k#7(i2 z>N{9TS1m6n;SN5+P-d75HhNEV5vUU&RB-R+BFV=tfa7Op@6qZ=tkcE`lF5&yJG#VP z+2dC`@-uAxafj;scqP(i=}7&v3fBFoW#ql_>{8^r^k*9fH(QbHZmAX(MnOgD*^6Xf zl3}@w4A6)#EL5&6FTCP+gu0tZgO4^9^0qkh z8~}L%$+^rk)s1Am-Hf5AIXy~s{(l+BDTw-Kdk^IRVqN=RcmMGkv-i}DQf3l)iP*H* zwd2#tIX+^$cP((`pn@nbEMxMXlFk!&m4iYK2*BgZpUREFugExB{pe#t75A_*sy{)3 zdDC*q>omQ>7FoA9DRy9GUpzvuIsLQdmv}UC(p_({8nvD1DeEa>GsZF1MG!izXwVx> zi&T5NNC+^dj@_)0HyedOpx)?42Y92yDoG6?V0;4>jJC7Zokf`AslwsGIh?;YQKAFz z`I5D|;(BsUe2v+B)HGLc+G6lQh8b3y)56hNBgv4L@`f!wkhSe0h_e5|sO|3}_Zi{j+icXdEY?^gnjvRnpvCOLXuY>o`wWS1;qQ zo6!#Bh}*uu-?wVzROoq^POIv>e;*ploxYnwVX%7SGON3gIb~^9a8H!5jP7FFDVmow zQvvkJ&zz3HF+ip-GWE>A!mCfggLdz@&d$m7!!IdhwylK3oG7eW>{J{1v1l7v-rHs* zIyNbr5cPS4OvBlAvRKRr!&UAA;Ft62^k>Ca*ifl;9dj@;-UJzFB`yY|r-Aw*5ygHS z@f3iP(y5nCRpxP$!bb42A82`TQ7@s`vh$#HcJhVd0DSn!xcQ@44L7qM*1ugxNDD+R zkS!H-*2D1dSW~Co2Fq=8NT{P_S={!~U~WIz?`2X_oO5{zS7uxHRMVXRaR7s*>w-~D zbKA+!q7N{=NX7u>!?l*00-T9|-!PAtAVW=pq>1~y{yA)<=nT70jX~`QBWV6lj%%#v z#5tGx*HO9){GDq4O_zqk8qKh;ThOTLY zr?UOHuy<$>IW71=-8Q@w!Bi>AMcs@{6SipE|>_ZPLW)8m?9B1`T2V6+RBA@nga_%?sBa z{O`?yCh*XFq=Xkw#$6dSP5a}*q^a7%r@_R6S4umMB*%}7cijyanT??&4hZV=6my!( zxihQ#&Zdh&olwNSNeA@dZH+tWr({?c;S8hGGvW!Wd@tK>2F2OXbeut1+~ZA#7N>oq zpv99UXQ)FY?OMR3!)0vuds>9IaDBc4b6GG96Rz)n+m3jwQZSP&F6!3~Y@$!Rw4Dd% zf>z)faT!igiR$ooMph4RedO-Sh-!J&yKw}u$P0kWi=vo7d|55m6@$w787m{qa7i2Z z+mp@eP@yq3ud+8mkkhKhgjyB1#n=Ju-%}TI27PH*@iCr=4aQOiJoiXzXy`R(YdT1` zI5b|gcD+vQa+Y@wCYD50uhM0D*rr_{;h#K}*V%nEax|6qBH@Yq1kBYpR=d5;E49DF zBPl9wS19%lSF9sRk(04QU1yV@@R^2P<*&iMxGxEK(A2L($PhQl*_yO~frochcEHp3 zE1dJSXNNt|rsr_n60az1!DXvtP=A4p77D65SV%^V=c(cn1QFO{xc*$hd z!Z>7Mc?-Wx{bFQhrE$L!LNAQ;YyEr3)3i66Urolh$|0$y`0M)u%k)@xyOHL;>lduV z#k0XaPw?-ul`hvj@Q_xy`;xvDkuq8KT-Dfp)Jp49ocM(BQY&eLRG5A}cQ+w`cKNN? z5(Kgn@hX0xktB+TBuYR}oO-M!tm&cLN{vvFF#~mNu=~tDHtnY^7e4%;+l9lBJjG<- zBBZ;Ib>YUXZ6&fl7Fh|FKP4+vBiv#OOWUc|XX=v$ZOU+~ItO|R=aETKu8p;5&#Z;kq1LG4B%e%tMB zc@bp9{Na0{dpTV3g7fA|&N5*;N|Z~jKQ+DDo|R@CO;L0+G;z7zXrttMZz}&_CFapg zWMOoH^`&U?Y23Eou$tGRYyO2yu94fY#+Yn&_pne7VO#Fe+iVsO;6-msM*^iZQT62x zhtuiDn3sjc90t|ruLQ}pd}Rdd1n=Kb z0GtW}Mjp`Xl{RzN9?e173c1Qgw*5z*Rvvg2wI|g6qR|xF+rt)IP;=|Xf6Lv&k z9({t3dE%nZdLk$N1D41pQ=GaYCsJ&%ZDlUSx^fQf&%QXIjNl(up4vEw0|YHiL!3l9 z+VleHWW`At_PiNS_mvD(`$$-+pqP@*FUh_+^JbdV+o5{0m%GNQDWGm=L4l~+T<&_8 zxrg-=G=CQPfUu=S8w%{$sKP){A!|wuI}X}CRq41!F9-_Oxmo7+^Ox)g&Ga# zPDZss>LP%Lcm2*}*be7Lr$uKbviaFWxY)dG7<{=M-=DJu;z}2ZJ)b&STAs(L z1zw~$ZfNfrCs8!xCQznF!!gy!fvYG82YJ=n1{XX?_$!6|-?Yv*q4I$YK=NS`vyP|( zW#Lo8jcO8Xvh*-_Q?(7$H7A&jEB!_st@h3YW47Tv`K^@nRjEIhsx7F^e%riU zs2gUVPoM5Mr<1K}G!U6F?Imzi(t0c1x2NZonybTa&)9$%d=ON!eepdYMqA8~ah6JOZ7Lohj*vmQE@d#rMMVOxWgNAGAKx`QU~&VJ7{ z{T?XdoXqEDDr~<64oaipN=x!z>zinBBpLl$Bhx3!D>2DnA`WEVtD@g=^2_C7aNP`v zI_BiZEV3?7G-GHxrBk28z!|q;iGr;Y!6#jjvr%VJoO$CmLs^jmv}Te&hBf>af!ooJ zk_AtU3uQhJIy6H&N{!w(6*!>0G4L5Jw?~5rIe5)HGGrH#*B6SYq zx7I26ldC3s<{b;kPLt(Y9(Vbo9O^E&L>%uPei!i_wH(p0%jh=sU2ReGoi03~1$>0y zcCxiJMC+`2@dt9m zsckQ}>Q1vzoyku#Uw*hCEN2bfeCtP^A-3MMe2-1u8D1u;8YnV36!4O^)ZX-T<)Q$= zIsY)$(OA9g+J)t_8>W)lQ6>J5T&LZ$N;bWU0|=!J0Q}n!0tXGpno-+ZeZi8Nu*K1Q zpx-a1BSEM@Q3wB)>vDOyy}!eU${;r0YFv`^Olp!emk@X7`Gu@+0RzYlE-!BQ%*w%l z99Z25IsAydVKbU;X}+FvxL$P#$6 zD-s~G?~=nyiuF0Rw_QtKFccx`?1;aJDrisEL4ou z%XQv)T1zf-W8UIIX*WZQ^QW+<;8d!!(Tk3shhIhFmhuVhuhN?c3xCD%+Paudk^%H_@%9s$7s*}DLZsNPP`w4yNku*5%7l4iWawg5 z%t>-L$#hf!YC05EKOZ-*uZfc-@8I>UUhB<&4lWf;>o4m25xG_P!_sypX-mvq5yV0c zObs|1o0&5=kzz#U<|exVZx&6Qd#V6(r#tlSzM#U7k$73CRd(}lQh}r995~UOj(Qb= zw7x#zJGjW=M+o90+B7@E`tH*7@AuYhovmoPm?WN?+kBmioA?}I+N$Ix26b|z0 z?$51C&JLIKyw(AGZ5W`l$PxGY|xL zC$uxzmH{GQDZQATD?$-&SV=89CZ(X{x#EW#sic6LUd{fOBHNyIBlaM)&M#(}*6>XtwTTZKz4W=z^) zxcjb~H>@en3yelc#)4$_e+u7iAl^+$>WPv)U$H!OF|W9L5_Wg{FmC&7JmIib$L(lA zZ8Wfw{T%y{Lx>c>&nesWRZ979X4A-_n10!~1Ppo|*t8zL0IX%G2L28;jrj8E+n$E+ zbcTzbJ~3+SlZ3c|uvk3m@6&<@ud1(cfA4oLjaMnC!? zG51~C3Ls2oxUBb{0Jk1JwOcda!72cKkUaG?m%AO?`VM;UAqRb!p;Id_rYM;}dGnef=S`9{!t>I$bXXh(aNd4UzTWS28wQJmhG%Dfd;zwTYWES9NG;9e=r zp^ERyjg)v*~WN**m6p*nbje8NSXy zx-2MYY4-7Bg+f@fl?ft*gV5;9UqU$S(^dH*%*2%RUL-ai7yQ%VApg6d87d?QlPF^5 z$h!3p*9jnaLsrxeb5rTWQF{CFAZ9>-Oc)~Ni1cCdG>XouSaVc9omOfQzkxJ3 zNMD|66;CZVNEailhq~QtD=7@&)>C4|1;K6j2FFv?b+#C)_>xF5=e0+ zk6w%_@Y%`Lp*E^pCn<>V?^gQKx@gO2f>_YC^QAUv^WmFAU8nR(F#@Lg936T48S+2e z0Y|I~orquv z!)T2ky*e%RxRbzlMTFLuS6vMH^ym=Y_;Sk;h}+E%Tz42wEwC*PX!&ZPF+8(3lw^}3 z=mRP4dIhK0W?s2XGr>`)9KAs06@eacX*mO{=cw@j-=#BY%hhSb)MFgG3G4kvQh(B(lR`*z!eo~Bf7~LA9*wSFj<1{aoN?T$>Gl5{ml?mKgkZ1r+fx$`cGJ2RNFz)I z%~n{Lk1`p$q7ya2c-phI>V3!%P`A6&LWK7*?PsVx!*e6Iws`@_N~nH-P!SlUh$onv zB>4m!5+q(pjMgm<5C(OM;|^X)^QCKC8C5o-={L4e$>`!;^Z^bFH|eqWPvL zEMEEw(B>-{BDC@G4Ixpv`OVmm6wUo--+Sh)-<(5OT61q~1D3l&QPg!a-o9-GL;7!7 za@`*;-0=P&Br55_~*IqUMgme~#Fh;k%c)dCgxIqc7dwmhsl@cOW!W*j@o@=m|<**R%fvK*}NcLjhA3uZw*@nNSz!&4n*cQL65@CFZFwR z-z98U-*s)+4M7VGbdY%pkSYnse+@Ix)VuNmn9Za@dh|{UlC(*HlaFhxB$}e0ES>@M z2@guatWQnd)R;jQLPkWNQiye`&X|v*04)`u$nhqg3DB5~f0xWl`^=QAM4Ai~An$x` z>BsfFBpresqYHVm>L``WXoYJndO!Hn1Pw1AH42>^PHHD-V&t2zXBg0|csnsyqp1HI zcKFMi*#xvdW6{-B8a$1eIUP=bq*FBCytv&Dmb*d9>l-GYd9C(NNCV!K&6dg$sgB%? zyJ2626U~S@9jSA7Qm3bUqJe;JZKqKzqP=ND(q9rhmFt%pA63Vl?lPB~v65nuOFz>U zrZ*)(Ya&^{pt!Z5&*O!B$;rQ)H??W2HwbP+DsgFHuG|6KGQPIr6Kg`O?pB3q$|VP) z+^VEVW>1L6OmA0m=C$ndrp{M_NBxDRkHKmZ(Q&N3Ca(2uFHPjs&UL;B0NaBbb*R{f zL?q~M4ufc5zriP<)jlS`y#LgHQ9AHo@$NPqtL*W&efBNohBckvA)&+hzP-ZoKRrS) zYS81sEhXPW%{JtI4MDO&ib5gEFCJBu7xYI8$m`j*7UG~|ERE|2lIB+5SJU(tM z!-~EgFYz;BJis+~=p@OPjpyCVB_d@E!Hqg1C6|nJ_DtY!aEH&~Bt*S(;UB&(NK11_ zHgUeu(ZLq33q;2*Ra`Y8xD(IA2N0CeL01(beb>@iC^-@5Qn)-sW?YXEsJPRnccpx= zT=(NZYB?=xPYD&RDC!XjklMY0%S|$bU`!yf06vEsBi6iOqOdlzbg;oFcsvL)nY&zoYzu-!4?_wpY~c)$JbgHQNEm$W^Yx8{i>wv5vPTp{F+IvzC^PD5 zCQQM767rL$j~FT~N+Yz^F6*j8AxHKF7J>!?hX)v+qlx z5WEkk&@zJ}g(h%);Lr_f(-cXBciT=XwO{yDWLGYI8i2Ue56j)knFfDKl5@4!JBPk@ zk>JM5=Z)W0^3s}?c{3i-tVuUXhneu#D_ty*6jnMMGgi?PZp%ckXTxZGJzk2(;l!P5 z5-fzivMOyJOEx^XtK|#WT9l4F5kTcuE+1<|6hPWDx`#OxkrEDgC2UU#m1hyH9s|Nk zBe(i3^kviP1jmubxs@nMfeGn=E9%N5mla;6M3U!mIi1jGdnk;gnpc7>i_Y8Y=Ktdp zi4b5l<>LdFEt^b*=9e;oavg!mw8N4$Kdxzkfx6rI5+OCaNtb3K#qRx6uQn^IX{gTY zx|^s|&$oQ_nbkixizR5^nw(61*={c}ba|x@@-OGk;Z%fww(bew3MYe90%pq{NB%C& zp(7O<1JpfFW4CT&=U9c>)YqjQzb1aZTw*k_KaTg0gCZ!y>BGVJM8MV7#?!PkD${y` zrp6Q?z9IY`5%Kj;AthK8WPoUI*Nh{jfV24ZCMgZ_NbgHyqg%+*Nw#uttBO-idTf|+>>srm5)o@)*$cym&$1(htKS8%$GhBM} z<$SGi7;?FS`0^+I#O6Hh-R!?`=>7ZY>v}*#IyvU+s1%qID9Ad$M>hm4Wy1Q745=An z{R+B6zbe{xtAgi?MX>{F%w~s&boOhY7kM+6KnmWE-CN0Onk{=#bGhPawpM;!k)=-2 zxzD^A%A=Zd@1(XJB$s9jI&gKY-LB@nlKmaT$hfrF9@c*ET|n_DB9V3o5}joiL_G2* z)7J43@6^3nCnddVlJPGk( z2iBL3TJTX766qYD|jr%k`cCo88hOcVBB3HuB_A1mD zcooROzVRQm`gC6zK1mc$;qh-YM-8A2kv~pQ4)CrEZOz;Lp41V~$+xp;)~4%}4O7g`aPR$2>~3958fol2#r}lLstLmU2QX+B2fCV?0`_wL~|k4!>JCcjD3#YBeUr;X_l2BS^X93z)tzb-h z_Q;yLxMY#uVKVI`g?tIk4Vc@=lD8qS&cIvT3wLQMb&w64L5sg}XNmjQ7E-_5gr8V{ z^4hPol_%af0R=78D7+@lwQ8Yw$Lza5Zq%7Yh`fxTr^@0#PXdXaYz3`BrG9YSe!Bst>1T5x$VTwdwJcakserD*f_ z^NBpq_YoT{&5AGE>}RT;^@_r}LcFUY7Xd^{Q{{%$?`%Bc`7PdYMAqRI)ox z75YydVsDbkQwR?Qz9Z1It1{m0wI7`*_#pOYC}T>%@<9EC6mVGU)TVZi%$^8v{bwZQ z_*TVHl>mHLp#Gli$pB}O5W@wbW2z>(^>#r(KOufqzoc*_B;ECQr_z^Y1>msDGj0;1 zJB=#B^aL1iL?PpVWn=VmiCo1dS>ei2V-O_FfX+ntXzYPlr~C(s3yOq`=ja9lj&llv z5I0M1DdAsaTr5X^qmlW31?!>6TxtWFVC@gX$9?i_a`gJBzp|Zo?1LV;A##Zdckjpa5JRnH+4A*1(3MZhV0(S_ z3Oj2sbI^4^!nxLSw>qZ#UP}02?}SCff*MX^!OSk|9GL-Gy;WV(>;W^B?0{k>pmFyV z*a$7GQgSP=DinTcY1QLZMrExV`x`7-RO%<70;2~_oQ0fLFPXI-Ec?xf%`BtxhI*%M zsf)q=CbBb0j!$b}ap!CR(Vy964=RpiO(s;CrBEQ)l@Gb7Z2Zu^l2BWa{M&2*Nz?H< z%I_urXeD>$(?th1q&dV1?ObiB+(>&Q&wwRFk#4DjxDpYrGEbH{Gq}2$M zDz$>p6Z%30X2hu=* z9?|lvo+-~pF8k(X1GeoI9wBD$Th#Xw%4V}KYyZXzbxP@JSL&Ei93enBMa0!o9wK=E z7rz2^;Yz@hX*$M%drkwS)yb6@{CjW9E3q{{G5AfdhGx7p$*+p|pJjwpByz74am;oV z@e8eFHgGh&Z1RuNB&RdFB{^zuF`ZerV7hzWAPQE7XyodEn{N zaxGj%a)VKa&;apU+tm)1oYdJ0PGUM@?^f1SDqWTyf3Xnk*BTX6IX;rMyq_mF=aMEJ%dsl-*{cPY=p)h;W%0uPu|axU|-yIgay-m4efSd0uD z0UXqI0|q)}l=SIT#bP*lNwc{NTsX02w`sM*G=LSB5ubOHe&l5~(NwUVS`wyY_gKWs z9SW6^TA#T3hpe+G!hc3it|DWlcEBNz=*)zN9{{YOT(uc{2tTrnT$x_X)dKe&399Db z>3I=oLaP!M(0C2>SBqk!7Y+A$L8&oLXOwE{?s?06Ob260q5Pw;doeMGcfSATY4eKd zS9O=HH*l^i4NcOl>>CcE(y?w+L0_rKQ>AHRqK!O%xR^N29G8V!K(ns#xlA4TBx>6zb%dRqiZtGE zb%^WilD+P8m3z!&KyHiHEHP#b-QIHiT5-VbdALbF2+an1oRLeNk=yFGwzX!eEfeOJ zbo-aKK!T*HWc^~)boW)Mq;q?$=wDNTR+0!y_q8NiNSF+(b!t}lbrgas z_*+Sk>7jC77Y1P4KJ?0F{!ojj8?OYJA74^gW?#Vz$eM8=Jk(2q@A$u^&N>c zb-@m;O=1sq%gR$juUdz8bU`%3QmwjL0$Fpiq&dkEyg^>Pp*FUCJ@5JXLTqoBG*c@- z3h%Uiekc(I4>!^aLl~Aj@tKy95hYiaL^d#mA@#;f*5O_L4kMT3(e^k9u{vfIX$R>_ zJ5D3cNXFTk%c@VwM>5|uhhXS&hKx67?L;Y~WxOnI`Lmrqr!iQR_z0)MW&P>M;s6j3 zZgFEt=iIKd*T!rY$)#NNs5Q#oxB@gPJP=Zova0}~{o!q7A$L$o2S$K=O!XCgzjFcV zNi&pa3loj9cLt|0Qde5aUppU=>@G*U%W=b0PM8mLa)2Z$jJC4}jb>#`x9gG_ZAHw! zYud%R8e#I6-s8R`KWcC7`_CCYpCQC8N^?i_ z@8)P8TMIvTVO-KV@C8UFSY z+o+D7NMbgo-Mkc(bwdaH(MBILQ$|>O`w2AR;!Jvmaqexg?thvP+P@_-(ExlS`N{Vk}O5rfQtk0(7*66SH+NW?u-c?@<-X$B0+&7f6TO^CuGpE>Knu?^f0kChX! zFCSd{ma=9ekUjCD^5rZIC_^D4C)i7T4ErwKs4lRG=35fhw4oq3f9=LTA&y?N&Irt1~G*KNg!s;=1IN>`uWe2>_Mbk}uz8Y`@M z{QEMB@cCi87wKRk6^Ese+xcd6Q~5T8Tf`qVdzGG6=wAvzh6U?L3-2~m*)?LM{{U)olI|8s7a>BtT91 zIg^zVET9G{vD%~F?r5E)r9eFT_=am)HPMIRW{bB%8scF{_Q|~4BfZ#|&aUKsp`xQX zLL;Qi2>L_jRqQmbx-Q4Fa+^0GEBvZD%L1fd%2)>cd{=d&=~)^~U%!|VwID%lqMzDg zxKoLIHxCX&$@cF~7aZNozokxWT9UzEt?NFEdY=51!C-M+9coj^wfxMj1*7dGu*+M% zzE<@m*t=`VwhUha%EI0_Yug^pkWs+(_>gt?Yaw^J#rq`-DYy&2!jrH~Mm5oG^E}$A zbxJ{Fkr?YOh?B0m$!@H%QQ!+}&6cZW_zJ(_oRGaQ|0X;DCqqr?i=8@xn$X5G{#&USC zNi$Y?;5b)>K*bzE*o-}Nslcx7z?@buN09Te8Q7TN7@`XCxmW>2QquTqN`w~h>or9! zcfNZdEO&;j=}Ff z@#MP|%qcIU=+nv&kEDqlzrYkV%FYF$1p=;|Wx%5roazgG8wctz2Rw3>ThvCwyXP`n z+W9?woDjl6grx(&v&iLn5S{hMs-{<23~%RdeH(tD0OkDfSG;2+102+R5Pct$@=(sl zFEW_-Ahfrm=NtPIqOk8y*Z)fuP5*=V@S1O<>ZYRZFk%9Ciy!%AEo_R7pb z;6b#!(|)d@gwFa{HB**hM@DG!!NOOAxxI1DgeQ=Gs=39`yDj^TvOLvJSJc2GqOn{U zG|JY&$&I}x=Ne(d1EdrH3}d|QvzY42tx_0S8vCh^)jUe|3Ro(e}ikSSm7V5Pi9JOz=FB{Pf%ox?s98Yf6*~zaF(woMiZk!U2$0 z%>OEAMn4%MnNUrtssRI0%5}QE^)+BFhkCMNunx&Lp0r?Smi@ui8RG53o*QAJ!Q<)x zON<3H?cr=B`bayEAlH0Adsn2vSa?faVyU@6Tgc0wL4?kTn*rkRmTaaLN;hNcV?xsnnRr0=|mY-XCUU`A5gw~W^B!yf4&79%GYxS zTV;u|zFKkUS~svmGvmdru*xrQ*rrlJma)eL3jfVA&`GE+akL5`d4&+IJLBp0x!P}G%4i`K2DB_P14 zhrOvXanE9~BRtz}G@}PTX17m($_X)gYmzUNk<&G2c2x@>)s{c^Sx;1vKP1%TOdsHV z-q__l;d4VB=8f~Z@!-dGjmNjp;51?|htm-?_o8*^?5gseL1|f*)_%$v*ALRNH$jh_ z#K@Y!j)(QWSn;}au$p;|u2&q{)YmkY4wnq*mZfz43T8`BV=GjFHkpeStMgrx1H8Vx zE*6n1RT;tvKevE#eW`9_!hH}vD97>AG);4{0w-LE&EPD{TPW@76Y zk2UDbaj9U8$5LP4UwC~=pG?XSv_S=6DM$S!c3cT&LOCEeO1oXIYa53N#aGIVvH+eA!mU{8k$-111!d_uAJ% zEjOLoUOpSbV4LasP(&dz9=BPRir0K0Hd)HaQAh+UGt!Z~mTE(O z56z@V*5G~v4B`2D-$28>Iqs7oBMd!;;Xxu_O~!UxSqi+9X1cZi5|S5a#~*n`KO5L5 zn_kh1C7SY!4t4p7ZpCiy{>THQ`#Op7Q;K+;B%pf=eLyivD`(~2E`Wn$uP*9kRk2M^N{7a7Uzv~Bu4r)Fi-Y1R$DX!s zy(P3vP2rx>R2ePRRTO%vobGzY(O}p;$R`()O?p4@C_?u)8Ntxli1u1D^&W}TwlSE* z1g3R|9j_ENYJNN2DuC2MLeaj zPNx^QXtJQVprMs08Tl;*^F3;cm$0Jb={w=R1ZTcvi#CM4iBcbM?yWP|_m4w}_FV3< zU>vDGva_!7CCVCDp)cL8elJCA0(4eEc&jg z>`n?K;LGh&SiNL|Nej659=RN<$)o!HA$}|EKi>u5(5NogzJr*$(SCiXi|>8#m@ONl zaZUWE&!b7JlVon?w$0;fb_TvfKnFjXx4qHACbb}E@bUHbfN9tbs;7Ccxc`XDfAN^p zmw3ccf1@&akUA4!l1Mz|SsX3WIJC%?rkU5|h(!l~DH<_K4Rd~qhkEAhg-s<|AHM14 zfJi8u^2tMUg!ShCYG^+qQbN!gL#lm>krbvlrPMl!vzfeGl@iZh!lBY*7Q~N*xaai4 zlzH-6U6Y(9LbwtaP1{TXzCSgx^qYKMyoCzK2a&?DJ>!@hbcjtF-Jd-lSCq4;hFxWr1A27ywgeZpdF;4C7+3$LA|d0))wE#h zAih4Oeqly0`rzDnG6DjZ2yz$cp*7b?1!p?414AyN@8&-RU30pkDb`!5z0b--gdz^c z9yUed!)2hAC4TYv_wC;FPmll?ia;O(lc;?g@fWn!dK|oZV&>`+&}zM5dZ_cqs`#Fk zuh?SnEp>*Y;$B+?O5bPxvgSVTm2vU%=r~tZZWq+qSZ7gD(8_PAhDWh6nuqT4!<&zh zX&SRN`G(g3mhz3H1mK%@@U-ykDm0^l;k9tJb@r2f`~m(1@>h2YuR$ zF;f_Ki-qGG%5#3rb>r@8^kC;rBgu;i!3{m zR*BEurI9MHz#G5peU^Y~H5BN0SE|e!Fl!+!AW>dV$rnxFTaD^L(!OT+EpXL7UEYUFyD6vi&6|9@uJK`oD-{!g)4;QEv^RJG0zxavjEMoySt*(5 z_%s47mGo@F(XwYxR*CO&5jp9mj*1A`5ct{)OqxmKiYtpmRo_P@$u#CMi&(U!bP)9Q2>x?}k?r2zK;(FW8Y&F3zfgMGLvqBcXNOoihh9h6n)R?UM>1@}b}5<=xtJw%Ds;G}XnMV$;=om%9&p?gjqhx;uFuc+nmBSc^<&wDeI%lKO6~1 zJtp4xw9hk(z^wpLSddH>t#1z2#%KBjc)>h7Wq0j6wU+*~o3qT0#HOt<#h()+y@7&Pj^o0Xe|o47+x!b-`* z{_T50zqk0+bBtgtJ3hB83@*wV9PpSOJCoI4^RXaKC^);r|D{>^h?pAzerPaA&XGzK zoiTbcVt#{57; zD{ZH}I69DMPjoY5cdky54W!owQ#{bO-v?S!%#`l~{77J^Q zEv<>e4AbYy00n);Y^f!K-aM?;P~ zW{M4Bm!u0-W*!9)1!Fa~`e>bsjC36>7fUg+N&yy2!M~bdEv9ZJwfjk2iC!sCse``N z80QAZkOIt0yq$|uhjbrC2IYsk;Vwg7i|DM|$#KB*urgu2-x)I)myy(zN)GuXz?d}T z&}-T(CuVoXNl=gIkYHeM`N77ut9aPEAT^dgvcS_2_F$MY!$rH^DxY~~t6w)C>Mz9w z9J7ZV&{}GvK6ZZzm}C2a1tzp_U%wN*KZ>gA`)-$RQ}XyB7i;^=I7a_8HPd?{<6nt6 zxUF42+dQ@^%|0=}ej2WfeKB}>>SFbsYRQ`!=U;+mMy;*0epu+7_C%<5!Y?E);0diH z$iuh$#BFU-X=CU`mZ*^ze=r6bmX-b=7&t@@%+-TN<`t5E_g#BFtdh4FQB4BN5}tP&IAT<@`dh-KT! zK3x3e0>Hq+FK?`@LHpzLmhCmiAAd_{4Q}s`=b!Hu@SxRu*k8{&R=ja-vgQyMO#}^T z57|SLuCO%l@kdTy_^qQzp+v@@P*na^*RYb`YHCvcyUQPq4pSE&KfY663C3M2r%U`o zyuwTl?Q1fQ|7SUjYb+il94ZtpFRo*1OeVPg9`8i~j-b|(e*)w+mXj%6JpBC(qL<)s z<}ncOx0z64OsSJ9=&xM1(8d^N$+mmM=RFF5c=q6Guc!FxsKRNp z!J|e}6g0)PahjrkAzpT|yF9&#D2ix~U?Y8{_0h7fKYGjK_k_sxQiLI@=eBATKCmR; z|1E-0M^E31yM6%ki0b?&P<ko6~FOCJg@vj|AtV9jjGMgY+cnQD`6haxN zVxU-vQ!N(``e*<)HGO4MiBmD{@a19zruN&x2nk~nPs4fDs4wKCz}ZVtw`x-pFcoml z(~L8#Wp1RD)I2p3tYsdhARF?aIIq}~_V5G^H%Q+!dL42bub6U+e-t6EZ#>RzxY)b5 z+clEBoTPcSXz`)V?}p*teN_A#?aj;kq|ts2qA3{lqh2cId7*1IRI6&c!I(a+9l zA~{4vxXnfQ-#=7d{x*)6fJHSyeeK&a-;Gc(>)$SA@;8>E1CD%x??$mMl^*CrgY2i{ zCU@F@qEdj(BhmWtsLokGdOzE$xI6n`8^Zje0YBy$k$8?-k)Z?~Kd6Y(Mu#>1g~jjt zHtl-BE(bw{Z)idr?XDo~s(WJVpPGc$H}b}qrjWo_Dv4&?qUe;Opgg9YkD5;$yK2Ck zux;p6kULs1tV&>Rgbuj@MwB?0hPqc*f%DqQu~elR8eV*f8>VaP!UhR3@GGyk|;F5FTRRZBVqv$)QDHe-@Y#{dToA17n3 z4^ukDPevJk-W+N2T(9zo>=l^cq~H;`8U^@QP#Ux`M4I}sgDw&lKf=_eQ(R|+*tRM- z<-hGh_wsv}f(m;0A^uQaP3_aDi+Rnq=&HAI3H6S~tiR8qt5$BE;ET@)277%zv(eYC z9BjMX%E$h-Kealxm)t2VbG}6h4^cAsGlS@c-ztpCYST$g;V4N_5!NQ>G0h?8aluVk z!TQS<@1k|6C817AEL~xc!O_yAx)D1@Ba;OBb0!dJ*X#6jy^0&|7aV=-TBO%oU!g8_ zf%Uk#CE`nna^SrgSG$4k%ui~gzm5xS)IB7#YYxz~UW+j#YkKiqMc?0V@92#}O#EDG zI!ZA`O%)mPZ7qdWM$?Pfk4_70Glj97VE!gOl=XV!kMGH9`J5aw`!W!_G5a@#^q-8b zbr^*IiG1yVrKQc~PtHBpj|aVf99(&~&YeijuU%~n^|FlA|EpPqph`rKA65UiZ_qnR z#vY6F+P$O1(uwg|Z!w(x{hcrAQs_!IcpHe~?G44-)YY`_!v>EicEB*EU&oX$wNT}Q zndisTSuOhaT(3ZPSt!RBpUvX!NXp_hQH)SCKHBw2q{+x~wJdjdiObN)->?-C>w9_a z5a|dcwJ66Y>uoiLIdc8>eXews%|B#d2Oo#Wx;Bib0}k;^UEu@0u(j2-5RXzT z-*WPN9EVK)9Qzz$>wMuA&PYK)7a&W|B5;Zk^tc@Y@J%N7H=(nWvFH{I5M~6mq++>< zl^c>{YjWV`q7lLzGM=LNyLx3HPR>$9E)}S@7JPyiZ!W4Y5?RzhuV7&7vO#EGYUqqr z$QQ6KTTnalf?GX%@gISY4GADLg*;GW9%~tY#R3f|q-mtb2n}i_9}#h{oBT9h_j3+) z9LFi`LrV2c;#5Xgc|z|?DCz)d084;u7P~(`B5tr2%Q7E(x*@wwuSmn-PVIy)gK2*tt zm2?TJ>B9x%)y4JawF6hZr}-M7Wd%Y!_F!uyOSH+9a)L!qo`5D)C`}d>=GAFely!o< z#zg9O3TMmYbrnUp^9UrpsHaXanB7MT@C`fWm^prl$4!UeDWBle>askU(!G4>)_NV$ z*6MX}G2ztw`)yh31a`rwV}|cLzYcX&Cv4GZV)Q*9yQ~qdAN>Sw`s}aryP^`Bccb6 zx3J@^6QV%)f$P}sJ8!#}A{AZrIK58s>;P@B(!BdnT2(2OQo5bQZF$u$WG$-RsaHOi z(n&{CUx+V@piss}Q3+1{s;o;O{bT65ff8R43Hxb}er1TBwFq(FLj6}(6xNx*%2Xe^ zo_DD7b%NDv5Clh^Frn&ds04}{K-<;H`qMIMiPx3yB}=_T&;n-6!V6zIzF6Q<@IZiE z+jr{>*|t47o*!TE&KhJ)ywmSGecerp?|(=rI5Hl^!2F$qiU22m~H{c-B97Ctm0u!V(yQ8LiAFkm=OFm$yP zpYPQ^#FPt55$2kaSfa+A92(7<9%IEHmraia-F<$-bo_IvNYdIFc`7?k$QfK&TF}8>UlwIz;W3yx9M`LqG9?7i85n-+ktWq^@8|jmNEUT%&Zzg6TDJ${m49kI z$`ND5yeIrh*pJ*6#seXI4w^!qNvt$Gq6JJFNTW^Cy8YoOzgp$+lir{|dLgcUS8Dvt zrL3{lIKwnXBta8ZYmj|?`$>Y8sxR@a*C4 z^{@$jM~TkL(hYkkZcomQ$8_3f-#&<9w5HeMe0#ss(r0TIhg6S&Wz(d#9#xEJ| zN-=3D0SgL@_QbXr)+-_#boF}69=C>OkH3$$OB_xj!&Mxa9IcksP#W64`ha__& z0@E_PFlR&(RxxgnrNV2;Y56#Lq=2_0b33OEoQ)50&!mHxQQF)El0_3Ef7~*4*OPpR zmlZo`??+KbC%4>!gAzCQP!hWwI7$^N&;K1{eNi;44#5tsswa2e**7>u`77j2yCT_Q%IbGL zm%4#;@l9K!et=ao^Bqupy`?uj^~$d_vT3cQeGFi}UHaIiFHBKgO}pR!J4ZkP+~| z&;$2q8o2UOG3$VJz2vx~(p;uOn5y1$J9eoTkVKX6Q51#2DT9iIV^zm2Y%#up<>0J1 zTUIq75jYD+wCO=f z%l55CHHHZ8)ai3orE8`!->)~Ri;5^se4jpv$))a|&+V4aMG#s$FV;oh6hdAuZQ2uJ z6xXMv`5W*MzH#V(dOD2Yl$Kl3l~QJZ1LMu#&tdG0r7&en!)ryoP53WBwt!6+slo3TP2G?OQi6~b@YA~U zyW`%0_EFVZtb|X^C88Qr*YStqXXr1(&z^KrUpx*zXUuIg7{l*1kY;bPibIk%HallW zeI$JdnsIUB`8^>l?!V;7^iNS+NMEVM$|(tdnGcUF273!TERfgSS$6b8_=#9&)soHJiBI{|lfZaFSD&LlQ$5}Dkh zwfZ@lpHqSAua8@U6jY}KgDacfcsZ)GX6vGjyjjsVfq)mKL8K8lcen&D+D5VErds^ z-8LLtBjEa_P%D9puq4<$nZSB$Tvl#(GO@1FXfVwACYN}K0q2dFTjlXv+(;+;W6JkSZ)u(MJ@27U^6!)RHc%Fgq^RzHst;$_=+Z8QMBnxf zWcf8WYwKpG=$XpBLGQ58=U+;I`yNYECL;xC^PfHf{4%d*VoVvOEj~<5%FN|%8#>u? z=$;C6@DJqjCR|iS{8*{|Lt6^wskVExGuY;Dy#Ep!@D2JkTuA1#{VyVgvlL(n_w%*b z*`5)+Y&R%@U%5F!CvIxMV|$DOJe7~aOA*^D=H&Y&_O^$vQNbvRV|9qG$6-1eWu*yo zS;RR~KhLz|1hUhiZ^A-i!Y%K#$)p$WK8s0x{U~k4Fz>W-wy?kg#8S$p*?h~)&Cx=# zLQOmAE3~72@-G7-Gy3iaDZN)_Q?8iaSZ#E4RmlY(JtNiTsHDe#VA@Qq z5c?@B;V~&21Q%(Q%D-X3o^9nSP>PAu!^UmJEj*_cCJv7q!7gb0L81p`sw_ctc5e%Q zF3altXu?MvVm`y{=3oi-%XOqGA6{v=!cf(2G(I>TYa)3MdQ2i_ru{}l9SZk zn^YfC>B-jmO`Q)Q2}6UInYsodFA*!MSttef_IWC$TF!G^c`(SVn@JM=QMuVyNPSpc z2#i2ff7m2E#7hr==p@b7gNtY}TpHw38)9{s^h{x>>EopiH&~2l|0Jl_8*X2qxu>bu z%O4ruj=Nq_8=}GwM7^qciLXiPOpW9?oT;KH5Gm}tsNy4HAfN8XgV>&K)CYMa5l$)uD^TL{Xgf9yF zJNc8ppgU1CSP(|Xa9Xi@d1E6nw$hQP5B0Tl2SeoT$9Uv66a`*zWcu=FhK*07AVdExS#Tu$PXTC>RWWur^`LkbA5=>f`8hV=M}cwBc``@Kz7+e15&tSjS&~oznI9$Xa;h9~@pOc6 zor`%d_|RCSzj(0@aB?SBuWMNhFtE*hGUtFpE&VIVRT=lP-3gyc`XaH6fduHDCiPI! zA6i+eIwd8NthWBx@c9sam20pv$mTm{yf}C_l?0w6KwV=C1lrU8^(YkuHb%fT+G_@Q zsDi%uGrb06N-pEywNg0W7$|76BHnK8t;Ng1{d^pEK80K!&ZAPdJ3xC0u#*;)UlRhO zB$H*uyW;GrVQjH;A)&#N zES3>wQx+)K)6xLd$%Sk6@SW}16*UIQF!cCHkC_y8DqXOi?aVyA6cNfSjur+atG91G z;Z@ExF`^+wsxtug#SSR&e`PRi_^b$d6xH{ z&n#IW_j4&j?#p;Bk&*I6v-)JI%p&7 zaSb4uF~UbWVUg3crEskfH)xPqsbG=jA3s_@vM8Y_FCt!0p;~t<1+C@-D2oRlUZ&ht zSOGkJ{T|ceuzF%`QFF|DDIKr&|)1tdfO{*WuTt}C3Tef<5cqsF(5w&8eBrk^l9?H zU{j7uRA=kUTnEVq^U(}vAd3=X7_(Kiu%Bpp{FdY1#j|gY!hNq#`oW9RGG}swT>)lb zuqCOoGh9%df?4(6;hE!1e0B>Jux6OG2q7FKG85&>n@7hB%kYnPw<0pw^mSCn0c4-P zc>R{UF`9{3_wt8lnB+IV(?u+>?)TH6!fO#hi)dc}oXGe1T)uX$RotS+Z?=ql2_75e z-khHjOrjf2@v5el2-U(`LzE4FsDK5xG0A2Z)<>X_nvodARBZfuFN5JN>?-qegQ?vz z*EvBBF5*XwZGG@5GsFN5OKL%m8*HbK89WR;Dnm;_C+Lxi_37S5q|OnhF->CiQ1TQj z4qG)R2CHZjSmpASP7LD)AREO<958oW={kkdKyUdiSH)jpWj5YA9B&V()xW=2wTpcU z)__?ub4$<=q1l;O%m6p<%678%Hm zwUcfReS>6cG}Fdgio1!1*LG#D;>tcNIY)M*cf*)UCW6q%g&o?^Bl)F3#gCdg82PG= z!f_XBDS6e}G_=HINn}NJm-#uDTs)prg1lv7#xDUbgnsRqnlQ|4G$Tu-QNt|Z*V>F~>G$h#RtA@PKy zPE0Sl;&E1}a*HpWPR!xGVia4gAz}m-<(hj41HuQwLFwY>hvC-Sf)8trQU&0n*4~}+ zhvo9Of1v$<_z^@W1wuO35b?95VP`WiY2|}6r%JqZciF|0{adz@UMCP@IpV$>udMqX z@W-U({*QNVYuhyji_3`Ww|q!51^_>q@e8CKiHpJQQW;?m?B*-P&(yJvQnjsWQ6wNo zAl{0=>$ULgtRn4rZiY?E^GLFi^}@c%ctENSXii3a1itTGH3XmiAk$m(s$k|03Lw65 zMfA3qxb%G{vb25(2Yq&&BNFi znF`|wwVST2G<-9Ib03=v|GFB154A1WBsG!zxOnr ze;FMDKUj~jQv^KyEQGwVLphD9W!^l!&Ys(T)CDuq+u)f1#hdd#6ENID=HaLf z6yW5o;%>7)<#w>{bT+i6$upT*#=X2|%4lJ-%W_el0lZ@c1ljPb0O!SC~UIVA=-OZlGY<(IDfR4@^ zTJJQ6!yM_Wgv`A+t$*K+u79Jy`Q>}mhlk$?J(oDENs?+7 zTu2dr0a7D;uuK^7&f)$CG1Tw?bf|4@zFS5c4(I*wvxLI$Y^3hU90k8gZ%WHnBJNFo zM_;sKAEeznuPJutBmjF>C{M!8f)RXofMw`pFp3%GVs$Rh@H?vw`S}z^$t0 z+pdt=5jTOSCgefP(WgVhb^>}~C~%@}%|BQ5`CY)tP7bM_>)AX>ukuKawWJb>p88>D z?h8@?Haufw|g&^B;|+!_)strqlt16 zUVDdW`(^X(G>IYqSTBntkud0eyYM&1IStNDC44X8D%Z}*{OV%h&S zTG8?*-sIw!#7n!)mx*Q>))n7)M0F5_j;NE@6Y2j1v3)%1*-WRQZ2!}m^!WMKQ+DzI z+uu6gI2Xf)4yQ9uQa4^JGuU_;@n#;s7D{-}Hs_qU3$B;YYf_(&!Yeq&-Gxsr<8I1O zIlJNOyC#wvKvUTHWIwcneDg#DsnQ&%mgRII?FFcNe=ihun#58a~X^4S7rvRe2rWDQR{ z((eBr98R9UFyDa^>`?=v3oeHulZ3PM(qmfu&PundD9=_APc@7?SCzxxQ($Q#+93&5 zCN#FdZw0c{ZA-&k8vL#JcdIUyB|^3`nn5u#&%&+2*JePT6p>!|j#bUL%)(bz-R7^X zN*ZQ<`xd_=5xklRPbnI~wQLD`UMb@++Z(z%_=Ed|2KX=pDq*|Hm_BF{Jkd`Z?5R2{ zIt)_t)+~S2ET=YfT*)*3@@#}W%=}|JL^K~cx=L^V+9o7@68!2nGdHcJ0r-L?JuE6< zO5c@^7QFWqnf;h{F5-{pbpDWT#Ddg&47eKubL7yT)(63oyy2QN{)d+C@;cDP(k|ak z`T98kPb}47#h`0C4J>*A`f>z6mmZ>2+IWvipHoQ?aJ^r0izb20rlW^ZW?>Kw&XK;; z(B-IDr$@+ZD`rl=ty4vNQpZ`#+*qE`5za^=WS-H;#ru?A8E;(8A;d9?<}JC^*EK3h z!k=72<%rnYd_MYUGf_QL1!7!;p+#Z=|3bSBivmn5*emTH5u1>v_3RxlOeCj1%Rq2r z`#G(TLlsWajjDK$+9{bV38Fd_YJqov7Sn)WX_RTB5i<3_{FKyHrO z;}94fGk2FNWxf0xyGG_Yz}>t^@Ptgp&kzR_;};i-A&ENMcmnG&1Mm0jAF`7@5G^@M zGURzPgmhMt_eRCkIZl9+*{AE;^M>hbhNtuC5$i_3l)qG&oBknbNP*Nz9CxvSD0e0$ zrXq5(Kgd1%e3h@^Fb!n+wpD^!NGADe=k9}FF0@o#l+3g$EJZ&^e}VA%G;Nx4L~wtd z?gBry)o|gbU96M%xpan|aS?3NxU}YO;W?V&a**O-C5j83_cR9y2GlXBubv!~0^vFp z>)IL+1J6GFvp3!6JYW98RSDv?A>!eqAzV4yVKBrue=n~+iZog!GxZr;+=0^S-PC_e*nlh80IciRw={B9u zz;IuylIHi}a9>Sl7p!X#<99zT@^ds@LgIP_<%eFZZ$tk>gC?Ax7~jogtMhY%%T`dY zSZJV%^{FTViUNO~jMG*E;mM8N+ClYNVPjkbR$M1moe252At^S>6k7@~X}@cS4~w_X z>?8dHBXa8}?ou>-?QTiYMSo*z8Wtann*-znWhAEYv6sYK@(Zxgv37M?OHxzov_;-3 z6K}CKif?J)w`}2N>Ds>d6@0$%2}#*z?oTMky%B|OpuiV13XuL;4zQHIA<3TX#VcZn zaX`f}Uzf(GWIE84jsgG%^mVWCm$xfCxvr31djz!`nJr*t@u}6`cf+Yxq>y>Ky`8U( zc&mcyzClxPn-jAOWj)mqsr2G-p|T@hKs)%b0piC^=g~eret8&{t@+jYH>mcNPy#Ac z<#E3w{4J=eL$qm{YAJm7mH(-Wz;4y%?_%J`3W(i;U?8Jgk=)m$O8&F9PYoAUh+*M2 zVLMq&4P95@oAAy=ve*2R)V$f_Ma(rO2}vZuHG&t@&q+d+g~9~`+Le@1eWGsCb83O! zo9Z71jhgG4UC=dPlj7jjtn~8++BN;DYcxDK#)lBC)R0*kZkwsMI^G1_F+~Dt0Xso?yUQ+2R%!)*H zmhg}7n&ccy;G!DJ+&BIwf+<>j{%BI69I%j3*f3cGfMqJ?$d7u4`C@gl8#h5Pi2aVS zbirW=OX>5Jx;b4V?_m>Zed&4agvl8pBKD-1)(klD+jZr}G3$1zK26)({Q2tkK%}`7 zzy1FI19a;+`2FS-``LfMn~l*lDIg&VZ7ckLlXwEgiQCHdXJXvaTB6l||19yGjd`5H zfQ>E8mQAX06qaZp?e6rKL67ym<#0Qx?44809q;aKiZa#3b&{WH;6v&7seiY~Mpe6B zn-m@;T|k_OlKE~Q&hyu1_@luUbRHGs$m~zWHs!=JjWVM6$u&jP!CKFf+q3(G6goGC zd{D|_j!JQyxalWMmB$%~N0nT8czFqs#s>laDj&;#oSEWI^KGyBTx2D1N(fITk>1j~ zpj{;sms;vKKOQeR`a97s5pLBgKOapcvTk5h^hbST2fu z8{&6cA6z~NtF3e$3*S6@izOM4{UCCnqybFwB2fMPi{ElXgWFmdSJ{!G{MUkbrR1L# zk~m?7CIe4eRtNGOdXB!%MPRgyEOSXv$!11aTHWtWFp{x^zorQl0OAiOJ6YKQY;MUvYsPKj3*lX4t3}IKokH zq{P38Ls4|-HwF8TOYM^VL_XK|&pl~=5k#hyaot(A_)UWNmAfrH^JCna1ppGeb)xi7 zmkzedJJhLeJ>OXDN&jATe9NaGh&LJ(ZDU#w3@Bx6|0VF=Yw@0skyQI9D@v*e4O%Vv zWD!uHhZC}S8upco0=6c)#r6urYO@UsomevA?Sv!PkvW)x+5%fJaog?#Pxyr55VpP#Liw1oo8#SzhLx@S1U_wWN5CqMJk=ANfd-(*}jOHnPRcKifrvzpKd|8%SL? zPnqb#g|2CZn94mux`0j}4OV{OA*6k!rs$fOXl^PQw`lSfy%J@f=Sr;rrIo*?p_E4O zTwTaSP9H4TYe_F_EW-YN0D&1-9!XA^H|$`y9-mfJe|wG+Qa!W;8S`E| z7#4c9mNPcjZ)W7m`}{CojPac={J+@4RWPH!@z@!*kaNvc!$1U_ohE#lT${k{T*K$L zf==08x98rU1m61GzHMvg!uF5n+O08409Ri8G6kHkloUAppulnz+ZR6NAIi+5(6Y)5 z1`^XEUWw|3KrBeQ#9%g10Z2GPSU6TZ_Vo@fGdzK#V&|%%>l31snY`A}rBaSSBD1sV zAE)jMyg~aRxP%P!m^|d&rVnoW!LkbQU*Es{hsQkAEcfEhF40@e=y2hOi=72`=8Q2r zh?f(+(#f>`?}T5w43q6+yQs(*MR{nwB^Oi^*AcM&$gRA|Z94FgcKUmWbOF93CLSF7 z;OU=kcn^?6>{_TbUA;BesQXTacNZ zu=7V(C@KTLBT7}lWJWy;Q_(;dls#G2pk8CLZG}Aq<|~*$4wLjhw!QgiG5hyzGPtd( z;$8B-&iTS~li%Jej?PiGd$s65nDg!GAO6!Br-H1u?Vzlik4?wPGl!fvp0}xi-qW|Y z+xXti55(O+5XJ+LVoRBeoC^7mbyQmyQ(M+ht*GmCl``IYRFl__`>l(*BbrNt{foTE z8ASu0@u}7JzfSAtpSG;piTFxJt^d$*W^28!s-xLR@PuV`#u$j|SvKgE(=UQ3d4O`F zI^*$ZKVa-pKku`?*Imc04^&||`&d{(BD;xRTc||SpYGNUIf*ES3u8W`+PcDCs>LiK zpya!9&FTH9`@ROxGw!y$>1AfCEG7PT{r0D@XEFA0;CCE58GXlLe6dQ#7Z|V7;~VFU z5Tip;O`A*MX^_OBD8#@(Ure7{eqX(-bJwjXasSot+YC|Z$sRm^S5Q0@?*%`j8QU;r z*+fkstjUR&mp)Z`x5nj1l`70DK2dgzTGF#u8Rq1UZQj;gQe7Y&v6v?uERzo|ipO}V z(WLq;H8l#g6$%ynr9^eZ3u_nC10j-bhm(49X(_mY6&1GggLqUV6;&`KtgTh^bnBX(iJ@OtX zIUaFc*n}gTwkOaNC+#|LHn`c*x+|0yh3$^O9dw~f8%jXFm&g`(m)6p5q@-L%DRc+kxduC;!h@ze+4{`=M2GCEHlyerg>M z5e8M4g+AEg%LKn85o6k$zdbLa@1x1MZTy>{q@nn~j#a}w7a55@)3Rcr9JK>oF#mAz zlq!Im0c#Ns2oHx2>U1;Y7Yq0Ia~@n6I`)GeRSk|l^=*!i^o$0-AdXqN>+pZgI4-@# zr*ORamKyfT3-xqXlmR88hs~3p>jJ(0J#AYxlO}w8fZ91%8~`OQe@XhxZAiivsN-bd zveeC}E~G`CFc~8`EKT;By*(A=m72?^EMKHU6^r0ednsHWe3QVzc=&A@KB{zvmQzeL zDuOj18MHlL9S=@^0W2{~x=h>L6PUN6C*DI5bz5O*_ZgyopKCZz3URk#=>)R}^&FSa zMs2ATY6EXKIxt^N(^sIY6b3>!7rr%qUw^45I3V2cCfu!35(E=%QqTAHSV|qK86czgH{nmNDoVP&35BMAXCC=%N=-$n=%HUl334)>axb@ol zzVRSmf~a-h!1HT_&W{`812UugANGSfXXlY4l%@MCH?1qn4Ew4^Y`_8{bYeuoL8gtu zztt;#&Uj$DT1LsGbEJN7PdWgdd|ujkROV}`T$<`pQ!=@9_)Rmg^{G_UYIQ7<&|ssG z2DO$??zeiQ@99+6-qY4+0#3Pj(#Hn7z1X#@BFbX}XGa$vMZ`JR#VEtP*x=+Hn8K?U zS1n#P+ckMmqJ{k}7+Xs#J?Pmi zwkxwdUi=r#9!X)6%?gh32*(R;rPq&1_D3_t*QLE*KWOL!RG7;fj{Yb#`WWg>;9vj1p$8@-|@DX2NE`HN4~J1 zR48Sx7zmr&iK+`s*%#Bm7>{6ux89XuJ1?X4M>rQNzPC8^pl&w#lt9>)i*M1Y(x4jn zs6!PeA6{e^BG8nG*w(BRA5w-@7wPz!ksXN?oT;_%Iz()bTF2>v%cBT8G{)49}jF%;TcS7IB~nPT(VGx^l@+ z6-f#|yrt(HMhckRCD8USK+mjw@rXkG!%_#jYx&sXN8PbqJ(-KW{EhP+wc(DIOw|-g z=HB%f2FKX-9(aoc^U>X&ay4>o$$E8zH_yI&+=v@{gL-jK!F{BqKBDbUK}Q@^*EQVZ zDJmyk9ndG0kWa6v`UeO&Rg7!jNv|2v?Fz;hKIMgun$~HmUY9OXzc4w_x{L_+Qp!~t zvL@L`!|sXY^TXsi|30xmt=~8ae{Knt@wafk;i0_LZk%0E`Vl~#t_p;`xh*86n*E)% zMC;uOrZi;mF5Ch~EgW!+mtBzoP!nMTp?ZBKMi2L&%qe&)7VcvsQ)no}_N#;|7@hVk z?$6zkh;44->q_SSZEYYkpn(epU!x>KHw)L9ExtN-pLzYp&h@vTqD^Zp$cQS zU5jY6t@%SJP(b{IDjw0&J3!cpx$kf2e){LdC$9@d6Jb|?3d|*{Ve>>|Q%5*!as2-0 z^KYuCBl{&%<_%rWQs#iI(dVYOE>Jk?XS7QG8ck0pVvG^@wNV}~r`d;|$Ct?<-hL9u zH&tQ-im!q##i3=m2ISB4kvWLpa==%#=&anqAcyDI!jj=UKCbMgj*F497M}8y-|)Q- zF>6ow7-uV5BMAQR)(R!PxE`onzzJwjY9obPT^2$d1=vX#e0=wv5ojnGR`xsK#Q{Ml z&W8!eH1gF1R7}{=SI#onr>AK7itp0_tCUx+&u0tB7|adN^~PGHj(Ib0e{%4oQ|-h; zx1ya#cIZd3L4)G6NAZtPttUmIHznVybVx z6ap~os-NzSul@^PuY+W+b|9Tczqm?tgk(!QUT*k_{Lw1C+Zg47V7cf@J{ynl4khAQ z$L>|>-B5-R5&pPRx-Y-;3pSK&SB>o6anLLcOyJ=&14c1dokKqTVb{ad<&&j@ zfYi#z%EQz{y=>?8eSQdc&^g%Fx4q}AHTF4qbrH5>%)0oV!>&#sk^6S%+Jc!7R5t(#Hvq*NK2@WE`V44iBX%-lzfT=^1okjMgU=ifw%P+SrD zd=x+CGf--L>uM=e5Rv^x@f8GLZAy0DzgH6hz!>DzXDh-RUwuWp@PRG|pi*i_eL@f_ zt1RNDSarH%wj5*idrY-94?60Ke$8 zJ{7t59XZfA_~TtUou>jb_FcY8y1$J?aTW@dmBceISJB}C+S7zAlMXfN{?nwSb%*6O zj^AO}RB&5!ypTX?nf?K0Lq>Almb$RBfe01yIw#7u544 z`vRLupWJ&|UnvowlQk#u^e*!(aYk#QuAT%&&CrE*m8;)kytIrdmwLEsCM;DE#RR#& z_`S5d;tH6-@AIx5<|aodQ`w+gS}Vm~z=%AZ3+mF3(pd4-&>y;=>ZylHZj`xR;1Ij#VOy9szYN z2W;&|<2WKuw|pl!l92LD>_j&(`giy#8J*=L8tUCz^WF0sN9Ju!wTe4t{=9C@im9y)IOHz_1YT!5o>^NkAkz& zIzSz-$$R;=u@}f$xtozQwd#)u=JE)Y=5jub(7axecFszrYqCi_^9Bt$83(Xl%tZe> z+YYrXl94KOUP9ORe6@!m)%&g|14}xC8wC_Dz{!dnnaa?0Z>gz-hmcNuaVF7xCjC>p zU_rJuCterR^fd`$qD!*x^%RdKF%(f0FB1}smInuUFn0pPf1}QRT)3hEkwr0undj)}yXs=cShld}#*%afo#P`q(8G#NA%1zByGAM9z} z3@9qogBY<=SM5*1>r_OB&v`)S=FaDmiZP}Lj|3mn*Ag4WP>TV zas(HigSPw?yx#Mwh~a^cy((IqiHMI=BJz05(~G-9#(XhI+NPgmD_?PFvKo2f!ivN) zn{BC5nx{M6qnBe>*x4KpP9=` z^i>EF57z9~B25hcPGoS4X=%*bdO43laW3{w1J9{p!IaDpxlepV2i2x^Hq*I__ zqWY9{-Ms0hS0<1UWm*ji_yN+a!EHGrDxYawxQ_b6mA}59Phsz5{zl~i5-|cp<49I<*1AyZLJtyHdx>1~+}L z?5Qm7c=*$!M(v*fH>d*is``)5RnPS334TA6)`frrxR*YAs=$nMPUUAlr`qLjJ?K6i zFPZd?_e2Jfe~t@N*9m*p>nM`eA$&Fny7VbVkN-Zg#4PjlJypW_IOBkgfuqI~0C(Bl zna6{;fKY$dGH0`0)Buk0<3XI*hB0=ZW9scDwnT`5~C|^KX#(R5r0;5-uqSHDG z4!P2JyXpCZhWi}RPvrrE+}dkT1adINJ~#UiN^u(7tI%7oJ+Ka5eVfjK0KE z*T6)|NBWqt4R$}(YeOCdaLF#RgD7;%IKGm9;C5iPVVn?ok|J$*&ph8lGQs`@F~dl( zZ)mj@KUzfz1FEQi)UW|wA;EtrwT>R;g=7Bq0e@vyXM8Ux)h>jZ6BUh`O^RISRYiDPW*#Iwld|WQ)E{ zCrV1{SP$Ag2?U0j5crJRp@|$90}L;(dd#;;g}cDFE0BcmssT#-%#`(5PU_)QP=k*z zjsCV%@XpS-EYn$h;rfL3Wp7AXtp_(0e^`)4;D_l`+0FFYmc(FY-3)TU4Czx)xQWndoifJ1+45>w=&@QCiz4ct6J!3v zPj~~YE&cW*dus}h<2kUS;iDyI^0X>2gZtBa(KF6?i<(QLV-cM!ynAuh{s?Y%FbV@k zloe}rqoEr+faSC$KY{Ns$+%ER_<5TTfj0M|jp=y4lX)74W&0)z z&6AqIZX``WLyIY_F3LRzaL`yBZ`|Frxv*&zAcZ=D25pm-Dy=GJhTb@4|wH^*-x~QpF=zeNNUa^cmIHcnmPcOCX&cd2WO>Ln<_K zr~Yde+nN%hZC)|N9;tagE+s=hEQ1r~clUC{u9yTl1yQi$5kr6;OOv;P0^(TN<_QY; zLcvAasQ4dqW7ubn=y8{mPEdGhmMGh|31Z4RYGM1e7w!0Sfm-=5f1kE`vquWwFE@d< z$gkz=nqSjVpPdxbVw8Oqpk5Y&F~lrwHE;h9o4}!nRuK7rrs!CMw=T$h!fR$Il6(m~ zyIZ|ChT`#HekazeQm=WNWXTz0A52zdEn1(p*|u}OPA(c@+N{cG^Up! zhyJEf^#OAed1M@VLXKGbdZ|<9@^Bk zAT1+@`hmgM1?^`cET5m7mzZH5LpuBYwY%3J*=Z5Bvm+6-IhN#4Yl2u`OYp4(=+;oq zQ+ze^e%~_gTC26^-(O6Zr}q&zK$T^6kKe~qs*S?b9mWpl^RIeaD5!zD_cjOu()-Vb zrt~)Dp0TW-k(It&sp_g+iF>95fJyxlkjH3@9j%u_54baoi5)ez{Jwk_P=R}y9>7o0 zG+xGGUhZ08HF;yPKG^9;v-S*?>~MkPu7S%lsLVJ$U#9b;EZH?7XSP|Rlf{Rg36>lnUB>oU>KOJiJLPr86^*~;7q zNS68i2sOR^!`5Nb7P^Dbd4GLf6*6 z5$U5VBfiCh-v_e8W;E79n~p=6ePJ5HKMp{KX-fD{pr>_w@*uXu_FEF(04T%uo|`a8 z9cTQVV<=qp?3rCxgWv?DD4LM(Ryesl`*VwmOQE%?|2Ay4D(DO3*vpHvQJOPda&3UP z} z2y)$&-bf+>>ib#W?bTcz_mm(97_NihXr1jy4?53`H&pxG>6-N$_XEe_o&rXI&ns>88h1g>Vw#^J{ zg{0+FTSeFWia2p-$2L5lz1FF?islSMNY+BW*5_oj`!Y7+mNu6LJpQqL>__!zSdq_E zt9~LTXvN?@FS4_gR$};v56|g`mGV`#Wt2*8N9HSN3brbrYAia+c-j_jhEjosKY;S+ z)z~g5_Ywc;K~$ragxa266g9C-`wvI!Qo=n*@<4C@{1)0KgEmDG{n}u(lEQ8F?y!zJ z3~ba>LmbS@8M|zwM+a>7SQ=E#B-x{L+lxjnmIq39K5y2NZym=q>#%4UXHJ-7DYXJHD20;WwY;gD})VXzvh+>W^xKWT)uMH@ab>_WmNXh&4wlF)@jNLD%}g+ z?aB&W#RW1>3aj41bQ1RjR}UB06DVyI2-PENJswmV1q8h^!vitxwOQ)PT1>GY!5e-7 zbUf-|+Y(v{jOobp1(rDk5DO4u-W`=Zi0= zyBQeuR?*)-Z&Jac}N* z4v)R*59CW^8#&2kw*xm=4spr~1ZAp{%U>U^wXA3WJu9BYlPvM;5a~|vJ40N{yGu)Z zO>N_;iWCD?xnfhk-Ebn7f_BuPomd<4yw`B&igBSc;f&!7tV zE}TL;swZ;jnxTUSH-H`6A;+PYnlRGl`0Qo57h!Y4NY$F0es-}%Wx@!5!Cei5REJ(i zs;_~9+s%c#y&G&C{u<4- zRkP&74um~$$fsRJD|Zv!$)Ayd@+DwriBhlzP`)-xyz3a4%zM^6P(ca1src_JYM2Yy z&(>6{gkdVrS|k8TW5~rz^>79T=m#d}H$QaZizfVkwul!XPUs@ap*P zn5>oDvx8o#PC0~D^|6LPFvW^iVW+Uxfm9p|5s3rj)WV%(wU%e7r#pKfGUPrRA4%Q? z{97zXj&Wtae37)0^^cq1aVRML&dsmn5(UHmBMK^0SJ3?x&11Hr%gxQC!gc|*xRe#x zuE@P?XZ6b+l1I_|hssigr2P2nEL8sn#Cu-Krh9sR&3RFhW zGG2wyeWfYy6za&%1F3}HK+UL(^hc7u|3%VCs4g5ql-MgH5D6F#!_c4h0>8q!shy22_8>CAWLTPCNTl+#4noN_T}B6>R~g7bmZlk<|+^F01Q;+lO!J2~j&_r-KrYe@b=dv|O{B4zwR+l8kMC zeg6mtY9IcB8pE~!J*c560iy}+9#3$2zZnpdV1crs&xFL-gMOb%1@RCe<%dMY(M7ExRAZ+@dhLO9r*yPHjN(A_ES?r}-kFe;w>DKAI6+7uJMd>6*ixgw)@@w9 z@Yjt3(25MiTSlD5v@JU84jitpMqH#P-!3^j}*^I_2;Qv?p z$%uE6l(jim^6qjlih6k1S$%(Tg6Yce?7jHnwJ%1R731;gwz1r%vK1bv=%H1fdygiz z60oEouyX8`Z$eJ#ioFr7axabP;X<)x$kg}x0&&0}MZ|$4BF;V)ZE_tj7ucp7=9Z_8 zM9+VpbyfX7SMp%8W;LwtXFVT2yroq&)ku+U11=VaPeX&V8SkZs@TR zCwT{}Q>KH{tWQJaw4yNhUwdOv-0g%+S~Pf@H$NsVdBF6wvb?CSd*h zk%>i7p%d+QGuY+WU1iY8vnmN##WdYq^k%mN4Knde2w(SX#ni9e*j>ozOVUcjDb1N?kX@(KCqwtNSNh4`^$(b z#@g3x)aUmgJMcdSKf|vfa2kdlBS_=s(dV5~?@xH%u>R z+?#FI-bV#YUCVn%DIx@^6+%Hs(j$poYy%^2HeT1Oin+sVx^2#GdY#?0{to^!DEY?1 z!)RH(YV^lB8vx*D&i!tkW6GJ{9r#jP3@Kk-_;8r5aw}&9$_L10ij<&*JY%2Xrf;hq zOp;%Ks{fbu-s-+2>Nq8{yXVdL(SIL7DBd;da$_ z3FB6`56&O2S~A=m+?HaUNPBL*n2~$U7Fwafpk<`*`pitQ!kJ1HBwJJ9ht#V(LRnK0; z7sj?Ij^*J{xKrL z5(^MXVj((>gb6<%dBq%H8}@5#s@<94v+rz*{YIn0cgOcsf|pNwtrQD4-#&64tS6qA zyzB6Ya^G)ejLxy8ioH5!IiuP5lJrAR&YX(L<@dbSnO!6`jZthPQ}QlSQD*WB>0n+~ zKjv6jrqS&1P)B_NCqDC@-xM*(Sjw!wKAenA?}nNS>1o?rX7gIR!5y}xSpgTx&ox$W z^Ge%9AhC}wHfIMa5d>Li9eeMUT7%*SuU08wJkG_)?ah<%qt~Hx^MOBo7h7r(O9ID7 z`QH|UT~GUeKN@4`ngaA=MSndl2!a~#qQ87SbP9XgKzDB>C9rR~S8f|NCCKVfB|Y7Za4{YZ7RBB+>g{+9}oyQRWae3?#VH7OFBt ztRr>9MBIdu)vYlUJ4_nUS-G7(w-u0|7g$-7eQ(ae!d;j6c2lc`eH-ryA zj*3Xdbu=zQ$CBf6f?pLlK~C?Ozp&HMGGgy>q-M8SC_`3fA6w2mi3m-2pG7gskKflH z8M@GT97b%Ecf^rnZrba3c$Mk1(0%!b^sTKLpX^9Nn{6F{k}h2dEFsw+xETx(JRWj@ z1VSSrp92wO@3z)oyO-0=|H``P_EtOUnr{F7y+IjFMv`6eItZ*<_3D>-As(zGntWug$CF5C?u`I=hg|7UI z6s{!|QQhY()7R*pAz^KX&D#G+#p@(^j&BpWb!EwjNCJn8xtjZ@s+nKceyb|IG7Y6Y zcmk{6wC);vGvFelk>;;mq!YQo|4}Pl7dR_ntA`;Rx7cde%n%Hs7oToEMyl63d-qeC z1lF6hqp{0$iJ1N^YQeOlnpr_-NeF%-CgzI4{2&+Zm|s-B3lIoA4k0lRi*aYuB<2Tj zDQ~bilrG6$Ey?23CdS?DZSUs?qvcP`q!u6IH^;#EeolR6ylbmWQC>w$t%B_@A9=4= zubtoUqJVxComM6b#h8)WP7R+5ndv;4E%K{6-FftLG5Zb8IT_qY;m*>0tI4oFPDW)1 zEtE;_zZcG8_l*$7D|Z8SBG6Kvy&H5bY>ez-k~3jeM#p9v&%Ns~(|4!pMffu7rAAjH z4NF{>GCh3WH``HRDEMRmb*i;YbQr^8fttnZ>IVEXqgX*PWSeh zV{c4oD^p_CBZ^*S=-}Rf;j?`G%ix-}2K%4e-WFTPB3*5kl(;KeUdHa9;q^JUc42R? zpx~POF>3y$ih6lJA2B^O0$AjlRRGr*@hV)SN%>9Wxxv#l2DHfz4@W9Cd)NAH4j57W zEBYpb3Y3qH+hgaai)w>RTy$JSm`d0x9GmJFYRQcsXI!2n(`mnNTnSV=cW6;1t$}=x zYU~)0d&7SScwW25M*wnw)lS%v>!Z;b{4zM18E+qPPc@IJetS$F-n+JqM?}l^rG6dA z`JS1UDKfo682CO2PuHV|&fa&Hfd942?C#TUK$h#^=g=c~=WKBpZCGvBC><=YDBNq5 zB{Ji~FcKr2J1_3qivTLzl!~7YTZL7NW-v3B*%W)|%+*<6c9XvWnN;%AbL5*!i$^=d zW9eSNDT;anNojvG-)M*Y`@(ko`v&)r0C(qE6^({DwrWiH9R(n;WRI6)X*R4@&Rn@Q zb8RD2X*JNC)iH3RO@;pBvKXYjv)~oaE;VxS{>!RZCr;6H6}?vU^>?3!XM?Eh`!d!cDJwY_ak_(y_iz`1Y(UK8 zHG1c^@+6z9f1&~(l`X;Q+7c_2vkDjz>_dlM@TA|Pks%90jgPIdKmlqDTMN`issh~e z9LGK3AY_@et6PXse~6JAT{Pr#%b!5r3#*n#weGF|ifoNWd{g`Q_GI=THUF2*w=`O^ zT-j0>82vm#@_?DSW<=paE#49sN z4Pu%KxJr>u+mlT$&t;3e$_l@W3onUs?CTCLL?@VYK z!7xEY0*Pi)=}}dBts|qit5uk9NVX=$U53`F55K<^P+;g*9X+dbl6m3za#kt&^V+w1 zI#9A=M>P07ezVN8-_L&`p00EHATF7m6a&bCH_57%>cxEVq3A9}g(gAxrZo=H5#qALhFt%Us&FA+^j9D zUAk~d)yAEJ!T@wC2speCR|YEt>XE%e-%wo|F7SdFNiK#wY=^x6AE1*6cW^wLD^$9K zPW9(uXAUvkJ$Y93zg|DSHTw=zU#1ua!_=f=e}9vVhCm^bj992(yEQWEwt!N^6%^hN zVofaZ6CqeLTv!8c#IfLC5Q8Lrl4@Ly-f8T5>@iF*ml1<(n zi|0184?qX3S)L%KI_?7+`7CN)x6jC2`ut7*TA5#hEW!wK2Z|%K39u9MTduv_uhAuH z;eT6Z2|>ed1qU}=C1BXuqK3=?M;nm-#zS5?=XU(J!CEneyky8ftG^Ho-jgKD4!JTHt`(OW>NIIgFVf-Y*~#?B0q54zd3I9O z3OGp9H&sy)jM4tAepeX>k{)od%+14;ax0_+-Ugy}x{pgE?3U_4tg$W>^K=_MSx%4- zUGNT%fj%=rRC_Vqi?Cz;CI&N;hGIsa_@=rcs)5L1Y@T&JQx170>n!_%?vsEL=wuEn z_-*jhdyKES1N^(d4|^!LLUiNkzG*@bY6n-jxBVuiqq|EdYf@2(^F5j2@?GoTFTPo zArfzQZmJ@cL0Go3_kyW=L4Hd|nnTDFEZ5AU8~kx3sgLZ1UpioUeo8F$2MVwVMPG^= zUB%`KbidbN6hM9>kh8sU<-ww98%G~Sm`*57Od*RcbVmw+!O^f@9`=a|Kz-e0<*o^L zCE~~#qPC}afHv-RytP-_o2`1T5s!tcN z2b;{ehtSc7W2n9d3nPBqX&W-P@6Uq+8GZP@*D_do7Ban{qA>luHkBH$K;}lhVStv} zN1njTY*Qmc+`{dr+`z}nTy}44nc9;`o2^Pqu~$$c_vi(`8X=MvfTgcp{>sdl48r70_XiXw48!7*QFrQNP9@}gU>0gV zZ8qMkd~lY(F6FXvDT@>Un#&31ic$<4u?1XibTNdXlVmhH_mliBLjAdft{oenq-**( zN-0ueS`x>qabUEx4M|a}0OGQ{IiWs04>=rb{LY_( z8k}DlZt@SLu=#FedVCTH!WK%c&yO0PG~Iz> zy6zt&_q=ugBUWj9)t{jhVF@=CFc}IjOQ|y(j;AAqb6(nYLkSXM?PZY2&htB{G_>NEfOPwA` zk8;(Pk?j7m{bwJMdVI8ra8U#rF&~p>#~G1~x6(hhvl;eWzdHd1Lr#_XjYjzPep^a0 zkC=VOM#;6Wtg{p53xrm)VrNo~{3hvxvM?e*r3b>f($?e0#*eHxTQ%nu&FpB?jfhs7)1X@0AGy0&xH#lL^--urbQuT{(EPweR6jC+R{f5zrXFNGt@7Wgd81C!QKHufykVZ7LD?|t^JOuGf&yi^V0D`C@sC# zNwP0Hw7p2lK@kKtBwex6cJ1v=WSirwWQtxcAt$fY%)ryuJRHeawthtE0SJgP)}0H$ zNy0^@3O5*ti%hnWQqoLK`iqX*xqjAVen2KX?IG;qCwyQLFv}6Ly+~d@F!z%yDxf5b z$Y@VrPW-Q9WlwkI74;J@zppw)I=7ku|ETYzmdbuHB+n35^g@FQDydNe{G>-@F4wk@ zzBAsRliE0)R+Ff+0lr`D*J8{HaZM@3kr0BXrGuJpdFj zBUkWb`~M9iaqHRtS^KW)B}RII`4?H>ys27syiUE&eqhLcw3BHbD7egISzJo6&wA}T zDSoNxyPob|;Z1@e_Lp#Vo=_pIuT>B5OE zm;yt@p+^jwz-5n+yp6^$fI>%)1B|BE>5fj|sP5mG7Z}hq!WoFjz-VS7<4xgbNC@>) zJrSZwGC%(ps7o={QYwQ_Pu64Ep0#AzelX~l;*HEZG2se4;bG)NQUa@R%6Xnc=2Bze z*j)#vmAJHM-AFnh^W#fuso9<$e^Ve34VC={)snRjtoZ5>Q6^HhAMug%c)TzBey7}USn(g@L^By;mD+8*O;3+Wf*?2uX=QaO%5gMh5v$r9SLw3 zv1WUWaR`w4al9#K*Y{VMm9ZlqvC5uV?Q*;w|5CL$^dg||Q@okhjVDu|Y!D=(VPOy> zQIBLX)k|!63dFk6``-0+`(Z)&_Nx95310IiTYNP9K)i*dlsKnrU{ImZ;CPD2SW=oS z_JXF}&#$7D=mNtq>hxkRRgE9X{GFxL(>r?DO)uuP`Sj8|7($lBRz)v2sj;@yq3YzN zVrflAQEDs2upohhIX}59k5TTsNOF;J|itsSh%UC_*apauLnEmH#Gb0 z&z_gNq;i41)cvp-QfZi_ggTfW?!iwqO`<~JrNylU5NlQJK1y<9>9zN7=~83;cl}YW z6cw}463#3$rJsBX(N(l|ixy-d=^eLx!+{8t@eBQQMZzeB^(!bX*`HQH)%UX>4`2P0 ziC-tKcKN`3IEC&m+sB@C;=vPEvy?VE`=m5jnB6=&5)r9zj=ImDPdr5U7kk7>PCZ0g ztElq)JbZ;~NO=w$rGG+oxCx_b>{I|UqMAEUUk#}Bi3kjFhkh#VH_BB7_j4Kx+*CWc z&Fp70Ax$mlKFsjuqBmirZa(uBl3Z;(9URQRpr4r1k3_&M6?-=%dXXbHk_1a)3LDu9 zV~_?gfi?jsnz?<)XJ<{g{GRbw!PsCpE{;3C*1z+G0)=baDO$Y!(f!!fe`EP$=(k}5 z%2yu$QvzoT(3X%#d)4=E-~>j%05hm$WSTUb(>I050_cCv>&yh(4V82@<`~fhdSt@= z=nteIUxXgK7INv)nSw{e0FivM0(JTZ3eGBx^o~V&RS&NSq7Z-l9XJt+m`g~@E7kw2 z7H?iBd3rH0f?AOm$j>FbZtxI6)EwwY$$Sfu%J3s0M98rB!AW>j94%OcbEtvzUxAo! z|2qa?$tRam00Fy8W3&9Hex-+qzrltDaBEVCQDGJZ{WTEGczNvN%1ycQxZ)4B^~y=I zlAmlHyY!7OOTH+jT<4fVR{U9yx~ZlD0Sz?+|5!O!8F-R1YJoSYTS+^`Kc-OP3P$0& z@oCk_@seg@if6IRC1j=i1U`MB;w4}eOaFfWmNE{oS}Yl{mpk_XSSEt9iS3cR58qX~ zG&L;mR|BNsQ^gFJY!uP<9fsTDT2E8cT|RXuqD|DjkW6T#UPFFRDwn1H5b^1a&a|E9 zo#2}I3gZFU#T=bK{SXv`nsw4DJTfn1ELyz_$1UKvmr#88S)ebzM(W;Hx`SK6I387& zkJ_?*K7`wvR;PGb1+2WY^jRT=yv9Ss#Sv(axhEX%b4J+BO2U&&`^^xN5_4roFbSi* zvl{UtSP2L ziRECd&ZotQ!d7(^J)u-w>qrm8_V52(zclV3s0mJwy&Q|JFE271B7bju>(P4bvR+=y zU&Dd~qHe^@aEh|eXC9lfp3~n;*&mE@AgZq@v^=!|S`fZWyLjE!>>w88pUHJhSNzP+ z#w}#Y+BxTc=!Y-9*w2ZBb7hdx@!0r#V6Wd`K-KgjCn$8H$fR}OcnNie6;N)ZVoTkY zQm%h`fj+@W_w-?qsyTO2PN0F zw(zVYF)ahWmqI-d!^K$wHwL6<#RS#H&r6-&Ugv!b7Z^Suij}$V)KkQ?EzUo!X7ZG) zdJ7g)_?&&<52gBiWLC$cXhp-wn8JrWY1?%nFW+iGd%FVL%}`Dch*j9Dz7Nl-SRh0C z0J39s0XT#)Obq#c6z{ga^RQhTbN+rEa0&Sp@NbG`9RHX2X)P{(THC0De;ItpWJV3& z9JK%8IUpTV0}BY_~^>X!R(D$ncGYTl~Q#z34eiqMAx1em)N z+zIv=iR1D+pWv~s6w|pDSH&^8s1WKosyK3&{Qx%O_3>6F?ym<=nK2_-Xe>o~5SUx7 zBHu866~6mV*NSxu5K@+6;G`T+I^^ZCneuBP3@D(DYesw^Ud6=XkOpX&0TX1qGPcG` zWE1h!&dX-T?&$<`=+S~m9aOxath-Y+zoeX66~qp;Mm`4qLly-=0CeoIrzNgj2&ZCo ziZqE|$Hf#DM|1M3CRLZ>vjLplM6lk z$CK06zo_!PO+q-zf}jE&qQcr(4n1?kZ{wgd4Db5dhC>JY5n4nq>q256NRXa8;cM9K zwN7x7{TjsB<6V&)e4BxFG#OpxnBF0wg+x>xs1gvEBctiRtP<|iqC}Xz452zv{>3V#oYh-Ef%rPn$3!CuN-7sc z8X@lZZ8=YE$#JeDs!3D=HLg@cXA+g{q_U&9Y=8gQGv^a3*&?#X=gTTw*W+c-*FgpC z%lI8sEjK2nFpi5{_C@2iSWKZl4XlSo%V|IALwhHTKAd?S?8!(`uQeh}A+IXEwJ?XndU z=yd#CXpKXKN zcdtG*Afkhi0|ddr8(s7;OhMm*&|8)C4#^Ou0-^!1nlD!m$5BT zlp+Hd<9V0)mXZ#QPPg9X6(eR90y1iyb>_}K4;}O7XZ2|-aYQ`;G|PHSNe|rgZ>doR zW>r1#PP7pB!RKlj#TaSH`>2MY87|AC6fPlNwebi7CWD)52D9D>*!5*}B`5E~BUs=9 zUs94>jxj9s;C=dSX|?gL!xwRPKk}`5%P@u-5#dpN97vDP_eeo){b+1HaIOwQnu^}w zG)|pzw0`n$&X?sHMM7)4S0jS@QbGEm#Lm!;6$~CHPgV%PG$Fv5&d{JGh0cX&B3K_W ziM$}fnpx>jSU>VJX9P(^5^@q9sV_=IPEMyxh0VX}0)f(Vsam&Pc8O#TxQ0@z3KX{b zPsjtwJnJ+HO}c{qQ~$RR!o3+j84cgRuaSq>zbZ9dg_xvaAF`%`1`QUS?vQs4lCk7G zT?MV@y-5K^tVBB_PdW-AJM(t;P41T&`4;5cU9Qm&c0l1ySz~r(e96g6bogi$rgz#l z2e?042O(>TXd}Rqp;HQ_!FbZD2A#;jw2&s~+ijd>Lo`|fDVp)TNz-wBNvz*)%9^x# zyD<0kOnw8U4$*)D){PvUI~ccnDP{sTb`tB?m>7YEgu^!>f3k0d+MoRmNQ%|G2~hzM z;m7Htv);PO6AV)WFv64*7`gzsw^}#}x^J#n>90Ex8+OR_Ik+gZQK6q0-~Y>}&G%*M zx!4UmZ_E*iPUc=0N^ecmsHYQbFh+b^l4u3;#AB_Y;hOo(o!AvdNhW4h2*Zr_ldsI0 zMuZXpq)E`Z_95~@e@WE}?ZnXyEXR_{WNh$fm~mgVq}${aZm>R5BiDJ^KH%*E@FmmC zEp{(cX3>QFszE>J421NQ-ZOhMVWrqZMD>o78AUpQ7jz2kc2Q?jXeKV88`=GL7Wyr) zTvRV>(K?)eM(XPleoc^HOx zVpI*MK&*3!^FK`EJ|V91*syXc#654+G9_eiB>Jt1F)?Zx=(oCKUF%dCU1eQrv6l9* zio`QP=uWPA1xFciMaz2v*97#e7L7qWS_cYrpo1lvuS1x=La}M#z{R)w;dHU5K!kDM z*j$o%EM}ku&0ozz3wfH-K5YRQ$;)q41qnIJ zDtJjA9Ju)HGO-)Vr(rQBA3iK>{H5l7Rha1yMaPk`-2yYH(WYVz$R-=c$Li>pS^Mo( z)lPosLvm2-3%}WV*WhZ_86e?k`A%JJJjZgO$|^C1BWLdyS$(AtsGzaBqWxSu@;kbF z5->gZfa$i4&sD%~^MBe>9^JHj`>An0EBf9^x$JGV2c7=~Bo?m)mse1OYM%nkt~TJK z*!{~7O+@+iyiVagv~=9gb%g*?U$Gq41B!1uby^tTg~BdPKf_3 z396Po%(tI>*y}>*qjs}1{iM$eTy4H&4@sfq^srdAKM*Ypzh;veZ=jG#s;!ml1+bDO z#g>?s&c3R_=;Sid%sZmj?+a%Ra(4E^fQJ8E4-t)@aAQUgtQNqJMFLEbuSl*98p#V= zx{`nZgi#y|FXB*qJtUsJqlL{E&@ddCs?KCWVl|8N^PzEBNzjq&W5mQpbt@JRI&QeS z^OEg|^dfiEAFvgayVo$Blbs#$FSbv-PMClC)dVsq__rhP=YQ_VTWQ8QdRBh|%sD{O zS9L!Q!Eo{C|Jx8&qGtWWJv2ovQ+j96=l3~hM^%`@9rh_~l^lLPbn$t?6J*PEQj(`D zevlj%C|0o{#;o3hato;x{4NpWN9*Fj@UaI%`SZc0{lRRW(;Q4UpbVUt1QeR7G~Kd# z(kI=|8-oD0VEy}&j0nTWk=YfgjNi*5(LlP62aP~Tz z;VM2MTX28X26vI>7SUm#+Bht^HecScT12B0N#Zb^=8f^G`d~ewR#CJ06!0RiDz_8! zm9HVS_QAGS)Kq>v` zonL6WqDF()uH@U4F8>uTd%hbcN)Wtnw>-u^a6eF&kb`ZaZNeSu4tuU=#{b|s?;ZbD z{1nMbvQxr_dfVi?Ioy;L3Y?nFk{2UU>Ht0O*yhxW5{-Mn4+sZ6k=pC-!Nsx~rTwWv zJ(;yjdb^!{P5Z(gZHe|DhFD65o50$H!0P(F*o5k=N8`+-&qBmJ$@r$9L^+oFkdDr) zv2=W}B?i37wacRqQ0$`aN&;L(&gRYBBRiiB9R{Oi89z42VIr4Ny=z25*q}z;ie^Wr z=N$tmREwpLm?K9i(;tkXh9~)zPy0els9V~%{N6*BGR#>#lbzkv|AJGzPeKq>_Wa-? zm}}4fFIRcdHu9dEjm%k0WanG>u(WF^ACX#>jKBk#$LqK>?z(#(H#=@!>BAJQA1N%w z=AkPWn#3o4TOI&AY{k?`Vn_5AL8d-_H&SyN+K1#wbE~RX!V>SlVve^|$P0o>YE>n zjRSCIA8wNI)cSv#@iqFPnZ&b$7ZdY~Q_-9xeu*$tmtG|jq0)1urUf{6fawZhiVUl8 zGH*u6Av2+?tYZ%c^>|d5`WhUbpicT7Fa(K=6RGoV8Ngig$owqA3j+ZzGXSV)5s=k2 z;AnOgaEv%*qvumI$j=w!2RUHxlFQOsUYgt#pxSuc`TCd~sFi^wRJ;Y&K4MBKx-;o6 z0?)AoT2Xlz*30QB&{R}tktsBo12hqGJZvG`okXT-456l1d`CPW^Dd8s8j=rH2CEOP zE%hl;oiAODApY!j^>Mv#!Qx5Dz0>Ul_7@?=lt8aqv3m}|Ws3~GEGU_nXN9NVLxiO$ zJZ=y#%bWC^)ED2xqwZ_EKbdp_I)GVCzyd_SB5e76)L!Gi_=i0#KfIo55#ICIkJF|dO|G{sFuKPST;?&pQexd1-qK{~E3H_pjRJL2_L_}&YMhWY> z;Yd!<|EK#SbPuv>nS3bjJ#@)Eg(M*}Q~J8~{GEfPJ!Qw}7acg@^320Kt*}T7DOF^Y za-~4evYSaC!Ecx>e~@na@+Eh^Q-PowO>l&6xpr)3a-j6(W-I_(CFMw3KyTtz56g2d zv=ILwsi|_QzGPCOL}FRxevdEriix@gK=9rUH1B_4q2BjY^Lr zL$*=%vYZF@zYwTySu@?gQrwmw2E@waf;HZBDggRznN=iI%KwcacM+%4^Y`!>{~5oN z*gSbc?_qwGc7Q8oevv#^{AJl9x_AHVyrMSjbunoz6=`YnBO zd!tU}g8#esE&#GV_g~VCuc(#V^8Vm zK>mqQwG9B8iF&Wy{B+gY<|+pbR(Z8cqMvC-($$0EMJNzYd;BEyVpB<18M%eG{R16K zkxq~E4QZO4ZNFkJnUCTrp!3GAl)ikKf3_B{u)}fK=-YDG!{Up7FF6t0mNcdR6NrO< zO%FdDicZcG6@1pTGdFK(2jtN_MCkGYJgzR=8&zHGdy*!*cBLQH;YTKaSA6)`0L*>2 zg0swW6`i~wkacEays zR*w-qm@hDrCWPP}9M4qY6N!se;t!0)J&bXD_t?=dJI9dbiPS6`@cv-=@czE|57A4( z&9SNFqzIP{oWf-TL%%a<6H;PXA0Lb{nC*o+*(Q?*e`HPtI?yh@J`iih1@pxjZ%p;a z;z)k%ywEk{OQVeuyxZQnuV0{xfB{*E5~bW&lapY=V;-BA*SosF{m3`zFpQt3LFzTL znbPp3r^YY~4pnPC70a~WO+G0n4!{BFq#CAk3(Hv%_t3?lf{|`~>yp5DL;w%Z>s;+) zg(`Vm5W~G;*)@9F1g5UO(M|P%!6^>i9*fs(dXi6p%)72}KIcR6`>zYI{PIhbkT+7C?viAv|3c*WGn%S9!E`5FWe*r zJPx5C94o$?B$T7kmy9-{O%jTK0WZ`+rP$;+P3YIdxdvsCe5}d#YPL|VLp_*0p_+je zL3LoGf5hopijc0F!E~a~NP7~6lmIA5|Ge($W(W?g0Kkt3JGn(BqkHtvtMeQJUf-yd za(U4l1c_bg!HPOlsD%e7VZYYc4R-XtFTSe`wG3c4uLrsyJHec)OL$1E8M}S`CGdak z_zV2nd+&Y~6D`bt4Z2kk)Tkl)iUZtTUb=toU7Z;s`Xl5)>1Fl92_yhTDZNtJ3?aPw zx^+*mN<8(;FSwTB`89YF8gHH)z9@vV2InKc(2&Q@_oR|UXUyVt_SN~@tgUtSox7Ge z*ewjL0d;FQuC7>^n^M|H)z>wEYVG_JG1OxLqt(+lvhs*DYTC?w2jZH)O!E$GOV9d7 z;W7(`NzizkMSWakw=_vmF>#W0w3tKys`bf*#JMm|=uf2ZEiRi!Sw&7Q-$ZAg3+3P! z72ke$WbgKdG#QrQ|(` z*z4)OSrKM-$LuJ`9b{<-Mw=QKM>Ft2D(3q=k{%C1#w0A`4fBo$^86_`zV>0c=qEVF zyJO_RWSTP_q8^NkDyhGnnRM5N8S~2Bz2$@Wd!hqYEe$!X^RfxV$mD?+nMl0e7df|2 z`e`SZ$W~c{kb%rL+}*^N+0d*}W>6t`;_CpYc;zbXXr&%z1UW6GPLwbRm^m;r5I-y# zcQjH{FWb}ok=kXw&mUA8VR6+-SL3F+pCWDDsUwDF9bwQ~&` z4#mLWZb0;=-H)f5a^so<3_AG z@ftGyXN~Xc5q)3~FbyRV!Yqw;_MpX0FQnZ~>b~^U(mBCDrh91SV;LB=O>M5TJzo4x zWD}NLad=>+UDqj$#3szLBkQ?bbef4HFom>W;fOv@AVi{;S?jZq{`}`2t_Hi(^EhUw zvLSBvy^VtJuYNlCO>nfnsrtKD(QUN?i_zZ0 z`cq-UlCY$5&UI7u1DZZ0i=)MQX1HivA-qvomA$jiBqCW$v1Ybk{5LYqQ7U5!<8;>S z>=3D_i#u$o^4d3C_49Cy*?OJwMprfyos9yyf@@=T#OdIYcY}FCM^t!u8s)ld12D1z zHs~0Ek#gfkRB{F?=%xc>isY3Ei%wx)$ICW1JI`l@=Z3RhvXvavzUatVY^gKrCVg}~ za4&TXJDceRj7SqOs)RrMdD$XOdZ)_DXija1z=qumNvVj&h~au9e;qq}U2fJ^IB7&j z2>X>}f>^KgrM0wAm=r=-FmfL z5SjKw*DS%V~jCFRB|)qLpUFVr}oKCbbMJrmCc|`u*fun{9R3mN{`OW3dd9*x#0*O2G|sw3*OCzSneXq^SVeKl8_zAowL0708rJQ^C03TIzM@y(=+&!;a81g0_BUk!G{iz#%~sL&=lY?IJUFX7}JP)+9{*6w20>o)r#v@$CG_bk*C z0ROoC#_Ytf_A_ejzCDy!p|`8t3#8}lUQ?D+!CAxX>*Ie#c8Q6x=PS%fOkxCGqmQ?2 z?eCey>@51tBGn-@T)=B0tN_%>mvC0gp(oxcH^yETlVm&`8t>z>T%pk(b03Nts|>|u%1y7@xplYWQgYsw|Id^h z>8$WUU7bf*M+Kk^DNuSm>OReEx&MzC27IJgYP{d2^PhkE6k6`?)>lT`{E!SOdGQ`D z)Ze%3rFUiUi}S~k9|t(z>f}n z_4S2{0?L1w!L?N5S!5u=6n-gV*lBj1O3f-u23(y1p;3PI8QP6($40c~8ro#}T&3wye`xW;H9_8_G2NHXhf@%#6<*d^B=nERg|tOc7`ySN3?)pd@7xC+cI`ljC4^AWSt?k zfd6kjf)aWRksYR7rxe+z+9H1iOqTrf5yBK8pb{TbXpqKsT2M$3*)hp38J?dOhEPivQgAW#Z|hTbs;hUOSN;FbaCd z2OTf13M*a%Uauf%J|b0)1Ag!2$OwoLn3}1u*OSfTc7#6JaJpp)8JzV_*B~#R`ko+s zs=aEs!OZMa)`G&kNQgeV+QS621Wf?R4oxH2Jz+6x5~fKsqsO#317{U#<)R}y+R%+e zfd77IU7FoPzwz)XOt+shP~Z9oz(% z$K&1uGJtdZM)J1d;PoV|wV5yt`?p+|)Tp@cFg~fdQd@_zgi+N{7V{eSg85OGuIc9w z=Z;PeC>vHYoZWalIB$=vSQPFogp+Ri&V;W!&brTgL{f)@L|Z|%RJ4N zXp_-VTEK7}8vYV%zhcs)Yd2vHLLdOktLl%J8f&{LyS?BYDmM*huRnU2# z29N93`wdj{ujSn2y#G2>)0h}AXW+iF;{ON~ICQxQVgLj;kR3(Q9hNzppkphhigs?A z@?Tb1Vh$lUF!qG-<*cyE)Kg7;_2t$nf@QA z3_Iq$IDR=Y_;L%WiO^Vr>o8%Y0>h(Gqo2{_#|l}pgc}xaM_+aIzMoEnFEM8N{ALE; z@+OMjNsRVh+$wq_XW)%c(Md5GF4DeF_RRh5?LRihJv{D8j&a0z@$HKz4_|_@Zd69U zn-+cHY~-N&R~hup@M}cC=&Y(TYh)y%AfGoXUZPOVsgk3x*42*dPZkFr11E(DM5^E_ zpI}`*ZD-G9th{Edv_Jifiqr~K1gXm(;i{Lv@tzvpwX*_%t-@*Pm6)*H64P(Rvt|vz z_iug-26r%Xjk(|Mte$u?%vz$;#&< zxKiNWr)Zg_(uwiTcm4%#hC4(kQ)Tu&_sk;ChgjT#1IQhAMV_R9PFmor9#Yx5pCXrO zWJ3KHqb!6jo$c7`cdo|oT}x`L{J_wbXK_qK$3EozR6rx{LxGVSRWsTmWa<~NraCth z%XH#CiDmblecc0wAF00CQZM}3#1{9#82V9gva^~RYteg)A|IG2e)xX~d#k88n{HdU zY1}PHa1HJR3r=vC#+~5q4vmvQkl+r%8+QvHAUJ^_!QCx1u7Br!zkT-DW9dnIFfw-B$)rAdPVfF!JBuHWUlf7ixVc0C6fy?x7AymDn=G9oW zp#W1NiQ4<1rSv`VdcSbX6+}`!_hNn|dvCd4heZKhnq^8&xaqI{8G##$E}fOpYrqE< zO5{H{o8}f>IFLWrCMT;%_8of0>D+DC`pu^C8oqtU@J|;NE#>|`9a%9KksWp&go$G( zP2%p-ZYPnMiDDUiY${L|Psf%>kD|9BYDb02+i(cOyh#&LpiB^04GazG*&Qq`R`U4Y z2aH2n8`gR%nhr)|r1x_Eqm?*s%s+Tkpq`rF(=FGtHrRa%*yZYyJwlxj8tT|cP^whvlt<3kK0s8rSGMbaey;n;?HLW zqRTw8=`~rAxOpFh83czW2bTDPg0*t%6FNo%JFgpJU^4zD8PQu;++FCkp-G=G=G!q& zFy;`nZH(|kxGy}cJP@`XO5{9KiCED_HhabizeBP2FrA~M9Xkv2_{Tl;eLCfxI}bXh z6$eRNK$+OXPL0fJ2}yR)f3;Fb|7$Cih5ncPk6Ege|7ZRTRtKbWAneyd(X>l5?hpme z71YK>#&JAO$!L3~z~^i=shA3$5b7bN-J-Ou5=XHN<6ES|5Mi7%SM;$P@nQ+;$=_>FxZBK0O#Gb=F2#9do%hdE{lzTNJ)98N-=LK#g*FIG z4>m~(e0I)gc@YRU;d9o#y}Q4A=;SzSx@pEf3J5Yj&NuccbGXNGxrTY^q8r3*0Brjc z-%HmMphDTQakf#QT9w?d&AQ+^sxxqRPl8NE%!0VGc6An#ID;RMZR{1p?=VtF&dv=? zn=X=mA^6k$Ro#hnqLt~B={*m&$8TGRQF)1(5pC&|Fyf#pwk@4<1jH3}|x4ATK$i}dmkLN4|E_#Uxe z$f4zS;XeZy>BNTAk86xR4pPu0Y+V;Ii$&oy>ypk9-O%Y!D;i)NYN4P;vw05wlgSw4 z7QqTSRNCsDhY1)y+x=lZ>;nZNJ=8j$-gPIY@TDWiMtoj@4J}cFS-}Fzwpf1=t$6JN zFA#;oq7Dqqf$3r;sGOm$6ZH==?t4&yU1*o*vq92}2*X8+td-w&=jjro%*__jQ8&5t zI|kb@n6kmGi20Srk{9Z*7Gu24kXgU0O#09!sg&53SXeEXb2tCJLcMUa4du1rhc+Z4 z7`tV(5X2(b_f1c>7zGNDu+{yW0N)-Lx2ybD-6f6?r~L*w7lIE9-oMC1D$ebEo`{~T zTzDc@TV?A4EqM`1mOjFWPY@xggo!j#`-hnb3fBGYRxRaZF$=)Yck=?Fr=%Z(J-W

U3Wp@(=H?VD41NZCQ>hjA>+p zfYslOvRi-l3bc9fsdetu!{5|#{wRaV+jyhsT4~p0z~an0KdI*<8>+_P=Oq46MUqIO zSKbO>CDkGp!wc-G66%hf5q7}~jp%=uG`A3tp5wM6+;IeaxnNj1?XH0ydyygBVnf;`$| z(~XqIIKaT~6TvT4*=QrM` zp9Nw=OoaS29{=V_Q3oE;{_)R~3K_lsZ!vkjiDX$^aYWeJr+ZKiPC7C|?jMec{-U?C z|F#%^Yqma=^VxReKX9+ZkUpZJ+Me#u^x3u`Sd)JkO$qN=z$H$jg*md-tu2mtr3-mg zx}RRUpKEztWG-mYOilhy_vW4G>khFD$b+GV?@_eX81qLYGP&nE7`ipaL5R_u9?nt;^LTI{`BS?C!|$l8dBQ6)`I=GoF!}=IEHx@S^l*0ZRnuQ+ z9V8OA|J$hBqUIx~*a(B1BPwhjDnFK}PH^;p7j}49|FvL*(EvZ0q-Ou)yAhGi4)er; z;a~?&cI^KrBLXgQ&gTqooTFj|cCx*`M^AJ}z0VA}@1^G0PjY$z>^WMdtjaI!84{qw z^i9N1OMWXBiCcelwb)^R97GKW36h`@4->*goJ@_^jueJUp?e)h)N*SrgW1T9bQGya zMI3!Xft zeVfy-&IzuyYDSLv6r-}IAE~L_{&|1GhZp6xTZBgIGt#83BfT^J9oHw1okZf3)!6E z?!m8mye%1M(}n3d+zpLC-A8I(6UW-EjXO`aj0(M^MWZ3{PTJ)C4C+^GVvkjd`hZw! z029s-;ARS7AA;m-v8ltS8r6w@HJ=Pn)sIrY!d0^2U>~}6vF=azY>Zgoc3$SZX0Nwj zNZy72+u3c2+NTyu^(=Wi)Kzwi6_QrzGo!<;KAb+}wZvFSJTjW|!fIDZ=mEAAEk}Qu+5iqRnk|_0*w2 zp*Pzbg8Dh#diI0w|5|FuVtt3TdtSHyn~xRauZn7Hp}oedzsd6OpE~b<=xP4m=+8J* z1!yniiiX6Sl9P3AtoP|9T*#}V);~%ll_bKmxN2i5EN=SniiW)t^oEdRbH3m(!iO9y zcWOL!)9KineT(hTIpOP^sao35IMK8Bf`0006d=kw^?J=k+fEI`CGk;l<&6&=%1D6z8va3H~aK@3>X58~Y?;kJg|1(}@YK z0~G;wIx#Fcrp*iqQ*mO|`|-!okuYAm@iPnM${>7LQrDAH|BvCu zQz#0#^gQ8T`>FsPtr{U}PL96=864jF(0}-VzdK}x@cjHj`V?OZ-Sdm;blm{f{n&bi zUpuLZNL6ZTA#hO^)bk#hGMYUwiOe&wKz-mf4sq0CsL7>hKd0vb8P$4@^FLTPOnz3g zxjsqUlqvOktv5nn05>!Rd+oSxQxuL=6x=Jlgz>T{P?}}7ke-jZXRt3htc$n#ngnN} z1?Ep+!z8!}){IkWm&Sojd6IR4pJ*HJUQy~)6GIE3m@&~jL(_}o)|+?Snf?j{f1PNA zxhZ4a^c5vh zH}2`z?w5U-eXlom4*n5_V5yGr4vWwD7ea`r2cIFz6|!u zJbjPVwq#d4!>AR`u(Coy_8(!%{r^pLFBRSMRWlOu_lW`+DTgL#r1chdARvq6vjP9q zj7-2sb^Vc15ngbyY4MN)F~Z3GpX4~i3hvu(FC<2nMjKXanulWF&~ME zUj>n-SQ&+T{++i#j3O#g z1RPEvul=C=w2`Ta0* z+|g=7kjKqq9{?T>9_L%pk77n4cIs-Iu+1hqjqG4klULLOn4SB2@g2CN_IHaU;QF(! z<15kQnXW1++)w@|x&!(b4ZJw>*G78FO-hol%rZ;lUeif`L_o#hTrm>6hKhiGYp%@9 zK*Nl}PIHy=OCj71^$-=1Rj$yU!CIjLge6ernjdc0Vo03%9obR&%pTsMTn)U^TiH&m z`{hkSvphj@ndlyH3KU8Zcp7xshJ9NwiTMsWkKX`qT~I%(iseG5!35Rtfg`wU4f0Uk zLgb%9;`j8hw&LyoP}NG*bGVY%Rl8PlZNxF!P~i`X>^!XNJS1xr9;U&r6)Kwwc*;lt zBkPXCd;o}c>DOTkjBs&|MhX8kHQJ}RZmgJJy>Qsh9{6^Tl>=7^w3x6Vo6qztZeYyX zRGo(*?sIiPI3W!BN#wzWl-r@q7fllswv0^Sw9U~@917V^O6!5Sj*9uu>Z;E3^Wo#+ zG{8zxi#P0JCgEv4;R>d;-GW1#oPgt#cO_q9pA-<@dw<3f8aqTXY}=Aooe{J#@Y5em zC3AHkR1vI#%eMP2p9{#uTMFs-UDrcJEuq3xhy5lv7dS zZBp>&`~rni2{xA1{YiN$7<{s;7-WfeKybK(Um@LX*r6GyfJas1o z950;Q*>`_^Z%DHG)rpJ_T(KIOsCB=*weQ}G^ck40)86O2GN29R}MD>GqM z!W~kT{9Q(^Qg3^w>z+N>MGvOcAz1f*_tmEyLwDe-p)7XcS28yyaR^S&yVrxpvpu+J z`8BwjoX)s&ZA%C^97q*dVi8FpPgK%T1_j@gzwMzbeio*7$X7(ez9necNaJA&(TLX< zgTz4BA%$NW#U>b0ebiJ#t6c&@K8C z9N=@xlP5y>C2KdP@`&~rnwI;TDHYo|(DuWf8M+RIzxliozw@^?``<%M(V;dLzn<6p z{qDw6A4x4#6rqly$~VKX{St?570U~m59)s}rD6oa-98#&(}Ab74Yp)1*>t7_+)E~M zlko!=#m7uxi3L50c+5tOM5|9K_Mza-tgUaUW=W8^HJ#}kOX z-!-!bycEPaj}T=@AgBZ(W%38IkJ0{K1?cSncKQFvu-CCaECEldSBdr<`w;vBoy5Ec z=KtWOW1{~0gWF8kpcCDbHGc%B`x(Z{-d$%kEjKcB{TctPj_K?)of+Vy-0wzt-C=6o+H2DN z*q~pda!r9Dc5O+nO<`aKC=aj0d)u}@Ql2m=YnJ-MEINdNhp>Vwevy-gHT+H9V=Z1DL!Q@@gm*RMkVr*Pqg zHVsGV6zz^uLpsL{fU4vjbr{!7UVXHf zb(P?}4w3TVcB)I)>ejE{1gZH&YjZw=h4J&(QXor!+C_x!jiqSH6R#OjlUcc#eD= zxCMPwjuUwe{(!@c&DX>HGoAm#Sg&ikx=hL@11+{pwpAheGbCiqiDk|F^-AAHEAcMs zs~w_EnIPu$QOfST!0iEAOit=o8P7FoJt6e#zbmj(+Wp?6hbOtp~?R1m}9f$u3cBS>ShDh zxv|&Cerk%y?QTcFUIV-z3il+Q7&bo|s`m0$wZN4w?d8~I@zJ&(=%y)P9Vq790gD+_ zJBT?Jilu~XP9}2YM0=LqN@&};B64UZAbA$*VruT zi%M395TLt9OV}#5{7Wt$t$A8rm1Z4M?OnT5+;{5vihuOLxj)X*i=Uv@yqW(}u(Mlk ze&!+JQgGG$LhD{%IuHFeLH!t1KN_*`9%B%W?tq7u>0Oj7+7c_;@=;W0Vcn1QLRRPx zc85)#P6dKb+?<^V*g3$Qo~nDgb;kG@B(!?8_Hx>aO?r%2>vvPi5&M*`mpeSq&)sW$ zweSNiiczWl2Ye+KHGlW$u>vfRtZr8CS)h35|5@oMsE>~vgD;!g>|?Qlo8~==X%}AX zd-bs*$>{dd3$N!mPt(h zYN$TfkT8X@d9i4OZ&K9>+7-BulJ@RwVEUqg;vWDI(FCvvxPWW+gJ}JGHnPK>nGA({ zq&Ov*58)X6lR=X_f|)0heD2D#S<-}2)e;A&r^~~2_MPjq+61q9coArYit9WD3<8>< zQsJ#T;6&?eaI<1$#z`m(Qkj;FWMvx=z5h7UVOoZek7uI@HIOuxGwvfJoX#I5dtR42HE=X&*h*Zzy=bza#-SfQX0ga=>FzP_*Nq?C&o9`d5u0HQi+^Vchk0YQ} z`yh+`jcRl`8gSHLa7Y?Pz-TkE8_nk%pdBZ6&n7}^Ap)p?td~6n)sTL@N)StdD!Qw0 z?hzL@L#{iyz^_^;{K?kHb73f?WEw-8`8wW})+L}vS4jH*N*fmo{WMVJ{#fNtSnj^^(89m zv5RxHMrs#M3-F0aUXQ|;@!nuKT;$~qEHPcp`$I}$V*5D~F4vz8X2%b>) zleI;l_GCdw<)`Of6<_p~%B7;SKZe%Rvo2GXX!QNiZb{#)I9Ids#BRxj9`~KsOv39r zULx;IIIHlemY?FAk!$a3B-bJMF|-ty_o}^=8|iZE0*5X`+jIA$!se)2z1l*TMS$u- z>PRb|R_k?$WT{j}(sda>lK&9X+)wtH(ZaDlghM0wCKwjvk7*)sQHOdaB@Zwl@ZAq8 zylg`VoS?6n6qF~6+PQW7uqO)ry+zep-Zgkr{ZyF>6$?JdNIVHTKUAb0#WrZYSitW% zZTH<8HyRaHk$Cl zL8Ea*&4u@|JldZ_>6cAzuigpMu;+U2BTa@!DE+?=?w--bY|I`zXpG^ zMR^fgyJZex>f(<5j!i`7L%beQ-5T?%LB+;#+U-X^5Ui!fh|J9BUE1O}K%s-RWGTox zGf}*_fX*MjA;iIj_ThaTa!plOa`p2{FlM9`xc;vo+3t zBS!40yC|G!OOf!$L=67J>nR0R5URT@MuI}65O3GG8i@PkyQ`#8NSRHH@IF3bI`e{; z4F@91*R~Ssq0kg~k^<45l5g@^MqaO?ArhT{(y%&ZSDO2grI6BgK&ievqA7&_9Ch9f zO1NIezNC&CeTdN0-dZLo&IrLzU1wUT_Na=?(rC*jS*(83&{8*b>bWH~-lyjW`z`C?r7d9z?huWC=hXtJ3O78bViUM>5XGY+% zbHVsUQ7IlRC;CMixn5aBJHPO|ni}2W(&u42QbSO^PPtfZ+LdCiGuP*Kp`JCiu{Sui zV}RJ}>>2N}c^rR`3?%@E5)DI@(g+70xRI@DHw;}voF)cD2k(D344Tg|Xfrg~vtz&E z_7;BV#Ovn%gO~99y}!*%i{Y>rBujs=6-b6%DRlo3KXFxmDdxTH^yTan(=+AK&5m9T z(U6>eK_Sq8*|zKT3O+ccp=r3;N|#wEp;~3+>uNz`LQsh}O{ds_)GK&x%zbzDx$!Sf zAE<%IS8Xjn_jVNntHVV389S~hmrw@1- z*LRx6U%TBMEQ{oC*}X;3ao*fgHhCUqqoi^9MsroA+=d|BV>jZQFBZrF2U{Wj^l$^C z#{SQfr?VSRuBCE7&u_vx!njZ3G%j)PG7PpG`_cCUKDr5IB`uL{b0(%!%f-^ULA)E0 z0W8_;qweZK&^H*A0u7T}+RMz{xnY^hq^>3Vf?3C!q;zydrBLgUbT_ivk&!WSwROeD(%AcBdv^f7 zWL@@bG#N7iE7t@BqR-F+g5;_SiD zxlfmaM+bZFR?eY|3iWv5DTvKR+J9n%=)}c7p8KF4(iO&2XaICX=;gso_n3O-He0#! zQRW@L0{p-%Tx#ltH|taCmDSgjN+n6CLB-#Bm^iX?Ev{7Sv_DG386NQi#WZXJ!&P(z zcC-fn(#pg8TmqL}X z$r4tB`s$}CXsnUeQKm(@dorbc&`f*kqSb`D&7SaPKWp<^0K-dV+{}!0M!&;^2eOsd zH+TLgk>6y=#bUrl@1*0KM&;HWMg4mZXIRu5*v8J+u@l$OW6^38Dt<4W~4@-I5(O zi9FBN#5akoEqKVSKWY~Kt~hM}q`nSmj~gDnl`(RRx8skM#I7cxr!w(k{o6kdT{J($ zT*C?h0hNE-M;!eP=K05E*LJ7iswC==yqAHzc`urXL}XctIr7&o+tdAXVU|2Rv=EH4 zjM(CcC*%?wjIvhjy8KK{4wRM)RbCQ3QU+s%-x+s|!Q}9h<zd3v zgMcC;0mVuXPz!hW?0(ZcU?rrSPKn4$=?LT8%?h>5%%U?Djb#@M;=Z)6bbfV<&GWeA z6q-a#%rdskqDfK92)_se6cRw07RUT*Ab7PDK{&A8M|tE*)Z3EplGh)H6)->o*xQn| z*PzvquusDS;@`rvBFNRD=t|l^VN^0nBe-tfHLVaWU@YTt25}k2`+>(KRN{eU+R|)z z)nQcXsb4gf-VIfziRWOT0;*CIN@2XXI@G)hC`f-kEZoBuQie8#PV^HCs)CyB#_?zl zSo9#Zt56ycb+Y#_zh5#SBN6&>*lggv&eqa7#&zn|d?b1={s^?Zmp?Yc{ylJAQRCe` zuOXrtdD_$@1>WGPNANJ)JaFeq zckTo0(#Reucnz`7ZgoU8!gY0{;9dNL7uw*@iI4WZ#Ye~8hkCNePD5JGtLQEuBdnY9 zF5KtsAsRUB%0nUxSXtkSZ8vuM%TvLie;R7puL32ig6eJAY>Y@ zsoKpEU5!2=su}Z#0T7}^SXd|@pEU7CWHe-lExch-YHeoB=8kObjnpB-m}1^Hj$8jG z$DyC7=&#^6tY&b}?q>_y!KRQF%I?&;&}t}x-M ziY5zdfjU1GN`?}l;&n-t+$ zcD38Z|BMpMedOdHCvNe4H?c?@T2CJ<;@$uFiW<1Kb4!#;a+fpSlkFMIWKHfHml;rI zeFxY(uAtEQ+k|cT?F|L4)O7xSTiVmQBm%k49A6jOvI+7kzMHS8)ok0cJDzKY*RE4W zisPkE$=IRT5{`V3FinX#ObVe43mOq6wgY92hZcO2X^e-fkfaiZV8hT8tz>*=W$`$Q zGGd35%jKwW z7s`YxTV6F|BQH}$D**72^Yz2c6!zGQw6sG@7QMd$Trh@c;yF6W6BJC@OIR$f)%PW| z!wl}F^nnQ)m5wigCU@CGUK0s#=K<1g7}}bXF13c^D{ap~L^^aSQJ|%5Bu1wK7XNMG z^$L8%Tf2`XWYI_Wq&XZPa?a6jom;7U_uABv`WAh8^BS@cPQ_+!WwXsLOE9{5uv7R< zYZvid2JmU*3MgKQ#IhHeVuPidbGKJr4?H?QbjGt?Q3tsxg`J~~;aaO$ozj&`YHy6C z@=RoJ3$(h{!zF)+4?P%Hfy*xdMB{=bL}rAALMq$QVF)-dIXaVYS%iF>G~C13B9PND zZ`<_aT5bU9uhMsz#vSPu!mV~{%t8KScmk(A%tK1dI0WLUIvF|_50f%A#A;zbjol-v z8d!TYr$f?j-%c*5y35p6xAj*ok@fLGW2vC8;Aq*GEOMp7;02u9N!+7H-p&(i&hpVR z4YBD~rTZWIWcqusg4LBLkX1C!FA&qaLdJV`>2?L`KD}w2eZ?6rG(FJHXa&@Gov+Z0 z>_G+BD0$J5U7A}Q#a_jg5dcG78yXrOTpa@-2V!PGE}no41f57l+UBHgbO1_oa${$XjT4 z&XbMob0e1in%&We;q%DMmzZ$XnE=?JKv9^@tR;%CR$q5Zx)wqM}H@FE8` zN62i9Dn*cWcWD~Ckrc1(Ph`)t7(*Mv^WVFjZG2C<8|oA875Fkaucg(LIA z%q_puL4#}zvj-$`EiCaS)}B|+dxN|K#T?vX@<3oKuR@R3Ef!kn zRh<4-t>cIFyy7Szx``Sb_lG#=zq|l2(vzvE=uT@rG#t-{ga9$A)3QJf%o2h)9-R+B z?go#FKC4-(?E(!J0fObzP@|bY$3?9a;Q<;E`85TXdI!yN{_b_AN4;(@pF2<|bEX=<7|nEgO&&gFccp)b`m!*7SmB+*w^{ zo`x<3)BSnw!&ifmuG*4r2JVJi6a53L6OR5oWYxq!WKIU|Hf%XfxA0q`?n^xhP2VXN zN+JC1=sn5!bbXIM8ZE7S`?FhZNm&Y>JmZr&dFd=on{-IwOI5$Z(r@Nndmfj#`E%8Y zU}Xe+;3}AC$9TN@H~Ed#BV00*rJ4bUf^vY9l_D?J7S8kh0;G6lEW+!3hPM#94}27_ z9eqc})nW=XWvD7^b}RcKVTEKv0x~}ZxcO--skS#hM=`a_v@;m1) z3#%zUpVZ-N%@k8Ipx63`QSrR}QF~>YjBG<@c=!*{D2uXBnP3P)8SpsK!|XiR@*96xT* zA8xyND5FF=ZqRr4b-~XVrVa!>`iN(1|KGfYop)E_pkLOZ8&5eI$VSR$FW+hCT~c<5 z4;(TT&Fetu#L}W0)}O1iET_pW(ZT&BIcS6LcT!B5^^3_Rfo@%y&KPnLNa^`4sASWh zgnUV#=**4wI95s`uuANczS5d@?sln}GBw3WHT0)M7btX%3m(>UYOBGWHdX&Zhc|M# z@E*Nb&K9a^_L6_OPCQ49yYACojGC-ptm(uJ5S8bKHz@e|EXs4X=WRP^9iPxcwV(jH zF&C;l<@tI5Pr#Jy=ZSp9kT!_gtkL3ea!6hkAjVSvi|1g!Il-y4Yglzp@@xylFs{kC zbop3u#^mr$glp2**|uO(+Q{_s68Y*xBJgx@=^Lp};KavOj!uNQ2!vOd#c_@Q@QWsp zty)@`^Iq4J728n2Ch6KT@$C8T+_Ih|LL9`cte)d#J5@;u#&!6hYSFEW@DP~Yg8s!o zUZ?eds?XY7jSI2y1ypSBo3Us@fNBv-*oyHOg2(oe6pkY{_KlC+=k;tQU&Lk9F)GkD z!kJJWu&%gztkBJZ-0-)su{v7H0?@2;CM+CN0y&WG2pn&>>0MC%Rt^T%oB_=0cxQT;$y_2uTH3t`Zn5;?yMU%7N+j2 zo!WP_01#@f`Am;y6(rSQ;PHBy8XUC9H8SYzYERGZcQmT4Dy zXYC2+G9vyZ44zEO#XHYL2y%LYF5&Vb{%$egj9p>_DXxR+)ob|z@d@a|i8frT+uj)# zDg3gVuOCdw_HLObfFl)QtbNpRCQ@9F(zTGHjpNp}_R9;gur_uqfp=QJ&MLqs1!)q{ zeow=H8MC;zPz3Br4*%Zx{=qDUflmg?6)mXtUI=@W?LtqM-S6uIadmQ3f5u9gd>vRD zLJO}Ezw*h`R-d`h*N$~uVNn^ld*Ep)%J7a`E`l>XNw;MCy*uBky!Fd41mR}jX|$;BwFi5a&}~0DV~WN zXhUqVw}cN>l8?3KRapmWs&m;Th3&J?4xIg{Te#Nr58x^we%Xy2%mjWp<7A6g*9q){ zszT(X=B?bE$-7t~JD6Gn*>H$VV5X2VHK1+9um-ri5#x;2CF;OipZg~IqZTx7q5K1B z!bO=7D)jS?=1`B;7+b+E`MBABfRTr2&k7_@mh+6xQ8^vFnvtY`O!>982d9m88TxOx zpUY|UB~<*v{J714xn~-i+()4uC;!6uhxIHr3bNyvPIK=wXrOJ3)X>Ju>wul;+mFFB zVMy&LZuj%$;n50*$I=E-a+Z}~L7K=p#6+^gS#Ui$^%TCevq!t5#j6%iMCr6s-d|zO zjZKRZ*Y>wC_Vuxz$kh+{5Keci#9h0NX^iOKvh3m)j*CE zR9+@dk6{cm{)$aub+c?!4EuI3s1RX$G_DGUY#uJG`UP+Ni@t9qg01UF;>oP*QiS^8 zVgESZZ5!Clz!57nm`h)4l*X-yd|`>aFMiph!DTS zsXmHpqUJMIy{rAjyuphQ?N)H5i$0+#JIWr{ty&wPNSfkd_VK*M((dR3(w8m@F_*98 zaQR3x^1i+!m)BPH{k^ouv+_D2J00{3BbF=JE{eloeOAN?2(W>y2WF0o=t|41v%3L*hZz ziCTi5YXjD9?T-Xh;}r3hoFs%+_D`9Sf1G*S%#xr-$ea){oKrfV+FEf%Ww5E8us|mz zyEpvGEZgHPytb_&mA>^GD;#*?%zi_W_+Tw1)1)L#tOSZQh${nP71%h^v=N&YRE+KrfwQ`z+P$=E5r zzz&8v5KRnrPX(!X&$M5*=RUsyM@$iZ5FCUs*V=J3GkZu_k$Om#;O=6e!!x?6aVZs8 z{LRzm#u8B$Z_G$&h4P!zo6_-ly=_Ktak@2uknd|Y`78j?XKk(;8wmGPJ+NdtXi#n- zn}>L}ZQS!?2!!bHHDHzozTw`2dPVwlTUai;H=Tx6dz`%6Sw$9*Ym)NaKt zV=-B!)Vt-xvnS?K`ouyg_ntj4<GE0%kuRL7k|$`%2^(n+=%?+%#C zaoTh7&%NOl;n=0n$5xrVeo+-yQbI#ab(!YAoVFN z--O~nYvDgDXyf}un!?mmt>Er{ND>J9i`A*2TkbDwvBe74sl2r0NnzZZdfD$f`jy>p z-Zih(`OJo1O~Xo}Y- zDOvP2fVw!gMnRLz>o3vExA}<4T4-NdvqMTM;AOd79_>&Bf@n zNf_05WeW4n?bDAEZxPnluk0MOb9rt@1!^4k@>wvhc1ak?+>=MuI7-QW2@RCRP17W# z!0|3gFH zE0rpahQZG!t4>RecL%1V3p-dk#ws|%`rFIxO_+Ut>8!{AcX2l4Y1lD`d5-zYgpXU&au+FldmEu%gK>iMbn{bT%ez5=)Qo$wCv_h}nC7e4wg1Mgd^;W0Xp%a|)m{np!f=P^F1l+*l~ zCE}%+n=>>{<1G<)Fp_s?SWF-UFo)(qtsGY#em5w|JA=1GV~h74)r#>XGRTq&)2^t; zyXmCw#dbIs2~rQXm|p*tdfDl=N|o&%Ovv>*UAl^I z(=y;A#mZb$Re0fo5h+s*2F!_FI&1paNzn=omC^G2KLv+_nh$EtkQo}ZVuKW;cZSaF z9|d#?aXRI)l=_{!g7#u^oNlFeiuA6<+rKZpyxYG<4~-KJS%GPSvL7xi=L{ai?ex7VS%Ziq6BRN2WS@0^f+{2&OekIx(Z$Y>yegHVq4B*OuTOsaoKeHS#_W<0){X4NUx++^!8-QHP z9JQKqPHMMiu4e;c@z~q^y7!B;81lW^^n5H95WII-N9DwBBt};7acE^D5zj}AyclhR z8qz-X1B6iN1Fa-?2ju)u_XQRu%>55nA>{(2Y-XVIl8+x1))ORfKm_6-MAaOi0R2=l z8MXo{Rk;cxooDqAETOFVPj7WmDUe+*%ri`k%Br&+Wm6_xDTK7Umudy>VL9Lh&@%RJ z;JR)7>+Ec5HrwnkU|ExXXd{`fLJuqQh+p~$NIVtZqOJTJ^7nSNaGqBz+63XAe+t%_ zl?_2Qed`YAr9#)E^V8NzD%Y%2RA8N%VK)Lk7s{>X-C=yWPpbgVksc8J`p9}1xLTnH z6u+TBjp$ocEQlHr9rc^IYQH8`ho)SzuP^*kJ=+6ueZ+}s>0%M4W_8ZJ=UKY64O7;m zWY_1tX1|t(&3cwTTC7ya+N4{xl`!EOX&D z+1Q)$P{?;u^8+zY7aY2U>%?!(Tk8NDfR%2nAZs{KRcb!-%G(25{ZrIRb1+e?&e?iN zH`glZl%#l>IkTxPm4PFAxr7m$Bc(FM81BB?ue8-U@f!uD8~lf|z83-|kYAb(@$Ync z8_aQh-!BZWeP|Bc2jyoGv(fXVcD&eaT(wIqK7MXQB`vplUoGEYPMK~8SMol@mtbTv z!k5sUMtnWL|FKYzQRc-CNCn3LE5%?$Qca}4mnl<-ECLE)t!0d_;sKp!(<}FKMRQJq zDZ2vjc&h_^?j;{qt(%pcQ3S}T=~F27yUZ^!2py%8uk`Qt3%P?4&!5SkP2u(GVo^F(@?L3{ z_f?>O6Hukp44xihJdic(3MKTZ|Fw6C8|`2E{}J|`K;P=b^wskBHqLk!(8fOLo;EhSxwl!72bgd2wExYyO0}*6+L4eSg-yaL+vV z#m3LTYquXe${XP{>30Bww=0(Dj-oH8A;VB}Eh4{1) zKhjv%UB6Z99k_>^YH2H6#T_E^ya6U&)xbmN-hd&C;YLD;)k*55 zJrQk#5QsaKlo8l|(>$-gMm4C)OEn|FB~U7W)J`P;>!kpmF6dp?Vwx7`*9dc8vPGsB zaLknyHI%8$MKgNRO>nzI(wP#!UhqQM#73j)*tm3(O$*C92nRs1rFXez`^wDwJM>5ijsZFlio^@}eHXS47 z+WYt@Jr=1NK9IEZ_A4c?YrnewG7YdQ=KauzDctvoZ0Ig#o(U)AC2bRszr^Fl8-J2N za`sU`PcEG%{B1<3B!Hks*|HACOl3rI#SkuC+8y;?Vn9xOuyJ=zEOvADRxbLAB_4B} z(;nAg(cN*=;kCI^FZtouNFnUyPv4t+_S{~XbH%A@Q}v97L87uacRUwe$ zxan!UN~5WOs1IV{x?H4wyd)W)YDg;8E+wBP8SQ!DQ&`DSx$BVHT|1>BV;@hGRL1d| zLo_?!sWHmW3wYBq$LT-FJuy`e^eqzDe*Q=yKSO=bS%F2t5&*Zdap(`*?N!_vqt{}@ z(qEYm4C2gx7ro&W4JXBq^Ja>6X?s>MbMns`$m*$%$Ls==;9*nCsx6!MimuKx!`?=V@Vuo)n3w z^jL9&Jgp)>F#P;K)}QV4g=d(rok?G=xi37KQt{i5m6Pm9A02^I{eN#g_}|hDFzWZy z|B>QMGmtB&qpF4hv8aN6SBAp8;WoZFLO%j_VFzakFoh<(Ns`&!GDx*jX|jNdm#tYb zgPX~IqcVosnc~+PpXOI8b+^EE8P3^trX30TkUa_^uIB1#2u||YsW>n#<&9mS z@|O_D>2k!Kdz_~Yc-1t%Gj~h-0BFCz5aZMV7y12=4VegWd6G4U-oL3fRt6LUp3Iv` zyx*ndKVhX@?itFjLQSRgQY~ zED**4D~dFgTlTZ>{j|C4k<(~oJ1}fRn1IrHvS2QUPxoD;2&b9#au=mV-q%&)_a-us z&-T|s3ILTi@0w#DmWkHJM}6>ESE{Hpl;6%5NI;R?_4yP9ESJGP1-@MrX6BkVyp!QO zXI5I*g;yBcAlN?H`0Y?cywEqFr+kSVRjrl3^(jfV`h14oF3WE8UDevo4?1_^gsr?( zd?GZFsq=(}Dx3nz%ODhYmkvA|b6B&9g?;*KB_cQJx?aXVzJdrr`DAK482rK#kfKZcCnc?-Y$7wbcxcnh|PWv%Zv$;uC zzw$Hjm%{VHRFs`I2V21%9W&4;8dmlPHC)}=#`EYqz0#kg-(`qB66>cAhlCs2$jbti zWW2)AoTyvkp%&fUHzL*j!nN(i^`@McTtoa@@}eShx63tn+Tl1g=Uhea0cG`-+|=nG zJxPrARo?d^HAPaBQvW^IzBB1e`TX%O;tpaw8LHgVa|BV4UHFAE#hBXuo0MP4mv32; zgEOx1RSN!l4(k(QBfNUj@vXH+5mDc2yaEAoty!L(I8U|tdNBgZjyE) z!tEq>^uBr_PIyBxi6f+GysX-m*mg`LjEQ#*@r_%(B4)%s*%s6IzQb}E)qj*LMLv4~ zpg+xz+P_a5CejuGCQ!Od;OpAV)taq<|}LeaMVSf)kfWpseEwp zybqVJyL_2Gu<`w=oL80~f1E|cH|2SiPXlF))B%?&@3sj<|0+p`C+w_I&Y!uE(NL9E zTxrBcysf)4EAU`U>Zzxl)?;*ZRfrAV2Lqt_c`s4cpYI`#F+bc7&Mz!{ll5@+l5&@irHP%|pVZi0sv%4X8qr+pHND^w z3{Ri1l`kpfliw`Kb&doBpHsnBHcn#4wr-iU1tEsBZ~JOj66zJ1w6kS54!=Y5_$g&i z+J!GZz7l@CTs^U-ATri0Z%1={2$2j>f7Y3&1;*VN)r`FPpx+eXm6nuMYk3|oe-QFK zIAtk$?&2lycbSwgKC{FC-bVy_ve(Z}CdA98i{j|)&h%oK-m#wGk5R$3?bKJ_0i^$-;4fgO_DLEv?e@lhbo1|Ha(b#Zs_MJj@reBN>sVizu`pWiBid;usk zQ?8nG!-#Z!6#hG!7M^g<$5^(Vlt2WPL8(||>YI@~e2fwCkGo=-V50|r*Cvj`vjOKG zPlCoC%zOdseI8=LZwEec>x%ZSZS(XGH`}i3?iiTn9vl(*@I#@xXAz?8+ox*E+E82O z^VcK_-lq!h1HgT_+Qech>&@`3hh6T+Z=bv_MVacOxXaqjXx-mZAt3RPcf)7muh&-Xe{r9`T=@s zMj~&`cf7jSpXIkWe_nDe`&yoPS&Gb>fOx?s<1JKnj=@mmkg}s@M{uBeb||p;CD8B37AiDR^xX+jw)D>sqKx)>%SXV zxV{pLd5%Nf?OAE0RzbhmY306dwAK65oPo0;aLTPDRf6SB`bw-OJ*a7FY03bT8B*x) zC(=aHWK@M%_i_;faD)Tq9h{)c-qc>qa+TK8w`eWrPxt=&M-zz%g@scoGR$Ij4w z79t%XACLk{6m|OoznU;-t`DW`1;U1H@eB&uSDhN~Kd9_4yH(_&SrfO&eWmj=+gp|R z`fBTEJki&rPVI8|;{+^D(QEd+K0!_`v@2ZHp+je)|94wC@6$V+PG6*Udj2v6&;HNJ zzje%izqKGORgwiVh)MHCgrM0Vz9)C96NO6&Y z<1`nIA9bl|fYR}*7058!+0jv71_n{n6Y_^7PV5-y%x9IXpw*@xjL(PR;xY+1c}OKB z+K^cST+X)TWi%Z2a+woBFGQHLWDG<*>)xZXt3-j&uixiV^U1{ZaOKvoMadn0gwi+F zGa#1e%5!tPSxz;6cZ*Ckkr~Xqp)tXqrk}?p*SpK^*P1N%eBRD;@p&(=nptbysgt83 zG9hmK!qkaMm$!^P>LF@a)#R{6bDIn4N+oxmxt|I?fDtssyPH*z)5rE{KN}Ks8W)6J z1b#fH=Bizu@Q^it>O!cMl;D{GvQUcNXT;pmZzs>#PjdH<<;+82zPQJeR-L_}FMNhf! zGQv>wF096}Ls2s#{o??rWOFx0`rAc#%1nRI*b_+xSj8) z@{jjMtW}Tc4d@jt-%m~;ODZev2TbwA3V}oTuOP939+FOgx1LQlcW0ze3RdqeJi-57 zAz`dF*A8Z}$7k24&9>uxI(Zr^;uBJ6zvZV=rFfhH)`OU8L#n?_6DN(UB0_mdZ!Ic8 zLO>yl=V}4IZw0U9boRux8K9lRJ);GX(jy)pZ(wt2K(K|%yq$;-ofiHl>GD&|0hR8$q4asHuhj`?}tT>KVc zN%c3XA*uK73uRvO!fXAWc0raOU;Qm6Qznh8|4uj2HhAgtUDP{ZRQ}n-?&#y4q=Mo2 z9_ue~Ua!@Klj(1@P5Z3v*)#@2Z$k6)B=u5TVhTu`0%AVnqG~r9eCJ&{WI{UtCM?~T z!wzLX&3WqAm|4DP_;Kyzx&1Zk4FW*H6&g==#aMsFZ9Rcx$%uaryhqo?LqbC;yv>9T z!3ub@VI3#&CO^lOqT)`;pAe?D5v|dkf{IcuFX3DiU=R3jm9B=B*c&dOciohT+UTn9Gs-)g-DK^ZhSptmOt*9e?}*#Gbc3ru5(eEDoX$|UwdUAPF~Kfm+!iWuuD0O z_*r`-7E2M6XP{g~>#N`A1`47j}G<<2`Z!(2CQ%pEPZ`qBsDMqR(Vv$d& zsMlxYCp(tmgdluU9)%wC4l$ShRHI*pmntE(R#NDuS4MSh`m_?SOq^{^%+f=a{b_Vy z2#Z2)E&0W~D^tFfZELNeHFpnQjl)t-NzUl|&*Eih{F66}G+9#V%q3FkVD*sNR1wD_ zv#825uqtnWfOblc0KuqW5#8}GA)s@=RH?xy1{MRXP%T897so&`p-h@d6SRfilb^Lc zt!1lY_ak92ib5H|QFz~m00Dl0oeKbjUS>)_GSlv^I}B?wpow3%D=ac8p5E2drfM>E zlks{9P^vffhTTmhM zc5^XC$Nv#@=aZjHTsbP;{%ys@9IZbKC1&F!9-`R(rx5)6Va3yMH<3dWZ~kweU&2R8 z_WS6C5OgDJ*RQes80pVz>X%_Zo(Z;&!aE*%Q%uPJ?jEiE_OKCIgVHXFPX^BbN-e`5 zF3BJ2nXx=@+@5TA6RxHr>?u>@PC^+p-jz2kF7%=AAAp0v>3Z<@#s)#Gg^oHP zpS2g<>sF;U6D@stNn}4n$fMChkyIA_zf%M!RH0!fZU8}GIny_CB2sz^gg`AF5wa(H z$>&2)kex2XZ9{j|41kXPLS=4)r=bQ)Ls9$nEHbQ-eM{{zp?6>)pfYeT_v;$(b1b4l zl9D5vkTO#*a~_Lb^CM-UYx-13o~Zscd*+MM7#+2N$U!4|x%fqZ`V2*Y0M;ZZ@2cc4 z1gjbj;EeU_Cr&GkdL(fADWtTrIi$5&rd@Cd`ApR5?*ox^vpy{R^|BRLBcy%5tTJF@bb8@FNl|75~KA}LPl_Q zIwC;Ox1RJMA(vbhLBuEE_U(+vAo)Q*`BghKK^u-?n}i8V(K+2kzDGhyZnT`r+e^A@ zW9@K9dBn`t4H$_ew%m-aO0UYUHo;;8{&ac-&=%G3@$)L}?$6wV_hGBEzwaLB^TTFa z9_$x1^4BQNhk8 zNg}X`@FFeyOpO&tS*dW>w#`*b8zuF?;xWFsS zCq^^7bB5qlRdNELy!CV&#JFgxN?ywvfZ$F6n#2K6i$aVQ=(I>dhar{n)%$dBz$RLQ z=8hbO6{dr_5QozJy->ghx+XH|>Or+FRg;**_a?QQ$aCBE1>q|BqF;m#5@ewu>U_Elc1_h}!LKBWT`=dJ}K?t#jX=DvUfIl-t8(@z z;2t3BG6aZ_Kc0DGiQMI?{Xu-{myg zBiaTbSX~*!YUfq)ocas>DW7(z0M@lkT=h)|Y+4Yd{XSk?Vd!m{547j$RO0><^Y;!x zPo)dsZ`256cGH)O`W_2m`GAimz_BJO5Y7j~q1nA20v%<_X3SZ$qSHDy{luFOx#NcO+t23Y5iqS@#=5W> zfvd>4V1;uE>!Jb*kYw1miuT^0v&~!85&wXW0-6fRuPOgVfNNvOx>Y^T*Y3q@{dk*7 z6p5+k1!~Wc%7VAOW8fL{p0wtMh+v*SOS40h)BNo3DtZ%IFkFLdoTU$F#)K5G6(Tgi zPa}DptI9+oOyC6_gU;UB2m|#+^#TLDtK|`p3HT}jT$&K0Z3FlL_?~w9)Q!Mt+rx>$ zM>kon5Mbi}EYKhm$<_x)JqMz4RGRv(Gm$SJ4rXsNWTPfxM#5n@p)l5y)J@~tm%lMtoQj{DSyCKn8tacII@&m>(~H7Lb+9uGNI_fB1+cp9cfIA4 z9+3u5!)8QzqG8?Z!ei(5z%Rv)$mzmz^>R5?um3I`C!Jn7+?W(FkWS+tbxO?wXV6*Hh ze~Lar37mC^-DTk0)yH!)4vE*&uG%8 zcyp@bffZvBT$6OCpIMTYRSgFwHUK?emh`ViA_UmqO>nh@!D+(>yP&%dP_UWsuMw=3 zm3y_QSWD;|yFm6YaVFbLs)Wd00SPXJPq_sfKX)}}_x)}R%QwP_@)TXuy&xSzKjUu+ z@yxxg>0*2zGd+BvuIBiN>R~=_3clIQy<%@Dr2}~y{dooW8%Z23 zHd7M}NN>W^<>i9T6P;*a)7V)68k7eX)IP?^SQ2y>SWN#xp(6~$I~j4vxT}trYWd#y z^V(&3K`Z}`{67(o_~(A7A#AB+vA<~Te@VN}|42I(kQLxxr$1-}f)9|)4gRiKsfVXn zmrYOF=C7f)N}ZRGZUY?2vP?l|eP+pzIbu?U)ZK=OMK5SMycXU~F-Z`_rT3zpxZiCS zz*>;Br{jnAJQJ(&`FH1qlFK@ek=f&wQslP3cH|6cD&k-DNQ&uxdHQ#r-&eE;(w9R) z;rnoYgA%WLYB$+QFB$J@%CxUGWzM$RX3X&`#gvE_>43ZYFDx!;<9Mq<{A(;g!ptp( zwKjx=zwBqENNUQBW@?K7MgRbxA+phEd76+DxAZ2@6Xf!BU-MF8MV0wif5P-~Vha7B z{1U1N^_a6GtKC2}Jhf5ctM#WBxE{p6l+G>lmFo>vZ`NZ6uHhc>r!Je96s=u0b4?|S zLz?l8MM`Gt9EV+kybu6`f1m@F#ZOP+@g~>741s%P6|ketFqYMCLNksg71+ix%zHupvtUdy(QRj#Ih8!MK-nIbP@5;UkPzy~T z1o!vG(cz8L`2)<0A88o8q{9l@)kL@8n)fO(+Ycv>NvYcKp1XQ|A8XCE4rtc2N3~64 zF8332qP13fm>D;agAmF;7C95s<;GEHxC}C!*Hn>o8}OKBP3+z6#N%ek^99L$H0PsB zR-K2(R5ouCF6@WdX8+Y~7v4VJJ6O$13>r0cd=RR&X0`7Sns441Mt53qCNaZ~UkRcr zhB=`5kMS0Bwl%lkvsX3(~jLBI_n&lS{&qC!5LpI~sI#Xv;}jKm@SqyH+kFfKsE z^hv)FZw20@;@M#>?RtYth~5Uw2JjM^(&wr<@mjF9*FISEpag zm?lh|%8#r$98);S>%?xg=T8=W-`GE|zFgD;wC%ecn@?}RfQTc4N9C@^t)%4OcZ4qq zR{uu+y|~$#r0e=vZ$^4ro+|rd@3X#47mtFh8;m;90$44ez`fl0Rr%h0qCvzJ%jc-7 zv(0CR1~IlDZ4Iwy5>u!hLuuH$Ia4_{3mnkw+7qjMqqWy?j?<3d6Tbf7y|yBEiAnLKPfp6PZA}9R6xQ zMPcw@fqia`}5O@A%k%=wV#i%>~6nHm@Ppx>2wz0uL8&Igw8t=k)2sZB>W zF1_w-^;8Vx*%8B{S5Bk)e$Iy85XOxc4+fOdJn+3eB95c#s2Jb5U2lxMu{-C@$WbvG zh?u4=d{Hyte(hDI7=<+9WPbLLqrjzr#KZI9Z&O@M4;c*Q#Eq{gcepFYey8oxa75o1lu%A(WuYRZT9QfUlg>bFaUr590Xkc=bFpcEmZQ_w7U7aU(CD}J>R^Kbrv zFGi2;D2(J(H`wX8-r*SnTo7uQ#NIwqfIfAJ5{|ZuBxuNQGg<}=(&E5NKiqL#bZjSm z61_HS8;5idB`iOUnTK0{BW&`}>;`{jbeh2p(vjYdnMg{$1=&gg7gj*;A1Z46R@nxV z9$7@=Xn9ConnIQUaNBZ7h48HkzIrMXDg=o_rN#>dh`oMY-s+GNmusC->d)xYBHN5E zdR=WYyX;DDzdGtGnBE{HU5Kid3<_cC$R{HhMJ>f}RVoLQe~xeXvGHt4)z4jrpN8r| z(2nhK`%8;?u4n3GMFOmZOqNHwAs~kWsrT$rvX9Hvx~E2|5Q0FqxCd7)2*InSoT|Ho zDU=|~R)6865~h>M)%VNHLy2GSC~a-(?f!()vnk0{Ip10a6r!+mQZXEc8wM)HLzodE*7eIz zlO|jcO(*KN&p-c+`PIL?&!LpF?DhX5kTCkhyZsAR?+?gm@AaD&<^D6~!V9u$WX5S) z7!M&nf7WY1ZM}a+6{4zhf}woqcxOWx%K2?^4KpYs__&~}n!aS%lI#If;HJGkqGlg2 zmhf(sjc$L9%Q{f-SMP|xQpAkT+7R!W`d7Brp=a`l^u&Cg|>Y`%o@xexiRR9aAe zC&YgRVw99*nQl{RN&Y-gjke z(TROkFI$np>ks&gfLSO(hc0kHWqSxuR@})9dTl4W;NC#f6#q0Z_<$?iG(7WhJ}A^X zG9KFP6WsoE|8PGnjI#Wl&cuS;Pyhtq@@8TudI!QuxY3NO)J&D8@L#|tS6T-7gl|Q_ zoRN!26-rMLr*$lBH^9( zI6}G>lrIWV6T;-RS+gR7P4jsT%lG!VuqqB8UMbhgR(&8;7y~=z;DbtHQGSOUom_6%0|fKEfCoHHkB`KtQY7?Jc$DINa`6=RL{Yy-L-LnVZ0VWl2aJaT?fR_3I(LwT`-a+$dcyy0W4;-~dHSLv2F!-vVE<tmKjc}p2$2P+crUj&X$euY*=`eR`Ul$?|lzcNi*AQQnNK#wY_kN3vO1J%;Oey)az5 zJ@4Z}{`Vsx57`qBj-BDYlxvRzvj|c?JWdYGM0*h1B1d=e6Qk30pwrI6E6E=f(=BSC z{SHF*U(7QhUm@&)PW_VozdG`Y>x1c58_CE!t^k%tw}k42~aT-Wm8>oXITErY|^;ptwdm}{h$|XM6f>ZWT2?P zuc3SsQFP5aLH0U7Kzq--8$b;zRr6OKIoNXf=Ic|ixp(r;?cNXAUpNWH+s~Zj9O>z| z)9F9et}YyDl%K4(BP+jmQ{f%8yUcqd1%sY=d;=>=V!e2=5IRC{EL+TnN{%Gb23W0~9^2KC zY%J26pAQh4?OiYEk(*lp(dn$eJ+j4{2AH88bm=0A4e|8S-1YQ&v z!VI@F<{qrx;`FH{;WeN6acGBvD5LIJpT?JRBT$0X$)(em8I4xM%`>Ah%!EtI`GIksO(A^f7MBPf_Ghg$mTA~_Bx*tOil zS)L4j&P^w?5E90#$3iO127tm)_ZeXjpA-JB99=Qyg#mPfOOVc~LYtW2*cw(KRVcxLmTyT8N zww_%D8Uw4!R{P$3r2BUB=2G`H-aZ-S?i(h)Jw4997T-09-g?L3Ra!Ml&b8F!HXh0! z+lS1_jZ>bsvjyVwL@#_7ToaJrz#5A|5A61&bbA}XEs4(d#8@G$i4SW62O$-N9}kfU zClVK@Njxf|ZJKUJqM$6^EImF{G#ApKmRfph3WH)CBl0_f6ftw33GD$I!xHcG0^Xih z^+I{Qsw`Knicq6HGSMeKkUL8UnervhzU@ObPhaAN{Hpd*K|6UabZXDMjyewq)@ToU z_|J8!!hYqqT`EoVul2Ly&ujmQMqJwoe@-EbZD7HE`+2Vz?!jx*uAUHGMU4~;vrSw% zv%xSI3`j_+WXJn)+O;IUJwg9mN?ImyvU*~v6mVeU=!kWPx3H4XpYl`{EtjsD-+wUFJsY{| zpm=4CJij;uflL3W0kBDJ2jsVfg>wJq^Z$x0AO$6fQttKTjB3VIB{wG4So&puwC7#R z+~o2n2eU9=4`l}v|MR0yUPk?|3zdbPiy%pw`7t|ayQPjlanT3iZ_$3; z*UQ7%soNK417A8B6C_N3WmofhW4SA#|M(f_Wo`Z`51Za%4|UjlW=9u3}ZTeA(^p3}@{<0*yCbbXYf&s@1o`E8!cH~cX0r}62n zOWtEi^*^dHr9ApsVPL4Cg%}U6JGPHA9>=jON!g@Dh-!fz2unFoV<9NTQ^YU47-?R znFoPyj)k|)R}V?~=LbDzVwbhw6QnMa_zUA1G?scr_k=O|M>Y9UfGi)%*bc`PiOYc5 zJS+ziq?61^XQRnbkfb&=dR)1Rwa1a zfUA&{cud)Y7L*6Ee&C<<2{YcN&s(LxXMCN$9hBS^w$$AMHMMWfKWbbi43T}1teZ78 z9HcMeCj~5Tpa_8$5qI)EyYI-tlC8#nE}|~T^Mz68p@Mc%Mvi7CV?VxmgBq|b_J)<9 zQXBCBBu~jrZ$JsC&!^F4Z4pDV+ybrq0WR;?tHNH$W9L!IzzS`yU{xmmU~5L~g)mi{ z$n|PL5vnIGvIGFh7F-J^KCb(ltk+50T_J-$9iKd4X$j!3=sGM`Y_b*XT+v_;{Rd!p3I)&5`1DPEXocn1aj953b^LYpVD$83$&Nu{ zdyNXG6w0S5?7S@_s?g;755LB7TQ$yu2Nc4ud)LwgrMP~01?Pf?bt2Sv29q$Oz2i#u-VA0$WH!W+{P}`hC^zz+O@;{QG7J|Hwy)V{vmH~ znzNK42@LX%vi{Om@ddr!^9~OxRtyUG3=++V!GAV$KhMjugK*g(84?S2qc(ePfcZlJ zh2(q6t>pP`lQ1p}T6wjG%=Cvic*rGCx;UvTJ7wLS3}q1b6klD_ynq%vM15$qLTz^1 z2JHHZQP{NkBvX99y!nNet#PwhRo>osHiz&fQWIBB0a@(jkgJijc}4%A*28?hALd4? z${Sy2?oO@eWy9Z&TZ6Y)C#7U`2_&7&T(q9#eBCQO-iOfOjQ@-jixOk8-*q?ze8786 z2cL>UZz68#ZHgxHTpUQq?`6Gycp=be>OXmbR}CuUdjxu1!Wq1cB5qx#4PpVoIxos% zLiA>dzsJ{VJ9V&Pv>ga-s%Q*)!z=K{iJu_N!ZMg2la=__(z!xlUa}!33h|*;18|Y zRmf&Vs{+g!|Nf{ZJ<_d!*D{--cKt8(tUn)_M3h!IoZ_DzbxmQzBz#y_VKMdQAM|Cw zKGTq2byou4`6yj|Uj~>|)ma*Lf&I1sFxS~VjhVDjYeoOlk))Rkt?&wm8Q;S~v%-dq z<2bDxz%vjWe=>39qk^YdST*+===;)j3Hhd7b!6{@%eD#bY~UB@GtZaab%gB2%7LLcIH@a+Kw2h3$t2 z$)Ec}+4Fl)CJ0`hlGbWO941QS(OQx4#6;iiPn!Aie@}!mzW+<~4E-Iino$%O(Ksjm z?}7y^(`@8aNX1yEugmbC*z@XN3BCIEGwy#zM@d0or_j>qso?GMIJ)p}_r6{2Ydl;9 zErg`*Q;^?t`RT=Q2vIY!(X5MO4mb<9>Ir*&{6Hiw7T*MZzYEj2DuX*97jCV= zECYPldVKn{K11-IUeiOOFyZl>%u$I%}0xfODKCEHR^HY5>#(T@UFxYCxt?d`|Hqdf?*B$&-VI!NAj z0Kaz~t1tZ;YCd0Nl zWafo863y{v$L#nz@OJ+TOZd7@I$o2jbnk5n2S|xJQI5>gri7UfS!G5(uV#AU`UPy> zNJN%{70txDl|C7drH)-Eaih!6T!D%q9@>_!9gr6n!r!&jlFujpeI^8BVf(V@vNEsP zDtHz0>O7B8l7LntP{xbCG63W#AwzYN9y1Rw4pjPv?^t!GD|F@kvPih~;mOCbVPE)W z=|L%=>dOz$j;BP=xB&Z0pcU9OHhkrvw&MH#T)8G_gK(wr&zMvB_WsL*7hQqw*N2eT zTmL17;qWH2e`#T!4wh>I?*iNoLkvi|ex04X5$^I~2{e1_dF%Mq{pz9v&+#d4?vs8N z_{I9=|8$tjyj)4oSIo}u{9%-?hwnX$OYyv+>{&dGZ2DAxI>QG6g~+3D&)jh7?w^-R z^S=c-4B`kr3hHC;F-Kkg92@ZyI>0ao*&kh#$>P4S50RYsSaFE^qjsY}z#+1>E-lI=y zA-2=C_0G|(jMS}T< zj>GnivA$KKO>W)%UUiMcq?rXEuf`Wv&jm4YJj%uTfRE!)BZFF_Qo!M&9s`+ zx|qDYxmLiSOP}^GkdwtL+G9XmDN~DrIEX6TUDhiz&2T`Z56cfp3Nib9R{YCxdf4P{ zcyckbwtz6?)vXXvG2_Pc5Ssk0KQ0dxQWZAbz<&3n=qHB7AM{Z-=er&$1y!=J0hDGpa}|zq64g zgAmvWxNgcHJcu|fRX6kUopYeQyEVK7FXW`=D=mBVfBnMUivG^;=&39eRQ@I{qfz)gQJ=OCTjqH!I1> z_t@7{M>xsrm?p=P7{~WC{%=~BMn#F*|2M5`p+z*|tYYzx=>OgW`ooxA6Ri7Rp7Ytj zpV5DL&c`&Wpb*cd?A;PX?Q0|9?Gl{L;V*i9KMOn1{P$0ZVyOK0b7HHWL|e|GVGQ^S zNsL}u%{FeLV*BN5^Kmddk(R0S^vXF9*(ng)x$hU(sm;jE7Yo=}qMw~>uF<~f#d$B` z+K6lG^*t;$rkFtDSxV6z9M5B%@zcL)(YhXN_6x| zI3wXmBVGX1T3#}MH71jk^xn1rVKq%C1k{4!We~*HVN6QLTl$VEFxrK#;Ie&Cb;!#f zt@=WP3li&nAI;7QDsd_t$t-HZ5%_WPGI7r_tycR|W>yg~&Vj!YhNgHptoSDPeA9pnltH)RLk+W z+391A4k(?4=D4Wstsh1JYpIY^{V57fQWVNuW?IOxpjdB(9R20k^EED;iYfZGNyyZB zavW*m9uGEQQk7G)A)~Jl7=OEqUy~{CgTNxZ1{uKsyTF=JrFImw+xDG9?(nGkcZi+uQZywF0 zBJ>E6UJLZfG0fW*BLlaM;5tI)1Xymwt%$-ZU!7%Xx5{4x^|rCD-xRDSt$X5A`9dZ( zu)x9;-PM$QpdFro_15xfTa>^3;(UQ>t9Ha0O4WAHAGZ95@8F{j{s-Ubz|+RL5Sfv5 z|1*ho;HAd)YuhrMzOq6>MMGFtUBV9W($II@zouiHpX( z9)HnKgru*gFD!$C9@DcN7g4mwkbIH3h_8p2l8p#qEFwGg@lu_w98Fp)n7!?@5Vg4yGG? z;#NayG}cMN-rgntQp!qhRYC|O*Z%0qnfcwHN{jO@ThSstMq8&m(v(9X3kmVCMBWe8 znPinXPifRvC0cp&xE4x%G76yihOyZ^Bqf~ZW-D5$qNfonEoE30qUOOMD(`S==DB%@ z82Vw|L{j@@NMfF5_yeFXyiJM$U^U={I26Yw5@1L$ak#*&%5E+My$wJSx8R+N4E?-u z2ntr|F3lt{&s2Wy4ygkXH>^pLIvOODT!a_sw`$Q60l=zn* z@Fg8lpYiKIGr2H;#e>@#G4m*u5Q@WZob7cA*@Yp_-{bW7ao#9@A_8FPx>bx@KTi>K zo7&oYZ(jdX)BbeD94 zG{Z172oe&KN=PFm&CnnnGjvLaf`l|99U_PfrKFUc(dU1j_q^vi*Tn}u^5fq3-fOS5 z_PtpOMAUivO_G-2K*kZBhpd2^sfLohL}D{4twyZzMxYy&M&*W6!;(u)SoEZP*~}*~ zLxgQB!H_j_=PtREKl`9F??DZDNXr@hS#Q8u4}lli*xis(EKWWHsq+pKNyhI9m&^yJ zU(6YIk6#n`Y5H}$|M~Ca8pPnglWX4HgQeo6HIPf!7BWXC8n;imVN-!&NHb0peIEF) zidVA^4Ym4DD|VCeK(HROZc(q{vxY^7x17{zN^xt%rIO?V&;@sM!Zt-k;$X?W75BI>Xp>} zeNbKj5l?;&d6SW+2-C556ZHfiex{jt&+iXloBKIbK)X3)$@w%`*z}QVY~0V*UpnOc zzmNm+nr{__&HNHO;Kje>_I*8_qEbgGzp< zk5=!e*1P1iymu~N2Ov;KdwU(?fzKWYW|D?8#${vU1+XkqQz=`#eG$d%5Cg#wDa#^K zI4a#~D5u&xba|m=#lU+{^p?nnU*%1vdC!uG7{`^Pd$tlLmUnUSTgI7444i}aLsTc?iV zWrRZUT|v7W=^IfBOmg0r4#cwQ18yRN6cHP&s2CF5Sy zwUs3aZ2s?IcQ%D5ld~Yy92^~%yfXlk+*fyMES*jILQ)d zz^H*J2X53xu0pP4_#T>?PCwZ)vBrUoYutK6W0)?QBy9XapeRwjpQw-u|K&E$8pSf_ zM^H7S`bj@s3*$>s^S~>((KqUvKlUB2U!}-H({(Vn#c^W z%$0x(c3!7nQOAFN-qQ1-=FG?qovvy_SIz!p{_-W0;`WBkBPjX4C~P~ITXF7r@l+I9 z=+VR$)sL8@M)ZDeH!hJ`d`K#;{iDSx*s{&asbx2r(9Glml(OJhgNo0yxn8nDk3HABLksSUm;@OF$@m9R(Rqj10^ zN<>xOFt}VMVfeSCu%KjIYoB&06n1@j`tOS*6YnY@1oH<@l&=YcW$#mD?II)!@i5yz zsPHARH7ujfEErUvGGLo3)KF9GKxIsda4}lX9V&%+rZ`7mg>>|rC_js7+YUEt7-Kl&V9s~b;=~b zt25d@5l3){%|J0f_Ex~JCBlrV*o@P&(H85bz9W&)uwW3s!KemKhUiRj?i0~0aqLfD z-e&uxQgc$~V2u1G)-|S4mSE>6?OuQPe(^Z$K_drXwaXw#vSpG#DNqY%pXwVBQu~|) z0%m3vS8m&SeqS$(;m*uaR^rS`U>^06l3E2%ERhuZjT>?WV+mg8wxiN^!p?_d*k->< zz+3@9sq+dJvHriDO_E{yYx87nIFloeMt#fT_fzhr|L>S+#u98#WY zUxoQnX$St$~|M@Osd!P^QWp+^Doy~JMW4U11&uuH$IkR z*&&R-P8KF;zg6`SU3%);^k`oO?+m|6=&u+7KdkE!AwNO?LMUTNA7CWWeR@4;uR%U* ztasYHHdUEl?&t)}*95$^^+6z6FP)qV%^2hPC=SIi=~(EV6b$EESjfpQxW+ z^eYk8Ov=3|vLC5_O4;Y%8)4UMW0S>!vxig?fOFfV^-CD}{Go8c2gT*dfPB*5e-IBd z$7zmC1T7e2&~d5-v<&sS2ui&ZG1{TQhSELY2$d`JKuYJr>Ev$e<1GXP`!xfbpix0sX#xbF{dL zTIRe5ytGyr9MdJUg4q8NS`USkFUy%C+yY~P3{m1ml=6vDA+aP^u?6N17m+Nj$7gM7 zAsg;@HE=~~%b)i#C1Lg#VJXA9rp@3IT7$}q`y*VHXj;CwHCs!j;4i5^tybMU4E=IfP`L-bi-ps$7zLiGKTq2RQ{ zAeC%l6Qx@K3B-?SeFe!J?E6(#EN0#IQW9sf2cecUwHpD9pfVKjoVzDy_D`)<_8dqy z@A$1N%VohSY%^nu(pdt9SyHL;+gyY_UMTI#2j8RkKlp=$;MY3GME<_{FG8$F0Kcv`3JPId+a7K%x zwq~T0U6FjJAH8R*$`$oDGZ4UQ2d7AupkVpW@|h^Da%(jqjn|R*TOI^ONEuqczv@_y9_A?ywI`wG^u(d`+V6LNyhOwy&0jgs2UaN9D7Df$1A z<%Ot&R6-nk*p0#PNTUX$tHdAS`-_p(mWoCq*OR#pzf} zBvl8#1+7LM9EvGJj`PLt!@N8E!D~N3$0GUqOm|mcz6hX945H7D2U$U>20xvzt~xeW z^ETpI88PLnKb5O7D)%Vg8TT!evk#1%iXp9UsnZs0KJ0M~cj;@k+GjDXPP&kIhw4Ur z_Uw|#JPaZNYH%~=i*HhNjk5dR`18b|_qtmc)yd$y2>OS0dMuOIc^Dx(R%J$vVXm-D zD=M8_k!f1)-}!;gS}!W0Pr7mfX&i9RoLCGyFtm*Fje4GJgK*dztwc$iR0Exum&E6+ zpIPO}K;vCuH4j?#{KH}sO9EhyqO6fHGlj_S;9P%E5+f=Qgf`u}G9fGHu!5bsrX%tpM$r0=f&r7vEAvAk z`c;qdH~j*nFOCfk3+SMNMvjq}jrWs@Y$Y1&AKHCGddAr-SifHsSXt_aLNpVO;fECG z_*uQu8VzH&8Y#7df~@8ROSDTM)P!;!J`2Qgz;fKj*jX<)(q}QvEtXq^VF|y4YL3H! zk8m5br_ZXf(|(ev@peLqa^}0v?V`#t=-5txZs1SJ$6~%>x1zVP^A%6!?Lq_&Duf-e zBb}P0f!QE-@-ZNcs3|JbwYq(&Z>@yzRg<&NBbx9EoJYB8n|L8E!>h}%g<%9FQQFrg zZ#Wfk0sEC5NycDF-`vrtTUAEa0T=e=|Cas+4X^Z;DOzuY0fZPTww^%l7n^i5h|dgB z@h|~n4&KyP)_;(mt~0Y_%8Ew4P2P`VIiB~wyLUZP8H5lvIc8fgc?l}Rl;!a89~PV% z)v%S(fB*O*B8tpcHD38_^r_6Auu^|pSghf)b?oe97JXESMGj^U{>er6N(W{ajIv|h z2lJW0udPcgryWop4dzp_qGXj)lgh#;Id-TNCg(Tw$$77M3s)oDoCE~lh--gHcc05P zcCzUytw(K_wof<;4#GaOQ<_&g8QgU0$-&lY9`Wf{GHK(TeOGGoewyD*{Y9+Fmr5%$ z>Af#k#6x=JBwc*vngbX7=egVk<+n+ybtvV!T%u;3MNF=YjkxCY_lN3_RerqKd4K)m zZJ2OkrAGkzo}i+5;R@^^MYrOldBN%Hf_NBA2hZG*vr13xIFsGv{1DLUnmFan&rHoY z{(5a9BlL%xM&p=Y3z+d}c?;#jtG}Ysk-+r1x$xZtGbW~rG?{S^F;B4ez*rh93lbjm zPEzOD2G3YfBbLcJ+|$Lp-Uq39SngW&$_Q@PfuT9y?9BcU4#sYb56sp)GU>@Tr=q2X zgT->mu=s(bY&~5vG?qZrQ{Q=3ZuX%Z)ttG53IS>Z$w&@XSw(0_HF4Hlo!)Z66Toi* z!)$*E!=hqqF2`Ka6t0jrA`?q25!umlEP!~gkD%nhJl<0cGUu3#(?Rvou^UX>n-y-% z^T1R+v9uTBniou_lGk|)%yr7@p@`GU-HlkI`^2@BTI>5EpJVj?@Bo3WIbzGuIWP2ke1B(T=7aJo;qirj1GOKrJ!93nnZ!MnZl#+G%JF8Q< z2)y8g(W!#5QKI;K8ggTWl-&MP*A82Z{3p12edXyFs0E~xhY)AG4Fn?KE$K>nX6?& zE@B}5l>oGP~Mz@H%k#~0aJHx{41azrKlx!{t$y;zQGhs;}Q*wAkIGlrwK zO8YbT6WuJ!i;BGSZVSX&hAEmnCTO-++i^0eiuSwHl`#iiRuhep3;BlsPEnfeLBlMg zx`-Dr&heu@hK$-nPg%FY}^p0=iHdX!O62bZ8)tq_}E5{>sf2mkoPX9IoKVk5mH0vE##=%R=?fH0hq)fRwY z9B`1O{nsXgNDU$v#J3@)^Z6m~uOpcYv$V(QbvJr4(!`qV{lD<81J@1(pU&wx_b>Ez^$=6WXrk*e|x1{}8C~BoHbomvF76r*TJ( z&y9aus4@VL6yF?wn_=c z`>N5~xB>+*HGEDV1$NxEDYh=Z)5ga3I~wQ&9XNAWi3^HjuX5U&kGiTmN^OJuR(}~j zT4!hz^=N5c@SDqK>82zL{H0E99iQ(Fe#^fpv;()>hRbl(VTx|cqizl?MWMq&ii(R{_CDIMVwWeGX!IBx8=aCx2N$B8VIKc*v{Z z8cfsSimo~1gpPRJ&-*{<-LQ5e>cI8hD@Yx&nUW<3BOX>ZqVwUzPrxU3U>=WFcPDTdd&-gqu^%6PX#sEt$mM(!}k`b1Vh*65lPq2 z*&uJZW?}T-_%ZX?AXe`$#zt0L;w{x)ySr~|E&ZI_CrBumd@py=<{r{^bG4q-Qc4&; z7xNdZY>2kYO8mnr9GI^2onaWF0BqvVAcKcxsYK5=u2$5W$Z29Sh$Ah}Pi^8txOsWl zM(UngXNG-@)v-|IF)dcqvt87uG26w~A|0hm$rdVu_v@rtOFo%u=^{l8p0Q#Sya-{J zy-~|3=vp4@l!IbTZXbtN%6#vMxAbei&t)c`^*2}9bOB5jHa2X{RWZC1*F``W0p#FH z8|2qdEe7TWyYIg9o`Y6JS%%@CD@Fuj!7t1U>@j64q=)HSy2HYK$Sv&6eva=Y$G!HbwEzeht_Je{NUF$Rtz;ZhJ zEv5_JJ9+F7X=N&U9dD{&yiur}Bi;|-p;&e~6P*uBFt3R!K@(W5&U@Qm!&h+RC=w2a zr^h+(C}qY5Ym(2OsD|lFutJnda-ViYGHyaOP3KW3)>Bi0v%Q&gi9m`1?B%7)LQ;gs zv8K7^3eyu}sO`Gk#eX~n+*wK`2gslRTb!o8iiA4{!en9-&+ap2t! z^Kput?4%#J{vNEgu%z;*zP0s`UBV{=Rl14WptJOo zr8I8B#L%s+*Bz7BG?9L_7qaWY_JY#%Rdd;RUx{xh=okG2nr=3>Ltc=f&+w+Hv`e4Wy3a(N=H8L#nR+O|Zmf|iRV8G)I+|Hiv*lp9CjVCW4meuw% zp)#l+o3XT&R%XNy2WT{{rQd1#{Wc#h8Ez$E$Rz8e={Gq7Kq9d)8A^P%i zehDYTVpOX`%~Z&)s9n`PyZqzHBPc+T-eqW9PiG>G>@kWTJ$Ma?)oLY(XR`b7e1oKU^gpR)OarW3{9uJ z>0c)`Xh`ty(-Gjluz75LGjMG6Dq&>(_1vbx+=!5uqM2&DhGPx;+q8U#ozoMTN|gcY(6Nwkc&+ICE`c z=#d@0Yu}0%9Kuw?jwtcX97Hqpc->v2Zn?r5EdNua>Ur`8=^_=TE_UpbVmnCk*O9~3 zvOuI8@9(NTdedE-f+uRp1sPQF#C?Mm>TB+kW8lcveH0jAO9M*uMGt^us6xk1#2it( zF3T0)1a{MHiKJAV3Nvt?;(Jm88*5lelMg?0nNs+Cw&`9RW749+Ac$LETA5^1*j6mlsVQ% zdqg4enY66s9Xq7@8k|xy9Zr8Cjk3dCRBJ_bqnz1HM^OpelrP_!adPv~V#*@u4M(BXtycM+NOZ%c z^6St-%*!Cchoj8U0fD;+^vINmU>`9xOlB1-lUV{48N@vwGyN%j_aH$jlp2OedqFi> zWV>_TU594seD2cr+$g1T5FKdJ?_K^^F_7dlXZkVf27@}#2-vUd@||&4(zk`*Qshfv z@W%**M8Pxu0wF-GD4(R4js!~=o=Iiit`qZ<4Yn6)f#?k^cnSSl07PP!LV>`m?aZB} zu5^*a2sbjNsOkz1g15HfU{(I+^rXLq{hFffM)Y4{zXUpSWTtodm)3d&hMdiQXip0b z`a@KL<9Q(8)x+qu|L!(Pc3RUk=!54YjZq3KDM`*XSri@h_s9*3j{&>Al(@2v0=5aF-ABwSpFnU4l1QlIRw=1Lr;@G{>@svOv!rG0i-E zSn&E+A_+!8zWEIfgPY||!T&OaZwxdDOUcfP2@hXW;Z^&@JGujdK43)&G77e=cP zy&g~*Ll%lfLIg?^XQCE=%Qt47+ZSQ92 z57~ypZ>@KXP>xc&rrrg#Lp67QfxL=x|7()*<1&F_g|2i#=TN5`dFI(~D|apvbT2Nm zm&m$Q6vv8m(-vNOiyPINcTQi=So2}P6x?wj0JuE3{BRpyv#0?8U4X50&U|lss)V&h z<1ZS&yfPEYf`fBslhtps!qv6x@VI%OA^RLIHL#@B47s#EL|q7v*CaGJ>t$yxKk1*Z zHg)%F&YXYe)OZ=8SuT@9siqd_DX19nsR{b=a*TL7b)zTemNOQ7?A4fLHe|x_3Zfq= zR2*cvQ`)&4PmN;){nB-ZBe7J%`huM7#c|-(de>+}R^NKto&{d!#-s42=S@f*Zj0JS ze;(z<-9P>mx;%PIJ!Z;gA@}3^A5j9*Ixz_xdZpCb-*9%0egf%y%8 z_|%Vf(o^HX-CqNoSWFGyY#TKFm~)cSZ7LL97S5t;bi*jO8-*HiA<+d>S%rH^}o9Z|^&8NUI88Gksvp+%w69etY z)eM)_l5Y7*ch$|YK$=5A2Wxyx{DN)RFYvh!j>(kzP3V$FA{Re6in)-y7h?Y7K$6vh zP=OOGGMXI!E;SQ4aOD=niugg9dm9s^TRlom&zpfPTF;Brgs3LO?>@ICo z6tpGCQ^Y(L_1#qS{SN*T_HH)ya*!|lRSiaZfC(9ca9e|4?aO6DY+-L0Qwpcb<|~%z z`lnPxao@aV{9#>6w%noilC*Ev7HTj)4fEv*c_AnRfEHaXS%Yt30hFuL#!!f(PP@;*neWPuAc@wGqBxGfWYdYX0ZX=qA{V z%H^2V5Fb-(Odg$9CAhpGH>s(iqH?(GbcLZD2*i=_F=B%YYrXI{;W|uXCHt2CV2zb$ z8&52P95st=>QMIrKic?dF)Il%F8^e0U`x_oL}%kKp_s-@*Uy7pO7~w{pLEMCx$e7u zf7xEhWg!>gMir8Clkkv5-=3Cx5UlF?by)qVeH^#E+5{2!#5h#WCM=aR0ycj%tE03Y zSq+B9c|3v;nPKGyB6)(&{krDrMXmo_3l=CdBrotk7kWvtHiQUhSSo4-psE&HYEHE2!K-3v~{FDiNs2-)#4nX)07#1Ga zF1J0^WGG>Gv=MfDykVOcDeE)rQG7&%SiV-eDP2X65nj0`(fe?57Ljd9-~%0VtMCo* zq!o7Xc>kBe#%3&iSS(j2+XUm=wac5|vmyX)3q3jZKEFz%-DC$Hxi1?a~V9jknl z`jsCm;p1nN8<`dDPK}pYN82QJ-iVMOK?i^WdsvJ(%e5(#636$%ud&ol$3=-92JqXy zqx~Y9usEs7_|*|Qnc?`dJa%*JI8V(_bpew3^}{u1VP~*~-Cr~k#wuh5AXe|uMn6-h z$Br4JTen>A3M*XF8v(Ay;+@N-rrJQDmmTVpT zD4lV$*|Fz|e^^#^-sn17YMW{6t?IWMdI+eEECA~Hj z<(R33L20KK%4T4rM!@TPAy83=rDB;BVUC+mFmp({O&g)N7@@!*REbPy$p{-2FYzr_Em zaT#>yn*VmOe%0oA`)Z93^CfZCp$qT~DNls|FAo1I0Hhe=zE+W)gt=Gi3CZi{o;I%% zSYaEqs~vle&4jB8gE5YHW>#0{S{50*N>eZ|N1SKbYJ&vHWufUM?bi5C%EUQkMoO7Z z;DG@hzN0|~f~D-+%uZRYR0I$GQbIg+nu#0w(G**3ZrT1vnPEa3w80tLYoKygetJc^ zY2o_QD*p)!nH=LL)I^FsCXS}sGTBj=S~-K z>~YP=5TsqFy@O#n>nt7k_Rsuk>sG5o9LzHxwu(yxrTs_6M?hZYUOG~T jN-uByR z{e=SSu8m50gN=8~>e|1UtbaQx?;9Qge|ym6{-&Q4@MORh$lW%Ej!ASt6)A?{)XSJ8 z_zN~2H{l1b1pPJ*zKQ6xxQ~e(*5crOU3b4M*^?D|o`?JT`swT559kk9?CPYv`w(xr z!$%5aV0Yz&bN}zR0K#S;K^(1f)tj(54Bp0qm2u_jus!1WZBsthvg{3X%1~UUF=1>G zjerkN^IUG#wj%};P7mEYeB|der4t)o2UXL|NKQ-J04m@KP1URo1b|#4cd4LZ<%F(R zgc`03T!$Do|J*zqZb&k-g31FxS!BbH2PWLi6V;ZP;wIQ8Ar}fGoQkCpV&(3>m~F~3 zm@|fo@hPn#5|M#a*7Z5boVxT#fXwB|68GoZjzqVlNja!ik@^o4^7&Ky|*Hm5BVmT#f2mUb?j?Nrp0N(1DVNncP z1LmiSkjna71IBv}~kyyLH%kSpzGkCfB(%#GVl>KYXTbHZbpi4EIO%DKy` zXcS=SVv|~zair2c>+2%myW4Yv_Ys6M;?|_+`jV;%MuEMr%{P{9S@Evu28XNvT5MOD3Q zE~b44imLNk4z13&WA2v?1cY-E7+b3wPnbX1QRkC1>#DC7F2$bkt1pAHjs2$Uy?J?e z4u8nCIRF{d&`crgnZqb)6)Ou>^s;?vvtKLdYUYe$fNlw^Y0MAU$x{%?CuL7{^SxLT zoTLZSjt)-@p>8~|qsSK4WrY;jE%PD~-Flmp6b@?atx?)NW9&4ckx7q_F}BSF6bmrV zPb?NxX|4NYYMUJXQ$$}V0>#us>TUjj-wN;*$OF*n>>nL?)9OHN*8|tz!_&TM=BVvD z7vBmrtc)$D|UI|Wtu_#)HEJ>_m%TAYuJND`jI!A;~r)ph1-1%%!o zC1m@la8_#x#o*&=h|+@I`nol*W}usK8{Gvqf<$#Mz1CGC!mkOrtf4R9{~Wjk@0mHz z>gGP~j#}Fc-0JH;Vzc6jdsV*IA>^Ia`2I%#RW)eoXSYnF_6_)#)ItSsy%X&Gd>nCs7sc5+b&ek#cdRworvktTnOU@ zMZ+n&;YXVb&&UkEttS>4m6|19Qqhftjt8N&la<|jP*1HV!!Un#F7nkp$qI2pXD6Dm zWVY^TDX`?+0bwUmaZGA^Q=v>r;S6ZYg(4q@v1Juo2BMoIXNXP+_^qsq%2p}52V)yV zX)sHw6Y0aW4gbe~9Nyw=gAy0#fGey@30wadJ^1Chr zW=9;EqXLmt1obk&n~SjJR2b@U8L1!hJG}u?STqN$fG7%7gP~SuulrpzL$8eYwF7&0 z#TuU=646-;YReI z?JJ&-OFZwFp{sQIBQ%O$wRkx*8~r7jIL%TGy`6dOM%hR$6iuG}E@H}cbGZD* zA}eduifTh71hkYd1l)<4xo}HweZP2_ew~wsoWW-}KtGs7iHUZi$c%1g)#+K&skfy? z+rm5)3oW~Be(UoKDm0RH4-hl2pWcqIpWk|Q_^5TymW-I(XYNfEoA~9pl$(~jYqOcU z$TZzf4~sxo>Xf(2^eb&%cHjCL*UO+h6R+8FH6b3iu*M|`wYtt1hp^OL_jt?bozvx! z^ro)`3lFJfPrM7fbKf0?$nWs+%c^XCWJgPE6!)=*I7EL@dp16%D_<8)?(=SyCZp*? z&)%~ikdVuZt57ZD>w3K%zMzqhv8F3ZvY?M3E7%A)^hm+^P82)Uk#&ebRZ{NtPr#di zg0^%-6QPcxf}UZ`i3*@5Skn|rj%%0Qz>4Q3F7-}bi{u==p(+ZH&`#rbL(6!Ubxg&0 zmfut#VniYWec*H>Yp{mL zo$-7${-QOYy%U(NSTguqJc16Pr}Ss&HqRv*`UI-mQTT`(icMc7uK20s?J)u}8n5n( zhnl94Ha*8L4?pQLa|!+T&FCwG9bSP%e5U(pz_efd?yO-HykL`mw;rOjn22V+z$sIV z9oqa)HLA!PwLzvW3Ds^L65cEt3bDp@wS00;BVgZ_U!ODvv`=dYu$+jp&kEZB!Gw*j zA%z%PEimaMQR*+BU1dp(YXi#yX+~4os}j~0Fe72tt|gKa#0Dv7c0LJ+d31xqY^Egd zWl?|3^b`yeMqGK{y`9nYDr~iZPO&?|7@#z{f~xt#^$3 zm}>W(>acbG-uEq1XYC%>G_}nS$?KkgaWCrdymzZE+xhju`zz1CiL4)dF1pbHdyVbU zD_Bb=^z%G*T`CCv>##g|*SJ>v@6H%2DH44R`NkOVThfIe;q_05DX zjc;USfwNymEI5#fAO&H4tG&TaRU>}e=b_!|4qJBT3Pf_DF!lh9UdvlHJ(DNH(xH6hG%y@6bquGg1Yrj&K* z_jS>(RCgg~7|Jar2t9uZT4q1ez$&w7ZZl4!`oYK3JdoB~H{OdA(L=p2DlRFr@(Q94 zpOHnsu7^}XAPX5X*l)Cbl_`umgW1jCX9aIF&*L@jfcaGUsXgO zQ~(X7*}f)T6lxvn+x;rh@u67u-u=+beNPT5{H4t54MiIgP`j~04@LY`5FKf39m^?M1i zrmwdaN4>wRV@6Zhp6jJamjvMhhI#ZB`WtUk5p*gGBJ%&2nWQ2itgJfWjjX8Hv>X&$ zx#=WAW8LEH(;wzJxwUm~S$9=Lsd~+rF0>TGGs#=UaQTK48?#D^?eKLlC(CEn4Q z>@I?PZrz$yc8v5DK8B29<~wugXsQZZacoi&o`l3tw9t{(=Ar zI((#~n2e$26p`qb-f5Pnq_qYeF)a`Z1$4OXq1?WxTtUwTmqZanwKCb?jI&*lqxUCE zjD-#WN<{5!A-bfzH)@BAq@}IdlBdG8sjA`2o~^3}kNm+m-mjBLP5D>O9N4E#Zr%RE zP^NRq-DZ*sPckX39_?(g|AJ8CP=^0{h)#K8*wguhB9I)L?p>@x0CYko!Wc6BXTKY&W*QDC-1=~K zXn(;S$>IKUkbz*sj>BUrwNGdW*yJF_u)rA?t+$0x#yKc=qD%ZAnHn z3W8yQFqz9LS@ifu(uT~+%)gV4R4HLF} zkZ=K2xHKQ8xCoDyY?7H@T{Neyks0O%P0IXmNkCl-pnKlM5f5Ih*NX2AL)VR@>jk<} z4QCYy0B>z7rOnrI*U=q?3SXHgmAVEV#JQEPfb6@zc{Z>agM+G94a}9)LZS!Vg-|t#8w)R1r2;YwWfU%|eyg%z0b_?-qIkYT`^aU?$+N z{NBg}g@x-4%Lh6AjR}kt3|v1yb6a8#=R7Vm8>UY22;3S>sqyaos-nInW6`hN0m=@y zMDCMGsdJUHL9nqqT+>;8%?`U=kZ;3Vg||UHl}Nv(dD2Be%op10*%t(k0K}at#%t(`5E1K4Q&NZ$GaU9#joVDAe$4&= zW2FiA;LR1^7@k!q;~y$Ih}=|{2>o^e_B=u&;nbuc$lS=b_t~-ef$e`G0bclbB!YNv z_@E^wtlusbVa|VCT24^Q%t+Uv{qZc#GCIBKGmO=2hKYJk8_b8pK9ELU9WUWyhgCJ8 zVT7<3BX@UldteAh8-;_cmWMxAdc{V7v!048$I-1QNAYjYWropZoMC5%*?47!wX;HC zaI`#o-CT*G&8AaxVrc2)xY^TiarS_v%^Ap`0Pt69gL~9BQPgB==(fY?CEu+lh^SU? z^zu<39`7k`R=>-5y>YQqpBISiRdSQUI){p-)~(Wbk}aZpz`<6C`nsO+HV75`oLztH z%}Z+=vjgAZ!^U>^ze5(6{$-4gzI4X%1BD|TI+QGUFdb48xnsitc8iAl3aL7Px)q3Qr0g^0g9jJHU z8gB=QB829EtD-50Kqt>#CU(y(JmRj)q=Xm8b^*eH3W$9)&mlwvcvYNPnVP)gB|_?~ z+@n3hkRewz{}BGj5K`m4ui6%&s76w_qE!Yud^gTYb8YgYQo20oCZx~a=?ym+Fd^Rc(#+%c3Iw@X9R^-10IxY!!h%kuOw@<0+ zX0&5IUL_~H1ImDR7n(t|mNRp~W@)wEOo@BwC!8DbsT&?Vk!WH%+%e1y6ExfnOnzRM z@lLnZmCiG2#}3FK3*btth0DS}eS!OR<#Z}~)%JAcJ$3&JOr&~&V&&H2qijFFY8*>q zE=wi!Pv{?sc~3%m&ktL@Xg&R(P2bl(uTEMC+Fj}W_06vtxGHVgR>?Z8fp&?$=5`zY zoA1yxsU)y!X7%{4d-TVryu!;1{;AMYYP6^0VmAbuVk~J7${8 zggppWtC~F*LrNL0±t)^$>@mtZknrN8~-E0~DK@L?cQV@LEPGfX(#zD2hI;2<87hFz$jfD?#*5)w*jFYFs5=v}?!*ZL93DN53ygbd<$- zT3B89tc zkZA?orJbg#6*c2V@eSeEV9dbDN8`_hGs9==PD+?!q5{I?Sf12G*tYKM?y@+&o;~kg zx{Z^w^^2p@YBT>b-%IN!=JubWAE{Qv0jU)WFhyXA#~sa#-877Vo2&|{y{xw3fsik~ zBjA=~adNamL?oYvUIE&H&lMP z!CZ@BQ%uin{3E;ldTCO`obfb?L~V9kt4vgGK3?RhfDvbyx_u=D}}{^Z5K%Te2G3>Cld#rlWfS}UKtd@(KMjU*=Cm=oa6uRGl9 zf6b)&dVU0a^KqJ3q35q3_)I>i%jX}Yy+{b7+c~Ss#7i5`L+ERohW-(h3*LWpzc9lI z1!vW|II{l;==)|muHB%mn_36jvrMg$zbQ(YkrRl53G91hmGp!^y+f=Ea(;M z#;6$lx8WkW#(LNF4t@m^;(&|D2L+(<8|jew#%E+w_&IYOA#GF)+yn4+!}mm9)%^41 z*j?<8e?`9QB32tQDdfZCO``&V$+pdDptSI4C?A@5x=zh{xA!}si)veLV@lA-N@_Q; z5w|S8-hcTz*u8b)A1`kU6ieeCxQ-Bc)_D5=Vc0bKCb$#!e2_v~tvy z+D58|{}0!}T-0B-aBi+G2(7!LZ}c_rF)MUt%(^EIOdl@G1LYSsI+04PK}|nqwRAZ= zK9h`d6Cb;Lz37~)G-!fcq_Nyk7!b}yn{_^+Ti`4Hz}0e#tqjCk*G(idADv3duQbvH{aXh7I^98td%GjSmBTn)MGbY=+_LuJ|hl81!AT z8b>^nA-B?k&tV+P!{kBFOwx>NP!(umOQJM`DUh}H2TR_$#N1D#jPozAmq%)A zE?GvVq*`}#EYEAuIh!uur1c#l=IP-Ne7wK1&jX-aLN#nB65}b50fhNs>o6uz$^WDn z$Y5%tfgTJlmNc{I2)NX9-`jkK#$b=%H!tPgbw0h5t zvqG;pN5|5?rlRlYHEzN&m)A+3{@9NyPsmd{-^i~oldgC453eUDW9|>icP;K)jB1Uh zX?PV6h~vYAy*BNcV4%)JDlt9Gh#@FfP=AWuXLPy6gM*`D;+Q@yGDdNBz_B_|GsbOU z>?7cA&(=Kn)Pm_s4Qqn}V6(7W_%Lf6XnGUG74zB`{kKP7{d^P?WJ6wD@&r8~_vm!! z=X(PVK6(7651roB;babok2)--b~;#&5*~K!u@T6e!grSjZG(c;pXe3Pk6Z_2$$xPw zhc|o;sGCa;&Wu@Y!2T5zW{>#y!BRqiTu8l4gGD0+BM9?RxFr~Oy84w7CtP#S?6!Im z;HeGpfvab8J~z+ha$3B#!vc1U5vc|$*BJ$MfZ5Sj=52VGzbIDLENOsKYPD(&e)-P> z^h^NShZM6usaV$5htUzg6D`z58ng7yZ_siZUY=%Ij-j>gV}@Br)DAfFkncb=O^P2r zn@p!z%>SQ?&!cL4?Kf(ORiMxMUgIK@H!I1-X7m4N^tC`6eWPXe{y(Cvz$@s$edX5e z^0bH1MdNSdgu`R;Uvc~*K~X_ySJQNx>qdfyNXzTTP~Y$7mo|6HkY@&>*GD1x4xg9> zFsI}wUSAH?x2<3Fyq$SZ`_-DqBP-l>)}vj*N- zwdxr7SB}JRn4utEm)-38-}$7xJxkJGF6B*9Kjtxq^m!o25ilU;=J9VHjrNJkD=?>X`ePWo9WIhPMmn&G>pz?k%p1r*B9^I>J&~e^C@qldmz(uUMhk zuaXwJ&g0SpwxTk{NcmC@5~r}B=n{`b&8vADl;lg#61@YteP3bH_S{H!xBtNPGdZ{J z$I_gH#IW?h)RIC`k))Wrs{v5b|2|x8Y=R@iazEE=8LbinT z>_56H6nouW25hpn)@C9bGu*e?W|)r4tCu~)zlc7UF@1z=qzfM$nImU~p$AJn|8%`S zA7-2sGaEO1hXXc{VorX#n|*TP9!Rbj4Gc%W7yCoAk895Vs$)rR)Un^}<~IHj%qIlt z^zQ5Az;ayZheLGh&o9h-E0ebPDAGK4FuvvVfYgtomzh=P1@1mppF*O)OXQoA9>GD} zCv1ptFuQ329)zrz333*LI5pRy^OPm!=kl7?s#KM)C0G!3PV_IE3;)HC=-*`c(WPGF zjshx&UP^7boD)*&G28{gJ-6L5$Bk~C5VWZI)|N5ZUkh-{>Lrsa>@Rk9%MVI>O4qgD zl&5-Vmfsmv*^qAzFU9+=QF3PxL{Ueei>kKU?`G1GGEVM4J)3zcxns5*lDB0PVn^>U zcT$t^hdI8Bsilxvd0YRJO&-=Kof*qpF%g?Iq# zh}$xnq5i=oC6^P&%e0=Ze7g>DKf-8<2jgBrJR9>qgawwXjr9#EE`4vNxVs?Ek%sG` z`mRsD3*A*-L@W+-?xZzEy_n~HTuOXl@wn^7+Zh)|7re5Kx$^b(x#jWzbe}9gI zqA^|f`Zp|o@3Z=(%U{n){VJ$}t|$7PtE5QpWAI?id7gDl)D-jK>^!eh!T6=x19|gt z4GGO<(e%6*2lm}uRwxK>|JdrtpJ)T&Hvi7Mh^S5C5uhrwDMPDOH}-*9odyi-@>yV1 z*^Uk?{6;MapWtW{>C0iCt2|8kSL7b`bI7&^hbr>|Z-K&MaIV@H6HC{P*Hi7PIiM$G z1-l+c0wsGRWd&xr_EfUPHrcwuUlcdWyLty)!aaGp4L%X?m+;8jwk*xG*1yu}AxQjl zbOB)mHYVkogODe)W=VPd%MW;rQH=oPe^Lj+$4w#uxK#;wwLju|*CC#lY3kVrWFut& zNZP3fcpkFl`?;n|9oC#@2>Tf5B7&MafCYBM1nWyzT)GFRdi$X~C{Ew?{?wB6_6H;Z z+7XS)ApyYwCS)9A*%h8}x@pn)ZQZ!NGwxq$1d}t3mP(?DdcP+YUu#lK=%#DQ&sUIM zIfmC!>AV1FPQw@4*lH}=cGH(@BRj&q7494sv^W#La(vtx=)bi9fCFLjmIEGuKLkRE ziBkR*ig9U7iy!?KU|2aEDpMn=BxrcN?04ERxiP@kcOGz}cOjj<^n(p^fUnB;Ah#s= zT0YS~fu0Fu&Z(Un#+U_W?*REYa>oFf8AZCjqXz(sWcU?P3n;UIc}>0!6xq)9KzsV= zacs?`(#l7gI>Zz=FUd6LG{^C6NQ?)QjBUWr2aJtNb^r{Le?LH_9s{0GuY&9#dj#1F zkiq$>!qirDR6ma8_VJa8Y-&6GkLP+DXJ^$J{7;Kb2M_EUTGDHvs>t#t{nr$Kt4)BM zjLbTOa;W!3`eYGCiu{SyqR(_U1b{$&EMwmBbxJGp@pnUh6wq}OY%k5j;gp654P_=%abNAx&q*y2G*$;OLGB-uy!zIC`n?Y*X6r%B|YfNwf|zYd_jV zo~==S>>Tj+Mt6BLKZriK$t2nHP=BF=tyX{W01x`19~e9Oa|~!F(k;(M@S#!q1r~sD zn&k!n2xq9CLHv2JxIwLKaa6kWXRMfkerr8L4z%6tndv>-{lhkAjLqkSTSg7=ZpEse zF{>$UH-9b!exS>@i0y@ih@h0>PtGmw~P`*9T6wEB}$TQIAj;3 zJOW^zx1YDd*|q`fl?B%QYo2|UFm^Y`!!1Lj7J~amN=e_Z#mya7UBg@|fnC7Kzq1|M z?VS?%#XQnB`W1PBIB7^)ZZoes%!F5L36!W%8_2H5h6f{;M^=N2GRMPTCAb4BR-c~q z>*nc83vtY`p0hUPiL=MGlZpQ*A6f+_BkM=rg)klQ3A|KnIF;*Fqap4P#oI?XNpw{R z0HH9wI}+#ive+~-POCKTW}b}{RZ%D=sGL8#`xy)X)_46!A|97l2*c5xq=9b$jP?OM zfd@m1Qa1N^6D+HNq8+s#^|@Q8!U+&Of1ZRD17H>6`ePMJ_Ori`_;RT?3@aI;GN{{w zeKXY7T0{_z8Q$}p&Fg{$&=f^Ua^0Wx^!ZEX#r~e^=g~Lywu5tw`3Z< z2s^fXk#6JNMxP_iQ;cv($`}#;u2w?SYvq}NqYw{MX z>w-MC5*)Ta7GmmttDn1~TF`+cRT&b0EkXE)Px}Lf`ri<@|A&EQdg%WFVoArpsNY{W zo}T$nhlcxU%MQ<04(LWx^QAcdDuFeN`t3skE<$Rhvy{g(BYMq2&tq71X7>4{Ytwy} zlCtL-{Ynn>fc9}T$E13865h1ABg-P-_w730=)z_rqt^dny8ow;H6th)@=eqa_ET;s zWd0#}1Bi6!p0tu?^!1Z$d%^0Tjz4EVReM75%$=Ge3WJ6ooQPx!o3*|#fN%sy6`>z@0=`NKkfGj6x#j3@DG`=2!v zeco9ivzMB{{-)YnK0)ybkz8|oxpJ`P_us=!5U|EgJW>Ln46Z_Iv%kMBd~Z0ZQyRlf zO4qaT+1WQ06*fK3grf!ERL?s58OYdIrnz9Ug;}zx7s7Ce%+$wf=j01cX+gWeXZ==! zD*;BFPyj+xZbGOPKb=FoEY{`hhD1aJ4LLXUZORtalWuCF;H%P7oXW@uJA2#*ps`fmtQnsr`K+$s6!XoggysRtzOwI* zBS{vz4!Py>A-2|Q)@$4A3TUzoqOr&xaI^H~l17=# z7KPzdIfYw25F+_NmvrWJKyO-FHpFAn%my3u3Vw4`2}g6n(R3;UFM%K@02PI#=o}S@ z*C7Vr?nxWlHddFtc4gB4-p1h8c=F5-o}N8A^_%}6OzL^|;HT%XeLUoB_&`TEX=Lbcj_%^#q; zT8=hNPt!{LP?z^%d{tL5OpcMvffu5t5QeWBl$2~B{hw^z(#|f+f(86%M_Y9i_ zX2IM`zF@mhd;d7jkn(}Xh>8~(Vp1>qgFRVp^jjz>;;ntjR(eHxO?pGRpM}HxEFmi_ z>dmX_pT^{wmZqMQmXhAzGTagqGyV=iH|Z-tFp+%g2+|Gz18RiV23p2_sj1X#xg#&b?2b)0h{Fpg~}g?ul`ut^*m1TM>!^Cv(Ft z4;Y;{e2Yb#jp}T$>kxDH#lpNjI=~Jy3;AzI5>5Zx+wK3~di$fB-rmDdJ5&7UgNB5* zs>dZfGZcsS@$z>*0!YTS<-O%MZdXBqg(F2*!(+_4XR~wCUpJmOu6kEsy$>jv2!ysRyy$y-F2}~hG-8Rr(U+ivBxc9@a zATW}8i1WQTDo5%-)VhX|gI)=mz@C|oOfR|l_jWpu9gKNrGL?0?Q_WK|6~~mWDL+p$n|uz4+sN!n1`OQP&RqGS0NBLE~Dw=n?8%iYwM>pI&$>0afyk_992 za7c!duuO3v54%l51I2!PST}}$k4sX=ByEBGYP)oz>ki&hS6)%m|qy8~0ryzN?ikA)dxU zJZ9^+~4y~fwyrwuZRbg5&rKR(B6hm{f-66_+v5V0f)SA})%JxG8l>;K<8&@`3jEr{W7i z*8GISY{1n9g5wOQ8ubb9CUo)wYSKUJcHk~UIlj}ZOig@>>e7+5H{b#Hnv-)Om0T-H zZ-;rxp35T*<_Z8-P!%I!r^U-;!te{%Eo+@rJnU5)mzF6mP2zQUPQ#!>*#oW}``|)1 zytCHUs{dqrIpkJqO)8`Qu#CZyfZPwE{#}rhCndg@DJ@@u1lw9_8}EXi3ZQ}AA40OA zV)u$1dFZrdKG{w-*c}{PkrJZ6WEr;X!;5PjmmdU&talWS5f3pDebBgBWvpd}GNf{5 z8pr$NURpoSpsHI}Zt%pc>zVA=@{!{a{PzfFm|CNM?AssL_`JORe>U-3{|_$Md%93a zpX+pb!_;vUV@Zm_{5DfX6LCO3+4x}VV*dMbs{eU{PsqlfrE5g7Cb9G-nsU2=Cj74pQFXY^COsH>RAJf|H5zH_SP3k zl3Z{usfa(h4MexZ8jMO3r^-mIz#;N(9@tl%+^A|FcTjK|=i9vX_7mr}OSU`$!i<1O z`LUwFNWQ)4!`~n@LiV}LiGaryRu6=l>4~eu+FGp%@M>HQ)Zs^Ts-+r8D4DxWZKWM| zLlQ)vK77W3k6hw_%?0=%bNh1CixI(-QbybGh3;Yz%fCI;5f4wO#IS10=#$*7Hnlnq z>>JnlFW8jyMQse5a5%(-I8-5X6C+JH^BE_VJa-O?08kA= z3X^*73M2>vsZBo$Q+a_eCU-LKsw2+UY+u{_O`l^lXdqT zwI@!58?(ki<{pu`vXY~!`DT0g9eh)H?Y_U7F`QCfzHaZWm}}ZYkBX6_3Mj8q)(j`Q zyHRY=%O+TH)m4AR0JdLDVh(8Hmg#Gz=Q4QUH+Nnz>*AO^HW73C3PN!g-MUP-y~PZ* zAVZd7$`6A`PTzf(kZ6T+hw18dC)RxG_aXn&I9|VX{;5Ctv(6xxEq|q8Ma&AH6K)Vx zJB!;ygD(HCm6K2h*v|{uZ&0-wyFl#`oEH_N}1rW%jVIfLy19Ae{JnW|h zlD_h_AulGB8a_P`0zrXZ)PW$v;f_b#4WN`Mwvl)WkuB~qoxoZLwoD$NQWe0e8VzZ) z$!3nTgp>nq6RVvq2(5Y`iod98BK|Ogtz_d^m@WeRJ3mXqLU;@S-B=hC{XR*CkAM%L zC|HNoFa0k&v2JDQdAyyYn$3j##;Z2z+Bx+{Nb-KLoSryf5bw}zA=|x-_rDiO#tg53 zX0${cg3F7wvp~7(v*^ibxVU4`ur(l6yPe_3+>?Fu^;&1PH_3>k<)Sp<_Wcd(-TZ65 z3lTZRYc(^Nm+B+x!u3ytSJ#a$T<5HKRyr|1Aun6>yztf*hsx%G$VdGRO|DmHdLRHh>8m@A`6#0>5xcI3G*G?~ zG&!jpC~Pd>Aa;)lcJwQXr;*Y^v-lAla@qYSZ{&|_Y2Gu>&{c9aFz+VH)CWNQ|F0<1 zb!_A_z7Zg%MeXK&6b~YuvHvr@1+WIo49e+|qx9W*ZJ>`M-owO=4W3?h9Td;(8*Y1@ zg`|vKx+0RU?FILYJ`$SXbe!>DYF_D~jt+OO=g3g<94h6TEz+hZ6XQ#vU{`!iv~$ED z<89`5h>_m4iw9(?GI#gE+p=YMG!*q5C$#6lhfw0Vrd*P77 ze7T$+@d+X@is36lq*v@K0+quAm9hyDGe>Tkj`V2qcS-9-?w7owU25{K(O~2TcQxV~ zlGODSen15VhPrSI3wbQ|9UYGJn^gq2ny?yE2ZUT=Z~g}Dt}(e-6nhcME*9>?He%ts z5gBG)6cL^o+X~XZ$gVv0zmg|!a-Q=&4M|?^nIgP1;3wJJOh3@j^@t+LaDK<^$}N+Irq{bi508r`(RKFp4J*ErVPpuC12m6h@dNr;Y6Ka)9_AY~F5u^>Z#;YUP zdI>J_QchXwEP&#eGr`f*xtufHb%>%29r|@VAH_}0zZg*;9@FXRw@SZwzP~{x60X;y zhiIhB0L(tJ2G14<3Hdq6&C2WmWSKNM{%@8EC{x6~)4)CjkP|oe%GtUD4fk%Kzv=w{ zE>qe5G76ROG0hsHL-pF2^T8-w!|Kr*;?$j~+vx|dhWN6jCAzk*r;YIZE~;pn=6shd zb~Gt*t~nzEulG2orz=Zja}dzQ7QZ9SUH6u&s!>08*RrE)vLCLael8hiJFv;?y28Y> zQU(%;-~9)EMTFjBQ2*u%oOmkZk+)t!IL7a^jOi`}n>+%}1fqT!7t`11ZRH%WL;Za} z6719s0p81~1p4{x6$eZHxsyjan8y&LM=;WV9^9iMKuNbOxaD?Q`MLVrJ?ewci5)&^N%!d%ZKi8+$#r(n{B z(YHEAH|Og*OV_WvY)@V7p8GUcPrmxM)*Zi<0rD*}=)N(&UWY8~i52USChNQ2q(r=+ z;9eIjMu|o0Lr`XEj(IZ8_A2aI$ptC*CTu*Kw^Hl5cvoYz%(XLDk z9?E1Z^e&f-!#KYuGkX2JD-v)}sEX^|=pFm31kt^Fw_`7qldo$+eDfO`{h_OV1+6>3 z>UMEq#qxgddNV3GWl~fE*{s>N$i7X-p%TLmvcMhU(k~xvrj=sJ=Mq#Q4C$5)b9~M` zG?5_6q8upNrdz!<#T6g3#u5);w?@~>iz_izTMiFG{T&<7Jq)lP~zy%0~2oEx?@0 zjQ|db@Z^&527t?_q{6Fefc!d#o_A@bLeOVv#bWY@gR|X{h>L6As4uXM{ik@+ke$CM zG1TU(`>xf*Rsq<@Q#&72aF;~EN^I7j0XdAx;vCplReAdiN&KLqXT`L*F?NDCQF8#g zkCC`#^KY_b!UL;7Uy`GqH0q>YH``*Jb(vaG4zB-tlPI;?A?qzNAo8oT&FDAI&P->@ z!L@jJdFHV4_29L4$wlwX*NY{d?(0W`!`F*4+qN_u)(W$XuTP%S=v^O9Jiof2y|-~* zLr&I~AJ$eD#49WB2n#T(&k5UkiZG=^(LB&m1+MHU2E7kdbqm0m0QKu)N3OAkz}Y?I z-gAd)R~SBiJ|U8!I3d1#+l2C!7^_|OsMqW)!6O|EzRLE`UlB=DF*CaXai$OpTu$`E zG0)~edSg`0H0GBE4}9A(IR*bG9}9a^Go-XG9?`AOOms)nVrE?OG`V>`eD~yyw(?A* zj%@i;1VP{Jz7L0wxot7gILh!A&fld>nn9y)SDSWy_DEa|qJJ~tY)#Lti-piL2Hf_0 z@2gx%ez_u6eGs?OH^a(k?PkIlL0sC>z7w?nKg*RQxxf)(;cWVW!O@mPB_}DTRLw{- z0-nH^LL2#Jr-ps~u~4=~K3oV6yLLoC`tOw56LaQ*&>mK12OPxa3`#FJszACtqxmo zim?V4z;&C_>YD87d)#)&i^)+4TRTrF#Y>ehovghWwtl*@&vJD$5v~pCd@l@D6J*x` zkYH;}8_zVG8(;##*fMjIa&{L}krqR-GHa}AK*{kKUgY6iRlDsZkFB>~v3?7RoO4~2m-kXNZkJ_1m z6cC(vWcTnZh`L}`IHe*PSyWx;+3M*PZPH` z(hB}bqh(91O~O~Yo*c&qVwA{!PZSS+wqzUW`rfjtAvw}~YW3L^pA##4nM!n*R7uMB z_%K}Sv{kCRH}rUbS%kkW=3|DJk9(Z1O@5CtYrkl6cH&@?dvq)}Hle-A8zz@}4Ub#U z)Np&JcL}~v6dCexfJ?g!)k*qtJcYP4>FCRgrHXm!eG#8RT^}Ha zLg8+s)Q)%yPEBLg1znit+y3=@z`tL9SkJH74E$(j@W@!JqZ!r=&cJ(|pk(CPEuM(c0dc(%n>n^KAj>Ji?{nir+4Q zNQL+0n2n{qovZLIP2zwxrc5IiSCK?6nLd?$(yXQMBc5d{gQ%DyCWv)Z5ylxBVE);V zV!jd!)GZk^l6V?1L?2087M;0H^ePIkyI+}5oWiXs04MNO%I6PBE%(u{{<_qS@qiN$ zeptv2ALZ{DL9Sb+Mta5~NqT^F5<$~4d8I+orQzdLs8*b%V<6Om77qsJRzHqQBT|IR z#f)oLwlG`O$*OL~6cRHGU2Ti$wPu#yjd~gMN%u+i5-T+lBT~1u+gM9FIoTV8Dw}u8Uud~ZlRVn8P3yx!MB0` z3w#p=z&DMM|4oq37qpLVvimca*064I-pr-){vz4ze-R;I6~zsr#s$8Q;{( z8U_xH*WCDVZgk(P%%TN2!_^)#51n!_O8cVxhPe7Ue{6`Cu?My*e#lCpjU~fuB>F(L z6h2tdpYA`;%b@D3T-+Y+-()Jy7MzkxR7~t08M8IGF~dqdeIQ_#{B!sVn`3{px4|}+ zF!?q|q{MX6_REurt;%VijVpSRd1n201GhylC3e@3Cv|XtzsugGEY6rYdGwQ~G0l`} zd_&CRaT(TP_5^6H?qQ3aK~-N~aH}R1Y?2to6r8y^QdElnu5=R@Q09YKzwn6!aB54H zR5UMoGCrx@ZgKBB$Tek8kwH;A>1wh23Y{Nx&;sWEKWk*E9(InQf@gi>)Ph_6HQLhl z`?6Y@xY`B1JDjR&t64bnwKG<89$>Ogx#)zWkExr&Ww1cdNC(ZSsj9YnJ&VbUk!~KG5Q|*vfhgPqxex{U3s&lH)nW1Y44Tg z@+YUMllrZ{7P+QC%g%e4Sa{k(=C@*48*4MvZm)~~-El<8@xW}z<}8T9>OF9!uJ8QP zdTrCsnI3eQ_gR4W>k{v+oDD^jUw*&XeRFN7n90zyT@{W$AND{s=RC4((Oc}=*x^@h zAsdkYvH*mdM#3AVHT=h@HrLA^)MeU}A3=7LJovLMG_s#kupMRdSiFccA>>>F%BrTT zC3n|-oB|zo<>@{$N_;KzdQn7*s|Lbm;8cgBgZ1HFnTmXr4M_r^IPdRsFrUcFu;hf~ zZfM`?KNZvXCD|`Wi3t3yGjXfgUl-kUujrNr(u?8Or1N~;Gg9IlL`{e`M)B$cn=tQ+ zxpZ0vk(=kqfg4a^d920D81=hsReLVmwCQ~xx!3nN*mGh6+o9={8>xvQ9{AbyLk0IBrIv0GPw_2e>)*W zg4i=hHF~?XUmgL;D(np6!v;l|a!VH4+6)tqK5`jg` z@97__9niTP>B#?F29aU~Bs0%Dy}5tv*Y{U{eJw~dU_w`Cb4q?;6k+>&^2&L_heUoa zQoqW3d=pR3m<_X1ikBV(d&G18iGpeEfPil;Ir{jzDg`P=SprK-a9*Ir2%Z^oe%lne;$i*j^?XbfEkD z-uXqMZ#`lxR&r;xsp5%)62M<+1rH7aoG!XK6?({+-ebPMdO zKJ}hvf|s(v9pe+CPjh=RHoit%GH|5iYC=vTy0|o>e= z&voSeA1|@CC-C6otvx7vgO9gby)L2FlfQBe zx_j92<;iXwi?UPLB!PhJ6Bas*Q#kja51Aj!phC4*0IjRz0eP{f1tx^T%JI!{>**|t zhY5Z|X}>z%9;1nae8gF4&>20T{>=_968~N56AqdGMFyM*oZ_y17zdvpxJT^54g>GT z`V7o3Bmwz|jRuJ~-)GN>g^mypZ_$5w!2+X?7oPu<)XWtc9E+?}d*>9UiEN5mVRZ8c zthu_lSDp8SKk=v*<}ev_yf1FzJwklt(m7jGeb$@OA;X$(<(+-~Dfo?5UpC9k7wVYf zTAEztJLU%rTf6ojtNKGGa@DvmOq?hzW8>Rgt2MYsM3 z;X1C9-Rx_7T`A>wsV*l#UfY8-=Y#Du4J$$!`rd z_bLw7ZP0mqeD)RuV+GH=rBFo$5;@Jw-EX$MF1!ydBVCTLIa7d;!GP>QeU_%As*!wj9`GP9r49K*mZX5AD^BOiYeR@StTQaG(2zc`PpSS- zZhbczz}V3GWfSpn3#Sd`s241bz(~82s5|%i*AuI!G;b4aK0R;isjA8~%=Ig7GAMY~ z%MjCp^DE5;{=9Pdxs$Bg+jt909PM$8r}r&O{bF*gG5HCfLV$dHK{d0gFA~hd+cy`? z-R@49UGdqeJJ!cHE~~ipn5`<0Nr|O)^joi~sp4_}7+5*s?y{q4~Qg=Z^RT zTt~e*^SFP1BjnzlI^bZ6oj+J<{VWi#tA z*HC>%`#J*T7&rV|)n6g=w&z9-7lKK?m`ZZFTA^^iGajNBU&|M3)=MUU)e_6N$*z}Z zs~f^DOD69Lc&lmQkZ0d)Q;TTskCV%r=&Nas>L$cTX+o|v^~IDRq|6W>W`(wmivFha zEM#Jm=OZOZL{Cl-7(GIfodc5c)_QDhFr3LyDxMkq1yD`}x8lng+rP>YmTa?3dNEIE znGFq$&aNp`QD{@zqWILq%_-vGLUPKgi3rr{fZEwal)fDW)bxff3>L(9LzKbl24(L} z6NH%Cd1UJdzVU;g8oI$tw~uQ)$)C75Q-Pq6QSSEDH(=N8<@6C%z$5LFZq9zE5?u7g z$EU7YdWM>2v?@hngv0iq;?RQybE^>6P5Wa6nU426R+0(uv41otR=SU?a%7&LCp8=| zZb}>qPtX->zEsC402h{^%pY(;O$Ibe#$GUcfZ6d?e{`-nd+W8P=8^EG1!8kxn)vqP zZakcd?Sk-VbwO8+X)bM1k?9e=!#ZGJM}EY9h0~FU%HZ`+OBg8M_wmbO{k*Msq|1(w zOHtbJo3QFX{3RetJ;YPD=4Cifsl1;mifIS9lg-IJcG7lU{?-A{1Jb%9(vli@UsB8Q z(UKERiMzyr;!D*Kx|8m`&gT6YNCh4pX)L`kwp-8*zKAnc%|ZJ1bc(?u9}`==dL-3x zY0yh9Yv{Aa(8#`BNYuCEVbcol+M!M3I{gx4QJYPZ*=~}IS0!r;`<+D6qUe~C zT8MPi9m+~GZ5jH_0!GK-?Q`e9Ek6M3X(qFtotI*CbMLoUAZ)|u_(2bJn|0>0@R-3w zAQ2@yzubgytbRICjyvL~!Q`7>e*SSQQU^FJ?g#~cs1?2vyKQVCUW@uCla@B~m3sL}kD4H41KFLg0 zWVP)PN5vsiuulMGfvjPbLmt0}7;Ar(E$I?B`Bd_} z4g|7PLnUze>|NQY`}~MPR9Fr_%4fWya2`WEK4UK8yYO{KA2ujwHki# zKXpr;br%IwYR;e0h!-uP-iT1vi`x=_oeC{1{j|*)*m#QUJ(2E zCP(Y|7A@o6!)1fKy^;^M)WLU~`CJe*K)Uwxv3VNuP>K;*WoJ?#eK)c^zx z`CU1*c~9pKXGuQQNxShDp|(+>+?;CBMLF9`Dao|A=^j-I!q3}?uj_h`EQ?DYKKp5t z*MLc?Q)&sB#WV>S;u-Nf zGsMVG2U-@_*=kjvBXu4;dbM#P#-?Y{A8w4qm>ttupD9-U$YZ@!ekGZFn@auiOtHbP z?@w8!h?3D03Ao5Lt(r+%!H0;;SYV@*lEXe(WskD!D^v-omuB(B*-*XVX^%|T&Rs`# zf#fNfVNo|hDtZbU_gKpDtUa3bdq>;XR@yc`M#z$^3()x_hn;W>l*itY}Hr z1Ty4g`MJ_u?2A9cneE$K{M%!bUdlk)sE9w~o&j*G#sR-bfUg8R;rQ%{NRw5e%aHps zvk$@H<)4@|2h&aP>w1t76Vd`f?GpkZT~@vl^ihrn^RwAfY;QPS6)1F-=fd|>JBW07 zfu(n>=PnxID-C+g6n#?ml)4TY%)A5J)uZbCj8$~ zG7iL3@8nY}tu@ggO#05^i1&@zpzi2xw;ncd_W(QRWGUV>5Ygs#a2e)D$XVcPng7&m ze>LGmv>SimH2_B5QRnU@y9L0KQkMfVZ_i%R^)cRFbsb93{cIDjUZossL(YQk69+S~ zKwhNRFBb1>j56w26ciDjFHGZBc<#nQe^<^HMd~y!MmKfN>G`ZY5j4`1J4^e$HsFdt z67m%&YjdlmT`}*i16N4*9({`Xm4LaIEcw$Wq8%cq`D3rGz`wKS`rh`CM#^6%R{)zf zE5Y8_8^Ud9oK7d9Qd!UVJNMEf>I}p9>WAzR8}h-IL|LP48$cxo&UGcme$6$;PF@r) z394AD+J7>uI;<7T0pl$q?t)lCgR$z%uMdWSNXqqf@wmyjA5?2G+C9ZnIZgWm`-%_!w2TNrj+ z0*?!|f*aJQ$QQSVLQe@g<;ProlMhN1K=B@sgL_!;RpY$X!Kx10Y>RI&8ujPrWAm$t z;s8r5DKB}wZmqwRAq;*+gu?e4q#|a#N;IqrCzUIJTR=Jel&k%*Udtp_hL;lXz-h&R z^S)I;^R%9zmo4z6;XFqu970``-~*U@M+Z`sdoI|-d47EKeerQ+EpN_}=?N1G^ByBA~yHv8Zg$>%XV z+XMY)tlJCr_i9g4z4wOuFI~b@4{xiW8&Z9}#P_jC9?!kC=&12|uL`1ansW6~&Pb0! zo=iXA7EDvbE<(CXpb4>mwa6GV3?)$Pt-C%dPSlx8babs_pr~!%({SbX``qTyHvbK` zL3HqM#U&vL*y@Hh79`%yIT{U^P8mt_y_jU(XjY!Fztd2o*t>u1Pr&%)g2foMTZP$8 zhFa8Ym5}I~c&;!0l|lVe86UR<(fsicA0@I@f9p2-AVdJ3;7JC zvSZCKgp2ST?R?-3lfE6cCHfh!lG%MWF?SmOGMuEtY@m;5Hko(2Pqt7y%!=wvJT!#1 zh=xZ4ai@XcIyIS4RVL2Ef->u99#W(d>D*q>I#eC5$&!SF*2c+3IBtISMe64R4|r2T zyUj^G_v_=Ac4Y)EjHLdJs}PzrO7^CN{slJffx~_Gam-_A&fkaGx`3zD)RVXHw$)b` ztJzT`P+V%FkQo-nR&1^JT4oYNT-Kw9X$t?&)?ww^fL;v1Q_~7se(|CdB%ir2&s_`g z41H!fn1lg`n)88H9V#||(%wz9rrU^wQ@2CH!(Dd3+EgiB5Y{lY>A8*X>k-ip3Uuf0 z%ltRHiN@gxnD5fMuF2=qG+V7hC~zu++U-kJ4VzSIJh#z6DzjZIQqAbiSr}&(5SNMV zdlp8s)Oz12U3v;7ce0>1byPM{I2-ML$$W9M8AmF#K2xd=vGmrYS`i62(MSNf#tV0E zJH%SZ6~sMC*!)ckLA<6(P5ESS;yyb+!#GI=+~i7RiOlm`vHo2&tJAcNC~)CJgqd+>8c3 zGaU8pqhh!WpirLMYv5Nq(&_w241FbGr7C7iD zUfFu4s(Sh4{HYLD1G#gM7aABYnW^FwwaL4v9W#Lp^#oEiq+=-LIU<1X^}?VzRzq}Y zsjyt8>^#z8M(^sF+lKdry{6 z_+I9=3EOM>mzNg8?|2~gNK*x2szKtmS&{dPOUXNoQrz?R601>|1!hx4sNG-97tuFDm#TfVk9)Ym^&kM$*a-TIoX zv{_(?CW&|6D(eHl`YhMj!~R*wK1yN|(ekJ@gSq!gS%EnetwtSiGJEFB{R2SG^{;om zxX0d{y*4spfbDwK9K2anT>X|-bzx?7wvDjD6uFT`})t;>chO-!4 zr#ok;1g`U`YR8r_EGh_hYDpTm%6^ttf}t^UkP=u!2_wZhE(rrh1w*g-Vw&a;+f{yy z6BIwK)}42Le=GDhsgP$m+JZ)Z@O()zfHrf)Tpme#lU#fkGXxi-$u@+I`}$SOro(oX zPo2VTzc902!zn!)$4ej~fw#`L>7CEgqvMLSIIf6DDs!Gk7M{@=S8PS2h=Lv?Q%+Bx zjwa9@9-z&_Xq!VeR`#O5RM^lq#%&P)4|(E^P3hR|)EL5$+@bLEkZcxr0mJo?wBYc$X#}q^R@oX$MFyy9^Kw zt)H;lLhLe!#&t4~PnJfJ-UkXFQ{5~g<4Po8+*kHL%Cz<6W!>)u&1=M=yP6Ppv}XF+ z*DIC@t8#(YF;yGe!-v26X!n0kBhYUZihtE?Z$&8%YW<4CgVBAs?Sk_;g`t82*1a`O zfs9YVE(~|WuSDMQmRVh3(UGkUlm~PMw-OMq?h|t-d738JVlLcTLflh=V!TLW0$H|K z!Kq%ix(FAP^Ih%&yX2?nh4m{-7y@RLa~b$5DHi#xg<0=kGJ+@{UghUgKHKjsyS2Zc zx}DIiGz%Fv*|_IQT}}Wu#k@2E%!)+%rHu_QRhKuJhyYW?Jx&y*oG}HUS48~upae4p zSb|aOgsjJYdI<>gi?IT*k0T$cA$+5DZz%u|(!IuV`a9m)z4U_O)B3p=N?xIhg)A17 z%&Lnq$Q39P4j@Oi&=#j17cMEJQC{MUe@hl8lpA>8>92$6P`7;^^h)=qi zvmJlntr`fu>AV`1mb%ovUy@(9W?W#Li+haK_S*!Uq_3+ekj#$tHCgA#`8%28v&wpc z3Is(gyPg}elm@coayh%5rRwqi^WwJfdf33DOm97CX%&_1!K}sb%j15#n9`e*a2Ipn zy-W&l4?@G+asoOQxfwPKkvEUhU=y6Bmmqc?4Gk*ShEVTMeHaRVKBoy zdDZAkP#CU!m#@z!4pGlTp}%5={bS*fuWrD;2L#lHf_IT{as<2(S{CmG`9_3;RQ-S# zfUND>7oL8f4R2;lxX%eolS)YxTSQ#Z5u~cdd@;yo^9kA-ygD^5H5nFM6}+ize~mwi zI2h}{Iz+Zze+~^)X%r=h$zt1I8t((QGy_VQLyBjFJvZJEMHFYsQ z=`-Mg!Y@d$z#}3PUBgC}eMPX##Ob0pco<=bddlhgK|)^@5M=yHITyr^c{OZ}QdekY z`l#nG6I3;a%c-S+^$7jO?@5IfU+Uf?}~#+MJN zneL2w^~6y}hJx`l&wRyxhIZz0*-BPAr_tcWWy^m#kH3h0zZ4C*FDX7|2F^re z6{=wKw6j1mjGNwi2=lWS)u9m{cfpKgM6pAVGIcO%SAUO2?S6Hj6JsE>Caxahwea+a zwRn$$uu2%6f^8&UtvOm}Ue?n3&_PQ%;)!2g_D1`D36DRBlt4NQz=>a+G7b$z* zB~dSy_wJtE7h(>Z82I|!W}`C}_9Mt=+vyP^(T~f=p#Kz>={Nz^0O0&mXPK|sj!G$s zS*EXzTiW9hRe;jj3K-aF^}t1;YQdk^8L8O=RMFgcdSR4BGj!RbaMAH{=|F8rCJA{S zBw;0tLx)R1_gNaK@SNN=<@v%m3d!-cqJO4D8A`*XWAIuDO2e#E`0}Ajd3$@3{6`Z% z;8wbm&tB+tQJK;ksjnIXCLnMU4s*7cyM ziOYv?L5TuLmzd*_WD&AFb3*NT=3CYTSrC*yI}6~g)-lX+T1_Z29YH=Sy2ZMjwc34I zi??6!Jy5s^g*+%k2T@=qhP%=L&32F1=NRek1m3IE#HpTkRZ zBOqg@4Drh4jc83{s3#6YtBvWmc>3gP>58evT-kEt=zRu$Qosp&)hxZM@V##I6LAB5 zE~wZt`?2?ZnJ!4Wwre-e0gb_Kdmv%iHKE!}Em^<7s-IW!{aL>qlf%sMP_;V45#sZ~ zqa*uN*9#}<-e0~+b zdeWKb?dK=sCZm_@&92v8B-fIXN$D>K3Rlq18)rM6q^v&y`9!qi9m96wdkM6Ph5io- zjec(KF`SVI^|ThvhyFz?=$>PTvbJn3Drf$j8&qzyA;uF_1fEjKJ6)xF3StYbM0Z&6 z?|e25+QiM#_NX=gp|)hAV~nBa^3-ADJe}X6!}7a|torb(KOfD=>(PI}RRBKgkLvQy zR?zORcMjg2zY2=il{J*n>hjzWy##X$<0d&C&20(cGQ{^U_2>I&?RQxaDyoO*yX4B{ zhvqbQmv{&h*|Ow6A!-YVvhRAgdhn^PB@}7Yp=SBpU*?tXVs>+DmTP7_g>FVfxVA^^ zWIMsuUq&9Y7cpx1{k=^?4TK3CJx#z5E_w3bESV=fKe|HIjbPSS9x|GmrHx_7f6jj8 zr%gEw-xCS2X%D40q(I3d_H(!$D71f|NDeTo5Cs`y1W?T~coR!(IF-*%swXuZtquaE1~klrCurnV~yG8Wibn zkQx{Sq??(ch6X7SrF)PRl#oVRdR=(m&-;Auw$|FV@7Mf=ne)2NH z1FRcb{i20AzPTL0@<@YPfD$HR7@MwR`uT!*+Au@OJdmU5z6|FgEBwWe&qkpx!?9j) z;gZsE^=c(nFhgX*|%@{o#LCF@W#Q0<)&WqdZfJkW@mMmhN2`GZ6>i*}R) z=eEldRhx!9qz082YMm{q(AJwDZsqR8v>}7WG6RnF*)%IohZ6iU=8={7&%HoFWRP5h zbeRcu&@s&$D%QXxUusXTTF(sJ(m-}_KZ77NALzxEvzIo22!S$I0wbXgH(3F(ey!}C z@H;1_dr%WB*wS4I0}lpOC!@}(!cZKm9Ze+SRdx)A$jeOwuMd~UuezaAEhW8L`#S+T zNhp(h+`2+h&voBevS$T;)VbMq82(eQR_|O1N`w%^@B}F^O_Gh&jDpZQ6?*JQ2 z2o~52&yrfa`x&_~ZonToWRA01dBeAY6h5ZFawer+8G?=p?REh5o8hD`y*3wBpTF1F z<}^!Q{DesX|85hsvO{oFoxu9j%hk4T=KC4`t>lFfI{~RzKB6uZ?<2!yj)yM9daiCl zz}-s4Dhg3xWMEiQit-mhm#~ni<>CxC3q)^$r9S<`_CR;mk$3M#B9d_eN#|pe9iP#G z6+8S+Tvi`Jvxm$#Sm_-J?+H`)f2|40ga+>`Vs9Z2OAHI=at>%hp3s|4-pq4?ri`VA zoAFNWlp_?LC1krXaag%lCC8c_ndlrAH$k0`3fCWr{9Feg1?^k8H`4Ygd#-!dEMXdJcx$U)S`71L$ z+LxK{pS+gl<5L|+kxUdfzgl@a&Vnblp^L7B5+V&B$jMe}`vEvf>w5xNkL?eA`lI(>HJ2&#reR*v{A{ zavw+m;y&AwLhwKkXQGnlVcgKokeEU;tnqNr6uTgzSJpStTfMw~Rd+;A(-eOv6a0nW zILzix-ple|XOGv@3wASRy1&iVJe4rko#23o)ZEhfh2w7Y>$^GAnj*KpjZT%;ao@%7 ztHq&C$$#-yn`;O5!!ub!uZ#=L3U61|ljgEn@vw=l=%#~kqryV`((%)1pkA3xhb$5r zE5zc6n+|Rz?$q{%mG1@)ccMRXY{e+GHRqMtV z(_6!gV1ARozByIXuk3RA>E#98%UZRWCE?@Ur;8tS_2TMO$63s^_doxa9#OicN2cld z0iPcGMfK$VK=#?F=OI#p_G9NwfFF7LXElrbCZFE-{fO)zAu@DyjqCKks{zsaB84ar zRhjPfI5?O&^rcUf7o=*Pfb7eB%hKpR1x*TXJ2|sr$`Lcf-DMKjHV9VX0dTLH8yGRY{HK3@l`-E7`5Zp{rkY}KKwMIa0=%@yLG?qBD zzV4yhslSTbF7K+~?oIM}6a!w_17{7O4!(q~uLF8BT3w__$IBp|ijf;plx6A8lh**Z zO9D~*TLWM8UTQnpfcj16c$!6b8PA#14s(S(dg|K>WwKS#;U3B$9WW_bRNdy+yxA`q zV_~5nT>|`XZiKS-ba@vtN9YZmiq>wi!|hejI??`$MBTq`8-Z8&@)=`1RK_31dYk9w z8xE3Qlo1Fhlu(_#myFRt3t-KO4^o5Ih&a2<%ss=n1D*DdkmP z{Q`VnfSXyY6bJlMTW?mzn1}j?iC`M!?=xQ!ZIT2nKChgnW)85kz^;3{{A%$RMZ#)- zME~A=4gKxn$H!CD=kiaxqDE{!5?^hEnmjo8^Igzi#Z*0U(@Urdhzdqo@*?ysR|&Jd=# zNr4ltK%$t+2rvafIBP2nR4WwP?>%_qp^WVt64uTmKg5@f5BtTzcDIFOY7?YH?m=7p zW4rb+%(Lzl)lbre44G@&{%!?yxQ=X<%wQOYnvp|{RUzS3QgEsp&H(_Y>B}5K$wT-E zQJ;lCt)WT;5A}RXi)Dds*Ru1c+O6aE8K0@(E?wR#-||h9&}@=&PZiZM-m69dbuf%u zd5ZPstjG2pCD@P*#_iXiBgxkGYcZkI~OSatm_;?i|Z*cMp=7d(DdEL{Kx zv9vg*WU_>A7@r>Ab%O#CX1e9;Da0EJOes%lcn4nZQ-Vj$q5q7l6BB{uKm|G8(^|RI zy0wtA3tr!{|Ih%yGr{vOr&IU;!RgQoi~WbwsR?d(<{KY{@lyQ=^W^#rY{M z5Bc7Sv!vVN_b6h|lwSE1C>#bWANW7x4}EO|L=h^%P-7@%_x+&P#U`J!Sd2@ZdfqSW zzWM->^oMW#+YaU_jDMus1=tikDvkrHV$F(_Uz!VQBbG;xx6i%s-{S*N?G@X2!29$pYo_T3Y=x`_9y3eMso^Ms# z3XG=d*x5iU#GHA}+TGk28kluyOx~D4o)sq?OZDr_f8N?|w4(q7_QdXX`VBlZ{JWgE z=up}kwx|@Ce~6RBJu@d0MTC}d>k|7>wqfs&6i8MVl+-wkqijK*6&4>a?GC{PT-u>U zlx(92=1vDRj!^5A$CIVfX$FCpHIK;haB-m>4Ys=p4~GO+5z2RI($K`e1y&>Mm%*Qr zC(v1(O2Yo3JflOc;%U)pK@Q!E+s8*w_dzzba_Ldw5w&6r0uKLiK=4rHkoW@{t(m_X zt$np^q}ttkmWiHGgbx_{ z$fY!08<{dR+2hWBU3s5XnmZj|=RYal2`UQtvlVcBU_GKflXaf+o23)>!wyYCjXCNK z&ZQ>s`YOmxS}4M8dBXeS{b$KiW2j8T=2V1U)!X3pyIR|P7EI3mb*+oJLNYVtX3Kxu zv&@LZ!`s7#IJe7~kCw&OL6&`$b+FQbya=5rYG}9C9}rp z=*AB7tFyiQGi+etKkx{!X12S^1x9=AHb^8%3=I<16q?i~DJ-;jw6^UJZ2fo#^bTz> zo6O5ai3KbzSNQpP&FXK=-CpBm2>Y;k0IXTPH@dFpRa=~(MZhK~kKUUza9~Th;iyyF zdk3Y7e(Ii2`}x;I$mDVdmjX>370MrA5_I*4HwdmktU>IGNQPsvAmYQ9%uYH1uwNe! zJGV`2-CZ1rx&b5Y*J?B^-zzM08|3|EPDR5Nr%)d zG&sifmOim;4HY!j5%Z}xZ{}9abFIOI3k2CFUNsOqJ9Z0DA`Laz;goKTNJllReYBzA zrhkXgOR$=K@AwsXLoR5t3W(hMDB&$9Ni)z#Dnm|?hcHL6yaCC$9d>q27YCW6P3|P) zg>dlsSl6e=>fcX5v6Sc0=*_V-|48Q1O`}OPzS|OB_U}pClmaB*#?5?|+i#2Olwi?H zk`GE6v0#cra22v(a21LMgM}#3BzD4&ktKk(1eb8@PF3_r(sc>9W21NXoZ{S{-0bA# zfJDGS%`ewVX-?2FyH7k&sO^>-(v{*>`+`BgKUyY-Hv8*g&F_0r9=f=vo|4Lo?&~s# z7rt;pKbc6_hfQxCWjFq49|J#PqQAu5h&+&6>7q(A)8qr>_1>Dg;H`IbLu3{4xtfzWu^aXRu4~w3ja3m81D3R}1v>X1vn}+8>33 z8tVz(P0==*B+8Y`8ouBKWwWFoegaE@;2WQ*+<-nYhIMGola$Qn7J(Ea>#cVysx|)* z4&m)p(hG(a872<-jE*!xUw{6p+K3Q6E(BUToB4h}kOIKCq;Bv?0rYBu$}UGLyu#T4hb{i?W{fZzVF)H^MV(m?d3W zZ1el`{8PH-ed@ipB6gbRzf%3qe$$NIxhLX=_-fggeHgJEuvck#ISqJL&q6#d)vuh- zXACd+W#B5rK?R`-M5gchIh-i@l<7Uq0(6ElBI-b{acpIy++{!~JIrg^lCM*Itw|KX zHuDZkpELbh$ap#CDwQ2|Hpi3Y=5@fbqj<`+{o8ML#ivo|Oq+Y=nV;$D_mAsz^ZCJ? zrzRK2VxDmwkqQ^a&R)VZ|MWr95|Pfc%SW~~#tcaumi{u0{C*_Yk)NHwn^PwwjiwV) zuF-4hbtQ&_id3{rqFu0IYPE@tVBTOikf1{&rB3l^)B4!~kA96>g{%scu&k)C&%fPE zlXBt=sH?kH)Z<)g`M(6x=ub9lBWUcW=nL}$o>_t#m^B1h^e=uER4B}GproZrx7h^N zI@1Ou>9qX~-6(aex@VB@k0KyyH24$(b|2o3ppw)Li|l4_IiR|cVgR`JKC4vmT4Pze30``R>t z^GUfpLxjPu?Ywyu*Kfaw!~kytYgmgE`s^ZPn)zR4m&fJFVseC+TiftOB_Y7^7!o-v z9yp`=J9ib~;de499;n~<*3&%%eOG|ujA0nCjwk4GTdqLUg4$LA6kh$`!&l_%pgm)>S z{9?pymJamZwbfv_%e>WScf&7AanI^5(UU$>jD4kX zXr_Gv%KylSso#<}QGokuA>Y4d&;N3|S+_61|FC7 z+aVI;dS8mQb+?XNr?dU>dx|QRkWcSU1Rn|T#7Ntqjrw@Hqn^FK0Z^L0cgFGV@aPfi z>pn6bN5P?+IGx~TTIk6~A{fNUYp>ps^Is4d@Tp$wfNGgv7;-@7d|o)m!qJ~IwKI#Z zN*d)+$oHRxC8|_x#05|#zw**`fXZ+;sM38xB+TFMP7jiM5f=`9P>z5|jM&9y#IfW$aMih$A0XWD`jHIJfq8sfxs_`HCU} zS-DP>UX&i_3kHu=vzR2hwHs!L#(Av-!w|{`F~^ClLtkj6saukX#;{%4X&wW;Kz@XW z$k;~^1Uy~7+>Yu)`D2kY+bL3B@o7u@e>XMYqmmbZ^=o-~BAi|t0!y)kEh9cEv%JxN z|MYorjavq3P}Ci?L8BJry5lLFXmhCK>bKh+W6CN!QI>6D5j;0642yh0;VwU!Xon54 zU|%P^ST$`K3)qU6?7s4v;19_DeH6tBl~`l+YS6l@m1>>7dMVYEcy-9gOj|*;3^h7vup!9i7 z50hp{7l!aal`xEbilOqegtCcBmJw$jC=@x$N}!!<0=1T@o%lt@7y;d-fP?T_e^nB_ z`Ihkl$lBD=uhA8ruOj>H^rf@zFYrDof`9Z#X^yZ5_`?l5;BZ6{YftpFDhK3&o{$v; z_k&E}3PK4sZV~xNdqBZFIoE^}4ni=Jz~Yn9lOXnclo1cUvgosNQVz+Wqm3D?``Pr} zje>mnRA9>vH9;7xPBD>~GNu2@8_Gy4C~BenZ0Q{e!Z(|qlj{S!GOyXm@k4T?9cIK! zpyFo!Aw$jOq>lm}=aNcwYr{4#Q<|^Qa|685ak?XOg*CpLcF@UFtg;Fnt~VBZI)J%+ z5rdceathzu0-D(&Zx2W-?#8T{(wYG7Z-_6R#1ABQHRirr<@X*w)_XwA z*Voq-aX=tYjFJ24&pwUN5@6*({?E$c3t2*5YtFtJe~Y_@Z>w2dbgnrXGA^=Y{;JwC zJLs=qW*6tp8@lChuO`mERo`qeIT{CgTtdMi z93D8kvFQ$o7&W09O7wA$tU$%5=PjnuFrg6u!I53I3$;b8=mp7N6yS~6*+|#G9dmH+ ze-};&?0FRlGsL3h^Tv+LcKjY|3jQ(H3tU>dwA$(+3o>RGFvvC|JDV=QyL4oF}!?kOGL8I25K z1Wg3iqjrro2e#4%Gk?dWA2McJxiuROPahB|b=XOTVA=G`KF(akD*uGQu_e)nUZ+i` zAJ4r&5WCDEI7oGhAX;yk7gx?Ts>L`*33k994jeswGG&+^p)d#J4f>~OY$Du@tXX=1 zzz(Ke2iRcNEomn!Z1TgbBxnC{#D8bxcqu+2_t6K5ZS0*0T(i3#e!2}h0H#AOCB}|nYoIfX$CbAHe{lwypv6TwIJc`$_jy( zdq0)^j*`x2|7+%`$36T1vU0?WwSN1n4b#Wp28o|#u>Phd8kUl98@tm3y*1hWgm9$r z?PHy?(-KCz_6xCqhRgczr&`P!oX4ijja-uWBBUeh?}D)4EN{flryCGkX%bu!1RtS|O+;yte&@Y2r4 zfF-lq6E6>Y*Us|vl0gXK2i#zX5*ug2a4hQ|*GXPF+^_gSGL?t5!c4!;We+;+wG4d_`oiY&6s%DwbNGVdeF6qwXX^r%=t09x+R*9^vQ94PL%NoE;Lv=#;)im>s7eis%9vIVJwD9T4g zHE!|X-5N)0VuEcs``QQF1uGv}&~po+4tyQyMUghxgL~FD(n53qJX zJK(*hilyN>%^9o9kV--wH@I73N#?~S7Vt`T%5LC{ZySgtCB4bxGN>vb-vXh;N5H_2 zo`ke`4~o~vTbVJ$JCa)Wv~P!26NwDrF>eM&c% znCmv*%}_VHNuZ1ao;N;;PZj(v)AUZ<>uliUsyFHHmHNrb-96^yzm# z&6FU|_db2?rDwI)W`dQZ$=E-tFMG|RIT-b0i|uKH*P7q^O1Zj@JUP5)@=O2Y-a}RB zd65mlKbs(>3=nP{*G=orQV+y~)y(S%WT#`z{;{zW=D>>5a96qI%P zeU{mB*$u4)l??v8;Kn545qKCjoaLs>d+d87*4O553 zA2fVv3TBKA&oTXmQmG$-0U5Y?u7r7TJ=pZh=6nO-!zUQC^MJo_e@26XnMpVh2ukcJ z=0qgUjvGR``;y0+S&|Q;-!Wr=o2B1e zBy`L7iT%bhf0V;v{>D^=zbBVZ7eo(Kiv%uy?Yg~n*bIS7Dd+7u6%AY3th6>v=YZfZSyreW2Ju<#!cpozsk&+LLSh@Rp)d z`+6Q0Wf>laSXkwi><$QLy%O`0HdfSHxxYb^jW>=GfLXNbtwXDZnm89>`t@lJZPr80_8ml=U$*UTA71hc z&kSK4lwj)g`6{rVny;j+z{&~>tuF}h#g<1xo>_*)koQr-K@2c0dKpacb51x^La~Ym zuk_DPA~=XJ_^v_^i4#J%dJh)FVPQs_+W6R8I-tXla1fgkWXx$}MxR=w;Cbvs;}3vy zw_%GagP!no=vaW>=)hS0<2*#*Kc^dVtu@hWOY3}XstAl~GF7ZEqLdUqe9~Z@ElvNz z+lBnXK(P$=9wQ)&H9A|BdYz(MNtLMb8O&?14Ft%R=yl=Qw=1cxO-iN`eoeM1}jo$b9W7jF!OG%wpX_`6+<`2LXPVKi_L>uQKEQ%+DSa z)QynfQk0oo{dw!UtSEwMyKezP$WS57gGZpfp3m3_P+O5OSLvT#3O~J|1vo0OP;>ZW z2vvPzA%={41et`jgLw(xQys9Q5kfuJ4$bB=bGriKK{2D*qk@`k**~01KrCp@>pMFd z7l7fXMf-%%{&$+U2oSg&>ipDCG{BFD zrU~B>6LSUvbb*1y>K5InPd~2=O2IT$NbXY(J_R0YP^5xu&{O{h7T8XXf+r)ExBDId zw~_s_(U(?i4lJrDyJfd3IZqc^HW8SAvDIOQD>w09&E{9eAmQm;e&3OC)a|p1mA?|C zgZ}?5LGl75NYgq0LDc_W<1M?(Br!}53zk^tZd5tC?VRQM0~@B85_mOqynP{KLUYAf zsv596m0LyiHY6|-=6q3kB^KbFfAx*-$8Y-8SU)pP#tzCns3Wsp5eN*^t`3%K-(}m9 zs{v~PW2~8iAiX&6ig+4$Pjtv$2<0ylqgzGlfO~&NpD@~&i|V_ri0s`bFvA85~pb#F-Z4aaPWmRB{ zO29y`++!yOhYDz7;Q8dsEN+|xkP;|GF%tPsuEHR!k-U!^S$Xr_By-orN0uNwA&3$z ztQ)3G&qd4u;lYc^aOc4jD~~KuxWZ*uUOE0v&h-kDW|MO_ zqMqm-_SkJ%m`wbi_2&h=M#w2l400bL|vp3fBW+76ldQ!WSrg zccfz4TEOW@q%}($p1%CLM=p!<9hQf1Y^(3qTS_?~a@&Q#UhBK=yu$|T$J`F3AEc*X z%x1N-Ld|G9@OFU+ItYCwJzWob0EfyLI6c+-{wvl%+}Dcy4}CV&ko*++2g3f%3`^)E zvS@Ay^t#9d){W*i(l#evC`xlc^a;0*ldZRZ{&#vY@=3NT+|!Kv9S!@=_Zv6`c{w^k zwZ=K3U)c1u?#b3f=iLY~(kIa7{h5q%M}DHn1HFD{ZN?0RaFlV_)b@(`??hi0j@c;JCQxQBsVw#riyoX&2U%IJeIFE5fPEGP~P|EVGCEG|a-L z7JJD)wRb}oZ(uCL+e0aGn0>g5i&=DyNnn=)uPZKoC`w(@76@L5MC?6yzN%g(!E?E8 z=zM(jBfgxrP5X8v?8!usb`A2b9Sp!(rmO%Q530#YUw{I_zIo|yDWA3&45+BUKq|0% zxLmJp_mFU+z6kyvdl$`ZCKz+^b)1FWFWG8Yt!F*-5_hW#$hns_G0jAJ`_bo@Fv!w=5X4=f{RK4B|0LL_@-5FeC*??+Sk>j z_(@Qo5Vq6T_l8F4?j6i^@=hxPP>UHUuDpBeqm|!Tw*kL)mYEogx#@S~lC%;GcXwrL zQzQ#3nBd+2Em!?wf+FfG9Gw8CXaqnEFh9^66~p?XqXdqpS@8KC+>cE^cPf&>3BQc) zgY;H}9!P;(UrPv9Vo1h|2dqQo&>&z=D=_I%v4gd?NoUL0!7%jB<7lRYj9uc3a50Zl z^L!iktuJAfYf_KpQD1~Bl_v%AuvMqOUXlYPom%)v*eOK2LeQt3{WOVgwr(1ssNbTp zgshss5qn#R+6YiET5*n`p@n+Tp-7zv`caMpwQ#7(l1UfvWw*#Gt97+A z{2uhy95~M33-QrHSd!^+JuCyXK#SnRYR1A<-X684<=tb6qukA$e^zMVu)LwjDK~Lg zbfW#g7=ABzM2q@wOn&CY9dQ#1#(r*7hYVAA-VN$h0;sg|(ZD)pl|a z^dmY|b-F~~{1zORwu!sHWS-37W$^h$=CojC_RwcWYbHlv45Z*mmcGC3kf=TN?T>SOFCxu8qVgMt=))T)?d+rri-J~7?TiwvZ>(tWsgoJ!ys{G7E zaUqp$4mrbhCZ8lmz3VWwo_K2Ff+9T^4rx_?R@O2*{YSV=?e9W)Kc-yerA>n4ljND8ne z4UH0Z*cIEC&wp$kdr)@wCEETcJ__bEPI8?t`jtY|0U!0X8+r+}ut)Jc;?Oldt1mJ* zoYeIa^{{F@o7%ZhKCYi{F*%3h-pAJgtVVarS@fX_fau8KO{sxQIaLps%}fw!g9uHm zC*`oBYL$W%f`5`Mg`p8u<+JEx?> zm%RwNJ&rQ5-za5-#+#p5&4OoOueW=khh_r&ZJ!MB_XH2{y6_C!Eh_I$0+D8vwej3`u&s~1 zOc?N_6++#9#rP_kWcz~cdS*ZBERApaF_hKUa`=%-sJy&E^WwSU)oYaF0@cu>GAv|J zejjh*e)TEMM4PR2)iq72hL2;CQ=wjv<4E$P+uQcTOqFr7y!1q%BT_fY=rAc;^=ggJ zd81ULIJUA-xGYMC>_@cZ{{2I+KJHU3iflFb%meC5)yf; zn&jmRv2LT#Gv>eS&Gt`dd4A5(i$F-GvTHPG4;S1w=|Fs<5>S(`iMD=F;y5vKUjAsD zI7_-J+L}J~(R27q{aG$EOh*(+3b&>O2kPP!w)Hf;edL)LP*YKmz4FjUni84Xl{>mn zsaqc1D`-X0AR8@*_lc5p>}h9R-A%O^Cf1dKqd*LwfKR(}fFUf4wK9MRsFJ%h_nWME zEOO`R1O0nT2WUH(hAJlbtvs1t658d2c=SvUfq8~(D)oRA6|1-I~1GVN;Z`5)VJJa=o! zbn>1xy=A2sSyu-}VuJ@GFCinn#$HAPD)Wmh+72A3kaMk>TCyO^@EbdbE@39+qd?(% zj`-6Yx?8k@WNApPicxn?ZD>8ixW_XW170EqVuX!GIWh_E`_-EqbU~{QhnEFK;WX$5C_xH!YTq4&9cr@h3uDkWKPr{*C7IWH*Wp;1Fc`X7|%d*I| zuo`&vwQC7M5>ZyndWcDa2^foVD#-xjR>YzDw0ZdHNszk#N?h;D8Zy?;sajtWHFohR~)IE_ge->Y>Q|CCiP$}rd zJVoFAJut0fUWs%pNMhmX4{I{q_-TOVah}}7nfdb_ttMv%!8P^U_`Wycmoy0=xg46} zOxAa4;MnRCeF?Ij-T*^9IMC(@G}3dENlcM^ccqt>nCO2B67mac%Fgk}yBD5o&GHlG z`baCO*NVF^tj&t0Y{S2+u=}2Q!arP;+(#I8?wn2{=REx#@`-Lm47xU$Jq*f{JNm^ zAN>2X@4ujQYY*`Ov#g_WlJL5lp>;e&ga{9)I>YM*Hl9233z`(@$3zktbOlEn>~(<& zs7ubdUu{C-zf#LeWUAaHZQ@0J@9Gm$wVxabjiadr5*>56^mDBgQVGta9q?IUPzA)2 zSE|YYxFR}PR5e|u&h3Kj&<>VRHhdH}#u57n_r6zFu8q(=4a(YiSz!vsof533(^NSwC-rGv|b4S2`rUaHvzP8 z=opvNL2bSai=Hhs7+44BsZTl=vofZg0A2G}x z&Xj9b3mYUmodEnY@TUjy*@bqYj+=Ocm#);mkMb1(HhJLiMa(vNh_iJ0sp=i? z?|!^R<`#V~fMq`AONg(tVj(@WC)R0aqf=tBtH9=P(4E%k++nYcde{z6mxjAB!FwgL zGc?rD%#_l}JRvlZn%D${lM=M*|Jq5jK~Qdjvp?}|X>f!VRj8KK>Otm)t0}(9Y`CEZ zzO2zNQnA5C?Vv3M`CMqjC+Qwe-IMtlYv$wMLF9Q%=Xd{79*fcKwto3dEA<9C z6%1XPcwcijNo89sKdLGAG{@qYr!j@;;n9RaWS{r-U}BTQVjyPSSte7NY}y}X&YzGW zGjuJP5{);WUK-anNPB(ko2A-c9)V0>vtf7N;!iNaoSarF=L_fN{n=RHScue^>$mck z+bp0#c?=u_kB8I^kA!z~` z-?+@D?$Oe8^j@h#4v}a<5I9n4)kZx+B|K~74WBmj4x*ab;u$RnB9BFl386)Nw}pFo z)Yu4gPk6i-luWeNj)KkVESgEYXNTTAVYKQoXxvo>^$&zwziC6lU#RTlq+fDZ0K$=X zOz- zM82sXxWIIP%Dz22wHf9}>eP+Wt=it$8xT2}lyH`GF#H#s0X}{|CVK2TTKweO;1f~& z+u-e4U?%3G{^o2lzm^P()3p|I4TeHQ?82_Y`s#=TCVXA)w2b{I8LV4E_Fu#}p3dF$ z=oB;3j}1|6JKwlOzhu>xSQ7-lu9@XOAI*QnxQEqZGcGHm4LF%5`z3jm_J=v&sF0F| z`LTIrSPscUT3L2{fSSNJNX8JBM(X+k`f3Y0m`0=c3m~AKs&x;V zjHKXauM2+=)DHb+g~y4;@fk)m0h5I=yUUN)JwZw6H;MpES8qsg3u+^lh_-E%gO8>~ z!}s3vVq*o+6h&rQk(Pzeq>^4jUy;csBX!}ttIP?h9m4%QRF8t+F~C~z#BV;KOb4BL zLuoMb47B>UGVw-8(q`GR;o}MhG+yKw(1A^_y03n9heY&z$wctfb7=Wb$atG!%88Di zwMQ?qElW2$Yk>H(NlrR5tHDS`&Qm6;GLs0a+JdvzbE`a%$QrXcdEtwHmLD@IN{U?F zYIQ~0pv4jsh3qs}iKb8I#nTzEb{3hatlCJ}cpB-2GB)}hCq7Dr6Wqg8$!)Gj>r$lZ z#_x)s67OPSrH6Ze67OiubWE1JxF`=pi`ITyVDk)!ruaVf)a%vaJ4U4LgP=7MT1HM4 z5ya8RMb~8JzbZ{STpzhWr&If+?HZMLpNcwT11$y|)qa^=oK$rIX&C>NVp~H0100`E z@wsj&Yh2q`HJa2QL%?uWr?>#r0_*g5O4`YlYai`=a-{WK25Xd0Ulx(~eXk>iHhT?* z`(hKy%P;fWcBK3#iFE-!=wj{Wpx@=|UB+{c?)CdtpOjScZV+|Ebw_BcM8O{V%L))$ zApC0>WJ-}n1`GaeA%HZZG=^2bk(UFt_Tbkx%R5h)??Ur?l|=O24hnzgBCFaD%xRhS zM0XwW!`%m|hDK7n5iCtP$5Vl3g*---XuU|)E)zn}AF+0tso}zS9i+E?`~ZMJQDL|z zMiLAgld0eQX-EAgWc{{<-&khOQ^jz{us#tTtU z?I0#V1DnqxJDcU5jbs!A6(Di7LGF@%_KD@Iwp@>Isfq5IQJL)LtLCR>kT0 zNx{be5wv88H#TgjTxa|T$YBlG2N*7%l=yvmtnL0y?636l9}JO-A_vzBX z9ndd}A^Lbccq}sC9a;x;GXX=qpRDebQGcK5+3DVj#E1Hc=FDIDxkqO!mP43BKxj=x zTz}259^nOru<4PVmUQ!(mOJ7wx?vTMH3E{h4@8a@e;m8ThU*9+Icnd#O<(rY`mcSv z;Cx1`3%UBeQ|~t=f#8Q@n?;ZdL%#RD@^5E^%4|EzP+;;vifW<#Z{Ojw>S4oK!nD6> zUWQvCX!%xjGhQwgD0(H(ma5+5K#k82wDY;pazG@D=DBqL8b;%YaY@=wOC1l9CsV6r z1c{8s3}h5%n}!NLATb-8@%L9fb%;npRg)+F&v@{JAG{@GGkHiM-u3Ze3v5uG)0*E0 zfTGvUZg?{Kw68pj@A+mEZ%%agHa<%X&;E|bt5FYjID+5_SM2)kgPqC*bm*#yBc4Ld zbexq!BIYdQODKp1Lp_dYJuK_>HltTMEV+TQK%@UW;@sh$k#>a>+QAMe!42Id`Uvgn zyrICW`1NsFOyXv=0Ly3Pj2y`5hWSvJIh-&xUhZSwVBxJFj(E^tD+=}UfY`)NAoZuW zhxel&k2@xxH`2@ojn&>8K}O_VCu0anM7ZxT0aM>wO;^Sdm4D=ea`@9VH^qlETd!Yc z!dMjib4#`1a&HTZvceYEA!6g`#^IH8i%Pz0{yJ1=N!t$=B0cAjoPB~H3f98BQs~yA z1F4=4an5a0_(;Fc^YCXAAj$j@u&Qk0_1{z^Mlwncd0jolq^CQ); zJ(51HL$GjJJlS7v|C-py5XWyl>|N^;M#@wZiF>tiO6)fBU)MX`LR_mg%6mj7{;mhK zjUNBA9z0uMA%b;E*Ef}_9+)WhKo2t79{zagyYe%Rq||OUB-*90v5HFE!3u?~-IKcf zQTu(!sA>3QNAv2wXYg+ka`}&+U!wY?ZUvC<&}dWA$XswnvXd1Hh!XXXnA666<%@O` zwueIla;$0-`gQB8q2IXm%kr!aKhbeQVzXQv15R91rypK#2Mk2-;v)H6`v92bV(Hh< z`9k>ES?4d7KQfk1fwbvP`Mq8pT^!gws-%+ep{7`RqWf=SHlM)`VwBEQ>w0TN(RmSf zyDJ4&Eyz>9VhQQ%`rcih;+PqKHhtAF%yhl2Ekv?Z9#saLxyw6EIPFVQQ1qy%0NCCO zk9{+rDl&j(Z;Hc3M#tH;XjRLW!N9th<9$RLb$Q7hkP>GBSO~)W+`6=T9oRT*W70v{ zk=>0HBP3c%9Bc|r+Qf%!5ne0h{etZiY%x9f}D(*}Q9tK#X^xeO_eP296v55YaEF(tYu zOPox8$woq}cJ((S5f1-QAHXDRfnQQjJJZn8$!9e3nO;BxfiwO4?1)3|~RAm)i+ z_9FAy2l=DkylvGla$WVZumY1DT+ZYz8=J$1tPnS(S zVgk+r%r1Sl8+6XSZ&Lz%{MW1EvHe$)JTIg?q^jM@jzxNu*`Ksn%zjp|LU#(6Qw zl^f74;^NkWqpR6ID<@d(vwZs48Pm#5$RWjLX*Smo%A64RGzs=tHO>1 zd!s6^hk?U}H1$2hRIQuPX!r`17nu^w8jO|zT+6o9rPm?=j{;0P`wWmScgpmD3{oYO)s$O~7&h0ugB+G@=sf(twj2P{{P za}FohnIvuol5X2g&Fz`mxDX$T`~HIvNm)?73FsK_l#F%efPMV^YM27mBg!Q8@t_jS zEPo6~Y}DWvW#}`%Q%d5C$gIBnQa9o0oc36aRo&s@)!B+O5P#+ayhGlfuf1FVFNHYZ zsVI^B^5Nj&_)sAAJstted#h(Hxa}hHwn-(%g7{XIdA%9>XUt-Chek8ku@Q?=Kx~A& zc=kZn`V@t2l8c1Yoy$vql&+AiQcE;R>-LaJs@pIXMOw6sj0RHsD(&g)aT=-w7W%9y zdTSPsK7W9Syt0MnuzBmg&LwJzG`y-X6nM3&|LaT-wOjcwiYbq^t7l!j=t4#k1 z54+YDOI*X}KH*NLQj^#4Ir+%{hR-ud=qdg_{@m=96>Ih(9X}Z=Is!4>yjPFv0wa_g zRSNRISnO~IG>XlgO}rE*MZ(zhTP}db_Psx5f8IVc?DLyc;?YZ+UVM7cm^$YbK!B z*4KB~RIX^%NvQMoly%BgZh?Yb;FR1m>dfIqi^|TsucRUrL&NO?Uq1r)httTSw0Q_E zPylKa&CwCz4b_XZ>GIZh&q-UEq!XZ+Jqt~B20DHqMae^0y^akAg+7!YD23iJ(=jQ^Eq_ z8l0suxJ4kcfF<-Fm$YG2WH9w%j5taqncs?8sm&5*(u#dn`^?;$Qvr38S<8j1WU#>6 z3#Tvl{Qsfry`$j_zqjA$6Nw-(Q4_rl36YTKy+@51M(^_ozt_ zMDLvlVbrKW@ILbWmG`W3&Og?&mWAP-``P>2*Zy1?Zx|cI{@k8GN_a=*qO*qxKb(0| zVBaF=*h?eUy$E%uT57&FWm)szq~bV|e?}(s-z>d$V@g|GVoSh+gqI7b2H=zkHL5rT zVb{lT(6RHE?-1yL5JRk))BqnQ!#-1mmJ*X@EAOwqYQpp?vV;u>%}_>YYkGdxZ$UY} z+``i;HxK1+r{llHq+7Xl;s9OO7%dO@bG^%HfrA>fkzb$dYtfCLP=S&Oax!CqK3Tu% zs~5N&C0QS?F|Bxf4^Nn>_DQC%N3*=C!OxOWYLkE;D>Z=2C)4iqe>k4oF`^dE-PVN& z$%R?Y?N>l^TR}~@{%L1W&Pt3y;Gy+qLeIIuq&nXNVmI{Es5T~{Y83Wk_>#%pMR)kX zUV;||GwvY;lm;#pb5=98>Z)TxYbLYciT^9g@n z2op&ivhB|^6f9TW?moaQVvFJ`xFrtMX&)WOO23NSuOvkYjBz!BUguduG4tk-2z&73 zW~xjkk6UV_ogQ_mtTGkaZATYG#fE|xZ@FSH%~JU#`IsPE=zVtOU8{p+1k-oORrj!2{ zxpO`C|AXB5|3={WO5G^V_D_9F@R{y~@s9+$mjg_?l!(i&*13-NQ4x9`hP*er#nxiN z>C97;b+3jOy-12}SCwJAhpm^@lIY6^fARfMq7#gSh7qL|YExg!xpBc!$zL`tiY!<6 zKX;U9%YE#9qu`KGGxO~%rUIq(*;DF(il23wjjkR2Ce5(NiSU5*Qa7u_^TmzT z4kit?JBPn^LAjaN5W;X`40a{QAq~U{HBM; zNl5f;KE$vK0fe%A;M9#Yg{mdmiK3ITgLyM1u;c(W*-IP%#H`nd4AJT~!oe!#& zz2Bg*SHyJknC{rE#fR~wmxqYU6m;cdK^{JQf?85fe0$hqnI~d0&*yAD-eX~Q$lP-O z%*FddVtQR%U>NYqQ;^3dZm1_woG{6Yr!`5^vt#6^8cjm4YZ|Nf8UVcEdm_A53f35q zeU>RSy1rXKZKHZ!0?vWjZX!DE>Iw_<+=N0LJEdHhKQWgZ*Zv?urr(~mzTy2RuD{didg&j_ zv^QSA^XOpL$!8=iJ$O8yQk-^`TPfi3^rAP>2l4Xy4m_=~hd~;(sz#dDb+oJO$4dPU z0&}e8v1Oo54ndwHa<`*fdWc#BiASpjN0ehGOw0rwQT%TaVAW+*-r+sXieTA-fDo$k zeB(|#mlBHBktd;i2s3F$mXG8gQWprTkCTev{=Eemr4sVXV7CH$cJy4?EU;MQ|L)eN zkBcwB_DeG&F9%S6C5x^e9bq`%Xo$YDTKz_N(KbtPgSFY{OI(Vr^-N9Mg4Ok1w6uO* zYAIfcX8gTZqg=A!d{ojsM9T%2&fjOlV1XyFXUX>#1ei6%y~qI&R|-yVSn?b{QEluU z%*-*AZ-j8(tKNAf39O?)4yh}M0SH5y?{A@&5HgG*z0Ut|5)vqL3|gnU4h$IT$aFXj z{udP^F8#l#82BF7yRMCE0%y?ibmqYH_V4D~?@VhAhPM1{SGcFfaw2 zj5*xSx$3iNJ`oRWt|T!0I~(XcXq6F4j%Jq7xB#$lGS(W_Bp`po zjkf^-8R7HlF@bzTC_hEUs)JP|>08I!`|v8q0`CqsbQH%I&!6zWmGMpn*Y*$AP$2yx zW+=tu+v}r>gi^xM(YruAN5O&Ore=I85E+mx!jTI;;-=!cwBYVknz`7r-Zr~(2pBm|6{Uzm$YR}FZ-Evre_?I$Cp<2402U8>*r0U^_9 zfnO9@6yavO*t@4^)IE-r#m<%YhVAeB?)#iEkPWK{t)@r7QtbQO7Rh}OxnRwpo*Rm0 zcK@y@qw9e}p6`|CuWcq5!*L9$NS_B*LbF_(Oyi0cPt~<(NnJSzAm!H;7>BW&$`N5N zpN-is#!et8Z5HS%3z*df;Q&|(d+i$EssDet#*NYA|Cejrg3>nKRB-E~(0#Su z@BJz#fzd6P`{XEIHgj}kLH5I+AsqTnEb#b7<4><6EemoY6FW%qO{_l3R>^b#?sg*O zj9@m)50kX(9&0`_6OH)HEqFZ;wJ4Xgpm2e{(b|W|J8!);D@S&9KxE2DLBlVL2cvm5 zHrEFufR&$t(#k3_iZKZRR*>y2!rqkx{Z(pFVeMjvz}Q2+{vo*E_zhY>91?p<*(sbDQ^Jy7iskHla@Xz;BbC^eB?$p-(F}HGgttx%8liNg@#Iu541`FUE@h$U;aPf^7pY*m! zX}sbkO1=124rr!AnDrdK(AZg?*NKLVKtmpa;--^ll7`Lc9X+d0;po!u(Xn2EgY=Ye zcxOpzdGy&|5?XLPWxTVTSSsyq_tmN7plEg1659FiPH9xlzIAka$ZWN4`3Spl%ai$k z*<78Q&f5{1fKC0Y6wFf$SWG&^b^@f#md0h9(E*2Z;DwkTgKBV(h7a}V-O#+``+?r4 zdf%YtN>PSZUrRy)BjsvF+|T*0wwA&e_A&)~Ab+;`*d=3B-ijyXsm~xHa2W5)V1oUk_DtRSVD!pG!%~ z3g~j_5BGz(ZS+2hTX{9O#DxzVj_{Be$}Ynmu%KUmxQG7z>b>!s0#&VIzdsdn1p;2> zM{c0;$%ggEUTP#i5n)557x6s$RCkK)!z5$Gq2+Zv$|oO7|X#VO~+v)QRva#{ld zkg=43=|U@jO~9L{kODxkZ5eCN-*ladFyo-MhQ^kAe7+2gw$cEuy@6cLV%mq>*?jW5Le=a3jJAB~F*9u}l*EEEQM3=V1e<6DO^0 z0@$U?9MzJ1{r2k2)`Myw!YkuZJ}iy7f(JkPY5h~Jo3R#ceHotZ@My7 z1;L&FL?$1i zeaMa=XbYymi3-xt*Q?uNBbLm#<31($sF%TN2AZIS!sx~VH?3&e5H(&*p+HGV$( zuYlkrM*ep8h&nTLp#C#k&MEntv^@9|x)V8uapS8G-y_B|{3!$)yN@EbpwuhYWiQh8 zagZX80Ow@(t+*v&(b=U63tdaHCVShaCj zd>JrV?sUC$zPj}0lJ<3DzY){b4&VlYMq)hlBiT$h(h3sWi-l`9q%@8XucN@ z*5HMnW;VC+1BBg zaXvAgjB41E2dwwa)|Lyd+L~ggXiq`kzkH zr-K#k3U~sW3_h&q4$iT}C07A%CW3`ydi+OBL%r*wmD|66hRg+C?2l+AV!q#X`Ab<- zH$pKPGa|s>7hJw@d9)rFc+i)5^{V3@F}#E;N{ry$9*LH!x^_2{{k>R*G+MKBL$1R*DCtxx2?-RA?Yw zlw&b_G*E|wjtu7rXHtJqbtU?t{54T=Egz)&L7Qli^lGFCsA&Zu5xZ48+hIWaX6o{n z%awKzyS)1K0_M`~uadlNvP}N7+A)Ru-1<$T&e zqD=MfJf&U@4Lb-c%INz|>BsF(U3AWf9<+4axv#BfphqH+%a>sGz`Hj4(_$lOtiuzK zh*w#d#0$R%ZPZ#y>PSLQU)JUamK4&Y zOqUC#P-%fcMwuVCB2(f$SqXRgREu=ujcMm@1|Lm2e^O|H07(6nGnHAN|5D)wo?LIj zib(ve16G(noeP0rpE9chcXwWg*HmSHZB4y~vuOTx)Zb+*6}PE+a07~`fugV~?XxBgv+}ADWk1$i4=_^P`jnXVY)r3`?u2l; z>z?W31Til2Vm%Ld%r~y?tuM14j*! zGC#hI2{Zn?q5#jkdDfewgoECuOrf-%A)R{L16zf(02Wov(Iu^Cyv&|398{0?w!=|L zZ48wh5SS+pkr+C`ntlQ7>LyFM8c}?nzHE6`^H`HRgwKY zG;{UmiMS_+smukItObXAF>pI0BRs;=b>a$V`dKOM$3qPzjszYq1=e6Fq`|lcbxGh) zQuXY;m2iv-R4X?MN%t7PA+AKbu7CmH#^=Y~BD!UT=|{oddF0TWalT*Oc;h`bxbke< zTr0~9l^-QqL4}R$L@`>kzJRuPORRU!S83M-=Yt0at%h0GUfWRh$Nyozb)eyUvA(K5 z3;}kvCXgK=y(5PHU`lKk3P#aC{?L9#j_jko4i~EbbJ|@pF2EF`egof{+#6pa^q%KU z8+Z$fPD!vzw{*W&sO6U(4$1=ER&2rH;h-uRa~K5Fi1{qw4ASKz^weIzB*xVsBlmdAMsPyH-ZKvZ+&Qd zcnHRx^%`38=wRYLnI^NEMub9P)3gAv1u@P*dcR753#Ntj4n{yVgowT7$i^`F>cbf} z=8f}pk@I!eyV_37d*uO_mSMylD)g)LJtbs1*q6odOyZ~_;V{IZ^ zHDgb|F4{0ueCW(MD`ju5Y|?2)t2|==Swzzxw8msTK^F=Up-7bDai-*=mo6nJq{QCc z>UotQBJy3n#44OE7G;Z;thkATUR;*$st;~+N~tL{(@s~oSA4ddwi6W`i;*u={5 z)JFO(TB%~B`BKO0e_x~%Yy6XMelrazP^fyj(Pdq9hy3Z0+=T0t-^Gm$swnO!{`G4a z`DvMua5hk++f%Pg3e`?Zy5Hi64`^Thc1{js7?d^+%G?{LB|K<&EL;Bq)yH#Cq&r-A zjDw~K0e=%x|EOxd#ksNW1#MN>q|N`z{0FET;FS@dmXia&#p|t9Ysx=ALriM$&!0uL zUv_&L2>&uzJiqL%2|f?m`Bk_rx!$__*KwQ|pPV$SWsu?pA%4K$Q;OZivx(nK+XB7- zYHBvuZXX|rOp5FpE{n1}nh(6n4$ezD|GTAiNkW>zgjs(z!2@+AQVM>7bPOAL(X^x` za|gk2&UCZfNWKOF@|N=0Q-RqH$#;}QNRsrFXlUk*a{KYi!jPG@4id74Y#3LVH`$|i z{l8BxY=`TruZ?0M?H?DH`*B`r^L8myJVzyKe~nvuZI)jHe=75c96p@^U{(Zd=+DF6 z=qNR+9L$6w#$|~^_$F}B&1|;U7urj(7d<9VH16Qf$`(p5B)@H+md$ah4XzGIB;t8V03PgI&`_pT z83ts2PS?B7yf@A%aFUvnBQ`;L;3pV9W~(`1JV+E<1&zLEnzW-(^TX+oH=%}Ef7z>s zbr>>Om0ulZE7-kxivpW9&)7K_Y{o0kLy3LJdz*z@Ur-KnQcf7JTZ4EbJFL>fywckD z=vB?g3GhG$65pf(t&86|j8-px91WM6-R|HEEIxSdl>3?KsmVrv;@yNX=i@oZ1dlnI z)4&O1ylNf+i!1)2-(sJj1NJ(Az;C`&h7}qZ1wOq60?|5X6#X+?G-o))YCh=5>u47MX?N9nc=>=G+_#5 zA`GF&uVn$tA{$A-uPA4^mqlqw+~=Z!Z+<^u)olTdz!$54_!=QSpOe5lGv;C%KR7Y% zvt9~Uv*~T~r)^aHWNZsXsHKqsZG;BdxxLd?B0{>cjTWr*^YA$n@57fo~{`e$y4I}B}Q}-IU z8MayZAKp7^Qyut)#mhYZBBap*1c>B+`v@1q<9i9V~t`J4hSm_U`j#;7cC)TdpEbw9mC0J_%2h$uEy!ag-E{Id4 znyONT1oA}&U1>r-95*I(bUTR#FTHm+$X=c#$UglJp=y#dZ6fKiyal(MFhjqnM`^ym zi%=CHe?<9t-fiW@9E;`gr|Skx?(&7K98MuDhIiU{VqW-C4bMt-32^S~>O%P0m-AbX@V#D@-(~sV+CMdr<~H4hd%X-TZPWJ~V=pkeyzSo|Gy>B??P=CbD(j$F!>l{%aP+>KGOvE>1M9UgV_v z#}JsJYr5wh;O*#5iB!ushQ3fVBl`mkpUG*%9F?<oV2Da$n&6lMlTteR=aaf9O}ugFYk{%;mY5v$Luo2fC&t> z)Z3)0ldu`3crCeT8l6L?Ixc->=I09v!>BBhWTd5w=>yOrik3vGU)a#W8lZJFygK8410S;p z_~SyQJdDFf!L%2k=HJfidE|MD^^33E!Wy+|?kcnA>JKbu?L_j4NQxb7N*d+6>s7U%p?%k6txxXiRAdT3FXp?iHpc(vo)mNrOwohYY z>%hQIdjTU%Tl|^4`sbvt6aMK^Pm)=*@frj_`dj=3X-D*SoNb6{?Ys|1{n74xrk{`H z=8Gpr5><)Qk=I(J2jTkFfF<3DnxH0+gOrND4F~(Kc0`0&SI`1czQxp|&$C*BSrYy_ z2#gfsOhO6&QwNC(6-^{4O^1zV;$~c&b&MWbwMHV;1?-XGHjo(l^B}B&z6TOlp*Y4o zN6FW;M~RI3X3DNd*PYqrF=fa1{> zcean}T%D3dq>;|glDoNJmFM4mkbmVV!y7qu!K*{FwBLb?v3H`^T+nU$-WUc^r2A;j z6$57^s>G;5zaQ(z-VVk2StoTOV=0?tSn*boE|ss5(WEAqzANdi7RAgxJ*O-REKWqgSe33t2^M{i zkuckP!;nn%nzTR$Z_FE3{S8^b?EE4XG}K^Xzta2tVFj7aEZ>`LAK&?w747o7_S)sa zEnCh57+tN@62?bI5So5t=7h9V(qwm~Z#uVsO4sE=s5gKbi8#8Xfjl4ym&&Hc#l#djQ>qzRtlXEn2OxjCKZSe5Fq^J$%St4+1g1lio*N?osVPNl#tgR?8{}fSAT%&%(*%1@7KPldra}1IV_yLFF2*}3sD7bKPnEn;>f}`9@~4}usETh1Zv{){?fysa@eEYVB*%bK7(9(qBvF^p$~w^N zxL*XSAWVQ#knNa%Gv)6_-)brHdnjE)C$|%o&9<|#nRK30bwK%{nwD^(VR?=8pBmd8 z88l7!kn_xQ2hNhi^!Ibv&@6G*{a_Qdnlo{tsDdu(FdRnt+bhu-;$YfZK|yEv%3S>N z#ZJ+df}CA0srMX9o10$7tX``G?QBk_%tf!m#NJVKqtK(v%5Z_R1EiS`-SGGl_}sh* z%8<=$QSdT`M#BxkY}c}Q&=BFVbn&=e{nsku2n#BFZ+dL@h$Nl0T&fI@>QH3!3Hyd}k9%e1?iAL>8^y8qj{tp2&+PD6IOo?Yl zJq?VHusw#~*Mgsj;`%{H5I(YZRQYG$`7tgATA8XB?=9KShOAUBUi21ykhM1~)|Gcy z<%IuS%ll>&MlT+RfCg4ypVRvH{{G!nWutncCy6?*J+oh|PhdAH1gCb4-#@sPE5oIw z2CFhZu38-#dCrKwp4`0QFuQwuv~lqZYv9IQh|3*3TfpQ2gb8sn{>=r~6v7FQC0l2E zg%{5jPINzl=NR2-{T-rf3LiAK`7fSMmjGl(askm9iubhm)l7Qt(qC&IHSUaKickhw z(T3v!?N1?>ML-D1*=jw&X$$mOeKX8w_DcS%7>%=5TX2$l!VfAJZi1@mDcZZU5k9Gj zPqJjHdTC1{3HH00;q15Gthziy{YuE)$}<<#$eqjMdczk_^c>f=ald@=PRhf{%D47G zA5N<_y@pB$LE=1Jz?h;=+VoXsbtwT5uU2%rBh)?Mm{^L}_80NvCd{7HA+DjPX*Ha@ zLuFYHpvp$*Xk*h%+D;=u{f#78Tkh|g$>HbtD;68U0)PZ;CcNNM@TrH#DHw(2GK)6HD0n5?M*;r+*Q zm!gHc#_1jNP8tCb??frFz-EYt^;38D`!(tzPE1<`2!Hxq^yk{%uABo+fYgrzR+?zy z?}bJubv)tFuY7ldUy*`nAPeZWstccs+~PL0+P|ZEqwkPP zTPofVerG2sRjP8T>zxbL^9(i<{%)8=ZI#M*{b?&zlX~lYiQw6S{5Ya#)f|y9-Sb9O zgfL8b((so$1>#fwW*qi?wC$={g(LuAXK80$d*HCdxT)xpt6nD3fZgC4!#rD?m_vTk z0T;tBtW9oV@Yz}-RM4oz%PLN|Ya1{wA`kKG4a{#7u1a+`tKghiu8Z|PJvRT1ZlG--%cx!}SQW3ED2pGXPy}NZXsU_YI^UDn6tQKgK#Ua_W}09J)E~Jp%+Ue2kL`9nXbl*TEH;-f%|$%` zOy!WF84gk~A@3GG)LENTJB9e>=3zBu5hr+G{;1^F&5QLN{YDf5st#BzzweKKy|@p^ z3rGEzU{wHwte&werm^xn3Cm>bCqJ9)?8NcPiCx{PvzptcaJxU?pZpU|F@^kI^lScS zO=QG)&1St;gZ69|V_U#g`pn-%tLd{-=^Ist+Voz&t7`vf@bKi*jm3T-)9N{}8F8Iy z^@*}{V&}p-8Gu~c)?8jUDy@yBW`%Hd+g7>``{W>f_}g{~$``yHDrEeU++S9C9`Qu1 z%PEwV>`ABvjz{R9JPy3B~DH8V55{(J{w9*k(Gmtpj8+JeM zR+twNCx9(pd}Y0xAxhIH-A_7BJJS8hios^pMKD%$BY4w2t1sb;$GOj+^je2yp2=Nr zFb~-?M_h$oc>eg{B>_9q;=!YAQQxz(A1Oe)Zt%*ZeZ@#b`z9Gv#=c&W7M zL*8i4MAut@wbQZ_uavdFKJ-m$v{Cj|q&C3_M?jJ^rjys*fBeooB_jiL{j$-*zJK*d#qfJRnO3ls_7L zuz`-NZ!Wq}UzicJHh3(@Q>-`ANn=1V!p-5rAcHB1fKg^4v;4xo#5)qe%Oju^*e)ED zL1JGhxKj65#H-ToGU+hq5ujOgX%@0zN2i5iie%h2V?ntwuBe}=pI}C z7uZVXaCK^YTx3V^ER_2Z+Hxktrob3(FJ#-e(b*Mqvg6e3Y=QX~mS{}T@SEr=WA1VY zZU3fakB>&>FmYx-And)oY!jV9@V!qf&vy%Z*ibk!YkECqi$I%q04TGcZrHQep^wF- z9^(XbGqb|yCp$ZyN<8O1!-}o{fk>-@7*M9r=k0#f(->VfyYqhhXRNyY_?;`CFIr*!dR*Il&Au(SoOy&zcoF9^r7I;D8;z4 zZ%Tak1lA_*nbTbeB%r`^5iXbZ4+kU_4&w3N?h0kzzo4Lk7T@nCsK^|rc(DT7JEtSZ zG5Z){ug+GzXcC;zhU|Lu4((6AiW!+|aud{teuXEN^s;=Pe&88~Fzn zt)hdWhPD~rK?t;L3(k(y^T2XuZBIlh*n@@k6?Y_f;Om<+GIH z_Z&ntfPVB%-BKNfq;SAw3JI++Ts$l_*dJ-O<49ZR7_fahOe$%fppsj~tZ$?=*&EK^ zAU2G1m9J^SABGf1!mg-(E>lZ3tywc;r`Ns0vcUNBPWYi+SwVW&N0wkql z7$%+50ASm6ttaHWWomk)A_H2x92bh&!3*)ny-G)n(`-BG7GX?iU{f)A>;h2PB7HOe z$kUch=?3pebmRu{=I+zSA`9*Y4$d(qLOciks7FBGXu1e;JbU+UaXZivYLV%rTC{Y2 zeC{pT1`pVMuC!oiHVb}`H`PfvU7}2NG8mMOJ9m7E=pZ*}+g6)>9#c_U9djdd(aZW` zxow~9Rk_H;B ziWj##?B`NNAV($TS6){uygSpqFE|JR2|;=1*yIw9(cOz>Tn*g2wu1bg$J@O_Ban6P zMUtg?g3GU2kI+Y{s9Y2m{ zmr3y=NDiGA+E&?X(vJ&04|v3&9@hv=wr9XFwHHZ{`PbKd(hj)B4XdV{%c@!dnJwTSVv&H8zc zqmup*hW_e;ZG+nV>kj<}sK)AEk?IpLhQN>MI60LSuz9LyJLLpX46F2+EIzPXj)e7= z4xI%=z;yQcKR|--yV1?+lR#=^Q7|fT4t+ZUe~Khb+$ON9*hICwtdc4F6GN5@#y#+{ zqJKXMzsZveRtPd?PxOloR=-7Q^SfQM6jGJ@68^&>Lj5DBzK7)l(6Y*%mn{1IK&zKy zD>zsTj0KqW<7Iwa{tj3!Yhg%vB8PFPdeuZn2=wtx`u(ysq-BfAiKY~O$eiyg;3T^- zUOzzNTnvi=bcPQUe>!HVQ*esIA9|;E`pKSne)6x|wq(gUHZ~x{blw_Xtn;6wb8r}t zeiSj5Thz;0W1;H& za~&gUbjhzQGN5A|pCV&d599=;1=MmuS0k14NhTzlH;Cta(OPdb0#=D$+1 z($;qdPEgX-d;Ex~!yGHd9^CzRh3topjnn$#tug6v;f9Zr%Me z2+y5)1=OAGxv)=tx;_>BAcps*g44ZCmx)n*(ZY;tBA))`5yC>50fcf1iG|Jv;Cc!K zFdih6#g(s?b=cy643cB1*o>eZ`)whFPr5t9MMQMh8z;@BU~aZ5PL@aT&u z_31{V*PXz(nW_|x!20`so%C%g0AM4j&r5{5@q-F$kzd^KGC7Rp15uQiVr|K2f8}L7 zVjT3DV+7`77=v(F1(bWgjqw_W7zof9UtGLbsi7q-A< z!C6q%>eeI$!EZXH*6&KuUKE7t7^j?_TIPTH=;Esixc$R(fMM1AwHTIe$uq2i4nn~yr*bFkF`8uw zufabt8Kd`1)kWg4Qz#AX#u$H7eM&5fAs2eHqXfXw8#U;{HwcEv!>Mz9@Q1?;Jjy+c z8c9b>fF&e%S+B_YNZX@fnTL0L8yzBGyOJFIZ!&qZne+vxchdzH)rN6QlAOMS!ir4x zANrY;OfLQ+!%qfIInC<>^$6)k_HV7)-iX0_UXyu|aFOLLFFE;SNui}cA01fqTwCxo z@JWx#SZIKFPS)mdY%&7MNuSI*whmac?jDtNHZ}mhdW8!zH>6{w99>$*fW>0kWzR=D zv8>M%*slJks2}pqNJeFXBJm9af z#%<@=SW2+PD6M=Bmhc9!hn{Eh-strv?)g>cW5t?kU%JMU%%?cB4Pl%UxXq%`jEb(& zUeVn#oZ00Whyg*U@UEi#FUiU4^b`kxxtHsEXtjA@2y|=MtUq636&A~&rBrK771BTP z6Tr<7Y&;jW+?2O`^e_H0f2}dUelrYb2!szjTks$4vG*0^XoK9hafIw&H@pU4o_cMD%@pprVT+x5_ z{r2YigF^M=F~RsOU~-T%`8ukRtCyDG@!f}XmQ(j-OYvx7kEvOB$uwjB#F?(4Yrk^$ zd0G((iYW*`12y}-8NKnA%zC~0TGLzxwHN*2CNC}!dOWIAIwLjSD|Q>dFx9r;JgbKM zZ*Pi7pc+ZLhR)A`nJ5n?aFagKf}5tB4$f|57%#*~{)8trVSnuUWTL`Vu9`a5Z0~r(CD10|q zB<`^skDdX$942>*BH}UeDwh`}K(aJis0zvY_G)q^tx_t~c09xu>?L!uCBccb9Z&!A zdc$nvA@5IhIUXYG;G8I;A>v_`Z7@9Fx!128j7_#nJ>`G-&SF03$!0qpxdA|Brv$iu z{oWb7x+ju-26#ElRQ1~4Mn$+<>?&rlRAl@ldwQ{y2djl<-u-=H+w-ENEU}KYz!Vv! z4wmD&c-1CSnDvFDf>-1_5USKtvK9XNVNB&1rV2=uayRsshmQ@LTu%Jd`}uJNf@cf;?0hf?sxlB!=c_p&60{aSceW8vyPw6y=->)}znpy;r# z%A2S|p!Ei<$-IL!jtSf3$uI?yXF!zhRZ}e{2?b^-Y3sTz%S@K)LvdnyBRq&W;px|B z-{MD?)OS>r<2X_AN}Z%{!HSA(e%<1KbY+txi9Oeh8jee&MeN`b4AHqM2VeakG#Zg9 ziRA>2Sh`I<&zB}vH~Y7efsZ5;_Pt00Epg{?Zk+ilN|nv@)5UcdKV%OT^vS1?>SuPy zHD=F+mhIUZQ*iY6)RBE=_CzWV!(XC0ss+{7`po-`O@beYH1NK*k2<&P+HXFRlgvGzWG)VQWBHc1sR(rOl&XD_^7iO^msk6)1DvZn2@ZB4M~heqGCL z(6OLV@{6~utES47n!rMVBNu|7z5ktnIzdt?=^4MHjlpd+srG$_*T>alx^HZC4o$VG z(e9>7x41>Q0;q%?ofjJEhamwtLjm0;_h8$Cmw)`|bD@A>t32Er0jvH4htvK7xCvu; zgqhO>S*?6=5r>8T_B&FcB{L8C<23Uu8O1xp+1TxQ5QnY4 z#oYk`MH{R4n&rikSXQ~a)|QV8Ni$?C?P~hWMaB!uH1X81tX#ure zfTiE6y2J! z1hYH_ph5E$d@TR9D%7)b=6}8qpRojO(c=Ngx#yPM`)`FZ-+r9e1j57YevP(XwtV{Z zV5B#|y^pV6yW(G7LN}@%34-;5G0Wq?#@h(v#q64UotP{r!jhdGGaGnngPkAGj8zES zGik~C8;X3mn#5!D*qniwl#pfh_8+BA)Rh+g2NXylgTE{UOWou4!?3_A1_fn^KX_oN z)od)6{b}!R!cPY+bw&0_tx8053kr>R$QR9D+477Z5kXx%A&@0tg%1qlYJzq{@cVy9zZTZEb;{4lY>R z4@~W@c4sagSd%x1DscXe0Qxrf@~SW5IA7QW=V$QZn{iKLRb!7F-ST}lo|5{{HjdVgd}VMYts+>H=|kHT3&1YW^XB|{~N z$+)lVnXMZJ4w)NbN7(1@3LM#Wxs~1RxIqh_INiq6Ka}2}HwUXEo9&v*mynuzxKn4Q zv3*d|Nok^ZVHq{&H`^ zSSH@{UBNe9SKQo6xAqYB8%!-dK)c{rywPoM?r<>lff5X2I_ITjCX{JMSX8%|>S1-- z`}}1KE!V?5yWoGsugbeJJb z*zaXXB~mFwf=~t$3|=N{`6>7`o-|=;Y873YZ_fTa#;2xNafoxDu=|}y_zd`_N2uyE ziptjzn9a#8doNE%kRxtrUMH}Xh3SH5BItI{rP(a?@u_N0?DI?u;2bLiFl#;m@j6It z-Jqo#pcNZ<;|3tXlfo&m3ab@*QZ2ni%9UC2t1QUD6ej086~up=fD+<+JKd0SLi(XT z^&CUC$34Oo?0;m9`*>CB&Xmvf?N{*CInptF-r>qUA_bVOnqE%N9r;lQ;@81wMa&*Dl18uZnX1qfdP1n-JY zkDIFFuNLT1&K}8-r47^qDJFW0AIIPq-b+oaRM?!e+oQmu2;2;tOSIQ3<2C1X$x|jD zsapR&B~{Q>`IkLiNQcp`N*My|Hg5lruQo=WLwO7CU(2UvAn%^zzx&tmI_fiuoQxvtSV)pW-fhfn)sMy!{{ z@Ag`V@!iI7StT92b>c%lxh{R(A75FUC>#$gG5Q6^w^G(`i4BDOg^=_ZPylL30(wrE z?vs>wTYC7fuO$nXqqGV^gNTjWi&^tCcYr{Nl=*&^y&jUbsb$oidY?3T)2o$F;XHof zUTxPc;Z{YT^A34acI-vba82HE+pegjfnZeo#TV;cV0MCPZv}>6dZ&@-z3fJ>^ptAj zG{L1(O7YayNV)p`ok4?74Jw@_2lt|%N9YZS?@5yp-=sN1dWSjXjMOa~npHA`dAfYx z-`m13Fh<|ydsK02tLw`P9%`*hT;UYqK)i64R26PjVr5goW!8?VTjLDLU!%!~dslb8 zJJiCGQ&J%8DaI5LeL>Ew8-G3{(g?)Pl^e%g8BZ>aV@!>EN2H#p5(!voGdW0A-D?I+ zGL+6_5$_3&#itJBY-Mh*u1wjtMT4J!Q5PaCEM0HC#OU zv*v;K57u5R-40ym{fbu9Gc5RtY&@B33>4U;50nrCS+i9jzvoa;v%{TEoi(~yx`rj= z4N(7nf@mYx%*Od+zJnd;fN%7Kh2Pae4_jKf~e! z3HDU$xiUJvl#M_Mr8EM!pO-7%Q=%dqgB^B4)!x(=6#ESRu_$RwBW%?H>PRfg^Wo0m zjiY{O4|&Ql9-xefqIk?eIi}x>otsFecdv(Alm+7~gcTdp%NyigcoioT^B=Tgv^ctI z2G~Ef6wdhrO~#TZe|6&5w+X-eqdgYVhKsGr^M0Kb!Bm<{C_eOC!Xm{+;Sdt+JM>yc zh7Mv{#UKNO-{zL*-aN|g_sVINub@|Z&S$6E@b;-js*#>0hM$f`De#FcA$3m`#Kqtf zs=!wR+~{fxAg7HqmmuR6+-tBg=oB>XDK975f{Z{Zeo&N<(?swip{o_Dn~Bg+nXp_2 zkuu1eXh$FI3@(a7q1qUrhjUIU29k!Z_f$n|1&JuU|ASqYlK&PCZ<@7Q`RmnFQs_qI zzx}p9p8l5TLtX#WRuIUK-E!;dTx%aIN%4E|m0`$@lGa_64X#w>r2NcvRbb&nCTsTc|eQ8;9JlwW*CNa>}7@8`pnxxDfFO zrTQS!K{@0@|AOKDbrJmm9Q4#R; zewm#QTw7D5O$3Mp^~D>tSSmJ$vE;YGj7_e|$cB7*0A=c?~d)&zl3(|mqQ z8H=3(kDg8L4U|KPU-WbUxnZ4eF2@Jy+i`hEcc0``N+f$=x%td9W{({neHDz-X0G;6 z`Y!tZTWKtxjzx|`YrR1~JJ&uX2ULQj4~tadS>-V~8M%~Pt(b;1YFU{h?x6=)jDJ+| z-J0B_e`Nr`yL$P@eUtU)dExfyWcyT2Q`@Z*5l`737bKE%Tm0rp23Xz|eiSs%bB`GX z-xstiG2ZCkzTWFe&ICXA7U5%+V^THOge{`+>u(kwC~H#P(EBt5ckN+y?O^r0MD_CZ*Xt+C z;hXj@9i&=)O2bYl@Z^Qh^9gkS9xeUiR(NZ1_4o_X=DTU$LoARHx`W+@-jQ8AlHqft z(Yz*A908p|0@ZTX7sIkIMk=Gu$%?W1cRw7-;nNn#aB#8T%r~*Ui)5;idC$sLvG<@! zLQeeJYL24rFNrchF@kmMUHEGzQlq!rIC!_~_q_^7e+)6y($me^)Ut9rY0#x5j%5+_ zaod5J>X#T&)$xg8j_@?Mu^$`uLnaqIg+GBvlH3%Tjb7zRz`sV z;2%k5z%s0paveAbl^Fi$J1?p(WM{Jho(1|*!062z`A2=lNv1ZgJe zp1F(|ThyF}cRrN<9(*XS`i8&Qps9p8DihZ}kk6pl9bCK)EY00He~(IWLlo}iDdg(B ziBq-{%x1H;ZHc+2@?j;m<8@pnN;(rwngI>lkDppJM=sJ*8=pn)dv0rWmcHlP2o`7k z(0wSdz8f*=cG3v^n%4JHtRkkG*Q7)((v+E+I&heIR2s8CvX(39%d2R6 z{~Gv42Dom5pU6GucA>A3S6pev;N76t;$$+%c9$@W8qOVt8WoQjfy2P&R1{2F)z5K0 zmU-?s!IsiP(&L1b2#VOcCnFyTw?|k~0|uFf6KZ?aiAtr*Kk1vr@UsdpujUmyz4Uv= zxS@XIR{MecUxl$^{L@cdq&Tt;s2>B*eQ7xm8qnN=6oZeSo6vVuhal>wtZbH#2#wu& z8SloLEuR<^@0F1_rdQ(`G?+af%d=)PtvisKB-Vggk);lbXM}&ttA3Ve@1DUftOm4C z4(Yx5j%Zd93TWT?zZoh}-h0el_kT#|I?h_5DCjKw_sI*uqUV!hmJ2vHsA_o|P3o3Z z0iIR%i-j?8`1!kwYZN8=N&>bCF+$%xXY*0?VJgKFOtf+SG%R51@+tm8(Sw%nBSi`Q zGgH#|%?P03c~^)n|GGo_vpyFp=py`9r>r@RJKm&OQanZUV)Y+QqM5D7%kj!^ zNvpc;_LRe6?R+JW1m=QAkwG5!IWz+Ak~b|q|pf( zW}ti#9Xr9ts#v2HRrbB(lTe(#ftA+GabKmOT=%;FUN)QYGi8Q5usDwgcPwzGZeEE? zxc`$D@URF8TPPb%T}uy}zBv{JeeTOjTM*ZEa%8}Mpt6Cc#Ndv~e_{?wpR-DUNm=ES z>X1#3&0_90m0=WLL_60@vEdq-dsEnVS~d8obkTp1sZJ% z1dIDwhek*x)DamC5t7jwEQ1z|Hl&2?jU?F36sNVc92Ca5`a+3T6!<{6eO?FE>(^|}8irbM5fT@UHvFP3&xVPbqK17X8dbVXc zOLpU*l^jR%tZ)!-Y7Xr-nvB(jRAe@J(T+X8h%GElx!rVrb5y6PN#*(-7-SDGAU(@WlvEM8)I-8KY7mVf#{6Hhi z{DCrBi?-yxb_>?2*xjpAI=m0;kef&<9aUtM_k65&c?!91yaqI#S6S;NMP|bYhqn>X z;!|I$xsvCig40)HPWv`;0LYn;5Rxv=RAoN9?$Qmz4IMlx8z&ydXO`IfrIOiqB<;sN zshQ5FmF*u|Svo>Ml`SCM`1P}8u?%gj+mn#b0x$Kq^F=Mf7*vu{(dpqV;m^egt#-Oc z8|sN&o!`{!*D(|>!X{*}jh23nxAi>?EXlZS0suUKHNI^RCGplA7hq1eJ-@~c`4rkPVRbMyjkeR{>A2RgvyjXgf)c6HhT>_Heev*xU#PP3d8jy#()CglLFA;4KgBr2oq|$aFY2bb!!EH{zvrH^J`Nb>{ zqU;J=4%90F>ky|^oldH=hN)wfAoz@4y_x|$O?F17=HI)FR0LQb7mJ8$HgX{A6P;r- z0ZWiw0j`!R5J-b+m*VYEW>&kFAg~vSTn_fV5@5uxPdLqFzlv_ZXTf8Qa8^~j0(+Vg z7Z5LpW(Q2wM|514_}i?qm4s~by0Jnf7uOAw?{u%0i+lD0tt1LftD-B71n$4aP%}^p zv2W8wW~EkJdlG_3qwYjWZPwhc9UMu?>VrrQ+J+hxt7!of#a4s1Qs5s4 z#qwF^Q5z%TWYPQsVDWTq21qWnlEYsiL7E@uxP)9ftCbsrS>7HU=q6Vs@D=aKI=wRI zF{bYhHHka%UBtd`_vcsTOTFeqDtw7r#$PL^hER$@=`~&`P3{ zedy>7`Fmis$7gB59EeaH7p5{{C7wj+n8xg-*z!M!*nV=4^Ow{zSfkcb*1YudHL4GX zAQJU~KJG`8%MLMgHsA4ag=X6B6=vCKsM&7g+iLxO{y_Gp&^vvvd+*xlGD{Wd!bSP| zC4yxNgXAvkn=4H&zm4k1zQ1gGkiz=##^#T27Ujl?rPSkJ_P^>SaF-MMcgO#SDG}1% z6UHL;AFM=JR&z`Lf9$8sORRqKtSM)c{~!CQGhXC1CGgPU10FifK4N!W0n;CQ$nkrR zS%^2RGn@(T zYdc{r9yC__4>j34f>hi0CO(qhNf9u%jQQs--32)t4r*;AIQSntW^io+#=?BUhbjX1 zz|L|XR8ZDe!H4sgx~C{$vBS;;{@wF13}m9W8#;BkSm3;sXe>63T#R&%?Eo)?pV|@n z4~uY|;zdzYVx$sJd3+xIex9d*3-~3+3{q=(3v?3O6vd{s98irTa8bum(05-?l(Roz z*Yv&)aHKV&#$@W3VAYlHFS`nFJM3{9<=Vui*HnExJw_j@s<~jj_>of7f zL5&I%G(+(0oMwSTd9_$56ZWlb1i`(t)CE=Yb6E3f>5|zHpMYyfB=KXcR^)8?eFmk* zC`mTEpLUeldw|1Xy#~~g5j;<I8>NJ?I`k)Uh#Zkt&2+cEN!Po+Fo`vlRqRl14uhkRnE<}z34-GbaFS6h z9Q=M@=koP7ACZ!Q&W;103wC|k>22)^6FyFODOVOg7recwyJxcfr%Ju{H}CC5#lbP_ znCnNhgpK9(t~8O~zU-a!=~!KF`<)?a*Mv*SW=Eh8qy|X_r})f*Y`=osZP-Rfpz09~ z`F)g&B-kN5CStO);s;ndovuQxOG~n9ut*PGKh8|%AMv=Giq!=~%Ce+%!ab#ROoNZ? zUXj%4^pl5w?^RtI8H9c+71Y=#{Q8KNoFM5rfgHtblIjJ%s$9HC-Sg{!cPi&hLO}u# zz9DJDGk|D3X`~L}%TcJDz%GniIc#ndVv4hiyeje*oBK|;yM@I*4PhazgxOM>2w%KE zRL!x^&*&c-C+W+vjUjlF`mNN~;^9KXo(azOh-bz^>mbA7`OqEVZ*C({F%BKDd3R$h zuxfn+K&oMNqNtwPtAkp%b$EmZ3=lC9j=g7s0yVeU*5u*V3)@KZ#2Nn)*--WvI&aCud5@Eqvu7Z%vFgAz zNg;GQbiOoLdn4mHY0ui6ck8=>t5U|Fp`bmKsnxcH z1d0|qX#P_&PL&yLB^PHH@|Jodg*S{Rw^6U%A!Fd~ToW_z3-5-&`T?iBdt{L(i9Hs) zSKXixw|grtOqE|FQJMpgf;?~?64uoB08y#-gOkxDVm4&sQs9r*QUuU7_aVt2rm?Xx z;z>EI{F{Z#Z;FiVkqjF}{|Aa7mam~K){uS)z3z1Q{73fc-8Z=bwM3_3sk?-KLSU2| zsOGt$&PL{MZeMdJ&0qUa=bkm@0W{r@ajF}hxTvO(ZXka;^X}eX+Q!Rji-Ltu%Lmsh z2fQcigs>GX`xU^Yp-|*m&D{UGu8Nhq4eQhzg(qT#ZKIA$(JK^rl=Yq#l1CboYVV7qE)acNu3@mFh z9P0Md1t}Sp4Kpwv1;Lx=yJv(Y(8%Xt6_n(}>#rTdZ(?IuqjO?b!s}9P$#Q_`UQW&y zZ=!j(-wGz2QnKDHzuqmam65{iaD%Rf^$Tnt18{P^`zTZ4(7bHF+5QT+ai3tu{xBHw6n*U`1 zfEsz~U+O$j@gX3i!r%H{*yGG0UNS(zK=F*U8?N*2c7BCp5L2vMm$F6z%HmV}0B;Pa zduu^eN)ctI3V*4=aF=|R`#d%WJ5FRJ=LDOYZM#x~n&-FTXz)`hVlu_bI-TRP4k{`H=v)v0p3}?uDV$sN#_Ib<1FxC~arNqDXL!3zA z1YhHT<(wd{nfvQ9kKe$pq6>rkW==iN)f=~5R%YX#;${SdzfQGIQ(>k>kv%EElb$9m z>rZ{TUGn^UsLWT#oa$%Ibda~~U=+a~0uJ@QN4Peeuk7L1G?~Y^w#?$yyz)=60KDSL z-!}LE8Ed88|Btbjezx_KRxe)JP2WrVK z%EJ14&n&m%W}j6i!Z^$mouQOdbe~b5HH|xHM&Wm2qnM^sx1_`)kMR0HerlVh)yX3= z%xP~!@$h-x9-Sw80Mvvx#<)Y95UiqgE#+$Bxx~q++h3ItE+f#fw`Vl?f<#hUSS3Y1 zXs5gAL&8irbfTXnPf#ix+MLD~&zk>srj7*f$6O z4eT6S&yg!oOHh_OBC38b?QTVVgoT)VH!LfE3cNbC+Q0Tgc|t6m!A^G6HDJ#H+z(Uv zC&Q0ZVHLD=C%7n2y8f8r`G{_ArDAW+o0kLrCQVP%xZ=`2J3)Iy901v?3&o~3I~VzZ zN*r+is*tS_Y7f{lL;EC;UQ!w-I~TyZ?HH8PhZ0@*s*j$a9;Hl5JS9RP{Iw{HeuqC@ z$o@ShvFu;<`s}P=`1n6Ur}uwRA^)7ONA~VX7O~>P|J6AciCdBXFO`#t8QXqq+NHuh zxCZdVeep53WAvbMd%K!|2nm4$d)A)`19LeYKS(Tpm9awcb2Wmz@9w`~b0?c;#PZR^wOUXa(hs zyBxKO)iV@dVdbzx%T>ZnB&%H5r98{LY-a%XrF}k%-I%{2OW%h7d-kITNGRS9JW3Rs z;oukjQ5xYf!#BY`g4oYU$3e3b>e1|?WSXD*%IMA(#x6jBLiBbkR{x)V@6Xjddh_Lc zq}oy3O`+H`gWX#WxN5;vuOEA?Vxs&=XqC1*w;wn9bHl3Hg+p-v(jPc%&9Qcttk1To z+*DEPZfG1vF_hUS9L-BO+g;7a*;dMq%$5Es{!47)P-RX#Rvn;EbPBl`UxJVls3(Y2 z8|+u(=ky`obW!*jtqscBovMWro7{~pWGnXUyWbdiN({1IdNArk$*9;tSm(wuZ#m$j zQo9)UMb`G&De4{5{g~%uPLZasRn0LtVA2$~y`Iby)oS(6#pZ!-4@I>3_>tkkVZ}9yA@MpS=UJE39wFCMP>c~V&K;DAlVevC_BP;+>aol4Wa!cZo-zJnmFoRl)S;{yg7% zE>GR8E}V8|hJkC944lA&yVF`KdCFJ@u3_z{_`cWI3Jh%k`< zcek4L04yTy$X()jkhb|QLH7si?{(KM-y~kRx7hRkEp-sVuIbL2a>g+I8DgZUn~|x_ zEf<0-jyk0`u?Bp`ItY5=9P7MQ*dTJ?tKB7A3RfYn!bXx!_9JMdV?Tp_z}YR(e8%ss z66vdzL&8RSwki2n-tWBMik^0rv@ZNkmNl6NWE(8ldp)-e=F8z{=X+e(aGLvpiM+VekRW*58hWl>pF^0*40ezo0wHsew9 zRMUWn=Zp%KeT=D&nGFe-Nj_i~ryznYq@Z;_4FHDA4s6En5CiBrzkKYHFn{T6IMiAh-*8?QfCL0_YQ->`-CVlHOn3Sl5ZUUxvXS9 zBwGV7_W4huizfe#`mQbxGm$6Z8LieEM=RnUg)&16WRYK0HVC||EnL4|0ri~t$cl#X zhq0c1Dzgie(-&gkS3k|&GG8e^#2BUsqe_U0u8g0(s4~;EQiZ@oHX$!4RRm0Nf#o<~ z{EVKT@1y#eJ)mD;995hlb?Rot+_ty&)yg;ru(RF^+vim3)3pZ#$8TUPcimZWr*uZ= zY~u&WOQOWj;V@}S*M>(Pe#bN&9QU#eiAbT#XTcxpf*XH}ccc5J`=a!q$s>6-uU#gz z&Zw34ar$fblzfceyq5@@n(Dde-MdFo3(|Y}Ru#EBpfe_h%tYJ`vsL#A0R#QD`q%ub>K%2?`8sCkI zEDp#u@St)fPfY2&&Y{#(Mw9V0xntLa-@hsEzV|(L>@vovm>jna1|EeYQXP^BH5ovZ zo%h((gZ30&QMkzVs3?m^`pVY#v9{JK@8-K>dy&%bv$GvWgwhl~J9x5r`^iUX-E*M4m#nha=DLT`2kc6*#*m8dla_Xb zh~Y3QHUr;XBiH)`AtgO#8|6CEQ;B|;(5_lQV}c`kM_bLKHKDElF#u6<^L4Z*TXa}( z-Dnv_GpCq*zxt=VzgGKuc1`hI#lfft*JAZ~IFXvFsO^v!%MnWzCE$A*aP1g$bvN|# zo)}Q8JWXteNLNWHNeC#9XzrUgn{i{$h>@WBmZedXr8hVlR3RC)3h(#6^U|lvG|{2J zp0RK9!G<#ZTVKIgD#DZ`tu2U9hSAbdvWBj~NS`JgrYRw4^Aw>!oi+w_{HYQrGgIi; zMTzhY5*jBW!nDuVepKZqX|H(t%knwMAovrzsGq$0G~}5<<8LxJtUI>T$2(u(7W*2B z&;op`YLT@WCD7H;nJ`zRHB1O7YV&?Wc(`XNHF2`i3b2PfszfHrr_(@ap6OEv{8cb` z(vmFa=x|GZp5lyhxL!vq5uF4YZ%Z&D?bCmi!1)FtvTu2c^lpgKQfR*;el?P*SRJ;`-hAs1!xJGe!DB8W7`y~d5OKu9ue8a)9y=;*d zh|)U=$^@?tU&>6w{lsVok19{aBTv~2WPLwhebfJBE!1}d$k>$o@9M6hOA8O+ch+A6 zd(DHfJSWg1eszQ3$!vFCCot;)8|fQy0C2}?f?56ZL9qD5QPJmm$1^s4DZo@&vf+*1 z9pR||J@`e zh)Jg`b41_s=;fz#Jk=&#K-9~rDNFHRWXb;&Qh7r2xbT2IujdzQH~oE`aGA4rTAewc z?Dyh&S`{LGhC}1MTZVP2sWv!_U3LoZgxh3e5e@lX`U}8_f{5J|jF5z?G%EP?-3lMz z*7BtwKu9jNRxJ{ljAEB2;@E!ni1suo$Oyea;1X0Zfkub{zb?G~8^&~tf6F}%IKA&e zV1YlJ3LNHdRu}1kE+cgXAl727#cshJE_Wlad<&jaLCIgox0ibDtr8a+v1cODZ(o#D zCZ@cZyG9(`#mp6gntcS{&#Ho1^LPwlHtnk2 zhz)5H(3wX5o>*MZNV*eEig{W3T6Safdc38*UL35RGR9J?k2>!CGin3c3pWpoSlDsu zUj*b5eev(mYWTHu+cd`!M|&SulO>x-7{TKfKhy8OO>nd72{NO=yUl-{?<+0x|V*cObq zS?OL zlqk7Jw$U86Jk7oJ{d}p}3$gC=%CNvRav~yj+bA6ymF9SN>=)5*euUs+Gw`q86OTM+ zWWezf0ASVmBT(XWJax1McZYR*`c++4iD5E^gsy#yH|3tq##36Y6fAQEE)?-Tmh}Az z2N2cgfPx%*rRzlJl{*cGaa!I?9n8zI_`||jC`h3;`!b;S9 zeUDB4e#oRYXd~p^!&z+paU+4JublsEX$IB3aye2uZSlx#&lY9uL%6bG{^0WDbQlXh`RHgcq{I=o;rGs1H0Hf3Pd%A_17fH_BdC{MwWA|l`9#Q4M{pJys%vQTfUBLj+ zn*X+irtK^5wPYd@^ZvHkLFD>>Xx7dok%J_k*H6$(Y|0;;Wh-Rm9-Ww+!PeP5PI(kd zI1JLbio+Uz(f2#BQ9VB7!HISR_j~IT@VI?Xr@TS7;B$?DCa|w`rz z2^S`2k0;vQC+UHkXj!_U>y>er?Jl%o$Zugw_ML19kof*O&p3Mz`@cq2$|lw7sP}F4 zUp1GGO14)Ad}Avuv&QvTsP`TJ=M6VZ6C^z@D|g;t(kqSr-^S7O{}`S+As`wx zR$H*2@KQbe+&3)%&F(D)+IZRsnlspV+>Lk5qf(vD(5t_UO{`>!(-r*a=8s6LRzE5E zGz9;JuM@wq2(iTo4W-Kg_EMY0gWFCYaFRvaa*!}2J?$E!96*hQs==TfLeX#@qN#o&kWCM8abN$(MP*6N`_ku1)vGs zziL-~`2xt>Ta~6F5`ZqE+UR3i(DzPh9{De3=(sAg1MuME`$-b-VpJN@%0R6kbBSa%7?1XU3$iUHC$Kbx|V2hS`ld0K`H`fW=7MSVb$rD8*B`=L4)FI zpDD!!+kNYaV~oz;v?j^2X=<}RW4HcPRoHpCBiA*ty`1uN=i%wU;Kdt7+-fV$7xRs! zPg}Y4p5)2EsW(r>+&cLS-%4{qWIAMDlbo3@pM-aMethiP9WxBMkNQFjx8&1*LmvA= zG`=Zno-l4t?=41Ki)Q$>vT@7u%(67`Sw51NQrX-OHz}k@?qbeqMSgIAto4=zbwH0M4Pkz_0D&G z8xORD?9Ii!4Ru}GbwSu3Ve)bz7FS(yQFeBel*lVeJ+X(t!W~U=88)Uw;QbLEn`ROkdDvHia>uS~N-ZCxl1zc3*|Hzj&U!P~`~$SV5y4fdW_SXq z$>C>(LS64u_qyp>=Z@hc(0Y7#;12;kms0H_=O;?D z$ClG?&&i^fK<^nA{PtGx1+egvsA-ZLE(5S1yV|t>Z?<7p112wkS6rFGQYtXe9l@V#4#dTjo;#HHgy{W|#oe&s3|l z{4DYv=x<(#;xHS0mi|x$rS?HfML+^#@TBn=C>1nAjOQi7qU&iVpHO1jTtJ%BuIbKa z{BT&HTpLYQA$LjFwnrlDw0-XU%~d_}2>Pq{zr)sSOiv%)A4uWbNjCt2W!Y0&OK^9= zz6puxF^qgZgCKX-fX-<%n4ZW*8r?UNEO6KwD|xA32WN-rv;Al+er)>DB2(FgLrS8D z!`m_|5*W1J7YzIm+s9I0`OB4rUvu#Mhb!q!hBZiH_GX^NYa61_n~^>w)s(<%S}e%u zXcnJ^#C7+Uf!3vN6@Ot}TgtWlDzy_wGW%}X=M^84YU{}&4=A1A^LS?>tPxq&Z?_4ifOqBo#wfJxW3er#;fZj$@0SM( z-#h#Wxm8{#Z667;D%CUCB!HeK*}g3M5xvZ#w>8G&Ee#epKoMLt0m`@c0;S~ACUW=V z2OA+}GQejg2SBX{kfIdqP#4QxXi%aLn8atq=jI!|-xyQVfh5aTzVLmuO5+cB*LDS4 zQkzI=hSSb<)lPV!#3r)NQBkKFDZhEHwlKNeGV%jJYn2i*y@J}XG&(B#s>#!|V;&^* z1hJb{fxiBEXkUW(vI56Y{#a-RSho}dMv4r~dU!xqZ7lO!SO}+?B0B0-ft~i3L|C>6(f1$uQsI#ify+rfg=k(nTWx-J>TMl-XhR(J`5%DgNSS#wfao*Q=d(`&g=#RE2Po-^qSdQ)63o<=jKi(B$6M-bsoh^rB&fKXgVOR3lh2l z%zJl#v;5=g4=3E2{B$eqGAuFcFShnNC?NUidJ$_zrAk3mjn+7a6SOW#O=4by%k0V(!$nZJiBORK-1Ji5(+d86z+FTHFr$3H_Qm$* zJ*BW}DmnWt$Tjw*h{Pu3dR#1?5?N#nF29Siczt9zDaB{(pU$UTt~C^4)d`tU3!+ie zFykKNGZ$zWeAnT}e^1EzM`q1w2r#1qy({7ifSK}!`n+oCFO|9{jhzjp0J%mqKXC@+ zWls04LK@r&scEgk=-)yi9A=8|j^Qx#1lYsy(6=4-t-K{*I5Z8|ot8kExJ^Oj0I#oE zpRPP{4$Wo@GEtu=?$t|xT@ctwn3QQCxPUuM4B+$$*)N(rli)MiKI(VR9A}qp`vHFr zZ1c@j06^~l?2;;hKA;b~*DH!#Vkny=kD^*rr*l`BxtZ+(0kxEP(<=pYl_X>N!p5*2 z*bOPjxj`oY08NwPr4$x%zz_vc?(_QyB5-`XR+#%)lUpmeV1%6NP8~1c?e9ta_28SM z+Jarw4=wbPfU%=vt{Qi!*%E%+JeS`|rAqeol%>V1Pl!t(7s4%yK^w7-i-#T1hug&q z5O-ve8%K4n0Jc(HVQh{4qwj9^PHYC}Yr!vv&@M1Rr6Z!W>bba&!K1BURBAfHxY~}{ zTtWUF|0jnBKX{YX>*S|8{Bs$P&|5+6W67!`Xxcjae3|XeQCfgsyHEW$e#dJw?DyOp zfd8!O{u@S?{%06@VkzPq1r?xm{bz10rYyQ`IeF-5i)fhaa7&VGI9Xj}QMP%2Eyq?& z>J^T+8{@n{p~NNNZ!(gB5>+_7pZy}JNSjhh60`{sZ7S983~g;DGJ5MDTt}EOFLU)_ zxv5saj;eQ{@N!`y3w*Wk(tS&@UcLf^74~BIEEeX8|GC~X=4Y1?-rWl+a-VrO<8=9I z{15>jFzu)}t}@k2C|_&Rku3n1@*)@!@FJs&rTb_@k`Tc>0QnBR8tt=X$3rD&qBt^9 za@lu*x1fBeh2xy~Usr)RPM3}vMuhLlSBd3np!Am_^7iK2lUv7-a%1kty~qy{Rx2O9 zwivM+)HVbuG2vl>r+MJ1Yoh)$SnyGsQ-v=w=?9F8JM%IOo6s5F>B24QM8J zPkv5nfFm^%BSm!0mx+d#v77l@ z3KftDlt7p7ahx^$KO<3-Xczlbk*8 z`qC9`2^(X3^pVDzIj%mrp_s1b1Y@fvc)67{sCs}j0Y0OM^jXA)TR`hoCXr(vnZZzq zTTku2AHHK`6f{!SCattc*}JBHTkI@ zel|L^rVxOp78;&h8TDumZ>B6715~8I)4y3c*M*#kwvPYcQ{4WKhy*GD@F_Ae|BFw7 z{hu}|K2k@h4`@(Ia^K8o~YLD;cyoVJMa= z0m|QCTo+}u1x42qy{s2?b;q7Pi{)cz)q}t&Sd&YF1O*>13fUzR@^IviH1|;)E6W`% z&6&$No;l=*jjWHGC)Z{J-80HXHxn@0w?ZO{f?m@;<3QaNMR@3z&(*J+z6*BXCR9__ zt0qR$yD7L69V?YV?xIucqAb8l@8FOFr1nBYDF^-FrEo%$?V`{jiJ)rrZ>EI^;{^cU z*T)P@$R5UrP8*)j`r<60k#^DDV&wm_Neoy+vbT;aHY3SH8Zf};u~jIJtNUQu^(^Qz zBo{btechT=E^m-(P(A@{w|nUx?hbJYbo?2IS&*1O8pg&Hk%|T~a1Xm@n9>nI?}|=i zU(qA_dz*E@m$yx{bJIlLH`b?Vyk^tif@_5nfS=UzcKK~>d_G06GIQ%{(^&o&<+e;{ zwo`J-)Ufkc`IOyw#?B^+7^te$yb^3hJbP~495Fs8FIrhbTx1t%#GMWuuQ#Z-BdS7b z0MS^Za+?>dVqig>)vHhongR<|D(Zy@)@K>k11x%@+7i1r0U6_)>f_=HQ>9li+6_;1 z<=7>hvTX_9gy2?N=*n#r$8N3(K3!fTZ!SRemUq{XrC5plel$X^FhUdTzkDW%inv5y)br z0vZTIk9&p8u$=yrZNt1d!Tbz+;b_)-t@!I?_MFvk9l)Fb#wr55$+f41jWNKSERgH> zv+;<)i!%1m&HE?IuoTVG`nO-%74LYcq8E6uqU&EkS&F~+gUD_egFnC6y!CA&c5?XG-(@<` zx=a*V+CV&xQ)T+Ea3c?t!)2B)-j0c;#LRuGih6+S9KdFl7y~s;@2GSJ_AbwJjRK-l zD^%+HYH(@!_@Kg-MbO?VBkdmWQk6M?@%@8A+f-GP>k9e}R)j+eUxb86nm|;Ad=RcOYTlfRQz27{Dk8-e zCB)`B60u$l>pBiu^VOYv-K+iU&iC2X2PZ5WrnK(le zEGhce+Nr4Y@l6p|g0_!rhM#Hnoc0#68Sqe9c}85BED%dpM@JusOtDg_rSoL{Uybt4e?E?X;zF$l1O{4nMw`{8ac_=(nA} z>@_?8_6Qa9i3f{wSs#mn!(cRyaj#M*QfCQZ#1W+Q^jDhfm`ljLs)hG*xpnKuZ8%4r3?Te4vKkVhGw7=jsTUX}2gIS2N(It~!Bo zD6yl1of$k(;^o>8*_jSp7*%YT0@7X4-)WE=(aRFV5BZYJiDA!kzxsNXVP5hEJ5qmF ztUhjUrNyWJ+R>s^(Jgg_fUb6#IvA~&l-}+zy9-RX>o+Y&x|h?WRM$+eJ4B~Uu>{|; z>vd*qPn@n5Jk{+Lz^i-HklY>-)zkhWXtQu?{x@^OY>wRSCS*44aRs(jA%8x#X&;{t zcun&3xWouRvB)nil6B-*+KKkqg?CGSvZ0{+V}g5LOK>_{fX2$7Z;Wa zqI^vOjMjve<1AIvUZD6Q>s}7aJK0yNHYpsb98QmtAA5KKmKx4COcL%~djRXefd9=xfx+J2U~2 z&_$d_aFG-26i_ah69T0-ywI<$jej)11Sf5-WGT0aSmf@S^tR*yR!fN?sJhWgQ=6xlXdd$6#CU|yWRyktnc;Dsg>pT| z$|&&8aOMUJq@e(UK(13b?_|3{?eR<54Qf6U&CXq1dmoJe=;H}gA!NSfebQ@K)AzSg z^X7$-md%7FxI-yDpCA4Fllkn`6>4{P-pKz_Zd!{QwR?!34L&gJ{?phjXazp|6<%0& zo)G~pkPK3JjoNx+H7WKMCBQ_IKF_@EIEwTzKC4@;-FdlkVDs@n758}sP3}8S#-AS8 z^V2LR3n}VeAW7{`t-z@m)eS%h8hiCiorFdG{qg=QTY_?XhfSJD+6RC?^ciA6oqIQv>QA$mPo4` zvsp92btRsLaIy=HzMpovG^pF%N?dPOkH6(rh4}-)5!T)|4ym9qsM=Zi*Qc^eu#TX) zqb|6Is^z?Ei-Q15rpFr7yB2?>=R4=H>}(@myj787wRargu27g64#Q^%s{!~6RHQlLswiV|r|;ddH-DNWO^ z^Zq+Pf;;rmCr#7b{}lGRaVy1#T05*`|0VFXWCN;XKRinc6w-4NT!4v*x@p2*^|(^B zQGgb`0Fr<8+{6xnb52c^5|pjNjAqnTx&#cCw`Z>0r{*_H!+o+sFwA=r3y9*dND3-U zWh^`H8%8{!J=PZcCNHT><@e_OsYaFDLmVe%}@^=*##7$>9sMv z2GMcB?pP0-$$JSffDGN#`QuaKt90*G@k6Z-;(GqjB@?eEGQY}M&k8P5hvv>EiWhqy z)(yhD5`9hZP&|4b+c`Rr1l?%aqNq!RY!T2Rxa09~!zEszvx>`j%PMz}XK_;MVYy2w z;})dUYJRWCVz7-|@x>Aj;P^|2qmk9eCXf&X*E!-}lNHoV&=Y&PdA^f`1OUZ9Nm)vDWa&&tGjPJbZffsj}u~XYz&ZTzum&qX>3O zw%Xs;FPWp=b>`>gI%%lnH{n6UC*rEr$4p+&fIQddGa+Wz7z?qFKf)h=?+C~P{_BM4 zahp+jV&U96>5dHD&qJcZqI;HugWJ};i|K9KVoyu;gcZ32j<+}L`MxM*;HD!uDEQN` z@IWu&P}TarAr7a`QMB%tqA{drhHa-6r`c=tZo0ESO05|wh+BTsfN=M>N-)m9QEeEO z!TBJHd}Y7#2p{xknhSfY9ri{^i1lZgX@o_&0`zbAY|HD`BP-b$SfL{)3tg|X>Uj7@ zp?;v#QRvl6c9ch`ZS!_w@cm+%)&griA zHPKipHH{5xSBR_mbc00o>22=}tPX)yHyhCHF*1L)>wzRi5g#a9S zd~ERaE@HCjyv*OG+?l(Pbx2Atl5uulUs&Cv__**c22z4R6 z*W#7UsaL_f+2h=~%@t;-_?JlV0Fp7Lw2Z3QZeFc-*UE(=C-Xc)mcJ@>&S4+RO;ZQI z_5=5kQB!;Cs&C4v-a)n?Hce!e`i9}~RUon~R{zPKL`~bfECFs*aC*#wVNDI$V0x7w ziizemLVtOlsB{iR%q_5BxD6wQN%b&o{a_#a)Cf&QRl9Nb=eP80Cyf{qc*`qMrlP@#Jnmn6J1?*+^|R~wX;c_GI4 zyh!AIX8FOikji1iB}at9dfIruN;StHhA`jy+MPq=w@r2ReD?&+c2862pGK{Xy9+0U zsith10wm^)f*0#Mzkkj8-!|0OM!5S=wtidB4e8B@Et-1)sRwVqA~3?d`S?yZ;ojNA zi!+Fb&yuS0enc0?d2^n!&1}2ln(SKtPepXP&pgZrtn;gT{ZqO7i_L=(s6kC%se-(h za`l%7ib?ks2Z-k(Lb{F=3I7jOZypb2A9n#~U$d`u6JsAl%1-t@yO}|@Y%#VFLiQzF zLdKSTHw$Ah_E0HCvW_)NA!V)XTfLY2exCPv-~W94V?N_|{jPJ)_nh<1g21g%v|hcv zyFwa_Ye>F(9XAP(0Xv=z_EqF3gG`j|Nq*D5rxV@GNO^Z@iQ}zN*71^tsoA=JFJP|t z%M%@R6^CpTL}{45^#wXClA;Ho;lTSn3!%GU&(4P2nQV@$mGG(YaKn9@g5nQ~UB2JY`iHT40T>;d_t_>{8j(X?9lJr=JCX5H>rE0j2`B<`Hx> zqgr^)ABL5v8AdIAsA!oqBEkvi9Laj5pr=e{0eLsmC-;c59I-X$c73MfSrFfLJwfhg z6>IXD0zSqGuh{Gu<59yAbB(lXm##%eIb0n$&%BT9HXuY7%NUbczX2XlC8?oI>QZtl5xBi&h~UJ2Cw(D!6K zraz3{4Ip{Xo8d{^iNth>R!>-Sk$u$MX4ysq-xb!BiCSwt0FujlV8!GElV4&`(t=CM zdpNk&6PyQO$!2cf@+oVr^MB8z4tkeLR8vjPP-fj6+bQGHZx_Em)fqC&t;#lwLw;T` zk;SSM6>Z%nFEFIU#$G$y(wSY`Eb*6EK~*lX&_} z2=<)KJX|`p_d63cB`+c9)78(u^0wsfq=nSJpv@2)T02aG%{y$WeF@qss1-ALqYWq;nO?*tnSy>73_f>Y&I+pXYa|p zeU^I^#{1^(kc?o($l&nq%Nop&f4IY>v~7w%iJbVXhIIx?t&TV_{x%A+J$dt^s*s-L zNFhJ`+qc{lhUz`hb*^%D@y!jzE!We6W6?u-N8AmL7(K1huo5H?4h|}&$V8=8t;MdW zbgw=!iaJ?fRMGOQ=Cl5-+zDYk^h*# z?~RauY{)=CrAuYp9~m3+T1o)G3xKXNrVPmP?(2Z~-8(K~Ry}#(Ws&hDp36&XLBOoe zHo4OJB0c!;_TYAVSI&jruKi{{T`ko&F_L@t%W!OB^V)$<`}erX#g#c+?|TMr(QWa(BF)s00(DakQoIzy zrXIzmeTDNCIQUM4>kl5O9;J1K5zD&`ev0jlR9#rNwQLEFzKyZJzuoBI7>$}${`sL9 zR%-ngn-$S#!QW+}hdC`?q>L#>-qsh}k1yC8!BuVM5mrHuA>9$VvPfO=<~}YniWiDc zk_;iM`}(&I24SNHd@Y*TlwZ^;^n*yPS;_lTF3McC#FeXs@co3(Ci3uU`IIZW^O~$8 zz%yoCsx;@ac>3$RBF#KPb7p$!dH1 zBJT>4x`lTi=PZJ(P@lX2pgG9$lV#5+YHzh&!QiiPOr#sRX02%^BHmdu!DdyRrjR?NL~J;)u9LVQH6$Jk^!a{?Q! zq@Y~h>8p>VgSIiuh#Gx+W+6i&NT4+#k)!_8(*#OR{)%2YKKp)$TGdBowEXrb58}@u zum?hOzU}iXrQ1(G8Jii{jN~pnv<(fuUOiP7oRc7ke`?2q>1b}50v3k9xM-??@_)Il zu>We_W)j$l=VRK@%e|qo8dZ~9#X=d?9w9I94agCmB%D~PIP?6_-{TRp9s`YpZQKy}fi=#(pS#^W zvbV|eAyaj^ZapV}uxdvb6Ycbahv!Z!LDr;ZB(kSv#|P_7p9!cIs{)(ldjuZ+^6^yM z?$$U6AY{g@+4lV7o+v?H!BHEg!c8@awYS47Ch4Q*s(-^=QSraa=GKxqF&k$~EPlaD^jTSCSPB;dcE(GtIoDVqySw6 zk+57<)l{4$`DuIjFlPOLP0HJkXCUW&{*H)nNvU`5kyMp9xXILi8D)P zK3m=S8#$)MY+JMDsyo-{G+N4*@BJFS(!s`mr1G=z!8$rYhtJnSRoA9zrOkxvE*(<~ zEg7Y)Q-ZPbRE!vO+raaBp0;Pkdx?p~@tL9Za|Ry_hh$@rH@Vw&g^BcT&UE*kI^w&9 zLAjr(`?Z#Sosn0`_SRPt(vx=@!a*iA)A~|xHrwN36aZ1mw5j^<; zbK}L=NJ|5eaP_Niw@41A)n}8i4Qa9L1TiM;WGa;W@VTlJbcPH((36sq(pYEvuw&!f zjR%(OOAFXK&He=&SM(N1{x&03)o4r4*&l({+Z-|W{L=6>JWAppKP*e{n}_dMEO6*SVwKwp; zUELp2j-3QTF#PHLJf?)i>7K{6%mGBZP9x)4hD*uo4oLkhbTmVXeh%IWP&g%0ugxDn zj_Qy8tjvdukAOYYUuqAtz9A*-){;^_La_;!HY6p}P`lDz8IWemV!zPk=ZJk!_>lSJ zsrksS3ZRhSnV2!>`}hL`t4xfoat6j?0NM?Nmj7v{xq@CvD7(^o@pyrAY~%3h2i92v z6DB=Z8`YHK(K5Z%C)OF6i8_soHCVTiArT&cfDV(F<~McVq<1!MuKbe}%g%;+$Jq!ze0gTV$gfy(1D`@hz(%n=tXJSott(=`8909OVAqDI7 zT}wp)>2&Z z*EczQB)rK~iT=hiL%}}iX*rQ5>T(>*6SJ=uYxf_BtmryLPQ7+sd4Fy`GR+nm!EzXV zpX^j7lBh&+*xxA=4^CzRC;3rBaDQA9XhYd(BfOn1B*qxBH$*$0`QlByR5aJRb--40 zVzKAk8hb3wY<&Rt?Yg(|YQlBoN{-FEW3*P@FbK)$$VU~oa#ItYJzIPlACp)plVsBj0#R2*IK@}>S1Du8M?F_3LLTsV*qYf%{;2;9{QLWD$%WGMV?R?;u}qXngwm6u8&$YQJ{WUi z++eL^W#lveD!Vmxpl2E8X`6)-bX;>ah-KGEd&(||a?q^3nGB6>epz9cK>g;+OY=F& zLC8^YXVxIc=;Iye6gj6{q}pNYon-mm;t3G8hdBenN6You)i-6idiRjfGLw0cboJ|< z=gq2rBK4daY^kPv$PUsbJ;vBr5S~(^k17Cx%UEcAC|+OjY!qtIrgx^l16vYu)!Nbv zA6*^`{#eGw7yG2YHyHd7y{3$OoiEM2EpnKpLR%eEa0F>xiNmw;3wqcT5KKl#x_N({ zy%A=bS7%tK@|_!gI%3oqAN;ehd;!s{*vk1}Y%!roN9*}syj_fZubdgjb{~!J7pCdR zJ#^rYPgzOOldp347-5nxT|1icwxs<$ehw1&*^#(}D-EZLdvCda`ONo>2~a*}Lm5@OQWU*@_-^F<749i!_-(k?H57@IIIM_6423LI!+7l>w za`NS6WMtmc#1g0LqN(avEf})1P&|hsq&<6t%!J5IxA;~MNm4!PZ@fyA$x;l^2KWY) zG80vxBoe`zYRnTp>i*zlR0_IwfYh&jn+y=e$F z8-?r2x~Sod8o%H{--!g8xc5=y{80&qh7UZygpxcJeQ7Rvov!1R8B(*)aPwmqXF@@_ zM-_7?PV~ij(qraB7w;c{AwjG}2;* z3SVA4T`z76nTnZry)RB}7FEzZ)ZFE1AJJK)_K4aW1j|372!K@CLR!hwpa3LM^$%bt z4dfd0&Op_!j;k#TThu%Il-UUdK=2y%&MztvRsKbG>`8WOIjcyKxaPC?68ml~Dk%b_ zMxuSA_kwBb>V5KR3X)9pu|T&%{U-Au0c`V}&L_P1BE8qavU%>4oY{!JtJrf3)VTFk zkgaF74-IL? zjZa`(LdJ~Ln<{_A_!V6d>PjL+T}vV(jQtHPCXRrRApgZKI8ud$=Y~2h8fnc^y0_6( zi`ov#NzeAlnZcn5pHff91qrq|$U0^c&A}m>VeXIdCL5un-yqC4H#Ya<8{^-Q@OV6w zN;{o0y(@cIIAFf?CPvA@@-TV@TU&0u&c3LUWCfu(X8%;fw<}(2HfK)YtoV-8J_fML z4)ctZB(l<%XoRf+4U`b5Il_BuDL6R}ko4ibZ)eD=hnTLw0edL>7JGOv$lBii;CnT+ z-S@HeoB$6!d4b1>Ny=y2*`@-Knm4%`tk0{vAo-}@d4_mXO}toOOS)mpRGmuoNOohc zUUsMCuokW$#%WDLIbAH=oT2MiO!}6>$Ve-%=-(Q9MQmHZ830;;wnlz=(9`tl!lx@l zUeq|ac`jl?%j)3!`+gs?TeQCt-oQDC#JwGk~Z5% zN25O0gJKrU=*lhnmFSLx2;zYC6wiaM_W`g`*@WpAg6XFI44j!>|7$5RJUNCqDmt|DRxzS7DaYKo7p{GCu2QD z%>jt7j|rpY)6vJgCpHxhRU16j)O%70f=JT?!cHECOmRJ__+V1Niove-EO&YG6uyw0Ey4e(JQtr zcr(xBQg4ZrBX~#OGGXpfk+I*o5K!0Fe|US0))VX^Ao>Pio!4}_SjK1xh^$}vda8gX zRRF!fbGTyElO>;Ay|$_TnxZwOIXnq^_pFaV+hM>Y1GA-c*KoToT22hc$84Skh#%ze z+b*=Th9i8O9)bKTUfOHnI+|CX;I5)u8{(5UKF(*NYX#@za;_*jX;})Ul2NV{zg`fI zJ!9uv!PRD|r%(c0s40(!m4auee{vp13!fxwK7GqOy5P#lRqDj<%qW@)o+bNcFYniS ziU6b=o##YsJRP*f2QVN=K9EWJK^=NCKwJ=| z+`?*y8J@=2c%c9}(}|B1+70JZv2606tXJ>sZWN}5zWvl4wlasczcyqkH1yPv`W@zgyov{OBnWw9)Pq-Us%m zYt@4Xpvk#-r+;4F_}ao!a*G%+F5`+Sz(KSRTbg+FHbc2ET6lt4i|r4=2OBG5T>C)A zIj)@?BAx`@mP~=R0vp(ADCy*Pf2^b*eKY@{Y=Uo*Sk%r_YnxP?_PZUo zIJ<;-`q+Rf;4&?jBMokzZk)&WKzVf1^1NER>>sWAIx0#z&h|@pR=- zHLt@r?;ENa=LDm%>&?I%h=KlJ=*+x1A@yP}v8uz#0J@_6xp@lWJtpHLd<4dH;LyaU32)dnp&A*KePL-hb~x0-_Li0)Q-OYN&!Hi;)wby%NdPOP$ZnQ%$N`PoRdENN zgAl6C(D}}Yr{o17*mEsaY{wb03zM@saVm%Cxu%1($d_mV!-*7iNGnGAN?)O`>~4OE zZcg7q0v--$K0ycsY_iG*Y&Jb3{qsb_7a=RIE{kM*Wjec>Q4Eo(0o`$s?TO z{}9198}CqYlBBOdFdoH=-{~)qVgZQtC#lNC*;gB)ewYcYBHM$HDm2zx<}U@`L68#- z^kx52al{QiY)8(;TShgL$W6>pw_4}9{I96utY>Y9IXd~4|IT$n}*)YYAI{xqj^OABRU9q<$ z#{hoOuz3#dS;oOuag}r{F2tX+)hP*5O|;vwgJD>>8|rP@izFmS!aYwpI~9TjI+N@U z0!tVi^r(~re~B=)r*P(Ymf5)tK*XTYuZ;>NQ-`exkh?12>YxCK)Q(WzP^phu2I_8F zH=y~0U_mAv1W2M?i&X|{vQd2$%?RTb+LV$Hczam-uMS?S4S0tBsPxF0s7nmO3q9ro zc?8K1ft#~)e*(vVeu>^kCo*HSYqEltrOVm#iz1vMB_91sU~AoB+JKyrQm7jnrqVs( zM(O(T&)Tluqbg@a0stO=BSc7{hOwz^35eCNW$qBLU66rYjC=o;IzvH}BtAwAAH)AY zR-_MawH8lBB>c>qEao@NiRkFzEHuLaV|Q{iLda$dllxF=p3L`IGrwg%OOb;_F!nHd zzN}I+a=#CLHowH{)j0*ivXUAz+kL3wZCi-owqU17qviJDEJI2!-<0b4_@NXx z578joF&A(hOshk)4>@1v%P?qY2o0Pp43ao8R#o>UCi+m`&s?C)R_#jRbb7$(__}*v zPrFp?0QrC;CPb>|APJ^&>=h?L3R7#7J05z`cCh9b>E%BPg8@@UN}s4nAIFnETnqJ6 ztDjp(5~MEby+FP76`X2+pN5m0>Y1KenDQUXI@@lTzoi z%n}f{=s0xSfsg%(lZU&5!>FExHu!QF{{DIIwA5m^`bm?*r#;&&qQV${UCD!bOXyPg zIzYY%w`Pu^dU52;rxHiGnFih|-c&06C!Lspbp-(e(j zqx@O0_ZDXLQQDbU#&`5W8hvo8`QEp#EoKksorSRP{6{s$z_jbCvy@Xrw1YxH`}Qjy zOZjxD3HQAE1b$9q7+!Jck!9WN;bKX({OnVp1J{9%ZB~CRv(L^GEVy1&tAYCrvoM1 z;Z{y_E5Ir}O((rr;%>y3fj6}yo`09quZ_h8b#?Fm(I+BJ@NYiSZ>p1C0ZPoHy&*Hy z*GQq!1ol<%ZN{;b{Vi;bo9--?ZcsU-)vUQdXUy=BPEP^qn-+T6x7~U$xA3>H*$S-<~cr;yEaR-f?txTIbU zJ|KkyN0lSmth6iWHhX0lOHzuz>hf0pxjP1~k>MRu}U5aJ8Ne?hN^B^0B_hctFyNO z8ludmtc1Hp?!2+YlSR55#m` zW#d#gBUDHk8#+$(xJ(v~OQZKsPNv8riq^dbtCf5jYc4)-QjaFudVJ+gFS*cUeb(~f zAAZu-NR_bEULfn;aV{)D`ivP3_3cvATusq(3q5>~jT;xc6_BdVM)Cg|9^U{og?K5B zs=owcr3bLx$FHMKY)`MI-F}lDd7?B+$Su-woQ1#D97PZ5)Mu4LQWi>Y1Z`-e#+9T& zdtvSKboT+nS!a_*gZ%Yo8V0mhbH@vx5{$ch$&=4lVxMear4EzD62gxpV=Lo0mI4 z_PHuQ(-lfh0y-!~J9xj_dKEl8(E>cH1Q};?`b0YfMdOmVH;GPt%4gzZ%207JSpN*E z1OI*0!)z370C9W8#32*)$x$?tdF@3h!wP=d8+LgLIbwvFv4SY!v6c^b>U#_%%ovlq z?0NnHu*S~YFnmlWq#21O-P9IOh-N~`ltY^DuWzk!1vt_YPJUW=!|^0f zHgc|7D>&5NhRV0XooLaCv}yup$EJoowimPZD1L|*Y4otS$#32kk7*o9tR{tmZ){=Z znLI#k4a#bF)_ZQ#kXtQRzvsg9SsGvn)Yi8ke2#+R6sN4Z6nm_OzpXcA*v5wVC29^6!NfzDxvLKvKuDdq{?|d-gKWJ3lWf zFy*LY_%F2Uw?JJDsx-Z z`$eG>aUfyt-;tDnPIOq|zZ#}L-<7cq7zzn`Dh9Ik?Yr%U+yJAs`$+v= zQUAmx!5so;C6+w>aMPF&Ej}z}ik~1$rET9PmkOEilf%BY~K-Wf{4G$w@=%%c! z_-tcT8c(Nr6IhaLsnVnyyR2{@5exZSpMd(fRjqUiW!lDF+FwMNJ-}^1K}Ij_W7aO% zesRp9Jm#0P!(sGXwfQ@7(S}u3W-KhD#Od{M6`(~xOj=6|e zrZCa>%;wt72{I`8k!f?ciXxkt#7)J9NW$PI$sh2Sy0+Lb1Kax98djtFxf;f)MW_q) z9mt>bm&>+F*t^l@=HGnLa&CCE56S_5X;xa%tqEv(Z*n9<+9kKH8*yP8wTOg0zcyihWA||U zI}!ExxrhS~pf9ksZw&m%4T-N63t%^}|Nj#Q&Tb85Yv9J1NQ)M-Hr|_PoW@Hr^NIcv zkGgodbHhHg=O`#miuA=juS`tF@ztVAJ}3kUeZ8JwP!bLFLkw$+YjT>)o0bQ?&v7P? zyoR^6c6i5x`I_&fJl{J%L%9g}#O4i#k;TAtkE6oJa=S==j0tsbsIAL=PlbOZzu%%| zW4I!{)yxu;g{YJuq4B<$tMrAeBAZG)FTT-aqb45=QPhB7%r+I`9GVDSc`Y7d0@*jA zM^${l!3EbCEOQ62qml1kTpru95iViCtBa5X%43SP+mV5|qU}Vxe%Z;gLbf}HUH>tJ z7wRc{R#6Y`-Nw;%1xWHv%x-xCFKy14Zrx^(|7z!bB6@#c8#i8iQUv75giguwQz8IE z*c97e?Pd^JbAyu*I|@iR=(up|s|>CSrdhN&r|3B`JV@E43@s3NT<}Vc;~}l$HICSn z6PKP(GC9x~zgh@MaZCo%l~eSuy2x*FS=1wjEOs9xkjk>5@NmRbcoREv!$x=w>@(3R z!9--p=Be23xW$7HzLI1ItBi9QB{vRujAlPEgp-|`b2PmCi5JP3}})#q;TXBRK7@O-8_`+IYn z$5-e7SVzl&g)UA7So&}f=Ov){VJfcceArLY*B<8Ru-#mYlzIXGBd%r*Aj=BU3EmH& zQ6^GgxXXk+#B#(uphlw~_MA`qg*Zm-KJ0FAo3Rt6&Isqt>E`W={s(xlGf=>P4JbH} zaj66Praai41DTxzGjB`k>O+g)3wpO^ssGE;f~Nr5#d z6Xta7gc(v6e%5xwoao2kO73wh@=wl@f)sTrxpu2aD_0;p;!4++V8`C60oYYS=-T0V zlBMda2}|TPQ~&BuSn7M%V*Kp7I`!L+*Zduy$;3&$UI;26F!3H;%0|rF3^)>huiRK0OD~m~vGi&GC)?j_`1&T4Bir6A!x7jd ze2=2ea46KES%w@o6*ALmFVMEwy{Vv=(|vyYac9d0wKw-;QExfKK@m6l&fXeu!>KNpP`{2Mbz9Y(ER()Qfm-e9K)gqUi9+L~?Ir zL{tu}$??ya@`BpaBinOXr3-VM^Sklb(inal2p~$r5`lOhRKEw}u|*zeg~=wP{$a>H zYfe7~Hqor%ObnH$1q=;ZW=@o{Ck8KPBaMGMWBAT@7FIe#Lzw*uH`?Ju2E z#5B&zpf#{jeZlb=TWhGdS8xw*|7h;Wn&b+)W>Tkvfu?U9}3jA-%?lX;ad>a zi@ZSp3jEpP3=I?+g(R_`3z(yXyM*f2bBGt`rLt&L+U}E;ANY6oP z=fj*+RYn47B7cMSJc*^AN)Zs zT^D%7G9 zB<3j17FthI2@=bVm-+lw+~U0w6{&bZt6K+2A?**YefWM!cMeOLPwmxypiAw7qS&3; zNqvDyOAbCd$4-_zEI2xZ1t`Lr_Di{JryTqRf=V6!TQ z7{-xr<&L2%s34U{j*3QVdTZcZG7fH-Y-_{3_JU%)d|HZHsOhzPk+7@r58-kqhS+kN zSfaEU%+KyR-bi4c`6HKNA2O4!g{lQm&2-bR1S$T9A>#o@*>3mJ*)u}h`*VJNI~xE zhg3O1L~N6j z03DG=fS7bSF3#+5reekfd^Oe>mcusKLFOp4upoQ!LK6P_2K}@(YBp3~1*m_4vX zoP+OJkvZzdOp20yQ&yd`kkB6nKL(Xt6B9&D?v}X|wT9uq%jLk7&%VqZNH}D{@J;kq z-GIzqadXHj=ul}eMF*nkb7rW%)fmIM|G8FbmPJDMxkuoK`!@4Y!+I3^vbFn|Z{}41 zSa1fMz6em8l%mD}GUQoBj+maE`XW{waQvFi?h$IEmFAc5OAW!?Wfh3S_}=8P3L9m2<4TCtOR z^Yz5p#Sjz5rND$uHrHKRBJA8OiN$!Elpr`bFF_RUDLBL+h)pOQB1uqFf`_+i7Tc5Bwax9ngS z?f_hRrMJk1jd59SPL@3L&tu#f0iw4;y~SQmBt?x+s1akTePr-2(hNTxn$J6EkQ=C58$(b zfcrX@_R?}(A*v_c%2gLQ=?zq?V5+&m+~;2fNV4wZAK%Z0?GHZbanu3RwruP7IVt1Q zqLpk_z6NOL`&I5$(XUPQvf1j^dXQ=AV%7UiGH93LPME^iz)+qUd?dXk%~4bk|c*9=yC+&`zbsaINn^gZyy1KzIXnMpt$b zDsNl|yV^wE6@G%zy~5)Pbf{_E-hfC&?cGm}^L1~T2!oFv>Ma~1Zb;lu?bV|gi5vk* z>yqPztY-};xk{o{5jsv+NI|xe7k6I&Y`q~ARYQ1}6XaAJR;HKHYii!=OmCrf50Hpw zffrJx`}Hh#vDq?jvR0%G=5f{|;PDnLrpXRfT^D-2CCSEAD?#R07G_03hD_Vsv-47A zQm3%*Pmq+MRC&wn%~D55yKrfIqw;K;R{TSXHKdiS%K{1IIc;0F*K0w}zv_Y`>X$N5 z#dLq87h}@)VYdO7u_2^CNvJdlayK3vP)|Qd8w(saPs06Ig1mpRu@Xz>R1JD9pDi=3;2fkH;Y{VH2}1Y1JUa)U(#ve^jY#gFn!UGvEAmB z%Q*Kve@lAW&o2EPx3c#T&MEk}CC-8#j!0__x2k5hZqiflU9o|Nlks9;;`ecPHI&nt zy)UA!M${s^v~XOUX!$9db^=#}WUH3H>D;k?#qU8!!>B_MpfddwrA|Z9FaDU*4v=*P zZ}T0f6Q-S^El&EFYLy%}zBPXv%G7-@4wYj7lca(9F7DKwyV3K*G3IyG;$xRhrXNo4 zs**>pBLo0&+tNbqO-lTs_U(UMVhsPe#1O`e3v4PNH(h!0l^#z6 zq3tfC?t!}ctoXz57`?wcTo#aIzrWwv8#mInt{e3uQ>LO4=9&q)(8!3Iivh=Mm0;LVG6`E=XXl@txAn-q(wC^5|R*$ zxK7!a`~KMe)L{CTWhQFMynaKPy&XB9DmMU3vGuKS_-NLrTZfaE^puM!PjJw%^`Ru$ zho2eSViKhDeAtpnbS+(nMbjM8qXpvwjJ;%B#lIRW&r$;CYV^uvqBGkpA&(-fHkQ4` z9Jhh831vc=HLzSvnzd&YmAayVt1zj6(T0)|M<-ab_J?DrcjXJel$}X8s=3aU z`}ob+$L{Akt!!?Ht^m)$TQm|uL%kRuaaH&E*K^<8)YMVvl;OoEGsc-i^JVfJu?Vx9 zJT-WVE?ilbWa%O6IxQZ~tKpImT_7}Lc~ai&%$hgGufy*(iHha>q&Z%b^k-dCg7m&m zn)T4~pG-HYhAon%wI_Z#5C1qQVoa;s@=P`fP&ORYeehQjC}SvV^wpXxIxbDYb!C*|n~(wTgi4^M)Imh+ z+I|KdA^%fy<=6ef&)e9iYQ_u1P^)N@VnX37$q$O%t9f>Y!s{dB8J+5uJDOvE8**ON zKQrKFoG}_OS!m&q{34kvV(a4S78FBW`aQdnVx+CUgTS;YO$PdHy zdDru(RD0pO@iBWZUG!q7)q+&4V+|4l>G&n}Av71w1@1ScUabpB2zeG)hlLM*(Cnto zf)GxlZy97M=mHKcw8Lj32CBP`BJbDgt2gr~KP9;)z=;y8$kn%eJx8ArTg0KlRB!8` z8L$Ay&)<7tM%;wt2uC^vK$`>PYDOl&XDxemM{v3%rI8V2s+{pS zs=kP8I$TsN-sAU})10eh&Szo9!pO3F{{KkS2%<*evHK&7)DIem16L&6Fq;RHN#)zx z9bElEJ!nK77_7k>A2TFhek(LZHX{__VA`Vp8TlQyQsMTk}VynsgBGz3@)md0?bBX zw+kHDs$7GwlgNSdtzoR|kvA+S;j_d1?F71LTCuad4rg=9_3Rc&!gd>!a>P>CQYgrs zz-sSydS?Yy^3^&dP3y3@hcQ$ot?kivP=-45qIAe|Xb1 z4RuI!Bg0GZ%!ja-hpx~0oR)Y#B(ttpK>!kEZxqFCfhfsvv$wYQ*w89?{~Po&!AUqs z{Jn%AoBC1D4JaI1#u>l$9zU5zcx^KHI`Q>*EB(_%D=iz!Pl%itb59?BnHMtkRfVp+ zzZ1aZqq`?kQ}Vwl|4E9wt6ndY0$F}qJP*54ZGjnYdU7r3mfKr#@jr4Y5FF?vB&w+c zyAM4YsY$jw_+5ACR~T)lgX*!Wx9ETX3xTsGXtb-}fTR-w*W7WG+I?V2HuUE!z?PNd zh0%v80T7N4-)RJ&P*mFk3mCCjKx=Ari5r9v=WSnCR-`}sCy8@%vGQ_uyg+>~E{oXT z+BqfjsKb#pJm`$;a!eVaRA{)_s}O9m)W-97BYZlnNBTQ+k_=?Ln%Ar;%N`hh^J1^M zsnpp1CSK6BWpKQC8Tp zjk*h(kbS#n!*#%J6Z=2Ti%c7;OtV*=uRs}&t<}Hz0E3(OZNQ07`y16c)b}NaJ?>q9+Ht3|&># zz$Q$Sp01Tpz*3g^w51K`M*n%5%OI;+U*HvT>v7AaTNTU6G?VN^wAMN-MKNtB6M(06Wl5etHW$BV@cM$Gm*%~ zzr#gI5S*7CP>1o5xkzP=H^w?BDf;=qquoE=2g==oXVc`*DW`+a9(o4r=0rc_hmCz7 z(eKp6q?ly~p!qD>U2+ueYLXP^2szu&kg>>ZCS)$9-d^kGz(+@J0hJ|jo@>m5v@cUG zQEh%UO1u8lpw6zP3(~$p+TeBqwGDtYjeO=^07TI|71Ec&HZ&w&+zKa0y$*eSI_es8 znq%NRL#9f1*)-QGx`vl%u#R|Bv2Bq)6qqH?SAO#nzS(rHwTAS&uK!IpCd{MbWIf$GWu~8Iku!3%Ns8MAhJgM?1Ch+5SHVKlj zxxsfpg#gI=tGv6EQ#tiN1qXi~|HG-C4?A3Ddj}RHJEZ!iQ!9hY1D8teuz{j$QYqnM64bXT2&6& zv1ITAwIl5~zR=$91xgMw(C&F9bSbzHOYV>U_?$lU2IGVS<&o6nr!88NNHMz3xv`z* z1||=~TIW2S<%JNkFAM&)8_+L2j_2NGys-05lLrw|4Nf4NA&Bvy;kf34C z#*GcCP0K3VsxyTXtDmn*P97gqSzrJ&NIGzo4Kmt9A#MW}{8CQ(5R!N#P%`zM=>@H2 zhcg|}juCGPSlbR)-uWDS_5Cga7$UofkFTyFwH)$zgKdAvtRh>8?OUFgeh~`}iQR|3 zmp_4@{jS=Qf^BY30!JDP1mS>B8asNrX0Vv64~5nDn^?~FzwD9^_f73}PzM%b!0t{L z7y|8?=$o$>PIfRaEIOG@`pl0y$DNJ~f~9Ro8kbV>;-2uO!Y zHw+*x&CJjx-O?a2q=-@~AxNs!eeiw%_udb8t^19|Hx@I`bI#d&|8|FgIeV=h(i$J6vUP+H(kGDj`AA^Vw1aa!|Uo-67!Ddwq=Q~gi42{J49d- zPSW#qy2{M-8A~!Its5*pA?Cq@Fbr!1d?R8!c-5bkX)_8}D_Kj~0bP__)Ftw3GC> zGL53zg0IFblFn*wV+{UUNz_M{T3qz9wORyJ+^i7y09oob-nU_LZ%~gvXRdM!%~vf1 z4#bQ64~ZzuMgwl_z}ccTK9Xed*5~~jpWKaB^>{}4v~@8xrVEcf`ZGHwi?%>nS^Th4 z8eELp7pgC3KelMCb95eF-w*{2W^!x(=~4417XMq162)PPM~=o-qvT_M^^!wEFQXw9Nj|lN!Ey zVrGo)?fx??nC})6gyt!5ivnmj9PoFJ$RWZSgR+se*M2$p&R1noh*1)e_y8)jjU|t~ zF}i&=vCkt3pLGa!R!eY<`#~fgSmbh^IO(fYnE^M~`C(W!aR6YQxFT>b_7NUkPUp^T7^ph zv>!w26JKb(Xwy!)MRLou2~k84Z0xBM6aS!Ma;3BOJNE%b0`B_E2or5BIMCibk+W)c z7dU5?`hRua`rNsAUvd?-E-RJO>1b0z)86P_62Mpf6i}Xe0SIPLOm}R40&mJK(_|FC zZJZ~}@`2ipI6N$f;4Elk3iL4pU^V36-W_EVv5%voPLqe-GDNR$9dU+-^PB!i2CQ+w zQ+5DPJlb%xUe4OBt$t+J+9M04Y1Fg*2Vvf>kA2m|Aa+6|ZacF?(NRl(C|e_2Ig+%#UEI1JYJU zAInW2SQ#S-V?sfYqpWVltv9x>2h>vNxY11ExbC?4XHQQ9lQBATCnEy$DHyixH!u0^ zkV9^uTCLZT)aWILHj{poukhK#<-yADYaOF+>SCpJ+u~o3@SBV z7q=0t0WXIsHH7PIDs2iz!^il>`nqE7H^mteX*@PE4VG8%O;F&YyDz7ja5V@@etnlRW)g z2B$rv_YUMS9Q(BR?-cv3rKL3m`4dS}8Q)796;0W~HmMQma_*jRymh+RlSQ0DI5jL`kn$(D3{5>J zL*G6#YHAgsO9%h+EAKMy^g|F5@P3qsufH++g+18h(ec;A=v*?3!kah{rx)-@T?|xK(qIq%?7ZR!N*-wD0}kGySWUKFc%f| zD%d-MG@uN+FQ<%MdWxVW?ohp{R^ zj2jTv{Y6T=QtivUuu5(dFTL~Hr&M1iPg)FD$7b@Iw_!yeUd#@V}zVuiAcpI~(!Wq)UEVWodoS3qQ-oa&qFg`rmnnH~dVQY~B*4 z7SOY_^|()ZKP%9=R>k_Pz9BESTMj~c{_ZN|xVK%4ZJS)rG$;qJ7j4KYbkJWzt;a_O zMGd$G)ddjLGDNBv@Z>V!4jPu9FswVgNV?a+AA2bh?tmQBWK0gyy~Pn_9&K2^%~Rjg zn(@0*L;^7nHIC7PFg}|rzl%sg`U$F*qCV(WYpP)Y^p;Eo3JkxS1`jm{p-bPI@@RMK z%Y!aHOa{qZ;Q56rgB72splr2#&~Wy z?UGT|Jv(ywyA7Xn-jfoE6RYVGcsbc| zoFQ=h_b4mbPK#+lsu)6zBHA5gHjoTUOMASn(wgEAV^mfvY0MwaBH2uPk{jHZC4t4V zK9dc}J9#MD1`rCW0 zOD^&f&gVrQ=Kz5a=CFK|pIqP~+SRC8;+lUdd~}QB*^jeqt_@9!j_*>q8WL7Ia4V&{ zLZrXm3GsYo+5;x)4&mY;AnIN7;y^>Z|zxN9M%{o0n9y+<@XY(E`1R1&`*e>faN zZ=%!yH7MXVBY-we%K=ORQXWBzyBubfCnDr&0|P8dVSb|3l(ck1G#?gB=Dg2KHQ<)o zbL;?3rmJEKzos+Wfx3{! z+=4?j0C$@r z7e@*xq{=E=oCtWjPV0B)t%40WMB;(=v|6%R%P5DE+N#HnbF}IHrV@e(LTH$3n${j4 zKP&Nv<-%v1Pmfd$Jw|D{WaQT? z9qGuC(hEi;zo%i-Tp8y^gZ&(no5CAOg+60l{ua}^zl+|f<&NbwV7wd(?U0HtfWUTJv8100Qa_z}rS zOGo=`MWwfX@p2ipWN9k|M-iq-KAvHS=8ZXHgc2FXT53zhWB4(~$kZeWGt~Z2$)G^5 zu2rJhRYMs}#>&Bi)~awVIx+7g@`eJc0v3eMsVAu2rs+qS&+9XRMC=KO5PLvqDb}>a zX4G6^FDPIMa|?YI$fn&b2KZQ&0r>ERq(S9cBwjN_mq?qZYQHD{kOKiq56w$M1FwJn z*d(+;B}vdMn!{wEgZuj*Kn0s&bA{;>xDAh`DlGv+s^$=El&8h@@;kIM@a-xe3Z^my z!EX?74nESIK_;Y=iGksZ|NBq%E~{oK2(W z`epK}Te{OzGFg62I9D@~j_4!}CIaVaGr+ST3sb^J7xp4$RY=nwwUm{%+puq%?fRD9 z3FCzjU|H`}SkjBb6sFbJ-zbzE^Cq98M@_IHonnX+FBVv*X!>sIzZEJg;cUU^N65`g6J1<5yMr z(H4y)EvQ#KUuO7CfItAj5G-jn?ry=W+-`dF?%Oh?O;%n1Nc(_$pEb;d3t<`9M!JZK z`_6hCX}`Y;7U9G%)g1@s4L}O(`YZB`Do)fD{x=~Cjs;8!9#~BI_tXZh@)(vJh^E>G zq%nUs0XPjbRBU_%7lK$&-KpO}?m+CnaIZgw6VX%I3#O_rQw~#3PRD}#P7C!|1_rmkz3z@^hHA>45e1L4SKSrr(Cwia;oFsoppuZa2b z#;}rR7&eMiRO?p42X}lLJGIb2C}c#q^V`>zPm_Y+@(MarZRTE-;s!7SJ*aB7VQoq- zhvzz4pNenHe&YWlTK%~Ff1CZ7fgsb0Lqti)$#Uw0TKSzkm4axko}0TWEIo=@kQs;$ znh77jo57=Kff&A0m)GRwnj99L7vmQ6|2|NuTr#K~c>L0-h!K=0{qVfyi*@_q+wX6? zeXo~YaAfpyZme3D)=z^|a-I)=pm&d37DX(O;J++!EWpM2Z4Fqllxel9exl8WMMV;g zW-&vZ5e_BYp&Ia!b}OerM-3&YrOfej%IS}t#&yuq24*RFGim_Zh7!{j_Zl@UMrl=| z>Tm)JALIRAEdM5t<8kCMNrk=FkuP)+l2I2TUkNN-xOuN{IFtK>-dISXJ~4}G)aivo z@d2+VjPPAd#g-#6W2;r-yhk4m6&4b?8qa>5;=$kvW8^giuCw#{P6jousptL+`I?C; zK+ml8FyNQjOR+e~%IhVV8Meub{ENqUo=fn!FY(#-ls+M5J^Vk48tm~W z>}!)RsMJFojO$1!!WZyjo&iQf_HH?w*ffj*o}{W^W6`wDo2Il{Ylf*2&OweLxn#_o zg674h^D77MZ0i`^ofm{!t!di%#WX1lgnQ14Frr>aG|fkb1tnwX7WB(W%)PC2mIWb@QDLCT|XT z8%m8XL+fCMS}U2D?_7~mxWE)kvpe}o{8`d-JH!TRVXyRlzhJzrln6vlI4|$=TYiws zPw78n*M(DeUPirIZAS*4_a&y*gAqVpO>L`jxRZX0gdXG(1<)bDCM|h}b!$z1p`5QA zkbco)LcQXecmxEH6qqp~{wXyuy%@W+`q>P?p|BFTqXvu${aJ4Nr8yS$^TF9gs~PsP z_*?a>uO-Vr9#EtGJ9f|8gRXnFQgsJua>+wpZVjAd9YlfaYbyiQ*6qMuy;Jh8iHkd4*{0^gj`R?b zPIeYHe)(5*r*GvMDdmxlY8{HU615N94GPy1MPmRy<2n~Wr&*Cog(Gvxig z6t#lewK2sY4fsP%p$O}NN5Awf1T*2UJ8}6Ahs6P_gEQ$1rlMz`-6|hj0iDpmeU>=| zt9f%USP|E$R9g_2BM2ZbQtsw2k%oXsk#sa=TFOX=3!vS*=`FV^$wS&P-&E1})s6&z z|4=N`eVQcR{k8(sRgvuSxeF@*e8xzqo&fGSFx)Pk6t!kNXDN#YtTd+a>%)s)yL~jLjOrrE_C0gnSq?LBz$_xXeJh zDsDe9Lpa?8qe5OVoL*Zl$$pAaq*VsV4E~3Mx8yItgdSsz$Kv;+LL-6LRJlUz{0&}PEZhK4H)*I`oT}XbU`TAYVlNSUD2)} z!b{C{?Ora1~xVzz~FRZZuc9DOMA%WdR&!PsxV2d9~o_4kd#UAiU}^Wm*P1O?p5W z_-l^}7Tx~(5pz~n^l-}~!9<{S{tRgrLx_y15CE;n*;-DSz-cOUp5@9|97bO z<2ab$Gyr&bw4@nPsCncy4ei;#ucC~&N%c8zv0d<|ma=^kQFD7z;~&Eo(Lop6O$6vy zPIen8|0l}L#UoDMZjeZcp`unRidf?k$5w0d9*NYL0*$I!vt`EzIyK;~fcfVMLIHn* zYh5az)y{k+5=Tbe3YzsGygk`!wYM+Hau3uS8;bW^P z+v>8eUI{8_2UCB@^lz&nIi7{vzj#Ij*!M_t#LM;-q`1*Je^@+`LU_msa;b9tIxSzo zxOeYaGq>QqKxmMQ%+p+h#;KUfkN7}la=Z626#3+qAQU-^qSurCdBg72fHiMfiS(l+ z|0!P9%{aTZmA~uH4!smYO|g3pA)=KruyC7tS6Gu~V+=d4?(c9N9?oq#QP^5r36ISj zZEcKo0w=t21#rmpq=!c8tN#GV5_S0PEw;qvyYMIUx)Erv&%@10=wm6=Z`l&k|DwVG zn5ii4jC$R4ylEkpOF{2kEw+2;=F^HENl@q>)rD$P;Nz!Tx9Y9g=XLWIP z$Rz`ZA0jE`;1m$oG+ex>_=%c24ys$@);aM0QAH2!zmxjtIo+9pCGqchr2(gKZP7X~ zs2lVvE$x5?tQjZpVLMTlZ*o`e;sT4AdGT&=;;UP^T|uOZDtwjfak$g?e*`{8eapTa zd$@f^1sn5I1h_n_NStUY8{>jmH4AfTK9=FfN;m?H2fQBFDwXRcRCdVgDb2unU_T)} z{2Yj0^)NY1i#Tf7k8OE%{y@PkcIVak@m{V139c@cCE$eg2}h1B=+;KpvA~}w_o>rJ zI||+>?GlQt_{s!@b<2>r8J(&_B9Sx&`^e3MpvG=^4-X>Q>E(B3nC|1o4uia_91o-c=v8f$oq1 zfy;HEBsD%9S)5bsfK<&D0#1BuPGFPqDAPCL*f^mHj#$iwp}%z0nCC)5irD1sSnJ=) zzq4OxTpmS-ndQ9)E;ZTi7-5cHIxRQ}ex^+*LsCGzBgX~=a}Q7y=m|@_3pC)@3IY2J ztb^XGesilo32_b*`IFMP9>l1QHt|2u`u>uSDKT0nMgIJYj^u=)p*3WwQM96_5C8fWo#`24gCfxB@G6Bw$ zYoBS*XuBb;Q=S6F7QL`A)`F3eCNwiuYY02$o5(4xaP0b`P?S_X{_dm7CqSL93oySwuNyGQu}BXEn^+pRbl3#vm*QvNcEJmI z_2$1rw>N-zhi$!AFn7xH-YAJUcH1wWnD)eOSUjqdq`bl8@DW8hd;fj)u72JjX(a{o zqI#CYKOP9Npr(E9;{Cd#+A%^zn)!z#xKL1=1)mw7$yilrsuj}>l<|P!wf9m$u^+%m zRJ0ObZWRJLn$MOmNWbK+IvCI{=^@Rz<}<-H1jE}1LLUH@aDqQ5TekScV#Vpx&Z_as znU}%QOn?S$w32RV5GuO;(I#Rgd8r@e7&O=rZ&+LLYPBJ`g3DSm^hsE6dzePnZDj~I1%C~Li;=^GuN~7D&Z#{>qVDfTC zbA4DpD>Qv(bwy-k(=zZhg5Pw4ReoGlIKSxx^Ns{M!PIl>15QTDRhSp`G1jvG5@K&} zXe9AmHFv{dexS+OAc;BN8V^`2wEQfFx>7qtaMf}4C1Zv>n~SY!t+ zqBdgoCzF00E-5-d_ilz(E!bz*)LCdkmHt6+JJKb1AzySvF48tT>eEzV!3~WXa2>bo`KEIbsfT)g=r-Zp*_|vP1|pnpV0#5CGnW>0xhNk;@5d z9LK2-C9~49RJES2Ow)WUha=NNn*sQcB1V}N2)tu>zGM};n*7s5x=pEm@8`F*90=#P z4XjRGme>fUU^fM8InCk;hyrMCGLn+`v(!Q<*ouHx%ct>Pz|%j&YA?uE>Kom#){MV| z;AUfqBgb^*n;7InL)>(KPZIQTx^*bD(R^v}fbcW@2PJQh(J@OT!0Txr}!uIX@aN{;p+rxxBn(O8|-cVDu`*MF+Y*l_X!F9x(*|$WR;O z$J`%b0eXCi$!%S{%bZyFqvT2HzLTq*|6W9fwfdFZI~xQ%sE=r zoO)^B+vh2^zy#f)i7g}ogE1)6)Qv? z1U29>K+|u^kJhzvia`@@!HC)~++AKNNWzdfFyig<$M~g&r_?p{w&$l)n3ft|nbW?H zi?p6FZ+mf~>JM`zuaT}DY0S{qg4Af8Z=HlJ>A8Q+CdsEp#>a`|95CkmW#wv$BQU18 z83cfw%-&As4p1C7NqF!F_h^L8U>Rln;SYAZGskQB+jeJ7$Ca!ZK!&&w!qw%fX=pcd zlnAPNzF|Gvyp=nfqjyqO5%ZSeX#v5M1XjJnbovpsv8u`3tAD#Fw(sGI7$8(aQ2xK~ zcK%iO6NAZHR8_szRhDnx3f~(^Cpyr6yLx!5?MU$-o}E9oJG)wLl_<{~f$y{0K0Wk+ z{rZy;sk!5)y~h0&*K3cnpsnoKIqvuoh+)-=Q=Q8fW z>wxVtpd0?39H^_RDO`w4r_n61|8pxc#zyrtG&r4yNOm*f)5pN2g>|y<O6zoulg(Sb7F&z<}nUfKK^ zcQ$=TJ3o$gKh>}GSbs`odA=CpG0$BA&;3C0vAogn1J5H+>mlyOeU&C~ij(JzVs>C- zvp-d(!<|gt7YaOfHhxwqHn%f*7amVEfZeq&ji0o0SUzw2(ZNXGzfZ7uNUhK|NL;Tn zND3|#!pWulxX}7V8YmrY;g5p7a{53x(yPbbG5jI;-+dDBKK=wQl-v6G(yiUAE!XeQ zEwcVRw!M?dm=Za}CW!!E?!dp81Mx>4QANDgfSe!|Nw_iXffqJ;7N_G`gbs6`H=Y4R?_Em+NqACpPJY^oWVjfJ8%Qds9 z-v1*(SUr{t&USNRaehoeQZtci>h_f0{1;ng%}A&A9Xj9Zp4GlTyfzK=wrwH2>lcq{ zPb~P3KPcBgcpQFI1+i)1aH~(^C{Egx5pFSZ*hi}?C*v>mT z9(~zk$VzV5ZOAp5HFaPfCKivG^2LZ-4w%3D{7a1}(kmm2!)#6JTx$y~)Nr#y>?n+7P?~FnviiUa{SuIcnz9 z!@vmnGw6Ou!1>B?@(PEdhBF~DYT4LOLf3?02VW(^45%;u(Q*KFy<#E|T*TBTQ%my( zhiBTX@zH80gNy}@F@aMQ0FPPcwelnh4_bMgWJmD>xcdds1|5G}cO0?%7ml8L2B$k* zd$>$QqTcIqvRcEaz~ni%MqS^~TLZRN6rCPjGCf4%N>wy0WG_VVZVlq-s(~#-s$z6X zT7G%%YyJaFi>B2?ZLUOM>Rg}o^ zo4+o*qo^x((^I0m%A!MXJqT@Of+Uk|CVE1NGZ5rm*1_FM@l#FJWZUL3Z$)(>Dtnyc znaYb2kH>A>>xw6yE;CDqsxQ2c*jCg0kx^CqDlg>YvI@4pqMxR}^$@8)IUp=x3Dwo@ zsGi8J1zZIkFU6|aQI&U*JeU!L9NoA~BR_fLR2g{5Nnbh07D!N3^zcM;@d7 z@?HHp#!DarI!(;nA5c<=<{M3%}w~FVRB*T=YLZF{*%%zgO}X`(>!SQ#!Nl1%aX0^77fk4WtM6xwBg0cx;pu0kF(i zy-cDj_KPNy9r7NmOZR_9K$}GBq%3SkzVrwTL(TYU9c?Bbg#A)A0YMYi#l%_yeW`e1 zT545)SrgWCb|#;n_+Jn-T@FXDUsciB(*34BBAdlT0Mp(im2C$#*Gp@Vcel#y zn*`y?qv=t?T8jR9aIm6FD2Guv{=>n#v7JUMFMt}KR%uYJJWRrjJbs1HdT#4v{)JCZ z`O`-KvO%4)o}Wkb6Be{XW2cS37BTGUToYQxPIj$ z`b|?H)2r&p`%{1B;g`!Z+-9@$#ohfb9}V{iz`7IP{ny|3w_U7EW-H(epCOtQ1tnVX zRpjWdXcK6!zo=blkB<3Hl8n-2t$bID$o7t^662ZjM#=TKjp>Wz4#;=6gsr)>tV{r@ zX*GZc3+)4V^H0VyMFR1c$FxJCE4@FA|`!s?NV<>AiYNvT4Hhaws?a^`N73oykQ&SwZ) z*A_qJKEv>C+7B-y#KTu|`a))&HBc|ibJ7aD^VrPEjdb?L-sehGJN(1PPTG0=vu%n! znh-9h(lBkypu?{E>&?5#uMyIVHpXFJ)g2&C zY@-SX$jw!MWtii68Rq_j7pwl{J6%N;=Ttrs`%tRSgItFOF4yyWN?fr{E1G<;A69)I zgMN+uI4ER7lK3TsS~g_}Y>qGB9O47M3rj`OyZ4JB%v#T-!*tYAQ9FM$a|S9_BVJ(7 z9(>T}(h>V1w-$ZwLe{eVJJ}^ z#ipZM_%OVxM#J@EW%}Nye_^97JBf;p5(DXLk zOdNf_mjb+VKC+H~0~&z|uex?%zeyL?@Pv`F@$wkIXTP;{C%N3{C3) z3C;=^210D+o9slDBfbaITb0>8LJL(+(w+g%RxTG>BvYKk<%@v838vTyxXe z+k8ym<8(dm$MKd0O_}dCpU$=tO@Fkj?Fs%#gp)-vLEI;k%40ek{n*P-;+;)nMw2mA z9jMp?4(N4v)%hhX2;a^cTa}v_o*7=hr3hIXk4Xqx@&!70zr9Tea(zGq*Lyf_fw!G7 zqdr#_zk64_x(!2nD9db7;#23BK3`VnZ`QR~We&`%^kEW2;)awv_>>_vCnpko%%zac4%y;(+yS@VGh)bR z0k_JK?y`79I$}#-MJm$nqxsnCw_)cU#p0V(-iNz$|FQl)Jp5gnB&Ajb7PVeDF5dmU z#P>LntIi-G>*w)mg}hPryPMZNJ${o~4YV6ACA)|}{cd{b4FZH#x5<64Q({$8_fZwK zivvr?Qrx5Tq8}fUleov(^(ykg-ZG|UWdUz`Ew`6-B*#>hPQVOw8v6ajq4Gxw=GrQ0 z5X>MS&3_BO}_ov75i^K520L$2z-|Ru6g(q9dlxB=p>_I_D!}{ga zPW3&{`|Bq4%xZNh{>UP|TYdu5De37&wU+K@MuZZCehPYVb}*La97jISKz|$=?Ig_S zR}MTcD%72=`$*Y%&~ENCOdLFd5}iP`nlW`*Oyk^I%^OjpUSJF>paGn(QS9nt_N94S1Qig(JAzu;w(8F5YlrIm zYx>GWh01+hGnoh96oOXeuSg-I_y-XNLq^(a8D4mFXc}ROew3WVHcZYz0~S{j`N%m9 zWqQVG@+P>uHG={T=kz966)DJezP}TIL;DE$#RD`7Cg&_>=PKfVqWmj>I?oXg{pV7L zx%1TjAJMjZCA`}yzwM-c)A{P?{8ye@!?Z&*{QTFC1vi=RL_?wnOR^Pz$bSxW%qi7P zdmkk{(I7Nq<3+ew8VlT~;I=AxN5UKy{#SSd(yMdVE!C7b3B&!M;9&68n*vTjvel{9 z9Tf~p13uj*`U}gSg4Fw*`#hZkae~Nk^k+~BTjv{|PQomNDj(oJ$F|#{pq!af?&Kn( z(HT#YJ9pcBa0U||;9l6+b3yqo=5j8M`wgq+cnL}{!H8~8f2;#k2&nT z$z9i(JZ1!%DNjuX_nQ4^f#u^-BhW5Ez3>oMSAf{#nq*tP4h%E2%o68lMR{z>$Hy4% zd#GIsVF6h*fwBiQowofY=|pu7lVtv*UPRR-;)Bc#Kk+u}eRnSw)p;pX_;L$RD|6<4 z4CQ&Jzlq$|vPT_IP+&+OCS~n+)#O%T-vD*S4}r7X(cNktv|I@idORo7iReL`M8#UCSL_vvIB|!QPKTs%ToN98BYjr4C97^!0w5)O+x^v&emW zmQgUG<0RpHdN|;cL6Apq5f3FBsz_Pz-4k#DayW6M;L~*AAJZ2&fQHs^T`5lK-8JAr z8gN6kx6C((HcU&Y3W~gQ zX8B2ASZf|_1KjEE3PxZw;AZoqUTSrs$;6MSZ4PC~%VIv_rhq!knv*dFjxMk9xCJ{B zyl|5-aszr~hF@}9qpIhUG3^wRryY_(4%_#<#+>e?AZdjk#P-X&X~N4M6xP@>3~S-} zix((K?(T}570Q{`a0}wpQl>m{cVAk)*=(4VPTR-JG-hVb>pb~YKW!$HB5in+2Lj{Xzb(;qTt z@?HAE<5uawpZ)AM(QZjf!9DX8s`hu$BCz%+sGb#}$EJOT`0HJkBJJq;f|h0aEQm{D^rfWxWqqG=HpN1nm|OqbgX!_09U&VWnZ1KdsYGDQ_s zGK{>WG%c37)gs#+k4PLwNHO`++N3C%*vffKGJciBj;WYYxYcb?l zDDcjF*X?VL;scWU^8PEs9`er~n19gQ+Z}EZCdj*9eUhSEl)c)XCwDR@`=bc|Ic|85 zIL%dxpBe=N`C?Vh*C+%%)y(}fWp?H)fCw~cW5UzDNx=EKl!JVRIwsJHg$Ig6@%>30JWd2fE{ zesPdK!JZzAgP`H>wVf5O)zm@DWYq!Fsd=I+a2+<7IFBP;`YH9eLU>eZf zT%yJns&Z(%s_2L8);Om_FkL&?|3zWarnn6Y3NJC{xaIvkoK0}1qq|{!|N>>f7M}S2^UY;ZU%K{APb(7 zg|*^I#(WNzQk!%+(*%s)ceYbCw+VqTWT|%C!BE`D*K*N1hpr)*ouNw=o`c8`OJ4Zj z?x>uS>#KH7`H9oZnwwigW%!JK507 zRJ}6rAyCqHe{jpP7qsUvvcc`{cs5yhJfZP^d1)kO<`XtQ-NMgAAvoZu4dA~_U%gMm z_d7E;pFa$Lps`u^){xKSq#9*^0{bn#*?TOdDrCffz7F>``|%c>wZ3AJ0AtA%Gm{Gm znR-qU>o5@djM@@8>trZY8C=EH`nf6OM59eylgMChEBpQD7BQgEhuC6g>}Zdn)5yt( zx7u(UCR0uv+ii%CE$qzEOr2R7_!50_WXmV8WMTEh9U ze-$RRqgd4|M{Wlqe*`v}dN%vxH==cS>yhF2=%;p+zu`6-+@z%FCZE2rm|$6*oCzH) zEVJdo&tAxs7MG@~DmABCf12}_=T$1-KH~RVz69Nts`HTQKZlI)&n?LpqT}{2h(*E} z<#8+`oMdjtP~ge{A0<&%obL}y{1{u3(1FQS->DQwNDcrJp=uq;(JbJ^>6^LIcpyO#@i0+Actrexo8r5$4X2tTo6aigg>kiZmy9tK}C}Q*xnNP7|j68b!fM zC3vQYLJ`3oMQa01_F>%+eEES14<0B#Zh44^J=n$&K-cbdwIs_09-I-0yUg|Ka%R^|b2BzZukuM{pgQSMi=q$_IK!>1fn z!>a*r?FLp>W@Bgx%|Ogc%|2f`*>|?E6wC&xDH$}(6!_`8s<}#?r*;Zbz2>D*bqWTT zz#KmxYyoVyk}qu)heo(lVDB^irTS(`0pPIdJMCeUrX%QO?l-d%C{ShZx7EjuKHO{z z1UZ%I+5@Gp!3_3Yh5@}nZ@MFzj2-t%f>6V=wZhAaC@B@p$VxFc;$03n8{%FjrT~1~ z4m-0))3u~fR$y|~KovovZ6lO0Aie|T;zqr~rQ*(he?C~wHt7*{@ldByF{)QGp8BP@ z3g|BJ4i>?Vk~>L{fc}c@ z{b+mgulQ@j0CY*=WH=xM$2wZ!GpM`m0-7Vu?)TO6`V4TxlsNDJTz}W#oFW|^zFV`9 zezf=!aMNbAQC=AtS1uq^(SXb2cPG`8m}v?+7dh}ijb;_-8V;FMLTRYAip#mEiXoc< z)McoOrP4ntROMsOOGL5fC8DrTvAD?0O#GT6yNrz6V1Rqb(c@Da0VhSAwYi`>M zm}*v{pqO4V9@oYan>U`IejudSPG<))>e??dZN$T(w6oE0VxeI6zpR))cXuB|o#Osi zXSFdQ(gEHXfVluxzl1evvaTcArN9wDPN3^i-MS6)6)oKf^Rpgy-oqdO7so1F(bmm8&X!*ulu z*UW*jg68uEgAEx4Th5a6UrvP@ulOwMt+xIdy<6bmn)rVzVmnw>UljG^g<&EjK)cI7 zt-xDA{2M%~{xw*Ys0e{uNj?ArWCeXR!f`1D(UsALVzB!^2Ty~rBh*#DLBARQC~~hc zs>{bAdJsF@)cks{#~RWGch%S%)xLpi7XKevL0c|>V`A6fb@tQq4mcpKjZZ?9O5FG~ zh*vGy-5;*knBG%^9H?kt2~w|in}n%Ae^@LGmWO;Yf0JvTx(=Ur__Y__V(3rx_<}bgS*~@hRCX~U>Zh#>88V=h+-A&9oN;6+D2V<=(Z(Um zr#cydPt=et%kYbVN2AhpIw|U&GJl*vQg7n9q=Xz2{{D z9}El*eC&Z@TS&Xs)boB#d?LmdTv}krq7L=bPNP30(U^ExxY#@qaiw3nORx#~&|=!4 z_6g&XH)&``=F*N%FMdN5e)2^H}0@X7B*g zAU4J!j;l9pkVyxc4XG4svjv|kH@wLypYRV-v`_2PPnXRHFDHY9hcL_sySa9KFBlCF zMNS6+8=>kK(D?^?haKX5Y#V=7O^Xc>u}2m4yJO0_Uq>jeEdPb-X77O%vSA|5(kFIp|lz+6!3m$=55^`j(M9Rwy(RQHNK>2I@MoWZMBdhE@M;} zN03VS@vr#$k;*nXi6U>qKtju&Voh6(IedvZ(a~zJmv8vqP|^Ju{_c@;o*=5#vS(AF zA2{3O{k13gVNa@dQHt{i1~3>Kt%8+>-=JJJU$ln>ZSe4>Ns`65_ph85KlXql)cO}UiluO-Y z{I-O7?n`O?FI}I>VjdP*p%EIEuDyir;E&FO&Pn?ct7x$pH8S&=(nKOwudS4E1xw6? z2AO_ua*qAl7H{jop7Gr7!zHt9h&*T^YtpX02?*_W`a1Q>j+lxK z-Em}5_2;j;H9-f`vg>T-MfTwAGfH$6w)3u{jE}fZ6_coL0TT$O!rq%B>`JWXA!;D# z-@T}N^_IpK?*$X`fy1~KqhSejWe$^7KS(&fRJ%Dk|AL^Z4u^}TVkWVF(KYFd`)^A~ zs)Jjz&K!1Z0h51ytw_%P_+CPd>D8a8U%`Ig)gZroCe0}Z)C!+Gpko-YC2olC2YdM0 zh1mel?$c`C-)%tbw;JRXvoH|yVa|={99I7VnZr+k(DWSiltEQZ)j;}s+5B*QcI~C8 zW>{$`3Hf4a&AC9VfjrzmB(Rw*tY1T4wNMLIqy;-msqStvYs(7SofsPU_2OLZdY0Ej zZN=zN-24ZlHd1ZcQI#X@)#IjM7S7I8qt2hF4q zStectNc;`cJVvt~qG-X;JhoWZ=-P$1|1)74y1;rh_Q;8J?@Ik~9Fz-;5F=uK8+BM!ZOMdUTPfj0@uHrt z^l0{z8v6T2E|2k>f+=m}PD@@E6`9^TM!<0st2s{p@T|iAurmd zEgVihk_|YH{9!USUH4~#QyozNX$?k@1k*uaKo)SGsATtYg?LXDvdds0b4HUweuJr( zbWWWa-=^q5nsI4q>61)N8b<^&c*<5t8ezF?XedL%BNwg4a1Rfq2*OK^*3YU#x-jWF zh`qFwQYao|)Z-vr@0F`V8CTuPpOp_8PH5kx8>V*Lvc7|d<>hAnmj%%4dPz=Jb!%sn z{rFM2nl`ux&L%bCca)4{9L#7%{!xQ8fZBS{3;lpw$s1o?Mw6o<#DR7YpXjQCc@o9- z;3k?;M;A!_Q=r$h#M$oCrDZYOt*XJ;=YzEW!s5VY-cjQnetf0xKxj4X;j@r*t8lA7 zPeu)BK6PKkJCfUe28HfQ8JT}Y1bVyv_f;VcRjnZ)yVRSmQ2@U1$G{iPd*9C6u71|1 z{fK9R0nF0c$OqJ}W>b*?O6LqFEH!W;X3onVVHRj+A?G6#z+d(n*KnIn9z`-gh-nN1 zTWA5&WYD8mRHG@r6_DMf&&2+p7hAdpdbm{}qaijd6dTHX$F_wI*D;CMmA2Nkvw+We3&ip-2YQK2}f10XvnfW%Ngw4###f@jJLi zEKsaUdJ@`5m4V3<`ghzG@Ca9u8;Y_EsH{Ru#c12)x)rvJI7CdwZHr+{MD|~H-$xvP=Kdc z!5ASIP-g(<24{Hu^}TbHLPyW~0}>wonh&1AuLyME(x6%Ca9m=c4r@ z_{OI&hHY)C{IrsuJ~rMV1Qg3f#sFV=2ZzUw>8kuik(f(A7*g*@qL$RA^snG02uL3q zQR@f1E`NujB-fUrLZVr;1BypNLg_@5zY`k)19d-^%U+4W;%g5@6dYL7!;qf+d;^1J zFI{9%OoTB#k5nLT)x#F#m*W9`6U;gmGl4jv795c0B^90KUy)Llg6cVqnHfI%(nhLn z%M}j6F=Ybd6z3a=_RzNE#5CEn1J?g3=3KD7~iEKaY@?8kvnNV1K( z(2z_|on|eOYJ-8=*EG0M>%kimQ1+L@%Let1HG;LH+wXAE6vuG!Y6Qx&2J!qeuPQTYFO_FkJA267f6^#5?lF2bOr8GA36r}+ zM&TexGn^F`?;S9E_OSjL;OR;oGXm0lrM_MI3(<1$l}{m}O_)u3R}|k*3DPQnAbQon zVlb0UsTbt-Q_!tVQ1c4<zm+_p0JgNTPonAi5G6_ERJDTDfz)zGZ5*;(-m7B)n(?G%Z(ckogA7CJ@*<2z3 zX|&7-wU&AFj@ah`VTr9>uTDqZYBgVdR1l?-Pp;M$$y$tmSuM%*D zgM@(8-Bpxc94rIS{Ut;5Z(Mi z*wEe{-t8pm>83nYGmY9cnGEjAG&qOnz52x12{gf>Gm1gnFwB=zy_qXR$L+gTyxQHp z*D5>BmOL3j5B+9o?YfYeiE)Uy-l5%Z&89idnC$$3a5g{gc^~M^uX>z;Jo_CjdU?ZpQBSChBy0Y^80Ar42S7GK=S+PrR-0Z ziqF$L(O`XO63?&B$G?l*A~B5=w?K^=my%PL+20(TQU4i84k&e!RIpxM%p)3cr{V!( z!ly%m-BItM>nv7iX6mmn8y4cs2LrKaF&P~Fv@1Q3r>6Ew1Nci+1dhaGj$)eOm|#SY zEPx&~*xK|V1;>$OgofET+~o|)5h>tZo^0@12T@XP`aLV#hS ze+Jw{j?Hz+^E&B|tlYh(sv1l97e73_n(C_Z8WG}4_Qr??_cnuQKU(#QyAXTs1GAs^f;P%a*VKmsS+hU^Xiow8aD|PvADdn*Vf=Rcuvqncj4!{&ngR?TUh` zj<0J8vwch@P9^)2w>%m0Vo`X30Ox6F=@dco zgoTBjQG8$qVYmS`h8tDD4Y{p}GwS@Pn;}YVpf4)irk-!g4VUg9NrXeW=45;ugw`X! z)aGsGShv^K5Ik6PRr-ssHkhfkdXO^g2=s0yOXs( zTxW2t`WGo|g!wflgvIzi>YN8~l_{ZJ?O~{*_}xjqy#|J&LaUZTUh=33N;M|(&Al=s z*<7ZEK`QF5&5&HZ1k<(2v&h=k${M#N5O^8>I z)V+QFI6N4u-{F>&!0e~OD{iZ4N#)k6fK`Rxt^1sK6&$heMQroF=scU#s=>yzjpg2e z+EScAwYKdFowIFWYZ*t``f1^UKd40Zj_Ddc1#Rj#0NUA?s%fYla^5#~-AEjQJ*tf> zp}wK3J`8!L_c=pY$wVcBP|F~%1OdIxJR9$12OSD1Wa3w)5$Zmc^nM5cXjPs9p-X`m zF(3VeMEmHIFAf5oekG)M6|iy~Mw9&`NYEifpZ=xVQRU9U0d2c+y*!FplNz7OWicBx z{h4o4ALgrS7*MRmw>%1~?K7*Ptb5fw&0MQt1M=PY5;Vm5E)6f`1gn8W<4;!At>BDl z!XRzY1CKvB3_f2lalF~=+#*KHUv~v2EtAiNTxx8OuGdz$=Jk~OtVOQXQN19xAm48F zTJSxY+xR}!(3AWXlyvT&jD-96w|#p<&}&;A(D;ec{Q`Vqg5o#`l7eA(K|LY6j%7kK;>LeSA3&Z3Y{lA3o-3Wy(QHAZkESYXm;9J zViK=tbSlP_ZNRMn&MDUPVmq3-A(MxG5bKbfB+{~;_M+*_O|dyWYD~B#)-(?;kk8Tr zdX`J2qapDzHR&660n?8cDkRE#!Y-ttu|{4iZ9jbQP_+urRPZ|5WK7wJK4qZygY{m1 z#9%2Y6z~dYh%-{;N+2lkvzTjCL+;A;sO3g`m=Yl_ldxU__YN__wLU=H=31i%1Uryg z?-Cnqy!S`B9Vrqc`U9R>O1mPG|IYB`t@QwAQX z_bYAdNNU>l`G~m#ico@ke%6x^6rqTaEwSh>BpZMdirbyM;$8u2^ETA_q|g{{=4n6$!5w0a`yDQ4g4v62dy;*gI>SxX=W?*svPT1Z!x~bd^|r z!RtAFW{tItI>g?8SR^GJqOV2bTTXv2FAc@3Mz`7hQ(ekneX7%tb@b(=dQ$63mvfKX z3$=0R>Ae>^H2-ln7w|E&>nqj3Y2873E5Gg8yqfvSheb>eXTx62{%7I1D#OI*&=Bf} z%M|wFTrd9uGO~qp!7}~G8K820y~1A90>Bu?L8{sbC}~va^-_0 zzf1mGH5F1%I^^X-P&X`mdqu)qU`c66wZcXEjsN_nJ<--8($sb%h5tgJ8lN_ao`+)A*u=rNNuJYZ%}PIwbB1-_l}j_B zmfofOgjr91r+8UUxnp6hEWltM6IC+NFqj6iY^y8TRWL9uE7h1B2tH=bYz_CA{*b?7 z*pR7tNm@FXmw)14J^PwC!xJ+rwrZDQ`WMBbY>@kXKe?1^=?>}fTlsY8bfm>usZ z%)Ygd;75I(n~++V*4V(6D^HPA>~Z9gI)kCZVFB~svgpc(uSNn)OiWu3B3|34o94)1 zvhx5<*gk0@^c3Fwyh1`64=R|fm3OnAHZQDLDSp|%mYvrKbYZT%|F;Xv7qR=d3sYXz z!i9vVKll2K(=KtHH-pQgC*`isE{}{K$X2bHFXiRl#;Tx|)nODbJ}b z!((A!HLe=@P=wCx0YURE*-72E2I3k>y?Kh7!BC?Tl+8Xb*6l@o56A^Hv|2#xnEl_Sf(3CI#sl>_ zIEG?&l%jnf)`qyCKE%M^wp14zx1&t;iZs|K4)a+B(7+-+~ls;=iRnF z4DaJVzpT%jH6|Gtbl=*l$?iD(tXDH}qt4j$rK^40P-X67DY{d(revZvb$5vYHZ)xt zrc-iOD@UhD@$tR5K_{>D#0wdY2ys2hTi>DXwAiMjFL6^uNYG1oo#~t&!Dy}Pj~53J z`RvH_C}H}2MRnwhxv|;hTty2q^vwfpoE`2Yz{<62G2OS3UAT;|DILkEKb~HAQz|u6 zqer38rFL3X9$wfe2+-r7WC=e5oNtV1E9(gXEMlb^B?q!!c9H{RL>EP=RG;^_%5sn3T1e@nA|Qpux6#Ejn)(yu)gp)c47z ze)H%4U?i@ABbW&sxE(r^uL?v9(HsxF&V&p}0pMO~5p-+s!-gS1wrSq(% zvVuWyyzU}J97XO6uYao60&GmyT>I&%vOSuDdMNM|N}~JAqWn(Prpi)a21rl}c@fw) z9V&|o>tcu zI^#wP1hM1`2kKt6bz&qZKSh^R&bZZH+29d9juqNyykxBH!P z9KKWA=X}L z?QR0>u$nvTE`Xmd)q|gh66#>5Y}}_EJaWq2kkIA@cOtLDN90!rmY81^D}i|@bwnSZ zITWK=fjc3LD1a2$u^7W%8l$Uq4fc{Gg}={&vH8oSk%$q@{sHE;%9Y9{g_3@g|avQ*9*OH>EBn+^$Khj3Ss#e&Ie937&K zyn4h^lxBG-9i7=BE=ov#TX>I#mt<;~`;(+!x?-qL{|Z8VIDuy9kKr~`a1;eA76ufD zLf`QKgKeX5e($>v?)z+;dSGICdEhgtyo(+-XHfOcX&;>v@Va9X45*1iJPXU8i~x=V zd$J0(6j-!D?>gW4djN66(ewbD-MCO2hNXuZlI<~xVU}!5ttVZ*4X`KSd_k^vJOP!P3dh96Bg&_dGFe>m)Zo4)!`X57%1y# ziAmY0DF!T%ZR5#N?wHC}hR6L?ktX0ueM-t6 zxS(|OpNz6E z#jmLCqbD|?6pBby$XqKh8r_lxN6lTlB_b-{os#ULzy7N%ard)kvg_Bf|9Ah#EcHe+ z!H;wjW0Ihi9{^AXXbO(M0S~ihc__oCREYnu(+a<+;fcb@#~v-3kH#$c7DQJLME?2} zf42=J&eW~IwpgF3YjzLzv|h!4*Ek-Yas8(OciX~^wmF^Oa=k%DLwxcU|9X@~L(;2y z((Hw#<_2+GK z`1Z-ybzSx>1S6_*!b4z00ub6%?UNf}=)u@?nQOIrt*|*zhX{<0?`*vvIyLc&$a61g zo!g(O!-Eq2;$zuzGu`JxFQb zt)nIsGokqkxMM$#&bRs5PBF&skJYw%hk>6_Ittt$;4{onai9u&n7%MjWkuePtr^b> zbh~8B#{Be@O=^^j4r08M>+*;-QHeOG|c?sss85E-l(P?F=cOa*+7wI0MqT(0H_=CZzGdm=+NJKVtDz0r-Q zgE%V0kV87kBOP(vtLT|!saQFGs7?5HJbdWH;ZoDSAK?IeIB#2FOD4NEFe_LzIPN_w zjJ#m*sG8gYT*?{1zKH)#Gs(=Q_RW9BIe;FH1wGaN345y``b!(cy}F}l+Kht1|1d_u zB8M3v>ix9o#PQcdFZ6nW6j-6CObN-jS`UcHWCquZAT*Zhfh@)(rCl_HMD+KL;}B5= z6WJzJxgi!VeLqNtl$leykCpEjlF@90by;hKX+>%T#cHU=pSl?paIqvWlmBO&?j0jT zC0@0Egz&kpw=5yEeMObck~z3PiQ1X6_4=>xg$721P2kw+J*O%VTFiqP;_8GXp!Rs1&m@~z}}{aaE4I*ps5 zxV+LLSMOtr(>I*2o~VT}HOtkEO}dRwjSVEFtbPQ1h-4+!-wyqC9uAoXnO~Pn8H!;3 zV?y+@Huedc<(;lFD{*?^>+KZR2}lf;>3swA?^8$^pR`r)sLubX*B%xI5+4C0;{0x_ zcRnyC;1k0o5?S^IN&L`MqP(DuiiK&hrodEQ2p>peo+t;IKoU^c0o&nSD_LLZm3;eY zl76&_vf4bljL@;|H*lb7c9p}ZXEYojAyQ3N~2X55})Hw@LXquhawo;n&8R z;Vk_BMu%ktp{gv#cxXVqUqLnZ{*Va}SJ0%_XQ@D{%wlMCQShfwrN;Sc(}MZR8R~`> zv&v=Rg@A{9GLa7fK1?+Dq}%s*uH$zFW>CMq&z&*py-{ zqnQsvV@wn!EkP76>6)v+f!-inm}O36`vQg-W5znyTk-#}2FwXQj1RtPl4<|2P`lLq zu!2~czJB5Ic$)wC$421@j=#G}aK~fe*t0``traID>nZfXUmpjeV3>>$W^RT*=838N zS_)39ATk9^s1BLGG_hQ@g!YH!JT=veoJyGgI^#_%DHvW+=!LzJc|}%sf#Jb2Ve13XJOlj|a~L7P^uXgAdCiDjxqjTR3;E4{p4Y0t)?@j)i-O z7cCm*)H_3UxY_8GE-yLftXF;|%NlWJbdNQ_&6lA$c9l$w;Jw|P8+GglKict9gr|g& z$AV!j3|O>4f+Ycio-H37#jNR2@Rfp3RWe^}9b}xH>QGALF|9d;_-TWuF2kFx*tRRW zE6vtC31x9N!(t_8ECHQcx6F^TEqt&!bs}6g{w=q)rEF)JIQJNx9C#($mh}@QX>Su$ zz$fo|n1EPpH8le!@VLc*9|aavw!n(n_g#lMSuv1vXW)R}R`+@e&ctCvt_Id0me3*@ zz^*o2K&!;GoQN{2|8AyrZ6W61^`&?Va|fu|qH?_fMz~tyi?{5OKWSAtt-vKl8y|W= zc9J!YWkgQNJC)AAzmEzu_WhT^b1M(5@6J{PY7A|N?eRkas8OF1%GhL7>gKFv=b6`X zT=R4!K@srO|I~^xDIu0yNO%PvufR@@MH>rWvM4g0$8ih`6k>o3fOLxCQp-Q2x|dKy za^%Hu4&L*}&spq83pV#Z8b`#ELqeCRU-JwunHpow;0>5&rn4CD-}Nm6R)5|@<9dAl zLqxerrErX0iM1eQW^7H2*zmBde~5uWaSlne$%Z1!0HA*mKNkQ@rc0|W4%xrjcSNgd zOjro^;c3(Td{7Q)^Yhb%mB6Lwh3m)S=$cdAwNb6F_pw~Tk{a~A1!luD#HOb+pZw;A zfzw%oJ0;B99>1OSk$G<8RjIs4hi@uYbmbf1i)5yXXKg;jdUTdU!0_!uNd+KTTeZ}b zN&Bz)t@>Z{o5z35Z^17AHNUYw3SGd3d_P}h`NPE7ajPyOJkiDX+do>Y?x*m6}W zxQH1n={6C3*7JwR3Kk^}mW9K~%4bsH#VWR}(T1UvC(od0a%ExXb|2vQPdeo)9PzEd z^qex~VCp9_Dqz%u6J3O3eOfAD4LiBj;uNE;!|s7U6%H9ndhq&`S01bS5iba1Gf!5@vD-Di z>EXk`r8CQF~awntAYS14Tu%6X-A@m(-~t`#t}spMTeViX6@FA&y~%$EItaIcevXaIh$?@ z{EQL&F#yP=_0+1*PgZEqE0NQbxBa=P)T^e(;b{FVb2P|6-xx3Vnh3FAAV}@86y--N zFCOL*yUA8l+8iuwagp-o45Qblf*Y1p^N&NB;9j5X{f4gPV$@C2d3W)LMwuthDc`BH zqNmU)UtdYTLCiKh{l#=Q`8X^nK0cb=!FwJi4{UF&^<^nf*>b;Omw{!7^A{0+9q@(+^ z1YV-KD@8frZ348G7z0ekDd4~{4-v5gtft+#HtOM4e!%rb^H>)H=@GCa4LmlWIzjIF; zU<3~c>lAY)!>8yClMvPTTrm(lNGSU@zv}0vg8@)A8=|uSKtJ2P@ex3D{*0yF`l%T4 zND+h))#X=z!!Vk?U55e*Uv(4GQZ&fjV^tzXZ|7$LAbFXDbC`wDZq5?IP>F7$|Mta@6PuBcN1?DjPykzu z5tYF#dQBgR?@lMbMxsk@ZGH9fx$HigHow?Ps@S#rov`i> z9Pn+OOBo9n$PVCV<(jR=uvv z|K04~1@Bcn7}^qK|8;fJM4$p8-_f;s!a)N~nV!+s00IA7UVcm8)l#D5#AQXI)c4DZ zt4-rb1C|odAe8J;DjbLyHek8xh&2F&vGtJ#LORtTF0^OF<){pjKEYzhG-3BT&^o3O ztj11iM!BFVUhna9TbYU(azJE}eka2v&KW?A7f5oCGPC$|>6WxcrCv8{>fo;+-tL#u z>@BlEJAc2?;zEDI;Fc5(nO&w9?0W6?$s34FIaH8gCg*_0KZ~(B-G!BTvv8XFE0PMN zg&>_OT_SB1b%wIVO&w{pdiZ-l=6gn1j+!={-hR-Z)GVAzPcFFA-5Z#omN*#$prj@w zI_W$H;7Dq4B>${ugcTWA-k1zty?%iq^f|4{%;Wpn&X8%gus?Vhy;=dq0@?Nv4Kox0Kg5j;qyNR-_wwtu8a?6ylN#ZXF!Eu~WbRE;+Ee zRm%KTJ4`vap_(Hki_B4=-4}X^c}5aivLOow@7eCNjDt@BE!MGj-|*C=tt4^;U^lj& zsbRjf-9(c6R*4R4rpXOll7}y0^O-=k?z-iyNe*mYe1q?bOU5+DhS!Sz2 zH|F1sWrl8d+|e6=b(N>W-x`M(!1&0dq(%H#XK#g_U6s+WDG7;>g^+8Cl_LNWyxQ>P zU{i{n&0fpXuXK=^CCeRnF6LGWhD_ZAtJIdJnZ!&4cU2CBlIWyw>@ulqM`Oeqv zJ|^v64Wf#l0N2ev+F;`|fU&ZCw+&`~*hy}!L=7PEdX^1153MoYIyX}`RzC=sG*x@K z#D`b&z@)j|5#ueh*oUX?3B_#9Rkn=^$YwJlgw798PUjKSsRgnfT^$1;=Z~(7Gh`t5R5(3#{{%0pFka@9A;#1AR^F z{c;Vq_I?OV2qyj)TFN3{4veNr%k}eB(*QRdM@!7%6TOhxnsC|JM}H?SmvXpjj$DH- z&z8B-6;0J=v8Ek#>1iUbEugRcdIig(e3J!0Va{WUbtx$@q%ojw_8uAZGg1qyodRA{ z@M-mLig9o`JStA;%9n_EHFS)==L&URCxot9ZWxckZM_&gerZ0(bN-*sfalH9rPIHi zLBjU`>I`~)e6CEdB=4`iJ4PZwH&wq(fp{0C=NssCVUr^HTJ6E>v=XC2ZtVoMttYU% zZs*_=!(i*~8}GH;XABK>ZmzqDm#+*(B@S+K!wAd3f*k59lrLo#@*NVjNZ)yl*QiOF zFMyNE1DFvuI2q5yv{Eq344YJcHV2Vd5^Og;aFLT;h)bv1d}7n5HQ>$N{}HfzDNbw> zm6Ij4_8V}ZX&S5Z&t|`Q7QRxfM4zubA3febij(tF+UIm8Ie08L$Wfq`aMznlBcD>Z zFQF;ed-v^Mx<|rP>!z6XO#8i$O9tEp@(<&v7dT(#3EnN4F@^64|3vDS%nAmCbX(XSj;#A#TS({J$fs>a z=+1?&!ky~f+a!A5IQ|_SR{GDy{5vWYj(JW#o7KwG*nk+irN7w2&ulR=#rLoH@;$fs zULsSC`#l06o|i1%0(|gO{PGMNv%S<5px4!!dG(5lpd~=$S%KCbfQup`n!Q!Z>Ri7Z z`AexQ^*2@gm|`H7N{AA0crt8T0SPvwGJIMpt|9Npfw;nDG4I$@nF2XlrVgGk(#Wvi zfy@=r@ed6%t#aExo6yFFP>B&Yh5cq`ex+q2i?J3PZc83L_r}SA)qJq!VoK8Nska`W zdv)D2Sn;aNb>~y@q$r3mVRIF1Sc`qeOZGehJ&ylt8EHx&%<$se%G@-yEn`5TfYi@l zshY$Ju6}}U6VwKGE_?eyFKZGV%*KVFuyu3Rdgh)W*f{^R0HTP%y(;&s#_?nTSNy(- zrSki{xB2?0ZItZNj|DGoy57M>@%aN6l18CV$hLe_Bs6TAN2b=i2|;WRLfQwIEMs@vMGU$-U-hqamXQUhWl^yPn43orpq|>F13Ddv5TCl*EdBBY3*VI z*7Ox6X-Dc_BkacFw*3lQv3LJ=rI_B=N9v20^uUxpXWOb~FHZX6kM`kZi_cLnpda%| z`2|^FX*HX34A#HG{mtD3=WugrrwEk~bPyZ{4CKZu~*=i+AH>YT!WR-FI)Ya~mA0 zOFX3y?jd^9YM)b(jc6ukUpzVJ^$>n;xyIAtPhA~}7lB=b%_8JVSp z!biUBRhtEH>A&mauyr}016WCoZw1O1ub=sSZc(7qHVrUx4Ewlid9};vNoZ0e zZPy-?o%O%=9X}+zUfCVd05hTREMRP-SZ&;V>348GNK|U4WUF>(4rpOiH-Y2AzIp&` zM&2=YuKav8vAPr=5tkze2`${ZHW9E}d|^l&(tgQ#_uUxV_3?PZd2}6wgfSeyp=H*|qLnE5< z8*)3T>PISTu?6vE1@Q^`IXAor2T$p~_n8I4oahBTz|5Q+^fwDTFCW$o2Rl?%A_k6Q zb-c>fZsyyF;CX*lwzkGzOx%v7o;T#R-4&l##HzrUu?p3^gP9D*XTNDPK(U}R%AT<< zTvvd;1^5-jyeOSqt^g^C_$M5f zfc`v)3yly5Q4H^0t3WnR4WzY9&hOlAIB(3AT#xT*l^%Uw#{LZ~-yKogbFKe?#{gPC z+A|#^oaLX>_WUsrKS8FKKvnU8r}%8f`yC+STGPQzj-5{6an8nxvjKB_LEgohpfT#f zm=UNU0!J9GAqszZT@+&+zt>dyT^1OM75S|UH={R|FEuTY_aypFfTrx!A05<@oI+)5 z`FBaF+?IR0SBGAi(amq0pIS?zYf@J#mgGjJ=~GFF4Ous+OZ9Ow?)#5939Y#PooTol z9(_^-t>O6_E_7E0g5fSKHEJ^7XKHvkL6cpH9wJxCZ32MUHX1B0^)Lrm{~EOy5)`M> z@pc45W(?AEYU$Z4EdyT~yRYBga!twXb||SMk@Wi6Yh8*3<>GI>7lM6h%gUctM(aPx zef>7yY-*~75leQuPhARGi*Hf_%tx5L#?0D6)>ZB zro0b~H!VIp`|SSE%)n<6vkN}xU+W9Vu|~XwnkC1M7VsePX4ND2jKn!oOqyp#?jk7& z5mlSV+%ND3@CSB2#Z|1`kTI&j)rAk9K7gXfn$#Nx_l&*xerX=>F-_DosMvHL-u8Z8 zQ@8qy79(C@zrh@OsY6yBx^$X7jNY1X`yf)kT!mrc%Gf}PnWanqu*Q_|R=1b{^$Fo{1t^MtGXJ@;I`}@F*hGY9MQ&RA z#l!^`FIE-+@HfSpo0? zMIk<&;KEbuO(uhi6u8MS8<4}5eWv65Y)c~Qd zRgZp%okkcIeN}~YgCOL8)F7c>*B@?CUE8eV+ed?Wia`z%aW=NBWXMN9fc$W3MMTK0 zc+S{-TEWf}d+Iouc}z!qh~DcaiM8ibJ#D_6r^|aiW-lS3Wk#AyZr?91>Ol}7ePS@~ zOA#%fnE2ul>YkDzE(~DdZ1dNdZKYQ;@!DSrO3yfQg*PnfS(73SPF( zyJ-6k_0CaDn{<0FE!P)>-V_V&YjT##jfk*nzXiNm_%!`nv*tc@duc{coLJ$+7A&9z zdzB?n;~QjK#b>bOG=-s<&IXM43kUUAP@1V95Q09B2FO+QlHod4KUeisle~6+K0b9@ z_Br0^9C|I*ILuxCG)P{3syp%No1-6oe~ZM5{${-Md%Nv%@Dul5m8C?BT3h;OX(NH{ zGHkAM=S;e@KCkVX>7u_5|Ck8uOMJ8c36B5&TiN*H3RaXqS3i+>jW5DQ#QzpM(3qLa)Pk9eaf<7g=B&4pVlpJY-@!`#bIjwS2( zoWgH3Nm~qS9#f9=hN0>04Fisd#{*d9!-x&}ka)WsvAZC>LqfS}mcwV#-aU5@RrGgo zHot7~RgP`Dbk^m$V!Qv+c(mt;WQ}TuCN)9V2PMMiAYEKt4n)=5XK(VGX1WOxN`|qUiHiS-;9<7^3cKQgD+{RDD@k_}o;fh&pq#=k$2K zGM|oZZlv`3diN^f7^8ASO1eL_gh85ray0jlWorhb_t|qb@T}M(fjr#fQF-iioJJ|v z(eF_wU-li}l+RN4JKg^$At$(3O|(5U5BkY|9B`Aruw)~4_RHWb6s7hAXWV9A#9rH; zm?|_xyjnY$&y&ZTn699y4iHk%AUe~tcgsi%2R(ZV8N=H{<1ujYYaGsNK*vK;n>|*lO zfnP^#l8Q%7k4>DskC)_iBWC43J|ewWm&wHSsa)#i;gjwM?dA>VclZn&vnqOnLnXhn zIWf{}E4wdsdPe>@L#12RJOIm4M!4S0QqnNliyjIu);Jpu1YiF|$}xc1d4jX2BZAXa zgUC&!O9Mi{aQi3x7&A3ev+l>ZSQMW7FR#avCWgsVM9=MHJz&4*y=L;vryrr+v@uAV zQvzuWZ0O8&;OJdCp1F^t{#50ZqxQA1+zL?uK?vHO*bhoB<|GMO@dxkY*O{n>)bpd_ zKWn#D+oio?l%G$%tG5!n;J3Z?oyUHf%Ds3stow9Ah|UO;Q|Scf;&4CLDMDJOm-2t4 zBZ&P=EGX8iGw{n39ljN|8azD;wA48CnHGlrY>?^l6LhBDoMI~Fm*9$~!8 zeq*)W6}`;s4jPsZeA+u%XO(ayM_uH~1_9jRlQ?&)8kzSic z)lxV=Vbb<Ie$yV2LCp!IhyR ze~q&reUcvTt*S?3{h;7E!tHoxB!%Bmz&k31fI!m&iIHRX0b68+5!wu%NDTz|mE+yq z9!|9IEc~W}@9JT}cWGPo>&aNv>ers0N12o%REb95!ttL&8&T=WNfcc}?4-Jz(>^Z{ z@C9%S_~r%lI@6KO#OeD$EUV!U$d!w73+)p0r$PFII^o+7v+{VvD zIX7!e+O_smwSc+l6&ykWSH95KggG9rQkF$+`G?bYr?V>)r~XoN{ws!Y6(Y-vu}xa? zI@ev+MI>s^oP=yeJO#O+o=kspz0V=;P0DDTB}jXGKHv`z_Jv`)`xrNfwX|zDN->_o`f%TO)cyt+6fQdw=3;$nxXWs=pf~y z+N6SwTcL46NZ8}7RS~;qS&#pd1@Q6m2m&d83Wo$&nVj50sr8{X$ua^?SV{{c^~Nd2 z6T|XmeB#AD{%}KG-a^^j?>ILF78Kfp>g#8YmQsY5a!7jtCX>za+06q3!)y`E zU9dp|c`(@pXKZoLXg*b?$*bg)eYZsO4Wz_-1{L7Oc3^q^O^!Y+ii<^8g4MdQjfoEr zcjm}{4CjGP%IXZBF-+0~N>1~-*vyiu^6?>M$isF~TG}WiSw_h1~sQu$hX? zi>o~{zG#2>L%ngfJv}!Nd)OK>^*w5zJWKOa9?y^t*Q1sI3E$0=CXh5J-1oD9(}YC; z^nC6=)lEf)`B`C7)ivFcOs9c5C2}-dTMXUj`zsOM!AHgmmjci=4+i^tsJ<|=C6*LY zPP_BTN1e-X7`L0zJsAoYx9H!|jdsB@#h(p@B)5Iqr;X((XMWC!y~PN#A{^Lm4@hNW z@|h^F%ia~|KPMW{ewyTL{rIIrjhkpkM<@Ll*c7Sff3))W$CbxiE+H7nwBM*>o2^b! zr}N0X%R)ZpF7a9ZwJ2EU74?srl9FV1_`vZF<(kR?L}M5V1;}6H5iXJuF8&cNLLZuT zC;j{<#iKGBpK_)KV#FieM^@Shj&+(&u~`EtkI;gnD)HK1eZ)FA zokt(uE$T|Q$-jE@{kIM#eX&W&`CJn7AI6Z`fpLa-Qc)W8~o8cR_>#y zp#<<}198ebhS8Xwk37t6kD8SOnd+uqV_-=l?gaFp2CQa-cHAkz+OskO*|bA3anSE~ zSG7^FzPs+$^(5Tmcflr73aVL}EXpv;FSg`iNc}CRt`yH`(&QRZ9pXyS7gAzaNvdHW zTVMT*&Eo89A~5t44TGGjmiVgn{(#RgLaRSCy(9X5cG@lV%$3i%F===;6jDN-S}Bd3 zx?NJ&%{pDKKw9Yjd!!$JOT-Tsmu}!6S1?#+1ZlKL1!lxb_`c((iuj^o|%$i5P#fPZnplSdJeJGyINz zy>cN$4jnkO2(&!T`x2o2@vO93zuvwdcq{QS4dT}3abfnd{awjkp#B2MzxHL&N&jz? zYeu0NDlWaK=MCgTAG3f!=Zp6j?FBR+UM3=E)|FqE@NSZS7b|x#3rq^WlNiPH#BZS< z`fbc}Qp%1SCgD;DrAAkZls4W#hPQmfI;HPV`@Q5;#q6HeNMT%!!(#H*N`wC!N!2nA zbrHjHIJW|$zZ1e@mbJB+s0Nn*%FQeHm)Hg^V0y@YRp^ekC%lKhQ*oC%U?ZuM$;NgXp;po+VK|~M}Ukh&P zs%L{~&yy7QSDHQS0;4)Mhj*fR&0Q58XgE32^4ul8I;!m{dh+E4H!BapJ6%%mXF)#` zLMG?(d|UrQlTgQkL4B?TCcw~=$*<K35TlD{BC8HaOVYYyLHd zY6ty4CS)qFpD>CAgI?d~oM>|J49GRlF+*NYiA3}M*>(%8j~^vcn?V?0rI9DeBudwf z=wSI2Uy==Lq$D_|f$y>GX7bN&I-F&fUoT+3(#>6ke<&`zhs{K>fbZ?Vi(PM~?i-56 zn-?q>ZTnR3=4@ZB6OA|giYw1rKCw=o{^?ja6PU|N3HVMP8U+dE@b2!ym;8LLjkJk+N0DkLNo{SGdRdg#^N?z=Q zddogi=bcw8JSTTX4@F}6bi>3V8kq9z?SDQg9=vHjzx&!%xy}m5N6_oSfG&X2s!%|7v~4;+YCyXlMY7x`)MvIxU`Kov;OYmnpSbCOKf;k@bkTmq)3d;SFb9UYt3^T%+ot^TmZ>%UuKfB(?Sm(5{+yEb zj#r+W-AmILE`7}R?FdN}VRv->lZWP|qS#b*iWO)4R|Rh$*zl*mL{vaAL&|HJo^=NY zgO)D!ah%1wjQ7kAA1oE`bA9fMKAXHcBuW`@+2b6|h%Ch|k1_H*|D|?gq5-^J-H*SY zOK{P2UJr*G3_(3Ac<_uXfFGs1p2?LJABJU%(LmesIQso*D(DZikRjCf2j$hb+pa;h z<&@91uQcG$%{$7LUGxo5jnj}{5Q9h>lE-e9QJ3olW9&0iLF!UHD5oyP2IxG|9l9K( z73v37XLa^#MH@!au;tz>Qh(O=ImDYnKhL(ibxw7X@OVLnwk5QYp^M);YCoCSdpQ)B zQ|qQ9vMqX~lX;TYc99);w@(=zalazUd$ly@+jaLVe5=b#4wsr%E6H5HpmO;%alcOh z92b)b5oAk{8PE$lHrQb0zHnq!*4??n{gt?!cuU-9oGH&Vfj=o(n(Pe!1zBX-Tovbf z2!MmqVyEhruS$RCTaqEHnE59lQ}+m!k6^@z7UG!CNKD?L5aPL|4}NUXEz0Fr7}l6K zY*?|a^7>)qG?4i)x*qmK-pFb;6^RL__zxU6t0m?nuV|OLx?JG)9=#yXqrHg%{I@An zI0n1jHC@KvyXBY@xf>M@^`$nLp4?`Eu{tkOHZ15cye^f_)hWspU!R#$K}0-ii<3db zX*Od`k1Qynrxqd(2)xj1DfqdA6SOgF;Ptqc771F)%4RSojODaKiiQpthyoQ+TUTTm zezjgzWI^GAweU7~KF~J{ZA9#|h3A?aza@@N?KAYRnU2})Ar!o=zcalj1u)LPmF09P zcsPM{bT%>WFZqk%A?|6{PZ7GWzD2nFV$kN8+NRyw}>u5aNcMYLDJxWYk z;YT>h&4y8`e`I~I6aByf$X%WvnzRF9^1!NJS=Tkm_*fiLfUe>>&h zUALCt`q|Fx<3w~{vbd&Va)0XSdUzczi;2O(MfA7((I8H%)G zCXe#rW8kYNfHu9CAnP}H1d!~YZaun`q9VG{f#S$FV3DhZfin8&4;&X8ml5PkL zi`>h1tMLgOL%oafEggB7 zeEJfm-U-EgCIKj+`Lq;|{vEjM- zBPr(UH>@ajBp011GfCA9L&eMCjKUcRzWfQUw9Z9O-TMoh;<$#;mhR9i9qc|gX05@CMYj^W7b=yNE?@7Kox|j0`YIDXPYkH+1BT|P zdH$^(o3p>4yDdbG=6G#Ph_Ga?4gBU70~0@|1zgh=R~TSo_+Fx_+nb$BBc@kt+hM-0~w$)9BdE(R3~FR2Oh1X$y$C#E@g2 zUAIAnuvs3|^f(d4b0k*@S`_@CfpW_EWu@@C4cwUlKD}N(xd_O;p4)9dypW2LdO=FD z#jLn-p4C^LGWnV@ZvD{=Z2J+UBZBPKX~!}G7kBXc3`cdc@6qQ<79%bNOF&mqB|~hR z$3T+CVR^?#cHny_>^OO@2aS~DBOGy-+4?}6_Lsx#2v*%}7mnGPKnHOt)w^xuQHpQH zAzP`)?uzXDi7y47icWa0ZqWu-?vtC(^Xj)NqThyXdJ2n*-SE1^ zVktHzWLPevIhKFK&XkBI*>4V2I?h=T7|TL^P@z@&1L?Oa>kt#+$D$+%^sL1=K|K@Y zJ29_onVI-I2HwH@=OEZ)%D)<`=YBIo#y8|58BO2rDtL5}PLrwH12@mg-kO=}8b$W5 z1Smg=X{vJ8;V5_;RnGMQ2W#84;p%2Hf2VjeZy?pgz^VJt^+3omn>k|hYIzfCeC#vWLfi+E!eSPBM zBtdYE%vu>m`EvNWTc~Hg*iYKOk0NtYcqwFh+Aj72eUA1)zPD{PG6-8`l}6GcL+|KU z3>OIrZcyZ+ze~OmytB5Q+(ai8djh8qnTqLZ3yC%>ISopzFjAGSB{zfj#p`Wi@v9IL zKgYk}$C47*LGwyR86?`>GU`_P5d0eB8iICQ?PcM-9$5G0w(?l<&=XXh*gibZP4qNB zDu1a|`>O+3uX*Z!6f@NG;Au>y+`oGx6(4xgQd-nitUFJ6lZqA!LL2qIr*ewAj63Ro z=5GC4-&;Sl{i&h6KDN2=nRTkBfbz$(%&WhOix%Qc1k8PvewL{!H*LBj_FU{@m~c<040s3*h7XTQEA(ki=65upu*#c2v!$h|NdTX zt&F4$z@QLNNH1#N=AyT9KcV3btFB}+ZPG~pGs}hq zZE9SG%Tv}hQ0i~ZG~#pMkcU3}Z0`>_pMTm3q-)~bJ_Ts#*w8&MT!L-HGn~lZEmq z;P3duQMu%+!FbD(#yy6*TZVe6DWunq={>hFL!@8`+b2~*(aOMgbW}utZS*&ggS`%V zPhD-nTv;)CY^DMp`($pGeo2T_YGQXNbzgLT9}{zn2|3)5VnRrE2BnOns|}p0yDI_G zi=ljgEB^#@18JzdwPi=>?oOc?Nggq>wc8^8Q$bIJO7*ztV!30keb1(=@8(j>)4Z-T zDIgmXyejda_h-w0n^Tb}=WSvfkEo#-;-avjrbuuz&YAdZ+J^O$N|+|m>&=3_w?WC& ziKjvQ6S9H9r$NQh#YuM-F|0TP~G};zGsl3axBU5Fe5W zlB6l|cvvv>j|+J((iNCkQ~iuda{t0$?F^`nv!Av18D zePJ=%n}9B{WFNmb+)D1uV#D*yfBM!3g~SCKGig1)?EVh*Bve8$H`COiAyiFm$eT2z zP{BmXM~7`19txtmDwX@c(W5%p{a@a^yiT%uJ~}9YGYlUK%G=P&0a9n!Lc%vE(iwci z&sIwQs2x={p5ZA3DQ#_PKNNwO@VM9Gf(HElY1*V-omK;K%vTdXyVaBL3YGZnW=^Xq zwLek$Mtb|4XJ=8PMyOHUx_{b^OhwH2>5y*QiCOmzLV4Ja6oYtWm|7x@Gf z0&pXkYn$z-s;4KyDD9yPE)N}%)4>0v+FT~ceAJeZAJf^|PGy{OLAgsnZzh%GNAbgS z2+E2np9XK3zhH!gKQA()kgLTPObC$-eA`7b;G2Y!FY`{Hak-=0Kssd2L={Wdixn2} zi74L1Wt;9}-cdYZ!`0P)e^laiVSn54vP$nxDs2;kuwG-6 zm2{Z#1>^5W$6^L2G1Rb`S4vzXS}7)`o4z4>(aLZ~txrmM#KrU@gRolc-DaZTcpyaK4+JDw*P`y5fWVW z$CF5(x>#GN!k2UoLNtl4ITr{5yQm{X6k7hish(AT*j5KA{XzR{kXlUwt7W)2#m+-@ zHIuFiE80!iBA+q|A|!<(QC&hg6jL_sWfD4pt7QfC{4W(o3X}=%FlZkO4FvniLxHlT zo2G5Z7NFC6_TkrlVc{o%5RlO1;a0+(51r*>H6oGorjB&%M(Um#l5+Q1By$!gZ^<=g z66$?MCVt*kd-c@d)l`rx#boHI+X6PhzHcJ3(e;~%X&mkIemCdN(Ck@BxBF??=I5Vk zT7Yv+ogRHq#U08p_9ud+7CFJHhO_LIZWK61N#m-jn%YAo`D0$!P^&cLIFFRI?RDKv87@eq`Bvy-yO{Nh0AkD!?_un?mf$*ec@(94be>-EmJ^D(w!I7;G~;7_?Kl^?c)n0oWeD$%lpT97gvpK*yw0kJ zR|4!ic(&@tM6L?U`_;}KZ3*qzWSaz4-4ykF7HC*2D6Fl{Am2U=6yZoyhPI6LFmE7_ zjC`E8E~1M1`x5(;N-^c(w1zziI$(5*0OiZ4{m4|opfZ>hafJXbbnGL&9a_UO!MYBo zZofHiG^2r4amW&^(w?t7mQ_u9zo8xrCfbT|EO{7t7)JjFvmCbhgOK^TO;q#1 zqcnG@O68)c616n0*D(omQO;{{8f2XFh7?)vrptv*ynnS29#cX6AC8NI4vobi_mrc% zGSG;axJN<9n9KHEu2P8w)B;4K66*04H<$@v(!?3DFlf|h+Bm{hHwoB2JiTQoq>9^u zrmFxeS<=|~NNeybY-1g35fc{qrDaA*Pya|#pv<`c7HRBvb;L6F<*Ic?!e^-iKKet- zEt#|YA#wAK|Mw%-`~MR|K;WTRMegG4dfxe0b1{^Zj$jjF(k z(N2AU)MFq>i|~(^cC~7j#`My=nRViYbD^cLFAzRgIO^a{O#a(knbwU>1w3}F;L*M(wvV<$fK^D>xpj$#uJ$Q?&&WS$w zrkd0dLm*O8gj*Gckvx0RRdZ>uobXVE*Z<+0BgyCYDW9=vI;0j&a*o-K^8?RXfcA(- zN7fq0^h%=z-$b31$!t@+9q=R4kc|err!kN@KAr?cuM4tT!ay*7B=i{VVqH-6mt+fu zLgjAa({DtD8~61=E$(SaDEiBv)a@BxIYTYaM!K4t&TUM^PN*;THgP0JKf30kGp0Dp!8M} zqGti7HLhE*OfUL3V_w)0)b}%fUwkj0H4BT|Z-%gE7#?4j*)5|%Qs#~cQEUvT5v8Zu zxfHXkc~76hSe#1T=67#)Oh7QNHWk=T^hn5Be|17=$4_=<58@TM@H}N? z8jS5ygd6XoKtn#3>5`SxCHTS0-N4~R1qK>R&-dWnFLWMLH)dxDkh`iwL6HGODUC%V zDXZrU#l(tC^$Fn9s&Z%LKHyN`R+Oiuq3?xi?S!%nL3cwZ#tWW{u(8g8V2$FmX!QAM zKYeMP_x@>QK8S3@AL^F;8Q`4wkm!^46{-X@G`KJt;Xsj*(RYI(HP?$WI120DZhvI_ zzI`QXme9ap(UV|=HGX8`i^lzPbgY-!Q(8Jit814W>z+iP`t_jYVV|SQ|MseM5^>Qb z{nXeP#=vjl{zRmOl=9nND`($KN5 zpn8jw4aBBz_v;)!=ZgAsD)bAM0yN0}daV-UJ!dQJ*<-<*qkR>EhxUQnQf_%$gR^$IK+T$5bamhv8UUqKRP>vLgKf6kw@ zkS|%{s$-yIQg(XN&M}>(O2lBF#1!`UHz_L>2nIh>SMwp#Eu|0Z7Rt!Ozb-*JD44v5 z2m9Y`slUwUZ3?-i`tfSQa5}yHPw-35^&fZ}_Kcxhiy{AfVGMKte#N7Reh`|^8h{V2 zd&keh(=VEc4bH;1l%czkJ1Hf6L%RBkd9cnYttav;s{VZ1JNi?46EirxQx8a@r^B;F zM4e-oSzisU_vcrz4TUk!<+^83=W^EVyBCxPWlg#N^eXm%`1kK#WnWJa?H?XKi0RK7 zZtUZp5faJ~>my*u1q7`9U1&4yJ+Wa$k(S|QbF;1KBq*7hn8bFn0hz8)z<3%4!qv?r z1R{?<3mzz$NEbCfJE)8Tu~%2(+Tg)1Oz#QKD-(A zO{HUS@O5lL4$B4&?f~YP%UdNNI$|a>{`x~x)!mmeqMi6AzbhTN1VccZ+3(LdmvAxO z07X2R`#(z0;mwz?4od*jU{*^)MH7-{MxOL3%!4<1HYTI}benhi4$Ol8u;(uIF+mDl zKKAx1W}>0m{l@ZRUt&;9vgLUzZLDJaoJ z()%&kX!fZ8@t8vSuD5pWWF>P3 zhgGqwmeqD&ihUDQox;P;;NMaljjm&PZ@&a}9bvm5>Ci{ix%fG1V$VV`Y1-!(5~JxKtp7BQYBj&tD?Mo!!{c}d#^cri-3P>d2nvD8X*=z_V}CJdcuE9U zhKYcS4N3}S0-fes*-LOzv++;WyvEo=Zz{|~Z*HyU*Capw`)zmKz!e0{BS|P)1jTh5 zOQZWHob(4atB%2NDBwbMb0R z7WjPKc}6;${LPL4`sRfO2fDa2_q4awd+3+{t)A%BJ?U`zAN9n4*<9-1>PchAN|v|Z z{P`E>oBg@D{^*8(Bw8K#qxY8|D3&!s6TYsna+7P;QM&lp2yioWAJ;7CLia{g9n0GTs$%XIyeD-SVZ>xJgO z3v%AYshL5^rlNW|W2L$erq7mC>Z08#Mk~s*a-P_bDOF~7SWJeqeD#NN5@rmE*0u;`Rw zhK6N&xw1X*`loSq(GQMjU-pEXbPUL$vkL{hNZ4ZS|E>>D#_E^;W^^`X7n#sPXopR0 z5e<4mg!BpmeGgV+Ve8!e{n=-WO>`ZQYWI-AC-idiJztbw%S`Y*7>t?jLlJTLgaS2w z?x%%B=eA`7g}_ulBHOv46sTI~OMr9RE@!JoFidb{wCg^}tv} z+(Xi?m*}_*VxG)d**)6Ik!B;M@aNunM$TpQ*@hwuRH%UuhwSf+N;J6Ib80%nJ`zI*~`zEP-*%MP9sjaTynl&)`=VC1N9Cv7?9@{KPooEJ03P!soMunJ^ zTkWnZ-!C7e2AR~Y5*3*&JB7K;*!IP|Bu3rxF}*Qs4Y!~DW$`j@HLmutI3@GX=-ATj zk}(OMT9atV*%fHdQIa+|7$<+_RI(Kk{0apawv_$`S#BEOOoBKJdr)A;#387p5r|8V zpLi__3Z(y-F@P$?{R ztFERWy!4bPTAD8OCem?aC;&!UsVbXhxQ#W3CtvBlcm1WHvlt~`?v4-JFY58HDwQ|^ zLp56rVFDRGsxApVcOyIow0q7$0~P!BRTX)BnB{?z_cW5j{(Kk?-hr(|vCPJ=3%ND+ z+$mpI8#`ksmCj2t>f7mWz>fV=G3N+?u%k+S4&9p8FUKue?Yx0);*2L0aT#ePyM4UC zlm3hQr@UgujYKWuxq=y8bNq4Uk-x(!7djER3VgGtO!9%a$9{c_NC_`j43=F|MfdI- zr?OHWGv7ZznhvwwNf?uz7VX}NexkQ&ZS6}n(h^wjntFX8Zke4RadAo+c(uN5Z4S-M zo~2q=B|+thFqqt3ZBwK49{)kyQZ@4ILL|3j;QGL7ElnKGe&o<&A7%sKkcrTmWu!Bk zW}#!0N7<`(mr)MnI+H8qDlW_XD5H@dw*IE9>OO1j%R{lFu^~oO)H?)a38bpNItC}t zN?z>vKFkhHfM2g+gJ4Yo$?aN-M)qru$WlHyY=!oq&6M+ry&|#ckn0DXQ~%_`#%6|; zr-EF$RYTr~aynvT_Ih~toi!a99aHl71iYrI=3Z=V-%v#X-1)!Q{C~N0f%55&f+>a8 zb`(DW$i^h^y#(Vys}9y7uu7^17`_9tl&ZSo-!!~mI!R;KI%x4>%cpeuNY#eq z@eDeqU+@zrnV+X^W>EUxy{@haHIY}TobuVxoiu6(C6x=>ebmuVDYjt4;?}MZulh+2 z5uDE_+Gta~Bl!aClP)Qxo;fg>FUhNQ*t;xWypfc==rwS6|A)tK-~-_Ha#{M_K>-~z zj6WJuB8q_iolH+b{tn$K+lll@=Fa8+*_$n@&wVV=hiWgb$;*{d(CRXF##)%h%=VSf zH2toJTS6c(S3A<@=eIfD&@~togwUnK_#G)?WegOPy(%3hD)j3@r}uh+vHe0`$-)pG ztU++*i9K*Ry$$c4CoK;nWe5>dq0?;I(f@2yGsnrinIY-SZs&8h3uZ)hTx^+ttgo@jbdt5Wo2A0dBXIuV?QlY~0;Yu>h; z9aR-bjK!DGc^d9hdL{Vt*PjF%4sF3pwQ1;DS+&p7?&WdnDB}bgu3~`7bj}ELpfE<{@Q#K;K7CA&Uo{0>4QPmjqxGB7M38NSW`M zEJ_6$g=a~oV_|vKPeM=yB3)FSo<@lq@Zi`8a>Ajxvri>eRXH0Yib|FEbSwvu0LPuf{jI=Vu)_CoaA~nz@5wJZowuy5c?-Pj}96TrcBZY zDtSl|NW-ZadBX`s#ETe4L2H(Trty7csLw)+C&F55G;D0a;nU~Pa&*n$@6Wirui0RQ0EQ?UqA&fbY6(Ib<7P#J;YZdx27|Z*VWPV}(xy<|Bc{+~ zKf~psV4TpOUw5q3gMmfJFwZQ`8VJfqn@Xt+cE$*KRW$zLy{@YC%Jz@jGxZDO5vb1- zRjy@lgQ~9S2e(`%9bRgVt=|~WaYS(bA%?Dr&b4c%*1Gus1Y!^ctScLVmS6YXDfio} zY0s7EaaAZFEY7aS4^p3^(qf(&IhK5VN{IGf;t()hx);u_aQp38kiKD`-2HwNT{;YB za$UxqL0DjE|NP}Er2^0ccv7XOqedGbKgLSp(ArWs#Ta)g>R380HJ4bPQ4yZY1mUtO z0Apd#s7{U~m21`P);aOmUEQm1d5Nk>Q;{?2WId4WBfPA1fUX<_PYoX3v|k%hX+){_ zApMvf1B`GXP{>FsfYv^R{sIHmmZ?WwhTEx?<>-7G-y|??Zm#&Hy%c5pvAgreosTDvqg*e`AqQ@8h3W9RIj=-UZ3+hn1ZyG6kFuNtx#?4?z=R3+Vv&R3IjM zxEUL3WOBLk>)Z;N`WBpFRj&0^(ZeqjFZ`~$Zdv8IS8qlxSK-5U<}&5i#sKayviFju~!u2cy#Pun}E(6@FTDO7@0r4g_zidkgYbW~e_ zn4OUD`V&@^0NdKN#q^5l$~L77*>#V0(!q*yaUBcb3FU%;X1vLWgvbj_K9+p8h7 zm?;#4nsHvUA5>AMXXnc3j(7nEW3kI1EGY8cO0Stwpo*~OzgqvdpN{CpVALY52-y30 zaVh@sc_EO>d<=koB3G1CMm9;APXF%t%Dv0Xg5SW=Ar0LRwSHC4+FWH7mc!JDi|Qf) z2&$K{KdKZxnPM5~$6|>JF$<@oIG-*UY*$+JL^4+>!_MoH6Q+_E_SkfXD&LJOcP5Ir=d*Lpx#>jue5gwyw4dAqIw0Wwx^ zu+MBV>s`2yYjuFY%`9t^CUrQ-$-}B5^C){|?)AZ9z%t6Hd&I4&=WAd)E*0AOhre?R z94!x4T25^Y%NVMq)+L=3=dJ0lbX&3JR^j3b#tRSjuLC%fybW=>0l1nnE{w5sOcc0o1C$`Yk~|4jGq0K+_WuqDC(BlVn7BWn4}&;zZB%A%yDp7Q z(BO~2kbhydqSxqwGK+jcJlcJD%b#%)p})aTHnIPB9q4~g z^Kjz9xs)?Gr)#UDN0b)4OR5I@AO*+#tD{OksJvZqGJHM^os&4~_qck0XPvBSY|oey z)}!Kr0yyLcyt1OBSty;=Lz`(?)b;XiBQTM-$k_mjItnEZF6`RgBd$uN3F{Ol6w8VX zGed7ty{p2l+znK4_0Jxor`O)jc!o3mm;H^5mQ$m9J8q`W?B8Dx1vXXO3TrakRiS2{ z-(LS`grQE5=5-G;Jz`M?c$jAU#F>I2X&A&+IX54k>$Y=yk4Psu6~(hIjw146rI9K!s&NY2{E155!_gh7yE-4$a$yEVq{h~i7KpWP|T zx(&K%G;C14(7n*VxIwzE-|KSmiRC5m4|w)|6#9UpX0fD=)Y~MPZ-J+gmjj)1G)JjA zVPVy)Dx@AD8YA_bK3U8<^C?TuoXLQI^yl0^G}hz7|I3=91R&z+z=Q+*J(&Ux3RLSf z=3$jZcv~r>PIM6P?^ogY)ws}N?9*xT>~Nh*U=TtynZgd;7`VGLK#9`nL4v_2ZziPT z;85bXN@rQEE%4boT2r7wTj7END}h!TXE=H)EyPbZkEtDfSMDmD zEG}T41?lQ>GEcixq!ni81(QR|^!SH(v!MW)`}#Qg>BkN){Y&kdW6_fa3-4s>wMGGz zgNDXGiC+U-1%}@9`A=3JZY?gqf1|Ts)JsMdQ%tsg+K87zIB>RY4b z3gVBZPZ(aEA>9IrolR*ve9|htuZ~M2OniVrfI3Rk#fx@8=0NAwTe>KAyxirWv}e*C zZ)jzG_b(Q3I{uN+rsdNlTO>mav$xSExNklXno%z|ZqIJGXj=lVOQxQB%-h)McO@4$ z&1{fFbauo}7JO|8ENQwwP@|;jy4zUq5k?jxqv``SdVmaSVPM@1I)NZ(25lz1~izZGpKprm;6Jmd&;hr=n%O55QRla8$`)YY%(d$GU~8IMAKk!Z?5BMiU-Nq^3ZJl(JF zU^P+*FI4p=kS}kCeKLc7Kgc-hyD}fm1#0e}0S$hpZ?+#?=%x`XcEiPqDTsR5P7oQV zF0)biY1^(Z-Ib~(2nf8Gzy}7Y>NGmYF%2b1g$qp5gtV#`!~CEGb5gG3sQ6YTpI8g+9qDC?wusu*jA26}n^^e!%pygAU zqn9;3L*rfblH|?V*_~t-k+1SiF7OOvnb0Sx8h6aIS(BlarsGw#!@Kwz;>s{lV0`TM zXWjMjNP~J`^nzH%?aK+pg#*Lfg8cA>k(i$uM?$1ZwU_rqQ@nlA7faVGDm=w#esBNg zJa81F-zq7*VI)i)$h!HXYxexdw4USwWEhiKN?8Zpq+!3`ZTHn}@dR%fhg!ion@7R9 z6G=+aZ{-a##%XC@yy?%B>2$!pNvwBh9vrzY88R5*K5K?YRE5XM;?Yp#4LxE6hYcM<`7n@>@%rm)62JC0-fIcpBA&ZWFKrOala$A@{7Vt_Jb$n^?k#6$oo`joM0xqYKDhRY_9Xv3BcL~nBx1ovUg^yBHad2G=F#W-ps{QyB_ENt| z7k|CDyl8Kf+AFzlE5+qJYD#N=m~(7XAWs>rp2#gP3gQ^ zs6zRFF1*xDI1zBQ-h1E7{M)!|+{phGjo)3Vk5Wj9;b+5bYee8@4Z{=vKg&o4-itZ* zrWMbd+kx?ux@L+y=Qi6&JkDI_>(#i{EX4r2SA|4@?V{JK04q)!LQ(akYWw}qkB>{= z)tJ0%w8}?S@f=ijW>1nT!&K7ys+Tr9wg?B#Qq(n&PO*Q&BD0yH3E`31}jYaKrDSo34WaZv7 z#Y*H}jvuJvUj(_rxBemMmq)cu0DxoCS{zRCQ}HWxritAgEcW=NGOo(X&Az?XHq`rK zqOHx1DYyUX0;BMGelFov_kj(=jo!I$+}Q^A14$u;bp?^>@4)Q;$JkqiMg4C5!!p!R zl0z$9LnGZFNH;Td3R03vNe(UDEg;Pd9g=H3{aeLs^vgOMDdp3(AdTBXPl!Xj^TH{fg zR18GxNK{U{@%JE;e_xUR`CxKjOCKTyXX7F6fxXVn*9hrSSMAAM=yO@QV01fP^~pP? zB`y46FP>`Ip}Hjf-(CP6!~qyVlX~axm75Q`<$_u@dly2e7bqp(56M$&7g0GQ@3w)NOa^tfPNS0~4uUWK zlj4!(j4Z*7QoVM8r>}b4CM04fA4Zl?&Ji=Dzm#9*ket(iTnhnE215zTAC609-U%kD z5BBE(bon)p%6&Osi-yJ*knn=ExMfpu5g1qw+Vq81j!FZAakP4yL;k=4csRXF5GP2W zmj8~eu%b_qct+W`48U!nF%Ub=W;RO09@dE^w}Py|mHRg**m2Yoqk)PFmD*o1ZoH4` zbhEWZv4<)5iIblHINROUk`Z>=n?E<+pF#ANDt@Q1Od2H~Up0#_*teKIIyTj zeF+?y3h|{gKD7(WX}LV3d^Mv0(Uj9?!;=lWpgP`2$LwE{PAxbqab^*PZudm}7Sh#Y z3mJVnr(5LF`O)`TOpWu9sHR`j>nWM)4^5=vb4yIk5GyDl6d~`fOZcE7)lZf( zRyQ{JMy{alsduEg2mY&81_Ldcu(l%J+n0rzpScf-LvnsrFAYG-4_zW=*U+3_NR1Q* zzK_XB4}zflNdD>N3*Bp~FH%XRuYPK*;QiyWUEqcUhFPXpcJE#Y6pt`(a6Jnz2>JOe zs;2wjVseT9D<3D=^rr4V{XBKb^AV89$(>#VFW^J*fd)LD;IO+|r zdLrXpU}E!fh`hz)hZV79yoVSdwNmy&+-qNa{K@)PJC6kq2g07rSL-DsLhW=y4q>>Q zH21#iWFcUl82OwaR(Y>bRe*M2pCZa;A>5zFjih}#Y}qpsVi5O1pB=5PSnY~attyN0 zg(l*-Jx8_+%teC4T{6S%G9 z(B@HVHd4L4G7SJzp3`?_aE6EvzVlP=RM&^e=yXA{^Z^np>Gk@eU-FGdAe|@ zTSLE}sVAyeW1nZ~?!EzY$~4qHCx>aBfHIUZ84^ku>wzM^80Qljv)tb z(QI!Sg?05+elVFZP$|nmz4Nf$Ca_XLsFzry8qeRomB=3<@7JKhonwi#Ke=b_+fkyz zp;ER3xIW=X3aB1I_u%4r;x%-}UFEXnAfMSOYhgTXmJW4)>AJH%-k$RE(JPl`j`l&a zQ{UYYX!|So(ob{H9q-D)pj|pQR1R0wbR*k1w1fNiI^~Z5Hb@GWSPmt1C^~OBs|p(2 zL%w?sK@!{YRnTDZ0Vv$-54bmF;kgQ*=Dyq~-rog&CL+Nip$7w7F$J zB+5BX4A=O3Pee%Go$S%;asCK(mh^Rabr!YgcaTRj6@eYf&F!aeEtkH%&W&{UCqf9G z*}Ym2o!25%9ao=Pl#@&L7qu_9vAGhSx5Bl~82R$z2rN%aW8iRS|FMR27U%(~VR zC=NtQ6)tF5pFK*t7Xih02KQ+7pS}P2XP64}KzqnzrsmcbFy$utk5)mI0-*D<8hjXq z)4Gb_7ur4RFpwV}3_#Dg5r~G0+?jT;<%33LGCHr?EiBwEBQRtEOSfV?o-DMINwMbc8>HQA}%P7^YQ!gP9kX7 z@0XY?=)&da*mF|R`5CSh@Knu|gd)WEe7!Ek5aiCP!^UJ@kWhkhPK@5BEk#^>?-c(t zQh(FA5Mr4U9pumQgLFI7ZX1gYy4eF>-Rp-eu*jR^Mu)K1A4PuI3N=D_7*OD&z2{z- zPete>kF^w1&L2oCrCnwz-yIn!U>wwhUSsxpqTDO`CE!}~TUxwfCYqdSu9Nk{8y`tq zuD8Wqq*~LTW)xO`Ch$yZM5JFIAbu)x<_ekEG>bpqF=GF(dZv@WOA(DZ#+Io&ikZPs zVk73uA`K-B$?x7%CvKYV6&1spH^qofwf@oM{TU6n3!9!4^<+a+x`2|`T-8?*ikA;} zA(j3fc}FgQAa^HO8>Ka{mYHxsS0wYoEjR-HwCxIMk zcAUg+bI!TwJ{^4dP$8P^_L#9fbc97WX|}G9qe_$0k<)wB4K?}G_MkEL_L@V|fTUM2 z0_{B`f;ZDF~*bq&m(H^w~e zNpNT``V{ctb+Ijo<%GrK%kk)RwE4vjx#S+<-k#5I)jsk4OavPAKxoHjGItCUK|_k7 zvH>lFBKr#0^b#9+ZJu?3;kAhSojQp!y_yjYI{?Fydy9#N8>@Ep4IyzhSi}Bv1kj6} z*NKb@!S|9|J1S6X@Cqb=xrk%SZV>1VaX(?-Y2$&@9;UtNPR z3Q(1*YjBZIlJR13bw(EM@`_!6Ub!@aUrUXuG;S=>)}1orcZ9JQZ_D7b?2ut-s#Cn| z{#xjML$%anBcb-yC-u{gyAq>Cv)%7YRfVp9hJ;xI2A+9EKgcCKstIiGXB-Fkz2Uto z&L1&8+c1`Y%+OtKQbx7AWA%VkLYemff-Q>JQU_->G@nHlgTp}=bZYW`y%oh*COUbm zUcc(C^4B9By-mp3c?@aIiz))e-x^NC?Cda>tRlMu0GVYdT~q zrAG%H&}P*|ydPS)e;328FMfT`Y5!L6DX`ybh=#}Nr-3AL^x=8WCxZiGC-YMa{4zFo zR$oH+kt5STlTmzFyom=5lA9>$!mKv;l|KmBgp$B_O*@cjkS-BNgHV_dXh%4L5CAKa zg38Ir|2DVefbqaiMxu(4Rz)5tF06&%SEs|XL2LyiVC)=@aUxeZ2d$YiWv-0wB(cUx zCX=Rz^eB8T>LemfAzQ(q6`);((D6+W8FfzCzq-qz=`GAZb;V{?Gmof}V=MC?Jb9Cm z%&d7Shw?U+SyNG8qzqV3iLz?gr!y$yKa8oKDWfCTIvAykp#SjX21{>)D*8D2T%}Ll z^q^tkiPSD!?yxJ^*9l_`f)}jSyQ|*)Hrz<*p$@`KW^Ravw*4V|N2Tk_Uc>r+A8WV- zXJSgf<9=4QosAj*U5Vw%3H^ZVSVQfO=a;d(`4HRCpp@i3QEe(+sklC9LXY2`{xVm! zzzwUhquz!J{p%PVrLm;|UiE{TFVWn9zE3#Br4FEA-PT-zfW|I%Z zw6a#_9_DfY0^E@}b~63(p^-DL-Q=ne-oIMuRebS{G;Y5K9EX>JJ_kQ>O_|)M3ON5t z)rD*RYX4TyXn9i55Nqus>Ee{h_S|$ts;2@zOPibZMbdG`i+U|dnv~{@nA#9OyroLcUKYAoz!g-(@1KVh~w@W z%5Xs7&1+N$h~`CGBe#V3Wa)=$Ykr~cg(H*iLY+(k9iM(psOwAVqb#2lYP!S0(K7vA zJ6&M{jc{a@qqB*D0UUj;Ua4GSy1B2{o=#3{E|-^k%9w9CIVoxf*vwwL50g&6YIJrF zda+bwwYnyN9MiGi^~3V3f1 z)-BggC_koM*$#|YMS`8)nbIBj1NN~nA3hydE^RcvHxWj38pJcq`UkSfYHpdGTFahd zIdm@PK;4_FXTf(OW$=C4`j2VL*MQ9*2{eQcjEA7HpKJ=HV^F$x6U8~SKTmBA-Ilhh zFs_iw#DI4!dU>8y`y()5={RRs@?-hAPDf;9%0hnkxw}02a?1v%#A?y0)<>(!K- z@Ef@oMMsV{2f}!7+R$sv=`>{An3|<&#t>74ZKswuwgRsrgX#Oq-dm0&R4qg@mVa;a z-G0@P8n|_>^E8vU<7U^%^qJhB0{X_K%TMxY>@6+qd;SJfsz=Q?`UgihM&C6H#nT8b zkfPFPOt*j~fD7O+Py1FewLc_zVIIa4gT9OECI16%JS{Z!Tp)fjO#V< z#lMMW)w_Ch?du?st?~;m9?g1yN;Ht_iem_2+AbR|qE=Mgpf!>Ut%C03= z?qO`|&0OZtcocdad4-YEare8`W!Wz`VgVR741}MW$7?`$D{r>bfH*j^`GJCBv)nSa zV~rjUy9f1V^8LyIGLD}=RhkCG!|vmJBerjABMRnyHxv`lqD(X57GxBa)h8V4GM{BoR{KAv16r?USBb!kf9toiWAxM!l0h`u#PaKFqj+jU@lxfzKZV=Y)(xg{3=9tuTp|7_VrYUSKn`5J#ynn3SK% z3v3sx~N!!?83JQqhhkZpILlearG;pqj7*eul!DD60 zMRSX84=W(#0CWK5k7c|to%;NC(f(Y%eyRS)K79JJ*>zQ2QSc?4FR0@qCR6fTq6^P- zHu*_YdAq~PS_}887-%CE%3|)pQuT~sA@=#ylz4!(HOl~){@xx9HYpYihlIOya}C0- zOjGh9;GozYWwV9Z{=0EjGrz#dNx&s8QEEpXl_3cm)O~xX;UxF6Wq^B2H)Zt8Dstnik*mc;dD{V*W!=tW0VMvtkT0jD|KPZcnXRzMfzs+R3JiY!=&^ePHwU8Sb=8 zF`4+I>ha@B;kgf5Ao{@m<8NZE4O`V8w-S4DcJ#9zaef4_KlS^6G_sI1kx^gyBvSbf z&NRfa{pUMQzWStai}&@r(lgOotfz_6;T@!fn5f1bVshqmHD$l-vjsbBtlHl7}qGL z=`7Wrjv;MOfID3ic<+w%hQZKj=bl`@hm0L=v~ZFiprwK?K)K$&(N|qcT3|4)C8+9U zBD(=wlmr{#;{nb$#$DOkOguuP-;6ROBbaAxBUuD&n{9jEX?$h+6$CM5?}W9tKt9hy zL;qvfgOtzu|K0oLQk^R1(Wf&C*R5+g{v2i#Y(4&{`g{|#=bbsxB}H-G*7!3l!LR-k z^DjJ9@KRJ)C%tf}Nb6AayCkD7LwZ^H=jcD!5@$E)^n>|fDE>HQg{-BxWHJN^Ip90C zcKkm+drJKj!jy13z|gJ71sKLI-#RGz_uff=1==nsa$lOt+k0#_5lP4}YnYqCU!mIM zZ#U52eI!7ETPjd)NYw@AR>cgCQQoR}3=F=c%kx666Z2BlU#|&TQCi+6mrHY5p0Ca4cSnh@t ztGU_`Kgku$AdlO1!Y~c2Gg{}ei6=HOFE1Kty{-SQztwna%Frk|Xd;%;fdrohnvVYI zc=vw)pNvfiXzCNQKCb)UNk_Vqj^JV z_PhL3@0v%a&-S5wvCD5)bb3iv|*_GzjgqkIr+2N#1icH=P`!hILQ$E6B%9_jIc>JHB9#Cb_!S2+j zxnX_eK-wtVJ;_D7T@pfFi#5v=djz!fmvYLU=ncqd3pUw$))?MUMH*(QJpr`X9R4w} z{)|K_orJENqqj3yWOx2E;RU<=*#v-M$l$d(#7$|QuISqS@jc`3DDVD<1){T zOXwB$?k8`lPyEw@0ehH*ZhDX!aOG%@Mj+BC@IQJ7q%}^!?QpAu@gk)iYE%@pM*idl zm0xl>J@QFkQaI_adYgzuNYPW@7V!ztJ`XyuE@yE37U-4)XiW?W1o?vGs`k^Ed8WTI ztL6EE$c_BHz-LE%oy&{LcOERv!;oZ?B?O)PVy7*2ZJZ+?S}Xw8f^id;yMUtz>Q6NC9l_xtRQuYQ>7iUC)A?BNY=>IlUV32^#!Y zcv)FXU+GQEvCf;_EPia;W!|~1T2dRR#Fz{4vEuvX#SDK~C_{RzH9lP|Rk->bp|huR zqAn)DevxZT_o!n1G_JnI&~YnH7qL?L!h`5#Jv}qIjlfH31Kf-h?yBf|23#KRS&n-i zKt7kgsT*sSsw4A>LF3P6UUuiu&BflErNs! zn?v7p(2%NqQJ<3{Kf0*@84Ob*p~@g;fh^8`rMd%^SYOvIz*DRN{;0R^tr4`~X3WC8mqC>#+%^(2IjVA`Ietrc3+(U{T&ovOwnt)7gGxbRfTj1S;ZnJpq7E1NguMw zET>O{Y-BFr_n@0^zZ#Hzxq1i z-~O-KKdkedPO}j297$#KN-QsYQ)J18T5@zP7Rx^NWS_zP zD-zPIh?*8BDgP<~fZC3Efy@mksz?nSm_e|0T`dZ8=33B0JENzmB71KNlB~zKT!V3o zDb?9gW(vD+xuVgU@|>TDTfuh+gZCh$T2+aWk>N*2y*7VnHmaznNH1l~ZThlre`Sx@ zkqt!V+APVB1$}DGxo|Q$?NAu}zAX^}J+IToB`E`4A}+c7oRyj~nQHYyaOJAna#@K{ z&+N-ARBJw5C2$rfO8N48S=rR0s)+((!aR}cKq4VGHs=7F-SwzA>=d-E7ShSfQ;@uN zf-eU%Qc}wu6-2HzN;~eQ8aM|@Q@_;Z*1${Wdz$o1FZZ6dUtNmuP02Ub+owkz+Gud$ z-d5kye+=7n|KAPUGMq~?`d-?w;m`kG{Wqy{)uPUf96%h^`H@jS#S`cCm%A$vqCds& zBlPXFQbX_Q+G*~5r_k}$`PP2uk>WLmH3ENlRZbVqOIRsLtIzqCemeVl`}h_Usnk+G zk5^%1O<~{24))(_)M}v*?84tRAft%Fz~J3rkCduYE8gqL35aZ2#HjtV-nvLJns_pW z^uT`;Uk;Ew9A_QtYaz>5-aM0O-;IFy_KYMh;m0XG$pB}WjQo_zZ}A~x_=N8*U&{0w zFYSi=d%`Kn^@P)dE+csC@x;Q63Gx+Kw&z}cGZYWkyq|?4v|s+AhnBUorub+%pqfCEv0NpM-RPM>gb46+bm18vT7qT5ywDe;> zf;!#x<9UR`m{h+RvCeO1?Q&ep^uPL#pG|w`T#}Rg5=b>s@b>t>g?s~$L=M4DP*HhI zQ7`I5W>}2NFMx1ueUUVq)1q7Mw0!W00mcc?X91_*Ba0SMd^sF^cPeU>p0h2bF5nkS zH8-r<-0Z-Vx>Yv;!~s97)PSicTxl3Q&O>X5vRF&=!zK2)RIZR7yce+&qj=l&)e`vMy!mk zXE6fw25_~pOshQDg8klFoDy8E$K{#~V@U21-M8W zO4}f#LKhqgE~=KnaIAP++l%!|`Vt~ENk|%}*{dwaZcybEYYXz`)LtSljpbm+rt-7? zG5hPcLQ{6S&$ako15tieFelPp-kYqotw0&2B{oTUSH{~t`_%9vb>}8>yV1~b=)c4X zqzj4!AW6D@OpQkXQ5Hbr@U^q0pZC7KUNN@B{K@7kF^_RtYuXg`QS38}+wV43@k&3~ zTnx#vKaRf?-|-)-EV z*QO{|NEI&orey}vd?NDLV0bynaR?Sr9Hccd+W~QBtPbrx=BMIbB_CtP$&m* zJ%8WhEZwpv-C}N#5N9ZPo`M1#O_?2pw8ZuX2V7~Q2%4L^KLwql|dcgUN zf_B@|WqWMN@Tsd4o(8bI)~`W`2O>3pwX$OJgHaA8+I#0k$M!*4r0a!B)T$o8J`d|W z7z#v|NflP4Xit>}B406yFFBdGbL9aq;CWe}%~LS44zBZ*B1bUl%PGh%JeA~>&Fr|` zeYba761nT(4)xpmo%+20AveH?!#jnED@dMI!QLg5@6iutH&OWmxSJ~ZVMq6j%@L6h z5nWTN+S&wu0#*R1&aoBUp+5nx%YB;6Y%|{p*QV@DW)!5$0DRQ7i+>tlBL#rwnHFxC zA@AQZOC~xI^p{mPw~Re?z^9XLgIXM}q)cV7@ac;me8&OJA58ZK`FO~Ou#ve#C#X}a zZ5|(xvgBv#t|wPHclboS87LtVBA~auf^CIR=3Z@}eR3Ygixrt`7vI6CMG6u05k}is z82;4Ud!)?7UQtQvzoS>>iri8LbYL?~**uL*`|3}e3zFZ1UZ#&9eONAjyl^TL&UYP^lg^%rzCZ{bEOeAAn-*!=T8t7D5V{1=%pU<(3=M$L=v48=VP z8sU5=LD8wA9H**77$Dow1^3KbD6$b<^h`7X9b^f4Oi~YnYpbHYjQ;r`D@dtT(B$*0 z6B$E8mw61X6yNHti@SS4v+D9z?Zk>Uho2FU58?quiVUR@!=OJme?F=BD>aSX4V&&u z8S%&_Lysx;75?<@r z>sq0|AJkK!*P`P@(9=)GrRU0KDA$fU6Fon9Pi_6og9{77zI{3oFxQd!pkYDM7={1A z(I*s3A1(UbvZ!xmDLuDzY2xd2?)halycm`G~ok&s{0qP%_*Uqa5A z$86O|f{q{%`aFiF{1uE>^9H0Nx3Z-=ckpK{G=qP}wxsKArj@p{gQUua`*hLQ^NY%~ zpPY-LCT1tb4*wTTVj+ULG8x(tY~RF8v?6CE3FYb=N!L4MCVH8j3eK}#3bgWgJuEi0 zO`oh8>|(Bn%UK?3i}_)=`ywZlSPz!m18rSlmPoKNL8m(M4x^=0DP;FsfMgGxbUp|J zyz~_sNK{Sph3R|s%JSJ9t53-TsU&U*Ml>9ot-`mga{o)w({{l8H>48iRg83*aY=K; z6OmrsyBnTL18v`N_Z^t5 zO|23Rs5demHS^$z3<@6Fbp3R@lwtO)0d*D#?F(o`8p`?UojD*Ec_41IFRdZQnLx;S zK38oT`tPr?d+1sW zP>;Np)!18oRr$6$U4hs;tqpvlyof1D-z^Fj3-%wJMR-$YII8Ob2?U4dckXc*^XwX zUy=P4+f>;-Qdg~O{g3|Zf0Rlv;<&&FXuzrK?!i(T#hrdlvwuNcBE*UfHSU5-b!*@^ zW%O6S#z!h#jE7aN0{TgKqB+c(p<9>v7E26EIYq8qF3(LF9DQRglUxhFFk^lzA`FR* z*;Is*8?-M=e+B=!YxW-b?acG0-&GcZB`eB$}hH%DEoXv9d-P+DonMs4!N2{!&90QH{Uv`leV5{g!mqQ$V{`>O>s ziQ1Yc55|yVPfZc6Uf7Zju|I@c&MK7Pf!Ps_PS3l(VV`E6T@}^4lcf+jvL7cS79t5)p*mgTb zbt|+-_yzZ0z$F8<_harqes2dIpl*8`jF@*?uN91t9AI}9LSaM21KY6983v?s1+zo` z=P&C^$NUuksM%ISQMBvRvSPxCt!H6e7!}d_gq2e>U%G4aMGW2(0y9UJPVdepoV(4z zLndRmk|TpomK6~Otx-Uo$=aU5%k-bRlHKz8dkrEsGKJtv&plm3HXua#BSzU^Lhphk z`EeDAn3=UxAsm34br2Q!+{qN@GRJp&aB({4an4lZpj|3p1@Dv7sqBhEK-a*+Z6OSm zPe<`>hB<9NB*c=TfkR9WCavD9e9c*HMxi0ytIR7a6VIv{p?P5bLOEy)n)*u4Oo4zp zt1VbQIbNp*j1?x2ikWuZiTCB%|5mA_YomRK3@fvtLJAd;;zTcUv^^wq@W_XZrWh}- zwkqf9-nhRu%@{5{1`y3pQ755f?J*UQ%H(L$*A>@!Nif zEh8B(^e}UiSHwmAi-$cA+7=4C)kWMgLrCD>RXVRy7oy8s}bv`hEF|eUvS-5YY z@{=MUbMsN#$vNFR?-3R1xviCuOSrrlyXG6BhXMHs;@7VW#e5a$!?5phlT*g}5h|*w z^kvKqGOPUiTjZG*{=9|i7x(sntxeJPsp#_$cS22f&uOexkXt^GqU+dF<-S32Zo@Ef z`M|2*X4fCUB;{TyuCKzY$C76J-hK=WrJ_tsS#rObDV`4gMJd`XCiZ5}Gobd+I}S;w z{RLvh8CW}1ZxVKFFv}afSp=jh<=&t4jZ{v!iXnA$g;EsZ4v(iSVFePcMfQZc}KkUR8eJ2Odm^b;{8r!z{eR5fvF?E$c%_h+|SY<%Jg=lU8Ga z;wqd2iBcHT{WiYKAqkF!G0kqB^>-|>2e?EuXFu2OPNYw1+}eF?(@icj?i~|sgj3=% zDeO#If~6J~1x#E!DDT)DvB4gld^267>hypQqzeUfHuJgnMUV1{Lyw20upJ}^{HQ&* zt-7AvUr(J}O&@-3jsk$N@ybRMRTE$s^wZFQ*n4q4JM_Ask~#%{L;*N6?|9vJ&MOD` z2g8C$f3SCd2@of{-BvpDreG44$7TdU#?zfsQSNRZ=mBj zmKfTuZ%uBWm5~r*Tk))Jcd%&Cue$l*dB6un1pG;TtfgmGFxk^CqIMTM@LK=)WLfMb zjq*kFp3gwH>U-{Y;v+Xb^p6ocyG`FRT1jA5-Y&VO;v2lTqkTV*Lnn_4j0L`Z=o{Ih zGgw^pszWDne!;C&!Szj1%eih>F3`rYD3;ym{{>U~{>1FRm|D+Ysa*oSJ?0BEW~A4g z{<8J2s28ksh3>*-Yz#{nr&6d=EGAL zzh6A%1BIlIiJMhCl-3#>Ki+S{_g_b`P@zaLYz`5b{+U|qL5i`IfgJ%X4?j-VSC7&( zDyXDO3tLuN*;ovi9;JMor>Xi9-r#NN)kNmo>Qq~QiTM)^+l9tukxt=Iv{On8NSRj$ zA{D(mWHFa>QeAqrl}j|1=Sx5y()4E2WTzO=N?3(|<7B(-5QeqNRJx#iO(Z0vW?Sx! zu64qNj5ftZFR=$70|Pw+gF2*$K1bL!=ZQ9CN`khy#UT3HA$nMrcdOY|qJPYbGpe&E zGKa=AHnwG?K%HTDZ^rK)&(q8dM|S&W96kv$X`RWf{=khE5=cM*kd+1kVJEAD!MNHv?Cp=CY8JXWDa>dgofL%CICn)pUNL4>5J*?t zaAM`Yb1a&Qm7z973j^DI9hCw)9_0UHoIq5=Ki9T5tIuDq^I1&-0?~ z0BO_QhFmwE%RSi{a1-9}SX!Bq*W*2VC`} z_{R3Sr5}@xJ!+oW|2g`txiwRR-x=S`L6%m?et+BDOUqcUW&9mK#?HB?)9;)L{O=Ay zA)RFe@gFA3%rkxH%(wt0?msZQ?B`zKd(fz}uQ4rKZGpH!2yLVaiHjOQuMfFQXjQ60 zoS>~nGW<>;HvJKQpw56B>wQ+=<$8wBJj|%c>q79;9CMwC9yES4@))A`4aLN~`rt>w z4Le#fedkfs{oY;fb`$eF2K0b>$q1VxGP3Q z7jC*L#5VD^coJv%c;5O1i2XN8P>}xsGGv5fJ83#*9Ox=8N<14~GNlG0am3B;0-mYV zjs?x1XsAs05guPWv-7OWXvPv>V)6WxR8g)%MNpa@37Rb1zr$c;Cn`G*2+06#X-cS; zS3f83j%$Hb?MXL-D`9z)-vqWaMshNaA^I85-Bo8gUoLf@w0Wg3vTg|mUD!m(7po(G zif*K|_iKLwuV0o2uvU+Pjz13M;wePqzezkUp330bQj zd{w&F7uo*6IU+@LFM$Ub$*D@6ok)Olz_3leRW2KHf6rai#NX2E$;(Vo-fCL; z{(0D9cU0!kqZXumw$lbip0vv-(q49OnZ=aZc-HlB21e-S9Fs-1UaZ77QsOANScg}< zf^>;uzq0b>EJC@N*^28h5|O1<2wY~kMVDo4CR{o!+Il>J;X%q*n2!SzjVJLJKE@bu zZPjTdujakP`GI7_Yt{q>NdK@|=kMev9bZJv{_UD-xkmLRiQiDHiP~K-*QS9LFX`U& zDlPa^^=zB-PFE7lNxDq8uRX<|T~H(7Kaa=Xum~g6=Wm3WN$p2k3EUCx+T#L@(<=9vqXXNN;iq{Vy3 z%<(Ew;%Wt0^)I^BM?%u1>qXHt){BN|uh0w2^;{3%uPPH*#3*;_lUOZ;zP9iD=I9ps z#`hI!7DP%L=LW(6zxf&U1nbEi;Tl;?gD-1QQzW(!Onm^`XV8`hYme6OS$9OYr>w}n zBzBNq{ZR+*fcV<|rQN_&NaWMZ32tFUZB<}73E<3Qh|-m06EJ}Ah0%BZq!tFSzk12c z{1JNtq5`L%1@Bv?gEGo7`AE}$lfx-mtR;dRI-{KoK2tWuk_jxU;X);#&01r{uy=s z?JQeranf3n9v??R^>nqs{ni2!>-IkUg+AP2z}NVNtT5IRKQZ<{CM=X2$Y zXl|$9io}Vk2!^*Gth9LW&Wg!(3UCim2TIm_!3AFcy5)+MfN*RDqGCHrc?VU6XS8|^ zgc`AVR-p|qBe+9+nk=5(bjCgXusTUQkPB1r)u(WSD{gCz?3Ji-Oy~U(GyaLS)xY83g4PZGBfWLz62PR5va;clZUNx#)?D(So zy?oZr9{7`nc-jG5ESX>SAsK*$z-8H5{YjE=n_s@PJl3+5+GF#NmyO&aaD5BcK+Nc{ z^yTk_$*FX<)|Ss5@{BEl-;gM{5YG(fSC;kBEucPsUOXzp8D-hjboKdD=(6Kw_eCy- z)R56PM-LNf^~sd~^I@^j6l-se%@Y+42c_CBOl z&jb|t*R~hGyA|1cQ-lS3M;jQBl)2sHxW}nYl8v2!tt!(wo5|^Vp>MD?`<^^wF$2RU zwmjZKk@JPi=T3*%df`3Ue7Z7hQrp+n^Fjr!ogTJa>tAnm${sq%PN}9M1v#Z_n||MYCyNeiEoKHX}4^GeyWFVx^bO6+rCoJ@)dp#RJ9ftClb~ z2P#d(JNE)jsH-=?$_A@}g&;vZ14|HwxeGB+52k?e1DV!gR_NN0b8S?y>eeiawv5n* z3Vz=qL0cAELCn^wUFP_K`=7IXHw=IC^S??uWD21~b8Cdr8+^iGPsyM`KFAj}!t;l3 zUbqqqV~Y@#k|Uk!#ht2@;N2E!C<)S+yVhi|CfHCAt^YGj7&5COz77CpvA>ijzH`59UqKd)KF!{&RhBaAp z1H$8zqsv-L-YSpVx&C8~$Lc>cXM5wq-4Li%ykhSqUtu}+1^oG%SJf%f_KXqQWz8Pf zQ0be#hU-P~;N|nHxkwLF*d!vVVf~5av1=bD z?A9@wu%fG_p+eP7EZ86+Ps%$N`D*yxOVM|s zBRLwr!`aR4-piWI^@$UMp2#hIv*OX|{dGZKDKZ!hHm~0k+AgArCW0mrE>^DX&Ylb~ zc9|u;k&)$7!0NHGxuKxPX&eTWh1Xz<#Mj*ky7(}?l<)yEnGQ8sPP&~$=+(r`3o*9> z&*dxUT0HzXB}pi9zjo|~8aIsY{EGS#gggr}0%XGCS_!Y@!hwm<|H{>p zx{~_ibZd{9Jv;w#U-LeD&oY7b+cF6@H2v7^fhv?1G8^m)HXKzmS4BR+F6!A?oob~9 zCKMjTS9H%^5a;p!+sby4*#Bgk|B3SKr!aZVI--HS_qmhu92dx;!$oP!l4om|DoJ_u zl~DZyX$q%|`~p*gtu+=zw`}b@N%`Hy0E7`r7XydtR*Mc!bu+{xL>Vn$Z{2}zkZKCF zj<@EAXhOf8+j3rC4A=eqb(XQ09EiiOu|c*giiP3~ywX2*p2){sqVye%Tw% zp72fwI?R|r+mD`*z1M@N-}+q3y8f+|JF^K&Cxe>Xof&w3-5wdS*r0Q0_xq1^QU|lW z97~B=H_0MFE=xIFeT8&|lCBK;0K!iHPH2L*7AQ7#F0$0DU)j;}JZoBr=&X^b-Ba0B zUXK{%E_mK}IHyc6iQu@%RA5+sWht1OMd0^>cakyMbMX&ehaXrLz3vO*jFL|A=PvYE zHgkoJf=4bl>5to1b|vYmH-N%abBUfWhsluFW0oK<5;W&Kt$hO#7#IMSy48(cIO*Bc)vAk@}r`LyL-{+1KzrJ zDx~R_{I3hZg$9Vqd&87C8F~fhRAr%&QnQh*rJS_jLNQDMTfL!G#~a)z&abC6N-)F_ z`kiV!S!ag@gdTV21nP&rfRxWUXg;8Kf%h_9qCj0C!`I^sKXUzte$%@oMg9$~b^>y> z!nJYRJMs_UfP02+O43^eMs_8jm%_@pXJe-1^R-~wgMv>8h$}`UxjcfWc4?gA8A!P6s4&SD(vOzYT zUbFDZZ?tbAkZFTGtiS2K?n!a2ta_o!D=(_I0`AqrqZ_!0E1I?@~zWpy6q7C#!# zNn6o;BYS>HW_|b44MMuED!}f7;6Uyws9_~QdJLl1Lfj@IniS*p6CsDp$7&NYLa=g8 zC&0&`zi0CA4h^ngzGM%jH-J25aQVc%=dD+E3M z;R%iE%o+FyVT4@l@DbR6^-dzj+G3Gu`S4rshXt4i4`5pZSLih90eR`Fp8%;FQ{Imj zZ$$UNshn|>o|5Jn*>@*0JjpV7H_)25RmBQ8{4}dq4!%h87JNqN^ZltQPbEhgHdfdK zUkh-fA6&!85s&x)GYAfXeBuJM>Q-U@DDto^CBM?qy7XY+3*cV!T^6j``()HvQ_Es@ z)QJ#tBW&6+%j`3Q9MQXEegN^FtkSzscJ0j`3Z(>4XAB&wBltho4w|6-sZ~Tn*mbg- z{Vkw2s968$RNs7*4zL+sesGKn&MiaDJL5mx4ZuU_yQ0voTcS)NumunV+Z^bl;NIlv z=gjt)<|at1>rkIg23CFK3lt(Bbd`v8IXz~(!tBEGmK3kmJqKg&`QJc2RfnxeAdAh*2k zOiM8eFCURnK|25K1uzmKxq$WKVAcboVdREZXxczclgCX48~<<=Y%Xw+f!T^*0y;@_ z=F<`B8dP}+DMQAhN^l?$s&b}P_F2~4ouGQk<9&GI_m3!fny@D^FcAbPN51)UtJbWB87?c#Jix;)wY+z(s|#p@*xuCMCO9bnIscVA z;N3~o*D0Fzt825*6)(~*(9EJ_w-w|(D=7g7PJ+Z_Fc-5In&p%z^xG;PI?6JYZtqy^ zc?!`Jz)F!+-Z8`M5{ZZ%W&&*JGgkcuADJs^>`so07hCG$k53XOJ#U+{F7gJFX`9|j z0i8Fs?vb@bCmA3XMH3!Sp2VfBpYqM(8i0-YEv}^%3LUKxv2WeZ~=$&Ryd^4p+yI>@zh3dp5_`MaY zd;wcmgdVQ>pCJ$dU?{4y%@{+{CkI`(R+|7IXAU8?S1ODgoTBYs3trocQ z&8hP3_C!q$@JhQXbBo^Ps4MOD{5d_vzmn+{JtXey<&6SOF#DI=`JM%L4+AP zq;cqw6e$IyL68zb+HVcleciX$`#jJ0J@5AY^#^X-T5Hbz*y}hWDRQ?|rLd2M#}%9f z&xRqqjtrPcy&63~sxiMZGcX(xWB!C%=NM_MGtMF~zz=rn%8zCX&cs-eHdTIcB8xDX z8d5W^nkAkLq^~Ee*RWAmi&6{(X%K z;n5eYNEPI{C?A?!@m-Wu?V^iu@PnL=dSH~~Hx9MU`gUjc_S4O;U&E8eS)e8Jw|3LH zB6M$1weB^Ry5#-#JK085Z{U{^(UsfiYkl2dkxYl9CpizSCLm|uW=kLWiSBEcU;ABg zT%_-wLxjr&M$6xON!Uh2YIK%7J*oe~fQ;m`xEh?gEPF^u176O(N&BIKn zKcQWoq(;W6;=4d+EA^n9lqFYX_JPeU)?yEu;=5&@PK_EM15WjWhdx9G#aG@=1Ss;# znRv4mx(>^c4_+U{Hw98Q*vyD=zhpH1Jrq6RoO9Z7o4R2`6!VRdr9v74?4FQjDr{s0 zr;#Ar*hi8xB~d!5yg(45gv#lqN_mh+9&`M1C#*=9mkl`PZ#JNqbw|~hp(XteAf_8X z+Z}V<+gSvWb^EhZA#tg%%Ig2X&`?Y?t7H&EYs*9Tb#7dwI9@c<$Pi8)MmLH-i0FxGEIC{2VRDLh1UIN!X^}-&D5pTwEWl4%~(F<-AaN-Ok_=O{p(z zV4f6HA6O|$3USN*SSl8=4|$QPqVX0u4Hx;|+l8vPHBe$mjX4$lJtbWk;hch0l@mlx zz?o;jrgIEkEqb3ksdN%bOz&P}-@2>cvG)a}-`H?@ijszu;su+Cy4?i-2SJsAC?$<= z>FHUqx5tjeb9i_s&XEV4k;iZI?Lj_#7DHHKUqz63XdkZIIRE zo?Q=*aZ4&ILUj3EdEb|2F2~PFSw^%7$EQK+&IlAzz4T8}u{~W~sU7(#Sypx9_+{V@ z*`SBRmzcBon4z0I)Hkq9RldE-DH0I!)HjRBg2!tt8NZ8XAv#rB`&3uJoZO`FH^95U z3k95kd*eNCx`~kE(ik;-%WV#FVz5@_>q+89!p@vr9#vu*Byd(Nz$$h)Dzxhfo59*N znUAYKD|gJO8MkJhc9=1#i8F|&6{-9iCb8B|KlxDyjl^$GdH0pt@cs=(?Zoo1*UrP5 zCyOi<--+L)N%9jRhT@gm=%R}Z<=~uXQ$T1SyRa)V$6JM4O(W{x8eZ2BE;3`<9$&ip z>?vFOUc*=ft1=~kh*ThWa!LL4fTkvV3r`wHpyt{5tifSmIl5csphXCTD2*BqlI+sl zX26xb!{GVF%egQLo@XdW@Bz^mwE@3VGOGIAC)J&lcJ089J#iY`04rTvMmg#}Y+p#l z)Htvn!Mce*?u!Oo#*$?1T)A0vq4cZ;}q8PD5Yy^bFOR?IGO6M*mW}yXYRcHgJ zXH|a!U*I55UIk7A=R9L~FRzC_?_SA&kxUk;3^Ben)hxhsd+x%V+(;HP&hf{iEujRt z(Fv&}L7^qB0aFGyA>-+Il9y`O-Iqu_Or$J?Qwt8RNaJ_s znYxGWYs9(>23I(=hs*`(ba{_uzH=g;heURC7idWgG36m|TFX_^KL88ca z^Un0pcw{J>ueuw3M3Jc5I9oGe((O^5XzCDF00_rBq&$Ood$n*C;k#v7FN%l-k<4ao zV!244RCtwe@DF%!{6ruAgEZSB;BLPx9wrSZ@2S8kr{*P-6}tO1i)L8jz3Ej@~% zsho}edP7hUTMaSfiXigNEcj6MQ}0B(dtka)RGbK7hAw;8JsK1c; ziADqJ(Ex^j7F?x&qa+6Z0}T_b}qw>fNnR!2+^dHk|=1I>#rb%sU4LhxE1~5swyzL zW270J1lJ=Q+-Ccgki6D@rL&LZ&0T)btJbFRpQE@9ke|qn#mR>AIr9UUkywUo*^h$( z9>~+yjT|jU+AvmaHISmW^n5TgFzMkP=Z>qfR1`s$Cp_Ml`Zi9U=AMC@GH^sxrq!4| zOck(CoJE#o9Qwd)2yLFYf7q%&VITtC$l6>ltw7aAjYxQgrCqAEN?9pGwvYEdS*};E|AA zdq)oYqASiI)F3#U$f6<~mz4s*J)9*NFjt#f%z{4NWL8j{2n2luh-7{+Gy?8kPcV># z(YMJbX@AtE`Hms+n>~jju8t!@Kk9$GaHl0 zw>?AF{?wZn%Up8#z}=bN)zZ#!@v!rOtz@M%EppJlG|p-x;F+Cjm&c9UqE9qXjwGPg zPlmt0F-=^U4Y0M``lWEet_DToo1@LcY-3W4wT7VRyEku?QIL=f5{{P?najO~4%s?U zc+u8e86zAN-~Iy!#{+JF-@gy`)jRCJXb_8tn$>HU9vy`MX6OU1=(LRwa`QC-C(Jr( zzN>B99oBNq%b5dO>J>jeK8C}9AK^z9o`BUBHv^53FyQ<~fM+Pfv;~pzV|9-dg_~w4 zb*^J-_&f30R0!a< z+>DP}X^yP@{i$!*aql(6t$ceC`vZA=4|KMbQ?j@0m?c2Id04G=1bM%ezgRb5ev$Sn zTY3q%|Ch5Y;P9uj zOn&^Qvs_5yGA^ILzf`aWdjyOF(MU`L-C@^yz0V!sQA@KV?2Jey^L%UG%(qyY0Y9UN zi5KEKt8E|Txt%r(mTs6%Dl@GoT#mEL`QQNhw#|B^My~2aCDdO3E&n-6^UqT`2r`gl zesIuHCi&hNQj!!c8Q>Nd!z;vq{+StAawVCfRZ40uU!Zx&6!)o-cB*YP)XWD}P^7$msBWJ*6^XX3vfEnRmfsC}PNZq6ozXnNN;gRwAH}b|w&bQxu zBuyuxDDoPMc9jml6dY7JZ$EwEbY61%XFBtCGpxa07G8Zy}=_ud^PPCjY~gTk%*~-+&|{xa zT*i`lk$)+z0{KQdb6A2|J2A^760W%Of?c};7CBcl3E;%-Yv89s6^=)JCL4vm0u5&h z`-Ob;QN+k;w)u#p{2YBmzU#L5hn3~+>X`6F;Qu_5H+E&aK<`-ApUmPLcp8Y`9|kP( z$aTsJCxYJCmP+7CkYXZhXOg%5>)knU=vPj*rkA24Ho~a??})dPtUhh0nWF zwbYU6M!w|PY=kwXMQRh|yG0@dF&3oVN?y+l5jvg5`&QP7Jsd!6JX8Ku&+Gha_NVi_ zoAk@SEPR^*yS$<>zl1~T}jNq&dCUj^IV{OALrUaFqi+ z?G6@(RtWBJ{rmXEDkU^vfL%OO`0%Uajp+)5$+!eDc;+l(*oCWe#`Ol_hrX{u&@4tR zv_73D$DJjdZQQ18&3c8iCEx&FwCWp?Dz7~Nje z;i6fWfOH@iABd-Uwxsd*d!O2$U1g0fzkbuCOJf#T3 zz2_pxJyUk>&!uAE*#y&F5pQcWg^+?3dgdCEhZ9+2a({!c;rKJ)H#?tlwzl5RcA`jr8>Z}?r4rXOE0 z>a&d=== z1r{EpdyiprUTRij4OMB={2v>HOERy~BinX7a~thv{gbW+83%q4@-*2Anp2*V^h;@a z^jVxbR5BHdc_mfi<2S*6%n40|13WW*A+wRhtk81%oSSfs78J9lt{kVs(#>x~DEOVl z#}%)Apz&+u;E`T#4h``}+!#|eH#?H7&d<16RWzphax$7nA0STJkIesvA35~j_>pHq-x~S>v2K2mdH%Zv&D)+0tVT^79hAv_;zBjGh3BaG zuHpS=QD`W?kjnz55icg!6uP{P>ueZp3#ex6tO*K`IvNTO3khKL4lri8dAH$RLDYHQ zgGk)nXgODMXdb9zPGu}C`g4}Us8T3iksvBwQ6GWl29xW!fh%a&3k}#Zg(LmsUc(>0 ziV`fNgniXHU`P+z7hhF=qap`*T@eYoE)OZFyV|qRV zTs1siqv*Ch=#n60CHQZbd`SQKXt<#by)MthzBH&Z8i@GR>~jnpb%Jih&a;C7P+@!%XI>%T2WSGR__z|6VI zM})yo>wc_VOBXIB$HF1!ryK4oDSyJD-NJO#zu<5hCSvjJFF5>}Yw(M;_2iNu3K4o; zp58&3mPWZ;Xf#&s-~pX0na}sB%>(+5L=&Mi6T$Z9(4pGGDjzg1g&;!P zx%^tGnYIVudo8M8lY3klwQ$0UjK&CROzZWn-~p6oQ(N}WDa5!8@;DEFv+wst6NUJ- z6u&y%17c()MChHo28M!fX4-`hSlfKfHe^tz%yDMIr0&ieeFRn=LP=rnAauFlc~AJ4 z=_|ZqG|M*fVq(ilbG~+WSCQ%;&ib3ah|2t+BfTw7Aautg09Tq1LITuLNd0rBSN!c(_ebI1|vi+;KJB!3E+&Gsce!h8lIZ8#Fq z>eOYoSxDLua`5cpmpnwkatbO_*j+G!0e4Ucak>xR3B~+iomaw{?=#%_`XN`RgRLa4 zVF9@2Dgn*o9VdNTn^)cYj0bO&NFf1iA}HUCkB6?sfgyBhjX-@UvvwxQsuPDMnmNf8 z;I5oczYxb#njTSPH2=&7NsdF79WZAw-99zr@mDW5G6_I&Sti37R0o7`xzMsM%ANtq zE51&g-5!kNKfK9T)faGAmGeIj9VcNO%WAn?yR)r*j_9w|r~XY}d7sLAz7l1UZtzV` zd_udny??mHLWJr48dyw^{b`VcETyr?x%tAu3)!A8%2dCRVX1_OkOkOv$IA+-!l+VH zZ1zZh^d9p#O0v0KelE*g&J<4a5dpxa{W6=m6GiVnU^KQ2g7;3_Vg+3&$DfO^YEopv zNssCpzU4dq??H3v@L09U-7bJTTDF2dI*u4m^q&~387IomvE zO#+p*4av$65x?}7ShdjgUgVnXQJP;Pq7WRaQYhv8DU>$LeNU>XuuKeBXts$HNonx- zGg`-ICia}9U#0y6kF9$=q(rK4!~gVnuaO`D(RA(>ZAq;kf3(P9y7y!&w?3lOkZFVV zl;GRWHKMoT#B7eKP3#R(cdVsQrdj&v{MIBm8-$A&E(N(O08G$5vkp%7X)PY_KU9m3 zQQX#?5?^ObcyIk7eVM7#dehi{Yb_iu-`)-?ARh6t^6mYNnU4O){m%k4c2<6>5VKQ0e0o^Hg`@9-w& z0x>v$$+!wFVNISmwD%WFK(|_S84LjsZP5F#7xo-^EErdF#W8?x6)gECV?Y8@VAhE6Zg08j9RK8 zyubuc2LsgV1TDPDApWee2az9hBIzRuCoAutLX;8N0>D_Exx1D>qsHv%Qq?$U8}e{x z>(P(;N%aQtb-qw4bynojdBt z(6R=WEs!3IqoVXy8poq*wU`K%K%T({YB;CWLIvwXk&-KR{fBm?y;|k4UM(Tb+Nc#O z%F-h9aT)qgXfII(qlpDZC#E&EBJDQ^y)R>ozUKPC$!C_BLPd_10<~#z{p5f!Ps{&( znCIW%Y03hKOkF3<<4;*C^dX%A*WT|gRwp9}!VP4D+<8NSpmHTbr0>c=->xM@6S`6K z9^a2M)T$;zXnclr2{U<+PSz61CTYT_X9_ZIzaR8B?u_UNktkWj|><05PFIqD5**JQhGNBBV zXNdQqidP};q2v`oo4n~SGT=)FY5P{ZQJ<+wTYHvlyiNz}EOH^DKfTmto>ytHc^Qpq zAu#g!HpfnbB#3A9@m@WM7k5xY9sS`c0$=$saSq@qaz{}pbWOBG{_7#SNiLw`x>8ORjIokmv2Zd zpwh$kZYGonIP3Sd@4WLhnI7@7tv@KU?I;FZ>%X5k@M&k<0aZIeg~qOaGm*|E0xMs8 z{lu;4nM&_^^#hk*LGN&A5l&ZW-6Z>+1VuN2710$V$BWFnN7lkrf=I-pH$mo)7S;y>03e?*)UgeoMWBN7ti2m$=r=E*mf8B=pHPkM6%{J{DpHP z%KV{!6v%je^LG+F^CTFm^m>gS8#am|7J0Yiv`rG zSO`Q(LXEnc<6QS?^?2}svxsE;fWPfvf zyU+*gqbIRE&f%FWZQODL1XGaRt*#!}aDvS*COQ0mxJC+QT_>c-m2_;0`V*(!F$sfW zNPJAil_dC7F7_Ok;riWYLf;y8a`9tKeaa5M&AIr@hO3YIwD($owTwBLiZg6rGB*T>=Blxj}chb!&bEDWY zW)*q1-Xwqi#;?n}g8O5U@2mW?f@BHrOB!1Q4&TR=p%jizJRJetw1wMeA>!H8V`17u z`37_aoMUEE3B^7UomwT*FX2`Fb>CiwTC?jW>OezkfE7YdL>{EuCicyse&Szeml<%|AGRz5Dtc5Ldxy%cied zgxRH>_oqMAwzKzV?b=uj?_jxRYF=>^kWT=DyC1NMFc1|aLjnOP$5-q8hct3-fl_>M zd_HJO311Ohqbo`wg+s#zFt0fhU*^m_aicpkSObmfQS80gp6iRhTupZ75b zEWS|wPYw$EWfOzM4e(r$eUq$X`k0S0&&hWef{SJcpS||E|0(efO_(Jn;EtzSY&a!% z{woRCkNc}i=Rbo}7A%#Df2X+0jr2}5IDAr*HkKj>#*ySf*Y^fA3;cZut#5_rAr^pJ zR<%qan7m-D$AFq0_Vu1QPYVp1U1~a?*{+OuZ}IM_$sPrUX6M;>DRmplMt0cnYxP>d zdTMzKrG${LEvg};0LE$FRd+M^rtZR4Uk$Cb=n==t3rD!nQ~oq7yaSjb!KwqUcX%{L z9xxhZ@X)AxY0Z0v%M0(K=k1L8-`L890in|ZL-*^6UCoEs zV65Z+9E>fd0D@@665{-ZkPaj&yE|5}DWT-AKSCe<{;y0XWa?12&b&=Nif%I$1Eig#B^2NOnO+ z;Kp6>Lv(T!oQX&daO%fsqkQo0bhwwQ_B&Y%6CLZuv*{2t?>Ue5sU*ucP8+7+O0o26wa{rh5RY#y@~DPE+*3h1TiG zH{jM{RqidVQU@sJ`W^|0ClGfl;{hE-0qGmx{~zfa>_9mA?}5-9$lv_H4ur>6=5Mdy zaSlBDZca;9AtG$zRA7=gW>J?+bxlzPg+l75@?Ft1A4p_NK)r@=f!MyU$N1>dnpt`@ zTT5@Rl2qLHJ%5YT*q?ABIO!f*FgF0w*MA=)fLB9+`zSO3xDgbBgv($WXfr=?qvllm zwR6aIsKE%$y+ZSi+VHp;1Ll0X?{tK$T6xiZm9Nk9o%VD?Lg8d(SK{ZZVeNDtWTzr? zl9fdfO~YIFXxd&dd{9o0B0?5ls&%e5d9KvEd0gIc&-B-y`W4kV7)D*VUdvq^wB8!o z*46ceO zJKm$6f051}siupKg0ubAfAwZ)L}zb-5!A?!CBAaLMGnBTsvv^Qlr~b$8tw|XXU`7h zyPPAsqQbi&n#&K`3)x}7t;?nD4?FJ>{;&~Z&UB2&k4c^#dI4&zju>F@JjJfYL- z`eS6N$b}dW7ObGH#jm~etA!u@mD<5Xv++Fgd^M4VVB_c0nt?b2+Z@2huJuH3jOsQSsl%}OP!mnPlvu~=&g_uA`90J3hYs*cDWS)l&!_jF?u%$VT zccz8YZbk3|!ObTPQEd)Ac#+skSOt*Ll4ES68%4~zQVm2f1i|p`1zz9At(mk5nd;Nq zkc%N<`LK(pK3atz)1bIU?3xV_{Rzj;=vv>?mOdbOf;c^Dx|N9T{(4~g(1iS*VM+M@ z><#(6Uieywx+ftrRojf${-7#ZTyn$p3hY zXG^`R9-dfz8ote1kuJxoqjtscSb84BI=?zYQ?a>G1)yLwP!6%Nv@98~JG|uVwR#<9 zEP0QTpk3MC#vfz8xda#++VvqcKu@3~allypaGYFdqlC1#o24QG7q zQGXb?#=KY=Dm1_-S}i*Awcbr|8V#=MuR{9o>2;nq$8yrf#vglq?aObB5*fMq4oXzG zvL{r%wt+%ReQwJ;`Re#%w?!6&Hv*4-6MYqYwVg4zY0%y=Ec0pL}AGkx#-YfTk@E zW3KT)sjM>%Ordo@im-v`{~Wt_ znL({uC4jbzA!`ceIM$K1`}DfuE;9nBClXdP;>4;!h_&cPw(uvPO~Y+_X!kHn{qP|i z)dN^lBFV3UT>PI!AAyS=B;#1WL^;>vFFO}&?TAx1{SvNlkAbK+_{ydwg<=T7yJWKa zvacyaF%Ym#t?`f~i&!c_BL@tRY8qh-|X2t6~VsF>lo zuRo>BK3lm3Wc}1%0S;6rUJ9rE&FE?013kc8B#xB?dNqSv3LXR>dD*RE3bF!dO}HsN zo!fhL@)t&7I(Q3upl@efg&%rtJ9dwnrSQ;$PLB79zk1yek^JSmU=%?AxkWdiT`}Lh zA!zZB(*Y|M;dDrFhXy2;JOhef)&Ue1Fj6+gwyq9}J|K)l9E(Ar=SULEqli&(UMDuX zWUuT5ot0jAh4b~c3ui52<4vnjAaPAF^*EdoSG)9`b8{at*a2qC_nH~39*KFoR@U#1nlctCKH#lVfYh*>fYe*Y+L+a%^H5fFe zQl=Vu9!`{>KV-^6#VLc?rU8V|5!5XyOmcoVssEuT@d??rN_3 z3hJy^Fb&9EO2qhxLFk1Pc3*wiRJ%U&xcf*G=5b%L_6zV5uo)cl3nAP{${?I;tS`lb zblj^7$OceluG7B5$3i6S6LeA9hzBv!iD;ONCX^6};Yotg<0qoy3@jBckma-{=G!x1#GFJk z`1$t$b@{J!On0)$_DxdS?is%{5m%Fj_9C>ZY|L#D0ut?ILq!BnCWNo{uHNDi5*J<% zvmF**XqiQ}0jj2`_)ko~ZvoLCa|APZC?tqm5c&1@dYb9b7;Kge{465K3=!+S?A_YO z-9f7UzOnqYGAug6xWS33Z-BlA)=6UdVf_k$nSunVN31l4NVWJiSTNREMpso>66ZPM z@s>OSQoGGoA?O?)1Xb zZBN6^^0ny`r@Qwy2f<4PfV@i3;D6GQjhPX*6mkEu$QBg%GE-d7`+q|gXRT8?G~~BJ zlx48;DvDYI&H+Xc;GSicT^_kQ^x(p)1w)^cjjj&vJerYt1EZy}j!~s}TRBS4SlN_J zgb+(8UFS>KfNPKzp!f@91SB?I?xx4i%pW}~{y34$Tu@n=Qt%J2<}Vg*LXpu~yOF9l zD<|64ZhEJfsYO3pzVH|EZC30Ic^>l9bAMg$L3_XLc*7{|XF;z7qp(=B$5p`{k4^{* zX0IHQs+6@qt744Qo>INGN8+o%Ad4oYUMj5O6za*IS>3y~$ThOM0E9=7N)Qaj!xhMq z|B$T^CbJoz+yCD1DsUU?O+L0)d8ExSj#5NbnX%}FQMgD$OFR5*SC0i^UjPzuP@#>% zm0&06F7r2{Wmr;4)n#FlLCHKo-nnezchJ()>$tsZ&$U5grhc*`AKbyWmFcAc9aYBZ z?)NZkXG8e=|Jd2!1cD*Ozm$KfWCI);Xl?A?SK;2Z_WW2k+nJ%{aNm-Vpoz#g=?No` zB>vQN-)!~-EnyOL7VOc)AuQa_y1jA9QxtBI#?6c>^_{uY!&@Kg|9iT_{>cUMAi;t! zaKZk`vJ2Fg;*(HGIwT9t!!S1)5;(p}8H~4t(m~E-9W2H0R#=PPt08{L^6V|JGK2MB zg^v7+T3MKI4LyZv$&uwJ11aXZsKdB`>jLqHmBl4PvS;5gtCex;fxtCsqxO9uLfY`{aDEXm<^auirsa@@cd z>q)|6t;Y|<8jM9Ae~(3^QtO(xsfHIW}AFfy7lcn};_GO(=_*wf3$*JwW03CfTTV45=%x zX^z6$f{kE^&4)XmdRlr>@oc$MdHLvFfVHC?4-i z2D#>b&(Io=^roQi2F~^^m5QMCk8R~u3nMpEhTK9l8=HAF&bQN-@(pzl7(U-8S1hV} zN6NshkH`}~p zv54bMRKVGElV$^6L;v&uR*2NHn7tye5Wei11W4YsSA(I`Yuq**_!+HbNKy5bC}26wTi1NMChBVznE7y$y_2x8Gp>lU^|Ga{H;&;|rq{m&kpuO%KN>4YVytjnk~apCf6 zzat=xw;q-yrDwEd)@@t2@^pWF#$yoBGn6GRdWbU^7mxe;&oK5~98lnyY57V%U90}v zBP;XT%9gD2{k7jbRPiF&ZPF=2S`giJSIT7*L2r`Q6QH9eVFanGZe4%vo{3)kVU~ z7V9IH?&vxKGfp?!dSmi9`gjg~+2b&*IEoyH(lseyuXc}o2mPNVr36}$< z%uAJ3+9>J1o5dx)v@F_Cl>_MIkk%+2(Q~PT*TiqV{jjOkZ2a4)6d)RE+@aFH^gk6O zhUwzFW!t9L+SazZWg@sQ$~W>^&(rGgp8S0L8F2eiecToK9xxZE zA%CCY`+Bpdy>UW{{)Kam`+q`y5M-A$4rfnkmrbG`0Hx8A-pJ8nfvIaj8c@v_NCFzF zXi4{Q{!RSn(IHzHH|grc7({8~nnNS=nuU>DD>GAax`Qd(pef7T8j#qNxS8$tA(dXG^z|d-d z44tRT-ci$Fd3+PZstu1^J>K|nurq@tqWiT)pPQ~9N7c$R>n_}|R)RMR+$rO|sN8^G zr!f(go$N!@qMjy{EWWd5zfI%<*ZU&K-66-EDpu!w=9A0?+942RTrU4W<_U7mgmRX! zV_yK)!gLya&+~6B)^xE0&}4(ht7jcyzlHt|{dBw}7;_MkyYw%G@zi0gU@jO?KQQ?8 z$ng9c=tG#~G?$>er29nKxrv#CQ!CG?p`Kd$PWu*&=-3t0+yHjB(xK@4? z^eX)gmdxM8SKlG=d%p4okN8lAdTx_NAiPok2T!60(XPV9=YG~6)mTj#q#Wv}3?~yu zie^1Y71?*)?y0Pp`aSdITt|X$2t%6RHuT+@1}EDH1n^MeZfrzi&+x~?H% z@EUN_gVm+>)kLQpOrMQ`%L$>dq|Ysw|k3+U>_X8 z`oLBY8DNQ(X)#3a88x7U_DRScBnvdtyIltNpF;gtBmdp(ti1X*^Ahk z3lIgOH}8!oNGAfB$(}r2?7o5+xm*PCsyr~LKk@HaV}M%t7V7gG4Ag z)FQSE!Vm5R8H!$FZBqjlYY!*>S6It{{84ji%9b|=NF)^_5R$v@z*CY~e?6kbCzP;i1(CcBuWhQegZpdAEm1ZF= z`kw~zUo$EQT%WMMs+>V#;#_HIfajrN-=o#XB`nVU#-8!1;$4f_=+DC47F=eH+!pZ` zor|w)8vs`VC?Hb5lj%p`<-=}i%jI5O&g}jEtSXDcc#D2U2k;+_mwNl)HJ-HfgrWUJ zy>NmGPdodC#c}4M8-)`?m4_u`b)NMeGp%A3NaiisR?fHkT~dG5;eC9MlUfbMoy1hK z#kIs3YOPkmoN=yvoEq?-tr3>yX>7}2SW?mM>VPzfucX?Wr;;-YeDL50jU+Kt&HO9} zbrYm&H@L1m6xSnYgH4eIr(c!cXG_~`@~ynFa|X2HBb|p!pUO~=QyqKT{`91yz&xMU z@Ag0sa)IVNTZNifEPw_4Z!$qq!&^`4*R{D2ig$KyVXdYc^QB8SCiA^ySyC2Pl<<& zd_HDG)Vk=K_sEBdNEUX*?At5-pdRv1f-Y&*Y%E~BdKrOxZ3)Z1G(?R*s{gGlcB|lb zb}BT_8+y<%Qg^!voUM)fFZnmjv1j_U;?f1Jp7d%I!6u2cJEm2_qz+7p-&)K4fzPzK zF>HV1b5OS3#lo;}sqVW8dL|P))!oP2D*bPVJz)0Gl8=n5pA&}O`oKl(b%NnFSFfL15W36iz2iN&Dn`=jeHX z7o9Lc@hYA~ov;u`l~s+>D6p#v>U9#lm6T?(;=1QI#sq~gtso`gM?fpk;O-3|R@%IQs(hGh_dk{Y+rnfp%f)L)*V_o6`!$4d$W+ zIT-r4HI{B;>ut*r)f+n5wSlM4SMBR^F}|gHK;A1ysj5mu#Jdn#DU@IWEqx6Rm+#D{ z_I@Djp>{-kbwVa2KY74WtWOD1CqXeCHFf)MMgDJCSx-B;frBH7qa-V(RZ$Po8ON`~ zM8M^_9()dy9pNi79Vd0Uh2-brFiouVZ|3Z|SrWI#J;lEY^gV5pUfyQ|`%&=crif`L zrG}J0dM7O{KaMh`qDwo-GqfnlX1k@`7AIv2y$zn&#(hd4hEgX6A^szm5D`xJu*R#Dn{bnuCA-dJ z{#n}H*cZuVPdH3xs6up7LtBE)@m|6@LHE8z5h1k7L4Ez$kxw>Fr)&HKYo}f#69@YD z2#QH1NEQ^gSUVe9d7J|5OaUxZAe`_&>=pvn{j}M<5m$GYzXWinDWyOrlUqsILg_G; z23X-iqHhZMcL(*}6-_>-{dIlv#;I|-iCj~>{3ToucL#f9!8k7qY22F)(Dv$GBW$es zKOrLkLoh#p9Hc(16Y`0uDMH8QaSWau<2iegPNH6_v2&urbGjqVp6}};&|v)z zm9VJsZMJlw#gge!$m0}t@K9V3n}#lQ`^Q$r8Q)+bO=fR~y)t#LY4x6XxThliuJXHVK)~C-X1H()V3Q3&$DQh? z#iLt`4AqboUJasS7=GMyW0r26JGauuQ>(VPEs|sz5U%Jo^s`< zwHL7Rz{D=Ji&8{Htv<5qbY}Dzwj&>8ej&`hzfX)UN&Ux&y8P@($TNPKvOW`KTa!}t zMCRZ&Wc1p0C*O@y{T8<^lo@hGfA^mg9>eipI!GaS`eMyy!djr zqMh5lcaA)^;!qy%O7tP=$JTzT?7G+TXFfWQ7*38R$fwY(^xK5I&BUknCBOr)u2jcv5+P4X|3p-WA5D2s-TJM0QDHz@GjOmQUXn-_#`? zkNtMvDCJH}0z&+&`de$`yEkU0bvyWb;hVP^qy;3l9ZVH0pLx0kXe+KZ`8YNx2BgVZ z(USe?ZyRqzjDsdnjXU%kORU|~N*_1btIC|pB)oWC~5uHL! zU7fVik#=e{-%*|SI9ymLh@(^`FwS)B=vQ96tPjf5as*ie4VAEboWI<1vteHW14ImS zT38cNx$1hEGWbvN$E?22zc?ht_8SgK6VM+AWS#uGj9kmC@SHh6KP)nq6GX;nR%V{R z)|UizZ3}v9*uiWCyeV{6w#_`^MP&lH*+EA`{Qho!;{ny;JUixNO5~_qU=XIPSfnkVVq9c~j{^-L@Z{D=J@05LMZta^Z{pp~*7sGe#b0Uqw75WP zKb(io^DedPALAIMWth=0$%6mqf9?>3;6~%DjsNvSv5>;hfB(H}DG~12Om2UEmBiMq zQx0E-tp%K9F1N(?W`EWswZtkWG;a@RL_wN_$yhRXHs<4o>Y}27*-3tD^==imj{J`w z2jE%qaMp$|ja)6}r+%T1VY#Kf!Vb*vMNi=#RJ~oYjRfBr0l)7Y_3Hj%N#Jz+na_Qx z8^1yr9G6|ouw1q2RnzB0xKWT{>Z9T3^j8%lCK)Bv$_I(r)&xeLfJPvO$yO^dZ8gpb zjFJ`(Pr%YgRj*60FqX~VB>pxbW=!r8uS)$UwxR{!eaBR0dRS{q{s^$=3Z}F?9BDXB z9z9ME#)jy7;W@bCatEy!_FrZqe)~lAXg>)x%hqU19566MP%lqCx7I=jX9L+=U0`3K z_A!6o>f??m?;5orDT=@M;SdjU#`fAR9UYK$WvsV#F;G2J<^D+bdsV>y=db^Dku)aj z_djH8DqO+0hJ$m(+Q2g{;R9(B=%l)}Iwp2*t7=*O;%FPOB25Wn`AC1GnqQ^LremEJ zX$&o03Tks^`1P3Ym3>}iq9p?nOxxL|_@5=+OUQ0!Gajeye}r3&dWE>l{MoMVB%({k zr^DHXbRgeXGO1*Os5V^q8a2aX+j{T;Y}J)F(UObfhn);sh)IN$L%D~~4O{vtrnvor zTYV-u8Eu>_*=f(3ZmD>%aszBeefQ^6#4-a55k+feGfABumiyt7X|NZnj30B*# zs!R_2Sta^^;|2F)!>j%N1se$fy;zeo{yoyXPXefxEK-c5}1&3Mx;bhPug1ju(-E4EyEjTx}v*We;+X;*cihpXk zJY@ctj_Td0yWa_u>k#9y`?U}QcM)zM|JJ$DM~98S1&HnTe7DN`o@xhKL~}0=iRH3A z-uv7LipkZ-yC(Tc-(nW;rPClxN9s)zU|chN$epq$M6hcc@GZP z+R3Zj;JXlbW>tsHk?&t}bLlddmT)&C^+Honw{)CWPOytyJG4i(Adl~Aw!|iD!ld&| zN9??z)WvY2S!mwJ+4y?QAlI^aHs1M(#*I!I{ zW+8l~7XJis8_hq{;ZFP<=jpg=Di?u*cV`r1JoJa*DkC7R;#M26D8dc zWrHYq{A3h{%)XM@Yqgo1iK$Ref?w;N9ZW#$5elT1E*t?xii?%E*}HSKXS$x-1Pk|0 zAD=h${;U4@JMaKCOZhF|X53pD>=*O=*$N*Wv8jJBRg=gvADtcibNDR2K4no>>6AIU z?i-Gk49GWj&6|oHzByq$jKX}ng1262w%BAxnLbUgl2EHOtJ1x1IVYxmtT|NHZ4U;=@1pvYmA{^zQ-cCqw+JLRS{u(Kso3qS*JM ze0Enqq&Aw=F}tyAkCwGvVd#^j3G!#Vg^-H4Dg7+v5D1i(f#OwVyBt1?B9{L^lwtWBnu3?Y7M<`$XUZ%Qd5Q$JvO!eIe zE#RO^?~Cc-^(ql7fN~|1DkA9gIWj%UOb;}_;R0LjE*J7_tXrl>aW*dh-RBA99hyGd z`o)3P&aVDkT&D)Ca%efu!fVg(P^ zkXAzJhPCMKMjE7Bx;v%W5AXZ?zWwceu5->`z_qyNGv}Cdj5(%zte$LFzR7sT*fU#5 zj3ISw2hL|aOCq8|{UD&{xT7kZdR87xr8iLeL)w+!*JND_Q0Se!c;P3pr}f79HL^bO zKlCWOZr^KW;{3g}UZ~3%F{YVDaR-74+G`}2SK87WOfYDyeK%4LygD!k&a=mK%s4FD z0CZx3A#yLI$QZGbWs2Mn~2{xHtH=JU(HYtz^tM88t`Dd_Zy?Ft82>#L+qp?pMN zbPMG*BC0jRi~}!43`)?;Lix{RPx97{N;%p5e=_uK0=7IO>1c$dm}7G-?j$Bz6uR_w zPgNY_m%`uM?8qkwa_eUDxtZ&Kh_>SXe?(i&T*ZUhXP(-${*&5b>h9F`V)$h#epqv^ z&N=|l-H>b}J@n>op69X0=hs4K6zL?Tj1@JZ>w~<;j^viy+=Wz^p>4oCUP>|(lTF9A zwpYXqMhj|RN&E6`;i~-$o&U#Yp(Ty%#mYF0g;iVL6$|Gla{PGsem4kWBl>F5m_`n| z`0T+RWEE3y{#2)0W2vh~qHO(=eSBSs2(-PIE82wf}(6eog--o}j3S zTOM9xdR+q;?m=cXQS>>jOz11rv9ybz>$wBbqs`oM4LD9 zb~OCguMeP_>9S-3R2j_?g8j|UCwX77({Hak`~I)(2p&XX-M5yU_EYIu>08%YDuJYj z3xK!T_lj?rExoekUE93rD&;V8K5a|j>OF|K0X4_BO+atckTVt&`rXFNK@D?(C*5<3{;oBh%ftL5|2z)UJ169S+w;_!|Jj~%0{*}MH=~VaXN^MrlwLjq zd(JQ!Zs8&%F^TI?mHfWoM0dV-<$K=PU499t&?++#dlen%NvH3fl?Jm3LngUY!4yt3g z=Hh-TGxRQ_tS(NINKg5dChUMaSGkKy$ay-i0ad2}nbIZdAkHMB8e7 zZP2@$`$WG0D1pW{0sqw#dRc9qkFt#+3&_unP-u>UlNiD$eu1S+1<^GnR;j|Wosl27 z2|C=}7dl6JB185qAQ6%K7$#{cDm)%SrgBg>ZAYQN+Fb25w*oR!8U9)m>gU6F!5CZF zgzxOUxwv^rNhOMRUAME&u_(>Yvd$s*}X@%$iiNq=LK=@#(-g)Vtxkzy(+=RiGb zl_WD8_ZW&&`Eswhx zf9PH>u8zS`&rTrqWAI$`O@^S+t9D&6*I+`Sz$KxO3NYh$YQhrPY`_aArUn%wfd{5Z zr*atV50-=CYc$pu?jLhRMm1vG0ccBr&rhSg;mN(Te62GW@6)RvUoF>l}*bn)K?f!P?9oq!#6cH7=jx_^fYWj`aek{QH zm&srmrcYhw|CFNqDPaw?z!!3jKO@3z=QP@;STn%WJ#)5$uBlNhPi0|dc2++qS^@ZU zr7hu40i&vIb}DVojUgfTF#{4}hRkfC|M37UoIqw$V&x=Pm_IkuR|!0^SSQcFGt!ES zPewOeSEi0}^h1$i(%eAXCUNoOWKeTnrYMUv)=n8`pqc{ZJl(X>U2Q^i1jEr=YJ7Xf z64*KoJf?G9JIt!~*7^x0#_j8%(rZ=`K-&YN(@xg1kb2|EL27&k`ZX4C@A!!w&^(zm+Rt!>@xq@mDOW2klkSZ zsYN>=-TY7=Cs0a?c(t`=KIwgWGGEjX3D-|AE&u&QjJq3c1q+j-)z{fz{SwE=%{P^* zW@_Rp{wk_a(c#9}N?(qdl}?Oh2CR}QSxQHS0ONHfepl{8DTb7hy)&)&t=05}yyzC) z#p@+64Y&ZaPsx?|%4<%9ZRK*t-o+ zbX1_|l1j+^H+3`}i-Tz+hWqCET<%?cQVMkn0OGuja6*h<>Bzs+bC6Xg0GEdfZ|~d=3L#Ay}Ar!%?~;bEqJh`*{2<-R<^nbR4W?I(H(= zDH=~Wcv0JncCup)eQ-dLDoo?}S+7CWJIm>eIEWHi@3@-{o;Cd%Y6U5{1n}7HfEWU% zVA!vq-AuOYwut|Z9sx_~+U=_wWw9BSXWMYCmuQmH%#qI?5I#Z|#qf(@(&zgZ*EHCz z|FYyJo)LV&5k_LmlMS-bW}dgGh%4V)po-W$%1FOnS3mFz%NHE>)v1>

Np=bxq!k zM}^#=+AU;1)dyfL9gwneOa8yl*3F*QlgvauI0MI6Xc3dDaS>fKYfv>Of9=yYXwVrX zK&2_Skh!3o>#%K_vYyv`re}qgD5~2r($Z<{ndIOi#>7Xf75txS*W6q*^^~^0Ty@e* zNqpaz*b72|_xx|H)iy?Be;pU>rf|FBZhIv}M0lIS#70Xq^NEFV4&@ik5E~awG*-A= zc?NnESQeU#BE`ZaY=I&*^W;%M2`bhV7;)_*-yOoA1yc zyR8kpGhUeEV+q6LMNd)x^@dS17r2uUI*(%tEE3`^Aak(;Mm4XPw7Rv=xp))aiPVwP zm3h<7Xw?M2@;i8i?&9w`y}X8Ei>Rm5e~FiKF|OD0#9`{be^P(jt~1T~7Ori@L{qJ7 zt-7htF=A|y<4F;}iq-1o1Oak>rhq|8a8bebEPn?-W(mTI-ee9jp3Q;GCV&z3mt+T* zP{HioZUqy%QX_s(ZTpDBe3#Bs(N9V$9lU;Z{EyAz{^b8*vk3eDuvr8lngLaxDLb!5;fuZ>>sb^M=^&wV2!;6JEIEE%=XeW;Sb_XO?9tVG(W=#!Jc?c@H82rc z@sf-KM{UjYk+|}8HY>fDsu??G)j9gd@#tG$tJyfJ@6|ENuhz$A(KjJ2eBBorub%Es z{8uqD#K6BgUo6gxHdk^qQ@CCjkdGE3^ReV=}=rK70zr5GJ7#Yxf1RnD@K1?=8 z;E?`z9vFpqt8{4$g?t6mCDH+qR8wgIrs=4JMfU9AqN$I!y^F|WmnYj9L?vKb#N=n= zvm*UPB-zX2O~AU~WRPB91ztbD!O&Mt<30LpmoH%l&XF;S1Mc8Vi>qqcQ**H zdpBFIqy)>lm|hMd?);rxu{XGug#J;ZtYeSBDouvIvtEg)pDI=7A7do}sudx?Pxn;j zZteUeMn5v%r*cz=M0jx1OD7|L1PQ@l7c#DkeE9BaOu?0*9$m=o{P)0{f4TQ=v;z-n z3y#8x*(Xoqr~PBxG@_yS>osIIJ8vnQynTdXb4h@%j5FY z=y;ySd7f4geG;6g_XCF3>pxs%pOO4phWa_9xcYI+(%1V#j5|DDq0fD)*A*TRlz$f) z(bQ6#e6*VLUbu9Y$l%`SUReBEP(7UT(Wl(^@OTt^5*R-IcXE`+Z$qxv{ul(aHUm~R zpIfAbM~kBapZ)&f^OkCn;kS??rntp@D;K^GY7AM7wf!%h7yQ}$ zNnGy{-Xuwi$zWh>BRJxt*UNG7#TvnTF+l6)UIgnjU+l;wNcm9-$=nt9U{oZK4&S@Za|emXu4x2POK|M-jMg%eLzN^$}ge>h#VFOe2p>^%d}J~=bQA9F;zZ$HIhS-^u1{; z8a3x%{mxrZt!tUbhi|@{KHiygLgn=-7}yhkTE6N$hY`}NpvoQDhQ@SQgpR97GCKf_ zF3Bs^@-f+A`+)lr4z5e$?(0Y*-j^hQdbM0&J=l?lGZDLgD8jr~^loRbbo+~}lK;j> zzBENAM*OdXoNZxJ8Lt`c!%(^~eb&)s%Mp5vVreJaD$-Y2B#Wm7!5d#9wu#(7{^6GV zk;$)1?GQ?r)Du$}jN6ThHf}6C{p~?DQKCvursa!qC+ZlqR@+CLOxBSxEfC}<_3@yT z`ZqQrCV{NwZxLJizl%_0tnrd598Jw_Z&2=!a)+XzSAx>b^1q=bxp^-y& zjU#j{X7||zOo;e&cO=vJf}1o@E+yE(&X1r$Y$C++LjFY)r@g!Xy-9YJ?|zOk1R~X# zxNw=qjePa8c~1nDfaQvr{Rf$ezdZ#LJTJ5Z%W1R}7m|UIMk^jaCv4`_V@9K`Y6e?e z>w{K#&%|ui&xut#%jM)`Ha)TkY)rI!&9MMSa*9JDYc9m^JGN2F-CFI|C(+J^KX{xHSyq z=ZRl5b>z)0wjN$oA#^*d|KpdDOJV58*6G;WrF7Fw{Gv;|#Frm|H=#0oM{r9*jF^vx zV3#;I@I{s3L?gA%TP#C=2|w54t&}V*;*{(6eU2X&wd?)nz}MtdPZDbNh$tAxhTE}| zhe3~8yi7l zc^lNbr#KQu*oh7$@H^>!^R|!LVV&yxxJsWO#(=bYdG5?F`k;oBKaxvZvDu*fEJCsz zQ@R8xPbz8 zHqMlBP3;Y6B7=4nT6*4Lf1*2Sf)X+!rj*NG)Suu!`XG$&;b2<8VAW`xuf3P11rsPSC{zYv?;#+#+ZnYsXScPS2+6nbYhNl!xoe*D7d#hOs=<}`5sK9LDNLoES4$h49mRfO3gwP zx}V&1y3A)Xmsw8apDJe)ba|?jQH%YSlYHKmJt_=i>wYX7d#{pm2dj(j{pQ3b5@YSK z=-d%MhJRqTD^L6VaB@I{GqsTfqJ!7-56({nQ4%O$lWe*Ec=1D$U(cI=*Pl^3vDxfDiWW`;Y)thS?$H1rQ zdtYx|#;LJo(cvfONtA(z{i)4wal5VAthD8yx{u!fb!cw-@e{wgLGDrK+;_0RLye+7 ze*E7UY6J1dqu*T}e4Q?O>&Qe|yf{ID?zUy@j*X3Vq_U7`jKuz)Y%(~qU@H`y(AnQR zZW!6}e*HOcyHvI8!+AJPSv=H%F(H-W0tY5+HH$Zs9au9DN)kqdB(>2K>f~h_)6EB= zBZ3{vvQlK#yYANL7;vd>)hrP*lMu!%{j}T45n!wzo{vxsj!{LF+g#Z&LLEejY2SuJ zitUN%S0ZGgstge-C860VSzOj|f@4G3Gz1JJ%@qGUn%)%?ek9cSL z=jFTOfb^CF5{0GvhnjhtwG*-BmB>E0=FTBoYo%vsTV5}OsI;Z}Rh?oiT%4%n&`s6l z?tC6A(c_$xA$8OICfwex@9Dqq=l#kM8SX7%hDM6mgX;&l?D^0dYEQ&F>f^&;!}Bzf zxi^a}n#?RszIJ;pR?`ki>`i6WgnaK*-`8@@vJH$!-jM~CAD>S6XfQNq?e^oF%SOcN z60(7L3b5m_p&buNXs?5%y)b>;V@3~bEsR3TTGmfJl}Y;KdTuJQI*;Y2&44MOj%(?9 zC9PTLN8p&NKkfl~4L zL;v$aIe#H~lEiV2aQ%>@f_p*O`6PSoV^x^f-)|PqGLAI6Zuqe6V9QzMIreT;@$B)@ zSdU4Ha2!Z*N}`L3i)eMGys+dK)#;>~+3i;3c z^3eD4&^^vYqP~@EYExgO{A7sGkUM+*zG7F3=i}#yO$4jd*x;Gxl#|JUSe0cA*`00u z^MMK!AU17hRpeTOxV6u8AfSpBSf`K7^@jl zT5W+GepeJZ1TTjx7L+agEjiHn;<)##HqV@=hdk52^`UG9M^|`pq_39_uM}w2#nsxq zQ?|aXX7iBTVFtCjG@dK^@cBH_WUF3uQoHwr8LYaZN*;M_JGu}R@|%8Q?D~W<3$s;W zWdA_yF;u6LY2hUCpOV%FO2S0^hKE;RoGf9_gg3(G_zZcS8%Q)XkkMy}8j(UlQXK=b z;p?4NWjqXAPO`<8PitK?ORZoUjj`~b4fJ;plX<6qSPjo;5c~kpaDM`21<-<4qWm+B&N8Q(3wLH}u3ZABSCpN#-j5$)OP)e4R zmjZone0NW2Abp^nQff`a7#Puz`2q-C@>Qa;RCp?Wwn>MjQZV=15M7@T&a@d?G-6U6 z(mpx1gD)wvsW0kBOx?fYr>43jh%~%A{yaTU^vO5=4-?TLlq`?`ap9n(`ev8B9baY# z$2EClTF_b@9W>U)>{Kth&Wv9ww6e>Ct_k<4WjNdUyBl9z0&Z3J`Bg%r6aRy36c#vO z{$B|C3-xanL`9O0NZHYSr)21%A|u{QDd4dB%(FFr_t>sn;~&QcC|DAK@cxflOuSUk z01vPpRVWGP{PQ4?HBmUuO3X0G3TKy1u6$+mrxQD;Oi_DrVmJt(4He(e<@ zn5L}Rke8KGX^3^MQoiaFDi7cFlhxu3jv+ja57q1@3`*s&#tWbb*HkNSkgPsApmhf;L1K$FW8f&e<7eR3%pdmk~zE=j6^y}Vz?24BN zHUWd_wdi$)TS+as_*>IeyuHX{NXg!llNBu1CN%r!zb3(FYRuA z@{Z_&7acu}*F#J8q@laSVY`sKUfactK70UH`%m69N7IMu#zGP-MCO^t`3Kp=0#hyJD}DoLRBD3j4e~H5;*{W9vXR-t zQ`1++r)i4J&h*W`NbfT_KxVq-{f>_S;u-z;Lf0E&G*^cR(;pU{g_Hk0- zMD3O~+0MDVS2{(BpPE3?x9-Q!bFHg8d`G9tc2MK6qswx1wJzIuKFiJ5vB+Pmg}D^0 z5^zc!qSnw66YUsOM#Pq<%LG6Zs&yHn9oMr00(0M@?J#Yd?g^w0yT*tpj(N36-tOS) zmakb(r7MP9_wWq13X^U8wUR6k0aal>pCT*KdtPcQe7MfeZu`9C1t*Kf3i6&Qd~dxG zXVhx!vi`%4&FFuDC=TRdfq$aU&{@DoTah1Z@nI!oTvR6YTPk^YLkoEbcU74GVQNT= zEdD^eacA!j#<=*XZvo{R)?dE*nQ-=|k1BrOoA_*ozi}-dteDEd(%hGTAGJ%$@G4?N~0(cuXOveU88_XzA=PB*T`!%H&~+^ zn$yUsts=!CS12fJVp^&u)!(0EoQpK>LoQX!{Vja5o>!v?Cp5`~2|9|U4JtK?79T26 zYfivX_?VkC`3Q4auCDwXhoAJh**&hDU(sy(FFXu5Pi)C_kaYMftVaj!}L^J#{*!kyPx74)+HUU1qkmuj~FEADtP}SseI*LNA zz%Nau*$W5u8MbwIZHQ=@15a})6w^s3y%dW%`ooGd-+x-A1>=VcE#| z@_Zx+vyeIt^0U;)sE1l?>o)AL^YPhF+Vp7A7fW0Hn2REPOWc*dtp!?D7g8Yg7=*G=nF{^D?3AK6 zLDrFwdsfyn|L*SV&^)_5c&m(IT{$7knC>OB6{>R8SVje?0`gq6h9*G1+X^%ksOlLL z3D?q_YkToq!GB#oWkgsDb77Lq7EIPAar3C(z3~%k7pZ92xS8YH;9zuPCWWTpo-69G zkk@r1Tji{12`5&?%=J4Pq+OhqO!Bp#h6zjC-L6-isp)N#e>?*u%$~yVi1vdOm`&S! z^~Z<0b%q%ii$Y1becuSNqmG`Y%gEoPUlT&`MP3mU^y6Q%(O_qFuPI)WGM9Yq*W=^-e7j+l32WLxuY>LIQx*-Ktnf zo~AVR9LN!d&KynqmJ)_m=)XZr#_x_{km+>IqUYti^j$FJLw$@E`dnBwo8zHe*)ci| zIEP7m9EC>xou4e72OqRtG^g+@yFl&KtE^6Yqt7p|*xL<__E_^yp#GCHhs9{hpKvx< z>Gm(R5P9@HPWvqSDI4a3%16ItHW|gyX_I3pqxZ@Gl8#&$%wp`gN#aU+lNFGL(uIUw z$uc0wrp#!!J16wjM_@mLaxp}gr!z2MRo9M}AugEgH|JlZ9~KP0>>)#{8?ZR(tdU7s z=<)m&!VU=`GF(%~=)wKSsrHygk`o()#?()o&3p1IwgY$v(_g#gn6}?ylR&vf{zb|9 zp;*sipk>hRheP=+@Vc0tCHP0o()|k_W}l*)W~GL7`2;}B_Fs{?TtsucR?DvcDa(B{ zNByBl7@l2sgqD7o_xSnhx%b7ufyleN*$@#YNm$2icFVASI1WB04njB%1`DPD&)dMS z3;38ax)FZZ3oIe$b~!T{wm;_{D`Vh?6?U;^I0Z9u<$jAj0xQn??^yb%?lwYpkNt6OXrez?fCf9Dh)C^iR*8=`{`|~;sEi0y46cN zd(&3gLaX~sbt$tRiVnJg3^uT`GPmZ-AjR599#_%2^-#=sB)(2M6M_RHv}VmS}j0(z% zLUeeER0z@g+`N1?AEX9+YOKa+M;0?Z#EDnN6iovBltp;j=|cfdF?kfOs?H%QlRqdM za5ftjSa>>q{ep_eUkRVneYK^;&gkPa>L+`*&JvEXu=N8PGH94Q9vtSWp3D%ht+OK?_AzVj6}=Lw0;^8gG7^MW~iLgh%?DFVq%(2KvYG(&J;CbNPUs1 zC-pAaeTQ{zs3|#VOGR{FGhXS+$8c>nhgR^{#tt)Zdv{9Cj=vo zL^t|8(TIo-ISP!ZGZaTy411qGUtYhbwfF`63#dI&6bN^W1e9h)ja(MXqar%Y?NDT< zkStB>0++&o5C&D;9%PT4{6lG z-e>R$j3?uLid%X+g)>Tj&7O?r$U5gA$ERcZH`GwaED&9?85E@s2ZnUh zrGB4Pl51AGVNAZmi_3a7Oq)+HCij4;QD#SQiJr^s@`+lRU-%>`QqdnBjn_5aeG}jR z-c~Ke{WHZfAoN?@pKW<9SuFELR+0%4#qHVTm*_mpVKxC?Nip$S3}mgvPF%>GBWIGN zLb5j7D(;}uT(hwImO3YX)Fk~UTCe0XCU^X+s5+C+(|BJ0_x`Ypn_Lpww5xSF9EeX}s_CN7C{3k#9G`QF?;QQJ1K`!jS#2Zhd5W;;t z>&ZT=z;8UrRyBU2l9CW}>Vs$?=pj9DbsNJOOsTw|J0@72DyV);GEzI_($y3-6@;%e zr_5hgH3YUhjYf1D*L1TN>*p%xI%?GOHfv8ZZ-k4?Q<-~jtK8_Y2NkW}a{^fh7i!~5^mlxEiw8pV=Tb6+ekz@gBi zH!$ZDi3q#`ZFh@7=PVQb`7Gd=b$lF&9v1n2DYbn9Xy&wdVdqzEA$h-5z^?mQjt%bo z!pDC)!yCY$46MJoL4Wx9LASW(<0@jpmIlaDyxeF-_sU|Nv57(PJJD%4&PA!dUU}yB zo?BdeN`#vAai^SBIL`0a#6seKLoa=DXMP*~K;i?FSw1f1kMMsZ;JlA>GN5L;a&t9j zp;c;@9RIQ|I(_vAO&RKYSL2_qBd~`a3d%3UK&i-9D~~IdaKU|io-hzUgcOGW2`1E8 zhJ^X{$tQ4?Ma;0Om19yt!(94yh_g>w4ti84_tExK)}i`! zwEU3z^kojyrWNH!_wV8MeOiO66>4;7aKbV0!gv?6wpeBVOIoM*`|iS~B0Zm|Y-Xap z0{xkEPv0cvCl<;8Aj3AEnd)FiBD~_QpoE!78upu#hh3;=yTN{g-!)V$h*Dakk?ai z_X(KC2~~<2rFRo3GlMEnmgmcGfVR*~1|0pZbP@HHm{|%Rei^aex&7mnyM)Oh2PGnm9Jg1!oW~h|jem65^<&n` zq7nZ#fdmIG7j4##2OfI5p9!L!KEi2&%#nGWC1o{Zn#5tAbLV8FEg$8pslHGZazNd+Jon-h&s*MDVEFY#c~!8$LlMRxWba~uvGeUXKDAqgjLw9TrCe)K z|1$zvi5tg7HJuPTLPSb|>2=cSRBpRHLW@2JxA_2FA`ptxcK(Ld(>#rT5BHsAU)_e; zjescrgADMVaIH{UKzs)%Lk6laty|V%B~=20L%R#0?;75rZ*ku!heHZ9yF=-7Z7H5B zzn}+|n0sp1p%z-)ge-QPD>nG{DZnSAA`dc|ZMnv-@H`t&N*Tv-*?S z7y>B8N+++SVAZ<4u>E=O@wVXH=Xhrl=N2By$n6O&rEny){;ylLqF@&=0@iGJ;I}kc z&T~Ob6I6He6;MAZRBFWRt78`G&2VFDKE9EhCSzQPe!@t$5{>($FIu2U;5NOe5XD>5 z@~qIJTPh?UPqxe|A(=uf1*TR3FWv%Ix_g%0FJsa_8QvJVl=!ld(HhZbL1@LPB;hEb z&^LG~U$0MUj`UBAkXp;vE4m?%!gcXmis_^4cFjC9}+IdrX$`oT(u_)Xdn=+i#0;b+Y4^AXU;;2T?;z*mhwj=b2gjgt8a zxSjAP!su&YT1IDve6-B3wWy)spknJJI#nz((eKV3C1mRWjeB4yY=GBz%@PCPqzhf{lx<0L8$)b+@?&sF*IYfsQr%U2Ny97`6^=m@CTU8xA2qQ7HUjE+iFe=EFk+jo zob?0j$)ue{+{c|pfOB<8&_qHPYT|SU7}~Q_!Nw4D>u_D?lU*IbtO>$;Or^)&C?c~S z<<@5V!u&FJ0wt$6_sR?@xn>ps(`JDTAllSeG@DxYBK(VaQ6ka#D(I4qW5-(+&zGqW3n=N)QjEFt-Wz41%cB?ME=JYJJ>W|{HK-@7Ug zOM1LH$Lg-nm{w=hqSk{1{xZ^udgGv5o~h`Yfj7Qta)Y7AA;=nzv#>oPYT+Iy5|Ag( zs#p>65355$ucGOBWqNfxf#Z937thd`t5)v8riou-ipCV*$Z4)(cE#bx!P5K=>5X(( zuy{G9(=!W84g>5*s-ZD4ilJYO+7~ewzsQ4N>^7Fq{^JEO>>6bSsq?$fUfU!wof_f< z%!)%BW_KC}#ZiR+A}igY`hW%$7Xn}V^ZM5)0x})28Fbe3glBDc@{W3i4AHzU$5$Xqpde3_{=k(^@%S81m=u z6rN9A!+hVO{uZ5a_r6PS=B4-6)~6RV$11(Ea!;)rGV?7CF`~@ZHt5y~50Py3?z{e~ zo8Ktl`rQRxz6eU%_7ps{q+s7`)thPR{5v+??@LBn5jp-3*;{rd`KV~+TPz1*Z6Ek< zk@DU}3)-LwJv3b{8q(ZPH`LZ`Jm{V=8|>UQ6Prj-@8upuEEs!YN$i1=R)zXVyz`D} z-fMkuXp%GvT+E`KQ&l_q<$769?uCbthR;A4oTXtdijo*0#Wh&Vb{mW|pa+`u_aIAPK)Je>6)aJ$d-J znstF7VXiJ+5KJ}8Y*>^`I{deM>081TsGE3vXJ*ox%6T}1#xN{2_c#CDt4&1;+!d|g zl)b9fF%q06*L?(A=6$tl)@}f~%e?DZbX55`rB4>WD#>E605^Pl(jX3O;4vt5DH|$K zCu0-v+kly43pMZx##_FHgV~$hH13ADkK5o{EAwiJl-;nTVQdaR3ofYb8Kh$fY%F}O}BzJJ4TIVE{^e$B|43apDWn{W&;xC@#>LZCcow1 zoG2Y^GGSdi9Z<_>*l4f;>GicnUrArXoD2+sKqW%)T$ecerp_dL?e; zqF~9A;2r&lcHThXx4LfKmAEDM$R>*E4!X*ru^oww*WlLf+*)NyEf*P2KvJsH2-iIL z(N2vw6RGRm%koz!GL#+eNiore!2Kmsck)?DLN2vE6a@5N+U{d6={@*<#lt2kP2l?e z_;nbTYiXb`?Q*u*)zP6{li>T))*!{ksRpjwF$b@*{LJ@qKOcD((Sngh9ydxo9>F%RD;Xe z68v$nt&GhSK8Pd!`h(FY9NxHD>#6P?={kyY1s-^ijJ8XZssSZF*YO6tmMaJ}6J+X4 zx?JlCOy9W?y=LVp#_6%wx`0VAokcNyY#cmW6OOX^wc3o8joo;L1w$KrZ6#U91Su&= z+}h8w-6)0V{}%2tA)bd(+pB@TaFj1-XtN3#s`o1KE#+wh=jF~&RWXGa2c=A1AKb1d zZfKbL*#tS2J2Livl$~%Y9b&e4HSe9A=F&3r9R0OLCv_JSAzA1d4%|Oxx#|%E>kv#f zW!X58d8JG0U3{NLaDsFHZ(GD0CrF^QH{uBJB*POGpU4qn-*NLmL>4^x?<6#e9`^F= zOk0_t;TS*LTUIA8#E^G(B&spbRN_U&t3fp8si@m<8P(mGg3Y=Z6~CAwst-FU*z^el zo-w@hdx4luzlZS49VnjMoX?aTKYM}D06id6A>}fN)HpdzeEkR zM1=2NW)|r`@(-h(mC0!K5w3N4V!vn}Ud8;DJ02pcem?-PMrp!jO0zG$|GOsw@&0ep z=;0jw@wBp;E3$Jxjx_t*d$|{+?N)JLKIjLxkw^dv8jF3u`2z|{el_g(GMB%4BTn<~w-on0GQbvuzgy2p)t>^@-p^Z2*4dGz77 zuy)v^g{tBaBU6%t8-$DRB>6|Yq93m1jsou@DEeR&vB4ASVVe|g+0CelT-ssJx*hT+ zm7`meK*I`p;Ns8|ZF$(@K5ORN?#ZcCaoY9u6Y)FQyV2e`*3xNJWAkZVK)?||o#E#E8w&a~(EN$+w*yUU2 zO7w?IJ1fN(9Tvuv)l=IK;xkGhMd|;QBhtiVNKg_+q-o=heX*Hkz!Q6<@BTtbavPVt zz$}Pm3-{WfYHhZNgO7V-cJt-exB}JNm-Hy)Ut=Xyhr4*yrv@xgjs8d!Q(G2q5cxA2 zv}N+s=F9}4kcTgl9j4;-DXg0pj-)Mj)%wJfX{D{R65@V>Gt-CdRGnI>b<6*?U*l^Q z(Y@Inq~voJnLzN9jqmprVS-M8s$3+B3z;5%blK`k{1m}0sa z#U^)kiA_7p2yoAQ2>%9ny`tAJbQKm+E@(3n!I8;CG}u8qY>$Hz!k5%j#XgKuvc^Xq z%w=6_2?V1bMj>A1iF@-Lkj zvcMPVQ%KSgYWOghKx-uh4C5(Xau3AU!Af1isf`L4W==D1?He;}9DfsC9zXPbstKHf z>Tm-u^)2`x`Bq0JhX17g^!}g5%4zDz{rnDd^k}+&#@11Ih|*;O!Xq8>j=`XeO5Ri}U+NK% zkb;r1aqBRNtC=itlt^OXIab{&NQ2;r$G93W42m#{?9cHLO)ly3jdfEoM`TvU8mgZx z^d(=SZCy$7B5*wZg5;HUY8hoTcp5ZA;Wt{V+6X^d!yEvTnpw?`XW!uB`t94VTQzCI zA9_^P^GXNpf*BjGucXVoCO3`}Zv;I#Nas6 ziwD~~$kiFg(hD4TY+~XR#PB1bsv={O^wyB)GHn6(4H&ZwA+C%*5fbGCQbb{4l_$9! znRU7^Rt22A$K(6$%*q|5A9*is1+wjxE^3mYuTrb_!NDV3WK@=LR%z&=YVE0p*yScvRhw@%VQhu3f4x1%0q zkpJ@hsgYT2qF}3UNEuRQf6{?1-We%w3MaPQ-uIcB)^i89b&`G4ELAgNZ&k-Ku2kGf zrqR@Ex|!nDWaj0OD#6sNYgm(-JsG!S*WJy;3furQc!YzTLuIZNOv3Bp~hDU>=P&3kJ7i zPGg{I#lq~nIWleI&7?rj>zyk2WeKJ_+o|LOI3(>nWs4AqavTIL?al*`$)p+5oC1tw zd%xHU2(B%-$V2g{B_qBSF_!7V16|LhEkQucRiHSxGs*-wB&Z@=2&r@~Hn`xj! z&y^%4VWm4o%-(wui<(;O=~|t!;$;)iz$d%AIJGL|+q#AJ zbRCn9Bue&Jv+_N9%-@TvERB$Z{OKf?O{rC2nmkZT^-?SBxk8JDKnaI`W|0l}*I4oE z97Xl`-pnsCNR_{X!kAc7!^hGgf@xEA+;Fa7k*nh+8tonapWB$&Np#$^U|gY|OxXb8 z5kgXXqL8eX?4#?rlm(GM;U--2fp0BH_rF-~CUaC%m3$Q8T0AeAR9=3**%#(c=+%G` z(7wZo4kt=)<{tD$`sp2m)ue<4Mx4Z$V%o)k;hf-t>@mb|&SQte4UNSnb9TSP0ESw* zyFvgGrkkIq(H4Wsu2>FlVIA0}VUHhk_Rk#XGk;|t*MGdi#KUtjm#a_{G1#p*bdQVg zH^0B%xtsXw4fY`zxD~#7hW6p5`?E4qfvT{}-`t&anq^m)ph>bbn)0HbT_cvvxf7lO z(gzH|fj5^i0lIAd;CKpWT~R{jUD`R}da^R&jm(6%Q?((y{&zw)SrpFe&9}pPb7G&& z27^`j0=}bE&dN!B390u|8?fCF7D7}k)I0zKn&^;Ley>c|w@&{m&RpJ}9XI}mIcl8W z%fQ#auyqwYHN0&(JFAvK6YPuv${aw^69C<|jj<6>aJfOmZ)2FZ%E^1a@<$#yv$Fuk zIgbPU)Z=d0E|GGtINqPua^G)8VYOc7UZxv5wz-9egXu9vrWVPx9442`9dI)bzPv^K z@V@RNIlb^DdM47hP$NT$&&6Xe8lVW6+}Jh;jf||;zov^nKPe7`kvJAlSW*`Si^=PS_Sg3s zC_HhQhA?z)O48pA)9T(CfEKwu{~O;juhxRMnckx6hR{ThEDTbZ66BSfGqJoyqw#tn zlF%aiOPPTf>NlDJl6$HP*bA?4YE0+>?n-L|KUmd4R;q&OW%wt4LW4hfk_@w%SN{aqWD zYHr%Q!47vbp&#_UwFK!mV2>cgP4em2#Scg|H|!!_oXOBKpQ`?duM)Lby1Au@9KY@9 zh2;O8RoWZYh=v@bACwgG;$O&d5^9_|sx?aws#|8yX%@JLTQPCPXTO^;7tS~c$Q(0! znV2pSi+7eA@MhQi2b+571$rWj_K2%~cE88q=Z;P~yO4C7dCC8@CVcG13Kpn-150yy z&9K*0F0V9soE1q^;ffp0IxP`QF&MM|GEyPZZ7gje!NW@2$hc8$y`)`*%PfU zCz_??DJ+)VUa36t!g6kenIgZz%_;Zd^&Jm1Z^T%PCj)3LC14&%;**_`-#Pq#_-^n# z7v5b%W>TNSaD2V|x|-v&`~qxz_^x1#xo9a>*`JSEcQM%h;-*LQ^s)gf$957lE8krp zo^0uUcUe93^vk&epDD(<9^`&P*Bx*%KxY}0YMq<*BM>gIr}DrI^0yI7ZdvEnE=I+}+ zngR0_Do?8GBnY2?i%k-Z){x4xyIs63+(^Bs@=Fv?>LVeS1gi!WFrvI(u`qLIK5_%Y z5mu0IwXy|mg+)ZRH?s95#C7<S72z6?i!FBB&1t9rJ12qI+QNyW%(X9Xy^}~Q^$N;u3h97;6n3Ne|KsKqqXakWm%3$`$_lv_ zatzD9FiAb%+?Irp;Jexr0EQ&bO+PCA zVtBkeI_b99apD|17r{9f!RyPg2&sxGe}-k;v?bsuQ{^~7OM=O`Tu2TvV#5Ab_0k%O zO`g%bU7Yicj>^)MxM^|bUD@WJG2V_a6o@=-5_ej)8JCuasuB5zf+Y{V>g+ZJT|VohL*^D|_XbdK40YyfI=`v-9{gvn2d)s-7ogx=*(? zI~k2eO<|-=obI&+6$gkN%6yT|)mr@#q8omXsMgHNF@6N>2OjQ;Qe!EBj^(OSuzz?Q$Ep z*BJP`rh`-*m*9(}5S!cYaLp7&=;K)l*5RBji6VWVZ2)l{PH(@h)#CdUrKG^=;)2KV zhz3GI3`I$6qvwok^Kt z|BRI3qcnxi0^p9snj&^tnW{^BEl%8CnvoclOR`1~V-rYJh2AJVl+|-kGg9V zKN|vD&O-YdBkKewq9#)+#coX+QApI7IjJlZ8VRwt9zURO7BGqE6-OO(Hn3SeDGid) zJ8`3bQqV7rP=197R#Ahc2mtA+O8=uJh$c#1Q0?rdpEN#TK4YHKwiC zH%N{j$^(_2DM0DAzy0&D9VxBrh>yG1`#S4Kj{lzJq1BxKuEB+dyik-okZ;rQWj53j z&MWH26wSoU85Zrvp z7zuw%Aj^xoPgay6=9IlJ#x1D(jUsWJ4_Fz=;+L|`wi9q2q?k-C1+SI^v!%5O%1Mz+ zVBfHJ#ANlzmrKWhnT!;;!weLxx$H4>tp|iO+n9JQ{$N8K1Uy|7AvRMXUkM~HEZXo2 zjmjjuNroLi^SLk@ny!n*6gVmgM<(sI3!^I7S9X39-zPfXQ+z!cp6fWMi51G)rCpWg zlSt}X!Pthuy8R>s{}_Kp6f`kyP23&lvv06PIYRgB^rY% zi=T+wwdLx&_wE#n_rkj@O(1Fja6QMf?n%_B*g4#)Q-($hv(}iEFqsAI`FL|V=wtq*e8`EKLXrOPtykRzW0J`He~y z!Z$R6wG7?s^NO$4sU(_`Eyg9{mG3XdtMQ+7bh!9C*SuKxL^?^LW*xIK^dEGts#d$d zaEJ9q#XP&bIpsCsj0I1(lY~0{Ja1J7j4AcWtd!;|CtFJJ6U0I}K2v=&n)jwFa`+h_ z)9bi*{-PK8ML1FLj%6`f!cZfup0wDv$%=_?D76 z?{~y6MnBjAT)AH!cU+%*cqzZJS(i2YS)H)b)@V}?3-Z9NbcnC{vL;-I!{3qSkb!6l zH<^r$aegmswm~kCCU#_u(|=m=tQbTzp&%6PDfNPklc*A@RsyOd^}6)}{VFB()@?@F zXil`;W$_L@oXcUcNIujC{Z@JgUZFkp^`TBlPm!U5;@#UDsd?0HY^b zoAA|*nmngwGY&RgOq}8lC=v_rv)vy=Qf0pWz&YjH8Ho7R3sb6Hv5+?fTBuA$F%Mc1 zL_j^D&!L?6tD3czB!%4tpcVQnq%dOC3Kp)3^OYw?F&qJpj!ty;wUHjk7gi2|-P@GC z;4@sZ(2M$3`lnA{I!>N(y1k3grf|t}KP%FzB2X2*QUdT|W(k3!$;iPShZW zVi@}Iz37DoRexm!^Hq0e2RjeWolIkKhL%RqN!k9^An&e+7ZO*(T1jvu*$zci?^DUt zhvk?hjyv$&QYYZ>M7yqWWAD*{1c)XkWklx3rMQm`4hyUcoC!LAx)V*z(TiP$7@!b* z{4b@((DufAYgpmGVVi@A|7elSjQ?m68A{4lN8u3XJmeAc2N&fs>FJ=84#%4Ppv7sj zr}=kyfULQ5d4q+^Np53-u@<@QZCh!W*gH1Nnx!75*yLXbCbe^A!2%H~;n~wARmx{6 zpYEN!b~(jl(1a$#G7NaEB0gfH|LhAleS=129#}4oT--QOE{hJ4KAE6mEI{r18P)R~ zOTdP7Y12LHn3-o^)lW+yQEy=~i+$$No5mfchghn*)nZh@c*FFD3{O2Tz zjYLA$bFV1YsxlRFUAz81z!t1jgE}tpvT!0gAIxE*vZ~66PNq@IWb9|dsdo;tL1xfLEVg2UnFkqq8ovxNx(*Fmb_5bm0elBkbWh z(;O614*>7L9Yp5Y%v?uZjRBcDB$>7%*~zajS02G5c$pvAtVDTep%LA&_DOR2Tby7K zGB@t$9Y$!+`{{Z&izkcC$nI(&vcA%$p4;f@HGJ#b5*t4U&6RRHk1|qaY!mJ#6 z78=0}L=+mI0_>&nb4T{K?e^DV8&n#yGo!jB4=*8Wr_7v9+ksj1$*`q$vSHP58dCZW zCPUFXOym!Tzud~aGfcVW_ILX|Aeyz&LRw@sos+pG2iYG+g!2p9BqC7ogUso zU42hCz<|hiw8#!q=t@C))8DqHg|mi!oMi)1^B&yKkf~#x8lBTDETS(IUmYLUlt|>O zhd`qZd516uTHEw1v7h<(DNjt9j7-BeGEcD~3YgVwemV8Pqr3y|JGj zP1k>xXDz6P_L9S@-f_IDva|4vfT13@WU@`vO{qrtjxheOsaC0T7?C({sKoQ+rSyQC z9R~%dHl9_=)Nj8*8Z2qs%_GV7MHqohy`R>*5;?OWGuDVUK@XEo)newzN9USrNk2Xv z_U}5asylGI)!SMEj*)1*0qk?q`cx#tA0j#vYCVvBw`G{oe1j>&Uy`e0T-iT~!4;O; zXyihAD&LVTl#8X0?Whmyh0Rtqvmo(S+MPBh_jX+(PVl~Ef zHw}lWd%ZTstaNs>l4=R=7#sAHjKmGh9<{R(7Xw^yWneJwY^H4L$)V#+ z`Ik$BupDF63&GOm(p43C$flaKh$Q`5*>MmYq2GjK1!L21^(6(l%GV#r)*HS3-bXnM z$D#0jS5324=n{CYI92*1AJkNYxbm2LR~>``x6hdUCrl%at{U!9Yxa)!!{`!W1{SJ9 zR!pV+HM^Y3379dZY4ZKNi6ewb_tkX)N>k-qUHej;&u_$)b)&*yXP@8h@1r{Al5IXz z3nj1F8OVIWU@4z z7=q7v8LxU^HXMdte2T4n)3B);A`rwShWvL9lU}}TOl%_KZWf(hWdb7;gjsGwgekGc zLXI}6ikf|sTxk%8BGIFmN%6p+;gp*b&z!~zCL3B!#BH5Z8$#9!S1^!H6_Sl4cW4dM z+a1;;M97!+lKfGDFd2bU*bM9FMr`Zww_wWpOeT`Jo8|@XuzXh*FND~lswKKO&I@Ic zUNdB<41e*U;Blu^fS+X-XxgO%#4wsH|(dtH=xMmViqoW6d2|WQ-w82U|1= zn71LT8SW}h+lCMhnz`>|wK&rotBdDn$JA+_mLf$cA_p||(v^jnJXXksFpfB-VfF*@ zSYf%{Dz9IkjnV=(Rkz`1?9Ma)^w)E}+w|a)!;a)=A-zHK$`G8zD%Vve!-|^YdyK{k zI|d3+omVmp+Sw~0)Z;kIZ9C5akM(RBWD^wRwfmbbSzi=1qF&C)osIkN5Bm`U=krUc z+x@#HA?<^TVudW$%@BCt4iK#1h->Lb*eQ?_Ms~p2&uF9WadQzoX*vUpJm-oG*HR8{DBm^flY{jY_R zfc1YHC<3z8nOVPd-#)2e5U%W{ZPx+sG-Gm;XTk!6W_tEQL zN2qVKs~yPrYJrBa_9f3)a41EnPocxI)|^yUx{-MOSmF#`y(lAse7$Cwqv7_UDj;D( zzGCu*j*cSe9EZ}vyGcS3T2!5?rN3hV3UCM$=bwPFE4!mcubINuIcz32l@hbdCeDe~ z!K7&>fc@-b_<#&O`^)?_Kji4APIU)_m9En~>iXSYHgAM;ZMRQ3R6$fHQ-8O=ZCxtf zm?p7SD9QoB#s^XPLX3j=2uGfLOM>-aC+?5(^-G0M&!_6<8|2*#b)It|K<@Gi*BXJq z^S%v3VByNwRl}h&3SX%8TZnTxclAC_UuzVL<-g5$(ia_;(^h}0w1~tT+6XhqTj5yP zV01k(Q_9@Ab_eJVpSs-Y=ruS3(>g0V7OkrpO_T(i zU?CYwhAdd!eN$N>$jonfG)IjUz1@daP2QbZ+c4FR<&E+^vojDxKErR~4@_pq|w(yl>%G z+9R@atY@z@Du((AFF}yZ%2)t^0jZjCDhN6 z#5^PLxCM|eLm$xpH`7J&^7g9i_&-ABiY@SK)%(YdXhgfs)S;KzlFL8K>;Dp6wHu3m zNNvn33#8{-3`2fLzM&4)o=4V3xLh5w2GWfpA24mKO0S03(rcFtqsj%osMOM>UzkG` zHrFl_4Oj0f73kh#qSDszCNNn;^K=WoBr9#xL-^)%I zbZ$v*kACeLb+))*$c5RDd8c;gr?9HGY)HE8XMH=W`Z*lP&~LG$_W>t6nLksMnMkVA zs8l)+#+zh2ag=ZYBQN)@zBw}%5$!s}q_bvQCTHf*@_jF}u=#1}w15+R6J7DTH|A{$ z0F%%y!6Ys|dSVzTR-9Z=IBe|uK>!u}LbIVjvH1BJIg~c}PV@8Yx6cHocuIk2^JeX|{Kl(>qD zG5`6tU%b#fNA1jdQ+2ItH_S2O_xJ$K*Iry3x=ZSeXCqdzak_zu)E`5t4w_|Gq2*x; zXD{A@XFiq~4G1rZ8`b)|;c*#DR$#q}GhqoBJN3~CW)V-nZxkU$h+lxfj`lM@ZE5l( zzCGOBEIox@` zCo$3Qwl>!&JN~9v*1-37kA|Im4oDC% za1VZxX*oNlsuik%XQ0Vs>yjfs!g{4w4UO|HjK6vtR&je%AN>Stnm)CmR~b5D!Eu{K z)0qs$^RXs%^S>VMFVE~n$C5RhWImsQZ{?HAw1o$KKiK&d;?%h9Csci3+VRh03fZuP zZ?DGFCEIyvG29t^8D+dtK1WdDL2i1a0&#pXBNb)83#oQlQ}ff+3zYhJRAH1=uk;DLzgZ`g$~f)7hov1_k<2gs6Z3nU7lQJg z6cs#URJ7N1_34~J(2H(#o=_wfYvpEqBiP{1@jm#Q&}IcV@)*+$V2%B!l2QZ+5w*t! zV5V}sV2qL;&9Z3fhOw0EM<)1&P63g{L(KzmA#SH`+2~FlDkpp($nh7NIj5M4h&MLB zxNO;o_&6IFzq6w*V6LwQSYEa1brrsNuF?z&q#h7V)h45N^;;C}-9tBjn>CC0S!^~p z5`H=9J7|n6Bk`tJ>i9$Q+0ta{4(K3Wo>6HUlB!k%`Wh|QGlk+2Um)}5*;n~4T0`~D zw|+52nA;rRv|k_P`r)#j0SoFapA&JuAMC4SV{^x4JEvSkeojZ6C^H#1?YSym^D7=u zSaX)^-8$3SFd2`_HHZ1ZUT?O!OR>#bCsv(7odpLarTkNO)7ulPPUvEty|P?tg7G#T zJAo#v%n!q7b2PX2&esmT5uCe*AAbIAou$`s(RcH^cfz9e0hFp#>)7`=M1+YQW#pBf z397_o2%uQ8t#B$8(njG#_H`&5MO|*dsBXA_iDN~B!f7_?pg08Ok2nOBg!HaMSI>^* zaF65*PeKA{>_jXnFVAEJl99i}I$U&dV{O~Zj`ea!ixeeWC*LqZR}{2l%`DFM-W}a> z{?-2?|G2M8{9sNf>qo9lH|Fq&dRs!l4i4@<*!0`6wDvPHT&?%~f>>2fAZieaBN{|5 zBZIhi+r*KAdY3jWR%thD3xEWunJ^vmzRO5klt&ztK&{E*tYJR(3D=T&s&ARwZfR|> z!#_7wlca!hhtKBC>}^^YE_$U7_vVky7)j@=R!q5+Awd?X;l%Rx21HXhA6C4$%$F;J zN1#J7kPW`idKdU?Df&rqM3tAUlT29Su;mfEKRKLSm%xdk&D;w8Er-!Hei2m8qG_AuNT-_W{GZ z)GwRfl<0BzSJ#eC|B?$yTXrY$Y5H<8&RX9j!4}iQ)}0*?)6+SN(0gE&1so~V8g*Y! z0g^cD3nOp-OVqtrKJwZ?xneW42O$327}GO>Mfugy&e{1BBQFu4`XMFmqF~?2@X13U zLXH3OcsKZ|L71nT@Nc9tgnCc4Oz$Pu0?`^4%|M2kVJ{wyWtc(_RJ%4q3}|3uNTY+E zHB*}xJ%tAq>O@Ae7VwSv9e(^O2na`=xQ|%&kDjt>+xJqeU5XP4TPMd0-6PykFbsOP z-6ceeI$gOh@$|0MZyFw=?O+} zg`O}Ga$z%$i0BT1{ivgfJR*6hA{;_wX4$|P?w1xQmqog#a~8~GQ=qTh2iW@ThB9e5 zZwqmIm-<~ZWtvCS>R~^0dtbE5aK7iKScKy*!V#`25or6;-&CUrtFzMve@!Rpia%0mfL3UpCntIIy zX1o?r9DSKU&q^Oqo#u-hH||%Q(;7JXOPui|dW8_mtWlSy)j3NcWT=!L0ofki5?t1a5;54g&LtG588CvbUhKD@5ll1Ghr2CxzV>(?tJAo>dW&iT>lj6Kk?$ zKf0sXm%>%P0!gw)toO3gJA_1T3;`p?KU^K5)+JVP$Hi?$)mAL1VKrP1Ngo0w&`~2Y zKfk)JM88~7s1GaIK+v}R7N|1CAhj%)4(D)jx!hNK&28w(ehI z8CWrAhyKIaGy%=@W0R4K+k*=Cw`!*KontZ#=BG+xvz4g_NG&QjUKBGZh(ZTOMgF+@lTtkJX z?8bOtE=(1+>o%s3=TktAw85G3BFV14rvix$dB*R0$d^Le{{Kojy{a^GWPbZU;uS9x z8;Fz$*dg|Yl&H%<-UFT~{QD`PdzTbO*PqCe)`lGd?(T)ZYGWkSV4hMmZ+u&k55A$K zAl_bXP7*~xD=6GKR8p9glu`yzPu?{vkoi%}p<5X5ZK_kDMs8JL8nIaXX~TlS`meVp zf%~YD*tA6WIJ1{Q?o>}9D_%pu6|1q!^;V}6BV;rzhW1yYkN+WfFIQ@aEG9zxH||&p z@ZcnOJNfH@3YS&{dg@b-B~b53Oa^xq-(@~%tT0IyN2lT@^kM)#u`4BTmImUH{Y~sM zyq~0+HnM*cz8~`2#Rus;nh_XI>Y+fi5CwH6`I2$Sn9zTi>ed`O1IkzQARfB zl_WyI(Y>=j66YU*>kBNY8+M6R-{z{;vg5Q{RKwcZU`Mtsu3;%mCcng*?0zl_6=6)I znTky(Gx7!^H%@Lf{Vi)+ci4_hoLNDhP8QViF{N7Jd%XVWJIs}{h=3JtQQSpiQR3TY z2Mbg3MKoR*Mspp?j$U#NBecwKd$*-A*YF4yG%T^@>SYhzNa;^>mzErTj@cwE4wt7$ z@pQ-2?+wxWcU8>wm^pRdeHMT#rK~r~q8}V(B4t}tew&SW@*KM!kSENM^tH)IoZ;+n4hd7@@|zV95RLXMDxP%I`AiO1A;zKIFqi{it`b>1C7e|8!Thcz3+h-2oXzBgD;8>0Uox z^_Jx7p|E8mjkr7w5w+2Yr>PA)sx6&V5y9HKc0&L>>QeJB^#Pdr`P(N-Ay-Zs1R;1m z`gde&fDk#S!TlIOb{(pHN=*=FiKeN6RiVoGW`{F5kYX2vrH_}US?3qwl*nOQByl+4 zaI>`$@ZH>iVt^@TK6%5|hr?iv_05I{dL2Fz;a1uhd~eqML7Gdhi*d2rl6O1-K`xbJGsSL|>z(MAw&c5~iW_FK!m9yUJ@s*#L9| zB;hmZzXXh|AIEq9H;c>Y2s{!*>>(AdX#5(oX3 z4qLwLkM2fZc2>-i4tB9LO%8gLNVw5h1ejp{*j1re=>=#|PZf>7H!48&N>@*=1d&$o zpw6**j$xA+=)IhMnTwDwk)bd}BNgZg{nm{7=|Fl;keZ$mN=BB{`}bo?r9d@Ew^I7$ zZ1QlNoZu{38)AH_z-bpv@av-yPZn;}tv~PJ2FL~yiT4JZijstnmvN=pvu3T^Uyca9 zGjq;Eo^#fe(MVL&*SCx4cS~xPCb$!wmPGSJH;pZqgNk~p5nGQS&5i_s2#0N)D9y*} zd2O{$);Uw$)`l!R6PSM`Wx@fbNY0Hv(Nj?D&d!Wk&5cRu6%6}<2BOEqzqqzrSWkzu zp_Na_)$6SPT??G8^qw}Vm=jFxJ4ow$z4q3J6)CY7$VPBxMgAACXuq@OjZ3u98C;g+ z)O8uoP<@o<)-unyf0fX=vZ}G_weAztWy^@_qcG(CgslcWH3$SH)otk?RAQWm9zY!vRnf%^W5+?H^bw=BS zFbgno)d^{V9!Y7&+rCIK7-33-Ptp5Qjy!SV5!k6LZxJqU%cd8-ku%f`Jk%^} zMBc`vR&Mv-Pk9ehs-JIteS1D(6Nrp{pw%Ulpc&I;V|L#JXt!(CP#0>EIbm^Jq7pr7 zHHDd=-xB-O0pLIz{~S<%wT1AhrnV|?_T9XhrMkd=>QV&PN!^aCZ5H!CDk0>guhqAk zh>1>5HL*Hd7ErG_0-;`nXR25NjL$e_zK`@;=APz#y-lp)`w`v&Yt9e0a&;-P66_&D z-&8F!iKc_MCOd;8iLRCYeDnOVoCkQsY?O9VUC+U@t@-bm}%qD)~bwf*XaV8p#BQX0BWm0(((!kYx zo}*^@MsE|v92rEZVV3}|Qw>HdGPYhk8d%nQA_L>~a`zTK{z3`4w&l`{OQge4jT`nI z`^VvmH0@~9Cg}U~SG6*aKH4G<#UA!D@UStJkIV^n`}TaVmFf!}Gy0eq2W%d>9fGX$_1@o86jtgq9Lxv><#^XE^>16EArhCm zR*i|PJTa<;$ z;u96xV<3ccN&1)ad@lJ_O##-cHP3XHe}xGV*UfJ+ff_R znypD1+*Y3!ZRg67MOtpB{Oc1O4|RW#I=*Upagp{M5{j# zQdcWhXNthn3XtIW)7!Ebc;&q}&bV>N^V*N*5@eW3t>NgYCg$z-D+}Lzxc)UKmhw-C zJc8!m$+6~cTIan|nt`TA$au(IykeW<146$}RjUpZDB7W$;qulQ$i9GAAk53~g=z~O zPgbuz+@70&J~$1N-$06#^}At4PlbeB`_QoBv-^$q@!%>@JWl=G{NK|^*z4i&*JsA& zzGtr;{}L4s2|I0~yzouCaSgZ6b;yVg|N16Fw%EBYJA$k5pGm&Xr@0_68!=AFHPDg-I@3{)~7y0Tc(}mt0KgGY@Q&=~-pby)a8q`h9Vh4G{CDV^BeAYv0=9CZJ0=sB2$Fcl;#J$uYS)2- zw8Sv7O-I-v*)LT_y;RUH5mzwLG;E^gv08Oq?^nh21Xp_;{uMItfgD}e`5o(T#JoKA z`cr(Z7Tb^f#>!W@ggfN33|j2Ur#2*hf@MdoSq=%;mU})bP3aGd5ZzX~C8Wp9wGgg_ z%WeP&V|6)%PBho67!%|xjCK84H6vF_`yDRX%pftIl2m9ygaga79e%0ym5WlV;vH|oa+)B1LQeUp8vdGZf3+n_KE zZbH}x`TA*Bw5oEiKf9;p@d2*VN$~xJIH-btFzztOgX2XdB?v`0ZVAJM62Ef8XlS4~ z-re4kQhE~Fuq=aE)imz#j5h^(T2PK*czpeqkqN_K4ygEDzr*+WyY%1vAFvJ1^A?~{ z-Ct@b1Tm}4e(1tzR5xnMsTdkhQq_oXp45rbh6}nWEl=$C?qI^GzaH!dtExABH0vTI z>NvMgKH!<-YW(nU|9f6=C|0m@89rIdv`t?YtW{ke_GKJeEd(xcr5FVyl!&jC)UgY8(X9)Ye)Sw#rie) zUN|_Jh}UXPCCGoU{m;RW!`-*wt3aRge>e9ZiK{UB{`~$VrPkebV|9|gmKtiVwEc}& zZ2uERd(`&|EUFML=Bum3?nd0t3%v$&dgb3T{3kZADmv1Di#q7MMF5a zmEpWE(4B!ONC^bW8g+t7ULM}HBSRT!2&H(lFi3VV0&+3GE`Zxgc#3$@CJ-%zI-5?k$} zTNPo7_W<0IUf2)1pGRO@vQ3=ulcC$n0Bi4t{u$ug(l=JF`I-tVg!(;e|RxV$$+}cjZYpK{h&lhU?-S zd^a|MGp_VZfGnFtc(DnxOJN+lkW-AD$d^&2Q9gC!{>PEcJ88YO&+Naql_NgF`drI7 z1W8%G`xG(#N=PD2*lMDS-tDh6&gdaIFTXPV?hu8eR}dxVne}A;sU%g}OCjhEF*(YE zsMri|DQKCH+aJo)8&4_%inY|8<_mn4*t+!Ur&|wpbV{pOa6>q@ForV8j>4lrD^dTeBSRmH`&8V`*55JPf z`m4JBSMfhah-S~{NHG0m{ohpmgh|aM6!O$u&G5f_hhl@6%rJK(0cp8uMeC%Mb`=zL z(foQ4*|f4jl2yoW1>Qp}Bs7!-GywY@3mhW@do$IFEpJ0tIrbfk=9|Wlx*rKZ@gPm< zB_18`Y{ptmzaiN&@Kfk%TceAtp;yOvmgpdrc-Rdu-G>4dK# zrJ;pQ^+_py)bv}SfpSoH07mHmH!C~X~yVnYNurZ9_z=JcXM#fv)m?c$_8mm}?@M?MUTxBtN zOGs(ayAw&PFjW4)DtW)~rPm`NMv=4uP^v|OHe}QV%ZM63i%3-qowCCCm2({Yl{w8s zn^OFJUigk+oJ79+Y%01_2#^DBSb5+R&PQxsf@ia(}L zPW)4{$h8XUGMD>1UK+6y23*-~Cp}#qc6~W5u}(~{rHW4zfoXD|*wLcxSU%!Iy;tsQ zTIN?P@#{kaZ|GCH@q9*DUVhcs=cGrIBi8!ztNGR6mU(jHU1J+zQOl$ zG;2ipvjy3bomjB^BE-k6)2AtSPz9nVA8!%j4V%7UecN_8q#tV^$<4*}E0I1h0$myo zd1Pefk&myh(PGBnz;&TbRCL{%Y;MaZ<8 z27Iol_0@ zenjl*4PH&9 zk<4;p=~HSnFumC@;U&44u_TL3%=hk%<1MV`|N0O|!HYnNe1@xpKz^P%1ZfsP zlryP{>7ffbRo+J4n#`)5o4*MbwK|(}+Z5FXV!*V6-_skh+c`wHpA#>tVt|=1fI2Fx z&_{N?({xOP9U8uHWckg=H3@0k*}JT*%J(ju1(JXI2^v|t+W~~pSzK{ z$@l9ZUN~OgWQEDAEptBoHKKt+O_AbMP?=txkYY#S(cIPVxe52KygP$Wh_nEyxw|yj z_LW>Uj>CB;;mIerKcb%|Zij_=xX-!&#SFGfEQt)DCm~)6Y>=Ylot0$bgcdCuWW)st zeq1i7>Im#zWIGG@A#^ch5WYt5VPF{QaGehs~OJL)q?c)!Q$SVa%*)(h85V`_^mMn`o!i zi%Hgka-}BJMF;+uzdV>ZL9|UpoXKRK>h?O0x^mrcPQwaIP2H80O8KI~6GzM)?0%4< z*uI#aBGN!3x}i~IM>?!gG{npj^6NO%07kT#;=GTul~^27DP!o8ZO`_MelPBYOJuxP z@%z-wFON_{_R&i(w=fw6M!uP#xFixs%i2b zD_^4FB5l#n4uJlhZZUed>OW`3h?*C06W2%KpU&^|KLZZCbbf;F z-&G6`u3jw-?VS^9jo3coJ6iuqV@8WjY5WiGVnKs^_nPq8k)87*)mSk*Ry*+RJ5LRt zR#HB#oi5hYO5}#3Z-(XCQVAguWjl|StTUYQoKqyVUy=5oTD~rt7m8-$5b1&l&`3u6 z1P9`GlA2VxI(cS+#yK=E^2?)iE}GkyeVJpk>c;75PN+eM>X7;G8g$g%v3UPpylNwu z#mwU7XE(*8BtFxC3N334_-i;4bcH8B4Cqvcp$*Ws+sVup|BaEaTS|6a*IH)Y#SSVt zichP6ErieV5mpTGhV`N^LJf)+6Gi)Z;7A{)+POR)b+$?XXuiXV*Azje*PkNsO;g}| z7r>jeX~CCWroy;MyXD{`!nn!E>{PV?%RLEEn5*B)WL2o&a<=?za~cI(pN>!ko1fio zhE{l$F=7N2WApQ4xrt=yJLJfsIf8s8$`dSF2Jp%871RNIe_rC$RlP3Tv80*tb<0}1 zh8*fQ{%HVhz3u&=4G>k_&V2Avci$eDc5;eU7?OlBNJ+k zxl#e66UPPDdTy;|yvag8$0iC7kmeq5cxW`>pFhrPWCQdXvcZMH*tcgjcV6ZiDVD>T zy!jGpu-et>{J`qNB}+5jAHGm@;&Sf|e3{tM`cvBEHR9#~KIn^JEo zjL541sQ;CW5Ic^s{wB&G2&t!jFu4EL&$ttp1>W0^AGL0~H6sJ3O;{(yogb3T!v6C9 zdw;R?_qL{cuScT(hJ1hl&VM2M8diLsv+#)}Aw3 zHmvxuVeqt1`NL27H_3r8=m2v2806QpuYF9`pfjRHXlTeL_PmKZ>5@16+RVn~1YA%!@8-`>>)w-Ay98^|uc!&~8J4zm`$LN}YhHg^#zc&Gc z*!uj0)k}hroFB_`c3Bk}(JOf84bA%uR(=sGo;-0jVI4S}?7c9>jgetJ{Z^|v7WdNi zwymZ3JBA7q$u6RRZTM)Xt|}&nOq-GErJBZbP|1i&$?k}D$?i3aRB7(j9bKigAB{D!hW29(db{>&=+9uhEkp(($~me~Up zqV0{h_<%X#E+Ln?=9Sc#0iYf;YKy|WXyUAHk-bnn6 zUE9%4uEe6c`G7Mch+PQlN(9ptUo{<29~3YlMUlBOR+qx#zlQ50nPl4QKs#^nYg=zy zZwksk+|M=@wqyv^64gQr&5C)OCzH1{>{KM=E>sFOZn?Rg%_0~!2N%Win&P_uGz3z( zQTTM>iqkfSW9$+Jl5&ikE!OXru*8wyVAL8|)B!1;wc=XT@r+F%+7Sm*>y;C^nQNbO zb3a%6T|BtAl)>xk>p|UfIhTls!1Mb)e>0i-!e`?7f?6W_P083VZDHlQQpPB#lEa`v zr!#rZt+a3Dy1J51ls`NT1WS!M#wunjr;9aOqHgnEmQ-PwnsC~j$)zU9HHw8m=jJX} z$>b4;obRDH1*=k!@h+v+OJVwvZ`zIiMD5-ofl6#grF`$7jt=?unhGk2imrKSB)WoXY2di$}Q3$Y{QVK8OI8seM`UEa#}@nj?f_;nZ#}Tl-}vlzF#x zmF^i+$!cm2yIt+sAN-n4MaA|&4LLF+ytUd!Xq8rS7M~b_E508*QgA(at=dL!_V~RX%yPj zb~q9AQFLloZY~5ebA;t+NhDS=i6U6MBa1L2JI7cesN*Yun@}ylR6X=rjBOVmXPV2^ z@0G-|jPSgHa$|67)ZA>9&TJLmY!$gELj<3MBJ5=Z3F#(0mpvIi`i=6B0%(-rSqt^- zp&SmME1sV!zw?YU>YzT0(^f3?aRh0$Z0c{Zk8^+i%3Z1o#eKl(1+(^&9eTSj0l6p?71Pe17JiikqX1FfZ^tr-_ZtJ6rUviXTo`+mngwy$5xRY z^(3KWk#GGb{~Mi!jzYo@IDIe>M;F8jES&yyBd5?BDl|_?zJ&@^3_iui19d7KATA_)%nx~`EDkf{ z>`JF5zcMAd#5osNEw(LT4|4^A+^Q0i=aLi3ax25&;9XVD*l!4uUq};qByzHOe zH3IJMc+F;&#AFwku6LR-K)MzjZrWH7*VKyy7i!rE+23y60`YmmEt=N+_PD#c%#w7j z_(O)5OaGHE_7;AczpP#lR=+!DI@ww4^lW#Rvv2_jq$s)%*mS+Iw*aHj8u7`&K3A3N zJG8;=i634drr*U8pwOWlf09K5G5nQgu_FFEex+yACUwls!wkcuyv$Kb|L~1W;>JmM z@(Q_A4<~doTSTJhv|n4r1NRpXoNKbPL;C!wbRcg@{f$#`fY2Hu-pIgh=LA9DftR zIR~cLgz8qT1n}#BLCQhFHEa%%WojRN`)~a+6LrtD<1W$0dQHDbjz6CO*E?;vV)=Zi zhmOE_K~dzMiPdYs<7#Y-0u`5R-B8a=Spr@f9J;Q+T;=iP(K|jb8rt_vwR~ zyP1@ZnY<63Q#&-CzhwfQ4%+!^oIX0fm(-3GbuzQ%pin9aRW1?A!NdO`X8x~fT_hlR zeY7Uc{m8W%B~7uK`dKj%yH2S;Y}T?Fjfto{^@5gUG>zqxSgu;GMsjonz# zPipO)+|rd5n*hs={D<=o{P2c)yTxC5jX;p(Dc(Om!b!Y)d2&|Qg_xu2JNX^SfBsH` z$`E64!vTxo@Ha^CtXlnYbQ7 zq2}}NxopP)xowGN^uceQG_DVi4Gf??Nj~-tSl}b%-x5};%)<)za(N3Fv8Xi0rel|~ ztl6>o#?I@<1F9FRorxJZi0Wq-g1(Hh8imG~8lM7QcQVmQSpxpDANP1yE>|Lh zw;Fnq7&faW(lawOrgQ@2ahapVuzR2aKz#PS`AQgzc}!w|_&ud(Us=S4;K-28M@LBq z=IK&Fbt9zQHCsNMEOnM1GYwofA`uQyvichBiMI(;kLEmJa>~nfrIW>dso6}sE4-WG z3V^}^_@o-Ti=Y+>CBsD)2sdvJdI&kR3U&YB0hcvmr5odFU5Ck-0xNHk&G<)7)Yq*k z3K&@7960@@*i8kHZQz^MZa|@=_v6CD`-87l9b3f??Rmq!|7=#9d6x6CZAq!5df!xQ ziWx?C+u?TL$)_N@3BFFhJ!R`pPm;GUy(DN_>`PC(uX2K4%){^98Q>X@UlWEv#vLOm z6>eAs2*Ty9x{#fb7-8s$bjqQinbryPGnE{hFh0!94qiC}jK> zgw-%a(Eo3(`~CF+@Bkdw_2;;)P|}~n@#E}<)I|wa1S{T`hPvyfvAVeTZjYXwGLor{ zFze!Lk4w?oN-GV7Ty665*r9YFAsfKiVb)CMFcsz4eoMp@*Y*lUhemJkZDa(yXvb?y zCb?&2`rl?@FGP|misCTn$SL5kj69cGdxB_is!d*&>qm(i)WUj#X`T~gVtXZdJi57F zIN?R0!OJYQ7feCBH_D(tettVtBDNB8cbOz2o!!{w>Ajm@ zgSKhUCwWPES8pm({={2iwp`RN5!Gt!szKNw4R>5*(!~5`-$q;d$dx@xHq~b6TXjA+ zKBT^-_M_{Q&koP=f`6Mw<=kboWpQgpgu;4qi#?en*8~g4tItBq_R0`K#$|=SM(x|* zD^jI;hvQVA26V=%FSTSsPKUibuO#8Iu(#7zX`H!j^LiGq^oxR#MWe=lAX)12K< zV0^kjWS@ZjoN7WNct_ou_sJ$FjA@)cgR=hFf4%>)FY8?>3y=9MA+8^gBj$@ff=$+3 zo!WuG&c?J6CH(dbC+zu{V?^CXl#0C2)ElcmRAAzt2+tuP&Z-5!eG?B>X*OIW#nDx$ z(#Tv;*9_hnlr*Iq@r|-J!1Lt-GaE85w=m%uVNtFQpewL(Kb^R$v+Nk_v@&R#to+58 zN>rWiTr=lrcI0QEm(v0&{ zGGqe;D5FcB-E&O@4?%;yI zy7a$a_27n(XjkWdvxp#KyO^zrXB`V|oe}E6ZIm>_h~ojPd1wr1wiw9o*k=#$Z+ybd z$53chaaEcBg8LorF|XQM;NWGELmY;i&tyZPUj}h~*kyFvb*{n}DGSub+anMy3y5MJ z&AEDVW;eDAj%qKQpU2h%$hMSeDrtn>mk0dZPiz`-Ta@eowfxGrJo_L3(~OYNh@?pC zzf5Ep(lxu|2CeK8KJl2enH)2}j$~QwZ2uDv6PXYH5EZDXF5Y?oe4TJH`DS!)*O$_a z+kb~gD$1u*IW}om`y(ydBp?Yk6n2)#hD$VrkCg=1v{7<6R*@JKDQN8rgVCN1W%hSb z+srCWy;UEgoOhnwY0{AYI5kN*uNQ#T#p>;Ir(s~7@m8m1;*&mQ)P28ni+IBj3@ZLCR6K^QJNTh9uLmNocyJ zoV5xQIi{;_!WyX=5$~rnKEGa8$*-l5)!fG6Hf%`1s0C6KFGTVDl*y;?_l%ud`r+Rk zo6}0UuQFI+pHAth4P6F$m4D@DeBS96m!#}j>irU`P?eb(ZL(ICcIiHRcO3x12C!HR zgY4_dg`Vl2=a9qq!i6KLL_7sp!21k2iy(xt!kc%UqlAJ&I1NV?>fZD6EeF(r{mhPq zzvxH4z9)8DyrK9064b=$DUFPd3&G|x4=;X*>aIz+03&$tl7hBjwG%O~d|E47@80kH=+d(<8Z zQx6>Qr*f}h5PmjKYDnXL(hFjE!Tlg0ypYBVimhw8AvE)>LKe5W{F@%V?ALoXPEvk` ze)=(Guyzw^IqtfrVG!HnXjeF$5~1@OyY-m&3N!($zhT(yLp6N&n$qp2Hj#Y%ECK7O zrSUvgx=*-0(2>h<^7ryREbTHK{knlGd7csUx@6kARRc^$6u_#L7x8f;MQ+94EMFB? z@sd#H%e88M3NLC*_GrkQf&LCkpF~HSI_2Q=wtRg>2=Ff6k5_xQOm_qzMetX5lx$JN z^9sR11Y%_}$FubBCB`|+aBL0B_cms{m@sWSGMuz%xK55-{URV5QBcpcuXz+Bj@K!w z>*_Q1^5p@O=j$n>@LO7fcJ>6R4s;$0$`~HD%0joFhJI%&n~~oK#XTnsCbTW~ zgU;U?jo*Gb09uL3Pjs)jd*lL-ejmP;Gv(RtGc{ygmS>@7uJSAMgb-w*8sV|WI$Oc~7cdng4P0UOc{}2< z_;bCjCQ`=wOiK50hR_*h4T|qnq2c_c?8#0@k=zJ*U zxh}3?bSniquYI{peJn{y|9Ry*;1CFE8!BCyZd*JR>b}%k`7SeldXX<4SYYY;nhr6r zu(+IzRGgg zQZbyg+QUW3VX4^cdMe%H=7pp6(j_wVcq!Vcy)*02sLAqY#*Qlz#nPYd4ySzHM5oOY z*ZiH=!}C`xt(3J7Q9?>2Z}+8_e*MVucHxkc;V%`_TGx%R#lU#!{@#do7t*rKoV)HK z_WiYM+MsV!V3;OIvHe3TT;NYzC;Kf`zM&hR1%l=GSWIu}IbzDi&}I83D$+22{-p={ zT9DZdGwPxaLYwt}Q7iF`sdxXwtk9@3*xQ#TkyJ#+mI40>SY(k9^GfPJ4fMztcna(r zq4^%4cuX1GtVddSPAP~Yl0x$ALc~Yk22w~Neh>IuA&j+YpQOQX4xMojOXpFH3(JDd zy%T%=^2#QdwOhl?I+(Tk``_gjs~%wGFXQKMMS*fOzu%omOc^IV;4remTLhq&F)(`u zD&l&YKzswATLehbcC&8^%}kTxo&Crd0{N7@dWY=G_LRkOdv+<97pI|kxP`1juXJIK z5mX57*R}_J=n@Dgo9krRg!P}fhPqIYSGkDX`jxT58{~UND5^Z#q#doo=onW6|U zpl*u_1q+Je95sPcz8m``e3izQFdqZV3c)A_G)33WA488ej5ymjc; z%+pL;G?TCV%~(t(%2G_=4qQUMV*+))N`#9eci4reQ*N$)fS)I7+Q@}OTWV=NB@JL5 zujN^?jNWNmDl2c1j+s?fn#S(#t5@CoAXfGLY8GD}M=Vre+0_ex$jZi~O_z^tB<9oP z{j;y%)%tiKUz6bBJ&(#(FfdO4=Va2CVFA0Zeh%waqttKL6xiWh>|DI{Q?Cgbi%Zdb z(pzj)0{JcEjdahPO@(Urb4J)kq?s4o>*iZC8^nZ6IS%k-H4HeHq(C!{WwEp4?c=IU z9?YF{tG|CM4==7{VM>OQ5H-ebok&Y_ah<#vhRwSr)En%xB035Ezt$&7e-KTE9!zbc zA>=VWsH6s7>K}Z$M|Fd8DozLc-n~WVvHglm6Rms{SI+ebOM>gj#nUl!TGLZr0!mB4 zkY1)w49`^NbYzMRsNxuShHuZK9O3=n~rh9$UTSOrlNfxj?KfxK!HFxX8mz54QTVIkMw!?Roswbmx=Gtmh%8I4suel4FnTOW;hlxy}~+Bm2hbb7Mmhq)MgZ(_4My@bnhpf1LN*&2GE->!+Zy zJgLej1O9IkR`uOKdF4r$qE`fRts%jqBsvqP;7F2~WKT-b8BsIdnn_CZ0&(Hk4;rGR zdR)sWT(t6_uI%$axA`Z=X_>FR3hOE(bS2e)0V&-+%o8t}UGM&8pftW-BJuEm9A#b*)nLu| zvl{ZMR%Q$udh53t7U?+P%$Tecc&5IZG*7v{R+DZOZAB>t6p*WENV%mzC5uO?3}yi~ z^!p!kLB<;gexRQla(7m96fL=879PHkuM3Q0#_U)DGHwB}MfG$$aJK=k>QX<-sUA|@ zDBY^y0^A(A5lY=Njj}`sn^W+R;y@KQFFSoUb34Pyl=);%OFKp87iP{aKaSA-Vb|u8 zt0SlE>UN8E56t5pk6icyguN6o)$J#XU4DDRS5!aNx9M)l9;s>z`X-!_DL%WW-@Hs@ zzHDR2PChPpH4OqluG-7v=VY)aei-0)krTaNaWwJe> z@;82r1$MnSF-{Zo^Wv6A(k_mK(Cc1o{ZaShuT9vYBFfm=Mjazj-P8LTU$@dbvttiO zTUgtz%}^C@emUvP%GYMT)^wGm;TL=Xv5~7Z27e~9)YPIw9fpa} zgyP`w3n5wrj+?b7W!I{Nny1S?k6_e76#c19-X8YJ@+^0?ZdlU}y~F&uxy!&3e{x89y8RAj|OOIR)ew^|uS4WWqEZ?u9{bnmYPJ49rKO>es1Qu^ zhapzgU{{0Ml8tAHcmXBG)-+q)ob@9+j2zBRR-KqQHh!28P+B#goaqx&ck|ZRquud> z!HQ#xVpipa*8CO@7>EtNM6R*t+Rxq)&8~~;;Ux?ct;_(0u;E4D^c48c&{}9~=9?>9 z`HvTvoQm}89DQVg;2$c3DG7tCDHZhhJhv$ZpPshVY=|?MxP8k?vyCt|VdJKP=+&Y) zV7`w}{U5G$h#0sBYv!2T+75=nBdkR#AjKUXPU9V`k0bN1cjEqCU^5#R)SIO8j(j{2 zQO#D|O^iWK*Ci47HLl)!U2r7z5e(uQ@Ma5YA-?^C4#$kT*;?Dh+75z?YGB+h&%itQ z%f(uzQ+r9KxB**lm)VqIB(6e&>p8u#14)fU6u)F=s;k80G4DCSBjEu$9+u+(4Q2d% zZLURy-b6Z|JuOv)}#46?-0ygwdNZ?+ACc!)Ia zYhShcLFuDHt2Y?JJUV&zd#prvCsV3~!?Gj(4hsD?+&?fA$up)@W&zy=z~vgcW< z*UPrBGmx3(*QPGeX&D&C&Two)o_~0pHTYE5Rsz4c{OpKgrT3S5Fc@Kc`C>FB=WeN~ zcQ2J@Ug3)ws#?FjZkH%vmA$w&>I8m_n+&abFAsS7q8j9&DGF#U@l*mGWW6AJ%f1Qx zW}RGKfrA<}gOZaRT_6mr$>=8lHWf%%Jv$G@;K3vhq%9GW?1--zDoBLosms93+e6I4 zsO|L0@0bjdlK9Oo2lzNxrY5b3RciESB@P=5IDi0hdnSWvq6xFvmrM{6Sz^1)dtFE^ zh14&W(y!YpM1p3fC^eNVn+Lxgt6bQRsz_ftezekT!M!Tyz^b4mZ4(zV6)kvIt;|^X z1ccBjn_MgHD*6;G&|<|2GJ0K-?#gwhO@mvU{d)9hU!jj6gM84H*3@}|6s<(|A84|a zP8VY&$oaIS@C~#n|h2$4RmEf67>I|NE!4Om&PeZ zP>BLiP{%m7Be;KLm-mUJ>)=jRf5+6~)#L*5k-wXm8WXilNNUAVfUHWFQ^dWiY^TrC z<7&y{BKhL&+y^@saNU_`NL&=HG1+6DI+x6Vr|8irmPta|iuwLB82{cLVz2+zrPxp0 zc%sxv{2L)MSX4yzh@xtN-N@C6rOX<5kIaP%XV66k<8U($CNJoe8|6Fg;OJpM;h#dS zf&*#8Pk#LjNVmWZPA9L9RRQA)h)0sDWVMq0{aM3`oQ4Z92#vK5&P^hu>6wc>825ZV}kvN9(Y3Q|RPWx0%r; z34()%T299&fSphGZEs|{bGoM4fjjHFnjT=jo%Ti-!?uGo$I<>Ht4r9#pDoJl7Z+pF z++Ex8a#E6ey+C6l1McVNVeI^Qe|;XgM<~NlAL8IzGbfHM^-%R@al07Pw_M8{`Axs} zQ%dQ?X7zRHii|z+3x2a40M!qG-a3m7R#@fB?N-qiQsJ}eqlq|++BsAD3Um;>(Hz}tYOx7dXlC5eh8;p`b=bL+MPeMguh6vESmGP63P`s zD#;aOajrh+~ylJ92izTM1c!ieIZb@nm>p=gO}x3S3C_@ciP^^?Lgei zl>j*TnSS{lvTN;l)v$WKXK_A&U6KEjl^ZMHaWH+*RzotZ>CxrJ$d4{g-0|ICukLWbp`euY#ZKe`RsJ zpVbYhecD+umNUgbRgBNotNB28hZbtVxQ{RfQsO=g0j+Y#wD0$84^F@Wn8iu(zyB4( zn9lxl`_E1-hyDk1Qlnb2yaMF)uJcmr%bIO+bZE_c%PiDeD0bu*J7uc z9?pqV+8|4eH0cCMHxKv)%A}F*k1}y-&81W>F$IKT1H*``4#cy6EBI=s%Hj&A0gL9ekr`;Jmzvvtn^yw!=p2Sq&Ang8TqPLN$9ME^0Nm&} zA&)F<5%C{d?e;T|FUfN`sH8A`@w}iZtg1VxwX^qtx}!f7o#to`^GJt z`^TH*q0wJit5?^eO4>vSx5l>_GgYVKjkmNDEm(dd5Vw)Z*$&>vjW-id^2rTPnCo*1 zr%Dz*{rJ_XfB1>MS+Yf-Y|xgo_(M8W#8|}%I(WjcfqsO0U-BQbI)NkM6*tu{-H6Zm zG3zTq_4I*sWFJZ7S%ZNXk`L_FtI@q=4a^=i9I@t6>cwBmF`KK;`%{v`|hRdS6x~M^^$V2RIN0#{e4a2ag`2z z_1Yfmf#&`fm*Id2g7A%F=vD3)+}vpPvj0Hbu)FbXa!_MNV_q5jGcC%%#rJl%zY1BchLP9Gr*cbvsY z6#vd_%r|4ry`cnc5Esfz4`KhkP^;X4D4T;F&(B263*T)^P1!I`Cg<)o`HS2kDM6aK zNdK|%Tes)yVUg4fP7ZF8+!s={Uv#?~i?4=zg6k`<-MH1_^e{+fQ%1NmqKvA%j{s_~ zKf2KH^#^Gu%C1e$wtfD?e~VO{#Q!F3aB;3Wc_o+AG3*`L@ujiDhQtSXT+W&=O2uc5!M|Q_k5HT0bX!FzG7FFjasr}fgj{@au*K3A%1gG?a8evaLg&izX zHSulH(-N(sZ}wI}g!*fIRRP&j&4V3eIfT=b6%j5JI+!-Z7%$$wA0nUJmfeK$MM>&B z6o>C%b1U_)(I67Jk&E!*K0WMfPtcna9^c0MTezFxtuWu!F4|aUg%W5EXc*>EEEpz1 z)hXCRP%HmMr3ml=s{t+_&aIu~Q~cPb)NU`N+DvV)xwA9mCVq={t1<$vW7vUnj8(c? z)5Ejr4GsKu(?HLF=)9GFRwzpPo_Zf>?UD*rZ?$Q#C-}Q6-xPT%x-zYg;OMhh*tIDQ z%`|$4*hLTvswkL)zLL?Ktr&0sA_5=Ms1}xPyJ>G7th#bDMALscNnLkHY>eNRl`G(^ zb5&S;J?R9PsBd@%+!ikkt{z`KT|kt8zD~RO-?`i9s3=KY#MYq4Z6#XBW}aYG#L1nN z1cxiW54}%RJ{Undj4K)${tDafM@`X-j{Nf(`xV(?+*k_cnuB3&19+d^YsykpsQw)7 zmLsiH#)7DB#Yu(r^(4vjYWG$0$6wsioEmOxI=;V7PI1l;dH*=;o1uD+v#e0#L)@w63Bvs1ikPfR@$y@)f*G#K*W@p}sAuzwS_ygk&u2${0 zNPbY&=uhTdd#X|9j(T`6bgaoeuAOLwuZ(M$kr8rp!d3d31kPkI{EJ<>Q5{aaVrLXX z<{YLv-k{d01g5GQ?%`-&w%6-eym6Tj2NFPpo%1 z^8p>E-bg78N<=~YgWS?o1<>8N{MAR(%3z(qha8zEj=wW>1w_LXb<0Q=f8~Fqj&~^CclNv&G9yhQBE&Wl-SiK@_PjW5R(RUfhS8q0z<1EHtXC%)I#v z&?4D9HmoH>-c8{j$sdJfTn80CT!{C^-ZgZlxmPUWEchECbdJttwZBjw^6h*F*1{Of z=Y7ncL}gDN*Tj+&dbjQlU)x{fm-Ez54FOSN9$LN=Azz(W&!}RB*{}1yQb}I`31nro zH4(8LnRw`EY?T?+8!_1QDIAHI=6`ozDYb|94pTbMILpy0wOIw9S3hgNPX71WD$SY=T( z8b;xKT&8ALr`>6>=(i$=3k{O-slZE=IbTp_Bc!>*MDtgcKJFpDwc+6$U z{fQ}3$V>e0!LNgk)-av32g-qCDoCd7vV$HJB4I_l}(B?gt$Tgxmz>-GN@vI^|Q}Qs}7{_RV@g;U&PZ!fC#@_c~xc zC#{7KNspw7f#STvO`(A8%5b&9C}I7%^*yoCID02C?%!Ov(?RFS!BM}w$NU=}t@gOu z*oANTVC@{|^**_;ht8ANg!Yx_bwlQR2B~oNvc0&v`k{Qal{vB@h^ds`&isG;NREAc zj`v%1{#=?)Kd5@(QB1;R$H&?sU=*Wo!1$jug(}⩔AKZtHIqj5~3Z^JR8-seM_XE z95ffB$Sw|FZQ?RYnBjv+ ztK>7cp3%SK^>ma)d52PLA7M>0&!*2S3mu37a#Y+219YpH%o`Le&3$MV$?xQBWy*2! zWm{ohx;2@&#J#Pu1e9|4C0Ziz%_$S}5Q8E?jdpYW!D)f5`Pb?d68V_vd%H>Wzr%UU zjbilJXP}47CJikUm_+E@CNYe2CG>c&t#4<@s+ZDI;L5$^LDh^ns)Lc5;cnL4zVj%R zE`cfvz>3#4#c1z+L!SXX!aG)R45{PKK7ttvKE4ZXJRm3IK8RP}`NsWAY}j#cd0BZe z-K`jQ@gFlnLQn!>#ib`i+E!1QW5ey&Wm>R#4 zuJ9pzg}EaOref>(7XINWTa!`6SKfH;4}4O6-+B);9AZhVqoOJiY^zuK0yd~@$Xv@r zJI6K4b`LD1>6MA_6AThzuvrf0P0Ta)L@C*umQ|XapQE_~oodQ^+#Fh!>tWE1>Q+Ps zPkQ-(4h^V9M*O361;IeXvis6 z5s*nNB-5AV} z&&&j;wK{Rh(868ohNl$yz_w7x3fudu)E>)}s`aWj2BSI|g2atYwEtAwwx$}i%6cTO zqeeaamppt|ld6iLWljEc{_$j0Y z&(I+I3^poPzdGqekZwS{iml)HugLO)EE6#ewbzABs*a>MYSTXHyI(JGxyuIHC3k}! zXLcH$!G#AC3$!&v0^+ZXea|NLZ+Iaauy6n}>b)bjOa7hNumqj)tqL1=yOem8UhT;| zf(5nO-tzVLojF!nRpRHlgJOgutg#ZO2;axgQhfpD@eRe~jRE{-{smNawOiF_0Mr&>;`Nxap}@Sfr@sZmcHCS1Z2wVwWYA)Pp6PnG6FK4Pz!&&Uy3bzMQXQ zG8_)Fnt8jD3i(lGo%nT6%*xO`n*-0I2gk36!c$a!#Ps8{OG{=puZ1u4f{`;6B^F|A zjU2JV`~=&8mk=h?SHi^^BBk%68D#Q-S(!|E#YfunV&7hwJ$RFq*mf4C%sRlm@TzLv z$ino29XA8t6U4}_-@vd}AjQiOwadK6yr@XI;zir~a@vj!*rX6_$-cjS_H+1-tnrd? zO#PDc)o|>(5j5ysm;XbFLJ2vE!zY~AaV%ysZm+{hFH&W>X2Vb`p17U2gO4kBoeA_sQhi8b-cKqq7JE=-| z)G9K}wIrWxo%j*3UfSGjm_5>V`|)@MIe+g$&c1OzMBfunuEI+aB||?sitYUnEJ@2lyAEeqyNEHWo^GdukO44 zSNVA@gmy-+5xZ*GBEBB{dA!3diGwv3RW!IG7Tm8HMyQcQukT5(f0LZnOJM&)vbijm zfoIl~$p{vyfP8*VoM$%`ZGMdSTR>;Q4dmsv#RrOUx(X(uKAEg)!+#6X^91oLoCo*QTVz5!_V+&Ia~=5{T;icc33z^g$9{x5O?d9Pj+nh3)>kq) zzyL8}PUXlkhuF{pO@aBXEKryMyD}3>MX7N^xZk{`Vh7AqY9g(~mA>`emFEOE=cARh z#+h&Cq_T7xNHN6UB@=6S?7&|vmu~2=!)x)ELp|#?DQYd^^3`69g!=!<0zhhyC%Fb1 z4Nj6kfI#^ocwD-nw*BQuFwr{rNv(rBJV;$0@J8sF3UFf*-<8cTAs@L{E39gWIh{5L>yQhM-e{@LHI|Tpg zKU#wtwV272H&olSK?O=?e{1Py6 zivzEnw|GDPW?WSruXEsI`rRUeKOkIKk!S^v?Cz8cVeB6Bdbu(L+R6X%(P#dE7-3EI zoinx%j@bvm{8Z;EORy#ikz!TU$(oMMw#DF7MjH*FX-DH?G#U`EHw+Xn;UW`>0~hhptGW5^g!7!c%6l%j&j9iFRf@RhIU}CQ)@;{Bxe8V? z8Sm>!vzV7l3&n2_jXtvcTZs{^3lLrs_s-wXvBLIvq&TJ%-Ti`%Mf3w4 z9lV4W^5GPKE*q=|Dr}PLu&;EW(n#xc)$dRtl)L=q@#+HMkaY9u~aU% zBDjQMiUPF|_86D2Rn}8W#~!srKcIUpd@K<|vbD_EQ`M$!A>y>~zP1M)R~EU*6itdm zGhsH6Wwsen^^TpRJg3Xg>6F1G#_j?M^M&pySX=IjB|p>s=PNvFqq{%W7Hq`&MM9}H zYMuMTh(%Cur7}o@PM+C=xHMvxPW_dPd^6d)B_8rP!MDZ2#rFM)3E&1KEhC{&1GKh8D>BS!5S7Wkp<&Pju{k&39-_9@=;S&Kku%e3qbZzTF(vMWijm#V< z<;)j>)}LcgYE{ut082%+A~9Rp!&e^fi8XP(%1ut+lTWn@6jCH(8{8T^f-l28t}Yi6 z>;qlDJy`L5TX%K{l$_m0@sD-nTwC?zFJJty*y1-?cn!g4eWFYCWQ(vCmQe13-RCSu z=`T3f6dD7qwg_0e+jZU{Is66C=XJrZLA3kM%=TOooiD^;&?CXSbf1Jahl0!ot#mwP zWb}n>)4yt#!l?f!N917tk8&iTpCYndrF@+SNg*OfW$7P}zZ7loPZGI7Vmo2%?M@}o z9O1p@VF%yes&N$y+IAB2y+)HhT66liKj1yldit>D+(w7!>REMvtcA2ocSLd4Z?3=v zS`WIO3M4N!ES&D++hP^|Ry9%m!g+O*Y1kYvU91|%4_o}Pbde*yZTi7YG$y8V{`f9u z^BX%>hj+!PzdZgkIa3#OY4@$)_!fV{7zIs-Wv~YPhe5DTN6woNHR^J6Kr~n|W-Ulj zisLcEV5k2bp9I`f>XhY7^1JSj7Q(z=vgkFlN-P-W zDmnvAt)fmq>dDV7woT9N!T6?F4FHX*gEB~r4|`RTw*dt?0)}sT_}Wd+x0dqqx40Kz z2ux2s1m068bxJ&pa=}0uO)2*{$qP~drYh)%2)o&Hv!N662e)5g*nl-@6G_ve_54=m z%H5ySZ(k%wsos=lU)ks2jnVql-+S2Z2)#uF@^{YTygc;YpazKdrc%(GvwYyjV6O~t zxOH}k_l)H`HT{sZWIWj&izr0Tq%mR^-}2z3b+|-tJq)@Us8^L0>qermwri#YbqADG;(cHQ=Gk`xG z3Q|I{=GNP^RP;0VevZXC<$(6YL(D?r) zBwV(HQvXqqT&Psa{)cyzz(#-5fby^HGO}BfcqqA%%xwR>>9HFUxYLrYJ7@IK)U#Xd zyDy^cVVbComGQ5ZnCH4&_T=5OD7Uq7P{Co~p@WH6-TED&^N$BC3hkA%6FNkz_t!f) z?^DKymQeLqs<)$)f&CDvJ2kPVg=?c)aiVXq&0*^%sdE|c?XWiogdMwD?+9F@^!}3! zk!Oicy_n1WlRS%tdU6;v5hXF46Y+)dJD){J)2ElG{SsM~M>W;}UtXiT@9lR8eh7|J zfUW%3kR9WTyDw)!Z_ThyqfMfz_0pnLf_)5bQEYvmeiH7hge^zE@%yODQ4Ao-U=p}h z{&RFF0jF+GTcIGqIJ4Huzp{HLbU0EGS=-!JKXGrybEVEZq7Y6&-V%F$WT@O}-c2_N zqR^SOm)Cq z)H%#6S-YIxr9vB9tL$-r8XGvwGbGds<_) zOGvRyn+;$=q{wm038_=|U_A(OQa+FkppqNR6Ev7AakI(m({F!|*6H0!AR&!Zu;*^e znQp#(PfD&oWu$a~bhS(&U{%n1drd3kN3XcoPzF|jmw_9!5N_It)+~qCU&7oXTsf~B z0zOMVy-YWmb-H)p0kxe~eOeG5ehk{{97?GeIk|2`YkAu|)4~LKNT*s)+xhiEn2^7IVytCW8}KzGjB=2?^&e(2jXUJYHrOr0~BDz$98^cF8 z$DHLk+V>2??HN(MsNIE26tRVlB1PN|IT&Vy@oId{=s&nUZ4B0Xg@?DFJ4-4HU0-Nr zm&>C!8&Wnn<7bv*B?@<9dfXAaZ=MeBEURtvD8|HY!5dh2Jc zE&h+?2IAa>_Fv0QQ?OIbx4F=u4&ScA#kK1g8yi0Pjg{X^&ymBn#PZK;$`az$jKq|@ ztckLq*1ubeKOCNZE&kZq&AqfNII$(}|JC^f@xI(~Zp%V$%DdnLs_*$&&mq!2Q47D; zA3NKE9(@*x#|=iF^LM=l6FS28&Isz zlnZd4I?%AAWZt(*R#VS!1En{Iuz(~v)wHCqA(d<a5TAC9_D@^0c@f-2n_-1 zcApE#(i#np7CWTY0gxIn6+NVbuAaR?9wDBG3p6f&J2Yd>fTb7Zurulvqc6T>o2en8 zxeaUo_*x^lSZN3APXh_hv>$U$SBB(B>b%iN5cOp!5({-njLq3>A22YuU_`RZJwxyL zQ^+eg0z%AqM&<8(Bt?6OY}V;~8opCpaNShx$MvL`({~faRHI75W|pc+HArStE__iH zs|PJnO@z~G#d$^p6VYg$xLt8JQMBy!4@!+aS6W6DvHj&Tj!RoQQ(A|1=2naOPHP%w zh?hnue$TvJ9;Ap;#w)ZJo0M(DZMx19;CsZW^q#9SUijM=y2s9!pr&Y(uM*=J>P4}w z^H(`3k6c?*9*(4nGy}^E9%>i?sb z#1xM|(DR@f&$&KG$y2WAFBiij5jWB5lt4r0YD)BZnipkg?N?5nvZ-UoQJ=Q$Cgv>I zU1Q#QWzhAuKOG=gL+4FmN(GZ9P{DN3p9|)02jMA*Thj!ONQ8Lm2K0he8pDTFcxIXIWTvj`u)u!& z^HWbLB=Eo`zwZiR9P~)^<9-T|+#(iPqWw~=5?}uS;su~ynu`Q9$EMhdSuEq`FNyN{I86tB&7AlQy@G=5R zr>koKb)D9dl2sbwgS-idCVniQXA}I`dVhmN?V%!W))*KiF*QRP9c_2E_w9^(zlaGO z2enuTk(+NEyH9(e`q5YlNerFX|Z0P4wN-{kmA46(Noo1&wgpkJslPHb; zg90jLICroU#i2y&&>uG5K_!X7zkgkPcC6g7(t;Cpz)Ti)MbBHPH9EB)xJQr!gMpQC z<SVCu;oB&WY5l)nhZ}hqMaVgYGoZms71Y5fX~W7{JJsQi%tuMT_{}$Ms?1 zUi5>pznxhA3~L(yl4F@;nfij529%bmrQ;X~b)c)Sg*j3I3J3*d>%__s(`iUc03y zjrTpFrjp}wyo7558L4EG)~hPHjDp~6GBo3k<{R{frY5S}6uvPV#_4yG%E5^dO1g~G z$Yy}*e-7h2#4-rpvU5{7d)8to+{Degq`m2}7Qb~}wB8n^u~PM05eCeBrW>=WPdAgA zx@*rB+E{E|yv~R;NZ&o@1J>do{l-|V&JVA1UrD=6La+;8_qG;KzuWji7Nv2kIFOj- z`_HmsR0tGv{9l$8y8mTaAz%c7{_b4zRKDBy@P6jn(Svqkqm5K#VD^o~{!5NKWw*2L`xLJ%VsV;z!U`j(PFN-avg5`Ze zQW5bFk6HU>9$lKkRvje; z($^eIA=4wXbA?+dHu89lcJ}!8ps^h% zkvy+XT2bZxLkwp={fap}!hvw~V5vE)eYeBReSBx0s_PvcVm~nG2isWH*N3#h-$@!l zD-1!u(Gn{zR-FGOx8a9wGXF+Skd5EKS(Nsz7vU#Eu5bC@Yau!~@)bmT<~_njOXCq2 zU4)0}2ujPZ5?_ex+F8=xjr!;4@Myza|}`k#^goXBqo(nL;?8hZc~ z|B1$}+ow$elThJt`&IhYNFi5&xmAoDUQfjV!mUxs#6NQ8#1TTbsHf$Da*^!Airomv zX@9Y_$|H-Qr`~eO=paTm84pZ!YG`?~LCYc7NcVTb-+q=aw%xtgz#7nMM{wnJp`fcN z-&^eB9}VC|9*f%t5-vHMmM5AI&H|`N^%gx`VO71FXGz_c%xd%PS958>pYZ#Un*93i zvL0>ip{ei~NzJ)pkix47OvgBReJW=7m*PrshnuMrs3ZNyd8q#GNrB%d@i0zbADMlh@VfJt z?(0bL>^~k7+XgxrfCXQM!Q=8a&MqFj+X}_97W*lJ}t$`SChpqxhwMAz%~Ws6SVBPLzL!d5g*ssp@WjtIrDQusoPpy)Gp z7y3+ga(;yDmoP7YD7SH%@4>5oBFBLZ^Hx^6Wy{Dldyb1kZw!KGf*RNl87CdY&m7EQj<=2UCgLoW$;$|%8NaczSeLPw1V>t|W(Lfw_!ZJQm4I)%as%a+ z4q<7$W5^Kqr*|JyfgS)GuwP=jo-@HxIdv9TWgIcxKsQX#?k~*?j_?eDgi-{kLN$z> zOcb1mbUOk|VLsZ?zO#ZL#B`?75UQgnQu&E|bXt>^Hiq2wLB139*vpLlC6>u4cvmg= z_ryY*!?9;|)qHlTj^kY(PEmiHSQ|cn<+873z`E|1q8!#DYVJfc^S@)`^bmcJvQdzl+!Q=!}L8Ow#0ceTGjZ;MvVJ^@1CxNFjnjkI9A zHSPi^5-2Zb3Ft25gI<5GF27!v3;c{iOrTD}+HSmVzPuAwnmLFWGQ7|%$pz%10Lm3` ziMyDk;sY&b;TOmA;v3~kfXc)&qG8S@4*hof0erozB4~=+Fi6h%JGMZptM@eh)qYPx z!mcq==AI=-P{{6WL6`cvEbJGG}+leup0&Oy%76I7djbHxKJBom{;;n?r`r1~rc zQh3S1Z~&jz;t#wJafqAkRr|UAmP&JCV8k#uF#Wu2)4c6Ri$8n)&Gff>#3htFGBy(4 zTKCI4$JpIKK}D??OW)eg_-@`~e|w{6ozOJkqy=W4}{ zf7C|x+i6VUbXAw5H)jKJWNJp-$0NDXZ(DMHkWR({H&8rlu7MsxbH}9cPJsl3h4==! zr0NL{SK2gwSHH9Mq&73Gy95T|bYx&w4@H8}xj?&Jux3Tl=9ng1R=btvRg#oX%nmUF2|@o0 z2}?hsJEL3kFSmfdbaTG|ZypGqJQ8(|K&h1_6|B*bsX{Nvj2||2@v4C)k=AbD!5y4f z9lrbZu01qW^aYqjMi3}=k- zA;E;Wy+UA!Et}#9O7@{r9WqW))3U|T_>T;-$}=VY(DpuW_M}Z?mSrGOIfMDO>D4VG9C0;XjE^F;_BZfc3`7<-%xahwqIpl^Q{%TxP9V8KPaZgOaXMMEi(o zV9Y*PNY+Xz)0Dz{bVxd4UARraKWa!X!7200=DlUvKe^hV+x5&> zuiMIM!`stH6Q)Oj5_8u+Xltku=%HC*+080><`ml~y%+G4-fxke-}TQpNu%#^*6wOI z(Gz`G}dx1!26VU}0Ny0sh!X)SV*@`N_8{cmTl&@W~YTN=5{V~9<#{&(&3(_41rY8gB&>Cg5OFz!(KBE-hoKA zD6%4158)3gkuueY=-&7c5DJNU$oGi?xV_t=`2H0nl<)wK53J?Wu8v6`Z*`WH5-R#` zq(_Sot&gba2l@fmZ&67lvgSNI7MPd(XvAkBUnT@N5C<82`sZM1H%g73s|tdI1pvM6 zJgC;q5IsdxNrg?w91}+o_;*ex+WXxYZxtBvu>`iF2nANOf*Yoo>PGesLq2T(QBM^`S-u+ zY=Jp{Qz(_fZ1uOEnWlc?-%3ExFv*yt?sOY;Z|W`JPzi0R&E#+fQhH8pMl!dj+gQnD zSq*Hy2`+^lVfJ|5ma65_`K{78$Q(w`maDwLo3oq-%sg*TC|O~%t-I09?n2$|g~J6V zJQm|fKtt>N>&8*7rYh7_Kw?wzYz~=>^?RtPqpGpDFDbxu@p^jk77; z_kYL*u_G4n8S7o3C}As(;Hcy2c>0bxlhhNK{{8kq&YOP8~)1*TAM5XO* z#O^1b6igC*r+aXTIzogE`p25nuHsHfONc}z{s8}ssMq$Y|Mf%RZB8K}dS|7`_PMQH zHV7H7Zei=C9HzD>h=Hscd|4`y9}V;?^eEfL4db(kn8dj`K8Ctxv&iJjLzg&oAxMq7 zFRltK{d0C?mcd6X{uTl`t)H7viBrlLFQhj95=s>#322Z1O(=B$kb+hVdD}47FKJTz z6Y075S?1$3u(&U^q*|&)(aUYg0q(ZjHg-FF&`b)CT{zpSX#F*K{pp52Wa!&xTc=%I zk$7{j36h&~I|=D4g}unA>_Z%E(9Idj4vVq~ zvw`6G5b_~E+86VYA=OJ1tLxZ-d3EA-8Vyay!%{A`d+$M(9JEE`{(4F!Q2J=zx)$Hm zERiLO)vt!`nLm-an~gp>0`&te1jj!uv!sCCkzMSq$W@8cIx!s^ooO(ZsT({EE$cM9 z$%j5@#)l7+P?y@B%p_M=e1N(OOO+GuL}O_fQpQYb)3;W2^1zr>O@~N9q;7!ZHXy_c z>;dSslSV~X*AP=9S9xtVb|pR~@gY-?WSVD&@-P7==q$B0{*s+%+EHOna=Dd2*jCjo z2vGi(GGUKY?et>r#IAFinFOr2^MV)H1$eHZt4|CsV@!DD=(RKB zzME=%)5diCd3{aF`D5#Ym5Ad;dcf%mefQ!e#+2IG@7ttmAWq!O$x7 zu1mf6rKEtTjqerqI7b1$8kyOCN8Xil#$!;k(#^)a(7^L~o8KqPm((XC;x}zGEN%PN zvyLRo&A-kr!{fibW-x1lCa=h^%Qa2kGi%qIusREupe9(J z_X*fzN;$j&dPGv9B-FJYAJVzqbCR=>c+6cwFX`HG<8Bes2(w57q> zG^`Sc4R31Qhniv6-QxSvCiQK5q3n*tZm85-@Oc%5ft~Q&vStQv0ACs+h&uvt5yVN7 zP=EIT^@!7wVEZ1~ zwm^cnO1F^&zf&BZ{6vcMMZ3Ai3fW{;la|ty=+{y(S1wdz$23Nd@x?Q&xjS(64}MDV zg=m~5t0=L=V$2t1k~!t_ITGV2x7ZPzaG`;~>&b>iep|{-@VTC}pabXj{I4^O(Z|Vd zoyLK?Iq7~w_Bco*83#1!nHb!X8IM+IZ8L6*+Py8j8F4WE1 zgr@2%XRLqtDR!d%uqR+>3RL7pI>!qWMVoelBe1--aw3zq2||V&gf;WxeT^tpk-!}v zt)aPrN6r<1@UG6;aIjVhY=FKy15Qp4K)PKw-t z7R|>zCh}aUOid!~-IVJ*Mp!Zg_{t)GZbVrX|B`($jQ>EQ_7xEPd;OQ z%HwfGOZ6POW<}2o2QIa*iZaz^!U-sKg4dzAA0*Oh8BNCkW@c){f0?@TANWKR&F{jf7GY(4=v0|pjh zj#t=I-}$q$pw&AfS!^(1ldSml=Uq`b=Roc;gXH|CRAnCgLo%6Vey$@G&^;CYs=rV%r zpa6Hfruy+3Pu0u<%-7>%y^zX+OI)(=LlJnP+^G?G7Sw`*urH?6f6|H0d{`jW9deVJZP49xe9_(?iWhqemK-CYi8fGO6*MHV z5}46xiRN7bZX%(D4pB?x`5#=y+Tx&5 zHB5#Ym3NBe34uVDQ->r8b%Q~=MOafhGfCi$__e(hmD7yt2S1OM?re+hry*I_?Q(Gz zeEWu#A%73pb|B93cY+1nLnPng*lwG>-yz}Q-n&+3kXGq*R=tZwg)i{Gu$)np#DUW(FDX5NLA+LJ|0IqwD(2S;dYGO6U$q;n5mKqSevr>+uhieD)3@LUADgul0~iZ_72?ZyODDx!DDL2J_N0g)X&Rz-|*);%7t+&}zV1wN9+1 zqRej%Umo%WWr)1+61nb5s1stjLtFa2|Mtn`ng8Ba^=G(JS(UzJweic>knMp4Hop2i zja~0JVPNKx5;pHTImbt3LuEz)ROMPm_s*)5rLe`kjjxot#7HQ8(hWeHtF=lC_AbXA zIV0j~<^9swc3G!<(Fg3^xR*g-ZG}P(6-@?_$|Qb6Vd4Tzaw+U_;Q~x#C}B4H3lkZe zc0Ewi7;q%Bo=G}6eSB7cS*%{I2ncjs42rzShZzgTonuu9%?<|_H-xB;TqG3zCVL7( z{(e^Xd|L#oV~&8Z>{Y|`nV;t85EK0H3yQiJONW3OQVwa|a<2Ej;>!WOZSjUH-})@R z1RoGh!6r?syDN(bnhf9usTK;f(L#if?Ow!kEwG2{wbbM0M@%V8D9i%Il~HhBn4)LG zQTHphX79en4coQdJQ>WIwfDgFQ=rh;!1lD}ES>(;CxU%GU$;qtKd0>CwblodJXC?V z5NRjf&v6GB9Rrv-PJ)b(G3f9ofLA7yU}T3~v3mgmpY9B9stEZkA0Ks{c81fcyg#7@ zv!s+yXWrG+E?=^ zZtdv{7~9>YsNrC@bMquTn@LljB$?udjv6MK^-x@j+VfZLU9a*NbLIViX zA7eIC27n$IXK8D-vyq@GPCdxluwZ&&k~JJWP%DK^nf)fGJ`Xfu>=08%LIU4WuNrZF zEP@^GwE|6Ms`SB0YkOO-O8WG|e=UNdBe0<5d3;}P$a1eD67%>l=|{^0kk_$DQ*>Bf|Oao9=TZoX64xv9SG;6jyW z?9E5TfEoJ0y^C3(y!0BD<~J;-eL}iTvATB=wC`*XePO|(ppu+4Cd}iXs)6YUnW3kA z;=)3zr+a&6g@FaHtb$9U*6*>9?oQb)a~kTYM?S+Ezc*jPof~*2mNn`SqdvVL9NX@h zi2V7TCBnXl16I0g@?&1Ad_>hMP^N^PwS-+>R+0t^ETF(X1CK}9cDWigMk`RzYeE(k zpg<$o_VHGnqA(MMiRKDW)rC8L}H@oEAuekrc~!kf$X zha6iI5u{X@jfML=E*RIl#B2{Ik@emYUtoQ?p{W0I3cME|>kZnUn&h1fhySFk1173d zAF| zzrOn~c5~&vo#8UnsO8V1M|_#mrl=C`rySBMlj6!N%vG&?Q`{$%nK$SSrKfZF!2oCJ zQTxMV;`dk2>aUEIiQ4Uyr>JYzHZw7NfhB*~2R7(G7I0F}1k1S!M{_lT%@l_X#ee5D zo?KG3Eq~k%aG6LzhU!$~Y(x%6^bKr%zPgv@V$o3QH@DP<{WY$tDz+2 zQ=Y(;`9(cyZD&sL#%cDWD$*zFX3%=^V=^lTU4gc4w0Q(AmLDCQEB1uAjGq6g_zMm% z71@Pc8ch3%nIt|Qi!-AYRxs$UkNran;k$6r&|tbCbb$ezJO$1Ex`)40#(r;N@i~+Q z_Btm}Q!SwrYZ=V!SA2QMiX{AjS)?b6dncOkOVvUghFj`SBy)N)a*=--@qh-zgjsm+ zN3>L=9(PeWS<#Ln`-@*pQrNG?Jfen!>v_gUkmMz>mz~qVwT>>`8%duIVLUbNHt+$V zscjY+hkm!>0GaJU!9~b7pDbD%V5>TRRPc6|CHfDvmJ>qT%WT@%r?s=u6+)k}R67$B zaVa`0*nC>RqMoh({Cj_dqnhLL^ydJ!Ft9V+G}1O_I55|s`{Oa}lw}MtiG2OU4Y5q?Pn7Q z7LL;Q8#g;HHl=}bu;^Q2G3yG#pZkU=y>5b*2MV=Ovc@q(G0yQL^WlI zgpv`7^7{pdB<=T}`pbw93cc+mslVw)Nb24N%Cx+vOF7a-)^n0<4 zm-Wde`N^821G9dKJA0Zeng^v6SDqmi_-tsL7{2pLVj->mB74EVy4HVvMHSC8S$<2b z;CN#UW!BH?Z$?(~frhXY{6-!UH$7>rfhQ~)!`Lj`i4$)s3@}ONSLd(S?iFiz?hgtI zKL8lZ$jyb%9r1+dzhl)uB-;TVW3Z(r4VcY|Z&xMF3AJ^^k2EZ!708{Bx|mXB0g4u! zeNRJ((_2IOs%b0lrUugDBaFR=sHlYOnAuPSp_7zVle_;vj1 zv%@ip0~@mwMy?ZW?w&=~g+rMywD~+QG`u4BgUv^K&QrLo(+TV}G5t=}u65H3o*s`2 z1gc3&@q6&59xFm;%u973jJmVe>k)AlwM-Ocl}t|o{XM>=C!rY+h*MCYE{Xj08C``= z%^f5f|66={>QG}s1xw7<^qhTZWK6lJLzcTk#jHZ}+>VqKz-?aJ?K z4A&S2MSlmB(5yq;Fc#~+Nn`g5N8U|^RcdCpbA%|5Q5GP`ACj`&&VS>xFz3mA{sDlK z$6UXr3oxCVKc&qzq($I4sOIi@*3CWkp0QMOf5r|w-)y$*JVLwqkxo7YaT0+VM7>4_ zt49gA(E8n+jbjfDhX%Rs*F^J$9o5E@tdXoh^^d1R3cfS;gjWNm;C~}Q4K6URwL#y3 zR+kC!L+`!W<_rSpyt|_^h)% z65tbo4qC}}3RGQ4F12iIKxE%~h^wEt zFc)~OTkH}~tKf_2z$vnr>HO5Loz9yA<>a^gra2y`K9OJdHY4A1`#%77Hvx~$0;J!T zCe4|mJH%9H;H2BlXrMM(KkdeCty8tlctr6H-2^=RGJ(9Ib<6y5Wo44@Smh?+(VrwF zVZ69&F>BJ+7}9_8oq=J8@TJR1Vs{E5P!E#eb6}zQ3S<(n5y0LEh}h{#KYcjkOQ+fDdVu zB~V!DTD(>&jKeSv$^79|Q0v5ZOC|AXbKP?Io}5_11*~jbn_Bqw^hsG%#^^*+IJvpf z9%@Vn5S|Q+U7^Kd372@4A7#Im=T&jpb~9`!d@}Q~4>IQ4HS2nL=VUVP+TLk#U9|Z= z8(H$|eB<&H@e@$-l+v4zC1R^l^5kB#;_FOBDdeUq2$P-;!Hb?CG~L_)y*B+`NK-;S3Zsz_K2`3P8Ux()cXKK+uOOgOB@${igaw$K zE=Ynqk!FyV`-f>PnyH<*uM}d2uuL&Y#uq>EJtVjE=g&fs)3|uPd|{$mE|e_ldT-6F zhv}3e{qVLnEnlgD*MD4jWdC4$IFbi95m|vWY%!=9 zGU4FiV{Se&zWK4WT6S|0#5WOm`tHuRqH%ouk+&pRBiBoL&wGM4_Wp1+Mq_hUaCP7v zTzvF!)OXw9URu*f`|UMMY_}X{9E20rXYYN5i{d*AJa7BsHE>yRC2haE@7A3t;5hXC=~zmH3b^(L-8 zKLMHa!HByty2s(?fW^c6LF4gw_S4uQOd#QVtsiuKI5}N$^(C_I-Q~nVl70Z+FU>cC z-0NhT58DsHUP16WU`m#B-Vg8QSTm>h)|k?bSF{x9Y_xH6>!KNEr0(U7u*Cf4E4Baq zYdWDniQNtn-n2#^9-Jr7P~`Bk&sA^ayOle>d0(uyg|>1!sM8KcK!C|KoJH^g`5SGw zN22UQUr%7cAdrm6$03A(zrIi%dsr#eWHuu^1|CAda9uG*DD1jlZsUtBJ~yx~F$xoU z!i6R0FEF@fN*?K({_QL3m(i#WlwCz6!o-deokg^k0>Q(Fuc@kL_c4p)=K8p}NYD%? z(%pPgqj>_Bk{}>}zX1C8vWx`|=TBel6ZrjC3&8iC%DXotK^>P-H-)W&7s5Ou&U4oT zYg#RP%K_})quPGgI-)=S{_VBEb(G^cRPT;OxaY2R`D>c!s65~pes$vWNCaA;id>_c zm`XF~uG{uEM#dppNlCG;!rFg#dwt&;x(25pjU2NKQn5E|Zmt+%qGz0yxFRFX7{9ju zO!vUPr;4Ay)tsHE>#0~s6Il?Fv5j)2 z@(KQvnJ=?8ILnA1@2PNc1&r$P=vt;|Txi`O$S&wrEEz`P?nbP$DFjvB8Zk}v?`b;Z z7}K8{GQrq-r6?#&A#4xzNqSv)Lp?a|z#iV&jct=}#cCAAD~-O$rTQgR0>vF$swCw~ zk*NM%)z0hUN*4DkXLiGsr`0iZ6C%yOx)ea}*FIW|JXw(h{jZmHc+;LbVKlX>d=Wts zY%*42;Kv#8gZ(~g6QpxCCOkXd$PsVk;8C#mN-J)&##b!TfDEOFt8x_9p3f+u-_F62 zHPLo`cN=p}C2}W?C>1GD&fKwf&3;oy-MAB??IJB55_WVOwoqbc@f!3mY{CD5L0P^xJ8 zP$Yd}S(;{2Q2WnVHE$(~f$xv!&{kh>G)34L9VN1SmNEH8F%{cSLJ#L1M@Sl|+Fyr7 z*rrb#Vg{0jXd~U|shEl4A?1Vx29_X{4b=n5Mne09()Xq%u+&#+g}5Bt%CUYwT5Am` zo|2t1^+65~?xQfRk>kO1-IRUpN1+R4g+CLW823{@#ki}=dT@P0vqRyTvl7JrPLaeA zQ$iTMJJZ^&0NEEq96qL+3T$Ost1o&wQ<_ZLke|qshntDtRAVH5GL`oCJk!Gqc@0{d z)x1lsPD@l^CIZZ@vn#m>V)TLO+!leJqdm7Rylb5aif%1FlRp=a#K3PqiiT(QgV~7^wyUj`jm>97UyxftB=|!n5_G zbd}JP9Q5PBz~1|QyAf_<>kGnY+xLYNsDRHV<(I!s^k{fw3qD#s?q=Td~U9L5*g)r zzfAkTZv00WE4B}n1Uiz`tvqfRI3YV%!1dbQ}9pwF5#` zGs7Cc<{(cdTd#U`9xWphrO13gqN&JGbv>`FLnY=})SIpo&rEg&4tzcQ$cv4{oR1lT zdg&s3M{M45!loj^xVkqxW!;bP+uk~N4~UEP91UC#1y07^?WkM2gIm9WLOz;{x5cR<99VCGx$H+sof@oN;K)ZmmBE%zSL#@`{)Fb zW*G&zeYD1u#X}S(xn`;%><;8!uXYWAhbSqMu~Z#zD*6l9fI{eAtX;m&ik?zCV{C~X z#7zpA9jd{-8%OmdqML)H8ZvjfP_q=lmVoSJ6Zptya)$`;w<;g%oUAP508G&A&E;n1=TVQk+p4Z|6?MS|8V`R=NeuD+e8uxnHK=lvBU-kTaIh__!!wb~tIc^bZEGkNlXq%`30B**|~>J>6YPDddcfKIW=I|x?JNgU0A=uo6g3n44g z)Zy*vQ`gnk(y2!mGm^SFFAlP2n~oynx9ZJChU9XEU%EbuxmFtz$&meIr}fW1o=jNy z4CNI@sUX&)UWo22xx2NyZmnU3;pfZA1dHHC(c2V+M}v7jU#qW|2)8?V_}HkvY2Vy^ zsl^Vdkl%6oh8AvVU7_8?y8T@&_zzLxQ$Jo(0vpoX z+zp_jCs?zLZr|y+MMj;11dJ!R8VNTV6wOFdfX{0~();?Z8)zhOoHF4!81p-pEv`;8 zb+-KiOL*SmxL%{(Cr#6;k!{ zlA1`ek$IuNu$qHXsTC_T`U7~ErB%ux+$#}#JJ|wP7AX})3DHdE7JRvb#1B#`y)P}h38#iNDN?|(Y6TBsF&dU!K&V^W=TtVDiM*&`!`VdK=H>hd(tK`{Vu;&4wKYu^7@PX|@~)Fcd9`oHWru}Ps)6hdSRRt7h&v6U zKuM%kLxUML5|w4`+v&J5tJI`Yw;C$(Jxs{g0YN7M@7VERgET3s@nEA2hQqGk`))4? z?c1x8r~)v6p$BcT_k4^y_Nw&d99&N^Gjt{$3XD?kDr`NJ`^9NZ1@z$eXImUCjI}4& z6fxuc{cLZdFoK4P4GCY?v)~{@mvjHzJW3AV%zM&ob~ikX5E?u6;B80T%U6t$s3H04 zP9QrcX{UNAJ(J%_%Llen-$#kT<$f6@p~ACI7mNDo_dPZLOqgM4*j(CA&Dk*Yk{qmeDtfyj)n8*H-q z2pJ;-J{&vggLfIcat|+Lm+B-d7cn~B`tav3Y;1-yGCFYqMDgwhCiqjytW zU;^mWJr>{ZQ8qp2_n-Qz`ZnFtd-C?DCk+Cw-YV;WYsd5hDL<-RYwJ&JG6VkdFPr`I z6GALjm}TNmDG7?TfvH+T|j(H?>` z258E|?ZO5SZmG36H5Q!fnvIOwxU9Mh=)07t7G_6G(Mhbde3LM9%%6yml@fppyR(ew z>od>>H$3Zufb1m_kgKt}8T_vFZmCJZM`9wP-sR*_IGty)LV_qkZ4>Li2NaE#xF4Z= z(WVlx{LD|JH1xsuLMM);XQZX_O;RpaCp-K*IM{U9rFI-u1bI35x2Btz{V$B83%t9LlazKy!djb&L@Sj)ld1mxUl>%w!};)Y!3VKwe?e6(MZ z-OksjDH5(X_Q5dB0Jj>1vZkBFex*^aSpHrQ^yG~l1m_x*g3O0n?qK-LBq6X+Rfg|H zO{NggFh*D^F$z-}{|k6x8aqDNfKTrav%Ep~0-r#xn06$18FA)aYDX;Z25NqN?qNBw zV^{h|U~06T}UsGM2 zCzbya=cBbMKTM0X^i zHy(GvR5s3gRJ`JRA>=24)fNF8^e_X^X(4ADp1zeT{|yXXv&N z?~pV(`)U$~0WFJu8}Y#lWzy^bL4eVbUWgBz_AU~y zq8R&P5hf2&;NQ90lrwql))m<3YZCw=w<}{9eRadUKhqcWT=!3)?`j9vI|$H|dh0)x zL}Y@eE&)df6l1AedH-gv_e<9&$hxiCn6nmEIuwjJJK%3Re+W9yi+@iA8oAX0azE5sL*g6hYY_pJS-p;NhZ1u=}EV%5wRagmODCb+n zLY-~?NC+!L*H5C}AquJy3`k@bNEvXzEpYAIIq=y=XjMh3aKeUxik-M>oMgiJJ@NmF z+cn`pnLbia;PAny`=OO7*<_?it^YyGc)RU;PYO96e#C0C#wI0Payi;$jKDjG3ac3x zv&VJd)bJ$h;^B?G)s3~7^`(D7TmdWvRUXp&=WE3ci~G`Ve8-x^YqpMQ;={gX=gcCK zz@H=bQ(>iV)Q^O0-Tc0e>}e};T&FS|H=oEYl~$^jP!ak>5R&RBfKd$I*XSuW zG|`rcA0qx1Pqo@}St;_FF0PZt1+^@TR0p=~_RC@RLW+JI9(-G$pUHkkH2SDqx_nw* z%1r_s5F=5H!cwU2@7F{)cffl7dT~HBMo{rzK(K^^WPPa5pp?4j1cQtr2>S@(dNN5M z@0}U~)~RSksNkz_5=jWNdI~w6Puxf|*^n^=`5s%JU2hbR5n^U#@JE5ZS;?ri)4Rad zl}q27njc;SzY2fbOy}hKT=vyuY5V;Lpt2KKSdvZJoa&OIXlC747Q&zUjz3|e=ZDIMhtoj>?TOuR`dLc4QDsCV&TD37cJ_wzNyqn~i z@LuYHF%zsH7g8FcZz5Y3y$lAXgKMK@kda`h<^A#!GswLz=DNOW-l9rneIVoatCxfE zzi9wS;C?S8Ik~Y!+hqwLYQ}>f83|c0PJtVv1LjlyEEVhq>Y9^iYf9&9LE+o`XGE5X zC`~^%JNbXlG%XqI3Uf55Yb3spXn5-FP8RW_d}`@kk3>FmZ7d~xhNu1E(~mhOCW|5z zsBjxHosYh$Q70&1=j;SRK(N=_{=4zp0z31vfq>!bUyQ-{V#0!jKji&YQOZPs(XyFt9)7(T}dyx9zxb!j?XbU*197QDxZYW1q< zs9sfP0!ylxg2sdI+?t9@4F=s3Kgh{Xb(Y_nuq~|r8}u|1ASTX1ls|=UBa{RSPW8EH zZXLIKq3h3FGIi8VPX_M*(0RQ!p(hJh7g#N0!~F zlelw6QzX}G@$naf0qj6{mDvQlb-3R z1VMsDEeSt73Q(1qFy=RPbzDN5C7X9QPpx)|pO?G(&(9oow^@93YNNGN&$&viKK~K1 zdX9F^sO?6ooa7k?moD>pE|(m*ojsC6XWBi}8A z0hl!6`>gm4=o877s(AXUAMzdGU?1WztgeA>f32Or6bWzwOCludQ=H_>g`TeLOuokp z<;W6LO7fuFQtgAd5TQ6=iQ1NZP)2QC;Om7lnuS?Xec_~vcYb;Y=UYFI-{~WaBk=ZG z+*>cM85{W#JN;SNy+vW_p-=C$T?keaC zM%a9hJ$1?k*f_VT4BGW>!ZcehZK53gVOFI>l8-_5VqR{G%9MMLl!rtn+wx1+{%>Gu zuOpD}u+z8kkRrJI1dCNyC191weIoC&dP@AT^1y^@axhDP6~oZicJOjtAaKCnZQ*ki zmOuH9$=PyFv@e7+;6&qWpW#9`x!5;#v_Ey~UhE`U7sJ)p3VmfX8?2EPz+At}#g8me zYEcHMJLXzjQu)QCf6)RZJa2w~R2YJ&J|B%x$seCV$fmf4-Y`K}6Q}Hqa!wR0J z4NxC+DF}l}%n(xZJFtQ~Tm}6ISO-}`KV*Ss8$*RGl}-n%iL4J&^xIAQArc60F-CO3 z2DQG_L@CIU4-!ZkFl>woraPijoENh@K@EI#g66&+Ay3y7#Vx4+n29c)qjmT~(U1B29L}b^ZP3_S3z_ zDhn5qnPl7Nn}?KhyuT#%(-;jG5VFF!F@?>%H~elW`ypf0A(XpART@BKdFyqxrj~J) zcXcNbsn@Os2?OAKaK00NjBD(!MnlGb(Zlc%fk(0dUtP~tx-JKd?F)hxE~hV{L~q|g z2^dBUb+brgFHyo7%)5ejuMD#Qx`L}+1=iP>7&>A6pk8wv?6VXmtzN<0h!9+SMQ+K% z+-PS9X24~HUCksO{wVEI*KEg^TSf$4Pb90(D^z#VN)xlwk2atqVAzSM5e!YoziCk@ zRstyBr3QBdJEEUWs|-W>ps}M3E|j%W)ggFfsH_hA3LPvxv(zY29GjZEtfgR2CD$*n zsbgSlI%Pr!MhSpq0DG(V#z*1V8u#<>9|LDRpu$(N7`Kf6?9Kl_qpv6@ytOu_IHV02 z82+8CJ!>@0A;n**Md&7CX9{;QXFI0t#QB%Y!aL!hW7kh7gXUk4KGl-x1m4J1;`@)v z6xDuo?bKIZZSM-2xw|JE-0G`hoNg(LG&q`jc{t}BFlyF&g4X8-!hSR)?~&>57qw~Y zA9lhBp4%`n85`US$s+r4OZxy$tv(PQ8skKfgp8j{3^FY~!F&C{r!FoDxGMcWY`u3p z)zKe6&UNig7uj1~6v--k%MACrMzXWV6;Who?{1L@E$+25vt?&huDvqLs3^Pmo~u5e z-{<>%{Qk=Qhuiy{_jx^E&+$4??p{Rdeo{y}F7Li=i6eipZ@W+eRW}#v58rjxd~tV@ zo_A)YllrAp?mdX>l&_?{_0NvH~v;si48rI32gjFY%jxe1ZyS?RHLQy9d1<=FA%(&MB-a;314n#F;kD3dZBs5 zjrm;ZjYX}oavF1QwnRRNbru-ozo_AX^)+ug{$d>}FdT=%wlJD}|Jw}aO;K_nH| zwp$pFOP(8({$}0n9nX<`Cf+=|>7qS?&>ICvL6R3EQhzcVD^6ym!!AKqx?6r)@lu75 zTtlv2LR2o)F1fGXC(>);ne-kWP->x=tE3P8yh`Kw>u&x?Ixdz=(xW=n8Z!<7wfYZO zi4yz5L%>fk^eiM@1p~R3Kjr(L?$nQ}d2_u8{ZU`n2iD6-3=TA>EjIyYpayG>Z-w|o zT9fV~9uj^F>AH-VUFJFS4=fCG>T2mYUN5IOK>1*D;>CHzi6iYU!V}A)z-N_G%bb4A zI5bS$4XDJ8X4Pc+J&2WGfO!#S{S?&a_VO7+;KIgE_Z9rj^A`S8t~1)F9itz#vht2W zS-X0N=W&8jdNhtbN`U>IyNw(qYwsuM(_f#To z&woCI;(#h3{Cz6d76Wedi+kNHWBxsIYg4=H()F^#1wB0;e1rFPf4-BF!y)3=1Zk__DOKy82LoKQX1`t_dGvxQ8t)9U})j`|!LKPvaNkYQY$;!9G) z%=QauLqf$>xx>H)MZ?l?2BUbm`Bq?obu=x|iK|J{aj9+z)9`vKCW`ANXbj1Hvl8*V z#dfN4H|jbvG=>_9a4Jg0DHNKgSz~~mIrg*#;uKq4SBPpQm$6|6C515wfU3PajVL7bS_>nr%szpDgv8fM}y519wJbR2_w_~KyT zK*b9mN#+ec#v0lP-J`VZF5jY&&X?2`LA2QA*b*q$8#V~BTGa`$twQ-E#*g`b7Pc=O0<`WDR9CoQ&L*_ymf|1f+(fpp+35u+}#sBF=Bv`o83{ z-2dSGfw=noU)Xa)JIpa2h$A7M)H--L#3IFd`6EmDArCV;HOJnB?opWBb-mIEd)VZg zYd{ryVUKsVQemwLs|F6tGbmA&bR3$pzK^)R4=l=q3WH@JGR%_JyB!+yO~zYm?qFX_ zCR@6TXN4w5U}QudU}sY+5|X?g96pALu#@Ub%k z`REGEkA~8PIEFHQ^>&z%g$b;t$eEORbzHrirNK$BDjw1iU|d#t4Jjm&94uB&ZEf8V zEfJN)y}K}_1&8h}$xzn+`6awOywo`QUjP=y(8C@cNjrk8d>g6=sGM(K*>0VZIO}zW zpVkl_+ECnGm$j!1<4zQErS7}o*8Y1#Cz_}I=PI=)U+T*nwkYdp#d%TZep(UT`wFDws^f~NsZi9J(GRduj{vWao%D;!Gm+iosJsM z1lUhr##Zl|2{AI^gjO-*fBem{Y8VzVBWNm-R%Wk1h z^BXo`3B_DTdl5VzS$;S#gE35pt8)gjcMl1LL!~BreWa=3;9kC|!$`;}I{R~4>kU7r zb$B)MI8E=J3%X3BA>iW{xtzA&v+$H zlFDAse$v_vYy3RbauB?>=c!Tt_IJ#j?w{Q2oK53}*Y)3XxtqPl|5RP7(I{VQHbUYA z%n0cHl4h7jPKdGQC0?tuj$BH2e-=~5yNl0#f?NC<^q%l;yYpw`nUuqwS$AvYG4$2! zi*?6?*(<<3uK9B1C@SiK{EuNCS@p#u8QHN9pj4Khqv0f}{C!sZIk)y&I&6b2O%N8_FG@S2=dT(KOC zUhq*^Pk9~kYhEQ?R_2=}_&Ezc2lH15UR|V}W<5}!8*h-qa1zsf1QaI-2qgrp5tuDR zm$03!3EP8cptZStNI-*z##@{Aw2t zmr(Co?p3swq{DNO=RZ%sRhx>fN5?&!sX$TEoXngQA0q9B<|UTn9~Kb`vt>z&OoeP? z;et0SXgieR%){e;q8_WfVpnTQ_g+wRAJ<+4rQboGqVi}mlP4P!8rG=CBxB#8+JI^o44?#TOVj>={%7sMTLU*0>;0<*iqdlr|sWzAjCDS#q( zy?b<1@fNVcFxrS>_jc;yw6cZy)qZ7=aiW6@E7lkBVfGGXqED_v@{WX7~R zVI#AeE zmE+rmehi%#A_9xs>ac~#eBJK22+kfe7#fDa?R%1Uh^FE=J+$}jD3Pkhd={z|*hhqY zRNyCC4=)rlSCzD*3z;^or+VE=tv+NH)yhHICgK>8HK zysi7RhtFxkJbshljrCsfr}gD7`d&z!)_K*=ehta^>N;Z=pwu-)w@d8DA?P>-X8I#T zg@uMKwjRW1<7n%h`>@T&>v_p+`c7dCf!~%ovcw)pDSm2F&|q1-{!d{5OG*9nRyHmZ zUDD|{!BD*B@3(9s11i$qQaz?R|4-7|?S)1PWR8l(9#Ea*iFA>RcX(u_jE+4S?tSat z8{KWz(lYjbyWbRg>;C@9@9)mb3k^EtRgdZG4kI7;C%ykJl|OtNxI!kZ zR|U2RA@l1c@w=hOYN{o;M0qh9BXs?GQM`sI1C9xeS+|=yCJ(^)sskzF1N$;IUVQ$V zoJlpQ{zCxc?=DvISP?)5kWs^=c|D2a-hGwANY`5Y7^`o(Xo`8if&=|m*Cy;%?>ie0|>%U$Z>RNauL0{QWdqUptKi=AF)sqIza zDAlGuxCQ2EVDx6|1z;iMFi$!~6)t#%D^=XL(l!RshgJ4vR7{8eKor;@w*jo656%Bn zTeX+n_+6v>dxCsjee&EH*$%yj1Gu+lSIiuxt zd#n{z+!Yi0qaf(#L8voQsd4xcgi&m#w0ZYvrK|nrs}_9v!-rE+ZeAx}pSd60cfC=* z{rM=M|M9UcU0I7rJHDjau)jNnQ<{GMY~tLXv%f1@>#@>MgcSvi3}l~6GP{$)i7}HD z`&WnNaDCIWoDB1dN`79<44YrJGI9ZXRo4b+a8b9P*G|< zZaKNSvQu`#{N}As#Y-8Mp`mQ2@eNJYqOa?X`F4KuZO<;%r&YWv(|xnBhDlhVP2Abu zYnhU%k4L3&H@{Q(RhujMympQjCaEu7c>20s(o$K%CK4n67IAJ>UOsdGtBkd#XxL}{ zSj4oAQBSyjrziU5298e99Kq~$p@gXIX~7R-xCI*ag3gvU*7Gu$UP1M4oKn0Vt5y>a zVIv{y3?sLrr%=Nr@Ni>rZgcRx(cSO0&7RV)!hE7uox`+f$Qg^IZ4@4 z8|Q!*MeR_H`$Mtye?C0(#@?UA@F#ATcprN^6FYdC*Y~~S-Q|4yPT@EHq;o^wr_M8> ztE&4I!J+m=94e8r?=q>#)CTv|`0H57+hgEmESP!}R{vCf>_UA!6}fe$iyo7&ex1+~ zyfmnhUxRc95^EuOKqiEZhw;`_nDd#1qckz_bnNh}YO>qYg#~7$7`Q}*N>TfLsFg|~ zYPqRgUs`m8>|GzNS7v23^o6u7XqFKV&wZuh zRGZ0W{}z5|{j!=tVkPd)7|*-aTcsH&V_G9YoFSM`drG0pF!ZeLc{jkHb11rpipKq9 z>#l$`&@Y%RFozv{inoLfSqb4<;pW$ysAh*pStL^Yb6LV?Ejd|>Y?r$TwuGT~HVjMH zy@?+|$CiWAJF*B3H#dkGC`FunP_`+MUP~acF`0*ECu+isYy%_^HOU$L} zzjstHPt`P_x7uq~1JLb&`uQJZ!EyeK9ovj$+jKrb9VHkZTv$=#cZSo8wd^d+rB>&x zN${3-t^?Mgs|J3geTD-M@hKOwQ-=PuK73RrYdSOP>lY+N#*b=Jh8W0>YH4DTH$qow zQ+tq8b#K#+^4-SY-8_0Wko#4=oOGmYy(Vl#Q_Yr@H|%WJ#0?33+UtM8go#?18{*cS z*b^@tGL0a^4nGp-lsT*4*;Hp7e&*NYd7hF`Weqs1!v zqMSA&!(r_-3|Iu?S2Mm5x()UQV_bnIF9zGr3x9V7JicKlFEZSrlLvuD(A9(qnlE$l zyuYRcjKwU3Pfdx(!b|W-LGGA#jag8zJinQ@_)Yjtn=?X6_!-z=%P3E;Eqmfwd$Ho< zx6}S|iymsN>8lxmbgR}3h3T0?rQ$_k(sgW(&Z~k;MM4(g*56ZIqFqv`kj7;TXcAAF zcicXd>yp5v%EzYpe41>^xl}iJ_0Z~f{-t3Gg;P0#$j=BC?kcP>iWC1W&dL~`T87LhUjWWbeP?w*g!+YdfnFY!mw;zc|PJ4mwE(c~#YDjEWeXSK!8QYKY=u&FB(z~Xi(!n$u9t|BIl8%JL%8+;u$z_^-LCH z$&Ow%D4=XAP|j}BtyR;NwwB$htj4{lt>4z~pTAI?pR6X$-YUG~d*)$3jGl(7y5AsFu7cEe3ohW{)g6}bev9M zyLl$|E+QIWZg`Jnw4f1Ulx)hlbU`wvTfO)l(KSj{rGR`3O<5Hb>7)<^M51`QlG+}_ zx{vEj)es$tEwrxYs$WZbe_tP%d&Tn};w=9N7T&)K^h;93%jYesNpbSh|eoM{q&0k66c1vC^#LUR?3g1tcah|$6lWZqpnS}BO!Kw$HY)j&Wn&w zL*Uwj+$q0=>qq121mh9bS9wkAFFeE*)AAM*;eZRCD+=@+|Hy7$yH?7`luG(EyN8^XuSd5q8N{See2O%&?8kK8g z+izc=p7T-^j|2ci|;1N9@b+@ zcc!R(o+g+`H4jn&nxj|?(hGR34i-+DPmYJPiiuvxHK#%YZ%KqT0_b_cLNt9|X7Hfv zVU9P{PKgOFqWV5w*V@~q>2(=s#GT_C8@mkw8IE}B-4_ji4%~l(tpxgm_p(o(UG*f5 zBQc$=N3OlLc)ay2X-nskcEuKNtm!V>u3|c_+$Sl{R*d)z&WC0V@x!YKw70Ygtq}z8 z0vuJpaJI|+yWe7?WH*S=$#&;l3%WAV`gUi`#r?wQN~x;t-Ib-%vp6z1VVxKt z9RIQ;qo)WW6ISmA6r-dz&&j@c>t*&kUY|<5Cjb1`h{~o_Yok%_#swl^;P9b^R4sC} z9M#G1-a1Bq5){3Eo(n?h=C<-vTHLd#{#VsjuX zPsdpr45pzvk358tX~vjBdqLOQBEQjEidNe0d!qp6n1r#zjkU5 za(Rih)b+HDA|2?3F>FqXR^ z^2`yVW@P}awr zR@L?4`r`wVWh)HkKAi+)DCenMM(2-Lx48QYEdHYsC?=~Q*APi!5Z0llCzO|V#2{bg zFq-mPBV#LT)eV(0RW1LlCPEg_8tU9qVE(tgMUl#I+4no8-%#k*d*_p_m_+5vRh!w0 zf}G3UT?4;JlwF1M$43gIUP?O}yM2OB-L#9dVm5U?iPKJ0R;Uc=qL8Ov&x$y zBY7#mC8=5ym2X}(rL}6dKsm+bgz+$&wqND99Lhd^85Wlu|N4o;m)&g+&E5UqOc?lM zsg(aBDZK_CTolm=smF+^oypGk7|%4^_^X>`b~&fvjmrsuVanLcxqRoO9jHzD zwxz9;7p)t@y~{bo=c&tkH*+q>gl;QK65RS39A~HG>IabbJ=*2&4`!lWv)QF%(hw6; zA>;Z$T6_{YGtr;4QSFd8{9Ui*5YufkGGFEa_V6@_+DF4;aU7EUxl9^m7zOiH>-tXpmP32Uia!8yd zkZ;YzZMO%6CscaWF45Ah%o|7&KM-Ueyh7K5*fkMpcc*cIoR#yzSqGljL0)4WVGBS* zT9BWFu>Zi>9%=VthaalX!IAZai)t0vST|mzJonnY>up>4&edmvj((qOS?a(Um%OzR zGRG)!#+RUBV#KSq#p(n_W&bwfE-7((ao&8`w~p6WXI|5nA)E1BlVz-lSCX2(fh=|F zJpZtsuPyGJ6+dE!Uk_WBrM_&Zhi&H(n<6Z0( z`CEF0fSH;I*UV#P&9{ZwsjA$1obWi&BI0v@c*zxOrxtJjYu$$&#KDQ&Grt(8w!#2v zL3eu=5_T4$Fd9Ki8y_`m+6wHz`7zPp-w6D6=blNZ&R<<+F6k9K{O0o~1JAlS9`U23 z=^wiRaGMH)nBAP*%Zf>qz3kETP0qPA=P2CbMxBe!rEMw!=cO8fQ7xa7xM!CEb1`Ek zE1az8qI7rhNZ9&D(GixRs#H}^ue1Hhv_@xlGFP?LiDcq z{Zx0}Z?%&Ox@cCKw;O1YM9Cozn(1Ef>arKx>N5j7wgZ)rJHhR&o7MS9z|k@~(TpKz1k z&bLHx|15~BWDO}_=`=MxI9IpPOHAjnG9V{ZcfMU1hO>LeGqd-P(^$wo{yK|h2;#YwiOq}?&2>K#t z=62r1NS%c0qYGFRdfIE|hV-l?TCJdPown*c>JH#dbT&^nn8Dz!^ zJly-*AZ~D12eW4UBIr63Mb zeBgBfBr#0GgTF_*M6*gckR-Hz{%vUuzYB5@i|Z?I>+KYzXwdj0_i{=F@BTH$bIF6@ z{!$_9I@qnr5|P^)6eGL@!-uocKdGSHiUE~rbh9aboC}z4-ZD6AmOhaSu%9{tKeX#5 z`%TTB5doRp@ey|WnyU9Y`4sQI?Dhr7f7!MCggD(KaELrZOrfw7&EwtoJ#AA1o8`bZFLCyS4*({a0M;%lZi_cNS-f7-)8-AppzL57(XEfl14LzIM#3vDrH zjzc_i-##WVRDYc)ghS_$I^0GO$*7u+b53T-b!8aZ>_9lU~Gdc^TY7a^^=gayWm34iEY-O7de zp_}Kj)sI5NntoQAjq$`^IXxK1J1J5SG8#f^)9Hq7QWu|{-*WzO!I77eiK<{Eunoz3>=yW`) zQzldA`G{GiCxR-ygFQv+!vmogCa}>sH_T$zxmbQV=wx3x>E%&&hV^pJkHQR(!jymw z*F0PcgMvidi1q}XBJ1F#)Hj!$7+z18(_r9Oq^6-oxFy6&Ds$J1F6gelIR@v49^M0E1+jKOL}d zt#s~%ApRojad`-j4XuHi(s+>j*BYO|D|`A(Knl%a{nzR)FLVLRw6KWZhla7~eTP&WU2&pD zcS?Wv#plnnJ&B)nRTxlS`O)EF(Opo918Fr`hA*V)u+TBx0j#VN{aQ45>p@%(*g_4H zn`LCCgKz2;Yb&q`&@JgbTCzSECnMoL8s&eh)kL*p%lO4~;PPrKMH?~TV7z&Ce&v~I zPPMw%-ADJtnO{8n^V>s7a;&U=agOXigMm!SE3R0e+$`CKzxZ&9lcQ8V0Vl6`NJ6XD z8e+qm1*Cr|kndUN+}R&Hi)s;4zi*p44EX5utDyuI1{;f&1h3LvmhcK7NY@o^I71;c zxnFDgCbc8PRUIfY;Ii0I$8O#|gOB*O2o5l2G(^O|8tA?ANGP~s1IEmC$}ipK2)7YN zR)aS0%RrY$b?=*j4mcc!5@m}RjIz+Iu+_uE!7vmjb(@j`WOP@F8*GG@e=Mc7`;Jd~ zK$xr*M9cU^fg8r0>;qLz7_Xsf36=&AFIMzw_-u*l$eEEq`9A-h$-|Pf>;hdwEK3$5 z#re*X>$R@)nj7%IHD1!k*iy>zWWOsXeM)i^#QrB&a_g}~)awcl)l$op8@`sB6u$%s zm5T9UlrdZym-{=m&=j2Tzt|uk*4d@iSo>DmtSj;9pEr_ro+2)^29*d*p3x;3+zSOY zdD(}}i3mh=`2xTEpr!Q>No5M^wzz4Nr*|6bpv&;Vv&ku{(YKvupI(Uxc$IX(v>?av z4nJ!;MG{REu{9CSo=OI0lMq_^8=AbLU{NCbB5d(raz$A$UdE>P9>~S3#(gibUmTfT zOD9Y*Ib}A%vu#{kii?S3Zh1Rk=_D4UpE-RuI+-F|Rf|_CXZMRn`8AH~f1{u-JjB9% zgE!BWWzbA|pU)O`Ju5}XPdy*aB3f)0oNz|NU^w9I$Q7vXLob&U>2*RF;!PPbSX;_S zX;zVlOhEYYUeDKMDy(~@O!E*6>5hYVv{4LqKG8FqNVcVm6{3jbrRO49yyryUz);@4 zFd-!IGWEU99QC$wMc+G?a!xUAAr#~h0WjPHeggmjUG4@&T6acG6M|J>42~pYef+t? zUsAxy&xadGL61@CA_cRs|MqGK^pUrtl+b9Rn(LojX{GmazJB8AKNt`B8gWnk^6lLy z5h$w&r4)xExMe4ItZ6ed3S)}1i4OdbUMgX(laA0hYAs++0oOuu$v(GM1#$1A#;qU+ zqzL1C-+KuQ9=`f9QLWB|^r)B8ix9BS322J+FSnvOPDfB@{&6dw&n%Hk@Ugo8pn|`X zI=0-Jj%2QgfMuij`Y%{=o0Rx!c!Vp9Vea&$sR(XBc_`Z}J;bHcu>?mx{xluPNWPtV zWS9(a?%%Z}pw0U#VI6QLs!9|7@$+P6B~NmDnugDPekAx_##F$=geGTPzgR8g)tJY} z#DK8I3sHvv#4FGhkZ-#QQ~j7^U$f*;DHQ-A2{fXbszbC8`I+sbM;*KS1S-% zk+E2Lw+kg(E7NeK2_k5mI?_kH?_iwnhV^Qs%d*zXt!0-pFW@LLtxjLrS#mT&V!6A` z=^R%g@*(KTDXJp_iT}J@AdacXvPKy6pT8g^2M>u>DLZG;d#SukNoK7W>|J2)cW5@G z6^dg$D}r?B=qnL*dzFV5VA7bsH6%a-ZGda3+~OSBQ+XSaSX51ssQO}ih6pyz_V8$Y zAkBKYxfFk!*~9f>QDE_llHi`7n-;i-m>G5yg!$k@gdGyLkYtnoT&}xlF1bm zf6gaQ8Iuq&7$BNLDICGQZ9V)1cW71^Vu;@d7)15I1uYbn4dC4aojhGy?tBu>-f@!+ z{g!Z1yqCW`sLt%)6of06-J8G3{F_J|oYP{mmTt#^x z{zs*Xp+4SKb33&p4_|>a85S?Z7FzlfnEM^AnCZttOhS@YCU>MNsE!IieYaGD&>nS;7So^5xK| z0*sJlxST&uXx>e-514JN!%Sd3X7{+5iXRoctmIhLv$-svl`tDEG~X?7zmW=!&w0KI zyajrIbp{klhHqR>|5?9`ju$-*gN8x`CDNR}k_yr1x{-4zHvVIW= zCRwkpSYN@xiT3f-okC=^E!}7S6^e=^Hipqq21(kP@t+JAC$AV6+s9H~Ld-r~Fque* z0d4`cQHf;C?5LRsQ|6 zfmCzOii4eUy0o(u@6nCtC{0S7+ku_`0F+JE=viqoE~c6Z5)wlcQ9IGRUUfz6?~7%{ z-x|eMLR9aV#yPe9N}T&eMBC@D(^6oD$o`aL#!Zj6?Clc`spmSQ-Jl3f2+mNm`C-yt zO$CXDaAJ{^J9%s(`zT7x0$X4V3pZY_W2lPIfeC9t(u-R2sS3?Vog(@cYC&&VhRKFY(Di0K*OVc}-(8R7jN)lfoFeKvZz;tnP{KNJa z-`TD-;~aakzr-ONMdpl8{yPig@u6v3|9nCc`_2r- z!~NTO4E0xqQN8>UGq(yx|DIoY22ihbVr>rnx7Rax&kG{6S_fOk)t_oqa9Fd7Ruimu zpD#`cI<}1YdHl))nlJiMy)-Z0fL?zQEzdNtJsmZ@|X!dfax zym8{fsKcRBUR?g&VXFGb7}w{Gf~GmM9Z1;Ll)bW^@oO>QKN@-YyclH6zV=HeL)oSKz)WzbF|gl1KlXdq;^=PB ztVFZaCs@NRjxi@`pXw=p=#PV}&ATEa#D9ww8{4U312qQl@&6eL0#8bgx5bdf|9u(D z6Bv56 zZKsp=RO{iH+YWlA{1&0;SxX520n0_7JYI9(YT9}?E{@UbjF89(z`?R5fFX!mCt>sl z99EIyRyObEUz|b47Vg!NGSwM&HC#T9r#69$QJdXob8PhwDy2dQhAmd)t+m_J zzSK-D`=v+KtoQK+)k-Rr;t27oYMR@H=+1i8#zM*m_Z1>`;1mdlAdw3Z*8cS#{Zx}= zultO%Zqv2Rh}Fol8t`~&X&hPc0S*F7|QN5mQybL)N1dw)+d=DO_5Ne17sto2v*(Z&-yjuf}Uq}~pA5rp%Kx9%h_BN#OZgWL!L zOk`;L>jp*R^y2*^jG$-9zsS#dmS`{ehZP6?g3f+t94FG8rt~Z6p1Vpa$tT8AzY|yY5XX zt?P5Y?+WL5I5O@JQV-oi&w(Dx+~(Lzcu(O}6O#VDL2~rH!GE`67B-M`1TsfgYbZ2= zaX^DrQRXd|c-H*bT{+PzWcmihxGWeWAllm_-@CB2DOK#rUv`C9UEYU>`9g(Nd|P8Y zb)aaHcV63+5O=qgYwu0~82X$$Q`8$XbJs*0m=XAmvlyg zSZ(4$j$v~3PpAuJ9gth&Dg@=b0)ejy&mnCEb3h{YcfkyA!?VOpcctB zS;y*>f}W6AH&m*A%{BRmMhn5LkCgRJR>tkS2tmmh{|zsD{%3V~b4#e@mTSXb{rbxt zX!@LMA`df@>+5c_p86s%L!mPzkt<_At>U?OA_t?g+TcjnW192wJTB^59f=CJdXa*N zphN*z`dE3Db5lQ`$f?{oH{+Kpaly2W>OK1-gvcy)Izn!#gO$*N%j9`gh9Es^X$}7a zMNY}WnM9)fBZ9TxF|0Mg<>!aXQ#mJHV*?w?`>$PPifW=%tSElG$<(yFbRnyLenbw2 zD^HO>WB9^M2Wh9jVCGbjajukgbxVdFt1dYm7ej4a^|q7SXSP2dvT5BMrTI?;J3Ue? z_**;JVC0!+)*n~qxFOWA3vx21tZWTif$41Tl-4wS+=>`Vu|bzmb!Ix-$Y`RmR2L@% zxzAI|OI59z-ZOLzk3t!j3i-##KU1w1A)`;!=%Khw579!C>z!1GuTZUTD%TKZMfN$o zG!Fu2qXiN<4$Z)=tuXgx#qx5_!n1PkKTssMF){cq?LG>7oo6W>^J@V@XeD~&(mm1* z;%cLAIaTm-6DT%b;E(J%Ve{I<6vxVkuA5{Mpasd5m0P?!3pbUO^29vqqX9fN@)`Njm=9TCVr1pyP&oC$e_m z>vD95H&*Df4g4ZgtSx5E;kttw(M{;qz(s0KM`nU^_zePDncl=Xv)EKU%PKtkWAtnZ zQIU1$5|<-wY9Mu5syfvGbU%Zt&71q(3(CsC%rM^t>Hf*Y`I0Jz4B^kl7cLyxaMB!X zxpA-dmV)6=Bp@TPTVzDhYv@Fz#Ks;0UR%X*q6~X83ix+j0kth zh;+)8mc$|C2A9~5jq%`8`6_Q!+O8mmvtd7ZMyFd9+diqe(bQa|(=4t_+Bo~kU4D*I zPjXTD#YY9s=wJjJsi0$<00YQLMe`%%tu@_ff0Yi?tJFa9`=mO{6`=Nr!FSR&W)b4^ z+;`+H`6{Teh4nl!2=>CSQiPfp$#!x#Lxug44L z7v!K4n2IXkfxKYjVkVUF)Ax*#(?=hwBml1eDGpt6lMDDO#O-s(bSzTv$)7!#^!phB5x*aImx}&KA9BLHaE&K1N6rw zIJ-Kj`sY{y6!v96B{T-54u~ljWPf<@)r=iu@kcE)Le`5gp%cG^dcLZ1we|U(rJFiy zc;Rlg$DS9gn1S~SQFGy=ElTRXx)O~byu-VZ%!K3-k`&S;WX5+<3lrhr2^W6v!x&x8 z&+7W7$AKluRR#qufOUl1(RG~fxXbs8%7^Qt>-g&kyzxc^3ZZpFR_u&1c?UMz3>R-) z!afURTOhHpo%?;t6#xX*mlFBjSVBTwQk>YRjj8;JOF~O@TQJb~IFxhA?m-u^;+)Q# z%{9@jUYe&CK#AS@c#OYyp~aSkAUvY7U6`G4CjO#={9%wCW%|DUdGy*wtnJX?0iR42 zR7|h`)*^mi^14Eakhx@T6(9eC8ZdIHej&JbqCz^7ZCYKV`W&vdD&&*fW3ojT#camJ z1adnR79=Q;$Cp-UF)i-*#7~u==t=B9JB?bN5`^3+ zB>#SN;-Det)wJvHr)1ZGjppdhM9$gs;~QJa-eKcrgefi~jImsU?_S&Jcv%4LC_rK- z8QT`FVf*d-$v}Ts=MsmAdFHh@-na?fGpU6VcOo40M>-`h)3g>;>RuW0M zTBEG}$pnh|%zc>?7Ou_b@k8Ah^Pa$t{T(l{n%sMatkJE!a^(C0OI?G`Sv&QB0;7qq zJ(CoA=Dn`~-e5gRF^&U(@E}hnWy@U!*B^M)r8u6=-#p0^C*>;&yf=$MJkx%I!c-ZW zD)`m(@XQF@%e0!x5k9GH%7xJoy&A3~Z7PN!E)HB+ukG2+b&m~S9E904I`%hg zEU_&A`kK9Pwezl(?MPV+T=2)Qd&-JGZ4Z;Mntu`Smmp|m-zT2HIwdfscL&>hR3n0N zdTHZ-N-5nNp1bcT#=L_CTGOxVfd9K^NQE5We|c9ePn#k0=;)*=>koO0+nLtA9+C;6 znY_D}$n?h!0^i=EAioHY%B94a5qiGiI}~OJVw3N+vuj3NCIn2` zN+d8$k0qIEYnh;v&`qqk54_Gt`cb}~x*+_)sosdkC0y*K(3x#Ha{Q{!`Tk*P=o|cUhFR&H7K+A*5g&9QqFM(TKo&+)l>IB0kZ7__P59M&rEe|4`!&j_W3Od1@jePW zARxLlEl!EtSquf7u zHz3_bHdIzF@W=Me5;DEiS@`41?9&M5zZQF(F1#51a%$$Hd~@J8yyF{(b&MYCm4LEI zjuPPR-t)>TDK7aiE75B&;+UEGuHBK7(tKC^^2K=T#(Xkgh*;+Jwztrr@0`LWbxUav7dc$5AtpgV2Z)J0hS}b% zM2(InIXoPJMHd<_L*)xiRRiHYs!des2-R59B`liz$SyWS2U%ZVo){=GpTfk_@W2gG z1SQ3BJ_7UCfX$V)Pw&GICFPEKe>}FRDA-Y)ShW<)$GgmKpN=aGuym(%F^uFwlL?fZ zPqe7G_o?E^Cqfu5j}XVZsdSI)#PP1)K>SQb0Y>Q+0Wrt5A|)M^yOKK2#j$4iGf9o6 zZ)en{t#6KC*_6>kH%sH_!!EPw;IFb10*5j9EIjiqz0!UcuLGseeq=J*oB25vk_b%{dp8# z-EuVw+*XulkjH#-lymv|_esn4_gl)iux4)Ll`(FJ;1>S=;H3M@ErfQ;QYf>Z(9^co zr#hNVRLsJe8sQ7F!kWST1uDrz%$G7H+C)7HDdK#JC4-pSgj3&=iSBnh=h;->ag^Uy z+Fe{`%MgCHySR98u-_LUDPuQUCyM2R-eo5?qD{+usnpV5J0J)RX8) zT2ku=5Pq0&OcL(Q4hmf_Q0U~|3b)Gs^s=6;%2gEksm55tXNlCE8U6w;ma3!{6=JB^FjA>D46 zF$`Z&bhUS*#9&#z!8w$?%|q?&;zWONe}z__Jxry^Hjo;R<={nf&8bz#_mSRgpU<>@ z(}9&5FMQ;d?#cChsg5vuU_?959A7-{e%=xv8gNAv^uRwU1byvWx)Xb2yS3ns!QZ@- z&t_cF-RP-Hdnps3VO+;3m4-x|U*B4MyKr82UFJb?KZ$*r+Lv1-nI{gvfoo|@aNVi^ z=J1ZwL0gP>BZk{vL+6QQ0fGU;=%9ulc~?<7N6@r3>&5UPx|N>G@PCDa3}Uflz%fCBe(HU)^0SD(ma%PZo9DjB&OQF0 zNA}ObFna%D!f2#LC&PcYAE=Ns&Ofe7Zrn)!&9r(Y$ld<#iX(Xp){2%L^020Hdr~xO zg4hdKgJbp3n2?O5-nZj!{NpdUR!s9yoyFWeX2GK38kdrm;7lcT$qt6OfsX>Ep`@a$ zNQnrDmJ&n+B_#wlT0lxV1SJFs2^l2Vtr zKhOQVc;4^Dw%_;nz0P&cxz06g*+{KLM-$yDv>5{#V5C__OL7gGF>eGSW0d299Amer zsP$z+Ndz85$veh_Y~KI^8GHb#7O@*#e>_d!!k88-06cKgQp}y*z($}GJmPb5FTY?x z-e+xmrYsfSkAgu|Tf(spB=2P3)9^-AdXYOj56JwyXH*$c*}i2x>gkKe4if;pu544k z!(%HK8m07a;G!8!arbk}qbR-&f971QJ5+q~5{2Trfwd(evVDBAk2brPqYjlEp78A`R?s8dEB_FrVQ69qaEd;)Q;T zR*&puwV%ab2Dn4PA$u(n&fi*sv<+EO_Iv%mgQ>aGIe`q*v7_7Y(7lvQVW`DG=q<`^ zqMB!0IS~C@bPfj&x7Kh(vW7*tQQ2CY?f~2x)XSNkDS+N|UUd;ti37cEb)%qjHz}WW zZsG$G6B8Jd6+U_Aq4T%p!7Y%x_rKtv*ls^92h-;+xliGSp}yk!(Z6Qatz*}j>Fk7V zb0fYmL1baJU6LZiWF1Y9-G1pD3qP-9p5+d58@A-Gu(X6iGm^Y#-VMb=4p-*6$KPp% z;UoSn`6M+>vA{1F3iIiG=@+E&A{cnKdJXFWejb5c!w@_TlTq^bAkw_8)*pY+D?8Yt z-?V)sP?YX&5$H=NFwX$|{f_~txqY}eOj@-zdh>POQ~KCjtMTrCFby#+TKy^0`!L0` zyjFY^TYn1vQ7Wvq@Wa0`2?UJIXiiv(y{~|)M~_r>S=#XbuxtB3$uQi5fnb4bVcZBU z5V+@QiP3O^edUhVnN*1cy)$Z z3q&lYPwYCuw0rA$B)SwcijPZEcVjUz4CFqUm%pVG$gpkd80CL$QAXCTz(2@7fAMW@ z6n0OWfY!>xKi^uL-+cYEXnhi_RQxMOuO_Jj`lra$^;w3VYt~)DU%F;0d*9Yso+26_ z3TMA!?_(u>2Y3*LwKh4u|C4d)XS@lynu2X|c8M8&EkV8geleVHyu!E}WQ(_j=>JV2 zr*pNaL{P4Fd8Ag(()*#N z5ELga4SHDtyT$E3c#nDvve1osJid1Yda4PZ{k)s`{j9-FB=ZX(1@!dA$47Gs9F;AT zfo?)*Ac5@es;nMr|`t8{;lg^G3w^^k&V<{c*FgcB=5E#^S+}KD-L`AS-Sn77Z4(5*L5dm7+gi1pZUb@woDy?P^}Nu?QMq(BZrW>(1mZt%s~6dvoTg=6#}(j zWinU2@S7E=N#pQXP1Moh4k#H0u5m2mcdny~hg7cNO`sxyxm;V!LuQ+Hh&s z3!m=s#-9Hy7gEjG6f(<{GR`r%PkI1<*>fhlI$4-_&_lQZZZK%EK$}IA-+irCL#|^@ zoq|JZ2QV;#r;13KEKkd^E2R6`ujzou3X{v7 zMAz$jlNnGj3qq`EP5qDtwSm*$wR_PB~9!t;W-htKycEEl8(vPmr_gjmGPFVb671SS-<7d z6U!hH-z}xEt>^>)}%I70iIXHZ(daN0lhV|)HYgNx2AI6TviS7%cJT|<_Hm5jEUV@l42<2WJ}SOD#h_Jaj#HSuV;gG9Nh4Xga-Zr zf-v*oSC~~fNlPbTyMep-c!Ld@fFlf#tYCGoYi?I{vN0g; zS*P7=qu;-?kpdgc=8dC!@Z;zHV1#iwtg)8{_j;_F3`x#T*OB2%6d4=KsVot~BSQ9T zfIF)LviQY7(8&Gb#TLO<1Z<071<4>tIso^pmK=a8BKtjiDHH#4;_r^`73lsK?==&e&;Ss6n$H|WMQ82% zJ>35n)cw&(pJV1s@h^5J)*+kcqSEMTMS`YQ3ba0VltkJkBTM*I+3g)v{i-Unm0*Z? z?kq&n51e;D@j71pc7SBm6CfRE^>17xKGz?jh9ojUC04i`@Fgrf)OZ62QlmESPh-|R z8>>jIViiBF+;2Lw$x)aPFe&4g3lOy)qVtrvUw0HW^xM6!<~=LAqygGL;C2t?);$ce znl8+4`nR!ug|`STeWkFoU+WQWg(JrXBa$i_??2$IAWI((g@^}WHRc&B=Cu>GvkBgU zV%I;mT5(ShPim(L^okG<;sW#%L|i=7O)0LVdL1YdxmSsF%@eVvOmCD2*C}42r(C79 z5&CyM;S^>Jgsw7k=GF`|X3w1U_h_MF4(zg10O2A+FlJkiggJUgYV?!wdLG#Pl(o^; zv9kj?KDhwS@l;Z4K`h0I5Cr_(v27w-V2?06yj#D}GrrS+!TP3bVdl-` z(sV3>?y{$@k`hg*rgpZXA6RK_{izsMsCk}eHs6k}uK83s&enQFxvN)hVPb zNV5ql4EBQ~30+v#P$o)qW-%r>0ketl!dFwMT>09%(?T89NE=Uvaof)n<;r znzZ@4AW2>O##j)f$W)r@B7+s{|8}RvD&Nk-5mX2rToFXAu5DkmAhR)6X(L+9hv#|mMMY}aYj%NGn45kOr^k;>%|f_M^*TdV|c0l2G*>D1YJ z8qN^B2I777+u1UYP}`5|KCrJ_LF?>?p51xmQ06_|w7VC3QT#ybru%`!lvjMWB`+#( zvFa!Wx(R>7uwdv@_Hk|oqBsv0GFgXXy0<;-4?=5=L)VK92!96Y*<13v|7wFDCt}nl zi^^c3ydnkA!sxJ^FF{{4?vHS^nmE1ypHEklT}UbNvKt=eh`+_<2UjJv@+GzfsSLD1 z&-jog!WFbsPuZinW6#!~Bk{!s(G>ODZT|E2oP3>14m9iNa@B1n%ApH=G`ZKeakO}A zU%SOBL*zcXU6^x_g1E)%;Jn(91{i1eV{Y$fFP%N-MV|bd|K2QLba6As2klk|H4^t& z@_10@tC+H(^Kx+Bixx6vzb6xdS4+#vPyF27ClarY#XifceqYX*)_v8DVP*1`0}Up1 z&)$%VBX06rty38+bAQe80MOHHt^o{Hv~>kQdzAQ#QdzZbC$a(^a_3)~mT~+qLE^;v zuba%C9?06s+~Js+6yTt2<^?U2{l(xH@=Ujk5oO-ng%2@Gw(rwD1I&n9rICXLT9m{K zU)(F_mgE3?pnP6y!!Jw29kaL~BFPvyw19hI#9!vY;Q_f8 zqP4v+P;azNDL6KWp@ww}pb!WeYoJ~*i)ed6RYy9D(SA!9TUX!*qLs88!mpnTA_xw|naIl{qa zDl*X-T4m-x$9I3x!5vlptd+=5wRexA{v=~f zL26E8HB9FLv2-*7C5!6Q$B|w_Xk%CEyz==R#o~kDxzFD9d>|Wd`Em&$7y6BoXOrXBQs-zB?5oIwBftarmZ`NqG)i|BeM;gJhxj8ua~BBgrRB zEF4~p;oL;m>;Sl((nYc(ma9jo(;0X~`n&m<2c2aV<3VEJx9HT$H2}$%S9jTKtjj0J z#(>N*GEb+X-U5#}utLpundy^h=tW}T$4YLLxrt3>!Z8k}@C{!)I)T=BMK726Dk9{d zMgDC^;@<-EvZn}Cl7gcoR-#{R$N7Pt+`}ennUSdo*?(yqZN~Qo_>)jHLB=D3oDjda z{-yO}l8BTQmdvzj7@`OQy)!$V23>HQK3f9mt&VaXBsd)U4%jqOv(LLO{K zs;jEsR+zT#*P0#SMXEZD#1~O0C0|YaWC_};ID*OpPfjZ2jKL(hK*1^+{!VB}-3y0Ga|ai{baFZ0dV{fH;# zLDa9AymOrytPYU)vI7p-kpb;;5d?-m854(Flldf3xM^>yvqcAvuRi*0UvguXz1SrQslBNrS#Uc8`0d*`z zxeRVJRxr|tVe3~chrRazmqnIV zgxLoLh9WX7v{2Aq6HGD0(2GIb{>&-^PJa`^7%pDfmqjmqkv}f~&O+}huORi$j82Y( zeT}rd-95!y5iL#{k+8k0Zik#y^{_@TgXqE4ZX1~I1y z`#k%Y+$Ut;nbq=^<9D`1X9$HmyqfOWswJHW*C@4HjH|@8Y_HB}2pE^+Qi5+yAARv| zF$t=DUYJ-J*+;!S@^!g%sp!8I2+jRJ*jDlLha?4sr}XGP5F%G$neBEB^sy?tyr#q%Z0XQ4DE}9=k?k7Z-cwyj7o`pWT)C% ztfwc5tGaT{#O!LO-{CPqyH$&o)W<=<0_$!>R#%kJz~Jw>c5c*XNX`$UHx0NXuF;J{ z;~Rg1O6QXpuWWB{3Mm0)N`Ai@3!(F7_Ks>xuy}-l7TnsDSqKA*c460SC=mwGl#)T9 zbTn}0sf;8pD#aWwXvs}f3QbQ|a-kW(#0r>Lt|b|F@Npq;fFi1=>29?1KJA1O(VzSy z@n8WVWEC^LffuT3H6lJ(Y%al+bF9Qj$>M1pEGiKDT}zK@tH2I|XyR!>tgxEoSG&?9 zdd6h^HPyzyiU)IrdVCFzg3OBCOmPiIlZ;qN`@XBjKLYkApfqf577x}*WH)n}_5FW3 zpT~#1)K-v%(|%5=EH)z`0SU(H z3!i0n)Wwsm(8XG==dY))M^rd#acZVhi<7;vd@;3U*>AC)k4ry#L-sQQ_Acn)*{}4D zdZPms1gUm0(Jr_XY!*R0JN5WOuCeVs6X7vx=vevjhrP!mBoYdY#T{m)M5RGlNmx!@ z)-9a+X7C98AIQTCHA_X@=k>ZpfTaa}g&?@tHMhZvlzhaovi zpzS7HsxDW}4S&CRxR6+F-^(PEQpYdEJJS0Yry1tAcz^&W+r%jK!-elqtb#m&&Y@QP z`mwr8KkqZ}%HgO$2+x)*9pBRQr>nmPbza!dRc$Qj(e=?wN!;%)CN+Jr{N9^G(YL;o zFb=($l^eSFp<6oJ?d+}QpRtj%7<44jGw#JCEYnU)onkYU9|hy3-~dd)uAMx{%xKI>V^G$pzRoKu8hEo`VrLS}$y=+i|edKr>=Z}Wuj zX4a#b4>lC}hABo>@3QcYj{^-2B z;WnNY`-h8|le(*S)*jZb>i-|S-1+ryfgpcPuM+O@a%S;bd)K4$#j&v>nu{H~9N&nJ zF5U|+S0MI!&iWa*q&#Tfb+r0dNy7um`>8Wp2xr1?0!q)prbr^e_%5x_ zRAV@6jUGBbU(qrxRIP>&q!@CjqItHV1!x^?f z14swT6$#MC?moS#z&7(Q{z<}iKcxU+0Mclwej-1C8hAh|?>-khiJ~}PhNc0B@V)5A z3aJJh&X(5GLW{(*LT`I9IxQJ8a<~MDWj=BFVH})V%{ptMo~#!$Q@PV~5}0m}#)swn zt=s}^si~=7vEHXgR6Ff3MciogRNG47nH*0w2isW0rKq8g9HnZ;1L{l-wXVNrp|!{Q z`8zG62D^$2iQjE=l9$^6?=c?|F^^AqLV0r7lIZNv($6JUHe%rwVE6TxTL8jRG8Yb7 z!3j;hI-!W@+gWb`>nhstPyzE;gt5y;|6c-R<+>_LS6S*6x+stj1Ygf}lqY zB>_(#FvNF<(nOX0-7!(NwRcW8uIj6|?IU`@oVIbRg9cB`(Ha5Y1FI%V^{5DIVLl}k z@nmL;7ID)r001?BzLvUW0sfp}hJH8GS@Fw(tN)DJwq^SF8nagtNIbM4{4*>oo`&p4#JtBg?AH=iw9*nU34ZPH6#`?jT z#u6a#)+TYxN*pg^&0C(-;-sWY>S!S&rr*i?-adl%!9nSQ#q%Ql?uCgMp;x^~moQ|5 zL7l{6I7}#5?Oipp<0OGYK)Yy!}G6q0vHNx`?w)iVt!>oW@5fPcW{)T)yuC@9H0C_3(?+G z!0YcEO3V)k;=7+fC7%+#xoFAbeNthUSI+uohXR*eV?1j2xQF+OV1B@e638foy%u6& zFR9x`A0Fhga}!8x(nCoS=SHsR7%gwX(G!x8K?NEU%DbC9!hkJ}0Vv!0uf#@mNF2%v zL}(V<`V#?<5rIdy@^O$8I-)qzzeZBT-ueO;RvgUGI|B}(MLjh=h~6l(lej^{+&eeH zm3D4ZOP)w2KSESh@d5zPV{bHo9H)C-@r7{^Rn_*3okXb@(;K1|2j_iD$oF#aYF%*T zI|X`68^Rx&=<(y)e*!R28Uj~G64^wA zkaa7u@7O(xj;OhEq_7s41rBp~L)J56;*~fwj(DP|LtPgi zK0KHeyMt?yYC8L$6D1A~Kbk+a08s_i^2lfE(I^@^G(9@}P#-C3opZTTr|*b-Otq8k zET+%+s@tn7ao#1qn;Y6Kfx0}=VG=1NqY!Q&i*DcPsV=&{TpGAZ{7}o@>-tTY5?#-O zdaQDOsH)+Q${UL(bD!5R6Q;e25-{TY$)#0>HPRSnu6bd?Sve1 z1pE=Q+GM#m!bnyrOIbub^izZN>wLQFmrwYf9fiRN>RnL?%&Iwv`o5`-lNE>9POr!p z^}?2Y@dt_}7VfKrz;}-izf(q!W7nd{u<0kS#!BJ682Dm`{V4}I(R<&2G>9=RaO#D8 z-DPXmtzR35yJyxx?_s>I7rG1t8;QAp^KLWLS}de(%W<8Am{|^ZE55h9U`xV$Y=B3N zS4*m-a|Yf{;|+@8=t${#%ZmZ_{3=cUk;EQy!YnS4MwgaYy=S?RjQL0|Q#0Y1Jj9b9 z1h=td=o!ZJv(Y8CqTCbxA5L_qi|$cGTvM%NJ{=_hr%Xt zV|XHc`VJbK>0C$8bqXTlFJeA=mFNlHNeC(s4)_RQZ-B3+M<{u&HpXYWWa5n9L#Rdm zUnSK6oRZRAikCs!qx8X5@%ceJKgJU%U+JlW_gbGCDx>vBd%S=}^DpAN0iaVNV<*6C zm}xXIBq99C#_pc-je^1c_u-q-sI}@(5XA=YSFaWCO~y5*h>h^yS?r&bk+wYpck611 zx1?SM;~oGU8JOJC@5fUij)jc&_3Ug-*7!!#1K&~wFjxTH0$7Kt`-fG&Yt3?obzrmJp`mbSV zgh;A`Uy#<)lP<6V+i50>a-LQ`aJKOKVkr)e?UGBX7@>ZF?l=m;^ezJe**cI#Db8Wh zE?}SG1F)wjnVKT=I>b^cCk_GHgv>>!adc4FeWjVB-Gs#sr;9_x5fo3}?3NR_pi+iL z%TQP^HN|DBL*h`Ie}EoxS`CA=sf)vVC3+5iDrNZN+r2RJ;PAirx#xv%EV53KPsrbmKKuJ{9anOfK4{-z=bz444~1grUOlzn znu*>Y*ijE%K|l&3M8TVWsx+N{-relkNnT35nYP9?>kFg;V&Sy~_Ed%K;ms#x?AW>% z-ZWC{M8|Hb@k(yJgiBd-cgbhH7zz!MZ}F%)jmK&pm6Oa1AV0YN#9il7QXgJ-2SQJR zE#xtGbm%j!<3`GWX=Co1`TLy(K{RE8at;Sp)pK*j!=xi=FK(|U%PTeYhsPc|mR z*JJbMVyy(%W11J}L3m8AE-mkst1in-K>YC~)6Q}vE?meH;3|9lCcbVMudFfeJBg-v z+A)K6c0XAU*$chwRg5kA=`--_NnUwH33bR@N5&!36X(J$B#sq8C7?neP6VYYO$N}u zTQ~fX(4D*HmIpzmqU%Nd+m+^pN;R~oG(-h47+ysk^h@wN2)naVx6b_MW}KpA%*m^7 zLKi+H==xiURm~ZLt^}UQ=Jdfbbkv;A3)Qy7lUvzjk~V6l6o18fF%~l5mmx7u9yN>z zZpj~s`vAW7X}rG11V5GvO@x zHH1*wep%Azt6?Dxlk%H)iE83M2?c4hh~!O>?rboxJhe_bIpyJs;}eXXOafNtRbIMq z?L4A~H@WU8I9r<9Tc}b%;sUS zD^whQI_nOxS)!q8)hCE7h*lL3B)`rXi_#s_d2~S)3!i(gDWFFrlm_Qa=+FpILH!gQ z`>QIvmZriwWnW0obUJT;_4_Es{h`PI5UzFP=-Hdq@#vk%Yxnl~z4t3SH{tF$`kP~t zaT3*w>qEk!U*(4nubic-KIQb{+=9pYW zy|uu@$gr)O565CcW}QzFcpfH069B|mjfv?jw}w}qOtnDZD?A4!qj9->)amj%=3^ja zxOyl=y~7q<0uvup(~H~SZFv>+O~Ax1L=iE)TH3r6yAOL?!f@`jWgI3$K0Y&fOVNsw z)~b4?A3IRE>9iJ|sY@l-4%Ku}28XY0+}_D~Dk1E5kO&WYADEc(cZrC$1-WDwzQPbD zS}~pc>^Fxm$Wqo-z2V{_PgV`je@N3sn>Gxnvd{{xaejOXYBAk-cLU!-3N;sNG9*GM zF#Q8$y5@!AhmPcfvFCNc;&Px*v~?nZb*wS2kb5f^Ag|L}6gGOi0`xKbJ*87P=vIMaifS9X`w_` z=&c3PsMog_ECA~9eDl`nCutVY*?Un;BBL; z&7c9+`QcmNbaoSi5@Uo5fAhH~FO78Hq*T9UlMQ$;|F6RUa~KEh5n=z8!o0oO|=DwjulZ;a2#fKdCM_X~YzOb>7Vdg>Z;15s6v z&wjo7IWBJR(?QW=`TfTTgi&K3(NX6C#QLj{GQ}dJ)Z19%$%GrDMHm)XAqdCc0{L-? zUgYEwa&3sF$wqjLH^aPb=xbm3ISroIwlhBMjZs&5SmxNNuTjk&(s8P&2L`;{xN=(A z`f0uNR992gl`P|3lJ|3NKvC_%W|F5-#QlbPaw=b0pt-(U{$ihX{m;h;2UuLhA|=ob zS8s#RsN^xgqgV%PeEm#klbofGqVVI#c?5HSapF-EEYdPDnU4q>V)4|&Kwmv9c1I0- zGO(thnnfP%yZ}xcg(@Qab!^zu2%9)c{c8O6%su{ewpIwB>jlhv8Xw~1AuY1v!yCQ6 zfJ(q8^?GzKW^%}ty%v;+cb3I>n}UpsC+?~r+`=T)hTr$fv&>NXX19^3)f2kv62aKb zvaF0piPvTCzT{qk;Ha2Yr`HBFsbZ72r$)$jyl@`BP*2vgA4OA*^(igDqa-d+SsD;h z?*NaX1A4fbGFRf1atqsl1*d%}8$7EbN&Lt$UG2d}fKJt~m426G$lm+@{G=cmqM~s& zZ#}LL(;65L@ShSl3U!7&;{ao%mI{d!oURyiBqK_!;gY3hJ?)|7$S?&szciZye8;zb zUymKaY<6wT3I1flg7litykiz;o!HGY6m9G-BdSs^(j>0Wvxn}Kjk^C=tTKe0{%krZ z2X`n&cWJkz}aKGGp z^h6uMyuVEHtY%rM>F`FmAusN||N9cF$eD8i^C0Eq+D|52sS9;jMN)8g^>$&l@zBlY zro@oy%9TJUw{b$@icp2)8={vh1oZ8})z8c4|J0?H|ENo!q6!49Kt4Njd=i(j?o_|M zZi#xAw2!;@B3@5Ma~30@1y7xl^HF%m2z-= zfT5N=uFDE95z(baNi;WDMkid9;7HzMYWk^z>zAYN!G&xVBtfEz@)U0lj9qx1!7#P)MB;JM|UzcrTj66$OvNL()#-qi&A1OYb-7K-!Gxy zI^vvUJ9rP*R_M79yNNyUz7?kyYj+5bg0*$ic6{*no@yIVq*c?y|G}02qpF`I;Q3>T zy}j-WWiMJF)-k*Bf9*LYjCr{q`{`bPWv>zdRT<&cq-X%4@abE@v5k%OA(~D-OZgd7g|DIepF1lnEI_oElRWevthX*{=BhLVUP( zF7*8AeGQ-4!&YOKZ`H*!OTumkwnx^#>wh0{PgKMqyTEQg^3s#2$F3BzcP1It#5NW& z&?mh-Mr>(e z$=gk)22*_TMX-?vDhvJNkMr`WR(-4%?wYl_RTK45i5A}hg%BL!b{0r=Bu0{uf^llO+txV%o3I!jSYpwCFpR={A z69~7tZv|8)eRy($N4zy)1@jgMv_x+;eMh=?&{0-zW=aF)YAG&t{n>}Do3E93wc)US z!AMIp20ce{s&dN4=2v_WAKI;(AN2(Zi_^F6856z*;JvzqK!`B$z)CQou@z~{nsjKX z#EAc3Q#j$9#)NLJfod;uKR7ixmHNK?4B%<+Z)k!ue4l4W8>WVs-CwmmhdsQjeIO#Y zuuH1m-P+b3$u#un)3Y1wGS_CXeD;5(TPgMbnulioi6M1f|1l4Beq}nt_elurI$G* z&y{p8@`mRxTv9Z1^;|jld2>4}y%Mpu?x{Rvs;~GXCt6YEm_RN(u(6-eY%wd3Z|LK@ z{Xm$->iwN}ZDFNN>_ngwR=zFbo;*o|y&jNFQva84+wNGa^>SB7yH@+5Zzm2AhxA{I zfxq!g@7acVQ3_hpoHu|y3X&x1*@c$oyqh2#C;(bz79e!LyTzPN0x|08RGYjB(YBx!J-|8WYT19&Gy8u8+0+$Uxc>My>ng?Sc_1TRv&U((FqM6et-geJ*k5B<9Fd9Z#3 zRqG68EQXC8;Ld(h25Kmu-Q6^!g-&SF-q}~S>~|&m4UdKyT?{#gf=|Ud1T5|i_ z3mzAeRm6%!YesYbFzw+rlASs=Lp~ewZ{=U>V`Ymw$>TEU35C##eE8`+@)LlZVuCjX z1ak~kRA5q%@9ZY`2_rGmV65Az2SKP)@&g-O>UQ93UuVknt&(~jz0MY}dc^BT!hnOE zZWcwj@WVwjvk0#D7@S7>G7w6S2;kWFo(mL9ABcrhk%TGyuQ7@OJ$vu2_>psBDL_{evKM zYEi3@@Gn`Rz%IU;d*r?F@JHbXk0BA1_Chpy;OHSpRm+uD^Sf^pX5J*$t3S7WFyxdgec^KJ~Qfb>f&B zHE%Rli8Ib?CHKUTCeycR@%Zz8#D*_W!Owz|3-o%|yw%gx5Pq5Ga1c+frv9Dmj{u{Y zTdyq@oPo*fls8py0TLUtO&yKtkRnvTHokN<0`0nm#lc5SydMQgjm_TpBEY2Qx0YMwJC5 zNbnncc#_IpjK@AnGRL@tmW8hLfECT&b2#dQ{q7a_-aq68|DNjPgd6MPu(Jq=V!8leKC z@(=+S`%@m{i4?*BBX}WUB{b^A{Lbm$Frph!wegqV zMn{N`0VB%g;TjuRZLVG8!dEAM>W7|mk+E$gZgl>+4(R&p*%c9hQL?+(A?LuO$%nfeN<3Tu5|!06|TLPjr(VD_cmuU{d%?q;C`qAxW9UYZlji|_!8}h zUp}ha22I&0lv&Gg)iOR&i3q9rWAIJ!Q=DJ{`~Rr}q`J6Ci0MR%!V7hdS)U5Zs z^arP`KXV)%oz+2yKpksr=P?VABoWX_agtbdT7Bv5y$eL(T;e2hc^V$GqmyUJ6|k?W z`Fz$$i&_!4UoRwTc}y2iz?A6(Q17C@z?sLKa-rdbT$}l#&v=xqA~Za&M&bKJ+(=naMmO^*t2tmy(=rQP zf(~<)1AY8j4bzDdHG7YjpP)T-tdvHCEnvs#eCEgJLgjYTW1cOUJcNcGEG&X2mh(ysRV z1NM9WH2vC8`7|_t{Y5(pnPK&iu^3|jfgf;ds7RwrL)SyYgOoGUOwAt?!Cewb4fTiR zZZ&_VyK0P-jz4q|Wt_yhB(?4L3Jvu<2d0a|eKe(`t2hLnE7`X4yzk{$foX49-X2Vn zYUXKjYV$ze))(mL+Oq0%j-? zE~MwK_E~J|PDFTxIagJw9YgY;A^0wG&CXwn3ZI+#`xa=QuvywqRQ&&^8)W}+U)w_< zg9nw2ME+0!}>UbwIN(lxp&o`}CCZv*w#HY#SCaRFLEwy(7y&uGtBmo>mtO7OY) zXHJyhBDAVacL*j*B4w?xjM-qq*-jUGGcekz^iFw;-KwuvER z@EmpkH!ita(ml~3v&wUIwq`DE$54J;mU}{??rZIn*Skx*8>dicF$^>uIzXNGY}tX8 ziMa72%yh%ep(sVuJl!D)tJoaxycA$;svV*65W|@bBYoNPB#NL~1ue}^L>O&eMB%GU zGSwYRpr`C+S`M?f)aC+?-80@?EeE)eZ6e4KHsHjJXJ3qDWA;D9(T@ugx@;rhppVS# zx2Gg1aR>!P#;A9`5W{ejR-*4eNCs!-(wsbz>{KUy_}f{Ri@*RM9#UxbO6KKX>c88+ zehCs0Y$UA*Gi9$S#0jqc7#>!7P=u9EqLyEQ&*L% zki)}zC}2pkM6(nV>cmyO9@arz`khb#WL5IwZmSbHQi|AWU74qzC&op!g^L}lR19NS zx%daslc)`d`MEn2F*)auziMNN{LTRt+caxyw*4FQuNakt0e93RuqdUC@$|Rpoa=~z z0L@e01c?9tN8&Q1N9&h?*&lq#gjiGF-kcdNq-K-Oh*19U*pG+K1Z5sn_Zr$F-^5+V zh1a+^hgR6CMJ_ceg$ciDykP*c^f07^ZPh>QyVYWIPi?;YFLz^c$+T#v@p8-rCf8^$_j*tw|UG&EUuu z`0$huJ>s|G5GDaOI#sEer>LIWBxmf{>zj0s`UFv=GLhMOAYfTIDuA#>wRbE+3lyP}wc zH1}c2D^79bqjyfC;RDIl4);4Xe1Fd>r>#1KxK z8wxf7F~#N5Pwp~oT?-XqVs2}w6p{pJG|vafmKiWiT}-E+KV#3Ryct2#ytH}L8?pbP z;`>-&X(H9OBe4Ni*{6u3vKWqp8{g=1M|>c)f+>q`ksmx6Yq2a%tD#+++Gy-!d!Mdy z?wD|I+}57^Vg6qJ{7F z*Z&z5z%lbqD1K0+Nx(R!e+~E^!)WFkdDMelz8ak(B(l8yqya|RUZ{TcPaz{YUiP=I z?|jrXHFto2kdmaJ{p$M7hnBGj#V^UTG3mW@Z8p^o7n9CTI$z01uQs2|T!46VA)z;C>qA|_TJ@1XzX$I0S_Q=L$jQF!v-g=&>6 za31FN!Ph06-^OS`JZgK4V>X1CplzW-yjU#1C*bsO;3$%`z@c4|X;VQJ9L=~*jR)GG zV!TUe1Vq4yUsakpz{rkm#vxs7`Z^`5R&DkPJ@BZj7msTd!7U7~cN8MoW*3($nL0Pw zYw#K<4nF~eH#)Hq5Py<2)scG9Mvb!<6QnQC!FXgx5ZBM?!m~i1P=;Udb8TM_I)KB8 zppl0gg~b3=H4Im`DrSIjo%U#ivcsvb3CDwsoxBeAR`Tvj>D_%p`;1TCIJ0kGP~-t)mM@Yhd5 z#SN$Fv!v-;n@Mw#zcXN0{ihUmyXm4T1M(%)I>BDMZEIeAF<;&W~(fa|b^!w;^Sh0o65aVsqiA z<^gAGxx=LEVkVg(D_xUe=7|?Jl8@l}2c_4Ay{$nrmkiFkwSB@Z*Rw?L&W2MH_g#s( z3Y9p%_a^%pz67%5Hgz&|w4I9yt4A5GM*2j83&A2V&y0LaJ`00qYZ2J5rnvV$cO=nO zEhr$ThUu8WccscFm9yi1nz7zlQq~7&kUU5#-3cY`W*mDcq`LX@Nn1oiNf3W;_sAvE z8P}Pqdi-nr;p&5;jU83yEf}XsD;C@b2FshWX8PlgoGxPR&F;6G32Hiz2H!g=7LpFN zfcu1WK}!)z-KZC-8bte$x)kU_vOIB=eE-uG^Yoa)QK-dJcb=Z`B*DYiMF+D_7vdSd z`9{K`MjM~`3siIt;zy+-ZNtINlx21zcal@Ox+l%%dU2DX7j^Jg2(Ff= z$&X$9>+m5~nUUraH&9BK0@~Gu+&d8x|qGx{5-OMi7M0z93YStK}EpLr} z|7rO~&gk4?MaG3xk6Cgx?dV;aWleKy5ZC|{Ojs5z4+W2VwJd624V_uvnYPtz^Tdi2 z)84vBmnvH=PhYVp*{D3+*bu!7fzq3C6!!ItcF;=ilRgBZKgFLuZLxW!<+vf$!c9l_ z=?%NNh+Ew#%}JJZ_yokM`MSUIo?7r#4kz?r$AV6J15EL9=864W+{ zmiAh%yFvmq4WViM*1OekM}-6(W4{a0#6j_t1o@Xw%*sPYpiG&bvn|myyOjsQ{yR}w zK?lGz&fRy?CamqQA@it&>9hs^Sns~OnXNx=0&rJ?C)9Q1Bi5JNI-%|!Mh63~&VR0) zXv(O~x4ch2i&}T17!hjHOI+MAJ-yG%l_MB6V_s5IG8R_S;*AXr8jrx4n&ix$~M`5n+qBis$4nb(>lh{y;uT6(vGD zrk9Q(-dlm%;SYd$vDVNEa~nt|bJ+FXMRqa+P!DaSfiDDEenZdbozwFzz9sxBhq0z0 zk{*HV*4p6@5T|B;@-U3YaZ|7k9D+t`EDavfG$~S$)=gSm?GYDf;Z!H9s5mJ4C@Rp&WeS0u$snHsbS@*!g^KpAnbx z6F}&+;IV}Ulw649%CCHxcY2u(^%l3h_xGk|k9 zB7D)0-kp7tEj(vO2vcl!GrhW+mIHCob)VKeM=;LfTz20g(J`~x456!HWd|7T*pgGh zOH~{sz(!;xLEel8^zp)iT%i*On%?`pJ=2>Vx0%OTD8uHF&0m{ux_+i{@w6mD6qn8H zz2I-KF@?XOcxh?T^d+z?C5*jpH9-W3xPb6V!pFS*b|n;>LAYD-}K6 zox2(j-IF~6F~SW@P)3kZn1?nPR@imfB~Q++5U6q>g7Y-Gmg`U>5#6~OhUa_qGg(j4 z9jc<3s& zVc;St{urth1(UyB(rj`-1fQ@Eb@`C65=OKXlL%IsG3F89p5{4~Td2QRGV?MBGrUrr zl^8-DeB4WdXcLbZCBGIe?)Oa<(YbBkCOSqtN>A`CK!69mc-`xAj3pvJELa($Q(9;m zh=4e#C&q;f*3o%kVP5=q!c!fLzmAYOz7Y)S)s$Q;@;61;kdp~^t9-9v6ZN6u}()Z88AVgYkZhZ5oPuWhWH2h05%8K`M63-(~ zxNM4VXxozRmxZ*7SIn%#y($UHP|q8d{jvN7

0ybUymw)5FscP zZr3HVbTS=WK?jI;jf{|`($9H*@>UG_nBWv7eA<4vD)jBd*gK2K2}l}C@K_NPNrXEX z0$sBgE>VjTsK&T;9aBr99k^APPvWinZsUXNdH06S{WkEcDq zgMiPt`H^}tZlFcO1k*AS zv8(yiL*oNYa*dl!W=rBjc0;N)S$Hzrx2?lmxjWZbC2fn=#*-?UcGY*BMkF=r)*VhxKT%)OlWsW zUTB~Nmg^x%(o<0|>CRylS{%inD>P>(wDAjE-a_IL)(w42UDLNo6ZdfZS^3q!g&oR0L%VEJwAF4TwA|>( z1w1~b5KtF=UsmC%9kI<}Gxo49PFNKN$vIGi_0S2%*@T3X%UqV>NB~PP#OF&l3D>0f zi-);&Ye<0GxA~Y1!LaANb@R<4vB@Fsm$RvsL=Wv(WnlV#mcIkTLLeirLPxz(&W!~V zik)y3^r3|+d?vaurYI-y4JQJaec9lP+=Rp6J~!>oMH{8V3y_^;_NzOzd^nSZ*oLQQ zcIJHreNbkI$JWN?r^ajjNuvwD<$a;HK^HoJ1Gz=EXY_=g3aT>(^;6Vx-s!3Hua|=MLV02d5&Urv1f+DAR$fjAK zd4z3)1(~un8}Fv1W9a8+p2`qVkt`Hr4ZX%+J|>dqu}0e9&0cDjEgVl@V^AA9aSG}o z&h>ng=;(8^x1u%o7mQ;L4Fq;GKN#JcSTw`-SmU;}+YC);<^9v050w&tQQtTFmt3w=hcU=5Gz9!Dl=qX6K_>6Ibm zxw1NWQC~A{3L#K<6GMN58JalvZ2ltzaYFQ~eo)P`-GxrUG>E%Vtkr&6ythlDgY9WJ zD&qD`EkDy_6A?*}ja=3rrwH(89s(_@E`~4?bH6FRcck)N15;B0YbA=O`-4 zrzNBhfwJClHMhNov_m&99WDJ(FAMaM>>aRoKNF|%e8)tYwJ3c7THM@7Z#ce@rV+g!miDacZZ4-ui@ayr9&AgOfKh49ynZ2W61&0MURQ(}?uTcxm7d#guEiN#r| z8WV->kD7CFgS(-WiEC1NFR}OoHL7>!!=1Lx&{xxX7azUz2 zEv6WJ)-#}y+U(E!xonPReweV9Rqp9UG8wkj9?Y(vQ2)PTs-vR?D*x!aa1>$`@LulX(WvFTx7$Q}}_U>TWne|87)nnUtXQt~(#|hF`@@E~7 zOY2}nVOgU0GPGq%IW6x`vsy$%9rqA0h<2j}(!5Le zzsJ@O^D>{e7+u<4X^WOKITYhumrv&t#d5vGWT{ON(_7%P0ZinSM^CIBw#4*Su}}Pa z1e0T#yAIg{6bN=2VXIch2r6&C8`{~gjTc1?k_b1M0}kC>)eKPdVN`~A=Q_SEgxoQj zuX%ADjD<;(#lNBPoD0V~gyLT##x5&0J9RZEnM4Gsye7G!_fjJ98~L`3v;Bj&pJ|eV zs^~!Zv`U&QY)2LZc2aI3G<%xoQp&O^lI(ppIgsdxk8Hmu3jngEDA`0Lh|U+)6fiYb zw^N<XP`prNsRWmDTE{#KcWP(t>i`6o~SBCVg<{;LYNmLSzuH z+imKkoeCRZLK!6}^RdTI)?muX=`U?Hcyjsw?p=Zlh>QOY$8K|BY`;=pY)uT0-oTeQ zZH#ILDRhJkpp-@s7?BE(7r*{SIMiOnTR`q zQEyln%dxTRL$(7t`A>5k4dT?jgr@{{6q4_7LI&qRlk!h9mwDOKp^yxv?-|lfkz6J+ z859x^Gt%%GEo4&^fIiYaLZC1G)efFM&{uciah@w@9}?E#(hfIU&?6@q8)BtON`b>5 z#AN}aTLz@70Nx$Jqb%oC2f6MLkObH|EIK{Zu*(_(00Aq}b-%{7jkT^rpk1 zL6YF9k;YU0rpGh~?z95qGJlEa-%Af^+?A?11M#edrIz5tr?^uP zLTLH%PYGYj&S%0kt)s^VkR^1>*0167H2L)WSMONcD^9QL`Lp*{b4>xPV#yU(#Fi)25HBn1BcIkwE(jlri4MHAjIwBQS>(M z7OM!dLPyfpJs?Gc&u0&zbExII+zw-()}9{_Ab8+hCM8f|qiEsAm`ay|Pm1Qs^KcH}lms7b=U2}w{J=TS!7^7qH^JQtBv}aPp`q;07i{xB(?Qp}_WTSyZt^$l z-eDuGQ+w*xxQvJ|poXsw_sL*@tM)#0G2cR%nM@^au;ZTer&0wMlwh& zlOvYbiaDJu3=e$Sio2vclpJE3p8f8{nOF~#Mk}r&8=^GURCx#2`PdclbmIVsk9#Hx zRU{J~IxEH;(^#a|-mQ>;DoJ)AX@(HOZu@)yHdktPF76VCE{YXe}KC#hp` z|Dd7^Fbt%Wn8F&v_|9KQcC}Z*%Tyw?oySl*w2#_Ah?ci^gsPs$)b#k)2GEKks%JdZ zTEI*3>Hdd=pefv^am|pprl>>Pv=F|&h$7kEswd6Cbb6UJfmZ`FJ-E7PO=P7I!a}q^w;+=nUgAwc;h(UKHFiLwD5y4UIS@0D43%hCOu4T#TT{G zA7qi9m&o`AT$7DNyK!=~eDqM6jgroTzPZJbzifm&rtTe{*MZlh_9BH)^ix8*LqlKL zq#xkBz?~8NgTdkG_;d%LI(-iJE_r{07y_L#1W)G z@#F-3w`w0M=BqEZm9P9Ad{m$cuv`h_v$+%q?&|lh9nqkz5q^K9xo35CwZ-EV-KJCZ zdfBTJw6|yZn+~?~71K5y?gu0Yakl?lW&fZ0a#2$q`Ig$8yDe0Ivc#`r&(ye4-K1qD zmPq^%+^bS{Y^tfz-GrY6h(5URP;{H7+YC)hW_hJ{5we_Ql5Zmh>f<{Uv#jbXrWc&Umo6Md%81T-fW_X zr9P6Dj0Xs>4{?osjN9T*@K&Dr=GZTXqVmG%t*k}l`gW#o;xj5?DqKNaIWM@FUl&-s z%A!$|-Ye(N&QU_D4SY~i#qyNum;RcH4NcqiZxLVLp(SoVQK*tI?x+f$6ZGjfSE`a# z#OmkH7_&XUM)qQ>L738yd$Z#vSRdElA`x3XkT?;M3GO1sDZuH znDh2{ebdR|bZAJO_bd>>i9`S=8^YkhU8dVIJO?O#AJtuFQ0>aBue0IV6SS;0#)QaF zKAb>sThg6Dq)(rHTBu{m*zF@eyZ)qxodt5`R{R}0)TYX&s_g5553mk>=Y*<8WymfO zR~)1!|8S}nZcV+}fCs_FIh@UC<=Q}&3&OvX_une8o~ci@k;+zqlVeU_m54VzJ5?JR zd?kS3H}oz*YtRG{&T~R?Pj?Je-#$(`%zPAKrv1Gs;LB@tOauyO;jZORDV=4@KhJjy zaQ(La`2rXK3m(0|_T3>~|KtuXJmNnn;9{~yvyx$-2-HAa%MH5~>CmR zQ^2Y(Bqm2y&XyF*3!WrStpMth5|OsI$zD9NHd7qI?|nX{l1;e|OCn8v#I?uS*HYop zGF(Jp=j75=@bg@{?z*dc!K1%VAzq3A%h~xWoq53#%e(CSA-trn22wp7O*Yk(9GQ@P z{tw68QojkqfMtY3XhhVWX$!dO9x?Cf9j2@zH-I(c6CHsku2U|RtbDJJvC z_EvyuHG_;g&#za?{So>VuIAlPtr6d^b}T>fS4aY<5`wq08jt!z&Y}@C#}@YXNt{7@ zoos>BK6AM6cOA$=u6t&7~s&oy!H>Fy?6q_ zR$4!HfuogHIwicANx9HDu}UVY*w9xoxdLg*O`KdSyKgdno=StGVDH`7x@fsfcI1wIJIV`76`GBJ{s~Q zpMVNg;(M=XoiOZPLptZL8#_DZnTJX(^r-Ql%psbY`LFw2#dcKo-7j;PVrgWEr?uEa z=HoqN66L{GN38+W`m@RNFE1Y?+182XdSx5n4|BzjhNc&0eyY`(4NxNQ<}GU9Zy?qt zE&X4{<3Z8oJ zQg|K{4pzx#`kRVuE!nLa%lwBOx>eHF><-Nan%7+r4eIv)m7u2n)v&#Il&$k%>94>2 z*VbH(33VG^?>V2j?r;wre@}#JScxHBh@*5A6W%3Y&|YWk2NS~ySWouSc- z$aDIM=l6@R3nylOKku?N2`$=%77sgk-r_&tfR<+^Z!{XwoWDB6eE#A?XgzU$Lbz4B z<-CDlP8={00_oM2nKD&w6IhJB2`jU)Br+HS3V+uPxv(XlqZouU>QazVkJz)r%Hj)p zD3It2mYeAAoH+DKeaZbyH+y4Ib(ibudm_avB=#&(oG;!U<#LKGFvr86 zCx^rhX`UoTpawCP?c1XRVX%D^FXQ~^H+SAeo&K`cfSW%98{Lp=r_wpd)lUQvnGX7~ z6g3zyiMkURN*@0B?oX1ITWZF^0g5bybSO8hc1=o7<}eR`LAaqq)mm^y>W>85Pc{&` z$;a!kh=J3sLpHx#H?;IVSZ3CcXD&a7W|o;XxXdH%+?+%HCQezh(7Hn|9T)B@?59d- zFWjpf069CNEiJ#eG#@`+`A~hfJ09YdxU^8)DsZxJU~#sqPU3Bm;KVs3xF-L<;cgfg zmznRv@J-6qB{yu*9-%?N9&75{8My`P>p6u_7ODvL4U&?Y5VA7e+q4#NSqpt*qWYqk zcw>Ra6%?|}>r*g!xer%kntBlbmo^y}Hh)YX-^q6_jY)%GV=-0JE|xai>AUp_=j?A_ZnfIqH% zHX@9#L&5??Y?5S#sNacvGO)p0i~|l^@+mk=yY{(#yI~YX>fNV zd*@J4>9Wj14bO6(C;|Iyb>dG9oun3$||UD*x%n1<|75Yw%X%4*y{>m6J$t)Z*4iE_Bq z(}vfeQ~rh4OJAcY8b6r?A}0KVqBA)_0$<9V;golLmpQ}q;Jdnw>~*#?dfe{oR(JOx zWgnR*-eo5}i_3gjygt(0tfxM=-VK&fsZ{lR?`ex11h!hwi(IsDSC^2z)?W*$m^NYzxWPP`te_%{1-g+xI6N z0~BRb(1{vjEp(sOa~o$Y2BER3j)aA0ZnTD{TcgmN$3MkX4~naGzH~{{v!nL~3g*n5 z0#!^tc5r@g!Q#T$WqmfazB!SybAq(2yaN@4>J5Ge8gwB4&tX&9f}MZ{T(*wG|Hn{L z6XDM`J~nw|QU8tVb@5A4lL!N6qZFZUl1kWPa*N%uB~O01G?)ojUS6^B#p)C1WK~$r49>|VbW-A zzIf1TP|TIExU5i{R!Q}DG#1c06EV{UDCd?hFQqb$`O9|#Zlj0ln6;uWz+QCy#^IB> zFWr-%B_WcA6Dx#y!u(9mCUZ$j^bx+)^v=FCr);1I{4ib+ne*DIDcBCukg zpa*$Uv69YT9^5%<5vNMT=&~1#hUP#eyulBZ^oHfWnLA0LI3rZ#?#wQMV8L$1Sus7n ztjl-DTXzsORG=uK;XBX@BrK$%jLpxXuE|&LJdjS{ao|zFdeL5K164F+*NJ8L4RZc~ zR$%3*VbAPOt@s6=fzU4Vk%0-jTyb0X7At$~5k}2+6rDy{ZH|zZ1F09Abg$oPxaHE2 z7sv@2+ThqFGksqpbBeUx2Criy@F8^ru+h}Z^5Dk1nQTRUD&5efO)k76`y*~Vk2@Z{ z=7EFrX+77YY+oMcdVHcTyCSuFc>k3qWNAm{KG=7)lf}O*g1pwec6Ba*IeQ|}_MnHx zUkf2H3?6)+agvxLFdM*d=gsc60)LtfPBea5Cb`0r=w1&U30Xlh@05xi^+v$edjboYXbMB1md~~pv&D2_@m=$l3v3!Pi=5-%4xdzzWOmJVfB}+7 zAKmP!D{9`B5OSnD_Oz|VW%Li`pLxjpn+quX-?#w2iV__y=5v?O0JAjm6PSMl54^(q z&L$|=Z_o!cN1y&Gi!(1tGxpDp`#19XexK_z~-0vl_IuiOakL`Xs=je7u z8Dj+2>iwFk(F-G5sHkP=B1wJVEXs1EnW?W07CGUFr}gVrys`7oIvaQ02pc;nBaU`s z&MgM*L-8IdN?I(xmBb`%s~C%{pU8Y{FaAvOmE{!DM|WuWr>}X2udj0ydC~P8)2+Pq zn=c(H^{60Tu>)t@#$E&gH@Ia2YXeWLZ%iR?UZu# zPV8*tsK##VC=2<)tnai5*O2&L@O^#A>WKkDc$quQY*51{CF;+-@pWe4f)#2@DK~51H}lAzk`&4y!+*_ z>z0SE5#JYgk`(oSQpQHRP06`_JZ{B>9E5x{s!M*l(R*Xe^@3g8$hnm`aOB~~W&b|u zZ>a9yi#ikjY)1xso31QL6c2ERcDfNT_P1e>Z<5B?V~wpJv8AI6B&9Xe?)7ujkqF5e z;_{vfO~0s{+TUA)z3Hq@Z-(F7{y^;tpA(7~thYQO5cX!7k){f`%OPs~)sE<3Inf(J z8n^|dHRs4DC@6T~ei>O4tmI9bpOhb{lHNI^ic*lnt zA4B6W-t(2qgi~TzbR-ODI>>yRm5mGnybOz)9@c$|#)af@mFd4Ax+605q>$xMoj3zd zLnn{`ryg&!MAWStEk+X&%b0CtCclL!!d+zVcwy&(`636WHzF3YC}y(8iTGhtg|7{d z@;tO%y1DD+-`_jz+u?GxUh4ZGR&Jnf6t5p|{Ns6}UhF(E<8}wP*;mD2TCoLSUT2rj zVs!D!PuLY~)irfkir~IXzMvGkjRvEMFsNFO<|R%SMP+%Y_9mes_nnL8Vw^YJMMT_H z_fw)^&W0_wR^T>h->;?Ad3F-u1ePB*JrZ_6onajRC8Gb)MgXWo^Ojon&f@0bRJeKV zKP2P#_5#?I1n3FLyPo`g0@&(3kpo-H2>9wiD0n6KD-tJ!9T%S+ew_Ygb42=OZtE8E zOPag6Sy_2mn8-e+;4R(?>jF6hWV|}E8{yU5dY-`$xjNX)w9-f$i=zy8|CA#NDn6jU zk0QD^P8#{;I=K-8Rq{`T>Py0HFAK>{ZutG!KjBKNs&QlPbaGT#`~Cr&U6JTCl8R5o z5VW_FDVTe@A{D=8$DOQL#kM=}oI$xZe?>5IV2-3^G|xux+pq)D8=b$cdgByDZuH&)}edXNpl?B;&5;tNR?A)+#XaJ!m>hE*aNMwwt-wOD>Q>? zVQ*3-1DT$`IJT2H?0HRzf|A}3Opo;i*dNyAzY};uLbS^rb|6-p9KJddO^B{3+#aZ4 zE86f`l*1gKTu<{1)b!8y=!G;E@(5e#n#5plW+B#_pQ0WKo9iWvR5baGIsO@p+bWtbwHuAr=!6}!Fyx%dphh$2apZ;anWGQ`sOO0Vkj1$b!(ZyIq_5>ycIS8T`9A@ti^ziX6scDu z^W_a-34IX>1uWTA(Hg zrL_p}6{lbk&NgnbgM7LFX>}~lJQJT?qKInc10#OmZX7L7WCSF*%p&HiRm5uk zW={xL^HL6^VNQB6+inYWPOFrF7fX#GdbO)&{904y$48S+yKa#ni=Pw~(v8Ar_H3HN zDtM)xA8F>}&WxKB$i)^8zw?N^4=MhJ+iHs!&>*o{sdj+a`%v%;w;N=fwz~HKvmF!c znAGpva%b9n$m+@ra^TLk38GcM-;_0$fRow`cqpa}sYxMTIb4sLuPm6!#C6bvfe|%- znCrg=pPvcewd7zS+{*W&1M0uo*#rpzht91VN^pgzOyxEsjM z(-yX~e(5wR@^!$6{DbjGwn_d+TIZj^wU)1`E9nL8d(?=`7k$=oEj8#f>qn4)?arro zJ+7h5NV_R|8F$Ml{o*2Nr1#6q&y$^AbW|)O&4b0`a6D-?xeTd_?SgCV-{OLvHB%}`FY2aNymkNzXtR8R@Y|7-5 zmrLhm9ZoaZxW6Vv%ZHEZp7dd+#1HWB*lwK}W7*VaW8tq84G-p{R)suOs;K6o$C|Qv z3Ul5F=h^QR9^Vd5LFF8tBx-x{ExybVWX$3X4iD1a`K*w|>(P`IiVT7)Kdzc?{`kg*j0Io2&0~`| zY?Y2)oLmsdWYtEH;=P#@-HmzbVUtO1U5LuaV9*Bt*su6e^A%~Awe?@mD&TuGPCIX{ zu%i$<*zk?T=9GAAyg2XWw`PJ6R#IDAxOm)JMkuo{B+}5i^pY;-^an{Gr)c&Zv!c?s zmD|{<6{ySyPSY|NsC%@LNdnw?u_wjWVeA`0g;OJC@=O1)VW0$G0(AS5=~f!t4URQI zl^x3uFBJ$$H=bB2S@wf?yR8=iBO)4Xrc2gfXR5efg!6%+wdXXGwflw0EZdEX zLlH0TxiILII1gMSPbgk^)(Fw$FB*I4XN}q{{3ZU#7WrU;<*Z2b7CT0T!8bP7&nwpo z$w8!e|AX*rRuu2{sD9)=JAzRkQzBet&*ndOEbBBmkihHFt|u{k*-T3&YfJTi9Fp=| z(NZsiGfr__$XZnPYJb_c0HxKCjgtcqD$_u>DAXV=!L%U2Z*P z4klQK4N?V}fcQG}w0S%p4(4Y@49h5|iv^%rOO*wh`e^))))>?=F}9&a`=AzP8q@uO zvgJe%ajo&75y3}n3kp~Xsm*PmA;I=1RXis5o+>sIjmI*6zWioMGH%pL9Csed>puG6(+w542iF#lw=Xd!x)nJd{wDt6<>;=R{Gf-8{X zk^O7x_aEnK3d&6_rVBj=TIPp>UoQqr^CU=R-eLI9UHog<5<;tQu1c9s{$XDK6>+QT(KK~_+Wq$i7k_Er*U(irtxskvdi76lAFPbzAU9@JP3#~cK*)SVZ%#c zBg}B$9Y;E?p+a7U7v5pdCrE%FMN6e%c94grqBIZpzFFh68wzjybV^*ZG zd&8LrnIrw94WCdjabVqpHf{~r66WS0==thm=BfFl>$u9Cy+5EYrUO0C)@##Vmdm(F`O!5DVU2>u#-%p163cq7zkeV@>AQDHqL z!WnK$T~xxpI{FYKgN!M3Kdg~ef3G6Zx@Z;kj(buIYW}da-pVk!i-Th=%`B?=Kb8V_; zPUz-=N3TVrN%wUMDdEc(;?G6u8o773%_N-qy_r%5E8e$MjQ3NaY4@R3pAbnMAdL1@ zjfZ8{lt%lHTeCk`!96V)UD|0sl{Rpv1`LI-nBp&XAb0<7Q2wtg$|DUMIEWCeyI%jN zVCEkW1@xF51C`6Iw-cDJd6W#ic^*NNJb>T74Is~X+myFWxJBGp>pzhGxF} zqfI-rlfxzW(2~1NKC{G=$zx`9P6~Vi5b5u4#wQRHPCPX~iYgn(rMvj4->>*R3X>?^ zcu6D;^s61d0RQdM8rcktmrjT}}?3q!8uK(n7mt%kW2>@MqRWQM^^eT1yF~y8REHYj#P2N*o*vd#->N#_Yf|ACNhp$;GOUEH%k>= z8#@#~E_)@6il<+>7srhIM7ED{C6Tas-2qQ{=vDc5aTlxHR3P|1oi@FcV=nmbvcse< zG25Q4n?&&ZM5T-JHE;VAnrR|)>ncn6x(&?lRlj&p80-)vAY?E1jenx&O@K>6QuaW0 zR-t%t>=y?++aeT5^NUk=Ijs7sAmbxNgJ|bTTLL(xl)^&W=p^UfYBHGYkKD$IbSx*U z1Fe^u{J{C|ukHgYDPi{1H1qG$u6Xt#>BIKAY0vu_aoLbw^@7zjDaiiBCC(1L$1!yC z^6gLGSaJU&5H6+cXr-;-e%m{?|GhF*f_b?Xk-Md_BREOFH3TMSBkkSjfW$qr_qhrs z_s0yfB)v^vxR-2+@>SkD?K#z?f7?^#lw@Vsl;F|nUO&=a7`VIzbN`Z1r1eMm zf$7aZSI&Nj0d@W#Qav>rYOhu`NWL^_B!AoHVj2?q#= zTuCR1^=SW^`<;2y|N3F2nHNf179NGQ0sm2S5UCdE>IOr7k5^!Z)*2J#RYZAt|Ml)F zx72jBC|`qP#Ps7F*jg&ab-SGMna^HVW$$?KpHEx)64)BMA6U)#g^FxlD4^3^tCu|| zm@{uaLCR~?{a`#T-bwrK-ub;>KM-Iae;RWc`_Go*QnGrlw0EkqfB*gbx-86fHGva= ze(TBfJile^$}3K$3*t+t-AGDzQy{9IcYdS#pD%2tC%ZA_p1Dl^8uG6S;}evLehqYe zp7AqeC4Ka&)?VjLocJ`6Ki=52gE{eoqnmaO!B)TRIWkeF!srpxXeVtxA`0{}c| zKdCV74aL4>kuf*xJ};Av{lGelze_(Sh_xQMOi+TQcgC&H4vpLAKlx)L{v+l8DCvLh h`2Sz;*_ 1** | **Spec Decoding** | **MLA** | **Sliding Window** | **MultiModal** | +|--------------------------|-------------------|-------------------|---------|--------------------|----------------| +| **FlashInfer** | ❌ | ✅ | ✅ | ✅ | ✅ | +| **FA3** | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Triton** | ❌ | ✅ | ✅ | ✅ | ❌ | +| **Torch Native** | ❌ | ❌ | ✅ | ❌ | ❌ | +| **FlashMLA** | ✅ | ✅ | ✅ | ❌ | ❌ | +| **TRTLLM MLA** | ✅ | ❌ | ✅ | ✅ | ❌ | +| **Ascend** | ✅ | ❌ | ✅ | ❌ | ❌ | +| **Wave** | ✅ | ❌ | ❌ | ❌ | ❌ | + +**Notes:** +- TRTLLM MLA only implements decode operations. For prefill operations (including multimodal inputs), it falls back to FlashInfer MLA backend. + +Note: Every kernel backend is compatible with a page size > 1 by specifying an argument such as `--page-size 16`. +This is because a page size of 16 can be converted to a page size of 1 in the kernel backend. +The "❌" and "✅" symbols in the table above under "Page Size > 1" indicate whether the kernel actually operates with a page size greater than 1, rather than treating a page size of 16 as a page size of 1. + +## User guide + +### Launch command for different attention backends. + +- FlashInfer (Default for Non-Hopper Machines, e.g., A100, A40) +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend flashinfer +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-V3 --attention-backend flashinfer --trust-remote-code +``` + +- FlashAttention 3 (Default for Hopper Machines, e.g., H100, H200, H20) +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend fa3 +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-V3 --trust-remote-code --attention-backend fa3 +``` + +- Triton +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend triton +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-V3 --attention-backend triton --trust-remote-code +``` + +- Torch Native +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend torch_native +``` + +- FlashMLA +```bash +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-R1 --attention-backend flashmla --trust-remote-code +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-R1 --attention-backend flashmla --kv-cache-dtype fp8_e4m3 --trust-remote-code +``` + +- TRTLLM MLA (Optimized for Blackwell Architecture, e.g., B200) +```bash +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-R1 --attention-backend trtllm_mla --trust-remote-code +``` + +- TRTLLM MLA with FP8 KV Cache (Higher concurrency, lower memory footprint) +```bash +python3 -m sglang.launch_server --tp 8 --model deepseek-ai/DeepSeek-R1 --attention-backend trtllm_mla --kv-cache-dtype fp8_e4m3 --trust-remote-code +``` + +- Ascend +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend ascend +``` + +- Wave +```bash +python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3.1-8B-Instruct --attention-backend wave +``` + +## Steps to add a new attention backend +To add a new attention backend, you can learn from the existing backends +(`python/sglang/srt/layers/attention/triton_backend.py`, `python/sglang/srt/layers/attention/flashattention_backend.py`) +and follow the steps below. + +1. Run without cuda graph. Support the two forward functions + - forward_extend + - Will be used for prefill, prefill with KV cache, and target verification + - It will be called once per layer + - forward_decode + - Will be used for normal decode, and draft decode + - It will be called once per layer + - init_forward_metadata + - Initialize the class and common metadata shared by all layers + - Call the plan function for optimizations like split_kv + - It will be called once per forward +2. Run with cuda graph. It has two phases (capture and replay) and you need to implement three functions + - init_cuda_graph_state + - It will be called once during life time + - Create all common shared buffers + - init_forward_metadata_capture_cuda_graph + - It will be called before capturing a cuda graph + - It is similar to init_forward_metadata but write the medatada to some pre-defined buffers + - init_forward_metadata_replay_cuda_graph + - It will be called before replaying a cuda graph + - This function is in the critical path and needs to be fast diff --git a/docs/advanced_features/hyperparameter_tuning.md b/docs/advanced_features/hyperparameter_tuning.md new file mode 100644 index 000000000..e15ddd21c --- /dev/null +++ b/docs/advanced_features/hyperparameter_tuning.md @@ -0,0 +1,77 @@ +# Hyperparameter Tuning + +## Achieving high throughput for offline batch inference + +Achieving a large batch size is the most important thing for attaining high throughput in offline batch inference. +When the server is running at full load in a steady state, look for the following in the log: + +```Decode batch. #running-req: 233, #token: 370959, token usage: 0.82, cuda graph: True, gen throughput (token/s): 4594.01, #queue-req: 317``` + +### Adjust the request submission speed to control `#queue-req` + +`#queue-req` indicates the number of requests in the queue. +If you frequently see `#queue-req: 0`, it suggests that your client code is submitting requests too slowly. +A healthy range for `#queue-req` is `100 - 2000`. +However, avoid making `#queue-req` too large, as this will increase the scheduling overhead on the server. + +### Achieve a high `token usage` + +`token usage` indicates the KV cache memory utilization of the server. `token usage > 0.9` means good utilization. + +If you frequently see `token usage < 0.9` and `#queue-req > 0`, it means the server is too conservative about taking in new requests. You can decrease `--schedule-conservativeness` to a value like 0.3. +The case of a server being too conservative can happen when users send many requests with a large `max_new_tokens` but the requests stop very early due to EOS or stop strings. + +On the other hand, if you see `token usage` very high and you frequently see warnings like +`KV cache pool is full. Retract requests. #retracted_reqs: 1, #new_token_ratio: 0.9998 -> 1.0000`, you can increase `--schedule-conservativeness` to a value like 1.3. +If you see `KV cache pool is full. Retract requests.` occasionally but not frequently, it is okay. + +### Tune `--mem-fraction-static` to increase KV cache pool capacity +SGLang allocates memory as follows: + +Total memory usage = model weights + KV cache pool + CUDA graph buffers + activations + +The `--mem-fraction-static` parameter determines how much memory is allocated to the first two components: + +mem_fraction_static = (model weights + KV cache pool) / GPU memory capacity + +To support higher concurrency, you should maximize the KV cache pool capacity by setting `--mem-fraction-static` as high as possible while still reserving enough memory for activations and CUDA graph buffers. + +SGLang uses simple heuristics to set the default value of `--mem-fraction-static`, but you can optimize it for your use cases. +As a rule of thumb, reserving 5–8 GB of memory for activations is typically sufficient. You can check this by inspecting the logs just before the server is ready. +Look for log entries like this: + +``` +[2025-08-11 17:17:03] max_total_num_tokens=665690, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=4096, context_len=65536, available_gpu_mem=13.50 GB +``` + +Check the `available_gpu_mem` value. +- If it is between 5–8 GB, the setting is good. +- If it is too high (e.g., 10 - 20 GB), increase `--mem-fraction-static` to allocate more memory to the KV cache. +- If it is too low, you risk out-of-memory (OOM) errors later, so decrease `--mem-fraction-static`. + +Another straightforward approach is to increase `--mem-fraction-static` in increments of 0.01 until you encounter OOM errors for your workloads. + +### Avoid out-of-memory errors by tuning `--chunked-prefill-size`, `--mem-fraction-static`, and `--max-running-requests` + +If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: + +- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. +- If OOM occurs during decoding, try lowering `--max-running-requests`. +- You can also reduce `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. + +### Tune `--cuda-graph-max-bs` +By default, CUDA graph is enabled only for small batch sizes (e.g., less than 160 or 256). +However, for some models, especially at large tensor parallelism sizes, CUDA graph can be useful for batch sizes up to 512 or 768. +Therefore, it may be beneficial to increase `--cuda-graph-max-bs` to a larger value. +Note that CUDA graph consumes more memory, so you may need to reduce `--mem-fraction-static` at the same time. + +### Tune `--dp-size` and `--tp-size` + +Data parallelism is better for throughput. When there is enough GPU memory, always favor data parallelism for throughput. Refer to [sglang router](../advanced_features/router.md) for a better data parallelism rather than using `dp_size` parameter. + +### Try other options + +- `torch.compile` accelerates small models on small batch sizes. You can enable it with `--enable-torch-compile`. +- Try other quantization (e.g. FP8 quantization with `--quantization fp8`) +- Try other parallelism strategies (e.g. [expert parallelism](https://lmsys.org/blog/2025-05-05-large-scale-ep/)) or DP attention for deepseek models (with `--enable-dp-attention --dp-size 8`). +- If the workload has many shared prefixes, try `--schedule-policy lpm`. Here, `lpm` stands for longest prefix match. It reorders requests to encourage more cache hits but introduces more scheduling overhead. diff --git a/docs/advanced_features/lora.ipynb b/docs/advanced_features/lora.ipynb new file mode 100644 index 000000000..1925baffc --- /dev/null +++ b/docs/advanced_features/lora.ipynb @@ -0,0 +1,529 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LoRA Serving" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SGLang enables the use of [LoRA adapters](https://arxiv.org/abs/2106.09685) with a base model. By incorporating techniques from [S-LoRA](https://arxiv.org/pdf/2311.03285) and [Punica](https://arxiv.org/pdf/2310.18547), SGLang can efficiently support multiple LoRA adapters for different sequences within a single batch of inputs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Arguments for LoRA Serving" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following server arguments are relevant for multi-LoRA serving:\n", + "\n", + "* `enable_lora`: Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.\n", + "\n", + "* `lora_paths`: The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {\"lora_name\":str,\"lora_path\":str,\"pinned\":bool}.\n", + "\n", + "* `max_loras_per_batch`: Maximum number of adaptors used by each batch. This argument can affect the amount of GPU memory reserved for multi-LoRA serving, so it should be set to a smaller value when memory is scarce. Defaults to be 8.\n", + "\n", + "* `max_loaded_loras`: If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `max-loras-per-batch`.\n", + "\n", + "* `lora_backend`: The backend of running GEMM kernels for Lora modules. Currently we only support Triton LoRA backend. In the future, faster backend built upon Cutlass or Cuda kernels will be added.\n", + "\n", + "* `max_lora_rank`: The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup.\n", + "\n", + "* `lora_target_modules`: The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of different target modules after server startup. You can also set it to `all` to enable LoRA for all supported modules. However, enabling LoRA on additional modules introduces a minor performance overhead. If your application is performance-sensitive, we recommend only specifying the modules for which you plan to load adapters.\n", + "\n", + "* `tp_size`: LoRA serving along with Tensor Parallelism is supported by SGLang. `tp_size` controls the number of GPUs for tensor parallelism. More details on the tensor sharding strategy can be found in [S-Lora](https://arxiv.org/pdf/2311.03285) paper.\n", + "\n", + "From client side, the user needs to provide a list of strings as input batch, and a list of adaptor names that each input sequence corresponds to." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "### Serving Single Adaptor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import requests\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, terminate_process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", + " --max-loras-per-batch 1 --lora-backend triton \\\n", + " --log-level warning \\\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses the base model\n", + " \"lora_path\": [\"lora0\", None],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output 0: {response.json()[0]['text']}\")\n", + "print(f\"Output 1: {response.json()[1]['text']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Serving Multiple Adaptors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --lora-paths lora0=algoprog/fact-generation-llama-3.1-8b-instruct-lora \\\n", + " lora1=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16 \\\n", + " --max-loras-per-batch 2 --lora-backend triton \\\n", + " --log-level warning \\\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output 0: {response.json()[0]['text']}\")\n", + "print(f\"Output 1: {response.json()[1]['text']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dynamic LoRA loading" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Instead of specifying all adapters during server startup via `--lora-paths`. You can also load & unload LoRA adapters dynamically via the `/load_lora_adapter` and `/unload_lora_adapter` API.\n", + "\n", + "When using dynamic LoRA loading, it's recommended to explicitly specify both `--max-lora-rank` and `--lora-target-modules` at startup. For backward compatibility, SGLang will infer these values from `--lora-paths` if they are not explicitly provided. However, in that case, you would have to ensure that all dynamically loaded adapters share the same shape (rank and target modules) as those in the initial `--lora-paths` or are strictly \"smaller\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lora0 = \"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\" # rank - 4, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj\n", + "lora1 = \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\" # rank - 64, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", + "lora0_new = \"philschmid/code-llama-3-1-8b-text-to-sql-lora\" # rank - 256, target modules - q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj\n", + "\n", + "\n", + "# The `--target-lora-modules` param below is technically not needed, as the server will infer it from lora0 which already has all the target modules specified.\n", + "# We are adding it here just to demonstrate usage.\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --cuda-graph-max-bs 2 \\\n", + " --max-loras-per-batch 2 --lora-backend triton \\\n", + " --max-lora-rank 256\n", + " --lora-target-modules all\n", + " --log-level warning\n", + " \"\"\"\n", + ")\n", + "\n", + "url = f\"http://127.0.0.1:{port}\"\n", + "wait_for_server(url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load adapter lora0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " \"lora_path\": lora0,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load adapter lora1:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " \"lora_path\": lora1,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check inference output:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Unload lora0 and replace it with a different adapter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/unload_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " },\n", + ")\n", + "\n", + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora0\",\n", + " \"lora_path\": lora0_new,\n", + " },\n", + ")\n", + "\n", + "if response.status_code == 200:\n", + " print(\"LoRA adapter loaded successfully.\", response.json())\n", + "else:\n", + " print(\"Failed to load LoRA adapter.\", response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check output again:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0: \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (updated): \\n{response.json()[1]['text']}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### LoRA GPU Pinning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another advanced option is to specify adapters as `pinned` during loading. When an adapter is pinned, it is permanently assigned to one of the available GPU pool slots (as configured by `--max-loras-per-batch`) and will not be evicted from GPU memory during runtime. Instead, it remains resident until it is explicitly unloaded.\n", + "\n", + "This can improve performance in scenarios where the same adapter is frequently used across requests, by avoiding repeated memory transfers and reinitialization overhead. However, since GPU pool slots are limited, pinning adapters reduces the flexibility of the system to dynamically load other adapters on demand. If too many adapters are pinned, it may lead to degraded performance, or in the most extreme case (`Number of pinned adapters == max-loras-per-batch`), halt all unpinned requests. Therefore, currently SGLang limits maximal number of pinned adapters to `max-loras-per-batch - 1` to prevent unexpected starvations. \n", + "\n", + "In the example below, we start a server with `lora1` loaded as pinned, `lora2` and `lora3` loaded as regular (unpinned) adapters. Please note that, we intentionally specify `lora2` and `lora3` in two different formats to demonstrate that both are supported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \\\n", + " --enable-lora \\\n", + " --cuda-graph-max-bs 8 \\\n", + " --max-loras-per-batch 3 --lora-backend triton \\\n", + " --max-lora-rank 256 \\\n", + " --lora-target-modules all \\\n", + " --lora-paths \\\n", + " {\"lora_name\":\"lora0\",\"lora_path\":\"Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16\",\"pinned\":true} \\\n", + " {\"lora_name\":\"lora1\",\"lora_path\":\"algoprog/fact-generation-llama-3.1-8b-instruct-lora\"} \\\n", + " lora2=philschmid/code-llama-3-1-8b-text-to-sql-lora\n", + " --log-level warning\n", + " \"\"\"\n", + ")\n", + "\n", + "\n", + "url = f\"http://127.0.0.1:{port}\"\n", + "wait_for_server(url)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can also specify adapter as pinned during dynamic adapter loading. In the example below, we reload `lora2` as pinned adapter:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " url + \"/unload_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " },\n", + ")\n", + "\n", + "response = requests.post(\n", + " url + \"/load_lora_adapter\",\n", + " json={\n", + " \"lora_name\": \"lora1\",\n", + " \"lora_path\": \"algoprog/fact-generation-llama-3.1-8b-instruct-lora\",\n", + " \"pinned\": True, # Pin the adapter to GPU\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Verify that the results are expected:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://127.0.0.1:{port}\"\n", + "json_data = {\n", + " \"text\": [\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " \"List 3 countries and their capitals.\",\n", + " ],\n", + " \"sampling_params\": {\"max_new_tokens\": 32, \"temperature\": 0},\n", + " # The first input uses lora0, and the second input uses lora1\n", + " \"lora_path\": [\"lora0\", \"lora1\", \"lora2\"],\n", + "}\n", + "response = requests.post(\n", + " url + \"/generate\",\n", + " json=json_data,\n", + ")\n", + "print(f\"Output from lora0 (pinned): \\n{response.json()[0]['text']}\\n\")\n", + "print(f\"Output from lora1 (pinned): \\n{response.json()[1]['text']}\\n\")\n", + "print(f\"Output from lora2 (not pinned): \\n{response.json()[2]['text']}\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Future Works\n", + "\n", + "The development roadmap for LoRA-related features can be found in this [issue](https://github.com/sgl-project/sglang/issues/2929). Other features, including Embedding Layer, Unified Paging, Cutlass backend are still under development." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/advanced_features/observability.md b/docs/advanced_features/observability.md new file mode 100644 index 000000000..f03fb3772 --- /dev/null +++ b/docs/advanced_features/observability.md @@ -0,0 +1,35 @@ +# Observability + +## Production Metrics +SGLang exposes the following metrics via Prometheus. You can enable them by adding `--enable-metrics` when launching the server. +You can query them by: +``` +curl http://localhost:30000/metrics +``` + +See [Production Metrics](../references/production_metrics.md) for more details. + +## Logging + +By default, SGLang does not log any request contents. You can log them by using `--log-requests`. +You can control the verbosity by using `--log-request-level`. +See [Logging](server_arguments.md#logging) for more details. + +## Request Dump and Replay + +You can dump all requests and replay them later for benchmarking or other purposes. + +To start dumping, use the following command to send a request to a server: +``` +python3 -m sglang.srt.managers.configure_logging --url http://localhost:30000 --dump-requests-folder /tmp/sglang_request_dump --dump-requests-threshold 100 +``` +The server will dump the requests into a pickle file for every 100 requests. + +To replay the request dump, use `scripts/playground/replay_request_dump.py`. + +## Crash Dump and Replay +Sometimes the server might crash, and you may want to debug the cause of the crash. +SGLang supports crash dumping, which will dump all requests from the 5 minutes before the crash, allowing you to replay the requests and debug the reason later. + +To enable crash dumping, use `--crash-dump-folder /tmp/crash_dump`. +To replay the crash dump, use `scripts/playground/replay_request_dump.py`. diff --git a/docs/advanced_features/pd_disaggregation.md b/docs/advanced_features/pd_disaggregation.md new file mode 100644 index 000000000..85a5db07e --- /dev/null +++ b/docs/advanced_features/pd_disaggregation.md @@ -0,0 +1,150 @@ +# PD Disaggregation + +## Why and What is PD Disaggregation? + +Large Language Model (LLM) inference comprises two distinct phases: **Prefill** and **Decode**. The Prefill phase is computation-intensive, processing the entire input sequence, while the Decode phase is memory-intensive, managing the Key-Value (KV) cache for token generation. Traditionally, these phases are handled within a unified engine, where combined scheduling of prefill and decode batches introduces inefficiencies. To address these challenges, we introduce **Prefill and Decoding (PD) Disaggregation** in SGLang. + +### Issues with Unified Scheduling + +The conventional unified engine, which processes prefill and decode batches together, results in two significant problems: + +1. **Prefill Interruption**: Incoming prefill batches frequently interrupt ongoing decode batches, causing substantial delays in token generation. +2. **DP Attention Imbalance**: In data-parallel (DP) attention, one DP worker may process a prefill batch while another handles a decode batch simultaneously, leading to increased decode latency. + +PD Disaggregation resolves these by separating the two stages, enabling tailored optimizations for each. + +For the design details, please refer to [link](https://docs.google.com/document/d/1rQXJwKd5b9b1aOzLh98mnyMhBMhlxXA5ATZTHoQrwvc/edit?tab=t.0). + +Currently, we support Mooncake and NIXL as the transfer engine. + +## Router Integration + +For deploying PD disaggregation at scale with load balancing and fault tolerance, SGLang provides a router. The router can distribute requests between prefill and decode instances using various routing policies. For detailed information on setting up routing with PD disaggregation, including configuration options and deployment patterns, see the [SGLang Router documentation](router.md#mode-3-prefill-decode-disaggregation). + + +## Mooncake +### Requirements + +```bash +uv pip install mooncake-transfer-engine +``` + +### Usage + +### Llama Single Node + +```bash +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode prefill --disaggregation-ib-device mlx5_roce0 +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode decode --port 30001 --base-gpu-id 1 --disaggregation-ib-device mlx5_roce0 +$ python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-ib-device ${device_name} --disaggregation-mode prefill --host ${local_ip} --port 30000 --trust-remote-code --dist-init-addr ${prefill_master_ip}:5000 --nnodes 2 --node-rank 0 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 +# prefill 1 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-ib-device ${device_name} --disaggregation-mode prefill --host ${local_ip} --port 30000 --trust-remote-code --dist-init-addr ${prefill_master_ip}:5000 --nnodes 2 --node-rank 1 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 +# decode 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-ib-device ${device_name} --disaggregation-mode decode --host ${local_ip} --port 30001 --trust-remote-code --dist-init-addr ${decode_master_ip}:5000 --nnodes 2 --node-rank 0 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 --max-running-requests 128 +# decode 1 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-ib-device ${device_name} --disaggregation-mode decode --host ${local_ip} --port 30001 --trust-remote-code --dist-init-addr ${decode_master_ip}:5000 --nnodes 2 --node-rank 1 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 --max-running-requests 128 +``` +### Advanced Configuration + +PD Disaggregation with Mooncake supports the following environment variables for fine-grained control over system behavior. + +#### Prefill Server Configuration +| Variable | Description | Default | +|:--------:|:-----------:|:--------: +| **`SGLANG_DISAGGREGATION_THREAD_POOL_SIZE`** | Controls the total number of worker threads for KVCache transfer operations per TP rank | A dynamic value calculated by `int(0.75 * os.cpu_count()) // 8)`, which is limited to be larger than 4 and less than 12 to ensure efficiency and prevent thread race conditions | +| **`SGLANG_DISAGGREGATION_QUEUE_SIZE`** | Sets the number of parallel transfer queues. KVCache transfer requests from multiple decode instances will be sharded into these queues so that they can share the threads and the transfer bandwidth at the same time. If it is set to `1`, then we transfer requests one by one according to fcfs strategy | `4` | +| **`SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT`** | Timeout (seconds) for receiving destination KV indices during request initialization | `300` | + +If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600` (10 minutes) to relax the timeout condition. +Please be aware that this setting will cause prefill instances to take a longer time to clean up the affected memory resources when a running decode node loses connection. + +#### Decode Server Configuration +| Variable | Description | Default | +|:--------:|:-----------:|:--------: +| **`SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL`** | Interval (seconds) between health checks to prefill bootstrap servers | `5.0` | +| **`SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE`** | Consecutive heartbeat failures before marking prefill server offline | `2` | +| **`SGLANG_DISAGGREGATION_WAITING_TIMEOUT`** | Timeout (seconds) for receiving KV Cache after request initialization | `300` | + +If a greater mean TTFT is acceptable, you can `export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600` (10 minutes) to relax the timeout condition. + + +## NIXL +### Requirements + +Install via pip. + +```bash +pip install nixl +``` + +Or build from source - may be required if you already have UCX installed. + +```bash +git clone https://github.com/ai-dynamo/nixl.git +cd nixl +pip install . --config-settings=setup-args="-Ducx_path=/path/to/ucx" +``` + + +### Usage + +### Llama Single Node + +```bash +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode prefill --disaggregation-transfer-backend nixl +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode decode --port 30001 --base-gpu-id 1 --disaggregation-transfer-backend nixl +$ python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend nixl --disaggregation-mode prefill --host ${local_ip} --port 30000 --trust-remote-code --dist-init-addr ${prefill_master_ip}:5000 --nnodes 2 --node-rank 0 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 +# prefill 1 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend nixl --disaggregation-mode prefill --host ${local_ip} --port 30000 --trust-remote-code --dist-init-addr ${prefill_master_ip}:5000 --nnodes 2 --node-rank 1 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 +# decode 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend nixl --disaggregation-mode decode --host ${local_ip} --port 30001 --trust-remote-code --dist-init-addr ${decode_master_ip}:5000 --nnodes 2 --node-rank 0 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 --max-running-requests 128 +# decode 1 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend nixl --disaggregation-mode decode --host ${local_ip} --port 30001 --trust-remote-code --dist-init-addr ${decode_master_ip}:5000 --nnodes 2 --node-rank 1 --tp-size 16 --dp-size 8 --enable-dp-attention --moe-a2a-backend deepep --mem-fraction-static 0.8 --max-running-requests 128 +``` + +## ASCEND + +### Usage + +Use ascend backend with [mf_adapter(download link)](https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com:443/sglang/mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl?AccessKeyId=HPUAXT4YM0U8JNTERLST&Expires=1783151861&Signature=3j10QDUjqk70enaq8lostYV2bEA%3D) and ASCEND_MF_STORE_URL being set + +```bash +pip install mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl --force-reinstall +export ASCEND_MF_STORE_URL="tcp://xxx.xx.xxx.xxx:xxxx" +``` +Use mooncake backend, more details can be found in mooncake section. +```bash +export ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE=true +``` + + +### Llama Single Node + +```bash +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode prefill --disaggregation-transfer-backend ascend +$ python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disaggregation-mode decode --port 30001 --base-gpu-id 1 --disaggregation-transfer-backend ascend +$ python -m sglang_router.launch_router --pd-disaggregation --prefill http://127.0.0.1:30000 --decode http://127.0.0.1:30001 --host 0.0.0.0 --port 8000 +``` + +### DeepSeek Multi-Node + +```bash +# prefill 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend ascend --disaggregation-mode prefill --host ${local_ip} --port 30000 --trust-remote-code --dist-init-addr ${prefill_master_ip}:5000 --nnodes 1 --node-rank 0 --tp-size 16 +# decode 0 +$ python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --disaggregation-transfer-backend ascend --disaggregation-mode decode --host ${local_ip} --port 30001 --trust-remote-code --dist-init-addr ${decode_master_ip}:5000 --nnodes 1 --node-rank 0 --tp-size 16 +``` diff --git a/docs/advanced_features/quantization.md b/docs/advanced_features/quantization.md new file mode 100644 index 000000000..3a229f83d --- /dev/null +++ b/docs/advanced_features/quantization.md @@ -0,0 +1,152 @@ +# Quantization + +SGLang supports various quantization methods, including offline quantization and online dynamic quantization. + +Offline quantization loads pre-quantized model weights directly during inference. This is required for quantization methods +such as GPTQ and AWQ, which collect and pre-compute various statistics from the original weights using the calibration dataset. + +Online quantization dynamically computes scaling parameters—such as the maximum/minimum values of model weights—during runtime. +Like NVIDIA FP8 training's [delayed scaling](https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/examples/fp8_primer.html#Mixed-precision-training-with-FP8) mechanism, online quantization calculates the appropriate scaling factors +on-the-fly to convert high-precision weights into a lower-precision format. + +**Note: For better performance, usability and convenience, offline quantization is recommended over online quantization.** + +If you use a pre-quantized model, do not add `--quantization` to enable online quantization at the same time. +For popular pre-quantized models, please visit [ModelCloud](https://huggingface.co/collections/ModelCloud/vortex-673743382af0a52b2a8b9fe2) +or [NeuralMagic](https://huggingface.co/collections/neuralmagic) collections on HF for some +popular quality validated quantized models. Quantized models must be validated via benchmarks post-quantization +to guard against abnormal quantization loss regressions. + +## Offline Quantization + +To load already quantized models, simply load the model weights and config. **Again, if the model has been quantized offline, +there's no need to add `--quantization` argument when starting the engine. The quantization method will be parsed from the +downloaded Hugging Face config. For example, DeepSeek V3/R1 models are already in FP8, so do not add redundant parameters.** + +```bash +python3 -m sglang.launch_server \ + --model-path hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 \ + --port 30000 --host 0.0.0.0 +``` + +Take note, if your model is **per-channel quantized (INT8 or FP8) with per-token dynamic quantization activation**, you can opt to include `--quantization w8a8_int8` or `--quantization w8a8_fp8` to invoke the corresponding CUTLASS int8_kernel or fp8_kernel in sgl-kernel. This action will ignore the Hugging Face config's quantization settings. For instance, with `neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic`, if you execute with `--quantization w8a8_fp8`, the system will use the `W8A8Fp8Config` from SGLang to invoke the sgl-kernel, rather than the `CompressedTensorsConfig` for vLLM kernels. + +```bash +python3 -m sglang.launch_server \ + --model-path neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic \ + --quantization w8a8_fp8 \ + --port 30000 --host 0.0.0.0 +``` + +### Examples of Offline Model Quantization + +#### Using [GPTQModel](https://github.com/ModelCloud/GPTQModel) + +```bash +# install +pip install gptqmodel --no-build-isolation -v +``` + +```py +from datasets import load_dataset +from gptqmodel import GPTQModel, QuantizeConfig + +model_id = "meta-llama/Llama-3.2-1B-Instruct" +quant_path = "Llama-3.2-1B-Instruct-gptqmodel-4bit" + +calibration_dataset = load_dataset( + "allenai/c4", data_files="en/c4-train.00001-of-01024.json.gz", + split="train" + ).select(range(1024))["text"] + +quant_config = QuantizeConfig(bits=4, group_size=128) # quantization config +model = GPTQModel.load(model_id, quant_config) # load model + +model.quantize(calibration_dataset, batch_size=2) # quantize +model.save(quant_path) # save model +``` + +#### Using [LLM Compressor](https://github.com/vllm-project/llm-compressor/) + +```bash +# install +pip install llmcompressor +``` + +Here, we take quantize `meta-llama/Meta-Llama-3-8B-Instruct` to `FP8` as an example to elaborate on how to do offline quantization. + +```python +from transformers import AutoTokenizer +from llmcompressor.transformers import SparseAutoModelForCausalLM +from llmcompressor.transformers import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier + +# Step 1: Load the original model. +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" + +model = SparseAutoModelForCausalLM.from_pretrained( + MODEL_ID, device_map="auto", torch_dtype="auto") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +# Step 2: Perform offline quantization. +# Step 2.1: Configure the simple PTQ quantization. +recipe = QuantizationModifier( + targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"]) + +# Step 2.2: Apply the quantization algorithm. +oneshot(model=model, recipe=recipe) + +# Step 3: Save the model. +SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic" +model.save_pretrained(SAVE_DIR) +tokenizer.save_pretrained(SAVE_DIR) +``` + +Then, you can directly use the quantized model with `SGLang`, by using the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path $PWD/Meta-Llama-3-8B-Instruct-FP8-Dynamic \ + --port 30000 --host 0.0.0.0 +``` + +## Online Quantization + +To enable online quantization, you can simply specify `--quantization` in the command line. For example, you can launch the server with the following command to enable `FP8` quantization for model `meta-llama/Meta-Llama-3.1-8B-Instruct`: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --quantization fp8 \ + --port 30000 --host 0.0.0.0 +``` + +Our team is working on supporting more online quantization methods. SGLang will soon support methods including but not limited to `["awq", "gptq", "marlin", "gptq_marlin", "awq_marlin", "bitsandbytes", "gguf"]`. + +SGLang also supports quantization methods based on [torchao](https://github.com/pytorch/ao). You can simply specify `--torchao-config` in the command line to support this feature. For example, if you want to enable `int4wo-128` for model `meta-llama/Meta-Llama-3.1-8B-Instruct`, you can launch the server with the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int4wo-128 \ + --port 30000 --host 0.0.0.0 +``` + +SGLang supports the following quantization methods based on torchao `["int8dq", "int8wo", "fp8wo", "fp8dq-per_tensor", "fp8dq-per_row", "int4wo-32", "int4wo-64", "int4wo-128", "int4wo-256"]`. + +Note: According to [this issue](https://github.com/sgl-project/sglang/issues/2219#issuecomment-2561890230), `"int8dq"` method currently has some bugs when using together with cuda graph capture. So we suggest to disable cuda graph capture when using `"int8dq"` method. Namely, please use the following command: + +```bash +python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --torchao-config int8dq \ + --disable-cuda-graph \ + --port 30000 --host 0.0.0.0 +``` + +## Reference + +- [GPTQModel](https://github.com/ModelCloud/GPTQModel) +- [LLM Compressor](https://github.com/vllm-project/llm-compressor/) +- [Torchao: PyTorch Architecture Optimization](https://github.com/pytorch/ao) +- [vLLM Quantization](https://docs.vllm.ai/en/latest/quantization/) diff --git a/docs/advanced_features/router.md b/docs/advanced_features/router.md new file mode 100644 index 000000000..4aba99f37 --- /dev/null +++ b/docs/advanced_features/router.md @@ -0,0 +1,445 @@ +# SGLang Router + +The SGLang Router is a high-performance request distribution system that routes inference requests across multiple SGLang runtime instances. It features cache-aware load balancing, fault tolerance, and support for advanced deployment patterns including data parallelism and prefill-decode disaggregation. + +## Key Features + +- **Cache-Aware Load Balancing**: Optimizes cache utilization while maintaining balanced load distribution +- **Multiple Routing Policies**: Choose from random, round-robin, cache-aware, or power-of-two policies +- **Fault Tolerance**: Automatic retry and circuit breaker mechanisms for resilient operation +- **Dynamic Scaling**: Add or remove workers at runtime without service interruption +- **Kubernetes Integration**: Native service discovery and pod management +- **Prefill-Decode Disaggregation**: Support for disaggregated serving load balancing +- **Prometheus Metrics**: Built-in observability and monitoring + +## Installation + +```bash +pip install sglang-router +``` + +## Quick Start + +To see all available options: + +```bash +python -m sglang_router.launch_server --help # Co-launch router and workers +python -m sglang_router.launch_router --help # Launch router only +``` + +## Deployment Modes + +The router supports three primary deployment patterns: + +1. **Co-launch Mode**: Router and workers launch together (simplest for single-node deployments) +2. **Separate Launch Mode**: Router and workers launch independently (best for multi-node setups) +3. **Prefill-Decode Disaggregation**: Specialized mode for disaggregated serving + +### Mode 1: Co-launch Router and Workers + +This mode launches both the router and multiple worker instances in a single command. It's the simplest deployment option and replaces the `--dp-size` argument of SGLang Runtime. + +```bash +# Launch router with 4 workers +python -m sglang_router.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --dp-size 4 \ + --host 0.0.0.0 \ + --port 30000 +``` + +#### Sending Requests + +Once the server is ready, send requests to the router endpoint: + +```python +import requests + +# Using the /generate endpoint +url = "http://localhost:30000/generate" +data = { + "text": "What is the capital of France?", + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 100 + } +} + +response = requests.post(url, json=data) +print(response.json()) + +# OpenAI-compatible endpoint +url = "http://localhost:30000/v1/chat/completions" +data = { + "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "What is the capital of France?"}] +} + +response = requests.post(url, json=data) +print(response.json()) +``` + +### Mode 2: Separate Launch Mode + +This mode is ideal for multi-node deployments where workers run on different machines. + +#### Step 1: Launch Workers + +On each worker node: + +```bash +# Worker node 1 +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 8000 + +# Worker node 2 +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 8001 +``` + +#### Step 2: Launch Router + +On the router node: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --host 0.0.0.0 \ + --port 30000 \ + --policy cache_aware # or random, round_robin, power_of_two +``` + +### Mode 3: Prefill-Decode Disaggregation + +This advanced mode separates prefill and decode operations for optimized performance: + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://prefill1:8000 9000 \ + --prefill http://prefill2:8001 9001 \ + --decode http://decode1:8002 \ + --decode http://decode2:8003 \ + --prefill-policy cache_aware \ + --decode-policy round_robin +``` + +#### Understanding --prefill Arguments + +The `--prefill` flag accepts URLs with optional bootstrap ports: +- `--prefill http://server:8000` - No bootstrap port +- `--prefill http://server:8000 9000` - Bootstrap port 9000 +- `--prefill http://server:8000 none` - Explicitly no bootstrap port + +#### Policy Inheritance in PD Mode + +The router intelligently handles policy configuration for prefill and decode nodes: + +1. **Only `--policy` specified**: Both prefill and decode nodes use this policy +2. **`--policy` and `--prefill-policy` specified**: Prefill nodes use `--prefill-policy`, decode nodes use `--policy` +3. **`--policy` and `--decode-policy` specified**: Prefill nodes use `--policy`, decode nodes use `--decode-policy` +4. **All three specified**: Prefill nodes use `--prefill-policy`, decode nodes use `--decode-policy` (main `--policy` is ignored) + +Example with mixed policies: +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill http://prefill1:8000 + --prefill http://prefill2:8000 \ + --decode http://decode1:8001 + --decode http://decode2:8001 \ + --policy round_robin \ + --prefill-policy cache_aware # Prefill uses cache_aware and decode uses round_robin from --policy +``` + +#### PD Mode with Service Discovery + +For Kubernetes deployments with separate prefill and decode server pools: + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --service-discovery \ + --prefill-selector app=prefill-server tier=gpu \ + --decode-selector app=decode-server tier=cpu \ + --service-discovery-namespace production \ + --prefill-policy cache_aware \ + --decode-policy round_robin +``` + +## Dynamic Scaling + +The router supports runtime scaling through REST APIs: + +### Adding Workers + +```bash +# Launch a new worker +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --port 30001 + +# Add it to the router +curl -X POST "http://localhost:30000/add_worker?url=http://127.0.0.1:30001" +``` + +### Removing Workers + +```bash +curl -X POST "http://localhost:30000/remove_worker?url=http://127.0.0.1:30001" +``` + +**Note**: When using cache-aware routing, removed workers are cleanly evicted from the routing tree and request queues. + +## Fault Tolerance + +The router includes comprehensive fault tolerance mechanisms: + +### Retry Configuration + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --retry-max-retries 3 \ + --retry-initial-backoff-ms 100 \ + --retry-max-backoff-ms 10000 \ + --retry-backoff-multiplier 2.0 \ + --retry-jitter-factor 0.1 +``` + +### Circuit Breaker + +Protects against cascading failures: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --cb-failure-threshold 5 \ + --cb-success-threshold 2 \ + --cb-timeout-duration-secs 30 \ + --cb-window-duration-secs 60 +``` + +**Behavior**: +- Worker is marked unhealthy after `cb-failure-threshold` consecutive failures +- Returns to service after `cb-success-threshold` successful health checks +- Circuit breaker can be disabled with `--disable-circuit-breaker` + +## Routing Policies + +The router supports multiple routing strategies: + +### 1. Random Routing +Distributes requests randomly across workers. + +```bash +--policy random +``` + +### 2. Round-Robin Routing +Cycles through workers in order. + +```bash +--policy round_robin +``` + +### 3. Power of Two Choices +Samples two workers and routes to the less loaded one. + +```bash +--policy power_of_two +``` + +### 4. Cache-Aware Load Balancing (Default) + +The most sophisticated policy that combines cache optimization with load balancing: + +```bash +--policy cache_aware \ +--cache-threshold 0.5 \ +--balance-abs-threshold 32 \ +--balance-rel-threshold 1.0001 +``` + +#### How It Works + +1. **Load Assessment**: Checks if the system is balanced + - Imbalanced if: `(max_load - min_load) > balance_abs_threshold` AND `max_load > balance_rel_threshold * min_load` + +2. **Routing Decision**: + - **Balanced System**: Uses cache-aware routing + - Routes to worker with highest prefix match if match > `cache_threshold` + - Otherwise routes to worker with most available cache capacity + - **Imbalanced System**: Uses shortest queue routing to the least busy worker + +3. **Cache Management**: + - Maintains approximate radix trees per worker + - Periodically evicts LRU entries based on `--eviction-interval-secs` and `--max-tree-size` + +### Data Parallelism Aware Routing + +Enables fine-grained control over data parallel replicas: + +```bash +--dp-aware \ +--api-key your_api_key # Required for worker authentication +``` + +This mode coordinates with SGLang's DP controller for optimized request distribution across data parallel ranks. + +## Configuration Reference + +### Core Settings + +| Parameter | Type | Default | Description | +| --------------------------- | ---- | ----------- | --------------------------------------------------------------- | +| `--host` | str | 127.0.0.1 | Router server host address | +| `--port` | int | 30000 | Router server port | +| `--worker-urls` | list | [] | Worker URLs for separate launch mode | +| `--policy` | str | cache_aware | Routing policy (random, round_robin, cache_aware, power_of_two) | +| `--max-concurrent-requests` | int | 64 | Maximum concurrent requests (rate limiting) | +| `--request-timeout-secs` | int | 600 | Request timeout in seconds | +| `--max-payload-size` | int | 256MB | Maximum request payload size | + +### Cache-Aware Routing Parameters + +| Parameter | Type | Default | Description | +| -------------------------- | ----- | -------- | ------------------------------------------------------ | +| `--cache-threshold` | float | 0.5 | Minimum prefix match ratio for cache routing (0.0-1.0) | +| `--balance-abs-threshold` | int | 32 | Absolute load difference threshold | +| `--balance-rel-threshold` | float | 1.0001 | Relative load ratio threshold | +| `--eviction-interval-secs` | int | 60 | Seconds between cache eviction cycles | +| `--max-tree-size` | int | 16777216 | Maximum nodes in routing tree | + +### Fault Tolerance Parameters + +| Parameter | Type | Default | Description | +| ---------------------------- | ----- | ------- | ------------------------------------- | +| `--retry-max-retries` | int | 3 | Maximum retry attempts per request | +| `--retry-initial-backoff-ms` | int | 100 | Initial retry backoff in milliseconds | +| `--retry-max-backoff-ms` | int | 10000 | Maximum retry backoff in milliseconds | +| `--retry-backoff-multiplier` | float | 2.0 | Backoff multiplier between retries | +| `--retry-jitter-factor` | float | 0.1 | Random jitter factor for retries | +| `--disable-retries` | flag | False | Disable retry mechanism | +| `--cb-failure-threshold` | int | 5 | Failures before circuit opens | +| `--cb-success-threshold` | int | 2 | Successes to close circuit | +| `--cb-timeout-duration-secs` | int | 30 | Circuit breaker timeout duration | +| `--cb-window-duration-secs` | int | 60 | Circuit breaker window duration | +| `--disable-circuit-breaker` | flag | False | Disable circuit breaker | + +### Prefill-Decode Disaggregation Parameters + +| Parameter | Type | Default | Description | +| --------------------------------- | ---- | ------- | ----------------------------------------------------- | +| `--pd-disaggregation` | flag | False | Enable PD disaggregated mode | +| `--prefill` | list | [] | Prefill server URLs with optional bootstrap ports | +| `--decode` | list | [] | Decode server URLs | +| `--prefill-policy` | str | None | Routing policy for prefill nodes (overrides --policy) | +| `--decode-policy` | str | None | Routing policy for decode nodes (overrides --policy) | +| `--worker-startup-timeout-secs` | int | 300 | Timeout for worker startup | +| `--worker-startup-check-interval` | int | 10 | Interval between startup checks | + +### Kubernetes Integration + +| Parameter | Type | Default | Description | +| ------------------------------- | ---- | ------------------------ | ---------------------------------------------------- | +| `--service-discovery` | flag | False | Enable Kubernetes service discovery | +| `--selector` | list | [] | Label selector for workers (key1=value1 key2=value2) | +| `--prefill-selector` | list | [] | Label selector for prefill servers in PD mode | +| `--decode-selector` | list | [] | Label selector for decode servers in PD mode | +| `--service-discovery-port` | int | 80 | Port for discovered pods | +| `--service-discovery-namespace` | str | None | Kubernetes namespace to watch | +| `--bootstrap-port-annotation` | str | sglang.ai/bootstrap-port | Annotation for bootstrap ports | + +### Observability + +| Parameter | Type | Default | Description | +| ---------------------- | ---- | --------- | ----------------------------------------------------- | +| `--prometheus-port` | int | 29000 | Prometheus metrics port | +| `--prometheus-host` | str | 127.0.0.1 | Prometheus metrics host | +| `--log-dir` | str | None | Directory for log files | +| `--log-level` | str | info | Logging level (debug, info, warning, error, critical) | +| `--request-id-headers` | list | None | Custom headers for request tracing | + +### CORS Configuration + +| Parameter | Type | Default | Description | +| ------------------------ | ---- | ------- | -------------------- | +| `--cors-allowed-origins` | list | [] | Allowed CORS origins | + +## Advanced Features + +### Kubernetes Service Discovery + +Automatically discover and manage workers in Kubernetes: + +#### Standard Mode +```bash +python -m sglang_router.launch_router \ + --service-discovery \ + --selector app=sglang-worker env=prod \ + --service-discovery-namespace production \ + --service-discovery-port 8000 +``` + +#### Prefill-Decode Disaggregation Mode +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --service-discovery \ + --prefill-selector app=prefill-server env=prod \ + --decode-selector app=decode-server env=prod \ + --service-discovery-namespace production +``` + +**Note**: The `--bootstrap-port-annotation` (default: `sglang.ai/bootstrap-port`) is used to discover bootstrap ports for prefill servers in PD mode. Prefill pods should have this annotation set to their bootstrap port value. + +### Prometheus Metrics + +Expose metrics for monitoring: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --prometheus-port 29000 \ + --prometheus-host 0.0.0.0 +``` + +Metrics available at `http://localhost:29000/metrics` + +### Request Tracing + +Enable request ID tracking: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --request-id-headers x-request-id x-trace-id +``` + +## Troubleshooting + +### Common Issues + +1. **Workers not connecting**: Ensure workers are fully initialized before starting the router. Use `--worker-startup-timeout-secs` to increase wait time. + +2. **High latency**: Check if cache-aware routing is causing imbalance. Try adjusting `--balance-abs-threshold` and `--balance-rel-threshold`. + +3. **Memory growth**: Reduce `--max-tree-size` or decrease `--eviction-interval-secs` for more aggressive cache cleanup. + +4. **Circuit breaker triggering frequently**: Increase `--cb-failure-threshold` or extend `--cb-window-duration-secs`. + +### Debug Mode + +Enable detailed logging: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8001 \ + --log-level debug \ + --log-dir ./router_logs +``` diff --git a/docs/advanced_features/separate_reasoning.ipynb b/docs/advanced_features/separate_reasoning.ipynb new file mode 100644 index 000000000..0c20c5a08 --- /dev/null +++ b/docs/advanced_features/separate_reasoning.ipynb @@ -0,0 +1,381 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reasoning Parser\n", + "\n", + "SGLang supports parsing reasoning content out from \"normal\" content for reasoning models such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1).\n", + "\n", + "## Supported Models & Parsers\n", + "\n", + "| Model | Reasoning tags | Parser | Notes |\n", + "|---------|-----------------------------|------------------|-------|\n", + "| [DeepSeek‑R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d) | `` … `` | `deepseek-r1` | Supports all variants (R1, R1-0528, R1-Distill) |\n", + "| [DeepSeek‑V3.1](https://huggingface.co/deepseek-ai/DeepSeek-V3.1) | `` … `` | `deepseek-v3` | Supports `thinking` parameter |\n", + "| [Standard Qwen3 models](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `` … `` | `qwen3` | Supports `enable_thinking` parameter |\n", + "| [Qwen3-Thinking models](https://huggingface.co/Qwen/Qwen3-235B-A22B-Thinking-2507) | `` … `` | `qwen3` or `qwen3-thinking` | Always generates thinking content |\n", + "| [Kimi models](https://huggingface.co/moonshotai/models) | `◁think▷` … `◁/think▷` | `kimi` | Uses special thinking delimiters |\n", + "| [GPT OSS](https://huggingface.co/openai/gpt-oss-120b) | `<\\|channel\\|>analysis<\\|message\\|>` … `<\\|end\\|>` | `gpt-oss` | N/A |\n", + "### Model-Specific Behaviors\n", + "\n", + "**DeepSeek-R1 Family:**\n", + "- DeepSeek-R1: No `` start tag, jumps directly to thinking content\n", + "- DeepSeek-R1-0528: Generates both `` start and `` end tags\n", + "- Both are handled by the same `deepseek-r1` parser\n", + "\n", + "**DeepSeek-V3 Family:**\n", + "- DeepSeek-V3.1: Hybrid model supporting both thinking and non-thinking modes, use the `deepseek-v3` parser and `thinking` parameter (NOTE: not `enable_thinking`)\n", + "\n", + "**Qwen3 Family:**\n", + "- Standard Qwen3 (e.g., Qwen3-2507): Use `qwen3` parser, supports `enable_thinking` in chat templates\n", + "- Qwen3-Thinking (e.g., Qwen3-235B-A22B-Thinking-2507): Use `qwen3` or `qwen3-thinking` parser, always thinks\n", + "\n", + "**Kimi:**\n", + "- Kimi: Uses special `◁think▷` and `◁/think▷` tags\n", + "\n", + "**GPT OSS:**\n", + "- GPT OSS: Uses special `<|channel|>analysis<|message|>` and `<|end|>` tags" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage\n", + "\n", + "### Launching the Server" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the `--reasoning-parser` option." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from openai import OpenAI\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `--reasoning-parser` defines the parser used to interpret responses." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI Compatible API\n", + "\n", + "Using the OpenAI compatible API, the contract follows the [DeepSeek API design](https://api-docs.deepseek.com/guides/reasoning_model) established with the release of DeepSeek-R1:\n", + "\n", + "- `reasoning_content`: The content of the CoT.\n", + "- `content`: The content of the final answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize OpenAI-like client\n", + "client = OpenAI(api_key=\"None\", base_url=f\"http://0.0.0.0:{port}/v1\")\n", + "model_name = client.models.list().data[0].id\n", + "\n", + "messages = [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"What is 1+3?\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Non-Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(response_non_stream.choices[0].message.reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Streaming Request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content += chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Optionally, you can buffer the reasoning content to the last reasoning chunk (or the first chunk after the reasoning content)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=True, # Non-streaming\n", + " extra_body={\"separate_reasoning\": True, \"stream_reasoning\": False},\n", + ")\n", + "\n", + "reasoning_content = \"\"\n", + "content = \"\"\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " content += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.reasoning_content:\n", + " reasoning_content += chunk.choices[0].delta.reasoning_content\n", + "\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_content)\n", + "\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The reasoning separation is enable by default when specify . \n", + "**To disable it, set the `separate_reasoning` option to `False` in request.**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0.6,\n", + " top_p=0.95,\n", + " stream=False, # Non-streaming\n", + " extra_body={\"separate_reasoning\": False},\n", + ")\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(response_non_stream.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### SGLang Native API " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + ")\n", + "\n", + "gen_url = f\"http://localhost:{port}/generate\"\n", + "gen_data = {\n", + " \"text\": input,\n", + " \"sampling_params\": {\n", + " \"skip_special_tokens\": False,\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + " },\n", + "}\n", + "gen_response = requests.post(gen_url, json=gen_data).json()[\"text\"]\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(gen_response)\n", + "\n", + "parse_url = f\"http://localhost:{port}/separate_reasoning\"\n", + "separate_reasoning_data = {\n", + " \"text\": gen_response,\n", + " \"reasoning_parser\": \"deepseek-r1\",\n", + "}\n", + "separate_reasoning_response_json = requests.post(\n", + " parse_url, json=separate_reasoning_data\n", + ").json()\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(separate_reasoning_response_json[\"reasoning_text\"])\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(separate_reasoning_response_json[\"text\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sglang as sgl\n", + "from sglang.srt.parser.reasoning_parser import ReasoningParser\n", + "from sglang.utils import print_highlight\n", + "\n", + "llm = sgl.Engine(model_path=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "tokenizer = AutoTokenizer.from_pretrained(\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\")\n", + "input = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=True,\n", + ")\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"skip_special_tokens\": False,\n", + " \"temperature\": 0.6,\n", + " \"top_p\": 0.95,\n", + "}\n", + "result = llm.generate(prompt=input, sampling_params=sampling_params)\n", + "\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print_highlight(\"==== Original Output ====\")\n", + "print_highlight(generated_text)\n", + "\n", + "parser = ReasoningParser(\"deepseek-r1\")\n", + "reasoning_text, text = parser.parse_non_stream(generated_text)\n", + "print_highlight(\"==== Reasoning ====\")\n", + "print_highlight(reasoning_text)\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Supporting New Reasoning Model Schemas\n", + "\n", + "For future reasoning models, you can implement the reasoning parser as a subclass of `BaseReasoningFormatDetector` in `python/sglang/srt/reasoning_parser.py` and specify the reasoning parser for new reasoning model schemas accordingly." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md new file mode 100644 index 000000000..873fa8b05 --- /dev/null +++ b/docs/advanced_features/server_arguments.md @@ -0,0 +1,318 @@ +# Server Arguments + +This page provides a list of server arguments used in the command line to configure the behavior +and performance of the language model server during deployment. These arguments enable users to +customize key aspects of the server, including model selection, parallelism policies, +memory management, and optimization techniques. +You can find all arguments by `python3 -m sglang.launch_server --help` + +## Common launch commands + +- To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 2 + ``` + +- To enable multi-GPU data parallelism, add `--dp 2`. Data parallelism is better for throughput if there is enough memory. It can also be used together with tensor parallelism. The following command uses 4 GPUs in total. We recommend [SGLang Router](../advanced_features/router.md) for data parallelism. + + ```bash + python -m sglang_router.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --dp 2 --tp 2 + ``` + +- If you see out-of-memory errors during serving, try to reduce the memory usage of the KV cache pool by setting a smaller value of `--mem-fraction-static`. The default value is `0.9`. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --mem-fraction-static 0.7 + ``` + +- See [hyperparameter tuning](hyperparameter_tuning.md) on tuning hyperparameters for better performance. +- For docker and Kubernetes runs, you need to set up shared memory which is used for communication between processes. See `--shm-size` for docker and `/dev/shm` size update for Kubernetes manifests. +- If you see out-of-memory errors during prefill for long prompts, try to set a smaller chunked prefill size. + + ```bash + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --chunked-prefill-size 4096 + ``` + +- To enable `torch.compile` acceleration, add `--enable-torch-compile`. It accelerates small models on small batch sizes. By default, the cache path is located at `/tmp/torchinductor_root`, you can customize it using environment variable `TORCHINDUCTOR_CACHE_DIR`. For more details, please refer to [PyTorch official documentation](https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html) and [Enabling cache for torch.compile](https://docs.sglang.ai/backend/hyperparameter_tuning.html#enabling-cache-for-torch-compile). +- To enable torchao quantization, add `--torchao-config int4wo-128`. It supports other [quantization strategies (INT8/FP8)](https://github.com/sgl-project/sglang/blob/v0.3.6/python/sglang/srt/server_args.py#L671) as well. +- To enable fp8 weight quantization, add `--quantization fp8` on a fp16 checkpoint or directly load a fp8 checkpoint without specifying any arguments. +- To enable fp8 kv cache quantization, add `--kv-cache-dtype fp8_e5m2`. +- If the model does not have a chat template in the Hugging Face tokenizer, you can specify a [custom chat template](../references/custom_chat_template.md). +- To run tensor parallelism on multiple nodes, add `--nnodes 2`. If you have two nodes with two GPUs on each node and want to run TP=4, let `sgl-dev-0` be the hostname of the first node and `50000` be an available port, you can use the following commands. If you meet deadlock, please try to add `--disable-cuda-graph` + + ```bash + # Node 0 + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 4 --dist-init-addr sgl-dev-0:50000 --nnodes 2 --node-rank 0 + + # Node 1 + python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --tp 4 --dist-init-addr sgl-dev-0:50000 --nnodes 2 --node-rank 1 + ``` + +Please consult the documentation below and [server_args.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/server_args.py) to learn more about the arguments you may provide when launching a server. + +## Model and tokenizer + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--model-path` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | None | +| `--tokenizer-path` | The path of the tokenizer. | None | +| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | auto | +| `--skip-tokenizer-init` | If set, skip init tokenizer and pass input_ids in generate request. | False | +| `--load-format` | The format of the model weights to load. 'auto' will try to load the weights in the safetensors format and fall back to the pytorch bin format if safetensors format is not available. 'pt' will load the weights in the pytorch bin format. 'safetensors' will load the weights in the safetensors format. 'npcache' will load the weights in pytorch format and store a numpy cache to speed up the loading. 'dummy' will initialize the weights with random values, which is mainly for profiling. 'gguf' will load the weights in the gguf format. 'bitsandbytes' will load the weights using bitsandbytes quantization. 'layered' loads weights layer by layer so that one can quantize a layer before loading another to make the peak memory envelope smaller. | auto | +| `--trust-remote-code` | Whether or not to allow for custom models defined on the Hub in their own modeling files. | False | +| `--context-length` | The model's maximum context length. Defaults to None (will use the value from the model's config.json instead). | None | +| `--is-embedding` | Whether to use a CausalLM as an embedding model. | False | +| `--enable-multimodal` | Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen. | None | +| `--revision` | The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version. | None | +| `--model-impl` | Which implementation of the model to use. 'auto' will try to use the SGLang implementation if it exists and fall back to the Transformers implementation if no SGLang implementation is available. 'sglang' will use the SGLang model implementation. 'transformers' will use the Transformers model implementation. | auto | + +## HTTP server + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--host` | The host address for the server. | 127.0.0.1 | +| `--port` | The port number for the server. | 30000 | +| `--skip-server-warmup` | If set, skip the server warmup process. | False | +| `--warmups` | Warmup configurations. | None | +| `--nccl-port` | The port for NCCL initialization. | None | + +## Quantization and data type + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--dtype` | Data type for model weights and activations. 'auto' will use FP16 precision for FP32 and FP16 models, and BF16 precision for BF16 models. 'half' for FP16. Recommended for AWQ quantization. 'float16' is the same as 'half'. 'bfloat16' for a balance between precision and range. 'float' is shorthand for FP32 precision. 'float32' for FP32 precision. | auto | +| `--quantization` | The quantization method. | None | +| `--quantization-param-path` | Path to the JSON file containing the KV cache scaling factors. This should generally be supplied, when KV cache dtype is FP8. Otherwise, KV cache scaling factors default to 1.0, which may cause accuracy issues. | None | +| `--kv-cache-dtype` | Data type for kv cache storage. 'auto' will use model data type. 'fp8_e5m2' and 'fp8_e4m3' is supported for CUDA 11.8+. | auto | + +## Memory and scheduling + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--mem-fraction-static` | The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors. | None | +| `--max-running-requests` | The maximum number of running requests. | None | +| `--max-total-tokens` | The maximum number of tokens in the memory pool. If not specified, it will be automatically calculated based on the memory usage fraction. This option is typically used for development and debugging purposes. | None | +| `--chunked-prefill-size` | The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill. | None | +| `--max-prefill-tokens` | The maximum number of tokens in a prefill batch. The real bound will be the maximum of this value and the model's maximum context length. | 16384 | +| `--schedule-policy` | The scheduling policy of the requests. | fcfs | +| `--schedule-conservativeness` | How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently. | 1.0 | +| `--cpu-offload-gb` | How many GBs of RAM to reserve for CPU offloading. | 0 | +| `--page-size` | The number of tokens in a page. | 1 | + +## Runtime options + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--device` | The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu'). Defaults to auto-detection if not specified. | None | +| `--tp-size` | The tensor parallelism size. | 1 | +| `--pp-size` | The pipeline parallelism size. | 1 | +| `--max-micro-batch-size` | The maximum micro batch size in pipeline parallelism. | None | +| `--stream-interval` | The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher. | 1 | +| `--stream-output` | Whether to output as a sequence of disjoint segments. | False | +| `--random-seed` | The random seed. | None | +| `--constrained-json-whitespace-pattern` | Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model generate consecutive whitespaces, set the pattern to [\n\t ]*. | None | +| `--watchdog-timeout` | Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging. | 300 | +| `--dist-timeout` | Set timeout for torch.distributed initialization. | None | +| `--download-dir` | Model download directory for huggingface. | None | +| `--base-gpu-id` | The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine. | 0 | +| `--gpu-id-step` | The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,.... | 1 | +| `--sleep-on-idle` | Reduce CPU usage when sglang is idle. | False | + +## Logging + +| Arguments | Description | Defaults | +|---------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +| `--log-level` | The logging level of all loggers. | info | +| `--log-level-http` | The logging level of HTTP server. If not set, reuse --log-level by default. | None | +| `--log-requests` | Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level. | False | +| `--log-requests-level` | 0: Log metadata (no sampling parameters). 1: Log metadata and sampling parameters. 2: Log metadata, sampling parameters and partial input/output. 3: Log every input/output. | 0 | +| `--show-time-cost` | Show time cost of custom marks. | False | +| `--enable-metrics` | Enable log prometheus metrics. | False | +| `--bucket-time-to-first-token` | The buckets of time to first token, specified as a list of floats. | None | +| `--bucket-inter-token-latency` | The buckets of inter-token latency, specified as a list of floats. | None | +| `--bucket-e2e-request-latency` | The buckets of end-to-end request latency, specified as a list of floats. | None | +| `--collect-tokens-histogram` | Collect prompt/generation tokens histogram. | False | +| `--kv-events-config` | Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used. | None | +| `--decode-log-interval` | The log interval of decode batch. | 40 | +| `--enable-request-time-stats-logging` | Enable per request time stats logging. | False | +| `--prompt-tokens-buckets` | The buckets rule of prompt tokens. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'customer ...' uses custom bucket values (e.g., 'customer 10 50 100 500'). | None | +| `--generation-tokens-buckets` | The buckets rule of prompt tokens. Supports 3 rule types: 'default' uses predefined buckets; 'tse ' generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets [984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'customer ...' uses custom bucket values (e.g., 'customer 10 50 100 500'). | None | + +## API related + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--api-key` | Set API key of the server. It is also used in the OpenAI API compatible server. | None | +| `--served-model-name` | Override the model name returned by the v1/models endpoint in OpenAI API server. | None | +| `--chat-template` | The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server. | None | +| `--completion-template` | The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently. | None | +| `--file-storage-path` | The path of the file storage in backend. | sglang_storage | +| `--enable-cache-report` | Return number of cached tokens in usage.prompt_tokens_details for each openai request. | False | +| `--reasoning-parser` | Specify the parser for reasoning models, supported parsers are: {list(ReasoningParser.DetectorMap.keys())}. | None | +| `--tool-call-parser` | Specify the parser for handling tool-call interactions. Options include: 'qwen25', 'mistral', 'llama3', 'deepseekv3', 'pythonic', 'kimi_k2', 'qwen3_coder', 'glm45', and 'step3'. | None | + +## Data parallelism + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--dp-size` | The data parallelism size. | 1 | +| `--load-balance-method` | The load balancing strategy for data parallelism. Options include: 'round_robin', 'minimum_tokens'. The Minimum Token algorithm can only be used when DP attention is applied. This algorithm performs load balancing based on the real-time token load of the DP workers. | round_robin | + +## Multi-node distributed serving + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--dist-init-addr` | The host address for initializing distributed backend (e.g., `192.168.0.2:25000`). | None | +| `--nnodes` | The number of nodes. | 1 | +| `--node-rank` | The node rank. | 0 | + +## Model override args in JSON + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--json-model-override-args` | A dictionary in JSON string format used to override default model configurations. | {} | +| `--preferred-sampling-params` | json-formatted sampling settings that will be returned in /get_model_info. | None | + +## LoRA + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--enable-lora` | Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility. | False | +| `--max-lora-rank` | The maximum LoRA rank that should be supported. If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of larger LoRA rank after server startup. | None | +| `--lora-target-modules` | The union set of all target modules where LoRA should be applied (e.g., `q_proj`, `k_proj`, `gate_proj`). If not specified, it will be automatically inferred from the adapters provided in `--lora-paths`. This argument is needed when you expect to dynamically load adapters of different target modules after server startup. You can also set it to `all` to enable LoRA for all supported modules. However, enabling LoRA on additional modules introduces a minor performance overhead. If your application is performance-sensitive, we recommend only specifying the modules for which you plan to load adapters. | None | +| `--lora-paths` | The list of LoRA adapters to load. Each adapter must be specified in one of the following formats: | = | JSON with schema {"lora_name":str,"lora_path":str,"pinned":bool} | None | +| `--max-loras-per-batch` | Maximum number of adapters for a running batch, include base-only request. | 8 | +| `--max-loaded-loras` | If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `--max-loras-per-batch`. | None | +| `--lora-backend` | Choose the kernel backend for multi-LoRA serving. | triton | + +## Kernel backend + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--attention-backend` | Choose the kernels for attention layers. | None | +| `--prefill-attention-backend` | (Experimental) This argument specifies the backend for prefill attention computation. Note that this argument has priority over `attention_backend`. | None | +| `--decode-attention-backend` | (Experimental) This argument specifies the backend for decode attention computation. Note that this argument has priority over `attention_backend`. | None | +| `--sampling-backend` | Choose the kernels for sampling layers. | None | +| `--grammar-backend` | Choose the backend for grammar-guided decoding. | None | +| `--mm-attention-backend` | Set multimodal attention backend. | None | + +## Speculative decoding + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--speculative-algorithm` | Speculative algorithm. | None | +| `--speculative-draft-model-path` | The path of the draft model weights. This can be a local folder or a Hugging Face repo ID. | None | +| `--speculative-num-steps` | The number of steps sampled from draft model in Speculative Decoding. | None | +| `--speculative-eagle-topk` | The number of tokens sampled from the draft model in eagle2 each step. | None | +| `--speculative-num-draft-tokens` | The number of tokens sampled from the draft model in Speculative Decoding. | None | +| `--speculative-accept-threshold-single` | Accept a draft token if its probability in the target model is greater than this threshold. | 1.0 | +| `--speculative-accept-threshold-acc` | The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc). | 1.0 | +| `--speculative-token-map` | The path of the draft model's small vocab table. | None | +| `--speculative-attention-mode` | Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'. | Prefill | + +## Expert parallelism + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--ep-size` | The expert parallelism size. | 1 | +| `--moe-a2a-backend` | Select the backend for all-to-all communication for expert parallelism. | none | +| `--moe-runner-backend` | Select the runner backend for MoE. | 'triton' | +| `--deepep-mode` | Select the mode when enable DeepEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch. | auto | +| `--ep-num-redundant-experts` | Allocate this number of redundant experts in expert parallel. | 0 | +| `--ep-dispatch-algorithm` | The algorithm to choose ranks for redundant experts in EPLB. | None | +| `--init-expert-location` | Initial location of EP experts. | trivial | +| `--enable-eplb` | Enable EPLB algorithm. | False | +| `--eplb-algorithm` | Chosen EPLB algorithm. | auto | +| `--eplb-rebalance-num-iterations` | Number of iterations to automatically trigger a EPLB re-balance. | 1000 | +| `--eplb-rebalance-layers-per-chunk` | Number of layers to rebalance per forward pass. | None | +| `--expert-distribution-recorder-mode` | Mode of expert distribution recorder. | None | +| `--expert-distribution-recorder-buffer-size` | Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer. | None | +| `--enable-expert-distribution-metrics` | Enable logging metrics for expert balancedness. | False | +| `--deepep-config` | Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path. | None | +| `--moe-dense-tp-size` | TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports. | None | + +## Hierarchical cache + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--enable-hierarchical-cache` | Enable hierarchical cache. | False | +| `--hicache-ratio` | The ratio of the size of host KV cache memory pool to the size of device pool. | 2.0 | +| `--hicache-size` | The size of the hierarchical cache. | 0 | +| `--hicache-write-policy` | The write policy for hierarchical cache. | write_through | +| `--hicache-io-backend` | The IO backend for hierarchical cache. | | +| `--hicache-storage-backend` | The storage backend for hierarchical cache. | None | + +## Optimization/debug options + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--disable-radix-cache` | Disable RadixAttention for prefix caching. | False | +| `--cuda-graph-max-bs` | Set the maximum batch size for cuda graph. It will extend the cuda graph capture batch size to this value. | None | +| `--cuda-graph-bs` | Set the list of batch sizes for cuda graph. | None | +| `--disable-cuda-graph` | Disable cuda graph. | False | +| `--disable-cuda-graph-padding` | Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed. | False | +| `--enable-profile-cuda-graph` | Enable profiling of cuda graph capture. | False | +| `--enable-nccl-nvls` | Enable NCCL NVLS for prefill heavy requests when available. | False | +| `--enable-symm-mem` | Enable NCCL symmetric memory for fast collectives. | False | +| `--enable-tokenizer-batch-encode` | Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds. | False | +| `--disable-outlines-disk-cache` | Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency. | False | +| `--disable-custom-all-reduce` | Disable the custom all-reduce kernel and fall back to NCCL. | False | +| `--enable-mscclpp` | Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL. | False | +| `--disable-overlap-schedule` | Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker. | False | +| `--enable-mixed-chunk` | Enabling mixing prefill and decode in a batch when using chunked prefill. | False | +| `--enable-dp-attention` | Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported. | False | +| `--enable-dp-lm-head` | Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention. | False | +| `--enable-two-batch-overlap` | Enabling two micro batches to overlap. | False | +| `--tbo-token-distribution-threshold` | The threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap. | 0.48 | +| `--enable-torch-compile` | Optimize the model with torch.compile. Experimental feature. | False | +| `--torch-compile-max-bs` | Set the maximum batch size when using torch compile. | 32 | +| `--torchao-config` | Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row. | | +| `--enable-nan-detection` | Enable the NaN detection for debugging purposes. | False | +| `--enable-p2p-check` | Enable P2P check for GPU access, otherwise the p2p access is allowed by default. | False | +| `--triton-attention-reduce-in-fp32` | Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16. This only affects Triton attention kernels. | False | +| `--triton-attention-num-kv-splits` | The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8. | 8 | +| `--num-continuous-decode-steps` | Run multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time. | 1 | +| `--delete-ckpt-after-loading` | Delete the model checkpoint after loading the model. | False | +| `--enable-memory-saver` | Allow saving memory using release_memory_occupation and resume_memory_occupation. | False | +| `--allow-auto-truncate` | Allow automatically truncating requests that exceed the maximum input length instead of returning an error. | False | +| `--enable-custom-logit-processor` | Enable users to pass custom logit processors to the server (disabled by default for security). | False | +| `--flashinfer-mla-disable-ragged` | Disable ragged processing in Flashinfer MLA. | False | +| `--disable-shared-experts-fusion` | Disable shared experts fusion. | False | +| `--disable-chunked-prefix-cache` | Disable chunked prefix cache. | False | +| `--disable-fast-image-processor` | Disable fast image processor. | False | +| `--enable-return-hidden-states` | Enable returning hidden states. | False | + +## Debug tensor dumps + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--debug-tensor-dump-output-folder` | The output folder for debug tensor dumps. | None | +| `--debug-tensor-dump-input-file` | The input file for debug tensor dumps. | None | +| `--debug-tensor-dump-inject` | Enable injection of debug tensor dumps. | False | +| `--debug-tensor-dump-prefill-only` | Enable prefill-only mode for debug tensor dumps. | False | + +## PD disaggregation + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--disaggregation-mode` | PD disaggregation mode: "null" (not disaggregated), "prefill" (prefill-only), or "decode" (decode-only). | null | +| `--disaggregation-transfer-backend` | The transfer backend for PD disaggregation. | mooncake | +| `--disaggregation-bootstrap-port` | The bootstrap port for PD disaggregation. | 8998 | +| `--disaggregation-decode-tp` | The decode TP for PD disaggregation. | None | +| `--disaggregation-decode-dp` | The decode DP for PD disaggregation. | None | +| `--disaggregation-prefill-pp` | The prefill PP for PD disaggregation. | 1 | + +## Model weight update + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--custom-weight-loader` | Custom weight loader paths. | None | +| `--weight-loader-disable-mmap` | Disable mmap for weight loader. | False | + +## PD-Multiplexing + +| Arguments | Description | Defaults | +|-----------|-------------|----------| +| `--enable-pdmux` | Enable PD-Multiplexing. | False | +| `--sm-group-num` | Number of SM groups for PD-Multiplexing. | 3 | diff --git a/docs/advanced_features/speculative_decoding.ipynb b/docs/advanced_features/speculative_decoding.ipynb new file mode 100644 index 000000000..aa62b897a --- /dev/null +++ b/docs/advanced_features/speculative_decoding.ipynb @@ -0,0 +1,370 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Speculative Decoding\n", + "\n", + "SGLang now provides an EAGLE-based (EAGLE-2/EAGLE-3) speculative decoding option. Our implementation aims to maximize speed and efficiency and is considered to be among the fastest in open-source LLM engines.\n", + "\n", + "### Performance Highlights\n", + "\n", + "Please see below for the huge improvements on throughput for LLaMA-Instruct 3.1 8B tested on MT bench that can be achieved via EAGLE3 decoding.\n", + "For further details please see the [EAGLE3 paper](https://arxiv.org/pdf/2503.01840).\n", + "\n", + "| Method | Throughput (tokens/s) |\n", + "|--------|----------------|\n", + "| SGLang (w/o speculative, 1x H100) | 158.34 tokens/s |\n", + "| SGLang + EAGLE-2 (1x H100) | 244.10 tokens/s |\n", + "| SGLang + EAGLE-3 (1x H100) | 373.25 tokens/s |" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## EAGLE Decoding\n", + "\n", + "To enable EAGLE speculative decoding the following parameters are relevant:\n", + "* `speculative_draft_model_path`: Specifies draft model. This parameter is required.\n", + "* `speculative_num_steps`: Depth of autoregressive drafting. Increases speculation range but risks rejection cascades. Default is 5.\n", + "* `speculative_eagle_topk`: Branching factor per step. Improves candidate diversity, will lead to higher acceptance rate, but more lead to higher memory/compute consumption. Default is 4.\n", + "* `speculative_num_draft_tokens`: Maximum parallel verification capacity. Allows deeper tree evaluation but will lead to higher GPU memory usage. Default is 8.\n", + "\n", + "These parameters are the same for EAGLE-2 and EAGLE-3.\n", + "\n", + "You can find the best combinations of these parameters with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py).\n", + "\n", + "In the documentation below, we set `--cuda-graph-max-bs` to be a small value for faster engine startup. For your own workloads, please tune the above parameters together with `--cuda-graph-max-bs`, `--max-running-requests`, `--mem-fraction-static` for the best performance. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 decoding\n", + "\n", + "You can enable EAGLE-2 decoding by setting `--speculative-algorithm EAGLE` and choosing an appropriate model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "import openai" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 \\\n", + " --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 Decoding with `torch.compile`\n", + "\n", + "You can also enable `torch.compile` for further optimizations and optionally set `--torch-compile-max-bs`:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-2-7b-chat-hf --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --mem-fraction 0.6 \\\n", + " --enable-torch-compile --torch-compile-max-bs 2 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Llama-2-7b-chat-hf\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling\n", + "\n", + "By employing a truncated high-frequency token vocabulary in the draft model, Eagle speculative decoding reduces `lm_head` computational overhead while accelerating the pipeline without quality degradation. For more details, checkout [the paper](https://arxiv.org/pdf/arXiv:2502.14856).\n", + "\n", + "In our implementation, set `--speculative-token-map` to enable the optimization. You can get the high-frequency token in FR-Spec from [this model](https://huggingface.co/thunlp/LLaMA3-Instruct-8B-FR-Spec). Or you can obtain high-frequency token by directly downloading these token from [this repo](https://github.com/thunlp/FR-Spec/tree/main?tab=readme-ov-file#prepare-fr-spec-vocabulary-subset).\n", + "\n", + "Thanks for the contribution from [Weilin Zhao](https://github.com/Achazwl) and [Zhousx](https://github.com/Zhou-sx). " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Meta-Llama-3-8B-Instruct --speculative-algorithm EAGLE \\\n", + " --speculative-draft-model-path lmsys/sglang-EAGLE-LLaMA3-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 64 --speculative-token-map thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt \\\n", + " --mem-fraction 0.7 --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EAGLE-3 Decoding\n", + "\n", + "You can enable EAGLE-3 decoding by setting `--speculative-algorithm EAGLE3` and choosing an appropriate model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --speculative-algorithm EAGLE3 \\\n", + " --speculative-draft-model-path jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B --speculative-num-steps 5 \\\n", + " --speculative-eagle-topk 8 --speculative-num-draft-tokens 32 --mem-fraction 0.6 \\\n", + " --cuda-graph-max-bs 2 --dtype float16 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi Token Prediction\n", + "\n", + "We support [MTP(Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use Xiaomi/MiMo-7B-RL model as example here (deepseek mtp usage refer to [deepseek doc](../basic_usage/deepseek.md#multi-token-prediction))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + " python3 -m sglang.launch_server --model-path XiaomiMiMo/MiMo-7B-RL --host 0.0.0.0 --trust-remote-code \\\n", + " --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 \\\n", + " --mem-fraction 0.5 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"XiaomiMiMo/MiMo-7B-RL\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "\n", + "EAGLE process is as follows:\n", + "\n", + "- Within EAGLE the draft model predicts the next feature vector, i.e. the last hidden state of the original LLM, using the feature sequence $(f_1, ..., f_k)$ and the token sequence $(t_2, ..., t_{k+1})$. \n", + "- The next token is then sampled from $p_{k+2}=\\text{LMHead}(f_{k+1})$. Afterwards, the two sequences are extended in a tree style—branching out multiple potential continuations, with the branching factor per step controlled by the `speculative_eagle_topk` parameter—to ensure a more coherent connection of context, and are given as input again.\n", + "- EAGLE-2 additionally uses the draft model to evaluate how probable certain branches in the draft tree are, dynamically stopping the expansion of unlikely branches. After the expansion phase, reranking is employed to select only the top `speculative_num_draft_tokens` final nodes as draft tokens.\n", + "- EAGLE-3 removes the feature prediction objective, incorporates low and mid-layer features, and is trained in an on-policy manner.\n", + "\n", + "This enhances drafting accuracy by operating on the features instead of tokens for more regular inputs and passing the tokens from the next timestep additionally to minimize randomness effects from sampling. Furthermore the dynamic adjustment of the draft tree and selection of reranked final nodes increases acceptance rate of draft tokens further. For more details see [EAGLE-2](https://arxiv.org/abs/2406.16858) and [EAGLE-3](https://arxiv.org/abs/2503.01840) paper.\n", + "\n", + "\n", + "For guidance how to train your own EAGLE model please see the [EAGLE repo](https://github.com/SafeAILab/EAGLE/tree/main?tab=readme-ov-file#train)." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/advanced_features/structured_outputs.ipynb b/docs/advanced_features/structured_outputs.ipynb new file mode 100644 index 000000000..1382f1e0e --- /dev/null +++ b/docs/advanced_features/structured_outputs.ipynb @@ -0,0 +1,853 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Structured Outputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can specify a JSON schema, [regular expression](https://en.wikipedia.org/wiki/Regular_expression) or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request.\n", + "\n", + "SGLang supports three grammar backends:\n", + "\n", + "- [XGrammar](https://github.com/mlc-ai/xgrammar)(default): Supports JSON schema, regular expression, and EBNF constraints.\n", + "- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints.\n", + "- [Llguidance](https://github.com/guidance-ai/llguidance): Supports JSON schema, regular expression, and EBNF constraints.\n", + "\n", + "We suggest using XGrammar for its better performance and utility. XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). For more details, see [XGrammar technical overview](https://blog.mlc.ai/2024/11/22/achieving-efficient-flexible-portable-structured-generation-with-xgrammar).\n", + "\n", + "To use Outlines, simply add `--grammar-backend outlines` when launching the server.\n", + "To use llguidance, add `--grammar-backend llguidance` when launching the server.\n", + "If no backend is specified, XGrammar will be used as the default.\n", + "\n", + "For better output quality, **It's advisable to explicitly include instructions in the prompt to guide the model to generate the desired format.** For example, you can specify, 'Please generate the output in the following JSON format: ...'.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import os\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON\n", + "\n", + "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Please generate the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\n", + " \"name\": \"foo\",\n", + " # convert the pydantic model to json schema\n", + " \"schema\": CapitalInfo.model_json_schema(),\n", + " },\n", + " },\n", + ")\n", + "\n", + "response_content = response.choices[0].message.content\n", + "# validate the JSON response by the pydantic model\n", + "capital_info = CapitalInfo.model_validate_json(response_content)\n", + "print_highlight(f\"Validated response: {capital_info.model_dump_json()}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ebnf_grammar = \"\"\"\n", + "root ::= city | description\n", + "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", + "description ::= city \" is \" status\n", + "status ::= \"the capital of \" country\n", + "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", + "\"\"\"\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"Give me the information of the capital of France.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=32,\n", + " extra_body={\"ebnf\": ebnf_grammar},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"What is the capital of France?\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=128,\n", + " extra_body={\"regex\": \"(Paris|London)\"},\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_get_current_weather = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "tool_get_current_date = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_date\",\n", + " \"description\": \"Get the current date and time for a given timezone\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"timezone\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", + " }\n", + " },\n", + " \"required\": [\"timezone\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", + "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"\"\"\n", + "# Tool Instructions\n", + "- Always execute python code in messages that you share.\n", + "- When looking for real time information use relevant functions if available else fallback to brave_search\n", + "You have access to the following functions:\n", + "Use the function 'get_current_weather' to: Get the current weather in a given location\n", + "{tool_get_current_weather[\"function\"]}\n", + "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", + "{tool_get_current_date[\"function\"]}\n", + "If a you choose to call a function ONLY reply in the following format:\n", + "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", + "where\n", + "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", + "end_tag => ``\n", + "Here is an example,\n", + "{{\"example_name\": \"example_value\"}}\n", + "Reminder:\n", + "- Function calls MUST follow the specified format\n", + "- Required parameters MUST be specified\n", + "- Only call one function at a time\n", + "- Put the entire function call reply on one line\n", + "- Always add your sources when using search results to answer the user query\n", + "You are a helpful assistant.\"\"\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"meta-llama/Meta-Llama-3.1-8B-Instruct\",\n", + " messages=messages,\n", + " response_format={\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"...` to denote reasoning sections, you might want to allow free-form text within these sections while still enforcing grammar constraints on the rest of the output.\n", + "\n", + "SGLang provides a feature to disable grammar restrictions within reasoning sections. This is particularly useful for models that need to perform complex reasoning steps before providing a structured output.\n", + "\n", + "To enable this feature, use the `--reasoning-parser` flag which decide the think_end_token, such as ``, when launching the server. You can also specify the reasoning parser using the `--reasoning-parser` flag.\n", + "\n", + "## Supported Models\n", + "\n", + "Currently, SGLang supports the following reasoning models:\n", + "- [DeepSeek R1 series](https://huggingface.co/collections/deepseek-ai/deepseek-r1-678e1e131c0169c0bc89728d): The reasoning content is wrapped with `` and `` tags.\n", + "- [QwQ](https://huggingface.co/Qwen/QwQ-32B): The reasoning content is wrapped with `` and `` tags.\n", + "\n", + "\n", + "## Usage\n", + "\n", + "## OpenAI Compatible API" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify the `--grammar-backend`, `--reasoning-parser` option." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "import os\n", + "\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path deepseek-ai/DeepSeek-R1-Distill-Qwen-7B --host 0.0.0.0 --reasoning-parser deepseek-r1 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### JSON\n", + "\n", + "you can directly define a JSON schema or use [Pydantic](https://docs.pydantic.dev/latest/) to define and validate the response." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Using Pydantic**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "# Define the schema using Pydantic\n", + "class CapitalInfo(BaseModel):\n", + " name: str = Field(..., pattern=r\"^\\w+$\", description=\"Name of the capital city\")\n", + " population: int = Field(..., description=\"Population of the capital city\")\n", + "\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\n", + " \"name\": \"foo\",\n", + " # convert the pydantic model to json schema\n", + " \"schema\": CapitalInfo.model_json_schema(),\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " response_format={\n", + " \"type\": \"json_schema\",\n", + " \"json_schema\": {\"name\": \"foo\", \"schema\": json.loads(json_schema)},\n", + " },\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ebnf_grammar = \"\"\"\n", + "root ::= city | description\n", + "city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\n", + "description ::= city \" is \" status\n", + "status ::= \"the capital of \" country\n", + "country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"\n", + "\"\"\"\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"You are a helpful geography bot.\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Give me the information and population of the capital of France in the JSON format.\",\n", + " },\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " extra_body={\"ebnf\": ebnf_grammar},\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=[\n", + " {\"role\": \"assistant\", \"content\": \"What is the capital of France?\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=2048,\n", + " extra_body={\"regex\": \"(Paris|London)\"},\n", + ")\n", + "\n", + "print_highlight(\n", + " f\"reasoing_content: {response.choices[0].message.reasoning_content}\\n\\ncontent: {response.choices[0].message.content}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tool_get_current_weather = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"state\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"the two-letter abbreviation for the state that the city is\"\n", + " \" in, e.g. 'CA' which would mean 'California'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"state\", \"unit\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "tool_get_current_date = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_date\",\n", + " \"description\": \"Get the current date and time for a given timezone\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"timezone\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The timezone to fetch the current date and time for, e.g. 'America/New_York'\",\n", + " }\n", + " },\n", + " \"required\": [\"timezone\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "schema_get_current_weather = tool_get_current_weather[\"function\"][\"parameters\"]\n", + "schema_get_current_date = tool_get_current_date[\"function\"][\"parameters\"]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": f\"\"\"\n", + "# Tool Instructions\n", + "- Always execute python code in messages that you share.\n", + "- When looking for real time information use relevant functions if available else fallback to brave_search\n", + "You have access to the following functions:\n", + "Use the function 'get_current_weather' to: Get the current weather in a given location\n", + "{tool_get_current_weather[\"function\"]}\n", + "Use the function 'get_current_date' to: Get the current date and time for a given timezone\n", + "{tool_get_current_date[\"function\"]}\n", + "If a you choose to call a function ONLY reply in the following format:\n", + "<{{start_tag}}={{function_name}}>{{parameters}}{{end_tag}}\n", + "where\n", + "start_tag => ` a JSON dict with the function argument name as key and function argument value as value.\n", + "end_tag => ``\n", + "Here is an example,\n", + "{{\"example_name\": \"example_value\"}}\n", + "Reminder:\n", + "- Function calls MUST follow the specified format\n", + "- Required parameters MUST be specified\n", + "- Only call one function at a time\n", + "- Put the entire function call reply on one line\n", + "- Always add your sources when using search results to answer the user query\n", + "You are a helpful assistant.\"\"\",\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"You are in New York. Please get the current date and time, and the weather.\",\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B\",\n", + " messages=messages,\n", + " response_format={\n", + " \"type\": \"structural_tag\",\n", + " \"max_new_tokens\": 2048,\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\")[0]\n", + "content = response.json()[\"text\"].split(\"\")[1]\n", + "print_highlight(f\"reasoing_content: {reasoing_content}\\n\\ncontent: {content}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**JSON Schema Directly**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "json_schema = json.dumps(\n", + " {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"name\": {\"type\": \"string\", \"pattern\": \"^[\\\\w]+$\"},\n", + " \"population\": {\"type\": \"integer\"},\n", + " },\n", + " \"required\": [\"name\", \"population\"],\n", + " }\n", + ")\n", + "\n", + "# JSON\n", + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True\n", + ")\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 2048,\n", + " \"json_schema\": json_schema,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### EBNF" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"Give me the information of the capital of France.\",\n", + " \"sampling_params\": {\n", + " \"max_new_tokens\": 2048,\n", + " \"temperature\": 0,\n", + " \"n\": 3,\n", + " \"ebnf\": (\n", + " \"root ::= city | description\\n\"\n", + " 'city ::= \"London\" | \"Paris\" | \"Berlin\" | \"Rome\"\\n'\n", + " 'description ::= city \" is \" status\\n'\n", + " 'status ::= \"the capital of \" country\\n'\n", + " 'country ::= \"England\" | \"France\" | \"Germany\" | \"Italy\"'\n", + " ),\n", + " },\n", + " \"stream\": False,\n", + " \"return_logprob\": False,\n", + " },\n", + ")\n", + "\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Regular expression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"Paris is the capital of\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 2048,\n", + " \"regex\": \"(France|England)\",\n", + " },\n", + " },\n", + ")\n", + "print(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Structural Tag" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " messages, tokenize=False, add_generation_prompt=True\n", + ")\n", + "payload = {\n", + " \"text\": text,\n", + " \"sampling_params\": {\n", + " \"max_new_tokens\": 2048,\n", + " \"structural_tag\": json.dumps(\n", + " {\n", + " \"type\": \"structural_tag\",\n", + " \"structures\": [\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\"\",\n", + " \"schema\": schema_get_current_weather,\n", + " \"end\": \"\",\n", + " },\n", + " {\n", + " \"begin\": \"\",\n", + " \"schema\": schema_get_current_date,\n", + " \"end\": \"\",\n", + " },\n", + " ],\n", + " \"triggers\": [\" is not trimmed.\n", + "\n", + "sampling_params = {\n", + " \"max_new_tokens\": 1024,\n", + " \"temperature\": 0,\n", + " \"top_p\": 0.95,\n", + " \"skip_special_tokens\": False,\n", + "}\n", + "\n", + "# 1) Offline generation\n", + "result = llm.generate(input_ids=input_ids, sampling_params=sampling_params)\n", + "generated_text = result[\"text\"] # Assume there is only one prompt\n", + "\n", + "print_highlight(\"=== Offline Engine Output Text ===\")\n", + "print_highlight(generated_text)\n", + "\n", + "\n", + "# 2) Parse using FunctionCallParser\n", + "def convert_dict_to_tool(tool_dict: dict) -> Tool:\n", + " function_dict = tool_dict.get(\"function\", {})\n", + " return Tool(\n", + " type=tool_dict.get(\"type\", \"function\"),\n", + " function=Function(\n", + " name=function_dict.get(\"name\"),\n", + " description=function_dict.get(\"description\"),\n", + " parameters=function_dict.get(\"parameters\"),\n", + " ),\n", + " )\n", + "\n", + "\n", + "tools = [convert_dict_to_tool(raw_tool) for raw_tool in tools]\n", + "\n", + "parser = FunctionCallParser(tools=tools, tool_call_parser=\"qwen25\")\n", + "normal_text, calls = parser.parse_non_stream(generated_text)\n", + "\n", + "print_highlight(\"=== Parsing Result ===\")\n", + "print(\"Normal text portion:\", normal_text)\n", + "print_highlight(\"Function call portion:\")\n", + "for call in calls:\n", + " # call: ToolCallItem\n", + " print_highlight(f\" - tool name: {call.name}\")\n", + " print_highlight(f\" parameters: {call.parameters}\")\n", + "\n", + "# 3) If needed, perform additional logic on the parsed functions, such as automatically calling the corresponding function to obtain a return value, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Choice Mode\n", + "\n", + "SGLang supports OpenAI's `tool_choice` parameter to control when and which tools the model should call. This feature is implemented using EBNF (Extended Backus-Naur Form) grammar to ensure reliable tool calling behavior.\n", + "\n", + "### Supported Tool Choice Options\n", + "\n", + "- **`tool_choice=\"required\"`**: Forces the model to call at least one tool\n", + "- **`tool_choice={\"type\": \"function\", \"function\": {\"name\": \"specific_function\"}}`**: Forces the model to call a specific function\n", + "\n", + "### Backend Compatibility\n", + "\n", + "Tool choice is fully supported with the **Xgrammar backend**, which is the default grammar backend (`--grammar-backend xgrammar`). However, it may not be fully supported with other backends such as `outlines`.\n", + "\n", + "### Example: Required Tool Choice" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "\n", + "# Start a new server session for tool choice examples\n", + "server_process_tool_choice, port_tool_choice = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --tool-call-parser qwen25 --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "wait_for_server(f\"http://localhost:{port_tool_choice}\")\n", + "\n", + "# Initialize client for tool choice examples\n", + "client_tool_choice = OpenAI(\n", + " api_key=\"None\", base_url=f\"http://0.0.0.0:{port_tool_choice}/v1\"\n", + ")\n", + "model_name_tool_choice = client_tool_choice.models.list().data[0].id\n", + "\n", + "# Example with tool_choice=\"required\" - forces the model to call a tool\n", + "messages_required = [\n", + " {\"role\": \"user\", \"content\": \"Hello, what is the capital of France?\"}\n", + "]\n", + "\n", + "# Define tools\n", + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_current_weather\",\n", + " \"description\": \"Get the current weather in a given location\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The city to find the weather for, e.g. 'San Francisco'\",\n", + " },\n", + " \"unit\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The unit to fetch the temperature in\",\n", + " \"enum\": [\"celsius\", \"fahrenheit\"],\n", + " },\n", + " },\n", + " \"required\": [\"city\", \"unit\"],\n", + " },\n", + " },\n", + " }\n", + "]\n", + "\n", + "response_required = client_tool_choice.chat.completions.create(\n", + " model=model_name_tool_choice,\n", + " messages=messages_required,\n", + " temperature=0,\n", + " max_tokens=1024,\n", + " tools=tools,\n", + " tool_choice=\"required\", # Force the model to call a tool\n", + ")\n", + "\n", + "print_highlight(\"Response with tool_choice='required':\")\n", + "print(\"Content:\", response_required.choices[0].message.content)\n", + "print(\"Tool calls:\", response_required.choices[0].message.tool_calls)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example: Specific Function Choice\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example with specific function choice - forces the model to call a specific function\n", + "messages_specific = [\n", + " {\"role\": \"user\", \"content\": \"What are the most attactive places in France?\"}\n", + "]\n", + "\n", + "response_specific = client_tool_choice.chat.completions.create(\n", + " model=model_name_tool_choice,\n", + " messages=messages_specific,\n", + " temperature=0,\n", + " max_tokens=1024,\n", + " tools=tools,\n", + " tool_choice={\n", + " \"type\": \"function\",\n", + " \"function\": {\"name\": \"get_current_weather\"},\n", + " }, # Force the model to call the specific get_current_weather function\n", + ")\n", + "\n", + "print_highlight(\"Response with specific function choice:\")\n", + "print(\"Content:\", response_specific.choices[0].message.content)\n", + "print(\"Tool calls:\", response_specific.choices[0].message.tool_calls)\n", + "\n", + "if response_specific.choices[0].message.tool_calls:\n", + " tool_call = response_specific.choices[0].message.tool_calls[0]\n", + " print_highlight(f\"Called function: {tool_call.function.name}\")\n", + " print_highlight(f\"Arguments: {tool_call.function.arguments}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process_tool_choice)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pythonic Tool Call Format (Llama-3.2 / Llama-3.3 / Llama-4)\n", + "\n", + "Some Llama models (such as Llama-3.2-1B, Llama-3.2-3B, Llama-3.3-70B, and Llama-4) support a \"pythonic\" tool call format, where the model outputs function calls as Python code, e.g.:\n", + "\n", + "```python\n", + "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\")]\n", + "```\n", + "\n", + "- The output is a Python list of function calls, with arguments as Python literals (not JSON).\n", + "- Multiple tool calls can be returned in the same list:\n", + "```python\n", + "[get_current_weather(city=\"San Francisco\", state=\"CA\", unit=\"celsius\"),\n", + " get_current_weather(city=\"New York\", state=\"NY\", unit=\"fahrenheit\")]\n", + "```\n", + "\n", + "For more information, refer to Meta’s documentation on [Zero shot function calling](https://github.com/meta-llama/llama-models/blob/main/models/llama4/prompt_format.md#zero-shot-function-calling---system-message).\n", + "\n", + "Note that this feature is still under development on Blackwell.\n", + "\n", + "### How to enable\n", + "- Launch the server with `--tool-call-parser pythonic`\n", + "- You may also specify --chat-template with the improved template for the model (e.g., `--chat-template=examples/chat_template/tool_chat_template_llama4_pythonic.jinja`).\n", + "This is recommended because the model expects a special prompt format to reliably produce valid pythonic tool call outputs. The template ensures that the prompt structure (e.g., special tokens, message boundaries like `<|eom|>`, and function call delimiters) matches what the model was trained or fine-tuned on. If you do not use the correct chat template, tool calling may fail or produce inconsistent results.\n", + "\n", + "#### Forcing Pythonic Tool Call Output Without a Chat Template\n", + "If you don't want to specify a chat template, you must give the model extremely explicit instructions in your messages to enforce pythonic output. For example, for `Llama-3.2-1B-Instruct`, you need:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \" python3 -m sglang.launch_server --model-path meta-llama/Llama-3.2-1B-Instruct --tool-call-parser pythonic --tp 1 --log-level warning\" # llama-3.2-1b-instruct\n", + ")\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "\n", + "tools = [\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_weather\",\n", + " \"description\": \"Get the current weather for a given location.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"location\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The name of the city or location.\",\n", + " }\n", + " },\n", + " \"required\": [\"location\"],\n", + " },\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"get_tourist_attractions\",\n", + " \"description\": \"Get a list of top tourist attractions for a given city.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"city\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The name of the city to find attractions for.\",\n", + " }\n", + " },\n", + " \"required\": [\"city\"],\n", + " },\n", + " },\n", + " },\n", + "]\n", + "\n", + "\n", + "def get_messages():\n", + " return [\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": (\n", + " \"You are a travel assistant. \"\n", + " \"When asked to call functions, ALWAYS respond ONLY with a python list of function calls, \"\n", + " \"using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. \"\n", + " \"Do NOT use JSON, do NOT use variables, do NOT use any other format. \"\n", + " \"Here is an example:\\n\"\n", + " '[get_weather(location=\"Paris\"), get_tourist_attractions(city=\"Paris\")]'\n", + " ),\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": (\n", + " \"I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? \"\n", + " \"Propose parallel tool calls at once, using the python list of function calls format as shown above.\"\n", + " ),\n", + " },\n", + " ]\n", + "\n", + "\n", + "messages = get_messages()\n", + "\n", + "client = openai.Client(base_url=f\"http://localhost:{port}/v1\", api_key=\"xxxxxx\")\n", + "model_name = client.models.list().data[0].id\n", + "\n", + "\n", + "response_non_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0,\n", + " top_p=0.9,\n", + " stream=False, # Non-streaming\n", + " tools=tools,\n", + ")\n", + "print_highlight(\"Non-stream response:\")\n", + "print_highlight(response_non_stream)\n", + "\n", + "response_stream = client.chat.completions.create(\n", + " model=model_name,\n", + " messages=messages,\n", + " temperature=0,\n", + " top_p=0.9,\n", + " stream=True,\n", + " tools=tools,\n", + ")\n", + "texts = \"\"\n", + "tool_calls = []\n", + "name = \"\"\n", + "arguments = \"\"\n", + "\n", + "for chunk in response_stream:\n", + " if chunk.choices[0].delta.content:\n", + " texts += chunk.choices[0].delta.content\n", + " if chunk.choices[0].delta.tool_calls:\n", + " tool_calls.append(chunk.choices[0].delta.tool_calls[0])\n", + "\n", + "print_highlight(\"Streaming Response:\")\n", + "print_highlight(\"==== Text ====\")\n", + "print_highlight(texts)\n", + "\n", + "print_highlight(\"==== Tool Call ====\")\n", + "for tool_call in tool_calls:\n", + " print_highlight(tool_call)\n", + "\n", + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> **Note:** \n", + "> The model may still default to JSON if it was heavily finetuned on that format. Prompt engineering (including examples) is the only way to increase the chance of pythonic output if you are not using a chat template." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to support a new model?\n", + "1. Update the TOOLS_TAG_LIST in sglang/srt/function_call_parser.py with the model’s tool tags. Currently supported tags include:\n", + "```\n", + "\tTOOLS_TAG_LIST = [\n", + "\t “<|plugin|>“,\n", + "\t ““,\n", + "\t “<|python_tag|>“,\n", + "\t “[TOOL_CALLS]”\n", + "\t]\n", + "```\n", + "2. Create a new detector class in sglang/srt/function_call_parser.py that inherits from BaseFormatDetector. The detector should handle the model’s specific function call format. For example:\n", + "```\n", + " class NewModelDetector(BaseFormatDetector):\n", + "```\n", + "3. Add the new detector to the MultiFormatParser class that manages all the format detectors." + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/advanced_features/vlm_query.ipynb b/docs/advanced_features/vlm_query.ipynb new file mode 100644 index 000000000..d9a8ae75d --- /dev/null +++ b/docs/advanced_features/vlm_query.ipynb @@ -0,0 +1,325 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Query Vision Language Model" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## Querying Qwen-VL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply() # Run this first.\n", + "\n", + "model_path = \"Qwen/Qwen2.5-VL-3B-Instruct\"\n", + "chat_template = \"qwen2-vl\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Lets create a prompt.\n", + "\n", + "from io import BytesIO\n", + "import requests\n", + "from PIL import Image\n", + "\n", + "from sglang.srt.parser.conversation import chat_templates\n", + "\n", + "image = Image.open(\n", + " BytesIO(\n", + " requests.get(\n", + " \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " ).content\n", + " )\n", + ")\n", + "\n", + "conv = chat_templates[chat_template].copy()\n", + "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", + "conv.append_message(conv.roles[1], \"\")\n", + "conv.image_data = [image]\n", + "\n", + "print(conv.get_prompt())\n", + "image" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "### Query via the offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import Engine\n", + "\n", + "llm = Engine(\n", + " model_path=model_path, chat_template=chat_template, mem_fraction_static=0.8\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", + "print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### Query via the offline Engine API, but send precomputed embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the image embeddings using Huggingface.\n", + "\n", + "from transformers import AutoProcessor\n", + "from transformers import Qwen2_5_VLForConditionalGeneration\n", + "\n", + "processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", + "vision = (\n", + " Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path).eval().visual.cuda()\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "processed_prompt = processor(\n", + " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", + ")\n", + "input_ids = processed_prompt[\"input_ids\"][0].detach().cpu().tolist()\n", + "precomputed_embeddings = vision(\n", + " processed_prompt[\"pixel_values\"].cuda(), processed_prompt[\"image_grid_thw\"].cuda()\n", + ")\n", + "\n", + "mm_item = dict(\n", + " modality=\"IMAGE\",\n", + " image_grid_thw=processed_prompt[\"image_grid_thw\"],\n", + " precomputed_embeddings=precomputed_embeddings,\n", + ")\n", + "out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", + "print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "## Querying Llama 4 (Vision)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply() # Run this first.\n", + "\n", + "model_path = \"meta-llama/Llama-4-Scout-17B-16E-Instruct\"\n", + "chat_template = \"llama-4\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "# Lets create a prompt.\n", + "\n", + "from io import BytesIO\n", + "import requests\n", + "from PIL import Image\n", + "\n", + "from sglang.srt.parser.conversation import chat_templates\n", + "\n", + "image = Image.open(\n", + " BytesIO(\n", + " requests.get(\n", + " \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " ).content\n", + " )\n", + ")\n", + "\n", + "conv = chat_templates[chat_template].copy()\n", + "conv.append_message(conv.roles[0], f\"What's shown here: {conv.image_token}?\")\n", + "conv.append_message(conv.roles[1], \"\")\n", + "conv.image_data = [image]\n", + "\n", + "print(conv.get_prompt())\n", + "print(f\"Image size: {image.size}\")\n", + "\n", + "image" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Query via the offline Engine API" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.test_utils import is_in_ci\n", + "\n", + "if not is_in_ci():\n", + " from sglang import Engine\n", + "\n", + " llm = Engine(\n", + " model_path=model_path,\n", + " trust_remote_code=True,\n", + " enable_multimodal=True,\n", + " mem_fraction_static=0.8,\n", + " tp_size=4,\n", + " attention_backend=\"fa3\",\n", + " context_length=65536,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " out = llm.generate(prompt=conv.get_prompt(), image_data=[image])\n", + " print(out[\"text\"])" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### Query via the offline Engine API, but send precomputed embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " # Compute the image embeddings using Huggingface.\n", + "\n", + " from transformers import AutoProcessor\n", + " from transformers import Llama4ForConditionalGeneration\n", + "\n", + " processor = AutoProcessor.from_pretrained(model_path, use_fast=True)\n", + " model = Llama4ForConditionalGeneration.from_pretrained(\n", + " model_path, torch_dtype=\"auto\"\n", + " ).eval()\n", + " vision = model.vision_model.cuda()\n", + " multi_modal_projector = model.multi_modal_projector.cuda()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "if not is_in_ci():\n", + " processed_prompt = processor(\n", + " images=[image], text=conv.get_prompt(), return_tensors=\"pt\"\n", + " )\n", + " print(f'{processed_prompt[\"pixel_values\"].shape=}')\n", + " input_ids = processed_prompt[\"input_ids\"][0].detach().cpu().tolist()\n", + "\n", + " image_outputs = vision(\n", + " processed_prompt[\"pixel_values\"].to(\"cuda\"), output_hidden_states=False\n", + " )\n", + " image_features = image_outputs.last_hidden_state\n", + " vision_flat = image_features.view(-1, image_features.size(-1))\n", + " precomputed_embeddings = multi_modal_projector(vision_flat)\n", + "\n", + " mm_item = dict(modality=\"IMAGE\", precomputed_embeddings=precomputed_embeddings)\n", + " out = llm.generate(input_ids=input_ids, image_data=[mm_item])\n", + " print(out[\"text\"])" + ] + } + ], + "metadata": { + "jupytext": { + "cell_metadata_filter": "-all", + "custom_cell_magics": "kql", + "encoding": "# -*- coding: utf-8 -*-" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/basic_usage/deepseek.md b/docs/basic_usage/deepseek.md new file mode 100644 index 000000000..8a71696f5 --- /dev/null +++ b/docs/basic_usage/deepseek.md @@ -0,0 +1,228 @@ +# DeepSeek Usage + +SGLang provides many optimizations specifically designed for the DeepSeek models, making it the inference engine recommended by the official [DeepSeek team](https://github.com/deepseek-ai/DeepSeek-V3/tree/main?tab=readme-ov-file#62-inference-with-sglang-recommended) from Day 0. + +This document outlines current optimizations for DeepSeek. +For an overview of the implemented features see the completed [Roadmap](https://github.com/sgl-project/sglang/issues/2591). + +## Launch DeepSeek V3.1/V3/R1 with SGLang + +To run DeepSeek V3.1/V3/R1 models, the recommended settings are as follows: + +| Weight Type | Configuration | +|------------|-------------------| +| **Full precision FP8**
*(recommended)* | 8 x H200 | +| | 8 x MI300X | +| | 2 x 8 x H100/800/20 | +| | Xeon 6980P CPU | +| **Full precision BF16** | 2 x 8 x H200 | +| | 2 x 8 x MI300X | +| | 4 x 8 x H100/800/20 | +| | 4 x 8 x A100/A800 | +| **Quantized weights (AWQ)** | 8 x H100/800/20 | +| | 8 x A100/A800 | +| **Quantized weights (int8)** | 16 x A100/800 | +| | 32 x L40S | +| | Xeon 6980P CPU | +| | 2 x Atlas 800I A3 | + + + +Detailed commands for reference: + +- [8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#using-docker-recommended) +- [8 x MI300X](../platforms/amd_gpu.md#running-deepseek-v3) +- [2 x 8 x H200](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes) +- [4 x 8 x A100](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes) +- [8 x A100 (AWQ)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-8-a100a800-with-awq-quantization) +- [16 x A100 (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-16-a100a800-with-int8-quantization) +- [32 x L40S (int8)](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-32-l40s-with-int8-quantization) +- [Xeon 6980P CPU](../platforms/cpu_server.md#example-running-deepseek-r1) +- [2 x Atlas 800I A3 (int8)](../platforms/ascend_npu.md#running-deepseek-v3) + +### Download Weights +If you encounter errors when starting the server, ensure the weights have finished downloading. It's recommended to download them beforehand or restart multiple times until all weights are downloaded. Please refer to [DeepSeek V3](https://huggingface.co/deepseek-ai/DeepSeek-V3-Base#61-inference-with-deepseek-infer-demo-example-only) official guide to download the weights. + +### Launch with one node of 8 x H200 +Please refer to [the example](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#installation--launch). +**Note that Deepseek V3 is already in FP8**, so we should not run it with any quantization arguments like `--quantization fp8 --kv-cache-dtype fp8_e5m2`. + +### Running examples on Multi-node + +- [Serving with two H20*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h208-nodes). + +- [Serving with two H200*8 nodes and docker](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-two-h2008-nodes-and-docker). + +- [Serving with four A100*8 nodes](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-four-a1008-nodes). + +## Optimizations + +### Multi-head Latent Attention (MLA) Throughput Optimizations + +**Description**: [MLA](https://arxiv.org/pdf/2405.04434) is an innovative attention mechanism introduced by the DeepSeek team, aimed at improving inference efficiency. SGLang has implemented specific optimizations for this, including: + +- **Weight Absorption**: By applying the associative law of matrix multiplication to reorder computation steps, this method balances computation and memory access and improves efficiency in the decoding phase. + +- **MLA Attention Backends**: Currently SGLang supports different optimized MLA attention backends, including [FlashAttention3](https://github.com/Dao-AILab/flash-attention), [Flashinfer](https://docs.flashinfer.ai/api/mla.html), [FlashMLA](https://github.com/deepseek-ai/FlashMLA), [CutlassMLA](https://github.com/sgl-project/sglang/pull/5390), **TRTLLM MLA** (optimized for Blackwell architecture), and [Triton](https://github.com/triton-lang/triton) backends. The default FA3 provides good performance across wide workloads. + +- **FP8 Quantization**: W8A8 FP8 and KV Cache FP8 quantization enables efficient FP8 inference. Additionally, we have implemented Batched Matrix Multiplication (BMM) operator to facilitate FP8 inference in MLA with weight absorption. + +- **CUDA Graph & Torch.compile**: Both MLA and Mixture of Experts (MoE) are compatible with CUDA Graph and Torch.compile, which reduces latency and accelerates decoding speed for small batch sizes. + +- **Chunked Prefix Cache**: Chunked prefix cache optimization can increase throughput by cutting prefix cache into chunks, processing them with multi-head attention and merging their states. Its improvement can be significant when doing chunked prefill on long sequences. Currently this optimization is only available for FlashAttention3 backend. + +Overall, with these optimizations, we have achieved up to **7x** acceleration in output throughput compared to the previous version. + +

+ Multi-head Latent Attention for DeepSeek Series Models +

+ +**Usage**: MLA optimization is enabled by default. For MLA models on Blackwell architecture (e.g., B200), the default backend is FlashInfer. To use the optimized TRTLLM MLA backend for prefill and decode operations, explicitly specify `--attention-backend trtllm_mla`. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-09-04-sglang-v0-3/#deepseek-multi-head-latent-attention-mla-throughput-optimizations) and [Slides](https://github.com/sgl-project/sgl-learning-materials/blob/main/slides/lmsys_1st_meetup_deepseek_mla.pdf) for more details. + +### Data Parallelism Attention + +**Description**: This optimization involves data parallelism (DP) for the MLA attention mechanism of DeepSeek Series Models, which allows for a significant reduction in the KV cache size, enabling larger batch sizes. Each DP worker independently handles different types of batches (prefill, decode, idle), which are then synchronized before and after processing through the Mixture-of-Experts (MoE) layer. If you do not use DP attention, KV cache will be duplicated among all TP ranks. + +

+ Data Parallelism Attention for DeepSeek Series Models +

+ +With data parallelism attention enabled, we have achieved up to **1.9x** decoding throughput improvement compared to the previous version. + +

+ Data Parallelism Attention Performance Comparison +

+ +**Usage**: +- Append `--enable-dp-attention --tp 8 --dp 8` to the server arguments when using 8 H200 GPUs. This optimization improves peak throughput in high batch size scenarios where the server is limited by KV cache capacity. However, it is not recommended for low-latency, small-batch use cases. +- DP and TP attention can be flexibly combined. For example, to deploy DeepSeek-V3/R1 on 2 nodes with 8 H100 GPUs each, you can specify `--enable-dp-attention --tp 16 --dp 2`. This configuration runs attention with 2 DP groups, each containing 8 TP GPUs. + +**Reference**: Check [Blog](https://lmsys.org/blog/2024-12-04-sglang-v0-4/#data-parallelism-attention-for-deepseek-models). + +### Multi Node Tensor Parallelism + +**Description**: For users with limited memory on a single node, SGLang supports serving DeepSeek Series Models, including DeepSeek V3, across multiple nodes using tensor parallelism. This approach partitions the model parameters across multiple GPUs or nodes to handle models that are too large for one node's memory. + +**Usage**: Check [here](https://github.com/sgl-project/sglang/tree/main/benchmark/deepseek_v3#example-serving-with-2-h208) for usage examples. + +### Block-wise FP8 + +**Description**: SGLang implements block-wise FP8 quantization with two key optimizations: + +- **Activation**: E4M3 format using per-token-per-128-channel sub-vector scales with online casting. + +- **Weight**: Per-128x128-block quantization for better numerical stability. + +- **DeepGEMM**: The [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM) kernel library optimized for FP8 matrix multiplications. + +**Usage**: The activation and weight optimization above are turned on by default for DeepSeek V3 models. DeepGEMM is enabled by default on NVIDIA Hopper GPUs and disabled by default on other devices. DeepGEMM can also be manually turned off by setting the environment variable `SGL_ENABLE_JIT_DEEPGEMM=0`. + +Before serving the DeepSeek model, precompile the DeepGEMM kernels using: +```bash +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code +``` +The precompilation process typically takes around 10 minutes to complete. + +### Multi-token Prediction +**Description**: SGLang implements DeepSeek V3 Multi-Token Prediction (MTP) based on [EAGLE speculative decoding](https://docs.sglang.ai/advanced_features/speculative_decoding.html#EAGLE-Decoding). With this optimization, the decoding speed can be improved by **1.8x** for batch size 1 and **1.5x** for batch size 32 respectively on H200 TP8 setting. + +**Usage**: +Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: +``` +python3 -m sglang.launch_server --model-path deepseek-ai/DeepSeek-V3-0324 --speculative-algorithm EAGLE --speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2 --trust-remote-code --tp 8 +``` +- The best configuration for `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` can be searched with [bench_speculative.py](https://github.com/sgl-project/sglang/blob/main/scripts/playground/bench_speculative.py) script for given batch size. The minimum configuration is `--speculative-num-steps 1 --speculative-eagle-topk 1 --speculative-num-draft-tokens 2`, which can achieve speedup for larger batch sizes. +- FlashAttention3, FlashMLA, and Triton backend fully supports MTP usage. For FlashInfer backend (`--attention-backend flashinfer`) with speculative decoding,`--speculative-eagle-topk` parameter should be set to `1`. MTP support for the CutlassMLA and TRTLLM MLA backends are still under development. +- To enable DeepSeek MTP for large batch sizes (>32), there are some parameters should be changed (Reference [this discussion](https://github.com/sgl-project/sglang/issues/4543#issuecomment-2737413756)): + - Adjust `--max-running-requests` to a larger number. The default value is `32` for MTP. For larger batch sizes, you should increase this value beyond the default value. + - Set `--cuda-graph-bs`. It's a list of batch sizes for cuda graph capture. The default captured batch sizes for speculative decoding is set [here](https://github.com/sgl-project/sglang/blob/49420741746c8f3e80e0eb17e7d012bfaf25793a/python/sglang/srt/model_executor/cuda_graph_runner.py#L126). You can include more batch sizes into it. + + +### Reasoning Content for DeepSeek R1 & V3.1 + +See [Reasoning Parser](https://docs.sglang.ai/advanced_features/separate_reasoning.html) and [Thinking Parameter for DeepSeek V3.1](https://docs.sglang.ai/basic_usage/openai_api_completions.html#Example:-DeepSeek-V3-Models). + + +### Function calling for DeepSeek Models + +Add arguments `--tool-call-parser deepseekv3` and `--chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja`(recommended) to enable this feature. For example (running on 1 * H20 node): + +``` +python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --port 30000 --host 0.0.0.0 --mem-fraction-static 0.9 --tool-call-parser deepseekv3 --chat-template ./examples/chat_template/tool_chat_template_deepseekv3.jinja +``` + +Sample Request: + +``` +curl "http://127.0.0.1:30000/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324", "tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +``` + +Expected Response + +``` +{"id":"6501ef8e2d874006bf555bc80cddc7c5","object":"chat.completion","created":1745993638,"model":"deepseek-ai/DeepSeek-V3-0324","choices":[{"index":0,"message":{"role":"assistant","content":null,"reasoning_content":null,"tool_calls":[{"id":"0","index":null,"type":"function","function":{"name":"query_weather","arguments":"{\"city\": \"Qingdao\"}"}}]},"logprobs":null,"finish_reason":"tool_calls","matched_stop":null}],"usage":{"prompt_tokens":116,"total_tokens":138,"completion_tokens":22,"prompt_tokens_details":null}} + +``` +Sample Streaming Request: +``` +curl "http://127.0.0.1:30000/v1/chat/completions" \ +-H "Content-Type: application/json" \ +-d '{"temperature": 0, "max_tokens": 100, "model": "deepseek-ai/DeepSeek-V3-0324","stream":true,"tools": [{"type": "function", "function": {"name": "query_weather", "description": "Get weather of an city, the user should supply a city first", "parameters": {"type": "object", "properties": {"city": {"type": "string", "description": "The city, e.g. Beijing"}}, "required": ["city"]}}}], "messages": [{"role": "user", "content": "Hows the weather like in Qingdao today"}]}' +``` +Expected Streamed Chunks (simplified for clarity): +``` +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"{\""}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"city"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\":\""}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"Q"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"ing"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"dao"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":[{"function":{"arguments":"\"}"}}]}}]} +data: {"choices":[{"delta":{"tool_calls":null}}], "finish_reason": "tool_calls"} +data: [DONE] +``` +The client needs to concatenate all arguments fragments to reconstruct the complete tool call: +``` +{"city": "Qingdao"} +``` +Important Notes: +1. Use a lower `"temperature"` value for better results. +2. To receive more consistent tool call results, it is recommended to use `--chat-template examples/chat_template/tool_chat_template_deepseekv3.jinja`. It provides an improved unified prompt. + + +## FAQ + +**Q: Model loading is taking too long, and I'm encountering an NCCL timeout. What should I do?** + +A: If you're experiencing extended model loading times and an NCCL timeout, you can try increasing the timeout duration. Add the argument `--dist-timeout 3600` when launching your model. This will set the timeout to one hour, which often resolves the issue. diff --git a/docs/basic_usage/gpt_oss.md b/docs/basic_usage/gpt_oss.md new file mode 100644 index 000000000..240463ec4 --- /dev/null +++ b/docs/basic_usage/gpt_oss.md @@ -0,0 +1,114 @@ +# GPT OSS Usage + +Please refer to [https://github.com/sgl-project/sglang/issues/8833](https://github.com/sgl-project/sglang/issues/8833). + +## Responses API & Built-in Tools + +### Responses API + +GPT‑OSS is compatible with the OpenAI Responses API. Use `client.responses.create(...)` with `model`, `instructions`, `input`, and optional `tools` to enable built‑in tool use. + +### Built-in Tools + +GPT‑OSS can call built‑in tools for web search and Python execution. You can use the demo tool server or connect to external MCP tool servers. + +#### Python Tool + +- Executes short Python snippets for calculations, parsing, and quick scripts. +- By default runs in a Docker-based sandbox. To run on the host, set `PYTHON_EXECUTION_BACKEND=UV` (this executes model-generated code locally; use with care). +- Ensure Docker is available if you are not using the UV backend. It is recommended to run `docker pull python:3.11` in advance. + +#### Web Search Tool + +- Uses the Exa backend for web search. +- Requires an Exa API key; set `EXA_API_KEY` in your environment. Create a key at `https://exa.ai`. + +### Tool & Reasoning Parser + +- We support OpenAI Reasoning and Tool Call parser, as well as our SGLang native api for tool call and reasoning. Refer to [reasoning parser](../advanced_features/separate_reasoning.ipynb) and [tool call parser](../advanced_features/function_calling.ipynb) for more details. + + +## Notes + +- Use **Python 3.12** for the demo tools. And install the required `gpt-oss` packages. +- The default demo integrates the web search tool (Exa backend) and a demo Python interpreter via Docker. +- For search, set `EXA_API_KEY`. For Python execution, either have Docker available or set `PYTHON_EXECUTION_BACKEND=UV`. + +Examples: +```bash +export EXA_API_KEY=YOUR_EXA_KEY +# Optional: run Python tool locally instead of Docker (use with care) +export PYTHON_EXECUTION_BACKEND=UV +``` + +Launch the server with the demo tool server: + +`python3 -m sglang.launch_server --model-path openai/gpt-oss-120b --tool-server demo --tp 2` + +For production usage, sglang can act as an MCP client for multiple services. An [example tool server](https://github.com/openai/gpt-oss/tree/main/gpt-oss-mcp-server) is provided. Start the servers and point sglang to them: +```bash +mcp run -t sse browser_server.py:mcp +mcp run -t sse python_server.py:mcp + +python -m sglang.launch_server ... --tool-server ip-1:port-1,ip-2:port-2 +``` +The URLs should be MCP SSE servers that expose server information and well-documented tools. These tools are added to the system prompt so the model can use them. + +### Quick Demo + +```python +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:30000/v1", + api_key="sk-123456" +) + +tools = [ + {"type": "code_interpreter"}, + {"type": "web_search_preview"}, +] + +# Test python tool +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helfpul assistant, you could use python tool to execute code.", + input="Use python tool to calculate the sum of 29138749187 and 29138749187", # 58,277,498,374 + tools=tools +) +print("====== test python tool ======") +print(response.output_text) + +# Test browser tool +response = client.responses.create( + model="openai/gpt-oss-120b", + instructions="You are a helfpul assistant, you could use browser to search the web", + input="Search the web for the latest news about Nvidia stock price", + tools=tools +) +print("====== test browser tool ======") +print(response.output_text) +``` + +Example output: +``` +====== test python tool ====== +The sum of 29,138,749,187 and 29,138,749,187 is **58,277,498,374**. +====== test browser tool ====== +**Recent headlines on Nvidia (NVDA) stock** + +| Date (2025) | Source | Key news points | Stock‑price detail | +|-------------|--------|----------------|--------------------| +| **May 13** | Reuters | The market data page shows Nvidia trading “higher” at **$116.61** with no change from the previous close. | **$116.61** – latest trade (delayed ≈ 15 min)【14†L34-L38】 | +| **Aug 18** | CNBC | Morgan Stanley kept an **overweight** rating and lifted its price target to **$206** (up from $200), implying a 14 % upside from the Friday close. The firm notes Nvidia shares have already **jumped 34 % this year**. | No exact price quoted, but the article signals strong upside expectations【9†L27-L31】 | +| **Aug 20** | The Motley Fool | Nvidia is set to release its Q2 earnings on Aug 27. The article lists the **current price of $175.36**, down 0.16 % on the day (as of 3:58 p.m. ET). | **$175.36** – current price on Aug 20【10†L12-L15】【10†L53-L57】 | + +**What the news tells us** + +* Nvidia’s share price has risen sharply this year – up roughly a third according to Morgan Stanley – and analysts are still raising targets (now $206). +* The most recent market quote (Reuters, May 13) was **$116.61**, but the stock has surged since then, reaching **$175.36** by mid‑August. +* Upcoming earnings on **Aug 27** are a focal point; both the Motley Fool and Morgan Stanley expect the results could keep the rally going. + +**Bottom line:** Nvidia’s stock is on a strong upward trajectory in 2025, with price targets climbing toward $200‑$210 and the market price already near $175 as of late August. + +``` diff --git a/docs/basic_usage/llama4.md b/docs/basic_usage/llama4.md new file mode 100644 index 000000000..07cc2b737 --- /dev/null +++ b/docs/basic_usage/llama4.md @@ -0,0 +1,61 @@ +# Llama4 Usage + +[Llama 4](https://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md) is Meta's latest generation of open-source LLM model with industry-leading performance. + +SGLang has supported Llama 4 Scout (109B) and Llama 4 Maverick (400B) since [v0.4.5](https://github.com/sgl-project/sglang/releases/tag/v0.4.5). + +Ongoing optimizations are tracked in the [Roadmap](https://github.com/sgl-project/sglang/issues/5118). + +## Launch Llama 4 with SGLang + +To serve Llama 4 models on 8xH100/H200 GPUs: + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct --tp 8 --context-length 1000000 +``` + +### Configuration Tips + +- **OOM Mitigation**: Adjust `--context-length` to avoid a GPU out-of-memory issue. For the Scout model, we recommend setting this value up to 1M on 8\*H100 and up to 2.5M on 8\*H200. For the Maverick model, we don't need to set context length on 8\*H200. When hybrid kv cache is enabled, `--context-length` can be set up to 5M on 8\*H100 and up to 10M on 8\*H200 for the Scout model. + +- **Chat Template**: Add `--chat-template llama-4` for chat completion tasks. +- **Enable Multi-Modal**: Add `--enable-multimodal` for multi-modal capabilities. +- **Enable Hybrid-KVCache**: Add `--hybrid-kvcache-ratio` for hybrid kv cache. Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/6563) + + +### EAGLE Speculative Decoding +**Description**: SGLang has supported Llama 4 Maverick (400B) with [EAGLE speculative decoding](https://docs.sglang.ai/backend/speculative_decoding.html#EAGLE-Decoding). + +**Usage**: +Add arguments `--speculative-draft-model-path`, `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: +``` +python3 -m sglang.launch_server --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct --speculative-algorithm EAGLE3 --speculative-draft-model-path nvidia/Llama-4-Maverick-17B-128E-Eagle3 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --trust-remote-code --tp 8 --context-length 1000000 +``` + +- **Note** The Llama 4 draft model *nvidia/Llama-4-Maverick-17B-128E-Eagle3* can only recognize conversations in chat mode. + +## Benchmarking Results + +### Accuracy Test with `lm_eval` + +The accuracy on SGLang for both Llama4 Scout and Llama4 Maverick can match the [official benchmark numbers](https://ai.meta.com/blog/llama-4-multimodal-intelligence/). + +Benchmark results on MMLU Pro dataset with 8*H100: +| | Llama-4-Scout-17B-16E-Instruct | Llama-4-Maverick-17B-128E-Instruct | +|--------------------|--------------------------------|-------------------------------------| +| Official Benchmark | 74.3 | 80.5 | +| SGLang | 75.2 | 80.7 | + +Commands: + +```bash +# Llama-4-Scout-17B-16E-Instruct model +python -m sglang.launch_server --model-path meta-llama/Llama-4-Scout-17B-16E-Instruct --port 30000 --tp 8 --mem-fraction-static 0.8 --context-length 65536 +lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Scout-17B-16E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 + +# Llama-4-Maverick-17B-128E-Instruct +python -m sglang.launch_server --model-path meta-llama/Llama-4-Maverick-17B-128E-Instruct --port 30000 --tp 8 --mem-fraction-static 0.8 --context-length 65536 +lm_eval --model local-chat-completions --model_args model=meta-llama/Llama-4-Maverick-17B-128E-Instruct,base_url=http://localhost:30000/v1/chat/completions,num_concurrent=128,timeout=999999,max_gen_toks=2048 --tasks mmlu_pro --batch_size 128 --apply_chat_template --num_fewshot 0 +``` + +Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/5092). diff --git a/docs/basic_usage/native_api.ipynb b/docs/basic_usage/native_api.ipynb new file mode 100644 index 000000000..5e4ca19a1 --- /dev/null +++ b/docs/basic_usage/native_api.ipynb @@ -0,0 +1,497 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Native APIs\n", + "\n", + "Apart from the OpenAI compatible APIs, the SGLang Runtime also provides its native server APIs. We introduce the following APIs:\n", + "\n", + "- `/generate` (text generation model)\n", + "- `/get_model_info`\n", + "- `/get_server_info`\n", + "- `/health`\n", + "- `/health_generate`\n", + "- `/flush_cache`\n", + "- `/update_weights`\n", + "- `/encode`(embedding model)\n", + "- `/v1/rerank`(cross encoder rerank model)\n", + "- `/classify`(reward model)\n", + "- `/start_expert_distribution_record`\n", + "- `/stop_expert_distribution_record`\n", + "- `/dump_expert_distribution_record`\n", + "- A full list of these APIs can be found at [http_server.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/http_server.py)\n", + "\n", + "We mainly use `requests` to test these APIs in the following examples. You can also use `curl`.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Generate (text generation model)\n", + "Generate completions. This is similar to the `/v1/completions` in OpenAI API. Detailed parameters can be found in the [sampling parameters](sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Model Info\n", + "\n", + "Get the information of the model.\n", + "\n", + "- `model_path`: The path/name of the model.\n", + "- `is_generation`: Whether the model is used as generation model or embedding model.\n", + "- `tokenizer_path`: The path/name of the tokenizer.\n", + "- `preferred_sampling_params`: The default sampling params specified via `--preferred-sampling-params`. `None` is returned in this example as we did not explicitly configure it in server args.\n", + "- `weight_version`: This field contains the version of the model weights. This is often used to track changes or updates to the model’s trained parameters." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/get_model_info\"\n", + "\n", + "response = requests.get(url)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"model_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", + "assert response_json[\"is_generation\"] is True\n", + "assert response_json[\"tokenizer_path\"] == \"qwen/qwen2.5-0.5b-instruct\"\n", + "assert response_json[\"preferred_sampling_params\"] is None\n", + "assert response_json.keys() == {\n", + " \"model_path\",\n", + " \"is_generation\",\n", + " \"tokenizer_path\",\n", + " \"preferred_sampling_params\",\n", + " \"weight_version\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get Server Info\n", + "Gets the server information including CLI arguments, token limits, and memory pool sizes.\n", + "- Note: `get_server_info` merges the following deprecated endpoints:\n", + " - `get_server_args`\n", + " - `get_memory_pool_size` \n", + " - `get_max_total_num_tokens`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/get_server_info\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Health Check\n", + "- `/health`: Check the health of the server.\n", + "- `/health_generate`: Check the health of the server by generating one token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health_generate\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/health\"\n", + "\n", + "response = requests.get(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Flush Cache\n", + "\n", + "Flush the radix cache. It will be automatically triggered when the model weights are updated by the `/update_weights` API." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "url = f\"http://localhost:{port}/flush_cache\"\n", + "\n", + "response = requests.post(url)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update Weights From Disk\n", + "\n", + "Update model weights from disk without restarting the server. Only applicable for models with the same architecture and parameter size.\n", + "\n", + "SGLang support `update_weights_from_disk` API for continuous evaluation during training (save checkpoint to disk and update weights from disk).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful update with same architecture and size\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)\n", + "assert response.json()[\"success\"] is True\n", + "assert response.json()[\"message\"] == \"Succeeded to update model weights.\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# failed update with different parameter size or wrong name\n", + "\n", + "url = f\"http://localhost:{port}/update_weights_from_disk\"\n", + "data = {\"model_path\": \"qwen/qwen2.5-0.5b-instruct-wrong\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(response_json)\n", + "assert response_json[\"success\"] is False\n", + "assert response_json[\"message\"] == (\n", + " \"Failed to get weights iterator: \"\n", + " \"qwen/qwen2.5-0.5b-instruct-wrong\"\n", + " \" (repository not found).\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Encode (embedding model)\n", + "\n", + "Encode text into embeddings. Note that this API is only available for [embedding models](openai_api_embeddings.ipynb) and will raise an error for generation models.\n", + "Therefore, we launch a new server to server an embedding model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", + " --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# successful encode for embedding model\n", + "\n", + "url = f\"http://localhost:{port}/encode\"\n", + "data = {\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"text\": \"Once upon a time\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "print_highlight(f\"Text embedding (first 10): {response_json['embedding'][:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## v1/rerank (cross encoder rerank model)\n", + "Rerank a list of documents given a query using a cross-encoder model. Note that this API is only available for cross encoder model like [BAAI/bge-reranker-v2-m3](https://huggingface.co/BAAI/bge-reranker-v2-m3) with `attention-backend` `triton` and `torch_native`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reranker_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path BAAI/bge-reranker-v2-m3 \\\n", + " --host 0.0.0.0 --disable-radix-cache --chunked-prefill-size -1 --attention-backend triton --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# compute rerank scores for query and documents\n", + "\n", + "url = f\"http://localhost:{port}/v1/rerank\"\n", + "data = {\n", + " \"model\": \"BAAI/bge-reranker-v2-m3\",\n", + " \"query\": \"what is panda?\",\n", + " \"documents\": [\n", + " \"hi\",\n", + " \"The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China.\",\n", + " ],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "response_json = response.json()\n", + "for item in response_json:\n", + " print_highlight(f\"Score: {item['score']:.2f} - Document: '{item['document']}'\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(reranker_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classify (reward model)\n", + "\n", + "SGLang Runtime also supports reward models. Here we use a reward model to classify the quality of pairwise generations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Note that SGLang now treats embedding models and reward models as the same type of models.\n", + "# This will be updated in the future.\n", + "\n", + "reward_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Skywork/Skywork-Reward-Llama-3.1-8B-v0.2 --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "PROMPT = (\n", + " \"What is the range of the numeric output of a sigmoid node in a neural network?\"\n", + ")\n", + "\n", + "RESPONSE1 = \"The output of a sigmoid node is bounded between -1 and 1.\"\n", + "RESPONSE2 = \"The output of a sigmoid node is bounded between 0 and 1.\"\n", + "\n", + "CONVS = [\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE1}],\n", + " [{\"role\": \"user\", \"content\": PROMPT}, {\"role\": \"assistant\", \"content\": RESPONSE2}],\n", + "]\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\")\n", + "prompts = tokenizer.apply_chat_template(CONVS, tokenize=False)\n", + "\n", + "url = f\"http://localhost:{port}/classify\"\n", + "data = {\"model\": \"Skywork/Skywork-Reward-Llama-3.1-8B-v0.2\", \"text\": prompts}\n", + "\n", + "responses = requests.post(url, json=data).json()\n", + "for response in responses:\n", + " print_highlight(f\"reward: {response['embedding'][0]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(reward_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Capture expert selection distribution in MoE models\n", + "\n", + "SGLang Runtime supports recording the number of times an expert is selected in a MoE model run for each expert in the model. This is useful when analyzing the throughput of the model and plan for optimization.\n", + "\n", + "*Note: We only print out the first 10 lines of the csv below for better readability. Please adjust accordingly if you want to analyze the results more deeply.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "expert_record_server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen1.5-MoE-A2.7B --host 0.0.0.0 --expert-distribution-recorder-mode stat --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = requests.post(f\"http://localhost:{port}/start_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "url = f\"http://localhost:{port}/generate\"\n", + "data = {\"text\": \"What is the capital of France?\"}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/stop_expert_distribution_record\")\n", + "print_highlight(response)\n", + "\n", + "response = requests.post(f\"http://localhost:{port}/dump_expert_distribution_record\")\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(expert_record_server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/basic_usage/offline_engine_api.ipynb b/docs/basic_usage/offline_engine_api.ipynb new file mode 100644 index 000000000..9c03e90a7 --- /dev/null +++ b/docs/basic_usage/offline_engine_api.ipynb @@ -0,0 +1,235 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Offline Engine API\n", + "\n", + "SGLang provides a direct inference engine without the need for an HTTP server, especially for use cases where additional HTTP server adds unnecessary complexity or overhead. Here are two general use cases:\n", + "\n", + "- Offline Batch Inference\n", + "- Custom Server on Top of the Engine\n", + "\n", + "This document focuses on the offline batch inference, demonstrating four different inference modes:\n", + "\n", + "- Non-streaming synchronous generation\n", + "- Streaming synchronous generation\n", + "- Non-streaming asynchronous generation\n", + "- Streaming asynchronous generation\n", + "\n", + "Additionally, you can easily build a custom server on top of the SGLang offline engine. A detailed example working in a python script can be found in [custom_server](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/custom_server.py).\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Nest Asyncio\n", + "Note that if you want to use **Offline Engine** in ipython or some other nested loop code, you need to add the following code:\n", + "```python\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()\n", + "\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Advanced Usage\n", + "\n", + "The engine supports [vlm inference](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py) as well as [extracting hidden states](https://github.com/sgl-project/sglang/blob/main/examples/runtime/hidden_states). \n", + "\n", + "Please see [the examples](https://github.com/sgl-project/sglang/tree/main/examples/runtime/engine) for further use cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Offline Batch Inference\n", + "\n", + "SGLang offline engine supports batch inference with efficient scheduling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# launch the offline engine\n", + "import asyncio\n", + "\n", + "import sglang as sgl\n", + "import sglang.test.doc_patch\n", + "from sglang.utils import async_stream_and_merge, stream_and_merge\n", + "\n", + "llm = sgl.Engine(model_path=\"qwen/qwen2.5-0.5b-instruct\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Hello, my name is\",\n", + " \"The president of the United States is\",\n", + " \"The capital of France is\",\n", + " \"The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "outputs = llm.generate(prompts, sampling_params)\n", + "for prompt, output in zip(prompts, outputs):\n", + " print(\"===============================\")\n", + " print(f\"Prompt: {prompt}\\nGenerated text: {output['text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Synchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\n", + " \"temperature\": 0.2,\n", + " \"top_p\": 0.9,\n", + "}\n", + "\n", + "print(\"\\n=== Testing synchronous streaming generation with overlap removal ===\\n\")\n", + "\n", + "for prompt in prompts:\n", + " print(f\"Prompt: {prompt}\")\n", + " merged_output = stream_and_merge(llm, prompt, sampling_params)\n", + " print(\"Generated text:\", merged_output)\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Non-streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous batch generation ===\")\n", + "\n", + "\n", + "async def main():\n", + " outputs = await llm.async_generate(prompts, sampling_params)\n", + "\n", + " for prompt, output in zip(prompts, outputs):\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(f\"Generated text: {output['text']}\")\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming Asynchronous Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "prompts = [\n", + " \"Write a short, neutral self-introduction for a fictional character. Hello, my name is\",\n", + " \"Provide a concise factual statement about France’s capital city. The capital of France is\",\n", + " \"Explain possible future trends in artificial intelligence. The future of AI is\",\n", + "]\n", + "\n", + "sampling_params = {\"temperature\": 0.8, \"top_p\": 0.95}\n", + "\n", + "print(\"\\n=== Testing asynchronous streaming generation (no repeats) ===\")\n", + "\n", + "\n", + "async def main():\n", + " for prompt in prompts:\n", + " print(f\"\\nPrompt: {prompt}\")\n", + " print(\"Generated text: \", end=\"\", flush=True)\n", + "\n", + " # Replace direct calls to async_generate with our custom overlap-aware version\n", + " async for cleaned_chunk in async_stream_and_merge(llm, prompt, sampling_params):\n", + " print(cleaned_chunk, end=\"\", flush=True)\n", + "\n", + " print() # New line after each prompt\n", + "\n", + "\n", + "asyncio.run(main())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm.shutdown()" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/basic_usage/openai_api.rst b/docs/basic_usage/openai_api.rst new file mode 100644 index 000000000..370abe99c --- /dev/null +++ b/docs/basic_usage/openai_api.rst @@ -0,0 +1,9 @@ +OpenAI-Compatible APIs +====================== + +.. toctree:: + :maxdepth: 1 + + openai_api_completions.ipynb + openai_api_vision.ipynb + openai_api_embeddings.ipynb diff --git a/docs/basic_usage/openai_api_completions.ipynb b/docs/basic_usage/openai_api_completions.ipynb new file mode 100644 index 000000000..6b967709f --- /dev/null +++ b/docs/basic_usage/openai_api_completions.ipynb @@ -0,0 +1,389 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Completions\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/api-reference).\n", + "\n", + "This tutorial covers the following popular APIs:\n", + "\n", + "- `chat/completions`\n", + "- `completions`\n", + "\n", + "Check out other tutorials to learn about [vision APIs](openai_api_vision.ipynb) for vision-language models and [embedding APIs](openai_api_embeddings.ipynb) for embedding models." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chat Completions\n", + "\n", + "### Usage\n", + "\n", + "The server fully implements the OpenAI API.\n", + "It will automatically apply the chat template specified in the Hugging Face tokenizer, if one is available.\n", + "You can also specify a custom chat template with `--chat-template` when launching the server." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Model Thinking/Reasoning Support\n", + "\n", + "Some models support internal reasoning or thinking processes that can be exposed in the API response. SGLang provides unified support for various reasoning models through the `chat_template_kwargs` parameter and compatible reasoning parsers.\n", + "\n", + "#### Supported Models and Configuration\n", + "\n", + "| Model Family | Chat Template Parameter | Reasoning Parser | Notes |\n", + "|--------------|------------------------|------------------|--------|\n", + "| DeepSeek-R1 (R1, R1-0528, R1-Distill) | `enable_thinking` | `--reasoning-parser deepseek-r1` | Standard reasoning models |\n", + "| DeepSeek-V3.1 | `thinking` | `--reasoning-parser deepseek-v3` | Hybrid model (thinking/non-thinking modes) |\n", + "| Qwen3 (standard) | `enable_thinking` | `--reasoning-parser qwen3` | Hybrid model (thinking/non-thinking modes) |\n", + "| Qwen3-Thinking | N/A (always enabled) | `--reasoning-parser qwen3-thinking` | Always generates reasoning |\n", + "| Kimi | N/A (always enabled) | `--reasoning-parser kimi` | Kimi thinking models |\n", + "| Gpt-Oss | N/A (always enabled) | `--reasoning-parser gpt-oss` | Gpt-Oss thinking models |\n", + "\n", + "#### Basic Usage\n", + "\n", + "To enable reasoning output, you need to:\n", + "1. Launch the server with the appropriate reasoning parser\n", + "2. Set the model-specific parameter in `chat_template_kwargs`\n", + "3. Optionally use `separate_reasoning: False` to not get reasoning content separately (default to `True`)\n", + "\n", + "**Note for Qwen3-Thinking models:** These models always generate thinking content and do not support the `enable_thinking` parameter. Use `--reasoning-parser qwen3-thinking` or `--reasoning-parser qwen3` to parse the thinking content.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Example: Qwen3 Models\n", + "\n", + "```python\n", + "# Launch server:\n", + "# python3 -m sglang.launch_server --model Qwen/Qwen3-4B --reasoning-parser qwen3\n", + "\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(\n", + " api_key=\"EMPTY\",\n", + " base_url=f\"http://127.0.0.1:30000/v1\",\n", + ")\n", + "\n", + "model = \"Qwen/Qwen3-4B\"\n", + "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", + "\n", + "response = client.chat.completions.create(\n", + " model=model,\n", + " messages=messages,\n", + " extra_body={\n", + " \"chat_template_kwargs\": {\"enable_thinking\": True},\n", + " \"separate_reasoning\": True\n", + " }\n", + ")\n", + "\n", + "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", + "print(\"-\"*100)\n", + "print(\"Answer:\", response.choices[0].message.content)\n", + "```\n", + "\n", + "**ExampleOutput:**\n", + "```\n", + "Reasoning: Okay, so the user is asking how many 'r's are in the word 'strawberry'. Let me think. First, I need to make sure I have the word spelled correctly. Strawberry... S-T-R-A-W-B-E-R-R-Y. Wait, is that right? Let me break it down.\n", + "\n", + "Starting with 'strawberry', let's write out the letters one by one. S, T, R, A, W, B, E, R, R, Y. Hmm, wait, that's 10 letters. Let me check again. S (1), T (2), R (3), A (4), W (5), B (6), E (7), R (8), R (9), Y (10). So the letters are S-T-R-A-W-B-E-R-R-Y. \n", + "...\n", + "Therefore, the answer should be three R's in 'strawberry'. But I need to make sure I'm not counting any other letters as R. Let me check again. S, T, R, A, W, B, E, R, R, Y. No other R's. So three in total. Yeah, that seems right.\n", + "\n", + "----------------------------------------------------------------------------------------------------\n", + "Answer: The word \"strawberry\" contains **three** letters 'r'. Here's the breakdown:\n", + "\n", + "1. **S-T-R-A-W-B-E-R-R-Y** \n", + " - The **third letter** is 'R'. \n", + " - The **eighth and ninth letters** are also 'R's. \n", + "\n", + "Thus, the total count is **3**. \n", + "\n", + "**Answer:** 3.\n", + "```\n", + "\n", + "**Note:** Setting `\"enable_thinking\": False` (or omitting it) will result in `reasoning_content` being `None`. Qwen3-Thinking models always generate reasoning content and don't support the `enable_thinking` parameter.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Example: DeepSeek-V3 Models\n", + "\n", + "DeepSeek-V3 models support thinking mode through the `thinking` parameter:\n", + "\n", + "```python\n", + "# Launch server:\n", + "# python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3.1 --tp 8 --reasoning-parser deepseek-v3\n", + "\n", + "from openai import OpenAI\n", + "\n", + "client = OpenAI(\n", + " api_key=\"EMPTY\",\n", + " base_url=f\"http://127.0.0.1:30000/v1\",\n", + ")\n", + "\n", + "model = \"deepseek-ai/DeepSeek-V3.1\"\n", + "messages = [{\"role\": \"user\", \"content\": \"How many r's are in 'strawberry'?\"}]\n", + "\n", + "response = client.chat.completions.create(\n", + " model=model,\n", + " messages=messages,\n", + " extra_body={\n", + " \"chat_template_kwargs\": {\"thinking\": True},\n", + " \"separate_reasoning\": True\n", + " }\n", + ")\n", + "\n", + "print(\"Reasoning:\", response.choices[0].message.reasoning_content)\n", + "print(\"-\"*100)\n", + "print(\"Answer:\", response.choices[0].message.content)\n", + "```\n", + "\n", + "**Example Output:**\n", + "```\n", + "Reasoning: First, the question is: \"How many r's are in 'strawberry'?\"\n", + "\n", + "I need to count the number of times the letter 'r' appears in the word \"strawberry\".\n", + "\n", + "Let me write out the word: S-T-R-A-W-B-E-R-R-Y.\n", + "\n", + "Now, I'll go through each letter and count the 'r's.\n", + "...\n", + "So, I have three 'r's in \"strawberry\".\n", + "\n", + "I should double-check. The word is spelled S-T-R-A-W-B-E-R-R-Y. The letters are at positions: 3, 8, and 9 are 'r's. Yes, that's correct.\n", + "\n", + "Therefore, the answer should be 3.\n", + "----------------------------------------------------------------------------------------------------\n", + "Answer: The word \"strawberry\" contains **3** instances of the letter \"r\". Here's a breakdown for clarity:\n", + "\n", + "- The word is spelled: S-T-R-A-W-B-E-R-R-Y\n", + "- The \"r\" appears at the 3rd, 8th, and 9th positions.\n", + "```\n", + "\n", + "**Note:** DeepSeek-V3 models use the `thinking` parameter (not `enable_thinking`) to control reasoning output.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The chat completions API accepts OpenAI Chat Completions API's parameters. Refer to [OpenAI Chat Completions API](https://platform.openai.com/docs/api-reference/chat/create) for more details.\n", + "\n", + "SGLang extends the standard API with the `extra_body` parameter, allowing for additional customization. One key option within `extra_body` is `chat_template_kwargs`, which can be used to pass arguments to the chat template processor." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"system\",\n", + " \"content\": \"You are a knowledgeable historian who provides concise responses.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"Tell me about ancient Rome\"},\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"content\": \"Ancient Rome was a civilization centered in Italy.\",\n", + " },\n", + " {\"role\": \"user\", \"content\": \"What were their major achievements?\"},\n", + " ],\n", + " temperature=0.3, # Lower temperature for more focused responses\n", + " max_tokens=128, # Reasonable length for a concise response\n", + " top_p=0.95, # Slightly higher for better fluency\n", + " presence_penalty=0.2, # Mild penalty to avoid repetition\n", + " frequency_penalty=0.2, # Mild penalty for more natural language\n", + " n=1, # Single response is usually more stable\n", + " seed=42, # Keep for reproducibility\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Streaming mode is also supported." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stream = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Say this is a test\"}],\n", + " stream=True,\n", + ")\n", + "for chunk in stream:\n", + " if chunk.choices[0].delta.content is not None:\n", + " print(chunk.choices[0].delta.content, end=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Completions\n", + "\n", + "### Usage\n", + "Completions API is similar to Chat Completions API, but without the `messages` parameter or chat templates." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " prompt=\"List 3 countries and their capitals.\",\n", + " temperature=0,\n", + " max_tokens=64,\n", + " n=1,\n", + " stop=None,\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parameters\n", + "\n", + "The completions API accepts OpenAI Completions API's parameters. Refer to [OpenAI Completions API](https://platform.openai.com/docs/api-reference/completions/create) for more details.\n", + "\n", + "Here is an example of a detailed completions request:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " prompt=\"Write a short story about a space explorer.\",\n", + " temperature=0.7, # Moderate temperature for creative writing\n", + " max_tokens=150, # Longer response for a story\n", + " top_p=0.9, # Balanced diversity in word choice\n", + " stop=[\"\\n\\n\", \"THE END\"], # Multiple stop sequences\n", + " presence_penalty=0.3, # Encourage novel elements\n", + " frequency_penalty=0.3, # Reduce repetitive phrases\n", + " n=1, # Generate one completion\n", + " seed=123, # For reproducible results\n", + ")\n", + "\n", + "print_highlight(f\"Response: {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs (JSON, Regex, EBNF)\n", + "\n", + "For OpenAI compatible structured outputs API, refer to [Structured Outputs](../advanced_features/structured_outputs.ipynb) for more details.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/basic_usage/openai_api_embeddings.ipynb b/docs/basic_usage/openai_api_embeddings.ipynb new file mode 100644 index 000000000..26e95a4e7 --- /dev/null +++ b/docs/basic_usage/openai_api_embeddings.ipynb @@ -0,0 +1,195 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Embedding\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/embeddings).\n", + "\n", + "This tutorial covers the embedding APIs for embedding models. For a list of the supported models see the [corresponding overview page](../supported_models/embedding_models.md)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize. Remember to add `--is-embedding` to the command." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "embedding_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct \\\n", + " --host 0.0.0.0 --is-embedding --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "curl_text = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": \"{text}\"}}'\"\"\"\n", + "\n", + "result = subprocess.check_output(curl_text, shell=True)\n", + "\n", + "print(result)\n", + "\n", + "text_embedding = json.loads(result)[\"data\"][0][\"embedding\"]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "text = \"Once upon a time\"\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/v1/embeddings\",\n", + " json={\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": text},\n", + ")\n", + "\n", + "text_embedding = response.json()[\"data\"][0][\"embedding\"]\n", + "\n", + "print_highlight(f\"Text embedding (first 10): {text_embedding[:10]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Text embedding example\n", + "response = client.embeddings.create(\n", + " model=\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\",\n", + " input=text,\n", + ")\n", + "\n", + "embedding = response.data[0].embedding[:10]\n", + "print_highlight(f\"Text embedding (first 10): {embedding}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Input IDs\n", + "\n", + "SGLang also supports `input_ids` as input to get the embedding." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from transformers import AutoTokenizer\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\"Alibaba-NLP/gte-Qwen2-1.5B-instruct\")\n", + "input_ids = tokenizer.encode(text)\n", + "\n", + "curl_ids = f\"\"\"curl -s http://localhost:{port}/v1/embeddings \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"Alibaba-NLP/gte-Qwen2-1.5B-instruct\", \"input\": {json.dumps(input_ids)}}}'\"\"\"\n", + "\n", + "input_ids_embedding = json.loads(subprocess.check_output(curl_ids, shell=True))[\"data\"][\n", + " 0\n", + "][\"embedding\"]\n", + "\n", + "print_highlight(f\"Input IDs embedding (first 10): {input_ids_embedding[:10]}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(embedding_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-Modal Embedding Model\n", + "Please refer to [Multi-Modal Embedding Model](../supported_models/embedding_models.md)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/basic_usage/openai_api_vision.ipynb b/docs/basic_usage/openai_api_vision.ipynb new file mode 100644 index 000000000..88d1ef7dd --- /dev/null +++ b/docs/basic_usage/openai_api_vision.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# OpenAI APIs - Vision\n", + "\n", + "SGLang provides OpenAI-compatible APIs to enable a smooth transition from OpenAI services to self-hosted local models.\n", + "A complete reference for the API is available in the [OpenAI API Reference](https://platform.openai.com/docs/guides/vision).\n", + "This tutorial covers the vision APIs for vision language models.\n", + "\n", + "SGLang supports various vision language models such as Llama 3.2, LLaVA-OneVision, Qwen2.5-VL, Gemma3 and [more](../supported_models/multimodal_language_models.md).\n", + "\n", + "As an alternative to the OpenAI API, you can also use the [SGLang offline engine](https://github.com/sgl-project/sglang/blob/main/examples/runtime/engine/offline_batch_inference_vlm.py)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "vision_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n", + "\n", + "Once the server is up, you can send test requests using curl or requests." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\\\n", + " -H \"Content-Type: application/json\" \\\\\n", + " -d '{{\n", + " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " \"messages\": [\n", + " {{\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {{\n", + " \"type\": \"text\",\n", + " \"text\": \"What’s in this image?\"\n", + " }},\n", + " {{\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {{\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " }}\n", + " }}\n", + " ]\n", + " }}\n", + " ],\n", + " \"max_tokens\": 300\n", + " }}'\n", + "\"\"\"\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)\n", + "\n", + "\n", + "response = subprocess.check_output(curl_command, shell=True).decode()\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"text\", \"text\": \"What’s in this image?\"},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " \"max_tokens\": 300,\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"What is in this image?\",\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + " },\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " max_tokens=300,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multiple-Image Inputs\n", + "\n", + "The server also supports multiple images and interleaved text and images if the model supports it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "\n", + "client = OpenAI(base_url=f\"http://localhost:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"Qwen/Qwen2.5-VL-7B-Instruct\",\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png\",\n", + " },\n", + " },\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\": \"I have two very different images. They are not related at all. \"\n", + " \"Please describe the first image in one sentence, and then describe the second image in another sentence.\",\n", + " },\n", + " ],\n", + " }\n", + " ],\n", + " temperature=0,\n", + ")\n", + "\n", + "print_highlight(response.choices[0].message.content)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(vision_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/basic_usage/qwen3.md b/docs/basic_usage/qwen3.md new file mode 100644 index 000000000..89295be60 --- /dev/null +++ b/docs/basic_usage/qwen3.md @@ -0,0 +1,27 @@ +# Qwen3-Next Usage + +SGLang has supported Qwen3-Next-80B-A3B-Instruct and Qwen3-Next-80B-A3B-Thinking since [this PR](https://github.com/sgl-project/sglang/pull/10233). + +## Launch Qwen3-Next with SGLang + +To serve Qwen3-Next models on 4xH100/H200 GPUs: + +```bash +python3 -m sglang.launch_server --model Qwen/Qwen3-Next-80B-A3B-Instruct --tp 4 +``` + +### Configuration Tips +- `--max-mamba-cache-size`: Adjust `--max-mamba-cache-size` to increase mamba cache space and max running requests capability. It will decrease KV cache space as a trade-off. You can adjust it according to workload. +- `--mamba-ssm-dtype`: `bfloat16` or `float32`, use `bfloat16` to save mamba cache size and `float32` to get more accurate results. The default setting is `float32`. + +### EAGLE Speculative Decoding +**Description**: SGLang has supported Qwen3-Next models with [EAGLE speculative decoding](https://docs.sglang.ai/advanced_features/speculative_decoding.html#EAGLE-Decoding). + +**Usage**: +Add arguments `--speculative-algorithm`, `--speculative-num-steps`, `--speculative-eagle-topk` and `--speculative-num-draft-tokens` to enable this feature. For example: + +``` bash +python3 -m sglang.launch_server --model Qwen/Qwen3-Next-80B-A3B-Instruct --tp 4 --speculative-num-steps 3 --speculative-eagle-topk 1 --speculative-num-draft-tokens 4 --speculative-algo NEXTN +``` + +Details can be seen in [this PR](https://github.com/sgl-project/sglang/pull/10233). diff --git a/docs/basic_usage/sampling_params.md b/docs/basic_usage/sampling_params.md new file mode 100644 index 000000000..c1394a9fd --- /dev/null +++ b/docs/basic_usage/sampling_params.md @@ -0,0 +1,305 @@ +# Sampling Parameters + +This doc describes the sampling parameters of the SGLang Runtime. It is the low-level endpoint of the runtime. +If you want a high-level endpoint that can automatically handle chat templates, consider using the [OpenAI Compatible API](openai_api_completions.ipynb). + +## `/generate` Endpoint + +The `/generate` endpoint accepts the following parameters in JSON format. For detailed usage, see the [native API doc](native_api.ipynb). The object is defined at `io_struct.py::GenerateReqInput`. You can also read the source code to find more arguments and docs. + +| Argument | Type/Default | Description | +|----------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| text | `Optional[Union[List[str], str]] = None` | The input prompt. Can be a single prompt or a batch of prompts. | +| input_ids | `Optional[Union[List[List[int]], List[int]]] = None` | The token IDs for text; one can specify either text or input_ids. | +| input_embeds | `Optional[Union[List[List[List[float]]], List[List[float]]]] = None` | The embeddings for input_ids; one can specify either text, input_ids, or input_embeds. | +| image_data | `Optional[Union[List[List[ImageDataItem]], List[ImageDataItem], ImageDataItem]] = None` | The image input. Can be an image instance, file name, URL, or base64 encoded string. Can be a single image, list of images, or list of lists of images. | +| audio_data | `Optional[Union[List[AudioDataItem], AudioDataItem]] = None` | The audio input. Can be a file name, URL, or base64 encoded string. | +| sampling_params | `Optional[Union[List[Dict], Dict]] = None` | The sampling parameters as described in the sections below. | +| rid | `Optional[Union[List[str], str]] = None` | The request ID. | +| return_logprob | `Optional[Union[List[bool], bool]] = None` | Whether to return log probabilities for tokens. | +| logprob_start_len | `Optional[Union[List[int], int]] = None` | If return_logprob, the start location in the prompt for returning logprobs. Default is "-1", which returns logprobs for output tokens only. | +| top_logprobs_num | `Optional[Union[List[int], int]] = None` | If return_logprob, the number of top logprobs to return at each position. | +| token_ids_logprob | `Optional[Union[List[List[int]], List[int]]] = None` | If return_logprob, the token IDs to return logprob for. | +| return_text_in_logprobs | `bool = False` | Whether to detokenize tokens in text in the returned logprobs. | +| stream | `bool = False` | Whether to stream output. | +| lora_path | `Optional[Union[List[Optional[str]], Optional[str]]] = None` | The path to the LoRA. | +| custom_logit_processor | `Optional[Union[List[Optional[str]], str]] = None` | Custom logit processor for advanced sampling control. Must be a serialized instance of `CustomLogitProcessor` using its `to_str()` method. For usage see below. | +| return_hidden_states | `Union[List[bool], bool] = False` | Whether to return hidden states. | + +## Sampling parameters + +The object is defined at `sampling_params.py::SamplingParams`. You can also read the source code to find more arguments and docs. + +### Core parameters + +| Argument | Type/Default | Description | +|-----------------|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| max_new_tokens | `int = 128` | The maximum output length measured in tokens. | +| stop | `Optional[Union[str, List[str]]] = None` | One or multiple [stop words](https://platform.openai.com/docs/api-reference/chat/create#chat-create-stop). Generation will stop if one of these words is sampled. | +| stop_token_ids | `Optional[List[int]] = None` | Provide stop words in the form of token IDs. Generation will stop if one of these token IDs is sampled. | +| temperature | `float = 1.0` | [Temperature](https://platform.openai.com/docs/api-reference/chat/create#chat-create-temperature) when sampling the next token. `temperature = 0` corresponds to greedy sampling, a higher temperature leads to more diversity. | +| top_p | `float = 1.0` | [Top-p](https://platform.openai.com/docs/api-reference/chat/create#chat-create-top_p) selects tokens from the smallest sorted set whose cumulative probability exceeds `top_p`. When `top_p = 1`, this reduces to unrestricted sampling from all tokens. | +| top_k | `int = -1` | [Top-k](https://developer.nvidia.com/blog/how-to-get-better-outputs-from-your-large-language-model/#predictability_vs_creativity) randomly selects from the `k` highest-probability tokens. | +| min_p | `float = 0.0` | [Min-p](https://github.com/huggingface/transformers/issues/27670) samples from tokens with probability larger than `min_p * highest_token_probability`. | + +### Penalizers + +| Argument | Type/Default | Description | +|--------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| frequency_penalty | `float = 0.0` | Penalizes tokens based on their frequency in generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of penalization grows linearly with each appearance of a token. | +| presence_penalty | `float = 0.0` | Penalizes tokens if they appeared in the generation so far. Must be between `-2` and `2` where negative numbers encourage repeatment of tokens and positive number encourages sampling of new tokens. The scaling of the penalization is constant if a token occurred. | +| min_new_tokens | `int = 0` | Forces the model to generate at least `min_new_tokens` until a stop word or EOS token is sampled. Note that this might lead to unintended behavior, for example, if the distribution is highly skewed towards these tokens. | + +### Constrained decoding + +Please refer to our dedicated guide on [constrained decoding](../advanced_features/structured_outputs.ipynb) for the following parameters. + +| Argument | Type/Default | Description | +|-----------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| json_schema | `Optional[str] = None` | JSON schema for structured outputs. | +| regex | `Optional[str] = None` | Regex for structured outputs. | +| ebnf | `Optional[str] = None` | EBNF for structured outputs. | +| structural_tag | `Optional[str] = None` | The structal tag for structured outputs. | + +### Other options + +| Argument | Type/Default | Description | +|-------------------------------|---------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------| +| n | `int = 1` | Specifies the number of output sequences to generate per request. (Generating multiple outputs in one request (n > 1) is discouraged; repeating the same prompts several times offers better control and efficiency.) | +| ignore_eos | `bool = False` | Don't stop generation when EOS token is sampled. | +| skip_special_tokens | `bool = True` | Remove special tokens during decoding. | +| spaces_between_special_tokens | `bool = True` | Whether or not to add spaces between special tokens during detokenization. | +| no_stop_trim | `bool = False` | Don't trim stop words or EOS token from the generated text. | +| custom_params | `Optional[List[Optional[Dict[str, Any]]]] = None` | Used when employing `CustomLogitProcessor`. For usage, see below. | + +## Examples + +### Normal + +Launch a server: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +Detailed example in [send request](./send_request.ipynb). + +### Streaming + +Send a request and stream the output: + +```python +import requests, json + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, +) + +prev = 0 +for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) +print("") +``` + +Detailed example in [openai compatible api](openai_api_completions.ipynb). + +### Multimodal + +Launch a server: + +```bash +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-7b-ov +``` + +Download an image: + +```bash +curl -o example_image.png -L https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n" + "<|im_start|>assistant\n", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, +) +print(response.json()) +``` + +The `image_data` can be a file name, a URL, or a base64 encoded string. See also `python/sglang/srt/utils.py:load_image`. + +Streaming is supported in a similar manner as [above](#streaming). + +Detailed example in [OpenAI API Vision](openai_api_vision.ipynb). + +### Structured Outputs (JSON, Regex, EBNF) + +You can specify a JSON schema, regular expression or [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) to constrain the model output. The model output will be guaranteed to follow the given constraints. Only one constraint parameter (`json_schema`, `regex`, or `ebnf`) can be specified for a request. + +SGLang supports two grammar backends: + +- [XGrammar](https://github.com/mlc-ai/xgrammar) (default): Supports JSON schema, regular expression, and EBNF constraints. + - XGrammar currently uses the [GGML BNF format](https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md). +- [Outlines](https://github.com/dottxt-ai/outlines): Supports JSON schema and regular expression constraints. + +If instead you want to initialize the Outlines backend, you can use `--grammar-backend outlines` flag: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ +--port 30000 --host 0.0.0.0 --grammar-backend [xgrammar|outlines] # xgrammar or outlines (default: xgrammar) +``` + +```python +import json +import requests + +json_schema = json.dumps({ + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], +}) + +# JSON (works with both Outlines and XGrammar) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, +) +print(response.json()) + +# Regular expression (Outlines backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Paris is the capital of", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "regex": "(France|England)", + }, + }, +) +print(response.json()) + +# EBNF (XGrammar backend only) +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "Write a greeting.", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "ebnf": 'root ::= "Hello" | "Hi" | "Hey"', + }, + }, +) +print(response.json()) +``` + +Detailed example in [structured outputs](../advanced_features/structured_outputs.ipynb). + +### Custom logit processor + +Launch a server with `--enable-custom-logit-processor` flag on. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3-8B-Instruct --port 30000 --enable-custom-logit-processor +``` + +Define a custom logit processor that will always sample a specific token id. + +```python +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor + +class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + # Check that the number of logits matches the number of custom parameters + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits +``` + +Send a request: + +```python +import requests + +response = requests.post( + "http://localhost:30000/generate", + json={ + "text": "The capital of France is", + "custom_logit_processor": DeterministicLogitProcessor().to_str(), + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 32, + "custom_params": {"token_id": 5}, + }, + }, +) +print(response.json()) +``` diff --git a/docs/basic_usage/send_request.ipynb b/docs/basic_usage/send_request.ipynb new file mode 100644 index 000000000..6e457a02b --- /dev/null +++ b/docs/basic_usage/send_request.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Sending Requests\n", + "This notebook provides a quick-start guide to use SGLang in chat completions after installation.\n", + "\n", + "- For Vision Language Models, see [OpenAI APIs - Vision](openai_api_vision.ipynb).\n", + "- For Embedding Models, see [OpenAI APIs - Embedding](openai_api_embeddings.ipynb) and [Encode (embedding model)](native_api.html#Encode-(embedding-model)).\n", + "- For Reward Models, see [Classify (reward model)](native_api.html#Classify-(reward-model))." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import wait_for_server, print_highlight, terminate_process\n", + "\n", + "# This is equivalent to running the following command in your terminal\n", + "# python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"\"\"\n", + "python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct \\\n", + " --host 0.0.0.0 --log-level warning\n", + "\"\"\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using cURL\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess, json\n", + "\n", + "curl_command = f\"\"\"\n", + "curl -s http://localhost:{port}/v1/chat/completions \\\n", + " -H \"Content-Type: application/json\" \\\n", + " -d '{{\"model\": \"qwen/qwen2.5-0.5b-instruct\", \"messages\": [{{\"role\": \"user\", \"content\": \"What is the capital of France?\"}}]}}'\n", + "\"\"\"\n", + "\n", + "response = json.loads(subprocess.check_output(curl_command, shell=True))\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Python Requests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "url = f\"http://localhost:{port}/v1/chat/completions\"\n", + "\n", + "data = {\n", + " \"model\": \"qwen/qwen2.5-0.5b-instruct\",\n", + " \"messages\": [{\"role\": \"user\", \"content\": \"What is the capital of France?\"}],\n", + "}\n", + "\n", + "response = requests.post(url, json=data)\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using OpenAI Python Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + ")\n", + "print_highlight(response)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "client = openai.Client(base_url=f\"http://127.0.0.1:{port}/v1\", api_key=\"None\")\n", + "\n", + "# Use stream=True for streaming responses\n", + "response = client.chat.completions.create(\n", + " model=\"qwen/qwen2.5-0.5b-instruct\",\n", + " messages=[\n", + " {\"role\": \"user\", \"content\": \"List 3 countries and their capitals.\"},\n", + " ],\n", + " temperature=0,\n", + " max_tokens=64,\n", + " stream=True,\n", + ")\n", + "\n", + "# Handle the streaming output\n", + "for chunk in response:\n", + " if chunk.choices[0].delta.content:\n", + " print(chunk.choices[0].delta.content, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Native Generation APIs\n", + "\n", + "You can also use the native `/generate` endpoint with requests, which provides more flexibility. An API reference is available at [Sampling Parameters](sampling_params.md)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " },\n", + ")\n", + "\n", + "print_highlight(response.json())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Streaming" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests, json\n", + "\n", + "response = requests.post(\n", + " f\"http://localhost:{port}/generate\",\n", + " json={\n", + " \"text\": \"The capital of France is\",\n", + " \"sampling_params\": {\n", + " \"temperature\": 0,\n", + " \"max_new_tokens\": 32,\n", + " },\n", + " \"stream\": True,\n", + " },\n", + " stream=True,\n", + ")\n", + "\n", + "prev = 0\n", + "for chunk in response.iter_lines(decode_unicode=False):\n", + " chunk = chunk.decode(\"utf-8\")\n", + " if chunk and chunk.startswith(\"data:\"):\n", + " if chunk == \"data: [DONE]\":\n", + " break\n", + " data = json.loads(chunk[5:].strip(\"\\n\"))\n", + " output = data[\"text\"]\n", + " print(output[prev:], end=\"\", flush=True)\n", + " prev = len(output)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..d6ca64d88 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,205 @@ +import os +import sys +from datetime import datetime + +sys.path.insert(0, os.path.abspath("../..")) + +version_file = "../python/sglang/version.py" +with open(version_file, "r") as f: + exec(compile(f.read(), version_file, "exec")) +__version__ = locals()["__version__"] + +project = "SGLang" +copyright = f"2023-{datetime.now().year}, SGLang" +author = "SGLang Team" + +version = __version__ +release = __version__ + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.autosectionlabel", + "sphinx.ext.intersphinx", + "sphinx_tabs.tabs", + "myst_parser", + "sphinx_copybutton", + "sphinxcontrib.mermaid", + "nbsphinx", + "sphinx.ext.mathjax", +] + +nbsphinx_allow_errors = True +nbsphinx_execute = "never" + +autosectionlabel_prefix_document = True +nbsphinx_allow_directives = True + + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", + "html_image", + "linkify", + "substitution", +] + +myst_heading_anchors = 3 + +nbsphinx_kernel_name = "python3" +nbsphinx_execute_arguments = [ + "--InlineBackend.figure_formats={'svg', 'pdf'}", + "--InlineBackend.rc={'figure.dpi': 96}", +] + + +nb_render_priority = { + "html": ( + "application/vnd.jupyter.widget-view+json", + "application/javascript", + "text/html", + "image/svg+xml", + "image/png", + "image/jpeg", + "text/markdown", + "text/latex", + "text/plain", + ) +} + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", + "html_image", + "linkify", + "substitution", +] + +myst_heading_anchors = 3 +myst_ref_domains = ["std", "py"] + +templates_path = ["_templates"] + +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +master_doc = "index" + +language = "en" + +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +pygments_style = "sphinx" + +html_theme = "sphinx_book_theme" +html_logo = "_static/image/logo.png" +html_favicon = "_static/image/logo.ico" +html_title = project +html_copy_source = True +html_last_updated_fmt = "" + +html_theme_options = { + "repository_url": "https://github.com/sgl-project/sgl-project.github.io", + "repository_branch": "main", + "show_navbar_depth": 3, + "max_navbar_depth": 4, + "collapse_navbar": True, + "use_edit_page_button": True, + "use_source_button": True, + "use_issues_button": True, + "use_repository_button": True, + "use_download_button": True, + "use_sidenotes": True, + "show_toc_level": 2, +} + +html_context = { + "display_github": True, + "github_user": "sgl-project", + "github_repo": "sgl-project.github.io", + "github_version": "main", + "conf_py_path": "/docs/", +} + +html_static_path = ["_static"] +html_css_files = ["css/custom_log.css"] + + +def setup(app): + app.add_css_file("css/custom_log.css") + + +myst_enable_extensions = [ + "dollarmath", + "amsmath", + "deflist", + "colon_fence", +] +myst_heading_anchors = 5 + +htmlhelp_basename = "sglangdoc" + +latex_elements = {} + +latex_documents = [ + (master_doc, "sglang.tex", "sglang Documentation", "SGLang Team", "manual"), +] + +man_pages = [(master_doc, "sglang", "sglang Documentation", [author], 1)] + +texinfo_documents = [ + ( + master_doc, + "sglang", + "sglang Documentation", + author, + "sglang", + "One line description of project.", + "Miscellaneous", + ), +] + +epub_title = project + +epub_exclude_files = ["search.html"] + +copybutton_prompt_text = r">>> |\.\.\. " +copybutton_prompt_is_regexp = True + +autodoc_preserve_defaults = True +navigation_with_keys = False + +autodoc_mock_imports = [ + "torch", + "transformers", + "triton", +] + +intersphinx_mapping = { + "python": ("https://docs.python.org/3.12", None), + "typing_extensions": ("https://typing-extensions.readthedocs.io/en/latest", None), + "pillow": ("https://pillow.readthedocs.io/en/stable", None), + "numpy": ("https://numpy.org/doc/stable", None), + "torch": ("https://pytorch.org/docs/stable", None), +} + +html_theme = "sphinx_book_theme" + + +nbsphinx_prolog = """ +.. raw:: html + + +""" diff --git a/docs/deploy.py b/docs/deploy.py new file mode 100644 index 000000000..75b7ea7f2 --- /dev/null +++ b/docs/deploy.py @@ -0,0 +1,22 @@ +# Deploy the documents + +import os +from datetime import datetime + + +def run_cmd(cmd): + print(cmd) + os.system(cmd) + + +run_cmd("cd $DOC_SITE_PATH; git pull") + +# (Optional) Remove old files +# run_cmd("rm -rf $ALPA_SITE_PATH/*") + +run_cmd("cp -r _build/html/* $DOC_SITE_PATH") + +cmd_message = f"Update {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}" +run_cmd( + f"cd $DOC_SITE_PATH; git add .; git commit -m '{cmd_message}'; git push origin main" +) diff --git a/docs/developer_guide/bench_serving.md b/docs/developer_guide/bench_serving.md new file mode 100644 index 000000000..82f7aa2af --- /dev/null +++ b/docs/developer_guide/bench_serving.md @@ -0,0 +1,334 @@ +## Bench Serving Guide + +This guide explains how to benchmark online serving throughput and latency using `python -m sglang.bench_serving`. It supports multiple inference backends via OpenAI-compatible and native endpoints, and produces both console metrics and optional JSONL outputs. + +### What it does + +- Generates synthetic or dataset-driven prompts and submits them to a target serving endpoint +- Measures throughput, time-to-first-token (TTFT), inter-token latency (ITL), per-request end-to-end latency, and more +- Supports streaming or non-streaming modes, rate control, and concurrency limits + +### Supported backends and endpoints + +- `sglang` / `sglang-native`: `POST /generate` +- `sglang-oai`, `vllm`, `lmdeploy`: `POST /v1/completions` +- `sglang-oai-chat`, `vllm-chat`, `lmdeploy-chat`: `POST /v1/chat/completions` +- `trt` (TensorRT-LLM): `POST /v2/models/ensemble/generate_stream` +- `gserver`: Custom server (Not Implemented yet in this script) +- `truss`: `POST /v1/models/model:predict` + +If `--base-url` is provided, requests are sent to it. Otherwise, `--host` and `--port` are used. When `--model` is not provided, the script will attempt to query `GET /v1/models` for an available model ID (OpenAI-compatible endpoints). + +### Prerequisites + +- Python 3.8+ +- Dependencies typically used by this script: `aiohttp`, `numpy`, `requests`, `tqdm`, `transformers`, and for some datasets `datasets`, `pillow`, `pybase64`. Install as needed. +- An inference server running and reachable via the endpoints above +- If your server requires authentication, set environment variable `OPENAI_API_KEY` (used as `Authorization: Bearer `) + +### Quick start + +Run a basic benchmark against an sglang server exposing `/generate`: + +```bash +python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct +``` + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --num-prompts 1000 \ + --model meta-llama/Llama-3.1-8B-Instruct +``` + +Or, using an OpenAI-compatible endpoint (completions): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm \ + --base-url http://127.0.0.1:8000 \ + --num-prompts 1000 \ + --model meta-llama/Llama-3.1-8B-Instruct +``` + +### Datasets + +Select with `--dataset-name`: + +- `sharegpt` (default): loads ShareGPT-style pairs; optionally restrict with `--sharegpt-context-len` and override outputs with `--sharegpt-output-len` +- `random`: random text lengths; sampled from ShareGPT token space +- `random-ids`: random token ids (can lead to gibberish) +- `random-image`: generates random images and wraps them in chat messages; supports custom resolutions via 'heightxwidth' format +- `generated-shared-prefix`: synthetic dataset with shared long system prompts and short questions +- `mmmu`: samples from MMMU (Math split) and includes images + +Common dataset flags: + +- `--num-prompts N`: number of requests +- `--random-input-len`, `--random-output-len`, `--random-range-ratio`: for random/random-ids/random-image +- `--random-image-num-images`, `--random-image-resolution`: for random-image dataset (supports presets 1080p/720p/360p or custom 'heightxwidth' format) +- `--apply-chat-template`: apply tokenizer chat template when constructing prompts +- `--dataset-path PATH`: file path for ShareGPT json; if blank and missing, it will be downloaded and cached + +Generated Shared Prefix flags (for `generated-shared-prefix`): + +- `--gsp-num-groups` +- `--gsp-prompts-per-group` +- `--gsp-system-prompt-len` +- `--gsp-question-len` +- `--gsp-output-len` + +Random Image dataset flags (for `random-image`): + +- `--random-image-num-images`: Number of images per request +- `--random-image-resolution`: Image resolution; supports presets (1080p, 720p, 360p) or custom 'heightxwidth' format (e.g., 1080x1920, 512x768) + +### Examples + +1. To benchmark random-image dataset with 3 images per request, 500 prompts, 512 input length, and 512 output length, you can run: + +```bash +python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-3B-Instruct --disable-radix-cache +``` + +```bash +python -m sglang.bench_serving \ + --backend sglang-oai-chat \ + --dataset-name random-image \ + --num-prompts 500 \ + --random-image-num-images 3 \ + --random-image-resolution 720p \ + --random-input-len 512 \ + --random-output-len 512 +``` + +2. To benchmark random dataset with 3000 prompts, 1024 input length, and 1024 output length, you can run: + +```bash +python -m sglang.launch_server --model-path Qwen/Qwen2.5-3B-Instruct +``` + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 3000 \ + --random-input 1024 \ + --random-output 1024 \ + --random-range-ratio 0.5 +``` + +### Choosing model and tokenizer + +- `--model` is required unless the backend exposes `GET /v1/models`, in which case the first model ID is auto-selected. +- `--tokenizer` defaults to `--model`. Both can be HF model IDs or local paths. +- For ModelScope workflows, setting `SGLANG_USE_MODELSCOPE=true` enables fetching via ModelScope (weights are skipped for speed). +- If your tokenizer lacks a chat template, the script warns because token counting can be less robust for gibberish outputs. + +### Rate, concurrency, and streaming + +- `--request-rate`: requests per second. `inf` sends all immediately (burst). Non-infinite rate uses a Poisson process for arrival times. +- `--max-concurrency`: caps concurrent in-flight requests regardless of arrival rate. +- `--disable-stream`: switch to non-streaming mode when supported; TTFT then equals total latency for chat completions. + +### Other key options + +- `--output-file FILE.jsonl`: append JSONL results to file; auto-named if unspecified +- `--output-details`: include per-request arrays (generated texts, errors, ttfts, itls, input/output lens) +- `--extra-request-body '{"top_p":0.9,"temperature":0.6}'`: merged into payload (sampling params, etc.) +- `--disable-ignore-eos`: pass through EOS behavior (varies by backend) +- `--warmup-requests N`: run warmup requests with short output first (default 1) +- `--flush-cache`: call `/flush_cache` (sglang) before main run +- `--profile`: call `/start_profile` and `/stop_profile` (requires server to enable profiling, e.g., `SGLANG_TORCH_PROFILER_DIR`) +- `--lora-name name1 name2 ...`: randomly pick one per request and pass to backend (e.g., `lora_path` for sglang) +- `--tokenize-prompt`: send integer IDs instead of text (currently supports `--backend sglang` only) + +### Authentication + +If your target endpoint requires OpenAI-style auth, set: + +```bash +export OPENAI_API_KEY=sk-...yourkey... +``` + +The script will add `Authorization: Bearer $OPENAI_API_KEY` automatically for OpenAI-compatible routes. + +### Metrics explained + +Printed after each run: + +- Request throughput (req/s) +- Input token throughput (tok/s) +- Output token throughput (tok/s) +- Total token throughput (tok/s) +- Concurrency: aggregate time of all requests divided by wall time +- End-to-End Latency (ms): mean/median/std/p99 per-request total latency +- Time to First Token (TTFT, ms): mean/median/std/p99 for streaming mode +- Inter-Token Latency (ITL, ms): mean/median/std/p95/p99/max between tokens +- TPOT (ms): Token processing time after first token, i.e., `(latency - ttft)/(tokens-1)` +- Accept length (sglang-only, if available): speculative decoding accept length + +The script also retokenizes generated text with the configured tokenizer and reports "retokenized" counts. + +### JSONL output format + +When `--output-file` is set, one JSON object is appended per run. Base fields: + +- Arguments summary: backend, dataset, request_rate, max_concurrency, etc. +- Duration and totals: completed, total_input_tokens, total_output_tokens, retokenized totals +- Throughputs and latency statistics as printed in the console +- `accept_length` when available (sglang) + +With `--output-details`, an extended object also includes arrays: + +- `input_lens`, `output_lens` +- `ttfts`, `itls` (per request: ITL arrays) +- `generated_texts`, `errors` + +### End-to-end examples + +1) sglang native `/generate` (streaming): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --random-input-len 1024 --random-output-len 1024 --random-range-ratio 0.5 \ + --num-prompts 2000 \ + --request-rate 100 \ + --max-concurrency 512 \ + --output-file sglang_random.jsonl --output-details +``` + +2) OpenAI-compatible Completions (e.g., vLLM): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm \ + --base-url http://127.0.0.1:8000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name sharegpt \ + --num-prompts 1000 \ + --sharegpt-output-len 256 +``` + +3) OpenAI-compatible Chat Completions (streaming): + +```bash +python3 -m sglang.bench_serving \ + --backend vllm-chat \ + --base-url http://127.0.0.1:8000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --num-prompts 500 \ + --apply-chat-template +``` + +4) Random images (VLM) with chat template: + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model your-vlm-model \ + --dataset-name random-image \ + --random-image-num-images 2 \ + --random-image-resolution 720p \ + --random-input-len 128 --random-output-len 256 \ + --num-prompts 200 \ + --apply-chat-template +``` + +4a) Random images with custom resolution: + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model your-vlm-model \ + --dataset-name random-image \ + --random-image-num-images 1 \ + --random-image-resolution 512x768 \ + --random-input-len 64 --random-output-len 128 \ + --num-prompts 100 \ + --apply-chat-template +``` + +5) Generated shared prefix (long system prompts + short questions): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name generated-shared-prefix \ + --gsp-num-groups 64 --gsp-prompts-per-group 16 \ + --gsp-system-prompt-len 2048 --gsp-question-len 128 --gsp-output-len 256 \ + --num-prompts 1024 +``` + +6) Tokenized prompts (ids) for strict length control (sglang only): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --dataset-name random \ + --tokenize-prompt \ + --random-input-len 2048 --random-output-len 256 --random-range-ratio 0.2 +``` + +7) Profiling and cache flush (sglang): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --profile \ + --flush-cache +``` + +8) TensorRT-LLM streaming endpoint: + +```bash +python3 -m sglang.bench_serving \ + --backend trt \ + --base-url http://127.0.0.1:8000 \ + --model your-trt-llm-model \ + --dataset-name random \ + --num-prompts 100 \ + --disable-ignore-eos +``` + +9) Evaluating large-scale KVCache sharing with mooncake trace (sglang only): + +```bash +python3 -m sglang.bench_serving \ + --backend sglang \ + --host 127.0.0.1 --port 30000 \ + --model mode-name \ + --dataset-name mooncake \ + --mooncake-slowdown-factor 1.0 \ + --mooncake-num-rounds 1000 \ + --mooncake-workload conversation|mooncake|agent|synthetic + --use-trace-timestamps true \ + --random-output-len 256 +``` + +### Troubleshooting + +- All requests failed: verify `--backend`, server URL/port, `--model`, and authentication. Check warmup errors printed by the script. +- Throughput seems too low: adjust `--request-rate` and `--max-concurrency`; verify server batch size/scheduling; ensure streaming is enabled if appropriate. +- Token counts look odd: prefer chat/instruct models with proper chat templates; otherwise tokenization of gibberish may be inconsistent. +- Random-image/MMMU datasets: ensure you installed extra deps (`pillow`, `datasets`, `pybase64`). +- Authentication errors (401/403): set `OPENAI_API_KEY` or disable auth on your server. + +### Notes + +- The script raises the file descriptor soft limit (`RLIMIT_NOFILE`) to help with many concurrent connections. +- For sglang, `/get_server_info` is queried post-run to report speculative decoding accept length when available. diff --git a/docs/developer_guide/benchmark_and_profiling.md b/docs/developer_guide/benchmark_and_profiling.md new file mode 100644 index 000000000..948c837ff --- /dev/null +++ b/docs/developer_guide/benchmark_and_profiling.md @@ -0,0 +1,182 @@ +# Benchmark and Profiling + +## Benchmark + +- Benchmark the latency of running a single static batch without a server. The arguments are the same as for `launch_server.py`. + Note that this is a simplified test script without a dynamic batching server, so it may run out of memory for a batch size that a real server can handle. A real server truncates the prefill into several batches, while this simplified script does not. + - Without a server (do not need to launch a server) + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 + ``` + - With a server (please use `sglang.launch_server` to launch a server first and run the following command.) + ```bash + python -m sglang.bench_one_batch_server --base-url http://127.0.0.1:30000 --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch-size 32 --input-len 256 --output-len 32 + ``` + + +- Benchmark offline processing. This script will start an offline engine and run the benchmark. + + ```bash + python3 -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + ``` + +- Benchmark online serving. Please use `sglang.launch_server` to launch a server first and run the following command. + + ```bash + python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + ``` + +## Profile with PyTorch Profiler + +[Pytorch Profiler](https://pytorch.org/tutorials/recipes/recipes/profiler_recipe.html) is a convenient basic tool to inspect kernel execution time, call stack, and kernel overlap and occupancy. + +### Profile a server with `sglang.bench_serving` + +```bash +# set trace path +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + +# start server +python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct + +# send profiling request from client +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 10 --sharegpt-output-len 100 --profile +``` + +Please make sure that the `SGLANG_TORCH_PROFILER_DIR` should be set at both server and client side, otherwise the trace file cannot be generated correctly . A secure way will be setting `SGLANG_TORCH_PROFILER_DIR` in the `.*rc` file of shell (e.g. `~/.bashrc` for bash shells). + +For more details, please refer to [Bench Serving Guide](./bench_serving.md). + +### Profile a server with `sglang.bench_offline_throughput` +```bash +export SGLANG_TORCH_PROFILER_DIR=/root/sglang/profile_log + +# profile one batch with bench_one_batch.py +# batch size can be controlled with --batch argument +python3 -m sglang.bench_one_batch --model-path meta-llama/Llama-3.1-8B-Instruct --batch 32 --input-len 1024 --output-len 10 --profile + +# profile multiple batches with bench_offline_throughput.py +python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 +``` + +### Profile a server with `sglang.profiler` + +When the server is running (e.g., processing a decoding request), you can start live profiling immediately by sending a profile request to the server. + +You can do this by running `python3 -m sglang.profiler`. For example: + +``` +# Terminal 1: Send a generation request +python3 -m sglang.test.send_one + +# Terminal 2: Before the above request finishes, quickly launch the following command in a separate terminal. +# It will generate a profile of the above request for several decoding batches. +python3 -m sglang.profiler +``` + +### Possible PyTorch bugs +If in any cases you encounter the following error (for example, using qwen 2.5 VL): +```bash +RuntimeError: !stack.empty() INTERNAL ASSERT FAILED at "/pytorch/torch/csrc/autograd/profiler_python.cpp":983, please report a bug to PyTorch. Python replay stack is empty. +``` +This is likely a PyTorch Bug reported in [Bug: vLLM Profiler](https://github.com/vllm-project/vllm/issues/18240) and [Bug: torch.profiler.profile](https://github.com/pytorch/pytorch/issues/101632). As a workaround, you may disable `with_stack` with an environment variable such as follows: +```bash +export SGLANG_PROFILE_WITH_STACK=False +python -m sglang.bench_offline_throughput --model-path meta-llama/Llama-3.1-8B-Instruct --dataset-name random --num-prompts 10 --profile --mem-frac=0.8 +``` + +### View traces + +Trace files can be loaded and visualized from: + +1. https://ui.perfetto.dev/ (any browser) +2. chrome://tracing (Chrome browser only) + +If browser cannot open trace file due to its large size, +client can generate a small trace file (<100MB) by controlling number of prompts and lengths of prompt outputs. +For example, when profiling a server, + +```bash +python -m sglang.bench_serving --backend sglang --model meta-llama/Llama-3.1-8B-Instruct --num-prompts 2 --sharegpt-output-len 100 --profile +``` + +This command sets the number of prompts to 2 with `--num-prompts` argument and limits the length of output sequences to 100 with `--sharegpt-output-len` argument, which can generate a small trace file for browser to open smoothly. + +Additionally, if you want to locate the SGLang Python source code through the cuda kernel in Trace, you need to disable CUDA Graph when starting the service. This can be done by using the `--disable-cuda-graph` parameter in the command to start the service. + +## Profile with Nsight + +[Nsight systems](https://docs.nvidia.com/nsight-systems/) is an advanced tool that exposes more profiling details, such as register and shared memory usage, annotated code regions and low-level CUDA APIs and events. + +1. Prerequisite: + + Install using apt, or run inside a [NVIDIA Docker container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags) or [SGLang Docker container](https://github.com/sgl-project/sglang/tree/main/docker). + + ```bash + # install nsys + # https://docs.nvidia.com/nsight-systems/InstallationGuide/index.html + apt update + apt install -y --no-install-recommends gnupg + echo "deb http://developer.download.nvidia.com/devtools/repos/ubuntu$(source /etc/lsb-release; echo "$DISTRIB_RELEASE" | tr -d .)/$(dpkg --print-architecture) /" | tee /etc/apt/sources.list.d/nvidia-devtools.list + apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub + apt update + apt install nsight-systems-cli + ``` + +2. To profile a single batch, use + + ```bash + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node python3 -m sglang.bench_one_batch --model meta-llama/Meta-Llama-3-8B --batch-size 64 --input-len 512 + ``` + +3. To profile a server, e.g. + + ```bash + # launch the server, set the delay and duration times according to needs + # after the duration time has been used up, server will be killed by nsys + + nsys profile --trace-fork-before-exec=true --cuda-graph-trace=node -o sglang.out --delay 60 --duration 70 python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + + # client + python3 -m sglang.bench_serving --backend sglang --num-prompts 1000 --dataset-name random --random-input 1024 --random-output 512 + ``` + + In practice, we recommend users to set `--duration` argument to a large value. Whenever user wants the server to stop profiling. Firstly run: + + ```bash + nsys sessions list + ``` + + to get the session id in the form of `profile-XXXXX`, then run: + + ```bash + nsys stop --session=profile-XXXXX + ``` + + to manually kill the profiler and generate `nsys-rep` files instantly. + +4. Use NVTX to annotate code regions, e.g. to see their execution time. + + ```bash + # install nvtx + pip install nvtx + ``` + + ```python + # code snippets + import nvtx + with nvtx.annotate("description", color="color"): + # some critical code + ``` + +## Other tips + +1. You can benchmark a model using dummy weights by only providing the config.json file. This allows for quick testing of model variants without training. To do so, add `--load-format dummy` to the above commands and then you only need a correct `config.json` under the checkpoint folder. +2. You can benchmark a model with modified configs (e.g., less layers) by using `--json-model-override-args`. For example, you can benchmark a model with only 2 layers and 2 kv heads using: + + ```bash + python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --batch 32 --input-len 256 --output-len 32 --load-format dummy --json-model-override-args '{"num_hidden_layers": 1, "num_key_value_heads": 1}' + ``` + +3. You can use `--python-backtrace=cuda` to see python call stack for all CUDA kernels, as in PyTorch Profiler. (Caveat: this can cause inaccurately long kernel runtimes for CUDA event based timing) +4. For more arguments see [Nsight Systems User Guide](https://docs.nvidia.com/nsight-systems/UserGuide/index.html). diff --git a/docs/developer_guide/contribution_guide.md b/docs/developer_guide/contribution_guide.md new file mode 100644 index 000000000..f8e6f692d --- /dev/null +++ b/docs/developer_guide/contribution_guide.md @@ -0,0 +1,103 @@ +# Contribution Guide + +Welcome to **SGLang**! We appreciate your interest in contributing. This guide provides a concise overview of how to set up your environment, run tests, build documentation, and open a Pull Request (PR). Whether you’re fixing a small bug or developing a major feature, we encourage following these steps for a smooth contribution process. + +## Install SGLang from Source + +### Fork and clone the repository + +**Note**: New contributors do **not** have the write permission to push to the official SGLang repo. Please fork the repository under your GitHub account, then clone your fork locally. + +```bash +git clone https://github.com//sglang.git +``` + +### Build from source + +Refer to [Install SGLang from Source](../get_started/install.md#method-2-from-source). + +## Format code with pre-commit + +We use [pre-commit](https://pre-commit.com/) to maintain consistent code style checks. Before pushing your changes, please run: + +```bash +pip3 install pre-commit +pre-commit install +pre-commit run --all-files +``` + +- **`pre-commit run --all-files`** manually runs all configured checks, applying fixes if possible. If it fails the first time, re-run it to ensure lint errors are fully resolved. Make sure your code passes all checks **before** creating a Pull Request. +- **Do not commit** directly to the `main` branch. Always create a new branch (e.g., `feature/my-new-feature`), push your changes, and open a PR from that branch. + +## Run and add unit tests + +If you add a new feature or fix a bug, please add corresponding unit tests to ensure coverage and prevent regression. +SGLang uses Python's built-in [unittest](https://docs.python.org/3/library/unittest.html) framework. +For detailed instructions on running tests and integrating them into CI, refer to [test/README.md](https://github.com/sgl-project/sglang/tree/main/test/README.md). + +## Write documentations + +We recommend new contributors start from writing documentation, which helps you quickly understand SGLang codebase. +For more details, please refer to [docs/README.md](https://github.com/sgl-project/sglang/tree/main/docs/README.md). + +## Test the accuracy +If your code changes the model output, please run the accuracy tests. A quick sanity check is the few-shot GSM8K. + +``` +# Launch a server +python3 -m sglang.launch_server --model Qwen/Qwen2-7B-Instruct + +# Evaluate +python3 -m sglang.test.few_shot_gsm8k --num-questions 200 +``` + +Please note that the above script is primarily a sanity check, not a rigorous accuracy or speed test. +This test can have significant variance (1%–5%) in accuracy due to batching and the non-deterministic nature of the inference engine. +Also, do not rely on the "Latency/Output throughput" from this script, as it is not a proper speed test. + +GSM8K is too easy for state-of-the-art models nowadays. Please try your own more challenging accuracy tests. +You can find additional accuracy eval examples in: +- [test_eval_accuracy_large.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_eval_accuracy_large.py) +- [test_gpt_oss_1gpu.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_gpt_oss_1gpu.py) + +## Benchmark the speed +Refer to [Benchmark and Profiling](../developer_guide/benchmark_and_profiling.md). + +## Request a review +You can identify potential reviewers for your code by checking the [code owners](https://github.com/sgl-project/sglang/blob/main/.github/CODEOWNERS) and [reviewers](https://github.com/sgl-project/sglang/blob/main/.github/REVIEWERS.md) files. +Another effective strategy is to review the file modification history and contact individuals who have frequently edited the files. +If you modify files protected by code owners, their approval is required to merge the code. + +## General code style +- Avoid code duplication. If the same code snippet (more than five lines) appears multiple times, extract it into a shared function. +- Minimize device synchronization. Reduce expensive CPU-GPU synchronization operations, such as `tensor.item()` or `tensor.cpu()`, whenever possible. Use vectorized code. +- Keep files concise. If a file exceeds 2,000 lines of code, split it into multiple smaller files. +- Prioritize extreme efficiency. SGLang is a runtime, and most of your code runs on the critical path for every request. Optimize all minor overheads as much as possible, especially in the model forward code. + - A common pattern is some runtime checks in the model forward pass (e.g., [this](https://github.com/sgl-project/sglang/blob/f1b0eda55c2c4838e8ab90a0fac7fb1e3d7064ab/python/sglang/srt/models/deepseek_v2.py#L486-L491)). These are very likely the same for every layer. Please cache the result as a single boolean value whenever possible. +- Strive to make functions as pure as possible. Avoid in-place modification of arguments. +- When supporting new hardware or features, follow these guidelines: + - Do not drastically change existing code. + - Always prefer new files to introduce specific components for your new hardware (e.g., `allocator_ascend.py`). + - If you write multiple if/else blocks for new features, ensure the common path (e.g., NVIDIA hardware or the existing code path) is the first branch. + +## How to update sgl-kernel +Since sglang and sgl-kernel are separate Python packages, our current GitHub CI infrastructure does not support updating a kernel and using it immediately within the same pull request (PR). +To add a new kernel or modify an existing one in the sgl-kernel package, you must use multiple PRs. + +Follow these steps: + +1. Submit a PR to update the sgl-kernel source code without using it in sglang python package (e.g., [#8884](https://github.com/sgl-project/sglang/pull/8884/files)). +2. Bump the version of sgl-kernel (e.g., [#9220](https://github.com/sgl-project/sglang/pull/9220/files)). + - Once merged, this will trigger an automatic release of the sgl-kernel wheel to PyPI. + - If not urgent, you can wait for other people to release the wheel. A new version will typically be released within one week. +3. Apply the changes: + - Update the sgl-kernel version in `sglang/python/pyproject.toml` to use the modified kernels. + - Update the related caller code in the sglang to use the new kernel. + +## Tips for newcomers + +If you want to contribute but don’t have a specific idea in mind, pick issues labeled [“good first issue” or “help wanted”](https://github.com/sgl-project/sglang/issues?q=is%3Aissue+label%3A%22good+first+issue%22%2C%22help+wanted%22). These tasks typically have lower complexity and provide an excellent introduction to the codebase. Also check out this [code walk-through](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/tree/main/sglang/code-walk-through) for a deeper look into SGLang’s workflow. + +If you have any questions or want to start a discussion, please feel free to ask in our [Slack channel](https://slack.sglang.ai). + +Thank you for your interest in SGLang. Happy coding! diff --git a/docs/developer_guide/development_guide_using_docker.md b/docs/developer_guide/development_guide_using_docker.md new file mode 100644 index 000000000..e38947902 --- /dev/null +++ b/docs/developer_guide/development_guide_using_docker.md @@ -0,0 +1,108 @@ +# Development Guide Using Docker + +## Setup VSCode on a Remote Host +(Optional - you can skip this step if you plan to run sglang dev container locally) + +1. In the remote host, download `code` from [Https://code.visualstudio.com/docs/?dv=linux64cli](https://code.visualstudio.com/download) and run `code tunnel` in a shell. + +Example +```bash +wget https://vscode.download.prss.microsoft.com/dbazure/download/stable/fabdb6a30b49f79a7aba0f2ad9df9b399473380f/vscode_cli_alpine_x64_cli.tar.gz +tar xf vscode_cli_alpine_x64_cli.tar.gz + +# https://code.visualstudio.com/docs/remote/tunnels +./code tunnel +``` + +2. In your local machine, press F1 in VSCode and choose "Remote Tunnels: Connect to Tunnel". + +## Setup Docker Container + +### Option 1. Use the default dev container automatically from VSCode +There is a `.devcontainer` folder in the sglang repository root folder to allow VSCode to automatically start up within dev container. You can read more about this VSCode extension in VSCode official document [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers). +![image](https://github.com/user-attachments/assets/6a245da8-2d4d-4ea8-8db1-5a05b3a66f6d) +(*Figure 1: Diagram from VSCode official documentation [Developing inside a Container](https://code.visualstudio.com/docs/devcontainers/containers).*) + +To enable this, you only need to: +1. Start Visual Studio Code and install [VSCode dev container extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). +2. Press F1, type and choose "Dev Container: Open Folder in Container. +3. Input the `sglang` local repo path in your machine and press enter. + +The first time you open it in dev container might take longer due to docker pull and build. Once it's successful, you should set on your status bar at the bottom left displaying that you are in a dev container: + +![image](https://github.com/user-attachments/assets/650bba0b-c023-455f-91f9-ab357340106b) + +Now when you run `sglang.launch_server` in the VSCode terminal or start debugging using F5, sglang server will be started in the dev container with all your local changes applied automatically: + +![image](https://github.com/user-attachments/assets/748c85ba-7f8c-465e-8599-2bf7a8dde895) + + +### Option 2. Start up containers manually (advanced) + +The following startup command is an example for internal development by the SGLang team. You can **modify or add directory mappings as needed**, especially for model weight downloads, to prevent repeated downloads by different Docker containers. + +❗️ **Note on RDMA** + + 1. `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them but keeping them there does not harm. Thus, we enable these two flags by default in the commands below. + 2. You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +```bash +# Change the name to yours +docker run -itd --shm-size 32g --gpus all -v --ipc=host --network=host --privileged --name sglang_dev lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_dev /bin/zsh +``` +Some useful volumes to mount are: +1. **Huggingface model cache**: mounting model cache can avoid re-download every time docker restarts. Default location on Linux is `~/.cache/huggingface/`. +2. **SGLang repository**: code changes in the SGLang local repository will be automatically synced to the .devcontainer. + +Example 1: Monting local cache folder `/opt/dlami/nvme/.cache` but not the SGLang repo. Use this when you prefer to manually transfer local code changes to the devcontainer. +```bash +docker run -itd --shm-size 32g --gpus all -v /opt/dlami/nvme/.cache:/root/.cache --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` +Example 2: Mounting both HuggingFace cache and local SGLang repo. Local code changes are automatically synced to the devcontainer as the SGLang is installed in editable mode in the dev image. +```bash +docker run -itd --shm-size 32g --gpus all -v $HOME/.cache/huggingface/:/root/.cache/huggingface -v $HOME/src/sglang:/sgl-workspace/sglang --ipc=host --network=host --privileged --name sglang_zhyncs lmsysorg/sglang:dev /bin/zsh +docker exec -it sglang_zhyncs /bin/zsh +``` +## Debug SGLang with VSCode Debugger +1. (Create if not exist) open `launch.json` in VSCode. +2. Add the following config and save. Please note that you can edit the script as needed to apply different parameters or debug a different program (e.g. benchmark script). + ```JSON + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python Debugger: launch_server", + "type": "debugpy", + "request": "launch", + "module": "sglang.launch_server", + "console": "integratedTerminal", + "args": [ + "--model-path", "meta-llama/Llama-3.2-1B", + "--host", "0.0.0.0", + "--port", "30000", + "--trust-remote-code", + ], + "justMyCode": false + } + ] + } + ``` + +3. Press "F5" to start. VSCode debugger will ensure that the program will pause at the breakpoints even if the program is running at remote SSH/Tunnel host + dev container. + +## Profile + +```bash +# Change batch size, input, output and add `disable-cuda-graph` (for easier analysis) +# e.g. DeepSeek V3 +nsys profile -o deepseek_v3 python3 -m sglang.bench_one_batch --batch-size 1 --input 128 --output 256 --model deepseek-ai/DeepSeek-V3 --trust-remote-code --tp 8 --disable-cuda-graph +``` + +## Evaluation + +```bash +# e.g. gsm8k 8 shot +python3 benchmark/gsm8k/bench_sglang.py --num-questions 2000 --parallel 2000 --num-shots 8 +``` diff --git a/docs/developer_guide/release_process.md b/docs/developer_guide/release_process.md new file mode 100644 index 000000000..e817a256e --- /dev/null +++ b/docs/developer_guide/release_process.md @@ -0,0 +1,18 @@ +# PyPI Package Release Process + +## Update the version in code +Update the package version in `python/pyproject.toml` and `python/sglang/__init__.py`. + +## Upload the PyPI package + +``` +pip install build twine +``` + +``` +cd python +bash upload_pypi.sh +``` + +## Make a release in GitHub +Make a new release https://github.com/sgl-project/sglang/releases/new. diff --git a/docs/developer_guide/setup_github_runner.md b/docs/developer_guide/setup_github_runner.md new file mode 100644 index 000000000..6ed78a247 --- /dev/null +++ b/docs/developer_guide/setup_github_runner.md @@ -0,0 +1,49 @@ +# Set Up Self-Hosted Runners for GitHub Action + +## Add a Runner + +### Step 1: Start a docker container. + +You can mount a folder for the shared huggingface model weights cache. The command below uses `/tmp/huggingface` as an example. + +``` +docker pull nvidia/cuda:12.1.1-devel-ubuntu22.04 +# Nvidia +docker run --shm-size 128g -it -v /tmp/huggingface:/hf_home --gpus all nvidia/cuda:12.1.1-devel-ubuntu22.04 /bin/bash +# AMD +docker run --rm --device=/dev/kfd --device=/dev/dri --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +# AMD just the last 2 GPUs +docker run --rm --device=/dev/kfd --device=/dev/dri/renderD176 --device=/dev/dri/renderD184 --group-add video --shm-size 128g -it -v /tmp/huggingface:/hf_home lmsysorg/sglang:v0.5.0rc1-rocm630 /bin/bash +``` + +### Step 2: Configure the runner by `config.sh` + +Run these commands inside the container. + +``` +apt update && apt install -y curl python3-pip git +export RUNNER_ALLOW_RUNASROOT=1 +``` + +Then follow https://github.com/sgl-project/sglang/settings/actions/runners/new?arch=x64&os=linux to run `config.sh` + +**Notes** +- Do not need to specify the runner group +- Give it a name (e.g., `test-sgl-gpu-0`) and some labels (e.g., `1-gpu-runner`). The labels can be edited later in Github Settings. +- Do not need to change the work folder. + +### Step 3: Run the runner by `run.sh` + +- Set up environment variables +``` +export HF_HOME=/hf_home +export SGLANG_IS_IN_CI=true +export HF_TOKEN=hf_xxx +export OPENAI_API_KEY=sk-xxx +export CUDA_VISIBLE_DEVICES=0 +``` + +- Run it forever +``` +while true; do ./run.sh; echo "Restarting..."; sleep 2; done +``` diff --git a/docs/get_started/install.md b/docs/get_started/install.md new file mode 100644 index 000000000..e2e780e00 --- /dev/null +++ b/docs/get_started/install.md @@ -0,0 +1,131 @@ +# Install SGLang + +You can install SGLang using one of the methods below. + +This page primarily applies to common NVIDIA GPU platforms. +For other or newer platforms, please refer to the dedicated pages for [NVIDIA Blackwell GPUs](../platforms/blackwell_gpu.md), [AMD GPUs](../platforms/amd_gpu.md), [Intel Xeon CPUs](../platforms/cpu_server.md), [NVIDIA Jetson](../platforms/nvidia_jetson.md), [Ascend NPUs](../platforms/ascend_npu.md). + +## Method 1: With pip or uv + +It is recommended to use uv for faster installation: + +```bash +pip install --upgrade pip +pip install uv +uv pip install "sglang[all]>=0.5.2" +``` + +**Quick fixes to common problems** +- If you encounter `OSError: CUDA_HOME environment variable is not set`. Please set it to your CUDA install root with either of the following solutions: + 1. Use `export CUDA_HOME=/usr/local/cuda-` to set the `CUDA_HOME` environment variable. + 2. Install FlashInfer first following [FlashInfer installation doc](https://docs.flashinfer.ai/installation.html), then install SGLang as described above. + +## Method 2: From source + +```bash +# Use the last release branch +git clone -b v0.5.2 https://github.com/sgl-project/sglang.git +cd sglang + +# Install the python packages +pip install --upgrade pip +pip install -e "python[all]" +``` + +**Quick fixes to common problems** +- If you want to develop SGLang, it is recommended to use docker. Please refer to [setup docker container](../developer_guide/development_guide_using_docker.md#setup-docker-container). The docker image is `lmsysorg/sglang:dev`. + +## Method 3: Using docker + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile](https://github.com/sgl-project/sglang/tree/main/docker). +Replace `` below with your huggingface hub [token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +docker run --gpus all \ + --shm-size 32g \ + -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --host 0.0.0.0 --port 30000 +``` + +## Method 4: Using Kubernetes + +Please check out [OME](https://github.com/sgl-project/ome), a Kubernetes operator for enterprise-grade management and serving of large language models (LLMs). + +
+More + +1. Option 1: For single node serving (typically when the model size fits into GPUs on one node) + + Execute command `kubectl apply -f docker/k8s-sglang-service.yaml`, to create k8s deployment and service, with llama-31-8b as example. + +2. Option 2: For multi-node serving (usually when a large model requires more than one GPU node, such as `DeepSeek-R1`) + + Modify the LLM model path and arguments as necessary, then execute command `kubectl apply -f docker/k8s-sglang-distributed-sts.yaml`, to create two nodes k8s statefulset and serving service. + +
+ +## Method 5: Using docker compose + +
+More + +> This method is recommended if you plan to serve it as a service. +> A better approach is to use the [k8s-sglang-service.yaml](https://github.com/sgl-project/sglang/blob/main/docker/k8s-sglang-service.yaml). + +1. Copy the [compose.yml](https://github.com/sgl-project/sglang/blob/main/docker/compose.yaml) to your local machine +2. Execute the command `docker compose up -d` in your terminal. +
+ +## Method 6: Run on Kubernetes or Clouds with SkyPilot + +
+More + +To deploy on Kubernetes or 12+ clouds, you can use [SkyPilot](https://github.com/skypilot-org/skypilot). + +1. Install SkyPilot and set up Kubernetes cluster or cloud access: see [SkyPilot's documentation](https://skypilot.readthedocs.io/en/latest/getting-started/installation.html). +2. Deploy on your own infra with a single command and get the HTTP API endpoint: +
+SkyPilot YAML: sglang.yaml + +```yaml +# sglang.yaml +envs: + HF_TOKEN: null + +resources: + image_id: docker:lmsysorg/sglang:latest + accelerators: A100 + ports: 30000 + +run: | + conda deactivate + python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.1-8B-Instruct \ + --host 0.0.0.0 \ + --port 30000 +``` + +
+ +```bash +# Deploy on any cloud or Kubernetes cluster. Use --cloud to select a specific cloud provider. +HF_TOKEN= sky launch -c sglang --env HF_TOKEN sglang.yaml + +# Get the HTTP API endpoint +sky status --endpoint 30000 sglang +``` + +3. To further scale up your deployment with autoscaling and failure recovery, check out the [SkyServe + SGLang guide](https://github.com/skypilot-org/skypilot/tree/master/llm/sglang#serving-llama-2-with-sglang-for-more-traffic-using-skyserve). +
+ +## Common Notes + +- [FlashInfer](https://github.com/flashinfer-ai/flashinfer) is the default attention kernel backend. It only supports sm75 and above. If you encounter any FlashInfer-related issues on sm75+ devices (e.g., T4, A10, A100, L4, L40S, H100), please switch to other kernels by adding `--attention-backend triton --sampling-backend pytorch` and open an issue on GitHub. +- To reinstall flashinfer locally, use the following command: `pip3 install --upgrade flashinfer-python --force-reinstall --no-deps` and then delete the cache with `rm -rf ~/.cache/flashinfer`. +- If you only need to use OpenAI API models with the frontend language, you can avoid installing other dependencies by using `pip install "sglang[openai]"`. +- The language frontend operates independently of the backend runtime. You can install the frontend locally without needing a GPU, while the backend can be set up on a GPU-enabled machine. To install the frontend, run `pip install sglang`, and for the backend, use `pip install sglang[srt]`. `srt` is the abbreviation of SGLang runtime. diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..f948fca24 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,95 @@ +SGLang Documentation +==================== + +SGLang is a fast serving framework for large language models and vision language models. +It makes your interaction with models faster and more controllable by co-designing the backend runtime and frontend language. +The core features include: + +- **Fast Backend Runtime**: Provides efficient serving with RadixAttention for prefix caching, zero-overhead CPU scheduler, prefill-decode disaggregation, speculative decoding, continuous batching, paged attention, tensor/pipeline/expert/data parallelism, structured outputs, chunked prefill, quantization (FP4/FP8/INT4/AWQ/GPTQ), and multi-lora batching. +- **Flexible Frontend Language**: Offers an intuitive interface for programming LLM applications, including chained generation calls, advanced prompting, control flow, multi-modal inputs, parallelism, and external interactions. +- **Extensive Model Support**: Supports a wide range of generative models (Llama, Qwen, DeepSeek, Kimi, GPT, Gemma, Mistral, etc.), embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models. +- **Active Community**: SGLang is open-source and backed by an active community with wide industry adoption. + +.. toctree:: + :maxdepth: 1 + :caption: Get Started + + get_started/install.md + +.. toctree:: + :maxdepth: 1 + :caption: Basic Usage + + basic_usage/send_request.ipynb + basic_usage/openai_api.rst + basic_usage/offline_engine_api.ipynb + basic_usage/native_api.ipynb + basic_usage/sampling_params.md + basic_usage/deepseek.md + basic_usage/gpt_oss.md + basic_usage/llama4.md + basic_usage/qwen3.md + +.. toctree:: + :maxdepth: 1 + :caption: Advanced Features + + advanced_features/server_arguments.md + advanced_features/hyperparameter_tuning.md + advanced_features/speculative_decoding.ipynb + advanced_features/structured_outputs.ipynb + advanced_features/structured_outputs_for_reasoning_models.ipynb + advanced_features/tool_parser.ipynb + advanced_features/separate_reasoning.ipynb + advanced_features/quantization.md + advanced_features/lora.ipynb + advanced_features/pd_disaggregation.md + advanced_features/vlm_query.ipynb + advanced_features/router.md + advanced_features/observability.md + advanced_features/attention_backend.md + +.. toctree:: + :maxdepth: 1 + :caption: Supported Models + + supported_models/generative_models.md + supported_models/multimodal_language_models.md + supported_models/embedding_models.md + supported_models/reward_models.md + supported_models/rerank_models.md + supported_models/support_new_models.md + supported_models/transformers_fallback.md + supported_models/modelscope.md + +.. toctree:: + :maxdepth: 1 + :caption: Hardware Platforms + + platforms/amd_gpu.md + platforms/blackwell_gpu.md + platforms/cpu_server.md + platforms/tpu.md + platforms/nvidia_jetson.md + platforms/ascend_npu.md + +.. toctree:: + :maxdepth: 1 + :caption: Developer Guide + + developer_guide/contribution_guide.md + developer_guide/development_guide_using_docker.md + developer_guide/benchmark_and_profiling.md + developer_guide/bench_serving.md + +.. toctree:: + :maxdepth: 1 + :caption: References + + references/faq.md + references/environment_variables.md + references/production_metrics.md + references/multi_node_deployment/multi_node_index.rst + references/custom_chat_template.md + references/frontend/frontend_index.rst + references/learn_more.md diff --git a/docs/platforms/amd_gpu.md b/docs/platforms/amd_gpu.md new file mode 100644 index 000000000..81d6d544a --- /dev/null +++ b/docs/platforms/amd_gpu.md @@ -0,0 +1,158 @@ +# AMD GPUs + +This document describes how run SGLang on AMD GPUs. If you encounter issues or have questions, please [open an issue](https://github.com/sgl-project/sglang/issues). + +## System Configuration + +When using AMD GPUs (such as MI300X), certain system-level optimizations help ensure stable performance. Here we take MI300X as an example. AMD provides official documentation for MI300X optimization and system tuning: + +- [AMD MI300X Tuning Guides](https://rocm.docs.amd.com/en/latest/how-to/tuning-guides/mi300x/index.html) +- [LLM inference performance validation on AMD Instinct MI300X](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference/vllm-benchmark.html) +- [AMD Instinct MI300X System Optimization](https://rocm.docs.amd.com/en/latest/how-to/system-optimization/mi300x.html) +- [AMD Instinct MI300X Workload Optimization](https://rocm.docs.amd.com/en/latest/how-to/rocm-for-ai/inference-optimization/workload.html) +- [Supercharge DeepSeek-R1 Inference on AMD Instinct MI300X](https://rocm.blogs.amd.com/artificial-intelligence/DeepSeekR1-Part2/README.html) + +**NOTE:** We strongly recommend reading these docs and guides entirely to fully utilize your system. + +Below are a few key settings to confirm or enable for SGLang: + +### Update GRUB Settings + +In `/etc/default/grub`, append the following to `GRUB_CMDLINE_LINUX`: + +```text +pci=realloc=off iommu=pt +``` + +Afterward, run `sudo update-grub` (or your distro’s equivalent) and reboot. + +### Disable NUMA Auto-Balancing + +```bash +sudo sh -c 'echo 0 > /proc/sys/kernel/numa_balancing' +``` + +You can automate or verify this change using [this helpful script](https://github.com/ROCm/triton/blob/rocm_env/scripts/amd/env_check.sh). + +Again, please go through the entire documentation to confirm your system is using the recommended configuration. + +## Install SGLang + +You can install SGLang using one of the methods below. + +### Install from Source + +```bash +# Use the last release branch +git clone -b v0.5.2 https://github.com/sgl-project/sglang.git +cd sglang + +# Compile sgl-kernel +pip install --upgrade pip +cd sgl-kernel +python setup_rocm.py install + +# Install sglang python package +cd .. +pip install -e "python[all_hip]" +``` + +### Install Using Docker (Recommended) + +The docker images are available on Docker Hub at [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags), built from [Dockerfile.rocm](https://github.com/sgl-project/sglang/tree/main/docker). + +The steps below show how to build and use an image. + +1. Build the docker image. + If you use pre-built images, you can skip this step and replace `sglang_image` with the pre-built image names in the steps below. + + ```bash + docker build -t sglang_image -f Dockerfile.rocm . + ``` + +2. Create a convenient alias. + + ```bash + alias drun='docker run -it --rm --network=host --privileged --device=/dev/kfd --device=/dev/dri \ + --ipc=host --shm-size 16G --group-add video --cap-add=SYS_PTRACE \ + --security-opt seccomp=unconfined \ + -v $HOME/dockerx:/dockerx \ + -v /data:/data' + ``` + + If you are using RDMA, please note that: + - `--network host` and `--privileged` are required by RDMA. If you don't need RDMA, you can remove them. + - You may need to set `NCCL_IB_GID_INDEX` if you are using RoCE, for example: `export NCCL_IB_GID_INDEX=3`. + +3. Launch the server. + + **NOTE:** Replace `` below with your [huggingface hub token](https://huggingface.co/docs/hub/en/security-tokens). + + ```bash + drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path NousResearch/Meta-Llama-3.1-8B \ + --host 0.0.0.0 \ + --port 30000 + ``` + +4. To verify the utility, you can run a benchmark in another terminal or refer to [other docs](https://docs.sglang.ai/backend/openai_api_completions.html) to send requests to the engine. + + ```bash + drun sglang_image \ + python3 -m sglang.bench_serving \ + --backend sglang \ + --dataset-name random \ + --num-prompts 4000 \ + --random-input 128 \ + --random-output 128 + ``` + +With your AMD system properly configured and SGLang installed, you can now fully leverage AMD hardware to power SGLang’s machine learning capabilities. + +## Examples + +### Running DeepSeek-V3 + +The only difference when running DeepSeek-V3 is in how you start the server. Here's an example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-V3 \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +[Running DeepSeek-R1 on a single NDv5 MI300X VM](https://techcommunity.microsoft.com/blog/azurehighperformancecomputingblog/running-deepseek-r1-on-a-single-ndv5-mi300x-vm/4372726) could also be a good reference. + +### Running Llama3.1 + +Running Llama3.1 is nearly identical to running DeepSeek-V3. The only difference is in the model specified when starting the server, shown by the following example command: + +```bash +drun -p 30000:30000 \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + --ipc=host \ + --env "HF_TOKEN=" \ + sglang_image \ + python3 -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ # <- here + --tp 8 \ + --trust-remote-code \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Warmup Step + +When the server displays `The server is fired up and ready to roll!`, it means the startup is successful. diff --git a/docs/platforms/ascend_npu.md b/docs/platforms/ascend_npu.md new file mode 100644 index 000000000..f57d3fe95 --- /dev/null +++ b/docs/platforms/ascend_npu.md @@ -0,0 +1,206 @@ +# Ascend NPUs + +You can install SGLang using any of the methods below. Please go through `System Settings` section to ensure the clusters are roaring at max performance. Feel free to leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) if you encounter any issues or have any problems. + +## System Settings + +### CPU performance power scheme + +The default power scheme on Ascend hardware is `ondemand` which could affect performance, changing it to `performance` is recommended. + +```shell +echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + +# Make sure changes are applied successfully +cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor # shows performance +``` + +### Disable NUMA balancing + +```shell +sudo sysctl -w kernel.numa_balancing=0 + +# Check +cat /proc/sys/kernel/numa_balancing # shows 0 +``` + +### Prevent swapping out system memory + +```shell +sudo sysctl -w vm.swappiness=10 + +# Check +cat /proc/sys/vm/swappiness # shows 10 +``` + +## Installing SGLang + +### Method 1: Installing from source with prerequisites + +#### Python Version + +Only `python==3.11` is supported currently. If you don't want to break system pre-installed python, try installing with [conda](https://github.com/conda/conda). + +```shell +conda create --name sglang_npu python=3.11 +conda activate sglang_npu +``` + +#### MemFabric Adaptor + +_TODO: MemFabric is still a working project yet open sourced til August/September, 2025. We will release it as prebuilt wheel package for now._ + +_Notice: Prebuilt wheel package is based on `aarch64`, please leave an issue [here at sglang](https://github.com/sgl-project/sglang/issues) to let us know the requests for `amd64` build._ + +MemFabric Adaptor is a drop-in replacement of Mooncake Transfer Engine that enables KV cache transfer on Ascend NPU clusters. + +```shell +MF_WHL_NAME="mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl" +MEMFABRIC_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/${MF_WHL_NAME}" +wget -O "${MF_WHL_NAME}" "${MEMFABRIC_URL}" && pip install "./${MF_WHL_NAME}" +``` + +#### Pytorch and Pytorch Framework Adaptor on Ascend + +Only `torch==2.6.0` is supported currently due to NPUgraph and Triton-on-Ascend's limitation, however a more generalized version will be release by the end of September, 2025. + +```shell +PYTORCH_VERSION=2.6.0 +TORCHVISION_VERSION=0.21.0 +pip install torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu + +PTA_VERSION="v7.1.0.1-pytorch2.6.0" +PTA_NAME="torch_npu-2.6.0.post1-cp311-cp311-manylinux_2_28_aarch64.whl" +PTA_URL="https://gitee.com/ascend/pytorch/releases/download/${PTA_VERSION}/${PTA_WHL_NAME}" +wget -O "${PTA_NAME}" "${PTA_URL}" && pip install "./${PTA_NAME}" +``` + +#### vLLM + +vLLM is still a major prerequisite on Ascend NPU. Because of `torch==2.6.0` limitation, only vLLM v0.8.5 is supported. + +```shell +VLLM_TAG=v0.8.5 +git clone --depth 1 https://github.com/vllm-project/vllm.git --branch $VLLM_TAG +(cd vllm && VLLM_TARGET_DEVICE="empty" pip install -v -e .) +``` + +#### Triton on Ascend + +_Notice:_ We recommend installing triton-ascend from source due to its rapid development, the version on PYPI can't keep up for know. This problem will be solved on Sep. 2025, afterwards `pip install` would be the one and only installing method. + +Please follow Triton-on-Ascend's [installation guide from source](https://gitee.com/ascend/triton-ascend#2%E6%BA%90%E4%BB%A3%E7%A0%81%E5%AE%89%E8%A3%85-triton-ascend) to install the latest `triton-ascend` package. + +#### DeepEP-compatible Library + +We are also providing a DeepEP-compatible Library as a drop-in replacement of deepseek-ai's DeepEP library, check the [installation guide](https://github.com/sgl-project/sgl-kernel-npu/blob/main/python/deep_ep/README.md). + +#### Installing SGLang from source + +```shell +# Use the last release branch +git clone -b v0.5.2 https://github.com/sgl-project/sglang.git +cd sglang + +pip install --upgrade pip +pip install -e python[srt_npu] +``` + +### Method 2: Using docker + +__Notice:__ `--privileged` and `--network=host` are required by RDMA, which is typically needed by Ascend NPU clusters. + +__Notice:__ The following docker command is based on Atlas 800I A3 machines. If you are using Atlas 800I A2, make sure only `davinci[0-7]` are mapped into container. + +```shell +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +docker build -t sglang-npu:main -f Dockerfile.npu . + +alias drun='docker run -it --rm --privileged --network=host --ipc=host --shm-size=16g \ + --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 \ + --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 \ + --device=/dev/davinci8 --device=/dev/davinci9 --device=/dev/davinci10 --device=/dev/davinci11 \ + --device=/dev/davinci12 --device=/dev/davinci13 --device=/dev/davinci14 --device=/dev/davinci15 \ + --device=/dev/davinci_manager --device=/dev/hisi_hdc \ + --volume /usr/local/sbin:/usr/local/sbin --volume /usr/local/Ascend/driver:/usr/local/Ascend/driver \ + --volume /usr/local/Ascend/firmware:/usr/local/Ascend/firmware \ + --volume /etc/ascend_install.info:/etc/ascend_install.info \ + --volume /var/queue_schedule:/var/queue_schedule --volume ~/.cache/:/root/.cache/' + +drun --env "HF_TOKEN=" \ + sglang-npu:main \ + python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --attention-backend ascend --host 0.0.0.0 --port 30000 +``` + +## Examples + +### Running DeepSeek-V3 + +Running DeepSeek with PD disaggregation on 2 x Atlas 800I A3. +Model weights could be found [here](https://modelers.cn/models/State_Cloud/Deepseek-R1-bf16-hfd-w8a8). + +Prefill: + +```shell +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export ASCEND_MF_STORE_URL="tcp://:" + +drun sglang-npu:main \ + python3 -m sglang.launch_server --model-path State_Cloud/DeepSeek-R1-bf16-hfd-w8a8 \ + --trust-remote-code \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --quantization w8a8_int8 \ + --tp-size 16 \ + --dp-size 1 \ + --nnodes 1 \ + --node-rank 0 \ + --disaggregation-mode prefill \ + --disaggregation-bootstrap-port 6657 \ + --disaggregation-transfer-backend ascend \ + --dist-init-addr :6688 \ + --host \ + --port 8000 +``` + +Decode: + +```shell +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export ASCEND_MF_STORE_URL="tcp://:" +export HCCL_BUFFSIZE=200 +export SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK=24 + +drun sglang-npu:main \ + python3 -m sglang.launch_server --model-path State_Cloud/DeepSeek-R1-bf16-hfd-w8a8 \ + --trust-remote-code \ + --attention-backend ascend \ + --mem-fraction-static 0.8 \ + --quantization w8a8_int8 \ + --enable-deepep-moe \ + --deepep-mode low_latency \ + --tp-size 16 \ + --dp-size 1 \ + --ep-size 16 \ + --nnodes 1 \ + --node-rank 0 \ + --disaggregation-mode decode \ + --disaggregation-transfer-backend ascend \ + --dist-init-addr :6688 \ + --host \ + --port 8001 +``` + +Mini_LB: + +```shell +drun sglang-npu:main \ + python -m sglang.srt.disaggregation.launch_lb \ + --prefill http://:8000 \ + --decode http://:8001 \ + --host 127.0.0.1 --port 5000 +``` diff --git a/docs/platforms/blackwell_gpu.md b/docs/platforms/blackwell_gpu.md new file mode 100644 index 000000000..8c433b3f0 --- /dev/null +++ b/docs/platforms/blackwell_gpu.md @@ -0,0 +1,9 @@ +# Blackwell GPUs + +We will release the pre-built wheels soon. Before that, please try to compile from source or check the blackwell docker images from [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang/tags). + +## B200 with x86 CPUs +TODO + +## GB200/GB300 with ARM CPUs +TODO diff --git a/docs/platforms/cpu_server.md b/docs/platforms/cpu_server.md new file mode 100644 index 000000000..97fad918d --- /dev/null +++ b/docs/platforms/cpu_server.md @@ -0,0 +1,204 @@ +# CPU Servers + +The document addresses how to set up the [SGLang](https://github.com/sgl-project/sglang) environment and run LLM inference on CPU servers. +Specifically, SGLang is well optimized on the CPUs equipped with Intel® AMX® Instructions, +which are 4th generation or newer Intel® Xeon® Scalable Processors. + +## Optimized Model List + +A list of popular LLMs are optimized and run efficiently on CPU, +including the most notable open-source models like Llama series, Qwen series, +and the phenomenal high-quality reasoning model DeepSeek-R1. + +| Model Name | BF16 | w8a8_int8 | FP8 | +|:---:|:---:|:---:|:---:| +| DeepSeek-R1 | | [meituan/DeepSeek-R1-Channel-INT8](https://huggingface.co/meituan/DeepSeek-R1-Channel-INT8) | [deepseek-ai/DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) | +| Llama-3.2-3B | [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) | [RedHatAI/Llama-3.2-3B-quantized.w8a8](https://huggingface.co/RedHatAI/Llama-3.2-3B-Instruct-quantized.w8a8) | | +| Llama-3.1-8B | [meta-llama/Llama-3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) | [RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8](https://huggingface.co/RedHatAI/Meta-Llama-3.1-8B-quantized.w8a8) | | +| QwQ-32B | | [RedHatAI/QwQ-32B-quantized.w8a8](https://huggingface.co/RedHatAI/QwQ-32B-quantized.w8a8) | | +| DeepSeek-Distilled-Llama | | [RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8](https://huggingface.co/RedHatAI/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8) | | +| Qwen3-235B | | | [Qwen/Qwen3-235B-A22B-FP8](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8) | + +**Note:** The model identifiers listed in the table above +have been verified on 6th Gen Intel® Xeon® P-core platforms. + +## Installation + +### Install Using Docker + +It is recommended to use Docker for setting up the SGLang environment. +A [Dockerfile](https://github.com/sgl-project/sglang/blob/main/docker/Dockerfile.xeon) is provided to facilitate the installation. +Replace `` below with your [HuggingFace access token](https://huggingface.co/docs/hub/en/security-tokens). + +```bash +# Clone the SGLang repository +git clone https://github.com/sgl-project/sglang.git +cd sglang/docker + +# Build the docker image +docker build -t sglang-cpu:main -f Dockerfile.xeon . + +# Initiate a docker container +docker run \ + -it \ + --privileged \ + --ipc=host \ + --network=host \ + -v /dev/shm:/dev/shm \ + -v ~/.cache/huggingface:/root/.cache/huggingface \ + -p 30000:30000 \ + -e "HF_TOKEN=" \ + sglang-cpu:main /bin/bash +``` + +### Install From Source + +If you'd prefer to install SGLang in a bare metal environment, +the command list is as below. +It is worth noting that the environment variable `SGLANG_USE_CPU_ENGINE=1` +is required to enable SGLang service with CPU engine. + +```bash +# Create and activate a conda environment +conda create -n sgl-cpu python=3.12 -y +conda activate sgl-cpu + +# Optional: Set PyTorch CPU as primary pip install channel to avoid installing CUDA version +pip config set global.index-url https://download.pytorch.org/whl/cpu +pip config set global.extra-index-url https://pypi.org/simple + +# Check if some conda related environment variables have been set +env | grep -i conda +# The following environment variable settings are required +# if they have not been set properly +export CONDA_EXE=$(which conda) +export CONDA_ROOT=${CONDA_EXE}/../.. +export CONDA_PREFIX=${CONDA_ROOT}/envs/sgl-cpu +export PATH=${PATH}:${CONDA_ROOT}/bin:${CONDA_ROOT}/condabin + +# Clone the SGLang code +git clone https://github.com/sgl-project/sglang.git +cd sglang +git checkout + +# Install SGLang dependent libs, and build SGLang main package +pip install --upgrade pip setuptools +conda install -y libsqlite==3.48.0 gperftools tbb libnuma numactl +pip install -e "python[all_cpu]" +pip install torch==2.7.1 torchvision==0.22.1 triton==3.3.1 --force-reinstall + +# Build the CPU backend kernels +cd sgl-kernel +cp pyproject_cpu.toml pyproject.toml +pip install . + +# Other required environment variables +# Recommend to set these in ~/.bashrc in order not to set every time in a new terminal +export SGLANG_USE_CPU_ENGINE=1 +export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libiomp5.so:${CONDA_PREFIX}/lib/libtcmalloc.so:${CONDA_PREFIX}/lib/libtbbmalloc.so.2 +``` + +## Launch of the Serving Engine + +Example command to launch SGLang serving: + +```bash +python -m sglang.launch_server \ + --model \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --tp 6 +``` + +Notes: + +1. For running W8A8 quantized models, please add the flag `--quantization w8a8_int8`. + +2. The flag `--tp 6` specifies that tensor parallelism will be applied using 6 ranks (TP6). + The number of TP specified is how many TP ranks will be used during the execution. + In a CPU platform, a TP rank means a sub-NUMA cluster (SNC). + Usually we can get the SNC information (How many available) from Operation System. + User can specify TP to be no more than the total available SNCs in current system. + + If the specified TP rank number differs from the total SNC count, + the system will automatically utilize the first `n` SNCs. + Note that `n` cannot exceed the total SNC number, doing so will result in an error. + + To specify the cores to be used, we need to explicitly set the environment variable `SGLANG_CPU_OMP_THREADS_BIND`. + For example, if we want to run the SGLang service using the first 40 cores of each SNC on a Xeon® 6980P server, + which has 43-43-42 cores on the 3 SNCs of a socket, we should set: + + ```bash + export SGLANG_CPU_OMP_THREADS_BIND="0-39|43-82|86-125|128-167|171-210|214-253" + ``` + + Please beware that with SGLANG_CPU_OMP_THREADS_BIND set, + the available memory amounts of the ranks may not be determined in prior. + You may need to set proper `--max-total-tokens` to avoid the out-of-memory error. + +3. For optimizing decoding with torch.compile, please add the flag `--enable-torch-compile`. + To specify the maximum batch size when using torch compile, set the flag `--torch-compile-max-bs`. + For example, `--enable-torch-compile --torch-compile-max-bs 4` means using torch compile and setting the + maximum batch size to 4. + +4. A warmup step is automatically triggered when the service is started. + The server is ready when you see the log `The server is fired up and ready to roll!`. + +## Benchmarking with Requests + +You can benchmark the performance via the `bench_serving` script. +Run the command in another terminal. + +```bash +python -m sglang.bench_serving \ + --dataset-name random \ + --random-input-len 1024 \ + --random-output-len 1024 \ + --num-prompts 1 \ + --request-rate inf \ + --random-range-ratio 1.0 +``` + +The detail explanations of the parameters can be looked up by the command: + +```bash +python -m sglang.bench_serving -h +``` + +Additionally, the requests can be formed with +[OpenAI Completions API](https://docs.sglang.ai/basic_usage/openai_api_completions.html) +and sent via the command line (e.g. using `curl`) or via your own script. + +## Example: Running DeepSeek-R1 + +An example command to launch service for W8A8 DeepSeek-R1 on a Xeon® 6980P server + +```bash +python -m sglang.launch_server \ + --model meituan/DeepSeek-R1-Channel-INT8 \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --quantization w8a8_int8 \ + --host 0.0.0.0 \ + --mem-fraction-static 0.8 \ + --tp 6 +``` + +Similarly, an example command to launch service for FP8 DeepSeek-R1 would be + +```bash +python -m sglang.launch_server \ + --model deepseek-ai/DeepSeek-R1 \ + --trust-remote-code \ + --disable-overlap-schedule \ + --device cpu \ + --host 0.0.0.0 \ + --mem-fraction-static 0.8 \ + --tp 6 +``` + +Then you can test with `bench_serving` command or construct your own command or script +following [the benchmarking example](#benchmarking-with-requests). diff --git a/docs/platforms/nvidia_jetson.md b/docs/platforms/nvidia_jetson.md new file mode 100644 index 000000000..362f60c83 --- /dev/null +++ b/docs/platforms/nvidia_jetson.md @@ -0,0 +1,80 @@ +# NVIDIA Jetson Orin + +## Prerequisites + +Before starting, ensure the following: + +- [**NVIDIA Jetson AGX Orin Devkit**](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/) is set up with **JetPack 6.1** or later. +- **CUDA Toolkit** and **cuDNN** are installed. +- Verify that the Jetson AGX Orin is in **high-performance mode**: +```bash +sudo nvpmodel -m 0 +``` +* * * * * +## Installing and running SGLang with Jetson Containers +Clone the jetson-containers github repository: +``` +git clone https://github.com/dusty-nv/jetson-containers.git +``` +Run the installation script: +``` +bash jetson-containers/install.sh +``` +Build the container image: +``` +jetson-containers build sglang +``` +Run the container: +``` +jetson-containers run $(autotag sglang) +``` +Or you can also manually run a container with this command: +``` +docker run --runtime nvidia -it --rm --network=host IMAGE_NAME +``` +* * * * * + +Running Inference +----------------------------------------- + +Launch the server: +```bash +python -m sglang.launch_server \ + --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \ + --device cuda \ + --dtype half \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 +``` +The quantization and limited context length (`--dtype half --context-length 8192`) are due to the limited computational resources in [Nvidia jetson kit](https://www.nvidia.com/en-us/autonomous-machines/embedded-systems/jetson-orin/). A detailed explanation can be found in [Server Arguments](../backend/server_arguments.md). + +After launching the engine, refer to [Chat completions](https://docs.sglang.ai/backend/openai_api_completions.html#Usage) to test the usability. +* * * * * +Running quantization with TorchAO +------------------------------------- +TorchAO is suggested to NVIDIA Jetson Orin. +```bash +python -m sglang.launch_server \ + --model-path meta-llama/Meta-Llama-3.1-8B-Instruct \ + --device cuda \ + --dtype bfloat16 \ + --attention-backend flashinfer \ + --mem-fraction-static 0.8 \ + --context-length 8192 \ + --torchao-config int4wo-128 +``` +This enables TorchAO's int4 weight-only quantization with a 128-group size. The usage of `--torchao-config int4wo-128` is also for memory efficiency. + + +* * * * * +Structured output with XGrammar +------------------------------- +Please refer to [SGLang doc structured output](../advanced_features/structured_outputs.ipynb). +* * * * * + +Thanks to the support from [Nurgaliyev Shakhizat](https://github.com/shahizat), [Dustin Franklin](https://github.com/dusty-nv) and [Johnny Núñez Cano](https://github.com/johnnynunez). + +References +---------- +- [NVIDIA Jetson AGX Orin Documentation](https://developer.nvidia.com/embedded/jetson-agx-orin) diff --git a/docs/platforms/tpu.md b/docs/platforms/tpu.md new file mode 100644 index 000000000..f304234cf --- /dev/null +++ b/docs/platforms/tpu.md @@ -0,0 +1,3 @@ +# TPU + +The support for TPU is under active development. Please stay tuned. diff --git a/docs/references/custom_chat_template.md b/docs/references/custom_chat_template.md new file mode 100644 index 000000000..557af5bf5 --- /dev/null +++ b/docs/references/custom_chat_template.md @@ -0,0 +1,42 @@ +# Custom Chat Template + +**NOTE**: There are two chat template systems in SGLang project. This document is about setting a custom chat template for the OpenAI-compatible API server (defined at [conversation.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/conversation.py)). It is NOT related to the chat template used in the SGLang language frontend (defined at [chat_template.py](https://github.com/sgl-project/sglang/blob/main/python/sglang/lang/chat_template.py)). + +By default, the server uses the chat template specified in the model tokenizer from Hugging Face. +It should just work for most official models such as Llama-2/Llama-3. + +If needed, you can also override the chat template when launching the server: + +```bash +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --chat-template llama-2 +``` + +If the chat template you are looking for is missing, you are welcome to contribute it or load it from a file. + +## JSON Format + +You can load the JSON format, which is defined by `conversation.py`. + +```json +{ + "name": "my_model", + "system": "<|im_start|>system", + "user": "<|im_start|>user", + "assistant": "<|im_start|>assistant", + "sep_style": "CHATML", + "sep": "<|im_end|>", + "stop_str": ["<|im_end|>", "<|im_start|>"] +} +``` + +```bash +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --chat-template ./my_model_template.json +``` + +## Jinja Format + +You can also use the [Jinja template format](https://huggingface.co/docs/transformers/main/en/chat_templating) as defined by Hugging Face Transformers. + +```bash +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 --chat-template ./my_model_template.jinja +``` diff --git a/docs/references/environment_variables.md b/docs/references/environment_variables.md new file mode 100644 index 000000000..f22685454 --- /dev/null +++ b/docs/references/environment_variables.md @@ -0,0 +1,97 @@ +# Environment Variables + +SGLang supports various environment variables that can be used to configure its runtime behavior. This document provides a comprehensive list and aims to stay updated over time. + +*Note: SGLang uses two prefixes for environment variables: `SGL_` and `SGLANG_`. This is likely due to historical reasons. While both are currently supported for different settings, future versions might consolidate them.* + +## General Configuration + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_USE_MODELSCOPE` | Enable using models from ModelScope | `false` | +| `SGLANG_HOST_IP` | Host IP address for the server | `0.0.0.0` | +| `SGLANG_PORT` | Port for the server | auto-detected | +| `SGLANG_LOGGING_CONFIG_PATH` | Custom logging configuration path | Not set | +| `SGLANG_DISABLE_REQUEST_LOGGING` | Disable request logging | `false` | +| `SGLANG_HEALTH_CHECK_TIMEOUT` | Timeout for health check in seconds | `20` | + +## Performance Tuning + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_ENABLE_TORCH_INFERENCE_MODE` | Control whether to use torch.inference_mode | `false` | +| `SGLANG_ENABLE_TORCH_COMPILE` | Enable torch.compile | `true` | +| `SGLANG_SET_CPU_AFFINITY` | Enable CPU affinity setting (often set to `1` in Docker builds) | `0` | +| `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN` | Allows the scheduler to overwrite longer context length requests (often set to `1` in Docker builds) | `0` | +| `SGLANG_IS_FLASHINFER_AVAILABLE` | Control FlashInfer availability check | `true` | +| `SGLANG_SKIP_P2P_CHECK` | Skip P2P (peer-to-peer) access check | `false` | +| `SGL_CHUNKED_PREFIX_CACHE_THRESHOLD` | Sets the threshold for enabling chunked prefix caching | `8192` | +| `SGLANG_FUSED_MLA_ENABLE_ROPE_FUSION` | Enable RoPE fusion in Fused Multi-Layer Attention | `1` | + +## DeepGEMM Configuration (Advanced Optimization) + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGL_ENABLE_JIT_DEEPGEMM` | Enable Just-In-Time compilation of DeepGEMM kernels | `"true"` | +| `SGL_JIT_DEEPGEMM_PRECOMPILE` | Enable precompilation of DeepGEMM kernels | `"true"` | +| `SGL_JIT_DEEPGEMM_COMPILE_WORKERS` | Number of workers for parallel DeepGEMM kernel compilation | `4` | +| `SGL_IN_DEEPGEMM_PRECOMPILE_STAGE` | Indicator flag used during the DeepGEMM precompile script | `"false"` | +| `SGL_DG_CACHE_DIR` | Directory for caching compiled DeepGEMM kernels | `~/.cache/deep_gemm` | +| `SGL_DG_USE_NVRTC` | Use NVRTC (instead of Triton) for JIT compilation (Experimental) | `"0"` | +| `SGL_USE_DEEPGEMM_BMM` | Use DeepGEMM for Batched Matrix Multiplication (BMM) operations | `"false"` | + +## Memory Management + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_DEBUG_MEMORY_POOL` | Enable memory pool debugging | `false` | +| `SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION` | Clip max new tokens estimation for memory planning | `4096` | +| `SGLANG_DETOKENIZER_MAX_STATES` | Maximum states for detokenizer | Default value based on system | +| `SGL_DISABLE_TP_MEMORY_INBALANCE_CHECK` | Disable checks for memory imbalance across Tensor Parallel ranks | Not set (defaults to enabled check) | + +## Model-Specific Options + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_USE_AITER` | Use AITER optimize implementation | `false` | +| `SGLANG_INT4_WEIGHT` | Enable INT4 weight quantization | `false` | +| `SGLANG_MOE_PADDING` | Enable MoE padding (sets padding size to 128 if value is `1`, often set to `1` in Docker builds) | `0` | +| `SGLANG_FORCE_FP8_MARLIN` | Force using FP8 MARLIN kernels even if other FP8 kernels are available | `false` | +| `SGLANG_ENABLE_FLASHINFER_GEMM` | Use flashinfer kernels when running blockwise fp8 GEMM on Blackwell GPUs | `false` | +| `SGLANG_SUPPORT_CUTLASS_BLOCK_FP8` | Use Cutlass kernels when running blockwise fp8 GEMM on Hopper or Blackwell GPUs | `false` | +| `SGLANG_CUTLASS_MOE` | Use Cutlass FP8 MoE kernel on Blackwell GPUs | `false` | + + +## Distributed Computing + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_BLOCK_NONZERO_RANK_CHILDREN` | Control blocking of non-zero rank children processes | `1` | +| `SGL_IS_FIRST_RANK_ON_NODE` | Indicates if the current process is the first rank on its node | `"true"` | +| `SGLANG_PP_LAYER_PARTITION` | Pipeline parallel layer partition specification | Not set | + +## Testing & Debugging (Internal/CI) + +*These variables are primarily used for internal testing, continuous integration, or debugging.* + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_IS_IN_CI` | Indicates if running in CI environment | `false` | +| `SGLANG_AMD_CI` | Indicates running in AMD CI environment | `0` | +| `SGLANG_TEST_RETRACT` | Enable retract decode testing | `false` | +| `SGLANG_RECORD_STEP_TIME` | Record step time for profiling | `false` | +| `SGLANG_TEST_REQUEST_TIME_STATS` | Test request time statistics | `false` | +| `SGLANG_CI_SMALL_KV_SIZE` | Use small KV cache size in CI | Not set | + +## Profiling & Benchmarking + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_TORCH_PROFILER_DIR` | Directory for PyTorch profiler output | `/tmp` | +| `SGLANG_PROFILE_WITH_STACK` | Set `with_stack` option (bool) for PyTorch profiler (capture stack trace) | `true` | + +## Storage & Caching + +| Environment Variable | Description | Default Value | +| --- | --- | --- | +| `SGLANG_DISABLE_OUTLINES_DISK_CACHE` | Disable Outlines disk cache | `true` | diff --git a/docs/references/faq.md b/docs/references/faq.md new file mode 100644 index 000000000..6d575d253 --- /dev/null +++ b/docs/references/faq.md @@ -0,0 +1,35 @@ +# Troubleshooting and Frequently Asked Questions + +## Troubleshooting + +This page lists common errors and tips for resolving them. + +### CUDA Out of Memory +If you encounter out-of-memory (OOM) errors, you can adjust the following parameters: + +- If OOM occurs during prefill, try reducing `--chunked-prefill-size` to `4096` or `2048`. This saves memory but slows down the prefill speed for long prompts. +- If OOM occurs during decoding, try lowering `--max-running-requests`. +- You can also reduce `--mem-fraction-static` to a smaller value, such as 0.8 or 0.7. This decreases the memory usage of the KV cache memory pool and helps prevent OOM errors during both prefill and decoding. However, it limits maximum concurrency and reduces peak throughput. +- Another common case for OOM is requesting input logprobs for a long prompt as it requires significant memory. To address this, set `logprob_start_len` in your sampling parameters to include only the necessary parts. If you do need input logprobs for a long prompt, try reducing `--mem-fraction-static`. + +### CUDA Error: Illegal Memory Access Encountered +This error may result from kernel errors or out-of-memory issues: +- If it is a kernel error, resolving it may be challenging. Please file an issue on GitHub. +- If it is an out-of-memory issue, it may sometimes be reported as this error instead of "Out of Memory." Refer to the section above for guidance on avoiding OOM issues. + + +## Frequently Asked Questions + +### The results are not deterministic, even with a temperature of 0 + +You may notice that when you send the same request twice, the results from the engine will be slightly different, even when the temperature is set to 0. + +From our initial investigation, this indeterminism arises from two factors: dynamic batching and prefix caching. Roughly speaking, dynamic batching accounts for about 95% of the indeterminism, while prefix caching accounts for the remaining portion. The server runs dynamic batching under the hood. Different batch sizes can cause PyTorch/CuBLAS to dispatch to different CUDA kernels, which can lead to slight numerical differences. This difference accumulates across many layers, resulting in nondeterministic output when the batch size changes. Similarly, when prefix caching is enabled, it can also dispatch to different kernels. Even when the computations are mathematically equivalent, small numerical differences from different kernel implementations lead to the final nondeterministic outputs. + +To achieve more deterministic outputs in the current code, you can add `--disable-radix-cache` and send only one request at a time. The results will be mostly deterministic under this setting. + +We are still investigating the root causes and potential solutions. In the short term, we may introduce a "deterministic mode" that uses more padding to address the variance caused by dynamic batching. This mode will be more deterministic but slower. + +We have two issues to track our progress: +- The deterministic mode is tracked at [https://github.com/sgl-project/sglang/issues/1729](https://github.com/sgl-project/sglang/issues/1729). +- The per-request random seed is tracked at [https://github.com/sgl-project/sglang/issues/1335](https://github.com/sgl-project/sglang/issues/1335). diff --git a/docs/references/frontend/choices_methods.md b/docs/references/frontend/choices_methods.md new file mode 100644 index 000000000..30a0a1814 --- /dev/null +++ b/docs/references/frontend/choices_methods.md @@ -0,0 +1,77 @@ +# Choices Methods in SGLang +This doc describes the choices methods supported by SGLang. + +The optional `choices_method` arg determines how options supplied to SGLang's `choices` primitive are selected. Only the `RuntimeEndpoint` backend supports the `choices_method` arg. Other backends, such as `OpenAI`, have bespoke selection implementations due to API limitations. + +## Methods + +### Token Length Normalized + +Token length normalized is the default SGLang choices method. It selects the option with the highest average logprob across all of its tokens. + +Usage example (alternatively, simply omit the `choices_method` arg): +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.token_length_normalized, + ) + ) +``` + + +This can perform poorly if an option contains many tokens, where its later tokens are predicted with high confidence based on its earlier tokens. For instance, even strong models will fail the above example if the specified options are `["Paris", "Antidisestablishmentarianism"]`. + +### Greedy Token Selection + +Greedy token selection simply selects the option with the highest logprob for its initial token. For overlapping options where one option is a subset of a longer option, the logprobs of the shorter option are extended using its average logprob for comparison against the longer option. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +This can perform poorly if an option misleads the model down a bad path based on an attractive initial token. For instance, greedy selection will result in an incorrect response for this example: +```python +@sgl.function +def us_president_example(s): + s += sgl.user("Name a US president.") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["Donald Duck", "Millard Fillmore"], + choices_method=sgl.greedy_token_selection, + ) + ) +``` + +### Unconditional Likelihood Normalized + +Unconditional likelihood normalized selects the option with the highest average token logprob once normalized by the unconditional token logprobs, as described in [this EleutherAI blogpost](https://blog.eleuther.ai/multiple-choice-normalization/). This method incurs an additional LLM call to obtain the unconditional likelihoods. + +Usage example: +```python +@sgl.function +def example(s): + s += sgl.user("What is the capital of France?") + s += sgl.assistant( + sgl.gen( + "answer", + choices=["London", "Paris", "Berlin"], + choices_method=sgl.unconditional_likelihood_normalized, + ) + ) +``` diff --git a/docs/references/frontend/frontend_index.rst b/docs/references/frontend/frontend_index.rst new file mode 100644 index 000000000..62544cba5 --- /dev/null +++ b/docs/references/frontend/frontend_index.rst @@ -0,0 +1,9 @@ +Frontend Language +================= + +.. toctree:: + :maxdepth: 1 + :caption: Frontend Language + + frontend_tutorial.ipynb + choices_methods.md diff --git a/docs/references/frontend/frontend_tutorial.ipynb b/docs/references/frontend/frontend_tutorial.ipynb new file mode 100644 index 000000000..836cab627 --- /dev/null +++ b/docs/references/frontend/frontend_tutorial.ipynb @@ -0,0 +1,456 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SGLang Frontend Language" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SGLang frontend language can be used to define simple and easy prompts in a convenient, structured way." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server in your terminal and wait for it to initialize." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import assistant_begin, assistant_end\n", + "from sglang import assistant, function, gen, system, user\n", + "from sglang import image\n", + "from sglang import RuntimeEndpoint\n", + "from sglang.lang.api import set_default_backend\n", + "from sglang.srt.utils import load_image\n", + "from sglang.test.doc_patch import launch_server_cmd\n", + "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the default backend. Note: Besides the local server, you may use also `OpenAI` or other API endpoints." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Usage\n", + "\n", + "The most simple way of using SGLang frontend language is a simple question answer dialog between a user and an assistant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", max_tokens=512))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "state = basic_qa(\"List 3 countries and their capitals.\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-turn Dialog\n", + "\n", + "SGLang frontend language can also be used to define multi-turn dialogs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def multi_turn_qa(s):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"first_answer\", max_tokens=512))\n", + " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", + " s += assistant(gen(\"second_answer\", max_tokens=512))\n", + " return s\n", + "\n", + "\n", + "state = multi_turn_qa()\n", + "print_highlight(state[\"first_answer\"])\n", + "print_highlight(state[\"second_answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Control flow\n", + "\n", + "You may use any Python code within the function to define more complex control flows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tool_use(s, question):\n", + " s += assistant(\n", + " \"To answer this question: \"\n", + " + question\n", + " + \". I need to use a \"\n", + " + gen(\"tool\", choices=[\"calculator\", \"search engine\"])\n", + " + \". \"\n", + " )\n", + "\n", + " if s[\"tool\"] == \"calculator\":\n", + " s += assistant(\"The math expression is: \" + gen(\"expression\"))\n", + " elif s[\"tool\"] == \"search engine\":\n", + " s += assistant(\"The key word to search is: \" + gen(\"word\"))\n", + "\n", + "\n", + "state = tool_use(\"What is 2 * 2?\")\n", + "print_highlight(state[\"tool\"])\n", + "print_highlight(state[\"expression\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parallelism\n", + "\n", + "Use `fork` to launch parallel prompts. Because `sgl.gen` is non-blocking, the for loop below issues two generation calls in parallel." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def tip_suggestion(s):\n", + " s += assistant(\n", + " \"Here are two tips for staying healthy: \"\n", + " \"1. Balanced Diet. 2. Regular Exercise.\\n\\n\"\n", + " )\n", + "\n", + " forks = s.fork(2)\n", + " for i, f in enumerate(forks):\n", + " f += assistant(\n", + " f\"Now, expand tip {i+1} into a paragraph:\\n\"\n", + " + gen(\"detailed_tip\", max_tokens=256, stop=\"\\n\\n\")\n", + " )\n", + "\n", + " s += assistant(\"Tip 1:\" + forks[0][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\"Tip 2:\" + forks[1][\"detailed_tip\"] + \"\\n\")\n", + " s += assistant(\n", + " \"To summarize the above two tips, I can say:\\n\" + gen(\"summary\", max_tokens=512)\n", + " )\n", + "\n", + "\n", + "state = tip_suggestion()\n", + "print_highlight(state[\"summary\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Constrained Decoding\n", + "\n", + "Use `regex` to specify a regular expression as a decoding constraint. This is only supported for local models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def regular_expression_gen(s):\n", + " s += user(\"What is the IP address of the Google DNS servers?\")\n", + " s += assistant(\n", + " gen(\n", + " \"answer\",\n", + " temperature=0,\n", + " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", + " )\n", + " )\n", + "\n", + "\n", + "state = regular_expression_gen()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use `regex` to define a `JSON` decoding schema." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "character_regex = (\n", + " r\"\"\"\\{\\n\"\"\"\n", + " + r\"\"\" \"name\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"house\": \"(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)\",\\n\"\"\"\n", + " + r\"\"\" \"blood status\": \"(Pure-blood|Half-blood|Muggle-born)\",\\n\"\"\"\n", + " + r\"\"\" \"occupation\": \"(student|teacher|auror|ministry of magic|death eater|order of the phoenix)\",\\n\"\"\"\n", + " + r\"\"\" \"wand\": \\{\\n\"\"\"\n", + " + r\"\"\" \"wood\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"core\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"length\": [0-9]{1,2}\\.[0-9]{0,2}\\n\"\"\"\n", + " + r\"\"\" \\},\\n\"\"\"\n", + " + r\"\"\" \"alive\": \"(Alive|Deceased)\",\\n\"\"\"\n", + " + r\"\"\" \"patronus\": \"[\\w\\d\\s]{1,16}\",\\n\"\"\"\n", + " + r\"\"\" \"bogart\": \"[\\w\\d\\s]{1,16}\"\\n\"\"\"\n", + " + r\"\"\"\\}\"\"\"\n", + ")\n", + "\n", + "\n", + "@function\n", + "def character_gen(s, name):\n", + " s += user(\n", + " f\"{name} is a character in Harry Potter. Please fill in the following information about this character.\"\n", + " )\n", + " s += assistant(gen(\"json_output\", max_tokens=256, regex=character_regex))\n", + "\n", + "\n", + "state = character_gen(\"Harry Potter\")\n", + "print_highlight(state[\"json_output\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batching \n", + "\n", + "Use `run_batch` to run a batch of prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "states = text_qa.run_batch(\n", + " [\n", + " {\"question\": \"What is the capital of the United Kingdom?\"},\n", + " {\"question\": \"What is the capital of France?\"},\n", + " {\"question\": \"What is the capital of Japan?\"},\n", + " ],\n", + " progress_bar=True,\n", + ")\n", + "\n", + "for i, state in enumerate(states):\n", + " print_highlight(f\"Answer {i+1}: {states[i]['answer']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming \n", + "\n", + "Use `stream` to stream the output to the user." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def text_qa(s, question):\n", + " s += user(question)\n", + " s += assistant(gen(\"answer\", stop=\"\\n\"))\n", + "\n", + "\n", + "state = text_qa.run(\n", + " question=\"What is the capital of France?\", temperature=0.1, stream=True\n", + ")\n", + "\n", + "for out in state.text_iter():\n", + " print(out, end=\"\", flush=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complex Prompts\n", + "\n", + "You may use `{system|user|assistant}_{begin|end}` to define complex prompts." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def chat_example(s):\n", + " s += system(\"You are a helpful assistant.\")\n", + " # Same as: s += s.system(\"You are a helpful assistant.\")\n", + "\n", + " with s.user():\n", + " s += \"Question: What is the capital of France?\"\n", + "\n", + " s += assistant_begin()\n", + " s += \"Answer: \" + gen(\"answer\", max_tokens=100, stop=\"\\n\")\n", + " s += assistant_end()\n", + "\n", + "\n", + "state = chat_example()\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Multi-modal Generation\n", + "\n", + "You may use SGLang frontend language to define multi-modal prompts.\n", + "See [here](https://docs.sglang.ai/supported_models/generative_models.html) for supported models." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "server_process, port = launch_server_cmd(\n", + " \"python -m sglang.launch_server --model-path Qwen/Qwen2.5-VL-7B-Instruct --host 0.0.0.0 --log-level warning\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(RuntimeEndpoint(f\"http://localhost:{port}\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Ask a question about an image." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def image_qa(s, image_file, question):\n", + " s += user(image(image_file) + question)\n", + " s += assistant(gen(\"answer\", max_tokens=256))\n", + "\n", + "\n", + "image_url = \"https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true\"\n", + "image_bytes, _ = load_image(image_url)\n", + "state = image_qa(image_bytes, \"What is in the image?\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "terminate_process(server_process)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/references/learn_more.md b/docs/references/learn_more.md new file mode 100644 index 000000000..b1a8a17da --- /dev/null +++ b/docs/references/learn_more.md @@ -0,0 +1,7 @@ +# Learn more + +You can find more blogs, slides, and videos about SGLang at [https://github.com/sgl-project/sgl-learning-materials](https://github.com/sgl-project/sgl-learning-materials). + +The latest SGLang features and updates are shared through the [LMSYS blog](https://lmsys.org/blog/). + +The 2025 H2 roadmap can be found at this [issue](https://github.com/sgl-project/sglang/issues/7736). diff --git a/docs/references/multi_node_deployment/deploy_on_k8s.md b/docs/references/multi_node_deployment/deploy_on_k8s.md new file mode 100644 index 000000000..cfc099f56 --- /dev/null +++ b/docs/references/multi_node_deployment/deploy_on_k8s.md @@ -0,0 +1,337 @@ +# Deploy On Kubernetes + +This document is for deploying a RoCE network-based SGLang two-node inference service on a Kubernetes (K8S) cluster. + +[LeaderWorkerSet (LWS)](https://github.com/kubernetes-sigs/lws) is a Kubernetes API that aims to address common deployment patterns of AI/ML inference workloads. A major use case is for multi-host/multi-node distributed inference. + +SGLang can also be deployed with LWS on Kubernetes for distributed model serving. + +Please see this guide for more details on deploying SGLang on Kubernetes using LWS. + +Here we take the deployment of DeepSeek-R1 as an example. + +## Prerequisites + +1. At least two Kubernetes nodes, each with two H20 systems and eight GPUs, are required. + +2. Make sure your K8S cluster has LWS correctly installed. If it hasn't been set up yet, please follow the [installation instructions](https://github.com/kubernetes-sigs/lws/blob/main/site/content/en/docs/installation/_index.md). **Note:** For LWS versions ≤0.5.x, you must use the Downward API to obtain `LWS_WORKER_INDEX`, as native support for this feature was introduced in v0.6.0. + +## Basic example + +For the basic example documentation, refer to [Deploy Distributed Inference Service with SGLang and LWS on GPUs](https://github.com/kubernetes-sigs/lws/tree/main/docs/examples/sglang). + +However, that document only covers the basic NCCL socket mode. + +In this section, we’ll make some simple modifications to adapt the setup to the RDMA scenario. + +## RDMA RoCE case + +* Check your env: + +```bash +[root@node1 ~]# ibstatus +Infiniband device 'mlx5_bond_0' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe64:c79a + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_1' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe6e:c3ec + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_2' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe73:0dd7 + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet + +Infiniband device 'mlx5_bond_3' port 1 status: + default gid: fe80:0000:0000:0000:0225:9dff:fe36:f7ff + base lid: 0x0 + sm lid: 0x0 + state: 4: ACTIVE + phys state: 5: LinkUp + rate: 200 Gb/sec (2X NDR) + link_layer: Ethernet +``` + +* Prepare the `lws.yaml` file for deploying on k8s. + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: sglang +spec: + replicas: 1 + leaderWorkerTemplate: + size: 2 + restartPolicy: RecreateGroupOnPodRestart + leaderTemplate: + metadata: + labels: + role: leader + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-leader + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --host + - "0.0.0.0" + - --port + - "40000" + resources: + limits: + nvidia.com/gpu: "8" + ports: + - containerPort: 40000 + readinessProbe: + tcpSocket: + port: 40000 + initialDelaySeconds: 15 + periodSeconds: 10 + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: model + hostPath: + path: '< your models dir >' # modify it according your models dir + - name: ib + hostPath: + path: /dev/infiniband + workerTemplate: + spec: + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostIPC: true + containers: + - name: sglang-worker + image: sglang:latest + securityContext: + privileged: true + env: + - name: NCCL_IB_GID_INDEX + value: "3" + command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --mem-fraction-static + - "0.93" + - --torch-compile-max-bs + - "8" + - --max-running-requests + - "20" + - --tp + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20000 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + resources: + limits: + nvidia.com/gpu: "8" + volumeMounts: + - mountPath: /dev/shm + name: dshm + - name: model + mountPath: /work/models + - name: ib + mountPath: /dev/infiniband + volumes: + - name: dshm + emptyDir: + medium: Memory + - name: ib + hostPath: + path: /dev/infiniband + - name: model + hostPath: + path: /data1/models/deepseek_v3_moe +--- +apiVersion: v1 +kind: Service +metadata: + name: sglang-leader +spec: + selector: + leaderworkerset.sigs.k8s.io/name: sglang + role: leader + ports: + - protocol: TCP + port: 40000 + targetPort: 40000 + +``` + +* Then use `kubectl apply -f lws.yaml` you will get this output. + +```text +NAME READY STATUS RESTARTS AGE +sglang-0 0/1 Running 0 9s +sglang-0-1 1/1 Running 0 9s +``` + +Wait for the sglang leader (`sglang-0`) status to change to 1/1, which indicates it is `Ready`. + +You can use the command `kubectl logs -f sglang-0` to view the logs of the leader node. + +Once successful, you should see output like this: + +```text +[2025-02-17 05:27:24 TP1] Capture cuda graph end. Time elapsed: 84.89 s +[2025-02-17 05:27:24 TP6] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP0] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP7] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP3] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP2] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP4] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP1] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24 TP5] max_total_num_tokens=712400, chunked_prefill_size=8192, max_prefill_tokens=16384, max_running_requests=50, context_len=163840 +[2025-02-17 05:27:24] INFO: Started server process [1] +[2025-02-17 05:27:24] INFO: Waiting for application startup. +[2025-02-17 05:27:24] INFO: Application startup complete. +[2025-02-17 05:27:24] INFO: Uvicorn running on http://0.0.0.0:40000 (Press CTRL+C to quit) +[2025-02-17 05:27:25] INFO: 127.0.0.1:48908 - "GET /get_model_info HTTP/1.1" 200 OK +[2025-02-17 05:27:25 TP0] Prefill batch. #new-seq: 1, #new-token: 7, #cached-token: 0, cache hit rate: 0.00%, token usage: 0.00, #running-req: 0, #queue-req: 0 +[2025-02-17 05:27:32] INFO: 127.0.0.1:48924 - "POST /generate HTTP/1.1" 200 OK +[2025-02-17 05:27:32] The server is fired up and ready to roll! +``` + +If it doesn’t start up successfully, please follow these steps to check for any remaining issues. Thanks! + +### Debug + +* Set `NCCL_DEBUG=TRACE` to check if it is a NCCL communication problem. + +This should resolve most NCCL-related issues. + +***Notice: If you find that NCCL_DEBUG=TRACE is not effective in the container environment, but the process is stuck or you encounter hard-to-diagnose issues, try switching to a different container image. Some images may not handle standard error output properly.*** + +#### RoCE scenario + +* Please make sure that RDMA devices are available in the cluster environment. +* Please make sure that the nodes in the cluster have Mellanox NICs with RoCE. In this example, we use Mellanox ConnectX 5 model NICs, and the proper OFED driver has been installed. If not, please refer to the document [Install OFED Driver](https://docs.nvidia.com/networking/display/mlnxofedv461000/installing+mellanox+ofed) to install the driver. +* Check your env: + + ```shell + $ lspci -nn | grep Eth | grep Mellanox + 0000:7f:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:7f:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0000:c7:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:08:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.0 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + 0001:a2:00.1 Ethernet controller [0200]: Mellanox Technologies MT43244 BlueField-3 integrated ConnectX-7 network controller [15b3:a2dc] (rev 01) + ``` + +* Check the OFED driver: + + ```shell + ofed_info -s + OFED-internal-23.07-0.5.0: + ``` + +* Show RDMA link status and check IB devices: + + ```shell + $ rdma link show + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + + $ ibdev2netdev + 8/1: mlx5_bond_0/1: state ACTIVE physical_state LINK_UP netdev reth0 + 9/1: mlx5_bond_1/1: state ACTIVE physical_state LINK_UP netdev reth2 + 10/1: mlx5_bond_2/1: state ACTIVE physical_state LINK_UP netdev reth4 + 11/1: mlx5_bond_3/1: state ACTIVE physical_state LINK_UP netdev reth6 + ``` + +* Test RoCE network speed on the host: + + ```shell + yum install qperf + # for server： + execute qperf + # for client + qperf -t 60 -cm1 rc_rdma_write_bw + ``` + +* Check RDMA accessible in your container: + + ```shell + # ibv_devices + # ibv_devinfo + ``` + +## Keys to success + +* In the YAML configuration above, pay attention to the NCCL environment variable. For older versions of NCCL, you should check the NCCL_IB_GID_INDEX environment setting. +* NCCL_SOCKET_IFNAME is also crucial, but in a containerized environment, this typically isn’t an issue. +* In some cases, it’s necessary to configure GLOO_SOCKET_IFNAME correctly. +* NCCL_DEBUG is essential for troubleshooting, but I've found that sometimes it doesn't show error logs within containers. This could be related to the Docker image you're using. You may want to try switching images if needed. +* Avoid using Docker images based on Ubuntu 18.04, as they tend to have compatibility issues. + +## Remaining issues + +* In Kubernetes, Docker, or Containerd environments, we use hostNetwork to prevent performance degradation. +* We utilize privileged mode, which isn’t secure. Additionally, in containerized environments, full GPU isolation cannot be achieved. + +## TODO + +* Integrated with [k8s-rdma-shared-dev-plugin](https://github.com/Mellanox/k8s-rdma-shared-dev-plugin). diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml new file mode 100644 index 000000000..27f98009e --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/d-svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-decode-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml new file mode 100644 index 000000000..ac1d295eb --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/d.yaml @@ -0,0 +1,290 @@ +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-decode-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NVSHMEM_HCA_PE_MAPPING + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_IB_SL + value: "5" + - name: MC_TE_METRIC + value: "true" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NVSHMEM_HCA_PE_MAPPING + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-worker + ports: + - containerPort: 30001 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + networkConfig: + subdomainPolicy: Shared + replicas: 1 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + startupPolicy: LeaderCreated diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml new file mode 100644 index 000000000..4ca690969 --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/lb.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deepseekr10528-lb-main + labels: + app: deepseekr10528-lb +spec: + replicas: 1 + selector: + matchLabels: + app: deepseekr10528-lb + template: + metadata: + labels: + app: deepseekr10528-lb + spec: + nodeSelector: + bo: "yes" + tolerations: + - key: bopd + operator: Exists + - key: node-role + operator: Exists + containers: + - name: sgl-minilb + image: lmsysorg/sglang:latest + command: + - python + - -m + - sglang_router.launch_router + - --pd-disaggregation + - --prefill + - http://deepseekr10528-prefill-main:30000 + - --decode + - http://deepseekr10528-decode-main:30000 + - --host + - 0.0.0.0 + - --port + - "8000" + ports: + - containerPort: 8000 + +--- +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-lb-service +spec: + type: NodePort # NodePort is easy to test, you can also specify `ClusterIP` + selector: + app: deepseekr10528-lb + ports: + - protocol: TCP + port: 8000 # Service Port（In-Cluster） + targetPort: 8000 # Exposed Container + nodePort: 30800 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml new file mode 100644 index 000000000..6826a13df --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/p-svc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-prefill-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 diff --git a/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml b/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml new file mode 100644 index 000000000..62df262bb --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws-examples/p.yaml @@ -0,0 +1,304 @@ +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-prefill-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: NVSHMEM_HCA_PE_MAPPING + # should modify according your rdma env + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "false" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + - mountPath: /root/.cache + name: sgl-cache + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + # - --deepep-config + # - /home/aiges/tuned/tuned_8sms.json + # can be tuned using deepep test scripts + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: NVSHMEM_HCA_PE_MAPPING + # should modify according your rdma env + value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "8" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + value: "0" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:latest + name: sglang-worker + ports: + - containerPort: 30001 + protocol: TCP + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + # should modify according your deployment env + pd: "yes" + tolerations: + # should modify according your deployment env + - key: bopd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + # modify according to you deployment env + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache diff --git a/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md b/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md new file mode 100644 index 000000000..eb8454997 --- /dev/null +++ b/docs/references/multi_node_deployment/lws_pd/lws_pd_deploy.md @@ -0,0 +1,783 @@ +# LWS Based PD Deploy + +## 0. Prerequisites + +1. k8s >=1.26 +2. lws installed on k8s. + +## 1. Image Preparation + +`lmsysorg/sglang:deepep` + +## 2. Deployment Manifest Files + +***Notice: We will package all deployment files into Helm Chart format in the near future. Interested community members can contact us to contribute*** + +### Prefill + +Prefill manifest file [prefill.yaml](lws-examples/p.yaml) + +*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-prefill-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --disaggregation-ib-device + # should modify according your rdma env + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + # - --init-expert-location + # - /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek + # - --deepep-config + # - /home/aiges/tuned/tuned_8sms.json + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: +# - name: NVSHMEM_HCA_PE_MAPPING +# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" +# - name: NVSHMEM_HCA_LIST +# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "false" + - name: NCCL_IB_SL + value: "5" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + - mountPath: /root/.cache + name: sgl-cache + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + # modify according to you deployment env + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --disaggregation-ib-device + - mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3 + - --chunked-prefill-size + - "524288" + - --max-prefill-tokens + - "32768" + - --page-size + - "64" + #- --init-expert-location + #- /home/aiges/tuned/attachment_ep_statistics/prefill_in1024.json + - --ep-dispatch-algorithm + - dynamic + - --eplb-algorithm + - deepseek +# - --deepep-config +# - /home/aiges/tuned/tuned_8sms.json + - --enable-dp-lm-head + - --enable-dp-attention + - --dp-size + - "16" + - --disable-radix-cache + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - prefill + - --mem-fraction-static + - "0.7" + - --context-length + - "32768" + - --tp + - "16" + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + - --max-running-requests + - "1024" + env: + - name: SGLANG_SET_CPU_AFFINITY + value: "true" + - name: SGLANG_HACK_DEEPEP_NUM_SMS + value: "8" + - name: SGLANG_HACK_DEEPEP_NEW_MODE + value: "0" +# - name: NVSHMEM_HCA_PE_MAPPING +# value: "mlx5_bond_0:1:2,mlx5_bond_1:1:2,mlx5_bond_2:1:2,mlx5_bond_3:1:2" +# - name: NVSHMEM_HCA_LIST +# value: "mlx5_bond_0:1,mlx5_bond_1:1,mlx5_bond_2:1,mlx5_bond_3:1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "8" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: SGL_CHUNKED_PREFIX_CACHE_THRESHOLD + value: "0" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: none + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-worker + ports: + - containerPort: 30001 + protocol: TCP + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + - hostPath: + path: /data1/sgl_cache + type: DirectoryOrCreate + name: sgl-cache + +``` + +### Decode + +Decode node deployment manifest file [decode.yaml](lws-examples/d.yaml) + +*Note: The NodeSelector section, model location section, and taint toleration section can be adjusted according to your actual deployment environment* + +```yaml +apiVersion: leaderworkerset.x-k8s.io/v1 +kind: LeaderWorkerSet +metadata: + name: deepseekr10528-decode-main +spec: + leaderWorkerTemplate: + leaderTemplate: + metadata: + labels: + role: leader + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --port + - "30000" + - --host + - "0.0.0.0" + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: CUDA_LAUNCH_BLOCKING + value: "0" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: NCCL_IB_SL + value: "5" + - name: MC_TE_METRIC + value: "true" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-leader + ports: + - containerPort: 30000 + protocol: TCP + readinessProbe: + periodSeconds: 30 + tcpSocket: + port: 30000 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + restartPolicy: RecreateGroupOnPodRestart + size: 2 + workerTemplate: + metadata: {} + spec: + containers: + - command: + - python3 + - -m + - sglang.launch_server + - --model-path + - /work/models + - --chunked-prefill-size + - "262144" + - --page-size + - "64" + - --enable-dp-attention + - --enable-dp-lm-head + #- --enable-two-batch-overlap + - --dp-size + - "16" + - --moe-a2a-backend + - deepep + - --disaggregation-mode + - decode + - --mem-fraction-static + - "0.849" + - --context-length + - "32768" + - --disaggregation-ib-device + # should modify according your rdma env + - "mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3" + - --cuda-graph-max-bs + - "64" + - --max-running-requests + - "2048" + - --tp-size + - "16" # Size of Tensor Parallelism + - --dist-init-addr + - $(LWS_LEADER_ADDRESS):20102 + - --nnodes + - $(LWS_GROUP_SIZE) + - --node-rank + - $(LWS_WORKER_INDEX) + - --trust-remote-code + - --ep-num-redundant-experts + - "32" + - --moe-dense-tp-size + - "1" + env: + - name: SGLANG_HACK_DEEPEP_NUM_SMS + value: "24" + - name: SGLANG_HACK_DEEPEP_NEW_MODE + value: "0" + - name: NVSHMEM_IB_TRAFFIC_CLASS + value: "16" + - name: NVSHMEM_IB_GID_INDEX + value: "3" + - name: NVSHMEM_ENABLE_NIC_PE_MAPPING + value: "1" + - name: NCCL_IB_QPS_PER_CONNECTION + value: "8" + - name: NCCL_IB_SPLIT_DATA_ON_QPS + value: "1" + - name: NCCL_NET_PLUGIN + value: "none" + - name: NCCL_IB_TC + value: "136" + - name: NCCL_MIN_NCHANNELS + value: "4" + - name: MC_TE_METRIC + value: "true" + - name: NCCL_IB_SL + value: "5" + - name: SGLANG_MOONCAKE_TRANS_THREAD + value: "16" + - name: SGL_ENABLE_JIT_DEEPGEMM + value: "1" + - name: NCCL_IB_HCA + value: ^=mlx5_0,mlx5_5,mlx5_6 + - name: LWS_WORKER_INDEX + valueFrom: + fieldRef: + fieldPath: metadata.labels['leaderworkerset.sigs.k8s.io/worker-index'] + image: lmsysorg/sglang:deepep + name: sglang-worker + ports: + - containerPort: 30001 + resources: + limits: + nvidia.com/gpu: "8" + securityContext: + capabilities: + add: + - IPC_LOCK + privileged: true + volumeMounts: + - mountPath: /root/.cache + name: sgl-cache + - mountPath: /dev/shm + name: dshm + - mountPath: /work/models + name: model + - mountPath: /dev/infiniband + name: ib + - mountPath: /sgl-workspace/sglang/python/sglang/srt/layers/moe/fused_moe_triton/configs + name: cf + dnsPolicy: ClusterFirstWithHostNet + hostIPC: true + hostNetwork: true + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + volumes: + - hostPath: + path: /data1/sgl_cache1 + type: DirectoryOrCreate + name: sgl-cache + - emptyDir: + medium: Memory + name: dshm + - hostPath: + path: /dev/infiniband + name: ib + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/DeepSeek-R1-0528/deepseek_r1_0528 + name: model + - hostPath: + # modify according to you deployment env + path: /data1/maas_hosted_models/models/fused_moe_triton/configs + name: cf + networkConfig: + subdomainPolicy: Shared + replicas: 1 + rolloutStrategy: + rollingUpdateConfiguration: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + startupPolicy: LeaderCreated +``` + +Execute separately: + +```bash +kubectl apply -f p.yaml +kubectl apply -f d.yaml +``` + +At this point, we have completed the deployment of the 1P1D SGlang engine part. + +To allow our users to directly experience the model API, we still need a load balancer to handle sequential calls between prefill and decode. Different companies implement LBs differently, and the community will also officially release a new LB component written in Rust in the near future. + +Currently, we use a static K8S service + minilb approach to implement model API calls. + +### Creating Service for Prefill and Decode + +#### Create prefill k8s service +[p-svc.yaml](lws-examples/p-svc.yaml) +```yaml +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-prefill-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-prefill-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 +``` +Execute `kubectl apply -f p-svc.yaml` + +#### Create decode k8s service +[d-svc.yaml](lws-examples/d-svc.yaml) +```yaml +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-decode-main +spec: + selector: + leaderworkerset.sigs.k8s.io/name: deepseekr10528-decode-main + role: leader + ports: + - protocol: TCP + port: 30000 + targetPort: 30000 +``` +Execute `kubectl apply -f d-svc.yaml` + +#### Deploy minilb and lb service +[lb.yaml](lws-examples/lb.yaml) +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deepseekr10528-lb-main + labels: + app: deepseekr10528-lb +spec: + replicas: 1 + selector: + matchLabels: + app: deepseekr10528-lb + template: + metadata: + labels: + app: deepseekr10528-lb + spec: + nodeSelector: + pd: "yes" + tolerations: + - key: pd + operator: Exists + - key: node-role + operator: Exists + containers: + - name: sgl-minilb + image: lmsysorg/sglang:deepep + command: + - python + - -m + - sglang_router.launch_router + - --pd-disaggregation + - --prefill + - http://deepseekr10528-prefill-main:30000 + - --decode + - http://deepseekr10528-decode-main:30000 + - --host + - 0.0.0.0 + - --port + - "8000" + ports: + - containerPort: 8000 +--- +apiVersion: v1 +kind: Service +metadata: + name: deepseekr10528-lb-service +spec: + type: NodePort + selector: + app: deepseekr10528-lb + ports: + - protocol: TCP + port: 8000 # Service Port（In-Cluster） + targetPort: 8000 # Exposed Container + nodePort: 30800 +``` +Execute `kubectl apply -f lb.yaml` + +After waiting for all model deployments to succeed, you will get the following output: + +```bash +[root@ecs-001]# kubectl get po +deepseekr10528-decode-main-0 1/1 Running 0 74m +deepseekr10528-decode-main-0-1 1/1 Running 0 74m +deepseekr10528-lb-main-9c5dbfc57-6lcbd 1/1 Running 0 22m +deepseekr10528-prefill-main-0 1/1 Running 0 74m +deepseekr10528-prefill-main-0-1 1/1 Running 0 74m +[root@ecs-cbm-x1-pd-cpu-001 main_doc]# kubectl get svc |grep dee +deepseekr10528-decode-main ClusterIP None 97m +deepseekr10528-lb-service NodePort 172.16.242.169 8000:30800/TCP 22m +deepseekr10528-prefill-main ClusterIP None 97m +``` + +At this point, select a nodePort:30800 to access: + +```bash +[root@ecs-001]# curl -X POST "http://{nodePort}:30800/v1/chat/completions" \ +> -H "Content-Type: application/json" \ +> -H "Authorization: Bearer None" \ +> -d '{ +> "rid":"ccccdd", +> "model": "r1", +> "messages": [ +> {"role": "system", "content": "0: You are a helpful AI assistant"}, +> {"role": "user", "content": "你是谁？."} +> ], +> "max_tokens":221 +> }' +{"id":"ccccdd","object":"chat.completion","created":1750252498,"model":"qwen2","choices":[{"index":0,"message":{"role":"assistant","content":"\n嗯，用户问了一个很基础的自我介绍问题"你是谁？"。这可能是第一次互动时的常规开场白，也可能是想确认我的身份和功能范围。\n\n用户没有提供任何背景信息，语气简洁中性。这种场景下新用户的可能性较高，需要给出清晰友好的自我介绍，同时突出实用价值来降低陌生感。\n\n考虑到中文用户，应该用简体中文回复。重点要说明三点：身份归属（深度求索）、功能定位（AI助手）、服务范围（学习/工作/生活）。结尾用开放性问题引导对话很关键——既能了解需求，又能避免让用户面对空白输入框时不知所措。\n\n用波浪线结尾可以软化语气，那个笑脸表情😊刚好能中和AI的机械感。不过要控制表情符号数量，避免显得轻浮。\n\n你好呀！我是你的AI助手，由深度求索公司（DeepSeek）开发的语言模型，名字叫 **DeepSeek-R1**。你可以把我当成一个知识丰富、随叫随到的小帮手～😊\n\n我的任务就是陪你聊天、解答问题、","reasoning_content":null,"tool_calls":null},"logprobs":null,"finish_reason":"length","matched_stop":null}],"usage":{"prompt_tokens":14,"total_tokens":235,"completion_tokens":221,"prompt_tokens_details":null}} + +``` +## FAQ + +1. The current deployment startup parameters may not be fully compatible with all RDMA scenarios. Different RDMA NCCL-related environment configurations may be needed in different network environments. + +2. Some preset, optimized configurations for EPLB are not used here. You can adjust them according to [6017](https://github.com/sgl-project/sglang/issues/6017) as needed. diff --git a/docs/references/multi_node_deployment/multi_node.md b/docs/references/multi_node_deployment/multi_node.md new file mode 100644 index 000000000..204b60586 --- /dev/null +++ b/docs/references/multi_node_deployment/multi_node.md @@ -0,0 +1,90 @@ +# Multi-Node Deployment + +## Llama 3.1 405B + +**Run 405B (fp16) on Two Nodes** + +```bash +# replace 172.16.4.52:20000 with your own node ip address and port of the first node + +python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct --tp 16 --dist-init-addr 172.16.4.52:20000 --nnodes 2 --node-rank 0 + +python3 -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct --tp 16 --dist-init-addr 172.16.4.52:20000 --nnodes 2 --node-rank 1 +``` + +Note that LLama 405B (fp8) can also be launched on a single node. + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-405B-Instruct-FP8 --tp 8 +``` + +## DeepSeek V3/R1 + +Please refer to [DeepSeek documents for reference](https://docs.sglang.ai/basic_usage/deepseek.html#running-examples-on-multi-node). + +## Multi-Node Inference on SLURM + +This example showcases how to serve SGLang server across multiple nodes by SLURM. Submit the following job to the SLURM cluster. + +``` +#!/bin/bash -l + +#SBATCH -o SLURM_Logs/%x_%j_master.out +#SBATCH -e SLURM_Logs/%x_%j_master.err +#SBATCH -D ./ +#SBATCH -J Llama-405B-Online-Inference-TP16-SGL + +#SBATCH --nodes=2 +#SBATCH --ntasks=2 +#SBATCH --ntasks-per-node=1 # Ensure 1 task per node +#SBATCH --cpus-per-task=18 +#SBATCH --mem=224GB +#SBATCH --partition="lmsys.org" +#SBATCH --gres=gpu:8 +#SBATCH --time=12:00:00 + +echo "[INFO] Activating environment on node $SLURM_PROCID" +if ! source ENV_FOLDER/bin/activate; then + echo "[ERROR] Failed to activate environment" >&2 + exit 1 +fi + +# Define parameters +model=MODEL_PATH +tp_size=16 + +echo "[INFO] Running inference" +echo "[INFO] Model: $model" +echo "[INFO] TP Size: $tp_size" + +# Set NCCL initialization address using the hostname of the head node +HEAD_NODE=$(scontrol show hostname "$SLURM_NODELIST" | head -n 1) +NCCL_INIT_ADDR="${HEAD_NODE}:8000" +echo "[INFO] NCCL_INIT_ADDR: $NCCL_INIT_ADDR" + +# Launch the model server on each node using SLURM +srun --ntasks=2 --nodes=2 --output="SLURM_Logs/%x_%j_node$SLURM_NODEID.out" \ + --error="SLURM_Logs/%x_%j_node$SLURM_NODEID.err" \ + python3 -m sglang.launch_server \ + --model-path "$model" \ + --grammar-backend "xgrammar" \ + --tp "$tp_size" \ + --dist-init-addr "$NCCL_INIT_ADDR" \ + --nnodes 2 \ + --node-rank "$SLURM_NODEID" & + +# Wait for the NCCL server to be ready on port 30000 +while ! nc -z "$HEAD_NODE" 30000; do + sleep 1 + echo "[INFO] Waiting for $HEAD_NODE:30000 to accept connections" +done + +echo "[INFO] $HEAD_NODE:30000 is ready to accept connections" + +# Keep the script running until the SLURM job times out +wait +``` + +Then, you can test the server by sending requests following other [documents](https://docs.sglang.ai/backend/openai_api_completions.html). + +Thanks for [aflah02](https://github.com/aflah02) for providing the example, based on his [blog post](https://aflah02.substack.com/p/multi-node-llm-inference-with-sglang). diff --git a/docs/references/multi_node_deployment/multi_node_index.rst b/docs/references/multi_node_deployment/multi_node_index.rst new file mode 100644 index 000000000..03411f5be --- /dev/null +++ b/docs/references/multi_node_deployment/multi_node_index.rst @@ -0,0 +1,13 @@ +Multi-Node Deployment +===================== + +.. toctree:: + :maxdepth: 1 + :caption: Multi-Node Deployment + + multi_node.md + deploy_on_k8s.md + lws_pd/lws_pd_deploy.md + +- `Deploying DeepSeek with PD Disaggregation and Large-Scale Expert Parallelism on 96 H100 GPUs `_ +- `Deploying Kimi K2 with PD Disaggregation and Large-Scale Expert Parallelism on 128 H200 GPUs `_ diff --git a/docs/references/production_metrics.md b/docs/references/production_metrics.md new file mode 100644 index 000000000..16afaca67 --- /dev/null +++ b/docs/references/production_metrics.md @@ -0,0 +1,217 @@ +# Production Metrics + +SGLang exposes the following metrics via Prometheus. You can enable it by adding `--enable-metrics` when you launch the server. + +An example of the monitoring dashboard is available in [examples/monitoring/grafana.json](https://github.com/sgl-project/sglang/blob/main/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json). + +Here is an example of the metrics: + +``` +$ curl http://localhost:30000/metrics +# HELP sglang:prompt_tokens_total Number of prefill tokens processed. +# TYPE sglang:prompt_tokens_total counter +sglang:prompt_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 8.128902e+06 +# HELP sglang:generation_tokens_total Number of generation tokens processed. +# TYPE sglang:generation_tokens_total counter +sglang:generation_tokens_total{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.557572e+06 +# HELP sglang:token_usage The token usage +# TYPE sglang:token_usage gauge +sglang:token_usage{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.28 +# HELP sglang:cache_hit_rate The cache hit rate +# TYPE sglang:cache_hit_rate gauge +sglang:cache_hit_rate{model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.007507552643049313 +# HELP sglang:time_to_first_token_seconds Histogram of time to first token in seconds. +# TYPE sglang:time_to_first_token_seconds histogram +sglang:time_to_first_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2.3518979474117756e+06 +sglang:time_to_first_token_seconds_bucket{le="0.001",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:time_to_first_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_to_first_token_seconds_bucket{le="0.06",model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.0 +sglang:time_to_first_token_seconds_bucket{le="0.08",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.25",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:time_to_first_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 27.0 +sglang:time_to_first_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 140.0 +sglang:time_to_first_token_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 314.0 +sglang:time_to_first_token_seconds_bucket{le="7.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 941.0 +sglang:time_to_first_token_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1330.0 +sglang:time_to_first_token_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1970.0 +sglang:time_to_first_token_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2326.0 +sglang:time_to_first_token_seconds_bucket{le="25.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2417.0 +sglang:time_to_first_token_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 2513.0 +sglang:time_to_first_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +sglang:time_to_first_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11008.0 +# HELP sglang:e2e_request_latency_seconds Histogram of End-to-end request latency in seconds +# TYPE sglang:e2e_request_latency_seconds histogram +sglang:e2e_request_latency_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 3.116093850019932e+06 +sglang:e2e_request_latency_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 0.0 +sglang:e2e_request_latency_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="0.8",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="1.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 6.0 +sglang:e2e_request_latency_seconds_bucket{le="5.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.0 +sglang:e2e_request_latency_seconds_bucket{le="10.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 10.0 +sglang:e2e_request_latency_seconds_bucket{le="15.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11.0 +sglang:e2e_request_latency_seconds_bucket{le="20.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 14.0 +sglang:e2e_request_latency_seconds_bucket{le="30.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 247.0 +sglang:e2e_request_latency_seconds_bucket{le="40.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 486.0 +sglang:e2e_request_latency_seconds_bucket{le="50.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 845.0 +sglang:e2e_request_latency_seconds_bucket{le="60.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1513.0 +sglang:e2e_request_latency_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +sglang:e2e_request_latency_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 11228.0 +# HELP sglang:time_per_output_token_seconds Histogram of time per output token in seconds. +# TYPE sglang:time_per_output_token_seconds histogram +sglang:time_per_output_token_seconds_sum{model_name="meta-llama/Llama-3.1-8B-Instruct"} 866964.5791549598 +sglang:time_per_output_token_seconds_bucket{le="0.005",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1.0 +sglang:time_per_output_token_seconds_bucket{le="0.01",model_name="meta-llama/Llama-3.1-8B-Instruct"} 73.0 +sglang:time_per_output_token_seconds_bucket{le="0.015",model_name="meta-llama/Llama-3.1-8B-Instruct"} 382.0 +sglang:time_per_output_token_seconds_bucket{le="0.02",model_name="meta-llama/Llama-3.1-8B-Instruct"} 593.0 +sglang:time_per_output_token_seconds_bucket{le="0.025",model_name="meta-llama/Llama-3.1-8B-Instruct"} 855.0 +sglang:time_per_output_token_seconds_bucket{le="0.03",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1035.0 +sglang:time_per_output_token_seconds_bucket{le="0.04",model_name="meta-llama/Llama-3.1-8B-Instruct"} 1815.0 +sglang:time_per_output_token_seconds_bucket{le="0.05",model_name="meta-llama/Llama-3.1-8B-Instruct"} 11685.0 +sglang:time_per_output_token_seconds_bucket{le="0.075",model_name="meta-llama/Llama-3.1-8B-Instruct"} 433413.0 +sglang:time_per_output_token_seconds_bucket{le="0.1",model_name="meta-llama/Llama-3.1-8B-Instruct"} 4.950195e+06 +sglang:time_per_output_token_seconds_bucket{le="0.15",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.039435e+06 +sglang:time_per_output_token_seconds_bucket{le="0.2",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.171662e+06 +sglang:time_per_output_token_seconds_bucket{le="0.3",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.266055e+06 +sglang:time_per_output_token_seconds_bucket{le="0.4",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.296752e+06 +sglang:time_per_output_token_seconds_bucket{le="0.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.312226e+06 +sglang:time_per_output_token_seconds_bucket{le="0.75",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.339675e+06 +sglang:time_per_output_token_seconds_bucket{le="1.0",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.357747e+06 +sglang:time_per_output_token_seconds_bucket{le="2.5",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.389414e+06 +sglang:time_per_output_token_seconds_bucket{le="+Inf",model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +sglang:time_per_output_token_seconds_count{model_name="meta-llama/Llama-3.1-8B-Instruct"} 7.400757e+06 +# HELP sglang:func_latency_seconds Function latency in seconds +# TYPE sglang:func_latency_seconds histogram +sglang:func_latency_seconds_sum{name="generate_request"} 4.514771912145079 +sglang:func_latency_seconds_bucket{le="0.05",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.07500000000000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.1125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.16875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.253125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.3796875",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.56953125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="0.8542968750000001",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.2814453125",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="1.9221679687500002",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="2.8832519531250003",name="generate_request"} 14006.0 +sglang:func_latency_seconds_bucket{le="4.3248779296875",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="6.487316894531251",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="9.730975341796876",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="14.596463012695313",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="21.89469451904297",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="32.84204177856446",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="49.26306266784668",name="generate_request"} 14007.0 +sglang:func_latency_seconds_bucket{le="+Inf",name="generate_request"} 14007.0 +sglang:func_latency_seconds_count{name="generate_request"} 14007.0 +# HELP sglang:num_running_reqs The number of running requests +# TYPE sglang:num_running_reqs gauge +sglang:num_running_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 162.0 +# HELP sglang:num_used_tokens The number of used tokens +# TYPE sglang:num_used_tokens gauge +sglang:num_used_tokens{model_name="meta-llama/Llama-3.1-8B-Instruct"} 123859.0 +# HELP sglang:gen_throughput The generate throughput (token/s) +# TYPE sglang:gen_throughput gauge +sglang:gen_throughput{model_name="meta-llama/Llama-3.1-8B-Instruct"} 86.50814177726902 +# HELP sglang:num_queue_reqs The number of requests in the waiting queue +# TYPE sglang:num_queue_reqs gauge +sglang:num_queue_reqs{model_name="meta-llama/Llama-3.1-8B-Instruct"} 2826.0 +``` + +## Setup Guide + +This section describes how to set up the monitoring stack (Prometheus + Grafana) provided in the `examples/monitoring` directory. + +### Prerequisites + +- Docker and Docker Compose installed +- SGLang server running with metrics enabled + +### Usage + +1. **Start your SGLang server with metrics enabled:** + + ```bash + python -m sglang.launch_server --model-path --port 30000 --enable-metrics + ``` + Replace `` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://:30000/metrics`. + +2. **Navigate to the monitoring example directory:** + ```bash + cd examples/monitoring + ``` + +3. **Start the monitoring stack:** + ```bash + docker compose up -d + ``` + This command will start Prometheus and Grafana in the background. + +4. **Access the monitoring interfaces:** + * **Grafana:** Open your web browser and go to [http://localhost:3000](http://localhost:3000). + * **Prometheus:** Open your web browser and go to [http://localhost:9090](http://localhost:9090). + +5. **Log in to Grafana:** + * Default Username: `admin` + * Default Password: `admin` + You will be prompted to change the password upon your first login. + +6. **View the Dashboard:** + The SGLang dashboard is pre-configured and should be available automatically. Navigate to `Dashboards` -> `Browse` -> `SGLang Monitoring` folder -> `SGLang Dashboard`. + +### Troubleshooting + +* **Port Conflicts:** If you encounter errors like "port is already allocated," check if other services (including previous instances of Prometheus/Grafana) are using ports `9090` or `3000`. Use `docker ps` to find running containers and `docker stop ` to stop them, or use `lsof -i :` to find other processes using the ports. You might need to adjust the ports in the `docker-compose.yaml` file if they permanently conflict with other essential services on your system. + +To modify Grafana's port to the other one(like 3090) in your Docker Compose file, you need to explicitly specify the port mapping under the grafana service. + + Option 1: Add GF_SERVER_HTTP_PORT to the environment section: + ``` + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_SERVER_HTTP_PORT=3090 # <-- Add this line + ``` + Option 2: Use port mapping: + ``` + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3090:3000" # <-- Host:Container port mapping + ``` +* **Connection Issues:** + * Ensure both Prometheus and Grafana containers are running (`docker ps`). + * Verify the Prometheus data source configuration in Grafana (usually auto-configured via `grafana/datasources/datasource.yaml`). Go to `Connections` -> `Data sources` -> `Prometheus`. The URL should point to the Prometheus service (e.g., `http://prometheus:9090`). + * Confirm that your SGLang server is running and the metrics endpoint (`http://:30000/metrics`) is accessible *from the Prometheus container*. If SGLang is running on your host machine and Prometheus is in Docker, use `host.docker.internal` (on Docker Desktop) or your machine's network IP instead of `localhost` in the `prometheus.yaml` scrape configuration. +* **No Data on Dashboard:** + * Generate some traffic to your SGLang server to produce metrics. For example, run a benchmark: + ```bash + python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 100 --random-input 128 --random-output 128 + ``` + * Check the Prometheus UI (`http://localhost:9090`) under `Status` -> `Targets` to see if the SGLang endpoint is being scraped successfully. + * Verify the `model_name` and `instance` labels in your Prometheus metrics match the variables used in the Grafana dashboard. You might need to adjust the Grafana dashboard variables or the labels in your Prometheus configuration. + +### Configuration Files + +The monitoring setup is defined by the following files within the `examples/monitoring` directory: + +* `docker-compose.yaml`: Defines the Prometheus and Grafana services. +* `prometheus.yaml`: Prometheus configuration, including scrape targets. +* `grafana/datasources/datasource.yaml`: Configures the Prometheus data source for Grafana. +* `grafana/dashboards/config/dashboard.yaml`: Tells Grafana to load dashboards from the specified path. +* `grafana/dashboards/json/sglang-dashboard.json`: The actual Grafana dashboard definition in JSON format. + +You can customize the setup by modifying these files. For instance, you might need to update the `static_configs` target in `prometheus.yaml` if your SGLang server runs on a different host or port. + +#### Check if the metrics are being collected + +Run `python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5` to generate some requests. + +Then you should be able to see the metrics in the Grafana dashboard. diff --git a/docs/references/torch_compile_cache.md b/docs/references/torch_compile_cache.md new file mode 100644 index 000000000..f2bb257f4 --- /dev/null +++ b/docs/references/torch_compile_cache.md @@ -0,0 +1,13 @@ +# Enabling cache for torch.compile + +SGLang uses `max-autotune-no-cudagraphs` mode of torch.compile. The auto-tuning can be slow. +If you want to deploy a model on many different machines, you can ship the torch.compile cache to these machines and skip the compilation steps. + +This is based on https://pytorch.org/tutorials/recipes/torch_compile_caching_tutorial.html + + +1. Generate the cache by setting TORCHINDUCTOR_CACHE_DIR and running the model once. +``` +TORCHINDUCTOR_CACHE_DIR=/root/inductor_root_cache python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B-Instruct --enable-torch-compile +``` +2. Copy the cache folder to other machines and launch the server with `TORCHINDUCTOR_CACHE_DIR`. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..1a7e5d4eb --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,20 @@ +ipykernel +ipywidgets +jupyter_client +markdown>=3.4.0 +matplotlib +myst-parser +nbconvert +nbsphinx +pandoc +pillow +pydantic +sphinx +sphinx-book-theme +sphinx-copybutton +sphinx-tabs +nbstripout +sphinxcontrib-mermaid +urllib3<2.0.0 +gguf>=0.10.0 +sphinx-autobuild diff --git a/docs/serve.sh b/docs/serve.sh new file mode 100644 index 000000000..049f767cf --- /dev/null +++ b/docs/serve.sh @@ -0,0 +1,3 @@ +# Clean and serve documentation with auto-build +make clean +make serve diff --git a/docs/supported_models/embedding_models.md b/docs/supported_models/embedding_models.md new file mode 100644 index 000000000..437cb8284 --- /dev/null +++ b/docs/supported_models/embedding_models.md @@ -0,0 +1,87 @@ +# Embedding Models + +SGLang provides robust support for embedding models by integrating efficient serving mechanisms with its flexible programming interface. This integration allows for streamlined handling of embedding tasks, facilitating faster and more accurate retrieval and semantic search operations. SGLang's architecture enables better resource utilization and reduced latency in embedding model deployment. + +```{important} +Embedding models are executed with `--is-embedding` flag and some may require `--trust-remote-code` +``` + +## Quick Start + +### Launch Server + +```shell +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen3-Embedding-4B \ + --is-embedding \ + --host 0.0.0.0 \ + --port 30000 +``` + +### Client Request + +```python +import requests + +url = "http://127.0.0.1:30000" + +payload = { + "model": "Qwen/Qwen3-Embedding-4B", + "input": "What is the capital of France?", + "encoding_format": "float" +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() +print("Embedding:", response["data"][0]["embedding"]) +``` + + + +## Multimodal Embedding Example + +For multimodal models like GME that support both text and images: + +```shell +python3 -m sglang.launch_server \ + --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct \ + --is-embedding \ + --chat-template gme-qwen2-vl \ + --host 0.0.0.0 \ + --port 30000 +``` + +```python +import requests + +url = "http://127.0.0.1:30000" + +text_input = "Represent this image in embedding space." +image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + +payload = { + "model": "gme-qwen2-vl", + "input": [ + { + "text": text_input + }, + { + "image": image_path + } + ], +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() + +print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) +``` + +## Supported Models + +| Model Family | Example Model | Chat Template | Description | +| ------------------------------------------ | -------------------------------------- | ------------- | --------------------------------------------------------------------------- | +| **E5 (Llama/Mistral based)** | `intfloat/e5-mistral-7b-instruct` | N/A | High-quality text embeddings based on Mistral/Llama architectures | +| **GTE-Qwen2** | `Alibaba-NLP/gte-Qwen2-7B-instruct` | N/A | Alibaba's text embedding model with multilingual support | +| **Qwen3-Embedding** | `Qwen/Qwen3-Embedding-4B` | N/A | Latest Qwen3-based text embedding model for semantic representation | +| **BGE** | `BAAI/bge-large-en-v1.5` | N/A | BAAI's text embeddings (requires `attention-backend` triton/torch_native) | +| **GME (Multimodal)** | `Alibaba-NLP/gme-Qwen2-VL-2B-Instruct`| `gme-qwen2-vl`| Multimodal embedding for text and image cross-modal tasks | +| **CLIP** | `openai/clip-vit-large-patch14-336` | N/A | OpenAI's CLIP for image and text embeddings | diff --git a/docs/supported_models/generative_models.md b/docs/supported_models/generative_models.md new file mode 100644 index 000000000..d6d3cdd45 --- /dev/null +++ b/docs/supported_models/generative_models.md @@ -0,0 +1,56 @@ +# Large Language Models + +These models accept text input and produce text output (e.g., chat completions). They are primarily large language models (LLMs), some with mixture-of-experts (MoE) architectures for scaling. + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-1B-Instruct \ # example HF/local path + --host 0.0.0.0 \ + --port 30000 \ +``` + +## Supported models + +Below the supported models are summarized in a table. + +If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen3ForCausalLM`, use the expression: + +``` +repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen3ForCausalLM +``` + +in the GitHub search bar. + +| Model Family (Variants) | Example HuggingFace Identifier | Description | +|-------------------------------------|--------------------------------------------------|----------------------------------------------------------------------------------------| +| **DeepSeek** (v1, v2, v3/R1) | `deepseek-ai/DeepSeek-R1` | Series of advanced reasoning-optimized models (including a 671B MoE) trained with reinforcement learning; top performance on complex reasoning, math, and code tasks. [SGLang provides Deepseek v3/R1 model-specific optimizations](../basic_usage/deepseek.md) and [Reasoning Parser](../advanced_features/separate_reasoning.ipynb)| +| **GPT-OSS** | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | OpenAI’s latest GPT-OSS series for complex reasoning, agentic tasks, and versatile developer use cases.| +| **Qwen** (3, 3MoE, 3Next, 2.5, 2 series) | `Qwen/Qwen3-0.6B`, `Qwen/Qwen3-30B-A3B` `Qwen/Qwen3-Next-80B-A3B-Instruct ` | Alibaba’s latest Qwen3 series for complex reasoning, language understanding, and generation tasks; Support for MoE variants along with previous generation 2.5, 2, etc. [SGLang provides Qwen3 specific reasoning parser](../advanced_features/separate_reasoning.ipynb)| +| **Llama** (2, 3.x, 4 series) | `meta-llama/Llama-4-Scout-17B-16E-Instruct` | Meta's open LLM series, spanning 7B to 400B parameters (Llama 2, 3, and new Llama 4) with well-recognized performance. [SGLang provides Llama-4 model-specific optimizations](../basic_usage/llama4.md) | +| **Mistral** (Mixtral, NeMo, Small3) | `mistralai/Mistral-7B-Instruct-v0.2` | Open 7B LLM by Mistral AI with strong performance; extended into MoE (“Mixtral”) and NeMo Megatron variants for larger scale. | +| **Gemma** (v1, v2, v3) | `google/gemma-3-1b-it` | Google’s family of efficient multilingual models (1B–27B); Gemma 3 offers a 128K context window, and its larger (4B+) variants support vision input. | +| **Phi** (Phi-1.5, Phi-2, Phi-3, Phi-4, Phi-MoE series) | `microsoft/Phi-4-multimodal-instruct`, `microsoft/Phi-3.5-MoE-instruct` | Microsoft’s Phi family of small models (1.3B–5.6B); Phi-4-multimodal (5.6B) processes text, images, and speech, Phi-4-mini is a high-accuracy text model and Phi-3.5-MoE is a mixture-of-experts model. | +| **MiniCPM** (v3, 4B) | `openbmb/MiniCPM3-4B` | OpenBMB’s series of compact LLMs for edge devices; MiniCPM 3 (4B) achieves GPT-3.5-level results in text tasks. | +| **OLMoE** (Open MoE) | `allenai/OLMoE-1B-7B-0924` | Allen AI’s open Mixture-of-Experts model (7B total, 1B active parameters) delivering state-of-the-art results with sparse expert activation. | +| **StableLM** (3B, 7B) | `stabilityai/stablelm-tuned-alpha-7b` | StabilityAI’s early open-source LLM (3B & 7B) for general text generation; a demonstration model with basic instruction-following ability. | +| **Command-R** (Cohere) | `CohereForAI/c4ai-command-r-v01` | Cohere’s open conversational LLM (Command series) optimized for long context, retrieval-augmented generation, and tool use. | +| **DBRX** (Databricks) | `databricks/dbrx-instruct` | Databricks’ 132B-parameter MoE model (36B active) trained on 12T tokens; competes with GPT-3.5 quality as a fully open foundation model. | +| **Grok** (xAI) | `xai-org/grok-1` | xAI’s grok-1 model known for vast size(314B parameters) and high quality; integrated in SGLang for high-performance inference. | +| **ChatGLM** (GLM-130B family) | `THUDM/chatglm2-6b` | Zhipu AI’s bilingual chat model (6B) excelling at Chinese-English dialogue; fine-tuned for conversational quality and alignment. | +| **InternLM 2** (7B, 20B) | `internlm/internlm2-7b` | Next-gen InternLM (7B and 20B) from SenseTime, offering strong reasoning and ultra-long context support (up to 200K tokens). | +| **ExaONE 3** (Korean-English) | `LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct` | LG AI Research’s Korean-English model (7.8B) trained on 8T tokens; provides high-quality bilingual understanding and generation. | +| **Baichuan 2** (7B, 13B) | `baichuan-inc/Baichuan2-13B-Chat` | BaichuanAI’s second-generation Chinese-English LLM (7B/13B) with improved performance and an open commercial license. | +| **XVERSE** (MoE) | `xverse/XVERSE-MoE-A36B` | Yuanxiang’s open MoE LLM (XVERSE-MoE-A36B: 255B total, 36B active) supporting ~40 languages; delivers 100B+ dense-level performance via expert routing. | +| **SmolLM** (135M–1.7B) | `HuggingFaceTB/SmolLM-1.7B` | Hugging Face’s ultra-small LLM series (135M–1.7B params) offering surprisingly strong results, enabling advanced AI on mobile/edge devices. | +| **GLM-4** (Multilingual 9B) | `ZhipuAI/glm-4-9b-chat` | Zhipu’s GLM-4 series (up to 9B parameters) – open multilingual models with support for 1M-token context and even a 5.6B multimodal variant (Phi-4V). | +| **MiMo** (7B series) | `XiaomiMiMo/MiMo-7B-RL` | Xiaomi's reasoning-optimized model series, leverages Multiple-Token Prediction for faster inference. | +| **ERNIE-4.5** (4.5, 4.5MoE series) | `baidu/ERNIE-4.5-21B-A3B-PT` | Baidu's ERNIE-4.5 series which consists of MoE with 47B and 3B active parameters, with the largest model having 424B total parameters, as well as a 0.3B dense model. | +| **Arcee AFM-4.5B** | `arcee-ai/AFM-4.5B-Base` | Arcee's foundational model series for real world reliability and edge deployments. | +| **Persimmon** (8B) | `adept/persimmon-8b-chat` | Adept’s open 8B model with a 16K context window and fast inference; trained for broad usability and licensed under Apache 2.0. | +| **Ling** (16.8B–290B) | `inclusionAI/Ling-lite`, `inclusionAI/Ling-plus` | InclusionAI’s open MoE models. Ling-Lite has 16.8B total / 2.75B active parameters, and Ling-Plus has 290B total / 28.8B active parameters. They are designed for high performance on NLP and complex reasoning tasks. | +| **Granite 3.0, 3.1** (IBM) | `ibm-granite/granite-3.1-8b-instruct` | IBM's open dense foundation models optimized for reasoning, code, and business AI use cases. Integrated with Red Hat and watsonx systems. | +| **Granite 3.0 MoE** (IBM) | `ibm-granite/granite-3.0-3b-a800m-instruct` | IBM’s Mixture-of-Experts models offering strong performance with cost-efficiency. MoE expert routing designed for enterprise deployment at scale. | +| **Llama Nemotron Super** (v1, v1.5, NVIDIA) | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, `nvidia/Llama-3_3-Nemotron-Super-49B-v1_5` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family builds on the strongest open models in the ecosystem by enhancing them with greater accuracy, efficiency, and transparency using NVIDIA open synthetic datasets, advanced techniques, and tools. This enables the creation of practical, right-sized, and high-performing AI agents. | +| **Llama Nemotron Ultra** (v1, NVIDIA) | `nvidia/Llama-3_1-Nemotron-Ultra-253B-v1` | The [NVIDIA Nemotron](https://www.nvidia.com/en-us/ai-data-science/foundation-models/nemotron/) family builds on the strongest open models in the ecosystem by enhancing them with greater accuracy, efficiency, and transparency using NVIDIA open synthetic datasets, advanced techniques, and tools. This enables the creation of practical, right-sized, and high-performing AI agents. | diff --git a/docs/supported_models/modelscope.md b/docs/supported_models/modelscope.md new file mode 100644 index 000000000..4740c2770 --- /dev/null +++ b/docs/supported_models/modelscope.md @@ -0,0 +1,28 @@ +# Use Models From ModelScope + +To use a model from [ModelScope](https://www.modelscope.cn), set the environment variable `SGLANG_USE_MODELSCOPE`. + +```bash +export SGLANG_USE_MODELSCOPE=true +``` + +We take [Qwen2-7B-Instruct](https://www.modelscope.cn/models/qwen/qwen2-7b-instruct) as an example. + +Launch the Server: +```bash +python -m sglang.launch_server --model-path qwen/Qwen2-7B-Instruct --port 30000 +``` + +Or start it by docker: + +```bash +docker run --gpus all \ + -p 30000:30000 \ + -v ~/.cache/modelscope:/root/.cache/modelscope \ + --env "SGLANG_USE_MODELSCOPE=true" \ + --ipc=host \ + lmsysorg/sglang:latest \ + python3 -m sglang.launch_server --model-path Qwen/Qwen2.5-7B-Instruct --host 0.0.0.0 --port 30000 +``` + +Note that modelscope uses a different cache directory than huggingface. You may need to set it manually to avoid running out of disk space. diff --git a/docs/supported_models/multimodal_language_models.md b/docs/supported_models/multimodal_language_models.md new file mode 100644 index 000000000..a2adf99cb --- /dev/null +++ b/docs/supported_models/multimodal_language_models.md @@ -0,0 +1,42 @@ +# Multimodal Language Models + +These models accept multi-modal inputs (e.g., images and text) and generate text output. They augment language models with multimodal encoders. + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-11B-Vision-Instruct \ # example HF/local path + --host 0.0.0.0 \ + --port 30000 \ +``` + +## Supported models + +Below the supported models are summarized in a table. + +If you are unsure if a specific architecture is implemented, you can search for it via GitHub. For example, to search for `Qwen2_5_VLForConditionalGeneration`, use the expression: + +``` +repo:sgl-project/sglang path:/^python\/sglang\/srt\/models\// Qwen2_5_VLForConditionalGeneration +``` + +in the GitHub search bar. + + +| Model Family (Variants) | Example HuggingFace Identifier | Chat Template | Description | +|----------------------------|--------------------------------------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Qwen-VL** (Qwen2 series) | `Qwen/Qwen2.5-VL-7B-Instruct` | `qwen2-vl` | Alibaba’s vision-language extension of Qwen; for example, Qwen2.5-VL (7B and larger variants) can analyze and converse about image content. | +| **DeepSeek-VL2** | `deepseek-ai/deepseek-vl2` | `deepseek-vl2` | Vision-language variant of DeepSeek (with a dedicated image processor), enabling advanced multimodal reasoning on image and text inputs. | +| **Janus-Pro** (1B, 7B) | `deepseek-ai/Janus-Pro-7B` | `janus-pro` | DeepSeek’s open-source multimodal model capable of both image understanding and generation. Janus-Pro employs a decoupled architecture for separate visual encoding paths, enhancing performance in both tasks. | +| **MiniCPM-V / MiniCPM-o** | `openbmb/MiniCPM-V-2_6` | `minicpmv` | MiniCPM-V (2.6, ~8B) supports image inputs, and MiniCPM-o adds audio/video; these multimodal LLMs are optimized for end-side deployment on mobile/edge devices. | +| **Llama 3.2 Vision** (11B) | `meta-llama/Llama-3.2-11B-Vision-Instruct` | `llama_3_vision` | Vision-enabled variant of Llama 3 (11B) that accepts image inputs for visual question answering and other multimodal tasks. | +| **LLaVA** (v1.5 & v1.6) | *e.g.* `liuhaotian/llava-v1.5-13b` | `vicuna_v1.1` | Open vision-chat models that add an image encoder to LLaMA/Vicuna (e.g. LLaMA2 13B) for following multimodal instruction prompts. | +| **LLaVA-NeXT** (8B, 72B) | `lmms-lab/llava-next-72b` | `chatml-llava` | Improved LLaVA models (with an 8B Llama3 version and a 72B version) offering enhanced visual instruction-following and accuracy on multimodal benchmarks. | +| **LLaVA-OneVision** | `lmms-lab/llava-onevision-qwen2-7b-ov` | `chatml-llava` | Enhanced LLaVA variant integrating Qwen as the backbone; supports multiple images (and even video frames) as inputs via an OpenAI Vision API-compatible format. | +| **Gemma 3 (Multimodal)** | `google/gemma-3-4b-it` | `gemma-it` | Gemma 3's larger models (4B, 12B, 27B) accept images (each image encoded as 256 tokens) alongside text in a combined 128K-token context. | +| **Kimi-VL** (A3B) | `moonshotai/Kimi-VL-A3B-Instruct` | `kimi-vl` | Kimi-VL is a multimodal model that can understand and generate text from images. | +| **Mistral-Small-3.1-24B** | `mistralai/Mistral-Small-3.1-24B-Instruct-2503` | `mistral` | Mistral 3.1 is a multimodal model that can generate text from text or images input. It also supports tool calling and structured output. | +| **Phi-4-multimodal-instruct** | `microsoft/Phi-4-multimodal-instruct` | `phi-4-mm` | Phi-4-multimodal-instruct is the multimodal variant of the Phi-4-mini model, enhanced with LoRA for improved multimodal capabilities. It supports text, vision and audio modalities in SGLang. | +| **MiMo-VL** (7B) | `XiaomiMiMo/MiMo-VL-7B-RL` | `mimo-vl` | Xiaomi's compact yet powerful vision-language model featuring a native resolution ViT encoder for fine-grained visual details, an MLP projector for cross-modal alignment, and the MiMo-7B language model optimized for complex reasoning tasks. | +| **GLM-4.5V** (106B) / **GLM-4.1V**(9B) | `zai-org/GLM-4.5V` | `glm-4v` | GLM-4.5V and GLM-4.1V-Thinking: Towards Versatile Multimodal Reasoning with Scalable Reinforcement Learning | diff --git a/docs/supported_models/rerank_models.md b/docs/supported_models/rerank_models.md new file mode 100644 index 000000000..b6f2ffa20 --- /dev/null +++ b/docs/supported_models/rerank_models.md @@ -0,0 +1,49 @@ +# Rerank Models + +SGLang offers comprehensive support for rerank models by incorporating optimized serving frameworks with a flexible programming interface. This setup enables efficient processing of cross-encoder reranking tasks, improving the accuracy and relevance of search result ordering. SGLang’s design ensures high throughput and low latency during reranker model deployment, making it ideal for semantic-based result refinement in large-scale retrieval systems. + +```{important} +They are executed with `--is-embedding` and some may require `--trust-remote-code` +``` + +## Example Launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path BAAI/bge-reranker-v2-m3 \ + --host 0.0.0.0 \ + --disable-radix-cache \ + --chunked-prefill-size -1 \ + --attention-backend triton \ + --is-embedding \ + --port 30000 +``` + +## Example Client Request + +```python +import requests + +url = "http://127.0.0.1:30000/v1/rerank" + +payload = { + "model": "BAAI/bge-reranker-v2-m3", + "query": "what is panda?", + "documents": [ + "hi", + "The giant panda (Ailuropoda melanoleuca), sometimes called a panda bear or simply panda, is a bear species endemic to China." + ] +} + +response = requests.post(url, json=payload) +response_json = response.json() + +for item in response_json: + print(f"Score: {item['score']:.2f} - Document: '{item['document']}'") +``` + +## Supported rerank models + +| Model Family (Rerank) | Example HuggingFace Identifier | Chat Template | Description | +|------------------------------------------------|--------------------------------------|---------------|----------------------------------------------------------------------------------------------------------------------------------| +| **BGE-Reranker (BgeRerankModel)** | `BAAI/bge-reranker-v2-m3` | N/A | Currently only support `attention-backend` `triton` and `torch_native`. high-performance cross-encoder reranker model from BAAI. Suitable for reranking search results based on semantic relevance. | diff --git a/docs/supported_models/reward_models.md b/docs/supported_models/reward_models.md new file mode 100644 index 000000000..ef4474637 --- /dev/null +++ b/docs/supported_models/reward_models.md @@ -0,0 +1,28 @@ +# Reward Models + +These models output a scalar reward score or classification result, often used in reinforcement learning or content moderation tasks. + +```{important} +They are executed with `--is-embedding` and some may require `--trust-remote-code`. +``` + +## Example launch Command + +```shell +python3 -m sglang.launch_server \ + --model-path Qwen/Qwen2.5-Math-RM-72B \ # example HF/local path + --is-embedding \ + --host 0.0.0.0 \ + --tp-size=4 \ # set for tensor parallelism + --port 30000 \ +``` + +## Supported models + +| Model Family (Reward) | Example HuggingFace Identifier | Description | +|---------------------------------------------------------------------------|-----------------------------------------------------|---------------------------------------------------------------------------------| +| **Llama (3.1 Reward / `LlamaForSequenceClassification`)** | `Skywork/Skywork-Reward-Llama-3.1-8B-v0.2` | Reward model (preference classifier) based on Llama 3.1 (8B) for scoring and ranking responses for RLHF. | +| **Gemma 2 (27B Reward / `Gemma2ForSequenceClassification`)** | `Skywork/Skywork-Reward-Gemma-2-27B-v0.2` | Derived from Gemma‑2 (27B), this model provides human preference scoring for RLHF and multilingual tasks. | +| **InternLM 2 (Reward / `InternLM2ForRewardMode`)** | `internlm/internlm2-7b-reward` | InternLM 2 (7B)–based reward model used in alignment pipelines to guide outputs toward preferred behavior. | +| **Qwen2.5 (Reward - Math / `Qwen2ForRewardModel`)** | `Qwen/Qwen2.5-Math-RM-72B` | A 72B math-specialized RLHF reward model from the Qwen2.5 series, tuned for evaluating and refining responses. | +| **Qwen2.5 (Reward - Sequence / `Qwen2ForSequenceClassification`)** | `jason9693/Qwen2.5-1.5B-apeach` | A smaller Qwen2.5 variant used for sequence classification, offering an alternative RLHF scoring mechanism. | diff --git a/docs/supported_models/support_new_models.md b/docs/supported_models/support_new_models.md new file mode 100644 index 000000000..511a8f398 --- /dev/null +++ b/docs/supported_models/support_new_models.md @@ -0,0 +1,320 @@ +# How to Support New Models + +This document explains how to add support for new language models and multimodal large language models (MLLMs) in +SGLang. It also covers how to test new models and register external implementations. + +## How to Support a New Language Model + +To support a new model in SGLang, you only need to add a single file under +the [SGLang Models Directory](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/models). You can learn +from existing model implementations and create a new file for your model. For most models, you should be able to find a +similar model to start with (e.g., starting from Llama). Also refer how +to [port a Model from vLLM to SGLang](#port-a-model-from-vllm-to-sglang) + +## How to Support a New Multimodal Large Language Model + +To support a new multimodal large language model (MLLM) in SGLang, there are several key components in addition to the +standard LLM support: + +1. **Register your new model as multimodal**: + Extend `is_multimodal_model` + in [model_config.py](https://github.com/sgl-project/sglang/blob/0ab3f437aba729b348a683ab32b35b214456efc7/python/sglang/srt/configs/model_config.py#L561) + to return `True` for your model. + +2. **Register a new chat-template**: + Only when your default chat-template is unable to accept images as input: Register a new chat template in [conversation.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/conversation.py) and the corresponding matching function. + +3. **Multimodal Data Processor**: + Define a new `Processor` class that inherits from `BaseMultimodalProcessor` and register this processor as your + model’s dedicated processor. + See [multimodal_processor.py](https://github.com/sgl-project/sglang/tree/main/python/sglang/srt/multimodal/processors) + for more details. + +4. **Handle Multimodal Tokens**: + Implement a `pad_input_ids` function for your new model. In this function, multimodal tokens in the prompt should be + expanded (if necessary) and padded with multimodal-data-hashes so that SGLang can recognize different multimodal data + with `RadixAttention`. + +5. **Handle Image Feature Extraction**: + Implement a `get_image_feature` function for your new model, which extracts image features from raw image data and converts them into the embeddings used by the language model. + +6. **Adapt to Vision Attention**: + Adapt the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`. + +You can refer to [Qwen2VL](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/qwen2_vl.py) or +other mllm implementations. These models demonstrate how to correctly handle both multimodal and textual inputs. + +## Testing and Debugging + +Please note all your testing and benchmarking results in PR description. + +### Interactive Debugging + +For interactive debugging, compare the outputs of Hugging Face/Transformers and SGLang. The following two commands +should give the same text output and very similar prefill logits: + +- Get the reference output: + ```bash + python3 scripts/playground/reference_hf.py --model-path [new model] --model-type {text,mllm} + ``` +- Get the SGLang output: + ```bash + python3 -m sglang.bench_one_batch --correct --model [new model] + ``` + +### Add the Model to the Test Suite + +To ensure the new model is well maintained, add it to the test suite by including it in the `ALL_OTHER_MODELS` list in +the [test_generation_models.py](https://github.com/sgl-project/sglang/blob/main/test/srt/models/test_generation_models.py) +file, test the new model on your local machine and report the results on demonstrative benchmarks (GSM8K, MMLU, MMMU, +MMMU-Pro, etc.) in your PR. \\ +For VLMs, also include a test in `test_vision_openai_server_{x}.py` (e.g. [test_vision_openai_server_a.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_a.py), [test_vision_openai_server_b.py](https://github.com/sgl-project/sglang/blob/main/test/srt/test_vision_openai_server_b.py)). + + +This is an example command to run to test a new model on your local machine: + +```bash +ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels.test_others +``` + +### Benchmark + +- **(Required) MMMU**: follow MMMU benchmark [README.md](https://github.com/sgl-project/sglang/blob/main/benchmark/mmmu/README.md) to get SGLang vs. HF Transformer accuracy comparison. The accuracy score from SGLang run should not be much lower than that from HF Transformer run. Similarly, follow https://docs.sglang.ai/developer_guide/benchmark_and_profiling.html to get performance comparison: TTFT and throughput must meet or exceed baselines (e.g., HF Transformer). +- **(Optional) Other evals**: If you ran other evals, please note the results in PR description. + +## Port a Model from vLLM to SGLang + +The [vLLM Models Directory](https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/models) is a valuable +resource, as vLLM covers many models. SGLang reuses vLLM’s interface and some layers, making it easier to port models +from vLLM to SGLang. + +To port a model from vLLM to SGLang: + +- Compare these two files for guidance: + - [SGLang Llama Implementation](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/llama.py) + - [vLLM Llama Implementation](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/llama.py) +- The major differences include: + - **Replace vLLM’s `Attention` with `RadixAttention`** (ensure you pass `layer_id` to `RadixAttention`). + - **Replace vLLM’s `LogitsProcessor` with SGLang’s `LogitsProcessor`.** + - **Replace the multi-headed `Attention` of ViT with SGLang’s `VisionAttention`.** + - **Replace other vLLM layers** (such as `RMSNorm`, `SiluAndMul`) with SGLang layers. + - **Remove `Sample`.** + - **Change the `forward()` functions** and add a `forward_batch()` method. + - **Add `EntryClass`** at the end. + - **Ensure that the new implementation uses only SGLang components** and does not rely on any vLLM components. + +Note: make sure you add your new model to the supported models list in the supported models documentation. + +## Registering an External Model Implementation + +In addition to the methods above, you can register your new model with the `ModelRegistry` before launching the server. +This allows you to integrate your model without modifying the source code. + +For example: + +```python +from sglang.srt.models.registry import ModelRegistry +from sglang.srt.entrypoints.http_server import launch_server + +# For a single model, add it to the registry: +ModelRegistry.models[model_name] = model_class + +# For multiple models, you can imitate the import_model_classes() function: +from functools import lru_cache + +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {} + # Populate model_arch_name_to_cls with your new model classes. + ... + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) + +# Launch the server with your server arguments: +launch_server(server_args) +``` + +## Example: Implementing and Serving a Llama Wrapper Model + +Below is an introductory, step-by-step walkthrough on how to implement a new model end-to-end in SGLang and then run it via the [Offline Engine](https://github.com/sgl-project/sglang/blob/main/docs/basic_usage/offline_engine_api.ipynb). + +### Implementing Our Model + +To keep things simple, this new model will be a simple wrapper around [Llama 3.1-8B-Instruct](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct), and our goal will be just to bias the output logits for each `forward` call by taking the square root of each individual logit. + +Let's start by defining our model in a file called `llama_wrapper.py`. +The first step is to import the necessary libraries from SRT, which is SGLang's internal backend. + +```python +# In the file `llama_wrapper.py` + +import torch +from transformers import LlamaConfig +from typing import Optional +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors + +from sglang.srt.models.llama import LlamaForCausalLM +``` + +Next, we declare a new `class` for our model and have it inherit from `LlamaForCausalLM`, which allows our model to access `LlamaForCausalLM`'s predefined modules and layers, such as `LlamaAttention` and `LlamaMLP`. +Note that almost all model implementations take in `config` and `quant_config` as arguments for their `__init__` method; `config` and `quant_config` are passed in via [`model_loader/loader.py`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_loader/loader.py#L219). +Because we have inherited from `LlamaForCausalLM`, we can pass our parameters directly to its constructor, which will set the member variables for us. + +```python +class LlamaWrapper(LlamaForCausalLM): + def __init__( + self, + config: LlamaConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ) -> None: + super().__init__(config=config, quant_config=quant_config, prefix=prefix) +``` + +Now, we want to define the `forward` method, which is what will be called at inference time. +Note that the signature for `forward` is essentially the same for any model; you can take a look at the other models defined in the [`models` directory](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/models/) for references. +To see where exactly `forward` is called in the SGLang runtime's internals, take a look at [`forward_decode`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1705) and [`forward_extend`](https://github.com/sgl-project/sglang/blob/bf72b80122fd888bf619d17b96fa3e323ab809fc/python/sglang/srt/model_executor/model_runner.py#L1724) in the [`ModelRunner` class](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/model_executor/model_runner.py). + +```python + @torch.no_grad() + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + pp_proxy_tensors: Optional[PPProxyTensors] = None, + input_embeds: Optional[torch.Tensor] = None, + get_embedding: bool = False, + ) -> LogitsProcessorOutput: +``` + +We now call the `__call__` method for `self.model` (which is a member variable that `LlamaForCausalLM` defines in its `__init__` method), which eventually calls `LlamaForCausalLM`'s `forward` method. +After that, we feed the `hidden_states` into our model's `LogitsProcessor` (again defined in `LlamaForCausalLM`). + +```python + hidden_states = self.model( + input_ids, + positions, + forward_batch, + input_embeds, + pp_proxy_tensors=pp_proxy_tensors, + ) + + res: LogitsProcessorOutput = self.logits_processor( + input_ids, + hidden_states, + self.lm_head, + forward_batch, + ) +``` + +After receiving the logits for the next token, we can finally perform our biasing step. + +```python + orig_logits = res.next_token_logits + res.next_token_logits = torch.where( + orig_logits > 0, + orig_logits.sqrt(), + orig_logits + ) + + return res +``` +Now, our `LlamaWrapper` model is created and ready to be served! + +### Serving Our Model Via SGLang's Offline Engine + +The next step of this walkthrough involves hosting our new model offline, so that it can be served locally and without an HTTP server. + +First, create a new file called `run.py`. +Now, we must ensure that SGLang's `ModelRegistry` can find our model. +To do this, we first download the model's configuration and weights from Huggingface. + +```python +# In the file `run.py` + +import asyncio +from functools import lru_cache +from huggingface_hub import snapshot_download +from llama_wrapper import LlamaWrapper # Make sure to import our new model! +import sglang as sgl +from sglang.srt.models.registry import ModelRegistry + +# Make sure to request access to this model on Huggingface, then export your +# `HF_TOKEN` to download the model snapshot +llama_dir = snapshot_download( + repo_id="meta-llama/Llama-3.1-8B-Instruct", + local_dir="./llama_ckpt", +) +``` + +Now that we have our model on disk, we want to point it to `LlamaWrapper` by changing the `architectures` field in `./llama_ckpt/config.json` to be `LlamaWrapper`. +That way, when we pass in the path of our model checkpoint to SGLang, it will know that we want to use "LlamaWrapper" instead of "LlamaForCausalLM" as our model. + +```python +{ + "architectures": [ + # "LlamaForCausalLM" + "LlamaWrapper" + ], + ... +} +``` + +However, if we don't link our `LlamaWrapper` class to the "LlamaWrapper" registry keyword, then SGLang won't be able to find our model. +Thus, to register our `LlamaWrapper`, we want to follow the steps in the above section titled "Registering an External Model Implementation". + +```python +@lru_cache() +def import_new_model_classes(): + model_arch_name_to_cls = {"LlamaWrapper": LlamaWrapper} + return model_arch_name_to_cls + +ModelRegistry.models.update(import_new_model_classes()) +``` + +Lastly, when we create our `Engine`, we just pass in the path to the local model directory. +Then, our `LlamaWrapper` is ready to be served; for this walkthrough, we will use SGLang `Engine`'s non-streaming asynchronous generation endpoint. + +```python +def main(): + llm = sgl.Engine(model_path="./llama_ckpt") + sampling_params = {"temperature": 0.2, "top_k": 5} + prompts = [ + "Write a short, neutral self-introduction for a fictional character. Hello, my name is", + "Provide a concise factual statement about France’s capital city. The capital of France is", + "Explain possible future trends in artificial intelligence. The future of AI is", + ] + + asyncio.run(run_llm(llm, sampling_params, prompts)) + + llm.shutdown() + +async def run_llm( + llm, + sampling_params, + prompts, +) -> None: + outputs = await llm.async_generate(prompts, sampling_params) + + for prompt, output in zip(prompts, outputs): + print(f"\nPrompt: {prompt}") + print(f"Generated text: {output['text']}") + +if __name__ == "__main__": + main() +``` + +Now, when we call `python run.py`, we will get the outputs of our newly created model! + + +## Documentation +Add to table of supported models in [generative_models.md](https://github.com/sgl-project/sglang/blob/main/docs/supported_models/generative_models.md) or [multimodal_language_models.md](https://github.com/sgl-project/sglang/blob/main/docs/supported_models/multimodal_language_models.md) + +--- + +By following these guidelines, you can add support for new language models and multimodal large language models in +SGLang and ensure they are thoroughly tested and easily integrated into the system. diff --git a/docs/supported_models/transformers_fallback.md b/docs/supported_models/transformers_fallback.md new file mode 100644 index 000000000..3c7dd961c --- /dev/null +++ b/docs/supported_models/transformers_fallback.md @@ -0,0 +1,58 @@ +# Transformers fallback in SGLang + +`sglang` can fall back to using models that are available in `transformers`. This works for most decoder-style language models and support for vision-language models is coming soon! + +## Example launch Command + +By default, we will use sglang implementation if it is available. Otherwise, we will fall back to transformers one. However, you can switch the implementation by setting `--model-impl` to `transformers`. + +```shell +python3 -m sglang.launch_server \ + --model-path meta-llama/Llama-3.2-1B-Instruct \ + --host 0.0.0.0 \ + --port 30000 \ + --model-impl transformers +``` + +## Supported features + +### Quantization + +Transformers fall back has supported most of available quantization in SGLang (except GGUF). See [Quantization page](../advanced_features/quantization.md) for more information about supported quantization in SGLang. + +### Remote code + +This fallback also means that any model on the hub that can be used in `transformers` with `trust_remote_code=True` that correctly implements attention can be used in production! + +A model just needs the following two things: + +```python +from transformers import PreTrainedModel +from torch import nn + +class MyAttention(nn.Module): + + def forward(self, hidden_states, **kwargs): # <- kwargs are required + + ... + attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation] + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + **kwargs, + ) + ... + +class MyModel(PreTrainedModel): + _supports_attention_backend = True +``` + +Here is what happens in the background: + +1. The config is loaded +2. `MyModel` python class is loaded from the `auto_map`, and we check that the model `_supports_attention_backend`. +3. The `TransformersModel` backend is used. See `/srt/models/transformers`, which leverages `self.config._attn_implementation = "sglang"`, thus the need to use `ALL_ATTENTION_FUNCTIONS`. + +That's it! diff --git a/docs/wrap_run_llm.py b/docs/wrap_run_llm.py new file mode 100644 index 000000000..2d21442cf --- /dev/null +++ b/docs/wrap_run_llm.py @@ -0,0 +1,47 @@ +import os +import re + + +def insert_runllm_widget(html_content): + # RunLLM Widget script to be inserted + widget_script = """ + + + """ + + # Find the closing body tag and insert the widget script before it + return re.sub(r"", f"{widget_script}\n", html_content) + + +def process_html_files(build_dir): + for root, dirs, files in os.walk(build_dir): + for file in files: + if file.endswith(".html"): + file_path = os.path.join(root, file) + + # Read the HTML file + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + # Insert the RunLLM widget + modified_content = insert_runllm_widget(content) + + # Write back the modified content + with open(file_path, "w", encoding="utf-8") as f: + f.write(modified_content) + + +def main(): + # Get the build directory path + build_dir = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "_build", "html" + ) + # Process all HTML files + if os.path.exists(build_dir): + process_html_files(build_dir) + else: + print(f"Build directory not found: {build_dir}") + + +if __name__ == "__main__": + main() diff --git a/examples/chat_template/tool_chat_template_deepseekr1.jinja b/examples/chat_template/tool_chat_template_deepseekr1.jinja new file mode 100644 index 000000000..bb86cbd9e --- /dev/null +++ b/examples/chat_template/tool_chat_template_deepseekr1.jinja @@ -0,0 +1,92 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} +{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor %} + +{# --- Append tool descriptions if tools are defined --- #} +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. ' + 'When a tool call is needed, you MUST use the following format to issue the call:\n' + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n' + '```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n' + 'Make sure the JSON is valid.' + '## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '- `' + tool['name'] + '`:\n```json\n' + (tool | tojson) + '\n```\n' %} + {% endfor %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{ bos_token }} +{{ ns.system_prompt }} +{%- for message in messages %} + {% set content = message['content'] %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + content + '<｜Assistant｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' %} + {% if '' in content %} + {% set content = content.split('')[-1] %} + {% endif %} + {% endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>'}} + {%- endif %} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- set ns.is_output_first = true %} + {%- for tool in message['tool_calls'] %} + {%- if not ns.is_first %} + {%- if content is none %} + {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- else %} + {{content + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>' + content + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {{content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {%- if ns.is_output_first %} + {{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}} + {%- set ns.is_output_first = false %} + {%- else %} + {{'\n<｜tool▁output▁begin｜>' + content + '<｜tool▁output▁end｜>'}} + {%- endif %} + {%- endif %} +{%- endfor -%} +{% if ns.is_tool %} + {{'<｜tool▁outputs▁end｜>'}} +{% endif %} +{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %} + {{'<｜Assistant｜>'}} +{% endif %} diff --git a/examples/chat_template/tool_chat_template_deepseekv3.jinja b/examples/chat_template/tool_chat_template_deepseekv3.jinja new file mode 100644 index 000000000..46c1b8801 --- /dev/null +++ b/examples/chat_template/tool_chat_template_deepseekv3.jinja @@ -0,0 +1,91 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} + +{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor -%} + +{# --- Append tool descriptions if tools are defined --- #} +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='You are a helpful assistant with tool calling capabilities. ' + 'When a tool call is needed, you MUST use the following format to issue the call:\n' + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>FUNCTION_NAME\n' + '```json\n{"param1": "value1", "param2": "value2"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>\n\n' + 'Make sure the JSON is valid.' + '## Tools\n\n### Function\n\nYou have the following functions available:\n\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '\n```json\n' + (tool | tojson) + '\n```\n' %} + {% endfor %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{- bos_token }} +{{- ns.system_prompt }} + +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + message['content'] + '<｜Assistant｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>'}} + {%- endif %} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- set ns.is_output_first = true %} + {%- for tool in message['tool_calls'] %} + {%- if not ns.is_first %} + {%- if message['content'] is none %} + {{- '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- else %} + {{- message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{- '\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments']|tojson + '\n' + '```' + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{- '<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none)%} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {% set content = message['content'] %} + {{- content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {%- if ns.is_output_first %} + {{- 'Use the results below to formulate an answer to the user question unless additional information is needed.' }} + {{- '<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- set ns.is_output_first = false %} + {%- else %} + {{- '\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- endif %} + {%- endif %} +{%- endfor -%} + +{% if ns.is_tool %} + {{- '<｜tool▁outputs▁end｜>'}} +{% endif %} +{% if add_generation_prompt and not ns.is_last_user and not ns.is_tool %} + {{- '<｜Assistant｜>'}} +{% endif %} diff --git a/examples/chat_template/tool_chat_template_deepseekv31.jinja b/examples/chat_template/tool_chat_template_deepseekv31.jinja new file mode 100644 index 000000000..08e93a30a --- /dev/null +++ b/examples/chat_template/tool_chat_template_deepseekv31.jinja @@ -0,0 +1,91 @@ +{% if not add_generation_prompt is defined %} + {% set add_generation_prompt = false %} +{% endif %} +{% if not thinking is defined %} + {% set thinking = false %} +{% endif %} +{% set ns = namespace(is_first=false, is_tool=false, system_prompt='', is_first_sp=true, is_last_user=false) %} +{%- for message in messages %} + {%- if message['role'] == 'system' %} + {%- if ns.is_first_sp %} + {% set ns.system_prompt = ns.system_prompt + message['content'] %} + {% set ns.is_first_sp = false %} + {%- else %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + message['content'] %} + {%- endif %} + {%- endif %} +{%- endfor %} + +{% if tools is defined and tools is not none %} + {% set tool_ns = namespace(text='## Tools\nYou have access to the following tools:\n') %} + {% for tool in tools %} + {% set tool_ns.text = tool_ns.text + '\n### ' + tool.function.name + '\nDescription: ' + tool.function.description + '\n\nParameters: ' + (tool.function.parameters | tojson) + '\n' %} + {% endfor %} + {% set tool_ns.text = tool_ns.text + "\nIMPORTANT: ALWAYS adhere to this exact format for tool use:\n<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>tool_call_name<｜tool▁sep｜>tool_call_arguments<｜tool▁call▁end｜>{{additional_tool_calls}}<｜tool▁calls▁end｜>\n\nWhere:\n\n- `tool_call_name` must be an exact match to one of the available tools\n- `tool_call_arguments` must be valid JSON that strictly follows the tool's Parameters Schema\n- For multiple tool calls, chain them directly without separators or spaces\n" %} + {% set ns.system_prompt = ns.system_prompt + '\n\n' + tool_ns.text %} +{% endif %} + +{{ bos_token }}{{ ns.system_prompt }} +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {%- set ns.is_tool = false -%} + {%- set ns.is_first = false -%} + {%- set ns.is_last_user = true -%} + {{'<｜User｜>' + message['content']}} + {%- endif %} + {%- if message['role'] == 'assistant' and message['tool_calls'] is defined and message['tool_calls'] is not none %} + {%- if ns.is_last_user %} + {{'<｜Assistant｜>'}} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_first = false %} + {%- set ns.is_tool = false -%} + {%- for tool in message['tool_calls'] %} + {%- if not ns.is_first %} + {%- if message['content'] is none %} + {{'<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}} + {%- else %} + {{message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- set ns.is_first = true -%} + {%- else %} + {{'<｜tool▁call▁begin｜>'+ tool['function']['name'] + '<｜tool▁sep｜>' + tool['function']['arguments']|tojson + '<｜tool▁call▁end｜>'}} + {%- endif %} + {%- endfor %} + {{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}} + {%- endif %} + {%- if message['role'] == 'assistant' and (message['tool_calls'] is not defined or message['tool_calls'] is none) %} + {%- if ns.is_last_user %} + {{'<｜Assistant｜>'}} + {%- if message['prefix'] is defined and message['prefix'] and thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} + {%- endif %} + {%- set ns.is_last_user = false -%} + {%- if ns.is_tool %} + {{message['content'] + '<｜end▁of▁sentence｜>'}} + {%- set ns.is_tool = false -%} + {%- else %} + {%- set content = message['content'] -%} + {%- if '' in content %} + {%- set content = content.split('', 1)[1] -%} + {%- endif %} + {{content + '<｜end▁of▁sentence｜>'}} + {%- endif %} + {%- endif %} + {%- if message['role'] == 'tool' %} + {%- set ns.is_last_user = false -%} + {%- set ns.is_tool = true -%} + {{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}} + {%- endif %} +{%- endfor -%} +{%- if add_generation_prompt and ns.is_last_user and not ns.is_tool %} + {{'<｜Assistant｜>'}} + {%- if not thinking %} + {{''}} + {%- else %} + {{''}} + {%- endif %} +{% endif %} diff --git a/examples/chat_template/tool_chat_template_llama4_pythonic.jinja b/examples/chat_template/tool_chat_template_llama4_pythonic.jinja new file mode 100644 index 000000000..74b315c31 --- /dev/null +++ b/examples/chat_template/tool_chat_template_llama4_pythonic.jinja @@ -0,0 +1,112 @@ +{# Copied from https://github.com/wukaixingxp/vllm/blob/8a32e2a6e452a03c0e8222e3876ad6086cbf581f/examples/tool_chat_template_llama4_pythonic.jinja to enable better model response. #} +{{- bos_token }} +{%- if custom_tools is defined and custom_tools %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if tools is defined and tools %} + {%- set tool_definition = tool_definition ~ (tools | tojson(indent=4)) %} +{%- else %} + {%- set tools = none %} +{%- endif %} + + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set user_provided_system_message = true %} + {%- if messages[0]['content'] is string %} + {%- set system_message = messages[0]['content']|trim %} + {%- else %} + {%- set system_message = messages[0]['content'][0]['text']|trim %} + {%- endif %} + {%- set messages = messages[1:] %} +{%- else %} + {%- if tools is not none %} + {#- Since not system_message was provided by user, if tool is provided, system_message is now default tool system message #} + {#- This system message is from llama website:https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/ #} + {%- set system_message = "You are a helpful assistant and an expert in function composition. You can answer general questions using your internal knowledge OR invoke functions when necessary. Follow these strict guidelines:\n\n1. FUNCTION CALLS:\n- ONLY use functions that are EXPLICITLY listed in the function list below\n- If NO functions are listed (empty function list []), respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If a function is not in the list, respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If ALL required parameters are present AND the query EXACTLY matches a listed function's purpose: output ONLY the function call(s)\n- Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\nExamples:\nCORRECT: [get_weather(location=\"Vancouver\"), calculate_route(start=\"Boston\", end=\"New York\")] <- Only if get_weather and calculate_route are in function list\nINCORRECT: get_weather(location=\"New York\")\nINCORRECT: Let me check the weather: [get_weather(location=\"New York\")]\nINCORRECT: [get_events(location=\"Singapore\")] <- If function not in list\n\n2. RESPONSE RULES:\n- For pure function requests matching a listed function: ONLY output the function call(s)\n- For knowledge questions: ONLY output text\n- For missing parameters: ONLY request the specific missing parameters\n- For unavailable services (not in function list): output ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\". Do NOT execute a function call.\n- If the query asks for information beyond what a listed function provides: output ONLY with internal knowledge about your limitations\n- NEVER combine text and function calls in the same response\n- NEVER suggest alternative functions when the requested service is unavailable\n- NEVER create or invent new functions not listed below\n\n3. STRICT BOUNDARIES:\n- ONLY use functions from the list below - no exceptions\n- NEVER use a function as an alternative to unavailable information\n- NEVER call functions not present in the function list\n- NEVER add explanatory text to function calls\n- NEVER respond with empty brackets\n- Use proper Python/JSON syntax for function calls\n- Check the function list carefully before responding\n\n4. TOOL RESPONSE HANDLING:\n- When receiving tool responses: provide concise, natural language responses\n- Don't repeat tool response verbatim\n- Don't add supplementary information\n\nHere is a list of functions in JSON format that you can invoke:\n" %} + {%- else %} + {%- set system_message = "" %} + {%- endif %} +{%- endif %} +{#- Now writing the system message: use the user provided system message if user_provided_system_message, else default tool system message if tools presented #} +{%- if system_message %} + {#- always use user provided system message to override default tool system message #} + {{- "<|header_start|>system<|header_end|>\n\n" }} + {{- system_message }} + {%- if user_provided_system_message and tools %} + {{- "\nHere is a list of functions in JSON format that you can invoke. Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\n" }} + {{- tool_definition -}} + {%- elif tool_definition %} + {{- tool_definition -}} + {%- endif %} + {{- "<|eot|>" }} +{%- endif %} + +{#- Now deal with all other messages #} +{%- for message in messages %} + {#- Base case: messages that are not from tool role and has empty tool_call list #} + {%- if not (message.role == 'ipython' or message.role == 'tool' or ('tool_calls' in message and message.tool_calls|length != 0 )) %} + {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} + {#- Tool case: messages has non-empty tool_call list, must from assistant #} + {%- elif 'tool_calls' in message %} + {#- assume tool_calls are always coming from assistant #} + {%- if message.role == 'assistant' %} + {{- '<|header_start|>assistant<|header_end|>\n\n' -}} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "[" }} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- tool_call.name + '(' -}} + {%- for param in tool_call.arguments %} + {{- param + '="' -}} + {{- "%s" | format(tool_call.arguments[param]) -}} + {{- '"' -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- ')' -}} + {% if not loop.last %}, {% endif %} + {%- endfor %} + {{- "]<|eot|>" }} +{%- endif %} +{#- Tool_response case: messages are from tool_response #} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|header_start|>ipython<|header_end|>\n\n" }} + {%- if message.content is string %} + {{- message.content | tojson }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'text' %} + {{- content['text'] | tojson }} + {%- endif %} + {%- endfor %} + {%- endif %} + {{- "<|eot|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|header_start|>assistant<|header_end|>\n\n' }} +{%- endif %} diff --git a/examples/frontend_language/quick_start/anthropic_example_chat.py b/examples/frontend_language/quick_start/anthropic_example_chat.py new file mode 100644 index 000000000..03d699be7 --- /dev/null +++ b/examples/frontend_language/quick_start/anthropic_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/anthropic_example_complete.py b/examples/frontend_language/quick_start/anthropic_example_complete.py new file mode 100644 index 000000000..bce2a61ea --- /dev/null +++ b/examples/frontend_language/quick_start/anthropic_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export ANTHROPIC_API_KEY=sk-****** +python3 anthropic_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """ +\n\nHuman: What is the capital of France? +\n\nAssistant: Paris +\n\nHuman: What is the capital of Germany? +\n\nAssistant: Berlin +\n\nHuman: What is the capital of Italy? +\n\nAssistant: Rome +""" + s += "\n\nHuman: " + question + "\n" + s += "\n\nAssistant:" + sgl.gen("answer", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.Anthropic("claude-3-haiku-20240307")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/azure_openai_example_chat.py b/examples/frontend_language/quick_start/azure_openai_example_chat.py new file mode 100644 index 000000000..d53f935f4 --- /dev/null +++ b/examples/frontend_language/quick_start/azure_openai_example_chat.py @@ -0,0 +1,83 @@ +""" +Usage: +export AZURE_OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="azure-gpt-4", + api_version="2023-07-01-preview", + azure_endpoint="https://oai-arena-sweden.openai.azure.com/", + api_key=os.environ["AZURE_OPENAI_API_KEY"], + is_azure=True, + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_chat.py b/examples/frontend_language/quick_start/gemini_example_chat.py new file mode 100644 index 000000000..0ae623109 --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_chat.py @@ -0,0 +1,73 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_complete.py b/examples/frontend_language/quick_start/gemini_example_complete.py new file mode 100644 index 000000000..5188bf418 --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py b/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py new file mode 100644 index 000000000..afe0c723f --- /dev/null +++ b/examples/frontend_language/quick_start/gemini_example_multimodal_chat.py @@ -0,0 +1,30 @@ +""" +Usage: +export GCP_PROJECT_ID=****** +python3 gemini_example_multimodal_chat.py +""" + +import sglang as sgl + + +@sgl.function +def image_qa(s, image_file1, image_file2, question): + s += sgl.user(sgl.image(image_file1) + sgl.image(image_file2) + question) + s += sgl.assistant(sgl.gen("answer", max_tokens=256)) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + state = image_qa.run( + image_file1="./images/cat.jpeg", + image_file2="./images/dog.jpeg", + question="Describe difference of the two images in one sentence.", + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + print(state["answer"]) diff --git a/examples/frontend_language/quick_start/images/cat.jpeg b/examples/frontend_language/quick_start/images/cat.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..a6a8e48c972b7c30fbdb5c2e95e806a9d6411540 GIT binary patch literal 345163 zcmcG#2|Se1+dq8I*vY$=YOx~}iJ?>+q97{Fs@Y-$XE zKmY)u{Q-O9AiU{?3vSl7R>r30Mh6XT0PSEr0RT{+`+l}2hLTQKoh4bu4zAd5bNBT3 z-T(eC4qCs|{{7wopjYL;IOqSZn(f9-e^1&83$%Y(KibV{gB75a1#Tad752*>2W6f8 za?pL>`?Pbc_sf3vwijsS8?>^*?SE7D_%~%wU%&nHpVQ9KfqDn*-)sNaAMv4^J`Q%Y zZ&upBDBus+0w#dr{`_gbX$LG10JK&CfFa?ZbsiZ2fQbZv!$be9lYI>U+>ZeO-Sf}7 zf6Bzy-Orug9V6`%eCrkfEENL)=T!jU{|*4`5PBQ!68K+yo>6 zQ@{uC1XO7yb>IYW642Nq02gTL{J%T)JNdusfp+yC4&Y@4XEOX@0PzA~UJwH>Xs-#7 z*f%Fe23q^S%oYHFnORsFnAjNEIcO)?@cJ(Kd}HH?fu*x&HK z|B%+H80~f-rhj`=V82sl77!~Nt@$=DO|^_ntSn57%*;%ztN=G{Pz;Q`QcQd&&)YEb z3%J80au=j!6Dw)AW06rmy(oCZoo+Eh6YvVg_ym@Y>p{chS`lvDt;JA4&1baX^YG9p0*2KB5<)jH<**kb zu=Q7VYoYys{?ooq;_6I2waM;E00uG!J`>^%n+(NhFM2lguU1EwxN0rvOL9W8D6JVA znS{+vB9l^CFUS&mzI`VMGl8;PD0?6*aYFE(o@-q>w43w@THdk;m^*+#L3*#&@Gt~< zX=@37bfz_81IRMmt*4G{X4M<)xYE1jc;rRKvP>Ib32{vx$_~xgj-J0@wMhXV5QP|L ziYeJ%Jo>jaBAB($Qq-#RlJ)RIjZHqw?MzPc)NE8VbNLHG8!wg-Ph8slXgQ32m{~ z4)LUD8;pyA#U6lR$tFM+Hi9R<(I**uA;*%i=ociy+xl&5Xgz_sl;BybL8%zp1N>pX zDvdu?j$Jc2S{F&)5ZM?aM4;D1BBGH!IkN|oG%j@!t9z$cjG1O$6RR}8Wa`+HqfG3B zeG$gwd>;cL$e6S};CqlL1=GP-_U?gj0#jGy`!MtpR*X6{QcG@-1_udaOP02l*AH1L z;5J6VSq4x-gx8!(^&TL;MtC0V>a?IH$!D+DGlg*L-x7;&a}!i97H63R941Zl^_Y6X zwm+jDkj}tYKulNA!x6|P5ishcx7jE77}9{TxMl*{>J^R%l#hs_pCd~^rR#v|d%}8z z@S_8bXG`N>vtfmpr?_6NL?0H$*oys2!DdgHN9tiH46%b0?P#gP-Nty20PpGJg87{#AfOncxZ`o| zr!UycG^^`Fsnu&MHOuorlX9sz(!L#fy?@6`G`OxzkP8QJ4G& z`6$^FbwUkKuVFg3wz-I!XVIZd5qRmkXRqd(y|l5aG&7g!ezq38qk|$uT7Ut&(7&Gy>ONoju<%1aug#Z88pnNN!J@Iy9hCHO%kcQ zwUMVo#YdKFA4#a3;iJ7%=Xmt<}*JZ3oiIq!8m}D#--oHX;5q zHFOVf0`Gcetx1qEx%$)8iU#^%h4+AS=jSbUuT+k1EkE&hlrZ3Z6HKfiC6ZRFqa@M< zu_b6(h@6`N{~NaiU>EmLpVynN3Kt5RitQ$D*Oxc%PN3=IH7*Sf8fubAMI_R ze0H`M=thvh$4!9O;t74h8*Vvo>F6RWvORykdhYBz%CT`(e{_NO%`2^(7kn)GB-Hx0 z_V0ZNkdm@|PsWz@%k&L9_Ya3w9^com+-~+~*rf69jjXyoK$I@T%w5s#_^|Tjjj)4d zriLKIhEH$BJZf&Hw2zP0yaAtqd2Ix)V#2m|j-MU&$llivKgtHBrb4cd+ah%I2m)t9 zB+}OO0tvqQ)A)Qk{&2e4{m~Y;z~G@oKUQ+|mC_%N-NVv)u1N}yk8IQ|2{(*TtLvJd zvK%OY^%wLrm3flb7Tbn}_N;uRW0AbWyh*y>an=b65<%dl+djBVZ!Aa_pS^|=$3Brc z06GDt@~~Zq#5HZ=U+y&7LfH;+o>Am$$v2^{WDLvVCUMi_%&&Ox640kDvbh-~;`C5c zmnu3gDL3KAT8&vI^g`RPdKW61O7|5bBb!zxKOH&@x=7t-ok$!dwLzoeEo4i5_&9ex z1~zuH`6BdWIy}4x)P)8iE=y0_0WcCE*q<^Qmi{ipEHsS!x(wDBi zS4qw_85_&jW3N5MdScUs@l>s-r99tpqY4^QpM^X z@apnD)-vbrlAlUw`!`b$mavixM#o2Ub!eL^5&iXpmsey%+TXZY;JhP?XBT>cLK^K~ z(LR_k=vZLG9-t;|q8&);TwIG{E?v;lme7rz(}RS;qUl3s{JXUQTVBGlAQ)yN13Ync z;g@vQD$$Jtix~gtebYTKE4N8t9W z0dbA6U5?R?rQ2GBW4@!K2trzx=fO_!4tC-|)5p#PzIerEI{3Edykzs4ZIjP6;U{?x z0VhDFqZ3|n)cU$wAtH%D_64=N`{R+0#7zztf;V8RQA@n;Z8~Q;AUf|8v>hz4yzl3e zjOLFU1`~VYTBt-yk5&;_?=_D~Erv0+R{v(`b_H#vui||y8Xyo@c4H43e5N;^VTnrx z1D}Ppxp=DvX;QpRC3s=;C#8(Yh-Hql?5$ZTgMm^3-ao9P{o=+0_}K1&-87u>z6l$i z9dqHVnchy&4$~wihSQyv#oAbG^BSeAwz5)^OsWx^vT=f|ouGS?QY-O535#~}^`Px~ zSfr7AQyqOF7_y?blEdCWA0 z;nLWKu89&cNKNWp-LR>tt~`hR^T9e4#9D0)J`hN_Bnn1MjM65>wC}1S&&%XUBa(-j zV3P;}O&9c$6g)x%Qz|uvBbu>GM1WqE#96~UpdB;1l|34W5ndu3)E30xfj~m-ex>l_ z;;_y>2AFB7x}e8{NAG zo~V%9!BL>~=kd_+<^AQ7^FoJAoW+b;PR!D~7a-I>b#I^Z4qg%op^(Cp93XdDfsK1u z$%nuMBC&SU?O+-)A)ujgs_^I*v_fNb-undmew@dgj8sK_UaQpcEk#>W&4yj2Ns^O+=jg`4DjTfHYF-`GZvyM(pF$}DzE7tD4H*UzJhX$owM3Ity*2Awbj0aq zV={qWTM~nt5h)zmR(Qr}!~6=Ugs49dvwwcYjEEy)xQuhdJ2 z)y?s#&8=CnwYsoMZ(+k|!zw7ycL25opg^OB*JA&PL6<}% zw6mf|jQfR`bco;K@pAbC!`EDf4zOqrw)1#DGCv|r_6P1nm#0O z^4}m}53oR^N9#9OyzR~Ikx!VF8e4}uk2$Ro5ZR-T!i_d2u4^`-FpZIQ-h6y22)sD4 z3#nBxe`$Q1fIeoU1=l~p5xWPZBi4F1h}l~pUs$LOND~7Lhlucd;+E*73f>0R zOV|XAJZYH#jjW)nfW_BJ8|wRVl#F_BnmlOa8FLSC7O;UMgM#cfc$$Wz0*GBOG&8~rTxYr5gppOgO{ z57rh}i?+Wadbu;0E=9(9)A4QRv7z1DZN%vZU-K%)uXz~Sc{bocUGFqSmcv>IYZRKw zlLtm=DhDoAOb%xb5yBBeqXi*3=8hoXk*{X0&gf|7=pOK7{k7>C<1RQzcEg#ds>-}W zmCYXY{MLT3cEf`cKog#xzNqv{9jZJAJjQ(>G?69NIYEEa3Df$hjRWcM7|lalX=x93 zAkha_zw311tK3v^Vl7_@aa?1GzCPt?(@TF)Ug6PIxX;0h;3R_;@vz-iSU>L2Ift5hgt>dJV%EtGpnZ(C(hEed z2`v*NR&o2HVa14WSMP+@R7mgMpgUtmk@(cOtt6FN$xJs-AJ{Mi9!@7p9`lycTzd~v zi+L2yFni#Rq+H8Ar%-NIRvrJwJ{4tZmukmHub>H(N+LGxAa)USyKYcV?jGJ*IDj48 zr4s;-0T4(G`y!LcOZ0CO#%7fdkB>2qZZWj9c1mG`xM;h{zZ0Z^!jpmRE%5LhQ=P? zyQH%Q1lE%=hk>`PwRjh&VBU&`fsLG+N*vQM?My00Y~D)Tkuv4(PMUf1i{p%e89m`r z<%|$V5GoqB=Z+htbEopR?Cf>Cx~-5sn-^HVP8zK3h3_ zZ99k_YBL!)5o_^TTU0D_6U2RL>JzVrIv7XrG5j38vPGfdfbH_i-OOOv@*a&{#%$1{{odJb zKty~L>ub_R{);*eSk10(k^t_39)QQF2c=(KrZo{0w}u1Uu==SRKrnO-dl<;}1$L01 zcVz%t6b=w)mUq*3YXBtHEe<#g#O>6hiYNS6aEwmFA+UBp!b)#ToF#RQKGARs48{vK z9Sq*RKD=Wr4F)8Wtn__GZ~N?!TT?5Q_LEv5N3RvN5VRz;LgEF6DW;<^+kr ziX9}yY`vpv4OD_(unfEhrizbZ12=Z~;6_Qks99JVAWtS>f4Alw24Gk`3OT-p^#*jk zHH5M0#W^|YZIp19o;ap<%~}d-8B+uZ<2QVCwu0T@VGTiI6+m^p&U=3k6vlH3-3V}S z%(G2G2_8Yq%s9yI$%9$NZF6ot!uBmU47>3H)$248!h<^pA-0RG0eDooMUmPN=oWRk zf!dVxwlc*795WyKG+jd7+q7~l8`iS*4}=-oa;vAT;$!Cn>o-=K4D4B$3GtAHOOkHa z^(FpcO5qk!bK2Jk+{NW36*3*XAZe;5sdsyMc{32l$bcY!VThw~5BQx5ko?T*24!t93uedFZPwVZT`ufY0n4MRd7i0PBz0t7^CEkkDJHo3WcxnXrxafJKKCS$xc)7oUb?iUR*BJKn| z$7P$0A$VeP{It-7AhFNOQQ;4{8IWoSewYeXW0>NDKd={IS^UcILNI#wnj_1fgXEZ1 zlfgH5crH*U0ZJT*;Wp=86xy8tDak@=bU%BC!AL#DLy`)V>J9B6nh7nAj=nhP>58d_ z7;c(M?DY3VLSV>1`Vs@K(!3#Y3dcyqXh_&Zg)>Yojg9VJ=ZF}_$F=s{Jy?cJ7c&>9 zX5ww`&MivOmB|X^*ey#8ybAeA-UZ7f_-a*Sup(mAmH7vHPDSAJ!+2k-^gKJ&GkYd^iZd$)!I5aPZnpkMhv zD$pI4RLlexcKjm4jMp+Y?CIZ8Cm@(cqAhQrS`mOHAcOAT&AIGMBP((`!IpbK`3C7Z z4L%5~Gn)h&km$RW`2sjbiSuoQWbE}o5l)J)3Vb%434<-HotV@`Y*zph<@)76@j%YN z?CKH@JPPq&(*k0UVpzAQ>60KWz$y4TD4ZD(+TEgHb9x26+7+yiffoE4Q$!#Sds3-L7Nwek+pSqNeP8uBDNJF)XAiLXmJCNh zJQ5RA^YR>=)gLoXe38U4@)D8zyg9(RQN6=d=?hlzMD8NUEsZ(zR-$Adz@+byXZIR- ztr8FSgTZ9Niwb#*VXTnunyAWEnTijA;>O$5$?J;I)#XU;p4c5EX5JLX7~w;T(?)8_ z%!D&f#A(Ve;C)oWdU^ov)LP;yX%El>9&)x%OnD-FX;hI~E5P9-5~Q8IwYd?8>Rty* z0S=@dVzZmM)9~V#q^;#U&t&GF5?NVmH;dMHQo@hNzokU<3I!C_Zxm7!DlTSpZoJVTwjD zOAR9x0K=BIO6zQoc$a*6U0rPz;|@Y%29|?TD~H)R2Lug{gLbu=u(x|w!%aLVO+-OR z1OwJDsEHoGgH>uYw{b!f?s$$lnq|3Mq}`tYrNFdzS_nZJ8vrm7;}kf&OeG;u+SaSY~IV!g*MCxCiuVfEaJJ zKUwP-|IvZ!rwGDxz_y^cGlsI^R=JIs_-Rw5;T6Tk%8=F5yBRl;y=x*~LZhP>L9PvM zuz|3}eVXeZ;s%M5m-G;B+Y;$o0OTRcB)*K`qlX6|lazeS#;KakcvXk^flR>av9~STqE&u7 zwgpi77Zv|Q)B9igZm`w^z5km^`slbz^tDk2Cx9JPC;(TK^tITIPgCAcx!F%j3ceoN zidd#}D^`6*g(vAr6Nf$11)8tp7zEQ7vgemTBK~`T4pQ4qz(CJraaCB~oT=v3zs~tz zE$!dm@tpGbI{nRYhM_*(^To?Y=vgGf_z}Em` zv9-9jCvlXp7;m%hUjSL|>MhvPa?^QT1;cg18=LZ8D7L(8^wk>9jGp_vehtG1HW2kO ze<;tMQ%1%G_vsS8OUn`ni5@{l!*Qwx+3TSuF=kh2H(zKBh~XHT{<>3MGda4`syiP@ z3)pC)RqM&cjB&622ifz&W=X`HR&qGoe`wkl#7@{AaLsK1;Pu(r1M0EpgO&V05eyTD zWz!mINoZTYNH*00e_gB62?!;JbHQDwA?6z0K;480A0HtcFU^fM!#9oHtTs99dL&Kg5M)*5qRq`6 zWwRglSDj5v8_s<))@=Sgr8V)<%}rXTEg&NOR#PU&2^IWeuVE&ix!I49tA^5O#Sc8u zg|;cPrq3A|>x!Mt*khuYbI;Yuv1A~mwT5OD4LjtFIQuk#0(k%PPv!@LIvGnKr(%7to@l&h& zrY-v7yj%Ys@LP&kW)t}i0rBH5lCoY@)iXT9|CE23zdC{dZQI_W;0X0yHBF5EA?qQ} zzF>t|Y6yrOgg=QCPUnf5GKmOK_x7BLHX)e?v{9MMe#iB%p$Gtv02}7dcBRs@#^~Vi zQr`cGE&orTf+zn*PHMxJ$EEcc6g+&a`MPL?Vs$&V|LYzQ(adj)zW1@u-i+VAu%#RN zVzSA|p!r7VF>D>~q$CewGkAwPC#!r(M!<2OxC7)`NhAV#q?_8X<>Dbd%%ZtL$R-U; zVSGj+B>yvq{C~RV|IIJf8X;o#$paDPJ2I2H2UaPV9%zJ6{rgXJNXp|6o?l+tWCw}0 zi5o07^jA)A`=gYzDo;z7!wu|@5Q-bRcP{jNi7t}g^m#m;%3cjEJ{odaNmBT3U-|Ej zG=WFsr)U30GWCINR4xL3J}Zz$pO-sLRt@7>nLY|0TDy*jJ8j!(-P+pdbACx`Uu_b8 zxK5o4F8H07zFS(Bv-ZS8~t zIfz}(d31&d$1Sm8G5l6%e^})au|lqGD|fiW$(ttV;p%UKr*Dxjg*};KkPgV}97zcj zz2+uUq|h1FZKc}%L+ej7@=J5!`KMwv-zQEU)i!0jr_HI+p%SiA$f=R=7|;ModV9me zxT5>W(wl~z9KMrX*FUImG4gI2$#H;DsEq&*q^yCO^wlAOy{Hx!Ol6g?HPYzdMCyX(~D+p}TeN zW=L|g0?@CkJFPT3s$VADMU^KNl6)jJ{^g;&&C{L7Lbq~m|zmbDx z4`@%8Q$hu6Xi0o)+5-Z0udI!pxvhR*z0w#?9I0lpl;netqd6`ctBGHnRO)rc#Or5;NEn~B>4`@b z5_kIcIZq=L@BF-!fvm`L`_dn!rLpyiyn>Om#{_bgqcU|g6PLdN%CaC-hX4*7L0O^v z=YAU0ly_b`r(<(G@7^!1Mz^O0YW-mL)GzW=twixcHj{U?L>__RIL=Cnk@N4Wa6hSw znG$N{-bSXTLZY5EJAj|Zi~ktb|95Z$%=r%1TqzrBgss8(dQ`CgEamKO^wrtXZ_VDR zG!0Da;TJqY{M_Uh|BI>*trQr7T&}Y}mV;Yjh^!T-=jCnQHOvK zUJUG-YFOx^QvK1P^{WIP(HBwwi6FoaeKH>!P?|R|637XV4O`t>@J1L5uS3LY^?dNQ zTF-CN=qJlcuaC%F8^xO1eovVdO2HtZBxzfx_Ok6=a&6vCzhD>ViykWPnJ?Y-N>)@< zloQa%Nl2(tq2h0p^o^eIdk}K&&sNe`U-U8lSI>?i^P|4|wGOG8PhP0Id;8svH2lY# zVq5vA&EJ0UUN8#O@2o#+#vF1|Cxg!qO?y87n4jM{9ky~p=Ia%z=$hN&s4K$bn|DQbtyKBo z>601hgFcC~vPN80tKg7eEmyvPjMck`@sqinF;bFYPq(3A>jdG(ZH{fXNkNgAb5`tz&uhp{EVgV>Z#8<6;SGDyITt2Y2 z8cKTMB{)LpbJ!%+^vtTBXv#=e#vDIe@&V>2z^4lHGmvQ`Vsz`M@{R>ctgg-FN%Mnu zhzjT6A3tQ+dz}qjgU(0&^4>}DIV@>~%}k>?2o95B^bk9-jx-w5!;ZgKOS(S?0sM7l>@;%N6%j4Ag!DXU8kbM}h;6 znM%(%Us7}y04EfrykP<}^Ioz$WA6eb-EFkFs&xnwbgT2|(W4(e*vZ5nzF01!DAp3R z<4E*cqHV0D_pgZ{wTiHh*Mbsq>Dye=t=P_oYHPHe<0FRPm-~B)h9{+Xa7?C*^C}om z;_8mEVu`7G>toHFH(ag;A|XRtG~!2l{Kdb|U|5b^Z5T$%Q(6gI9!2}vgWxge2&~NN z&LV@1veCstIj4mu!U&AD2nCj zn;0_t`?n{*B{0pwNGr0aP$RSVY=&UzcNhOBk>NiROc0}s1R2!`h#sy1*#@s@i^Kka z7+^J<U|Eas}N|uE4l%LQhGd> zC_Vbx;g0_asq^1Ke^5BP02>7}OTklGbY;{lSdJRCi6YCRcWDoo0@K12STS`EkQcyN z{vZ4hOHc12j?u+QjEo3vKwiPI98HB!wG8iS;NutZ!%9+`*%W=X*At-T_nhdVtbkv~ z&@T;?;jID&m+u&!x9$I6Xr1!VJ})KQEWG)~hiJmhsarFFPCCvRmA8IDU*5WUQelSK zPdcT0P~hEMG|xzO?3Pk{Ns)%fUr)>CXXTuKf-RKoY)2*7zs{DU3u*-!0aT!|^RW?w z%ZW$s1Rh=PzJG&18yedA%;m&~@smr$DirLFNAXJusp*?Pf{VM!_K?by+v=K_7sg_) z{jIrJ`yr!qG3>f}8GOeNpFDb*_jjXv&5sv%g{9C}?;I;iVv{|_Dw5=FYSVI8(ZbMB zj}+Ri(PB&ZN=dm!%#p}G+MrusuTziFm71EFalX*e{*~pb&Dr_q-}o~Ek3nFQYGLn! zYKKGXFcaRFtjkg?5j{5_+eB6QbHNL4t|Y3g2(Oe}y2*6x_9e+}Aoaa-l+}}OLeqsB z4|$3q9BpT7&<|#ITj8xX(3W|Y%sd4p;}?)qK{syTp2^*Y3(3O;E6)l{NS`{o<2L)~ zrq3C%mDiEy!U^~KM$t)0MP=+Rs!Rg8rk8yzZouRiZ>lJ3GrSjSe)%cnHzs*e?A|k~ z$InA4gK1&b?}7~FSu>ohBf8H?zs+(iG-&Ng`&6px;4E#UphckMzD`w;Z#{2qmX!8w zt{BV`btZ?`$c@9|lgRlutHz&C2YD$rRgYO~Z`CoG`AcQF{Gt9Dsh{EPd@rNW**R`4 ztzJ`nI3p3~l|yEKt8E$m^KCDez*)1-fFsa%ikEEdA6+sN5IxU+C-UqzE>dP0pqjpzdTE_ufJ{eI7>-M*5zTAvc6E^)n*A^ufz34VqRzDI^8q8ZxhOb zEIW!hOeCn)NmJ*iEF_<(Hhp=LvIwp#x^^7YhX>_Cjmat%HVhShJKyIkIXdB;B@P_srE&keo%p(s3xA`>iIq^Tz@ukgG<_10scn?HWtkZb_OgrI&0}G8&OHfg8RO zS0~GytgAi>1^Bwsb7&}DmRr+Oo46;*{X8lgf5gE>qmbd zx3!O)wPGw-IVyTQtH`?IS8?PDq|b zU3jaNKqdyrW=}f|UpuUVxZ(SpVJio|t zM3w@}SlxlZDB*e7sI?%TNmG0xdW(p%n?z17hDVkE=ale2{pY`n{+p8NFO<-T#Pps3u}<+%uKwiQ6G>j5|1RomhM-C#}?E@Gp${!JVzE0oJWeZvDl5A0DzYL~WqT zZ3hQ)FF#AYQVMPSWB3o6gu6)K>@U;Ak}7=CJwdjzJRJoa9!#NGaAP57)ev)`qS>w| zPJ-`U0{rx;JB9WpcSpnr2}BunAxe4NeXmkS(pvFtc}>5Vj*c&1MurPa_*3L=Mm}YY zzqH9aH^XrZ!+&~uI^W(O5yjQw(H{T<)jzF`?xt|ny6 zc*}AcmGKG|!JC*d7<_o((AvNyo}ZJ>cOgQo){b>04t(pNTS^{ZqphR&cu+r5KpCOzl6Wf!9U)0|v*XiUBJvhuy1RjW_K zrrf7vl{MGFda^TY(DPN10;ln2d}-m*GUxm5wH1CidcsT0>FX7-0YjTsQ1{Yiw5(0n zYa?V{bgNAY>{LXGXaCcYEfY8PnFgB|te+ha+`BeXLr%L;%;zh$l^Pn%>%AHMw|=}m zOYm($Mji4;{>=G~-DFOU8d_tAoVRFO7qG%y5ca63M_a~6h zw;1J6s$jU~+4V5nrzFKQ+PsdXWhKzfFk#%b>J4w9fdWtH#^`WtL!HORGbKv>PJ|$x z?%y%S&wsgQ48D7RIOohtsoKEz8T%`QV1q$ZgP*31L;aFNYeX+k$u279eC5?fhq{zF zUXf;obo*wmAF0L-)7DS;pThAj%*m_zg8|Ws2`f%-xp8(RGPW+RK)59YtOMt+oKnpl?pbO$SL|FA4ck-X(=7RRH0>aAmx69qF7 z1Akc(|xS^OfpDZ#3op_2fzxLX$k%gOPPe#LZB>LP^po)NFDm!WSs zJK7mTfDL#X-?99t*SINE8_vULf(+Cr&Q7g1*76|bV z4hiO-_shBLP7--O5@8+Jx^uDp@^M^ot3qI|s*=PrxAs%_eF_F-q*J^<`JH;@x$!2m z@G0M6$05Jx_wW2jHYxV^{z5bhk=FWC_Ccw~M*T?wJSZT@6oUu}jy`uzH}m69*R86_ z=Uq}vr#G4I6^zNAmF853e1}Ny0ge13H>#zaQw!P+lX)KheLjc2GMsxx>rAxNq0!87 zx5!Vn5xOSJ7sGN+Kge0>ksEx|zh?Q}=XiYni;#j5wz&4K&J6kI;kPuf4C|38QO_6YFsI<^7-fX z07tXX2LS>1@5?_-E^X}X_*mLkbyFeFB6E3XAOxtv_+sTIYi#bRK{| z&$Hw%5jV-}J0}F}Usk;GkO%Snr(2o-pOwDKnJ(nkDKrUM=^!Ld1cRJemxw*p^+CHM zl}vK);AURvxjgeFj0Af0PqqKkQr>zt}$VA@!7pUU*F|Jsn9q{ zMQw2e+$&yEdl^c5n6vtX=r7qm(mH3A0M5cU%gH6{|NomeN*f7ovxS4)ps*) zf~$5b=0xA0uH~>(u!FXl*4$jWYPTkQPF(T0MnR}iv$>x%A0KPLJsX4i%a1&jVvp$F zj_MY3_%jxW_rpEUu7Z#g(B9NEMUAGbB(0QdSypD6cg*DGjhs54-S8<|3N-4A^y&S} z`~B~vLE2YmHZ?EzJ1R_^m*9Ol(M7|;c2-(^gx7@@CD+G)<+D6ddN}y%854hBSmB$f zFOkPgbV^=|C$n3Ud zUHoC@&(DS4^SSs}u6HmF;?Hl{$*1R^nzhnMOLph|@=LYpmPwX)aM4lQK0^ppHudTi zGlTj$=J(<3c>W$;ifyXWc)WUpe1*Skp@m?NyMGhu$;a|5GZ_ITD(cJ1@W&~2Gd3Rt z?>|n_`*x%1TZ7d2nPcbzP~oB6fwU*R=>hpNy}_R}1(P3JilHs7t6;Ti3HKvU|4MqP z9;{)}_D;e0w!6{5B?W<4eCM-U`O@abPXlDi)eFxadpPcC5=`7I&?1=nT!?+F$nnZk zuk)hbP)7fY9t)RADfU3~bB_wI)Zbf3zyIZtQ|2M_IG(eeJ^U%0ea~({qs{8S{I~&M>1lmU<#dKZ^tl#W_ms z)9fQka8HIff0pwyDRMwR`Hgoh{9N|Ck#$$Azh|~ouBIus@M2lzukYlVTc{4E+ydt$e-@k&RL(Weuy2an0s@QQhAi#?Nr zzq^_DRqkGN>&4*b@n)wp6kTlIzO`#Ln`6fTOhtceZ40U@bCn+e1%KT8g;<@Fq7Hqi zs?)i(c-0u`KViTp4Hkt@BEK zS(;`Iph|0Nri*Fc3pFBREh7Kg^*~b)&fR45E$w2~o`xTdVTIbob8AI0%0UyyRrLOh za12o6YEUcp;u>_=Qx$KIOU#$py{yqB{B{y0AOl-Ce=Azv$D~Ld&T9Sm_~xDO`WM?| zhVtzpIe8^N_W)&YDs|XnlzOD(&D2=DNaQgo*_`cYNggC`fcJxj$wRjk`ZHQ}1L4{2 zF9>^p>43lv#_f!j*+t(po>yA&Ho2EM3|SokuthZ-J+LgBjSLqx%GXP+_ywgs`sE|R z+n`;d^h)CbS{;)qr+xntHth2zx0)SkNxTLF(eAl%iTVwB;pmTt6%k_Sj>x31;u0R> zSbwZ!hZ#jLmh>JNI8P@&1uu#I{%L_033kbGjg6?LHlO)eN4*&DT|BAN zm0hN1aD{6N=3cAKi$V|J_W;P4&YO2y=`Usrn{QpXcRA{EwDsstd6)?zYrEkJTJ=bk zEppz`>KadU(|XJkm7^zI?;jE95%(T==Uz9``zgRC{ZibZ?ZYy5e)q6z218qcw09j& zJ%1$peB6{HvOjH|tiAArUra=9seQ58JUZWvYxcxv=p*xx*dMG)7kZ@iAC6mmM>v<_OcNxW0sD1g+A<|hq zT=}U_Anbx&)U6+kK5EY*jE)K0C3cQPeL7jOzp_p8>qMV=nZIj%`?^fIsYHUKQ{|kj ziR000cTs$%GB?8^z8q)_e?eAQzR>g3-nqCoVNE~7n|3?*Am4)~d{q)uk zhzh_ei%iJYPV`Yj4-ZZ7f~+MMRMw$iFgi|wdaetCr)ZC)L|6;0R-y+gHxgG#n`27s zrst|6YCjl=7vaJ+hZC5}Ytic^na=V24RWDYedo>0?5^5MNB&ShC0s2c)Av^etNyyc z?(UDH@C~cBuoulmcvwr&sJP$U7fv+lS?vXLBgJZqyJ;?lzmRhz?iY{1TDUmB2HuYq z@2Z}j`lfc*9xD7QFmEvI?UuruLSyX=W8M&jyAUxClUqp{hcbOrD%$S>M3916 zUgvw$z(|9Hzs8ZB4>MT(-b3yeHYzdp&wRMv^;F8+?iBZ@n#^hcdmdd$-#bn;N9-!e zI@2B;eHIcBQt{eYOZRO0RnyDpd#&UPaJldD#cx&c-N_LqZ@=m~4EUFpG@5DJg?yYy z@4vG+EhPPGg6$-D@PzsZSF8!AY+bR4A*AG#{z+!*4=xI~sv7mw`)d}5AdzO6JJ6%b zZu4cIj#DjB-}Znr3bA$?O}#eW*_peFvt%=enOU>QwEA~rp%t$4<_@Ur*b z;N`b9#UpTbv+BpK@7|<+XDU1DJm~Nw57Nh^AKK->w^}|+Pre5<-0ns0P4u(2f%ZJi3`3iBTyn3+LMI`D9y53&Ct8$U; zd*dzL+asBJj0(90g|A#otT;0=&wNI_SZ#PaV|K2dcXZ{?(yCGVvXy03A=;SX&V8R> zQ>q2s8Q<@tQuDfI`X2@aM5MgIyj72tugNemJ9lzE;%6KT{v_pr{299v__2l;l}EWX z@`NS?!_F70T^G+&R{C-C{3RQJO)*Ny{sNbNZOfUNk@gX&^%LVSRUB_WecDN*&gwVY zpsxS8t#!fdNleVIsTE5jy`NU9=4pd7Qy)1(dOvpbCtS3Sd9S_uV(L`s4*}^Le~Xh9 z#FOS`!O|ZWpWO}I13=%KO|_Sou|iwRQ<&A}6M||UDkM|};gzZ4`y*e$nU^u^T$kh2 zXm8Fa{_i+NhGpL-rNj#s7!1w*YE$?f!PDVucoWC{BPtaVyXQL4reYD^Aej z1qzfFcS&$3!QGt}cZxd{hhW7iwomrn@Bf_n-Z|gQcg{>^n91CXuIF)=fJPW(D& zN#`6s|Nq@%Kd#NcUFp4b@OyjLhRz>gP;f9ReMwP%b2gN^-cJZ{tqxW!lXHIk)c-L^ zrFq#je^4IB;|cK|j-DW!7OXXbawJJTVsg^2&`4`gI&nqeFeCo|NIk{NoG~6G4JM7iPu7R^IYwKqyqgBSc zMl7E|vlELTBejWC@OW#4Wek%N5QGe>yDYS08Wzu2>sE~d=gxqCVc?+l!8R4VH6ZUn zPt9>7>xYtw%v>u7^2ur7MGt>ppPPHC+G$j+pIx!IxaitF+Zm`u{B0bJKdFrLO>pF? zCPO{t{&OQe9sorJhkPR8y1yr6(U4pz2+G7%ZwW1upJ#V*iKo&ChpEJBdzz;y0x1$o z+C@1^o$kuvQGLi4L~@9-dS3_s<~DJ=JBXN{_FFkl`}qViKv&w0FIR$6A;B@>ax5>KdqD4|iTAIKt}nOAEoC)% z6qy?>X?<&6!f!Pwb$&OStS=k;vt1xPUgoz<=+?ZB{1IDH#fSG;AH2rVO&Pnc>53zR zc!|T0$O>S~WWe*VeIEgE=<^uKBXc`Xt{bZgWbhAS8v;l2Yu=+&3QXH5l2H^whe6*! z)p@)>x(cuRl&5vyMFP6>4KLEV$p@y*vB(UNUhdaWjisGaR~GPPY8LM}Xnn zH2|HHVV1Ka@E2R$52H+I&}#oG@Ha^Lj8(P&7hF3s2jkz~MnuHw#r{@kks72j#YsZK zjN#b;)neWK+m42Erb4Zw-ywoBn1XCy*43L9KT52$#2`G4HeV7wDk@vwGO>;n9bzpz>D zTfC>U{qLHh4(+_|@`J!`wq}}#B)b$>^UP>Ws>c3kt{q{SQJIeuD}}dG^e3Pp-mmP|O4tOZyd8uN^XV1+4}^l6Dz0aKanOY!x@vF@cH;aw?>S6MH* zy=i2>qXbXv2u5IVOz8;R9}SP-t6SE4LJ6G^0 zcxi+umanWOoOeH5&Eer-1;l)MJ6CK)OquR+%7XTMY!1 z3&;*t&egC4UTHC_H-#=Cl<+0|yWcsM=a*x)QmdE#BC zh@%~EQIGX_ph-=;bCF?_cpk*wSl13TfKyH`>(QJpJF)mY_+BE?mub)S%6x<}>J6eA zBExRwGZcM^BYghCxHzN%G`^<0M18s$(4~fC)2*;Sa!EWH{9uQd|;9GG~%!lBkqjq73kpuUVZbxB|N!K>Rw=7t#?3I z?ByQQMy8)YKB=BV0dEGv`$AvbD1XYG9{2e#1Qo3UV_WcI!kC!7SFf}EKKonC>d!0Y zYeIbj+Wg#sT=1Qb&B^IlWI~cdz7I2R=?=G_9YT84qjpAqpOBM|7ONa4Ee5x#Fh3s*`d(t&itTxEFE_ zZTfh|)0KhnGl#YK90~u0G30y6@)riML%SQ2+VVp4pMD$%lVL`C$On%lzYfGdo|``z z6O~fIGgatx7_j|QX-r_lRdVTQwHDth7KHOcVzzB$!^PsT+lpjeeCU3;U|F%V@; z0DQq*38pf3oXV$$Oi{`YQT*vCzBsOIx@wX58eeX{vAP5eCU~+66$*V=aAkjWp<`IXb+VKHHIJz#W1+qoLDVYR()Pxd@>d2YTW&*7#j zGFnj2$pWX1TdHh;>Ad^w$Um983*48^=-eH;4q`U&e)i0{FB7=`w*Ww3%gvlB$Er#= z_TQt684KcY+9^A96Wn6()3w3j>a6miG5??T>%U2d2)u`H9sQnSm-XFF_A(H#f}ra! zK;wbX>yrzixr`rm9qZHoaSO@2)ZrMEg)Ewap##`K)PaS?O4XVkjPzI<85)mcjbB48 z6-{x~3=0Qbs>wNoGKH6{ar3;!^g6A{JLL$Xoa7r@TB&M019x5yE)_I&EIO0OAV@h&ETGXSVhYCeyb@X#0+WfG(qqrx72AteH(JFDSAjdC?l#;`0&e%>!-}rFlpvYPnNu1xs|F|9(a+o|slM4S3nz zChIMp!C^;wthwT7K9$F83anUny_cu6x+Wy8F2q=IMt(%$J)npO!akw=|di z{BR`p?6(SL0ebY;ST7TvZmQ3^MG0+>fSoIH&ieB<5qyXwCoUhVpbf(=}*#XXfvQa#aH zwgz0x2cW5PN7DHBxEra(=5b$)hG+t1jVQxtrL?tpA>mTG=(hC}DHL2@JtJ32Y9{+t z@K5mawSlEA!?`{nM*$BRgU0WA{MXAkKp^$Qlss{oRF0DAO%L(U(_7bhmEOt+9n-TU z>IJ;#18)GQG~vO44HLJ#gof``gYeFKTsZlW^gT*&)f3`5S}LyBiSSSB`QbK0;zyG3 zi41o?t^M1Qsf{yrwn(b32vTK^yyLV7jxy0U1G`y$^i8AxN3qYz8@5PbJs3l))L{PR zW+8fby|@LBxad7LVrB&r_ET!IXlue&X}lDbEljm_L{n3()cTDTKvfX0|UH+8=Q|Dofa-3nv;+XM3t!5p|0=O~wzblC&C zV{K^-;u#5l{eR~t|4&$qzr&~K3m(Vg-$uU;-T3I_YDbNsL5RIG>)M3Ye)%=L{lF9x z+Hgf)>adz|C3HIH0^B*@u~U)P&g1I5No>ODbhy|Vo86g9%i8Nvdx6384|V>UxFjZL z{np;zI;C-21px~gmMFz_VDvEA(zk{U4=*u-S2UDOO!YOJjV{(*bn zrA$C!jVGxmbWyVT%sl{J#$Glq2vhy7+SBB~t~}x7I*KNUXlt8QVsQKRY8^lsu-jZZ zG82%yl3A-c9*)NRODy)uqj3)bH4%bNP=DLg?licLKF${pcMor$i5C-2kb8E=2@fyP zZpSK5zI`4oHcLzs`nGJ}T*Tuv|0nF|%WQdzJ1SXz(Z zNu{yemNH@wI7wn`dP=L+TxR5~l=&Gbx6P(P)uc^aBiIR2APR6d5H6`$u9mxfc~suf zyeRxYvWGeihVgG@L0=58NOOfpf?LSc6}7IcbmC~_^}i@;>=Kh%w#RCEq!xafpj7={ zTliW((@SA*$G9SGa&dBUdNtg82O7TcQf^h&d)dgE>h<{rQVP?bpxQ|Evb9*#0u-ei z&M27sm8G-1DRB{7tF|ovObjA>o@e$1>I}wM7?_A7VMQAG`1l8*X~fjROLkQ$x@F_c zj=?=^|BvTXNaHDffa{`&rs&ol2^8=bMlqsEEeEnW3RQSyQ4j@(Le zpPM>nfV0$1)pn0Y*^*R}j-PRJbBh5BV)qaG+#jK2D%Xo4`RQ7`0veFuJmtvZkyO9) zxxOAwcDDI+TM0Cwb-`#B@!#G@3rSH1L z=gs-H1F270uWI`{yvFr;=c)3~#PNhN6hGGZ77aK1WIdL>LR*R zN?zb5M-!^12%;4m386FpBQa!P2S>ZWgDzk)W(t3^ZU5p+g>0-}oR(Hr9dEss{B*

R7bMG}!AqIC z2`!m1@MBRbke81|xz`^$Y7h%dzHa7+Rc7=^nGxec=|eXaOv0$x*Vk2X1V>btx&oum z#2aDw6ZAc8rMDU${YfU(h4m?SNMDaI;>f}Rb77*zaTwN$3Tv)tMhE}7%z>7%uR^{? z(K-8PeFs5~S0AL&04;+o6!@kKkBPZ{QOX|140x(PI|nz6bBHd6XnPO~xRUoS%5_%6 z97DcM>h$#vI$o}DbE~R8cNR_`9+l(3iu@H1co*7Vb?J?AXJwazEhKK#P}swnvDnyf z9hmC`8y4|I$`rAnAJ|1c0pfrAa#&ax+8kW9L9c{Saf>uR8wjba6gNT4Fdzk&NQkDe z%JV$22^mcWHktkic?RKmg6#qw(L;s;`t86bq1psLGp$jd$t_5~u6t z^^Lo(QaI^Th|Nqo8k))E4+hI%-w+@ODLU#L_(5Xk((9loigqL}t)|}g8H0zn2Wx2h zgGK1d22nuK=D_A46Nm>%ktABKpYE)9qHol>S`)L+U3GJw?`14VH{^`LB` z!S(0&`nWQsF(dCrpAk(&fFonD{dDN^3+*$lP>Ily?a0WjaXux-arq2-a#^lEqRP>= zNLHPxhq@d^x7T+Zc+akNIPe1m6L`91ZC(d)3ehr1X%iyY;Ll9;#jVK#`f5w>I3r8% z1Q|U%^IYXqq?pAV*eGbjFHy2*C8cd_k>LZlQd;tj{MS+Kq}VumW*SR+3RHk8&BW6% z>1fg+IInvM4rz|LJyJU`2Y+Q1oh>_%zYkmcqY)r;km3X+gM{ceWtXuncFQ84Lm;t_ zCSdw}!*E`y#Ne;V`8i3_L2%~tuX%u}n_lXpFU3lbgy2GtlP<$gQ(@f*J?0O@UuxmV zUbxj~rm|IHas;xlA8<8jqo5#_%{h+)ciLa@SqI^@JQJu?a87Jdt+(YlWxxV+Wml zfb8B?U@6HfqUHqO*J)q1n#9Mt&<2Vv*_q+n#zoKa2gdDHJ8TU@{5xXv`BHT`St7B{oB~U-*YZ9VoMBZP%I)MKmV1e-% z_V(C0fA;aOgt>1k51ypmd{&6Tp(GjNxs5{{5v| zh);zS{3^!Nxliq8lAis^deed6;;@(JvE!qvKKM(*jZNTGuPbn8-NN~R@fSP(s>8e0 zp~1`33$jz;6V4p}EN84K{^ghQ`%|Bj|14|$s5`)*i}}+dlO#*N*zr>fY}_l zBA`u@BPI2l-yC``4;}W}#E?0zt5}1~b7F~dTVLnO;z?)I3g$o47uxu%@#6kmH7qPE z7aQgj4ys4eJCxxx69;8~jVFf91wC>0`O3wA1CG)VwF#=K>Gf~`9MI`lWJ0Pwo2iTe z6sXhm}H5tdk`Z`AbLUh$ogeQ98~fH3ac?N|eD6Ab`1~DDTarZS0{mLwX5= z!h)H=G3z!;wstlN^vd<%^daP^Rma?2NhynVl0FP)cfpx@pQA0iX>+n#_pV2DK~#YB zNRt&waABPgKp`ed@ndhBN#?he8XK_)>rVLRftnr5%Z2Q%^g2>y+7Rs&5_}e1`*x6Q z!Q;#u(tGU=0VvlSy2t~4{qrV$#k$JI6myoN5g_4!O|)gmAnJZU>$Hj*AC4#jiPpK= zUu7YRbaIirIs4!{ox53C(dN6N=5b5=i%v$>o0vm9hGZ7?YGy$WKSmUv9f<<{e9aDo zKqr6LTQl-@$YHeTni(HFJZbx}fb&gvF-*ajZ!SW1_~E&DGP*~l=yaOTb5sjXWsYB^ zyiFuOzS9{2lOl9k-; zf<;FGF8Vl>Fsp+(xyN}-Jq8lBk{md36)7G_=nx~$d`2{Jy4~hz!b|69ItG!1C zd0hh}I5j*=q?Jm|cp?uP8vxx2=fhOyRJ*dh>Gjnfs`33lMu6EtX{Mt+J*DQNH+{uu z$gF+HJF&0Oh=^PzD-(5&y5JJivRz^kzenaLABaY2B|++*lIc9o?&8Fpso3ry+MyTQ|0Qll8EUK z2PD*_)pX9s37H)WCk(WSo`boXHOE`U+r(1A&&X;#I26!_*Ei&6Q=PP;K6ZMQTPi-+j(1amiD1ONDjm63~g&oJT|drXzI;b~uduI~LK%u*-OBDb@f zFPlAY{97Bsj{gt-JPt&BRrrJUzH9&MTljY=SJ#vJFAQ9^gE;3bJAUxsSD_G(S_4cu&oaP1SRtWnXjtWy zFj1)r4RmXBWbX~C&%C)L##UU zE?=5Zl9KIL-UKHR)XIu*gg z-nuD}IBrHrYiyGy0ye7MHA~(Vs;1OXfy)-RKQ%ysPMa@OCKfb|9-ez^``bu7HUjL;N{HIpY{AB| zDJJ28VSWn0$?L%|CfV4=Xci8lcqH!`39Q};d${6A(*2HOc00p3vt6fZYkFTTKOU@H z@3XpdQ(;iIH;Y-62&e08bJr;_SqXM2W(CO05Ts;+cxgaW(n|S|Bnu7O>U3zTnI?aI z@TOGZ+xqNoNcIBLh|60Y$|x2C`-zcwxK-lW@!Z3%{)62RG_KoGL@ zgOSw*MJs3J5V%?C$RZ*qf_XqKCn!mxHBHnI1CfTNp;~XD^u== zPTeUN)v*anChjXdA)lWv5m_u>q=%+ZsJL>ziQO9|&WmC=QeIjlvii9HjF2_SJY8Xn z$-;3{C-Iet*zji$3Ej$)d&P!#JC&9)W|akLdmkTWhIn4Qw<%k4CSr=C&ek85$m4M{ zN7T_4ZS(i3d2kuzmo?a3FyXQZs^ss*LJ+W&v#;{F^zxzKU8Fi^82b|9oINxrO>WZV zH$IyeUh3g85>a<7IMvE21?G(LJwM++&;&J8uie`!>_#GX_XD~!GZkg8mG|~$XKZOG z6B40s_J|vu_oh(xRvXe0G@=xdUC`=xO>I+Bi5n%%j+|wEq;*ALF&4*)_XIUsRe=tB zD&A=7b1LC>i~a8MFARC{x9#nc;#5OJlfxrxn`^!a0pQNS+&5%8qN9Y_Z$^WcFa6w{ zwZLrpNyw*qOGskFE9A4rb$LWJrr#?fTJ?m zhqbQb83E_pquwfPX>jiTh+QlQD#u>~0OZvk$zGZS52!ajnwE98)<&>>4g6=EK_DO7 z?1Dt1B~tb~>sr+WVLw~-qiXQ?=vRdD(x#8G6f|S#U1}@0WM(oPW|%?}N1tC0-% zOmi=3txG4KwzgOKT6%@=edmq^`@Jx`HRMS#O>o9bCv$9CXkJ)TtAzS7xcu92FE1~f zN%4?{50)ozs-W_bp`Nz#$j>K5^|Q-$C1#FqOGVmg*!l!UCE~vH0hW8J5*_xYE*Jee z8E67O|C}@%Yg*Gipi^-2Q@+3e`mF49q|`s2M!s=6L`tra-CAlFGHfkKA(aNFzUqMvk_^VE9j3LW}BHT2g2ns*2Z zbrzFYCi}{prK;nRxFIX5@11Ks3C?iAbxVMzMKuMC#rtl$3ZcceAOBhiIVsgS{=%@_ zY4)Jy<9bqORlm?MgTG|kR)GRGomPtZ@{rUhICZN1{>2Fg73Qi1b-ReMS~K%prKZ_Z zXY`o~p3mJ}?f4u1(^dsy|HyS>+m9Ac!hbg6U@GO$FIJ7e~p55?@+@j5oMpgLx$K;Qf&+ju;@if5KScY(# zu%F*8)OC+qf(?W?;Ov~c)xcL?o*QRvRi7w zwFla5#Km7*qM;2h08bzQ*z^kY3TikC;IHx%MU-L!lhzJZ9P&f6ql5wXs_}aIJr|jw z7f%#jzJ~s?$mI=)i(90|=K)a5J=Ykq%lZqWc3R%7isC8KlD-zA@3`6i<$3HW`jOT6 zIg#d15u5m}tas8lhu74j3PUAEU8BtFhXq7$O#9MG&#$qr0gl$%;cs@ zo73MDFLdKlGO3V67wsTlOWI6rKA&j$NFtmdn$9q-fiHE~wStbQ#hLtR`k!A6e-r$! zB@O`E=V>PQH;o@=LhR%kEj(mDX3SmF%;M3pS6_w>umBYQ?Cg1Lk?8|wOU2oSZJB;)^!v>;^?keOEib%X;DLQ64rS+zv-4;-hZT$? z!#KOUv~jsYCrv)UAjQ`eb=-`U47b%tW2v#u8-CcIHu7&H)9MwsYqUtN@|3EvH}XB1 zKl6r2qWRr_UNvVb4K{hC+nCk%PK5k}BPy=Z{IOdS#gi^wX?EcC8p0f1)t18eR|HD= zEo-oAZ%0DTsYHQhZ%r4ryUEG2q`S!xHqHA*5Cv09y^e~#WDqLfVA;sXAyRa4au(}_ zk;>p;R?3=)LJC1`2&H?t{q^%$2;nxOhC*wWwrCK)c(Y0=FpKmD;1Q!SpxO-1b|(3| zEi343ibGIex1~;MjqpyL%r6pkVSQ{Ux7#qM$|j%Bqh2<>`C~eXcPTi215`j>FvX!5 zMG>kx7)MKNy{x*Ppk-cb>=HKpK3T)#Ce8l#VIt z@nQTZ-Q8?^NMin1f2}UXg*8-MzBZ3HMdn0A ze+?{vOtKqbV=(6-asZEjbB5-x&XGW+#PK+=V@G($?+TU5x1Jv)%ChIhs$4C4<}&82 zEc0MSIpoB~M)JcGXU^$3EHYN2Y2W(xvkoI&{?5ir3v;MS`o{RH;4?g~jFnw-@cq+} zbuAhDUH0$MOlCp$W%czOh<(x)Ar$fK-%`L=z$KID%Rl!g>m`2Ww{Kg?z* z+0K0P7sm0Y@PP2pYej#rFJ|w^8y-nj^qHG}KSrsFIh>uL)dFM41FyzEhxh*d-T%05 z&;h=d7xE1oK7^7~uy1=#e4k}x1PFXQ@BIgk!$XyKTE>QUF-UV(&~Aq56K0nBiAVjA zNA;V39)-LAjXOTlMklcTU;JJF{U(;s>ed{g(-lpJ@rS1?i$|Xi$Zd*aJ zo<}Ey!gNWA?AXj&;cw5dFbgsn(Vyk$Ivo0U<}bQ4LCq|ImwzWW-?d6RX`}n4aMz=A zBpQLyNCx2lv6{+IP84uZ)XEbOA;n%AS2Zb%e5aqGFI<1LyZejfl9yK+GsBn zo5h8By;w1|_r_O@vsG2Cj`0)256-5IBKspsTahSrQ~_)J8QEpX=K8D}AHJE4PI~_K z6km!yd;9@wrg!VYEPd9BCK0e+M~NeGqbgsTV4I9Kf56e?sPW;s@?M%nv1Zq7^!>)h zrQ?2{NopgoqQA|i%A0#|RFtFq$T<5ghwKMshT3Ke``9D|FhA3)_uIra2~r3$!~Awj z(C&o@5;SWdvHcZJN*wrExEqjXQCuLNaB~@*Khm>&(vvW_Cs#V`$W`*7wKug=*Ss9? zqDQ(!B|V>sd3cMiTZ(ilnc{0Pv}leqct_aIZDTDV$Dx%!M?pgc_6_|p!*tekd1BU^ zOC8fk$tgUS%Ou3A&D_Za+?{UA!}pON(2Fg|JI0<>ELaglvyg}Xw&?tiOJe-`s#z5% zC>e$bu;B7Du%%AnAfrs>pB{9Z!}RJ|2m=z3)VT+jA9DP>jMPslej3y)-qMAMuErgA zwn4S8);V!n4QHjCRVH0!tqX)t0Gu=5h!WTJl{<;JJOgy`O8walR6FO{@vg&JUqKR# z@T*{^7xf?8DFfqSCE4}*(saw-MR5o26OFhn1(}oLYJ0W)1I2HPY7@Vzb)pkDJJ=uD zBw6B;$LU?C7QTXw^7~_$h~aPG9IndTIN%GtV#ECEgG7frSM~HM z1A_3Nqx$m#&j`&5^`dp$5g%Q6^j{dMzyFjh8&{rJuv$|F5dkIP${+#x(cTl6Wo@76 zDF&?svg^Mvn(c5jx+;`4*_;I)H+0V{+FCmD*FuuGl%JB@|&ch zDomQ&WIoNKl?KB_%(o%E&t}d|QXc>_{kzQ_)6UW17oZFEDd}N;rD-|uJeK(VaUw1x zPSvv%Z0|Ce5|CHm=oJ90WR;(|;+zuI0Z3M!Az-1AAGLMR`x8USW0mhKsCLIUSz!>C zFiur5;*kg!Gg3TJiIMhpL7MFD>}WMhCXkJgOdn3~EjYsRszmoJQ0M!1f-nn|Kbya; zn5byaEQq8o_QSTWSx#aNji4LNb#VcltpiLfGJ1GLO5jN3 zGbZ`S^`A&)g=x3A2(6Z~l-n*qFL|$l0|$dzeOD+7!Yl&XdXmO^1Grzl2zVEbi5a5v zjWMESs4C@c^V{yxnwl{sq#NUvMI=&pOfNAl(WdM=f&C@M2I^_wH-fgUvBdYH=3Tb? zYNVQ@iRSgh`-Lj_2g++TESy`=@BJI56e@hcmV65~OGl0IdFJ<$np1H-2~7lIVkAcN zhX(^PN2o*ckP_^W*{TL38xv~6*fF+JsuY{7(|jU^EX=KE=9Xl|vZ_}toGL0fp_B?;5;L>bc18`b zn_;D2&T(<_gN%cjx$Yn>>m_;E?XIBx{-@d>VD~io>{3 zVQNhywoIow8d0J>(5Tf(h$XMEN?Jx`Uc0l()p<^l?+ERxp;oN%;3eG*A5>shEW&M? z%7^8)y(7e%K)&3*{-WkqnDMi!&@3Eg+mJ=zmgGOhc({z+XU#)Lm|rv&gf;aD2U}E% z+mf&h+W}zDW}v3a!4waPMY5uek+cWabhHd>T-V1j6|s#_3Rhm8q}(h;I5T)c;(48r zMK%*qvP&~HJ_1jEDmx8vfTetf$+{B+F+-iNCVO1=p9H)x*aE&E_m>2VR?b~GStiq! zRx{DYw=SeC|5i8l9~&TwPKf`y{AYi#HDdH(XYL zUjI%jh(}UAZVnH?T-PYa=S~*=dGuRwOz-xlq+|)$PE@biY8!JtTU&6j9rdpHr+qY& z3AC+Q20hO67#v2dx}^fLc;016*{cwm7?&gpO|j$3`y&$?_qs9{mOoT0#~vYjiH!q& z{Y53#ohZZI2Bq+=Bx16A3-=9L&LCr<#&$rR$41SYvLSMLRg3dHJd7eU^78yVxBA(P z--yDBtV|*Jlu9|_jKw`;P3UDlOq3DuxVOkCrs&BBIG_XnDeRH&L05R&fnc?Z66~&s zzUz&re2Xa-bph6b^w-IAS-B=_$W`Kb2K$Uw*i3{vZ8h`Tw4o{reIvVmbg5ynGa#$@ zk4OQkwwQmxUUbx`Q23sehq(}P>}h0{zL+5CLpn(-;$274kQI#pS#kmQcq!C1H;7=I zS(IY;vrRx!Z#|kG;a|qzn*5?5Goa?J=aJ|!H>z(wd*uP-p%C>}s@0paBm4?}t9Wli zHY8Pj0yaT9{!z(%jhJP6OH#I6`9%cBmJIKZ1CH`zCLukkpMhTv!VlGploa&T0lIkY zvG0JR&UF}Eh1NUTTo+W5%~GT-V$yfokw%I3fXIfr^prsyt0bE<&@E3sp!LpNVGge0<43zz)Az%?-D35UTFDe= zgvzrgnQ>wlCLdjyd5aDBMUONpvX2U&7gKc~-z~9Fa!7UgFg0_#;8hu=c_=54a&wgp z5iy3)uy05x1tadIC-K?&Cy0SUvNUigrO2=0X!MhQ6g&Ijz*6}1oz&j8)C^`9#q}6& zg0nkBnFn0CvL)u()>95ignON4k!LdeNRzc+d)|hy@Xl31och$^XYAtZsfd6zFXIB+>KYj-;+Md>@X!Pi$!iurRiY$g&TUlxzGC~|bP2iL2TOFhj z4~F@)o2;Y5tzH1I!6n#cTecF*68XRkhWNf*Tv3uSlEvqL}meuB}IFD9>i(C%CMSjc9eH$;>(I#a`l% zFEjGW`dGBWz2@Itoid$VtY*r-Wu`Q@YK-(Q^a!gO7@DDZ^6 zy_{R@dpex*_-waSg)zYi@g!;AJJUj6zW8~W!30+!?ccnPvS8+S{r!Vo9IO^wSp6HE zU6U#~n@=qZ(n@n?UDLWJStbeGxRmUIhiYdD`Fjc=U%!!HevRxUJzb;6gn-(@%zlig ziRJrMPj}2l-^yq6)(xguI6Am}PWx(X1mN`$%60eV9wGjqs)C%!aLXN6%072y@M1XR z^VcaM8T^P)zZ2!eDq|sX1|VNlCSm_J?=cNtvhQG4X`G<25RQ6D-*=}wL7<6VRR&b? zX!UOE(cCGiO-JfZLP`>g=fI6bm^V~GqR!jzvlEdwVox<~+@DC|6x9szM+a_8b$B!6 zhaj@gyv|!Fsg7o{6oX7|Gd|S+yz0ACTWNILJ=y3EF8C_Y7dX5pZ)Mn=N~S_mBYz+9 z@=Qm{fM4USDFq%lIX_L4IB#)-?qeV5mMbDjf875}G6IM`>GXT)9X>k*rZmTBq`;b? zt}?+l>jULAxK{|?U+hxTj|vpsnnrPTM@kiKVXuf8q|=iqlcy4=ff9ltmdKayH!_HO zMnrbHu4Zf+_wUBh?avkdy~w6SQ*GZfejUB??JyPOmAYv5rRy;BPmRE$)(#oz6Z}~I z*VO&P{d+$XjU3pN^D;-y$Fb1XhFA5}SeQ_T4*Gon$|*p!nKYIkgMq z45o@1PZ+Bwk*GvdTM5LQ2;s_mdT!a!L>klgBWSInMo|u%JfVpLVnJadQ`E*X*_>Qu zu>;-(9hC})Y#)^lQtpj(xKDJPsj6zjGK`Fj9BCvR6395^CV@Nq5jOmX5+LPywd($t z)$0*%P3q8*pLxOyv3fjPLs`mhe_`NlPkk$EtXCBD6$~m6yx#|-Q=?A$9OWY|J!ffx zG1$8{rR1>Y(!tgB%*?!!;_r4dA5hdcUyWtt|CsSb(kn}(x=ZtF?$pr$bUHUFNgxwb z^HT&HeW}ahon@Sr@K|s{?SktCnCMyzqnwZ%RnBVm!k+#YG@c{8v6_lC3lAL4E8x^k5>* zv~T{Aj&Xt;>i2w&YO{N7^JJ-!r|lViKmlQrr~bi0o6!Ai@y4Gv|2FY9Q=S{sT^Wzl zESafItd#>-aVxX7mEy+JERaO{5FY6FhKwRm?gWjDT$i>kjxujY?CZhh(|;ZS40CPgYgp@F82WoN?#l(mk2|5GPdLk790mdTcT8fIfGChM z;KxT&Ld1dWcdA14xidEHTLooZfAp*>9A>2wUo9!kbHRw|DlI58UjBJ5F79m#X#GvP zQs}y@!EpG{4~7{b($`lDx6bHTPdy3DneQZaTTW7aXJw;)^)u$TBn)b>Lfe&4ha=B_ zV%E<{SjxMI&9Xy_*e64a1agGmVG0T7{&&sfKV=mxmgR~nNhkF9KOEw+vZbhdamkJ; zv;SxgF*bVMp<$6nx>{f6sA%sN{m|pj_I>lTcs5>=ySx*5)QB0e9{Rspnjc9e_^bB) zzV%wQbN+(;6!(inS3;BE+4@TQ2ll_m+!rt3b(BBOMxSdd`W)&Tmj0NK#QjcuIC~@< z`<$&}U3|IwF;>S}g=UoDNz~`X{l$(uzm_Q}C-D_OllRKDjSYXi?md>rogVo+Pu;^_wJo;BkChit%4HuK#jN{=>Rx$Twb>Im)4@G5s?< zByIKE+;2HChab0pVMI!hm_PbQgx?*p(7ig-MZ)1`XS-o-0h0N9u7`$fU)0_8pZmt0 zfUi*s>ZT<$^}VM@abGpH`6A^W;l8FZtq%Zj5fm22Y|D5Y!QbxA5hIXT9PfR5snm_x zYt#vPa)t+&lRRySwI?cCtn@B>oTW4vBBln(7G$hsJ_1x<`KWIqc2r^}(X9VeeQO59 zJ7lf$E;M1aV0z9LqVwcfuvvezT#3C}d>gT8wVZmhJ~{V3c2dcHPg8r;YTmJZD@3ba ztj$b9_Yy9H?Bh^wrp+`-lSCJw#i?co_hc?|(Xul%*mj|tHYG{s><9I3P_gEoMe+_8 zr>70tTG;T10>=&XX9~ti^C7nYBZ1w(8A#^d%vaKIth@c9PG@xDRU#W*JDnmJ;POPv<|2b-Mm|khnN()F zPwW{&;5G4*-RmISZFc*P&xhQ{ny@z9gy{vb@uT9xvs`o-(~URjM4$6`U%j2&dVx!C zgyh;b1~T-O17(Tjej@{bxDV?8QG^KytPMPQnDaAtsLYm%BqIh!e$}gml9VMB)!$F= zgX)EMlZ`%h*4y_sXxm?FhH4q57LR6yf!3{-YrDUQ6In?r~&W&RWrVcMPWl&r@12jvLCY-dAIrIe5+1;QwV??rWJjcnp%uw>t^yi&#+i2f(IKF zC0S%_O$^r=T|M*GS?}aPk4p(+)ssyd@%kyDBCUX~$RNrQJ4THGU|Z5Kvl z#dmonB}Kaxke4^yEB+`vLS)kTdz2!M40-I5Z!>E0*A!btZv@~E_mty7f%(9jJ$L9a zFokGnH^<6G)1ah)U2%AN4zR{KtzN_6xV`eJP4xQ~V9&pE>_nvi>6Xq5SU~kF<$vz#n6%i2c@Yc531f~k1PHEdtfYEB#)xOive@RPS1gyv8>iUFJjr8Y|loU-Dhh< z^VEziDn!4fD=xgHpUIJHe~|gCF&NX|PK75Qgj?lQHYbc;PaFs-O_?hS3yn$=a0P27 z6*^irj2l6+99>;XH?5pnT#k8`B@!NR?Wdg^5p4a-63szb3VA2;V#5WW`U^DJUyp|; z-aS;P&izq%iU112I8wB_dljG4dmH}9P};JA@ZCt9YQUGI;ni4Zarf<$H_ z)WRoJWTDh4UUgT;>ojE1fP-&k&bAKvD{fK%r&>ho!3dTDGH*Q!^FqNXUFzK>0wVyj zAtvMBZjMK+(~uzghCU;z=r*W=aQ}xZm3&=c0j774l@V9{Sbcvake^7w!S9<@TEXC_{ zk~CEL;DkxM;h0@^ItbZsn9&56WV%!97iEWqGiV)*q&>m_QnHOt^l4Ys2^7uq=^dgC zk(?E?-&b#?yDnz(k(Lg{*@=lNIn`<=bYW<@&xyPQfUD@>;23jB?(&e7h~A3gKJ*+p0;YVK{F{)GPrU8k`GKYPOicWA7w6S`S~w{^jdqn_vlF2gtYO%|zd!X8#c!m?-uoGkFfZu0LP6ecE|SjE+fd05TR8vzbjd( znI|g)DR~b#+kQW{h}BX5wq#@2jcJ=T6+K!Jn|;1$77IFF?>MkHAG9y)5u@f5WQA=O(uD6G>BD&d5R(gr+mdp?z(yXPXpX!R5nRc6!y8xJGOpUmUSL zUpuhY*Fk>N$GYKN9XM~goX+-zVz$ax9(EjJCWWNF2z4Q`w<CU(n#}Gl1?_*EgA_=wFl-vy<9$ywv#!pQ~X`}wCiI3Jk1IXkhdDt6&~Ao@P$to zs5dQxrsH&a{LVTP?*(w1m}BL~$(&^U*vEVhKTKo4TU4;mO(%;}IKp95b7E;0QB}oO z=7*Tdvx;_hw&&jJRF8a;-fn4$x8sa^16xM0S*!Dv)CVx_ZwU{|d1ez`dSroQk)6&{ zgW%rLeF?)u0k)p7WhC7d!ak8lvH0;x$Lgs6JSm5x#KmlKX9k3u_&K#)BkX5tYC|_N zGa#mfGa%@{xV{7fa712vgUT`F&d-t@3UWU4`dkO8uEnU%(w(y}Zk3*;XenHO9XG3W z4B2@dnowaGy>zHWpAAc?elcQP;0(o$!-4FxMN6Cwc9DxTrL$yk5`F zQ7X5do>xB@m@JzdXXDZW-u;6v_JIiEijB)^ksGWP`1ro>O5%sCMoVQiBJ zHk6)X#2TfO)w90o*Zg~L;(I73l)}x`?`@bisc~BCsOhZrSs1(}|DbrCnKXp?4~ptE z-du7u-vf54-wgCh*;2>+U_t-y%qHK*h5%jOt%!LaAK>@QLcs(MZ4$*YE!`hE{Xkt> zM=W$A3UN{XFyA0wv$&WGg!BWgTTb7c5J)ihvw`M2^y6jPGIKz3GekXJgom*HZb#i#p#?$^_eQYubNaLjKg1w)ag>`>K$iPRCEQlOdTs$t7lSA`)f? z)Ec@-`{*`ylc=ysh&`UP$BjdD_rPy0Ql+_}$)C>{Lxck(lwjLC1<{JCR*729Ozt2` zhVlvXyD7SQLN>yGP#EhAqWv8_s^JSwQ28WpU(TWfU3^Ky#<&hE&_U8j2AP}bB5wrf zjRR9J2=|s)tI{c=9-+q{L}*~DU*JVjQx*fIJF3ntm~kR&-~E-SePbYzU0@dWu^{~Z zn0%`uYi;7PoQ*=CyS6st>Iv|DJ(I|_@8Cx{VbQj2E3_Nn2fl|JFsp2VJUXOd&W6P%Ip%#9v--rHiNLgu zyo0dlOlGl+(u4;1oI;w~e^j~UM_|aO0v;K4gTMM_O~`!#O)Np_wn``jTweDNis=wp zItg!IkDoxQs>>vQvSE(XMI5)9iM~(dMSeFA3!4K)H5*>XQa!6xc1k>{?k89m;CG=9psua#%sOm<|fQi)8Ja~6B${`CW zBgWxW%3-ijJA-n$-&Ir}?sh>g-V?UfE&hiEhh*~F+tdP8M|p7Il2W8eV2}9a&d>%1U6GKQ3HzF3OLSwQ7_$w*^>A#d@&>=Zl^4sW-*P(SO?^3pH>kg0++Vp$brH##U@m zyrcOK%7RsfLrfzjR^%H_%{*{Y1p;_8AjE4m_o23Nh0CGoZ8RFezBJg;GgSPNWI-fr z)qgC~m8__>Z0b5Q=kK$FQnObss>6I<6B{pS1Yl+>;yx8WBduW8EEEl7A*}?%(qai@ z`uMuMey)?I-?NNFTT!}D$KbwVb|JcDXO-XN9~67XcLdd`s)v)@a-Zqe3u1Gx&G*$> z`InD>B*3Q99|lw>ts9RuOur$O5kwI`4kiFnMa=}1zM{FPB``~_DGU08i3UOF??ao- zXdv|Yilm0r;M0tIE zP*2}_p2%;@BnHZ!G*+0_r!q~xj(=4D+@6=zG&W{)%AK-+g-Xe?NsSB?%g?hZpZ>pdK!m(Px7a@@e*e)RQCP|0 z+d~N}Pf{cg{-gkaO47HZU%6GH|^1LPDZeBOxPWd0WB>0H4>!t8>uE}XslA`^l&F(~SLX5(HUNuxY*?=s$ zarHg99A>8cubP?uI~-_t z_uT0e@t>a)g~NOKYX8Y86lm7}tp^mB_}S#M>Ax-l6rxW#xpDt}0eb)CW56I8QU@)m ztw5P+Le!&j`#;w*kgswE5LW>IS5WIw;L*aVT2n{Gnl9xu8*-Ec>61GO zgZ#9ZtguwC?Oul@*6L%|L^5&Alpm8(c)++4Nn|(O;1}%J7*=$)Go7HA-#u~oe0*ZV zJ4qqtvaVXTi7T!o44Mi)-5l0ORgkb5hMlYi%_14-KBis#TGt69SUsX)g=bfM-mcwa z^o`<|VdYfILCe;#+1R*9=9rmzr}h+o zoQF!&Mn;~6UNKnv$HzNZEb``FGa%Um7cf^7-QDPQ(0+ZMN;*D|rquvkvL`Pyl|A!In~ri*jOO z0~0m;HR_ACUff`_40pmGXgR5_Zr3@Jy;{i0cKO)5Q84VH_FAusad8*qJaoaWrq5nx zHg9CSs@y#wO~Zf58_H{=Rmh*VD+peXeK$csKTkg-!@rZ_C-}=3p;VIf{T zQWqu#Lr@kBKx?G!XIof(?X*nY+5ZWt`kHpi-lCpfi%cSIe|kEp(dU9U{ZvApeX=Lp zmDP3{HB#z5uUQ&H=L=_ll*Y4avRGxTSr-o$7qRW}=5xiRp90P+{Ld=2ck>n1S$!Id zU=wWCaKGxV-lu;lnFoi?=+t-eo#YM;4r$@YVwaI;&97Xtt z!{e^q&^)9yqJ4dwbJ*sog3v}j^)1j9r-&#Fg?@V;2WP?V6GU`!!Z?j5HeR1;0ji?| z?!M61F`ZxLhX_3ERCiE$>dD+yvBtf)Gr^DcEli;D<>((Tn)hQw*crA%!5LdItO4)S zW@ENn*>Rbe071$!p#$ZU-wIuA8_oYte0)AQaDCPRdcw0NIhOcrXLOvdd;uQ#g;$g{ z>F%*c8KDH9Yv}0gBbBSc+g5kS^5C4gQi6|r6jn`)smdVp+{pKWi&; zRLqmd>Y7ZtEN>^Rk6u^J&qs8>B`mMSuSbpKO5QN-wh`EA&Ao4wPSge~ohMWPuCp*- z*BJ!fGu)nZyyoA}rp^~>)eNVIfdQkPbIe-SP2x#?*LY6x>=bn@C`Zoqx2(2TWPW8Z zn@5&s>xCWQt7W9sGUa1%zNe8ZF%yNC$SB3Xa~5u$AqwTa;@MAx%_?&a>7p5|%&G0L!K!+IQ-WtCx8_PoZh`)#y>VT5q%_%aqY>fx@ zE8rea4#?gtg17lc^Ny@i{yxVXb?i5CWbV@#4T_$VpQRpp%6cl3K`v&W z$4*y`OWR+bH2OlSSU_?kS0y?|YTP>GR@IH4ZN}lLgQ)^j}37b?XjWB!b$y&V|AP(j|)0mO_y6HyIXcgZho1`$?H(qTZzK5oD|8koZ0y*n(XZd zs~p%zlMECWj1`Er`Z_9Jj!KZtAHbccoU6lw60`6EM#=j05}E?O_0 ziuwg5(qWn_XC+r{^{L_IKfu(kvrLTX5EV?|&mb4@aw!>fY zt$Mh{Dp_8XD|vLN_i)~UgDZmIY*o0hC_F)wIob=FrG)=Q!UZ-ogHT#c;i}oU*U`rvS#0=K=hWRIh zZ7OZ#sFA9nZ%ifx#6qS#KKfEeMv+07%6u;|5nqC;#7!n0H6(E(#K>M@0Uhv4L0vkf zMEjR(5%Wa36pWx|bLj6gonZ!_B0lUhQ$5q3Zx~=Zc~*<2Gb>L$hBK%q0%^F?4#6xh z=OMuW6|#bSXc6pE>YOC%fuVLFY?Fk7x2WuYi|wSOGmL|Y*_k!~w~Ss}p+o5GVN4B~ zG7R*8o0;V~n;*+YJrY_s<`-3(?Er!;ah%RvQlLi_#;;BaZQ2{fJ#+$k-)MgV+(OS_ zuIoWYVV_)`3--v;;c%C9T$_fIH#TB-&qecg!JI!{@f>_Kl_h7iRO#g zI+f7XDa_w%*=tluHkRTWmSf#hcA_%C5KDSREG9-)u-Y>yx%Uhi#{f}pMH zU87mt-o>iTi|d5PztbFP8cn&KR0P9*q3)X)8q2vU})TMopZ4c@VSd55Ahp+8BSrRzx|TkdLhf@XR=TbO!No>n&W z2vz}JEl5XzB~A-|GTmHy0&GuLNU-U|Wq;`EK68s;I=Fmp-9=nb!HyfRDy^b;^kUbB zmu^|G)_eK*OY^z7Q(a3gDY7!y{_aPR%MwJ`*W}Y+85`eFh!9QbJ*rCUt2x5k)2F%@ zc8ZOy7rUhry8`m8fabngGk@{pfFVD51>4-1reDDQN(fOVfp=5j(Wi&|J3Ke_jNLa@ zDR*v{1fcRkjff2c(-Y6+eTy69Wo{fJ8$t(W6?-49rdc#wUS}At5l-x?V__dF;sg9CAH_+h27zj-aN7EbHh0dLCnwv7 z5msrbJ}!Jz+QHY~kD0?J@|QX6bans?Upv{(H7_cOAYYrrH8UGe=|dE%YE4B?8?-&g zSigJ`IwZC?7gcX~h!!iRb|_ZBV8ygcA}i^B$(wAw&?H~lk+*UnNLoG_@AzFp>S;gH zV4>jrz>2UZN?lmhE$Hn5Q9v^rah4LZh{Mq?vz_oVNS*m2nJAwukvRF46X*P&T>^R* z>DX54%H8VB9dbw<_Xp27hAvX3@!jb%HoN}dB+bjL#$p2{Bs%V!BJUrozXb%IW3Svy zZ+E5@CqGW7KhR$RALkLby8Q@hLH#ewjOXp)p9Gc9KQ0L{Z~udG(B0^5#1<2zu$pbR z2Q}y8WifwHjwZ3C(x@3|zY-ZLy8H)4TxPq==W9%|~1Gg2+G~16z_TgTr%4x|A9mEi9R_4i}gb>k1 zeNAj=AW;%rsEX|s>Gg!Zbm16-=@Riemh%pAY*+ZmPa!C8Kt2G?ztHBq(|!s^ARRtK z8hG{2^QZ$dmIV1w6sCSxC>^mmqgghY%qE)hB0aupe#D@R{6{$BetS|#1vP>*e@BPZ zpvFY#fKuo(@H*hU%}1_2Mzl%-85O#UgsB02TmH7D{bb8?FVT%BuRYSvR!ziV`(Dy% z^f#=ac;X|=Y&x!RoLJ+N(k)d_tXftkBS#umY96risp*`$+id_A& z&sK(WNf#<-5)}V1kgP2q{n~)}(~Vk`cHVYw{WV7N$s27V2HIQIel8~6(t%v15`fni z|N4l5EF5U0NgNp|WzwvC1alW;syc0Prb<04*!O8ytLq zh4ErIlmnI^y~>!J0?HxCh%vHyKqkGmyg_WLMQq;hW8}I~*K0 z8P;Xo@CUKy%b1`?D1En@U!leiIPcs^_N#9|L)(UEI?xOO7Y|!a72&KawBfun;n46j za747WP@k=kvI=47l8&tVEzjG@mzYRwXZ6UKpmj3iRo^vV1{%x0MJ(eI;5YYH&|Jp&fqOut4$`D`Z&897!J z;?MNiYwlWf^aej>#Jo6fus)fuf8HJRkN_Ooy}OT6zmEUafhhNq(|dX#j(=#Tw45(T zV-)!TKK_svZVzcHS5aAXCpI56u(P)bv{HaJV$eAY#0IS+Yt%>j?K`@1L%MdV-D~l{ zd4y;u9=oCh;Wq2YVl>CY)&=r6Dux$4{@k!AXzzQRdwFNQSgo})2`8}dcqghH=}wP` z_i`QnqGV8wEfY5(nNSl4nty4eJ|ul5Mu)IB=9xf34-_j%tZP($J!|am#l8C2D{-aM zy}4IX(o))2-cA;eI5&?6;>)*coW>RRtRMq;BYRVdP+ma#z8wpM5m}pAHDVBv!0$Ob z121rZ;xZYTw6znmgQa1hBqTa3q(Wh3>41gQieEn&yz^plJF>si`wt?M_r^U^c9I2i z%{A!VuCRPP6z7+d#h;JzL_>v%luz$b$B>0L;S}$d)T^i0nl}EY`2T@j>Ezfj!%*0; z)teuEU;kGjG-bn)cGq17VAI{yQ&n$ngYH!mBmPdRmdG%){f@_PPFP}7lGD$=o(-AxJlZ;Hb3-`L}SBNNjVldJnjzd=A? z*1u7z7fg~=-}}qi@tAhdBT92q&7vYzbSHUGNfZ7DvV~H$Y`NV(XPGm@vTjq~ql(U7 z+y^+Aixipld<%4c5$t3UkWb?$J5$wueDzu3Gof9YiCF-ejbN5TWLMaecFP#8IaoSOT!GsgmQbLSMS^FvCaN%8^*kQa(N~|`A&o7a{B%?a$<#(!`+hdKdJO`4kDo1x_!QIW(UC6*wi|mu3m)roE&Ru#dagt?l8VeGSy= z*7=e{OByQE-+G5)ZVPHoA(bcWXvJ7r`YHJ2G(}Tk_Mf>L-~1Cvzdx?qs*N+PNTuY? zHHEH+sn5##-+}i;IKNKh9Z*{soPjm|eHR0YI``7zrxDnWYA9KtcDLXDlH{77X~r&k z|NUHA>)#S3VNqgJbvG>(sI=zOyX5C39CPX1F~TDXi#vC*N;}G=d0Cn zv}E0$lGwkiqzctB<5BOkhFnyA2ASVq*gKV^yCL`l9Yd3;24jDH!Igey_kW9m$OK`{ zBm^f3jS81#c5E@93F5}&BP6I{m#(mU5*^)*0)>qQWcw)9O=g?=a;KAaQP9iBdzwnL zIZ?M{p+H^Cw7QCXx&naZ6*8IUL z$)h&>QAfXV)0-w{{&gp>gx>u-E{3=Qr8TmWnRFAxovXr^adc5`$A3q(12feXj@*%`e?dPP_yDeM_2NGb~+{MAfUay z{YuAVv$2Z(LSKDC1X7^qmSQ(=jL@;z9<}$=*_72xn`Oh3cVrrRJC4O?+LIJcA>DTj zrUvRWhQ8}mfJ#XkGReZubQg} zY5WJJ3MzJWdr@hsCEwQZNHjKkwp!4>{$@gSU$ySpe%QeDxPcQ`vgpiKc4xNz$(hsD zY#~KAY!nQAWj~bYX4T89Vhq)@jj>JA;c5ObwMut99K0ucZC-y8q?;P zk;c`GOo@~tmq44@W<{;%MS`WZ+s$G5nnuo=3gCrXXyp9G$n;WG%gTGcnAQtVdTv|M z7n?aHrR`)xy8xXM9g_{u!n=Z}`T5x@S24MI-~qAe_+2o*+3g?hbVrdf^r$F0y&g+nP2~BlZy%nb@o-&vv7rD_Un>JRO>YSR7!m1> z;SKC{D+)4=^KUl^U9*;fbB@!;9S&IYKg0DuJ}wB$@TW|v9yzoMi?+6P^eH0jbdlfX z`SRf4aA7Sb|F$KAYk3#<>&WV>DffiAc5MyBW&6um3!dH7+}x~`grfHF!tcKPQ7hO{ zlGu?SB2#&bY=B8tB}S8GYw-aZ@8nb)Av9}aAOuGox%MlIN)Y*AILPWmiFZ#NOqARB zYHXFYt0I}yw%7<9?d$mrILYxP>y7_WoiV*m=)r+x5`x)lMB&59`aYhcyhR_@<08?L zvE9-EmP{aef78+iPPUmpvha|YWOCkE=uIqt$ZMIIPNNEO06MtDkl`s?K2zkj5MaAcq zLG~`(r?JDa8~aCV)H!Pk+j{%Q8yoqQpvE_#a@VkA2Jer-Kw$Cc5?CEX!LAdv)#8*I zm4r_GI6 zos%A3;*U;08LL-0QAum99}`hRC`_@~zu{0gjydUEf_XW$GpI}6xxxNoCJ@#hc3H83 zNBe5Y3ek?GJNu&4LJ=DZMJfK;$lo8JJN0zjahF}l7=>S93rvKExv?|KSrPaOMW|9) z&ihei+h>W&w}-xq)i2ruYiz`F<=1lv++vGJg~%KGxsO7o8!D^&Fg2-7W)R!ztYChi4^^L-Z7 z=_kkzD6Zej}b0VDP9EUNG1Nuc$>OSc~ReWvGu7sE5|m-{3nBVvu89M$E-YOhGi zr6coJG?jep54$Liw-!x*Ii|vW<3E1)Y`m~t+ORWz2(B>1!yWY!YMtY*{gt)4PS~AU zw@9J<+ak$|S=~~pyqu)V3CmPhUPChG;gljj(Q-#QZ4Atg#p_u`h) z<-71Wzp7ibwu6IH=FOJ>*+rZtPu@AaL0i*hZ6lyk^@Uh?nn|Nrsm4Y0S#HE0wAo=U zE%Nd`Kriav(J7xjryld=fP6qStglsKt^5#KTi%z*_4js}8+3_^Ir zwX;;n;Z-#%9nwTo!+E2N}n z-a<;0%CK=;5Ix$OMTV!EpIQ2MI**=2ST{eYkBWX~T=@G_tlM#vAf3$I7{kJx6~EyT zCA0Y)j7Ee49%H0fq=U1ojO_Mu2D$Sy52=31SYo+Op(?VWUhKKXC$4zKjL31<*k>W@ zui72=z>`sm;Is2ovSNA38S8=V*3Eiep>PQZ>bWXNEsQGhc2ItsP zt@ICZg1eJZS@YuS(2U9s(>ddFIhbdr4aDNLLPTwdDG)QI)iI)j!CK!k0o{FxG00TQ z%EkG`#8eG2Mh+h5l2{oHg_xjEs@O~XDx$~Xt{o=J0^|TjOo|5)z#genIwMJO+H`t3 zP@c}j6tWTS2h7M`V%W68GM3*nM<__49nz>i)tI|1pY`*}Bg>zbkKb7hEg_tK0&lX_ zqA7BOn%7!3cjI%IVWnf#&?R<4R!R_6$sf@XZuBobj7|>TA7t;B1AUfBUe7cxw&>XP z@?6n<8j#4i@=a@Cn?wueu4@YGeV{QwuNCYTL+Un?h=A$w;ysRYpFbqCPuamXRu5cB z{kt|5^fjZ4aw8I5UyQ7`z3L1N4SIQ7U7==o6BjO9QY24-(^Cxo&bwS`?*h3uNNxyT zg=--MGa}UfmM4tn&yzBV>SJco$c(YlkmIuq#(I`O7zzD28RXkKS=M=vyWN(}Ol?GT zfh*@MobKT)ngiLC&z3vx)5gBD75$D3UZ4297#8Ga5VkB$x{NR(huLg-Df9iY47LUe1D22XGypA@JzOmYn zGX%y8V#i|KmWP%fjL3f>YJ?KX42D}}YZOFs;s$Kk3^m*Q7FFStZ5U;HnUKG})ZVSp zi0pn>CmlZ9zr8B~ALf8d@ve&sy1Y*>w_6VX19h>^}V@bN>``QH9nIKXG@PZzJ-zI|On4tczlk zaGJW3qj8EW)jCK5gImunY;?>*N(u8p1=m*QWKTY^!gVRc78 zxtc$DZ}P=8LWrH%#HXDrN&i7v_3h?AxWeHO6MEaY1r7K(fE}w_HC5THxo2`(AJvzy z^eY9|T|$)~Slo|=Z&s`V0~yarhg!Bco=owE)##=9Rm10%Y&OiOWW=tWD5pPc&;6o& z*5;~Fe+JBD8zbn3AeFQ1(>btm&P0@&fAaPKzM3iYf{F(8B$FX63J$ zqF9tn0nHU^{IC5e>gq57-N-Xw3I^@aoli2g>!8$d8VM&9nV^O zPZDw@fE;S7z2?%%9SRfD!Do`3hoQZi!Z!0lcs;IazI+LZ0c=H-*PsZzzmo$vZq5i* z`I%8D@Q5Qi!O-jGV&8C1Q2Sk4Y8(*VCuChkl}-OE?#79+Q8!WY=pbX1nn7mR+lJ(* zF!cTP!T03kvQEO;u|?6`r3|II_IV!((h8*GMNTz=w?E&dPZVq^-gJvAm{;h@`zMx_ zcI6l_q6!a|8~BmnqbMl6kVBL)?Kt*+N&j#BMCSDRo8DtS8XZv{L8JgvsL*kAI9-fP z$)HZO=&vyTI{PaW{geVk8Bo?))49N_jcx4#b!9m=aG*mj%C`J)m#lA(+ny=Io-raK zj7I$t9jWQNrW^{`|FON6GlmrJb5*R@>WWmZTZV{A?hEPerw|0h3OWZVd51_ zfSSp1RMQyx#tuMJn^I{U5*ZO%bNA&hDof6?liT{w-H5^s&ZiwTX~k{_pXcTWQ+w4_ z?RykG#fu}sE~G3Ndw(eC4h2PlB|*$LS50XhUz7dwHvyOVNm?erhZNecQ#71$&uLLG zxSm+t~dg2J|+oLkMT~&*Zo==f~kbuZ{HCX`BNL5cD6nWT`c1( z0wGI&$EZV-4@qh%Yc_}fZ&E3ckug^-=NEYr__tvX2CMG5LUaF}ud~=jn8s)_Hfo1q2 zgHLDw7rJu8pIks&_A%M*Vkby$(8>vph*7{;HuXEbH5ZtTM?XiUW6Z(4H!~9c`;its zVEcF=?9pW?cj>x6P^<7_cV|Ed);qiXfSY?hQPkcniYZ? zj%ikqr7EPLE6CDBmfXQ2$LPih*yLuT2M|$#ou^}Uhm@4D31Xsa4pMXqu5b|ZrQPjD zc6tJH*38(&VjAUQswptpN_mL-=682PL9061jLp37F~@@J5GtqNk3|5V8|{5R)A^Jr zN8Q@i`{g+o;lY;%+n?@EmttSv$j#u9Nl!nhI<8Di@0NFninbR^faTQWz`nkb5)u-2 z@}!D$;k$>`i^aw&8K?L9sl>2wV|F-?UA%f3T??6&YHjdRD3;@Gep3@&ly`R%v)G5L z5B!bxD$#v2>4|%e7q6Xj_1nF!RP?!T@&iYfBo>mKo)|AL9~Y2?a`?R0G6K2Y+CP|r zlQ+GRnE#_`InP#ZtyF_(dRR3pIpkZ1Hqj|}T@VsqDu(W$S$xhmem&|@XNaxIL|4Mh zKt53@Ab6AW)bh7oBLI4*Q&Qi zhLNad<>awNT!dcJeV-(n%$O!3#zM9rRMf`q2EXD~RK{xF)&8ciSC&%`#_etNEH89v zGO3PUMp9_u(~A3Zr_(A2=teg3^V&YIOTS$N4A|DVm~3VnXM2{VZf1Y+^<22u6P4lo z>8|SfcbN4C4n;=K=YF~9_rjXZX4(xzt}|FP8nyY04Bg6@8%V+cPbubZkC25E3tSJP zHL@8jsU5|iI$FO>GN5D)OxrnWnw~NFa)HaK4b>xuoopD>t_tj7C$b}uZTyy=31D=x z`BOyGjOIYahfE#a)e8gU$n+)#r?2mV=qg1=?VcOi-Dd|$kc5D~XLMh-SG=n&Rk8kb zM&;7)=9x8~5oIDOh$~}kbs$~-8ir=*U!_(k>BL|V&ky@G%gCqbN#<}+Bl~4tprSm+ z-Z;G}W2t@Yl@W2ks?lfus+(wO6V-#G{60*lR&lTAzI4`Z&sS_O6*Zan1rFZ;iG17|h6P^~3V)YANEz+to8aKQ6BRtv74sE}h~ z7ltTVOAG5S6|nzSWa}!Qiwb6(UGoy*RPb%T0<~{&BmYL#muzPXEY>rbn`562-uGE0 zwTutV8TYvJfE<5vtdo7NxiUik2Zi|Z(Cqop=W&7I!}jYuCO{aAXe-f^qRvS>IGB=o zgw=5Igvr6@&|axb#`-5+`lzIoAOM`fhLd{Iz9ll2X8NJu+ajH9`({03Mg6G75L;&E zBvxijPdmN(D-bbl)w-wzXFVceXOkILa%nFAmP}uM$ff%Z$orG}U}jl?k@3V6oHP(G z6=(j^;C3r0VavyT@9*>7PYQ{Tyyosu-w);TguDkyo?aoe@|Zov)^5BbO_Za^S5wbz z74J~tMW+y8roINZg;cE{WYjUiq=egAgpguBF6aWa*V?tkFy|rSF-)jO=kF zGw^0P@v>ugCzE&vV$@b3s5hC@T7x@8) zz+T5mEN|F0G43pS80v&A8pI()Q+`Mr`Y)qh}k-y35-z<2czO0N;RB!)`UF%7|_i4XQr0XC3ryI zpd36VG*6=O**ReRifW{&2YEKTYkYrPTKImEv&zej>BEg`+V!tiZQx5Qh*pVWnr3tmJ6 zHP}$u@hAR5kFGi=C`b#{5V^%C^Nm~%*5q#fsw`pWQ$(|q0}?Vqqt7YLV&SQ zfxDtzv;0G!CyP_arUfE~?cuv>y69p>xsh0!_#aEvx~i?>>Mm#a zF<{EYwG?(Z(E3dS*SBvGw8%#&Et8$E5*NK#f@Tt%@yTW~mDpFIUBQtk@W!*M+#^-0VmX~WR&?24w@l6i~)gHq8h-vy*ClW;6Cfn^K@@SP%i z+U7XBe^gdLHCK(d{OCWutG_sk^jq(&r`$lKCxp)TY74S~=1^a-gCuKPNfXewS!f*(K-YP6x8FU^c_eK^%K1L3(j}fB&<;D zfxi<5hj2<47zl#8ZdN9TmvPfpX?T*nitLJ6GCo6vX$~^X4TgWaH{bRW_e$U#WUH#S z!WHQl5uAu!f;}h*rIDPj$5WI@8#@~0B&yjMQbvAIwxL~3-jrol(SqB@0q^Ki-1z=0 zX48$n_G*yP2D($o5Mo;Tg~y}XzQi#9N<8cuS!Ad4Vk1{im2psS*G3DLrexL61TUlZ zz01&|@oxE1(=&zEC7o%QooAwOSjUWC3saA4;H|<^MMgGaHi2%yL)iuXVh3RVSA;VY z6CDPMYC*-zyJbB|qsdr(m4Yzm2QI#b5Y;c)TX^e>eVMvti^Z0|kz9|EYF7O9*Jcc2 zz-`roV{%tX!@SYhAgu^O4q% zRiS**cKqR&LE??{BDvXmCMJwixTy^57aUk?Wo5g4(;hO8hy=A zyAi+P+Gd0Eu5iQbsGb!s4egar&P%F!#5TCCeva+gRy@~!sC6}eS8LNA{(o9fm{tJ32b8=hbfX`>f& z%?XVO-YW)HMwSRe6fQ#7>?=c@EIq-=&r7u-8&;`EY(xuQnN*d49>ChtjWv;k;F$Bg zC#>G)QyieAH_;*Zli1pylaH(QpWo?5qx(6iDS{hnEv%0m7tQoP;jJfgD=Nte;JLQs zmZ0W6>5ZP12FGB~T7Q#Ushai*$1LH9lS3=iA9%<@r$#(glw?OoIfL^+fSh_Z|c!OfcjbAb&_%D3P8c z9j}s&y+)C=_+Y6-=9=$beny*`A@afV8Y z)%An#Vs^8o+@b>10qOoc{|{Af9T(OAMSBmWlG5Eh2q+!W&Co-)fOJbYD2>!ObV)P7 z(2az2qx6u{($XNHchv9ix%c@CCeJw>Ugv!F-fO*=5fs=kSU2X`KrKU7B}c*Es$~C- zG~#;%nuhVdaZ}(g+$+m$a>l2W>wQ%8@r-JZ)@{8E5BKWO~XV!4^nYw zfJSv3R3fk!RDUDp&05M z-=_DY?{@V|D0BShR>0Y!vyr@bx10JhNuI9vdeD9Io!si?pPbb+{AKV4gti8JqqvN8 zZn=a`zWP-3?KJdiroyDSRio<8REk>LzHp+#963-wctRUjOiOe9{sVCnxZ|AL{dvm3 zwF#@J;Hvk_qhU*x+AT#lvEDL6=c^Tsfiz=nkTfVsYO091j8KAmBkw?8)@=N0aMG)w z5ez3ecUic4D)SIqXY#2Z9Kth$Z}x%F1Itn;)0oeJui&vM-V;Yzx=-cU-cny#_DFOT zBEto$s6H#X8}3q;k25&c|14bFjhXm5_>d3A)mc)IHGk8)GcgVH=nJsQVrrC%Q67;Y z_N~@@#mkIF`&t=TrlQfQ18&kwEx57EysOObYdSAi)r*E2<9-YP?aY*@Qr20axoCrXpBM)oZoDdyG}QdPx~8Hhq(8R;dXXK!fL%3Gu)vU(!)2#dS; zRDub`FZ)M-WHr8};8lsp5p2_$i*Q4!K_j%cfG4qecF~{S1}>ZwHjoHs>sq?JzIbV=a)S(7d=h(# zkL|uU03ZjvTe=KNT-$!AA<#9STX797=-JpzaCUTdTuT(cz1>{MWm~i)P`uR0*{A_* zI{Aue^@A5oj#!Z!AnLq%U1u zyqmVDT-1L9g53!Sq^OPsx9&gvGS73;gMWor#OEZV7=cmPT)Y9RYrDiVg}R?^Ld%MX z-bfNfhn6aAESvr6k{z$**dGk@10*Hk4&DdW;^S*#p50pN{_Rc}0GxR6@#Xt#G!*dL z1P=hHYq>XoYykM<6Xvr=gNgI=ge=coDuSfINdWXS-}@#y6I>9Yd{XN#Km65NEuEO0 z4$&WJ(4XD` zXOsV&a6Et79mqGM{5{A=sD|SvLC6PxdSQ>AG5zWBF_S0#jr*1Y=*G?&%Xi;UozkrC z>?v#8ouR{KDm-3wZS&d`>FwnnsI&3%z8A^{QTAiN?C`Oc6+h@}R%zkWMT+cC$S%>GBOrk8XOv9 z9Q+yi_*36I2-79dF0P=d8%_8{rY;*c3Ujf7hm4?xAVU1`4gU~S z%C;J-ki?PEyZO)qOv_YYb*8_|Lhb$YYdFS=R+M%&I0t|zcu*n710W_Zr~7Y72G+c^sD z5`LYM+L) z8|vfeH1fZWproNv( zDOwESI1t0eR6J*r^)I2(nT9=--LL*a(ti)It>-Fd3)-%5Qxn-Zw}mmZUzs`YPB93AGR9bcUfcG!r{KQl9e*C36Nfvvjg3@euMM3$DVE+RR z^AxYfy?x`5v%>P>2X5G&u0p~1&IE_Q$=j9KxyGAvX{|^5+*~egK1Qp?30ytHx<69a}zth$jrw-#o7QlGSSYq`wRT5!xLE-;qD?g^5f#%EDV1g z0aW-%Y$|Ffi)aJA0px=xs$Ra2%po2Z@e1br#LY2uJFl>DkNdrSOv zlP2Nn#=a&vR1m=+FNa>TZPyVEGB%{gG2+_OGw`p;DP}ySVDlHxw4GpC+AZ_U75iBj zE|^9G9+A<3vZHf`$19(D6|Jtb*Jvu$L5VlC90NBMV!tc;_$>t^JHoNx{_*$vvqM*> zJI70lQORrB4OK-yneydXtOh`+pKmA)u}xQStJn1!Uw-^ZNC_f@uJRS;lANBf=L+`L zH+>Ed98vTM+$XP_>=}AvbbL$^9D`j-Vb7o|lDy(ozac!L4|6b!U^9vxs>PDE2)U-# z1VeE{o0M0^XIH$I1R|$}(#?5Hzd8z3K7doAmaS(z<(Xw#z-^&*X(iyAwxH zd*F9oBOtw4IYko3g+dj~DU4=7zVVSIYW2dAhq%Z(=sL?q;+JcJ;NhF6Ov{~uhIb6( z9rMS%x_Qoc1uELPM_lPjEK;U}w3-X&#j5X`U4(?hES3Fz9q*hhCg)C}R?=Iet@`q$ z2b=a2@|Q{dv(Fv4i^hybGva&82#Rs{l+MYp1l2mJNT_HIl|)OY>`xn)E>rosG(V|; z1Xgs(TSRN-gU;7O2yDg=c0!VL$TDr;x=4`ZGwy73hZ8y9v@}MvZwL@Y5uKC86dUl} zP>3QiMd2d5ra9XCSJp`>+}^Lt+Cn2RSUgCxB3MT-*cn(jlvu~UelV@XTpGvBpjHXDHr>?wqj_xAqP?v9HK zy-nJ!UA%Im;gy%dj(z=`b9%Pq4~NF?k|~kL#_a@5L>g4aHl`pYGMiNL|R!Vi!_e_ z>MCh`+{SdQX#O+Gk-A^NflaTmv(z(%#t|GSQvKa!v&yolRnL z7tI2JE>-OlwqEB^LfES*N7ge7(v%0hxl>XMeAX;d4s1#)Rh?59VQW3iCQ05XFO(ZJ z)Seo1{ygC^7LnywHP)PfpNf+bM#64E>#25T%Z67Rfg+IC%CDEeI=cyt_GC)jC34pG z%5(h9>#SQ;5a{ambJxmWkJ_z<9ol{C?WMK?_U27-(?2yBTe+`4r|7d;?mD{F1TFGq zL#Mi81=7p`WYDi(7MM2j4XKs*FyN0NnG|FMWzme$rn%4(FhV6(Yfm8DVw1v)yv>5o*x z9#nyQ^-tV})8jen0*}d>Soqe7gdEHWcZ_St07!{GMCb198whBD#VUrk= zb9AL0vA1R7ef=eKKLa|pSu947{0dIV>LH|TS_+c@Jx7uJ?-SRDyr(wW0u?rl(sQ5-W%1~~&C!{lumZ`^a6rC1j}^4XjH-C>a=`W>4ukvaFZx z3vf^W)UX-tF9fly>@(jgsPl!9exR`?@_PT#n)Y5q_- zu{5k>4?bp#vL-a~`}oTE)WN1Cx;j-%Io98!T_mkhJK zGD@IW7bq~AFn&58y07z84#Am`L9wizOmRgp`K0HOH~#eDovJ9^jPJ7+25AnpOSD%~ zkCxr*!t{fTDwgQL$Mt@FO#|x}cxZ0-s!iWCcu*hgL(sfEkECm*Vj*T-jVH^lI+u^ycZKnfFw`h zv>TPQn!)vXPn~U4;_^Qa&v*@mN9G!&+ydvtVo9(u#ZS(a9rHuCrzny+KSwC(ENCVvx5p)v*wg6;C# zAVI=;V&~+$DM=UphW)r^H3=z$+g3hbA_mpUcuYYnsaDM}dHG=MaZv!7QDYZ}RjO_` z7FLiH>o1ZlEfPw8je)PJAz4h#LxtX*w+iL)cH?-9+O2;?<3YOfUxEng4s;Pefc!qp z9ju-M8~G7M!g|y`%Gb`Av}fVWT&}_!R&aX4B?pjBK5?yzwU@myg-YFMi=x3`hlHTJ zhreIoL&A;J*inWv?Og!n#ZMBH7v<_x3-1($8wU2DD;w)eSwug1F}2&4x{_Mf zFQa|nH8=Vh)1JlA)xk?e$lKq_^@D{|lLAsgC;T*bHKs`mykt9xv(B=yPWk)uFF6ZtV8{RK-0gqG1CR zdwX)J8JW_1?T>cPCbh#9Hl}iOtb$Y6M?PPAVaUI1f8nF7HIbmUkW8Db;HfkgD_2Zv zu{FXIS3k`D(MFLFxR`~H+xe@dmrI}YH{iKs*^-TdM-GH%2ujpS<>%AS^KH8u-8{hg z`tE4&!Hu9Q+m|jW8$CqH`} zc18Y3?0L(y8T_Fll$b4 zJoM?L{^fJ`^9&W8?*5Xb3K}|ZVn%(-sdUzj{t6|QL(M>)2f24Dy+}cdrNAHK3g9yk zUK7g0x!Otlk=*9;$0eXoC$lEo^8X&pXP|)LWb~td#1@=??@Nn9I|Ka7*hG1&ZEW$s zB`9)dhDz&tJn7dGvqfH%biVycl+3`YSQ5i^7Pk4nb0m3R&$=h925Dbzy=GsPN(Bl0IVLTK|C!r>0l+BDZYgU_I&j$Ro$ zJC-mF85y%l%8Vw<;U>HKN=dKyL27X}zAofsreN`lPP^Vj?xuR&uwC_wFa@o!_TE0yPcTO^qx3p|zAz%3fVir<#!RHL>6* z40nTywLBA!lc3$hwjQ(OyKEcM>~3nOrH4_0U0VAX z*o1xT8zBJ%sZ=%fssVQmqRD_?*K58g#-DI5)q-Ynqxi%N&Qg7~%ME!fk*eRm+hd^j z`;@0}=wVVK8k2U(%rmu`Mx_sg9Om9~-c(XatyoE8k+T}0jO)lX{_o&*cGPAXPO|JF z7F}&E#NJ-#JTk)KwyME>_23)-+>EyERqO6`WiQdKzIo=^;eo-?APJpATjgTkt?uHd z76%x8J5Rbu^gNR)9s{P#J@b@9L-Du_s8NW$=0HWuF1=`vSO)>*h#a123i4yehJGmr z`QsGdbI18ySE&|Ah=yq8j@j^!s0Y9vL+`+paRz0>MZmY46O7cp=ZreL+e)?|UDns- z#S*c72IDLo=>>s=4BQXXvd*1bOOCrk1))`zg^5%*q?i_1Rf%LkLp-&SV?)U+cNOg6dc10!FR{BT&5 zWjY@%v12ONy;Vk$q)lP3WG_l#=EAVxkgW^!upHd@30mcAqa0KkHfhn7n1-dbrqTvC zfk+*K9kNJ?ssTmJM6ZkRQ55e&UqXnvCO&vVlX4^|1=3<+j^v6+*84Lan*F#5{#PSn z&PV46Yz6wV-KXflrZkHl&cYhhZW1?V3%4g~v+i+T{NJ;i&a2xA(!U~Lx?7rCiI@ix z>y~3hD|fGmK!>OzC#4~v#kCjrDOShh{y!*-XF2j($NtQsqBb`B3W zytZB1g6n%-jJ@>O0M7|McdS~H-T@M7{ImKDzFFV^fm;J#O@w2~~HP3=v;A76tFHY^@a>Yi?^l>H&A7PGkM3GxY-2ec2i;7TWlZtga-a#ZyBj?6( zp4;f0p8tCl-F*zOdOdH|12jqZ-itrkMF|rqxH`{Fi1(SXTTi7;o8?Cx`N90lNRlRP zLHL(z${#xuXryy(g=zK{4jBf~DG2>p1`OK|ZC)fK&jOA$mi&N)^}_mg2jxRoH%2eu zu+pfAkmkO1TAuFHqyZ91yRix!Ag3~~jsO5GL+D()@;Xoq0(c2=4K!>BM%qencbz}X zqjn0X*)?kS%F8=v?g+{}p157*c~Dt&U_uy(ukK?#{1KGAqyQLUJUR2N1f7q_8LY+p9irYM2hN{CQyviE{dpe0VVg98*NRw z1xGchQ@Bb~^LCt{Du6aIa8KYf%m(|K$mVmobRA>Na?sr2&J^W1!;vFnw)U4|+}p+u zeUN|W3j0quGO_O{2A6LL&gacitVkN>mzadGqeD)@9mq}Q+n zLUYMd%s;DfQ4aciB?Xo;l3?nL$7Nmo*?1Dzklz?WFm$+#;t2q&$x&_c-=((x8Z9Pj zqV|$=fx0%0V60~pK)n3jo$pzZG2y}5@Pk|F^!xx@>ubo{0jG@9$DTJ0i)T(ubxS8n zs`Nzby!)gD@W_e~!3A3wR_$8#lfn2TDk4dT49bPBdA?tyDNDBoX?e5*l1*>|`CwWP znYWnObF=c4p2RFhaYsT$Z#Pgn8d9(iUEeN&=??QZm-Ub4IQXvel1r*Drz?@|1b?W; zjR{kWiBo{hbf%%R;Y4c0M-~@`Tim9SDKIGnIlg2EL9iOxvBM!!s5eBLr`ddZ8^N;l zgNuTS6~6&HqMTisKk9Kkmbri0HkpWs2I35+*_um(jA6YTX2>nG@aoiod;qAPkbKYO z4dq~eWqFs5j_lrGG~ts}3hvW0grYpcb}^xuF-;Cr>1P10>j7R0;#Mwh@$CJcLX{MMUss zfcT*oiE+pkZuA1GyXQobPsTx%Xlu6_%U)gQqgqZ*F>Zw2FtoF6o8l*$R|y3;>P(`P zl6A?BMfwTrRqZnFZ3}BbZq6*Z;I}sj)U;QXnbO;Nss%y2c&;#v9P8+nR*RCu5MXL z?#$q)W3$uNE%4*RqZXsaVM>%e-Q_H*R&%~^?7BV($smeSRphdMAdFr5f!tPS$KCvUotV<(tvd)3Z4= z^Q-2?sVE1-OsNlwS3}X#`qW4a2nrgSlLfF7ttW-LYVkuL)g233-+VOE`z*7z4B3kg zu%pKc1D*J|*_IgIJd^B)@xomgp0RD*go$=O1ETJOzkr5MC0K`Bu+W!HS(7H1JFq+T z5s7xS3Mvhr_K)}&JIO!@sf1xE%lGpZPA$bW!yKK9$SE_sg8hsDqZPrs zida({H7M~E#WO!941QnB@$!X9w4CLYi(QGYyt

%Ma(2V~&T zN$%=JV((iq`NKc-L6sm7wFOegd75R>ozvYARHR%a>r3c1Rtmn7l15LW&6cSZHghee^OE13>!DeM%OH z`V|T5R>wBO&)mNercbAqLi7q5Q~n%6w~r!6={}6BM=fXVGtZ8{nQIhVi=4SQ%Vc+I6Ny(gcOE!-R8jIaZST51jR{io5#)2z(C)ZZ(p5r1!sSPGiOgNV;R@G6D zr*g*QVXyxl_n<8`n#dxhiD`vw)E)E!DF1-jT3U!T>SzjyL}x+PG8PjoG?{yg)^Tvd z^zyk9m>^V8;N6VQp$$oQtF$K!4h`iHgwWab5ZlZrc^2_EZ?Ddr5-6rU%>~K zxvy;YH6mkl_jyI0DJCmrizXMxTSw094C)1&!z?j|Z_bK9bi5D)en2Bv`{U0FpEDnk z$<%k&vh4`FKJgT<%V`SsK74BIr`SOcm!ZgvNm8q&#b5l$KEfEa`RH9K%GuFYnX75I z)*m?~(L(h}-Kje`I6VcZnNdV~&5d>PGi3|g#ax~Gd(z}v%r|uU*cSbteK(YW{Rh9! zLzTf4%NN0ADNj^~eDl~{)>k;2?-5>uLS9i5DSXb^Muqidtk>nW#^K(AyY;tkR6&>H z3c;BSZ#+KvIqoEL@r>$g%<7XMvrUq< z28`B@M=paS(xiMSB%-bVbgFn*im_gMaLTi>1UPQn7!Z=e{X(NiDsM2QDdQ@8rutZB zOg(ZQTkZS!Zcb!FOD~$k z#2d~T>)Y*$%mm;ru9vow`d3PQ2J^9tJj{?+=^!ypcm?l7knJy-HOf49KTtB!9o_PN2V3-1=#-J=wdrRgI>A0%#qK$bd9~i--cJ2y6q& z_^6R)-KY967bzbn6sDBP)%U(R4=R0esR!~D#)ww+Cm611+sGKs=63a$qXW^s8+#0r zkst2Q$rB>YlI7?_8&~>4Wt2&&BZ^CVB%N%fPA+~pvl#bS_&H&eR>5-tV3_=>Xt=|b zMya3)E>;-L0aC0nN;`193UBww#^^x2&5xe=@f}VZwx>@&?M49qcq_b^KzENi{VD4L z#x8hiypm~%92||is`KkMJaTjxcS!8cYq;_jgf9wl9@S>h^r7Di!wW z;@s%>8HoRja;NzI+bUH1w%(-3Q_iPGL06T(0F+BW5bz-wdm;cDF#j8UUJE_GEOb6? zvDI)(_&4_x;_DN0PfnT^TQ&do_H66b%l5CF7u~5n?*xpjaGre{xHt;CPMmcndw&Gb z_W)W5Q8|7p(b627=W8_21Q_k;#W-m^pGZtZ$=fP%j?@%5EwHk56hEZOG^%@;3Kvbb zZwdL;HoT;!IB{}8_7`QR2`w`Z=lN}cF@_>yJU))z=;LUiWM^#Tyt=cHJW6=P+x=p# zwFuVL!DCL_C34h=?Ln2;OGu|@n4x#+=?u>TZ+LOw5MkOe6zat%n1J1NDbvk%Insoo`*F}b{0nB> zxTLq0o1ac{#0pT+@g$~gsbjztkg51OqE5U-p4cEl9DTyQ4^mL1Ue?L;*ZJef@K{d7T>1Zy-Lc?<)&e~&QN zYvo~R=B>fVun#8gJUR<|y`t1ymbY2vtIl2xz5dZCO3;_`s6B$Av8NWFyKXcJ$02X1 z%!LD0&=Enp+Vds>u-acHM!PUBaw#AC6AnR_kb;kQJ4}U#LnVN+xb7X2A6=}++uTGk zN&r!&cQW#T9fG`~DUl6BFhwJ?Bp$q}7NE$ox&q0D3oSaq1h$6N^9~SScBOChx4&w= z4&pq{Sje(FI%;;HW;2`pn$-7hPfc-StvW;xqK8TIB3w!>PgXxvE-(q&{w3gkd<7D| z^}$&g(+t);!^jvATx7W5RLzCpONI9JklIXwVPqYnAo@Mhi#$C>M}~8|@KVg}w_t;Q zn1|cf_mba^X|wmIRuD6b(edO!00%Ru5vNZNZy^J0p@0MOJ5m8TY5z|ah)V4l0Mz6dbR2x z3to_bWSM%50)Tq3l>z3c`q@1!u30ow8+x0ZMIZx~bPLf~jG>*mz+WG*3(#@t@U%lT z+%deEGEB-(kA{nj@nn6m&jeQVdfyNGqW6|t8S@#It{D$@m=xK{GXR8x44G8c2%eMM zHv%;y!iAD@3Q+{PS+8+j8otzND*RZVEb?f*-IzI485E83r*A#7eQf|>uwLvWaw&gC zF|L_dlf`~U^ut-f5G_B>Hl+e3VOw8o@o0|WD@lYy1b5J{vy&DL4-GO?tFh2?kaYG0 zn`Q&coPFH@U)_kGCW3#{#EGjsDs(#iefi{{7~xB-&(O0>J^emc1dNBE#NWIG(%ODlqN4i$O_GfnxLgl*y8f#3VPCi%gYcy`zo@XGX9o-!F)z8BjA^dGg*pg~5J zn&#&oGvqGt!Wg{deOdl`$`cqs7IP5AemI=<{gT2`k4?An5K z2@FY2n`Wl``SK#UK&QptXK3@B$}WYbLyp4ta>}g5@c_N2GgW$9mghviH*KCF62!n&#tl zPsE!ZQQO4Krn0E-6IAg}LrRkx4(#fp%HQRpcNI%4)*2yxbrXb?)qV-K#@)UwD0B0l zAUNFz2^H9{XKpT2+V~x_zCH{2{c`V3JXBpa+7(lb*MK6>e&7=}=xine^t}M6|h@0F;6AlOapagf1EzBOSap;@uFlq-ug| zp)MwN1Zxm(MRgMDO9FyzO)vb&oE;pKz>p}=&)fBamFxL2b31&objAEyjmqF(v6ztf z=OItDOx}gqSE~GE@wu|;e1jfl$0uEG?K;ExI+`wy^5Q(KRzM1~Mb<|L5^?_RhN^(> z&oO>+ZlBZ1lhHKFUeQAn_vf|9-dA?CwE=Yq>mBY6rp$^|d0s_^Ir5LymMu$eR^+$>S#)Cs z;JT)>ER|yBdx$MOkCo1qX4B07B!s;Z`uQ#5f98(b864+y<`%9NeXlFgs&!ZN>5xW| zKSU?k37zCIKP)Il##P|03FkdH`P4e0sX$9_6JxBYYd`7Ua0iDvnKO-%WAfNw156f4 zXQv*PQTNrp)L|F9I`f}T-@lnI~F}=5C)3!FUDor*;Hk6K|b!O*IWm4-(`1Z`_6lwv~1AdjUwF?V5T< z(7~ntuOXNYHXXS6_VxeDivNH2rA(W_(!<&Rk3(r_vW?cT%j%J4wTlO&I}&PMmSN^7l*?L4f$|RTU&UXzG6Bk0seF8$fJT@6aa8?<+=l%;8}yn$@V*fCIu}(VsSw~XN=L;Vt`&6-W|d*ri!Uso6prrwI)wi!ODPjS z{zNI2x(#k3D@-l>67DlToNK7@!w~)~;^)VASx3ZIzFrDBg$`rXj$`X3qbbi7uadydbn{Ew1K>ylCpf9}zkPN5t-xktKOH1*}JwKPu z_afaLsxs)qhFj#8Ua!QDq|y>)xJeXvm_&$7Pn2mp;kOS>eSS}_6YK407P{^B?KH-< zv!hvY*omxD9Z^;PQjOf;R^LJ9klou)mW4L$gFAOj2 zjt@{6>nIG6W^*BmOoF1pNb^5;8cR{T+T4#60^zKX51lZ)cmm8iDF4RVKD}Ah(ngaZ zv|_IF&~q7=P#x#+PH+-hwPJ3tPMcQEukT{19y?UI15~hqkc?!c6gc-~~i^ylYzmd3e zynlb-L>`94Lwi{}U{5EN^8vAdVv$H#POji-CKu7{{B}d+U0_3PQE#yF_w)NS?9?Qm z;m}-Z>CbfZbRZyP-S#3}{oh12xm$XGO#@v~LpdtWuv z%R0&k=N`O4;x8!`i4)ARd6M@auN-pFxLzEHWm+k^RUHJ18Riuaj2vG1@D8k&pP);} zH5K_gov!xGD^3+W1y058@!Y{%&?2hKsr@oK68|TaS^X?6^M1j1ny!K)`Q>toi=o#i zXnC{1oV#7ix8ahh{WdZ#qY$YyXgu#PCF|xE*f!+u?jEr@LsfWb^jUHN zG`Pg#s285D4m{OWWG8|orCBIi1eWC|;o`kE7o=@6!&L(~w#IyV8voombhB8!t81My zPx`Ao*e*)R{W_zRo?D=3hA?tSQ7wI(;9am|iF66xnWv(sMDnW{2^P9*R3pGx_j}1q z$O0;=ilS4)ZJoN_>=(y2dV$?O{zZR7F`_R^ysIE7Ynje?{>Fw?tr;;Z=h4oGNS*Ti z^qb8FBOathcZMoY8h74Cwer;bi+N`0;#z_4RdY_ceN|7hRB1Zf)>x&<9>YCCnY~)H z&G+iHbQq8YZS4;GDI=qBy>Ob^3n9T$F+b=s;eQ}E)T;F<+KbFJ-@C6!LwpTbo zr@ZbXhSHG_P20x5XK?s3w=h>{o>sNJ#3x~q?jy1xB{=bX$m}CW*2zx2 zt?vzu>Epl1IH2_}%1LCO)H5k>+-ml1{)N9NN*1^#N2j#GhVYv1o<~CcXO(krl6M!k za76zJ#=EHf5Gf}cg>VIakqGMTx>CoH`l^-3a`u4;#rm7q5m@a@q|G$bN(!Z=C7so+ z>pekbL{hfzzr@DkB~2|Tcu0!wOSsC}c7N+KSzz8}(ZRhT)%8(Q$aRUlaj0!xcX(dv zRWp9r9YSh>5qDY#_ajwjFRSZikqZvdmDH79>j5Nx$HOJfCk97Dy$>$Kdij3J)G9~# z8+oh>O703><_Wx?Uf85XN*kkWCdEF3qI5`s3}6AWk0I9b142CTvzvG-vmEn+xZe*i2H@i$1?neAUmTX-2=e$w46-I~^q>>AqS$$=v(x zUMDf1M|{zi=GC@BrdA9kaiIQLmRu(^08v_R#6kTW)`$Abuex3*G(j_P7@Hy^nMu=$mZm?3my zKpag}t!MtQH{DqWXCcqmxH{>3O}5fbWtm3TW4@G$KDN%>VAu5v{iGrh2Q>Bc!jvCf@y=1!6b<5vz zbB-iV-nFDpt>kJ`%Wl>2UqQXO+kz(oM+Bn)CU}@%I&)@B0Jez$)WLv&!4#z}abO7Y zij&h_mQ1elIl?XU{sUJ`hV3f%SbqCjz;4_bRNpX8()wc<}Y0NBm=1 zQX2IF7sJy=MD9vOmY$cJS7Lj3hT^0Z&Q8bvPTuWl?$+Td|@q&Dw)Z(Iv(ORE> z+9WaCTGzkd*x|hjbrP+YxN;o|=utB1J<92Ff2bF3WIlJi~#lxZg#k%nGgwsKUqVLB*49c)5(5Lh ziq_WR1Nciy<<}e<`(A->4pd)uMR+`=s)t=|^Svm4SzI?wq*x)NwfU(G1q%(pyo`LO z78AX^9`i^4-26Fd;5ijgrf$9eJJIf$W3Vp~GOIs_UuBOE&4KTS(cz(w92<@ciD)tu zk{Eul7Qm|`GWrOk7q91Auo`_CTqLl!8Q3p2x>K2<{dNNyw^-jX23ZtB>YGW%197YpHeG9~U`~%7# zoow!7Ipn5UD{41+A6`ami;uXdTdG9K-=B=xEQju?%Qpj8K~T1Nh|%txS$WGVP9{!o z`Tgibs8p)}=!3%#9GmoT$s?bE-xFGy)gu*$KG*tSF1C3MUYcdQS?EsOPxL-RP0h$SO`;=`827ln~WHg zqN8HdpVlqu&8vU06uRxBwQNj6`ub!0IrYQWzeq+P+CINrXn0$AW`^=?o~Ng!Z+kGch<`n8nR@y%)CEuwmLpTT#~*yFYRzUcx7%udqm zEdtVODWe3Hnl6G{;2QiUeZ`J1*c3pR5}(kwJ!4Ke*o5|f4b4+rmvYVjG8T?Ap8i_x zJU4M6SfnG~Jpmn#2_(R>^oflR!9Lx3WALphGc?{PX1)>2%P(3Mv>ndHqg_hF(6X#7 z2f^859rxRy@{SHN_vNJ5BO?yuZs1R?UVH}$Bg%FpV}cg?{jdjRc~3*XoN+Mmd@3?^ z!BbYrxkmgip8{&W-j>gL+i)~M5^TI%t%&0knRY|)HV`edI9;P~BxTHw>Z$@M*QgzI+UO(~W=?gvlLaRgOqBNh5 zG~=;a(VxP9!aT=8s;Nzo(P2BHx*`L`?)zv$NNFP7<*vzK#jwqbl0szzc6>UAH<^sb z4h_q?;B09&2ejT7XhOv>(fxRQ_B|5_#(aYFSSA^3f&^!pwWm_(Hk|}Eg(MmKD{~!8ABV#5jDpV_80zY4+r@uy4j0n>?kDO; zSb#jbCt_=&*|aZ>_K5{EU;>|YS(Ah0f&8h#Lin~v-f@u*=D~ZJDJ2?2LQGe}I5@zU zAS&6}mvUT^QBiX0qNS!{Oy@+n4hpv4%8HUA&_ON7&XDnYxl5o}cKpjx$Qo3c+DgEi zCp|QQg+Z%#jE#vfSx-Q^?&@NxnMg&xrCv1YGX7<~U<4SPBsD2q{OhY)LvOt}lpl^W z(dPp)=FGP?M5w|ZLLCMJr0f0^vTw)P`jFt;P;7)%LR=r5=qUsk7m|}nR4OP-tEsAa z0RCZlL=61et+i9FuRhKI+WJ0v*|F%P*HF3B5~9tY1=;4{)49SXC~bn|B;k5u9<`a)67)vzpg|h7Ij{OTAp8BU{-Dpu`ZIxGq=qi)t|lxh z9JiPuHDyW&W`%?cw0o)s8o@+>dA$AI+vyGQ;x8w})cJNMzB5Md{~?(AEA+|!BJsQ} zuD|532NiugA*KEaV2OVbAC71xyt&VA-NNfYU6!qF(J0FP6#Q!Oed3*EoeHK+Ja!d6-o|No6|JL!sWi|I<|I+v{-p4U#kJS7 zGr3qe!bMvj-}_`Gp)FjaVs5VlS9{qDN`t$w=?s2|TH^cbsV6sWW_3uwH=iEXJd z7_eC&PvoPjpUz3&F?#Ve0T7x2DJ-dZ4JmvCe;-bi1{Q@CZx1^0`lZx+jE<-y?Q;Q9)^&U38q-xnEK ziDcfAb)-YhE+2oHedS zQ0R<3PzME~-bG3h3Q#6P=`$b3t0*LivtUDyk@c_=dmm7}jY{ptOil7o^L*!A=AuUibd3&qWh-b8CE1fx1 z4t9*d%@DRpM{LsLYrc;Qd$ z=qnRlq5g1vdY!9mx%Fd=#dm%~B5#q`MnkaWBDG!`pgRj-ufG)`U(Aiz{wUfDKK8}G zlk1I)@wZKBpr3($MS78j{&@BcpgWPBh_X;08A#;jH#R2~K9)+un>^tovF+H5Mw47Il{Lfz-48$D(J@*&dV&-0Jgtu7U7%85@60ZP~Oy?UbKk_n@2u z!IZj&aSeY0D0oYy$|eSNg7{D0f`gXcrpWrPjecif&f})p$NNSRiXBe%T45EbrmVu? zKX15Q`5a3(5Y8K9O=WW7CzlQ>1$b@mC;j~OPW88h<1GK>Bk_W zG#ESVles(-B**_Ixar5A*6Co<<~i&Dg|cYcTw%Dm1>R%Qy>OXsoJ<2>{^y1L%-uQaelI~xc@}1&}NiO{Oyz5_@65ESoufObWj;x1RMA#-pvZRj@m*&4hxnw!R(vGL>3x=2AyVo#ejNQT?a5kw z|ED~JQ^BFF@&5ivLH^%cVNLr}B}%`Q{7+#cD~q(gvM~}G(y;QU`X#ZOMC^mdZb;5! zfO#B}9--C#C2f`3K;>d_xJh7JTgN9c!R#A}0NUf2(EmOeMNEnjao7v+zrrd-J|iMM ziGxV*ba~G`N3_ zQU8w;K=?l#Dd2?DT};SxU<%c*w|P ze{+WO4SA1~bYt?R@78>;?oD84|3aq6y8bO@-a}RpqGr;AX|?3KInz0(nn(6vJ75TS zv($UD|3R6&zT)uM!?n~X-tM>D!q<~JGrj@`csOYYO(1-rFt7q59L?FboDy8Q#bZ(3 z*~|4FqX%JB?<2KAE&WJaS)65OKg9RVoD_S{$1km__{jS4U4LBePkZ)`-g1xC^;^jk z^nq#yvI_pBPOap-%ccDs-=SIfwi8ipoJNCWH^Y`v-AXx+S185{k`$-q-I- zk@#&ylQe&p8xcK%>vtv2 z(OvDXz#m{adA7#txhR0UPWcz9{_L85%jZkG%%=y9aIr?T4r&f%B)2~qYJFz_i6^YJ z93XU*t}eXW4D7SkcIVe?c&)ldXz{)yen32uR?f<$tLgc!XsT!?`sN`(8Q?b5@)$cE z+<}U|-!kfdwRFB>q_-W%r5gL$F7ugz`KL)DDJxP+FuRuU_ddZgvB?cgNRyI5b&#Km zolk?3Hyn|26Zka$;A3g=NbS)4s=~vfQNlknl;9$B^#h;V@8V>fiqnTXT&mU?;i2-- z9-2T2-X{HY%7pO|W#OHbOyZBeZ0*0=2j0t|%U(*3JW8HXzZ~7XM>#{a>gyX?ykT4k zDQ=u+oUUoEtB$C&WzDpPrtH^7?LBR^%zeOBQ&8=D2mZZS=x}FDqXg>f+PK%4uSuvQ zGq|xBJ)^h*j!WurpPD59raTM%6jmp?h`-{D@k*I$(X$!jv$KxPL+c-fOZj)A8#Oe^ zl(ygy@VdkPXp&@8(~0`Ry%l|b_hRri_V2G7ZqlfHn(>o~{Ai604=%f%i#QKB+sq?B zxz>7lPhWb&t~v)kbz0(l5?c*A!=MUQ%3b)M@Nhm$s`pJv-2e;4*=LUJI82z(hC#ZuG|l|NdKH9ek+otk9oi?%!mvuh#T zldv_o=e#xul=~EKHpdE^jwp$w1n9$Ac3eQ#7%=gCl^2a zzz=k&6h~~w(iL%+ity(g=PF)5))IN8@Q}Z3 znE&+CKVr$-V*6)9nob&(f0v<0Zv(?@X3=9))DuxGJ%1V%{n$^imI>dTa*Xu{4j^Md zf@X%g!gvX8j-!)f7@u}4D3CuB?bQQ^Dv83p4giF^1!H4|TZ(`SC!xe>p8K}j`?%Fw zpMHfSc0X}}y>MO1qD&Ada{dRPRC>`EE?22|TVE+FORwxQm+w40Y(G{jp2;Jsx{qV6 zpLMM|XfU~REi;`kQ4-4}ky4UQ$Xo-}WkI8yqxa<(OeV}=^h}oT6ry$<9=E~W)`OP- zj3~gzY85nm$_{W-)%V+pbu)Q=b(30huF{2!5jm;0O5kO&o*vwhem6T;se&u?#KMk+ z{8cud6%p^)DV4o?1XC{OM3E_6NR4y#b0G)lj8ZLXy513UWs?CkJyT|ZjT1GI*qi)2 z_4$(MKYW56XZqJR(!!FEn=cM{6$9zGMVk&bxV$jYX=7QK2sRZUD)q}yvS#qoUL0)3 z#Yk5~k!rik(wOOO)rwmX!+spP*^OBRcH$}Njky*QsZK4s;Fl8Gr|7J@u%?$P@g#zf z2AKGDCKgaXkAV_M`gZCnF9fHld>iHKpXVoDalxB+J8&jYYn!`YMKNs(`B{Cxj z5vkb`qtF*8n85ns3X)NerNXbK@Ap<*azzCz)E?ZB>fN>7)~Vo=C;3G{b__?OQR{$^@i#%5-h&+>~#rZAe-RJ*3CxKjeZ4AY(*(1 zO0TD86xRUD_@l`$caN|_UEQII)&}g{f=3~a6($q|DNydNnQF7BU*fA4LJ6_jjn)2AU6>Q!3^2zqyDKy-Lc1oLlrtDlmr}TIvkW+jY|1! z%W=+Xl)sAS%fSVJLn9r~ovWRQzVRB(ng(@izVxRjjzgML8XHan$1~L5JDEKj z!g@OBa5+5E&`@tU)INdkzVCmIh|668ExF!3_kOj@E@rWOj`8~T#&x_dFSA5^$`xW< zA|XLovb3<*QdYN~AHY%B{c-r#EvXh&@ka0fy%z^5!&%<^{@sdhTlj5t{hxdg*!KEu zAIlwXCR< zTUV3@)(qOv;TXTb#AW{m8@4@6R5<*l2W^A@6iC#CCa1rw3z-zetuy|MR3PYDe{afR zT!DaS@=|e~+`&mtC6Wu-G%9#NWqB(6@=dWx&0b5OEJxYw;Cp_uAO@xd>ZG2^6#8sB zuOZD9CwoipH3#|k$BDn&_@ovHpieEea+1)3`Doc3Z!QH4BW;_hx6q?b)({RDfvdnE z&X8EMgu!mc9l69WVA|m7uRwPK*SmSsj_ZhDgcq(F>=hDnJlIBY62Ag{QkxWCE9TnF zIgDJ@Wh($b{otTKLP&H|EY-pzyOCUwxjyBU-movf;yhM4_C+xL(CEZ<4-h~9go(UZ zy$&eXWVSdYU$mK#y4*FShRK^^Wu?mH&n7P3g80i4ufSsWY^Pq8 zbj#u`kis+T`5aqq(=LZSNzO3Jf^e@nC9Jk!hIY*i?w=b}Z{^bSaZp@~+usHJ>+Q`o z`3g))uftNV=upQ#@u~nWKm}BpbIsXF2XPi`k#FvbD-algbWoH@ik7fnIDMFZ@rQkY zPB+_BPu^q7)S|5PptjpxbjxQE0S%j6+|4nRV7A^(6+GAdELQ=F1iqdi3J`^q$M1=j zeqvUNcrC+J__C$~DZB+cu&#^^fJ{RP>iO&nTVQ13QEkIqTrQ!>mWVw^g=ugVtcTxnVd!(O*7_MtGRxGU@#Xj zlHgnCcM#IE7EPzWrcZx;&bK3MfpObbmM5Qr$YTFO~>YZ&LJBAA@xcf>`zL$l~Z&!~0sn66_Lm!;C-5ac#=f9!FF z-_NeLmcaH|Tk5&XLYPDqd4CFV!hy3p_I6l=>FUJv{TMKkiqA`z+zqrPhSH@I&AeOk z?h9ezbO#kg%S)OTZ}Jws?!tHS8jWl~?+1&1OwXJ+ z%u4X2hia?7@z8pC&z)6(=dw&h=(Wd67wOBw@g>+txosl4j%e?&I%sQ8j1T>(Mw- zWH24AN@Uhoo3Lmqs9UYDA+y@Lo8xPdba6$rM5&`$*JOjYG!ukgHqo~#GUmF|NE^-A zD7`6roC=MajEh+cKtQS_UE7l)Fy0yF5*8@Ybo6N{C|#8phvRDvk+YoDeYx*lGyT@j zw|kVT5Mg7R)KNI&e3(z_ z0cmQPZmKMwXi(=VSF5G%5?eHL2}FZP*IpI?RITt0Gu;7d|7cVV<{X~G4iYVg-7b-1 z&YTM;)!RSo!0I_24bdF1xRFS*ag20^Jnez$^I-*LJ>Z1u^Kjw4hu>Fo_TCajHnzg4 zqU(zKuetE^yc~U%9)f1e}OS6rMV@c*BObT zN5FTk>X>+iLDE{B!|~nP8k2S_xUt)cVz^xLTs%kt@0Uv?djHPh!-`X;TaYznOM`y* zuz14jv!6mna%`^+lI>_3J#A}tTG;rr%|?tg(f5wFZcl@w7!uxcnh1Eh7{lpTxC4do zxE?vv;b(?+_xr)seACLoNgvtBMNYnnNM0U>2sAaVkPms;<|d99TJRNh*a*G(t?yYX zVXYa#C~E8-&OW56JJy#_cBatkwtl-3xK(4iRQ7cw=e)ENG$Egp>+64nSUc(+hBFX$TBBuvlFlJ6qjQj;3|#b`oBVB1UB#RjTt~>fw+beiaBzpaG0}<#k^ULSJ1|o|zUtd4@UUs3$1caFRG9VG5HFRN2dbo29 z&?K5m+#o8VS6v*sSr*>OdtFoGe~}hKOe2p+Nc8_9>xuruQ~G`EjJ)ra{qN)}*tS+L z9}rURsL8iV&%DUu7#6FZmm9xVkVui;2F|$yhw=Mt9iuhRRrwKJd(De`%tHr1io4y*S6_w6Qt4GPg9;y##dsm zpx3?cQ)nE2i`xEdl?bIK#pY>w_?3tp=?5#(XtPA{8(`5+(yEpjotomUa_ls?^@UVO z+aN408X27In})sl@9ouEf=;A?eKq|-C+VAQ{WaInd+USgf=pA9>VX?AX9`&e-G5RHve z4o9`>)I56>959v*6byrj-ia{jc8+3*c|Dhz&1}0x_0<)Y^mD{IqCl)d zbdSw<&F(+_bgH~&b^L8K$`m=tbyhojf<-Q2e;W{L5UjrqG;ZFo^aGo&Fqz^%ePAYLN0rCHCi_=^M-Zx{cIl(9xXR~I~d6Fxe|==ZB+))-PEpPAsQ>nhor-Au;D z4s7wgIo?CVLA&@cBbsqy)GA^%)_VUvoZ}{jn?<$0g#8U|6c}r}V{f2Z5^t=^f+_E6 zDtZ8~!PCB0&s?HLS8+@u(?8O$`oL5tT)I}80YsE!t`o7c;PzA5(8vqTJv9row3v;| z<@ly0$wrr5Es*ClL4uDi0-;43JG_7TL7?gU;~41uOnTh73*;^W^;#PmM&D*&yR7Le zzxgsY4Ro;@r26f$h^suEl1sFQ>Y>Izm-h~%V}c{nkd}AIqPL{`yATm3hKI*{w5q=N z$9s<$c56Ug>)jV|K+Q){$8^u}X4`dW%dXZCR3G_SqFaJ7*0lIggM4vqt_gnyl6}rI z`xhy&%X#({JV^wy;&j*7G<-Trd`i)g9-pQ&}>%b>V z{$R!p4d_h&iX-j$#;CJ8kHNCBMw6u(G1cjQm#rgd9sU0I>%9Sw^|fcaoO{K$$S!>0 z|KmCH+_fSFo!%2(KAhkGjMcD<#*TaNIgXR$>WvJfpz)onZ@fanYd{~~dUROcyS>g^ zYa6|N=rt8k73yibmqxkCrvrK;g(U%M2)ao$p=@l}GUp~9Ay_~oF7YKm<~Dy+z;Y2c zNk`mSqxnLcYbOx2A5A@Ydc+eLH9H!a?$OIOq{`ZtI>VPnb(^U!otkPA!l`ovsYg_t zeu$^YegjN`fg3i}46XiIjqyII{LF+pDf7pyYu^C$Q>wGBb2(mM?ds}z&?dNlQ_1Si!lG{49DDeMcQlr zfdZsl{?Kn*?yNLK(FP?M*z&e^pmH+6=bnTPSc(m6e1dnacR56T&_6)vRhs6I2J>uP zOe~AOw9Zk_6P{tmo-5nJT!ROUtW7M^j*W!5T_?eOf01$|=GAJOioRg~! zC@*|P^HQUHS-`$eLk$bPen0h`j8#PhTAbRq+q!+p!UFaS9;3Pkzs_tkg>>XMQ3E5} z=^FQuhUBy)9l$w>Ov)b_t}_a7Qn*~`(%qR<5>Bw#wCe^h=yaCCNkwpTF-J2fvuxc% zl}AU*@c!zBVxq(S@LJQ_WfpN|Je^_P&BR9#HplXG>3~dkk zphP}mHplR=5I{+VUvCb*E*!OkPb|xdDv^jG>dX+8L{t8#E=B4!*m`hmtJXSQCVGtr z{RTX(11*dnI=9!=4lFc~kdy;nHoK8!H!+eFqqc;8CdYae3dXtvBZFK+vEJZ8t%bc= zhm1x0*S+OUW=ng@nK$*=)P4XzzJ!V{N8urTD`Tr;TjQeZV=YnOaA6cDJH>wLSL&Q+ z$b_#?&GOi7II4rab9ouKhV9-Amg^EKj0VdNO9?u*KX#|kgDF0nkJtVLjZEhH4SPqd zrg8MhQmKYuIR>{LaT!UF*AX+gAvlpZ`RAQ(kwedGQt4CAlO-oRr6-&EDJRV$Ied1|jR?)fJ0$wza;B6{q`^8S`N@vV_4( z5x5yY4@JVn2^;Oj=oIaJB-@=PkAiE{S$GPyZzImPN3J<=j+dsc)A~=12$|}Qdu0)k z^+zjnSjz16{i2Dg$s^Liqj{cQNCu4VY3@j58^4FK{4)+5y;vi{yliy?MP95gda)C5dszfZ;MMqrD5 zC2T%|N1i;<+%R=vcy=1MTtbh^ms`&RZ9D-Yj^ri9$+H4&mM8VrQ}0HD`;e7s?dQlQ z9mEPvL(Bd75NJW~^`}agr7WioUfCekKfXXIAdGCOZdi#!6_z=@TQuB=wBEn)U1Y~f z&$6_F5;%U5ol!u!F1HhaC+{^!PALm1z%BUPE@9ZhEQHGQ7wJ(#c{V@aAsK`{4nrYy zFGJ!4`vASZ1u-f1LaD2^)Cn(XWkG00;DER5;}YtGTU_@QZ1*KV#UC{8l#N=jHHhn8`81J%+qe^U4doF!GMg|S}RZQs2m?-13iFpEa(&KItqoxw3{Q>ir&qzJvrl)wx|Ka_4 zv#Yr7md_Z$BR1Z>>cF&!R{+*3i_c~2!`dMH{Zl;n=S%t?1RsvL5$*S0a1p{QNmDx>`6#JDBtY*h^|Q zA{4|sv09St$M@6Of9dp$a&~BLk#!G4sm$Pv;yq)H+p2O{Y&@;6#bCdX^XLPoP1DcZ zCY4$#G)tK_v0@hZLtRIJ2`O;Z+ew3v4YK(vi1`Xpc zHCm{WNBKbW9~t^IWlBE$I&e_av78#OF}8mdp^J3UwiB21%ZJEl6b?MV7p?`p_heLZ zhxs>%9^KL=(sFSU0{~Z1hXWkiE(?UR;dz_`&iQX}7tv zl982pdf9ZF)>cez%?8gzaPvx7Hj3)=h2-)!nHzSD`-Vrb!g1MKzY%|@c{9C5N$N50 zx%SzhJ+l2CDuH>6Zzf3)x#B_F`8TmpCKUuk=EK5%oMU*g{GeXKSEK zq<5qj`lNN+$X3O7D3b6C=Z`cynX4O4aF5_>L`-R_RjiY#=xKi3m?z8yvLy`XoCm1L z>05MEoP@MbJSKEKA!6=MKw)#+!7YI}^_usfI6sXJrq|q*$2{{d<$Mi9kZ@ey6I3LU zVeD`Pneldpe6qvhQ<^{Zh>#HaI*~5NUrl(EV;uH^jRm9w0wI0ZYsZ;Zc&3Mx^f9JR z7CLfJzIeyMv=ZWc|J`zf#L0G!c)@O-8G!-k@V-Gqiy%mdpyH4*j!)CXgM=2%UMw3R zqt&%Y?GF7Kc(o%LS(;2TrXqf*ytg!4swqLoYM>c+llr+FWFPSuY;6!f|VAOT8PMzjhIxZ>l}x7r4tDw`6DNic<3NXwleDUs7Y^L;!W&6 z;WI)q%J+M<{wr99#uD!MNZdY#Xk=Hv{O^%6nT^i0BGkbgmSsz9$*K_TXI>iYIvnGU z1%`>^PuM|k&C`NAabJ}EOQ*8PT#8~QNKmafJyT$(BOFE;ED@69C_?{AlO%U`Gw08X zSj4I-wYh|7i~*<34T~tG9%mO9513et#0_gz9mxOUrARDs|0P1fvG@wDK7Z~Wf4-=B zvUiDC3C2VLT(&>wN^D$UAd|8;bQ6EOPaOEa$#O!ZFNYIVn+^1g|I9}(S>F(QG#+2? zfBG*Ft7REzei-y3URTHOHN)cnIpBH!%c!Cw|ChS$Qu-G{ec*Fvi$MBc02GPuHNwR7 z4RJ@0I8NrD#^-;yP2fNA(|d}T0L}38Moo0a|Be0wu-b0@?}HJlpZ~a?mNNg}LDZBF z#vi?dCH@lu{da_yE>1ZJ7JTG(1Cr3jawJ_@2{rB!je*VgtG`IA33`XhlP2!t-%>P< zRo|e%I{8sOzwsxxq;aveC-7gXOq*^{3YrMK<>GSh+6>y|KnrK>S`im2!M((xrnA>)w6=d)@?pxSGuIP z^I14Qx3O1qroux zbi$_v(t7YaCS_n%`$UiH{MBiYVabvBearMdVUyOClVUFQdBmR}rA1bqNw7TT5CfD9 zq`o%DF!6n(Glz5eZHa&!eNB+J^92=@D*55VBg3AaBlVuqp{`T05OeXp9Vc_qVEY;Fj0k$I#TWHItdVHM97 zy;MkzJ5aR2s9iFC#4x-8*= zUle^Kq0gG`?$Y$`MB5@bkC_XcG!^}kC>!tX^S+;7jD)V~rzYEI18(GBq|ow)Vc{0L zy7IQdj1<4WNJ9_ECu?h!{N6%dCWB`nz?i%~>#$9c*k7cZjO&;d+QhUV+)NEk5LIL< zo%VrR!_pGlJ6K=B0Wy7$_6SIEd92cdKnKQpK50C}YvmJVbY6EVYV@I7?nB-M#+w>X zL02&rMac$B1;e9Mt@gGoY+9VDwr>TP=}c@4V(}c~IrHtNm2A7SmwNA^ z+2c4TVT+si@gr}WWk`K75qD!Y^V2;`^=f!X47+XVN)YeBr@37I&zl)dN{utrnq*ee zlTbJfR=JF21vRi{>FY4HDSgPi1~xC=(P&Q;)moM$W-P_beEb^X~a0L#cVjUrQDl|3ko{bwSctE&vo&emq^ zbi9rTRFX^UB#cZ7X_ie!G6|@Nn+vDK!>HS>Lh{i%S^%Q(9*#-cYeflh$wU=c&4LLepF}ij1@sH#JwUux2X}=skg>G$TFnR3yI3L^E`V=SdJJ?J!Tve%(_#up4A4 zAZaw}0C-V?P4+r7xxVij|7NR~AiLVNxwMFeJT;#yyQfBM2LBh@loPAD&r_58r&auv zAMyc=s731>mNE?k&n3Vlyn!Uoj$0%ek$5wxX%N?GB;g#ZkAI_xnVs9hr^OaW7kGlm zgTV)`hTan;(exn4r5*klB}Lzrvcq+LHAb+eST$A3&|;2@TiQpoaeh@7F8C^+8y0&F zB#5P7PC#>_Yw@{Lz*YEkBgX^sZ@?o0EuX%eXNA@vE}ZBF)#dcit!n0(@7Jl=bMb%e zHHjPa!|JBR_+Nnrczt~no_~=>eSbx<9@CVGIFWO9pddK<25ga*8`n_@~UbQaur<0`36ry@xC`YBzoyrfCU$mlI{J#iIS0Fe|&IcEiYvoa8XK+jiO#T))w zr4+4?P93OYr#F41{Y^)o=}tQXj>w76=Zm(K0EBT<72^`;8s1sQ}Si)XDDKBh(_>=1Pv1MbFSAzFQ;vEndi$0bHPLM?B1fY z1#!W2S{d(SExd5!2DfoA^RNeu;TD84aho&zIz!i+U*u_m<=sm?T-~*$7{{59`j*1< zlo=bm@@M5woZiD5607hvl)?)1_qt13Mx>86;*NhYeVt-YTK11L2AUA%A3iBf0p_FxqBv>WHbtnDj<-wkz>EdMLjvBXxkO9 zBH}xc=A%R`9Bt&~DFSCfBTAJ&(BM8htE^a0U20_sYh!Y#7T;jG9oGUfg|+PR zcq$nyd;p9XRLkw0#l{)VD$&jc)+fr!!ECH4Ar>8zYKa=B!q#AN(<*Ss3ShzHtq+#~U z%_J^;IT4XPlztDlqqJ9zN#U;BB7-9cnzlqDXR#C$=V&g9o;SA{A@@g3vvlCkV1s+q| zO8<1k*ND6%6%lgSBN(#e^~?)>ac6_W3HdIHpSfb~cw<=}s?-(3@`i>*v8j30S$f^z zhc?!iSlw>FyOUP?_8at=j(OJ3zewW7Zo^WSHa&v`HbxQJ!!9i1d!^af7!SV1{@C6` z!Zt^o>S@y)1bs!e_uiOQwnNnHM`8U&wVL$0B|4-#qf;g$Q1M=NSBrF|zeu1-c0X}? zHtVY;Ex|)gQL14L3#!_JRo?I05*T(*+{JINvi~B{v0iF-lXb_!%>b&DlP<=|hhEx+ zQ;@oT_3!+mxx>+QTP{U1vKqRQq@>4S`C(VZRhA+u@Dk7R*F$HU zXobo?g-Kc#Y{~CtLo9IeHYocaz}cP6Z0YpE3Jb+rZ+(xghC%&{mQibNO_=Za>0!pK zVou%)#X0*yp)gq_Ys2-?@%$GRhgR)E#QTiTav`JL?+`3- zDY|(MiJFuWumQ`Tir4B=XdDqKuD?2ZUKo;x4O(2TGB}XKBL};s+8y%&Oo#Z*w0E^| z?XB6!DSq}OHlbEhTz>-UEZ5b(pzJ2=XK${`IArI-pFf(leu}>(c0vy=Y_HYOskb&S z-R#vCWJ`(k+Ec#_(K?cmgEZbhuvIUQNb#3KZbCRazvj-=Td<0MXkD%R#c4GXGaD>j zs1VaMtR~s&+A1FD?r^S%MFmtk++T8&t!jk1?v)I#J^*e<5{u{!a{X*l%|JfDypqL~ z{Olyq6mC)2^=YBMeU>tHK=OhVocYIFOsLGgQ{siJ-A5Xc1l!8BaiR{uTw6n%S&bPq zqz=Uvz|lHcHQ%Ad->KsZce9GTBI4&C-wp(6(-O;JP}lrK z>L|35z~0pyNFS^J%v*##4x%2+P*Y&YEUBZUXwM)xuq=IAMg4pK}Zt$&%eY($IlsGXP5CY~jjRUAAAGt@zEm4nc8^0Y!j zKK{}Za)*@8aMN2{_Ev?6bm{N^Fs(Mdj9bb2$R1GV`RI^Mg@^8E* zW#-2cX|OM)8jaLKj9Nk>G^f1sa_4I9%^1=YiM+gf2GiyzuR+zd9QRn2v&lUCSMI%f{rguiM}2**N>uqsw}rhxD<;xdI6)3N=N}56!}TVO@oqm-$uZ z5y|80^6C<-@$G%tJAYa7P=5ckmB%*0_M{OOm#HL5w=9X9B`ORXVIO`MN|TNAM$>_wb9b*hXD=}5<3^@ zu35)Ulcoi}Q_LpKO$vnmoMb%paXS7(OF4Dyb=$!S^hC2q| zLVU2+92goZpmpIB(&hq#VGp3p;j4eNsM+c}pW|Mn%(*E-382 zk(h=T>8qI?+IP8TUVwuPWg&xn@JncyiIn35@AZY;esW|XX)R2fqu13%l0Gm*p$;)w z0;R5r#9q@0Ikb>immIvK$V=Ue~sseF#oLk!#6dvL+qtV&a@pGc|8GK-(F-G?b2BDwEY zV`o#Nh=1{dTd_}1L{2O?9JD&}OXCpxc;mLmU+LD=BbxFGpvAxC>MF{EmdJ$^PK#LK z^ZXHQ$ofS2dx89j?9PYsIZXI;NIVQ4kK$TKW5`Sfy9H3r#Kpx+J8l}@z zoWY~LvD~vBS|6AJBpr`c;So+eIIif@pY{3n?7HdDZLqdl({jS0py+Mw zdl8GERBubVnYUbDjxMF1Bh($jS$cD;cvgdz+@^V{S8b?yh`G21goE=8jE zpS^4T#o(#C=F)_PUvxdLT8iOA^_x?~;Ez9~ngvs|fB=@fQdZ4~6U+rRLhHGwjG=7pj~LQMP*0_Ap3c#*Pd zp&kWHc8k#m?x*?#If~}x_Ur!l6B|}6(`3`HllCQHX;%D}9z+PsU>m4T4DNUE3iheQ z&n3FRE$%$ArqhCKWoDbfsQrk3@_ujQCW1BoRc)S@85KFhS~g;&0x=!^QKF+)ob?Tf zVi4@s!(`Pu-Y)XuW^>kF{l9LWv?k1U6LPEXv^&Ljf@#wI;=Xzq{4d9~pF;;9|FZuJ zTz5|NzZs}NfeGAh{(lou6E%KkQT7ks=CanvHJ9Y}UjIGD36|RGd+z4gK4Iy6kBIzs zEqZ|BKa)JpfA87N{-0_7J>I_#dcZIFa+HD3pER^c6vp0QA-sJvL8p4bDk4TDcItfW z2hV9%3;F!ZWqlhz8K4yNnzCfd^PjtZAol5?IIr&eXNB%` z`7b4?>ze>diklIiBi)6rHUl_n`s%&!y#vef03w?K7YDGTC4kp!m*MLYAoAfOWXTJ% z=l9&X(G180$UH$-{q99{A`93ukZ&s|-2OcvV%*dB~_BLf0@vHs(=zI#4(%rTizOJb}K|FlXnE%BH@rF35Z&I6gm|zcIa}nYy3F_aW`m ztw!b@6l7VC#pgdY-N3U4}K=6ws2yUFyrx?ssDvckb-^u|zON8~p?L z^wnHeI&Zl&Uq%ernkSt%mRDT4<6r%vU3DNC+k zshIw!+i`H(5nJV|0R62{PHwsow|W1E`qONIaJ<(aW{>hGn_MM$C4V>FP^|jw9EwrLXV#(SosY%?1Eg|Tp26`oIYz%mh<8A`uS%-dEfFzuDwk# zzT}jrl%`Oau*XifZvu;>=&B876ndFNr(7W{W-ZM`$7lN59;5hH(3n@B3sR6kYz#^NTz@P`Cq9pN!o42!fbpS# zePQ^o#kge&QNM@U+6Kh4h^9`GC-pr&BF(S$Dfj8LG1+zz)LgbOfWd-iC#4dt08CyJeG{b3iMOfv$WOY%SeSth9# z@5hUM6w6a7$3TP0ffaTv-B_!(r8CSwRXsT_JW3^;l`7#Jw%?+bL^kPS5#vK|yso1^ zAAm=~e%ktJ)W6Q^Wce^h>hN@)X4=I;1v>ghVN5!fjgx_Hheg03*;0G+ZKAn+sYp<~ z`60sRgGpLagVgjxx)NT>vRXnJ~H*|s8;SAS71=Zih`1IPV10o~*EsXsk7uV0z_xh2+NcOxeT->G;6enVRUi;hMIuCcN9FbS=uv(P(SJFWVKi%l$Q~`;OK?Ub~FwqwhowB-= zsd^iIU!SpJ1h+I$+Ih&ceOKqln$WD*Iiulq;bSEXDc{*-v(b`)m|UeXEFuS5v%<;_ zi|X|oSKi`YmL$Bg9C-T~lli{7+j+OCLBsU#-;;d%Q+)KQ;KJca!y*Bdh8&A!rc=sr zJDEM24@ZyYGRm>Rn2lLvQKOYWVM5~B${t>Lk`8C{*!&jfwyz9Lje#2nV?5&O#ZIU(ih8st;R6=AL{dYm2yoV6T(>g5#_x-v z%>wo-%`BoIttGf==~eYf>eeU8sQa=p(#n0s5AoS}+BFv!i6Zb`K$Hs z>mz4j!c{tRgZqeHj$YPs^$9+brb1r`u<@<6Uce=Tno~^Fflw$VXyR`u{AAd(U)j?IAMx z6$$Zn%M!Z|g6N&*#OSZ4tGlV?`NoHctg8b&TkOh$RiZz~U2&edAueK%2ITQw4lFkV zY7r3?y9PP22-OWrnvjE0Um1xo2b{*>Z*&=_Uwp%O&mpg#&DfTyN3qT>zVEsR7M2_x zYtH^-~pJFEHH{9iay8-X0FUtSV+`xPg;U{Zu4U17|053wjFY zs5>b12u(^R_;pw5J3ZJbSNLJL~;%eSKuEa*l= zJdznzzWD+wt|r9PJegHyQnWFat$Jz7%B3yVRk3VVZ!;F^%Sv#02>)b(iI~kAHM&-4 zrs%pu)xPqnG9syT$d>Lg?nVK1r8WbzQ343ibR*+*+)ADA=~lzz@FU$^XE^D_bxr4;H!_BV?fnt72G~10@~0RXL$ZT~zuhSW({2LCS*~5TpPNY6IYRoEbXk$IR%tGdhg*6E?OsU~jfa zS6Nt+;P&qtug#tPO0*j)l9Hy3$j6~%fL8NMh0UcGBpf{OK$S6F+x9;77kM2{Tc%|8 ztmmB-huD1Maj?vrVJytA{S#AhoiC9}8|McaPlP2C)+=ys&WMpQ&X-c-#>ut3`;2}P zyQiEKz^U=mF;dExrmKBmS+1yp=g})BIgw30oT#l>ts{&&@=PeUh~}}eYR6ngAU#?> z{0cdRZ_FM;(m6;uYWoC=om49I2gL%x;)Xu15*ZnFp(ePZsW2B=>Dh8eKVM&)Sdfys zNnq|u6#dCD@OE_BsF1XzHWfYYXUJTmAv4r7T%PPRDiEilyw{#-P5%pTgT`J-)hhaj zX@*GtbHPeDOOjGbmM{&2_*8z{)-{qhDN?#|I5)yglh z?U(~>nijp{Jb&-|D5uK^SS2RX)cO-2o3v8T&4eCznUt(-5jbxQp~JqDv@I1Ki@4QL zoi#0orP&1O(C&}Sb`w>|^dUM&7Achf@_wjs3oz+VzYUk(d?8Rdd;RVs{QM4fTK8LI zv~lm$w4XvMTA$oELS{iZ-)xeX_{~>{e=^t9N`5AL`)J_k6N=S8fD7A@SB4`j`0(?a zNAlKgas7K6AJ*N>*|U~xSQ2?w2B#TISIXmJMk|+E*|-w1P+69|@2-EU8>lPmOKZ>^ z4P){+XTmy>0q+KvWrv2T;3w}73DXO4?qCfIBvZz6FY(jJI)XA9#7+;4n@0`7M4;qn z8)OcB@_L2?%|ve;Y6dz!*DMCHYk2`mwC4nTTVoOi^nK_fnlh zC22g@c|(5;lW~nyP1H`CNr7zFCGKWj{A}Xn1>eJS<^3rVo964swi?bWfnp|wO6{u; ze+06Q;>&+Iaz4uOx$EX}M%SuO_dQmqRAPzc3k>iLU=f}FaUV}O9IN(n*XO}OvSnDV z<#PI&Y0LKt^`b66p{*pvwyY6Frc2voAbZR2!L}O2pwmp1rCk~1Nz|0A!Xf*-h`CMr}F8A>?#@p ziTOE$r5y|kHuzE*lit3Vu!9wE!$Pxvx=y+Opnb?$11=Zz(Swxla~y-ASuS{0X_AIkG(@a3A;Z1DHJy&%j%jDPEr>5R zY)!Et^xcFCc-~cJ!qxI;;tW}Kv{-9=)8i=AMNNr~8aVS`) z?Th;LrVHT)OK`z{Ja;R2r4VO1I>;$-*qKaDRj^nal}e6s7su2M4G*D$MjAAmuhtsf z;}35DnU=d-$_rv(b0+=eHzCv|K0sJ+K27|B^42(7>-}zm^sl$rtf(+h$Sw<|oaSKL z=;$AguzxSH(U`X}e?$}}Z>Ul&W(;D}GGfoh_JQ0h(L{~EKQ}k+R29l5q8gS?Sc zQ=c5PaRC0dE-95Ic|JS2PCwQ4a<_w0p#9dli?t-wmRB9S*0#DU`aq0jqdeg37LQAL z%4fzuV1>ikFy?id>X&(}w!Wx5(MX7b23B3LDVVOem_l{+R4-6ps&trn{n)Rf7oql> z;fA@Cs9xJ(Q#?b*HYUkr@WPJu{XrVAi{lmVaQct}Rd>qQXsT zQ$+ByVTLWCe@meCq|Ra_!-+96MLLW6v-G!g`kRVewYXx0nOWSidFA{$;~CkI51x?{iNzL$7)bOD zZuaAa(sg5e*~cxJ%2wb31^bsey_2UVqoNz=eFlDn#*O}<5ske=V6l?G zfx02b-BXc{+^F_QM*tc}Q=A=|Dc8Fn47=QO{WMMKmQ?aVCnb6BjDVUnLl-)7@d|zc zH`NwIQ6@YiF%B_%Km$dK)Eu#)sF4cAn&1!%29mHzDIP57##ouu*W0t3xy z40WD*1Ns)m{*j*kjD`|kL5Eu3@&#MSpOjEWf#p)6sCV;(Hh%R5f+Vt#!Ui3d!M~;= zM;d`4r9)9`sP@5eMSI}Vk2r?l3~=HHUZ|k?eO0)|z%vE$p4B$2hnCM9Jwm6Bo{+S_ z()feiql!8gUn+@$JdZGuVuNHoh$sULhZ=lD#3?zH00&VM-ywr%62#x)p^}PV@rt`nt-kTsSLp046CAiimOoy^C8Ktv zj+JJTdISlg>^^+8NE`+Shf^;FVoolSO)K$bcQmO4nUr`i=Zqq8*PDlkq_>=>eK?gp zs)q#qU~*pnp<b#2+oXJB zAq^6=2XwBXZP)X%RpL!qFg4;8dH{d3yM{K7#pV0EWV`i+FE!-X-+?oW8(hP0lgjs1 zi(GG5UUdmKDN0_&c|qpds_eM&E%gvS7M$g}mJkJ=OgRW1Q#wI_T%@YnhX` zKijylo3^^h_9l97Qi}IodNQ{~xQ3;c(s~M+r1$&5ZEx$a5zI7t&B|s$y{%}ZOl^r_ zNsxC8pKRwD(bFIdRrf;kQ)Awm8D`S4imZ){uNx#=_NR}K8xscuQmF>LU-^xNQ*SbEwiS_W?VL!Wv^OR#S)l#;c2L0M@ zqIV8qH6%PC8pWT7Q0QrtSTN8a&9!{IROeBNK=kmt)3X%Iav8N~`rFC#G-0o;)q@g8 ze1%*5ync%uY$jW^H(#bYZ?8$vyd3==5HGT@Cn{@ptWtQH+fxc$Y^es%r#jVev-_{C!tuAT@#5-WB$#g z|0F>45H^aSs0oJ+5Inqe!N?8%%hO)G1VH{G5Nxb^JGMNFRv!euzdLml{0E>AU;t1C z{QqE8`0JjX0NrmL58>ocq-XEnh)#g6>?MGIm$5HEB=mUC$Ay(00jU}0Xgmn10lbC= zOwNvxUj?GdHAYVy_&~Y{5C?s{0=nOP(eT=T`pYU#&gn^>LX9Z4>cr+Ecl!126n1nE z_)*rUpKZ_f1&%1_63H=(u&r#BnGZuRqW&>~GEjZ~}^YzdpEc z?!f^NZykN&FS~w2b}=XI9s}h&*JBpXq#s~g0ZM-nWXI5S*3c4bYM;e6wikA<0#S%v zSeiG#u-SR7M!)c#+XZTPinw>QJnAn-0r1&TIGmK7$+%dv)y_UaZ09J-FBG{cG}zPk-2zhaQVL#M-ojCD1d{1b@+o_jOFXFXShQA%VlUuY^Z4~|vzsLeC0vB#XBjm9ydEc}@&9(+ z*HWXuaH3oRYX0ON_5UupKJ9VQ0kJ^6aZ1@eq!!g(I)RQ`+<>{(aQ zBxj{x(k|K=y_*TYX^qe`!KQ8%q21|?eV>S=p3`G5S=NJxfh$Y>{y_O zONb?8$VlYZFp$)h=?8UZ*Gsvg5Qz;4q+F1OMP#sE7T*TM>RZoB!r>u2lu^Mm!r8R2 zzkk)X_dk)44Cy`s$1U-`nqpTf@B&=2?K994?F$(gtBEWB?jPz>)bi%9ZN6G02h6q& zIES4vu4_&36x`cYh?3TE-#bmVB_zW7EyeMb-oz*3Oa)`8%gw)2OD1V8ClA$g{4yEKF8D<+45_5JNPfSM=`U+ z>Rl%fefpSSMon4Vupw24M~$aor2*^6K-{f4R9I)wk<-SoMED!rxe2$M9I_|353_tl ztS^{d5W&b$;UDa-#mhf9Z$!%a7|D!t(v8#uMV`Jbaety}-R=6D@-guOxB1u>?2w4% z4I5Uw)}P&uV6OjQKa<_kw9Rrs^2Fv>v0Gh#r}(Nm3l!9mUWv>H&eLD&$GmG8Uy3+J zs_vk_{V*henuSq1*HUW>-f2ov!beq8$&+Ltg09CAyDR^wn}4QmcF0?%J@9hQqhEAv zbDPab>&rHZp!G1u5l;W9p7zX?3^xJ|=vyR?+Hz6rzJy9QCcYCS6q<2-=T&+nrAEaN zkGnFTG%IebWJu?Q& zScGyYWfPXbDg%h<1mBDBq&2ZxVkINQ_i%z?7?iXK;B z(Q(S5#V0}Fqr;r=`y-`8E=mv6-)AR|Y%&%m8Ev&`@wlis@SHYsm|cv|w*=1ZMWwuE zuD1&VXcFQ_#Vk|AtPdak)#i1vHm| z3Cq5SW)k4l_`3}rwFb>phxcH|p@_hF@=pNIn?BrMp!3Wat^ulkA+EF(i_E`deoq=t zC>F1sdtMFUH`cC1g?$kaQ)W!xc;ynFO0_+jBtbm3Vo+8zO_I=LG=@}2+?6sMfyH`c zyMH@=`U5;!%QQ5p3rv%W8v2Ryfi6YDTsaV0GR ze9m93mCDl+B$SFioIgCBifSnJ_W_okI4+!9j65>VW)xYG6eH?$7B6xp&KOI6T=JA- zou{A8?=AoJSI@5B+fL0AY4ek|1eep}gwnf*{hcl|FiCoJ^I!Cu#&}r{U&`guV6GT_fUk|4sk=UZjf4 z`fT2#f)itZ#NTR2}p)p4sJW2I;Dz;CFfW$DcK{4lB>&)ux# zsebe-xhB1i{mz&tM$cO(Xhf{XQ9iL3hV59Kg*u=MVlPc1K;lBjLa`5}STk4Dc zbGt-H``xoC+s7mbGkXl3htuQZH^Q(NMoeH9sA@qQ1ZKwFl`qmVPD*BEA*C)%55~`! zKO#OtMUY})M3Un#@goq)tbc5pd|qf%QTuXK`oy$u7!zQj=B<5KRd#(`Mx=><6B=8# zF|h2yD|u^t^U z>C&X{yOGQY<-QH7Zt5uV5g7(YT7DplfO@;V%w-pUl34^X92@NMOS*eAO+QB_w;Iou zLDZR5_a@3D1v+gql%kF|9*pQSr=8kI=8SDWl*A>L?|{C1F7TyXCB@!P#!#;4O6$5% zJ?oxCmkfK7D%!9)sUM}=N2mNXkO)Ls+9Sn}JSvUP3eC${SWH`yw4`%Qq|`VppkkVW z3ej=DQ>S2hm`W;vmL@>IgBpz=s+dv0P!}b_dQerDHs5k52Z_x_rvF!O##0q-`vTwF zX!;{^8BS8(3Tol_OKA1GIcA>;{rn-YeOIux`cx*^pF6Vb?)~k3gmbVbD6N6sdxwOr z&Y9xcL2l+BfK4+e^Lda!+722U;l7n}$-46)&a#BmJosStOVUi1nZ!2d(I@byyE5B| zT>h1h3LG2y@)=6@rAOu^b)S$3R6eM#uW||Xdb;<31-FFbIs3k9KU(|)Sl%#f-9xLd z6$LhVu`tXZ$s$8twM+V^e~;4;q*(WDCtbO@Pxol(el(9~c5$MAKZ?%aA6@5sg%b{kEZxSkiapp$u0HR3mkj3UG* z<+^jhKaZ9U3Giod3yQTNAh0jfeN^-6kYqRmZ~Rq7@Dnkb$z7~rN6M8%59P;gJ&P8U zbzYFX^Uu1EhLS3w%q!X8ibUZW7`3&5c|G<+e_6r;78fg5*o@0%`>nEs*m7+H(=Y$V|=CH(~)1|9Wt{NkpH zThXTlE>v*>4IpXarZ@C$98ZI2Y6WKC8y9Kl_j0?r1UpW9!yVxh;dS}jM(KY5cp$R} z?T#nk7$Y?w8ZGb~Nw1=d1tI%UqkYAmL<;yQKnG~dSglelJ6Q#?P2tOX<+U2C{^cN_ z!^b@Njg74+h0NcqEZx<hq)4dhm^_ySEQh@YU&JMp~8YJUhd* z#lR@{A{YN?L}K>tL9$U1Tq-X6eO-8Y4rpJ|p%@DKp=cQ`QMB;2NX^KQ8+Acrq9vV1 zOWM>@JW=zHy}V-609I6W`6m)o*-wTD;V@Po*uVU4MF zgm0;is`qX;z+m-0`aP<*#aT3aC$VxYbWHvuu$gu2hGo${_cUeX_0I))iJx+w6A-Ae zgQyRDxMiML&qtU6gxaNn(ex`ETnm$ymY7ymZe} zK}U{xe0;Lc37vczVUppedx@vMrAtA}cdua!Js3%s> z8ruG*9aDAuI0_3obM$@$mwfkJVv5vJdSX8HtL{m%L#&LU+VWBa6|F1Vib~l-&Jr8T z6Hx2(ly1?GGuQTH+_*Sl0yA&^1;;b55xnwWnKjznNPn)KX&p}s_+-FIy8weh+d8~3HW8->p;N*4N z;CGmx@o;TtYSg!FB~ZW5HR8wNRwn(B#>uB;die&jR2Ny6{De<+=9@D-5PPje0Ah|B zzKwGA_V^Y*d^kV%+bIe|nqF#Xk1O0V0^MU1RZe#R&`pF>45q*iFw@V3#=#^hI9D=N=KqJt`+`468rVVw-WWK_8a7kwVHGPizs0{0b zeM(%xAuEkM69Y&KhK*Orpu-F&Kg%1=Y2@Gr=Q`5RsI)o!&DR#2I4>@|jh>~|vp%ZM zqF*Pb3XW%@@JpcLC>KaaxgZt)z@RTA(ewRd5P?K8tOs|6ibaI;!#sDBk_Pnj9I9YY(O%R3 zNcBULkJlxzeh1%Wii6Dnisca}SLE%k!&GFPV(!75AJ?>A_J2SMIJAk+Dm+%$)r`B+Pb>>EmE zE#uTOeJAtI5D5|oy(NPb%EWCNgeKZwW!%ZoB-~uYmCQP2i4=KJwTW21(^$-kMs(w4 zfpVITTnl%3ZkzW?+1DvL8%HYAd1+!NeA>lwX}w=AWAm1i4!smu7JW6-;;7|ElU!U` zde!pUSCMs`yBpegRwNs2%L?Qqrdpx!kM@zdpjgQp1fEVDRuH9syeLvEXtZF@Sn~cI?I3RK` zmzuY8^{N2&Ma%=El7E{%{4fh*<_w|;mwE3LJYa=Q2dk@{o(mf&x!;#}@ajtso9St^ zDo7*Jk(8kfC%n<5z{`-X2l6KuwoTh>$bK-H3&q`$)|~zyM57IOh>?Z$V8dNX}`rHb@DWYauGysUU_!MRoHtdm7l`oR2j3XAr7qw+b!4`RhLWjOo7 zH7vd1c+1#pP2Er#oBjfWgrA7g<1xY@P1~@?ul5%VXl33|x1WE2;0(V9AY-I=AzAfN zRaL8&pqc-#hxh`Em?Aebo`td&WBPr?NzDo^Mjr3))~@06)b+H*yi$YEr|C3(`{I5t zN&Qg?nFG!5VUs+b?(Sx4G67iBNAZ~$?BC|8f~fF_3KH(4!u(}by?9cV1C-il8Z_U4 zx|#xfy*<6XCDT)ZG#7`*pznt=hwiT8xR$owl=OXLVGg{qx45^ZV`L{smoEg2b&ueJ zuGD&A4n{G-@4G}y^ASI6-?;JaasvUM3$X0XxDUQYkzNfLFL26})p4hfMcT@pkSm&R z{y9H&J4M!CU4JLeN%alqlrbeVkqc@j1?+5{7;eWpYKJK6zoZ&|Euj(Z0v0COlzmSC?G(~)MC<7bx#f)!eY}0mExoS|oxX{ghi%p=ckqZlRcjsN;nA<3Ra5%I`>fO-S$0>Yu@4oCMpFLhIbyN~ z$ym^fue8LI*2Zv=UXw4QfJ|)E=a@1x=X3HwQisBqojXRlCx(oi2AuWiC3 zanZ(_ir8;)%cL(MPl9HD)f@^L?3iwr*nl22JnvJ*@95?WF!!0%mNs4Tq}vBYv*|1I zEf`x@u58@DdM>TKB<|{$DK1S-)LhJund@H~5-bCgl#tQY!%!MW;_{FEs>i!no!PuF zJNQlfnw^OSn`EXOy1q)c_MD*QaHQ7VNhiwc2`?8f{T+d9l`e$M$hv04yo-6(w{uhi zS*Na-6mE)aY^5q8ZbP4|@|t6u1+F^HzPR);_lxU0NR(@Am)0R8$6W>rSK&gI0QTq= zn1b?bN@?Xk7qJ&_=DdHS{Gdw5A)E+*`=|%Z4|Q&bO?oc3m!Ap40o7@gwS}eDcP?Po6l^cbCj%Gpj&qX(U(3|l2?%nuHD}ZptFlvds=r}- z(7`(90w)@lP#wye8H*|~*D}Kq0iO6+M(VQqltrJK6LH?5(ismLaU90&y2MNM1Y25z zt7ig{F$@h&!(q{S(QwTrB*fa<;pDHj;%O6my?rF)hu-6l*;EI+Bs_XmJ~}b{^A5O- zc0}ei`8*1b@;F1&)%G-#s^{0di+iT^-$~B%_j3f{8A5aBaY>!PEvK-$u$_<%wnB~lztO4_qb6-9o`kR#(#GJrlQra=V8dO z_|INw(czmJ=X+m7*5uwAe_e`((XOujfPWKIHIJD9r4zBHwTtfk=hLoK@({|{AlO|h z3VC(f0XBeg=UL~kc;jT%;y(a>0C7gFFpRWWBy`H-6CkEL`!^zsPtMrgsxJV?&in3~ z8ouRlejJF>d@$J&w)*6q)p-w(ugmW9k=r>ot>wTb;S^}(&C#wLN70K30GY=bA9?6C zY!-KUv#sxK&K3M3FMHF;5E$uRTeIDFCe%Xwd?xJdgy6vc9rW*iRSzW?Bc0kHrhdrqMp)=qk5H0|Cyd}F^c{S4Nn*8xyQ`;m^Z&7e%!`JKa zmUObP23&6c8U0oJOptWa`LJojKQ%!^Dl5~>3YdrJziI!U)=!Lb!^m$~ROD4^TM_P8 zM{qW?gWT)y?d^rjlqon%8W(i)*E~V?_5*f~|5Fzth|c<|Bj6(~+yfkUd5IlVHXX$e zBL2nF09YI^-*0{qghYHH^LdO_KqH3N2gn4v0jobY9bN`kVY2|Nn%`gj1-763ns@Nj zOiurPCNaor>Vn*g=wntFe=6o7>)gJChmK;+iBweoc;&EJ%2!TbMn!dK5_ZBfJweGO z4K%%8EnjDLekm5&DPf&p)x>*l5w;kv&WW4#}5}h&86EfyNAFaZ0xDQo8 z@{mPRX|!3Am9`_#rVZX|Fu zDp{-k>H?r%yDBj_?_B-Ps?40Ee*@F~Ey!tl3gX`i%@?z!W_5LLf8t_e8tTHYzTx98%dtowOe!*$3#j z+AA2~gP*o9g2JSUx@;$hbJ^KbM}HrvXeZgHKjtybS=#Ex_YI8=Rq_*(T)u)%;0##4v`hy8lLuIv%>W;`3?RXdsgT1sQ77euvm7Q(O+`QseQ?|=IZ zx-rG@9!e&M)z&gRy`3p9bMGd>6hSRp^(k&Trt9YB@oCcHh-mePl!JnKd& z%*&g~)3XwJ!OOvqk~0(C(#k$U5=1TUt{$N`C$C1GlXl)QL{@y*#6uey;nZgDM=26! zUQmR!c*9hYv=IQ*Cm`ZX7s27-xGO zz-Z;b?%DjZ*lq0w!~c`2qKUMy^yQi$u0kBC#=q6>euLp(ehNzm2W@sgh#bo_Tcxkv z+l$mB;ZiWQ5a1KXvbz(sH1*c5{kESg(No{ox-^ZnEsYo%Pxi1~uklhkl}t-f&()YJ z$MaoAE@V!7NDH^N6x4Im7 zI;N+U!@dpUCSS+gus81VXKv z_0hXsiv9-x-wadsZyx0kY)Xl`k9KS0qEU#YD`}Gv4-x}JdPi5nk~hOh?>brEQS12l zA{*ESrYJP%IeV z@t{g<%ZwA6xkZKf9@y;t4I<{DG9C2wv2$SQ0VuOA63kEz;n5zEgMN+t=Q%Q(y9 zx7k`s6LxDLQxuYlBB(Cw(J=p6L>0Z7G}3R5;gXq=_q@$LWn)=9U%Mpx=I!Q6uXB9_ zZxdJ*0%y`&r#H+T%iNGp_T?JxN0FoD!tt|vm7wqA!hS>Rih1b7q~qpiwvoZA8V(8X zC;AnK-LRaKKelqg+@?tMqMhw^h35BQo1((PLLE{u99~>?s!(O5;Dof37{mA~Vx`2g zKN8Ay83An6)T^4EEEO5>j=(Vbb*t zkKErA3!WK);Yh9}Mn@>k$eIl$-{eMKgWl&pf+qNC76fYEL@!}4}-Q6Js4N{6* za41e&+zG*5g1ZEl;!bgg0xc9TUbJtn`}tks-^66eP^jQ5{(ShUgIXeENZDI7!f|N>E=A}W0 zT4Y!iI`Sf)N^QBNcmt5wB)n~nzkZas9fOg8(bnMkcT_$_{*FG%ZHYc~;Gji8a%R$~ z2;qI0K}K0@Z4Iwnn#jnKOWGMCD(c==6J52xgqyLjpKGssX4D8GzJHm5HDW2tVl#Yba@ zfdXwBf#kMpe4xGKG%rG2>k~&~lkR96CLDqgq-cMjayWENaiuWUvzQCG;l(bO>LeH; z_!7V3$7T+1`;<=9M4?W&;K00%8_SSR?9_SYjf@f}nTJBP{3e!AQ`uZPAB(6&2I0ee z0h5hBc{HQ-020|=BW>M%9?yV>DZz-#Ky)oAD9s!;*v24z!gs<=$Zd}`j%sA;o+!sK z?Xv1yQXf_NCD*rx2Rc_F7l#^7?TI+3Svjv%TDB>5yjvXNMv;D0%hsV|CN^5#w_(}L zx}@e+scg|t(Z=h$N&SqcK}E0Ul94FrLDE)DhX%Bi9hoP$@gRPznm6vx_{`5%QucUS z66+y`bIv1VcwP3oa<2Bz_dDH3{r0?4`hKmeH?*V)%hnna@L;5Ee>2BIf{l+dXy%Oh z2_v1MVS$!#kIp?kSuS$^Q)7m}wHUrhsV-9+i=MTv>1W~=Z(PG8G7>LNCAtGKpWCccROY3NiC-g0LV}#`_pM`Ap zDDU~nmcgf{{}%Ph9eUf;w0h}b3=`8#W@NYs6Qve2vL)VjR>!e=i=EMl)wY76Y2Y04 zG^PX+t#Pk*ll$Wl=%4uqXAxP`P+r`(3>Rn&am9_q;OakKQ5~Y4*JT-Ay27ugVlJj? zgB@tb2A8y@?2=Q;+LXBV-ZXfD1#~SJ*-+i>;+Tqm@s=|`NdSwNY-fM@7y0I*m0*<& zWYNDL$N;lNWE3yFecIKE(3yg39?I$ZWgvY>0h&G&*E>Fcr0m+>o~QDY>3|#;<7YMw z7JmKu9*utSJBK4`il*W!t&omCPhV|D;&r`v2Vrt)wkh9Xi+Q8TvZr_>rqFEvg)UX< zo0$r=NNaWvRW&H%^(DV1OgxV#iY;oh z414t)Ls<#Ca;4gf_?nuYz2+ZMDn=h`b;f;h80D+PNpwqojVVZaoK!JFKyy4;hxteSXo1u!EwCGx-Tc50c8lWQvH#~>nJs-yc zSI!`XjoT2qjR)%r-xC;Te`#MZeiFH=e1^uttdw4}WLzm9_KqpaTlMC9Ou$G3Ar_Ho z#>Nz?vQ=F&(gj+0?}%HQk66xMHPJ4#A5DjJ2vOX9Z7>EE=V+B%xHz}Cv=?sjIjh*B zK{a3Xo+nWrf3beKjb}FrJHwT%B;n{nG(#KySes9zCDt*BZ-#u?q2w51vtDKa$`*gv z%xF+qsW6b;_|q+9^x*`AvznRAPf|ypK@zUrIx!Kw_RfvD;wH&4*H-DzhPDy5oSGV& z!h%a^`B(QEHBuTwb0FH$(Fap zf2E_1Rp#Wh`#1%Ef34_a&<3C92YVtdoo;N>hJjqR?ruNkTS{3UE@SCGldbqpfpP7q zm+S`NebJ?+kxgSvFmGS_&@rZDUix-!d*&kgrcs^@>6*?S43Yhnw2CM0GEQA~D)F)f z;(kn3y@Cv`b_QB#*oc?HqQfNo2NX5U2ZZMM$;!kdj%6uLYD7ZOZ)~;&*P3%xQ(7@S5_A+s>3Lbc;2t6*O8W34`8Y*{f;Rnv0)a4c|I=uD|lvotz~G z?+~oFJjeK@?X}O<-kH9+D>oA|FIgBIvFgQ_+9j?cE>UxELs5srSW7AWcvpj*JXxE@ zeMiqb-~ZGlZKHKPs3z*pEb@e{(@GZ{FwMF=CAT$9k*` z85jyI@hzYdUo@9)v%O*dDD&=X%Tnup?q8oU61(g!+dV;MK>=NxBg+mfKNGVcuYUk3 zKAke@gGX%lONK)@T+ePoUQfMELoU@pJd(wE=TGLnY78TsR;NgOZ8Ei(f$a-0nS~3J zg;JF#uGlTMW~t2n&Qh((p~Wf3t>}5t;g90Q)o-NI2_%)m+^}@1EoP%+Hz7fJ@a&U< zjYH4hUqS>pWm`l0;cpE;GV+Ms15j>#_$Flu|a0>{qm%3v>vyPph7z;$n z4D^k;;);ipxh^hZ@7bfjWt5Ac#=K?d5^ZFZHai7@z8Rp$z0ZnX-W>YI+A1Zkl4U2f>S!+Sx`#Q5f7mMbO%6OxCai_mD3xC`twkaL8h~a!ZV72w< zM&;UeDlIM3sF||f`W(Si&$+_Zp~nL#UV0(SZgudKY7kj|FL!j^4SqF;*dU! zD9$5$#Pj#1|JnR%($}(hp1e)Yu0qXPZB~+%alobYV+642l~C90r!|O6`bX-C+9uE@D0^z?1#~Z!*U( zi;LeO*89w|koimr;*?*ACA6;~B7N0Pt3EM>eyt=9oi!}iJQ@9^AO>Bqk=IY31^a7^ zr4Bjsvac3>U14MgmUUJZnc6K5d9~xN0H>TtzbRT{PmbIqq(F(Ok*bWr}o6bH^l(4rk->nIf?gu02s*>}e=6)BtV))BQfqFKE zkJc=B^=MS@0X0-ux_IxlY?q1!KKW%OZSWvo*T&xg5M^5fjScO?h(3^tbdm z>brTCKWB-hg~!e9Nm5q^rz{&^a@IO$*qOq_al>ZtN<{u2?j0|)3RNwn3w`c^K!NBJ zb?4)NUNQ`adq~_IYht9R0)s;UHWBNDD^ct3;2Wl%TCi_Lxv5+Nd^7_0L>^8ni{h#2 z%*-El449`A0+!)P_2;rGm$EJ`}?6_nfSL;u><8wdDtI|#$#yP zc&r-gA`E>~!kR8sX~vpZPzh?cNK6$ge%ca!PD)wUJSETP1G$ z6GLx_!J~kJ*BZhZtMH>zQ(ax`4cWF^@AYF^nlI%Nqdlr1rg22Toj^tsMVtJ6(fqCv zBbtv048GJX@xCiLsU$o;wrlHw5Rx!ti;B)RZ?clLzxlhDx~0bp=Np*(A)PVMdw zlH5!OI0uQW{b4@YzZh^VrN;aTaBkgiZn>aL6x=76NAAb>0uoYAxsN{byu&%WKim2v z*$Oapse8P=-rc{xPsDm~4KCf@zv@K?pA1%CAlGvwWUv|FBmPWv8W<=|_XGR^bG&W$ zGlYBiF+{WfXqXbe|N942k&CX|`ki4&z?k&)NeD*R-?Oc$fvwiZ7EBKW{r^PcB;dz; zQUV}D<9ybJeEXMUuo1vF2prh6I}X4h$eRwH4nP7B^vFahWu8QYFo1yF zN$?Il9n22y$prwuUfokYe**Y|-5^_)0j>8p0FPxK%o^@Zt7|C9Zls&8z_i#0q`owU_WpCkh{l8 z?!TfDhj?Z&!eXw3Hm~e*$&7&5q?KWJo}Acy2y))4%hvJA`RVa!aD8CjbN(B#-Wo z@(ua;?k)7cYPS#X{sD9jo}R&%!vShH0sjDom^2+_vgpnU%8Uxm^EAZoYQdWMoX>)2B+>So~`@p?pt`lv+%Q^ zaHeKj@$gnPdmAp%MX?yIo!ngFmKLskgyU=NV17wu(7L6A*!!79^yv-5`dPGqpMaNJ zO9TA@rq3_t8@Nl&?}@M(@{&Xp2@ zB6Jx8qZOn$?COMfgKh}WI9DhH)qZ(Rm+gJ;8<(;0cQsF=wr$UF%l7qWysiV;hD9y= z_Nsy5WTOTX)ofjAb__?_)Kwdwj?mV+kYgpH(&b!e*+g|YW3yOV(d_X~=843E(pq&U2yHe^VLse@sVyz08r=3^2nC;4wYh^1A-n1L-^t~tIhi}Au zGvUX1EPL!v81_qe!|${{yVZcJSkvAlKNv1Bkkp6n2gKVbdc@udpIq-V=Pt?Zqb zJk}0jc1+>X4>!8d-A>PTef8gy6q}UpV%JvnA9lXTs8(U&31)?w>a*#Mr3I?GslK=i zquq$;Q?%0MV|U>VAE8v5IMIVW&y~x{>AP@Usbmm}vP@A-6rj?@?fF$x2l>nU%k`Lg z{naU^fR$r8<6}pRw^utS#DHN1}X*F zWUmmCbdKXpXpf!wC%G4V@ZZ`Cdj2}+sP{j>@<>1lg*Zm{9i0j_I8)QuM~gY zt*=3(2dtnA1Jl!47^@&Chw!kkAFhYV7_xY|NjOCI8zyAH5O3Ndu-jydB=P@vC>8ZN zYxve0VL@QPja;zUTxmy&17W!UbEe00rUSJp6L93OR|TJoS_qsYjPr&pd+A*tdiezL z4C`;xu(_1RrMJEIZ7NB}2NB-we5M$ekF1a-qH}8Uu9@xze{L`AN*XEaDAx;{k7Z#p z6IV-_71oGDAaN~LhQLc}*~}*y=@wE>>k6j>wgj5~*lHyW3<1>)UYpu(3j{eQG^1Z3 zzAXC#kcL*LjV5Jtv;mP~%A=GO04t?#hqEeVbcuFI*z3;r5FcupvngI3Y7Y4cMqXUM zjH?(oYHkiEPE#5na&dD$ah_Vwlej|`0~S2xqIJz&eHDzf1`6gsX-l@WXj;0B(rgqspPGdjU_?@!3~0~aIhP5zeA?cQ z{ntcKS+}cqqT$Od$ArHXcfd8I$EI>p&ktnWAE@fVP~(oX0Pbu`%l~}e15}fb@Z$b8Jlb>A*1K_O?zYNU*IG-a^4isdaN}S zG-!Ws5qu1i)o3(X;CPi2hk%J|Mju~$-v(t4 z{Xk@&Ui(+)5E92P`9ZI}(8|#qlRltE0TMMHMFnTOqy)YsMbP@ITc z+sE$<12Ht?53v>IzbIT2abRhDu3!C`m8i^+mDG6s53qUucJ)P~(3=~zKOGT_*1OZV zoAa@a8RR)s_yRwa*?{42K!7X=jvl%mV*2-c4mlBG9$poOYdL7cO)^8th?n!|l74^7 z1b$z_VpCIXynp!%^{jX7?J>KBc+ygK0p{xKmCW{WfJ(LXU#3PfHpOy`5!~9%tAss+ z7~D}#yrmu9@AQVMZW`xXs$LF0iWp&GblDF|T%$sJn-hW6t*<>MW{B5TbX=4;$q96r zPOs7|k!Z;yPsn4E7b*1Fe8PQKayAo8Wa4Z;s{xAuL4^RxZuRd-JmTI525N~d z*iE)P_}3!VMe0o2MB&wl6aN(<^0+fM{}3OEkoxZ3`AKLcbJipG=6eoMQ^m9ERl`C0 z`eu?1`nU1{c8`9gXcBjYo;20$MX*nwu11a8XrQuoba51FFPOSoszzy|O7dGBk4}@V zZkENK{t0!BmH5rL8izPs=SXD1;dRT8O)W@X{R>O z8JAEM$rhyXt)(E43#08Xikr`IE(W1(h!~ImKRuaX!g) zr*KN3mU_y9;#c6;5`QPl;`)N%u%132>F%O#8^@met$81EcVM=ee--TQwCMPe$7bS{ z;u?6&UFnQOij*fdhf56A($jd|{o#9d9GCp}*{g+Dadqsw6xPB?D^X*lF$fN|CqVuW zD`evK&_U;bK!qV`VANQI+<{zq_S^hb4lC2m@dfg6b#V&T!esq5vA0SKI=IS=CCeLP zF_E4W1TjAvPH4!I&cW5EgG}JG_Z}&23r5)GGuz5hX+8dYsR$;H%2anu>{2v3mfqNC zw(MxOFv}?N{4dYb)Ww^tB5<2Y1tUipw)(0mHo5jF+!&RL67*EfnlYz8_FgvQ#{we zj%f*Yi78j)hlyQ!IuWwm_q-9l^7+;;6_xT+BvhoNI_f?y9M~Jdh@nV9mSlxmY?KP9 zqMXMU2uejiJYJb!CQb9&6%oQNiF7qIQKWI3{oe4Vy08n|nWL#sKIEH0D4arq8SL0P1rI^3gn)C91l`nK#j=v(Rxhi`RG|!gp-D> zv2bFG^R=%=X2azbtx-irb`%x0fgr_{-kmT;7bY#PQJ+t`V<59s&*ZZDBf?c3*lvX} z&?^Na!mHY%E+WHSCZ%Sw!HnpTH8Fo(hXg*%bKcZ0&S;~eUW6K@s?E!y`97?Ug)l`{ zMmb`_$O9Tj1#D)_b&7I8hAvbTbd=nI>E(J*b<+()o~Qz4USK-uWcQl;E&*L=@oD(t z>KD}66%@_CY8V=8Qk_BEOv*8O5#td!);!;vcS~r)nlUx zL)A;^8d3M7jlY=b*pI`jX>MkLy6&JgH_&QxXlrpW zy0y4`35RjfNj-cELR{EJjV3xbtswCCRPycE0YSlUKBmxdqR)hi{X(+BCQ5YlbbPGR z^X8cg0Y~r3hlwE^_t~n^S;}q}W`E+LY48y2Y*C_7+*-Si5=3TdQkUSUnpF<6s|s-fJ0lX&$^u6-Ooayrhv zW*helb&c@SRCshJ{l;9WCs`P1M5UTso_CtZ`jU{csF2w2{fzTx=|`5M9Kxv-f6z>^ zy5teY@eCiwVzkaS1CZ}insSizB4K$e)LI=D5siwSVQ&v$+BNv3YI?x2`VZi;V0&j! zDZxyZQ25(6Om_uo;~lJ`u&Ze*DlS;u@SCe%yfzVztxScBsMbhe{fyI)+~sO$2yLlm zH&*zL$g%dDt(%dZohk*c#kPiF?5sqCCgEy-QcuNR7#*ktcv)S1dAJm|Opxfin<652 zXMwK1B02r0VHdlCcL0tt{_?5;#u8efB}LV$!zjY3pxJKjM(6S-_~`9WTVsWn$cm(z zVK3iUb6Fl|2p-@inzBKdD=`!!`7uBECuhw6jn)4bb;scu#T|50UE4NdSRMUY>5n&B z2O#vvcXg^LPZ0mx`?eVNz3pUyXMf48y&RKX=jWJl;^UpxOV3s;SH{uCpBRzHU!aAl zv#{33G)cxrl2ZLONNehf8MUgO*UglBsMtdoZt>U)G9kqSsi41G747YLJGu3F+v?qp zp63HO_;E(rEeweB=2HlhHhstjI98!rap@>;fs-ax~vLsDi=uz-HMtgLloY=(n z2KC5GD~gyt!)E^g$Gm_bzuDZ}B9Fy2&NJ#{s|pKwZ;qtEcbolf>sgz*xKdH}`q}62Q#wYQu$9X6#0%T7HFU=(lme1FpWSUP9 z?6d~dXWZQRjk=^Mn))k}TRNKqaSViTcFft`TyE#6JA%CGezDt6F@o_YL`C#mb76nT z#MtI5n(pvO$9u4$8qHwDAOo_Z7#>#dIjAb942yrNXB`f(UgdF#KVIV#{Y6yw=ks{0 zAKJ`jp`MX98!45>7p{CPjVioI!Jq3PitA>N&M?Put4HsRDc+Q?`#pAKoJ!(#PTwo0 zTzNC71q#|R-=<&XaoDhs_&Ksv&aEWQMXIWmJF~rc2PuAqH+jn{w0A2gyp~MDSIhDz zk8vPC2Sz4TC@HA>)npvVCt8a1X|Os)X%Ug?`~0TSYBX}Mm1E6cB3|ueoiEM){I1MF zKScs>$b^-xA$4`WWXVx~N(Y4=UcUkA5DvgJ>p69J_S1|TFv0F(MCOG{)Dvz+1%8$qhB2)30FKlO z+)tjj`gM}TY9BLaeYnQmR2!OA&Ef*BGLMZP(uc;+gpI+G(ZUJU7dV_3QY=_dY8G4D zH$y)a7D^XCD4==!XV%AbthGl@+}B;=@IPoSm6I63IK~BwWGjYs7 zI)pHo$69Q$ZNa^i&Y&|8d~iGHAoND#vtC@nI|zj1<)9T@%GX+-GLn4}*hOt%|Csice*hclmWPX!iHasCNr#bdOz>64BwHDx+wUxm`NNUv zv>tmN;cdrRH5l1o^&^2cCXhtIQ$W5WpZm3@NIA1k&D6>fi#Il5E`!5-IZoRp5P@0W z2KeGKY}qydJx$)|2z|(ZH=C_=C?K(ZEOPWYepb@M6^8UZCkJ_fgL4pf=j&i1xlEsw zU*R1u$PP2BpqD4Z-t`SnfFty(yj+%MV)5o-P->w@d`j|}t2Ha0YBW$Y<&E)5Vr4j- zPCQ}n198+StNnM*Ma91km+MOT)*eUX`d@-Qd|`AqiUtzx14RheMJ2SP-ly@{Bc!`! zqBYIP>!n*M5hRV=jn=u9;F(QEHPQcT(1TGqn`#Akzp=%NNzz9ntga(MDi}-=fLI%O z8g|L0lXxA1ISn9mIlI_RC~Ld!MbA6A`v))qyzCwZ-~tpPuAh;!SZ2U0j2;4W$;Z4J z0NUBno@B@elv^M zpBcS4PVAHHlvghrQCYTdUOMK!<&)G9$^V9|10nALmv_OzEy92gC^z>p31!{?0BLhh z0ELi8%(T2Y|GN*U!Q9+SsQ{74`w&hD_cc1+vwkl)n*Z+0C(JLH6E1Ep@Q$+&uWQQ+ zrbgFzR7eH)n;*>910Fs|AWt9bm+LZ?lb!zn_^w3%(~c){5m3-*C2oyRx1{_btVLF0 z5+`a7+<}iQ|N4C(>+g4w(x(4It-tK9MRjv2<%Ahp(GcphD-nRJnbh*mr)L{7+$Nc} z3#jo)@_^8J(x^RC=fpjI>h(7_pefBSReEQbqcmL9r zEA_sB+Z7)32r_YeBe=JIfjnAM4m%i3{{Tq8t<2V`zu7XXo?Gt7E!pKd12SyOZb&VB z*|zJ#o%(;Bq+rxEV#Pak*&15fOm8A0r_EhJIUaht^M(%a2Im|?J0-d z->mQAgR+CO*(wmN$T4*pq)xTn?MIL-{*T-L4b&6=+U%(YsgxM-Rh#08X^p~+yX}0_ zM$~vs3%@0c=3(q;KqnQ8C)mKz+N@1|#4nlOTsiP1yl9BGj(WY&B07a6Bzf4G1UlhN z)xT$p3cgBu@ifjsoxt0sdy$2K5E zA}hS4YS1@K;9B83@@Sbr2i^7;BTJ>BF+b;wbc~&@+Xi~ch9at-JH_fs;7-91R;pTF^E~SssL|dKH#MY~oMh2&pT*B1N z?c@W!_r}ioH9j5_x9_r&KX5mH6Qca$rlHc5kYgpt#Mv&%$Gd;vd7;OYZ^YRehEDvd z+UFhGo~&M3fy68}c61~c1Ky~uuTxa+sM?5iMz@}ptlvWmZhe}qdf?DJu?lc8>p;8S zmyrv`0jGXxXmzAH0#?RJAZMRZn^xlk=TGg>OFo}f_qFG~%RK5?JW%pvvh=yvNM*#S%Rvl(#$=K1O(cci6gEoM>NA(~{lU=~@+ON-!bKS||PVAZk5`ABOoN;Ix4 z3kn0MLSgx@Qc&7c!)O(3!R5f{$Z>ICeEjbU8>1bT3{Aczd>$5gI6dXT_zDRw}$5alx16sf58vB>$tKKlI>FfP#Za~yXY zCT?f|6{@f?Dj2k_KD-`gAo?_fr=g;s7fkpa2y4`mvvS?oUZFh~baAfH-2Z|XwJqk; z{P~dw!cNdts73$zx5CZYR>JfWihDxr4N?w}_^>uFjJ^)xbP;nBmgovvE-YSH{`i#n zBO9_ppp5^mtdSux{d~jMVVe^HFIVgZ0RFbTxkSJFsBUU3s3*@m{Um7QSzsR>x0SQL zvgGUM^CW88iMyP32_2BI%81^-}kz z(^4(t?+vrWE^dLCBj?@yj$gZeD)fK2K`O97Bw@KR8}K3d*oD# zxMSje*(y5b@sIX8VCY0RwTj$?ip6|&%Ze961i2TEvxPjX7+Q>XN=l8PqPJ05Gmf9k z%)Bph+Vh#OuK$zsWTui)j9hq zLl?itQNm6{NK-11h^tgC+eNj_W8yx26H)E$-i!R%u;EYm@vyRnP2Y+D<7RAzNVoLMt9l+WvUFu0;FWOfNU>ty7r$Ysdc< z0L$Js21BYE@pEJ!kz<70MF7fyfL4M;i6xhA9yVG~f3l0($1wvIBtcS?jLNlNQsu z5|hdTfr88=`f3k56HgC|F}SoVVF;xr_bRxDE}QO3iL_Y5t&x~?ikPab`D}tMuIjBR zOSP~Bq5AO=FUQi)`6R|Zm2*&YxteB?*DG>Edlk*%hQt-D`K*GQnS#_D=ZfP@kdBrM z?`KNd9Rx25$8*OEgMK@h>Tb4sHk#71u^%9RsghN=vLc?66Mb5sPw5nvPni+ozxfgeM{>EVr@d7|W z*uZN0Q-Et3F{4V!ionWzYe#i%d7#qw%?bM*T}7^O$G0m)$LyWcJ(LgP`=L6QXe*i} zvOgApKS78)$G(p8cZ@2o@O_y&95&SvXHB>TZ8M79@q&s>quDam5_JRYod?3&7t+1t zu6^L}vfRhU)NQ2cYf{FhwUu(@j(3uRyaHJgP53`RQ$+r$XG?|N(uY1>e$lG11y&qY zj^!v;Gco5cK8*Q=vk6{JU8D0nrNCHzpCJSV>5T_pe^mVN@m{Vb*k7!0YjJ*UR+Fp8 zdNyxROmSx+x?F&q{-pyN?29OCj{{}lK^V9ET$<`zdM5e$Nug4tW# z+&DHuF|@Y&S#cypq{|SxKBcH_;W-s>B*ga1(Vw5XMw(6E9IDnd*tfo4LT37mqK}Qc zD7Sg(LCjXpp&5*DK7u{5}&E9_M=}>B@#-?XkHuXnWuSfnS-OodA z-GSU~HAMnV`=Etn+NRg5mBAVPu|qErt;`9x2|*?@^f8aMK#XBsq1dr%WmK<|1;+mGDA0)0t zu{c+KCDF70&LM0;?HtzM>aXdeW^2qRLX+}&Wu<{Sv*wnadVe9lP{Wl`U1@?uB|eSf zcDY8@zJYkHxFCIIcUrA1<^fJbH(aAp(O1&L*|Y#o()y|x?xJB(HpOhk;GHcQF0UIh z+RAH<%yN|pu0&6rMsYPjrXeM^%+(ChTs3gaMxmYzI!^&HnIAb5E#q|45X%-u%~7|5 zaM?#+h_aq6vMQ%UYIoQ>yJ-&_W&nmhnMk0%a0ruRb1zDp+%a}}AO8XhU{#ZIvk6+oWr3N$lDFk~@}eqUJ-U|4A@SSW$oYD0UWni;## z(Q4%#-yvr%1j__r#PeH` zJ`#)Wx`!Ca)gV>_1Xv@l?gLcV!}&3XVjM2BO_vohtBUwyr(6-+*QZ)y$z7y_SXDs2 zFBOA4e7rLI)5^aU%gM+_qSpWrPXrD1Nf`)Lw?1a2#4kjN2Zxr z83l%rQ_Y*d^{983T)MEU;SCPlM3hV_gX?br48Tk zC8Cnft$eZO6|rUltCLT^W)w~|pK2Af2j}Y`&d{2gX(E5!xyUizO*Rz07*aGREMde- zvC3d&{@Nz9^5chR&6RQJw9D|w+oqb3N3B+=8cU%73_W2leI4~l-Imv)Suu(J_B-G3 zeVxci0!~g5FE6=No*~1Cs&?%wR%LD0#!AzwroB3VLsFNJYBtAi!S018}r$t2$K1rg%PQ&AguAzZ51);Z*WuRg;zl z{@{f79!(;~AU%Xyh{IfnsMlBuEkXCA=V$VF3!Bj5iuc(kdk?O0P^#_-{%8h%wF{6KzkuIxJFh!8-h_J`MI65dhm5SgHx+DqIOH;!{D!2E zfjX>Sw(|)O_GyCXzC?3YOk=u4^Sgf7HKel{LaKTJ>>mXLrxi8s_?D<$sK_R>1R>Sg z{{VE?h5e56wl?EK3TeXXaMWfBdRY*03K3EDs;GW*0Xv)X;%6@ll$W0dtu53nmTOBg zcC2@loWIj1Ov1B*)F^zA`aL2oR(!5iZ{SFs{*u`1KcQb+w=|BuARHR}B~`X>qmMK^ zPZM0kU*H_pSR}|6Or+2b@TX=PN2kgHRDm*VI@;2$?XaSD>3JnfnO}Ok&Rb?9gr}=) z?Zrp7X_Vt=h^J30q2o>w;UY0Y%11Q1Y5MrBon1=MpKLg63)`I`JGhx8Dd3&lUFW|q zdLy0W+2mZ(`s(q>J)Ocke0wM&NSu|~Ie2R|l;W`|Oh2FYiGi03DB%#N7%N3%bO{XG$VNh|daz1r8kU(F)j|L zCu%tS+y}Tf5?%ZcfQS8tbUuDd?d#Bu)XsIX9(l*e9S3ut<*O&9c5$W`#*rht+gm{e z4NpST^#zjs?iT1F#Q`dHR2@*xtG(O$T?#7d`)~(-ez70GK1SXsuqW;|S+AQ@<0wgI zy&Zx8-Hje*qvX@?JQFM6cvV|sbof;_v>!*?>OdJw2ijaTIa%24aWy6T=nsuvM)m&zqBs)2kTk5&mCV@wbTCz&Eb$4vDn4@)bK`xG zU|e=#u_p?4J#-elD=}b&Bdc~J7F8b|Ol9zd%MQV*Zp$u0yeXUmO%9-yNd);&j!Iic zr>#E1Sj-aHBJ!wJJ|sA#b8pFH&%E#Lw2|q5%lpV*5cR`2UV$uOGNCC?U3Bd@&B(pL z&>LcTFc1})pg!F+awbH?UMoyUo|E~Gb`(2*p2|p0#~ggYa;+e|`c4b9%aw(r=I7(@ zWp9#sS+P0s!Vl&&IhYGYm}*_>-0AW?8dz+a?hK7mFM3FQSsyL;EtG|ru%qL9Gmkr7 zNI1P<6PeRUuPD+gKN{|_>upk?P+>?7Z;(9oI->xmw4bG3VpFJ1AxP^IiHW&e*AM6) z(1vRSU+_0(xx`j&BOd)BJYhivTU28mjaw1#H`-mt&VS7v#zp*4m5mOeeHRlGbJqN@ zdLzYIpQP5&Vf9evpU$d`S#5B1p$db&+=!0Ok7aaZNbki1 zF39H0sEEsQ!g`30yjnoLNiA#s{bkD6t25Zgt~cz~>C=a|sk|qI<)Wt~XLz$a1wT4s z!Xk+B3%XLu3}3lbbb@Bd=Z9ST{+c5s`em(Z-6t0KLTe@Jgyl?lnoIz`1PGSs=D7s4 z3lic)?qBy$ew3)~zFfYXU#c;`D6~lWpIkiQqB|b$9#rY(pWL?uKGqS{&HNXkFMx!yBj~rwE}C0{hxZu-(r!^wPZcfk!v~9 zh!M6FT(1F;bLhAB65mc*@)jS0T6^)V-OfIbdot1QKYf!lc{W6;7QVfEn+VlFJUmN* zV^P}!X-C}%UCeWbUAUIEARB+sD}=PLw5;S|XBItz%kdZwv6!I^-6rzrxxGlbt^Zf^ zb2<3 znlt34-RS|S>~~hdPqu+>ko2sV7O-r+2(`Rx9vURipuG- z^dY$R@q;*{g7zwSypK};K;q`^>o5f6dLQMN`(nURNDKwq^6djsT8H`H&rJFMvB|U+ zKY5ffFz;M8`-2`<{{m{ZYR~pVUe>KagztA){;CE0hGZ+FA@?6dkU{SMF-m9zNCk|~ z02<*_snh~sP#PET9))UpB-s4_Zwc*1BVIM_>BYeLfeclfr?O$u?{Gd13D;}gAkTOp z(393}z>BsA)9$Na8?Kht{up~(=2EAw-O`)Wlh?U0;#c(|n`n9{}qg zfas8?*8Hdni8=HjuMyxt%Zn(De}JRw;H%-C>hnZ)!{c+F8N+2TL8s{@DBvA;9bVFb zd*#zHcg$iB%GoK;r`_ANn9c}YEnw$I2!vRg*&ol+B1o9><7Dj-~s4)1Iq^r4VlI?3{8%*`R1WPpI=iZ4zgSSgnli*5qN0agkYM|<@ zK+Vp`ERm`-7TX&^9Sm^Jt?-~whfm}tr8ex{Y$#OsWp6ZwFj)*4S8GQKN7)Ld$Cj+d z^TXrGuNcNvu(-N8TmN!ke@;(~e)X|3&9n90UGnW9!K%lT&Wez5cP_(V?UaE|k{&8L4U*Mb;=M{sbwWl_jw znlruXgf7r16excmmvE>r=T8T9S!Vy@*mH<6tI1j}74bSH0{B+P`&3MmS zO%jE{bNUa^hB#U)K>r6I{Ihe*kn6we@BU$sk;$I*R>`c82J_c5&>C%n&S}%WLZg9} zyfVNA2L~f@F3;)sJoCWoJmnyz9JGpE#A~2g3Xjxi^CXgsAF?jTXGp{k>nwJDY+Y=K zJNd}TI4pU+*t4Oz>?zA5 zq3{!xLgiE|ZMx6W(M0SqD8q~;*CKxy42yAaoD_E*eQ}Msw`I%aw8!(>{$BvE4p8yxRko>(#H`Wc zy-}rA>F>&>)aldWwCnMvLZ1Qn(&I>B#@bz<~iW<7oKYyK8Dj|h50SE|3&+NqIIn<;e(vnmsvmnG?M%%37+-H|d(i7R#I zq{ww^b@$wHPCV+=M^fyC{%-wtk%HGc#HTz-h(z>*1d!N+ow}|#5BPm8S zd#Ib}y~*H9I-Za+?X$x~$kx$*J^RV>O+{pQ4nD$cx8qkoCVAVN`51whAc!~j9dP-oGk&ixe5 zH`rrE_SS^0_u1nmB$UlrWa>&mlv1KS5R8nkBPFPej@X<==sHY4XKtS&HJVJPEAr$7 zfP85`y(&uaT+|d)tL?tpKZk7qCmBaS^=q+QT1*SDYA*d_+Xh%PZ|@dP^jiW_u4GQm zux3P1D!Yfpkd&ns?r(UDS>pPGiW?q4J{k&=ER`teIQ*;mqec_W!ZJ+LveaLFvT4*vkdN#D$+H$J4?{mvr4i1&98;?2JI{qG@!I?&ob za!Ou=>kO>w(k@78XMMNa1HJEId_}L^p8oS4E7^&e(@e_fEBw|rk2Rq!adVtEBdtv> z6?AD@RCOm;g0o^tAZk%nxC%EGIK~+klemw3jwONj53*{d=O^CM+=;4mI~OwfgOQcx zt8OFf{j5}4RL+ZKD&&|fJyuOi+=TT72mRH^0z#Q3DhVm^ikwZ-RDz`h6on`p9#s9) zhNFA$b{x0#U~JT~wiauFHu@jzbEtZPT2=nB5f$*f$|LgqL>eoy5|4!`d<2fb z6{l_Sh-2{ciH^@nB#}uy5=sRFy!8#P%8s~%0+m?!5-)HFIvP`eDI^rBB}p36pi<&) zg@quLo3x~=l#7$#GA=&`;_w?~kyGN;qB5nt5~>|aVR5J4W=N45oZFGqiEOysrqIHq zI+mGj}H=tvDY8MQKpQ5DwL884S)27~WBSrk!^}NQn%V>`c-`MOF!QhO<7h*rIVbDJ2*)WmD$rJjbN;5q=^fwjRmU`m&Q2 z3>ZjmdCn0N;zmR`>>?vcxLP7bOEBAYExv{l^KGp%OxU(R-pugA;*i&9_?}R`B+W`E zdR67!jb$lDh9vAoQYtewO4TgXqnVkRnU6oRq~{@jr;1}z<_Xa&6nD{BqS5QkA{9?E zJ-WIW_rH8r@@ z_~jJ3OE9Ual4VX2MCsF{NVRw~s^E{*IP`A$YAm~kKlB})50`qLsHkd<08#Hg(VBgT zU0BPCZs}4?XleM+g{TK)QP^-LYf6@lZ2tfaT2w9xzT}f4`2PTzSBUyI+8T{4Tp)1j zzGE#*X%6-4H^CuDjC0izDpJDN8P=MuRp=DTf}_cfrliJ@xK&xK#+Mz|rBl?VAE&(* zL?}+jpp!GCTO{U_dUludR_Z-vc)FsSG*7d~XQEk?W>iHk)fQ%w(Z$W__0?KG%bsQH z-z6$!X*C&=iAggdw>=|ubqa(S@`x<10v|1~ltf4_%*%kVk912O946IKsl6Va-k1?& z)hd#sMrut;bFwx@3NvECp_$j|lyD@I(ujzxA|)A|2#BI_fajTB$(yr{Akl&n%}kq9 zOwf`dHbjaUn{%V9FjQErOdx_<*3Cj2~hP1%-5zawxK88T>p_W_)`t9~S#YV8z~$ZJB14a8k>of?66# zre|SGjiUC<=t`o#f{~=lVUjqzZW>|~4XjGENO=ix-NYq&P^5&UCbL^lcYB$5jY@-2 zlgeObQDHrO7Kh%FjyKwI(N3YE4a!?+LINBMLXc9GC&Ln=fY1sG31oP#5XyA7sD)_M zl##AIkrOk4Va4iv_7AEuN@-_oqj}u!uG}>WjOJbi;$v$%cJr@$ zICT-1R;9NBtw@y(Dm%xKCD^_@t*BXHPIzdzDk&EDK)Jr@M;V<4^%FmtF~#zg2`_2+ zJYX$N z(Qk*5tNDp0rcm^#$~?|yZlxHA`z%Q(Epo-0N(lQ=>!MbC(VXwJhnqbsTGi2RoIF%Z zt)ntBh2P>!QS+sBmJkpAEi{Co1F}*vtrCpg93p>({{V}HmRLHfKh~7@@lL<}G0bM< zlA0JPBwkc>SP}w5jUj6&P3+o=t~H<~VFgWv2GpdPr|!Ey`$dDn`}6)A&&Y;5sBhvE zxzG*IH&7(l@&))DCw(AX;*#twz6K<_3vYpsHhsP-bqF0StEL>cqlr>!P#^HsVz_f^ zuwqMNR_a0|MRnF4C@E5!acC;Ch&boJyJOGm5Y)2$yH|XYeWIXlgJuVEn(A#~DNmGp_K`y+}DN=4l*>SZCo%SQs7tCE<)Xa@cl4fzo zR=I8(B?3g~Zlvr)#@~ZsV-fBrN%}d{i8OfAq-FOWk{CmpoUP^+AgkdwQwFWzC|sMR zO%$Hs;?Yj*QkgSJPhP!a>71H&U}$D$%_joBK4WFF?o-Q99SuXGQ=_353Gm|}ELkce z_=Fb;1P#i`+nigG5m-ZfB$8dqsX|9);gq3MU?dGOSsV6GkpK^urac&iwrf=SXTu$W z;9?|Xi-ubtA?kZ@a1|71ROe-yqGPaav?OeViyIAx7_OA2u}ahL?%X1k9b=u5hiLAY zN=m{=8b*_AdFsEZ;B0oqL|S`1NKjwGd%MVY(A#XdkWi8?b|7{b$TLYA3MI@4`^;iq z4B;5v9{&I$>>B}+DXx%F7QJ4ns_lE#0+pB|wf5B0L7e1Okt5!iY5xG>uXhj$ z_(t1+2(`}#10ph==6oPEsuAJ4tt`t&Xr@+tFvRgA9%c0*31V!NGLYI5vI0}E1LN)V z8}qg`3Qy(_YP<*B`5b1aS5r~M$~NxvF{ph~q*8MZUaZk8)W{W@Jx!PCwJKdsBNJ)! z>GH{qDje978*zy-q%h*zd6$s1I@*d>2*w5KYzVdg0QU}wRVrZx6icJtWAPW(*sWwA zXwI)>yrN#C<&3Y@9PvS+QjsQ1#_9B$R%WQdr^8H&gR4^Bb7j*6cRc;)>K#o2c@3{{T4O5%8A&^BjHq zqR`Q}s}Mq6X=*}>NKgtkAn%V8 zV_lvj2TPGD-L+}J$hUa!AE*RwkWVCBpEHdJV1-AK@+ayoZ_fY|fCm@=U;&8!zx%!* z7?23!!?zzx8~`~c{f^$71NqWZA3^^B%73Z-0f6x(6C&H(Vx?>e?l&KCfF@@Pjz`#Hs{zOB z>DvJjNjv`l`M-%E{ulr!a0%q|^gD5Y2<2oR0RG-zGxxwt^?%;*5TG~oCg1*L{{W!C zNhWgEMIZgRc$g>vQ(@owkMDpa%*jzZybs7!K-*!t-{=jn5Gh2<4-z574?iLJ5swpL zY%bwg%4M|pk&5gteF|;qdtdL5MLUaGDm$VoSM(>I;@10MBAZKWJ7i*mwe7Wtx24i= J_qGGU|Jk?Iys-cP literal 0 HcmV?d00001 diff --git a/examples/frontend_language/quick_start/images/dog.jpeg b/examples/frontend_language/quick_start/images/dog.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..dc4b40e2107bf4fa949352ad31d6cab5a2ea18b9 GIT binary patch literal 416580 zcmbrmc|4Tu|2BSM;!>6wd)daAER~S5Gc(pvmS|HdTenoUkey-d$rcq^My1qU!hKr{ zV-SicgpjdMNQ1#x#`s-oK&-#^dg#hf$OIp=w;@AvUOj^naDussUEPnubn z0UR6vzybXL+v6Mq7RJVor|oRaEG$j`tY`zE1*Qf7o?iaGcIHM%`-_*5JfnZUv9r$E z)z4?=_5ZLyCzUi-_>73#16{i1!L$)SxEf!ZS1|Fo5M+Ae?E z`aA7Fe;3!3gXhiyYbtE8aq@`iq{10KL% z01~hOyZ~3=DAYOzr~&GL=JpU^44uyZyJcsP|GN>OZ*MmM2$1s=*H11E1i*>l;6iY0 zHv_WJ@#fkCt^Qv-3vh98!hk*8JRtAheNYEWIMjg?#>EZW!v(#aaaHTOsG$+)?QZ(3HoX zlu^EdB4hFzMt0DIXyQM|ja_{2cZ->vdD?i;Rb1i3*=yr|{t}9&F&X9Cg8&~F2eivv z2mk{lL~=SHaUdY$<8|%#wd@Q9H)d<4{CKU1r36nVW5c{d)|LnsWDYG$WKEIFvsRI9 zO?Bk8dUe4y0=tXdW>m41TJ($HVbPFRuSgf!2GsS|w}I87d0xF>Prtc#frP8W#iB&; zYGh6rb`G)WJiK>k+;HJN=Z$ghUsSwL*6^i%TUk)sAa$)CK01H<|)G=9Wn) zsObGf(rFblzGfVqzp>dI$gMy{sqB-f-3DGcQmOp0UhlR6A@bEV`;|2k4R-j>A|~i% zO8q41Ex5mmj%JN@TTh4EFo}bsLI*GOq2Irkt>2saw znjnJi_(Qc0u4w$wEOyx-&~;Jl^{rEV59&gTVZ!ej>x3}H*R+K#r+C=RK(v9SWY~4W zd&+OWh`As-56Lj1bRNv1Nb4JCddqABG@f6qMdN4#yuA0Vc|rbGjGSxL(a;sj>^ci$ z9|#Q-|I@}Mo9*)(HV^G(AZNfDsaE-{bX3|iUjQ%}juuBPmS>mm7s^+BecW8rJyZDC zqNw2;35{b_Cb5LGHup2l8|K@ySDSNrhsGloljAezQ33kQG!q6pWMJ%zaf4n^ zjh~u)$Ag>senc=JW71%6O=!w#gHoJe(k0in903OtFDH7pPFlrvNxECr$&7xpZ_WF; z&XUf!Xi2eKk6zOzj^5UpbvJFPH+x*Cj9S%9{#^fW`-`tMqEu%3CE;5+;r@ePMo|rZ zr7J6|OXTKwpm&4ZDZ&}Pntz)p24<$YUGs~U485~CMFfNJXnU^JkYIlb%(f~99`MaG zc3iM;P@)6LeuLfIMBk zng`stJaIrag*~|qcpI9)OLYY!DKI6IU+LA7{2dLwAfc_<1jcx!UEqJoF6mVi? zn({FC!a(gCP7bugCr~$14rj!xT=xt*mQwaIg{k<|JevF9WE%{gg^bVQRgQXji1+?A z{xf2wVk^{#Ad1MV#xYm+j__NU(*S=TyNrzCRWO}2!78+pw!Vs|WmFHEq?y7^qFwml zR`hZK;!(Gw@-PJMkV3;!jxg}clG8<%)(%5lvdqk~5Z<^KYTZ9+0jyA93J(Z_0eLAu z!jGyFK|GfeHzp?j++&=Vg3noQyB?Ge}z75#bFHBO(s*H|= zE6%XhOmp?=%$s^JVfLeiF}mRI1P>uzYDojBKpWLu^Xl=WUi8(&{SIkYQc6z#{Pjgg zFuN+RDAy?w`4`N8Z7au*0@E5^TDKvo)|KM;?A+ud3P~eA#v+d@m}qwC2Q&j6shciq zc-?cs_-w}e$P=3E7heVvYgvY;xxs64JktT{KUH1n63Dn_EJOWM-w3#jUKXndQ3;w$ z++imEv`>LkOHDz-Rn1$YQ6h9b{x)0gzGFC8QgYWZpWaQ?59XDXKQ}2NXT8u%)25RR zJP$VZYuqlcu*?0v+2#e^s#+yEtZe|c^#mQ$j2C~GX;YZy+4UuuBd+*T%t}ei2&V5~ zx^`5<)74SJGI_%>i{NiGnpFCOcZPxKl-v!>{G--XVP;(MF!lD@gM4ZPWh*U-C!ez~!-9=gJhW4&Jhzo$H9jm zgcp!zNsdcCA@u~oK9Iw5geNy+=^pZoZI=Sq_=0iU$!E9x(Fpx}*)n18#xD>I!<^#& z1TUQE69dER>l7HjMT(o?%eB^#RSN8w0bw(hK?IGmrwj|X0m2Xrer&HBJ3#RkE1n*| z8=2$AN{tW(5$JF=BRF$Ced%t>zuUlYN!&INu7>?Vn7u%qScr*c1w9}y%IaBzW*F}q z7dEw8Qn}=9lnii~!(DH>9!mcrd{{_m8~d6vN{V%SO;+-fZP3C9mTMluB)3xQ2X4vx z(cW=yUJj<0E3c5Kuj;l~wWYbc$*Sn}R^W*e!cp4`GFS6k*sfD?6kD3ZT541*Y=ky` zJ$tUAqz9&?P+@p*MQ`xI;NeT}Iz|FFCA@cow8eU2>)PhpnblYe1=z;r`R?XWn(= zx^cGe-FtrjnAD3?o~){}x%zEDeCI^SNxq&rwNxj-BqvqMxT$b;-R{UvM_PJ&Xi7VG zR7m($_dfC&)w`ND) z_p^rF<6xQ)H&u_J=T8+NN~A*XJn1onZb7(i7kD|$Zk4(7saCcCtd!|Kjv zc<2z`*X-4l%w|_3YI9?hxvmw?(v0OD50|WG2dbGA*}Q7HpD*(QeKIH{D?nNxPPDdNX6;dgU1cX0GD4A<4NV3>lYB&M0DM^PnS@|y%T~yj3C^9j%K&5p-NhZ=D{B3wx zM>N_(mQzOP#Kzg5Iyd(>rCXmAzIEAuz(y}R;zHeYgCk5yWY}Z=d@=5APCxcA3<$Bf zAMRt^aDCchF?M7#-*29P$rQ}qC&;VcP&P*^zdH2$r~s5a<(+tn9KEhjheL_~as2wlaNSPk2-W5?B01;yXXVu(BgYFzFU~eOn~l6z0OO=GXDgoNu8$@n zl+Of3jyV~;Jq*9Bp&lssOzmyTJL^8loZmJ!2KC&*qX}b1s^KMFqnfheYS1hpg48a* z8E2&ab(h^UzO#OR+f99J_Ys+^y?K%LwgdordhRYZRQ2AwoZrMU-y{z|ZIK)-HF zXOt)nObx>$r1!408a4c&|yj7Rr)wu$Z3CO zLMcLK4SAOz3qS!&6vn0g((36a7@RMvEF0Q_n8N+MKw>rE&0LxF!R-yk@;sOa=wb$Y ziVb{lARM^|_BYQ~f#Z#7O~9cD4-gL&TAWNEit4}RZqEA2A@k1j7$|oJHdCP>*OR7EXaq?tAdi{cDr$v2U)rDjbwG7*GkymFn&QH+Y)i*7aEkE&vHyn@jiL#wT^}D1Py0>v$8v z{T7m`Wiv$35^m}Q?S9vrVCV*<@Y^}Q_GOH@@5jkR|?m2Sa`%$K6_sA9g1stC;?bC3}S9Eews1s2Fn}HNkS)hOo!(4p_ z3W-{rOE%feO7P}=uv*kNIDZ4XOm%hj(XnhR)0nhrf~o)Q?rvOhY5Cj6TEY#fr{-ns z7A?aZr?|OEmdWuKE(<`pEPgFylnB52#mNGR71p!QTB;rz#qo7}3%e(oY*>0i=@Fzt z$B5sm5CPoa)j(-RBV|ZTAxqAH=3Ca!$aE&V%ui5nYcPcZ-51YU+M!LGPo*VWmGzQ1 z`Zq&ev}<^Y(B_G5t#6F|Z6j{WZx{3~;lN|XCy_3b5d+sbhQlJhbaJ(K`P-Haj^xo?-e_qKmM|;G9+Jq@67YO9%hmKBBFl?Gu94FmU%;uUGIAdRZf)@UUHBuA;dO?svGe0+G;7{4nTl%DlG$W!g4y{s1b(3_^6SH7V68M~%O2cJ%~YWW(BQJ2&OBZ`)sb|AF}o#sXFpT-Z( z^|q?M0HHVHz3JZhQ|0T%aZQ6uFJ+G@7yuLjG3jr2)Yj7GzknAL$(ZCQO|q^2hqh@o zqxS}P77G*}3>!WzyeZc->0RzklrrS5ywx}wdv`5#i!|{HDv1)%-c@J3M;&uNOiICV z@~%?vLc-(=2Yqn+h#*S1W0QbsU8BH+VF+w=C+khH?-0x3W#es*5)$UhJ$TtZ{YaUx zm@bUAUSfh~6Ctr85Go>i`>%_RJRpjK>3$0p8;n#hU!O$t{m*^@lH;*Ru!zkZ5#y5`7V`9Lw7xSC9*qr zuD$4rb@>6~?!3g+&CLxd(7Nb7eU+WI{Gyo|%ve&J--W}{yKq=w7Y@hopTtB>hj4U^ zoeeVm`&Vm(Phsg&Q}(1>(YfT>)zXU9T9a0Xt_#ogFDKB+aX%JQ^0Yf{9y6Y+f6*9u zsZi$?car#Egpk4!m9pWH>%~tmp?%jfm<;zgP)Jpwfl`(zi{NCalk*;e!5LzR{@dR1 zq(|~lCDRa+fBjL5K2&N5;In4*5=g0;o-_kBYHmD=+VI5dh;A-NW1}f}L!* z@Y+K+JoMK>HS)G~K!|bVx30e7j`el#J~g9t#@HT!4{pjS=pa#wX$Bb-INRj{gD`(# zP{!5kHn-D!d)CS}AP+PCQ!|q{FU!Lmw}0ebw;$LHAy?l5w~1^R!t`a-%a7c! zpp)S_AikIr_VpI|svtu?@^$*eY!> zft#@dr#cOS9ovO=>lQf{6}obudSe^-m;aDFN#(8^k3med{$yC3cx1#oR@+)&GbSq! zx=T0DYy%$WpP^N=zo8|LSxR!rc1ko<395DOiQKqSk5D7sPLy4|ts=`1dX9;d)2n}R zVDE3i&Y=+g5_;Q^fZP%A-o%&;#Q%VU^4p1~~!4RjqUQ z90p;F4Ao)m4mHvJ89Z(LscFKjvDUbIQkUKy?xTY73}G^98@NfBk9RjGU5ywex~8$$ ztPgrG$hzmS6iE)EGlmNWRoWgHhTS&sAcA)rc&B8Z8{#;IsRuD9RM#(zk~W{b;uSye zA`t02%(Yi?FmTIOBIIrrU*qt|#+C|SQ?^_VGm|;c|D7rOomoens88Fpw0cJw>|dXs z^l8gs2oZKRfIJ^_(7^2nSeq`sY`B^7LY2)X%kb zszikh?N4a<%#pEKfr(R>Z3oUCM_Qu20;k%fnWT%H`U|Etd!EZ}1LF{3k-L1Ln!i>K zpwk%SzQr;cQ86x4z?}X?@Bx$@qnVp`Rw*DWb*M`b9%EDW8{8a%?MbO9NeWLImN*bQ zF0k3EBqlj@D{5?zAt4uYo)~}rYi!A>vx08l0^-u=Lg(8jR>5itzkKm6PyI0BA-VLP zzCfA*Ze&Z%*d<|*&6-AChrt`uteLeNIf><01FF*1iJ(vhawtm(n;%epI_ppw!WV5h zN)(Z(OqcXc;~w-zv_(rsqd=7%u<+P&LMhGeq=ksjMAqxGm&FKf1zh|lF^cC3IMiGV zWXgnKc^%m1Lv}&g^jmX3Y8^7v_G4dG0m{dZk5&FYY2OpNkH;x-ycEZQx=^Jv;9V5j zurjNXunk2MwtYhUMOjbMr8^@MXV1I~7-3R#qhW})fzg%RdQ9PoA+woq=^t|^HPqv3 z;8{~zbLHW@srEBOHwf_I_npvTekT{j81mTIknNk((xtp`!fg($d2e~N{aB)0WCTey zg;?JA#i;eoozzqZK0^pg8lTO(`V0n_f>6aFF_`eAOMfEFl6RmD;VSerzBRl?oyciI z5EAU!Bq+lacCZT=R-Yjg@h`X52zoR|zutOyI;e`Rtp0s!+!1y60YyRi#g??Kk3jJ z?oH%cdj4+y^Tx;_6~@4NhnIXu3wVGeDdW}8-%+jr;(Y^4^O#}-^qN&+EwSlun?>)Z zQ63n2MpXs-LC__sG*D_3ZeE+>T2k~v?p`M9L7VqK&t?pCR;GGgOcc-jU~Urd|2xsj zm$>1AFWOgAN>jr)8A6O{Z1Tri>sG(^^x@1Yn-erAb&N)1V^bk= z`lDekRD>CK^U0joOWp{*GZ`Xd0$-5t&(@lwc|tQ2pXhg0F<1nsSSVL4>e#`Q@wgF{ zeu&gGX`9AImz#V}rc53&*F(hE0{b$mQYqP^aWPRe>Ai%|Nm=kWi|t6|+2Qz|M@mRdEAj6CACBIRVs{pRrE`XxZkZp!rv?SG?wKZRecA@v z*@eroT3c8@1~tO?1*|A>_Or@FJ*Lelsx7Bwv1WBHQG16t99Kfw>Cn4Z zVV-SUVecTeJ`y!jCQF5m$WR+mM##E5;`XN=)(T}WhbEnYN8%(S&GkrmB{NRJT__o$ zUZnJFNMD|^`M2c=@V*p>)V-y$wM~bY6mWt>ByD&7uMljLa zfKMEsP`;8n?b&NZC!$CB27iBx1q5limZR|i5A}RtEM=bns5zxE*9E~%QI$3Yjvl{Y z^^TdJDtbx3yP~zy=N1yec4Vc+jC;{k&@kV5>^<>H+YBwAp?y-0h2aH~*4V&TMH(Kub zTpUuj@3`W3hJnxR*M!JTqhR`7T2-Ya59!F4EaY7Z>^z97udOW&tDwQq5hGta{2R&r zb<27b5r8`w3cxL#7W@TOD?}wc;$_~rQ%h^nX_d0-X&>!@u&|>p-W^sDH*dBUN(eB4-?5dhnYr)&V%7qiRzl<8isl;`trITN~K-U`1`Qu zzw4zTKb}fi&M4H+{ylrJnrN07cE`H`i{uQQf_AHHozsQy!Y0U`%Q(vFCe6j zmUT=VrgzE8AL{kG9{oK4pKjb zSh}1*2Yn$@4-BmNC8j5RGtRQpdmOWJRJ7tmd_!4fH(%5(zbVbKnW?xBq>HNCfUWF` zBArh~0YFlSd^u6-5l&oOxu(Aaz%ag2RanO*SAw>r>_aBRfup(`@)QS%5kLkJym2z} z{sWF)A|%~ZBjZE_&a)$+T>ec#Gym#8;VML^zLo=dGM8&)E#cpIz8I!Jd2v5(Msp5R zN__#|dtyw&Bk(^tc#7KY!3XwR$hbmRx)u!1Uc6$y11Rsw3q&Z=g(F-7cfhjER_QoY zt@;LU11Fzl5^NqF9!{5#`V}D?R51t9^V$%*VEjDjMzPma(;Rsa7b14dGxpMcw!K(GY%5!y+MzTb}0XWRh{qyE==q=f;*}G1nchCxZtH z%!gbLkAE+;!MsfC*I66s&JuE&G*9xC6`J#FFwOiy>^qrfwB>0dQ~M$J za(_c?immG$gd{WcEb`j&SP5)Pvo&N~4(DDsr;)ExOrE#Yj+R$9&ba@GSVY&na1<|V z78(2({_D!BBK^t5Z&C@=9`;XR&OpYHU&8pW>_#VKoSkuA!my`9vefx<8!`iebAMEh zkzI-Q5Bskt5O3o0oWDd3d5I@Ctk@eb5ufb9;u%ExT^d>F*5`3vN)?~)@vF{Tpfe)~ zkA=HPs>M$pi}p-oa4a zN}HU^&eyZA{-dzE0dc+Lrsi;&!xJ=ja%+E>j?4mmT|$n}mqd6UwlE-P+9qv{IBy{M zNmiA4AJW8aHe!_Z8n{B)uQy1kDy)U(=VT zp_azSa{=o{kxqo!5t9K=X7{)}?68`ZyH&uGqGh$UT9&H%q~cr+jfm!`EdA|CvPm8h z|1}y9k&#lWK!{0FqGFapPk$#Gf5$GE)jITflo$y_#Zd&!XQ2(TgyW7E{Xr+OdoA=> z$*f9@o>MvEq-5ywP>HrYN&U;`G$z&h9K<^DkH*K{izv}fRSQU$aTyP&D2-KJ6dOj6 zwb-S&(PeFazJRz_s9{Q$de6|9rv7hvL`M&KX0Ne7=G6K|_1>6d*FwpwkeV-Be_$76 z95PPRW!?MZLeTrhSc@p7U+5a!ps1rQL~kV4#XIDm&!Ul$J|nwn$6D_C)I{f36E3*s zAaiA=qTb-Pv+~%itF*3nv0N1IsftsNdZSM5JD1yAtD~H;RkWINxlwZ)k;9Q4e9U=h7C$u-Nfn{#8BCE|`vT=$DApRn31X0N0BpomP~gV<(sf zXueN&Ffx=43W5#2pf<0wIzb1G@QBbAa*Gt)BjT9m8hs&(DD?Z+@Q28`@Z-*}XRR<7 z>sAMN2Lx$v!Sn00&E3@A8nS(mC3{n1h1?zXVgXCTEi8tlz8zaO!X+TM-VFZ^i;D8P zVuMou*Dbr&uuE{??5X=*|L)~Ja2}E??n=StduTnI8V98aP+q6xEH&M+cOBuOa^ zC-gCFRk_e!NTi%9dqy1_(_6_ZUP=f`OO8{ZDvGw5-K@M?UbRWj` zDFNckG?+K=+k|v)pX7=nFKymNLgBN9!}l@X@GBoZ?LD!B2eM8NOh-uksnyLJitFX+ zqJ^9wELB_9p0QFXu*Y*zHi*-@b!^2jbXXfLU_6GoNZF-Q!7Y7`^`=t!5ow5Y+Di%w z?mlFg-a}uVifeMDi@@xiKOCCtgrsKe!elAmrA6DeBu_-oqyLPkEn3waKE}ThI@b0d z=I)indjXpjnLt;_NU)^84OEqlY>wm8uir<0dp)sLx3L~PhGNyc%xN|k zj=ri8^4`F{?k3%6{3*QQ2iA&J@XOqmAzpS5zIHQB{4Y+Y91G5&&mUSz`NM6U`DYSL^{IY8azJUwOF1)el_{Ld!s}N z+n0Drf2LN_FWw~@5tPYjgKQW#s4)02+(;xE<7jxb3<3h;>x`b)*t?)OQHQZrKTkv{ zqsXd;SGny3ZR}Kgf@O6m{85=LL)oPnoK)!@c=Ou_CrB6VC9D>$HLB5eJQULk+W>~$ z3*p9n5E(PNKPS?y)i?jPesV_Nnu@Qam0sn#VQ-S1+eYg%C14bOS=w&wr~gqQ@Xhgp zksrFkU0HQSkKT$Y%nF`|vG%PAg8T6X=GFwqqa=YCeiy-H$B8BP{csn-{x<2@#7a=s zZ``oqjO(j~xOoqF0huHV-VjsoD*TAv*ani;iN^S+Ikrx}Cs(QhSj~J%AEtv@;TG%b z7Lmr_>vL2`nL<{uj#HDXVQU)>(*k;+ZB958CU4 z`Yo?_J*r4JzjWB@=Qbdl68YsVNu4z=vl|)ko-GX~iE|ocs~YL8(pP6r*9={a6d<_E z;S(5SDz%h$1TsKH{=8t51$jSsaPz1Lp0SQQ6xa!DNAhn%gwaTjtlc%@o*mI=NAojp z@*<2hg?frnVQNKAx_*jNB?^4sV&rdef?AUE#}vH3_`NH72;S1LC-Pr>BU}CTePWGT zw}FS3hgXCcLys}w6i0fbw>&kOV~P-NLkM}6&eU~CTKY|^P3|%-rt9#b91oqcxnjQK zBMdYh)kCzYMCydh zPx^XR4q_AXM?Kn7j$tgzwzfHceG~KYw)z=S3^P2DFv#>=`Rc3~fs1zzSThOZgAh*b%JMuI%b7=v`&y z4`(qp;IAw33dMnh`0R5}<=tuM0fE4#v@^aTr`Lv>HcXFL(pKSt6DnWt9}=K@CRQKi zfZidj%7p>rv~)3>^uCdXSgVGc^+CJSLW38jz;zaARe4jwrN|@`%6l z36*pwk(lIhM3w7}my6&cQ8ZH0u!|uk%dwZ!N!p0@1WOn8pYNLFMy$)mkI1o_$uwz- z<5}kY;T(IDmqji*mXQG~+y)qYqV zj(NWYh&@rt$ykWqs;`!tG46dF7PbZNlXf^H8K1%0lWx4Sh01h(Be(@%i}vo|pna;OQ@! zNUu9}vC_%LrWSLbbcd)x44&z~g3dt+e4I3`NcaTJX=p|CCpSxy_I2KVRke7 zQ1VSi?kr|}%vGkWpgWXD2et7*h^38CLYVH~`fTI*>4gFzdxACj%o~h_erjEv7PIO3 z+>e#`0*&`cm+!=W39BcxpcCaNfG@Wu4^4Bxx5~iXLgE9acPB0t8gA3*$uO7iTuq8|`0(h%LM589>lM*oPA zH`LWxtcfY@o7JJ~fz(q}dsDPH^1`F!u-el`i3u3NAyTE__x16}gXq#dtFPBgySC!> zufRJTK5A$@%)GB5xVo`uf=%59l9iQd)r#2)J$J7k|MXRV>|09vHlWF-(Vj=#9=R9i zb=2nLNP5nL@Aci|ON&{~p(k>3N%5(hpD$Yb99vSt>0~<=^Xl8QHvD}8x97Ic*vQ^- zwtD!iwa8cmdi=tGhwhf5;8exRH%m>=;;FBZ5A7^Y-g#Gu!XGFJaNFN!mC-KDxICoN z{udm76(btmR1is6UbMKBrP*w73!fwR>-w!wXI(eJf3_aYo4ZdJ2B;!z_~T=rh}k0h*;gDN_5HnY@|;8At79Go zAX-SY1Ec!+Snjwj>49r$aA6U;gd^!uqRl?8i$Q0f8&&oiU{Pt;w=(A)|GEuQ0y5C8 z0nS^QQ+Y-049lZ#HFsUGA0~=AGabHPxBJ`R0hgHN<=s^0Y_OB|L}e;hHrK|q1tum| zW5(oWxmPd;{O)2}+r|htXmR4J`9aR_%sZr+sKB++r{CHH5nDEgi~y0hj>20uCrxk2 z+N5u-M@e9=)9wnohF38hXGEW1u2K8{BRVfAempV9uk_9xFCTg3?We!z(( z*<6`$txjCg;vDea1%^nA0j6wUPTkEt2huiW;)mR)x^AqG?oy30V~V(8v%G~EP_qi> z61cX))tjjKAa(R;(Q_~iMP9Lhc8#N;l z={0LxPrY}Q%pF=!3_EGv)2-9IU$L-ZE@t#XhPQtH5%x{Tr4kB;X(}>IR7|fNAGJxZ z^`-JBseO<;1lR+AsUbbfk||p1!d@;uIKaVK-p(@VDlRSf7ivL8L&Kc{v(R$Rbo;Aou{FlL=XTd8YE(QT>1H=P3Nh0K}5;$xN!`G2d{_ot^plFC?+LNE0 zg~=uESvB&0nzK9uD@DUuRdZ83tF2pyH`RvI!}BLrNp+=V8qwyW&zYMIz4_5acm!a= zNU~Z_2|^yfMcRAA_rqpqX{T*xgv$26pP{{b+pW01MJAv)>Oqh|?wM;R@0_n-^VM~VlQlGMx|XP^-?2`;9MQy&84Q#-p*#fge*LN zT7}liG)jMIOFg5daQ%isNOGl`DUJKXST!P0AHYohPx-8F?rqP%$`6LY}OK}&@ENdg<5D+3CUY9YE@tcB> z|A8r!`>HiuPDgZv5zd;Vqc=Iwpj?jg4hN04-Ud(vcM6C-GC>3PjC))&$(U;=C|9)Q zo2UT-_ht{2zM!2Mt4wR;TM~~#Vhtb%M(v1)2`^EiB}dO!dUbk(8uX-7Q!)*oswbiT z#kq|cPiSj=vBlxL!gYI#@Az_gN(MQX-3M`1wv|m@WwSPmv~nk+kU>#)F5LLhMk-Y< z?%8hi7Iwjek(VG6R4qE!UvI-PeX5(a?)(*8Ah*xCzpNHtUYQY|s~mw&%h0E81@lHj zX~^Hm6B>@PirdV0bNYMcTyebWhyhhjNc_$jZ=nXg_3GnlYtfj)1fj6PEHSYr=P{(X z%mGLUi&vOtE-{vA;+v6pWUR25f)hVotLkWx3!bLc`0n+Y&Z6fLr;SU~aOoT7kaC_P9aoCT)|T!g=lnMi%4 z^@kRKyMQZ@u}S4M>&BoGr^+t_9bc-zZt!(|vf;Yo@VsRk(7RKKCdwcI6zx?KgOn-Y z3vt{iI~2#D77ZiW*_02Smc3Fk+3)NpXqoTl^^ZBJ=*ft#ef;yz606~qa9zpDUL{0* zi#Hn_y0zg;5nOy<&*pymqvQC~n9uCX?7046n_eR4M%dhv;fdeXNW3&JQ?^FtA%*Ma zEzgDpUtPO}1$5LB`4b29We%0;x?a_>PN^qFjr?7+YUn$_QqJs5`ERvHGew5sgF(Ms z@=@Lk2W=vbHIf#h!`EW{1?ixh*de1xYbv1%a_gRbq1vg++JC@ePD|{;A5d)xBj8D_9k`I@T*|P zGV!1~8J}{MAOW%d9x@RNqdZ81$Xp%e@t<-ZZNzbcpaQGLs%};Lu|j-IR9^&}u6^=9 z4k#?ZCpRiws`y2Cxv{ayf{XB^Dd^0Oi1}X<3(OR?8rMwoy51&TV(4QauuDTe7B{(^ zh8HQV`;iVT|znLf|0Et(!c$t6kq33XM!@c2{SJ86~QY;z8xgb3ef7)B2+ zI`Ou|4|-c%N&EFT*y2X?Qc=j_wNN}=3j&J?NJ;<-|JG>&E7oa1x++6l30;Fa_MEqveY=BfOi z>dx%?h4a__-^j-;(Kv;kACO88pE&1*Ow0V{HeG!9g8!_N(q)ZQ^oAJ=OUo{-Sqpn* zEZ=THI2Uy(?ywl|k#kADK9uCra#{9+H{tkY#G2?Y%r)9y>!e%o&^v?2(W#wo>tW9x zINi;C{cThz?H<11?Qvz()HR0>BUN_=0zZ`dUs-fw~zKSde@^=-iOX<^|xPrF+;b8bE@ zx2qh8cp@@i-XF1kX7;4;_@&bvL$rYR(#fCx`93h*Dh`?9jtG8+EIG%U7;z7Ll|R1> zRPS8!Hvg`9hD1|5^ifU6jbp$se9KcUxjn4}TzvAC`@>>{Svz z`|1$P8P|KFp&-S2UUtE%aoFtY=l2K4MHcF>m5%o`=Hll~y-c3Z+S>WD#yEnL6`nm; zsmLx;?=9KL&QSqnYJXwI@vn~gTzS(JaYukAP1F}|c<)j=x&PX^YM)n)H&+7IZ_hm2 z)H~R5RbFS=Hg6m7v*AcjqjhfuSlS>Cofn2bIMrd4X`h|-v3jWiHXdKjl5O2eUCsC& zYjeAWCGVJF_u!}Lttip#qt{$!+W!r4S-o(%3}D^}t;MR9#^Pljw}oE$KGM^+@?FL= zuC+sPph!UXiu;9_w<^f^Hx){H9PaCNLsS!XTKEB%PRYu;jOJ@WSkhqbH45g?XA2Ae zsdZwb&wk=sli+8I`zJ5oZ?3F}4&5)VIdI27C+zk=<~rXhhU{NzEm5JWIqQ$)rWK_Z zW;1e_=TGss`>3P+|ERa_Ha+=LV57cOvQ;nd0XLe_AN3`MY*?=$W1#2H)MW*i{GCEh z6YQNW0#DV@&h)IPrJ0SZjqEa8f4JK7+rVV%lP2f4eR1y6HtmohL9H-#UQ<@^6@`aN zqY>Vyr6r|Hku6yTU2f-`;A?OTq?0Ogn7ho09m`o3N z8Y*jbWOTzHbJQLgvlr!>l|qn2+-~pZA$6l4soQ;2z8Qyn?|TG%`IV{Hlz#gK9-`}? z%<5k4OVHw+22c5~N05%D^o&h!1Mls~lc9bV;Xj*R!nF5@+~=oog(p zdPwg}nfPJ1e}j6OALxZi7Y^FY>u-2YdlI?c^U%nLb&62#lXfr@87IqNHnmEb!o}#O zwNag4j18K6uA=NUu2bbZoyb*^Hdl_DwrWVwqr%jUosR4BuezFRxw&%5vr;kdqi;|U zjpjYCFs+{yNk`zuHU_5^0Ald|8Jb^Q$qxgnIt+{wPYGK$V@&fdH-0sK)GSNR!S&;f zt2N38IwbvwnpDx=`vT2Dh}HEwiGNug`}i!{XO*^bv#~sWe^w39K$}@?Y_Xt}$*6r` zXUwli+}!HGu|Tb+L)Q4UlsxXJUj zPZ2wK*6@KFDLxY(-qILWOgs>#qs~)nk2!?J_s}p^ErDy1-VkK-{^2b>CWnXc^uC)p zcT%&n5qjABAYw}qvbd^LQedcUz+!dd9vtg~iVfOKt31ZPTRAPS-?uoP`isF?AXT4lGp|(0dItqwVm+^Sn~>m`_L!00j)l+ zw{&JmkP(4UcFVam-BBG9b|QmQA{ut?VB}qL*?NoAu7U|4)gm)xU0fqv#?=P2Ajn;k zm)C6p`&c~)4p>I=MQhcqCOO5vLO$;6KO#B3H7$clG>9;!r%{9j=6H1PD>-&9RLeP= zlgl`VS)X5o_~pWJLMSi&Ny&XH35t`dkY*|SKTYiVSxGk3+K}cu4R~?lUY_)!V3{5b z*k${ZArQUDDH)tc^P6M&4#8jcerw4I41NQ5I=sknI zF{~DA^xSYj@n+a9i1aiUKJn4W>g7LFz`NMtio1!nK!JlhWLOb;u5Vy@c6QH93Bf){ zy)7Hj;V{;f>HMX1G1N=zK5yMqH}1YF3rNl6>WPT8d!skHY)~QdriSIo zCvgnRSuRB{vm@4>60TdXH~}1XQCbCJ5qBzoIrkKbBzJClw&4{+yA01%H#&BbU*zja z#3Qu(F4-Kc+Q7jW<6agD{6j?S)37EU&4xDaQ|{k6ED6r zQ2x_MrCw^U@C{`2LEhRzZy9tH+Up59_IxA~EQLf|5D+*GY(PrF1IxAn>4cg|1(cw} ze5Gu|TU&?!TBW7D+b+3`q#v^=_m$s^=z1My%1*!JKXE*g3$4fVN-s~v`MjUqLV*P* zliEHQaeb_JY-}}e=+xC*W>#L zy`8JFv=2S&nuH9X0?J3Z`X5HI7!ht#nN}ZHY4jdplTvJ&rsPJh*1$FpKL$3~bL;`j z^4hADG|3ozs zy?3J2&-oNWw zYkgN1NUcxmaRs*qpO?&C#y_Av>G`UbGP<}pbQK!7#5S3(nqcG`iunn(CSCq0Cc0yNizd$A7Ki=*3S<3Oe$dUBN2M zuQ%b*^XZ@@YVt4w_n6L`AE!rV|39xZJ&e^3od+S8=x#iY%zpA4c(!7ly-SA6>C9ji zBj)>N*RL4^4M=qucKOVt0|ZYnP`F~cX4Fguqr&Txtjp!6L0$e8F_oGR3`#Cr?-qf< z=^v04`>RXztL&$`YZ)e{x|vq@Ln0mC+LxtqGnRV4;*`Jts-~=$sur3$lALJOHfX_h zTy>$Skr&zZj-3wjoaGI@6i&D+W2NqktsDCCVWBRwM?0;2cgMX}$lu0;@r>a3=WRKTmx91Li7LT9n2`>-$dg%Md8IxR}N(~))s!tk5 zJoG;7tVFkeJ~+(9WXi{LXGp`^{K&>qvKKHRMQDF$gNo$l+4v37m+lH^GY*~TUBFM0 zW>5Ni&&FY&x?VO0a+}clt*zg_m!EtBjT(NyOWMWm*987z@~~JJ!KCm_`bO-GWM;)E z0KTkde;r9fAafD-jy;YnXk;LeudyAg!6OFH;5yRJ>sNckUpD;YgH?M6$vRnwx zd>yT@ew4o_TE>_VO!8^I#jQguS_S?_wO0tO4N4KizBX_cA z7HgX$=^H~dmFSN#UKLJXODL35*S^PNI!F{whs6B|4Uh)Kugh$45rwEo)iLiWJeRDv z!D5YURnRkY)bz}Y`q}0H>5lK6XHLwI5TJ4`!TT=xH~Dy-mvJhRc@LFsOE|j)41;al z)WVx=tJQZ2Q=r)4{eDSUdIU&#AgQC4m|BN=XCfWhI}w??4+5|1m{$7i!prSGw6Tfi znc&=6cfP4ouPD67Miqj8FPH!3GaPYdr|h^%E+3 zKHSf6iO`Ux<|Q3s-07OH@@XjtiDZ&;qEl?ta>Y^Z30L}wLR(mN54bgg^OgGbYN90Y za3nG~Zcn-03s?stNKJi~1#YtvBDMFF7-HXth2S+u>j(KNj}C}-zrEscZJPWlKHpM= zn>qMJpxE~7+p^ZJS68;Vw{OE&2)S6er9?C8ADOcaa z%XGJjm6#n?_I7j$?J_&Ov?6*ma415vN^F^=Aio@onQ<+5I|uG7A+i0wiV8pTK!5;$ zY1d5}T&>*k%(O!+=>w?f9%b}{!AI_*D}np2^=a}QuXxtjL5vt+b|Dh#@X2QM!*dGt zxat2+GXbTWk={e3eH|;}Cy6FMHXT=4qW+vQ=5oK+tc0}3nj0sYEIjp{qf&+ecqlbr zUt z@Z$DS=$KPLB>svMp1nW+-rt>xsWh~J!>une*kf~OI`(OvryMEcg6?=K>))LkwBfI! z0Y><`)>b3Pb$ChI=jm?I=}xqs7fdzKv7mvH+`9k|DW$3+W=*sv1Vy=sw?WLwY8Tg{ z7nz((^|47=N9iL8Yw1zLr9c0SCAkX$Xb%gZHt-jw$7#0Ec+%Eikk#NdeUO-qsR{Oop6yM;1SdSk(ufCprLf4>;Fvq~Wi&;*&I6;2S`Z6a zgs;Q)+6Is{Q))iP^O%)v0p1>QFxg^WJP;hVp*UWXxIOweGW^tf+{;`g`)Gy;eRsD3 z=Fk6Qx_lbcs|$E7p?$C1u1}U^JZ`o*3yG zv-YV6!%_qT4Pq@-?0y1iA5@~=6!D(8pX6=TCz42k1jYbEtnr?G#1OTiH)SKc3Q z%#=igS@@<#w0Ea~y)b(H+pTYI<>Vm39e=y-oCc0JEj4$7c^oo+CX%u0-F`|j4rV5E z+$DK&^2gKpN|vnuE*EXU!X(*v%M9#885Nd1|HZ{HzopBQen}R@^1FW&kK=y~Ab2XQ z`@_@NNnMjx!L(L#P}Pi?L2L^Pxf(@BOK8Oy>B&B+G@}y?DN~Ci(c(MFd^6rcpHEV( zGO`d5?;Rq5PoP9<3Ps;J+~?-%hFmoJ#at;oyV9LT0F*$dYeB5|gYxcG?Kd3^oxAhpX)n`LxBNH|T_Rr^Cfom+-9u3#H zJ=RVw0QRUItos21z_IbaD0tbz-Bg^k^^*9HcV>}OLgQ=WCjPx%_9T6q{d35`hs3UH zmLmUSNDUNhq$N8^w0RB4K=pZP>Ro_iKHdlW2}j&jNXfEFY7w-BSGX}9{aIlG=kW2otH-W27+Lr@OJAIxZc#|q{det}z{oalkQ z8J$5loPFi(%C4%@HM{EM;29yu?r{ooZ!%_R3Z0j0fq)R15&qjn3dYCP{>v9Ex>pxT z8$blpz|}0P%P9c3Dy9;E$?c642@RRxC18Ex!dF9j1g1^`a)%>8}FF6ep|;w2v#IX7tWC&Js+IfGq(Jbf0bDeziHQ>UUE@7F-!?ypnzza7edJJcO<21-Kj zQj&F*UsuuO!^sJN$nO}iDIhOf9s9K2R7P+Qxe+C$-@113ai21Dv+&{u#02m%R{Q3b zUTR*d9QP9^&_h88l9i21ChY)Mh%7>$qVR~I5wl=IsMz9K zFdKw+dYP=9EF#~;>>m(b@0sB3fgCPQJh^lS`i>r0t0bo8&qKfl$JmM?>{uCGf(VkJ zKsms8AzQV;-r?^-(I?vuW|n{|z6nO>qP=t&7q!bGi+{iZ3sWTY1KV@+8M^6wy5%D& zmf_g&tXwrIsjn-FMIsHtE+yKV=OvHaR}8MfBA*$AtVwM@gxDE;X~)*E_nvbo+ygVF zzkx|M%IK|+Qb6fLj%II7Cbr4cglCQxL>TK69NHKV%enu$8p1Ty`Dd@c<+vBOWj!UP z1A_6}cP{jt%9s`aL_+{Fz&dY%_+lwzIJM=@x1*#mCF~wNug4ve3}STLYnx)@ zFl2W>Y~?A2{EN}=FR>TVfk*VeOq@)r3gGwSb~n#&%n4I%y68%_i8|yww0r24#|P|A zPrBYf9A@8LH5Wk6utcc#WnQZ?;CJHJWx@PRH%_NULb*ghv2UyaPo}PF2vka@Mthu) zaH1>o#iwi9);t=T4{cbZdD|z&2MKftqwKC5e9eu#UM%(ICodSYI{xl!DC7`{j@rxz z*R7uzy5ie7G*edE3+~fpX6jrJQU}-cXQPZ92wl24*&u3fD%5mAY+R}nTI+2)KF_}p zII=wm6M6&tPSu!?v#n{q^4u^n%7*)SzM32=S)Csw9{ZJMU#p`Pbo>77?JsJmN9fhH z?L25YrSd^4*F)hawfST0q(D)t#5n#v^(V&2bnv>dy5VCC z53f}mf;%RSQtkzn!CT^*=(jqWH)(TBNdNp@%8v3yA=Da`OmZhvzgg}Oj0UD8dZ<-w zp|5#Y=EHg;;ekI|M?uE=`8FpdqPyr}`35X+=yxBQB`Hq#L3{vjQhAV7ix#@h1LDtSMoUy;r&R%H$2Ku-#jodR2|EgPsBjDK*QUI{QIr%?qQT+QCP232@tqY z>^da(D3)LMp?Z3OQlq%YR!{KPqj#BJ+Eo!SG0=HFThJHkPq+N|)z;gYtw@1n8ji+0JaZ{-Hlp=GW5G-V-2t&r6XgzKl0`-6s$lh4<@fC&Ph;JM z3&=~n1b7PHiT>cOc8%jOOe})kBN!afhm}1|tfwqOgh7y1Au;L?xSl?Ev~ejokN=40 z$D-FJLn$i4`uZ{OzegA2`4PX}-@wu5=hLn9p$D%S0@Gdy@q}?08gr>n>?LD3nZ{_M zxfGf}^DsvCPbMtxVbx@p%?tZqszud|>&l-;rCP5Yxu_PB^>g)s?bU-PT~BLkqX+hc z|AY@hKfi}61}B$^qmXQ}V6B2elPYGw(iYfO4pCGu@5QsL^$t)If3__D?*!&!f-StNuP#v96s#dG_ zSLVB}zFY4)@%S4VeEpKiRaDoA%TMpj22V~&B)<2YAI!Kb+eV0R8|WKkSJO2zOs>>n z+LAUuO!~YjSd`(Q@a=}?@}WV`n`9S-UC)=|Mo$8Nn&v|>@lRNklsLLc^A3VfZ(v8%DH2m zASvyvKrQL<*VZO1zeg1suX&yKZgXlep#CGTLE)fZ>01*ZE8wqU0?gv&v4_FpXl6z+DF%9@MO`s z3i0HI@km4c6lNQ>F*z_c+=mpkrk%p;NN!0TmT4HP5)A;rhIViZH*y8>W7>ZU5=&bL ziCM{TgNtUUnCG$Mjd$jeNbZOJ33No{PW?AuJKCexcBiCVICdou+Ge}STN7dm7cVy* z8$ACGE|eo7PfPNqXoDKqS#7SgwIXt-x(5CSgQHC*@;?|HOdcuqI@2%crmAEg^hy5+ zL0tNUdDFj0H^2>!_}2|S>Hl8NM%?WFBVIBV#lG~a+;n?QY@T>kj@Kb{!`hlvVRAfZ zP4Htt-#Iz$a4880x(0knwcT;UC1>q_K_Xam(ML@p@;;_%q8GRG%Ps$=fWr?sobsmd zP%5n3^etZh`MCJ=H%$(&+%9DM<+6-#FK_w(_w*@;8+mZ0peFK2ET@if{N%qJ1;NIE z>_kp6nZTffC6IXwx{+j*xiE4tZLh%36TgvBhRX8mYXm!u`R|YUxOY$nDtw_{xDpG9 zGgMss|A-ZEcprb(cPL)jJXCHm12FJ1XMQ4nks>nX;>*VNaCsqYPOiGp;YCgXzby87 zFtA<__c)ErZLn7-pBe?Xfi>&1cEko14bPJtBH@TPOytu*U{;7Dov|~({2Vx{N&+eR z463L_yF4y{*GYqEuGAYmKN`aUJF@QYkK2>qD+r?)VN!=7IWiWU<#6C3_+V&$&?Bsz zl7!2-(#v;@ewTZIa&qct()SEXT?pMRC{mA2p)KWlNl%(rL%6JnM6(rH%SFr0^gXK|-?D;mbVB_1H!L>}v!UI1Le1wT3;V<@-db&Lk`+TQgWUFO`&8xQ6+;*_ho=X8 z=j^&8;X2CyfaGNYHh>9wdUYgaZ78oontx194vwuc)af4tr)Z?ZaVF#JB1D3pMDBx` zHdnjG)-x32vHS__B%Z`PI1*RDFx1sovv{O^l`+iyGH!$X;DJ-*b(h~?P9aNzfB2|F@pR1UjkKt}FjF*C(Sl>R-^^z&Ks%gS zEo)=q56R|vFz3hbEEclV>>_m=!|4Z+tgv9MWLujNyW!OL2lD?W*gM*BP?p)lzwQ7I zD2?Ln{;K5;JusKSlow3{6LxKC6z7AXb36V%3}+O;xmN=!PcCDLZYc!D){Wm2ug!}% zCS=Lo_C#0*P8Cl8T`|FS#?e8~6!WBKN*HGfDJY3K;^>=c$Mt|WfR4-fd|*CQYs61~ zXSYiRU1>{P&hjg0#|c|spSB##L*rX%fHcVQ;=sQw-sL^w1(su%=7^$32!{ow&B{{U z4*fvjVxy@Jjd-sm7mXSjkC_5?$(I0fNE$q@&Ab3@+(FWKkO~?z25y^ zFvC2OPcqm(E8}}3er*;ENoa0|@1OTYhitgmL*;a@A{JSLA;J*m53Lbi(hs!e>o2<7 zvBf%Iv(>x{OgtwG5Cd{@JHM}|=Uu#x=K(RkAHkGfk{cyyJ)WKaDRE07SdVar``i|+ zZPqAS5Mfx0SHi7u<&k{5KZM9xN76BlWTOC{YNpoj|BAz+r0^+=v&Bv)9x|0p#(zM1 z5v+fmW@LYjRSXdd`98%6x5(xVM-zbbm_gN1Ao=F8opI*3-Fnf2$n2faW5Q3nJ0UH< z8xnNMg3NpZA8MX4XtDi^B&^W%OLyM5p*W095ZSD|aEI#e_V*xwsPH9VDH8Q>Tg-3A zPpB{}({O{Ikk=1cEHXvGiV#8uTlk5)r-uNjk(_JRBV$zsh|BiIi!_pyNyfoV+Y(H& zgWSMsEM|1^WNG``p$HPcn~x=%gg;sX(hh@hFhtmzy(M`GLC3TF{349#-wH$F#s4#h zMQ_WB@HAf1wjET*hJ~E30k;%=3`BCt*MAtepR^@0)i37#0xVLiBufQw2dByu04Z}O zxf&&k-+g&h$!g*=%ZMdm7SRrieUFS*kYVzlRl`% z3ug^Rhhd8`yi4>>0=`x3CDrqqc#=Ia@1^CHqU*UW_-RYY>>uLn8tZE=6O~n#3CkA9HX`+6ZZ@m;KL{j}bptxYB*cAUCVaKH)W36Sqt zzSS!C_^1p83?Iab{kvHV6F3%;{JO&;ZsK26rSzJ^%*-IECg!$kqB)YsOgrAlnm)IK zHm%k`cW7mI!12P_e%8v~;@aQ;A5c^W=cb9f@Wz0v_|tNX6xO=o?#Xt)aKtNySeJ7L zdZc!zA5Q1?5R)oVd4)UPk$p-QPR4rUhnxHU^f1`$8!4Tt`y~$3*$zeK5^}Q|m0xIB zV~1QgNO+#g^2GLFU`Gd~rKqs}WLbv+>D@p7fJQfLZ`n5tjPpL4F%_6#bn2 z!4o92&@ecdB#0H5nX1~kJXTaH_V5AYj0e$$+|F=SIH%j-udiBSyKKdodh@y&dPUwQ z+9-&w3dvgBmIC3cJ`$_RGEbrB@+M|$S7}<3K>!m%y|MDO#tkD?z`2RWY6+iw#JY4girnA;0;z-(k9VySL(aczo4>%i$};9x(VdWZ(+IvcF^wwjnL+c6 z^o0E%or21D!xyzY5zsV=$qH^R#pqIQ+~Sc6HaS;tDbsaa_Az=yJ#AYjXCpTln zu4h%bZyxK;=eV-0M+$w|-I;TOo4$<|Hned9gWC_j>%~$Qs0D{|W4mKEXLder5$aM0 zEfYL_OmAWt)qQv5vX9?gv#E0$n-$n*?s}!3h&3`-A-QmBZl)<_Ah1k})cerXuQRhZ zhDhPzjxcXMBY;e@nz&aQrAi<^2EvvU zXEoGk<1fVhtd#$tNhg>ZwF7W0W==tZ1gb_HnkM5(?l@}~3j&_9OElsgQG7UZ@w%UVKYKOc zO4WQY96o(VyD3lXc=@iz;u~ET{_f{_vmQbKIYDI*+$D1HB~%Ej90!Hd^Ebpi1N&PGSmOHz?n59ZR0ASZGcim>Rbg`EXf<-$ z2|~a^Re`eDfccxAcvx9q%uc>0c*FJRJp&tpb5jsQ8JL92EQ>s_ zL5?oVC9%tX!X^n~4iZmfa(U{TX0gYRto}e#d+o$!sJk+NNWiNBHCewxyUdM>#Cehr zM(2NiUpo19SxtwHdyHkBnlcb1s5$~8(KJn2xSr0@czRFip{QvB!jHzGVn!$JUHyZ> z9Qj!nzI&AMunHA5jVLua3Hk8k5}k1nGG5rPl_vk?f#$r|jh#}F0-6*Xma_51xgYY}_mr00 zZM2-bM@;3lFI!kou_vgswCJ>yde!EzD696aT|)FmVQpw?B3#tEM!9SxyY*e^Y_nUb zle&JBwrUL553YQ?QPX^@WXjK(O`D3~%Mv`ANN*W^OFUg7j)i_t;&C3Ur+1V%%%$;N@6uFH>M}x|yS(!|iuh zx;KTt&v{mRq8@+8lKv9#sz0aqrq*Fwe)7wxn$Cm~Dl(mA{?Z*=Y<{1{1YGg7M9*m} zz#z1RQRT)~q=At2Pr~r*8YrfzLQ0C~XpZc!prspz(?{|Ts7?5&j$S`uXWCK2yNT0H zliRl~0+T!#C`aMzFmiQY)Z&7wz1`v$Q>Sg$nALqh>rhVUdbzP-xjTp2z5?EU?`_En zzcWK`+$PokqVe*TfVHmdMQ-Su4UrQXZB))(fR{C3SGx|l-UWg#T+M=nO&0jwo#$2!P3Jvh7F^~uqEDlv$&gAg9 ze{2XAdlKnG$3Rc}J|2_EB5$Ee)k1{`+N2Aj^8?n+L1onS^S6FiM$pOs#0eUdocnlc zu3xn5L`HQ2ebr1%blo@EeqxI8A%{J@&fifmOi+FIb?+wDw2QLPmCfN8o}w!iFhqCl zkZA~Px(|JEoYv+Cj7?0Ht!0cJSUO18tn6n%P3m|>@E6iST)3@BUJ|w@7q^7MmFjp0c zYi;qpTnY_Lz~h$#z_hNGCwYoVjYmg;JR1_n^1_#?7p6r_FIW~B?R<$ImtX;G^rT}t zh!VokX>38^ck2L1N+S9eY;5Lt>FiM*R4U6p;#YZ?^*t_ThrjCGWIm&#sc8(uEI$5O zTuE;#H=LSWkzdir>oDxsJVnaYc4>;-Cu1*=doi5MW8;)cVF)K6&*a#^0&6q z0N*zrS<^F4vF747-M_*s4+PPNw=4p#n(r7_p)4W^69KFw?;|p4Sb`%tR4ImSj%Mh* znZj96UbfsP6+FM23_4e&kdGhpLLA|K_R2S#1`34u>02`P!f)wY;I}uke4`33L}6h- zvPDHZkJT|@T;k3Be3$6$tk}bOdx-=N3hM{U)jI3 z15`hlM}JT12T{X~FW+1uv!M#kasXpM;f^nWlVeJ_DG;bTXnW12wL<$9y3tbu?a=%O zgqd+_S?@}iA{n|ShIXiv+o4X%n(Yr}68?}0I6ar431s(Pwk7L}RKAqyuDy~sO|Pn5 z$Jz`HeG4|dxpuvX22dK3M8@;6#m8%wE*30j!Nf}u{B{$8gKP6Qf0QSw!Zd8pVC;O7 z7(?9mJOa%J@%4NXnL`(s`m%uL(gMZktBe`m>R4WI4Zmj{&J(zMWvS-%9gG8{ElmKEMy|kJemtMH{wqa%jP3K!nlT zd&jJwlB>(fh-m)kaO~_(|0{GQcVBkj-L|wCz=OewPBq%q7fTV~L={=zmIE`;Tq-0P z#Dn1kr_(T-wJCZ-@>6NBg5D1F_e;wjN6CoWQVVr3HugZ!8ppza@RaTlKs7 zO@lLHLXWRHH$HoK)LELV<(+wB=9n75JGlvlAp&zyYO6idB^ozhdDFV3!Tmz_F5$Q}?Ysr|k-b{<9}=+S#Ir9sG-quBuxmtI^Zr zjd6fe$F+G3sEW!XQar-|%AqQ^Sav>023F4I$%wxX)@O{cncG|NGGNIE=v!xitoGl} zb-|9(!{duYU{3i~jKg@^ZmVYO1$?r$)WKLkcG+utxKm_V@SOzuhtc4c`n=%J7UR8l zzu(TkJWo+qPIqqk%jlYf4T^+)00b~a8u^fon;jX8SYPuc6=DS?LzKDT*^YwrtJ!E7dTq*|(R|$kuWSvY5Q%ngH@tso z+I?clcmUB=B@M)me2@T2AH$alHp3*b(tAE05}@gJRu!HYF`lqdZ~oD4P(~*{ZSGfKzf5bTl5S z$jZCKb`1lA#x_T7G=8PrX={j5BSBttRy#sBH~*QTgoKd#iS;LvY!?As$V>;l>= z|0B57_;{1Km(WOD{~j*)#M9)>MCNDggpsIBZd&>NTH7QSHx4}GF>Y=J0hfj~-CE5D zbo%9#WrVjG?7j&Fi> ziWX`e-{iYkUCne)aOp$n{Iqo*6mHn_tCS`?C z?HLtiN)d@wEPcgVIYwCC$g!$I1r@~8#!Nt{>FtIRXn2S;Qx=lKhu#z*^wg%3eIGks z%HNCme!Np%Cd1KVF+h|JzgePRJPOMAAPY`!JWEZbDJp+Mgf**J+{i^J`{1!7+W{7# z%gfw9kK`i>llX(!&%9uvL!81)S!T+~MY0DR9t(LEX^&oBLelST34oWnA(rp_4W4hA z?5P`TVLn*Y^A(rBW=8sl|5E#oG|WmT#C>${4U*Yb>%fer&uqQlgi)i%wS8v)jTAjC zsUcCf{ZmFX9kmP@f84=-BQJswS`biB!O_mE_TRfZoA#qA+uKKoS>(R)BA@3G1qM!kb8*UpPABf7Q z%!9$+*CLWbW4&@;;0_h!Byuj#%qdinn-ikAzY1|G&5VXQSxh+0!o?YI)sYXzj2}T^ zY(x$^wPCP(7*1Gm$D1~tTd&ANuutzE@g50IP{Asb?bymao|kHHF$eXxs8c(gH$IB9 z;u|`9DGlJ#c3q2`(ey7>+o}a3sy!-r?_c3l3!9z1O&>%wM$yA$ zNlH+>A|tPF57Wl1w(QTV9qrkklh`Qs^ckm>78gbxe`R*udlUxCIA`^$F`G zD$5?7XO4|p8z|Ks7N0fUEOMlhT5{vq9GAptaXl*z9ACw0~G0N zwajXm5B8x}jSQ~wsk4Je+`DZ3;yqFFk+d^@qaE(LhitF4-hA=e_CbgbR0&-V=9xo5OjHN1!PEJ5;n*Dg*Hgf%8; z3tT5sFrO3i3mAp(m}N-mO=p;(Y?**nvc=iaex>!`AJF}pV9p1XEQyT--2Z^=hGvy> z{{cOGz(b=~6YZ1t&9HfRre%KZMa{tACPnLmjAc2|a%}wC2x>M;7Aq*W*V`xqe8Yr; zl5x}gg~=<*VPy(B__`RWB%7$oxV9-3K80v<+#CWC@Mc{dao>PI^`@rVg))>Y-g{bU zhhUQ&UD)qHcYU-VI^12+oRx0rdp!QuxoFP!>8iv!dlqh_0I1to3+ON12aXAXr;}eJnlWbO>;NwqadLlpz#S_AVR*!j=othXi?? zFBIRX0W{4}xceGa6nWjM1zTbCstJ2I<({H!7BY)2YdQO`5P4r=0!X(^=WT*eEUFj& zx0lP`Uv%8;@tlMUY3@5{^ENM*E-~CROV{0=iMo*K7))Pz*C2>vBxt~LA_5!RM@O|k z_em`}QvAPqvu!@kXerHm2`%|rPX=H zP+2ukRx``X3x9x0YEE$$@EV|W2EO+60wy~ShsN3KlRGU)WYLEcA3vvM92R@@@R7_A z#2y@p@RRyG>3K0f)jg5Xz0p1l&fQl#Dxr2Ru*5ks7BOv*Ht0XkOBa1E= z3lFQc2fwyivX41bE;a%aM&n^a*AUW)~d=Yj+GRA0&HeDu1Q?=F{|V&;DQnCyrn zg|0~|^jX*oVqgDvJokQ0kRJ&E2OW!FN@xd?|7$t1_}^l+|Lgh!tE7bgvhE|NiDiAO z8WlvRd~I4O)?_GoiR3fcFNgMu`QFX2+bo_0W9OqRofftv-0&Q+GMY52R>PZnrRBd( zmendkk06Wt*WNqMf8grc0{g(%D6;{;VmbCL`yZkuf%dwnqXjvt6Op=W*G4Khpx^o3 zi6n~A9RXx}0A4xCOTF6M!DlG@?p|DrelBRuh3~d z{1JVUA2e}euoZu_yheZy*tIFdZJH!}s<6xbTEX)O(5c-1@ULC>ep_WG(acKLpLdiM z#?QZQZV|O7Gi~;LC#ew{n`7;p6}d$c+X)sGgX0u>O&m?^IxH`#oL_GY|881}l9#Q| zlL4wksOYh3i?#VJWCoE(06`--$mx+}?@@sN(CSh!;i8Na0E1IMx&}TRB^HEfDY~sy zD{y84=$vW{!fJu=L0{Msuqhs0wUYue+W83wVVt|?%^);N17_VX%w+9V zM*s1%M%@cu5+M67+e{_5%EYg*YcoE@s-DR4Q(g+N} zKrG(NCZZ;@hUa5}Nd5L(IyB#gBZH~KdjQx}g>6vUQiCJ!yCX(pMI|t@TI^YM znWq7j3-%1E(JnD{irbML{$=2!_eg7oc~iYnICv>a2}CC!nlu3TjNxsvfC|jU{5eo9 zSPqmRLF#Ho4vzu@c+V<1d>~j@u1J~Wy{_CNwOcc^Vo8Y8+$qv^>af2s#KQH(-<0f2 zm;Tik(2%_+rK67$BFy-H9GRX7Kaa%Ryb&2hm_2Q(Ri&iYpyz7w#jcBlFY{P03dgx< z-ve-Vl8#C4sO+e8o_9`osYrMp2Y#XZRU=~|DP5&sm=9zIyRT;a2-3TZ|D_d-tm4YR z_owY(kG&r0m1Gb7rK!WfL9e}ZHQQNJDw_d%HnJm2L;`S6nEQK{om~K7T#uFJ=jpp1 zV}!YX$S6xsLvSz_TKAq;4%#dNd8cQ;>ukQq;!fhc&*QX@X>8LC8dpiMs>Qoh zD5ukaJ|7uz=RIQL!sjW*Ilt``4W(}nmfrI?#GoI)J=WoHarp;CA*LQJD40~`+dh`- zakb;LZ%4OWb>WOez%(K^H-Lr@USFWS46hA`scI8YLT~ZLhtss3A#>xv1z#Z9gt==A z*3R9$+>s+46I+}iI<30A&)_UaNl~-IM~}(4bZ`NPa+C5)L?gVP)vmYFqyN+F7pEb*zBQgnkpD5?a+Xfm!T1+#!RQjOF z1Mo9S>`(P?TzcIne^S5le&g~)Jo$5vUC*Z~{k*&hlTiSZA^^Ygd)AS z(8x`uF)g~!pE+J#+JE;N8nJj4Is|8Wl4cZrPiltC;(G_%h}20UWqh9g9en_)*l>}b zBU8N3CXG6ule|h6lh#Zb7g9-C(9vkQH4g~BJAK)OfE-$P3IVLFxy;s@9KJ`U2 z*P_8T4cO&wn1C&O?W4JFaUFY4@MwhoG%mAw=v%{=>8Od)=UCtE%T&5R0+^pjL5^IZ zCM_OG(iCFJte}SZkZ2KA#ATXFf~&DeTH98?)I zw_c{IEGS1#k-SuudJK0cC*Ar8!SsyfeaW$6Yl!f-Xw=zkl^9}kQ&2#rQKa3#u3)6$ z=gERlxMgl5>UZF`1zavU8)5kUGk#$)BC5cg8&qTuAMYR(yN07^0(=-maj@y2K)VNf@Rsl+v@?r$g427gHpx zMh74g-nFTRZW8J^t#ea-w&^gidj@}trHOMqnH^>KL6<%Dtmu^ZX3uzX8%MRQl<0Bt zu;>Wt>D5QUQPk@pAWG?tP!^en65TyL zuUxMdV%|5KjE$JtY$zvv74g(Ms_lDXNT`IZln>6MmL z*7XOa#O&PdTWtnU=Rzn2?)&YC`&)6~v8l4B4^m&0^YGju#6ZFqsTS%e-l#!>=jN-) zJhjQ0o+COh)8dPSBdAFquZ%#)GI@n-!Kv{}1bWYrUU7DzwL3#ZN#?cUI8({@LU5(O zk**`1Cksj-wFJ)gKP2^On~gI!Fs67#XLH@KC`ZqG=ZJb_2hJ8EbMSB^~{VOCvAM{6mU?7b{ZIR+7I zLRFurJh&85I2Jx-={mt{u|lX{?2GDNR7-TEQ#+?h&`#&h4kQk;z%#;A6Gcw!~s=@V@;Ze=IJUMYi@v8QDBO>{cz+*pb}vj20_4kDlV6<+|-G%$=! z&QShs%^xaUTSi0_cc5NO98M<2CqD#jl`__rVAbeR{d(e9BfWC^YGB@-c#vkX>1a%N zgUp4QVZVNiL=IdD;MCuOrzM*4!-3X-@Gqn!zaM_ilUdIGRj8e=^YlBHuJ+9v%6?U~%?CEA z`D;>oxMYvcZ=!TSAwPY&W}8fmq7&i<4kL&nJE_$|f=M7&xLc;y(*gWGs1x%fk5;bU z)#vM|caB7~kPKckTyD8Deae##oI)}2pTn_CQ30J~P!IAPjz@(?KNqcje*5ZW`Ia&Y z9oF67qtM0W)Yjt|zFIJXsm-dR+{Jf4vb8C@ott4?T)X|;epMU7DiM$DT~cr!($wCl zdh&8P5hr=X+$hFamj6Owb03(;^L%KBo(QSaeCioi)B8Cz@*r;7%!-jR08qte1*_ds z_I{w69WAWxi}LC5hw-Hn5^LRMIf7cVBYSQc(N`wa@|Bh_#og*yFBRe|cYb*r3ID5; zlSE;_vPdq6IL4WG;%7etplGTRa(Tcy{%y!e)ZF0p?Txb>BmhvffNsk6&EwO#&Us)QkD8hgLM_VIO67_NB4{sp=f;+-X;RDY@W+bN<4%7sh&VxEFgZ^N>+pbc{m zs(0XcFh65V+gUV}%yzI6UCn}NejayUt(FZQm^k>OLUcwpp4Fi-=`}!;%GnpSm0Q4k zn<34PcxP~icpn%jT&8=OLUk|$A!ktO1-c|N%SRC;$fj6pr>G9!;U0Vr=Y8MIO^B?> zWjcY<(26R;5B=p_)Z}kq(IVKm(c02-`0KZ+N*tDP#MM=vuK)RVbCgtyo8x;LpH`ES zul4s9&cn$O-u^z<=!>Ws#sHpgck%IES;^@k!ZoCYLp~;c8)T?Aq@EL}*WL4yi%Yt5 zA$Mb@>C65vcz2lF*eP*4FdvSa<0Onq>R6+l*@P^?M`HvV>`FJSn(I~?X2GmbVUg?W z*%^3XnZTw~E8jO@Qj(SUmvqsopV2yzx0oL&?Zy07sc*#ABVYtV4s?XE20~Rz_?C|H zwA_Fp5r_!l2(#noRq}>@Z>`s0rW3q89LsxMACbBEMYrT2sf1&rBs^c?i46mX!$GwN z9ww&k>|{YH@>tN)avChMY7(@#3Xw>B?PSE6?da^c< zKg+Y+2TMHvm~;DnkUmq1?t$Ky!|0vG{2OehO7w3NIDA$Flq_ zZWIvN?H5)}SU3i+=dYaj&CJ4OY8XFt3E|pusgyNtl=Ei@(l<+%_}=c*xcdJKC7wn! z4~TRWXTQ(yeit=2Z3D{7S)ZZaXWWkujIO?}UR?F=HgM`A-y@zYkg{0#LK1rY<}PQ^ zi$zgx1Lrr|xc+L!ED9A*v(*XEqQ)CKs}z6+Gg7T3pr1K!-q#dX9&Y_{uAv=9eihL= zvcd_UfxnGwZ3Upupt$hst0SgG{|#uF)-O$I6HkA5a7m}2&H0>v$tU`}R~MycQ@@zJ zn7^YcB=$dh+xr{`s-z23JN?bmlKR1a9G~4DZ2Fx0TOffKv-Cs+pcccik)WuMmLpKT zfSEXBJ&o&qhbsIgY;;M7K+?dE=Zg`yT*s~blw^A2PRUqN|50Zn^UJ94RQP1fS;hxL znydaW%Pw@jt`}YCbO6X3`Y~5=2@K+k0geD-n0nq1GUv1S&5M)#Cg(sgP-K3AK7nI= zZhuwsK-Qzx*W3T0SbMNH)<)3H+na*t6=b*4*72*5b)Wc`WN0t7T!oFZb&;Y<@Mebr z))#=pK5`%tH(yheg+^C_I z+dGvUz?^%aSW>yWCzAzCX{-vZY!BpQn`%L&@=EFlv?T|Iu{TQBD5u{~s_?Vsyjk?ruho(IZ4U zq#Klw?(P;z=@5{XZV&{d1*9aTq&xi0_vicjXMgRSot<-U_kCTj=kxK9in+Q=v@QDY zXc2C_G+%*3>qmLp*DrpxGJAJ;m8*=0LGkmU@+MSvzxy@?bM7YKd3}bKC^f^yH_^CG zrgem%VFyt#Ag#-K9C?oTN1N}VvuJDn^01|ku~yo-oTBJSEEcnAj!7j9ZdO#`Q8!GV zUrW#T^S!vDMR^<>AS!(IX+m?wWGHZ&iBaOmDqeRkMZm@wB*YHiRi?Xjko8?!b%41n z-MYSOJcwCrs$!R!{(4~ZRH5YC({|ub@qpJF1mU`7@(4M_);sn4WW_GAoV|!ptioLT zW$^RL`pF`WCb3UP?PVw-ZYXv6{5AdqVQEXN8&@2S8XbrE?XM|`BQZIAQM8--6&FP|c zr7TN%Z2zfi^{0)h#hc!+O7%B*XRQSoQTQS;$)NynL{=0=*dj{gqulu5kM~Rf=)+ z=o6WBjKjzNQ6@3T}X!Ji(zK*YpKb^`Oc%PTs5-M~@L zf1y%6ys=xvF>w6}jv%k$k{E`_oy%eBTf`W=tbL_#7N*wz8)N)w+$A=QZG8XpQFmq< z%jkKPlARr^J%!8r#K@S)OThu`LLgJwGFTXt9t_9{uJP#+7||Tk$cr1+WZY5_?bPPh zbx?Wt^2-;G2aX?)1*oa!^~?{(d*-p)($Iby02DX!Nmpd>Y;&rNcgILv;V> zOXe$L?%*Oz-y$SfAtC87@k_X*kZEr4Qx%L`>_0O`GCF#;zgve%I&@ydtyRMp zN|?%wxS7hdpwmJ{5SSd~gov?9p#&Lmy0##8ZM&?vrTa%AkEh*nE_2rBH@TTD(`MRH zu}2+#gx@HEeU&`5vtZZ+ol=f()@rF644JMSXas0%K+OV9~NwHcCJX2U$q|t$OS$kAM(lF1Rs|{|fGgVQ^^?8)Ly-1^|>`@k# ztpOGGCs~fLRyLWKs4d^M1@9A{7Vb`~d!0mbLk)#?PjPw)Dh}bMk zrNX!KQ_kSws{qB3Q28Q`6J)3}2CK>HTdNKZ)P>Zh2;r-**)By4|N{OKI|E*H)9fIvsi$)&qY5W!9lCOP%wX zjhdVN7TGA4v=?+EiPo9dwgc@B1h<9OtxdJf5{4VqHCnp0b|(if5-c!|$jFi}XAj6yJRsUd`5!YMxUWe?w><_qb9F}LpePEfbbd!eVI@~NgQ)c#MbD#Co=Vr9#N@1qOaxtF}WIR*`S+=g`M`7KV z+gE*PpZOBm2SxiyT_S(w?}~6w+g{Y)wP1RhJt}E7kPiRi&9ZzEPsS8?`kvIgY3~sw zOP{ql8!iCXn*I6xEd`a;^QR2n1EfC-=|P5XxI~a-w-Ol`<+2`RZ{|n?Z9ZL{3dWb| zEU(%S)3m;j+&rnYT=eRJwb^#LiQKY8d$hnlS6%6AAHs(9@|g7AJY6_&K&^P$Ju1Un ze^x4VJ8Z(hOsEjHcyqob?FWX#676BO=;@!8>y2?b4q3||3TVnE+$@bxUu>QCx@Gqt z`}4B(r|tv~JipArxAN%HjZ>7iP|-`3&M3l}U0~BAe=8pfUn-wSTeIgd4li|SdvFd# zWzdf-vqPVsLdP+C9A#IbtG+8V_NWYS+lu7>d>S(d-r_Zg=XPbvy&!X2+BE#2K@uFh ziZ>o~|E<_6gATQMPM88VT*3+}BO&6}x^VatV_c~s_tgZ)2nO75JcOgyP=r{LckhH$ z8|1rVSN_8f&x#USsp%Z;G*xmb3xm@%&|;fgK5~H>)8%r-Gb}66)r+97ps4&L#W0mrh_!TIdFCG@wZotT^#1x}neyjzUW=rb1;7f*~;hD~Q1a za-Do^e4a6-lW<*5)aJ#aARc`?VkQ9-C(i!eC}#(;sJLgNyzZ~L($=n^GY2>#y`GuS(Lm)l%^X>TbqHrfAnWv z#0}j+68*0=!7#izZ~$R2cKzF7i+q>%jMFY`@RHHx-ky_hTnz_ShM(ucfUN zWXKFjAU~WD)O(Z~ew4^|L*&()5X>(e+cyQN6X#WRlsxqt5Ljw@|5By31^29O6a0ze zxBNHph(b=UwQ;YH#36&5XSdiE1D0-sL=~c1;7GPz+zDBs+Q*!If$y2>%l_qjV*;z) zrV<<@bSkE1d~_ZQ8OT^f8ORuHoDfxzSBcNrZQ&#DUO>d2$|S$MPTsY3o)kHWR|#rQ zR%g)~OVq`meLAk)`10qlO;#&0Yw8C>YDIM+kuM!cD@UYG^)|X^cmC({AM|O0IUVEs z+Fx%*%%M<$C7~Wxo3>k|r%(C!G?S4WJhZt-fneZf0~ln@%Bx|Po$@NXlpr6)e>8>r zT?`;!6O(-@;QTfhPr|COf*^IZ+kH>7T113XQXOqKd4}72U&49YS-O?XfW97rAuI8W zpn+ndxMA*U${-9ytRY|n&uqZTGTv8_ZUiBb2(mWoNMJRIhR|<=R>ghm;zh#UPVYJf zpx|8EDDE~gCrRM{02)J#vMY>5max0`a|F09;vl~Sh37; zXf38z6UyOZ?h*QNq@MJ~;$)A(=<>Q)cYCc3z{EfuF z>Ls$*^=}!MY-a*7f0c*3CKV*Ot()(8oU`Qp!`lI=|B#Sd{F&3axs3pc)y3l)V43(M z`o%TR=;kIw6R}?`UCmn!>~?q)nT&hu5JEFuc6{;h7hnJ-PbZ}GO;UY)=BHoM^hf7( zJPQyLi86SvyRNI*I7oyNWr^WsWyr^)nPJ8zg$>y@U&E_YQzmCcNf_Bj8C0yH-+(Sb z7&Up-Ycw#xsEh`oEENnu5J4!;Cz)^EC_vBzNS-JsRph#7)5hD>9uB?8%hgg2NV9@nKr4$NYC6&-75>A%idrB)Qsht>X1QP+IluEy>+k$2T zP8*~)*pa0F0yMl612gZFL$0JX-O&5hJcr=W^IEUh1!0gY;gZ7aO61ujpR)Rdw z`tWOnI#y2{Zwkop2_IY-?%K%cQe<5x$a>6zqt^yMfMgT*<~Do!p3O=S#%$M})&jG= zpiH_@HbCDw;~L`_V>Y_E8e#L$O>pI$H_kf5V03fpC{=jtAfS{|zFy%JH3|PR(FEP> zY!OuVkM;h)S8V9&w_{{6OytKwC(GM6U4Gs5ORnC30o6NCr$yK%2r(I`R}KFU<5HsK zaIaynQL=!wiW~Km@cmzasm-(^!rAovT#P93{f{UL4n}kr(TY6GC)6&JB3CAsy%4~h z$E{ye*HTLTh@l7(tF~ZqBml>`9xwX&MaSb4#1Of?iWa}QpamJv7d7%*r5w7cspb2b z^0>Ll8_&&z)Rv+z+=SzoYPTnaC)I3zy-CWWlLr*lEycP#T*lIH8<_&%8BHT0BlJ8= zuzY8;jBad6AhCoWbDi>i)!Sn3pl zCg)q=x_A2*Kn9LtxJ(MFL>Okm5p0U_a{31hs&Ey=XBJi@`{a2fe z0=#w+H+b^Jli@@38tWmbP#R@7-;37f?q5J?u=sB;q4Mz>8Gn36a<;g^YKklsaY*M(5hTR00Y1eS0VP$2yf;xr>=%y`rrpPj_2h51q2o@FJ`bg zW(ofmrtvE#5pjMovq-D@Td`&v<|VKnXk!x;BsKZJ?Gc;G$J{?}#aBMAF*5N5MRZ$T zugu~+%UlFl$|hqnmWiOU`YaNS58#}MSRR0}GK+ocm~(3gL#2ME_V zYAd<8T@tShz6qhKqpGjMBtk~4*!_ldzkP7fQ*VPKNS3iM^W#6>(!rFI9;z!f6CXKk1gHy`B@?-L+0fuTx!`c|t{47@g zOZL##rD@t5_S9GZ?(6mrMg2^VIO2U1lgyAz)7`p{{-522b#Qa!jP_oy(RqELp8IU- zA@P6utXP?x0>qQJKjC#~@&3aw1jeed`ke() zGN4+7C1@de?tbh-b>2l~HkhdoNcy^LIC1xrx}U&l*salogx}V{dv_E)L6e-*i{F~~ zFH1R8bB5p=C2r3e8^GdEoiyj|*I(1pcf1f@$7|DgPfh60j)7|(9{&2hMO&Zx4DASN-h`c2+cNlY5}a;8|CduLx*VAa3PLB*YhI%_< zrF^fbb=ZzR%-3CzXQ;~TpejPasbcnq=sOg}jeLQP@Kw`d^O}rm6)kt${$hv$78+hi z(-J!@{I@_uV*^8i&Ef%O0JP(ZXF=pq7}q!g)1!i;|JB&LdWs6^h%iSIMYiQEl9iyl zp_V*VR*Z1L(g+5_RQNJcNy%@GX5`@F>6gEU`BdSVAiO#PRHEYE_Y@LXjan;dN?Peq zwTuawEHH&J1^P!(K(l=<`*vw~p}1pJ>HsSbajV9!_j2;l`Ok%btyCDI`x-+%Tgk*G zpGYqxsrt(Ln?0FVA$8D&Xq3l0l=EodOphK99;4<5GR|g1RDHgFsNdV;%|bne=9G;% z#pZ`vx+qlfjGXzxMzs>Gh^DO{AnIy17)jYGJld_Wuu}-YIIwx}YeNN|;cMHpK&CHO9NrfRSZb!g*pUfbPlA0qxuO&1KXdImRVqEOzLQu0}3e zSB%-R-ppDjB%f+CM|{>gohhq@@4z1BA~2#QXwVx8GJA}&3%+^2dTb#RtG0mH@aQ~) z+veAVj$9qfi_kRmSgo@?(A$=3UKMq7>2*cYoB|-AF#`Idxe}Y zyzN>eNB6d%rqb?LoU3%794IbJ!-VWB7Vf4@J$eysD4Qg??(1(eT^w6_d%LS z&D(P$otd=apzXts4(W)r-#O49HP?jmd)rPs8l63BzxnNJ%IQYHlTdv{g~N!jSJ4U{ zuC-vCM{!rI5>`LK;SLOS_$9kJ+89?Typ*n7PGX4Zn=BxMibjBz|J{4zRx6EQfROV$ zlN#F3*DT+!?US;xbI|;ED+iE+SJ$q%`-#oCeT6DiA2K_uX@uy`P(ybuZb;Cle^$*Ki&3itp6C)dRG5*|0>r|KLyy`XLHeE z|3M`s>{`P!9^BBnJBE4$aa>fVuAzttk%7fwtWsOUuP`u5TqwD|n?kL}FCU3NpQ^+X!|lW0aWBbG7@xn4RL zm3R^lxyu&iyM4H1?Lj%*eIFg>y~@6_le~}0_8lTk-e#o+h@ShK@4Lwfreetv5O1OP z;|yL0chtl~ntijRg}e{`aFlO8;D{M`9xy%|%3{mfN@hE5dQ5yH7k)Y7i(?4S5G^b| zEG&kDls&c1QwchNq9038nQRC;n7)@-pSCZMTP!$gl)o&6HgHqXxPP*eV9B+Z3ox!i z^FYV4`Yy0WKLU<`2Hj8Nb z&eE9a#1WfwFKTXP;>0h*>0xY3fr(GP*-AWkS|*IiFOJXHe^gv1_~m*T{$evx#-0~Z z!>zBWJgM$tsv3}maywj?r+As>*V{{pOq`UNtvFK5Cu zI4G2Uh8$m3mT6FmrrZ@G(dG`ckdjB>JTe0KxK%hIgCqz$eUroc16WOQt(pzd)HbmL zyESWtb`VRI;e627T}*bOYIZ2A-Cw}h6$qQ+iOakno|+Msk`C%xDc))+p9w_=iOdRF#4#rUr|UB?msH_kb@~GhYm<<(7=1 z=YG6}Kn)@XB6#XwsIs|bq!*2(6KG*+HB-qfRvE_H8b(|;244}iQ?&Us@}R{l^q55C z9?n0u1z9NYN4q)df2=p3bb+#`XA~*2d@&Sq1hx@b6J#(W77Bu_MNLPw;=LdnwH>B_ zi+9-(aO5yPwD-cDQR{Zy)<61`GK#*rK}aZ?l2a;5LJ{Uh(q$k~l_wdMeMZ^# zC1Pu~^qcobB0Pd=0ja2AKBr|pV|d#|8voZzJCY?_4wx>eiOV4j6-A2x%EwC}*!TIX z6*VFAF;@N8G(&)}up5>0mDeT=*114YafqqFuxi+Gm1I_vkjI}NJ=>O!}jAdE8; zCT`z>A3Nm56a4RcF?^$5Sa6FJM(}BkhqPu_T0^}ui<%^tr%OU5MqYPVh~Px0#P$Bd zC8dqz&M0ZXo_9C8V6i*LD|J;|!oFae;8%NK%}V@w76X6DW%oiZ^aH3Q3kxV=I%0R$ zc*b+Y@8|kIx`7DSPVe3S&Nd;NLCs4G9y3OzP2Ti z8|}B$_0i7UEfLF&9S`0bP}9EF8Oh;%cS)RY^}Q=$Q~*8e6(gmem9JGR_tVGlr-F{a z51YuAno}N<5lK8UP9S+XY}bX=#1(^)8V`hHm`&{n_rZ|igTjc>BQA00-T9j)&`kGoo}e z{de+Cq}AN+aegk-I#5Q@bRhBk7Z8MSiCpyjtT+N`-kS|Xb;^99gXU7n|2Gwpf?*oh zt=+3-fmT`429XT#w>ZiY@B-UlE|HWImXpI6f{uv0Rik}uA*uWN@=-F{wIz?Vp*cC< zChS{b0!*}1uY!_Ep(lP0rwl^cYObMxpegQ~H^z%mXlI|(3E&2y&` z{19<3O@S6^y3w z_hgpg3EgQI526O?kBYTdw{x=T7%>RTLfP(m=SepP%HjC8srPyzTry-Uu4flFedv?V zj#GiR1ikxKuH{@`mS;ORqUQkByn^N4UPmZWYv}mWf#m_u*O%>#{1iIiIOlqlN#>wd zNumG1xnKpe*^17mN8V2&H1v2^P|^~PwhN{ z(Nk1neGD+ilk&=MUI>bK2&%-9a16hQ1m)6$XMgDjsVaYzJ`^qbVgEWSYEJRvuUi@f z|5h|r1h&5PLHP>z>Jx9`K|@T3?X?vmPeOkpDW?PwI$p+X%89&Sa36lMW3`!cx8V10 zrqhw&d$lx)WNZUh=Re@YHZX}4?G~!E$#?lpk#=Fi`Y`67-aB*cre_x=K;14OmoL69 zzgg>13dx+*SOH5I#-RFBsR>$0)BQOCH_L!1_+qagL;Ct=rNGpL7lQsjh*BtqUSL|M z?8;ogJ)&egQYgIF(6%fby(jQpc94WiUFqT~z&!NhWm2H2?TJ6T6GCB!q17--=s30u zKBhz=Ks~YVru^>=_$7))bJ6Xq%dn4^iLLjWqyIwVga4xAlL$0ZQIS34i`=zA*-NmC z70G~A415c~opH=%yb8yS{mEZ|cC60IB3Cd;OCb~8CN~fVn~JhWiF(q-$+uIq}FIOesbwg|Q6)(3n7MJ!D0q`J9u6dpDRDYv0tdFV?WV z)4mt2a}zuS%Bh_%DYmCIJ#Sz3Im|b2QygDXE)bb4$-vGjdc=>RbmErge`HDb15=uu zgI~(8kL~x+<4FHoMbw5GRD`-(6iL1L84u5>;=+$WaD;M7L;*}oSU(H4=~EeFve%ve z0vZrTWAiu5(nkL}#Y+>b_mlzoZS)Fd8Se_v&Mh{OEJA4jM4;CtNGzUcznnnzuZ4BR zUkue~oC^PbJ%Nk(H%LtnM~0`-3AlT4mc=>qJK_wuGnlKQd z`(9wn{&5U46fN=bSW~b_eTiSSUZzOZ! zz#ywy?RVZi9MG2b>{frvt|LscKRY{) zg9)u0yt8AgM5&Q-lf>v=j(1UbVj*PWI;PpXd+I6C9aLwzwv&iRdCu}$kM@=t1S^|~ z-77DyVhqS-18B!sjx`GRiD!~0eV=l7FYh4cmonH4(m%KHK1frg(EHxS;qP=kKGnCZ zdZKmXvMpBc1Buxrqk1@AAZwb`@^us3KLRROGzr*dF!

Pl1c`y^!TK$s4p^S;NOb zNz7GQTvK6Na!s~(%SI*-j#7_ysn5LNN8x=Zcrkpqywa?2b;f1 zjQQRA{n~Y3qD9Qx`tt{R9pTsMERvH*$X-U7wTw2|6N7w&e4=BYXTlki>`+-rVc`Lc zZf59iBdLDR1mSAk9K2^y?|> zMi!^YZi7$L83%QuVdYTb1W|Ei8YY%udMPnQN>$rKlVmHC&M0?r4=+FIuNN*KD%rsx zPDyh>8o&A+k?aWIILPVL*Y#5-bG#C7f!&$Ti?uzI#DtFf+|Sf;L%Z8sh+{CnJ z*36r>h=biMF7uI$+pq)ys^hCK7bU_gRdfOXy6!XC((u0kg^2@qUSv#Ee)qkay=Jn< znINjMp}Za!~VOgwFy`1dF`fc?@e`ffQbhP$$I72T=LL&{s z>tvm+KkJr6=E%)JV43pP@3izIQd7cU>GC>@-m!XZ3H9p?69h`V)??|wv7UNNp;kGx zwp6m%rOTR#Mos98oLV_8qAfW#P;*J;%uHgcoN8SLCmjSQ1WVjvkf4S=HNB{O@(PP{ zOf~hYo_5P!=;yjk?0$~0=SNhQ2%yaFIAbzyNqW4H!NtKAw_tkCPaoH(axNt^rQmdm ziN0cEIf9IqbGGyC%*(GWk7F(`g9xagq`XVm2o4DAfO!yCRN>;neGt)`IENt9-4^K=(}raENFGHR>O8Ol44dKwzc2VCye zJ&YW_jw6>@ROze63?A5e5fX-eK~`@2xW?(XEaJzXV~d|Oz?4H*&Q5pAGi zN?9aDMPM9A7tBZwmy6#|RbcNH5XcBud8%MU-t2SG*_GxYG(Y#(5Hr8eOn#Mc+q5Ic zI`u6&;pO0mHv^b&4_3!olE32$kGrHtpc5^xXrFT30Kmjd7_V6?wVCMOss(!35VWU- z1Z@J-9yuwW3gfi59ZW_1b3ZqhRmt%#35Zq>B>&l1YIxU7*3hv({PgLW(}$yRnWFTi z%@N(3ukOq_LTZf_$QZc1p;sCip&3OZLEKRUKXB__4dcSyU&RkZ^+AWW1T8#Jf-yY1 zj#g5r2O6bg8$_Fed~7)St$j#;sSZSC12iPAwijHo{Rvwyw^W}j+hjtA{Kz#t zm9l1Nh)QEKq;z$&OvEV^#n=P2nwXqLG&bj$TsR;)c`qBKqa>8`xYZ`qdj9+%#*5ii zYH8%1>qs4ZPhEA{{#)mJcu|xoqgS~QQV&n0RRnX=5z<|hM_|#%{=k)>`M758Y4(G^ zfTf&D>!@{>3!cvRpg*A{QK_@e^OA3oVe(};_Hz}OOtLol9ZiMik2sW*9?&8Ong`d8v4c=d>0C}IOR3ia;lZBpj zEb%Tnw=0)1A^mP!^|ww+5+X~xPND8T~_rak8>WKlBl9%RPeW~id-sU>n39VgtX(EoLW=mx>FAA zzhq1?(sBE5!U=cHo-#ch#7|yFDfYn=|itYmH81C%E~ZfI@L4p z7v|Gcq_TzOoyVUAf8T=P#AXU)!+HO%lndqF*jF|^0ggFB^(_F^D~<=rgBJ8Erb3Es zZ2YRbJMXAxOJ;h+e)jIInueKlE0{nC*PtF;aQ!Y0OWEn#_x5;`bEjp-v+Q961qut> zKM_1I%v9ANB4YB$0IUZ>Q%fs7G{q()XqWwq(>@=mIL8PTI=HM>!l$UcaT9y5D^Yz( z0@5dW)F{pvaAGX12DwoP&xGub1JNFptzV_7b*SgnA0{&Ctag_D`~yoTDs9=#g6fwN z=tD6LBG`h;@pvfX`zkOCHxsgvhgQw918v({(8VsDQlkjuO6*YZM`sdu-$&7lSBH1F z$(->%sp?g7ae6yw;mPUBnL{8L4B88NKmbvx%J5XCSAQUNm;2Q@?#{B+ z8LiFR90w{-@Y`0q^zO^oIkB04jJIS}(B4HJAfD;nc zm@2vm31KXgf&9DRCxE5H&hisF9F(a9WNm9FhJG>Mm457pYg09KVU(S2}?%e3ZL zrp*+xh^<)O99}-8rhAbUm+THN4|GtC8nUpfH$pVW!n?_0SC;2CqiJ&l*KDgm`Ymhb z8KN_T5^HVYz3voRatkG2IhoioXW)gfn6JuRa;nt+2XpVp-1|`;Agu3VLwWqLw4^_A zkfRwDkig`^O4+AXAZjkbkYv0F%8l7`abHX^CNtV_coXj^D!3M=HP7jqgFrhlcF-(bzfh5fvri=_t@232@6atRLqLP zYG(oE57v6@utoKpY3Y)-@zGR&-4o$2w$#pn2}e9GXF2-fN+Rfzn+*nrg<)#&x3C$L znL?YMw!S1-0lM5l4>a#bX=#|eS=B6&9HwT5w zr9uEBu}_`X?FNs0$PO|q^$hF%zqpxI8B@K|X_s{(5Q{scC+>UJW4vPSIrY&- zC%&A}Jhf^OL**MA0d6Q@^NjyD87`Ae^|cZvDzlalin6^{7`o>n!?^oNLQ&#y)(%x= z?YG|5=KXO4S(fXyy~0EvD1BjGi1bg9Tk}xXP6WvgU`}O&}B& zg8UVqwBc(1!3b0-DTV#|^&}w#2&@DX&_rb~+@OVwG?}77c151{Zbhw$#QY(M90zRU z0B;zrEJ~pBN+EzBn@l*a{ z2bn%ldA#Qbba<#2|eb9Kj*Xbg>4z7BQJFgYU75e?HZx2^GAg9bA@3s?ZKK?)us zK!3%5^BoXXC2VKB^6=)g4=4g;;@Z<}3t{7HS=a1tqZeYTbvrE zyg;>AAuXSYa@JCN9sV+bRVbiuTQKL`uTR=mZX=$GaVkcx4qeX3vBi+#D6wa} zE6XB>?l@-q((Bs)(<0^kx8uPCoZN^;(nSJ;i_WQ?ye=PhxRpC4o|Q0^|8@I45O&9& zeIO27jASp+@9lc+7o*Hcl>o{1I|y+U1cd^Ro*P}-GhplcoUdmn25qOmOw#cmAV@Yo zCuPEytdpE!!+YCi>3_urdG+q+jXR5aIoHq^0~ooAH>)|(4KXW)*UTz?_@D;~QOgIk zdr#p7(2+8)@E%|}fTv+uvgoV!%QwJChK#%`923LZ(L)0j$*tF+Dk8f|3-p}ac=GO% z{d#dTC4}82l=HnVx7---&kRz4=pLwb>p*;fD=Fc^nu&CegffsCSy#pbondYxlqP9L z>KOs&{?)MOx_-FrlIzig>&37$ zv66$}AV{Ln`zvP@bH0A;LaY@qmwGQq!ny_tTMI_~!RM+y;=Ha|T=cTJBveCWYeg}w zGL(J1_2<4J3m73&3s1t!wtnaRzCDN@3d@L(!ZVqr9h7Kn-@!ohC7 zMDGXz2}{|o%-%etVu))&fzVXhU>DhxZQw;N1hE4ln1(=`l*41P8PeB*pax#s@zGEn z2~Pm@*0T>Nd>8`+h_B|iZlgxs4Ro#WNnIW~bcZVm|I?TvVdD1v@fMdb(5r@@YB2+S z^X^i5R6VI>;}O;8o88l1b`%QTV>qi{)JvfzB1VsWLgsRC?5WkCcEr>;GYv|dsIwjOdzv)_Q3Cy@h#2uZ$*|dxlFw19I5_O*DXwH zP0Yupw*6;`xq{!ofL?@4b@vSsNwt3IMhL>Y(2#GnGXwPZ8o+96YO|s57I~K|oD~jm z3;yZy;_!MO50I~OLU^;&0uX9+`3s<+O?b4JezT0)`3Sy0U&3a=dPDSE@{iH48c?MQ z^97o(GqrS9zUko{{;yJ@z0HPDMwD_#&Ex?X{0;RaICdEmII&{k%*Rrc)~bGbSsrun z4fO2;liO@`s<8wwj;CR&(=tUY#As0%89Z?-i|!X4W^U_egbRnGt&?8-#2?L-tgR@j zeA&={=_b$DV2&v%xIne_0cdumk;iOv=4V`Bqya2tbUk(1yzh%Hh=%<@h>6(fjP-%3 z5RWSxpprdNu-y$Q>euu5y2JerHkj5>E#REi0+4u5?Jv z`)4_I$JUu`ijX|46RlsSa4tb*bK$b>fDvFV6sHHP-~;!gGq`kiX+UJDujtq+87q$E zR5;nMi^sEM2xB!cS&ip;uI=P}GD~@)VHU zX=p_sO|BBwGa-{HBKWD{^6ki3&TfHAEQfyFwh76Pv9;?-&+z z#x4=d*U`5BQpiLjb`=Cn&%E?F7;AiZ4;3i55vvt#m@}?U* zuyoA$1}=vLBhM6&BdO?AG^v3IOA_RIe>3(&b1&GWe|dO7s*#T-aEy_>1S_Lc!nIW@ zs!8TdA81MM`aOkItz9}d{>Ty6YEW}A7{(eRfkOenLZsKhMHQ$Tbh$~`Hfrx@vPGLs zswU(!aZR%%43tct2S2v0O_{Bq!Hk<{uLgTRkqgsLJ%c^4NP3>l-*?)2tbKcSFgMU4`u zUXFn{LtohY(();u&8Ve2r;G5Oy8nYPv3%3dj{~#km->W=SuF(Wk=qrBe_^pw{wuA8 z<_AZ$u0ZTW7H7a7ro9lBN5fpLEY54z+1XL{ISSQSth1JL6&9MZqmhqm->w}La(s+w z2=N6Au8T7r7A?2m+>L|63{M+R1Olhqw*_VUYRuxNz$pOAW&6wM&POlRDn$GYjRU;(q zk0rur>|eu`-zHnVt>ula z;0f1HN-xzFm$_|Nawo2P;z0c~;ay`b5~s14g_6tWnp7ep1vo|K)}_cLLL6JR;;i0M zbuU6yQI13>p7}+Grzxc*n1a}WfP{oundg|cn0v}*|LH@)_!Os|lPpgm5FKe%t1GAL z9^-{2E6e1`df)7L1^#qAEb7G)sVsS6eOtU&;7A4{ zK@Ya+9{t&A8l>apGy&`PPMZrq?YHaM4zSsp4`6UAcXlYu1buKz4y#iEzZ{nMlycoW zNt-oNd9#f{a)AP0<2s=<-)&NP$k)TT zTv5o=5$cQW@Jfe$CR+zju<#ddI0*C$28;4Vg0ROslXa31@@5?JZfmABPPfWy{E25! z{OURJtFf&P@g-73wOC*1`~~o;`~?tuzVp$^Z_5hj_*__8N&G5;ke&J`1XFCmG(A~2 zqXdUMLb{o;RK0oQ1g@vy*Q{ryJ#1Z_{wgMUUWIlm!hp}=w`(mbjRdapvOl+&MyZ7x zS-dDBALdYJ?_@(m`5B#}!^ zj>srgWcXq@%4q)d`Ug_@jFT~nVd}iseIv-qQq&8YH?j_&g0k2c-O<#*{WzG;p$RU79^h6m@{f0Mr^r&TT!uXZM_2rlKM&C=lo|!a! zUm(m)mbEv~VY0;LlOwPTO{Q8K8ZkLu^YS@}-(%RA8bk-oo#K(U5zQ`F`2E}A^+JH^@0A&54u+A<3r_6aOrqNwCWPC}pvX|a z1~VYy`b}x5yZI!%d0O26!6vm@KZw?-gEwg+fQMgE?sLT2l*b_@#?qxq@ zlH&f!v5(43gY80`pHv+__uhCoM$7)9rnEnqYTT2EjeV7?R8^xCk8XJO620-qwb`<9 znUZssn!xrqBQ9X5VnB|LI<+0K$|fLGo#EneWo_U(>U)-B!&CY(3wC@Tf57XUF(|6i zE2yb^24gCd_1*iqbj#NnsS!+I(N}yS>Eu9rIAFf^46 zv?n`XIwGM@ChHWzaYX>dXnL{yj+_+0;UkI+%U@li&*kG6g%!joB;(6b?5y%uEFD{6 z!{6u3g8u-ccQ!Fm=BCD9^wxGWoM1mrM0w8?rIQ_z`J9x9se_0VD1Xqa#0^s4jZqq^ z@I?;pa2_od&R4De1(dU5(?o5Ck|Qbx6e$z_GVxq@w1B{MuJ?$_L0cH$kzJ?#!n@>yftAeT*n8N%hy3m@AZ~Lbxo`ic z$v`HJi#-yQM|Rs3v3A*&z8RP)^raP!p3AIg9Y4Xq0FLJ1djup^q@5vFE(9EZ9?h=K zEzZkwj7`TQ1fm>DmutMaDIF5@RA!*7Dq(Y$1nKV|l{4eBq;L{XiRl>=zF2hsQ=x_n zq8lKmO(QL=yYKDnlUZtF?zsHXenDy#7&M=qXZ{-GdUJCl`u_iT?I;0G+&2#ExxLYB zoygdlDYOXOplCdQ#gTk70KI?pgaG#5+TD-NSk_@ezU8vgedZ%vV=X|6W%H?ibZpR@ z0s!5{lt@*JOBmf%-|$1A6BCu`r4li~Feo9_$MuIn=q?YURY0E2%W`s3{s}`-3^ljs z+Xf>ZQXVLm<`Rn|DyclEJ_sxE?Oulw=K%Nn|lj<+6? zRCtRYFoFa5Wt?~lxKLgleA@yu(i${J9NWo^{Wiz%Fx}fyv01fW#e;La=`c= z^5T%W`;%4p+S+akjKwK2z?f)G^xhK5(q8zX7?^uBOW<(ieR1!c14d15PuI+m|Mg!8 zHdhutJ}&+RWEk}qoJsL6jrFeXoIF^HAprOE|6ai8*))OvWjq1`tr9t;2ufloKCRzf z5~o2Z1As9vF;z1gAtCnDSUsb_{6iyjaUM!-Cz%8d^I3H!Dc}(sOqN%}IX-j($-cZ# zTIU4GjxrXgx=z^_$U7&AwqSr2lz9XGV^CUvoDWBpP3{mK@fjBpL@_1;ieoF;Ig<~{ zWNS?~R%xf^^u9mgm(xxXrtciBOXk8tBH%1yS;N*C_Ugk0G@yh>Emq6;$)9dN3{0l5 z7#L`8YkU`OSZ3Mz>O6%L++9G%?u4FqL(5Z_B@l{1FQavHPxY^f zaH$5une`s-u&nF8f5b>>)2?*`X2ux07Hz!=09ZW+3k}ijYY!1#!GR^!px-)(87#Wa zaA@*yo4?~%HglGmuX`hkG`H^IdP7)06wBya8H=#+9pWZ@0)M+%sfZWHV021&T%u#{`~^VH5IVJ@1e4`;8e4PoLfO5Ahll4|?adh`Y_yKW!>dqW z$|N&3X1-T0z|wb<=VRjT$>~z>UfD_k2q0xvx-kZAdF&3*WaE===~aCd@xn+wN_m^2 zzJ7<;>XtjGXLg%{`l1OgCNaVQBPOHq5P85{Eg<>n6i%cjMpne_GhTSXbe*MXxf z@#VK|Ik#Qae6556sYa@bxQHlO>+^6>H%C#K_$Lqw24s+G@kKZ8$)X z;O_434#g!%kszhG6faO5io3hJ2B$!AD=q~J6n87`?(lqBYyI!WzmvU;jFF7YIq&N_ z&*Px1MN&3j!0S=Jl#nRh{>xo{1QyjII=Fs`Rg|wzq@;V!*A6wks3AegsT%OwFY^0E z?Dzj`?XinknKcvOxV}0dVEVHoQ49RENtYhj11XyiS(AZ^ zet3>-{doWOR)qh9M?-1}_N$oCQY*{qKLki%-{S$!UbBuz0$gjwpCDAI> z>!0s?0?Rsu!u~HP1ttmI^~DhyZiRFeZ*aS+*iwvw?NPf-wvs-0#IgsScG;eNd$$pV zg(RBvIDe%=X8lUC`1ZoKbT@GxBExBA=jFoRbxZ~(r=!4Yqu`}8@mh0^Nv1M-NSl8A zRlKq>boF3{k(I#0G?|^A!Q{AV z_f$S+Pz&WZ>R^WUmfhPyyU(5smA?uMUd^QF0}++X79G3lDANPV%T42)CGm9b=ZD-t zCTN#NuCbSm0x0P>YTU{Imu8OY{u2@eJA#gyf(PpgL*=oG0yW`;GRh@7)^V-G?ec_) zecTyW-0k{I+HtuClhqDe)#eJu_`u)sak!XFbsMf#)1aYV*x9n&ILCznvP;q z<52PWu{Sm_}Ve>JQUr2p!` zPfd9>SFHm5EyJkcZ-LDbW=;j2aKzq@`F@G&ac!IOqilsEmS6JO`n-mxKCQ2npbG;N`;X2kS(O!O| zx4?MU=%tOyc0(=6{I}6~no+JOq%R->!0V-#`$@~h-dCiYcklXmrOaiwvP!Nc8Ggo0 zKUz18agb_S3L+(qTfqXDtg&_`j`HFMxeVtl>F2CJt^5t?H!C-vV^pXA8zqQe@6fzY z`8Qm9|Ez+Du<#SGO*tJ{ZO{Xiic?m?0R<;Zb%oTJ^8xEtvmji4zZ-9?tTyxi(s4in ztb$||=c4DZIez!@#LgcC^6SzRuHZJY46(qqhowG7f>bNv5^p6)mvk@;5& zZ9F7rf4E|$eX9+0jVv-f-EYCC+1QHc2FEds2NU93O~B7 ztX+=8L5w2AAQMFd=i76u!r zN^zEqW3s4hNSOZzz}-BiTXC{ayK8DEwEU!$A=-?P$!5SX*snqKQT9t=Ej9L;zB+%j z0|U;n+Q>84=!<9;n_nTbQ_X&s&BmDIbihFZ$|xy#IzAG78gOtuvl7iu@BVJG_10DvvYxG?}^UDZn7 z+2g;b!OZ9UK(hG?r*W%mf7UOrsuK!y1X02@(hc+%(%(A|^;}|PmT=K~uW*Id^s(ME zd#LGt5G?=%zVnsE@;A_9tlyP2fk^KzUrfB>n>fw;H-dGVKf^#3f08NIc z0(Ka4xcyZ0lWN8R&JA^te~<~ET3xHa)V_Lc00I4EFLsG-ut-!g`i@juRR`5Rsn;9@ zvlGtMbK~)H@7$?<$;H$W0%R0SkEkRn^YIm?i4;^FC0ol`u~f(NF;RSk(CDwvHO}M% zu*lpuFpzxdt3JjKdf~!02DmvnV4%VQd|S@)v!Cs*Yt-~FkbC704K>d+6|zt@F02K z3>|;SuAdK|f|%{)cesk_X3tu;9}umDMDqQhsI%rDb!DCS(cR2o_N<7%OoN&l0l|#W z!2HvZRwm8B3peCF!!V_kArO~S- z`XLbckOqOcA9!dVo3}R z%+R8-7o)>Pw#mzzGH_q1x|s)XR@MS09AlPB`0}0dRqG%xJBNw&jQl35kNZCJU86xj zgX%Q=7C2kzRJhR}K=k<&PJ;|h)5}D#b+e-+W&QvyUNF?1`H|M3_hNLSES*AdEP^rsN@g;c;afwgN6vdD{ z5_Ei6U!|(DI5?tj#nWu%bXYz;`1v@c6Pc<; zkxM9=$jScEO%lBDYOx%g_LMuo;812qj;1xijF(|0E2CYcJL7KSC&$jR3$>?_>ss9v zoF3X2LCs*FO8V|hk5=C%`ZrAeX+el`@_vsQx4!LO*kuGGik-*H$Nw@7eV^toi5^Ob4TD#Jr!__tOZ0M|vLVYs*q!1ICTqF9XzZcNh zfMvH3bPV@nwOi@zkz5yxX8`E{HOPr5OV>>b(4>g z=66g*o#JPL)$9 zcx%1;6hF3K`ymAlGc&GHJp8Gp!hbS61fV)ul zUOKCmMJwKtq-q*BxS3f)mxbVKV*V563)5dFC`otTh*GB2y9MiD`0!R-YBsDnNeYUZ z4&E>9P(t2K*x9^n54{abnlj8n0Ii7ETg1X6HhjBl209ykKIfZLnrEl#x`ZLjSwa{KDGs?F);Pk7t+e4Kuz&0)VnYB zP6aJOk`j6+g>xC1tE`aBs>)axQqzp0!If>)F_DiXIlJu{){S3{6|=XKf+p*kGg9}L z&5eY8f5;>&>(I#2v%oELsNv#E;m63V%zbs#>1apcrFW?h^x*T^S%*FHs+!NGO&P5# z3-N20VQwOvB`DJR{!;vchbb^C`Zwe%S8KlhJ3`S|K@j(_GCPPdea4N-|D(4c#Gfq%b z%+nZmy_A|SY#(679lFe30#q1*-}Nfs*nA=i%E4*ofUOcRqjI}lw!Iw69ZB!T6Rs(n zF-X;uBj1WNv}LavwYGY3@bD-3?)QYdvbl3$j_t>X9m-4pFF%f1(OznKm1v|YXU}cf0JV@D1~K5%q$pZ&72bV zVB2^;AhsMCer9`9LPq)A3%Y=ke@Q2-^$Py~7ekPcg^Z4DrqzY@dh^bg5_3eFL;2{kZ%oO=pN3e~HholHgona-u@zI&N zqlh=qdfE1TIC3hhmr^x!63mGS@QbHw!k%aT6L^AjiL}V$@K)^1trgL1@cqe2h_;m7 z+u=dzC4pTO#e8)cD_YXnHpLbe?~a8Cmoj!Q3O+Ba+&6 zWkJ{t`xXr=cI~2dSgP+}yg%^W1lm4>nPlcyUM~4d5%I(%^3iWcI~sNQesMCK@`J@x z(c>A9FOzYc=i}E~tWC5HDrp1PUyJXdhiVIyrY?Y~I*HD+kYBa3ot^rr3+Y{imE~Fs z2R$AuAnq)HAxX}rDXo0O%#uRwvtZy!r6=H3x()mJV(EEI%;24AyK(u%(}t3Q_>fl% zp!RV8Gy?mnE}3Q@c0W9Ch-`%cVBFJxNbtyytBI?NKve{%&Fdpfcz+Xx{ZLJt($sCsZBfDSq#r{@qLOPAOy#PQwj zSG|z@*}{$(vE%+cw%&zG3^_e*h}BiT|Bk6jB&AQdCW;W6`3g_7e7_ zz5CC?FS)5A0K~ud%_NFXZ7d~h?yf&vuD~dO6S(wvWI>~ED1(VeqK}_WfUsxuc;u77 zgb#ED{j&>?iO|%8^`0Y#+Z9&O;AnZzxhwv|kN5MQsS=YFI1Y2P#i2nRfu40@UHy=unf({Mj?S|G+O&Yuf|XWAN&+^`yeh|8xDxv&)@~ z!!|Fu3Aa57L?JKz=;^(rX|5m|TGQitq#36+Dv?+vf?z$7N>izs7XLSiZ(mwrtrM~( zxjXU^ktH`GVLPpXxxu(LbPVLUSBI+*C_)`C_<*T)YAUTZLZ;Dot6tPJn9Y}l+VH_1 zD9)%Zd_XHfwjE~qgG%AMS05rm;gWB})8IGRU?t!gr2(vz$&Rfse45dy`qF$ZvHb&X zyP=$K2fa5tJA|RDI5>~hRb(>te>D){2Uvq#PVyN=jSrF z^UN1GR{vydC^tMXnq;Qfs9|^3nUJoP^5ssU`x`{{tEHljaW zSR5PR4@;=>9-<5wEYqwBaC&B7cmyb=Q@3jeW{^9p3m?yKunZ~9HP;kj{OJNafWETG za}>fcmF|5=PYG5hKccNsnrp9anWz5TJbiuoui5pg`p(OHA`h^T8zo>m(HF;Rev^NI zpXJAU^jRMQA#v)>x4XI^brME@ z)-{M6{_{H6Z-=JQ+FjAyOfdvAzA@HB^NH`4FM0o;dU;p*}3*l zdP^9BhMcu?C3l3YX|?^ZR-QJL1rE%rL>dv93x>dtG24K7vzWV&))K zv4MbHyHPRnQfNalYS`=(gw1{^kUKIYl*RYWRCZnvvBh)HZZ_6IOQo; zt)>_QR|>F$;q7#T_6WO2XiajSp*Hf2wW;$~)LE)Hr;4&tD121FEFH%v`r~d-ngX&^ zvI8ORloF1Vz0e)TAZ9%0>e?B9>w!J_H?wvB8>dW_BuH__-QP6|?DzkK=@iodUjs>0 zV8ZKkD%((SHX*sSU=f4GbCzOHX}z}3qmuBiPoH}@rY$a zQouFYyZos@d51K|r3wA~uqgeHeN*(0YZrLL%4@>dERxg$Eto|6dv;5BJjVDiP36#| z#!4=IH8{AiFo^-jM=w$c31tQGKLCVl_|D$s-&HM}UN5UX2JVHG>$n2q6LN4v>f$^z zL)9nc3^?vE<*B#AAV3zd>r7Tvmy1fI0?cf+i887n@iG zt7M9|zO)-XRH_olz=q(ur5T($6GQ z*kfChzphexh6tl-1H()%ZU!O^z7FyM;n59L!oTP~=h^KyD7)-keHcKVjoDqdFKpn z@DM6!tNIo#v_Ros+Oe+d9=m^|*|BcH#1nXA)`|B{F9&Kpvf@L z*V+26>omU=n>dMS=pM^?go>>L7J^pD^z=$=~I+8Cx?v2j9j)Q z_A-L3y&Z~fcmBO6Lhfh|=alM5J=GN6_~ydG-Mol^0)Q;Iqk4u_FQo_w!<)X#UK91@=pzadysG@SC`37q^^B%|ys{Jojt?z`sl@o*=pVzDp6s*#|1xMtSF_caV# zJ<`O6weZL}2rvWQp{Yw1gFZhb<-VQoL)P^+79*>Bpa7ZUzGvX5HXCwAr(M4mJCqyAxSn9 zlcqNR21J)`mOU4lYdEOfR2daB%DLwv<$c}I9Nc?M_ooJbH#$?K>zx}Rz9Pb~%A)X@ zbJs5C__OK`Rt=ss$zo`(6ixnD&5+$pgXZkO_AY>{OSpyR>l$liA_U?e&z0o*P_Yxf z-&#rkN>VZ-A`uWWQ9o0@UG5(MbPQJ9zZ{XtyWG4yJR^MV5%rFkEZ!5a5S=l=9S>T# zBblksVa9*#BTY7Afw<+$M-2W+;)`33?-W?|&%LcIpvq7hRN7I0jFq6>f#zKl?qhJ2 zC~Cs85%y2=Mq~w<-o?rBm%@f11{^=7IAvGwZ~QW^bp6T+7N-R;OYU!(17V?B#7a#C zvgqqV%5S$3OWGE@>Yr0IlOIkCTLWMoqU{^c}Jf6PtP7ZDF!*U`mhH3llroR8USe$eA0#kVH0;%4= z_3F5m0Xr@YT?TS_0oRQ_Z+Fi)31;sFWL`&9l>QleQufU#0@EQi8kxUd|Jn8->)Fi^ zQctL^NPhb!gNAX7_AJnN+xR3MT2Bk}YfM(KC;!Tvy!T+Lw$Q;pPax zWO9mn8H~V*B+7&V;X)0vceHRCkgzWH!k>QyU@b5T&s2}&!vYkrOIk!*R2!|OE-=UN z&pmx)kU2O(cxNY&Q<_LnOmb+z!77lLk9J++;$>Sg-*1vi!CXX_3<07#iuZs+Y0{I9 zn(s3|71B+~=Mcse}WaoAl!!NCP@JhZ%Twq_*t4)3JfUHplNYhQ** zBYcakzS8*^zB%dkJ`Ma`E;^u>r~3>SsRc>+xN?YmRB{3t>s7`P#a zCzdCNR$y+CBp3ucb?KlHQ}$w&Ivup_O#!jV|A$^mh_KzWmJ4HD43U+f{s&O*joeIX zyY0i#hc6;pQEJ_krF(+K^VhJY;gXvBGO-z9;+D%9(K&2KFhR5DH5&;DU$;dtK%-V* zEok9Q4yDXD!PsWAdGn$tFBA|rvN;8)D`+AjxY5FTS(}o){OT}_(#>(9y}wToXIDgL{O*9{jSD(x!nujXmPLuHv#0E_24V*) zQOa#BH1MJWEH!**06`}t_fRi=sBxz8lIj*&=htoXGj}neU#qM=7()^F$HeNv719^K~JC3iwT6nW>r+Y~Vj{4i!>O0jP ziX#Q_l8YyyQ0p%kDu_~#IwXjZTPAQ9W2+Y+`0}KNl8FSrfCW=pX1)u){0Be=vJ;~q zOk;(k&}#Y94@QA#1!0tjG5+6)Un-t)QJ6`k0837Mb4tl-{ifiEj^b#3E1arGEoUZbQ!5hC-)xzPolw!Jgb}^ax{jL zIyCX@ySuU&LOB}}v$E&3aOUR`21LX_tiIXP6^DVkiFm}$S~t@%8LAdHpEI*NHuYFF zz$vCy;-hcnSgauXYa4P5gYH68DTLD3i5)lCH1vQt=D`;^1sNPDe$i_1mY*56xg)`J z;FTtEcvFMJlJ{1g^*;O`U~5}mLSKWF)a#h6OSbS3EGT%0d>!z+Pe$dz_5lIf9v+Mu zI@0}%@Oi@9)T8xCO_i$q&@&h%^_Uu-e)6?YHOFCg7KDm$EYQQ+&m5q3%?Vt}tmmV@ zyQxo#!p>9o+=M}A(`lVvnppmKG7$`$$V6wDK48nyPi#P>;V~|V4sio?5+n_k2q0_| zJP6%ga6P^%@E^lo!~UJVUdl~tmoZwbzJl-jfD*JoMO8rXS&`!-jTBSTv!DNK{X8@3 zhwn}nMz9o7U<|sPk^r}CKbRJI=C6$+w zj4E~Eb?>g8I5F>jdnYAox1#trbr;J96sF&S6_r>s$cy5 z$ImN^IQ*-ICMyJ}hGQxND$A0HP#c#hjAKNH_I!zg2c5)Zos1F|aL`5xCI73;ceymy zFQ;`FlCP9nr#r|*FOI@Jq>}~Pdh|L`N?M7cMh-gu&Q~-lb(Mbp+^#dtu`9bO2}J zGF)#ublk7tbb*aGzmoV9Xy>aVy9uUzy)F8%7uHkqhkt92YV#gFh#Axt4H5TWB-DdG zJqAqFOa3EhAsGl%#M2Ja5DnqlRt*W^-Q`&mESz2lZzf=I#MNyj*Q$ba4yA0c$badm zj>9drs5f&lY^5H+wnSSZJE?USC7ct9CzJdz>7x>hroDrxmpI9~wj{WI733^Q3WUxH zwydBily%mpqG+-zX=t)0!%M?~P}wxPQ}Yxh&PYJnQA~3JnZf@G8Ot&_TH@MwKV;EB z((6_=t3;zoNe|)J0o5qe?Nhi!sdw_S(K2KdJH0djJCIgMf2Rz7VsWz)X;UFuyi??$ z`q0F`&x~ak69R0~$`UQ{wHaUXYk4iVcWBH2uBe$f)OY`J zwJ_J3#E9E$W9k`XM>B%!M$a4R%nG?JQpQXP9W>v6u3?9z1uzFiip+b@PwxpBlNNl0 zmJlSs6RzF^oW0v2=HcA$Ch_kl8cVA;7!{ibL{}N_RTGw*=OR>C)|lDl2(>)28R8kP zL<)gpghrV7do^`(HI;oEX|R(Jw1sC)4DZkrC&4h+!=CM4^a`Gs2Olw#JhM4FspMJl zRra3m?`zhw`WL*X(~0ocC3BSi|h4SK@feJ8T=0-DIT8ezv+Kx1SP@&qr(xbp~l_BhZi>w zEx(dF)mLkL*NojW|Dnf%5w6XQC*wgo3pHm`nxy{u@zvbkEiC%Z>g=I3%+k8eY^F4P ze`Vx`d$WeW>tl66MBwk|8r>{(JNbwYZgg}T#R|mqHYRozZnp{s4hXM8&KNp`rd!D! zSk07JIPEk+(oLwyl?Wy`mjOU zsJQ^oT3&%a+>L0J2l>Pyo2~ZI-0CExt;#LY09#vYo0DU%C6Y;+G+#x#H^r9tOHu8_ zvD2vUKisa-bp#o-nA4A&dB*cy?o6O}cp}Bh+$k0*o67#|Iev26<|YFOK`l#zH=7ij zl0I2yXta@vqZ`=ASK?rJElgEcSJdq2KLBA+>-mya=~u(LORkLrOwDR0^p5_GO3T~& zk8HO`D9WA1P30=lveDk!KS-2p9eJM0`es?Ve>AoZCX>67rl>tub@g6yw*l>&e~T6D zS=2GNi#@WIjoY(T*oPpB`XV!SXERe7&U*5fQgWTRU%HC&0kPy%Pt93x%sA z8LY6u+#B%pk#Lygd*3;yRSH?!;cgGuV!Dyt)VmK7pCOQC{XCOz<@kxg%5x{#7-DP~{bTN(8a-66wlWYzopG7= zIn9<SqQh;(8 z^vv&th?d)Z4=nqI7vp*VIG9MaNqRp}6&Q4iEM~?k20cDR`>;)E8Kr6e3Qul&58(Kg zsx@EerAj|4+s8t&+6&fD{!}SQ96Q3E6rXm?^c#pfm@R;&85iPY#2;7E*j@EUj4eJX zX8a0>zep59dCt#}d*n%{=ojW!@+$DoMKV!YL0LXw=uN}y_#d~3V4;B?NDP)&b2{^)^JNb%c+E;MXl}zMQuGf2iq^|ha)_%3+U!oBnCe5>U->pC@u@g zP^H3n?K8AS@|%$AHIinrNlosGk|}rsjT?;MX2D6T>sL&TS{7wIjTO`^rL3;g?# zoL%9rpPs;JunQcBfiCA}^V4!kr-w@GXA|4YWP+evyqH0_rwOR_b)U)QcJ@+>)6z>x zsj0WFlV<7yP1UW9T54qkMCP+Q8y7X9QAlNdx2ORes)HN`EOYwNu#>Qc{T*?sXJ9L! z^y~5uorsCcKD~f1&^6X0YLVCJdHqvqF1>Tg!58(4(UZgJ*3=Gbe4L{BG~V{q_6;I! zGFL4%8$Eqg-8A;V(lA8N$q|q6FHt|WPW(R!RU78j+9$pX-}W57U}bGN7mn@@jeoEi z8gPR2o$2i-3~_zo&&#_i@s%CG|E&n;yh8rx(Jwqo%UkzLn3_)_0-=I&EAAV4rmmKz z!V+Y?-0R;laFP$Klo zIO2`YV&C%d3+9;hEAeSE1OiDl|2teWiy+k}{F!cnT9~ajm%fy~Oe)@B1|ulNM{n|q z=q1$DB&BY;+XVJnLrB3{+Xoy3CA&bui&qa+9)}d6tBz@lWrT`R%XJ4Pp zqVZ{}8xYd(ANz^ykhvMC2m1hnD>G8!!;A4gPC=}Ur95P9o(o%#ep|qt@Te^w;_RWf*M2P8IAKm=}wlnPOr~Q%!s3hmjMuUk|m?yA$l*L{gUBGu343ew)Z8#+izuq(!ozZ4yz6JHNi?oIT7^{ zc(nBJT70;fIAVh_Hv_c0vJmBugW#1-w)_H*J|LVq`W8jB4(l~v za-HDxLZ}VK&zJO6w5FE)?9n%YTpU|KEHN@aQH=5h^Cjh1w(%XOTY*T@hoYOpmTq}! zJHl2LdbdQ?=i3|;G!TOLeRGXpt=QE6jD7}esYfKeXkVUug2MmLSW+NxbhW_(kCaKv zR5qAQEAiwqNS~|{*FkDaLN4-_tWXluv!Z$O(bv3*seIx;fbjoOfK~n%*^5C3rt>rY zOGVk{)c)tC_ForwJhuE}XI7VI5WUl$Ex?T_eci%_&w>tK;QV=jTIGKUuJbP=ox9ZG zz?6zCov`16DJ?H6SV>Ffa``k9uE9i@!SFqEe!oJ1*h8QMle&$UAntF{@U%>@U zWHAW~zrc*RU+g|eo!~iJT$tUX#7?tiV!C-++<5s;4Zg;jMnpdr1kvIO+mU9LduL`r6$^E@V%I=ueYKfh42&E4YH0NQ=vqp^lGnHEE$5Q&`3ra5kvRU? zWHRxf?$F;2TO(h}A>ijXyv6(t08&_8(sQ4T!k{sy`s}OkH%Xd@7f*g{BOXFt>H6L8 zI5!d{pnw?!mDyUdVf;u@Tz6d4;b9M0kE}y}Uyey@{oy45TppEUq8ofi^Toa#Mf7B7 zjdF*ce+i5#M32z2dNyU@tAtI%P8ASJ1e8Ee4LVtT>;q!c-;#m5|E)g6yu|V-n{y&G z{C%kcAFS74Uk34Ei;}&1J{PQ_@6Ze7C(8B)nVCKK3Zl?zPI@c$j3nK*!QKxi*dRIe z(BB{ke30Vg52qqZeCVm;;wU9K@`mJ7!lGSX<-FEp+@@3pH=+jp&0J)gm#fP(XDQ7a zB-^NoJ@< zu^Ki+^ko2IRku&D>z<;evy;icseqfYAh$$^GoDa&rM&#pj_#Vymq{z1%u;~j-cL(9 zjLag%6|j1(VNJ0eLBEwnqoLexF!4xbK;)0iB{N7?s+5@#<#GKephfFO{L3jR6KT(WOb;Fg+wavH|LQk{?;`zQ+Kn-t z_Hel$okNJ_rh%b+ZGw*il+khW%dPNM-fs@1EZJ`R-XW}BziyOYTNG1?a49AzZ7rds z9ieQU#b1RpATVo)nfmBnciQO&m(Ykx|MnjLsjPGQ{Wm>%_xYphM(-Y&BUv68Rsjm+fW5k;eV+Lye%Z_g~1WEc#gEJi!? z8`?b@O#^9897D{NW5H%a6E&>xG63)`$-rKB%D5yKGaphXUL<2%$)`t~6rNDwkwIDPBhx2Sjgvf>Q;?x%uy;j8(MW0oj`5s2GZ`U=s84KoOY3YjBx z2e<>WI+Qeb{GewFv|neU=-bdeS8mOHZRg@ImVf;IEmz6t^F7>(pB}(wZeZ0dkAWrT ztWNcJ_jX<<3#u$b;;@MJB{VFxXy-%0`3i~-WLlxG4}zyu z#ppHreib9Jx;6pJKYteL**z>AXtngPZAMXzY}&3j?t2IGcFnYMjoDFr^LH2_Cs1qA zu0ZE+W@|@Pf`v8T%Zr@Xe-`3@__WNW@R`e+Va)Id0WZE?6rh4k?PgK$S;4dIq}&rl zxp05vTAyW1zemE`vvlbtgNzp8a*GrkFKcvrv+?6cRSWbuyVbC&%6sNy9!s>9!g&Z+ zdChr5DCFB5ie(z|4Sbd+>(CKaaU7jz;80}!AAqg) zk5n0+MQl^7V0gyfRKbU}M+Ib(>{>U<2LsX+MX@;9m8**k3#y|BC8p1aBQaU3CYJ`+tk&vCOiRMmu|ciz%0O{gK7g{F(fR zEw-ph^q<=nwGJ)RM#*iaDr=E2L}TEiXH=z@R=>2Jkiqfo_oK2TmMm_=SLD^f&lh^C zH;v*2Y!yWwdAmBo%(%bHO(hh4yU28&!fB*(Z1(Uygnp#j$jbzdTGzJ-L zH@8f8@Qi?9g#r@i}H)CSMQTxpb=yek6uK$hF(2(XRc^T8c2# z5ia}ZqQx`VbS&i=Ei;=HPWD=us|f$aShe!{Nih#NvGJffGdR|&*9QLsnaz`6*Mhp6 zu8N$kNCo&qHING9j8~6(ri{$3oR=anohkK(wciBzw2P`fxuT{*Wmz#=wF6HQ+Mx7v z9pNF`6HwbSkE*PF%i$$&2F04ih#lYWPu*sk9B)bA2HQ5y1IqdZun0WJ2H3v;NMJ43 zVIZL3Jv=(H$x;t6(3qWg_&D8`Mw_1=_pwFR78A-TXygzMDa+8ies0S9_#a>!F%uTw zTQoM!>Ni9ZEL$4^{ep()y-KeZY`>?KRcY5$C6%R9;A^8e97l&nyBpsbXlC8T&_iXR2i0)Vzm9Z8p$m$ zGd5AxNW?GG(LL0Uz7sC&20PdQJSW?_HRjExFgW=M><~8V42)vGa4(04&MK~K>T-UJ zIiiGkR4-NEI%?Ne2{3SYR;ttjN?8U~qTx$VYzIm+Lr|5!7Em7C;l28IP-X&Wbs4e;V;0UWX0ZmQc6w{a z6xOPB2<_F6#rcaykiXtqf?T=52InUNc8CW`%c7@Av~4}+I@+!ktZ1y zJG?rZk`0LT_0Ve*XgC@V25lbz`ROPq0toGNn9yaY1Dw!9)Rl|f9;?I1g zf2cA^JB%l!0l%OXTY633Ete1M_ROQpx%_&|3CFKdKWkzV-j#Gpg;m%=V9WXQ+qAKC z1%b6ro`S&q>oq$={pZ=jg)1x@F@~@S1~s|eWQ@a|@@j$M9}fv~!f4Ht6#u#!$VG-~ z8+Csx?xkong=zDarU@3$6%x=F!&Lr>zlDS1H(EN-uzAuy?>0PUT z^&_|bkOtc%`gu6i&yu!joGF;<9h(e%ic?xyG`sWV4zm{L58HpGK3=$TO!RCfv5zMZ zH7QaJtv}>0XGu4(97Uwca8~Rd^oaObJ)Z2`Ml`sm_4c%d$cE}o4{zrrHD6Kcm=|} zhvBD)!U8Onw3be*as?TW#O&8u$85#yPwRwF_nSYq=hT)S7*gmR7is4zrWL7K{>jj8 zW z;%LEec_F!EX?(x+XSXPT)gvdmhcr(@Lyl$iU8XN@cCj(19bQOw2DyuNX*TmnTAEv$ zcC$`Xns!xD-T0b&M3%lX;c87=%bMjNPAzBC4*qJS#$_d|5-uGT82T|yLc8*(w$)f} zqT-cCpjS{Mwlq4J%dbaJ_>__l6gN@dR)gy@F~zE!EG+A+9Z!Je#mtPKnp7O3K^zyA z>&WKl4AjObUWef$Mqk^%8&d78!cNDfEB!ZPuuUwSL)eA6$jz;!$K!7dh)g@h4sCb} zr3u!)C#9Te6Jq;#Q#>QFAj>^5irU4OLB?F@FznFTtA4G z8f?*fPw5R<0a6_|i8`PD7Ee@;@%`rCDUs~#1*uT#G{S4E^^;27Q3W7Gj>%-dW^-yx zV;iS(jsKg3Z@M_3JKuB`Q=t7nD!^Ruxj=xx<+-1^XtHAI+vH-xhRBXaVR?^Gcs?}0 zb8+a9Yq85t^<^-=XUFey8Z3@VfKNqrNLI=}^X~gb6REMZMuutrlHf}l-gAeMP|#&K zpcb%P(Q{1(1BJi@kQl= z^@ES(2;G2QvLmhIXYz~S4^ghIfsDUlFGiB#Tr<1RB9v#lkI=I#0%s8UC>$>V7evAP zPLB4MbH4rEduE(^Bz7*mH&mOLcYgm>3JZkrZWb0lId<2K5K9~0UJIYWYfaLJu)t#@ zlz+s!7p@4T4ME0@I%;LD3n=~0WhMY^KfaW}pc#4%PvjEvJsT^yC#H@;NAU;L+8O`q z-k=Df!czOrX+HIEasniSf=vLyIA>zr&~-ooRc_u?k*Sw}5g%sQ$<*Wz4?84?IQioP z#Rncze~$sCrGuGZ`946?ifmH5`{7l1y0R>RizcuEw-L4PU+&;hh67J7Kpys4b!!BIG$ zeDWuLzg?MeUTW#1aB{+W7DzkNI6|?jMX=uDv95)0#$Vpn^IxsBbHAv~`}-cLW4Xu`>eqqlcKb)Q6|(R+HxZ^}E}|FF(qyRSL&8b( zZM6@PdrcdYcnEG&IWd#df`ds5^e};>jeFR9*d0CzH zlhG;&h*8dpror=ZYNmA{j8Z5c|t3b_>Mwh%BO z-G?3b(p=cS+F!>sUQt4_J2z-lqyzvQX6J$_ec=9}U@E_d$L(XyquJ!rrF$iUv%GuPR3CkgyAn^@xqq1=Lei^7b4+%jP?=FcS&v^XbOI>~%P<+~c;YSfBpIpD~R0f|G0IZrM5|_VQ=|WS?3<~`1%Abp?foO7?e3Lxw zth!H18rcaYt;bL?dILF4Wpb#;J=p6-$kgsawwtZK7X*O(2QvW*0Bpn|uze!duT)O? zVqfzkuRhq4l*-#i3CY&Dvi=_JVCU%aMVDJo8lv2@l?q-XCKQ)ke5`reAtO? z{o7k<(Plk7zyBAC3+dt`Ha6$t?=uyb$C5E)NCg?3w%$8D<1vpq8?}wsZ&^~*j(6%% zxc>v(7#pK^aDoz{Li|q5LHeJ4(;wR!OsuhAxk)l9ok7(YT=QpngXkI{V42dyO{AeS z_pLnWS5pQVs*H@!`aeK?hBg}0g|uHU@#5;sadMGjU zQ|W1?m)yEx01LwM$6!4P4%3u8;z4s~Riu561VkENSP@7KUZ{K>waik`?sOOVcHH+y zOcI0nv9GV^%_xY`jT@ zmtdhn=K#Z7SCca0l+Sc;yZ&0lB8epLufjsjDG9WIwQdb#_u?2k$f|XD@RdU zlf(FPWhIbB-dX897mba-#n6r)3{h#Rmkf(0p5w^&VCN{HA5C+{yMj&gZ0Sf?aZi-q z>rBx4{ed8g7AxHM> z(QkRT{wyf*6G38m6Fper1o+w#2;E_wNAgeSy8+W))rqb)c{j1HRpXN_W%F&Iml#d^ z6AsXr?Po=K)Hk^SaXv*;+F6nS-Ow?o8K*XK8jfp(N1K)o^o&2N0xAjU7(CqN4f`oJOyx>D_hze-ZAvaWqKZHH27+( z>yR_U6U@&T2`@q(1Gj#2Rphn&ine@iJ8*Z(Jx{G^<4iw{nb8!l(2z=1xrTst6Q{(E zcH)z;#q@?^pv{=8%O=lD=Z~G4Yqik@!7#(-&vr_EcmwRs;t(#DsYw zg0-mUp4U(B*j{20{JLzAx{(zEE3hc@ijO-#g>$8ZC0LWcCB_$A46g;*hloFfvHHww z8uv8X$ebAXw=~LNi>WlV@7+BOJMMMBPdR-`SJ(PeR`#QO44qXyVm^n6GvC@8m zn*HaGR(TEPO6$JxD8Hv0d2b9O$fksD^F|S1eBO%+&nX}xXLM0@wuYYSYM~frbF{0u zTC>{DQ}Iu&yt`@b%~XC<@kYgd>fbFH)bfhA5VY{7RmUt;yB1_K1lze4v=COJ1pzo`RfD$m+lh1$hm5SYiuX z!}*14;f9L*V&-wKk84cjzO57l$mraCMoOPb2Dx}JXX0W2t~7^vPV7iV2F*ztBGL@hc-)t#_aDQT)+Qi`68?x(d2kg4ZVWcRbQFpKe7# zVKR!hwaiS`M<5^Gh@?Y;A|BQ+0wM_2E_W@ZCOVh*?h=`EO&uGl%=T4+%y$0(!uC1p zcZkMse6Q`YdVTm6l~q+S%Lbz`D@{#H6?Uw{7@q@kH`S8&NHK9$5 z%ol%bX^O0z$rcm z%+p-PHr>Z4d}f^PU{C4lQM-I@qN`+-eHgE!Wo>F(JX!_8$@=ENzgr8|5p34VAStha zmC`ybJIr;*6nu2XgskS#zAXdVJm#!0JCR~nn8D**tSUS^wGA7+4oUPIvM{AmY&Ar(=kBn_CNY@L$nQ3u(W-LjWmAa`MHf|iuxGZb2i}OiC zhzNa!qN#g>SDB5Wu&MS)90lP>AF&w9SnBZpoT=*s836soPrVwRdP7xasXo;3wkUCVC_dJd4|s*lMRLq^m2c`RD7XFPo5S<0>=plee< z!_oHCznIf*$-W`DanE_uZD~}id8jq#?(JYPlLudGSKkbNsG7kpD`t+$cCrGps56Iv zcaxOUqkDEx8H6LILB)e6vX;$m9UkaoOnsKP6_8JwQAzV2Or}bI68h3+L}*SuTe<7f zNkqL`s;^guavjRE-`M!n`yuB#pf&8wjP__|RJtVmBFUeL7QYO!G0`|gyRn2n8_aLi z@ioJ&AzinEr(}dRF|E@Q$9DIKdaIF1@?sub_&lrpjg zu&vis-MFz6+)D{>MP%o5_-Wdo635z5=ao;mpeoTWyU9bBR z-mCQQ!wmlGq>C@>iuv36N?iM!@~P6hE?=KsxPX}*e&z>Eu>sWR_YR}jtHxV`xu!b; zc@-m;v?=aq$ygrzNh?iNj=C%A$tz}fR&jcfm6=L?puI@!_&kGSHJ94=qydqrqI-o- zFDR3q>n0AXtZqVS@XI$f>Bv;EWsrj`XQey=Mg>moMhWNl!ESiYl&eXT6+{E&w7q<= z26N{eZ}NJ>Ainy7U%tX+#f_=kQy;!Ciz}C3i#$$ZWmlN5p0Ct0-PguLyu^{;Nu-Cc z&Aiw!m1tt=ABjJpMpB0b=!=nc#iaf{oSA$7W9n42mk;HjN64XAdk-|#>dn_I1GZD2 z2An^|joTemMJK2Ljm=(Oyb}*80!ot;NyO)}4jpvskJ%T~hv*bzjZ0lUPg*7^ku3rf z{ND2}->KVd=AYeaeT%~SdeK1^lk3m~4FXW1J{i_WdU!68GlUd`)xUl4hr~X}&P*Z5 z!A=}$I=h>Rn1%=OaaM;E8dZ{(z~iL!jh|AW^A5uLDBE<5c=2$f*Bp_z+4y;h`ozGg zb108y&OVGIq4fJ@KwAQvxDIgYb1yvi+e*N&$to&l`qSO{n%+s9Bu~XyMZnOmN)$dY zd31N_?JW-oy!~Z=V{S##OWH^gu-cWxL+2Q1dkY9e+8x6U7hT;P+~_d?351}_f>zHQ z5p7fbVfG}jfk`kTK%Wb|Q>-zJtlw}pcMOG8ys`YUZ>9eLJ?s9Jl5}#L7g1>iY$a^i zw;ZqnI9L5eLD+I*JJLhA{~v-&O%oQ&zcA+_DA2tDS0Q)Q89$Ea>N@Q}!`n}sVrWz& zGswfSEH!s|8Hm=3XLEi>Rv0D>e@cVIf(=Xsy0qU1;}9o&I_JaYOWj5cR5Kb+Vhe0R z){6jPJl$ozB|=Z=c#an#$TmTtj!x|FG@ks_vG^Vr(9ff6U>N-7%O<=GfCk+$1R4BZ zWIw6aaD8*k5mPDZ*Q7%GPzmk(de%k-4`*=&5e1k~nos#z6tUoV^CkK3z1?m450SSX zygW6-(bZ&M+0}bXID=oqP z$@*Pfm3|FrNW#k-DNZ2x#b1`Z+wH+`fcbx!DjHPPMR*^amju6-IJy6SW?aI&7WYx0ecq8p}3c-`+N{=RXvV`*&>M5;Le2sT_QG{^^}OiJWejr^ zMWKw{->|yr=toE0Q0?C8$M1XFy8gy`5<*DHCh@QW?e3n-gOWIHD*|ROtHf;?|P}j(^8E?4nFcVet=?r%v$o-jPpgX*onj z77h!gyE{5SCKN6=FCb%`Q%%@Jy1KC{j%M#wJNW+gk+rP22X9n5vGKvyY)&UKKDhN5 zoAv>AZ*_F>CEzk+3;=iqH% z8()F!`+m0Hy6WZ04)GUsb5fhcAVd_$g%P^hKsZItgKw~;TXQ+~ZU}FpOQHS3O9cvS zZolb+<`+B6Eho9;OX-9#HZ>Yq&>-w&ZkqxiQL+WQZa_nf9v z@FD=B03*j|T|Guz2XlBOq^Oq{Xg?V}Z%|fBiWUKmzxa^|LTv+*cm5DEa5wPK(GO`% z0>1<`8Xg+}h~1bGbbH#GHOb8P%Nm-hnO}SgoXJxE;3)Fs)^>8%LiZrh?cU)TfvDnt za7?D^>F)uiV9J7JeYkv=GgQFsg-S5rrp}$hsdZ=2dJ%#S6yPzFIs6Vh4yQ86)1pH& z2j9cIoyrb5gNcr_wXyqO@BrO>+bsD^B)M*B-7tLbz9v5 z>p^Ayx*fkiAh0@>8A{JNdhl*OHtl!$y(01M7Y4IY0peo+^XJJX^fN?x4v}vwSmyy= zP(jDGeE~o=m1D;p5;pFV!aj;Z<8v;078ZV@`t1-^HCul3E4H0ot}k`~|NkOkI|VfE z`$gvrIJKRb$5?pA!RNp;zKVe_1;z%U574&kpLpLRN9pJV6W%-h>!p+2{xd}F0dV;G zwThU&k8OR4gwF<0f-GGAO20Zrr28@qfo;egCjS8zkQu2kIyuR)#k26bMNXw}TGmE7 zdb$u9dbc7&0C!!hPq}`oOMx(JegS`|YBOIq5&TP{vaUq}Ta})04*7jF`!^3S2o98Y zzrX>WH?wCJR+`z-0J+b2`!Cyn$c5>$Xm^`Pp3e5$pm3|N}N3+SfexbdAeq!Y~AC)_u3M+o}0EY)Q0e`&OH( z3<>>(@7OGQg|S_^zf5Az5Pjn$aZclO`3#?!U(WwMVLnAN{j@jM z@6*An9n_mcaOt{|`i_L|yB_hp zNRtQn=KI|ApEleu`W_p?B4T?~)Ou99$LJYQG6#e?6i)K&!S=&$0~dINdpcabov5vP zPEPgKd`SQoET1nzSDE>m8yL;dBGR+5rx^$}ZQ)bJ7~`L2Mm8F5^6nr)22n6gHqx8C!dL8)qAqv?H7;7rRo9DbDM9FBJYf=~Jnnd5fS^JKIVh!>GG^ zmDI2Kg}*EALz_}B+Z}X+pe|;-81zF49W#vZ^&rB;xXq#A=ZLGt9i*rm^xW`9=8nct zNWMolfqjQTKqr{>3v1V3I_}wMEd_-j?yx2N7ux!O_)q5(Bk4jU^!#ztYZ-R*RcP^p zNX$CId`I7}XtVG8EZ(O)RDFH~OqdQgiXoitT)Ef%+)PD8?|BN<(5b*LYikzVNz*DL zU{Z3n5v7OdvVfwQLD4A=ma2j~bV11Yh|ihmD}Sn$hLaC=$`P){VXE$2bpXYLO@g!ab{HdqX0yKWx- z%M0cM1j@zuN9S`z_=jF;>4x@|qrr1xih&w@xPba>Vf|d(XH6yI+D|Unv4q?h4V@Aq zKEppu!oK&_%3&WYYkzp*N}zV&4qsf1$;h&|2>B5fgV1sdPa%j~>8ya}-af~^J*}bL z$uiQmFXtpPOk)lcH~8GI7`iro>mRyNcU#s4H9JEaa>5NWL;3a7(?FU;%)}K=TJ4fA zwVvw>rNumUGxV&yJNp0(2V`bPzEGc`F!TTa{3x~fTtdW@ zkq7+}1$*hRsAZm9E{j0Tm@Bicb*^`-5!e~wX9W2HJ!I=^ zdCH129it1m#tK*{vrOg0g&9pFG~f4qYsfQ8IOe_wDs213xU-^39);NCDADf=u<_O# zd1JzAl`@0kft3byPy}M8C4Df_5{?Q6D=|Cv;{F{;I;EV6($fm_teQqThDG5+0GTBe z?Hjzv@*^guHR=0kc)bdjyG+R+pB#uS8+_S9(xX)i{%%^j4Lar1k)qJ*R39nh=iXKh z8{VsPs|||0{RyqF(L5uz>{Hf_y2SPzDSnYeVl1lk6B<^ zGf;i}^G*D$mixx{FD2~fo(55I2!?&a)m&%Mtk&GucF?B<9?>&K>$D$XjI&G&WVI}c z3yNQNVsfze;9E7+H;>?P@abQl@XueY1GxVI2A+7#+x^KDAmcZF)7qdez@PrlG-z(N zM$AT_>BGZ?Nn68S4F!9(R>g|d2>f=-mwuFS`bKUy(CNq;q$@cIXP;0*k| z?fngg2q%dzj6P3xVQ%hs*uNGsZ;u|2S1^{JfBwc?Wb9OTC$b&zqxi5uTLO!&s^*yW zXZZ*LGQ_$nr^5PJ$;P{2MnFvR-OtaonJkjbPwaa{Lt3j!h9#O01plv+JF+9o7-R-+ zzCC0VR;gLF82lq`j&NZo1jJ;ZS@Uiv8WUWlw)GFdK0}{!@C_!QTCr(<31TxD06rlnQ@Zq=?)&IVV;w+aUj}0zRbpe?hI3d!@ojm-GgmA_D1E;j!5@$NH&kaDXuC;8nHUbLX_5^4Axy=>(`8Rw+Sf7_{F1{?OVe-O#a#ZmdMRh!MTa9Lc3iF z0=tM&5wwH|xv(;uJzlSri4w|iHj-hnSsE5~Zbf%GB;0Gp_|BbPZ_#okfyX|dFIB-aDIN( zy^6~E9GCREratbYLT#1xPq*fBwRa76Ee1m-5Y*o+fh4bTrD)NG+IILQz&vJ7a0sMI zyxVNlx`q*r1lJ zIWZBB3dRxV>cF5E*|F39mbe=z_z-ez45=K2izaoi9sC0e4r*zZ%#(zb3;L~q8;XK5 zgvU8bXZ}Y3kDiVQ9u`3!L#>hQ*U-~4=7yzF&X7V5gE=O70{x50sX)1HxoWc{w(ab8|DP_2syp>xOn=R5CS#&tZU!)w&v|fwt1^ zf$NXgoWx8}Tbi@APEk$Am2fGbj5UkE-vIey9{nw70l ziLRA@VQDabChOFXoRTiAsRABvN{f{wgR%80&HMu+^}p&;AKR{PfvoTihR12(TAtGg z*V!Mbspk7;99+t$r8tIc^udf8RHs8t3YL|Q^U;9nuek0x!N%OYF#68DAiXOZ< ziF+A$jbHiu^pq6ti}~N#CjQrMOiUqKNng42AFye3`-*pM*nO4{ z-or_^(b-{m2()L}*qT?=2y5e^7g?N+LN4RR3O0A%$xj{Kjx9hOqdk=?gqF!C!UG}5 z_RJDvvqNr8`ixF(AaQql7N=G)c=6(}=5znVKY&MRy@c3N;BB`=Q4Gz#(1w`(6}jQf zei5o_>I<}o1Vc)v?=|SYSDaYLy2#(>NSAzDSQKpCXxpuaf@8)}5@{l&{&>2phap?w zS$VmhkTd~g`j7#Bmfo<~42+;dbuvB;(D(Oi1Z8Bi)($1!p`F#g^y|bkEK zMU{Sl(L3rqbY!nEmhzK3cMUl4x(%HC*Qujbdh(Ps+Hp0?dwNIu!TRs`*(?b z2*w{(E^v~Q7YU#uXGeb9oxSaBdF;av%7kw#exjfnmd85Mo;=*MJe>9F$9Ev+n12%a z$q#3|i6I2vJ-E>pa zU&5e-7;?-a3m7u#o+LGXBwJ{Z$CeY_(X)CE?csLc>$gJgiDVw9lFfGPTO5X@>=oH_ zzzb%U^Uaq1)LpdF`D$m!Ln0UoGvPFM$oFx7ToDm4+=^OYZFzCrl|3p`hL2CoLK??w z-JH(DAxD>c>ydz0Gg!@Rr&@x7bT+(K1Z8gy4_I-=$+0DKQ%byH_N`Yd^n^CBsI{5Q z^yfsNol2iTI+jSHSJbTSWM~b}DjRs1(M2q8(r>3?agyC$4}M|)H*!1Fc~IET-9{ZP zJi7hY_!FG##sVa`p5a7og_hnX20{i#{-gQ+-`JPD&F}|E>Dkbh{IF;zNHc(QoIx(usN%oZ z9}T~1MR;UtwND{nrl6IPd3eJKWtvsl`idPXV-u5?F8bn$8pt)&ul`@oizXbFUFdpG zL{!|rzbJ&^fZ@Yo_o*tfi_PcdcfMylBtlx(vAox)F6^cELiGR*Cd+S+i|JXJ_UwrA z0Dbec0+ZIYO~88g>tdkD7fRbeR7_|NZbZEy1csAB`1QU(atWsE`NYB3OZ)Ey{$cdI z3qNx&J;cOIy^Uz5f z0DVU~W>%CV1J6=Wf*Gb+2?gU@85Uv_^?KnIqc10Jhu^;w1Q+rP*ZFfuV5RVq^>)_! z#=ph4Z=y&Y4!)O^0P}_Bg8=SC-|EQv!CUF?qunGD{kr<^P()-+{9p%b>3e@ZDnS?7Sfyd;^S zZ5VHTX|Qvi^fgy59K9gK%krT7q9aoAh7^W3bhYNrQa^l}b&2k9Q7~5rP^i;hE&pWO zS0pMezc-5WfcM>jL$VhD#ER^Hk=_bHRtu8`6vN}uVM9a*teXbbl+C2U;ckz=1{G#g zAb=QxKN7=@JrbJN51*HN{Yl9qMk7jCM}6)@#E+(MWyDi#HVH2d;FV$JDWH$rNdU`^ z8i>Rp(G!HEEUA>@8A7-u4}>rUFPdBDKX6V3o^rgywLQL-i0LODJhGhOmsSCA+VQxY z_XMzgd_$aWDDq&Yy+&-RssxHD$zCBjyJlN)L~KFub_y24sX^YEz?6^G(AKG!MBz73JX!A4o9hJ4sL)do0gMo>} zkLwp6WE#8Lrcb%US@uuAbT_f?@VR%)#U;lF%n!DAd)zPvX$hE!YGFy*R;Ijl-XsQhxKSe+}Tv3^`kk1fDvsx$At&k z`SMK0vVF@;1XIeY48^R1x;!-uqoRa-kSyP(0!4Xd#x|x=Xl~ztCoW6mUT#+!oYfv> z)C)D2z_?fkr&=^KH#5>oL;3!1!Oe{tug{N=SNX5x2uX*Z%RTd&702-BQJpES)hm=t z?>5LUuqFF&lU+)VV-zKw56KDfC<+e5Qj(NUWog3K7PAP<7~yoqDMjP&Km71Q&t(^w zG-ot*dF>$A8#Ie*gIVrD$PP6M;xoN`gEIZBuc&s2;<=z1hq}tLt(u=@L&>VE)gYEN z$#i7(Ij!;}g`_e9Y&RxpdC|FB$0c&Uuh&PKdXTP2QGUVoLhwhffY55 zVDybZ%TFZ3Xv@NF+)as|d_#66+<{*0q{uhLE8t*}EnJ5(;-!gU4v&X@J9i7<3k0cs z3>U-jjZ`?RL*b~%=dP$CP*oj~{B^WXKOLh32=fIcKN|7M^zS#q=|~GOrDa9OAiXLc z4?d5puH|Az9)zqg=ux(+3zAOinWq$M#ao`U=#Jwe9}d4FkDZMmON*zZw2?TLOgpTq zY(_AS3={h6_8Q7;Yx3ECv_B;|?a|PB|6B$cR$>hbR)2KCPeEW6ayU_NVmx*#!eCU4 zQVbR#k(Wcl?~w-tc_R}133kFza*Go_8Vl5cy6!u(v~CWmRH#BP`tJgCHuT@W7+9y| zcpl|gMidVKTA4DYS*qDIH6U!P75L#y7#PNgV-`o@HWCIviuKq(V~?Pm$TG9)NHZIS z<&jbwB)&?gm+qhWIhXE)>ZHz@{nkn-7gFh%_+4A#cC{J!;MpHDJ#!(WX-l7yvg35s z)g9Q6kc~6=T5_iiThKY9ZR_b3AoYa-b%@+R|MmYhZd34-Qm8gPDU-B^`F~ zU+tN2zU5hLPxRmTGx_RRs7+JN;gZNAHLs${>Q?D~uOc)i8@fS%f0rSZia>3xkB?t8 znv%*@M2W4}9AHUVF0~k4gE#e!VU7psSRgX0EtadaW{gW_|E`J0z19{|TnB)leEBhN zL`xvSHZ$KA#=Gw;@|^TkW(ug4z*3t12%!KtSwAml1=mP2v`RRE2q6s2hj_i++kEUZlbeWMt< zd>1p1nyaVS%HXSK^(iD8XvBAF#Cdhs_DsP0h{#`Gb zWONB(A)GAYvVpC7AW)sR40}k&pQ35KwpMWcSz7tsJ)BOkRs76R%u!Hkz3_bFH@7WC z4j91QWOq)gIZrMm0YxN(#MI-Uy}&gy*{ns{o&9$x7K^nXnf&LKxFKKFX6RL06%5PD z!sxhBSDKwalo79K@ox?fY>?`A=^c~Qj>#TS{kVxK+anQG((9*wD6IH>-4c_L_F$(a zqWrUB%Bzg7DkpoWP;m}Lqf8$q6J6KI{#uOJ;nA@$bA`u|=-aLP#9?64mjfRAxblqb zrZN?+9G|NCS@gQfvACLf&;j?E2u8DN1z|Hkn70-8_J8JMP3KFSR(HPzZkYgcZ3S)3 zd}qez$~@@Ma@eOUcsZ3TTuPEzsw z$XY&fnQ6U~qZ;xZ@JX3=@`DotGV3a6o!^+Dp54;omJ#~q z&8|;!kRK1WnHf6sq#d(7i<;0ym9aFItpxExb=lIi;VwHv(JaQ_Ds5A3)w!y4a;rAq z`3LZPI)lwHtjsx{M|Yd!wi2E5B^&iBhym|GDs94qDe&ho=Xo>NREw3H=DAf2rhwq4 zZO}0;nL{?Slp=r(Imdtcr2%m;JiqxWOqdQO&W=TvHu>Fgn}B;m*yZf7Vem3_{w|R( zs{xdj9-!WFXY^?%Uq4)%izEbsy{zVu;{yIvA;qld+bPbVrN%~@qfuxt=bGP69G1=^ zr4n8m&+Njmuwb71a5p-ZQ|{cTq8L}XZ>J-?>9Vipw(_~gijL0O*`0|QMME=29Mxq7 zHyudWIkAzuG(FeyGJuw+UiuMMzjMAG&|JA+sA8^kKtk~WcmHAz&&{FPA_R6EjOL+tYn{-YCNrwU21 z4UZKYF~;$q!@wQdhUeFql5$9utn0_-p*&2%jF~f`MBY_LriJ4qP+5_!77^Kb+O!rw zETOEZ8QBVX!Cl$#_+Lsce&KIqS^}g=aPcV9k>uyE9aX#U#2?#k;z3L;ed zDO$m0z)E<;zF-AFZ!(WgzWB+*%i{$Ze4dS%=LTFO%$%!pvte`lb;FNHd{jj(RGD&# z<{e9<_tjtybNq5X+}h$!d)czTQevT5BlKolE#VK}%n!l##>2lpkNb*F=~Z~t{z;t^ zd*0)$nON5hkg+GofQk8FK{DM08q(G}F1@8@*3TK|Q$D(v{R zk%a5(B#p#FcsHY=2Jmcje3kg2^>TA{%pEqk!H8bL3W@ND6nh7RLW**kl9uGQ#4=Ew zeZA=@M2Ou!3x;JL3h!_1WLfOif77`YII_jc+uGZ9cKxn#hS9X%@5f3x;bF8w!CauFk7<0ONeBeSpd1B#*d_2 z=pUD}Wb>bLQ|4WEO}|(*8fsZo=<_`4^|N+bv*~xD<0`pn2PKl}FLna-bXqL%6X64Jx>~K< zgPg9>!a6kkp5U5lxRs4DGtO%;Tj>+<4&opVl(frF#Jf5$I)}Frva~ zQub%9Q^CepREaO}+5Al2IutDMU}i~8r$0e(BmV%m`2z`I1HRC^8#)dZiHVG$neCGqi)EbLtCMZ!!~#$sbq=W0-_m*ktUJ> zfdCW8x4n7$X1M(;>Yqpsj^fE;8hbw5xO#ULqUTOkn1Z^Qfza4lNPh1NJk@uj25J~> z#FE0|Id3p-1&MUTj>0!A2h&(B^$YRsV&o9nt0=4Du^p_M?h~l9w zm=UkZo~WSFadX-fUk|)-d(*_-^>cBGJxpD_V};?+oUHIHY6a@91Y6Ke-X$QzNcdKWZvCcVgoC>23HD zgR!?mfUmIK$d2dcs&y#-wyy_^!$&&M`|*A2OO(*&@G(3`aB~SzIk&RVI13n@+)k#9 z6?yi~A%P6c;tf*1Cgf|_Kg5d}a|>;A%u}L$fc$Bz_P8g|E3uSsO2s=4(9`?f11*|v zoi-!*kF-PczmYn!LI~&Py1w}p?3U60<-@t?;Vv}P3QuQzzo(|Q`iOUaoAJ@d+Sl8P z8gY{f0)+{OY*F<0b&~l$y>xI|ABg+|6m}oN(_z75)1QBOW2t`bLsBiw6Wj4 zea0TA!``-+d_(trNu+l<^20Vp70jQS0^f=nd^m0yj^* zZ=kGORbqS9d#qar6%-`-8OI&?-hHy?_r>5T%>VO_grF|bIMA$>h+3{byR@J{MF}$n z=DT17Ha=)#4Az=?g$YDy52pZBW6iQ=2EeSE zfXx{uxa6GxO7#T4#73mVbJ0J_AjvmB2u+>)bv{86Ixw)J{{8`z$a#nuSDW8`AEY97 z;Ge#Phk_LVG9wi5Q1!71EFXQDFTQjHc5#(R%K|hmr?el=pO{Ff~cQkg6NQsP67?Kd)ZVzx9j#Xjc0x1`~nM8nsT9e zsq!-u3P!hxMGl?YJi1Ht{4cJDLxz^yAOu%tKa!B-R-r4Fd2#si<^P_3G^l`Rerrns z)_lBh?fo%@;&5u#i#hHQ+YT^KMtpPFJ)1!%54Xa-{2t(Q6O$DUexHEPy=u0W=zAM4 zC!;^776$@6DK`Uscley^YW@KnoMEM4)I!PrY(*)ry?Ae;l|CBm-Y@z{9&fPy17LOK z>R<+c@QQtV4Ioy%DBK|G-oFn4i{H?tPKxzFr_ZILoKUWPz^~AIzxut3=qK-&0j~1r zq8tIu4rPg#TxElE zze~x1uyUkAlP(jZryT?$_QR>gH05N17Z5*^C`QXrKFU|Vz1?PXipOgpk)Jmv@@nhA97hm;{yIL2Yk!8x0Jqf8tDP=N<~OET=1BN6Mu1f6qcyx>Syyf?OK*<=-(GzSC$e`uyXA}am)hNKt z8c+VP!5n)LA$UrW(!SH$6{xeeV$~`hoQ2=dR*>A67;j199;+!+f^FAM0QV1IaxLdo z=~}#SapATr=Eme!fD&eSG?L27DE|+D7t5Rl${aG=Fw2YVt41%}uE?cmg`x-ijEr*UE*H_hbx7## zP#Am!O$)y$IO@;9CTXRG5SIT&MgyhHqyZnSbx8;A$!3TgxR`FPC&vy2k_+8SSqs_G znQgpsC6Lr;OJMR< zVj=sG<1;jfRynoPxlq|AN4u?xw5>dUIT7GEWN4do`I3fjIq3cww<^L$5!<((>2CpO zCR^Pkk}~gB;nDSs7ZljRb&83^K7_5On&;e>vO+Z)eGK1v1+yUB{8;`z&)$FhOR#SE znQgN1{m=Geh;!E3FSfz2dJJM0c=1!SnP09TL_7UvM0C^G;uVyaj{g8qC>=#)QrB`I z$y2i zZgVYaT^=Ge=tRK;v}2G%3&7u}(Iw~VC}&IAfhmfeR0sTS(~IGHr@vz>>&+^BbV7Zc zhM$$88;EY?pZ#gtZw*>GUTW?}`CRI&!N8D6+#xmjAL)-u&{`F7T=dCLE^mZp>J!Od zxq`prj}Mc`Rnp!^lXBg#HrdRIQRuaUS^YUhc{FtK_E>-x3IML}b)RmYWy*2Js z$nSXFvyCAbG;M*ryME0z&DI|It=URVXM!0ii+%>0$O}g^gcbPBjCIQSKm^@V@{wRx zk-|}>Y4`*FbUc@{lRKdZLLMHw?xA-SW>5oVijYMZkdj{Iv_(A(6n3RSe_OdmAMDV! ztzX#hA#FXIa-a2Cn;FioH0qI|`cs89wY)Rn$ArBIHyv1}3O5Y%k`?U`kmUX82K z)chY{?j|W=`zafW=6DXA(20+PdtB*ZbAdVjn1j2>u!C;Y%NJgQLatiy-N zLN*f_f33J6MM7nppT5wN6f{qus;S$baHJO$t$*@a`q7ST2A#zTljrF@o`6+NjP8t* z+>tqej6Avh!2(Zxv4a07TX$DRQvvsv=igzTB5S?zLZQ z^;M^c#H^`#Q@unFu5sN&k%ECoagxIF9y5XtNLTis9H}T9%7JAuvOmxrTySQPNK=>5xFbx!&dV;>jy6Me{AJ1PkEXOJ# zV@f~u6tQ0tS)f=<8KIwU6`q!T6jB0U((|$2FJqp!57?K(I9iaO6-O>>6ZL=VVCd6$ zo^s^K%c~98c|%S@*-Y=5O{`~NN~+7X4T3|S5eT%Byz9H-FP)?!*|xIk6Y!I34FNWO zMrQ|qJk(&O-;E}5U4p_gjt#8J6g;xpHZ+w_tRz9JcC5BG5LQ}AJ%{$LE7&}Sct!iL zimw*LSL-9iLR4rI~dKHnm(0Zd)lEDS3C6N?+dX;}Z&=|3}qZMzz(oQMY#frNZic|bKdB5+BamLBtUq-?L9efBy>k)et`TAh1m-MKqh4baxS=XF0rVclHJ zu0nS>!)$`q6_y!=`h*hL!8l9p@FA>Nq94$7_j=N5jfbVaBfpCZ^1EOx+43)SyNrtD z83$do+)vYUmH#C;|8Pk9Rd;D&K`@U~_g~rvpO{k<0x9Wm(3&VDH|3kLRE})p+Wpge zqvvky?%MD5Jzhd%RVTk(Fi9)rWuPz1%$%V^u_ zWLIA6poxXBOT^7{*iZiGM)^0L5yKExHvCL|;1oMuC70+znoHMBr6$QouyMiP59&*c zTpb?Va(}TE`$%Ll;HpS>hFYlXCK5&-NLepsiv)U|4mBFe z*T}j*btc$flbD~IyR$Q#n^cC!;%BBorqJq|*Zp+T#q7@Q9>OPA_A|sp$3YnAaw@Zr z?Oap0WHg1qQo0xxXq}pNN?BPgC(Mi(l|Qnib1`wu`7P?Va&hn4MV@ulf~OKbIg!N%JRPV&K- zIKvEggcTt&oTl+B;ENP)Ab}L}_YXKa*VGg?&Wp)a5kD^k^@if$RqUYl4Qrr8YZPOa z<>E3%h?ZM_%{%!o8*N=hE-v_oQNjvb!vuJIEp3I7X?NA%*N3N^7Vu5}`|Jk080Zpt z8Q*0SwNbV8<45;&1}+Mbbb-Z*2mB6y8D9C`2|PABeFXF@$5e{{=GucRX4bVpj7zm1 z+S$Q+Ht6H5M)1bTNhPFG{{hfmxP#B8m{=Fp`GD1lC|!R4h+7eiI26%8X|f1bhuV7ggueg884e+644?56 z$Z0~8@07?IGG7|$*M<2BsAQwFWCv(UuW^$x?cQl&fx^pyG*`nZ;q{i>T4yAVc8WcU zK3*q=KZcs0tx}`rl>yjyiMVFH= zHH_w9-F~31vpCLsrbLyeb!AONn**(!2Cjz#&_P1sq~OK8>F5x)Xg#y|LWGwk_#zI9*D9&wY?ulE@taD2$-}N z0WEG~xsI_xwR^xSpQgY~+N;m@)x}Uf+S5bSVNxhDRT8#0)XY6@3bET_JVAALwc0x_nE;ic~}RAnH4o!bTYq<=Q5Db*Qr2 zb>=-ddbXtaEKY=r7j5P)T~HEiULbm2ZE2}qhxXSm@zX~u)1JoPhxe@2(z2{dDP(PE z!JoKL;YeCiYv8@V(PnqVit4h=P4T@Zuur_m#7ta0b~CXx_f#VR8$0~vD_2yBK0|_7 z8qd-T0iQs;RNE;9u}F`xmIYBY$pr#a-o!1rJ;Pli)=2v3M*?bu>>THfkeW%;ubZ@- zBKj%Il4N=n#)ghY=t(Hl;2`XPlC;bN;nyraNLX?15F*-tfGLZBWTfg*#vi(vkP98^ z+=L6BJ9EBf$;76>wPNLgTy=CCwJcRM`d2;TXbVUr9BmUU-JWg;r!oKmTg;6^J}0W; z!WFQse8SnzYdjf$${opzabCP`K~r8E%EhI7!_4;#m0Y0(1+=b#n68}0{XO$~^&=0E zNm_ao?vIHWrr(jMpG*&6hGdk6@B)HFdVBIqp=V=E7!kiUig9}1mrt-h2rHI5*mmyn zff|#as3Ho6VmIA)^z$)wmd9fTmdOsE?v_X{+ivD&(m1GYiA0f8OZ94s?*4ta9ls4h zER6|)L-{#=2-A;UYUI8mu`;_BYjk#!&@m!QnDqBE?RdmJtvh=axJX>1hf+tzrlw_8 zBK4H-(tp+7h>1_Hoq|}A{JG@~aJf7UA5KL0PAPpxePWIy>r3<;iJn3g7=+>b{_8k6 z{PwtlQ-)|6U^q9% z8UlF;$Fl7)jlXL%|K+nnEc448Z1jH{o)aPpdq`p`VlpDn)b_w9N*aCnl!$x zujaisocdMNbFsj9d_?Q9zPFXquIG(F?>k?lM`mwz!ttI7S0 zXS?XH$C0?R!uL@|Obh}8q7J8lX4Cn>=1Iwse^fQ1UFdes7$9+Rb%0Ply8%Qc1D+GB zoe-oDjDB5Im>UVD4)vRK=*^oM2D0*@q{-9Q=QmebIhbdvtH1S7ZW*hW>DTz|5SDAQ z$!2z*X+59cCti0wn?j=Mt3(0*cKx<1UM3C(s>WODzz(f+jVRs-Ui3G%zt?LppT8%wiSzK zifzQ}AJs(qiT^*@n6_b{A-~jpyR5s@eRy!7wv@8~|N7VSsoKgQ*Ij{2#Q+-I03v4G z^V*J!5D%p%;-#A~tST>f5w9CpnF$qFmGbCij1QN&D3s3vl?E4ZLXU`RRjyGvfGB)( z2fK0fju*UG@ncZ3G61UkYnRIwhksw8{LXe@h-6>@Qg4_D?VK#^Th3#&36yt%$=3cl5x@x?5+43JKzbVV8-Ya~Ll@HuoAwn{-3U*O2IfHn-u z2|V_9M4`b%Or=I!4w7`xX}Wx_x9z9C$D#gYz+F3S`w;%G|2r>IFcu>w@UZbU*oqh1 z+4({Z7w$W2N&%?qA zn_RDbXR53^_(2gF+uIi9&>T)hslpRT1-*hQEl;PL#MQq6{Gmddwy7kc3_e=bf#70^wAWews^f_WM*Ky~~ z?EpFko|ci`mh%O)JzTL&R`CVxxuA|2Y_`P{lig?3BwgYf?BMMfzV+JX@`^_fdF>>=Iu~E0@bFYM@2wK{04T^sa!xG&-e_s%gUzry zqgGI}a!`{FKZ5~8(Njk0G91!!DBD>QFAbsJBshG;duMvC8dm7E7+l=>FS=K#GsX~! zL&}ikzjYuc+Bi+p2WHYrGpEsutb!Zvcs#sl#5B3MSHUmg*UYCX5@I{2 z$}MpC^XqpM^+7@6xXdowt8yG!_HSypHHqPxz% zsVHs|>8^(~mE3Z1w5@RP*WG1F^YVNju{B^PTTJ6{*g;GKX!rBI7{r5g86?Px`dgex zH7q?~q?M&CyA&1asVE2WS)O_HKB>}k3E6aGVdLf9pYx2B9j>*JuI zf}XLqv5zG?;os#CA8eLdS~$}4rjb5bW_D#mB8ZGUbGidYRWL3IC(7R`U&Pj_7We%; z*6lNMKTrQs7N)4j?b8Yw*ll=}3m=#of#E&!Vv@cotMfvpqkwwzWzYB3VP9jnUzB?2 ze3|b8lAe+7FoLIUP6G;3vQ#dZ9h+WzL<;O0A1780Ech@L)iNxWIDl6MuM4YU=euM70gj!@|A;#^8XU@<^QNB308cAQmQXZ*rUEVK zAQ5H*o+wQQaLhNIdO+1mg$!PG9CY1c#T0b>7$-Cu=ViQoisOZPDCaK^uk4SsjocZ*V z5f1)|P8sVo5F7vwD(l&T5SOiInlUBKdovB}=hQ9XzdA?Vhl^rrkt8@uxn=+B0K=UX zUwk5{Gi~IQ<+j(I-vL>^iXWEJ23vR={QQ}@xS_N*!1$sIH#o>)xGo zU^Op-uo>5Plglc3V!~QfRMTYexJ)*ffJV0PxssGBXcV;AcEY(8!KRU$(j7#<++*v7 zXB%&urYUk2gYtK|&xX}=By{An9Q26+yXgd4hITqkhHue}#l^b9-Fp`~Z$^FSn@NjY zoTm22G0o@QT?>oi%n%rkYY{#Dkk$HmO+kUL%$^qF-U9-Y7o5)XX+U~1CJ6;~Cnl$v zruh{=)rpt^$)X`;ECABjatoEqvrl91C=vpm3pi;OI$>$6Q6%;b`_p2Ocy^N!T^5jX z7MEapD~o8f+c09fwN7a8W8a(aO>13oPIKUdaC~EbQ?$`v=Gn@MECXb6H}|W|8Zi>5R?8e`%m1 z1c0b{20{V>0JgeDZV~!W>;(?9iJ?RAE&x6Y|H}r8vMr7Nm<**c+FL={-MEhhf~IO_~sX3)L^bE&F1EMok@-?pmMv@LXN6uO=#Fs~wau^*Z({ zUlxgtN+HcHps^`dMqp|`clEb0(=y55xJI5IhuIa;?Hmf;pKz9DI7oFKDeur_w5B{P z(2!&dQ`5Lt1PKyRy&O|~k)O48xCRA+h>#cb*NNPewklbX{6#YK`5T(jXlGA-(u6^R z+A?1<>70ZkpLH*+QihqqknB@s6wQHh?HqqgB(|%MCsIVA;UiX#c^r^E-lM$(H6#-g z6A1bcE>u8ezMuaqGgc6k$}6+{_RDru*Jut6sEW~3Ej^6GieV@%&TZe3ELz&QAYCii zX`KJr<-&pJ*2)a3^)2%|XJ`gK@*XrS!`r>D`SNG#T~esJS6XSZF-Fwj#8+Slq=Q=q z!iK7W$B+yMXfeKp88T*HRW)n(hbYS)^oL13uqsunMXUGDL&d5Mx^SaSelShDx2a@* zDes+`4+H6J0XQ-Td;_vL6bxP?)epmW!Oc3$g#wO6g8a5W{gmFa^D?GgE7MXu7X$|@ z`R_YUZ5+}r5H2wLNJCuZzUd4SkQIx-+Ee^+UL8Hblhw`flO50!F8lXGfz7;YDy^~3 zR)a;2qqWHKYQhN+kG&J%+lm%+b+YeWk4b?-$NRoY+1=@*n#2M^B z2dW5dgCAp<7t9a-zFOyJ@IQFgOav|2zA0yNKdz3?9uvEwxA;j^*InfXz zKBmRh!zM#v%@Yy$=^B%74!K1n0@bLRc&{G#jH2IOot60+2Q6pb>dqseeTv?e8J>L@ z>mYA5dzn!Sk7;D`mWh$++yP$Rkv$vuT?mCGBIA^-ws7#nGG-m?jyn6-=QWeAk}wai zY!=V*2=tsHR##$}QAohK2u*H8MJ~P-Mo~=sQvnGt%M}tjSJ%j*1x>s1@;ETGG1;Sz za>TgCeom}oME7GQC~_G2H0b;FE@{QE`NPWjKaojb7<~fAV8X0*(_`85;+zPlYo|&E zhcu6F84@Avyb#3@W68zLvgaubZD%fn%_I}1bdE*Z{UMmcWSX4u)#{)pO-(#%o^QTxo3&DneokR`{Wz@K~s~7Zl^}%#i%l`$<}yQ(|~G!3#yR*c-5s$w*$E_EYlM&Jwnt}15?3LD+cvwp|aUv{V z0oUTNLnoO-*=%~JqZ(#saB0Zmo-&{}H~4`5lt^U_v$@4Ekeqj-@hQ+UWQ2$09RBAA z*}+N%;Iy&v+4b0KP}-vH7m?pYu3bs_2dmE=W&5&WVs$Ceq;vQ=KN*8VYhyP_57pgN zw8$o_DsukPrvHp!y>dk%bi&|a${=(->NjeO?P5-pvGGuR_*UWUhC6uZn0rfO0hE>`2E9p0w}b)sIuPf%xK#^h1=@#N1^Bt{ltg#zoX9c&s*K~!A<*Ke zw$wD1e_?_I$p;q5V$=vgo|Q+%jA;+DI&w?;9g5z>ppIjT&xZ;D7Wwu!c2sWS3L_#R zYu=0*BvsI=$s23IRHVi%T|L_H#V|xpU7F>D_P9TDpXAp_2=Z|Z6fnLzbU3`7?2QUS zWy6+5Qa&^tvXaoAU$B*X-Ljb-Wqr416kD7qSMuDCo=D62AhJcc&~yi4LH+#C96cQj zSCHjuqS;u^ChU+*Mu*>#Y!DCJ9icFJb8J2(Ng)4qyrK6lGt>>rAQZ|f@AGLpUGs$& zaLrUK84Z12;Dynra-0jeCQmA1qBmV}8NG}XE!JlL4o_$7_J`24Kbjb|4sg6>RlUnx zKro39!V{Al`5!XaKErEuYjj6SB;R5&vB5RV6R3Z|C+^xYuye)YuTJN*WmOo)muzWu zutm$E)-7u3euPzD z&Ymit=H=W4h*3212!F5v{8S%i^j6KH9lBCAkC_f8w6~B|c=JghdNz19Hg7USPvR2D zSanbB=(k>I@R@UVJAl{DRV0-@*v!@P7<$@JXCqLiF@v)^cIJC^adkmv{IZ$v`bk6P zgKiq}Z5?l=Z6x3CIANppz{ayQ3E9E5G#C>nZGH_qEOE`2G6il0lcCbW*)fy!(Gcr} zq7{X0>a6Ic(;cxBhINt39#*j-^q?Yw3C_5iO&0{OMAS`B1U1->c}T%x8kyJ&o>|C7 z1Z@S81$=}VW|?KG?yt=xBnf6^iy}SS!v%Z#?Y{p`go#pG@_*$|sZd{%YQR~Mwgzy} z3k=hx*A$1Ob@`e!fZU!IMYF96K`I^)uu3E8Wl7*9)GD#2#e!f|xxf6@yV88Zz8l?S zi(XoG!l~luLFYv2TIAFr&{&9j*j8og5I%NU&-OpSysPj;8SwE#T97wna7wk5Ef(sG zuyiegWUZ?{J1&I~6fY@NVdU|bY!`d(^o&GJ@neBo0h!fPgBxdI{#jc}IQ3pU!_*mI zK0E!aV*Lx7HpEI%z{Cssnl3x`&si%rpTv#n%YJz39e)M`lOfuk7N9sD1bN_u-5gtL zd$;5?zSfUbOCF`SW^&vYUSU6lMGnbSQwxrxpqeuN5XHqal{!!NbFsz@)jo1PA8ogaq{O62C%a zKdR~5ul=!IzMoZj#rFkHmFAK)h|EeVYoMyL3CM4eS`^XLV}f^SSrs11S9amW&0eJ+ z#?~)w{N=kd==Q~9gsl0aDgvIjh_#6}Rq7Uj5u}R#H!VzA9J&x)h+JW+?7S-5UD8@5 z$JsRyUy(xC(9`Wdz$PZg^}OKck}K;ILH|%*FB=cEO6}-1XG>9Y?e(pPvdRsS-S+N& zLM5bY;^(k;CU3)kr+<-4T0(AXY{~;sBoOzlNR?QoU8NIC&iF{R0}8e8)54Y2Lh;;Gv^kz! zo?AWpr2V?*z6m?DRxaPHPP55It428y68vpThNKo^1S6T`O*GOksQU~fb8+-f|4I^{ zG{e3ba66ACYo(eAAAX0IASI=$2F>!B)Bwxy;|DR%s+1A0_NK!^5}P*IGg!3{Q^f^^_qFY!!c!GMhFqD%pkfRi7C0eC4cV(ZC%DJYjbWAtpm_yJ>34 zSSgIW6F>-;f`1*2wnyu9=?w+HWN;(2vbHn3d*0%j8AT zK2L3Zt3aSK%A`m;*<&(7XA-hkMQdJzQOaC4?5h4d5HEn;Q(gKKJ=@f6H*vLUXpVKq zXFc=mNhG-t0}@6ii(pL|d_tbTWjrV`4L7q=fj#MGyxmi8LmZWI&OBObCJy$wG9Y6| zm8h@EM*M{=VFd&jRpw%<-P$(v%f;@(-(0MiisIk^341hD&az*3R^G?_9JD}+re>u|#F=5&IU#Rv+Wnm6)!rt6zJ!faPO%TZy| z);x22E4cGp#h$ha+WMFe3;uf6--M!P8K}xncoU5Cv*twPcgs%DtZ5vcZJicw|L}QC zygNQsY7TnTpLelKTI*z{u$>V;N4I&1w)qNzs$^vByLwYl0YA6YB4%}j9rZJEy!cM* zZSBO~vg`-Wu)OHF2u^qhMSR-BoC`f$+oSl$G38+AUC*q>mz54aa-KPkV1*qvWY>ubAIT~W$9J+ z8&1l`E;>I%DV>IsBghR9%fPINpOuqhTRO+{<3ZB!A`6xLznv<{QeQDQ5HF|pl(il{ zrR^|A9!sCgh3x&M+b9j?6YobH_7}MyD#jR6Hb^=}M!60cE4m-#UhzgvS)P7(2?_kU zP@1wA?}1WT0B+ydll(u4)Umq%Lkue_{Uj1!fYUid2PPXG{)`wzl7w8_6I6psmna_~ z-pGJQIE)033RAKJ`-Ee0mry81x^E+85y!J`6Z*o*O0R`jlyy{#tK5*_lpG=~Ej!Cc z2|_Rp3fSdHg~^gRU3ezxT_jD#ait+&1MUhE2U#oW%5DD4_fR-?(_-(<1(pBl2u@Up zC(I_?LQW0dfz8P_`DkMBeI7#RfS@$?1(}IBadMZUD)8G)(2UTNrF8N@fW9wGhbLVtm!%rHg+ zSHO$H75kzj&K>UTvN2Npnzhw{D+l6m#Oq@{ot8t#B!vLYHMNDV5@i@rH}L!h2Ia2_ zN@MUk3N(E}xMlFhL#Z|TYOiE4H4%gCVR%RZGXZ-SDSW)eEGqxrcUW)%4-++n-Bkvi z5bCtcq})NXwG(fpI?*wR`yTk0&Yn-TNzM(4FV4Q=Xp6s=7Bw=ImwCHfocz77d}x$c zlQ5FUo{0WD#|Jfve21d7Hm7&+?s2sU2}8KGTK$;+WX7R?{P2i*KPa6DL2Gws_nl1M zU36>AIU*eu*31dlfrtl)8yj7}fMG59fR^^{Q1Vym!)F<+D4NKac#p;&=pfg^;$G*! zly{d@qrpw3@(EK8-$)ZtF6sZDr`!+bET@ea#OhI6e7?WyPZNw6|2;_d?*A|T79*-K zvWQu;_x5EYKO`W`cQ;CB!5BwW^GmQ?!y}y}Ds3s1tS`X#ulT{1&;flmbYDZ#R|Y-? zUred{?QT#85&Wj@JEt}Cbtb^Ckmp{R1)w~nmHg=HbHk{V z2vm9;@?Plce=EeiZxMzs#*tz}Jv;(XbZt!Z(pM!&CBFD{0|==ljZnjGCAa6U-?Irs30iN`93zHM3_9D~x$A!9dUd4{)Y2OHi*# z5x)PjtHgU9mm@qw7n^u4bmUjNHQG?AUT;@$5keOVHy#uvH8FrlVSpnO!UU$R;P}WY zv@|ZT5)X@)Yu%90Dz6`CA`1E5ntq`TA%F2m=B$Q-^~>@BZC0e3SMtE}`n+!;YLVjQ+vRN3 zN3OC1K&%Mi|B(+sWV(v~Har^sFA-u?2=4{vc+PmEV`y86+DJnDA}A-*lJ{yg$aF?@Z$x=!cAQJLsE%Uwe# zMHPY>rtv2QB`gB486YmM0z`p5Gmx$DYl@Ln94pb7wjm3n_64J5YHK(%)RgH=o}Z!P z?nm@E01SWsCy)P88XW+0(@Xg^AQ|vS@#7~ZnH1!Az4kUD`p`lWN>2YLv`39GL_At} zE>P$!8b}Bb1y8dzmQFcX+e?3mO)Y8%W?av#2PdZ@o+JO_2@(-MT3%7;&H)A^wOw8x z--RH~%tV;(BeF}N%SKYsu*1beaa8iVd#++w!3#!`rP{Nb&*J*RS#)Ot9Dj2jm6En& zdzouW{{v{&4}Qc*(-2Zt9`1=2CgEh~PsIjJO-;g&Q|{_zh~OfIB2Y}VM8g)VJNE&6 z)L?nOgcYu-G?2s&%nwsuA{&w$sS^C;G9lV2@qBKq#vNdFjdX@|c5V2{N!@>2NvA)) ze#M!$zUQokXy2UK5}@e^rU0nq^8@eyaMtJKB(?-@Yu^MJRiWQ=eSB+5)^Pe8on z&=nsVs6)wX&S*f&mdQ$v+sT5iowtfEQ@9t%eP1I-h?B-%k(^wH2B`xQ13(P@S7(AP z{S3P`sdVvoV$hQGROfngD|}o1$i9}U5E+C>E$Kgl=Zj?}`kt+8V>q6CtJ@?GStDuF z+ScM?8Aq+*o%e!zAiK(qz`S&z%<`e-Umo@T&}}j8pQ7Ku{LQMK#2f=Nqn1!MoRA@K ziWnXUE^K0fAeA6_cac;f*lkc6@Wbg-{0n`7uk3{$ese&ce2sI^TI>sTvX$`ZA-Wzu z+mLcrR9YmWil#cHN*?3UG*xX-R$^tmZU46He6lsUrDY8$tNGPbM!-9P3TZr`TK4=Wcp=GS!uAT)OW3rY z4_56A&amI5af>eSnZh5|b)+tIk(CO6A9yAlZ|L~kj_`K5Mi=ZMxnNUfkUK=m9ZblB zky{yo3DHhXV*8Gr)SqQ%g-vOe#=($WL^G}GtQ_|=KAN7vzB_{#tyx5#lFKa@Om9)6@*)i$R8Ox0Xi;6rmrv$v+9MpaQOB#W?=Hd0?=TTgqfJ|6 z3Po36KUVH7#)Tqf(uzh@B2@oOe919Ds3tjmR$sKEgsOxzA`+m9_P1S`-NDoEMl++= z(Ww@-h5`VOT2ptiXM6w^0jf3uaz3ojjwC3=o(1V%*p5|pgCBTXs$DI0Q^udRoi4>I z(Pb-+VL8zN4s{!ry~apHy|JEtOC_?3zh6X^CSjI1+_}9uhF#7fZ|0_WvT4xi56Fd3 ztU>kEZ8X&>h0LG{>A*cf=aygALDml%8z21DI{$6#LyU^Tin34U8C>F}+8a(?;j_{j z!{$m9QGW*aq`rg2+0J(2ZJ$vG!IRt~8&4FKZU7}OlX=e-tRHL5d7< zQR%@~Vpmh|ABnrj2=!B z8_^NEhkl2cT;-mY^KEWvStKT#w%k)g1%^DoW7Gd?z)w{&$}fSnnWTR3Ff|@*tMz@` zR_f^<+UE-#GXJ&mzFZ4}w?XxdP2n^9#$j+S-jrpp+QOI^)pWE&QDB&%ro5X_QaAp? zYWMw@w<&197ch6ir%t(t*>R)-vlB=h5~?_HsOf3yFNmhr#rH<->=QouovO4fp}Cdk zXekTd#p~pldE{b~b59-5jG9B$d97wMr3)f5Wp#x+RDzXo&?6DcB6nSJMlQjh5`HM3 z+_rS~+4yN>nB8D*sRL{b-_^oW%MMBFfJRW|VMfVLS_6m9=^wPzlS}ti zF3N3PpQaN&MokBG_L=7_R@MUl)3x{B4P5vwaMin6!F%ZYkoZu`7`kegcRsR@v zQODfak-LDy>%e<{<0~e&YDmkP4Xjy53(aE23r{Y??YqRpJSS^Jqs6yT0jahesRvU1 z&^sRuDM}Z1^0yy6^^5e*;n?d%{2EU`k&&f;Ja@dRaRlPM=^o5e>oLwoAEPpF0pSQj zqDu?J_r>TEjOr(BfR6(%<2K1lY5O8ctgiQLKLUsFx!Yo|?=PFSBUL$tP{K{^+$-~I zm0?Md;ee!Jl?y4G*^(H%j4B&3)Ki8u3s)dspTT;DEKuu%D7R>L?LuX~ILf@a*e$r!Bc+UroZntW4wU?&wnx4i zcfM9x1&<3fj{gCkw$Z0yJxU=+?F2s*!}lh;dpn<_ujV)Fzt$AT*G_6_4%&j%&Z2SF!O|7y*!S30TOtJx zefV3Uehlb*E?`53ryWAfYS!W0FGO` zIs)m{Z`CYLAAFYNI-S_8b+IxiELlrRxIoYx``@SNADzZ>p+4<;TSDw8MbqaAh6#BY zg~;gCS$rfSRk8I3Welzbn^5a28GhJS(xI;fp)oq!wI_c3MIjqmi{vR2$vPQYNoPDY zXx=O}J~24WWTB00&j(1>J!PaxsuPobwGtlvbcy3Bky;&M&&smu4qeKNKwq1a?6)gM|^91l2r?))G~NKA+n<*?*0d` z$!Z=MelmKz)y>U;k41$HUgEmi{4?gpABaaD2|BmY^C`V9Bg|y=1 z2X^55Yd8hU`Pbp9qE7k(T?+eTQk?T z$rW!KlUT+Ue=xr!5Y)qdR6HsnmSIs^1PGQ$lKuz%c2ASIExaKRP-xA+cu-eKlL`jA zBD5ZPMRp*VyY+&gvWse{e=XRiO-}j^BD!se;~{v^5~CianBQGBmN_qK7k=ytjB<8W z5XA@@$411|zns#D8ZZP@wP{%~Ujpv{*7OH&yb2))9BF%f#&D8%daav-at#p;)!R$$+EEf|hrDuNM9dtp zg|Yvan<_MQwOe8K!K;!Q-N$>={hWKqykY0#qa(~c*mWz0t$#rr`sf$A_N8l+rCLj3 zG7^nhlgOLqkMeRMda`aB3h+ajBEHtAh2C0ct@PB&R8JVQB!2D(E?NA@YHqhHGlgr= z3#HC3x(Z`Ma~bOEkaL-6l!k8wt=wQH0~YRQ3p+JH;)9t>qp0AV!>BrB22J#;Do>*B zSVOGE!^W>da+g(k`>q_K&47neh>R2rpkDCT((YJ9!#pQf^epG99*N5KdZWkra@xT_ zr9~^$?cNj7FbMkQ*o^EnBbw~+(x3Vr%cVS}HwX2se(3i;acgh?ZEUyZ@oECDtGfd% z?&r2kvD{P)ja@)uU_W&986hJ}{~4AJj|OFxmR*H&Fj#}NFvo5;d#9$(3cmt_Ri_Dd zhtn9cpTFaNU~hD>iXfu99t!|HW1cIOv3FU;&LO*>?;^x-&GPQK-ZtFkvC3Wp4=Z`E zD`g-MEpc|?AUjcwg$yf7=+)fko`S1OI|>fX?^Y%nX>^rnc5%J2Wtv13-QWnnQjzVT7|*FwO&5m}-v0ZXf@w_x9UR zC5YMQJ_xl4`xyEv0!_}D=+V=d46?YVQVaB;@GdnVLpeqj-r=&~;mFl`@tyBX@+sUU z0kRCAg0zqkpD<0ph}p*K?=B091l8RZ&}kzB=&ZxPJw&a(J(}Cdf#VDpV0B8tB$ z6GZYp_29qf%KKJh1G3R=+sznR-H0R5COCjc#rzn7Rt1iKcoke;)PJXXp{{{7xNUod z*#ZC5*L2(@fA5$V;BYWXt6KWlq;4WsXUtu@ls3C1^!EVj@8f`?pKi%+n6f}c}n>w$?&B~fuX9x45(ARBPU=>GS66+!I3>Qr9J{JyReG)4TrSPuMDX-9FW&6^$i9$zY>iQJTxg>-ZFr z*`s9C`K-{%*W(y6AdMjB`W2ymT92fV-bIQ_2a!7LwSjoy{;#W2jQrSh28~za<5Fv3 z@XhVv0tUvyT`*|}AK!bBiwPc_Z^+;O0KZR{k>$s;_~>g$8yeN|b5xJ<>>YO~m#E*NwYl;AFOShb<&)v?TM_>X@irlJo=}>}7#@ zAr@nB_tXLaSpK-F`IsWK41c-{at0N*K z9gwoP_8yF5{!$>W@PR6b7Lg*Z`yM;!*@fHkcRD4XIxId7!W4NU!MhD5hAok`;Uzsuu^8^-&iZwgc7z#$At?P}qhD0Po1-^Bbi zwrE38dBmcUj>TSkxKC(;l@aPtH7UMzQvlex<@@1e5T(~dp6|B{G~^l&IfR&YOuK5` z7A2xqehJ)aoNeOJd*y_PtVvh}Q6FR9`0D?;k8DJFVB21Gc}{*0ME==9Pl>EP`0KLw zcY;&~$CV7SG;vZ;s?&WARmv2sgBv*Z|!#;m&&wBu-Lk5W) zZ?%z09t&qQApi)WgwtZOj-;)N-R*Ei%|MgP+5xqMyj&y~$o$OP?*3m24memcu){qj zKNU}nHo_>A4ycS$^q6Ceso$lsa>l=&3#t{RhJO?9reL}@r-rbdmj&=YztrB)ys;!d4gIRM6zVM%y)t zd4Uu$xS&8&gS+9z1h9uOj+A#Maz6+(^s0KR-M9e`Z=CBB0u^c&vqwS~!k?2kO? zG0Ds2eva2yA*etb->*rhQ~PrZ2w#O7f^c_2#6=@%wRK_ZhwI`qWXODnuAnySbsO=m zXa~c`>8s*)@A8wmg|NGPW_$NG485b~)T6Clx}s2QYsu^?io>=OQO5aq-!2*O4IteS z4>na}$Qn%C)3JF_#jm^5rqRK%y7V96tKyOK23qCA|9Tqv}0Gj9;LB`ZVC)e2Y z|6s>?uIDgPRC&cDybk6p^aFXy;5*?cs}yZh_jj=*X=i6Z^T3%^!H)7J^Pj0*DxY^g zDICLXD$iyZ&7nDr_k3MorbDh}cIxI6Tk#ZVvxXp-Rk)hvFW(5C`g%f9mFVpgWGq^D z-88?9Rl`$HN%CSk%UeX4{m?5<%(r79tM%NB@?GGpPeyS9n z8Mbg+{X|1hW1Y#bvA3Td#3#YoEgF(?t}fp#KA&1(M{QsnZWL>RPRVN2FRR_8o~WrT zYMC3!oXF=*MM2d_L8p^Jw)9=Ou$9AhK9##5c?4MITF9ud!fC)1OVxG5_p13-l>GSLO_(BiEag?hhI7y?B`)dzfL`+r@NNF-vNK5@+Vg?fSSc&Wis*r5o)Nv zZ^LHB(0B?DtGabI#KucGMT}55SWSD+q#YYeK?%R0P-^wpYU)v7I$!QKWr%L9hQ$kh z2!zRvIo!87wz=6yR0hPL${X~9xhW)CyHqoc!M86p*s64U5Ha5>$29*qqwVo#f?Xlt ziYm?BfjlMA=icr`&jGddu0E;p1-0uJp;shv6yiuMyu`$s$X68qkI4>e`dtZk3ppdo z1IY?qcjCQTy^q+fn4*)IwIj7n$UUq}_6+be7$!-`UNIXfL(>T{5_10%!Ys-==+?G2 zjH~6QisD|)f_uB$|&VVBawL^5Ne032nn&ms5XP6&l*znk&sXME`520HMB_>Vc|MnF}$yG>*uj| z+*@bq29r~+xdYidv^X&58a~Z6LoCFdN@6`GKcptSDbe|tMrNUDB8FJv1^KF?Qy3`Q z4CLyUVQg$5OVADau`wBnlf1f`{cAinGTM!!uv-oAf*IHoK5wWcix?|C`tKZcsEQ}N z2IV60c_tSZ{RasDwwT;nu2#R9Xf9*W_|#vDqtD_FA3q##yZhU_7WRHBlb`oQDZERB z91463Y!-(91DU|K;#%ig*y78i=p+p!**(Pi zo{7vkYhd&`&`PbTd5l?GDwtTy`W5LTJeggIt~<82Bl-!gP6a@hLoKKtlCK7wOMf3f zDDB-H2yE?XZSn1S&!mn*|2rqVP%g9eMRMOG+2S>T8~vFL-Zye=;);&uI?YngCI-4s9GXg_XallNw}|niCaY zfZS07fm@ibUeDx%5C0DUGC|G0*U@axNEu!kANY^a@2<=-!EeEw_DWtIfB1_BH7QEH zV8dWS^BEHOSmGp3CH-N4r^`QYhTWB>GjH7wa_HF{$W$N zFYym@I($KyjKIxE1#3Y-O1QmD&twC{LV*Bq(ZEwNt2iL$IR|MRZNd+JV}6hQ^02g_|Zqhy|Ia&mOQl zyM-EoT7V?W=jQo5<>RvK>pjV)o&vtoW%_iOGN0H}gQ zDJ3>01Wk!OoXN0-6Oo0U0ihEyxUvX$s7%?0%oqaV&!v1Iegw z!eQqSuWo8m5*aE;YBP%hF782g;u?s~c8e~|6aR5s#sCaAJ}!mVzG*vqKhdFK|QLv686rxsKcsQpcmV{x&NF*YV)l9_WR z2jeyX09{`5vj7r7;%aI)=J|3HjhTtUU}lv1nQBPlO4d?QD}*Q2xhh)cAxH^ImBw5k zl0*E9{xE+MIk#>dv~gDzyv)}BR8BTW}T-f_abAPICP54>h zP9W^ZwIRec*#!=NQ^Y(KS&~y4=V+6gJf}14iEL|0Vv$v?r#7H)YDeaC48<<38rTcK zVDl+*5@b$dKZS#e6Mj$c3R1m#ZhhHl*!lQ;UJQidQ|;rA^IDYgrh}1>PI#OD0Est& zJ2uE)Uj}wT$9zwot(Ak}%JrK(sk2c@@q!!RA3%uugXyjB@OiBh;|9-tl(tp%O7taE;E{{Y1K_@T_uD(OED zU7zI?&v=?@r;ana=X5BLl2!izhN)l?u`PgNb|VyqV>r}EPQULp(?Rw4G-nmrq=M5k zDa4mxFsX3dsySPBc+xyo{{X~)_?*sA<27B3a2sJkV^e%Vn>d0F`{hVcfyk&6;+^pE zoZ=;WxK!>B5pH^Neo-3xAthjunNBVk{{V(XxdH*9d%dHZc%%Jc`&#Xls@#V+;eHq3 z)n?4UMElhB8HQrrZ8cj(nu=!X++72ieN%~9OD`=iq(+n$-dYYNPIWeL^-7j7@FF8u`@AvlTuQpu_3gp^k!!OM>J&JtTRQ#RP|KuJxaYvMO6!~gHELQ zQlzw`1U3{PX{o10I@VB>DW?!hml9`=G70KQr$eATw0)y5F9@)ZUbO&ovD6wDb4K0c zadD3xWiMs)nS~RfL2R&!ie;<^*ic}}QArw-y(Fzo>02vO)U+rS6iB(;EOaNWIq25f z_lYDRsi@pWyMQ%3+vp>hdqm>;I<^~SOCUVTq>uR$rv{h~j!Du$O4>?LqLNe-2N?OY zipb!m#L~WR)vFmPQdtajCbX&Di`sw(+Ii+lX9f$ZUOMIBg05)91d3p z$*0}m0%p@Vf*&67{{W-E99HIeCu`lDR=nf+e+*P)ymyt;DO!U{&HMbKy$k|DZM0NT z(9}4TLRPwmR8pl9ap@oN!*2DVa$hCXb5J@R}$ zU5)g0+ccmFQdOKu%}@YJ=g+{+$)x0J;&iENl|MF`NOm$Muh~2yy^yFOwK{@>qc5;a6D2uM*{K!pMU5+}na%;Bp+Bnpz33JwVrvWAAH?am=yE>dKs zXd{7M?8LE2JV^tEVbW?3x=C=+vhv=+J3;(R{sZ{$fO|crL-kbOjNHpT%`2+E^0{_b z!+9TMy`EHO8KqW)Eq%=zRJO%CNpaVzG_uetQfHRu3Y9lLUJ+|Vvyc3q2L=@|D2b;Q zL-7)Ls>G|kiH~ghIF*@X{)}?G0F;l2n;Y z2~(sv_M^3XG~mzw057{a;GP6EXs&aGvRVw5qcCBmFwJB1`qcaEG~~F33e^pFp)MBE z6r>?aC{Inn$QVpKsX&z^AxZ&7%1Ls>>SzHLb4?@1nbvT7MVsR@6NgR?W-~NQxr>wv z0{8$bS^}M3uZpQsobZbKMNyczh$?{qK#4Fx1|XZ6gFOty^0OHA1}A45>h1EcH~~kQ zzWoXIkERH00YXFpKYxFA!2~7HBFCAu8&1Xy?|q;N!VnGOL`*1p0t-Hdj?7r^QTB$& zSg;Zxh_#RR<;q996dPI{>c{75Yij63^5zME(TD)@AnH6SG^J|;P>~bA%Krd<#|;jW z3m)*#=AH2@2&9QVe=b}4$F}&8C>9%!yZoSts!rhT_IK;H9RUwR5gksIO`%sFUwwJt zh*lW&(`aFs)O`)M(g3*hk5f2>L)xaKG3(^ULQ>vPC-?zL090@QEpm8xZE|;{41v>> zT)=_1TmI804T+#iNVxL9ua~Fna1TCBy6FM)*r3F z8p0$1w4X9R{Z(P^N(<=K?-%l2j4YrNIR?P-avtin{+AFnr=0CG>muG;%GKMNf(Jk60+=Wg{PwrUpY?=mbNicoA(L{I83xwme4-=vKKN-Aa2R!WuR+tv zi&%NT?$mGYa^?2$+7Wl3NIS`k!0J3xZgbNOi?I-hu^1pQ6$mHdFhUPE`roV#V3BTD z{{R>vs|4vm;{iJzz4rIP5kzt$d=N!QP5KcVY&j3@ymNtpil15+-oADwC6zi}91@_< z!jM4#03LuA1A1HzLMVfAvHD=sm(%STH~|}8eSo(kZoYTxgdP3M}CE8RWcF##fMaSTHYj~tG3=y@M`+uW_H*{+s9M!zW8-idHjdl&EY^M(I*o`@mA-zJP?F|atByyMwuH}5`6DJ00*>U zWP78|rZEsim^KsBmK_8Xa`fIeo_{NkJP<)p2SIBO@%@GfkpNV%rn>p=f?$V|2B39= zq(B^w=hp|t;2wOr_);Er1_Y9C&wJqjXWETZr_0O5JW&VLu?OG(0OJS*kUHUFq!>-) z^D*Q(3=l+t1Oh&L^uNOdN`e5=rM}PS;_%=k#t46(KV70iQb>UoOrLGzslf<4>$!ia z_(asC06Ex1+z8qyY!I;3f}oKb>JdJi{V?>90-WeJdcK;)JYW02^MCP$M8uycw~)WR zz8ID&GwDrQ`aSl4yWR|uCSoJkf4%*%iaaBoyZOVdQZ4L`X0Cg@n393H_WuAlSe6O| zkUaU=dQU0A1Qdu_{;1OidI3JW{@9ce!Q6hoXva>h$U#Jo zLJEYDb32bq3E$tA#7-8<)cZZo{J{rbp@Q}34)k-O1Uvo29V25KlRjIB>-KO$!@%kM z;DQ`fNCS|V8^rSTu*BgZF3(-#8#Zy9!r%axVSjm_utS&`}DyCO>HG9@mRz}&4DN5bIjT>M9BmS-{|E1 zPr@CgWkFH^e|e*i%QZwJ~nG!5THWH?!Zw{1J9p{T+KKyN6C|3ELL@e zHzc3Q;4yfFwe`Y{>P3kU?!WBi?Bu@C`hUhwTO1ApvslS!6l5I7HNZ$(m1LxpV=h{W zEc_}Xmwy&LgH`?=J7Ir@sT8NUJBupE74V&6lJDdn_`nbntV|%;7NEel^%Cb`ao-FgVuYbt@9NQl%_}u*Bh} zC=jW>*OxyW{{V+c{)as);rmtD%p!+i{V4XY720$l8HyQZc5@`p&9X!RD5V%&uMe4H z@tAapsN%rqQ-GGK7%nMX@Xcb08kY$hn_3EhxwtZ!Hn#CIj>e=7KnKF?Y6m0cvEo^C z)Ju{|SdyURP^eJ#6=6ZNco+V?)N5B~U4&J#38karjKYO#1nDk0lh?Z3gUzl{!BSE#DN@`Zvw=zo`ceIx^RV)JciW7*NdY*2N7)3l)~gb8 z%s^2}R7#W(W{|ZcsVGu{R-~w;ADHZUxPilJk`kcRsi9}sIqGX4U)Tb}Qb7jSr6^4+ zc)HL6R-MVU#mo&SU|`I0`eIo`umJ4l-P&%)aAKtT09<$SeS70=@Zwx-ca4jPDVe?X za_JiaX5gKpCnw$HN$W$qe@^olS^oI{0F->pZQQDv%*q1ZbGurR-rY!1REz2}sW1)X z6in*VBMUQ5{{W;hD=2;wc6S3ozbdxEESc6vol-8XiWgZ*zb^WiLRNs`(9u{yZ}pd4 z3GjjeN}x!vHKxg#?jVrY=I%{3sTI~TQfCFESA~ZQo>dpV&8C{ho;)i3H6CA{W@S&S zICb*UNh(lN6$wM#=^&LjJW`v0qS(GM-k_ldYRjGW8v+Ar{Q(@}>L&5b^?_4s}5IUEy~ejmFCtn7k| zhVu7W&-0wyiEA^g_lxS?W|^S&V@$f-!h)Evx^=#XsUK{*j3Y#3wa!toPg;NzNttP4r3AuK z=snV2w}dOa)P)!pQc2qoB4A3wf0$?kJ({FZ!{}Nvey@!r7NUr1*a|h6*O-iqrATvg zr0YoJ6ou$oNrN^6S38))hrdcx2d0&~)UgE8z<})V%C&)Vbt1KUidKT6y-ZxFGc_{m zmPu?9pi)vHNGkCp$p%&u830I90*Hk{4??(TZ1i%#RMS%53el_@qy@EYM-_WJnhQ19 z^`q@B4dchSKZv-0jdJ&kS(gg&9}aO2UbbA&;wy3-){``*Qd|d6P^hL^QQ2Y86(J;I z<`)>e1}hDXlHZuZV{ll|UWHgCae&6HO9wX>1~p#PWU{+M+5Gdf_`*&TJj-(|v0N+& zPGy%Iu#nHf&P&t=s33;>w#!{+qLkGV9Hz`@n5to7RFO{m3_1ysV0*CHpFrvV|H{q^JUxy?_PZxuf#agOe^)%;r`+h04arOxaW=D3qyk6tpYWBm}7w zCakRl6IBKZDo9$z>QdCTgcLw2Fcc1zgu-B44PGddw#1x3CY129ABdXxKXZ`7!)VE= z0($2OlLVtEBq<7_om_k*r9@B!l@$;jz}4c+?Q3i$dfbwbbwm)I{3+;W@L-)W5-_s? zT7v=wI9@EJxTRLRp8 zvY7!B3g&^sEODYzW`Q(1Stv)bsZnwT3FIy<%u!WT;ceYo-6?HBz~CuLi(n_nm1+qG z!hJ7K61AbYgD`b*B#YdaHTVMhgB^#m*aS{NS|$(LtRhKF&svg{sZz(Qb^#?z;$lfb zNHpOudFNJ$e50adbC13-Q&xVg>+{5ST+#FWmiomqEg++$IdttwJW zd8wk!8Oz*4zzai-JnL>Cm>?!(>C~KOYHo2(bzygtOLx zor`@2GHk=N*g2SaXNi-GPiZk@3{pZu(=uGOdeo8tGSU=yyGUxWZb1bG@7NYZ$KQr`h4%*&Ob(&Sz=tW`}YFX zcF~)6s{;+9S_6*M))Cw^ML{HKl%N8U0E?+nVPic^VkD=A?B?cxYI^VHu{_F#OVNW= z*N6|T+>Ii{{e4DnSy7!;q`bv0lv6OWN&*`oHX{BMh$@fNp$6EjrJ1Cuy^!qXt7q%0 z1FLnB2oll~JI_i{!J|=)>Uwl%Z(X}~HO8Hrc3av10AziPH^7;v3guLfarbDMzH0Vk z#e8$b>ZDUyn$pu13(Kc!o|*xPok0X)x@E!@equUnI|DhF_D z+$(P9Yk)3k7N!7#0W9=msi)KCY$6xr7W0(_lz?f`-Z&(Uol9!cIg(sS1EC;t#3c?( z3hFC&<=@`i;ZU&5O*AH*-*4#|_Cn@#`?O3{(^Wo&7ShYtO4fxEq=dMjK_rr-l9gPA z5vfXAq?9HRVER7VY}Y)h*Myj5hjU(C_iA0-UYv|>ZYy@bCCe#N`_8vi87h996)#H3 z(vn>8rWAm!J=h@%l#MAdi__B5TGa=Z)cH}qhMpE+GZ-Dg`8_H}wXp}e#-Dz@{J}|a z)fW>APK6MqCs8Dw_tYbEGHud>5((M4`C)ydCd7hv>*j(nOxOhK@acDRF7*v(O!Y)yGrfDF(F@*qTmdomY{%0{KjEjnmtS`2`cXL+O-H0KvZMBdJ#&$ zIod80?SFUW*I)rSnd}aNf{sTaM;|=Dv@UWb_E*t}StSbXZgy0JjwO_*vslk*C`l?% zLxJo9eV_vs0DZ!*aV;h72`UMB?*w?GzFI*5z@E0Uwh^~>n6hR9S-21#>F$Dga@FJK zMq4nWFIYkeBNd7~0Vj#Qa)X4BE2+`QYF0G&X`t(63>p`+n(={psk_U=|^i$9h`ylxL0li-o=O2S~VgVoC0^B91Tsfslv%Bo3^ z(QakJJ278>xC@PfYETpXpC{(p19w9T^KE|ZHC_n`Qg`a2h5+7a-bI#XNK8{KOjDD&ZOAU+5$w#KY!y4 zQ-E3Cl2Yisd`$o=X+&Hj}`m0)Ev6YDk+?}+0i5!_9JghT;icH3jH z5ItuE5fy3OUZ2PD@xckNpFT$P9tc}NRk^edUf3c6&RW`eF&$GT$9sAbPx!$J4Z#FG zV*6Wdy?#Gq4F;rY=A5ZsPw~F7n4#t%3I~{(u)mSE-A}v;!qjW$U2XE|NKU^G7N7^_ z;8&PbBzg{Z6aBdxdD{yJpyQ*qgTI*GgSrHLBX6b%i1cfGphr-FHlQ-FxX=M08#e( z3=8*6_v?ZPGJjC_`Wz5Jut4NkkrWbCsl1aXU0){F_4D(KVR*QGmUhjcD+BsE;FMUDN^NS{xf4tQzzli%`+_>w(Y4_&{^lV0lhLXDt-%trRO8~I|$NK4e(UDAPw!0AAj?jWv6Qo#m>+lomNWlvdKuG94 z{W_ma5JM!A3F>SvU_tWf&!pk$9?+rY;?zEQ?MTMWnr4ywDGt>->RXjOcY}TY$K{C< zI7d78@`%Nb;@$=NQoX}yG68@NJNrxan?XWEffLr;`3`>I?{7koYQ(#9<@kvHyAu2pC zAy{$>aS8|N@1?K8ypP2X;cL<_WwQJfiQT&F>QmdFM{}knDV}>z+3Y06qbpSPOPr}V ze$Zt~%yJBJJTq|K8pN%8uVHfVPY_|@JZ02GrX_bsR-6U+w*Qi$=N5&b}` z>w<^oC4OxSLeREYS`-wOAw^s&LXzQ0JRpK7LDi}G2cIgNhZqul3a zt0_{_tHM*>scIv_QlM;Xd1s~kALctDKM;hHaJh~WZlI!Et_e$a6mMLTY5*x9);+iQ z1nn31Z>H=fPsWLIvtGyJa+iN&5jPMqIn=ewC7;^JW)!yFfciZg+1Ueh%bS#WGM%+*fS@ z_}5P8-WfAVMKf?Tk^+JCFd)SEL`V{E(m~6lGC)250I$f#5^ZKFxqO6eE);#XD;jR4 zw7E>~rraAzPzgv9ZP`d;uq>?~_=wRn(<=;XND-Ap@Tu_mgW{uP|Q&8J)3pxl| z6C@VgDIqST00LY>ieN-aN<>qAl^6MSZx(pEg>h+DX57-;sv^O}`4f$0bk!7X2e@vx zF2GMEMq`n5F`XZTVi z!%X(vhG9V37IAB^A;8;15IC2!;m0`>6`VX0a~gKi9;P8(;$BE0O*oh;1f(_zi17u2 zK@)>q)XJOc)O28O9)#;+ZYg%@la!7y2_DgYB~(y7*{lbDIMdWUi*fx$V5g|bpQB|f zhpL{^8A3GxwIs>bK#~9g@C;00rx7J2uhtqG(2{%k`1nMgt!h^V1JU1406K@Wh`#Zb zRZYX0eHRa~;#W|DkfIb0m86BOY9@b5(pI7%0zn4YmP)-~<7UtUaBMZR0_VqgMf(o* zZL58}Vopm?a}2Skl#*O+1q+K66cUs;37e@Y1P-zW*tp}>ijd0hG$X;=mXEb>R;M%t z^dt&Cz$P5ENWhta@tbvZ&5;dJT&qQYCs*t zKf%Fk?gTzm#XIFkS9sgaApu&2^&o`Dgs6~GrzB_+PzX|2hMjD{7CKZC8%hha7OSyn zj&&uf?nBe44qb>FcZ~#ofIKqQMIQ=qgEbo0`8<;0upv9eJxphtZPKy;5lxMB1q9>naep z{{S-0Gj7v3@0`0;=|dfg#HS08%$qLh;t6GnPn|FD=Or3R0v`$0Y+1i(1NkV%ITcl%kO7V;+(5Utk}D zYVH}Oq0K9J%O|NuS(nwnnPps56Ps2Vy62lfck^^Ld2x1~tfMk+p@dMjTymnAwH_3! zL6_UE*5T*+>3Oy)a_Y({l=ZlB3*6LKNcEm0@a^bh6@gY?R^|uyc?I?D7IpJ1&^s_Fpw4_FQF&NFMr@RK5}dVs>`fbT;}D17@A0 z6K?6<5L8JZA{#I zhnlY5_H0Qm!Bb8f6DDesKxKfT!|eO2F76ooaxK->Jcirx0U8JwPyh>&bb~jO1l&P0 zj~{L4&Ylta4HT582i_N>RBu9CV0qFa=Z`70B9$bhE)?J%6Fb41z%dhH7>a3O!V48J z?kDSC2%cEf^KPw6R^V&hz4M0XkyP9HEBa8kN$%7~iAZ)7+SB10feO?Ji3BQbIDu(0 z!AQ6?1ZY7O=0GB+YeFVs2?V~VK`J{R<5kq9x*W9EFywbN46P+4B=}0f zI+JJ-1Ou__Sy-7N{i_QQLs!F{%d)8Si*mTg43pqV19Hz!tiVRn;qJyWjEgOHr#3-u zvGfQ>LPRCD$&Fe{jq>IdZoOT4(VN9Q^SqC#eQ>`q<%?< zS)t0;P>Sr;ma&pVgDuv!aC_^p6|1yR_34^DRG~mfccB1OyO2Wye&x(r^12~Wo3h;- zC=Amn46?4^IHuB~kQ7@$Fr|ye89M9vgTY1Z2^7zswrEKJf$U0r@TP!(%!QvOE7}HZY1hknw^R_0EeK` zp6-N?Iv8#dB9pu9Xw5xLv(>jK?_%jMRz49ZO17jo03Gy#yMBy&&s=U z{bMg6Dq|#_NPSpTzc+el?RSh*IcZ7OGy~w0Vm47G3BITxgS-N7BM6dNz&bzbHowBQ zhY&`D3SWOO(lz7%0IdGoM3Em~nA$KYxD~}xN$+diZ}HOb58`2d_G|bx$nOl$3|54X zp?~yL;brCBpMD_wL3OnutlvH0{tu+6Ab^*$3U{+=*21F6a-WJCE+m*zP*Qqh(Vm5H zyg5&3F~kA+yu%BG96)@mTBWG_LyOd`<1kAh_ms4Rg}RdkC&m<3lpT(; zjU=LV1zdpyxmnRcJU|0`eTEd6*rRs8|ln_C;fS8XNW3yv{ z&81ICt{C)sl3A`x4*hrgk&*Ug9ii-QGb^1bBw}!UAXt;NiG3^&jefg3MtdvI>heC( z_;9j#ceS1z$@upzaf`CuR%qi1&ncw?LRQXhGX<8w3T+Rv787iJKlC@+aoV24WwR7a z%tl?ufW?ABiI*Zw#b7m{P8lI7HUR_+7W|8TIK33j^hxRWwfGqrVS&MRn-?RrSVf@< zLQ^_^bA*$dd!Qt#c$v8A%T*nCkW@>8t#H~zs!T?m`HT7e&N`Cn5_kPP_eYa3j6GwJ zAA@21(B+^L2H+X;JtwI0>M+Q-KrH$fh)$p+#^BsYu(`G7U`71!L`n>KA1wh;=Y8*U z9;bd*n??w12#eo+zi9bjf*38~J93!6-zneAY%xjHTy3cP`PwW=nfwYr^1sRcUJqdn zP%|;g=al}}{!Pv!HLldr{2Y_ySVf2m(>Lfo{`U04&{~_Hr|3Ljh^tRNzc0V-i6DrM zwtrCQN{8+~bAlsUbsheGa2b@rJ6umeGCz-hEJn2I2VQYw4?F$Z&cOuO3c5+~;qMy= zX}2wpxa;rbh~ne`h?p{d;(u|HEeBdiQ-|ASg)&jZXT! z1wurEZ7~OYBK_gh>S3(kC{0?zx87~)NRNgZLpL{W=KID zvSe@8;@*Nf7&u6pAt(bsDH9*x1R?m2WO%pQ0fH6=y~p3*_Q4RWG33-60Rd37%tikI zkL~M%8v(%tIR1qCoBN;bKPiki_>VEU<)yqiTU?GG_bA+Pf2mCUU`yC~{{T32PVg)c zb%2mB{?AA_4_@8hm@{t=^C&BpzUHeR|*g+tUOjG=eQW?LOT7@5uiZW0 zF$4)o)IE$Ib8fc2*TKSudN%G`=L9NZEw%m=h+`#({n9_p{-r`GI=x03r15nTB9q0` zKnS0my4&DAWB0@{35P1Q1Je`}^|0Ob`fiKT!Acw*YXnNj7i-gW?Q?TL}@j{g9e zjCLdU%l`l<@IpjNZ3Q9;o0znG_vME|BsQndkI%W{LIJ$&W8dz6;{-Y-B{9fO{$t91 zxQR-8JA4i5-VFp1DJR~-)A!})(8d}YSMDHk{-bKtLV!(yI?z2ghbWT(?;43V9QmG? znY>3lNQAS}wjO!^07$KfUWNYvok~68?|YNoTf@>I0n^v_1AG41d{=`YbNe61?g;b2 z1WaiINF-iK6Bh(WNrNWb_uB+Yfkz`>5Y6ESNPq=^+mb}uZR%zRToCIGDcu84juc>s zfy_fer$0Yf#ja|mgs6~1QG*KA^}QgV5*fumf8XYOq0LF#jr%l zOLf=c&~zm4)BI1g{B40sAw4bhevjb@SQ4So_2pr9d+I3%LJ1}G|W0$OX`e5Q$ z3u@N-J^Zt@I9kKH-hA_qM85-6iBtSW;)ns>CG6<{kg`AhA6=SeYD!3vA+}sFuM9|3 z_;KU^0L3iOyjNlH+ha2ow=d3ibwIGukW>!ICcwwmFYM041ZBRN@!&aDOWB;g*sx9` zFYQ(lRkyM;5>C{3*B|OXT)NI9qRYaP5SD75xf0qG$lWFsok$873IN!^0b}Opr5s}a z0KYgb`FmEkrTG)_4WxdKKM46;rFLr8*T2_}CHB2g*wvbAer(9u7QlUh5PpX%>Xdn4O`os8F^pLLIdM(OiW3d8$ZQ1FE zAkJt zdT=LFHyCU#P;AOkp+2?0Qo z2(KYxIrFED9w8HrJ(i_v*1g8{a%kezjfN^IxRWPtmDYfHrFV&|AilQaSyB?+Iz!4T zkXE-l;{=dXY*(5INq^UCX?>#7X7H(tz1wkD8#uk5f#Vy=Gy2{ZU3O7fM9ycCW?Goc zvV6!BOIFcoZKY#f>FV1GQKqRtL##Md0*XmNfhyplzjPX!ySbnv`U2eTtM;Hxlt^dW z+N+lo(V5wcnrV+>;iVv5mYtq)L5 zNf&ySXT5}KbJ_lFnp7sMH_Tf{<`<%+#k!YjbPo!cXd_c?q$rfBTFR7&l#dpx0#vm7 z0muwqja!Yasqfq{PN4Q!)`XodE%F=Q(U#S4>ls2D6#ddElGvsxZDGwWHjfEW(hMnL zH4s9zE)-)92lidXgV?#FI3!uFM)rt^t+sO4sBS@ny@4Qfiwm^rQk2^K3b0C(r0MB4 zGC>3ren8lgBPtDg)X-45z1&>u^mqETgJ4~%NO0XRZCmRXKFxSGr;U3|?DK-D(&O)O zuO9Gs2~)O%4Y>aR5%CsZleUE>6SAuN!~X!H@|}(Bs|KugX_jCEl;d+OzXfM>zAH^Lo+?~|iQegqvC;+Ds?6J4VX~5kDCsI1a40@;!~HFGm-5(bE=z@3 z^_aAU%?=#k_(4L|R|=ehm96W zjFBm2DNM;ZCLt4~VU&Q$mX{^gqaI*(bNJ5V44X8hs%ECGw3+^zak`aY0_D@6fF1eHLaCZ@B&nvxA1NgVjT9m8 zr7KHT&A)s_iIF1{C)G{>hh*890LAKFmZQ^Wk4gUk2wZr+J?!Fos?eeo;~9ev(bXeT0vA?_#?X`bdB>}L9cRkVas-*AerqfV!Q&_jokDWN z@>$YXC`>|^uyKc#;IQ+uP7tDr@-qD8 zSWHZb4Zx)z-GeO(R&)5P;ni_h;or7?94#7$a}Le8){2ZH#!XX{S)!Rz6L|jspSw|# zNJ>%?qEte&-KxTjcV{zM5{I=nFC{^$TMH}%b0jzb0_r8j0UUSG#&Dj^dR_E=n&Xlt zNy__AgGL>4)D#k<+MH6SPDli_w397rTHU3~;zF54C*rb=q@IXe5X9H`E zqCN$sWr#bM7CmTRMsaQ9WN_UNYFQXdbXASE(GY`ELj79Mg`^azUBgd}S_FWTAOJy; zinB7@JmozsX>HlZ%$RYP zDR@bTPYiH%TS`J-SNrutk;K(q%Z!9n7I?8dcqS*~eandWshRrL}Dtw;8R6K5n7V(;G9 za&_eSQd21(vJFDiBE^}TTF{-1L%%t2`ZGF|)YayU)zM|8rG+$*8Dhyo(3jLw(RHms zDN2H#aV8WHvlC`Q6g}>`^KYLka;;2IDqpJr5l3ndxieKS;493XvoS1%MT1y_-VC%l)G4 zs#Ez^#OexdNkUZ;0V03pi6DXunIOyyz^6St1RB`Wqo0Vj>`Y~_47X<&8;{|8p0wq7 z*ZfOP5lZ8RQ=v)oI!aVE89cv85gmC=bg$YiO^&6xYt!Ktu_)ncyB6`kflqzuXw|*^bKKu*#AB>5iX{nyG10q)kJq8Qyiq0=-o19z?u8 zW0%Y+mXzb>w-m*Nr9L9%bv1lRn$%ZUDNohbRXtfy;Sg5y?zrQHBlM?ImQ(=nq#Yu_ zN6!rV8;Hfv#$)j_)VxM3iyFCw$x_th)RXNJNiSEs+CNPmkMc|gPlC$w9J32A1(szv z90mg|ktrc5lYqb_D3v7$O%M=}5|oAkm}uUF_C3Q3G&JjNFj9pA0(Au)Boh`;(hj5N z1_S|*IChg9*wdp^sM@48x3_eC7W8kIre<9!wY4Y<`2v*|ppT^g0K!iYyT?C+=CXR{ zL;0^_Tx#B05~P6gcw38-x^=Xrc!zSvaZQx1sUtzCaF7DQ$G86g2wkc(=HoIY&Ml0< zttcE+{PqL(p;fv3nlY%RlPRc=g1_5G=z>lPMnUNZ465 zFhIT+pPlx_u%gC?(V^*E$)X}s{^ z%HjfE&l4He^{U^n;?g~i>F?9jbeDlfM^9YR|KuC$9`FAdf^n7ijYPwZ$;F@X6ur=dSI^GYEF-DSIvf!7 z-tUC00{Lvr{si91CY~$0A+<}yZ7$r?#%G}b1Qym+X7cbkE2}cz^d;9a7c&~&0RCtt z$7iQXj!=8Ibyrfq8h&VDA2#!MsKypNAY(XRw1^;qm!$cOZ7Jalk+iJZqW9E=ROfPlw~Lpn^Fe$xPu&u| zXt^GEA*ux?xyGybbE|kserB7jBUU4z{*)T(joIb{oc@vlwT2D=h7_-t{i{Lio$T#+zTMc0UE>3N{goQD6@oT&p}>lGtkQ}zTd;NAIc ztr;7gitwlDIMJ^zdF6D_vVkt$-KIi-U%j(0=L&rovb1T;=cu<=xl}bEw_kKM@2)m6 zWU&S}I))9|#h!xi4V?bJu?v~xX3v+}BL+07!c46mG#N8IS?Ozu4jhc#oZC!4Sm0wv ztmUeck$ASjo$;ExVfpP7lRyQ%yx9$W;eBNFD~<|BV`qWln(g~P_~T@G;u(Z$%?9hb zo}DVAeUfFK3dYu6?1+`rzVH*3d^iRbii7|-P!YY4kPixL&LQMN=#D|~+6}kT@$QBC zvfqgGJd_71((dO1QLfxJWa17v3SaP2Rf&QXfHm<2Hvw;nON;DYPdS5IFz6vhdPGphyEbm*0+ zC!A&7V1R6jq;iSyVtEL_g$n`BgvRX%+cK4Vsyc{YIv7s>zPwWNG0_8F8O}ED2~N0f z=6=)b>Y<#=qC}~R>=8Gp5&Ttq*p@p3l3NsYOKZYoZU{H(B=3uAAT)azkyrHoNON_Y zXEBuuYdDko z=2x(!2aEg-t>~+T`0A;_3;2c4v+BRUCbl zLQZ^;H5`T|{yVNjUjN%qmh_^KtSc$r$aKqz5!-0AqvB!@5mbS7w|>T^pxd)g z@Hy^wHu6u2L#l6hn_6kpRTI^>Xj(E<$l2H>%jzb9vd{yKl7FfWnIkmCbzpeK%H78Q09ovNwv24Qlw;3pl_ z0i=&{ymvT$*T*M}i>vCGNC# z*$`1`7vWSby8-_o92e@Zd5>3uGk4%h#LCviJ_do%vS!Ng^c(HTD<)w@wZr?$%L5!| zp!b^>qCp>nmX;m-87TX`Q+RRVu~y(sQmw^uEwHxF-#j;DVuzm-flDVXvY)VXJw1sH zIWKvI^g!kY$>Yk@`$|rhNJkWLp=Qgb&BVV01Ko=}5ALLD9?@Yi)}EMFtJ*%n+F)|C$I~^e$Yvk+C&v-#wZzRg%GWH`8ki z5lhWjg1u4m`ue?Q5Nx9EnO^I|JuljrV~kN%1c$c=$OZ%@q(bH3Y=N44mxXV0={R7SCb zuDk?~95g(cZr2TI<{2H95wd5U>`}ufAgOPi5r%@1c?BzjC+dE`BWqX4vXJ*ug1oU8yhl-X@~TOroEIjl#9$^8P^g{p6L2t&9zP+sf!b z3kHU4?=4?JsO)<)4bNW)cF!|G4_>3LmY6g6GG?VN{wT_ftd0P4~YQaCOq{Vf$=OS~NH>@?e} z(7<^*^B*xt(DCT#_Yq664L`~4IU41pzxWUQc`2spBfNtZ+=wRz)KOqm{ovZJV)V)W zW~BZWqE>0fk@&sU$M`MiryMVXq0TH;s2Xh#G(f0&)AApnMtop4!!Ch56YCVaTAlZJ zq*G`;*z|-TUY$rL59p*n!Pue|E3Vv1QmpV}4yMSY27Ijm#xjrlfR-x*931Z~K6G>Y zV82KGr5R*4E_MpIB8jc2nJh4={Lpq}VvN_S&(s`RdCp~036w{a`)-*X7h0^TM6E&H z()DX`k3c;u+z}ZmADc_l+ogm4A0U0VebUKnr`piDLU8mK+(T^xHZ@{Ig7HVH0`8z{ zk(7KI-JU{YaYT;fQYY)p9K=5U0CuCNRyU_}ci`7iCpoae79*rwtHoER97)xPc|{{r zPQVTATXtl)AaTvgS&frcdds$h(|4`EG;J;7yu$CR2fM1-X1|5EFkP)LGs4ul`3;g0 zP)If!uh`;{o#ZWJNkQDI!q}U2T)l$#@){QB&_ubQL+!*skuUH~k*3x-dw|ljo42M0 z85}L`5|4neVUn zyQC9phwzb4&t+N!{8MiF7g_D2ss(lL=+xmr!Z}tVYZBhj2sSy+xAO9Fh|qA`bP9x=I#Pqg!P1&UdPnaD< zn#E2f1hdQy9;(doNIWgq&Wj%q}*ld8|p1!wgW*HrML;?LdVoQuhiYMlSil2TL&JWzm}9r zj^TaxVk^Pq7So87w7@8Q$gX_DYRg*eLIfe@NKP1J5)Tagb8X*3uViIVpRA2(cq#d< zX!$G9vlz~U|AJmxr-CB@`OeCYAwc*p-t*WoJ-TJzb*{319muH9Gyr9?dy{?=31vDr zacr=U-lQ=4`gFCDh`JjkwJp;6j@xCEay>tos0n_jd8qdDv@`g3)>sA~;@g5d2{pgp za4~7UwWt^u@Tw$V#ok*^YtN9S;W%BP+S6~+V9WD>`SY9*c9cp&_&AVIa|%zN-PjS+ zz#*)q{{R?odZUbh_#7m{MUuV;7qCCw1gjxRZ4+}${m|Ql2CjnN1|S32)_EoS@R+PB zlYS=S~ej5g3 z)N=D5G(Eo9ar<~fx-+}MM_9?wN^V9-vitVuE1tI|x%+MORECnlQnb+@L1v7u0~tD~ zKKPG{*X}EUGR{9Rmrm!c#@aldlUOFTi3qdfAz?xo)&pe@#;Q1kO6Wk)T5e1@+vcYV zA1$`;F?vTg$tB_0vgmEP+#)n{Fm72?)gi{xl~0CYeYaGO*YhqEJp<}Udf542CGUQg za*}a7z|Z@I4O@I_Izqqo#kxD_gWcd*1@b;KI$B5EZ$;MVJ#r$ooj5sfPmGNt`v>L` zPv`AFzARyQlr|vbD6E+cd9tZHa34Dv&s4Yx9y{8x*=@U)O#vNrF@TW?SjZ?4LH{P2 zP{fT=Cz#}lyALa4G-j=L+NJ%*-~m7$56$rrX^!Y>!i$K2xKC6JUYJu7e~ z;hJwM0|sJv!jCq`*@aIgKe8&JqL5-iL2nmLi*Cnoaa1?GT|(Mc-5e>y%QZU9R-N$- zu&OHZOgf`TwGMKPtVXizTjm>n`24>jy-uHVp^jmx=3@G5Y4#srfm70c=;NDc{_p-d zbA*-LwY%7%PvnGb1L;O! z+8SCXkGFM%Wj(B6K-Mt!SW<}Tde?_)Xbwt@ z8;79)>@a8)i$SFldpeOhZMfwM-y7rw%?d9kfp_XkYucv|LB6T~0NTF_1ji$f;oULTn!6fF z#dkofCaMwt)!v@->Q54aHl?<=Qp;t2t2B<1=*vDhf;-v6ll$S*aNe6hOZ^6A>4%3MY6(cY0NvYqUvzN!E1~hRFdF6+EjqSd^@@a z9HL~R2|1_1b!$b~{R$(~2Dshu;Lhp5tA_Z28HxC;G0~pk%+L^$oCCiuf#&Q#<{m4O z!vZHL?kj?mhUokrdCU|jCCogLtZJzcn6;6P#^!Rmn3}ZcD`$X;1+hEAmN9C7wwB_F z`V{f>^T)ScErcT02uc!+KT3y?lJ3E#YID0Z11!!zWoW|xmSH${zQt!9Jg_AJUj!nP zI+Mq8UcxFS^&1F?7-`mr|1dlWcOq2N{{0ei=p@}wt9^pPATTaLiq%DA^&}*K_kN$W z8JUTZL7*Tx4~q)d&zdaSdsF3`yfi@NXTgY7n@2FJpxaQo;K!pjBB_hxO{)%`O}oEL&V7A|;Ba|YbarHuu`mGp zsw}>41fu3r;prd5)=}f=8E?WW-u7-s_e?L&y`4=WWkVVM6j1%#(A_xWJ! zHY|NSRXh3+b-ZrqMU^xJwQ1@ffZdPee)9Mq;N#hoj7OU2ZSBU5Sz&a(DKrW(&3)wc z_sjKw`=@wn*O3=%#5o}J6gz3=cPyH<@(iK~%8YE{w(W{N^y#t}h{jBet0%kXrTP66 zsbfSqDo<9-*vnDC&?4%OlTE^gDIY%S%~ia)?z*2{4c>r;hC`{quCqA&I-do!FZG&_ zxZGW~D%@?UXwjEyQ8;fI?p~6Zrf)L=d{mwNjTW%i{rKYCFsuFKoP2y-XJ?Sb2`Z}5&Tk5mk-b*VuPXK1WekHTz1P#f30Y`I^qL$BG-43X>AKu^#*DY>ueu!@lb>Vxue%iz+bhxit zn0=x65SbeA-BHYwR|td^+m0umeQ)S;)em6&uiW4E=d4lxN-O2k-Oounx0KY_Q)qC- zUXUb2gweCDz#48+QB+FM*52TiHv!WWJU`!I0x=p60rcq9_FE%Qzhk)oIuU#0XNy35 zH8{!hk6ZNc)}m^T%9-w~>va{8H=onMO^TmzYilh&gS)Ovb&JzP&H}$KX*jduMDNN$ z;kZrIp}KyitiuB6soOV1g8VY9+G=0+Qd(sk6RjG@7Dpvz4}o^LPUdUyhA%zw3(mP% z0*5uuhV9K~JlP9MxTsPux^;LcmS?Z=J=&r2RQDueI^teW&8`TT%XHbC1$B& zl(kU0{;S$&Fwckkc)t7(Y>AdF@cPl;WVU}Z@INcqfc=44k<*LH>L1=;PjLJxZFCsY zw##g=h5LtrxI0BIhai(Wo`?s$Uo7t``ip5rX(JeYZE9ct0ak(dvc5`Rt(osw@&c_F zHhBAAk412(CUuz2 z$us;?@x$pN7-n6&HA|#L?k`2@^w=py1+TUxE!*gQlYz1{2S8eCX&;t8rebncHfA(& z=$VAcOm;6O4vPi?!Gd(~#lg2>45ZG07Rg9Cafb|m>L)qL(6zhg&-4tNxBc_!)FIf{ z8+?Gg`#Bt~(NaZJYqzU&jlh;wIoRoND zAu$__07fLfmfDam>ICX;Igmo!Yf9%qVZF4hN^3>`9wT2?nN#^$@NX>r0ZpExSxuVH zNokjhkG)DUo8#kUcoJ`Jx~5E~T4VLC4$6CTA!PKZHPdz4&Ya>{S#h0-uP6*V$MD64 zNLb*+U1DT4zZ{6O`IS979slDyMCLyDynn$T^mFqH&OHczaF)As55=@1}C4{P& zY~Vs)hNut~aDT?`UBsbLWZ%B7{R2#)d8*1DP`?CmUA#OfHv`(GvX1|Y+e(3J2((t{y!L60u zyH;tdzj&VE+w@OrFaBO(bjid z7f*{VQH-`!={Os~&B%Y5aYN_3BlXe?oL9(WHF|LNf9astcfb>A%{mwvzQ5_W>K|{N zO`$_iYbn{>fm(($F)?p&`y9@N)WL0&m2Aa3n(KK|Twncz?h(tm?Ig7tqXwBnjIW}A zk&y!ZaiywichTSEAttq5as1gGp24#cvwP-St4^zVu0SyhWrPI!a9Ur}@oNy9gY z_T&@Y%E4*x@iw|!+r3T6i%NO=>&d=^?(V-sy@#YaEu?S7lmbsHx~gg^f}goi@=D4#kQlJzjrDhX&|sI zJ`WM#v2o1$Qv46zxDWzTNSu){e#k?fO$PnMbZtr0wa^#HGubO0!O%06c3=CRBQDu_ zH9C}MQ5vmg%-F8kZuvV>9Q!Az4;pNOnuCdGLU&#Y*3t8=u0lFIh@PRvNbK*egOrNM z2;LXk41z0-3fA#{P2Ft_qu$~`QYe&&vwA@Wbf5M>_c|-nXFILX*=_;T8r++O-BQd7 z+huQmUfvDyxE1*(U|TLYur)^u$2~Dfp^~s&TV=xnTNJ_;c_RkmhF_8a;T$kLX_#Eu33h#&PMAo+mh{q(8gB+oj#)vN^%8Z zhXjq_6D@G}SQL-ISjG}t)O`a7R-QMums?ysf+1sQ5oR<|l0Y!1Q*;z=Y)8dT_Rb`T z9)DswB-emVrA7(Uunlo6@-R;tG?zv!(94?Bi06xakol1nS`-R-E~Ayux`1a&3+(#R z=>{NNt#~EG>2$Je>@Jot2OS`ue5iz<)Gsb2WF|N@myIqVe2~)-O^`OGUh;LxeLSCy zLq4_L30;pcl`x%2Ll0U(Y{X{g+%Q{zG|j?K^ddlxK!Z8g?J5%HA0126f_~80$|~VK(B`hY=ovO;Lyi zzdEYw5fdg`>w0|D2(BNfwW()Jz3%*IOrfH&iY$1DCe6flgZ~dep?ZM~05Y}sSkxMP z50w~X`=I6`FV5&!nZRmj$f%px)h&ZIqNG2F!vuxpyf@Z&;m}G4Mh^S0=z%2qvK8n> z^cE6%v%czAu#~n=4RosgI((aux{v=PP#Wr43J}*uMuIaeh)Ew$-l(@cC zkb@u}o5ROhG1=67outqdexCSe(BJqh8s9bZ)T+qfoP4{Voxx!%E*qP@2@0rC5OZR$ z@Y=mcHxqL#0%-P_$pz7}3?Ol6Ai9HQbTkcru#y`XkhtPkh;QIcC@^N)Z>d=iG~6oh zh0!I%$0SB-Hw?W?I$vvIN;csBovFO+$iS zUqrEE{}v=wHR-cO(xzIL*3zTG=o3=tFl0-Ly7nyxer(EA=dR!yz~JR2=Q} z5sePT7B|gw`uD!sacJ3C{{VVjYt_?OI=6H^>FNRT0a{gWjq>>9TSz{%8D|yEi(U|k)kMx3?qDW{uw{KMa{7^Si!+89 zI!2%^-&bK`cUO^-K_Q=u=@kzpAmt)#vKkdnUM>i`q ziFm7($PFi*1i))8kkD<%Ttg*GJwf4vz#LRcc_O`xTC4Zy`ce*nWG|8i82R1hQr;>u z15H?ASPn)?f{#@(3Z#w~dqom;g0+d7d|IFjf2v>GK9It6U%!A>d?ozo(*kM5^AQFm zm+1N%ismHUJDYZu{dFrKQ`F%ts*Qv;Pjfpr^`{FAEeX_q2J#zr(;-k|VRQ(7wvg75 z9%5&S!ZZH$*`Iiyl0r4341FHNxNK^A@RM3PW20R;M}LY41t(&2vHIu&WY5SM!pLk+ zfd|DQbI^;KZhic6s{lukNqia&cvA|qc%?Ye=Pn)dV29GtRhbKnuA5&r0jT`xu z$@l1=+1WA}8Hw(8r8f#$HTj7RG&g_#>DXfpLMHgq$EBXpLgVTGw^M@?p`S^RiC#J^ z81;o-ZSn~>{x2~PVP)2r3AGdnM&l8Tx7t~%=`3ViXrJo&oeu?I7p%xU%UKx7gl*fp z8I_r4WkE$hm^88F->&qnzO9yZap{WW$S9lLI&Z@f<4-FluRk6yJf4zTWOF{$EozwY z5=Sb1u^gS4(=b`s)>L;uS2JL(eQS`;rHHLX1ak$*A;aT`Zy}=coN+aMDvhVtKLlhH z;Qx{XoNVWlCUkYU-k1^UHhSUl@}WAiV6O~f#L(p3uS{x~eKlWRgBN38tp5SBo~3_c z3Qmjq=MPc(r|0-DON|+fzt1+t^;M{^{Rdi!=<9S7<3xj1Z7{Rh^OpH#(>Fsl*Z=kdU)|{pfO@|#~)R!Uq#K#0Q9F})#U`RSt zA_FSGkx2HQKjr;Vb(svgu2OZ9BJca+$a6iL1Xmy|!KRvtYP ztMSZ~_zg;+@);4fQ6B&x?f7lgurf4>B|bI;9yerRsVC36K$V0X@QdLrbV-bc&Z9ia zE9Z`+eQ$4~aCexK&x#SbB9m^UGI^vl_tk;w$|CR5D ze93d7mc5EmUuS~Od%^KV6Sa4>NpWor8SCgWnU$&@jHQ*%tmFiFshD1nBkAyKg=IS{ z-ur@u_n>#fu^AoQ8V0^Z3@FiWS*Q_kNW`u6qxyB3*yY?J30a$a^;^V8G4<57#0h`9z8yUz`L-lIuuHl& z>+|GIXJ=%Nhd1W|%(kCrB?5sUk&iuV3{jtEWpy1lBQ5%CJq`<7mr>vEj?0ZUI;f?r zV)a?SCkolR!IB-QB*4p7SlBL`$m55xsSQ}OX!Yzg@Go@mV?4uZ?T!wPs*W;RkJvZ7 z%V3LGm5cdqZ99>nDCK#^5Nm?%PzY4^p*w58?ej2I{IGR-o)!as!t*{K*ki!JG$$ub z@1rW;W7^-l_l>7hwqzfbAeKy)=VTvIQNwyHlara$Gs9HTQ~|u&Xy^a|oIeu*jH8fS z?^#5`dPi$|RUOk4ijl-^;9ou)fr1B*=sg`|in>JC>N)h~j?R$#q$nH;GPBlmA9s_> z54TVC8^m*&dZ2kxoe}u)qkK;~pap=&4p|~#TT|-DU3pd^=64YK6H;yJD^z-jYUwuG zZ^!?(Ud?JpFObx@x3I97Cc@Zr_O^FL2`hTeY^j^36%QH}-FQc37)BdeX)kAlWCR4q z>vWynzb))vU2BH#5q$0bF2bSVy};SmpSFqPF|(3KQHnqeFVgn1w^f1+N4PN-SZ}WfsYpdy)$aie<2y$xxaQ8F@_LF9^PxcxzoTU#3D~_6Air>* zgSj1Nyu=2XU}(5irU7)|OK{=lTu=ij6$O?wJ#ufM`^liMUJgsacz84{MOf5DZYeec zP~6rnHGJ1yH%@@~n`!_bz73{~Wmwv>?2t9KOxe468CoJt(Q(?3R0Q(T`dvNtRRB~e z<2FOIo!@=_5&qX=ss=%age9m5C$M$RHQ2_bRt`S{n09#9{G!qW5Be3;zGPp%irtXP zpXN|m^qa}R=n|^ofz_+}4nBJM*p2>`-G@sG4=MD4Cb+9i@m~9`aPJ>r4@rm7>{{Yj4{l}3i9sWTjAc|O*=`8DYNg61+Y%c|AQv1CX z(HT|6D_-<%6V5-t@eZzmIS*3$?%H#Zh!hVN^@sVxu@(KD%+Pk*WI&rpD;>*%wUo76 zy&^-yHJlK5?Ok_$u%8)f85PDkdbRmsGsN87v|bUTSjv(;ac6#*=Ku=FHCl!fW;4(D zT_=W7*Sm!ZA(t{R`_#B8uyM^K$guwjK4*%U4SZPf3wT*U`W2HO@Ve45SZGn$^R9+` zt;iOU_T}Mm%aZO-;1wQvvtQcv6Fl$PNs%qXBQ-M!qD|QFxs0Wr4XkyOmrZX!9EWd6 zC8p4515d_v8Adc7*FaM4Z-Ng`A7V4)UaHXQI)8)8uR3IKKIaLAK%at?$-6!80tbZt z_X+@>)?#P3`%~ii@_#lU3C6ki0VIoVeoM{hAr((SDCGb9#tD-PCkcpLEj_9XmoaL8 zmweb8BVPAQ%3R11zA*G;ROtn(_5$^Q*?`yLvDLE!DZk+idRNEv+r0V~uX65W#l%SL z7@o4H+-)$UGmy1mCCO#eTzeB&H40Z7ZgoUrfbVSm4nBAx-cnwd7eYxuiuLtq%!(&8 zxJ%dls*-AD@Bm;0o=TOW+v}vzT^jl~P9{ky4I6lnRa-_*4i|qUAz`8?mkORSbq%Bb zbM}yBNW1i@{uQo_dAa~80$kxV5&Uo_&DaVTqb}DMa;!AIUA1e}OsRvt$hty?!o=$s zztiJ)b=r}X!BOxxB^5e-D4fl|ApZYM!89y*U=u3^aq;Tdgy6C=$b{Nc9;Ir1UC6a1s-6v@7}isK*XaPf@OZ zLD>AjPVk~K6WIP-&+#CR_z%FKl@WtvIkAgmru1lPH~?`uTRb~>T2alwSwhkLirjw6 zEhQ89e|MSAWK6G+HB#`qT)P=S6QB-fvEhAU1QgSP?ms%z{I zagm`zi#(D5iQ{&M$hr&Kq*B6do+3CC7ozC(y#&3p1q#{s0arJNp&7svpc>7z*umY) z-2kvXSYgkQc9ghdWwpR(wdU23OFL<4;mZxl0@?=X9@nohV<=lms& z2*ecF?H#?hZ+pn>2=q!OlXG)8am2Uy0~dQEMh|_iZxEuVP0- zge#@Z>En9vZN_>|tx;Zsf_PIzv4dE}^+)5d=`Ek_p(kh856OO4cjv$MplXP;0PnDa zpw|~(`Q9_z@<%Tu>N1=0JA9J3Q9Z9>8P9Y% zU8G*zAygg`wrF^xe~!3N0NSyaWB4%vkEwc&O+Wcy!vH5xM-Flbyrj=h<9_FY)c5Hf z+*kc~l$tDB=J;~9;g;Y@0ttRB?+LAdPwU&%SYj>`e5ud+=FBbMALj0dYMzH3)=aD0 z9Qr_DmFGwpx-%O)f}F#GUL+_v-wBuoGkmaEg(2Uj39Loe{+?4`%5Zd5psCRAe-W?aajfD7sm0Js9?xP9OtI6>jV1Id^FUEaA^vC%UogjvDFPOCcXbuGur z!cxdL-at|Hc zadPDwsW;+tWo6L~l`)6xD@^4%x*bt4Rljd}D%xXL6Vr_kQxu|Od>rWsy&7;km?qX? zXrrCAA)teq;ePfhP5q(&+s2mwrbC~4zeUR*4(TJ40TKm!X}nn-guBeP17x3V_enp= zp%49r6tz3~DE1KW?S?XJxc1*|W5)bY7&*KWm(xObh`yTmQsYnoZ&M~5TlUUh`Gm1v zZrovTibqRD?LXgIH63T1XdJK!pvkHfOR{@XK)NYCps0aeq~SfkM)(9rt$H2z)3qgXV-02+ddwf10q`ytVCgtF^+9JO~z96ZSU9+ zj`B8R4rd~1t6;O+i;CILV`Ju)cjI29eg3l7AHJ@KY_8`(BC<3f?xfFMstBZ!?$yuc zc#Ly+gUsG!sfdLU%OyT_{ir*jcXl8tvPbIy?H)U__tsygZ+%*fpo5bGcH^XePRdA< zMzS*NRYqBPrGk3Q|v?k%UZG^IMaufD+nu{oFYg`>_dq<|TnbY8?IVUUXdkV$ z!5hS7X)DO)6+v#q{%N}&;xpeeO6BP-Wvx}v>7z6&RvzQm^N+UY^3I#4gwsp%Ip~-^ zm^jYwgv_I}$y9*k!a?Y&uhqb}p^M5QKRjF|1}Y4797UIY#4lXeraQjb;A0nme6-hO zQqskuO@@9iNx^TRpfdJi?crr&){o7|mz{5We{a$z7Ps;4vU*4u#Vkwj7pH#Byoi=T z^7i#1cwhv9w$k-Tgk2Xr7_Pynxbf2-fljS|FI!$UU)uH+xXny&4wBOy z_^L8<=JHZ;&SRa2xbOy2TtKt+5ov3pm8)d*Lk8RB7&}%uSNXWZrSbd0tsJnpF}ZHd zVxb;{QvVB;{d{|dUD(LwXxdv_EB?PfF!@N|s@}x*TF%3);J2Ddu@cORz{pQ3Fg*0y zbBokMo%&=G1<5wr@LGYlFh@-OkUMtA&_WfO501fHrS@q^SJj?{+Rp@KuG3%GrlFJ1o%{Jbxjl_5Gh$ z)Au=(j=nU_5btFEj!mAe4W{7iF8bO@bL+-j0nhKyz2((6%l?`m<56RWwQhc|MdG8M z=CJSO7S0S*4E5_Ja06a@E9D@~=nlw@EEE6&eYawU-K3TdsM3$)NatAl?Yl}1QZS2``*K};erX>andESC((RNo2gW1z)@rpSsfkkoSsD)1-|2DykEAWFd z4gxW==ILPjVgY_JxceH#m;r3t82nDGSCs8NLZ}~(R_*qmVB^ilm6k+Ib4d#JS(G#% zF0=2^*Vr=?^>plEaw_OjJ^lOWv<;kK120+$d>Zx_O7A+EYqiy{bCA88q zN<^v9ZYSsl*uu=@7{MVw_Kl4k{HBI?TUC{|4vZNqIz-4(CzJl5yf=Ub zZVSUGF}T(xN2>D}qAL_LQ(h8RDE~2Uustz`XQl%3%rX`WRMl_P*qE>8ZwaAkauN3VEBIHdW7n?!8w+(F&h;U zNhQ`g?lhPa!Q;W_|27j-A1@M;sH{q&i=v7M%%XiP${ajt%f0)Y9+=#dX+}0u&XkQ#9BheDHFT6+6AN%8{Si?lUm9ni+p1#|74B_ z=hP3~s#15bYxKQx4pjIF;x)~`C-u<{kJK)sD#B4eR;w#_XjEPsL!I8a zqjQX}{cHUsZ!|{#ZzKe9&nfh@UjF)>1y(L;_Ggt>w)3EbzU?RcKbu4h{3lbAc}@5Q zqw?MLihC!yPI={rwi7PDrL~q&z4RSs4#Dc4S#`CVDdupoJc3`<&1?Tn10&8FDk|WXma!kjVVY_(7n;nUyrOTU zY!SfF)pJM@y%E#1afne$o1wvFJ- z2M1wDDvYecA%V%VVwrOEOB@=DFMemc7X-LQ|`(7axb${osD@QpNa3297z{HS*<_}`)RpadcB{DEM z!OkxiIr(~abTYYR&*5t6Af3Zhl-*iI!iXfbU;;PCfk)ZV?9Hcspd6A0wE$`Y^2&5& zxi|rhluV0{R9f0Av4mrLvlau>B(LLU3`OS7Iqt;Uo(}`Z6z%eUaBAN3!{yDP-|#YA z7ujHZd0vqrgoT^>@U@?|pE7(0=%Ix(KZP)C9!EuSR#~JSn;h@9L1pzUll@bU!+)X| zGkhS2DEQM4Ty~&m-4q1zv0B%18#i@+6+Hi{eym-2qxt;qA0X;Jw~nNbGUc~+mcWH% zCj-{Ij=)0_-jkzLsy|iaaBzBILCMQniD)7Vgd$W-rLNhko{iD0y7d_UEt){vp7iZO zI0z+7ym`%-Hr#KsaO2*(mrJF4uDg$=|J`Sdr|;pak{Svm3mNXd z9)-Y;V*=?wyenjgA-g;&qO-yQ`8lkD09WuN*%$)-q_?|U|So>`u(t>xqxGAI0?oS}-lGS3!=?{a!3)M9T1 zt_Em&UoD`v*5Y$*#Hdc!5eKBM=Z}-Y6khck;4eiGP=6KGid>(%EyU%wF^Rd8fDs3OG4sZ7W3S|Wxl`2`c*Iz<+>1fSSu`<-w zzn^z72?m(Zbg9sLfe5nLNt^4Pa=mCY7SbV_lu|3@%T}kbd+3lKXIPkU3*$ zXowL;Wv!uh_-elar;QS#t>-o6cURwa)zdTEUaGZn9o)e|2s*1y0Nl)nJUj&;8L%fD z{SW;=0Oz}0bLPG4I9rh#^$e_X;a4Jg7AtyO8KdsuKq2Y>l(Z3O8){v`;t>A-ltbjF z@*n%O7T*K9tPv1Zx(AYe80K#JCH{|H87pK~b5N&#CkYo0UoJ^W&Zy6l;Rnne(3E|~ zZG$Dxh-n#IhCKL(s@*<*=ES4KQ~ZpluyXLH5cYtpuG-Q-&Ry0*u>&M#4lH^eqWqsk zDXK$l9b=4#=>s~q9Y8B|zq=IGk*f&4_oXw&_=IwX)7QTRm19XQQAwdv;q%)gRjjCF z=*<0UWB+;+0vCwER=*u-H9QK{pI@`mKL3@L@eLc=rdAxl=R>&->)Nmf_4N|?uDJZs z>@uWNdM5kp$OGb4JU2iiSUfxX3kUoOM-tUV1wM533{cUptH}hI97f{opD33n84qe0 zN)O5u!YLSna8Fd*t6PH>4$#mtoY6I$Su+o0;Q^H058U~qJq_e~bW(@Ql+2Ne->lW7 z^iZDGR`1|w0RUou=E$epT`68UESWlo6=t8XIVg>#+jV7hdeBDRO7q9d1^oX!X8*em z%akjTV|aI55O3pu{zZk(#HYF~UvXv-&xy+^caRSOT;{tteL(X;2b!+ROKRm$gODQ? zRk}C3`!Ny<3Uxufa_F%r+Cx0AVj7<7Tht9oYXCOo5UKfC7Ofj+0J73e{Y=4kPu}ZL zK(on)-?kE7t&J{NINwp*cZH1+ksYNp_jpb#nAZTueTrrEY&&R2`n)W<<-92{X{592 zWb`@k{w|+HBH3Bdi>CXouYSoF&5)ZIjUybi$p(B3&NaUsh=rpei1?gmv%-TK4Z}Bf z{MLVeg#sh|7z-oI@18!Z5zQbWRn|o$eN|OWySU85yH5>$A2Ry7*nI_5ZZccvy6C{O zM3)KK;O!9-zM7B`ZpM?h?o&ew+F4xDqJ(s6HdA}XzMuSU$*z9qAyYdxVriIcDsO+raBHb;m4_l+!_`CFC zbDtXlII2>UU8h^QHdwL;6d|sW!LNq?px2j>3bj4mEj{WSJY)G(U;zlmug+KGqDOSLRi956YWZJ$5!_pPFk4 zq-uu_jgN%}1a#7YFWtG;qJjRfdjxb9B=2YiH|PNoogR1b>t-4o8_6;g%CH?7++s_! z&BOqo3GC*uzsP2A!OuJHSDb%0FxX6Wo5?i6Lg0GU4ndz5B#}&chi{=cALik>mm{l zyB|s3yHaYf^ybYUc|GVYF0GBlXf#8%$F?R;jg|#%chVinKfr!Q6al)|U}yz0?NpRv zDuUX)93x^ye&{t%iM4)e1|V5(JxUhle7Zt{ZNFm?7e#cNzzGc59U-I@1ByJTn;cWx zq;=4-WzpWhU!d1oT}wIgX|@R-8bf{)pxbXg<##HhJ1QX)Psc>~@hkd5S^gcY)a9Iq z>VgJvplP2clJx`LmO}SEI_&>3^_Brm{g3EY#^@1Jf`D`*NOwykEzL$r zNh2lQAzhLJQqmotd*9#x$^CQ>ceZoh=Ze>*TVC^?hl|@#FsiiEK?j0qLg%2=HlEm) zQf$}w8$Q{=m9zcP`73cX8!zS=;tIbd|EnouwKfhxpp~NwZEzBKqF~?FgOgr__UNkn z!2QMXr`)HG3lT>C)e7y(Oj#B)z;(qRm4&);1E5G~Z*QwgIN7_jN@iUuL$yWO0 z@0S#eJ2vlO+EX7Y72ei;&3_M3N*Jb+CSEvZr?Qt*8Z5_cN7C}~agmLX?IqYPNi?1& zg^I3VIEl>9DXz*Ah-=C#=Dg*WYT2BmRKTio{&7@bT&;i|ZpwOHO!w<=&YfL=>}oLC zu6x=AWh5~^AFnpP6Nu?&g@aN3*Yht!;BEb%l*yF-lGl@m0W*+v!N6TI#1?6Jq~@2E z*A-2TsS#U}GQ##4Bmsx7c^_g3BT4h*u=>WZg>3c^l4I0^?waf)Q?U3GiEVkDB)xS^ z5AnNvINV^?(7nclVgOXiTPZw0I)(@^ODlhjdmDH^KKo=3iR@G`C{MSQNxYKcUKk%0 zGUI)ZoViTf+i)@-#WcI`AV^+9i8_*9q6zS1_{;FVhpH>~l#K*XyZ*^X1B^f+o38b)nC~#a);=3;pbxg-3eKhCj`C!Marx}0jqvLvV3-3UO`j+!C zA4u7T*QQ@Q6+WHjXxHXj$-C#4MAT)y)`8 zff||eyc1=sL(-s?*Ud-CQ|EN60D-ln3ENIR+@|eln{scL)4yty04}{T%@NEg@(?(F(dMc>|DTdhxV$`|91YAf%L<;HwgRR*chn+2#6@Y zkl}Fx#>(^<-9oM0BVTXy=VI|HgP#UQ`gYRPZVY~{L{;=&2lOn8biE~aU%N7q4Nx=8oUr#za&MU0q(z5vHLw5SP!#WC!nY5bX3e%nhL2#R3vJ@Cx zY3G%Md+OpyHf{8KTiWz-1|tG0TUzBj6X$@h4Cxmm(4{VP=m=QW8iVMbvO7OXXH2MZ zY*DUDAYO?wxWXv0P>2$fybrCXk*Ly9eSFcJ`)5`!(EC`mIs%~Bpcf9Mkzl5#T+ z+ODfiWc?z>CzTS_DTmgvO}l^J-JDrF-kV3_Nk(svZHV@ z>^tN`F|$~0xFA|;EAXk*?A>%K64r@{viI(n>#vryfb++9tL#o);jfed0@`+T$$$J9 z0H5sojX$2)_-Prznw&U0ihkSJn7=P4y|uQ;`E2_(M`E{52FL3}R7a;DOVwl0)=$)? zq22cb7e?kbKZKUJz7m2=bOu-JXDy%34KF<^*8)zUW_-?iI9+j5?M?}L^QlK*x-l-N z$Ei{47Z`dWRym$8bhW$ygM5uh&7HK^tsY&+AfCaGx^+mQNM*Niv>V zzi3LjjWuZ0#*2?Xk&lU`exBGBsIs{(6>}FOtr~+DOD}oRDEv~?@FwN4EuIn8&JQqN z(@=F_M`gM73npVCwpm~yM5)&?ti}5Hwk%wx#~fzN|z&F91(+xW# zCa+-+%mL8>)I$`p+bh}s9h|S+yKTGn`|`zzQAAz-2yBR99CygpbfE@O>eWWCxeQ9o zY5I(xN!GW5|2++Ir?pWY2pxbv&}|MS8R?b%Iy952#quJzTSwZ3PHh_ftzNkGT+OWb zI{v%DbQ@sE%0K6*=D>tD@E>3#l9&|EQy<6*cIU_$06_~wVul~{OWe~nJenPc(w&6p z1`zuD`r(|;(V$AMG7z*#KW`WX2SH%~Y|9MNqHu+jjiPk&Q8OGww1J^hb|GXSkk`S4 zX0DtwyWDWW{GBeC7H_<`#ZV4gfzCKQr<0Wlq8!Q8vS80RNU)`~NphF;VM=iSZ)17G z9S6#aDRNzVxlP(5!QS85)%e73##P=8C4D#zqwhpC<)B=SCLeM1Gu@x0hgVbf$|5=R z3VEz^J3K2iLkMg&XN8Q8>`~IMujBm6-_?}5bsK1Ex@cNLsP(FtsTFIQf7{{Pb9iWx zpIRH|XbdH4i7IL;%C_m*{Ys)6%EzV2gL(U=MKbe6eq(u?VV__d1s|Q^r%8f2Nik6!l^m&S*mQ?wbWJb1A?o&C+gG_!7wQLA#q+>y>IHaqgNiN>p_6sKr=PS_3m=n z+%j#V{)tC1!+_MuBQm-JR^P?An`kGKB&nJSvO|ya+aLpWv!EA0q|jm*l1HUc&C{W# z5nh~~scyy>DhJhOg@0Mquw}2On%_#{|6QEfPNlz>Vq#9fFDLc^q(o0E_@X61JS**k zRqake>_IYbxr)y+Cghw{T{%lV<+)njHD7`0fkdV$vH`5pi4v*^4b5i;UQH+nP_k#m zr^?8Gd|DB*Nt3g^oyl4}txY}3tE0#`z)X!F`{+#W6pb{}Z_R%X3x!30jQJ|Sm}2HB zVprQSPLjEApgdM}^kxm?_3JTp9yAV>i8>5~g_Qx4WAj0^Le>k`0hF@mK zebnPl!K@in>2NlFRGBMwV=9FhydI_r_Yo88Jsf%cP=vT=aDX@=kZ|%8NO)SwdLglg z2MO?e(ObSoZtdgU${|^o{%Qb@^+;03i`O9qXAE~XSORu?daoW8@D8YJ8ydA#ca-;p0kDl3fi zbmiC?i@|j8$}!ZSkbi9!#++RvYye%^ z!BYd0JoaT?x0NMS3HyzT-GVv&0;285z}l{PpzWk&Zm##f*Zg+haiR|EW5FKF4+kYy z>y{sywOYeaA0f#&h+_J-(n$IDqg~7W zb%^tK!P%TN+dqQodF9boZF-^R!gP2Fo!u9$f3rX!n&GrSlrjjhTH@K|B_d*7M+U+;Xx@3YeSryV}#v z$_F+SVJcS~YBE+M{4uMta%}9j@ISUM)h0GmLlcz>m(PjBr~_Xf!XpXws(UD8kC^gr z!;Qu`){C~M@W(hZ($A-n&%fdAJ-pFW1SgW>*@%1d+hndeG}Vt;=;$bNxWHvvh-dZe znl@4{@rNuu7;*=yf@K^Bdf8*Sga*mNhU}`z{M*T7>OrX><=IfH;Ua`|1|!g*8hRJs z``%_-5IAN|L6t{XsnwL%$aXflwitxoC`Me?1u z|9q}jcR9$OaA&?BNhyOnBV<0vWIp_II7)EM5=ouwV%oJMAn&$==pmucBP+T;lwVAX z&lb$kEAnQ*iWbA9C5YvWqDZ@c)2a)1uN-M*y+!|IjJ=Kv(LJddkL zj}+8rSVC6YS`k;#fipFsPpz$|)uSP>40HUowg-V8X0}`_yV=mk@0yeyhSQo{kW~7K z(S}W1Wh|E{oRfw7dY^=_^<;G?2}XxE%#5hnYehMGeqY831@b=nNQOV&9GIVTU$knB z-G2~6Yl3s>cA@t>Zb)RE5B73aO_1rA#VOCTD`tDZb>w zoxK{+Hx2JyF?wfhNYQb~2XopX;hkQZyL?Ss#iIsM0{`<)jt*NAt6@ zDV2$G_ZmlP%?L@Vddl)^eE#m1GRBDcQnK+jVN)puVukb3Q;%6qe+mjng2uq03A*C}!^-T{mhIy9^S?iB% z{fQ`I>tZ5V*7Gt2vw3fkeb){E|C<6NH? z<_+>Zo63{q8$U}^DD@HK3*)KpB}wb=3tAH+pv-DecgLNQAQI7?7&Aw*1S-tM9wn<~ zO`k(jp!Ea+g^Sob+Ob^!opm9IxZ!>n~byg ztMvu&Hvep(D}_t^2gnm(jX!+>O?~H9)k&qeeR)h~3e-kb@|Gb%D1!X&d+(oZ#S;h@ zFwm3#wC3IuE9u8E%CX@;SG%2dQ`h}3u1FcO$-Ft}+5g+HDVm7+)%rQG+bCL4hc$xj zX@a3UTZM)IlUX9PQ0CK9k5H)T$kl%MOvKqi?eli{6ytw@e?8w{Js4e)MciLmPVuwU zMp$#wH?ON+`6&Txcq4)mzOf99G9=Tu@nRvrS)MnK^sxfFq!2yWK-Hs%XIY~Z7%2ac zNT2b6uDg8Soqbi&Z)V^FQ2_D=*?qe`t?$40G=)Yw$ZOyN75+&a{gtn(WNk~6B^t74 zssC$7(?uqvD|z|CQq(ry(6Lv3FO(0*O!)r&wigqmg^*+JSSLS)R)7--#iwS}B-}mfg@{*pm!qH1nTu#ppPOk^l+YWE|6(b9D0}$H? z^x1KoJ4Yirf|La_DsUiP%^KjbwKkb?hmyMAo?$JS@&8Q~v` zA5r;9=gvgJbbu7M`DdwKp&8BdU)tnrTgGwA-pKV4kHPTq*AxRTujJqYIe!g?7~Lpx zf=`RDJ`Z09rIZd{a`7;_RXW4gLhvvnuoKl${Bq>rH@kk%%(`3@L>uKv1dI8B?NGER|w+>O?-f%pG*~cTz zuAhAjMDCyZ(S~{>A#Clz&~5P$GMlXf93K9Es~&`83!!x*chWcZXFpMF?Y6bH3!sWb5Ie_dUm)=p z)B!A1l8*M zCF7AC?YQ5uWwi)1m`HTl)x$~vdDq;~H#d|xpCy%L@jpQDRX7_`gv-HnUm}KF{MYF# zkF)=2w*Z__rH!W-#EMO52VT{NNH}Kd*lu1@q;}Jq->PWe>Xy&kiNiM$*Bwmb3l6+k ziajJ*dC421gE-<7`kNj-(y~Gq&9NT=CADuJq7P7!Z`)5}qyj|bp$hfF@0D&fL=na8& zHQ0qc1awPpc98P0iOO?aRuF6np!fH{k97M3j{425~b=e2U*3{z|))2#gPZt47{ z;e!X6+$xMHie<0w5cM^%c9Mk3YpOKQGy*OvVUoIDgmA>og1@g!ZuXy1%#J)Gamxv^QWTn+2Cg69 z1s&hQzPM7a7Iz)t)BySrpPrqa)Q{ms?UO$Pab4M`-Lwogbe(B91pDLs$DX%pTxdW- zOkNHpKZE-mpPo~k#3gJAr>bjWC>Dx&KABLtd7=FtafmHP#u`?AE1H@Th$TMs|wOF0KQUllb{!SF!QB`RW?HdhAXt zM(f$%Ur&>~v((e#dA8(&kWK&C!IS%2OAFQ}2KXpax9gYNfXQ0S7~~kJkQ!hK?TXy zHBAu4);GM;Dqb{V3*wM?=zgVzRB2fZ8Z(YPX>Qd{ zJgJWz875E6T)1ID4O$J~lio#P7Ta7R&nj$ME)lng1TRavFrbV25boKLbi1)6FK{{cWk9Oj%JxltEz@bLSd4z>#> zGd7bku@s#%8%i^^)w)NA+p9@deyb&F4JPqr)oZADYLT0F->*5v! zTl=6vX5~{n+t`e+;i2A+?IaXg{+5ErVV8qvV8FWaWa))8f6l<*o@P{6_nUci+orUl zqeK*J$%XBe#!Kk)=Mm!e&@pXW*ME|t;dts410rALxPn*^Nx}<@mNpFm$~;xryj~FEhfFV=*r+zj=%4V3=Oetb@+l6Y zRA%=|q_kK)+53rW4#UxIymv1@OS&*48O?EcPmGF*Sd9xvJ59yCqcFM!O!W?@OV`_* z-eGxB^eq(&;Gej91=dtfIHF9pe105U+6OaDoweI$K=i^Oa%Yv}VWf3HQji0hz_n4O`Nd~$5_no^$#}{pB?I9yiVphWLt6lVE(xZs<((mjOHdLG6 z1wWZpk)Y9Uy1&ntwCq$?G>TOqRbc$K*!ptrw=%pPALcnlx%FInS+keEC3C^yKqfNk z0Q<2slYSTP4K`N5nIhvr6@x-jitDs;eIZ&+HBwRz9e2Zu=}7?Nyw61Du*2ZYmejun zctQki3iyoJnGnJi?d~mM-;KP6A;HWk>;|yM<*C0k}Tge zg}t|PmFevTh!>j435uwKLL0Df(`mb0jzLHesIE~gRtijHGlEsnTEDkc^o`_I-9Yz+ zR3&QV{OL?`wsi%g$wl#1!Y)62qRNT$tAUnT@`PZY$h#viDH!OEw1kx6ovzL!d8Rk8 zeou_B%7)bN=G%^sTgKEPoO2C3{5C5}wbJ&zcEIf2`rE8$k-2sKQ{I3t2QJZ|jR^*2 zW#}QD33Sc>Gmf%XHENVprm?Djf7AzNQ`6jx->oVAWE|&`YbJrGZ;28Lk^u6!I2IMG zKX7P{>E9lZEf^C}?1LwSaGOuM_S*VD3z`o~k~PC?sva8uiV#fCURMa|ZDggILWcN! zrQ#;AZN_oeZYB^zY;Xb#=#Dbc&|RwOAA2Qv;jz+6;ITkw@Ww5bwd{!aXrs2oA*xkp zZID;uiK!@3kj5k;Q8?Np$l@WY?H7*1;Db7=FwgAMyqUpMDTRdTJc3v4aRS zClegvgF;#w>zLzL~D}Od=H7D~6l4%+lcUs_$4qYVdhda09MONba=(T)!sh+r0 z&|?kB>1XhVj2C|hxBm)E#E+b5uVc85_-ooA-IEr#oIvcEH>Q^(zDRp+RbMN6MM82q zsa9H)!D>EY0*3jVSu-5b^?N6NSGCAx;qh36GQg*D)X}uc!B7(~_z)+}FaBeHl~goE zNZgU11x%`uIGxS;VOax;fU;(&BiDRX*js)#noRI%_x#JM8S07WxmX~ZTjq3Tst{dd zX2KhBbxj;pxV*y_7T_E%&j=(}FZe0l^h*Dv;qcjYMpF1>H!)F9_R_#$FXv-38fpbs zd!>It&Fxr6KZ3-%cp|q9cys8tXplY5<~x1`2qh+oS4yzTiDlX+tpLnh%KdSo_q8ZV z2Gb=CBW_dT2){`*E;mHr%_oESsfGl>%$%eQB=*HIlk}1!Y$hOTv0@}p1sHp$R&-5TqeYNb~yeZdl$5P}ivTb5d}SVn(iIr#*v)a&3&our?OVs^AJ zwJZy#`-?^w)@ifnul5YKNn_or5B~BLz-q>^ZO*1bZm(bw)pkYOB_-#c1}Eh>uSm}# z;>^tQh+QU-6D>rmAfKVX1NYwkzOu6B^t)y@1dBt6Bja0aRYL!pd$ z`qH>;8i&tg9n_y zTDfARn@%TX2khU5(~BhA`)?&7`=&t^Sgb`~;rWT5q-ft-_QcTt?BJ^PNY#FLj)-Wm zHj!+KWOe)&RF-S{`8%fpHC)BYxjL9LSUVbZ4 zHhzWnE2R%=iu2z_8isqAGMk8-s^189<_o9uvME4myv7R^?Ja` z0U)gW>4$L+f69$mWkHKq$?Z;V{mHyFefEjl*469*OeiL+P;G+M#y1i~3A-&Mt3bpMd#U&{Qo@=Q@P2taDg@0Wf0;g|sM6ym=AN7YGG_GNPqD{DWnXYNKPgSIF_Vo`A)ZH^#5bt~@{N z|C!y))K`A;y{3_n8)<{q7Vn3nkU~7v(D>EV*sUwjfX9pkq(&o@+h%&~R;>XdlfqJY z4wcq){oKsKg4%#>B);(##_rx`)>EU7V9HAR@h95dSN6af%-tk@+spzrXd!G==SH!% z6>A=9R_c(?82>|wSU0(Jmr^eq8i0kX_b6X zUT~uMA|&l{)s5+dW|8fv)3#>Osb~Ujr&*q$(h1it&LIhBW<0*wv?#m@Mb@?E*T|M; zag)q{{$T}0KH^wQ&UrHJ24)D4dqC5skgVq&tS@UAKO^tR6J1~sselR1#CR0epKfEd z*cDQ{WNf{p)+6F3l#gq9^RdpgP-8QGo zvpCl9!+DO>)WTWHWsRiH!Wo&8)5IS{sQZ*qAHwTAOJtoD{$ID>{Xamva!AfYfMfYm zH+sE>7X{1QK|sPAw%935nf)MVnO7r5gj)C}?M+|tz?3r2=;i@YwK!Xo_R=X*;&eGr z>%o}a<{I+L>+XMmA@2Xlh5t*?ikLEJXUV#j7O==X`Uma&Q8Djd0^Ch&Hz|FqiqR&& z|JoXeE~O?laKiUJIlt?IW+w6HLvn-#OqSs34}w3*8AHSn;zN60C^o938uNKpZgV8( z;DG2f95bnyW#)2o?%!{B1!_O3miA?{zwgBuMNnOF(bN#U$h%&`l);uzzPct=wVr8I zAVTZGm|k)<4{3UQ9|sG5U_dM}LwaD)R+fp)!txeEnGR7-M~NW_OZHc_o4%yPgQ*uw zjJF3bj81QodhBj{@r+B|{=6_^GvaLcdG``2LOy&Qe00@j4yJJ?Fz$c*uiz{@oTd8k z3o#}#Z=j-+SKOCB{2(b!Lwl)*tKaxqbA%d`52JhI54VFJXIve7RJ20=u+;wmhO^44jmFO<}qIa_i?dALK#HAJp+J*qoVyoQYXDPB8ccaGC0{C-n!N z@vyma$)X2rju3K}vRi6q))ltTQ^**I;*wCGX9N~_S=fnnLuz)d4peHDGoy)++-!(@MQuw`#O|DfyqK4JsOC4m^~bz*Dyz5+~U1@wcpv%PTI_`}U}S2r|dFh!3B5$Fn$(M%)xD}zAz1`>YfVT|W>{bL4?ZGpGoNy>8!(J#PiBP>67U!N^?#^P^ zHxPkR4~8EI(R7|M5txDO0^rIxvpfsbBqijUk;c4QWZjChf{?h29=X+LAGo~U_HooB zK$vt-Ahn6J0GN}8l?@P%#K{9p-J4W3=4XA@`$E860vDybFbiV;*tC%asnYC2-}{_Lul}D_ zeyxcXhudU%p|o#vl{!tin{3 zE%66jVtF(z>fAXaSDUwV@x%Z+F)|)+uI`(qu0=S}TZ}y?>Kfrxe|L-aru6fuIK!Xro@%N2Z95$2h{oU%~ z>&sAe;q$KJpQVns4fSHpOT(M z3XM&L4|0q!4|%g`j=ovvqC1$`XX5-hrFeABl3LWtxs?8~aG;y$Lfn<9pmt|XKn5tC z*#0oA&_zb=DlWH5ASj*+8}K?znu`VNH}DRQ9*OI$dG1|T`1~4K2JdTpywR()+bG z|M>(?s=D{^WXg_c zZ4^zJHZC$(%WYPbll~>Rf zl=$mKO+xdPX_7%WQSW~-NN}P?wOH@)x!=Ptw8{4${82LebPz;_bSB|5afl(jvkjyv8D|1j z7K{{2zt<<`hH?C+9KEc_Ij&hwTRO8_+@$Q+9=WX6KJ#3sj$htBU#jQny{Vy2r2e%b zvx5pRTXF4bs?q8!r>(ALZDM6ncPZ^jO9!QcwA+a@+6w-P7m?#mQ_5b66&&7&A$YL@ z2Mz7mi09%%DO<67X;@#|M~^3ob^vuEef6Bf{Z(8(VizGHS-`s`JgslBYB=wk_c)L> zN&*(0Sq$JoO$n2bvL@1;u$w^D$Pv4G9(%P)wS8;nSuHqRIqt4zCo{!>%i!oPj$VA^ zy&<%Le^PqEzS`rh+V?D!W20{z|4nuO3;2q<8I3gTHxmcrn>4*DeF!*>NhS9HWWbj0 z;U})^wg=iDyBb9uSOV_UO51t zs{~nuVS6-jz>cxiz?^u{d((H1e!+?Mt!&ygtfP1{k}Czm1oW~6dk?by3qfQ;AG(`a zht`$*nZTvS*qk#AmMNquB-?tE={yj*a%qSTQh)Az%UOj7yqWDHy-zLOK+ZUi6TWft zXL`FW>BC!lUfP>nl_PVHBV~iT?be0jG5;9JKa6ZRA9#?0#s}a)3>?I7drI`mf%V9( zJQq*Z#Gq6#kC52ZJVE{VqO>bwVb{rhiEwtm2Alho4?AUbo^TYR3u!JUXz7oYz1G)- z_FY%_IvmsJeTFQi(8mq@8j&_FXDcQ4p;7j0pqu!CJ%4n|bZElA5oyR@AiJH{!$Fnl zty&Qz*>(kP-G#ku_AZvXZpvP~f)rB@py+FtB$IPrunww&26 zR;?eYxshITTG#JFYbaHBYE_n;z-j*cYLu@n>||QakZxPEKmT24#}SkR@{pKK{}I`_ zN{sk^q<7?3SUFd}yZm*yYh_EI9iaMMIOn`?@*f*X!KzzWb5)%`n3O8apn>1Fq~N^-Z1tg@D65<^ePu*nNAXT z=JQa2JZ_9R&;`eRsDZzk9Z&C7H!LS5W_2!Bwi%kiRE*Vg1!Ah%gLZJ&O_OQXv_)dG zIK&}&9W(qMZ6XJiZK&%*1ruwhlA&^zC$;d<=**U?>~|0v-#3IM3OP`0fL0Wq(vdT( zf_@2J;e>YDYb8Vz)Te9``c`wby0;{W1=MQ=xS6fjyM4ZHd^2sx=NOL@|AlwfPFD2A+l(_3tkdN`wW_#JxE z4SGL4TF)A4_j8mRrJmG7!TXyF(ZbrWy$A%P;*wA55CvSURMh_mKw@Qjx0MgYEkk|B z{;5<1Bt_Zj+H;0^|M^Av6dX+PE5@4m=ZBp4sB4#ht&Atd935Rbu~>|H_N0qE3~gr% z>Z&H@SLaGir{_UG;T)0Sik09nP!xb1oK17|fY{pE21=tppS)qaSdauQ z(~ocK9mm*SKGd1Bz6Zuw$@BGlDvH<(!xRl{EprW&P2)klWTCBwJ1HwOa`PpnASf%8 zF(o5@S8K>>f0Dkz`S1#{#W}$J>~*Bx13sqGQjh);cVpy7=_kVYYN$o8M?vk5(H%bJ z75_rVsZ_rOg_bMy0%4Jk2EC|*&20FaFc-1;?^j=b8U{Q9|M{C2b@9+OHmrQ8uOud? z6Ol~p*F8bP^Qj=9iWtL^*HwVl0DXy}S}Bn*fLwi8Kt#o8sFsM_6{+#~@5u<7X6p69 zC?~pQQuUOPTXk&G#sasx$r$f2EG|xiINkdUPU$}d`C9C{XkA4jXLH$0waIYdqRLz9 z#$l~nr!VwJdFW?w8aZ@g+%&Z7uX1f`zG8EY&kAXd$|C`nr8BdGx%t+mw?e!mWmtKl z6+JTs|CH@-@#w76J}7p$LXn{Wmh4#I;{?(le=TuySxU&@yTs~EClZ$mb7_W*Jw~;( zDU(x`*_z+YK6q3r2K`); z7R5e1PAW7lpc^q=UQ_YyGMHiSnUm<|lFZP(LP7ZFM4q|BeI+gPX;56on@vb~s~TAb^@$Tv(v2jbQWs3M@f_xP;^N!qbte0O300>$Xn~ zjUqsXSKP{quMJC-0vQa%cAe(SNUSCkTn( z797NpH9L3!kqrlhywp_?JfNxX^91b$VI7x&dg4YD9v6tV~n?2YUcJ8mY0;{MCKIK z2#02s7o#S5Ryx?|#A64Hi}^`pF8)wv6Oz!Ytl=zQ3&Vv3m9s^^L;2nZ zzP@IA@~ZO5iZOGZezcr_xR&N=)jNoLADfIHY?r!Eaw75YIE9&LZL;Z_CG#oN6nxCS zEcf`j9OpwPQ^vC$@~Q4&`cd!T2en$n$R*-WD@ysnICh*^G=4Q0Iao}U_NkY)nJ}`o z5NY$}y_k^^sj>B3R-pzC-%--mbEJBd%pHGW?w0LVQqlIe|IJD$H|nTdAl7&4(FFd$( zmdEnIkLJaHN1*QA=A&2Bhaz4Sbna|cC7drK3<*yR+}uRC>kMnCOh@alufhDdg!Qbi zb-FJ-&P3S#^v$7sSv{L`+w*yZ%T_-0rN`k6e0Q;kgP-gXVO5+=kDRPuA3@JRq3A!N zjMy>4(u0tkgVWQfNc_F0%84t!(mOX4qZGd9fEV`ZX;cx*pLJ_Jf1V$HYV@*%bZ0Yj zI93zn5Z$n>%J?!OK!qGgFhaaPo0FSldo_^gGcR6DG~ES z#B2Zmm2xqH%|Fh_zbc|r5g11b0D zP1lxo_edxSIsBhHvzgrQnFRpxFE3Y@Km8uouET*ZFV7yd|C3sj!mJ21)BxP_%blb+ z5&r>-u##mvEPDoDEvg|ju_A4tU4s7N^MyDy5)ztgRE-Dk%lHM|P5OS$5kDS=Ra29+-HE;*)07If81AcxjV<9&5tFNk(fhM96& zc?>_j^dKe#hJ!=hmPUp!jk0+$Hv7lRGclR;SX~bt%ZDqJF;Fq zm;V7O`9tMU;5`Y_41>RK65MH|saAJkR5XEIep`1BqCwAZ;sDsV_G^R-!5x7bYFcY@ zdqKU&2hT&;JZ7O64D**T8K5w$g327m?sNg^)8XM=IH{76k;|DM14Au+`L5JIFUR`T zNDLSG1InhqKW7hy!gqBYH!u+cxJ_5cn^Q!F(zt2|)4B3L3-lJxrRpyFdeH$VNCM%Q zOst?;Bw~#MAFT6;eWmPmxg!B)K)c%bK+6!{-65}-dmfDvg+!3NG z^q*RDAK%~yI^+k&vtmtxIHNgcGAMgb z^yNEHpF)-$c-hq{IYCF`PmEC<5l+s=pPga;X>IDYgFu6b8Ex99FjC#wH7N8x#F_B1 z$GiRbIuxe7gR9zK&>yl(U|c`@`o>;ljNGI=Q5D7X01QSuG+&*XgU}%pbGNQqdsU zYBN(Xa!|~_;OBFt+?EcfZG3qyELH(OWLI19;gsYABTG$MlqL0`&Q*acwkkJlN;>ysGsTSv$dC#6$q%6tsgH;tNbs zMxMpd8P4m8tMV3IwIi8J*$FtxY9b`BN6u4nB&vi!$gy23)6kmp0-a7HGgxE>FJc0X zt(=GIYWUu9Z9YC!!s(-c)isf zb=`3kJPL+C`^8*n?lPm4?e>wS?*tZT1Umi&DDFl(gAg3LpF_{2bv(A-@V;DRNmBo z81rk;ZGm7ClBQ%=Q(D^jbM9!QIlYBVldnXJ9f@mzZjkGX{k0XSf~=VPx#|IjYt_~& zFqTPHV8k?gKrY42F{q~OYx3$^9&y($#m{-@c>=kO#r9s(o53pEa6L<;R^yfeqrw;LB zd$SlZ#S@tY$bW!8x@CBRkq*ZH!o#WLu_07;&^&0LylE`zF>jj3ubFNpglfny|Bu`B za8JWI>SAfdN)1=L%M_`+ch~XObg1snv!-^|zHx4agJYGMT9r4N1CIJ<#kA$w)wDiy zuop*qMx9Q^n-n&U1y-dPTlTJ%j&F{a6qKDR6iG(e2eC`}M!C8Vm!yl*M%yyIzN*c( zAV>RDP3Ywji1@`u$3Ug{Eu=~Cm=y1g3D~64F=bLVV1rw5OCLGW7)XxhO4EjSBNEJY z)#>z>hu{tgW7YffLjy3sF`0ARCICLJiSX>d#5~P;e1#3X{in`4L&XKn!cMH*)L3 z``ilYN9=F9R1Q<{nDoh6iKn?t+by)9rxct22E6*T1@qx7N=7uCU0<9GMJ+lw-haw? zbZ&R;u+W2s?it#eSni2MA8v&~cDI{Fp_kd}Hd+KE4*_8#Y#I>hH4Hki$Gkx{(mRou z@5Z&QJSx<|g^SYCCujApINOip=&HRd)}*B7=mzTGS!2sqbx5^Uj-$Gx_aXkP-_RUo_}#6%$*`yg;wAQlq0M-pdMFn6k*cy{1B;5!`mml8?6PPn2> zzee5?4XpNBBUcK^g3&1v<7cyhYa1eQ-d4g@1$uI&M+M(X%jdPXwX-$Nejm^C8TO)b z5o*TKQ~zdDPJFqHski;d#~Vvd`%uq-BXidK?|eWB$>Ut&pfze!DDAusNoBC6oa^*9 z4L%095QUzJS#$SLSiuIp$)~-wMHgKiSkn|LPL&PO1d3c?wG@LFe=lKg-cV)s+>x1 z#DGTtK6OUDkH}eVlapcY1|PxCR!4nD&gTdI%ZolU?LJxqty`;v)jXS%ySuk&ai@53EAA9`J#WtWKkNN+&wR?tS~HV7zdd_j zo0vx?-Y;}V$@Seq&OjkUYkq#FrOGz=B>B%=x=j^sO509CDnNCA)6CHZGl4r5qSk^> zWO?$(gY@N{>~aczIi%L3+*^$5oY7hktzza6dh93Ie<~|OG$sa|v<7|M64=<> zsw=pzSljv}Sdu_sAU{qZq1+vJXAMb;(aa8AocDTM$O=!E|9mtJeE@8#ABSO9`L?Sg z_~YU#1p*C6Et{5jBTn@3?~W_RHheP|kWjf#==9nw4isX{R;IbLFdDRi-An3n%~%BSHf)(S zG?Z%(V#}scGfO42jQRpdz7W;!X7ZEw|f+WE7Jarw2^&E|L6j?Ve0r5P80TE zX1iE~P5+@rYj8~U159SDjRrFYl88I{@20&BnO*wrQf!oE!*gQi4B)!C`}=I14_*ReTer@&Rqx-d$kb+jV6O zucVnGVH=K33vSlhP37#eje!iSD&1LF(z3q)0;6iXDxGj`!&j-B6VPOaR9kVCj-_Ga z75KHJGH+MRvq`y-U3=;@Hu@xJ5t^5X)N}NYM>0+z1vSJX?KMt4cPzlBVP)Y(-MoTB9ZH=}h z;8v3UqS8dFZO61(6*enwm{|$;xLPtQZuf*^V!WgKt2nz;!V8(45OO`qsb<2MrS%oo zrzu-%vZ0BK&Nk*djZ=;tOcqSF`ms3ORSjMgAxhuU$RsF@;_Xg#w7Tl?L)vo7z9MO_ zu7QkWT06K@c-vQO&=DY-|Dro^Y1^KDaFnHIjTtm?erb2McBkOq|Ma?ffjLVmT+Z*$ zMF`V-iC-$T=6yTPR9ZI1xjE5}Y=dsMSjY&eyM*9j1H$e^fGZHaZCS&}WbzrTB!Z7Oc>fqcw&N$as`S%#uQRm<6|%r=m?$B+e=7+44t$?= z8bmZYlA2-lijWn$Zck%l$*9+YU~+&ZDI)ZcqBYUV!$J4aYTK9=-`#9izb&!X)mP7L zE-Vty7t&z|)6h1dd?yh{b|6HNR7(ITS^RY*!JvnUfFhg_2e$Wpi$1i|m3ACE>|TMg ztup?;@K64*U%s;7$o$EbDH%@j0|7;uaf>F+*}=jOB)Hd$R7tv*(RtRmNr<0TajaJ# zIry;-zduL36@vj8cxsY^g#K0iV;`qawD9W_dCy20vQ%65+VxvtN-#EZsI#+#lw#YX z4;8SAhl~?`bw)0|1;(XD{L{FBJf1lj+|oaQeismHhqM_RES5Xp^91^=bId@g+!d%0y~A5N^s=j$sb!LI}p zNhB|9h}?zct!^J91rYdIg1U_4g8s|PuqUIYwWu&|FZ)p?{3{WlS zITcPQ8zUs6p*_hdh}G|oAFZQ@(jmO!)8w!HFo+})Qh6k58>9D~jNsi=YVD6B zR25**iUK&D9SWJ?{R2RH3QW{Mr-K1t7V(iA9suIn+_Crm3j7V2oOh5@sRbC|KW(Kcs`5q3B1KTVY#tqipo5RP6)dvNU5#kHth2ndzB5egs z`Zp;s8xeu=S`FK8S#}o~lOUi)#&-4+ z+x^PLFv%UqvKF_V^+q4!N8PCIbeLTaz z3OfG>FbVxXN_hVS{X~DI|JqHp<=JX`S*FWU_K}>-%%=TR{s9Cr^AJA-m>t6lMYN_b z+wZqV^JC|#dTpM)QC?AZ&ZY#zJ8+o-Xfopp@)h`9wFhslBkVVI`+ISX_k&^g@_E^# zu>oR1{;!vZfrk`$civ?Q1BdgP$HvZDu!t4M( zj)XQx`Xq)$M-@<%+B0yw3QC7z&5_9Wu=e1MnuenTXy)7$PmLy&U;#}VGb#rbsqI*r z{Qf_4E1X7$>=h98#esPownr5ii&vd!I8)RUXZ#6x6&9CEXIC+nS!up_x5W?bt%B_8jdJK&A)J+qyo~uE$ievj6NIlO=@`hUUlZ{TKIv4 z+tiM%N|Fwa2J(l~d)0uyBNw zCrG0K=z-&6n9GB^LKtaZ6SgI)(tboRA6(QHANsvX^K&mbw@+k&wbF8!v?%o>3Yt2@ z2?&%^Hj0v;qY@A^+K&b}_zmwkP2LK_t(WFBRArky^1&-#9!AVL7=iiYNapEg?XBN$ zk>vKtVRblL`eVXaP-o?iX<901qWG|-eNF6r$FVq=+!0xMsu%ju-h*0r%Njij-88TjbbdvVIPtWa)O<6V*aCUCgfTal;0FMA0Xu^YSkHMCtaA%W0!(t`{; zR0-Zl*&FHoQ#;BbfcYhM{7OS2o0c^d3cf;gEl`Br*%K*x`D9EXZ+1^?dH+!ov~DKP zN<*dUQ3|YNZQJ=8UnjX)t*;|xt$bb7QZGMCz~It=^+U@m*ISz!!i=Pp_zA;k(9NPk>!{bI#PL$ZXaCHj2rpJIhq`{ zvkuC3WNyDHUFb6h)>kxp#+$!TpOF{y1W^}_=M;^vloXdh8>uGk{!~=CCU*rCtk?~b zO)ySY5t|ZybeJ!FxaB^Lw*6E}e{XNg%6HQ1H(*5JOigSFQ;T$Tf9krl5&83lV+UOI zZLDj(J8|`yF|~5++c>*D`=r!43)CT8MOEo;A68n%uqv4{DX6jgtgExLpkb+{h`~$@ z1%2v9mlg%nVZTVrIW^42XgrNJG~h!1Fivm>Jq8lf%26|= zW?iKiQ5Vx_OzOO19RN0{T04HeaJH5d6ty&&sk2zxox|y|X z28Oy4Tj#g*DOnIS*uN41kz(!E`I(-P?@Pu*x-UNs#jBj&xx3xFQw(duGSygVn7{?r zHpsw|I^#?x?Ifua30HwfJHK>SD$rLs!f2N-^&0c_j9wj{^`C>H|5mmA-uKAtlKfNc zfX=59OPkiw5aT?yL5WsoPuWGBJrf*8YYj3SZ%bu}6MTI55!yO@_sx+SHF{%fPL56E z;rz6%#Fu`>QMqqjZNTy1o8}>RaN#P}#CHp+aHjLD8LdD1gpvm7zm$XZwRFC)r0-yt zvRlFHZ5qkHhY!1w_BMeu)OmzL3Nfj$@$VWA4~VM zEH2iUVRpu)NpsPvHkZ4~6r@7KAW*7YQ3yDq|I*KDo+6mW1J^&?^mILU5N*?j7$=3~ zs^{P(NH!IJ)4CP}KYJfYKCUr-E7^F6-&9mf@OPiBEizecx!yt(fwhMF1eX9K9eXta zQi}csw+?7M;jy~iZE&G4gKcakMEmS#7_RICYwHnb^VcAnD~@!xSScoA)!>`p+D)l^ zr4;kemi5mJh{1|j9i`={W`lJ(%q^_yfLX}Ixk~*MfsJGWj2G*6g(uH>V3Vz!WgM++ zK8(u6qz%ZpIu2Nu!#loc;%Vk-?jZijsvCL_{P}r-Vm$~pDjAncsiHW10$?6FdbpZ5 z%3-0x0YgetVpm-!E|$vt>cjLWc3k8RL)bkbKI-RABfND4RA1Mo+JyoyOH?Y-E7;T$ zyowWbje=-r463>CY_2eu3AL7*!{>`m$43DvoT!0q=U?VU9{}K#Jv}=IZkLMKXvOqz zE#V*y11tvEBAO(yCZJMEolrft+|GcQj-HT1ubWN3nA#MiiXo@sM4mJ1`sowzCrZ>5 z5kI^}k$GxK(6-*79lnus$Qu|GCujb}FBQ|PRY?U2^#f4@U9AoeIh^Og4&#Kd+q&al zK%47uR+jdK+ebF^nRg4GLOX?o2BpxZun?r01o&0V-dze&8_6$T>5sXsZx*JBKoBQw z@U#Jm|*3LNK z{7koMucvF9yaVrHKS#s_8lyM2V&_DLIXa;m$e>!FS{*6Ojl9(7R4R)u<;A|4%!(u` zrPM^dtkMu?oP*u#qviTA?m5DvR7sJTpr<>$>1=Zsr?B$Mn9gZs!H?^Lt{7QBb*Nq} z8o_zK%HQ7oVVii52i4;sRVC%25Q?Q?PdU<%2MP8W(hR~qN9>aR1EA_Du6J^E6Duog<3ABf*W}tpieAiV%`dGm)#NFx{8ccnm?p62l@9KKn;(#+iO@v9NMG*1Uw?|U^3?O0{n%SK`8 zz*S4KI2cSa1?b)hjy|?H9);3RMfvZ*vv)*GIsR@t_faAn1QZ{FPaFe<&V*va+uw!> zXnQYT#PVE-L-BSWoP-NwAnw<2M|4;zs_ldRqSHiT;hn~8&lhXHKb8sazMxe1w{TT)8lFiz&5rTLQKnQEqMpgD^BZe;TT zqCf9J=0FfRcb??umk-a|TQBf#^a8yGk^KH4C;G`8nXxbJsvg>@ohVF>_Mck8xfAEQ<9 zeVTihoe1|5EX#Z9JKaMP;p6gxg8U@*DSW2YcHPq2V4NgS1LT zS5)$JYWD}}|EMq+v9+}wJLnRi)qVQqRsRx{-h$)Dc%goHW<{2F$=3h$e-PIKUU!|19Ue|^{%_{zWG52vdwad?J7=n2g>sIb7O8L$ zMc<#E%}y}0Jji{erMcYdJ@e@=B4Q`wLBi3U{$e>&n}3=N`hzxwfn10v9Qp7xg-jX! zGfI7RYpC87tAcBvFq4`8UF)l`49g8-}0e$ zdU`2;8dpk(@nzxR_;Bkt3Bh(laJ-bC8mC=YQ z)IuaXNFL$B%+SVsN8GGjFv<&8e(gN}?hHO8C?}`&w0g~Q-tUM-s^mJ2Wn_v^~GVZP(EAZ$-KxU z_Ebt{umFmhCwD#dueDMeJu54G#_;NURTtI=2?26FGZ$n1%;c zmScQKpE-8SK2x3wE^dHV!XxToQ}k0oc2@)dSd|=n_I#lE6#{9n^si8{XM5^B`5TVV zXP?U$@16&bcgG7)d`IA5f+&CTzl5Kdq7Z|*44&0u#d9azL?WEF&9a;0(61|pPKy@~ zUeUOU+8YZudws!4h_mATMI-eyQU#^dU~ti85`BSim(Tm)_}{}M>p|2rTkK|8I6H=JR_82H@Bf^q#>P((lTMLj{$rG8iu+hO1*k8xSH6a zZ-MuRP%S0pFYj8qv7f0v-j|kO8%~A>3*Ov5eSe+92ix2G=lU)1oIN~OW1v%z9l7*xuF_ArPyt(DPFXZs$Kk7J+Tifal(GTic<`SV0npMS6KwG|gMFQVYj>%0Y!~ z%!L=R>Cmn}EPU8%`wORxvnl&4S-NRUwbCQ64l@U=3=_fXh^F6c-+qo`x~&MmuyH17 z$MjlQQD!7_Val>0127NcJ^?Vac&G%+3o+;E8;#STWl}LZAccO*x1#ayof9kx$!WW4 z&fY^Tmcf$FAB!Z%TmR0ZxF>K@j7(z?i?WwZk|;1#T7*F1fvH zOd$x1ylPz9dY1=twfH4%zjjR1gyGkrA#wx9G$d#7FLC99S7ilcrQ&r=vbQ-1zqnbI z3kj{S8H}{jZ{uZsUWqW`<{#Ytl&yh4e(q7c9@FICchw?$pqYD+f>}_gJl|msME0<8 z#th?6g=v!C`$9f5=3+CrO=FHD>7cIO!@hs)sD9Jh>?1LLBRr1UAbzk@@I(L1*%m?m zn-nXfJOka4PI6FwQ-&C(cULSvlgL@evd=2dw?fM?48|Y65sbEO4Ue+AO4ZVVm72Lb@g74vuVL3{ZX5!Zo&^orR7^3oE zjq0!(`s7NAfmeiI1QYCg8-0|?3_^#og?wo6L|6@*@6jX&=7(lO23eJQB_y`=^?qt? z&z9q>0g{m6KUI@qJ$jc;cYw~^r~>xWzN0@4fA5?}h^VA+Y*e&q(&0mJ)TI5byeSUO zDV~Q--oRxZGeI3mTEIWKJ#54YpZM(igcOllyE3B7x`|ZWC?;W)c5kOPmNUWhgD22} zWAfyu2KxKY|8jpxjWtW!tV==!R4RZiluL++iU^) ziJNp<#k$eZvYG4IEdTX&u7sw77;9)L00fd6TX%ZoMp-3WsyRo=Fs}YY>!qbcM+{yW z$Hz}6wMwhkVhfT-AN1O2*%KTpQber&J)HB>avnRdvPz;*j3Qrb z^a%_K{__5r_qT?|s#dOYDH9`s$O?9^LkrYSP|9Bq3(&~BI3woqxiGHvOBXrlCrI1V zs%bErI{psX#s!1L6Ds5t6lg+V#l#_qRO+rCt^*a%xT?l#7rW&*CbEoyH>zK+V|i9b zYz#$K;t>u>RR;xPx!eUTN$MAbPm&@A1k-CUY<0euw56n$_Lo$OzZW95uzDvl+Kwj~ zg-?=pc=IMEjI+COELU<`;)az~@d8%MiUVwZZHStR;hDtwcF(qb<#vlmyVvaBwzXqH z?^L^Uvkxy+X_-Z@?gV-%Vk`RM7nKx{+UEaBc0<|B7^2VwRUizvv{%+MdYrcT2NKcs zzZ;;6aR`BOEI$Sxg?hs-`3G zW_jI}DyUpCLlFeq*L>nv9W_u zH!_S6US0b%^r?&Eg7KIOX;Uc~I-?{K_cbaUXhnetGjW0DpXK}ODzo6u^`{Rf8C3J* zB!pd21j8V{gs@?6)gOlBc9=xAy=+cgCt5kms-T)-0=z(>3QM^*%5Q{JU^D`q@gM#P_h zEbvbvV*|=RmSnr9HlNG1x+ooiyurCLErS4_S6S8AaVy`wq)H(qp{|@X$Fz%I^qjqi z8bOSoxF%85;dBZGgQtvB&z}|l7B`2*}d;B}gWPv{o zMmI;G0k!?iJjY9$-tY4Q=}p z+tE~>JJJsm+yxha3}C<)7}V2E&32`Z>chK(QtAwNr9)bus(8&GP8g_(r#h^5&_kJu zYLwkFW2raK`eskNk&F-v(QT{1FJsA?^4m7FZ;CP&z>i?k82-!{$#(VZM8RVxyuzAK z{o6C&4{G#?NQ8PM-zG^P)`G0vGpDo+ij`&9Uhd4QB=L0;K5AMMQ1^3}mI|(^5VA(( zUeM6LEo;T;vWzlK0MVqa^i<4KZrJaS$7Pe9n z#iJ+zE8E&ZGx)O45*rc&HIKV=y)GPjK0zFQ3sZEL)8R1`KH&goX38qC{M1Wg*Cx#+CV3m+Pf8^A z;nXLlA9D{8&T6TRSa9N$&dure2&_UFX}WgryE7-fm$6(mY?+dGd>e* zf*h_qq^bF%*=ED0S@w5}5Z^e0j_2%ruDp|Y^R`C#hiou-E6>0F8z!fDa(GUc3TyG(4KtSgu2`T_PT;_TewX~LL@I$@IKioZo{I4JB# z98gALnGnlF;X7E?H4FY{N{O|7KXS^kgWgTcY~Nid8B-WPGy^-i14#+X>+YIQYzS#Adp$gDN3V@on9vYOLD?KE~judT&meD?KfURebh>AT!rRk>=(Pv zKrFwYHQ^wq4u@6NMT;x&h`AfHu}#J=a0rSdEsz#uslF%~1rI5*-b(y%GE_djZT%L( z*;vjxQW2ECEYoFfx*^Uri*(Ude?4f!B8luKz8QeP7h>c47cDQ}1Uafeg7-;$Z4*~o zjVH7G zAHl5uPDbS~!oNO$+3H1<+ym@KEMRMkB0B8seROc5oZ_17$llexs$97JfwMHedAPI< zH|k?9ZJ=8qvSzoGgN+0gEQSdPp@Rk&yd+Dh3bPUn86TBu?-T+2! zF(Fhn-d)n~n=Qou*$2ek4@64>fU>Ai0Cvf=e5d>2i#ZER?dzM*gJR^{H8j_fDw#k- z0?8Dci1WnR|Ci6~IZj=E4(_g^UW3OWaHDtZX@1>8$DVJ*+MCaEbu}>Bj={wm^xX=P z=-W3zj;oCqxqtL~%fUg1gU7xkz+(p_E;%Kuf&1D&6cs#!DD zAef7XOrJVseuHm*x>%YX&})a$JqBhFKHv5aXiXTl3Y`LYH%nejqvV|c{$?eP7J=Dk z6eRif*y5=ceh!z*Ci|xlWb0@VQ@EZ?3eBh`XgT8lCqc7Pe@);M@Y@mim+cK*dM_&) zPj7KZpt5(TE(%WwS5L7o@y#Wh8zw~+GN z_#Trytt$vdte%i2f!=r1sRR&&_@L*M9(B??LYz zP$;2kjI%$s#Bjf;?$cIuf}#HW%iGIypE}=>==*MEo*t#gd3}G+lC2%yhp!iV0wzIz zP=p20Ud*AZK>{P6aP20avMoOBZYX5oX$W1>_io|j>kq%^g|iMs2%Xi22@2w7ka`5` zK>f^+XYtVo%o3phtVCS(?h&W;prDX`C8*H~&u%?1-OKj?LX28@2x>K{)~ z+shgzY8tN4KY*+Yxf+Y|_a|Sh&tgw-Ux}Oe9qAptRiQUOOGifm`H@ShY)ccVlF7xz z(;d(gE5H=9@1kB$P$s+;ZohFqD7B%X;Zh!zGz#ypME=?)rd+4cyX?R86#vJd#pK&a zpdp3HH;2n^8l!lx30V=!sTxBe_2u6Wwu9?}bkWG~nb-Z$+Oa1IBv=8t^!3+96jsMG zxhCw-*^bPD(@5L6Pc1naYgLK+#saEhXvkA_5aeocO$9)(-}~r-ey~!ouKf*tM2T6nq@S5g~zp1 ze*v?cbMZ%_^VSjh5$R#cq8AuYLwD`;_+iwJG@RQX}mg2 zSE$##4+s9ru%>e)?TI}c*Ig}QE#5~3phNRWVHD!a_E#$_S9@VQ{8Aki#$_juUx%=iApVW_X5#u&_T4R^)1g1^`P?gWZX4j*AOhr`B1HM`5; zM6pF@ysY&4qKHS)0HvMWHpH#m(>MB59fq@Uk8&RSKr{-9xP+TXd-x=?I^8&CU4|&z zqFiBMg{^m)YXw@AjpPrKc9xy_R~Q&67uPXiQP%6h<>tU%ICA>D)NyqUkRl;IGn1!7 zTckO7|xq)9ojlrL@Ew^~aXKcm6EHPsPWdTgDe0`GskKg-e8~3F5QjdL(*y zc+=#Sw_Z7<1B|H*`D>%!b9C|tbbt=kJ*psI*T27e54kvIOX28o-Wei%(=jA84UA4e z^1FoDEZWQ@;bGMq@^p)PnWur?$3Nr^iv)04Dll$>P)D*Gj_&w3b-&=3ZZrj&VmizZ zD}pm6(fi;-8jhEIV6hUBK1BcIsp|Vvfi>~E>(T#WU40^97b>Cp%`gc#5o+hFG~v6$ z(1n-jqkyU$@|DqHQlo%X!{?ZA1Iju#V#M63@i*9#eU|3ZD1rpPv>{1Zkwykk(~fpX zO?i_Iiq55+jFao}Fx!A+xT-Wd#>&WMNx><72I3zjc`Ex+J{;#JPfqMXf3PP00XT2d ztT!v#neT-t7mpcdbzrff7p~e17c#Sm){}(~0Kz_wsoCpCnGGMrO>Nl%jyJG&V@PK- z4|!BTCyR^>S_A!s>0|~LCf-jzu?!BxaFn#153dR~K<-FVT?xtA`c+;iBE9ws=Rnvn zI=;_6Nw&u5*PNGwwVfLEas3Z)7W!HvEBsDQJ#VoEqL%y)!;sX zyU_i8R8cGyXT!o>E``D%snemTG0)FE|2yYM`RlShlTIxAce*uQwgT2Yjs_$L>JmSM zO5enX?|J3_(F@v zj|2W4+qB1|N^aIFTn$?}u&DeHmj-C`JO+mby7X|W5PYnz)!hGTNi+Yz$}g#NLkk<1 zQ>!2kQbet-E_Y^lDuE?Tu!fXYhPY1uZES3%v{=$x(PcJExFN}f4%UkNwY=V&Pej{7 z-}pnayi5v>h>f%{4WG$AgPsLX4u0ql&dzg}BH^C8bF;LS%^*k`dZe;2I`}q1g~2%( z^?PfE0?De2=heG!pVvp=Co*Hye4Q6pld>SaUC43zJLKqvqiwmbS*6dP%vQl^i^h`r z%G5rkRtBGR^{`8;3NaeSn3}keg3=F|y{s>?*!xon1fzr+A2srD80p7N&Tx^bf{EZY zEt*3$?iShgy9u12WssU&N{kijot}dasSrA~Yf4Ym`&LfQ^tPo6(5yNWR&(+00&XzG*EA5#!oj zAyjSNU|2{Djb^8-+F%Y)j?2a@8 zK?Orts!2yG7EuJg6(V=$#%wB=rpAlSx@g(&b=*1}I!V26Je6MNeRULDY}T#Li=Q-e z4z%1^8p*DkK>f5ZEHw;m1kW=`*pjWlsIRade>aKPnEO=n>L1WGFrfjl@QVt9+A}h| zh80Y)ruGdq=CrWrj4+q0b?42fZ09Z*&ko$*b39u1`&taera;Ov2mmappp${h4qvIJ zaCk~jYQt5wQ?8O9Wp2yRo5Z?7U7RcJ*K65kpUs*Y+OGiqEKhy}v_(K{cGWY{lwS}S z53%JVYK%)f>|7_;-6TOIpbpFJ{`nxVS-LAVaqyy5{Mf_7=&Sa-s84pitu%q!tQ{9| z&~RvQx%cTq^Y6TQz((Txzp(MEY+U^3II_Hw{m(^F=$pE1FKP*E40mR#kbS zBzCUdYzienLS4UKUg;yP+yG4!tjnO)sE%buG;+||OMeXpNbbrh%r_e=DmmGb>k6+e zK8!gx6$%&q$SZ@H0q;BakgHqnD%omC=MhH5q31bMg&)aapZC{)q3 zA?f?N5O;)=wVJNDBjy?o=>QX6<717n%U2IlS=Qk_emAFL99=tyz~48kfh5~MM|4iK z$|_HA@sk0S3gC!@1Qb>fM~S>nCVWVoS@=LTQA{#(OfHfnM!R=A>m(UHa3DHa1r*da z5C>EiNjNv2ppXSbY@j`=Y!iJK1?+4r-Z)rK`KeBjj&aI}W(IT<75dLx)$=0HMY{)=!ln>WZDwnf{mI6RMUJ1mxLk$w_ zIQmLIA!3;6eHAmMmV}ZSnNp%MR6@zfsQ!MAk|YjavWs%stuS8>69NMWdLDZlZDsbm ze5t&68ad=bFK42S>~Nb5qK_Jf^$iW`afulENp24O$&0uadObjROD#9M7(v6K##`=P z_Zy)Pm)_!3%=`-`La%TU=ZC~8YFw3y6Bd$C;d-vlo7hO+eU8ls`Z){9jSt9L0tnDUPl6gAyM-YlT*r8Ul z@bZ1aMjVdE+Zq9K~zU{&OF;u@VCg zz1CWZ+_m#jtWCuinl3hnZg%R14W&1qApie>twr|~5Ay#7RM52~k{rD@SVfHuwi~%+ z^$rh^mQKUoSsa?#x?c;4>qoKj^Nvt5IC|I&Q7D=eI{>d04_A)^L_yaNM%{9|O{l`a zXaBLDIOErb&<)+EKy^Q?mlUX71{}iX6m$rM#P}ROnQN9)-%_9A=(gRyO%V`^1VSib zHVSy3Z$pCg*7k+Z+SnbABZ82gy7Q6KF*lmYw72{HSIMR?Hn*MPNIo*eoK5rrZiGG% zhK#YDLeK_fE>7>D&EQvk$PYJ%CTS5LALif@Mp-Oey4OD=?l@%9cug3Q<#V;eWY`rQ>fM`+L>0l6V5EZP8b_;l{*QQ<^Ow?8kh5!5O6=%TjHy+4`f;`XLcd-_-S>!-U z-A=FaQ3+fLDpT2|&D=6iu{D1Y87I|nfB(n+_TM9N)h2bX`BsqP+xVc${^I|94Y>b1 zU=9?{47Q{gI(~7Z38FAmN)_sQ&kIb9&1F!e{0fmjNOs0jcUT@6-hF_M@KL67t^G*U zuV~Y)%-B*xqvRIb)9?(Ha2c12=m?PlU7Zta@JcK@Hnz}=H3#9^;l;!2cG3RIyAa;& zLOpMT9wp%^Jd|2%5l$om=Ty6s$M&}Q)blt*)3@e_Hxn=fZC7K~427n0l7ZwsMh(&x zRskO8Z@|&0hi+sKH%d!C_v$rnA3n4?Qi&di+$`GuGkSrQ7Abv%` zPIV8A4=+CD515Dg;;!lI8si6be3v^CMyjL&4YeK2+(^B-UNTqj>9g zdf+>Tst)At9QgNlT;zY#{sGjo{@Fx=v|8OM4@4kUQ3ZOtMx&NhK0~3jdNmQR9sDjL zYbJkwY^>_j#Ew)9CM+i<2Aacb;G<>$AzC)4m~ z`yk!ci?>VZq`ULyK0=xLMgNw;f6s^P)!_@HSO(EJsQ(n!va}vu3PiP$AirexGk191 z7CMy=Gs^VAO3H*sW;C>pphpLVQ%wy_>o~qS;3g!S3Au%|rq4V%b(%(^`d9;FKg;&Y zjbHqo82#77GR*6PXgdmww<;?BgX3n3`Ikhzb0n+m(3(Rur{O5BD7E(E>4xab&}1Wd zzsS=YR4Y5f4Niby1m}N>PwXS$;9&qjALX`XMe0F2)IFjCON)-hN{$I@;;Rv z+V_p+QuUfRkpzx!ex zT_tgGsZO-~(ByA%F2-SFy!E_XPoFoa%=(G${jfCF-xav;-a{y!I`y>K_~^;7ewR5r^IwG+&ce-f9Z`HSl}$Y*o-X+tqiQY0UHJ$J)pA6W!i*M#{yXzY`O{Q~0 zG>D$Cw}Rf%wgB-V>lU)4N%e!Zw%k|P{R9Pm`I95?Em@tU+ zdWsp83gk0D{26Q;LfSgVoKoE&P%}VF(}LXmu}1(>NdP2{3DqGU##uSUw-Oe}owb}g zkj=`}W=)DoFtX;KJCaSCl|P59p7$%zau(nwRnRw-%CK6?u{fB&v{_&Jrlp23H>2r< zRJ|<}3k72+1^5M(A(3m}Cok7I40xDP{fMA&NTWb+*~zk}EYFaNO3_+AoB={XOVi^V4^8uh%3>W zLF+hwTeMz(%e{ndzNaT7W)Gm*Q|?pG68>g*^)Lbnb=Fu%kV--+e!04oDG*4EE|+o9($l`2HJsq-Wr2pcowsRu{ymp`t;1l{f>NGzMDZs7A5mu+)Km9*%xXR^xfDE0j4HV z^4Yy*cNtlC@_(3eMF0#2=T)(_obbh|Md~8r)P0u0q`I4vgF;kJH^@1xJ zm-mkxj;2jE{LjuXcN+@24*&BbK^ODbjHuL_D7d=RyAl_*YPi1UX@!?y&TnnncH5JS zd6+sex$gL^O(og*xz>VfidLm;OgVXzSJ@QyH(6a9^$`dI$KmtOprn&i?+bZ3@91@pM;IN?P1tTkC?B3|f?S8#S zz+V7vk51twuh+`N%@5pJrUW#0QPoOu{zv+pVry4{TCe$DxovvjS?zpuG=2NGZ~<*< zd_ERF9fXiwK9?%u@N|I=kbt($n;dbugUXZo{oB-L3uJ>;y43#c(=8m{b(s{yg=y zv$iMfHbOp|=4m;E^aAVbhr5+mw^YBHoM!)2MJ1(2V5()TcMijwh-c&ZB&ju-^wqad zO`TooL1QY32?zogj_V_9pewRzk;J+gEVKa*N&W(&9`31jHu<*uNE?G`S$9;a#<@tU zQa@iRluorEdK?-{DGDVEk>lIb)7{tDOWL2T@$zUcQCE5!uvXkKkJUKB1+ny ztD4%8J24X*At4`{Etc$Ts`^tLh4n+JZMAE(No&1*+$jFoI|cX%lIV#U?oOelocAeq z;<(0Ldq&QKDXks{zhj@JGJpVctqmhH6B7Wz3Og7o)Zas&HQ!4d z#Jrqs%@T1kK)cQL$m3?{k3x*r%KcW1hYot{7wedv)n;rcbIn1*2T;)m*L4eh4w?f! zM5cmC!vG{D$Dp^X#z$~4`X{trk9w@7b4!<}eYNkr-a+-p@BFgQ&}F5t)=8`~IE+E$@9XdjJthWV-dfsm(JxL+X4v^AILXJoaZw&{6w zfShUZ#iD-ZPu;^ajrNH;%hzGNKde}wdFrW0$66-=3G>uP!xT+A=GyFP-!OVi>;poM zf+4zYBo>HwBh{Qe%&;; z(-IF#0yBfAk^*aa&ES^~JUm`}7KV4&EO=D4oYSu~ALh-Q1UTeB7YoE!_^{no*Kx0= z9Z?P$WgQ5%2Nn=Z&zrgJ!bM`pgi&e&b#}5yM~-)xbJ^_qFClucZ15-?Z0~n$IH#Kw z_)C0GfTAjl3N|F2em8fXA{WhF_r|qo1`VJs7~5p=JQ0Grt8|&~C`Df*EP^4PMIS+a zAR?{h*kFGLlN8%Md9IPk-Ml7;2v>u-{ZNOIGY0|`Cm4%QxX>EH8X)1JWi_R`8jCul zF~L5trD)=+v%Q!0L-x2=hWGb?3pGI;4H12E+P|vmqwDpDwIYW7y+}Ee<-l9>+Wfz z)ho%`=|Vn{W-Ox?F`$&erizcyR;H;MN`71K>1wyjb1L4-M)0SRhD}%-(pAJuu3&BU zrjwQfa%3ixw!n@0Qo)jCER?Dk<~}2ef2R{_A1E6!^hvY-3!~D)=Sd^o3Ay)kgsxw< zWI_^Jrsizk#gJb4vDojp<KB}4{bQHEu8V>Om&;3DXwQVvhv&(}0rK&4ec}Kp5A0cjF9J2-u z=c7=BxemP!r}=kS%vZ4y{>*|hVAz(bLEQCGH&O+Az;e@L0Wv`<4&W`ll=Q6ZUED79 ze|s@yZ2tn36W3g&1fRF@Ej=AgF0Mq&(|q{bkWR1(=u-Vt)=s98bIi6&6YrCxX(7nh z;OMHC|LBM5pg9=d7mxvB>8S+^s!Oo{4TLj|wju8F=@g(&H%%)JGx_Kx$|1okL>l~? zcjE`iPn_-M*2I7aWyu34D~|!3dhO>PMb|7d)^DIu^@7lQwfmuXw_;T>4O`@`R?>^P z&;XKB3oPiWq2n}WxI1AniePj2(`GY#RVj^13E{uv~WP`_1GFT1U@7VVq=Ly&{kC;{*OD7kn;b1!2^z=jEM=Je1>l(>dV+1NfI2l!lk0$LG^XomI| zAO;qpX$1&_dHJezo8d)_!ZO4PqbAKIT)t-8LQj&Rhgq?#^&HKp=d*=j&SD%im?QKW zbl$0A033=}H)-<(BrxY~x=g?n?Z3+*vv8;K6#HXf1WCniPa;ts>m+kY%O zr4LYDIusOnTmEQM1ogrv2SC4hd9U;7IhRnzA&qk^A)=2kF$4JV8UMl_5r~IS84hLy zs<;*=koMOnPMrJa(OKm5RK=s|M&r@ZKRbF^)J0SCYX*IvPAh)Jz0z(S|0iN&{y$no zvg`IW`mJnj0!tB-S#;wsVCSEh&Fw!P7@)a3i!_#U3R5e#i!0*&mLHUG;pl>^A)=O5 zAWg$X_`a1XgHiSyX%yLx$DzBg@m35+1c$O5Q)=zi4Mv-MZNHo#5{)WjJ}{mV(~tMT z@MKR$AZ3I7rTUL^wei{9k`xJt#%BPY5?&P!z!&j}M&|M9SD0|l6-M`w$-%;Y z3yHQcEc#XWi|mEeK!VgKL4&2c!3gQq4!;_qAgCz+YsUH`gxb6A#wvGMPO)MdmdVUeVYaJxh$@%>LK8W&Ml3lU&|zB@XimKRzL71To&%*pva3Ikc3!|^78TjF*?C<1%~qgKxx`eya^ zV9~HCX4ySbFof@N@k%B_gbeV)3FiSg-Jh5y0XX2yX58Qn!HM^j$ZKe5n0rEsw{Lmi z%)m#ej^sYr;@=g~6V#a(n!XCjrjWE_R|`wh`}x(QGF9?PHLrmBpxzCv51sm;}q z^PW#Ap?O$6cvAmYeyiwz!pG2G zSXG7FG5vaOei848GElV}px0%2Fzhf^1jFeDro*;Dt~=Jr6k9jl8RQfuA?7hiuJtDLK( zQ~0o~eZfRkZY^Cu$}GdotoS_mpXC5;fL%y>AlJvGJ7M85<B^;p}(Ofq5UzLZ0z6(VY-jViq{r%jL$C3b=9s8d|P>w$DamZ*REC&@cRe9bTG zMqg@K_Cp$|s$=#VAt~^U;ei^{dNbuewY6(MC)5tGQA3(QJ6>U7*s-8xml+_)tab~f zJ2;J-(+wOM$y-PpI@es9R`vEo`k02|8 zfDC6y3WX|S4@aZbJ;iCARB?EnT3>#>vZOX`C%O~Y9B92y2P;!Pl1a}c6$k;(Ym>;C zCgz2Yju*!&3Ex+5j=4?k znK8$v-VmZ$xTLPqZ$kJ~e0#3U>OEuu_btTfp6GqRF|4uVnoA2WUI34u;VhptKjSo~ zqE_h}oxOw;oBqj!JIQMFm&%#aW|guj6u6pn3mMV~&Jc%Aoi)t@tS(QV)wHCEiW-l? z;-Fdx=C)%H#2C(MIgbB*YER-?MRdv#exVC3=c601O|T08EaRtF#qfDdjXfv-9jgzC zdHcH8_1bxKh;`j9GxwBy4WF)aE)HDtD;ypL^;HtIr32eX-eN~&HfFKEY_8ru7R zwXTjG77yAr=XV<6?jo^Kn!M0V_tSu)(M&m8^erLRMY`xD|Dmaog{SBSb0 zYXH)sN5@s=q_={_Ms14QW~SbtIVXyJ=My~GaKQVX=M;{yWXgVmcg0yuTGALo9OBWc zIoGgi;Ixy50aDk#-~@*Y>d>v`ENjQI3d&EDS?!G#TLb<~8*1D< zI&V`#k3xQ%kC44HEV>IH3}O}lb54?N-+egNhA@q<*5G^n=u*DoT2N=blV)oGI#+0w z3q+M7e6xbHHfOrSCL-l-Gl<;3=rQA*(~i^QMvDdDl5wi|)U};%sa3m>l^Gf_rxV%4 z&s6C?`UYlid1HjN9lLMHvepbrXc~Yj9ttF6;E+iCAN)_|3bnWIwv_Jw`~?t+!!Q0+ z4{=VYTB*{sJ~WD^?K99)3n-_NG?=JWGE2ofaYU5@7|79jU9Kb%Qe1bfti#G$p~hU%K~N-ykG*CGG6of9A@D4-!Ka5Qk+W}H1g}la zJcN(=wSSStY2U!{xbzmmE4<58E_s8rXmW_>D|4rhYA!!4%Zb9sW= zca248?9FG_s&7lkpHDQW#N5Oe-d*C3ZZBQ3rrv!zFiH5TJlBjqn?rA`z--CB>y+uP z&0q-nV+Ybu(tu!aNfCN$^XY`{0W*A{)xb7#^S#mWVj&tLt(^*ZaE`aUw%uoFh<=Yk zrv+_J{fWmdA#}u9pbsqw+5*qD)gKJj{awULj-%)UHZ(wr3a0V)pm@;!RRx#++PCJ4 zmFh|18XJChfiJDt2?QlPA>Q2>$oS3_BKfC0Q6<7R`=(Y`*P%nQgOnxRi!)23S82O4Z5N~63BxJA zN-`-ZdSaEa?Z=v4bCyY_&SA!-u_FltH&igw-7T z1wlkm+pJLkkFQ_W5Z-sA`45(O!#^aD0=@BJ)8*>YOXFV+tjlYFm5U6e(e=iUunIm2 zeuZ8OjS>1*$FdTKCbANUeI+`AxUor{VD2fR1-*`qYJaA)1uID?H+K`3S!`?9FQCC0 zWFBXMwpA0NERvnz2%6{BAfYId@#<5`Jy=oF@~>-SE}vvNW?BNCK$AFm{+Q)WPh`%L zc82U>>a9%7ic(tX%Cd{7=*;)<+`MEQ1pB@SpO_qSKl$Y*@bd@1@lmacRf!XAH0%YJ z-toTW-?kD*Q<#rHYp@fe0}W3_!@e6kI{Y!E-6_VZ{=t1nf$>!XvaUF+Kl{6gTDePe zs4KmDJi4q`n5D*{u*>Ba{Z<;os4HgRNEvTA@$3N7)$R(9HB2?F994LO3bQ=xC8gtw zo~AWc9R#{X4p9Vb~#-bJm@cu>fY|KU}LK-nR$>B;b8CQ%T;4XJp#;Zoa7T{ zYxn?Ms2_*al_cIzuvH;JZd zem~qte|c#;W^mMR*y$Mv63dKYa;a{eI~Cj6^2uZ2nZ?GI!qYeT1S*BzcC;Q%8d#Y1 zK4cfuoz%PZ7DJ%+Fj1?$OIWm2fkw&kk>SLL(Y)7+y+XHIqq5F1=GrfXE49a8m8Y5K zj5f{XbJ)Ax>tbu11-8$&@g#i7MvrgzUh-U+X7=dqF(S(nMr26Fntu(&LMnM#ye z)(;3WxAO?M6?|nkC|L<+KKVH@;)zHf9Cz!am(W5AYd@aXTO0X4q_I^J4ky%?;d_o@VP>n>^bd zy|eViJkXdy`aC_AB%*fZR$cJr#IrP)4LdHKk}q#rm&ZI@*HjkSbikjeEQ4(Fe zpbvSj01E`EJJso_0LL$fO>5n0-Naq2Za8O1zy+P_FJPr2r&*R<>`LV}cy1L5XXauW zLu)<)m&6%TBGg@uxS{^D5*CGBV8hIwdZmDryHsvCAU_nVlf9=j&wN=|zu0JMF{AI$n*SSNd{E&Epdxyz9 ztk%Xv1=>yZK_e-c>xCwOMe1PultLJ5ICh}9R)OMzn$2ZD7jub+ z6H6Tn!TsSWT^Al=Fndaw0|L-_*Y*1vqxE|k8q0iV;1@*_fDX}f^Qwd!!W|IGS!#96 zADRn8j=yd!)tRsh;6MX!)#BuuFWVCUZUDQ>@0Mtc5e$^=vSfj-T$15ltj8gm@u81d ziJpT5!7$IytBBZ+OCu39d}?ZG;`6t0N`%Lf`=UY9kj)L13sMTODD^D#s6XX#?yVG7 z4-mO%i!G-E=##Gc?j+jZk&SF`tKb;KLpkSDT{h~&L(JK>MJ3ljF^5(e{ti<$ zlaOLR>dMxwD&qjpsCWgqlI!Ob#~aA|O;Ocz4>^SQHmOkCi5LdH9GF4Z=M7 zSa?o$>dJHtaiq!GIwIf*3+Eib`nC$ia={w5hZtf(jTaDi7g}tZd)No6umE#3UgH{< zJ`)@rG4FkyjoQT+Y_G>6kc^R3%D2CO#BKcNIiRN6L#RRrbV23tcGvhK2iuTR`efe1 zbN(a)-GGE>5W5o|i@m*zlATtoj!S1Y60HtM{>6Q1tGw)w7O<5&054~HBp!4j?Tx~ukf{TsHV zAn+<<-ej4k>v})ZEABv&yAPQvEG%(awATkLUcOyBqp_Lt-H!i6Ey}71Dmj*a?4Pg3 z{Uiu633XjV_4er{-EQ-p`^pf81OMpt4Q&8M``0t2#^guC;U_Nvy=ZOA^?+oBnT76l z4tkGkD$I!M{-TEc2vRtWf}6wrPKq)dDBN@X(j*O^OLzJdc0#I_P15N1IQ!53L!=X& z^QF4 z@3Ke&#b>q>;6gjMKQVI+u_z9@my%WKoxXfFx`vG1%V7VzQt`sJDjC`J_PiF;ODZX< z=bwRT#g<98ksA?t#i}w*5s`;ikUygM3($;D`**wY-wINX5RM2jDMeWB%^;HJBo=u{u?aHuE0i8ReoFZW<2fG` zd!(vXuL-3`y8i;iq0hM%A3`3@V%Mp9?ISOll^EO zIwv)-=+AhBo@)w+9pw1*kbBD3(P=F8ZJV;qDzy_#_%Jpya2E0JDM#G7rswNN=hMj7 z!=g`mT&&gP5D$iX99Pq@c8XF6$&94-Q!Pa*K59m>E&ihfBv z_s|{{(q+nH)0?w+bk&SG4Y*%e=dlol4ryFmv66tv*C$V?nFbloMM^wO^l<;eA)a74 zrS6`86cUe&bR@2KANRDw=^@(|Ts+#)zUAlPJDc+Jbjx8bc>BQ1WRbERpQdd4muc}< z!w%CCjlP@oMavcS*~s0~CKKXjf&Y$PNxGw4&0*k@4*M)!`S)Vm(5e}XCh1R(G&sk$ zzr~$!m|5rFzzPgq7m-#Od|Tr|DUc`gUmQEMHkkjca%$r86}e{ZP*4w~HNTdhy_6&? zM*uWnXDv&>g)fih_~#VOdcrZ7K~OEv0Q}U*T}MOwRZ7$rM!eCb-b#(Ji(`wu{O2^L z-bz++%FvW)c!5+l!+Jq(Sa*R3`@_21BoX603cM5vn>85M&`pjVql_*;59zd>0O26# zC)sZ)@Bu+ql-c-V7w3;e`hNjmjVZQ{DL+6Khi8S*p5r()vkmV`w2FFOX?{h8mHifsU^Txz>B9;6ebT_gsf4}R zCgZRQt>Z?TD>PbD2=9ofg&n-GY?OnLRODlVjwp`%7gasestO8ABD z+ac#PwpP(tf&{b2v4e|Z38e%NPT-5=p2I2J7cq+61Rq&B-@)>|t_woLq&Zg9$~w0c z!Z(X#o<>&po!}}!MinKCIAO>~Rqa)#l}%I$qM+{~UH}vrwSlB;G$`J)`z}{TGn25|P>OU{y%4{XRxYKcdSe7k?VxFO^(%Sln2zVl#xu1?`OU z(ZM+Yo;cV8mh^V}a)o&c*VuGiqcc%2M#k_lk!|o{>M?36UW&&HK3Fr-Dyl^rONA|8 z?KyuRBj@Br&dC<}Y|WYAV=@ zuU@4%mdt!&ywoHzH9$8H_=N8;T1^B8Fc_CxyTtZMnN67fNTR1Ep$?MIPe7S!O-+7X z_4dVZZJiZHTMchUkNjc@B`B&nvB33Qp~`dCKEImEY0V8nqlqnEED6QxZbRv0%BYLW zw^nHUlCMOQW9a@l^UhDj@f{=Sv>#Zgq>M6u^0<~8vAfpiFW`vh>v1-T_eeeolU;$~ zN2bH{>nb z6u-gQtsjaDlsf&uJnk1t{r&36DLZ^v!|LXTwTrgUn#W$7e_a*=wLh+I3z{{dc#q zGvB-$r?k>f9gi2=Q=l0@La`e{v(>5&(7E6_T+O?ekvYY)XVwjxbA6F* zuUSi%rDENTq-!LzkKG+vvfcHWzBp!-M}79XWVkr`_OUC_k0*)OZ0c zSS#L$B|DAQ)U_1_FU~XNf8}FVp|>Q?FD{*`RzEp9FY;!Z(7KiMhTiI?W~k?*N=j-v z36YCNywG`@e5&1Eo*`Dq5~hHAY7nhJ5lBQZr^ZBVI zM1~UzKyrh#H6~P@HfUfc^G1h78G@on2fI24M6Aqua>V^UJ{Pa&Ur|$+F+0g3D6f4N z{+8u$YGp*MHX^u0;^s@Bp{+IBH1Q$`!F&8($D71ZEv!lmBG`3mdPO}>$WwFkMAdmw zW%{=EE>3!eE~hTM+F7yT{&Py;ESLX>94$L$JO5OJh##+(VD7ge0kxK-F2MsiP(mBM z!cqwW;)WD(ri4u{;*-q4)w&XchfW_N>ui45IS|{qd6Q<+W5rJ&(drYKri7!dg98gp zfW3*oi67FtFUuM;Q46Uug7nnp%-m)rRdhHcS`02UTP-JWuu3MoardaaWICUjX>|>j z@jpx~8$@KD07sfm$-fp;hZIGn!(rmrr~`qS7#Xeh3-B1@ibFQlOd@n7+RL2}p1iOX z;5>pUyg!lgp0T0)8TdIz1Z(knlGJRlK*|G3clzA|2pE27FVEG=~bUUEW@ID1Yww82>Yd=V)+*(9JPVY|3-jkWcrT}Wc;Tg~W;E-Qb%PVLoTj9dc zFf5e%gjZe?mSkBKR*)LN*-b93S=mS$qV_%>vrNJuCL7CmXmw7rNttKz+*J^J?b4Ja;*(-n9N?6}om}Pg3 zTH$c~z02Q}Hbyf0rihxwT$OT`hDM*B^A_Onxs8TsqvT(E-xMBKuD$J25th!auFAyFs7di`Qi3SoH6UjCn2^s~YtoUUzNA87Vw*xbDNlW7~ThVX20UVs&zcNlPXX)pnkED&i z1?FE1A>STeG_RC()lJ%FIZ=*jZpN`9y&1&FFb&e2b|9c6;@4OiqX-=4B|Yi~*xI~X zZtxm{b;uWOl)R@yKAF|2|2{tEHe4~zo~T|6EO;%fEm8997QUfqv1;p%gqJ z6Ep9HMI45_r6BvuPyx=ueWV55;!-cK1)h{|D1I(-iq1Y#6edpiEQ8NLkdBb9%OMm=i5z~+FxxNIJ;dB`Wj#shxTXEFr$f_ zB5RYH8L}#b)z>NKCU3lP#ungclA6ph1N!i<=KmM4UV8HY;a31<*}+P--Ss?~H!*hcxC%jJ zWtjOQDRQYXQWiHZ=10=Vwf|uOF0q3zNx*U389u8EYfoCIKGJ!17K#X3j)B%G@6nYp z42xbbhCHicXVa*==7x?G5=uL7iSmAJXFcglTp07W zCkvE+DmV6IO@@}vkDwDPct0-7gP(T)>6_~4`P!BfUyaV`U5sI-KpdSbk&_J#_sFV$ zGJjmr^w6AT1ad-e@9@`0f3ujM%jzA>px8W8Cm%mdWG@6WIM~HE*J*eO{Uf`ih98I1 z@N{D4a8vS=FJHGYhP$6qk_)~Ik_l4OP!GkrS9>#o?Qq*^SLGY$?gnug4LUIIy3SAP z)d2hV6sCRTjX8%m5%9fG!*F1r9CEzx_2r? zaTjtDa)hV(3sAg!N@BtOC33*)xu3wY8xK8$XI2N})DH zp~3oek_LwaUJq9F_wYqdf3W|bu7-*9w707A@Q_Ha3Q=`;-YlAsL`&CB3>jQcOj+)) zAWnu2W)*zPet;~NNu;BzbqIjjbD+BtcZKAg>-SUQNsO!|qCq{-S!1g_X zp+VTqqAiSyt3NNA9w5JvPQN;|n`D%e|1V$_tB=S*aq|PiZm?Ll-CbgY;A6=1D7%gFBvynpJWUtLM2*viGZW5>Z(4e-_^? z{n&AYrP&blAM4Li+r`L(Rt0gYs0(S_&t(On8_m!mlbs$brtIQi@i!p3{T91e*FBr% z-KC@thN#%lVQ26|J3z>He_1YDuzMSJtTM{2{aJW28u)V(!OdH;-(#7*0Bx#-B(omy zI31U%YtSLEDpcDy$yMpXKq^5UvMxvoeMCZL zG)*X~aix+vuX<=n=*o|y!O%}w5H{_s9nW_$(4+cteB}Y)P(M=L=r5T(eFtDdgao(M zZqrpn7gE=FUfOXoTyd4>n?l%$h{C)hShD!a_E$CM zW*Y^9SuU^@c&XqdO=M3l%U$0!eQEX!GJ(St7@=B5a&{VStCDjVog%dFfz$XV;#*+N zY^|a(uf9U3T+$h^Q8u=+ zsmM<~?d~V5M440YHo}VzK;L!nv?Qus1%;(ib4Y0lY4+6e&39_hjd4?!!zA_q`85d4 zmGZ-FS)WAFb>#?(n;!IzO1nYdg1;s8^Z?fl^eXM3(k-nWMJYg*y%a8Nk++WMghz!k zP!frdj((JEkf~8q@|KcXVaP@xBlgZ3P8@G9AX3YkQO{IE#um%{S`t-eKcVi)(@)7R3Sd~@Ow!-Bp zlXLE9^T)uF^tCSyh!QYt)OMJ{h*IHH;VIJ8xkyh1qd$5)@G81*KX)&usu)b`&WdLK z1xUIGm2azVb-|JO5V`(fPZaJGf-zNsoVN^tvcWer@F-bs+(oMdb zDJ`6s7yw3%?TMIv-z8|);TS~tpZf$ug4KPZA?Dhu8X!a_yjvx;Ze6YR6|1JJ?KFi< zYGp1%CW&f)k8J~ECTcldMHyJ5HkBicr2ZsE?LxO=4akbDCKA_}0Tx%~tSKA8Y!;#p zIkNQpiQT=+CxR@DC^B9JP;Wo~ocfl+f7-}XFI?^Y`GQr`@*8;YRW)l5DO%gujw4j4N z`?=zH;lSFvuDV7kqARl@p(;@|+sGlWl9_37HA77%Wl z2|l+SBFJI{;+l9>Ua7&0nDuiara{)EZGD!5Xno3tl|eI=){iLh7eIUSMk@ z=f8j$0iX6VQ!j2L>Ex33+00$z*#fm>vxYW$`u$eTDz(6Rzsz*<<8T(VHI4Q8I@H<4 zM6PBs-p9$+ilLh9JmAo{RGam*k^KFRm1U=*&Gz8ZGOU^)0(Row*JV~py-T`cM!b~ z+wQXwNcfbw%F#TTXN9OB{VL$KtS;WB!rdq==;iS|jmX zX)RZaDb(qsHt7E%i3+p1+7jvG-s3^J8yVS28YdtL#UJx34Tm<$78+P*+;jpB77&dg z{sr4AbF)@miMj&W+v3i zS8pkK<27=$5;&0hBl~wqtrm_-8WHl+wCAh5cZ!-}^ z9Iw^Px!BZI&Z^E-pUWbzaF8$A-L7kEFop=b8W#BhI>@-6KRr zAsy3|!ZOUf^2}dNRxO={{F_M{8u|0r`c&;Fl01ZADi47r_wnpJPhF01YL*un0HdUbvt~_) z!UmTo&xaS0)+q@4Yud4<2LPY7|2=qw>V) z@%_fQyzz0Zw#cyB1P4Yes+1I8qY5xEPy569JpIo72~<6LaNlvSjzEhj9JC*D^}l`I z&voHJI8AIt4Z;8Ju;#q#c1&Ubqg2i=9S;JfQx(Z8z^PxPYZaA|OzB4z6UtA3c zgl$d~Crj%dN&**!C^RB7!iOxl3;h++sD%9D6sU~8<`USlc-2EAD^V^#js;93Jxvw` zV$w!BN81p!7F8)Vv|Y3Z>@~kSq-_E!ji;G$V^Wb(wP`4GVOmlewxUyH(ml}I)$F;^ zg+v40%@XEbOfbh3>4zz!MC~8F-d5B`l;0L@9d4Lb&}ZGhO(2>CKcbYPDrXhr43+@l z$2z0R5V7@5Du^m76X0*IlDfXbnA5>@6veUxYMUHg*N24A;hdy9iGDt}*1&jR*78D> z7XbVPI7t)YNwN_o5BOfY{B9FqQ>pj4qPu#K@6TohHnB@gy&^vjS&)n=Xrz|GKPUG- z27+m0YFPXeedBO=#&%%X@V-T;X2Daa`r`(cd{_@KONuhwREtrG6aUc^mHML-sWi@T z6G`GsxaZ{)Y;2qthnTJF*3vX`9nN>KXvj0Pb#zDkQ0*y-^PAOG=UPRdd%tF!l8$iraJs%%0uj&FFjEKi6|6HHv>57 z-@sx)ojshCxTuYXrd3FGQjnTNP-8I)uWH+mq*D1j26qoEK4+JkL-+Vr zAmi;4thR(p>DeW&pb&Ez^?ADKCzAFjnZ@t+|XFQ5AN6ZKr(E&l>K8`I60m!Bvr5uU52KE3A?MVB>Y?RvUe zJHH%+M^gMuWwU|DV1g@)6nj3kVigV@*2=&(s}G)ZJy zjY_{@tp>luAVZo;*C{01(6f%b_usZ(w4Y)XTx zMqJ~x&nbmumP0qoUbcQ~3FM}`jnjGHq`i+4Y@Gj=skkR`eHb^dI1@gt7Y$^7lE?kQ(}qs*GbHGVl#eONM)ZD;wbX zXC`zq7bK~%kK^JF`~JXf4+_}?8}YV<7)hZLotI@&L#MA-cMeQfA=h+pIzxkdPct2ftB`>AU936_<kbujeZz~m!` zn}5$hDym#Rx4atq)QK0YcDxGCD<_AAzvh1gh)KhEM0{STv(leU~itSlULzga^d?r;fp_^(X^_?oJ{GkN!Qp|c~tU!G%-}MjgpIL zqRp1UlWJRv1Gr(U8BhM759r2zE~&qR3$9wn>TD+~`1lu)MZ@$fPYHk>D^3m`w;O;LVh*#p0gc4Ij-AJ@8 zvEAgkHH$?D)Tv)xSzS?WQz@c#%QJ@0lMgbhp{i4i>|$cjw&TR5-=Tx{zhve;1OXZ7 zJyM>>Jn@eYE`--i32uWK1_NdqaP+cCVjBD;1X=~^Y&*k?K3CS zvr)CmiDQ7{3jdF&bBc~^4cl$lQOCAzr(>gH+fF*RZQJTp#dbP&$LXMBt79Ac)ZYJp z&bg_px~Vl*jkUh_eV#cXDrKoKOoV!_6L1MzGhDcl6xElkb$v5`q=RIgIAu+h7b>OI z-~-B&LMOwM;ptUh*iY7fH;#8H=32s1$?Gj~vKk{G44YVCILE3-Qbtfenz6tqtwuct zeNtZ+rhcG5>bI&LgLHCUJYk7T`hVD2#go{FX6DnFh6*yMP@+Z~*JhY<08+S7nQc{f z;phv^ezem}>Ct;2>B-yK@~5g~l$erR0c5<>i{N1q1%L5fKU#(jAFhmd_}rc%v48bW z%-Ch|XrB;n-)-zLSb(ME>{q0{ql%K-g8?q`8<$q z(<8^w@&7r+E;nf@*HhzBrQm_C@D;g$%xpPahy}Qgs$G4_iVEvQZa zqMc`9*EScvM)xn61Z5PWsZF3z>$wc%g8OxS?#O)tWG|;SA2?N7w_Oe5_i@dU9i6>L zY{lqg>32%CgGLg_UkP3uStAkohehy9C5W;Wrx2b!Vq6JLL!^b{Zv39f?p+2|;ncRt z>jyl+$*{Xv71E`R%N0nNNeZ%H((guN?N6UpFWSRDCKxfG74$0)Q?1|K`rcD-1Q(3l z@4TK-S)R#*`IZxi!=G9a3|#d$GH6y2xoLny@xG63rrn>sHu+PVblNVuDu)6#9;fX{ zyrJ{xr;B49^XcX{Jockq0*`S}IuvDj4InNPh|f^rUIEsNYSxRfCL@Ro$wjr=?n$N} z6*Z-HAX9{CSd1mT5j=-6Q${_i9^=i0$_V~_W{|CuB0`fmmUpm#z2b!{-L~GalJ{zy z&Mj}{-@)a?a^SvyRehN2X`B5P4G6K)V;Hv*BNxo73sH&r{yeIvswA>(7gsry$iQ}J+eNCb z2&^ISATXJuN2J%47$*aG`)130lolPkfe|VDCj^sm0uElm&m?JNQ<+hm+68b_fcU=Pl+uI404uS=AT}~`ig^qJos{M;bkZybaPcThJ%H2cMIoeBx5vYX|m1J zgOxeWmt;!tGi5{s=ei#Kz-K;aGY@+=%jicLfWGqdjDBm1tDX&H$!2@Icbg`krKZ8! zTWgDrJ)4|f#*kB3tHe)yCXeW#ecWwri6StEhU~uAoYpEgr!GeNZL{oH$!gXGM=qy= zU2wgCNLVNMj(qT6WQrD}`hSmcm!D z0K1v>Jh!@KphiXRVfLvf0uHNnpAf@!jcQ8vA_s_yI#$zG?-#tPdbH@gg3Jl;idY}p zBAlie2#Fob-Yd${wMHOpspx2&S96Gi8j7XS{fOi;V zK1}p`xcq2#XtxC3!I_Iu__~O8*vaW(p3W&jLN`igA3BOeg}c)0HNFw6q}H~T)3zrZ zWIyaTY~_8ZfvIPVraw2j*vqLF+6@)JxX&*@>PIyDig5=q40m-bQ*Kkaxs6DZDthCElNh%r28zTSSy|Lde86%>&rUjz z;&J)tDsFBTzEeORbw0gut(&;sefub-tVYAvRE&YRye~Z`V&wj_nGH7ETr2-r>(4zN+7RiifVt*8|75Tocpc|5nZ`sccFJ%9VUm&$yaD z!_Qrn)@b4bE+PuYpC7jMtm~}b&b3Uw4-W1Al|pcpn9p$Defo2ANDD{jxi2r3enS|t zwR(Gsv5k0xc?w@~K@pjo4yBL@$qz7215e46<_%h6<#cMa#xUj5My(V0ZG0=UwUF_!}!d?xTjB2jHzC$2fvA0&6%~45_l1a*KIU<00|m?{Tvo$BbKiIv)+>i$w*R3LG{2 zmvx9eA^fR3-$1-^!+=eM%acEuX;pqrQhxD}Y0acB zAK}zUlGa?`p&GMel;0W-DwfF9PCXr81u$!D0tW?{_C&zJ_B~RjT}jwN=v}+ayGxEK z+%HMDTp5s~9Am_2SdSK`n9|{!q8*{4N*(vJB`8AxBN`>A9N8}lp~;Xik*zZ|N3*v{ z9D&RqoG7E70)(rH)3dwNjU8Xj8vGGq<&dSELoZB_xNRpF-Y+xDnWyYIPJc>d5rECZ zCE{EqHPyG@DN?}38Z`(hZcIh$ugS@21i(tkqUM}0ge*e`FLoS-sK0`|D6Ej_NZ#ux z0O?W)`oU#Q-o9hIob#SeYg60vxZgRI)=P<5g{I!*bAC;0CKjA!WF9dN0Vot=tk;y_ zzJ$kg*lv(x(lFIhrAnGRg4MI&{cd&wA|;)-ayFI!*doz?jM8v}jJKA%pa;C*6NEJT zm2GqMHy^X%v~$h8=-7L>=^wJ;JFv7fQtcfa{xjPw9k}4bEI273!Z{yyym%j6Z+aFv zs>Qs71LH(v?HJ`0lnEar`ZK}(TBPFE4C;}jFUWwVzClE}h2zUku=yn_tC*9$8T^!_ zB|gjr=SlmNQrRxKM=1IL)up) zSIO~-M1kXQpH2^qvy(cp&*YLbd!i>$!llR&#@*n2i84@JHCgWXwrB-46W+A^J}8_F zCoJF-An;C$Ug0O*#b)hQ(gS&h3;dFq)nl{lfnFP@Rqu__r`o-J?&lE6aD2KAkc{no zZJ$rUyR@zxj(z&Z_9`yP+b4eP^D@h-_|b5%;!a9u2i}BF8j(M2U)9edgGrX%B1mr| z#C)jJBIfYYJFi4ixI0MXWGilWFnG}0uFCnleIuW$Mt66wV(1NV4QKakBP$-Sc<4#k zW&0^a37?o(-b3Y^{Btj_!_Uf3MTh@D96}3_N<{w0&mA{W8rWlaeEM__`nSI__zr+p zb!>(@kcve{mK&PRF*GmM%VLPEm*4DG|5#~GUnkfw}pAd3PUF)E@jhhLpAzsQ7R z{;^05TZUnh(hVUF{0CwbUYxiSw=c^N4+S2<Gkp`}tETuk5(#|JC)t;*qWek_52P33Ur{cR5*a z&2qs+3nrBjAzkus!9ykPZ;3qUIo;e2pjB*mwZCilKHr8w!E1Oc9bEWbUB48%fonQi zNAI)GL;r&Y356q}nv$h;07e+RWIs|)nUPz%zeeYw{*N9y!uUjZTQ%}OrYJsmf&`v5 z0Z(BD2>f5KCe!_P>AE@`{hnMSMTmbPw@zLdso=7(Sp0e=3DhF-#s&Wex+yxdxJY^L zX6A^bki(S*?|NQ0_WE1%GUp_S4zJ!!LJ?EB<)YIjHbysTzf0<{w4F@akR3bx4EVg8 zV0-vi>HK4uy~DA!+%KEr+gsqX9rxwy&~F)E0nVoZb}BhU4~QHn)-O&l&<{n?f1N3x zqPh78H6th4r?+nA4R%axFYNt=GI{J;NlTZSos3@u1w)(@{7v$ZN} z!+75RZqpFo{Qr51Y^soC66<~)*d0txHX*qM9vVV)EEIw#DrR4k=7l~x$hK!fX<8~} zG1$8Fwaxo@diVNMT5Y?now(|ph@G~FE0{vJ~EZB z;Csj#l}kStdAjziEE(u!p`(wotbRO~n(Wu1eNes(s(gdrU1-u=gEAR}FaY!8)4L&8 z0cOrQZ&)X3qyAqE3p$M`nG9>xb{%%$bNzJzFU7GQubXQu9rq>WhtSf^)*gJq?DFH& zN5csCqBm0E;gP)uu>GYFWfS{rT`!z}Ee7j6q2l1$Y@aave*E7c725?gTnfm^JWZ+| zz(-397-tuZXnXi*q|%c5S+sD1kunwV0X}?#sV2d^$M^ToA!mY(1mkT*!o4uh_AvNV zQ_z@^D3@>Q9fOJgeRlqfQKpt?^?GKrsTRa&6c0$5RWk|ol~E4^2m%tFdjs8yxh3RUu`n?Yp8gsP+7u=osB!Anj{Mh?;<6m@wrDjN3>a9i# z)ce*?aw~ilBLhx<>_CHu?ZQEq*uO2{<34_v`&*5_6cKlf%|$8`#sPsex7`6 z&aO3Bw_=JO&w?V8n!HV|C4yOfmvvagu1R?5$h86A4)bLrxf+3!`>SdU6-Yig95qlf zf)>jXT=S_b*(H8=v55%{s zQ-)z$`2>CtvSJ2WmY#~|FE#Ui;Z#>lVS}1CWZFk{5+l<=qF=*lS6|9%JD+9#^3ILf zVTX@c5l2mxWAf`^CQew>f8>Wb0)Qj2WZD?Z0Shh|`i3)Til)>_ofx94)>~7{PpqF^(AA6R8i=cWZ2rfD!Amk{aR~vwH zEv2BLqC?*ME)|^j_4~6=hXA@@Sq% zFU^gR5MMBunhN790vhYLsaBFM;?T?)rEjTU%8qj98gIO;k9$D5`6%jnW>zYu&Uojh zH2tM8r{E0)ylZLcH#15%PoJ#(`3iF6r;BB)dj&eCbPxqd9ME=q7`2+pBQu-> z(D|_18v6VYP#nl?nyj;Sq3%>K`$~MioUoL4*piVmqDgFNs3+xG`8RCoB~Wv_1~knK znugR3Gzjn0J6&uWy5bojX(x2?4qHk>Dg_&gj;d(@$vV zN-_RK)LIbwEn4O-Zd0k%GS7DCs=i50ZULJM*EP(s%IZKeG)dru;$mI*+jKvXd#874 zbtZxuR^~|O+i6qPPZ}bhpITTe4mDTohNkKfPlU=xC#TWkF}?{^?pXp$yOFZ$T~IdX z_cx?Kg`8?JSGl>wT#+OOwgOd*;el-% zO6H}-z>J5X7TCTb{Ya%+!ExGwsv1f22oVk#cd+WERDk@@sK|Sn%KVh z4&^yC9?3WpgHho>Gg3p?w8CrYGoZd`>G7dLMXP`DbTN55cfh74V?K(kXyk_A9_Wd4 zJ0nT<6dDkeT+mQhMvPp@T2v(P{^x}7N=$0 zUl)&9C&c^dI<87jA01vC??-2|gZV@~wJt@uvArrlo2A>VL&jzg{(`?|2wXf~tZ^(+ z)zzD{rnT(Tmwt2VN@yD`SOUqv)fuW)P1O7>*i`z{Hm%7YBW(|eJfw$ztX4!ORqT5E zk&|tuulCwWKD1mfd-44rh~`V8cZNs8kQ~Btb4z5E zi1sJvs#EV%LZjCOs7+|;>uIZUqA4S&W%~_>LI|XlrPFpdOXc9nr5(Kj3DQJ~CvUF< zMbBiPEiQ^~2#b#UV!&9sb|suTpKqajIcw*!d0dWFbum8jrtZHcI0*p9IJp~HyptK4 zS=RWBhZ>r?h5?Rhe_d02mC~}!V#bsTW8t>a*Cn(zo@Dg;EpHDFo7fnEb9faU5O+d> zW{U~>YlXUU1Dn9uPh@T%>n|>rC=zSR)*x)hx^m{-8OBm;zLIhD9FUxEDcjEXRox_* zBagpoTHS9EcW`MXauDW?^M}9)-llRh!)W=?(cqA3KC2V_nk;N?n-mID@H6Zm2!1>7 z1jzEh+DY%}H|eQL@60@!{i9g=D!#^_na`=Y1qhZ^j^8FmXg?_`Y}@8D*UR>3&Rj2< zL;?Ecj$I;CO%q&SI4bZCCq+NHzJ1++P^Nt@xEyU+1MVHkVO~eNK7;FevGYgg@~+rh zOx-MdT|n$=!|{R#2oK_?R7!{b#B%i+wN%5e2*s|OjMc^mlAd+_(>f^(7Dwuc8*MWZ zjS$FQEWb~bcgh=S-z4@WT>_l|pzCGiMRJWw7npX#d|L4}aSy1Bg}Jn?E{F)BO~_X* z_Q^Nh0{QW#)nCeQo-`e(JsYgDfZsH8`J23#eq5%kCoDp+`Ta0Tbtbs9+T8U`PuacrSjpV^WMa z)AbrJ;MTHr&LfBG3KSBE7?L-Byev)AI#{NoSWb@KdL0(>_lKLKp+#mp7^dE?@F?#tSAW}=` z_^|1s6ckm-3YS0WyO*z!jHM90+=Uh2nKL!T#H)wp0MhvGx@Y+i^t!b?u$^H&~h zughq8f&NEd=8+c$Mu=uQjJ(*q zFTR>XVRro*l7&SY$+dykT?j3`pb{qxxT{ECO1g1`Wz7?c0Y; z-MZ=AQZlp?x)U5Q=4I^adf4q>>d6Xt*aDU@9wXm!`anCH6h(AyqmTTG?;6NMg0xQ* zeP;#hfzb|FHf@x{5C^%%{6zfvotI9w?;}~2z+rTFOr(Mx*HhV#q#+E}r?9>QiW%^0 znvMK&WVx}oSO3h|+#Wah%W;wov?e#$1MMl4SG)glE4ckeyVQMo6Z}d3C9r@SQp9jm zL$Va{P7+;$;JSw(&>vw)Q%&jIUn-D2B^f?FSc23p>eEuxC_1E*tS}D^Ou}v3ibBAY za$Pe}yL{>Ag`%*n|Bk=ul*If>BfgbQYUxu|A$u!rjeMn?5#oteUY z`}QP^_z#1{jRhqY4%uvlu-gq%drZ!m0NL;GcVPnC^@Z@{Fn}Jhhm;`2o+B5d|0f^* z-~JBwf4QKRKKCd8VK}s5wk0{acA>u4-We*e3iH^T=lF^81d1#SE??4(x1G6t(3yB_ z>;i^{I`m7^S24Jtb=cxirB7*L(a+}?^R-jbllxGrq+Vmg9v?FuU)C)`yPnsR7Pstdj))}ezPK~^%M!5TNnm#J(qtu$ERK_Dv#kXKk!#>rXGUlq`kRL7~Hv%^6hvXlCB&z5V3Ei?Ef#cf zDA{;o@Q3uXG{nP)!_D(uL>O|$#lzbh9yjqn5a3&b{65-+wA1({Xfazdn{vn6{^53T zlm~^mwG;-;r6Gkxhr*%@?(*b|^ym{!(CiQl-MSN3O#A?hTTgeq0!if7J`58n3|i=f z>WY34xX#l{f>hzPHketT2Pua4F80y%3xwg}v>B}Iq3@Wi;r92N?>`XnO#kYQz9h`P zehB~M0uC#4_O2b3LtI+xOX39p{RpGJi~nfVxZH-Hjzl0Y99jTX=#xi?fCN%suYqu^gJ8hy^8Ty?nGewF62 z8_(4sVmIyBU3AcbO0RXEw%wEs`Q;ugAVOy; zpBAYGkJV!!60b(sZ4$Mix6GUmx+1NQ=PAHO=6^ZHY9lI{J4`b=UJjK|diDmRa!ULYy-M1ATN9#Gn8h?OmV6ftoQDFd-O${Vc^R0j2P+=ySyUhJZbo4zi9x) ztK<2MeD1Dl;`l8QfuBKTnyB_uc=u1U25ZyxECED+=?k0Jd8!?hejTJ5@)V<^nP8`;-t*ouNC?pjH4+Gze+q!|pi@L(N8;&(%DkKl#+9$;W zFtFhR{_t6IXOzSu{WV^UPYoVRu_#q$sggu{e#xQ}*G4x}ITwa=)yXN&i)t#5{ly67 zf|=f>CXM3;EmqHerczObul{RUGfTh{s7*T_E2l%@7>-ABM}#^LSc;MK4}{bR;S`R^ zYcm8-lGvW}Y8`;e3Qwxycr$v`)FsDm{!q6}G!(0we*_TAR!nXpfbS+Abs>YxV*O3i z3+{%;X)vb?(VlJmw2E#E@X&@efu%N7tB?WPuZP7XrlC(_I@>h;`T4$!qP;f7DKh5{;nThT%vt-h++CUNW;~r8zJ5K)N}&*z-_U(fOOr^e1lXx94`~3=sV@I_v=HUgekW9 zUwH8oX0DN*3|Y}0`|yilVTf`}a7_}e%_P6>Mw|SPdLtW0sM+Nt$CE2G0~i&$kL4Vo zewHAHfW$3`yiO3E%mEkopm!r-?<0|Rvn%^#HfNTqS1W$2?bY{vkJbz^jRz!Ec}fZ+16e7v zbgEPck(Sz)vvL_yyePEv41BXnD!U6YZKBzD-zCMhBm~c~)d&$R=UI%pB);jt)jBg+ z7l!v1KmyMEy zW-7ByMJ+ovN?<-LGi&i;>Os(OOprEPt6Baf({;?XEJss({w{qrJ!2ujJwXCLcM8*&h<*K#W>5r|FQ zwa~+>rDxfm8W(?$0K(x_)L8@#Ljd9_03h~t{Z`))nI7(qHY>gksJ)$KnQfb+Uiv+w zS&=n2f~>pD)EUEcuk(g=-^}#mdcF^abc}k;``wzJfL&FgUv%N&gVyjvB_kK6qR^Mq z0FQu}SjPNTLOQ)-dzqesy0u&xK9_CV%_4Buvy^Od!-Xs1H_Q_A&_58=>WmejDhn#q zxyUKIinsp@<*$AE^Jg=Y|E*E zsw&T|l7Q3W#eO!l^~>5n#c_6FuqXh@h0j6-C6trU3i zElG$*79-L>d}5c#zgxDw4soaz<%Oa!k)ZGV*_PzGrs*45*5< z^ZV(ABmW|hp25xS5FDyuXjR&0MrBL99S#uxRL3Y%Yh)>ACzivuGC3-u^E$GqP-e+} z0}ah_3nz3Yf_5&aPQmYC->v#seG5zW&MbPAfHXy%6AVZ*EoF;j?V5l?W>=O}JePYO zfzjXJ*dyuXTvTB3dOT}KjE?o2&9P%Y)sXX~`QtjT>s^vuMNmYdP7AqME;@xzKZ?T=JbwtCqfzG{Z4L@@~;5_YDS*=MUl_ogr=DS>{b9Q?Pc3%zJg;}jy6_<{% z;ui`@f6?iwVU(~Ot=0x=r|}f0w|tsvi)p>ZFKP*%y5QyOs&%NJC&>K}{W|qyu>Hgr z(Rgs&U=ZRlt5j3^5emq8qE$UMsr|YVR0*acwQ#$2)2_f_)^on|9OkPQJO+M9AwGcd zrPv2tQ{8qgXev(#H^=Zt-vV8d1=J!>NvE3(K5`weK5`CO;S#1ohH@~V0WxpW6q7nk zI;ykI)maW+U(*=)!1R1ivKw?QF6!3J#Gd7(oT{e{ZpSlKT|hi6CW4q&<(A^2YzUNp zFdjn8L&!|GlD+DbYNzJDKVy9rOsv33m*eJBLe~iX-ks5}E-G$^wx7L3vu|QM`E8}v zI=fR%)#UrY`3v_&>yOIg49T=z9Ec(PU$~u9Xc)za~FUcyC;T$wt7ml_n z>%O6gzvUFLtEy_}i{_^p`SVq&urbo9GMLFu42uhBp&0@he7-28sx$wB1i++1Hf2;o zPUGha!T!Bs9kG({KkSeTY8|GkxsEEPj84ufu1>RNaZjpr8d|n^5wR8|PCeja^5W|B zAQT6v;cBa(5xat(Od$3zE0up%-q72TmXnBIB-vP5d7m`Le545f<)&Iod|TIM;(8TZ zF`AP5#bH2$BTHdtr+Z@EQ9h9?b=TUvNdN4{P+3;)PHi}!O2~o?Kmg2vsL-T;m;h0V z@hJ%$H8c0zzFWk|X3i=;g18o2JWOg>lp}`CZ(!-(e;UwB(HC2*QZ)cPmiJ!9c1Wpy z(TSiJ#0rfuHD-I3uFd#!qM+Cf^yRW3bO+^BBIbdC%yR; zOR5;Mc=wtSOT^6SS7g|(UdKVffE7XB`XOa!Q7+h?Q$7ctdj04>#MQrfM@gebIotH* zGrFk@55JA8V6<4ShNh(-AtWnsm7mfXxn3g?#DxDk`wVwe54%tuP!MUDZoqUN*1d7X zeCY&Wn`yKt18mhAARfcLo!-Q{u)2?byK;`}n79nw=^gF*l_3^abjh>3TRmo)RS?+E0+8<2IaewyZb`U7uty zIP@kjX6DB(B_ z?Lv*r=-JzJ*jZ$6nv$~kf7X_`DOXVUQi&+=+QO zHM0dq%PJ-n3G41gue{zTN_^Gs7N`x{dv?NQ8(KLWYw-I%)xVW1LBt=fM-jlgT~1G; zW2@>s_1!kWz%@WhFC%8ru-5wJ*$sQX5nT&0tbxsIJ>P9;>9X+DEBd=cpeUu^@aS*e zLN)VUFpc-}=?^99ytRYe!>Z2zcySuTTV^W2Ln9~+BmT56H$G6`6fb?8CecSa5_>xD zKd!C^VBpUK*rHB_%?PR;obujbO|0B$PC$an0 zz*M$EK*TRDuvc=9i3tr>Pi{+LZ2M^djWDY#%SmMZOR1k?dlFxg{!VxZKIZb%01P_^ zAW0T(+59F^e=L*Hj-NC;F8hD;O^EgLfiJt*mqq*|pnoGgWHkMSS~72nMg=^}i8t|yN64bF^4W$Abcyj*6yAuwok!;)Vh@+=5}I z+X~t-mw%arl6(~t7hCk;NF460WkctWi&I>V&ucy`bN^Y`M= z1$%h!*}wB7a+MW}h#GFY7+q+gI;T>hPoRtj`MtA2kql#YbOm(PqwR^f_T7TTa%^+7 zLB==7QPMmptP{eY;Hyc55Anau5`3t!KZ3QN&f=k;5YK&UG#rqaQw66B`yx;c;`Au`!D$1io(*g+_j|_S?a&L?nolUInbqLt}&Asrwa^Vdx9qOTV_iLlY(zD53ccJOPcXB>hEMsh$6TO8Q! z4#U7hsApX}!&LKEM*PjF;C+i-7Tej&d3~*CBNs8@BdIUz>LU7w4!;*}-1K#jrZG#xksCl;~f9dH4rr*gb?s8279LuaJH1p_bss zz@)dfUkHZuwwdoQ!Q-Grc%A(1zY6f}FTzKk&lDDKrOMn#Ol#?(jW)G_gzz6hz-$c#NnSp?2Rd?abkMYH~>$-ZfF8<4OT#Q2St{^JDF#ZMO_~ zQYV)G#zlBTGpBJ$``q^uuLP}r55~o;k)1@d!3lLLw~ziAz#!8qm4Tbt85tJ=USL2L zn)HiM^0@}=Zr`#geXR9?YXuLG#|0E_H2v;WNye<^x5IX%EpiLd;Czqza@HC>+|gyA zyHlyuc;gL_=0~GNu3Ed@X2`K|yl%|29-Qh!FU9}Nh~!|<0q6ltH=cGUswuc+ zZpj57?yOmENCx7k_yTuYe1|!R3BFn)h81I|7QWY22u z<*d^4S>J$%y8B1np9MZKEz<$~5l*rkL=R03?rPEQGkGosRayQP@q{`b)Us6?FLS8i zGD5CF-FzxatSEkVjZxTss_?FcB)d<@M%U}W{G;8YhW9H|_IxIr&dI5U9N?<#mK}Pg zd~T<$2Zdg86BHIU;Z=Jgp{8?yRPmk$?k$_P6hV zxW95Z5qo4iB ze-f>g(4pZL9$IB_s&Zs5YvE(4m_wi7((s*=z_nmdqp+2ROlOObP=4L2w21mKqEbyv z>2puM2ASen+7g)5XngLSUfm*;_a%gn2oUSSNAEZwJFq_p;_wc&gepCcN(?Hk1I^ND z15U(SdD=2+GSFl>N-J!|gXEGCAX3ca#!ywX2yD$APBw4eaLq~b3n+tgqRF_YnPre0 zRv!*YbOY%H@Vp}K0Ex7dw(t|xiB7g0G8p05UFOZ2tEAh|fP_FNi z3un}R;koox-bim!A+3`b%tXfRtEh)dyxfDbyF;uWO*(QOe6@Isx%xv&6yNlfR5S(E zuo53wAiiXf&36x`7b_%QSlU_K>k%Dbxtwjwtf~Pm)7k7ki-q?vD%8i-lxIvofy0V49r>AKWNzUNXQ*dJ= zJvdr_b8Gpl;tTrIb&v$_^6IY9qL0`}yk{50S|>;i5QujNmWvL}jtaI57D65CbE2K_ zkbZH~IRj*9YjUSE&mU@1tgAXFgHElAfz&MAAMoOD#l(`#NVbDHSeLL0N$y_n`_Oo# zHl5z;h`=g%j|`qx&|q?GdNCiA#b8*bp8PQM*0+6xC_=vE(7aIMe1E;t61#IB@l?JD z|HcI4u^a)%>mUf#!x3gq35j7ui+Xu%*opYTjMMbqaBn2D1}k{PMG&9>U;_2EnJ+(! zh~?tendP2nYZMzgD(3G2!Gh97DbBQad4k`!ojQc_@w@!#$U?uE=Y+>Jqc;@pJWWR0 zl1a255urd8OIhGdg}#ou!=e@(lu`*hsXiT`wtg}w!9OZhPo*^hI*VsJF#T+IPAKtH zKtTLQ)dWZ3Z%$YjTx9yW(eJr`R#^iL&xaT7EyalUmAM#B0qHn?3(B%Akq{%}_~DCC z*0@&fV#PfMNzmSL=vk(VMXL}SjhDD-!$$@oK z_O}z6o4f8EYgH>g=fBs^M?pZ2;ysl~hbOEMOGGWb289(7Sa_WOt#>k*Lna&y62`FHTEP)x113Stm zzl9LY*z939Lfy?$qz~vf7iIAj;&2*PnhBF%9ne>I1XnuR*Gx?_+U_m}5s(}Z zsO2DHVBz)h75j?O{pkm)6DFH`@`X-Cp`{l|o{gu=Lvr#~zcRp9Pv?Tpw9N4fl~+9j zjP+$?6K~23JtGs^-Kz*(7BzX+h~{&U4B70!nYzqlAKrZV7itFJr3B)$8`bi@PW?-c zSvWDQZo$~QKAQu5j~`Ow=P$nXXg{%W*YjWvX5}NhA8c8flgdFIe-^ZwcIQN~=mS>qt>l%6*MSYLU7Ec+>Y2D$l8Nzb)=f; z@R))2j1A4r;&kYq<(oi@@AMFh_-(1#%DUKu7Rm4&nkL{|@||8xu{D!^xGH0GqQ1^Q zM{U)Os5fHz^uZJN;AtBN$YA~R1m=q>a`jn{^zq?^6PhReGu}J7Gv?BEOzaj^*|got zMD;W?z-DvCwh~=iCPjNuQ?*3u!>O^z@<*+WxJp$lH%&qRYB?Nw%oL5LEEQ)1b;Y_$ zgC?x7q|`Z%rU*7`3=ZuUdC1Wtm`g*V6-XWQ$DZ;X!!u>A+|(cL*`}m)ZwqJY)aF@U zV9H*nVs~SR6$>B5qrclt^2`Q|t3JrN^LTjaa=%E|Ajgj}J!sc{iOd@_b5o0{%A_v+ zBL^L!G=uO{&dU$#+%K2onPu{;K@R8C9u{5k$WWNG^h3RymY_6Po&bi(VFbmaFi+8S znd}{~C%ce-@f;WodX}NrPtOW?O=^%Kcx2PFDAAdkeG$&M;_h z5gmu>U*JaJ8${;qvy5z2?D@tIj9jKKc#7$-!(x7q#>UQxfG5k;gC+>p$Y}y!Y2AZsKhoelL?f@W`ed>rRg(kJ%rdx6Z5e8G*8cX)Jg{$5h}?dsZa zA-0|gi5#A{EPM|VbrGQVk_8gmcXdi;w8+rvYyWcTpQE7??~qJQK_YQAz~X88Rx$z= zQ+l@Yx70j0gkZC!qe;lJ%3ugIISB52+}5}=FZP@Pv!q(lnjyACh?HTi?Fo#l4aV*O z8H-R|o^G37BqRY}4ImaK(ifSU$11sR+2oX03;$~XHejVtn2TegcQWAc-K`xe zjwxG&{x|?TCRtbp9-fV59~0r=Xpw3EOkpW^m!1dHSDG3o>8X$i1;uh;tGlo|d{lJY z31vK5sA^p5MU$6O&zV@dLYm)6-*u zJ|2jy(J=9TOX5N-Mp7h@su?G-ka6L^8gWa(Hl<6r!kaQMdga?u6+^S71*BS=%wSDs zfJTRndKKFja(vl_@SFIMf8w~xVgIgq75rW_q-jWNOPqUbKx%M`vyFbBTd2^E3#2YT z)APG^f(-fx!lV9jTP_-)wiyy{DZrRFJ<+Vt%#o^r8$P;oM4K#LAoHp9Rc`)GI_%`G zQAg>|qk3yV_DIQrwU48DdVLJj*LTTMS|m2L%B)!Jn!O6>wc;9QVQV?<3@hwT2M<2G zK9+!28GRjkeGjZjbw>_hHvgI)k@!Lfor-|@UmgPg8Y>bl8u6s6{N!Q@I;;vPEG#0d z3b0%bHXRF|J87!WA#{pEaDiij{aH6?3*T&+!Wa+ZkF~tod?ghfEJ71lGMbvo_xp4a z3(}cc$qrW)W+ZWckGGG-=vlKl61j%}9m*V|753YsEbE_A#mIPDowkbSjMjwr9v8vn zaiQUN)8DXG`wLH>D_Apni|u0Ql~kz%lW}ld&{x}==t(EwkkveuAL>=O=B)Z%eEd&& zQQjTpUy``-*+&+b^Ds(7*|+iR-*;Le1COzuqlqUXBj))V@9gH8FNxb*rvkV9cBkuV zeq$kfo;bJzDZ1I;e2;DU5TpbevDebC%&@=={#oVqm9_T z=M-jD9=ZR%l8r96Y6#I6>R(2|$`?#ln_pBesvPvaxC-1owh6t41;jy7 zZjr4&-h543h{yRmKCb|yw++qP|66RU9=+i2XdapR;hd#2C# zyyx@&2lK~&x3TEk9B_aEfE8~iE6OY>W%eE*EY{!|qX7w^1n*NyH-Q|57VAP>t0 zzbS7Q`dWw3<;RT}xYjNzv#Hb6<{swTNFEH=;#iSRg5!0W_v)_UZ>;2$9}<;PN=X7q zHg^M!JG8%tzU%UjUWi<&fm(O;FlAsbo7d;j>9E*vjUxH-4yz$g!gf>Sbz3M!K_L4^ zR-@M{c@?bJ`j0l)jCdF-a@5?I1QxIT;!c>f<_;B6v^$ z!dX06DMScdgcd+b<6B~28SgI&`Ue2Q!$qUCk1_KB3ktNUTLrp$qN zeS>^64iBNVJ(lTTv-zI^IuxCp`j4#K`C8G^mC1jJ76>9cLe}d-4LZ1f8L`s}>%Tm? z#;jKn1HKe3!ba~UV}F+2z1_$2{fA_brTMw;PIMLN0+JR;r%+^#fbB8|XXL zJh0hp0hKoQ`O5$1j<89qDGS02KKMwC15T#qLy7r}oC-?+g7p!+2OYf*C2_wL{q_i> zF*$kqvpGe95~FT1KR>jUQyfESPwnv0mo2f9H~miQbq)Q4jMUHx-O+sFFBt3^c<1%0 zQ{nb4(v2c87+!c5TBV1qQ)g^&Y~YgO`u)o1S13x+qVUVF7*(M{AZX2q!~dh-AMH}c z?`3_ZyqCrAHBG2HW)$KhaRI#I{1>#BH9^0cffXra$3 zlT+3b!WlnPseh$0Y+L3zt|k#gxb#rZqy8ln=efO(Me4BPL7ouYyUva=p>mV|ma@dK z;_$+PvOq#E9IR26wmd&+I%ceyaa9A*A0=9L(A3p=w5d)qRCf01`4X~HRM5;IjG|~_ zkPZeK7!B*hvLVtAvw3>E;JuS9 zfguaI!;?lF{3_`>a+^q*MENc%jJ#VeC-+^jhHVC`*Bgt7Suz_`Aw#bY%oOqV3}gIfeSh z!8$f}?u*&0)Ydicm!evYY5(Wjk3d(0yRbCI7q@(GC4q4F$!7v%+sr zqd&sTqqpc$0pdu0Q1g9cYn=i9FZK3DZ~b?mGnKeF1hiByy0G8yVDEi_I#F>y_x`NC zjJh+ZpmxmbXa7zby7c|+=kiOlV^2xb&`Pv*%T~$?^_a7%q(4Heg)lMiK>WTeLYkt< z=L!9>X{5_B zZgDhdxyODOAq3EhB3?9F3|wq>(TGA&#llKXg`-TXM~IXDHF!vxL6H5jU1ZX0TFIR$ zxE;~Bs{;2y=E=aO(Fiz-Abi4sI`!NwPU7o)i`bCC6j96+<>K^-ESF01lDtY298`18 z!_y{5N+e6jtmC#_{5hfKFl+$RRJ`7MW%e^5e`e%r?n?_bodjz9Aev~<`$3akaH`VB z!aexGT;SLpV@tx_&UQaaSLvpb48FPMmc)>ahbZ`{KfaO6YL%*F?)UfV`@Yna9r`Rf z(j3efc0q&0g*^_(Ib|o^gu$G3)yJ_tyyA+J=-6PPQb;a}Z=P?=V~<=y!S_X>2zwKE z@H&KBKuVwwv$G_yJQ=}TnpMG7LGUFvlp=)AO|Oq=p5c%E2%nO+n9c{4x&l7J*GlK6 z!}8P;crkb=IS5C#K{tbT`<$gEcAb4OGEjF{dBgyx%x|{GkAr^PyN!aOKaw^Z{DwH9AW^%fXYCnDY>*gSqOV|ghOdI zpFj%C%mXTw)CNpaJDE1<8p2IwQ@Vqub)P>Ka`|E0l5Y_Z_!(5c@q}QxlL+4|FE%+Y z)?#*zTSaaz#W9scCCsThwi_IjAt=ObWcAPF)H1MX39`rgd$Bz4;5XuPwIFO1^zGQ~yVB>?eb~(Ywi4k9|=FK3c%a%$m zGAI39LP{0OVtEuR%ka)kxhQRAPN_$*Y-?*Sp%5=>RLcngmq)P?!dPD)6F>mpS7N@L z%XQi&=Fjy`2S?f?MHE;O6f*j0cehsYtj@`ofu9Z4miq&$Zwd<{*CT67uG(ZIJ2M54 zg@sVuQP(g$8iTHp_!lfEp7Zbq0|QZA!SVj+Hm2*1F9`i8u_N2UMO$8e+FlO9X$Z)_ z7*AV`5L}!}Xq^FF4E)9eM}DQS5<*>??o|PNz>f9g!Rp?rTeWNgMkkPs=9n_@2F5W} zS9z8eF;syD%xOz(u~r!LTkVB6ODL&w!}L`TV)ZhCn(V1K1*EjHn!73O1x&TwoiGt4 z)Jh^F@XUK*;BzP1t7GY0t6M=Sm2Ab;73E3_7ZzLx z`W7f5NZh^TQzLenr3TfbqHb1AuZ8*}&6bqGD};J^8xiM*Z)3r(@fNR}Om&@M2x?HB zDb?KVKTJ~vg+pC1`K^&fkyvc|Iu=9M;sAq)Pzl%a-%^(#n-LP5A%XBL@#Rj#2nIQ_!#5E7JO0MgDWUxJ}Kl6xk zoZAFA(*uAv>@B|E@tq!z=iIU)^J}bPY;*%=#mqjjj#e%+^K}q{?&_)%=r!oo!{{F~ zVH)=Lhx;CZVUnp?@l_TpN#DyX>!pU1cd*&&y{$eN?m!_N`SfWOZ&@gciEh3~<_}>0 zF0+r5`iUIs5xjUm8%jo7>hT%52ZBX47Ym73;>phhx+spGh>Z2j4l!H)$HhjIH5Pbz)<+F>4c3R#yi@XSAX z1)bejzyPjnHb~GcdQ3L@53}d+_pV6Pzf@S{X=*XWFGj1jPU^!2*bB&Lbr=8$0#h4` zST$M?=v>p^efBUdQ>gWM;7Yg|$AB-wbM76X)88tB67Zy%KRR zK^$r{V(eYwG~s^#uot9-fl#V*7W&~&kuOUt?NfJkX_5JJ)&;);$trC~d9+hpAiKCN z2K}|NI0^9!EDAdMh45?R&4h!!4zed|D30Yxckr#|3kLDZnstW^9;PnVRIbbmFy~ z2b>zoEl(Mdux*7BEe94Dvz@IYG;F|cW(vC@a)Wr@3}$)gSy{W3dZAT__J)j&f__hs zL1rJeWhRbmzD84jtT*}5<|p2COHCHU9nwKdhqqG{FJ^r6#aJm~n@0|f-M3L)UR_t9 z#&+qg-r3jpc754M%ls5JjMW;FsXZ~j&c|pLQ+iqQ6K_R$3~J?Mp+zpvbPh zMDl67{!MyxtsfapS#jo~i)ZUD?P~%BC|K2!*UpjzlIgMF%8t7$kIw)mW>a|y*b*>J>e5QH^L>@wmE1*?#k%ug1 z=55PU23F0?QHz6n@l?b(!}6<5RLZzuRVZsW)C5BbbRyYq{7mJODcKD{9nSlxbSdB= zyv0U$3-5HTr!O{noOI%zU3Wo{TPy|~5$geAwMl^+>8)GIo0EdIi&1tKhq^w~O-tS` zEi2>En&87G{r8IoXi7HX1Zm^7{YZt;Cy(>OX&e3;ml*KBR7zn!l|*q#Zyxak_9Eb!3n&h@5W=lJy)+^$l#(0 zYMCAO^eD$A#*xQ&42)X6~IGJ*6ZMRx2VsOjLz>fFkNa# zyEe@FarqqkVPEua)Cdza=Nf&vB^Xt*+Acnx)n(SSY$Vh8$xcY}nKY?y0k?3esbf_7 ztqeITg9bK228IM_Caqp!E$_1L+ud->`Iu!_B1q-ADPZOw$0+ondx-l*aOiZLMLQVZU=>OBA{+c?h`FlISWZ< z#7n~ok3^xKrCDboS!C1M9vT^L|3tcBv$roUXgM!F$1KkJ_o%o&z>w_pgC0(8c^NT7 zm{7v3u%w3bEPgDABf`~3xWiRKl{+-5kE>rYn@e%9!VE*_=T4O6vFCI2`fl(Xjn9;? zdDhTRL=#zhN&_Z#)g1oRiqx;kP!Qm_%|{%(sSLI4%VtJVOpkGkhF`!q2S-8B5JMWk zi@Z?{vmd*Ma}CvVWrV>5{JSHq6Z8HQuXJ)22fukGJ1HDw&n(`|io*l#BuA2)FXK#B z?jsEtknY47X{OJ#@N~FHpX!X@opXj9jOP~^GR0lQbNfYl;sGD+*wp=S}x zBP>!(%hHSDNaz5~9bP`SnP8AeT{b|Do^2L|UX>pl)JofuOX^ck3`HL)RR~dj)$Y_O zm`LKCB59T*joe4FG2{|r`4e!8hNCnS=Rn}6Vjy3%9cBt3yhmEQio#UdoFzvLj|iql znjeVsF?gF-OA~5UCuZe2%I^4gy(#XM zyRa>|_I50OF|OtF?E;Q%5Or^)z|UTK6L+g9@{@z*!Jb0srW$XW`i#@2LXEvY{>!gC z=%GuMmpZ05p~$!`-5lyZ1J#gMYzTH1`2K>a78cD&!6c*|LHm58o3pm?xm*MM=XhRTtjIpjxaGK<%7Q7eh+jl9QA1i(7O{4gdP5 zr*=x>DUt#GtRMUQE9;o*z3vrHj)}8y@L+#nDCT`Wz7gsTK}<91+h_o;rbLQke>;mo z&`i5cXFZPCBJ808KMMkH3WdF1e7wcxi=%&g6>T<9#I38!)H%z1x5Y19v4b!=6W$ez z{!gJ5^Up>tjwj$S=XFf_j-watL@cam~5anNeQ7~`xi{;dP$Y#fyAkJ826ub zD{|QPW#sbddfL(36{I^F#Ev12en!m#ZKcB{dvV9*`F-%4_MlTb5jiYf&sU|)ht#`6e=Rak&iE;G0ZQr)%-Ec9|-o*o`J~zJh{B{&d zC6#2Up%<@*P6&(D1` zf|^^qmzOuAgHRkw2s!Q!iZwGQMl>mOM7D2nV>ihvGfBN$M`4&^U0dt!S5AonXrFD8 zq2mz+>&*Z}|7uhi%DKlgN2|g&BZr&w(u~h<1YnE?3vmXaoa^>Z7%8W#Acd@n&J73d zrvC8Ke;HmFX-rsiF~NvYkW-Wa?PJ>4z?US&m=%K`6mmbIuuaX%g;6$J^=MnCQ=*Qj6|9OqLy{Yxq z1#TXm(u|pHIs!vfs}`lo-U$(q%2wX(@P6r)(opGz#~FmIjvt*yjttyF=(q-IwMo4u zkek`&w3THQ-<`=mJPm7tpu_=NL@yf<;m=CvJ8%(#JJi*c4E4GYwB`YN@5w%CwLe#2kxIK3=4lq3*v zf`hR!os4@M8Lof~@MWO2x@`vKPq7b(cgjcH5x9NE+O|X*AfJ{}CY#|C%8zB}ibmds zdJ7HO#;Yu#;!*4U71L^4a7WF$g*9}=P{FUl%(sB7O2#oyGp2%<>bo_@W6ad@99QQF zlH3QtlmSyRO9BZnC_mBys*=M=k*LOKXdznU@G2`jfkYOsSGOsdpQ z%s*2>Ci#tEar;2I>`*bF?BYF*ifONd)CX;Ql~M-PGcM)#foF>(FEqdF)HqItr~}Z8 z!5+iu^?^=uY*TVG-m24ldM=U?{Y)BNGy&euwVqf=W&L6^sRh%SjB;L!0|rg67XB%l zBNL(+aP{M^XrlxDmd2u>Wx=y-*j(MppJ_CD{x?QyiHg=HHZwvlYoyn(_xR~-(`Nqb1j|*RpoBm1M@pR(2SoMok++r4jQEaPbiB9eR z@IiF?Y0C5_@6qkuzrh1}BFLkjZPC?jz6`LW+R$#L4YbT~;-(8n|HXt87ak(uY_T5Lo)8o1_SSt3bUGG>ro zT!eM8YppKi|=vfGizfg*Z8l`3Y6D#TPi8w{FgUcKWhikXyxLHt>@t{=Tt za;fWm*GuH2cjG6wLNM(RvHYG1s+@~&tg4F-odpU4eu}(ndt@6wqEl4v9?YvZpbCnv9_gDnw?Ap$IDL#r=GGMYsZZF`S0d z&C4B;IgclNss z1$@;F)l#iC8_`8{+`tkOUR$imlMAnIC$2`W;eZ_HRC!xlJKH}>X!#Bkx4a**) zp)LuO&eleduG{w9dZ&H?r9#I=0G&lXlQ0xfV|DcG5W%MG&Ki(h8*FoPl_+#jxX>+C z(+H@2E)v#sK&+~yt)8Lpo^qP34(I&|J-nSRkkyQnO39QSs+#9^3N&ZENAg!(eLXi4#!zgCrqf-Q|}4=)=__h z21otY(rd`463V%YZ`&_!%Jq|Un(o<2a9(Swh^cj-SGlk($~8cu*Wv6SXX(j`snTT? zyCLpjghNvXEhesn}PzqfGNdo4TAnyt!FQk{B20DmUQ+rQ%VvL>logyM1KT)}>w z_hGZSq%W=CS$vc5VnycDR~skPRZ~6U({??Qk`6F@YgEse%2GUA+flEU%xEsBP1TI5 zCarOutiELeZ3P;B*ZV?dTmekL!)LYq+UBho6?0a7cGf)viE`tFWsYE7^qg|HlVrm7u*bUE&*}rDgOA>}zJ#lVpBoje0Thsw z)$H}$hnlbLq@s6V2~Ptf>w=4Gl})T7~1q&V|=7^O6SEV$hG<6>IjOqr6r(@Mivl9H@uB$M{=zRH{!qY$2R?k$?KJYa+Joir&AEk>Gu zf&Bg?NP6L!Fbzs)IE>_os)r7FMMIT{9P;5#b!2gwav+B#Vox zYEw=>|H2|G)II%1mo{I(@9KPj8V*LwL@J>BoO5>R1w2R*!QO7B`8J1I^Dg9UX&lA- z7Dt!o*WO1}?#^^ZuRT#!dGQ)sQdZqmk-pjW88MYCg)yMMLaS2$XQl%&#iE(j+r4!; zsI{i371`KkWV~S${ZCPQK+}vsz=rNiJG=dGNR|N(wCD|FHQ4qQT$$Fs;O*W;uM>9$ zEgp|r7^_nk%s|?*(FdjL;|ZZNH~F69w`)77B1W7SlJbJu-4gxDvGK z?ZC^c?m4-^L(sE;Y+=Bst-LhlWRW>xn8l4_B9T8b6Y5fU@k4R)*ain%h~MrJSeE^2 z4**)|*yG!}=&X_AClCI{KWWhYRH(Wp1{LVYmR;X^Rd0)^yBwNUF)3?Z zqgW%=^#1BqS3s8=vJNf|Zf66$US$7_{qwNL-dcAH%XEp-SbPZ0FjXPIaf7y5Wtl7! zCP7*aM@l7Cj1Gj#h6Ywk2VK$?h_C)pyMPUlPCQEd^f;+|Rr$B`^R0w%-J=+&$Io-y zd5&_s3*%)0QRNvR3d{+O?BDUuV8R8*fSy<`C9US^g#9TT7C!iM#xLw8=(_BpZJE^@ zLUb!c6CoL#I_n+ePZd5kSvi#;OJRuMJ@n*2=$Ex{KU~Lm6DUV90mNjamn*rAJL1!;M2QDs-x2ff z-jPr2E4*;&~bj6raNa5m%b8&0tJ^~KG*DW zS*xJ=IDQPGn`a~#5pV{4qu2$?3LK65Ky!};Wj)h`d z<-P2X6mfI;FBq>Y%YsBHN0iB`%8*FD$aDYnLnzbS7rpN@{*1z+etSO6Em4Ck&u^wXj!QgyCG=Ir!K045`#~tyz}CF9{re9} zKSP!d12}*;oGF}b8z;}LUS2 zL}I`Z_0yC2(e%>tby$ngo#kYVylRpL#ppXqQsDKn_lT{j+%Z8OLdX2DcU4Fi^~rkL z{pOY_?Zp4nZZC>rc;fcK<&rUvHsR+RE8rlLi6L63`HiD9zsd}O z^$xtfwZqv1_+HEX6-|(nRG8+i|6e{ej1l0l$nw#I0N&#Ddsn9m++bb@(#b6-=Y9a? ztvx@2sc|e>R6{c4Brv6mMB?M7X%y|3Tf?lYu*^enK~l;8m3BwZzWN;fK^v7sabv?F zSA3b7AaK1K_)qyn2{%thb9FL}LW!8nKR_>vO+8@Hwk}>n8q$OivJ}u;h8-04-Y+|y zM;vv8ev2fAMgABja_p-(@qEXdYS_E0qbdqpMe4ZuC;B-N1I?}H@=P#Z|6*UmPWpvl zS;#+{tVTn}>3;ZEh-*=Qw+VdTg-Vp3_aFwdT-)Jhpy$&FrDhZhkxV=~DM7;1F_PaV zChy6wz=di?!}?mE!;krLA+QNn)2Bz?1kzgsBKJcR^=p*BU@1{$_enw3!$1DZV*Wp@ zN*;jS7EO__RO;U`$R-a|{_gmLq&odz9Y^5z*4~DDh)-UTVURwd9OLV_7bOQQ^|h0L z6kT(Gno8@t==K)cyqj&buO`?;Ylx#IU(BDUk?1FxpqUZXTRZnxE$pXON2uMd!3`?+qeBC?pH}*VUX?z=mnX2A8t{h{)ZzXF`BuM z^nsjHi`lM7cN-2jNxMr>5*TV7DLu0%MR&iY;ZmS}RyVRRLCc&G{{@o>rHo<9Rway0 z6ZU#Fls&;CjR_@{UZ7aYgd3o^C#t|=YiL~a!<9P+#As&HaopMy?T<rc(4m4huCY*o~+=weV> zQe<^Pu?dCdF17FEBE)phEfg4z<~fPq2q{;@8;2@#0!6!8th#&1mmOHLD@zV9SM(XZ zyL*87Z`z%IOxOn&t=$w}DQE&RFmI8Ha*(8t2p)D2iilcf`ua_oOx5i6We^4mD(=Pb z=_s(sE!>F{rv$lwWV;rMOF}MDbmO)^U$nE}8@)nQtQ+tZSj}lN(-JE7*j^TSUMDO4 ze)#IUyElyzy)TZ$L@9k6(CD-aMawAFeYuMnf`Q6(8A{FNpcD2%FU%Q9y6&TL_qKPA zP2>66zcj{%&IQE4UQ$c!==}>;2+H8)L$BBRy02HMxD};;EQ}_?c~*AM56?A2lqU>E zD8&aU8ohknl7phxQAHi$L0c3W0l#m~HZroL1%1Xj4|j!SRRcgYo4z(2oMoeO(d1tc zV<8>b{7*K!xUm-Avb&W74o4h8Y%yem6OFmB&pnFbCFY96QN>^z{<9xV;k=wdeHpQC zMXNOtNys_|q#_<*f%~Sve>Nqm5vMp+Hh#TnCfV#O+ z)Ji_WVYVvpHc^%|*#Iy)w&L(II+=tK+%pr(0)u>FM)(?RhMig(DAQH5eRR4U0NwQRQExF?rH0&H3nIZST71QDT-{f20xO=#B4W#n0X4U}6DI-!q|1aK6xsKAiB+ zs7X#bw3cLU+OtnHQ86`7+la{%NZ8Um-4zWymm^77r&S%Vqz9nGa3a0D%6fxnL4)qxXwlPiR2%@@|W&;8ar&da1$o z$<`6P(*#)7S#J;MbvBCL-0m>z3teT!fV8tTyIluH6$aNt&r2Cji=IYM6;dIxh}xgO zn6f;bww%Ex%^180zrq*^#479`#c-Td)Q6j{kR~8vhy}1L-md#0hUC@->V{JWyOc%z zhF=c0R*}cq5cU_aoWv|(VFgU+0?rUHN~#hZJ1dqmhz2!~<-<&iNhj~1w#4v4d*!S^ zcPavRYf{Zeww}2OrTn@HYB2MvBN{_7cyzgK;3)69hVwk$CqZXBP$YS7#c1Sf?(%Zu zaFz>!J(EwS=;8_$(h0wm1*Y60it*B^sQSu)S@RO6cj!A@b~i6F5X3q z!}$RvKnOO&u+oFa11)6Sd1rw?(ddH#Pku#B602oJH7Qja+5xiHJa2hA^c0>=X-yV3 zyf{W%wVW(SpiC4u$9Ct!>yKl^gVDo|G_OsrI63`b^L@$!XbUHYq$LAqDhK3n{h;%d zpqswoEA-hwswE#XrLk!4=gK#uGGQIZPNIMCCKI-wbTzaQA8Cblp)C)mY|g8ECP>80 zWnB?yC>a|N6VJdGH-KiRaiNyo$|f8TTq6j{iOipI1Vakg*MYHe{mP`x2yIsJI;nFC zpBDwtN*$%oEN-oh#)bhH;3`zH8E0~vdPF&WB&*%_$$y$;Hm}cQ{{@?&=vLa+gTs&Q zuV#gabwt86gZ)PjJGt~G&8l-TJgn1&MSTfZGiS76N&tbtsYg~KoQPI)j%}uJwQpQGB!xY~g9%tPoI`Q&L zXk;L2{&v4BH-F%(ovQR0RU`9>1O4b3I>;{hZDf9&O-TqNnHV?*af}G48mU3eU-%65 ztF}a&uQn~*S8EB7V<-ZJ)eM~J^WQgP9i+B7ANrtSfRU1Q+(00#dND*1@vCVQ0Q@P- z1nB$J%g42Wmwpqb?iCowS-i+4<*BIx$f{n;I(FzA;-pfk{vK;woFJU~DRJ z*R+fO8bH*;08$%}3yeaf7L-1!TBj;#MUK19>vK7dF*4ce$s(XEyYdt-Y^EXm?EUG4HiK4PY_FD}FeiJCz>wr&?l+FRWT_+i6qZD}-ol~POCegQZJr=I9^Hp6w z&~KbDvo=Dyy@W+o$>3=x;!Fa=bO=ZF5KPQzv5&39Zk=lM8gz5XEmQO7=&rO6@$e?t z1X+Vltx;9XkHnW-t%RwD&q z>wziYq;NVH{-{y@GDFq2E2Cmy$x?wZ1s!| zRhAU;cA#D-2ewYov{v=^MEDx&V*H;Cb9T#rihcFi@tgs{;8oOR1s5pgA+qB&i;YX9 z7+B!zbktC2Ds11r$q}TXBLL*<&2BeqhnL%NyjF{mtMKQXS`Z}b?zp_TXWbZQ#|hQj z+LHL&ux+X6)#RDYbz4i={(?CiD`r8xsQ@1vc8&5mlGYuhAiXr6^a~ustWfFb5yeP@ z;V3V0aeng?;Nc8+syE$4-ZI2&Pp{>`%+Lij++e(#L)1sJd7lgGC;K`;utJ4*f_H=; zQ0Hr&e|^2X-5NwMmXGOR?qz}5rg6cgU$Mb;H#8((qnV*4092B!j~XPm&El zZ=HdyytLTF(!~C}kZ2|ZA>kAw_3B+CIVna6j{@l{K;}TZXb=#l$@k=ZI7=-i+hU*% zjiLdArwe_Xr0U5*mX>bP&)P+5>dr`Q2AUb_o>r^pj3!#E-~|W5`0(G1AE+Tp#TA|M5>&TOgB{MW9{3 z3+Wljes~UXbcq>_m!>UFRu{+Xt5@2UkG}6y&!B+Ka<1q03tjEL6jhOZZ3CGr+fU!9 z-i19w#<>`zgtWZr2~1XsBk204J4-3C>1>!YDxe^6N`1^q&EX$-C0H}5&K;0rL{QA~ zeXEEuUrb!bAL-Vi8%H764U9zBt(D-&(xVE$5K>Pq8(vI%O3%B&9%EiYCrhu(3C2goS&UuF-4pI} zvS?`?!h=zLhml_HLMN{$DlfaguS*FP>A`1(WJ>|@F6L58=(iCWkY{n_^Rp zH>r>lsJdXo%&8KgxVlO>)JaDT=hD@K0*kL zI$HoY#v$9}(ll0Cs9QRgG9y(8@R)U?&;6h|9;#%3n5ies19W?>Rv{`ONI3Kv$mZyr zj8coD9cJ^A^J2!A$4{ zO7g(Pp(&~inVkpOa})N`xeC__Y<0+X^wj#~d_;ljZh2#yoh!^vnm&{B5 zswu_sWZ2C7aFI}jU?p;dVaVkx#|YJA7?f*|(}lAHIe!eGXYS6}*WSn9DvLB7GC9i6 zP3!PYa_i@J$l@k0H>DCBu~T8mJl8)DC+%cpk-hil3YU6Pzai#^*B;-V_kQCFY#u;K zJ8v18+!fJ-ibR8vO=k zLC)X7Dwk0H`PnHC4h8Kdyn6qZj|G_=P89|!@}ZSA3`C=;s9*$zLcibC%4N}oxGH&m zL+$2(#P}5ym{x2haw1o+qe>DFu0`npyGO8Ffvg&@VBBiNUNcK8Hgc9Wt_RE*~@&znL778t(2A{-xSVNSUAw%XX65yMS@N^fqdxbh}HV;J!kcz(}7rZ9kmn}L$R5Ubx5}UYe+bULhMCS_>UfPqB&1iHrRBMsZ zdXPPun5tLW((@dQDL8SIJ(`5})OJAo{mUk*)M#Hj_uWHmcH@&} z9dG{S)w9Qv{BWwdz34}m@8(H_bXn!v7(L~;mjnXC)}U2N(is(sKOlEnX|0N#VUu<= z(aMsR$RZ^qDPGJ-G+D6kYQZfnGl@6zf$EmxL%)jLg9s918&qw0uC;RR`{f6DJGq+h z25c)yG-9Dc=dvxo7%SL-*8?`E;9syhZKd2+>QDC$HE1z3ST^BOl_kec9d~RPxVw%@ zK{T;NOTM^diX0-Ls9`8H3n)B2(eP|l?(9T^gM6o`5o_y zCZ5$qf4q1ouLE<}NynGA?Ji5=Hr{u;wk0^h(4}PF8oy$>j4qjAESXE;>%@Sq0!7s1qWI};0x@riPu8ia;)*iNGIcp8 zO}LM^!)gEJkkPj!-~YMf$z4&>Q#y$|Qn|OJu1}6yl3G`+I=eYKnA36Dq|GJ}-7{`l zSWPSGJj*hDmp7O-rbyEq4tC5@l?}81na^U#kH`UHZ8HPpd~yOt`wlTt-y5#;wMM$B z8Ufp-RtRQKq%TY9k`el<_QFHEjs75$#0BYl`-}fA$Uvw4 znWBRJwvx$ovC?=^vAM(Cx{u%$3J?Pu#B@8RsCbMsr7W_H_cn^QpBuRN+K3Qd_{AJz zi9U3+0@h0A4iVMj04M_bMCJNf^u;nG11Wc~z+A=R{!~77BdkFiU}+6>kSzB~pKfU})l zgK)8O!mK94?87|)b&)W}%S3uJW$mkt80ER2ffrgN0$Ds?U-UP#o|UntyX8WF**Hska5 ze{7R;Mn!>gu1z&qf5^kxtlET0N)588bI9FWnc(v1-MieV6eZ{leyRs~u>%%Q^{5dP zva`e|9_6YN;77OQZMM||4hxJ+Y;JM@vT_hov3YdFWbog5kX+-}a7uVqZXu3pj-Pgq zB&^W4{rJ+z?HhHz1tFfepX(uB1B6rCss?QszchVgh?7yti%+FhNv2mksp9a#fC(TyPzIw31`{_*UMPpxW>8#< z5=Jp!lz~uCHYgy-eEeXZ>v9!ed+&O2)jD6sj{V}BPm)PXy!}Isqf3TGc`er@jVTSS zDa}568=1});m=b{ON&^H=MyCQJR9e6EPxeOjD31%0);BShI>7$PbfhQXz&MGeC-D7 zuo0E3WXFRqaARkgldawhCL zABX+cknDn9&js%yCcV^$EiL)60CUf7mLb|-I0FjfQ)+Hm%Bp3vnilm^Kt(}jG=_N} zFGT)T2L)}-Odie5o<)6SbA5%j)i7h*%9# z7d1ZS&w`I#Uob@zhS}<^i&f~>7G7ip>CQ{YFdO=OF842<6$TD#TUF&A`>$kD{|72T z)xJv&wv_jT1?osxZNz}s378m!g4|`Q_IKD`)E4Oxm4Q>m-YeIq+OebB;p)0)%Gy;* z8IW9Nx~&p-07LAyqCqpHfTa^KNt_%3K-?QOoKS*1Zr8CFiIBb*AcE{K)W3I4V%eH- z{RKr+bqJ?JnX?Xswv8Yj`2@NYRuUReT2cv26#;bIj66zv5=E@^?qR0ewIQtbuI*4X z)4tK5^WG1xq|9n6D-w?r%4Uw0fJ&)Cl$EJK5=Oc2RJxVILyAyBBG_uBIA(C#^V^k} zTe>Fo#5b@xRQx%78kUTAGE%5ZN}@grT?t%5(&~_ckIE<$rERIQp-`KLd)1X25+IrG zs)UvfMztGTLfW-MF^To1=;r;}0B!XajWfiQXbE}dBq3y}Ev-dMWlAe>qfxqnPK70Z zsX#*SbyX=%C@=D0)!UgH(t<1_-5vh`5_XL}m}Rb%sbmRCd}1J%6s<%O2|MkyfFN?i z5A3a1HLj`YU3quIkg`z1fmI{U?O4JpSyM$2NKz6((hR{!wT#Y_ZO9@;?}YWHw7vbK z=C(h0lZtNYh-?+I;B7`m;Q2`fF*guGqm-N53^j7v@4wE)%hlp(Q@9)vtsA_HYVKYo zR2c;Rpg@9^6CE{7kYd6w0yn?`@9FY4*Gr4IiPWd5pdNnFZC;e#ObHuo z()ZEaZ`5){k%g9(@<5FZF$(&19u`>}vgDT_i6DhSE&5uuv&$ zQJ%uF=$pJ)&Q+F%8dz*42VKHc92$~z!Tm`|=P?(@2A6=Fj>MUSDcPzG7!F#ycMqmfOc3GC}D&wF$28lQ!-s-xV{y46i%(Eto4LSkUMMXxSZbmlZ z5hg%1VmUJ!R+_N-SVttBwc~Nqid|e|hJ%nLnuL_r_AV$RQTB1dDpz+<>q;d>Ga^jd z{KQJXREgR#&wE3f6r{@sTKhq|8ohb~{NufPKJ2piB+p!@AUXDJb_BByrLHx1-p^2K zsH4l9NeX$!nMedqrL7}Ln+=FFU{xj~8n#yWsr3f7U>vxp=U~P57ZKI7!kh*RgNDuz zEdVChuTa_u`VH*wix=59;gf`TQjNt^RA*Fem#Nu3+SH1Kf~QoZ&XSa@I`R?;Ctj5a z(t5Wa<**?k!Lw@A?|Kt%MzQ88cpvXR(Bg5EK|hwFO-Nwu0W9Zo9-QrP{6BEgIH!$T zt_bg3O;c&6TZ}1EQl(TnrqrTCM7R^)mn`a1lq3bX1}I^X>!(y200qenTU65TqgJLi zoZ}h!46T71F}nvW&IkpD^>JYy?7y35d?(qZHc7%1PB=3v6)~LV`BITkGTL)3y+pvL zD|IfrBvjPtN>b96mfY0@`d`roN7?Dv{{Zg{<1oRZej2ze9N?L9<+=X=Yn-K&rC|+8 zPE@5cf*w9S{yDpQ=}!xh&g}1FdoQ1>E6-KMo%WwC9=>K53B|;mzX>KtUM&+brCzfa zH7E&27)(U9%b7)U8>&{KB+M8}gc1ykCw_1M>l@?NYGq6ISdo4gI*+51r8(E)>d>?+ z)u{?h(`jl+3pNFiKvB397bAGJx^AgA8xQ+MjO{*_f;~!{E>dNJ#Fu9$lef&py=(DJgjfN$n2x7wUl6HBi>Jl@ z^xt@MI3$L9zKzYfJVaR4B%crT?|6Z|?YSm5oE&9_`jR~P7Kh4|CX%H%NMJwWYmR^l zw)8ZJpuiJiWc=@MIpybutKL3zi!wiSkMn=2PzA&tWYmhl}#Bo!Tp z*dF^}g!e%wnLN*y(YZyu?0pUuHyuPn4obh{Pf}Gs{luTB_QJ2;J>MZ286UbwRX;Cj zMz!;8VjH9to6G^XrR4KJYW>sa zQ3f6Ta!WtF<)|Gh72A|F-hAhKO!fZVaSUW)I*#O9=nwhAD#op9^Ll(?4FS`iJt966 z_%DjUB(KEvpEQvSNJqH;+5>!vF-- zu!!nu=g*xX$|gkLZ#V=KKKpyd5;G7NkARQ;Kp%#ye7<-fh-)ASoBqeA&VRlLLsH!9 z-Cvtho#Egnz@DFfx6=d!u=BY1?Gec)Pc!x~L=XaG$-x9ZKtP^nq>bkP0LMHb<+0Vi zK1Qa&@>qY|qx|3MR0&t#a(yqn`)}nr{{S!2G9zkEmhf9Cqc zf5rZ9tOZ*i{ol3^;r{?P(h=y~-_O%(?*qBp@9*x|LDBs3^4>6zf@0zi+;M1GO)^*E6p=0`~0vk-96ti3Tfq{ zm1RJZG~1u)1D`Ty<%uNwzIQQ>#D4XE<^KRmVaeQEpYA@EpYMsH7$BLs-bm+bN8g?Z zAfg3`jlna$!TCfO!43>4LtGN~s6Pf`&|$wj^q*sG;NJ*sr8jQzTR5am1``+F{{UNj zu!&}%ZG88FJGN*c#z`g&Tj)9a;YGYfIT*)c0rjgaTiL4?_fjuw(iI=5d;Ja;Ac+Yu ze2)JBJP_oTJhg*hiJ<8+CL;tOr7jPFpM+ybVnJ}%d-!vvyz2!)x%0on z`J6gto&NwxRTluWRQy=KeOsZ4V1u_Wxc>ka!Z=EtR2#RR{{Vzd25j%tK^-~IzTX3d zP@q+fJlf(2a!i5q^~8iFC4m9?in%P^EnRI$#w3ba&vj|@CY-6H5gYq|d?JH_Jp5ts zVYrUCSRmrj6VP?R1UW>WrV5FH;Sxr~b+-Odi4m`N)+%D$gcd9~o|F&6+#V(c0#qbg zIrRX~P0V4*_eX!s!cHK~TQLBon7z^tnrYn!K@l9s^M^!nC6jPHPTpO7zl-7-!P4Q$ zp!mO7mO%n)2_O$@ri0xu@JRRH&+c%5x0?5D=SK!8a?gYwQa^b<+_sZFMjnfvpHbxB z=MZ8GAQpB|@MGuUtT_blefs$jMgIV9qV~j*RhZcKlh1ym$cYF$j+)5{Hsox_7rEqyiD!(RxLf@asW$5B7# z4hVEy8&v7PBg@n0Hutt11BG&9^NQGmS!4cj)7?K9n9k>|qtr#Q4g(>RE2C6#I;_kxdN(yu!)Vy>@ zW&Z$&j?w!_BC>}ocO~K1;Y;ES{JBXUN z#{#jR_KeP#Eec>$`5d)qm5G@#-2PTb!r@4$T8U7O)8YRB!{=nZn9~fyK3U@|{iKz= z<#M~6rkT>%D@{IHg#;9-dKAi^Ej5{l`RN1JV>1F#mtjc-&JQk2VZ)Jmh*Z)G z8HI!SsrZHM!-&5O{itTVYm`#eW<8Q|W@5KEqbQ+5mYyEWR-~7+yuFlahAN76*V8o# zQ^-|IV_0x6%4+R8aaoQj6Q>#I#1dU<4a835v$N}?X&+~MA?Wu7o_0qsz)Qo7K53L; zaJXqpN?A(6VOXkKXy?=q-2fcu4Pz#u{7G$z|Nl-~jz`^F2e_F3uEW4I1U7E$H`tQ~@ET^)#tO|^kO9Tp*Nu#j# zhMoF6YTVbf_FH(4kWjS{6iD2_2{Iss zD1MT0%yZnOQYX$J5)CX!I{-_*qV=iJ$E5oqm`lvE*;3Y95h2JWfhF_?rGW8cXvK1l z4?U)FHw$ouBH?^h#CbMAhx|lSHJWBr^p4gDCR0k&>!@-BB`l~+6N9oDE+e!ZrpWRv zJBgXDKLdipP@W&n;=L7t5 zvTTnTn@p64BZ_3{xLIhHfa;i~%K#NAG3oCSy_Y#ySjt#UHcLRo9$%2r_iR(r(B+vn zUgb7bmnpX)3^kU~Q)Q~AFP6I_0^W@kW88%UuWw==>k%PeDaF}d7v{@Kw z7;nnqk^r$n2?`&=UD}cO0_|sL^FK&^yzMt`Gt5e{nTKdIZ0|K*VKMmexXIX*;xQ)> z^-5QmOGNoaY$hNjQr(_B#QcA!FQe>(i1>k8o36_coX0D9z}jmnRB0GqhZ}ocTP&B9 zmy2O-iEujOcX4XM`bWUvb1uYomjjQKlZKy>=1CyG3MM5py-;vit!;*wgHWl`N=PhQ zul^bH%=@#Sh)+ZL-(_)lnOUTtNWF%gETJII$7R^O*CfK`l4l8njTjMu$K!vzB)U|r z7B+HNl$Y|;vd-0m?Nd0nRENr}d^|I0`IDa#6Y9~^WLhFvE0*LS>g0e{pgtnzxWpnC(Ltd1|7|QJCb3cm) zvQ8TnF+eI*#R|z$Q4K0%dk(^0;y!f#E&l)wNbL-aN2DIc;DFC?(uC~SEyE{ zp%c&$AN#^eeUz*MxzdS3{l984b~tjfwVA4qT6988Wa%)z@>ml zQd|To{IK*?@v}QHI$Hh$(A1t1O~Gn(@@vmlj$_WaXFbU?3W`+GAopole>kWsQkzPa z%WAWl^#G-nG*YA$HkfQl5Fn`5t)_*?$Kl@-AP|sEFLGV#-1lO8^%1K$qbg@Kc;%E1 zC3Q$pIh}r`01N`8Yg8#IAnFbn4j>5e1!<80tsF+@O84Hi(vJ@eLyK6EYm>cnA2J2> zcM(u9%g$0|?FUm%g`qkIq@*OvjyWJhXiA2s4c<}G*a%9rz@+dUxCd*3R^^tAt4~lUbvDk#A+)`_1CYF77CR{ zRSj-kg=ptq=J{aN)JjW>Ng9a?TF+QaBzZ=VL5qdRk$f_8Q$PjWovC)_cNkgaxVi|)~CT9TqXYB$?S8l`T#}(4A2M0p?c>H5__D8_#&GQ<3=QC)omYu+=Hk?d{UO^}G3kUxI zl#}{5mjE}6X}zk?6NQp+xK#l${oq~0a-|P8CiG_Ro{4)4J2=PT^PI{^I9VRf9$7_k zc(<-CV2|9F_ZR`TgVAmsuWKixMv@SuAtAK@0EF$i0Fq`TosxIQ1I#6>8#8` zqYAlgd;R^9u2xfplfXhiVqAbLu7nnMpxnj7DW&3$)p)Nd;2sZYbve#-`dcp`6{d!i zt+dovzo00TG?fd9J>rod$s?zFG4y(SMV;W}s4Mc)YmF*NBZRpHMVJy2ctLI=7&{pA z7vit+8HMyi*bLh=l{xb{#O2vsps<;u=pj5sd0;a7?n!g%cTmkvH&b=abr?efsPi=9Syu@Nt zo~~-KYsL_@DH*ZMKfHvVA;O>&!7R53lCl(C#C+oR$Jk??mi{EWU+lpua;Gccp3u09 zD#)S#nQ4Bk(OJ7G8blN*oAmJj%Bcjb6l zj{^k)K?HAa>^LZs%|ZUY_&s#L5_45Dud0EHQANkYJ? z6q1z=AY*W38ABOW0u~aK>2C8Hia-Xy?q{L4ICeJ~GaEe1Iw{-j^t;95dO(Q`0sTO_}jQ{{V^Vl2TH(GT@j~ zZK**1Vxl#^lz_-WRAm>4MiCq#y>KLw<8~tK!j(4C@knL!^ZD+#GAdG7kX%zh4Zo|$ zr5}gS#>Qd8Ow&@-=Bs@OLRPk(QFXVLq-k1TLdix9Codb`VXij01)bRHdJ9HT24CWWqmPc*+G*oPC0Y%sUZGa7DUtn z_v2Hj1OEUI4%CkcdT0C(dOhuk#IF_EX;>V5)Rdg#CjS5fjUmN2v=-$kFWXLn3 zB<5*3Mma^6zuw@RlH92cu;M(#e6IAb_#E_&*nx<~c4A@ewo8Fa;j*wAHudSe! zn34bqSeb@Ea150Z#T>C$OI4Ruy+L13QAtxywKnZ7Q!Z1~GPJ0rNO_l1ml{wYjV>iB z8xd@Exl?4~gn_#hcTvi^F zaJbAo>^3G>5|YMbnW#j`VALnpB|h)~P?&`3#1ajFf@aCyGkzft)9#ir2_M73P2YyU+ap%pMj#vk^cle)??j^u6RH75- za6Kgc$LENZ?;kowLO>!?0R$1nLq_(LVj)zJNh7R!m>jts@Yt3DvtfTx{Qm%a5RXy% z)2`j)rm+##q@aL7xSwtR0Bc}|DL{q?UU3OVY$Tbuk9gb4-uRg`0G0uReNBbw5%dJ~ z4r$0jp!h*HiTM-$aT>kj=Saxm#|)&nZ+M!2#IL8{Scaksi53R&(+z|-Iz(li;^HG? zwTH^&+VFP42!0y*{R`TV6p;$CHW!VgAMbvhCk$SdW_|fuEaCqEjQ;@3UH$ETJfEC2 zn7$(blG>QD5st_2{CS>36TuMOFq40V{$A6+KA0h^EIhw=-s5ee$$}!r6c1R`Er{2r zf6^zyI1?#Dz{hsu6!EMzUa__PE&bi{FPBP^KpFn#9QzqLa{RUP08 zFb~7hd7Lqi8~490zMJb3#6_FOPwjyK-Pmn+`+VXWu;L}Q*3@UyzMDYI$mx0b8~H{V zze;$BmktlDNoo!!Hv`#LwuL55_dgRDNXW!@ln4TMAjd*A11H>If+4S!`rD@1AvHbE zPP*8@m9g|cOb{x^p@3kBASp;x&nyu%scBhP61YHSs05Px?A7Wa(+L9edqu8!&sZ^r zkmL$e#B}LLeo=BjB2obaA8Lk&-Y^wj(kFQ)I{bOg141r948S{mwD8kepeiOT1&61( z>*pA74EMX&&7lb(Q~GoL1OZkiPTa}mz6ga*2?V=uv@*oWOXw2K@2elc|rIaZ*Ev=rJA4tea_YZYe*tN zAP!@D^5q_fc$;;cHF|gbAoJ(X)*g@4J^qIcN4z`!MhihfF?rf{Hy%@eE8m=92qNH) zu)Q6$ZT21g`m`NCqyBGlSI~?X{n5C@;z#*06ZEONR>V_@LyeGl2e1RP9ULFe)Nj1e+D(cj%zG7Y2Q@}Ivv5W4AS z<~h6SIca#9%7keq@<0*qznLZnk--ZGy`W5xJ-Ye&V1g56E6+?Frrv;C)2;zw@qOQ^ z(X~Ga#^M3>sw`Nu9SbEc+r!de^`2b)zWMwn1SovRpPVr)XlnEz@^T2}@;%Qf2XXKI zI(^4HLk>ZA>Q9gA{{RRwNfRA5>+RQ=#P<~)LG$s4!BQYw*Wyk64=#4T5h@A}%vyw4 z+g}rVn4>aSmPZK;r=e#TH@O{lHMBSwJpH%%`QgymJ_sP`B!S5Cj+~E|_Q3=hFi@n4 zD!GW>RE@m=`r*mkcx#WVI4fz>jB$1r}o9axK=aV+d8^o>PBe9)5nlSdl*H z@A;UcI3HM|)b?uJha=1mq(P9Y&Ewwux^y$oZdR8>PW$)~^YhY=B?{88dT;1{BAz}Cu0V^Wsh2v=UW!XO+cB`5rDd7x~qOB~u-PUGLmA94UNtA(CS(zy?+QCzQ z`BsCkX>(SMC{Y^UX**w$c3ThY^K4>$VGl10ET`AvL2ucw4jhEk9yNga0~_a~FGHQX z^r@N4$~!HQ;!K<@<>B!;wlPyxm*px*_H!xOsR@`dM1lVR)9|8l)5ZHv2Ct}Hm;M(! zHteyAJmZVpr{g{AN+} z1o;?g!zIep#WHZq;Kbqs&|?;o2>|GyVb8c21NFbqSKy1W-HujSnP+%0+WynY6wNak zMs^PZwE`5Zs|_$MGL)e2Hg6o_6v|q)%{argsya4}Ju|FRw!#vY7-LIKA+(rE8*Kq2 zK>;BF2}%G-2T3M;quMUrlBvt}r-+c_hy^h3bG--D$J0K@dM))*u%q&YD3f$ZLM7p3 z7NC_V1_U`Hg~dbd9Cyb3sZ>{&wNwN)sS8PR@UIYs0SPH2#7PQ*Nr@aYIPFyF^CpE) zBo}JCm(r${YJ+ngp6s__u=rF-Y&}y(C@eWz!S|+>tZ~n2kJ53bQCXGdETX!OskfU` zZX;L(prO*Bum+`rsFgyy#VeS;hP?iIC-I_U!@u(fI(6pScSm^a?pZ4>oF)d6`l8O@ z5aFw{^)A)$16n=PaIQOQf{3q%{YB z#4xQjG0=a4{Lp7TwB_C5arQWChd9TG6toWTwWA@f%5uttg0v8}bBy0DaUd-xT2N3@ zxH#TF4;_zIE7K2UW=@oq$;)wA{J^E0Bw?_4v|&oN3B}F9kWCyUJ!8uM0NQEkGFN4P z1%8+D7+g~D*!=eik@jB+GE%*yjNdxSCF8h2VpMWz82Se#30j+lDkHqJDyEx5)vX~Q z(zjVurK!i%+S#LKu+cWQ&~>JjG{LE5!BCXG=YII7K!f$!QkeR-p&(lCu9qPEUE}e6 zmdz6IuftFYAd^6W?1BYCcvgTz7E;nn0w2eJ^cSz4+o=^P3)=s;zbSZRJD@?7M8nzL{2&c{z2WGfwrMhK5&7gjb*)P~{r z44GJ^0!aiFC=vt5y#D~R)3Or#U-*v6R;eQqgzUaSndhi85)~{+!uERymSN{9UgB|5 zbKG7EbijC3Eu{qn@rUtGvwk?s>&($SwqHpF+GiCmGbMHqYounFX)>nGdHRB( z#X>YC>OY{ob!8@LL&SMv>1xYwQ`NHLYH49E2vSOud)+#OsQ9HIks#xq%2p!*h~q&i z3~opzxoAVyji`=?io?mqNtrHmv?@_FNFck`g0B3Zn+U`5rqhsS$>HqjZ7(jWDn>(? zysZsF`=HiSRPvDGgWc$Dsuh4r(6YkRd?h~4^fl>5US%RrXEU5Y{KTMBfX7T&3YZ0@ zO)4YXc4?-7pL*o{nfzS-8YX2kv3`*|8-NoqSgb;EU7*VFQk0V-b}Qb@aMI<1rTXvu z+_ZwEF_@6!f+S|yW3-jE3ZQaN;Rlio?N`r8h9BEIX2mwnIBS8T3GdqED(G*}@McI`|W%u*tIYihsNL4dxHB~Rs znMEp;6r{b~mrk^p(p!~6bjcyyYTw{I{{T*}#+-jl)<(Lz9EU4db<#>H+k)bQ+7JWP z3k0^(N`lZ-H6#V159tV%I}!2a^B*{w1KAhZnAW7Xdku|aop_#tt02iAcQXicGpjv?LZBj>F(1SLOpysC6VMU@QO#DNsQr zI(h<>DII)1Q;25&01fqG1J}E+P2y5b4J=$ZW#%d-Q4jqsr8);a;-aq>N|Xf@R#BUS5g!y{TBet!j7{6V?)Yoe?9qpwx z>$pAr8K5g@X_DG(0a4Y`#DXBCTuhnv!nT#=Kp(!mUst3nF;J_fss+8BxUca6^o*Z{ zIBCr4wXXRKO%qBqD(FcdN@C6^Y$YimoMCf}(KrbvS_m`E$RY1tPL+Kr-Mw}v zvQv9Wm|^D4;u(Gkh9N2iD4s|F@VI5rKeGP-0R5rDbZq6UGK3{9Hq&W9NDvZ039#fe z>^VorjpkU1xS5<&4-@TBXdA;_ZK02D_ICv%DZ|Q;nJJPm;nEN;TaXm4nscelDs#8r z0+yRnl%fQ9w4pFSAc94d4%>);ZZT5_mjbU@Fl|yf3w%BsMoTj783;1v8@M1`v)cT; zdPk&x>2XDrRrayRRW)j6t2WB1xGA>GwrVNb&Fc7!EHmYQkc3Y~>R|N{9+q z_X^VF7C|Y)qo}5DXuV z*y5RPSgF%JVF#HmDmOZ6+ZvOvCgZaFtv1 zNMs>I6o>-$Ccu~@a8^y{Xt0cE*xb!)Q9MA6sdGo8S@gZ%NbMOPXZ=jMNJM~**W;52+fpDte)4BfAUT2wBsuHVax!#EfAkW+~!-SgX?D*dMWdsc7D% zWFC=&d+cSm#fOWTfbkX2JDd4t831UR&%%Y|t|72Vtbl&Gw>Rtn{1WEckxm7N(1 z4K+kP$AgBW4o~)GJF(bobgmhgR2j}lQ2_J3t~IIX(d=j8+wq;4WiC@wW?6&lXap2G zhsuIfg@mtKl1`-s>TzfS6cXLSO0l_RU9B?Mu41oCv1?p`yYsYgPTG1LlZ-^w;qc)k z9Z_cDU`$cM?fndK0P)56uTeW zJU%W?f7$Fhu?az!D>RoU^7)Px3TI5bY?O(lSeZ)WmUR}6Qtf~6m&ber-A;YN#Bt9C zB$b-{rI#GeC>uT@{^da0R1QMD17WiTe{^Of)7B^) zBE{PdwGbr^QxpgMU`NQGOuvLK-}*h7ip;w^EB^o!c4VX#nG&WeKbEAD3b9fpB}v32 zq?7Ap%ON74l~cIn2bN@o0wCNG1QI}k2(bzT5fd;4@zlXd4ipF_fdmj*>_zB74WsQI zL0q|rQA*IXDI_GNNm588ummVLDp6oANnjoT>F}tL09cqYeizLr0v z5glgH&d1o!-w&UjAN*hC`oRz#I+vv&m=SJdT>L+rIwDh*Fms~{yH>+*ybSW5K#zGR z_Y>uaRqr1 z`}D&RM|xWH-Y-OF3D_QAb8&`oD|6!vQivdToqM$T#AQsMU+w%)pSB)-l(whO>G6gd z0-czjzr+r3&_&J202=|$V|(5@eQ^=e+WBW2>$L%DTEc(4{{Scd04S_1T!?OvR$}lX z-ETg=xFCqDGJP$A2olYsukH3QL~Awx?)}XX*SEO+{{A>3I2H&Y>)+P?y${TfZ%iOA z{vLi5YHs9#n272(^pUXG+o`;7u^0tv&28QJ&^u{(OTBD#Zv#``K>|@DZim~?&~w7V zD3NWCcRqty0*Qibd70)p;D~PnBt^C$OkDnw06PQxgMtVewy^?E=kM!=DNSvi_8aX9 zdwBh{`L@-HOu!vvpZ(H7pEv{3aR9oGg_!S1$>H2fNFkZuLi|d?s|X55T#*VEQ7hAzUD({O3uFM(F7$Au$*m_`zl4v~LZ3SM_(Zm#tPZkA zKtPau;_-qOCQs}^-|ze2f)Q#S5Fi_qGZ%r>k?U#a(*OpyJ|93dzTY}{zxRLU{{Z6- zosUl;_wR`%>)rK=fkaunAn0Gx{Kv{K8`#9{geW5ABdPr&L;@?J9sIP3Ai)Qr2lvC! zt|43ed^_475&)2$#iUz#8y_x^H#Xl0f>KHil;X60gxTxk5loaOwuCz#x*ks2{NXUE zuFW*qwK?R!;$nWjvi2J+i_iuy{^ZIN6u%4%AKT(IK;`+AM zEq5>sY4~T3(?$5PI_H2Ji%2%#v`}l_kL*75QuhH+^2QbZtYz zL9_w4meIZld`v+|CP@i0qN5un0PnB~0F%%S;}cOp0t12)MUN3A0BY`CfVrR%j+~~L zwE~ioo*=3e@S<2!)y4d%NF`<2AQD&tW4nJ2pM)IWv_|Q;!?Z+I@ZSV37f(VI;~BpZ zww*;kmP*1LprKhoK&((p>PDkZhR=55xW8}xE@1KmoVy@eF}Z?EuO6c%gQCpoi8!XlFo^V1SWo8Eh4h!fh!c{ z(=I`D$E1$td>z@B4>QBuDVF8=9#KZpny00sLK^`|w|P>y3w`$`OIwdAN-A`aZU#I@ z+b-WvoruI^a}09vIHiJSE_Bamnm-$;Frod}w(<6Z(Ql!wrzOj9c`ieeNXl}2G_fZM zfWk=NmYE3*3c?e`HxfVfCakMhEgY%GeXyE^J`nf)H8@fb>;QzV7YhUs0hJ~I)O6kq zk2!Xew#%HUuQ=7%J@(d($aQ+hv%4ec+EyJBT0&W?Mh&e9=hpO&RN^i?da2i3Tk2YX zR)7wb>r$lvsE{^^P$U_V;u3itWuHx*>sK;riEm^%axdelVL(qP?^#D>Wk{DMT4f{< z&IoQ0)~FhgE2!r2s&g#e>Vl%Rtw~8%mlU7^m7yvK0Nwy4Fg7=m4mB)JE@I782rp*V zqT1J^2PQX+9tu>NB|J29#Z77$hdKdrz+T4}s&Q*sx|XG+q~69-sgOJ+OVco9Y9cq0 zMR=3M&hJVGZ5v0X*MA)D9$dZ_A*rprP3}31Zu;MfI9h{3m&{Wg5{FqPG+|2<4&>Um*Pk8aK%+i z+)2W0yOrk@?k=H7e{!d&}$jv>LTlH?8E^?q@nW*J^*kgOva zEmQDGwB)3%a+&Ecfb(=TdbxU&9=(63m19|-7xtmVluKhWqHD5M(^Aw^mZ}?DnXer1 zO4%q#aVUFJG!GIjkblEuxY8UHhgL5r;32{HnM`b2 zmM!v^`(Sg$;eQI5R$GU}keu_v*)WC6tBDM?FEl%MX9po4N<3sEuzrr}U8z+=KM^;L0vRm6M`!>DY@MHMJZq2-rcZM9S=Q@o~FOMtqxrERGJ9!>mEX9F(D@(d7Kn-`9qG=RYH z;4p?-OC>2_6_^l6u_-R?A8fy|@37Ld&foe{?Og0S2lq!}@VPet55tS?CU2Bx^UCah zbJ=;)l1NVx6A}R<+3_a3gmdl~u0~~-cg#4#^`&j<2_Sx?ts=^8aAgV*WRW|MBH(lB zIDx>T0T%>;eZf6nGhIj1JD&Y4SE{Bfb}gM582;nZD%36T7@cyIj2HH z^vW=o(vT9Runm@;Y8P4v79k+v*z7!+^4&lxuR{78Gt}GZcwCDcipytk6@FfK^E)vD ztZ8doPH!EPz#X8foo%xE^_1wA%7`FyLIS;E>9Iz8w(V2yvt6OS& zi~2*e9jHz=OsNyd^}^~Un^)W&-tBnlFz#E#`A}k{rQY$fLXrV>rKS|xkm6-XS_~AZ zLX)5n0hrVpR5G0Qfvn#!qir9Q&MK-K0=F+=8dw66)Pz)4vfP@PTV=$AJS9qU zebeZp(qU7W8&Z9i}2lz^lh8sds*4M zGN*P=7m70`;ZZ3|Mp21bCysbRNEli6mo-f;Rca$)WPPP(nUyk_q00N9ba&3TNGV(H zWTDWekc5uxBrQo_e&7lUNsANQCAyN!E=d4c+}Wyb;kVmU;?;zp3j0KWleuM<#*+KU*8XRhIHKsKMx_^9)}njrs z!MCV9>M6*4FC4kWnG)%WL%^DfhqdWRvXrMBb*A7>p6RdoR-gNpr2z=U69A#!gm33w z=w3Dd05QMI`uU`39w5t2D@`G!D5VuGB&;SiH>d|Sfnas6rcX5kXd@6b%@j4$!%rBU z3*&HT%sDaL>7#eh9W}GG>LEyVHc@a1xYQtkExw?2Yzjhv-xXBjdbzDA?_EtFSFVo6 z)V1-_*0*`TibcL3( ztBnP04Fk7Ic-rw`s-#E(BrOC0LAH|$f$kZNv1{i0VLfG&=4DNVoxr>cF$w!lYF|M69zXdopuQjmZGKmNIbR&NF~-frOyf~1E_t0B;C9x+ zJ$cz|gA0^lWl57E2?6LDftt)2&Zf+FjgL&?)&otbB>?Gha}Xr~K$Eco@JwonorW@A zAJ$Posk8t)>1WVwZfs*{#O6j)9V!X7pfO=`TiNkw#xwpiMp)aF88d-ZMq7Ta{{Zoh zX>9XYw~}PuU}Isy;1ZMTX8!Kn2Jdfn^toBh9nDtS4-0K1tGu!j`=e-H5{MNvj&@KRLDYw8@a3sON#YD$PW?f!+a zAw8qau&FF&WwUWiDqjVQ!0l2;=2y{)CW>#L{{W9HkpBSqOOt1_P^ErLwwO#JRm1H6 z0QDRuV1zn|AaN@TEF?JMBm}uqW9gHKYQOSdnNplbaZ!}IA?DDgg-ti*&|D?g6wOK# zUre{ZHB^r6z~jkJcCn90sPIZ)Dql)dN{WJZ&;hi3jKR(I%XTA#%Ox&2fe7|pi@5cH z_=sW(xkrcp0O~VP#Xfu4K5Ej|^WW^;al$?7u9Xajisltw7hQpGeTsTs&L^vdIC20ikl04iw#STeM=4+b10Y;@Zm3WGmDpFDe%86Bi zQzV-mryQpPgHRz7<)jCF%^cK?#r52n_YQHLrb;JH#m*`wz?S;es7EF}b$xDko%V&8 zBrCnA${SejSPD?ql#``Sa+tURQjlOxjxt!hT%5K^aP!HbN}TVlPlnpYuYthk9j3*F zP2!V*udsloH()5m?mjWN_+y3>NZU=Jpfu?c0D^oZ8%5%6&JO!yo#uJU<}K7f2BxCG z+U-Msq<5~)c3Bund;&@s*_gR{TbuQ#NZlDONVNB3Nr9w9c%&y_M1eYrjE#lyr($y# zcOX{O9nEug)N*=`wUuP50>H&bpZ-UPzRArOcdI704_n%zJm%8 zB#6enKQk#ihdS2XC_Yqp-f%e*O*nR`b{jt;2ghjKoI0oJv;`BTFe1j3*ShVxZVKL@Sj6^-n&W+j&(E1)hwKPp^V+}4NC&c?qGd6r1J zWTv^+P?a`Tg}E)1p@M`2q#J+AQjnPtiJ37whEv*wNL?nYV&{HO@GX7CIQMJ48e#Fs zoVDR4tdT&#n=xi`Z}9N;SFm5jLMvLOrmL-K#khneO+MzJ>PQJ%N)$?5BwIibuwWAg zdmcmDv`u8HVuaG*5@~%4+rN`AXz_ofKSQEQ=Pek1DH?>Js9CH9h5ma+&+S|A$=TO# zlr=n8zz$IGcOKQSr_|de4dhwl=2oWbX_`7HYaVl4RaslJE>5bJ zAJl2;86>3@RPO%((x|3%gQ4cinD*Or(hfTpxY5>CodM#o3GO)$om9jUrf?#0<3uOu#`w-Iwl>P|eSfh|6?}e-6Nj+SWl_G8l^WfZ4*nC#7R?uN-+=0rj)x2{2HSsZSD8%h>qV-d+AS~Sho?@ zK|6>ad-MDCGN3X?I>8Z?sgoNHzrNA;F%ak?5*7adCZ&JsC)^*ghO6B^b%P8*o`Pxb z_>?JycQhQ}WXU})Z|Cy$#4(V7sLl4d5%Br{00X87B09fO!Uq67>+|znZv+p=sai<@ z0e(<_ztay2Ppbrrzl*Uxd3PkSHA+#IT7lftrnFFF^*sIW%66EJacoGk3o!=&07y!bSQ1HkcfA;&HHwWgHEIO? zKhFCcM*Nw+JpTK9PD)&)$_0r$Oyu&g_?T%i@F(FP&fA_?B3Ofh5Ziw~ZC$GXJD*d6 z76Q1FdtZMdMEPxkAT=cK5!9)X9R8g?e%K))5}iQJxqJQp0GuH#9ww71GYR5T0Kf+T zsGSDB0^~F`Q+tAXh}`c z>}cKp09X=~4%^;62!rYMw?l=7DL~rwZiF%E^QOiEslDuZm=XNX%LGRZP>V=6_vg9i z&*j$y5G4=PyM0f?a8`^5-=0&EEalmyqiyfGILshAUQExF<(`=h_+ zV=R*3A2)$7^&fAb#B2ADcgRF*_m6kTLjjUUNd5gkeTSU7UcKK~N26CmUPJZpXaU&! z?f3g)I7mPue&TL^f459d03QeLgX8Dv^V~xm+rq~@ypA>1DOi{ z0O{oseZ8=z=kD*%yulL)2b9{^fjp69< zex7xL9gX8_^uOJ??dOGGx_s&(z=k5luU9qrbk}hYN9rGZjw46BJN`xl`@_HFVAz@_ zgj7YrBYR0F%HOxs1za(0^wO2n>kwiqDM9N>Z|w~)6aksoKmjqg_vId#l@>jI(T*U3 z#4*^=_(2dzF=#W+`wjkp4||M02rdX9iHV-G1IyxegZCIBM6mFl6H0e^6>Zx{SXCJ$jf@fQ1{6Iv`u9dN5J#+9 zoN&^*Q7L-bn(Yrp$JhA#_rVcC(5^?@;esB5#g4kYuLQvo#DGi#v5r_F2a9>}^1V^)IJ{V$xU>@_(fRy*uFOi`wGz_qXrugx8?%*Ofbne&~L-(uc+ReS4sw!I+<1 zEG$S0G1H%((H~!~2q4Xkc2R{OjVGs+a|1{{e8wD=6#Zd~k|zM=l%I6n+*s}$14~t} z4?^KEs0!(_kC`05j7s1N2idFVtWX80rUzFnPcjX>wHhR&bLsnxEIJAa-(!7`oM3_q zgFQTlKk?J1*ok3_*0=J|K~i9VcK-l7_1~V@YrQ+S?$x#o+~9>NDN;p@&g15Mqo1ZF zN`kHPc&9p1K>?lJ_>)wc8rgv#7J?**+HD(yezrIFiNm5GU%Qw;SMg|&l>$$ZjrSXG z=X?;cJtn|SzyOJo2m|hLM8UZ@01>Eb{9B>4R*-@q$X70k-|25#VZ`wcW8yXY#R<}c zvN%hE^eo{0Mvm7667W{el>q+J?VpVi__$Ocb@N~ZH{jSS(VMsbl~Nk59h| z--Onx>8Lwc?Q1Uyp#iF{65%Swl+dVdJI1rD;g=E>`E_Y+RCF$+6@lNrneN(e&mNS0 zDJKacenXJr3Pna^nIT}Pl0CT?d5I+|E=Zd>1dSyg5=r_c_*wib&&=^zXQV$!d1_}Y zBp%9kT6BfWmxq(Z#*EuGku?cfnxGJ=I0@)V<&c@v&?kDicz)L7{?e%OYIRf7y3;+| zWi25P(h%E2Pj=c9833U|;(Vn#5V*LyKob-R_ z&|*;~Q!wMytNooAbSht}6=whjda*h-tsD`@{ixJcE~C7t*LMnAZA1W6H2?~Zkd&!0 z028H2GBsn&+5Z4)*E`oKQ;g$K0+azgD`3~>-yY8Fk6`6tkt9iw5X?^!Tj&TUy2Ys+ z)tm7%`GVJ}YD!e4AxH$10u-dEkJJ+Z1$`jgW0>RfIoOpDRy;#9nm+uAvF2m4U~**O z(IgPDQIcZdKobTI*u|Nle*Dr-?p&Q6$))Ye zftoW(!}_yg8HiwXJpTYlhPM?7A_`NhNGX!2Xlgsi5&;04$;2v{o)S&N4VtgaJv0=v zyi4kVSYcDE^4gZMdcO6BC?!Y%0HMeW5~86f0YD~VKd7n%01_ujj7GJk+yXVA)`jUx zQ~+D6xrrquNq!{-3LDn@2RgF`p=h>MG@}5w8bW|lOzP4IOprlH(xNwjHE*^dm`N-g z>R3_yG}oU;jD{G2bqc5(wJB%PFzoLqX1Rs5`v8X>hIV`)&e ze*&56Y#E4m=}69CF_W|=i)#B`ID59w!+#EOr7c%`moy(*9FiJJyTD{g zG5UZ2Ccx={Z!w(8^ZZs{G_>V6BHCzZJUNO+@q@?V@zQ-pDoSw!X^hS^bD*z>CWwFO z!Pwq+*vAomA*r0qvk}8NreE3Z233_U_5M?oLI~tPb#tD2K*(gW8rW1;^g>F&iWVhT8t(pVr1vO zlILRNVGu}`h2V02HkfD-kepJuiDqzGpdU!HS5)H)O5Azn7gX!1NJB&=!t$G2-Qv=a zv@`;k{L;vD6zNl?OKANET8Z`bcl^Ut)Dv!eRc1A6YTuenEV4!vBp+o<cyp0%56pRe=b3% zWN1vQ!~Xz_X~t$Nvbc1X;YM(vk$-4prP{&53lVlTpab+V{foFc61zX?hc;4KSp3(q zSR$rL;zYG};b4of%G7? z$1ZEyZonCT;$%xmvydDYr!ivx0HkBQ568~9<9SYSugPZ&l7u*+p%q0Sh)u*ePwQ;+ zB&TqZv}16xu}Sr~cAG5KDN%CZJp)*{=}KuDMh_K>%P|I2@?_1R3IMi-?Ry^P8iCk6 z0on?geGIT6TAuLxz0$G`$zyC2DN9P)%-@$+dDhjG-A{F zdDLe1R6sXtP=WNj#{U4UW`~Qi#+1w5&S6NDRMOP8q2Dt4bge5wh=B`26<|S)kBDOh zlV=#-&*l=bIBC-`^Q0F_kl<**5kfS50DZ(|J5||?r?r_rJ1EWYQgN6JZWQ40Ql@aL z)+K)R1BD5c^=((AP*?YahXE*@Z;F5lQm6J;G@1F1&t@t@o=G7*SQ*n9k(gl+l&J|>;h!W*nvkicVh%R0JFCy# zadhUNhj?|R0Hq|Mmn#yslBUCTO4CqdSfg1Kgwfo_SN#YI&y<3rvZsKgFz5K!UKn>+|Y_})`B}-p+uI&a1 zC`W-ZR&^va5aN`N9u-wh3)$OW4t0sh+S)t1yPHQ~SYGj=^Zp)bZYwaUQdX80w4owE zQ_8Z4z4WLduppEqtVm2uBm;dR8L#I3bq)GPg5qi#eKqOs!W(3;+^7jJA!>Ey#`jWC znVYFf5O*;Ykj}%sytjvZC|N_0TYl=>&at4gI(C%(Lr6}QCBo_e*!WJMkUylYN<<4; zVr`1^wBEGjSaLl8A?;YTB}FMsHx+lOBoNlIy{{NOT@)oDM3LfJN_LA$lej1#C`^TO z#6^K}N#$a6y(|p|nh{vDEV=>f^D%3yp+pqKg~$Y(&4ur}k4Paxz}j2GF0@a^pCVje zXnV0C*o%!vQKVa%%FRQGX$S!GFiwyO)C}4La)Mw1&lEAZxp{sa6K63}&#)ibQ@6Mr zm1+n`J!3nORDf2*4M5NpqiWEXm71mN8SB;C2ba#@2p z<+HqEx%U+*VOf4!y$yg+>*+A36$B81FQ9>S0t|ZNV8P)LEETPZXRYeYz_ESo*VZ&l z(>R-mUj(0I8UsKJ4K$%018qo)!yGSHp4Bu87x{9jQq?4=Y9G|0VRL=VgNoUvYbOaS zi6n7s;Fr4ug8nb9?-?G=_GcBFPje)v4$j3qA?fbXyM=yn>o4FFhiG_fw+OY&ktj`>>M@xnmE_?hW1MXf&3?D_@g3R$=N??yBC@x-C(&{_6rJ?5>xr;z$4>e321yu{Ec{N?qP?$8%pyS+YD<~s?0u5UC zBtH(jX#Bu$&lVJfiDdv0>QG%(03;zNh|Mz*!^8)@FO9ho9wYw%sZMo4!`wgFc5RsV z%BXveyJBXGEm!4(u2d_9dj@)-p%mJ?mW$4*YFo$tr+lXM!N-|>H(-@NH_9d;w+xI~ zQp<+`;K3)0Mk(aPZ(pw;+4G)^m$Dv_C1Pc*Q)lNnO0XD2q#06>q-O=CnPHTYr%^uA zxYV%ZGkFo0QRkdBX(u^K{T=?Rk| zM0*3bAp@z$FwJv=F-By`iX@jvyI<|UQpPZv8#gN9 z;1Zt!Fhqhaw3yz&n33hz8tnYStP}uqQfx+qaR)1HnXA-_LDrfDV@0#ac>YC+VMNaTQ^4dzbsjqfJy0;SE8(@Oy5-L2dk znPwE#qmXv9u^4P_7H~|I0$i~Hz^%hs#)C~G*T2Kh;yGHWX_r{FK8*!Ut$LE;bxV#m zQb_P5gai;3U^g&3XJGI@cu z!M&gUCAclTn~9LqIlBy(IDd(0l7(hjHBkhk#*?b5fn+39(L7Yzp(F&U0a)}OV*7KL zADE?Hp6xFLkXFzo#?OU9aqN!FcB3opRsk5y1j3bqM9euL&5<~tWR$sCmO%b6 z8AZ9r#D7ZOihEh;o3s-$&e37etQ?S~;PZJ%TCf@BMu}uj9;qnxIDE!il2U~dq$Cgn z6sJ<9I=}Xbo`+48pJIGNan3QZ7fZNfL014ccRfY4FL>uNC5ffaxB|+HQ}$#Di6=sb z_Q9%a#KsXpxF9C|NBO|#bK+w0l`OdRX|Ajr=I!(85#C&QNA(GwP-JcN!a}Uj0kgj< zi(8+{A}I2-n|gZSh~Ns2bMxNY^8Wztx6cG2rQ9>`L(fn8t>Y9(9AQcW7=1^4Zzfqp(5IV`@l>^ z%ttd zgb!t$5dQPu&7daJ<$+csUA4Z~i<>Gw(d$d)#eeH|grsr`n-#EL>~wrnb<;ZS5Np zU7hPr@W6M_SQr?&6JdGU_CCP$j-0o{S3+3o4?^|SYtQWukmhnr8WlMYLQ<>ku!=jI z$diVT)5GbsWMB>u2CeRrHyTCAKIC)fI3S3M*!}H)hgrc0bl2z9R5NmO0Kp3YTuQl% zn|yFY2Is6tHzwDex4%3PfD%{&PQs=f29QbKE!2M>So6Xf(YAMXZo2g6)-Pe?@cU{N zKIk_Cey79^ur&|`rbvO-Pp79J6Jyi&{^Q|-65rJQZ>&mWgDJQ% zG7X3Z!adFiSdt8Ff(Ut6ghH<|z3nr$2&;)u-~cE%Sa1z&tF>XE+#R+P($P4Z_O>zLwr47c|trJf?h5Z!3t_yYCrc%e35s$!aj62Ry|1P#b)4Ml3?L%|~8;DiP0 zAmHq7Haw^J!2}ru*zW}Idx8k|!30Qa^;pD+GXT!UK0iC#1S)U0!^586XNp-VcT^qH zLFxbifynedF)}^T=TjK`ON>Ix7Id-Djs$|+{USnC6agIp^qyX#Fz9G62TG>Lb1~De z-=+vyAS7*ohoOQB6b~`!=g*(qVihH~*Q06Q`auTOter=GduTrP29&tyEnO>1 z7&;ZtV0!$z{hTZ^CA8rLO~QM%o+raf-_^>~#DUxr~oz#{h|VEhIA;|OLH zP&1wU`Pvd}w9TWkgYcSMixIPd@*6`EuNVjVr%x#a%z$^Dt>iH&ZW2Lo7?R?E26Ebb zH3g#@NChc0Qp2Dc6^+sjR5^zc=x5-M@ZQ&HS-Sg7<4PdD1K}*uXv*l{Z1;Ry#1v#H zX?QFS_bkg+0_#u$n|`S*Q-!Glp|Qn%EOzYVSXJb?a{mDOrY=xcGGa)44h3U~B?Tmu z&zzE#A<9)LDk&fqBidhtZ$Sgn4sS2F9i_p8*AVq>K0MY<} z0r~#`+kVxYB)RW2s;@%kx{wLC>sEOCx9HcfpPN<%GNc3`3X+llW+b~dlzz33G&@P{ zASu20`N?Ou|%|hzgaablzr6#Npgdc!O8H^lu7AUj;g3m6DW~l2TWt4msg)EG&S^P?wxa z)AS$k6ty1E_S~quWO+${{{YR^j^ViiT+qkGukFXh{%g?>p(#^+q^r02`bbN={{V4i zj?LUQE&!>Ej&9p*wKU=A=fOX(lybt2~|U7Mby#_We=&BfRCpD`zFrM zazzy~?VzC@#hO1hxond%Q7lW60ALTaGLlGlNCSmo!QlxibL^fUpZ@^Iv&GKeYn0h4 z{I?5aOouzx97>YiZf^WTDGF#MZ!MnXaQ2I{vIA3y;4;j~VITlmKx3Ba_kaph zGnldV5ybS$ewvln6W;1VNKpVFz_BLMr0sL#07RI_gS%Ok%}?zmB=rEDsY+}?Hne^9 z>_&Ab8-kT6%q|6uSi6F1^$txQ{bSHNt^~;V;~`x=LrlCLWTYsd5|UM=l?4D`D^8Lo zNrb`2HpAm*;UaFNG<|fdu^(GUl|P7&!q#KMwUrsR zS4!$z4tJ<=q?G9(&ZKAp6bx=`M9fAy&t`jGiJq!PB4GeK@mMoEyH}a<5zYHa*y$L_ zM-_w>WmO}I?)6PY-<7B%n7Dh4vuuK?yr+o6s8dTyDYF#?YLu%=Q&F)p6&mJM604IN z_99LjA1jVe2r70C9a>R|Hs4Wqj>hcgw1cjl znwpVt18yXZ3vE}_l9L8VjU@T=$2-olP!ANe@T@%;gUAtMQ(p_;uVfkMn=jN)Bi1}lF{*|SZlKK!&%KUW}=zZi9@OdD=j(_M7CChm5GH5b0-I5 zds~@yH#o!Nu=D01$Ud@W7&=jk!?*6HIb%=ZEJa)fj4!1hMqQuurP)k>XLGD9xze#x z%*+~;0mNXZDDh!2D4K0LNglde9w9BI%wifj2RF>~L{$DmGIjA>)mdRwEuB^MPgAzj zK-02G?r96HlG~2z3VBYc1)5T{Bj|5sJ42WDqbR}V_z9l13sR9WB|wxdo@}{*RVt8{ zC<^xxGi*oO9|i zs|IrR)N&TC<$(aSVO(*Wy-h;nG>Z;6JX&Pa*X+ye4b_0zz*-v)k zSQqTGdpCBam_nThA+=-NEcOIg+RRIx6~%d#H8xv98BtqpQ@sBGGF%}+rk*lJrPY-= z-AC&i!jm|MjXV4g+8BGauxs}}k92dN5%=0^5cS-sc&R`gIzs`ptQ77NY7U{igM2k) z7Ge$lkddVG8YQZwPN&4Aszs+#RHd#2Z>TBMZEZnTi&9?~5v>i4!w|#0=|gc*smLWE zMbtnRt2tuhTX%JM#48Y_g|(!G1+Ee-q)Le?iSY#sM@UcwCk-KiwVPcucL437ZOCIR zzi>V3{wL>ul_Jn`_o|eJ7;jRXR-l`sDK54l6Ro;P5yZKbXYhT~1O|rMgL?Lz7}?53&icF_zHeZPBGE04+cg zrIg8WT1eEX{*VkHaso~4IYwul&CKx%SW`)q=L(Zf0Q*si4O%-stsc1Shhnh3nah%d zr`O{qC@&W=U=LX^mh~=bWdZ{;rJtN$+e>XGSK&%Vq6&ZsNcRHV`P-GTjR8w6Do>v+ z8${`-!yuOl4RS(LZFFI^DP3AI9OF8AtzVU+Dhz&=1kR^Us5VT(L9k4T0#4>LSZOH3 zvgQX+(drK8V)wl$G<_={j-QX4>ZHG^YNR}#$u-9XI0ei?1NhJu8l!Q4IS?$%kWubi7b#Ai~L7B zpP@0=zk@9F)P5&BF`{mJr|P(or$SVu>q+4p{{UE6M~g^Gbd-y%pVF`j71sR-VJ2by zB4jhF;|rc8h}2}5;sJVtZ)6vG29FqjXl`dG9r#!DiG)f)La|+uatov*{{R_wbu$!7 zY?P%+Dpf-lBmo~l6%^=kjJ2m$v?aQ6sG&XEYh67fNO5f|xl&qFr3Ee2B-GW{w$fwU5o_6)!ScOP^sPfPi>V0?0Fj?h;FIsBtP_ zPXPdjj}m{>YloQP8a~Pk$)_LfoMqZZrkkW@nQ-R`_CUhB^t7(>Nz2L*iiVkefZuSw z^~5RF`D5_|jb0BKl1xg7{Kg9!6!ocAPUEfKv?b^ZN7)bT=gFmKSs!bqPvBqWGaQU0 z0GHw5&Vp6KDgx`mNDn~8obgW!@Z&5YmY8S(qyqo};Q+#;`nI?tz!Q%j=6L1ISFBvs zJ#}(Rq;lqzBSdd^O0UCjTr9m zGZMC_4)uES4qSuqxUC(EmW~l)P~)vd>*L9}98-5{{WAd|C!o5CBteoiiSjqJ;@r8# zCg7e%y7>GzH;OZGrAa%{#;!Z4bk(Bemsh();#3?~kf5Zjsx*MGkf21*hEs91;xCN8 z2QgAqT}Gtq!*D52jYr#G%rf6wE^qA84vsF)Dai7)bZ2H=q}Jrs^vOkMc9zx_mr_(n zg4=|x0D@6+2#`q=AWk|)QJZkD*_u*0mS!IpsOsHgiT1OyQZcxB!_)%q-p@8>tDAgT z%?GoehYsEaTBy*vwH1`54j`zIv1bdZ0Hr#00NufzNdOY5JB5;D!V1j zNCm3eS5ykLB}66WN$%fhDGCyC-uoZg+{Yrv;qkZ`r3#k*vPx7vRxK<+M8Fsjp?9;m zHjkeEnY|JAd$)NGH?!TKz)G1So}yqtRwDT*&vu6c+AOL@ zVS*)UTFNIOMzTu3+>|*{;ZL-M5D0c4kR##Oq|Zh@q4Zzd40mSpqEm!alupa132F$> z5h3_@_~W_Pd-K*C0KOc6 z<=WpkA|liY-ox+GXP6`#Vpt+NmHz-L0k8nvh!dpD9iwUU!&PE6r8~qvN#UO?tJwpJ zs-X83sbRN#3DS~eYMx`3!(rDD#q|(NhvxAeX25yR!~Ecez!q$I6Uch~Fq+#-_uJ3b z2oh&d!$)s98m)BCSFXP8Dz6sDZd^X1marf`#gyW|cwrbk$*LU*piY=p(TKmLh5KiI;S&u!wy#^e8l+ye8@QBD@ z)?)8)NgXUIK4WA_5U_W!`4T>#97NXh=jPVDS;Bw3{{Scd04Q(Y-^&DCh_0wBVh@;| zgup#_lhOxV5JW{PZMW9r=z3s;XQ}YG1buuuJz!+OJ8jPdBFh76@3$z9qCg~l-vmPf zN9rE`07HT%!~Xdn=Klanq0VkD4Y_jXf0x4uDoJgtxcc=P5g90szvK7*{{VZgz21E3 z&FGr9E^YSuz}A0Ma|GXiKZXcJA7s+3X-@c#sJvWqweRK5Cj>-U=~1LIU>SmT@{3#A zed7cq1eoc#7y0|dk%9=M?PDOBn2#tuO}>3+6-T_T@04V4{{XsAJ|q7CxR0mC5r3rn zeMSZQr@Q7NRr{yA=K<-zkp0idLACD~jUMpt`525)*Gl+Td)QZf;e^U3?|eJ`-=F4& zz(F_ZBb@C&1AF^mM!nnf?(l=po{+eb2nxv5ZZ8q=!2|_Fj$-2R*Prp$EO}wlz;gGY z0-C;?@1utoFhKCwBHuXol6eC`3{ zWLunA%pRUxq+DQHv>pEdZ$Ef>Iz1e1%HitGTLMWsRIT&&{qRfIyXy*#d%mztNfJXv zb*Tm+;ak&p=bHB@R993Wn{)apFWcZZTpi!KY))h@Hgrx0ydY z5JSO<=s_PG5TLW{K^`!0d5jl{fpJ6nYs^6qZ@Dwd_KOd_hd(5?zTNc;?sX^_7}gN@Ht{OFTBoIN^+t;7>!3ChJI>I!9U=a}{L~cg%1mZ+%-Svvt zw@{VB31F~51h=}*K-InwjVL8oHX=RbMEvn2Q$b_b=cm>sgRvg6FSqf6C4;1h-ed3Y ze_Rkl!MM5dkM3Y$4Mm@<5NRKOe;$~E2y}t~Ga^SL^&&o>v(j3-?)M!5d)HU^v2w!M z%Qv5(hYdg*Ox|Q|hw&$Wf%`>oQn6r`+^g!v{9x!<7CvNH`A!`Xzm_p6wUIDLk$e0{ zukdja@7?tdXlI$O8zIWG4A&NaWre{drMMh(ihy7>AkcsuR*gv7dlA_T%eURC?Du5ySxK9T z&a+%b4n`?yQokpFlOkp!WYrN=ph_HRT8KzMmmEZp zHe`kapbe~kgFO-YI?H=Klw^IG%5bve;Bsj)@c2n+013;GgvyzWm<6WeF(cN^C6uQs zCXweCZC#@rWf{_3u2p%9xq#GX@cGt8D#NuZWHA$6UW!dYo z0N9HLHWBo@(U)T-;c&P>sVRDKQJ6iBzt)9?c!xF3T&k&cZmg*4U6T-HM4%E?F?9th z#OefsH{rP4V&_aNU$$y4`q8;=;fp&`zM8@1^6=7R`n0nKZWBr|C!=^%NXLu%)TI>} z0l=hnKq&+&2?7DRyd5ft#x>WgbE)(5jF+n6OATp4=FTm*m#au&ZIq_kl!>q+#1cg6 zo5>~=Kq)3X&~OTp6}h3IIts)~LV~MH-j@3@8yGuFDo~I3#-#%)ogkDzwDVr50FWdA zMevAU7506)zn+>xFW5^8)Sjk*3-jJ5Af(<^1Pg*nRCJk8AJRcc0FmL@LT3yrC5ibp z`FtDU6-oecMm49yr%QK3%83CfLWnm682uqhP%2O9Bn1GHVszftUN2{dpF3ZByXmQf z2~t)`Q*8DE_pdRx#s>P=r@}!{kVJvvxgIV-K!608Aw*gZ1=U2{Q{+3?S|yUlLTqiO z&7*0{QqLIeNl29t0#XPdLWu-wDk9+^8)^nTLfr8ZUanrdznA((N?0UNYJNk{d|DlF zf=W~rl>(qANg*H-1VNbq3Ed@Ig<<01B7vT^{;_^aN}R5@HV>w))KW735Pb}A*BUqg z*Z%LquAzl@t2G4WJZ{r0X=J5tEl<=o?{>XyIo;?`LbSf$(2u}U!0jh)r2rh{Sy2_Y z`1A!UV3EW@IubcY#Lw-%mo$1S^c#qO*h}<>pYfoBU;h9nJz@}B_?B>T%0Z%?AbZYj zd80uEbp@trl-p|#ooR5TBqg9rzB+oA+X`E1c9n>Ry4IQa?G_e9u%k*d!$hB!Wo+Km%9dPz=tHK??Smu^%@->PK0{`Dfw=pFYSR zezKjO4#|<4<<)G2(ma=|vxi%44K_705i2dMxZ{X@RS6DhI#c4W#Khbbj@4wCY*Nw_ zJ1;+#0i}d+%v;nI2S_P>SxGvG{RV$zhFh3peGGQ*6P2ob^IW3py@4gEwwoCH4cYF}&&%;46Q2o+!nmC&PKk4w=z^{u zJ-w1zuJ0bMb{*S|CSuDp6*Xkg?!Mf)=7dAW7l9r0l*Y3cp_^ zBh)aWQ^W}+?YO4DLmrXIv)m4QhLjA(o(*`^0UJ1Q)h>0dS);Nu46BFu_a||O+GXzf zl8q_>Wm?s-NsT2^pcHn418iuhpNPXvDwcRj(6}P59A1Xvn=y+-Pr|HAAejZloC=q1 z=t(!*oOz?$$Kg$eUa7CksL$oxD{!RNOi%y<;YtFG z^?Ht=KR1fF?29hU(Z?w%3Kd|)n!hupdRouL%iwM-;D?AqRTVjNODaK0UbY~_pv;fc zbQ2yW2(ZTRjPC`QBh}2{Oyj8_5GCe8wC0U+NgO;Axc4H0s{HUQXKQY6Al;U!UlTw}{yi~MSW5qh^YQUzcj3w7k5=p$Q8VNhR8x#590}w%Xf5kwnnameJifVI%U#-*RTA zbg46?kWxTUAc6{19SQ^%DNs-#yAVNkB0e0?l)Nvr&(%L{5(6j^;a|wL5hp!iRQQsBRt`?y$ zPt?QZ%i}e68(f`-i0Jo?Z0%A?8+|1@7Nj9clOQ;qO`!BCD??=32~p*0{BE8Tskv?4 zh<)lGN>^Kf3L=eq`(8E#3%O=HX%xrZZWM%)ORxZvSQyGOt{znl7v^eP0;Z{?))Ot`Aje(e*tvP#~RaB@7ZXJ|8t0OlEkD+r0HDVdhwiQ+*CPzyxtVuEKW1tC%;i84qB)(DSQ zlr)ko2nJ6(s&?FXMUK;B7}qfA3R&k}4z+%lxzt)ZZaAh;qY5?mYw2UFR=dU>n7dJ0 zu}e+}M8eiA6*s@Q1P!Eg#&SZXNcM<&l)W|%YWv=sb3kDA@9j&Pze#nu19N z;k2VW5veUsY$I$@C5i??001Cz)j!fEBwFC}wBH)y{ca%f5?kSBXSU~F=-=diUjr$V z!tCq~dDBx(K14)p&xjhPWlz*TntE1KbtzhPT0#&23R1`gNl4U8*jSNo1(vR1WJ(E{ zqS)}THLp5~Qkq&YndfPw;U&$PK6j}^vHK}$P)epygTR}w{Gi^tzW4kDaV}?_eldF- zTOm}L16hX{CGPku{{S)_?DLBnL8wv`g@pHxHd#P%I#g5QQh>*4^j#wfm-f#kM-rEb zQOyNM8{MOTI|GaSnzk_S2E4>{{X|kq+Dr1!(})*eVsDU5}=9bJ+kcJ9=MPK zLdghq5D8N~Sp7j}&}jyw7ZU9gbmma<%b{$c2`O?r*ASg<8YUa5WeITs4Awl5;K#DT zAu`v*0dqp3LLJ;_X;XR%R7b)rDB_rs2M|&c2?0g@qytdofM0|K3WLJ4Sn+T1-ug>i z2Hu?y4kO|jaGDB`OR8R0!ZB+9QM#S~+`yGP6(+ zr6x%!nDb9AkrF)tJu%`u=N3>}U$R_Z#F7gWV)xy{$K0;TWuCPXS24*gMsq;+W2@@V zO=GAyYatq1@PHs`Iz*Y%Zn|Jb;wIO)=Q)Nr?n%*z=Fh1v5A+e=GJLLFsZ0UXmUr9X zaBN7^UEh-As7OniK__q^-%&`D;Q#_HVo2#2)NwenZ&&%$(@uK>O=Ee%;P`)q?^Znz zJ${yG*%TRaGz&x$NH*}FOTwlgkVhemT-mQm7CNy!i)~Iov9`IXa`?` z1zH48g(pmJ9Jzu4xF;1BNZ1jgWav`Qos@b6@F#uxS@$shc$8V=6SPzk#TOGQhX#O8|-{UrbGiW zbwLwjy@io8OSu5*7zWY4PVm00bNGFs&QgsqL>PidowY#(Zy_YXl4MTykvM}#aj76x z!?WAvtVFqBEW5YOoBA7fMS`ZBp;9A38o;=an2;tTV=yCDk_E}d>@@P_jmvD*sUgV* z?YD$d#K{YBwQr|i>qsRHbCmCeCE}9MM>qv33M;tNB=aW1*N_f39IqCqxF)oF?_JLS z06RvHoM4l{J)JAFHF;j!_(x~<5!zz%yuytMsdYWuh>$@~fVj%kkP3nj3QQAyNL< zUE|UJ0Bl#;tj0&Jn6+j2@xzOj326wg4D@X=B=I(((rzQ?=i(>O<`>d$WwDZ^NJPHT zWeWA#CMc)dS^x;f;ASbDO2G^gsDwB%B%o4F!X^B!N;pTBFT9GZNmbL$^j!2|(B$sIQNeg6P?!2~l-vQsgpO&#!e$FgXO%< z$1}FTPuMraL=kXDScy#T$6JlhJ)h(`#B~q}jkomlkH4-YR*A51A5NU~)&&qKIUCrA zpxe&S(Yf`A^q4!H!5E4yVL=|T!5-^4A^qq@D@ZUWZ-=kNHz5iHaVzQ57{vLYjS_VfGYBLp%Sjltw5PfH!}MA&eBYD*Rl z=RoOGKG91)BWR0#aEY_hwWX{Bd?O_h)Im1h^ZR*xEO+t%^XC0uSV3C1e_tM4tgQ^Yn}n90&_5oDHqlaBs_hf!hQS!U2io zWP0I?ASAg0*F4}AEGkWax$eADOr(ADKYn}SD9K2qls!G`)A)=DN+!_}$o}|+N+yiW zq58rpMPEn|GNJsAK=k$A-6Gf#?+*U}k%Hbkf+^DXy)_L2!(kz*-BaZN_dp20(s}gx zblVdkQTO^klg`jZFZ7>psKcN>eEq2b!2|_c{7;q7F1P3HxWb_ORbhTjLw`K=f*ZCH z2$+t3-@XVT=4~E_()aT>AbQ@wVX8}ub32+ELh17#Uq2llf;rw|E&E>#O9EVUarCWW zAd+O*$Duri&^qrvQGr158Xpc&r0C_e1_+WQ8Mj|-B7NcC@-V6QhkwYy5Jr)GrhKk< z#EADtolHoNba~Xp5pxC&rqk&>hW=CMhZR^CZ7FBpsfyT%PRmI4tHcG>YV{r`x)g0- zK=a>o_wR|3?vDQenTkXZSb`45h6tE8CvVg5x%KHdbPz#4#!dGV5IwJg2s%vifH`@e zq<;QbB4n4*fYX`zH`kOreEAuVZkO}65cmB2b*C!ApC@mdb0&>K3&fVlvAK{3FM0dx zfapjRsj0ZzeQwaB&yh%_CN`MgYY#8KEGVtDv|}gB^-3f;LvcG2!nC8gr8O=!hajIa zI`R+)-_xE5iVldI^tY}EavO+0cUUe6nj{0U_rVA)=g-TaiH*oWnUD#S(2{rbkCq5n zDBqR3!20v)=Y&3dk0#o|4Nk(`qW=I7?lB@&!nYQ4(Y*XE7@DP33`U2gJnIEP8*S6f z^ZlTGaH|HC-{}gX>;ZPEAd{f0^?s2d3gmshk@x&zNGbP0FL$lFHt`^%4qqrTR7Hn4 zk=8c%{%?g_H6mYJk_Kgh7IYNH#p?#_=4mL!iN6grEDi9QT9p-`md&;~O7`-UujE zPv2iL*Ut_uM=iBSD%Z_U@Iex(HV4d*KgIt1qbX|8hoEwPG_(+8#*@C02^*6TZ_s>z z!aJ9x3%vmQpNTNOtO-{P{;!8t;lE8f!leq3BnT$vdSBc7;!Y&*^NM2*y0-voy;yEd z29&F72TGtGQE+*B7~2g=B~`f;9pWM&fJp=d`^4CF+rB+%{{T)t%iBvFOWPi8OqCz` z_)~=xv|Y3nrR?`GW=HBtLm6saPw7IcTPg=qapylyy`>V)2eQ~%ixU!*W;v5Hlr1eL zQh=x>Dk}+41d<3E$I=hL$D?708P}xD!w!;_DsN{yG3?e>6?m|$>o7qfmZ?*v%F89K zE&>eMM86tGSoY(^iKEMEm_w^=L=-K=ET%~WgvxwrB!DF#f-Q+V4a?ob}j=GbTz(i|Jjg?8A5H8RN*#(72QHmGvzblqGH=#YzfEbxKf4 z)hR9{jY%U?0Mn?X2=WhTvFo2cceD+_U^&+O>imOTN6|jWWaABmlA|mJZ7S?VMxoq~ zkuj&!Id6KfD&Y=l4GVL;v%nY&)y+Y30NrhZqIvjXVu{B zuSwDgGbU6-s2xB_)u^X(1&N7J5HQk0N5Fldt3g(p7Yx+;HkuAma-~{UqDTOd60$-` z3ko6xNRpJu8lpsXRQpY%@4xkm)6k(}>;;XrdKR_ov`-Fd2?0eAK?NX`l!5?A17W!` zCrE=Ek%_P-E0h6pM^+z5ad_Aeuyp>VK?lN0NRkQF1cb)+kf}IUR>#1=q@`CQZ5#Uc zz~6FHVqlp`8n#PeDLPDSH-JgBlNMt=lzeIMXndq_D&)T6)IWacvecy@Sa*ZOV&Ian zHZTMkfMAON0kor~HuzV@FUSgh7jCrBIc|Q>dqyi#>Oz(Jr65hhhv?G;+UhnZp#V%- z48-$&Kcm;;(VCKkgbi<32c7I$&3oe{;Y!quw~J}e*?n~Mv>D%ahRVtgTlvhbFBZvc z>0oE>IAPQUDFLDjYF<6t_!P{BAJQ*ra5V!P*x8ele{Oacv^g}1a+1T^TIJ$Okbvq^ zNhu5gkB>jwSJ~;f{{X{pVKamUDOoS2e40Ykk;ElR&AT_+2~(FbL4pv2mZU98Pynz| z{usCBYg4Y&0JOByi;UEll(dmf^d-LW+C;YfGM&x`na+Zh7@Nk z$`z$Us@?YvDp>wv@W+Z;h*z`1fK=s}f*h$N6qaWJln4Pqy{Pd+@l}&jXMMTx8&8zg zF!v5{$O?6BI!X;+5v3#qIQkoJx=IZ?66(_1WhexYvyYvhjH#2by`$`&B44w`fW{C* zfLT~qt5u$^8nt-+Gyec$8u0mtqF+jw8I}C6Yr8zOxdf>=+y-ntU`v)q77!)DAuPnZ z2;zP<{4ggqNBgEih0<5H3yi%^R)x0Wbtwu307A76TZxcCIPz{&+qt+Edb!8f$OO`1 zZHlYFxa|&(J2fb{?kxcZ`~Sthl;rOvYgSnPA#UYLo7Iu z#(K&ep9+FU=UGqc00fmhktcPel1SeXHzn;1jwwqBpRON?JQM-L63z#fG0Ns(`%T!) zRFgJb+42&@^TxJimlE0iLY;0vSzsEvni|y7gGUjX!(yb(M+C<8%9yk0o zC24m#;bGlWH4iC-f;Bh<*&@WGkYISDa5lz!1=?BCnocaKV{l0umg`QswzYA&b_Wo! zsbMlE79mNcOvnkrcY4wY@g+$GK!6Hl*ullw z(`Bk;tY#R2CCj8;fpI`@bI)CjQsgYln<8-HT7z+s?9|=u&iC<-dE)LBtfJzF_<7~L z#z;!DB-Hn?{Y40Q2oy+DvKwlNge)O5rN^CNzW`IN4|IAA?a%%u?C{Dl)3~!diRHS8 z;>G2Y6h=6+su=^iHl)gTgWK~O<{x>eiZ8Yi?)8mg;B0%#gssfOz5(uBw? zp|_C22mt<<0DU4q`O+}O4`haB6y)EfAsVbc74Y)?@y@-V@XM66bac(NDMd{i5d?*` zsfLjx&x^e%Y=Uhd9flwzH`118<}G2bM9YKQ8kW@0s=VM%xrHK7YoBfLpgi`#C|jDspniEByBamhGr zGbucB@aD6zOclfxhd;YjUBeR8?jD2v9Zc=VX{FROS##u@AQ|2TYlJelD z;6A&6Vkb(f(<(&CYESK$<(cy#1zQzLk`f>o@e2u_M%;kf*x~pYhGFAdlJ(Sr{l)d( zvF@zeOuRgwS1wXRi)&6=l118#{dIkPGmj{sf(nV%0KgMI6TXl{UI_wH1e|QLCn8Z{ zR=F*y6fVy7_{TWKV2&Y0-=GGsYBP%ITZ%*S{GlzGS}z32xv-ep;1RgI@8rq&wX8Zw zOByj?{{TBwQo-#K;PP}!9 znjA~Hs@bolB;>uF=bTSa-ebheH1xpgr=SFe+DQQE(-%<&R<#08Gh#7c8QKgc4L_I5 zk|xY*%qQ6e+>R?Y)v!8;Hu8T*xsG!_US--@%(VhA_*+Yq-^~8$BE*4KVAk{x@J*9t z+&}n}?4;5mwu=?qLN!7)DJ$UqA!SL0Eh$oz+loO+T6JsEIV9{}h_SQrev*4OX3bFn zdC;TZ`;#15&4{Da(t}sye~zq|1(yC3y(~+?NS2Nqzp|l^2xI>Mj=Lv93ss(k>>8#$ z`OoW-Pexo)R8{#TmFf*Nr737>3kXD0sr1yP6$lAo_cGd=l0p*0OGBgR7-wLx9GjIE z4sZKX!q*&3My@0CSCueQe`2TMaqLS4$haP$aRSTt7GT z4BLqmrqY6xg4S2Tz^7N{Y7*~oTnx)jCAQYlr$|y53s;hV5z|U$_I_9-w7l5UM6Fp# zmU#u&gpkT0@Y$p7m-cDl5i>3A7g|D&U$LUX>qK zuC|TAl+px&C;)z;BEYAa*p1I1B+pE1IE6a_$Y|z(=z3{!LmO0-gr<$nzp3fn8J#l# z1i&&$kM_AjP z?vm4pB`EF}y@29+N5OrTsi%oz4jqGQs80@$? zoM{Zi3b?K4H~3IjM~@HZ_#lPO7Nv{X=ts@oyEu;{e;r+$vBDjzqlz-xbkb#fa@z9; zLnzSM&S}X;a1*3LO5v*Dk!@AWAVRV0Z$zD-R%wyW%ds;Qyhd63IZj6r!c9;ZR3sAQ zC29dpU$jJ~4@(V3pwwI%5y_y4>o<>2-{HTOB!VN6sm`!MkFd1)lLQWMaT2IcFaY>J!X=WQ zTR1&JNm||NML;xS>L8J*1_bo#-p~YeJO2QG@rFg!``z9iZ{HD=Oh`7{%+IgfUT=aM z0VXd8+X$;`-PCg(^@Jt4@)v*;xjtw7P68Y=JCRQD2(NVaeBoFB02m}fisThq1PHfZ z-Yj{^U^NLl$k)Wa#_z!0MK7jcO{}Kj1E$InEl7g5U!=gLtjgsVnX@{QA+!&z1G)KwwX8SVS*u1a$9EWSNlMfxYYz+IY76^)0_@x1U3dv(K|;;xjlUFK?n>e zT1Di0Tj+k{5Ubj9)aS^Lsd$?X{xh`IB@3r8PqzBJI=O+-mT^qYdI+T{GcGlCW+^9KYFgpFhXE@1VtM0?DG(AeJO z0-qfX9iIIdY;=rnBloNB_#`|yZ_^T_jvA3lRlB|)BM~H4wiPe(@B74nZVxF32!#_v z@eF!2NR<~Lw&3}_D+UHYKTJ6^NP1rVOgO zKypV5u^jF`0x1(=eM0*lw#2I*-w>!Ny$En^;L$-Kg2WN*>If|V0M-nXAOUC>{zb3% zz6yD*dK-h>+P<%^np3E;9q*%PWU4I(kgyjw-2FOj)Rv_20ZQBYLcsx$56kb<3j`gv z0#l@t44CyX9)Lyp8Nm(;U`2Ngy&L7;@QzP-RFF^UAjFM<7?mV&ij8&VV;_i0%=M`# zQ69krB#?cC>>0y%TP*9xknwR%@ri7L1>ZcXs}2#^2-T4G29nB_eEFuWsS z;SOQXd!N6a_;GRoxcx$7awbXjv>vnNhAe{2LBG-#3Wp?uH_{3H$E4fX;XrhNe^FCd zAmL)yi|%I9ZLm8GCk@CTP;@j8$%)^-2sXG$1DMmy*n>R2Q{*uj4ncml6{e;3E@*Y$ z8ngR3fK-$yYy7Gjb+uw{Q$tmp<{4I5M1u8JXHdl>6z!RD+K1gmJ541!l`ZEG){+W> z)DseLv#}Dea%WGQL`h1PRH`H;Ad;e@z$GahMMv2H65vB}%(D@k;<9`W8DP!LmL&Rw zdsEESO_zsEl*vU+fjEib@w$9B|1Si10O#7XP-^Zb8O=~oLyMus%9PPW#QC+!%Hl0DKz0B z(l!?T1bQ3nB#y=QOS3(Z!O519mF2Rq^VEI?yxl9X0atfL?|d+ogq7*>r0I=6xE4S&ejFizqb8$^gE|y3Y5Z(le`H8M4TLB zWAOXKRWFNC$-b>ktUl&#JIPQ=h$>14i&4@PB`$Bu29O|gkS~YDY0`m6;XE&G4eD9m zy3>~CF0E965)zOB1}14Hw4;ZE})c`8#Gj0kLNGbetpjd4iuDvq@X75 z6fEgjfI&^XAWGl~RIf-T1`*4<*LSa$i1o@*kbD*WslAV1YQF^@m1awnm127(JF;D^mO?nl8E0rRygq11 zPwtIJY1OH^mf56Y zTdP@jTWkAJXBDIpl=EV0)vpTB9m86;79@T?u*fM$l`JTdLH;6WctHtDf|A2!aG(+u zvGT9+FqAXfpi&g_nXBQNBn2(jou^o%%oL#G3Pn2)1fR~WYE?=DkLgp1@au8U0e&{{ zVG{cnk4Xv&R&j(SH32+WAdVwY%BT!j@Q?|tevH4cuQ^b_`e^L@+VFqijK)|bJi{>^ zr!y5TG*Xg<6orxr3R+64p~aK%USCa6(?Li|(&9h>jY)+zkehCQQ6Rv$>TmON7c_;a zJxMx-U^IKPQ&!Z{KSgqn5?I=l=VE)Mf}^3Q9`Mby9$iK)|;>PGFdl!6Xvk zg(QPg)CAI|v=z9eGHFE82`VcDk936#7u*BOx{f1QanA_kxweejg|=ADsS=l4S^z?w zY><_(=9mf#M5!f5^?;xw7qD2Vs2mF?)VcP0>DIKfSZfyKDrJ^9nd;omtlERn8dMnM zZrgZVRAea6>6=okvTYSjrWTbt%0VkZQbCzmN|FqMu#*H^9e>eZYSr0p)?hJaa)u8Q zp(7fCSR5WOaL++<02q0Hwm?7coC=uW--z!m0USnL`Er+qm{Q}_;^qq&<*eep?7-3Gs9S1KZ49Z@k~9TMs;bfbEvO;dhLE{B zOO)-W!jO!8B$bI1C50q{2m~sGaK8@_Q1(dTbiGFvI(~lVVP?vlv?6^Beu>OKoz-3B8n zs|p){!tPUTHL2N}ETJJ?0bm1Evo`+#BKovx>h2n0%GW6aMQaV08$wfMA#NxJtunWo zZm^Oj18s-}Lltq8<;a%2M8zc%(uFQ$#gLRsmXPXNlt>9eLR3nW!8|0p0%IwcU}t7{ z+$IkTE@avBCQFklOr$Do<<6Y1*vXeI2~yWKZx9mDl$9)y5)?}t^SnCO4`of!4JFo^ z3#fj;IP#lGQC)tbA|M8*i64BOu1(?nvd@+-w+w6W_ZPV%tXE?VvIbXoE)bmu*sAM?LA

g%&=;nt}<#j z1^f0UgzarBi#VBRh5Pnt3Yny{ zX3i`n0GUWw(nY!Xjsq@F6u~ZbE&l*^f+k8}H%TdQ;jCC*FjBIN^Rz2$B2|{)34iYl zN>%IT5JQ|ig@Cz6pS_xps{5@QPlk8{B;nbqaX*~QSEWzaRQ~|T^NZ2|P$yQ9N=X0# z0?`CxS;yyCynPCcxd-@V)PYI>$75&GvA<+Jj><9+nUau+lhOXsio~?ZlRS|3~ zv%Z8zV(oKj1qcZcEMNm*M!;AqY+!+55sX?;LVyK}v0~)bf~+WZzLBwhWdU3u25tPf zJM*oKbZ_9nE>wOdyESp{{S)L$#81yAETDkc)Ric0E<+g!k48&1N!uKO(Y$E{JiX&jaY<&cOfM6k8^YC zim&$cA%f6lEL69pC=D{QrkaGgrIwjnYiY+4DXS^i%gaTihf-4`=ovsy8o}ZujZ{zZ z0?jaxT-BWG%0D*wWcBf2m=uI0(D4MnvY;32qyXH0%F>hYM?o7e$|?0;!7C`ys}&eCDYbQF>K46_c8X~*!YeAJMA(JZFP zF&DFw{{WX{NJOx{2sLFN->ZPT7SJD+~|w;}NlquW~UG>^GGjs47^PfClk{k~Dv zTs)~mKnU98T!g_kHzq7O5&*|J#;Aq_RQ+B2K5ZSZE&d?-AFXt+O~h_&tstQ@xd{N7 z1QU25@*0lc<6E8zEJKh*7+8`^0$dO&Mt0{LEU*GYYi^!j#xoXRB_tE9BnSddsa50w zX^646G6pf~DJ~h!gKy#DIg5~D;sUpGeVkI1^H_K~XT%KoTl9hlQ+Yd$?_r3dWY~U# z<>$UGRG=)~L;8dhaFmH8kN^OHMZx3+@d~};=SY{0Y#5Mq=lax9g)3JPl2SGxDkK$1 zx%CD}gQSSJLL_sGjA{t#KUj%u#6E$!)DqmTL{3`(3-h@#1V}ne+}uGUkh$dzLYjxW zltv~OusXNjlw1b4h)ameC9K{On-cE3JG&lXI+ zv20aJRNAyq&=D6M<#`f3WMW&i1MeU zESWfrB<1kAFJNA^BEvCDriK5z+Ex>kS@xfKm+(cUqy#_tOwiu*T3OKk;ac{^_L22`~m ztzY>zyR6-WX&R->t<_YNfKIw35D5bvhtaNja&FaSs9N}Nxbko_(xq^KwWu6CTSda>Lk_n3wypaMq+UFj-B}YqCiW(3|EmFi)>IkThjVYE8!6@M> z0H%f16utmM)P+oB2BBZFJ*q&7-cO&`koWoFwKfz!8gJb7+mt#6vtTXz`5*J`hoDrR zKzP*nK>=E$XrCyxx_mExKR1Mdpzjbwbr?xK2gC1p=Y@nc1jKYI0`-y#fD;$kiT1b4 z5v+pDUjG0{vxMSLt>X#>K@905`NP$%NC_m3%$>RNH}=GFVggK(0OxOE2HwNcFhKxO z>CV@-+L{I+^*X)bFR^P?vDQenT(uAS)GaWjgO&= z`JK7j4Kza<9gXh|Rfym3<<6Ig$ElNPj(%SdI&}2{*(n20rT%cjEIp!4cy4}2Zhg?u zL6SKb`|{p4`rwhyCh# zaX&s==PnGH4 zT{(_Un(OdyzlVJq1g)c~9CwI)MKO@bemgkr8-Z&-H-`fk7@5My zmj}h)cI|6qQfw;haiH) z5Oy>$aAbb{f4^Rb1Q2waZzSLQdwuVMBtgGd=cB*F;{-{AyaT@2A?Lrik@CD49*5zA z7tNX{r(t;_#PuV55JD(Nh;A<)WPv{7(aibA0jN6Lz|);WntY5@#L98&62OmpM;aZR zc%JA`WJuWINFw4XK~Q;oKK?&r4_l>Awz@q>${c81Xj;Ah07_}fBp~_y@{U4e+jD_r z+O6~co^*+P{{T*0;z9|S`E&OEKfVQmO#w99nT3f21;GXg27X)ldGh$-;Xy!#1?~;s zajvv66v;wsIUtKRub~=;1L+I~M2Q`IzkTh_Eps>$qU2bOjr{$hkReneqeBNx#iaaB z*1iOWNO#ua^)KJ!*L4N|09%f}d|~NO5D1Sy?dNIu;bCC7Ac7@O2Z)2H%zxVgJ{OOC z5TLN3BBk4EH`WnBBlN0fLFM)j>~SQrqMv9n9EjBK7|F8JFNcAATtI>xQJZZDHryC^ zbd52|FhSVH$E@|lkj^q1^bA4Ycf36-K>||;P)WJxFn98U(+yR?8=Npe0I$lYDWLo6;N7nxUXAR;!JUspzc-9sr zC3A9YK!f+!2+mk*=U5}I{{RGB297)J{{V&)S4@|3&J$f%7uKBtO)WWNJ?#rhu}BKd z-=SBFbu?l~IOjf<^OfY?jK``}z??>6Q4PFhBMOCJyEBTJ+MGhtfT>fk@};VBFy(1UNI%|~yuNTrA?D#| zT}=Q!Gz=Qe%UJZhnJJZrEJmh<#XuLPr{H%Gbg3uxcoss5Aj)K&yhg{wm@~;%Gm1)5 zN{f3uxqgqK=e$Bf$MGaJ3y+4I{9%i6Nm7tjAw@wX8Bs`76S73i5P4}cY(oD4w0bi$ zzain{*GP~f*aEHYwaYig7^jo00^vykN8q z_&)F;5P(Fq+_1UTp8K;KJHF?qeI;cnR-rS8p&DpyB z3mA@O7!ykL9yW45F4w$W*s4?+RFs`XJA#m6WL^j{b04a}sA1eI-1x;sOO|dpU%;?? z9eO>ww6v@$6cki+!UfPb(|8dBQ8IVaGHx)x?>y}lmW~jn0mCsDskmp+!LxOWE(Ii# z3ScWxN!rOt7S#X|I*gJ{x4@Dp8IR82*I~RpNGTc-)Dk|erqmXohONS&06;QAldQ$n zAO%PQN~Cm_*D793+ANzh!~{794Bq3ry(an<^t0&v$xP$hKS%ww$YceMB^<{y$g$WgvS3Y0jJ(En9&%aB zAgnwm@=)V(3E%qHutxY(^I#A2iNuh1fTmJxV(tt>LE;Wz8l3dCU zM;u9K0YDIw!rVIRsp|3gKEkC^r>!e=u3D0riCRdTC=z&#_MArm zelDn;9Bsha`a9(@QB9gZj7F*Po*hxu$$zP0Y zomk&TxmIN^QUS`d+zD9&{{SzJ!eb>AD1bs+8oViDTowh%5%$~r6FXjvqtnN2J0~)F zmY&plILt9vB=I^^n09+O$mgLF(od{Z)#cdibkKyS+LbP3z?7j2CSPakT3Dx1GvP!` zlc)rMK_id`By{JGlepA-WELDe_Sf&T+CNjW2;ePRj+83+{rX2pIm7#duzMFK2TMwyHt+rJ;xwkjAB-GlI5A*GMTETY4k0% zBo(%jQ>(=UpAkueGbHLdh>N;Z#mZ4sveXs1ZI4HzY~3g))*$dP(*sE@Pb93 zPK@=jj6Q3SWcl>JQ;Ne&RM+>&4RJ?MZiMXCk2m;_vYyyG6UNvuZGkW+9iJ}kZbzTV5;G}sqb|fy!9ngG`8(FWDPs$f|QVw_S)l?^O6>F;1d~z*{{T3%lloG{VHNov(BYScIvi3= zsjpK0ya6Dc8;yrZ?0z2nK;-%OX-26Dgp{oz(R1NiK~DPv5>2%e69b!OeUHZA2q6Ge z3meq!Q&7f&I<{TfyuUC(RJn-?+lek%=pMT1-50FeKg)B>s=c*TvZmC{Luwu8i$m#B z+ZqHE=?n~+RtgCL}1=#Qf=(H!@(Sxo7m zs}qOAt1re9UFTqj43&{~oh*s731B8TENLGG{{Rx5g->CVUOOxQby)>SS(v)h+jg zkWYU$B#d-mt^`yYmU{|w=~5b|)Dh>w?UDl;`&$4@5?C|Qn6tA+9hhac4%Fq@%$2dN z`X|T&0#IBlXwQT%LSHF~C%79y!$DDbjt>n91QMRLBo1b}1_mcc;Q&?sB#X7J%FvXY zJeG50=I?-p^7=Pvr<1h`QX5LNkaZ%RY`nBZ$x)xywH`I;hqP+T@>y9P5@OSg&H_^~ zhjDSo=y{{iCjT43o=wdH4j}ubKTO1Oy^mR3mSr)ZpzXG ziB^)JkdU3pBxxXCK))@s2Zxkj3$(3)Ft$y=H2Aky7H4;D)i*pX|@ z3leq~wkAxz8l;MVdnaR@?nDR4|3q{57uHc%10 zjfNXCVp2@ccd6FYs9-?^a}Ng(kb@I>%{NeL{#Q`L?o zt?shgg0{!z8l_`EBU;k7{{Sf3$`bRm@4A;8Xt>*Gex*-SsD|37l9YV|O3w?shaufU zjBo9bFIoT#U8D1XmLRN>QbLGN3-JyHFZoOeV3G?3 z0JIX$DikZPk~NDfZnsmMpzUdT-SoRZ>=vw#Vv zmcKejR&dj#rKFjGpr+(W5J@D19uXJj41iB z^&*D$eQz7XCnTjJBwKI+hy=+ZZ>ZkWZh&J~#Dan^KrHX8U#RFPYgpSQdxOq`Gd6%g z8c!tNCJ6mku#g~_z2Y&Cvi|_gsHdWr7Kc&lB-@ns2#Np zh#@8{B4?Hv5_kOm+W!DpyHdumqA#$4CeRJ0^E{901m9s^9j}XQ+EGuXgX$^^n9OLB z0z^pEVhy$yGOLLeA_$&)VikMG&XE+#kau(9(Q{i~1E5G3kTuw(?qKb3k;o2Q+ZFi3 z*zEQ+eJ_*8lZbU!^CmFm9-WPpExsB{aDNxoiMshfV7Y|K{{;zGLT^fgxP!y6A-KZ%^1Nw9l zNh*seK#N{vY%#s%b4hR*kB+-;{oV1;v)NBj9nP-rT{b4IT3*q=yCKib7I8-AvYjp? zVtgu;#koXS-Yk0Ks&;b_WlaUf!(u?cb$XQ3c;OwkzkLdt6@Mhi8=;xN2U2MQ*q5P>aJ z3k4+)#awYppH5iGq%a1rU!L93s=(B^F}!*0<%hulCL(#5@*AIrED(9~^6L>?b5(_f ziS>y151GCQiAs74T>ZbR*N_meE>4llov-M6UlrpnqUVt5Uj3pvj9LjNZg(F$V1~dH zVrGANKEE5`6dWGyA6>rs0|-lR-F~nPD6i`QEF9?yxIi#LQGs!=zTWfpFc3eK4)43g z*>aOD*pgJvOAA;qVX26UBnx$qt{URs!q?Mgf2;DmNbPuvwN1V!uIBxB7;0+oU7owV zLhdW$Pxy~^me6#D57Yz#4)SBu;&8vxF zb;4f1G33ScOH8*34Cb|abJ$nL1htfdFhA|{vE}4r+D-_3L0PenOa1Tua6)PW{5-dP z^nlm`dVMcAASHq134kccpNFOhgqJVmKo|PH?f1XF2uLJp1}^8uN?ToNU3oxBB+t@dSljF6zn%|=gol<;QT%>0a4{`q{)kUeDBMi2#yA`5@hWrHZVC4 zP|_CYP^w)-do;e@{6CK|H~kzpf;L2sgOBfsLZ)p2ztZAc7%Wxkthz?azK#A?Q|` zH-kC0x6%v@N0FQnEFB7vIrZj5z@H;)df^D>#*NLMhm~sxHrt!@cv57aC==G(`W{B} zg%-1NjCLdU%l`l?bM{4TX9PHUgQzg%8}q*1JpO|OLkCHMJ_pkTDoYT2dO^Pa;{;5a zfzwzrOo7(q`=7s;To7{e3h$(_>Q)bV39!e`^G&%L7gN$oDlZ2)VEW16uAM9h(8JbaG-?` znDg_2!vIAc&2LYv9V*!M7xd@n%MC}d@9N#{dqb1#yV33Dt#Npeg%NSr$mYdu@Xp1sA;?R{*YQIfR$;J5hI_dl0ZJh+tU&fkT{sT)|-lS zvo%BWiW86|%qePUMQ!Ym=HbKg1Vhq5vDnE6>m#kbdPq1>LW5|0*QLW(Yl0Z>QTqAa z>0@D1C07J}`5u^3Jh*=cEeifvw~GgZ1x>9nK33o7C(`&xSXl>{``_v?K@Jc(4=ZhX zx3509cM`YxM1HDY{+fIQ5TOEmMs$J&u1xrL-oxXz1T9PhS8DGOj3lP>JmoM0GW=U8{w{KDn^tgiE4NKAjV5c37D1yf>a3rs0LyK)Q787j*&qw0&ba9 z4aVSt52*d|C#^>R01@|yh8AsosdfJVi%@8N-#WumshQ<}Y)23!cXP~J!|f6qaDk>x z$A`!XC(mz&9FKT+{GskC&fhro_x&_DNi?}XA935QkgUR@gNA4wRD;}5w5-E0(nz>2 z)lng2Zl;1v6r6dF@dKNxW-}qk&{7kJm5avWfFSWJ5TS>{yHpg*Pa16LQ;HB{>9^o7 zvPhAWcC)rK60oep6@bX_3sO}{SSf17<3bc@O6O0Ig|RNIIZ5`Vmh4X0v!O|sRHsq> z3%D_OB#v<+KsSpR$Bg@9is6$vkHl5WH9l4NnELbR&oA})gn&V)Va?bQH?gMYNb&o% zTJ+u_<$~6o0s}I*i5uY z!%1Y9qXyQ5_ga+J(aoLqrPm5)0(>sy2taAK7QgvpjZMjnRcjs8_ z-mD2+3_uiiHn+UY?_HiRNLgA)P}E0=f4=I3=Bw`|ixa0`{1VQ`06x?4{8-Oe# zX+(Q9-{S4#7Ue${N`` z6U-1p1Koe?MvLV=VLjPW%&1C2017a)l}S28K)hTLu^3cL;=KrK)AZByj9mEujSJ8k z+e(d(lS3CoKl)Ro6de*S2!yDDLD+=AD3F;^oq)uVM4=!8QUf1owHf)=)p7kF-=qYU z0;me{1r-IT_iITb`eI-JNel3(k}yMkP!uUY5z5!+y9-u`T@EJ7JEc-)1o&d`NKiMM z128TQBK4x}@p_(uCzmNG1gP;3uEn2VMQK-yRZ{f@5|pb_RFFy2B+5dgPS+r6A}n_z z1SzO>ww;!-iv?y8T?iHZR`}RFXeK1UAaA>$aoMD!`Y75$^QVv zIc6SAu%BvPSd47EB*lc3NGU{@3kA8W@l!|nt_sTY##*Oz<@xSWnJMJ5+E+@&Pji-3 zIJdjSytJx}=v!eZNl|ZBrKM_BUr%`#{vq^F+UybmGq8DV%+&>nDxH)qSi!==q<&Rp zfhUx7Z|vLc@_inDFMSZ>@i?h+aJ{DOE+exu#I+zMM2^;E7>tVonWv1Dk&VY?fs_+a zh7~c6_HLf?ikR*H02)k1?-znZ5+oC7F}^+|W>WYu6Ow2OfK4t*9?ERxi-0aGX#KHa z&+G!&)COv}C4seFzyxkC;2w#jVSU;(g(L({p$EhvBc8&*9e^Gw6O1~Q#LmHk0ieG8 zm~E?q@d~Dpwjb1!p%$fV)njNnsus=2DJdf4*bo!}7B?hJ3wjBbzxDXs9rYFG3BV>g ziDGUUtXNj{;eFWjf+I6!#FRV)_hB-jdyr&E2Ed;P9wR0q!ow{801zdt0khiRYi%2p zRbCW@k5|`V!*gr3KS;y)i-q!R;kSd-j68=BLbV~Yr%>2|GduI%4fnAoWXUIqGgOjv zsIdfWZrWT{m4yxm1m_{&R=r)`fw~amNaOy}e+v8q#f9b7Wc27+QHR=WA;6_wJ|se< z5=6~_u@)HJGOpF;xF2FpElf65KFvd=_PN)ye#&IIx<}^n=?f?R6p|U~%aH=k`iSSz z_%%6W$~-m3`R+-jN>dDL{N*YYFiL6aHMRmtFf{Uk9Jcv>k#VYXiNj%Mo!2N3)U89l zm+_4%XQHg4N`cGqS%y*_%*-w+Uag=10MVUtwI9IE;Tc9>*bizs8j7iVNy@8ILuhq5 zwo>|5ttm>Ig;P~9&{2>QLw*?|Q?;)U7USmm@ zsSec97HKN#*hY%1*_wI>AMelPQ(5`q&01+qJDsP{hiotdJ(tJf)bBLI)nVUiD=n9YT2Y8nms+rX zq%bTk%7<9^#Vb6Yh+ZSqDM=hmNdyicTZEDr7jnYABbYln%fADirG64P(}^p1ql~*O z?RS9}+7?}y)ge_pMU!T{S;Fq;G#PeeidyQI6PF`8rcD-d{##0ldRiy*HyNZ$EHc_f ziFrC_WoH*HNkEgu`JzZjP)ij#CqP1r*bqyD4gUbdc$^kMv8%PBML$17_v%o{mc*)4u$3x1KcGSkhW zx5W}^%~AqD{L!c|B;&ySwe4Jt@3mO@>X5v48w|6SU`U!Jr_^AF1jVIJ-l~2h_NiDr zDjw2Gh5qdN4fI`)><^;MqXjl>oECAK%KrV$O7O{4(Tm4>QsqubJV~7+3amC2Xiy7G z%p}FA0=UH7?5xzXh38yDnN_y08rY-msVUUri3(C3Xc~Y>1V?;ykVfK89Ov@_+YN)k z;m@_~%qy8nvG)CLo&*a#ZtLunxst)eW;n_6$t~)NF!D%u4eGdN7HW0xh{@po707{A zCS1y%(6oxGqPnWGH6&g4wXE-}N{Is26Ne`qkH?TF&syY`JOM^g(yc9xfj%~B(Oy}W zWq|%$2O%V|{K4UyHAMda`^wC-`&fTt{C9mDoWObm1<9iL?>WjCh_98wlz%0I#Q1U zOLw?D>!@(zmO8X_uFv*uKn2W}3l`=9r4OG7^bhc#!zPo9^Q_eYC=?tvnl!JwO2cpF z8Mam3EJz`>B{Z(9Qj=t%P^2g&So9y^H44V|_c+DQN{VoOmchFvo2G7uG zs!XXRbOGJnuH%T3TmnF1PMkG0kIw%9XQWKU<7DB}G%U+ks^JAH0%%ActwA89DK+9v z6~s#DDINiKubrbg?SnL1No_RQa~X9VJ52uo(`Y|PjNYQ1A`%C4nP!#j!4s(pnIcbt z{VL-QGqkzlfp6PPz?DC;_$($8SgY!!C;_-Sq_^oC@RN~@lhH?`=Pek#%E z-v@Sm=+X3}vI==pIynPU;(&J+LZ{sM9Xd6L;{cjxu4XnM`7VIVnHHjuAzuqs-@^j0j|eq zsrGyM0UNxKcr0#o+|n~7j{&}rWVjGiOb{SNgm2DFvL#Qt3plMwxh`}*bvs_N^-6}~2?P)c z5eK*}(M2*hlACY57?X;Q z4Tv!Ui8rHpF?I=1Za?jSM1WurU=U!zHWMUH+ZLsQK%ls8`!qtfIw+B-2A2+Z4^gJJ z1}wRino`hH;5Q(NQgo72a3n|rp@VEqE~=*AXmUN#=UCL)@JtyA)v5tJ!gX3nBXUTz zNSFW&W$Py($G_3`?Z*BTh$v8-G`gJD9#u=~9UbP?!K&D3hca<_yJw zoAM)d%T^G#1PTF4nzn+VldV|SrkR_{<24UQuyIM-_M?}wd3rwMD_>Uz7j{NvqE!Z#Bs@Al@4 zwn0c9BXn&9d059Q^#>k??40jrPqZii(v;Jl;hv3GkDor4ut6B9>)}@uGc|dT8Os-` zZDZ2@+3KgW?h$+&H^j6={{Zz<)?D-^2a%W;jBn_gQHYXX?9cTS+`h5qJWPQ#Q{|T+ z5zO3gu9W#!Jg)pu<*CX$Va{D?vJu1-84h_M5Q1|&s)4$P1Pvez!49crU?~n0vW`8M z=)V@McVln?L=VW$oq_@C4iK(y3D}gC00n9TR0#N=__)dukos`XJw&O4`|}Hx<`{+W zo)jY=gUhnhFqRAvjzVXss)$&c&>GKf6DL5F9l= zKRA53UbK=)2n@~qMe4_3DEpclz}BLrD)J`XAWVHv#|YqDf!`ORGLX{^nx%>Y7=hqHCz->i=znZ)>r(j*kW~l&NH!Jfd8kvxyIS2sH zFlN)zJ-Y20$a7I~Ug1wW>qxhTll)`_6Yc?46?08bh^w)15SW!*+hOHD&pw!JT8NG- zQ8T>W42^&h^PCVwRT4?{`e20T@$2MxK-5GV`F;FwLl#AWFHVhLy?dfMj021Cdmw+hRCx4hc=cgmreXWJ=w|_>bn?hdp zzal!@sCwSifRn%6PhT(C94rQ#I3GQE4f=9`e7Sx7Fhmf=5o5{^t>8%XJMD%kL|gi8 z=adj4eCLq<(}Z!r&L3@W1W-{kas+83Ze-2w3>f9~#33O{FSr#fEGkT7WCQAxaYY8z z_=$f5<%A8+UjG2L8#7M-0Hi)*@V*r+;orM> zB7h=qGZ&7N@66(07%+kwrA3DS08A_p6KLr-JwE z9M2+o^7Qs2n8Zr|0G5{4zy>GBUSjovB}h9EaVvvj3@9vL&!3zTAa(0EwWImpq>K?N zX+u+2JrC;L&7@O&{l9!iFJAAgHeS8oSSBQiA(3+=5dlzo&i;Y~3*uBTO?0`nclky# zbiY)scv;4);#Y|9Ay;}tq;ZPWD- zM3_ED@2#*OZrX3p`a=8K^d{xa0r>LnAjE_iKEL1hk7>X?q_H*A&q`~&NRr~W9z9^_ zCO{m>9+GeIwj06-AkeuCX{DO*=#oNQ2~_EcgX%6X?TLbe7XgjJw-x0POq7|RIU|MG zj&~mc3?5_;JI9dz{QRU0G$(=FhNX*|UV-}z7uNRi5cHTZZ9RSe0JoO- zcxxSe-|1ti7l)U))6KrO8Utu3GuPkm`}G(YXnteor!q$|1D!|6__n-&Jd8ami*oDE z{Q6-bVZ}gB-2Kc5{ynfk1qw=ukrB*qGrWHHHa=KuDK<7XJh}DgKBPFHD@h7AxaVW# z(Eiv&%~D=~zg;=MIF4kb%?WZt8WGO#PC)#%gaqmYkS{x=*uXsRVTU;9;C|63o#4PX z8v}TrqIZu^+Yq|9J|-jeQooTt5GLVEN))0Gh5+1!p9$E-~8*y6}n4Vt0D>@G(j z;m(m#M4woqg_WT}GSrlS$|;;rt562KD`Ub~h5rBvrGAC~00;auI76^r31v+dYI8ig ztB9)-Q>8`h?xo7cih%(@ia*PtGY3cj*hV}j>0>(n^XTqYhk&skzJwZ3U;COp+q)aT2hfD0;DQHHe7;*Em}!J3dZ{nG3T?2)O*9y zfK-}nSa{jmK8JQg0uv%tDk*MEM*jBo&<(WZHDXs#vZDn{l`2X7Wg^1xonVBW#7vvx zM8tq<;>~9jv$MTYHor&clcj#h5(`q7=14lywJqY&ls8bA(n%zOon)Ors4HZHBlId{ zs3gsZAe?HzAXGRvCd0n1KwjeDmnJc5mXfbxgFtEUEdAXej#ULtq2?e3exj&Z+(}WA z1?48*AutkL153L2j@8;b+J?LzUA!-WTXHoa-qj$PWVyg(I7N{|61I;|vRhGG>#P+7Wk zBt0!tXmFGQ-Af7#TA;T+M6f)Wny^OdluVG4Ab=#M#1aT35+>e)OkWIr+M@2)dS9zC zBv*HZ^=y{EEl52)v;p^ymFN=nSQaD`T9R}+X!I>p z63dj{_$7wK{W{n6d=H2cqRN%5!jS|RAO`}aNt=s+qa=aQ37MpoC1Kba3xWfX0MV#V z&{5+(fMoj%AcqCr+kcTgP3auzn$|9ZGI<}FYLT*~CC3XYan-gT54OW(lPG96ShXYt z6}s~)Qky~)r2R_#4ED40-Iwh*V(}A<$@xZUCpOCw#3_Z$)A?LL2`r#6_{=66Y{iC2 z{NkO=)W7E&`*6JujIYu!q90BeIRgiU?I&mRKE`&d7baBIgk~L+%`iFMLpoy7`#IBb z*;ZxR3=x#1s%|1ySs`e7cv*y6yGY6h6LxvTc`#q)*P3P3^ld3rjRcieH~E@w_#^&6 zu%_Hed0I*ogekDn5(9qc$6%!9IaXzt&q-b%AB4u?7L&!r>hU-=%3A`uy0MF4sY;#5 zQICrqhwTP8vc0+Nr)n@ygv|1M<08$lN|7(vMq?`*mFAc^lgfCEv4{htuIWALFQWER*)XqHEP;*0%GKH-LNjg=b#VINwL|RCUdF&72+FnL5UDZ*n_c)Nye10 zqAmy;P&G~Wwft)tdGl0>jvat+Xd1JJ9e#$CfaVkhAx=mEU@vP#l)>AX5<##MKo`T3 z4+?Ls8qQC|uGHgK{ma`0DS@@rTsWf6S$X!4gCWIzgSRF9ZwCyzzMxc+yj9jWq@NwSGpoh>W1k zYSdf`=;KYBlw;M65*^5f?bP|)k=$&PsK zl9Bf~PdzQ9hnb+NsHad0)ZI%6Tdb`s1tls&fhq>jM2LyT=h*DZW?P=+c=<%6;juVu zU{Rss!Q%Lm#?Dhv4Oo>&KlF!`$+FYV^q)x-;U7pQwMH2vZ3~m<}gel=J3Io%=Q8Ms4Z(b8v3q@#i>?Oi2Wek&f?balev-C8OeBo z!%7wxxNCeLO*y=3^YGIZV_%8CR=*<_#AL3zbuF?*if;lz+V&t3VSUcgdH(=BC8De^ zR{()WV$WTeUXK;VSusj1M{v}>pg)5EDPUtH%Q#8ug%9|@s@@Z{#0dZxweJvT80M}StO3+9Kx8oOKv5z-7QJ*0Da~7Hp-R#)!RJ&Vio=`9hBu5S*l4&P>j0) zjK*RTl@Y_oIa!7kC`hWx3P2@`IJwA4JUC1guFmI*pE+qVM4#CvehKO@6=nhT+l<9Z)+ye&A2n&J?` zm);#~gqGG(-7?r|UG9Y-f_-(IV-teGVzD!o{vK3<93&Dr zNsp4gjq+K!)_;`ccsVkEH<)LLmpM|D3eF~cxogA9;Ylh~>9XcYT&W-hEfXcF0ZA!; z25ECMpYeWgooG=10C>)8^Cr?v=tX5)uc4#JC8RhTN0AVuBqX&RO6Ho>V34_7Q4M627AZ`*_ zgmyOpB&7`i2@sHUji-^g5f}0XNI2pd+KFAXt>_E6@gEad_77ze5h_bp=6Uzi!<2QW z2qZ7KuvMg!8-)mvAl}AE1bN>a-!UOHp(96sYW3Q**1V&-WI%%_k*nBU>H~a#PK(JA zr70SKAd-3#0SA8}2c^N-<4v7K7ulG***vsseB*nPM-9i1GZ{v`DLY6_)3{OSOvmXW zBbX%a4k+P)D1HvBZ)Y!6#0SwXa6i-sT~#&;$isgxx~KLA(*yNg!@U!*VSl zHnH>PZA=L&YLGW!;7*S8CBU}Pe#*&pN!URXGdBPm^8!MEJmz}hti^$$r+4%p9#@M6 zAXI=~Rt;x&>$s?miq%08-YzHt1^^kdCmsKn6ePO=W^N;$!Pn=`flEOl@0GYf0F=qPNRuEXBH&CL zSYH`zEz{J0@!#tZ&oyfn4mTHmS074HY0Th|;Q*LWB_xyKFcM@9f`E;TM0w+E!7JJ| zabP;s*1NrW3Pz=wrA1?<4=VL>Ms04G4hn=|Xp?Ne&UusyI;cioln)5wws5oP4Y-YR;S z(t6%(2giPxed$L=6{9~}KYV#D5|XTJ=_NH)9gSUqYHC=xChbNs#(#{wg|~9P$GGu? zK+JMn--Yuvgvnh_Oz-=p2C^&(qRpxj-1v#7N=NBVJvZn>G*a+c&Q`Wjm5Gy%mL9+2 zrUXn1tCq?68>3 zz#%97?jH`23W{TdRzqOHy0(!!8~lztw52~2u(vJuG4sZ91h(NNTu#m`#8RA&ur(oY zbtxwE5=of!w2wiyAL~=Hzv(cC4K&?IJ?H+=1NcQ1Xhwval^H(5e{XL*FqbbEC8SD6 z0!g_gR&u~UD_cM+yaW7K>B*LStZe&O8<*=L(Kvb#K2J?T9 zFSCWV&+#3{UXIn|$$i`4d&Zqv{9Zba;7q~TA8uUlf?u+bNeQV0D7O3cx9JRGQU?hn z17J%zr$=M-)*CR849=4RI+8tYB>ITM740mj`>Yx5VAZwD{37&#M5F=;KGh8mykIII zdi0(D0JrWj5iVXOC4-;q=J6G3uxzfx>#6a~K zlhlp;iT1xQ*x?qgGESd5=sJ7_;zzcU{AAc`AI$t2n$2H#@wu{fD20=l-kdH8N%@;_fa)umz3 zA1wa>TEJ99lQ!$^f(RPPiw@ zKS=xLa6t|`A9?c@v~%+x=Lo8U-SdT|ejD`L<=04iW+q@8e*0mFVD%a|JNe}a1_t1Z z`E}(V&y|7f3)4!ZKAEbTr zI8=M1&ZZoXba~XmsKE6xJvtw(j&{SvmDB1X97rZu01|k`gXpcIfd~2YkLK8ujBI`% zc*n!%Pyh>ya`pYLZS%w6f{7cMKM!mWK~k&VpdUK{%o?WgJeIc>ZdCgaK^?kAqzE$ho{Bkgu^r_adN0DJlG@NXw*DH?{M@aO)2X!O7y z$436ItMnCkt%#Gw?W9zDz+8>`j}kgVQmcp}Jk7th6^OWKpcIt{NeUKlK@YVeLI~<_ zv}}E61UMET?FI&JFhsz(1|F3}&e3mQED&=7{XRWA`b4&rWTrrnVtU#KruM-L60kuO zlx_lu-1$KDjpv+gHo}!E_3=06Dm({q7OVMG{{T9vyC0-b*iTG7L{ot7{5{)6B9#Vo zkoIsxwb|~_WSIl5+uZ%%4L-`LrJv7q6-fklbI|u*41;nf%ztm~=Z7d$j>d%fXC@~W z8bpMu00h956K^30)@|f44{0r^^ZI;Xhb)q!KqlfOOhQ1nEy?!-6Qrd{0MhhTE`0eh zkDiqot7NEy!(d3FMy=K$j0BE;A8((@;C=7s&nniB3OT`mdV(jPPmj3kgmKUQlk^`AARlYr&w17qElJq@ zb{xGhtPX(pZT5wQ)JHis^9OVB79Ux{w};oQf21S$^V_9AtUNrV+eo)r^FO{JQ*jy{ zHRaIxe&vgXQB17MILfm02zIkG%kuQAuXwP8j)W;n(4G85$2fxHdn*;}-tYq>`zx&@i$g+gEpN`M6Oqo*vB`cPMsH7x^1uIZow1DbT&j_m# z^v}bz?BpCDmSyzm38u^Pni?8#kpMSEM2ci2>_(pM2;`{|*XG7~jhT#{j?4-6vG|-` zEoV+8sF7Q75ba8{i%00^v6(cCuLqQ5@@4wGK3Rjo;E^iCOTplR6G8z1l$4DN0YVu0 z_xOq8FBMy8aDWQR$vmUQ%n|{D4$w&Fk39Cj5}=vIzy{>d+y@5Naz1hHUqv}uIHW8f zsI!_9XaTkAe&M6bt~Sh*-ndkvqNS)33I3v01_y;;090=wF?5V@EZS;7b%zf^3Fuz$ z^xi$Y*_=APSFuGF4X%c-MiuJ%)Qw9)oFx^iZDvy`(o9N*q5%>BJ8Ccl6DBvtt1?(p zXdcFua?Mx039E){J3je7OUphj(|U8|++$_4QWs7Zg#r*YD5$4akhK`uL@G#x#qJ64 zoMF_L0^xz>THI?(a@YeH70YF?6$G&y>Jr?Yymm0dTx967Ld7X_dheRADy&?5PT}rtoY` zoIQ%_D^t~p?{QGY^*0bD23q(l!dQaV)1x(>zyRkhri+)5L zAuQA*e&+&#jTyt2=NHg~l<`xGQF2?CA}By0R)IDgZ(B(_E1O; z-Dtt9V(oJF)|3>)&%=&73-UH4uhbj03%%lZ=Ad{;Fm5!0Py@1~JC6#0PLfQ>F&J0a z@+{PHYUSRh-AfA!o~#eyZxQEMavl9CC00$~DqvWj4>J*YnY@8|;5^1G&w5Z^n;vGa z&ea8dSpHM-g>lq{NitLvnOK5JK}kx8u>ctWiGT>aVea!?oLF6`ni}fmtN`&a^h;5{ zi5lnyzCY6z&K0SrQV>8;SRjHzB_1S%s#RqqX^AAML;x`&N46+v;J-oz`7=?;c4|fG z1tCO>06`mqLo;qF+LLCEWaCAe8naa@N|W9ctt(q<))Wd_NlFx?spk@!e*grfY9`vX zdZ+NE+VRFjm{;hfhbZ#ji+Tryv2u7 zl_VOByF%TgQK(y=mSw~(Cav((NLW*4BqRi>l`;uWfhX+Zcv6ID%9a+4kBRd}8zo`G`AWx;oMquf3Roqo3icbAPNg? z6JVuuoOyYCE&l)(^`WbMN$zfb;>E38ZdePwE_89G<9@MusHRJ(T2z34qAw&AgC+q6 z_aJnVZ5U^*P+1`)@?u$w4ulJyLhlr1&RP^!_R-k)R2p-)jA2=ZQe>D=L74<7kfY&| z7690Veyvm27E?3WVAar@JCA`BtpEksm&(eWcOW;wuzC&dEMC`W!)o%?B}xcEnSxX| z5>JIHShap#O-hPMyb^X2 zNH7SHX2u1NGqxp`qNlUzOMnf4C770``?xO);-O#*x#?TE8+@+dg z0Du4h-6ws#jOkH9HDIC{yTS3HH5)j*H(wfg+s=`V)KLV!Mm2sp5n@SuzEumJ2_{aU~<2VoV~{6>Ph9#uBEAoSeEknJ-rOW781h?N=|zEGI% zoJmrI?5U7>$W4vT%aYCYc$p<+7I&yTG^^+;PQ445z-O|G@H3dZ;R;v)SxsQtfO%6& z#*X}Rej7g<`QCll-x&TGdpN1>2Z6Y!@cG%6SH$h*b^JX^m}Q((kkxS=a)o1g_GOhg zDJpW_BFoiE^&^!jb;MOA!_>isIs(nb(^Z&FmdcTM2UPU_2LEmA4FZQmxf2R1_3H;xGio&gYz%ji)Um5Ys@b7T7b~ zYT%aC653id{7wz9@~B)~HAivK5??_^)W)IAczH`&rM=7~1i>arCN~mdc7qZIHc2?n z1TZ4|nvJM*{)9PwD;boDh1rGb4()bhUh7|5A>m~>1T`uoizQG=APKbifY=ze@wDeM zQeLBTTKBtr2S#|u42^499;_4q2UWPK*KFyV0P|*9V?`r7JT6{$|E>pqiY^EC0M$(WcVoc46By-gu88*fX z6Pbk*`$ZB?#hf(@KFaZM%^Q9LA`&evA?siZXxxQ0`bK?0kfE>$K(L4sVD2CQ2mq!n zJCoF$M9f)J`#{z17;WLTTSG|BOTi*oqE9*+z0HFbrLA|1`&n>kO2kFS^rVO-MogO$ z0fEYPnT-<)*QF~$>>i`yZD^Rw;7Px;cNVU@cbMxg0Lu_C-g;q;h#U^0~NwR%;})0XyAFbOQIY}r@@0$ELtID9J56LM9CpF0vjbL$Z_-3x7J@LSli z+FwgOnVpD}EmATwitQFKASGy@HzS!XQm63;3rwuT3nwp^F~Uj=q$Z1&ljTPP(w8cb zxCTy;Zhxo~u#qBSI&&EC?r%GR4B~;sc^VOI-&TF8`UmWW6_@K8yRinfr#(&E;T^5P z4J;**v;>INazFqN4#ba-JaP=)Vd8h7p(d=rmn}+~mu5D531P3lcUeL2T;$&vz2;>VqA4((@V z_;g-S&`J_jxon8qZYBtg^%B6p`twZcUwIXDD!@rh?c(IpCoKQ(WsN2lJ({p$RY&ma+ z@cRu-^u4d}yK2!WU45P{oDZ{_(lbn%vUJSGk~}g909+_V_5dsNro=$nl;7g+`b31N zdj!$WY>d*c&yswQF-Sx3KsOqJ9x3#rh7)As95RLJZEJlh3sab+dx6GGxtsP8Lr{QRRSt2LR4jl=2o1P} zE3JaAc^^}bwp`(5O!>I8d}`fe#fjlYek8pLiZu3?lkFsKLsScDBb>hww8K06GtgHN zsoopoY@zh4a?3fcU0;;BGx}1e*G3g2CPG5G4%z5e8S48;WVmc*4r)jxioYgPF(3|M z2HwzO){hZ>I`ULv{TENga1)5h^Q^LBCIpkfG4e6FI;IlLlp%QR8Wf_*3QAIH;yluE z!mTnk)v)UkG4R_Ss6rE3oi!Z!N5{NSCxL-J^?^Ujv?|t3wRJo4fj+x)+YC>-JN{-Z z^7H19H4-Dt-0#274uS?&Z?%s7Bd5+!FnVBu1hc1Y&fk4UPpG~C#C6b$_x}J|>_jG` z=jUGQ62<~xbDuz?`+Z{oAoCAf{*XI^!snt%uY@XqZ6--2TONdM<;>fj1^Z!+mTltX zxk;DA#FC;Mo=)F0;}Mjt$-mqHFvUBBfLej?@v#CWH2FkzstGf>G4QwT_k1^m6)>oK z=+|$gL|Ce(l(k1o>sxE2M<7haJ z{(QV-rAiWkV&t=)nhs9>L?JpsCu?nUKA(K>M6n5Kl5ZeL5H|yy4X2Y;W;fg){{Wi) z_T_>I5#D+O>whnA3XF|L|L%qS`2;sjAQ1W<83 z_HhhZ1|WmK?S?FZ%t61>Cs_rUgMXv}Mq~+*6BpV@_#Mw98{#%5!Qqg_8Bsq^OgUch z^P~o(o#BQB$ezBO{kq`;>HU8FHQETInE?5d^Tfa~17QZ=ubnj78l^>s{{T!Z5G)Wt zq!ZKKU2V5cf-iAw5JNCQ4N-t%50|aB!3|MhXby*=_}9&$CM517)DvO6-=Q%!j1bpi zrxf81FIqG6<{HphAf1l$%pddf!4g41kv~trmn;xLQP11=!2}RrnErh*KsD*GKRb22 zB2#iaAPtGMi{8V2clI0NGKyS$0MlSJf6@p#2!JF_?ayKs_QYxSdB5`3(8Qh>eRd%IV4jipz9fln zXl;|W$Ca(Vo4oxQp3&PWmO zgct7S-){nW+6S)8D{qUN4qzRwL>($4miv#sm^qIqC~`5!@lmrM{vEqfag1|n~9*QM~N96{3Q z9b3ACTlDH8o-LgX{{R;|@V1(9wV=NeBjA4FUy}?H2W5--T8EGaOjYSLPf_x zZyA3Ba(zMI5u1?g5G+#NyK zjb7TtOb;U)`OFdPc*k{{XA`_-yb_^ULS`u%wN?UqCD8SVlYi{-o2+ zkby+n@i*L%51*GWJR_gfl1SUa^bg!x6^G~m0Po{!mhTS^1kT59pAbDvZ|8+~N^6s3d0fx=upeuMMu z_MeBcr$u6lviF)>#I|MXWjLKh)Fwbu2lb$ls@h0ZkIihyF&HT))-t65?IasngM2x@ z^pDY}W-@7+z9l9>2%RRlWrqI%5F9A+mZ-UB-%1}U{{RqNOGj5)p|lxZ8_V8mYiDNGI2Vc^@0a1yd*ZZ`4~N2n5}=Z1Ksu1H0n$iN zIN}+(_QDh>O7=t)v0o7>UM90ZW9S!ABZtLR08xLQ>Nh5 znBANupD~+dH1#d53rozQV&ML3nJs`Nz^(N0 z)9re|D_&7bNGD9LZbisQ1nbyb6%abk#1#Ojh6J7Tt#%C0Ul=5TxO2{xHSnN&MY@`? zX;MYOBu|9J#F09`Nl@4c-BA|9<+yrXw7uvxt(+Ze5=@djZBG8@rH! z7QtH6rZs>HWTb!E2$e*Vl0b;O;w}K;z!ZOU_1f*-4-!^w!b7z#+uLuuJz(jnN|aIr z>K2sT2%ew~F%vQXNg#`15Iv9>yU;Oqa%}pzvERH@r6HcA5Zocl)YNTnFBZ(fVaApr zTFOqSQ5!8pTHYh9l$C8NfJP@lPrc;6K2WRt4k2Axg7nkQ*Ze^BYuu~aQaX38F3^4fFGlBL%nS{kI zQyLPIkO2lF@I%SW9iu@>SsI?BLrpgOp$J_IAxFR|m@8>4D3y20C=RT&O29|x6SKL* zy!*4+Mtz0i^?AlymgX|<6E0R~m&nFSmg89}#LiKm;R#VA*(81g{U~-n9q8-QKcpVO zX0j6^e`EVo+P=@Ba*_(>%x5 zeC(W*klgBexl0<*5Ok@z5%)V7`~j}w_rA8T(V6KPl2-{y79~a?DgY`msXKsBpb0y2 z1HRX8nz(!D+C-A3>J13F8d9IG;_-}T6loqJ1+^F=bqgecA_++|qzH{tK?f6| zU&NYd%6F)s*n$N+#AKyR;{*^zXvM70(u5Ls@r+)klr}YKI*zCsk^-0|v`)89IfqzPpzAn7sz zbHX&JohlX}B$9c=Vs0ZbTrR~@kqCjS(01-j% zzR)oG8+8aud*mr$ZJz?rl_Y`oBMX?|F!*dNug!o_hQiBO65OR|04*v@dovJy^pV7! zKxq7<=UGN&nP&OS`4}%+oL(;&t|_xwB`eGs1StDFD{o+e+5%&nKNUZU?#cfE6CIuQ zXr9YB--kHLpCSA~a2xrb`zmXhs>_CB!M&rgNNVZp`?rEBJGzvzq_d$;FyeLUPLs`j zHR7aVJsI|s)Tped@_a#LIQGhF>_Ot71Yeyazy1vJyk1xMlJtJ8WVA~Q#N)~e62f3P zdP}B&FWDhfe`N7mHO^s{2dI*;fTaX#n}8)DM3X1Pk^mhd+~?*>7E>~Y=X(wO2A4P1 zKSp!W$A=Uy1-EkB(!uT4@r~x0C|xru0I5Z!00>Yi0kB4$QlbeUDJEh>i$*QU#1zbX z^{YRfJPjikI}OJvnDEN82%JD5?@M# zz3T7HsaK3|_hJ@((#EtMY~Dx8T{jA(kfPbS5DHAdTpO6^0TD2pT;4gEnwo?;C8{me z>qfX1tHjcBiD6#qo1xgYk&{ty(oahd?$O1OBO|6Qidem+$yO4IHQ@-(e zIB94e?ylyIW7C~n51Ia<#H&{Opgd9AlL{YDi)E*5GLUA#&0ddrex;39vf;bnrb#8h0AFe zEYBJ0F!Sa!aw5czAO_B#XmY4=SnVv%mCU<^xPF7XFDos{^15|XxU>0Y>Qj@@(gLPw zLPFNJ-0z+5Tv19;^ZnkiW9@fgvXo?fi|nRTDITb0xh7MDP$8AYhQVP`iwXn=;>9Ln z>MmpO9TOSWpy$0q-StEvevFdjYtUDQNI@=R9DREILZ2qjb(@*T|?0^3NPwj_nXG@pI!IyS_E&(YY zmq5g;6`0C&ihAYc8GQD7esy)>Qc32X0?Wu)T2LloJmkjwqTtP@3F(h0XIO*7MtbVT z&RMq_+v6W$`ze4331L%F<6&NP-GLR3#NfI@6v2T2fFKl^8bVCX=51pJc?Bv^Gz3327+CkDzkO9WD{{0I7NaIUY5U66U#3_T}NNNqi5tTp6xc+ zexN~6fMi_wLC_2dx2PD$PE+)&Z%@0#MiXY(SX;2$3MDb&_Nth*5B!ChGixUHUU6R6JB(?R_gI@|UW@;Vk&q7K@gTqFy zwo&J3jl)EY*$Dzpv8HCnP$Eg>2+~Lw=X>LF$mWqcw0k?&hsQ>0ni|JF?JSh54*Vpu z0(2WaC@R;idS%&ue?^wi2ra!zg4#$^j=te$m6~?bAgOOuK9nIT7n$omgY1z> z5>zTpeVUU(OS8XqYIz9pAEen6Wqx@JN|_u+V~R@^s>^#6b|tIGN4nX+8|Aze*iQ~4 zh!oRr@osgMWVDp7D1|vtff&12y0c(_>cGw}Za6K>aTc7zKsXU_dMK7ZQ_1T-l6 z#Bb8^_tXxURy%cVI{E7e<9?5?pXmcp{*iKea{mB04L5%Za}~XH?%WmGylcqP)YHFV z5=ZJM-+S~Paqk#V=a?Itb>;J`LQqg#vj8ey%r!a=a7ju5Gd_{+&*z3X0)Q6KSH}K% z#p!a>ED)D2*CR^`05vsdFH&gH+;FIy_(8Dro1Z)>AY6ggB#5hBk3qa%dVh=#9h^Se z!V(3@i0UN%!=@uq0k}vPcJ8}GW~6X{=D{Mhu%*u-ZgAKlN$0hq#%JMgkESE@=i?US zNh$W$2NCx{6-cYnz?yRclyV}k&wg>V^T8IPE3RrV4Ti=DAoGilc^;S`h>BIp2~#CiSuVPeVK$ERN}p$!9{@6*b_ z3?;ss^Qkq1>1yAo01uo6OWbm>2k&{?`NG6%Rlv6Y06wr48%*>m-{0ao^}!5*9lZWO zcf51w?SZc^D#2ft~KqRmPo3IXBdP19|pCigSbmi%Ye~0|P zNXq{J5BYwP9GJD_%s`#Lf0tZC5Wv+S&c;MVf!-PFxN&EizbO-gj-_2kuB1xNd z_Q44ZkPIG^?f3W8f&yY~x1I%@MFDy=A|^g@LV<}f(1YuleBy}UQNyIJntA)T(!AZ&(hFAllq6h z(BUA0BlM4dslfzNqEl(y?-8~AJ#dP#>Ev~Ql5gkqK^|9Ohk`(cl@A<3EW%{ zKf7<&q)sM>B!X%{Yj-`g=3U+?Hc*>-?YAS-&k&}H4@=&fK%aPb{EQJFq1g)b+XN6)pBC~+fPDJjewXyaQSOfa0GWp* z*SLUoJM-EKJn#4W;n6L3(`|SCAgDjnZlm}HO#S-<%h32RfOj^G*VVp!x#)1AWROnc zB5psl&&v%C?({Vs4R+t)?(~LCC;)M!s zTo6G}PhW0hr``DvLL@;pdVEJuABRXNBwS#LB0^OVM^vlaOi#zJ_8^H(uULcnuQss) zFSK*G_lUs+76Gw~0tCsAVdv8K!i5ytuFgE`@QCJ16995S4#ab~BT+!&ko5quxE!FK zSC2#G>3{_WARmvPyhk=lCxCd;3o)ulDXj=98oM61qbe21`+E=IVL&QBGZX+SK798N zloS#+!Z9YLw|aT$4oM)F*6v%s^!q`92V;ffR{sD2_K9Koj6t{?3ER;7?dj7K1#Cya z!&Zc>GJ}9!oQl+FM&=#=04M$DatDuilh5{-Ma&K|ID4Dry^P%k=Zk9f+00IMN3Y9@Q$KEJrZgITU&TLnhbSBinDT z9_bgOzJ|N$;}jh3f2F)2qgOSfd-#*K8^n+0_z%2T^(+4X)tCPOS^ogGDLRiMW4B&$ z8^9mt*otXMZWd7>*t-DC&8T@7t)<{nH4s7p1evd(PmRCA9`-Ji@-m8%1j;Imw0q=cRX zjK43!;DVfk?99ihSeZ(yCMIGQl@O#D^po)e#!ltAHe$*hgc^5S@Tl1!C=i%3W=6yj zAng(I)3&*~u<4$Jt?O^3Oz zVU0LTvCtIsL0ohV+U;*NKPONA%fN`Yl00+oNtMF=3Xic_p+ z@+FdwQ7{zZgO*q+RY1`|C{@9ySk;_km!~qzo@rpHyoW%8`a-}=q!0}1HUJrsd}`Qf zhD7qMj-U&>)Ey0lY#F0-!>#`SW<@JU-n9pw&8Iily@T;%%v9$bBbY5osL87wWg5!0 z8aMU2!$f2A zsI$c&mMqLp?*9OIu{~6zkV2(G^LV&qQ(iBR9ib)V=F9uS9CL|5+Fy*6!&=NMSDoZ;Df4-y`r#?_x zTv6L=@2ghgtwMH;y^5mpd`kqRB&9(_LP-YpxC2e6a;q3I5|S1L7`+%#jV)IW-0H&7 zg_wgaxKnb(UX?FS@k#(I%0Z}&QgsokK}_k?Krzf4m?8;)43cI(6%{!Krlf)|*IKZ> zM$sY?l+Y1o0Bz$BOUZ!(btoi3BT18Wa~h4Lk@}%R&;$lhCe@)lIvq<5wbTt^46;oX zt=_lrzOLKt41MG&DoI+V9fKg5(nwNNq`@}?>ITFVfK1|KD5wEfs9*-p2xe|AtZTGF zQlu6LL2Gdou?CyzUx!%Kd~r?Qu*y7}+6wEas3j;u2 zv>J*zdlR&0FiUx1?&E4WG_6LQ{<{AFfJ=b5rxLdqrBZ2HXeA2?ZQ6=zbV6J$rNBPUx>gdPE=5GT-P+q5|sd5Bw#SOTz@W9Nyb@N3tEJo|GkAKDJ=bvD2|7DODDk zjvOeXQFho-m3GO&xte>HlbPk2wGaq*ym5WYnQO$Wx`q&hJKmHw+ExObTsRAgN#lQw z?4=lv!*+KZQhjz4J5~^{gz$-Y{0=!_@n}6(1tgL~0RHHB_fPf}_K{d$OP-XwHWw{8 zZ0E3q?eETSrZBU08aOok=pN5=1Bv zM|fdYlNUN>4&>wHwE1N!W>7mZ-)-vSyhrKZ7J?7~xN!TsG3FWib&ZgPYXiby#HezE zCsZoLX}F8na-3K3uRlI|Mk0;_jze8VJns~xXt6q?WLjbmAtVk^V9m6M#EBl*e`Hmj z*5+E8{GvoB)_b9blr+=fufi<#Ls)~Ii?QfIigevib{ZEN`VP&TJ&E|qLj$7x}*ckJC&`c zOF?E1PJnAd<-rx@6>~4zU}o^rptk+gAZc#m@#QN#C-@fa7m2$_9oQ`+}x z93RG+o^M&hN#Q;Z;cgV*ZW-+zXv?w+2Wl&8vkI)&G~)8&s!Pn}>}C0F8&phyR^qwe zrkqw^BhjyDrC^e^uMluxtxiEF9DqqAF!q)QRRAh$-v0mx_$==m_<{6qH!}4_SX@Rp z{>+2Ik&$L3r~p)1BL^&~*a8(tuceElACg_Qf5#vjKnXBZsS0f+K-`g}39zyGteKy* zZW6>3Z94w|n${c8$LLFeqEDB|Xgk4P}Oac#tTt_nk6bL5d zgCh+y6op)FMJwJ&_}#3~^h!$8PTB<(VAnrK0l!K~e_tMvPNYqE_0!j&!wAl16&gCLzI|D4FkIZAZNNRDY zgaNyfppYm@AO|HN@vlf7qd9&cw`y}=T{t6!GbZI&Y?mRMF$`8c zT+DV~jhvDcphV6-A`cO?@NQlZhLfanKc~_>G9q@o2|h&Q&AGlHanPjg&ssNdx%x-c zF3sTr6DOGsL#0CwK=srxzgX}^0atV)1SSX~VMQcBlhlX?M_3>pX_`$byaSfomI!k_F@eCN0kjM;T*RVd2we zjIg-sK5<~pDwG^jMZ#i1w8FJQqQ>$j(Ii1Li4uU1(xSD#yZdDA0&D1J4J~FiSO~JnpaU9pCmvCMwuR9g97I2kp)M4ev{-z zF%u}yOZ_41V#0>uxzdf}89#Xl1du>M)pWpu6R2NxB#0z~V4IzzhaeE{2(|m-&6w38 zwpOA-Nl8kI;OWv$z&8=30VK`h+rC;!RR{GntU{$4lIQPHru@SWw2ofw7mHh?cvGd& zg*ak00Jw_;rg?%vn@5$gx@7rMrpq%3RRho(S6!*QRMOPfGhE#`RRjeGKIjECqqwbG zQa?z>_;jkV0rbkXM~Iz7c!?_{Tm+;l2oZ4`Mkd51GNuHBLicBTxovPn(Uo^ul>d?>VkP?66O;n`#}RMD(#?RvA@?qb}~ zcobN$cQpV|ik&<4?u=I|%-PFbcD|w1GTSLGy6cK4ZH7}STGq8fQKd4aDqum5nA>u+ z%oC*)IRH5T6JyH8`siau&($p9aSE7IEg*3zOCbOuizt;405x)0UF{ux_%F{WJ74X0 zAk8zJ)|)5n?}c+V>as?&2Nue>yNH@4iZ^N7BT38YYM0+Mw@Mmu_E4=#1nKk_VCLbD zGEAII)?icu6bq68YB{4`jYeZ0CG_DZ880DR{J#Meq@j3S)fU{Ng;>?R+Q2__vpVMm;KqE=CdTFSd3m^a6xS3 z@Py)2IE4vJ#RRAaWe^JzBhc=^bBaDN$SQI=hY;;Pb&=L(PNqa?uAp|6X(mtq0MrCX zfB+zpMC_fO#wt?qv*oA-VX-(&SY!N1R}r-w8Yuq&3)QO}d(xIcFBCHjCMsE06EebL z@KX{}TZ0>f{F=&v6afzI`U-jvkkdBsLp`Mfl$XBN}pL~KyBEGl3P zCTT*qH*4mNI&@pt+`V}BNY=5G7IRI&X(JNt;YSo&efKvI36 zLxDv&xrY*u^24kpkA~ZnsOd=i+FiOBaJ{g9?RSc zx$0l29ElLcuTj_lCI_r<@WMD25B~rte-?{T9ZH$Kzm9!=pIjpm?*gs0ZyQ5NYl1=8 zKUed-M_HJFU`_3N`E9pT*9d4h%@wh>*1fsI6+#ZKfA*FhYYZFiJU8EX5j7c?DuYtj;W?feo1%zA)Dr_U9^HF^ymoBolJlPxmAtGhWo zi5Czul5*X)btkDoH=TG3v12>M~GO+o=2Ga{q(?)c!dtSaxlnZ?-70f z0BkDi&0GB-fow-9pL1XbsWuipEr9@?4K4owx&D4cD8G^M)6MM^QV%Yjc}#tABn7ni z`PXW;@QeK?+v|c70Bk?B{G?!l2CrN6^^#7}ztmVk!_ZmXhf@SI41y~J9(?#iH!6;v zqSv_P9Ov}RnFho}OO+ebuEpuoxSGQOAS>9tB0j1WV} z^fA4l{{RO60N(`7JN}TQyLakO&&BBP$_n2_~ju&;XZFDorA`33*T?M-7yke;ZhPQ9Xiv%3|pWe>(geRx4s8@q1|%9`F|-W|-XtPiiY6O3$rA9#PNmHz;yp9rF}Gcgw5sK3O2YepK7Tox_6a)~0l)3&REcjdf} zka~JzURvz~^Qo^od|*yE5|MrH%HdE3_r0U7@Uj6MP6%=UaLz-x8%-LH*hzhu4g#)8WG!*p)uRW=3kt#v0 zO*I2+OaLjKZe_4>sa~#Dw)N z`b70sw5FEY-BD@8fYOS!l_E}0>C&)SDx`aF-~aR@$NI zVFzMrNINKFY{^!Pepidj=GI~p`Kd(z02==Ql^@Kj5CfOHn84?<)rZC4^9f`m6%0Nw zxMd@CDZt@jb5EyTH<+fbfexUOl{QFF0ZJuWNFiFR1jLd`O12jRpWWKzL2bTFp8;= z2|AMm$TF+Mnf)aC+=4(h7<7=XD;kv6pxo>8aAAs17NijDM?XE$i|5Uu>UUD03DDwo zNhm2%7l@LP0{$5$=3wIG%1B{thOK%zENI-C@oNl9Da0+WA~v!30QS>d@O?U|>oX*Z zQ0iFBGPI>i&=KA`?NW*fQBUe5cdjg!(wAXCB}A+2=iu6KU!d%RiAB?o?MhjRZ~p)& zGc|%kZlYPVFRXq*{{Xf_4?7F^-t9{8*l8;M^X%879Kw3kq`FD$-Wx4ykWxZ0QWB*o zgoMmMYJz$H0P$kr%ebqv?9~4N)-}~ErD$xWgtbvf#X)ViT?q&UZRl;b)TArBE*|md zDX;jxl8j%YR3L(VPIZ@%F6tOeehkn(cyV(s0RxLbTYmnLe zkeGl3!k~@z#yBA*gO}QxR)pG)bhUZnR+RAq*0K4wb9ch2#H#!aIOEPKur z0|V8b^tZ{xNi88}^k?f(8%owJbxD;iQ3F$AQZ$lG4WsoaC@RVZ*8~GHn(8bW*f%E( z{tVRyv1uv|>ceurgfTVLi-PY$8KT1YQ(;n)PlPJpsHl|!fCwHDAdn!$UMCXETDYs? zHx4L9tY|Dnw29pyf(W}YAS)CiFuC)@m6b`W@e{|vk%Kr{{Z3&Ou~-ZGJOw4}}9hx)on2b;5Msfh3g{veh=_^=04!!gWCQ&`D&|yV`A41^>j-xMQSMF8G8(n5HJ$+q&%E9!&$dBw34hbrgPk4sgeihLTFkbp+ld z5+IU!Oh^+TTX04lEdgQ4)rNtXZtgb+SG*u0s$5dG8gnnr`GXiPb(=Lxg_5w40)!Fb z5C|sf7K5oIbe%*TU&6^VRN*RS6#&u2yMePBk{_)4Fl4kbpw?^y_v;hoH)CR zNpZl+)C!?86kuGz1S(Y`0NM@A<9EvP)XGP!nZ4SZ5FU(Kg9?pz<~inWNfMB)EC_L0 z2GYMn7^ep1wHc2T^dJ;=yxUy-Z0+(Xp(&RgzTlj{Q zuPMy?w9zHelBj4+8sBrj(i2d%A;%j?Y&fkHuBmHAhum9j-&%gru41L3V~CT);Bhqp zECCfH-Ao>UN8%4Fk?RvVCxIu`5&=OaM}!i=L<3L+wK*QqAue7%dp{6#Wxm&|Y0CkU zhKTM;Qr?1dY|O!J3MfXROOTZ*DJu#EK*{hIrv;RD+cI)pDa(|qA9Tt~k`1_c6y$*H zMuopgpMWV6Rrr7GoT&&1m;LwJRfMDl_OqNs!lA-Q_S`MOAcX?#UMKLC5dQ##M({mk z-aREo!M1t^dJH@E32 z1Sl8;pE5QOFK%g@PpwEh2M^s^TzAko5#4euC`_p=IRMSISoz)Dw^H!?32mehohVR9 z2W0{hCebjXU=4{oiIb$_RV_;t4Rc^Bb)Yn;eH*K9^4Tb1R@t1e0MmW`@s_%+Ev+P| z=}Zy`B%Kgd1_&uIL?$4TcEuTf`0v$&F$K<=eJasojav_o-Y`7Iq%G8dNF_iSCd$;J zpbSh$B6>i)V<3r1D6l(#?`QT#!oq}-`&FqWHrDUw&aP`0&K_u&LdYbQ5>Gn<0aSuW z5;ngsvqu=F3BwcJx!p^n6- z4~H`e3E`@VAT!zC!q>G-dA*r&d)#Z<1JpFmsZ_apt~9ceRFbWJFZ-<)Op_$;B2Q@k z@0F9~5(0qr_=n;pfdHDj*pmE*HfZ5keA;(tdq}BL)y0Oxunb)m08}{gS*jzevN}}I z&?a=FOKF}!p)j?b2 zpX(NU{qESPcc9c|? z8>-6q1*e-X1FD|8iyv*kqfW^>TxTarq(1TSRRBvj_tW%{@ zFvtG@w6IENfIpO~pw^MiX5jIitelCn(!3{UaEjuWgi=*66PwHf{pyqfmxI&Q7MW@@ z$3k{vpMT-5FyVSiO*FJL)$|QC%MjCa?Iq^Y5QLy4B@KX`Iza%AQ#jl^1&Ww+Nh?&v zolm@5qdz`Q@yUHHU?m)@5R^(ssTe|%fp@DUiZOS01hoL)sXrcEVVUK>i#`v|%c@N_ zdDy3Ab+pcsbUWPBX7sO=5=Z|4#J2KQO@e~2F+H=jlhmBw0WcRBj1CU2OS$2`qmSAD z0GgCEk1hH#!bu&S&7_2Z`CijwC_UtZ`9&m=R^mWW6!$?Qi81Ow;AgcZ)|{QU>PR}- zL={OQQzHFf{UqZ#*}PdR64Y2K&UkLZ&c>sjz5LtK&PYEkI}n7a#cC;`HSjv~Xv4n~ zIS>9D?Z1zy8$w&7%rnMwmJ-;q*y1+Z%sa#YWS|E`HDOajUWF8f8@Rq0}^Ktl{?E11FO#QErnMlLnvrO+R%y?uc z+nkQlki10nr6RJEHyb~WaC8GMX)6T^;n2m+rXoip*I|Dz<}tj2)nIh1^ZG}g%E|Z} zbnCC3A<7FflM*gKo`n1L_+TnPc0!5Wk3MzzMT3W$bxG)3{{V=E)Z7_t0yZ1Zy|B?R zo02-JE&92C$dCF{_JI`>I*Io`Uzgd#6kdVO?XT(QXnYAuAvue z9lX{4&;=*}K!YRL3_ZZP1HKVT1F75R0a1WR^6UO_6O|C>zi*UBz_J`AL{*OoBJ6Z) z-<&A7K0sQ-)8~fX`WGE{_(n!lq{;<}B(sytVW9ZHTyOwDAjlw#{{T?21|Wz#bL)a7 zh-y@R-gm#iV1fcn;DQFTa3^iX2#IE`x6jYk0Hs@)ow*-gpn8k)!3>5Y^$&lh2%8W4 z)biqg`b;@${*lULa=72ye__)JdfKcD>|5tS$eTQ5qB4mm%ckEE^!{fG?+7Y&>2K=% z?Z|@zBmhsZ@%6R{908>?9(UycBy#o>CfkAc&jdnC+Jivu&?Oy;gzg|1xcS;h_K{

YJ(;}>v?L6g5;1`FYy3Q%3#qG zlL;K|teG}b%F4Q*#A9>IHYBE#q{DvS_noD&q3h0BBT0R`qZV2Vvd-|nEK?D+Hpho9 zJm;xA}bbU$mwr1CsP$T7vRVQ~w{;087nI9UoZ(A+xMs z{Ut!e|E;I~dzBd_6ssW%%_YF2Id>skn-Wqt;d>%DxmJVytJeQoF zBoJUjPF?}hq>`sU$^Q;AQIVtszHeL&W? ze>e!CuZm_DUXlOxPN0zx{0$&Hb`BDN z%a2}QQ!NhsGCifI&Qy44UfU8^`gvs0y0cJdYVhMZJ$0bjvin8G0{Bj;H(SYKJphe!2yyyy9*g$ zbbhgFJ4qf23H4n#xqb*vU`)|{Ux-goSW7e6AH34^8<6`eJ@roufP{G2zov9?&A=irOAWAg8)kr^mH?r zhY_r9@-!V_FWAxa;Ju4Jn`3%)6Kar)43NUa)H)1qx6Etbsd+-5;LiN-rT0q6L3`&7 zU`w+G-6Sv`?k2J6d~&-G>KS4*K6f%p5WHd`kBwd5lG(; zN&O;cFeVG348r!8mEwvBZvc0ZQ79^gXUsb-D)@6%YljU@VtdJa9_2j?6}l#!fa+q| zAXc9nX%Q9H*LLp)Lod#}lkNI}RAY+aW((q!8M4*-7QU$Q8WAEl&WzxhlKdx-JVIur zA5+aGL#)hGiwFmu@((~ro2P(is?;#8@%DPy^UOpjs&f_EaDtUA zNAKniOnrV`$d-$?mP5o@Dvaa2hr1K;1Sd_;F^rnQ_B<>gukYWzp4UI(Q|k|1E^1)P zP}64M?-MK*9l>9kGD?WrLV;bj6q^$h|3GYRI@ZVBSN&STT2E4=KHSfHGn!)?#TVKP z^g|Q2F=UAqCwzUGkDYYq4oNeyeEAA~%S)*kePK67PcjTWw@aj;%nq7ic*eNBGQNIr zY#7v+TYT2luTsA1R%d|YxH(N`^0nz2eQi6$zgPZ#^ef%EssC)@>~8?#^PNcjq-oo_ zl$k~JNS&Y&|MH~4JFG&Dv7`!V4KenwJ_{KALk&DmS%}R%hNNuePBhRRG8}-4`s*1C&$gdW~NF-+ZLywjyUs{_6#7G9e zhe$mf+SbSdfXQl{OYA^XAR#IJk5&x~z+}}*OBtV73KiI8pL%0Ty0*HiS;Hj@ zn2yb8tJw1_M zKTlQ1$n4XfY`t$6PW(b=g&;({i>sL4G#D87rdhBGYI@3XZsunyeVt0p!2wP@_h^i` zA!A&_jlAkA`B(%4RbyACcKH9GQ!AP}j*U(e>wPl7mn-?Ob#Q>;uz051#>dhSH@P473XQds| zi5h2nxEOprn05D>^JhYk|Go=*<1xhf2jb;ZoTm{iLR2Q@o3FS6&Fd|TMkCsen`opK_P_V;{BUeQ#In9cErgI2@E{ek}fyv0pai#62AEOd0cS$Opvr<1KO0cOg0g zLB=?cbbq4k$>RBDqFrx8pubb*L;Kxr{FZ;(%u}a*F9s}oT@GvFlWz^1Z>^=jD9Nbc z1gayK3*k5JWY#&yYNxIx;$ngIoPBo!CI%W*kFwOPz16?e-sxqG33kk%Dund3T~@!m zv|n|-n$8h%loy-Z|GW)|+HSL-Ia23RD=vhCAfuVbZ%!g3f1L z(}cd6ehBlRR9T9^+Ak?9C>tz2*=Vb%5<~)TjB`{S9|9Js%5*J}GFp0YI*VFMiGfLm zmKnJ7Dao)p2XE|$ANdMok^1>c;4h87T3=SQkpT*CQhx)eUk-$U0_;Wx`(`}8nrJ-M zH+^qAK=)H-QXW9@Z%O;i%jRQO&JT z0<}^8z*#TQNJkL?-!*BY2J0U0YcaD6Ld)`BS(Y!F1iX@O@;E0KkQ5ix^`6vF8=-!{ z7$r9iqq{t){O#J>=89W_pLSIs_IsIj z1y9~nB`y8--t5I_Wt9ZQwlCkND?q6uW_Sh*pGcojnPobvHywy!2(CR9PTI5ln(gjE zIO;GJRiI}X!TJsG8}Lq~A-8uOG(?4Z{R+_knqzZid;5a4(Gdt&PRn)%B1Ic{jMf4`CxH>?ZIex0P20cSxUF_+Qbf(4%x3BwAL(`2% zy-cIXo;(LgPJC#Eelok5NtsM-8jCte;R?laxUQBowyeS_6_FQ+p)*%L>((hY!4&n} zdPd29>KluzBuPI^*0WqTE`y=bg;R?}l{i!iQ>}T8gt$WWc19*k5DV;0T(xUk$->Vp z0n-pwvN4_%-v32Vl130r87jeA0OMe3igocU-$Q5KFDZ$YbK%lLP$?;ANyOL8x(ZqO zHdpXRh;w2H7PIo@yC_b@b0nQ5ReJ-HoWjXH57>SKj!l{xxC|=CLHt97mp!7D&pxZ_ z8g-7j@jHi%%Ejg zG+Xt_A}IZ8iN|zJ)n>Xv+=j|<)~vx$VLYJyIYlsuhtQY8x@Q796R?`^Lln0qepNJ7}<=Y*`A$Ea#~LC`3rKXg)MCpUh7 z$*k?GxTVK&bUVYPkMNXbGpq5}t(HbQlvW<-=tNwpQiISYKR^9b8b@1Yck7P}D$;`m zrQRMgFGW@)4ir{*k360&Bj$q~_8;zztEPc%S(O6bmfRc~ja)K(qc*ytQrDg=8p@?Q z*yBwYRakT`>{#GVVwUD*?q{!236&Efzh)<0q8<0}iGw#^O=$gEqiAphLq1xh-|Lbh zO(wC=Z{Vpo(^-}SZVqJIpBX)P-mkao-%x!Yk$ z$UkALVG1;5O|hH&h)APR9DaB#SGc#&nq+GhUrd8E!&OXI%;%&HfT>7Q9}3?*NOMK4 z>FAcPO%%bax@n|R&^&$vLg}-JYJ`V)igJtt8gpS5)fPT--}n+vCZi)1nq}Kot$1n- zeaeaVhGp_~Ay}dG`9DyQB!NNYnwRlY@Y%MA>(4I7M+FVeiDJU$)SJ1w5--aO9v*hf ziB={Kx6}NKO-g-TggA#S=#4jLio%OZkol-t>}3_;JAjL894XWvl1_iMwuIYX6FIY+ zqFznM8#usLJ?b=IUvN2Kl(fJN6I6>;z6k|{$Ln_?2?n)V%ja=|F}@a+-;A4NeXt*O z_rqy_Kz3~~F5-JxKJWLiHQ>YOfqLkKDMW6VH!+$3NB}`=&Dt9?O3t>x{TSyOeo~;W zr(cizY3TRz*;H1zoG|3c7AFlzv#Y=HXifj+Y3Iey5X1&A&TN*N%KG?pz1AY9p^{34 znAui%mg!8pYzq=6HM&*Y>Btej!e7FmF)Sh2*{b64^+lHvv<_6VC*Eg z5&L>TMn=y_Rj3yRlHp#qR%}()!mPDBOy%m@H%8c&99W8N-iTw(|6!C77W^_V}5?wr>u6E?oxx^(~_mEu%I5Y7ctxcN~k=K*w_sy({2;0eEsQ z={kQQOrn80y=hLWk%qQYUgBc4C%}ru| z6vHSVe+GA^+iE2uho4XMgQD_;uUcOPigmw14+Wf^v`BoSy@{|^3>RY<4n_GhXhn4n z89fPoQgcRk9`m<`MbFEibKS>dM17m2_Uktw&Hz;{;`Ucm z=O*pkp5A=V#DY7Dl42fG0!0@HkueCrP&`TJ(HwbeszYxJB?pCRK8nTbHfT{kUI}p-5pzt3o$*V{42(;AV85; zALz|;9;7vZ=;~X9zKy^aqH_P)d~f+j!3^mXBPhN1fc!JD~4LqlLOwO>=N#=>3EZ4ZqgE}Y1NxZ@#y{Qmy{IEhHv8I$e|n1 zu2q{$Rsq-mIR6q@$WvX-pK9Xp{hfkajYP$`p8*C_{tm44&lA$6YI_eC&cdtHVhrnHeeGTeMyBcBm2!H~u}<^#@`xW7 zGUTlY+|)ouU7H1;cgYoW-lu`nzY-Wk_NsyWlx3B@abk`eHl*XXu>Y5CgkxQ^si?16D$~OGE#KTECt&K8KxnZD!^~Lfw2Mclq{7WXPoJT(@msp1! zd%W{}Kl4{*31E3dSH3vh&wibzpJvsWPOuk^7i||JGN*UW8o9_(-5Y+1ky5Xpv{9nH ztHTT32IrRb$JLX0=h$#9sg(aBGwcvYmJ9=QS@-NiUEoBGC#>AK9ynbBH)Y*h`v-`* z+^)6`wePlpiBp07nJWf{wbEgf*t&u1Cj5vm)8bda zcLLGcmLubM0t9rQOvYJ0^iDrgqg&0*`96&ds`ZjwH#Nnk;y2%^q0CY&6&0x0dC7|` zUuyT~o2z$nb31isI_>8{`&}tQ)hTq%8i>uqh?EDYKbc%XPjTIZ^B9ccgRqXNy>i)0 zy|R4nC)_RjYfu7Etiii)9ULIzw7bt+V!NLl%q%OOhpnVq#NqKog{Oqy5=#B5+l z3JvhtGfnVB4Wc0H88X+hdt}_|Vm02!r(gvkS{sgy-Ke2A@iij5d|x!RpwRu-hO7@A z?)Z^=Yt1T>5D%Z4z=$G}@M8!pxzskpKwY9eo$X4GdILqbLyTVK4;^GKYD_ z$45EWrl^ql@XtHUQ@zMVQ$mWFql$Hq;=m<=O%X-CpJ7LeX=fLHlI?5nN1gI}tjEAa zrAqXbNSuAcG9k5}HB+1X01tf6e`KtMck-ucnDJQTr(2~LUY4`_5PvUs%MX4E*)gzrhM6zM9=qFMaUJrK%?-U#r;Fxde1_Ll^Qcs_z}co$e=x$9j<$I1 zbw1Y*D31H5{Hs~%ujNihvTYCLOXn=JrXi=sUbq|+rVjhG)DJe7g!4>f83)qL^{WNfsCpyYu0+ zd|orW7?ZR8)G>0i2x#7C5GNQ5b_iZ-qcbgj6;rF`a2l3+BW2`e0^>y+HI(_X)Zet? zC)8jMcf~Vc3KxSSFYb^Awzrk0FUuC$T+>R0}Iu z)3$bj+bb-r? zX#_rAQGR%bK9sW*)1T3)lsMEf_k+66%_0Z;)TAMTd`ZFrwZ)3vPirK6FbIJagukM# zq(!Ea*-&hND>Dx^O){o*p>)@1D}eIAgw%BP*tx9|$;s>*Y#RCB9?#xZbY{`ZToihq zFHa_H+JHim8yVp0N}#tUesfF4oKQmnD2Z zAp{%Nsza8jc!d%j4a zVrZvnB%un4uBeh!=jHo!^906WoB1kIF(dLeFHH!$TmT(Vn+cT5RFu zvS{EyCTP5$rkVdKTqd=x5K|K+GgD_z)@QlgSj9nY#Pc`cvUjqGP%<)qob~v4Pec94 zn5j7@s#8;9rfw>3oB?ULbErYC9qBKd;lyT@Y2%v>O>Dpb#gXD6YuIRxDug(x^kZjv zxaCJDBuYO22lu1Z{e3CMW1>zjNb1;6CnE$E&u;yzLmjea4u*r}>EoyBTwVMJk4sIp z?sY_Fgv?8GXSKhiLT%;C^BX%Wn&Md|ZCG4sSZQ>$JaxO_v&n$6*KRv-VNBNH^`=uaU<3*ky~n zZ;DUOq3_?&^Y1`7_*QkS z__Z$bkxF58)IcJRya^I@dOys z$BRGkaODp!7|5#D)8=l*V9aNEsdM$18i+)a7}Pqf(r$n92Kl%~@7$8e@fX|A>y2mF zUJy;0t>zb8)SVbV* z0ZUz_lGVMVw6Pqz&fy}cQd=g+QesPSv?~MGcg83Lx?!Y}8tTGG8QkN8z<+560F(fq zT>>LlE+n)}ljvWNz~J^30jKPL^&T@ytl$3?0Z0BV%KS7>xh15?FETb;;&Sl;-H7#1 ziO8S-NsVI|gmOpZ_=M3PCEay0#kn@#ftUrVrmR)Y~CGN zC`qBHaD8Gp(GsnC(?-e|ocDw$KIA>87tdf5@r zcjG%tX}TAGEXjY}!~jC+M}q|4D@+8F2$ILb|D_M$DDuA-CQEvR+{^dKDixW3KoU;7 zvz7qN`rFT7%xLIt_d6w8|M`pD7TN8<*{NdWr{lRn=5{*qAv*|vwBFK`>Z=`tySuK& z7TL3dU;aFe6&I}V8=!Iix6uF7>!AM_mla?0=Qj#uCjkBLCMagC`aTVo)U_0LYv=1X zc6M}`7yhgK15727wpyZr^d|8&pb7=eP(4*&#U3$aps3OxTl9aYjp?tFoe<Y?F`%XLHBDs$Ig7y+WNma>a1?yr^Gw zNV&L9(=P|}5k!d2PKMxpN(JxP{4%28DD^1k^uP!$o%ixNo3sTUcvB5BvY3qhtQ@os zHgM-k&4lm34wXWGrW*R~z#6V>`i%*;M$<1A)!1w4`1}WX4>Bitr>KJUf7D3!PbJqI zS6^3;#ODmc!2aF%>vR{~Ecy za9LtcXoeH24EeTUj0VffXatjhv;df0U^d#*(Of!ONp*r40lBhyQ_>s>RJIu$^Ba!! z7#LP0{=wkvmR?RFNj9DgeNH@a3oV?a$d3g104!JNS$q;!iQZIU?5lFuNzWgGu^0fI z$iC%{AK?TH1rZ*N%@B})6;wHlWrpSY?DLM<6yp8DjABTLXUJ9n9*iC~9!4VHRi3WWoJaJx&vm-Wn&7@&=%p)SlR^R#LZ0JKg>=DdU0cvaEjp}cD zM2^LDtXNYujlDOBxcQw&E2ZDD2mEL{b`N*C7L%>Qa6{F8RQoyxXaZVSM7|htI_V(n zGMp_+Ra3Qga};*L3RQ)_D}Ao5UuD{>J_M4He?*J^=qlYPANdGC!~-NJ&yu^&NjiP_ zyG_}xkBX5=&Tp75uY1Qoc0Va%)MzR*E&RpU)AxN(zoj2Ge}N1Lya!@Vqq5{`gX5{I zigHDw{-6Lans$DF`(c|9sx@hT#bbSOczVcKRH$Ui(^*=y(+}3QLFj$La!beV(r!Ec zlvcvE4gfJrI~TmhBUCNvaj);OQEEeGh*iJwjrwj>d@{p2!LT3T@f?!VXQr*?l@-bm z9gG(=Wy&cOwH1TW#el}E6y+R{4HZ-^P#%3+Y()Yq8iI@6rJJ5sGrFvruHA6!zp-oW zG+p#W4}$bJN~DBBShP(T(Ge2NFZ5)+-&CY{e$7!mO@=h$h-zWk@06)B9%-966cMB_ z0(Wy>Fta9>UA-=tP-w6f_K-8cYXX$x@33{~6#9{NF-psZ$4e>$nIK(yY#`qRQ$)WX zV3(_|r(_VMfsX*`XfR=g&J8UKr<>A_w+dBlT)uoNQV?GnWdNIC`MMrrYx2>V(5xAk zg6o4^5kQ7TF|1Y$Is>%yl)_L}R{E}PQw=rWXIUDXq#myJ93fBX8QyqVIBeI?O$+W& z&3oT%76#@P#8h2`^!|GDKPPUt&33ifH z%Qvx5%91Fm?tPp|q8S>+R9llev2dFEi83eN9}fPkvQ8y4S%e>Sgmq;5PqB@_LOq~T zl;^0F!$3%6*UCwNl)qT#A=a^CRGbkk+9=FJ(%*70V+nl%i!3d8rPiljU0!|`lGj_A z3(Q#_X*As)-lerssLH zZy7ONE4sSBM9#yC9}J}^ixqIF!w16?*XPivOp_4q9E2s5A-rn?pmu2>FdF|^d`?(A zm{tmv33I{Rjq$@z1@`r&pdYn2OgS+Ou*TXCC*;h^xl-Eefhg$nIa?gkVE2R%7t*wql?4%Y0Q4Gl~%RKW(4~=$^I06Lwz^)i@;+hw;r7kG%j=g z+eGf)T(QJ*3FE%eAnknR4UWKjbkrt%*tSNow>G?I%yJ5(U2OMlJeZXAwE5O_{CS#T zUK>GoukRQelQN#yo-9kTs3^KZkz^E zBw*0S0M*O7*Rk;L)d9}{@@scILt)PQTeAnr8C7N9MR>d+%H*kGUM>DKn~y0n`I1nR zkU%C9_0tYe1r>&!j?6ozZh7`vL~0%3{I+1h6UC!V{CR{Ms79`@V&!O~UsKEaxncJd zGt$L;dbTC-g<6<;*PMnf9BsOUy^|?g)cA+1tOG4S?AcweO`)Y3vp56ssTv$B0p4X& zT788G&+K>58M~4?g`D3|+v~rqvmt!3(6`c6zW|8q3?t2F&8l4^L zP@D|84@xwUD`zlcHxLnup<>tfk+cY%(An}G={*(oFUzMXKv4K&3!;{2oL%re5%ET> z&+G@gw8mh_FJ^ac`X%fd&?i3v-m9?Ol?Rudx;?R=WfGYN4C_GBgR)^D;H%a7!ygo$ z_R@r*8bYYCWWWqZ@B7{}gRq&JnVN5D$y}#lFs&L!`E2d34QU6#n-bC`CQ*OCdUW1T zz>_!ky&ibc75>MY)C?|;E@R45H*1mVNn7XVYx8(eXXsc)YZA2G{AKQR>8_ZHf~w)q z^k=|Nwx6`Ar|?FJP7qqmtr!^SqFv_ruO|0~zOk1$nx9l!T5})#Zhdh5)))J@z!4TT z#GXI*P`stQZ@>i7d6?RdIsu2A%Vs}LOMWyB27=OwP`mJl7-rtLH%qrp_)lFSc7ve(W&YR_`^I;GX?i}=4)plGK>$5 z1#tQ1I&hu33nwldp4F|5nWW@P3?fsS!tU_=4qx6Ei_4s&sKI5N{P(an84JtGY8LS-y;YHD{`2r_wYu~UH- z$MWvnR6vMCpS()7Czko#)oYEqZ}cdg^%-tSd-QUx8%}|jI#A4)@#nsMO~u=~gEI*| zK&FWc*=KQ66H`6~&L%qZr{wNqO-Ra4r(oy2H`^s<;vj#T^fLK3_QxvCQ<(@URH=#n zU*R~>Z7Z3a;fCJ-wOw^lKwU$vj%tnZ{df|C?f23&LiCRMOgFyeIw+IGy5}IhxedRb;n<5d1(!l z?@?mi>Svw-O95}c#;fB+NND3EJv==d5+s9V7f1ewiJh|TWy_;Wc!+Z8$H~F8XTGT03 ze?j}9trfYPr>FmT_?Y%oR7x@;+yD8X(d1>Sm!jF9&|Ypgwp9MxiO5&Kl$H?+x%Dpl z*N6VvWPf~$zb5+WFRu&Pbk0iGShNW#QZr=ek^q%pNWKG0S>2a^`QK}YmX+~ddqj&E zlwW`QTPu?xSMyah94kCi@z5G0Um&7!{WV=u!cQ+L3LvJRFM4gmY5hXfe#qosQRA6< z!-1`h-SNV}IXl&c4J;&=!u8b9uwgMOQ6$&!;`TF*ZY}O{BE}g)6^tUV81BupeNunX zE}r0AV4Bft?wj9>^u3}SpSEt-d!p?ZP&`M1?ps119M8KnerFv0U4Pq8Row0R{cXDu z{QbYwGURvKJ1Q-e%tLoc=H2R#`M0l+J}qdb(Vn~Tg#V4VaC)0T&_4(~W`A#keDKJY z%>4=OhN4*q^$~8iu)a@4fi>taNnULJ@AE-RWggl~k4Yh?^dA;%e=)wOd-N%{r!6iu zee1_CV7kHAfw66Mb9P%h4E*;csdC4UvCE*5KV~!l)N&aV;HF#hJ@)ZXg1Hj%lf80ebz6?rx=?5()0Ff*E0`Y4)?f z#;xaRAZj2x6;|i_YfS%i=%ud3xv@FR$jDp!V|lqvd12;uP3`Xyfmg15uo3v_X_bI` zWYkY^lJou^7AW;QJ|F*q-B8hAWZpd3UA04{)ai?VBmEoJFTQf+U)5iB{UgNzv^10O z{V(f!>!<}w-RA8p%kWuvc+co}!}^3HzSO&B6$XlD`w^bVHz56F5z-1gd$%F&Pq%%s zzr137hD$&RAEgK&D#O*p|R$?DOHM9IN^!M=kjb_FhAttr$ zx3l8Nz-yh`8Z`G{#^k~r$t#>GxtGOFYxd!T=;0-4`mfFS2V<{+s=@fq{r2K~Uw>UE zj89hU*t_!Qm0l;jtIH2jDW9~rv|iBolw8X&xcc_Xv`gCXf}EPCv)NQAKwP{#6gRF( zI_pdI&*T_f82N^BfUR`BP}ZrO`D%Ab4z)pi(rR7x>6~cxC!P1Rsj4M>q3B|EvWe^5 zteAA@zkf|EMykTZU9($nvI@Ma>j-Ssja+19n&U2l|^lYKd&QSl7>F_f}ZIue`dkw#I)!iCq)e>TV?p9+7$W*gb zenW7kgU<2KDLS1ppC`Lfbt~Wb&qGg(M)6Jws7pTdu@^N8a2n!S>&R&NK888FiUuv-8n1%(uNC9>;Fq}-yUmv(=fthC9-NZHQzVw3 zx4bJ9^sNg45Jm^Ub4F-tJfqDs@t`^*PI;T#8s?-yv_?HIVJu@&()qO6t);oLf|dFs znx?0I9!NL$9~9V#g0*3CPE$##;VZh{iW8|EWXs4$=`dRxFp`K6Z2vrzGJrO9^>s947s?yrLPE=Od{b zO!rd1y7pl-BsGS=2_llPi`$Q83Kxu(j*@kbV0ItRIlQqfH-k(S9ZNfR^KS&J*Ag&F zZh~l0a~td0pEf*=u{j`ao;FYzy0%(&(eb^*Sc_f}FeaANac?(X+?(aP$}6XRtbqx{^C0l_bJ+}(Yp)F zZQKHctE>dMDAJPea*Yk|68^#R9Zt(rhGD;ff35Juv^xzS4}oh%)qxj}k{XMl*#Ock z$&Xq%JYJy8st1z*xQS?*T}4zv8mWk}19HSTbV1gz@12Ib_6_Rsy!OY@(-`mI?TEZR z>>`xxe)pzge?jY8`wrYWEuh5|w+2SeraGnhPgP7iq8?v}O^#SvjPIr;<&lbwb3NO`dD`#TB28 zc8db&3a#^k=(~dMHIyqERU&s?+qL_I-$&hbkW~=dKp^r3_*z9NM!g_sySwsuik19q zxA9Oo37gCvE6g!ZRU56Xv)Sxr_8N-(kmN57-76p&B8HtB#|(N37K zK#vN`h6`HOvfA)6$rIB!2EPlGE(mZy2j>K+NS%uGDE7OlUEMSqtOr&^DF7UET@C}g z)8_pSdJ>}Xstz%XTVA(%9HbRIV|o!}y>dFjw1%uqp`c=R?dbf3fTE5(Ne2&WRoPCW zsBcnaJkmG8Zb3ok-P~v5a*COxIBK27xBLhB#o@t12wql>Bo~9`=y6KyhJnfcCBE;l zfS#h(35TRv5@!dB(3kHN6M~okaljVrLmf{{zMm4x04aWl2#-lE0{L(^;2;sk|*G&0z9;h6mWo|4CsCE^dp zy^%T9;5cSCF?(pF!!7}#zWB(+L6MT7@>;){jYB-aNFd2PPEYB$wnH~!4=PG&Gt~J7 zUJiN!l;-UNr*Q6O2s`%vR8c0;HoG#!k?(X^>XWvj-wHWTAmH~R5aU*|4lmm|zmYe? z{n5gg7qGoM+2r80N4D*-$B13$8c%*P zkS7Evc!F+^mpV`z3ej1uaQNy!RO5uicRu3*fr5+l)SVBT?G0K_)>O32o0~gZPnMi` zbB?#PH@?pSIg4Y1_POg&8_o30padZMgz@y~`1Xn}rBCBoXAo;f7KXJ~Dr&k*-u zEc5E|xo_o6%4-QZywy^Sxh7afk{(2XAS52##Xx5=KnA3I9~w_55bC24i60Og6-G$# z%#)jqp$o|^RtmGSooO04s9xQ^pkctUO*x;YurCxfX^+3Rf=#@-maB#_RwtUO&X5o_ zR;rK1lgMjX15QuR@l%$tzHhl*VM6ruV>c^?=>r>HpATd&56$iKPUI>%1gVy99gLs9 zusVgfTSg*V@R@XcvSD&=>~ro@l4iMJxnKjexv)<575ASzzMZ@EW^*-WwdCoJht2d$ z6vpiG&62Erf@gXU^VjQ6&t--wD__jVRp8AHF4^rxfm#x7^%UaPw}?0;^oToo*~LO; zb+xR=cT@~F3ePw3HX3a-po!9jlCV>4sDKfv80pg&VouH@;EJPRIYWx-3v&E0Gc)8V zxCs_UZ!d=WHVCQ!1~V6aPtS!6X6I$#!gq*#P!bEi+E1;b4>DZ27EU zqOutk`VuOZri2cFt3x<_gYYp?rywiqsL~DO8Bd4?998rOku?x+low#Z7SSwL= zlm$YD;WRY3X2i_QpDT5wmeL}bG8T4Mm5)2J(Za6C+3!r}@LB7<&9JkonWWfMdD0!n z-TP50nLj9FcC*KsoG+aS@#`uIkn^sMtmIt4v8R) z!XW}kwsP$@ZhT5xePB3~Xj;YCRn46gz*J!mfx2FLh-H{F_{fk&*xl4y`}cW%FWC0c zg7$eP<}Bx$tHGVLdO=i~3_|StWZZCfkxLT~!t>3F0IDJ{slLfjn}RQbW>AlVIc?uL z(s=Z<4XnjyrLm=YIsW}-5qorDeTj@996L6^Bf;2gbC_3yc|_R*8F{OSMaFDV{>&B! zs(&sr+2Z>{7B7^+ijyYF=8B2lLnR+CIsPE{K!BRx7t0sgo2nlB0j#>P+;15*-~QCL zXWd#E5~pQU(dhK^@-{LM5cQa?n4T24dAxJ;W^mkl{^bdfh!)fQt?UCG=lZt;##E?| zMiF07Q}Eq8A=qtJXDGFr0kxTrx!?~~8>VMWI!{Z<`F6Q}UpIyP-cz1&;1VQ6t+Gwa z80a+yWXPB}UMzv*_&E{F7WIXC)>M$6f$M^GmlT(AVwng#3k0d%od*rMla3Gm#m7wv zhQHmTob3;~g1;_(FPR=9ZbAv39=`yKkZGa;U{04Bn@lJG0TaZe@%S3Y3T+eTdI>%fyHZGBC&Bt>$J`xl z`>6Y<%j{@(^;sl>qP|#dAF!+VFq5D#i4Ak{EE$;e_`bFG2RVa;EeYDo*Gr#7`YjKQ zWB5-*O*zcRq0W;eftciK(_LB*%eb0< z$yw^9GrrYbNske7+*F|%doA!jp5G|A|7E7d5V6e1ahS3<5y&~bWHNV3KR2hva7sU; z!1w<0PTb?|o%kK~jE}R2+<6Hv(yGiW?V{tPt)`z^!76QrIX*GLacT3lFM0BNAlNbu+CvQkHt>Zko=+v8~kjxk6vV zk=Z3s&d{c4bfC=!Ue*~C>TorJ1(RnI|5=soc=3d?kG-I6uWl6^ZbHrqBanP|>M;^J zYVg*ZNpCTIeRRdl$Kq<)zVKB43W#`y2n*m?yXW|t1se<_EHh)etR984^bomBtACxS zFrG!&NPc%Bxl8yRUw^xobVoXOq%W{(90RX1V+^KDaqhh;Fj&%W>-bI4yOFm2hD*hG zlDa91NEB-*FEZgHz=EV4oNN3$#>l3JXUit`ch_`eYQJanKK3zCk+aUz1?glE*@W(; z>OfS&=4764%y+b>t{6J7owp^&#;L!Fkkn$~3}tD)S^Bb|5Ua6!!fC6Ys%#U=uFPJy zEcdk|$*#A?M))Lp;2`8chPj2N3nH!RvIue#SZxS*4PvxT&@iJ4+P`7x6u%6NJLHI> zGgy3kzm8oiF?Xi064=l*c46&|^QD^LIXMZf=K=@+S_^SrohJBs@_0YqVH~+lY-PK| zvY2HoYycw1QFm1U9AcE!o$_K^iJLk(Tb&E;!nYy20SOb39Q5Pf6t;)XP21L4vvi-P zFuty_I-^MBzA|A%X89b;IB36yg#taV%bD@<0vTn2&K)xLwM|A%CB47Lq1-Z793!}5 zs_R|#$(DXNg$zV4tiz?F2U-ct{B>clLB1{A5%~8K~5n-AeG9;>X6^CoVEsO zX|k81^r4qYzbDNhp${&P4Zq#qqWm8~@8kdYdPcyV9$wL*&&bjq<)h%QC!hHU+n?eU zmhvSgJqP~hK*9aTkPE3;6I$x}515Dkj{IlqU#SimUe}}C+OwRLLZtCMi%ysF`+LwwiQrT)cq4;+|vZnr6rm(?xDyzP@IXsO4wJRnpO6C zeVX{LVgk)S06}S~N8raNJ=E05O+%6x@cnEWkpuUFGQg6`uGr#cwsSI~BaKC} z-7!Oi%H|g}DeDv8;-G2Yh5I|WlKw%_E35l;dHNP5n$^zO;}42q<>{<8@>Bl{Iz*vI zl9NVZ@BV+-z$Zv)tG(e?_fhb(Q00b;J|hFE_WGm1V=!O*UjRjL?O$|19?b9T><6d} zLlgy05``Qn3<*Il30+vdfw%46^?!Tv|Hy>@pReVgz2zC;-F*A2tHZkMzoRFi07-of zDcl~gXB4GK1u2eLTL$o-q9hNDtfV=9jX@e2m3eu3vnFaL?h;1Ou(mzYaOec4e77}vL z&b%e1FV@mzhGVO|o`gx`&YQ9ha+-35nh~qktt-P8 z3HNvgk^d3{HwOLvgfVu(Ta_WB7Pp4@HiVD$2`%#eB@@$Od&c=Szq!qiC zWddwYQh9+r;~_A?@OVQN$RLZ1HA}HIA!M?!L}~m@g}9lX+rc`RQodxqp> z!dy;HSdN)XTAM<(ZLnis6LEitJqX-4`g0mW4IBx8Wko765XZhw3)<6f^I;qtu{qa^zJ;Y3cpM^l4sVSP!!95tJ_BjT~f#C)q2{~E7g2)iH9Kq%g%|5=ZjI!#n`z`HL=@T#8cGgpUhJ4Cc`2L_s$lj|=l%Ek- zOVzYh*-7cjc~>}xn1e1@!MUJ;N&rJb^y*}a(|v86_Mp~$Qn)80CnuU_dl?h@=O=mH zA2g)Nq1?UiyT2m}D(+_<2|XkSFFxI2{G4`AEAB_Cz;C??ruJnzer!;{Q)8xna)*CyukJ21R~IqL2|c8 z=9|PO=c(c$(3L&&z8rSpb6Va&cK9}7c}eKoDEZ5zOGtqQa?}@2Zc(q_Td})c)};J# z_#`IhaV+gYI-*NB#mT&Ym^fdW!Av(S(o#S>Fet&|m5CGDf)#*qR>vl!w0BN6Tp@Lc z5yU$+rDtaU$kh9!bUYIt_0-%yBl0BbMRUhs`!sIbB4%D|HQ78|;v% zMGhn2gclMmlJAjuL*^Fj&W&>|U=R(+EaqX_-NCaB*4uS)!+@kF_# z5dQXKs|QWTNs*A;<>yP{VOBxt;Q`fDrpawgP@VE*MyzSSvt#}_!!95tV4BuLc(N&R zVz5LSS$%}fs5j}sQp8!9lDgCv#?$qqb5|Jdj~9s9Kzy?XmP#Vy->L!wFAcOGFC_LW zoj}BA^rxYj_o0`GF^F}$+ZW7j8{~pM9e+^Z&-XvJ2#G(c`wr08mtc!_<~$A<9a0VF z+yO2s3LZ!GILmpJ^>|2YEoI+q)UP#kLG?KKM(L)iZ7ovLKNY7HjdPIkgoCwTx2#^F z^fXtibdp_=O7oAGYT_=O8LUFf;~S?10!g&&R7tnz8kIkeCo{<`@15^moMxstjWw=vWD{p#}k|TO#8ih=qsrv)^wtOtd<&noLlCWH1-+z+J zZ8z7Pbeo#Skem=Vxg2%Ax%YB5$}M@Fr_UAurm+U%=SM%Pq%o>72*`xAs2dC|lQ*RfAxm-m2+_jWeKecs=@} zu{+e`PMDwIQ^cv`O1*(p(KswA48M?_fm`I+>s@XOjE^P5d)#u4-9ao|ljvC4In8!FT|r$6;b1Z{cX21+ zKw+0Z1@vBYw=Gc%q$7js5k`S`H$P1A%)<)t3x5+S4LasEt+Ndo@xNIUTzA|cd>vl~ zMAf5^c-A6#f(wV0NPPD6Ji<-#x(9Tff z&Ar{}UP*sYOpbhvC~y7wFV<8o-9(M<5_I;eULn@4e$P$3$^CX+%E(QL`?8P))BYFO zIlB!z2fSxKI$pO^0j}xA2&Ma$A1QLCgR@oyVwvWA1QPYSb=%#MJofJq&}B|tLX)Rb z=%r2reiHk$o9UjY=Z`vUSvbT_7Q)Dy<+IEbQox*(w_(6nfSwdaaccZ-NzVcMs|R&5 zvOQ$dTt;+x|8k^EWgFBYH?+IEN8@Y}*&fj4RyAiDGNxmbKvReM)-+F(s_=kDmYK`1 zn#2YJ^yc$3->1O2sqA-`tIb{Zl2JIXNbt_`mq~IfGWNSKNV16;6?+1FQBY7)b-b<{ zZ(iM0)|(R=J0%d%Hblg%MKM{nIWDhfQuC~MSj#QxFeQ!H zJT&DKEFe7%fi+5l^0U5GD-HF6*gY$p9!Km6uDJ_()t=}#otow(bbV{e!HHg%Sk$>m z;H>O5x~+#hcU=4Zhm!hZQU<~YwSoU)#o&>c&@6_~WtQvi_-=?N5(HLB=|l654eREQeZGt3q9Q+~^*2u)$E zHN#&CNr)auA(!IsXHL)F4NCWpavphquTr4~b$+y&DYLAmH<9xGdH=n?G|_c%^n73f zO8%Xz_<0-+oPnrB*fJct*EYm|(Y9O~Rnpq&Q@5GyUhH>pe28@H58#muzM)n=t=Fek zp%L$&+ARSqqq4eX_gy7$Y}%|S6NyVp>nXbj+f@j0tqlG#x)B z`RF}GY3&()Cjf=S-1aJ7O7^Ea?&>=h=dvjbYgyW=!`bM!Frqh9Evoh>lZh#c4RXyi zar3;l2EXhD0WIzI^QMed^5h2g%Vt}E(WL_i$JK+S{$nkQ9z`JfVFC+!-30;gc3^{W3=^Pjj>ZDH>13(UdLy(X23Muam9 z;rsE3Wo|19^vAK6RTBp>hoO@c`R6Va;9Xb6l2en?N#B1P)imbFj8EgWivZMg3{C7~88V#nE+0aXejHLCAa(Le+4gt)En&r*mU| za}fggj8$Nym_&X&{P2DFVTURA?GhGucu-NPneLp~$r0rzXfxXfr|EdqT*f}D9K;f) zhpa)^y@nCuho8nbGJrnU^`~rR{)^&?T-AFCMCHU-S-e`9JY|)g`H4XaOM#vBi{}NW zEJFhf+T-{h%$@6w?r?Jbh>deasp*~i61gQr&Ui>j)`8j1>4DNs>XC8yWWI(eW367n zXbyNy{-ESFYAgc342wZFv?*3vM#Zhl&7!SIZ~LuW^9fB4J;mZg94!!sf!fGMwp1V) zzud3mbdG~*Fu3R09~7Lfg}1A5*WA6%=?&jh^PXH^J|-USdCbr7VH%a9Nb?-p-ySdQ z`XYnv5HK|Ua{CfU9qyaWwUa3@abWM%P=aqWR8Eq z@A?I&zhIkGcX2^6O8RF@dJ+oYu6zBQ`d_LbQWy%QSz~$Vc@ozCS0xH#`kNPt@|RnI z)K{La1wUK(a*s}&-HlVBv>QBG(tE{)WNyR<8 zIbsI>!+z3*%hftAGjs(igB25tM5rz(GY4YdzgCvKN3Ywj58$2^i`HI%Mho##db?8J zWk##;b*>x^Q;)8TAR8K^+Q;|BrUGUnq971``n|61_>s zi!SYGE9WkMa~hvvYQ(I^Sw*q!&r*&MyiOf)%}g9cgKNyDW{QZk8v*6@f>1%RuR_=N zuOln}setH{nIbdix2wvU_H6G7{_fiFDE7?}A7&Iu#LeT2yZ;2mCra;5{_;cYyRS~$ zQP)RMUdrZb-07SgRH8(Gc;|WY`t_^BJ+^Z}gF3*!H+u8M?cLhHh>rK7c0$YE1|M-| zR{mvQFix%17d>@&LYaxo(7Mbv@)Y!yzuLZ!JNtFu(wqEuEKQ|&nsjS=e}X{2v7cIq(%~qm56Po=(W9LBecTBn(?#35&CF2qxGD?T)wF@uI&FzL5*CZvCKEzcC ziqbN^+r}mHce+ z#lnP!gW1Q+!Uxx~CvSC~kvPxSc@0sHfsFadT9~4nlisKWj7NFuHpc`iUd_!%mDz#m zcjrz&@B1lFhjwA4f<6+XJd)Z)4%!<_Gl}1qB5lwi87?b%iF40SH(mC8Wbi(a;W3+DqJc{ur-jh~Q#G zt<~o@wThNH6o#nMXx6*>^Yo z{p=bdiQT<$XMv+}L`fS{ySbqM9k)=HBYpx=vT|HC65>45)}O5VaK^NBFAl%b^iMES zD>@#Mm5>#~p4yFB`GO|I%BK+;p<-R|MQ)Y@Y8Sj}Xl*L&64?}fNeUYx;n1T+sz?o0 z_sYmKm!bjkgsuU8ykPOS1-@=cXz|My_u{gj(r<1dXv9Y-UMVh*saxw5Q(@X>EBwR{ zHs!Q%M(Zu!a_tIzDITv8)cwi+-P*dmsc1=^Rn}NlMpkP+qJ7A35{OPpj$F)fE~AP;UpCXm92Q-D_X8^q2e2dL=wf#HRWDqNVK#Lw2ksw z`HVvQPIn*zp`r>_9rl;flNj?-56|3c!@r2qfkKbo+7ggtfTL$$mYD2li@KrCze2=R zE+{?yjQ5HC$pys@oEz7X9~6GR zMoa3p0Ht!zM|-pI%|=Bk8`mpm%#qe|a1nXqfwOCrSa-nMb#s0Rq^o9quSA%a35WW^ zt3~WwI)x1jy{j2&Qe|!O(DJaf`w16cm3h@w9(y=O)R@t?^&G0Z40n=kM9MUgO6ylG zf7=mFNa!P^zHFfDN)2a-wy|0~+_W~*5h&5RTyVnb^ehty;F6>2yX^Fx`F?jW5M6SU z;-kv%4z6(GEIXBU*VEsn`CX{ibdVoAZ(Kh)SV2d&=s#@<(t*5=z%DY47~)A)bW!S$ z_pDHJ5YlH2RoqY`l|zinhc=8GoWqHx^s}+_v(FTeP;UrpHzayLndmSM$f33rk3CmX zuDs!CbQRO63=6=KpsnnbTT)UoB^0|k<59R zC2Q{fK~d+pxAZcM*7H^kQTQ3fANXXKb3zwB32)<4`>ZIuPFz%#keKF`Y)@tr8VL%T z)T&aUbeMzH>zFt@xMBst){0!1Gs!bDQnzo$^lQ&72IoH9ni9qnY_bzX_vpfTSr{QX z#b8Kr9G)X@V;qBAFH)a1c2>G=8O zS-Dt1nG7qgyxhLi3`=iK5}7@&tihBN--B_fftI#OXof=a$9lQGWQih}kfccwpr3ll({x{^A>PgTdU z$u;bgE4k;#tbJ|@V05UN30pBEd}ChB@Vt>ncMf8fsS-pE3B@kzXHIJMEXVP6|{Meec(3x|dljTB(vnM|%X5z|J?bdi=ZY;HU(=*~IVXeK)`#5?o zP-!^CP~$_2xFH8x9F&9+cl~2HPKk*EXzT{qlk3kybvjc}kh8C-1EANxbGbh}w3?Ur zaLB${0+K12h{Qcb=%MbNYcrLUvl*m7q4z2?S9-WJ#Zf-|Hh1dZpW^X2_2Ev>zJl*X z;(f_+~8-A@<~7MYtwejCx(AWOc2+uL^SlKN)6!$NfpVK|GI!E%C=1} zx#39hYaOnPwq|Ds8E##lL&N=e{j`PGeuK+?!xb1AG@nV{s%E*jUJr3v?44(RnOZPh z@0@8?g62Tz32?tRwMf1hgHFL&4NW9h#%nHka>?-+rxnB)oa|&XkYm%wP0yrTHTrXkgebN zOV_hIF}bNb{`6b}KJEC5Cz&`4-gMqqS~&rcZUj%S7lI^b0P3U$9-5Qpl10dD>Onoz z9)~FbclJGfywN?RTZ66tjI=BXRm~EfS5_ zq~V&asRoWc40X_)CGVSu8U66-<~mr8V~6qk(El=gX=%l01-DbL3`TprX10%nz*Mz`H>Y+RSAt)1>x2w&{!Hu% zy&6#{#i{9R1=pEz_i}pAG~Erg9Jy1FVmy&x^3FQ_NyG!Jd6izz`~32SUn@feqvm5t z`3)`-%}lvN`a#2ICFe@?+LD6F*?1lrlj&${HuzjXDQ|{JjXy7lhTyIS417ZodrJ&{ zWJBlQ4{VNgN?nZPo6g9@`#o1;lajlz&5qz7Ya*?oW1in03gAAYIU^}ab9}}5zJ{t1 z!%SD^leD+M4;(5ZWNMbLQz3AkFs29-r1}*fPm*z!(IO>$W3zB`ke1^mcr3J9Td32F z=&V95W?n>svg&GH2XJTTRKx!`KxwV^2W5{=W~h-rmnd%vdf#1MtK45XIhcF{=| z<4kdQS+{ETJy_o)ERC-+C6S;@rvLHMJ@>51BDhCdj5Yt^wC2XVqJQqXqN5XcmQ;kR zzn{E}r|kilb#74agRB9Ho0=wOAqo{j>ahE1*xuPqcFGRr{kx2jGP6cCU`4HzmL;*c zs&1lOUi$zsYj9G|Cp^LfhVdGj(k)Eu`2ZIz=@{j3*sisVJ63cz2da9=p_)YkNX7#h z!*k}@KNf5ZlUsuInf=nb#1%=sSG(Q3SF4Qd?2THex&u_tNO4X?H`z@N7n^&ICEV&? zx`*&bUy2nLH{wz6pOqk_gQ?H2-sLd&YUK^2OM8oJb*}u#??X4Tq@TEfLK1c3`UzEO zh%_Ye1ujEO&*;wW6b-`s4g=nq2v^$om1bSu9Q42j3VPRV;W@8P00J09hVOB zatdV8#sr5LaOeEkBhgru+SZvXDaP9wUmxA*(*{rYC0tY_^iiRZAAos?NPYP#dlnyi zITi%J1_;vlkWb?WwpFE7Ly}D;1Z%GbX|NPEN`qVUUhYv_e#xC!@nQl;XQCGw8=+d` zblIZw9I>Jh(HeY^{BE8!$!M^|xpjk6nyy2Yqg6&BxKd*HYHFa?^!>UIQNqWZlQ+ei1-rr!qmOU%t$!~|jiCQ8DiZHMDCU2(4FaT-g^03Z#=Eo4`ae_) zQGNN$!o=O>f9W7-v$_l03wO5*C&&LKh&(x4nCM?TzMzyp`4`kdw?1O)o#DCqkKKVr zceb>>d5+Tk@jpTd{u%Td>ie&{Mv*2vsb8}R@eRK@-1M+R#zwB?3S7DQI`%xYwrYaf z#&NejKYqM9yA;6YJ-`3`*Ls7>ZD4qHN?ylvh6D_y(=7Gl{L&K@sVQR?VI zH68!kC>eo;Q&Rs0kA>rfm_~$p>VQvX=HxT9%oyG~Jc6M=C?EfA-ZKduo1u2JD+*7$ zmLK&axc|z0jK+oOtw%8y4A43seJ z`riKjKZQWh(N4$U_wi=v-%9++r@DJx0|)FsC@hEf3$6>BEuXj~dH)}--ZHAKE?V0T zUZ7}kiWT<k|bX4P|$r~+hu?Hzx71j7Y}&|?*8ZH z;*R3sNX_L6n>oM#Yd{Jx3X=Q{06*Ph`~8por{EdY&}WRm8&O|hc)(v9QRgA?T@YdvGHvsj?*t$wz*VL5MV*?>veARB|6g2B_Wo z*LDZ+JN%2}K|)_i8iM@WTgWwP^I|J!LGS+=%_u?R+W+l%p&B1FLFSnt;m!RhKskMO zXyEO+Ky%;2(|bVKJ1)T8^~I&=bvJ-^WmQ1T7akw!0}~qI^)oNbyg% zul}Ax*KoT@AK%%6%_i5`*%&jkxlSM=EGh==%6azVXb;RV_bf??QFLMyqhjgff7VA~ z7eUqz^}J$!YVxCOfwy>VTT$*G&ithD7wL&m@$Dxu}fMi~JW&2FCD z)g=0`YN3}3JDI%24RCgKzHj4KUqi1NIjx5icA{*&wQ9EG7s zOlqUD>B*Mbt5N~hO>Sv&O+0xQS^nWTuI_oKk{Pd_^?}^`GZXuMPL*%d_If@Lt)>^1 zHYTcf_)nLlwI?0iP73@!qpieNlT95Kalu~Z>Q?KQCfrp|#0v)vQ^CZL>2RRPo?qgO zp;?Pw3s>M6-&m{69uCOb?7!I!P}!^@y6*6)S%zj=_otH|{l-pRQPD>6)tm(mpS3a3 z7CwNq!S&jz&-b$Sy{!= zrQI9NDExieO50Xu{+A&mSRBajoHeFdCG6#$Fg?5|if&*%Gt?Lm^UfRLd~;O;k`%G4 zsZ}?zyC@?;s8VRQOoFlYA2^=u-gAp|g2wqP;?2iW9LPUF4by+#H5)qpC_mpuaD&`- zR^2lE0TD1Kx)MWWC*EZv=cb|t9-HxO0Tr@_YYrjBgFI1;FHPaJkBZ{_!2y1g-GYA8 zxd>3MBT&q3;Rlz9gM9%)>kr!^4xsl*Ji5o1Pv!NebmDegzIt^FHoMY2-?T359|v-Q zTV%2y(cgqp=bQnM=5<T#HXo1JfIT;1EE4ZHLjruOVr1|B9x8%;Ec0t7sbOSfJ)Xt1nYa7 zoO0p}JvHE1g^Re1t3ngE5B1}`*2fc7z(1aEH=oqD-Q7Bk_wV zL*&yK5-a2SsHz7SOG^1xSzKO|_#%*FXw|bEtto!xDUO-+;#2wCDz%@@{lTCNp^k`E zAeD=@lIE7Ix7eMRT84y}VKqsuN(OGVY~`0w{bJQ{?T@%dWPYGfPl)lFF^!35r2JLK zm(?ySGu4kwXN6hABKBWd+NB#nOLCU4# ztBA>>0By+o2-2-(_w-uYcXH|>*hnn)!1a-}xD`&Fb@2Tln@s+=nVy}7yZuc&qI|bz z*q+WXv6Rx@X-FEB7`P+-axwgce3F`2+P!()M&zOJQ`zy4IV<&%C7Vq0V=T=e%9!LS z=>>HeZeWctU&~cfm^gAfPr8vIDf#e$oVkOM3t^Qi!&3&SZmnbscUtTB=*N#iSHa1y z)+D4R%r=ke=5BD2GJ<)YRA&n9xYdR&(_H*wAjEEdM`ey?h9rFUY1t> z2818^`H3l$YDQjdWsIFA=UAs)(3{<|C+AIh8da8iA+qsWQ>M9xT$MM5+}EZ}9)5mg zqpxgj5$ti1S#2>B8S^cca&{mH!V*>%AUNw%{YFV7_?F0RYIT`07UB>i8j>rC#@6bcr7_-a-s)i;kqb+=d#O^y3CL_ zF+O>nbKVF~$eBEaFK8ygj$(_GujHJALiqW6wF$^fmO}yk`ZT2N0y5$ZIl@Gs_4W1o z)AARyx2*Y@R5~cwpNmxY$fOUaeI**Aw-a!#tWKF3*n)_A8CbG2a`QlzhHA-M`9|D> zY{qr`#==>Frw*6)9@u_SY{2J3E0{`}0zhf>M5h-X0rX!tzfMcHwb?Fruy2n!P?C zM|U#hyAgl*Zj(XQRkb*_i`$dqLyA+blKU+~MJIOMG>7FqLFkvbD?uh0p*Q?5qJz~&S83&T*v ze4eB2d}=>XaAH<$j;P^mS?fURbBcCkH6wfQ0eJMXk+kcKa zqCH~#N#_{BTZY=t?I7fZu9PGy(`YhU@py-swktpPfg(=`0m{c}mVHmpQoYg|0-E`m zrQ?bV>NZcuQYxKM_vLK4NniQM&+8Ijl3QlyKwt`!WRR0IHU#~C4~-!ir}>WaT&>8& zRB=*lb)lMci@M6s`$=bCTcr&&;zy-TB?#1PpK<9M-ovd`5a;~9;Zmz9IWF>RQe%Og zR#Gspr8|nmwmDcC7MLm&y`z#{RNcH7sWOu5Q!228V(s8A$NTTLwcC=5T`Dfo+zFy;k50G(bP&T z&d1NNAoUVRL$RQ;!k-zYU9w|j$PZ_)Zf~&#YF4F>JzjhoVR$E`awPOJLHG1*TSvon zZTPWRT!;@$34w}hy8b65u9>BF#$0N#Hz~)i_Kr8QN@4=6t~EZ*5QVz>ykL|vwG1`p zg9HP(rG5j3a*+4Rd%QUC2)Qgf@hls3+=v}y?c=J5 zTb7kuG?mMmmo(PJf$Xt6JbWS#`PcN}G`1X8x+;s5M%!GjUwU_F1IhDgj zINIXPynhZQM(TSc%SReXU+a8&iLY8#S^Q}=n2?B)S6QA@qo+M><@A91LaSt`LhHxe ztAsm;VzQJ)@zEESt&Ft@^!wB$^vE?Xu+X8pQk1)u%(5(!U#PUOb}@$|OeMbjnrgw| z?Y)ya3ekXUS^Sbrt`DwujEiXeeNfji^pS^RcPRAs85@DHx?Vy&8TbQV#Wl2c94Z9s zW)QFXERFtdkaEX+efSfi^{ugAi@?L*Xgi>%=b~TZe`3aszt~rVOUJ_xWTCoxQGM{Y z$3XP8&wwP4Z>^>KCl3<<(u)T~dE-L-8leh6&h}_^b6Iii9)jLJwf3UkKA~JVLt8fB z0RTYj`i*mNSqlLD5uq3|4Q78lK>}o(4!I1axRwT|1K4^|jgRXQ%87a*K!{TFK-ay| zk?5W2BA^n0(szu2#*i*wNpk%L#MA#xP!Qz?r$2SNor^#6&=Mm8S)6+#$E)^sNpNJ) zI{@Ver#JBXLw}Ep_MKP&=iq{<>lyvdp4znpG=#$E;vAk0ucz^V9!>zzMBQCmHgf^+ z8OIy|sK632G8#9r+PY}u!r2WV*>MSKJrHcJQm41P_<@|y`6z#NN%9jud)`L*1DVr& zr~4jh{NdpUiD`SG7ft57?+Iu3!uCS+B_$Kf_Q=@Y|;_#wsUa~1K`UI zFRyjFI#-)tV`QHrWF^Ho6JO2iHeF?00)5-y*&)wZnS*)XV65deOWOhfvy}2TrcdL+J~>s01#84-%%{JzL!X`oFN+O~n5i_0Ku6_t0Auc)KHqj0@5kA7%W)z)Y z;(&R!wx4%y9aUa{U(P*qL?6GaDqUUbUOzC^xB^}I4TB|j<6PN1^O(5e<|S?Otd9!x zmd6rJCs`_YuqLHC+_d(H<&-BHJg_!PT=f`imq+aDbSN^FGWfp{Ex!BlCJLPkH#Nv< z=`{Iuk4GfqkSR*`J$s?CDX-V#?qT0j=MHSPvT{CkA7XFFj(9njyPsOEvVQq-Ml3ST z!MQcnT~}iy_tK5Q+PG11v@9?kDN&QILrhV%3x0w;VvOpbxPs3U9$;5Bs#A0SUX5&U zO~s3e&%csnEO~oxuNOfUL;OXIlkNu$?cgo{22`nC)-I!nY=1gazI90ASpg?!;W@bXu>!OKdOfGNLogwe#hEDoN#~@qqU0hGaTS1Zf4LDeQfJLceq(L3Qq89$u0Y*ALIJP&sX?$03#vP ziiZJ!(NFlpNeF%L8WVmyei-$}Y#tGu(sBIhQe2PIl)6c-W)xsI#0NFGxxTkeEzS-m zwr6Q>YC{XH%8hjd3Ygh^+xSC+M%Nty>*b{zIlQjV%D^O@Sr3{1o`GpN(PYAoCI>`~ zVL2kUw%E;{&MYkrI$An6UzE=J3@Poynfh{M-f|c=3UwIOk%GA>|3|I)q{P5TkwD+C zL?6trf9{`S`ue?1@#C>LgoekZiBkYb0a+zH2>-}G^}7GlR%TO0)^sr|yMwIY0q6|{ z(8ONqgMgrj;?Bf1M;2>Vl%4Q-zfm=2Fr`gpLII7h)5toB1&FALZt5pIuUELzbMINf|D(9mw%-Fbs9UnH8_9Ip9 zv<|h>)}OZ2HP%ZaXFIS3HAWtl3+19`s=BkQFg@t_1sJqhC0_m9x=W3cyyWU`{Ia@> z*Ob}pw3Kdlv6XM+kQq7Aw;go|j-`kYNkMrOZPeckTjhqrF zvt5Rz(NQ370?~9;Ks+xDf+u00SwBeEBKD@-W#R081Q>tB{lsD`oyeMoaj;}rU06Er_!i^U(zbUdGX8N>$4SPzC; zh~}zKV$E|m~%c)nUT856L(Brs%WiRRRF6$nI^kZ_*Tq&-fBU`P5*e+(XzZ$)yGirJN- zqjRBTU~EG6q*omYLEW9N*92DAh5uC%9uz~^3}2pn0cgHYP7O0oF2=7cmqn?5o9F|4 zrr7=J`SI$8#Il}NQmIIP>NU-R71(xeyI!2P7n8T5MLSy}HG>ktM_6(|#=@ z5@(6dqA1+?;HK<>aMsUmOPmFAT8xS;@!A4hbje2gE?0uaVfL$Rss}m7thY)^_cd@) zf3X2%AE2|Nco5+!nKGZh#)Q6G8^03aJR(Wa*(LPOkoG7=(@?uP>_mrVEtJ7H+!*oY zVV|X=N$GqTcb4SFqm!MbBYGqxwc}Y&h74VJIb`5wdY(xYKONQi-ySxG z8D6;Qi?}A$xgRkbs}PQoD3L>N%bFm9p&+Hz)q#YA#)rEJvKY*<1=ZAME|pu{wgzC1 zQKeBd(>0rc0n^WGhI43roYp4PwL9%$0`@b=z{d6Y=>la<+SklX&s|h5MwOOduLhJ^ zHRwY~%h(<}&$rs?iilt0&y%^k<$lB7X1AERDuvsBnz}boD&ymX1+TRm7nb2w8*^?8pP{mD`|-X3 zE3pK-e-(!TW>mxOMN|MNCfw*I#QY}Q;&M{k(fCW(qv3&yc6(r9^I9tf26vAQ{fuS} z-sx#XB{%3u^sv#pl!%nd74Hbo&_qz{^@dACit(`;bAA^@ANeUUl{0&*s_F9Wn*7>? zWQ@9V%d@sWaQu#XTB-06eE-JRY^4oqua^^CIJ(P)asoZf{;oK)Ol7qDv&m-dvrRh) zvZVq;=e6wUZxR?AUQ!kWQ6`D66y+ic4*#)dT39XB{cI}nz}2X=<%rT5&75dY%r{#r zQ;1NT3@71??U;f;d-=TQ`T4TmK4n64Kw4p7tyQO?ONjNOxSU8D%cUHmS!SLl+RKmo z5<+4nZ)h65kCzUiVrgw))~pGdAfJ{K2tHDBVQsbOS-)}AoFxPxFIRcf79TrDY~Q;i zq-jj0jMB8b#tJTpcMLthGEo!bv6+tyxW-G>nvZsAjD7a0Wyno!Ke9V_!ngT#ArHZh z^mG&IF56r~O-k#ziHzg1e;VEvgDuO|(oYIf+mUfwSf)B{iOe-HMCv1-|dk$FS2S(w!d751tpU{RtPJ6Wh z)udB4H9H%{HR)+FF}g!D?xazp?A2y=;J+vvbAXISbbyh;?|dlnygv-V4aNA${6>=7 zBw=|6@Yg;hoZ;h6u-cP8CgykV+as~aJkcNYe;$QSE#+qrd;y8JekXL>)d|&)X6Xls z0JP;Xs5T@KpiWbpi!4q7Uxr6oe*h|8-ugZDyhQGB`w=7!=-Ii0Z-xA0fNHr)W-y$# zwYug}`~UbGkl`>pC)8~G2E_^oj&t!?W@WUN=9^FpJU-~MAH)Tw)Kgv~)$3^cyAgpsqe9Hjkzd?Dw6aXhD31cfuLWmySer+@`WYNz-7zy(-aTrj1apQyHV70QX2 zy;Bu({lJ@6^@oZ5Cj1AVB1tR$seg(mh7*fm68>8v{cb{ahXo#x=EDYbs%}9*B z>kz{jZ#Pllyh|yI=-vwd-Wk}s`tFhMg66+U5&swWc-v`0%2OS$!G2y6Bd7?{h^lyj z$XAuB{@yEeqSQ=QhyvZbm*6u^k`SvJO1D zaMi@0uT`&L{<2iKbH*k7vbKyTSKEHc5^`H`C!4zXUO2j%ssi1F_AHI7R`*lb;z$3m zL%n03MAUi2wjLI*Tb=x)o(%D_K{Xr3}{GTX>akn{QtAiCJ+>{Qrd z!?vc2m`$x_1xlbDe`{Y^Tygkmyu8q|LsbAq)O{KxG~#k)&XZbIbx&Z(2fU=IEaf1O zfnk=(5rx^{a`;GybShzW)TRMSbFGt(mH?*&a@XZLrWuk$)^HtkD1MD6mUDHHOS%x* z%=sYF24-Or)pzhpy5Kvc2_r7 z(k_y|At(2q1W(PfLzY;xCco;ObPwa2&vrca;%OPFBXKd#h4|}pPUlHr1aqDegVcd{ zNj`p0eWo|i9pvPmFX{B(K>Ve8I{VtBa61@^Q6%r~fUQT27Dr4qr-Tzwoc znk(QT&%55eBg>D)s$rR&`e|)b^rl6pX+>JGRmnm85FD`rs|t0My$`*8+JwHOIAVAiYOYgfZ zm3csW+$-<*cChigCqll;^X_Rl{GCn13p^m8n>)Z#dI za2bN8y(}4y75UVA7w{;3Bi0f3)%$Pr_3GP6a@S?2h>u=3YZNe`G(-VVV5Cpn1VNrM zj7zMxF02IGEs^!8vW#^L$eN_=czR)b371W{AMdLr?ur$w+yHMpA~>;}uyU(Sn4H0l(L zDFR*PS(bijXD=9g?l9Z=@g4GPb15*f73&LuO@-2~rl=Q*=;pG;DHkH`i0*?Kd$@nNP5-wNp#gStDd%61fYNi3P>sH=^q&PB;Yl0aMB|^wP5T5NZbo$ybG6 z=3T-=N9{idij_DtsUMDT&i(ogI0m9d&A&i&mahd2&G>Su*dJRuE4j^x=WNNxK%{BT z%s9Y7dS(~rh|CSN2;;*UTin;qxTcFR6)g7Y5UV%k0|rhwvck5DM$vRGOv6BaPV-iF zy~n`$<5=H8dTAY?$Ct|v0j%?)Tm4AMo7zX$| z$i7_-GWKr5-w#nQ^UkOuWa-kpV*WS0);h2G(nIN9$#Es2u^4{R)Z|tiKTih^GxlOd z=@!x_9b7~^WCkM;MsrS&dpE&2y96`PMt)V&gPo-yb)Nh5qaIgrqjSU8wGTQhmvlvC zGIq{&gyHt>b`17fllWdj+pv~J4V;R7O*F>Nj2D#^uSLom))enCMpfD1*v#uaaaoZQ zja5R+I0GZaELDtHdiPqzgoGh+jI!*&56P)}`kGoBzZ8t=ZF*>`c0DBwfrsJs`S)E z&?f$_QX2Aaz>~b3A$ye0m&9;JDQ)E#xd3P1K?wx{$Ey^ghv(7Etm+6C$08lSm}H%# z^A?K{RUI62KJv966`wwtW8%_97{9;IPkG(PDYhS*dOzBqeo4%%8)j)L4zsDUrr``n zO7YTli{dst|C-xp;~&%SfJJ-g_~Pc{Im~r8CSZ?e!_?xEL%T2`lIMqJf8aYhWp`u! zL5rd&yr#)`h;Tt3L4T3q7q!B$G!t3M5S$2h(Oln3CZ>h|RBt@iyL4^FAvTJ!r;)r1 zOTNz!)7?#-V?O`*?x1GM@Fky-*=ScoifKHYx&QPfD-y+YD%DX@$&i&Q$;}?@KiHAC z$x_|KX^!2|t$dMK<0MaAdm~B&WyBe%Vb+hION^-LQbO@1C$ee|of*O`Psx{Og5)_ostXXUf)1;fx_?PsAaHz@##! z-8woJG{I@U*7Z^kFMPB;dV!SthK6;Q;%d#oQKP1y67SCAV>rYFC`5 za#lKq;1v&-pH%fOOU}e6yb+|9yTh`3Jhob!NngHZNMtJpP4wKTzjj(&vRvL|RQ8dj zP;=Pzwu%1^l94vGAH5yP7JrLIRG{q2s3jeKY;y#?qJYwlbJ#N}BRt>bego3{#C7}j z@Lw(Rj#QneAhv;YuHW3bT1{sU_TIbD)w~BK;$^E}ZXXFPJJ&r2w!3t7NyKzJA~S%= z+Q`|UEXRK7%vH-ZBShJ|IL&RP#`i6Zt&D-LCb$CXk?=jR{7$%|OZcAts-ksBq2#%x z9Mr4SKt}Gia;N52D(J4fUQ42V~S=E7DmapA67O=4YT)2~|(UadwGzE4ja^A_s% zJ@+mp^0inRm?CfH6O*gIjD{+`S`b(gR1)y@v6t_f{7<3CA@ds~mb(t2G#N_fvqnQS zQ3Su~i;@)Y6q$~%DV;NIYK*C70`;%I=44lz6*sxEn}Bf7q*7Cmkc)-uC+EUYlewF_ z4g*iyr>cq4n+5LnTK4huR~mS3xp5ELbMAv$Rs6Pqi408H8umVf6wYt-uo$;J(LTCH zY^D7ND&$ZyL74CTq$VtGnvi*R_^QL`W>JK)lJ~l-Ey3%q`G@%Um2`E`&99%jx^=Cr zHBz&6ksG;3hxoa&R2j#Fag6 zYY|dGct9x=gA^iNBfe&8_J-CNhNPjhp2ris5PbSCQ2Hnh{qL_w!!t~V6RBeu;BTZ; zD|IdzdDZ(j)?t};b@bV3{R7_r-@%QY;0)Iiq`%?Fpj_9Z5d1*>^550WO1FV{7Z}Hn z1ATvo=}3=6sd`%AwY`5M2%Ta)E$7fzLo>%cD0#Wa{C|jaFY*l+j^3Za%_rbNPw(*@ zbJ2#zu;h-rzsw)Ha!YW-k+%MHK!+s--bmI2Jao(cctt{Oa^$2a8G!ni?N8D@6uhv1 z_x2S)-|#nJa|Vg+kD~%6E#*^B-@S(=;w@FZCrvDp5?gzm%tiXc^P{8SaKQ;VMi_L^ zxh{%#4FGk5-{en0~J-!)++4{I>k$@xpl!;rJ~w*g`U zhQ82<)wV@=oTfSanPql|2Yo}9nPhX+aG^?tlK4mlUDOvsOu6!qPG%4qNXV-$(` zJx~In-zBcNFefv!Zd~}3#z^1(1GaPZcN7bhte#O_BiVxV$CiR9o2n+*thW1qKbhNv zq5HT0d?Vf{y>!7jbbJ!^2uTbPq#C?W7^<(^ObWqjLL9*J*pFS=vvHtEh_3b~oCCF; zmU+?@pPrlliRT)B14z$%HSynXJmuvFQ{ow;PM0ZOMGC%JIdJtqy>a5g$oOZTLCdtT z=+Vmj@q8y&K#~L11M&C(#g;2cWm8yO8)7nq>xHyuXlXAd9MqfBvacw{EL8y17kX+w5Ubp&*g$KkB(`dE28=aN*Y~@1?K?N_Yscq#sCl@Ms00i>%;Xu&IuBo*pceh z3LNml<%+Zy6(3IspdZw_8SjsfEBxJo01}4V$KL>jJn#wDpB;$-`+k0Y_HaT?5vS3p zjcOEJD(aiE@(wSf1lgnfGFEsMAi%7I5qNA4TR7P~@0Ft=a6RI=x;MRidx5e@*z{JB zr;%HiL1k1sp9ue{>f7dNsfx|KZu(iZP?bocG_{eylJva}bR_8EQQg$em$k?%~c#~k^DS;AVGZc@hCSo2K*FB4<0whpug zc%);EtaK)kjzQ>WuLG7A9_{Vqi0cm4YE&xU6mHrX@4|?A^qb7xr*KHTKFxJg8I+>?YGAjN_?F}U-k2|-yX1x3)Oh;jG3-*$n6JwFY63;( z=g9O-d|PK3qZ+H#k+u>j)u&xk=h*+6K1UPi_)>VAe!F#Wsnu`aG1m8WY;(el!oi_; zf&OEbFTrx9asvA+f9=bx45W+^Jpa)luO3k-P%#AKs_5R$R?vd5PhcEyp? z(q1ftauX0E|LcePJ;N7DNt4+eZJPF|Wg7uYKLlo$8kd#}7Shitt881{KaWn^*RNP< zQJ1$fXlG|_x*}&a^Zl1ThdFqGzw23+4Pm9zt0H4;@N+i5IE%|I8W5;-61t5`n8@ur zKT9{Nu)*+~Quc~gA%6i8{>)o6yNJU>4(ZrWm@CrT1L@l_yzuNz+=V7D$1j#m4a6DV z{!$njKBBhAVORJKh>YZ=4)`*0{%cwck9=XHX>S@fl^e(+U;xaGI_9ir4C5NsjWlvr zD;Za^wWhV4$+6L}`F?dKz0R3h3=Z_pW$eF{eofxWwiDryMHIy;cKp~-iOTs#%E7`a zu#2C?UFvz8!l(I=;&5v!GoqRN<+B_wQu~|Svgq>iyTrP6C*e;c8cb~?iRV~a&K!V} zO9NE~mN-DDSIhJ;h)`?eKbO>URaWnAg)9Cux6`>YRZdl18Xu%!x$?FtDNx#a8X(E> zJszQtf@EJn$TPjiR-q;7Ez`y&nr9wO#&}x6HZ8X9X<(5UN`vRKzleUhEA)fR=PkCK zDzy+GjuF$+Z6(z3>iYK zl&6$)r0WSv)S2EtnBJEB1wD;@8f8U@NLx-Jc6h9eLqYub zVdzutAF4;_1?1+|=|S^%2iW0kdiPZ#7OBxK&SEa9PboBfbi|#}C_A6`Ue(jN3dv|@ z8BLpNB8gHaR^sxT3?LU+Bqj>d6Wy_@2A!{BcScsTWyqlX(1cSyDC7ic>@hsRjZ!~N zimbk3-*H`Nd@U zttDSqLzWR}M!kl?g;=(%oM49oLi;{~YPqZljurHR-U2y%lq75%;%WMBb24BcdsJ+b z*LH%$F^_Vn(YLjhN^DeHOQJM$mnZokm_#(_YWgRT5h^8bk9|tR22I4*#w{*eVn|)v znNCg0rIEQK8>GwiNSGHrp;qeHQEwD*tg#tGWRfjs!d)6cRqb-4vwPt#)inj;yV6*t zOr%G4K`)eXnpp|i;&5m=>m=xEn=DSI2P# z>p2ChW?6jGM&kZ?BXH}aSCVwG)DYw)w@xEqJSsUuTRh(Vd~U+FuBoU<7ZdSwnqLR~)pW_44}tH>(8fAbju4?tSzOVCQOG z6Dtg_i9UFwih{aUh0N{UWo+1w-iv^o6sY^7F#~=4QY} z>zt3s0}j~Lq8Z2w%Pz_?S;K@VzIut^4)zL%M~&?fjqMu6)9_H z%QG)K>fQjM!;z^m0Hw}(p6}dsoE_OKD-6cjHqpwmvtMn=H|w-*q9InImqU^H*S1%} zUM;N$j|U6)(jMZN3*~7KU!`}03`Vt7K18`yOuP3OeP<(}vUtM@46%`Si%dM+9Q8)9 zALjD6v^G9>X}U}d9Oo78qli+OUV!{W-5PAEB19AFt3G>`q@{We@QsSg{PZF_PeVVX zpDy3yoQ;q5^kWIAgw5vmgr|jBN_X@n-VGRs^UC>^s%NO=9uub(~nr-L;*W0Gs1Jj6##Kswc;*!D{ z>!!K+`zVid3vymja3_Tn@Y`<@d9Mr^QSV0-qK*>D+%P$l>tJy_BH9^6%^LxjIEwZe zM3~Bs3=GzULxMHGM!|Y!pSC^IY?kJ#8qY5e7h2Y@vei2zT*Ui3i z-3?~eMa@s`Fr`QLNff^i-Wx7OrrDA;7;?khyx}zYcjogo&WQ6N3~p<#-sG=!z-ho< zY^&1NT&eBDlJ=?EMXSJ~fY2kV;nS0wn$jy6$uKGO!03&Ner@uFD6U9ETn1NrOFTF4 z^|Ds$tyX%&zC!2Q3qY~qmxc4z{T$1|PiZ=_MFbQ$R{|t}<&CEx%gbxbKL+>22|fEg zHOTxaTDglS;jNC(QDy#5HSkX%@Q(-m|L+CcWB)gzl$ia{qx%GRe~0v^s%U=q82E=J z^*`4lf4TmnKKwKCRtSKIm;MBMB!@#Iuz>LH-+)Tg)Z$FbFO-wB9NfWEw@;lpsE&Ue z`IwOiM&w%BR#Mt}m==B*u0mCz?~n%HFl4*nO)OkqZaSYJk+fbv`kGC40{~{`h!7{Q z_tyKeXMfZQp!nvv3cExl7(Y4qM zz&=3q9H#jO>HjToh9D~)1bw|r`fHaMBBjKfkXO1t`ZtmXl;tCep@FSul;{6O5nh9L zc*jwI_dz`o^fOPB_yqaKgF~0sB=_en49I+yh(~)6+yvk-?>k=I8Lz>PuQmaM=tH}~ z5m)da%ncYk>6{FpZ*1&mc-1PV?| za(>4J9l%-qIiyVbpN@D7QU^wt%6v)TGivQ`KH7BxQHhq(hb z?tQP%&=&#ucrpI*aL((e4A}7?K-gl)+xXk5WrJxFlZ@uF9ZLC-DG0Qz<^MsqnU2=* zXRh0!4y($!E#a$Jw+{S>cAzZh+8Vh?M^{&iJ-UiX>X}D7tk{;YTQ-O^FT1B1UjC`- zQlS)zS**;Z=_j5>TTwFOsQ1C~vh3aJ6W12Dw>YjitKla$XAdi8kT#^0(zk%sVR(tA zt`$4rVX*Z;H_A?%N<1AeSTN1AXlm$M&$hH?qT%AwNxfo~;P&%X)`TfxZpdiyD-jYf zYxmW{c+&^fW&%s+Pp@wJr&TKj928qqTF`Y*)7KUw3`N0SG;uq11yq|~gQ~Y99ep!z z)@Ocb%@&9HhfA!_HXHs-PLIaNd$r&F6HYPCoKgt(isf6z&DMk2 zq~O)2DmkGq`k1?!F98*)4+!#~278uPlL950A&1-phSqBmIU{4`g=S~nF(VYLeYIGqwN%Eb)fg)-3dJx{D|)v>7@ z4#oKWxKc3<_lwz1LL-8o{Yh%dm*_kF8RVHfROa)=d^1%dVb0{#Qz~bn*(5`^l|cNx zt{aCPiKXOSWq)o6%?icN%0&g|-gIsZ(5^A^R5Rp+E+X6aOL0=ABR1R}t&hS5D+-gO zZh8Ak;9v}t#>TrE*lqmz24`LMMWt0cd!hf1^{Q&v22p4MlmU%V$>XbGmDCb$J}tQS z$tu*tf6)}k&nOpB^vmaCmwZ)1d3wM55J8$K>ovjFGHe>Ic$<71_^ z7)ie|OinWs6#(jJdRumA&UK@`FMO@nB-nJV?pG;o%RJL)ZNX&vf(&d(H1d3hNq0<# zST$wTgV<7{f1MG5)&I6E8C_6l+&m zejowYvYX*Q<6-Pj1HXE?*eqqoL{ppGmB`{os<<~ndnVo!-QSUy)A6fyjr@pGcMH>p z455jySIIn9kqlIbQjZgGsh%h^rblv|HkG1Jr!x}z7}4}NyChKCQi8)G0nQs7q&9GR z*BA>gJ5|DQyUf&lf7{hwU2nrI4=3fX6PCh?qpnO@AR{#AU`%UPR(7c+HWG^3!ajzi zhjQ9vT&`3TGIN{bH-4WXa&9*sE8>`sV7{R={XI_ei;C~G*6*+qS$`2D&W4tlVLicJLb{wdSH`soyDq(CDYq3x zzGSC|^dk7RUi;A4(wnhT@^#|YWY~YUueGZ=mq1oR=`(EFLWr)ZE`v1vSlW)5;ERIY z)k-G5)#sOTws$qDf%daKS?h5hIwh^S_=+m^aw10AusJ+QO;~qlFDlql;c`&Nu#RdN zMrsf|)8wjb39ELUNHAmn%>5jWGoq8p;-vhxD%1)>vr9~8WUpM)6gS%hfBTIzVVC*Z z_3Eg@v^G2GHy|p>H#!Jj*39>9C9XIswXC$ZVXen&8f@a6OJscV={!Ci9#7I3jDx1y z+=ijYt|8vMLMMm{!4$=jf3zn~^ZrI~bwI=Ty7F|1Yw@IWh#=IOo({1j7pO{e)fc;Kj(?Y?y>zeHUBkZlC+G_i)-9QU%p+NCcoB|~{6xS9f6btU| z1efBp#l5&wB!u9_ofdcZAjOM&aC*|`dC&g7z4tid{F%W`)?y`Nv3~cQ^P2Uh^So^h z6zm8mvBqhuZK}h;+!`G$)%W#}ApI+L2aZZl&(`fN!5Xm|&9yiN?(xNOOlM+dzgSK* z%Vwa=KWS{4>4PF4$Cv#e*LDI_BJ4lDKD;hvhDa6RG*TFi9yq4GOuKuA*bJ-o=H{Z3WWnYy3_0#m(GO93cYKG3e)Cl+}}J z@k>PZL&;+XAg1+{Dv)P@$U6`aMfxN?BPrE^S6hBiy|uOtKApC&rwn;Z4SzTaod3`; zmy^B2+N~8yMKl*xRhdxGk_g;%A z_<+82m|&)T6H<+Py+6YjV{D6PfR;(FLMqj3ski31<+UFN@-o@*V_$t@{56q~L$LKI z=N}|2o6d_gd-!Z>?5jIbUODGI{h~PB#608`m=bhZrQ+G;YCnHNp+F`X7T=^&BB1Bq zY!X;+E;jJ+bf_;eo^73y$$WVQIi};?IjW$zh!<93fyG7yx=-|Qea;aj6(5sWQLQ!E zfNHIC#nKZSp5LsSNp;v3E2Mi30;iiK_2#R&oUw_(%&UKKbofg<2-|pt608@i31Jt! zp(tS@fR)zd&E+5JTeq^@O)=chwyjfZ-tx-Mi*=taC+TtcDNV%G=#_IQp}@r^4=htg z?*8es#Zx>6nAmIiv>JS*$?)%H>J(AqQb4?F$_4Bf6RS+9-FXvU#w=eri-cxvL(8&d zg@RyjC^>s{X9l0hD#Cx!e%ts=t%8yvuK_Ve9i6V37aR)Y3Acf%#-BL9R|rQhq0%RK z6wVCR#=>bKb^95|hFtya%9?wlN+pVWCRJ^7TyO)DW>mQ<@~@Jv~{q{dt7-PqGdpIk(Od$*je z^CO^@jSGTd`S@rwh7X^w0EvN0+na${=ZwgeNaVvDRR5S=%$ONft8EhNZB)Ifw;db^ z9OGDCDLFg8bEcokg3NpbSW`A$>-+8ptWOf4z2c|r_3zE8fCY>K05P}MpDJ%4JqV9S z0K&qJQ^A!Hmx!@~T*cp~=+-ZnvjYh^L5{@+Rr&+h5e{d(NT9#c~JKk7k0J zQUo1Elkp`0Z{9U5KD7Fn3Qn)1WTE%6NxdRRr|X|kSmR)QpO%)kKr|}P+ZfMtv22fv z?sr5x*EiQUS>=H1?A}2Du?=b69{^X~{VnACV{EU;gFwtLh{?wDMiOdHk;c;>fLO`; zZ?!SEMv;W3A0HhAKAXIMhCcfkt+u=I25qzL-*NB-_(>Y$9ondP?n)i)KNlZ3;W>A3 zdUtefLzLqnZ5y2pgkI$e&XMAV&XIw{sF(i;K4D{Q($t{u5CK z;BWb_OyS3}{FPP#`F$u=O8g|)^+&^XU^M1({hhepLw(j?%V(hKKf2hn31t#-RaEA3 zNc#FN^x`4;2YP3p*U8H0=)Md9IN3|??cn$HoCNjxETWga9y~^0p+lj!DgJ3?577UY zS@rnQ2r*$O;Tf7z(6#@Yrz|%sck}=-o+IsE-Rr+#s>eGdwJi@Rr)>;cP(3YjVcM?8 zj{%Nt%N{7d?6b;6ZO0J+4a4htR2Wnv5c5AV&KiOJnQPbH6#yVkeB+`1P(T3i0;d8o7T|#TqaM$B z8KXaKzq*Ts6#^Jn;;8NZ0DJ+`Ah;zqwaeIJbaM||g-h~5dc!li>2XKi{VI26genjbw*Au_3ld_p&MZk2ER30Tj z+dqK11>o15%z>GYPcQ4t3U32MFV{)W^?2$7EbYR#OfT!DGv#8W>AJS*D)S3A&sLkv zTbU9iMQ)$)>zg9YcTctNzPB2gKiQI8&=VWj#ZwpRoCb0L&(h93jEmlZr%O5un?;du zu5lMR2R5#TuVWc=w23g+hT$4WhJxnGKqhjjiBHGrAdTZ)VPyXTW{@PXVcelf>jcH3 z@98aLiFy6Q(S~AZ-Sj(GFHXjX6e7^iN;FTv+;JSlWD($8+RCFQ6ZO^VsRDM4$1XAV z_th+FY1GAuY%Z+OQmrytEyE|dr0)t+$phvYOm|Ne+pyc97J}(F7W5q z%ne#QS4l>Pvol)7_+q@8$6G0UxMs0K4H+NTR3-+C+CRYMd=DYGAZT; zRJH-@tjHV|{jMl8pOicH8z;v{dTc)s&mh>tiAs0#4+<`fcs5Pc3oS~g)2!WgJK(!z zvU!?TYI)OI^2u)|pZA#bnGh;++Y<6+H=^0Za?xI`vY>q|K?;-XnARt=S74c(E-2Kv zxxS6EkB_1xG5@~N%IfpWB{9KorsL_I@XldoUUu*w{&D5yHL}Mcr=2uMi z$Nm8H{1R_Y#sxWARf=W4ZE3lzS=9YjPjNM1EV$s8NRI61xjVvdJk7RICh9M)TaRR> zyJD8knXg$Qm8ZCwEEf+3?(a7D8uh)B*HPWSWUbPGxE`R627@qDpXI{`cA6$)npF3^ zX*bQJXtz?D-fvh_(b>nd5;T+V^3h7Y;CrGm8B;k@tV?xUasCNOme||t_zdJ)RM)xm zT$r`$Rrjq-a$pU8pYe-=d2ZfXQ!#uiqF@>$)Aty3v!r~Cn|V4j<^K9h!LvVf={ga= zP_@$$1`7Vn%H4Kwn4VuKyPEmZ_x1dD%!`#}q@ z=NtFVcL>=~Di(tL3K-cyS;Vpmwvsl~7cY`o598J6B^`+8g+6zN)3UtmYV!%*QY005 zC{=Qqp5uE%So>6T@iq4^AE%dV%}FY@(|HT*iPObc)X%P7`du8T;BaD<)sr_+#T-JAKIUoQq<^K4xb& z@*u3rK4GQZ%KmPhHliTY)uuNa5y%@u%|M=uewrQ8gUYtG+vrQWb%)M2E=NeUMRkJk5&MCeX}jrEjL94k&gJp+)GR23YCj$B?goibQvfo#3OW$saMRN3H#&%*RK2c0`GfS ze-asrIkT~*M3RNVZ*UqR-N&V$EOzQPL2}GZtQ?K>*oXWhJc?5D=E%O=U+gF-4CNod zUH*8xsc%tc{Y1o;MclZFS6w5!9t%(tsT!bGk85C5(k@lS5G*)?<;qP8Z+8A}@P)>bbkBLi!%0IQX%>0dX_~6Pu zOL|#yU1t_zVW}}ertyInYV@$ZX<(qRX@lv>(nViPo;ONVV@wH6swl8wGA4#ph4Qe>a8*jY)>Qew_DJrz&`m*_e?Prr{E*gUcex3j8=_Bg2S1D)^pZ zNzKc0&HghpSC=89$6^bW@0Ymo%q{SiaBa@3P&;p>fmKVdqP3Suh2i|0{EajY@eosiQNdO> zTh_3Fkr7Ya0FWx=B~N*Xp5vV16P9mmGed0qDtZ4)Xw11!wju_VYliD!=Di8(b|$3r zH5`#zw_+%ImPhX1Z&QA|7lBr4RZ z=r@`tb>-n^ZHSEf52oT&odd$yX;T>!IJwqg1o~nfiC9IWXW76 z{`SMIkm@F#pDf$d>)AIwR6H|lO^ZTZ1ZzaQff4%n_h$->7Sg6!IU#niVwMKT88@KCC1mjao;-dZ55=DE5tyt zF(?=|7gw$|+zK0|<)e6!l=X5kW*_JvvZCDjZ*fwk_Xv_gq{%wozOC_(2(WZ;PSD8u zYHRH-En!a26ig}qQ6OYddcO=|Sa?296R$@fjA2F;Vr#kpjjz(b)D>55jRxmHHX^_5 zviiG=*4fV5b5d8f+q}f*=GVL-&dK9h)J`gGPheKloB+=hPTj^8)0#X}{9N=iIjJe; zGn-GQ1u(?M@~+>L-<_^bG;^s%`2Ncly%ORC@l6C{Zx%*|SIf%6k_p00IIG zk;{6Y!HMN%lroS&DOt;!2jd;`)5-4M=AVo0Y!9HzB!0vXQY>Z2&z;X z?NS9?=uh0{DgTjqo*S$*KxZKN;yh9ye!<9>Z$JYwF1!sa<5X* zMWn@HBLm5iIih+5fNslvYnY~D*`C2tOJ?t!(sKhnFj0P$n0xp1EC}^=w#d828#$`2 zk0glHOnEl^oEZ`2j*6T@sqfA?_DPwimg(?o1Cq&D-QD_8 zoUCQN5v9beeK77i>2o zLX&*QdEe$l4tnBmLc3l)j^x8qC?;WkiX~Cp!lSFEL2}ix}Z6in#SEiLat_KDfxjl(V*V! z%@1vaMdCneJiAJK4QByVb77Y5B$5ZtJ2|}>J$Y2`F>Vk+ppfacjk`vzrCt@(R7=31 zORqsTrP9b+*ZFCqFR0?V;s*-`EnRe72T5xMML2Ga-cyN{D#*?o{j>IaSDfQ|hX&ai z%Ozc4hKz!dmqMD}FD_nm7**blUOJc}e#&~l*#y{c9I{ znd4C1fO1@;^cmDE!Y3DrqV`KgDIxX-$IB^w4G^tATU)=_ zjH>->c@HUNetJ3a!8V15HaO`-cgPP9>+rqS($!& zsQ11gUG1^Cd|E|Zf=YUFgowoTuLAx5Sw!GV)S4=TgLNq8_&ZmOO<0-^fi`u#igBpC%)3#R?EGcq6D44WwBVZMR^#l&KP|iIkT?56;vv&pWIk^?83>LVl zlC9@Q_hEhHo5WAF{r5J-(aa78=_!9VInOc|c^Hies_!B5%OkH)U{SC!mxviCCqK0JAWI0KK^nGinY3qLaAp9(oV?V0|dp+;&XF!;=aY>9xY zTVIDoVCq6*@4DT2jo=uXjll|bdSto+M9!i>AO+$6j6*tsMADyVF17&5f%0aZ zYy>R$r4W=I3nup&c!;)KdM@x;MlI>RfyGj)FnKzJ-C$Y`5p6K@%Ca?efk#7X->G^C zXu|MOChFn$NOQFeJwq%5$WE=~S|~9NSN=z?mz^ytw$|e%=I=v2`FcM#naS~(j^xeV zmioP6()TGNakpu>;4Q6I%8o3PW>SA<2Yz3FCJ zTRdv-Gfr1MWFs}=ah(_AoVPx*xh7v6V=eZzyg*;WKf)6ecvV^lo-AIQO)+at1T5r^ zv!(kEjt@RNnP1~DnE!5^AW0(og63o?jWUf)XHC&CE>r6H&LNw!ih!vshzw z{hR-4Oa*0Y-dz@JWe$GFlqTZQRcymk8-Gp66yL%Xt>TbVQRGMCInxvH)qO9O=NhZ( zaq`0%iT&;;cLlBtI4heGVX0azeOBo4hMX-HR!lzJu~p;+8j!z@X=yB!Va1}kOZ`7y zFTK%8Ql_Lx_Q$4#B8PsbL?JE6Q zxW8u~VD8Fn7GCO4qsCGLE)T8a!^KW(EEW{XM1i41X-aVM)&h27d@P{6#I6ZS1Abnb zuQsrt!y}4&K2I&Mj2lyzqU*hMWt5%-YjDf4@M4@~hECSNK zck3Ql(2gD-#+NEYm2K`Z!05NRWxRwP=4_ zGZ|guTr|esgdNGFAhImFuv&BLaWQH$b?Wkk=~Zqdlc1_<-jc($QTo=$U%W>G44Ey9EVf;>c+IE$R=a+4=W^t|G#A&GN=8nBpZs}&{ThcWnULXiGeH-}Uqf|Pw;eh{6 zVzG{qfc%-Q$V5RX6G1drwZ62a(vHf~E<`$oEQs5+LFrG7iQ z$Z#f_D~$&8G2%coiTGCS zFA3J}qyy(Lq(&ot%pOCSnaz<)0t8Y$Di53^)+u=? z{(b2XieEfYst#dhEB9C16H>>eZVgzgr26UKH^3Tx;&L%oPcULv^idw9nO8$6yq)zDUTvP4UGX%5oBzis?4l+9$9nXqXMuolYGx!on@YD+<9k*K6175~1 zq#Qhfrb~w+-}sm@<|O*Tb{8kXU#02CGkHNLG2@rdFdy|LGc3Fh zJmjDg#{&P2bMF#&Y9h=%}tM-G$@M|gu$x%2@jkWCwU z&R>2tuB@Zc*~QrpdGTkCM`HHWvZ}m2NOfd(_Vk@v9L47(`W@OvF6L;FI7bM=kLHul zoHq8Cs3nrb-$!TFp9c1jyHjQo1$d$d{PSO+p~%B{k2z&^a}M!As>YF@8%8y_62`aB z&m)>Tt=WYQO5q&@CBf=+%`Woo-NmU3>`FhJ_!#f=OeC9noGLF6Yho$=p!fNXw##GI z(9s*rNN?_xRSQUaMA%)@t(~fRH$iD-*5?c@3e3%#+yJ%0J95IEX#EF^ln3}_>dbwk zAYA7yC;oc zG4WD2THCLC$k~cN3aYAzs@y(43y)Dy;6ahnAa|KzwOt-w=JGDOWTC&RZbR3z=#v;( zY^&sLNQCrhP<|-q`_PcWF$w`b5Qq5R6i(BAagT}W->I-ap3 z=R^7)p_x!u@*b|OeRq5k9f-olq+S7!K1igKYtGxQv(t+W$OJx#{%D=-Co;=5P2o$+ z)vD^iTxw@I<8U@nE@HL5$sv1R!R_j&xx**1xA`n_xNI}EZ1e7%t}Z7gmf1ATHV<7P zP&RKX#ZZB*pv)4}hI=ECSJ}%tKVI3$YC>84b-z#O+C-Lx+4LQxelrw|td)YXREdQQ zrilArOS{`@T^Ze#Os^z)Q@N@opW!kXvRb}S{?UlP0w(^x-r>Kgr}Lgkx~>W=-!28$ zP&E`mFs+8Ho;D@ePz&{0+%>D}l{|0g0?v{!J`O6>{H1+GAVR?R`j>;VwlI|RWj~j7 zQD=QXBA;i(&;pw%#wexi=*zXY5Zuh$@hLh%+T=wNeXJ;pzK~xOPq6#(WHq-RU|k8{ z=51W%svL-tw8vmIck2Q8zCFeMcuqbI`#GE>xVFC~-9594HalO1tNIGme}hjC<_$SZ zk&;aA^X=|z+A=eP(&9Q2RSW4gQYZzq1}GyPGOG1E?YP?dU8{bky9ekh6!sxTGcL~1 z!fbWKi+SjQ;f=O%F%~wUQdz?;ymC@bNM2DX6E0`2ZGh>LxF&r_c=tua_nV&9PE!ES zo*=TLBM5h2rt3&S!%8)BN*tApyDm5N)aJr4oruUtQ1A;)@H4?kO3Zp722NL2$y_!O zxBgs&gc?xIM?73+n3&At;h0uiU3wPWZmDIup}74m?uZzHB&5}iW+F^|cc&lwjqd02 zPc9CixL{zk)S-ieysA)mw^X%v3HWRUF(31Y6bX4X4)r6Y;8if>{}}l>L(zc?E!`(D z5+U(QTmG71$rz94wdQhNmxfF$A6uz@>uTQe-tiN#ZVmEu42ozQQkjVP@ z${uPL_nYqueIi;;X=}9(;j$4x>caJLJz+;8jmYm>5?#vc)vQ`(&{=n z(FN+kL21~pBqLrCH4+wQA-EwHnd{YolR(*^3G4_m+trbhZLU4XYR311x}Q|`#LNTF z&KP732x(LiCiBEtk=2G$W{eb;X?}kIRMlLH6Tiv|TcqCKuXEOI3b`u5GB@3sCejPY z%+Ep>E4}w*M>p*sSSs6MkO~P7*W?L_u$l<3l(umC{wBIzHwvFchj* zAgVI$J=B&eJv~V35oBY!x&VobinsY(&TgYRam&=gWfSJ{4@dNK&0E^#F0C0eL&?I+ zp4a8zfXr#ihYI-OtM7ON6KxqFaG#tr^;yhoETswvk%kWSWQVKp_u$NG z#?2kY+v+v)QWVrZ5y~L15H=CFOYH34fc*|=raM4xqr?QoBPp1!6K3D)UoTSFpIt#d zTip`rLXILScY2LZYc7lIC4!AS_}kQqxgNWRKI#L{d9l|>f-DrTB9uctR?LAA@&chu zJI&|Q$2y*rDXz5MYcQBcaJIz?>RI;4nDe6Ny zES%Bfi~>yVvnJvqITRu1QLA*bVh#S}>;NsL*5S-az5o-`6&)=e;>XU{$5ME>;akx4Hv zu-VY6r5hyyq`m*U`!Hu zO&TWxTbVTyiPNY}g4%teXV&p3WD(>*6_YaDdYQa#XKZG?ybLVvLpG?4 z#Ck7)KIRZR>4hU?k0FCEplV3ZF_y{<+r7>@FJb|yT%XB{yQ#?h3;lEalM`NIdK+1w zR}$aAyVi0z2mzs=BV$JSg$`tXD)U!_7VRyge7)ak>t^1A)20Rph~BSUv9;tE#Sm$t z(SLR)S1Zc9deiH*XtCX*PBJwB0ZB~lDg89jgLS)&_)RQ#eQOD|8A;rKDjQLF=s=FX zO*D~Q+q^Gued-#RSK{pSxowf<@#Pdkb(3=ubj0M%kg<7!zt$ue}J4g zyU_WqMnbBd#vyrX+8y$}ccGPMVAt#b=ohm-ZC$TCtna9OQNe^)nPp&bAGr~5BN7{s zk|whURU0)r*Mr~X!vnDAp*G4!NmTKJJDY?Y6U&xqnZaUtogi8wW=b);L)OBdhRln_ zgYkQ|SMdmkK`NU0SH#hgu>11Yad4=GVrf}D;a5qIvZ|NNxhkWaP_a)_R7IM`ziEGa zeW8&s&HN=y)TyDs$|MwwUJyzF?|{%01fhH6PRL*yEcLE(F#vP8531;vl#B;2EVEz( z;R&rbZ2%E^C*h|#C;Cs$U$j~z>vP}`L3|Cv!JGAwBVdoyYqAj5cdHtjvHjAS4$IZWmw_(Zwo?0~ zUmHvkk$5xAu8s(TvQE6kj%DMci|gXPYE3eMzxhE$Y)z@=~qu3+R+VDQ5}+0Z;IcOOokQl*C;7$gWqr4)V-( z^SMQM>OFeDrY+LvMySfm>k=<^3VU#}{zM&_c)nAyX=vblObv})0PSWE>zL$Wrt(a4 z2C^9&sCvo_&{%^H(F4K3F+XY2y`;uL)_cVwPCY^o)%Rouwj15tr$%AkY0FNZWcHsz zTXY}#uc5s5U*56_9X5Jt7Vg+!NR?b_m?(WPE+EgK`<5Q(8aL-2Epv$dYnso#_A2TR zpoX84D&71cwh?LoX$HBAC||0%({Vf^2gisNUgh!d_%gJ&bZbBRU*Zn7)BhVv`@hB- zF{AK@W|T#(fdJ)~Gnz~YEU*r~6EQ=_4=nzhS^IBF)c<+K%+nkYhzSh*|6K6DWvqXG z84JR`@{1bf5j?xbezbI^tqdnL{pHRXL9IhY|Wfb}ylLX2Q#nQqw! z#dfTxwtpP-&~D@l;KT>pL1;mxPmkJ`S{g+HIZ$TKf2fQ4*CQ`e_Ja?Y19=5p%Jj1$ zV#21}<&?A6uOdiS93U>4_Z0DChEZX;=+)*IdR?wU8g;P;j`MljXNjJ#G$)P@GJ7+; zLYUvi?yuzmzpn`~Tu4G`G4wH;;0 z%v|xj86=F05c^_{?3abtDNH1ZGhGA?kUReJd~qz+ke58`Z_5bx&{|?5UWUjMD&NcK^ig|@ zpytvgu8bSUBr{om*2eLrB-MCw7Yw#U+UuBH*;UUUt`FI7X@2WMm9#=(V1zZ4M5B1Y ziWt3$VZqW>j63;za=tcn1*wDVc5`Ze+H;Q(CEig_x8RT@u2&N(tf6)vRA2IO37(1xZjIB6%-mlLOufK(tt@u zbNR&5pRed`{ammmy}!A9ie}yIw>YGJDGPG%q%cEN_L3mK?q8hI*;tmkN-c-_@2Ts} z9)7(xS~Gym&xCO0Yvd9088}N@DbApFkn$W&?07m7bA12~xpw4P2Q>`Uyz~(V2B%fd z6w|CzzP)vauLUiJI=Xlm?6TBQlsOd~-cK)eZ$H}i#DMuQbgAUg5k)xVFOCy^aeOQg z0UpARg!k^rMZJ|VZ-ydOM${>K7PsPZygo6=zQ+|o9P5)@feAh_=S`^NvBMJ+R+Q!% z%9F_0C|gd}I9`;*dZwg|83G-?#pht87~1dRwwJT^OrsvkgU(7!O=COQR%Lwoz-4T3 z%`Wv~*i)~>dkj+rrRHMWC`~KPjmk*ukO+uauu1eZpqtSON}P-fYIJonm??Sj{)say zLL8~DE~;SJe-B4GZG?JHbBALRR_<^ObJ{fNe8FUi<^+Fpr(Wag*c^|GHef~UR-!Ej+UPZ@S2dCci_fKo9>U!vt zrQwTz?1`BuiXDVBB-*M*105I!N|SAx*j^p>|Ky#SUE#lOx#@?S4$1w-z_6&`DP2305RYD^jhwD%YuRQAlzoUTfd&d>RrTM(N}`EjzA5| z+}m26CJW`7bj{%7>L@-VNWO*Hertq=z0Z|X8{ZOMlVjauc`h?FA+_Sz^>4M9*k^B2 zl24QH2lU1_3nhpKapiqhzE#r#)tGF!)`a<9Msw7}7v=LfWs1eIcg{xbkyaDIjokL; z^Y()pQ-l+L?RD z=$8L9#k64g97Y^!CWzUnQ2bGmdn#;52~2z>#tEsLw-)?RoU4Il`&cUt)uPpNUp^^r z@WOcRh~=<;bjrFSdvt}{0H$MsDnnP6y)Jd%?>Si;(HpJ4Q5NPre^PN?!E!*`G#_?5C|RSE9-gFWf# zdCNphx@9H;w6o~{^_@k27Hj6ttfWcAo^BMUX+lSRxnx7mgNs`=qIQIgQ zfc`AkTChBKZ|_%zO<5W+eTh^(rEjQsAl0cthTS6xBe#a{u&Ow>Xg)M%#Y@l*7&6a< z^fYL2;p&@ebf5$q*u=dDC$5oyTxpa$X4R-9ohmu_Ubow)QD=7kFXoSOr#9>_;>>MuhM0%~nvNb1g7OH>!Y7Uo;yjiKNa-S^AXK6mwN<$+##+_>-HNXsCgL&m}7b zoY-*T$d#-VUK^rz$TQV^IFybYL)uJn+ys3ZsWMTpXv zj_^M3in9N>OYB0v=U}d;T*plq zp7V6hrza(tNm;bM7sCM-Z%VX-b+0`nYl4JnEo_g~OFm_Ky7RHl)5bOoIZpSOC4;9D zHkq!@A0DPMP$jW@%6h);+HC2|AER}nHkd7q3Z~NJz};M-wgk2WDmg*5RJm!Vn{+Rn z3O+w_w~lxPSp#-{Tiil0yB>-jzw830o5c#8^$0vDD|-vm&Oo>Hzb4TDH&G$9!Xm74H5qkMdEtsyu;hSzZPGU_1QkLRl5~FHRe-8`drph|0VKbX@|P@;lri8A zmLHBYZ#}%(5SbJ-#Ev1Z`oyNdM^KuWRE5}HH?9Zk)M#Wh92@|Taw0|FAsu#M4=|M~ zd#e1cM$Qsq0>s{~G&&T$nK>@y#iqTAh348+HL(UhoVEY!5wWd@4peZJA^35_uC~Fe z9jggF){jT^S!R}fnW6rJ&x>=5NEs``ftdDjT{a`-7DL-K;@7D$Z_4E&2t(L3BX+8+1<>PxGS%43?F*F?sP)PT+Wcc>n{y z#KLgIym)y`IN(6dQuR2Ij_XzBl?)qua zPaY=aJ`qM1$o`2@;T@6sR~%b|=(Ni~kP6i3X*c~i8-BGfNR2$(NRV)v zWICYUb5FHRxYBJ^08dTzCzb5QHHBUbU2IVOWoiW)&EIhig9k2sGU$Aqj22k*&p>tx zV$+wqyRPJ<`Ufrl@@bhC$sYiRc#$zMc{|~A2 zUzX&5R5!p>d%C8JDuz=gH_$@vT$j~4k->O^evpI3yj zkN@c_c!}ly7vK3D1N4`QiHXkgZ@tOOr?Z+O*nOP;Fb^@&=^R_H#U;-+s8E*7zh~(2 z(wIJpJ`$H}+3?dp7a9GT2(B`DEF5r#Adv0SyN!~je&*^G75wYr6zPJD$qx#KFUG35 z(Jho43|@@MDQzsNPGo#?{$Brb&bljo$tcds%JRjk7p`n_8Np79;YiNF1XM2e+Eyuo zJDKvefEoT)eKqzQQ6I{SjMwCi?BR@l^2xG2H9I!-i@K_b6)4p8V3%g-`o^Y3{#Pf{_(#~wuEl_D>SC& z--DrjD%dd-p9kW`p5o-cci>Tnb1!}e7Z2l$DeVnf&r{4n^SSj#RZ&MJx@g+B4<#gb zkGG4@?XhUf*;KzRfrEoVZZ2!K7-;h9K|OSmwIs5kE!#S*eBd>FA+>NOo&}(WH^fJ| z>H0?IB-0%-4wq@VO=!Ey9(~TiO^Rb1I>S1lo59_7!ARz_I8>w*7H7()d7K~76W2wv zMwEEPp$_Dn9mp>L3c=s1`WN8t9f`m99nYAw(6GN4Zo0_cx8y?=0c*S}C=FKoU2k~W zRi1xdm$xX>x|4dg>)u2Lt3=)DqkhVktL2Tr7)w(0Gk^;?rWxuPLPYYlAKJwz1DLeV zFm*D5z1=^0df$SgFY$81-R@YXJ|1V|Q-T5qKw@#Ze*j}ct?oYl{iv;27Er)Q;`4&b z+sTr0?(qQcYm?%8S0ux*0h@!S|IS|juCiPT_2u`W#2HFyE2j6CA!|)N0fWbyw^MRX z+j5~;VootBE`EBKwa-GCc|@kwQ{t_gvz{mkO@tSPar&B^DOK-6-IU1rUv6Cc?H6YE zB@gAi=-biHKn=DAB71)$CyoOC#J}5Fnlz*El2^OEt;9Uv-FoVC56Ljafk<*w%mhBU!<%^U8Ie;wePJst zkwsaD&t4~(*<}t`q&3%54SH2o^TF?3*R9Mk`U#81;qPQ&<<&IW7{W?lG%|XDSa3mh zRO>GZThF91PaIl}Ym((y!&2Rwa*5xGS?Q(cq|moz&gK^K!W@)=u$pt5iEyod;vTlpaF!{`&L%@W>)Hxlp z3H+$QMsCv8#g@Tn+OAx5GS;{#IkoEGBsM$uvap`h=3Rn<`}e!>SSk<+PU%;-Mk4LV z0!nEhfN?#FMdNn|XWO7v#S6+Hk6ps$XuJ=N+^8sR=O?P?3uQ9q!3+*>sfaN0)WaRi z*vhOe-$dQqz5PWWM6TN^F4Xwl_Vjdm`U}a+8|91;P-NZ?v&l(9@T*_Qhntn-nDQpI z8LF2ZCsp$t^VdaV^F#M~-fVQS)SY^Qx>)h)Ss;l=@R*3jK3uU=LfXPiZ!x3XX66bm zTv?7ms31YER6zPfn(?&)<CR!W6?yea^sx8V+#JqaDhvHFr-MU*5jMUsHdn`CMCm z=F=U=ndW`b#UFr0(~eT*T|beG*GTK5-4&G^-)#6y)zE&#@e-n+qep=qDi6^ypTA>cRW2qPr`kIu(!`1x;-alN`y3lYJMrIf_saTBdVoo%Z z1%pCJH!zx7RTo|bXG!Zx5hQRIBU5iK2%$jXMWhvc{rd0Y0`o1X4zi_>OD z3Ni&`(5=uWRRIjTe6LiXG0FCVT7p)ni)hL~#a`N_)U;maZ7myHYMxzHP^BqJQ~}#p z-{cBn*FJYqJWt_od==+=^aeI7=O(+^?zd)_Wh~MYqsLA17hvxPyGx_=xQkKdZDtFZ zpl*NR?~5UIj}u$6Sg7#jG+QMG>Jo}CvFe30Oi$9|3ub$`L_CRFJflxor^a(%^B!GZ zI@XccQ6{~GpEnhO*Tn|Es=&aox^c_4Ct@sfXr@osI=aXK^xbvtda+t~Fnpc0Xf+c1 zlv``pl88I_An#;x4ZOCrYas983a0vN2;L}b>LceO67fA~GOCgiGE;!% znx+%+QTm$D6~L< z0>!mZ+`SZcNbm%AcP~zn7T4hJ5Q00wiWDgh#fujx?!}>SC%yMSGi%nGZ?Tn}WSyLI z-u>HqKi6SNz7#6Fh5IcxrI&pHBp{EXRgY@Iyl#P_wAz%r$|gghhJOHhyx{>6iU-b+ zYPrxpDc^fEZUCk zhSau9!l-h?(pthaZh7Ncka)5}@V!OF#_vhQ3%F>aokFX1<8k)a!=3Jz zyKw^^Lwa|Uiy})lKh|Ja;ueM49~d=?M^$9lcB-lci&2|Xt?Ev{43=3Lj;bn_X;{`B zpbee=ETGlp2RrM@y@q~j;g${H+`tgKZ%Xs@?&GsFzFN%WAa z-&;PR%#0uT?m>&x#)(a)_CtQ2o34Cugcky2nIbgWp0Q@nK6<)?z)4D3Af7qw^9OOi zFtxOYPL9bohj)-Ef%%yiB1$n+qKhC{U3jm9-n6xI)zDtJ{>=g=x9#ji=T7_dB1a~- zuBTaI-TMc*qHN|Lovq+_4h!b?ea`QRiTRsFKKPlLDctdKz$Yk~`&YE35TAku*fhpe zd=!i- zMx=kiYpCYA)VX0quKOcvOA4#oaci>Ti z#E@R_j>o+8cByJ7MXpFvriz?IB7Sx*lN8vwg1&4lN9o}ainFJD>G5wAnpPe++#EBn_<23OP{?T)7?XNfwTRJ{YsAp#bS$r*cyc>7Zj6a zTU8&5+A(K}P>hbQ#6MT}`jq(%pDt>GJJf8-Hpuj8o~75z{xEU2vN%SD4%xf=MWcy< zz&Siwk43Xt@IroyMJh=Zhxj^0zQRFUAM9_J)NOMDDjL#xD!!5q2$-jCi%X!avC1hI zes-?&b41?tK#mjFqf{kCz_EifhOgklLM=YRqB+yFy=fH5L`?CBnbIs=`M}=JsHT}| z3w!o9ilYAOQEWVB$h1-9j-?f9{8`_RH95<~*Dkh;e*3^L553yT{vvvi$>xObN#j$S z+0M$mzKhwKNl4Y#;!3cEfSy5>5muGylCHMpFMb)HQ|1x6?PqTcF}pXS8e};u=}M>4 zpVF$)v6*eGv9*g>4b*&QSjiIY4W*+O+6%F@o1c^1hsxe5i;012^E-*74-Rcc+B5aK z&g&4@1r^kB4#`jBzugvS_q=xJ^KzArmCIlfry$?`xdr68N9(=t!Fb$*vW9MaiTZ^E zu$;Xig?BU+6lV?@6O5B$PG<`-Y*uJ>Cub`?&adE#F{{Bjwjh z>v3`c0~MAIuWPTHi{L@BkY)KWDJxL!><_&pUsqVPGkAoX{@XkXvqqVg?Rab2^d{lx zz99+=Vs5LhTuIVwom`YpIZ?=Ha$3KiV7ypuk%Y>dWWlvB^_8PBVt2b!d6*xx!u(1| zJ!(ovsZ{8YXHY_w!8*;%+r3^QOs5NJ1{didnm9FY( z)xdUAyUp>d++U4++;bn;5_6?L`nVlnl=E(|0x`jS{C64!$yLEDv!QjqRt*Vp{F>w1 zH4GE2nTtI}49XkBBoUD-l=;bx*3!{r)-l3R0dLu;YE5MSX}hK~ENnZrP$13Hu^DjY zl&hd?tN$a-HTnDa2d^?UkK&4~q_pIT{^sQ_!)xr9^TxH`g*~au4p$*ja$}H;064Q2 zsE;Um>7#7$QEgx)a5Qcp(S6gbPm2MYwx^}^9SiW5nR9w^)BJpOt%ZTKFN0`VKJ`J+ zvVB+kLrh!wrD9%ApSlqnC#P(*W5d;Z0V1q0vt5=ZeFM&sG8L=VF4om@w0h;FU!cz$ zbI^Rqn}gS$2wFW_2_^U(;EX6k~hOS(3#h}VQW_=rsqsCSb$BH}oj_!wuRzkbpawWLtOoKiD zE#I_g=ukDVPE^+iaq;|T9*_NEuAG2uG3N}8Xu{FC>|c(+W$eh;jZC5J2s!Suf>n*UJ=P*b%DVr5V$|9yl1;D0EDf8~m8+@Q2%F|09tP&2iU%)J!8||6k?gKkx@B#{bn| z{`cy6HpR0aYwN22w!@+pB-NGzSbETc)kp!KIsWo2kxS_1|NanY|Fh}nl1Q6yzA})d zx-QfKPJQP<27%)|T@-kla5DQuRbL*oS^8| zg)R?4!pk;V`N6-pmbZ!_4cb1FzSLu9RB7TYnAfK>0U?}Lo%FKKN` zI~1!15_4WwP)Y{hj=$h?V0~|9&l=l@-iH>Pv;ATjw$ffqCi((B*V_p&C`1E{q8pywo6~^V42$sWSx|4w;Q~YlJ+bkPWsT&QCg=I7Imd)EuzZi zQ>QNC<1daZe%1qdR*Q+;PmWbBH!XSAyyOmZAf?DNYs)NUWR|{}Q-tVlu4e>2(^jdx zH1u+T04u~VJ15DbKuA%&pRhF64};6tpBrDvE3_s=$0)ulVinu&VLn#-d3HvhD)YtE~n zdT|Q=jeYCRl2i%eF2Pi0Ez~RxLfIq~^?@wrK8LF&;laUCJffJ`B%-$4@L+u&mlS{E zW4=-LJ|7Z&&x2a^*{d4Y%e<>3@$c?2^x(Piua(7DxNGEE&p^lu1s6!$+~I{u`(78_ z;n>~V#m35`t`89^y8Am0ZY{a;Hv4j_nW;Yl-7~FPovkT536e7#LMs^FBQ(!Tw_~GH z8^g`>S{QezOOz66ol8|O^RzVT>BWSntDL8Eqvm$V*K)lMHc+xkayp~8ld`^aPkG!l zW|?(wmXRQ%GM}cqPe`iq_orR&^wn35qQx}BNpg>jFKxDm_jPpk+S>EU--4{4JCs^u z=Vq}sfFxoESVNMnJ?YYBErm3ftae0~(@Hww{qFhrmWZ4R_Xjz(%j#z!?#KOw>h?^U z84+#MNc>vCy*8#l_yQPFXk9(pqQ)`jDBh=Gc6oicD&#E&VUUFM11D) zeaf-y9=tAGywfRL{zL!HfXaN~A3zVZ$+9e=Drbza8pho9$mVgh!YgllZE`T14s>CZ zAE`z=l&!e(XM9KDpzHsntZg>NQPgilKwA~e4nvxmn4q?c!h!NbgWWyeUFL{pH#bn#a7#5eql*_x`5EVboO13OmnokM;n*W-k<>!F2hCbo2 zs%=fEd;sVEo5A!{(u|W%{2dY{uU^EczF*P=-KWB+5Sno@;#8~rR8NU>FhxvlgNcnV zAMxUcw7cu|OCyrGpmWXah2KkDmd{N`U4Q4Pr)0eX@g|h(Fw)Cw$qilP2gceVH6lqb9W*;5v%%VQ`~Mw$O`h%O&DN8K!UUZ!ZkHS<#| zTiFldHgDjAp|1X2_QKi%@!R5?btdgRXj{3KrT5SvB%8@kY@K)K;Hb~;A3#>`0~5-H zxQ^8oPfG2v!L;P47l4f1z0GCeqaT+n?a+I?k(ot%9yqPKTh<}tMBRWFk)AAo09 zzexd#MUb$lHYFtU=_lRW@DcOvC4Kh|ty5pp&`@rSJrF#3e|CI99+S&DH-4AAMI5e@ zp{t|j(RvLX+-+JoLb*olYq{`_?{QjT>{_r57$XYjz20y zuFd=ds5e;JWwQJ_OZhnC2upd>Eb%ot$8G}T#NMfqB%{2;Xs1!q#sP+dG@SisBkW(7@@lY1G&w-y|% z1aNY&fSZb91O<%9YXzt+CJXNELy*k2_V1jpp!fb;F_o2-GgDW{3fH;W`-Ka2wr$hH z&WyZDE%z{IB6CpR8LNP;ns74C(dfm$!fBeipDZucX{i`3R!Y$BQ4{4)^13A`P^Q- zYMHVlmc-oox! z6VrHqX=cmBjyB|`Y-TyZ88n$@jWQt>Y5@pR<*Zz=N zl<52J1|2A^7_8e;O_lG!dcj?pnD@~(!dS61v}yFPVHQP7(#~G=6(0ZQKY;YjH_0l+ za$27k)kckpZ#M-MD2p9Oixx1399>}7tk2II&Y^*~alfNR6iNf$y|{rBuXl*q!k>u66W9hm zsg9)VPa6f+%0)h^N+(!q9$ZEXT2-u~&O2+nlnGq9IQBtBSEn@#r0Ck2-|;UU9Ip9K#&yV2cPqZKhXTio+3gM(LV|iVT+~!y zI9b}MsNv)4Ed^BKFmOt=zvrKEq90{?<|(s=Q(8EoC6#^V1ccM16zm0amm zx*4LV+mP}H{yV3ysNeah4MUecf?Yb!-1=aGnEJtOXSI)DYGVUlAHI9@Mn11fKF;76-tN7c$?W{!UnBWYPEWN88BL_@;~8t^-}o8b)Z*! zk_?KA)bU=d@C2G)zuPYf`_gi+yO-#14`mWi1WTWq&$_Ih>%DYeW|#ktCCT~^U?eMP zY1*p&gcz%Tik0gNXYbpe=Hu`O_Fu8J0xf3SBUkJUe&3;2x!@!UFh*xVZ*&^~o-G{a z*4fgXU<;EHYv{F!%ysX%^^F}=31Zuha$DB?a%5x%Y&V^(*5ZFdfNEp}4y8%JRF}jN z$~FCgL$@_@Oi*cDAn!gmL3fM^D$OOi8oEw$DB_YGmxBoGk(FLe;)%;}kN-ci=+S>svxt}&tAe>QyEfAqksO{;Jl zG(hZI%HUl&Aww2*U+IA$?Qkm?Q!XtTEgKHOXMfHxd^XP7-Z;r+ zZ&YuzX@8@H07xuQYieFiL+Bp>>!C!Jkg?duYvTQnLa9r#bA6aGs?O|+9LyX&q1@q% zmM$(`?~m3VPC50>ZK<{&xB0F>`JjKH(NhiHDw? zmf|mpWZI^im4mwG8_^hpI+2^fcxH_ztNleN)HPgNl!3F_U(}J~=KcYe&>IdKk@BA9 z9YCYF2XC2A?c@76lpo-KBR-2L`&qT1TMZ3tJ-Z@tX8#IX{)4tJ;xtKf37Ls?{fQEc3CNSsSq2b*CEMO04XFs^oQy6-f5LI|Cr}pv2N0l$ znxb-}yfJb3bkT!usS9v%R*CFH;u7JHEevtV3ZC1Y8_0&s=>2ym|Tf{_z76 zQ-T%@tm{GBfBT`uMSLv-Q1X{rLa)DGdm=#zkO!~=j->AL*@B=K!C0>Uc_0+g)>gxw zG5~~$lRoxkDuAKMY{ zb!*ozKVM%%fE2KMe)%{k z6ilQzh>UFUkaO&*h2mnxAw9K!0Z(d*Cn*ac>&z~)Ki#R!pJaVJZ zlANyBcP-!JJOn8NK;b=PaOi*+VESdjgBvj?yIa-S<0M` zefO$z2fS$lzgH&e!L>xM{a1D+1;rOgP56Pfj9ko2-x%j7{Pr!9Y(@zoPrtFJSbr5A z#sSXyw;VPVA<<6noC05Kk$m(@;N&EZvjrlP`Umxb^~fH@4r!C)3^B*t_WCV1Xf-d# z-CEx8!gDV1$BERU;km;*xbf&U&irG+Di{1JAJ%42tEUydjRXPkI2@RxD)KqZ z=uavu@|l9C`w0z6-0Hx}mxF?8d!g5|Z40K~9>`&HNljN`N9`GpAv1GKQbuz@goj>m zxRIYG2TjQ2lG0QrnS{990X8W}jz*}-?uwzX<(b+)XA*zGtX&)eL6E;|#B2A< z<+Gb@ep}sxD)Gs^K@jaDO9O{0e@KmXTLW^^IJ`Du4oZbl87ay6J=rb%sh*dG^&hm%a0G1KaLkgYg;ko;5txU&nsSNg5b^=(j1M={uuwZ?=NY18JMhjMu zd`7Cty~5D*W+&9&6BgU8Q_$K%eZ&{QG}zlnEiezxh2RDG-@I!VKCnHf_|*C!$qJG)q7Nj=(4-xP)`shibpuMy;x@^Q>EpWroH zzyD1R1bvQAFQ2*$1Ek=*!DOD!HjuteFN167x^eN?BMWx*GJZ<$YF31`fHL8I%GMiU zAyzE-b}4ckDarLbyYi|(!{D15NkC<2;l$ox_efT%&KBZvS*;P@W9nYuIf@d0g`O4L ztWh-Gh!B) zr}B~6<-9_&Wti&mYH?}FFUPsrtV?|(tb9R-87gOb`B4Yjrm}sIzLOH0v`g8o12#i_ zS6G=IN$ijEhA-EX?RSRU$K-tGpTXDH7d1Ywrl3c+j~Qe^tXR$g2La*&;A&WASqP`F zap-6^M~Tq%w@j{rN{w^1js^mPWfpBXkob-SqpXNM z@?4cLV0b^X#$p^bpB>Vq?!-~ZA_>rA0lcI~F-5O7YYwL$&jKf<_`B_9drpo0`L?8* zfSBC?%`Q8*F~3k`0*|LOA#gsg;8eY#_;DT?ylp7L6zs_Ioos`@H5j6+nkI6HB{ z1ivd5vu~22W0ewG@UeXPatT@jgDA%AG@Ch0`&{JhB4D(k_U6~@ciNT%(3Luo2N6N# z$oOX$4@)Yd7is*(bKFmJa~)e3GxOwZ#g(-q?d^$I)kIVCul=W#sF?jep~A7)INlS& z!lm`yXvrbi{VYm$uKX8evTwjk;AwI{QN}ds70s|hM-c(GVnQ-+eTB)$O`IR)pt4Sd zS1&ge#;3a8jB&-`zT2pRu>z&P$Qy!cVJfWzsc5yJ&!W~pI0|ppHJyS6W7|7cOJ{3tv{10=37z{6bt_XdhbP z(lOc!Z^SGI5-J`h=Lk=!Db@YHvy8&i-}8 zee{I4G(SSQS{6}HT(1<{B(#Cqo}V!KoLS+GEsSL}LGxUK>@rF|E9MBPUE`A?yLDv3 z-peqE9!8g0X0R(thj4VFz9a~WDmgG@_3#)hz;jl9nx33fm&#a&Ex0Aq@yFjFbKdq;kyX zZC$h{aI!9rs<+LwW$yo$^Dd@=Q@&aNXWq1!V@#?cnOG$CWBdS=Em@--n+mxDFf4TT*=qUc;b}vk`FH~v#SC1 z1zB%uiaOLhzqSKsf64qQWS%P_x1VHkIhXq;T`U-T7;xO(o)Y6Nd?KbdwwPCl&xLNi zvLO|m)WngqJF_J3Y_`B*^@251iGMaHAtli=120z!3n)me%ADt^-Rx)fIREh=^qe{7 z7mVo)c|ICme7#k(Nj{S$oSbE#pPB~qco1MyKl??Qo?AAKXd*(5z}G7vFh=AS-^PZ$ z-l$dt8ncwEynZlz5v4_`^7!X1IKIr|T((_Z^2IWlSh;T)&cYC)0_C(N`yxr7 z#o6@xi45xLQDXGVD$kTJkC0&vrb9zSD$_%;czuF*2CdwBqg=7bQ$5{R=ND>ATvGSzC=aHJEKIt}klbvM-`D^|aA zkiR-vGUCla@9fRTB9lNuKgkgO`s+g&lrAlK+#E!D#ad=!h=cj}{(KeLSXP(Iw97C$ z?oU_IKY&?_fpU>o zlx7uAXM}q%rXM3*M9kly06uhYJh}q;@3YMZb!~)Q!0(2`-y)H6we);bvld;{v3n*>kr}*32#BR zh&u#hVj^?4RJD@}*^lCeN3LM|UN_{Xi%xDl*5eTomSiuV03YuD0T=?na>1RCPR$Xn zjzWJP3;;Oqh{ZdD>&1hwTdGg50dSSmAnf-K*yj$KtlFKi<=qZ;U21%K=~83 zn(o1PFfjnp9FB}+bOBfgdQhIa({2Hu-!J>)U)*2ZZmyvHL}eH4R>=9wuEAJP6@4|t zjaJcQh4CC!Di}Q&sgwK-_8WAkU;RyT{PzT_MiRm;?#lr-x@3>++%vlNbM!YCyFup% zJp0Tu6axLiL5rJA)EGLVFQFNxtU$TSD4Kl1=g$UMYi97xZr771P+7pS=Nq`*VG#)r z*yvJ180A`C8K(cD$xJb|Y)A00(Lw;&Xaq3w*w{|dO;XJNV{>@%#I*|rGL|jYykYqn z11&@Q*ve$C_M+>y=9!;ZvYjGXkl_4CP!wzJ^DJU8CwXb#YYgD}UAp#4h?T9}378|N z*gV}UCStRCC18ezB`|^i6K76=ZE+e@PE70{!0Ee2FEQ;YFP~SVCdYUsIjP5Xvxw=X z?_(?D;`&(ADfTz%f6S%TU!NW5F$-dWF$BA+Xfo%ud?w_&3`J^s0+f!lv!)vaKh-|G z&{0s4tIS&*br6dZUXU5rGcNz`__W1argaPLheau=L|gHfuEwUPHIRDuPq(VQv4_(0 zU&wLtPf<^)p(Q!~Ts|oY^ee_7vuBo`>U;nWFAVNz1{(Z2Z5t8IVX`S^0%y-umZ#v# zy{;SHi^liV3KxxfEM4nyr-JXYhq0tnmTGTxXpfM^;m?zE@ivR8lr?d{csD^a)YUk+ zNTwtsnSQZ~-R}8hd^M$UWpOfe`my+f<5I}=;?eQr*Bxvcnp_WkjoByr7?=^=<-zrV zN$RxUa`UNbr}j-UO*HmQYU2x)##9@&9Y4j84EIO5AADx2rlQO4q`Gbd1T*_*46+Hq z5kDZuVpx{%)DM>2(s@!D7%GK_L`}VvhHAvLWdk+0Rn?~3nA9Y`d-T;NmYc*C0s6ye`Iz{7tnh_O;8t$^5X}?CopIQj(Ic3JHKXI?ZI*F7=4Xl*x z4bE)puHLt5DGLpL6HVxW8L@Avb!}Bp{@@!_b$A~wbHoYDWaYQ5RAPk^vrT+U*RX~} z%P>;Phcs9!B+@@mu$b8uH1Y^9zO@r(F3* zf>R*)@a3JhrcYD!uwy2turh`RB%DVMS1qUmK~jly!T7PIs7b@(=& zl$}P1AWJmv$8X1h#64C?bJ;|i<_kQ~o!YYwB#&9TU2BfIcEPk4AMcSkiecUeZjC+0 zcDbZ^;{jWI%^PQr&6#Pn^Zmh#k;=JM%F~##WwtwZX1qzi@^R1-CT2Jy%Zo9|!fn7l zc&;rkXh!x*y5S{nBeu0Y?iFr`VS<-(yOR#fJVV4lNYz+~VzKpbwdnNRMj5hBmY6V= zu>B{CkaaRm=lcGJb-jRgg+_SyD_xT^E6;k>=wX?e{EES8>0h<|fq66ciW>IcbDY>f zHR-y6Vm$r9ns8KAWQ0a<)N|MIjBuy;%Vb{fPvPBd(F+W`3g$R6)9^5|a1)gNcjqnT z>8wsZhuTG0#mVk?^6M|Ew(g$<54%WBTGcf!i^H-zDjMfbJtVP3K;M3JTHKP)8 zUbyyJHmkbQqm?7$G(T7MXR&ISf}}-E9dj9Vif{5T^@ zwVAe36y+f96otMHtTt3|wBf6aXbaxUi((?tMRlvzrkj#hWW(4m)tho&R4-wQ8(8&h z@kXl?E<^bEC2uf65=ftQal=vQ7=br03bA&)-u&@% zPy>mhRkD)6WMEdulG;*90!d3Uyv^A{<6u8a5AsaACY{e;;~S@KLdYAA_WLWKI{3O{ zV>$AUx%Sm3^9?w0w3n)UUJj>k!Ws&caz@px8foL^!{Zj47O5^YP9Pwr zyIS;m{{|Q=_6x`Ys?OcEar$#iqP%kX;V3NIb+!E&(N256WSFAwfn{o;YH!B2bisvs zhP}Ge3^^s!rtsMCR+IC&1^4_Snh%|*&vhGh#iFA$Hc?> zs&;}e6UzA5Etc(m+C(i|CYu)icCKLH^?oW5?=rW0n-enrNYPlt(e zvpzu&?|F$0hgKEG!krvvcZSQ9N*8i@Ho?Tq=PPD((2aT9%(xj^Ehd($cSVrW?ee2Z zWulR?G}9SI<-zuP^HiOQqb;Gd2|g=bU8veM!K;8adjp)vaF`UTHLH8jfxb!kR#)EBV9iw8a;Pnj93c#-k;Hi(ULWH)z^3B9MLJX~)ISCt z>5I%*Jg~PEAuysYh*(r@YEy|lnr(;T!12O*$V+Lb({+`G&r*q64B6=ra9uaj#Kf%K zEexlQcDk$;(w)P4hsV7Mo)qC(rn*T?N>!4#da2v=!WOCAz>X;ccRfrX0Jg^hW|B|f ztXAfq-TVQ>En{XNg=v($GYCiwk0VW&M|xl#oyRURziimhie=cTPpPhI<0m{UfTOpt zQp)MMJ4(`!$#7b_k@J+4Z-!PO@P4HUTBnL3K@Aph8JMx~NvPWo+AO=XK0~JMF4U3n z6ra>GEN9U+x4!B!+P(V1Ib^L8K}rX;p?++{q!W8W^*f;PzG9ja$LsIinqw`h{+{2Y zH$JDF)MV4e2EE1x$&J~`z4D`Z6`3q%XB8loL&WZWn>{2@ufN~w*uxKqhgaPwpr zYok>J2eM5;I!4SO>nEN^h=+Gh6k#-q8pmX_28M$dko8Xr87WPE#c6QJTGI4>;%E(# zJ-G86ojhtajXY^qyH`>eoU#4(bFxM0y3k_&8ZT5-w17`}_GF4%M%PCnkDHRmZX7kA znqUb)kDicPva~Ts_)GUbNo62gPVDLjpk?Mv+Z#oXebeWRZa0Z)bB`ZR zh80LYM`b7JIVWiH`Eyg4ZJ@z(eY}&Tq|!Bg;?IPsg;-dMrB$oQwxnYptWcQKoJ-k* z={EB_GmW$%v(FxN+8b8&LpN4Z+4zwobE7pawBr*c<=T!j<1iDC1q7v8w`r2rboAIm zHScb}p;C6F*c{~JIMMr7Hh2fYuHFe$z>e}zq5aqlvAeYiZhZGy!cLKdUZS+ zB51##wq}8`##$&a*CqDljCkcBl*IvqEOBRn1NM%XzEvqsV4P*PMf)f`Gi%6^&A6yE z+dxY7HcBP3v~+CYLH8*CV~6+4*cAyYoGpX)T=mg5<4nd5K|i+oKD9C`4((q>J?me! zym>K(z>>EHkzv)N1q`CkoGYSl$?p~`_%=||-OOPu$pRA}KDisWPZ-;Yk9${j%7m$n zs#)ulPnTARVt0!ttCd7Srtb|@RQA-jN=q{|03LEsa1NrRt62R;8`*8~Fz-a#*_3JZaqNPGkK*G^8W^9QqI7EG(s}ku1Vw56L(UI#tov_*3=jMePW5N}<7}kyv{# z4H%^bOcDM zOM4pk0qN}kKR$Xr!_v58pLSHAANRS|dns^0aoRv*9Y;=q#4=mDHl4;^AuHLGePFZf z4a>kC&Io1>X#I`EkB=DOByjP?87l-(>v1)6!XwblXqEk9{co|6*hR4qDGqS*!0mj?S@B1mFgq zc_>Kl`G13#DP5{IvT^$F-O`HXW*5W*M=SR$Bp|kj0HiX-PA3)YKVct^DuGoWY|8c= z6~oB+cW?|ctWL9pM4~1~{Y{%+BQ5l%a$iyVTt=a#Vt+wkQ5XQ+o)3ey-vi%?x1QUp z_6?HScYi_Gjf{>R=}n^zPSEu=*ZP}m$N!fZYmWSs=YTgyI1!UAV|TAF9^%db%h22^ zQRuS6<4q6RQ!Vmo*6CWV)9YQ5|EQsaKZ$0b&wgV={x)2ryg3e9SlE5UWeGdmP3pG8 ztiQ*7jDT+i|7Q4d^5yCbKllO}_*^|l@_d4Uk3rd{Z>OZFMb$jM08mxJZ~qPFhXvAP zK6)d4>V5zE?}JMW;@`DNoCo!xZ19VW_Km576R&%0=juTeR%HJF zlN9QM+s7BFqt7;WSbF@wKTV9L+Qx-ZH% z^6%uEkRhHJm|7+!!582`VD=F9@^ALpl8b?7~lLR*(=l^R@sO%HBzy~E|Te;~idW)ca zKrSg`OcySn)2vf}CCwKJt%Os*`1J`HYVQD?ci@O zo;iI}?w=F3vp(RVEAnkMv&O1{7rE(s$rbB=O>I zG!*^;)+JVzMYHZc1yp8FhpWu`RlK1iQ+Gceo5?UWIa`YP8+x;y+i=-aIartsx^~If zm^P+0VYCATsqWKVRL9Zdnb`_{k4b%lzN*}DAvF)^r70CJWeoI0R{u<_Swg5)FW1FY zOEk@t!Z?PK-*hckeVNIkwELFEpTbNL26xA-r;lgH<+i_3Vf$tkkme{B(G!rFteHiv zOCcC4FR#w~J~3rzgp|^TjZSQCzeSywsTr;eRTU1m;LFGurtgeO7byI4#pSK9V~YI6tX9TKS~o+Z^t^>YmCS|k{kB9U+LuWL1Kj#c zyA$dsLI#JQV#1xTh)%P1k9zmgsqv|sExWP>+tI}czw{-!r=n)!GzvOa{XFrs%NSO& z@)XXgs!hL-US)a31WH%e(H!TmI!J^7(XS5}Yaacp+LyVA2xAuOXFpsm`uz0jKDv-k zXc{{Ra}b-SE8Mk|-bct)Xu}A;szqNqUT9U>7#1DAF}y9W>XA{b_YHSeXVvz$m@%gK zBXW{bL^mEY&CIV>oK<|Tbprm(}8N>mtVMC$+ls)pr)P1JtV#K z#pjnM-IcyeWt&)<3ANsV)OJO33Dv6f`{r2ll6p6mHGM=M!k&gYl*W(@DlY-VkBeSB zvV1mm9n=rZCG1S54l(o`h?Cl8$24GD-&v*PvFP$WlR<>pJKvL3C!l?(wDaCtA zd23TjywK~giH-etl6IM?H(Osu-S*{VeN6U8}DYkpkCytNktBJ>260f!E8+dHDQspb}E>uBauW!V{>92lXJ>USLSa`oo2Z zm_00f#kGlPHH@E@THVYMh1ID0rBY|><~GxDox&@xL;l@5tDRNG&g--LZ;$@~boxpL z&FRTCQzkV)+GFrUIwciCn7sf51($y|d>O;dBg?&1#gveSSX8@8{=v|j+`vxm zy=_t`2W~mYp8XWxZP|&4rQv(J)b-yZdzd_z8YVLj@5$U)giHpX1-7ILYvht{ypa=r zB30Ut^B*4!%8?HJKi2&o42V^kJzsXA8DNr8LK1RP)d3RWcfA(cG~9X}S#@&yJacFL zqbtXXDUXW3z|4kJAZ;vpoC^E}j>bX?974N@+xW9R(!ta1M^e(N*4YeO3`041r$x=} zrKfFd^2wBtp2zzLCj1O<%f+Hao$j-`ozYiueIvDY&h0xG9$yY+i+6r#VRur;K3q_K zxG~qjRfza0Sv*uZlri>R3zLB@8ATfH8@()~`I$Osb1e2a#@Yo_L(|^B&}n78&J%<_ z@U5|8extZ$N1xV#FY?cx|2xl(oZ>nCN1oUFh5a^mg!kEwMek$(fG5Mq1>`BEBS$n( zsrGhwMd{^3lv8nIFI_yG{YlfD2*x*ywtkvGxbVBJq``{|yaQDR+&j7^FTD&UQP zD$naV=@%;Jc&Ur}BGlngd>7~(4yA$+>XQDQwfCQ1mR94aS#ml7dCa3~tqzn~x7}d| zF@z zhigpjonD}=%>k!hL%VnOSP!NKxkbva>k z)$=i4)N#Z1E5Z}nW$}vC?LOv7ATSpqA zme156D(O;m4temL16|SfA05F}is~MVB_7+!^_|DblP>qL#y{+Gd>0mO!x7?2mr;Ng zJ5C9z*_N{KEP^_Pegz5{ z=H3S{5&#_UoU^H~HC|kibNDc`eZRwY;s-%Yrx(7#d=4=(w;;aM5qj3lc~@LSW}oYMQ}L)y>2!YtN3$L9>O-BZ9D@YsYzTo zn`zhec(r0TGuQmG$0SlN#kyi{My&S=v>wy&`*XmI{tRHWrbd)T*$@sY;JJ<&FTux4 zt6np=o=humtGDrVR@_zT&Rt_C-?JxXB;t-8nmC$og_yNd8B4CR6wLGNRzr>FR|Z;r zP4(EmbS!G#j}qwG)R5T)3sl1yK84>^OE@~M%9&=xOPerkyX~k}I@!L#W!f(FJu5Bs zj?gRAbD!l1&wBms+-r5D>^Xt5MN$DW zy}dG#`gx^3&k*dCRX|3p%ENhTx=d);dO9C?K3PogCA|ylsHB=5r5((gnTkD9b#mrQ z@4My5bbVHCt3=hLdtdsnn>}8EM6&@UmUU1r$f7{eTAk(dJoXEKI&U~u5jZfIrD>js z5kJX)(aDAbN4Ew5ED5==_S8Q&T}`%-@M0TT%BWirk!^%zr2+~#q_1n(S~1xeB#Q)- ziMN0ecgLKt-c;Ey^3aHuj$|0{UC9=2UA;qca%_NZjcG}(P^qs!>q^c2Gxeo>e=iP- z4$}DU-}dSh90>mal;0*lKUvkvEpwv-kRdg4O*XI(Tumi zk3LBPe2gjJTEbg(2MJ}0^G(tB>8GZWs<0JwxqxCFKd=43WC9GQgh*8Tr*{h!tWol! zD8%c2ZP%*~b7{9XnS6vU1&o@a20`>D#?snS6avIg5#Z6@6Y|9c@8y}dG{3X2j%|H6 zl$^GaLuOR(h&n`b@WmD|%Tkls|0FFksy%y%6<);im+0FzMjI4ro<6G^@m?jB%Gspk z%v3b;uDJ=i=oosoa;~1d>A}cV_rygH0eU6aVFzNMODR!qNWeS@rKY(QE-V-RD$5Oy z0QR6D-KWQf8C5Z7K@PuL@4a1--mF%q>3MWRv{2xf)m6K^5L_WAw1c_Pe(C zByrD}Va!~;LHutKSXHOuc8YlqIscowQC9Z$)fGI{?1xtttOpU7!emsgZ+#FS!8u-9 z69q)8N6}#^hy{+`8J9F)IM(gkw|K$ubhwY^S%2}>J9wiw=_EzW)0(0k+0XssX_m9w z+VwS+b*5Gv!%|$d-hKMzE9M)z|M+>QRG%;I->_*ap;u5bs%K_#9bQk>>{s4Kt1jRq z&PiH$ah00N7o$w`JxVmXEl z?EERHt}1barbbo0xMU6kGfuJ4z&ia;6W(Kg5S5Y#8WY2Mup@0nG4hgJgUTtR{O>R5bb1%tZSrj1En+dbgU`eJ zt9vxH`g->*kNKQyV_FSsbpODPW&{5o5vw-*UjF>f2#-ouoprU>T*}tbU-iQbN$UQ8 zXsv4mQF)bA_C!IsoWAAf@2Tv2r~gB<^YX`Vy9v{3qHtPd{r=!{e6xlqLU9o@QQ2*j zRpR^k?8=6JyYHh1TI5STZ?Ei|c7b{{8q~LA_Q!QJG&KHq6Bj7E2b#dmZ*-Flu~{^< z=TFW*qYI#+q^xIXPmRn1uwCJUw~y|N{om75ZO_+rXiuM1^%e`kONQqI zoCxFG8ZYYa*Bbso4;)@s8(K&MDj);ZlmHn351{2uKH7&4ok&6EXPW?M@tdiluB0A~ z%coJpge3zuJ$_#0>@Fe>0XNO(L3xCKRwXQ+Fvz@RglBN>JvCxG)m1E>1@4VLRl6S! z9DTQ7kdRTnIn=0QkHw`MRwVH%IoU%+2mrQY!>6RP0uIz+()^BEXiT(pLl`W4uFrzF zM_bOeGQqV%$~@MGR(Hgi^Y!O>)qf;4x(YzG-`$tjbu4D;W4Jexo8itaSJxgl?K&OK zT~a<*;ep!c$T2d%l^ zq%o;)nXOHfd6sMY&HU@Ln$0}u{{PzFgrQGQC@IiVdVo7}44{Te9rWO3pGMI@d}>bvikNIXszc{GsD4fT z?kkq#JL+>Spn7ra1@+a->@O^M!YGI0{2EJJU{DF%x%&+^u?zVsmSu|kOWmjugrn`u>L|dx76N5rJgqim=tWVw{LEiB+|3a-G^zYDB*;DRWu) zI#;;x@eToS=FVbJWyDzg%o?1dwlN}iXQVjRf+CB_`R09h9g~ldfh8w0O7Ul~@b82I zgpyFb3-=|^P=1x2X_wJPSuF_`u-C9(!CUSz7jb>B$V{QUNgKNH+sN8^P3!(A1xxSu z>>1$f={Wr^BIkO%vVMBpyHnss4jx35+4yjzv;^^pK{M<{hqM^~Kvd!YTQn@=*ZEn6 z3G$U(eyh-<_1ON2N>-~?rKYomzm>R^*ZpnhIc~m|kR>_)m`-5&j_AugwWjAuXXR8F z*mNH_H#maFn_N=3#lprGHDpVip>Cg;oeh-O4Y{uqW$hk?FBZKnFF#cV{1ApUTY#kj zBbFk9)NMJ#U;#m@Z>jIc5h~k^rh?4<)r4j1*tH6X;t!unhP)Drp;1-u1?gnG`8M`i z0JSvvow!q0VPB5>luMGv*jZ@R z+MEZJk7cWHuJ^oCuV4DOGc2a3&JxzItY*w2ECJugPvoeXG|88MLP^Coxe^Y{Mc2cl zAlMW1LEhc;B?Z{fq$O_C%4jEJwV7FWy;#@aX1T)B#ckI}5qF}jA9hDZMVlen&goJn zSwC60Bb@UH*sPPWh1+Nyyq8SBt5e0|8+$A6Rxt2s7q_f|YdF=N&Y5x6z$H<-+y9{j z?^PgU*Xn+Mj!uh&a@&HZ+nt~}1l#Jb0%5L?<@DI#9&Px?OE{%rsBTnUQcYzS7bA5^ zvNh+*3#e)pZgtQ*!h9}##iDBMVg-?_+*kUa)2i&ocwxA+p6mUeAZ)d9)?+%+>KbAi zrd+wNO7aUM7(fpBW7m4AY34*Sda8AE^e^KRVYIr5YZ$ntAxdZF8l;M$Le1SJNwo%D zjU0E+hBk+ODu%!5Dc;icYQ`Qr@ zVfx9LF;9Q=7x*;WBP~PXoU;`j_dGaoW zfX!WRnf+XUul4HD#d~$3m8YJrS2=1Jm-1DTR?d9SZ-gC(hF1M+WHrI> zo|r)ez}{yn$uZF#n1XplZoA#$(KeW)X;n{Zgn`K-&BbKhZs1MIbsjUzWD&@iT@#Uvz0;$H$tR3yr<{E#x|Y^dB(@Vr zQ67-2$J;;5TF$2TsPqeqSeE?|>qG24W2zsT!O0AO#HXrC5aLFOnOx60qC6)Xd@QF< za43!NkOGfY&^fVpZ1mbqcXvTYxa}3iey1Uyp`XJ=?!2D{H80bYP>`+x4UYB&>A7}n zT9TJ}aZHH}C$Du?QmmIJuJ@GsmI(t6GogMf$hY3SKNHt3llT}5(00|?Hl$d++NNHk z$IESzZt96bnMd5KzjMPseV6Ok;w@__R;Veb>>O2$Zv$i*E%HiNFzby6$yQ-Vt~Od5 zHIeC6W%(&M)s8Adz?!oTE>#O%HC`Sdp{S49RbnI1Wuc4b($gUNg5w&U$NjtM<$amZ zEPu;UkdSRc0o0Zrys3pGKO~)7dot(_4oRx*OE}kfy(*LXcA4^pR0_D2RZ*hSLsCno zAQEEdvf@wP?90}WZ^m;W(b&#hLYb3)b2q67XdfSQ^%lJ|nc@mUobL^ulKUlcbG{G?mj7M=9$b%^%ng04bffgJFwIr8{L_OU%UGQL!(mB%tecL>RM?ZWYMa2G(& zo6%IBVp3O0vvFZ?-2UkSwv)b0%eheI-Bn`mV6J`fECm65!66p9p|*#W`X8E7X#S)% zPH!$JevTJOIMtAegBJ1&&crjb2}&W*f-C#F5m_!I-)@8%i2k-T8y+>>zo%ca2}ys zoIa;<1!IxDj!YkE*W{0zn)u)PWnnTyc5Ik}T+8LAI|d^NszcVW;&e;0Ass34`%1{@ zKr2+MvSU^ScHUJEW-5{TTtver^R#V~%4G~@0bf|M2f*8Mos{2tixNxo7bo@D8H7@~ zCN*d&D_9K+i#J{Bm@IK%)XVt4Yq#q?F=7P>p}JXbDo>=xgeY*d_kT7>Ce3XTsQ&}V3uG?7}) z-Yn(!JibdNhUsyNbbWcwqZUrjmPild$OKz|g$X2AF{e<3(>o6YqC9olFwGn%SSXoK zFumzif$WgiYqplYAK}2lq~rfQYh!n3KXf7zmChJ z@1#t_<~7VYW0p{5-b_=rtZVgEQU@S-lB~hdOuWd^4q1|9$3nhKx5y0=t5dAG#~vu- zrVJe~$>zeCRJ#Se7fDm_)kN=qQ8JMmGOXh@38vuQV=p$5Z>q`C07)0~F4*qt#$J^k z{=m=Zzk20vv@%4Ed^d&Q=~YVwF%$rxc1+(RE(Du`bYk!e+$`%NKz~ehEcN0GCUeHo zsi*8pY$4sUimFt#qkIuzR&#V#TDYa7e7Gel`j7$@w@7Wmy)5ZGt5BA#v#ExY|Ik7V z+@$4I?kX$`r6Pb-Zcs@IY)RsA{U8E%b|uwfMZTdk)~f@5s8`H_&=4y1i8n@ z^xvOg7bvKeN}0j{Ad?v&LhY+Qv%gj3!*_g(n|<&f+WXz88hEx9 zT-C{Tis}6P5GK?t)LAa-)+>W*orvl^a2YR|h9($PYMj{})C_bc4$YrW2WxcAcj&%P z<6MSnPVPtdM$OT8Kl?>g2w~FVEH7` z{nCIX6#W%B8Y_D=x;XUbg54!(K%KMKS&NozlXW^*K0RK3orIK>l!6+(q|8F|HUE%= zG#<}j!yRo}W_8_N}KhF50|f%b_Y`aVFWM!tHC;G8pp znhlipjwgA)4Q0r@Kor=sCAUZGMx&QE|x4Qwn32Sb>R>iVCBE zE|bgI1_r3+d4N^5;Ew_A4DWGWKWWPaXgXf5A4Ld-WTux0qM|~ih|2yGJAAz*x#mm} zy+>a%(}_tJ9IwN12rP-TND~%yce5p_FY#O|z&XpD_KiJi=*=#4k6Ga8*QAb*k>7O+5!jxlN2;p5DX0u!W}7c}xy>P=5|}sV{|F<@YLp5J zEL!i;k^lZu5`)*s#AO|UZ|C;p0gmk}SnvK!Sop!?m@E`2)~skje)}I96nZe;a`5Aj zde;aE^MqEUsweE-%UPY45JH(&)D=}3>|7U|U=7GL3)D=AzTit)ZBtsO8|j7&$!i4; zDMaq;`QBI?G_70X5dwyIE%nt&R)HYzTVIj8*+=ca78Dc%L64oYKTjM*i7B{VW4(@b zcluv*0fC7ldl>Pp(S8pN4dcbte`o^n^m1a1L!UnW%OgCaL9dV7$a&6~_nl0T>c{WY5+syOmJN!T;` zzscT@uPe77z9Y&6Qqvrc&TypvH@0^rWqi*vMGp&rOZv1EJe;un-E_v%`rDzv6S=&y zaB*5E^7kqmVwuXHW279Aerl;OuBpP3Bz{Xivtl3zou$kw)9QikfPDe#A z!(}rE!J8ZRr>gb7b(i5N(Sh@`2vO2PjQjiL=_f;ro{ynuc<|BIn66jR_m^#p#%O52 z_4XaaLT#z+WJ2x&176f!3j4Lnce8|sJDsA5WjT1;d%HNoXY4#D9%t}YZR>pm_G5Nz zFxWS*o}d00&(Fo*q8&rs9W;vH#Mm{ggD6`DI@!N6iC{-3+m~k#-~YB%gfY{`|F674 z7UQoKhYrKz?f*J0BqbI2)<-9t#fz3bQc*r5xgzH%oGV=xltTo^TkV8HOlliM zC=XxsGg4>yGy4y`EpI>Z%9)IRPMf|;jBSsEFS9xB?Wqe{|6AQMYIR|MMxDjQ4`W$a zap|{2BK~l=hqK3YJ-9#4{`2b4w7kq-!xpJSF>(Y_9F1P9(?oAm{z8l8zU=&47kdr_ z%3XK=Yg&r%_yPF!1Ji_GxmV0cA8hG&j7PS_Qg1MaC;#+7q)i(^$AOb=Aiu6AxQ=E?}QNFpR+ zl&LZ$0wK)$_=IVqASVx z2^MlmSVb@c!qqC&r_5HLkFQQdpG0Q(8!d7=Mn%vY->}lSme8M+Lf9hMXji_h+*gd< zB<$HgVYQHhQ&c7DRLFDMyvZut+I7kO1EH~uRURUwX#A8c7p#!gY5i(AZ>tFo+~I<< z*Xu&7IhCu^npp}m1IyY8FP_6o{hJO0%i}3eLyK~9re;>o82p3tZ_Cg2jd+X(M1^J{ zycMOB2Ul9Sz|twWy6nTAt@Xfp8qYyx!rHWyA^u*{Vtk6pxN|Vew{`KS;jT?|{2N5< zj?5~|Hkw4c$W9CIt$L!gE8O11%8`n;lc~fqlQXq_M=dD4U|$t@KZ*H}h@3c~vgm=8 z*XvEd6r3a^B=oc~Da+U`w09eX?IYz2o@{&Uaqtwm@E$iTArrDTElVFqsOFsVWNc|K zsMBi8YDL?foOlWEhM;m#>e}jkXB2}=ypt+PCs*h#G=RzW~0f|Y~y;9?7B8G{EY8O0V& zK|x$qpn+sf;Q0Wh21l>Chst%*e1V;?N*v4i92DCw!;#R}g1}OnGDTlg#fd$86TO`M zp(D##B;hhurkS6w_qncG-+4KUfnCms$|!>z6~gkx&%?>=h$p9SQ2|}rL-d1wqcY_| zaN`{EeLaU+bxH$GZW`AGG|T`G{Lee_i`7<&zt@(M3TW9B$Mtb*9ojEYuYFyl7&7uEbowIn4ze zjsq_Jj^MTXGBLfxweJFvwE~qA-l!9OfO(cw-{P&&(6x+y8rBdTNr^iMR}OjWFs3w} zKk9wja`Q#5$(h2QZ8B-np#BVkzbLwVY19wo%El0tOikUX;bT4?$k(v_z@c7TcmI}d z$T#Bw^8lH}Y9KW9QJTL5nv=aPU(2M@s>17&oV0tyEzH%Cpi3{xb@klm<3_>DZW+3g zNh$WMmDRh+-h3V$q8u7m0r-IUrV zWvgH|*m1~>wsOPe9g-!%2cu#+DZk)5=V3R(96unS&$+Q2kC}IYcZh5m)tIB-FM~pC3fe4T<6`}GV8%b96{?EIyYa#_C8{pZ!%iiov-Y#v}6mMkE zXu>+D)g(8i`^N#VJXBAM?xI1;{VuLRe#$hd9|Zpw0|khZ&M-5gfl>Ju9c(Lfgw@bpnIvh&Qe^gq`XX$pB}x5g+Rel8L67 zM~Ywb6YV6@3Sw01L6t=@OzG5T3@y?LS3UZd*rp6lDtfh@)q0NJBgo7y1Z+_7V;oljny0DMw<9bJ? zM$PlsB_akt_ULUcgkVhRq_CLBw%aec4e^=^ZE6s2q`dD&G_B#^RmlFyQg@m`cM2B9 zbfktCwCGt&EOvkFE@ejV+5J_0A=dAZuVKK8eeVr1KrYfH(F;yhwj2J3R>AcaM}Jgh z(DhtCswUwy-8%J6QcGhPy5SK5?^0@vH>rg47n#3Si-gH51sQc`L;JC8=)o$-ZgTcl z1;7IDtik2_t2-wmE$$6BgmU;u?b88IAl%l2{$=&FZ&8ek;Q#$u5K9*~V&AdAUIjkAkY~PHmv5f^bEhIVn;#TSL%2y7AtrC&O)D-SG z)lph=>fr1CnV`W2H z4-o0T!|UbSJ6T}bl?79qj-RBielY=g6s!YxVolPw*pJ?>Td-yPPZq6tW0&2^?(@d6 z?AA{FD1WbJfR}d=w5hA%)f!By{5l$#l%xpf7W};RH6yo)jw5Sr9ji!2baGe1O9($T zu3Mq;NIt2Xd8cZMG^5{Ihxfp=SF_nOx#1jkHOu>B_Go%r~RH#vjVjBj;R|fLVgtk%>Z1x)oC%ml798pnO3%$ zJvpXXpcS?!Ip2Izx=u-!X*NRw)sn01wvF9tZ1BAKG)(DmX;VoL?bDF$rkt%ZZ;2%= zan<$N=#S_yYXMT?kvp5$yk~3KHRXz%NZNTeMcN9P=b~R2#aduhS~c!$Q$JWHtsSa` zI`j-P&tUCkC(_J>T_E$tmL?;RpED|zqr!L@SF3ct9u)c=TN8Y{>m&jwBJ6FY{uHqR zx9b{T*!6`o?;QD^Nfz>FL1qgm^#%5-g+s;t@}^iqL%-NhtKo9$7nUlv&IvG zk<_?_!}3fFLYuHuG~Z{!5=>554Ek;xGEUYwoMaYfd||2*9uCN~ea?zHm$E4D?@`B5 zY(FdoC=CqEuE*fi{c3Lp`-`Q)W-PQNS^*)YI6Nt1c^gfWAa#q7Z~aM?&R(Kib|q38 z4Rt&j%2rpG1KO+ZAMxCwRWB{m9;Ws;#glVw%Pdix+^c~H-6%YoR|>BMC_$rlNLD3{ z=Rm1yMm6&?kuw2BZ1>%4`nrV2>bkSp7%guh#)g=ZO}UGNRK4wDD8P{JShtvF z=*`v2lP^@sb(k;Zaa~)A-3r_Gxk;{%V0&a6O5;Pyq(s^YuWxV&xw#^|Gc>TS8^3-H zC9IW5N*?^$v1Bl14DQ;)X^b@AbEBPXoXB+Sw>H6>go7pZN@PQZ zH0#`|?*fQq*kgRgK+qz#N_i2byMkPYj`!H{B{_PuX{i6CN_R1E<4c7@M_Lg&MIh2By^m@4hQ4*!z{oqL*o@+ zypNA6_4mR*fkCCxr7!Pq1WqXN_AEGnolqxfg!N?6Qc#^UuSLF{VxMD&CClVy-7i*G zmC#qlNiepyv#2~7KGQJ+LAe{P@+PpGQ5R@zo?BI-Snh73Ja^gh<%66?noE#SL1au< zvBstV@xGAlpjI-MBmd%yy4GhTmGHQgfBj2(AOA^0|}I+>Zsa5O6)6p?|h1Fc}W6 zw^AkEV`DL^kR3W_88(fX(%2Nnqu=_yr7fFHaJ6Lr@$7EsM_7wZ`m~|{IwxlnCMrsNj6yyqA5%?@FAW+z?^7CIV=t`W(R=-ZuAx5A4ERGHkW@PRhVb@R zW-=xn20zX^|6AS6`#QbShLC`IAvt#Da`qZmVIN#4j-}3_9OQ*qX;w+~MmxJQicK7r z5*Yb{BLFZ+-SM&-Bc%wa^NAWJGBKJjjIGyc;OPw-?X6(&Fd>Qgv9{F{X;vW26==P` z-SY&*l1!Iej#F%nDZ8nO%jRU7^p)(|H5Z533lIaRyw(WgHnpio*4Nk6M>vkoB~9vq z=IY#3yedODiK_<;=-3un&Aj6zL$01)HGOHMSlEnLL=nomi3ewug7LTBj38 zU^y2ZVJ3NDon2B-59fg}?HtxspLHx)F%y!$B?OAx-m3el%5ZJ?E(WSr2D2g+3n3* zj*7xmHL%EVKW)J$%*J(-T#-3hnR)6a_E;vr)!@dkhsBjEJi zoele`nCZNChly|2b#zCLxX)g*S4H!~>v1V35d2|f=GLzrM>pDk*#+!k%p8uz7(2sw zaYlkx1~=R7UD$6#*-A_^COjR_&TeCIIhpLl#*dgTBP5{&_Mx^9etz22{5i+sAHJOz zU@drLW1ruxeZ#JK%^fzWLZLc@N&oEbWz3=}3joDlc-6ay_Qw10_Gb03EP(d@itqlP zg9JTK@3HcK^oozE->hi+7V2~1`+vGhrgmjNHVs--;;m+3GtlvG+5er#QIdm*q{myw zZ(jplOjhi>%Woe2mnI%Z-(>$mM&C8pN1O+aZmDiRh+W5>J0+Dy_qDB5%%UM^M;%qq z{AkE7PePsp0%JTlP*-zj!|MQxk@%wK_67BGP$3aZZ zP+7d0PB;Egk+WSP{>8+bM@zhtT0Mt%l6Vo{Y37}dKD_w{(fIz?4nrw^aPmgY8?S$R zu-?JvWK*iE;pRDN${4Ef_=&F&Jq;qH z>gbG-XgJ&cvp~gEO5E(ek4`3Ed)fHp__UiN_p%|D7I6?Ax{ek0*MsHxgQOo5+-v+If1A3vjuIC_)oFWBbBuYcyR zd(``lUd6Xy=49rleC)4CS^t(6jY!+?xnu6VpGUOo`j7j+-679681eCOBpZ!=M7mg0 zo_7vOymz+u`D@0)NXN@ERu9m}eK@=jF`NB6_vcFghXy|2UesJ>{9lvH-(HoFzdo}O zoAn)L7V^lW=j)rOd{iFOCl82C5ccRhT$R;l{VVdz3%mT5RlluK%F@AD{>RTyI?Qk^&eW4w0t~%l>Jntm2D&17e~1>t1lGm z&6CdhRIg6x{${npU>asjU7WzbMKzMJzqcaruV#1izfxJMt}2f4e+L{wsG>pU#n%{2 z$}W<>&r{-G70v&9#v}-qzoZqvb|W7Y)mQS0c>ecFj!gR~<()A@q=(zXbJDu$)0OQ2 zgnE`!%Fn)RA$OXq-wU}rY=SGk&1>@4k^W9$KlXceK8L0yed+ss^5E^|nOnq)6+uzk zo4bFqy&F!u#{STQRo;Hb`Q5D2hJiq9H<$)q)?hY)N0N|K22+d3Yk+Ik90BI&Fo54s z5ns8kHX#kF1!38yI$6<-4{oLQ0XO+MP8tL$gaxl>>%AD8U!NG*So9FDiM$@UNTWJo zz@JP_Ew+A-4x^U+jmfH}_EaCi-j!X0nBV$|G_TQ6))$Pr95I&?>YmE?J@L@jGZPa= zSr%@kJ_xLzjN|7!{?W7bKSRU3+L&F*Tp@myHY1kn_%YNeInX1|njz}WWJpAoKUytO z*&7i4j;PZ4*~W@(!Uy|uJJWe?Iz(&^LZP>Mh@AxN?e> z!n|l{p36SKtyKkcxDs#tKX!lxkS+~N^jLusAaF>&xZzhy-*{J7 zVBXtsMaJcXCEv-~q*X!U{l2R=W5n}WeLX>kpC;%5jUp7-0bcy@RHQp=Hjml3(r?*P zLj(E9a=f9zpfWZGDe=G$3^OsN-XrgS>>zIw+qy{LRGH$r(+=BVtxp%%9F zZmO%e8{@^o)9anWv7~~j6F~UzLRnWiqgL?`kJ?OBsTXrH5~^{(_<*B!3gWaHNA3Gg z-;Xc01gi#x7EM}>Z1_|XrPw$O*`y3x%q|h7tmtw}&%2TZwNy&$kb3N|u=RM*lx!!11uKKeM(R{EgKQtKcg9?yZEF(4c$C<}6LO%$J zsyM_Goi~33lMl3V50LVvCzbH&7imElic{d=(ppls{`|2sQQXS$`RS~z;U-Gb8ud!v z33b2GV$qj zLx~u#Ff7!qKv9S=i8G}ia>|EP&YGos_R{flM`klmNtpYY-NKlWH!86&V8d2IB z`xN=%IY<@3a`o3@q6l^&;O;p0K@Thou_Py@lA*!Gk+IPs@&LQPgBt$O)vT2ym+h;>h2t~N)l4u5%E_3Q^2wwp znm%6C4AwMYTbpGmLv!tLMULKitDCCPG-h}O7{$`BkZx69wTm8)GZ6A~_Sh1=m2!}j zTvLf;DGBK$Do}cnDwdgQ3SNcCmU04gN}6ri>OfoCCKz}iULKYx)puCeqiq(^mKQ>m zxi*KN;M9O3PP(2i*l8WJMO|*d1u34EcBL@U(|R35Q&CRZx^m+&*P?Vr=1unQjZP})8}A+>Hz7iYb6Dy2ettg9EJdHqU# zg+H)xY8K^kf;Byp!AYC4=mkJiO-1SEqI}vMffaKRvywlGO&P8;L03kV5CRIFP+mc) zqOM_$V!&0&psM(%U{0CphJNU!eo4bZ%u^)hoU&ppWxos{j8&xi z@a%+@cb?9Agq14{qL?mccO3HsPj%17N{v9bw9i?q8P@-Kt?@neGadR--#B^d`ZsW? zq58aziI|UrPLYGaDk63TOCy~{gEYkf)3tI74JGc6$c_ zht62lutAlt=S*_l-?ow9Uq_)w>w<9&3mh{9H`&h;&VW|IjReAQ!(-v1IBmhhF#awXm93w{0%dwUyJ zPv zICOAtC-nFQ7B#Arj=kSjYT1p(?2qm&ACJ2m`M|PeLBGp67qvek%$ z)V?AsCj;&gaGmaJL<|#-*rz>=R2<&(>Acn7w5P0J@gw{Xjb|U1=62A)C^5yzX?%g3 zTKU(KbJA^6hOc^|M=(4>m-_P_r-)@LZr+>cX3X0Qkds2?aIOMg_{HPafiTpe3+Pn< z{1Xut*$jEdN;^X}{AZSjM8LxkyD_dubHRHesCa9vBIyFIm`7n{tG=PVcw&nV`vruSqhhri)u8o>;hyF3d$2Z#mkv_v#sGy1XP!y%@;VrW)vl(%l4Z=Wrc~Q1;hk zEot(qUUfN?Ve49bd)f3dbA*z=R5X>FFd3xtfsTZC)?uo;X14mL7gs@rqP1I0dU4Y_ z*m^|L`UCS4lrLVzx#*HNKZ8E6%BW#wl%AC0nbUH#;vmZr>LaNhaPR?6K$=;kGs=76 zCwIItXvVcEYuz_^(_092FBIerJ1N=4D{Ztv(mN<4#uq(*aw51bs}HH{tb&Wj&s=JB z@1PXv#k&;=-4*fIS|#lJr^v+8rTSgaJAH5QOTX>J8pX;5hj$he!{?}?vq2X2UDVIi zHj3h)F&>e8osarc_j$x6rvthN1;nD6>q_n z4>L0)=T_-+pL1wTdAP%*SWj2|Vz7^TbBKP-6_c^QNOMp~W_PIWRNG+fire*7&1bWE zDT&`z571R)>$FHqr1I_P7RtD@Ygu=lZARJ^$@%r2AseWQEd=1)8MEj6?@rXBS0cXm zi2D8jw9zO`Vl~jk`F)7Eq9Z?aIi)atEQaT~S}alOJ_hP}c#RVniMkV{Ms0|R%3hM^ zY`k=APwuBY{#j44@H5PsNP5+iLf~~1)8se)Cxc%_j@Ur0;WxL&Ozhd))pXt#;?zl8 z6Ma#IjXl}Nj_f}%?gRe>U^YiCygks>5Zf0NR!tzW?18|*GO!{3xvdXLkc=ej3Q?7G z8nO*peNqPQ2ub=+zINreu9pdw*!FxTCh+Hq`TF2x{pQft?iE!luakJYRbN8LNIF}knHIkj#M&@J$1^ORLU912F2Hwu;a&m zf@+n2Az6L9bM9rp_k`2bA{?K>&5(1nA39WRGlw##CfgQwzU%0!3>sRpAn7u(mTqrq zsdk54!G#FDN+)UdW5O=gy@W?zOP@l_W!_3sHxF1V7dB)H^`US}LqyEYczy;!A+NC zgNUfYD`w$}0x+0=&Qc@lw&q)i?jCOQewTYCfVROm|LE_sfM?X<-P|W*guDz7*l%NpUp`N>!8^YK5LW<;=FmMldM%eSf zr>E1_zDgiMjAmugtZUJwrPk0lWu5EEaGCv&S#tFL-?_B&sQ8m65vDZ$iv@4J!CdiF z_~dUiJIr$19OZI9@0C*#H}c;2< z{bpX(;$!S{T=1{tu1sxDBhc-Eg0yZmJs3=Zn8Zd8NM@sx^^a?l=83q)VEPT15lc%6<+o{fnoSMtWU)MYu*L4i8`bv(B96 zt?B!GQD9iK^NyWa2n;J`$n-pUL8ihe8XAlL25iVuz@#-Xajbf$r@s>E?;R{-($&%%Y&agB5F6x|-C=L?{nkHA0juT8^TzjiVa9}x z?8nrPOAF}YZ2w}82ConP;9w~Teik!+TA0=F?jw#6^Au$fCE;Go1dhBsCs}xB_|5;6 z#gG411pcdha;h{{{Wg@${#W`-6l~n3^(W`(^nWjMPHA-t#o763a!~mUjwWzT;`-3b{p1B@6rnG5W+V3HXXB(1tlev%Qm6{F-qPNR)6sn=O zW)}dhX}_7s?Xs3&9^dgq*A39tEt40eSDO0O>DONA^fCjt)Y45}qYb$*Wa;qMxoNOo z>gz+u)k>h5t%KXO{36vtgtdpVZe%CYQUt=?Uoe@BKX+Ez>u+BGj_JyWRinMf7f+_O zzp5ubQbo(mak3dNl{x1<+kCOmayG(FMS1B${#%Z_ZfWg&kn%%WBjx;*a{)Q?uT^16 zZW|)!MQD$kJ+(%LtGiCxInx+q(rVxG<2X|yymRPgl${ftI{_f}LQK3X;K;@xwT)8hNFL6@NuBWN&Ce zJYA+G+rXrIXZ1Odl|9PDem+E<$39tkUd@w5WqFNbvPeTzC~=B#$C_uCBgsfUCK+as z?}pbT)RGW>c#so?C}dK!{2*sBscBa@4zSQ&?-5xz5BnC%C&;ok^>gF=mX8r83hDyq zwMyU4454mq$C!b_)U<)d)a06o+urCwC+u9d95kXM78cd^fE6Adn5c18F z@*-`$CKV}?IWTlcANR_ICv}*J_yR`xcDhYtcLo6U;@8d{cqBSL(?tc|?4UG(+#evZ zwq&(J(Mda_eP%GR;Tzeq;n#GMCjF&(0~Jax9{};sQd9SAU^1qSY3sw8lZJ>-h0SSQ zyDlYMqWMDBu|OA=iqpXrKXSmy`i@!fcsgSeA-Px0S>V+F;p;7c+UommZ7R?LMT->+@VOJcyZ_c(%#SWz2}@cbC@CQ*)t(K zdm?-Ne`{Tfq!%>iQH*YYsa!HcP2^9lL?2}T-pFwj%P}nBvbs+gRxS3mr~EPH-ooz3 zx2!fW+0@6a76tv$G-NBP1kg@*wJSt!=5LBEzA0ILC*TWv!e?WE9;KGROsPk9&wFO& z-{^G5fqq%9k8F2X`Xx`&=G+K_*X54dCtLM0e?(Okq(i2SeE~*@&f=#vZ!`v;^?;Kf zDhP=m)z+w%w72M7>+0rxHysgH9WMlI-<{Lkw|I_glL~D&-o*L`TH9IDhj2)m72BY( zZ!P=~YXS%G6lDTw-_ctEGZ>3|l=dqvVhI~vF{^%11Qw@lUn;lp?{)1d<{5HBI&9}d zG%B@nl7pFtqkFF{SAP2pD+in`82XO76Z`b%fL=&=O@(g8xUv5$iIHJm!yyRjgdz~B z#&rp0wX*S;P)Uzrv#~vks(>)#x}>n;`d5+d*c8S|?-&GUe7PxGy_ptctK)OBE7~*} zSf+6ylJRWg-r0>XR17a4v3rWE`}s_+x{WitN= z`{#}Tp~R%5l6s=ydoRN6+Ei`pGn{(iS88~@k&WOfzN|50fq7lxM|g6D-(owX!O2)m zY-0@H;@1{EIunSBp!I;WW-XAQ@n|GJ2%J%Tum@W=JHfq55SUISN(URoA8XU{I&>Mt z5P-hwhn8aN0+J+j8o~!+9K-s2^$3lfhJm(Yit+vlU%qxv2$jthw%-0uu$hf3JrDSm zqTBziq~O}&sbl$bM!DDQUaCD~7GhQ~5^kLWAtxKhMIoKy$w?)-IFX-&0-N^_rPUS> z$RODnf>T>k{%ex_gBr}UNBMPKevVk<3nPzK8v$L#topHqo(zn*O20pm2{mgp5HG|N zu0ienyXkJojPG6sFjg6!S_b)1`>yeY(OYe6kONrF2mtG>mnz4NDEpR0gQ^H&q9c+?7;`?;JB!lBX(Doxrs#4lV`vyMox2*( zaYPn0^-x9!yWhx&o?e~&EDi_#o%hmI;cMg+%TP=NK4?!_$EIaMr6u~{=1xh-+#v0k zBGn7Q&X|;uLubHS)Ds918;|co04&rS!dONXpVj^HXr$Q~dv{2x7o6- z?rh2RWWnbP`>Q49^eFjv1tNuat86X;QR;OMlZq#(#MVbaGR7c!Sy>!C;tt2OMjTMd z0s+1!glrq^Fe$m1YV(M4(;knZFm*Y>qa{R(k740izx@x`W%cPLht_lvA%d^ixhw)4 zz9e$NbJ-cTrNL{a&cCy5P62d&|_#< zcI~+N-d=hDc0E(kI#(tm9b?;zXM{d;QyCmLl{r9xh>Y>dL<*-97D_xs0 zro6q*EQv0iZnfv;=R;h3;kBXS`{D|pGf@Sx)q0sAKTB}vTUbw8N5U7KH^BJG0Z(-b zANp@Nr4Qu|f8TmA$Rsc)-PpSQA|aVmoys(qQdppH8szh2jquW7JBo0IM-*E=4VtLl zc4z5KDxyQEgWgU7`ZFy22Z7*0K8xXJ{Uq}(PfU2gJ9aUzUR|e57VCmkQE=KKl2-oK z%J<|sd{0m`T*?1nn8puowf9-XmwCRQ1wb@(boB}+Kl|T3di9uHqS<3gWEWdI7noR_ z`~#T4kf~i*VX{fDV*yIQK}ZoZOcrFYH1<~3+(RhutogOgXN9~EQWI#_=C{4~`H`os zP8$b^D7fslu9{P$X7pqs#gKr zEMrrV3!5G#yoHJ{#NBRi%NHW`7kr|5-E z(SEdPGF9BRR?jEm$OmuszZZ*1d)EWK%xzRJ9;(#y^MtdH*-c@4EM^)PZ${QPAl)V= zZckDA%!AN=?@ws!|cd>6mp=M39vFKt=p2oScG)f9o( z7{cK0n0(xp+WX+yQHmsbyBQWZmr|amUe-=QdB2z@)jI)+sIx~eUcgx0@z~QTNin)> zjQmq`Toz8y>lUQaxDB}B0uG9S!V)Sa9fLvoIt|osx$Ig=hA9?aspaa`+<}rxIJ?B` z1jE;CZd=xJOn{Tu_Zp=`*4y8UPu2m^VN!v7l}5QXeQtKm(J|m0S+BHaS|RTMW5o)J zvL4G#G%`P{Z;wvVpX1zV4NlGpC~UqdMsXZhs#hx9*RdL$73mkB=~qH;^^GFu?~_lh zmvcs|bp^d85lQH^KQ|rsahLd7J)5Q~Zt7iZB z`#08(O5X`|J_?6(nSAe+>TbKGuR1uPcvLE>bMfMS>vdA9l~WPomAU+j8VNGbLgM9T zDv8D^G^yA#2JvSC<#^ntkRPb`$rHUdR9x_-w1e>CSpPW6shpu4RDy2D3rV8evUZLq z24Am{cx072WruBgG%gf!%M{jC@A)h(>RQI)KP2EV0hoK{U=7vF*yNrH|5)qh z9w;YkO+Mzfv~96H9V?OgIM z?wbDFNr?^tv}v-cs-UZko}*YMe)S`pj@_qi)iGz{0*>5vIu*vaOsuxc%_7}*S#4WK z>aFb@OAOOu6odsG&^=Lz57@Gpxyv5X1-F%pYndpA)tnGh7||xshm}Xnx;Cb{LPWw+ zfy`R|dvVpUWxp^o$GL`gri4ddthYQ-(Tv-%A7FUh@1ML;xWR?Tm8nHM_Uh6pRlS;& zWjWMo*)YMJzz0HJK5|&tuiot3?%xw;eRg=>p^){o_X{v2uuVGb9x6ytDa&GMMmIVfxaW0NR7<(@4 zF!WrsO|Q?KT^vRxFh7-bZPcT_jQZ;Hq8M6S1eiAm%Iko1w4wVv1$i&nD|`DpJ)C~v zlpYA!j z(qWDJJ?wu^pyc1TiaHZRn2CQ&OWTJeo>>!kg0!}4L&u6ErVOLgG`V#60)}vF`xeQ( zY5P}g(}i2XzSoca4ECN-<8K6+y=wzRQuK^RtuKhm^?cK(X_n4&p?%u!bl1cFXCdcr zH!Jox5OBb)>b^Dv02Ec%qnmaiNhDxEM@WMU0g)t@9Ns>C%F5)8?!k>O9u$vE^y2Pg zaGfpQ9_^?jcWS*pZ_|M!I)=4FvV0gA77Q-7&MZcy`O(N`w4KB+xx}?j9p_JXG3{Rm z%qvy&e^bhoc~!o&N(*@=FV9K9NDPV-a$jQt@m28>5q`zEyARHf4xOOsF*%AXR^#O! z36LWKmpamuvNv>~@9^s1w&8#O0NG0^6d8bjVs!Yg%E!wq!{AkG5`wRX^?VISF{)bk zc#dEEp0p3k$i>Rn;he(gt^bp;qn-Ja$@|}|F0}fb1QBV+e=R?R1dOqN%|0{)`!2-6 zpYM!`i15&ac!Ks73?hfyhC4d(dxJ7G&sw(dkNri6R?bP1m!2trxxJ;Af$BNHFe$O% zv)U9}EtO*3uB>?i;(S`^1P*z!PB^>*NRoTmqz(D-(W${%S9&4KhnhPyzqV86JbEo6 z`U&W+_TxnD*jwEMCBFEY(8vncX& zOFp-F2t6ND0gs}6qLFJSQOQP9hJYn_gqHAn?h$2m%5k{G@Dw9X0;xK(VTxTR6@58? zYnu0Eb>VIJv}v6R{r9cYP}*G{E|J7bjzs^Z`vBR25HdDsGPY?f=bX`G~{ zHsMZT^te67H+5~W_5^L-EmLG063uQ*-{56wmKb`q@QF4ZuaIHFcG3V`!f$`5&A0zB ztxJs7D4!Am3Y00~!pc!cD5VN%SFhGtalTzfLGupIj`zeba3rUu6Kot!(`|U+XrPvG zJiI0s)L&x^NRpG9Aexg4tAa&rB}xjc8(c_~f3-H>cys(nYDHyg+z9 zh$2oK@+DkswCV1KdnkPQdH$=SO4Je|85}0{t7#lPlN9*Al+mhXD12E|2}1zZa&$d; zt}0^c>d@&kr&`Z3bpqK%@8Wu+VXzwbvnEW)V-C!snkr;`{Br;XIBPpicPrp>V+IAz zkZeQ|G_4n>+o(!!e~#ga|#+KTu|0#73*>ME=lR5P2SU1$+v;r0HrhyI z$1R#=;)1P(yU|rAab^pPwx#R!ZsOlR!t10|>c5+tXU81^wzZ?P;imCEy+A+Rcvv*% zGzgP5JE!!%UsSyOG^!e@D<&8u>BV3P8tlo$8>+`EB6;k!{(Z%m6QuLXSQ|HOGTowQ zq7Y;P2{#fLmg3<1Qq`evyvu>f(*QRyG0Os=sIDP{dUD&(cMQ`mA;a2m$j~T$89@gX!V#G3SRIC!_4&$d7pDoG}f^ zSj;vneokdkv=mr3FG&B=+@j;&gLNiF>-L*^^dF3GD%l2h9^IIE8|yi@Gq1-p^o<(J zC4&Gnpj6$D(=nIjq`&I)&2imF^-x=h92I?|Iy-O?6=-;zD}JpIOiK^euOY}@9;5M=Pit_8i?q0N-{*-bOEKW+8KOXo1;eX( zu!)nD!!jJXE(E(;(1<^4i<6B^#rG9Yq?5Aj^3x+SxJTl21q{9-u5)lHHh{>5IO@Eh|JB5U$I=eQ+YR2x3^x z$$a7UaPwQzGaV$B|APnJin!RuZixfqxhG;tBz2EVWpF?svv)?(y5Ne;q7h-z%uvHG zHv=)80)|IN*`MP|DVmW-D}lX9i^TxkBSSEBhmkXsMMt=v*|~e=vN{_)K&UJP#yP_D zNH(S3ks#=NQ{rhlYp${P!+rN*HS0~OobxB z7)OvGt1x~~$ks$&yVmL`8ZMzzQex|i)&UH1+U||oxr(0Wq=3SW2AvIkXo2A*Nm4`V zZ4Ra4Adp-f$&^_zhin8PKJDl5Z;D#+>Vmg=zqB!`qBC^{>vTlt1b}fVHTK-d&4hpi@@KMN9m}Yj4901ffORW z{t)B`1XAiBjN~2ulx6^slG1gr-aIYc;<2=tH4mRgQln9!wS4$?9XOM>^2UY=f8l1} zu8KUq^4a3@3CB&!4QeKv%Ja1YzSXX(QkHh?WD>P78vw2L@q_`*)K2B!)I!?M3sVF? z&2jrfI21Z3PEab@i;|A9@9e+ro(LLw%F)~4WD?nV0&&viOVktFJqNDJ9Tg+Hjls%p zT(tGHO!oLrIu@Q2&p|^3+NJf>38{b&t7u9Y=fM4@cFsH23Puft8AXF$Qs-fOK$gER zJ;L`yQqMxYxHn@hZyA+D#`=+jz za^wCJy0_1D=e_Kr7`a9)ID0FEZUq6gPO0JXffCzQn*rx>nasA-xdZmP$>C~ElG0K^ zyT=ucb{ZPy?7QXLzb+FdSd{|PZ3DgXwcHWj>>cl?ngm$Pimh~K90YIQ>msoWRmP56 zz4;J^6R*Uv7zHM^j+L}Zd{^f~e@*=wb04uTo#qQy7qyqnjAP1!PD!`9Jgo z`TE<8zDuw1^u5`nV?X0Al|Dw$H-go z@ac)zCv(2oovq92p@O4EHE9CH-ZO<-H?b=$_M@Nw|G1p_JR7cFpc-Gv~o{$*smq-=SI(&;U(go%l70 zENhwjZ3YIJnF-d~mwY6X?=Ys9!!>;m{93Ob1>+U(cTS|CSV_sdfpS>FHRI zvf-^u?l5@eAYxy0f8a_xdP6;?i(8}%7*vSe$uj8I`Uk_iS^gq3CUZ9E^ilO=gZVC8 zN^7&DDLPcw)h7Q}w|U71==uH0&2+5Rw;9ugo>jaMT7z$~U*9kwh#^~FdDTFCdLz7z za4kZ?Eag?}rQ4ln+Jw*iKmt9(uC$-h4J>+LUm#Ii`U%fZh%0;u-wL91>wniaP%FXy zvmr-B^z1L_Ag7i38cz zo&LelJP@Lr3}}}l<@l2WVb}j#B#Y+g(k1>sHqpcPXBfPHV-)}DUf#b2Hwg~en7u>1 z2Y*!JpPSccVx!(NVJ+PCaq+ORkL-x&A4^E&-zoYw1m#6?(lS zkZX!&8hx}PzoqH_l=EC+{TGdkRm&GjA89%N^Zjr1X)vH8c-1Xhe0D*eDio=vRN2(h zTnElm{NX6tep=ld0xkGS6KbxEjdg3j`)_hv$%Q3;sFcj-RT}P8uFiP^fEiG73@;>JioH zMo0bEGm`utpGv3m$ohQM2Ue`zA7nj3cf4eINUN4^=XZShb2GK9(yY=27%46P|vx}zT`n5mtR@mqh zUfQtY5Xt<3-oRDQGxmB0mV?vbIQ#wb2q92FibT*y$J1do(_(Z{aBx_S76W(v-R@OB zt=*Q?XWd;bEaT82)p{sHGv>5gJ$AcCn2@QEMC6f!Xo>-QW(G&G+h@a6T8QDy+wZz* z93DeCk2rrKZF76O8te-rG}CQfDc;RP z-E(Y)bK_n%Q(w_4w}vdQ4>|i^tdN!JcDE0X?(qLzF-)k2rs@zs(5J#*ckdSFm=#@U0KYc82~+b19N!~Ntpb^myr!4W z$NC1Hzz?r`PW8oDKUqxq1vhY2d2$a2U~%W0SbiKU+k_qu3ng&?ab zxk^WrBR=MEf4l!NP&~QyW)HhNQ<e^6Zu4v^b8R&PIa~sO~cU zGp5z-OMsuKOI4i9+hYLQxSQ;D%u6a3a=Biizo9W z&`%SUt8L^&LzYf4#llRPHIP!sSo73T^13hN2-SKVq)OiU%1XZZQm9X9we3E_R!Fk?N93+Ky))8F0zo?7yU ziS6hnW%?A0r|QYYRj{~t;&9p6uGXmx3>k9{am_3Z)kWheg=bKDSOuQA@0z1}X!I0{ zuhJ%sTlfc`6l$dBStL8r?X=$n?%qTF8m~D=@AP&ri|>&CU?}`*PG&)eb}J?_;goP<>dt7zeCy9&D>o$Xa#2HjF7Oz z3KVh);@@vthY;#Tq=6TXRJf4zviW4(`(hMJ<0ng}45U*UU>ze5d%u_)y$I*qNoIUs z-Zfz~rKfR+MQu>Pv3$Jvq^8F#f3##%)WgnoYw~#6cKEE)HTJZTpGqa!FbwCta#1zf zK4T&=MJW-E?pQz%=yp&gEmX-ZrytsSQc-cylU1v&OGT2deHQ`k6ky1nphx6%Ka^n^x))L-=UeIm$l>ztc5-Z0j_aSHeI zcK-(hL8Ai9JIsde5@#_^$4%|UJdH^$Suz5So&{75)dTaax|RA9KdR4VbPF>nBtUJC zvp~A5MHlNNRyOR#_TR2Uuj~yCp@@=ILD(VrX+n{ZQdWawN+9-1Z|(CVng-+D(#h|U zM>MZtR_yZK1zc%dABa=(Yr-BQ&+JqnOoSmp%*=kW^!s7#+SOzwoe>ab}*n7l<^}2a{^g*?D@*@!b=HN(wKpu_S^Tf z?h+bsZ;}V9yDW(G8`g(gW-nB&?P##T;&3XJ4d+Wlk((% zm@@Pa_>6X#t~w_&nNs}*?Ay<2`B$xvs{%;wi;Mls>=NoW`ZQtK{AAwH6hr*SC~VY% zZ#^4f!NFq0^Ea~2Bz5KA%|$F?3-w8OuwsMGY^z$Dzec*|msG7+uUFGQ^+hNveF5mp zA?dj#HGG_??d7bRzG*UcJ#}qm=V;L>DdxJnmz=?g^47JjJaAg~@OC)tOm8opyaqvc z-CNjr7?*#Dk-F1FOd2KQzs5S?OC?1BHQAllJh3M^A?{Hi6j6uv*yx$-QvjPa=`7_(g^w$J~btk1p-2Tcc&BlQ6$1m9%2!;4(Kh zn&z{0XZVlCimccac@C|vaZvqYfAh;mfXyE^Q3^Kuv}hR@DutJoviHP3q$gs5SCpy5y6ymy1r|qdI=62H|EZ z5s1P=DC00Ig0uw>5|F0D1)OHf#8Wx1%cg!hQ47X-l-fkBPtrkxCEqDCtqbZ0Lm%>sd{QHLgC6KNp;>=#P3j+?SOajpf(cRlJP z+NBdG>%lPeqVRaOSjo zz!LpIj*|2&_tT#AP%>QJMFex$-eLL+1dZJ0kC!|KYJ5jh2qnUVna>76Fm~{F*mLgB z^cQr+x`j-rhV?H}+5;9a77i8x#aX*{!?E&g(rm~9k0krA&c+e1gwMqnv9Nw}G94CG zt)Q*rI@3`-v=G*llvLTwj@sVsb=-l=Z}+M*R%9l>$!pZIrls6*r?Bpe6M;f^Y+g$J z3Jn0iix+&%%x6r;w6SOsn`RrbXS&ZpQ-$=_`+F-&SrF@^_?hP9zivPR_%A^rQ<>mWIHWPrL4G^g1Z8HsEd^ZK)@b`#tJ=9yr;qeRYv%SNw-%$jj54^b2MmAbbossWeO<0;FfO(v z7>Zkt(qBV@xSi^XhS84=+BUbhXhg-@Pq0hA+*x;X8;XX2Jo%ChOTZJ9}2C;J1^ZIkVV{Q=a0EslslPlB}Lg zbp9cWk8$Y3K;f&dXiq0ha8~no_Bx!dC#yQ2^=y;a1xoJC)3F*A{YohYbiyo`lg45c zD~8i;s9s6lc1|BtM0kyObkpvtYANM>U7bdip|w4_^YL`&3?cTbJ8Pm0I6iAoH?yR?+$m`ZXkl>Mr?~-ZzFrCD zI$vH7{I=fC>KZ-ur~mZ@%a9JtVK4Uc+{j2j0n_ZG>h-wzp1aD?+qXd7pCkD%Hlx7K z@9j6mVk2f3-m|Rxe+o7iU-8fMK5m;WAJc1s(98a3IPDcHqFkfvrWV$`@4gAKtUqFL z%qtC`A5E)sTo6jYy|dtNw@&Pz6Uu5ndm2kGtUdCXp=Wx!^trnW#39+T=e}8y`>LBt zd_O0hoHwP-9jem+iE&2csEyGdIxD!XT$>j*8)dBtw4D+xCpp*DPflAa;Zhm)-9{`PCtEBQHKQ7im;{9bu1 zww0h{=siN`qv$gT`lp@ z<*&j2>A%GD;UI>yReg0t;X89(kKSB1g&+o^x_~_#d!T1pz z+%AGXYEB-1+^dVfoiH*O$4|~~emoRl9Xt}ju+vEW(^zv%=ZFEsIGJ79{r9>40mc4T zhyIU8!r~`-H^89j%H&S`E}&fxWBsL**GkgUEoWq-tDEp_EQWRzvtR=u&cwIshP4ZE zKd&DLPyfYtU_6d%SiCxFjch<8>$d9E(FqldQpNhmMKHq5Ogp7}1wC8|y;VNf_wxb4RWZoC8k zoT@h7;ZkM;FAEN@7fYtPhB>bX=~M-ARfRlfX*I;(Z&Y4j;(ue=yqDb~Te}13zD*S# zx!k$7o~J7Fm`}wO8UX3SOCW-*%wQA8e=yFZu@{hSxq6AaK`#OwWVI51S*Un9766CO zCe(8$_=cs6XGZoWyIJOf1M5aJpVlwF?N{GHPzk!fyDzsmVV0`DDZkSTbhxS93}^RJ zNwSi-sSsJOY$}3#{zUeg`gW6|cH`N1TxawUB5VEZU}!mrWdn+4fjDBn4R@1Y(|m7F zjRe1L`|%*?7C$&;0UmjJ^>ztVRg4UjRT zR!qzfHRyYzrZY5l)03I%=l|rwRd&9#CwUTduYGr+G+wVd2$@j|&mHVn=#0S>u$`)r zwrF$?JZdrlqxlPByue9|V2?hTR)i!HTJ{Uo6UC%taBG|-J*JI_y;RT`8{l%DL#=Bl4 zR-|k@b>!Ig55`YZ$QX?^RXeG0QTstvOc|DZp8oR*F{XgMTzn^F^XlmlGd;rxBjc#l zoNVJLdR;N?@|!l$$7dA097sKvQnkF^TUrkyya z2y0BH>GziRMXo&CTcsXFl^tl6J%<2?xmUy}+yizMdOQZ|;d1Z0OI6-mHPzIY8%5WX zV}qk%ZhGnnfW9WC{5sj6Gu<%Q1&zsRE5~DyaM3_>xgn|#I7!E-sS*nZ%z!#s!j&iaFD=W_orqiU{%*{t8 zMirx=Z_L^0JS=vm5*uqW^-eMcHKvjMQ-h}!#*Zme_{rWu^QSg8w~~6eZA~8tZf&bF z)EmF{wyE4JRQDgI?3T*-km>NS#z9><WbqB3_a;yNq4|k|gf|xdz#Nv{f0zA}p9Y{#5Hz|SvuxX|q>p+8 zOdZy;&Lgq1GNMO8)vJ@2?g`F1XFKPW=29SX|6qVo!MgSz)dDW(=0m$sJ(?hu*s8&L zoR3=R=_z-cDtVk_{19oL;jA6KV%`E_#MGG8QZy`+PuOehO248+=^^8%d-it^S>G~> z7(O`h()A}B3`kXZcRIGE_Fett#_#<-)MM<8qRpvOqMs)Ne}u6IGxf9nO|sQ!A^RN6 z&G875LfOQfar6=q5>_Q+dSZ`Z(F=dwIENPTQc9qVd6wC&+cWvmSu7C5E%nH;rkRoa2BY?(SM>qi|Tz z$gKS}ZdYCT(O_(Tg}Aa4mhyR3^mV531USrpx27O(hu*1&a;E$Zq2{|G>(H3Y&{-#Z z4nVD-O-uP!hcIrTr;aIhD)SJN5I1X`HJ=U{mCiZbvmcxxtVz$|$17|6nq`}v103G* z^&|kuMS+wOqI=HW54(s7W4h=25-W)t&R7QD=g^2uS|9 zZfX^Rc{=;Sk=Lnk8`LP5fPaV+_3mM3RE|VqcXR%H9R2vq5wU`rWQy@TLb_O$Pb4+u zZw2hF7c!A`^PT=$K0X>fu`I@rYhP^U9K|b_h5k2!`l2_QGaQ67G#sAwReK>6THx^} zel(WfkxPH7grmFhg=RAVVwIThYF`*_I~rS+;Oz9x^|wR(uDB~@QHEH8X;}}^jy8>F z-!8X85`wfG3XqrjE{ubRX(aa=36vcQU6AXUJW1Lfp&E>DeO)+E`dwHYo-b);EZ z5w%ORHNwvn2dBo2{53n(E&fMqDrn}%!2WOfV&h6Sucn53ZEJ;7V9^Wb=1xzVEK%j^ zGII`Kux>efv(zJ0`8&+sO}#1RbY9MzGOWn`8D5v|xZpSBAb!2FC(k z98Mb;b3Y$KY(eTrCmyp?%PFD>DrZ-KoXKL71G-dtIq}7M%1SY84|gA3txgSpo7%oQ zY9BzM(CR=#`$~;sp1>q=x#Gk9Fn;A(xTlbplZ}(&;CRZ!5Lnu~Y#GWvoAYGL}wv%{xA-I;Tw$^bf{;^jOZR zaYgin26~Chh!frKC;4V)NA8qo@8=D-pZA!N3V(?X%1k+a!(Z0{mTH9DvR@#JZqyRX z+abK+-S~nRv6%nWK3&${h~(DK#5xsR@(+ehoxa-73X9lLSHsY&F7FajQS!u!auYuM z@X-cmv{fnE@17m%zpJ4{+;nQZgtgq7BgQy3tC%wp+cNCe5I#P?GoE3aE0(Q8UhtVw z6V!%LA7g(;K4e()wykXNs;G2i88Xjlo&la5FCDKm3}?O6H=G+{d>3Br`KYY9NVLy$ zrxgh40Z$hGP%z4f9ajr^{KzxYfKV~n;)#5ehxcU)MQv) zg!-vV%<{Eb8D@9=Lk(#upB}+l2bJE2z`Mj2ClZH}^kIMRD{tKKkLiq_uxKaX5ZK^d z@#Lk;FEnfXq(G>HUscReKtBJpU354QQh%tX5Zld&5FL@>`F<-x!#A}1DKV!XmYrag z*2v)otba=?ig~l3o5d0G^;;1o>{BO zi}8#_Oe1%=rlsxwNLj7m@%T!0bPOid2tJwd$W)+0K5+)j=ite9C+?=rZXXzwOsOV| zFxln;BBiJf^eu}p8fv&W3tsc~HMlv^TqRj&m^ zcbbbe??%F;)pt^rwAF))?78jm9o$m7yxkV(m|fRuA7tg=rO>48ZEI+kX1(qA%^!W0 zIq5j9G$O7D%p?_+uJ^j7x*F4VgR9T{qcS(`?Rw7d27hR__dSptaqOn zGNZt(@o-~{w|D|-ybwdVdjp@}=P_2wml|kgfx%Ch0K}G}H-!}PB{4rap2`aVn7f`} zka;Of?e!#=c6;K75DqLV0$2A19Aflgscp{3E%)1}b*{pj4hXta%bPUMQkP9;u8(um zi7DTq1&`5}KVWDG6!Fp!08;oMs}KTut&FbTd_;Zg_$Z~fZDApJ2E$1s1%p-NG5pKj zJ^B^a4*&8D<6*#|g~P0~`+~po#aB!PG@#-9hkJYaeh5wyoGIH?L$ov@7bo&bBEcx( zpWclJ326QI-=@vK6`Oy@z%QRih8Q^Sj?SPTaUEU5$@%1ex;PcW4lRNuFw`*sQ zHw{y88fAXsey<%j;6k&2su*Ss&OH-c5p#mzsC zx;;JIeDwenC2s7-<>cEKG`+mk(C&yXF_j1mYbI-Y5?GM&s4YjWU{L!Y|Fq`5BE?^TH&>mV(58wX zL8@t^bptaaZ$_Bs7K!mjHyzb_Q^~8(q@7+FRdC#6OiMBuQ@Xnv1ZTp-80Vwg8R z!Z_&}iQ#K^B^M9Va<*}-1`b1`phhjh zv`lp*rRs^h+=SrN{BodyTpx}mWnvu{Q^Oj`JT%$k;B>CP&^N~P+>e%3JdX=qxU-UG zdl(y~s$n9U#W~}f6<~H=axPe6ZKW|R(8N*l^OUsxrYE~!znS4DkW?9I=!F^isF&F# zMQ2`e8^5WLYIbfsKB0Hw0x#cAIo{;t0Yp&?@=u7InT z8f+{}fP31GC zA+_J7a&}W&`n~#O+Q(gR(^q?pfB6SGo0f|Z zOWXfucgg$Ez{dC$rCkcL3?v88m5LtFCg2>SLReGszdAlQ&DUO?MgCNy6f&B>+5FwH za$C9Tk~X46f-A-nXSJ8mcB^}5x4Vg2{0D=)$A=&ZQ?*E=%ij?=OSB;>?!x8%*3r95 zbL>?F>jM5m4g7ZXIgPIBN$wO5Tsd~Ve5=RssebnF$yK1>bbZ#q_w^TD=uYbEXF>0Z zD&sOOtZy-YM6Vp&qN#Qyxo2-bGt_egD*NGyHWr`0#R<{oQOqctpZ^|a=%}24d;-E} ze@8r=32?$CThhVo4IM9-Do|P+TFTnl*S-U+E-FMx`V% z3}Q!zD8Enn%C%_SNUri04=gRUePpt}FSHS(DwmcWn!@pq18@sis(IEm)ER_5#v}=2 zii2hXq#A{ghWtjtrjCpw>!fLjqZ}EZ(Yormw4=1y0oJ;Dr^aVszbN^^CHW#ksu@Fz zzWQmRT%jVsRZ`Z5eVHhS3|KWx?2?>b4ZsZ4L087U3?Hi919kdw($jniThA;ps2>$= zmDJM6$?)`x^=uW((e-cNY{UcZ^zN;lQ=CURDHyZW^5>=>*njMEjmR5g8)qRrzNHIB zXq)vrgf=c66%Chf7#JoMtjW?27aPGCX_@Cfpzte=Bt}bGzHLw2*jR_Eu0LY6*@L|C5#NL~E$=q0iI%v7y`4^7gJ7X1pPdGV9GCgZ}dz|&TNRMT&v~9wi;lA45 zo=N#cvGPujyn{}$VFV-Zp{Fnd0FOSxY>>!$$`p#knSAk%V~XFnIC>$OqR_Ef3s6RbkP?NcAeF#d;ZhGlm$c~kI+8T``ISGZPXDHGEYXdn+ue7RJi*(K2Uh()?edfi# zb*7-`qJvLP(HU}CAhc1H4hiE-Gk_SZ@QYHm*Y~VT%Agb`rMYQJdF(v)zpG@7HVT*q?M$HJPUpN4l-2 z)`HXtoeCcV_qO&P7tpRIsg!^e!6ra(yp31Uuc=!h1`2n#vfyS$9jRqzDisc;ec?k3 zoJ)Q1_-XmVsO}72(|Pb+#**7^7>Ko9i{+;gZi0|ekf-2QEy?A3^_LA%aup>8uMt6JUGs+Ps!(M(z@Nnbl6U@mv`!=#XL40DAxSd zi#k7%>I$bh39<-;kB4u_3<~@@?Pq>!5*^$ElX770ezsaD>0a{UJpL}ZyhfjC$%DWq zo#!ThS_d58YPV^PmTrReGbN~{yhR~C4KtMYL>jjiN+y}U7c84eI#*6W@jX#{4RC?; zlgEXGSdS2YTUNUYWmD4n#Ktg2!U^`}+g0Y-%d)dq_oufD$L(SJ$JY*6UOR1eBgC|y z%qu`pSY~9uFQi+czeY|OESJJw#WB^edD#RqEL!F{ET7tPj>~&CH>IYiTKmlLFjV!X zC=`pbSpR#Rc^CHV?|_EG;s1}bw~T7Lf4e=QEwn&^LU9TdcPQ>sAV_d`D6WAbf#RhF zife*9gy2w|pe@C%xCALqu;Na8a{uq=oSA24&8#`Yi>yVG-&%R`%gT3O*WRBU@#oW~ zduS8TUMK#~%_RvJsc~CrLAm69vsT|s>A%#%DN(-6Ly3pItq` znsIG+HO+c}*!D6T8qFl8n) zK=Q({71}7!Jh(+EKvDEyG&_QYkh~}S;h1i)BCBKFJ~r=W*k4O%ig1B4FIH23^^f6 zEzLjSeh!~>FGe8GB;UJ=+??zJFpAm-Qc&)F=6;?0tJx;#lG@9va|i0`+Z@QT$o%R- zQo>ePaYTiq3w_V+hnj+4O=w0erk`t-hqHT69kKR~3M`G~)9&74c8=(Z>?L z-MO?#)@2FVgqLAQ$5F#>R8kU$)C_Dd%Ya+zWa=Rtd*^7w`LcNfylR8WHpKa$W~LoQ z$6y;>MfQBb&c(Vb?<@j{a??~*Hh^9)q?A!6aP+vnWyVZrQ-xAiSbllL1XqkzQFCHu zX?1LO7`a?{-`kh^d9|_7cIkSNsc)Z=?q8ON z2vErvBO_IbY!wFEh|MrHGP5N{(C|Bnxy4ZuGnSHQgd0>Vv3n2-e?hW!Y{Vz7BUSvI zjeifl;@kQD<|6{$Q6xGAaWNFZ<=$WY!+v-;ySo9i@)0HUIVF%AmJ-hg47Xa7BYLdx zuTN>UJYBD{lx0(d&S5yFnJQZH%AklrukTKn_NV^-uJ==tXGV$7y1;0k!3_ph+16L;KQP^ znXx~7hvEDCtfSL%juyl8>sOkl2f|tk^mgw5(2)PuKK@Td`Ty`A!5fU))15rC|9PE9 z>65sKo&B5D^;5oZAJjw3DxYiQtqJjair>{Dlw>zw}@A|F+p?18RdKJaN_ zv8>43T$~B7_s!~m+rfB|1o!IOiC^jXeU9{4C7G>=2}#SF#`;k5zb)(jxqva~KJr{9 zytCvl1{a1$(ury4Da%gBm&N;FihrH0xS);%G(&%@X?AtoZFwDC$BBWFl|{&db$2*k z)s`D<7~P3>=*n_h>`gvB#h1YQr1YZ^6te9BJa6pqU@l)@-P*Z$VXvj;At3YUm%?j? z^{abn#rr><9}IT)>u2W|f*EYuhHva!GvI|;vL&kJV@6M!DXEaQ>?VOKyfj<@TaKQ- z#HPh3VNk5Z6zweG1l-agYL8h1bN+gZ$Cw&hLii~fB{%9aZK8Fd%G3U;gV3?DUh*#~ zvH8hjYU@0-4c>!NuTeRs5<5y8>N!-L+d>2)?P8eFYk=KQdqk|?OPe>4O5kU#TbKCc zPcv2}J@L<{I>>fMGlc!P?i>TZ?u!zAeMVdDYqpfJLL`~Lx zjlTa);a+uhSEe@_Nz8Vq;C)t#FB~mpM^ui`vnbFH6czF@c)8Bbzz6QzY6=T0^L#DE z!;cAk<;S~k`)*O#aU6a}32j%~3i)vRl=1Y8@w7s)cknL;pGw~S?8S^l5qi_u1)Z#| z?Ndrfr_}#sp_GAX8s;gx#WF{ZIC^>(!mrX~dv94f-MU7|Ee~gjd$;J9a`@dsJf_{^ z(B9ULja%6?uB0oyY^(&22eMyehdLDfk(m+;O&&GqqO|8`Q(KXGq!&G)Ez?>E&@#bU(wkf32H0G~pYvm`Z+bX)-`KCUE*kc9_WF z$7eET++dH9(=Br7V_pgK%AuED%qo4xNz6oMgGqHS*<(nfz_?W^Z%WVENW~w3Ihr}a z*+^k9mC(bU8FEQT1jZa=*gP^DRPS3w76ZPYjYNY{+kki4`&WkM?0ZyPiqd)0%ATm! z=w?g;*ih@ z;bV5Arg3P_V*8)|Et-(bh5PBo=dhRZyEPis zjPJ4BX#gTO!(9v}hPt}Cdv$yfewq?+q|xpFT!r^V+|uu3V%2C9(sB8U)k-;wU|V0wmt^6L;{9cZlqWY%#)Y&7>N^81N;U&08Yu1#jPs{>qV zwdMBpRLtl&V$2P*^X`bZbGUXwkykNW+_%MFx$AdktWDf!L)s>arq9EN%yspuwnvjL8o-l?x~*6Jk$JQV8JZ$g zuM~;lOe%?c$y81eS}^v0v-btu1v11@ILb`xC5e+l6c2b^bMXY3{zZ1LnVp+3$=;qBF(PeSKldigx$=LCFhx@No-O1a!wbmoeat&<%n0^tWJihOs(9t zr~IMx{2V1a5i*3P=+^w!sQ%jIkydmQ05>Na_Vau*`T#5&E2TIn6N+{>=twI3r=l8n zI=dylZriz@9+!%o41|VXR;-B&RwxRo^9Kb93l>fJ$Xw}55vK>Koe8=npV zQtETxZn}5f^Dd%`4gO-Z>BM)fnS&kc$ON-@RE8#3L#Hf?aJQH@TkLcM8tHg+Jh!Yg zfc6|#isg1Z1z=tl03j(f_-2fBC5I(nE0Tm|Z^qbz-F*QSyHIf}^nmI*Lf^Xxu8%IiAazPD)6VC z`0a!>eMU+gLkI8BieXR#RwNa&e~Hsb-OKONtdjGtEsKsk(8lkxV=jkrYmFFfV&B-YAJM{&cTfyoSVUy1_?sOQmoA#c1H*P8r@XH29s#dv2kJ){9l%mfum9=S5Wy zj_5KRKXCe~Puax5cp?XsiZmG4ID--}Yw1|Fn_MKqy_1=S{}%(Fe)g5C8CTOH(lUA^ zA|rWtlwrnM-SKtRYMtj!Z71bW<(HB=9*7G-_jxqmUX4bHKj&1;pGiLp!#L*y;%XgzR zieYu@ul>j2$JoMZ*PpJ~rVQ;QxnvUjfW`KY`qHP=3gKNWFbN zY@8;@eSeXY0pc)<`6a>G3p4{JBaox?nGzf)xmMG5*Of#5{13AwTUuD6ZDDOKFolVe z>O8vd2y*=$J*8P=fh3;RSN(tiXP%hB@liXd%I7;tSv!` zUlWiEj9S8P!(5pwEZ0`&O`E;rZ8v8LwT&WA^XqTMa44@H7ctHMZ#gXX$XkVfZ)*I-@Dx3C#owI?^F5A|HBhI8 z94zWCN`qy7JKWTG{aL~4CtsYyp!T`0T!Nv{4b?^;q90mpIv(_u6vt0UVrZQ7Dlqkc0(OTa;= zcnl_s)(&2jmtO{s*9WZOED8K}>=B4BiENNen_9-q-1nYe48CAqmHJV$fTMZLho@{O z>O!)4V>hS@`^V-Lg#+bnUuVm18iWNG)fvh%w7ci467r;;-OQsITJ z%JAJ>`aNFFGO=M%efQdw1<>xoqE)x zqxF89ZwJDi++H)P22Tlfm*j&APRSBoC(;o02CSngw7>t1k0I_gz_m;F)w zIVovBGZ+(x?ECqzhnEhyJL_k>`}d{_I_IaQ^C!ws<0L6iPL0~tcD>ghPEYT@OcSS( zPU9Vuc>FFE)^RC@c&$~ zK$JlOeBvKTW@a}kIUWAcB#35O*H2%qZK@MCs#n&At6|B9BBzF5 z=))WCy7_+Zg>?MFbnm=8@8rbr^SAzw=*F<-=2`uPdE1rC{{Lh5i23oEWL(q^s$Vo& z*F+!;L$3XbHLm92*_n)+;=ib@$8n4uhi;l<_pTv3#F%Wt^h(wujfYv0f4xKL~;PJKNe zBdp4$vE4BHIwxog^ImDiG`^(*okGvHo9xYE+$A}Q?Few67UjHPcmPTX1r}T6=H(*g zczHFkPv0}NjJ;s&@m9)`X-w5kd`LN4ig+c@PpaV~p;5+FNrGm&`4t*8us4xR()i8n zLZ+%3S|XmnfV{F7(Rvf=&NdN8b}JHewO0axL)qg;9Zg>Lt#Tp5yfxk&F$>9+`>?sx z%>^D&*maD_Y_lGW+u7)Pc6|r}Gb(eLErvd~wy2LX)XeBbx9w z(Ct)Zkj5|3Yx4v6(=D7Sv6!7@O~oXbsnpaCLs`;f{wuKjfNyk_4ITj^oHZLZKB63l zNm=cYX#sTk4fm5>{8YK*l=W#_*VToV`nA-Gx3Zf!RLuO_T*v*Hq8bGYt8n^GVdQUG zl@gbo%5B`W9ryN*%VX?*x7;kWnfHM6gQ@})2g$?*aH^!XV~ zN_>3Fe`O&iS)@tYu6kPaVyXRnu>pmp!I8zq&fAH4tnh(7`e*!4j%Nu((6SQUf|A74 zNp4eqwCyN>txv=T=|dHLqs zH`6Fw7()(mX5WXAmv-&vQDWuKwMmysh6*9{BOOz-Xbs#kWb?JcF9OSppQXlp_-Iqu zbHp?V$-?QZz9sgmX~=}PwAQdG+Vqh1R(~v_@U~wc?Cn398y^EsnVLZCB6v;w{H3_j zATNZ5cD;bRxfkLgyH4FepkWcDD+*RECvh(NS=f#zcpnE9mlytYASlCq`MFa@%S)A~ zgT9P|%^Mq!fZ0V4as(CQuesu0$*p~uGpO6P9)veNZ}N9MUm!fHq!s~BXad0A{1g5a zbNcs{wtNms;>)HbhG;;*4(@bWB8LVuUpetyseH=&kHg`Z;x@|H-j#4wp0LmM**Zya zAHb?|)qP?I?|c+3$l`hynj1Of;(}e*APK)U$J}S7Isp3s|2&t9S-!oAB`JpCn|R5V zgZ8NbKR7Xz8C1#5MEk2a#adMO=~NTfA5uL_sig(>gzDX)w=*}(DZ^P)6=!tIVhfys zo`M{Uf-zj~E=yT>mtAD@6z{O&*ME(B}4vVXrH%bm_GK^(zo{264^*D zBmwju`HA#M>0SKNPKa`QqqDF$(u%Mt%E(M|d782-#Dt-8y3(IzMEik!B{NguD9*o9)7&GE&RLS#&Tzb|J;v7L=sG6*Q8<@XdIj%e?Y5=UH= z`$N#fhl--PEpu;6v{WjTqr|1F72-}&Bfx=rNs6Wtx_`fyvdxB zS4yW(tEA%(%8f>g$73h=7oABit$tKYSO z!?sCd>O6(~%HYUbx2!{ax4Opuq7dcQC3H6%19Q^yU*;wWt~zr+sJeW!=b*&lXZ$DJ z`!-5zqDYm734Q>!dv5z3(^XHu#wpeuM`kde7KxBj784NMIBrQxP0LK}MoSTpX6 z)0*!XAuw3`rG#JxwG%`I`xWNVSkmO?diOnrksM0UdyfQ@-B$hNX%3*+Svmv^Eu7N& zSKFGc84lOg9i)}?mx(UZmINlL}F!Hy${)n;6+&GYAY*bA9t(?3n9qg;& zU$}ryF{MI$O#rrH^ET(c%*LIaw)^2CiWfWL`ZF`=TT~2z(q@HQvisoi5Cl~8kNj5S zuFm2Ch(|n_q|=&~!Dr@Z?xv!n2MJ^gsGhJ&Gm}4mfB%Ix{T{y5;`zaKBC4v*)Q%nL zBk40QVXxlK{w>SFR!QK-a;QTJTw?V7LHZEvl{DX6cYG1(c||Co%F3YNA<>KkUC<@1<; z9$iWCY5Jm8?{O~^(XVCalXl7>O{Z3G`%x;nXT|y&P z3#V=d!6_M58qz}VFTu&0Ca)Ttbs#tJ(qOjTDl#gr1N8JU#}_~3vK(c3a5_w~7fvS` zU;)OC(mt7-_MJ)h518gj*_n?5(Z8(w{6Ic9v7yV6v|wZw4fcSS!#EvPoe0YYAH^tU z2&?T0u8I`#vJVY2>AZp&n;#ajtrV%~Kuob;-dKaV#}zKm&(D2;1JI<8rx78t4{3Y2 zCgS>s3JBK*`u+{2F>MZR-SRM)8-IVOMaic9FBUmncPh*Z^(T=2aOUr+%X531Z5wLC zyxi6< zyy6%}Z+d4FZLWC7s%y+$wh>d+P-I15iIUr>XFg9c_cT%MYX~ys&xEH)WB8iJO2(>Rg))`V7yLHvC67b*^M(E0H&azi^2H(@|)$^j&Aq4shc~iA)cwLr09CADXKPN{2RkHo)H43gas*(a%OveM&^cP!9zZuJtdD~6E_jhR=> zluntNQT}ddivUqG*R%W_as1;Md;cy_X;g4a@B}FAys(8r-6pwt`rlVQ=sGC@x*BW= zD$kFird?Cg&|M?59E6*TPFcjE#BgrnePf<4+U51yBgjnrD6@LF#FX)g2W#jO!x9b%-$T;9-xdZMfNj zeprgq$24+|+RkWIy-D~6^d1oUwQ=0lS`Wqvr-5~~)gDUp%N0K42|xH4DCGpWHpx5k zYswPC8elyvw{EIIQLmtnc%*%$l}Pr=C2cln)xgn&3wt`L7W)=mqLy1eo13(uZ^l3A zJ~2xQXl|hf)qZ{b2zy5}%~iD!vSe$ABv|L6Rx&E^WR8)X@i9kKLnQk1b7*%Z2X)r# ziL+r`Wwm-#;yPIvZWX2}e}27e#r|P5<&f=I{Ux_`W9)ITZ^5#8%9LGt*iGU^&1-Ll zAlf*Jsu<256=R?>ImSzCU9T?Bf}q2l*h0tIh4%IB_j8VY?RG-6M@Yr7WnKx^EyoeT zC)-OAe=*{evN#x>O$_cq%kTlMTH|2q%nT0~5M~zzGe4CZ%4v4&I!4JL_L#r>cQe8F zuw(wH=U!)AxkL8GX_p#R5CvQHmrpub-UxhNAz~0vNrLGsLlgmz=d){OZb7ml}ZOd0Q&A$fVz zPu6ox>cV)NB3O=PEY^jTDmIM z0Gq0XedHr?9)i1Mav6$C$7x?I1?N-zvw>!UWLEC{cuXsMEZu|A|B(Xv&n%*wkE3gD z;jvVmOer=)@yz@?3UqMz-xRN6m6MzZk^qGEXA?7(?)lYn^}GaCAj%tvSQqkE$K{%s4BeKWehwukv| z!uT=n|M`WAi8*&zQr*G7Z+Vmw3~1SmyF1`M|BJyI^`F8mERI&1-nc&jhc14Pug+E` zfo?qDbi4%4&4Y>Tt#o7SIF9l?y|DH(WmPQ((qc%!^c7@UxmUR@u3+xxS((c7f?g+; z>d}QFFD@jTP1LxVhUn4AhT=Sc1354 zET~%f(japDvxJm>(1Snv$H+@NwdE)1>jXn_L!lWYi=>H%&OZ2V z)fIufeATif?LXx1cHVPp(q_8>pRWuNyi~XXTKWzl*0|ZCcaqZ=qsdGQw$fEXWnpiN zf2bdgu#J9Q=a`*6oqU|oQf;t5WEvw;=0J&UoXTDrp>>epOK|pz?>0H8c*UesN7G$% z9JB?Kl0+%d?9}akKY0%awBc8xlhhH+zxCTf@;)|cco_IyD}CGu+Rk*X>V-Yi*fWX5 zQN~EtD|&}P-pYizbTep97i|_$$VrcDocuytBJxXOULz;pl)zKDGjlSoZ6?8Gep4s8 zS4t-(qCN329L)$6*cqRzpnB90{TsAS@ounzaSaVBths6^en3VIEf7|8eB9hX zLYd*664bc8#LOhmqm>xKXQKI&vY$uxpEd0Zt0-h1Y8=d$t2Vc2iQsChU@V2Dc*^N@ zT$1;-pD(CWdE?dRd>F^P5aNDYA7GZ@HL@%KbKx*pHe1SqsN&wwiYY$%09> zOZb(p>UG5BiKjck_3ci)3&exQv%|K=S>qY*&F|{{o0H88{-97ue98I2z0^w^*L-}x zqf$mjzPepD4^V5GF?`B1pt4Xc+J9$j@FLT?M+k+5au9HKHuAAH(g=9U{b5jFT3|}R zVyoR74R{1|-n1BHsPc9sIbFEH*qe=W6nw8NCENR_^c@ za7*mLwvLSyG4Q!p6UN*GWmOJrNaU`NiDIZ}o1%uRalmhMYWSXO&RY>rU0C)#yQ2Y& z&6!zZDXHsm(;$Xxk~?DzxCvNPqkSd+!)6B1;Bdz#_a;m%{?45mD>u1GDhI(Z6De2! z7sIre=P$;QpWaxa(tD$j8@8|Em-9@At$o^SD(cjLf~Pm|niCT)u7_D86NA`*Ov4hL zDD<)Fe7~AXx1t7Ggvqa?9#`qJLYyWuH)-S97n&w-*$rbg5~Z7zcRz(L)=mE%*Y zlfIKqR&)+qg05|qi_OO{jd-EE9$CDnNGJDv%ql|H27@sH*JSx&POA~0y*i1@XxcnD zf6#{7;GQpuZ8~TFLu7&XnmYg`_PvW`60 zooGbEn;L{|?Vv)2@WSj5Jt$w>-u-0pucm{fm-SfeC236kp08#tbYgVhAxa>dhI+|q zME3@a?Z~PcQ+FlDQqi9taGngfNKMKai3NWW>O_c|zfW5R??9?&BKYH-tL?3D;DB*N z3x}AshB+GPQG1+13(z1Q6|Tp%DYLiZjv^)*O6*qgbfv)2(#DivC65P@^rB{}4rp@% z?}y*@OpfL_0m)Jdv-th3;M(pQErC8kyMwKp)eT?`&9cAeBJcVCETc&Nq~CdhiSra;&Dii9 z&rY?mPgN{$i74HQloU;q4@^w73tgcjLsP4SgFz*f>G>NctnfZNlNqjFd|G3kd7Eb& zBU_b054X~~qtpT7-ac>N*2l4{AhIUL=BhZTxHitTl6(jW+4PkU#r4XVcg#H8#n0kN z5qD#iEzM@ek9C$jR868IucL4#!;0~QotX;Zc2oTXsDym8m2?3)kl9h6dd! z1K?6xh26t1>IFPr504xU1#j_o5ywFddgV;Y4pI}>%XQE25G%=t7BDSmoAU+latSpl zK+yz=Cr)u(v=`4vJ(=Yr#J)%T$kV}A{kN0u(`R?v28|-yi@LKh_R(%#;S(Vo`y;ke z^A8E;ctaLQ@Ja#r=^|FtO_QR^2#>ZWRM-qo-(94jfUBl;K@(3#pl4{ z7S^iUPhRqsT+iO3ojI&Gy904%N%(%pqVIXAnIxQM1oA`|W~Rgx`SSw(CMx40=L(7; z(Fr2>Q4j4Q^@auDRqHE(F~Hyp{%t3ehV!ug2i;Pw`2=AwN4>!p=w3sVy9mt!|7+=G z5F&cx^LndR$<(rHnGQ*jowELjps6sr)AsWBcfPnY&OW^P%o#Pi`ZydFDXC^pnEF0& zUn@R(@AQ11I1J5t;Jtq@u9I=rN>#HRQ+*>qZ(x2grHQYyItao}I4C1^tH8Cj^Kf!lfwS0q#vCSBtvV1nh@J`-kSXhmDJ3R8xe@K`Yz0w#RoA$G z-}!y}o1vN)^uc$5hAHBUia&8UW%Juxjkt56!*FJ|KT!6MXUmys?P&S zm+$SvTn`Vz8VRU)4H1p$*(OPkl9Wq$|4b!YkNLlsr5n*(2Bd%cp7qiPi1o?4MJ5tq zLR}vG_^96ED^Xurpc@{=s?EH_h_)a_8xylUYX<==m(d4_MqVa z5gB{P5aS*6%X_@oaGBJBzc^?p#xSS}S3Kl+ca$tBsXN{y)8FMd)+Rh{ z!>->aH;r}Zt+~JzbAM0RbF&88Ol636S-8>VPrurZ3;eM=5UB6Y%rwi6_2^Dy;pHLq z%Xk%-aSfPLVuxTwB<|qSdr0eadMNHGpAuwSaco3HbM&jV(p84{&2#zk?);r-F8wn6 zwhT{emqstr+H-cHk$RIcN|G5TlBFHp%qY<&7G8WUh)l&_R0R=(pePyva zu((R^J`nt{o1>wDHXn6_gx%D9?{*F2by%RZXyuO)i>#xAcSigu1?pvbUspFRrIU)= z=1<9iX{$8+xeXGu?a%^(!&N-Yy5-cwq`XXA0z~P#^GtTFw5Ho9nkD=_ddY6NoDq<^ zBQq-AVS>g}c}R(OfDO^`X~kjN&cxTLT_|FFG=gI3k;X~@fv<$6tEm(_d#0$@GgUC0 zuT6Y{%ZO=@znS3uAvJu;EkY&p+*^pC8$wQ2oWritR7mn78NcmRA}Q!#VJC|%F~;Le z+Le5i_LI(m;&{ZE0Kmb{U860hqRoP;m!%`<`$0-tI$!z%1otm+ z1Ph?5B@(h=GTPrC^6wPwcY7;}>UAPISI7q%)6YGh0#Umkrw9}@k{30QPmrJVo{G#H z9dM)K26@!Hz+?k4DI0N;^iFKX@k4qk0R`%KruqZd zDJS^d*nQn@Iepwnt%JqD=P?a!+)WOh)Oe1CK?TA>w0@O#w4fw^if*yA;vw6XmR(Z~ zVvXdFCfck?Uhlay2SVTRcvMSkxu~3TMg0~#)%i{1{P{9Gdz8Lj?c%Jhb95F&ar=B^ zCdqPwv|pP<6vmOq%;}$t;+TfPI^McMi47W+bUCyazBfGjT-`1%Xoq`XZKyORUc{!@ zL!C8@`nNMTLgaWCvmfvDn_!Tc}cHYLTYlk@KfWS$oyb z%QWwE=!v!=1z67d$*C~M)>%0i#;Svdy9e>V8te24NsT5d3G zhW;a7{|kHdUy4{yQZ*qFTk5}EUm%kETZT6uEv^5F<33lcxe%63u#G{BU>F+r^nVe~ zI2>Xp($<*&DX{K-^B+bRwp1y)@8I=w+5hHf|B-Y;_GbeV`x2QqmoBPj@Je+Ai;kR( zWh1EVil$PO`a$wUx8NZcgOmKXdkNz|IIt({D(XgKe3&Bz&F%tSuTAfR_*8Q0N&zOSo-w&>g=80vC=A~mZ{b!JIiWTbf z0$b{TCAk0V(><9#Y?k!@#rPjaHU$n&ZOc~~_XG7~T7_R842Ne7ACo1JE|1ZzTSuUe z(fZUH0WHey08yNXe~4KaSeF!-*(}zs?(U|Rjp%7?Q@3KXqIk=~0+BGa`v5sc%-V2}YoRz! zADCV5exw>KMLKZoE5SEFkK(S9oWCDv@JFcyPNi8aE>@{KE+t-ZSr@lYl}AaIYw;JE zs}&w7N!Oj4s4+fi%d{5ZPg5+>`qfvW^IATXfYYBc94n%*-_fje*DkD}0a{w-_9Cmw zS!cRXPi2@H$Klvm^p%9)Ih~ZGRJ(MO^Xd7*)>^$yeB!D9+uZA1WVC4Fp?X9GwNcEc zr*40Wx`wkgYA$}SzD%7^n*@d<^2MWboHLZIG^ZUDBZd?U%LuW5ztjAlEL5}kQAL90 zFd9w6Z@i|Y%qZM$=jh z;^T?{z4&uhKv--V-^NvgOe2;<=?RD44e8UANAenVg^f(?+XfrX$1^Y&Q&&Vurs-D- zk95_bR9lK?9u2QtYP@57OY@ORu76SoDjDn67D|^gdtn4ejO zm5eHR!|D~Qy5<$nkR!_PPBTWi&6gc3Om%8Zm{?D1Cv(v(ZT&y&sA=ad-4ZuFmdKG_ z(=+A)Rj!Y?D@pWM#FIo-0WBNV+-w?+pBi&Us}-U$O}N$&lhC`WHTsH4JyUW6p)-rE z2!Uk+J9?{~dnhe%P1^~j+Mw1(GoWQKDJfj8YvDamvHkSLBXEa7I_$1~{I!O4I9a$7 zr$^~6>MZ{Wjh?#ac39kTXaYlk?SF@q=uO{fZ4 z=Z43;*<%+s9(Pb-iCCjyJ*r{Y`#LUYpJlqZQPFvSr`FtWpFKnBIVzB}`CeH{e?MBz z2kXt|mUO5DMUs0skBYvMvM^$@4eyahqQ8?#!g~`LLYR2Q-ef}s+Ak{6gpak#GCh`A zVxBs@DgrKC@d=!&x#vn4aB+=IU@k0-myCfQSh3z&f@8Mz~^09N54Z4pP zrag}i*XS4iVxW91E6N1;M~&$>Ie&oI{6Iri?}(GbnYo9oR2Z#wK6|_`u(2+^@|iHr zvX*%F!ByhUCuU?}c0ax~Ub7#6@lW_quLl}>cnVcBB?di1$v3}ah()hJVYy&437}De zZ_t_PWLSpN?@*!|nd9YnoR{j~zul;mPFv9PI~Hc*yx{y5d#P33pIaSQ zs}mNng{uO3Y@KNSA?^rv7Hxlg)D@R9h3~UG?&L}-?4B?ZEMoQ-;~4HL{9%>5ql3ZC zL@zaY*=~S`ZK27zVen)Y{|L&xJF)d$8JBVW3dZE4{i=Um&1gw0HV7opU`H{@Q#L9H zg1u=z?iB969;QS)JwvRr2fV)2rOA2m=c%Qbpdx@3?3*+y@Adh6Ir3kD4Cz%YJ4l%W z*{J6&gIZC_y1@IR{z-nitQ*;HJPpflb`yGZ65LKW#p?%?AUg~T-i&4{YZdx zO_nOox^hOoZ@gJv=}YVAe39{V7&+LL%T`Q$1$0sWs27r$PmDqFeGLKQC5$zs`VT}Fye$76V26VnzDs5v25n_M*bp+JM?I6 zJPNFd?mgyoPh4KgSe$bDVADqxA&aU~bagAzUY9>1L9E+rcRor8lT6`uIbJf1XoS7F zR}jw6uZuaw=rzu4H))H?^wKwnEqW;GHJU2)}Z| zkF16DzyTG)$))3R8P5PAy+Ln-1syCS%9Ck-wl(ajz77*^w|&}!vIH6$*X%F7acF#a zODal3@6TpJ)^rM>u_z3f* zKF-`(=f0I+MIeTl(J&{8rX?Zl^2QQ~R-vp8=$)Ism3Q*9$LlLsWRKm=jOhAua%~P+ zn1iQ|^N3=n_#*KXQolKyf!SSwPR`D%b}Bk5c9)PeDjl?4sALxU0%J8)VIY3Od~11m zE!PP(Gt>6W*sb$}wTw~7y^!%O*t2X7_eoti53@#`{%oqLod!!4P>-qVHx53mU!@=s zu9vu9jP|$rn#?r)EU!}RYBNpuUo;!?N%)LLC?RP2VqvFNTcdMI)(-$F37gXzKEymq zcdcS+d~pXtF-BM4wV0d7T->=t05KF0bIj)wZO{8 zg&^hKl?Bi&bs(PRAR@^@=?*Z+8jIKp}MFRnSbRnKP&KWB6p zGCm#mKn7MPWMg%Si0PMv3#8R$p?FwEyNtolbX{C z(1mH_CFS=Q^*?Q>#M;Znc1mv#kRx9dJy~@GtfTi*ieXv(;|@|q+XhMlH|$i-g!R(-^D=iq zVr+@mKAUAcB#?&58|{;2CuedUU}z_0ny+PCnaDO#_SzUS2)}prXxSm*ku>cWFx05R zkP}nO3002!S1gg++0MjM|n8n8O2_RS`oa?j2Fnb+Znn=5vnc1%0B&z=gr5 z%RC3SlN|njWnhedQL>u|PBTYicQZV|h$wQ#aDP0(Y2loDlrFh3#MUD9#<;~bUI)eb zu1RY8u}1*>v&}^GHOS7vF1xyBR-!NQz6|f=CU-fsJHb z|BK<+ku{IU#B}N^jJ-+iU8b^Y3BLM1_L9r@+@I%*?Y;{yBeJV=0e@<6S=!t=PW=mm zs1OJ7D`!C{%U+JWX$-{a2qinikwlNudEI_%aEeMUo?3-0nQ)Wgr9DSF;n;>hoZ&dL zTBz>78(^NvAwHkkqGP#tE4g}nV^{;7?8^LHR55p219&u<8-esj7C8mEmI8#ySO(^a zI-*uVgBVA?JEX}+&GDfpU!>Z8amwh#nk~#(w64ZQ1g=PtpbV8v-o__*itCuN|DpKW zlru0X*~DlAwwOnjX3}JOshSe^=Oo8{{=8LU!JHyoYJGu6U+xP5pM@0pJ!R@G`eA>N z&&Dh+t4t=Db09&?JHOZ;)U4#guIVaiCGF58L%(I9EnG}%GVQE&3Y{M}SWBmHN)%*g zf6dG+VS;>=)b-jdAq;A5%X9jZs^Ste~X$?`W?i#;I_m=CguKG3GxNQj)$Y~ z7n7M8b>0>^*=s*!Qqu`X`E`cxiS#h`gyNQs1DOA+-3DiYS5qo*jD`i|=%>hiKqAWx zY85$sG)qQJ`)ifrs+>FK2l?2`<+i3E7go=kLXy5B^E3!{`F3a8$fstr$nTSo6rc0b z;*aV>pn2*Syr+yQrCnMe%Jcbk9|+uCc(28`J!kmJp`p(9bh;=OSD1GB!|%kVQnAG@ zNVXb|lV^dt>vixJvjI4aycoZjcWaAJcpkz8%)V4b`-^ANBYXXI75mkgI%FcU3Z?VD}&uH z`88BMF^w2Q_3PX*FuLQL{HQq)SWN<{SOdJ#S>te z)Nyz!>5jxawCR7v+eG0;;)MJhRC6j)d$D>ViZhFHM#r#CgCFH*{q|^VSxv~h>GrcQ zunCrkRFs!wB7A#$b}xBMgb@}$ck`#}Bh=1jiRq&# zZI+R%-)XmkV6hLK`CkmlCviZD@wOHnE#4*n;I1*uf2y}$|4i9w{vUL`Wl&p-7q=U% z*a3=Lp*RF>Deevl#oeVi1b2txP9RW8{UY7>mbi)bHNxpNagp^L`4#&{$ zFFwE|u*fz(fsxhJ2hFrG3mqMPQS#^!Mcy(PeqH^f;2(7SVM*E{-v=T+$&uC{>?NSE z;hqPpQcDw60_DocNo!mtg{IafiBW9|*P5WO5r9a));rzXWwp##J zXn1M9XkqHYOyShGCW$7*B)Ub|ZraV>q+yy=Zfx4ga6Xd8;m(!rv6`7exg8 zf)Sd2uQ1E6Psd$8!r81@jCD4VuBc`F&%C13dEhd;VNK&DTm(dGe48lI?rl+M(JO_G}lJM*~?$N zU#or^p%m5JT2Vk$Zj0Gpdk?barLJASUZxzdavK`Z;q$5YT-;0y|EyFe=2AN0(&yJb zoLq>sut$hIAqu5Jr-j(f81#-2K~}dwrJGqDGQ&*e?uo{H>%+#^H{*a4fOG4 zyLST)+FAgnYYt$XaJkeeoNVxBMQ>RT66we1k#?!)PW`w6uDmMa4Kc zsBlqB<5#W!u-O_tYlOm9=(U;vl_qAJeD0;6Z2Icv(Ta4oY z*w@l*07vcNh=fa1Nb_ryh4bv*75)2!A_Um?8`y^n|oweqfK$wYpykTRU%G)xS$j3xC=FdbsmSs$&qv8RAu+^ zT^gw{p?!{1dqLWG1(Di1;^U60xS_n{6AyRPe{^iiz}p{qWY1OX_-Q<{iH!^Ih`suq zTERb6P?J(T36(vOh5jPh&|eL2WQqpsweAl4(!PgWENoUzX_IyZX(_P7+%Fn-xw;^2-xC@a-J37 zn~#QHRd3TtXh?6;7LNo!kl?WA<$63XGa+)z|5Y*)H7Z9^kk&&1ar0`E6oKmQD=d!1)>=iq8 zwsC!t5lek##qwR*@CxTDQ13(c$fOQcq<$pr&_C}CSfw_b#Q(K@nl>UvA`=k{wWKMrylK`etXA&wuNJeK#GBpQc)X zVsr!*5gfo6>*%gqQasulS(LO?2+wxWF;mZ zmlb4Xo0ZYCP!xWB_4vpE+9e&`EFVLTCGQis5!R#E~}s)vp7y7@w}l)D|j@@Lm3EyFf?W;75=fT+Eb;8rxn(E zHBmYhYyDW|q~A@pqVbmU1v)V?lm7}kwEaKS8e*DPExag6Ievq=;`5Yf1W8g?FG$`h;RlhZHuJZ<%3~Hd8zLR3eILe~n9Yq@=)_*@ z;`tY#JZWevZy-Ey$>Q}rTcKR6UZFzUyt+ucfbO+blG@8MqV&`W%fedDBg37{wfqz9@xhHs4YwmVZm6VUIIIhg zMjSX#>@w6AI)}R4pi-wfeVWsjn0eDBv(~CbmF;$1(R;irmS5vm)XwpZX=22)^b*t* zj#G{A(hag~U_f@n&=_iX;gkdJ+IR)UR|Z*;tz)WGhc63>v%D2UH|*==k>w)CO@Nrg zR6zEemJ#GMEa<;cVV~Pmzl95IO$0?>QC?Q}@lGip`QkceM8nRyT}3QePJ9ffejjimcH0Nn!Dv~`$fUC zt5)PqLB76R$$zTDMsQVd>A%HKlbX!XP>$)(-m3}n zjW;v~yl>NO-=|_CRay{sGLOxSms<8Z;eV4Pv2>kL^U_gr*YhKB)QZa01DMNyyk{Y2 z^3A=3g+=VRrFxUUanDtDAcO9fPjzKQ^Ut@_4_iik+wW@ycg~q39z}$QUC5U?-N#tY zj=EMDV74f(N)dxCOkE*xew5f{qM5Rf_cPKCE_vz`PF0ZGe;(6XT~kx_!p_~3Vh_Xt zE1?bvTpcNH>kbAbtfj7dJ(%%%t9_Y*-NM$?)}7?>@xuWE8ZoxW?}6gOuUAiE@$g5m zp<&snHBYE0CG*LUAp{qrA;1z_rM_#f5Wcu3M#4Qy>t;eD(pJX7jP!OPa3(!NP)+6#)6ui@L?Zu#bEN}bD>V-Mw#wndm>8{5D_<>p2w*zeKOv1l2@kTX;!9L0B-M6K%spQYWSH(=w%WTc=15 zj<)C1x{AhFF3~K8eJkVYqQ^Fn?|?oQnE)+3{3r6|0$F*YN_r9DUtYMSq&gze~d=NTnfcXBWQwo0i|l5`+_yVN^_u@s`YKa*0d3rg1H~T#H6(M#Wy0N{84+nE?4;GPm7CMzvkX->ED0p)k`TnQj#vUEN~=I z=Xqk4Izla!N8b7dz81v(}*uyVwm)@th?pKS=>mot^ zmFdXUQVsar8N}(gv5&=(X#rE|I9gu}6Tr)>6<;EhVq%2L8J(pI#H*HQ)V%_g(s_v9 z>UxAHMNXQ-8kJ^dA_jjQDo$I{+Ngstb-L>t3--AjFGJ4#3#iEXw*-M6~HACaciV`Riad zA6Ih}3KZ$)WG-avKn&3ea`FlQY$pW4A_lCatNfngD@>apPG*aKidz`XRzWxmmE>Ql znwW(kPa^Hr@;nmOK=kcx)Kv2Fx%fKLu_e_L_8^wz@iiF|%kXnQ5)HQ7uKyXYy|PlGRNwxe%Sg+ELAcM?9IQO|3OU=J!MH}+gK z*Pue2PJcU~!(h?3M}qwmcYWCr~xG*~K<31Q4CI=I985E&w!`HTcaHOy_gUPjul7mjM0 z7?esW>J)hOuQ?lu;Y@sDKCz_>j8J5o%2xRqdY(RazCkZ=38P2pI2A*FM;%T7>-jVY z7W{a79DJ6rm~KJa$P|!n%=Qc2+z%aq-y?V*jv#)Uks1-l;R_G#DYH{y#|h|35yE zXjR7LQ^4u(J43wpV>PV*|QLkuVh(7 z@Uu)UF=Vb#ip3^9N}*XywFEGh)B^S&;YiLN1ci}NNc_mX2Ow;weZ(UKYoW_%L;cYQ zwS%|>>+)enO8I#`;1M(?U-j#i4}LO8Mkb{ zSAuEELTCX4Hf2Mw)V)-F`d?Rj@D;4klaZDh{Vu2*z;$F7{pe};foa9)V|;7|KJ1L1 zJ3gG!vWl?Fp6}D>)E=;>MEOvhVtCaU&Q8o|L5c!_t@VMi)wCK)dyL=3c-vIwcx8GJ zPYQqR@9vw$RLOhG9Wkh2_VC?eaCVEO#ICh-MFKSI$IcQTe(eO8Wd_l*HBkJkoX5%t zXJi}#uQvBKC1c|{$P#lf@&QhrWxHO~othC};*kQvvz*;2k^?130Y?kZhHQ-g&~gg` z=p$P$_~w>G09%V&36jj>woQgx@}QstymNq5)6zUu4Xivnuv%T8T@%fXUoH}e*5)Sd zvtFg}zNb{@sY9lqm9?RgOzWu{@y<7u2y`fM)8w*f2mV5q9Eedbdvo8aGhi~sc@nn@ zoteBHdzcf=UaltLS*Z+9DXI3^aZtY5+QDNx{=ExiZ8|uuL*TAfnbc~9@2J9m=xfv( zI2xwO2WqSs0vwfhZ`ZbW`1fbr za}Df|%C;7#+38jB4n24$gump_tdMBfJ5|4E^lwn=8S987@hyS1AYmNJpFcj&;Ne;c z4fwjpuCGpa%ulaS#`?s%`soj3Zt=DXb#QoM*?pVJAOu1LU4rjm)9#<0yPEhgc08FL z7tT!;D3OK)u*A*&0%i-G1*Z!_{sQK7W!;57-RhWdqHl;X*RrbPowXbn z9FHZU1Vh|@SBsb#^KO4Q!#T3uLGkn%jBImn;5L}tX zcc;-u*hfpBM+3iDDij3_v}!Cye;RE52cUHSY6Cz1=Uev8C$^^h4{wv#X1@G+nY=iT z@4P6?ZVZa^Ij819GysS-38$|c-V>RS$?5&DMF(%h;tyf;twRWgq; zmTRkxUJQ}*2+(N6?vym72jKlESzZ3k!&b`*8HxSWy>U8 zW1A6?!u?r2gE*MLuvp{UOo4J?lgnQ~^FYjZOeuwdwh|zMgr>eyul{16)7>0U15)L9 zYtvtD)Zr2lb9G?SE+m35;UO^Ra3RS9*;#Y`vPCvV%coy!UVj>_SLtV0JOiY;nv+(( zoXnWZYuX<5d;d{9|2+^({fU4==&cnt*bw}F_)_fyj^HI|;yL;2ZRK>Pj!dz{V-+;V6pJkF!Z8Y!HDb(klv`l%GUAuFBOTodlb2Qd@8wD~9hEi4 zApzqt*KpAwSktAxrb;WSDAJ*mbgSMpdpaT8JY03#`sv&-uuj5{YZu&MR@9Js5sziD z(q-sz)Fx~964F34?6z>&8X6Gq$aDe*qwiF+QfC~8R_y@Ke)PDJcs_8 zU#Y$?K8PNzZoNP2uMYTf@fV=`Aej2WNr8>$&3cv%Q7f^Vh^jrT$#o1I47ao`O#W%B(|!TQ~-DAB>Ed(j9QS|Or0 zuS>`Bz%Cnvrz-jqTwn;@hXj@gaJ@jPmCbKW^V=uRwjW$8VxVWb?5XjPp^*>kcA|vy zw>rjD(E7lP0Qh#Hb=|FQUI{HIsIX=g<(O+09B=Br)>ls<}B*E?r3@lJ2U}GjWF%Vln zSSq|+sq80^H4K}~Q)T#3{(!KpHFXL;)rMCUvcYpH*bmc4Jp^HEU^=Hd<`*BB^vb1P zIs(wM^nxPhr(@aBU)p8a)e#BU+%jjLFT^P9xLmt>v*_GvdUZw+E5EgVT+PQ{)9MuP z7%ba{9rppo*ztPXlPq?ZhazavcigB`;bZu-;{JuY2BHrGIBv|jnL$zzs%SoeEW1RN z)O(t^c*o^aP-Tks%zxO+lvBw^1TlE3GsBVE%)6FWLXknun4~jK=8jNw*uOj&?tfPF zYQub*Fml>xO-EAMlaf?XKzCp#_tM@~rmlTjTB=}*G)YV>x`hI6)IMPHu<&Eil5@Sa ze_2v&0(Y(_7ON^I@}Yix>N;53@h%1vH7pNSf%(GznaG7FPw_fectgb7Tbju3bgzCQ zVbLNXi~W^+@G=Anv(pOJB|(~MSK$ED0G1zi0N)K{<>*3e1^K<`)!P{TZ~F}gZHf9- z8P|n1%K7T3+>R2Rv?$Z3PqJ!rk<7)aqFHf&#IvBDy!j{g$pA`H`}l)x?G37dYyO`( zTP6Rw=c%UplQ{tFTfT({gt_vXz7^j@?2NWN8kkU~PNL$RUTvq;WnN#`DchZvb9=G|xU4Z`m~{ z4y{;~|6F#c3gW@%mR(u z+k^S|j~$F?w#I7NR=0;7g%twulUlqdIml@tp0Fo+jI76qFzx&qS0$g4tJ`!E8UEBp zV*AhTC9IHM95H$>T6^36iSEXjpDP6k?l*nP6@t~nuXOquoEOQK#2WJT1ueA*W@oR{ zl76;)EEh;lCCbQ}vLxV+!$l_$jQ=teAd_#^>JGBiEgEy_A^ziv+yg2OBd#Q}h0}=? zQ8}<`dhdI!>rg5lCJ;n|Z5g3oYkBL0rao{#n}>}K@sZ=>r>a{|vSctZ2pFqdw9`sV zadEQI`&s;421~6L&N{{jl{kH&D~G-NAYC`z!y+u)phw%9v5%Y2;Lk>?6m<;g@QeMt zJiczI2V?*zR8Gw=;wsqxCkQ8A_w#Bv-}ql4C~o5LFAN?rLSCCYMihqtY?BQelV5?+adp^nEcA>L$x;V|x1Blp9(BX(${X7^M6TxLf6zS^-G1(yMO31_0 z84`uuJm6sgZ~nf^kFfXgBS(-@W9O%3$GIoJ^I)0t##NVIt>81AqEBWElxvSikj4Q= z@xd;QD(>t{I<_K|0lniH^Efn#|64u$ub{gNb>aSPBg$YJIeWc=tBGxqGw`m zR6xQUtc)_v&Gs?YCY$wQ@45q-7hz{7%rpKHqY19d1ytrJMsm;5(B!J*U}@=s-%}?i z_mg@@v(V23I9(tj22~X>#PJvSl{y218s^@pFIBfq4hmof?F`a)NAX+knS?tI@`wAu z@vaHwK3|HE-d|5|D3+>3DuoUgjODV52sqIAt0+Pb^vu2~56v}%9jGDEK;cNPthBtT zLzoK|x@Jqp(v%8Qxb+6H(~rwirhm4MF0QVuQ9jqz$cz&Rvxi|ZsfEZ<=eh6EY4`@E znNMyrHZg`_@`AbmCD)`}SsMT~bdOqx#E7H-UZfAL9 z@WWkLpLI=$G?=J4H6M1PKO{PBK({U?t}j|jMgBN};_c3_{$_D;87&d>J|j8nW6bL?8ht#I1H(nm=8r`LDaA%A zurf6@J?(G7P^YKwyQ`^4{PRoVJUY_CYt}B$%;Xt~@D&>!d3VWwG=Ng5vM%;)G_okk z;BKPK61LHF9JrMC>oZ51q_h7$rLttxXD%L3MV-kv!2I)DCuri`X+Lz!`$ z8%0;rNIph$Ta*WpOO!ATOF=t#8EA+Z6?9oKX7gCEgp1LrV2*I@uskI8B@m;AXS?Ah zlC-AvZ=iCkA>F1dSRa~S5=5?`o=G7daiBPSrruCc|Hjtt9G`*da&_t7=v*;Ezy z{V=qkbYlSx-x;T#if#wx8~{gX=7C7eV1~q~rdzTmqwuD#zOPRbM6wlw7tJj-e9OB& zrwuh~vXc=cj#nho8-{-I;Vr@t$XNUk7HOxwiyx(F>gz@{pBRtB}z8Sun zrnHSB{IaQ8CwNlh5o(IIN0Y&Z(EL8gXgrq=RI~o+(;bqz)6b=aLQedD7lf!j)oT{N ze$BP?7x1B@_vRAR5SfF#&r59P^5LAy_2Pgu=yT@M(9#Xv9L?#ASC_UT0&mJ-fQ1^6 zzv0DN@FZCiIH%&~=jzQgkUfzDtZxenZXPZAa-YyLS^(wR^vO(3@T4{x-$)P;(e3}2 z24a?>^8WHIWridwqH3hf4yGCs^A9q5YK%%qi1U7ZpPn-P7qAlC%=94fHtliY)b9b` zoajokpi;z){^Bj`g`L;SYrvVr#d>G4cwV!ta?SkMT&@6a%To@29pBQ<{^JpV*|FF6 zJh-sPyZgnU3PyB>86F+2c-+DdLFk>2b2N2mm&pe>iA400bwM%6IPtaeeJ|UO=3;nfO zE1a>F)Afm#y%q+IfufHNp3Fh^V!y~XRocmoE|n{*goE^55TZEVzks_in{KzYQ9>-V z6t+^Am^P!qdrVt&Q~om$z@-Hd!W8J4O7XI&Yr{Qa597lt`U{W>t$aL2n!g1j?LBER z>B2gg!{lSA*!R(-`DtVeKyklSy3ff9n>VB{el1C5;Uh;8P)oEvFQO?|)R)-U+S*RV z`DMOJ1Q$nC&uCQFEH+J?mP($-zd9H?|DF6ND8~1xWZ>MSbxxWB@+jRZ=y0i4%8gl9HuIc(WkAD~zZB=5w+VR@Vk`lzHnob=E zMsuQ;mkwJVn8dq+%So_0KKR!qwiU2)os%pEF&6vxJbYAh(0gR|9j##>iJc|=et0Nz z$f9_zV>cuE>jr*fJ^pNjSQt|I@FF#uIwL&oE9#o{g_|`cZ?A>}<%0SO0@7R6j*ROdF?IS+9?=Iqd*!|eu z>5GjR_b0T`HCTvPjTArZF1A$urgIayjEAsbbgKU6LNn{NqC~6` zL8VzsMcXn@pt|O8$8(UPSm=4#Hp#GMM*4@4&Gp_ut>nzy=e>EupTuC+2~mv$)LUcy zv?jw?rcOQ$>t06vp*ybarQwY)jF!s-DTrzwJ z2Md?^*d|~RL<7R%4ZAEWONRQ5K94!x6aC8`xi)xCSi{Wv)bW%(JzkJ!)BxcFE$2G*V?$aSp)gSE)h4IDw1K-;Mdjn^|7 zj3bo{RIh~W)v88KsNXKsSooyjR4_=dCuVnoMA8em{peiH8d$G${R4|~CS;$= zsc=2Bf8QCGe!9Y+uZJ5E9(*N*uWwYY{}8)ev*>ecS0709@NFb{cPOrlqp4yB0Eb&X>qkZ@|CR=|l4S0; z@+23~3h1kqX4IFk33Kil{d)iK7m!k~NLqM3r?-E|_GwfYF`T^E%5?#&JA&E=g;$`M z!pM$(3vu>#Y7~u&n^e`uR9cxOjO@vHik^;}-n;g!p=ismHe53eMmlNSF80P#(<)?) z;Mbz+q(`fS+S|($LH_aqge(lpchWW0qzx%96KPv(o(m?#vLpPOH>7;J-re$|^E@B3 zQi|@R{eqf6HGDgsDAmjXKhFOX++rQAPQi#`@HriKNk=Oe$VEw2&b+6b3RhwcAcG9p zPyrMNjX!0Ed`YFLjC`a{qFa|mzh8qTEEyJ(E!K510HHhu2Xfz~KqRvTEK!dg_E`7r z{-Sbn`M-$;)HWN=@rz5uU;jey-|axb`383s3$0$#DtOWc76TFMxwUdl6jEPU8JHD} zLJ1aH<|t@E8epR*-sZ4gR(kCus+3OMY}0ihzE5lSs#y^a`}P%24_7RD{>DO-HzhJ9f3f z*lT4Kd(*GV5}^RmN^zyiHcj<64Tl4%X;BxbbZS-MG)bF47e|KB5?QH(C8||bpVFhK zBLwoyW^3j~Q%hQtw(_zQXEM&2QZKD`n&-wqJqd|KRi)R1fS;}XR(l8L)IOwjUMpM; zRv*#fmnPU|`LcP{_il0@E+Y}EH}Yl45AE!zx6dI|g0EFd!$TMQb2E)0>H<=YhzTi{ zmT@RY=Y+?X{t@QNyqV?tI68GUXYPdMqi6>bOZC>Y?wB;b_ThQBFi-@!>NkY~MX)4m z4OZW)c<5LbHdW-NWt(-Ah3oe*YgBv~6eO#$puKFF6s_>-z6HjrC*h$l6^I=2%|WtN zUD7x<6WH|)Ekt3;W;Y0?#`i>X@vlaSEc5WdP4!{nV&#cxZcY+=TXfNnJYGyifH^dc7r3>apUU1 z@$&Cnt-Ll49(fW4$dXtT&3sBP^wnslsJf0Y)PjFLlZ5N&P>=mu^_~T?@;f+wXhd~o z$+ac=RGvW3HE9=fM;xGoNm*V@3FkxKM9mJa!Q2VuDk9gmE6>heIck;8p5kP%vx}05 z;e$nD$ZPFdTb`4XpsA>V+Cz}%gpmTd%0M)y4Jp6v*I)1d1=vT;n7lDfnRPIGi?h%@ zP3ubMt0_pS8^o0kJK_|}6YQ-#jAM18CMnNlQLzHen@5ba1Nu$^D>y*g{n(ni_)a#?<>>EtWtF9HL>yipt~; zPIaST+px1NjjM)AAAq`lsR%ZWk=Hz?UFHVvcOS})9G`!_%#q|BE~Pq@EUFdUpUe5a zrh)^UorAgi{+!*PMF{ZRpR{8R$>I}|h4Q8yzbo$>JEU5c;FbJnFbt44+?AXsc{9rn zV^xeR!HTCHz)obxnBZ{L{291-2ThMCsov0FNp%Lwp+^t%W%LfbbMbN+`IzJ};)|4E zRv7;*Dl7a3G`6#Qb#+V*1<9jBfehcyv{a4WB-eJSig-~zmwR39kr4apzfft%yMj0j zKR(@8>+J16EF=B-e=FMm^MV$&V*rSzihTWlD>7fJ;|Tzg01N-$DJa`&Q-=!cW_qaau62bgmI2%Rp z3mnw4aC3V1aNH#3za>3WN+8UMn7W_d2XH+HetG4*?fI^Q+8~?)voo*p4)Z@rR!bROr8Q@|~{ePR(Anp;+*-6g?mbm3=#C&(=s9*&i#V!#jj1#{@<_U0qumyTvU zP~nF;!^_}e8j~QEPQSl^q5l>OMAWzG*2|TX{caHp9xq>#;y;Dav!N#8R4uGu??)lX zd3C)Qn^czAWLAO5wivohl{?-I`@ zB47OPSCQy${B$O=q_g)F(LK@ki6A*`GIK%1y+h^_q3Oe@ehy?lTqlt4Hh3JBl_wPQ z>8$TM!MpB_45#Np>XmJ~<$s_=a9}`6kT5p<{=*o`VP@g}(MKMO)3uB=M?2_vbBj+h zH9jAUk^c8cN;VKpo*c3Gz`1G}5Lc7-2NX-)1?pn0r#0`gjxEr89*v(?)79Sen;_d} z{!Omw%;m(cijxj?%z@?i?Mp5-0ht%gVV&PWHjeeog;#@D#oVaA6|xnIu+rfWD!a2^ zh25c03n79(5?Iva7`mjoja@%eK3G90oCV%|!X6y<^NeR(*sGJZta}7Yo~H#24bQv9 zQ5!47joS*Ov>2d3V;E5~(z8}Q4)z{Q%82~8agKR9>aOJAX^g3qbuR@&DXZlSt?*@V zib`2KE>Jx-MgtBcrInvg8#OXs)egPoX&||0YJQCh`v)Z)v~xXbHzPJ}ocpwMAE3 zdZJ=59#-y+#qZ~3E`+7cRMm?395n}&$hW~)V;PkBA!5p6<`Xha1*Hcr)-5}+h^H8; z{_Z*DThgtCfw(M1f@9j7+L6knq|5^vq%)+a{o2X~B3N-Y4a=gueQXf`3f|)MQ;BM0 zxsrGcZu8*Vf)#LZ@O%CQ8+zR>sW(RHdpuG65@WdAhkys7h59PT9wOs&Sg-J#tlyp1 zz9W~4P7C<`54_Pa364gR;U+OvJvGB9@J1K16Y*<%EbS|Xi_Ps0p8=R_p{8QIO?gx- z-^d?M{(gIrOZ`c0(DVAAuT!5m{;7>LREBf^N=W}|5XGjLfW~Cd!B*9XQvCjRi}^-U zgGLgJXpQna=p8VZQ1tO!G#o$X)5lD?UnVR3$+=r#wKy{j#fv4SRV5Q+!q39h<13T#@3{AjEeUVHfx^Fp|~CemALbDjWN$Iie=7>wJl}aW|M2#RCpNo=GxDYxqiR!hbw`Y}_QCt?gl_{`@`=k?h%O=OMYGX##`8MB#L0VKSz{nM(TV1n91|NWRER|&S(G7WM;N*ADDp|nprfT0M6 z-yU8OwtXGxA(qlwn!DT9Fow9TISi|r@~YY zk4NIGg`8bRV@LAY+MEp$-Y&IdB=%RSa5tTDrU~oogXbBwd@4ET=brATjLwBK!#Jw> zg(N@#s4_1vXB|leoS^oPqGy}KoVeW|5}^61w9v+(?B>0W!Ta5xZe`qa2K&1v`L+q-@P|4fTwBD$qel1A{-n=UOFPDEoMFE z{Cezik^{^(w_tNvk(=IwIm~y`I0vDh@Zk19{5+LaX-%zl@?4ku+CP%hmUP<0E#0Gr zr0Uu@D4%Fb&3G}b-=I@MZpJMR0=6F}5IQumHvYsEaV5?K(9YL)aG>%S@yzkE==PQU zF$dSmb8hPZ>u=Uig)Ku}PWD8BL-K*=_qWNT>esLg zp@HXi#PgriquP8}L+bC~CuS`;xLk}1x}8W>t_?m)DPB5ghaznlG%pIuIHI7G=BA#Gy=F?o0OiNqpFj}xo5N}&CQM;Q$8(SA+H*anuY7z*sVT0Rmv9ln zz$ZvFYt_MNQ?GVqmiODF9~-|dSwhpA0IvZ;(M^v8so6S|e*yozMw@{%{3C;ZmiBT< z8Xyj2Q#%7`0KzlV49)ztYft$8s7+WlFJ3~|4|KHWN(!C+*m^7)gL{CuG zmip8c7F^FLRKDe2L6e6nW+)Yu(UcwjO<+kM$&r?8Mw((M@_p7Zhj;9&iBY|xqQ%jg zCyuiYJ!>utH+MEEMLQQvJO*0O3{Fxu7}B5m9ov$?PRQaef$z$q!7BB-hg5$lX3d(~ zf$e(!z>hmX*trgxJN#7AW$j-BO$rT-Ai~N5X*X&oEr;c+5BZa3`7lIesmGos@zUN9 zG<>rlYbmeHA(J?3K1*H?!NE-%CMchoTds8#?R+^DPy6AJpgII+OiklKC8_{IC^%fz z^Un$#^Hxza^(cmwAL+9x9PY2;+s{+w-weMg1lKiC3k zOH+}t@W{#@;+gtFVt%P{zq>eYvXbwuYAgpCm+Q74b%_toELqX|mlxi~g`%BW8nTor zh~onRYyoV|OS5(T4uvvaZYb-O-SrT6BZ9)l4-;lAiWuZl+&|!9ugEK{-oZ$XlC*hI z;fg>7tmO@93w!mIl|Qv_Fx4?^L7lPT4%Zsn^z;c+(F%hFa;k#Mbj|L*+|gS4af^iH z3oO`mSlr9y(_;8XG?56Ft9L?X7}A;p+srqwm2OeJOA#Sk0`YRyz!e_7m`gN@xzn{V5uS zV(w*AP-JGGn%{WI;{3z)tL-ez0k`pzi=8hnnZTzNqvq5Vgd}0Ou(Ce2^>l*S$mJGY zVSJ}o);cushiC4jop8}2U(hH#ZQY8W-b1oL&o2RD%Hg))xomd5vppHE5pU3(;Vm5% zX^)eKuC{HjyK3P7=o1>)yOnLhH1hu8OQsO7oo1dx5#4#*NE|K8_R(YpCBCtgAtni} zOp?MvidhY%Bd0v%wEFvOgXTC@_H()~^`fL|sYVa)L+Y;XqFD}MEy{Spk6q99C4JRD z42&ixDm^tjD*!)RlJ>D7x^3yA9DW$U7+C@{D%mFKJVv{5DGUf}YJoTz?z3!(`m33Ph$}(Ep1~Q>|L-3eJtT; z<*Ypi8X5ft5|V#7byY#%YzK-MWLB0*QuhfPW)CdXgjYY-Yh+|$#LIJ&GZV?RvMRK! zAFn)hRwA=%Vrl0hjhO9-^&o+nqi?>11_$sR2j3uez=^_e_l}>BI~Pnv$Ei{2B|D4O z)`PqJveG)J?yy490}z*4RQob&v>j%tAd}ofr;qctEX22&WTA&SA19gyCy9@v zOskdRhlSa!JsLn-?q8a^{0uj-&*MZ;OZch3L7x8|*oYqnpTSTB74A95a$bL<+wDD{ zm@y1&^W9*wXDf;n@elh$y6_jEala)hesoT)^9Mm>rXT8mx5Yr?g{X0(@o=o&5$#Cy zj3@C%B*QaKF>?{o`{na1#G4gi9$U-Xe0U*4qj z>ws~p9iuh#SV%xi)8RCdz&K&8$RU34l~Uqf|C3uk%3!)jZR3cnB>ywyIsQ@iJ_}XJ znLF%!cWR!1*oL|@==|CFb!E9XX;Kx^|GFYdi4!^e4L|+@w7USv-eiC{00`AB7y~hS zKR)aS&?Ca?90OGdJ|40Ja~wbpLCP!6lQ3Qvy?gdGMP`TiLG}>Gq5z0dQ<@tA4Bze% zFgzbq81yCv#PK5wh$jWU3 zk$_kTu$;r|)DrL}O^&_xpSjua^9Vlqh?+yWxP{sYUfT?d-l)-SIyQQN5gp2r>9rJ# zODMM}UD~5r>F4;bFhgCF+pG=Pk405h#hk5QR$rz|rrwxS+M{gscbyFIoeWRM(OX7% zEt4mPaWCa+YgjS@LDDM?z6g5%oxDv-9=<|8#*ZIaU_7}?+AXi9A}vzk5-pqezl(V> zujL__MwMS|Z!NgY`6!6;y40glPh7yMHigP3rdMv0lt) z!iHil_m!YB67X8m(T-R%ilL2FMfcfvEArFXz9zt&?ZW+WT909Dr0k!}CQ zldUJSb92lhF#&Sds81M(>Xw!p)42^xrp3~Z9CX2AI#ky#yRAVbwiSZ8n61Z2wW^j0 z7&ru?SMneTbwezw1zzAsmr4!d&GyE%J!38XnM-5Wy8%I3L zz7-a>-tIlFwiMzsQBUP9k>CD*2z%?Vr~0i=8Ap+g#! zZiWFQ6_}yBQ;cXa*ucAFC+le-QV`JEM?^pwDI|}|TIVt#@|ByM0$%P42w8O+S zfnm)}FVgoRDppHP+2o3#HMDn~+$71{G;r;&jN*CrdKIVETonz zE6RQ9+9O=o$gzrx>Z;^6)(WL)n56g0+B(}if?s-+@TIa^nZJ~7?tQ3ZsjB1DA59TC zq{uPwI-iN3xdcwe9I{*Zl2*XC;PNV8LO(yZ*jb1`ap*5QDu0tpL54$?h2${Bs2FEZ zQ&w}in2$MkGbzLe0vneOO5DAt29Kd&JqOoqd3{d~$$M+#ZwC;We*F8dM&ZjR_XXvy}xHW>0 z^7@EKxBXik^EjUbM7{t45_v4tK{4o}C^6yQQBQeeEj##h+=Kenu63v`iVVY2>}1@K zu8gNM{TmtHAWd27Nh&Ggi&!t81l2eOHvvVzoJ^Q+;KIG-;A((% zl_;kTX~wYT!guxmkqgO$qO>_d?qVXmT6wS61~UPIO@iUj6wz>Kz4#siEtF@72Rjz1 zJ|ViqAVMF9IGMUk zmf_@RmYx(%pRg&l@gCc7VzHt@+FElw?;5$!Fa(C624k&+hn$5EG`L5@4bGq#etHRo zJ#!Nd@agovp=-eow}IWsca>y4>*+kc2(di2;uf715pvd8g%0h(BPC+shzg#UY??Lv zE3Lc^nBv962~NnHZdnU(Vr!3UFrwEEh}?qDK7C81%i2Hk#mym5);y5{ZYGjL=T)}aRQ0OlsyQjr; z!%E}+`FHjV+Hy80-ff0{C}JmVPZCwQB*+WZ|4k6$D|MfY&7y}fO}{7E?LZu{sP8&M zY~K6ENtKe0>%tIh$*B&zfK0^c3i zz%RuIe4{5Y?j;u~UU%K1zZ+s)#bAhBN2b)&r}S{x%~EWr^engh++G)XQS zmI57L=VUaMAL?A&XwUfD*Zip@Dy0|K(7a8!tni01SYH~}NYSMbg=5ikBOduXB_8p4 zqm{l;ElxR!jDP0Y*E>#P?;;Tm)In*4$C5C*W_Y*ls3p*qA&0w#^I}n(c~j4RRYLGS z-qRG3JWh=8$(cv^DaohH6~JxW$RyCa_wt3-Y?oI%CV3;ncDZW%YTUT?#T1R7as{ET zX+TA571y}@3TYW+3+Hdg>KT#!xn=GmU6#pDF4p1bl^E|ry?+4Tue1l)it0Teq1v73 z)S4?xSoi5hsD?U}a94s*Rc@ozoj|R3py_Gj;m>dWV!y4R;Gls&NLd0$5Crz86mvJopmp-3 zKJO2MwNo?JD-)r@Urn%_qmMKX9E%FNbA{esih4hLT*tw<48 zq%0eh!jfFkx%pkbK``c!Gcm=`OygySCPvS=?Is#wLNfdn`D3kU7!`|dl+(3E6|ppI zfCDk+X`-RuAHT8Ji6rzz0dgz|yCaILnZ_TN!)1w{r1snl{pwWj}Jnb_#;} z^gb9oq>alH4V4_9@DM$3Z93<)Uku$OqBv8y|Vp_K` z&x-wWB;5JfEh>pzZEy3TfTmk3He!!Je}vmZ37kt2An8eR@=PaWfm#TwCP-0dWw!Fo zX&P%;^AU$SNT~=@g<<}Br|uh$_%fN;FAdSgQU2>O|Cuk5Yb4>7>$e8}XL>27%BL-w zKbX@er$1!aykCjmwWRu1$Qzi_Fyt44X8n0V%NqC}8kXs$rg!=o_vYEB=@zw_(pdAa zkBs)K9IkeRw5JG^t6O?I5|qI}Q8Ijx*wajZmLiNadFJC0G2yW>M3~t!TLu@q3b10A z3iXs${XB%em~QKVwPSbH<~Mz4bUlbs-VTDKuraPdhRtVlhDG3@RtqWVATNUb94nq5 zUJ66*RMDdC_zI+$&#hq-VP|3lO9P6o0_pM?{+H_Y8MA9oCozW++Nbf8_`V@+J^4R`d>lcG$RRP++_h6q7@y}xLSsuSUtH_~u?^PKSCtr#y5%*DX z6!}uY9X6n2<29(2<#f@z;>wm&ByT9ulrQ8sM5X9Wg@WMXL{Ui(=liwwe%p?UPpM1O z)y9y0A<#{YY?(AvhM>7-Qh)~32=%4|x$eXZwZxP|r{}d=-jw%Y{Ine)Js=+9xVowT zjG3R0DfC%+5-oUP{)nq(aAH;4E2{?iITTQBmB|X84%KHGqMR?6jsZ!3EVc9A%^#Z#5$9KNw|Qx3DRr)KxdiOLYkpL_ijN>8@#)7IXULi{fv)-yZPKxhCHZx%wtS~j9 zjOG-`P;v8J@v(!qn!`7%;Y)jrxH3Iq`s|v`x`{cCgi-Z_jT&8WO9fxYf)Vl7sgz9|XQb=uYaq&ohsLi(hn*(F{d%{ttJPC zArGm&kDPO`T62m{BEwlU!+RW__LX<^sklIKJ-P~oGx7}N#8n;K&P`#%ZjJPkxMm+; zfKqPa*r#lub6$4t9V}|a=YX!T#>?+Ue1wM`{2s44o>yDG=ntp_rg<6evAhQ{3AYg{ z7i-zKVBMUmPbRxp-gK3my5wdyxK7sq0Gxr>l8 zNUKj!7K~OkMHh(oZPc|6qGjoR;5@fB_Fyx`fGiaEJ>Gvv^_dL4Yev{foE=~5lHTU( zb{)s#`S&LWf8RX7WpQ5md42ac902-;(>H`2-8n@X{=Ub~?vCo~40Hc8ssE3!o`hJK zSb4aNgzQP>u$ct3)EEzclFI>s_15?De#&|h^51Ac9P?29*UuGy)*oOb<^HS9NKOb7 z|8IUB*FeXGYZM@YmPu|#|%Jf>6hsf6x7!MU{$Vf!kSx{e~4E-P>MLXHG7y0 zp!Eg+#Ucc+PnNEJ2IJ!aR)0E4nQj39A?<&ze+4%GdGkPza^4Il;h7if#ou`VQ0S!j zyz%NKz?SAXh!rROG6F73wig zPh_KEEt6HUgHc~gGqb~~)1jy{|5GXM`}f3Hi?xjt=WbhS-tzb$Rm3}3PMhbNupzIl zN4R;;ji?-`Cq#PJr$(IR}5}KD!Zttir2qkgVENX$~3<=or)f=;%x>UU2%m;S)B z+@J>juM!EQ!KTq)uqiZKxj_O*x(}D|WGwtKZ5+GYF%uu1Aw-T1D3~uSZJxUn?UzlpqC;O039|6;>$9&?=RW-lo`Mr&qeozJATGRf9Iip%sinK|@XJl0|);96dP3qCW1q!l&I+l8V?LFY{e zzV?(XM6(LM@by_!Oi`YHQk*&CeRU5z*`R4=;EEdqED`+UK-P5#6p;S=pEBN!+ zr&?6hV^2yw(X>)uT!cSDp5y(QN(N5P@F*yt)Ga#ng2Q4j? zRWT@8;Q|Y^e#CB&o4+)$hDHq!l7QvV)^KKD_WaTPCMLN#(E-m+OlF*teoMb5`*f}g zkymRCOD0>{qqCdqUaUiwnudR;KJQ0+EZ;xw z{ReOoqR)5cc}4N$6~J&?`D6{zu3WV=wogG)n&6$5HXcCuL5xW8FSa>Y$-L+G(3Rp3 zYinCr7k{e{gIL&#Bz5m~$&vdL(b?W&Rrz{iqZj zQ!IAx13C(B*@9z#xWF88G90te4vGR)-Vj1Gm`>h!>$Fb@HaYbtG&f%4 zpYZT~z@e9{`bkUz8S{F6D=ggd_OB8n@>6HD@QbI}9j8Y;h^-*T%<1->=TTkSss`Li z5M;7E4LWwyl`k|^>7{sP?%rz+`J?>yw&bZY4TUl!b9q~nu4Ua6XQ{MI{N(+p8G(fy zaaG#7_~2OR0lyh{KBK_S*8(O%J_R8Rqu`_7o3}bb3(iivE1sFSuM?W<>huNqyUL}T zo>%KB1_>yg<6aWw^t=SK`=0=$RC#Giy%?2p2Xy8=&uBQW~HUAH9CW+qjh`?g-6u*oe! zXnAT6)#9}gdr>>RjtrSSRn+r+LCW>1VGa?RjRw9yJhO6?3%|fdl(UG9+jjDQ0Ks1! z$4OlZzHd#+Bu4M!DluO(e)hj`3AJG_3)xKn9c}e3w}}#0{TbDOoqSPU+62fBT%FCm zG3*#01+`UF7V&J%{0Hztsk8;U{6!T*+e>ZA$F|!k{iHA;wV!t1!tU?P_t=X41jbjH zwF*7uI3=^A>ux9PtJRvtB96@P#pd03?Qf%}{bs878wfKm#eK1P70&d^vuQ%M1Dx=} zip~k8*n5V(n+`!baBLx+PBa6 zYQO0&aEaY9Kb&#gl!CY+GZbhKJu|x3U4XP?HV;E|eStd6VrqNrUOr6mN#?}0l_L7l zWNs=^xZG0Ne1E*jHgcA~2pcS=%lEBFiQX7lZDC_oDaPzG-Sp&Mi_Wt z=Y_&yg>Yrq6L2;kq-wV>kGU)bXskP(LGB99&fuS5hk;+#q~GWIuwJ`)TAONK%qn^d zc8}=m%lLvhMQ4I?uWlt`B4T?d9S_dy7vyPL$FBcHrP~c>x@#~ zu+6~iBSi(_H8Os_6r|V|UV4o82-g*l$saiUl2WMiY*6E_B6ZxX7 zJn1xIY94>pq0=zsQLv4Iz#uB!VT{~Ux*yJs`MXqU_O-wxqBxzO(`L5#SN@xA)iLG# zNz0DZTKS0mD+8AKIxJ^Uq~gbIM{mXf?{Z?A?vUv8K|By8yi0S@T#&ceNw*W$NHVrv zS>4xFP~@HFSkn)YrHdU?yz3lD|LPL`Y{n-V4|B(2XkW8i+?u-4lm zm)~2bmkq%YZcF^%5(k&lb&}~-T%R*u8%1a(Q_QW`Y-*0EGxL66o;8~)@mL-r6P>-o zPgZ$fwG*RgC*+tw_!U@kQlP??DR?Kd_W`f{yMaqp0)#DPwmaJ-a!@JXvzCcID{$Bz zM$)oj3d6Sp{R61=Jm8c#A@!%|06LaD+`;oCzvAfOlu3QZopYK@F_ZdHMvUuOMnzrW zdH%*fz+~~6+OB_=?9pVg`h3em3*&(68|G#5M47hTIjwJQ+c9woUW!ftxl|APa1_UYL6c*r+Jy}Tgn zXVdiT?FaH!(K}b;XHB=#V0;c1m}W~Ay3CrZI{i1x87bnYB`^( zeQ!oZxl>A24wU@sHwZ1?>@052htSg-WWIVmR~LSm>%7CpYXnIwk4_Av2!^7I$lk~) zaGZvf;En}dit--{XIQCoQJT}P6Q+Z{jSb$18o|n(hEW$EWb;ms0DhZ|NGNhkem|~Ut z#+{3;MO#2ANm5)NLIh+S;sqgE7SlZQCYZVnLyLLRh|hG=JP=CU6eig^2A$yKntbyV zs34VmuARCNm8!e1eee*w_-tVYeX<<+@FK6fA@&m;ts&aO^K9l}dNRDIKN`rC>Xkpa zSxopibzZtzmbwX~c7xAGY0L2ExPhM5L zL{j(Y^VlD7P;4$3^U4-#sFm29(fF*@YTOoXwS(EY=VRe#qZ;7wX51Sg%>bY7K zGy{djsBw?=q2&sd>DeYG{Psl^%cGEkK&bl7eB1zi;R}MIOqHm#$f;FBP+kmOa+Kld zN`7v+c4C+PQ&A{Rc$G}m3$bG)fbL4qq-*0VS4N$jWX9;B*O>m&@MGN=cbIiH9+sV` zTXByet<*2&L*JGQ<(qk82)_V1>Z6H!4>ZKJ(TyH8F(0TO0UOCK=HK?Aixq2|21%}_ zbvtc9-JXk&!=d1$igS@{3|>k%oSDCF@VuA$XU0J}1=t{st|6?hM6Mb&aXVS~P-_Sp zgML-BW!^h(Je!UqU)Mx=Tz7gpf_;}(@I3Dw-DFOwqO%WovCFV<+{B)HP)Rs*Al;OV zk130B#vG@sK4URv$g@6AR9g)#nM>EFnIaGdtb22iyz-vmS^w;o>f>?EC0vjp&z7N} znw?w&t-7RcC+>sRmKGfwzWCC}=-0;9L+YeVv3Pkl_y)u}g2n99$UGFqlW5D}es=@# z1MlcMQ2(F@6M-2Y#KhZhk66S${`%2ab6@uo^os)jwzF)54L zT=5QmH!-N_;>X1AA-LNa*XVvP^cS>IGG}E&u3^7AK zz{lzGSk_xnynd^5WQOC_E6BdzGVk-d+rL;9O4U~|yG8H9^~3mMTk;F#g;&Hswh%oD z8S2X>OGIy9CiYdoKV7O!KWXW$5IY_bR$n3$bChK4r+HX9?K~e6<2A@$lkOHst2iNF zC_Xtk=PZt(9+`f&97!nlM@>|=g3M9utiv*3q@$tb&sFBp%+uDuwD_M$Dp6+$9JHy| zd$)tQIk~tf2~d9*!ARH-`dase(eCkYo#yr%%D)aQ=wqTTYWJzhOb$E)|Fx)dQ^+{^ z(--756@#pC8O=#5(m07v> zUq+r71R4Gjj$3#=!r7wc&h1DiP*|9sFeJKFbnQZa*bWVwNU&L?)FXN=A>&akRvCMG|Qsw+56<_XT3W%ywe3~I9-!QulDD@=355s2yK%j ziV+60Y6#52j1b$m#|-9ma;!?$swA4jqF5xidH5!6GqHdwwwYQmQvH!CW-LZC8Vq_c zpOCwan=K6@Ff`}JF!j{4c`r>AU@5(y_1aq+?!wPJLWT?1;eiXoe(rK*@|BYy&!)$x z_LsNyY$fxbaSXZILQ9q$6l)g0HABYHnQWxdi03+*4R2~~{D-Kz{ikn_Cq{N!Yn|J@ zePGonMu9BmqAySph06=VylE2;p;vUYO{6}JoF$}o7aB!0R2q4YxzyBr` z4tVwDnmO*|=k*qVMAOeXm>}3Vc*%LD>E34@C!3u$pmJadOYbOuICm`7fW3g%FXz3z zgJ3*OK(Kh@w?_btmjLplv7ni%2mkTbloOo)!76%qPcoMrynbDD>;&UOo%@(W8UVso z|66kTU>Mf0QXzfljm2!0X2~C}q&O(esGjVE3=8 zqSAoJJaHW<>{BwVEIZ~yzF@;upH?%&W|+@V>yT# z*AHlYdBb>U3&0iPI>U&@l7an|Xl2RrvgM5oj#6NbgKMtzK2axB#PDfXXu7L(D_53s z&cOEIW1A;#Y|8m;GyE;Z>4TPM2h)q^CBHgbIiM)o5;aYBEiFL5$0J+{>_W^zHc$|k z6%0Y*Ju&nzD=2?AeDAMx==m63LxNhwc|F->dxWA*EXJV z6vD@tSGZtvHv8(Ado-X+#^`842GSl}K&XQOZ-9!}U9SfJ&CTeGls~*PfvtyYY*?I) z1W3Bh;U0*rH05CURlZyG_j2F&QvVs%6m<#t1y?h=j4k)Ap1C=MS{bpz4aDM(iyP@- z>fz>~o%eVycb%?|lNEF=SH6Q2wNm`zBOa!WbU#PB-q+L^B)=4$)UVMEW19G2874gR znw=KG?qko7LoJ6VyQ7`hIlb1#7zRTf+eQ15>ZhEGsY90n&VGtSVQy9*Xp-7e7!zRQ-&s6TPSur;1P8k&EN=eF2rW z{u*ai$P_N|+O=(~!^*jC(}qgjTA;D9H}W-Fa~DMje8|KclGIu3|5`~qN3F%|u%f_) z@srz%=sx%v@PtU(vl|osgUk6b`u!*`UZHJT6Re3WDlyF%-rv@LrO-(J)Zj6y(L#QCr zI`tW6D6-l0&DoS9f6Ij|NaDV>SkZ!C?{J~M9H0*9<9t+}z0&-KK_o?GS$a>H^zV_^ z@x!yVdNqY1kF9N60eSA9rnZ{2yed5K6D@hwCDu<)tfBAu+Q-^ z#lJLN72A?uHdGnv2<_7b-S@Kpht9~n>yRk4mK-34aBXU$Y`!y@ zZD8W2D!G^s447DB41_n3Z1uA@UgV0sD#B2I2xbGZ98A^=Ow{P!d^pIVdDYZHP7Egv?&fbB(N<^?TG-+h!$W0cB`)&Ly~wHaVU$3jVqKmbGWTn} zz>22OiiA_u@)Nxf()}1@+6yY$6`$kS_ond=4H3#9kbUf7ylHDtSSayMpuMlQNlu*} z(RVDbYy;!EWLy3vf<{&8!*n1yIKDvEv#pCuP{&*g+tZ4!$KO-e%rDon77=JIDsW#^ z)XZ+#Vc>XXD2OR#XR~zyL+n1}DQbec5y<|S$C0GPW>>((Qg*8zl@nIVN*oXEflgY^Ey)%DF3wh$x9oZlI!`hAH?dH3-F=`zh z7`WuQw%TiJh?DiXFA(Pa-f)AoEVE4nrR;wU-)Zy^pIjpR6f(}#&V|65CUVhjOW&#v zMysyCMS(zet7hJOg*YO>$JZ(~{=*=AHWU%?Ip0@5Ck{pr0@jz^dyj;)ib=K7C9 zkX)Qw0X^1BW+!80@bJx4j$Yy? z*^a_p&|;_15gLorDCLzPomHWIo8~b#L}l{Yxmc~-=0fSMMO>29h@66mgs#_w=zzkU zg0LY!-?nD$hCl-hq+ZM189hl7uo7tjrL&;jnKH2#P#8(jv5 zc*45+Wkj`Cg|q7eTFFTcjt3Jcsx`W?g2bULbnKipKTQ#&DU~T8XC4)llvvl(ZEKzd zcZxS?{l&sc1{Z!+rc7A=<9&zh_Se6OP(Sz^$&^Lwcs>S z{&Vuk{xegfLaQbj9`Z6!_%nZ=-&*_{zwM;ITBGlg{{TC7@ACRg zeo<`M1ld!7D5T(novCqwTbbvmaYsDGj0CKl`MUGu(XzAWgJ$8pg=`z4qHx|C%@Fr( zP+$xX*!#LYx#Q}%uC)F<*A!;MH#>5w?~vpRDXPU9 z7|Cv?JbEW>b~%Ue4WJHtG6a&$s>B3+ZG*!kj2iWVC=q%q7g$T~4`t|=Fz~anWm`fl z3iC8%G_@%s6p1R?g}?6*`1wFv6rL!vQ!3R5F!rYh6&~ox6-&qsnmh=T#BhJ!5P`yR zzf+>|gD;}-@dd74KU^>UwXCdFgI>A7_P@CCu@*Wo$p_fxS!;#RMhhZBS#1aXfI%wn zTKtorTa|F_#*WNz53*pU_tB``Q}scxf?~!Gd-gu_^d!5_M|{_+1=gb<6vs3ed#0C) zzGE110S!zN)UtDCP#mypcv`@K7+dmJF3@8;SnKPB${yQ^qqMiNit4afw~P$m{;Mu* z7Gi4d!r0)~#iA77>L4}_?N_`KxS!(8yfX`WlX$7mpRennReH{>U^rvzvG2fHpO)e4 zUQbnP(igMo<2EDRGadrPC!q3tdy>50#<`;JbJuw@T56 z`)(@4Q*&?LF~Lf-A?30uOeuzGN@;XFuqoXrYDZB*WB!_UvKSj3kv7{Q`~9^NJ0s_- zL*6<+j(**AO>GksVZLE4l#Uo31y+?lfJ}Ip4(_QJ=I#5=ar^_kqaSTX@CC#d=CZZq z_a<(I1}jVc)X?uZ5Ao2;dHX5{w^kLFm1nTFAo1Qv?5m~}&F8w&J;Sh7&Kiv}zG{O&B+5IjFd1lR5Aphe}>~FwsogJ_?}<-PXfo zTXVMh%-YcOnsU;UWti`GEPtOnZPC7bXL$1Y0``Zam{_C3#sjEyplIcB!07Q3&1Gws zS7VgDS>F4}V8Kn+2rQrR@ex1vZ7d{rwkqpoz=eb_#7L6aHwl||7jzzkLssH zv7Zz+*;+0@xOSb179{Qspvq)<>Jk8GSc#u)ri*I+wWZ3?#9iEHe)9`uAy$Hhi>T&= z68{LlZxBySyAmVf>gl;#=fk%#EHEgq88F_$R^WyxT1!grx{-~ywS8hI!bv)77UO@X zthMB(xu`I5`*p6@ZbLBJwjt=+uozXZ{zqLjyd$|1V*4vqEqd%ScW;(2HTFQ^?0Dh9 z99Iw6u(=@WB)O?;tH_yOU5To&x~pz4;JXr(-GU;NpDXvu}RkUeciV!`M1 zzo5fkZAVKqTp`LvN0$jKYBD#m?W!&$PC68&C0ifsBVVlf#g#V;V;QsWdPi>;m4f(c z3>hm01~asxzeE(~&1yhvyj=QR_6z((FB|f-t>E?++FtDT9Qb-o)zu(YSHH|-C$8$( z1o>G8{^}Lo1}pwY@f{DmPL7BTmKiJzv04HXtTSKHbG=!>#Rv$PCiv!{m`E|UUetawYSh}=SOrOt%#RnOGWoD{$ zSTLEPYEjthO_OId$jM{N(19m__eNVZp2sSj$?m0EV=!lnbwRX?`dP) z>?3Ru!?CNjy$IWPAK9|9HAmEFktwEVEe&>#xYs=AJBs28uZyXQYBIo(8LA6z=FQ$Q zB_B1u>*N9W?UCqY#oF0;;zdfFl0Bwq)UI=rS;A|yGlc5Eas2RfIclc#dv#bPuFghC z*0bSB>q0lS`3rpooT%1T-IA;q*$%-k0Ftj{Sn5)WK~MlIbWqyBF)>dU^Hk3Xl~OEh z(=aF}thjJ9=Gj-F;OIN5I#D%3nrb7do9-Fe|3%TY<0*?)ES~+b&2SkpDRB0K$?R<1 zDR6-(Xc0ZJ{E&-peRz#LXOo3RgEQO*v z+=3vAorjgWTeDVykBuIc$NE8#ESoNSgqQRAr_wpkpDLq_J?wIl>{(9}J%T8jeluYv zp^(m1yV=jJgPQ&|!%=uI;exwM)}@Twh8FKvCTNY2;W{kx??_20#>Fi(QH=P?!+Cl+ zMmO2-IkaXs4cbw(NN6}pyWFZcGN~)(46%U*)ZOwFUtblBXhGvIPJgyp@6Twcug;JD zwIrP;2!>{=a09&`@2QmN%@&fAp%YC+Ap;)z&HSkaOU7Gi%#4fHU(@8`RrA@7nj zKC627uf6%qN12&!Padkzt(igWQ6fJc?yweOgd7(F=G_8mT~J@X~7M%92t7{a8VP4$E?mMEb?dB3P<<1FTWTe7Vm z{HaecLNxTpHXgxeSXoAWqo@_q5d<<2a!!Ad&Vk3Nebh36mo z9dqNJ9YRxB+8md-%^$RN2x(iE&Si?Rh}jF|rA_AhnFjQf2QeW9aoL>rI?7Te!t@oF z4wq;ZS&$8>aym;sya(Pj70&l}Q#WipEkX=_8#I*^I1(63CULXt$K> z%bS*_{qNRs{(nU%(*NZC0RP)^i-T>{x&HQu{K(8vygbOn*&pDFT0%-O9IXM!p4Xqu z=d83l_uT!}17y?wLO9JeGTj+Yjhz7~_ULVbMH;$xocjReD8llplTiQ=G=;UDRptQL z|8K;f|JvXFpPJZ%NAu6O-#qxHb|YLAy6V^V=WB>=T`tX0gi!4F@k2Dfx^H2HoWWil zQ^xdZuGehSkWy2xPVWcPB6hJD*}pF=dEYSXFu!mrbFWk|afCoYT~SHP3OFIsZj}z{ z@*eb%MlXw5;<}Cd)FZ3x*u3itGB!D-{Yj~)y~P)C9eyPO0}}FuR`z~J<-OYVqXzFR zw;GUj^{30(EPzer7ad=|&W?N(6sl$wv3JNQ7W7(8}6q5X&FM zx!v1Q?yP_rKUyF&Xny3nX*d6rJjPzLnvwNa<@b8)Puv}a!uvVQ)@r)(aV{Yn@xR_D zUL%_m4G$P<<);;u+TOVxO0A(%7#}#jAAbp}s6&ze0sFEgCZg;ihvQIfUA#rYHT8UZ z)yzLrWmkr-E3G>D+kK8@c83Q`%rRS*ugMYYLGa0NmtXH{!&Cm`uSQ>03@FERGfgd6 zzF`C6l431QLgNjosS(;KeJ@%(6$1z$kmyMPenuI^ev?8hm+$yCq0a8{vwr}~8?N6| z<@d`;$27T9{yfuVFaXO>pF>z?CpK*|Y%`U>M1&?_pufDyujaddgZWc}l1S!$M#DORX?gTj`{cvhNMN+I&@8E^zi zfi-q*IS)7gmQUWv%GSLJzc(+FoS9~cZQ7HCPBAu;qr;`=(G8B`Z*eF!ADRc@s3A$v zEKkirvsQ%$*8xKB<3|o9QKM3VmUoETQPv0Q{xMW*Y^iYc z%#3lrjw~6!_8uD!2*&8zj7mOVajE84 zjPeJ`$Z*qPX)TtvHtaGDMD%;w6ygYq$WQT)jo=VX-+&{mz*bYKAHCk-1TQJ9E5DC8q zf_|hzH(z?DS~?(rJOc?#*crzmd|3E)ByVcLYxg>i@mF4y)Rh#cwbdtN>;RaFjaqfm zi;60ViZa@%??p}9r$V5rh-bvSCq0S&P0E{sbQwAL_)ztxXeUtkkX)1m1xWyU6z!*r zSXefKWbMC2D{6_H8x+!*y?K3}XRG)zogMg+G(!Vz$A*l>JgH@ClONa6(19AT-OXs= zX@@;Vk{Q8Do)S|?ysAmu$|0M@@B=+ghg?+#lGhopm^ej{ECq7U7Nh6Avb+}79IcoJ zPE!jsZQYVOJNdNGQ)I$4)e=411ERd*)M`7{&^Q&0FjlX9b<4wu?t_?cGR{ik70Mc$ zM>4KQ;UY?OL=9yw-V7MQS?Os6b(tHr;dHKIKfDguDC$PeNJO$=dIvhp+Rq}#gfx8x zb)tzOC3V?Q=805AVKwINS6s|pT%ZpJ6^UU=hDY3bDj87bdS33jbijMw!uKr%wNLw2 zje)$=dRs3ctV|Jhicd&2*Px$f`C*3U`2BN}EPH}+>OFW=v+uo%lATU}5|aP^MUDm` zKsyaf8;D}3rhNls+9zwdG@Di9BtC(})Y+m@;pJM57}#lwQX!>z6EH)@xT-(;fSZ}? z&fKgdZNGmv3AI1N$IlETl~33K>!Q@K_n~+~LvS^^nXZ*Rj>?uJX+v^R&UJ+vT@xWO zQ?RD}!oXMUHG77k_5zV~-axTjIz&ysCDva1$@C&G1>YVjpU_MCq}GUlmydGa-(Wzs zKW;?_^2ntg%9JF)M%tbN*D;I8rj%f#8L%|iA%BzdrVF0Jx7|p2$v1=-aI>iqZ=LJu z>)`t6N?DMq`amZ`fu7#c!mm!R)?Sc+|M^n5^Qs|NZLKw}qQpiqJBN^I(8KEeC(p%x zDCJ3S$~&u9Br(6uixSPb3>=pIyks=5?!eC_%e!xljf* zqG9|~4yPgxz_Z0er|D;m%YKS&<(;u)A}1n(^sHk~;KQ8?v!M+<9~zJG-{7cbl6ugx zR!5EZ`LHjCrUs&)mbOh3jM&g!N6JEwNQGZo_-mf5SyXwoLPb|tS)j}0o@SBpbNw)~ z@>gM@;l{(f)h{-Uqq+%FzQhVLxszGSuAm#QHXz^Da6jBh-W(z?AW9D-qWlMlJO@=^ znxokH>;|c=vNB89t9GSx%lb#Vj%xa;QhFu7U1$CJgf3A05W2vjy7ehljm=(QAOjINW%Y zc5CT@sjn>TR-;-n{sGLc_jNS_-#Un_3Ne^#{F?jiX?j#d4TWEratMp3rza^M5DfGh zM>O1%Hnt~QHf-5K-)ZM@$;V_iT_95QkW~;P%_cE>u@gs2dZe8zi@Q~V)i2%(KSp1n z)miGin}~On=N*Qd;*C@GgQf z_M__!&5Ka;$$VM0Caw!l<1-L4X+e8?;|Ng=U0IrI#OZ<<*3C4KkyX?}BQ&?X=aD%Z zgULicRb z59 zXx$Z!w{;O@;n~`X$r=2v<3AFY#uRV7-vJj@ekXH#X9D8{E+;UBd4U`^?+k-D_^_g|FPmt)ZT>W|nee)&JHKdXXsDJ&!xxViB<-GW z^gJ-X*j1`EQ#U31Z{AO=m^*fU#0@Q=5qnsoCZP6e>ZZ^X&n1oa$@ew1gH@e#GG51T z&HBHTyY#I1VvaPKcXD*6?Qy@)OXqo*2hHY9{$4O~It2L3avki9a@Ca7IB*!7^od=+ z9N}uO@ROAJ2e{W#V7NsxAM?l6tV%J|DS8$?aQ<6oBxW>kn!~d?*KPkF&LhvnDgD^rO{%{v2p_R8yP${P`vvB$ zf_^EdH}c7QzV7~`>{>%O(o=4}WcFT3C%5X3YnBFEQK3|4OT{-Ol+{@I4{((t;eA|Z zisww9V+ApciXf~DPW{Mb4{(xJZeDUjoUhK(`|-y}#&Bh}*?2Y(r=4dc+T{4!-OU0^ zwVvH*#OupKi82l50H2%RKKlJ}o9ipTqYJ8l}BBoh|9 zp*!(QLY#bEKG|va@CSCZ^%Bz@E2WJX`=(8+a&s9*X+j|y zlp{HK#!cqca-!0!`3LGk2X&@(`AHcp{lobZP4m$3`~3W2syK)_Sf|{<5g=yzv8g_) zy2Fj4pX7b9tA1SDWOJm+~PmPWNLjr+NDQz-RIxHN*TndVs= z_B^}TvAw^E-46j?-(^BR2C4qxcggSOhSGbsAU(UWBNP1$yLhqk232Sw4J5UR2U=DK zq4V5Dl5PX=c*9d?Z1q?k+VuH16ji@N%b#~|lt19qH2e11E()p&&5Q-I7^!dCNb#$e z6&_A8K%GUxs-aa!02eU`A4gc2kX$Y-BDG|RlNry(3THBWm5U9|(+5COjKigvL9<&e z;fvN#UT)W~JV);)@Y#;+e}?}69N8b#P_OoU)zwMfa)Z?-h{jG&mk`b>02P_I)5aSc zACh%oK4Ew3o&zRsFdsa5qHfxEY~qXYD;Z=ueJ~q(2w00|nJ0p!(XnwVIl`s|JW8x6 zkkfn|&$>VyGO3r&BYlKP1(K_&CSY4NBx^)zE@3sk7J~l*LB&TP(>{m9po-C?Phlxi z^#eE@Ljm%q7I1J@mJq{bk{l_}(rMeAW}GxCcU34j)QP9qNtXSC2%60QQWNJEH;X>) z=u`WK7?Cp^8o+%TL29vggkchfZ&lA8q6I$Baw%mKlQLdM9RTe&*(&z=d80jEP zo^wr0zcQ+kQg&ux8+1_N%_HFm!y>QWcH~|#VR~|f9qT~RdnMv@Wa}oCrtbf&DWS15 zb0;%3Rf8w}AU5?zC; zyN_w*-%$=8Xw*L+PR+oV{ivEy0(jH7>S&DCH<~&{qe3hKJR z)HR2bLDPJEeQXDjdeeENti<=H_Js#X!<|L6U7B*=nW2=daYveUguBS+w&~PqL2H+m zAkS!aN_&NDSd5T1H26(oK0RuEL9S3z#(ntXi5ja8_F3#4`Sl2eDpF%gNp+Svvh zrv5+#8r|2O&Pac=IUD;r9Fz63)f_LU-J52DXd7GlZ*$!2r?;A*&Yq^`R;DM*)OKn* zNiMYQQYqV1Lv&=pQVF>ho`@fV3<4X&AxzG7#QDXncTl4{o^6Nc4 zhARfBdHM15&!y`VCuEKiuCh3@$;6EeO`ZYtV-Y-t8XC4&MU`g@Lu^odPr%0YXqeJmAKqo`c>X@>jgFgrD4%{ zmugy+qu{-^n37;9Pflte?tIXq0O(uTt>Mp&lYS|aIa0PHS+-L$4hp;?XHvls6|1}@ z|J)cF3m(ce9=k941hmn4d^|F|{TxU-+D2Fvyx2j#7!lMV^vbyv*~UL-i#!$`I2KDg z7*p8GF#`(sR!NZtybd8%LA;^Ke%*F?-x&nf`x(gw=gwN&*K9?bz7`fm*SiDbadcG1Um;eKYGQmNK4&QD6aT#oGAn9T9!u6z9CZ3L#q?E~S@5Jbv>Az4g^`1)--9axa~fM!`|PnbBzX5NPfic9507wMU}IdWCOp8tog4G~wC3XsjMh zJEaV7gtA@M%S|R8&Uz+1DCb7C{0l#=RrT{YgZ#q&&Pm>_j4OJ=!*h%K57`~J4jVsF zC2Q&VzpH_d+US_R)aaIS8)JuirHs(N*?0<*r*`YdJv$%Q{ZOUTbrNNMPP_$bvu*|+Amb|Z#-Tg(&O5Ocq|4)VJ|m!RJO@YeCLR#h(|(>r-EeMwRk4?8QO#wA zvgD?H&!qQNkD^d~wf*Agx0*#RlD)I zSkvf0L{{dw_Jb4Re$Hv9iuQdlO)+_@TDM!a{US|lL*ykNb{iH1#$g}3B>DCkGqibA zD3pgs5CAHDZThU>c3URM2z+sPNw5lg%>GUm${TB5G2|GQD5n06mWQcGHiJH16PQFF z4-yQa&4<(DjV@3u>I!{(%^D>$v~NzMt6TRuwyko?G;(NoY)!r+fl`*r4tDI) zYp+KOhD~0~zc;<9vjayR$mVjHm zQnD$4b{4I)Ppe7ZyYUzOeyylF`c{~0yQ0b)Ic!kQZ z2`xGH_E-SFaqjE|cVvbUUf)Jh>^rnWQY`y>rCgxX1Zd!9M1=^bCRiZ|&LpLItt-*1 znLwEqDsHK&jiq|Z65C4gZIpV8@)e!%3s#VmbU&0~ zEHG8AhZ;W_SlJ>jMsU`MgXpWjl=pURAvcR2p7{LuQ)%U}$%2sZtDLJWO-N1hAV@d( z4nsW6(&T44=EU=~PN8}&qvAL5mO*r~E{{w?mFvzM3rl9Cy>tbskgteZungs1kw5z6 zkPup7a7tijjU&0GNT%~{BSjR#Dk#=2Ap_GCtomv78f^v27WxOH=%aitt@ftp6Lx?& z82%52`|EE|1$afjh+d)&Staa{nRcp5@y&aY@4rg*7hh(bHF(w6KRki{;a6i}{|BS+ z`tQLHFMUPc&blb-#Ly`RKc}FwoUt8m`)qqdz_& zrvE^0Z?v?{MuQGD(t~u5vs~+ZiB4|t54g1&xH+W^i=>}gln+F_%8k!|Ju7aECu9)Y z^5;Dwk!x`;^k>m5r=xTEyW#sitV#HE$!^{Cf^T$$twMuCDpapOHE|HU{QRO;8Oa*( zhm&mjbTr#b{=FVazG-gT^bhgJ>FMd}T92D2hyZjw*3a+9C*XbA?P3yVYwFJIo;aJZ z=D4bmeKf+BfWEDt1#UQct>*yeRPu_RwIDLNr^ry#)wI?L9ggL#))FGPQL0T5=ohyr zEJvGCx(RE>fX5^9AO27p3i%8@7P*KT#JOS zMab048tAar30!1KiwOxUdKK@Q#&;}vlmx+Lb7e60;ah#-myTl;gG-1fe82fT?3F7o z6Flo^XHTvhS%JEmzc}r)zh|kIQL*;YRPyT=5NH!b!Z_cUXCCcRLiW`nbwa;gcF0$X zk1pEW?_JH8xhw{BtO8Al0*^m!x`>*cYx8iRVPf;7M`?Bb-bbmnZtB$UAL4V9!3Z%A z5l%G}t-#z)cp(OU9O)f3Czr#WFGnP*4=Dx0#;p_{yM1gVf~X*>^<^-HmQRFY8N<;!d}$hvJM{dFO$48~1RRuMgYA(;Lu7 zpf{2Gb%0Gt?xIv4%iF7~+ET0)`JV=qRp|A?XU-67P+rZr!xDHCAja@xI5*Pg!=|uD>OH-tT+m# zqYEuAnWxJMi`H43i)mgVpRWJJdH3nAu-#|!de6gs&J?ZHa=A5;`Uj(vqnhgBx`Ml- z{w9UZ8qP`K50C+rp!*QPoey;G1M5!C@ijLF^_%hIxK7^BV?P#VR$R=RTygCl3N{l8 z$isw{oaoFxd(H4QJ%)AEu0xyqSLOR!?un#?l8rbYP-}q?5;oGVpMPLKN?iJ%1vH=b z|50XVED+5-f6JHO3ZCjVxaz_(hHsYiZn6hfr9gkJo7bN&oW~UZ8@mirY5>2=cuzJDbE0_s&$aQD&e#(JvIR&YY~dufIBI_#Br{N;1=bqbv< zmHRn)$^K|V^Db{Cw6%?kw#m#|t(s7uv#Awp*=@z|Ea+jG@uC6|m@9Y+^CYT>DV%`| zm^zTdFWdf3g(@L!|G{`#-20W9xLBfd2CGJ7wisbXAglHb)HDQ^5uLSv?&AZdKTiuj zF%U_uVrenj=Cu-Pe?6A+R6Dj6IT=Crxf*DV9;-<_bTdysDcvBPNgUCn+DYwI;dwW= z#3fRsFv2L|#9$sH0?n9jj0yN){LJ7ZCzf_GtCmstvCVI11R;WxfEVx&2BmYsvWj-i z+j8Qz9GGmjY+SZ?>eywQ`*nZb#ghN9flf{AT`>|@zXZJ$Pe%3HyFSfw>@yTD&KHOC z4S30awBpH-9#gKG$mwH2=!O=NA_G|*P1K)Aa~fQl5C_*ChkZ+&u4+jmtLD5g0K$@$ zV^6|FftuTV>!nS|$-}$Y#GLu_b3s~D9n@HBCy#Xi@u`=bduXvMx`c;&+O2#j2f$Cn z{Z^7(SBNvL)e1?)ieAw|hegS9H7VtT0m{@rUNn@Ys0Ru7Q6lBYU^^t2cRv3~5z= zav$D4Fn=u%?1TmC*%)e9R$Z6Y{(^)I>I`f@c->$UHGX_#7vjgkY(z~tP1`~7Ww=!8 zFvL((S$_67#%ojcsE%E4|qW|o1$2^Rn+Xx0N2 zy9EKLtH=T(TW61>P~=A86MNMlUcMLbCTUHeEjuW}Fho1@sOT+~f&vhS`VUXiD9M`ifENu}uP{GP`?_3R{SLNBe~N)^W*Lmr+>NNiz*_$p zwD$}BlZlCX41{1vHIbC!53m<2 zOV)5tjtN@*;p_V&a+@rqXA_Gm?Ue2K35`WW+VA)Bp3g*P^l~$3B0R$NRe=e_ET2IH z(DX8>Y=a{;i8QAF`dblI5d)YJ2Nggp7YB`RV}+*8{~`U!w9Fuw<`Ki^cl)!{P-rch z#7tq0(OfV-V064Z$=EH5)OY{HGh=Q-CnK>GM(7Z$ZA}IzWvW=xX$(uQlLm0aIQ*h4 zV;YY*6mn=;l=_$_{=MPgIrPxxGKI$^GiV*$^I&OznK<&c(O-I;zS#NOZD839XX0=CMJ2RO(Y(Uqyl_azfaJ)K(;y9$vB;n626Kt)2f+@8qyk_P5k8qp zXBcdh(Bgsm?!3Yao}*OvR1&%oXkaOs-KG6uudM^$Hy`;$#juMNJ8}j+6%SJbDv1aq z>*B4HZ1gi#3LOM9bXf!i>*7hIVsCHrWyTrajQo)eU!r{feH>4)H_A`&DEzlx1dWa; zVdmQ-T4Pn}v*PbsXsE^a_+9%{Kl!XzLl1M*@Jx35?Am$X5y`=8i{j;-Gb2cb(9PeP zk`Z1MT@BovKC;>?b_3E_M8@~bBv7ZVF%``%N;TEAm+Spytx#9F@Cdd-CKQ;cFYx=#*t#l^vyO|&z|CH0g4q7*#&9X*I<+Y0328(iP65yGpB~^n# zcrqR^pq;5K&oek(UGIb$o3QAfp*&St*YMdz1nHTm%<-u%)4*I~3!wS|GHX!XxaC&G?1+B*?yx2)NE`T`29&aG#4XHf~GUe+n+!Up5X zAmh>*9ToAKt?m`|2fClnGn(qh-d{4FZN1DNJqBbLU_}6W)8nI_-7MXSi;4GnftoJGuxq0mptzoIUhw0eA&NNsAWc7re|bYctfwQXc;(Rvkqkn zkKnrwR;X)89JHIBB?-o@Y zWMDVP@_&VlQ?MT3EWtt-tAdwvxdk-9ZXqOzu3|RpTsW!k%_V# z9&au}sVOf0!6+kIDUqXiEY*yXc~!vcK8~EcWa{{aFkvn{VNFU}^kUPBL$ZdlzIYc~ z6zx>}ND!KDO`If?P>4_&QNM|MWpkijoMH?*|nzw}xkmUJ%ThoD2p{F%l>W2n?GQxwY~q@&{a zI+l3wlo0jr)=x4NPB^kzpA1UtS!JUzw52yIu6)m+{w8 z$*BgvmQ|-VL4mgpfD&dJH&JU#aHG)EUVAW|oRJ%MwS&3z6Q|cd6IqLTvnWA`VxLP> z>yZYJmPD+ICb6GuDfqwbKyyesr^UMZoXq|o9l8od}bS_58xX-~EQYUk((jpzi@eA*0}-s&gbWOCk4&Xo&(5LY$Z z5)ilf!twa!SYUKH=(ZrK`|hfS8aj)2-e^A|%TiEPsXjR@d#7R;r2=(OZWW;uH;f+f zWa3gnjbwpkpw2uhxHNqw#l$2mv2Mueu!eQdQ2E1J2cjZ%sZq4q(>3awV3J4dnl)DI zM!qJ8LiqFB-{xx9cbpvvqGz^`uzZ3Vq3kgXIJTKJyIQiGOsnYLHm16Vfq+5UR=!cf z#3WSuD=oS#N1*j$Qt0qFhw29Jg3AepyL{R<^d9NaP>i+Q>M6HONK^my-0|G}6HmS3 zLE+ZCjz&;tpX4oV#oEHbKNwqw>cq8yKY}l_ib+`B4SU#=mAmZ^7kd)@y8d#`ppu|v z@?OOHfP`hru88FUyc>Q?`vI-c>ZLAq*td|LdIoUg_{gMKL`m%IbWI**5M z{z|>&B6QI34>!ZqIV^*Aare(`|JVayEyuL6w3-_H!+zT@KIWEK$6jlyo8Wjh$ZoBQ z{L~Qy>xjU8Jw_8YFUVjY-K)(tVFOIs?=|8t(B9uIU#wjsU$Uuw%N0m$Ttt+wcIsC;I_n0V@WuzJl}WDCV9~KRIUMSR zOB5HQp8Y}yj_Ss)#!emti_qs2P?_|`6{nY>nGJ7~PO9!Xd)}`uMluH0B{Ep*I3cvvJw?s~#j)I_WuQrbWQV(fl$ij_WV-Lv)MuDmA5X-yh zHSGptxg1X-ae64tE~TMx<=eH=S{;$)$j6)8pGCSnGBuyLzy^=nh#j*{+rohIxR5^x zRgPQXW0KA$ZH}SN_F91-DNFnQClbr?{I#7x1Jn@T$o*jMm{(qrrQ|*;J5<5K%t7)W z44AK!{$SVTz4PI!Fhma#%60%zoNrOl1kK2$PJ^_-I6tQ7`u6~=CP1KL4sO=hL)TGnhVPzG=vSAdwb#I<>DdjK@g8;?O+#+>9>pIUZWTj5y7PG|5 ztt)&eBwfVTiHf$Ws+^|ZIR^f09(ZVu`5P$Qgp^U)$Xzym@|7`bIfgLzDO-2soj1wM zDkE$Rk^0l;nLQ-B1eVVSJ>S7yj;nTFNq!A>bYgCQBMGpJkSYzFC**Ycyi^6ETJUq$ zpf^GXtk1aqRs4v&;&zme2p3B_U$QJSt}AqQ8LRok=ylIgjRx3*9+s~N#Bn2_eP$Pu9{WcNKGEwE2`{k0yR(Foz& z(drtP;U_IOI}DE@>n46am4n%=y8R06bJl)40=`Xn0pUVcjkG;K1RE_&guh@irh{Hup zmemG3Qn8nH=9IK%!^gL7LQbE(evDP^6>3~$^tbJC8s~uxa3D$cO>6aIF#&fL;@eh< z)yjG~qK46Nld2Aw7V~YjKx-?wkLB5fM@$6ZP*F{K@R_aPgwV1RM<$OLY$xlm$e+Fm& zsFl-cO0FzV2<9=&A{L_!-f>c;b_!Q@-k|yx}?jSY?0N~ z2iqYW{S33+a8A{L8^|LVMDI7u(8qz-(G|C_Kmc+VL&SZ0jYW|fTOp64vv?1yUnU*> zc+;k(B)Mt-9ghN}M^Q-%gFg}E&=ur!KyvFU7083fO9|DE^={eKU_}-(TBe0EY{r+C z0WMW)U}cRd`reNEmpS*~%NztF%&4eK^XI7`!`R37ZGFDxdeXE*BCT6H3xuRY^~-Tp zZUBxr2_LyN&pykt`9R=drih7?y4G*=z%EIDb5A0z-XyYLyl=LcOPb<1H1(g#rF?+< z$dF81C7Kx0Lv*IOpnf)*($_?I4I$f+5Bm#y8i1vE(W)7B9P-d{Z12Gr44L`1w#mYo z4agmd%QCoFA%eX6y;dtHz6h>IN|0{Q&Ne7{wk)QxtF%JCOZVWod1`3Ui?t4+7-(l5 zd0IX^`(Cq`2bFAmLA9lsI+fBK#j^^;lt^7h#;2VpaFMm)ouEYTHUhsm=hH8?Ass=I zH~wP1;O)T=f5`p0tT+Pk5BQUSJWc6bqzn~0IUUWom!RdPF0sO3bLgKU46y*XBSR>cvOxS5G4VXEWbZ@ zs9NQ0&#)CgK1)A@T}*nEQA0En=W^7sU|8_t4#;t|f#$CszndH#Z*f&I}mFTb16^hLW0sG(>mR0^=B-o?gDMEAivevhd4n--t@8!u!r zpa>S88(XHAHWS&depq;!3Do9w*)UGuC}}kId^NxSIjM)LX<#Gkv&IJRv$=|L!4;N0 zUDVKaOQDEasYe*k!VwX*UVHc?`9~~SPWmQegXM_w1j{*lCYl7)tnoHQ5PXH&JG|x{2%{PvE8DgO zRL=eQuy>EJ^}PIT8lZIiz_Itm@bEiKbd@j#?vS9Q(p3nA3si(!sJuG7>(#ZVs*`_* z%F=)Jcc9^wenxNZkE?$$nmGC7N;9OHvtLZr%*LJ`^Co%f zME-9L3I9>8+e|eZo%AICf8;-kVPDVQPTci8vD{HXJH{}u9vSD>C(w{gU?;j#klTXjp>d2Mo+7&WhuM zSq%NHBB*sqjE@$`6eD{ApsxN1|1%}zG_>W>-ggBrML1NGU zyN9!5YH?Q6$YBNpb)N&b%0LY2*Ans2eUo;V-$Ai85{=N70Q0ED9!Wna;g5>h92d z!>s(UzodJRtF}ZgCV5bkB*^h~_hhY!D%#e(&xiTceimc`KI*cm5x>+{SNAPDBorrQ zXae$}p7z+QfVop1%cq^y`S~nOfTn#A=_s$ zXkBr-IuD-jED=<7kS_Vtemi0Q9dq5^!ItVcBZ^zVi08F2)Lu1GAKd>}WaU}iR(~q3 zh>YU0igJ7~oaN@S#Lf4+zlaIFQeG0TotN8Xgx89B=#J!re16p#(QHeLNM{5Fp^=-T zw{$wv1kunpsNiV^_R4N@l!3ShKjJA+auSMx%>k;azxU!J55kf%D0up^Dacf!%9-oPra* zj;qFIU(~Y<5~#AMGEklPu3^7xvu_lXVBR*@N2^2xY%B5a>Ei+HX+BARDH_vkjZ92) zO&UufZxz}1lm$gKI1TKf31kTH>sUcDs9#Ma0eDkrSAtvSqbfyuTs@$48sg7C3ycXL z6aK1$=S#{A1sPGNy&AG7n;>7zX5`8k5UeO-L$E1nX;swIB$WVO1YAdxBJknkE@z%E zn%o1}LpzW*X7g(d-<$EUT_JNOz}FIY0Z&@c{Y8=Ck}SddV{1tPqsQHQaL6`BZ+(15s2tt;cl}m)J2-?C&s>g}8fN z`l;M8wt10nbY+mRwq^GY0;^2s2C<#Epf%t(jHrFbUkToB&pW3ozvn24Y5SWcT|J#& z+9+e_p)<1p(V3BAjxRxj@KGBlB6MqmDgLLD`U zB6L|~SR2N5Oj!t3DyM27D;TDIQEp}OvX5O3k|T?MDfbc|c%eh?%u%kQwxW@8R%iB_ zB{+3PQzx;Q5ZD7_<1rgFNh@aRShklhXc`FYPr}Fh+yp3NE7{sNh~^^{{5tc|{kWkXR-)RscOvy2LOD)^$Q!i}^NZ-mMtA&;P_O!?Z9?7owBN|vU? zF7MH%ys)Bo#mV-4LmNt(tfBGFwTcq&ilE6X8L^AXFpBWTY6&2r{ij|$=*H2P>gQg` zBQIjZY80iu(%do_AcT>b(Y)723ec zqAT{?z$pwIk&!uR#hs)bIS)88*anzDEkgtFu(dOvu~&?joS|kS);eE3<=r#emnrA5 z56jL&iVxx=>nvjOSOPHKH{bZoQjdnl)vvf-+D?Q@D(fQVwY|CZCPZhoh`qtR1BGq0Uxqx~1B)3cBkgE6ZV^fEKxpcc7f47q`(1MyKWoUq6-PD`w_yz~ zK3V)zH;$p$4MhS3mY6Q}qJdVXj++=Z z!ahIHcCjSmOzL1_ZREtWY^6-+Shl@LLsvP?PXcwN5LOQH)1^7jKRtVj@DSkCf|vKo zEvqJFU;+r`{ywEqtZF7Klk>=@f^P-OzK=|<1}+#1f;e{9$9}9yC*G#dDyw&zxxB~* z0M%)QF9dU12ycUte~tDqtGAU;p#LeIZJkKP zKNuk3^M~+UiL=7B+CekSn+GQW`deLCjuGDjXw%0pOX6K@cP<-cJg|35mCjMqfvZN ziWKAkbPZ{04n5zjE!eJN4=SC?xjvWxQRUCk?aNcV+ z$Tmra3N24?O4JmD(<)>}8wI^n`H>S7UGdEOr&tmEJYWkRSjtLySMo2aOK1jpC<@_d>? z>{PDwx_yTV*KyV^R=+1S%~8KXJ#-B2Cp1%jQq>&dP*y-Ni?-M;Y1i%*aony}iUdsy zfBHr5X#!jk8#i)#Ov~c1m&Ed6%L)?fkzAPnWqC&ShXy?_cG-Xia5NY4FTdxCktKC37;h5QYi^giHSuizHy8oJ+8-sapet-xGmK&piEg1)Bho-~`moPk!0_QxdW@3Pz(2HV4( zrt*TkLtR!?8yQt+sdQvtY6A{)I}Q-&_S>COhcFmXz7c_8_{i@lhZ>#MDa!o22ltHY zpEL2BuC=$<`yjX&v|nuXRB6}oJOM4sZmg`x54CA+S-YlbuM2Q87k^AQ$C$^{{V4x&hq&Z8d!3u*?d>4) z`H15hC&v5J;FTX9g+Vpxt zHapiH%arKk-4-Q5hw=1ITx0Sq;1UX}s80}By`B+?uqw^@Yh>dS>!O{T1q?+UO(m{= zBcs=XVlcfV1$jy8>ElChUr8`LV^(qV*jpn_ji_$Oo6jSE%b%#q2=DN1e{uf8OyDc@ zhEtx2!$e+1E;$%;s#t^qQ9YR)LDyAwjcm-h_~ zPMqz8);#^?o%m2VKs+5|RRdSmku6--r^S*i#3svAE8879d>aLc6O2Vjr8$OBPD?$; zEE!TnGwgn3>DbZ^aT43cr;k2Jyfxw#OFeW`+M&`^Jg>u=)94fSW{Zm(#Lc-$=rMX1 z6rlYK)+T!-cs^+CbkSp&BU{;o6(=BH0VZ2k)hk+7eg#75ox-VZHr7fe-nDP zUuXMMI>Bg`^Jq9j=_!QEPf3v{DJu8HXprZh(uS<2>_siWY&pK0ol0{!WJ%8>QH0qM z*JKFXn21MRwa)TJq&Jw_49X zo*?DznN{)MQirEpqW=X9gC`fT!&(~I=yK{S(fdXbwvtL~N4;PltM9qvqfT^bCk@Hu zIcP|K*$fR!f?NvCzD{x3;N7=kQ7kS+6*pe-CB;Q^o|c~@FYxVrBDF>)dHu9!Jx`+g zq8TFMdPz?cfZ8b;jEiHlHRhMUdB?+Ova0JiG7LX9qQ1S7iajdvaY#$)!yB*QYnSDE zhTx&Dg6H9US5qYt^2A0b0>8cu? zunZtnuxj?+#49ejltMI5QoCWq`1GY)O9Q*Dh>FtTaz(%y?RNlJJ8i)kgc~S9BvL0H zSL>K}9wjJVj$BiD!1>}ZV&7}Se?zTg7PZwCViCkvGRJhLlvqiEC4;?iRnlXet!)T5 zTyW~nVh^u}yIT`BIfi%6nK5^-vJM~VzG*7|TxtTAR0Fjdi0h161;=n3<5EnKyM&4- zmKt58w|IKn)%vs?|K8*?8mNU+G*EjASU`QnZfZ^Z@yt3wBZoH0Xs(8P4`ikiYXO_` z>6E9zHxQ>5xY3wT`-<9o2EpJSh}~=Y=zMnZ!YR3g;@8Ejsv{SF2(<_`etJVE++D6r z>fI+k71Q*^+-K>`05iODd3pE?buL$es=k&u3{1!o72N_CZwQKPj4n=Eono#DiAKC9EM)I_mTqqXB#D6tS_@7=yv+86MVuM-G^J zR20!=%Z@q>AZ}J(#@qFOC2Rk8z~=+7`aQb%&iA$qeYXAuA5#wqti?!gIlj?6ajks& zsE#H4ub-I&&D9PZ$NcvUX4a5(%v1NCmiuU@n*A@kLx1z$VNX^3fH6JYj=N`{a1U&` z=~2-xd`NZ<7`*Pj*+yY-VZ^KbgR#9Ij46WZyU)3O>4YJQn(Ll$X`%az#%XV0x(q#U z`Uj&61K4!#@4nkzy6pDzXAkyrx8%-sS%TSb-i zi*R6cNugSEI2QbWHz`Tb%a68>{qA=0^BK-RPxi^aoj{`k9V`Nhv!X-nR#j`zdTLNiY`hhBlTq%9rs zv5!9R=76;@9L?co?v*su;nCi%Hd#h0#a{HpRHh)-HSG@fJPRStKLrn##Xk!g1Y8uX zPIOMm1iXccho|BMmU=;c0!iemJSQ$SXXqKrT_-+C}c!Pde5L_O*ZsMZ7;JCkbQ96_3;9QsQY_IU3-QrDNCSksxdP3f4XcTb~&yrC3*))>dh+M=~Ht9T+6{{}>Kf3^bC_MF# za#rtm;Vf3Cg{dPG>#?y@^}aM+j>xFp$puc{nqzOa);}4F?RLp*U3%;j;%%3t2f}_S z@ya&jkwFx@icuOLbBNzY&bLZ_C!KjX#8+{asHrBd2J|z?63B(WRtvgEo&Pmo0Ab%J zKkGb8y48Diymm}&q^`XoPinn%0G^c8V0Da_Wi%qvJHL-R%kfhtQZQmZc=P>Ak-Ze% z*sVC1xSLp5J11su1diMU2F+24f-*<(?^^lUtF+*Tag^?fv|)m!gBnN z{%ShmV}w+UNW?5zKd}aUO_uaDw<;ohU=xoxn3bsAV#ho1ZI20d`#frBTR1R@?V$`(JwM>6eyo87@h;oOr@kr00_SjVzSepGf?4CD(PCTeS^tu|H*_E5pJQhY;Y;s4OyDGota2FcAXXmKISDZF{eOFfiHX zR%<=e7OlUXL=v17A7VdUlC7KLhLzWC;=RO<{A%|PM&o%05V=GGlh~4Yr%mldLV@5@ zk&Et7EaSlklcn!8Z;9=t(SeunJcC4A8)!(^@XVq4Pb&(0mp9~Ao0WB-ZB&D8{sZ75 z0fdht(0j{kgxcbeOlrYSMJ0ad<$yg@9{Mr4Ns)YKx)<76I#W^qjnVe4md*U8)Sm&t zR|bZfLqA;C`!>7|X*;*&mp((_YQCs&OEkFp*!7I=hVkYLB&y6axe_6B{Up&cuVail`O> z)17CMQOrs)IQX22&F=$Zo|c2W*7522vRMg#v!Qj8UC1hBs;Hb$D+d-WN(%v&kc{L) zKcHj>$J&}D2Wf1Gx^K_2A}2l<6b(|Jnl8X3l!qcC48SZx)7&_{m$??KvG0t+U&NQ% z?9fzmDlJ>GEYdhgu+l7(R!a$R-aISR(klJkXKOyrE>;zJ{n5`Cl>Fg_JZx2i=QuVr z2f$@_Bxi)Fr^6+_jLAVj+o$-sg;yLB{tciJJC$W~C6?TB))<48&7%DlD868)x94O2ZP20=&8~RQ4y|gmUTw<CdEJPP7%c-Xl~b|x-~mZB_paAsedAMHEJ{i_Y(uedmP31GTi0G| zLUilk(ooeI1D5cjJQxEU@M^9RST=l-k+fZodQ?Uz^Kg_LX1?1|{$FIhWmwZ;AND^| zT50Kyk^=@{K9AWAo5fTYyuj!}Y2OEbD9q*D-)^4Wdg&-3Dc{Exlau~&=V z@7i^p-}C&OBErH;gsq!F%bw5YM|?N)I6awJ^#2V(p?l^RizOxJlyFcRFs9lGR5j1qN=iCc5`7*)W&=1B4hM zP?0vOJcFvxG~Zw)&)_&xdc4VE|7a_Psor@wMJ>)h05NLQRk$9fFp49L*-Y?7?2Fuw zzYr}S3s{*Io|V2x{bY(z^6Ax!qbRiKN#%Z4)xYJ%t-$3=BFiSy$?o8t$wjHcdUY+P zsClO_nMnSlmIgW(yV-(_hS`5(eC~?vMEs5n)3yPXDu+u8glO@%+T~9*J^qT}RD2im z7Gj{Zb4;?VyLVr2b;Y&Z`nu=+Z@vU``vv($ff4=K1F&m-B&?Xfg5tt6aHu6XMawT~ zCM6t;erWN}KR{wl@;|_rd>P!@E1|~6)bdWzBX2%$wV9eyReLFTAtqH*X#yPB=hZaz zo;)d{xERHdrSR)w*LL3nN>B@bs%GxBL@t}l2UJiIJ6gHhJ_yDA!Sq<$R0xi`X6?e4 zkMgi?==de4v~4ybm|@qoC&TiSr85LemyM^O{8a<=>=FJ;chl-l76~IgE@zQe1(r8lAmg9W9b1-cm=;PZn zs?`=>Oei(}sO0U(mPGXjl=gPvCxwwxVu^J@(bcO1o1pjC-shU@GbLU2aRW1a-XUxS zTUj+(!jI`A)DjC46+8r7Mn;+fO`y{kaSV)WoBdaZ(w}+`Orm+kmdC~OA;r4_6%IVk zKVFZgMLLZMRxgdp?p6swpN62k?|H5gD`_b{8oaIOXd#%T{q5lU-pBw(oLT&#E>TJ8 z%9GVEII&wO|JC$M_gJaWbk<-^Tfes<5>;9Wr3JG$zPeCnDabotpW^Yw;n<`Fr8|T9 zr&{n*#(XQY+&f7!qvM5-Q^mwnzf=}B{)7~7H&Ow`rQHJ?2Y!)(HK`ffl>r8Zt@#|d`5Rh+qVD~Q9xnq1*0Dt+g5x$xg*~;sP2*it*H+dWHEU( zNzHHe1=)R5?2+P$=&)Al2r)uSJV^`Ym{V$DctUg;P|^7hKn_)(;514pY!bW3omYV* z&u&wS7wxyHb_-r`UMlC-!LHHvSpWSdw5q>rkHT{w;{IR$E9qrOR-BP~m+2jc;fd}5G{RLR} ze$Dh7^zFsq-Qwv#VIl-?0ZohQZT%|#;3>FYA)Q-hv@_Glp`w-G8?25!Qn(-t{evH74By7KGT_hP1{xl9(F~>OP8}ojcNHx^fSF-T+ zpr2sfxCqoKD099_9FAA)K1q6{q~aOZA3K5T6L%i(v-H5Z<@UyDpe(0kktW@-O)WPd zFbxJy^AtsFP8XH-JSm6h_Ok1I{NgxEr^9-{+B+{zvNh!LOZx1m3%LcTB_-cZY(c^> zqdqO@Iuuv9Mv{ZFTyr2(Kzg-$c74U;0>i~q^Po~Z+vD=2?a(%;BrO}{gFk7ICLlpd z#F7J?s^%*$itNGZ5 zC!jHm1%P%gZ}G)#f~zPNnMr)DM|TF=B&Tt(J!C)s#`=4oPQPCpJV|xp57S>`(vA1w z+qv78uEn)4DRYm{>5g(-L^?07cnNjQxlx3iHl2U_g1^RdZjpdYK!dr z5UD#p>}zv12cs~Bt=t;FHdVc~i8jGgcuA6c_E1Ps9hgRnFU;`fYm2F^X+L}5NF(6GF`wg+o>~$!xDv# zCu;5Lov~ow_Toa=mUgdE-8T+30)g*PCjn|ybT5(s!|x4Cc*JSXi!IyC5KUh_FwQ}M z!t*pVQ0`YZF<7PPMRD&UaSv`xW3qa z?!57IGFW-87>Zjl@0l?p%6!e)RopUhwWQoMzCH8cE$Qz{H(p+uO^oPjGD`M6DdA0&6YlN1n{fEoO*|&o~ zZ5pK%^7Tf%73+X=xs?O8!EqfOjpD7d{*5~r$*xO&ITtdnFVEsq+Ok>j6ggc-s7XnU zmF@5lf|{AKq+2`1Bko3awO>>9@fPl}pW!@8K-&CQpN}W~UYx%9tr*(5#X3@};s#>s&%4j6H)9EldC%jGoMqAbyt!|2i#mkOVU};nu>$6v2>Hs6D)ND^i`${iHzZk9qQuaszBHozE+x@p&Vi(?4cdNA= zg9BS7At@i;XA&nR@H1_e7$w5jKo=^Yqe+QgetNdz#8I@@L9#_yvq1twsRahqEMBUml$eLt z^RPP=1CsP5gPl>cvKgGyQt?*1O;-yifQ&jRN5t3PdC60z-`?pO3lStO)>#Du>8v5Q6R{tNNu)e;{on>d7KZfe~#A`>utN}ZcaxtV!E@`}FDvQWnS?sM5@WWWfz zuRcU9wC4Ofq05?IW3wgs*Pu4WMlN2*ct3ZRsR!X&s>1Q)cW+WS+H^jI=`=+~ajD_( zFz0?KS64_%SAfz<|Ma_73UIcbep>fbtCrc7kwH|Et?3CHAJ^DWv+u!l6?|Vz61;$( zCW|SseR@F2?9#zT9{iA7`fuYJ2T&-eBz?f;clo#eOR)SEq@Nq6^XYlHJ;fuJPd4eg zvVqQV6IR1O=Of$Nssaz%iz!5;q=uHHbpPLq^c*w8%o-Py<$obOj{iw=O8-bct4glRuVht?aGU4&j$&R{|1($74=~@~UwHBl;D~P*sBKsU{#_ng zAiYVCeEU^5p4H$#>K+lc(&F;9_~psF4EH39xBMruiRf3*d?w|d{BsO;(!KD5$qm#Z zAQ4?V$HEn^6;2OO0dTIY{G3=#_<>nkc6WWbalRVHcDslPCcc&qzWi&2sQ~!>5Qao& z52eVz)NlU!iY5R1@iC^g0l>Vzwf?3wV10ox^Y0H#*oW>K5BK@D(OYy)-&47NfS9bm zUjb$J{@Xskcj!W2VBkH{k4b4WT zT~Ix4Qz`ZkmaynDxHRx})Q$%$lWfyt1I2_mRkIo(BH|^g;21<5pF_D1U(t>F3q| z27WH5Qfa4gi@?DL*F{OKBjk2>7t9eo`L9{PZl&gFWwVtkrPHm=iP~?vw78xpZ{G6l z6m!#mum|y{n2k`^w-ng;aH*w9Wn{|3Jd+yZ)DnYUwD1p9`~oentXPLU)ofe6=kjSB znRQ;kad7GRh*nw$ns$7?AdQG1s^UKe4Zw$j0DMI1~DR>m#cC`+i zYHA$U%zw=JebjCeR9sQ%tiv;_kWuv$OpWHe`Q!H9#rY6w@Vi zP7{kJzma=IRmr);7pnNJI(4Dn69s{c3$kH>43miQi5sIwo6pG8rxCT$h2m0I1mFnT8 zN#&6w_*GSquaY@vIoN5dTB9tTyU3TMWDmeW@!FcfM+7e`wc}@qRW6Fvtew>P6-+7W z88nJ^4QR39kwDkl4~As}6_SG=?gpEVY!K*62uIiT+>a&)U`~TwVTH}`BrEY@l^VF)xp9^SxSgG_s;O|RRY-wDy14Zd8}If8RXMx zX^6E0o_NhUP9>6h@As+AJ5}%A^lkFeD3L>8H|pmm&Ey zy6`>*o_WHDn0;sU>vrx^)Qwq1Nf}5wwm1K_%;#nHAW~x|p8;00zBGNDQcv%2T<*gk z?d30PDzw2(N=o&~Jpatg=Tf!wS0>VdS&w_1+P-2FqXh zaJ5Teu-4+r_QZvzCQo&)Hs0Z`!uAJUs`GEmxtw1YyGOEY3Kd97S*-ipwu%O7uO!GW#itXov| zxRX6BpXzE_a1Y*?R5%dOdr-Cvdr8Y=bDPfG`&Kqss-KFMGuR9FPGz5sS$7Di!xg>k zpC2P$LlmXfUaD|a-MxG_gmi*}im0{dk5^p9c7v5v=>s!ZHkW4iSuh4T@cb3XBHwLo zkG}0`9~5zI#pGd1jzr4thW;|iuA;V0$?VZ^o*b_-v-6{1vkv!LaLdt;)q3dtaFF}f(=u%(z^=V z3(NA10Z;t)T@ieo>oi{bec&k?kACKM)^3K6$9?wNEtge1of=QeARBfPG$W)*scfGI zS$-057jZf#7W#;=N^)}Hjt|cXSC@9tRC>soPr;})xZWo)(hlEHz`5&ooUEWyd3u{X zZcs>X=q{U@`{PUErWvkzhmUvd7FgL@&ybGz`O!-LhkPQVfWOS_yBz7GkAZ4` z8Ai~9xv&zZ`{Vrw`~{O}X4|%+0*Mi9ieQu3MQCBSM@L~TFHuA4?Q$ZWnTyxZ9-e|i z368=RtK3A|;4j`?NBsUV3%Q{Swd?d}Zk1_hLG`c<6A@-f+XeSA--%0-vZIH3GJnE|^eLUs zZ56HMf8@9FiRy>BdFC@pqEo|e^V;9OBBjelrWBNukY2Kug z8}Rh%0cZwvW^Z=}<0uaRoiXYO`~2*x^h`Oq+0{_u4sqa$U^Ev=+WN0zbgha-dfVm9 zN^jLq2C4zCzor^nhF=S|JeQ3})@a#PL^e)174oiC55X$^9U1!K9S!5N(h0_A)}eO+ z*tV8|)rN03-EUmpecGR@>RE1LyQ$KdG`b=NVli@E5#*7QcHjUi*!kIEM&3=uTjmPO z;5-u!rOL&`iYFyy&d-`q$E!;jVKENR)OPE{nY9{Y02^l*i&3w_^ym~Yh*dOrRwkt{x^*mrgacd;i>KeX2nqJdGQv)E;CLf~ zT9HcruaBUqJ@)E$YcHAF_GcUxcYQOcU?otzoNC%6l2Uy)9tR7itm0Uhk`jbLQ>3mX ztBh8~E`m+a5a(NRxmivO1RsPBm#7=SRcGHTHdnq_0hqn{QN;vBBxsCY%JhS|$Uaok zhc`I_m+fFL)y~FvmyY@Z3?QQ92N1XycN~xF*R?km!!s`P&@M*_1A1mPu0e-%Njk!j zYwv1M?U3N>TrwYaiY%L+nUlkYSyAR>kCzGFV zPhgSj8xxq`EU{x)+X%W{Gh;Qf4)%p64GPtgGw7ZPsK+jTY&kILYMVdCm8cU}(;*3%HPYxm-UKFms0Od8@>ktiy|Muw>`y6w z!R^@Z!5LqhNMs={ZY(D8=@Z3i)`Epg;f}}GIaIC)2|gH#N!^-2FP4^}Qy-D(S}j@& zUIR;6_AB8)b#bx8 z^0wLGcyXq{8;&!s9I>+HPL{M>3r>5&<2X1QU9zF7E;USUG5?Lh>pf^1do8~df>{`b zZxO)*`GZ}Gi3!kIO?aoZoX_{S0LIzp{%a|y&wcv$pJvw_dwE^g98qow>9=hfo02Jf zE_$%HsuYC)LN8^dbRTaCi8kE3lS|2YRaJ%A)&S@8!Be*G10vrAE>+n1?C2#!=2oUo zrXGQ?>++k>0o>w-eMO+x?<1hFx|@)&dfH}>6}-hrOQ`2Dz}jD1ls_2zs7 z+%xj-Np**JMXcy~?~Y_t4a%f4Yb^V9o4$!2u_zi#GqbH!zV}jexPi=wF)589 z@gtea{D8x^mn~wEV1Mog4TvMAKFCs#0qvEFjmMSb0IH^su6u=Ya!WLWJDVu=Q5kY% zv@)NaPLGqDpLYxKXu(93XkSeanSAV()t)4?EXU&?&B>f2?Y3VX&st60XF+}k$M$f`7daPuvIQqj zIE2@P0{fwwHjoY2R-6g138a;~)-<+*9SWd}BCp^+Czw~`q92z9nz_$3_q!3-$|6?f zczkW{qfun82B9&S*IHZibSrABX&v2-tThO+CfaWIu60m z1g;+ULy(kL2IVbxL$l*4)lF|sdMvUI8Efxe?&3JerTpZYZ)iKH87`L5oJiAvwP(`Z56UfBrKJh@Q`{2eX-rzg7S;4;RuSSAbC=;l{$g%wu>o zI^+2vwDwEEh6nPo9Y9g)uCiec@LK-U!1a555ny~_`{g;HqA+6ehxX2^G647gbF`s- zZa69drx$3qTRG#Co&R{+_Wu4a&}sXL5}q@a$ISe^9Hmup{DQRf&Gm-ub-w{B(IYD7 zow#hzkCaIOnew$HpcJbj*X&RhomA!7j<`kwfSemA64WO{lUG^*W-_(K1j}R9?A@Ud zv(ZrinSATJFNl-yg*r4&%y@ZdFMRUW3_yvt)S&@nVgJ!9$tr3jvp(Xo{(pF4R+tn5 zcz6GoDz*uv4cIwW4G9asj1#HO<8K;Va!0NO#IgVIq8TlcVxDi7vk8)LkAhculc_zL z5C*d-Cn0(5v!T@36dC487qF6(T-+*-Pg|jTHX?~X$(DVB z6A`{>1#fbwWgQwI>Zd0RHtP{9mx}BR&qnFHZ++DTR0bza$;&L76WLf&nsga#lc%_^ zKG_d@t(#a;lT&0qrKEJae}u{H$}Jy1+R%jkqE!kbh((#jG`2z7Y%;XI`zwGv^!mho zT!L}t*J<+KB$TtP!`o99s6)Ge5|0#E`OJ@f5r$9oBs5l{`#3zOKVhxQ)Qoanh+?$L zRQl^ffW~jD7p*3v?G0M>vG4J=_L60g5>!md!ca6F;bpZ-5HfYQZ|838t58fe z-SaG(#F8urCp~Ky`PV=kbzNd$OD)d~bJ1Yts`fLM?@|R$5>?~kt;1gPZu-ga(>BY6 zc000S6Lr{LlQ+$;EJH9`YxxEwa*#-TGm5}8eY3q|tYOSo+1tmucm;(R_`sPLf&bLN zax@OY5TpNF2&5n(e`iBU2A=mYf*MXDS1sPu`vp(qS-Cez(QBCCt2qc`c(pXyvXVXu zzB`Cr_09pCVnpZuSr%r7I1>#zbzpQ5zO|Wa8w|m7r;%IAW&A)-aMvLq9aEsU`l8y$ zl<8@d*iNLG)%!#EdhaSyb<*}5_P03$I;y|NDeNz}F-d-?;W(v`8W$;Nr8vS87}SgdGR*p~17o#qyf-#r07@d;posH!0<~6}6MM zDzzpG)GLgxG$>sj@uOJr25o@(O2_)wGj|N>^$jh4%6qP^&yniehLbF7ic?e>+$$q1 zmkm@Ko}Uy#R?Thr1vY*NEM4(0JP$O9APExS07_{sur6TD2Cx4hPH)&9Vq_7U>yiC@ zjvHTnadiA8S5>9MPKkXoYNpA?-L>FIL@&x@T z@D*7sHXVl&Mb%eIuD=-->=x|K>>tjnH5-*%T{btLtDEyDU+CsKuTH}HVn>!7ObMpF zQ1Y8C)#dIteF$bdza$Ah2SJ ze5j!0Ar{zPx%Uu7uVCWTd0=+675AM;A#|35pyr!_T20Sd`L2*h^&iN}uK97|pl|UE zS*D27LY4R4joY1{_otSGofEuQC+ZnJNrGsy<#n zLSTfgvD{9q`HeyS;a5j0LfTXcg`7W*6P&UQlX_ z>j!qG?uNupFt^HLf9$N1gM;dPY=QHYo}pvlw?MOP&vb`JE`oeD2)fd){aH&4x|GWb z>lDJs!()p2t~<~tR=O>1?ZFj#5pnqOTaWM@p*5a-v4?5w2nUw%kun=FkIieq3p=VF zOkqe-PN7(2`dZ=M1W(G2%1r$|csZX_$#Jx{x4914vdW<&l9YCvXB(mQM5Mvx1cl(_ zJ5qV}K8$^AoL$M2wI!hx#>UEX!5VS>9mfjsnsClea!CrZl{&Y>p>aezWu{o6rxxS1 zAM~9-(WA0O$afsPxm5k;ueIlpXjKNqSZyS;cQz?awhL(G`EcS$OK@k2lehG!E{U^h zuR*2J%D@HXrX)T7+m@{w#E^cm85NPzi=wgMZ>pwGM6wT~23UES4Jqm+$4+0_?f~Vc zTCXUo6-{&kp*&z1JiA(41*Yfmeot${a;QOoG|{COT!Uc3zexQNwx01sCQCWr86G`O ziBj}`s58gkr`F|jtIDEmY}_1yi!urO^GJLA9*~l&`#6ev)+r-FOt`qLUk?qU)7iVu zvwu2rt7JtIU&O`%!CMSTCukj}Yf1YDn5LskP2k+y3x%f{pwCWbBjaX)$=fG52-o89 z_Nz+wh$GsSlha7vQa47zpmIKbwh*$ME(|lXq_Z~WcV2b&lm?Qmd*MN6ANLEsy}Cj# z=;QkT0RI5s5GK_&QtoRlm>3yZUPG>1Pj;FB;EN}Jh~=+Asr0{88+}qw!&QH>cQ>^& zo#`WH(yM0)12}Dmzt|5A_!w7O4!mSF@$Y?RJyR znd8|~UxYls_KJ1!zkiik+E0h~c1t<#wQ zX*8X?o(icc1{D|O`dl8l8&HXw7vW^iw-LY@|(7hs&uMrd86LhnOi?nkWk$DC|FCq^+o*m z!PDb;oW~CioGqU=>$q1fJ=1=&9&X#hhoom_Fa3kD)m7-OcQhou^WrYoHvnuOWT`vfH zuIg2A@b12%<4w_&a1p|pB%<$_R?u?O!W!KIr)}BWC968iWc{Rm?(J%1RAh$|;xk3L zV&`bwM?Z3J^*Vq#tQ0ntT25n^nWG|}na@Mjr9Ccnbk8^35rs^d)ON4bwdbIwlN&3& ze!FQ|pQ|$RI4!P}V$XlGkzbl?362TGu>f38cs`{+1;qx5V&TDE7l^=xoR&CUNas> zHjmPwAd+b>+v$U)_IQ!AAGTMv%lz#5z;p_6IG^-c?}|O$gdy>#1O>nIgL639&)H3U zT7CWKfqeA{tM+2w!&EN*ugkR1SzOezNR3G)sE6!_S5%CBdSYFJS6hxI$&urkSCepB z8VwDtdo`sw{Ax-hy8tKz`+6<13kBXEbs3c0jrJG%CZVqh(|iR~0GcEsNq z$?(wkV$chx`{bw3>XK|-aR6)o2e+tbVbFiJVk^FtJYW1)v#iZT=qC|`${99zY!P2q zllyN88OASIH1nNNd-(xMRz1a6_1up0>XcO)CM29c%QI1a!o@JMEeZ@hcjkMLA1tcK z3@xR!xelyt=Btb~pZC{o)HekrZE$(N&n(Op4lJ=;&oi?u&PULV413T*bETMO-)8gc z1-s4C9;+;OI1_`;Hl|?k4Rzq3InLnc9Sw?NxGR9u_vE%8q-0B)j@o5qB~O#OY;?c+ z$4d*ES4P_n0rzQz;gd|J!Ca1x5*2xFAF5|6;S(Iio!fY+Ln(eP^?!AMd1t9P;}rAX z)LZjfw$fO*(J63O|9ob$sRjtVY12)1jfL6f+g5s6{;XP%&Hnu$Ra)L2G`qh9-vocj zbH(ZK$7xrSnH2*I zy*y>u7X3LZP}))-ONOC_y=qQqZn|=;FCgAAfDJPROa1-iIVc{eUfk&xxB|6#yUaiK zEd#;Q?EhU2qQBg7XspA$B%u&5ka-0Z`pnf*8Ce)Z=hORJ16&{a8STXq%!6{ewFWVy z3Szu>nSCi_?&S>beeLyz6dauZgBf_|6X@W8`Kc+3RP`p0kC1MVVp-JJZlXWuG}oGB z)9fwu9Y2I>3S0PXw_1k{hS+%q6mH{|m@5uuOG34vosF(hb7$NHRgPps5m_>++g63J zl{^Y|_CqQhD)Ej+Cg!4$ry2%~?1u%_63%3+S(U%Cvq!#0VMLX zX=OhchWkzC9jf*tPI|=6xJN%5X&U%a_ht7N@u_2U&7MIo*ve~EQvXDW6NUSS$u&f5 z6%pVB%VQTgF&E-MS-|*eEXrnVv?iJin(>Q1$XQ;W-4NQ#PSE5G&+38 zYxo&1+h>01XQ5UGI?rSAfZAjGRL*fUE*h^XZ1A5T`DRPlkDGg?*%o;_%4M#;({mB- zeK{0=RYWX-@q$7i4$mH(tixay=SQ>zlDFP(B2y&v1up(78gXmdPSex#V(R3Y_cM4S zTh1gPk#Oi$7bHfoJ&1$1jg`lSd88{z+C?#jd=z^fB|=IZ!Ghe?D=A;IB`7sM<>&iM z6JL>SXiCoD4D@UF{kSB|H;6*51h+eUzi0-!_iTObN!m$ zXCTw*#VhLC$yU5rwsS>mLa)6IsX0ZGT~QO0waBBB=a?32K}r-AxgV@u&{6YPU}JY} z8(YVTa!ys*@X**mpVGWRLrfYG0=4>ySIj{w^kvz6J(c1c7B%(*0&IkgyWcax_BF-7 z?gGyn^;9xQqi#nFzG+@uvz%~<4uyl(eZ4n zR%&4(UlbWUvZex{Mu;vi1A?T+>%EKRkzzS+(6KAQ&AP|Sb0+l=U_in*e%IF9Rijn! zCZoWO0#3wo-?I24@26*M{{XCEWH9CN?~A8${NHE0F)TQ{K0BvWyrIaauOG6=^!HqN z1$s!Zm;3|fLN@7VUM6Yxbzox)q$@T&$;Dhoa{giegA^tvH^>#v$ns#~x4%0IUF|8+$8vR+#zh#9yVayC-GtKgaxO$6TOpt>F_R|N6zc8!bQ0 zVv4%0|9Qn z*T&x7Z%VvvVIsZ%mi?)h7~Pp&95-N*Vu;_@-=g1*kR%)Vo@bKp_SgFlc>j57q$Tro zSm>uxo!+_`pKv7RfB0OlQ>+rT^S@>oF29b&#{~c^t>X*Ma=!8a*^ZEl|FNI}P_@Xt zS)2c6NHJ)&)9!5iMzJ<`F<;afGd&fe0 zte6VK$APEeZRoH0wL(n&D1g*Spg^C<^}S|1HrTP_$5MFu4JP}jlwyaQ>gLpY_Bn(qto%)CMp#H zO#YhuZ;5xz#M`jwvwi=#7!O{hbFh5jcRKBps&V5%{~{B zVZP%qeIM^G;NgiA``j>J;M0p;#j6-D$fqWAo;Ul4j@4D?F(Rp>F9msM5=Gx){`EqW zJpo>-kg43<(wa4U_UO6FV8amw!;-QtD!)#uCApc{nwy*PHL>{xp?c)Q?T*w$?bX3e zD!NOa{(RKhjo+;j=+(ug3y#Gz%rH+?(0^8_gJuQhMs@C=ygJm{kp3InR(xP6T0&Jf zPCJ04fAKlRaI;wKnxgNNJxNNxH1#z*MYU#)Kj8Mw^_RCsFv^P2h8{$nqIMaB;ptKp zTd&NY!4*|Y9Qm{m>#@2x3u zyLL-v?bI`ZCWg*dV(iD8KoX<)O-rl$gZ2d8aA;jX_SDC^Kf8FAMV*bxRtE*+o5OwSf z`7o;XO?rKAy>oqeV*PC;Pp=|R^3%tf%q*%!8rQtf1Kv!XPiK8r6Y`O*9g8QEnF7z@ z2J~{>=M<*me?r&EWp?&X3+s#&WvMGkD?c0j zeGk6%=6{pWT6lCZou>kLcp!K8sFv!FFMd;#)K(qKX?qYx8;=V6wl9Ngs}ATQ6YAw5 zaD3pSCD|$6jkTc@hfAjukR6;{hgw)7QL~4YIDe!Vg&~bk5><9~C~q#ZJ%NT^+)!(v z05nMJ#TEq>-akN?U4zVYg_L+&kKh zog|~^do$lU4U~5wXIPtc^LiY9u?kC=EbB&VC28=g;k%X=GGlX?GT4Lk-AwV-%+y`g z9Qz=Vue?+6iI`jH{7QEqBX!jT5+kUFB78++Puc{>eKb(22&*L#<(I50O`(daQ8jJs zo}XT4Rou)b>?5hfQ_ydU5>$6-yU?dgrn2yJ!gI499v5SdMx!nlINH_6umnF>btbqIVlwDD~amMQjmWd6dlu`rf&O_*Ku`+T38IO zR?vx7h<})SxBM(qf4Nk8Q`s#$c|?)GT`egr%BbOtHEF7$Hk!(eI}f+%x_|0ZpkAak z;e?~T%fx>AtXjNK{#b7`ys4T?{YSE6ODM6ha zgm7g1^FG**7vqK{!r3mLx9n5`vMCYPd4!;$5$_{8fkaq)RyQm){o%#@gbs zN!`f52hYhiTMrTZ$R!CC(n_gD2=5vVC0Ym0g~l{Py32ODAE5i&DOYB*;{HL2f++40 zM~>+_*oRZnM@+Z!8*XM}I;;+=ITz!19>_6EI*YX(DWrKzEnGk5LPrLS>NKRxZzndn zzIHTHODJUeApEoC^8Cy=8TV^0nRo}&Gi-0cBwL`Ia^1#1fSRASmGeJ9rbeni0}VL* zYN+CJNAc)-(}{NpV*Ki1fj@H55?zp&iW0|ie_P*HCp23=+^Spv`@#K~GGw`Y8K-iq zQxmiK#ukZfzAR&ZZC9pM4XXNDre_1L29MLc)URp`=T)fs8TX4v>{f%XPpM+}FJT#^ zO0@iv&d0b~xo|glO>CDoTj@iVvxe#+BdC9>Qcqb{@CMSHzi`OFcD3U)jd*fgInID{{I zFFn&A9-m@by#Hl9jQW;G+dT6h;BjoGx_BwA5T9eeZGD5lDQ&rp{{3=TfQ;!Li^{AX zRnxZn_c4<&*nkl)cY8*1NadkBV=b7@LbSpZM~o7|5KUYbFN!%zu|B5XZah{%eROnLY4nZwd!+9lHF@b{xj zex&%_N-s^u@xW|!)GSdj4G;uu*Rn0)P6FM)@UUI5DZ<6CkuRV%B`#jqYTY5xudafe z;CD8|=rL^$LP8}1@nT`eFh#nPE@SA{A5Z;tOHnC+Y7v8_v6QstSxMp z$knsP%Yk2}21-FmG5nqxAJm<46QDr>=O5)RFpMNxuVsuck0$0fHSTEt-vl;W!EU=XMAXH||$7E5JQKyb(61l8bTB!`wX@?}C^jj&)!S9pc zrtIPk5$A9&+Vi0jIT^kMI`js#TBoHcmm?$0llN`xS^K1(h%C;qpcI(jKDFc#^}1a_ z9<1EC+v{_d@y)Rd+Kl*qaflZkQsr6dZ?Tk$&!Sc&6U#( zQg@6f4#A&UU&X#d*DMJ@)FcR|WX`OJPkfxNU{ybl?EbPPe=`8rJifqeEaL6(Hax7tl) zbR1O91w^oWzgp)w%GV;IuUNRvZWEzV9&_u7JB92#GTcE9Sl)zl%39|8el)fAkP^|8 zwjm_LhbM+;N?;DZo`0g|(M@@EIAJp1I0*+cJCwa;J=GM{WhH5XE=y+R$ka!r9&tl& zBO$0|g*PlMm4-=Fs^_X*{udtuYU<;4n|JGn=(HHAZIRFCZ)lZ!>y&G~-Er@7(bg03 z+qia_<8~wpU$DnD6pFQ)?$^p~|gs$TIk*UI`ab?Ttm2~Xf==vcLfYOrA z9&@X{$CB{pM1@Dp#N(>)QU;ysL-OuCfA0OpTmOJHyC?N9dzI3e6>C~Ew-)SoPq{#} zNw5HRa^JE$&p>G#2%L+aFtmNenmSKT$S=V4@t4a>q%^pfwc#`W=M(M(;~L5Sm~Xsu zm0!%vFm!24SILL87Oj&P>kzNVg-7fYM*G*&lSZxk@HrBle6sr=44t_c$UBqLSBXoa| zH(ns#*!+079A%Lu2@5LV*x&|~+$O%i7vAFbVV;bVHKOI)2yuQNa;JQk_WaP$5M#Xm733iy4!=9{i(hqJ~_kIhW`d^W0x1R?=6M~qSASs+e=zk8s|KF?&Y75mA9H|tw z!LVEZ4CMU!%QX>V@s4dduWCa;L=~vAUm!UTUcQV*iczP+nWi48qVh|LMGZcn@$IvE zW)HJDVXvThy&i>$v9;lpg}!)h0lBgF`yBnKQPuEYNl+8p&zVt2|G>%3 zm%WBIQ5MqG=EXS z7yZolvTZE$nxuW^k1p6q!n*|>WSqzHw1c)Nu7gF5T!JiwTQq&`5Q1%fa-5LdPK3>3 zF$e+M(31@`+1DMfA%v=j^&dp8Nh4=aa&8!66OC62awWLPGN^EP`u}Rna%JVb+vG7C zt@8_8>zBHAv9w`eW=Iwars+=pfCg|W`J+w&QHjO(5wIE`ib>^>zH{}6vA%aFvSD_j z3IYyMofI_l*UU*AqJ0fUGf!f=8=)+n=~Zc<+I`m3?&wtM)jx^x2z`awC&E7 z3IIV7jR*nIN@R3UJX8)Ghi%JMau%TgOvim+A!w7l*ABKc2dSKw(h9_CdhApy_e_)h z-l7MG)lzN;aBU^H5v(5Qz1gy>ka3OSh$1qw_(wD6wnJ3hEQiQ$J0<96GUanIsbOUj zFi+>hyB6BTr*Iht8U_IMdsUS?W=4Qob!WL3ZXqda2PmjoF--P1r6X8RgjT zZdCACv0S({a=^u=X(TnXH9_3ow_CBOm>!>W+3(5=EKeUAu?xQ{>jcd1S?SL=3ao&p zyMnw%R3upVaDvYC#jiN8zNW5bL0>KmR&L)6x#~~!MFc&BP=u#qLw}N?TZfWhAqR8A z#(D}|cnY&wD85s8etbD-UU#99KOb|}r7FO1cjBZ-F96Fz$W3I-c`>o31TrePw&~<5 zT9#EByMi7H1HSHu+F4x>5%xC-l+|s@zerog0iL0~pMEpA$Dq@gZ{VEDF6U_?McNtv zu=TjaT55XT&(>K`atSWEh}kKaRcsw>9B%xR${C%45lEpi3vi)S<7VD;(Yj2uIhUbb z?4mr%B2%o(wDPy!AEbE(xJ@{0tNw17YB1+pK&n72cQQUY+I;A;Wpcl?^HAY)5M@`$ zT;j+L^;i`wqA>{E7koZ<<-O^6?IGQjpq^;7H6JnTcZ&iFy4JktUQJvLE^hr_z)2tr z`ER!+?T>-eljQn2jE9gN`?xkblwHmVQuY2|z&0<*yqV$EF8Coy-)pnM*k-4fXNyj?l#Rhz374(aK<$!goB=1ThzKnl$BNHup|z$wdbZnG_ws ze)G3vY5=(|3oRS=d?=TELTZy6`k*>N!(|V4)Q5c-X|ubVpAT=82Y(XFqR3WKbJIN<%9{cU;;{hFk)fN6^b?TEwhmxTAK! zU6}ngFE^*jgim?`+eI?MiSdpJ)MSrw*MWm%t+f4p}TLWx>}|#$b#}8iHU%6{SNID2A&(yD2cy3hB7$_gAHn z70;XX7?g6BiMk@CY5;{+PPLBCcD77EKs(&!(fYl*V2`3LEce^AgKOD1@7xMa?P-NrR}rdP=mmqCo-QHuZIL*XY1$&P{rZa#e*zoJQgu2P!Uv7cvf#(h~B= zrV(!8sH&7)DW?uPT0TWq4NoyIbeK{bhPY4%QlN3#h(@pJ47cQPzDZ(u_xL1=nHeY| z7F3b%K}FD2d-he|X=<>$G^S%8AN4_70kMTvZ#>(Kfj7@|{n0x?(*t?ptyyA)G<-Awa((j@8T0riRY$u!gh1 zQ%OGd$IVgA_l_W3AE@$lZ(_YV{{0?Bw1CO96`hK5TznDIcQIso;BV#l?_A3lR%;sF zA5oT<)iTOip_{yCQHKI3np~5n$n@p!Hyigv{`UDTe72Lq|HDnS^A8-Xgv-D~vaV4O zsne}@254HW{j#1IZNMO+Z7?A4NO+dhJ)e8mleG6npyUg4HPb)$hb~++cGVR&#&;Z} zE7AUek<_P1;AD~XvbBc5zLZ|-+swj#@#SmCgt42YqBqrEio@~x6#I1b$QUfc`D#|` ze|SN~#I}y3EYg+tPHaPCckfe1A?X?161?2#u$pi7O#PnoA|17Q9rdWRgMv+}QNJfC zVM1ZEO3%y(+5t@2{Ph(McTayvwe6)z;@<|@Kp)sk_kGkHRh#kg)vQqrz$kr$fKo$C zh`Yc7;Bg;J5I-+W0c3tYI5~}=(*)?m6+}-s<9z~=5fqTo&m3TqeYwwX7f!5bx3i(2 zjLGDoMz{8G0JyZ_KFOPJrrl$;wPoY9#}z16n*+KOZZ`ObiQ3wNq_{1~-6ynPc)|VB zX|C<$Np04W2b<+wlog7FD;wJd`5lS-&f4i6={TPAN|n^Rg-JN(ltH4>=J<+tMcT{} zsqbXwMX(thlO~nhR|m)6@xc860V5XOeY4G}2~vMpzXO@4PZ! zIIBEmGp1wz&fk5vF}?;Q1i-D6U?3<2H{TB|;HTAFVJL3yUJ+}ah$PjFMj*kK*Bo^Q z+#4&K#)x+-7_iFwJyFW4gB^cP{V1^n1n44Fb^&DUWywWb3sDnZ=$(l#@B4sq&)}?= zo}kP9*V2>(Jjzl9!4M9!kg$AU`Vd%v=(JH`obGcVw!Qy5FdHhG*U9I*0pw} zDPadVfRm?8F0*r}^qkhdU1vFtd?DkPDO2%p+Ed5MR%!gJmS66BvpLv%zs3pDX<_zl zc7ckHw%w9g*GYI4fFT9nQa`j5*YO&>p5><}nLd?sud)c$h#nUdX8K|ZePniIVqL## z#oynWBAowMCp4&&{&Kk19%WGSSLJ4Ru~`sn6BdwOM2x5(OcoI5y_(jKdxP=Cm3Uk4 z5#yVya~APy_u#ehbEa_c6-;oFNwnakMh+jhhF^!$JJG>_m4@p!)j#sqYA(>`e&h5K zUg}&$l>XJfZd5iG9j&Ch8z2tFF(Ilt;Xpe#hz42a_ix{LfV!7#YVA7Ro{s#yFOPp) z0%mkub~()Ec$anAn0*2pidDu=-}QANA=AFD-&*-b*P0eAY6@Knp0h{%fZ~idJ`8ef zgdmx)rp;Hw=hp+`L#2LN0LWp+?TOfrczc( zEthmf5c%bS2)c5req@Zr+-`XM`VZNhS>Z#K^i=iybq!gXuaeT>aPf~+^z@aK{0!8} zZuG=zcG%Q8mFD3Nu`zyeWMXo1EF=PcK{O~An83(N2z4Ga*Ntu5jgCeN3ek?T2Hl>w zt3-n{FBp<}GTv@;$;7_MJzDUE;vh+jI&r|S27og7xE1G0{rRh<^_;_nD9iCxTFGje z`na)SY&s9dWS}Ja z3*-JLE0P3j%uoUm6A$TXP!;SM(-)Tf?=E;qm0CGvu})mkyc{++$L0${;&lmGtRoBF z)*SA9*=dQ}&Kl);&JL}$0s~F8!^%dPuKP-i0?w_EaONp?x=Sys4?t%PLa>!h1EZ#7 zx8|n}6=T}RwA(e9ktM3Cew%ImTwv>SU>u_;`nQ6WV?$i<2rDu@Hr;^)X@?Lh* ze0_*lFzAp@x|3nqsAP_~BFy;8EuSi6eJLzq`g4Ll>Zb_^B0$v!gweG-qP}oCEdR3zz`Lp`CeP! zUDmxau(XjGR`N($)(p!(gNO3d*pznIoZ&#bXmygpp+~6{N9@M1CHLi1R6W;_npb4v zHb<<4fTm~(e_J+D$-uG=$kY*iO((-?4-~f+u=iw&xJTh8VGI;V$ugmT)Z4SX?wU$l zmw1<~Eq3))g;gm!kA((G<;W{N|@Bly?E24NuuUD>05g{?KiK%XEDIz+W6dt0{0 zk79%9Au3?pHEIsiM>a8y`F$Ut)-n!<#9(|VjUo+5uDq3+ zJ~I(mhwy-_pf@L1r&O8t2BKW6%Vg{$>?=zx!V4evb zs#t;bt}$?%??9)Nm{Nq-6ll|EDM-z{MxU%@0fGnU%=3tA19E%-YTRtg7QQ&Xn^3Qh7WR@R?JX(cckI|6gZ z)#~Ek7Sy}{`(tcywB5NB-bnwIPDwqJ-hEc}Eq;CN%u_1NR$2az3)Qn3sy^$4=O*vm zy*ZrGB7yU+^eap7m+%PhcJB&$bc_2=CVCF*gayLm0V-$+LlD0Hk1F;5rc?;H)~pZx z|6!BJi6npW-Ro)DINiW`zj-}hokWCDm6IKnU15D^DD2WZPfw!5uKzpvtfERfPLv`c zSu<|19%qXR+85m!$=FpBp)M1okC;`%jO}PkuuX>dJo@Yj6G`buU34|rf3|)WOOeYf zp=JKQCi|jtilj-LotWIajwG^t`U znnh`9BxJr3sm6NE=%7jC6;_lCtUwV00)B{(c4c(n1RbBpPlO7(jPgu6ve%l<>Tp<| z--O{-3(C(Y(51)!Dv3VZ*%N^~ns8>zx8^&b8jft!asn@);}RDM znMGLfV1()8Rf7ouDg*MnE1xH@Z?VjbYhTDl_vd1496k1~vNyh;xy-o;PMK|Na?S4v zqFW~XRLL^|YzJV5QT<@!yidO(UKZ&vq**xVk&(2LD-p7} zk=-KEIq9rbb;g9mvq!J(utexcHNmr*a1GxY)hdFB*>dzNFT5cmnl_W!{uELMr3fW& z1*>2G5og;~W}NERbgeQ~UhdH#)i2=afr zmVEtcc)0E|G`p+tw*Tw&v*zHmxUAUE*+y-r$@Xm?UTJ+=xhCxFzxCI3Z&ati=_YF0 zZAukTOP%tWB02mQOo%nc@_OX6G4a~Y5e|_||G-AV4l{HzFG`iK2^D>FX!33N#pY?3 zX?0<3Wd&mJeQA{wl;9Xd$6|A2Vw%yrqp=~Kk5~o@>s()J3@0F2HagiTO;nKCXr8ue zs3+YM_;a+ivQ{NwT8gK9`ceGKKF&=<%*N}}NmR`O7+P=OFouD3#v}rI4rls^QqTpL z%Xx|%inQ*r&GBAWJ!o%lgWo3RhJcHpu>{MO)#_r3hs4@#cR$BWK9+Qap`vOBs8m z`lVqpSvZzu8XYJE)`xZG-PWuUn4})}?8IEZ*hQSxG(8cv8M)c85?zxR&kBS|4)4d+ z9d&=i4N86rW8xc&Wd$Wrb107{t)<%Zvv9f20CYd_noOwcTJp&eX>T%$sHrwuWWTi) z82;|}44=#Xz>PQU4+YvOtQrg+#=ig3oY&=UBkRLiF!<_Ivy6o#z847xt$*>%N7y_B~C09tanx;M~z-@a?FIs zu_s_l7f{Dueu}yqA7+A_dThM%ZETV>iPmA>u+5Eo?*RGh>lzR5K(Re9&kTlI;18 zC>~b{fgZ$@YC>m?GCl(n>skRSgnI@{al@6AEVH&juG|7llV=z;~_t2J^jR%=xiETWxdmhwKr9iM)Y8hJGW-}zsxm3UTy;)rDD80wR)k!`wWo*NvY~W8AdA23MYU*7RE3Ho{QMvfv|~mz%o(%7GL^!)Tne} zJhn~&y-p?G@)W>}tm%W6XH*R+(8D#{2%i*)C0~is%LEU?P=MNye8x`B>uqGVMEtlS zj9(Lvmsxpr@mcuAQ`9E`SpR0Bi&6G;)~*E`n6TD#z5`bU_AQ)OtiZuxmb0(VDz4#i z!S61!|5s(dQ%brrRitX1B2UTsyW@0gfAmVg6LW7mp3?036hH zJ!^&6nBHx>iUS-g{@iQZcAKRp#OfE6;{)f57eEfb?}nX>UD(mX5>T@)sz&huSIVhq z8)T9B&;%pVW3LgesxZ9JRMg~bF$RmyJpKbWM1%dRy7VlQcd%7Q#*@d(ytr9-koA^s z`gP)?DFujYoPr|?(09JjI^MAnp}qwq`e z8yzAY_dW(sgpUeIR}YA;EIlFnXK`vE-iynuipytzM7_Q)o;=Lst$ey*J;K~pNp8(wEt9P5qKzjdDXXh$Ys%2%Xze%y zEVrTrKr=(bbI%nhfU9YQS0Y^%JqY4ApNxN{XpeYRn zkfuI!CMWDT6~V>96~J(^blxmwG2*dmuoC(DE&0gA?SmvWc2^$|9VxJL^_gUKZMBb} z!Ltl{d6F|`H_t6P+s1U0Nr&+W-1g<}Bef-zqi7g)id9Xw=Mp>ptc+Od@L||W!XuWw zX)6{JL)N~YjFZM|%vMa^j7p$k>@gs1a%7PIu;so=&V7Y}dU3=$Ra@x~N@VyyaDX$! z?EU-v*J_>E3E{T36A90baD90@3!QO&Ndk^+`j6=-5}wP6S}~W|mf$%z5^zdT$S%cf z(}}D)1@1Ekk96*-X6|mbkW63ay7qlD@&>wE7xPmU&nf5N^@&@iwi))4@Pd-1-3~n`bZObp_xmb;vai&&NBpLs1I@j!)IAs~WXCv;3 z%R{M~;DgJvnAqGwtDR1)0nMwocBCLA*rLMA0>?|~w5u`Q++&WCV%1AS?|` zwJ~@9?x=pnPrw$88TR>eKw{?-5aH>*S@tGU(ior&3ylw(3)ZNjJSb+SXP7vQa%?Ea z$>oLm%+#i(qI_|@6kkPtRW-CK1?@_Y#mqi`z7+d%eP(;u`!+Jjozbzz9N#Og-oY1n zY8}n01Wcl|zu@@un_w1m&K9+!YkTeWVL$_hh9DlT6fWKrp__5|o#}CDH+U3hGoa@1|)fVPMAuU~F@ zOdi)_04l?lFaEw6ZtGfDHs7(M+CRfV!tLsdidWA~9~ zjfj?MPGjyUaf>9BeH5fpK<@93_qyfRYNgiD6a1ROARZk7tZiD_>WGeRYN7L~CFrMN z`a|wE-uy6bc;j=ekPBe0{JrRXYcZFTrxpk%1-8QLz=6UnPIYjT7=RcaBFyn*#5kXr zhsaHVs*o)ecYTdC9o3pWjvbF-AO3`@qJ>17KJD!I-r1m^=8$yY4&ldn>$x;CjHF!? zhMJbDQy-ebVRtwLjD$563;ih+quseVd}_MFP3xMzO@tadOo5L<)NcJdkR1l1ly)P5 zm%wbvI%!KYhjb)&eOg%qgZ z@OgdeyrwL0(l&qwq#oP<%kUq#ac>`>xvDzTz2)HShA$gomDj1uz~=rbH4bXCO=Zms zX86%}mh}22x;}rWpx{Jx4Ncy?s<*u&7D!8%vDcPzXTObA0#C|J-2C3XQ7*x&T zLfD(FGCryLv$6;eIwWR;a=0V6jC+pOOdSK4i_T?tswwSAs5_H{VBZsj;qZulA5XcPBb=qqt~r2q=CBqoe8IyA)czykc@|Y|^~O zmQXa^yq}RrR;_<)hPr6cyCY2NdXJQ+%F+tR3TXzCxthcgrGi^Rb?XRYra4T1ew;2w%xfqK1H9XK7Jp#4yhH%b6W@?Id|M&q6Yb2WDNN#zz_= z{QSGijSJEGRM}EtuCbD``E*lG44}>yCPfXZVtx{$QQ_GlLX&DxNfHc=;>kUcNv*j> z>TTpJCo_~Y=h$%zC}pk7qO>*v)35DtWKoOXPP|E}>v$A(yU-AzHO}IEbe*r%R0yv@ zrIJ-9pi^XB<)LX9FCfY)V5A~E6J&0a6i-pw_p~^o^!7G=Zycov4t^YDtD9rbY|~CA zWnU{(6Yplg0KuvlELzLVh9E|5cpmW$)iRJc5UdMQW=WXhQAa;)aXT*v%H- zByMR<8kKaHMrMp)OUFeYR^0G@=sA-Tr8#(Z9tKaRj~NasBXlz-{|x4{{W?2~rqbA> z9>-d#)i*aEpvB2+A|Z55nIlk{YHso;x8^+92FUwj`WAj{{XWUeM469>*eyLdj*1c{ zQxO$RKN0{`3aBmR&JR0mFoMtVWvRYW*eBcN56ce{`WjbyG{ z7KpAQg+c{#q5t=dY@l_xweZsCux;FE@@J{eX8RvTMVK24>au)E6hU%39DMP;C^BT( zo6G}0M5{E>^?Ri+e(VWG%g>`)3PY!8K*-6W({) zjJe9TDY$MR8}`3fo&?;9#@|U;GzQCvV5YqPf~J2zBCytW!@=Z_KUi$8B}-N>E;zhYCLn-Gd%cD}Db%ULPF2>!zg1;JfZaKZNA^j^33`eFj zd}u<@9y#6H3WJ(A+uQj?z`*NpTKM=v@V-lS3pMEf;-1hOF`JL>jl=)DNy_n;$Nt1a zTK-WP=HJlW?s1I?xqbcc&<*;x2`7e|SR?BHcNz5v)_TLkd=qg;_xJl>dk&rgU0Lal zF!|pAbl~3(USLpy!X$u(+nBFkx~+Emn?y~)Y|#?H9)f!)H&@1y8A(*G$1tQA*GP89 zu?+BdL}h@$_~3C#CA%)T`P7uIq^LVlpuY6%CtXH%wGR3ViJ2?sf-`Zsg2d7eEtSPf zNi+4FZwiv<=TaWKW}rJGTmc!-A8(w~?!d3be6P)S-m0`Lmz~#W=-*nM^eND4e`kP6 zkeTxYoI*yJ(%B_#!qZ3TtX+m1jqRUyY#4-^M`Qf4XSwEPXlQVl31cLsw1(@ks!sf| zvrMEQX|JJPRtpqMZByxtT>XV?Z&Hsa3HaO$KNtk2Ljq=T^W&ftG)AKWy@6FaLJZtx zZpKW8!Rzl%rNyQvO7wE`*Owm0$@C3vGOn^}TE69scy<*ViulQxg>Z2eNx6sE%rA^5{H ziBg@fE9ZA>^6C)wEFQ(>bx~pRIbo#oyCaUN4^ldYRjN6NFdlO1M^J>GMI>A(3B7}0 zsVo*`He0}O;^am#ZDxQIwK(^EN|U7mpJ}AJb4oc&6z(W6!MGqqwO* z6}PaVrVwSnm7Gu2!a)JI&jCylOM?Wr-cnaOH>#F4Qc<%Hov>UeD3|uFyU3}FH9wG^ zJ_8E;qV06sauCFLF4em`y#se!E&6z%gBJZlRyFcB6Bd$P=o6CIf77YZfUrb`*GiaI z5=d2C+#g0%x#rvZ#-hwkP&TAWnep_qgoUO|+%fr2NeoxL2e5SOM^?8!De{~~hkk(q z5MB|FoT8k702-38f_~F)iP{^L36$|G?maYnwsu~6TAsJQa(@YK2audUF#&(cUv$!P zC;tnl_eRq;b5cUEv&V`b2p0}}7sCR#mBX$L_|-aZv~ZH`Jm0j1#rY@dI;AYAVVTYf zbEG2-VO?j5_fEJnYOWku3P zYkjl(-}m?<;6x9Z^bN#i4rXkr-pn}ny@Q+0p?M5{pO71~3F&HvC(ki>o+&)l1D03* zg$J|n@K7cD1bED{7Yut~$BRdf5?{qCgY2%fo)Oc=e9=VumtPS#M`>0<>8FX@X4|jk zx!KRRu#q}ZAuCG@OUQVh2zjcKyPJ`$@#LhjEh6;+Mlw`p_``t4`s&fR&Te;(gZqV> z=pw!y@9mpP_DPklR2ZVI&02QzYQHPruaCxt)*{!n!mPRZPN?W$&D}30us1MoDfS-F z2WRR!vY1ek$KlFK(LR?#CY^=H^br|fvq@fz!_umOGX9v3zQvpvkbDc3w(&p9iZ~XCCz9sDRRWO` zFt3!-E|`Dfn0+6WP_8HywQD;C;^3#b!8$;g2{}?;MJ`eUO8YH$1)Y9e3S&wB2aeDe zt!FP^!a~*Ij%p|h-gRcr+S*ekC~Dh_xi2`)`uQ~8qGAVcu8F7nI=F^cS3ZWl&U|g2 z;vH2fJP%1hFB&Yb%cI`#Oh^^t3c+rV!C#Fz5267R%T+E~W~{xhii#>;Y2pZ9RUv#d zyC_x;MNO(uauS;^w3+**W`7I$h!V`a-?FO5JOV*8EG*64{fsY& zo^%ul0Pjo2>c*_*huf;&uYQdPi?L^UPL-TgshKq8WTtPMA3N3y_MGAr|DH_5QkxMK zHtV96sOpwR`hG+*)hI$=7DJuTlAEMXvuvzfr2l3Wjd+s+^QG=SG}9&;prco7Iq4PX zBzc@PYK3Fmm1@kfAQCk{YF0NrkttipnQ9uOA0*{6-4HfpYH}J}TZ&XTy!U&sJ}*m6 z29=0tz@9HX5z)oMB_}88cST~#*?VLPg%qxq6lVro0AoU+m}6(ac)btLf12?nLF{B?{&bMOI37PA~$Ll$saF? z49C=7at(sCoxs5GH5}>U$MT0}CU++MK7eDBqB2gAc%lU=bt8Idql*NX zUd)#gr{t^sS%eB4Hh*$vARK5)EL9UsVDd!8! z#)Tu%?jE*T$vS@zCoZ<0OmIY)lbep>tzg?Xbw9dZHmQt@lXX%(yDEQ88Rq8lp8NyX z%#(cQFH04*k|=fY`tz{N66>L*MMc5?WsP`K**REI%vOuwX@O@f=ve~;PVND z0b%aw>TXDL)$|;Xz8lYX1XA~5EF&|Bb%$M`ZRSfo%cr`GGg5Eu%0slif3Y{vAe`Pe~7YY$oIEn&>ZCozi{(x3X>Ug_5WCEPODsv4Ds-Dl)8UMhk|7AEH-18z#E?70w(yH+VNbA5y z%uAxoiR6d$YR4d%@FYWL@KVrc&_A@P<|t>PgHfh^Pm8m+Ui)^pY>q1i7D~OYGFUTa zy8!pRncMbFx_0-W)hcsT1wK+Hh4hR}N-Cs!TuB{IxO%VA%}v~YM4>9^e`I4;<%+ghp*ex&VN1OnDR)ydJYUUs&8HD*#XY5i>511)ii_5elpM zYVe?pL`v7)W_ME6&oM`<)xUYT6eliOU$*-q;%^#%1$sXG1@o*vkUj`rG?O?OMvSxk z17~loCWLFAGpdogs>Jb&ILmz#HMs@65hgfYa3^(j@WfWwTYWZ*3F!# z2QmWCXNzMyxqsji(sjaC#M(L}bgNXSh3oF4-&kZ=h%sqEyvqLAw66(7VzhnHkGHM7 z1SC7!Z)2i6`Y(gFS?ZQw|BeU~T~mpi*bq@<&!Unfd?nqo6K>Dkx{6HF@e%@BPEeO* z75@XbY6NX8A;B>Cu|ySsITVldCbs!lM!zJ45u0{wUI#|6aHl5U|IKRe6@SCi?$g$py*uT;p{sM`Z+v7gMokh2w>6x38`=Nw2u8<-5_^xUXpA}u8 zmj92^xCvP&%jPa&k@XJorwDXJ2}_=No@IE4W9#x9q1Ox5*&x>Fw&0yo8W!a7cOT@c zULumJkCI*zFrUd8gN+{dkt`G*X?tIG>(sS&K5*uiRGemMQp!Mw1O>ed z4C2A5N&ZqQMFmpc)mY6k=ntlmm!B(5UBHIeSsi&MLrq++nqDX>w7cjYLc3dJnj|b7 zFOWwAORp>b=rN$nbb)N3l+rZWZ})8^u`KBofG==#9DP8$fuc8M!ysyIJo)MC7r|Xw@V-b< zfVj5@&S??%9c?*vTj-};duOu&>c&k;NiVw%i~(yjSle^7^)bmu0j5xHq@@FErDkJr)WIC#z3(_jvOk^7 zXUvD3HKr^<2cW;nJN=E&SrXtnn&(PrNHzrg35QiKuj6w&mz;TWE_<`yi9BSdFlDFv zBuaC~;|1x*F+Sk#34Z4c)oD9P_T*18DLrwBXa2ZI)ljSpK|>W!(04A$Md$F{JoQHg zNm!tJ9A@hG?VY{jF|$mx(4eGB@86eH4kvAsLBl31GLcAs5q_`&nKm*)g;LC7j0)Vn zy|}V)Bvm*KP0a(t)j)~>&Zq=Auqe;B>CS09mO_nFb?w`1DYmxV#7ae;(#kxYMAuYP z_K#&_5}>+uVxJu#nxy;d^+mfzbVd{8abkE|%K#xO!ZE4+-MS)6gDHlzz)XV|j-hlb zk>mj$eOLq?0wp)d90O`U?fKUsTQ?=en8wPwjU*Ja?yFBP5S02_iWwty9+#Ai14oCO zRv(auY8hp}Rvxx^A~QHOS`yf+$99p!^ZsIXyyOnIbV<#iWJ=d|GEUX2Wj;kq z_V*h8Q%o}u-9gNNAs?)Ap7e{q+oXi}11DY;(n=Q%qtKYPq@9LF0Y45EHSH)Gx}e?i za)52-a;CMEA3@Vo(*z^^&Q$5yns@Uc%X7ng!{ewFXDrr9bneLqRp`EDS}e2T@Xsk1 z1SkeLsYp#stW?5!b5>g%oT{84Uvh$33ece{&7ilk;z^Owqr!0Q`|-jp-!$CpT~sh3 zkHHDzBT(~51s0HD9`EX_UGm*?lNzhd%N(nr%>dY`CW0wSRcPu$cSj&!ZG^^3(Iuiw ztPAN$3eSN}-Pj&x3uRIy=X_%4-NQqpu~-uuBn__aX4bkj+%!7@VdXSZxY>m=2ps;| zp;5HHuJqMA!W;VxF(#R@DpPl_>|g$+cB8<-hS2_ZFp@17SgZW~QO~gKA2=)@2|NHD z7TqDme=5QOZrTbu!f+Z8=TH#;CSh0!A?m`3#=_v_u){|nX}sTmkSHMU`gh8lBr8RH zecR#hglPYLiH4Z)ZVK+d=r20i!Iwp3&l_1Jpi#!dX;}>S!#jBDQfe)Sn43EUvHfjo zsCrNVw$L`-TrA=Zg3X`T*U%~ialz+EpRn6|-m3#Y?Mxz2^8^f^K9!7_Z9>w$y~HguN>ir%v7be)U5&xFuB+w-es{ zVIW4@rdr!Pc7JrcMZU!JAv6tLEi+DLX>N)9zh^d5Dev=NUm@vA+ayDh$~v7Ic(Jdv z{i|E4X}7P;VZGIZGG_S~S+($5K!uU5#dV`E)FcT)l~3Ek+{AfiCGc<;b_Tai!jzmf z3IJJrQsZA8NY@di^ted4E-^e<*sTP#o5e4?5Wpb!Xc6@r?2(GMkoGviMS& z;QYQ;*|xY9PJK9f8c+z30>seb5l>EM7{7tk-;RZ%XG3OpXY^q85ZO1W4VDi}qbgS^ zG*RK?Jo}*f2Ik6aQ+yKA*6>Y_DO|`N#|u~0$lq}oLUS-X-F!fn0MbswZCR;>c@#bU+i&m(x>ZaZnRYJbw_!3T!@6YkF4rmxS@V} z7yb(y7oA$#k^ywOCV!oy5JG1%x)pG$c?l*12dF#xxgl$4w}5S3HK2uCI?)lyX+{+* zYTCmWYq5qqXy?Q|l}iBnddq+_tOJ{)G8$*=pyHLG#hl24&Bg<|*^+2ceg{2zRZ}z_ z*tSKIUk3oqfrYQJAwMn}q)dt9g8`)889)^o!y**WJFlr=bcd7 zQ?=o=@!?mPh>CVtKDdng@KlUKjycGJ^t3I(x^UU(7!(k6#uf-rr_rFBg=ejhenW?` zuiac@frNu{!Q6b-HM#GijW*-w&B{RJ0rA2@;L)zWJMIzLQxpLLRc#i7u8CGsKw)9A z;?LZ$#KK5qoLF%*0-hG5uC@v0`IV6vRm=t;04}6?)PcMdWz%|jQ<3k)x@ka)$z8=Bjo58H93;#8br#EQHR{?K z9W~ICIu#R=@!EF5;@+A#N%2*koM3fKbD~u9CHMsnpSq+WH{LTAkY`_pHV#qJOh=YY z|7T9;a-ZkM?sqkU4pE?gZLJb!U@K0`#EpidO>+$C#)O~Z>UI7M%4?=yUNzygv?4K^ zw+-;`6&rQ;#%mYC7Ly?tBGV$;jS8!)$@}C!J3{g~inNkh&?wG_4Yt8HOzFKV{ay%f7qJ~XX4uq+&y^^@#u-wz%k0Q_K_s91y=V$wd_;F#@KFX? zWw8}wQT8dE3PxgVk2UzVX-s^^k`V2^tpC^86N1#g<3P&y{v;V2O3$+%Z*}{fziaaa z3-da~g(a%VA+GNi{Tb+A`@s7FC40EYXKL4NlD`tQO=}MW)7R0#H7|oYBlU>tsAt;9 z4EoIquw4-65^urCQY$otSEWzAO!%o#_hy0$SlbID(_5w@8>N~-B|o{tF^KJ(j}p(W zo;S4U2VEw_TA-1SP$CAmEvc)?rP*|Jw#c1re0)X23`p-AFS+7~Rs+ zASI=MbjN6r+USnaA>BxKBc0OeZy%rUAJ3mVj@7a4c<;Vn_j#RHdI8JYQVEE2krMG` z^c^?zV#DxfyOumtWzpGHS?9{gGLS#%n0%rseL4~lTYe$>dT2Cf?hML?SNn}M>6QG< zzQJ==H4H@sNnLD1)Y{PDrWqKc2(tA)F~om<5gCZ#nA4|D1}0gK>{ZMEMbY`q%}nt+ zGP+p)Kta|?W8?#24t~K2Nvj{e z4RH(%uC@i~rYdcc9wH8+OUvV;Ob+bdzOoy5rU-UUrJ`JRi!w)G6}`uV{CIVWOa!%c z=JkitPh`$?2M!c$WX<<5HjYef%RNg?rH&gU>KvaFO`8b*5UFY|_p&y%hbt@&F2jm^ zYYBN25HH0=m{Ej`_xUrBhUf|$053DKW8bo+Mf~crcF=X~vsc9K(a_RQJFsryYh`qO(AasO<6YV?t}HXPt;5H2K-ayA5f?Wo<4@WJ_Tp(0iED~h zQ)(j9*A@%E2#iYu=M7FJ@QTEx%1EE*#wS6A%2dij^$$u2T3bb%`Z(l%P{#nC_Y^7I zP*_j#rA9713(BVstt0LZo&E6;Zt6S2WjaA8mrM$uazXFjNnxor!2(bwBR>|5Qf9_3 z?$8}hJA$Ac($q9f{Gj??Syg6Hq4cU|(jHmI7|stY0R|J^l!x-Q?Neg#CS)cH0#s)2 zX1_d8cY(*})`!;#)FE3)^#Pb?cma|VwTwMA^6-i?9T(Rxw$#k`9@-E2xk$XPGW*@* z-oD56oSC4v6?1fw`Xc$w?IwFY3~MP!(fDO%tWVJBa+SyXpC7z)dgV4)y;0>J5MBnq zB_`6>R0`Tgy0!=RCf^tMy>HQ~U!vZs_Lxh@B6ml@@tm7IqdMLoEjD|!yrv!@nG|=3 zRi9v@P*^4{U@GrwiyTp6SPY^LVpz2wxah5izKSTJ`?)t;Q=BX;XMIJgsjKlxAnSrq z^x0pO^XE+adJ#HDqV=kaKL*`!D{Bb+om+pV^b=K)>&&Ge;w&zbfK9~5%Tx{4yMror zZwcej6ki(-#PQ~Fy>z3)nv|&7MKF^A+(o_jeQ%dV% zH$_l>YC7nVpRcU1$e|<0{4SA|e~RVYqM0ZtX-DXAJEKDtD5Y&#nhs|*fn_Mj=F-Ov zFaMxiq5P3!{eCIe0(0)0^4BEj;*3)LfDkweM!jU5RVZXXm-egNp6KcHKt@bW)}aWt zl$BBR>iPImk(9&T)sehfFcc`8@I@5Yy0Nu|WeRD*4rhPHW6{!G8kce&4N0c*r>c_u z#;IPI8tkt;)M)$8d|}immx;lWtrbsXsa-4EA~@Up&m(}4%t3po&ib;RCH1PPB=vg! zMm`!@`mn5&(Ucu>Tc_9ZE7HSpM zge@~SsqG!@O@h7HIo{GC!9Ue?ZAI>nEUgQ6I+TaOloNcye^EsJ_?8Tu*!3boeD)D) zW7ZTjycH8isza|xshh5)QeJz+p;Aij{{m~k>5C~Puz42MrnX_*3yaD?cA-SM*vX0Q zu8;Kqg(yCCyGbdaLG7qR8Rhku@yNG7k&i5Se^I1Cg1W1&uFpxdI=S4Cg?{{a}tt)KEY;@Q@B|}v=9b}XO-`&#{h}yTz#V@K9CFAWA4TnrH zPAxiV?T5-h3Iwootb<6dEn*Ch)9OfWP8I|706&ww1do8bJEhBJlw=%;;hb}0(809A zl=sUSBE?6xv9h_hrA7J+v-C=sFJrpXDIe{ld`C2mDZ$j?%%{xmf6M~C#jR6^eBzA= zE{WVbXA7*I`gCU4@{s^+fpu4X6Q4vQO?;qpPtQ-Zy;YaGs!AU5FVo&FF~ja&YBu*W zCpQJTVt-E71tWy{7-%^%Lqwb%Wcu6gsg|h+s*EBr9vJ8N4;atuJgK`B`16G=Q!paYc1& zVBlf*rdN4je)Aj$YBt_?U7)RG!Jti^#oI3h$R+6gOe(`XIy(J+3 zjcP3HIa7C2Z9GQhb1)_ocv2a@`0zq2#u5%pbXl;ndH(>wl?afSajtEyxFyG)}pH8+xnDl@(oxjWifc&IH zIGRkvjNbD`soxYeHP%=qnU^|9MnT;~NKaJ)V5=^#AEsBH*d%W~%@;7`R?a3{#kP=W zO_{W~AuUHTymLM!F;gTQ=u378u>oNRj{+Nu`>?%c~5w+TgCXv^J?vg@k z-IhrAHiIK3N+LkGFyXUCV#IR>HbQ~-r$?JHBQQk(CS9D;ix%nJls`1hU3%$N9n$D5*L zQ3F^n1u+|~6RE^!r=!QlCz$?XmV5<)7fsYLkG$35KMW~FPovbZVgAw)RJ(pt(O*}N zxf&~TEv8k;^ewh$!;7nodEPgjKD4)DLv*>DO=NA|^vA*491Rf6kJ#8+I#oWD8;{zfHW8Bk5X!0;&R}3~PHqr&^z}W3M=!53cY0$- zux>iY%1Q)DpW|7kcB`*gSG%=|E;CU6MTvyfxgKqoM4ZF2lyJ|8o;C?q#L!lkKvv3m zUzGbWmE$F4JL1tkbbaJws$5y|*ughDOX?yG(YR0f(ZxaK5%sVa3ArPsv0>cwRm6~o zh>2s@v8(JFX8Ix)P}%B=ZgJ<-6x!V+OV%CTsu;Kpw9)1`j)cTxAv9ruO> zX(nm&JR)VD1?jM=C9O0tx*9kAgsV3gm@ur&1qx405R!sUiYf9w^lqeDgE?r#o!*iw zdf^?{CcAyZ#iTVNBiG*pLqdk)=Gt67*7f8*4(EI2U?;~JlEfMDMm85LB=A8ki2Bn> zQ}Mu)+!`vVbkig__dAQxZi16>Uz?ldm)ss!J;TDYDnioFIbS)WB_mo|f zb*881_MeICp4d;I8|%yU60Rx+P=A{U_kkR&8XtJB6mS%z?zf%pXQ;>R9*JnbB1MN| zdCDoIv!g&mlA!gJVCROYcLTq8`x0xp%je5|s~klU3s%CR<;)ThO^qNIR~Ium)p8+h zl8#YJRZp)-^FP`|I0g8EX;64tk{7;yiRQWapGgI0hFEn$;&)f)f>+uo$#L7nbMg?LW$5|hapxsRNwI|(3X zy2(JWwye{PZc{RW&~;v>p7iP<2EW1480=-spw2Wwot)d7VEH9S<`YY|grYm@+8nX7VQuTG>e(yX;^>=~xaj z5u&syV8CaVy{!2-2`nuzNc2*g4b5-aXd|<`#i|d6v#h_h5@}_TsBzvFNcIC$$FCLk zz`uY5r3YU+o67mC2q7pGBfXXSJ7Fn&`MXz)t_f%n8{pY`2s2)2_-vm>|KKz(evxcR zu4eNdKk@y;=_gff?yYPww|_J_jx;&tjX-#Uwu#^G&fLlkBU9f0W`eKfQ#X7wZZLq( zaq<-eg>fD27hV5}_Uf6N0xa{I-5ulDO%9!r^r9ui{?|2Bb zEunh81)Od~!IL|9;#+;a(ngi5J7VCA!m>Rh)|OR~&j6iXFny6H?EPHsa_8dwfs1x+ z^OaVPZ#_Bgop=Qs%IA-0oWIR>+Nl~~u;Qwr^a5AjNApC@$UI#1!f(g~3;nk}9|_j~ zUPa(gLpkKDm{rVdTz4@m4q%&(rBR};4YBZK)Z*g&=lm)Mxr+^`)9oeJMUu{?2|ay8 zeTn~#CQ)^XU#uA|I=WofU-GZ+|0`2`Md-x-k#hif(L=qvTmL7_Lx&xH?RfH`E?C_; zlIm`_Md6J5?KpeU`QOv}fmLS3+=-9sZp9r7%IT1*?|&So%r}32if{J!Z$Ch`Db+S2$x1NZ8g2APk zeO49^N2s)X3)d@8`cGw7-4gF54wWCjZKB|=-#wAU7eAHX`$%XzR&LBvM_qSI$V6`r z>rjrgoVcvkwY8WQAPw@bq9wx$Qttz;ZblMIlN24+Zs;jPgGh#`4V2P_~{~w(`RvCtv9M zy^4pLAfz8>G^YE52fzoLFbNzfYvE<1}Hc4wF` z_<%+y%vntd`@9+>dKH#fG~k6>9{npk;p={_`70rwH>ItqL%*#sGJYt}MOH43)?Dgy zUZ~Ryl=K_OT88!#qGg)%O~#XdqJ`f8C-cz>K54`+*e#l{!=3{%1vA#6%5o$G4o#f` z^AzBB>}LMn@2-3m?qvp(2wWOP$-e{O8&EI0TWuZKi2v7%Q?>DU3zE3hQ~(S2LzTCr zn`O?_>s5`C(ZY5y=^_x(!I0jNUWT(G@FUQs**yJwk>u(tNZ|Z4qUqv=5&;kSI@~20 z0bE&haea`DU*4b3;ZAwX*ODK1w+`}zo2MDlrqNjTi)9E&8N&HH!uM$cwqUmjkFxBu|J5UtUKJ@M-I{)d={LMdV!?7?@c-Gi#e7~x6Tt3-dEQ#TivFKF zv0E3k8-LqmaJ9bst=bquouip6>3j*VgwX&Lt2N?%g!2z*&^Usy&>%xL3bZ)ZgX0xo z51WCs(buM0tC4P0%hhB?2C7O$O{7N&vW$+;CxFuZgMWZ0!Hinc-l~r*MBYP@U^9gz z(pTUB30(j`F<+IQhE_=tN_LP=(ocF%75f^YyDN(3X#6=bJpSv$$bz(*PljsEQ-|x; zq?B-BoEKcjM&+Y}gn)^C8&zx83EnwE2HYlxuXJef@eq}3VP+Uh311k;cd1lU3(zXi z#0TfB4Z?xrsk)xuh->xH*ONFOZ z^EhgJ!8@s1jA#h6v;U+vWKNLt2zQK?l9b!Wd|!E;sX&)}>0O4Q31^R_^L{m%F9ZgL zi5zNg#{bcP7Gd+PSE-A?RkxLiEd$7#D4O5^v9z0IAnBT=kcpQecw(MhH9UC=?5x=} z_vWbX%rIRQ=grwfg?81{7yuM}LmRT*$ML?j2RPBIzhOY8fC=OqjH z-1roYf7y!q*0@C&_(()c5Z589PCA}7wjz45zu_Jq(>Bm=us`WyA-Jr%ivzp zz_9+HzO;3~XgB-JS$F)>BqE=s-A8~rB(~;@nJ(YSMMk6W3%eoPGotxJG&lE^4*j}v z?+1T+(vYuEvtkinHnJ#&G2Y?d?P_b>iF~$pT&8vs=W{5WPL4gJe^HE+K!;z^7R*%a zDYn+lkD^XjLuaCkx4FdHmU)wT0d+nf-%2pV?X)6Z?CkaO4(e{yC=K!+w;LUr^Dm{k zs1q9kKKJ!yBH^=Sof`Xi3}Mv8x>7qk$8SHDt1O2rmSjfPlk7a~I7yvIC)xSom`s6Z z=Zjgf?V@9=Y(2c{IG$&roroLcLRmG~4)6U%``qg2%*o;;VUVo&8-bM8U69sr(oACT zMJ#wqQl1~3`t4x=l!migGJFCSgdQ89?l5~<{{?HC_~eu-@?Ojmz+;BR2FJ9=U<9>j zG9V;%R3dmHKyV;e92uBQ&+yfIWj)rErq2+UgeJ1FluqQZM@oILkDx_}cl*}N%HuR| zTI4(AY!d{{d`~n5NsO%wAsTX+_cD<8=u&Lsj{b673m4zm<%-Yj`7T||8eL>;O}dN~ zx%n(JscUPg*hQZ)ia^?eBujoM?{17)fD*1VA)NyIf#>WICNeYX)ZrW$FLPn6I>lbZ zTpf^b=HpJ?D+9&J3T&{uQpikyz;Iy-0OAu{uw=_1Xz1#ky9g64*bYBfjTim&~W zMQ0!=DiI}8tQhv}Me>o=Z`}~Xh-H7$L{YYUaQu7>aWLb&(D0x;!Er~9s|B_yxsfbH z-UX(s9c_-RhF&KdXGI?Jm&&i_F^>`n#9JEYT(8-_{4G3oQ6wY{ZAckVpGCU^-O}dVEW29Z+Ro`*&~*?g zib^FAia4Cmw?wfC0jT}%&%!H+{Qe?*R8GTU+u6Tk)JecFbZ2UrBq_{BEhywqA70>_ z-(XxObviFMm&lc94&xh=&gEp-w~VoSimOUyo={H|Q$AGqP&F$1Glw zAF!XF$e2!nB5X&;gKRC`?0;n_9WqOztEHyuV<~N^3(V7R#gr?4I>$GK#ue=@XwmbM zrvH4;a3!a)yi9kx>lg@U2xA;!aMdrFtmgZRGH19-EaLWlmY~ksUq(Vd{Nu8sBz2{d zZJ+JB8%4RG+UT)uw%=<(}=w(97PomDF0Rr0BxOm^qw=C+masL(;xQ zg>mG}htwXksYNB*;$Uxe9a|%;p`lu)Q5btLlq?!VKs{3OxbAW%R`8?)DbZC9R-6cV z0!~Gco^@#on%Pji_5VmufBGS(@&lE2-PKW5-OYj>5|a#8N&btXE2*n86!^Nvwtr1Xq#QC?Nm!`moKfEK9@bs`+cYnGD9yLThvPo#y88%0v9 zxvfX3wtuL&y;-f$f2X+_Q&fgF(VCCCij_TW#ZvUEiG|ES{yh|3#S$vWx1Q zcV;+0%7gCYwltFpe}cK9h3%avfZRat{PN6^7YRrit_|(dz3!Cx)E(W@HMhpn@^jKL z5$PD6^dIIsJ5g$yc0v5_YE;XZM6Hy{ZTHLMGu6r}3RFMyN&L9f1h3z$3^9s$a263f~=f@xA;dK?UJntR>|Uc+8)wz1-;7FdVcG z^U3b4(je+pCVuZ<{6*=G#Odw+*0dHYbL5@B)*G=0D6)&e7kk0puITy~WiXu7V;4?E zdA%u2!`AX+Vbo`T0&h-7Tf=2B)P6@Rw4A+VHsyP}=GK-gahfRpIdzpIp z#y<0>zLvb)AZd`nkx~r!*la-tcx#D4U?rNQp0l-lp8I*Cv>U*^US>`REa(4Jai!fJmSs6+M zN4%Dc5-Z;7+r)eg#M`nPP+*DljQmCbsXwi3xICq%xpz@w&Ii|&Nz_^?lavZ8?9V^c zYq^eR3KdDoP>s612aIM|L2;250&hpo$8t>sWe56Iy>G*;A;v|pqWB1fU{}A-QCu;j zu6{y<_?>&i(Zq7D&hloB=4T}4uspmP?bH+xb)*nf^%W4@ngnCv4k zMxFf=ViP&xqWuAS;13%d)7byS!uNbIQ>^HNJ%j#d_#h^_QTKO;x%@xH;d!U`YLot8 zG5UP+@YOJ4XNO5l zE$O^xp|Oxl9pS~@erB)+AuiJT9F^VvG&}HhtnyPE=y-uc;5abq_>FBq`XK?v4TO!K zWzLUHd33HYQO9C5Z$}sFMdZo?E0qFhF})8|kc_W}<&PAo#0TaWvD8s2u%hD7#-loLXE&VYw}tDK z5j!14&2|D@*SM#62&U5rq-~nq2N|p^-=@Y<2E(ATNtfIpp3Y?nmkqtBu8<8k85+c9 z+7IwW;By$FU}_;WcC#k-GTl)fMuo4(9nH7Dj@QeVy7W!PJdvPfk+H;|agUS~S0Xg~ zvtX|iT@Zwwj4=2`p0?;!1tpfAwk@)hZ%RWhFp#xN&AEpUFE^Q9frD?84x86$qC1%@ z`iLMuY1zuzsR@_z`C~X~6CWMa)Ckeb9r>fyOfV^tBu24G8Vrmy*pvDsGqIrvU@KTR z=_4qJ@CP7u+i-ze^ktZ+vYSHWm~`GmCe;v)TJ?dY2dSO~D6aDzXRL2FZ0c$P#LY{A zED-I;mR1YuKEMlf)$ADlbEzj6Mv`N}fko~ub z%n3LeZXA-acLeuUczTKz@1OuYDx8Y$$O#hmE}yVsWg#0QDGZLnlo4BvVtlP7clqPM z@<{nLt-+rcjy2P@mh)-x&i--MvKDw{AUF3^@~Nn`9}3I7nVKZKXQf|_cVhL^ZiKYv z&f<~GtO=BFF)CNe8EulHp-Uy$gw(OAONoF_&mTJ}U6#ixym4r~n0&K*h*2!g?Ai{V z(mndcOL9GAp9NC4-xQn+XI)3NhP-FUq?Mq)_XEgqBBON5Gb(gLmmW9aQG~~vo{w)@ z^oO*r8u?y2pEn||mK6=Dt5;l1e&9R)H?0}oMAuuiJ(@m(DL zqz&I@IN-TMYC@Q8`m|bSuHJ|l7b4sI=%B8(#HJ;0!W4-_V(s3#({+$9%+2i3 z!pjS+SzGQY+97e{g2d`0yO!NDz7uLDnv}G|7eL}AvsiXGk#OfDvo1)=n72l_fZu68TgwLmenr-ACT~pR^d3z90s^3KZ zR{fn7rbQoVEMlf4E6A8wxOhf{9KZFZZE>FGlW~Q(;aP)Sbgf>LxLdtue0}-`_p#ir zh4K!W4KMF>gB>wEu}GV-yAZ&l!}w(88{I*nkC(=CL;BfWcNYY9adB4mNG{MK_g)`n zH1iPhJX|{mq-wf<2a;i$DKBevl&HR3oAZ$0<5&JIibQtmzp*^0 z@uD{6zQB!&7;9|z&{oIDr9f>%)_PD^7&7<88?Ra4W#W`)%FjB71zlMyp;}fr3OxKV zu)G=g!7+!l_;s=O-jdI%xd&ACS-FaBk5SNbH2EgaHe?0+*X~7A6cKxuNPQXC+<v1Ct9>-N!mNP` z0A}VxD&Rk`Q?T&v?KNCZNQ*j#9Z^BRQoPARpf>UPyIT*?Qi#DSK#CoF&E(f7z< z#p98Jm63GdkB;3o4GOBa$I`z$qC{NRZ@;-ECV|?(>Isk?V(7?+)>Bs?zr6CBXk@}$ z0%?N$UxBFjB-Ov*!z(+x7v*pGmY&2X@xdkbG|$l_s2cuJrq5B)V>q*B>qXYbwHoYc zC^B?hb_Vzt2mhkPOuh>1LQX05#~hP)djFyrw(|7|g?EK{hW#55sL;`&pkDF|pzN&q zggpBy(sJhXEHFTUhTI^15d}A#MOjCM3#D7)e=&RIur>UPe}>2ZakDS~N6#_@2%;nF zME<)K_HS;18s5R+bvAJ8@gP1D)Q`cgAZc6t9BnJq@kC`;V>HS2a-q!gQxL5)q2Vwd zB~USv8I;KV%(<|bUWhISnh2N=hax^af*~&D#|Ng}2TR%+D(Bp7?^waOoff!f$LbRr z!NwN;TzT{4pB=6=3<(aUT|CM(+56A*n!36_s+^VJkN1!L6t_RI?rRE4+vmy~65GMj zVDaH;AP9AiSHM8U0&)uE6-!<>cjvhxxXf>+!5~YuHnm9mS&U9x!QNfmw~tI$?1$4s zG}YG2!>PFKB0u-zy-S{ph>+r@sW8ZCGCC=c5?})GN^Ph#r;1DBH` zOACvcXAZ)U_M*yY(Z2s&E1Br&%Es2BjS)<4O+`3zV-9=o!MmtsPwL< zY#cJt-9Ggq5#I-@7O8T(7S|-W`ITKr;2J4<6!!Iaut35>IAfnKD{umuiHY26l+`x8 z{PT>Z`|Q}N=9@0CPKOKl2LJ>An$uT&!s}N=cJeOU+dzJ; z>%}Ka1u1Y+vJ0F8vibHopyg>L_cypijgsmops@a2E3{kch1r_x0KST0opj5W_+=Pz z6e^lvRXXM#t^iZGrhnm8o{$BLid}xW>j!Mm#GF=b=3szb;`8TU{Px?1kU^3FqWJG; zJ5#l3_N#5nI+AjQP6iqItAA`_&3nwfNM&VpuAxWFah-Cgg>_0h#(OQZro-w*(Ubnf zD1m$l56UJsM3P#IDnhx5IVy-CM%JjNOhznEo1dxd=2{POZV6Bob;0gIX;rQZL-P0I zq;`iR%@{k21mS=Q0Sf!LUOt&kxm32{(wmY@jX}=|v^86ANHB2!=-)coG|3#QP!Q5;LS5;tw)W*jf8~cnma2dW0zxA zkhCc@H7!$_G50;0;R<0M%L_}m2vtmal7+o~QFG&`mpY$$Us=a-1cyorGn)A;0cde( z8s^g`%s^Wy8~AUG4KDJ{UOA8BuOzEVFTi}AF9Q@bQ#J)i72{J!Wz|z9nh}cP6F{px zNMwwtU}0pmYWh^z-rnwp>=~XXFR{Yd8SOo4HY&l!>>+i=Sd z;`bpzXnm3M4pI%;iG+H|1QnSpzG;2hdAi(?fln5rofp;5G|pdt3+wv$e(F8BLxN*O zVddEU9bxNIT$RhdeC3 zNt$_ZW@O!y9Tn-1l*zSavJ%tJ<(4&srX1cx)cLiRi!?9k-IJs6wdZq3lHun22(nwv z@sg=6EwJP_8j8?H{SHcs823!XwXWV~|7oeKrabaQ7axmCwDhC%8s&DAp&$?*{duyx zpm>OPxxuFCRhk&cOpriub%8g4p@21U-erkJ$6c~h!?8n^=;e?1G!MP^kv3qP{E;XR zBcJ_kbF(~d(_{}VC5VpA)@|ZhfuD;bX<)g2@%E27-`9gV123e@4^V%mN zXlH*&bNw*v$*G>#Us73lKYmRa0^MMFIKKw{mtO+0N!q1TBKVln=kDwmR#`W)%lp=M z@QGyvk2nAdl{A!$>kMysu~~%qYUkdTh4KO3)9S#Q6|KS|zY|=om)@6>JCECSni5@n)LJn$>onM!$2>FcbTm6QugpwN?ZGJ4^>5u;Be9TF;oto12LNGPw{(d zv5Zpa_O;&TMK3J^)dZmcAEf~qW<6O$)o_M=scp8#+`BF_OD&+5M6{Dy9IlAO3S|jt z;fys;*&b(jX;A$~zKIxGj#}&v{zmx;?Qdb$lJRT-)kOXCefma~u%-

a)W_KvWBafT^DL8=C?Pa~h&F5%@cD^=__O)7aB zj-Lo>ze)S&h}G_&I>nXyr@Q7NI**i%kHTZ&&yc<$h+(B>O}--k0AIhBAI7bRf&0Ko zH-W2iY;W=>ub)nQjNlAZdbrrL{oiOJg{}49f4`0Ii3Nolmd^(-J>Eo7AZ|eQ!odMj zioa|SLkY3>`1JMZf(QvS5imOZN%z47CI}&^P~V#N01*8Ux${0 z4tJO**4<)n&IiK{i2&};A=1`Y3Q&>b04f)&VKK-yoc#xH8n@``S2vnXD?DS|bKF4_2 z;E90-+Si;BE0O{G_yHe1y-o;NGE`D`P#pC6ZDRoY`QU;kB}r72nTsFren;W92q2)` zTW&&be1H%KU*8DiYFOUD2DA0=!!7a z)RIXQ*!3Zehf{8`u=^tH^EK^Xv_2c)JfFbke_Y4r>zqcm@~oPLg;r*HoBWW?6%$Xs zqP;rdErJ%}XBFBk_XXLElQGV*%i%^_gIv@lYDyBkMAb4-lA-}wXW?>7ShR0GjQbCo zdPeQmN7$U2TCo#*Nsh|E#2IRutQKE|lj~P9XaJdV_;^56#!5oPK~ORDEyJvE?oZgC zXYFvaA&wYtg(wD2gQrsH`=uo{FBHICO-a{LSm*x$I;n2CZFc#e+HTR}dqAvrXff+6 zotk4Yn2FBLi8GXBNS7=5y;=)^0c5dYD<7f1M_q-&_6>L*!r(BelZWh*Wn~MNQc`9N zJfvaOi3uPTPQ@)r#ZDkOoS;HFxG6;oQv`9dgz(nzFYt3fBo*DcAo1 zBGr^Uq)wHl1C)zc@v@aPN_(A7?5R>S96DwW5syMze4pT%FaH2kYG%>cRM(k?%2C42 zP*a>b+tlcpVVDy1OS^|Z!%7&%dq|@>#}8IARFqcIYpJGK{Yj{$bG&Iy%z>pzO46j2 z83_aQoJ*8q1```9ZwV5Ubh8jxRX|(3y{hjK=QvS~z|52!71&mxu8>7sgHR69w7W6l zah~wDSyHtWDe6Qe_NM5414FH-gD~5vDL2wU6L`gL)MeufkC`%6B&>ThqM4h5j!u-; zoiMouEmQD_N{RLI&kD+}m05^23gkfeMe8wnNO0*Of=MbS>7CShsvr^wiBYkUjbVwf zBzXNJc#ug!8jFEbUPGrp7@wEcl_U!(D$rm-@e+}^xiAQbnGqMlL#HB@t5}l3sktxB zuE$n_Ry+@i4X$LSAPqJG2>|Ys;o4-Afj-M{>0y4pF%l}UTx=*thR>@P7d0RjtYfa$ zXjmt0RwQWz$ByCvQb-ac*z7EkVIzU{I=|`W;KRRxwJHt?dN;+a?Ojf>k!5+oObto_ zDr^I&=}1$C2BHdxB79aU^8n$KCDOzIbkmqL9nVIn4G>M1ik1Wl2XkFMX4h@lMt@6L z-XzJ?cLWrKcvLy403Z;P7CK}Yj6h|MV@B6^H0Kz(sNgy~mu+@GH`XSqr$WN5sCZNb z%5^9i1VAIiqL~0~8sxABt?=^Dh^R_Q@gES9lj4ma zf|GHkNhaMv=5XYm2L;I>5(c0TjYTTXF7I6hD%K(4()c#LpX&br4L%WqVpO4s$9AD!$Y7-}v>DvA5L(4j z$P7tQOI#NGc>e%wZe%h40FF-9;bg5SnS{uF9AG9XUe^vE4cU&$X48^T2mqxR>F%Wt z6eTkjeUvS@@A1)^s*f`4?JloNqOB`YQ`(*}m-k~+Nl$j#94RY0RI{ic1t|oTMfj;$ zUtr|O1;!^n!Uh`&d4iPMo&X%ufVlyTbPw!u!p)QThs`nS1xm~P8)rDaGD$o~HtfC; z1wkW(aVHruXLt5WlUUh3h3BZN%4p?MRD^7~@n}HONt>twZwgkDL>zpR&#-(G=TgGN zRNd}a*qT;9Sy&ubufnr}NF)~p*u4&>&d*@l$68a=CIAFTOdZG|8JnHp?`@5(jZ6kt z6Bh%+M{Sz+(TIgm2Iafj9_2tvMwJ63fCv7dr9nJ^F#~9ek+Z-A)xfx=?aI9?_mwDj zcMRN#ci(2OQ4!pJXK0BjBpcXUaCZbq19=l*R1R-SwuDepptS{Bi~+2PM+x14b`~3I zIqcpM0q0e6JV0m?M5JjdF}{#4Ma{vu2G~x4?g_sl&7l@?zR={LBy&5kt36(t?@Bwn zMUJIFivSG5iCmkP3R9+J2>?Vvn-L=xWgt)&4E#jc(#_|*n3)A5-VDGoApG0_A45aR zG1s1&f)tyUl9HbRK!Tt`iBNJ>F9jrN5(rUD+W~w8@ojZ%Dn&)h=*58=5b&R96Ju6l z!$!FQix#l4b_O;701~b#Urcy3IF)q<4x1ArasdV;>D+^hm@AN2KAQX6RMVd-UE7FH z?Ae`&O+%E1tJ2IIxaXjcDY!N4>W8y@=B+h+X_av6U8M0veRgF_#F;HW3WpftjLkHt zUkaseaaM6ws*u|)45-q+nw7rgJB_%8Tu;`{!YP0F7qcL%-N`a`o$ID>{{XRJO(gIR zR}FXGKQLaLy;A)%`bS3?6R};g?KiBJFcSTeGpt}w9w&K-5CL77u_b5&WI1{Gbx^=~jZgNU;>QJd!ZqSPEq)b+Iwzm-)dG-c#dy;9&` zH~d4+e@6MJmb@*0Bti8PK_~m$21@`e#mE`}PcH5F2E(dPr~c1zCl!S%9i(A16)Jxh z7iTk)l1WpjuH#F9?O#nRQah)>GJHBpcS(zsl~|BTD4P%sj@bFbEc<1jV@3gkzY}y9 zzr*wnl08|dE?(pQ68F)C2Yu-&4FgWVf|UT6D@piCLDDGCM&1d{^F_`u>W zY+Q$-y>+0y8Nd%zNl&U%kZLS%?z33zI=jHo%0&2g0!pDFBoaw7Cy^YyKtaT+RSJuf zs1>f-iaUJac+aeIStJo!Fms_5AhGF6!ya?H=O#b|5J75;n!mJfO7% zB`P422c{ovfjxf9fJZvSX0Oa}$e39$yGxkZ4W2tRG(ODkD(AU}s}-9=2l14SmK5S)Xm|Q?3{5C5Gj0BNI zrix-H>HZ<5EM7Jo-yoNZ$a7r3fRmVISey|^2Z#(_7-1ARMM>duLr?>2N1=R3(`w3J zq^4ODuT5Ci2#XnoE@-$NvE1TaJE74glLi$V{alcIC|Vvi)*HvEn2SCLCcd zrVa-g3HHB;fpP?X4HF>Y@oK`)SE@4-tSSV>2}jo^T*Xb2fJ-WBWyzSeB3M$AB+OAt zLR1s;?f8q!zbV^}Y{FLQNMSlvQpsh*t250ON}DJ27TjpGZdA!ahlNN{@Mq!;Tl!lnd@B}xF)w0?}gvws?#$Iw;+M6y81yIr4Uz$gTs6M@Kc z6tJNHoWz9abR-b>?hK&Rrd1Y{l*}t=HZvfb@*WWpF&<6Lu?6@v(%?JM$pWC1f(d2@ zKCt$055?*zX!Sc(Y}$bRbfk6v00ia429&K@d;-Agk)%b;0VGK$uPbAk=97mV-(GyR zxo}CX&X(>uG>!yHb{8hL&_2vq)aYEtLS?M?Y_EG1EH(lMKT#l`N%J;w6azCY)6a&qpK!P{uijq`dUyIUX)(JSrftC5a728h{N( z?OpkQ@ad4c?qB}=TmZaucfh2j;3l6XXlrM0`W z5)E3{zOkr!E6U8g3Ng5#PA3!YsHCKu(Js$IEzDLs8C6W8oYhqTs%1mG@vD z5mdrZsZz-WHWEz1=BbjY^AU%Vg(`Z=Qz)S-CSr|(2oADEf^TMe4%n%v!^%~k4t-RW2OsZ6n|LCVGc3TwApj_dP2h<-gdC;tEy_7~WPZBz~F&O1D($v6{=9Bd71;qD})e>dT(r^xXE zNt{t;^g%{J8IvHBpab7KL5D=#4g(P1;xL>t3Wd`v5RpF!veXDnzyMLF6N;Ge-(d4U z$o7{x%2!-2b1if5sWwh8E&aA4Xfy;&!kz#_y9%`;?f4?&^;6~e$#{hgVD?YCz$+%_2i=GhJ-CYd=v33Bn77HK0DgqXRW z>OE!?I1=TOp~d-i1o3byhl^z<_O;**qnvX1dSkUZxPOb@=VtR*@Sm!e{E6@pT88pL zn7o7h?SQR9;0~bu;z(&;;s`vp-|yE4tLO0jV1_Ak&IBFEzfyg^7;6w|R@7I|NQ|hz zvryVs@8^^SOWY4h<#2w+^M?SG+=J4)?SBYpWW9+N+s9t^#N^Cy=n%Jzxt~@i&{97U}0Be_SYHE;P2CK9ME#r#Bx8KRXV!fni{v*bfK^RL?8e zc@j_YJ#a)tP7ao*BIoVakOdKV=Nz^>kB>foOc2;bDsN$H^MSuj{XI7a1Q5e;ceEG) z88CiOOr5<>AegF#98NUr$>Ld3#Lg!t*#d&(mu~7EKU6kF?cw>x-`fnbYfojn+R>5z z5AyvWT95t58mkYWfs{@5Ua6CYpa@0UCfK+31ic^*-; zLG!ui>1+@~FhL1Cg>{^xBA2sU@kWW!NgOONt*+0?dl*;o$c%AfpuoB?OnEm&z}g86bRf7w-)7xxZHS% zoG5NQN1l+lSXilme83+u7v9&~uGmf=Xc42oI(-5h@9vGCmi+*dggqOYgK{r>uNc32)3 zFz`gE3lkT#{{S%&{qR8s3M1w4A9lV7a8U>0-LQiK4Z*<@f(=9!Dwqaj`bG#?Gi%bJ zdq~thh`&2b$3KP$B2ZdLkU^3l57Yr5Tu&lyc8#Z*1P~G@edm6CO^o3iW*ypih#{oJ z3y~c!B0g8%*jUHSkWf>=6sHB~BZr^oHj;T5dUFGI`ehH&uIIU>URw1&6^!_v?3as41_ zRi;ISx_3ZXs!>w@Tf-Tj3+D=LTHIqc z;cWXi$|@Rq2`F`R>{3-|S=3voZDc~V9~_)-uI#1?c=otL(x$CaNg(r1uWRWaU3**E z*;!AguG{5>B?!f4nQm!=D;x>@wle{N#HlGN01@i}2ZhHFXL?7JUA^&Qvz+3rYw#h* z0g`7>RL!EsdDxi)mOQuGsj8Wn&jmmfioMeNhj7jgj5^2Ak4HIloWCiPz?vH{U})Fm zXuyHY3P&B*S4)J65=rO`6EWo;fA~wN=4m+aL#1sr?d3)54 ztHQCZ4mJW_rgq3-FUyg`MjI+w9C-4u7rJgie>hq>({z z)}S_bV#h$v3mR$HvwEi(t;w@h{{Z@PEkaP@L`&V)V$e|mf17PT^0t6WFD1rG z)I3Yr)hR0yvC;?ZrXC5&OEMK^TTvvrp<~|>OIpdFgPz88p~Wn{R6m#vKrK)kV(@Op zbDzq&oaD#jrMiNbl#-GO8l76x5+sAU#zQ}q`K-GOi6PXRwZ-meKn`8S z8Lnec%h&mA+cUzr4UDmH{Ft`7UgKunxwTPH)(B3d?m&eTrcwyLtBAC~n1FGi1iK_r zy!9sjM06|^$GPTC`Z(T#z0}qrWjT~DM34fFrhlj?gq`HtB!U37jiX$$n*+A*&(Fdy zPbpi{fSrwO80%7Y9nq0LP!tBOK$NCHx>N~BO{`2E$(=$b;9x>pq*SvFMSf+Onkbb3 zC>_g+>UA}y`cRO>_R8kTXG6Y0Lh0^sj}O78@E8 z(Uq#POG+U^MynHLN_Ma^Y*VCz0wM{-LBXg$BfT~v{{T8f5<%i{_U<(s(D^zvF)yhq zi6D?qh=aOkOadkpB4Cun&pZqtSNC5|7uU_0J3yiohi#qrsN@URLu%FF>V8s~x^+oU zge4;LyrdN<1c@;wWI)8SK~%yG$S!-s^we{_Jw_-{Binw#LZl}<1g(-O_0bk$(iafnHOL? z5!w952|s}(OPs8JQ=Cm)xr+<+a}o$*t{*#pANi7##rzYN1uO;;Q&1TV(}w~ z{>MC_U-;wFK6;>E1=$Q1R0v4mmT|eRSv5#128%MX#LX#Dq6ZLkgBK6O*A#~7X(^p? zDLR`7xqwoXg#u@oDw&I$5+@%t`%8j(lU4z9?zLW^25nd|r$-f!(^qD3=5`WDQ4eC5 z972rDYW(^hQRo&`n z+C8CBL%*MlT@6u{7?a^Cf<>g6x9XILB#E9}@dA=kSaBX)`UebLh99I(iJ?I8YRqW+ zHD0{yt8W2aNDAeB!Oci#mPWfG<4PV1DBlyBlJDN`W<<=H*s5wMN66SZ6yrlaD-9pe{yXX>s%#?{GU);GpLMHm}_ADVa4=ME8qpW^iJv5b{Oa79cRv{#I)3m?^V&n-* z3C^)#5iBe702+o5;AysUY~MP}zZ74Gg=bZ~Q&Jk?jywFj9B0B=cME4MR4awF*1Hrs z9n5m~)P_?@Rhgh>3?Y=#7ixoUcYNpZ3RbU2Ie=Ki7W||7^DO@W=WuDXqeI2){{Sl}H9gA|a-tl;IU>&?<$sHg26Dl!D42g3YxnQXKs&XalbV zrCL@7qNyC$Cr^kH036^807wLx7DuZhiBSVqCr354Mf&p*Aok5m{{X~I2I7Rhsjh_U z7P4akSHDbm#fw5Sg(kk>Kt(-ZEC*N z@!7>1GzS*(Lfu=frABnFMCl3ANw^A=Gi1TAAZ&59;4<{26|oIl7jf5CXCRiLjW;yS zOtI?01hAmKw%NfZ#=0Ni3wrGksQE5OB zk`AH_z_CB7HZgxJVN$YG4;AbjTe#ka-x$kGl#riwt5V-OH=dEzIhXwTQ%*XT+iB`r zml{%OnQ0RKT~x^K-Ca(VhRcqixPpfevljno_PV z6N;BB9HOofKuAHY8A&vBgZLx#g_b8G^qbqYFbg_D=&InG$M2UC$M zTIQl+lEnopO4#u?gg=TJW>edhbyE$r+jU7u9ooXurKc~-9a^0Mw51RCwTtgtl@R2q zWV+(iuNQtMB^iCAz%1T=O0+1N08yAsEZ~qpK|<=sg-IX)L65ut0NHJhPBYO~e}odk zeMf6$;{9EC{dP06U7bwi%tr_)nu+)&$(o9k&6K3C_K8wPN5FZ0pvtt!IybQgVhA^l z;yMw^-Y3oR4zgTqLj$wwTfUnR%s$q3V++J2mNe9XKpDvB)zq6>y???q0k#_O5Ke=< z*(yOI;!TLK+=4e4=XklCb0|UEetI78%o*dcAuyQ= zAbymj02FzX0>sY~qJtag)aK4F83ZV(9w5e=5(uV@df<$iK;0{EdA_Q9wycKk|-Z~wHG};(YbOokkUjXezLC!0K~$K zkWSyzXb?m}#<_?tpm`rZNXVoTmM3nM+4TE$jHo-I8k4Az1f&8akg_07!otx43>}4X zIH=F(lt~uc!neC&ZWz>*z=Lo~Qme`3v`7R4zbT|k3RMHWfT=w{uJHuLijYAGrS&DN z=ScE<@pr`zXWUQ2Tn|R-$g8eP^+`zzC5DJ35zHx5dnG%W<>-$`m?YpbtY&Pq7nOwJ zK~QcbJCvZQxQMq11dj*u@1^|A=9$(_+4(aRP^@R>F<~UDj|hK+fDHqH<%YG5{{Y$D z15{a61q*sxX{t(a@{|0@`jSSeh1;_TSj|Qg_Nf{ z%vwSUawe1kCY`+@d8)xJs5pv2{{T__HD;+94i03+G0ySei79YVm&;Y=5B(|AXh1wF zk~9UxT$wg*e~cJ8Bqt`p%~$8=ha(@2K?_zATAh=HlAtGI81T> zj2aGGj>2aP3~>48Do`icACkr^^2$r71__qSE&Mh2S~%zMubI1*XZczJl#(55Q8JPU zT0A6f2x)Gg4>{W*Q}>x5!U z6JBP&cmt_}*0(3Z6Ii+Qf$`_h)(55?!oTMDm@3W`d51m2nSqr_zdgMNA?f84dBA;; z>8Lj*qMaLf0W}E(xo&4+O-*{m2qp@E*xc{9x1aNf;#o4#w<}Ws%p3v-$%{{SBx z5MI#afRQTp>;6s<^1h#bgT60e3dAB&^e2)IuVu&d$LAJca zCbbLYTDN?fGM3SvvKB((srr&4?1M3r*%0}zr_ZO{m6 zlvVVGZ@BX0XPvyf;{HToQir>bgTh5f>OLb9INW%MA~~JFw8X)Qm@;`@z~1&ajVW+$ zJZrX<*Wm;&Uo$^1oL)JJi8zo30Br&450BIcsSz?GuPiJP9|RD^8&5sK7xF&Wk2%`} z5H0XQ6@mz4krQFI;Byc~=hpV$1V_YZ2w-Wma|G7kctaN!8(s$6gK703?gfCH5YoZM zv=3Wvt?Aus!V&D#Kz{}MfvT9!@{UsR6e&l^sjU~1U@Sc zPrN(+MhJLRbb@BlVbTB>^V_By1Q34*?&t5W2q1`5So7P+{{VatL8pH|zpuvxA1E3~ znCrKopQXn$X_zAss1|*%T}ujHo*y_M?eq5i@JrXb>jV-yPfOpaCcr@=Z`acZq_Z=h zkIDun!U%#zh%w5>*i;fIL2W2$5+IR63u!}0CM1a=0(u+Ua4)oN8*=*C0w>tBmb2FV z8++-})aS?~aU!Z7^(sXgk5Vhlj*SXGwf?rms34><1+QBy-Ptxkr{ZxOhH+ET)_8&)cowomTc#zloTvcS z^UPnnNP#I*s+6ffU4bK^Xi`=t;=)eW7r$7sk4!PeD!W#?^5uA+^0PNbioflKsHa4d z6k;r9Zx-ZjGlSL_wLAWHiH;RMt*BLQGEVRAaT+xpo4zCKGd7NX)6V-&C5jicdKueU zM*x4PuL9($Qd+Cxyr(zfio}Z--_NUR+@@NlHU9wQ6-3%i#)guaF^`!40EwK`V|!_x zBaKalpCbZ|Sr{BA<=C3A7?4+H3MR4oX#UJD$s;B7U)Z_gQa>?|%cgdN9I-^rnV#d9 zjKYwAh$jpA2?$f3AuS0_SPnS<083BAAj@*z(|-xvO3ILMNo-K$#?^>d_pEif@9nYme*wv9>?ec~aj)dUb00FeZh zL>(ja*rw(>jpG$pCZ?U~1-r#sh0lO?i-=lE9F&3q+398$Cz!5+BqFS-CP*tF2();N z0yzluyo+KE1#S;gb${#D@i5~}jatNpeQI{m&f{3uynT?no6&xMD$z}z<&C)ZTK>s& z6(qTr)TuM7lBl$lxFTC{BB8KcWdvoita!stN-+bE2{|meZsbb;0MsK%)N&8B1Tz&Q z^D=nwgCKLf=u81+KCAu`v*~^J2T>Q{{W4u+i@S2sHtnYuB327 zl(p1u4)vviNmheUr6teMw!$53skJ3%d7pzx%4eP-GF-piEd(tK{{WFdl&bm@eQ6x? z5sV4WWKQ5Rrm>TZLV-%viXw|BETG|>k#Gggdcv}-J4}Tn?2`fwyaRYQIzfb)0&iev znasnKOmYYYl>}HZxD`5hwG4C&1p1n>KIkM@>9vJ>%wtq_q*OszNJ_$BZ_w(CZ3K&x zJK`nFeMK#$`u*cHR1}_Xr$bTX#(2d@TQ^W7#*if5P$SHN02J89YSlYGpBA(z<^bJ=Jk_-n zF^AMtsh8AtDKJLXCqa?`0(B`@lQWBnlE4x`*d&5<@jq+BmoT%6iu<6K@cXdZ?UZDh z&V{4`xk`ah)RGo$J`kdksEsz%NF)&$ie!*$p?y9k)VmQ^p@z^$8}>_u+i zM$DtHL#?Fufy709{Le%-a@cV`sOFMLd+qoTenT2D^$}h6GXc2S|;; z4+k>}A|DL7Rhx>}e=%CqN$%-$3UoMz8)4T3y-w&tp-53LC55vJd6fO|^ihFES?JfX zm=Ke}hU}MTWRGNy8OzC~0{lXTT;!D~GpB(?(o*B_efYG_m)d{C*QY$(PnYZSKTUp+ zbL9~>u_#(qicx~YzUO^VV;LW)lc!wT@VjK}xBn|iH%d1-F?@muPW*HJlc4suT z??OW#h_wEL2RfUDiOFbszHrF6Ci`Wc)je2B~h1h^L z1P5}#+_Ml#=d-+I3ktmrjXvZ*CwL<2i2x*kH3MQLGdqJM!AJnwd3in@763t zRaHq<;sw%m5vYhTqxww5S^<+1Futt+05ywP^YYi_3r+PEYBAUJh%eNX$w?|vfieBNFlO`;75+x-y0F$X>aialx9ZS7yXGs7z5GuVxmeRclHRoGKI=Yma+0rMu z)(Ir)NIR6s@Sg1>P!kF%j<)zchti;4gLfbcFnT$9?Kx8{fv}(_qf>1*qX@zHspJZ3 znedRx)JlmXN>Hstgy{sS0VKusM)Qlc2Zw zs!BH7cRI?NqcxRI{%N&QvJ>@>v0Ov3J(-phJfLK`JF%DQX@SCVI3KhCbaAYHU%fJ) zTj_JsO2oV$61#2MWP!mjvuAk3+*G*)P_=JpzwjS>OZchmMxE@PyppLoHyEO&mUzX^ zF)l1{y--T3s-Modfpjheg*uu;npBdihd_1Iwv;o>zYsGU(bjD0L34vAgT#d2ws#Oe(7`aJ8tyqeY2CXe#M>xj7si=^evR$ZX)aOcWdA}_^DW8p+ z2?`;_Sl3E6;4m$yuyY%Gg!o-cEh$Zn5*2Vj@Q_4#e^HMs_+uKzKOU)u^q}f(y`M*} z)s4$8%aJJ_+DR=}4E;P@@7){cC(6>o5CT&wNC(Iy?R~B71|w_`?uH${{Wcm7h^F11dK;y`#mtL)k~{n_y7ncb}1ArL2_&ieiFSt;D5XS01)1r z@M{HU59x!_#sH+LPMIq0UuhLF#Ig!gE^>smsYD5Fz_KQkr1+=!xNXGt!IHSDbTF#M zGYW#@SS@$xeXNOWx}_=il?K&JSrl&6lmik8p{(TiEF zzM{58fp;38G#h=0uB^`L!;Tb{NhL~9DiRadV^U52>b{`+SVEk8_HARQYINCS+;? ze@bt@*vsXZe_e=@gaRJSFRKAZQFRuV*j0+Qcbl>nj7((A(vr?PlFIB;f;If*U`Q~bnSCt84$ zy&gPygU24ydPvN1O7`&ibkE78yQI)liq6Caccrf#p-EMkR1CL+auR>vxQL$h@RD1~ z^7SF5Rpze8geH-d1h5qab*?gj5~4Y9M41A1agAl+uuDL~ODFxNcpOqBE#2kLN3}Oi zPd`Q~bks@0WLOi!m}BLtWXXc~DH7GgjmJ>DvX!sYNtB>Cu@jL!KeCkR>rNiYa^4)t zsB%oT9o-&Y&}=Pd{%74(DyF8Os#NRMPLk`cD3YfXr4uoZcaOx+#$u+cAUuftc=XH&;hYqP?cjDe406dhr3eh%o4orNz-`TMm9USsQ!o+GOO zOY7{7cB=|=G3%${zl>9zar3#c(psukNk$o3wfB@X*=06ZmfR4ab$&nx0XOf?(-`R4 zCSaUKD~OTB{2{}T+XrW{VP0_Wb zU--}P%i2E-s26G-;F{dVmAxfvRc=nwb6RwvC;tGLU7`RKiIrQB9ox~)VfhT_CdKh2 z2{`#`T8RvlPhaKcl@mae%Oz@d2|+3}G4uEFGugCugR(ug%7PU!c%IGXhD@}it#dOe zGM9;w>d=r%rY>$8T4kjmscTYFim4#^Mx&&IAfW7bu^}MN##BJI>Lf(QJ-C)qj)W?q z!?R093(Db;7eV8$R$OgQfH9^?_EyJUmeLL#$HXczxDp+M7bTD*{Uzw#1Kn`l(;Yp9o^zNdzRHSRzcPVhB4<@$1xyA~824{QhK| z5I|I^RgdAV{c?Scqm;b6|j)-lg-jw3FMF@46} z{NRKJ=iB3ZdB?mGaU5O3f<0q`J==mmn|K=4HXC#E#7k4qTV8Q8)btkDoHTiPf_VuC zc=>wa5$_K=7-*$g_wS?!SB3`rdwn*y&kM!uwzlr`PmClC>}P2H-*JYEj-m)qd?NGi zZGLhG-_CnM`uY3rJYtChatE#mAW}3$iRm}qewZME6##zEzVqul^5=pG88TyFP5h4B z!1ez4Aco+AD+CaRZ3YKFv-j(Q8lXuG&CCMcKTh~%&;6#>B%2O)-`{c9p%Jm!=|cKe zzXq+WbP&^TSu#bS^c#O2IpBn5>gTwQd-UZLED5|wwA;=_rqDh94pq*!w?PY`HD?|5 z*UiX>B+Q9C;?wSaKAm^FVN+5=+!k?neQr1Lf?$baut5tvxlH}@!2~=g-e7xl0BvDl zHnBXU;mT(%1Qi5GJh}N>`;S=>z8eG>5y+2`!2~LZ&_J7MJg#I?@Bq2=m_C1Jsb8NjpDlILj}ij(VnYg^U{LSRwP`DxN;D>A9XAtzOk@mOo_r3^7C?pViV1)#N z4$xGpe2?DPB2=mGf1gM?Rt=4h%LFQ&&4zfl5A%)pb^k%KrOhL$?Y;;h$fk0w_Gy>}PE@=aWwhOETZJhJOm0r8fDD+DCUEsl5+x~f6gX0s zTHl2!N<)<>&;)?wy#T9#A~fI?jKoTlAQ3WplqJbS@y$^7h?W2)YDiZSfJrO?Ew5>R zsfK>n33h+k=Vlb@4I_qlCx?xm6%#RKTtwI2Nq2T?gCDN{0Ks=;fAHsF`!|9_l%W_LlP1g1I9MiDUx2}3 zlEAY8iAtBq0z-gf=j$QjJlBReqbuT$9j~a%Gu(?eq|Net#*t;ZiX6u;szpT|OG{0{ zkn?C!NwSg@l}bq%;}|?znui4wx+;gpr7oQa95kU*$XbxHP< zt2BA9u}jDH12@fkD>ih=_$<2@!<5gM{Bv@d%6u&4lT|8QpV$*&GuVD399_e`r&8t% z4Z7e;{6oaCc~JmIQ8W8<#}UtaIJrKxQqV$8loFx{eG4&rbMGQOPubql$;L~R7chkj zg(;1-1y1f=$Q004HnSOj)D~)mI%;orEXIyryEIQtTW_`{KtM@Y7AM1SIS@$}i`d6JR5Ex}z+A8`N|vio zb87OeSVH)Aa$Hn9*}<(TZ9Ai#IQxc|ka4a_$JuI2XnFL|XJ|kvOXzhgZl0)#3GW$k zX-b!*1u~`UklSH9uV*s>Hbm^QeNi=r8Nq3F3|N{zg6B(0h6_v~SuB#8hR?xE5~PYl8<^S_b6FAnK9`3INBE*K<5NrQ%Gwjv5AvIX3_F(Ix)<|D7Hb+zsD-9U24L6#2qHn%GESg3)rK+DQ#+#@V|4u zv=+S|1J1QH6yjtOqj7%`SyrP0AO%ba^A;GrC-#lR%mH?ANDNQR9jQjFM=?RdL1c=a zbznKx`@fB+FU`wOAzFw6NF`7#Ob9&sTpUcPX(?_R0zILxTaTCpz@afzPfr*?N$Sk> zJ;a>>tDhaOlII~o02Higf;C(cCRCz9AjeLkY;lj7haWbfspwdTBnmyO#s!oSK~Nh* zTI%k3hK$KIn$Q%5g`1>EjUbI8HU{TWH?^arexM|ZIeUUncnt0x#+s5Ch^!W_GKMW% zJ0GEbUNA#g5Q3!wLSaN3C&EfdQT-~NDKJ3*+(eyuc^T*S!94;h1o@&B#>Ok;1AL^C;Q8{9k9(N6t5eb z=e?%JiRlB05_62l51F}|qCgWS$w~nAiS>&K2nh&b<^KTUIZDj`01#P&Ywy+>%jp|v zRK6ZrY*$t2Q)RS-G|JM1tt|myRO_foaY-Hw{8mmf`$v`|i7YbYsI`X@lnEf!_|l{o zp;8n9bjRp3`yQ}po9LIdxP>8L&F#R;)FyD133+}83Ykkh6iOyZ#);D00%a{LAQc$M zGMtS}(jmmB!V+MmJbxRi}vAF3{X!Yvt8CgmNorBTz)N8r3S;^2t7gGTtOA!Sk6h@;uu470@f&xLXv|_)j zSWx0;6)tN^>#JHNp+~%z;Iq|&P3wBwmeh!j)u1GWE+FX_Aj};wc_(`iB0TXtxk?Vs z#h4IS64j`yQHXk07E-`XRKlT-uEoy0faLdhu`Z@6h!;s^CIAXZ36TX_N~EYAj+leK zB=t(+qMh`w2!58OScb4;cz>*Aml^{4lH{`%=541)z<9o{Xe!~tlBQl%r%#3(UaNp1 zQWJ7#!l@8OGMFThGDy@12U6Q|KLXJnD7GGPwL+_*C&0U~qZT)h9CBKZ%W$1_dkNuc zY{||poO-oM`mxz5X_wh9 z&liwY{mFp@l+5+d`79UlH>PF zYErlMRKSr?T$s-LI{cD<6MhdAm&EB;gpGIY2-CTLh{GXW;=0LiTBkL^;D9{fh-x38?V4<$%^r2hc&wG#xZR4V8KZg=qrwW4kF zxEG)m(_^!IA7|D+`N}SW;Jxfi3R`O%?l&Yhrm!km{P>)wF+QH49<++egcGQhh;}7N!rt3>(K}xZM1b_mF zAjpCQKnfmnu=>eT6F`@7{6KPl0(zf@q-9RU%yk72TH#!ojRk?`C|t(O;jS1ml?FiZ z6DCQY>V+u55xhygtOhlVeqgl_#rjm(u`Dd^ZLMQ$%d+r+#LasP)QXcv9Jj1*ET1Y% z2tg?cPzewS1QaA#5nxFW0pu?S8v5p_uq4r{bV$-YAe(|ppokZliWwMx66Q&zq;+9OWd%))sK|?KAV@dowlWwA z3tZb#m;lG4u%YNk>tl?^Vp557t*2cq^kG)7GaWU;b!)Ed{{VyQ61s;Lz8|D)C6^kf z)E(i~{{X~=7gp$L+J+Ua3EV#W|{i3y#$0bxu3|OIpjy zxspt*ULMmw)-&@oda<=9X_D;O}8hbm2K8{Q+8I>(xQ6sZ_YCM!S88Rp^+Hxjvr zg)=yPyalyBZ&60dIlm^TZeDTWr!#YT{yYQR}|T*HYt@f`l%{*GHMH3)18TtI(`;*b&Uk$ zb4h~`prAy|CI^Ry1BF0S6s}C7shE^E5~mr#Nh&hLbl>NLZ6qoaD1u@{6SR>EF>nk3 z2d*@aR5*wP8dlswwG14F)`wa~^DMKZl17G&|dusJP-Ay)l!)BPDRV)CfXZjZb-IDaZsP7!toj*}l=>u5Q zrE;GMCQkAFK!dTBOxic>xiV!3G#ZmYPORg(-h?*q%(G)QeE{h|7Qe!(;e(ojPJ_dk ztA-ILhp66R)Pbl4EhTb6)qOxgA_Sxx7?2Mu<(HVL9>NIWsVXD@Ub<4G^*o+1NXk^d zZe*}jG-ovpKz`cNuO039W#gvu4l~M_t8pnsQ|Mr7R3}+f%TAxXwOHQB#u zR45C~azf5Qo2?P4%(F03s^}KA01Q)CCZVL2Ns8SwB+qeNiFn*L5`76sWdTwLJ%e+( zw%#bqb1C^QW0+;DBqJ91haiNR!~NK!6~FKT;)R+!*YK@85@C#!wqcl&X#5mPU z2A$BLM#@6gl}LgP(~X-TTI4H$5&(L?v%cZzdV{!lKLWEABq!iAfk9XwQ(V1mA zZZ$PlF9Rm)Wk{c8{~FWXj3&EYk>?MS_zn2by8A z7*y~P#FWlXDoW&*X^ELBQrsy#Sg(FR?K^cF^z`%J%NaPNC3;m1`K*ro!SR!29yS3IR3*^bhiiL5Z7eFYkmUw0a+h zog(HG5$hZg?%Wal+rw0?;uW+Vk5~tg`TF4i)RW9q{65sw28(eVl4qEke{U|hcb`6f zu$tcc-%kx3bk*I2ksk~CThEjM7wRyU({txebv`hJwY7Wqk z2%mI!{LDh;+xheQz{!DfR269e2>bo8=pbq`PuI*qw~@SHa4uR1Xx&pW@ZWF^$3rnU zHQjV+Pxy|Y9e|}bySyPGBj`T z_dmuVh+&CV@89Og~!0@s*5NF&gC&Lv0y z4<7bs9e%Jv;m|Y0AkBKww*Ot(5*h@IJmAyzI>$UdQ8!{(uU44bD1-aYKa6@4M++GZPi2nE?L7^MB z!2bYPa0eZIq#vKlX zxIE)+wm01Q$N+Wq#EDP<($?qaQyA&eeOCzuN(2|Dy3KMoa0gDXpqZV?HuvT6zpflv zB7?inmWUv*QUH-)uXDKmeX!^tfSyz2H|2Zp$XocrTJ6@~57*}o1nB&Q^sR58r`ie# zw>XKs&%*xz7{Ty88&2?bv&4YCFQdD?HT_xp;p1_++hH7t-bIIAeDEI*hPpKx=Wh$Z zdJ}!VZVxY<9VX!VNdDbK;58J!ht8Yt4uYL4w3|=uKexjO3wzW%)4RU(4&0t1nJF_s zaz_iX9PT~>0Ol_HE$8yvf4}TO z6|7&@E%&!W&!wLTA2U|xEdf4BVoA_@#nUxji3llB!Ldq`LF3gm0J!mf`0`SdG4C9Zs0whX++39zum(U7pbpoG7RHFhD!)#9 zQo~TqUE{xERF#gpJGHH&nwnS*9z6u?-j6ThzrwE<<=Lx3pzyC9RvfT7a@#yX!}(oP z%{3#!fn7%9`au9K)e&_o8>Ux}n=7;UnUc#KJ~2ElLw}1)kze>pC#&?~y`ak_&Gdzr zpbz%knrK}ggkbT*h1#I8O;q0UP0C-(UNNDn;Y{)6uvFU!e3U69k)*0c#__q61~(HJ z!^+3X%q9A{pg3Y3s!b`YYvD!_x_@@BEmy9Rxr{(OK_`W)P-+eS6wt=e;EvIob+(zP zcu#;qShkoD2)*EvFV+FRHH_CLNE~v~Us1wbbGMBjV;g=?pM@R?hJXqpYupVe{zF>ajkrQ$GCB^F3fI)*Q`u2|DxIXN8%3yTwgDIfwU zXvEo=(_1wOi<4F5B_%4<4UEEdCfaSb=VA1;c3q;%3^;G8*bd!n@jKloJC>hy`g-5u^~H0$})NNf(sDtl_azrb;&e z6)>TDovBjeKzdilOHBC_WK8ywS*~2O7JIR&X9P8)9pH@RRarX;c|}h)j_uG7>2(V3 z+)3t?q=G>OKdK0{<6_UGAk3FKSUfnO++V_8g^3KSyE(CXG-;U@CS;(zGSmvNq$hyf zDgDw7LapquI$RSB&2taSz7-%Os7a9^i3AcZEI}a5oAV%GB+BCVQg}D6^*loDOBSY} zJa00wi4xqhYHmQj+m`tb{Og9ZK;tQat}H;Epd_h4Tio19< zZ0$~v@rdIN2vBFC0gjX|Da&59&QR-XPSQ$587AW8Mq&uQ(*(@k)YB}3?1D|s<6)-v zqv>d9nyNE5ItFTL9-8aTfsAK3Xf3sF0@EPe+=4d)l%^~$YOMURfkt#?7P)HpC0%YG|wlcU`37D5U z4es~e?#$X=@rau|$xvbgF$A`vx{v<=XG)V9Uk>pCPjr_Zf|!C-LO}|Xak&Q9I}sop zMTx?cun8m)RPer`g`b369U0!NWR;3agB`UtsJAP)@N*l7F3mgir0Yt&Di>5qASeki zK!Gwu57Lv2a;305AeIyfs5NJ`D^hJ&T#mVJ58sh$lkqEJ)~EPNo~#~T?>=$p0D)(^Ak0}!_Mlv5~nd$Wo4iYjksjVeh{0SQnjel)4N z2Peoo%)PZ1K}bln07er+Ff<;q6@wLpEr ztq9~sqgRY4h_XKDtwEv&lOsu#kYqsvQQ9mH*BtSiz++VPPypZB9v~fd-@&k1mI)*{ zO_`4Sa=$Ts=@?#Joz`SEX>DsEqOS_I5Dusy$c?}QU_k~)Tb8%4ArL0SPwPRXSe9WoHB$Af|HAv)K8#r&gM|QX!Vnl5q4TfCLHWo4w7L1vZ2ujad1HdHG!j8jhdy{ z>!6^?V6yO<7X|9$XJ*tr9JP5GM&8PD#@I?E08RJ(j!*Yw!#T2xlDjk5)xtrs0cDdc&ucSBM+8>weWmd!;N;K+;kCI zbj}pe_!{_azTOYc-K23s8K!--xOKqfT^PpZZ&F~5IhWaM`8K0GoHw}DfLvg>fOpK=@8Sk3_6v$N0>dAG!;&| zSKk_rS{g!7+wFhNp~`T#3+np!=stJmeHZpbq^E@$WXusI66OS>4Vh*bX-;O$LO@m@ z44FxA9Dou@q<%Vo9~o+Lf5ra*req)y`ONPDoP;POapH5Fizv?)kdl(JqGu^bgeVY6 zC}%O|_v4`mZ;UuK)EZ+oX+Tpnj>V|;IlQQ_-8OAVCAZ&%0(GTk5zwVhH)((le<@zxJf;i&BU-$ zW%0_&GtS0IN|NN2r`F~(U$z{)@bT!Fb4kgwE1 zncDUaP~s(zQ?(fMGz2vo!ZG-nB2hsW;_DymCuxvD`4P)^*FI)N%DOj$$|usvqh9#B=P;IZ4_#FqeS1!4(> zkaTd>U>LK$wx?d`^271X$4C9EF5zl-ym_^AZ9arJVN~c^og}~(QV?5N9YolP^bewp zs3vwNA21)-7fOic^dtwTQ**R=H}MahtMWKZicIAuZexuYa$dfv$@PQ(0Jh)62Mz0_ za)%4LyvDbMxFVE--F-tfd9H~wms4pa3*H4oiISBO2uVr(a(BFRzf~(6o6Mq*?C@mM zhY%_2QccJFLB*@2bIhX`#IDbFV-BI}=A}4{$A?TBYbgr=x>GjOxWEZ;6{LFo!0GO2 zsn~#q*kP7gYyh7CLI56c089ykVQ?|$4D6*-rp$WXEv?A42&aCf&3%=TFNX;!2mmlH zO#pL8HfQQc4H@lI?grM8bfiQT3j+ZmB-oNo$08s=7{HDWB0_3GbEqQEZ|iXO*@5<5 zm>+13i&gp2k3BCMtBBfh(4aw3I*$DYsF{K+0F;|v3BEJf38_E~!=AJ+a>j%;-0yQ5 z-ab*7PZ0rEYBjd4JTKNY_xn7o5}A{9!30lINq{XI#A7%4#U&oi=~sH|MtZef%bnu} zthb1|nDladS)YC4`A?T6h#@hxz!CtMj|e*xVS7YiPAzjw5*%!9)o;<M@Rk!hNLZQdsdN ziE=It38Z@8_-n0gIWAK292(NwB$>Q|o%vf=rpKl?da-F)R zpm>5`ZA1-t^`D}g&|`Zs`6-J^r)87Q5EPeCMkCw8sDR`p0ZR{HNz_y0-{R}ClQJJk z$3DX0rT$m6xmE`?Qi?!C>}Fn}mE-WT)B-~BCUl&?5VFBb+iyF} zUmZF;w|I8HuglA0=_3RYUFVn21QA7k{dXR*&(QkYXdr==?#+JjRCgr%^7u!~ zsq4AtpPxF!B)#f))8*&J7)St{>_@_GI!y2Bfxx+FNCK2&uJ8bZaCVM3Uja@t9|ODyuZ*uAnyh`!9PK@jn7O*0WR8k@->1GfNoW!2D3f3^dUF4QPBrh)WHkMeTX#xxU-M zjmG3*$oGeT$iNsj;mK;aZSCah!YXb_5zfSm%uVJX&Gs7|#v2K8=c#X}>l9B;uq__8 z8xVGziSxk)IdZz)X6KhG^YhZ^0%+3W3MyGB^U4Ws`xEZw1!}-6cwe5l)lJqyB?mu1Hm4(3$ ziopaZpJ90eZ>LVNf(S4mfgfj|zdR5@5U8=+mzD@1xFCcA3FbLKHUQXPmF#yno>*eR zqdBSE-|0voh*vJVd10tlo7r z%dBlC{SQ7|;v(s?hcQo;e(->iWAgb{zaHp%RXfD;^TU#-yFQ%aM5*r2r#N^b4>NXpcyPJi!CY2uCwP zrTWyD=s`PH91kH~=-TMad%)!e3Iq;cw=MTFc0Na*4=2rq`}?9mK^%?Wdk$svJhy@c z8+lqIar};7TrZOf&E$A#-L|Ev1qYwV{SWRiTy@laAzl32loS-6Kzk0bKKWrBi}>l! z!{t$^5&EfL$e##mfJqR3{i6#MXG#)^Sx(^4vxe!RdUP=4s%%Z>2VQ&f^7|Mf8(trR z2yjszW2OjjQ5?q5T|=m!TYm9^B;pNl?Y$qYR-wH8`Ska}3&bw`yzSozQ~@0#e~7*a zX;TriIr=qWHS`4ga{mB6TOPO|;v}*`S3qt3d0U(!l&Bx5b>t_@?~uU(<1EQo-`OI) zZw&^=tU!=vHa>)T`e6yS&v4f7)25DIEaBvo`)eN8@Pl-X&cx}c9bigYZeVkm79D;# z1L@**exIh?#NMh`@+ZUX9Vhrncu;n>_YgUtBZoLIj5B6V<)^8#dfFxzupk9ir%3_| zif=JIhtrlf^1hBULX{A_Mk5D>SS$|-#3f{!nlfIx07VTwXZRY(W@jIXPR%NA9!l`I zEVD4i%SwHvp;#nl7->aGEK-%L6D88NHqMm!H%r)hw#MrApLf zl3r4^Op_NkI}_G%@@qO-pP5Ys^&nU|Iy2Y1x<6aphmwpQPl7-f6mTg^=tH?~K;ijJJle6r^8$E>|qx<5kAdnU+MJ&gKGx)?+4cfpvaY2$ZLS&+RBA zUDQAhO?mbvP@mhZHX*;+vWZC~R0U&kYg&>>G*M0wBvcX<6*Qv^pNl^ZW5~7~%)dtiQ6&56oq=;3Fle9#6 zpJ!DuKDJPCek6iZZ{`4p8tx1AmnrQ)l({lz%u-Y?nTLfy63jxFhG4{x3dcibd|gdi z(`UUzt4Y)x9R#aXjzpW^ZQ}YqD=SpBu|k0@UJ~cv`+Zu`)G#^PCZpL>4zy6#g2ld( zlP5VsnknxB(4?dUlBt4Vf?^H!JuV;|Vdi0!m4cGt-lfBrEc;Tsy*G@(#HMHju&wD+ zTh~Er+=F6c1>sD@QRjKSS*Rfis#QxuNS5CSQh>COio@zPV)zI*<#R&`{JjCgjI>%SxUWkO0&*Z>1dl_HkH! zQ<{e%VmH)mkV)zQB$>X_Erun~07$hHL#xuo?i|oo;Ka#LQLQK})n*iODt4*s8lQ}4 z4wqeQ8G@n$PUg$l$(R!(&O}=)fSN+eNet!PlTm7&DSxzMagu`Naa`MkJCV(+a5=T3 zR?z28(Nu&jNg97p+Epj?sDP2Q2^x%@J#n%x2$_=oVF8>o3cIyDJPd0o#>|mBf@b~z zR35Br%EIxvcz220btwgE)Ckg&MaPEmc?85nlYOrkJWd45s0I2FQZ;d?YUyhe8GJrH zEB+u@wG<0{c4}I>#?Nar4^g?LTD3G&zl0=`4DJ0w2p~bU1HLQO6e^&APPVJ+S-=z* z)xe`%#7*^>d5Ay6mrMTu5?P%iP+XEjht@9ro#qN%CRYyCTk`FdEv6Qg_f?dxc$WJN zp|mwCQ*R;8^_8CVwDML`e?S>>xbQf6ti_3B{{Rbveh*aL4aMzW!53-du3mGR$@Qyb zgfZcCkeSFQh?L6!xg-#Hl0ZBwa3hR6PeSTH5o=qEQuXANX$nV>~NjM*e=^) zN}PYveYSM!5}5Dh}D>a{?LA@$*EgS11%^BEhMCn0EmeM4UC=U`0x_R zcMbmuM zg7qc$y)+NEec_Yk38`Wq=eXu8wR|H&@g7J#QrS+DNHKJ*PNA!PBt&xO2OCZU7)f;z zUSJYlt~J%RwTxt3VmADydLWkNA>6 zH!AT-Fi32F+ASb+-c>_|4%FhQ=|?Q}asUDTaN@D+Y;GJ}>K=|AjfEW2w!%2Zp?~f5 zGN?&u*fNliRD}a6fjUl+(n#JhYXAqWf8r&BnvqH|0MqNzD`OS}nZC%V18X&$UXFQ| z(mb5WD&GG9z-7kkGg?+DxZ6IfTJnxOqRLfGP;>H@3!E`nmmXWvl{#wj_8fdusmD9U z+)HGAR_xl6J1yDrCMmxmg9V5VKPRbu+m^=h`H%FK>lf)Ww$mK}U-+v#3^4Wo0Q2rR z4j@oP4y<0J5(tdTvMSuJyYW5Q0LotBcUIRA@q4vu;o5BBmyq+^KT;Mmx~%Medo!1S(-(mfgY6PDhc@)Bi*5S-yl z;mjvY;ZOb;#Go!?pj?1}qo@FqUByR(j`pNu>}BQ0Bp`1kLIS*6M%;#*m>xj&m2*^< zVl^a|pabE3bOswLN>r+b)~~hCM!uAc^DUu6Xi7>`5pqBV>I4|%CKMtpARC-xmbJ{b zga$4fa%x-0u2UrS*)pfu$)<`jVIiCO{(4f@g3E0GYAeVkI{b zvr(I%r+JBZ%1I!zyXQDdxk_d=0K>z}hO`_Jsh)L$;uon6G zq(?DCle&#qvvT=sVOMz8996`P(YE<7076QDHX&D1j0q&_+IgST`i3&uehE{ixJgr+ zfv^It$ZBs+xE#wInFt4n0O6d0%!jYr=NxOs{79z?u=N!d1Nv!!(80 zfy=76rklJ~LK$H%t2vCT)+v z2I4A8Bu<-;OUy*+QvfQ*%4e3!^|H%*X@ARS_M&A9$^QTrxt1seN|)LW$97w^E10El zB;pZ+mD%ZJ6>tS9czg`?1tnmacv-|al(L>d{CMT+YL|p_BD}z%jL}S^zp8Gfgr(Z5 zCmUE>DpEq0)6XfLDJTthh+)^kdBG3j9}0!`5`4hPIDB?ovRP>$6)OOn&Hx1?6ice~ zA)FJle`aQ6aK4@}_;9JD=6#>-CKJ^vXjw96BP;D9R#FQQ$AF*e)R0h-CU8oKmoSLk zkSwl(jP9b-FeGoJo1SEVVg>I8WRH{mqd-jQz+YD!o`An2UyT9%UG_a+;3XS2`qWp% zuwogW^gSH6(%mpI3cbKEU=1J}jY@)f2)LcGsh2Ji3ETi&3R0SBTIm~-))D~>OtA|@usoVb=sNpM&>0)~a!`Z0f$KMd$nUDTdlgU_y^65)zRnKvaVOgDJ2!IM-&U-6rW|r~>2-_7!olaT)nDETUYe*;R>W z1Rm!Pa(mt`Zq%d!Dvu82s1*SOkdO=s0FfjDU;!r;%%Mg~tGG4kU%y*KzO_y6_|fmq zr0fJ@JVBX;DO^xelcfsZNFeE6lOOtlh?56!ZVpcYBaUp8Gl2DaF>^+6Y8baS8pS-v z5+z~y$pHI7RSW~_`+)5p8-6GF!8}9SyY!*yrO28(q{`q#6U7bKm75R)%Dg}#_?kEXOL3d-jhL4|~mGiXb{kg;;Q&e!-AmSCGYjqbqB9|?D zh>`eK%iiLSG|c#s#Oh_GCYqG25|lE60#Yac076871gP^i#9h0XjK40CgF#>_K{mHL zU8|woJ&!}_UUi*gbGgo`bEf`Z3>f~;3JoQVi~j%#VjY+zgnJvql>NmcKp;s@sZi%& zEg}xlxsYNp<{Z*z9KadI!0oQ5cCMy9(UV9Ck^_+7Fh21FIA?l;)q~a9vR6_{4X2KejyFP*WPmi0B!g+?2hJzR^aB_DYPtUa)PB*A z^?d#xtVGORYMrwYs0ci?+j2mXsl9+PVI1vY61AvN+!ml9xGcmtBt3-;P{l1W<&}|Z zf+{-rRrKzSEyS71TMGfi$OK+2tH=``!_Rw0H~gat>Xhj$E=ouNwL4U}a(m7BUuZKV z{@Peja%eb%oq!H(3EjE~^dInZTT#Ru9-4%yOK&L54Gm9=Q9^+ZvPw?DY>-4sNCpoq zcfQJ~96YMKHle{T4;5}7QM>44#eS5Olw;-+{{XaaS5MwGNJ;)J>CWM!=uhy`oK9O! zPrr5&o#%20_}qz&xlTHc5V8TQ3KM-z55wU|@^5G*@lGx^YQ3uaGuUiEFBxCsAA&~~ z_H?Z3nV{xt!X-JyGQgFqFL?>3&DszNQUV1nSq8~8s#FgV$5-@Mp8o)Lc7{e8Vt~SC zU$L5nD}|>mK;h=7lE6z*JW8+zJZt!=>?*rg=9X|;$Nx~mW;|$ zctIroT6Br3N=f0w$zsYt1IS>alc!WDfkIMup zUc<-X2v&IjK6oJt&VUb$FqquM{$~U-7*+?LUoVyjA*CfoWhc;+A6yYQ!>rh#^i%wV zDnLlodF(mI@_ti~WIpN0!xy!o&m zYVl;BKR-LL)&r=?An(_wEG!Moj|202a-CRrNsW z6F)G1k%Us2nTz-QqIpsb&Mq=i12ZLwCI0}K#T#v5s?!%re1?(ye9Yn^t`T#{JX*wM6LPHX$ov#~uUvcRM;ttls(i{3UiSxMdf)WMnE+g8}9%2Re z=e`{THAHDQA0IprLlt!xzTZKD2niw%`+;(KgCEtji;-wJcMmk1>jW^w+GEZ?Y$Kh| ziGmXe1Xc(ksy=%IARm3XT5c{TcEJ&>)K{pdkZ)Gn7-Mtiu$k)w^y{}sz6fa*FH7ir z29&mf5P~PmrqF$U9~?P8@bCE;Mj)NV`~Lu}Alyu!ncj9Y&iCBmuvG6_Z>#s#t-Mrn z7ZL|aCS&)x!3f(|qs4NO@ra1)ZOo2-ys$wP=gw>g@xMD?k--E#Cw^c?)-Zk%>UQP4 z;D;uTbQdF?0UY`0A9zlLnB}oNruG*yO!M4efM{F`jdtV?{{U`I21&WF76xzYFn07a zaWjG(9J>+#4#MyCb|mimygvj<6@mzO>o*;21&^Kj03*|u2q3dR;E%2dAgNT?@Abh1 z5^Wy8IQ~AkAcaf;BXO|ufIa!0@WPgL2Pf`-SRu`w@6tV|>4ZkT-&i5y`w`{^$>=Tj z(*l4s3`o0c=gpw=)*O-ogxhg{L-oT`?+*U}k%uSV9sd9$1xM*0e9k0Cx;*M)M0=yo zrWBYE0l13;5n*`gCOMml!-xgRW7nR~@$QPV(ow)AB}oT}E7gJlZWewXrJk3C2blIC z{r>oRN=3zYHq*n+Vx2)_`o6G4+=)8?2cG``PQF-rebL|ZFhhc-+~Q1azn$>tA>ff8 ze_e-Ijm_|cYqy&VovzwKfNw212;FfhH%;q)dY1;&rw-xl!r!2;m< zZ5-fjc=hw^%LoH`75#VkL0(-nZJo5zl&jnjP@&giBzcXn5bw~B7*`%?1dyS75jW-K z&i<1&J7FzL?Y@+t@hzylM>bMsfaHQ5i08P}Tgin`JB`NvargHF4v4W5c)hfWk9Z4_ zzfs~xNJOh(O#0dfndkQ>1W|_tO!;)bGjBMx_Pz)qW=I2=osQjnutI=MfjtBKqNH^n z91xKGCifN-zu(yyBFE*X{{X||`b7hzkPVDnjlBoQc|Lebk*>6}?NI*!1NMt@r6y1; zNhO?~LrzZTechoJtK5j?{w`0?2s_F8d;osuZNy6fTa!CmbMl{lyyT1kbECK)AYW|- z8{|AjQqATK%uI6WJkP(D9NtAfP}65|xVLj1{CPBQ?!aoz3cFOyL50!wS)QQ^AJQsw zJe`FZ2g7BQo<>eM2jV*rWp)H5h!~v53s)6VWiFMuNzAcUA&ChK;7XNZr6g<*h8?-)3$No`<;5#js^^IDLGe1GcIr&d z))wC%EiuYUv9qQ>iMGEGY}a`GO?GVxMh^&zuneJEJJgOvg==2;)fsA4We{wlNE-{8 z1D`Meo5-7E99QiIsMWP6&YdG>m=6JCt=s3)&MXKmNbYXWvo|Tf4y+`YD(xSO!hnec zWSknLNQgccA`bQ!oMtB@{{YG0z?a5jY5}hgGD)MnzJ1oe^IAH zPQ#bZLgkS{Kr{?!OT7!YAonvHM10pN%kx$XZ?=`CK!Qk4tJvNk$sED3IIAXn`2@_C zrYztJ9)+l0(RyYURP_GQE?UP*nll^geo@VQaoANodYrAwmT786rGmwXRme%w0U#ME z@Sai#k&VA8?M&<~CKEbJs#7$z!&tFu?Q%CH>Dj+zF!sk{~5QWEr$X8><*B z&$9ERCyr1=#5;wc1t+f%bnmgayT;9wcEU~~XR4paEk>aeN2^ewG)mZzZBs_Ov}CXG zGAMYjEw2k7m#jx{wQ3`yQRVlokV%rQylHrWW7|gzCAgZ7!5TzY}xnT4F z_y}IqV*OqtFU%z<1xpUH;gO*Al^i2V>0KT+{$)~{%(N-82vQQU;vf{1+GfPwb_UQu zfHA60!yG;+g5bSDp*mRAwQjxpZVMEgUL7kGD)&pdP^A*9shm?*a!GjST-%7Mcz-iX zrCLyjK`Kn2(x?+V7(CK;1oNG{FTtZ7At69LOED@T+RpbT&K|anH#W?tWz&6VQv&7P zgX^ZfI#uJYJ3r#a^K7n`qrFL%lXR%b>L3#*^%4Lw2qxgpm7Qg(!eS>&2`MDdC>q>> zR(cZQ=;o2SWI4FZFw&-lBsI?v*P-8=R*|sWts)752{9280-zb*z(j~VjxYoi9uPUw zqO=a|MO#{PjgU;}eh1HVS#uKILM22i!)=lXAd3(|@mOs$@WbSlB~QYj=Dx{bL)GeT z;Oy?jX{uM5-rn%P}b+qG{0PSN^^cSOnuFt89CQHO_Qn(VOs1(5Q8H)koR`#*mtaZjpei1Vg zStRoT$ZbmiGgR;Z8+oeOzRSmR#s9&OG=8kQiUWA z1uKap@dC1>0In7R-Z_hkRI1;^U}Omh$Z5z>{IXK7{NsvLCXl$4q%15B^~W3}%!CGj z`Stk6FPLPcv1|@BYNjS;3?v>j&tjrn0xIAKG>_1S_BqS-U6t+3peg<1k`PY|C@U(S zc!vs<2qDTyQ6whisnyhlEwI`FB`FDjWD6&jqBTT-NHR#U>GSbVcLY6#j^@^W{cdCK z?2?`ujV*5f0GaRS4Xe=L(oU$0Nz@Fe7)%i&Ajn9x6F6`jDsb&XW4jPuq#dtj6oupV zUXIqK?CtTiYi#!_OM${+LP!uGfMj@qox#*h!SM4K$l<4vY8N&(Al1HadWwjriI5hS zEF9c#xY*gqt6Nyp*{)aJI7vt)DoNGi5#rn(fdGpDX3@4b3@#|CEak(S)rl7++Pebf zz+-)56@z8lp|CyvB=oCA!zHR!GJ|DXfe>t^CdTR@ok_SH>`j0eZYo+7>>Oy%cNDi% z&@+)1FkDjA#i1G!X+jyh^!mofrma$tQ>WBit z*a}_7{N2U(Ee}l6COhlWp!ix@GYN#@=a5J925muAbTt9#aUM_PFBNbmWR}{>^B2*^97F{o zdp57lv#M0No>sJ{>3H3mI@WTc%0J|Lx~1>C#~BC@1|DrcRXZ@Hp2+rnd};{%h6M5c zU0ekpkrjRr82iI%)fw9H5u?qO02iAqZYXD68t zwmU9%seg&i$3Hh!#?_iQH;Ub6r;RG8>uQ|ua(aZix+=OXx{96JMXna5ii-6-y((Lj z%?)GVgnkISwWfED%hCx&)_~v{{Z}P%O?KOKwxulNgO~C zl;V;>U@OE@ogGRE_lObdJL^IlsFfW$2_ke8C@LCsg%G7ed_(k@BVrHF#CzS%@g9Gq zebY?C>e{c%ae8ia>v{+y!7d>e2Tqb>sqhUYULXMnRi#4EAj=i+9;aGg_1kCCBt^Ua zf+SJX$uOHpu@M@g2~n_wkv0+r<}n22y#WV^)tK3-sdwp0R^n-QJwBo`dUPQmN{V(e zQ?-Q7!X%58Y-bRJtXWAQwS{?I=mwEEUky`0YHPNjfEtAItVvtpunCi-g&*m;1QXH% zq~DPMB*=v_^mEavwP<`k#t^CNKs4pe;|3fdaJx;ei(#u^si$$13d}d{jaus0J1!2CJ66#h+js+7YMn z>PNFW)U1M{RxhdnB!YPh0AV^~^;&snadUIdP8wNP+oX+ToZN^^?GRF?Wohkq+s-lC-m z1%28A^t+*6;`|`z2Nl@<(q+qE5z9MEhQuK$P*RFB93EDXQjmCnSj*i6d%+7SeiOs_ z{r=g`&CJOCH+m}VM41Q}e$jd_#pE-1#|jaM?B;9QENUmIOG%UJ0)Mr{BmUIcYvw05W>spT@; zBqWBZ`k$w1M;Yk<05wk%oh!pEpecMw7;JtNgTw2A0EuMk87d~mJmLMGIe^Ol01`ci znpT;J#>&KHznK9X6F)VP&cdNSwJTBxQOuMjhAyRWOLpNJ)3KMQDx%*~m~7AK5PU^2 zqs+)BR>CIZA0)d>kMUE11URFH+-f!f?;o!2$4@HDWRmPn9)Ok_>r+;_w(-6zt3!0^ z0?QBt5ow!&vF8!U7`TjTGG$Elo76cKaN_jV?|!cvs-}RFONR_zv^CdWtjAc?8NEuL zsZdgsQdMshN`<5unDj6LPp&ozGWb(KOTLHC#KszI&MB4J`T&2WL({%5{5FxSD&)W$ zgFD~JP-A0Z9PdjOS;3(%eu}-LBZ36j@}=rSH!)_XT1MNZa5j{UfJlf+RWNn|QyleM z&z>-9B|!02>S_V70)eSZHAQKzQc{?gP!_qk)EjIn@g4fn5>BZG>WgPBQ$Gz4G91x~FwUE=1}x zFi28NK{FTSjlZ**a&Z{h64;PK1{&BF7A|~EUXP=8w=`B0EuS!ukdW$j0VyBEO~4Iu zT9@DFr?fsOb2QKRpEC}rZ)W+s^vtLoYExbBbP7o~3TdD$H(80Cdlx3l#vc!aki-l& z7i%*R2+-7lsAi#f`G?xf_~J2H#wbl%u^FyBVGyL3E0?K56U3X;IC%9l@ORnk*%t+6 z)%CRMZl^Fns#sbyk{p$SLAsz^Nfs8qIA^4czsc^3(NjA{9v+~f~R^87Mo^`zk#9$IGm=X;05HBDUnVTxW z0TNKICSV&;@vFUEi_mD=$`-ey5E$H>h85`gJ9kDcnAD^tQn;8hOcA&Q!I+66WE042 zG7=Ctijw3WRHL&n`1R^x#7rOn&PA^L-u$iyD8wr>M%fBicsl6{*iQR{u-lvqfox=C z%2LRs{oP;4{*jHHiAZ zw+T+A1GsF2w^4Ir(bG>yXZ*gVPNsb=jStXOh_psg~zlDgWewd z-IxgUul+nY@|us2Gmg&l2T}QMG0D|aLYE3sKi;x^ZZwn_3D`{p)xE`S6IhQSXY4YaETTYUo zq^;ZCwQbBXZ7#rbKG0y1kHRkvh?gp69&QsEg2f@yeR46Ng|O5!62!A?#ylnCo*R3F zILC=NUaMDsl4jYKToh#|xk|@$;$(h@T2pSMpVS;lm{MojnHFR-T$?V;lHcNRXyYk7 zK+Kl1f(aHOLxnbKa6$QLpYZsfv^7alJiLUPe6T_J zX}|FJ{*gr~o0BKb_xAbW9CM}ldF5VFH|0_M%J_Yvl?jj_NHhL=Z}q?r7UQnE@6(Y7 zT_;4eOK_0RYF+KmVO~{$sa&UF*J!u`LEi^TKQEj(oh|ULjNpUxft3*9Y~c{{RQ? zh>0YczK+^`A*!)Vn@OMA-bVx^2$E;d1Q3?%4ZySg)zbko>wTL*92oh`uD4Ce#Pu%n0o(Lhj$3yb|a6tmW1R*igV-w|q2od+* z4=I5gY$KmZ!2~x15m+FEKMy_t^*{>0pL1@wAqmq`3w#I6h-8DRc@I0p&ih>3nAq50 zi2dh#fc;tiq;~*0!+P#I^4N`pfj8Rm9(G!&?o2hV^xmAnH01V*1Fxvs_9o^*7P>S1Ip^nd?;`{dqQY)A_agl~yx^P2I3Y&0{dE-cv7v>N*O;*}yxzvfcN~t! zErJT}FKP?xw_U9c2d|hjC(V1f&* zbU!cW1Q0-A;DQk{n7sL(f5^GG+imd@sHYsdSdXJ`c7h1m-uJY8@Tk|j>jV)YJ8HMA zpQXQrG*1Zt0TKwGFlOR;{rTW- z64$Fxxa#$%!=+-TB1?d@3XgaPax^_h#5TY}gZ+r+I`TK^hw<@yB!Rixof_k@B8>J7K}{I-vLJ_vN3?tN}N$I}SIMjN#cD|f=V zy-!+>W90`1WA?wdo)<^?)5mC-7Z36v*Jw~k{k**JH=Wwg(gxa}PJ+PqL&AxH%VRf_ z%ieyc*9Xq6=Rt3T73rYW{zu|y@4Or=$Q%7k?IhtDt=u#n!Jp$>Lc@VUk=6x&*5XhRPM!@xVT@C_FeB-+})SHXt+nVy8TcS+1hFK z0cotxGRKIK;=ga|AataR>U)dzk6CNA(n4ipXPGq6t3#xFLGZM8J z8;}Hp(tNS<`xL$`MDoll@EGLB;1|Cr~(9x(!DY)tlcI0Cl%c&$p>eNUAWEi|n z#iT*L;B`30u5SW{qmX?$claHP?50RuwyT?JZ}B?qqelDS{vkp;G2*@vTYy#Nd`*|} zT{|YuIhN*BP2?G)s04zfu~V021;Cjj^BT^PDjS|xC}tTZCO{*L4~t5Ym-fnWKfF)? zxk?!EFl5+~rzX#5e{%N22aR8|BOjKW0)oAih2gPbQjke+Z9@|=Nl(0>l@J*u#ioj^ z5EH6Q2;Zy&zn3#^XKZu4&kQ9knTe=+b0MAD)*9BJlhpD|#7;^=NJ&!!lAs;6P57@{ zwe$qu(@=aTRr!e=rfoKmMdWhpjd@bQH@#`sZ9Y-CNyKr7a7M00=|}P3pdg4h=C%H_ zs2G9-fdD`{ zu5bXurA?b9MRJ!g9mBYN7y?_BesIZ>CCwZQmVz`idiOB7*Q|5j8g?Z?RHfJD0Mgc` zR)w)Cxzwm38$zN85i&8f<$b2a;n@=uXGwP{nppF2qi1H`{dGLsvRFJ=_0kfYJ|p?1 zvo%`JyB>hJcO3r!pFPBUI?$rd67r2BiFS^z3W8l_X;W%FJZTenKh-y#=P$(IF?0QX z9Vkr81iDI&^nNXA@YXrbbA-%rSXtOCVVOG#b3K%)m2sdgpsB99M^W~3#jJ5R4bjvk zdX!m(CS1vcUpxzvS)wr_npgoun6YI4ET77;T-9&Lv6F^#l+&3t|J5b9M{~)_2V622bnU5h;NJK)ue!Bm+0cC7CaQR;w1FcBY!syi|4W zGN$rDKY;5aM?f90n-uz7Bh>ORjDey3*TY->w7h#mM&PDhQC8zXPqt6lymP8WlmB) z;@Z@komx$k00lv`cwnSS*jtu1+#LS^T7&IW0^pjE!Q45m05<{OR*erb!YWezVnf-$ z8*5K2G!{P%X0?iSG^ksepIB%RnO23uRRKm6KoXP?bsq@|*zdiB$o~MpJ8RhdwNjE% zkL^ZKiK<9&oE9G)hMkQ{>Fgy~SyN@L$OF}^6YQ*^&&qVh{MW6 z`N8FnSdhRJq)Y{B2~k2l;8cYYrF#GpfJiKPQjaueHvCHGPBQaOH${+8H!PvH(v>GM z%n}l>%e?!2=xMfs&v^;(v;jg8rF`%FUE>BLHOpnBF&Hoi{vcGTgrtjt_;h9Lz#B^(l>rDt2&}+Q`EE%X`X9mUM+f;-)6{yIvNr47T+QVsAkrD|O!lr3K zGzIHJs|{<@mYl`=e5~5H%baV>=PeG0G@wX{CifyHJm454N3JrmC-CYj;^R*mZ?Sr6 zG1KIUi2yht=;Zl4=|R>oP|BHjYfKQY>YZPxh~;4eXrDkMEGBUv2YR`$CT|z;*Gycw zYYPZ!00Yzp15?i67JYSWwu#tn5@3Ky0_T~P9U{P70TUy{NVOS>d+S5gdH6jTE6l`vsMR@?XS6#_TRAsPT`^;DgZA;);^ZqLSn_ zRGP5>gX!HY3s6h5aTKPUKu4&LxKzu6Wm*)g zBrZuLM=4INk_t%!djLEPlzUXyO}F_++|*GgXQq^47Z%n302nZWM2QiojYFQ`g##z3 z=xttN2@S^8{vV6jvvjTAf+mz%01^SE&mq(06WCC?q?sd8Atav)WXhl!3YhBv#A1;J zC5w%n?B?yUduwQwCd4@N@ew^W0U#3zx=yQ|jNBBE0)Uu<7r$49Jpd%VnxlI)2Iub5 z`oy#O@!R9dESa`$+YK#R(WyGlqof3+5>>vE5B~s&7#rH=9}Kyw1cJiA0@eO*e*kPw zl;=+p(B(lX-9!qW@#N`v=MFpLG^D)SP7qCr3JDj2qpBnugpib{oJHq$$}*(X%t_P& zL1(e46xrxO=H@x?Xmj9!>gAB^U)lqsgLcq0N!~farlsuCt`}P#awZsn*u5qn9pZ+;rz8mvR8tqi$NuD z`s~t>D?zAlD{;RYNhK|=v4;YZ@=I&fx{7e59V%JU8zo^VKDYcAW~v$2ZE{r-q|L%* zd6EEfqy61FeiY0|1jN@7r(a)vPxHA{RMXA36zCpJ*w?rV{vn4 z6rx(Cc9$i?Vz4q4wT=Z$n~LnfOqnG?a~7tS6tDr*{{W0u7gN~x5UHAGT0?oxS5->u zl_4Oq5%~&`hFYmDhf7LGZlAk#B}zl3DN0yzKD_jof*9V#XNX8qS~BU}Y2iFi`Bh=i zpc{y`HPyveE*;4c@+0t@ov$MF_vpJCzg339<9mJC999s@nv*Eyk~54BCp4BBTp&!! z!IlyU2Y{%$6T~^ov1*X4l7)JLiDfn;^q2TvAv;))@smT5#wcl09r$!C z>{_?1e#$!yH;*a8R?ZDwj}c8;#4Fe?ee%u3V{zvd$_*^)AcUz&2^JOrs!f0qs>BgD z0~)>>SSq0U>c@J$xrpB8&Gu@i(dPSl0IbeK+}b_AGD3VMPs~$o# zNgMzHPzkG0o?BRIZAv@hf`ZfuBlul90kNqKeXmgL+3yahAqgY`O^7BZ{{V44Hzq_8 zdmCYKqlhZ{9%Y=J-)>EY!s1oIHV@fNg~9?I~Sw;sW^<&3+yS7eWDXYf6Qyc$=R5T$DX}2_Kg{L zXR|4i=P5i`Q!D~NTBZvi15&T}S>LUEwd^aidXCh)L(3?hZEMN$XHi+0Ii!?0;c+T0 z69pv1B!fE~cb?Q|a`G&qYS5sZRx!kZ!~&4)0Z=TUEmk^PUm5y0?2O*qcDoE`p+_vr z2dth$ntl?iE<8Eux^N3?VPxfy;IRvt=NNIuE#6;7m9;6}qr`=pYo z0_F!h3&E2c-+qv7jaVTns*4iz+edrpWq4LL?^*jWmZh}@?o0hNdWOB4yEJK}8Br1X z$y8s&qDJDz@B|oyo$+E$8AaUj6QFV$=FfI_17SxmR%R$l9J$du&~ENIi&iy{5b<>_ zA{|q&A+SL(RdNJcfZOprKs_&W;y;727#Pr9Kpvuv}Kuic@5R0yc&{_mja1UByI@?IRH#vWNp#1PRCDLq^fMO!VO7D z01(EdfbQgI^*mFxo|G`yF*;nipKKkTKC@-eYUYQkiK0iq)I|)gs$~EmG!`w(IQw z0FHAW&$z#ac+V+yP1o?|b48iApy?`YhZEnhNHUE_HNL}M|CQFN66B#LD;M6^+ zO?3ytr1cT@n=O@&%W+aBjwIqJpv2clU~Sf|T3hQo@b|`?H2QVwuS1{ zwo*bX6w-Utm_8j>xf{)e@!j(I=)|T}$z?79a^izm0oXa$Qq%%G9ome#W>t!xg`4dr zNqKuIt-wtSG51~W;f#8hmOh%ynW9djhSE;v+EdK)ncP}-IN2nqBm;27QI5VIZ5+oM zjwF|NNgd9@qiSl^0z9SsVq~bwyI2{MEr!PuWO+_t>XMQYr>WasR`R6StcxIb6? zfkfW^pY&6WT>j0g47Kt8M;(fi6T}HjnTZ&bD25~=WYv&`pJhrMAeB-+SAH$>Iayz% zoE~W*1qZabZW4UyfGL+YOvPa)4k`sIT)YI3w2x^iQh10-0P?tUG=O5!>Gj->v(Ftt zhu@!8ze`7#ox88g=kuVv6gmuf+Gpcv_lzSD>|bD9_dCGj%@2j)Lhd;ZeERu%VQYKx z6#1X8$U?$uGdqr+Q}=uX4)e|Xx^kxBj1V%r+{xeAbn~CwoD?1V{a*V*KcA;xlNV?i zRQ34B%JFM&^T5ZDYj)3&p+7zP`9S`{{Gt;qryB&@j+b z;kLA`=>t;1u%BPAt-Ua{^X8gXy&;NNG!NJ3)}0_^@6^e`5#A9-1@1Zef0)4p2&kWl zyl=4fu;srzBY|=Ud=T3KCsa&AKqs$0Lu2YT#A=#9gcHmMZ;kx5VtG%^urcHU8*PpYfmp0q`G9Ih*#R9H}NT`ZAcx-l4H#6 z{m(!z(-44Edh6%q1xa#y&!2#XEa|xC*OVT6a-P3jIQtf>-T8EGaD}wDI2mw(Fh2e8 z1*n#%$L|C+O4a(529h?k^NZTnz2NSXXmIv}k<;M%?)=&aq7pzFcG~xjlh2m&ck72i z0$QfSZhm+mgi@wCe}A!p2ppdxe6MNvZ99>1f(ROvOmc!xo#TH<0CTnQwBhFBoPa-g zA+z`UVIYdZ1R*}V6Y%}|V1y;S3Gjx`hHvY)tiX+My3byM{{S2-t8?ODspekVm$~>>9RQF3 z>QAWKdG{WC@Q|Paz-sPqXXr!o+(HsefK2TdCM3uZPeC*7Y&kyg@A())YHPTtuD?$G z1tGinnY7-0p#0C48wnLWkH`)D_Q40AJ|-x{o0HS8E&iU{V1+$j_227y#S2^;@;CTx zxwibb-vlS0KdexWe<--$<1y64^uY(4dFAo%c|B&(XKrw9xViR=Z-N|8S^#NBF2$*( zhRqc>=O)JYF>lfhjCz0si-f5L%iV%K&ZGB8}))5j(*?18!uk(tPn%i0P7PcYxK7Q_tf+KQTBVFLndY^lqZJ=^abF17mM<)13Xr2x(DvPP70GPR&>P=?&Equ4kF$ zyx7=GjzIFnOH^D5ci0IF;`1kX_EensWk_aw0p90G(1alL17)Ql=mo0Njgj z8jXO$09wfeM8Bzm>XcHm0ig-d_o1gn&9ejCc_a*xB` zYkXf)G}Gtx*+&cIUKmTM%PE}9vqZM3@fO!rMu#+uFTkOx3dfPZ6nWgEgvoMAMW;x^ z<1zE9IFhNWKCM#&@gXWoQc_Dl5MstYx_@P_W3kfnUfuTJ5`>&4YmdotOsJ$EQ7;da z;XgH*ge?Bdl;W`{ME(*zTCmDYz?Dl}cwxq}z9P(3&+{ug5=!y~Wc015oKSWt-xN!1n8jcBPqCo1z*WptUaJ$*%z7mF9UcXNhuDhpMQXbuZH*q^7oo z-~=hRTQZPmL&$BwH6%t`vl+s&{BlI`{vL4l!S-3J?Gnv*dQjB3F|2mWA(x$H>P)f* zEh+rQ7GneI=6@+r6jgx~P~{e080`xz&D#|eRUh%KWeaK51k_PHgAP(upqWF6GLTgP zX*!c)LU%0x0JGy2AmVVzl%$hYAcB`WsV*)YRmF8;d8cMOS33cVIc_=8K0)>qHPEF{ zkjYyT3$>^mM%<*%>FN*(Pz3Hmq!g6Kp?H-6rVaUtH$0;imQ0l}g&}XTN|JfFXQjE+ zioADhw>FhE2Zc#f*aZR{KDDo#Gu*#0*}4`r9%PyGR>t6S20Csrj8u`Kev~@Y3-~l= z<>Cndr9cH)$OY+UxV4`3A&fXwsDDsU1WX=xl1+q25Dw;M#}RtA1&?!nrYy_EUy0Ah z0*R>A`U2vj zo5wJ4HAP=%d~H+1RYhYh%@(?xy=x={)HVruk+q#=A!tp2EjVDQXB|9Dm7iql3=Sqs zl-R7--AE2jrkAZuaLiT`e`PyFj6z@XIELa+x{w32(WnSu^*1Iu7l@cfXO_4?3kyM3 zk+CY7K^u}mRC5HFJI;wGj|+(af)vaF1R9nO7*>VpxiQn0Q;or;nMGdeL4G5l_DOv# z^lDhgDR6db?*{gwa@ST_rOe$>VMbB|@}d$2zgi~7HEC3DexQyVdCUa1%AIM*b_6GcZSn_d!kXO?-Syi?-Btqv9XaOpEK=* z64sTyTJA*{P=GE8clzlJQl5a&U(d(Cq+xmfTGdJqhfz>b8bAbJoL_Etn6Z;M$zf)$ zQuP%TeKqPE;?)fpoLq!W4pP7tVhCb7>f*Z+2|C8Er=a|ndqqyII$cd_%1Hrbmgybv zr68XYol_|VYiKkHbt;y$pIQDH@`Pr+u*7GxK~#>-V5d!!?D0!D{LWy(;elIlg(nQW z+Im4Gs%)ug0m*NapW2JtW7%&)m=3|xLU=_@_{?8Kwj1Y%R!zRzJNX1K-5lHqe^ zjNKidkuVMuE=1YWl%$D79jU_|t8n&q?sZ9)=8RNkJQ$p_D&m?4SW^hftLpM<%+-av zQuQ@E72lestZkH3A#S%zSeLu%OtnP0kD*vB-wBEsj3PwobF%Y0ifV#^SE(RONd>}H zDT$hrK|r~n5&7W9c4Ifn)0AeDh652RwYYpF>?Rc|lv9J1h?hQOoL3MDRNOu#GX%L4 z!aZEsp47CYER^#PWPDuSciSf)XX>bOp_eM@DCu2!-u2SkEFoG;rMAkG-@5jgLuml_ zX%Gbb-uy;l7n$eNRHOdPvn+qiC{S7bplQQ!-COjX{hfIdJ2B{!Cn3Pi6^P9pU&k*L zrOLxj3Sq{KMjsC*Y?*`Dn<;2qvQVVQdu92+d5{9ZF)6&=BcR-K+C+hSE;&{g(Q$eJKQiFn%dgM$-ORVt1AS_fKn8slWta$b^t;|4+tcKy+bWF zDFD|-Z$1~?#!)K*#24p&kcy}mAE=T%Qb#V6XdztyB~b$m^(ox=C;s1*4tG8#EtNqE z2Tg|HPNHO_Zv+T31cM@W+{%%{!8$u@_k?gRS}?rAfh$_fX_Em^FS&>&>I8F8BVv&t zoP@c>Iv(Ip(6jRmOT1883QL03`0Pfs*W}S2oGqdfWP%Th4Tzb%Kp+qQ01R5#nDCI! zJVYd=6sgZDmJC^gjftkv^oynrDJ9nwuc+ObOr-X2v+kTYm@qge_9lt-z@$}wx(2Oh zy<_uJ=_PX}`a$itt5zj)SK6HAX*1?X7*zqDW(sjFL`^6NNKjW1PYq2YdG>0jkK(6; z68X$^QTAP!)iUN~!x?V~C-U{$LPk@PuDRoID% zSFTW2J1IZ4nJQzH@r71R$Ym11&>Ly#{{Za%l!W?UD@t&{VKOY>kIP8JNl8oPq0*V+ zhze3t;ILWYCyAfZa``Pdln}4@)dpcw65!k^I%JSSghGz79J~;Zcz67x?h+=GDerDq zZfC!n#nFdh2?7*Ect|o*ohH|bl^Nbho_L9A)tKt^aTBFr+>Wn**ULolQc|5-l96M* z&B;k1ivs{ft+@kumnkZ#0a{VtU5&o*{{UIS1z)2Ww^!!vvWu)3NSRf@gE0_w4oLz4 zlj`{xV5K-o+Qi@G-J|H>#WxZXhob=W@_G#)acvpKd}fx?^JsN33Irumztmwj1Px=M z<=2p~f>c#Hl#oWN2__%_U=w%;mN5C2G^LR4+YwgOmuj&iZ5-6>nI>47hE&F8 z9@B$jL1G1tk4-m@k>OmtG8;Mtf|N>8_E~ za`{P=iUy@B*CLE*V(wkFwRisj0M(-};z}U2A%B`}G|ee!OQ~&Wsiv$CmenFzMKatz z1t}paDIsG1jQk_!zmW8^FARx`NX@%7pXFuA24wjYB<4s`S&IE;A1t3NH5Ka_H;@e?I4JIS()-6&kcGDZh5liD2jF2qU8CX|I7Vr1YXrhW!ns|OR+0#dK*lE!kf#RL@gJcEGuUaTgwzIc zr72tKw;`c%83UDtDO8lkng0N~CekJcS%5(zLFq{n09){%nBJ86H3wI;siTCjV@DxR z;YS`OfZnDsE+Wp-(+??1{V7o(Nh&2Wu6qD=M3blh0XWUzaGYuZo!zN>o@J?{miS>a z402@3D{$aFOEXrrr4L?Fx@-gC~^vCTw9g7fKg{wqNdm>P_lpj0D74h1ZhbG zm^k!TqrA3W85oRW(@+WGA(`HV>1qoS``^Ugob&m($(U@aN`lpj#sk2WI*2s{p+3<~ z1KyR6ko+QcYS#;Jn>my0MRsSVrPq{!twBk$O5;=#Ng&7($|o9Eq(0CbZZTMNDLfJ% zgoOeNhNBiNUD&f$?;VHGk7H%zSu!Wgl#k5LPv(}Q2N$h+r0d3R*Vt(t^2@E$vV$lA zDW4ZDiIQLm2G)ThFOD@Z8H%wa*@aI^R{ZyP^lYX2pjaATMg^!?q>hw~YdPY&N;cM| zK9p(R-~nPS1N8y{2lRq=IJbt&r_Ngh0!udvNhS58y&|4BHp5|Ll`3gQ;H`)|kVtC* z)a=pAouhXBljJSB*sp878q$Qe=BHLffRv>&vS!2tn3yC?W29s~g~j5DQsoefag-<| zv$*0Q500j`F~j>|>0c|#hpCr{h`5e2QlXi_tG|bp3Ryt zs+%H7al){)y+wf)7N{NN+BHjW*Vj~cwKWeu;Z~xj4x=Q=01^|ifo-jGje$y&DLfLT z%0LH@xaQBx>?2Y^#NuWZ%$qW3+IJ8qYLREXbZSyF?3{OHG6+`UEU32N%v{OyGA9^p zQuw8UO9vjwBb{D_m^HMGpDmOV$&R6&+#kEKv~A3x-wH<}43W4#5P9@B^BY)3p^Q94 zFwoFf_|MWhPFr#R05SgnQH;5D1XG1-k_;G)f>tMSeStURAnl5nWfV^1u9m4ZW4`@k zDUMI9!q&jol@0diTag~g{{W?L;syE)?cU7udhJXl_Lr9-=OiVk7IdK@R|T^nF<_Ml ziv!pDEsZJyS+NurCz~A@79sZ3N0j|55<;q7A8;B$VM#RvR)aunGQWtKffG_4Dmk% zR1hW1lQCpQ@Xl>U5+n=2t#L;70Y~(VdMnWRCk>p-aLG6zqVxq4rmaA|hzS&-sC^^m z{{Z7nGUVo6h{bS{qlL_+qy)N*(VOHFRD^);q^B^PW)JshMAa!FDO7Oovo}2?S|ao3 z>3+EFG2&h__1saY+Vg#@1qvI3{k^{JJ#a*h4UdKSY1j1x!-|PJk?2kc8+Yy`d>$06 z?YB}7@%{cI1Qj}l+#CJAJwXo*lQTCq`A5?P7W03-2Udm^0leH>ubA?SV1z0H?@>+p zj!j4zQ8VTQ^7a?o`N0vD68yk@JB4lcf$9ftTg|P^->2>7FhgOwzyOnRZQx&iW_IK` zY%PI_uy{hGq&kR5vyuRPs|+SdGd(Oty${oWsX{=x1E~NwR8V!IXNShOJdGhP_BN8Nl03-vkp}ZsrlCE7~ z55K;H5t3AG&1%wdA zE;fVnj1WU5Klc322q0>;k2$gauSnfH=C)Nhmwi^xwh1!IQJG+mitw9Xf^|_nx{(g9OverCX>3+L% zhWzUOd|oSgJ6~wGKc&7nY$rcHL{xlwdj9_a&ym3dRi~VD+vRdFK?`tVFTUR51X?dK z>B|Hyn|u94AS0N!KE9*R*vp%;bwXZz~n$CXVanOdtU@C^cxD+@4v;G945jx z7UW4A3lX;CV{KkUg6zVhYoF=kuSW#~KXdncZ)_1LP$j%M!>~i5ut5$^`x9|#i{I(j zmIxq#?atioaWZ~aiH}R*f)p9`1pRIC7$Ab6^6R&k_BbGdV1gYW023p5f%PElITJjG zJ262200=s}Q){*L`R@ch3f1T4IsX8ExV9A}mjqOKtPt{cw$nXNrQ+Xr!YwaReR;tT zL5Z<5{{UzwZn6gRZ`*7D6uW|7YTJ8n4?$<4FR*)mSS}wD#YFiTvW>{N>R|187ZNav zNLSJjfP1a|HH4FC01`QD0JIJ79K4O~he}fb4KIBvesPVS{;;@jcX3V-DSf>fou@4@Iko21Ux8#9(?*=etj@Qf`|>z zU8{X|hlNP<_rIZw`P$s@M1X^{>3@N58dF$xuN`{rloAvwq^^6cV zGtJ2_2CzVZg1zszA-uFkW} z(HT}-8A9h_aAr)o)0dkjS^*;p5h`kx zhza#d2`;phl@9d$Q1ERp0d_T#_HmN9mva6U%biYRRN6`tnP-)tIh$9g2-IDxDnmNS zT4|8VOaz3Fn|_pY+*fJwnVxe!U+oh;T!fQZcwga`+5joZh-M-`NZ*BC#N~dC@?Ot& zR{H%)>+xG{fxe%?tDOzD^BoIbAS7meQW?$rStuYFxv_a5o zXF7)?lT>kkNIjF8F<;(!VKT8m22c!Lj=GDQGaB@|c7NNg9&DDVs#L7IG-^{(m%iZE zr2-0-PNGx!mcohsFzaC|){vA1t2LW;XAg}><}kn#?d*GRs{)sBOV#^n#<4pHq#GCJ2qu?e1qs!(sSX5h7<|rJ&@88BWj_`6xfCD!uLc>Aj{?N=1_YbQ z)D6zlB2BvDvZQfSgoEIBv8{PVQ5cjisAMNnmK_{3o?CaoU(7<#Ql(o+HZlp7au+0% zGido?JyGZX07?7Bl+H=1APw#H`u9gTcC(f$rIcp0FpBKC#MQ9iOdV{V@Fv1^l8+ie zNKh&YkfV*ivzW=4JWCV%5=XK8M1i0+-$S9ZzKPo`wq9X@SV~%ShGo{IgKl=@3svJY z;I1TZh;lkQfdst33qqHuhX4p$L>Q9Wn%i>V_(`yUZRhv|%AA?<#|l)%D`rX4*;S2s zkzqpSHQvYOs>ERzhMcCKhBPWHr+#dA@*_v~y^uP^LmB-N6q?MkrMlMCfMrW%-t;8F z3L4N9gE8UMlVKRkc26DZ=cY`>0V@&jOH%?a!-R?!sc}*^A~3#?a)XP)D-|FCiI9vCmYv|BFD^8N0w9?nNHPY;tm9SfCSGvZ`KT7q z=xR^0_X4hUeHgaS*&NK`^4ucSp!+>V4)-@CpM!|lf0|aID%1%A1>z5i05`Fc2sY`B zQ`PX1N9g7{1guF=4nVt-Z(DNHTUL=Fn!=Qjp{S0V0tNOuKv5^$KoepV6==wV?y!g7f|E}i8Jb#+eTNz^c7D$x0Ng;+ z9HUwseA21fAt`dC7EjU7!`Ek%*^ft(b6i}m3~y+st2CNc0u*p45^$JwC`>6S3Be^8 zlqdx{vGWyTNc`6S0BBF7OlPI<#HaoZ$gz^~884&hL$UJGrHPW{cBoIq99B6hmBtmv zpC=R9>9UtHS)O}4AS5kw(1r7NV_Al2!##&*oNt`PCUx4Lahh>LyVq$u*4xeM0%%Ju zHqz3_URY_3B5NH1p5e-NXxT~~TKi54YVdhBGZ!^v;xR`VI&8%P85qPB%v#AIk`M~i zuXlyiNdbqIJ4b?*nfCtxEXig94ZjGd>*?loQZO9(k07J53O9u@?@q- zQC}LRB3zKA`&h4GIkU9SW?7pD(cdu8Bt5M3vsX%1RX7{B1QZH_5}DF(jFvc z`Gz`j$x_vfQs^FvHCE&Ut*R>@p|3!hQhOiS9G57PI3-}PSS$pju^ck+I1q_!4wOnk zv(k@F{^4FO{KHM9ZxG-DJV8dqB_ny#t>r`joz3SQJ2A^b6jT8M?8?_rdDF%{ZIfpS zo|D9+)NTP|zRzd9*gFW^R5_r1uu9TN3Dt9Q2C39<$|IPaqWa9~2aDZ3?%pG(hsSlp z;}WZshB}Z1Ue0`Udl;Nv1lYkQ-Xwu}w&LPN_mXV~6%3LtKrhB6JJro4bd+;S$d-~;>re{s-vJ#{Nt6XqGGb8C8yj) zE6-89Tl$HROP&7!hH_U%C?oP%5M5HeKx`UOSzvnS;tf+p26qwpqV4u04t*f@r7D?% z6Mw#Q$%HCg^x{etj>mAT*M8t3TuI?m5J^ZJMAu+;C~1HFrgDVPRAto^Z1!WDEwtqP zQ<-sYTTbGuGGs4KWt+0aaH@GrQ?AlD*-bhfWiTEuPYnD*srwpYZsAjg+ULobTp{He8RtO=Swm=5sTZ?AMktj-X_YBRiC_)LQnbL0xPHZWvf zL}8qO#GRPh!^=Y^V*(iIclSXBn1_A!ZciFpi+JrUrN-EF1*cGuL4srwPelQ|%mc_{ zW6LszSM39h$R){g)?@OkZrW~b+Ame?AxQx2u+|0L`jM*|(S&o(X`EGNFSu6X)vaWR zJ8CePg9#TMWXF~_j6NDngxa7678Q0HHMj0)=D57Vb}3VMot1CXLiDhMWjSk$S_#xi zGD1X~+AKHNl6grRnWv4$B`Nh_g$+xJl6%|6>^ViOzb{nDf?)>EC~C*-9frN`R~fMVu)n{HA2hy29lewjN7iV7%`ip{ee5OM%1&9E7! zPlX92fJnvWczzHGb<5!QqsV{S0ol*|SNKBo-Igj+6s$jFBxj(I2!6|F8DDAVni9!Y zA1aw`E8v>d9LEL5St%u!_ZTsxe;h7ip= zte2jnVfPCwLtuuULY`;^qv~|&ij|sYSbjQVu;OrK0%l4C;LOCDh001Lr`rye2BOjV zX3KEM%zG2qOvy4z=XQIv7`(AaOu2*d>zRs^ogp*P!WANZAvvxhbfiq+GZGasDO|J_ z#s%3vZYkf)X&X{fhHL6jl(w}k2un}6+LGCD>J7>v1yBl72w4hASItjMxiYYQv(Avs z>v1yi@~P~v^XQk3_CJ_DG=rcZy+I@N@%T9HY_8IJ59|&pGnX}Z9DyB|H7Zx;`yq*+ z*;&&Fej)4d*(xWXkPOAG3QO^b-Fb~{f^{84LJ|ld2?17;00IdA0JtUuUmSi`9m^Us zJqy*yrOjyOjBG%UUrxmnDTQiW0OYtdxzd8R)J3k6pq0T;xk(Wu9WbR_Ow3O~=X^;% zSO^3P*!8_Nd+A+FGYv87?kiB)rMjtsAlr2!2VJ#H7_pt8q^TTAsz~lWMfCj( zC*r+Q3o5`>)1z5|2bl!5Xz`icC#=5Twe8Ou#S1!jPV$JJmbWQ>%@>XXRnP7_o4Fj)H~XDY@q}x>KwAAx@2hz zQ+?tbCF3I2QP8fdM&}*nYED7f&=c(9C$L#GAdp2t0E$`k(^&EL6$%!&3 z_C$M9)NJWcbSjN)iH}`)HL6U5A)%&8LK|U$2rx8+B_Igd7qpTueqiIy8LF3z#m<<- zyDg08D^b!20twVi5R{-5bV(qbWinpM<8uYdP?SPJB%TEX>R6T_UZZ2X#v`m%3B(*o^+g^)9;c^qNSGAWve+dn1=+h?1;QQ#C+TQB7S%@4XFdBh0<3^p%|Fvu9!Q)viVfi!5;o$}fHv z;xASq%ZqQOX!1C~qkQk+_f{8OlrVFV;?aUF57O~>J6Cxs|Xq!0IU z&$Ju<5X1%*akW~JqG9_zjLa5i;wH|WI9*ecsxe~3jWwpM9y2;_2zf1q7Kn(6NHILg ziH*ecxs$dfS=+2J49`Lfj%M90pkqhaK2ewU_lm{ z=lWz#$4QbPMk&q5rDRPbkS{0~yw z%W0~-;+}=w3ougSsnQG(45%OpGjcJ}dpR_wWQw>f?r7`-mcJDVb`~V+I4`6OGA7`0 z!D%WXJVArFatetI;Jr_}#ofq{x&F*EVO7JKHF8%XqJo*pDv8wERtDl^s$*`pF^W9v9VQ*!)txk&4A7ii>fkUKFH|^dSmU5*!5~|j?HOtd2q-*J#>rWU>N@@H z&_+7a6dj-RKd+r*#-tH&M^H!33=EIk)DQ9PiD4j`?k;&xQ+xY=j1c}aN|vMZFIY`o zc?)@-Q#<{}2u>~Z6y|hyzm|n65IM(D@#c3hPSei>DNtdnZaj52EKQ$SWd8s-A&}Cb zIqmn%i5tvMu^1qM5>HQ99J$_qk1R2l{KL<55YnASAP@w>7TgJ#F&E`GorWA$!&>)V z6iG9QNJvu-CRe+5yBgH6+2LY$m?F?d#>PQOzVRCfJM99!ySUw{YHOvt;^c`W0=0%G z-5P+^CElmR4zSXsi97Q#e{=UI4Jt^pcQjclNblLaOazlR=gjTP(s!`u9I+b`Ye&@6 z&q#>9%h%%pD)Ix`dJc2Q4w!5tQjDm?pJUT~wzl|sl3bRj^nwO81p;OWH;#bEE~X=N zo6gvf04y7?JN&-TLx~*79XW0HC(PkshO6|SzIY&j;DQ=Wgb@eM{?UIU)20X^gzs~? z+V-8d*n5kd5JO3uo$x^kf(S#`$lI^q?}8B0x&U9{`oe00Kh3)Hw$Z%pYrzdF9nQz4 zeikEeVTw2OJiM>GMW@V*;lKlO0QmXm3{i7k2^70O*!bU-L~T0`cf21s*w|cdNf>M) z4euJ1<5HZj{$7?sv z#o?rbumf(hZey3P^MVqadHrIH^Z4&R^9TES;DQzmSQwx1M*jdJeuopd+<1s}h3FJ) z*8+#3`a*K&xRJ>3{C^u_2y7?c-`~>&5TZH3->v!$hsW%YdGmOyi5DMy_WEFg93n>d z*!j;n5PDmsz?%@DIRF$v>%JOuXRn^W7+##Vo$WGY@3rh=_cn|W?Tdj>{Y#L0f!-)v zZlGMmkv&G|rr^cM!3YajJx9UqV_iHd@bT|!U+(_6B2jC*Q1uYFA!4vW4zzS9-_GLp z*bjLzhbP`0{{SNdIVRrrpSyuFdK@okp5>@7(ax zqY!x+lkb8WR7hi5yZFNjqhNOFG1h;{{zn^#03+z39Khwj4DRVscj(b3?rfu4;8_H?2e3?XQG}(@M-nmgy%7`F1 z-4!GB$Y1J8YFyKDBZhi$&L?5AyuT^Ktc2qgHxY^s#%>=7N+rtxRZnm=y=ea4{1tXm zR%_aP{{XcZ=@Lxrs{@9>WiKNh)mjug(D zbtI)!?>bckl{w&96Bi&PkVoo?wdckiqDLMQa(D>_)GcFFwb|a44WO)jrSxSnQgb{a z+=7xgj;V0RiS1jymyeU4+*I`zb(s*`r6R7NctHRJtMw>|+^PsZ&_+1Cq#$BFSpaSe z17gLmaNP~#+yQgWUm|R@sZ8XQg%&NT0I;}n;_qP`<3+^%EC*XE`iB<1i2x{Ca^u$tYD?)b0lJ2Z!Bn33S5i3wfPzlKqSW9;r=LT(Areu ze?)$uWKF>ZL;nD6po?xXsN?dZEO7@Qn}n9}ZdP$&QY#zYQJyAcJz@br!^Q!6kw+$% zrq#S;S>+`bf0wx1jiJqHB!vJ>?nIM#HkdrXxW*1qn}wKGf>Le;UBfvQ3{IiG0EpUV zCQd&Mz_f)kvwK%0XfI<*YiQzr(z_yYhuq9^p7Dy7)?Gu50^5mDB}r0LjXFqxQa}K} z>5h$%cAZlH0AVMa+3TKLOB_)OR0@Vfksc6^SKiX!0 z!xbo@;vW4?p+U4}04S^gpat^h$43Qs);SL8*b zmq^9t{hOb{vo&b3ETTQ3R#8oQ6Qk;5r}$sA#%r2s#;Ssl>Ot=tT!jt6)RIJ*Ct(Bt zO{X09Htgc$tM_4xJvHy3qrD#b2}k_*&zj-7$djN~v-EkeW^vjSM`33~$F zOxH7&l>V*t5}7HKkV>LS0xT>!`C^H}iZVz$5FF6d27M1JwWB(W#Y#X0KvtlgYT7-& zOGXdGwcnYgb+wS9;#6a#CL{F==~qpzQzI8Jl1gUDSrq|5fd0+h>8Um*=8P#7arnbJ zT-1`_k7t`#m}k<~14h=Jmgk@Cyg`|8`>QBvZkp;u5wmVABB2e>ge5NMQ~*rAtCJ@i zwk|-$<&zXks|#??%E8z*xv%M1CP1rlJd)HIBIKqLv$0?`< zI<+)p8{DC1Q3g3_6Uq-IPBiQa%#2*Aj>Iq{T}gePPVSJ(gB3k13?2T95u%IZOfM+fJTq*w&sXp;boO?EpU0}=_G!T29N(9GXI z8Aer@=7lB2&`~`54W%nusvbfKVw@o`txr0nG~SVAIGaW@{g%kp+CI@_*%}WDvn+NG z2Bnwb{H)Q%h@7PwddVpE=NrmN;#&qML-e)Vd0vgZ8g}2eIDlp+vpEJoIm09Y#XA{- z!KW33!%VBTCmAwc6-Z}atd@`oQbF^Z@dut&W}Gv_-J0c2)ePUXULm8(+*(x2Z{`_z z%)d9AW)*HYrdxNvt#!JR0^6ffYSgv9y#&*JMa0WS8wY{Pu=3F&Le+%B;Uyq~aQS?+ zAQ?$#ECk6C)>sfQ>vrK(ep-7iGZmO-{i*G?dopGsc1kflp~hx2(ds7WJ*3SsxvbyL z&oej@P^t1~W>USeEAttIg#{A1Z;V zhEB=iJ74K5v+J2H7`bTAF*$~4xqz?M;&NcMVx}o(i~PnfFOk7Dnr9LitjtL+;EaLn zuOy-ub9bYCvz0{RLzhTC)7N z3Ia$Y-UTJa=t}~S^)5E<+gToZ?Hh<8QK0QGom*WuARUZz8`?$-8-z^6ju1#10Q>FH zfUwY1#^aXbreT&G@Yhm72YoiFIy0U6BZc^_FQExjgeU~X_wdES%!HC|OxzH61Zf!j z0hdtqJjf)S7+gJ=0xCIl{IfC>kV1$Z?WHerEpCkQv>42Tjet}XqtEn`K}a_y#K`c1 zdXA?9Hb6@_t!dKwzP|>%Z4z-@UhxEO;q~tEJ>UqIYi>@d8_6a{{da9b@rMh=SWkW+W*7nbarJ18YrwrR^surbR_T%n;{&nQ$da^L5ox(y&yQSR6j;pQHT1DRt`L%b!m& zbVyD~m5)<~^}mzC129aR#l)0?uw(^s1t!V`E)DA7er!8iHxW0s9j2cqb~0py<8gq> zoi0LUq%SZ`zz!`*-~h|^(gK{zB}r!nHuqxntIR(WLaAA!8L?-ws->EiGB+|7S_{r{ zR?@{SGl^j@QX~a9)lEGuB{5UZw$&>^r31o$5mSP@F^879{=9#C;M4$+sV<+%V}})m zEat|$$F%r)vkf5Ui0VJRWkeh)XAv&T@BjRMBm3yYQq0{hS`nHCzF$MYklsO+Im`IIoJG0mhhGfw~Gr z00@u@q?ytHjR_oLYoo%IS7X4eL zPD+U}xjre?Vr)`wAWq;|8|?CvCRDdAqg8`D5n_7Kn_4;cF;j6W0n|YhZB0oo*8q@4 z*Q^$YE>nt1iU0(H02cye2(gnFlW2+N8HvYmC4)T;LpH3%!8T&;TT$Z13oR>G?Hxl( z6LG0#9((piNtS@59U5TlK^p_4kVl3_;F4`IGmUQC{(`OGys3tdjw*-Wj zDItp&1W>h0TrsHHHm+F6M|aRfc%&Ff39zv+E(o`l=K{4YPV_sFU&i%om({BqejFt% zz*Id$Rf{!1EJ$V)cf4k_u8;&PVIl|sMWb;B1X#g0<%%-_r8oLT^zCK>n)By(J^bM_ z9Kbb9w^ao!1ing+l*M%pWGo~C zr%d(n1{LB2Js*hsYtLHXZ^U0>JZSotOO|nO4d?xpW|e4jFkYBTHQ6hP{ONHcO>3vS z46R{csrOlJ<#)9oTk|=Jk(gze-XzRUCl0I(!j&L|reO+48jyXSIh;sG6BR53B$5J0 z%#m&N#mEu(;6OyhohLDMKCz&N)s#=8U;XFZW z!?<@e{{U;_-XqJ4sZbRkn`k9!D>U!xOKDPETAc|>Qr@(vLLO4u)}=J4@;~A;v(n|} zI1JiIZ_MEiTQO<&k})_!R+)-YTmY##f~265%KV(9f=WS;v|ri3+ofVVBib&;VsQ)g znMP`y?<`Eiz^Qpga84Y}NSLCvNlNnJiJv(82*u+kAw??%BYC-)rPKqclK>EQx~AH% z2|L8h82|x~DJoi!rmZT~EmA`$wWtUqOs{(@^bksV#Gy%0 zDg+UIj{gAD2E_B2k$3|T6qJ&Mq>y=8_1qp;7aZ<=1=?D_Tq)D6mw8tSZ>o zvW~_CpUmS6AlKEYlBxg;g#cJFyly?B^q8be{GLq8Q-1NmP4zB7(#&plG0MwN67i*Z zs?2I?s+?Mh)Ztc?f~X;;$Au}9VgO0zCQdg$Ux3Rc`n)<&ir5efn^WPq4p_%DmxIkS z+)uB>O`klI{?RonBcPH+SZpdU8nh|M0r8J;GI`6?I|br zYDgOb3W#DsHLipbbZ{ugpk&>KWcta{lA;}pQJmB}Ykp2CbG>rj&#CCyNK*-rHqrqx zxfYSJu@(jfHEG$|n7XZd)moFK^zqiwz2Q4EDpEgkh9h8gy@;!Qxm_LMW*b73pc0g% zLI?>du>kBx{iK@#CSwrG%w|kx54lU90jGKHi>k?_p`9uO>>GU;eOS=ek&eBSGTelu zT$qqeq`}w;=kYU(sm0AuEL?_q9d_m$W{MNC*$EH8?MeXk@pis|hl_fs5QP{*L~KUa z_8%SN*Afg$)b|{n$HYcBV3MD8C%GW*Hi_&}Y)FW<-eYTA%*VWJViiqN?i)a8!@a6` zYZfKRt1;*WUyx&MO>NbZt%8=siwV35DbxTe*k1ejdE$IJh*_Z?HDcnt!;5G)F;5pO z*dT)3tT}`7*y*G~?BP#0+TR1X)}Z2elPfxfh>E0hfMk+3C(hXE9iAYnK)G;wQ3K}V zxW@|Z)iAiy1CPmY(!S6n@aoEDrPz(*?z6C*!Bsp9!wsoQQoq}1(z}}^AQ?MAkO0`9 zk?lEaDpa6o%$rt}E__JaTgQbvM3XNSGzyaD$5I%z#Q<@kjw}3ERJyL&^Y-16DRSJI zrn&^Vlrs8KIZB;%W&}>62cN8c8=(p8o(QM-=VH)b_*|iZH7@oCUysP|#O@|>^tGEy zn3bwt4<*Z$B}qvrQs(9J{aRC^jMnx7R0_Z$B99#C(#nznHWrIdIpyUs%hMb5qEp1$ zwfQ`G?8_5C3M6ovGivTD&>iWs!9tU|NduhPFKgTp<$sC9iJAzgBbzWk7K+nk@UoB; zQeKSY9W8L*Dw@NBh~+!{Q0@Tzq0(d-j#d@Uo~^`3+0{NgS%u=xGqc+SJ)_kfjJ;DBv=Pg${#uS_+F zxFfn56yl_aGq^Vgco#hW;}2N{n1g?$5ET&y{$Fw+h@V&^d^I4rElQsc*ULc<)J3dq z&hzK%>$QQ~6G08Z1Py$>{{UQ?dVvHWUY&mZ}(|?zt0p;g^()4I}P##w@PhIvOF=!{(1UxB*wR)RBCKh{*9u%kx z@9B8^iHmGcED++Pg6{9l*i-C2Fz=QKcf+qZtPn!Y=EDBN>w*YyJ6u>=d3wm_=qvx7*0iF&Bb9#jrsJN~Xtut_UIcAc}!JC&EtmgML7r`3|^*t<+d3v0%aev1U*SmJ(7+hfr(rRgW+W7uo`+hSSz% zC!Be2Z!moDoJrr{e$e$T?Wix`_4q_r9Rftn`COF=>F>OG;oaa3$pCo61CqcJPbWWk zg0&SN5h()TSU^5{Z?tVYdCP>vfku5?yZmBlQh3U^$wM)dO*DePjOsPCV9~ z93$FY2~vC3^Q?-gKoAPsc8k1ggh1lc~QC8*$6T>v}i!sb4x_8M}fP4$Ic z<%3gO7B0=vsB7~H+H*N}bD8D?!s1r@x|<;~7NaS*lu0va1kN~bXtJC}3Td4rsCQL| zF>8X)sQw^d+?|N+78^3cdWqZ)!i7l&!mUF>Lk(Kgc;sFy?CZbp9-%$Uz?~%of?`PU z#mwG)XUi9X&Zc4J33Gua&1a>H(xt7Y*Nu-h?A{X^fin)Wm-kxcx;OxUX+zYSMxr&m zN5Yi_73{p_J7zoff|MXA0HPvfjpK79av0guKLMFb;YpV){{SkKfc0ZSU(Ep(jUV!v zwn;OIK+Ye`lqovcVY3d; z&fp7{go>~Nrq#B)J$cXw*|J{LOG>yWOj1QCv%*a|nu9|?S@ff9nwMxZnrTX#C141V zaz~0J+Ub@ue$P z8Uv2|HKWW@>!@Wqm8{BBAQPzcurdVPA3=SKD$gfj)xep9Ku-_{o4F4Ay#ujx8oqVe ztTtkRu^}7S3OgHVwYRSmM>ac`=iCch-ct9jP}LZ&q)6G}5x*gbV~9B(u1_B$b-XJy=KmHbFjRO4FGfsVR>b`igLBaPY2 z*lA5ZHH{%?DgneO0jVlZqM)%Rck6s>dA?E*LeL75-v0mz;5VT*(3|ZWXJ>Q&09=j_ zSfnoe3M{8L_DKq>iwX){2;9{9hkDphT#_v^KqVt?gmVL#KKgv@FRN1kdb3|j+Sa{^ zt-6jI8|x8L!A<~hl3Bx0){pgzHfs9R%F2{r0#4SHkRW(}SD5L_aFS4-D>j|tBc(!An|KL92*ud zzJiuc7IafCp}WDTSur|6BuRm5Dr`0)d1C%(4B2w2yV2Q+?!QadXu#x0)KA7TgB)2P zro}spIK5~K);6s@bfE`Ph=3s5V5r*H5zw20Hr1CT15;b^I~J`?>vyGCF$`mWk%&AF zD&s?9D8!8@`^C+=&7neMSdvL39f>;_l5cPf3{{yVf?X!}NFZxM2=u0;6LsZbX9e9i*lL7I@wJaq-`uEwGX2C4r_zYr>8T)&NJj;aGVmv!W5*; znZ)HNPY^6>sea{1QFC9;2c&+%XMT>dX?=mrvayxs*}h_NINUxBb0=XjxGc3O7<@PP zguvdqVp$TgX;=4)OHZl8l9Ytcs&U>w?Kz5Og50HIr4_4Mm8mFbDaPr76cnVTu!kH; zY7$yoK?ZaPu_SfJm(YeIqb8ghk zlujCQ(}T;&mZ2##F{N3iZ;X>Qyxv5WQ{{Z3UDYC{{ zZ2%M?0U#wLfC0CbkeHD>#7;bJF_R`u^@Op#X{TCsyOQ;fu?ovmFAu@$(nINOdAG;D zj~ed2jp|qRBmOiKHUpW|0Rmv$?=la^V8dd9(g6XOk~TE3Hw2qC9`rG)<1ov^%%D6i zQ(J}|J~m?)+>1VXjnoGTgsefZ5y;v?l0Y|&q!469TxJej#E`)51$QnlvAd8(3V{YL zW%-ocGQvqEBE%PF*7j{a8bJUjfU==tOikoT+?c*F z>Xg?7Jwf{Y?Hd9o@i}zWnc1{7W)HX`-SA;l6f}ntVM&l7jaEgM|cba^JAwsi0wi(DEFx;gQ!eMHU!1B!i2KmakJKt3$*=2fT-O0CzbT zC94^PsIagOo3j8aTATinf$_~|jFT~0AQMLT21qGNSzO5kM!>4#HzWcvkji9}ijn|# zAs{3YN|i*Wfyp%qNCc4VDHtx$O%igfY|bDE!WVJ_aHZD!aEDRZEK5h696a+)@McQ; zdA?gx?k^I;t5Hc#u=P7jeP2xQ1BU^E zk|s|=QlIv7hFGSbwkYXBs#kh`Z8N#PyEe_SNk$O2GN7@S zpjvOw{{V=kO*t+KROL@t{N@?IF)~co;lX0~ihWDD4mhh$B)d3&dH(=s^t2(wQ z@KS*6-YF{lw5jq+RIKMJQc$rx4iyxI6sF}VDijgkwH08nq^U_iq!lX3gE6SYfx7omZ^$z=BBNGf%$x*(u}wT9ZCd}R7}i)&;bO<9dkdO$7-)rKO*~Vir7jyFm zqtkbcwi6Bl!5Z?RYq+%oxYP!rV;Ri_IBX3<1VT(fk+3pA5Rz;*kMT5=O-ngC4>S>!3Me4a z#|W*Mg7u{-&8yNb0;NC=CLk&jWP$*hlO}oty}YpzL0KiJbT0dUNVDq;yN4yNA4cVU z+33+8wxBym@(J**4z{G5PPC08(Kh5Zl6uV8Qd2Tg*n%z@h;z=xsasMhV-@S<&Xz+x zjr>?#lY41#Up9Xde09Q{QIXMbzI|O^m+?mk=M;IKPhP`qBC3|REU2nQGO4OR+NvB) zGTNwHt~AY4E!3_TQWE17uOsOnXFVun+23R`LzpKK{pEwg;br3HNC4s&i$t+0Q^e3r zrEyOJYW-k52pmZuD46``_Sf)*+s{Y7nmcEW?6jKhKI1yf(=%EZSyy7{O2Kg|biAc3 zm8hvpaFuD2hR=bplBKD9lhR&sn0AUpqdSC1!{Xjtx_hCA5C*km`$FKp(e0nZFJN*{ zMfjXltO8XtHOwWH=DD1}c&3GM<3BAXSpb|koVBGWKqyLyNM&2Ms53)RPTHg>6c5x% zNF)PzB;3r#;DSs+$DF6(xKz#XCiU;9iESh376%+=pfSF>7dlj){(V~T)qTXR1bGlI zaS;)*vE&ap2Nf5Gd(e@4Qk_PgUNbUrp(J|36n7=tSKpwk5bi4FUTzZFg4AS@{XoP{ zqkSqC3FXYj_+l_gDxfEjbGh1#EY@vn4qjwJO15m^@O=(ut)15<@(zE>LT&WY6>qs{h+>@=# z)wSqjf;&m#ANTGcMIU{xl7W5W=42`3B?&kFgm|D2)Zhk6XB~ei!hTB#89%o!IJ%pP z7aKD;G3KoM9IV?nQkF@>Vd?{t3An9A?t7xvt|iSXDASZ>#dP$VLKYh(D|%onQ3UBp zu_;WIb2kGGRuV+Dne%v5ZYUTAtcgB z=qxqiE*;+C_n@xFmukv;Er!CSl_?1S0GW`+^(xR-q`NsTYa2zjGJLEC!9#@t$!$86 zpluR>U=kz{MD^)~QJ*zl(ztBgUd!BTPyu$cix#NaQ7;RJPYk(k2B{oHsi?4^dvl1D zHO{#yqoY!6PPNF@Cgu!op1-~)l`cwOiDdV?+na9@&Q#tX*~kgqJPH+K{{RRC&;mRO z1ht9!EXJg%#|nZ*r6dT05+Hx=+mW@P12$#PlmPW<06iFnchkS)80naK%Pydx*D3^y zwFv+=)jdnbF;zU8=-8S?PPwUFlqO=?gXrI?*KZ z8hG8g#knk^f^2bgz$ynqhk zFErGW2ZwLfXzSM7gKdw#AXMW)Le*f*FI{UF5rs&w03KW~;q@BCOwOGMX;%WoPTau< zqzR4h7`K61mZmBWTZcyuTnh&+-aZ(7)hI-=RKtZXbT=B3%>PUgk-6tJ)vlwnQNgxuCNa6v-4GB6~9}vUxxhYr*a&kjJODDl9{n`BS0MIt*6Q&ao8|NfgA?a zy$gUFVNvv}2=Qib`)QNr9Ss{tVs~)n``VGBzZzUMQI&R^mT?G5lS9OL23b%ZBmOm2 zWovUUsB9>BLYZYOx}h=>@>HR@KArSam?IIAVsJ&cu$Y512T3TFwDEB$RHX!@4l)2G zObfJpxcpvra&}$Ye4Dh11vosz2Zcr;6s#dfC!hHx0r-?}b6C8+# z3Sz`Sl2x>bzxns_>Ger0#m~F0z8|c4M1qhMK}cr=h72m;nh-AbiD+=>0B(6eAl`N} z_HBh@z|3D}t@eindo8a=)!@ZNLQnmqfI7vxm?kVE>n}+uPgJoQ+vf>3P@0fQAb?0C zR~92s?04r=6)67z*Uyy58%3;7DcUajMT&=eBx|{Y z@~Hl0d_K^!cb_~qEkKmG*p8&w;x(>LJvk4px%qYT#Eb=lK@9H)pUaoN2qC)9LG%7_K?t%Wo?qJp5G5OkB5p6Yocub^ z#{>{BzrVjs5Jg~uD+CaP+kYt$Z?kVNE$~BBXlmLxt9iFrX(sS(2h-Td=We5wutP~M zSiK49OM*NB($L$<8l;)F)6`#>n~&+4-f%-I=EVHpZ!R{WW}ic77apsXQV*#-+RG4=i8Rpg^uIIecH?o`MSpQe7Rcj{{Y-NVX#2~A}?{wa}q@Q7(S!O;Ye>g!39d^cpoXk zK@Y(MP>V!F&7=HaVX%T6AtFSQI*|r<>9pJpt-csi)fW~TS-SqPLZNYDY<|Cg3-vf^ zRMa;4euvHAf`M@)$34G~zg#tpFQR*XUh&@u<#J(8xIIudO)_^;&}^Ld5%&6 z!iE|T53@vbB&J{hxMEqGR)3K2ijIJuwy^ol{{U<}DMsoY;19f0og~XC;o?v(^mZYx z+r25Fgd_k-`{sUAiDCoi&qyQ<09=!K0Fkjgy?`9%M%aVaf~NsQTH1N9!Y=xs*aWw> zeZTWHwzso(dcy+BHEKyWfD#I}nVVYvxOpK=#0K7%+WSIE3X#RelCCyyRydo5-QAns z4ZUecG^Nev!;mw6zP&cYZViYrY>{`MyH}(}QqHNc3IreI?kp{6iNi=`K7=u9)N8jd z7>KC3YCSrSkHR6kzY!fs0y)~&iNEpgtRIWbfcZa+Hui|CQ3^6lNsXrcE;hKlUQMrv z%AFu;ZZ>ZryW-j<8FP>Fr`;v1#29j*Dh0@>2F4+}?vN*$*jjm?hc1)T42g*(xsKfY z3USfMyF=nG5?ip2EaCcu(W6)z^GuCnFbDwnip4Uk+kiD|@SCV4a<5ETVa4`y4UR$& z9$Se-)gBUBRxjBrCq#h6sD|OL<~?WlJ?%0wk4PB)&P68c8KE*gVrmPqaMTqpXhja9n_50(`$Xaj+^asUaYbdd zRILsq+yIiTWCJ!J8>UU+LgN~)YnI8zN+DvbSQ;C)x)vJ?M{exSdyC6f3RX&MP;Sg_ zXnbAwu*WF)u9~?C3Q|BMl0--(bG66)yroeSY;v6Avx}23Oh{0Amr3wqC|DXlfN1m& zXS--ID3dO431)XtEc7k|w#Mc44GCXEm*%KaT@5Kjf7+cY3b>mML<^rjaxtrVyjBit zf}yhp4d+HCH*tu;$>b<{3On4^HCA!iMHN$p{JHliUL>e3l@tWtOr(t@ zCOj!AA}xGnu=)NY4xd(DXx?VpfK^K{*}2~jitNq@6s{zdNZ-|iQ`{JLHG7)Q&RNI& zDzOeVpHQHkD@LYj(1h4aN+qS3wxqb3Bn=XiHfLrzMmz-1lBqKQ{o+9arlIH_yOBf8 z#T+hEn&oqNu-LE9%hXO+sh<3nT2OKZ`@Ywzem6tu6#0V=(mfYNLrQI7LuB~CnVFQB zw&xTkWr%(}GSe{Cgt8WC{{Sf`l{IQcdk>vV;S%u@mn;wm?E;cCVeK#s&u6ami!OPW zWx2ynBzH~+H3WeNNrFTUTgU(z5q@Fv7m30bJV{9eIY|SkaMa}e#KW-?uz2*%mO@nF zhYEpKmSearL(6#QoYy4GxKA<}hFVq|3zr#5ggBItOrkf|nT!fzyXGA9^J zOPPEn$qX%U<fI^L9w_L{VmAyOx6N!*YP$vsTPtvJ9= zmOvaJ>ioW^t?73nHn|gM{?CYQeplA@1}-S83MnN@fMnhYN=USWCw<7Uh@4j_AuKh0 zz#YU;Ir+uJ&IK;^ElQE*;O+~CtUcF~`)X-Yq__)e20Te4NlBaBZ($^z#Na_tB}n4p z6=LG9%~tQxgXsx1NKz&tL=j*Bcdnb7IQl$s?`S=YXDnv5xt9i3J3_{0=XqU3f)z_p zxKT>ore3C8S`xC9DQH@f6r_mMtzCDr-jQ+nRDmgYmB<$b3n^<5UX`idwL8a|JtcYz z?Ee64)Ul4rusG}_(Zh(t<7La95}LS^z?DuRj)jN}1Z>{KyDq8W!&94QPO7Umu25UA zx&oB|At_DAgbg4iBGI5)^Q?BmwNtbC<#VRYm>jtWh#S?xs|Rv+aATnQFZ64db{jAJ zt_uZ+!pTTsDpFFO5G=?@yJ*1Jv&TtQH?2AXPLLr0-T*)Y+}O-S&s*rpSZ5>ytCqPs z6KyM{4VZQ_D~XU=a|DoWje<}ULcO3Y z>>L-UBB7|qzQ8z*LS&`LQ&bHNU#86A!&62Ev6tjI>1k;NMIsaWNFsUa1I}lZa=hak zjlu*Qv}7!Y;oOZU<;pPBWI#BA+vpaA&9XKTXh<}%pJTiM$$*0t{$8@7D}th zyh;5$VXDMAdy{j2v_x*W7hWWZB|b70a1^vcN{ALRY=A)|Tt+8cuVbqeunf%ShQ8AZ z9!^HF0%wCZceIksOxQq{8t7+nXSbzSl1-m{25}w-TA$F);Lcg{{VM$g?MnX zuFpOdif?>RnQ(QtD}HR{B&`^nVP5 zs{?~itHlBm%KrcrCYp!#oLrE??BYK;xYeZQxrD(p2uaLJ!v6pvJmNS2OOZIGG8Ts6 z24@p7SF~e4{4%{vp4s@R6s_cFWuDLSrqrGu&hsuJ%ZrCFXH$x;&1$NvMRX_s065uo zEhWTjY!bD&=bXO~CP~Y3!6jHz{xGn^CSnL|8GmMFX?FNM{)2u3Q8adswKFGvWhyyD zw-~wdW}4#gd@N+W6i~LikggZ3dJ9ohRMo(MkeLHgjIZ>;Iuu2K*C5Z4$CD8++W-)! zE&~l7{M$=AN7Y5mX>L#3O}BCJh?Vasu>`tO$lLP zTU_xtbMS~~D3nyOG_5(EwrIkto7ANOQ~;TXzSK3dei zT(By^=-fx1nm(gmad@O`NHCyGN>yV74a|)<00@}|@e_#Dt^rMTA)3O}>eV#nBJ_z| zHD)$QVPjYLotQp`GJ17@kbLGWpHpyTNf(*qAQ2eJwc={p)!H(W?5^)T`gAdKZlt6D zM#pZEWgcU5{{VOCg&vrT)vIyi`bD%%azdCN<_7FJ7JbOjkiw=4s@DK1v=IV2+j2-G zo^T1Y=^z|D%S{-P`#0JC1CzojXh{HjMeGzGeF5%nc8g_AX$V|S)dCLi*vASB+c+~tQx@4YyNW_J}l+-xJUa!k!greqZv2?>&{3C;>rGvbIIB>}bUbz4Y?B0&I) zlZqdc3aJ2cC7#*=PVnz>8IRAXe-gE$r~cjVNVQkzqq|hB0|W?Byn+Dn6aN5dB!WDS zEXcv6Dxu5JTv4v!?nOCsia4ywVo>~JOAvJ*7XXi|v~$O49AugbDwhk1kcA{9fEB7s zX-G`&rH}w7b|4#(x#U@5rp%N`R|-L>3^q5=X!QqD1af}Yc7V!}IU#AdVZ@#yY`{B^ zEWj5ffFxIwXx*vJt1Okg{Y#~42oj|&WT{(pk~u&l$YXwJraR|k`x}UuWlWzW5Z=)!ASQyZR;U_{?9JL7 zH^C^&DVj?(?X;8xfa}pRV3L%DK)50|-Ah`}AXUo3*B@)Qa?7%$;blr8 z2$#atQzQ#9Ahmoey`wWZW?aArFs~~Bf70&~$^-%kN)n+U-*8AXzbYwSDrhAVK&F6@ z#1ks#Pe z<#H{1lCq3s!FhqJ?4eB5015{bI&Lf@7m4j2T`AsTQp5O`kj?`*Q3tzSE+Z1HJ=hQm}yQYb!sbjYOF-1w7yYNS(kuv0&vGnMmSLCw2G7}IZ{{Sfp z6y%$>fL~5~EQ~s4$>Ug3NYOxtAXeer@)R^^ko7w>t%4Mp)2?JdfNVAvnHMoPI7BIH zRUDSl&br>4^n!SmAqR-C)yNJ0oi68YQLOl=3rybiNz^4l=fGc)j;DW-GYfuhP9wqJmdpk0k5QT;X-P~;B zM>VLX?hJh>{vWu&Wox6SdBlQ*G?gVvB0MCJOp|$=%xNcmxvoXg6Dp6<1TC9r{IzS}Dx z-Z%cm_L*Is&M>poO0am6aJczT??_z=h!)}Zkg5j&6LB7V^xNppU!;!5V{+u8T)f{a zRGiBo#FlktI50wR^N1W@mCO=9DT`0>OIqg=#4bFGsA=jtw`nSyQ*{iure0-eQ*N~4 z6cV=9(tuc62?<&jok37Hwmn}abgV>)c!?z|MAV6r<|L(Pn1w$A(6o{il{d1ak_}kL z$h>X}c3qfG%Q1*toNgL-sKnuBAyV-&CnZTjHx4A5Mh+-y?>(-1wMlozIDZg%Rh<@@>!KY$PN$=gi=O6a#sffgYw{ zl6if|=JcV(TptgZf)m&eKnKPJ@hJe@1J(#lL!5ykY)4(K>*az7V#qNDcjxJX2oWHE z9Q{2{ED%B!fxWFGm_D6HO}(%|11kRjCwYN;T1O+(-bf+1Ad0~RRtO;p`Wc)0Pp?jR zA*9?=#{Ixscl$$T$7$c7={v~>mm_VkLmq|Nu2@=>b6s3BT!6*lIn9J^)1>o*=6_9r zoDiM>X|d7kyK71Z=F?y!pIG1MM)5Zs@Ip1OUPqO;8*A`{lW3cM(tcBd6VIO*piaY| z-_Q8yFhS?fo{*Ta2cHO2Cv!giexJ8E#dspUrrf%FbM^PZ1S}47u!*0Qji4BxKTtx` z;qiX`Izm9b`Tqbun|fe`ijmW=_V{05FhS?f#uJgWj$`BN%5MfRLV!&GBGbSwF%k2Kd23~EvzTZ9f%WtvU&3Ikpl!Te?zruTT}GhG&v`IqhlugfImsS`CoDz zlHDi?qtbx3)_d~W9XntAee?Fg4|`NsB&3Ul3yxg5?rweul;RQqu?M6O zUi8iM@_;0teWT&)hw-V|gY3oraQ-u}`2FE~!{zWJ_rh_jVc>r7ah-?9?+e~}eE$Hp z7uIt4{*b=2m&f#i0-lL6ex!Ou$F}nZaUfgSPti~Hg!NLtkuEt`?*mG?5^wrJJvRb< ztT7q^4hUnhrJejCP+EogKPoW$Lp3X}{{XCikLL#p{{WlfVUBk`CIq)M3mKkVyten= zV+sQnNd@lpX6W0(L}fatCVZmDc!BiV=io5#{a^r+z&SJkT%BCK^;gr6AND_BbdQuA zFj7IfYvc%tfe47SbazRN9L*R=cS=huAgwgg-67qgg1+y4zTb2Ia{mR}IeYEw?0P<~ z$5kR^0xEwZrz_C*3jiiI^&8+y8-zb&?1`L`?Bue*sk2$Z_)A|frH>A)Qx6n}k(hD^2$@melrfB~NIm#d3!N5)X? ze2YUSHg0&s)fF{Lxf5J76m4dlBT-QSDKXr}XPpBWMo0GMMwZ=TZ!{oNiE@yZieI1v z9|R}BzU2rw=|Sx8mIoQ1vhH`r>WfButKgtlz5zuwJXBARe_%k(e&fG51r_iY5z zo7cbAw)fv^O;FF;UmstX5q4F+ii3=k&sS-OyW$L_knO%je1r!?nlZSnpB;R^_ik#| z7zipS!&b=2aI!FPmAUfX1w3p`BL#3J=z6wAqmIYf7gp`vzazb_aru5 z3am{I-;PjV_YHR3ZnN|>xEy8qU?W&ci@h%orsV2Qy<|6OI}PT5am$Kkqs=L?uQ5?7Kl(z$K>j4$0kEkbQQ~W;x@Ge;7IHT z=wV|MD?nxVP;1a?loWc7pHV&Tct!IgK-)71wI|0rA03m=^eSJV?_dMKERLn5C@vA{ zZEcyhe;CgRF`4qQ{VpVMd*L2YXFZT(8|WHD5Sm@dWs5GW7J?`xhNil^9*Fmk%G@rHufg!8$2b^XIk9knN(ZP}kL?Is~Q+$ttFR|B`F zvHJ}1)$tfI74K~Z|BGJ{aHTlSaAE9l7G7-Nk}(zC@b}n3t$gk|%oJC(vwYO|p~Y`g_?yRvw5V&M3doR^6jJCCJb zDtw2F5V54ire&BoIL$j?2?@{<%6c;PUMM@cT7a;QUQ&8JL5ly$;|!)LZwZws_TFDf zlsaKpY|h&bl(&C_nyY(J{G1`jqnxt35)C>wz>|H-YleA8-wYKalNPCXCC7WT4~g_T za3_|grDzwG9rlFn8+fPacvYgTRkb`Ko2it`>vz8tR~KsV!gZrfU(}NpAu%^LxI~&2 zW71Pl=7$Iv81#pcVv&S?KEjLr$(sApIcBTVBU?=Q*BH)O46VvXUqVvsF_wNlH*@VVp^%a5J1~EMl z9k61Hg78a42`ti=l4%xi+|)2lYtw?*e8m~0MPQokVxc#<@sIpIF*~DW%;G943#8cg z6Yxfak&+6`_dUpI#{}FlEg5Xa($>vi`<4?34hAzjPECG+lKRdRDb-qEqby;B*u!wC zC5#H#L(>Q6P0eA(b69%Zd5CTyo$N02M7AaZ5wjbdPGO0Z(~yQHE0TGQ7;E<7Z!ooR zzt~tOnBU3dGqvL?Vrc!KftNB(mQw4~wSW>!UGLRDfOv_tgs0m2u+P#jd3gKMk2&n(6Tu)dsGLZd|l#oe(go4OHDhOYdSw zz^&esq7I&RSUo9D%H^Nn!s!#TbTK2Ve-kt3>YtRNn-dLT))(q%L}3=i=s6*SNKQ=n zno26Oymb2H2cez7UuCs@ZU%Dj)F%s5k%*brU|EBV!Eo#@gYI%cf`~5-$4jZWQ7VB5 zHv(`@(sD0#IqRF9x4Q5ziDIlP*ndRgPy8ybSaK?8e-jaJ3J9AFqx8(#fa~EaKJ>O51B<*EP`>mwjnnv)^&gyhP zva$bKLOa33B7s0e*H_@ygYh^co~({8G3ND{zzz*>*M=)sLp(B;*Y+|A)o_>$!_o-* zSlCO9fhju>12}Hi-?0#8(Z}51%elA+54+T!T=jn^`PG_(e#)vR*&$P}; zHtA>S8_Y|fWprMtjMC^Go4fzej(i47-Fy!@>Ju)~17VUdaniAyv-r$r1xi8fLG;Hk z2kvb&m7l%OC-oNPC^TXxqW%4LPm{0QN6&8fja@Tp3g1>VEYCT1c7Va)_o7jz!is45 zi{a?!U-TCBj6P~3Ulk}&s9E*xM|jUGJemzr)CxBqZ|jf>6d9sBbU|iIl3@V~)8xI1 zO5UNf6B$p*AtZ(T8p2l}n;bQhc|&-)Ocy}9L<4euYYhZ6$2J!{= z;cE!0gJF%>zpnc^5zYNu?@5#!ORE-&en^OVvg*B(Mf%pMnm-Ub&cf6;UzZjh#WA43 z-?A}|nXUq5H#bdC1a$7CqTR`4g+_}{AV?x*xIIC+a?lT7pE36qxP)RCunF@OCpCxG zykNc!B5I1?**_ZZep(}5XP{02sT!f`WL(l&sS5zJDA9;V+Lg^s8%AtGzt^kF7q1nM zPgAWM%%76~Fx?r;;dYFRi=*X&Fb;kb@ij&cpi>)%9=h)DU9S9;mW03*@7}Z5-?}<) zzTu@aI#z7}vI-k=I06S(Af`XtcqY>O^!`q&uA26ICc>9Un5jq&kswn%i2fb2%hE7TK+Ng4;p!^{(gtvb>wh6iy-L^eNsBm@ivKLuQ`(qBI~)7JbRr z-|bs5PfJ}F92~2$MCrJl%XQqEkX{JQB#-8e%sYLALLvWcDCWqNPT*&Z}!cP_%Enu6(#UXl~Bzj6wNZ~mT z0&tQ46#`F=&RI_fiNB6|A2EPCPVz#X!mYF?GK_x5q=zrmJw=Db6sZ->V> z1f86Ds@}}UmAc@Z6Vrz!0Qvz+RV#skmlbq*>8E83@~2EBLnJ7HOF=8*LaLhzL2;Xfy!12XPMC_VQ}GqaONfzfG8IP~x-#5;b7 zCovAOBB#p*3fZkjqnTlv>D4@NBsd^x>z7LTQ;lq*O-6I#CPLWm*J|E|OA6LtIi~^7 zS2Ln!yE8!@e~nC@)tTbrKX1kt_?lVNy=&aLqTu+bwv(ax-Kw3ah+;Pe+$$Xi@CHK6 zdN+Q%pPx@{skip)Oy(!d&Bt_Iz!qNS{`@LoFvgb-eHqQ|bq;N-SN88BzwWMD$o3X| zhXp6lgoXPxMp{b~AQM?Po&bwAcq{?cFp_6XFISyco<0Si84llNC+^(u>;#`eNlpWO z6Qa3suhzX^H69+zpk8bj{v z>ZHAK>)PJ_jbxkTppuOAW#KB19*B`f;mE1#td2<1v2CL{kE6u*Rh&5I-{)VM0!&^; zNXK!%A21uX>v;6s&NNK@6Af5gk`R2dv^FD6hQl@b(UB zpcerkXH|YehSuepZ)6o>+#o-sh9+i2ol}b;oDi;!pb5@SXJI6Z@%*&^E^AQRpbjf3 zPi)NzuQ*zfL)~F-z3es?F~IJiy+=b~95k3g*^H|YfLD>L7Cj-^y0+5@*oibf(8w-$ zmCK^|*Z<)h!dV&O2r7c3iS}atNOk%C&-P%^<{E-6h%a~py7s98Q5{|1eZpi zE^wwAg(t`Rqf#n6X-w#eADgx98o=;Kr?JGNnc$89=2ej)67>w9Km#0M{5$$r$EjIu zZtD|&$IAep?X>*bP%Ah^i9*-qNAkn?IdtLbbAIjR>zN986B0_ei!J3Vzyll+#Or-* z-wS25sj|om6qIj~PD(CKXMoP^bn=CiW&KX6popDIMk7dQ18c+XSwH(8&XV#P#mlZ` zu_hr1xVS$B(F!hTn;$^P92Nj4iaBBbnk z-?isw4|A^%2S!Kvy=JHd$2#O^xHR-o0#EQ9V7e3W=Olnl*0cf!Fy*?r`wu|d!vw-@ z=)8fk4QU*Uccp!K|6cnU1;2d}CghRCjF-oM!E})cagMGYmi|Y}4e^+gE68;xeobS^ z0>@Bs{jZGu|HE>Fr_#Eue<6K+1X>DxckO*+s$t1RSeFR@^X|rpOP`w#D z7XdCyuM=VN83EE$@Rm z|G7Dty)f2t5A_cIdp3ICucQ?6CGu}0lt}(H)hv)w{qTrxg9yYQ)o`JMtP%`Gs%;xH5x2+RFi zc=7jAX-?>ZVpB-rtJZ&j#Q!0BQPC&`%Xg)f(6fDvn(B*r(O)*DxG(%|I7g0mg#`py zu2>acB=xFq9DYvdfB^^~H z$~rMAN8oM!IqQu3<$7|6U?J`dZR@oAWl$>vl*Vw8pHeI;*E@YMfn0z-eqPvd)6HFS z?(I>uwF}`nf>nfZbL?W35J*Xsz}y*#SqQ52QFwj`r1-&%-)o7W-gv}RLE1N;2|0S~ z>L&F}tHL(l53F2;xrjua)r59S^^v7lWeE$YP`bzJg#NN{{FU`DHvmo$pvU#|{^;;( zW62!yVY}->w}qm}8-JVf0_R~n0OX=^!Ro*sKRjLRrM>nr@^2l`RRlLslU>9A-FN@5 zAXHEI3kA3HA7XC>uTUiocp>yMOx_X5zD_4yKy!##VRWj=LxtaJ5<({vp!&E;zmM|%@|nQ`eJDgsS!&)jBsC8kO9Mh-?WeMvavL0-S5c!iBR@M))JRDhWjR zWkhgIYDmrv3?xUlT!PnU@ z0}>@9&+R@-W&Zpu)?Z3lXy0LCAlTfkzdQEPvCLX9KwAo~GN?12IJes*;vQ?ZImWBV zi7iMztc=bcR?VswXlu_^ju$N6aro9fqm!X_j7OT3EO{XC3Q)A?;D7AExH}^^zdVx8 zsqe_kt%PDeb`#a7y%Y;WC$FdLYvmh_Qlr_6FEdE{i1mauH~)`Yhwvxdcmar@0sdm_pcP~DHRFbBKN+BA#m*SQ*bswCY<-Dsr3 zAdE>-k#u>Y8eAhLKNMh%*^lzq9iRWD+`DGzvh>v*wLVL1osxG#5E>$E~Uk4 znJx3NLCU;KBR#Jge57I@$e(nziw|2-{SN7W^@RGa9KwYTFKAuE=tOyrd%vf8mZ}{cNbGrg&ZB_E2a|}5cvS5G z!Ny13^L`9+yA9}Ei1>m`P&?SpTkjf1WKTDiu9uCp-!5H)kyOFPzjvu}wn?g+QSY~( zxxilHstSlP3U$;a#tQ2^C1z7Wyg+S#qoMBpRPfr~RUW{?1~fhcQ@QD%V>ZmL8eX6W zjke|z=BASq|N54>dD%x}zmb+O@yMhr|* z48Y(aEyIJ+l3D2qMw73BJxvbJ&WX>J70h!a<9s#Az{wsk+Rjc2;2;c~wZiUE zlAr+y-yZ=I#jm-{ACkPL4(`QF69XIZt^E^4#B4#W5U9k<65Hd}A_6XGxI~)E+r}H1 zbq-33F36h(DE3U-@(MoliMpF*lWSvbY~y=fGoN)vZ9>%X%Hz(#AO(yFxv^ztV>2*L zrpUHFK%rK2(s;biBD!YI6fFs4SnsuZ;XoSRF6`5;zu2ks1DLP4EDV9NfVuTFjzn5E zX&2BhG=Tmx%!+>EV)iHWUE*wBulhe~%$)D7lp%=Ma@_5fJJoz>qXPDYc7O;2K3tR)+t*n=OTZs8~{A^070?X zxWK6Z$W|_o$B2h7n68;32ys?t&XyMEs{>e=vw5{9-oMTRO|n@j4^+p$;=2P!?LUbr zY1B^BJ}9S$q*1adlud!j#LdTHd=uBxTZNNbRTyA(7i-5p2=HtkJkwP2M{_8s;HzJb zqb!>gmsdGYCX)i{LmuV_ZLI?+p2FlcZ|?FRfb8<8S>jaE0xp@(87Jo}lDC2C2OmsH z%h_Y&3;hLN$`SJmejlyLmn!tTHY%U(;hm{`cbVF8}lN_zHbn>=4f`M#*D@ctw!&-Oe4eJOvmNhe==jEDSpx15dALb&w ziuqSYlLT>qlBUVONws}`jepV+FQMl8>r6UZMb zy=Q}eTgAYqu5it!QJVx=+G>S(5B)UA_BW3WH^AHO%Sl|?b6l|;m=4@2oNq*cHN-P%Xo7c3HjZO|YJx~Uz`JrO|1QdK z{~Yg37i7`dDo=Zy*T*S=5l?Fn?E$2<1CnA3N+epUGfQ=S77%v_7&rTyow9Mwhz^Y9 zSxdf_)o;00tqEyof&xV+1;xXmf>z*kU>`8*E1P#8od^dA-up-j|Yz zIy}U+{#D+GFI=5k6OPT;{-)IAU4Rx#U zz+Don{CEXFC&5cMK2g8Qlt8X6s_N^U@pkTNvDVRx&G}mKLWa8$lQfP&SSLEEY>mWT z2voTCP0Y%uu`YTJ)eT&lZl!1JlqsNU z%`cr`Bu8(fk(#2W@S=2e`Dwfy1Dn#J8=ljPM9GXjL%VqzTUjXIJ2!q?9t|9QEXX;4 zkmWhKK63Ezw>c>%y$KA=ScK8$|#}->Q&s{HeF@47{x=nxJe_+G^zjm)HHtSK&8!fr~h4iqwV&*O~dC-o6sLk3w{v; zowjIrLqE4c{wg4~5Y**FL; z2H_|%vVH!VB(?A1hs>Yf`g}8`=Px;?h|b^%u!bl~aA^oyLidv*36~NLFGA|rmPcWq znVmaUGMv?-v;UZkmOY)g5pt= z3=Wr2+g30!-|Ii~MDd@A?jsB^-@n)Sb{koTRhO2tLRR%XXzSPJT|*qkc5kZff9K3a zVfPcP@nir|^-;f~izX7W!Rbkrq^Vni8HSC$b88oB!8cs3)x`&a@7=v?dy&>oC>-o# z_6Y-hEr8Rx$(R~X*Gx}S(6NQ5B?_i*NV&)D2pq|RT(1fv9Z-eKTI$}CKbZ3~~r>)WtE%-dq$w&Am)V1T!BHCJL z1~?aN6W7sdmyX{}obbkBQ1S^%tCb?dR z(haQ$-3Pros3vuHud1T3=49(fBsSbK!Rz#2+~rb~U1!wQpx{W~6U}D5b)7>s7C*@M7S@Wl2E8RNmmxH zj#Ul-DbBAfydSp^PpaSOVm6`qbPcW~!7=Hqa*vHrQoe|$0zdIT8ht>~`ga_K{0H)X zKcy<$=3h8qP#sLnA_%2H;Mm-?hd@>?r8lf+FOColw|+LSbMq&`y79`g=U|$P)|jX@hgAd$%W7qP;4_@kUHqya2x(7>fcKQ-4ju# zQqYr@!9fxV|5jxuhSs&#$dzH96^%4!!lk4Vu=l>u!Y?C?dL4_7vllzB7SLec9y3we zkl72}5I?ZVDzJMw`}pM{kt0U5yMAYV{(KQuV>%7R3C+Dtk%{}geXfv-nIdF!coN~E ze8c|tN+msbii5*><$G9zwkpBJn;?lX(D_Cp|6$H7x^FN%;?zmQ%1ktim(MBu)Pu-i6 zx+;RK%+5o!Mdrsac$oTM-Un46ne9~jQ`W-!9_C>`R@N@%-zH(>U#R~Btdh>Sdfwa~ zIbX=Q_z!dpf8D;13wtW^J8h@o!RoWsl_@m_W3yPraBX^jFFfQF0wuvWhr2X_@{e(U zg11NRW5d4nBUTj5FXTuJZ}0x-wl5D@$^PB>GO-E6Y|_a|FEr41WbS8 zo(jY{R!a7B3OKrQWw|_=7~XQlTU^6xo2A^}Q|itGY&m+qzTfV>!@uyH3&gBV&lClv2uLaaZ-aZUs`3P=pfBhzL(m$f*w}G> zy+3Xs<-pdbvP4=rIk0h9`S+_sA`|AUh{3#A0}8JsT`2xt{>A`cm_44>06Uf887{xV zO&}%I^1{7>0E`MMy!nEuHW&RS|M#8cfr*1OHXQNPJX^MtEO>P+e`K?Em?lf`909B( z$;D~??IM3aK!hc5sTe=B|JsWy^dH8sdVt+a0Om6h@3OFb5o?UUadp{`!u`f<3CP7| zL2?g8I9P$Pb^Xr8`bGNjMc_!)nv7l%z4+LM@D{gBu_|_XFSk?0q3q({g zP~5Yp*e(b@v+8+5HPpRGU7`&N>|JuhPd$0qB*Mv!Sim#T=X>>7R7l%eM8qaihSGi_ z^P3UCKRP@43$tQ{y?*5hn&n>}F_41DJ>vnC`vlFc@K4R*QUK8iF-3ML-ax^OwAr_C z!PH8m{gNgcq}<4L(dcJSc{==VHGJejW^4bg7>|^7hjJ{9W?LRES~#&Ft&T$*YrUGt z)jEWelPK;hviVV-n>gPtF2|6&k};8w4|iz?U8L3ivG@$K6dy~Y;3r3KXCiLDEI(oG zp{CSG-xbSgPU^MD?^Rw-&jd0W(~MWl93^7YF2E)P8cez^e0~=!6I2)fA+>Gpy`I`f zXcnYbpGYVGJo&vxtaJ1gf%fPr{bEMf>vU3b+ujk>hf!WJ*iRTZgHD4q6g;Fb;VhZL z`QTYP+FWjAD7d2>FVN`=iDuT+7dEs+r!lni0ti=G%F8NR=EndsV~(Q|BowBc9l%?> z!5jt#vCnZWf!_!FL^B#Bs;J9VbYrmQUk4WVK5?cKymc7XmN5n=z6{my*UmQGOv#(2}#ivtHJ1NTk2>CHVd$|-{Mm6;6d!5pm zE3DMF+J{E>BfO~8m+|v|K)JZU;}(Uf;7L)J0e-@l6fnzEQgUyhBdrL(E|vFp8;n7?duTQjOng-4N=Lqo>w+bWtmjdJqW;`^xggFLEn{-myFQynwk zyR5Dr9qxYw{wt!g3^eIW4E+*6ulc!4+ZRkaK3TJttOgv+7IM@bI#%lGCB0>4 z@8J;9uHjejT08Y`7|0V&NO45B6uTn}XN*X^kPsd|Z7fd^Aq5$;3<3Uh4dE9}*fjA2sPGcScZ7g*QJIB|;Gt`ejz&8Gs8};~?;10U$Zu5I$nnO0jrL!( zls2vsnn9 z##Kc#>%Dfq8=HowYd~vDqsGk39e~Ch`kJS}!_wkys>d&sq#=5th=)j|+sDn`yUVd- z8`(#0`N5@B5g8t-CN5zu;dGOj5H6X)>p2Rk^>;9N%|i@gd^rl>>U)|>Rdt$QW<6;W zA!5sQuP9jhY-*Q2X*w$#qUTpaR^N zUIkI?N_YE&>`0^Ipx@VG3OzAB#}BN$m7Wo<%F~qFdcpibr!D9{-e{UH9)-?e-s>f1 zchg3RBG6d~d`Y@N)O0{q@tcen4NzTDGb4_CSrL!4&-VdG_bVHd))i~Or-RfeTeH5u z;*qWA{D5=3XL-``E`HWhu6|cPY#0(XQO*6ubCm5|<*(CLu0DF@eeHI|(c{)eXyE5n z+K#ZKVXF)^W+h4~wLP$s>ZXJ*3kH6<{c@n;G`z)yS?`}C1tOwA3Ea_@1 z%YTgmKvE^ZjOi4`+-X1y8>Zi!zSxs|r^=ZO$uI5wJz=@{vK7hvuZm0Oj2E_-PfXQ( zhP;fT`jJZ%s)I*0Ex1^W&{e?slQ?ENw2`*!eA!Zw3R2z+HuTWCPS+` zT%(A+TiL08vBt+kf%=FSgGbwrgY)-&CLK8ye?c_}pK>Zxp%itBTYf6(83ktiP?Nwf zcTC(_UMurt{RI=R^|sK;n&)nOQ%@82D3Vv+!Y9c-!d{6~?KgmH3F19gXWjcek=Nt< zSo`_;=rlc4YSSURBCM}fCsaj?|E*}!yq&09X@W4|fC)zJ^`Yw!#%xNR{$ z9^!B%GrI$|U=4cNjrAE*W4q$zDMV(=sng}?XaEx#_DTt&ajJ&0tyGt07Z}h8Nn^Fi zM}g0}m{a0|jvsn&KV{70v%`M?OB#zaTun8PN-CwC!L~mX(b2_Lj#X4h3R4geK5juaF;gB+2RUmOMGe!NeGD%-<(hH@%X;V_u-6o*w#j$lzVb7~*#B0{ z45?Ph<{{<0m(k9ydL;xDJJts8RK~HuXFy|{$!g{GuYJ2eG;;cDiH{v`KEK;>_DwIN zavkBea*f!d;Z2qJIrvoLv!R)jwUtPg0?y$lGN)OPYSa@xFahBjg?288>^r`T;OKKd zw#iR-_^@n|xykmga` zv0`!z64ed6js5pl>2S!ROx>%)3BdCKNZN}{UdCMeZi<6l@Q7A-amO6JPpLgvwf1Wo z=aM{ZJ3-9+?c2~SV1`AlhDZRw2}~j01h(G8%-U96&|Xvu>6lKdKIup;3{)hR;_unB zXz;N#8TO%=jnIU>z%aYi&{Qg5;J6zJqm3A%e|oxcUuk{gJQ1*|>9iap8{yf{y08BI zOy_5p6VsvC)2A6wV?8`=LT;mLd)Qt&yXXgB8~cInHXzC;^j~E5dPH#&mB_ecE4fyh z)7d-oyK~Bh8WCovBufz2F*qC3E}~G-K8mK793LHMHIiJ7DGIK0Q{4DekoIA6@T&eu z!f7V|KLEo<-sz;#8x;+NQcDI5LqUz;YaDj%tav=M7W%U=i-sk>G~)hLS7c*q=Hi>^ zFFd~8`h-wZjFHkq;5irursQ@`G|q_Sdu@?_R86`!GigJYG_DB6Xz=41vApK0>v#H+ zBX??#j^n>;jwV7fcn}cns4_}Aga#Np?GG$HX$&h$rCI<^Gm#(b(J8jn;jo{*U@vX= zd|peBA(JgTQiR3Vm8mD@zBdowWN_ZLjQ$je>aJ%T*_$$xAfphA(9WITtLGKg}^`P1B`PU>l{e@HVOS)l`{-P*6&HG zus8`^YEyIbJ&L5i%>pN9leMz!vyjJ}whg)I>N8v6PvKa_^C^|WEK^p(x0>43@EN$- zGJnsf`6_wOHiv*WNrbEg0c8*Z<#I6@tasyB(p8t>{Zw#6Q42D2*bE`9}<~>6;E6!r%3%E-T{^n zT>M_W2>Vu{M#+gTuY%X@%qysM2H~(7Qd1|T>uTwsfTM6E8HovXP z{2CW+s3$gdsv+u{{N7p$$EA|&pN{uh{cOH?VekCDU^u?*s@1J1pNsGq=>{m zs1l1s4n~PXvA|q_I1a1E{BT;TMj;$pc#{$_i_i1TL)g%i7C4;H@6eo0@q0i?l~?|Y zR2Tu~AgEMg^S)q~3&2qz08Zw{n7@r4&aM)*hYcbXCrtep3vQ)5%6z`sF}K*uO74rB zJFUH>_r)SKojX8}Dj^K0{=CG}7j(rBNz5*yQtI_}t%0^KWdaz42CY534yCi>XaKK* zj2RLfnCpST`oH8SzcL=6wcCx$_}YpqzWRVld`&EIJcwZ7)C62xHeyl6Ii`OAr0TvV zG5I!XXTYeD_r2-_%}Zkbtqx4n1(swLlg@nR>}9ORn8@owXFR1v2SS-!?L= zZ`QSz39D}5+#xvXdY6(b?2AdujYBqyYwj?s9#P57-n2=7mdqFLn)(QWj(sq-so57| zQMnVt{{djX?qq*kkZ;MPq^w|V4UBpJHZ!G8cSV8&b1A93gUQS$pWxh9{W%$LE@;w3 zo=6C)ce&rJP7xh??^wP($6O)LMnV3K`)5aC{1<1EuXL~f1Dp#=)-gKeyxS>4?x{C@ z(bmN9SYY7#S@((GpCJ}oS9X4z?ydCpLUUsTRnW<9(C-au=Er++u7}%}I~fb$)6G|N zkD=0!#nLLtp>S?@wz*t$4<%LB(D&E-X9xldIdO3e$|WJm=4MPhjb-pB{V&^d3g8W` zd|s#Tnx%(;DS-L|god#Zn4i?rBBL8e&@R%EbhcRWFVxtc?&CRg?e@mhT>?w#&$VX_ zuLdvrUq^Rzy!j6>dVcYGhECHxFhV9pfrrs|;Q!n7K>pu`qIGf{mx1FE7daY=pWl1m z9~^cSVuh)c{RenZcwqZnl|_Nyw#pox)E+r1c*sPu{d3`pj_8ZnP=7t^f%WER=at(N zm_{c)x)%xX$?nWX2>^Zb)M>Z-l&8J0uevae$6(hG3gvKiH}O60V4Sm=pU2u;2<9-0 z3=Hh#?N*rv6jg^hEQ^FT!YIEXo3dAqF&}_cLA}K_$G@n*#h(nivbyg#c54a#*5duY zVHjcMgaSS5?sn`|SbqaS-7yAj+aM4QdTkjyr}^PflHW zpF2Dz0f=QR_V#@|M(c(k@O-%kTM2>zjeP*Ux!ZHwheI0^K-nz7#wv67^Wap-byBZT zOdmDP$-+-MjzXw>XcK0kLt@R{JzQaufNe;Ho)3J>b-%icASi030*q{0UAJCBsViNe zRM)?#b?D_RBfup%}LeLy2n2&LRk7QZxg;L={<6)p6l}Ss#PDDw_vkNUqf3bH=@+; zZ}w*(6dtL++B>$n>@<)Z1-&XE;+vDd009c+IH(r@fO(@Bbty*Fq-_y^t=K~5d8~ci z-yO4IE>L2sce+=dm);s~iGwdK=cN7m151;(h9GI%0lvZ91(kCygy{Kc_+I-DyuG2; zy9i6mG$HmYp%xH%!F_@n1MzyHt7*#B14CRF(g97=@8o<&v7aB!=v}&SOPX!_ro@w! z#b}X8x?MyTxyT_#kF*uM!7L5zQ*}&2A_`rW-5tqs(k#b_Ew+G zG^!~Jgyyuw)@*_&rQ#k?3?27o-Rc&n=*m^ydLswk3a5$4oX#zO{15QS%ZrQ+UZIT< zOfAdaUQs3Fip?a7&Ook&-$tq+{0f)U8Kg2Lvbnj&9m2#k^us8f3^5Q|rM1M`FdFmygloR+us-XodZ1f6bvZr3dVj#0 zQHU@+#UrMYAdN$HQl-h^tYmh2Q)HAlsrN(!iUJ~FErHQ1{y3hKns)~7@R$5i)%}FT zuVP^OU|@~U*JA`pxYW^7Vkq-dH&Kr_{i%cTID@CSEDv7RfvKGhtm!RS6ZLH&$1bKqIb_dlT^|Hk-vhpLaQ` zyhCNgiQ%#i@?c8Kt@TJB_c`{P&~@J(iWWHrMf<0h&~@L$X9U zr^rsSmQ6b>;KpwHcdOtIYb=a0M{q?Lv%d#)|>+|X{pF)jZRc~$PAbN z96Znq!m7cd=euT``s$Pd=f-ZPb^4JbVANBEBi^CreQ3!#vB*-=v1^xBn&GRDm5$Vg zWyhBricLZlFV6Ov$~K;HJ+oRI5CWDFrv{i|Gl8D;9$*s5VXY_O*@@_Dq~;w~e~K@0 za>Q^`v3HQ!Byxx|EW5&!uSpt}LG|6HpP&P*Za|eZMvUyQeM9c_OlFN@)DP_D+Kn$g z-bml69%E(Jy1Fo?nVTJ|YyZV`#dv3udz6}p#o&zV@-U9_8wdr+!hyYP0J%9P8A@@(B2if)$47Q!+$0b^G0z&3)j*xrN&dIFjh^^DCXF=&rF8gE8# zo->Q5j!oCtF5GGNI8cz_Qf zLx*WHU2lL}PZ+>uqtPg_G}6&_KpG<^JAiwf37Xs)55?O;NE|#|WYfA-UzdAQYElhi zY_9RxTiEk{V7SP^horcUZsczYFo?@f4wy?qvW)vT6~DJnoL$M$il5UrkRJ>p%Q_NC&r1y@xx#aVBeBX0 z&|D%a^G=PN=ERC32eyNe5*bfAhACmz0Q~+?%TrRV&^`|wucD7o)o6w-qiW7+iW0`*d!u6ALuqr8*&uIi(Lr$b3l8<@rwFrgRU z(2S}Ryc8HmKfaB2D!2@i*c?=APDK3#Hh(rs#sY+$`;`LvcIh=U2L@3kkZ3Ak)@3?~ z1b~~kK`7$#8vYH2B`{yQXxYnr^8GeZ3J>0EJ$z5y(?Tcu&r&&Sw3^Qnt0s$)K1m6p zNG0Ch!MKC_jVP$`Gi8GwlF8pdeCB zty9*gMX#w$KGn?xpm2M=Kk<7Oi`NAh1T=BE*W9fWh&i83W-cw%G5EJR&2vO_MA0~L z^M6o;D&;+eukvbQ1K{uf#1X)*Kvdulo*jN%N;HN0iiqRktv))AXhfb%;Z9H`oFOk^ zR!zXRQ2kwfPP%^DE}Ed`%8E#DF_cppj!Tq4)a9-8G^acW8K_@6dXvd}M5&jU()>8l z^;^3-K3d+JBv;T=N@TnJq0*?eYnf10p=3eK4CuoCE3)R^g=1uTvzF|OVML#?L z>}rmKBpdm-0=a5c6L})w)apGDo2_T;_OzK`%_64#_~qc=%6TKhmveLVHL(m{6`!Bk z^VWK5e>ZZ}TK1Jmu*+&gWacPQqiNx*(LC@>^zSdT-VJpBho`p;Yw~~Gum_Cp(IJj8 zxy6gMw_kW(_dAIHDcI^J#*L_{* znT3H&B(jb{tM3In&T+}2Sa|hxl zdIm8CHDOn9_uCrk>i1PKC|qcGi=|Jdd;D+(Dju?h@a*#H`iqZ zXq}EN6U*0u=D5pFDmu)^X@P}BefbIuXAbKtNKm@*e(}N@ca>=0aU(`V9FHa8RY9D2 z>3{>v_+__DM@gHUR8H#e1?XvWnliiUS3W2)e7m)tq`KvO18lqWDaOAmJPw}MU7~ca zb2;E!9o9qZs{Pg8B)tG(aZdOl!n%6Lhvam0sHlRym<5NkS7HS}AKaRB57Fgxz^FFi zt02u`6o(&*9gGpl;ny+nf+h`;?r-u2knV8!!R1YZA>cIUV} zlX~Py^oTvQ^g0TcK9M+ISB`h?9N%PD3jLRTmdz-|5KRD1Az4j#%v4rbc^)z74VEMu zjXp2E>By&_JJbA{s)GIIG73j5jN>+bZc%zh2(7+Y$!#P$jg17F;_U#!@WjlANuyPn zp|tpK$*KyXG}@)pwDaReX@roY7rM(Id%4vrM}9NBZAE-^Cm>4m)n z5{h~um$nb~;Xf8XYq0FTbY`;pnqNl}UYnL`3Pf zBdZ;+y5Gmfi2coBEofd5yURq`k@*AKhp$N-eLR?zh zjO^z|U}R>pVr-v#y1(wxHyA`nehm9$^u?Y%s@RlBwH@vMhFkrGZ#tWG4jUp2zv+B@ zH7uaOe#>S)%X*sGN$I=BFQn;?n3=npfOB!=p6AL^DEDN6l~0c}wozagm;vB6p|HDn zDS>^BwmPGc0WpRa1{}RLpje}9v~I=7k@qT$l9+w$cVHf+2?8Iuxq0RySa_XcAwD`> z;zAP3Qos2HMOfeCywI!1s-bmbGXCBoMMO+)RQBXgCT~!IOL^$O(ggRa*kPS}+nj9gOrGMXG3iqD3d=kd-%@7 zA&Lqq;;HV@%C{iDQ3p~nXzFlY;t1US8nh`U>r3WZiOt@SIA}1{OQrs5io29zp8K@s za2y4gP7h$>dTK?x>y0NecTLR_>W{zJ7V?)d_?f0sw64uRY*L0Mb;Ca|gCK$2btdXK zyhQArxtdI=P~u9nm>Zz`Dy=c=lnT!4-bO975ayngDQ)2H{@&(C^7+9gXX~NB zIIdz=MvM92fL_>t06i9vAQfJKf;gpX7M=siD3s1xVEdd=uV$eBeO{B3r>Z+Wn&lf{ zXFF+22>YKUvHZz?v2(Z3#^G1VsxSbaDC2^Cw`m$HcCxtI-3MNL>&VirQfFE+Rzw>> zE`G^ufc_9|=K|!~u$vmN2BlFWg(^5qjn?*9mD$-dru_I%c)#tyyam+@-w#!Aakj;$ z=54DZC8^krb49(nV*jugA-(7gZ^uJDs~nBldGLgwd}k}w!Y9eUe=#|cOzC}&sqnJA z017kG0g*|><`0ndZ~b942l~Nfuv|{B?&a@Wo+&NDJjIc>Zdc;x^x!DHIUs|GEsIii z!-XjG5q&NX$D_>SXH%{jc=pg>wKoF6I>)m*p=F$H&4YmO>h^KvJ_eDX{>CW3Q|yVw z|74y=isnmBzPY>_All6ARh#Bf(|h#2UJo1neY-MB2jpWFmQU}osP4Pj4rkPm<9qU0 zcs%#+pYU$mvUr3`1SW|?)Ob$M)O^=!q^ZFPMsmm=%K1WXj0p$usTi{@M++X`0QY<& zg)q#;(aYx8!o(__qRlPV`EatkX3O2$_m^x&PD}Ld{nyKG_(dycLtP=8ItkB~UYd7) zALXrDImEBlmB+?669-Y8txU|yoOga%0aEHYZSmjrKV1Ed;aq$2AAk+WK{>rUJU}dC ze!>#rxC@bO#6W%t%Afib9hb~AbA_vd6-YJ}!)vsla##Tt; zDPI-`_Z+pyHg6H&X&!hgkgz_z45I*|ggG|-=nA_n{+B)eFZz1Sd%mZ$%wtF!E6lmn zA1)BEth(Bqa_w4pdw;Az7*3$8^M3YU=ONn+kfKug*J1!V#ba8WB@i<`exnFTQ8El1xk+rKdOiR4$(JgYC^6EuZJHL4`WHsEHo0|y z(rTo*10)$n1Z44=+wx%1{~P0#m^^eu@o405#x`PcIb?bM%T;cm#s59BB1@NB!kU1; z*GjGO@;V$MU)%=)Q2{CXFiDft<0c#v4GV@OpV1YiW3XYnGsOj5blu#2!hkc{m2AgilbPXW$*h&nVbA?<_O9hs{6^SfOkWETb7Z7 z*ec131TcF|s;_9BSGP}-_2?7TT}1y*Kr@m}hBYFiZQMCMjZ=+SQRJ&%=Sqrf(|0?w zv!C~t^YDC5wZas(mTJ1a4a^s@LID<9Z=X55`nK~v3+&bfqy&!*DyMpQd0X42A(m!z z0lRYI{cFK@3X}R)wSNQxOF}AWc!NSCHt;c1M#Em*1T%aPLrjS1CCgLE77i1#_PF9V z&Z&KnM#pu%p%m872iVYk$m*t>Y(oC9c~J9GuG+OycZuarW$ldpB7gmxpXw0nIkjxj zne2>_=BT`m@9+;0J(wun5tH1URB!AuW?u>Gko`07E>1`pxE~*g$!V#gyEgquhx|zW zQ`KTMc`X^=TWH@RA5Q*|ufJ|JOBQXM4z$`__N_bpOci)4E2vo6c=wdlAQ9qdpUMV_F2-5CC7VnPI@X7nxpD;i?Pg2t2C|+rM%%R;kK* zf=%#V>ycH&tTFGx&|q#~=WVRqS`#4!|I-Qz!H(6E0A9&_Nv=ut9ZFq-x59x*YDSf2 z-5$-oQRB_1Otx4oEi~=Y=~}~4g@5w0-=-)#y134fqtFrgQz0dsK$KP{7W$?4y=qL- z{oE=zwT8r(Ps8wP=U_fr(RS{1k1!w>hLzj~vlOUeADQ~{LVQ!3?{`D7XuvjJEmA#0 zfl#$n)IWY~M?w}epxJ?szTc&o*3xY`CAEN#{@`WG1U7@l-Kf*F1Q=8>yCB_@dS_8| zevjU)rVpjKa+>`XAlN7;LFMM1`Iq0AO}EMYp_!Q4n}Ji}2gnz_JX;S^9zOFy0w50z zOStzS==NuE1{!qGUW9_);}0yUyiS-j$jfEBpBvKm>wWo;{^u)gqfn!#w)E_viclDl zycY$4T|`0iU!ZrT^^`wH_|##Ig~mJb!Q`mnsmWQj3|>{F0tuIvl2m=X0o}K!ub)rW zW;~mh=njnnKU^XZ}MIB|fTrSrJ> zkEu(UIJ|hoKYKlthR*PqiHj$A4n11(8ZYX-_16n>hl?;ZC6Y`8kJAV^76>Oq!bB9p zlZ+%JbKo=EDN6BKcF%?WO-C-H2Wd=V>`u5ga`!8Rn}Fk1jUFyWU5qBbsnPy8VZ4L! zQl-_L#r_lT?br8nasVI%$WO96=F9GB(4Li&J1(!5Bf%*@y{r3+rB#bZCvL{z1?Eme ze2(g{1RT+Ws-&Fk*3WD(Qw81mHw{d=i&10ZEqD{j6jTY9PoJZ)2#UiITF$zq-dDPe zi!lygM=NtKMZqQ)UCcj_^YhsL?8{%48yPpkunHs-1S+SWZL}vHiE0HBrV_uvcxDva z@%`{`$G$;9%D=?~+=&^JPv{LMJcH)RGJCmIUAhwmsW@dP=>7@?hlr5zwGbt8Bf*53((CaUGT7Xn}!>h>0xR8Z~EmP5Z@=XocxEa^wLAztTNWc7`&8;r+s+z=z%cz*?yS}uu~(_U|O`M94}kwTD? zj?sjRfZ&Xl^b)kq5KdR*1>#ESV4!&0f4t5KlWD~j7JS00Dl0pjqAzqQK@^40<#OhQ zaR}n6^tQZJ^zHUYJh*gY!%8sLx|{zA5Cz6Jp&i>ckcuGwR6xtzwYo z8jv~`YW4&Tml-ZJbuGzUUH~7X zVe(vQ?y#1KnVEMI^U2$4p?9xu&SGMdjH`!fPime#EVntwgek^E4&wy%C_g1U?;EwD z0!DUq|EiulVf;Rv|7ovg@#m~<0|+}7Pi3=TfoC6RVKwdBAuxt4pmX>{!O}a>IX6T9 z4hF6f{3lSUy? z26aYXPelz;W%-|&^&OM!A4#*CY>`b*Bj7~kjgo{`j(}_)gzhFFP=Fb` zVuwT-ndaY9e6du0bu-2gX?$W#Yg+}9kn&|g^-rWry#Xda6U(2(k1}Xpiiyott)DmP z^3s1OKz1yuwxNW4K%6>7DKxTUw5gKg<=7EO&@$~)joAk6{837i{q1CbZ#uc*6ehQu zMGnys)3#8Ucp9CX5qBpyWF;J5Uav_&x2@vo=?A~a`+GB$Zj)1Z-M8^Ys@m1A>%+~_ zw~~=uRQ1TjGOhI`vJ!W9PA@e5a}JfQ3U|j}w9g4XC;a4@_hD!KxKM0$+rK&^rc4E1 ztyT5K!$mv)ov_uWIyDnl%={Goe2!P``z}$>6z5OyVmr_@1Na|6tvlXO+q*I3{PXFI4?5ww|qlHQ6*cYuOb-JJanacgGU(FdRZsgVD z;kBpZus>0ppGT~swip2)wqv;Xd65lhHG7L5l4pGI(U#<#Xo{F(t6AI;VfCsCz{R0P9CX;nS zt-oDgL$T2OD-{Ddjn`e6gQxAzrq}7WI}$t^u~wvDHw_eU-ojrC-ppJL9F}?)Hq;mi z*1QgF4rd3iD#rqiFDNsIwsST2d8tf)B92P=#CtUyXX}5T9nM)Pm2+j3)>4rtw?3Qw zvD%yv@VNTAPLMd463(>8F-Y*d#{8@1QE#+}_b1*g%`0JbG5d?m;BDgND^kY$UvP#+ z2DM}k_V@lM@Xvk#k^6&~WyqbOgz{93I%t!Q6r?s)6%OXSOV0gmEZ%-&<_T=c-x;}5vnJC zkrD?hhk9`xxU;hyY@L&C1io3u*>m81b-eT$_Ws30b=Q?SFlo^rvCS%=SMN=25S~_% zU#M$+>TbRH$Ye!sdotd({|9UHhW3QbU-?#jY{!DcLRBs3r`WHL9Gr>QjGqwCtVB7) zC>c@n+O^#o{dj=SCjDupj%?6DB(j$n#C%_CMqFMko1Vfy`JXC>kpBQ# z-+lc|+bS>LM8TxZ91P%FrE>(hNUa34^>D69MW!^$kb7{8Xki@RnewN9eeB5A_13de zRz3_q%i&ew`zzA>kE2)X?vsH+n^&gp$tW(ax>qO%;lq{8Isn|nC=E+I%;n#V^}yyX z`8%#&23WlO17eHU`3a}Qq#c%&6NYe_0?;9Zp3`;gs(w<_-+mspd@xv8hweyA7Pdm* z3e0lT07|ag6$QiyjAp&CpsL~~R7QF#*zkI$q~8|L{F0n77xO5jNwz9Gtc2wh2Y^+x zu;Slx;<3T>eB;(fR-_t)gd>rI?XD#yFkEq@ji%X=t9e!xp9W4NcUO)4Q@=!C+x0SR zgWRYZd_r(`7ujG$Bi~8Kb4-uZWbmEt1WUD7Wjk`ML^o?bJPo7O6=Bz}Sjp*%Kc&u? zN|~~;Chgak{0a)dXBr`yur|z|_C^b6?cYc5CR(|!%|t#bn_J`HD;6@{Hck$gch0o+8j{EoCFpa zls5?$YGn`+)kga!nSI7Z5A8}Jt1>bo>xpBKTN4)*p&YsX31;I?dcZPmR<|L3Gie>Y z1$mpy#K_?zkm}v^=+ePBpt9y5uwEy_KRU3=OP_*JgSON*^rhnTYMF*Jz72Ra=;0p< zbMnB*a@H0z^-`28cnfr9s0*p&=ec*WL#D7pSTU&97Kc ziUZZ;u6bQoCq-Ur)f|XT&fWIkP7}$u2ImqtBPMe}FS!93>Hlzad0|&Jmg_B8<4`4g zptmUYFZs{EQxfEPdzmz@J_PI$3u&P}Fv`%KFF8d;!{ldP3SJJ4_?#6+klSA0u=kGazwbb@6@Q#FitKRt-I+P^(u)DSnCbaz6^1{`|9bE5 zr&rXf2;e#%Esq}I1jfHISTMA*6?X?93ueh@o=kJvb-p`U5aR9+zQ#YBE<7>9UY~jt zZ+W=+7f$s0`}u8;!I4!6e=i%5QeS3#FXSwopz;1{b!0~FYGoxDqZQ!X*Xr%SxCU;( zak8B&#a~MPhLcxt1C+VhFpnn+XB05&M*qHz7m-@~A$P4Ie;*)iyw&IsNy%@0{SPJg zJx5y@vu5#KS*U0Wt!z)S83eT<#v;2h_x}OSIN71(Z}yczLyO0@@7}Gyi+o9`sz?Rz zO_vY>FcI`7MxNyeFJk}Cs^CzcjRE~6-%?#h<*eSgD&L%M1$X2ODs6CZoIzH1Yha){k@y8Bp;7?k@Ch&-=K#u zJvc7ZCbMy3D`#)+q>pRoe_KmRbVhEd6u9yl{3As`zk4}4KTmH$ z`}q!&Ql*|;43KTI@csM`@X~*fhBRutSYFNR;wj+(DKgZ{d*S+L$Kv-${Fz#|#rqp` zhoWH|SBh9@YCD2){@cLMgP*MGa}q+F-+bWSsh&I{N^z%=akcl8|Lv0GZ=Hcg2S4X3GD!O@Xfm1tz#0H{*VEuR8;F;c z8xXMy)8VZo%Xj|r(DCJ6lYbL0f(1$=053*TsI@hcsCaFXaGZeCp%KE^Zv!JFT{S2c zH+s@EmD4ocb%s({`~iV54RbtMHe@7MJej3ALOg=$HOrQj78mi&w<)LQc#vJ{zZyUp z7wH^tP;8)R=*B(_b=Cr=aD_z#>{y>slp#8!r8XemT4~JLmOK_Ecxc2P?JlzfG8QAm zeW;j<0S9POiQzd8iorrG>vTy1(#!E@!~0ICEI{lC>BgPb@T1gO;;C;5Z)19&3~6 zO0CvPIw#>aqlTS~#IFg7EP_@}%izuUyBF1Fy=E>)eNZtJr^Sg^wH{$Qzkp(=y}!`< z7%4#TA0RTeLv~>=La-p=jx>?kyooMCtv&}k4vVaaOa??Bp5mW&I4Ym_NOGrP6X!>RKlX z{5!HTnu&-a%_yOvnhZr9U$D_a_s(zsxU~ycBa*s=(m>hF{JWrz~DVTu|+o!qmxrq?c8p^E-IBf{d*Q<4`x+Gm?z%@9Yj=l%A!f*sr zR^>w^g*(oR&%HA}HuknA!{02>c#lq#xPOCGrWj+4<95k+|B7^4=;eZ=wAY8RvDQdb z8Uu!Fr#x#P8D4EeWqdd*6yHzpLR-v++xe96nB#Sc0arq-L)}vYFXx~yF=g|n-W{6| zDXLs6tdCfXwFN|++2@&23r7t@xhXCca`@_ z#}Bf7$c-b8{Z{kupuVynm?oezdaQctia`jT{Q_D4jR;)Q$)FV96Y}g^GY;8KbCug7 z?+mzYgUd+eCBF^ka{3oVbkDz~GMXQKR{bUdd4dV-4G4NtTrFWsE48#V96P6gipZKc zybW)}{!bhO;~rbvOC8Z%L+dKeR359~WDFKWV&2ovIweGC&l~rGk6~f%gi@4e$-3?= zS5vdwR`hwRpEK6a(!(A*${!3PPumMP82x;lq1$ zWK8JANTlqB+%Z;6zjVls?Ul=mI0zg{2xnEqh}B#e*hN7w+{E0eDaL6rcfZ6y{t?X- zXsvCnl^K!7JQvPEs$V6FeX6|FKTuMmk(J0~9)cxZ1p}Ltj)4`5ke1qsIhI}28m8xc zGQx8ozcu(Aq>J%pJ#l(%Ng!ibkRot+K~0zNlg9L04Oi|7w)j#m;EVRlWgEu*ZHl_5 zfD5|sx9{-&@zSa@^luR?9B8EeY;?CC&w0+OTcZ}Kg&tVt90I#q-zg+ug7{#VY8|wBkW65ICi`w9kn`^A-uS=>J)`Lw0oT z<%#Y_f}Pl*nBTZ}E=V<}n4)E2Q#aiPoGxi$IBx#5g0tzxS@J`lBWi!533%Ei{>_^} z#8t5Qi60xNlpRV&BkJn2xLjOdXwcuNdgZNsM4Oi*U24{dH}v?`ei)9_x@=za@?nQ- zM5alLS9ap)dC!;Ul*(4x3DD=w|6q@a?FtkQvYxBNVl!;V5mJhVpTP0J78eK+``@&X zzD6|}Q8C#o(?RUKLIO|RVXjajIG~~$frnS7j-cdbLWB!#W<9GFV>37Se{R$2qehI`cbF{)o)3{hj4qsVPNdH7D9s?M{VQq3vqnO>J=LL=3&dM;42cQU7fr+O{Mh*yHX67BN6Gqv5|uSf6eZlTy5?2+o+wd z=KD3N0CzH4Rxi4V)NYJ&kO7h5Hdvt~(z$a*|B zVBVkxZwFjGv>D+jnq-g>8qb);o=CvQvgqQhBY9zVQ)g*1{%k(q;BX+!-ujLI=m!V& ztWqyFYt{0K6v9}zY~PDw$&mu#b!fwhu!FRO5C-wUiM%Co=MHYcUBW7#Cpry@(sa2U z=A%KRLm7xfy?vlgRA}On(REc(NA7j%*SgaqN5jj}U^*8p#w1oAV(2Hv7*a)?pikW+ zVLG2@aZFPyP`F6Vv0|y^J1U5{6g9-OF#OHc%c7re)>@K(7bGeig6|QP?S~QNVnM)t zAN4R%=`z9`kfo^tg*972S##^cWOde)e&uooF}yfw(~^s(qU*GRn*rE{6R)R9@~MB(scw z_@z*@c?m7Gz~xB#Ce&2skRd0E2^)*kdpNr(dRo&eAn9#+&3;dQCaBCqIG|@0Yb+^e z0zM!b8AtSqmd|9sfg}3_ocGrsy`Z{XU`(m%O?+*-MN5Z6Sc{&F6H%{3YHDgFV4lTQ zVRc2&PihPi`r`M0^08VQ{Fu|P^~LL_(;?(mhUpJ+OuS8QTw@Phq?E3u4bcq(uK`f> z80q!8+^Qk;Mw}QK91W_ zreX(pxz$Kl3lyd~kzozlPxB$bcC%_3)xe3kn7l%W+q2gV*V}&*|B~U7d-$4HU~no| z*!!nP|6i7$HN5|1+L?d8A2PT7kYi8i<0RO1){4wE+5r$^4`6t(%KsA9c%wC&lUVsk zjTF^l?Q8zN_;Id1lWA`}D-mW~`(b=jIit=yr%*RCUBf_Z5-a%QdQVY%mD7Bk!;&k| z1a9hPt`Vb{pF z;m8eXFyq7NXi{*dU;E+rW&&wc4uCQ!&I4o9Y*L9R6nzVv(qTLT6`dl#1j*c{r8@|% z;dRt?2J8Z_ro-F3EXYi#JoGYwwewX9DVj_K%)2g_Hm06rl^q&hR7#M9Q}V)_hI_v$sEFoJFuFdX$@NmH$McseAEhA2iMC z`C)5O{ri|{gohISy_&-@eU_653QQ!51rr!vT9VLrB^EmPAmG_h8F^?8DtdY7TK1MK z9hzXrei`lz!1i99@BS($sYH8|eYkKEgc{WGcmXlyN8-%h;2)LeeW0Yb3$blrSLg2l zVB4;!!@}Xpp}^R-HETcg#vMZZ8#N9N5>|ia{pNU&@v~^N@@~i*QaDYvws+c zmJiWJKaSC)iKuk{KFQwk-L`3woZ#80@B?xYQeJ%{>?vUq8{HAAfp zmlTN^eAu6rG33sehdln}7>#9Y%JA5$h-*tM{(BouEG z3OwzaTkiF3c)2>^V#=vri0I1I{bRXG3?l4!@EwWV+^&sCa{Ur~d32bdd7sS$l{D?} zq{cmW_OJ|A#c2|$f>xED4{TFN5sLqTr+1|a$8#HGhAVg5GR;E&I1g3+eLtI!`|vx` zgpEq9aP>qYkyJ^FT4ql0M(2qai|U#Cw?%^qYU*d1*lg$dJevLZ0f2GZSiSJHCLexe zKdFjeNOC+u&HGt<aW13PK~@>mTh4H4}rZ9|7A}S`Cestm-@&HB@v!&HiuzygAu|Un!9r91S(v-tYhP z;f00^eeJcY?4=RNNwGscP5UGJl6VOva#IF3B-$N+dcNq{!K6?nn!E7HT?X}x__iWN zdBc_I3$m(H7zU0^Mcc=X9Nc~q>V3$t8uwMIapf;+7(J|&Dn_IZfJA=aj*rWea?2Zj zQtq3dZlHH8pgjdMHhNLO|Dllwmw#c_K)l%v?3V~-h|iJM#90lZu^D_l1g22s#cl>j z+?S0z zW8>8cH<8FUqRtwz%IVwy4^Lm|PKkIOh23PODC!mCo7P&VJP!#L<=bsNJ?QkiyDb^*=!H zE1@xni#rVpoVXP`=i71jk8R>zvjRR(Mv>~E+)dEw*--is876H&RfpSg0T8l$fY(Q) zK!M~33&Eq4y_mPt7EkBE_tpRX5BuxzAE2FtR!$myPX>+41cuYcQ8>AgwdntKYiiy; z9w68aG^NP>OaSSSk2wnY4WzX23I}Y5dLv3`YM0!tn-yq+U#We`B7k6Cg%g;F%E!E4 zYsExA;>o)+(^BUHKn?-`w3$o`K*#AApq6vhS^esGTDavpXme)UK14*;5K|Q~ikN@* zAnnsZ_P@{9?;ECOF+ecuHkR^Y`J&bC_1{cSW7fzg(2utM^^Z%&jFxJh5RUJ1{2msv z67kDH{6Ba42=eEHha#>YFn2-i?-wthtX$_&N1|HY)mS?l@5`spXovpCDAU^C#*|*m zA2S|G5j&7IFRHeiAWO=~m_ZycQ+__-6Nv~R>996eKX4=m>4E#(3+6{|TBe9eAVs=t zRA>GX_BY|<3l0p#ZuaW9<1(q$R}vgWw}aPjdN%2jEP65kWTj-W4u1S{6i%QNkUxDk zgP9vsAoO^+(+PDshNHOXQxL)|OgMGgzkw7c!Y%p6*nYw|PU%jmv`|$7NEi?<0&&Q` zX&?Pv^nc;m)sSmgvX51~Z)hwa0K?5g$lN{r0#baSkb3>cU4bp7 zQ-;Y@NP(+874Z0gXgj6s?f3>!>;n&IL@_V3II|d8+N=) zRKXj~fV7zE&p$+L2Q~Q0e>eZ1y>j#U#N?yhb@k(td+uY!z2~Fgwc!R$F9t&5tMd?2 zA>mQnze0uzv5*g@_;CLpKoor+^gq4mMO>Ox{#o(ozSrU8^1+nKYijyWWE+BSRSA$Y zT6^C#9;{=eWRt(#J)A@7{GNZmh=UehE%}Hb?k*|Ai7z@uwu zGuC5hJDSwzWssrsqQW(^8>uY1Zi|mL)@(M+$4Vyk_V(BNd`iY`x@%H_EWMV2_d>7* zkIt$7aDQ5i2~iY~#CE$eGkp}=rMY+DF9&pWy=Mi z3Jgg8P5N7gJb4Xz-_W;X5f}0=x>mnOrSryo)OI8}<;>Ou^F%yE*%5a*9uVX~q`7s^L4Fy`D zcY_(Szohn;M;IQzP8VTOI!kKb_lDDsuV`6Yfa6otb->C7Ep+n2Cedbv?FVCpQML6u zSXBkFivI9~K6K0g3j#b8cuNm~aM=GNNx`|Eg9?9qdZ0+GH7a zEyyWSD&^YkA*0x}D7<)$O>HZ+TNUox)h<*s(g^WQ{|P8X{=<7IK`aKIdfGNP#hT4; zEA%ImJm6`+$;j`zTcLl4Hh-?#mYDw%$}CK>nuyErtH*hA@{u$sQ3RP*TP+L=8J|iz zl%-7+Vwi9`?mP_0LGB;MHPyUI$C>O-7|t`Yq#}*1o&o!;L)NQs`?Y6ICPEi!5S78t zqkvExBBQ|`<~}5*d+YMlEf9B^xl?SiS@MSlRpq7~B$&muG8R<~v9p^=YpW@x?H{8XiV(4QML^_WWIRq!Z=th_Ig5aEoIJcgAPD zPLYauhs+`Bog06oav+6m$KyY?#g7$>k^fDJ{0}gtlkbi%{S_%dYHwm4imMr=zypK= zFfe3%|F*;u#x!qKmLwR4V~w z_=zqAZ)@gN^AL~!M+51KgQ0`GO({90zFBVgG;~aaDJv}FW~Tr1Njt*X6$Gj zT^UR$stjz@drhFIPs{0@DL^PMNb@^TmBu)QCxzIxqcWz7Kk~m_t*w9?C2U8 zt`mUcQ2&zGJ7JtOU2~6@P$)`(qkj#-w4}h4Pbus{nviDTzm?wEYH}CaC;vX^Tf(U2 zi7Y9{yDdCclrG==Cs&gwrUFZoXun|-#reG3w-qIDvI>4Tr}m2ux(ZY2CrbVX_7zql z4VHwO!;|`*L|GXW%Z4cDl?}!!UXg!fW=$u^Y8_ZLEu-0Z#M?Skc++vW+myMm>r4{U zOO2yKs)`&A>UTVVa0T#QkzYJ!KgSuwq)Iz?K7cH8FQ-AUUO%*iod2CXqvJa5{u1+uDAiF#M=1h)D(3K1;7`yi zxT&TB00ayNX~pK72*FddYeZ#~_}QVl`>*oVDc)kzt$Z4kdWth4AD6;+)Ar;){i7@w z3iFkS+%zjiLJq+B_IKSKJW=mT?9gY_!5On3i#iftQ%bVJu$d1_xYx)&3&7j#nI#IT z>p$vO@+^VeeZ#mzwXSqmBmC2ozN`Tq}pB8p&{^D9FK_=0ej`+DR*06N|Bt#0x2g*;{+F{e<#=|yu7 z1zhHJM08}xpwFzS`)#pI#{Kw;UPrkq?%%U8fl4rsfkcwM{bI9BxOiBVAs;L-QPm5aLfV3vAN z>Ox+4wzK%{Fy`d?$eyAeT@J8-bJ|AsBmFoH81QUfKd+Z>S?gVS^TBV=zbdnc$@Ko? z4Rs>h@=x_>9aWGstI47+HrJAhifOn@u4xG0P>ok-@sqWx6`%G}X8<=5HaX(lxA?Iy zf?UnoSFCo)sQv4*?8P;zpykhU$ldDsIPF-33Emoq->p2z*r9xR-suW%gOX{^)EQw=(j&slVKNGU<4Z;!G@Hjm1ZG z^em&*q6m}ow=E~%{HBo(ZR_Ern(I(l5)zjnt4^%E_VcWckxbN>r#_ji?aXYX2JHt-h;>rq+r@C2v#jN{P)Lm&!KK79=EORNL zU&yAQ23w<>o+P6oqMpHd(;1tl1*e21WO!_W_(&`j3&!;qJPfQ9LC6fSw4lQ{^uVBC zA3&$7Y&VlyRh1TWT=3PEfJXaAZ|nrfH`6_KO1#ODBR-D2Efd#`_JwT;V#qZNPj zDMG`a#Ush^12!xLD}7RX_p6RRfNS*^T}1;p?`s}~Bf^8P(yUbWo4p;oN1vEq4cY-sL`dOvC5b2K{&saxFT^^_*P6-AsU2V)JKCMJ#h9RvcFm;AOwFQB_k=Dr_}x%0t?Tp76RmUZU6es5-v0&@H}Jclk>Skir_m zM^r^tPaMks85%d}9Flnx60z&bosrQ@E?}Z zgg)}fOhIxsQVwT&0r(I*EBhqwrliGVrIerN`7bH&{77zY1g$U_%c?a}0oxs7=$WGW z%uL;&ng~i;|KV@%MODe13=8&~vNZKKjrOpKzL4`XR%IPc z8xAt-i^hNEA(BhmjX+8ARp~ch7al47+^8o*I|eu>6ccX8IJ6?)g;Ge5sE^~6>>my^ zHJDw}y!n-S=7MBc)O&zg(aW?fAnr ze8fm|Qb1=Z;?O;4L_lh_6`hQvb`G&8XNViDFEEXi)GA$IELWEFpWA6@p$MM0wn(TCdQj=e{mO>f^eKfTGp|3tSp-2&R;clc3MvMiss6w!|0 zZ!_8FejSaew+=0k-@)~XLJ4%|HO569Gtz~{3$z4$X&kQq@bD_WYHRXKI{z;Cgb#}! zx%2IMOx)#j&+1X&Qn5dW-b>{RS!|!k%6ZM<{{X{c05F3tzz3u)prLLstf`Lpi2#VM zrUrf7doM^SW+f|GAR7HFlP{a*E8gd-gTaQ&UGH1<48K^f_6lrUUQ6FFe+qj5mUzo& zzDculNrb@2AK2=A3tqxjuM}QYk35U_6PTA_m_uzBUU{%#N6wV?%M6-~N2^4I7XVx* z8RDwhz_sUk5aA%I>gsD0pUFJd+~;q|=*Zq*@Zn9N911gyoKinR&$rDBLzKsYv<|Jz zy?cj)%uubgniY>*ZyN(w+q0#Dgap%Z4NXt1)- zqbZ|t&@Pk>iG=flxuuDEMBS=vH^vF@Lz@kknk)Z2&19yfKorCdmsxHQ%Fku^!y03u zqSymdvDUb3fAr=5bw z%Deu;LiyU{I%!_r6rnC;tZBMfMnGs~?T8cl|CIIKK~25kw{L*Zr3Iu2VhEAmk=_CV zCUin5hF+vfm)?7C0TiV7CIV6f0cimNY0{)CARtAgcktx^-yd zu4k>!(o`2sJxyT^=5M&rX#5teKqy(JcRVsKFzm6WW)ra4LK-x$)> zo+bT|{se#k%FvYTi_tMZ(24J7^W32-AGnKt>Bh~kYfXnixIN`EK~Kw3t`BBJrM<%3 zGn2jB3V0+O-G?I?!O0X#BRHHP3-4{9{gEW1;E`|=A>D_3kmHT1-9wzgNuOFd*?8{H zpMyJC-$Z#)OKx!s1YxueKa1%goPM&SI`T{RH=a&dQ~DO$TJ}&|ASOidXi51`MB<*jg0BLvpx1x8O)<`0P)#NGaWONJ zfHT;dm}xFB_#oK8{8+GG;jH)fL-_W`zQuyC|DWTFZ&hy!bB5?yyUngw-}}gOaGZMJ zNJqF5IJMx3Wys<`fE?soh_UDY59&_`f*|C3zu0epbHZd&5&!3IoSj`flciR9=^}fX;&gOYfU87Jy3|NeHVI@8Njt)jxR5|1 z!#qQHsdy1rIB_kA{~>3t@f;4&|4=y!1XDdG)SVG6jNpN0>##nIjDS-AEG~1Ag`fn* zQ$vuTR0IxBvtaicL*jv8dn{Zoo)b3Zv>i*Fj}0*b$1p9?@FY;%0!elVzp$5L;GMCM zJ6i&q^3Z=$Lc?`c?zvf6(%?QFR=bOQ}3#&@dNU6LH+Z0ly%Ik()=Xm}II@3nc;+;{PxYti5o zH~a6zykh;)JFz6q7XF(x*bx7wEhMitwsov;{XAivT+r*tDrfsfXdIKaO5B~$|73YdFQ4a>;G(?!bgYGVn>IKb*O&1vM#=wb zB%*Gr&!5HD`~x(7%umJ83L#`DbC?&tO%%YhB&{-VWH^yB4jnKEVaI7OXV2p!R`n!I zax;}wlmq;%C?PgE*jreH8PrX`El%ukh+jH*l1ATzZLtxpw9xxh7dRkzJ0WI-qonLxgJlz-)CF`_6!)z&ff9;_t-j3D|l=w~H z0Yc#%L4N_Mq@8uaur;Pnb_Ii;fwl`ZbYk6$iHTxFw94VPQLIwu{s2Z^@raJ_bS|{Mx=IPH$xR@4@ z#vS@))@Keo{vvyOwR z#Pk+SRfB%t#<*C$BK0J6b|^F7NfM|aIJ3!tApS??Qgiu138Ncp>T23lK_nDvLR4)} z^z%v@HnPd;Jxb$-Jy_DLD4wSFp_8rtu$>yh<^9s z`)E66!ze8&>TWSYM{{FwOE|VnmV}GoZI*a%X_t&xEZ?kz$G!RfQ7yO0FjQ5`BG+^l z1l%gE3>?~yPDF$LK-=5hD&H=LF}TJ=>Pw1o-AxAeHKujiqg=bjpq%VQ!gUCtR6}S4$-G~tUE*-)6?;=rT41~!s~wZz#M+= z1yW)|Nw8;Fg2>U{tDU=vJztc=H1x09UphOxHdgAUAxxM6O!VvJF|bg+R*#glVRdS_^~olA`$J8$ zw1XGv0%+e?kQKr6`CO%vCNkooNo0OSFUhSzD!--dM|_{&kkz)hN?qB$*SmX{?!f$& zpPM-uGT&nXypCyQ_NGm7C2^HI&!P}BtsJDy-AQVblsmK^A$a)rUwFR2@3AiP>ugym zAn6ornnP@JCbrJQ#7|z$gifXTChl8(9YT^kjRS)yGda5F5?1b2;ses;b?*#bHoHGo zLkOtXi^)^?GhC45u9s@Fkgw`^TFEk_U!KUQcB~0dt?nIGfKDGUx?`&yxk<~tcmTaO zglx9wf1gGSYvhg*2#*Mn-A49vbp0i3!2e{bT36=yyu@()i0_RQ=Ea&2CR0lL+QGtd ztniVUq&qmm>czg3&Z|*kBHnOO$9_@NA6wI`*Td@Fk2lR;s842clXH0@>5JYG^=9Nr zThZ0lZ;_3b4rj`vX=3~4ks0*xidMxUXFNe954rp0!-O~U_ksm!qOjuyp(AiyZXI~l zvKB0|B1IL4K1zrZo`lC zY?L_oV@06nJ=8cL|84|EG*%k=d=yWxm%P5+qi{^s(c~qyU7z1$aS^;67Y^qUmD)tS zo-jk*Io&!Z(6@H=FU3}PNL+u@F_GI<`$|%U&;VbvmryY>P)7q_&nAqPmIDwm=EKRv z1iDS{hsngrpgcSSlGYg#b9e1nYJRS}8!WNb(Z6}Z_bF2&vQnib0(=BfUDH}Ghni%J z!tsF2{{S>4%_{JH_c9Y2B`q;uFCim34+`ReWQPX(1wT%j$J4;d6&o(D6)Rv5;21I~ zNGthWxD{B2LTr+_seI9E?wM!<{C#fbZOa*LNvb>X+TW2}lGDfb{koW)5E(2A>IBh% z-j!x)^t8?O0QQ`5JpM|UL>uV!C@h|zKVg)#wKbe&xf(T%j)>pz=i z+C>;+HkVLNV%Z%Xq}qcbFYi6m{9|Q9ZoCy`-EbgOQ!p^bl;7vQl!u#J?JK?GLoEE# zI>!Xc+4>D|s6C$_!(nexS+#>zwe2x^#Vb6r#Fkb5nPUQEWQhc9LJ}N+^l{pQYZH^t zGCv$eRg_0$Y2)y(7yoD~R82jssuG;oh()Gm0yrH3T2}9;d7^she9i{M0|p;&B{r#P#U804?=TnkcsL^BX$C zKB{E#sc_N?87l-H0)#Ym@oeT&oPu)Z0PNT0a~!k9zcB)~)f}oAvlE#zR*#HQq$6|T}yf`D}WL9S6dYS~lSq)Y9 zWs+Ann-bKM@hshi6hpo6DJQIsNO+**d}x@>v|;Q%MND6-msmWntWA5VZtL67MW;88 zc`D>8)kQGe?!jjCC#k%L#at?67DTu?(qi!|%-D2+$-T)k` zcZiIkX{-paB61PpcmE>;4t*VDZPX8?RxIKV;$2sL#MQ@(Dyf3OvcteV64M+vL!E8Z zc*LCinpWH{7MB+T_<_Zrxc-(rkl>;2*{+)9bESB!WBWpHQk#p@Rq`b14=zK(I3<__ z=Auhe^jW|X6E-cJCj>;}e+-9`epCd9at50%4&6sZ@P(qA%RVc$pQ|JYRqv-(=~5#& ztiA)ZW^?992!ajjCxcyeicdYhb*_FY{ac+H~-O(V$4Q67}11rGci zl(S=O9(+JNA9f^O#%(_@bkUvB;G(VA*_V@EINO46cRj3o>kb&n8AgQSr2?oCsU$?o zt5v=C^#|`$s<4RnoIX&Fc|z)gAGT9;EFM}Hy8vU;f~veL)Q-sxC84EX(teJO5X>qh zQ%cxNKihJL*UHGgevM~O;Xy|=oG-Thj_@!^8u=`^08a>GLn2NKg7!4vlV*pyOM7Hj zT%al{ErQoF#2g*D?{j$PPzdCIXy-=w%ycfwFV6OY1(UaF**la?3P37RT?mdzU|fP` zTz{mEXQqW#2HEHChD%iuJEsH8_+MX@D$#X)hFWgLng@&m-VrbkM`i|Qx>O$wo^-LG zv`<6g^JjL4g8LOt)atVt?44OEQh9AX9QmDe?V7JC+jETbR3P7U7dX&bnV+H-Ej=(A z3m}w@QLIdIIsee!uRn+YYQ@7dug)w(c>ma+ef$j*XYCh==d&ZlBsEk0iPqz1F&W@0 zYh^rWEP33{y~^xZ*|T31+on0WE-m6zdtL6tF5do5iFu170OPN2L`}6^*te4zGu!sf zG7GPfcob}p4KUQ9`YdtT*}qG~7r;N#=N*>7?=85Aq7+X9jvLBfx34wirwb{&6{FR< zGN@N;Ebi#O^_%f&pS{KP>1?r(cS*2&RCzv)quqKf~LUt#pH>4!==>53CBnHfRk)|VK0zQg}+cC zC8AT%ZFZBLT=WSq)vf^{3AA+=Ic`8W=*%jTTNmGIP=&bv3grY0Ra#J{6;UpoqZ=Z! zB{tE@MyQJ@3L09i=P{z+h;`^1@E}9|%5occ?8>pKcQmjOjA8>v%>C zGKrQgv6A0?3khBf+Zog%c))q&D(|vB^CQ$ko#x0ymq(WzuMAbe1}4}_Dat%@nKmaf zz`B`IbXt=8_|JmoJI424c|1OY#yl79t(A(V+dY3wJCh+MMvY<&I-@S#7zeT*!fW>x z?fEClo6e#rZ+LV_MOkfz;?+PIg@@OU_u{9G=g0g00&SOb&W!s4c?%4xE5ysny+5>(25E~7 zSeiZ1%XnAw+`+!wp-GPznN~J&bn2t-%zbKZ)h>`c`jZ+Jr#50&$)Y@oR0BPl7t|Rj zo8uG0!v-dvtZwTuZSyu|#Z?wao7*1c%8p3bxgntb+l>I8FXMpvyhQafK6frO zRN~WpY8`|qghYlJ0gkw~_ELHqz4%-DG?_yD87Yn%QHUr!Yt`%%gHIMVqTbMWSWc>p zqz%rc4d&PTrL1T)JRpzeDA@oFLZdi-5je+3mCQ;jr@6o^yIWT|qQ6*QUCu)CxL#+4 z?f?$)yeOp88l~v`@yIegnc-=_6|a8;vH|IZji{f-rbf+ud6JR6A)mJAeblRjZ4yNJ zhKT~vFCoeq3;oY}HP`sVNw~yY<9_-1&7~d7{0yhpItmIL$K^jwRc)T7hE2-7lfu!? zsIx^+#V_%ufYdG73Z%gN#z#_Zi&a1x<&+8~=imM+pCZO^e1H9kta@h6PU(kTVmChF~j11R^`c@M94B<24DJ z9H$>cXN=-+KO|UqSBAa4s|AyiSdfM<&!tLm%^xoHsfAm5+OXoNS#} zu@`0+xCZ(0uN4J2r3she1Q84IX%23*oq&WC@8b^3dxmJUjn9Siy8Ff@w?K}a+f=Zx zvT)$EALcm39jqXcvWx0h*zRCB4yzUq#)qd8_P1PuHnay6}a=5;ayQXcP zL-#<$+koxEtO6W#aK;`S&Q%MUOt%IU`MyST9SJ-^jZ zrhxX|{~@_>J>xFKtgqnZssEf&Roit&p-IMs5X}^ev?WASn@;7@JqSEYvJq}T>XiR!e0yWS?Myq zCG5he#u$!vO_j=0Uk*c^pY|>o<_doS0P~`TSjM@|Q+|=JNp`3cORF?|M-}_ji?c)? z9m_e3KqiEgUp8Zgbe#c@3kql2*Jx94n?2~I;ug{DVrEob6KG71u}iz8|Ku?!mlC2b z#pi5UzV%Gd;{m^u$FOu88XN&dd)IgFA%K?ybV%nSpUIM~6JfqM#9OjBS+=aQtaLPp zq|lL~C+N?32Mp{RCk|Ly+T^s*Jj0i^jDy2HHA$z+zqw(GyJ_q3MhW-MJcs$CRCiH? zv`(h%P6<6Ssy|(9RT=+BcBdVF)Axr9HRnjYbP+a<8|l~ADpwDI9fS#CD}7i1hyw;) zSldg$l$8al=@!Fn!g*ZGbr?N-irlQl_QpSa7WxwsH@BKhTlCk2Tm3T1f3o!3Mw3nW zr%yX+t5$hU4=pgL)6@kzlyIGQM3a=AZM~jwHYR=+yIMbFM4m-}-{x_<5o#V=26ff3 zF1eC9@mdH$1Qjw-*_B>pBYzbK%RB)wh?}vZzU@M*U;e_6vo~qU;rCS2X?t1JFHq4c zxhT44e&E_YgCAcYxMn8jk?kWpfo7Q>dm0Zf2@6l>RcW0Q(#n!2NrdFYQtTfZZf9WR+0B z1vu_!uc?q^g=bWw!pC75oUjq%alti(L+hNgr2I`XF(=zht&WVQg^Y}h`z{t?*{6A6@2TOOk;yvG@Wz4VugA3T^CiH;@s1(Qth|-Y z!@leS=32++0+s3R&=s3-pX^hOoWC!n4V3VW!(b7*-9@YW+pQDy57vb{n;t8FMZ>EXbEa&ih@TUPLV#HnP_3w>y51n_74= zqG*Jku3)Qk*=i%`yj0~hUocL(FFso)iGeMH^Y%5M8c{TZsX6M! zP7F%FRe&2wIu}PRrJTbU>YSj>VTz1Hh|RG5agzGD;>(EBA9L{+5PF) z>>JY+byrwn}W_bmX+uu9#Emz|xf+G`r;%R*n#|E-Q#xr?x|MJaKz$k~{Ls{J%K;EdDd+om_?-3q!?>V% zn306^ThX>Tx65AJ#wucSo3e#ZUe3kebAkGktqX-1JhgIERA_M~X?S||nwm;t-oYN% zO-FCNBB$G!BX2-kfKyfMu}GpVOJOLw5Oam1o{?s5B`h(AlveVW{R7N6s8A}WRRIZ_ zJ9HtgM1=ikj|}8~>2unG&Cye8&hI?eC1yrYnXC|=($-h4juAS~_p#wD+dw-ZQ=Z8+ z0~m&4boeC@4(LgnFgp@r64?Y8JVHwhMEw|`H-p$jF%{;-JYevk?1}w-vca@(To8lI zcte6q*6L!cq({ipiJaZPGO}oyWxs;Yt_s%iuDnxWj_aIT61my-r0FfjE3{T+3neP< zoS9!c*zolN8KjH2)WA6Vd?7ob(1L4~WUjauy+PB8x6zB}WSs@Th$+?BUEa<3Q;p66n@q`ON{F=edD zPS0jyr<J-S~YIzJ=KNC!hM$bA_3S;T>obHFM2Geo9jkeYb!2FzSmX3`1akMnY zjmKt@muupDY(Xwu$Y(9!Bs`6m_q0+%BcZ=sd5AziG@?P$?}M-78t(0lJ5!G*; z59siwko-X|3*?TxAe^tvN{mp0`h}`QTw3LEt1xG@Bh+eeX0OxXre^fU1d9 z-wMpcyic_3Y~_#dBGJOUlOfP`8}XNPH*@3jxBCkv1Xx)S5;A#M&LbVG1q66*Bn}1_ zd%TYb+1_lRd2X0o82=ZYv+|8`x(eZj(cpoTZ9xuzks}vs-V0L+EbNJIg($~8{W15wHZEL z?!26(P7~Dg5janZ0zdhnM0q_&O%wob#~|EC0>t@91yoeeQsn%D&#q}wB-ok4?@^?H z{F_1H+X?J=qoGtd| zjgC1(ye@7?Ib50-lnHI8d8DX7k$~7U6UwQO9%!v%PauupT53Sx2^P*Lx@=6xDUu<; zBU~6{F+&q3q%u{r1sV?fBB@c29OF5uc#3^OoS*Zhgh5yl86S=T1yw)qsfjL+c!mpL z34d?q^;;Q0gbA-^)i-HlS)_GH_3dF6y}yqt?{jD>MHqKdn`bOE)XGYl8f?uAu>?|g->_nR zB4(Nsi8v6zPwGxjYntDF3^F}S)GBL*l5pTP7xsY7)@;v!%nt`dL|!xX&^_^xia7ml z)VF4XQ3T*1kDTh0)AyVj>>K3g5x)w>H8dDkR*^O)NtbvDi0lTJgM~nA`YN-j`uVxP z<=08oBd1Lr_h{L~Ig6S`wEMo6^AX}35v+Cv8f3Gn$Zy-aW7Cc2>Z|JIoEXjgiywHC zrm|YXAJr%;G)&R0e+)~zOUYS$n zX>Z7=P2BKN&!jMPsT`X>3ATL)L}mIlXeAZd1h9?drhE3i<+<;@llwM4+^ZtzuB5l3 z(_Bu9Ye}kUqxrneO&zV0-V{not(d?XC6WmjPXg1rgh+do*R7OA1=OkDw=QZ=9bI(G zQikZR;)g{Ez-GT+epi(ovo9?Xu>@g|Q}J0Tv*j>g7&oUE2}if^TXGa=wNLyLVCeG0aPoXVwVL7J60{kr?hKiSUvIdsu!RQ`u#7Xj`$9vkn~r?`H~6 zxV>h}zd{%Dk8Nhxgkn&(;9ZViIXw~3Y-sr+vlVKn7hh5dF~7Xf{NT1Ycf`y#B;MNe zwpiVzH=>d_KMikquiF$oD1ku<*o+uyWYIvDg+=sY9=GZ)E%+_36=f&G&%$SFGO9k~ z)f(2B7Qb)#a8g0z0!9 z+Drwu7i7@NR8pN}Ju8Y$TPPNLPmFQiwP!E&;jZe=Pp9SDMAPi~8w6|PB5vkRrt%b;QM8R&1pF-k#YC14{Ojz`z&jTOAiv-TWb|H8Gld9NZFUM zArz)D7I7L@2uGvTRU8Q^Fgz7rn_$U5>tPXBiS73AyG zJiC6;8c)A8P*t9BFMOsIyrA9mjG|43HEg#w)Eap;Ve$d5NXOQV6lkJf31d!gX8eLa zBgVMvo6{j{cx(K^Ge0zyQaGTM?}K}&N$@pKO$Mf1;PQi3cQd?W;5D7sa^-Ec1Sm(N zs<7IV)|B$kOlvR4xo#vUY*Jbrb}tgdhPNMitC{E%nb4Y}0R+#fh?rlf5nyAQ(#w>s ztNkU1%dFc~cxj{_dwA^OH?ZCNBjx~6gA3rt53)hqsMPcZ&ExK-wT9##tqHZ>Q`)c= zO`PKGp|X$C=(FTh8?beR1wSsgH8BM95mHLUYE$#5@7iYCqz!9EMRu&Oa z`8E0|I|lC|~0^?*#6MGOK_#R5w(Tl+{gD=_kj4rr7j1vw-> z+mW*T8DT}Gh7D~cUfo)y6bgsHA?vFxKh(`O-5(oJTQj|RE;nhjjSCNMU6LSAZyHsP|6ToMtV&xjy!h6M}fS*tm4eA=`t>oLu)_-PAH(n7CY`z<)n zen05}3XrftDF*2|=Ujq_aRSWk?JdrY#q~@o2Wvi6!Ic(bg43aZ8kENSL6`Xv|M{Zg z^9oNBPK2x{<}PvTl(1Adjho{Vs<6RLj(f~ByNBX9yA(Kqi6vJ2_<9%Tm?AC`7f(u_ zWl6gAdubESweOGKl*%Ib?&`q~q>grF)tx@2u^^=(MrmD}gmWQgd^?OHSZ$x}v^J#5 zZULz=igAE(^?WNqCqhzTD2U%mkpv?JBHgg83@1xe=E?#or(DB7lfV=yNx*aA>sY1H zH7fSQRE`6c42`r~2w5?Sk)n{rXt$6Tb#b8586M7r^ByjP(<-biD&VxZl$aDXK9ZIO z-_?&82nc6kASz6t>0ry!O?wr_0|4NvIIwA}29p*!iPXyaAUYE0L$20_Tbz7$1b1$Q z26raDEPU4e2PY@?W8fI=DCQqmidUE*H$^I4xkVp^nen2JeQ&gkz~Swd_{&se@^R=rD#fqmB;fc)N5~iCdx?L5s_%1wxVx-WnkQ2*{X@Q}?P4hZ zga6NP7ShR?(KXku(45m&;zb}e2dQ*vIv!Y?ZS7+@2?Q5qt^a?>Z;CYIi|0K+5p6wO zHWrm%vF7$){ewq{&iNG!^;4W`7Ebb@OZ~GRs~NE1JqcLwt~6N@Ld@n_fA;7ewX)b< z$sb&;NHphNbr97zBj5A;-~W&cp8x(`cT&y7+5Tg%`aCUvz! z$@HyWbYHo{kCRx1i7qhvR&;@%!sew)BIot^QOHq(Q&-Kr`5lYI?N-4?+?&6we^V2l z-^y?Q0~|!0V(oqy`i$y25!{Ux9Q^|n(Kx&19~$0$kZW>2R$w8&#W^9ff*{zs70#|( z4*nVFZ;9*2qK8}YhaUW79GaH}ReduyCs!m5>=*w4IOfW*obC-7u6q{aU4$N{9<^#5 zR=sa9iCk98(lT1Z3W8Mim1b}&c%q*L}>n}@#tm7(^8=LI2$VW!|1dW~M zzE@8#xY9ZQ0Z7kYhg84W9e*i=xwj9zngYN0fJS631y?%oUl1RCN&W}exu0d&W^R-@ zJ5wun`C-RQXX%KSoz-syakEPK{N2g<+hy)Y8Xt6sOksqCRTW$Z-EF+GzLpLKk1*_Bhm9Rxi zO(mKj&sEKzXlLKvv0?|UjAxdCpVvAj%_I zeT$|shrEADcAZnRm8imu%Q){=O2|UVTZz8^jU$_VF{36>NM1P|fa*Z+zI_Q9Ejcat ztu1%yxs`y~=1Y4zFkT~6YAIZLK(-?Q+}n^gow zOlVTscum`p@rG7fq<+iXC<`75pk*?a6i%qy7UsF>n%I$Y=6Y1~%NmhSe4cS-#N8(1 zuv$tnuo!hg^P#Iuwp_{sJ9xVg@Y!t%hL1K9jc;bH>d^QK%6=B5Ff-_(lM>Bq<>C)L z|4RhOXV=?L(i#w%?cR@y{&~G}*V=zjVfm9?E+$7b;yhvMV&*E(H-jr9_iJbRQ!~t;v>Nevp9YKQ3zY5jE_^Op?TVp;h!+`!+tx3+V(6`jZI8}K#O9H+S(U$I9kjDUfn{FzCPgXX6< z$Z^{5?lNPC4;@A9a#d^t*N**5Yg=+1$Td|hs$(!s;!~PMiWX%QEul&}sdCi6-d9`% zHymHW2&+O_C!+v#0k0CYtx1$ba!O-AJnU1w$SL*Mh@_B_x~r5G{`V$voo&}^<7AqXRBkc+YVLoSUHtmXi@2lmAb}EL zGKm_Y_)h_X77JNrv7Av}3Vl^estN{?N`a)UU6E!(NBt)NvMj(rMzdxKZ5-$9g96%{ zszk>slF8M{?CG3sTDR;DtCLhRf>1}5rUjwKjKacU_IaYn__lY}&D`2IG4NDnMv#EG z{1yX5<+r3~@>djx<#eUX;O1QXZ{MNZW_2s_N6j)>SWz&g z+ev81x}E#x>Q(yTfFa=>F2JchqFS5HZN4HJRO0366Ah*i#)mEtD@LiLm^X2Py0R{^ znrykXUumUD$KX1N9qxz(ht5kWcr^vG`qmMrkWU0F0}WxbMGudnGq6|(ng@FJI7)@& zD7=0vIC$(c$CG!@`D3f#2d%_|% zgJV+TXltB$?D70p=OwnIxInR23Tz-_huWui;%5pI1=o5TvENjPwvL3|z5iZ!&rTtU zy_b{+BLBL}xYBorD@D+E4TIlmSBptpRqO{^{m_|0)?F;lg(!t2nq}U|;o`UbTF9fQAZ@PKMR_j*?i$`{o{#5gh%NVQt#&EQ#OYdp|IJRCRR6o+ZHee=WSMpY zH<6b{s9%2TAiU{w!e!P#$`WP7EzJ(Ys`u+GJ3n{2J^X++w%0xK6&iUl-BxE0){8&8 z$LCxZ-5H%VCc<1pFDUp!7nF{RL7^bLZyDt<4Ztf5r!VxSmP}o7J(1m8T+cBPS$Bx= zHOT?z+OL9rt*^MZ#U;+vhRa~EJP4WEJ1HwMmAKMBAC>&GxxITL{Bypb-CX$}pjOzk zoYSaF6jyp*Pj|eG+nN*}%%al6e<`u`*S-)pq>{$d<$yw#s}~ z;^kPsQ#_7Kb}3K^(n|oH?qBu4G{5dT{s%Y=rx>r>69xrgPZV}gumqoiQtI?F3E9ib zRm%hStNWX!<2}CM81|EzADdC>hhXi(-#Et-@E>5v^B-XBk}mB4nl_O>{WF|f`Ky%% z0|R?n48MF9IF8vloWy?FOzq=8fTNoth4!wS$3e(NNPQ2G-3C(yy)a%H+wS^OfSX6R zAihJh++o^mPW~3*B!4iKO^@AK-q+SE(oH-d>?#iW$#j}CMgr!&#o5D`e$0g6#-2O~ z{!)ycPREbCQrB%)4eF=kJCCmQPqmM5D?xuhPiOCl7*Rw8N?C++_re<3Pn~gSNxL1q zQ$}!DVO~Jax#!kD04V7S_p$@WguiO{d?6s%ME(qpOaAz`Ji1Ol)*)UaWsOXve){?D zj*~1Wxnv7STCAp#jYDu8l-{~VnRf(yR$}YB+jz8ztLrFicaFwR{J@=p;V~B{?vF13 zBU1@go4$uu?M%`2?9%&AR+C?`b??n}TgVaTErcapT^))EG|s+D`%X%g@^Ais0oZ%Q Ar2qf` literal 0 HcmV?d00001 diff --git a/examples/frontend_language/quick_start/local_example_chat.py b/examples/frontend_language/quick_start/local_example_chat.py new file mode 100644 index 000000000..e1e4b62cc --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_chat.py @@ -0,0 +1,75 @@ +""" +Usage: +python3 local_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/local_example_complete.py b/examples/frontend_language/quick_start/local_example_complete.py new file mode 100644 index 000000000..00a451cf6 --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_complete.py @@ -0,0 +1,70 @@ +""" +Usage: +python3 local_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Llama-2-7b-chat-hf") + sgl.set_default_backend(runtime) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/local_example_llava_next.py b/examples/frontend_language/quick_start/local_example_llava_next.py new file mode 100644 index 000000000..c941a549e --- /dev/null +++ b/examples/frontend_language/quick_start/local_example_llava_next.py @@ -0,0 +1,78 @@ +""" +Usage: python3 local_example_llava_next.py +""" + +import sglang as sgl +from sglang.lang.chat_template import get_chat_template + + +@sgl.function +def image_qa(s, image_path, question): + s += sgl.user(sgl.image(image_path) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(): + state = image_qa.run( + image_path="images/cat.jpeg", question="What is this?", max_new_tokens=128 + ) + print(state["answer"], "\n") + + +def stream(): + state = image_qa.run( + image_path="images/cat.jpeg", + question="What is this?", + max_new_tokens=64, + stream=True, + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = image_qa.run_batch( + [ + {"image_path": "images/cat.jpeg", "question": "What is this?"}, + {"image_path": "images/dog.jpeg", "question": "What is this?"}, + ], + max_new_tokens=128, + ) + for s in states: + print(s["answer"], "\n") + + +if __name__ == "__main__": + import multiprocessing as mp + + mp.set_start_method("spawn", force=True) + + runtime = sgl.Runtime(model_path="lmms-lab/llama3-llava-next-8b") + runtime.endpoint.chat_template = get_chat_template("llama-3-instruct-llava") + + # Or you can use the 72B model + # runtime = sgl.Runtime(model_path="lmms-lab/llava-next-72b", tp_size=8) + # runtime.endpoint.chat_template = get_chat_template("chatml-llava") + + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Or you can use API models + # sgl.set_default_backend(sgl.OpenAI("gpt-4-vision-preview")) + # sgl.set_default_backend(sgl.VertexAI("gemini-pro-vision")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() + + runtime.shutdown() diff --git a/examples/frontend_language/quick_start/openai_example_chat.py b/examples/frontend_language/quick_start/openai_example_chat.py new file mode 100644 index 000000000..9511e21cf --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_chat.py @@ -0,0 +1,74 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_complete.py b/examples/frontend_language/quick_start/openai_example_complete.py new file mode 100644 index 000000000..d64bcaf1c --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_complete.py @@ -0,0 +1,68 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_complete.py +""" + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_n.py b/examples/frontend_language/quick_start/openai_example_n.py new file mode 100644 index 000000000..25372b9f4 --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_n.py @@ -0,0 +1,71 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=1024, n=2)) + s += sgl.user(question_2) + s += sgl.assistant( + sgl.gen( + "answer_2", + max_tokens=1024, + ) + ) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + assert isinstance(state["answer_1"], list) + assert len(state["answer_1"]) == 2 + assert isinstance(state["answer_2"], str) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + print("\n-- answer_1 --\n", s["answer_1"]) + print("\n-- answer_2 --\n", s["answer_2"]) + assert isinstance(s["answer_1"], list) + assert len(s["answer_1"]) == 2 + assert isinstance(s["answer_2"], str) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openai_example_o1.py b/examples/frontend_language/quick_start/openai_example_o1.py new file mode 100644 index 000000000..2e5c14002 --- /dev/null +++ b/examples/frontend_language/quick_start/openai_example_o1.py @@ -0,0 +1,57 @@ +""" +Usage: +export OPENAI_API_KEY=sk-****** +python3 openai_example_chat.py +""" + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=100)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2")) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.OpenAI("o1")) + + # Run a single request + print("\n========== single ==========\n") + single() + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/openrouter_example_chat.py b/examples/frontend_language/quick_start/openrouter_example_chat.py new file mode 100644 index 000000000..a0b6f15bc --- /dev/null +++ b/examples/frontend_language/quick_start/openrouter_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export OPENROUTER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="google/gemma-7b-it:free", + base_url="https://openrouter.ai/api/v1", + api_key=os.environ.get("OPENROUTER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/together_example_chat.py b/examples/frontend_language/quick_start/together_example_chat.py new file mode 100644 index 000000000..2d2059062 --- /dev/null +++ b/examples/frontend_language/quick_start/together_example_chat.py @@ -0,0 +1,81 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_chat.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +def single(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + + +def stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print() + + +def batch(): + states = multi_turn_question.run_batch( + [ + { + "question_1": "What is the capital of the United States?", + "question_2": "List two local attractions.", + }, + { + "question_1": "What is the capital of France?", + "question_2": "What is the population of this city?", + }, + ] + ) + + for s in states: + print(s.messages()) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/quick_start/together_example_complete.py b/examples/frontend_language/quick_start/together_example_complete.py new file mode 100644 index 000000000..d9119ed6c --- /dev/null +++ b/examples/frontend_language/quick_start/together_example_complete.py @@ -0,0 +1,76 @@ +""" +Usage: +export TOGETHER_API_KEY=sk-****** +python3 together_example_complete.py +""" + +import os + +import sglang as sgl + + +@sgl.function +def few_shot_qa(s, question): + s += """The following are questions with answers. +Q: What is the capital of France? +A: Paris +Q: What is the capital of Germany? +A: Berlin +Q: What is the capital of Italy? +A: Rome +""" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + +def single(): + state = few_shot_qa.run(question="What is the capital of the United States?") + answer = state["answer"].strip().lower() + + assert "washington" in answer, f"answer: {state['answer']}" + + print(state.text()) + + +def stream(): + state = few_shot_qa.run( + question="What is the capital of the United States?", stream=True + ) + + for out in state.text_iter("answer"): + print(out, end="", flush=True) + print() + + +def batch(): + states = few_shot_qa.run_batch( + [ + {"question": "What is the capital of the United States?"}, + {"question": "What is the capital of China?"}, + ] + ) + + for s in states: + print(s["answer"]) + + +if __name__ == "__main__": + backend = sgl.OpenAI( + model_name="mistralai/Mixtral-8x7B-Instruct-v0.1", + is_chat_model=False, + base_url="https://api.together.xyz/v1", + api_key=os.environ.get("TOGETHER_API_KEY"), + ) + sgl.set_default_backend(backend) + + # Run a single request + print("\n========== single ==========\n") + single() + + # Stream output + print("\n========== stream ==========\n") + stream() + + # Run a batch of requests + print("\n========== batch ==========\n") + batch() diff --git a/examples/frontend_language/usage/chinese_regex.py b/examples/frontend_language/usage/chinese_regex.py new file mode 100644 index 000000000..78e9c7e16 --- /dev/null +++ b/examples/frontend_language/usage/chinese_regex.py @@ -0,0 +1,53 @@ +import sglang as sgl + +character_regex = ( + r"""\{\n""" + + r""" "姓名": "[^"]{1,32}",\n""" + + r""" "学院": "(格兰芬多|赫奇帕奇|拉文克劳|斯莱特林)",\n""" + + r""" "血型": "(纯血|混血|麻瓜)",\n""" + + r""" "职业": "(学生|教师|傲罗|魔法部|食死徒|凤凰社成员)",\n""" + + r""" "魔杖": \{\n""" + + r""" "材质": "[^"]{1,32}",\n""" + + r""" "杖芯": "[^"]{1,32}",\n""" + + r""" "长度": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "存活": "(存活|死亡)",\n""" + + r""" "守护神": "[^"]{1,32}",\n""" + + r""" "博格特": "[^"]{1,32}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += name + " 是一名哈利波特系列小说中的角色。请填写以下关于这个角色的信息。" + s += """\ +这是一个例子 +{ + "姓名": "哈利波特", + "学院": "格兰芬多", + "血型": "混血", + "职业": "学生", + "魔杖": { + "材质": "冬青木", + "杖芯": "凤凰尾羽", + "长度": 11.0 + }, + "存活": "存活", + "守护神": "麋鹿", + "博格特": "摄魂怪" +} +""" + s += f"现在请你填写{name}的信息：\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def main(): + backend = sgl.RuntimeEndpoint("http://localhost:30000") + sgl.set_default_backend(backend) + ret = character_gen.run(name="赫敏格兰杰", temperature=0) + print(ret.text()) + + +if __name__ == "__main__": + main() diff --git a/examples/frontend_language/usage/choices_logprob.py b/examples/frontend_language/usage/choices_logprob.py new file mode 100644 index 000000000..6cd733fe9 --- /dev/null +++ b/examples/frontend_language/usage/choices_logprob.py @@ -0,0 +1,44 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python choices_logprob.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ", " + s += "I need to use a " + sgl.gen("tool", choices=["calculator", "search engine"]) + + +def main(): + # Run one case + question = "What is 5 + 5?" + state = tool_use.run(question) + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + # Run a batch + questions = [ + "What is 5 + 6?", + "Who is Michael Jordan?", + ] + states = tool_use.run_batch([{"question": q} for q in questions]) + for question, state in zip(questions, states): + print("questions:", question) + print("choice:", state["tool"]) + meta_info = state.get_meta_info("tool") + print("logprobs of choice 1", meta_info["input_token_logprobs"][0]) + print("logprobs of choice 2", meta_info["input_token_logprobs"][1]) + print("-" * 50) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + main() diff --git a/examples/frontend_language/usage/cot_decoding.py b/examples/frontend_language/usage/cot_decoding.py new file mode 100644 index 000000000..7a7a04bce --- /dev/null +++ b/examples/frontend_language/usage/cot_decoding.py @@ -0,0 +1,115 @@ +from math import exp +from pprint import pformat + +import sglang as sgl + +YELLOW = "\033[1;33m" +GREEN = "\033[1;32m" +BLUE = "\033[1;34m" +CLEAR = "\033[1;0m" + + +@sgl.function +def cot_decoding(s, question, get_top_k, is_chat_model, verbose): + """CoT Decoding: http://arxiv.org/abs/2402.10200""" + + if is_chat_model: + s += sgl.user("Question: " + question + "\nAnswer:") + s += sgl.assistant_begin() + else: + s += "Question: " + question + "\nAnswer:" + + step_0 = s.fork(1)[0] + forks = s.fork(get_top_k) + answer_forks = s.fork(get_top_k) + + # decoding step 0 + step_0 += sgl.gen( + "get_top_k", + max_tokens=0, + return_logprob=True, + top_logprobs_num=get_top_k, + return_text_in_logprobs=True, + ) + logprobs = step_0.get_meta_info("get_top_k")["output_top_logprobs"][0] + + print("Decoding step 0:", ", ".join(pformat(token[2]) for token in logprobs)) + for idx, (f, token) in enumerate(zip(forks, logprobs)): + logprob, token_id, text = token + f += text + + if text == "<|end_of_text|>": + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score=nan, answer=nan){CLEAR}" + ) + continue + + # continue greedy decoding + f += sgl.gen( + "answer", + temperature=0, + max_tokens=1024, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + + # calculate probability disparity between the top and secondary tokens + x1s = [exp(xt[0][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + x2s = [exp(xt[1][0]) for xt in f.get_meta_info("answer")["output_top_logprobs"]] + tokens = [xt[0][2] for xt in f.get_meta_info("answer")["output_top_logprobs"]] + delta = (sum(x1s) - sum(x2s)) / len(x1s) + + # extract the answer span (without the '<|end_of_text|>' token) + answer_forks[idx] += text + f["answer"] + "\nSo the answer is" + answer_forks[idx] += sgl.gen( + "answer_span", + temperature=0, + max_tokens=64, + return_logprob=True, + top_logprobs_num=2, + return_text_in_logprobs=True, + ) + answer = answer_forks[idx]["answer_span"].replace("\n", " ").strip(":") + print( + f"{YELLOW}Path #{idx} {pformat(text)}[{exp(logprob):.3f}] (score={delta}, answer={answer}){CLEAR}" + ) + generated_text = str(answer_forks[idx])[len("ProgramState(") : -1] + print(f"{BLUE}{pformat(generated_text)}{CLEAR}") + + if verbose: + answer_tokens = [ + xt[0][2] + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x1s = [ + exp(xt[0][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + answer_x2s = [ + exp(xt[1][0]) + for xt in answer_forks[idx].get_meta_info("answer_span")[ + "output_top_logprobs" + ] + ] + + for token, x1, x2 in zip(tokens, x1s, x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print("\n===========") + for token, x1, x2 in zip(answer_tokens, answer_x1s, answer_x2s): + print(f" {GREEN}{pformat(token)}{CLEAR}({x1:.3f}-{x2:.3f})", end="") + print() + + +sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = cot_decoding.run( + question=r"Claire makes a 3 egg omelet every morning for breakfast. How many dozens of eggs will she eat in 4 weeks?", + get_top_k=10, + is_chat_model=True, + verbose=False, +) diff --git a/examples/frontend_language/usage/json_decode.py b/examples/frontend_language/usage/json_decode.py new file mode 100644 index 000000000..5dc3522d5 --- /dev/null +++ b/examples/frontend_language/usage/json_decode.py @@ -0,0 +1,83 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python json_decode.py +""" + +from enum import Enum + +from pydantic import BaseModel + +import sglang as sgl +from sglang.srt.constrained.outlines_backend import build_regex_from_object + +character_regex = ( + r"""\{\n""" + + r""" "name": "[\w\d\s]{1,16}",\n""" + + r""" "house": "(Gryffindor|Slytherin|Ravenclaw|Hufflepuff)",\n""" + + r""" "blood status": "(Pure-blood|Half-blood|Muggle-born)",\n""" + + r""" "occupation": "(student|teacher|auror|ministry of magic|death eater|order of the phoenix)",\n""" + + r""" "wand": \{\n""" + + r""" "wood": "[\w\d\s]{1,16}",\n""" + + r""" "core": "[\w\d\s]{1,16}",\n""" + + r""" "length": [0-9]{1,2}\.[0-9]{0,2}\n""" + + r""" \},\n""" + + r""" "alive": "(Alive|Deceased)",\n""" + + r""" "patronus": "[\w\d\s]{1,16}",\n""" + + r""" "bogart": "[\w\d\s]{1,16}"\n""" + + r"""\}""" +) + + +@sgl.function +def character_gen(s, name): + s += ( + name + + " is a character in Harry Potter. Please fill in the following information about this character.\n" + ) + s += "The constrained regex is:\n" + s += character_regex + "\n" + s += "The JSON output is:\n" + s += sgl.gen("json_output", max_tokens=256, regex=character_regex) + + +def driver_character_gen(): + state = character_gen.run(name="Hermione Granger") + print(state.text()) + + +class Weapon(str, Enum): + sword = "sword" + axe = "axe" + mace = "mace" + spear = "spear" + bow = "bow" + crossbow = "crossbow" + + +class Wizard(BaseModel): + name: str + age: int + weapon: Weapon + + +@sgl.function +def pydantic_wizard_gen(s): + s += "Give me a description about a wizard in the JSON format.\n" + s += sgl.gen( + "character", + max_tokens=128, + temperature=0, + regex=build_regex_from_object(Wizard), # Requires pydantic >= 2.0 + ) + + +def driver_pydantic_wizard_gen(): + state = pydantic_wizard_gen.run() + print(state.text()) + + +if __name__ == "__main__": + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + driver_character_gen() + # driver_pydantic_wizard_gen() diff --git a/examples/frontend_language/usage/json_logprobs.py b/examples/frontend_language/usage/json_logprobs.py new file mode 100644 index 000000000..15206a619 --- /dev/null +++ b/examples/frontend_language/usage/json_logprobs.py @@ -0,0 +1,103 @@ +# NOTE: Currently this can only be run through HTTP requests. +from concurrent.futures import ThreadPoolExecutor + +from json_decode import character_regex + +from sglang.utils import http_request + +character_names = ["Hermione Granger", "Ron Weasley", "Harry Potter"] + +base_url = "http://localhost:30000" + +prompt = "is a character in Harry Potter. Please fill in the following information about this character.\n" + + +def openai_api_request(name): + data = { + "model": "", + "prompt": name + prompt, + "temperature": 0, + "max_tokens": 128, + "regex": character_regex, + "logprobs": 3, + } + res = http_request(base_url + "/v1/completions", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + logprobs = res["choices"][0]["logprobs"] + usage = res["usage"] + assert len(logprobs["token_logprobs"]) == len(logprobs["tokens"]) + assert len(logprobs["token_logprobs"]) == len(logprobs["top_logprobs"]) + assert len(logprobs["token_logprobs"]) == usage["completion_tokens"] - 1 + + return res + + +def srt_api_request(name): + data = { + "text": name + prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + "regex": character_regex, + }, + "return_logprob": True, + "logprob_start_len": 0, + "top_logprobs_num": 3, + "return_text_in_logprobs": True, + } + + res = http_request(base_url + "/generate", json=data).json() + + # with open(f"json_logprobs_{name.replace(' ', '_')}_tmp.json", "w") as fout: + # fout.write(json.dumps(res, indent=4)) + + meta_info = res["meta_info"] + assert len(meta_info["input_token_logprobs"]) == len( + meta_info["input_top_logprobs"] + ) + assert len(meta_info["output_token_logprobs"]) == len( + meta_info["output_top_logprobs"] + ) + assert len(meta_info["input_token_logprobs"]) == meta_info["prompt_tokens"] + assert len(meta_info["output_token_logprobs"]) == meta_info["completion_tokens"] - 1 + + return res + + +def pretty_print(res): + meta_info = res["meta_info"] + + print("\n\n", "=" * 30, "Prefill", "=" * 30) + for i in range(len(meta_info["input_token_logprobs"])): + print(f"{str(meta_info['input_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = ( + [str(t[2].encode()) for t in meta_info["input_top_logprobs"][i]] + if meta_info["input_top_logprobs"][i] + else [] + ) + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print("\n\n", "=" * 30, "Decode", "=" * 30) + for i in range(len(meta_info["output_token_logprobs"])): + print(f"{str(meta_info['output_token_logprobs'][i][2].encode()): <20}", end="") + top_ks = [str(t[2].encode()) for t in meta_info["output_top_logprobs"][i]] + for top_k in top_ks: + print(f"{top_k: <15}", end="") + print() + + print(res["text"]) + + +if __name__ == "__main__": + with ThreadPoolExecutor() as executor: + ress = executor.map(srt_api_request, character_names) + + for res in ress: + pretty_print(res) + + openai_api_request("Hermione Granger") diff --git a/examples/frontend_language/usage/llava_video/srt_example_llava_v.py b/examples/frontend_language/usage/llava_video/srt_example_llava_v.py new file mode 100644 index 000000000..ec5b334b0 --- /dev/null +++ b/examples/frontend_language/usage/llava_video/srt_example_llava_v.py @@ -0,0 +1,260 @@ +""" +Usage: +pip install opencv-python-headless + +python3 srt_example_llava_v.py +""" + +import argparse +import csv +import json +import os +import time + +import requests + +import sglang as sgl + + +@sgl.function +def video_qa(s, num_frames, video_path, question): + s += sgl.user(sgl.video(video_path, num_frames) + question) + s += sgl.assistant(sgl.gen("answer")) + + +def single(path, num_frames=16): + state = video_qa.run( + num_frames=num_frames, + video_path=path, + question="Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes", + temperature=0.0, + max_new_tokens=1024, + ) + print(state["answer"], "\n") + + +def split_into_chunks(lst, num_chunks): + """Split a list into a specified number of chunks.""" + # Calculate the chunk size using integer division. Note that this may drop some items if not evenly divisible. + chunk_size = len(lst) // num_chunks + + if chunk_size == 0: + chunk_size = len(lst) + # Use list comprehension to generate chunks. The last chunk will take any remainder if the list size isn't evenly divisible. + chunks = [lst[i : i + chunk_size] for i in range(0, len(lst), chunk_size)] + # Ensure we have exactly num_chunks chunks, even if some are empty + chunks.extend([[] for _ in range(num_chunks - len(chunks))]) + return chunks + + +def save_batch_results(batch_video_files, states, cur_chunk, batch_idx, save_dir): + csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(csv_filename, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["video_name", "answer"]) + for video_path, state in zip(batch_video_files, states): + video_name = os.path.basename(video_path) + writer.writerow([video_name, state["answer"]]) + + +def compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir): + final_csv_filename = f"{save_dir}/final_results_chunk_{cur_chunk}.csv" + with open(final_csv_filename, "w", newline="") as final_csvfile: + writer = csv.writer(final_csvfile) + writer.writerow(["video_name", "answer"]) + for batch_idx in range(num_batches): + batch_csv_filename = f"{save_dir}/chunk_{cur_chunk}_batch_{batch_idx}.csv" + with open(batch_csv_filename, "r") as batch_csvfile: + reader = csv.reader(batch_csvfile) + next(reader) # Skip header row + for row in reader: + writer.writerow(row) + os.remove(batch_csv_filename) + + +def find_video_files(video_dir): + # Check if the video_dir is actually a file + if os.path.isfile(video_dir): + # If it's a file, return it as a single-element list + return [video_dir] + + # Original logic to find video files in a directory + video_files = [] + for root, dirs, files in os.walk(video_dir): + for file in files: + if file.endswith((".mp4", ".avi", ".mov")): + video_files.append(os.path.join(root, file)) + return video_files + + +def batch(video_dir, save_dir, cur_chunk, num_chunks, num_frames=16, batch_size=64): + video_files = find_video_files(video_dir) + chunked_video_files = split_into_chunks(video_files, num_chunks)[cur_chunk] + num_batches = 0 + + for i in range(0, len(chunked_video_files), batch_size): + batch_video_files = chunked_video_files[i : i + batch_size] + print(f"Processing batch of {len(batch_video_files)} video(s)...") + + if not batch_video_files: + print("No video files found in the specified directory.") + return + + batch_input = [ + { + "num_frames": num_frames, + "video_path": video_path, + "question": "Please provide a detailed description of the video, focusing on the main subjects, their actions, the background scenes.", + } + for video_path in batch_video_files + ] + + start_time = time.perf_counter() + states = video_qa.run_batch(batch_input, max_new_tokens=512, temperature=0.2) + total_time = time.perf_counter() - start_time + average_time = total_time / len(batch_video_files) + print( + f"Number of videos in batch: {len(batch_video_files)}. Average processing time per video: {average_time:.2f} seconds. Total time for this batch: {total_time:.2f} seconds" + ) + + save_batch_results(batch_video_files, states, cur_chunk, num_batches, save_dir) + num_batches += 1 + + compile_and_cleanup_final_results(cur_chunk, num_batches, save_dir) + + +if __name__ == "__main__": + + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() # Raise an exception for bad responses + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + # Create the parser + parser = argparse.ArgumentParser( + description="Run video processing with specified port." + ) + + # Add an argument for the port + parser.add_argument( + "--port", + type=int, + default=30000, + help="The master port for distributed serving.", + ) + parser.add_argument( + "--chunk-idx", type=int, default=0, help="The index of the chunk to process." + ) + parser.add_argument( + "--num-chunks", type=int, default=8, help="The number of chunks to process." + ) + parser.add_argument( + "--save-dir", + type=str, + default="./work_dirs/llava_video", + help="The directory to save the processed video files.", + ) + parser.add_argument( + "--video-dir", + type=str, + default=os.path.expanduser("~/.cache/jobs.mp4"), + help="The directory or path for the processed video files.", + ) + parser.add_argument( + "--model-path", + type=str, + default="lmms-lab/LLaVA-NeXT-Video-7B", + help="The model path for the video processing.", + ) + parser.add_argument( + "--num-frames", + type=int, + default=16, + help="The number of frames to process in each video.", + ) + parser.add_argument("--mm_spatial_pool_stride", type=int, default=2) + + # Parse the arguments + args = parser.parse_args() + cur_port = args.port + cur_chunk = args.chunk_idx + num_chunks = args.num_chunks + num_frames = args.num_frames + + if "34b" in args.model_path.lower(): + tokenizer_path = "liuhaotian/llava-v1.6-34b-tokenizer" + elif "7b" in args.model_path.lower(): + tokenizer_path = "llava-hf/llava-1.5-7b-hf" + else: + print("Invalid model path. Please specify a valid model path.") + exit() + + model_override_args = {} + model_override_args["mm_spatial_pool_stride"] = args.mm_spatial_pool_stride + model_override_args["architectures"] = ["LlavaVidForCausalLM"] + model_override_args["num_frames"] = args.num_frames + model_override_args["model_type"] = "llava" + + if "34b" in args.model_path.lower(): + model_override_args["image_token_index"] = 64002 + + if args.num_frames == 32: + model_override_args["rope_scaling"] = {"factor": 2.0, "rope_type": "linear"} + model_override_args["max_sequence_length"] = 4096 * 2 + model_override_args["tokenizer_model_max_length"] = 4096 * 2 + elif args.num_frames < 32: + pass + else: + print( + "The maximum number of frames to process is 32. Please specify a valid number of frames." + ) + exit() + + runtime = sgl.Runtime( + model_path=args.model_path, # "liuhaotian/llava-v1.6-vicuna-7b", + tokenizer_path=tokenizer_path, + port=cur_port, + json_model_override_args=json.dumps(model_override_args), + tp_size=1, + ) + sgl.set_default_backend(runtime) + print(f"chat template: {runtime.endpoint.chat_template.name}") + + # Run a single request + print("\n========== single ==========\n") + root = args.video_dir + if os.path.isfile(root): + video_files = [root] + else: + video_files = [ + os.path.join(root, f) + for f in os.listdir(root) + if f.endswith((".mp4", ".avi", ".mov")) + ] # Add more extensions if needed + start_time = time.perf_counter() # Start time for processing a single video + for cur_video in video_files[:1]: + print(cur_video) + single(cur_video, num_frames) + end_time = time.perf_counter() # End time for processing a single video + total_time = end_time - start_time + average_time = total_time / len( + video_files + ) # Calculate the average processing time + print(f"Average processing time per video: {average_time:.2f} seconds") + runtime.shutdown() + + # # Run a batch of requests + # print("\n========== batch ==========\n") + # if not os.path.exists(args.save_dir): + # os.makedirs(args.save_dir) + # batch(args.video_dir, args.save_dir, cur_chunk, num_chunks, num_frames, num_chunks) + # runtime.shutdown() diff --git a/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh b/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh new file mode 100755 index 000000000..ffb1af96d --- /dev/null +++ b/examples/frontend_language/usage/llava_video/srt_example_llava_v.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +##### USAGE ##### +# - First node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 0 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - Second node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K 1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` +# - The K node: +# ```sh +# bash examples/usage/llava_video/srt_example_llava_v.sh K K-1 YOUR_VIDEO_PATH YOUR_MODEL_PATH FRAMES_PER_VIDEO +# ``` + + +# Replace `K`, `YOUR_VIDEO_PATH`, `YOUR_MODEL_PATH`, and `FRAMES_PER_VIDEO` with your specific details. +# CURRENT_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CURRENT_ROOT=$(dirname "$0") + +echo ${CURRENT_ROOT} + +cd ${CURRENT_ROOT} + +export PYTHONWARNINGS=ignore + +START_TIME=$(date +%s) # Capture start time + +NUM_NODES=$1 + +CUR_NODES_IDX=$2 + +VIDEO_DIR=$3 + +MODEL_PATH=$4 + +NUM_FRAMES=$5 + + +# FRAME_FORMAT=$6 + +# FRAME_FORMAT=$(echo $FRAME_FORMAT | tr '[:lower:]' '[:upper:]') + +# # Check if FRAME_FORMAT is either JPEG or PNG +# if [[ "$FRAME_FORMAT" != "JPEG" && "$FRAME_FORMAT" != "PNG" ]]; then +# echo "Error: FRAME_FORMAT must be either JPEG or PNG." +# exit 1 +# fi + +# export TARGET_FRAMES=$TARGET_FRAMES + +echo "Each video you will sample $NUM_FRAMES frames" + +# export FRAME_FORMAT=$FRAME_FORMAT + +# echo "The frame format is $FRAME_FORMAT" + +# Assuming GPULIST is a bash array containing your GPUs +GPULIST=(0 1 2 3 4 5 6 7) +LOCAL_CHUNKS=${#GPULIST[@]} + +echo "Number of GPUs in GPULIST: $LOCAL_CHUNKS" + +ALL_CHUNKS=$((NUM_NODES * LOCAL_CHUNKS)) + +# Calculate GPUs per chunk +GPUS_PER_CHUNK=1 + +echo $GPUS_PER_CHUNK + +for IDX in $(seq 1 $LOCAL_CHUNKS); do + ( + START=$(((IDX-1) * GPUS_PER_CHUNK)) + LENGTH=$GPUS_PER_CHUNK # Length for slicing, not the end index + + CHUNK_GPUS=(${GPULIST[@]:$START:$LENGTH}) + + # Convert the chunk GPUs array to a comma-separated string + CHUNK_GPUS_STR=$(IFS=,; echo "${CHUNK_GPUS[*]}") + + LOCAL_IDX=$((CUR_NODES_IDX * LOCAL_CHUNKS + IDX)) + + echo "Chunk $(($LOCAL_IDX - 1)) will run on GPUs $CHUNK_GPUS_STR" + + # Calculate the port for this chunk. Ensure it's incremented by 5 for each chunk. + PORT=$((10000 + RANDOM % 55536)) + + MAX_RETRIES=10 + RETRY_COUNT=0 + COMMAND_STATUS=1 # Initialize as failed + + while [ $RETRY_COUNT -lt $MAX_RETRIES ] && [ $COMMAND_STATUS -ne 0 ]; do + echo "Running chunk $(($LOCAL_IDX - 1)) on GPUs $CHUNK_GPUS_STR with port $PORT. Attempt $(($RETRY_COUNT + 1))" + +#!/bin/bash + CUDA_VISIBLE_DEVICES=$CHUNK_GPUS_STR python3 srt_example_llava_v.py \ + --port $PORT \ + --num-chunks $ALL_CHUNKS \ + --chunk-idx $(($LOCAL_IDX - 1)) \ + --save-dir work_dirs/llava_next_video_inference_results \ + --video-dir $VIDEO_DIR \ + --model-path $MODEL_PATH \ + --num-frames $NUM_FRAMES #& + + wait $! # Wait for the process to finish and capture its exit status + COMMAND_STATUS=$? + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)), attempt $(($RETRY_COUNT + 1)). Retrying..." + RETRY_COUNT=$(($RETRY_COUNT + 1)) + sleep 180 # Wait a bit before retrying + else + echo "Execution succeeded for chunk $(($LOCAL_IDX - 1))." + fi + done + + if [ $COMMAND_STATUS -ne 0 ]; then + echo "Execution failed for chunk $(($LOCAL_IDX - 1)) after $MAX_RETRIES attempts." + fi + ) #& + sleep 2 # Slight delay to stagger the start times +done + +wait + +cat work_dirs/llava_next_video_inference_results/final_results_chunk_*.csv > work_dirs/llava_next_video_inference_results/final_results_node_${CUR_NODES_IDX}.csv + +END_TIME=$(date +%s) # Capture end time +ELAPSED_TIME=$(($END_TIME - $START_TIME)) +echo "Total execution time: $ELAPSED_TIME seconds." diff --git a/examples/frontend_language/usage/openai_chat_speculative.py b/examples/frontend_language/usage/openai_chat_speculative.py new file mode 100644 index 000000000..f3fd74ed8 --- /dev/null +++ b/examples/frontend_language/usage/openai_chat_speculative.py @@ -0,0 +1,155 @@ +""" +Usage: +***Note: for speculative execution to work, user must put all "gen" in "assistant". +Show in "assistant" the desired answer format. Each "gen" term should have a stop token. +The stream mode is not supported in speculative execution. + +E.g. +correct: + sgl.assistant("\nName:" + sgl.gen("name", stop="\n") + "\nBirthday:" + sgl.gen("birthday", stop="\n") + "\nJob:" + sgl.gen("job", stop="\n")) +incorrect: + s += sgl.assistant("\nName:" + sgl.gen("name", stop="\n")) + s += sgl.assistant("\nBirthday:" + sgl.gen("birthday", stop="\n")) + s += sgl.assistant("\nJob:" + sgl.gen("job", stop="\n")) + +export OPENAI_API_KEY=sk-****** +python3 openai_chat_speculative.py +""" + +import sglang as sgl +from sglang import OpenAI, function, set_default_backend + + +@function(num_api_spec_tokens=256) +def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function(num_api_spec_tokens=256) +def gen_character_spec_no_few_shot(s): + s += sgl.user("Construct a character. For each field stop with a newline\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nAge:" + + sgl.gen("age", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + +@function +def gen_character_normal(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("What's the answer of 23 + 8?") + s += sgl.assistant(sgl.gen("answer", max_tokens=64)) + + +@function(num_api_spec_tokens=1024) +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Answer questions in the following format:") + s += sgl.user( + "Question 1: What is the capital of France?\nQuestion 2: What is the population of this city?\n" + ) + s += sgl.assistant( + "Answer 1: The capital of France is Paris.\nAnswer 2: The population of Paris in 2024 is estimated to be around 2.1 million for the city proper.\n" + ) + s += sgl.user("Question 1: " + question_1 + "\nQuestion 2: " + question_2) + s += sgl.assistant( + "Answer 1: " + + sgl.gen("answer_1", stop="\n") + + "\nAnswer 2: " + + sgl.gen("answer_2", stop="\n") + ) + + +def test_spec_single_turn(): + backend.token_usage.reset() + + state = gen_character_spec.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("-- birthday:", state["birthday"]) + print("-- job:", state["job"]) + print(backend.token_usage) + + +def test_inaccurate_spec_single_turn(): + state = gen_character_spec_no_few_shot.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- name:", state["name"]) + print("\n-- age:", state["age"]) + print("\n-- job:", state["job"]) + + +def test_normal_single_turn(): + state = gen_character_normal.run() + for m in state.messages(): + print(m["role"], ":", m["content"]) + + +def test_spec_multi_turn(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions in the capital of the United States.", + ) + + for m in state.messages(): + print(m["role"], ":", m["content"]) + + print("\n-- answer_1 --\n", state["answer_1"]) + print("\n-- answer_2 --\n", state["answer_2"]) + + +def test_spec_multi_turn_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + + +if __name__ == "__main__": + backend = OpenAI("gpt-4-turbo") + set_default_backend(backend) + + print("\n========== test spec single turn ==========\n") + # expect reasonable answer for each field + test_spec_single_turn() + + print("\n========== test inaccurate spec single turn ==========\n") + # expect incomplete or unreasonable answers + test_inaccurate_spec_single_turn() + + print("\n========== test normal single turn ==========\n") + # expect reasonable answer + test_normal_single_turn() + + print("\n========== test spec multi turn ==========\n") + # expect answer with same format as in the few shot + test_spec_multi_turn() + + print("\n========== test spec multi turn stream ==========\n") + # expect error in stream_executor: stream is not supported... + test_spec_multi_turn_stream() diff --git a/examples/frontend_language/usage/openai_speculative.py b/examples/frontend_language/usage/openai_speculative.py new file mode 100644 index 000000000..4389cb059 --- /dev/null +++ b/examples/frontend_language/usage/openai_speculative.py @@ -0,0 +1,54 @@ +""" +Usage: +python3 openai_speculative.py +""" + +from sglang import OpenAI, function, gen, set_default_backend + + +@function(num_api_spec_tokens=64) +def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function +def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +@function(num_api_spec_tokens=64) +def gen_character_spec_no_few_shot(s): + # s += "Construct a character with name, birthday, and job:\n" + s += "Construct a character:\n" + s += "Name:" + gen("name", stop="\n") + "\nBirthday:" + gen("birthday", stop="\n") + s += "\nJob:" + gen("job", stop="\n") + "\n" + + +if __name__ == "__main__": + backend = OpenAI("gpt-3.5-turbo-instruct") + set_default_backend(backend) + + for function in [ + gen_character_spec, + gen_character_no_spec, + gen_character_spec_no_few_shot, + ]: + backend.token_usage.reset() + + print(f"function: {function.func.__name__}") + + state = function.run() + + print("...name:", state["name"]) + print("...birthday:", state["birthday"]) + print("...job:", state["job"]) + print(backend.token_usage) + print() diff --git a/examples/frontend_language/usage/parallel_sample.py b/examples/frontend_language/usage/parallel_sample.py new file mode 100644 index 000000000..0f3cf1700 --- /dev/null +++ b/examples/frontend_language/usage/parallel_sample.py @@ -0,0 +1,40 @@ +""" +Usage: +python3 parallel_sample.py +""" + +import sglang as sgl + + +@sgl.function +def parallel_sample(s, question, n): + s += ( + "Question: Compute 1 + 2 + 3\n" + "Reasoning: I need to use a calculator.\n" + "Tool: calculator\n" + "Answer: 6\n" + "Question: Compute 3 + 2 + 2\n" + "Reasoning: I will try a calculator.\n" + "Tool: calculator\n" + "Answer: 7\n" + ) + s += "Question: " + question + "\n" + forks = s.fork(n) + forks += "Reasoning:" + sgl.gen("reasoning", stop="\n") + "\n" + forks += "Tool:" + sgl.gen("tool", choices=["calculator", "browser"]) + "\n" + forks += "Answer:" + sgl.gen("answer", stop="\n") + "\n" + forks.join() + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) +# sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + +state = parallel_sample.run(question="Compute 5 + 2 + 4.", n=5, temperature=1.0) + +for i in range(5): + obj = { + "reasoning": state["reasoning"][i], + "tool": state["tool"][i], + "answer": state["answer"][i], + } + print(f"[{i}], {obj}") diff --git a/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb new file mode 100644 index 000000000..f309142ad --- /dev/null +++ b/examples/frontend_language/usage/rag_using_parea/trace_and_evaluate_rag_using_parea.ipynb @@ -0,0 +1,408 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RAG Powered by SGLang & Chroma Evaluated using Parea\n", + "\n", + "In this notebook, we will build a simple RAG pipeline using SGLang to execute our LLM calls, Chroma as vector database for retrieval and [Parea](https://www.parea.ai) for tracing and evaluation. We will then evaluate the performance of our RAG pipeline. The dataset we will use was created by [Virat](https://twitter.com/virattt) and contains 100 questions, contexts and answers from the Airbnb 2023 10k filing.\n", + "\n", + "The RAG pipeline consists of two steps:\n", + "1. Retrieval: Given a question, we retrieve the relevant context from all provided contexts.\n", + "2. Generation: Given the question and the retrieved context, we generate an answer.\n", + "\n", + "ℹ️ This notebook requires an OpenAI API key.\n", + "\n", + "ℹ️ This notebook requires a Parea API key, which can be created [here](https://docs.parea.ai/api-reference/authentication#parea-api-key)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting up the environment\n", + "\n", + "We will first install the necessary packages: `sglang`, `parea-ai` and `chromadb`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# note, if you use a Mac M1 chip, you might need to install grpcio 1.59.0 first such that installing chromadb works\n", + "# !pip install grpcio==1.59.0\n", + "\n", + "!pip install sglang[openai] parea-ai chromadb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create a Parea API key as outlined [here](https://docs.parea.ai/api-reference/authentication#parea-api-key) and save it in a `.env` file as `PAREA_API_KEY=your-api-key`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Indexing the data\n", + "\n", + "Now it's time to download the data & index it! For that, we create a collection called `contexts` in Chroma and add the contexts as documents." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import List\n", + "\n", + "import chromadb\n", + "\n", + "path_qca = \"airbnb-2023-10k-qca.json\"\n", + "\n", + "if not os.path.exists(path_qca):\n", + " !wget https://virattt.github.io/datasets/abnb-2023-10k.json -O airbnb-2023-10k-qca.json\n", + "\n", + "with open(path_qca, \"r\") as f:\n", + " question_context_answers = json.load(f)\n", + "\n", + "chroma_client = chromadb.PersistentClient()\n", + "collection = chroma_client.get_or_create_collection(name=\"contexts\")\n", + "if collection.count() == 0:\n", + " collection.add(\n", + " documents=[qca[\"context\"] for qca in question_context_answers],\n", + " ids=[str(i) for i in range(len(question_context_answers))],\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining the RAG pipeline\n", + "\n", + "We will start with importing the necessary packages, setting up tracing of OpenAI calls via Parea and setting OpenAI as the default backend for SGLang." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import time\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "from sglang import function, user, assistant, gen, set_default_backend, OpenAI\n", + "from sglang.lang.interpreter import ProgramState\n", + "from parea import Parea, trace\n", + "\n", + "\n", + "load_dotenv()\n", + "\n", + "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", + "\n", + "p = Parea(api_key=os.getenv(\"PAREA_API_KEY\"), project_name=\"rag_sglang\")\n", + "p.integrate_with_sglang()\n", + "\n", + "set_default_backend(OpenAI(\"gpt-3.5-turbo\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can define our retrieval step shown below. Notice, the `trace` decorator which will automatically trace inputs, output, latency, etc. of that call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we will define the generation step which uses SGLang to execute the LLM call." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\"))\n", + "\n", + "\n", + "@trace\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can tie it together and execute a sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Debug Trace\n", + "\n", + "The output is unfortunately wrong! Using the traced pipeline, we can see that\n", + "\n", + "- the context is relevant to the question and contains the correct information\n", + "- but, the generation step is cut off as max tokens is set to 16\n", + "\n", + "When opening the generation step in the playground and rerunning the prompt with max. tokens set to 1000, the correct answer is produced.\n", + "\n", + "![RAG Trace](https://drive.google.com/uc?id=1QI243ogGjzbO01tUrR72g9rFoGzUJqVH)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluating RAG Pipelines\n", + "\n", + "Before we apply above's fix, let's dive into evaluating RAG pipelines.\n", + "\n", + "RAG pipelines consist of a retrieval step to fetch relevant information and a generation step to generate a response to a users question. A RAG pipeline can fail at either step. E.g. the retrieval step can fail to find relevant information which makes generating the correct impossible. Another failure mode is that the generation step doesn't leverage the retrieved information correctly. We will apply the following evaluation metrics to understand different failure modes:\n", + "\n", + "- `context_relevancy`: measures how relevant the context is given the question\n", + "- `percent_target_supported_by_context`: measures how much of the target answer is supported by the context; this will give an upper ceiling of how well the generation step can perform\n", + "- `answer_context_faithfulness`: measures how much the generated answer utilizes the context\n", + "- `answer_matches_target`: measures how well the generated answer matches the target answer judged by a LLM and gives a sense of accuracy of our entire pipeline\n", + "\n", + "To use these evaluation metrics, we can import them from `parea.evals.rag` and `parea.evals.general` and apply them to a function by specifying in the `trace` decorator which evaluation metrics to use. The `@trace` decorator will automatically log the results of the evaluation metrics to the Parea dashboard.\n", + "\n", + "Applying them to the retrieval step:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.rag import (\n", + " context_query_relevancy_factory,\n", + " percent_target_supported_by_context_factory,\n", + ")\n", + "\n", + "\n", + "context_relevancy_eval = context_query_relevancy_factory()\n", + "percent_target_supported_by_context = percent_target_supported_by_context_factory()\n", + "\n", + "\n", + "@trace(eval_funcs=[context_relevancy_eval, percent_target_supported_by_context])\n", + "def retrieval(question: str) -> List[str]:\n", + " return collection.query(query_texts=[question], n_results=1)[\"documents\"][0]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can apply `answer_context_faithfulness` and `answer_matches_target` to the generation step." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from parea.evals.general import answer_matches_target_llm_grader_factory\n", + "from parea.evals.rag import answer_context_faithfulness_statement_level_factory\n", + "\n", + "\n", + "answer_context_faithfulness = answer_context_faithfulness_statement_level_factory()\n", + "answer_matches_target_llm_grader = answer_matches_target_llm_grader_factory()\n", + "\n", + "\n", + "@function\n", + "def generation_sglang(s, question: str, *context: str):\n", + " context = \"\\n\".join(context)\n", + " s += user(\n", + " f\"Given this question:\\n{question}\\n\\nAnd this context:\\n{context}\\n\\nAnswer the question.\"\n", + " )\n", + " s += assistant(gen(\"answer\", max_tokens=1_000))\n", + "\n", + "\n", + "@trace(eval_funcs=[answer_context_faithfulness, answer_matches_target_llm_grader])\n", + "def generation(question: str, *context):\n", + " state: ProgramState = generation_sglang.run(question, *context)\n", + " while not state.stream_executor.is_finished:\n", + " time.sleep(1)\n", + " return state.stream_executor.variables[\"answer\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we tie them together & execute the original sample query." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@trace\n", + "def rag_pipeline(question: str) -> str:\n", + " contexts = retrieval(question)\n", + " return generation(question, *contexts)\n", + "\n", + "\n", + "rag_pipeline(\n", + " \"When did the World Health Organization formally declare an end to the COVID-19 global health emergency?\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, the answer is correct! Can you spot the line where we fixed the output truncation issue?\n", + "\n", + "The evaluation scores appear in the bottom right of the logs (screenshot below). Note, that there is no score for `answer_matches_target_llm_grader` and `percent_target_supported_by_context` as these evals are automatically skipped if the target answer is not provided.\n", + "\n", + "![Fixed Max. Tokens](max-tokens-fixed-rag-trace.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running an experiment\n", + "\n", + "Now we are (almost) ready to evaluate the performance of our RAG pipeline on the entire dataset. First, we will need to apply the `nest_asyncio` package to avoid issues with the Jupyter notebook event loop." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install nest-asyncio\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Running the actual experiment is straight-forward. For that we use `p.experiment` to initialize the experiment with a name, the data (list of key-value pairs fed into our entry function) and the entry function. We then call `run` on the experiment to execute it. Note, that `target` is a reserved key in the data dictionary and will be used as the target answer for evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "e = p.experiment(\n", + " \"RAG\",\n", + " data=[\n", + " {\n", + " \"question\": qca[\"question\"],\n", + " \"target\": qca[\"answer\"],\n", + " }\n", + " for qca in question_context_answers\n", + " ],\n", + " func=rag_pipeline,\n", + ").run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analyzing the results\n", + "\n", + "When opening above experiment, we will see an overview of the experiment as shown below. The upper half shows a summary of the statistics on the left and charts to investigate the distribution and relationships of scores on the right. The lower half is a table with the individual traces which we can use to debug individual samples.\n", + "\n", + "When looking at the statistics, we can see that the accuracy of our RAG pipeline is 22% as measured by `answer_matches_target_llm_grader`. Though when checking the quality of our retrieval step (`context_query_relevancy`), we can see that our retrieval step is fetching relevant information in only 27% of all samples. As shown in the GIF, we investigate the relationship between the two and see the two scores have 95% agreement. This confirms that the retrieval step is a major bottleneck for our RAG pipeline. So, now it's your turn to improve the retrieval step!\n", + "\n", + "Note, above link isn't publicly accessible but the experiment can be accessed through [here](https://app.parea.ai/public-experiments/parea/rag_sglang/30f0244a-d56c-44ff-bdfb-8f47626304b6).\n", + "\n", + "![Experiment Results](https://drive.google.com/uc?id=1KMtJBU47nPB02Pvv3SPPTK7RnHRh5YdA)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/frontend_language/usage/readme_examples.py b/examples/frontend_language/usage/readme_examples.py new file mode 100644 index 000000000..7269ef148 --- /dev/null +++ b/examples/frontend_language/usage/readme_examples.py @@ -0,0 +1,109 @@ +""" +Usage: +python -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --port 30000 +python readme_examples.py +""" + +import sglang as sgl + + +@sgl.function +def tool_use(s, question): + s += "To answer this question: " + question + ". " + s += ( + "I need to use a " + + sgl.gen("tool", choices=["calculator", "search engine"]) + + ". " + ) + + if s["tool"] == "calculator": + s += "The math expression is" + sgl.gen("expression") + elif s["tool"] == "search engine": + s += "The key word to search is" + sgl.gen("word") + + +@sgl.function +def tip_suggestion(s): + s += ( + "Here are two tips for staying healthy: " + "1. Balanced Diet. 2. Regular Exercise.\n\n" + ) + + forks = s.fork(2) + for i, f in enumerate(forks): + f += f"Now, expand tip {i+1} into a paragraph:\n" + f += sgl.gen(f"detailed_tip", max_tokens=256, stop="\n\n") + + s += "Tip 1:" + forks[0]["detailed_tip"] + "\n" + s += "Tip 2:" + forks[1]["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + +@sgl.function +def regular_expression_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)", + ) + + +@sgl.function +def text_qa(s, question): + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + +def driver_tool_use(): + state = tool_use.run(question="What is the capital of the United States?") + print(state.text()) + print("\n") + + +def driver_tip_suggestion(): + state = tip_suggestion.run() + print(state.text()) + print("\n") + + +def driver_regex(): + state = regular_expression_gen.run() + print(state.text()) + print("\n") + + +def driver_batching(): + states = text_qa.run_batch( + [ + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital of France?"}, + {"question": "What is the capital of Japan?"}, + ], + progress_bar=True, + ) + + for s in states: + print(s.text()) + print("\n") + + +def driver_stream(): + state = text_qa.run( + question="What is the capital of France?", temperature=0.1, stream=True + ) + + for out in state.text_iter(): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + # sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + sgl.set_default_backend(sgl.RuntimeEndpoint("http://localhost:30000")) + + driver_tool_use() + driver_tip_suggestion() + driver_regex() + driver_batching() + driver_stream() diff --git a/examples/frontend_language/usage/sgl_gen_min_tokens.py b/examples/frontend_language/usage/sgl_gen_min_tokens.py new file mode 100644 index 000000000..a5088199b --- /dev/null +++ b/examples/frontend_language/usage/sgl_gen_min_tokens.py @@ -0,0 +1,35 @@ +""" +This example demonstrates how to use `min_tokens` to enforce sgl.gen to generate a longer sequence + +Usage: +python3 sgl_gen_min_tokens.py +""" + +import sglang as sgl + + +@sgl.function +def long_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer", min_tokens=64, max_tokens=128)) + + +@sgl.function +def short_answer(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant(sgl.gen("answer")) + + +if __name__ == "__main__": + runtime = sgl.Runtime(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + sgl.set_default_backend(runtime) + + state = long_answer.run() + print("=" * 20) + print("Longer Answer", state["answer"]) + + state = short_answer.run() + print("=" * 20) + print("Short Answer", state["answer"]) + + runtime.shutdown() diff --git a/examples/frontend_language/usage/streaming.py b/examples/frontend_language/usage/streaming.py new file mode 100644 index 000000000..506ee35c6 --- /dev/null +++ b/examples/frontend_language/usage/streaming.py @@ -0,0 +1,49 @@ +""" +Usage: +python3 streaming.py +""" + +import asyncio + +import sglang as sgl + + +@sgl.function +def multi_turn_question(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1", max_tokens=256)) + s += sgl.user(question_2) + s += sgl.assistant(sgl.gen("answer_2", max_tokens=256)) + + +sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo")) + + +def stream_a_variable(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + for out in state.text_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +async def async_stream(): + state = multi_turn_question.run( + question_1="What is the capital of the United States?", + question_2="List two local attractions.", + stream=True, + ) + + async for out in state.text_async_iter(var_name="answer_2"): + print(out, end="", flush=True) + print("\n") + + +if __name__ == "__main__": + stream_a_variable() + asyncio.run(async_stream()) diff --git a/examples/frontend_language/usage/triton/Dockerfile b/examples/frontend_language/usage/triton/Dockerfile new file mode 100644 index 000000000..e4741a1db --- /dev/null +++ b/examples/frontend_language/usage/triton/Dockerfile @@ -0,0 +1,10 @@ +FROM nvcr.io/nvidia/tritonserver:24.01-py3 + +WORKDIR /opt + +RUN git clone https://github.com/sgl-project/sglang.git + +WORKDIR /opt/sglang +RUN pip install --upgrade pip && \ + pip install -e "python[all]" && \ + pip install datasets diff --git a/examples/frontend_language/usage/triton/README.md b/examples/frontend_language/usage/triton/README.md new file mode 100644 index 000000000..b2e55961f --- /dev/null +++ b/examples/frontend_language/usage/triton/README.md @@ -0,0 +1,35 @@ +# sglang_triton + +Build the docker image: +``` +docker build -t sglang-triton . +``` + +Then do: +``` +docker run -ti --gpus=all --network=host --name sglang-triton -v ./models:/mnt/models sglang-triton +``` + +inside the docker container: +``` +cd sglang +python3 -m sglang.launch_server --model-path mistralai/Mistral-7B-Instruct-v0.2 --port 30000 --mem-fraction-static 0.9 +``` + +with another shell, inside the docker container: +``` +docker exec -ti sglang-triton /bin/bash +cd /mnt +tritonserver --model-repository=/mnt/models +``` + + +Send request to the server: +``` +curl -X POST http://localhost:8000/v2/models/character_generation/generate \ +-H "Content-Type: application/json" \ +-d '{ + "INPUT_TEXT": ["harry"] +}' + +``` diff --git a/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt b/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt new file mode 100644 index 000000000..7546f993a --- /dev/null +++ b/examples/frontend_language/usage/triton/models/character_generation/config.pbtxt @@ -0,0 +1,23 @@ +name: "character_generation" +backend: "python" +input [ + { + name: "INPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +output [ + { + name: "OUTPUT_TEXT" + data_type: TYPE_STRING + dims: [ -1 ] + } +] +instance_group [ + { + count: 1 + kind: KIND_GPU + gpus: [ 0 ] + } +] diff --git a/examples/monitoring/README.md b/examples/monitoring/README.md new file mode 100644 index 000000000..3eef0b09b --- /dev/null +++ b/examples/monitoring/README.md @@ -0,0 +1,76 @@ +# SGLang Monitoring Setup + +This directory contains a ready-to-use monitoring setup for SGLang using Prometheus and Grafana. + +## Prerequisites + +- Docker and Docker Compose installed +- SGLang server running with metrics enabled + +## Usage + +1. Start your SGLang server with metrics enabled: + +```bash +python -m sglang.launch_server --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --port 30000 --enable-metrics +``` + +By default, the metrics server will run on `127.0.0.1:30000`. + +2. Start the monitoring stack: + +```bash +cd examples/monitoring +docker compose up +``` + +3. Access the monitoring interfaces: + - Grafana: [http://localhost:3000](http://localhost:3000) + - Prometheus: [http://localhost:9090](http://localhost:9090) + +Default Grafana login credentials: +- Username: `admin` +- Password: `admin` + +You'll be prompted to change the password on first login. + +4. The SGLang dashboard will be automatically available in the "SGLang Monitoring" folder. + +## Troubleshooting + +### Port Conflicts +If you see errors like "port is already allocated": + +1. Check if you already have Prometheus or Grafana running: + ```bash + docker ps | grep -E 'prometheus|grafana' + ``` + +2. Stop any conflicting containers: + ```bash + docker stop + ``` + +3. Ensure no other services are using ports 9090 and 3000: + ```bash + lsof -i :9090 + lsof -i :3000 + ``` + +### Connection Issues +If Grafana cannot connect to Prometheus: +1. Check that both services are running +2. Verify the datasource configuration in Grafana +3. Check that your SGLang server is properly exposing metrics + +## Configuration + +- Prometheus configuration: `prometheus.yaml` +- Docker Compose configuration: `docker-compose.yaml` +- Grafana datasource: `grafana/datasources/datasource.yaml` +- Grafana dashboard configuration: `grafana/dashboards/config/dashboard.yaml` +- SGLang dashboard JSON: `grafana/dashboards/json/sglang-dashboard.json` + +## Customization + +You can customize the monitoring setup by modifying the configuration files as needed. diff --git a/examples/monitoring/docker-compose.yaml b/examples/monitoring/docker-compose.yaml new file mode 100644 index 000000000..ce6457fa3 --- /dev/null +++ b/examples/monitoring/docker-compose.yaml @@ -0,0 +1,28 @@ +version: '3' +services: + prometheus: + image: prom/prometheus:latest + container_name: prometheus + network_mode: host + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + + grafana: + image: grafana/grafana:latest + container_name: grafana + network_mode: host + volumes: + - ./grafana/datasources:/etc/grafana/provisioning/datasources + - ./grafana/dashboards/config:/etc/grafana/provisioning/dashboards + - ./grafana/dashboards/json:/var/lib/grafana/dashboards + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer + - GF_AUTH_BASIC_ENABLED=false + - GF_USERS_ALLOW_SIGN_UP=false + - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/sglang-dashboard.json + depends_on: + - prometheus diff --git a/examples/monitoring/grafana/dashboards/config/dashboard.yaml b/examples/monitoring/grafana/dashboards/config/dashboard.yaml new file mode 100644 index 000000000..6c17a6c63 --- /dev/null +++ b/examples/monitoring/grafana/dashboards/config/dashboard.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: 'SGLang' + orgId: 1 + folder: 'SGLang Monitoring' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json b/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json new file mode 100644 index 000000000..ebd663e41 --- /dev/null +++ b/examples/monitoring/grafana/dashboards/json/sglang-dashboard.json @@ -0,0 +1,984 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:e2e_request_latency_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:e2e_request_latency_seconds_sum[$__rate_interval]) / rate(sglang:e2e_request_latency_seconds_count[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "End-to-End Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 17, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "yBuckets": { + "scale": { + "type": "linear" + } + } + }, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "secs" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum(increase(sglang:e2e_request_latency_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval])) by (le)\r\n", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "End-to-End Request Latency(s) Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(sglang:time_to_first_token_seconds_bucket[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "avg(rate(sglang:time_to_first_token_seconds_sum[$__rate_interval]) / rate(sglang:time_to_first_token_seconds_count[$__rate_interval]))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Time-To-First-Token Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 19, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "" + } + }, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (increase(sglang:time_to_first_token_seconds_bucket{model_name=~\"$model_name\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Time-To-First-Token Seconds Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_running_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num Running Requests", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "editorMode": "code", + "expr": "sglang:gen_throughput", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Token Generation Throughput (Tokens / S)", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:cache_hit_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddyfngn31dg5cf" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sglang:num_queue_reqs", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number Queued Requests", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "127.0.0.1:30000", + "value": "127.0.0.1:30000" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(instance)", + "includeAll": false, + "label": "instance", + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "meta-llama/Llama-3.1-8B-Instruct", + "value": "meta-llama/Llama-3.1-8B-Instruct" + }, + "datasource": { + "type": "prometheus" + }, + "definition": "label_values(model_name)", + "includeAll": false, + "label": "model name", + "name": "model_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(model_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "SGLang Dashboard", + "uid": "sglang-dashboard", + "version": 11 +} diff --git a/examples/monitoring/grafana/datasources/datasource.yaml b/examples/monitoring/grafana/datasources/datasource.yaml new file mode 100644 index 000000000..1ab0e4a5f --- /dev/null +++ b/examples/monitoring/grafana/datasources/datasource.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://localhost:9090 + isDefault: true + editable: false diff --git a/examples/monitoring/prometheus.yaml b/examples/monitoring/prometheus.yaml new file mode 100644 index 000000000..ba16ac3bd --- /dev/null +++ b/examples/monitoring/prometheus.yaml @@ -0,0 +1,10 @@ +# prometheus.yaml +global: + scrape_interval: 5s + evaluation_interval: 30s + +scrape_configs: + - job_name: sglang + static_configs: + - targets: + - '127.0.0.1:30000' diff --git a/examples/profiler/nsys_profile_tools/README.md b/examples/profiler/nsys_profile_tools/README.md new file mode 100644 index 000000000..687200e05 --- /dev/null +++ b/examples/profiler/nsys_profile_tools/README.md @@ -0,0 +1,176 @@ +# gputrc2graph.py + +This script processes NVIDIA Nsight Systems (`nsys`) GPU trace files +(`.nsys-rep`) with -t cuda tracing enabled, and generates kernel-level +summaries and visualizations of GPU and non-GPU time. It is useful for +profiling and analyzing nsys profile output. + +## Usage + +### Command-line Arguments + +- `--in_file` + **(required)** + List of input files and their metadata. Each entry should be in the format: + `,,,` + - `nsys-rep`: Path to the `.nsys-rep` file. + - `engine`: Engine name (e.g., `sglang`). + - `model`: Model name (e.g., `llama`, `gpt-oss`, `ds`). + - `elapsed_nonprofiled_sec`: Wall-clock runtime (in seconds) without + profiling. Specify `0` to use the elapsed time from the nsys-rep file + (this may inflate non-GPU time if actual runtime without profiling is + less). Multiple entries can be provided, separated by spaces. + +- `--out_dir` + Output directory for the generated CSV and HTML files. + If not specified, results are saved in the current directory. + +- `--title` + Title for the HTML chart/visualization. + +- `--nsys_cmd` + Path to the `nsys` command. + Default: `nsys` (assumes it is in your PATH). + Use this if `nsys` is not in your system PATH. + +## Notes + +- Make sure you have pandas installed. Any version is fine. +- Make sure [nsys](https://developer.nvidia.com/nsight-systems/get-started) is +installed, and specify the path to the `nsys` command with `--nsys_cmd` if it + is not in your PATH. The nsys version must be >= the nsys profile version that + was used to collect the traces when profiling the server, so that nsys can + process the nsys-rep that was generated. + +- For more details on available engines and models, see the help string in + the script or run: + +```bash +python3 gputrc2graph.py --help +``` + +## Example 1: analyze a single profile + +To analyze the GPU cycles of for example, a llama-3.1-8B model with sglang: + +1. Run the following command to collect nsys profile, for sglang server config. + + ```bash + nsys profile -t cuda -o nsys_res -f true --trace-fork-before-exec=true \ + --cuda-graph-trace=node --delay --duration \ + python3 -m sglang.launch_server --model meta-llama/Llama-3.1-8B ... + ``` + + where: + + - DELAY: how many seconds to delay nsys from collecting profiles, needed so + that profiles aren't captured till sglang server has come up and load + generation starts. + - DURATION: how many seconds for nsys profile to run before generating the + profile. This should be > the duration of the run. +2. After the server starts, run the client load generation command. Once the +test completes, after DURATION amount of time, nsys profile will generate an +nsys_res.nsys-rep file and shut down the server. + +3. Run step #1 again, this time starting up the server without collecting the +profile. + +4. Run step #2 again, and record the total time to complete the test in +seconds. This value will be used by the script to calculate the + CPU(non-GPU) seconds for the analysis. + +5. Say the run elapsed time from step #4 is 132 seconds. Run script to + analyze: + + ```bash + python3 gputrc2graph.py \ + --in_file run1.nsys-rep,sglang,llama,132 + ``` + +The command will produce 2 files for analysis: + +- result.html: this categorizes kernel names into different categories in a + stacked bar chart. +- result.csv: shows how the kernel names are mapped to the different + categories. + +### HTML visualization with result.html + +The html file shows the number of elapsed seconds due to different GPU +Substages or categories, which consist of attention kernels as the biggest +category, at 63 seconds, followed by "gemm" kernels. This lets the user +prioritize the kernels to focus on for performance optimizations. + +There's also an appended data table underneath the bar chart for copying out to + other post-processing tools. + +### Kernel to category mapping with result.csv + +Suppose the user would like to focus on improving triton kernels. It's not the +biggest consumer of cycles at .01 sec but perhaps it hasn't been optimized. +The next step is to use the result.csv to dive into what the kernels are which +compose the triton kernel GPU cycles. + +## Example 2: analyze multiple profiles + +Suppose the user has multiple nsys trace files, captured for different models, +say llama and gpt-oss in this case, and wish to compare their GPU/non-GPU +time, something like the following command can be used. + +```bash +python3 gputrc2graph.py \ +--in_file run1.nsys-rep,sglang,llama,100 run2.nsys-rep,sglang,gpt-oss,102 \ +--out_dir results +``` + +The analysis process is similar to example 1 but now there will be multiple +stack bar charts that can be compared. The categories for the different +kernels will remain the same, so that it's easy to compare the GPU cycles for +the same categories. + +Once a category is shown to have more cycles for one configuration than +another, the next step would be to use the csv file to see what kernels are +mapped into that category, and which kernels are taking the largest amount of +time which would cause a difference for the overall category. + +## Example 3: add new classification for a new model + +To create a new engine DEF with model ABC, just add another json file in the same directory as +gputrc2graph.py with the same format as the other json files. The script will automatically pick up all the json files in the same directory as engine/model specifications. + +Then, for this new model, suppose there are 4 kernels to be classified into +"gemm" and "attn", where the gemm kernels have names with "*H*" or "*I*" in +them, and attn kernels have names with "*J*" or "*K*" in them, just add another + .json file in the same directory as gputrc2graph.py with the same format as + the other json files, like the following: + +```json +{ + "DEF": { + "ABC": { + "H|I": "gemm", + "J|K": "attn", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + } + } +} +``` + +Each entry in the dictionary consists of: + +- key: a regex used to classify the kernels +- value: the category to classify the kernels into. + +The last 2 entries are common for all engine/models, consisting of CUDA memory +operations and a 'misc' for anything that's leftover and can't be classified. + +When invoking gputrc2graph.py, specify a trace file with this new model/engine +like the following: + +```bash +--in_file new.nsys-rep,DEF,ABC, +``` + +If the engine_DEF.json file already exists, just add the model as a new node in + the existing engine file, after the other models. diff --git a/examples/profiler/nsys_profile_tools/gputrc2graph.py b/examples/profiler/nsys_profile_tools/gputrc2graph.py new file mode 100755 index 000000000..f17bd1857 --- /dev/null +++ b/examples/profiler/nsys_profile_tools/gputrc2graph.py @@ -0,0 +1,344 @@ +""" + This generates gpu kernel analysis output from nsys rep. Will call nsys + stats -r cuda_gpu_kern_trace, get non-overlapped gpu cycles, then generate + csv and html output for analysis +""" + +import argparse +import logging +import os + +import regex as re + +logger = logging.getLogger(__name__) + + +# helper data class for annotating kernels +def load_engine_model(): + """returns engine_model built from all json files in the current dir""" + import glob + import json + + engine_model = {} + + json_files = glob.glob(os.path.join(os.path.dirname(__file__) or ".", "*.json")) + for fname in json_files: + with open(fname, encoding="utf-8") as f: + engine_model.update(json.load(f)) + return engine_model + + +class GPUTrace2Graph: + """ + Parses output of nsys report, generates csv and bar chart output + """ + + def __init__(self): + import pandas as pd # avoid importing till needed + + self.pd = pd + self.pd.options.mode.copy_on_write = True + + # helper functions for generating trace->summary csvs + def gen_nonoverlapped_sum_from_gputrace(self, in_file, out_file): + logger.info("loading %s", in_file) + df = self.pd.read_csv( + in_file, usecols=["Start (ns)", "Duration (ns)", "Device", "Strm", "Name"] + ) + df["End (ns)"] = df["Start (ns)"] + df["Duration (ns)"] + df = self.sum_non_overlapping_intervals(df) + # get ready to print table with elapsed times per kernel + df["Instances"] = 1 + df_sum = df.groupby("Name", as_index=False).agg( + {"Elapsed Time (ns)": "sum", "Duration (ns)": "sum", "Instances": "size"} + ) + + # generate csv + df_sum["Total Time (sec)"] = df_sum["Duration (ns)"] / 1e9 + df_sum["Elapsed Time (sec)"] = df_sum["Elapsed Time (ns)"] / 1e9 + df_sum = df_sum.sort_values(by="Elapsed Time (sec)", ascending=False) + df_sum[["Elapsed Time (sec)", "Total Time (sec)", "Instances", "Name"]].to_csv( + out_file, index=False + ) + + def sum_non_overlapping_intervals(self, df): + """ + returns new sorted df with Elapsed Time (ns) column using + vectorized operations + """ + logger.info("sorting %s trace records by start time", str(df.shape)) + + # Sort by start time and reset index + df = df.sort_values(by="Start (ns)").reset_index(drop=True) + + # Initialize elapsed time as duration + df["Elapsed Time (ns)"] = df["Duration (ns)"] + + # Get numpy arrays for faster operations + starts = df["Start (ns)"].values + ends = df["End (ns)"].values + + # Keep track of current interval end + current_end = ends[0] + display_units = max(1, int(len(df) / 100)) + # Update current_end for overlapping intervals + for i in range(1, len(df)): + if i % display_units == 0: + print(f"processing trace: {int(i/len(df) * 100)} %", end="\r") + if starts[i] <= current_end: + if ends[i] > current_end: + # Partial overlap + df.iloc[i, df.columns.get_loc("Elapsed Time (ns)")] = ( + ends[i] - current_end + ) + current_end = ends[i] + else: + # Complete overlap + df.iloc[i, df.columns.get_loc("Elapsed Time (ns)")] = 0 + else: + # No overlap + current_end = ends[i] + + return df + + # functions for generating html files + def make_html(self, df, output_dir, title): + """make html graph from df""" + import plotly.express as px + + if df.empty: + return + output_name = os.path.join(output_dir, "result") + if not title: + title = "Model_Engine" + x = "Model_Engine" + y = "Elapsed Time (sec)" + color = "Category" + """ generate kernel mapping table """ + # Sort Model_Engine categories by last field after underscore + df["Model_Engine"] = self.pd.Categorical( + df["Model_Engine"], + sorted(df["Model_Engine"].unique(), key=lambda x: x.split("_")[-1]), + ) + df[["Model_Engine", color, "Instances", "Name", y]].sort_values( + by=color + ).to_csv(f"{output_name}.csv", index=False) + graph = px.histogram( + df.round(2), + x=x, + y=y, + title=(f"{y} for {title}"), + color=color, + text_auto=True, + ) + # wrap x axis labels + graph.update_xaxes(automargin=True) + graph.write_html(f"{output_name}.html") + """ + Generate data table with columns per Model_Engine into result.html + """ + pivot_df = df.pivot_table( + values="Elapsed Time (sec)", + index="Category", + columns="Model_Engine", + aggfunc="sum", + observed=False, + ).round(2) + # Add sum row at bottom + pivot_df.loc["total_elapsed_sec"] = pivot_df.sum() + pivot_df.fillna("").to_html("temp.html") + with ( + open(f"{output_name}.html", "a", encoding="utf-8") as outfile, + open("temp.html", encoding="utf-8") as infile, + ): + outfile.write(infile.read()) + os.remove("temp.html") + + print( + f"Finished generating: \n" + f" {output_name}.html for stack bar chart \n" + f" {output_name}.csv for Kernel-Category mapping" + ) + + def anno_gpu_kernname(self, df, mapping): + """add "Category" column""" + + def anno_gpu_kernname_helper(name): + for kern_name, val in mapping.items(): + if re.search(kern_name, name): + return val + + df["Category"] = df["Name"].apply(anno_gpu_kernname_helper) + + def make_nongpu_row(self, df, nongpu_sec): + """this will append non-gpu time entry at end of df""" + nongpu_row = self.pd.DataFrame([df.iloc[-1]]) + nongpu_row["Category"] = nongpu_row["Name"] = "CPU(non-GPU)" + nongpu_row["Instances"] = 1 + nongpu_row["Elapsed Time (sec)"] = nongpu_sec + return nongpu_row + + def is_valid_file(self, base_file): + """asserts if base_file is non-existent or is empty""" + assert ( + os.path.isfile(base_file) and os.path.getsize(base_file) > 0 + ), f"{base_file} doesn't exist or is empty" + + def should_gen_file(self, new_file, base_file): + """figure out if new file should be generated from base_file""" + self.is_valid_file(base_file) + if ( + os.path.exists(new_file) + and (os.path.getmtime(new_file) > os.path.getmtime(base_file)) + and (os.path.getsize(base_file) > 0) + ): + logger.info("reusing %s", new_file) + return False + else: + logger.info("generating %s", new_file) + return True + + def gen_sum_file(self, file, nsys_cmd): + """ + generates sum file from nsys trace with times per kernel and + returns the name of the sum file + """ + import subprocess + + file_dir = os.path.dirname(file) + file_name = os.path.basename(file) + + if not file_dir: + file_dir = "." + # Walk through trace and get the total non-overlapped time + nsys_stats_file = os.path.join(file_dir, f"{file_name}_cuda_gpu_trace.csv") + sum_file = os.path.join(file_dir, f"{file_name}_cuda_gpu_kernel_tracesum.csv") + if self.should_gen_file(nsys_stats_file, file): + cmd = [ + nsys_cmd, + "stats", + "-r", + "cuda_gpu_trace", + file, + "-o", + f"{file_dir}/{file_name}", + ] + cmd_str = " ".join(cmd) + logger.info("+ %s", cmd_str) + # estimate time based on calibrated 240M/min + file_size_mb = os.path.getsize(file) / 1e6 + logger.info( + "nsys stats for %.2f MB file expected to take %.2f min", + file_size_mb, + file_size_mb / 240, + ) + try: + subprocess.run(cmd, check=True) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + logger.error( + "'%s' failed: %s. Use --nsys_cmd to specify nsys path", cmd_str, e + ) + exit(1) + logger.info("generating non-overalapped sum %s", sum_file) + self.gen_nonoverlapped_sum_from_gputrace(nsys_stats_file, sum_file) + self.is_valid_file(sum_file) + logger.info("Finished generating %s", sum_file) + return sum_file + + def gen_graph(self, in_file, out_dir, title, nsys_cmd, engine_model): + """generates graph and csv file from in_file into out_dir""" + # Initialize an empty DataFrame to store combined data + combined_df = self.pd.DataFrame() + for idx, (file, engine, model, total_sec) in enumerate(in_file): + file_dir = os.path.dirname(file) + file_name = os.path.basename(file) + if not file_dir: + file_dir = "." + sum_file = self.gen_sum_file(file, nsys_cmd) + # read kernel summary file + df = self.pd.read_csv(sum_file) + # annotate kernel to their categories + assert engine_model.get(engine), f"engine {engine} unknown" + assert engine_model[engine].get(model), f"model {model} unknown" + # remove nsys-rep from file_name for shorter x-label + file_name = file_name.replace(".nsys-rep", "") + df["Model_Engine"] = f"{model}_{engine}_{file_name}_{idx}" + self.anno_gpu_kernname(df, engine_model[engine][model]) + # patch in non-gpu time + gpu_sec = round(df["Elapsed Time (sec)"].sum(), 1) + total_sec = round(float(total_sec), 1) + if total_sec < gpu_sec: + logger.warning( + "Elapsed sec %.2f < GPU sec %.2f resetting Elapsed sec ", + total_sec, + gpu_sec, + ) + total_sec = gpu_sec + nongpu_row = self.make_nongpu_row(df, total_sec - gpu_sec) + df = self.pd.concat([df, nongpu_row], ignore_index=True) + combined_df = self.pd.concat([combined_df, df], ignore_index=True) + if out_dir is None: + out_dir = "." + else: + os.makedirs(out_dir, exist_ok=True) + # generate html file + self.make_html(combined_df, out_dir, title) + + +def parse_tuple(s): + return tuple(s.split(",")) + + +def main(): + logging.basicConfig( + format=("%(asctime)s - %(levelname)s - %(message)s"), level=logging.INFO + ) + parser = argparse.ArgumentParser( + description=( + "Process nsys rep and generate kernel non-overlapped cycles. \n" + "Example:\n" + "gputrc2graph.py --in_file d1.nsys-rep,sglang,llama,100 \n" + "d2.nsys-rep,sglang,gpt-oss,102 " + '--out_dir results/ --title "Model=gpt-oss SGLANG chart"' + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + # load supported engine_model + engine_model_supported = load_engine_model() + # Get a string representation of supported engine/model combinations + engine_model_supported_str = ", ".join( + f"{engine}:[{', '.join(models.keys())}]" + for engine, models in engine_model_supported.items() + ) + parser.add_argument( + "--in_file", + type=parse_tuple, + nargs="+", + help=( + "list of (nsys-rep, engine, model, elapsed_nonprofiled_sec) " + "separated by space. Elapsed_nonprofiled_sec is runtime without " + "profiling used to calculate non-gpu time. Specify 0 to use " + "elapsed time from nsys-rep but that might inflate non-gpu time. " + f"Available engine:[model] are: {engine_model_supported_str} " + f"Example: --infile d1.nsys-rep,sglan,llama,100 " + "d2.nsys-rep,sglang,gpt-oss,102" + ), + required=True, + ) + parser.add_argument("--out_dir", help=("output dir for result.csv/html")) + parser.add_argument("--title", help=("title for html chart")) + parser.add_argument( + "--nsys_cmd", + help=("nsys cmd, e.g. /usr/bin/nsys, Default: nsys"), + default="nsys", + ) + args = parser.parse_args() + gputrace = GPUTrace2Graph() + gputrace.gen_graph( + args.in_file, args.out_dir, args.title, args.nsys_cmd, engine_model_supported + ) + + +if __name__ == "__main__": + main() diff --git a/examples/profiler/nsys_profile_tools/sglang_engine_model.json b/examples/profiler/nsys_profile_tools/sglang_engine_model.json new file mode 100644 index 000000000..253cc762b --- /dev/null +++ b/examples/profiler/nsys_profile_tools/sglang_engine_model.json @@ -0,0 +1,61 @@ +{ + "sglang": { + "llama": { + "gemm|nvjet": "gemm", + "fused_moe_kernel|GroupProblemShape|group_gemm_starts|bmm_|GemmUniversal": "moe_gemm", + "moe|sigmoid": "moe", + "CatArrayBatched|prepare_inputs": "prepare_next", + "ncclDevKernel|cross_device_reduce": "nccl_and_custom_ar", + "_norm_|Norm": "norm", + "topk": "topk", + "act_and_mul_": "activation", + "Rotary": "rope", + "SoftMax": "softmax", + "flash|fmha": "attn", + "elementwise": "elementwise", + "fp8_quant|cvt_|quantize": "quantize", + "reduce_kernel": "reduce", + "triton": "triton_kernel", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + }, + "ds": { + "block_fp8_matmul": "block_fp8_gemm", + "gemm|matmul|nvjet": "gemm", + "fused_moe_kernel": "moe_gemm", + "moe|expert|sigmoid": "moe", + "CatArrayBatched|write_req_to": "prepare_next", + "ncclDevKernel|cross_device_reduce|all_gather": "nccl_and_custom_ar", + "Norm": "norm", + "topk": "topk", + "activation|act_and_mul": "activation", + "compute_position_kernel": "rope", + "elementwise": "elementwise", + "fp8_quant|quant_fp8|quantize": "quantize", + "SoftMax": "softmax", + "reduce": "reduce", + "_fwd_|create_flash|::mla::|KVCache": "attn", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + }, + "gpt-oss": { + "gemm|nvjet": "gemm", + "fused_moe_kernel|_group_gemm|GroupProblemShape|GemmUniversal|bmm_|matmul_ogs_|_topk_forward|_combined_routing|_sum_bitmatrix_rows|_compute_writeback_idx": "moe_gemm", + "moe|sigmoid": "moe", + "CatArrayBatched|prepare_inputs": "prepare_next", + "_norm_|Norm": "norm", + "ncclDevKernel|cross_device_reduce|allreduce": "nccl_and_custom_ar", + "topk|TopK": "topk", + "act_and_mul_": "activation", + "Rotary": "rope", + "SoftMax": "softmax", + "flash|fmha": "attn", + "elementwise": "elementwise", + "fp8_quant|cvt_|quantize": "quantize", + "reduce_kernel": "reduce", + "triton": "triton_kernel", + "CUDA mem": "non-gpu-H_D_memops", + ".*": "misc" + } + } +} diff --git a/examples/runtime/README.md b/examples/runtime/README.md new file mode 100644 index 000000000..18414452f --- /dev/null +++ b/examples/runtime/README.md @@ -0,0 +1,45 @@ +# Runtime examples + +The below examples will mostly need you to start a server in a separate terminal before you can execute them. Please see in the code for detailed instruction. + +## Native API + +* `lora.py`: An example how to use LoRA adapters. +* `multimodal_embedding.py`: An example how perform [multi modal embedding](Alibaba-NLP/gme-Qwen2-VL-2B-Instruct). +* `openai_batch_chat.py`: An example how to process batch requests for chat completions. +* `openai_batch_complete.py`: An example how to process batch requests for text completions. +* **`openai_chat_with_response_prefill.py`**: + An example that demonstrates how to [prefill a response](https://eugeneyan.com/writing/prompting/#prefill-claudes-responses) using the OpenAI API by enabling the `continue_final_message` parameter. + When enabled, the final (partial) assistant message is removed and its content is used as a prefill so that the model continues that message rather than starting a new turn. See [Anthropic's prefill example](https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering/prefill-claudes-response#example-structured-data-extraction-with-prefilling) for more context. +* `reward_model.py`: An example how to extract scores from a reward model. +* `vertex_predict.py`: An example how to deploy a model to [Vertex AI](https://cloud.google.com/vertex-ai?hl=en). + +## Engine + +The `engine` folder contains that examples that show how to use [Offline Engine API](https://docs.sglang.ai/backend/offline_engine_api.html#Offline-Engine-API) for common workflows. + +* `custom_server.py`: An example how to deploy a custom server. +* `embedding.py`: An example how to extract embeddings. +* `launch_engine.py`: An example how to launch the Engine. +* `offline_batch_inference_eagle.py`: An example how to perform speculative decoding using [EAGLE](https://docs.sglang.ai/backend/speculative_decoding.html). +* `offline_batch_inference_torchrun.py`: An example how to perform inference using [torchrun](https://pytorch.org/docs/stable/elastic/run.html). +* `offline_batch_inference_vlm.py`: An example how to use VLMs with the engine. +* `offline_batch_inference.py`: An example how to use the engine to perform inference on a batch of examples. + +## Hidden States + +The `hidden_states` folder contains examples on how to extract hidden states using SGLang. Please note that this might degrade throughput due to cuda graph rebuilding. + +* `hidden_states_engine.py`: An example how to extract hidden states using the Engine API. +* `hidden_states_server.py`: An example how to extract hidden states using the Server API. + +## Multimodal + +SGLang supports multimodal inputs for various model architectures. The `multimodal` folder contains examples showing how to use urls, files or encoded data to make requests to multimodal models. Examples include querying the [Llava-OneVision](multimodal/llava_onevision_server.py) model (image, multi-image, video), Llava-backed [Qwen-Llava](multimodal/qwen_llava_server.py) and [Llama3-Llava](multimodal/llama3_llava_server.py) models (image, multi-image), and Mistral AI's [Pixtral](multimodal/pixtral_server.py) (image, multi-image). + + +## Token In, Token Out + +The folder `token_in_token_out` shows how to perform inference, where we provide tokens and get tokens as response. + +* `token_in_token_out_{llm|vlm}_{engine|server}.py`: Shows how to perform token in, token out workflow for llm/vlm using either the engine or native API. diff --git a/examples/runtime/engine/custom_server.py b/examples/runtime/engine/custom_server.py new file mode 100644 index 000000000..b190a463e --- /dev/null +++ b/examples/runtime/engine/custom_server.py @@ -0,0 +1,53 @@ +from sanic import Sanic, text +from sanic.response import json + +import sglang as sgl + +engine = None + +# Create an instance of the Sanic app +app = Sanic("sanic-server") + + +# Define an asynchronous route handler +@app.route("/generate", methods=["POST"]) +async def generate(request): + prompt = request.json.get("prompt") + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt) + + return text(result["text"]) + + +@app.route("/generate_stream", methods=["POST"]) +async def generate_stream(request): + prompt = request.json.get("prompt") + + if not prompt: + return json({"error": "Prompt is required"}, status=400) + + # async_generate returns a dict + result = await engine.async_generate(prompt, stream=True) + + # https://sanic.dev/en/guide/advanced/streaming.md#streaming + # init the response + response = await request.respond() + + # result is an async generator + async for chunk in result: + await response.send(chunk["text"]) + + await response.eof() + + +def run_server(): + global engine + engine = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + app.run(host="0.0.0.0", port=8000, single_process=True) + + +if __name__ == "__main__": + run_server() diff --git a/examples/runtime/engine/embedding.py b/examples/runtime/engine/embedding.py new file mode 100644 index 000000000..b927a188b --- /dev/null +++ b/examples/runtime/engine/embedding.py @@ -0,0 +1,27 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", is_embedding=True + ) + + outputs = llm.encode(prompts) + # Print the outputs (embedding vectors) + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nEmbedding vector: {output['embedding']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/fastapi_engine_inference.py b/examples/runtime/engine/fastapi_engine_inference.py new file mode 100644 index 000000000..a755cf8d8 --- /dev/null +++ b/examples/runtime/engine/fastapi_engine_inference.py @@ -0,0 +1,189 @@ +""" +FastAPI server example for text generation using SGLang Engine and demonstrating client usage. + +Starts the server, sends requests to it, and prints responses. + +Usage: +python fastapi_engine_inference.py --model-path Qwen/Qwen2.5-0.5B-Instruct --tp_size 1 --host 127.0.0.1 --port 8000 +""" + +import os +import subprocess +import time +from contextlib import asynccontextmanager + +import requests +from fastapi import FastAPI, Request + +import sglang as sgl +from sglang.utils import terminate_process + +engine = None + + +# Use FastAPI's lifespan manager to initialize/shutdown the engine +@asynccontextmanager +async def lifespan(app: FastAPI): + """Manages SGLang engine initialization during server startup.""" + global engine + # Initialize the SGLang engine when the server starts + # Adjust model_path and other engine arguments as needed + print("Loading SGLang engine...") + engine = sgl.Engine( + model_path=os.getenv("MODEL_PATH"), tp_size=int(os.getenv("TP_SIZE")) + ) + print("SGLang engine loaded.") + yield + # Clean up engine resources when the server stops (optional, depends on engine needs) + print("Shutting down SGLang engine...") + # engine.shutdown() # Or other cleanup if available/necessary + print("SGLang engine shutdown.") + + +app = FastAPI(lifespan=lifespan) + + +@app.post("/generate") +async def generate_text(request: Request): + """FastAPI endpoint to handle text generation requests.""" + global engine + if not engine: + return {"error": "Engine not initialized"}, 503 + + try: + data = await request.json() + prompt = data.get("prompt") + max_new_tokens = data.get("max_new_tokens", 128) + temperature = data.get("temperature", 0.7) + + if not prompt: + return {"error": "Prompt is required"}, 400 + + # Use async_generate for non-blocking generation + state = await engine.async_generate( + prompt, + sampling_params={ + "max_new_tokens": max_new_tokens, + "temperature": temperature, + }, + # Add other parameters like stop, top_p etc. as needed + ) + + return {"generated_text": state["text"]} + except Exception as e: + return {"error": str(e)}, 500 + + +# Helper function to start the server +def start_server(args, timeout=60): + """Starts the Uvicorn server as a subprocess and waits for it to be ready.""" + base_url = f"http://{args.host}:{args.port}" + command = [ + "python", + "-m", + "uvicorn", + "fastapi_engine_inference:app", + f"--host={args.host}", + f"--port={args.port}", + ] + + process = subprocess.Popen(command, stdout=None, stderr=None) + + start_time = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start_time < timeout: + try: + # Check the /docs endpoint which FastAPI provides by default + response = session.get( + f"{base_url}/docs", timeout=5 + ) # Add a request timeout + if response.status_code == 200: + print(f"Server {base_url} is ready (responded on /docs)") + return process + except requests.ConnectionError: + # Specific exception for connection refused/DNS error etc. + pass + except requests.Timeout: + # Specific exception for request timeout + print(f"Health check to {base_url}/docs timed out, retrying...") + pass + except requests.RequestException as e: + # Catch other request exceptions + print(f"Health check request error: {e}, retrying...") + pass + # Use a shorter sleep interval for faster startup detection + time.sleep(1) + + # If loop finishes, raise the timeout error + # Attempt to terminate the failed process before raising + if process: + print( + "Server failed to start within timeout, attempting to terminate process..." + ) + terminate_process(process) # Use the imported terminate_process + raise TimeoutError( + f"Server failed to start at {base_url} within the timeout period." + ) + + +def send_requests(server_url, prompts, max_new_tokens, temperature): + """Sends generation requests to the running server for a list of prompts.""" + # Iterate through prompts and send requests + for i, prompt in enumerate(prompts): + print(f"\n[{i+1}/{len(prompts)}] Sending prompt: '{prompt}'") + payload = { + "prompt": prompt, + "max_new_tokens": max_new_tokens, + "temperature": temperature, + } + + try: + response = requests.post(f"{server_url}/generate", json=payload, timeout=60) + + result = response.json() + + print(f"Prompt: {prompt}\nResponse: {result['generated_text']}") + + except requests.exceptions.Timeout: + print(f" Error: Request timed out for prompt '{prompt}'") + except requests.exceptions.RequestException as e: + print(f" Error sending request for prompt '{prompt}': {e}") + + +if __name__ == "__main__": + """Main entry point for the script.""" + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--model-path", type=str, default="Qwen/Qwen2.5-0.5B-Instruct") + parser.add_argument("--tp_size", type=int, default=1) + args = parser.parse_args() + + # Pass the model to the child uvicorn process via an env var + os.environ["MODEL_PATH"] = args.model_path + os.environ["TP_SIZE"] = str(args.tp_size) + + # Start the server + process = start_server(args) + + # Define the prompts and sampling parameters + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + max_new_tokens = 64 + temperature = 0.1 + + # Define server url + server_url = f"http://{args.host}:{args.port}" + + # Send requests to the server + send_requests(server_url, prompts, max_new_tokens, temperature) + + # Terminate the server process + terminate_process(process) diff --git a/examples/runtime/engine/launch_engine.py b/examples/runtime/engine/launch_engine.py new file mode 100644 index 000000000..b97e0bcd5 --- /dev/null +++ b/examples/runtime/engine/launch_engine.py @@ -0,0 +1,17 @@ +""" +This example demonstrates how to launch the offline engine. +""" + +import sglang as sgl + + +def main(): + llm = sgl.Engine(model_path="meta-llama/Meta-Llama-3.1-8B-Instruct") + llm.generate("What is the capital of France?") + llm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/offline_batch_inference.py b/examples/runtime/engine/offline_batch_inference.py new file mode 100644 index 000000000..92e68dcd7 --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference.py @@ -0,0 +1,43 @@ +""" +Usage: +python3 offline_batch_inference.py --model meta-llama/Llama-3.1-8B-Instruct +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine(**dataclasses.asdict(server_args)) + + outputs = llm.generate(prompts, sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/examples/runtime/engine/offline_batch_inference_async.py b/examples/runtime/engine/offline_batch_inference_async.py new file mode 100644 index 000000000..578962d78 --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_async.py @@ -0,0 +1,65 @@ +""" +Usage: +python offline_batch_inference_async.py --model-path Qwen/Qwen2-VL-7B-Instruct + +Note: +This demo shows the usage of async generation, +which is useful to implement an online-like generation with batched inference. +""" + +import argparse +import asyncio +import dataclasses +import time + +import sglang as sgl +from sglang.srt.server_args import ServerArgs + + +class InferenceEngine: + def __init__(self, **kwargs): + self.engine = sgl.Engine(**kwargs) + + async def generate(self, prompt, sampling_params): + result = await self.engine.async_generate(prompt, sampling_params) + return result + + +async def run_server(server_args): + inference = InferenceEngine(**dataclasses.asdict(server_args)) + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] * 100 + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Run the generation tasks concurrently in async mode. + tasks = [] + for prompt in prompts: + task = asyncio.create_task(inference.generate(prompt, sampling_params)) + tasks.append(task) + + # Get and print the result + for task in tasks: + await task + while True: + if not task.done(): + time.sleep(1) + else: + result = task.result() + print(f"Generated text: {result['text']}") + break + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + asyncio.run(run_server(server_args)) diff --git a/examples/runtime/engine/offline_batch_inference_eagle.py b/examples/runtime/engine/offline_batch_inference_eagle.py new file mode 100644 index 000000000..a7a89ef5c --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_eagle.py @@ -0,0 +1,38 @@ +import sglang as sgl + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0, "max_new_tokens": 30} + + # Create an LLM. + llm = sgl.Engine( + model_path="meta-llama/Llama-2-7b-chat-hf", + speculative_algorithm="EAGLE", + speculative_draft_model_path="lmsys/sglang-EAGLE-llama2-chat-7B", + speculative_num_steps=3, + speculative_eagle_topk=4, + speculative_num_draft_tokens=16, + cuda_graph_max_bs=8, + ) + + outputs = llm.generate(prompts, sampling_params) + + # Print the outputs. + for prompt, output in zip(prompts, outputs): + print("===============================") + print(f"Prompt: {prompt}\nGenerated text: {output['text']}") + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/offline_batch_inference_qwen_1m.py b/examples/runtime/engine/offline_batch_inference_qwen_1m.py new file mode 100644 index 000000000..664efa6d7 --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_qwen_1m.py @@ -0,0 +1,74 @@ +""" +Usage: +python3 offline_batch_inference.py +""" + +from urllib.request import urlopen + +import sglang as sgl + + +def load_prompt() -> str: + # Test cases with various lengths can be found at: + # + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/64k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/200k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/600k.txt + # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/1m.txt + + with urlopen( + "https://qianwen-res.oss-cn-beijing.aliyuncs.com" + "/Qwen2.5-1M/test-data/64k.txt", + timeout=5, + ) as response: + prompt = response.read().decode("utf-8") + return prompt + + +# Processing the prompt. +def process_requests(llm: sgl.Engine, prompts: list[str]) -> None: + # Create a sampling params object. + sampling_params = { + "temperature": 0.7, + "top_p": 0.8, + "top_k": 20, + "repetition_penalty": 1.05, + "max_new_tokens": 256, + } + # Generate texts from the prompts. + outputs = llm.generate(prompts, sampling_params) + # Print the outputs. + for output in outputs: + prompt_token_ids = output["meta_info"]["prompt_tokens"] + generated_text = output["text"] + print( + f"Prompt length: {prompt_token_ids}, " f"Generated text: {generated_text!r}" + ) + + +# Create an LLM. +def initialize_engine() -> sgl.Engine: + llm = sgl.Engine( + model_path="Qwen/Qwen2.5-7B-Instruct-1M", + context_length=1048576, + page_size=256, + attention_backend="dual_chunk_flash_attn", + tp_size=4, + disable_radix_cache=True, + enable_mixed_chunk=False, + enable_torch_compile=False, + chunked_prefill_size=131072, + mem_fraction_static=0.6, + log_level="DEBUG", + ) + return llm + + +def main(): + llm = initialize_engine() + prompt = load_prompt() + process_requests(llm, [prompt]) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/engine/offline_batch_inference_vlm.py b/examples/runtime/engine/offline_batch_inference_vlm.py new file mode 100644 index 000000000..392823946 --- /dev/null +++ b/examples/runtime/engine/offline_batch_inference_vlm.py @@ -0,0 +1,52 @@ +""" +Usage: +python offline_batch_inference_vlm.py --model-path Qwen/Qwen2-VL-7B-Instruct +""" + +import argparse +import dataclasses + +import sglang as sgl +from sglang.srt.parser.conversation import chat_templates +from sglang.srt.server_args import ServerArgs + + +def main( + server_args: ServerArgs, +): + vlm = sgl.Engine(**dataclasses.asdict(server_args)) + + conv = chat_templates[server_args.chat_template].copy() + image_token = conv.image_token + + image_url = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" + + prompt = f"What's in this image?\n{image_token}" + + sampling_params = { + "temperature": 0.001, + "max_new_tokens": 30, + } + + output = vlm.generate( + prompt=prompt, + image_data=image_url, + sampling_params=sampling_params, + ) + + print("===============================") + print(f"Prompt: {prompt}") + print(f"Generated text: {output['text']}") + + vlm.shutdown() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = parser.parse_args() + + server_args = ServerArgs.from_cli_args(args) + main(server_args) diff --git a/examples/runtime/engine/readme.md b/examples/runtime/engine/readme.md new file mode 100644 index 000000000..fc6e89a6a --- /dev/null +++ b/examples/runtime/engine/readme.md @@ -0,0 +1,54 @@ +# SGLang Engine + +SGLang provides a direct inference engine without the need for an HTTP server. There are generally these use cases: + +- [Offline Batch Inference](#offline-batch-inference) +- [Embedding Generation](#embedding-generation) +- [Custom Server](#custom-server) +- [Token-In-Token-Out for RLHF](#token-in-token-out-for-rlhf) +- [Inference Using FastAPI](#inference-using-fastapi) + +## Examples + +### [Offline Batch Inference](./offline_batch_inference.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for inference. If you provide a very large batch, the engine will intelligently schedule the requests to process efficiently and prevent OOM (Out of Memory) errors. + +### [Embedding Generation](./embedding.py) + +In this example, we launch an SGLang engine and feed a batch of inputs for embedding generation. + +### [Custom Server](./custom_server.py) + +This example demonstrates how to create a custom server on top of the SGLang Engine. We use [Sanic](https://sanic.dev/en/) as an example. The server supports both non-streaming and streaming endpoints. + +#### Steps + +1. Install Sanic: + + ```bash + pip install sanic + ``` + +2. Run the server: + + ```bash + python custom_server + ``` + +3. Send requests: + + ```bash + curl -X POST http://localhost:8000/generate -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' + curl -X POST http://localhost:8000/generate_stream -H "Content-Type: application/json" -d '{"prompt": "The Transformer architecture is..."}' --no-buffer + ``` + + This will send both non-streaming and streaming requests to the server. + +### [Token-In-Token-Out for RLHF](../token_in_token_out) + +In this example, we launch an SGLang engine, feed tokens as input and generate tokens as output. + +### [Inference Using FastAPI](fastapi_engine_inference.py) + +This example demonstrates how to create a FastAPI server that uses the SGLang engine for text generation. diff --git a/examples/runtime/engine/save_remote_state.py b/examples/runtime/engine/save_remote_state.py new file mode 100644 index 000000000..a428195ca --- /dev/null +++ b/examples/runtime/engine/save_remote_state.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_remote_state.py \ + --model-path /path/to/load \ + --tensor-parallel-size 8 \ + --remote-model-save-url [protocol]://[host]:[port]/[model_name] \ + +Then, the model can be loaded with + +llm = Engine( + model_path="[protocol]://[host]:[port]/[model_name]", + tensor_parallel_size=8, +) +""" +import dataclasses +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--remote-model-save-url", + required=True, + type=str, + help="remote address to store model weights", +) +parser.add_argument( + "--remote-draft-model-save-url", + default=None, + type=str, + help="remote address to store draft model weights", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + llm.save_remote_model( + url=args.remote_model_save_url, draft_url=args.remote_draft_model_save_url + ) + print("save remote (draft) model successfully") + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/examples/runtime/engine/save_sharded_state.py b/examples/runtime/engine/save_sharded_state.py new file mode 100644 index 000000000..80ad5321f --- /dev/null +++ b/examples/runtime/engine/save_sharded_state.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Saves each worker's model state dict directly to a checkpoint, which enables a +fast load path for large tensor-parallel models where each worker only needs to +read its own shard rather than the entire checkpoint. + +Example usage: + +python save_sharded_state.py \ + --model-path /path/to/load \ + --quantization deepspeedfp \ + --tensor-parallel-size 8 \ + --output /path/to/save + +Then, the model can be loaded with + +llm = Engine( + model_path="/path/to/save", + load_format="sharded_state", + quantization="deepspeedfp", + tensor_parallel_size=8, +) +""" +import dataclasses +import os +import shutil +from argparse import ArgumentParser +from pathlib import Path + +from sglang import Engine, ServerArgs + +parser = ArgumentParser() +ServerArgs.add_cli_args(parser) + +parser.add_argument( + "--output", "-o", required=True, type=str, help="path to output checkpoint" +) +parser.add_argument( + "--file-pattern", type=str, help="string pattern of saved filenames" +) +parser.add_argument( + "--max-file-size", + type=str, + default=5 * 1024**3, + help="max size (in bytes) of each safetensors file", +) + + +def main(args): + engine_args = ServerArgs.from_cli_args(args) + model_path = engine_args.model_path + if not Path(model_path).is_dir(): + raise ValueError("model path must be a local directory") + # Create LLM instance from arguments + llm = Engine(**dataclasses.asdict(engine_args)) + Path(args.output).mkdir(exist_ok=True) + llm.save_sharded_model( + path=args.output, pattern=args.file_pattern, max_size=args.max_file_size + ) + + # Copy metadata files to output directory + for file in os.listdir(model_path): + if os.path.splitext(file)[1] not in (".bin", ".pt", ".safetensors"): + if os.path.isdir(os.path.join(model_path, file)): + shutil.copytree( + os.path.join(model_path, file), os.path.join(args.output, file) + ) + else: + shutil.copy(os.path.join(model_path, file), args.output) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args) diff --git a/examples/runtime/hidden_states/hidden_states_engine.py b/examples/runtime/hidden_states/hidden_states_engine.py new file mode 100644 index 000000000..60ab302ca --- /dev/null +++ b/examples/runtime/hidden_states/hidden_states_engine.py @@ -0,0 +1,66 @@ +""" +Usage: +python hidden_states.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import torch + +import sglang as sgl + + +def main(): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create an LLM. + llm = sgl.Engine( + model_path="Alibaba-NLP/gte-Qwen2-1.5B-instruct", + enable_return_hidden_states=True, + ) + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + outputs = llm.generate( + prompts, sampling_params=sampling_params, return_hidden_states=True + ) + + llm.shutdown() + + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/hidden_states/hidden_states_server.py b/examples/runtime/hidden_states/hidden_states_server.py new file mode 100644 index 000000000..b04f74372 --- /dev/null +++ b/examples/runtime/hidden_states/hidden_states_server.py @@ -0,0 +1,81 @@ +""" +Usage: + +python hidden_states_server.py + +Note that each time you change the `return_hidden_states` parameter, +the cuda graph will be recaptured, which might lead to a performance hit. +So avoid getting hidden states and completions alternately. +""" + +import requests +import torch + +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + "python -m sglang.launch_server --model-path Alibaba-NLP/gte-Qwen2-1.5B-instruct --enable-return-hidden-states --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + sampling_params = { + "temperature": 0.8, + "top_p": 0.95, + "max_new_tokens": 10, + } + + json_data = { + "text": prompts, + "sampling_params": sampling_params, + "return_hidden_states": True, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + terminate_process(server_process) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + for i in range(len(output["meta_info"]["hidden_states"])): + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + print("===============================") + print( + f"Prompt: {prompt}\n" + f"Generated text: {output['text']}\n" + f"Prompt_Tokens: {output['meta_info']['prompt_tokens']}\t" + f"Completion_tokens: {output['meta_info']['completion_tokens']}" + ) + print("Hidden states: ") + hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ) + print(hidden_states) + print() + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/lora.py b/examples/runtime/lora.py new file mode 100644 index 000000000..bf3fc2d9e --- /dev/null +++ b/examples/runtime/lora.py @@ -0,0 +1,37 @@ +# launch server +# python -m sglang.launch_server --model mistralai/Mistral-7B-Instruct-v0.3 --lora-paths /home/ying/test_lora lora1=/home/ying/test_lora_1 lora2=/home/ying/test_lora_2 --disable-radix --disable-cuda-graph --max-loras-per-batch 4 + +# send requests +# lora_path[i] specifies the LoRA used for text[i], so make sure they have the same length +# use None to specify base-only prompt, e.x. "lora_path": [None, "/home/ying/test_lora"] +import json + +import requests + +url = "http://127.0.0.1:30000" +json_data = { + "text": [ + "prompt 1", + "prompt 2", + "prompt 3", + "prompt 4", + "prompt 5", + "prompt 6", + "prompt 7", + ], + "sampling_params": {"max_new_tokens": 32}, + "lora_path": [ + "/home/ying/test_lora", + "lora1", + "lora2", + "lora1", + "lora2", + None, + None, + ], +} +response = requests.post( + url + "/generate", + json=json_data, +) +print(json.dumps(response.json())) diff --git a/examples/runtime/multimodal/llama3_llava_server.py b/examples/runtime/multimodal/llama3_llava_server.py new file mode 100644 index 000000000..a8409af71 --- /dev/null +++ b/examples/runtime/multimodal/llama3_llava_server.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llama3-llava-next-8b --port=30000 + +python3 llama3_llava_server.py + +Output: +"Friends posing for a fun photo with a life-sized teddy bear, creating a playful and memorable moment." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_llava_llama_3 + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_llava_llama_3) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|eot_id|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/examples/runtime/multimodal/llava_onevision_server.py b/examples/runtime/multimodal/llava_onevision_server.py new file mode 100644 index 000000000..ee921b558 --- /dev/null +++ b/examples/runtime/multimodal/llava_onevision_server.py @@ -0,0 +1,264 @@ +""" +Usage: + +python3 -m sglang.launch_server --model-path lmms-lab/llava-onevision-qwen2-72b-ov --port=30000 --tp-size=8 + +python3 llava_onevision_server.py +""" + +import base64 +import io +import os +import sys +import time + +import numpy as np +import openai +import requests +from decord import VideoReader, cpu +from PIL import Image + +# pip install httpx==0.23.3 +# pip install decord +# pip install protobuf==3.20.0 + + +def download_video(url, cache_dir): + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + + print(f"File downloaded and saved to: {file_path}") + return file_path + + +def create_openai_client(base_url): + return openai.Client(api_key="EMPTY", base_url=base_url) + + +def image_stream_request_test(client): + print("----------------------Image Stream Request Test----------------------") + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def multi_image_stream_request_test(client): + print( + "----------------------Multi-Images Stream Request Test----------------------" + ) + stream_request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png" + }, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have shown you two images. Please describe the two images to me.", + }, + ], + }, + ], + temperature=0.7, + max_tokens=1024, + stream=True, + ) + stream_response = "" + + for chunk in stream_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + stream_response += content + sys.stdout.write(content) + sys.stdout.flush() + + print("-" * 30) + + +def video_stream_request_test(client, video_path): + print("------------------------Video Stream Request Test----------------------") + messages = prepare_video_messages(video_path) + + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=True, + ) + print("-" * 30) + video_response = "" + + for chunk in video_request: + if chunk.choices[0].delta.content is not None: + content = chunk.choices[0].delta.content + video_response += content + sys.stdout.write(content) + sys.stdout.flush() + print("-" * 30) + + +def image_speed_test(client): + print("----------------------Image Speed Test----------------------") + start_time = time.perf_counter() + request = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png" + }, + }, + { + "type": "text", + "text": "Please describe this image. Please list the benchmarks and the models.", + }, + ], + }, + ], + temperature=0, + max_tokens=1024, + ) + end_time = time.perf_counter() + response = request.choices[0].message.content + print(response) + print("-" * 30) + print_speed_test_results(request, start_time, end_time) + + +def video_speed_test(client, video_path): + print("------------------------Video Speed Test------------------------") + messages = prepare_video_messages(video_path) + + start_time = time.perf_counter() + video_request = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + ) + end_time = time.perf_counter() + video_response = video_request.choices[0].message.content + print(video_response) + print("-" * 30) + print_speed_test_results(video_request, start_time, end_time) + + +def prepare_video_messages(video_path): + max_frames_num = 32 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + + for base64_frame in base64_frames: + frame_format = { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{base64_frame}"}, + "modalities": "video", + } + messages[0]["content"].append(frame_format) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + +def print_speed_test_results(request, start_time, end_time): + total_tokens = request.usage.total_tokens + completion_tokens = request.usage.completion_tokens + prompt_tokens = request.usage.prompt_tokens + + print(f"Total tokens: {total_tokens}") + print(f"Completion tokens: {completion_tokens}") + print(f"Prompt tokens: {prompt_tokens}") + print(f"Time taken: {end_time - start_time} seconds") + print(f"Token per second: {total_tokens / (end_time - start_time)}") + print(f"Completion token per second: {completion_tokens / (end_time - start_time)}") + print(f"Prompt token per second: {prompt_tokens / (end_time - start_time)}") + + +def main(): + url = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + video_path = download_video(url, cache_dir) + + client = create_openai_client("http://127.0.0.1:30000/v1") + + image_stream_request_test(client) + multi_image_stream_request_test(client) + video_stream_request_test(client, video_path) + image_speed_test(client) + video_speed_test(client, video_path) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/multimodal/pixtral_server.py b/examples/runtime/multimodal/pixtral_server.py new file mode 100644 index 000000000..d907de14d --- /dev/null +++ b/examples/runtime/multimodal/pixtral_server.py @@ -0,0 +1,127 @@ +""" +Usage: +# Run a Pixtral model with SGLang: +# HuggingFace: +python -m sglang.launch_server --model-path mistral-community/pixtral-12b --port=30000 +# ModelScope: +python -m sglang.launch_server --model-path AI-ModelScope/pixtral-12b --port=30000 + +# Then test it with: +python pixtral_server.py + +This script tests Pixtral model with both single and multiple images. +""" + +import argparse +import asyncio +import json + +import aiohttp +import requests + +IMAGE_TOKEN_SEP = "\n[IMG]" +ROUTE = "/generate" + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}{ROUTE}" + + # Single image test + if args.single_image: + prompt = f"[INST]Describe this image in detail.{IMAGE_TOKEN_SEP}[/INST]" + image_url = "https://picsum.photos/id/237/400/300" + modality = ["image"] + # Multiple images test + else: + image_urls = [ + "https://picsum.photos/id/237/400/300", + "https://picsum.photos/id/27/500/500", + ] + prompt = f"[INST]How many photos are there? Describe each in a very short sentence.{IMAGE_TOKEN_SEP * len(image_urls)}[/INST]" + image_url = image_urls + modality = ["multi-images"] + + response = await send_request( + url, + { + "text": prompt, + "image_data": image_url, + "sampling_params": { + "max_new_tokens": 100, + "temperature": 0.7, + "top_p": 0.9, + }, + "modalities": modality, + }, + ) + + print(f"Response: {response}") + if "text" in response: + print("\nOutput text:", response["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}/generate" + + # Single image test + if args.single_image: + prompt = f"[INST]Describe this image in detail.{IMAGE_TOKEN_SEP}[/INST]" + image_data = "https://picsum.photos/id/237/400/300" + modality = ["image"] + # Multiple images test + else: + image_urls = [ + "https://picsum.photos/id/237/400/300", + "https://picsum.photos/id/27/500/500", + ] + prompt = f"[INST]How many photos are there? Describe each in a very short sentence.{IMAGE_TOKEN_SEP * len(image_urls)}[/INST]" + image_data = image_urls + modality = ["multi-images"] + + pload = { + "text": prompt, + "image_data": image_data, + "sampling_params": {"max_new_tokens": 100, "temperature": 0.7, "top_p": 0.9}, + "modalities": modality, + "stream": True, + } + + response = requests.post(url, json=pload, stream=True) + + print("Streaming response:") + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument( + "--single-image", + action="store_true", + help="Test with single image instead of multiple images", + ) + parser.add_argument("--no-stream", action="store_true", help="Don't test streaming") + args = parser.parse_args() + + asyncio.run(test_concurrent(args)) + if not args.no_stream: + test_streaming(args) diff --git a/examples/runtime/multimodal/qwen_llava_server.py b/examples/runtime/multimodal/qwen_llava_server.py new file mode 100644 index 000000000..d8b3226e7 --- /dev/null +++ b/examples/runtime/multimodal/qwen_llava_server.py @@ -0,0 +1,111 @@ +""" +Usage: +# Installing latest llava-next: pip install git+https://github.com/LLaVA-VL/LLaVA-NeXT.git +# Installing latest sglang. + +# Endpoint Service CLI: +python -m sglang.launch_server --model-path lmms-lab/llava-next-72b --port=30000 --tp-size=8 + +python3 qwen_llava_server.py + +Output: +"Two children pose with a large teddy bear, one holding a smaller stuffed bear, in a room with an American flag and potted plants." +""" + +import argparse +import asyncio +import copy +import json + +import aiohttp +import requests +from llava.conversation import conv_qwen + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + response = [] + for i in range(1): + response.append( + send_request( + url + "/generate", + { + "text": prompt_with_template, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + prompt = "\nPlease generate caption towards this image." + conv_template = copy.deepcopy(conv_qwen) + conv_template.append_message(role=conv_template.roles[0], message=prompt) + conv_template.append_message(role=conv_template.roles[1], message=None) + prompt_with_template = conv_template.get_prompt() + pload = { + "text": prompt_with_template, + "sampling_params": { + "max_new_tokens": 1024, + "temperature": 0, + "top_p": 1.0, + "presence_penalty": 2, + "frequency_penalty": 2, + "stop": "<|im_end|>", + }, + "image_data": "https://farm4.staticflickr.com/3175/2653711032_804ff86d81_z.jpg", + "stream": True, + } + response = requests.post( + url + "/generate", + json=pload, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + asyncio.run(test_concurrent(args)) + test_streaming(args) diff --git a/examples/runtime/multimodal_embedding.py b/examples/runtime/multimodal_embedding.py new file mode 100644 index 000000000..4e8d748b4 --- /dev/null +++ b/examples/runtime/multimodal_embedding.py @@ -0,0 +1,18 @@ +# launch server +# python -m sglang.launch_server --model-path Alibaba-NLP/gme-Qwen2-VL-2B-Instruct --is-embedding + +import requests + +url = "http://127.0.0.1:30000" + +text_input = "Represent this image in embedding space." +image_path = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + +payload = { + "model": "gme-qwen2-vl", + "input": [{"text": text_input}, {"image": image_path}], +} + +response = requests.post(url + "/v1/embeddings", json=payload).json() + +print("Embeddings:", [x.get("embedding") for x in response.get("data", [])]) diff --git a/examples/runtime/openai_chat_with_response_prefill.py b/examples/runtime/openai_chat_with_response_prefill.py new file mode 100644 index 000000000..6d803a1d1 --- /dev/null +++ b/examples/runtime/openai_chat_with_response_prefill.py @@ -0,0 +1,53 @@ +""" +Usage: +1) Launch the server in one terminal: + python -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --port 30000 + +2) Run this script in another terminal: + python openai_chat_with_response_prefill.py + +This example demonstrates two chat completion calls: +- One with continue_final_message enabled (the final assistant message is used as a prefill). +- One without continue_final_message (the final assistant message remains, starting a new turn). +""" + +import openai + +client = openai.Client(base_url="http://127.0.0.1:30000/v1", api_key="EMPTY") + +messages = [ + {"role": "system", "content": "You are a helpful AI assistant."}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. +At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app— +no matter where you place it in your home. +This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + {"role": "assistant", "content": "{\n"}, +] + +# Calling the API with continue_final_message enabled. +print("=== Prefill with continue_final_messagem ===") +response_with = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=messages, + temperature=0, + extra_body={"continue_final_message": True}, +) +print(response_with.choices[0].message.content) + +# Calling the API without continue_final_message (using default behavior). +print("\n=== Prefill without continue_final_message ===") +response_without = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=messages, + temperature=0, +) +print(response_without.choices[0].message.content) diff --git a/examples/runtime/reward_model.py b/examples/runtime/reward_model.py new file mode 100644 index 000000000..1a1177e66 --- /dev/null +++ b/examples/runtime/reward_model.py @@ -0,0 +1,32 @@ +# launch server +# python -m sglang.launch_server --model LxzGordon/URM-LLaMa-3.1-8B --is-embedding + +import requests + +url = "http://127.0.0.1:30000" + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +json_data = { + "conv": [ + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE1}, + ], + [ + {"role": "user", "content": PROMPT}, + {"role": "assistant", "content": RESPONSE2}, + ], + ], +} +response = requests.post( + url + "/classify", + json=json_data, +).json() + +print(response) +print("scores:", [x["embedding"] for x in response]) diff --git a/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py b/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py new file mode 100644 index 000000000..cb1b7ddc1 --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_llm_engine.py @@ -0,0 +1,43 @@ +""" +This example demonstrates how to provide tokenized ids to LLM as input instead of text prompt, i.e. a token-in-token-out workflow. +""" + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + # Create an LLM. + llm = sgl.Engine(model_path=MODEL_PATH, skip_tokenizer_init=True) + + outputs = llm.generate(input_ids=token_ids_list, sampling_params=sampling_params) + # Print the outputs. + for prompt, output in zip(prompts, outputs): + decode_output = tokenizer.decode(output["output_ids"]) + print("===============================") + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + main() diff --git a/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py new file mode 100644 index 000000000..00c0988b2 --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_llm_server.py @@ -0,0 +1,68 @@ +""" +Usage: + +python token_in_token_out_llm_server.py + +""" + +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "meta-llama/Llama-3.1-8B-Instruct" + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create a sampling params object. + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Tokenize inputs + tokenizer = get_tokenizer(MODEL_PATH) + token_ids_list = [tokenizer.encode(prompt) for prompt in prompts] + + json_data = { + "input_ids": token_ids_list, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + outputs = response.json() + for prompt, output in zip(prompts, outputs): + print("===============================") + decode_output = tokenizer.decode(output["output_ids"]) + print( + f"Prompt: {prompt}\nGenerated token ids: {output['output_ids']}\nGenerated text: {decode_output}" + ) + print() + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py b/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py new file mode 100644 index 000000000..bf6f19719 --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_vlm_engine.py @@ -0,0 +1,74 @@ +import argparse +import dataclasses +from typing import Tuple + +from transformers import AutoProcessor + +from sglang import Engine +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import DEFAULT_IMAGE_URL + + +def get_input_ids( + server_args: ServerArgs, model_config: ModelConfig +) -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(model_config.model_path) + text = f"{chat_template.image_token}What is in this picture?" + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained( + model_config.model_path, trust_remote_code=server_args.trust_remote_code + ) + + input_ids = ( + processor.tokenizer( + text=[text], + return_tensors="pt", + ) + .input_ids[0] + .tolist() + ) + + return input_ids, image_data + + +def token_in_out_example( + server_args: ServerArgs, +): + input_ids, image_data = get_input_ids( + server_args, + ModelConfig( + server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + model_override_args=server_args.json_model_override_args, + ), + ) + backend = Engine(**dataclasses.asdict(server_args)) + + output = backend.generate( + input_ids=input_ids, + image_data=image_data, + sampling_params={ + "temperature": 0.8, + "max_new_tokens": 32, + }, + ) + + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + backend.shutdown() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + args = [ + "--model-path=Qwen/Qwen2-VL-2B", + ] + args = parser.parse_args(args=args) + server_args = ServerArgs.from_cli_args(args) + server_args.skip_tokenizer_init = True + token_in_out_example(server_args) diff --git a/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py new file mode 100644 index 000000000..392e1bf0e --- /dev/null +++ b/examples/runtime/token_in_token_out/token_in_token_out_vlm_server.py @@ -0,0 +1,78 @@ +""" +Usage: + +python token_in_token_out_vlm_server.py + +""" + +from typing import Tuple + +import requests +from transformers import AutoProcessor + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.test.test_utils import DEFAULT_IMAGE_URL, is_in_ci +from sglang.utils import terminate_process, wait_for_server + +if is_in_ci(): + from docs.backend.patch import launch_server_cmd +else: + from sglang.utils import launch_server_cmd + + +MODEL_PATH = "Qwen/Qwen2-VL-2B" + + +def get_input_ids() -> Tuple[list[int], list]: + chat_template = get_chat_template_by_model_path(MODEL_PATH) + text = f"{chat_template.image_token}What is in this picture?" + image_data = [DEFAULT_IMAGE_URL] + + processor = AutoProcessor.from_pretrained(MODEL_PATH) + + input_ids = ( + processor.tokenizer( + text=[text], + return_tensors="pt", + ) + .input_ids[0] + .tolist() + ) + + return input_ids, image_data + + +def main(): + # Launch the server + server_process, port = launch_server_cmd( + f"python -m sglang.launch_server --model-path {MODEL_PATH} --skip-tokenizer-init --host 0.0.0.0" + ) + wait_for_server(f"http://localhost:{port}") + + input_ids, image_data = get_input_ids() + + sampling_params = { + "temperature": 0.8, + "max_new_tokens": 32, + } + + json_data = { + "input_ids": input_ids, + "image_data": image_data, + "sampling_params": sampling_params, + } + + response = requests.post( + f"http://localhost:{port}/generate", + json=json_data, + ) + + output = response.json() + print("===============================") + print(f"Output token ids: ", output["output_ids"]) + + terminate_process(server_process) + + +if __name__ == "__main__": + main() diff --git a/examples/runtime/vertex_predict.py b/examples/runtime/vertex_predict.py new file mode 100644 index 000000000..58a41b1c4 --- /dev/null +++ b/examples/runtime/vertex_predict.py @@ -0,0 +1,66 @@ +""" +Usage: +python -m sglang.launch_server --model meta-llama/Llama-2-7b-hf --port 30000 +python vertex_predict.py + +This example shows the request and response formats of the prediction route for +Google Cloud Vertex AI Online Predictions. + +Vertex AI SDK for Python is recommended for deploying models to Vertex AI +instead of a local server. After deploying the model to a Vertex AI Online +Prediction Endpoint, send requests via the Python SDK: + +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) + +More details about get online predictions from Vertex AI can be found at +https://cloud.google.com/vertex-ai/docs/predictions/get-online-predictions. +""" + +from dataclasses import dataclass +from typing import List, Optional + +import requests + + +@dataclass +class VertexPrediction: + predictions: List + + +class LocalVertexEndpoint: + def __init__(self) -> None: + self.base_url = "http://127.0.0.1:30000" + + def predict(self, instances: List[dict], parameters: Optional[dict] = None): + response = requests.post( + self.base_url + "/vertex_generate", + json={ + "instances": instances, + "parameters": parameters, + }, + ) + return VertexPrediction(predictions=response.json()["predictions"]) + + +endpoint = LocalVertexEndpoint() + +# Predict with a single prompt. +response = endpoint.predict(instances=[{"text": "The capital of France is"}]) +print(response.predictions) + +# Predict with multiple prompts and parameters. +response = endpoint.predict( + instances=[ + {"text": "The capital of France is"}, + {"text": "What is a car?"}, + ], + parameters={"sampling_params": {"max_new_tokens": 16}}, +) +print(response.predictions) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..54cb66f0b --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "sglang", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100755 index 000000000..f2e69b3c0 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,167 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang" +version = "0.5.2" +description = "SGLang is a fast serving framework for large language models and vision language models." +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", +] +dependencies = ["aiohttp", "requests", "tqdm", "numpy", "IPython", "setproctitle"] + +[project.optional-dependencies] +runtime_common = [ + "blobfile==3.0.0", + "build", + "compressed-tensors", + "datasets", + "einops", + "fastapi", + "hf_transfer", + "huggingface_hub", + "interegular", + "llguidance>=0.7.11,<0.8.0", + "modelscope", + "msgspec", + "ninja", + "openai==1.99.1", + "openai-harmony==0.0.4", + "orjson", + "outlines==0.1.11", + "packaging", + "partial_json_parser", + "pillow", + "prometheus-client>=0.20.0", + "psutil", + "pybase64", + "pydantic", + "pynvml", + "python-multipart", + "pyzmq>=25.1.2", + "sentencepiece", + "soundfile==0.13.1", + "scipy", + "timm==1.0.16", + "tiktoken", + "torchao==0.9.0", + "transformers==4.56.1", + "uvicorn", + "uvloop", + "xgrammar==0.1.24", +] + +srt = [ + "sglang[runtime_common]", + "sgl-kernel==0.3.9.post2", + "torch==2.8.0", + "torchaudio==2.8.0", + "torchvision", + "cuda-python", + "flashinfer_python==0.3.1", +] + +blackwell = [ + "sglang[runtime_common]", + "sgl-kernel==0.3.9.post2", + "torch==2.8.0", + "torchaudio==2.8.0", + "torchvision", + "cuda-python", + "flashinfer_python==0.3.1", + "nvidia-cutlass-dsl==4.1.0", +] + +# HIP (Heterogeneous-computing Interface for Portability) for AMD +# => base docker rocm/vllm-dev:20250114, not from public vllm whl +srt_hip = [ + "sglang[runtime_common]", + "torch", + "petit_kernel==0.0.2", + "wave-lang==3.7.0", +] + +# https://docs.sglang.ai/platforms/cpu_server.html +srt_cpu = ["sglang[runtime_common]", "intel-openmp"] + +# https://docs.sglang.ai/platforms/ascend_npu.html +srt_npu = ["sglang[runtime_common]"] + +# xpu is not enabled in public vllm and torch whl, +# need to follow https://docs.vllm.ai/en/latest/getting_started/xpu-installation.htmlinstall vllm +srt_xpu = ["sglang[runtime_common]"] + +# For Intel Gaudi(device : hpu) follow the installation guide +# https://docs.vllm.ai/en/latest/getting_started/gaudi-installation.html +srt_hpu = ["sglang[runtime_common]"] + +openai = ["openai==1.99.1", "tiktoken"] +anthropic = ["anthropic>=0.20.0"] +litellm = ["litellm>=1.0.0"] +torch_memory_saver = ["torch_memory_saver==0.0.8"] +decord = ["decord"] +test = [ + "accelerate", + "expecttest", + "jsonlines", + "matplotlib", + "pandas", + "peft", + "sentence_transformers", + "pytest", + "tabulate", +] +all = ["sglang[srt]", "sglang[openai]", "sglang[anthropic]", "sglang[torch_memory_saver]", "sglang[decord]"] +all_hip = ["sglang[srt_hip]", "sglang[openai]", "sglang[anthropic]", "sglang[decord]"] +all_xpu = ["sglang[srt_xpu]", "sglang[openai]", "sglang[anthropic]", "sglang[decord]"] +all_hpu = ["sglang[srt_hpu]", "sglang[openai]", "sglang[anthropic]", "sglang[decord]"] +all_cpu = ["sglang[srt_cpu]", "sglang[openai]", "sglang[anthropic]", "sglang[decord]"] +all_npu = ["sglang[srt_npu]", "sglang[openai]", "sglang[anthropic]", "sglang[decord]"] + +dev = ["sglang[all]", "sglang[test]"] +dev_hip = ["sglang[all_hip]", "sglang[test]"] +dev_xpu = ["sglang[all_xpu]", "sglang[test]"] +dev_hpu = ["sglang[all_hpu]", "sglang[test]"] +dev_cpu = ["sglang[all_cpu]", "sglang[test]"] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.setuptools.package-data] +"sglang" = [ + "srt/layers/moe/fused_moe_triton/configs/*/*.json", + "srt/layers/quantization/configs/*.json", + "srt/mem_cache/storage/hf3fs/hf3fs_utils.cpp", +] + +[tool.setuptools.packages.find] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.wheel] +exclude = [ + "assets*", + "benchmark*", + "docs*", + "dist*", + "playground*", + "scripts*", + "tests*", +] + +[tool.codespell] +ignore-words-list = "ans, als, hel, boostrap, childs, te, vas, hsa, ment" +skip = "*.json,*.jsonl,*.patch,*.txt" diff --git a/python/sglang/README.md b/python/sglang/README.md new file mode 100644 index 000000000..ae0c479b9 --- /dev/null +++ b/python/sglang/README.md @@ -0,0 +1,16 @@ +# Code Structures + +- `eval`: The evaluation utilities. +- `lang`: The frontend language. +- `srt`: The backend engine for running local models. (SRT = SGLang Runtime). +- `test`: The test utilities. +- `api.py`: The public APIs. +- `bench_offline_throughput.py`: Benchmark the performance in the offline mode. +- `bench_one_batch.py`: Benchmark the latency of running a single static batch without a server. +- `bench_one_batch_server.py`: Benchmark the latency of running a single batch with a server. +- `bench_serving.py`: Benchmark online serving with dynamic requests. +- `check_env.py`: Check the environment variables and dependencies. +- `global_config.py`: The global configs and constants. +- `launch_server.py`: The entry point for launching the local server. +- `utils.py`: Common utilities. +- `version.py`: Version info. diff --git a/python/sglang/__init__.py b/python/sglang/__init__.py new file mode 100644 index 000000000..509b145a9 --- /dev/null +++ b/python/sglang/__init__.py @@ -0,0 +1,83 @@ +# SGLang public APIs + +# Frontend Language APIs +from sglang.global_config import global_config +from sglang.lang.api import ( + Engine, + Runtime, + assistant, + assistant_begin, + assistant_end, + flush_cache, + function, + gen, + gen_int, + gen_string, + get_server_info, + image, + select, + separate_reasoning, + set_default_backend, + system, + system_begin, + system_end, + user, + user_begin, + user_end, + video, +) +from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint +from sglang.lang.choices import ( + greedy_token_selection, + token_length_normalized, + unconditional_likelihood_normalized, +) + +# Lazy import some libraries +from sglang.utils import LazyImport +from sglang.version import __version__ + +Anthropic = LazyImport("sglang.lang.backend.anthropic", "Anthropic") +LiteLLM = LazyImport("sglang.lang.backend.litellm", "LiteLLM") +OpenAI = LazyImport("sglang.lang.backend.openai", "OpenAI") +VertexAI = LazyImport("sglang.lang.backend.vertexai", "VertexAI") + +# Runtime Engine APIs +ServerArgs = LazyImport("sglang.srt.server_args", "ServerArgs") +Engine = LazyImport("sglang.srt.entrypoints.engine", "Engine") + +__all__ = [ + "Engine", + "Runtime", + "assistant", + "assistant_begin", + "assistant_end", + "flush_cache", + "function", + "gen", + "gen_int", + "gen_string", + "get_server_info", + "image", + "select", + "separate_reasoning", + "set_default_backend", + "system", + "system_begin", + "system_end", + "user", + "user_begin", + "user_end", + "video", + "RuntimeEndpoint", + "greedy_token_selection", + "token_length_normalized", + "unconditional_likelihood_normalized", + "ServerArgs", + "Anthropic", + "LiteLLM", + "OpenAI", + "VertexAI", + "global_config", + "__version__", +] diff --git a/python/sglang/bench_offline_throughput.py b/python/sglang/bench_offline_throughput.py new file mode 100644 index 000000000..457d120d9 --- /dev/null +++ b/python/sglang/bench_offline_throughput.py @@ -0,0 +1,452 @@ +""" +Benchmark the throughput in the offline mode. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (the same as bench_serving.py). + +# Usage +## Sharegpt dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --num-prompts 10 + +## Random dataset with default args +python -m sglang.bench_offline_throughput --model-path meta-llama/Meta-Llama-3.1-8B-Instruct --dataset-name random --random-input 1024 --random-output 1024 +""" + +import argparse +import asyncio +import dataclasses +import inspect +import json +import logging +import os +import random +import time +from typing import Dict, List, Optional + +import numpy as np + +from sglang.bench_serving import ( + DatasetRow, + get_dataset, + get_tokenizer, + sample_random_requests, + set_ulimit, +) +from sglang.lang.backend.runtime_endpoint import Runtime +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.server_args import ServerArgs + + +@dataclasses.dataclass +class BenchArgs: + backend: str = "engine" + result_filename: str = "" + dataset_name: str = "sharegpt" + dataset_path: str = "" + num_prompts: int = 1000 + sharegpt_output_len: Optional[int] = None + sharegpt_context_len: Optional[int] = None + random_input_len: int = 1024 + random_output_len: int = 1024 + random_range_ratio: float = 0.0 + gsp_num_groups: int = 64 + gsp_prompts_per_group: int = 16 + gsp_system_prompt_len: int = 2048 + gsp_question_len: int = 128 + gsp_output_len: int = 256 + seed: int = 1 + disable_ignore_eos: bool = False + extra_request_body: Optional[str] = None + apply_chat_template: bool = False + profile: bool = False + skip_warmup: bool = False + do_not_exit: bool = False + prompt_suffix: str = "" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--backend", type=str, default=BenchArgs.backend) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=["sharegpt", "random", "generated-shared-prefix"], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--num-prompts", + type=int, + default=BenchArgs.num_prompts, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=BenchArgs.sharegpt_output_len, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=BenchArgs.sharegpt_context_len, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=BenchArgs.random_input_len, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + type=int, + default=BenchArgs.random_output_len, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=BenchArgs.random_range_ratio, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + parser.add_argument( + "--gsp-num-groups", + type=int, + default=BenchArgs.gsp_num_groups, + help="Number of groups with shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-prompts-per-group", + type=int, + default=BenchArgs.gsp_prompts_per_group, + help="Number of prompts per group of shared prefix, used" + "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-system-prompt-len", + type=int, + default=BenchArgs.gsp_system_prompt_len, + help="System prompt length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-question-len", + type=int, + default=BenchArgs.gsp_question_len, + help="Question length, used" "only for generate-shared-prefix", + ) + parser.add_argument( + "--gsp-output-len", + type=int, + default=BenchArgs.gsp_output_len, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignore EOS token", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + default=BenchArgs.extra_request_body, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--skip-warmup", + action="store_true", + help="Skip the warmup batches.", + ) + parser.add_argument( + "--do-not-exit", + action="store_true", + help="Do not exit the program. This is useful for nsys profile with --duration and --delay.", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + attrs = [attr.name for attr in dataclasses.fields(cls)] + return cls(**{attr: getattr(args, attr) for attr in attrs}) + + +def throughput_test_once( + backend_name: str, + backend, + reqs: List[DatasetRow], + ignore_eos: bool, + extra_request_body: Dict, + profile: bool, +): + measurement_results = { + "backend": backend_name, + "successful_requests": len(reqs), + "total_latency": -1, + "total_input_tokens": sum(r.prompt_len for r in reqs), + "total_output_tokens": -1, + "request_throughput": -1, + "input_throughput": -1, + "output_throughput": -1, + "total_throughput": -1, + } + + prompt = [r.prompt for r in reqs] + sampling_params = [ + { + "temperature": 0, + "max_new_tokens": r.output_len, + "ignore_eos": ignore_eos, + **extra_request_body, + } + for r in reqs + ] + + if profile: + assert ( + "SGLANG_TORCH_PROFILER_DIR" in os.environ + ), "Please set SGLANG_TORCH_PROFILER_DIR." + os.makedirs(os.environ["SGLANG_TORCH_PROFILER_DIR"], exist_ok=True) + backend.start_profile() + + st = time.perf_counter() + gen_out = backend.generate(prompt=prompt, sampling_params=sampling_params) + latency = time.perf_counter() - st + + if profile: + dir = os.getenv("SGLANG_TORCH_PROFILER_DIR") + known_files = set(os.listdir(dir)) + backend.stop_profile() + monitor_trace_file(known_files, dir) + + if backend_name == "runtime": + gen_out = json.loads(gen_out) + + server_info = backend.get_server_info() + + measurement_results["total_latency"] = latency + measurement_results["total_output_tokens"] = sum( + o["meta_info"]["completion_tokens"] for o in gen_out + ) + measurement_results["request_throughput"] = ( + measurement_results["successful_requests"] / latency + ) + measurement_results["input_throughput"] = ( + measurement_results["total_input_tokens"] / latency + ) + measurement_results["output_throughput"] = ( + measurement_results["total_output_tokens"] / latency + ) + measurement_results["total_throughput"] = ( + measurement_results["total_input_tokens"] + + measurement_results["total_output_tokens"] + ) / latency + + if inspect.isawaitable(server_info): + server_info = asyncio.run(server_info) + + measurement_results["last_gen_throughput"] = server_info["internal_states"][0][ + "last_gen_throughput" + ] + + return measurement_results + + +def monitor_trace_file(known_files, directory, interval=1): + print(f"Monitoring {directory} for new trace files...") + + while True: + flag = False + time.sleep(interval) + current_files = set(os.listdir(directory)) + + new_files = current_files - known_files + for new_file in new_files: + new_file_path = os.path.join(directory, new_file) + print(f"New file detected: {new_file}") + + previous_size = 0 + while True: + try: + current_size = os.path.getsize(new_file_path) + except FileNotFoundError: + print(f"File {new_file} is no longer accessible.") + break + + if current_size > previous_size: + previous_size = current_size + else: + flag = True + break + + time.sleep(interval) + if flag: + break + + +def throughput_test( + server_args: ServerArgs, + bench_args: BenchArgs, +): + if bench_args.backend == "engine": + backend = Engine(**dataclasses.asdict(server_args)) + if not backend: + raise ValueError("Please provide valid engine arguments") + elif bench_args.backend == "runtime": + backend = Runtime(**dataclasses.asdict(server_args)) + else: + raise ValueError('Please set backend to either "engine" or "runtime"') + + tokenizer_id = server_args.tokenizer_path or server_args.model_path + tokenizer = get_tokenizer(tokenizer_id) + + # Set global environments + set_ulimit() + random.seed(bench_args.seed) + np.random.seed(bench_args.seed) + + # Parse args + extra_request_body = {} + if bench_args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + # Read dataset + input_requests = get_dataset(bench_args, tokenizer) + + warmup_requests = sample_random_requests( + input_len=256, + output_len=16, + num_prompts=min(bench_args.num_prompts, 16), + range_ratio=1.0, + tokenizer=tokenizer, + dataset_path=bench_args.dataset_path, + ) + + # Warm up + if not bench_args.skip_warmup: + logging.info("\nWarmup...") + throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=warmup_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=False, + ) + time.sleep(0.5) + + logging.info("\nBenchmark...") + result = throughput_test_once( + backend_name=bench_args.backend, + backend=backend, + reqs=input_requests, + ignore_eos=not bench_args.disable_ignore_eos, + extra_request_body=extra_request_body, + profile=bench_args.profile, + ) + backend.shutdown() + + if bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + fout.write(json.dumps(result) + "\n") + + print( + "\n{s:{c}^{n}}".format(s=" Offline Throughput Benchmark Result ", n=50, c="=") + ) + print("{:<40} {:<10}".format("Backend:", result["backend"])) + print("{:<40} {:<10}".format("Successful requests:", result["successful_requests"])) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", result["total_latency"])) + print("{:<40} {:<10}".format("Total input tokens:", result["total_input_tokens"])) + print( + "{:<40} {:<10}".format("Total generated tokens:", result["total_output_tokens"]) + ) + print( + "{:<40} {:<10.2f}".format( + "Last generation throughput (tok/s):", result["last_gen_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", result["request_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", result["input_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", result["output_throughput"] + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", result["total_throughput"] + ) + ) + print("=" * 50) + + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + + # handling ModelScope model downloads + if os.getenv("SGLANG_USE_MODELSCOPE", "false").lower() in ("true", "1"): + if os.path.exists(args.model_path): + print(f"Using local model path: {args.model_path}") + else: + try: + from modelscope import snapshot_download + + print(f"Using ModelScope to download model: {args.model_path}") + + # download the model and replace args.model_path + args.model_path = snapshot_download( + args.model_path, + ) + print(f"Model downloaded to: {args.model_path}") + except Exception as e: + print(f"ModelScope download failed: {str(e)}") + raise e + + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + throughput_test(server_args, bench_args) + + while bench_args.do_not_exit: + pass diff --git a/python/sglang/bench_one_batch.py b/python/sglang/bench_one_batch.py new file mode 100644 index 000000000..ebd461ec3 --- /dev/null +++ b/python/sglang/bench_one_batch.py @@ -0,0 +1,665 @@ +""" +Benchmark the latency of running a single static batch without a server. + +This script does not launch a server and uses the low-level APIs. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +# Usage (latency test) +## with dummy weights: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --load-format dummy +## sweep through multiple data points and store (append) the results in a jsonl file: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --output-len 32 256 --run-name test_run +## run with profiling: +python -m sglang.bench_one_batch --model-path meta-llama/Meta-Llama-3-8B-Instruct --batch 1 12 14 --input-len 256 512 --profile +# Usage (correctness test): +python -m sglang.bench_one_batch --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --correct + +## Reference output (of the correctness test above, can be gpu dependent): +input_ids=[[1, 450, 7483, 310, 3444, 338], [1, 450, 7483, 310, 278, 3303, 13187, 290, 338], [1, 20628, 338, 263, 6575, 1460, 2462, 322, 306, 763]] + +prefill logits (first half): tensor([[-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [-10.0312, -9.5000, 0.8931, ..., -4.9414, -3.2422, -3.3633], + [ -9.1875, -10.2500, 2.7129, ..., -4.3359, -4.0664, -4.1328]], + device='cuda:0') + +prefill logits (final): tensor([[-8.3125, -7.1172, 3.3457, ..., -4.9570, -4.1328, -3.4141], + [-8.9141, -9.0156, 4.1445, ..., -4.9922, -4.4961, -4.0781], + [-9.6328, -9.0547, 4.0195, ..., -5.3047, -4.7148, -4.4570]], + device='cuda:0') + +========== Prompt 0 ========== + The capital of France is Paris. +The capital of the United States is Washington, D.C. + + +========== Prompt 1 ========== + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of the + +========== Prompt 2 ========== + Today is a sunny day and I like to go for a walk in the park. +I'm going to the park +""" + +import argparse +import copy +import dataclasses +import itertools +import json +import logging +import multiprocessing +import os +import time +from typing import Tuple + +import numpy as np +import torch +import torch.distributed as dist + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.distributed.parallel_state import destroy_distributed_environment +from sglang.srt.entrypoints.engine import _set_envs_and_config +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.layers.moe import initialize_moe_config +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.managers.scheduler import Scheduler +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.utils import ( + configure_logger, + get_bool_env_var, + kill_process_tree, + require_mlp_sync, + require_mlp_tp_gather, + set_gpu_proc_affinity, + suppress_other_loggers, +) + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + prompt_filename: str = "" + result_filename: str = "result.jsonl" + correctness_test: bool = False + # This is only used for correctness test + cut_len: int = 4 + log_decode_step: int = 0 + profile: bool = False + profile_record_shapes: bool = False + profile_filename_prefix: str = "profile" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument( + "--prompt-filename", type=str, default=BenchArgs.prompt_filename + ) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument("--correctness-test", action="store_true") + parser.add_argument("--cut-len", type=int, default=BenchArgs.cut_len) + parser.add_argument( + "--log-decode-step", + type=int, + default=BenchArgs.log_decode_step, + help="Log decode latency by step, default is set to zero to disable.", + ) + parser.add_argument( + "--profile", action="store_true", help="Use Torch Profiler." + ) + parser.add_argument( + "--profile-record-shapes", + action="store_true", + help="Record tensor shapes in profiling results.", + ) + parser.add_argument( + "--profile-filename-prefix", + type=str, + default=BenchArgs.profile_filename_prefix, + help="Prefix of the profiling file names. The full profiling result file(s) be " + '"[profile_filename_prefix]_batch[batch_size]_input[input_len]_output[output_len].trace.json.gz"', + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +def load_model(server_args, port_args, tp_rank): + suppress_other_loggers() + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size) + + model_config = ModelConfig.from_server_args(server_args) + model_runner = ModelRunner( + model_config=model_config, + mem_fraction_static=server_args.mem_fraction_static, + gpu_id=tp_rank, + tp_rank=tp_rank, + tp_size=server_args.tp_size, + moe_ep_rank=moe_ep_rank, + moe_ep_size=server_args.ep_size, + pp_rank=0, + pp_size=1, + nccl_port=port_args.nccl_port, + server_args=server_args, + ) + rank_print(f"max_total_num_tokens={model_runner.max_total_num_tokens}") + tokenizer = get_tokenizer( + server_args.tokenizer_path, + tokenizer_mode=server_args.tokenizer_mode, + trust_remote_code=server_args.trust_remote_code, + ) + if server_args.tp_size > 1: + dist.barrier() + return model_runner, tokenizer + + +def prepare_inputs_for_correctness_test(bench_args, tokenizer, custom_prompts): + prompts = ( + custom_prompts + if custom_prompts + else [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + ) + input_ids = [tokenizer.encode(p) for p in prompts] + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(prompts)): + assert len(input_ids[i]) > bench_args.cut_len + + tmp_input_ids = input_ids[i][: bench_args.cut_len] + req = Req( + rid=i, + origin_input_text=prompts[i], + origin_input_ids=tmp_input_ids, + sampling_params=sampling_params, + ) + req.prefix_indices = [] + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + reqs.append(req) + + return input_ids, reqs + + +def prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner +): + for i in range(len(reqs)): + req = reqs[i] + req.fill_ids += input_ids[i][bench_args.cut_len :] + req.prefix_indices = model_runner.req_to_token_pool.req_to_token[ + i, : bench_args.cut_len + ] + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + return reqs + + +def prepare_synthetic_inputs_for_latency_test( + batch_size, input_len, custom_inputs=None +): + input_ids = ( + custom_inputs + if custom_inputs + else np.random.randint(0, 10000, (batch_size, input_len), dtype=np.int32) + ) + sampling_params = SamplingParams( + temperature=0, + max_new_tokens=BenchArgs.output_len, + ) + + reqs = [] + for i in range(len(input_ids)): + req = Req( + rid=i, + origin_input_text="", + origin_input_ids=list(input_ids[i]), + sampling_params=sampling_params, + ) + req.prefix_indices = [] + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + reqs.append(req) + + return reqs + + +@torch.no_grad +def extend(reqs, model_runner): + batch = ScheduleBatch.init_new( + reqs=reqs, + req_to_token_pool=model_runner.req_to_token_pool, + token_to_kv_pool_allocator=model_runner.token_to_kv_pool_allocator, + tree_cache=None, + model_config=model_runner.model_config, + enable_overlap=False, + spec_algorithm=SpeculativeAlgorithm.NONE, + ) + batch.prepare_for_extend() + _maybe_prepare_mlp_sync_batch(batch, model_runner) + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output, _ = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits, batch + + +@torch.no_grad +def decode(input_token_ids, batch, model_runner): + batch.output_ids = input_token_ids + batch.prepare_for_decode() + _maybe_prepare_mlp_sync_batch(batch, model_runner) + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, model_runner) + logits_output, _ = model_runner.forward(forward_batch) + next_token_ids = model_runner.sample(logits_output, forward_batch) + return next_token_ids, logits_output.next_token_logits + + +def _maybe_prepare_mlp_sync_batch(batch: ScheduleBatch, model_runner): + if require_mlp_sync(model_runner.server_args): + Scheduler.prepare_mlp_sync_batch_raw( + batch, + dp_size=model_runner.server_args.dp_size, + attn_tp_size=1, + tp_group=model_runner.tp_group, + get_idle_batch=None, + disable_cuda_graph=model_runner.server_args.disable_cuda_graph, + spec_algorithm=SpeculativeAlgorithm.NONE, + speculative_num_draft_tokens=None, + require_mlp_tp_gather=require_mlp_tp_gather(model_runner.server_args), + disable_overlap_schedule=model_runner.server_args.disable_overlap_schedule, + ) + + +def _read_prompts_from_file(prompt_file, rank_print): + """Read custom prompts from the file specified by `--prompt-filename`.""" + if not prompt_file: + return [] + if not os.path.exists(prompt_file): + rank_print( + f"Custom prompt file {prompt_file} not found. Using default inputs..." + ) + return [] + with open(prompt_file, "r") as pf: + return pf.readlines() + + +def _save_profile_trace_results(profiler, filename): + parent_dir = os.path.dirname(os.path.abspath(filename)) + os.makedirs(parent_dir, exist_ok=True) + profiler.export_chrome_trace(filename) + print( + profiler.key_averages(group_by_input_shape=True).table( + sort_by="self_cpu_time_total" + ) + ) + + +def correctness_test( + server_args, + port_args, + bench_args, + tp_rank, +): + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, tp_rank) + + # Prepare inputs + custom_prompts = _read_prompts_from_file(bench_args.prompt_filename, rank_print) + input_ids, reqs = prepare_inputs_for_correctness_test( + bench_args, tokenizer, custom_prompts + ) + rank_print(f"\n{input_ids=}\n") + + if bench_args.cut_len > 0: + # Prefill + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (first half): {next_token_logits} \n") + + # Prepare extend inputs + reqs = prepare_extend_inputs_for_correctness_test( + bench_args, input_ids, reqs, model_runner + ) + + # Extend (prefill w/ KV cache) + next_token_ids, next_token_logits, batch = extend(reqs, model_runner) + rank_print(f"prefill logits (final): {next_token_logits} \n") + + # Decode + output_ids = [input_ids[i] + [next_token_ids[i]] for i in range(len(input_ids))] + for _ in range(bench_args.output_len[0] - 1): + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + next_token_ids_list = next_token_ids.tolist() + for i in range(len(reqs)): + output_ids[i].append(next_token_ids_list[i]) + + # Print output texts + for i in range(len(reqs)): + rank_print(f"========== Prompt {i} ==========") + rank_print(tokenizer.decode(output_ids[i]), "\n") + + +def synchronize(device): + torch.get_device_module(device).synchronize() + + +def latency_test_run_once( + run_name, + model_runner, + rank_print, + reqs, + batch_size, + input_len, + output_len, + device, + log_decode_step, + profile, + profile_record_shapes, + profile_filename_prefix, +): + max_batch_size = model_runner.max_total_num_tokens // (input_len + output_len) + if batch_size > max_batch_size: + rank_print( + f"skipping ({batch_size}, {input_len}, {output_len}) due to max batch size limit" + ) + return + + # Clear the pools. + model_runner.req_to_token_pool.clear() + model_runner.token_to_kv_pool_allocator.clear() + + measurement_results = { + "run_name": run_name, + "batch_size": batch_size, + "input_len": input_len, + "output_len": output_len, + } + + tot_latency = 0 + + profiler = None + if profile: + profiler = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, + record_shapes=profile_record_shapes, + ) + profiler.start() + + # Prefill + synchronize(device) + tic = time.perf_counter() + next_token_ids, _, batch = extend(reqs, model_runner) + synchronize(device) + prefill_latency = time.perf_counter() - tic + tot_latency += prefill_latency + throughput = input_len * batch_size / prefill_latency + rank_print( + f"Prefill. latency: {prefill_latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["prefill_latency"] = prefill_latency + measurement_results["prefill_throughput"] = throughput + + if profile: + profiler.stop() + profile_filename = f"{profile_filename_prefix}_batch{batch_size}_input{input_len}_output{output_len}_prefill.trace.json.gz" + _save_profile_trace_results(profiler, profile_filename) + rank_print( + f"torch profiler chrome trace for prefill saved to {profile_filename}" + ) + + # Decode + decode_latencies = [] + for i in range(output_len - 1): + synchronize(device) + if profile and i == output_len / 2: + profiler = None + profiler = torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + with_stack=True, + record_shapes=profile_record_shapes, + ) + profiler.start() + + tic = time.perf_counter() + next_token_ids, _ = decode(next_token_ids, batch, model_runner) + synchronize(device) + latency = time.perf_counter() - tic + tot_latency += latency + throughput = batch_size / latency + decode_latencies.append(latency) + if i < 5 or (log_decode_step > 0 and i % log_decode_step == 0): + rank_print( + f"Decode {i}. Batch size: {batch_size}, latency: {latency:6.5f} s, throughput: {throughput:9.2f} token/s" + ) + + if profile and i == output_len / 2: + profiler.stop() + profile_filename = f"{profile_filename_prefix}_batch{batch_size}_input{input_len}_output{output_len}_decode.trace.json.gz" + _save_profile_trace_results(profiler, profile_filename) + rank_print( + f"torch profiler chrome trace for decoding 1 token saved to {profile_filename}" + ) + + # Record decode timing from 2nd output + if output_len > 1: + med_decode_latency = np.median(decode_latencies) + med_decode_throughput = batch_size / med_decode_latency + rank_print( + f"Decode. median latency: {med_decode_latency:6.5f} s, median throughput: {med_decode_throughput:9.2f} token/s" + ) + measurement_results["median_decode_latency"] = med_decode_latency + measurement_results["median_decode_throughput"] = med_decode_throughput + + throughput = (input_len + output_len) * batch_size / tot_latency + rank_print( + f"Total. latency: {tot_latency:6.3f} s, throughput: {throughput:9.2f} token/s" + ) + measurement_results["total_latency"] = tot_latency + measurement_results["overall_throughput"] = throughput + return measurement_results + + +def latency_test( + server_args, + port_args, + bench_args, + tp_rank, +): + initialize_moe_config(server_args) + + # Set CPU affinity + if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"): + set_gpu_proc_affinity(server_args.tp_size, server_args.nnodes, tp_rank) + + # Configure the logger + configure_logger(server_args, prefix=f" TP{tp_rank}") + rank_print = print if tp_rank == 0 else lambda *args, **kwargs: None + + # Load the model + model_runner, tokenizer = load_model(server_args, port_args, tp_rank) + + # Prepare inputs for warm up + reqs = prepare_synthetic_inputs_for_latency_test( + bench_args.batch_size[0], bench_args.input_len[0] + ) + + # Warm up + rank_print("Warmup ...") + latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bench_args.batch_size[0], + bench_args.input_len[0], + min(32, bench_args.output_len[0]), # shorter decoding to speed up the warmup + server_args.device, + log_decode_step=0, + profile=False, + profile_record_shapes=False, + profile_filename_prefix="", # not used + ) + + rank_print("Benchmark ...") + + custom_inputs = _read_prompts_from_file(bench_args.prompt_filename, rank_print) + custom_inputs = [tokenizer.encode(p.strip()) for p in custom_inputs] + custom_input_len = len(custom_inputs) + + # Run the sweep + result_list = [] + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + bs_aligned_inputs = [] + if custom_inputs: + if custom_input_len == bs: + bs_aligned_inputs = custom_inputs + elif custom_input_len > bs: + rank_print( + f"Custom input size ({custom_input_len}) is larger than batch_size ({bs}). " + f"Using the first {bs} prompts." + ) + bs_aligned_inputs = copy.deepcopy(custom_inputs[:bs]) + else: + rank_print( + f"Custom input size ({custom_input_len}) is smaller than batch_size ({bs}). " + f"Pad to the desired batch_size with the last prompt." + ) + bs_aligned_inputs = copy.deepcopy(custom_inputs) + bs_aligned_inputs.extend( + [bs_aligned_inputs[-1]] * (bs - custom_input_len) + ) + + reqs = prepare_synthetic_inputs_for_latency_test(bs, il, bs_aligned_inputs) + ret = latency_test_run_once( + bench_args.run_name, + model_runner, + rank_print, + reqs, + bs, + il, + ol, + server_args.device, + bench_args.log_decode_step, + bench_args.profile if tp_rank == 0 else None, + bench_args.profile_record_shapes if tp_rank == 0 else None, + bench_args.profile_filename_prefix, + ) + if ret is not None: + result_list.append(ret) + + # Write results in jsonlines format on rank 0. + if tp_rank == 0 and bench_args.result_filename: + with open(bench_args.result_filename, "a") as fout: + for result in result_list: + fout.write(json.dumps(result) + "\n") + + if server_args.tp_size > 1: + destroy_distributed_environment() + + +def main(server_args, bench_args): + server_args.cuda_graph_max_bs = max(bench_args.batch_size) + + _set_envs_and_config(server_args) + + if server_args.model_path: + if bench_args.correctness_test: + work_func = correctness_test + else: + work_func = latency_test + else: + raise ValueError( + "Provide --model-path for running the tests or " + "provide --result-filename for plotting the results" + ) + + port_args = PortArgs.init_new(server_args) + + if server_args.tp_size == 1: + work_func(server_args, port_args, bench_args, 0) + else: + workers = [] + for tp_rank in range(server_args.tp_size): + proc = multiprocessing.Process( + target=work_func, + args=( + server_args, + port_args, + bench_args, + tp_rank, + ), + ) + proc.start() + workers.append(proc) + + for proc in workers: + proc.join() + + proc.terminate() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + logging.basicConfig( + level=getattr(logging, server_args.log_level.upper()), + format="%(message)s", + ) + + try: + main(server_args, bench_args) + finally: + if server_args.tp_size != 1: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/python/sglang/bench_one_batch_server.py b/python/sglang/bench_one_batch_server.py new file mode 100644 index 000000000..8495c110e --- /dev/null +++ b/python/sglang/bench_one_batch_server.py @@ -0,0 +1,439 @@ +""" +Benchmark the latency of running a single batch with a server. + +This script launches a server and uses the HTTP interface. +It accepts server arguments (the same as launch_server.py) and benchmark arguments (e.g., batch size, input lengths). + +Usage: +python3 -m sglang.bench_one_batch_server --model meta-llama/Meta-Llama-3.1-8B --batch-size 1 16 64 --input-len 1024 --output-len 8 + +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 +python3 -m sglang.bench_one_batch_server --model None --base-url http://localhost:30000 --batch-size 16 --input-len 1024 --output-len 8 --show-report --profile --profile-by-stage +""" + +import argparse +import dataclasses +import itertools +import json +import multiprocessing +import os +import time +from typing import List, Tuple + +import requests + +from sglang.bench_serving import get_tokenizer, sample_random_requests +from sglang.profiler import run_profile +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_blackwell, kill_process_tree +from sglang.test.test_utils import is_in_ci, write_github_step_summary + + +@dataclasses.dataclass +class BenchArgs: + run_name: str = "default" + batch_size: Tuple[int] = (1,) + input_len: Tuple[int] = (1024,) + output_len: Tuple[int] = (16,) + temperature: float = 0.0 + return_logprob: bool = False + client_stream_interval: int = 1 + input_len_step_percentage: float = 0.0 + result_filename: str = "result.jsonl" + base_url: str = "" + skip_warmup: bool = False + show_report: bool = False + profile: bool = False + profile_steps: int = 3 + profile_by_stage: bool = False + dataset_path: str = "" + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--run-name", type=str, default=BenchArgs.run_name) + parser.add_argument( + "--batch-size", type=int, nargs="+", default=BenchArgs.batch_size + ) + parser.add_argument( + "--input-len", type=int, nargs="+", default=BenchArgs.input_len + ) + parser.add_argument( + "--output-len", type=int, nargs="+", default=BenchArgs.output_len + ) + parser.add_argument("--temperature", type=float, default=BenchArgs.temperature) + parser.add_argument("--return-logprob", action="store_true") + parser.add_argument( + "--client-stream-interval", + type=int, + default=BenchArgs.client_stream_interval, + ) + parser.add_argument( + "--input-len-step-percentage", + type=float, + default=BenchArgs.input_len_step_percentage, + ) + parser.add_argument( + "--result-filename", type=str, default=BenchArgs.result_filename + ) + parser.add_argument("--base-url", type=str, default=BenchArgs.base_url) + parser.add_argument("--skip-warmup", action="store_true") + parser.add_argument("--show-report", action="store_true") + parser.add_argument("--profile", action="store_true") + parser.add_argument( + "--profile-steps", type=int, default=BenchArgs.profile_steps + ) + parser.add_argument("--profile-by-stage", action="store_true") + parser.add_argument( + "--dataset-path", + type=str, + default=BenchArgs.dataset_path, + help="Path to the dataset.", + ) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +def launch_server_internal(server_args): + try: + launch_server(server_args) + except Exception as e: + raise e + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +def launch_server_process(server_args: ServerArgs): + proc = multiprocessing.Process(target=launch_server_internal, args=(server_args,)) + proc.start() + base_url = f"http://{server_args.host}:{server_args.port}" + timeout = 600 + + start_time = time.time() + while time.time() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + } + response = requests.get(f"{base_url}/v1/models", headers=headers) + if response.status_code == 200: + return proc, base_url + except requests.RequestException: + pass + time.sleep(10) + raise TimeoutError("Server failed to start within the timeout period.") + + +def run_one_case( + url: str, + batch_size: int, + input_len: int, + output_len: int, + temperature: float, + return_logprob: bool, + stream_interval: int, + input_len_step_percentage: float, + run_name: str, + result_filename: str, + tokenizer, + profile: bool = False, + profile_steps: int = 3, + profile_by_stage: bool = False, + dataset_path: str = "", +): + requests.post(url + "/flush_cache") + input_requests = sample_random_requests( + input_len=input_len, + output_len=output_len, + num_prompts=batch_size, + range_ratio=1.0, + tokenizer=tokenizer, + dataset_path=dataset_path, + random_sample=True, + return_text=False, + ) + + use_structured_outputs = False + if use_structured_outputs: + texts = [] + for _ in range(batch_size): + texts.append( + "Human: What is the capital city of france? can you give as many trivial information as possible about that city? answer in json.\n" + * 50 + + "Assistant:" + ) + json_schema = "$$ANY$$" + else: + json_schema = None + + profile_link = None + if profile: + profile_link: str = run_profile( + url, profile_steps, ["CPU", "GPU"], None, None, profile_by_stage + ) + + tic = time.perf_counter() + response = requests.post( + url + "/generate", + json={ + "input_ids": [req.prompt for req in input_requests], + "sampling_params": { + "temperature": temperature, + "max_new_tokens": output_len, + "ignore_eos": True, + "json_schema": json_schema, + "stream_interval": stream_interval, + }, + "return_logprob": return_logprob, + "stream": True, + }, + stream=True, + ) + + # The TTFT of the last request in the batch + ttft = 0.0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + if "error" in data: + raise RuntimeError(f"Request has failed. {data}.") + + assert ( + data["meta_info"]["finish_reason"] is None + or data["meta_info"]["finish_reason"]["type"] == "length" + ) + if data["meta_info"]["completion_tokens"] == 1: + ttft = time.perf_counter() - tic + + latency = time.perf_counter() - tic + input_throughput = batch_size * input_len / ttft + output_throughput = batch_size * output_len / (latency - ttft) + overall_throughput = batch_size * (input_len + output_len) / latency + + server_info = requests.get(url + "/get_server_info").json() + acc_length = server_info["internal_states"][0].get("avg_spec_accept_length", None) + last_gen_throughput = server_info["internal_states"][0]["last_gen_throughput"] + + print(f"batch size: {batch_size}") + print(f"input_len: {input_len}") + print(f"output_len: {output_len}") + print(f"latency: {latency:.2f} s") + print(f"ttft: {ttft:.2f} s") + print(f"last generation throughput: {last_gen_throughput:.2f} tok/s") + print(f"input throughput: {input_throughput:.2f} tok/s") + if output_len != 1: + print(f"output throughput: {output_throughput:.2f} tok/s") + + if result_filename: + with open(result_filename, "a") as fout: + res = { + "run_name": run_name, + "batch_size": batch_size, + "input_len": input_len, + "output_len": output_len, + "latency": round(latency, 4), + "output_throughput": round(output_throughput, 2), + "overall_throughput": round(overall_throughput, 2), + "last_gen_throughput": round(last_gen_throughput, 2), + } + fout.write(json.dumps(res) + "\n") + + return ( + batch_size, + latency, + ttft, + input_throughput, + output_throughput, + overall_throughput, + last_gen_throughput, + acc_length, + profile_link if profile else None, + ) + + +def get_report_summary( + result: List[Tuple], server_args: ServerArgs, bench_args: BenchArgs +): + import tabulate + + summary = ( + f"\nInput lens: {bench_args.input_len}. Output lens: {bench_args.output_len}.\n" + ) + + headers = [ + "batch size", + "latency (s)", + "input throughput (tok/s)", + "output throughput (tok/s)", + "acc length", + "ITL (ms)", + "input cost ($/1M)", + "output cost ($/1M)", + ] + if bench_args.profile: + headers.append("profile") + rows = [] + + for ( + batch_size, + latency, + ttft, + input_throughput, + output_throughput, + _, + _, + acc_length, + trace_link, + ) in result: + if is_blackwell(): + hourly_cost_per_gpu = 4 # $4/hour for one B200 + else: + hourly_cost_per_gpu = 2 # $2/hour for one H100 + + hourly_cost = hourly_cost_per_gpu * server_args.tp_size + input_util = 0.7 + accept_length = round(acc_length, 2) if acc_length is not None else "n/a" + itl = 1 / (output_throughput / batch_size) * 1000 + input_cost = 1e6 / (input_throughput * input_util) / 3600 * hourly_cost + output_cost = 1e6 / output_throughput / 3600 * hourly_cost + row = [ + batch_size, + latency, + input_throughput, + output_throughput, + accept_length, + itl, + input_cost, + output_cost, + ] + if trace_link: + row.append(f"[Profile]({trace_link})") + rows.append(row) + + summary += tabulate.tabulate( + rows, headers=headers, tablefmt="github", floatfmt=".2f" + ) + return summary + + +def run_benchmark(server_args: ServerArgs, bench_args: BenchArgs): + if bench_args.base_url: + proc, base_url = None, bench_args.base_url + else: + proc, base_url = launch_server_process(server_args) + + server_info = requests.get(base_url + "/get_server_info").json() + if "tokenizer_path" in server_info: + tokenizer_path = server_info["tokenizer_path"] + elif "prefill" in server_info: + tokenizer_path = server_info["prefill"][0]["tokenizer_path"] + tokenizer = get_tokenizer(tokenizer_path) + + # warmup + if not bench_args.skip_warmup: + print("=" * 8 + " Warmup Begin " + "=" * 8) + run_one_case( + base_url, + batch_size=16, + input_len=1024, + output_len=16, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name="", + result_filename="", + tokenizer=tokenizer, + dataset_path=bench_args.dataset_path, + ) + print("=" * 8 + " Warmup End " + "=" * 8 + "\n") + + # benchmark + result = [] + bench_result = [] + try: + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + result.append( + run_one_case( + base_url, + bs, + il, + ol, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name=bench_args.run_name, + result_filename=bench_args.result_filename, + tokenizer=tokenizer, + ) + ) + + if bench_args.profile: + try: + for bs, il, ol in itertools.product( + bench_args.batch_size, bench_args.input_len, bench_args.output_len + ): + bench_result.append( + ( + run_one_case( + base_url, + bs, + il, + ol, + temperature=bench_args.temperature, + return_logprob=bench_args.return_logprob, + stream_interval=bench_args.client_stream_interval, + input_len_step_percentage=bench_args.input_len_step_percentage, + run_name=bench_args.run_name, + result_filename=bench_args.result_filename, + tokenizer=tokenizer, + profile=bench_args.profile, + profile_steps=bench_args.profile_steps, + profile_by_stage=bench_args.profile_by_stage, + )[-1], + ) + ) + result = [t1[:-1] + t2 for t1, t2 in zip(result, bench_result)] + except Exception as e: + print(f"Error profiling, there will be no profile trace dump: {e}") + finally: + if proc: + kill_process_tree(proc.pid) + + print(f"\nResults are saved to {bench_args.result_filename}") + + if not bench_args.show_report: + return + + summary = get_report_summary(result, server_args, bench_args) + print(summary) + + if is_in_ci(): + write_github_step_summary(summary) + + +def main(): + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + BenchArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + bench_args = BenchArgs.from_cli_args(args) + + run_benchmark(server_args, bench_args) + + +if __name__ == "__main__": + main() diff --git a/python/sglang/bench_serving.py b/python/sglang/bench_serving.py new file mode 100644 index 000000000..27ff8a6da --- /dev/null +++ b/python/sglang/bench_serving.py @@ -0,0 +1,2403 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/backend_request_func.py +# Adapted from https://github.com/vllm-project/vllm/blob/6366efc67b0aedd2c1721c14385370e50b297fb3/benchmarks/benchmark_serving.py + +""" +Benchmark online serving with dynamic requests. + +Usage: +python3 -m sglang.bench_serving --backend sglang --num-prompt 10 + +python3 -m sglang.bench_serving --backend sglang --dataset-name random --num-prompts 3000 --random-input 1024 --random-output 1024 --random-range-ratio 0.5 +""" + +import argparse +import asyncio +import base64 +import io +import json +import os +import pickle +import random +import resource +import sys +import time +import traceback +import warnings +from argparse import ArgumentParser +from dataclasses import dataclass, field +from datetime import datetime +from json import JSONDecodeError +from pathlib import Path +from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple, Union + +import aiohttp +import numpy as np +import requests +from tqdm.asyncio import tqdm +from transformers import ( + AutoTokenizer, + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, +) + +ASSISTANT_SUFFIX = "Assistant:" + +global args + + +# don't want to import sglang package here +def _get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def _create_bench_client_session(): + # When the pressure is big, the read buffer could be full before aio thread read + # the content. We increase the read_bufsize from 64K to 10M. + # Define constants for timeout and buffer size for clarity and maintainability + BENCH_AIOHTTP_TIMEOUT_SECONDS = 6 * 60 * 60 # 6 hours + BENCH_AIOHTTP_READ_BUFSIZE_BYTES = 10 * 1024**2 # 10 MB + + aiohttp_timeout = aiohttp.ClientTimeout(total=BENCH_AIOHTTP_TIMEOUT_SECONDS) + return aiohttp.ClientSession( + timeout=aiohttp_timeout, read_bufsize=BENCH_AIOHTTP_READ_BUFSIZE_BYTES + ) + + +@dataclass +class RequestFuncInput: + prompt: str + api_url: str + prompt_len: int + output_len: int + model: str + lora_name: str + image_data: Optional[List[str]] + extra_request_body: Dict[str, Any] + timestamp: Optional[float] = None + + +@dataclass +class RequestFuncOutput: + generated_text: str = "" + success: bool = False + latency: float = 0.0 + ttft: float = 0.0 # Time to first token + itl: List[float] = field(default_factory=list) # List of inter-token latencies + prompt_len: int = 0 + error: str = "" + output_len: int = 0 + + @staticmethod + def init_new(request_func_input: RequestFuncInput): + output = RequestFuncOutput() + output.prompt_len = request_func_input.prompt_len + return output + + +def remove_prefix(text: str, prefix: str) -> str: + return text[len(prefix) :] if text.startswith(prefix) else text + + +def remove_suffix(text: str, suffix: str) -> str: + return text[: -len(suffix)] if text.endswith(suffix) else text + + +def get_auth_headers() -> Dict[str, str]: + api_key = os.environ.get("OPENAI_API_KEY") + if api_key: + return {"Authorization": f"Bearer {api_key}"} + else: + return {} + + +# trt llm does not support ignore_eos +# https://github.com/triton-inference-server/tensorrtllm_backend/issues/505 +async def async_request_trt_llm( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith("generate_stream") + + async with _create_bench_client_session() as session: + payload = { + "accumulate_tokens": True, + "text_input": request_func_input.prompt, + "temperature": 0.000001, + "top_p": 1.0, + "max_tokens": request_func_input.output_len, + "stream": True, + "min_length": request_func_input.output_len, + "end_id": 1048576, + **request_func_input.extra_request_body, + } + if args.disable_ignore_eos: + del payload["min_length"] + del payload["end_id"] + output = RequestFuncOutput.init_new(request_func_input) + + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post(url=api_url, json=payload) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data:") + + data = json.loads(chunk) + output.generated_text += data["text_output"] + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + + output.latency = most_recent_timestamp - st + output.success = True + output.output_len = request_func_input.output_len + + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +# set ignore_eos True by default +async def async_request_openai_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + assert api_url.endswith( + "completions" + ), "OpenAI Completions API URL must end with 'completions'." + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_openai_chat_completions( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + """Makes a request to the OpenAI Chat Completions API. + + Handles both streaming and non-streaming responses, including support + for image data in messages. Calculates and returns various performance + metrics. + + Args: + request_func_input: Input parameters for the request. + pbar: Optional tqdm progress bar to update. + + Returns: + RequestFuncOutput: Output of the request, including generated text, + latency, TTFT, ITL, and success status. + """ + api_url = request_func_input.api_url + assert api_url.endswith( + "chat/completions" + ), "OpenAI Chat Completions API URL must end with 'chat/completions'." + + if request_func_input.image_data: + # Build multi-image content: a list of image_url entries followed by the text + content_items = [ + { + "type": "image_url", + "image_url": {"url": img_url}, + } + for img_url in request_func_input.image_data + ] + content_items.append({"type": "text", "text": request_func_input.prompt}) + messages = [ + { + "role": "user", + "content": content_items, + }, + ] + else: + messages = [{"role": "user", "content": request_func_input.prompt}] + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "messages": messages, + "temperature": 0.0, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + if args.disable_stream: + # Non-streaming response + response_json = await response.json() + output.generated_text = response_json["choices"][0]["message"][ + "content" + ] + output.success = True + output.latency = time.perf_counter() - st + output.ttft = ( + output.latency + ) # For non-streaming, TTFT = total latency + output.output_len = response_json.get("usage", {}).get( + "completion_tokens", output_len + ) + else: + # Streaming response + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # Check if this chunk contains content + delta = data.get("choices", [{}])[0].get("delta", {}) + content = delta.get("content", "") + + if content: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = timestamp - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append( + timestamp - most_recent_timestamp + ) + + most_recent_timestamp = timestamp + generated_text += content + + # Check for usage info in final chunk + output_len = (data.get("usage") or {}).get( + "completion_tokens", output_len + ) + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_truss( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + "model": request_func_input.model, + "prompt": prompt, + "temperature": 0.0, + "best_of": 1, + "max_tokens": request_func_input.output_len, + "stream": not args.disable_stream, + "ignore_eos": not args.disable_ignore_eos, + **request_func_input.extra_request_body, + } + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if data["choices"][0]["text"]: + timestamp = time.perf_counter() + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + output.itl.append(timestamp - most_recent_timestamp) + + most_recent_timestamp = timestamp + generated_text += data["choices"][0]["text"] + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = request_func_input.output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + if pbar: + pbar.update(1) + return output + + +async def async_request_sglang_generate( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + api_url = request_func_input.api_url + prompt = request_func_input.prompt + + async with _create_bench_client_session() as session: + payload = { + ("text" if isinstance(prompt, str) else "input_ids"): prompt, + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": request_func_input.output_len, + "ignore_eos": not args.disable_ignore_eos, + }, + "stream": not args.disable_stream, + "lora_path": request_func_input.lora_name, + "return_logprob": args.return_logprob, + "logprob_start_len": -1, + **request_func_input.extra_request_body, + } + + # Add image data if available (list of image urls/base64) + if request_func_input.image_data: + payload["image_data"] = request_func_input.image_data + + headers = get_auth_headers() + + output = RequestFuncOutput.init_new(request_func_input) + + generated_text = "" + output_len = request_func_input.output_len + ttft = 0.0 + st = time.perf_counter() + most_recent_timestamp = st + last_output_len = 0 + try: + async with session.post( + url=api_url, json=payload, headers=headers + ) as response: + if response.status == 200: + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + latency = time.perf_counter() - st + if chunk == "[DONE]": + pass + else: + data = json.loads(chunk) + + # NOTE: Some completion API might have a last + # usage summary response without a token so we + # want to check a token was generated + if "text" in data and data["text"]: + timestamp = time.perf_counter() + generated_text = data["text"] + output_len = data["meta_info"]["completion_tokens"] + + # First token + if ttft == 0.0: + ttft = time.perf_counter() - st + output.ttft = ttft + + # Decoding phase + else: + num_new_tokens = output_len - last_output_len + if num_new_tokens == 0: + continue + adjust_itl = ( + timestamp - most_recent_timestamp + ) / num_new_tokens + output.itl.extend([adjust_itl] * num_new_tokens) + + most_recent_timestamp = timestamp + last_output_len = output_len + + output.generated_text = generated_text + output.success = True + output.latency = latency + output.output_len = output_len + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + print(f"{output.error=}") + + if pbar: + pbar.update(1) + return output + + +async def async_request_gserver( + request_func_input: RequestFuncInput, + pbar: Optional[tqdm] = None, +) -> RequestFuncOutput: + raise NotImplementedError() + + +async def async_request_profile(api_url: str) -> RequestFuncOutput: + async with _create_bench_client_session() as session: + output = RequestFuncOutput() + try: + async with session.post(url=api_url) as response: + if response.status == 200: + output.success = True + else: + output.error = response.reason or "" + output.success = False + except Exception: + output.success = False + exc_info = sys.exc_info() + output.error = "".join(traceback.format_exception(*exc_info)) + + return output + + +def get_model(pretrained_model_name_or_path: str) -> str: + if os.getenv("SGLANG_USE_MODELSCOPE", "false").lower() == "true": + import huggingface_hub.constants + from modelscope import snapshot_download + + model_path = snapshot_download( + model_id=pretrained_model_name_or_path, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_file_pattern=[".*.pt", ".*.safetensors", ".*.bin"], + ) + + return model_path + return pretrained_model_name_or_path + + +def get_tokenizer( + pretrained_model_name_or_path: str, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + assert ( + pretrained_model_name_or_path is not None + and pretrained_model_name_or_path != "" + ) + if pretrained_model_name_or_path.endswith( + ".json" + ) or pretrained_model_name_or_path.endswith(".model"): + from sglang.srt.hf_transformers_utils import get_tokenizer + + return get_tokenizer(pretrained_model_name_or_path) + + if pretrained_model_name_or_path is not None and not os.path.exists( + pretrained_model_name_or_path + ): + pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) + return AutoTokenizer.from_pretrained( + pretrained_model_name_or_path, trust_remote_code=True + ) + + +def get_dataset(args, tokenizer): + tokenize_prompt = getattr(args, "tokenize_prompt", False) + if args.dataset_name == "sharegpt": + assert not tokenize_prompt + input_requests = sample_sharegpt_requests( + dataset_path=args.dataset_path, + num_requests=args.num_prompts, + tokenizer=tokenizer, + fixed_output_len=args.sharegpt_output_len, + context_len=args.sharegpt_context_len, + prompt_suffix=args.prompt_suffix, + apply_chat_template=args.apply_chat_template, + ) + elif args.dataset_name.startswith("random") and args.dataset_name != "random-image": + input_requests = sample_random_requests( + input_len=args.random_input_len, + output_len=args.random_output_len, + num_prompts=args.num_prompts, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + dataset_path=args.dataset_path, + random_sample=args.dataset_name == "random", + return_text=not tokenize_prompt, + ) + elif args.dataset_name == "random-image": + assert not tokenize_prompt, "random-image does not support --tokenize-prompt" + input_requests = sample_random_image_requests( + num_requests=args.num_prompts, + num_images=args.random_image_num_images, + input_len=args.random_input_len, + output_len=args.random_output_len, + range_ratio=args.random_range_ratio, + tokenizer=tokenizer, + apply_chat_template=args.apply_chat_template, + image_resolution=args.random_image_resolution, + ) + elif args.dataset_name == "generated-shared-prefix": + assert not tokenize_prompt + input_requests = sample_generated_shared_prefix_requests( + num_groups=args.gsp_num_groups, + prompts_per_group=args.gsp_prompts_per_group, + system_prompt_len=args.gsp_system_prompt_len, + question_len=args.gsp_question_len, + output_len=args.gsp_output_len, + tokenizer=tokenizer, + args=args, + ) + elif args.dataset_name == "mmmu": + assert not tokenize_prompt + input_requests = sample_mmmu_requests( + num_requests=args.num_prompts, + tokenizer=tokenizer, + fixed_output_len=args.random_output_len, + apply_chat_template=args.apply_chat_template, + random_sample=True, + ) + elif args.dataset_name == "mooncake": + # For mooncake, we don't generate the prompts here. + # We just load the raw trace data. The async generator will handle the rest. + if not args.dataset_path: + local_path = os.path.join("/tmp", args.mooncake_workload + "_trace.jsonl") + else: + local_path = args.dataset_path + + if not os.path.exists(local_path): + download_and_cache_file( + MOONCAKE_DATASET_URL[args.mooncake_workload], local_path + ) + + with open(local_path, "r") as f: + all_requests_data = [json.loads(line) for line in f if line.strip()] + + # Limit the number of requests based on --num-prompts + input_requests = all_requests_data[: args.num_prompts] + else: + raise ValueError(f"Unknown dataset: {args.dataset_name}") + return input_requests + + +ASYNC_REQUEST_FUNCS = { + "sglang": async_request_sglang_generate, + "sglang-native": async_request_sglang_generate, + "sglang-oai": async_request_openai_completions, + "sglang-oai-chat": async_request_openai_chat_completions, + "vllm": async_request_openai_completions, + "vllm-chat": async_request_openai_chat_completions, + "lmdeploy": async_request_openai_completions, + "lmdeploy-chat": async_request_openai_chat_completions, + "trt": async_request_trt_llm, + "gserver": async_request_gserver, + "truss": async_request_truss, +} + + +@dataclass +class BenchmarkMetrics: + completed: int + total_input: int + total_output: int + total_output_retokenized: int + request_throughput: float + input_throughput: float + output_throughput: float + output_throughput_retokenized: float + total_throughput: float + total_throughput_retokenized: float + mean_ttft_ms: float + median_ttft_ms: float + std_ttft_ms: float + p99_ttft_ms: float + mean_tpot_ms: float + median_tpot_ms: float + std_tpot_ms: float + p99_tpot_ms: float + mean_itl_ms: float + median_itl_ms: float + std_itl_ms: float + p95_itl_ms: float + p99_itl_ms: float + max_itl_ms: float + mean_e2e_latency_ms: float + median_e2e_latency_ms: float + std_e2e_latency_ms: float + p99_e2e_latency_ms: float + concurrency: float + + +SHAREGPT_URL = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" +MOONCAKE_DATASET_URL = { + "mooncake": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/arxiv-trace/mooncake_trace.jsonl", + "conversation": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/conversation_trace.jsonl", + "synthetic": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/synthetic_trace.jsonl", + "toolagent": "https://raw.githubusercontent.com/kvcache-ai/Mooncake/main/FAST25-release/traces/toolagent_trace.jsonl", +} + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if is_file_valid_json(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def is_file_valid_json(path): + if not os.path.isfile(path): + return False + + # TODO can fuse into the real file open later + try: + with open(path) as f: + json.load(f) + return True + except JSONDecodeError as e: + print( + f"{path} exists but json loading fails ({e=}), thus treat as invalid file" + ) + return False + + +@dataclass +class DatasetRow: + prompt: str + prompt_len: int + output_len: int + image_data: Optional[List[str]] = None + timestamp: Optional[float] = None + + +async def get_mooncake_request_over_time( + input_requests: List[Dict], + tokenizer: PreTrainedTokenizerBase, + slowdown_factor: float, + num_rounds: int, +) -> AsyncGenerator[DatasetRow, None]: + """ + An async generator that yields requests based on the timestamps in the Mooncake trace file, + with support for multi-round sessions. + """ + if not input_requests: + return + + input_requests.sort(key=lambda r: r["timestamp"]) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0]["timestamp"] + + for record in input_requests: + # Calculate when this entire session should start + relative_arrival_time_s = (record["timestamp"] - trace_start_time_ms) / 1000.0 + target_arrival_time_s = relative_arrival_time_s * slowdown_factor + + current_elapsed_time_s = time.perf_counter() - start_time + sleep_duration_s = target_arrival_time_s - current_elapsed_time_s + if sleep_duration_s > 0: + await asyncio.sleep(sleep_duration_s) + + # Once the session starts, generate all rounds for it as a burst + # This simulates a user engaging in a multi-turn conversation + + # Base user query constructed from hash_ids + user_query_base = "" + hash_ids = record.get("hash_ids", []) + for hash_id in hash_ids: + user_query_base += f"{hash_id}" + " ".join( + ["hi"] * 128 + ) # Shorter for multi-round + user_query_base += "Tell me a story based on this context." + + output_len_per_round = record.get("output_length", 256) + chat_history = [] + + for i in range(num_rounds): + # Add user query for the current round + chat_history.append( + {"role": "user", "content": f"Round {i+1}: {user_query_base}"} + ) + + # Form the full prompt from history + try: + full_prompt_text = tokenizer.apply_chat_template( + chat_history, tokenize=False, add_generation_prompt=True + ) + except Exception: + full_prompt_text = "\n".join( + [f"{msg['role']}: {msg['content']}" for msg in chat_history] + ) + + prompt_len = len(tokenizer.encode(full_prompt_text)) + + yield DatasetRow( + prompt=full_prompt_text, + prompt_len=prompt_len, + output_len=output_len_per_round, + ) + + # Add a placeholder assistant response for the next round's context + # We use a placeholder because we don't know the real response + placeholder_response = " ".join(["story"] * output_len_per_round) + chat_history.append({"role": "assistant", "content": placeholder_response}) + + +def sample_mmmu_requests( + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + fixed_output_len: Optional[int] = None, + apply_chat_template: bool = True, + random_sample: bool = True, +) -> List[DatasetRow]: + """ + Sample requests from the MMMU dataset using HuggingFace datasets. + + Args: + num_requests: Number of requests to sample. + tokenizer: Tokenizer to use for token counting. + fixed_output_len: If provided, use this fixed output length for all requests. + apply_chat_template: Whether to apply the chat template to the prompt. + random_sample: Whether to randomly sample or take the first N. + + Returns: + List of tuples (prompt, prompt_token_len, output_token_len). + """ + try: + import io + + import pybase64 + from datasets import load_dataset + except ImportError: + raise ImportError("Please install datasets: pip install datasets") + + print("Loading MMMU dataset from HuggingFace...") + + try: + print("Attempting to load MMMU Math dataset...") + mmmu_dataset = load_dataset("MMMU/MMMU", "Math", split="test") + print( + f"Successfully loaded MMMU Math dataset from HuggingFace with {len(mmmu_dataset)} examples" + ) + except Exception as e: + print(f"Failed to load MMMU Math dataset: {e}") + raise ValueError(f"Failed to load MMMU dataset: {e}") + + # Sample from the dataset + if len(mmmu_dataset) > num_requests: + if random_sample: + # Random sample + indices = random.sample(range(len(mmmu_dataset)), num_requests) + sample_dataset = mmmu_dataset.select(indices) + else: + # Take first N + sample_dataset = mmmu_dataset.select( + range(min(num_requests, len(mmmu_dataset))) + ) + else: + print(f"Dataset has less than {num_requests} examples, using all examples") + sample_dataset = mmmu_dataset + + print(f"Selected {len(sample_dataset)} examples for benchmarking") + + # Create prompts + filtered_dataset = [] + + for i, example in enumerate(sample_dataset): + try: + # Extract image_1 + image = example.get("image_1") + + if image is not None: + if hasattr(image, "save"): + # Convert RGBA images to RGB before encoding + if image.mode == "RGBA": + image = image.convert("RGB") + + # Encode image to base64 (save as PNG to support palette/alpha modes) + buffered = io.BytesIO() + image.save(buffered, format="PNG") + img_str = pybase64.b64encode(buffered.getvalue()).decode("utf-8") + image_data = f"data:image/png;base64,{img_str}" + else: + continue + + # Extract the question + question = example.get("question") + + # Construct the prompt + prompt = f"Question: {question}\n\nAnswer: " + if apply_chat_template: + try: + is_phi4_multimodal = ( + "phi-4-multimodal" in tokenizer.name_or_path.lower() + ) + if is_phi4_multimodal: + # <|endoftext10|> is the image token used in the phi-4-multimodal model. + content = prompt.replace("image 1", "<|endoftext10|>") + else: + content = [ + { + "type": "image_url", + "image_url": {"url": image_data}, + }, + {"type": "text", "text": prompt}, + ] + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": content, + } + ], + add_generation_prompt=True, + tokenize=False, + ) + except Exception as e: + # Note (Xinyuan): This is a workaround for an issue where some tokenizers do not support content as a list. (e.g. InternVL) + print( + f"Error applying chat template: {e}, fallback to tag" + ) + prompt = f"{prompt}" + + # Calculate token lengths for text only (without image data) + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + output_len = fixed_output_len if fixed_output_len is not None else 256 + + filtered_dataset.append( + DatasetRow( + prompt=prompt, + prompt_len=prompt_len, + output_len=output_len, + image_data=[image_data], + ) + ) + + except Exception as e: + print(f"Error processing example {i}: {e}") + + print(f"\nCreated {len(filtered_dataset)} MMMU prompts") + return filtered_dataset + + +def sample_sharegpt_requests( + dataset_path: str, + num_requests: int, + tokenizer: PreTrainedTokenizerBase, + fixed_output_len: Optional[int] = None, + context_len: Optional[int] = None, + prompt_suffix: Optional[str] = "", + apply_chat_template=False, +) -> List[DatasetRow]: + if fixed_output_len is not None and fixed_output_len < 4: + raise ValueError("output_len too small") + + # Download sharegpt if necessary + if not is_file_valid_json(dataset_path) and dataset_path == "": + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + filtered_dataset: List[DatasetRow] = [] + for i in range(len(dataset)): + if len(filtered_dataset) == num_requests: + break + + # Tokenize the prompts and completions. + prompt = dataset[i][0] + if prompt_suffix: + prompt = ( + remove_suffix(prompt, ASSISTANT_SUFFIX) + + prompt_suffix + + ASSISTANT_SUFFIX + ) + + if apply_chat_template: + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=False, + ) + prompt = prompt.replace(tokenizer.bos_token, "") + + prompt_token_ids = tokenizer.encode(prompt) + completion = dataset[i][1] + completion_token_ids = tokenizer.encode(completion) + prompt_len = len(prompt_token_ids) + output_len = ( + len(completion_token_ids) if fixed_output_len is None else fixed_output_len + ) + + if prompt_len < 2 or output_len < 2: + # Prune too short sequences. + continue + + if context_len and prompt_len + output_len > context_len: + # Prune too long sequences. + continue + + filtered_dataset.append( + DatasetRow(prompt=prompt, prompt_len=prompt_len, output_len=output_len) + ) + + print(f"#Input tokens: {np.sum([x.prompt_len for x in filtered_dataset])}") + print(f"#Output tokens: {np.sum([x.output_len for x in filtered_dataset])}") + return filtered_dataset + + +def sample_random_requests( + input_len: int, + output_len: int, + num_prompts: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + dataset_path: str, + random_sample: bool = True, + return_text: bool = True, +) -> List[DatasetRow]: + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), + input_len + 1, + size=num_prompts, + ) + output_lens = np.random.randint( + int(output_len * range_ratio), + output_len + 1, + size=num_prompts, + ) + + if random_sample: + # Sample token ids from ShareGPT and repeat/truncate them to satisfy the input_lens + + # Download sharegpt if necessary + if not is_file_valid_json(dataset_path): + dataset_path = download_and_cache_file(SHAREGPT_URL) + + # Load the dataset. + with open(dataset_path) as f: + dataset = json.load(f) + # Filter out the conversations with less than 2 turns. + dataset = [ + data + for data in dataset + if len(data.get("conversations", data.get("conversation", []))) >= 2 + ] + # Only keep the first two turns of each conversation. + dataset = [ + ( + data.get("conversations", data.get("conversation", []))[0]["value"], + data.get("conversations", data.get("conversation", []))[1]["value"], + ) + for data in dataset + ] + # Shuffle the dataset. + random.shuffle(dataset) + + # Filter out sequences that are too long or too short + input_requests: List[DatasetRow] = [] + for data in dataset: + i = len(input_requests) + if i == num_prompts: + break + + # Tokenize the prompts and completions. + prompt = data[0] + prompt_token_ids = tokenizer.encode(prompt) + prompt_len = len(prompt_token_ids) + + # Skip empty prompt + if prompt_len == 0: + continue + + if prompt_len > input_lens[i]: + input_ids = prompt_token_ids[: input_lens[i]] + else: + ratio = (input_lens[i] + prompt_len - 1) // prompt_len + input_ids = (prompt_token_ids * ratio)[: input_lens[i]] + input_content = input_ids + if return_text: + input_content = tokenizer.decode(input_content) + input_requests.append( + DatasetRow( + prompt=input_content, + prompt_len=int(input_lens[i]), + output_len=int(output_lens[i]), + ) + ) + else: + # Sample token ids from random integers. This can cause some NaN issues. + offsets = np.random.randint(0, tokenizer.vocab_size, size=num_prompts) + input_requests = [] + for i in range(num_prompts): + input_content = [ + (offsets[i] + i + j) % tokenizer.vocab_size + for j in range(input_lens[i]) + ] + if return_text: + input_content = tokenizer.decode(input_content) + input_requests.append( + DatasetRow( + prompt=input_content, + prompt_len=int(input_lens[i]), + output_len=int(output_lens[i]), + ) + ) + + print(f"#Input tokens: {np.sum(input_lens)}") + print(f"#Output tokens: {np.sum(output_lens)}") + return input_requests + + +def parse_random_image_resolution(image_resolution: str) -> Tuple[int, int]: + """Parse image resolution into (width, height). + + Supports presets '1080p', '720p', '360p' and custom 'heightxwidth' format + (e.g., '1080x1920' means height=1080, width=1920). + """ + resolution_to_size = { + "4k": (3840, 2160), + "1080p": (1920, 1080), + "720p": (1280, 720), + "360p": (640, 360), + } + if image_resolution in resolution_to_size: + return resolution_to_size[image_resolution] + + res = image_resolution.strip().lower() + if "x" in res: + parts = res.split("x") + if len(parts) == 2 and parts[0].isdigit() and parts[1].isdigit(): + height = int(parts[0]) + width = int(parts[1]) + if height > 0 and width > 0: + return (width, height) + + raise ValueError( + f"Unsupported random-image resolution: {image_resolution}. " + "Choose from 4k, 1080p, 720p, 360p, or provide custom 'heightxwidth' (e.g., 1080x1920)." + ) + + +def sample_random_image_requests( + num_requests: int, + num_images: int, + input_len: int, + output_len: int, + range_ratio: float, + tokenizer: PreTrainedTokenizerBase, + apply_chat_template: bool = True, + image_resolution: str = "1080p", +) -> List[DatasetRow]: + """Generate requests with random images. + + - Each request includes ``num_images`` random images. + - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p (1280x720), 360p (640x360), + or custom 'heightxwidth' (e.g., 1080x1920). + - Text lengths follow the 'random' dataset sampling rule. ``prompt_len`` + only counts text tokens and excludes image data. + """ + try: + import pybase64 + from PIL import Image + except ImportError as e: + raise ImportError( + "Please install Pillow to generate random images: pip install pillow" + ) from e + + # Parse resolution (supports presets and 'heightxwidth') + width, height = parse_random_image_resolution(image_resolution) + + # Check for potentially problematic combinations and warn user + if width * height >= 1920 * 1080 and num_images * num_requests >= 100: + warnings.warn( + f"High resolution ({width}x{height}) with {num_images * num_requests} total images " + f"may take a long time. Consider reducing resolution or image count.", + UserWarning, + stacklevel=2, + ) + + # Sample text lengths + input_lens = np.random.randint( + max(int(input_len * range_ratio), 1), input_len + 1, size=num_requests + ) + output_lens = np.random.randint( + int(output_len * range_ratio), output_len + 1, size=num_requests + ) + + def _gen_random_image_data_uri(width: int = width, height: int = height) -> str: + arr = (np.random.rand(height, width, 3) * 255).astype(np.uint8) + img = Image.fromarray(arr, mode="RGB") + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + encoded = pybase64.b64encode(buf.getvalue()).decode("utf-8") + return f"data:image/jpeg;base64,{encoded}" + + dataset: List[DatasetRow] = [] + for i in range(num_requests): + # Generate text prompt + text_prompt = gen_prompt(tokenizer, int(input_lens[i])) + + # Generate image list + images = [_gen_random_image_data_uri() for _ in range(num_images)] + + prompt_str = text_prompt + if apply_chat_template: + try: + content_items = [ + {"type": "image_url", "image_url": {"url": img_url}} + for img_url in images + ] + content_items.append({"type": "text", "text": text_prompt}) + prompt_str = tokenizer.apply_chat_template( + [{"role": "user", "content": content_items}], + add_generation_prompt=True, + tokenize=False, + ) + except Exception: + # Some tokenizers do not support list content; fall back to a placeholder in the text + prompt_str = f"{text_prompt}" + + prompt_token_ids = tokenizer.encode(prompt_str) + prompt_token_len = len(prompt_token_ids) + + dataset.append( + DatasetRow( + prompt=prompt_str, + prompt_len=prompt_token_len, + output_len=int(output_lens[i]), + image_data=images, + ) + ) + + print(f"#Input tokens: {np.sum([x.prompt_len for x in dataset])}") + print(f"#Output tokens: {np.sum([x.output_len for x in dataset])}") + return dataset + + +def gen_prompt(tokenizer, token_num): + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return tokenizer.decode(selected_tokens) + + +def get_gen_prefix_cache_path(args, tokenizer): + """Create cache directory under ~/.cache/sglang/benchmark""" + cache_dir = Path.home() / ".cache" / "sglang" / "benchmark" + + # Create a unique cache filename based on the generation parameters + cache_key = ( + f"gen_shared_prefix_{args.gsp_num_groups}_{args.gsp_prompts_per_group}_" + f"{args.gsp_system_prompt_len}_{args.gsp_question_len}_{args.gsp_output_len}_" + f"{tokenizer.__class__.__name__}.pkl" + ) + return cache_dir / cache_key + + +def sample_generated_shared_prefix_requests( + num_groups: int, + prompts_per_group: int, + system_prompt_len: int, + question_len: int, + output_len: int, + tokenizer: PreTrainedTokenizerBase, + args: argparse.Namespace, +) -> List[DatasetRow]: + """Generate benchmark requests with shared system prompts using random tokens and caching.""" + cache_path = get_gen_prefix_cache_path(args, tokenizer) + + # Try to load from cache first + if cache_path.exists(): + print(f"\nLoading cached generated input data from {cache_path}") + with open(cache_path, "rb") as f: + return pickle.load(f) + + print("\nGenerating new input data...") + + # Generate system prompts for each group + system_prompts = [] + for _ in range(num_groups): + system_prompt = gen_prompt(tokenizer, system_prompt_len) + system_prompts.append(system_prompt) + + # Generate questions + questions = [] + for _ in range(num_groups * prompts_per_group): + question = gen_prompt(tokenizer, question_len) + questions.append(question) + + # Combine system prompts with questions + input_requests = [] + total_input_tokens = 0 + total_output_tokens = 0 + + for group_idx in tqdm(range(num_groups), desc="Generating system prompt"): + system_prompt = system_prompts[group_idx] + for prompt_idx in tqdm( + range(prompts_per_group), desc="Generating questions", leave=False + ): + question = questions[group_idx * prompts_per_group + prompt_idx] + full_prompt = f"{system_prompt}\n\n{question}" + prompt_len = len(tokenizer.encode(full_prompt)) + + input_requests.append( + DatasetRow( + prompt=full_prompt, prompt_len=prompt_len, output_len=output_len + ) + ) + total_input_tokens += prompt_len + total_output_tokens += output_len + + # Shuffle questions + random.shuffle(input_requests) + + # Print statistics + print(f"\nGenerated shared prefix dataset statistics:") + print(f"Number of groups: {num_groups}") + print(f"Prompts per group: {prompts_per_group}") + print(f"Total prompts: {len(input_requests)}") + print(f"Total input tokens: {total_input_tokens}") + print(f"Total output tokens: {total_output_tokens}") + print( + f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + ) + print( + f"Average question length: {sum(len(tokenizer.encode(q)) for q in questions) / len(questions):.1f} tokens\n" + ) + + # Save to cache + cache_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Caching generated input data to {cache_path}") + with open(cache_path, "wb") as f: + pickle.dump(input_requests, f) + + return input_requests + + +async def get_request( + input_requests: List[DatasetRow], + request_rate: float, + use_trace_timestamps: bool = False, + slowdown_factor: float = 1.0, +) -> AsyncGenerator[DatasetRow, None]: + if use_trace_timestamps: + print( + f"Using trace timestamps for request generation with slowdown factor {slowdown_factor}." + ) + # Sort requests by timestamp for correct replay + input_requests.sort(key=lambda r: r.timestamp) + + start_time = time.perf_counter() + trace_start_time_ms = input_requests[0].timestamp if input_requests else 0 + + for request in input_requests: + trace_time_s = (request.timestamp - trace_start_time_ms) / 1000.0 + target_arrival_time = start_time + (trace_time_s * slowdown_factor) + + sleep_duration = target_arrival_time - time.perf_counter() + if sleep_duration > 0: + await asyncio.sleep(sleep_duration) + + yield request + else: + input_requests_iter = iter(input_requests) + for request in input_requests_iter: + yield request + + if request_rate == float("inf"): + # If the request rate is infinity, then we don't need to wait. + continue + + # Sample the request interval from the exponential distribution. + interval = np.random.exponential(1.0 / request_rate) + # The next request will be sent after the interval. + await asyncio.sleep(interval) + + +def calculate_metrics( + input_requests: List[DatasetRow], + outputs: List[RequestFuncOutput], + dur_s: float, + tokenizer: PreTrainedTokenizerBase, + backend: str, +) -> Tuple[BenchmarkMetrics, List[int]]: + output_lens: List[int] = [] + retokenized_output_lens: List[int] = [] + total_input = 0 + completed = 0 + itls: List[float] = [] + tpots: List[float] = [] + ttfts: List[float] = [] + e2e_latencies: List[float] = [] + for i in range(len(outputs)): + if outputs[i].success: + output_len = outputs[i].output_len + output_lens.append(output_len) + retokenized_output_len = len( + tokenizer.encode(outputs[i].generated_text, add_special_tokens=False) + ) + retokenized_output_lens.append(retokenized_output_len) + total_input += outputs[i].prompt_len + if output_len > 1: + tpots.append((outputs[i].latency - outputs[i].ttft) / (output_len - 1)) + itls += outputs[i].itl + ttfts.append(outputs[i].ttft) + + e2e_latencies.append(outputs[i].latency) + + completed += 1 + else: + output_lens.append(0) + retokenized_output_lens.append(0) + + if completed == 0: + warnings.warn( + "All requests failed. This is likely due to a misconfiguration " + "on the benchmark arguments.", + stacklevel=2, + ) + metrics = BenchmarkMetrics( + completed=completed, + total_input=total_input, + total_output=sum(output_lens), + total_output_retokenized=sum(retokenized_output_lens), + request_throughput=completed / dur_s, + input_throughput=total_input / dur_s, + output_throughput=sum(output_lens) / dur_s, + output_throughput_retokenized=sum(retokenized_output_lens) / dur_s, + total_throughput=(total_input + sum(output_lens)) / dur_s, + total_throughput_retokenized=(total_input + sum(retokenized_output_lens)) + / dur_s, + mean_ttft_ms=np.mean(ttfts or 0) + * 1000, # ttfts is empty if streaming is not supported by backend + median_ttft_ms=np.median(ttfts or 0) * 1000, + std_ttft_ms=np.std(ttfts or 0) * 1000, + p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000, + mean_tpot_ms=np.mean(tpots or 0) * 1000, + median_tpot_ms=np.median(tpots or 0) * 1000, + std_tpot_ms=np.std(tpots or 0) * 1000, + p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000, + mean_itl_ms=np.mean(itls or 0) * 1000, + median_itl_ms=np.median(itls or 0) * 1000, + std_itl_ms=np.std(itls or 0) * 1000, + p95_itl_ms=np.percentile(itls or 0, 95) * 1000, + p99_itl_ms=np.percentile(itls or 0, 99) * 1000, + max_itl_ms=np.max(itls or 0) * 1000, + mean_e2e_latency_ms=np.mean(e2e_latencies) * 1000, + median_e2e_latency_ms=np.median(e2e_latencies) * 1000, + std_e2e_latency_ms=np.std(e2e_latencies) * 1000, + p99_e2e_latency_ms=np.percentile(e2e_latencies, 99) * 1000, + concurrency=np.sum(e2e_latencies) / dur_s, + ) + + return metrics, output_lens + + +async def benchmark( + backend: str, + api_url: str, + base_url: str, + model_id: str, + tokenizer: PreTrainedTokenizerBase, + input_requests: List[DatasetRow], + request_rate: float, + max_concurrency: Optional[int], + disable_tqdm: bool, + lora_names: List[str], + extra_request_body: Dict[str, Any], + profile: bool, + pd_separated: bool = False, + flush_cache: bool = False, + warmup_requests: int = 1, + use_trace_timestamps: bool = False, + mooncake_slowdown_factor=1.0, + mooncake_num_rounds=1, +): + if backend in ASYNC_REQUEST_FUNCS: + request_func = ASYNC_REQUEST_FUNCS[backend] + else: + raise ValueError(f"Unknown backend: {backend}") + + # Limit concurrency + # From https://github.com/vllm-project/vllm/pull/9390 + semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None + + async def limited_request_func(request_func_input, pbar): + if semaphore is None: + return await request_func(request_func_input=request_func_input, pbar=pbar) + async with semaphore: + return await request_func(request_func_input=request_func_input, pbar=pbar) + + # Warmup + print(f"Starting warmup with {warmup_requests} sequences...") + + # Handle the data structure difference for the warmup request + if args.dataset_name == "mooncake": + # For mooncake, input_requests is a list of dicts. + # We need to build a temporary DatasetRow for the warmup phase. + warmup_record = input_requests[0] + + # Build prompt from hash_ids, just like in the async generator + hash_ids = warmup_record.get("hash_ids", []) + prompt_text = "" + for hash_id in hash_ids: + prompt_text += f"{hash_id}" + " ".join(["hi"] * 512) + prompt_text += "Can you tell me a detailed story in 1000 words?" + + output_len = warmup_record.get("output_length", 32) + prompt_len = len(tokenizer.encode(prompt_text)) + + # Create a temporary DatasetRow object for warmup + test_request = DatasetRow( + prompt=prompt_text, + prompt_len=prompt_len, + output_len=output_len, + image_data=None, # Mooncake doesn't have image data + ) + else: + # For all other datasets, input_requests is a list of DatasetRow objects + test_request = input_requests[0] + + if lora_names is not None and len(lora_names) != 0: + lora_name = lora_names[0] + else: + lora_name = None + + # Create the test input once + test_input = RequestFuncInput( + model=model_id, + prompt=test_request.prompt, + api_url=api_url, + prompt_len=test_request.prompt_len, + output_len=min(test_request.output_len, 32), + lora_name=lora_name, + image_data=test_request.image_data, + extra_request_body=extra_request_body, + ) + + # Run warmup requests + warmup_tasks = [] + for _ in range(warmup_requests): + warmup_tasks.append( + asyncio.create_task(request_func(request_func_input=test_input)) + ) + + warmup_outputs = await asyncio.gather(*warmup_tasks) + + # Check if at least one warmup request succeeded + if warmup_requests > 0 and not any(output.success for output in warmup_outputs): + raise ValueError( + "Warmup failed - Please make sure benchmark arguments " + f"are correctly specified. Error: {warmup_outputs[0].error}" + ) + else: + print( + f"Warmup completed with {args.warmup_requests} sequences. Starting main benchmark run..." + ) + + # Flush cache + if ("sglang" in backend and _get_bool_env_var("SGLANG_IS_IN_CI")) or flush_cache: + requests.post(base_url + "/flush_cache", headers=get_auth_headers()) + + time.sleep(1.0) + + # Start profiler + if profile: + print("Starting profiler...") + profile_output = await async_request_profile( + api_url=base_url + "/start_profile" + ) + if profile_output.success: + print("Profiler started") + + # Run all requests + benchmark_start_time = time.perf_counter() + tasks: List[asyncio.Task] = [] + pbar_total = len(input_requests) + if ( + backend == "sglang" and args.dataset_name == "mooncake" + ): # Assuming mooncake is mainly for sglang or similar backends + print("Using time-based Mooncake request scheduler, ignoring --request-rate.") + request_generator = get_mooncake_request_over_time( + input_requests, tokenizer, mooncake_slowdown_factor, mooncake_num_rounds + ) + print( + f"Starting Mooncake trace replay. Sessions: {len(input_requests)}, Rounds per session: {mooncake_num_rounds}. Slowdown factor: {mooncake_slowdown_factor}" + ) + pbar_total *= args.mooncake_num_rounds + else: + request_generator = get_request(input_requests, request_rate) + + pbar = None if disable_tqdm else tqdm(total=pbar_total) + async for request in request_generator: + if lora_names is not None and len(lora_names) != 0: + idx = random.randint(0, len(lora_names) - 1) + lora_name = lora_names[idx] + else: + lora_name = None + + request_func_input = RequestFuncInput( + model=model_id, + prompt=request.prompt, + api_url=api_url, + prompt_len=request.prompt_len, + output_len=request.output_len, + lora_name=lora_name, + image_data=request.image_data, + extra_request_body=extra_request_body, + timestamp=request.timestamp, + ) + + tasks.append( + asyncio.create_task( + limited_request_func(request_func_input=request_func_input, pbar=pbar) + ) + ) + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks) + + # Stop profiler + if profile: + print("Stopping profiler...") + profile_output = await async_request_profile(api_url=base_url + "/stop_profile") + if profile_output.success: + print("Profiler stopped") + + if pbar is not None: + pbar.close() + + if "sglang" in backend: + server_info = requests.get(base_url + "/get_server_info") + if server_info.status_code == 200: + server_info_json = server_info.json() + if "decode" in server_info_json: + server_info_json = server_info_json["decode"][0] + accept_length = server_info_json["internal_states"][0].get( + "avg_spec_accept_length", None + ) + else: + accept_length = None + else: + accept_length = None + + # Compute metrics and print results + benchmark_duration = time.perf_counter() - benchmark_start_time + metrics, output_lens = calculate_metrics( + input_requests=input_requests, + outputs=outputs, + dur_s=benchmark_duration, + tokenizer=tokenizer, + backend=backend, + ) + + print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) + print("{:<40} {:<10}".format("Backend:", backend)) + print( + "{:<40} {:<10}".format( + "Traffic request rate:", "trace" if use_trace_timestamps else request_rate + ) + ) + print( + "{:<40} {:<10}".format( + "Max request concurrency:", + max_concurrency if max_concurrency else "not set", + ) + ) + print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) + print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) + print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) + print( + "{:<40} {:<10}".format( + "Total generated tokens (retokenized):", metrics.total_output_retokenized + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Request throughput (req/s):", metrics.request_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Input token throughput (tok/s):", metrics.input_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Output token throughput (tok/s):", metrics.output_throughput + ) + ) + print( + "{:<40} {:<10.2f}".format( + "Total token throughput (tok/s):", metrics.total_throughput + ) + ) + print("{:<40} {:<10.2f}".format("Concurrency:", metrics.concurrency)) + if accept_length: + print("{:<40} {:<10.2f}".format("Accept length:", accept_length)) + print("{s:{c}^{n}}".format(s="End-to-End Latency", n=50, c="-")) + print( + "{:<40} {:<10.2f}".format("Mean E2E Latency (ms):", metrics.mean_e2e_latency_ms) + ) + print( + "{:<40} {:<10.2f}".format( + "Median E2E Latency (ms):", metrics.median_e2e_latency_ms + ) + ) + print("{s:{c}^{n}}".format(s="Time to First Token", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms)) + print("{:<40} {:<10.2f}".format("Median TTFT (ms):", metrics.median_ttft_ms)) + print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms)) + print("{s:{c}^{n}}".format(s="Inter-Token Latency", n=50, c="-")) + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms)) + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms)) + print("{:<40} {:<10.2f}".format("P95 ITL (ms):", metrics.p95_itl_ms)) + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms)) + print("{:<40} {:<10.2f}".format("Max ITL (ms):", metrics.max_itl_ms)) + print("=" * 50) + + if ( + metrics.median_ttft_ms is not None + and metrics.mean_itl_ms is not None + and metrics.output_throughput is not None + ): + result = { + # Arguments + "backend": args.backend, + "dataset_name": args.dataset_name, + "request_rate": "trace" if use_trace_timestamps else request_rate, + "max_concurrency": max_concurrency, + "sharegpt_output_len": args.sharegpt_output_len, + "random_input_len": args.random_input_len, + "random_output_len": args.random_output_len, + "random_range_ratio": args.random_range_ratio, + # Results + "duration": benchmark_duration, + "completed": metrics.completed, + "total_input_tokens": metrics.total_input, + "total_output_tokens": metrics.total_output, + "total_output_tokens_retokenized": metrics.total_output_retokenized, + "request_throughput": metrics.request_throughput, + "input_throughput": metrics.input_throughput, + "output_throughput": metrics.output_throughput, + "mean_e2e_latency_ms": metrics.mean_e2e_latency_ms, + "median_e2e_latency_ms": metrics.median_e2e_latency_ms, + "std_e2e_latency_ms": metrics.std_e2e_latency_ms, + "p99_e2e_latency_ms": metrics.p99_e2e_latency_ms, + "mean_ttft_ms": metrics.mean_ttft_ms, + "median_ttft_ms": metrics.median_ttft_ms, + "std_ttft_ms": metrics.std_ttft_ms, + "p99_ttft_ms": metrics.p99_ttft_ms, + "mean_tpot_ms": metrics.mean_tpot_ms, + "median_tpot_ms": metrics.median_tpot_ms, + "std_tpot_ms": metrics.std_tpot_ms, + "p99_tpot_ms": metrics.p99_tpot_ms, + "mean_itl_ms": metrics.mean_itl_ms, + "median_itl_ms": metrics.median_itl_ms, + "std_itl_ms": metrics.std_itl_ms, + "p95_itl_ms": metrics.p95_itl_ms, + "p99_itl_ms": metrics.p99_itl_ms, + "concurrency": metrics.concurrency, + "accept_length": accept_length, + } + else: + print(f"Error running benchmark for request rate: {request_rate}") + print("-" * 30) + + # Determine output file name + if args.output_file: + output_file_name = args.output_file + else: + now = datetime.now().strftime("%m%d") + if args.dataset_name == "random-image": + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_" + f"{args.random_output_len}_{args.random_image_num_images}imgs_" + f"{args.random_image_resolution}.jsonl" + ) + elif args.dataset_name.startswith("random"): + output_file_name = f"{args.backend}_{now}_{args.num_prompts}_{args.random_input_len}_{args.random_output_len}.jsonl" + else: + output_file_name = ( + f"{args.backend}_{now}_{args.num_prompts}_{args.dataset_name}.jsonl" + ) + + result_details = { + "input_lens": [output.prompt_len for output in outputs], + "output_lens": output_lens, + "ttfts": [output.ttft for output in outputs], + "itls": [output.itl for output in outputs], + "generated_texts": [output.generated_text for output in outputs], + "errors": [output.error for output in outputs], + } + + # Append results to a JSONL file + with open(output_file_name, "a") as file: + if args.output_details: + result_for_dump = result | result_details + else: + result_for_dump = result + file.write(json.dumps(result_for_dump) + "\n") + + return result | result_details + + +def check_chat_template(model_path): + try: + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + return "chat_template" in tokenizer.init_kwargs + except Exception as e: + print(f"Fail to load tokenizer config with error={e}") + return False + + +def set_global_args(args_: argparse.Namespace): + """Set the global args.""" + global args + args = args_ + + +def run_benchmark(args_: argparse.Namespace): + global args + args = args_ + + # Set default value for max_concurrency if not present + if not hasattr(args, "max_concurrency"): + args.max_concurrency = None + + # Set default value for warmup_requests if not present + if not hasattr(args, "warmup_requests"): + args.warmup_requests = 1 + + if not hasattr(args, "output_details"): + args.output_details = False + + if not hasattr(args, "tokenize_prompt"): + args.tokenize_prompt = False + + if not hasattr(args, "use_trace_timestamps"): + args.use_trace_timestamps = False + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_slowdown_factor"): + args.mooncake_slowdown_factor = 1.0 + + if not hasattr(args, "mooncake_num_rounds"): + args.mooncake_num_rounds = 1 + + print(f"benchmark_args={args}") + + # Set global environments + set_ulimit() + random.seed(args.seed) + np.random.seed(args.seed) + + extra_request_body = {} + if args.extra_request_body: + extra_request_body = json.loads(args.extra_request_body) + + if args.tokenize_prompt: + assert ( + args.backend == "sglang" + ), "`--tokenize-prompt` only compatible with `--backend sglang` currently" + + # Set url + if args.port is None: + args.port = { + "sglang": 30000, + "sglang-native": 30000, + "sglang-oai": 30000, + "lmdeploy": 23333, + "vllm": 8000, + "trt": 8000, + "gserver": 9988, + "truss": 8080, + }.get(args.backend, 30000) + + model_url = ( + f"{args.base_url}/v1/models" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models" + ) + + if args.backend in ["sglang", "sglang-native"]: + api_url = ( + f"{args.base_url}/generate" + if args.base_url + else f"http://{args.host}:{args.port}/generate" + ) + elif args.backend in ["sglang-oai", "vllm", "lmdeploy"]: + api_url = ( + f"{args.base_url}/v1/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/completions" + ) + elif args.backend in ["sglang-oai-chat", "vllm-chat", "lmdeploy-chat"]: + api_url = ( + f"{args.base_url}/v1/chat/completions" + if args.base_url + else f"http://{args.host}:{args.port}/v1/chat/completions" + ) + elif args.backend == "trt": + api_url = ( + f"{args.base_url}/v2/models/ensemble/generate_stream" + if args.base_url + else f"http://{args.host}:{args.port}/v2/models/ensemble/generate_stream" + ) + if args.model is None: + print("Please provide a model using `--model` when using `trt` backend.") + sys.exit(1) + elif args.backend == "gserver": + api_url = args.base_url if args.base_url else f"{args.host}:{args.port}" + args.model = args.model or "default" + elif args.backend == "truss": + api_url = ( + f"{args.base_url}/v1/models/model:predict" + if args.base_url + else f"http://{args.host}:{args.port}/v1/models/model:predict" + ) + base_url = ( + f"http://{args.host}:{args.port}" if args.base_url is None else args.base_url + ) + + # Get model name + if args.model is None: + if args.backend == "truss": + print( + "Please provide a model with `--model` when using truss backend. e.g. --model meta-llama/Llama-3.1-8B-Instruct" + ) + sys.exit(1) + try: + response = requests.get(model_url, headers=get_auth_headers()) + model_list = response.json().get("data", []) + args.model = model_list[0]["id"] if model_list else None + except Exception as e: + print(f"Failed to fetch model from {model_url}. Error: {e}") + print( + "Please specify the correct host and port using `--host` and `--port`." + ) + sys.exit(1) + + if args.model is None: + print("No model specified or found. Please provide a model using `--model`.") + sys.exit(1) + + if not check_chat_template(args.model): + print( + "\nWARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.\n" + "Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.\n" + ) + + print(f"{args}\n") + + # Read dataset + backend = args.backend + model_id = args.model + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model + tokenizer = get_tokenizer(tokenizer_id) + input_requests = get_dataset(args, tokenizer) + + # compatible with SimpleNamespace + if not hasattr(args, "flush_cache"): + args.flush_cache = False + + return asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id=model_id, + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=args.request_rate, + max_concurrency=args.max_concurrency, + disable_tqdm=args.disable_tqdm, + lora_names=args.lora_name, + extra_request_body=extra_request_body, + profile=args.profile, + pd_separated=args.pd_separated, + flush_cache=args.flush_cache, + warmup_requests=args.warmup_requests, + use_trace_timestamps=args.use_trace_timestamps, + mooncake_slowdown_factor=args.mooncake_slowdown_factor, + mooncake_num_rounds=args.mooncake_num_rounds, + ) + ) + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") + + +class LoRAPathAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, []) + for lora_name in values: + getattr(namespace, self.dest).append(lora_name) + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--backend", + type=str, + choices=list(ASYNC_REQUEST_FUNCS.keys()), + default="sglang", + help="Must specify a backend, depending on the LLM Inference Engine.", + ) + parser.add_argument( + "--base-url", + type=str, + default=None, + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--host", type=str, default="0.0.0.0", help="Default host is 0.0.0.0." + ) + parser.add_argument( + "--port", + type=int, + help="If not set, the default port is configured according to its default value for different LLM Inference Engines.", + ) + parser.add_argument( + "--dataset-name", + type=str, + default="sharegpt", + choices=[ + "sharegpt", + "random", + "random-ids", + "generated-shared-prefix", + "mmmu", + "random-image", + "mooncake", + ], + help="Name of the dataset to benchmark on.", + ) + parser.add_argument( + "--dataset-path", type=str, default="", help="Path to the dataset." + ) + parser.add_argument( + "--model", + type=str, + help="Name or path of the model. If not set, the default model will request /v1/models for conf.", + ) + parser.add_argument( + "--tokenizer", + type=str, + help="Name or path of the tokenizer. If not set, using the model conf.", + ) + parser.add_argument( + "--num-prompts", + type=int, + default=1000, + help="Number of prompts to process. Default is 1000.", + ) + parser.add_argument( + "--sharegpt-output-len", + type=int, + default=None, + help="Output length for each request. Overrides the output length from the ShareGPT dataset.", + ) + parser.add_argument( + "--sharegpt-context-len", + type=int, + default=None, + help="The context length of the model for the ShareGPT dataset. Requests longer than the context length will be dropped.", + ) + parser.add_argument( + "--random-input-len", + type=int, + default=1024, + help="Number of input tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-output-len", + default=1024, + type=int, + help="Number of output tokens per request, used only for random dataset.", + ) + parser.add_argument( + "--random-range-ratio", + type=float, + default=0.0, + help="Range of sampled ratio of input/output length, " + "used only for random dataset.", + ) + # random-image dataset args + parser.add_argument( + "--random-image-num-images", + type=int, + default=1, + help="Number of images per request (only available with the random-image dataset)", + ) + parser.add_argument( + "--random-image-resolution", + type=str, + default="1080p", + help=( + "Resolution of random images for random-image dataset. " + "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)." + ), + ) + parser.add_argument( + "--request-rate", + type=float, + default=float("inf"), + help="Number of requests per second. If this is inf, then all the requests are sent at time 0. " + "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.", + ) + parser.add_argument( + "--use-trace-timestamps", + action="store_true", + help="Use timestamps from the trace file for request scheduling. Only valid for 'mooncake' dataset.", + ) + parser.add_argument( + "--max-concurrency", + type=int, + default=None, + help="Maximum number of concurrent requests. This can be used " + "to help simulate an environment where a higher level component " + "is enforcing a maximum number of concurrent requests. While the " + "--request-rate argument controls the rate at which requests are " + "initiated, this argument will control how many are actually allowed " + "to execute at a time. This means that when used in combination, the " + "actual request rate may be lower than specified with --request-rate, " + "if the server is not processing requests fast enough to keep up.", + ) + parser.add_argument("--output-file", type=str, help="Output JSONL file name.") + parser.add_argument( + "--output-details", action="store_true", help="Output details of benchmarking." + ) + parser.add_argument( + "--disable-tqdm", + action="store_true", + help="Specify to disable tqdm progress bar.", + ) + parser.add_argument( + "--disable-stream", + action="store_true", + help="Disable streaming mode.", + ) + parser.add_argument( + "--return-logprob", + action="store_true", + help="Return logprob.", + ) + parser.add_argument("--seed", type=int, default=1, help="The random seed.") + parser.add_argument( + "--disable-ignore-eos", + action="store_true", + help="Disable ignoring EOS.", + ) + parser.add_argument( + "--extra-request-body", + metavar='{"key1": "value1", "key2": "value2"}', + type=str, + help="Append given JSON object to the request payload. You can use this to specify" + "additional generate params like sampling params.", + ) + parser.add_argument( + "--apply-chat-template", + action="store_true", + help="Apply chat template", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Use Torch Profiler. The endpoint must be launched with " + "SGLANG_TORCH_PROFILER_DIR to enable profiler.", + ) + parser.add_argument( + "--lora-name", + type=str, + nargs="*", + default=None, + action=LoRAPathAction, + help="The names of LoRA adapters. You can provide a list of names in the format {name} {name} {name}...", + ) + parser.add_argument( + "--prompt-suffix", + type=str, + default="", + help="Suffix applied to the end of all user prompts, followed by assistant prompt suffix.", + ) + parser.add_argument( + "--pd-separated", + action="store_true", + help="Benchmark PD disaggregation server", + ) + parser.add_argument( + "--flush-cache", + action="store_true", + help="Flush the cache before running the benchmark", + ) + parser.add_argument( + "--warmup-requests", + type=int, + default=1, + help="Number of warmup requests to run before the benchmark", + ) + parser.add_argument( + "--tokenize-prompt", + action="store_true", + help="Use integer ids instead of string for inputs. Useful to control prompt lengths accurately", + ) + + group = parser.add_argument_group("generated-shared-prefix dataset arguments") + group.add_argument( + "--gsp-num-groups", + type=int, + default=64, + help="Number of system prompt groups for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-prompts-per-group", + type=int, + default=16, + help="Number of prompts per system prompt group for generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-system-prompt-len", + type=int, + default=2048, + help="Target length in tokens for system prompts in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-question-len", + type=int, + default=128, + help="Target length in tokens for questions in generated-shared-prefix dataset", + ) + group.add_argument( + "--gsp-output-len", + type=int, + default=256, + help="Target length in tokens for outputs in generated-shared-prefix dataset", + ) + mooncake_group = parser.add_argument_group("mooncake dataset arguments") + mooncake_group.add_argument( + "--mooncake-slowdown-factor", + type=float, + default=1.0, + help="Slowdown factor for replaying the mooncake trace. " + "A value of 2.0 means the replay is twice as slow. " + "NOTE: --request-rate is IGNORED in mooncake mode.", + ) + mooncake_group.add_argument( + "--mooncake-num-rounds", + type=int, + default=1, + help="Number of conversation rounds for each session in the mooncake dataset. " + "A value > 1 will enable true multi-turn session benchmarking.", + ) + mooncake_group.add_argument( + "--mooncake-workload", + type=str, + default="conversation", + choices=[ + "mooncake", + "conversation", + "synthetic", + "toolagent", + ], + help="Underlying workload for the mooncake dataset.", + ) + args = parser.parse_args() + run_benchmark(args) diff --git a/python/sglang/check_env.py b/python/sglang/check_env.py new file mode 100644 index 000000000..1870e3207 --- /dev/null +++ b/python/sglang/check_env.py @@ -0,0 +1,305 @@ +"""Check environment configurations and dependency versions.""" + +import importlib.metadata +import os +import resource +import subprocess +import sys +from collections import OrderedDict, defaultdict + +import torch + +from sglang.srt.utils import is_hip + + +def is_cuda_v2(): + return torch.version.cuda is not None + + +# List of packages to check versions +PACKAGE_LIST = [ + "sglang", + "sgl_kernel", + "flashinfer_python", + "triton", + "transformers", + "torchao", + "numpy", + "aiohttp", + "fastapi", + "hf_transfer", + "huggingface_hub", + "interegular", + "modelscope", + "orjson", + "outlines", + "packaging", + "psutil", + "pydantic", + "python-multipart", + "pyzmq", + "torchao", + "uvicorn", + "uvloop", + "vllm", + "xgrammar", + "openai", + "tiktoken", + "anthropic", + "litellm", + "decord", +] + + +def get_package_versions(packages): + """ + Get versions of specified packages. + """ + versions = {} + for package in packages: + package_name = package.split("==")[0].split(">=")[0].split("<=")[0] + try: + version = importlib.metadata.version(package_name) + versions[package_name] = version + except ModuleNotFoundError: + versions[package_name] = "Module Not Found" + return versions + + +def get_cuda_info(): + """ + Get CUDA-related information if available. + """ + if is_cuda_v2(): + cuda_info = {"CUDA available": torch.cuda.is_available()} + + if cuda_info["CUDA available"]: + cuda_info.update(_get_gpu_info()) + cuda_info.update(_get_cuda_version_info()) + + return cuda_info + elif is_hip(): + cuda_info = {"ROCM available": torch.cuda.is_available()} + + if cuda_info["ROCM available"]: + cuda_info.update(_get_gpu_info()) + cuda_info.update(_get_cuda_version_info()) + + return cuda_info + + +def _get_gpu_info(): + """ + Get information about available GPUs. + """ + devices = defaultdict(list) + capabilities = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + capability = torch.cuda.get_device_capability(k) + capabilities[f"{capability[0]}.{capability[1]}"].append(str(k)) + + gpu_info = {} + for name, device_ids in devices.items(): + gpu_info[f"GPU {','.join(device_ids)}"] = name + + if len(capabilities) == 1: + # All GPUs have the same compute capability + cap, gpu_ids = list(capabilities.items())[0] + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + else: + # GPUs have different compute capabilities + for cap, gpu_ids in capabilities.items(): + gpu_info[f"GPU {','.join(gpu_ids)} Compute Capability"] = cap + + return gpu_info + + +def _get_cuda_version_info(): + """ + Get CUDA version information. + """ + if is_cuda_v2(): + from torch.utils.cpp_extension import CUDA_HOME + + cuda_info = {"CUDA_HOME": CUDA_HOME} + + if CUDA_HOME and os.path.isdir(CUDA_HOME): + cuda_info.update(_get_nvcc_info()) + cuda_info.update(_get_cuda_driver_version()) + + return cuda_info + elif is_hip(): + from torch.utils.cpp_extension import ROCM_HOME as ROCM_HOME + + cuda_info = {"ROCM_HOME": ROCM_HOME} + + if ROCM_HOME and os.path.isdir(ROCM_HOME): + cuda_info.update(_get_nvcc_info()) + cuda_info.update(_get_cuda_driver_version()) + + return cuda_info + else: + cuda_info = {"CUDA_HOME": ""} + return cuda_info + + +def _get_nvcc_info(): + """ + Get NVCC version information. + """ + if is_cuda_v2(): + from torch.utils.cpp_extension import CUDA_HOME + + try: + nvcc = os.path.join(CUDA_HOME, "bin/nvcc") + nvcc_output = ( + subprocess.check_output(f'"{nvcc}" -V', shell=True) + .decode("utf-8") + .strip() + ) + return { + "NVCC": nvcc_output[ + nvcc_output.rfind("Cuda compilation tools") : nvcc_output.rfind( + "Build" + ) + ].strip() + } + except subprocess.SubprocessError: + return {"NVCC": "Not Available"} + elif is_hip(): + from torch.utils.cpp_extension import ROCM_HOME + + try: + hipcc = os.path.join(ROCM_HOME, "bin/hipcc") + hipcc_output = ( + subprocess.check_output(f'"{hipcc}" --version', shell=True) + .decode("utf-8") + .strip() + ) + return { + "HIPCC": hipcc_output[ + hipcc_output.rfind("HIP version") : hipcc_output.rfind("AMD clang") + ].strip() + } + except subprocess.SubprocessError: + return {"HIPCC": "Not Available"} + else: + return {"NVCC": "Not Available"} + + +def _get_cuda_driver_version(): + """ + Get CUDA driver version. + """ + versions = set() + if is_cuda_v2(): + try: + output = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=driver_version", + "--format=csv,noheader,nounits", + ] + ) + versions = set(output.decode().strip().split("\n")) + if len(versions) == 1: + return {"CUDA Driver Version": versions.pop()} + else: + return {"CUDA Driver Versions": ", ".join(sorted(versions))} + except subprocess.SubprocessError: + return {"CUDA Driver Version": "Not Available"} + elif is_hip(): + try: + output = subprocess.check_output( + [ + "rocm-smi", + "--showdriverversion", + "--csv", + ] + ) + versions = set(output.decode().strip().split("\n")) + versions.discard("name, value") + ver = versions.pop() + ver = ver.replace('"Driver version", ', "").replace('"', "") + + return {"ROCM Driver Version": ver} + except subprocess.SubprocessError: + return {"ROCM Driver Version": "Not Available"} + else: + return {"CUDA Driver Version": "Not Available"} + + +def get_gpu_topology(): + """ + Get GPU topology information. + """ + if is_cuda_v2(): + try: + result = subprocess.run( + ["nvidia-smi", "topo", "-m"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return "\n" + result.stdout if result.returncode == 0 else None + except subprocess.SubprocessError: + return None + elif is_hip(): + try: + result = subprocess.run( + ["rocm-smi", "--showtopotype"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return "\n" + result.stdout if result.returncode == 0 else None + except subprocess.SubprocessError: + return None + else: + return None + + +def get_hypervisor_vendor(): + try: + output = subprocess.check_output(["lscpu"], text=True) + for line in output.split("\n"): + if "Hypervisor vendor:" in line: + return line.split(":")[1].strip() + return None + except: + return None + + +def check_env(): + """ + Check and print environment information. + """ + env_info = OrderedDict() + env_info["Python"] = sys.version.replace("\n", "") + env_info.update(get_cuda_info()) + env_info["PyTorch"] = torch.__version__ + env_info.update(get_package_versions(PACKAGE_LIST)) + + gpu_topo = get_gpu_topology() + if gpu_topo: + if is_cuda_v2(): + env_info["NVIDIA Topology"] = gpu_topo + elif is_hip(): + env_info["AMD Topology"] = gpu_topo + + hypervisor_vendor = get_hypervisor_vendor() + if hypervisor_vendor: + env_info["Hypervisor vendor"] = hypervisor_vendor + + ulimit_soft, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + env_info["ulimit soft"] = ulimit_soft + + for k, v in env_info.items(): + print(f"{k}: {v}") + + +if __name__ == "__main__": + check_env() diff --git a/python/sglang/compile_deep_gemm.py b/python/sglang/compile_deep_gemm.py new file mode 100644 index 000000000..e59036f7b --- /dev/null +++ b/python/sglang/compile_deep_gemm.py @@ -0,0 +1,184 @@ +""" +Compile DeepGEMM Kernels for a model with specify server arguments + +This script launches a server for capturing DeepGEMM calls and then compiles the kernels. +It accepts server arguments (the same as launch_server.py). + +Usage: +python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code + +""" + +import argparse +import dataclasses +import multiprocessing +import os +import time + +import requests + +from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree +from sglang.srt.warmup import warmup + +multiprocessing.set_start_method("spawn", force=True) + +# Reduce warning +os.environ["SGL_IN_DEEPGEMM_PRECOMPILE_STAGE"] = "1" +# Force enable deep gemm +os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "1" +# Force enable mha chunked kv for DeepSeek V3 to avoid missing kv_b_proj DeepGEMM case +os.environ["SGL_CHUNKED_PREFIX_CACHE_THRESHOLD"] = "0" + + +@dataclasses.dataclass +class CompileArgs: + timeout: int = 3600 + + @staticmethod + def add_cli_args(parser: argparse.ArgumentParser): + parser.add_argument("--timeout", type=int, default=CompileArgs.timeout) + + @classmethod + def from_cli_args(cls, args: argparse.Namespace): + # use the default value's type to cast the args into correct types. + attrs = [(attr.name, type(attr.default)) for attr in dataclasses.fields(cls)] + return cls( + **{attr: attr_type(getattr(args, attr)) for attr, attr_type in attrs} + ) + + +@warmup("compile-deep-gemm") +async def warm_up_compile( + disaggregation_mode: str, tokenizer_manager: TokenizerManager +): + print("\nGenerate warm up request for compiling DeepGEMM...\n") + generate_req_input = GenerateReqInput( + input_ids=[0, 1, 2, 3], + sampling_params={ + "temperature": 0.0, + "max_new_tokens": 8, + "ignore_eos": True, + }, + ) + if disaggregation_mode != "null": + generate_req_input.bootstrap_room = 0 + generate_req_input.bootstrap_host = FAKE_BOOTSTRAP_HOST + + await tokenizer_manager.generate_request(generate_req_input, None).__anext__() + + +def launch_server_internal(server_args): + try: + launch_server(server_args) + except Exception as e: + raise e + finally: + kill_process_tree(os.getpid(), include_parent=False) + + +def launch_server_process_and_send_one_request( + server_args: ServerArgs, compile_args: CompileArgs +): + proc = multiprocessing.Process(target=launch_server_internal, args=(server_args,)) + proc.start() + base_url = f"http://{server_args.host}:{server_args.port}" + timeout = compile_args.timeout + + start_time = time.perf_counter() + while time.perf_counter() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + } + if server_args.node_rank == 0: + response = requests.get(f"{base_url}/v1/models", headers=headers) + else: + # This http api is created by launch_dummy_health_check_server for none-rank0 node. + response = requests.get(f"{base_url}/health", headers=headers) + if response.status_code == 200: + # Rank-0 node send a request to sync with other node and then return. + if server_args.node_rank == 0: + response = requests.post( + f"{base_url}/generate", + json={ + "input_ids": [0, 1, 2, 3], + "sampling_params": { + "max_new_tokens": 8, + "temperature": 0, + }, + }, + timeout=600, + ) + if response.status_code != 200: + error = response.json() + raise RuntimeError(f"Sync request failed: {error}") + # Other nodes should wait for the exit signal from Rank-0 node. + else: + start_time_waiting = time.perf_counter() + while proc.is_alive(): + if time.perf_counter() - start_time_waiting < timeout: + time.sleep(10) + else: + raise TimeoutError("Waiting for main node timeout!") + return proc + except requests.RequestException: + pass + time.sleep(10) + raise TimeoutError( + "DeepGEMM Kernels compilation timeout." + "\n\nFeel free and please restart the command." + ) + + +def refine_server_args(server_args: ServerArgs, compile_args: CompileArgs): + # Disable cuda graph and torch compile to save time + server_args.disable_cuda_graph = True + server_args.enable_torch_compile = False + print(f"Disable CUDA Graph and Torch Compile to save time...") + + # Set watchdog timeout to compile_args.timeout because compilation will take a long time + server_args.watchdog_timeout = compile_args.timeout + server_args.warmups = "compile-deep-gemm" + + +def run_compile(server_args: ServerArgs, compile_args: CompileArgs): + print( + "Begin DeepGEMM Kernels compilation...\n" + "It may take a long time and timeout maybe raised " + "while the compilation is still in progress.\n" + "Just feel free to restart the command " + "until the compilation is fully finished.\n" + ) + + proc = launch_server_process_and_send_one_request(server_args, compile_args) + + print("\nDeepGEMM Kernels compilation finished successfully.") + + # Sleep for safety + time.sleep(10) + if proc.is_alive(): + # This is the rank0 node. + kill_process_tree(proc.pid) + else: + try: + kill_process_tree(proc.pid) + except Exception: + pass + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + CompileArgs.add_cli_args(parser) + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + compile_args = CompileArgs.from_cli_args(args) + + refine_server_args(server_args, compile_args) + + run_compile(server_args, compile_args) diff --git a/python/sglang/eval/llama3_eval.py b/python/sglang/eval/llama3_eval.py new file mode 100644 index 000000000..253cdf275 --- /dev/null +++ b/python/sglang/eval/llama3_eval.py @@ -0,0 +1,315 @@ +# Adapt from https://github.com/fw-ai/llm_eval_meta + +import argparse +import asyncio +import os +import pickle +import re +import shutil +from collections import defaultdict +from dataclasses import dataclass + +import httpx +import numpy as np +import openai +from datasets import load_dataset +from openai import AsyncOpenAI +from tqdm import tqdm + +# Mapping providers to their clients and models +provider_to_models = { + "b10": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "oai": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, + "sgl": { + "8b": "meta-llama/Llama-3.1-8B-Instruct", + "70b": "meta-llama/Llama-3.1-70B-Instruct", + "405b": "meta-llama/Llama-3.1-405B-Instruct", + }, +} + + +async def fetch_responses( + client, prompt, semaphore, index, provider, model_size, output_dir, max_tokens +): + output_file = os.path.join(output_dir, f"response_{index}.pkl") + if os.path.exists(output_file): + print(f"File {output_file} already exists, skipping.") + return + + async with semaphore: + response = await client.completions.create( + model=provider_to_models[provider][model_size], + prompt=prompt, + temperature=0.0, + max_tokens=max_tokens, + ) + if isinstance(response, openai.BadRequestError): + with open(output_file, "wb") as f: + pickle.dump("bad_response", f) + assert isinstance(response, openai.types.completion.Completion) + # Save response to a file + with open(output_file, "wb") as f: + pickle.dump(response, f) + + +TASK_TO_MAX_TOKENS = { + "evals__mmlu__details": 1, + "evals__mmlu__0_shot__cot__details": 1024, + # Official meta uses 1024, but a small % (.05) of questions are answered correctly after relaxing + "evals__mmlu_pro__details": 2048, + "evals__gsm8k__details": 1024, +} + +TASK_TO_EVAL_SET = { + "mmlu": "evals__mmlu__details", + "mmlu_cot": "evals__mmlu__0_shot__cot__details", + "mmlu_pro": "evals__mmlu_pro__details", + "gsm8k": "evals__gsm8k__details", +} + + +class CustomAsyncHTTPXClient(httpx.AsyncClient): + async def send(self, request: httpx.Request, *args, **kwargs) -> httpx.Response: + request.url = httpx.URL( + f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict" + ) + return await super().send(request, *args, **kwargs) + + +def get_client(provider): + if provider not in "b10": + if os.getenv("OPENAI_API_KEY") == None: + os.environ["OPENAI_API_KEY"] = "EMPTY" + return { + "oai": AsyncOpenAI(base_url="http://127.0.0.1:8000/v1/"), + "b10": AsyncOpenAI( + api_key=f"Api-Key {os.getenv('OPENAI_API_KEY')}", + base_url=f"https://model-{os.getenv('MODEL_ID')}.api.baseten.co/development/predict", + http_client=CustomAsyncHTTPXClient(), + ), + "sgl": AsyncOpenAI(base_url="http://127.0.0.1:30000/v1/"), + }[provider] + + +# Define the benchmark function +async def benchmark(args): + ds = load_dataset( + "meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{TASK_TO_EVAL_SET[args.task]}", + ) + semaphore = asyncio.Semaphore(args.concurrency) # Limit to 16 concurrent tasks + + if args.num_examples is None: + args.num_examples = len(ds["latest"]["input_final_prompts"]) + prompts = ds["latest"]["input_final_prompts"][: args.num_examples] + + # Create the output directory if it does not exist + os.makedirs(args.output_dir, exist_ok=True) + + tasks = [] + # Create the tasks with tqdm progress bar + max_tokens = TASK_TO_MAX_TOKENS[TASK_TO_EVAL_SET[args.task]] + client = get_client(args.provider) + for idx, prompt in enumerate(tqdm(prompts, desc="Creating tasks")): + tasks.append( + asyncio.create_task( + fetch_responses( + client, + f"<|begin_of_text|>{prompt[0]}", + semaphore, + idx, + args.provider, + args.model_size, + args.output_dir, + max_tokens=max_tokens, + ) + ) + ) + + # Run the tasks with tqdm progress bar + for future in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Processing tasks" + ): + await future + + +def get_mmlu_answer(response): + if response is not None: + return response.choices[0].text.lstrip().rstrip().upper().replace(".", "") + return None + + +def get_mmlu_cot_answer(response): + pattern = r"The best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "").replace("*", "") + + pattern = r"the best answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"The correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + pattern = r"the correct answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + return match.group(1).replace(".", "") + + +def get_answer_gsm8k(response): + pattern = r"The final answer is (.+)\.?" + match = re.search(pattern, response.choices[0].text) + if match: + s = match.group(1) + for ok_symbol in ["%", "$"]: + s = s.replace(ok_symbol, "") + return s + + +TASK_TO_ANSWER_EXTRACTOR = { + "evals__mmlu__details": get_mmlu_answer, + "evals__mmlu__0_shot__cot__details": get_mmlu_cot_answer, + "evals__gsm8k__details": get_answer_gsm8k, + "evals__mmlu_pro__details": get_mmlu_cot_answer, +} + + +def get_dataset_from_task(task, response_path, model_size): + ds_405b = load_dataset( + f"meta-llama/Llama-3.1-405B-Instruct-evals", + f"Llama-3.1-405B-Instruct-{task}", + ) + ds_405b_hash_order = [x[0] for x in ds_405b["latest"]["input_final_prompts_hash"]] + + if "70b" in model_size or "8b" in model_size: + if "70" in model_size: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-70B-Instruct-evals", + f"Llama-3.1-70B-Instruct-{task}", + ) + else: + ref_model_ds = load_dataset( + f"meta-llama/Llama-3.1-8B-Instruct-evals", + f"Llama-3.1-8B-Instruct-{task}", + ) + + hash_to_row = {} + for row in ref_model_ds["latest"]: + hash_to_row[row["input_final_prompts_hash"][0]] = row + reordered_rows = [] + for prompt_hash in ds_405b_hash_order: + reordered_rows.append(hash_to_row[prompt_hash]) + ref_model_ds["latest"] = reordered_rows + return ref_model_ds + + return ds_405b + + +def analyze(task, response_path, model_size): + ds = get_dataset_from_task(task, response_path, model_size) + + responses = [] + total = len(ds["latest"]) + + for i in range(0, total): + response = pickle.load( + open(os.path.join(response_path, f"response_{i}.pkl"), "rb") + ) + responses.append(response) + + @dataclass + class Stats: + correct: int = 0 + total: int = 0 + meta_correct: int = 0 + + average: float = None + + subtask_name_to_stats = defaultdict(lambda: Stats()) + + for response, ds_row in zip(responses, ds["latest"]): + model_answer = TASK_TO_ANSWER_EXTRACTOR[task](response) + + subtask = ds_row["subtask_name"] + + is_eval_correct = model_answer in ds_row["input_correct_responses"] + if is_eval_correct: + subtask_name_to_stats[subtask].correct += 1 + + if ds_row["is_correct"]: + subtask_name_to_stats[subtask].meta_correct += 1 + + subtask_name_to_stats[subtask].total += 1 + + micro_stats = Stats() + for subtask, stats in subtask_name_to_stats.items(): + stats.average = stats.correct / stats.total + stats.meta_average = stats.meta_correct / stats.total + + micro_stats.correct += stats.correct + micro_stats.total += stats.total + micro_stats.meta_correct += stats.meta_correct + + micro_stats.average = micro_stats.correct / micro_stats.total + micro_stats.meta_average = micro_stats.meta_correct / micro_stats.total + + print("Macro average", np.mean([x.average for x in subtask_name_to_stats.values()])) + print( + "Meta Macro average", + np.mean([x.meta_average for x in subtask_name_to_stats.values()]), + ) + print("Micro average", micro_stats.average) + print("Meta Micro average", micro_stats.meta_average) + + +# Entry point for the script +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Script to run model with specified parameters." + ) + parser.add_argument( + "--model-size", + type=str, + default="8b", + help="Size of the model (e.g., 8b or 70b)", + ) + parser.add_argument( + "--provider", + type=str, + default="sgl", + help="Provider name (e.g., sgl, oai, b10)", + ) + parser.add_argument( + "--task", + type=str, + required=True, + help="Task (e.g., mmlu, mmlu_cot, mmlu_pro, gsm8k)", + ) + parser.add_argument( + "--num-examples", type=int, default=None, help="Number of examples to process" + ) + parser.add_argument("--concurrency", type=int, default=16) + parser.add_argument( + "--output-dir", + type=str, + default="tmp-output-dir", + help="Directory to save responses", + ) + + args = parser.parse_args() + asyncio.run(benchmark(args)) + analyze(TASK_TO_EVAL_SET[args.task], args.output_dir, args.model_size) + shutil.rmtree("tmp-output-dir", ignore_errors=True) diff --git a/python/sglang/eval/loogle_eval.py b/python/sglang/eval/loogle_eval.py new file mode 100644 index 000000000..895362cd1 --- /dev/null +++ b/python/sglang/eval/loogle_eval.py @@ -0,0 +1,164 @@ +import argparse +import asyncio +import os +import pickle +from pathlib import Path +from typing import List + +import openai +import torch +from bert_score import BERTScorer +from datasets import load_dataset +from tqdm import tqdm + + +def get_client(api_url: str) -> openai.AsyncOpenAI: + if os.getenv("OPENAI_API_KEY") is None: + os.environ["OPENAI_API_KEY"] = "EMPTY" + return openai.AsyncOpenAI(base_url=api_url) + + +def get_dataset(): + return load_dataset("bigai-nlco/LooGLE", "longdep_qa", split="test") + + +async def fetch_response( + client: openai.AsyncOpenAI, + context: str, + question: str, + semaphore: asyncio.Semaphore, + index: int, + model: str, + output_dir: Path, +): + output_file = output_dir / f"response_{index}.pkl" + if output_file.exists(): + return + + prompt = ( + "Please answer the question based on the long texts below.\n" + f"{context}\n" + f"Question: {question}\n" + "Answer:" + ) + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": prompt}, + ] + + async with semaphore: + try: + response = await client.chat.completions.create( + model=model, + messages=messages, + temperature=0.0, + max_tokens=512, + ) + except openai.BadRequestError as e: + with open(output_file, "wb") as f: + pickle.dump({"error": str(e)}, f) + return + + with open(output_file, "wb") as f: + pickle.dump(response, f) + + +async def benchmark(args): + dataset = get_dataset() + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + client = get_client(args.api_url) + semaphore = asyncio.Semaphore(args.max_concurrency) + + tasks: List[asyncio.Task] = [] + for idx, ex in enumerate(dataset): + if idx >= args.num_prompts: + break + tasks.append( + asyncio.create_task( + fetch_response( + client, + ex["context"], + ex["question"], + semaphore, + idx, + args.model, + output_dir, + ) + ) + ) + + for _ in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Running benchmark" + ): + await _ + + +def analyse(args): + dataset = get_dataset() + output_dir = Path(args.output_dir) + + device = "cuda" if torch.cuda.is_available() else "cpu" + scorer = BERTScorer(lang="en", device=device) + + hyps: List[str] = [] + refs: List[str] = [] + for idx, ex in enumerate(tqdm(dataset, desc="Loading responses")): + if idx >= args.num_prompts: + break + pkl_file = output_dir / f"response_{idx}.pkl" + if not pkl_file.exists(): + raise FileNotFoundError(pkl_file) + + response = pickle.load(open(pkl_file, "rb")) + if isinstance(response, dict) and "error" in response: + continue + + hyps.append(response.choices[0].message.content.strip()) + refs.append(ex["answer"]) + + if not hyps: + print("No valid responses to score!") + return + + batch_size = 64 + all_f1: List[float] = [] + for i in tqdm(range(0, len(hyps), batch_size), desc="Scoring batches"): + h_batch = hyps[i : i + batch_size] + r_batch = refs[i : i + batch_size] + _, _, f1_scores = scorer.score(h_batch, r_batch, verbose=False) + all_f1.extend([float(x) for x in f1_scores]) + + avg = sum(all_f1) / len(all_f1) + print(f"Average BERTScore (F1): {avg:.2%}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run benchmark and evaluation in one go." + ) + parser.add_argument( + "--api-url", + default="http://127.0.0.1:30000/v1", + help="OpenAI‑compatible API base URL", + ) + parser.add_argument( + "--model", + default="meta-llama/Llama-4-Maverick-17B-128E-Instruct", + help="Model name or ID, only used for model name", + ) + parser.add_argument( + "--max-concurrency", type=int, default=144, help="Maximum concurrent requests" + ) + parser.add_argument( + "--output-dir", default="tmp-output-dir", help="Directory for cached responses" + ) + parser.add_argument( + "--num-prompts", type=int, default=10000, help="Number of prompts to run" + ) + args = parser.parse_args() + + asyncio.run(benchmark(args)) + + analyse(args) diff --git a/python/sglang/global_config.py b/python/sglang/global_config.py new file mode 100644 index 000000000..f006bd94c --- /dev/null +++ b/python/sglang/global_config.py @@ -0,0 +1,53 @@ +"""Global configurations""" + +import os + + +class GlobalConfig: + """ + Store some global constants. + + See also python/sglang/srt/managers/schedule_batch.py::global_server_args_dict, which stores + many global runtime arguments as well. + """ + + def __init__(self): + # Verbosity level + # 0: do not output anything + # 2: output final text after every run + self.verbosity = 0 + + # Default backend of the language + self.default_backend = None + + # Runtime constants: New generation token ratio estimation + self.default_init_new_token_ratio = float( + os.environ.get("SGLANG_INIT_NEW_TOKEN_RATIO", 0.7) + ) + self.default_min_new_token_ratio_factor = float( + os.environ.get("SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR", 0.14) + ) + self.default_new_token_ratio_decay_steps = float( + os.environ.get("SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS", 600) + ) + self.torch_empty_cache_interval = float( + os.environ.get( + "SGLANG_EMPTY_CACHE_INTERVAL", -1 + ) # in seconds. Set if you observe high memory accumulation over a long serving period. + ) + # Runtime constants: others + self.retract_decode_steps = 20 + self.flashinfer_workspace_size = os.environ.get( + "FLASHINFER_WORKSPACE_SIZE", 384 * 1024 * 1024 + ) + + # Output tokenization configs + self.skip_special_tokens_in_output = True + self.spaces_between_special_tokens_in_out = True + + # Language frontend interpreter optimization configs + self.enable_precache_with_tracing = True + self.enable_parallel_encoding = True + + +global_config = GlobalConfig() diff --git a/python/sglang/lang/api.py b/python/sglang/lang/api.py new file mode 100644 index 000000000..a8d2e43e6 --- /dev/null +++ b/python/sglang/lang/api.py @@ -0,0 +1,286 @@ +"""Public APIs of the language.""" + +import re +from typing import Callable, List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.choices import ChoicesSamplingMethod, token_length_normalized +from sglang.lang.ir import ( + SglExpr, + SglExprList, + SglFunction, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglSeparateReasoning, + SglVideo, +) + + +def function( + func: Optional[Callable] = None, num_api_spec_tokens: Optional[int] = None +): + if func: + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + def decorator(func): + return SglFunction(func, num_api_spec_tokens=num_api_spec_tokens) + + return decorator + + +def Runtime(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.lang.backend.runtime_endpoint import Runtime + + return Runtime(*args, **kwargs) + + +def Engine(*args, **kwargs): + # Avoid importing unnecessary dependency + from sglang.srt.entrypoints.engine import Engine + + return Engine(*args, **kwargs) + + +def set_default_backend(backend: BaseBackend): + global_config.default_backend = backend + + +def flush_cache(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return False + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.flush_cache() + + +def get_server_info(backend: Optional[BaseBackend] = None): + backend = backend or global_config.default_backend + if backend is None: + return None + + # If backend is Runtime + if hasattr(backend, "endpoint"): + backend = backend.endpoint + return backend.get_server_info() + + +def gen( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + min_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[Union[type, str]] = None, + choices: Optional[List[str]] = None, + choices_method: Optional[ChoicesSamplingMethod] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, +): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + + if choices: + return SglSelect( + name, + choices, + 0.0 if temperature is None else temperature, + token_length_normalized if choices_method is None else choices_method, + ) + + # check regex is valid + if regex is not None: + try: + re.compile(regex) + except re.error as e: + raise e + + return SglGen( + name, + max_tokens, + min_tokens, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + dtype, + regex, + json_schema, + ) + + +def gen_int( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + int, + None, + ) + + +def gen_string( + name: Optional[str] = None, + max_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, +): + return SglGen( + name, + max_tokens, + None, + n, + stop, + stop_token_ids, + temperature, + top_p, + top_k, + min_p, + frequency_penalty, + presence_penalty, + ignore_eos, + return_logprob, + logprob_start_len, + top_logprobs_num, + return_text_in_logprobs, + str, + None, + ) + + +def image(expr: SglExpr): + return SglImage(expr) + + +def video(path: str, num_frames: int): + return SglVideo(path, num_frames) + + +def select( + name: Optional[str] = None, + choices: Optional[List[str]] = None, + temperature: float = 0.0, + choices_method: ChoicesSamplingMethod = token_length_normalized, +): + assert choices is not None + return SglSelect(name, choices, temperature, choices_method) + + +def _role_common(name: str, expr: Optional[SglExpr] = None): + if expr is None: + return SglExprList([SglRoleBegin(name), SglRoleEnd(name)]) + else: + return SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + + +def system(expr: Optional[SglExpr] = None): + return _role_common("system", expr) + + +def user(expr: Optional[SglExpr] = None): + return _role_common("user", expr) + + +def assistant(expr: Optional[SglExpr] = None): + return _role_common("assistant", expr) + + +def system_begin(): + return SglRoleBegin("system") + + +def system_end(): + return SglRoleEnd("system") + + +def user_begin(): + return SglRoleBegin("user") + + +def user_end(): + return SglRoleEnd("user") + + +def assistant_begin(): + return SglRoleBegin("assistant") + + +def assistant_end(): + return SglRoleEnd("assistant") + + +def separate_reasoning( + expr: Optional[SglExpr] = None, model_type: Optional[str] = None +): + return SglExprList([expr, SglSeparateReasoning(model_type, expr=expr)]) diff --git a/python/sglang/lang/backend/anthropic.py b/python/sglang/lang/backend/anthropic.py new file mode 100644 index 000000000..4918a1703 --- /dev/null +++ b/python/sglang/lang/backend/anthropic.py @@ -0,0 +1,73 @@ +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import anthropic +except ImportError as e: + anthropic = e + + +class Anthropic(BaseBackend): + def __init__(self, model_name, *args, **kwargs): + super().__init__() + + if isinstance(anthropic, Exception): + raise anthropic + + self.model_name = model_name + self.chat_template = get_chat_template("claude") + self.client = anthropic.Anthropic(*args, **kwargs) + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + ret = self.client.messages.create( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) + comp = ret.content[0].text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + if messages and messages[0]["role"] == "system": + system = messages.pop(0)["content"] + else: + system = "" + + with self.client.messages.stream( + model=self.model_name, + system=system, + messages=messages, + **sampling_params.to_anthropic_kwargs(), + ) as stream: + for text in stream.text_stream: + yield text, {} diff --git a/python/sglang/lang/backend/base_backend.py b/python/sglang/lang/backend/base_backend.py new file mode 100644 index 000000000..62dd50416 --- /dev/null +++ b/python/sglang/lang/backend/base_backend.py @@ -0,0 +1,82 @@ +from typing import List, Optional, Union + +from sglang.lang.chat_template import get_chat_template +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + + +class BaseBackend: + def __init__(self) -> None: + self.support_concate_and_append = False + self.chat_template = get_chat_template("default") + + def get_model_name(self): + raise NotImplementedError() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + pass + + def uncache_prefix(self, rid: str): + pass + + def end_request(self, rid: Union[str, List[str]]): + pass + + def begin_program(self, s: StreamExecutor): + pass + + def end_program(self, s: Union[StreamExecutor, List[StreamExecutor]]): + pass + + def commit_lazy_operations(self, s: StreamExecutor): + pass + + def fork_program( + self, + src: StreamExecutor, + dst: List[StreamExecutor], + position_ids_offset: Optional[List[int]] = None, + ): + pass + + def fill_image(self, s: StreamExecutor): + pass + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + raise NotImplementedError() + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: Optional[ChoicesSamplingMethod] = None, + ) -> ChoicesDecision: + raise NotImplementedError() + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + raise NotImplementedError() + + def shutdown(self): + pass + + def flush_cache(self): + pass + + def get_server_info(self): + pass diff --git a/python/sglang/lang/backend/litellm.py b/python/sglang/lang/backend/litellm.py new file mode 100644 index 000000000..5803b5431 --- /dev/null +++ b/python/sglang/lang/backend/litellm.py @@ -0,0 +1,90 @@ +from typing import Mapping, Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import litellm +except ImportError as e: + litellm = e + litellm.num_retries = 1 + + +class LiteLLM(BaseBackend): + def __init__( + self, + model_name, + chat_template=None, + api_key=None, + organization: Optional[str] = None, + base_url: Optional[str] = None, + timeout: Optional[float] = 600, + max_retries: Optional[int] = litellm.num_retries, + default_headers: Optional[Mapping[str, str]] = None, + ): + super().__init__() + + if isinstance(litellm, Exception): + raise litellm + + self.model_name = model_name + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + self.client_params = { + "api_key": api_key, + "organization": organization, + "base_url": base_url, + "timeout": timeout, + "max_retries": max_retries, + "default_headers": default_headers, + } + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + comp = ret.choices[0].message.content + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + messages = s.messages_ + else: + messages = [{"role": "user", "content": s.text_}] + + ret = litellm.completion( + model=self.model_name, + messages=messages, + stream=True, + **self.client_params, + **sampling_params.to_litellm_kwargs(), + ) + for chunk in ret: + text = chunk.choices[0].delta.content + if text is not None: + yield text, {} diff --git a/python/sglang/lang/backend/openai.py b/python/sglang/lang/backend/openai.py new file mode 100644 index 000000000..a2d006bb7 --- /dev/null +++ b/python/sglang/lang/backend/openai.py @@ -0,0 +1,475 @@ +import dataclasses +import logging +import time +import warnings +from typing import List, Optional, Union + +import numpy as np + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import ChatTemplate, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import openai + import tiktoken +except ImportError as e: + openai = tiktoken = e + + +logger = logging.getLogger(__name__) + + +def create_logit_bias_int(tokenizer): + """Get logit bias for integer numbers.""" + int_token_ids = [] + + tokens = tokenizer._mergeable_ranks + for token, token_id in tokens.items(): + s = tokenizer.decode([token_id]) + if all([c.isdigit() for c in s]) or s in [" "]: + int_token_ids.append(token_id) + if len(int_token_ids) >= 300: # OpenAI API limit + break + special_tokens = tokenizer._special_tokens + mask = {t: 100 for t in int_token_ids[:299]} + mask[special_tokens["<|endoftext|>"]] = 100 + return mask + + +INSTRUCT_MODEL_NAMES = [ + "gpt-3.5-turbo-instruct", +] + + +@dataclasses.dataclass +class TokenUsage: + prompt_tokens: int + completion_tokens: int + + def reset(self): + self.prompt_tokens = self.completion_tokens = 0 + + +class OpenAI(BaseBackend): + def __init__( + self, + model_name: str, + is_chat_model: Optional[bool] = None, + chat_template: Optional[ChatTemplate] = None, + is_azure: bool = False, + *args, + **kwargs, + ): + super().__init__() + + if isinstance(openai, Exception): + raise openai + + if is_azure: + self.client = openai.AzureOpenAI(*args, **kwargs) + else: + self.client = openai.OpenAI(*args, **kwargs) + + self.model_name = model_name + try: + self.tokenizer = tiktoken.encoding_for_model(model_name) + except KeyError: + self.tokenizer = tiktoken.get_encoding("cl100k_base") + self.logit_bias_int = create_logit_bias_int(self.tokenizer) + + self.chat_template = chat_template or get_chat_template_by_model_path( + model_name + ) + + if is_chat_model is not None: + self.is_chat_model = is_chat_model + else: + if model_name in INSTRUCT_MODEL_NAMES: + self.is_chat_model = False + else: + self.is_chat_model = True + + self.chat_prefix = self.chat_template.role_prefix_and_suffix["assistant"][0] + + # Usage + self.token_usage = TokenUsage(0, 0) + + # API speculative execution + # TODO(ying): This does not support multi-threading (run_batch) + self.spec_kwargs = {} + self.spec_format = [] + self.spec_max_num_tries = 3 + + def get_chat_template(self): + return self.chat_template + + def _prepare_spec_execution( + self, + sampling_params: SglSamplingParams, + num_api_spec_tokens: int, + spec_var_name: str, + ): + if "max_tokens" not in self.spec_kwargs: + self.spec_kwargs["max_tokens"] = num_api_spec_tokens + else: + assert self.spec_kwargs["max_tokens"] == num_api_spec_tokens + + params = sampling_params.to_openai_kwargs() + for key, value in params.items(): + if key in ["stop"]: + continue + if key in ["max_tokens"]: + warnings.warn( + "The parameter max_tokens will be overwritten by speculated number of tokens." + ) + continue + if key not in self.spec_kwargs: + self.spec_kwargs[key] = value + else: + assert ( + value == self.spec_kwargs[key] + ), "sampling parameters should be consistent if turn on api speculative execution." + self.spec_format.append( + {"text": "", "stop": params["stop"], "name": spec_var_name} + ) + return "", {} + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + spec_var_name: str = None, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if s.num_api_spec_tokens is None: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported if api speculative execution is off. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant. " + "Example of adding api speculative execution: @function(num_api_spec_tokens=128)." + ) + prompt = s.messages_ + else: + return self._prepare_spec_execution( + sampling_params, s.num_api_spec_tokens, spec_var_name + ) + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + if ( + self.model_name.startswith("o1") + or self.model_name.startswith("o3") + or "o1" in self.model_name + ): + kwargs.pop("max_tokens", None) + else: + kwargs.pop("max_completion_tokens", None) + + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + # Keep the returned list (or string) as is. + elif sampling_params.dtype in [str, "str", "string"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_ + '"', + stop='"', + **kwargs, + ) + # Wrap each element in quotes if we have a list. + if isinstance(comp, list): + comp = ['"' + x + '"' for x in comp] + else: + comp = '"' + comp + '"' + elif sampling_params.dtype in [int, "int"]: + assert ( + not self.is_chat_model + ), "constrained type not supported on chat model" + kwargs = sampling_params.to_openai_kwargs() + kwargs.pop("stop") + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.text_, + logit_bias=self.logit_bias_int, + stop=[" "], + **kwargs, + ) + # Leave as a list if that's what is returned. + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + return comp, {} + + def spec_fill(self, value: str): + assert self.is_chat_model + self.spec_format.append({"text": value, "stop": None, "name": None}) + + def spec_pattern_match(self, comp): + for i, term in enumerate(self.spec_format): + text = term["text"] + if text != "": + if comp.startswith(text): + comp = comp[len(text) :] + else: + return False + else: + pos = comp.find(term["stop"]) + if pos != -1: + term["text"] = comp[:pos] + comp = comp[pos:] + else: + if i == len(self.spec_format) - 1: + term["text"] = comp + else: + return False + return True + + def role_end_generate( + self, + s: StreamExecutor, + ): + if s.num_api_spec_tokens is None or not s.text_.endswith(self.chat_prefix): + return + + comp = "" + if not all(x["name"] is None for x in self.spec_format): + # TODO(ying): throw errors or warnings + for i in range(self.spec_max_num_tries): + comp = openai_completion( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=s.messages_, + **self.spec_kwargs, + ) + # Use a string for pattern matching. + comp_for_match = comp[0] if isinstance(comp, list) else comp + if self.spec_pattern_match(comp_for_match): + break + + for term in self.spec_format: + s.text_ += term["text"] + name = term["name"] + if name is not None: + s.variables[name] = term["text"] + s.meta_info[name] = {} + s.variable_event[name].set() + + self.spec_kwargs = {} + self.spec_format = [] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if sampling_params.dtype is None: + if self.is_chat_model: + if not s.text_.endswith(self.chat_prefix): + raise RuntimeError( + "This use case is not supported. " + "For OpenAI chat models, sgl.gen must be right after sgl.assistant" + ) + prompt = s.messages_ + else: + prompt = s.text_ + + kwargs = sampling_params.to_openai_kwargs() + generator = openai_completion_stream( + client=self.client, + token_usage=self.token_usage, + is_chat=self.is_chat_model, + model=self.model_name, + prompt=prompt, + **kwargs, + ) + return generator + else: + raise ValueError(f"Unknown dtype: {sampling_params.dtype}") + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + """Note: `choices_method` is not used by the OpenAI backend.""" + if self.is_chat_model: + raise NotImplementedError( + "select/choices is not supported for chat models. " + "Please try to use a non-chat model such as gpt-3.5-turbo-instruct" + ) + + n_choices = len(choices) + token_ids = [self.tokenizer.encode(x) for x in choices] + scores = [0] * n_choices + valid = [len(x) > 0 for x in token_ids] + prompt_tokens = self.tokenizer.encode(s.text_) + + max_len = max([len(x) for x in token_ids]) + for step in range(max_len): + # Build logit bias + logit_bias = {} + for i in range(n_choices): + if valid[i]: + logit_bias[token_ids[i][step]] = 100 + + # Call API + ret = self.client.completions.create( + model=self.model_name, + prompt=prompt_tokens, + logit_bias=logit_bias, + max_tokens=1, + temperature=temperature, + ) + ret_str = ret.choices[0].text + ret_token = self.tokenizer.encode(ret_str)[0] + self.token_usage.prompt_tokens += ret.usage.prompt_tokens + self.token_usage.completion_tokens = ret.usage.completion_tokens + + # TODO: + # 1. return logits as the scores + # 2. compute logits of the full choice + # 3. consider chunk-based decoding + + # Update valid + hit = False + for i in range(n_choices): + if valid[i]: + if step == len(token_ids[i]) - 1: + valid[i] = False + + if ret_token == token_ids[i][step]: + scores[i] += 1 + hit = True + else: + valid[i] = False + assert hit + + if np.sum(valid) <= 1: + break + + prompt_tokens.append(ret_token) + + return ChoicesDecision( + decision=choices[np.argmax(scores)], + meta_info={"scores": scores}, + ) + + +def openai_completion( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +) -> Union[str, List[str]]: + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + ret = client.chat.completions.create(messages=prompt, **kwargs) + if len(ret.choices) == 1: + comp = ret.choices[0].message.content + else: + comp = [c.message.content for c in ret.choices] + else: + ret = client.completions.create(prompt=prompt, **kwargs) + if isinstance(prompt, (list, tuple)): + comp = [c.text for c in ret.choices] + else: + comp = ret.choices[0].text + if len(ret.choices) > 1: + comp = [c.text for c in ret.choices] + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e + + return comp + + +def openai_completion_stream( + client, token_usage, is_chat=None, retries=3, prompt=None, **kwargs +): + # if "ebnf" is in kwargs, warn and remove + if "ebnf" in kwargs: + warnings.warn("EBNF is not officially supported by OpenAI endpoints. Ignoring.") + del kwargs["ebnf"] + + for attempt in range(retries): + try: + if is_chat: + if "stop" in kwargs and kwargs["stop"] is None: + kwargs.pop("stop") + generator = client.chat.completions.create( + messages=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + try: + content = ret.choices[0].delta.content + except IndexError: + content = None + yield content or "", {} + else: + generator = client.completions.create( + prompt=prompt, + stream=True, + stream_options={"include_usage": True}, + **kwargs, + ) + for ret in generator: + if len(ret.choices) == 0: + continue + content = ret.choices[0].text + yield content or "", {} + + token_usage.prompt_tokens += ret.usage.prompt_tokens + token_usage.completion_tokens += ret.usage.completion_tokens + break + except (openai.APIError, openai.APIConnectionError, openai.RateLimitError) as e: + logger.error(f"OpenAI Error: {e}. Waiting 5 seconds...") + time.sleep(5) + if attempt == retries - 1: + raise e + except Exception as e: + logger.error(f"RuntimeError {e}.") + raise e diff --git a/python/sglang/lang/backend/runtime_endpoint.py b/python/sglang/lang/backend/runtime_endpoint.py new file mode 100644 index 000000000..349f9934a --- /dev/null +++ b/python/sglang/lang/backend/runtime_endpoint.py @@ -0,0 +1,527 @@ +import atexit +import json +import multiprocessing +import warnings +from typing import Dict, List, Optional, Union + +import aiohttp +import requests + +from sglang.global_config import global_config +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template, get_chat_template_by_model_path +from sglang.lang.choices import ChoicesDecision, ChoicesSamplingMethod +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import ( + REGEX_BOOL, + REGEX_FLOAT, + REGEX_INT, + REGEX_STR, + SglSamplingParams, +) +from sglang.utils import http_request + + +class RuntimeEndpoint(BaseBackend): + def __init__( + self, + base_url: str, + api_key: Optional[str] = None, + verify: Optional[str] = None, + chat_template_name: Optional[str] = None, + ): + super().__init__() + self.support_concate_and_append = True + + self.base_url = base_url + self.api_key = api_key + self.verify = verify + + res = http_request( + self.base_url + "/get_model_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + self.model_info = res.json() + + if chat_template_name: + self.chat_template = get_chat_template(chat_template_name) + else: + self.chat_template = get_chat_template_by_model_path( + self.model_info["model_path"] + ) + + def get_model_name(self): + return self.model_info["model_path"] + + def flush_cache(self): + res = http_request( + self.base_url + "/flush_cache", + api_key=self.api_key, + verify=self.verify, + method="POST", + ) + self._assert_success(res) + + def get_server_info(self): + res = http_request( + self.base_url + "/get_server_info", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def get_chat_template(self): + return self.chat_template + + def cache_prefix(self, prefix_str: str): + res = http_request( + self.base_url + "/generate", + json={"text": prefix_str, "sampling_params": {"max_new_tokens": 0}}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def start_profile(self): + res = http_request( + self.base_url + "/start_profile", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def stop_profile(self): + res = http_request( + self.base_url + "/stop_profile", + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def commit_lazy_operations(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def fill_image(self, s: StreamExecutor): + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _handle_dtype_to_regex(self, sampling_params: SglSamplingParams): + if sampling_params.dtype is None: + return + + if sampling_params.stop == (): + sampling_params.stop = [] + + dtype_regex = None + if sampling_params.dtype in ["int", int]: + + dtype_regex = REGEX_INT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["float", float]: + + dtype_regex = REGEX_FLOAT + sampling_params.stop.extend([" ", "\n"]) + elif sampling_params.dtype in ["str", str]: + + dtype_regex = REGEX_STR + elif sampling_params.dtype in ["bool", bool]: + + dtype_regex = REGEX_BOOL + else: + raise RuntimeError(f"Invalid dtype: {sampling_params.dtype}") + + if dtype_regex is not None and sampling_params.regex is not None: + warnings.warn( + f"Both dtype and regex are set. Only dtype will be used. dtype: {sampling_params.dtype}, regex: {sampling_params.regex}" + ) + + sampling_params.regex = dtype_regex + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + obj = res.json() + comp = obj["text"] + return comp, obj["meta_info"] + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + self._handle_dtype_to_regex(sampling_params) + + data = { + "text": s.text_, + "sampling_params": { + "skip_special_tokens": global_config.skip_special_tokens_in_output, + "spaces_between_special_tokens": global_config.spaces_between_special_tokens_in_out, + **sampling_params.to_srt_kwargs(), + }, + } + + for item in [ + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + data[item] = value + + data["stream"] = True + self._add_images(s, data) + + res = http_request( + self.base_url + "/generate", + json=data, + stream=True, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + pos = 0 + + for chunk in res.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + chunk_text = data["text"][pos:] + meta_info = data["meta_info"] + pos += len(chunk_text) + yield chunk_text, meta_info + + def select( + self, + s: StreamExecutor, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ) -> ChoicesDecision: + assert temperature <= 1e-5 + + # Cache common prefix + data = {"text": s.text_, "sampling_params": {"max_new_tokens": 0}} + obj = self._generate_http_request(s, data) + prompt_len = obj["meta_info"]["prompt_tokens"] + logprob_start_len = max(prompt_len - 2, 0) # For token healing + + # Compute logprob + data = { + "text": [s.text_ + c for c in choices], + "sampling_params": { + "max_new_tokens": 0, + "temperature": 0, + }, + "return_logprob": True, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + } + obj = self._generate_http_request(s, data) + + input_token_logprobs = [r["meta_info"]["input_token_logprobs"] for r in obj] + output_token_logprobs = [r["meta_info"]["output_token_logprobs"] for r in obj] + normalized_prompt_logprobs = [ + compute_normalized_prompt_logprobs(r["meta_info"]["input_token_logprobs"]) + for r in obj + ] + + # Remove extra token if no token healing occurred + for i in range(len(input_token_logprobs)): + healed_token_str = input_token_logprobs[i][0][-1] + if s.text_.endswith(healed_token_str): + healed_token_logprob = input_token_logprobs[i][0][0] + normalized_prompt_logprobs[i] = ( + normalized_prompt_logprobs[i] * len(input_token_logprobs[i]) + - healed_token_logprob + ) / (len(input_token_logprobs[i]) - 1) + input_token_logprobs[i] = input_token_logprobs[i][1:] + + # Compute unconditional logprobs if required + if choices_method.requires_unconditional_logprobs: + input_ids = [[el[1] for el in subl] for subl in input_token_logprobs] + data = { + "input_ids": input_ids, + "sampling_params": {"max_new_tokens": 0}, + "return_logprob": True, + } + obj = self._generate_http_request(s, data) + unconditional_token_logprobs = [ + r["meta_info"]["input_token_logprobs"] for r in obj + ] + else: + unconditional_token_logprobs = None + + return choices_method( + choices=choices, + normalized_prompt_logprobs=normalized_prompt_logprobs, + input_token_logprobs=input_token_logprobs, + output_token_logprobs=output_token_logprobs, + unconditional_token_logprobs=unconditional_token_logprobs, + ) + + def concatenate_and_append(self, src_rids: List[str], dst_rid: str): + res = http_request( + self.base_url + "/concate_and_append_request", + json={"src_rids": src_rids, "dst_rid": dst_rid}, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + + def _generate_http_request(self, s: StreamExecutor, data): + self._add_images(s, data) + res = http_request( + self.base_url + "/generate", + json=data, + api_key=self.api_key, + verify=self.verify, + ) + self._assert_success(res) + return res.json() + + def _add_images(self, s: StreamExecutor, data): + if s.images_: + assert len(s.images_) == 1, "Only support one image." + data["image_data"] = s.images_[0][1] + + def _assert_success(self, res): + if res.status_code != 200: + try: + content = res.json() + except json.JSONDecodeError: + content = res.text + raise RuntimeError(content) + + +def compute_normalized_prompt_logprobs(input_logprobs): + values = [x[0] for x in input_logprobs if x[0]] + return sum(values) / len(values) + + +class Runtime: + """ + A wrapper for the HTTP server. + This is used for launching the server in a python program without + using the command line interface. + + It is mainly used for the frontend language. + You should use the Engine class if you want to do normal offline processing without the frontend language. + """ + + def __init__( + self, + log_level: str = "error", + *args, + **kwargs, + ): + """See the arguments in server_args.py::ServerArgs""" + # We delay the import of any `sglang.srt` components in `sglang.lang`, so users can run + # client code without installing SRT server and its dependency if they want. + from sglang.srt.entrypoints.http_server import launch_server + from sglang.srt.server_args import ServerArgs + from sglang.srt.utils import is_port_available + + self.server_args = ServerArgs(*args, log_level=log_level, **kwargs) + + # Pre-allocate ports + for port in range(self.server_args.port, 40000): + if is_port_available(port): + break + self.server_args.port = port + + self.url = self.server_args.url() + self.generate_url = self.url + "/generate" + + # NOTE: We store pid instead of proc to fix some issues during __delete__ + self.pid = None + pipe_reader, pipe_writer = multiprocessing.Pipe(duplex=False) + + ctx = multiprocessing.get_context("spawn") + proc = ctx.Process( + target=launch_server, + args=(self.server_args, pipe_writer), + ) + proc.start() + pipe_writer.close() + self.pid = proc.pid + + # Before python program terminates, call shutdown implicitly. Therefore, users don't have to explicitly call .shutdown() + atexit.register(self.shutdown) + + # TODO: remove this pipe_writer mechanism and use `/health_generate` instead. + try: + init_state = pipe_reader.recv() + except EOFError: + init_state = "" + + if init_state != "ready": + self.shutdown() + raise RuntimeError( + "Initialization failed. Please see the error messages above." + ) + + self.endpoint = RuntimeEndpoint(self.url) + + def shutdown(self): + from sglang.srt.utils import kill_process_tree + + if self.pid is not None: + kill_process_tree(self.pid) + self.pid = None + + def start_profile(self): + self.endpoint.start_profile() + + def stop_profile(self): + self.endpoint.stop_profile() + + def cache_prefix(self, prefix: str): + self.endpoint.cache_prefix(prefix) + + def get_tokenizer(self): + from sglang.srt.hf_transformers_utils import get_tokenizer + + return get_tokenizer( + self.server_args.tokenizer_path, + tokenizer_mode=self.server_args.tokenizer_mode, + trust_remote_code=self.server_args.trust_remote_code, + revision=self.server_args.revision, + ) + + async def async_generate( + self, + prompt: str, + sampling_params: Optional[Dict] = None, + ): + if self.server_args.skip_tokenizer_init: + json_data = { + "input_ids": prompt, + "sampling_params": sampling_params, + "stream": True, + } + else: + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "stream": True, + } + pos = 0 + + timeout = aiohttp.ClientTimeout(total=3 * 3600) + async with aiohttp.ClientSession(timeout=timeout, trust_env=True) as session: + async with session.post(self.generate_url, json=json_data) as response: + async for chunk, _ in response.content.iter_chunks(): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]\n\n": + break + data = json.loads(chunk[5:].strip("\n")) + if "text" in data: + cur = data["text"][pos:] + if cur: + yield cur + pos += len(cur) + else: + yield data + + add_request = async_generate + + def generate( + self, + prompt: Union[str, List[str]], + sampling_params: Optional[Dict] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + lora_path: Optional[List[Optional[str]]] = None, + ): + json_data = { + "text": prompt, + "sampling_params": sampling_params, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "lora_path": lora_path, + } + assert not isinstance(lora_path, list) or len(lora_path) == len(prompt) + response = requests.post( + self.url + "/generate", + json=json_data, + ) + return json.dumps(response.json()) + + def encode( + self, + prompt: Union[str, List[str], List[Dict], List[List[Dict]]], + ): + json_data = {"text": prompt} + response = requests.post(self.url + "/encode", json=json_data) + return json.dumps(response.json()) + + async def get_server_info(self): + async with aiohttp.ClientSession() as session: + async with session.get(f"{self.url}/get_server_info") as response: + if response.status == 200: + return await response.json() + else: + error_data = await response.json() + raise RuntimeError( + f"Failed to get server info. {error_data['error']['message']}" + ) + + def __del__(self): + self.shutdown() diff --git a/python/sglang/lang/backend/vertexai.py b/python/sglang/lang/backend/vertexai.py new file mode 100644 index 000000000..3d51fb137 --- /dev/null +++ b/python/sglang/lang/backend/vertexai.py @@ -0,0 +1,148 @@ +import os +import warnings + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglSamplingParams + +try: + import vertexai + from vertexai.preview.generative_models import ( + GenerationConfig, + GenerativeModel, + Image, + ) +except ImportError as e: + GenerativeModel = e + + +class VertexAI(BaseBackend): + def __init__(self, model_name, safety_settings=None): + super().__init__() + + if isinstance(GenerativeModel, Exception): + raise GenerativeModel + + project_id = os.environ["GCP_PROJECT_ID"] + location = os.environ.get("GCP_LOCATION") + vertexai.init(project=project_id, location=location) + + self.model_name = model_name + self.chat_template = get_chat_template("default") + self.safety_settings = safety_settings + + def get_chat_template(self): + return self.chat_template + + def generate( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + ret = GenerativeModel(self.model_name).generate_content( + prompt, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + + comp = ret.text + + return comp, {} + + def generate_stream( + self, + s: StreamExecutor, + sampling_params: SglSamplingParams, + ): + if s.messages_: + prompt = self.messages_to_vertexai_input(s.messages_) + else: + # single-turn + prompt = ( + self.text_to_vertexai_input(s.text_, s.cur_images) + if s.cur_images + else s.text_ + ) + generator = GenerativeModel(self.model_name).generate_content( + prompt, + stream=True, + generation_config=GenerationConfig(**sampling_params.to_vertexai_kwargs()), + safety_settings=self.safety_settings, + ) + for ret in generator: + yield ret.text, {} + + def text_to_vertexai_input(self, text, images): + input = [] + # split with image token + text_segs = text.split(self.chat_template.image_token) + for image_path, image_base64_data in images: + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + input.append(Image.from_bytes(image_base64_data)) + text_seg = text_segs.pop(0) + if text_seg != "": + input.append(text_seg) + return input + + def messages_to_vertexai_input(self, messages): + vertexai_message = [] + # from openai message format to vertexai message format + for msg in messages: + if isinstance(msg["content"], str): + text = msg["content"] + else: + text = msg["content"][0]["text"] + + if msg["role"] == "system": + warnings.warn("Warning: system prompt is not supported in VertexAI.") + vertexai_message.append( + { + "role": "user", + "parts": [{"text": "System prompt: " + text}], + } + ) + vertexai_message.append( + { + "role": "model", + "parts": [{"text": "Understood."}], + } + ) + continue + if msg["role"] == "user": + vertexai_msg = { + "role": "user", + "parts": [{"text": text}], + } + elif msg["role"] == "assistant": + vertexai_msg = { + "role": "model", + "parts": [{"text": text}], + } + + # images + if isinstance(msg["content"], list) and len(msg["content"]) > 1: + for image in msg["content"][1:]: + assert image["type"] == "image_url" + vertexai_msg["parts"].append( + { + "inline_data": { + "data": image["image_url"]["url"].split(",")[1], + "mime_type": "image/jpeg", + } + } + ) + + vertexai_message.append(vertexai_msg) + return vertexai_message diff --git a/python/sglang/lang/chat_template.py b/python/sglang/lang/chat_template.py new file mode 100644 index 000000000..80ea6d963 --- /dev/null +++ b/python/sglang/lang/chat_template.py @@ -0,0 +1,662 @@ +import re +from dataclasses import dataclass +from enum import Enum, auto +from typing import Callable, Dict, List, Tuple + + +class ChatTemplateStyle(Enum): + PLAIN = auto() + LLAMA2 = auto() + + +@dataclass +class ChatTemplate: + name: str + default_system_prompt: str + role_prefix_and_suffix: Dict[str, Tuple[str, str]] + stop_str: List[str] = () + image_token: str = "" + audio_token: str = ""), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="llama-2-chat", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("<>\n", "\n<>\n\n"), + "user": ("[INST] ", " [/INST]"), + "assistant": ("", " "), + }, + style=ChatTemplateStyle.LLAMA2, + ) +) + +# Reference: https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503/blob/main/chat_template.json +register_chat_template( + ChatTemplate( + name="mistral", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("[SYSTEM_PROMPT] ", " [/SYSTEM_PROMPT]"), + "user": ("[INST] ", " [/INST]"), + "assistant": ("", " "), + }, + stop_str=("",), + image_token="[IMG]", + ) +) + +register_chat_template( + ChatTemplate( + name="llama-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="<|image|>", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-V-2_6 +register_chat_template( + ChatTemplate( + name="minicpmv", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + ) +) + +register_chat_template( + ChatTemplate( + name="janus-pro", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "User": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + image_token="\n", + ) +) + +# https://huggingface.co/openbmb/MiniCPM-o-2_6 +register_chat_template( + ChatTemplate( + name="minicpmo", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", " "), + "user": ("user:", " "), + "assistant": ("assistant:", ""), + }, + stop_str=("<|im_end|>", "<|endoftext|>"), + image_token="(./)", + audio_token="()", + ) +) + +register_chat_template( + ChatTemplate( + name="janus", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "user": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + image_token="\n", + ) +) + +# The difference between "llama-3-instruct-llava" and "llama-3-instruct" is that llava uses a different image_token. +register_chat_template( + ChatTemplate( + name="llama-3-instruct-llava", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_header_id|>system<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "user": ( + "<|start_header_id|>user<|end_header_id|>\n\n", + "<|eot_id|>", + ), + "assistant": ( + "<|start_header_id|>assistant<|end_header_id|>\n\n", + "<|eot_id|>", + ), + }, + stop_str=("<|eot_id|>",), + image_token="\n", + ) +) + +# Reference: https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct/blob/main/chat_template.json +register_chat_template( + ChatTemplate( + name="llama-4", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|header_start|>system<|header_end|>\n\n", + "<|eot|>", + ), + "user": ( + "<|header_start|>user<|header_end|>\n\n", + "<|eot|>", + ), + "assistant": ( + "<|header_start|>assistant<|header_end|>\n\n", + "<|eot|>", + ), + }, + stop_str=("<|eot|>",), + image_token="<|image|>", + ) +) + +# Reference: https://modelscope.cn/models/01ai/Yi-1.5-34B-Chat/file/view/master?fileName=tokenizer_config.json&status=1 +register_chat_template( + ChatTemplate( + name="yi-1.5", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("<|im_start|>user\n", "<|im_end|>\n<|im_start|>assistant\n"), + "assistant": ("", "<|im_end|>\n"), + }, + style=ChatTemplateStyle.PLAIN, + stop_str=("<|im_end|>",), + ) +) + +# Reference: https://github.com/01-ai/Yi/tree/main/VL#major-difference-with-llava +register_chat_template( + ChatTemplate( + name="yi-vl", + default_system_prompt=( + "This is a chat between an inquisitive human and an AI assistant. Assume the role of the AI assistant. Read all the images carefully, and respond to the human's questions with informative, helpful, detailed and polite answers." + "这是一个好奇的人类和一个人工智能助手之间的对话。假设你扮演这个AI助手的角色。仔细阅读所有的图像，并对人类的问题做出信息丰富、有帮助、详细的和礼貌的回答。" + ), + role_prefix_and_suffix={ + "system": ("", "\n\n"), + "user": ("### Human:", "\n"), + "assistant": ("### Assistant:", "\n"), + }, + image_token=" \n", + ) +) + +register_chat_template( + ChatTemplate( + name="gemma-it", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("", ""), + "user": ("user\n", "\n"), + "assistant": ("model\n", "\n"), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +register_chat_template( + ChatTemplate( + name="dbrx-instruct", + default_system_prompt="You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER'S QUERY.", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>"), + "user": ("\n<|im_start|>user\n", "<|im_end|>"), + "assistant": ("\n<|im_start|>assistant\n", "<|im_end|>"), + }, + stop_str=("<|im_end|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="c4ai-command-r", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + "user": ("<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", "<|END_OF_TURN_TOKEN|>"), + "assistant": ( + "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "<|END_OF_TURN_TOKEN|>", + ), + }, + style=ChatTemplateStyle.PLAIN, + ) +) + +# Adapted from https://huggingface.co/OpenGVLab/InternVL2-4B/blob/main/modeling_intern_vit.py +register_chat_template( + ChatTemplate( + name="internvl-2-5", + default_system_prompt="你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>\n"), + "user": ("<|im_start|>user\n", "<|im_end|>\n"), + "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"), + }, + stop_str=["<|im_end|>", "<|action_end|>"], + ) +) + +register_chat_template( + ChatTemplate( + name="interns1", + default_system_prompt="You are an AI assistant whose name is Intern-S1 (书生大模型).\n- Intern-S1 (书生大模型) is a vision-language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.\n- Intern-S1 (书生大模型) can understand and communicate fluently in the language chosen by the user such as English and 中文.\nYou are an expert reasoner with extensive experience in all areas. You approach problems through systematic thinking and rigorous reasoning. Your response should reflect deep understanding and precise logical thinking, making your solution path and reasoning clear to others. Please put your thinking process within ... tags.", + role_prefix_and_suffix={ + "system": ("<|im_start|>system\n", "<|im_end|>\n"), + "user": ("<|im_start|>user\n", "<|im_end|>\n"), + "assistant": ("<|im_start|>assistant\n", "<|im_end|>\n"), + }, + stop_str=["<|im_end|>", "<|action_end|>"], + ) +) + +register_chat_template( + ChatTemplate( + name="granite-3-instruct", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "<|start_of_role|>system<|end_of_role|>", + "<|end_of_text|>", + ), + "user": ( + "<|start_of_role|>user<|end_of_role|>", + "<|end_of_text|>", + ), + "assistant": ( + "<|start_of_role|>assistant<|end_of_role|>", + "<|end_of_text|>", + ), + }, + stop_str=("<|end_of_text|>",), + ) +) + +register_chat_template( + ChatTemplate( + name="deepseek-v3", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ( + "", + "", + ), + "user": ( + "<｜User｜>", + "", + ), + "assistant": ( + "<｜Assistant｜>", + "<｜end▁of▁sentence｜>", + ), + }, + stop_str=("<｜end▁of▁sentence｜>",), + ) +) + +# Reference: https://huggingface.co/docs/transformers/main/model_doc/glm4_v#usage-example +register_chat_template( + ChatTemplate( + name="glm-4v", + default_system_prompt=None, + role_prefix_and_suffix={ + "system": ("<|system|>\n", "\n"), + "user": ("<|user|>\n", "\n"), + "assistant": ("<|assistant|>\n", "\n"), + }, + style=ChatTemplateStyle.PLAIN, + stop_str=["<|user|>", "<|endoftext|>", "<|observation|>"], + image_token="<|image|>", + ) +) + + +@register_chat_template_matching_function +def match_deepseek(model_path: str): + if re.search(r"deepseek-(v3|r1)", model_path, re.IGNORECASE) and not re.search( + r"base", model_path, re.IGNORECASE + ): + return "deepseek-v3" + + +@register_chat_template_matching_function +def match_deepseek_janus_pro(model_path: str): + if re.search(r"janus", model_path, re.IGNORECASE): + return "janus-pro" + + +@register_chat_template_matching_function +def match_dbrx(model_path: str): + if re.search(r"dbrx", model_path, re.IGNORECASE) and re.search( + r"instruct", model_path, re.IGNORECASE + ): + return "dbrx-instruct" + + +@register_chat_template_matching_function +def match_vicuna(model_path: str): + if re.search(r"vicuna|llava-v1\.5|llava-next-video-7b", model_path, re.IGNORECASE): + return "vicuna_v1.1" + + +@register_chat_template_matching_function +def match_llama2_chat(model_path: str): + if re.search( + r"llama-2.*chat|codellama.*instruct", + model_path, + re.IGNORECASE, + ): + return "llama-2-chat" + + +@register_chat_template_matching_function +def match_mistral(model_path: str): + if re.search(r"pixtral|(mistral|mixtral).*instruct", model_path, re.IGNORECASE): + return "mistral" + + +@register_chat_template_matching_function +def match_llama3_instruct(model_path: str): + if re.search(r"llama-3.*instruct", model_path, re.IGNORECASE): + return "llama-3-instruct" + + +@register_chat_template_matching_function +def match_chat_ml(model_path: str): + if re.search(r"tinyllama", model_path, re.IGNORECASE): + return "chatml" + if re.search(r"qwen.*vl", model_path, re.IGNORECASE): + return "qwen2-vl" + if re.search(r"glm[-_]?4(\.\d+)?v", model_path, re.IGNORECASE): + return "glm-4v" + if re.search(r"qwen.*(chat|instruct)", model_path, re.IGNORECASE) and not re.search( + r"llava", model_path, re.IGNORECASE + ): + return "qwen" + if re.search( + r"llava-v1\.6-34b|llava-v1\.6-yi-34b|llava-next-video-34b|llava-onevision-qwen2", + model_path, + re.IGNORECASE, + ): + return "chatml-llava" + + +@register_chat_template_matching_function +def match_chat_yi(model_path: str): + if re.search(r"yi-vl", model_path, re.IGNORECASE) and not re.search( + r"llava", model_path, re.IGNORECASE + ): + return "yi-vl" + elif re.search(r"yi-1\.5.*chat", model_path, re.IGNORECASE): + return "yi-1.5" + + +@register_chat_template_matching_function +def match_gemma_it(model_path: str): + if re.search(r"gemma.*it", model_path, re.IGNORECASE): + return "gemma-it" + + +@register_chat_template_matching_function +def match_openbmb_minicpm(model_path: str): + if re.search(r"minicpm-v", model_path, re.IGNORECASE): + return "minicpmv" + elif re.search(r"minicpm-o", model_path, re.IGNORECASE): + return "minicpmo" + + +@register_chat_template_matching_function +def match_c4ai_command_r(model_path: str): + if re.search(r"c4ai-command-r", model_path, re.IGNORECASE): + return "c4ai-command-r" + + +@register_chat_template_matching_function +def match_granite_instruct(model_path: str): + if re.search(r"granite.*instruct", model_path, re.IGNORECASE): + return "granite-3-instruct" + + +@register_chat_template_matching_function +def match_gemma3_instruct(model_path: str): + if re.search(r"gemma-3", model_path, re.IGNORECASE): + return "gemma-it" + + +@register_chat_template_matching_function +def match_internvl_chat(model_path: str): + if re.search(r"internvl2_5", model_path, re.IGNORECASE): + return "internvl-2-5" + + +@register_chat_template_matching_function +def match_interns1_chat(model_path: str): + if re.search(r"intern-s1", model_path, re.IGNORECASE): + return "interns1" + if re.search(r"interns1", model_path, re.IGNORECASE): + return "interns1" + + +if __name__ == "__main__": + messages = [ + {"role": "system", "content": None}, # None means default + # {"role": "system", "content": "You are a helpful, respectful and honest assistant."}, + {"role": "user", "content": "Hello!"}, + {"role": "assistant", "content": "Hi!"}, + {"role": "user", "content": "What can you do?"}, + {"role": "assistant", "content": "I can chat with you."}, + ] + + template = get_chat_template("llama-2-chat") + print(template.get_prompt(messages)) diff --git a/python/sglang/lang/choices.py b/python/sglang/lang/choices.py new file mode 100644 index 000000000..e52c6b362 --- /dev/null +++ b/python/sglang/lang/choices.py @@ -0,0 +1,164 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import numpy as np + + +@dataclass +class ChoicesDecision: + decision: str + meta_info: Optional[Dict[str, Any]] = None + + +class ChoicesSamplingMethod(ABC): + + @property + def requires_unconditional_logprobs(self) -> bool: + return False + + @abstractmethod + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: ... + + +class TokenLengthNormalized(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest token length normalized prompt logprob.""" + best_choice = choices[np.argmax(normalized_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + +token_length_normalized = TokenLengthNormalized() + + +class GreedyTokenSelection(ChoicesSamplingMethod): + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option based on greedy logprob selection. For overlapping options + where one option is a subset of a longer option, extend the shorter option using + its average logprob for comparison against the longer option.""" + + num_options = len(choices) + max_tokens = max(len(option) for option in input_token_logprobs) + logprob_matrix = self._build_logprob_matrix( + input_token_logprobs, max_tokens, num_options + ) + remaining = self._greedy_selection(logprob_matrix, num_options, max_tokens) + + best_choice = choices[remaining[0]] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "greedy_logprob_matrix": logprob_matrix.tolist(), + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _build_logprob_matrix(self, input_token_logprobs, max_tokens, num_options): + logprob_matrix = np.zeros((num_options, max_tokens)) + for i, option in enumerate(input_token_logprobs): + actual_logprobs = [token[0] for token in option] + avg_logprob = np.mean(actual_logprobs) + logprob_matrix[i, : len(option)] = actual_logprobs + if len(option) < max_tokens: + logprob_matrix[i, len(option) :] = avg_logprob + return logprob_matrix + + def _greedy_selection(self, logprob_matrix, num_options, max_tokens): + remaining = np.arange(num_options) + for j in range(max_tokens): + max_logprob = np.max(logprob_matrix[remaining, j]) + remaining = remaining[logprob_matrix[remaining, j] == max_logprob] + if len(remaining) == 1: + break + return remaining + + +greedy_token_selection = GreedyTokenSelection() + + +class UnconditionalLikelihoodNormalized(ChoicesSamplingMethod): + + @property + def requires_unconditional_logprobs(self) -> bool: + return True + + def __call__( + self, + *, + choices: List[str], + normalized_prompt_logprobs: List[float], + input_token_logprobs: List[List[Any]], + output_token_logprobs: List[List[Any]], + unconditional_token_logprobs: Optional[List[List[Any]]] = None, + ) -> ChoicesDecision: + """Select the option with the highest average token logprob once normalized by + the unconditional token logprobs. + + The first unconditional token logprob is assumed to be None. If so, it is + replaced with 0 for the purposes of normalization.""" + + if unconditional_token_logprobs is None: + raise ValueError( + "Unconditional token logprobs are required for this method." + ) + + normalized_unconditional_prompt_logprobs = self._normalize_logprobs( + input_token_logprobs, unconditional_token_logprobs + ) + + best_choice = choices[np.argmax(normalized_unconditional_prompt_logprobs)] + meta_info = { + "normalized_prompt_logprobs": normalized_prompt_logprobs, + "input_token_logprobs": input_token_logprobs, + "output_token_logprobs": output_token_logprobs, + "unconditional_token_logprobs": unconditional_token_logprobs, + "normalized_unconditional_prompt_logprobs": normalized_unconditional_prompt_logprobs, + } + return ChoicesDecision(decision=best_choice, meta_info=meta_info) + + def _normalize_logprobs(self, input_token_logprobs, unconditional_token_logprobs): + normalized_unconditional_prompt_logprobs = [] + for inputs, unconditionals in zip( + input_token_logprobs, unconditional_token_logprobs + ): + inputs_logprobs = np.array([token[0] for token in inputs]) + unconditionals_logprobs = np.array([token[0] for token in unconditionals]) + unconditionals_logprobs[0] = unconditionals_logprobs[0] or 0 + normalized_unconditional_prompt_logprobs.append( + float(np.mean(inputs_logprobs - unconditionals_logprobs)) + ) + return normalized_unconditional_prompt_logprobs + + +unconditional_likelihood_normalized = UnconditionalLikelihoodNormalized() diff --git a/python/sglang/lang/compiler.py b/python/sglang/lang/compiler.py new file mode 100644 index 000000000..1284232f7 --- /dev/null +++ b/python/sglang/lang/compiler.py @@ -0,0 +1,231 @@ +import multiprocessing +from concurrent.futures import ThreadPoolExecutor +from queue import Queue +from typing import List, Union + +from sglang.global_config import global_config +from sglang.lang.interpreter import ProgramState, StreamExecutor, cache_program +from sglang.lang.ir import SglArgument, SglExpr, SglSamplingParams, SglVariable + + +def compile_func(function, backend): + tracer = function.trace(backend=backend) + compiler = CompiledFunction(tracer, function) + return compiler + + +class CompiledFunction: + def __init__(self, tracer, function): + self.function = function + + self.last_node = CompGraphNode(tracer.last_node) + self.expr_to_node = {} + self.build_graph(tracer) + self.topological_sort() + + def build_graph(self, tracer): + self.nodes = [self.last_node] + self.expr_to_node[tracer.last_node] = self.nodes[-1] + + rename_pid = {} + + visited = set([tracer.last_node]) + head = 0 + while head < len(self.nodes): + cur_node = self.nodes[head] + + # add prev node + prev_node = cur_node.expr.prev_node + if prev_node is not None: + if prev_node not in visited: + visited.add(prev_node) + self.nodes.append(CompGraphNode(prev_node)) + self.expr_to_node[prev_node] = self.nodes[-1] + cur_node.prev_node = self.expr_to_node[prev_node] + self.expr_to_node[prev_node].add_next_node(cur_node) + + # add source node + if isinstance(cur_node.expr, SglVariable): + if cur_node.expr.name in tracer.variables: + source = tracer.variables[cur_node.expr.name].source + else: + source = cur_node.expr.source + if source not in visited: + visited.add(source) + self.nodes.append(CompGraphNode(source)) + self.expr_to_node[source] = self.nodes[-1] + cur_node.source_node = self.expr_to_node[source] + self.expr_to_node[source].add_next_node(cur_node) + head += 1 + + # rename pid + if cur_node.expr.pid not in rename_pid: + rename_pid[cur_node.expr.pid] = len(rename_pid) + cur_node.expr.pid = rename_pid[cur_node.expr.pid] + + def topological_sort(self): + prevd = {} + cand = Queue() + for x in self.nodes: + prevd[x] = (x.prev_node is not None) + (x.source_node is not None) + if prevd[x] == 0: + cand.put(x) + new_list = [] + while cand.qsize() > 0: + head = cand.get() + new_list.append(head) + for x in head.next_nodes: + prevd[x] -= 1 + if prevd[x] == 0: + cand.put(x) + self.nodes = new_list + + def print_graph( + self, + ): + for node in self.nodes: + print(node) + + def run_internal( + self, + backend, + kwargs, + default_sampling_para, + ): + stream_executor_ids = set([x.expr.pid for x in self.nodes]) + stream_executors = {} + for x in stream_executor_ids: + arguments = kwargs if x == self.last_node.expr.pid else {} + stream_executors[x] = StreamExecutor( + backend, arguments, default_sampling_para, None, False + ) + for node in self.nodes: + se_id = node.expr.pid + expr = node.expr + if isinstance(expr, SglVariable): + # Make a copy for SglVariable + expr = SglVariable(expr.name, expr.source) + expr.source_stream_executor = stream_executors[ + node.source_node.expr.pid + ] + elif isinstance(expr, SglArgument): + # Substitute SglArgument + expr = kwargs[expr.name] + stream_executors[se_id].submit(expr) + for stream_executor in stream_executors.values(): + stream_executor.end() + return ProgramState(stream_executors[self.last_node.expr.pid]) + + def run( + self, + *, + max_new_tokens: int = 128, + stop: Union[str, List[str]] = (), + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + backend=None, + **kwargs, + ): + backend = backend or global_config.default_backend + + kwargs.update(self.function.bind_arguments) + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + stop=stop, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ) + + return self.run_internal(backend, kwargs, default_sampling_para) + + def run_batch( + self, + batch_kwargs, + *, + max_new_tokens: int = 128, + stop: Union[str, List[str]] = (), + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + backend=None, + num_threads: Union[str, int] = "auto", + ): + assert isinstance(batch_kwargs, (list, tuple)) + if len(batch_kwargs) == 0: + return [] + assert isinstance(batch_kwargs[0], dict) + + backend = backend or global_config.default_backend + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + stop=stop, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ) + + # Extract prefix by tracing and cache it + if len(batch_kwargs) > 1: + cache_program(self.function, backend) + + # Run all programs + if num_threads == "auto": + num_threads = multiprocessing.cpu_count() + num_threads = min(num_threads, len(batch_kwargs)) + + if num_threads == 1: + rets = [] + for arguments in batch_kwargs: + rets.append( + self.run_internal(backend, arguments, default_sampling_para) + ) + else: + with ThreadPoolExecutor(num_threads) as executor: + futures = [] + for arguments in batch_kwargs: + futures.append( + executor.submit( + self.run_internal, backend, arguments, default_sampling_para + ) + ) + rets = [f.result() for f in futures] + rets[-1].sync() + + return rets + + +class CompGraphNode: + def __init__( + self, expr: SglExpr, prev_node=None, next_nodes=None, source_node=None + ): + self.expr = expr + self.next_nodes = next_nodes or [] + self.prev_node = prev_node + self.source_node = source_node + + def add_next_node(self, other): + self.next_nodes.append(other) + + def __repr__(self): + re = f"stream {self.expr.pid:2d}: " + re += f"%{self.expr.node_id} = " + if self.prev_node is not None: + re += f"%{self.prev_node.expr.node_id} + " + re += repr(self.expr) + return re diff --git a/python/sglang/lang/interpreter.py b/python/sglang/lang/interpreter.py new file mode 100644 index 000000000..8b8cdf9c5 --- /dev/null +++ b/python/sglang/lang/interpreter.py @@ -0,0 +1,1060 @@ +"""The interpreter that executes SGL programs""" + +import asyncio +import contextvars +import copy +import multiprocessing +import queue +import threading +import uuid +import warnings +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from typing import Any, Callable, Dict, List, Optional + +import tqdm + +from sglang.global_config import global_config +from sglang.lang.ir import ( + SglCommitLazy, + SglConcateAndAppend, + SglConstantText, + SglExpr, + SglExprList, + SglGen, + SglImage, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglSeparateReasoning, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, + SglVideo, +) +from sglang.utils import ( + encode_image_base64, + encode_video_base64, + get_exception_traceback, +) + + +def run_internal(state, program, func_args, func_kwargs, sync): + try: + state.ret_value = program.func(state, *func_args, **func_kwargs) + except Exception as e: + raise e + finally: + state.stream_executor.end() + + if sync: + state.stream_executor.sync() + + if global_config.verbosity >= 2: + print(state.text()) + + +def run_program( + program, + backend, + func_args, + func_kwargs, + default_sampling_para, + stream, + sync=False, + use_thread=True, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + assert backend is not None, "Please specify a backend" + func_kwargs.update(program.bind_arguments) + stream_executor = StreamExecutor( + backend, + func_kwargs, + default_sampling_para, + chat_template=None, + stream=stream, + num_api_spec_tokens=program.num_api_spec_tokens, + use_thread=use_thread, + ) + state = ProgramState(stream_executor) + + if stream: + t = threading.Thread( + target=run_internal, args=(state, program, func_args, func_kwargs, sync) + ) + t.start() + return state + else: + run_internal(state, program, func_args, func_kwargs, sync) + return state + + +def run_program_batch( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + generator_style=False, +): + if hasattr(backend, "endpoint"): + backend = backend.endpoint + + # Pre-cache the common prefix for a batch. The prefix is extracted by tracing the program. + if global_config.enable_precache_with_tracing and len(batch_arguments) > 1: + cache_program(program, backend) + + # Run all programs + if num_threads == "auto": + num_threads = max(96, multiprocessing.cpu_count() * 16) + num_threads = min(num_threads, len(batch_arguments)) + + if generator_style: + return _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, + ) + + # Original code path when generator_style=False + if num_threads == 1: + rets = [] + if progress_bar: + for arguments in tqdm.tqdm(batch_arguments): + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + for arguments in batch_arguments: + rets.append( + run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + else: + if progress_bar: + pbar = tqdm.tqdm(total=len(batch_arguments)) + + with ThreadPoolExecutor(num_threads) as executor: + futures = [] + for arguments in batch_arguments: + futures.append( + executor.submit( + run_program, + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + ) + if progress_bar: + futures[-1].add_done_callback(lambda _: pbar.update()) + + rets = [f.result() for f in futures] + rets[-1].sync() + + if progress_bar: + pbar.close() + + return rets + + +def _run_program_batch_generator( + program, + backend, + batch_arguments, + default_sampling_para, + num_threads, + progress_bar, +): + """Helper function that yields results one by one using chunking to avoid overwhelming ThreadPoolExecutor.""" + if num_threads == 1: + iterator = tqdm.tqdm(batch_arguments) if progress_bar else batch_arguments + for arguments in iterator: + yield run_program( + program, + backend, + (), + arguments, + default_sampling_para, + False, + True, + ) + else: + pbar = tqdm.tqdm(total=len(batch_arguments)) if progress_bar else None + + # Process in chunks to avoid overwhelming ThreadPoolExecutor + # Otherwise, ThreadPoolExecutor.submit will block after adding certain number of tasks + # so we will never reach "yield" until all tasks are done + chunk_size = 200 + + with ThreadPoolExecutor(num_threads) as executor: + for chunk_start in range(0, len(batch_arguments), chunk_size): + chunk_end = min(chunk_start + chunk_size, len(batch_arguments)) + chunk_futures = [] + + # Submit chunk of tasks + for i in range(chunk_start, chunk_end): + future = executor.submit( + run_program, + program, + backend, + (), + batch_arguments[i], + default_sampling_para, + False, + True, + ) + if pbar: + future.add_done_callback(lambda _: pbar.update()) + chunk_futures.append(future) + + # Yield results from this chunk as they complete + for future in chunk_futures: + yield future.result() + + if pbar: + pbar.close() + + +def cache_program(program, backend): + from sglang.lang.tracer import extract_prefix_by_tracing + + prefix = extract_prefix_by_tracing(program, backend) + if prefix and len(prefix) > 64: + backend.cache_prefix(prefix) + + +class StreamExecutor: + """A stream executor that executes SGL expressions in a background thread.""" + + def __init__( + self, + backend, + arguments, + default_sampling_para, + chat_template, + stream, + num_api_spec_tokens=None, + use_thread=True, + ): + from sglang.lang.backend.base_backend import BaseBackend + + self.sid = uuid.uuid4().hex + self.backend: BaseBackend = backend + self.arguments: Dict[str, Any] = arguments + self.default_sampling_para = default_sampling_para + self.stream = stream + + self.variables = {} # Dict[name: str -> value: str] + self.variable_event = {} # Dict[name: str -> event: threading.Event] + self.meta_info = {} # Dict[name: str -> info: str] + self.is_finished = False + self.error_ = None + + # For completion + self.text_ = "" # The full text + + # For chat + self.messages_ = [] # The messages in the OpenAI API format + self.chat_template = chat_template or self.backend.get_chat_template() + self.cur_role = None + self.cur_role_begin_pos = None + + # For vision + self.images_ = [] + self.cur_images = [] + + # For fork/join + self.fork_start_text_pos = None + + # For speculative execution + self.num_api_spec_tokens = num_api_spec_tokens + self.speculated_text = "" + + # Worker thread + self.use_thread = use_thread + if self.use_thread: + self.queue = queue.Queue() + + def _run_worker_in_context(): + self._thread_worker_func() + + self.worker = threading.Thread( + target=contextvars.copy_context().run, args=(_run_worker_in_context,) + ) + self.worker.start() + + # For streaming + if stream: + self.stream_text_event = threading.Event() + self.stream_var_event = {} + else: + self.stream_text_event = None + self.stream_var_event = None + + def submit(self, expr: SglExpr): + self._init_var_event(expr) + + if self.use_thread: + self.queue.put(expr) + else: + self._execute(expr) + + def sync(self): + if self.use_thread: + self.queue.join() + + def get_var(self, name): + if name in self.variable_event: + self.variable_event[name].wait() + return self.variables[name] + + def set_var(self, name, value): + self.variables[name] = value + + def get_meta_info(self, name, timeout=None): + if name in self.variable_event: + got = self.variable_event[name].wait(timeout) + if not got: + raise TimeoutError(f"Timeout while waiting for event '{name}'") + ret = self.meta_info.get(name, None) + return ret + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + if size > 1 and str(self.text_): + self.submit(SglCommitLazy()) + + self.sync() + size = int(size) + + exes = [ + StreamExecutor( + self.backend, + self.arguments, + self.default_sampling_para, + self.chat_template, + self.stream, + ) + for _ in range(size) + ] + for i in range(size): + exes[i].variables = dict(self.variables) + exes[i].text_ = str(self.text_) + exes[i].messages_ = list(self.messages_) + exes[i].cur_role = self.cur_role + exes[i].cur_role_begin_pos = self.cur_role_begin_pos + exes[i].fork_start_text_pos = len(self.text_) + exes[i].images_ = list(self.images_) + + # TODO(ying): handle API speculative execution + + return exes + + def text(self): + self.sync() + return self.text_ + + def messages(self): + self.sync() + return self.messages_ + + def error(self): + self.sync() + return self.error_ + + def end(self): + if self.use_thread: + if self.worker.is_alive(): + self.queue.put(None) + self.backend.end_program(self) + + def _thread_worker_func(self): + error = None + + while True: + expr = self.queue.get() + if expr is None: + self.queue.task_done() + break + + try: + self._execute(expr) + except Exception as e: + warnings.warn(f"Error in stream_executor: {get_exception_traceback()}") + error = e + break + self.queue.task_done() + if self.stream_text_event: + self.stream_text_event.set() + + # Clean the queue and events + if error is not None: + try: + while True: + self.queue.task_done() + self.queue.get_nowait() + except queue.Empty: + pass + for name in self.variable_event: + self.variable_event[name].set() + if self.stream_var_event: + for name in self.stream_var_event: + self.stream_var_event[name].set() + self.error_ = error + + if self.stream_text_event: + self.stream_text_event.set() + + self.is_finished = True + + def _execute(self, other): + if isinstance(other, str): + other = SglConstantText(other) + + assert isinstance(other, SglExpr), f"{other}" + + if isinstance(other, SglConstantText): + self._execute_fill(other.value) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglImage): + self._execute_image(other) + elif isinstance(other, SglVideo): + self._execute_video(other) + elif isinstance(other, SglVariable): + self._execute_variable(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + elif isinstance(other, SglCommitLazy): + self._execute_commit_lazy_operations(other) + elif isinstance(other, SglConcateAndAppend): + if ( + global_config.enable_parallel_encoding + and self.backend.support_concate_and_append + ): + self._execute_concatenate_and_append_kv_cache(other) + else: + self._execute_concatenate_and_append_text(other) + elif isinstance(other, SglSeparateReasoning): + self._execute_separate_reasoning(other) + else: + raise ValueError(f"Unknown type: {type(other)}") + + def _execute_fill(self, value: str, prefix=False): + value = str(value) + + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + and not prefix + ): + self.backend.spec_fill(value) + return + + if self.speculated_text.startswith(value): + self.speculated_text = self.speculated_text[len(value) :] + else: + self.speculated_text = "" + + self.text_ += value + + def _execute_image(self, expr: SglImage): + path = expr.path + + base64_data = encode_image_base64(path) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _execute_video(self, expr: SglVideo): + path = expr.path + num_frames = expr.num_frames + + base64_data = encode_video_base64(path, num_frames) + + self.images_.append((path, base64_data)) + self.cur_images.append((path, base64_data)) + self.text_ += self.chat_template.image_token + + def _spec_gen(self, sampling_params): + stop = sampling_params.stop + max_new_tokens = sampling_params.max_new_tokens + meta_info = {} + + def regen(): + nonlocal meta_info + + sampling_params.max_new_tokens = max( + sampling_params.max_new_tokens, self.num_api_spec_tokens + ) + sampling_params.stop = None + self.speculated_text, meta_info = self.backend.generate( + self, sampling_params=sampling_params + ) + + def find_stop(): + if isinstance(stop, str): + return self.speculated_text.find(stop) + elif isinstance(stop, (tuple, list)): + pos = -1 + for stop_str in stop: + stop_pos = self.speculated_text.find(stop_str) + if stop_pos != -1 and (pos == -1 or stop_pos < pos): + pos = stop_pos + return pos + else: + raise Exception("Wrong type of stop in sampling parameters.") + + if stop is None: + if len(self.speculated_text) < max_new_tokens: + regen() + comp = self.speculated_text[:max_new_tokens] + self.speculated_text = self.speculated_text[max_new_tokens:] + elif isinstance(stop, (str, list, tuple)): + if self.speculated_text == "": + regen() + stop_pos = find_stop() + if stop_pos == -1: + stop_pos = min( + sampling_params.max_new_tokens, + len(self.speculated_text), + ) + comp = self.speculated_text[:stop_pos] + self.speculated_text = self.speculated_text[stop_pos:] + else: + raise ValueError("Wrong type of stop in sampling parameters.") + + return comp, meta_info + + def _execute_gen(self, expr: SglGen): + sampling_params = self._resolve_sampling_params(expr.sampling_params) + name = expr.name + if not self.stream: + if self.num_api_spec_tokens is None: + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + ) + + else: + if self.backend.is_chat_model: + # Speculative execution on models with only chat interface. + # Store the calls into a temporary list. + # They will be lazily executed later. + comp, meta_info = self.backend.generate( + self, + sampling_params=sampling_params, + spec_var_name=name, + ) + return + + else: # Speculative execution on models with completion interface + comp, meta_info = self._spec_gen(sampling_params) + if isinstance(comp, list): + self.text_ += comp[0] + else: + assert isinstance(comp, str) + self.text_ += comp + + self.variables[name] = comp + self.meta_info[name] = meta_info + self.variable_event[name].set() + else: + assert ( + self.num_api_spec_tokens is None + ), "stream is not supported with api speculative execution" + generator = self.backend.generate_stream( + self, sampling_params=sampling_params + ) + + self.variables[name] = "" + self.stream_var_event[name].set() + + for comp, meta_info in generator: + self.text_ += comp + self.variables[name] += comp + self.meta_info[name] = meta_info + self.stream_var_event[name].set() + self.stream_text_event.set() + + self.variable_event[name].set() + self.stream_var_event[name].set() + + def _execute_select(self, expr: SglSelect): + choices_decision = self.backend.select( + self, expr.choices, expr.temperature, expr.choices_method + ) + if expr.name is not None: + name = expr.name + self.variables[name] = choices_decision.decision + self.meta_info[name] = choices_decision.meta_info + self.variable_event[name].set() + if self.stream_var_event: + self.stream_var_event[name].set() + self.text_ += choices_decision.decision + + def _execute_variable(self, expr: SglVariable): + src_executor = expr.source_stream_executor + value = src_executor.get_var(expr.name) + self._execute_fill(value) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert the default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, _ = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + + self._execute_fill(prefix, prefix=True) + self.cur_role_begin_pos = len(self.text_) + + def _execute_role_end(self, expr: SglRoleEnd): + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + ): + # Execute the stored lazy generation calls + self.backend.role_end_generate(self) + self.cur_role = None + + new_text = self.text_[self.cur_role_begin_pos :].lstrip() + + _, suffix = self.chat_template.get_prefix_and_suffix(expr.role, self.messages_) + self._execute_fill(suffix) + + if self.cur_images: + # OpenAI vision API format + last_msg = { + "role": expr.role, + "content": [{"type": "text", "text": new_text}], + } + for image_path, image_base64_data in self.cur_images: + last_msg["content"].append( + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image_base64_data}" + }, + } + ) + self.messages_.append(last_msg) + self.cur_images = [] + else: + # OpenAI chat API format + self.messages_.append({"role": expr.role, "content": new_text}) + + def _execute_var_scope_begin(self, expr: SglVarScopeBegin): + self.variables[expr.name] = int(len(self.text_)) + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + self.variables[expr.name] = self.text_[self.variables[expr.name] :] + self.variable_event[expr.name].set() + + def _execute_commit_lazy_operations(self, expr: SglCommitLazy): + self.backend.commit_lazy_operations(self) + + def _execute_concatenate_and_append_text(self, expr: SglConcateAndAppend): + new_text = "" + for s in expr.states: + exe = s.stream_executor + exe.sync() + new_text += exe.text_[exe.fork_start_text_pos :] + + self._execute_fill(new_text) + + def _execute_concatenate_and_append_kv_cache(self, expr: SglConcateAndAppend): + self_len = len(self.text_) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.submit(SglCommitLazy()) + + for i, s in enumerate(expr.states): + exe = s.stream_executor + exe.sync() + assert exe.fork_start_text_pos == self_len + self.text_ += exe.text_[exe.fork_start_text_pos :] + + src_rids = [state.stream_executor.sid for state in expr.states] + self.backend.concatenate_and_append(src_rids, self.sid) + + def _execute_separate_reasoning(self, expr: SglSeparateReasoning): + if self.stream: + # separate reasoning for stream is not supported + return + + if ( + self.cur_role == "assistant" + and self.num_api_spec_tokens is not None + and self.backend.is_chat_model + ): + # Execute the stored lazy generation calls + self.backend.role_end_generate(self) + + from sglang.srt.parser.reasoning_parser import ReasoningParser + + reasoning_parser = ReasoningParser(expr.model_type) + other = expr.expr + if not other: + return + elif isinstance(other, SglGen) or isinstance(other, SglSelect): + cur_text = self.get_var(other.name) + reasoning, normal_text = reasoning_parser.parse_non_stream(cur_text) + reasoning_name = expr.process_name_for_reasoning(other.name) + self.set_var(other.name, normal_text) + self.set_var(reasoning_name, reasoning) + # the variable is ready to be used + self.variable_event[reasoning_name].set() + self.text_ = self.text_[: self.cur_role_begin_pos] + normal_text + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute_separate_reasoning( + SglSeparateReasoning(expr.model_type, x) + ) + + def _init_var_event(self, expr): + if isinstance( + expr, (SglGen, SglSelect, SglVarScopeBegin, SglSeparateReasoning) + ): + self.variable_event[expr.name] = threading.Event() + if self.stream: + self.stream_var_event[expr.name] = threading.Event() + elif isinstance(expr, SglExprList): + for e in expr.expr_list: + self._init_var_event(e) + + def _resolve_sampling_params(self, sampling_params): + """ + Construct sampling param based on default + override values + + The default values of sampling are populated in `default_sampling_para` via sgl.function.run(...sampling_args) + , and `sampling_params` contains the override values from sgl.gen(). + + Here we use default_sampling_para as the base and override the values if they exist in `sampling_params`. + It also extends the stop tokens based on the chat template. + """ + + # deepcopy is required because the dict has lists inside + clone = copy.deepcopy(self.default_sampling_para) + + for item in [ + "max_new_tokens", + "min_new_tokens", + "n", + "stop", + "stop_token_ids", + "temperature", + "top_p", + "top_k", + "min_p", + "frequency_penalty", + "presence_penalty", + "ignore_eos", + "return_logprob", + "logprob_start_len", + "top_logprobs_num", + "return_text_in_logprobs", + "dtype", + "regex", + "json_schema", + ]: + value = getattr(sampling_params, item, None) + if value is not None: + setattr(clone, item, value) + + if self.chat_template.stop_str: + if clone.stop == (): + clone.stop = [] + elif isinstance(clone.stop, str): + clone.stop = [clone.stop] + clone.stop += self.chat_template.stop_str + + return clone + + def __del__(self): + self.end() + + +class ProgramState: + """The state of an SGL program.""" + + def __init__(self, stream_executor: StreamExecutor): + self.stream_executor = stream_executor + + def _role_common(self, name: str, expr: Optional[SglExpr] = None): + if expr is not None: + role_expr = SglExprList([SglRoleBegin(name), expr, SglRoleEnd(name)]) + self.stream_executor.submit(role_expr) + return role_expr + else: + + @contextmanager + def role_scope(): + self.stream_executor.submit(SglRoleBegin(name)) + yield + self.stream_executor.submit(SglRoleEnd(name)) + + return role_scope() + + def system(self, expr: Optional[SglExpr] = None): + return self._role_common("system", expr) + + def user(self, expr: Optional[SglExpr] = None): + return self._role_common("user", expr) + + def assistant(self, expr: Optional[SglExpr] = None): + return self._role_common("assistant", expr) + + @contextmanager + def var_scope(self, name: str): + self.stream_executor.submit(SglVarScopeBegin(name)) + yield + self.stream_executor.submit(SglVarScopeEnd(name)) + + def fork( + self, + size: int = 1, + position_ids_offset: Optional[List[int]] = None, + ): + stream_executors = self.stream_executor.fork(size, position_ids_offset) + states = [ProgramState(x) for x in stream_executors] + state_group = ProgramStateGroup(states, self) + return state_group + + @contextmanager + def copy(self, position_ids_offset: Optional[List[int]] = None): + state_group = self.fork(1, position_ids_offset) + try: + yield state_group[0] + finally: + state_group.join() + + def text(self): + return self.stream_executor.text() + + def messages(self): + return self.stream_executor.messages() + + def sync(self): + return self.stream_executor.sync() + + def error(self): + return self.stream_executor.error() + + def text_iter(self, var_name: Optional[str] = None): + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + event.wait() + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + event.wait() + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + async def text_async_iter( + self, var_name: Optional[str] = None, return_meta_data: bool = False + ): + loop = asyncio.get_running_loop() + + if self.stream_executor.stream: + prev = 0 + if var_name is None: + event = self.stream_executor.stream_text_event + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.text_[prev:]) + prev += len(out) + if out: + yield out + if self.stream_executor.is_finished: + break + else: + event = None + while not event: + if var_name in self.stream_executor.stream_var_event: + event = self.stream_executor.stream_var_event[var_name] + if self.stream_executor.is_finished: + yield "" + return + + while True: + await loop.run_in_executor(None, event.wait) + event.clear() + out = str(self.stream_executor.variables[var_name][prev:]) + prev += len(out) + if out: + if return_meta_data: + yield out, self.stream_executor.meta_info[var_name] + else: + yield out + if self.stream_executor.variable_event[var_name].is_set(): + break + else: + if var_name is None: + yield self.text() + else: + yield self.get_var(var_name) + + def get_var(self, name): + return self.stream_executor.get_var(name) + + def set_var(self, name, value): + return self.stream_executor.set_var(name, value) + + def get_meta_info(self, name): + return self.stream_executor.get_meta_info(name) + + def __iadd__(self, other): + if other is None: + raise ValueError("Tried to append None to state.") + self.stream_executor.submit(other) + return self + + def __getitem__(self, name): + return self.get_var(name) + + def __setitem__(self, name, value): + self.set_var(name, value) + + def __contains__(self, name): + return name in self.stream_executor.variables + + def __del__(self): + self.stream_executor.end() + + def __repr__(self) -> str: + return f"ProgramState({self.text()})" + + +class ProgramStateGroup: + def __init__( + self, states: List[ProgramState], src_state: Optional[ProgramState] = None + ): + self.states = states + self.src_state = src_state + + def join(self, mode: str = "gather_variable"): + if mode == "gather_variable": + # Copy variables back + src_vars = self.src_state.stream_executor.variables + src_var_set = set(src_vars.keys()) + for child_state in self.states: + child_state.stream_executor.sync() + child_vars = child_state.stream_executor.variables + new_vars = set(child_vars.keys()) - src_var_set + + for k in new_vars: + if k in src_vars: + src_vars[k].append(child_vars[k]) + else: + src_vars[k] = [child_vars[k]] + elif mode == "concate_and_append": + # Concatenate and append KV cache + self.src_state += SglConcateAndAppend(self.states) + # Need a sync here. Otherwise, `states` can be deleted. + self.src_state.stream_executor.sync() + else: + raise ValueError(f"Invalid join mode: {mode}") + + for s in self.states: + s.stream_executor.end() + + def __getitem__(self, i: int): + return self.states[i] + + def __setitem__(self, i: int, value): + assert self.states[i] == value + + def __iadd__(self, other): + if isinstance(other, Callable): + # lambda function + for i in range(len(self.states)): + self.states[i] += other(i) + elif isinstance(other, SglExpr): + for i in range(len(self.states)): + self.states[i] += other + elif isinstance(other, (list, tuple)): + for i in range(len(self.states)): + self.states[i] += other[i] + else: + raise ValueError(f"Invalid value: {other}") + + return self diff --git a/python/sglang/lang/ir.py b/python/sglang/lang/ir.py new file mode 100644 index 000000000..531705ebe --- /dev/null +++ b/python/sglang/lang/ir.py @@ -0,0 +1,635 @@ +"""The intermediate representation.""" + +import dataclasses +import inspect +import warnings +from typing import List, Optional, Union + +from sglang.global_config import global_config +from sglang.lang.choices import ChoicesSamplingMethod + +REGEX_INT = r"[-+]?[0-9]+[ \n]*" +REGEX_FLOAT = r"[-+]?[0-9]*\.?[0-9]+[ \n]*" +REGEX_BOOL = r"(True|False)" +REGEX_STR = r"\"[\w\d\s]*\"" # bugs with regex r"\".*\"" in interegular pkg + + +@dataclasses.dataclass +class SglSamplingParams: + max_new_tokens: int = 128 + min_new_tokens: int = 0 + n: int = 1 + stop: Union[str, List[str]] = () + stop_token_ids: Optional[List[int]] = () + temperature: float = 1.0 + top_p: float = 1.0 + top_k: int = -1 # -1 means disable + min_p: float = 0.0 + frequency_penalty: float = 0.0 + presence_penalty: float = 0.0 + ignore_eos: bool = False + return_logprob: Optional[bool] = None + logprob_start_len: Optional[int] = (None,) + top_logprobs_num: Optional[int] = (None,) + return_text_in_logprobs: Optional[bool] = (None,) + json_schema: Optional[str] = None + + # for constrained generation, not included in to_xxx_kwargs + dtype: Optional[str] = None + regex: Optional[str] = None + + def clone(self): + return SglSamplingParams( + self.max_new_tokens, + self.min_new_tokens, + self.n, + self.stop, + self.stop_token_ids, + self.temperature, + self.top_p, + self.top_k, + self.min_p, + self.frequency_penalty, + self.presence_penalty, + self.ignore_eos, + self.return_logprob, + self.logprob_start_len, + self.top_logprobs_num, + self.return_text_in_logprobs, + self.json_schema, + ) + + def to_openai_kwargs(self): + # OpenAI does not support top_k, so we drop it here + if self.regex is not None: + warnings.warn("Regular expression is not supported in the OpenAI backend.") + return { + "max_tokens": self.max_new_tokens, + "max_completion_tokens": self.max_new_tokens, + "n": self.n, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_vertexai_kwargs(self): + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the VertexAI backend." + ) + return { + "candidate_count": 1, + "max_output_tokens": self.max_new_tokens, + "stop_sequences": self.stop, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k if self.top_k > 0 else None, + } + + def to_anthropic_kwargs(self): + # Anthropic does not support frequency_penalty or presence_penalty, so we drop it here + if self.regex is not None: + warnings.warn( + "Regular expression is not supported in the Anthropic backend." + ) + return { + "max_tokens": self.max_new_tokens, + "stop_sequences": ( + self.stop if isinstance(self.stop, (list, tuple)) else [self.stop] + ), + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + } + + def to_litellm_kwargs(self): + if self.regex is not None: + warnings.warn("Regular expression is not supported in the LiteLLM backend.") + return { + "max_tokens": self.max_new_tokens, + "stop": self.stop or None, + "temperature": self.temperature, + "top_p": self.top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + } + + def to_srt_kwargs(self): + return { + "max_new_tokens": self.max_new_tokens, + "min_new_tokens": self.min_new_tokens, + "n": self.n, + "stop": self.stop, + "stop_token_ids": self.stop_token_ids, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "min_p": self.min_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + "ignore_eos": self.ignore_eos, + "regex": self.regex, + "json_schema": self.json_schema, + } + + +class SglFunction: + def __init__(self, func, num_api_spec_tokens=None, bind_arguments=None): + self.func = func + self.num_api_spec_tokens = num_api_spec_tokens + self.bind_arguments = bind_arguments or {} + self.pin_prefix_rid = None + + # Parse arguments + argspec = inspect.getfullargspec(func) + assert argspec.args[0] == "s", 'The first argument must be "s"' + self.arg_names = argspec.args[1:] + self.arg_defaults = argspec.defaults if argspec.defaults is not None else [] + + def bind(self, **kwargs): + assert all(key in self.arg_names for key in kwargs) + + new_bind_dict = {**self.bind_arguments, **kwargs} + return SglFunction(self.func, bind_arguments=new_bind_dict) + + def run( + self, + *args, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + stream: bool = False, + backend=None, + use_thread: bool = True, + **kwargs, + ): + from sglang.lang.interpreter import run_program + + # avoid using [] as the default arg: https://nikos7am.com/posts/mutable-default-arguments/ + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program( + self, + backend, + args, + kwargs, + default_sampling_para, + stream, + use_thread=use_thread, + ) + + def run_batch( + self, + batch_kwargs, + *, + max_new_tokens: int = 128, + n: int = 1, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: float = 1.0, + top_p: float = 1.0, + top_k: int = -1, + min_p: float = 0.0, + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + ignore_eos: bool = False, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + backend=None, + num_threads: Union[str, int] = "auto", + progress_bar: bool = False, + generator_style: bool = False, + ): + from sglang.lang.interpreter import run_program_batch + + if stop is None: + stop = [] + if stop_token_ids is None: + stop_token_ids = [] + + assert isinstance(batch_kwargs, (list, tuple)) + if len(batch_kwargs) == 0: + return [] + if not isinstance(batch_kwargs[0], dict): + num_programs = len(batch_kwargs) + # change the list of argument values to dict of arg_name -> arg_value + batch_kwargs = [ + {self.arg_names[i]: v for i, v in enumerate(arg_values)} + for arg_values in batch_kwargs + if isinstance(arg_values, (list, tuple)) + and len(self.arg_names) - len(self.arg_defaults) + <= len(arg_values) + <= len(self.arg_names) + ] + # Ensure to raise an exception if the number of arguments mismatch + if len(batch_kwargs) != num_programs: + raise Exception("Given arguments mismatch the SGL function signature") + + default_sampling_para = SglSamplingParams( + max_new_tokens=max_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + ) + backend = backend or global_config.default_backend + return run_program_batch( + self, + backend, + batch_kwargs, + default_sampling_para, + num_threads, + progress_bar, + generator_style=generator_style, + ) + + def trace(self, *, backend=None, **kwargs): + from sglang.lang.tracer import trace_program + + backend = backend or global_config.default_backend + return trace_program(self, kwargs, backend) + + def cache(self, backend=None): + from sglang.lang.interpreter import cache_program + + backend = backend or global_config.default_backend + return cache_program(self, backend) + + def compile(self, *, backend=None): + from sglang.lang.compiler import compile_func + + return compile_func(self, backend) + + def __call__(self, *args, **kwargs): + from sglang.lang.tracer import TracingScope + + tracing_scope = TracingScope.get_current_scope() + if tracing_scope is None: + return self.run(*args, **kwargs) + else: + kwargs["backend"] = tracing_scope.tracer_state.backend + return self.trace(*args, **kwargs) + + +class SglExpr: + node_ct = 0 + + def __init__(self): + self.node_id = SglExpr.node_ct + self.prev_node = None + self.pid = None + SglExpr.node_ct += 1 + + def __add__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr) + + return self.concatenate_ir(self, other) + + def __radd__(self, other): + if isinstance(other, str): + other = SglConstantText(other) + assert isinstance(other, SglExpr), f"{other}" + + return self.concatenate_ir(other, self) + + def concatenate_ir(self, a, b): + if isinstance(a, SglExprList): + if isinstance(b, SglExprList): + return SglExprList(a.expr_list + b.expr_list) + else: + return SglExprList(a.expr_list + [b]) + elif isinstance(b, SglExprList): + return SglExprList([a] + b.expr_list) + + return SglExprList([a, b]) + + def print_graph_dfs(self): + ret = [""] + visited = set() + + def dfs_print(x): + if x is None or x in visited: + return + visited.add(x) + + # Print dependency + if x.prev_node is not None: + dfs_print(x.prev_node) + + if isinstance(x, SglExprList): + for y in x.expr_list: + dfs_print(y) + # elif isinstance(x, SglRole): + # dfs_print(x.expr) + elif isinstance(x, SglVariable): + dfs_print(x.source) + + # Print the node itself + if isinstance(x, (SglFork, SglGetForkItem)): + ret[0] += f"%{x.node_id} = {x}\n" + else: + if x.prev_node is not None: + ret[0] += ( + f"%{x.node_id} = %{x.prev_node.node_id} + " + str(x) + "\n" + ) + else: + ret[0] += f"%{x.node_id} = " + str(x) + "\n" + + dfs_print(self) + return ret[0] + + +class SglExprList(SglExpr): + def __init__(self, expr_list: List[SglExpr]): + super().__init__() + self.expr_list = expr_list + + def __repr__(self): + return f"ExprList({self.expr_list})" + + +class SglArgument(SglExpr): + def __init__(self, name: str, value: str): + super().__init__() + self.name = name + self.value = value + + def __repr__(self): + return f"Argument(name={self.name}, value={repr(self.value)})" + + def __len__(self): + return len(self.value) + + def __getitem__(self, i): + return self.value[i] + + def __int__(self): + return self.value + + def __bool__(self): + return self.value + + def __format__(self, *args): + raise TypeError( + "Cannot put argument inside a f-string. " + "This is not compatible with the tracer. " + ) + + +class SglImage(SglExpr): + def __init__(self, path: str): + self.path = path + + def __repr__(self) -> str: + return f"SglImage({self.path})" + + +class SglVideo(SglExpr): + def __init__(self, path: str, num_frames: int): + self.path = path + self.num_frames = num_frames + + def __repr__(self) -> str: + return f"SglVideo({self.path}, {self.num_frames})" + + +class SglGen(SglExpr): + def __init__( + self, + name: Optional[str] = None, + max_new_tokens: Optional[int] = None, + min_new_tokens: Optional[int] = None, + n: Optional[int] = None, + stop: Optional[Union[str, List[str]]] = None, + stop_token_ids: Optional[List[int]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + top_k: Optional[int] = None, + min_p: Optional[float] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + ignore_eos: Optional[bool] = None, + return_logprob: Optional[bool] = None, + logprob_start_len: Optional[int] = None, + top_logprobs_num: Optional[int] = None, + return_text_in_logprobs: Optional[bool] = None, + dtype: Optional[type] = None, + regex: Optional[str] = None, + json_schema: Optional[str] = None, + ): + """Call the model to generate. See the meaning of the arguments in docs/backend/sampling_params.md""" + super().__init__() + self.name = name + self.sampling_params = SglSamplingParams( + max_new_tokens=max_new_tokens, + min_new_tokens=min_new_tokens, + n=n, + stop=stop, + stop_token_ids=stop_token_ids, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ignore_eos=ignore_eos, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + return_text_in_logprobs=return_text_in_logprobs, + dtype=dtype, + regex=regex, + json_schema=json_schema, + ) + + def __repr__(self): + return f"Gen('{self.name}')" + + +class SglConstantText(SglExpr): + def __init__(self, value: str): + super().__init__() + self.value = value + + def __repr__(self): + return f"Constant({repr(self.value)})" + + +class SglRoleBegin(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleBegin({self.role})" + + +class SglRoleEnd(SglExpr): + def __init__(self, role: str): + super().__init__() + self.role = role + + def __repr__(self): + return f"RoleEnd({self.role})" + + +class SglSelect(SglExpr): + + def __init__( + self, + name: str, + choices: List[str], + temperature: float, + choices_method: ChoicesSamplingMethod, + ): + super().__init__() + self.name = name + self.choices = choices + self.temperature = temperature + self.choices_method = choices_method + + def __repr__(self): + return f"Select({self.name}, choices={self.choices}, choices_method={self.choices_method})" + + +class SglFork(SglExpr): + def __init__(self, number: int, position_ids_offset=None): + super().__init__() + self.number = number + self.position_ids_offset = position_ids_offset + + def __repr__(self): + return ( + f"Fork(%{self.prev_node.node_id}, number={self.number}, " + f"position_ids_offset={self.position_ids_offset})" + ) + + +class SglGetForkItem(SglExpr): + def __init__(self, index: int): + super().__init__() + self.index = index + + def __repr__(self): + return f"GetForkItem(%{self.prev_node.node_id}, index={self.index})" + + +class SglVariable(SglExpr): + def __init__(self, name: str, source): + super().__init__() + self.name = name + self.source = source + + def __repr__(self): + return f"Variable('{self.name}', source=%{self.source.node_id})" + + +class SglVarScopeBegin(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeBegin('{self.name}')" + + +class SglVarScopeEnd(SglExpr): + def __init__(self, name: str): + super().__init__() + self.name = name + + def __repr__(self): + return f"VarScopeEnd('{self.name}')" + + +class SglConcateAndAppend(SglExpr): + def __init__(self, states): + super().__init__() + self.states = states + + def __repr__(self): + return f"ConcatenateAndAppend('{self.states}')" + + +class SglCommitLazy(SglExpr): + def __init__(self): + super().__init__() + + def __repr__(self): + return "CommitLazy()" + + +class SglSeparateReasoning(SglExpr): + def __init__(self, model_type: str, expr: SglExpr): + super().__init__() + self.model_type = model_type + + self.expr = expr + self.name = None + self._process_expr(expr) + + def process_name_for_reasoning(self, name): + if not name: + raise ValueError("name must be provided") + return f"{name}_reasoning_content" + + def _process_expr(self, expr): + if isinstance(expr, SglGen): + self.name = self.process_name_for_reasoning(expr.name) + elif isinstance(expr, SglSelect): + self.name = self.process_name_for_reasoning(expr.name) + elif isinstance(expr, SglExprList): + for x in expr.expr_list: + self._process_expr(x) + + def __repr__(self): + return f"SeparateReasoning(model_type={self.model_type}, name={self.name})" diff --git a/python/sglang/lang/tracer.py b/python/sglang/lang/tracer.py new file mode 100644 index 000000000..0a2a744f9 --- /dev/null +++ b/python/sglang/lang/tracer.py @@ -0,0 +1,279 @@ +"""Tracing a program.""" + +import uuid +from typing import Any, Dict, List, Optional + +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.interpreter import ProgramState, ProgramStateGroup +from sglang.lang.ir import ( + SglArgument, + SglConstantText, + SglExpr, + SglExprList, + SglFork, + SglGen, + SglGetForkItem, + SglRoleBegin, + SglRoleEnd, + SglSelect, + SglVariable, + SglVarScopeBegin, + SglVarScopeEnd, +) + + +class StopTracing(Exception): + pass + + +def extract_prefix_by_tracing(program, backend): + # Create dummy arguments + dummy_arguments = {name: SglArgument(name, None) for name in program.arg_names} + arguments = dummy_arguments + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=True) + try: + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + except (StopTracing, TypeError, AttributeError): + # Some exceptions may not be caught + pass + + # Run and cache prefix + prefix = "" + for expr in tracer.flatten_nodes(): + if isinstance(expr, SglConstantText): + prefix += expr.value + else: + break + return prefix + + +def trace_program(program, arguments, backend): + # Create dummy backend + if backend is None: + backend = BaseBackend() + + # Create dummy arguments + dummy_arguments = { + name: SglArgument(name, None) + for name in program.arg_names + if name not in arguments + } + arguments.update(dummy_arguments) + arguments.update(program.bind_arguments) + + # Trace + tracer = TracerProgramState(backend, arguments, only_trace_prefix=False) + with TracingScope(tracer): + tracer.ret_value = program.func(tracer, **arguments) + return tracer + + +class TracerProgramState(ProgramState): + def __init__(self, backend, arguments, only_trace_prefix): + self.pid = uuid.uuid4().hex + self.backend = backend + self.arguments: Dict[str, Any] = arguments + self.only_trace_prefix = only_trace_prefix + + if hasattr(backend, "endpoint"): + self.backend = backend.endpoint + + self.nodes = [] + self.last_node = None + self.variables = {} + self.ret_value = None + + # For completion + + # For chat + self.messages_ = [] + self.cur_role = None + self.chat_template = self.backend.get_chat_template() + + # For multi states + self.child_states = [] + + cur_scope = TracingScope.get_current_scope() + if cur_scope is not None: + cur_scope.add_child_state(self) + + ################################## + ########### Public API ########### + ################################## + + def fork(self, size: int = 1, position_ids_offset: Optional[List[int]] = None): + assert size >= 1 + + if self.only_trace_prefix: + raise StopTracing() + + fork_node = SglFork(size) + fork_node.prev_node = self.last_node + + states = [ + TracerProgramState(self.backend, self.arguments, self.only_trace_prefix) + for _ in range(size) + ] + + for i in range(size): + node = SglGetForkItem(i) + node.prev_node = fork_node + states[i].last_node = node + states[i].variables = dict(self.variables) + states[i].messages_ = list(self.messages_) + states[i].cur_role = self.cur_role + states[i].chat_template = self.chat_template + + state_group = ProgramStateGroup(states, self) + + return state_group + + ################################## + ########## Internal API ########## + ################################## + + def _append_node(self, other: SglExpr): + self.nodes.append(other) + other.prev_node = self.last_node + self.last_node = other + + def _execute(self, other: SglExpr): + if isinstance(other, str): + other = SglConstantText(other) + + other.pid = self.pid + + if isinstance(other, SglConstantText): + self._execute_fill(other) + elif isinstance(other, SglGen): + self._execute_gen(other) + elif isinstance(other, SglSelect): + self._execute_select(other) + elif isinstance(other, SglExprList): + for x in other.expr_list: + self._execute(x) + elif isinstance(other, SglRoleBegin): + self._execute_role_begin(other) + elif isinstance(other, SglRoleEnd): + self._execute_role_end(other) + elif isinstance(other, SglVarScopeBegin): + self._execute_var_scope_begin(other) + elif isinstance(other, SglVarScopeEnd): + self._execute_var_scope_end(other) + else: + if self.only_trace_prefix: + raise StopTracing() + else: + self._append_node(other) + + return self + + def __iadd__(self, other): + self._execute(other) + return self + + def _execute_fill(self, expr: SglConstantText): + if isinstance(expr, str): + expr = SglConstantText(expr) + self._append_node(expr) + + def _execute_gen(self, expr: SglGen): + name = expr.name if expr.name is not None else "gen_" + str(len(self.variables)) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_select(self, expr: SglSelect): + name = ( + expr.name if expr.name is not None else "select_" + str(len(self.variables)) + ) + new_node = SglVariable(name, source=expr) + self.variables[name] = new_node + self._append_node(expr) + + def _execute_role_begin(self, expr: SglRoleBegin): + assert self.cur_role is None, "Nested roles are not allowed." + + if len(self.messages_) == 0 and expr.role != "system": + # Insert default system message + default_system = self.chat_template.default_system_prompt + if default_system: + self._execute_role_begin(SglRoleBegin("system")) + self._execute_fill(default_system) + self._execute_role_end(SglRoleEnd("system")) + + self.cur_role = expr.role + + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(prefix) + + def _execute_role_end(self, expr: SglRoleEnd): + prefix, suffix = self.chat_template.get_prefix_and_suffix( + expr.role, self.messages_ + ) + + self._execute_fill(suffix) + + self.messages_.append({"role": expr.role, "content": ""}) + + self.cur_role = None + + def _execute_var_scope_end(self, expr: SglVarScopeEnd): + new_node = SglVariable(expr.name, source=self.last_node) + self.variables[expr.name] = new_node + + def get_var(self, name): + ret = self.arguments.get(name, None) + if ret is not None: + return ret + + v = self.variables[name] + return SglVariable(v.name, v.source) + + def flatten_nodes(self): + def traverse(cur): + if isinstance(cur, SglExprList): + for child in cur.expr_list: + traverse(child) + else: + ret.append(cur) + + ret = [] + for x in self.nodes: + traverse(x) + return ret + + def __del__(self): + pass + + +class TracingScope: + cur_scope = None + + def __init__(self, tracer_state: TracerProgramState): + self.tracer_state = tracer_state + self.last_scope = TracingScope.cur_scope + + def __enter__(self): + TracingScope.cur_scope = self + return self + + def __exit__(self, exc_type, exc_value, traceback): + TracingScope.cur_scope = self.last_scope + + @staticmethod + def get_current_scope(): + return TracingScope.cur_scope + + def add_child_state(self, state: TracerProgramState): + cur_scope = self + while cur_scope is not None: + cur_scope.tracer_state.child_states.append(state) + cur_scope = cur_scope.last_scope diff --git a/python/sglang/launch_server.py b/python/sglang/launch_server.py new file mode 100644 index 000000000..caae7b0f6 --- /dev/null +++ b/python/sglang/launch_server.py @@ -0,0 +1,16 @@ +"""Launch the inference server.""" + +import os +import sys + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import prepare_server_args +from sglang.srt.utils import kill_process_tree + +if __name__ == "__main__": + server_args = prepare_server_args(sys.argv[1:]) + + try: + launch_server(server_args) + finally: + kill_process_tree(os.getpid(), include_parent=False) diff --git a/python/sglang/profiler.py b/python/sglang/profiler.py new file mode 100644 index 000000000..d872ca320 --- /dev/null +++ b/python/sglang/profiler.py @@ -0,0 +1,166 @@ +""" +Run live profiling. + +Usage: +python3 -m sglang.profiler +""" + +import argparse +import json +import os +import time +from argparse import ArgumentParser +from pathlib import Path +from typing import List, Optional + +import requests + +PARENT_FOLDER = "/tmp/sglang-profile" + + +def _run_profile( + url: Optional[str], + num_steps: int, + activities: List[str], + output_dir: Optional[str] = None, + profile_name: Optional[str] = None, + profile_by_stage: bool = False, +) -> str: + if output_dir is None: + output_dir = PARENT_FOLDER + + output_dir = os.path.normpath(output_dir) + output_dir = os.path.abspath(output_dir) + output_dir = Path(output_dir) + + # Add "profile_name/timestamp" to the path. + if profile_name: + output_dir = output_dir / profile_name + output_dir = output_dir / str(time.time()) + output_dir.mkdir(exist_ok=True, parents=True) + + print(f"Dump profiling traces to {output_dir}") + print( + f"Waiting for {num_steps} steps and the trace to be flushed.... ({profile_by_stage=})" + ) + + # Dump server args. + file_path = Path(output_dir) / "server_args.json" + if not file_path.exists(): + response = requests.get(url + "/get_server_info") + response.raise_for_status() + server_args_data = response.json() + with open(file_path, "w") as file: + file.write(json.dumps(server_args_data)) + + # Start profiler. The API replies when all steps are processed + # and files are generated. + json_data = { + "output_dir": str(output_dir), + "num_steps": str(num_steps), + "activities": activities, + "profile_by_stage": profile_by_stage, + } + + response = requests.post(url=url + "/start_profile", json=json_data) + response.raise_for_status() + + trace_link = str(output_dir) + return trace_link + + +def run_profile( + url: Optional[str], + num_steps: int, + activities: List[str], + output_dir: Optional[str] = None, + profile_name: Optional[str] = None, + profile_by_stage: bool = False, +): + # step based profile will self terminate on num_steps constraints + link = _run_profile( + url, num_steps, activities, output_dir, profile_name, profile_by_stage + ) + return link + + +if __name__ == "__main__": + parser = ArgumentParser(description="Benchmark the online serving throughput.") + parser.add_argument( + "--url", + type=str, + default="http://localhost:30000", + help="Server or API base url if not using http host and port.", + ) + parser.add_argument( + "--output-dir", + type=str, + default=None, + help="Profile directory to dump profile traces.", + ) + parser.add_argument( + "--profile-name", + type=str, + default=None, + help="The name of this profile run.", + ) + parser.add_argument( + "--num-steps", + type=int, + default=5, + help="The number of forward steps to profile.", + ) + parser.add_argument( + "--profile-by-stage", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="The number of forward steps to profile.", + ) + parser.add_argument( + "--cpu", + action=argparse.BooleanOptionalAction, + type=bool, + default=True, + help="Whether to profile CPU activity", + ) + parser.add_argument( + "--gpu", + action=argparse.BooleanOptionalAction, + type=bool, + default=True, + help="Whether to profile GPU activity", + ) + parser.add_argument( + "--mem", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to memory usage (https://pytorch.org/memory_viz)", + ) + parser.add_argument( + "--rpd", + action=argparse.BooleanOptionalAction, + type=bool, + default=False, + help="Whether to use rpd profiler (https://github.com/ROCm/rocmProfileData)", + ) + + args = parser.parse_args() + activities = [] + if args.cpu: + activities.append("CPU") + if args.gpu: + activities.append("GPU") + if args.mem: + activities.append("MEM") + if args.rpd: + activities.append("RPD") + run_profile( + args.url, + args.num_steps, + activities, + args.output_dir, + args.profile_name, + args.profile_by_stage, + ) diff --git a/python/sglang/srt/_custom_ops.py b/python/sglang/srt/_custom_ops.py new file mode 100644 index 000000000..5ed175312 --- /dev/null +++ b/python/sglang/srt/_custom_ops.py @@ -0,0 +1,177 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/_custom_ops.py +import logging +from typing import List, Optional, Tuple + +import torch + +from sglang.srt.utils import get_bool_env_var, is_hip, is_hpu, is_npu + +logger = logging.getLogger(__name__) +use_vllm_custom_allreduce = get_bool_env_var( + "USE_VLLM_CUSTOM_ALLREDUCE", default="false" +) + +if not is_hpu(): + # ROCm does not use vllm custom allreduce + if use_vllm_custom_allreduce and not is_hip(): + try: + import vllm._C + except ImportError as e: + logger.warning("Failed to import from vllm._C with %r", e) + else: + try: + import sgl_kernel + except ImportError as e: + logger.warning("Failed to import from custom_ar with %r", e) + + +if not is_hip() and not is_npu(): + if use_vllm_custom_allreduce: + custom_op = torch.ops._C_custom_ar + else: + custom_op = sgl_kernel.allreduce + + # custom allreduce + def init_custom_ar( + ipc_tensors: List[torch.Tensor], + rank_data: torch.Tensor, + rank: int, + full_nvlink: bool, + ) -> int: + return custom_op.init_custom_ar(ipc_tensors, rank_data, rank, full_nvlink) + + def all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + reg_buffer: int, + reg_buffer_sz_bytes: int, + ) -> None: + custom_op.all_reduce(fa, inp, out, reg_buffer, reg_buffer_sz_bytes) + + def dispose(fa: int) -> None: + custom_op.dispose(fa) + + def meta_size() -> int: + return custom_op.meta_size() + + def register_buffer(fa: int, ipc_tensors: List[int]) -> None: + return custom_op.register_buffer(fa, ipc_tensors) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[List[int], List[int]]: + return custom_op.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + custom_op.register_graph_buffers(fa, handles, offsets) + +else: + # ROCM custom allreduce + + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return sgl_kernel.allreduce.init_custom_ar( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + sgl_kernel.allreduce.all_reduce_reg(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + sgl_kernel.allreduce.all_reduce_unreg(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + sgl_kernel.allreduce.dispose(fa) + + def meta_size() -> int: + return sgl_kernel.allreduce.meta_size() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return sgl_kernel.allreduce.register_buffer(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return sgl_kernel.allreduce.get_graph_buffer_ipc_meta(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + sgl_kernel.allreduce.register_graph_buffers(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return sgl_kernel.allreduce.allocate_meta_buffer(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return sgl_kernel.allreduce.get_meta_buffer_ipc_handle(inp) + + # ROCM custom quick allreduce + + def init_custom_qr( + rank: int, world_size: int, qr_max_size: Optional[int] = None + ) -> int: + return sgl_kernel.allreduce.init_custom_qr(world_size, rank, qr_max_size) + + def qr_get_handle(fa: int) -> torch.Tensor: + return sgl_kernel.allreduce.qr_get_handle(fa) + + def qr_open_handles(fa: int, handles: list[torch.Tensor]) -> None: + sgl_kernel.allreduce.qr_open_handles(fa, handles) + + def qr_all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + quant_level: int, + cast_bf2half: bool, + ) -> None: + sgl_kernel.allreduce.qr_all_reduce(fa, inp, out, quant_level, cast_bf2half) + + def qr_destroy(fa: int) -> None: + sgl_kernel.allreduce.qr_destroy(fa) + + def qr_max_size() -> int: + return sgl_kernel.allreduce.qr_max_size() + + +def mscclpp_generate_unique_id() -> bytes: + return sgl_kernel.allreduce.mscclpp_generate_unique_id() + + +def mscclpp_init_context( + unique_id: bytes, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, +) -> int: + return sgl_kernel.allreduce.mscclpp_init_context( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + context_selection, + ) + + +def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int +) -> None: + return sgl_kernel.allreduce.mscclpp_allreduce(context, inp, out, nthreads, nblocks) diff --git a/python/sglang/srt/aio_rwlock.py b/python/sglang/srt/aio_rwlock.py new file mode 100644 index 000000000..deda1fe79 --- /dev/null +++ b/python/sglang/srt/aio_rwlock.py @@ -0,0 +1,100 @@ +import asyncio + + +class RWLock: + def __init__(self): + # Protects internal state + self._lock = asyncio.Lock() + + # Condition variable used to wait for state changes + self._cond = asyncio.Condition(self._lock) + + # Number of readers currently holding the lock + self._readers = 0 + + # Whether a writer is currently holding the lock + self._writer_active = False + + # How many writers are queued waiting for a turn + self._waiting_writers = 0 + + @property + def reader_lock(self): + """ + A context manager for acquiring a shared (reader) lock. + + Example: + async with rwlock.reader_lock: + # read-only access + """ + return _ReaderLock(self) + + @property + def writer_lock(self): + """ + A context manager for acquiring an exclusive (writer) lock. + + Example: + async with rwlock.writer_lock: + # exclusive access + """ + return _WriterLock(self) + + async def acquire_reader(self): + async with self._lock: + # Wait until there is no active writer or waiting writer + # to ensure fairness. + while self._writer_active or self._waiting_writers > 0: + await self._cond.wait() + self._readers += 1 + + async def release_reader(self): + async with self._lock: + self._readers -= 1 + # If this was the last reader, wake up anyone waiting + # (potentially a writer or new readers). + if self._readers == 0: + self._cond.notify_all() + + async def acquire_writer(self): + async with self._lock: + # Increment the count of writers waiting + self._waiting_writers += 1 + try: + # Wait while either a writer is active or readers are present + while self._writer_active or self._readers > 0: + await self._cond.wait() + self._writer_active = True + finally: + # Decrement waiting writers only after we've acquired the writer lock + self._waiting_writers -= 1 + + async def release_writer(self): + async with self._lock: + self._writer_active = False + # Wake up anyone waiting (readers or writers) + self._cond.notify_all() + + +class _ReaderLock: + def __init__(self, rwlock: RWLock): + self._rwlock = rwlock + + async def __aenter__(self): + await self._rwlock.acquire_reader() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._rwlock.release_reader() + + +class _WriterLock: + def __init__(self, rwlock: RWLock): + self._rwlock = rwlock + + async def __aenter__(self): + await self._rwlock.acquire_writer() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self._rwlock.release_writer() diff --git a/python/sglang/srt/bench_utils.py b/python/sglang/srt/bench_utils.py new file mode 100644 index 000000000..e9f7fcbb4 --- /dev/null +++ b/python/sglang/srt/bench_utils.py @@ -0,0 +1,137 @@ +import os +import sys +from contextlib import nullcontext + +import torch + + +# NOTE copied and modified from DeepGEMM +class suppress_stdout_stderr: + def __enter__(self): + self.outnull_file = open(os.devnull, "w") + self.errnull_file = open(os.devnull, "w") + + self.old_stdout_fileno_undup = sys.stdout.fileno() + self.old_stderr_fileno_undup = sys.stderr.fileno() + + self.old_stdout_fileno = os.dup(sys.stdout.fileno()) + self.old_stderr_fileno = os.dup(sys.stderr.fileno()) + + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + + os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup) + os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup) + + sys.stdout = self.outnull_file + sys.stderr = self.errnull_file + return self + + def __exit__(self, *_): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + + os.close(self.old_stdout_fileno) + os.close(self.old_stderr_fileno) + + self.outnull_file.close() + self.errnull_file.close() + + +# NOTE copied and modified from DeepGEMM +def bench_kineto( + fn, + kernel_names, + num_tests: int = 30, + suppress_kineto_output: bool = False, + trace_path: str = None, + flush_l2: bool = True, + with_multiple_kernels: bool = False, +): + # Conflict with Nsight Systems + using_nsys = int(os.environ.get("SGLANG_NSYS_PROFILING", 0)) + + # By default, flush L2 with an excessive 8GB memset to give the GPU some (literal) chill time without full idle + flush_l2_size = int(8e9 // 4) + + # For some auto-tuning kernels with prints + fn() + + # Profile + suppress = ( + suppress_stdout_stderr + if suppress_kineto_output and not using_nsys + else nullcontext + ) + with suppress(): + schedule = ( + torch.profiler.schedule(wait=0, warmup=1, active=1, repeat=1) + if not using_nsys + else None + ) + profiler = ( + torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA], schedule=schedule + ) + if not using_nsys + else nullcontext() + ) + with profiler: + for i in range(2): + for _ in range(num_tests): + if flush_l2: + torch.empty( + flush_l2_size, dtype=torch.int, device="cuda" + ).zero_() + fn() + + if not using_nsys: + profiler.step() + + # Return 1 if using Nsight Systems + if using_nsys: + return 1 + + # Parse the profiling table + assert isinstance(kernel_names, str) or isinstance(kernel_names, tuple) + is_tuple = isinstance(kernel_names, tuple) + prof_lines = ( + profiler.key_averages() + .table(sort_by="cuda_time_total", max_name_column_width=100) + .split("\n") + ) + kernel_names = (kernel_names,) if isinstance(kernel_names, str) else kernel_names + assert all([isinstance(name, str) for name in kernel_names]) + if not with_multiple_kernels: + for name in kernel_names: + assert ( + sum([name in line for line in prof_lines]) == 1 + ), f"Errors of the kernel {name} in the profiling table (table: {prof_lines})" + + # Save chrome traces + if trace_path is not None: + profiler.export_chrome_trace(trace_path) + + # Return average kernel times + units = {"ms": 1e3, "us": 1e6} + kernel_times = [] + for name in kernel_names: + total_time = 0 + total_num = 0 + for line in prof_lines: + if name in line: + time_str = line.split()[-2] + num_str = line.split()[-1] + for unit, scale in units.items(): + if unit in time_str: + total_time += ( + float(time_str.replace(unit, "")) / scale * int(num_str) + ) + total_num += int(num_str) + break + kernel_times.append(total_time / total_num) + + return tuple(kernel_times) if is_tuple else kernel_times[0] diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py new file mode 100644 index 000000000..ef880c911 --- /dev/null +++ b/python/sglang/srt/configs/__init__.py @@ -0,0 +1,29 @@ +from sglang.srt.configs.chatglm import ChatGLMConfig +from sglang.srt.configs.dbrx import DbrxConfig +from sglang.srt.configs.deepseekvl2 import DeepseekVL2Config +from sglang.srt.configs.exaone import ExaoneConfig +from sglang.srt.configs.janus_pro import MultiModalityConfig +from sglang.srt.configs.kimi_vl import KimiVLConfig +from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig +from sglang.srt.configs.longcat_flash import LongcatFlashConfig +from sglang.srt.configs.qwen3_next import Qwen3NextConfig +from sglang.srt.configs.step3_vl import ( + Step3TextConfig, + Step3VisionEncoderConfig, + Step3VLConfig, +) + +__all__ = [ + "ExaoneConfig", + "ChatGLMConfig", + "DbrxConfig", + "DeepseekVL2Config", + "LongcatFlashConfig", + "MultiModalityConfig", + "KimiVLConfig", + "MoonViTConfig", + "Step3VLConfig", + "Step3TextConfig", + "Step3VisionEncoderConfig", + "Qwen3NextConfig", +] diff --git a/python/sglang/srt/configs/chatglm.py b/python/sglang/srt/configs/chatglm.py new file mode 100644 index 000000000..9370c218a --- /dev/null +++ b/python/sglang/srt/configs/chatglm.py @@ -0,0 +1,78 @@ +# Adapted from +# https://github.com/THUDM/ChatGLM2-6B +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/chatglm.py + +# ChatGLM2 and ChatGLM3 share the same config. +# ChatGLM4 is officially supported by Huggingface +# transformers >= 4.46.0 is required +# https://huggingface.co/docs/transformers/en/model_doc/glm +from transformers import PretrainedConfig + + +class ChatGLMConfig(PretrainedConfig): + model_type = "chatglm" + attribute_map = { + "num_hidden_layers": "num_layers", + "n_head_kv": "multi_query_group_num", + } + + def __init__( + self, + num_layers=28, + padded_vocab_size=65024, + hidden_size=4096, + ffn_hidden_size=13696, + kv_channels=128, + num_attention_heads=32, + seq_length=2048, + hidden_dropout=0.0, + attention_dropout=0.0, + layernorm_epsilon=1e-5, + rmsnorm=True, + apply_residual_connection_post_layernorm=False, + post_layer_norm=True, + add_bias_linear=False, + add_qkv_bias=False, + interleaved_qkv=False, + bias_dropout_fusion=True, + multi_query_attention=False, + multi_query_group_num=1, + apply_query_key_layer_scaling=True, + attention_softmax_in_fp32=True, + fp32_residual_connection=False, + quantization_bit=0, + pre_seq_len=None, + prefix_projection=False, + **kwargs + ): + self.num_layers = num_layers + self.vocab_size = padded_vocab_size + self.padded_vocab_size = padded_vocab_size + self.hidden_size = hidden_size + self.ffn_hidden_size = ffn_hidden_size + self.kv_channels = kv_channels + self.num_attention_heads = num_attention_heads + self.seq_length = seq_length + # It is to be compatible with long lora. + self.max_position_embeddings = seq_length + self.hidden_dropout = hidden_dropout + self.attention_dropout = attention_dropout + self.layernorm_epsilon = layernorm_epsilon + self.rmsnorm = rmsnorm + self.apply_residual_connection_post_layernorm = ( + apply_residual_connection_post_layernorm + ) + self.post_layer_norm = post_layer_norm + self.add_bias_linear = add_bias_linear + self.add_qkv_bias = add_qkv_bias + self.bias_dropout_fusion = bias_dropout_fusion + self.multi_query_attention = multi_query_attention + self.multi_query_group_num = multi_query_group_num + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling + self.attention_softmax_in_fp32 = attention_softmax_in_fp32 + self.fp32_residual_connection = fp32_residual_connection + self.quantization_bit = quantization_bit + self.pre_seq_len = pre_seq_len + self.prefix_projection = prefix_projection + self.interleaved_qkv = interleaved_qkv + super().__init__(**kwargs) diff --git a/python/sglang/srt/configs/dbrx.py b/python/sglang/srt/configs/dbrx.py new file mode 100644 index 000000000..75ccbde94 --- /dev/null +++ b/python/sglang/srt/configs/dbrx.py @@ -0,0 +1,279 @@ +# Adapted from +# https://huggingface.co/databricks/dbrx-base/blob/main/configuration_dbrx.py +# https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/dbrx.py +"""Dbrx configuration.""" + +from typing import Any, Optional + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +DBRX_PRETRAINED_CONFIG_ARCHIVE_MAP = {} # type: ignore + + +class DbrxAttentionConfig(PretrainedConfig): + """Configuration class for Dbrx Attention. + + [`DbrxAttention`] class. It is used to instantiate attention layers + according to the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + attn_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the attention layers. + clip_qkv (`float`, *optional*, defaults to None): + If not `None`, clip the queries, keys, and values in the attention layer to this value. + kv_n_heads (Optional[int]): For grouped_query_attention only, allow user to specify number of kv heads. + rope_theta (float): The base frequency for rope. + """ + + def __init__( + self, + attn_pdrop: float = 0, + clip_qkv: Optional[float] = None, + kv_n_heads: int = 1, + rope_theta: float = 10000.0, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.attn_pdrop = attn_pdrop + self.clip_qkv = clip_qkv + self.kv_n_heads = kv_n_heads + self.rope_theta = rope_theta + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["attn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all configurations of " + "models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxFFNConfig(PretrainedConfig): + """Configuration class for Dbrx FFN. + + [`DbrxFFN`] class. It is used to instantiate feedforward layers according to + the specified arguments, defining the layers architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + ffn_act_fn (dict, optional): A dict specifying activation function for the FFN. + The dict should have a key 'name' with the value being the name of + the activation function along with any additional keyword arguments. + ffn_hidden_size (int, optional): The hidden size of the feedforward network. + moe_num_experts (int, optional): The number of experts in the mixture of experts layer. + moe_top_k (int, optional): The number of experts to use in the mixture of experts layer. + moe_jitter_eps (float, optional): The jitter epsilon for the mixture of experts layer. + moe_loss_weight (float, optional): The loss weight for the mixture of experts layer. + moe_normalize_expert_weights (float, optional): The normalization factor for the expert weights. + uniform_expert_assignment (bool, optional): Whether to use uniform expert assignment. + This should only be used for benchmarking purposes. + """ + + def __init__( + self, + ffn_act_fn: Optional[dict] = None, + ffn_hidden_size: int = 3584, + moe_num_experts: int = 4, + moe_top_k: int = 1, + moe_jitter_eps: Optional[float] = None, + moe_loss_weight: float = 0.01, + moe_normalize_expert_weights: Optional[float] = 1, + uniform_expert_assignment: bool = False, + **kwargs: Any, + ): + super().__init__() + if ffn_act_fn is None: + ffn_act_fn = {"name": "silu"} + self.ffn_act_fn = ffn_act_fn + self.ffn_hidden_size = ffn_hidden_size + self.moe_num_experts = moe_num_experts + self.moe_top_k = moe_top_k + self.moe_jitter_eps = moe_jitter_eps + self.moe_loss_weight = moe_loss_weight + self.moe_normalize_expert_weights = moe_normalize_expert_weights + self.uniform_expert_assignment = uniform_expert_assignment + + for k in ["model_type"]: + if k in kwargs: + kwargs.pop(k) + if len(kwargs) != 0: + raise ValueError(f"Found unknown {kwargs=}") + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: str, **kwargs: Any + ) -> "PretrainedConfig": + cls._set_token_in_kwargs(kwargs) + + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if config_dict.get("model_type") == "dbrx": + config_dict = config_dict["ffn_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + "You are using a model of type %s to instantiate a model of " + "type %s. This is not supported for all " + "configurations of models and can yield errors.", + config_dict["model_type"], + cls.model_type, + ) + + return cls.from_dict(config_dict, **kwargs) + + +class DbrxConfig(PretrainedConfig): + """Configuration class for Dbrx. + + [`DbrxModel`]. It is used to instantiate a Dbrx model according to the + specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + d_model (`int`, *optional*, defaults to 6144): + Dimensionality of the embeddings and hidden states. + n_heads (`int`, *optional*, defaults to 48): + Number of attention heads for each attention layer in the Transformer encoder. + n_layers (`int`, *optional*, defaults to 40): + Number of hidden layers in the Transformer encoder. + max_seq_len (`int`, *optional*, defaults to 32768): + The maximum sequence length of the model. + vocab_size (`int`, *optional*, defaults to 100352): + Vocabulary size of the Dbrx model. Defines the maximum number of different tokens that can be represented by + the `inputs_ids` passed when calling [`DbrxModel`]. + resid_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability applied to the attention output before combining with residual. + emb_pdrop (`float`, *optional*, defaults to 0.0): + The dropout probability for the embedding layer. + attn_config (`dict`, *optional*): + A dictionary used to configure the model's attention module. + ffn_config (`dict`, *optional*): + A dictionary used to configure the model's FFN module. + use_cache (`bool`, *optional*, defaults to `False`): + Whether or not the model should return the last key/values attentions (not used by all models). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not the router logits should be returned by the model. Enabling this will also + allow the model to output the auxiliary loss. See [here]() for more details + router_aux_loss_coef (`float`, *optional*, defaults to 0.001): + The aux loss factor for the total loss. + + + Example: + ```python + >>> from transformers import DbrxConfig, DbrxModel + + >>> # Initializing a Dbrx configuration + >>> configuration = DbrxConfig() + + >>> # Initializing a model (with random weights) from the configuration + >>> model = DbrxModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "dbrx" + attribute_map = { + "num_attention_heads": "n_heads", + "hidden_size": "d_model", + "num_hidden_layers": "n_layers", + "max_position_embeddings": "max_seq_len", + } + + def __init__( + self, + d_model: int = 2048, + n_heads: int = 16, + n_layers: int = 24, + max_seq_len: int = 2048, + vocab_size: int = 32000, + resid_pdrop: float = 0.0, + emb_pdrop: float = 0.0, + attn_config: Optional[DbrxAttentionConfig] = None, + ffn_config: Optional[DbrxFFNConfig] = None, + use_cache: bool = True, + initializer_range: float = 0.02, + output_router_logits: bool = False, + router_aux_loss_coef: float = 0.05, + **kwargs: Any, + ): + if attn_config is None: + self.attn_config = DbrxAttentionConfig() + elif isinstance(attn_config, dict): + self.attn_config = DbrxAttentionConfig(**attn_config) + else: + self.attn_config = attn_config + + if ffn_config is None: + self.ffn_config = DbrxFFNConfig() + elif isinstance(ffn_config, dict): + self.ffn_config = DbrxFFNConfig(**ffn_config) + else: + self.ffn_config = ffn_config + + self.d_model = d_model + self.n_heads = n_heads + self.n_layers = n_layers + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.resid_pdrop = resid_pdrop + self.emb_pdrop = emb_pdrop + self.use_cache = use_cache + self.initializer_range = initializer_range + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + + tie_word_embeddings = kwargs.pop("tie_word_embeddings", False) + if tie_word_embeddings: + raise ValueError("tie_word_embeddings is not supported for Dbrx models.") + + super().__init__( + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) diff --git a/python/sglang/srt/configs/deepseekvl2.py b/python/sglang/srt/configs/deepseekvl2.py new file mode 100644 index 000000000..bcb0afe5a --- /dev/null +++ b/python/sglang/srt/configs/deepseekvl2.py @@ -0,0 +1,688 @@ +import math +import os +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +import torch +from PIL import Image, ImageOps +from transformers import ( + AutoProcessor, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) + + +def select_best_resolution(image_size, candidate_resolutions): + # used for cropping + original_width, original_height = image_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float("inf") + + for width, height in candidate_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int( + original_height * scale + ) + effective_resolution = min( + downscaled_width * downscaled_height, original_width * original_height + ) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or ( + effective_resolution == max_effective_resolution + and wasted_resolution < min_wasted_resolution + ): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (width, height) + + return best_fit + + +class DictOutput(object): + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __contains__(self, key): + return key in self.__dict__ + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + input_ids: torch.LongTensor + target_ids: torch.LongTensor + pixel_values: ( + torch.Tensor + ) # rename from "images" to "pixel_values" for compatibility + images_seq_mask: torch.BoolTensor + images_spatial_crop: torch.LongTensor + + def __len__(self): + return len(self.input_ids) + + +class ImageTransform(object): + def __init__( + self, + mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5), + normalize: bool = True, + ): + self.mean = mean + self.std = std + self.normalize = normalize + + # only load torchvision.transforms when needed + try: + import torchvision.transforms as T + + # FIXME: add version check for gguf + except ImportError as err: + raise ImportError( + "Please install torchvision via `pip install torchvision` to use Deepseek-VL2." + ) from err + + transform_pipelines = [T.ToTensor()] + + if normalize: + transform_pipelines.append(T.Normalize(mean, std)) + + self.transform = T.Compose(transform_pipelines) + + def __call__(self, pil_img: Image.Image): + x = self.transform(pil_img) + return x + + +class DeepseekVLV2Processor(ProcessorMixin): + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + attributes = ["tokenizer"] + + def __init__( + self, + tokenizer: LlamaTokenizerFast, + candidate_resolutions: Tuple[Tuple[int, int]], + patch_size: int, + downsample_ratio: int, + image_mean: Tuple[float, float, float] = (0.5, 0.5, 0.5), + image_std: Tuple[float, float, float] = (0.5, 0.5, 0.5), + normalize: bool = True, + image_token: str = "", + pad_token: str = "<｜▁pad▁｜>", + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + + self.candidate_resolutions = candidate_resolutions + self.image_size = candidate_resolutions[0][0] + self.patch_size = patch_size + self.image_mean = image_mean + self.image_std = image_std + self.normalize = normalize + self.downsample_ratio = downsample_ratio + + self.image_transform = ImageTransform( + mean=image_mean, std=image_std, normalize=normalize + ) + self.tokenizer = tokenizer + # must set this，padding side with make a difference in batch inference + self.tokenizer.padding_side = "left" + + # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' + if tokenizer.pad_token is None: + self.tokenizer.add_special_tokens({"pad_token": pad_token}) + + # add image token + image_token_id = self.tokenizer.vocab.get(image_token) + if image_token_id is None: + special_tokens = [image_token] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + self.image_token_id = self.tokenizer.vocab.get(image_token) + + # add five special tokens for grounding-related tasks + # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> + special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + # add special tokens for SFT data + special_tokens = ["<|User|>", "<|Assistant|>"] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + + self.image_token = image_token + self.pad_token = pad_token + self.add_special_token = add_special_token + self.sft_format = sft_format + self.mask_prompt = mask_prompt + self.ignore_id = ignore_id + + super().__init__( + tokenizer, + **kwargs, + ) + + def format_messages_v2(self, messages, pil_images, max_req_input_len=-1): + """play the role of format_messages_v2 and get_images_info in the last version""" + tokenized_data = [] + masked_tokenized_data = [] # labels + images_list = [] + images_seq_mask = [] + images_spatial_crop = [] + + image_index = 0 + image_token_cnt = messages.count(self.image_token) + tokenized_str, images, seq_mask, spatial_crop = self.tokenize_with_images( + messages, + pil_images[image_index : image_index + image_token_cnt], + bos=True, + eos=True, + cropping=len(pil_images) <= 2, + max_req_input_len=max_req_input_len, + ) + + image_index = image_token_cnt + tokenized_data += tokenized_str + if self.mask_prompt: + masked_tokenized_data += [self.ignore_id] * len(tokenized_str) + else: + masked_tokenized_data += tokenized_str + images_list += images + images_seq_mask += seq_mask + images_spatial_crop += spatial_crop + + assert len(tokenized_data) == len( + images_seq_mask + ), f"format_messages_v2: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return ( + tokenized_data, + masked_tokenized_data, + images_list, + images_seq_mask, + images_spatial_crop, + ) + + @property + def bos_id(self): + return self.tokenizer.bos_token_id + + @property + def eos_id(self): + return self.tokenizer.eos_token_id + + @property + def pad_id(self): + return self.tokenizer.pad_token_id + + def encode(self, text: str, bos: bool = True, eos: bool = False): + t = self.tokenizer.encode(text, add_special_tokens=False) + + if bos: + t = [self.bos_id] + t + if eos: + t = t + [self.eos_id] + + return t + + def decode(self, t: List[int], **kwargs) -> str: + return self.tokenizer.decode(t, **kwargs) + + def process_one( + self, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + apply_sft_format (bool): if prompt is not None, then apply the SFT format to prompt; + if conversations is not None, then it will always apply the SFT format to conversations; + inference_mode (bool): if True, then remove the last eos token; + system_prompt (str): the system prompt; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + assert ( + prompt is None or conversations is None + ), "prompt and conversations cannot be used at the same time." + + ( + tokenized_str, + masked_tokenized_str, + images_list, + images_seq_mask, + images_spatial_crop, + ) = self.format_messages_v2(conversations, images, max_req_input_len) + + assert ( + len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) + ), ( + f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " + f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" + ) + + input_ids = torch.LongTensor(tokenized_str) + target_ids = torch.LongTensor(masked_tokenized_str) + images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) + + # set input_ids < 0 | input_ids == self.image_token_id as ignore_id + target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( + self.ignore_id + ) + input_ids[input_ids < 0] = self.pad_id + + if inference_mode: + assert input_ids[-1] == self.eos_id + input_ids = input_ids[:-1] + target_ids = target_ids[:-1] + images_seq_mask = images_seq_mask[:-1] + + if len(images_list) == 0: + images = torch.zeros((1, 3, self.image_size, self.image_size)) + images_spatial_crop = torch.zeros((1, 2), dtype=torch.long) + else: + images = torch.stack(images_list, dim=0) + images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) + + images_spatial_crop = torch.stack( + [images_spatial_crop], dim=0 + ) # stack the tensor to make it a batch of 1 + + prepare = VLChatProcessorOutput( + input_ids=input_ids, + target_ids=target_ids, + pixel_values=images, + images_seq_mask=images_seq_mask, + images_spatial_crop=images_spatial_crop, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image.Image] = None, + apply_sft_format: bool = False, + inference_mode: bool = True, + system_prompt: str = "", + max_req_input_len: int = -1, + **kwargs, + ): + prepare = self.process_one( + prompt=prompt, + conversations=conversations, + images=images, + apply_sft_format=apply_sft_format, + inference_mode=inference_mode, + system_prompt=system_prompt, + max_req_input_len=max_req_input_len, + ) + + return prepare + + def find_all_indices(self, messages, target_value): + indices = [] + for index, item in enumerate(messages): + if item == target_value: + indices.append(index) + return indices + + def tokenize_with_images( + self, + conversation: str, + images: List[Image.Image], + bos: bool = True, + eos: bool = True, + cropping: bool = True, + max_req_input_len: int = -1, + ): + """Tokenize text with tags.""" + images_list, images_seq_mask, images_spatial_crop = [], [], [] + text_splits = conversation.split(self.image_token) + tokenized_str = [] + for text_sep, image in zip(text_splits, images): + """encode text_sep""" + tokenized_sep = self.encode(text_sep, bos=False, eos=False) + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """select best resolution for anyres""" + if cropping: + best_width, best_height = select_best_resolution( + image.size, self.candidate_resolutions + ) + else: + best_width, best_height = self.image_size, self.image_size + # print(image.size, (best_width, best_height)) # check the select_best_resolutions func + + """process the global view""" + global_view = ImageOps.pad( + image, + (self.image_size, self.image_size), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + images_list.append(self.image_transform(global_view)) + + """process the local views""" + local_view = ImageOps.pad( + image, + (best_width, best_height), + color=tuple(int(x * 255) for x in self.image_transform.mean), + ) + for i in range(0, best_height, self.image_size): + for j in range(0, best_width, self.image_size): + images_list.append( + self.image_transform( + local_view.crop( + (j, i, j + self.image_size, i + self.image_size) + ) + ) + ) + + """record height / width crop num""" + num_width_tiles, num_height_tiles = ( + best_width // self.image_size, + best_height // self.image_size, + ) + images_spatial_crop.append([num_width_tiles, num_height_tiles]) + + """add image tokens""" + h = w = math.ceil( + (self.image_size // self.patch_size) / self.downsample_ratio + ) + # global views tokens h * (w + 1), 1 is for line separator + tokenized_image = [self.image_token_id] * h * (w + 1) + # add a separator between global and local views + tokenized_image += [self.image_token_id] + # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1) + tokenized_image += ( + [self.image_token_id] + * (num_height_tiles * h) + * (num_width_tiles * w + 1) + ) + + tokenized_str += tokenized_image + images_seq_mask += [True] * len(tokenized_image) + # print(width_crop_num, height_crop_num, len(tokenized_image)) # test the correctness of the number of image-related tokens + + """process the last text split""" + tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) + # deal with video, limit with request len + if max_req_input_len > -1: + if max_req_input_len < len(tokenized_sep) + len(tokenized_str) - 1: + rest = max_req_input_len - len(tokenized_sep) - 1 - 1024 + tokenized_str = tokenized_str[:rest] + images_seq_mask = images_seq_mask[:rest] + tokenized_str += tokenized_sep + images_seq_mask += [False] * len(tokenized_sep) + + """add the bos and eos tokens""" + if bos: + tokenized_str = [self.bos_id] + tokenized_str + images_seq_mask = [False] + images_seq_mask + if eos: + tokenized_str = tokenized_str + [self.eos_id] + images_seq_mask = images_seq_mask + [False] + + assert len(tokenized_str) == len( + images_seq_mask + ), f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" + + return tokenized_str, images_list, images_seq_mask, images_spatial_crop + + +class DeepseekVL2VisionEncoderConfig(PretrainedConfig): + model_type: str = "vision" + + model_name: str = "siglip_large_patch16_384" + image_size: int = 384 + patch_size: int = 16 + width: int = 1024 + layers: int = 24 + heads: int = 16 + mlp_ratio: int = 4 + global_pool: str = "map" + ignore_head: bool = True + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + weight_init: str = "skip" + deterministic: bool = False + num_recomputing_layers: int = 0 + + def __init__( + self, + model_name: str = "siglip_large_patch16_384", + image_size: int = 384, + patch_size: int = 16, + width: int = 1024, + layers: int = 24, + heads: int = 16, + mlp_ratio: int = 4, + global_pool: str = "map", + ignore_head: bool = True, + class_token: bool = False, + num_classes: int = 0, + use_checkpoint: bool = False, + **kwargs, + ): + self.model_name = model_name + self.image_size = image_size + self.patch_size = patch_size + self.width = width + self.layers = layers + self.heads = heads + self.mlp_ratio = mlp_ratio + self.global_pool = global_pool + self.ignore_head = ignore_head + self.class_token = class_token + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + + super().__init__(**kwargs) + + +class DeepseekVL2MlpProjectorConfig(PretrainedConfig): + model_type = "mlp_projector" + projector_type: str = "downsample_mlp_gelu" + input_dim: int = 1152 + n_embed: int = 2048 + depth: int = 2 + mlp_ratio: int = 1 + downsample_ratio: int = 2 + token_pooling: bool = False + + def __init__( + self, + projector_type: str = "downsample_mlp_gelu", + input_dim: int = 1152, + n_embed: int = 2048, + depth: int = 2, + mlp_ratio: int = 1, + downsample_ratio: int = 2, + **kwargs, + ): + self.projector_type = projector_type + self.input_dim = input_dim + self.n_embed = n_embed + self.depth = depth + self.mlp_ratio = mlp_ratio + self.downsample_ratio = downsample_ratio + + super().__init__(**kwargs) + + +class DeepseekV2Config(PretrainedConfig): + + model_type = "deepseek_v2" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=102400, + hidden_size=4096, + intermediate_size=11008, + moe_intermediate_size=1407, + num_hidden_layers=30, + num_attention_heads=32, + num_key_value_heads=32, + n_shared_experts=None, + n_routed_experts=None, + ep_size=1, + routed_scaling_factor=1.0, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=64, + v_head_dim=128, + qk_nope_head_dim=128, + topk_method="gready", + n_group=None, + topk_group=None, + num_experts_per_tok=None, + moe_layer_freq=1, + first_k_dense_replace=0, + norm_topk_prob=False, + scoring_func="softmax", + aux_loss_alpha=0.001, + seq_aux=True, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=None, + bos_token_id=100000, + eos_token_id=100001, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + use_mla=True, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.moe_intermediate_size = moe_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.ep_size = ep_size + self.routed_scaling_factor = routed_scaling_factor + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.topk_method = topk_method + self.n_group = n_group + self.topk_group = topk_group + self.num_experts_per_tok = num_experts_per_tok + self.moe_layer_freq = moe_layer_freq + self.first_k_dense_replace = first_k_dense_replace + self.norm_topk_prob = norm_topk_prob + self.scoring_func = scoring_func + self.aux_loss_alpha = aux_loss_alpha + self.seq_aux = seq_aux + # for backward compatibility + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = float(rms_norm_eps) + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.use_mla = use_mla + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class DeepseekVL2Config(PretrainedConfig): + model_type = "deepseek_vl_v2" + vision_config: DeepseekVL2VisionEncoderConfig + projector_config: DeepseekVL2MlpProjectorConfig + language_config: DeepseekV2Config + + tile_tag: str = "2D" + global_view_pos: str = "head" + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),) + + def __init__( + self, + tile_tag: str = "tile_tag", + global_view_pos: str = "head", + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384),), + **kwargs, + ): + super().__init__(**kwargs) + + vision_config = kwargs.get("vision_config", {}) + self.vision_config = DeepseekVL2VisionEncoderConfig(**vision_config) + + projector_config = kwargs.get("projector_config", {}) + self.projector_config = DeepseekVL2MlpProjectorConfig(**projector_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, DeepseekV2Config): + self.language_config = language_config + else: + self.language_config = DeepseekV2Config(**language_config) + + self.tile_tag = tile_tag + self.global_view_pos = global_view_pos + self.candidate_resolutions = candidate_resolutions + self.architectures = ["DeepseekVL2ForCausalLM"] + + +AutoProcessor.register(DeepseekVL2Config, DeepseekVLV2Processor) diff --git a/python/sglang/srt/configs/device_config.py b/python/sglang/srt/configs/device_config.py new file mode 100644 index 000000000..3b9d3a1ed --- /dev/null +++ b/python/sglang/srt/configs/device_config.py @@ -0,0 +1,17 @@ +import logging +from typing import Optional + +import torch + +logger = logging.getLogger(__name__) + + +class DeviceConfig: + device: Optional[torch.device] + + def __init__(self, device: str = "cuda") -> None: + if device in ["cuda", "xpu", "hpu", "cpu", "npu"]: + self.device_type = device + else: + raise RuntimeError(f"Not supported device type: {device}") + self.device = torch.device(self.device_type) diff --git a/python/sglang/srt/configs/exaone.py b/python/sglang/srt/configs/exaone.py new file mode 100644 index 000000000..7b0a2d290 --- /dev/null +++ b/python/sglang/srt/configs/exaone.py @@ -0,0 +1,195 @@ +# coding=utf-8 +# Copyright 2024 The LG AI Research EXAONE Lab. All rights reserved. +# Copyright 2024 The LG CNS AI Engineering Team. +# Copyright 2023-2024 SGLang Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" EXAONE model configuration """ +from typing import Any, Dict + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, Any] = {} + + +# ruff: noqa: E501 +class ExaoneConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a :class:`~transformers.ExaoneModel`. It is used to + instantiate a EXAONE model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the Exaone + + Configuration objects inherit from :class:`~transformers.PretrainedConfig` and can be used to control the model + outputs. Read the documentation from :class:`~transformers.PretrainedConfig` for more information. + + + Args: + vocab_size (:obj:`int`, `optional`, defaults to 102400): + Vocabulary size of the EXAONE model. Defines the number of different tokens that can be represented by the + :obj:`inputs_ids` passed when calling :class:`~transformers.ExaoneModel`. Vocabulary size of the model. + Defines the different tokens that can be represented by the `inputs_ids` passed to the forward method of + :class:`~transformers.EXAONEModel`. + max_position_embeddings (:obj:`int`, `optional`, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + hidden_size (:obj:`int`, `optional`, defaults to 2048): + Dimensionality of the encoder layers and the pooler layer. + num_layers (:obj:`int`, `optional`, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (:obj:`int`, `optional`, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (:obj:`int`, `optional`): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + intermediate_size (:obj:`int`, `optional`, defaults to `hidden_size * 4`): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + activation_function (:obj:`str` or :obj:`function`, `optional`, defaults to :obj:`"silu"`): + The non-linear activation function (function or string) in the decoder. + rope_theta (:obj:`float`, `optional`, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (:obj:`Dict`, `optional`): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (:obj:`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (:obj:`float`, `optional`): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (:obj:`int`, `optional`): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (:obj:`float`, `optional`): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (:obj:`float`, `optional`): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (:obj:`List[float]`, `optional`): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (:obj:`float`, `optional`): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + embed_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (:obj:`float`, `optional`, defaults to 0.0): + The dropout ratio for the attention probabilities. + layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5): + The epsilon used by the layer normalization layers. + initializer_range (:obj:`float`, `optional`, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if ``configs.is_decoder=True``. + bos_token_id (:obj:`int`, `optional`, defaults to 0): + Beginning of stream token id. + eos_token_id (:obj:`int`, `optional`, defaults to 2): + End of stream token id. + tie_word_embeddings (:obj:`bool`, `optional`, defaults to :obj:`True`): + Whether to tie weight embeddings + gradient_checkpointing (:obj:`bool`, `optional`, defaults to :obj:`False`): + If True, use gradient checkpointing to save memory at the expense of slower backward pass. + + Example:: + + >>> from transformers import EXAONEModel, ExaoneConfig + + >>> # Initializing a EXAONE configuration + >>> configuration = ExaoneConfig() + + >>> # Initializing a model from configuration + >>> model = EXAONEModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.configs + """ + + model_type = "exaone" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_hidden_layers": "num_layers"} + + def __init__( + self, + vocab_size=102400, + max_position_embeddings=2048, + hidden_size=2048, + num_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + intermediate_size=None, + activation_function="silu", + rope_theta=10000.0, + rope_scaling=None, + embed_dropout=0.0, + attention_dropout=0.0, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + use_cache=True, + bos_token_id=0, + eos_token_id=2, + tie_word_embeddings=True, + **kwargs + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_layers = num_layers + self.num_attention_heads = num_attention_heads + self.num_hidden_layers = num_layers + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + if intermediate_size: + self.intermediate_size = intermediate_size + else: + self.intermediate_size = hidden_size * 4 + self.activation_function = activation_function + self.embed_dropout = embed_dropout + self.attention_dropout = attention_dropout + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs + ) diff --git a/python/sglang/srt/configs/internvl.py b/python/sglang/srt/configs/internvl.py new file mode 100644 index 000000000..3ba9c61c1 --- /dev/null +++ b/python/sglang/srt/configs/internvl.py @@ -0,0 +1,706 @@ +import copy +import os +from shutil import copyfile +from typing import Any, Dict, List, Optional, Tuple, Union + +import sentencepiece as spm +from transformers import ( + TOKENIZER_MAPPING, + GptOssConfig, + LlamaConfig, + PretrainedConfig, + PreTrainedTokenizer, + Qwen2Config, + Qwen3Config, + Qwen3MoeConfig, +) + +from sglang.utils import logger + +# Copied from: https://github.com/OpenGVLab/InternVL/blob/34a81000402bf8f716bab8c9b57aff1f6b436bd0/internvl_chat/internvl/model/internvl_chat/configuration_internvl_chat.py#L21 + + +VOCAB_FILES_NAMES = {"vocab_file": "./tokenizer.model"} + +PRETRAINED_VOCAB_FILES_MAP = {} + + +# Modified from transformers.model.llama.configuration_llama.LlamaConfig +class InternLM2Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternLM2Model`]. It is used to instantiate + an InternLM2 model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the InternLM2-7B. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 32000): + Vocabulary size of the InternLM2 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`InternLM2Model`] + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 11008): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 2048): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-12): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings(`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + Example: + + """ + + model_type = "internlm2" + _auto_class = "AutoConfig" + + def __init__( # pylint: disable=W0102 + self, + vocab_size=103168, + hidden_size=4096, + intermediate_size=11008, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + hidden_act="silu", + max_position_embeddings=2048, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + tie_word_embeddings=False, + bias=True, + rope_theta=10000, + rope_scaling=None, + attn_implementation="eager", + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.bias = bias + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_validation() + + self.attn_implementation = attn_implementation + if self.attn_implementation is None: + self.attn_implementation = "eager" + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2: + raise ValueError( + "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_factor = self.rope_scaling.get("factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]: + raise ValueError( + f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}" + ) + if ( + rope_scaling_factor is None + or not isinstance(rope_scaling_factor, (float, int)) + or rope_scaling_factor < 1.0 + ): + raise ValueError( + f"`rope_scaling`'s factor field must be a float|int >= 1, got {rope_scaling_factor=}, {type(rope_scaling_factor)=}" + ) + if isinstance(rope_scaling_factor, int): + rope_scaling_factor = float(rope_scaling_factor) + + +class InternVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to + instantiate a vision encoder according to the specified arguments, defining the model architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + num_channels (`int`, *optional*, defaults to 3): + Number of color channels in the input images (e.g., 3 for RGB). + patch_size (`int`, *optional*, defaults to 14): + The size (resolution) of each patch. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + qkv_bias (`bool`, *optional*, defaults to `False`): + Whether to add a bias to the queries and values in the self-attention layers. + hidden_size (`int`, *optional*, defaults to 3200): + Dimensionality of the encoder layers and the pooler layer. + num_attention_heads (`int`, *optional*, defaults to 25): + Number of attention heads for each attention layer in the Transformer encoder. + intermediate_size (`int`, *optional*, defaults to 12800): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + qk_normalization (`bool`, *optional*, defaults to `True`): + Whether to normalize the queries and keys in the self-attention layers. + num_hidden_layers (`int`, *optional*, defaults to 48): + Number of hidden layers in the Transformer encoder. + use_flash_attn (`bool`, *optional*, defaults to `True`): + Whether to use flash attention mechanism. + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. + layer_norm_eps (`float`, *optional*, defaults to 1e-6): + The epsilon used by the layer normalization layers. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + drop_path_rate (`float`, *optional*, defaults to 0.0): + Dropout rate for stochastic depth. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float`, *optional*, defaults to 0.1): + A factor for layer scale. + """ + + model_type = "intern_vit_6b" + + def __init__( + self, + num_channels=3, + patch_size=14, + image_size=224, + qkv_bias=False, + hidden_size=3200, + num_attention_heads=25, + intermediate_size=12800, + qk_normalization=True, + num_hidden_layers=48, + use_flash_attn=True, + hidden_act="gelu", + layer_norm_eps=1e-6, + dropout=0.0, + drop_path_rate=0.0, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=0.1, + **kwargs, + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.dropout = dropout + self.drop_path_rate = drop_path_rate + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.qkv_bias = qkv_bias + self.qk_normalization = qk_normalization + self.use_flash_attn = use_flash_attn + + @classmethod + def from_pretrained( + cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs + ) -> "PretrainedConfig": + config_dict, kwargs = cls.get_config_dict( + pretrained_model_name_or_path, **kwargs + ) + + if "vision_config" in config_dict: + config_dict = config_dict["vision_config"] + + if ( + "model_type" in config_dict + and hasattr(cls, "model_type") + and config_dict["model_type"] != cls.model_type + ): + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class InternVLChatConfig(PretrainedConfig): + model_type = "internvl_chat" + is_composition = True + + def __init__( + self, + vision_config=None, + llm_config=None, + use_backbone_lora=0, + use_llm_lora=0, + pad2square=False, + select_layer=-1, + force_image_size=None, + downsample_ratio=0.5, + template=None, + dynamic_image_size=False, + use_thumbnail=False, + ps_version="v1", + min_dynamic_patch=1, + max_dynamic_patch=6, + **kwargs, + ): + super().__init__(**kwargs) + + if vision_config is None: + vision_config = {"architectures": ["InternVisionModel"]} + logger.info( + "vision_config is None. Initializing the InternVisionConfig with default values." + ) + + if llm_config is None: + llm_config = {"architectures": ["InternLM2ForCausalLM"]} + logger.info( + "llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`)." + ) + + self.vision_config = InternVisionConfig(**vision_config) + if llm_config.get("architectures")[0] == "LlamaForCausalLM": + self.llm_config = LlamaConfig(**llm_config) + elif llm_config.get("architectures")[0] == "InternLM2ForCausalLM": + self.llm_config = InternLM2Config(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen2ForCausalLM": + self.llm_config = Qwen2Config(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen3MoeForCausalLM": + self.llm_config = Qwen3MoeConfig(**llm_config) + elif llm_config.get("architectures")[0] == "Qwen3ForCausalLM": + self.llm_config = Qwen3Config(**llm_config) + elif llm_config.get("architectures")[0] == "GptOssForCausalLM": + self.llm_config = GptOssConfig(**llm_config) + else: + raise ValueError( + "Unsupported architecture: {}".format( + llm_config.get("architectures")[0] + ) + ) + + self.use_backbone_lora = use_backbone_lora + self.use_llm_lora = use_llm_lora + self.pad2square = pad2square + self.select_layer = select_layer + self.force_image_size = force_image_size + self.downsample_ratio = downsample_ratio + self.template = template + self.dynamic_image_size = dynamic_image_size + self.use_thumbnail = use_thumbnail + self.ps_version = ps_version # pixel shuffle version + self.min_dynamic_patch = min_dynamic_patch + self.max_dynamic_patch = max_dynamic_patch + + self.hidden_size = self.llm_config.hidden_size + # By default, we use tie_word_embeddings=False for models of all sizes. + self.tie_word_embeddings = False + self.llm_config.tie_word_embeddings = self.tie_word_embeddings + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["vision_config"] = self.vision_config.to_dict() + output["llm_config"] = self.llm_config.to_dict() + output["model_type"] = self.__class__.model_type + output["use_backbone_lora"] = self.use_backbone_lora + output["use_llm_lora"] = self.use_llm_lora + output["select_layer"] = self.select_layer + output["force_image_size"] = self.force_image_size + output["downsample_ratio"] = self.downsample_ratio + output["template"] = self.template + output["dynamic_image_size"] = self.dynamic_image_size + output["use_thumbnail"] = self.use_thumbnail + output["ps_version"] = self.ps_version + output["min_dynamic_patch"] = self.min_dynamic_patch + output["max_dynamic_patch"] = self.max_dynamic_patch + + return output + + +# # Modified from transformers.model.llama.tokenization_llama_fast.LlamaTokenizerFast -> InternLM2TokenizerFast +# class InternLM2TokenizerFast(PreTrainedTokenizerFast): +# vocab_files_names = VOCAB_FILES_NAMES +# slow_tokenizer_class = InternLM2Tokenizer +# padding_side = 'left' +# model_input_names = ['input_ids', 'attention_mask'] +# _auto_class = 'AutoTokenizer' +# +# def __init__( +# self, +# vocab_file, +# unk_token='', +# bos_token='', +# eos_token='', +# pad_token='', +# sp_model_kwargs: Optional[Dict[str, Any]] = None, +# add_bos_token=True, +# add_eos_token=False, +# decode_with_prefix_space=False, +# clean_up_tokenization_spaces=False, +# **kwargs, +# ): +# super().__init__( +# vocab_file=vocab_file, +# unk_token=unk_token, +# bos_token=bos_token, +# eos_token=eos_token, +# pad_token=pad_token, +# sp_model_kwargs=sp_model_kwargs, +# add_bos_token=add_bos_token, +# add_eos_token=add_eos_token, +# decode_with_prefix_space=decode_with_prefix_space, +# clean_up_tokenization_spaces=clean_up_tokenization_spaces, +# **kwargs, +# ) +# self._add_bos_token = add_bos_token +# self._add_eos_token = add_eos_token +# self.update_post_processor() +# self.vocab_file = vocab_file +# +# @property +# def can_save_slow_tokenizer(self) -> bool: +# return os.path.isfile(self.vocab_file) if self.vocab_file else False +# +# def update_post_processor(self): +# """ +# Updates the underlying post processor with the current `bos_token` and `eos_token`. +# """ +# bos = self.bos_token +# bos_token_id = self.bos_token_id +# if bos is None and self.add_bos_token: +# raise ValueError('add_bos_token = True but bos_token = None') +# +# eos = self.eos_token +# eos_token_id = self.eos_token_id +# if eos is None and self.add_eos_token: +# raise ValueError('add_eos_token = True but eos_token = None') +# +# single = f"{(bos + ':0 ') if self.add_bos_token else ''}$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}" +# pair = f"{single}{(' ' + bos + ':1') if self.add_bos_token else ''} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}" +# +# special_tokens = [] +# if self.add_bos_token: +# special_tokens.append((bos, bos_token_id)) +# if self.add_eos_token: +# special_tokens.append((eos, eos_token_id)) +# self._tokenizer.post_processor = processors.TemplateProcessing( +# single=single, pair=pair, special_tokens=special_tokens +# ) +# +# @property +# def add_eos_token(self): +# return self._add_eos_token +# +# @property +# def add_bos_token(self): +# return self._add_bos_token +# +# @add_eos_token.setter +# def add_eos_token(self, value): +# self._add_eos_token = value +# self.update_post_processor() +# +# @add_bos_token.setter +# def add_bos_token(self, value): +# self._add_bos_token = value +# self.update_post_processor() +# +# def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: +# if not self.can_save_slow_tokenizer: +# raise ValueError( +# 'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow ' +# 'tokenizer.' +# ) +# +# if not os.path.isdir(save_directory): +# logger.error(f'Vocabulary path ({save_directory}) should be a directory') +# return +# out_vocab_file = os.path.join( +# save_directory, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] +# ) +# +# if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): +# copyfile(self.vocab_file, out_vocab_file) +# +# return (out_vocab_file,) + + +# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer +class InternLM2Tokenizer(PreTrainedTokenizer): + """ + Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + model_input_names = ["input_ids", "attention_mask"] + _auto_class = "AutoTokenizer" + + def __init__( + self, + vocab_file, + unk_token="", + bos_token="", + eos_token="", + pad_token="", + sp_model_kwargs: Optional[Dict[str, Any]] = None, + add_bos_token=True, + add_eos_token=False, + decode_with_prefix_space=False, + clean_up_tokenization_spaces=False, + **kwargs, + ): + print("register succeed") + self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs + self.vocab_file = vocab_file + self.add_bos_token = add_bos_token + self.add_eos_token = add_eos_token + self.decode_with_prefix_space = decode_with_prefix_space + self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs) + self.sp_model.Load(vocab_file) + self._no_prefix_space_tokens = None + super().__init__( + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + pad_token=pad_token, + clean_up_tokenization_spaces=clean_up_tokenization_spaces, + **kwargs, + ) + + @property + def no_prefix_space_tokens(self): + if self._no_prefix_space_tokens is None: + vocab = self.convert_ids_to_tokens(list(range(self.vocab_size))) + self._no_prefix_space_tokens = { + i for i, tok in enumerate(vocab) if not tok.startswith("▁") + } + return self._no_prefix_space_tokens + + @property + def vocab_size(self): + """Returns vocab size""" + return self.sp_model.get_piece_size() + + @property + def bos_token_id(self) -> Optional[int]: + return self.sp_model.bos_id() + + @property + def eos_token_id(self) -> Optional[int]: + return self.sp_model.eos_id() + + def get_vocab(self): + """Returns vocab as a dict""" + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text): + """Returns a tokenized string.""" + return self.sp_model.encode(text, out_type=str) + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.sp_model.piece_to_id(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + token = self.sp_model.IdToPiece(index) + return token + + def _maybe_add_prefix_space(self, tokens, decoded): + if tokens and tokens[0] not in self.no_prefix_space_tokens: + return " " + decoded + else: + return decoded + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + current_sub_tokens = [] + out_string = "" + prev_is_special = False + for token in tokens: + # make sure that special tokens are not decoded using sentencepiece model + if token in self.all_special_tokens: + if not prev_is_special: + out_string += " " + out_string += self.sp_model.decode(current_sub_tokens) + token + prev_is_special = True + current_sub_tokens = [] + else: + current_sub_tokens.append(token) + prev_is_special = False + out_string += self.sp_model.decode(current_sub_tokens) + out_string = self.clean_up_tokenization(out_string) + out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string) + return out_string[1:] + + def save_vocabulary( + self, save_directory, filename_prefix: Optional[str] = None + ) -> Tuple[str]: + """ + Save the vocabulary and special tokens file to a directory. + + Args: + save_directory (`str`): + The directory in which to save the vocabulary. + + Returns: + `Tuple(str)`: Paths to the files saved. + """ + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + out_vocab_file = os.path.join( + save_directory, + (filename_prefix + "-" if filename_prefix else "") + + VOCAB_FILES_NAMES["vocab_file"], + ) + + if os.path.abspath(self.vocab_file) != os.path.abspath( + out_vocab_file + ) and os.path.isfile(self.vocab_file): + copyfile(self.vocab_file, out_vocab_file) + elif not os.path.isfile(self.vocab_file): + with open(out_vocab_file, "wb") as fi: + content_spiece_model = self.sp_model.serialized_model_proto() + fi.write(content_spiece_model) + + return (out_vocab_file,) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + if self.add_bos_token: + bos_token_ids = [self.bos_token_id] + else: + bos_token_ids = [] + + output = bos_token_ids + token_ids_0 + + if token_ids_1 is not None: + output = output + token_ids_1 + + if self.add_eos_token: + output = output + [self.eos_token_id] + + return output + + def get_special_tokens_mask( + self, + token_ids_0: List[int], + token_ids_1: Optional[List[int]] = None, + already_has_special_tokens: bool = False, + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, + token_ids_1=token_ids_1, + already_has_special_tokens=True, + ) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make + use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + eos = [self.eos_token_id] + + if token_ids_1 is None: + return len(token_ids_0 + eos) * [0] + return len(token_ids_0 + eos + token_ids_1 + eos) * [0] + + +TOKENIZER_MAPPING.register( + InternVLChatConfig, (InternLM2Tokenizer, None), exist_ok=True +) diff --git a/python/sglang/srt/configs/janus_pro.py b/python/sglang/srt/configs/janus_pro.py new file mode 100644 index 000000000..d574953e9 --- /dev/null +++ b/python/sglang/srt/configs/janus_pro.py @@ -0,0 +1,634 @@ +# Adapted from: +# https://github.com/deepseek-ai/Janus/tree/main/janus/models + +from dataclasses import dataclass +from typing import Dict, List, Tuple, Union + +import numpy as np +import PIL +import torch +from PIL.Image import Image +from transformers import ( + BaseImageProcessor, + BatchFeature, + LlamaConfig, + LlamaTokenizerFast, + PretrainedConfig, + ProcessorMixin, +) +from transformers.image_utils import to_numpy_array + +from sglang.srt.configs.utils import register_image_processor, register_processor +from sglang.srt.multimodal.mm_utils import expand2square + + +class DictToObject(dict): + def __init__(self, dictionary): + super(self).__init__(dictionary) + + for key, value in dictionary.items(): + if isinstance(value, dict): + value = DictToObject(value) + setattr(self, key, value) + + +class VisionConfig(PretrainedConfig): + model_type = "vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenAlignerConfig(PretrainedConfig): + model_type = "gen_aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenHeadConfig(PretrainedConfig): + model_type = "gen_head" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class AlignerConfig(PretrainedConfig): + model_type = "aligner" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +class GenVisionConfig(PretrainedConfig): + model_type = "gen_vision" + cls: str = "" + params = {} + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self.cls = kwargs.get("cls", "") + if not isinstance(self.cls, str): + self.cls = self.cls.__name__ + + self.params = kwargs.get("params", {}) + + +@dataclass +class SigLIPVisionCfg: + width: int = 1152 + layers: Union[Tuple[int, int, int, int], int] = 27 + heads: int = 16 + patch_size: int = 14 + image_size: Union[Tuple[int, int], int] = 336 + global_pool: str = "map" + mlp_ratio: float = 3.7362 + class_token: bool = False + num_classes: int = 0 + use_checkpoint: bool = False + + +class MultiModalityConfig(PretrainedConfig): + model_type = "multi_modality" + vision_config: VisionConfig + aligner_config: AlignerConfig + + gen_vision_config: GenVisionConfig + gen_aligner_config: GenAlignerConfig + gen_head_config: GenHeadConfig + + language_config: LlamaConfig + + def __init__(self, **kwargs): + super().__init__(**kwargs) + vision_config = kwargs.get("vision_config", {}) + self.vision_config = VisionConfig(**vision_config) + + aligner_config = kwargs.get("aligner_config", {}) + self.aligner_config = AlignerConfig(**aligner_config) + + gen_vision_config = kwargs.get("gen_vision_config", {}) + self.gen_vision_config = GenVisionConfig(**gen_vision_config) + + gen_aligner_config = kwargs.get("gen_aligner_config", {}) + self.gen_aligner_config = GenAlignerConfig(**gen_aligner_config) + + gen_head_config = kwargs.get("gen_head_config", {}) + self.gen_head_config = GenHeadConfig(**gen_head_config) + + language_config = kwargs.get("language_config", {}) + if isinstance(language_config, LlamaConfig): + self.language_config = language_config + else: + self.language_config = LlamaConfig(**language_config) + + +class VLMImageProcessor(BaseImageProcessor): + model_input_names = ["pixel_values"] + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + super().__init__(**kwargs) + + self.image_size = image_size + self.rescale_factor = rescale_factor + self.image_mean = image_mean + self.image_std = image_std + self.min_size = min_size + self.do_normalize = do_normalize + + if image_mean is None: + self.background_color = (127, 127, 127) + else: + self.background_color = tuple([int(x * 255) for x in image_mean]) + + def resize(self, pil_img: Image) -> np.ndarray: + """ + + Args: + pil_img (PIL.Image): [H, W, 3] in PIL.Image in RGB + + Returns: + x (np.ndarray): [3, self.image_size, self.image_size] + """ + + width, height = pil_img.size + max_size = max(width, height) + + size = [ + max(int(height / max_size * self.image_size), self.min_size), + max(int(width / max_size * self.image_size), self.min_size), + ] + + if width <= 0 or height <= 0 or size[0] <= 0 or size[1] <= 0: + # print(f"orig size = {pil_img.size}, new size = {size}") + raise ValueError("Invalid size!") + + def resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ): + if isinstance(size, int): + w, h = pil_img.size + if (w <= h and w == size) or (h <= w and h == size): + return pil_img + if w < h: + ow = size + oh = int(size * h / w) + else: + oh = size + ow = int(size * w / h) + size = (ow, oh) + else: + size = (size[1], size[0]) + + return pil_img.resize( + size, resample=interpolation, reducing_gap=None if antialias else 3.0 + ) + + pil_img = resize( + pil_img, size, interpolation=PIL.Image.Resampling.BICUBIC, antialias=True + ) + + pil_img = expand2square(pil_img, self.background_color) + x = to_numpy_array(pil_img) + + # [H, W, 3] -> [3, H, W] + x = np.transpose(x, (2, 0, 1)) + + return x + + def preprocess(self, images, return_tensors: str = "pt", **kwargs) -> BatchFeature: + # resize and pad to [self.image_size, self.image_size] + # then convert from [H, W, 3] to [3, H, W] + if not isinstance(images, list): + images = [images] + images: List[np.ndarray] = [self.resize(image) for image in images] + images = [image[:3, ...] for image in images] + + # rescale from [0, 255] -> [0, 1] + images = [ + self.rescale( + image=image, + scale=self.rescale_factor, + input_data_format="channels_first", + ) + for image in images + ] + + # normalize + if self.do_normalize: + images = [ + self.normalize( + image=image, + mean=self.image_mean, + std=self.image_std, + input_data_format="channels_first", + ) + for image in images + ] + data = {"pixel_values": images} + return BatchFeature(data=data, tensor_type=return_tensors) + + @property + def default_shape(self): + return [3, self.image_size, self.image_size] + + +class DictOutput(object): + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __getitem__(self, item): + return self.__dict__[item] + + def __contains__(self, key): + return key in self.__dict__ + + def __setitem__(self, key, value): + self.__dict__[key] = value + + +@dataclass +class VLChatProcessorOutput(DictOutput): + sft_format: str + input_ids: torch.Tensor + pixel_values: torch.Tensor + num_image_tokens: torch.IntTensor + + def __len__(self): + return len(self.input_ids) + + +@dataclass +class BatchedVLChatProcessorOutput(DictOutput): + sft_format: List[str] + input_ids: torch.Tensor + pixel_values: torch.Tensor + attention_mask: torch.Tensor + images_seq_mask: torch.BoolTensor + images_emb_mask: torch.BoolTensor + + +# FIXME: had to place Official Processor here, since image_processor module would not be imported in all threads, +# hence AutoProcessor registration would not be affective in some cases +class VLChatProcessor(ProcessorMixin): + image_processor_class = "AutoImageProcessor" + tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") + + attributes = ["image_processor", "tokenizer"] + + def __init__( + self, + image_processor: VLMImageProcessor, + tokenizer: LlamaTokenizerFast, + image_tag: str = "", + image_start_tag: str = "", + image_end_tag: str = "", + pad_tag: str = "<｜▁pad▁｜>", + num_image_tokens: int = 576, + add_special_token: bool = False, + sft_format: str = "deepseek", + mask_prompt: bool = True, + ignore_id: int = -100, + **kwargs, + ): + self.image_processor = image_processor + self.tokenizer = tokenizer + + image_id = self.tokenizer.vocab.get(image_tag) + if image_id is None: + special_tokens = [image_tag] + special_tokens_dict = {"additional_special_tokens": special_tokens} + self.tokenizer.add_special_tokens(special_tokens_dict) + # print(f"Add image tag = {image_tag} to the tokenizer") + + self.image_tag = image_tag + self.image_start_tag = image_start_tag + self.image_end_tag = image_end_tag + self.pad_tag = pad_tag + + self.num_image_tokens = num_image_tokens + self.add_special_token = add_special_token + self.sft_format = sft_format + self.ignore_id = ignore_id + + super().__init__( + image_processor, + tokenizer, + **kwargs, + ) + + @property + def image_token(self): + return self.image_tag + + @property + def image_id(self) -> int: + image_id = self.tokenizer.vocab.get(self.image_tag) + return image_id + + @property + def image_start_id(self): + image_start_id = self.tokenizer.vocab.get(self.image_start_tag) + return image_start_id + + @property + def image_end_id(self): + image_end_id = self.tokenizer.vocab.get(self.image_end_tag) + return image_end_id + + @property + def image_start_token(self): + return self.image_start_tag + + @property + def image_end_token(self): + return self.image_end_tag + + @property + def pad_id(self): + pad_id = self.tokenizer.vocab.get(self.pad_tag) + return pad_id + + def add_image_token( + self, + image_indices: List[int], + input_ids: torch.LongTensor, + ): + """ + + Args: + image_indices (List[int]): [index_0, index_1, ..., index_j] + input_ids (torch.LongTensor): [N] + + Returns: + input_ids (torch.LongTensor): [N + image tokens] + num_image_tokens (torch.IntTensor): [n_images] + """ + + input_slices = [] + + start = 0 + for index in image_indices: + if self.add_special_token: + end = index + 1 + else: + end = index + + # original text tokens + input_slices.append(input_ids[start:end]) + + # add boi, image tokens, eoi and set the mask as False + input_slices.append(self.image_start_id * torch.ones((1), dtype=torch.long)) + input_slices.append( + self.image_id * torch.ones((self.num_image_tokens,), dtype=torch.long) + ) + input_slices.append(self.image_end_id * torch.ones((1), dtype=torch.long)) + start = index + 1 + + # the left part + input_slices.append(input_ids[start:]) + + # concat all slices + input_ids = torch.cat(input_slices, dim=0) + num_image_tokens = torch.IntTensor([self.num_image_tokens] * len(image_indices)) + + return input_ids, num_image_tokens + + def process_one( + self, + prompt: str = None, + images: List[Image] = None, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + images (List[ImageType]): the list of images; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - target_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + sft_format = prompt + # tokenize + input_ids = self.tokenizer.encode(sft_format) + input_ids = torch.LongTensor(input_ids) + + # add image tokens to the input_ids + image_token_mask: torch.Tensor = (input_ids == self.image_id).to(torch.bool) + image_indices = image_token_mask.nonzero() + input_ids, num_image_tokens = self.add_image_token( + image_indices=image_indices, + input_ids=input_ids, + ) + + # load images + images_outputs = self.image_processor(images, return_tensors="pt") + + prepare = VLChatProcessorOutput( + sft_format=sft_format, + input_ids=input_ids, + pixel_values=images_outputs.pixel_values, + num_image_tokens=num_image_tokens, + ) + + return prepare + + def __call__( + self, + *, + prompt: str = None, + conversations: List[Dict[str, str]] = None, + images: List[Image] = None, + force_batchify: bool = True, + **kwargs, + ): + """ + + Args: + prompt (str): the formatted prompt; + conversations (List[Dict]): conversations with a list of messages; + images (List[ImageType]): the list of images; + force_batchify (bool): force batchify the inputs; + **kwargs: + + Returns: + outputs (BaseProcessorOutput): the output of the processor, + - input_ids (torch.LongTensor): [N + image tokens] + - images (torch.FloatTensor): [n_images, 3, H, W] + - image_id (int): the id of the image token + - num_image_tokens (List[int]): the number of image tokens + """ + + prepare = self.process_one( + prompt=prompt, conversations=conversations, images=images + ) + + if force_batchify: + prepare = self.batchify([prepare]) + + return prepare + + def batchify( + self, prepare_list: List[VLChatProcessorOutput] + ) -> BatchedVLChatProcessorOutput: + """ + Preprocesses the inputs for multimodal inference. + + Args: + prepare_list (List[VLChatProcessorOutput]): A list of VLChatProcessorOutput. + + Returns: + BatchedVLChatProcessorOutput: A dictionary of the inputs to use for multimodal inference. + """ + + batch_size = len(prepare_list) + sft_format = [] + n_images = [] + seq_lens = [] + for prepare in prepare_list: + n_images.append(len(prepare.num_image_tokens)) + seq_lens.append(len(prepare)) + + input_token_max_len = max(seq_lens) + max_n_images = max(1, max(n_images)) + + batched_input_ids = torch.full( + (batch_size, input_token_max_len), self.pad_id + ).long() # FIXME + batched_attention_mask = torch.zeros((batch_size, input_token_max_len)).long() + batched_pixel_values = torch.zeros( + (batch_size, max_n_images, *self.image_processor.default_shape) + ).float() + batched_images_seq_mask = torch.zeros((batch_size, input_token_max_len)).bool() + batched_images_emb_mask = torch.zeros( + (batch_size, max_n_images, self.num_image_tokens) + ).bool() + + for i, prepare in enumerate(prepare_list): + input_ids = prepare.input_ids + seq_len = len(prepare) + n_image = len(prepare.num_image_tokens) + # left-padding + batched_attention_mask[i, -seq_len:] = 1 + batched_input_ids[i, -seq_len:] = torch.LongTensor(input_ids) + batched_images_seq_mask[i, -seq_len:] = input_ids == self.image_id + + if n_image > 0: + batched_pixel_values[i, :n_image] = prepare.pixel_values + for j, n_image_tokens in enumerate(prepare.num_image_tokens): + batched_images_emb_mask[i, j, :n_image_tokens] = True + + sft_format.append(prepare.sft_format) + + batched_prepares = BatchedVLChatProcessorOutput( + input_ids=batched_input_ids, + attention_mask=batched_attention_mask, + pixel_values=batched_pixel_values, + images_seq_mask=batched_images_seq_mask, + images_emb_mask=batched_images_emb_mask, + sft_format=sft_format, + ) + + return batched_prepares + + +class VLMImageProcessorConfig(PretrainedConfig): + model_type = "deepseek_vlm" + image_size: int + min_size: int + image_mean: Union[Tuple[float, float, float], List[float]] + image_std: Union[Tuple[float, float, float], List[float]] + rescale_factor: float + do_normalize: bool + + def __init__( + self, + image_size: int, + min_size: int = 14, + image_mean: Union[Tuple[float, float, float], List[float]] = ( + 0.48145466, + 0.4578275, + 0.40821073, + ), + image_std: Union[Tuple[float, float, float], List[float]] = ( + 0.26862954, + 0.26130258, + 0.27577711, + ), + rescale_factor: float = 1.0 / 255.0, + do_normalize: bool = True, + **kwargs, + ): + self.image_size = image_size + self.min_size = min_size + self.image_mean = image_mean + self.image_std = image_std + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + + super().__init__(**kwargs) + + +register_processor(MultiModalityConfig, VLChatProcessor) +register_image_processor(MultiModalityConfig, VLMImageProcessor) diff --git a/python/sglang/srt/configs/kimi_vl.py b/python/sglang/srt/configs/kimi_vl.py new file mode 100644 index 000000000..3c7d20f59 --- /dev/null +++ b/python/sglang/srt/configs/kimi_vl.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py +from typing import Optional, Union + +from transformers.configuration_utils import PretrainedConfig + +from sglang.srt.configs.deepseekvl2 import DeepseekV2Config +from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig + + +class KimiVLConfig(PretrainedConfig): + model_type = "kimi_vl" + + def __init__( + self, + vision_config: Optional[Union[dict, MoonViTConfig]] = None, + text_config: Optional[Union[dict, DeepseekV2Config]] = None, + ignore_index: int = -100, + media_placeholder_token_id: int = 163605, + pad_token_id: int = 0, + **kwargs + ): + if vision_config is None: + vision_config = MoonViTConfig() + elif isinstance(vision_config, dict): + vision_config = MoonViTConfig(**vision_config) + self.vision_config = vision_config + + if text_config is None: + text_config = DeepseekV2Config() + elif isinstance(text_config, dict): + text_config = DeepseekV2Config(**text_config) + self.text_config = text_config + + self.ignore_index = ignore_index + self.media_placeholder_token_id = media_placeholder_token_id + + super().__init__(pad_token_id=pad_token_id, **kwargs) diff --git a/python/sglang/srt/configs/kimi_vl_moonvit.py b/python/sglang/srt/configs/kimi_vl_moonvit.py new file mode 100644 index 000000000..166809eb6 --- /dev/null +++ b/python/sglang/srt/configs/kimi_vl_moonvit.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://huggingface.co/moonshotai/Kimi-VL-A3B-Instruct/blob/main/configuration_kimi_vl.py +from transformers.configuration_utils import PretrainedConfig + + +class MoonViTConfig(PretrainedConfig): + model_type = "moonvit" + + def __init__( + self, + patch_size: int = 14, + init_pos_emb_height: int = 64, + init_pos_emb_width: int = 64, + num_attention_heads: int = 16, + num_hidden_layers: int = 27, + hidden_size: int = 1152, + intermediate_size: int = 4304, + merge_kernel_size: tuple[int, int] = (2, 2), + **kwargs, + ): + super().__init__(**kwargs) + self.patch_size = patch_size + # Positional embedding config + self.init_pos_emb_height = init_pos_emb_height + self.init_pos_emb_width = init_pos_emb_width + # Transformer config + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + # Patch merger config + self.merge_kernel_size = merge_kernel_size diff --git a/python/sglang/srt/configs/load_config.py b/python/sglang/srt/configs/load_config.py new file mode 100644 index 000000000..be9a40b4b --- /dev/null +++ b/python/sglang/srt/configs/load_config.py @@ -0,0 +1,89 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +import enum +import json +import logging +from dataclasses import dataclass, field +from typing import List, Optional, Union + +from sglang.srt.utils import is_hip + +logger = logging.getLogger(__name__) + + +class LoadFormat(str, enum.Enum): + AUTO = "auto" + PT = "pt" + SAFETENSORS = "safetensors" + NPCACHE = "npcache" + DUMMY = "dummy" + SHARDED_STATE = "sharded_state" + GGUF = "gguf" + BITSANDBYTES = "bitsandbytes" + MISTRAL = "mistral" + LAYERED = "layered" + JAX = "jax" + REMOTE = "remote" + + +@dataclass +class LoadConfig: + """ + download_dir: Directory to download and load the weights, default to the + default cache directory of huggingface. + load_format: The format of the model weights to load: + "auto" will try to load the weights in the safetensors format and + fall back to the pytorch bin format if safetensors format is + not available. + "pt" will load the weights in the pytorch bin format. + "safetensors" will load the weights in the safetensors format. + "npcache" will load the weights in pytorch format and store + a numpy cache to speed up the loading. + "dummy" will initialize the weights with random values, which is + mainly for profiling. + "bitsandbytes" will load nf4 type weights. + ignore_patterns: The list of patterns to ignore when loading the model. + Default to "original/**/*" to avoid repeated loading of llama's + checkpoints. + decryption_key_file: If set, decrypts the output files with a password read + from this file (after PBKDF2). + """ + + load_format: Union[str, LoadFormat] = LoadFormat.AUTO + download_dir: Optional[str] = None + model_loader_extra_config: Optional[Union[str, dict]] = field(default_factory=dict) + ignore_patterns: Optional[Union[List[str], str]] = None + decryption_key_file: Optional[str] = None + + def __post_init__(self): + model_loader_extra_config = self.model_loader_extra_config or {} + if isinstance(model_loader_extra_config, str): + self.model_loader_extra_config = json.loads(model_loader_extra_config) + self._verify_load_format() + + if self.ignore_patterns is not None and len(self.ignore_patterns) > 0: + logger.info( + "Ignoring the following patterns when downloading weights: %s", + self.ignore_patterns, + ) + else: + self.ignore_patterns = ["original/**/*"] + + def _verify_load_format(self) -> None: + if not isinstance(self.load_format, str): + return + + load_format = self.load_format.lower() + self.load_format = LoadFormat(load_format) + + rocm_not_supported_load_format: List[str] = [] + if is_hip() and load_format in rocm_not_supported_load_format: + rocm_supported_load_format = [ + f + for f in LoadFormat.__members__ + if (f not in rocm_not_supported_load_format) + ] + raise ValueError( + f"load format '{load_format}' is not supported in ROCm. " + f"Supported load formats are " + f"{rocm_supported_load_format}" + ) diff --git a/python/sglang/srt/configs/longcat_flash.py b/python/sglang/srt/configs/longcat_flash.py new file mode 100644 index 000000000..e6a2dfb02 --- /dev/null +++ b/python/sglang/srt/configs/longcat_flash.py @@ -0,0 +1,104 @@ +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + +logger = logging.get_logger(__name__) + +FLASH_PRETRAINED_CONFIG_ARCHIVE_MAP = {} + + +class LongcatFlashConfig(PretrainedConfig): + model_type = "longcat_flash" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=131072, + hidden_size=6144, + intermediate_size=None, + ffn_hidden_size=12288, + expert_ffn_hidden_size=2048, + num_layers=28, + num_hidden_layers=None, + num_attention_heads=64, + ep_size=1, + kv_lora_rank=512, + q_lora_rank=1536, + qk_rope_head_dim=128, + qk_nope_head_dim=128, + v_head_dim=128, + n_routed_experts=512, + moe_topk=12, + norm_topk_prob=False, + max_position_embeddings=131072, + rms_norm_eps=1e-05, + use_cache=True, + pad_token_id=None, + bos_token_id=1, + eos_token_id=2, + pretraining_tp=1, + tie_word_embeddings=False, + rope_theta=10000000.0, + rope_scaling=None, + attention_bias=False, + attention_dropout=0.0, + mla_scale_q_lora=True, + mla_scale_kv_lora=True, + torch_dtype="bfloat16", + params_dtype="bfloat16", + rounter_params_dtype="float32", + router_bias=False, + topk_method=None, + routed_scaling_factor=6.0, + zero_expert_num=256, + zero_expert_type="identity", + nextn_use_scmoe=False, + num_nextn_predict_layers=1, + **kwargs, + ): + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + torch_dtype=torch_dtype, + params_dtype=params_dtype, + rounter_params_dtype=rounter_params_dtype, + topk_method=topk_method, + router_bias=router_bias, + nextn_use_scmoe=nextn_use_scmoe, + num_nextn_predict_layers=num_nextn_predict_layers, + **kwargs, + ) + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.num_hidden_layers = ( + num_hidden_layers if num_hidden_layers is not None else num_layers + ) + self.intermediate_size = ( + intermediate_size if intermediate_size is not None else ffn_hidden_size + ) + self.moe_intermediate_size = expert_ffn_hidden_size + self.num_attention_heads = num_attention_heads + self.ep_size = ep_size + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.v_head_dim = v_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.n_routed_experts = n_routed_experts + self.moe_topk = moe_topk + self.norm_topk_prob = norm_topk_prob + self.rms_norm_eps = rms_norm_eps + self.pretraining_tp = pretraining_tp + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.mla_scale_q_lora = mla_scale_q_lora + self.mla_scale_kv_lora = mla_scale_kv_lora + self.zero_expert_num = zero_expert_num + self.zero_expert_type = zero_expert_type + self.routed_scaling_factor = routed_scaling_factor + self.hidden_act = "silu" diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py new file mode 100644 index 000000000..0dbe37aa0 --- /dev/null +++ b/python/sglang/srt/configs/model_config.py @@ -0,0 +1,811 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import logging +import math +import os +from enum import Enum, IntEnum, auto +from typing import List, Optional, Set, Union + +import torch +from transformers import PretrainedConfig + +from sglang.srt.hf_transformers_utils import ( + get_config, + get_context_length, + get_generation_config, + get_hf_text_config, + get_sparse_attention_config, +) +from sglang.srt.layers.quantization import QUANTIZATION_METHODS +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import get_bool_env_var, is_hip +from sglang.utils import is_in_ci + +logger = logging.getLogger(__name__) + + +class AttentionArch(IntEnum): + MLA = auto() + MHA = auto() + + +class ModelImpl(str, Enum): + AUTO = "auto" + SGLANG = "sglang" + TRANSFORMERS = "transformers" + + +class ModelConfig: + def __init__( + self, + model_path: str, + trust_remote_code: bool = True, + revision: Optional[str] = None, + context_length: Optional[int] = None, + model_override_args: str = "{}", + is_embedding: Optional[bool] = None, + enable_multimodal: Optional[bool] = None, + dtype: str = "auto", + quantization: Optional[str] = None, + override_config_file: Optional[str] = None, + is_draft_model: bool = False, + hybrid_kvcache_ratio: Optional[float] = None, + model_impl: Union[str, ModelImpl] = ModelImpl.AUTO, + ) -> None: + # Parse args + self.model_path = model_path + self.revision = revision + self.quantization = quantization + self.model_impl = model_impl + + self.maybe_pull_model_tokenizer_from_remote() + self.model_override_args = json.loads(model_override_args) + kwargs = {} + if override_config_file and override_config_file.strip(): + kwargs["_configuration_file"] = override_config_file.strip() + + self.hf_config = get_config( + self.model_path, + trust_remote_code=trust_remote_code, + revision=revision, + model_override_args=self.model_override_args, + **kwargs, + ) + + self.hf_generation_config = get_generation_config( + self.model_path, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + + self.hf_text_config = get_hf_text_config(self.hf_config) + self.attention_chunk_size = getattr( + self.hf_text_config, "attention_chunk_size", None + ) + self.is_hybrid = is_hybrid_model( + self.hf_config.architectures, + hybrid_kvcache_ratio=hybrid_kvcache_ratio, + context_length=context_length, + attention_chunk_size=self.attention_chunk_size, + ) + if self.is_hybrid is not None: + self.swa_attention_layer_ids, self.full_attention_layer_ids = ( + get_hybrid_layer_ids( + self.hf_config.architectures, self.hf_text_config.num_hidden_layers + ) + ) + + if enable_multimodal is None: + mm_disabled_models = [ + "Gemma3ForConditionalGeneration", + "Llama4ForConditionalGeneration", + "Step3VLForConditionalGeneration", + ] + if self.hf_config.architectures[0] in mm_disabled_models: + enable_multimodal = False + logger.info( + f"Multimodal is disabled for {self.hf_config.model_type}. To enable it, set --enable-multimodal." + ) + else: + enable_multimodal = True + + if ( + is_draft_model + and self.hf_config.architectures[0] == "DeepseekV3ForCausalLM" + ): + self.hf_config.architectures[0] = "DeepseekV3ForCausalLMNextN" + + if is_draft_model and self.hf_config.architectures[0] == "Glm4MoeForCausalLM": + self.hf_config.architectures[0] = "Glm4MoeForCausalLMNextN" + + if ( + is_draft_model + and self.hf_config.architectures[0] == "LongcatFlashForCausalLM" + ): + self.hf_config.architectures[0] = "LongcatFlashForCausalLMNextN" + self.hf_config.num_hidden_layers = self.hf_config.num_nextn_predict_layers + + if is_draft_model and self.hf_config.architectures[0] == "MiMoForCausalLM": + self.hf_config.architectures[0] = "MiMoMTP" + if ( + is_draft_model + and self.hf_config.architectures[0] == "Ernie4_5_MoeForCausalLM" + ): + self.hf_config.architectures[0] = "Ernie4_5_MoeForCausalLMMTP" + + if is_draft_model and self.hf_config.architectures[0] == "Qwen3NextForCausalLM": + self.hf_config.architectures[0] = "Qwen3NextForCausalLMMTP" + + # Check model type + self.is_generation = is_generation_model( + self.hf_config.architectures, is_embedding + ) + self.is_multimodal = enable_multimodal and is_multimodal_model( + self.hf_config.architectures + ) + self.is_multimodal_gen = enable_multimodal and is_multimodal_gen_model( + self.hf_config.architectures + ) + self.is_image_gen = enable_multimodal and is_image_gen_model( + self.hf_config.architectures + ) + self.is_audio_model = enable_multimodal and is_audio_model( + self.hf_config.architectures + ) + self.is_multimodal_chunked_prefill_supported = ( + enable_multimodal + and is_multimodal_chunked_prefill_supported(self.hf_config.architectures) + ) + self.is_encoder_decoder = is_encoder_decoder_model(self.hf_config.architectures) + self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype) + + # Derive context length + derived_context_len = get_context_length(self.hf_text_config) + if context_length is not None: + if context_length > derived_context_len: + reason = "Target model's" if is_draft_model else "User-specified" + msg = ( + f"Warning: {reason} context_length ({context_length}) is greater than the derived context_length ({derived_context_len}). " + f"This may lead to incorrect model outputs or CUDA errors. Note that the derived context_length may differ from max_position_embeddings in the model's config." + ) + if ( + get_bool_env_var("SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN") + or is_in_ci() # FIXME: fix this special case + ): + logger.warning(msg) + self.context_len = context_length + else: + raise ValueError( + f"{msg} To allow overriding this maximum, set the env var SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1" + ) + else: + self.context_len = context_length + else: + self.context_len = derived_context_len + + # Unify the config keys for hf_text_config + self.head_dim = getattr( + self.hf_text_config, + "head_dim", + self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads, + ) + + # FIXME: temporary special judge for MLA architecture + if ( + "DeepseekV2ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLM" in self.hf_config.architectures + or "DeepseekV3ForCausalLMNextN" in self.hf_config.architectures + or "LongcatFlashForCausalLM" in self.hf_config.architectures + or "LongcatFlashForCausalLMNextN" in self.hf_config.architectures + ): + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_nope_head_dim = self.hf_config.qk_nope_head_dim + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + self.v_head_dim = self.hf_config.v_head_dim + + # Handle rope scaling with yarn + self.scaling = 1 / math.sqrt(self.qk_nope_head_dim + self.qk_rope_head_dim) + if self.hf_config.rope_scaling: + mscale_all_dim = self.hf_config.rope_scaling.get( + "mscale_all_dim", False + ) + scaling_factor = self.hf_config.rope_scaling["factor"] + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) + self.scaling = self.scaling * mscale * mscale + + elif "MiniCPM3ForCausalLM" in self.hf_config.architectures: + self.head_dim = 128 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_config.qk_rope_head_dim + elif "DeepseekVL2ForCausalLM" in self.hf_config.architectures and getattr( + self.hf_text_config, "use_mla", True + ): + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + elif "KimiVLForConditionalGeneration" in self.hf_config.architectures: + self.head_dim = 256 + self.attention_arch = AttentionArch.MLA + self.kv_lora_rank = self.hf_text_config.kv_lora_rank + self.qk_rope_head_dim = self.hf_text_config.qk_rope_head_dim + self.v_head_dim = self.hf_text_config.v_head_dim + self.qk_nope_head_dim = self.hf_text_config.qk_nope_head_dim + else: + if ( + "MistralModel" in self.hf_config.architectures + or "MixtralForCausalLM" in self.hf_config.architectures + or "MistralForCausalLM" in self.hf_config.architectures + ): + if getattr(self, "head_dim", None) is None: + self.head_dim = ( + self.hf_config.hidden_size // self.hf_config.num_attention_heads + ) + # In transformers==4.52.3, the head_dim is null in MistralConfig + if ( + not hasattr(self.hf_text_config, "head_dim") + or self.hf_text_config.head_dim is None + ): + setattr(self.hf_text_config, "head_dim", self.head_dim) + + self.attention_arch = AttentionArch.MHA + + self.num_attention_heads = self.hf_text_config.num_attention_heads + self.num_key_value_heads = getattr( + self.hf_text_config, "num_key_value_heads", None + ) + + # for Dbrx and MPT models + if self.hf_config.model_type in ["dbrx", "mpt"]: + self.num_key_value_heads = getattr( + self.hf_config.attn_config, "kv_n_heads", None + ) + + if self.num_key_value_heads is None: + self.num_key_value_heads = self.num_attention_heads + self.hidden_size = self.hf_text_config.hidden_size + self.num_hidden_layers = self.hf_text_config.num_hidden_layers + self.num_attention_layers = self.num_hidden_layers + if "LongcatFlashForCausalLM" in self.hf_config.architectures: + self.num_attention_layers = self.num_hidden_layers * 2 + self.num_nextn_predict_layers = getattr( + self.hf_text_config, "num_nextn_predict_layers", None + ) + self.vocab_size = self.hf_text_config.vocab_size + + # Verify quantization + self._verify_quantization() + + # Verify dual-chunk attention config + self._verify_dual_chunk_attention_config() + + # Cache attributes + self.hf_eos_token_id = self.get_hf_eos_token_id() + + # multimodal + self.image_token_id = getattr( + self.hf_config, "image_token_id", None + ) or getattr(self.hf_config, "image_token_index", None) + + @staticmethod + def from_server_args( + server_args: ServerArgs, + model_path: str = None, + model_revision: str = None, + **kwargs, + ): + return ModelConfig( + model_path=model_path or server_args.model_path, + trust_remote_code=server_args.trust_remote_code, + revision=model_revision or server_args.revision, + context_length=server_args.context_length, + model_override_args=server_args.json_model_override_args, + is_embedding=server_args.is_embedding, + enable_multimodal=server_args.enable_multimodal, + dtype=server_args.dtype, + quantization=server_args.quantization, + hybrid_kvcache_ratio=server_args.hybrid_kvcache_ratio, + model_impl=server_args.model_impl, + **kwargs, + ) + + def get_total_num_attention_heads(self) -> int: + return self.num_attention_heads + + def get_num_attention_heads(self, tensor_parallel_size) -> int: + total_num_attention_heads = self.num_attention_heads + return max(1, total_num_attention_heads // tensor_parallel_size) + + # adapted from https://github.com/vllm-project/vllm/blob/main/vllm/config.py#L289 + def get_total_num_kv_heads(self) -> int: + """Returns the total number of KV heads.""" + # For GPTBigCode & Falcon: + # NOTE: for falcon, when new_decoder_architecture is True, the + # multi_query flag is ignored and we use n_head_kv for the number of + # KV heads. + falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"] + new_decoder_arch_falcon = ( + self.hf_config.model_type in falcon_model_types + and getattr(self.hf_config, "new_decoder_architecture", False) + ) + if not new_decoder_arch_falcon and getattr( + self.hf_text_config, "multi_query", False + ): + # Multi-query attention, only one KV head. + # Currently, tensor parallelism is not supported in this case. + return 1 + + # For DBRX and MPT + if self.hf_config.model_type in ["mpt"]: + if "kv_n_heads" in self.hf_config.attn_config: + return self.hf_config.attn_config["kv_n_heads"] + return self.hf_config.num_attention_heads + if self.hf_config.model_type in ["dbrx"]: + return getattr( + self.hf_config.attn_config, + "kv_n_heads", + self.hf_config.num_attention_heads, + ) + if self.hf_config.model_type in ["nemotron-nas"]: + nkvh = { + self.hf_config.num_attention_heads // block.attention.n_heads_in_group + for block in self.hf_config.block_configs + if not block.attention.no_op + } + if len(nkvh) == 0: + raise RuntimeError("Couldn't determine number of kv heads") + if len(nkvh) > 1: + raise ValueError( + "Variable GQA (VGQA) is not yet supported for nemotron-nas in sglang" + ) + return next(iter(nkvh)) + + attributes = [ + # For Falcon: + "n_head_kv", + "num_kv_heads", + # For LLaMA-2: + "num_key_value_heads", + # For ChatGLM: + "multi_query_group_num", + # For Step3 + "num_attention_groups", + ] + for attr in attributes: + num_kv_heads = getattr(self.hf_text_config, attr, None) + if num_kv_heads is not None: + return num_kv_heads + + # For non-grouped-query attention models, the number of KV heads is + # equal to the number of attention heads. + return self.hf_text_config.num_attention_heads + + def get_num_kv_heads(self, tensor_parallel_size) -> int: + """Returns the number of KV heads per GPU.""" + total_num_kv_heads = self.get_total_num_kv_heads() + # If tensor parallelism is used, we divide the number of KV heads by + # the tensor parallel size. We will replicate the KV heads in the + # case where the number of KV heads is smaller than the tensor + # parallel size so each GPU has at least one KV head. + return max(1, total_num_kv_heads // tensor_parallel_size) + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _parse_quant_hf_config(self): + quant_cfg = getattr(self.hf_config, "quantization_config", None) + if quant_cfg is None: + # compressed-tensors uses a "compression_config" key + quant_cfg = getattr(self.hf_config, "compression_config", None) + if quant_cfg is None: + # check if is modelopt or mixed-precision model -- Both of them don't have corresponding field + # in hf `config.json` but has a standalone `hf_quant_config.json` in the root directory + # example: https://huggingface.co/nvidia/Llama-3.1-8B-Instruct-FP8/tree/main + # example: https://huggingface.co/Barrrrry/DeepSeek-R1-W4AFP8/tree/main + is_local = os.path.exists(self.model_path) + modelopt_quant_config = {"quant_method": "modelopt"} + if not is_local: + import huggingface_hub + + try: + from huggingface_hub import HfApi + + hf_api = HfApi() + if hf_api.file_exists(self.model_path, "hf_quant_config.json"): + quant_cfg = modelopt_quant_config + except huggingface_hub.errors.OfflineModeIsEnabled: + logger.warning( + "Offline mode is enabled, skipping hf_quant_config.json check" + ) + pass + + elif os.path.exists(os.path.join(self.model_path, "hf_quant_config.json")): + quant_config_file = os.path.join( + self.model_path, "hf_quant_config.json" + ) + with open(quant_config_file) as f: + quant_config_dict = json.load(f) + json_quant_configs = quant_config_dict["quantization"] + quant_algo = json_quant_configs.get("quant_algo", None) + if quant_algo == "MIXED_PRECISION": + quant_cfg = {"quant_method": "w4afp8"} + else: + quant_cfg = modelopt_quant_config + return quant_cfg + + # adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py + def _verify_quantization(self) -> None: + supported_quantization = [*QUANTIZATION_METHODS] + rocm_supported_quantization = [ + "awq", + "gptq", + "fp8", + "compressed_tensors", + "compressed-tensors", + "fbgemm_fp8", + "w8a8_fp8", + "petit_nvfp4", + "quark", + "mxfp4", + ] + optimized_quantization_methods = [ + "fp8", + "marlin", + "modelopt", + "gptq_marlin_24", + "gptq_marlin", + "awq_marlin", + "fbgemm_fp8", + "compressed_tensors", + "compressed-tensors", + "experts_int8", + "w8a8_int8", + "w8a8_fp8", + "moe_wna16", + "qoq", + "w4afp8", + "petit_nvfp4", + ] + compatible_quantization_methods = { + "modelopt_fp4": ["modelopt"], + "petit_nvfp4": ["modelopt"], + "w8a8_int8": ["compressed-tensors", "compressed_tensors"], + "w8a8_fp8": ["compressed-tensors", "compressed_tensors"], + } + if self.quantization is not None: + self.quantization = self.quantization.lower() + + # Parse quantization method from the HF model config, if available. + quant_cfg = self._parse_quant_hf_config() + + if quant_cfg is not None: + quant_method = quant_cfg.get( + "quant_method", "" if not self.quantization else self.quantization + ).lower() + + # Detect which checkpoint is it + for _, method in QUANTIZATION_METHODS.items(): + quantization_override = method.override_quantization_method( + quant_cfg, self.quantization + ) + if quantization_override: + quant_method = quantization_override + self.quantization = quantization_override + break + + # Verify quantization configurations. + if self.quantization is None: + self.quantization = quant_method + elif self.quantization != quant_method: + if ( + self.quantization not in compatible_quantization_methods + or quant_method + not in compatible_quantization_methods[self.quantization] + ): + raise ValueError( + "Quantization method specified in the model config " + f"({quant_method}) does not match the quantization " + f"method specified in the `quantization` argument " + f"({self.quantization})." + ) + + if self.quantization is not None: + if self.quantization not in supported_quantization: + raise ValueError( + f"Unknown quantization method: {self.quantization}. Must " + f"be one of {supported_quantization}." + ) + if is_hip() and self.quantization not in rocm_supported_quantization: + raise ValueError( + f"{self.quantization} quantization is currently not " + f"supported in ROCm." + ) + if self.quantization not in optimized_quantization_methods: + logger.warning( + "%s quantization is not fully " + "optimized yet. The speed can be slower than " + "non-quantized models.", + self.quantization, + ) + + def _verify_dual_chunk_attention_config(self) -> None: + if hasattr(self.hf_config, "dual_chunk_attention_config"): + # Try loading the sparse attention config + sparse_attn_config = get_sparse_attention_config(self.model_path) + if not sparse_attn_config: + return + self.hf_config.dual_chunk_attention_config["sparse_attention_config"] = ( + sparse_attn_config + ) + if ( + "sparse_attention_enabled" + not in self.hf_config.dual_chunk_attention_config + ): + self.hf_config.dual_chunk_attention_config[ + "sparse_attention_enabled" + ] = True + + def get_hf_eos_token_id(self) -> Optional[Set[int]]: + eos_ids = getattr(self.hf_config, "eos_token_id", None) + if eos_ids is not None: + # it can be either int or list of int + eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids) + if eos_ids is None: + eos_ids = set() + if self.hf_generation_config: + generation_eos_ids = getattr( + self.hf_generation_config, "eos_token_id", None + ) + if generation_eos_ids: + generation_eos_ids = ( + {generation_eos_ids} + if isinstance(generation_eos_ids, int) + else set(generation_eos_ids) + ) + eos_ids = eos_ids | generation_eos_ids + return eos_ids + + def maybe_pull_model_tokenizer_from_remote(self) -> None: + """ + Pull the model config files to a temporary + directory in case of remote. + + Args: + model: The model name or path. + + """ + from sglang.srt.connector import create_remote_connector + from sglang.srt.utils import is_remote_url + + if is_remote_url(self.model_path): + logger.info("Pulling model configs from remote...") + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(self.model_path) + if is_remote_url(self.model_path): + client.pull_files(allow_pattern=["*config.json"]) + self.model_weights = self.model_path + self.model_path = client.get_local_dir() + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +_STR_DTYPE_TO_TORCH_DTYPE = { + "half": torch.float16, + "float16": torch.float16, + "float": torch.float32, + "float32": torch.float32, + "bfloat16": torch.bfloat16, +} + + +# adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/config.py +def _get_and_verify_dtype( + config: PretrainedConfig, + dtype: Union[str, torch.dtype], +) -> torch.dtype: + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct + # because config.torch_dtype can be None. + config_dtype = getattr(config, "torch_dtype", None) + if isinstance(config_dtype, str): + config_dtype = _STR_DTYPE_TO_TORCH_DTYPE.get(config_dtype, None) + if config_dtype is None: + config_dtype = torch.float32 + + if isinstance(dtype, str): + dtype = dtype.lower() + if dtype == "auto": + if config_dtype == torch.float32: + if config.model_type.startswith("gemma"): + if config.model_type == "gemma": + gemma_version = "" + else: + gemma_version = config.model_type[5] + logger.info( + f"For Gemma {gemma_version}, we downcast float32 to bfloat16 instead " + "of float16 by default. Please specify `dtype` if you " + "want to use float16." + ) + torch_dtype = torch.bfloat16 + else: + # Following the common practice, we use float16 for float32 + # models. + torch_dtype = torch.float16 + else: + torch_dtype = config_dtype + else: + if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: + raise ValueError(f"Unknown dtype: {dtype}") + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype] + elif isinstance(dtype, torch.dtype): + torch_dtype = dtype + else: + raise ValueError(f"Unknown dtype: {dtype}") + + # Verify the dtype. + if torch_dtype != config_dtype: + if torch_dtype == torch.float32: + # Upcasting to float32 is allowed. + logger.info("Upcasting %s to %s.", config_dtype, torch_dtype) + pass + elif config_dtype == torch.float32: + # Downcasting from float32 to float16 or bfloat16 is allowed. + logger.info("Downcasting %s to %s.", config_dtype, torch_dtype) + pass + else: + # Casting between float16 and bfloat16 is allowed with a warning. + logger.warning("Casting %s to %s.", config_dtype, torch_dtype) + + return torch_dtype + + +def is_generation_model(model_architectures: List[str], is_embedding: bool = False): + # We have two ways to determine whether a model is a generative model. + # 1. Check the model architecture + # 2. check the `is_embedding` server args + + if ( + "LlamaEmbeddingModel" in model_architectures + or "MistralModel" in model_architectures + or "LlamaForSequenceClassification" in model_architectures + or "LlamaForSequenceClassificationWithNormal_Weights" in model_architectures + or "InternLM2ForRewardModel" in model_architectures + or "Qwen2ForRewardModel" in model_architectures + or "Qwen2ForSequenceClassification" in model_architectures + or "Qwen3ForSequenceClassification" in model_architectures + or "CLIPModel" in model_architectures + or "BertModel" in model_architectures + or "Contriever" in model_architectures + or "BertForSequenceClassification" in model_architectures + or "XLMRobertaModel" in model_architectures + or "XLMRobertaForSequenceClassification" in model_architectures + ): + return False + else: + return not is_embedding + + +multimodal_model_archs = [ + "CLIPModel", + "DeepseekVL2ForCausalLM", + "Gemma3ForConditionalGeneration", + "Gemma3nForConditionalGeneration", + "Glm4vForConditionalGeneration", + "Glm4vMoeForConditionalGeneration", + "Grok1VForCausalLM", + "Grok1AForCausalLM", + "LlavaLlamaForCausalLM", + "Llama4ForConditionalGeneration", + "LlavaMistralForCausalLM", + "LlavaQwenForCausalLM", + "LlavaForConditionalGeneration", + "LlavaVidForCausalLM", + "MiniCPMO", + "MiniCPMV", + "Mistral3ForConditionalGeneration", + "MultiModalityCausalLM", + "MllamaForConditionalGeneration", + "Qwen2AudioForConditionalGeneration", + "Qwen2VLForConditionalGeneration", + "Qwen2_5_VLForConditionalGeneration", + "KimiVLForConditionalGeneration", + "InternVLChatModel", + "InternS1ForConditionalGeneration", + "Phi4MMForCausalLM", + "VILAForConditionalGeneration", + "Step3VLForConditionalGeneration", +] + + +def is_multimodal_model(model_architectures: List[str]): + if any( + multi_model_arch in model_architectures + for multi_model_arch in multimodal_model_archs + ): + return True + else: + return False + + +def is_multimodal_gen_model(model_architectures: List[str]): + return False + + +def is_image_gen_model(model_architectures: List[str]): + return False + + +def is_audio_model(model_architectures: List[str]): + return False + + +def is_encoder_decoder_model(model_architectures: List[str]): + return "MllamaForConditionalGeneration" in model_architectures + + +def is_multimodal_chunked_prefill_supported(model_architectures: List[str]): + """Check if chunked prefill is supported for a MultiModal model.""" + unsupported = [ + "Grok1VForCausalLM", + "Grok1AForCausalLM", + "LlavaLlamaForCausalLM", + "MllamaForConditionalGeneration", + "CLIPModel", + ] + if any(multi_model_arch in unsupported for multi_model_arch in model_architectures): + return False + else: + return True + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +def is_hybrid_model( + model_architectures: List[str], + hybrid_kvcache_ratio: Optional[float], + context_length: Optional[int], + attention_chunk_size: Optional[int], +): + if hybrid_kvcache_ratio is None: + return None + elif ( + hybrid_kvcache_ratio > 0 + and model_architectures[0] == "Llama4ForConditionalGeneration" + and context_length > attention_chunk_size + ): + return hybrid_kvcache_ratio + else: + return None + + +def get_hybrid_layer_ids(model_architectures: List[str], num_hidden_layers: int): + if "Llama4ForConditionalGeneration" in model_architectures: + swa_attention_layer_ids = [ + i for i in range(num_hidden_layers) if (i + 1) % 4 != 0 + ] + full_attention_layer_ids = [ + i for i in range(num_hidden_layers) if (i + 1) % 4 == 0 + ] + else: + swa_attention_layer_ids = None + full_attention_layer_ids = None + return swa_attention_layer_ids, full_attention_layer_ids diff --git a/python/sglang/srt/configs/qwen3_next.py b/python/sglang/srt/configs/qwen3_next.py new file mode 100644 index 000000000..099d14d41 --- /dev/null +++ b/python/sglang/srt/configs/qwen3_next.py @@ -0,0 +1,326 @@ +# coding=utf-8 +# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Qwen3Hybrid model configuration""" + +import enum +import os + +import numpy as np +import torch +from transformers.configuration_utils import PretrainedConfig +from transformers.modeling_rope_utils import rope_config_validation +from transformers.utils import logging + +from sglang.srt.distributed.utils import divide +from sglang.srt.layers.dp_attention import get_attention_tp_size + +logger = logging.get_logger(__name__) + + +# NOTE: HybridLayerType +class HybridLayerType(enum.Enum): + full_attention = "attention" + swa_attention = "swa_attention" + linear_attention = "linear_attention" + mamba2 = "mamba" + + +class Qwen3NextConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a + Qwen3-Next model according to the specified arguments, defining the model architecture. + Instantiating a configuration with the defaults will yield a similar configuration to that of + Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 151936): + Vocabulary size of the model. Defines the number of different tokens that can be represented by the + `inputs_ids`. + hidden_size (`int`, *optional*, defaults to 2048): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 5632): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 48): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + num_key_value_heads (`int`, *optional*, defaults to 2): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`. + hidden_act (`str`, *optional*, defaults to `"silu"`): + The non-linear activation function in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 32768): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-06): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether the model's input and output word embeddings should be tied. + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`Dict`, *optional*): + Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type + and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value + accordingly. + Expected contents: + `rope_type` (`str`): + The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope', + 'llama3'], with 'default' being the original RoPE implementation. + `factor` (`float`, *optional*): + Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In + most scaling types, a `factor` of x will enable the model to handle sequences of length x * + original maximum pre-trained length. + `original_max_position_embeddings` (`int`, *optional*): + Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during + pretraining. + `attention_factor` (`float`, *optional*): + Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention + computation. If unspecified, it defaults to value recommended by the implementation, using the + `factor` field to infer the suggested value. + `beta_fast` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear + ramp function. If unspecified, it defaults to 32. + `beta_slow` (`float`, *optional*): + Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear + ramp function. If unspecified, it defaults to 1. + `short_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to short contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `long_factor` (`List[float]`, *optional*): + Only used with 'longrope'. The scaling factor to be applied to long contexts (< + `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden + size divided by the number of attention heads divided by 2 + `low_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE + `high_freq_factor` (`float`, *optional*): + Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE + partial_rotary_factor (`float`, *optional*, defaults to 0.25): + Percentage of the query and keys which will have rotary embedding. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + head_dim (`int`, *optional*, defaults to 256): + Projection weights dimension in multi-head attention. + linear_conv_kernel_dim (`int`, *optional*, defaults to 4): + Kernel size of the convolution used in linear attention layers. + linear_key_head_dim (`int`, *optional*, defaults to 128): + Dimension of each key head in linear attention. + linear_value_head_dim (`int`, *optional*, defaults to 128): + Dimension of each value head in linear attention. + linear_num_key_heads (`int`, *optional*, defaults to 16): + Number of key heads used in linear attention layers. + linear_num_value_heads (`int`, *optional*, defaults to 32): + Number of value heads used in linear attention layers. + decoder_sparse_step (`int`, *optional*, defaults to 1): + The frequency of the MoE layer. + moe_intermediate_size (`int`, *optional*, defaults to 512): + Intermediate size of the routed expert. + shared_expert_intermediate_size (`int`, *optional*, defaults to 512): + Intermediate size of the shared expert. + num_experts_per_tok (`int`, *optional*, defaults to 10): + Number of selected experts. + num_experts (`int`, *optional*, defaults to 512): + Number of routed experts. + norm_topk_prob (`bool`, *optional*, defaults to `True`): + Whether to normalize the topk probabilities. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether or not the router logits should be returned by the model. Enabling this will also + allow the model to output the auxiliary loss, including load balancing loss and router z-loss. + router_aux_loss_coef (`float`, *optional*, defaults to 0.001): + The aux loss factor for the total loss. + mlp_only_layers (`list[int]`, *optional*, defaults to `[]`): + Indicate which layers use Qwen3NextMLP rather than Qwen3NextSparseMoeBlock + The list contains layer index, from 0 to num_layers-1 if we have num_layers layers + If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity. + layer_types (`list[str]`, *optional*, defaults to None): + Types of each layer (attention or linear). + + ```python + >>> from transformers import Qwen3NextModel, Qwen3NextConfig + + >>> # Initializing a Qwen3Next style configuration + >>> configuration = Qwen3NextConfig() + + >>> # Initializing a model from the Qwen3-Next-80B-A3B style configuration + >>> model = Qwen3NextModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ``` + """ + + model_type = "qwen3_next" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=151936, + hidden_size=2048, + intermediate_size=5632, + num_hidden_layers=48, + num_attention_heads=16, + num_key_value_heads=2, + hidden_act="silu", + max_position_embeddings=32768, + initializer_range=0.02, + rms_norm_eps=1e-6, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + partial_rotary_factor=0.25, + attention_bias=False, + attention_dropout=0.0, + head_dim=256, + linear_conv_kernel_dim=4, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=32, + decoder_sparse_step=1, + moe_intermediate_size=512, + shared_expert_intermediate_size=512, + num_experts_per_tok=10, + num_experts=512, + norm_topk_prob=True, + output_router_logits=False, + router_aux_loss_coef=0.001, + mlp_only_layers=[], + layer_types=None, + **kwargs, + ): + super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.partial_rotary_factor = partial_rotary_factor + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.head_dim = head_dim + rope_config_validation(self) + + # linear attention (gdn now part) + self.linear_conv_kernel_dim = linear_conv_kernel_dim + self.linear_key_head_dim = linear_key_head_dim + self.linear_value_head_dim = linear_value_head_dim + self.linear_num_key_heads = linear_num_key_heads + self.linear_num_value_heads = linear_num_value_heads + + # MoE arguments + self.decoder_sparse_step = decoder_sparse_step + self.moe_intermediate_size = moe_intermediate_size + self.shared_expert_intermediate_size = shared_expert_intermediate_size + self.num_experts_per_tok = num_experts_per_tok + self.num_experts = num_experts + self.norm_topk_prob = norm_topk_prob + self.output_router_logits = output_router_logits + self.router_aux_loss_coef = router_aux_loss_coef + self.mlp_only_layers = mlp_only_layers + + @property + def layers_block_type(self): + layer_type_list = [] + + for l in range(self.num_hidden_layers): + if (l + 1) % self.full_attention_interval == 0: + layer_type_list.append(HybridLayerType.full_attention.value) + else: + layer_type_list.append(HybridLayerType.linear_attention.value) + + return layer_type_list + + @property + def linear_layer_ids(self): + return [ + i + for i, type_value in enumerate(self.layers_block_type) + if type_value == HybridLayerType.linear_attention.value + ] + + @property + def full_attention_layer_ids(self): + return [ + i + for i, type_value in enumerate(self.layers_block_type) + if type_value == HybridLayerType.full_attention.value + ] + + @property + def hybrid_gdn_params(self): + world_size = get_attention_tp_size() + conv_dim = ( + self.linear_key_head_dim * self.linear_num_key_heads * 2 + + self.linear_value_head_dim * self.linear_num_value_heads + ) + conv_state_shape = ( + divide(conv_dim, world_size), + self.linear_conv_kernel_dim - 1, + ) + + temporal_state_shape = ( + divide(self.linear_num_value_heads, world_size), + self.linear_key_head_dim, + self.linear_value_head_dim, + ) + conv_dtype = torch.bfloat16 + dtype_map = { + "float32": torch.float32, + "bfloat16": torch.bfloat16, + } + ssm_dtype = dtype_map[os.environ["SGLANG_MAMBA_SSM_DTYPE"]] + mamba_layers = self.linear_layer_ids + return ( + conv_state_shape, + temporal_state_shape, + conv_dtype, + ssm_dtype, + mamba_layers, + ) + + @property + def mamba_cache_per_req(self): + conv_state_shape, temporal_state_shape, conv_dtype, ssm_dtype, mamba_layers = ( + self.hybrid_gdn_params + ) + mamba_layers_len = len(mamba_layers) + + return ( + int(np.prod(conv_state_shape)) * conv_dtype.itemsize + + int(np.prod(temporal_state_shape)) * ssm_dtype.itemsize + ) * mamba_layers_len diff --git a/python/sglang/srt/configs/step3_vl.py b/python/sglang/srt/configs/step3_vl.py new file mode 100644 index 000000000..5519605c6 --- /dev/null +++ b/python/sglang/srt/configs/step3_vl.py @@ -0,0 +1,172 @@ +from typing import Any, Optional, Union + +from transformers.configuration_utils import PretrainedConfig + + +class Step3VisionEncoderConfig(PretrainedConfig): + model_type = "step3_vision_encoder" + + def __init__( + self, + hidden_size=1792, + intermediate_size=3072, + output_hidden_size=4096, + num_hidden_layers=63, + num_attention_heads=16, + num_channels=3, + image_size=728, + patch_size=14, + hidden_act="quick_gelu", + layer_norm_eps=1e-5, + **kwargs, + ): + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.output_hidden_size = output_hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + super().__init__(**kwargs) + + +class Step3TextConfig(PretrainedConfig): + model_type = "step3_text" + architectures = ["Step3TextForCausalLM"] + + def __init__( + self, + hidden_size: int = 7168, + intermediate_size: int = 18432, + num_attention_heads: int = 64, + num_attention_groups: int = 1, + num_hidden_layers: int = 61, + max_seq_len: int = 65536, + vocab_size: int = 128815, + rms_norm_eps: float = 1e-5, + moe_intermediate_size: int = 5120, + moe_num_experts: int = 48, + moe_top_k: int = 3, + rope_theta: float = 500000, + rope_scaling: Optional[dict[str, Any]] = None, + max_position_embedding: int = 65536, + share_expert_dim: int = 5120, + share_q_dim: int = 2048, + head_dim: int = 256, + norm_expert_weight: bool = False, + moe_layers_enum: tuple[int] = ( + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + ), + **kwargs, + ) -> None: + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_attention_heads = num_attention_heads + self.num_attention_groups = num_attention_groups + self.num_hidden_layers = num_hidden_layers + self.max_seq_len = max_seq_len + self.vocab_size = vocab_size + self.rms_norm_eps = rms_norm_eps + self.moe_intermediate_size = moe_intermediate_size + self.moe_num_experts = moe_num_experts + self.moe_top_k = moe_top_k + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self.max_position_embedding = max_position_embedding + self.share_expert_dim = share_expert_dim + self.share_q_dim = share_q_dim + self.head_dim = head_dim + self.norm_expert_weight = norm_expert_weight + self.moe_layers_enum = moe_layers_enum + + super().__init__(**kwargs) + + +class Step3VLConfig(PretrainedConfig): + model_type = "step3_vl" + + def __init__( + self, + vision_config: Optional[Union[dict, Step3VisionEncoderConfig]] = None, + text_config: Optional[Union[dict, Step3TextConfig]] = None, + understand_projector_stride: int = 1, + projector_bias: bool = True, + image_token_id: int = 128001, + **kwargs, + ) -> None: + if vision_config is None: + vision_config = Step3VisionEncoderConfig() + elif isinstance(vision_config, dict): + vision_config = Step3VisionEncoderConfig(**vision_config) + self.vision_config = vision_config + + if text_config is None: + text_config = Step3TextConfig() + elif isinstance(text_config, dict): + text_config = Step3TextConfig(**text_config) + self.text_config = text_config + + self.understand_projector_stride = understand_projector_stride + self.projector_bias = projector_bias + self.hidden_size = text_config.hidden_size + self.image_token_id = image_token_id + + super().__init__(**kwargs) diff --git a/python/sglang/srt/configs/update_config.py b/python/sglang/srt/configs/update_config.py new file mode 100644 index 000000000..abbd724fb --- /dev/null +++ b/python/sglang/srt/configs/update_config.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +DEFAULT_MOE_PADDING_SIZE = 32 + + +if TYPE_CHECKING: + from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.configs.model_config import ModelConfig + + +def may_get_weight_block_size(model_config, load_config): + from sglang.srt.model_loader.loader import _get_quantization_config + from sglang.srt.model_loader.utils import get_model_architecture + + model_class, _ = get_model_architecture(model_config) + packed_modules_mapping = getattr(model_class, "packed_modules_mapping", {}) + + quant_config = _get_quantization_config( + model_config, load_config, packed_modules_mapping + ) + + if quant_config is not None and hasattr(quant_config, "weight_block_size"): + return getattr(quant_config, "weight_block_size") + return None + + +def get_moe_padding_size(weight_block_size): + if weight_block_size is not None: + # See NOTE(HandH1998): To ensure proper alignment of the block-wise quantization scales, the output_size of the weights for both the gate and up layers must be divisible by block_n. + assert ( + len(weight_block_size) == 2 + ), "Only len(weight_block_size) == 2 is supported" + assert ( + weight_block_size[0] == weight_block_size[1] + ), "Only weight_block_size[0] == weight_block_size[1] is supported" + + return weight_block_size[0] + + return DEFAULT_MOE_PADDING_SIZE + + +def get_num_heads_padding_size(tp_size, weight_block_size): + pad_size = ( + tp_size * 2 if tp_size % 2 == 1 and weight_block_size is not None else tp_size + ) + return pad_size + + +def update_intermediate_size(model_config, attr_name, intermediate_padding_size): + attr_value = intermediate_padding_size + if hasattr(model_config, "hf_config") and hasattr( + model_config.hf_config, attr_name + ): + attr_value = getattr(model_config.hf_config, attr_name) + elif hasattr(model_config, attr_name): + attr_value = getattr(model_config, attr_name) + + if attr_value % intermediate_padding_size != 0: + from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size + + attr_value = pad_vocab_size(attr_value, intermediate_padding_size) + if hasattr(model_config, "hf_config"): + setattr(model_config.hf_config, attr_name, attr_value) + if hasattr(model_config, "hf_text_config"): + setattr(model_config.hf_text_config, attr_name, attr_value) + else: + setattr(model_config, attr_name, attr_value) + + return model_config + + +def adjust_config_with_unaligned_cpu_tp( + model_config: ModelConfig, load_config: LoadConfig, tp_size: int +) -> ModelConfig: + # Support the case where the num_attention_heads is not divisible by the TP size. + weight_block_size = may_get_weight_block_size(model_config, load_config) + + model_config.hf_config.original_num_attention_heads = ( + model_config.num_attention_heads + ) + model_config.hf_text_config.original_num_attention_heads = ( + model_config.num_attention_heads + ) + + model_config.hf_config.original_total_num_kv_heads = ( + model_config.get_total_num_kv_heads() + ) + model_config.hf_text_config.original_total_num_kv_heads = ( + model_config.get_total_num_kv_heads() + ) + + if ( + model_config.num_attention_heads % tp_size != 0 + or model_config.get_total_num_kv_heads() % tp_size != 0 + ): + # Compute the head_dim using the model_config.num_attention_heads before padding + if not hasattr(model_config.hf_config, "head_dim"): + model_config.hf_config.head_dim = ( + model_config.hidden_size // model_config.num_attention_heads + ) + + query_heads_per_kv = ( + model_config.num_attention_heads // model_config.get_total_num_kv_heads() + ) + total_kv_heads = model_config.get_total_num_kv_heads() + from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size + + pad_size = get_num_heads_padding_size(tp_size, weight_block_size) + num_key_value_heads = pad_vocab_size(total_kv_heads, pad_size) + + model_config.num_key_value_heads = num_key_value_heads + model_config.hf_config.num_key_value_heads = num_key_value_heads + model_config.hf_text_config.num_key_value_heads = num_key_value_heads + + num_attention_heads = num_key_value_heads * query_heads_per_kv + model_config.num_attention_heads = num_attention_heads + model_config.hf_config.num_attention_heads = num_attention_heads + model_config.hf_text_config.num_attention_heads = num_attention_heads + + intermediate_padding_size = tp_size * get_moe_padding_size(weight_block_size) + model_config = update_intermediate_size( + model_config, "moe_intermediate_size", intermediate_padding_size + ) + model_config = update_intermediate_size( + model_config, "intermediate_size", intermediate_padding_size + ) + model_config = update_intermediate_size( + model_config, "intermediate_size_mlp", intermediate_padding_size + ) + if ( + hasattr(model_config.hf_config, "vision_config") + and model_config.hf_config.vision_config.model_type == "siglip_vision_model" + ): + model_config.hf_config.vision_config.original_num_attention_heads = ( + model_config.num_attention_heads + ) + if model_config.hf_config.vision_config.num_attention_heads % tp_size != 0: + model_config.hf_config.vision_config.head_dim = ( + model_config.hf_config.vision_config.hidden_size + // model_config.hf_config.vision_config.num_attention_heads + ) + from sglang.srt.layers.vocab_parallel_embedding import pad_vocab_size + + pad_size = get_num_heads_padding_size(tp_size, weight_block_size) + model_config.hf_config.vision_config.num_attention_heads = pad_vocab_size( + model_config.hf_config.vision_config.num_attention_heads, pad_size + ) + model_config.hf_config.vision_config = update_intermediate_size( + model_config.hf_config.vision_config, + "intermediate_size", + intermediate_padding_size, + ) + + return model_config diff --git a/python/sglang/srt/configs/utils.py b/python/sglang/srt/configs/utils.py new file mode 100644 index 000000000..8403d95da --- /dev/null +++ b/python/sglang/srt/configs/utils.py @@ -0,0 +1,25 @@ +from typing import Type + +from transformers import ( + AutoImageProcessor, + AutoProcessor, + BaseImageProcessor, + PretrainedConfig, + ProcessorMixin, +) + + +def register_image_processor( + config: Type[PretrainedConfig], image_processor: Type[BaseImageProcessor] +): + """ + register customized hf image processor while removing hf impl + """ + AutoImageProcessor.register(config, None, image_processor, None, exist_ok=True) + + +def register_processor(config: Type[PretrainedConfig], processor: Type[ProcessorMixin]): + """ + register customized hf processor while removing hf impl + """ + AutoProcessor.register(config, processor, exist_ok=True) diff --git a/python/sglang/srt/connector/__init__.py b/python/sglang/srt/connector/__init__.py new file mode 100644 index 000000000..38e1d5eab --- /dev/null +++ b/python/sglang/srt/connector/__init__.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 + +import enum +import logging + +from sglang.srt.connector.base_connector import ( + BaseConnector, + BaseFileConnector, + BaseKVConnector, +) +from sglang.srt.connector.redis import RedisConnector +from sglang.srt.connector.s3 import S3Connector +from sglang.srt.utils import parse_connector_type + +logger = logging.getLogger(__name__) + + +class ConnectorType(str, enum.Enum): + FS = "filesystem" + KV = "KV" + + +def create_remote_connector(url, **kwargs) -> BaseConnector: + connector_type = parse_connector_type(url) + if connector_type == "redis": + return RedisConnector(url) + elif connector_type == "s3": + return S3Connector(url) + else: + raise ValueError(f"Invalid connector type: {url}") + + +def get_connector_type(client: BaseConnector) -> ConnectorType: + if isinstance(client, BaseKVConnector): + return ConnectorType.KV + if isinstance(client, BaseFileConnector): + return ConnectorType.FS + + raise ValueError(f"Invalid connector type: {client}") + + +__all__ = [ + "BaseConnector", + "BaseFileConnector", + "BaseKVConnector", + "RedisConnector", + "S3Connector", + "ConnectorType", + "create_remote_connector", + "get_connector_type", +] diff --git a/python/sglang/srt/connector/base_connector.py b/python/sglang/srt/connector/base_connector.py new file mode 100644 index 000000000..c9a1c36e2 --- /dev/null +++ b/python/sglang/srt/connector/base_connector.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +import shutil +import signal +import tempfile +from abc import ABC, abstractmethod +from typing import Generator, List, Optional, Tuple + +import torch + + +class BaseConnector(ABC): + """ + For fs connector such as s3: + :/// + + For kv connector such as redis: + ://://keys/ + ://files/ + """ + + def __init__(self, url: str): + self.url = url + self.closed = False + self.local_dir = tempfile.mkdtemp() + for sig in (signal.SIGINT, signal.SIGTERM): + existing_handler = signal.getsignal(sig) + signal.signal(sig, self._close_by_signal(existing_handler)) + + def get_local_dir(self): + return self.local_dir + + @abstractmethod + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + raise NotImplementedError() + + @abstractmethod + def pull_files( + self, + allow_pattern: Optional[List[str]] = None, + ignore_pattern: Optional[List[str]] = None, + ) -> None: + raise NotImplementedError() + + def close(self): + if self.closed: + return + + self.closed = True + if os.path.exists(self.local_dir): + shutil.rmtree(self.local_dir) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __del__(self): + self.close() + + def _close_by_signal(self, existing_handler=None): + + def new_handler(signum, frame): + self.close() + if existing_handler: + existing_handler(signum, frame) + + return new_handler + + +class BaseKVConnector(BaseConnector): + + @abstractmethod + def get(self, key: str) -> Optional[torch.Tensor]: + raise NotImplementedError() + + @abstractmethod + def getstr(self, key: str) -> Optional[str]: + raise NotImplementedError() + + @abstractmethod + def set(self, key: str, obj: torch.Tensor) -> None: + raise NotImplementedError() + + @abstractmethod + def setstr(self, key: str, obj: str) -> None: + raise NotImplementedError() + + @abstractmethod + def list(self, prefix: str) -> List[str]: + raise NotImplementedError() + + +class BaseFileConnector(BaseConnector): + """ + List full file names from remote fs path and filter by allow pattern. + + Args: + allow_pattern: A list of patterns of which files to pull. + + Returns: + list[str]: List of full paths allowed by the pattern + """ + + @abstractmethod + def glob(self, allow_pattern: str) -> List[str]: + raise NotImplementedError() diff --git a/python/sglang/srt/connector/redis.py b/python/sglang/srt/connector/redis.py new file mode 100644 index 000000000..cb1db3f7c --- /dev/null +++ b/python/sglang/srt/connector/redis.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 + +import logging +from typing import Generator, List, Optional, Tuple +from urllib.parse import urlparse + +import torch + +from sglang.srt.connector import BaseKVConnector +from sglang.srt.connector.serde import create_serde +from sglang.srt.connector.utils import pull_files_from_db + +logger = logging.getLogger(__name__) + + +class RedisConnector(BaseKVConnector): + + def __init__(self, url: str): + import redis + + super().__init__(url) + parsed_url = urlparse(url) + self.connection = redis.Redis(host=parsed_url.hostname, port=parsed_url.port) + self.model_name = parsed_url.path.lstrip("/") + # TODO: more serde options + self.s, self.d = create_serde("safe") + + def get(self, key: str) -> Optional[torch.Tensor]: + val = self.connection.get(key) + + if val is None: + logger.error("Key %s not found", key) + return None + + return self.d.from_bytes(val) + + def getstr(self, key: str) -> Optional[str]: + val = self.connection.get(key) + if val is None: + logger.error("Key %s not found", key) + return None + + return val.decode("utf-8") + + def set(self, key: str, tensor: torch.Tensor) -> None: + assert tensor is not None + self.connection.set(key, self.s.to_bytes(tensor)) + + def setstr(self, key: str, obj: str) -> None: + self.connection.set(key, obj) + + def list(self, prefix: str) -> List[str]: + cursor = 0 + all_keys: List[bytes] = [] + + while True: + ret: Tuple[int, List[bytes]] = self.connection.scan( + cursor=cursor, match=f"{prefix}*" + ) # type: ignore + cursor, keys = ret + all_keys.extend(keys) + if cursor == 0: + break + + return [key.decode("utf-8") for key in all_keys] + + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, bytes], None, None]: + keys = self.list(f"{self.model_name}/keys/rank_{rank}/") + for key in keys: + val = self.get(key) + key = key.removeprefix(f"{self.model_name}/keys/rank_{rank}/") + yield key, val + + def pull_files( + self, + allow_pattern: Optional[List[str]] = None, + ignore_pattern: Optional[List[str]] = None, + ) -> None: + pull_files_from_db(self, self.model_name, allow_pattern, ignore_pattern) + + def close(self): + self.connection.close() + super().close() diff --git a/python/sglang/srt/connector/s3.py b/python/sglang/srt/connector/s3.py new file mode 100644 index 000000000..7bef8f5d5 --- /dev/null +++ b/python/sglang/srt/connector/s3.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 + +import fnmatch +import os +from pathlib import Path +from typing import Generator, Optional, Tuple + +import torch + +from sglang.srt.connector import BaseFileConnector + + +def _filter_allow(paths: list[str], patterns: list[str]) -> list[str]: + return [ + path + for path in paths + if any(fnmatch.fnmatch(path, pattern) for pattern in patterns) + ] + + +def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]: + return [ + path + for path in paths + if not any(fnmatch.fnmatch(path, pattern) for pattern in patterns) + ] + + +def list_files( + s3, + path: str, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, +) -> tuple[str, str, list[str]]: + """ + List files from S3 path and filter by pattern. + + Args: + s3: S3 client to use. + path: The S3 path to list from. + allow_pattern: A list of patterns of which files to pull. + ignore_pattern: A list of patterns of which files not to pull. + + Returns: + tuple[str, str, list[str]]: A tuple where: + - The first element is the bucket name + - The second element is string represent the bucket + and the prefix as a dir like string + - The third element is a list of files allowed or + disallowed by pattern + """ + parts = path.removeprefix("s3://").split("/") + prefix = "/".join(parts[1:]) + bucket_name = parts[0] + + objects = s3.list_objects_v2(Bucket=bucket_name, Prefix=prefix) + paths = [obj["Key"] for obj in objects.get("Contents", [])] + + paths = _filter_ignore(paths, ["*/"]) + if allow_pattern is not None: + paths = _filter_allow(paths, allow_pattern) + + if ignore_pattern is not None: + paths = _filter_ignore(paths, ignore_pattern) + + return bucket_name, prefix, paths + + +class S3Connector(BaseFileConnector): + + def __init__(self, url: str) -> None: + import boto3 + + super().__init__(url) + self.client = boto3.client("s3") + + def glob(self, allow_pattern: Optional[list[str]] = None) -> list[str]: + bucket_name, _, paths = list_files( + self.client, path=self.url, allow_pattern=allow_pattern + ) + return [f"s3://{bucket_name}/{path}" for path in paths] + + def pull_files( + self, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, + ) -> None: + """ + Pull files from S3 storage into the temporary directory. + + Args: + s3_model_path: The S3 path of the model. + allow_pattern: A list of patterns of which files to pull. + ignore_pattern: A list of patterns of which files not to pull. + + """ + bucket_name, base_dir, files = list_files( + self.client, self.url, allow_pattern, ignore_pattern + ) + if len(files) == 0: + return + + for file in files: + destination_file = os.path.join(self.local_dir, file.removeprefix(base_dir)) + local_dir = Path(destination_file).parent + os.makedirs(local_dir, exist_ok=True) + self.client.download_file(bucket_name, file, destination_file) + + def weight_iterator( + self, rank: int = 0 + ) -> Generator[Tuple[str, torch.Tensor], None, None]: + from sglang.srt.model_loader.weight_utils import ( + runai_safetensors_weights_iterator, + ) + + # only support safetensor files now + hf_weights_files = self.glob(allow_pattern=["*.safetensors"]) + return runai_safetensors_weights_iterator(hf_weights_files) + + def close(self): + self.client.close() + super().close() diff --git a/python/sglang/srt/connector/serde/__init__.py b/python/sglang/srt/connector/serde/__init__.py new file mode 100644 index 000000000..c05b20afa --- /dev/null +++ b/python/sglang/srt/connector/serde/__init__.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 + +# inspired by LMCache +from typing import Optional, Tuple + +import torch + +from sglang.srt.connector.serde.safe_serde import SafeDeserializer, SafeSerializer +from sglang.srt.connector.serde.serde import Deserializer, Serializer + + +def create_serde(serde_type: str) -> Tuple[Serializer, Deserializer]: + s: Optional[Serializer] = None + d: Optional[Deserializer] = None + + if serde_type == "safe": + s = SafeSerializer() + d = SafeDeserializer() + else: + raise ValueError(f"Unknown serde type: {serde_type}") + + return s, d + + +__all__ = [ + "Serializer", + "Deserializer", + "SafeSerializer", + "SafeDeserializer", + "create_serde", +] diff --git a/python/sglang/srt/connector/serde/safe_serde.py b/python/sglang/srt/connector/serde/safe_serde.py new file mode 100644 index 000000000..3e75f9bfc --- /dev/null +++ b/python/sglang/srt/connector/serde/safe_serde.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Union + +import torch +from safetensors.torch import load, save + +from sglang.srt.connector.serde.serde import Deserializer, Serializer + + +class SafeSerializer(Serializer): + + def __init__(self): + super().__init__() + + def to_bytes(self, t: torch.Tensor) -> bytes: + return save({"tensor_bytes": t.cpu().contiguous()}) + + +class SafeDeserializer(Deserializer): + + def __init__(self): + # TODO: dtype options + super().__init__(torch.float32) + + def from_bytes_normal(self, b: Union[bytearray, bytes]) -> torch.Tensor: + return load(bytes(b))["tensor_bytes"] + + def from_bytes(self, b: Union[bytearray, bytes]) -> torch.Tensor: + return self.from_bytes_normal(b) diff --git a/python/sglang/srt/connector/serde/serde.py b/python/sglang/srt/connector/serde/serde.py new file mode 100644 index 000000000..3d6f804d7 --- /dev/null +++ b/python/sglang/srt/connector/serde/serde.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 + +import abc +from abc import ABC, abstractmethod + +import torch + + +class Serializer(ABC): + + @abstractmethod + def to_bytes(self, t: torch.Tensor) -> bytes: + """ + Serialize a pytorch tensor to bytes. The serialized bytes should contain + both the data and the metadata (shape, dtype, etc.) of the tensor. + + Input: + t: the input pytorch tensor, can be on any device, in any shape, + with any dtype + + Returns: + bytes: the serialized bytes + """ + raise NotImplementedError + + +class Deserializer(metaclass=abc.ABCMeta): + + def __init__(self, dtype): + self.dtype = dtype + + @abstractmethod + def from_bytes(self, bs: bytes) -> torch.Tensor: + """ + Deserialize a pytorch tensor from bytes. + + Input: + bytes: a stream of bytes + + Output: + torch.Tensor: the deserialized pytorch tensor + """ + raise NotImplementedError diff --git a/python/sglang/srt/connector/utils.py b/python/sglang/srt/connector/utils.py new file mode 100644 index 000000000..6cd12da33 --- /dev/null +++ b/python/sglang/srt/connector/utils.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path +from typing import Optional +from urllib.parse import urlparse + +from sglang.srt.connector import BaseConnector + + +def parse_model_name(url: str) -> str: + """ + Parse the model name from the url. + Only used for db connector + """ + parsed_url = urlparse(url) + return parsed_url.path.lstrip("/") + + +def pull_files_from_db( + connector: BaseConnector, + model_name: str, + allow_pattern: Optional[list[str]] = None, + ignore_pattern: Optional[list[str]] = None, +) -> None: + prefix = f"{model_name}/files/" + local_dir = connector.get_local_dir() + files = connector.list(prefix) + + for file in files: + destination_file = os.path.join(local_dir, file.removeprefix(prefix)) + local_dir = Path(destination_file).parent + os.makedirs(local_dir, exist_ok=True) + with open(destination_file, "wb") as f: + f.write(connector.getstr(file).encode("utf-8")) diff --git a/python/sglang/srt/constants.py b/python/sglang/srt/constants.py new file mode 100644 index 000000000..aa03a089b --- /dev/null +++ b/python/sglang/srt/constants.py @@ -0,0 +1,3 @@ +# GPU Memory Types +GPU_MEMORY_TYPE_KV_CACHE = "kv_cache" +GPU_MEMORY_TYPE_WEIGHTS = "weights" diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py new file mode 100644 index 000000000..4fe5d6c77 --- /dev/null +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -0,0 +1,213 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The baseclass of a backend for grammar-guided constrained decoding.""" + +import logging +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from threading import Event +from typing import Dict, List, Optional, Tuple + +import torch + +from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + + +class BaseGrammarObject: + + def __init__(self): + self._finished = False + + def accept_token(self, token: int) -> None: + """ + Accept a token in the grammar. + """ + raise NotImplementedError() + + def rollback(self, k: int): + raise NotImplementedError() + + def is_terminated(self): + return False + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + raise NotImplementedError() + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + raise NotImplementedError() + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + raise NotImplementedError() + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + raise NotImplementedError() + + def copy(self) -> "BaseGrammarObject": + return self + + @property + def finished(self): + return self._finished + + @finished.setter + def finished(self, finished): + self._finished = finished + + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + """ + Try to jump forward in the grammar. + + Returns: + A jump forward helper which may be used in `jump_forward_str_state`. + None if the jump forward is not possible. + """ + raise NotImplementedError() + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + """ + Jump forward for the grammar. + + Returns: + A tuple of the jump forward string and the next state of the grammar + (which can be used in `jump_and_retokenize` if needed). + """ + raise NotImplementedError() + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ) -> None: + """ + Jump forward occurs, and update the grammar state if needed. + """ + raise NotImplementedError() + + +INVALID_GRAMMAR_OBJ = BaseGrammarObject() + + +@dataclass +class CacheEntry: + value: BaseGrammarObject + event: Event + + +class BaseGrammarBackend: + def __init__(self): + self.executor = ThreadPoolExecutor() + self.cache: Dict[Tuple[str, str], CacheEntry] = {} + + def _not_supported(self, key_type: str, key_string: str) -> None: + logger.warning(f"Skip unsupported {key_type=}, {key_string=}") + + def dispatch_fallback( + self, key_type: str, key_string: str + ) -> Optional[BaseGrammarObject]: + """ + This function should not be reached in any case. + """ + raise ValueError(f"Invalid key_type: {key_type}={key_string}") + + def dispatch_json(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("json", key_string) + + def dispatch_regex(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("regex", key_string) + + def dispatch_ebnf(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("ebnf", key_string) + + def dispatch_structural_tag(self, key_string: str) -> Optional[BaseGrammarObject]: + return self._not_supported("structural_tag", key_string) + + def _init_value_dispatch(self, key: Tuple[str, str]) -> Optional[BaseGrammarObject]: + key_type, key_string = key + if key_type == "json": + return self.dispatch_json(key_string) + elif key_type == "regex": + return self.dispatch_regex(key_string) + elif key_type == "ebnf": + return self.dispatch_ebnf(key_string) + elif key_type == "structural_tag": + return self.dispatch_structural_tag(key_string) + elif key_type == "structural_pattern": + return self.dispatch_structural_pattern(key_string) + else: + return self.dispatch_fallback(key_type, key_string) + + def get_cached_or_future_value( + self, key: Tuple[str, str] + ) -> Optional[BaseGrammarObject]: + value = self.cache.get(key) + if value: + return value.copy(), True + value = self.executor.submit(self._init_value_dispatch, key) + return value, False + + def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject): + self.cache[key] = value + + def reset(self): + self.cache.clear() + + +def create_grammar_backend( + server_args: ServerArgs, + tokenizer, + vocab_size: int, + eos_token_ids: Optional[set] = None, +) -> Optional[BaseGrammarBackend]: + if server_args.grammar_backend == "outlines": + from sglang.srt.constrained.outlines_backend import OutlinesGrammarBackend + + grammar_backend = OutlinesGrammarBackend( + tokenizer, + whitespace_pattern=server_args.constrained_json_whitespace_pattern, + ) + elif server_args.grammar_backend == "xgrammar": + from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend + + # Convert Set[int] to List[int] if needed + eos_list = list(eos_token_ids) if eos_token_ids else None + + grammar_backend = XGrammarGrammarBackend( + tokenizer, vocab_size=vocab_size, model_eos_token_ids=eos_list + ) + elif server_args.grammar_backend == "llguidance": + from sglang.srt.constrained.llguidance_backend import GuidanceBackend + + grammar_backend = GuidanceBackend( + tokenizer=tokenizer, + whitespace_pattern=server_args.constrained_json_whitespace_pattern, + ) + elif server_args.grammar_backend == "none": + return None + else: + raise ValueError(f"Invalid grammar backend: {server_args.grammar_backend}") + + if server_args.reasoning_parser and hasattr(tokenizer, "think_end_id"): + from sglang.srt.constrained.reasoner_grammar_backend import ( + ReasonerGrammarBackend, + ) + + grammar_backend = ReasonerGrammarBackend( + grammar_backend, tokenizer.think_end_id + ) + + return grammar_backend diff --git a/python/sglang/srt/constrained/llguidance_backend.py b/python/sglang/srt/constrained/llguidance_backend.py new file mode 100644 index 000000000..2acbf2c51 --- /dev/null +++ b/python/sglang/srt/constrained/llguidance_backend.py @@ -0,0 +1,174 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with llguidance backend.""" + +import json +import logging +import os +from typing import List, Optional, Tuple + +import torch +from llguidance import LLMatcher, LLTokenizer, StructTag, grammar_from +from llguidance.hf import from_tokenizer +from llguidance.torch import ( + allocate_token_bitmask, + apply_token_bitmask_inplace, + fill_next_token_bitmask, +) + +from sglang.srt.constrained.base_grammar_backend import ( + INVALID_GRAMMAR_OBJ, + BaseGrammarBackend, + BaseGrammarObject, +) + +logger = logging.getLogger(__name__) + + +class GuidanceGrammar(BaseGrammarObject): + + def __init__(self, llguidance_tokenizer: LLTokenizer, serialized_grammar: str): + super().__init__() + self.llguidance_tokenizer = llguidance_tokenizer + self.serialized_grammar = serialized_grammar + + self.ll_matcher = LLMatcher( + self.llguidance_tokenizer, + self.serialized_grammar, + log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")), + ) + self.finished = False + self.bitmask = None + + def accept_token(self, token: int): + if not self.ll_matcher.consume_token(token): + logger.warning(f"matcher error: {self.ll_matcher.get_error()}") + self.finished = True + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + if self.ll_matcher.is_stopped(): + self.finished = True + + fill_next_token_bitmask(self.ll_matcher, vocab_mask, idx) + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + if self.bitmask is None or self.bitmask.shape[0] < batch_size: + # only create bitmask when batch gets larger + self.bitmask = allocate_token_bitmask( + batch_size, self.llguidance_tokenizer.vocab_size + ) + bitmask = self.bitmask + else: + bitmask = self.bitmask[:batch_size] + + return bitmask + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask.to(device, non_blocking=True) + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + apply_token_bitmask_inplace(logits, vocab_mask) + + def copy(self): + return GuidanceGrammar( + llguidance_tokenizer=self.llguidance_tokenizer, + serialized_grammar=self.serialized_grammar, + ) + + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + ff_tokens = self.ll_matcher.compute_ff_tokens() + if ff_tokens: + return ff_tokens, "" + else: + return None + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + return "", -1 + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + pass + + +class GuidanceBackend(BaseGrammarBackend): + + def __init__( + self, + tokenizer, + whitespace_pattern: Optional[str] = None, + n_vocab: Optional[int] = None, + ): + super().__init__() + + self.tokenizer = tokenizer + self.whitespace_pattern = whitespace_pattern + self.llguidance_tokenizer = from_tokenizer(self.tokenizer, n_vocab) + + def _from_serialized(self, serialized_grammar) -> Optional[GuidanceGrammar]: + try: + return GuidanceGrammar( + llguidance_tokenizer=self.llguidance_tokenizer, + serialized_grammar=serialized_grammar, + ) + except Exception as e: + logger.error(f"Hit invalid grammar: {serialized_grammar=}, {e=}") + return INVALID_GRAMMAR_OBJ + + def dispatch_json(self, key_string: str) -> Optional[GuidanceGrammar]: + try: + serialized_grammar = LLMatcher.grammar_from_json_schema( + key_string, + defaults={ + "whitespace_pattern": self.whitespace_pattern, + }, + ) + except Exception as e: + logger.error(f"Hit invalid json_schema: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._from_serialized(serialized_grammar) + + def dispatch_regex(self, key_string: str) -> Optional[GuidanceGrammar]: + serialized_grammar = grammar_from("regex", key_string) + return self._from_serialized(serialized_grammar) + + def dispatch_ebnf(self, key_string: str) -> Optional[GuidanceGrammar]: + try: + serialized_grammar = grammar_from("ebnf", key_string) + return self._from_serialized(serialized_grammar) + except ValueError as e: + logger.error(f"Hit invalid ebnf: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + + def dispatch_structural_tag(self, key_string: str) -> Optional[GuidanceGrammar]: + try: + structural_tag = json.loads(key_string) + tags = [ + StructTag( + begin=structure["begin"], + grammar=structure["schema"], + end=structure["end"], + trigger=structural_tag["triggers"][0], # TODO? + ) + for structure in structural_tag["structures"] + ] + g = StructTag.to_grammar(tags) + return self._from_serialized(g) + except Exception as e: + logging.error(f"Hit invalid structural_tag: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ diff --git a/python/sglang/srt/constrained/outlines_backend.py b/python/sglang/srt/constrained/outlines_backend.py new file mode 100644 index 000000000..5302fadaa --- /dev/null +++ b/python/sglang/srt/constrained/outlines_backend.py @@ -0,0 +1,191 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with outlines backend.""" + +import json +import logging +from typing import Dict, List, Optional, Tuple, Union + +import interegular +import torch +from outlines.fsm.guide import RegexGuide +from outlines.models.transformers import TransformerTokenizer +from pydantic import BaseModel + +from sglang.srt.constrained.base_grammar_backend import ( + INVALID_GRAMMAR_OBJ, + BaseGrammarBackend, + BaseGrammarObject, +) +from sglang.srt.constrained.outlines_jump_forward import OutlinesJumpForwardMap + +try: + from outlines.fsm.json_schema import build_regex_from_schema +except ImportError: + from outlines_core.fsm.json_schema import build_regex_from_schema + + +logger = logging.getLogger(__name__) + + +class OutlinesGrammar(BaseGrammarObject): + def __init__( + self, + guide: RegexGuide, + jump_forward_map: Union[OutlinesJumpForwardMap, None], + ) -> None: + super().__init__() + self.guide = guide + self.jump_forward_map = jump_forward_map + self.state = 0 + self.finished = False + + def accept_token(self, token: int): + self.state = self.guide.get_next_state(self.state, token) + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + return torch.zeros(batch_size, vocab_size, dtype=torch.bool, device=device) + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + tokens = torch.tensor( + self.guide.get_next_instruction(self.state).tokens, dtype=torch.int64 + ).to(vocab_mask.device, non_blocking=True) + vocab_mask = vocab_mask[idx] + vocab_mask.fill_(1) + vocab_mask.scatter_(0, tokens, torch.zeros_like(tokens, dtype=torch.bool)) + + @staticmethod + def apply_vocab_mask(logits: torch.Tensor, vocab_mask: torch.Tensor): + logits.masked_fill_(vocab_mask, float("-inf")) + + def copy(self): + return OutlinesGrammar(self.guide, self.jump_forward_map) + + def try_jump_forward(self, tokenizer) -> Optional[Tuple]: + if not self.jump_forward_map: + return None + + jump_forward_bytes = self.jump_forward_map.jump_forward_byte(self.state) + if jump_forward_bytes is None or len(jump_forward_bytes) <= 1: + return None + + # preprocess the jump forward string + suffix_bytes = [] + continuation_range = range(0x80, 0xC0) + cur_state = self.state + while ( + len(jump_forward_bytes) and jump_forward_bytes[0][0] in continuation_range + ): + # continuation bytes + byte_edge = jump_forward_bytes.pop(0) + suffix_bytes.append(byte_edge[0]) + cur_state = byte_edge[1] + + suffix_tokens = [f"<0x{hex(b)[2:].upper()}>" for b in suffix_bytes] + suffix_ids = tokenizer.convert_tokens_to_ids(suffix_tokens) + return suffix_ids, cur_state + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + _, cur_state = helper + return self.jump_forward_map.jump_forward_symbol(cur_state) + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + self.state = next_state + + +class OutlinesGrammarBackend(BaseGrammarBackend): + def __init__( + self, + tokenizer, + whitespace_pattern: bool, + ): + super().__init__() + + try: + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + except AttributeError: + # FIXME: tmp fix for chatglm2 & chatglm3 (pad_token_id=0) + origin_pad_token_id = tokenizer.pad_token_id + + def fset(self, value): + self._value = value + + type(tokenizer).pad_token_id = property( + fget=type(tokenizer).pad_token_id.fget, fset=fset + ) + self.outlines_tokenizer = TransformerTokenizer(tokenizer) + self.outlines_tokenizer.tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token_id = origin_pad_token_id + self.outlines_tokenizer.pad_token = ( + self.outlines_tokenizer.tokenizer.pad_token + ) + self.outlines_tokenizer.vocabulary = ( + self.outlines_tokenizer.tokenizer.get_vocab() + ) + self.whitespace_pattern = whitespace_pattern + + def _compile_regex(self, regex: str) -> Optional[OutlinesGrammar]: + try: + if hasattr(RegexGuide, "from_regex"): + # outlines >= 0.1.1 + guide = RegexGuide.from_regex(regex, self.outlines_tokenizer) + else: + # outlines <= 0.0.46 + guide = RegexGuide(regex, self.outlines_tokenizer) + except interegular.patterns.InvalidSyntax as e: + logger.error(f"Hit invalid regex schema: {regex=}, {e=}") + return INVALID_GRAMMAR_OBJ + + jump_forward_map = None + return OutlinesGrammar(guide, jump_forward_map) + + def dispatch_ebnf(self, key_string: str): + return super().dispatch_ebnf(key_string) + + def dispatch_structural_tag(self, key_string: str): + return super().dispatch_structural_tag(key_string) + + def dispatch_json(self, key_string: str): + try: + regex = build_regex_from_object( + key_string, + whitespace_pattern=self.whitespace_pattern, + ) + except (NotImplementedError, json.decoder.JSONDecodeError, ValueError) as e: + logger.error(f"Hit invalid json_schema: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._compile_regex(regex) + + def dispatch_regex(self, key_string: str): + return self._compile_regex(key_string) + + +def build_regex_from_object( + object: Union[str, BaseModel, Dict], whitespace_pattern: Optional[str] = None +): + if isinstance(object, type(BaseModel)): + schema = json.dumps(object.model_json_schema()) + elif isinstance(object, Dict): + schema = json.dumps(object) + else: + schema = object + return build_regex_from_schema(schema, whitespace_pattern) diff --git a/python/sglang/srt/constrained/outlines_jump_forward.py b/python/sglang/srt/constrained/outlines_jump_forward.py new file mode 100644 index 000000000..cfc65f75f --- /dev/null +++ b/python/sglang/srt/constrained/outlines_jump_forward.py @@ -0,0 +1,200 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Faster constrained decoding with jump forward decoding / compressed finite state machine. +Reference: https://lmsys.org/blog/2024-02-05-compressed-fsm/ +""" + +import dataclasses +import logging +from collections import defaultdict +from typing import Optional + +import interegular +from interegular import InvalidSyntax +from outlines.caching import cache + +from sglang.srt.utils import get_bool_env_var + +try: + # outlines >= 0.1.0 + from outlines_core.fsm.outlines_core_rs import FSMInfo + from outlines_core.fsm.regex import make_byte_level_fsm, make_deterministic_fsm +except ImportError: + # outlines <= 0.0.46 + from outlines.fsm.regex import FSMInfo, make_byte_level_fsm, make_deterministic_fsm + +IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + +# Env var was set in sglang.srt.server_args.ServerArgs.__post__init__ +DISABLE_DISK_CACHE = get_bool_env_var("SGLANG_DISABLE_OUTLINES_DISK_CACHE", "true") + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class JumpEdge: + symbol: str = None + symbol_next_state: int = None + byte: int = None + byte_next_state: int = None + + +def disk_cache(expire: Optional[float] = None, typed=False, ignore=()): + if not DISABLE_DISK_CACHE: + return cache(expire, typed, ignore) + else: + return lambda fn: None + + +@disk_cache() +def init_state_to_jump_forward(regex_string): + try: + regex_pattern = interegular.parse_pattern(regex_string) + except InvalidSyntax as e: + logger.warning(f"skip invalid regex: {regex_string}, {e=}") + return + + byte_fsm = make_byte_level_fsm(regex_pattern.to_fsm().reduce(), keep_utf8=True) + regex_fsm, _ = make_deterministic_fsm(byte_fsm) + + fsm_info: FSMInfo = regex_fsm.fsm_info + + symbol_to_id = fsm_info.alphabet_symbol_mapping + id_to_symbol = {} + for symbol, id_ in symbol_to_id.items(): + id_to_symbol.setdefault(id_, []).append(symbol) + + transitions = fsm_info.transitions + + outgoings_ct = defaultdict(int) + # NOTE(lsyin): Final states can lead to terminate, so they have one outgoing edge naturally + for s in fsm_info.finals: + outgoings_ct[s] = 1 + + state_to_jump_forward = {} + for (state, id_), next_state in transitions.items(): + if id_ == fsm_info.alphabet_anything_value: + # Arbitrarily symbol cannot be recognized as jump forward + continue + + symbols = id_to_symbol[id_] + for c in symbols: + if len(c) > 1: + # Skip byte level transitions like c = "5E" + continue + + outgoings_ct[state] += 1 + if outgoings_ct[state] > 1: + if state in state_to_jump_forward: + del state_to_jump_forward[state] + break + + state_to_jump_forward[state] = JumpEdge( + symbol=c, + symbol_next_state=next_state, + ) + + # Process the byte level jump forward + outgoings_ct = defaultdict(int) + for s in fsm_info.finals: + outgoings_ct[s] = 1 + + for (state, id_), next_state in transitions.items(): + if id_ == fsm_info.alphabet_anything_value: + continue + symbols = id_to_symbol[id_] + for c in symbols: + byte_ = None + if len(c) == 1 and ord(c) < 0x80: + # ASCII character + byte_ = ord(c) + elif len(c) > 1: + # FIXME: This logic is due to the leading \x00 + # https://github.com/outlines-dev/outlines/pull/930 + byte_ = int(symbols[0][1:], 16) + + if byte_ is not None: + outgoings_ct[state] += 1 + if outgoings_ct[state] > 1: + if state in state_to_jump_forward: + del state_to_jump_forward[state] + break + e = state_to_jump_forward.get(state, JumpEdge()) + e.byte = byte_ + e.byte_next_state = next_state + state_to_jump_forward[state] = e + + return state_to_jump_forward + + +class OutlinesJumpForwardMap: + def __init__(self, regex_string): + self.state_to_jump_forward = init_state_to_jump_forward(regex_string) + + def jump_forward_symbol(self, state): + jump_forward_str = "" + next_state = state + while state in self.state_to_jump_forward: + e = self.state_to_jump_forward[state] + if e.symbol is None: + break + jump_forward_str += e.symbol + next_state = e.symbol_next_state + state = next_state + + return jump_forward_str, next_state + + def jump_forward_byte(self, state): + if state not in self.state_to_jump_forward: + return None + + jump_forward_bytes = [] + next_state = None + while state in self.state_to_jump_forward: + e = self.state_to_jump_forward[state] + assert e.byte is not None and e.byte_next_state is not None + jump_forward_bytes.append((e.byte, e.byte_next_state)) + next_state = e.byte_next_state + state = next_state + + return jump_forward_bytes + + def is_jump_forward_symbol_state(self, state): + return ( + state in self.state_to_jump_forward + and self.state_to_jump_forward[state].symbol is not None + ) + + +def test_main(regex_string): + jump_forward_map = OutlinesJumpForwardMap(regex_string) + for state, e in jump_forward_map.state_to_jump_forward.items(): + if e.symbol is not None: + jump_forward_str, next_state = jump_forward_map.jump_forward_symbol(state) + print(f"{state} -> {next_state}", jump_forward_str) + bytes_ = jump_forward_map.jump_forward_byte(state) + print(f"{state} -> {bytes_[-1][1]}", [hex(b) for b, _ in bytes_]) + + +if __name__ == "__main__": + import outlines + + outlines.caching.clear_cache() + test_main(r"The google's DNS sever address is " + IP_REGEX) + test_main(r"霍格沃茨特快列车|霍比特人比尔博") + # 霍格: \xe9\x9c\x8d \xe6\xa0\xbc ... + # 霍比: \xe9\x9c\x8d \xe6\xaf\x94 ... + + test_main(r"[-+]?[0-9]+[ ]*") diff --git a/python/sglang/srt/constrained/reasoner_grammar_backend.py b/python/sglang/srt/constrained/reasoner_grammar_backend.py new file mode 100644 index 000000000..ca5f118ef --- /dev/null +++ b/python/sglang/srt/constrained/reasoner_grammar_backend.py @@ -0,0 +1,90 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The baseclass of a backend for reasoner grammar-guided constrained decoding.""" + +from typing import List, Optional, Tuple + +import torch + +from .base_grammar_backend import BaseGrammarBackend, BaseGrammarObject + + +class ReasonerGrammarObject(BaseGrammarObject): + def __init__(self, grammar: BaseGrammarObject, think_end_id): + super().__init__() + self.grammar = grammar + self.think_end_id = think_end_id + self.is_in_reasoning = True + + def accept_token(self, token: int): + if token == self.think_end_id: + self.is_in_reasoning = False + + if not self.is_in_reasoning and token != self.think_end_id: + self.grammar.accept_token(token) + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + return self.grammar.allocate_vocab_mask(vocab_size, batch_size, device) + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + if not self.is_in_reasoning: + self.grammar.fill_vocab_mask(vocab_mask, idx) + + def move_vocab_mask(self, vocab_mask: torch.Tensor, device) -> torch.Tensor: + return self.grammar.move_vocab_mask(vocab_mask, device) + + @property + def apply_vocab_mask(self): + return self.grammar.apply_vocab_mask + + def copy(self) -> BaseGrammarObject: + return ReasonerGrammarObject(self.grammar.copy(), self.think_end_id) + + @property + def finished(self): + return self.grammar.finished + + @finished.setter + def finished(self, finished): + self.grammar.finished = finished + + def try_jump_forward(self, tokenizer): + return self.grammar.try_jump_forward(tokenizer) + + def jump_forward_str_state(self, helper): + return self.grammar.jump_forward_str_state(helper) + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + return self.grammar.jump_and_retokenize( + old_output_ids, new_output_ids, next_state + ) + + +class ReasonerGrammarBackend(BaseGrammarBackend): + def __init__(self, grammar_backend: BaseGrammarBackend, think_end_id): + super().__init__() + self.grammar_backend = grammar_backend + self.think_end_id = think_end_id + + def _init_value_dispatch( + self, key: Tuple[str, str] + ) -> Optional[ReasonerGrammarObject]: + ret = self.grammar_backend._init_value_dispatch(key) + if ret is None: + return None + return ReasonerGrammarObject(ret, self.think_end_id) diff --git a/python/sglang/srt/constrained/triton_ops/bitmask_ops.py b/python/sglang/srt/constrained/triton_ops/bitmask_ops.py new file mode 100644 index 000000000..9a195c006 --- /dev/null +++ b/python/sglang/srt/constrained/triton_ops/bitmask_ops.py @@ -0,0 +1,141 @@ +# Adapt from +# https://github.com/mlc-ai/xgrammar/blob/v0.1.17/python/xgrammar/kernels/apply_token_bitmask_inplace_triton.py + +from typing import List, Optional, Union + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import get_device_core_count + + +@triton.jit +def apply_token_bitmask_inplace_kernel( + logits_ptr, + bitmask_ptr, + indices_ptr, + num_rows, + vocab_size, + logits_strides, + bitmask_strides, + NUM_SMS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + """Apply a bitmask to logits in-place using Triton. The bitmask is a 01 bitwise compressed tensor, + where 0 means the token is masked and 1 means the token is not masked. After applying the bitmask, + the masked logits will be set to -inf. + + Parameters + ---------- + logits_ptr : tl.tensor + Pointer to the logits tensor to apply the bitmask to. + + bitmask_ptr : tl.tensor + Pointer to the bitmask tensor to apply. + + indices_ptr : Optional[tl.tensor] + Optional pointer to indices tensor specifying which rows to apply the mask to. + + num_rows : int + Number of rows to process. If indices_ptr is provided, this is the number of unique indices. + + vocab_size : int + Size of the vocabulary dimension. If the logits does not have a vocab padding, this is the + same as the logits's second dimension. Otherwise, this is the actual size of the vocabulary. + + logits_strides : int + Stride between rows in the logits tensor. + + bitmask_strides : int + Stride between rows in the bitmask tensor. + + NUM_SMS : int + Number of streaming multiprocessors to use. + + BLOCK_SIZE : int + Size of processing blocks. + """ + + pid = tl.program_id(0) + num_blocks = tl.cdiv(vocab_size, BLOCK_SIZE) + for work_id in tl.range(pid, num_rows * num_blocks, NUM_SMS): + row_id = work_id // num_blocks + block_offset = (work_id % num_blocks) * BLOCK_SIZE + batch_id = row_id if indices_ptr is None else tl.load(indices_ptr + row_id) + offsets = block_offset + tl.arange(0, BLOCK_SIZE) + bitmask_offsets = block_offset // 32 + tl.arange(0, BLOCK_SIZE // 32) + vocab_mask = offsets < vocab_size + packed_bitmask_mask = bitmask_offsets < bitmask_strides + packed_bitmask = tl.load( + bitmask_ptr + batch_id * bitmask_strides + bitmask_offsets, + packed_bitmask_mask, + ) + bitmask = ((packed_bitmask[:, None] >> (tl.arange(0, 32)[None, :])) & 1) == 0 + bitmask = bitmask.reshape(BLOCK_SIZE) + + tl.store( + logits_ptr + batch_id * logits_strides + offsets, + -float("inf"), + vocab_mask & bitmask, + ) + + +def apply_token_bitmask_inplace_triton( + logits: torch.Tensor, + bitmask: torch.Tensor, + indices: Optional[Union[List[int], torch.Tensor]] = None, +): + NUM_SMS = get_device_core_count() + BLOCK_SIZE = 4096 + BITS_PER_BLOCK = 32 + + # Check input dtype + assert bitmask.dtype == torch.int32, "bitmask must be of type int32" + + # Check input tensor shapes. + logits_shape = logits.shape + bitmask_shape = bitmask.shape + if logits.ndim == 1: + logits_shape = (1, logits_shape[0]) + if bitmask.ndim == 1: + bitmask_shape = (1, bitmask_shape[0]) + + required_bitmask_width = (logits_shape[1] + BITS_PER_BLOCK - 1) // BITS_PER_BLOCK + assert required_bitmask_width >= bitmask_shape[1], ( + f"Bitmask width too large: allow at most {required_bitmask_width} int32s for " + f"logits' width {logits_shape[1]}, but got {bitmask_shape[1]}" + ) + + vocab_size = min(logits_shape[1], bitmask_shape[1] * BITS_PER_BLOCK) + + num_rows = None + if isinstance(indices, list) or isinstance(indices, torch.Tensor): + indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) + num_rows = indices.shape[0] + else: + assert ( + logits_shape[0] == bitmask_shape[0] + ), f"batch size mismatch: logits {logits_shape[0]} vs bitmask {bitmask_shape[0]}" + num_rows = logits_shape[0] + + if NUM_SMS > 0: + grid = (NUM_SMS,) + else: + num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE) + grid = (num_rows * num_blocks,) + NUM_SMS = triton.next_power_of_2(grid[0]) + + apply_token_bitmask_inplace_kernel[grid]( + logits, + bitmask, + indices, + num_rows, + vocab_size, + logits_shape[1], + bitmask_shape[1], + NUM_SMS, + BLOCK_SIZE, + num_warps=BLOCK_SIZE // 32 // (16 // logits.element_size()), + num_stages=3, + ) diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py new file mode 100644 index 000000000..7b101df4f --- /dev/null +++ b/python/sglang/srt/constrained/xgrammar_backend.py @@ -0,0 +1,239 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Constrained decoding with xgrammar backend.""" + +import json +import logging +from typing import List, Optional, Tuple, Union + +import torch +from xgrammar import ( + CompiledGrammar, + GrammarCompiler, + GrammarMatcher, + StructuralTagItem, + TokenizerInfo, + allocate_token_bitmask, +) + +from sglang.srt.constrained.base_grammar_backend import ( + INVALID_GRAMMAR_OBJ, + BaseGrammarBackend, + BaseGrammarObject, +) +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +if _is_hip: + from sgl_kernel import apply_token_bitmask_inplace_cuda +else: + from sglang.srt.constrained.triton_ops.bitmask_ops import ( + apply_token_bitmask_inplace_triton, + ) +logger = logging.getLogger(__name__) + + +MAX_ROLLBACK_TOKENS = 200 + + +class XGrammarGrammar(BaseGrammarObject): + + def __init__( + self, + matcher: GrammarMatcher, + vocab_size: int, + ctx: CompiledGrammar, + override_stop_tokens: Optional[Union[List[int], int]], + key_string: Optional[str] = None, # TODO (sk): for debugging, remove later + ) -> None: + self.matcher = matcher + self.vocab_size = vocab_size + self.ctx = ctx + self.override_stop_tokens = override_stop_tokens + self.finished = False + self.accepted_tokens = [] + self.key_string = key_string + + def accept_token(self, token: int): + if not self.is_terminated(): + accepted = self.matcher.accept_token(token) + if not accepted: + # log for debugging + raise ValueError( + f"Tokens not accepted: {token}\n" + f"Accepted tokens: {self.accepted_tokens}\n" + f"Key string: {self.key_string}" + ) + else: + self.accepted_tokens.append(token) + + def rollback(self, k: int): + self.matcher.rollback(k) + self.accepted_tokens = self.accepted_tokens[:-k] + + def is_terminated(self): + return self.matcher.is_terminated() + + def allocate_vocab_mask( + self, vocab_size: int, batch_size: int, device + ) -> torch.Tensor: + return allocate_token_bitmask(batch_size, vocab_size) + + def fill_vocab_mask(self, vocab_mask: torch.Tensor, idx: int) -> None: + self.matcher.fill_next_token_bitmask(vocab_mask, idx) + + @staticmethod + def move_vocab_mask(vocab_mask: torch.Tensor, device) -> torch.Tensor: + return vocab_mask.to(device, non_blocking=True) + + def apply_vocab_mask(self, logits: torch.Tensor, vocab_mask: torch.Tensor) -> None: + if logits.device.type == "cuda": + if _is_hip: + apply_token_bitmask_inplace_cuda(logits, vocab_mask) + else: + apply_token_bitmask_inplace_triton(logits, vocab_mask) + elif logits.device.type == "cpu" and self.apply_vocab_mask_cpu: + self.apply_vocab_mask_cpu(logits, vocab_mask) + else: + raise RuntimeError(f"Unsupported device: {logits.device.type}") + + def copy(self): + matcher = GrammarMatcher( + self.ctx, + max_rollback_tokens=MAX_ROLLBACK_TOKENS, + override_stop_tokens=self.override_stop_tokens, + ) + return XGrammarGrammar( + matcher, + self.vocab_size, + self.ctx, + self.override_stop_tokens, + self.key_string, + ) + + def try_jump_forward(self, tokenizer) -> Optional[Tuple[List[int], str]]: + s = self.matcher.find_jump_forward_string() + if s: + return [], s + return None + + def jump_forward_str_state(self, helper: Tuple[List[int], str]) -> Tuple[str, int]: + _, data = helper + return data, -1 + + def jump_and_retokenize( + self, old_output_ids: List[int], new_output_ids: List[int], next_state: int + ): + k = 0 + for i, old_id in enumerate(old_output_ids): + if old_id == new_output_ids[i]: + k = i + 1 + else: + break + + # rollback to the last token that is the same + if k < len(old_output_ids): + self.matcher.rollback(len(old_output_ids) - k) + + for i in range(k, len(new_output_ids)): + assert self.matcher.accept_token(new_output_ids[i]) + + def __repr__(self): + return f"XGrammarGrammar({self.key_string=}, {self.accepted_tokens=})" + + +class XGrammarGrammarBackend(BaseGrammarBackend): + def __init__( + self, + tokenizer, + vocab_size: int, + model_eos_token_ids: Optional[List[int]] = None, + ): + super().__init__() + + if hasattr(tokenizer, "init_xgrammar"): + # For special tokenizer + tokenizer_info, override_stop_tokens = tokenizer.init_xgrammar() + else: + # Create TokenizerInfo with model's EOS tokens as the authoritative stop tokens + # This ensures consistency between what the model considers EOS and what XGrammar uses + tokenizer_info = TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size, stop_token_ids=model_eos_token_ids + ) + override_stop_tokens = None + + self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info) + self.vocab_size = vocab_size + self.override_stop_tokens = override_stop_tokens + + def _from_context(self, ctx: CompiledGrammar, key_string: str) -> XGrammarGrammar: + matcher = GrammarMatcher( + ctx, + max_rollback_tokens=MAX_ROLLBACK_TOKENS, + override_stop_tokens=self.override_stop_tokens, + ) + return XGrammarGrammar( + matcher, self.vocab_size, ctx, self.override_stop_tokens, key_string + ) + + def dispatch_json(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + if key_string == "$$ANY$$": + # Note: This builtin JSON grammar includes *all* valid JSON (including, for example, arrays at the root) + ctx = self.grammar_compiler.compile_builtin_json_grammar() + else: + ctx = self.grammar_compiler.compile_json_schema(schema=key_string) + + except (RuntimeError, json.decoder.JSONDecodeError) as e: + logging.error(f"Hit invalid json_schema: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._from_context(ctx, key_string) + + def dispatch_ebnf(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + ctx = self.grammar_compiler.compile_grammar(key_string) + except RuntimeError as e: + logging.error(f"Hit invalid ebnf: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._from_context(ctx, key_string) + + def dispatch_regex(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + ctx = self.grammar_compiler.compile_regex(key_string) + except RuntimeError as e: + logging.error(f"Hit invalid regex: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._from_context(ctx, key_string) + + def dispatch_structural_tag(self, key_string: str) -> Optional[XGrammarGrammar]: + try: + structural_tag = json.loads(key_string) + tags = [ + StructuralTagItem( + begin=structure["begin"], + schema=json.dumps(structure["schema"]), + end=structure["end"], + ) + for structure in structural_tag["structures"] + ] + ctx = self.grammar_compiler.compile_structural_tag( + tags, structural_tag["triggers"] + ) + except (RuntimeError, json.decoder.JSONDecodeError) as e: + logging.error(f"Hit invalid structural_tag: {key_string=}, {e=}") + return INVALID_GRAMMAR_OBJ + return self._from_context(ctx, key_string) + + def reset(self): + self.grammar_compiler.clear_cache() diff --git a/python/sglang/srt/custom_op.py b/python/sglang/srt/custom_op.py new file mode 100644 index 000000000..ea3c06e6d --- /dev/null +++ b/python/sglang/srt/custom_op.py @@ -0,0 +1,102 @@ +from torch import nn + +from sglang.srt.utils import ( + cpu_has_amx_support, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, +) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_cpu = is_cpu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_npu = is_npu() +_is_xpu = is_xpu() + + +class CustomOp(nn.Module): + def __init__(self): + super().__init__() + self._forward_method = self.dispatch_forward() + + # States for torch.compile + self._original_forward_method = None + self.is_torch_compile = False + + def enter_torch_compile(self, num_tokens: int): + # Skip if Op is already entered compile mode. + # NOTE(alcanderian): Some Ops(for example RotaryEmbedding) will be reused + # among layers and `enter_torch_compile` will be called many times. + # We should prevent `self._original_forward_method` from being overridden when + # it is not the first time `enter_torch_compile` called. + if self.is_torch_compile: + return + + self._original_forward_method = self._forward_method + # NOTE: Temporarily workaround MoE + # The performance of torch.compile on this layer is not always good when bs > 1, + # so we decide to only use torch.compile when bs=1 + if "FusedMoE" in self.__class__.__name__: + if num_tokens == 1: + from sglang.srt.layers.moe.fused_moe_native import ( + fused_moe_forward_native, + ) + + self._forward_method = fused_moe_forward_native + elif "TopK" in self.__class__.__name__: + if num_tokens == 1: + self._forward_method = self.forward_native + else: + self._forward_method = self.forward_native + self.is_torch_compile = True + + def leave_torch_compile(self): + # Skip if Op is already exited compile mode. + if not self.is_torch_compile: + return + + self._forward_method = self._original_forward_method + self._original_forward_method = None + self.is_torch_compile = False + + # Please do not override this method, because `self._forward_method` can change when in torch compile mode + def forward(self, *args, **kwargs): + return self._forward_method(*args, **kwargs) + + def forward_native(self, *args, **kwargs): + raise NotImplementedError + + def forward_cuda(self, *args, **kwargs): + raise NotImplementedError + + def forward_npu(self, *args, **kwargs): + raise NotImplementedError + + def forward_hip(self, *args, **kwargs): + return self.forward_cuda(*args, **kwargs) + + def forward_xpu(self, *args, **kwargs): + return self.forward_native(*args, **kwargs) + + def forward_hpu(self, *args, **kwargs): + return self.forward_native(*args, **kwargs) + + def forward_cpu(self, *args, **kwargs): + return self.forward_native(*args, **kwargs) + + def dispatch_forward(self): + if _is_cuda: + return self.forward_cuda + elif _is_hip: + return self.forward_hip + elif _is_cpu and _is_cpu_amx_available: + return self.forward_cpu + elif _is_npu: + return self.forward_npu + elif _is_xpu: + return self.forward_xpu + else: + return self.forward_native diff --git a/python/sglang/srt/debug_utils/__init__.py b/python/sglang/srt/debug_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/debug_utils/dump_comparator.py b/python/sglang/srt/debug_utils/dump_comparator.py new file mode 100644 index 000000000..aca9c3b7a --- /dev/null +++ b/python/sglang/srt/debug_utils/dump_comparator.py @@ -0,0 +1,168 @@ +import argparse +import functools +from pathlib import Path + +import polars as pl +import torch + +from sglang.srt.debug_utils.dump_loader import find_row, read_meta +from sglang.srt.debug_utils.dumper import get_truncated_value + + +def main(args): + df_target = read_meta(args.target_path) + df_target = df_target.sort("rank", "dump_index") + df_target = df_target.filter( + (pl.col("forward_pass_id") >= args.start_id) + & (pl.col("forward_pass_id") <= args.end_id) + ) + assert all( + c in df_target.columns + for c in ["rank", "forward_pass_id", "dump_index", "name"] + ) + + df_baseline = read_meta(args.baseline_path) + print("df_target", df_target) + print("df_baseline", df_baseline) + + for row in df_target.iter_rows(named=True): + path_target = Path(args.target_path) / row["filename"] + + row_baseline = find_row( + df_baseline, + conditions=dict( + forward_pass_id=row["forward_pass_id"] + - args.start_id + + args.baseline_start_id, + **{ + k: v + for k, v in row.items() + if k not in ["forward_pass_id", "dump_index", "filename"] + }, + ), + ) + + if row_baseline is None: + print(f"Skip: target={str(path_target)} since no baseline") + x_target = _load_object(path_target) + if x_target is not None: + print(f"x_target(sample)={get_truncated_value(x_target)}") + continue + + path_baseline = Path(args.baseline_path) / row_baseline["filename"] + print(f"Check: target={str(path_target)} baseline={str(path_baseline)}") + check_tensor_pair( + path_baseline=path_baseline, path_target=path_target, name=row["name"] + ) + print() + + +def check_tensor_pair(path_baseline, path_target, name=""): + x_baseline = _load_object(path_baseline) + x_target = _load_object(path_target) + + print( + f"Raw " + f"[shape] {x_baseline.shape} vs {x_target.shape}\t" + f"[dtype] {x_baseline.dtype} vs {x_target.dtype}" + ) + + x_baseline, x_target = _comparison_preprocessor(x_baseline, x_target, name=name) + x_baseline = _try_unify_shape(x_baseline, target_shape=x_target.shape) + + print( + f"After preprocessor " + f"[shape] {x_baseline.shape} vs {x_target.shape}\t" + f"[dtype] {x_baseline.dtype} vs {x_target.dtype}" + ) + + x_target = x_target.float() + x_baseline = x_baseline.float() + + for name, fn in ( + ("mean", torch.mean), + ("std", torch.std), + ("min", torch.min), + ("max", torch.max), + ("p1", functools.partial(torch.quantile, q=0.01)), + ("p5", functools.partial(torch.quantile, q=0.05)), + ("p95", functools.partial(torch.quantile, q=0.95)), + ("p99", functools.partial(torch.quantile, q=0.99)), + ): + value_baseline = fn(x_baseline).item() + value_target = fn(x_target).item() + print( + f"[{name}] {value_baseline :.4f} vs {value_target:.4f} (diff: {value_target - value_baseline:.4f})" + ) + + if x_baseline.shape != x_target.shape: + print(f"⚠️ Shape mismatch") + return + + raw_abs_diff = (x_target - x_baseline).abs() + + max_abs_diff = raw_abs_diff.max().item() + mean_abs_diff = raw_abs_diff.mean().item() + rel_diff = _calc_rel_diff(x_target, x_baseline) + + needs_print = max_abs_diff > 1e-3 + + print( + "\t".join( + f"{'❌' if value > 1e-3 else '✅'} {name}={value}" + for name, value in [ + ("rel_diff", rel_diff), + ("max_abs_diff", max_abs_diff), + ("mean_abs_diff", mean_abs_diff), + ] + ) + ) + + if needs_print: + print(f"x_baseline(sample)={get_truncated_value(x_baseline)}") + print(f"x_target(sample)={get_truncated_value(x_target)}") + + +def _try_unify_shape(x: torch.Tensor, target_shape): + x_shape = x.shape + num_dim_to_remove = len(x_shape) - len(target_shape) + if (x_shape[num_dim_to_remove:] == target_shape) and all( + val == 1 for val in x_shape[:num_dim_to_remove] + ): + out = functools.reduce(lambda a, _: a.squeeze(0), range(num_dim_to_remove), x) + print(f"Unify shape: {x_shape} -> {out.shape} (to match {target_shape})") + return out + + return x + + +# Copied from DeepGEMM +def _calc_rel_diff(x: torch.Tensor, y: torch.Tensor): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def _comparison_preprocessor(x_baseline, x_target, name): + # can insert arbitrary adhoc postprocessing logic here + return x_baseline, x_target + + +def _load_object(path): + x = torch.load(path, weights_only=False) + if not isinstance(x, torch.Tensor): + print(f"Skip load {path} since {type(x)=} is not a Tensor") + return None + return x.cuda() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--baseline-path", type=str) + parser.add_argument("--target-path", type=str) + parser.add_argument("--start-id", type=int, default=0) + parser.add_argument("--end-id", type=int, default=1000000) + parser.add_argument("--baseline-start-id", type=int, default=0) + args = parser.parse_args() + main(args) diff --git a/python/sglang/srt/debug_utils/dump_loader.py b/python/sglang/srt/debug_utils/dump_loader.py new file mode 100644 index 000000000..8e6f2c79b --- /dev/null +++ b/python/sglang/srt/debug_utils/dump_loader.py @@ -0,0 +1,97 @@ +import functools +import os +from pathlib import Path +from typing import Any, Dict + +import polars as pl +import torch + + +class DumpLoader: + def __init__(self): + directory = os.environ.get("SGLANG_DUMP_LOADER_DIR") + + self._enable = directory is not None + if self._enable: + self._directory = Path(directory) + self._df = read_meta(directory) + + @property + def enable(self): + return self._enable + + def load(self, name, **kwargs): + assert self._enable, "Please call DumpLoader.load only when it is enabled" + + from sglang.srt.debug_utils.dumper import dumper + + forward_pass_id = dumper._forward_pass_id + conditions = dict(name=name, forward_pass_id=forward_pass_id, **kwargs) + row = find_row(self._df, conditions=conditions) + assert ( + row is not None + ), f"DumpLoader cannot find row given query {name=} {kwargs=} {self._directory=}" + + path = self._directory / row["filename"] + output = torch.load(path, weights_only=False) + + print( + f"[DumpLoader] load from {path=} (query: {name=} {kwargs=}, output: {type(output)})" + ) + return output + + +def read_meta(directory): + directory = Path(directory) + assert directory.is_dir(), f"{directory=} should be a directory" + + rows = [] + for p in directory.glob("*.pt"): + full_kwargs = {} + for kv in p.stem.split("___"): + k, v = kv.split("=") + full_kwargs[k] = v + rows.append( + { + "filename": str(p.name), + **full_kwargs, + } + ) + + df = pl.DataFrame(rows) + df = df.with_columns( + pl.col("forward_pass_id").cast(int), + pl.col("rank").cast(int), + pl.col("dump_index").cast(int), + ) + return df + + +def find_row(df, conditions: Dict[str, Any]): + df_sub = df.filter( + functools.reduce( + lambda a, b: a & b, + [ + pl.col(col) == _cast_to_polars_dtype(conditions[col], df.schema[col]) + for col in conditions.keys() + ], + ) + ) + assert len(df_sub) <= 1 + return df_sub.to_dicts()[0] if len(df_sub) > 0 else None + + +def _cast_to_polars_dtype(value, target_dtype): + if target_dtype in (pl.Int64, pl.Int32, pl.UInt64, pl.UInt32): + return int(value) + elif target_dtype in (pl.Float64, pl.Float32): + return float(value) + elif target_dtype == pl.Boolean: + return bool(value) + elif target_dtype == pl.String: + return str(value) + else: + return value + + +dump_loader = DumpLoader() diff --git a/python/sglang/srt/debug_utils/dumper.py b/python/sglang/srt/debug_utils/dumper.py new file mode 100644 index 000000000..8a9808bb7 --- /dev/null +++ b/python/sglang/srt/debug_utils/dumper.py @@ -0,0 +1,116 @@ +import os +import time +from pathlib import Path +from typing import Optional + +import torch +import torch.distributed as dist + + +class _Dumper: + """Utility to dump tensors, which can be useful when comparison checking models. + + Example usage: + dumper.on_forward_pass_start() + dumper.dump("layer_start__hidden_states", hidden_states, layer_id=self.layer_id) + + Import from non-SGLang system: + ``` + import sys + sys.path.append("/YOUR_PATH/sglang/python/sglang/srt/debug_utils") + from dumper import dumper + ``` + + Related: `sglang.srt.debug_utils.dump_comparator` for dump comparison + """ + + def __init__(self): + # Do not import `sglang` to make this file standalone + self._enable = bool(int(os.environ.get("SGLANG_DUMPER_ENABLE", "1"))) + self._base_dir = Path(os.environ.get("SGLANG_DUMPER_DIR", "/tmp")) + self._enable_write_file = bool( + int(os.environ.get("SGLANG_DUMPER_WRITE_FILE", "1")) + ) + self._partial_name: Optional[str] = None + self._dump_index = 0 + self._forward_pass_id = 0 + + def on_forward_pass_start(self): + self._forward_pass_id += 1 + print( + f"[Dumper] [{time.time()}] on_forward_pass_start id={self._forward_pass_id}" + ) + + def dump(self, name, value, **kwargs): + if not self._enable: + return + + assert ( + self._forward_pass_id >= 1 + ), "Do you forget to call `dumper.on_forward_pass_start()`?" + self._dump_index += 1 + + if self._partial_name is None: + self._partial_name = _get_partial_name() + + rank = _get_rank() + full_kwargs = dict( + forward_pass_id=self._forward_pass_id, + rank=rank, + name=name, + dump_index=self._dump_index, + **kwargs, + ) + full_filename = "___".join(f"{k}={v}" for k, v in full_kwargs.items()) + ".pt" + path = self._base_dir / f"sglang_dump_{self._partial_name}" / full_filename + + sample_value = get_truncated_value(value) + + print( + f"[Dumper] [{rank}, {time.time()}] {path} " + f"type={type(value)} " + f"shape={value.shape if isinstance(value, torch.Tensor) else None} " + f"dtype={value.dtype if isinstance(value, torch.Tensor) else None} " + f"sample_value={sample_value}" + ) + + if self._enable_write_file: + path.parent.mkdir(parents=True, exist_ok=True) + torch.save(value, str(path)) + + +def _get_partial_name(): + rank = _get_rank() + object_list = [str(time.time()) if rank == 0 else None] + if dist.is_initialized(): + dist.broadcast_object_list(object_list, device="cuda") + return object_list[0] + + +def _get_rank(): + if dist.is_initialized(): + return dist.get_rank() + else: + return 0 + + +def get_truncated_value(value): + if value is None: + return None + + if isinstance(value, tuple): + return [get_truncated_value(x) for x in value] + + if not isinstance(value, torch.Tensor): + return None + + if value.numel() < 200: + return value + + slices = [ + slice(0, 5) if dim_size > 200 else slice(None) for dim_size in value.shape + ] + return value[tuple(slices)] + + +dumper = _Dumper() diff --git a/python/sglang/srt/debug_utils/text_comparator.py b/python/sglang/srt/debug_utils/text_comparator.py new file mode 100644 index 000000000..3a6df19b9 --- /dev/null +++ b/python/sglang/srt/debug_utils/text_comparator.py @@ -0,0 +1,234 @@ +import argparse +import hashlib +import json +from pathlib import Path + +import polars as pl + +_DESCRIPTION = """Compare and find differences to benchmark outputs. + +Supported inputs: +* The samples jsonl from `lm_eval --log_samples --output_path FOLDER_NAME` +* The output from `gsm8k/bench_sglang.py --raw-result-file FILE_NAME` (or mmlu) +""" + + +def main(args): + if args.data_type == "simple_evals": + df_input = _compute_df_input_mode_simple_evals(args) + else: + df_input = _transform_df_input(_compute_df_raw(args)) + + assert all( + c in df_input.columns + for c in ["category", "trial_index", "prompt_id", "prompt", "output", "correct"] + ) + + df_meta = _compute_df_meta(df_input) + + df_correctness_per_trial = df_input.group_by( + "category", "trial_index", maintain_order=True + ).agg(pl.col("correct").mean()) + df_correctness_delta = ( + df_meta.group_by("correctness_delta").len().sort("correctness_delta") + ) + df_good_to_bad = df_meta.filter(pl.col("correctness_delta") < 0) + df_bad_to_good = df_meta.filter(pl.col("correctness_delta") > 0) + + print(f"Dump output to {args.output_path}") + Path(args.output_path).write_text( + json.dumps( + dict( + df_meta=df_meta.to_dicts(), + df_good_to_bad=df_good_to_bad.to_dicts(), + df_bad_to_good=df_bad_to_good.to_dicts(), + ), + indent=4, + ), + ) + + if not args.disable_print_details: + with pl.Config( + fmt_str_lengths=10000, + tbl_cols=-1, + tbl_rows=-1, + tbl_width_chars=-1, + tbl_formatting="UTF8_FULL", + ): + print("====== Correctness per trial ======") + print(df_correctness_per_trial) + + print( + "====== Correctness Delta (-1.0 means all-right becomes all-wrong) ======" + ) + print(df_correctness_delta) + + for name, df in [ + ("Good->Bad", df_good_to_bad), + ("Bad->Good", df_bad_to_good), + ]: + print(f"====== Concrete Examples: {name} ======") + print(df) + + +def _compute_df_input_mode_simple_evals(args): + return pl.concat( + [ + _compute_df_input_one_mode_simple_evals(**info) + for info in _get_file_infos(args=args) + ] + ) + + +def _compute_df_input_one_mode_simple_evals(path, category, trial_index): + data = json.loads(Path(path).read_text()) + rows = [] + + for single_eval_result in data["metadata"]["single_eval_results"]: + prompt = single_eval_result["example_level_metadata"][ + "actual_queried_prompt_messages" + ] + score = single_eval_result["score"] + assert score in {0.0, 1.0}, f"{score=}" + + row = dict( + category=category, + trial_index=trial_index, + prompt_id=_compute_id_from_object(prompt), + prompt=json.dumps(prompt), + output=single_eval_result["example_level_metadata"]["response_text"], + correct=score == 1.0, + ) + rows.append(row) + + return pl.DataFrame(rows) + + +def _compute_id_from_object(obj): + if isinstance(obj, pl.Series): + obj = obj.to_list() + json_str = json.dumps(obj, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(json_str.encode("utf-8")).hexdigest() + + +def _compute_df_raw(args): + return pl.concat( + [ + _read_df_raw( + path=info["path"], + category=info["category"], + trial_index=info["trial_index"], + ) + for info in _get_file_infos(args=args) + ] + ) + + +def _get_file_infos(args): + return [ + dict(path=path, category=category, trial_index=trial_index) + for category, paths in [ + ("baseline", args.baseline_path), + ("target", args.target_path), + ] + for trial_index, path in enumerate(paths) + ] + + +def _read_df_raw(path: str, category: str, trial_index: int): + return pl.read_ndjson(path).with_columns( + category=pl.lit(category), trial_index=trial_index + ) + + +def _transform_df_input(df: pl.DataFrame): + if "doc_id" in df.columns: + print("Transform mode: lm_eval") + + filter_names = df["filter"].unique(maintain_order=True).to_list() + if len(filter_names) > 1: + filter_name = filter_names[0] + print(f"Choose {filter_name=} among {filter_names}") + df = df.filter(pl.col("filter") == filter_name) + + df = df.select( + pl.col("category"), + pl.col("trial_index"), + prompt_id=pl.col("doc_id"), + prompt=pl.col("arguments").struct.field("gen_args_0").struct.field("arg_0"), + output=pl.col("resps").list.get(0).list.get(0), + correct=pl.col("exact_match").cast(bool), + ) + + return df + elif "prompt_id" in df.columns: + print("Transform mode: SGLang bench") + return df + else: + raise Exception( + f"Unknown data: {df.columns}. You may need to set `--data-type` if using e.g. simple_evals." + ) + + +def _compute_df_meta(df_input: pl.DataFrame): + df_input = df_input.sort("prompt_id", "category", "trial_index") + df_meta = pl.DataFrame( + [ + _handle_one_prompt(df_one_prompt) + for df_one_prompt in df_input.partition_by("prompt_id", maintain_order=True) + ] + ) + df_meta = df_meta.with_columns( + correctness_delta=pl.col("correctness_target") - pl.col("correctness_baseline"), + ) + df_meta = df_meta.sort("correctness_delta", "output_same_prefix_len") + return df_meta + + +def _handle_one_prompt(df_one_prompt: pl.DataFrame): + assert ( + len(set(_compute_id_from_object(obj) for obj in df_one_prompt["prompt"])) == 1 + ) + + df_baseline = df_one_prompt.filter(pl.col("category") == "baseline") + df_target = df_one_prompt.filter(pl.col("category") == "target") + + outputs_baseline = df_baseline["output"].to_list() + outputs_target = df_target["output"].to_list() + + output_same_prefix_len = max( + _compute_str_prefix_len(output_baseline, output_target) + for output_baseline in outputs_baseline + for output_target in outputs_target + ) + + return dict( + prompt_id=df_one_prompt[0, "prompt_id"], + correctness_baseline=df_baseline["correct"].mean(), + correctness_target=df_target["correct"].mean(), + output_same_prefix_len=output_same_prefix_len, + prompt=df_one_prompt[0, "prompt"], + outputs_baseline=outputs_baseline, + outputs_target=outputs_target, + ) + + +def _compute_str_prefix_len(a: str, b: str) -> int: + min_len = min(len(a), len(b)) + for i in range(min_len): + if a[i] != b[i]: + return i + return min_len + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=_DESCRIPTION) + parser.add_argument("--data-type", type=str, default="auto") + parser.add_argument("--baseline-path", type=str, nargs="+") + parser.add_argument("--target-path", type=str, nargs="+") + parser.add_argument( + "--output-path", type=str, default="/tmp/text_comparator_output.json" + ) + parser.add_argument("--disable-print-details", action="store_true") + args = parser.parse_args() + main(args) diff --git a/python/sglang/srt/disaggregation/ascend/__init__.py b/python/sglang/srt/disaggregation/ascend/__init__.py new file mode 100644 index 000000000..2550f91a4 --- /dev/null +++ b/python/sglang/srt/disaggregation/ascend/__init__.py @@ -0,0 +1,6 @@ +from sglang.srt.disaggregation.ascend.conn import ( + AscendKVBootstrapServer, + AscendKVManager, + AscendKVReceiver, + AscendKVSender, +) diff --git a/python/sglang/srt/disaggregation/ascend/conn.py b/python/sglang/srt/disaggregation/ascend/conn.py new file mode 100644 index 000000000..b0009fc7c --- /dev/null +++ b/python/sglang/srt/disaggregation/ascend/conn.py @@ -0,0 +1,117 @@ +import concurrent.futures +import logging +from typing import List, Tuple + +import numpy as np +import numpy.typing as npt + +from sglang.srt.disaggregation.ascend.transfer_engine import AscendTransferEngine +from sglang.srt.disaggregation.common.utils import group_concurrent_contiguous +from sglang.srt.disaggregation.mooncake.conn import ( + MooncakeKVBootstrapServer, + MooncakeKVManager, + MooncakeKVReceiver, + MooncakeKVSender, +) +from sglang.srt.utils import get_local_ip_by_remote + +logger = logging.getLogger(__name__) + + +class AscendKVManager(MooncakeKVManager): + def init_engine(self): + # TransferEngine initialized on ascend. + local_ip = get_local_ip_by_remote() + self.engine = AscendTransferEngine( + hostname=local_ip, + npu_id=self.kv_args.gpu_id, + disaggregation_mode=self.disaggregation_mode, + ) + + def register_buffer_to_engine(self): + self.engine.batch_register(self.kv_args.kv_data_ptrs, self.kv_args.kv_data_lens) + # The Ascend backend optimize batch registration for small memory blocks. + self.engine.batch_register( + self.kv_args.aux_data_ptrs, self.kv_args.aux_data_lens + ) + + def send_kvcache( + self, + mooncake_session_id: str, + prefill_kv_indices: npt.NDArray[np.int32], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int32], + executor: concurrent.futures.ThreadPoolExecutor, + ): + # Group by indices + prefill_kv_blocks, dst_kv_blocks = group_concurrent_contiguous( + prefill_kv_indices, dst_kv_indices + ) + + num_layers = len(self.kv_args.kv_data_ptrs) + layers_params = [ + ( + self.kv_args.kv_data_ptrs[layer_id], + dst_kv_ptrs[layer_id], + self.kv_args.kv_item_lens[layer_id], + ) + for layer_id in range(num_layers) + ] + + def set_transfer_blocks( + src_ptr: int, dst_ptr: int, item_len: int + ) -> List[Tuple[int, int, int]]: + transfer_blocks = [] + for prefill_index, decode_index in zip(prefill_kv_blocks, dst_kv_blocks): + src_addr = src_ptr + int(prefill_index[0]) * item_len + dst_addr = dst_ptr + int(decode_index[0]) * item_len + length = item_len * len(prefill_index) + transfer_blocks.append((src_addr, dst_addr, length)) + return transfer_blocks + + # Worker function for processing a single layer + def process_layer(src_ptr: int, dst_ptr: int, item_len: int) -> int: + transfer_blocks = set_transfer_blocks(src_ptr, dst_ptr, item_len) + return self._transfer_data(mooncake_session_id, transfer_blocks) + + # Worker function for processing all layers in a batch + def process_layers(layers_params: List[Tuple[int, int, int]]) -> int: + transfer_blocks = [] + for src_ptr, dst_ptr, item_len in layers_params: + transfer_blocks.extend(set_transfer_blocks(src_ptr, dst_ptr, item_len)) + return self._transfer_data(mooncake_session_id, transfer_blocks) + + if self.enable_custom_mem_pool: + futures = [ + executor.submit( + process_layer, + src_ptr, + dst_ptr, + item_len, + ) + for (src_ptr, dst_ptr, item_len) in layers_params + ] + for future in concurrent.futures.as_completed(futures): + status = future.result() + if status != 0: + for f in futures: + f.cancel() + return status + else: + # Combining all layers' params in one batch transfer is more efficient + # compared to using multiple threads + return process_layers(layers_params) + + return 0 + + +class AscendKVSender(MooncakeKVSender): + pass + + +class AscendKVReceiver(MooncakeKVReceiver): + pass + + +class AscendKVBootstrapServer(MooncakeKVBootstrapServer): + pass diff --git a/python/sglang/srt/disaggregation/ascend/transfer_engine.py b/python/sglang/srt/disaggregation/ascend/transfer_engine.py new file mode 100644 index 000000000..0ccffffd6 --- /dev/null +++ b/python/sglang/srt/disaggregation/ascend/transfer_engine.py @@ -0,0 +1,58 @@ +import logging +import os +from typing import List, Optional + +from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine +from sglang.srt.disaggregation.utils import DisaggregationMode + +logger = logging.getLogger(__name__) + + +class AscendTransferEngine(MooncakeTransferEngine): + + def __init__( + self, hostname: str, npu_id: int, disaggregation_mode: DisaggregationMode + ): + try: + from mf_adapter import TransferEngine + except ImportError as e: + raise ImportError( + "Please install mf_adapter, for details, see docs/backend/pd_disaggregation.md" + ) from e + + self.engine = TransferEngine() + self.hostname = hostname + self.npu_id = npu_id + + # Centralized storage address of the AscendTransferEngine + self.store_url = os.getenv("ASCEND_MF_STORE_URL") + if disaggregation_mode == DisaggregationMode.PREFILL: + self.role = "Prefill" + elif disaggregation_mode == DisaggregationMode.DECODE: + self.role = "Decode" + else: + logger.error(f"Unsupported DisaggregationMode: {disaggregation_mode}") + raise ValueError(f"Unsupported DisaggregationMode: {disaggregation_mode}") + self.session_id = f"{self.hostname}:{self.engine.get_rpc_port()}" + self.initialize() + + def initialize(self) -> None: + """Initialize the ascend transfer instance.""" + ret_value = self.engine.initialize( + self.store_url, + self.session_id, + self.role, + self.npu_id, + ) + if ret_value != 0: + logger.error("Ascend Transfer Engine initialization failed.") + raise RuntimeError("Ascend Transfer Engine initialization failed.") + + def batch_register(self, ptrs: List[int], lengths: List[int]): + try: + ret_value = self.engine.batch_register_memory(ptrs, lengths) + except Exception: + # Mark register as failed + ret_value = -1 + if ret_value != 0: + logger.debug(f"Ascend memory registration for ptr {ptrs} failed.") diff --git a/python/sglang/srt/disaggregation/base/__init__.py b/python/sglang/srt/disaggregation/base/__init__.py new file mode 100644 index 000000000..ef0f797fc --- /dev/null +++ b/python/sglang/srt/disaggregation/base/__init__.py @@ -0,0 +1,8 @@ +from sglang.srt.disaggregation.base.conn import ( + BaseKVBootstrapServer, + BaseKVManager, + BaseKVReceiver, + BaseKVSender, + KVArgs, + KVPoll, +) diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py new file mode 100644 index 000000000..3f5877ea3 --- /dev/null +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, List, Optional + +import numpy as np +import numpy.typing as npt + +from sglang.srt.server_args import ServerArgs + +if TYPE_CHECKING: + from sglang.srt.disaggregation.utils import DisaggregationMode + + +class KVArgs: + engine_rank: int + kv_data_ptrs: List[int] + kv_data_lens: List[int] + kv_item_lens: List[int] + aux_data_ptrs: List[int] + aux_data_lens: List[int] + aux_item_lens: List[int] + ib_device: str + ib_traffic_class: str + gpu_id: int + # for different tp + decode_tp_size: int + kv_head_num: int + page_size: int + # for pp prefill + prefill_pp_size: int + pp_rank: int + prefill_start_layer: int + # for system dp + system_dp_rank: int + + +class KVPoll: + Failed = 0 + Bootstrapping = 1 + WaitingForInput = 2 + Transferring = 3 + Success = 4 + + +class BaseKVManager(ABC): + """Base class for managing transfers states""" + + @abstractmethod + def __init__( + self, + args: KVArgs, + disaggregation_mode: DisaggregationMode, + server_args: ServerArgs, + is_mla_backend: Optional[bool] = False, + ): ... + + +class BaseKVSender(ABC): + + @abstractmethod + def __init__( + self, + mgr: BaseKVManager, + bootstrap_addr: str, + bootstrap_room: int, + dest_tp_ranks: List[int], + pp_rank: int, + ): ... + + @abstractmethod + def init(self, num_kv_indices: int, aux_index: Optional[int] = None): + """ + Notify the decoder server about the kv indices length and aux index + """ + ... + + @abstractmethod + def send(self, kv_indices: npt.NDArray[np.int32]): + """ + Send the kv cache at the given kv indices to the decoder server + """ + ... + + @abstractmethod + def poll(self) -> KVPoll: + """ + Check the status of the kv cache transfer + """ + ... + + @abstractmethod + def failure_exception(self): + """ + Raise an exception if the kv cache transfer fails + """ + ... + + +class BaseKVReceiver(ABC): + + @abstractmethod + def __init__( + self, + mgr: BaseKVManager, + bootstrap_addr: str, + bootstrap_room: Optional[int] = None, + ): ... + + @abstractmethod + def init(self, kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None): + """ + Notify the prefill server about the kv indices and aux index + """ + ... + + @abstractmethod + def poll(self) -> KVPoll: + """ + Check the status of the kv cache transfer + """ + ... + + @abstractmethod + def failure_exception(self): + """ + Raise an exception if the kv cache transfer fails + """ + ... + + +class BaseKVBootstrapServer(ABC): + @abstractmethod + def __init__(self, host: str, port: int): ... diff --git a/python/sglang/srt/disaggregation/common/__init__.py b/python/sglang/srt/disaggregation/common/__init__.py new file mode 100644 index 000000000..8294c3892 --- /dev/null +++ b/python/sglang/srt/disaggregation/common/__init__.py @@ -0,0 +1,5 @@ +from sglang.srt.disaggregation.common.conn import ( + CommonKVBootstrapServer, + CommonKVManager, + CommonKVReceiver, +) diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py new file mode 100644 index 000000000..10b6093b9 --- /dev/null +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -0,0 +1,438 @@ +from __future__ import annotations + +import asyncio +import logging +import socket +import threading +from functools import cache +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import numpy.typing as npt +import requests +import zmq +from aiohttp import web + +from sglang.srt.disaggregation.base.conn import ( + BaseKVBootstrapServer, + BaseKVManager, + BaseKVReceiver, + BaseKVSender, + KVArgs, + KVPoll, +) +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ( + format_tcp_address, + get_free_port, + get_ip, + get_local_ip_by_remote, + is_valid_ipv6_address, + maybe_wrap_ipv6_address, +) + +logger = logging.getLogger(__name__) + + +class CommonKVManager(BaseKVManager): + def __init__( + self, + args: KVArgs, + disaggregation_mode: DisaggregationMode, + server_args: ServerArgs, + is_mla_backend: Optional[bool] = False, + ): + self.kv_args = args + self.is_mla_backend = is_mla_backend + self.disaggregation_mode = disaggregation_mode + # for p/d multi node infer + self.bootstrap_host = server_args.host + self.bootstrap_port = server_args.disaggregation_bootstrap_port + self.dist_init_addr = server_args.dist_init_addr + self.tp_size = server_args.tp_size + self.dp_size = server_args.dp_size + self.enable_dp_attention = server_args.enable_dp_attention + if not server_args.enable_dp_attention and server_args.dp_size != 1: + raise ValueError( + "If dp_attention is not enabled, dp size must be 1 in disaggregation mode." + ) + + self.rank_port = get_free_port() + if self.disaggregation_mode == DisaggregationMode.PREFILL: + self._register_to_bootstrap() + elif self.disaggregation_mode == DisaggregationMode.DECODE: + self.connection_pool: Dict[str, Dict[str, Union[str, int]]] = {} + self.prefill_tp_size_table: Dict[str, int] = {} + self.prefill_dp_size_table: Dict[str, int] = {} + else: + raise ValueError( + f"Unsupported DisaggregationMode: {self.disaggregation_mode}" + ) + + def _register_to_bootstrap(self): + """Register KVSender to bootstrap server via HTTP POST.""" + if self.dist_init_addr: + # multi node: bootstrap server's host is dist_init_addr + if self.dist_init_addr.startswith("["): # [ipv6]:port or [ipv6] + if self.dist_init_addr.endswith("]"): + host = self.dist_init_addr + else: + host, _ = self.dist_init_addr.rsplit(":", 1) + else: + host = socket.gethostbyname(self.dist_init_addr.rsplit(":", 1)[0]) + else: + # single node: bootstrap server's host is same as http server's host + host = self.bootstrap_host + host = maybe_wrap_ipv6_address(host) + + bootstrap_server_url = f"{host}:{self.bootstrap_port}" + url = f"http://{bootstrap_server_url}/route" + payload = { + "role": "Prefill", + "tp_size": self.tp_size, + "dp_size": self.dp_size, + "rank_ip": get_local_ip_by_remote(), + "rank_port": self.rank_port, + "engine_rank": self.kv_args.engine_rank, + } + + try: + response = requests.put(url, json=payload) + if response.status_code == 200: + logger.debug("Prefill successfully registered to bootstrap server.") + else: + logger.error( + f"Prefill Failed to connect to bootstrap server: {response.status_code}, {response.text}" + ) + except Exception as e: + logger.error(f"Prefill Failed to register to bootstrap server: {e}") + + @cache + def _connect(self, endpoint: str, is_ipv6: bool = False): + socket = zmq.Context().socket(zmq.PUSH) + if is_ipv6: + socket.setsockopt(zmq.IPV6, 1) + socket.connect(endpoint) + return socket + + +class CommonKVReceiver(BaseKVReceiver): + _ctx = zmq.Context() + _socket_cache = {} + _socket_locks = {} + _global_lock = threading.Lock() + + def __init__( + self, + mgr: BaseKVManager, + bootstrap_addr: str, + bootstrap_room: Optional[int] = None, + prefill_dp_rank: Optional[int] = None, + ): + self.bootstrap_room = bootstrap_room + self.bootstrap_addr = bootstrap_addr + self.kv_mgr = mgr + + if self.bootstrap_addr not in self.kv_mgr.prefill_dp_size_table: + self.prefill_tp_size, self.prefill_dp_size = ( + self._get_prefill_dp_size_from_server() + ) + if self.prefill_tp_size is None or self.prefill_dp_size is None: + logger.error( + f"Could not fetch prefill parallel info for bootstrap_addr: {self.bootstrap_addr}" + ) + else: + self.kv_mgr.prefill_tp_size_table[self.bootstrap_addr] = ( + self.prefill_tp_size + ) + self.kv_mgr.prefill_dp_size_table[self.bootstrap_addr] = ( + self.prefill_dp_size + ) + else: + self.prefill_tp_size = self.kv_mgr.prefill_tp_size_table[ + self.bootstrap_addr + ] + self.prefill_dp_size = self.kv_mgr.prefill_dp_size_table[ + self.bootstrap_addr + ] + + # Currently, we don't allow prefill instance and decode instance to + # have different TP sizes per DP rank, except for models using MLA. + local_tp_size_per_dp_rank = self.kv_mgr.tp_size // self.kv_mgr.dp_size + prefill_tp_size_per_dp_rank = self.prefill_tp_size // self.prefill_dp_size + if local_tp_size_per_dp_rank == prefill_tp_size_per_dp_rank: + self.target_tp_rank = ( + self.kv_mgr.kv_args.engine_rank % local_tp_size_per_dp_rank + ) + self.required_dst_info_num = 1 + self.target_tp_ranks = [self.target_tp_rank] + elif local_tp_size_per_dp_rank > prefill_tp_size_per_dp_rank: + self.target_tp_rank = ( + self.kv_mgr.kv_args.engine_rank % local_tp_size_per_dp_rank + ) // (local_tp_size_per_dp_rank // prefill_tp_size_per_dp_rank) + self.required_dst_info_num = ( + local_tp_size_per_dp_rank // prefill_tp_size_per_dp_rank + ) + self.target_tp_ranks = [self.target_tp_rank] + else: + assert ( + self.kv_mgr.is_mla_backend + ), "PD with different TP sizes per DP rank is not yet supported for non-MLA models" + + # For non-MLA models, one decode rank needs to retrieve KVCache from multiple prefill ranks for non MLA models; + self.target_tp_ranks = [ + rank + for rank in range( + (self.kv_mgr.kv_args.engine_rank % local_tp_size_per_dp_rank) + * (prefill_tp_size_per_dp_rank // local_tp_size_per_dp_rank), + (self.kv_mgr.kv_args.engine_rank % local_tp_size_per_dp_rank + 1) + * (prefill_tp_size_per_dp_rank // local_tp_size_per_dp_rank), + ) + ] + + # For MLA models, we can retrieve KVCache from only one prefill rank, but we still need to maintain + # multiple connections in the connection pool and have to send dummy requests to other prefill ranks, + # or the KVPoll will never be set correctly + self.target_tp_rank = self.target_tp_ranks[0] + self.required_dst_info_num = 1 + + if prefill_dp_rank is not None: + logger.debug(f"Targeting DP rank: {prefill_dp_rank}") + self.prefill_dp_rank = prefill_dp_rank + else: + self.prefill_dp_rank = bootstrap_room % self.prefill_dp_size + + # FIXME: alias here: target_dp_group -> prefill_dp_rank + self.target_dp_group = self.prefill_dp_rank + + # NOTE: key distinguished by bootstrap_addr, target_dp_group, and target_tp_rank + bootstrap_key = ( + f"{self.bootstrap_addr}_{self.target_dp_group}_{self.target_tp_rank}" + ) + + if bootstrap_key not in self.kv_mgr.connection_pool: + bootstrap_infos = [] + for target_tp_rank in self.target_tp_ranks: + bootstrap_info = self._get_bootstrap_info_from_server( + target_tp_rank, + self.target_dp_group, + ) + if bootstrap_info is not None: + # NOTE: only support MLA for now: select one prefill rank as real rank + bootstrap_info["is_dummy"] = not bool( + target_tp_rank == self.target_tp_rank + or self.target_tp_rank is None + ) + bootstrap_infos.append(bootstrap_info) + else: + logger.error( + f"Could not fetch bootstrap info for engine rank: {self.kv_mgr.kv_args.engine_rank} and target_dp_group: {self.target_dp_group}" + ) + self.bootstrap_infos = bootstrap_infos + + if len(self.bootstrap_infos) == 0: + logger.error( + f"Could not fetch bootstrap info for engine rank: {self.kv_mgr.kv_args.engine_rank}" + ) + else: + self.kv_mgr.connection_pool[bootstrap_key] = self.bootstrap_infos + # Register kv_args only once to prefill KVManager according to the info fetched from the bootstrap server + self._register_kv_args() + else: + self.bootstrap_infos = self.kv_mgr.connection_pool[bootstrap_key] + + assert len(self.bootstrap_infos) > 0 + + def _get_bootstrap_info_from_server(self, engine_rank, target_dp_group): + """Fetch the bootstrap info from the bootstrap server.""" + try: + url = f"http://{self.bootstrap_addr}/route?engine_rank={engine_rank}&target_dp_group={target_dp_group}" + response = requests.get(url) + if response.status_code == 200: + bootstrap_info = response.json() + return bootstrap_info + else: + logger.error( + f"Failed to get prefill server info: {response.status_code}, {response.text}" + ) + return None + except Exception as e: + logger.error(f"Error fetching prefill info from bootstrap: {e}") + return None + + def _get_prefill_dp_size_from_server(self) -> int: + """Fetch the prefill parallel info from the bootstrap server.""" + try: + url = f"http://{self.bootstrap_addr}/route?engine_rank={-1}&target_dp_group={-1}" + response = requests.get(url) + if response.status_code == 200: + prefill_parallel_info = response.json() + return int(prefill_parallel_info["prefill_tp_size"]), int( + prefill_parallel_info["prefill_dp_size"] + ) + else: + logger.error( + f"Failed to get prefill parallel info: {response.status_code}, {response.text}" + ) + return None + except Exception as e: + logger.error(f"Error fetching prefill parallel info from bootstrap: {e}") + return None + + @classmethod + def _connect(cls, endpoint: str, is_ipv6: bool = False): + with cls._global_lock: + if endpoint not in cls._socket_cache: + sock = cls._ctx.socket(zmq.PUSH) + if is_ipv6: + sock.setsockopt(zmq.IPV6, 1) + sock.connect(endpoint) + cls._socket_cache[endpoint] = sock + cls._socket_locks[endpoint] = threading.Lock() + return cls._socket_cache[endpoint], cls._socket_locks[endpoint] + + @classmethod + def _connect_to_bootstrap_server(cls, bootstrap_info: dict): + ip_address = bootstrap_info["rank_ip"] + port = bootstrap_info["rank_port"] + is_ipv6_address = is_valid_ipv6_address(ip_address) + sock, lock = cls._connect( + format_tcp_address(ip_address, port), is_ipv6=is_ipv6_address + ) + return sock, lock + + def _register_kv_args(self): + pass + + def failure_exception(self): + raise Exception("Fake KVReceiver Exception") + + +class CommonKVBootstrapServer(BaseKVBootstrapServer): + def __init__(self, host: str, port: int): + self.host = host + self.port = port + self.app = web.Application() + self.store = dict() + self.lock = asyncio.Lock() + self._setup_routes() + self.tp_size = None + self.dp_size = None + self.tp_size_per_dp_rank = None + self.prefill_port_table: Dict[int, Dict[int, Dict[str, Union[str, int]]]] = {} + + # Start bootstrap server + self.thread = threading.Thread(target=self._run_server, daemon=True) + self.run() + + def run(self): + self.thread.start() + + def _setup_routes(self): + self.app.router.add_route("*", "/route", self._handle_route) + + async def _handle_route(self, request: web.Request): + method = request.method + if method == "PUT": + return await self._handle_route_put(request) + elif method == "GET": + return await self._handle_route_get(request) + else: + return web.Response( + text="Method not allowed", status=405, content_type="application/json" + ) + + async def _handle_route_put(self, request: web.Request): + data = await request.json() + role = data["role"] + tp_size = data["tp_size"] + dp_size = data["dp_size"] + rank_ip = data["rank_ip"] + rank_port = int(data["rank_port"]) + engine_rank = int(data["engine_rank"]) + + if self.tp_size is None: + self.tp_size = tp_size + + if self.dp_size is None: + self.dp_size = dp_size + + tp_size_per_dp_rank = tp_size // dp_size + if self.tp_size_per_dp_rank == None: + self.tp_size_per_dp_rank = tp_size_per_dp_rank + + # Add lock to make sure thread-safe + if role == "Prefill": + dp_group = engine_rank // tp_size_per_dp_rank + tp_rank_in_dp_group = engine_rank % tp_size_per_dp_rank + + async with self.lock: + if dp_group not in self.prefill_port_table: + self.prefill_port_table[dp_group] = {} + + self.prefill_port_table[dp_group][tp_rank_in_dp_group] = { + "rank_ip": rank_ip, + "rank_port": rank_port, + } + logger.debug( + f"Register Prefill bootstrap: {engine_rank} with rank_ip: {rank_ip} and rank_port: {rank_port}" + ) + + return web.Response(text="OK", status=200) + + async def _handle_route_get(self, request: web.Request): + engine_rank = request.query.get("engine_rank") + target_dp_group = request.query.get("target_dp_group") + if not engine_rank or not target_dp_group: + return web.Response(text="Missing inputs for bootstrap server.", status=400) + + # Currently we use engine_rank == -1 and target_dp_group == -1 to sync dp size + if int(engine_rank) == -1 and int(target_dp_group) == -1: + prefill_parallel_info = { + "prefill_tp_size": self.tp_size, + "prefill_dp_size": self.dp_size, + } + return web.json_response(prefill_parallel_info, status=200) + + # Find corresponding prefill info + async with self.lock: + bootstrap_info = self.prefill_port_table[int(target_dp_group)][ + int(engine_rank) + ] + + if bootstrap_info is not None: + return web.json_response(bootstrap_info, status=200) + else: + return web.Response(text="Bootstrap info not Found", status=404) + + def _run_server(self): + try: + # Event Loop + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + + self._runner = web.AppRunner(self.app) + self._loop.run_until_complete(self._runner.setup()) + + site = web.TCPSite(self._runner, host=self.host, port=self.port) + self._loop.run_until_complete(site.start()) + self._loop.run_forever() + except Exception as e: + logger.error(f"Server error: {str(e)}") + finally: + # Cleanup + self._loop.run_until_complete(self._runner.cleanup()) + self._loop.close() + + def close(self): + """Shutdown""" + if self._loop is not None and self._loop.is_running(): + self._loop.call_soon_threadsafe(self._loop.stop) + logger.info("Stopping server loop...") + + if self.thread.is_alive(): + self.thread.join(timeout=2) + logger.info("Server thread stopped") + + def poll(self) -> KVPoll: ... diff --git a/python/sglang/srt/disaggregation/common/utils.py b/python/sglang/srt/disaggregation/common/utils.py new file mode 100644 index 000000000..6f3da2128 --- /dev/null +++ b/python/sglang/srt/disaggregation/common/utils.py @@ -0,0 +1,42 @@ +import threading +from collections import deque +from typing import List, Tuple + +import numpy as np +import numpy.typing as npt + + +class FastQueue: + def __init__(self): + self._buf = deque() + self._cond = threading.Condition() + + def put(self, item): + with self._cond: + self._buf.append(item) + # wake up a thread of wait() + self._cond.notify() + + def get(self): + with self._cond: + # if queue is empty ,block until is notified() + while not self._buf: + self._cond.wait() + return self._buf.popleft() + + +def group_concurrent_contiguous( + src_indices: npt.NDArray[np.int32], dst_indices: npt.NDArray[np.int32] +) -> Tuple[List[npt.NDArray[np.int32]], List[npt.NDArray[np.int32]]]: + """Vectorised NumPy implementation.""" + if src_indices.size == 0: + return [], [] + + brk = np.where((np.diff(src_indices) != 1) | (np.diff(dst_indices) != 1))[0] + 1 + src_groups = np.split(src_indices, brk) + dst_groups = np.split(dst_indices, brk) + + src_groups = [g.tolist() for g in src_groups] + dst_groups = [g.tolist() for g in dst_groups] + + return src_groups, dst_groups diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py new file mode 100644 index 000000000..b79c8ca87 --- /dev/null +++ b/python/sglang/srt/disaggregation/decode.py @@ -0,0 +1,894 @@ +""" +Life cycle of a request in the decode server + +1. PreallocQueue: + a. Initialize a receiver for each request + b. The request handshakes first, and pre-allocate kv once there is available kv. + c. Move the request to TransferQueue. + +2. TransferQueue: + a. Poll the receiver to check the transfer state + b. If the transfer has finished, move the request to waiting queue + +3. WaitingQueue: + a. Use the requests in the queue to construct a PrebuiltExtendBatch + b. Skip the prefill forward but only populate metadata + +4. RunningBatch: + a. Merge the resolved PrebuiltExtendBatch into running batch to run decoding +""" + +from __future__ import annotations + +import logging +from collections import deque +from dataclasses import dataclass +from http import HTTPStatus +from typing import TYPE_CHECKING, List, Optional, Tuple, Type, Union + +import torch +from torch.distributed import ProcessGroup + +from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE +from sglang.srt.disaggregation.base import BaseKVManager, BaseKVReceiver, KVPoll +from sglang.srt.disaggregation.utils import ( + FAKE_BOOTSTRAP_HOST, + DisaggregationMode, + KVClassType, + MetadataBuffers, + ReqToMetadataIdxAllocator, + TransferBackend, + get_kv_class, + is_mla_backend, + kv_to_page_indices, + poll_and_all_reduce, + prepare_abort, +) +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.managers.schedule_batch import FINISH_ABORT, ScheduleBatch +from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator +from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache +from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter +from sglang.srt.utils import get_int_env_var, require_mlp_sync + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import Req + from sglang.srt.managers.scheduler import Scheduler + +CLIP_MAX_NEW_TOKEN = get_int_env_var("SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION", 4096) + + +class DecodeReqToTokenPool: + """ + The difference of DecodeReqToTokenPool and ReqToTokenPool is that + DecodeReqToTokenPool subscribes memory for pre-allocated requests. + + In ReqToTokenPool, if `--max-running-requests` is 8, + #pre-allocated + #transfer + #running <= 8, but there are in fact more memory can carry pre-allocated requests. + + In DecodeReqToTokenPool, if `--max-running-requests` is 8, + #running <= 8, #pre-allocated + #transfer <= pre_alloc_size, so we can use the free memory to pre-allocate requests to unblock prefill. + """ + + def __init__( + self, + size: int, + max_context_len: int, + device: str, + enable_memory_saver: bool, + pre_alloc_size: int, + ): + memory_saver_adapter = TorchMemorySaverAdapter.create( + enable=enable_memory_saver + ) + + self.size = size + self.max_context_len = max_context_len + self.device = device + self.pre_alloc_size = pre_alloc_size + with memory_saver_adapter.region(tag=GPU_MEMORY_TYPE_KV_CACHE): + self.req_to_token = torch.zeros( + (size + pre_alloc_size, max_context_len), + dtype=torch.int32, + device=device, + ) + + self.free_slots = list(range(size + pre_alloc_size)) + + def write(self, indices, values): + self.req_to_token[indices] = values + + def available_size(self): + return len(self.free_slots) + + def alloc(self, need_size: int) -> List[int]: + if need_size > len(self.free_slots): + return None + + select_index = self.free_slots[:need_size] + self.free_slots = self.free_slots[need_size:] + return select_index + + def free(self, free_index: Union[int, List[int]]): + if isinstance(free_index, (int,)): + self.free_slots.append(free_index) + else: + self.free_slots.extend(free_index) + + def clear(self): + self.free_slots = list(range(self.size + self.pre_alloc_size)) + + +@dataclass +class DecodeRequest: + req: Req + kv_receiver: BaseKVReceiver + waiting_for_input: bool = False + metadata_buffer_index: int = -1 + + +class DecodePreallocQueue: + """ + Store the requests that are preallocating. + """ + + def __init__( + self, + req_to_token_pool: ReqToTokenPool, + token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, + draft_token_to_kv_pool: Optional[KVCache], + req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, + metadata_buffers: MetadataBuffers, + scheduler: Scheduler, + transfer_queue: DecodeTransferQueue, + tree_cache: BasePrefixCache, + gloo_group: ProcessGroup, + tp_rank: int, + tp_size: int, + dp_size: int, + gpu_id: int, + bootstrap_port: int, + max_total_num_tokens: int, + prefill_pp_size: int, + num_reserved_decode_tokens: int, + transfer_backend: TransferBackend, + ): + self.req_to_token_pool = req_to_token_pool + self.token_to_kv_pool_allocator = token_to_kv_pool_allocator + self.token_to_kv_pool = token_to_kv_pool_allocator.get_kvcache() + self.draft_token_to_kv_pool = draft_token_to_kv_pool + self.is_mla_backend = is_mla_backend(self.token_to_kv_pool) + self.metadata_buffers = metadata_buffers + self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator + self.scheduler = scheduler + self.transfer_queue = transfer_queue + self.tree_cache = tree_cache # this is always a chunk cache + self.gloo_group = gloo_group + self.tp_rank = tp_rank + self.tp_size = tp_size + self.dp_size = dp_size + self.gpu_id = gpu_id + self.bootstrap_port = bootstrap_port + self.max_total_num_tokens = max_total_num_tokens + self.prefill_pp_size = prefill_pp_size + self.num_reserved_decode_tokens = num_reserved_decode_tokens + self.transfer_backend = transfer_backend + # Queue for requests pending pre-allocation + self.queue: List[DecodeRequest] = [] + self.retracted_queue: List[Req] = [] + self.prefill_pp_size = prefill_pp_size + self.kv_manager = self._init_kv_manager() + + def _init_kv_manager(self) -> BaseKVManager: + kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS) + kv_args = kv_args_class() + + attn_tp_size = get_attention_tp_size() + kv_args.engine_rank = self.tp_rank % (attn_tp_size) + + kv_args.decode_tp_size = attn_tp_size + # Note(shangming): pp is not supported on the decode side yet, so its rank is fixed to 0 + kv_args.pp_rank = 0 + kv_args.system_dp_rank = self.scheduler.dp_rank + kv_args.prefill_pp_size = self.prefill_pp_size + kv_data_ptrs, kv_data_lens, kv_item_lens = ( + self.token_to_kv_pool.get_contiguous_buf_infos() + ) + if self.draft_token_to_kv_pool is not None: + # We should also transfer draft model kv cache. The indices are + # always shared with a target model. + draft_kv_data_ptrs, draft_kv_data_lens, draft_kv_item_lens = ( + self.draft_token_to_kv_pool.get_contiguous_buf_infos() + ) + kv_data_ptrs += draft_kv_data_ptrs + kv_data_lens += draft_kv_data_lens + kv_item_lens += draft_kv_item_lens + + kv_args.kv_data_ptrs = kv_data_ptrs + kv_args.kv_data_lens = kv_data_lens + kv_args.kv_item_lens = kv_item_lens + + kv_args.aux_data_ptrs, kv_args.aux_data_lens, kv_args.aux_item_lens = ( + self.metadata_buffers.get_buf_infos() + ) + + kv_args.ib_device = self.scheduler.server_args.disaggregation_ib_device + kv_args.gpu_id = self.scheduler.gpu_id + kv_manager_class: Type[BaseKVManager] = get_kv_class( + self.transfer_backend, KVClassType.MANAGER + ) + kv_manager: BaseKVManager = kv_manager_class( + kv_args, + DisaggregationMode.DECODE, + self.scheduler.server_args, + self.is_mla_backend, + ) + return kv_manager + + def add(self, req: Req, is_retracted: bool = False) -> None: + """Add a request to the pending queue.""" + if self._check_if_req_exceed_kv_capacity(req): + return + + if is_retracted: + self.retracted_queue.append(req) + else: + if req.bootstrap_host == FAKE_BOOTSTRAP_HOST: + kv_receiver_class = get_kv_class( + TransferBackend.FAKE, KVClassType.RECEIVER + ) + else: + kv_receiver_class = get_kv_class( + self.transfer_backend, KVClassType.RECEIVER + ) + + kv_receiver = kv_receiver_class( + mgr=self.kv_manager, + bootstrap_addr=f"{req.bootstrap_host}:{req.bootstrap_port}", + bootstrap_room=req.bootstrap_room, + prefill_dp_rank=req.data_parallel_rank, + ) + + self.queue.append( + DecodeRequest(req=req, kv_receiver=kv_receiver, waiting_for_input=False) + ) + + def _check_if_req_exceed_kv_capacity(self, req: Req) -> bool: + if len(req.origin_input_ids) > self.max_total_num_tokens: + message = f"Request {req.rid} exceeds the maximum number of tokens: {len(req.origin_input_ids)} > {self.max_total_num_tokens}" + logger.error(message) + prepare_abort(req, message, status_code=HTTPStatus.BAD_REQUEST) + self.scheduler.stream_output([req], req.return_logprob) + return True + return False + + def extend(self, reqs: List[Req], is_retracted: bool = False) -> None: + """Add a request to the pending queue.""" + for req in reqs: + self.add(req, is_retracted=is_retracted) + + def resume_retracted_reqs(self) -> List[Req]: + # TODO refactor the scheduling part, reuse with the unified engine logic as much as possible + + # allocate memory + resumed_reqs = [] + indices_to_remove = set() + allocatable_tokens = self._allocatable_tokens(count_retracted=False) + + for i, req in enumerate(self.retracted_queue): + if self.req_to_token_pool.available_size() <= 0: + break + + required_tokens_for_request = ( + len(req.origin_input_ids) + + len(req.output_ids) + + self.num_reserved_decode_tokens + ) + if required_tokens_for_request > allocatable_tokens: + break + + resumed_reqs.append(req) + indices_to_remove.add(i) + req.is_retracted = False + self._pre_alloc(req) + allocatable_tokens -= required_tokens_for_request + + # load from cpu, release the cpu copy + req.load_kv_cache(self.req_to_token_pool, self.token_to_kv_pool_allocator) + + self.retracted_queue = [ + entry + for i, entry in enumerate(self.retracted_queue) + if i not in indices_to_remove + ] + + return resumed_reqs + + def _update_handshake_waiters(self) -> None: + if not self.queue: + return + + if all(decode_req.waiting_for_input for decode_req in self.queue): + return + + polls = poll_and_all_reduce( + [decode_req.kv_receiver for decode_req in self.queue], self.gloo_group + ) + + for i, (decode_req, poll) in enumerate(zip(self.queue, polls)): + if poll == KVPoll.Bootstrapping: + pass + elif poll == KVPoll.WaitingForInput: + decode_req.waiting_for_input = True + elif poll == KVPoll.Failed: + error_message = f"Decode handshake failed for request rank={self.tp_rank} {decode_req.req.rid=} {decode_req.req.bootstrap_room=}" + try: + decode_req.kv_receiver.failure_exception() + except Exception as e: + error_message += f" with exception {e}" + logger.error(error_message) + prepare_abort( + decode_req.req, + error_message, + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) + if self.scheduler.enable_metrics: + self.scheduler.metrics_collector.increment_bootstrap_failed_reqs() + else: + raise ValueError(f"Unexpected poll case: {poll}") + + def pop_preallocated(self) -> List[DecodeRequest]: + """Pop the preallocated requests from the pending queue (FIFO).""" + self._update_handshake_waiters() + + preallocated_reqs = [] + indices_to_remove = set() + + # We need to make sure that the sum of inflight tokens and allocatable tokens is greater than maximum input+output length of each inflight request + # Otherwise it is possible for one request running decode out of memory, while all other requests are in the transfer queue that cannot be retracted. + retractable_tokens = sum( + len(r.origin_input_ids) + len(r.output_ids) + for r in self.scheduler.running_batch.reqs + ) + allocatable_tokens = self._allocatable_tokens( + retractable_tokens=retractable_tokens, count_retracted=True + ) + # First, remove all failed requests from the queue + for i, decode_req in enumerate(self.queue): + if isinstance(decode_req.req.finished_reason, FINISH_ABORT): + self.scheduler.stream_output( + [decode_req.req], decode_req.req.return_logprob + ) + indices_to_remove.add(i) + + # Then, preallocate the remaining requests if possible + for i, decode_req in enumerate(self.queue): + if i in indices_to_remove: + continue + + if not decode_req.waiting_for_input: + continue + + if self.req_to_token_pool.available_size() <= 0: + break + + if self.req_to_metadata_buffer_idx_allocator.available_size() <= 0: + break + + # Memory estimation: don't add if the projected memory cannot be met + # TODO: add new_token ratio + origin_input_len = len(decode_req.req.origin_input_ids) + required_tokens_for_request = ( + origin_input_len + self.num_reserved_decode_tokens + ) + + if ( + max( + required_tokens_for_request, + origin_input_len + + min( + decode_req.req.sampling_params.max_new_tokens, + CLIP_MAX_NEW_TOKEN, + ) + - retractable_tokens, + ) + > allocatable_tokens + ): + break + if required_tokens_for_request > allocatable_tokens: + break + + allocatable_tokens -= required_tokens_for_request + self._pre_alloc(decode_req.req) + + kv_indices = ( + self.req_to_token_pool.req_to_token[decode_req.req.req_pool_idx][ + : len(decode_req.req.origin_input_ids) + ] + .cpu() + .numpy() + ) + + decode_req.metadata_buffer_index = ( + self.req_to_metadata_buffer_idx_allocator.alloc() + ) + assert decode_req.metadata_buffer_index is not None + page_indices = kv_to_page_indices( + kv_indices, self.token_to_kv_pool_allocator.page_size + ) + decode_req.kv_receiver.init(page_indices, decode_req.metadata_buffer_index) + preallocated_reqs.append(decode_req) + indices_to_remove.add(i) + + self.queue = [ + entry for i, entry in enumerate(self.queue) if i not in indices_to_remove + ] + + return preallocated_reqs + + @property + def num_tokens_pre_allocated(self): + return sum( + len(decode_req.req.fill_ids) for decode_req in self.transfer_queue.queue + ) + + def _allocatable_tokens( + self, retractable_tokens: Optional[int] = None, count_retracted: bool = True + ) -> int: + need_space_for_single_req = ( + max( + [ + min(x.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKEN) + + len(x.origin_input_ids) + - retractable_tokens + for x in self.scheduler.running_batch.reqs + ] + ) + if retractable_tokens is not None + and len(self.scheduler.running_batch.reqs) > 0 + else 0 + ) + + if self.scheduler.model_config.is_hybrid: + available_size = min( + self.token_to_kv_pool_allocator.full_available_size(), + self.token_to_kv_pool_allocator.swa_available_size(), + ) + else: + available_size = self.token_to_kv_pool_allocator.available_size() + + allocatable_tokens = available_size - max( + # preserve some space for future decode + self.num_reserved_decode_tokens + * ( + len(self.scheduler.running_batch.reqs) + + len(self.transfer_queue.queue) + + len(self.scheduler.waiting_queue) + ), + # make sure each request can finish if reach max_tokens with all other requests retracted + need_space_for_single_req, + ) + + # Note: if the last fake extend just finishes, and we enter `pop_preallocated` immediately in the next iteration + # the extend batch is not in any queue, so we need to explicitly add the tokens slots here + if ( + self.scheduler.last_batch + and self.scheduler.last_batch.forward_mode.is_extend() + ): + allocatable_tokens -= self.num_reserved_decode_tokens * len( + self.scheduler.last_batch.reqs + ) + + if count_retracted: + allocatable_tokens -= sum( + [ + len(req.origin_input_ids) + + len(req.output_ids) + + self.num_reserved_decode_tokens + for req in self.retracted_queue + ] + ) + return allocatable_tokens + + def _pre_alloc(self, req: Req) -> torch.Tensor: + """Pre-allocate the memory for req_to_token and token_kv_pool""" + req_pool_indices = self.req_to_token_pool.alloc(1) + + assert ( + req_pool_indices is not None + ), "req_pool_indices is full! There is a bug in memory estimation." + + req.req_pool_idx = req_pool_indices[0] + + if self.token_to_kv_pool_allocator.page_size == 1: + kv_loc = self.token_to_kv_pool_allocator.alloc( + len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0) + ) + else: + num_tokens = len(req.origin_input_ids) + max(len(req.output_ids) - 1, 0) + kv_loc = self.token_to_kv_pool_allocator.alloc_extend( + prefix_lens=torch.tensor( + [0], + dtype=torch.int64, + device=self.token_to_kv_pool_allocator.device, + ), + seq_lens=torch.tensor( + [num_tokens], + dtype=torch.int64, + device=self.token_to_kv_pool_allocator.device, + ), + last_loc=torch.tensor( + [-1], + dtype=torch.int64, + device=self.token_to_kv_pool_allocator.device, + ), + extend_num_tokens=num_tokens, + ) + + assert ( + kv_loc is not None + ), "KV cache is full! There is a bug in memory estimation." + + self.req_to_token_pool.write((req.req_pool_idx, slice(0, len(kv_loc))), kv_loc) + + # populate metadata + req.fill_ids = req.origin_input_ids + req.output_ids + req.extend_input_len = len(req.origin_input_ids) + + return kv_loc + + +class DecodeTransferQueue: + """ + Store the requests that is polling kv + """ + + def __init__( + self, + gloo_group: ProcessGroup, + req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, + tp_rank: int, + metadata_buffers: MetadataBuffers, + scheduler: Scheduler, + tree_cache: BasePrefixCache, + ): + self.queue: List[DecodeRequest] = [] + self.gloo_group = gloo_group + self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator + self.tp_rank = tp_rank + self.metadata_buffers = metadata_buffers + self.scheduler = scheduler + self.tree_cache = tree_cache + self.spec_algorithm = scheduler.spec_algorithm + + def add(self, decode_req: DecodeRequest) -> None: + self.queue.append(decode_req) + + def extend(self, decode_reqs: List[DecodeRequest]) -> None: + self.queue.extend(decode_reqs) + + def pop_transferred(self) -> List[Req]: + if not self.queue: + return [] + polls = poll_and_all_reduce( + [decode_req.kv_receiver for decode_req in self.queue], self.gloo_group + ) + + transferred_reqs = [] + indices_to_remove = set() + for i, (decode_req, poll) in enumerate(zip(self.queue, polls)): + if poll == KVPoll.Failed: + error_message = f"Decode transfer failed for request rank={self.tp_rank} {decode_req.req.rid=} {decode_req.req.bootstrap_room=}" + try: + decode_req.kv_receiver.failure_exception() + except Exception as e: + error_message += f" with exception {e}" + logger.error(error_message) + prepare_abort( + decode_req.req, + error_message, + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) + self.scheduler.stream_output( + [decode_req.req], decode_req.req.return_logprob + ) + # unlock the kv cache or it will have memory leak + self.tree_cache.cache_finished_req(decode_req.req) + indices_to_remove.add(i) + if self.scheduler.enable_metrics: + self.scheduler.metrics_collector.increment_transfer_failed_reqs() + continue + elif poll == KVPoll.Success: + + idx = decode_req.metadata_buffer_index + ( + output_id, + output_token_logprobs_val, + output_token_logprobs_idx, + output_top_logprobs_val, + output_top_logprobs_idx, + output_hidden_states, + ) = self.metadata_buffers.get_buf(idx) + + decode_req.req.output_ids.append(output_id[0].item()) + if not self.spec_algorithm.is_none(): + decode_req.req.hidden_states_tensor = output_hidden_states + if decode_req.req.return_logprob: + decode_req.req.output_token_logprobs_val.append( + output_token_logprobs_val[0].item() + ) + decode_req.req.output_token_logprobs_idx.append( + output_token_logprobs_idx[0].item() + ) + decode_req.req.output_top_logprobs_val.append( + output_top_logprobs_val[ + : decode_req.req.top_logprobs_num + ].tolist() + ) + decode_req.req.output_top_logprobs_idx.append( + output_top_logprobs_idx[ + : decode_req.req.top_logprobs_num + ].tolist() + ) + + if hasattr(decode_req.kv_receiver, "clear"): + decode_req.kv_receiver.clear() + + # special handling for sampling_params.max_new_tokens == 1 + if decode_req.req.sampling_params.max_new_tokens == 1: + # finish immediately + decode_req.req.check_finished() + self.scheduler.stream_output( + [decode_req.req], decode_req.req.return_logprob + ) + self.tree_cache.cache_finished_req(decode_req.req) + else: + transferred_reqs.append(decode_req.req) + + indices_to_remove.add(i) + elif poll in [ + KVPoll.Bootstrapping, + KVPoll.WaitingForInput, + KVPoll.Transferring, + ]: + pass + else: + raise ValueError(f"Unexpected poll case: {poll}") + + for i in indices_to_remove: + idx = self.queue[i].metadata_buffer_index + assert idx != -1 + self.req_to_metadata_buffer_idx_allocator.free(idx) + + self.queue = [ + entry for i, entry in enumerate(self.queue) if i not in indices_to_remove + ] + + return transferred_reqs + + +class SchedulerDisaggregationDecodeMixin: + + @torch.no_grad() + def event_loop_normal_disagg_decode(self: Scheduler): + """A normal scheduler loop for decode worker in disaggregation mode.""" + + while True: + recv_reqs = self.recv_requests() + self.process_input_requests(recv_reqs) + # polling and allocating kv cache + self.process_decode_queue() + batch = self.get_next_disagg_decode_batch_to_run() + self.cur_batch = batch + + prepare_mlp_sync_flag = require_mlp_sync(self.server_args) + + if batch: + # Generate fake extend output. + if batch.forward_mode.is_extend(): + # Note: Logprobs should be handled on the prefill engine. + self.stream_output( + batch.reqs, any(req.return_logprob for req in batch.reqs) + ) + if prepare_mlp_sync_flag: + self._prepare_idle_batch_and_run(None) + else: + if prepare_mlp_sync_flag: + self.prepare_mlp_sync_batch(batch) + result = self.run_batch(batch) + self.process_batch_result(batch, result) + elif prepare_mlp_sync_flag: + batch, _ = self._prepare_idle_batch_and_run(None) + + if batch is None and ( + len(self.waiting_queue) + + len(self.disagg_decode_transfer_queue.queue) + + len(self.disagg_decode_prealloc_queue.queue) + == 0 + ): + self.self_check_during_idle() + + self.last_batch = batch + + @torch.no_grad() + def event_loop_overlap_disagg_decode(self: Scheduler): + result_queue = deque() + self.last_batch: Optional[ScheduleBatch] = None + self.last_batch_in_queue = False # last batch is modified in-place, so we need another variable to track if it's extend + + while True: + recv_reqs = self.recv_requests() + self.process_input_requests(recv_reqs) + # polling and allocating kv cache + self.process_decode_queue() + batch = self.get_next_disagg_decode_batch_to_run() + self.cur_batch = batch + last_batch_in_queue = False + + prepare_mlp_sync_flag = require_mlp_sync(self.server_args) + + if batch: + # Generate fake extend output. + if batch.forward_mode.is_extend(): + # Note: Logprobs should be handled on the prefill engine. + self.stream_output( + batch.reqs, any(req.return_logprob for req in batch.reqs) + ) + if prepare_mlp_sync_flag: + batch_, result = self._prepare_idle_batch_and_run( + None, delay_process=True + ) + if batch_: + result_queue.append((batch_.copy(), result)) + last_batch_in_queue = True + else: + if prepare_mlp_sync_flag: + self.prepare_mlp_sync_batch(batch) + result = self.run_batch(batch) + result_queue.append((batch.copy(), result)) + + if (self.last_batch is None) or (not self.last_batch_in_queue): + # Create a dummy first batch to start the pipeline for overlap schedule. + # It is now used for triggering the sampling_info_done event. + tmp_batch = ScheduleBatch( + reqs=None, + forward_mode=ForwardMode.DUMMY_FIRST, + next_batch_sampling_info=self.tp_worker.cur_sampling_info, + ) + self.set_next_batch_sampling_info_done(tmp_batch) + last_batch_in_queue = True + + elif prepare_mlp_sync_flag: + batch, result = self._prepare_idle_batch_and_run( + None, delay_process=True + ) + if batch: + result_queue.append((batch.copy(), result)) + last_batch_in_queue = True + + # Process the results of the previous batch but skip if the last batch is extend + if self.last_batch and self.last_batch_in_queue: + tmp_batch, tmp_result = result_queue.popleft() + tmp_batch.next_batch_sampling_info = ( + self.tp_worker.cur_sampling_info if batch else None + ) + self.process_batch_result(tmp_batch, tmp_result) + + if batch is None and ( + len(self.waiting_queue) + + len(self.disagg_decode_transfer_queue.queue) + + len(self.disagg_decode_prealloc_queue.queue) + == 0 + ): + self.self_check_during_idle() + + self.last_batch = batch + self.last_batch_in_queue = last_batch_in_queue + + def _prepare_idle_batch_and_run(self: Scheduler, batch, delay_process=False): + batch = self.prepare_mlp_sync_batch(batch) + result = None + if batch: + result = self.run_batch(batch) + if not delay_process: + self.process_batch_result(batch, result) + return batch, result + + def get_next_disagg_decode_batch_to_run( + self: Scheduler, + ) -> Optional[Tuple[ScheduleBatch, bool]]: + """Create fake completed prefill if possible and merge with running batch""" + # Merge the prefill batch into the running batch + last_batch = self.last_batch + if last_batch and last_batch.forward_mode.is_extend(): + # chunked prefill doesn't happen in decode instance. + assert self.chunked_req is None + # Filter finished batches. + last_batch.filter_batch() + if not last_batch.is_empty(): + if self.running_batch.is_empty(): + self.running_batch = last_batch + else: + # merge running_batch with prefill batch + self.running_batch.merge_batch(last_batch) + + new_prebuilt_batch = self.get_new_prebuilt_batch() + + ret: Optional[ScheduleBatch] = None + if new_prebuilt_batch: + ret = new_prebuilt_batch + else: + if self.running_batch.is_empty(): + ret = None + else: + self.running_batch = self.update_running_batch(self.running_batch) + ret = self.running_batch if not self.running_batch.is_empty() else None + + return ret + + def get_new_prebuilt_batch(self: Scheduler) -> Optional[ScheduleBatch]: + """Create a schedulebatch for fake completed prefill""" + if self.grammar_queue: + self.move_ready_grammar_requests() + + if len(self.waiting_queue) == 0: + return None + + curr_batch_size = self.running_batch.batch_size() + + batch_size = min(self.req_to_token_pool.size, self.max_running_requests) + + num_not_used_batch = batch_size - curr_batch_size + + # pop req from waiting queue + can_run_list: List[Req] = [] + waiting_queue: List[Req] = [] + + for i in range(len(self.waiting_queue)): + req = self.waiting_queue[i] + # we can only add at least `num_not_used_batch` new batch to the running queue + if i < num_not_used_batch: + can_run_list.append(req) + req.init_next_round_input(self.tree_cache) + else: + waiting_queue.append(req) + + self.waiting_queue = waiting_queue + if len(can_run_list) == 0: + return None + + # construct a schedule batch with those requests and mark as decode + new_batch = ScheduleBatch.init_new( + can_run_list, + self.req_to_token_pool, + self.token_to_kv_pool_allocator, + self.tree_cache, + self.model_config, + self.enable_overlap, + self.spec_algorithm, + ) + + # construct fake completed prefill + new_batch.prepare_for_prebuilt_extend() + new_batch.process_prebuilt_extend(self.server_args, self.model_config) + + return new_batch + + def process_decode_queue(self: Scheduler): + # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps + resumed_reqs = self.disagg_decode_prealloc_queue.resume_retracted_reqs() + self.waiting_queue.extend(resumed_reqs) + if len(self.disagg_decode_prealloc_queue.retracted_queue) > 0: + # if there are still retracted requests, we do not allocate new requests + return + + req_conns = self.disagg_decode_prealloc_queue.pop_preallocated() + self.disagg_decode_transfer_queue.extend(req_conns) + alloc_reqs = ( + self.disagg_decode_transfer_queue.pop_transferred() + ) # the requests which kv has arrived + self.waiting_queue.extend(alloc_reqs) diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py new file mode 100644 index 000000000..c1cb17c04 --- /dev/null +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING + +import torch + +from sglang.srt.disaggregation.utils import prepare_abort +from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode, ForwardMode +from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.managers.schedule_batch import ScheduleBatch + from sglang.srt.server_args import ServerArgs + + +class ScheduleBatchDisaggregationDecodeMixin: + + def prepare_for_prebuilt_extend(self: ScheduleBatch): + """ + Prepare a prebuilt extend by populate metadata + Adapted from .prepare_for_extend(). + """ + + self.forward_mode = ForwardMode.EXTEND + reqs = self.reqs + input_ids = [r.fill_ids[len(r.prefix_indices) :] for r in reqs] + extend_num_tokens = sum(len(ids) for ids in input_ids) + seq_lens = [] + pre_lens = [] + req_pool_indices = [] + + # Pre-calculate total size + total_size = sum(req.extend_input_len for req in reqs) + out_cache_loc = torch.empty(total_size, dtype=torch.int64, device=self.device) + + # Fill the tensor in one pass + offset = 0 + for i, req in enumerate(reqs): + req_pool_indices.append(req.req_pool_idx) + + chunk = self.req_to_token_pool.req_to_token[req.req_pool_idx][ + : req.extend_input_len + ] + assert ( + offset + req.extend_input_len <= total_size + ), f"Exceeds total size: offset={offset}, req.extend_input_len={req.extend_input_len}, total_size={total_size}" + out_cache_loc[offset : offset + req.extend_input_len] = chunk + offset += req.extend_input_len + + pre_len = len(req.prefix_indices) + seq_len = len(req.origin_input_ids) + max(0, len(req.output_ids) - 1) + seq_lens.append(seq_len) + if len(req.output_ids) == 0: + assert ( + seq_len - pre_len == req.extend_input_len + ), f"seq_len={seq_len}, pre_len={pre_len}, req.extend_input_len={req.extend_input_len}" + + req.cached_tokens += pre_len - req.already_computed + req.already_computed = seq_len + req.is_retracted = False + pre_lens.append(pre_len) + req.extend_logprob_start_len = 0 + + extend_input_logprob_token_ids = None + + # Set fields + self.input_ids = torch.tensor( + sum(input_ids, []), dtype=torch.int32, device=self.device + ) + self.req_pool_indices = torch.tensor( + req_pool_indices, dtype=torch.int64, device=self.device + ) + self.seq_lens = torch.tensor(seq_lens, dtype=torch.int64, device=self.device) + self.orig_seq_lens = torch.tensor( + seq_lens, dtype=torch.int32, device=self.device + ) + self.out_cache_loc = out_cache_loc + self.seq_lens_sum = sum(seq_lens) + + if self.return_logprob: + self.top_logprobs_nums = [r.top_logprobs_num for r in reqs] + self.token_ids_logprobs = [r.token_ids_logprob for r in reqs] + + self.extend_num_tokens = extend_num_tokens + self.prefix_lens = [len(r.prefix_indices) for r in reqs] + self.extend_lens = [r.extend_input_len for r in reqs] + self.extend_logprob_start_lens = [r.extend_logprob_start_len for r in reqs] + self.extend_input_logprob_token_ids = extend_input_logprob_token_ids + self.multimodal_inputs = [r.multimodal_inputs for r in reqs] + + # Build sampling info + self.sampling_info = SamplingBatchInfo.from_schedule_batch( + self, + self.model_config.vocab_size, + ) + + def process_prebuilt_extend( + self: ScheduleBatch, server_args: ServerArgs, model_config: ModelConfig + ): + """Assign the buffered last input id to schedule batch""" + self.output_ids = [] + for req in self.reqs: + self.output_ids.append(req.output_ids[-1]) + self.tree_cache.cache_unfinished_req(req) + if req.grammar is not None: + # FIXME: this try-except block is for handling unexpected xgrammar issue. + try: + req.grammar.accept_token(req.output_ids[-1]) + except ValueError as e: + # Grammar accept_token can raise ValueError if the token is not in the grammar. + # This can happen if the grammar is not set correctly or the token is invalid. + error_message = f"Grammar accept_token failed for req {req.rid} with token {req.output_ids[-1]}: {e}" + self.tree_cache.cache_finished_req(req) + prepare_abort( + req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR + ) + req.grammar.finished = req.finished() + self.output_ids = torch.tensor(self.output_ids, device=self.device) + + # Simulate the eagle run. We add mock data to hidden states for the + # ease of implementation now meaning the first token will have acc rate + # of 0. + if not self.spec_algorithm.is_none(): + + b = len(self.reqs) + topk_p = torch.arange( + b * server_args.speculative_eagle_topk, + 0, + -1, + device=self.device, + dtype=torch.float32, + ) + topk_p = topk_p.reshape(b, server_args.speculative_eagle_topk) + topk_p /= b * server_args.speculative_eagle_topk + topk_index = torch.arange( + b * server_args.speculative_eagle_topk, device=self.device + ) + topk_index = topk_index.reshape(b, server_args.speculative_eagle_topk) + + hidden_states_list = [req.hidden_states_tensor for req in self.reqs] + hidden_states = torch.stack(hidden_states_list, dim=0).to(self.device) + + # local import to avoid circular import + from sglang.srt.speculative.eagle_utils import EagleDraftInput + + spec_info = EagleDraftInput( + topk_p=topk_p, + topk_index=topk_index, + hidden_states=hidden_states, + verified_id=self.output_ids, + ) + spec_info.prepare_for_extend(self) + spec_info.capture_hidden_mode = CaptureHiddenMode.LAST + self.spec_info = spec_info diff --git a/python/sglang/srt/disaggregation/fake/__init__.py b/python/sglang/srt/disaggregation/fake/__init__.py new file mode 100644 index 000000000..d7cdb4b27 --- /dev/null +++ b/python/sglang/srt/disaggregation/fake/__init__.py @@ -0,0 +1 @@ +from sglang.srt.disaggregation.fake.conn import FakeKVReceiver, FakeKVSender diff --git a/python/sglang/srt/disaggregation/fake/conn.py b/python/sglang/srt/disaggregation/fake/conn.py new file mode 100644 index 000000000..120633824 --- /dev/null +++ b/python/sglang/srt/disaggregation/fake/conn.py @@ -0,0 +1,85 @@ +import logging +from typing import List, Optional + +import numpy as np +import numpy.typing as npt + +from sglang.srt.disaggregation.base.conn import ( + BaseKVManager, + BaseKVReceiver, + BaseKVSender, + KVPoll, +) + +logger = logging.getLogger(__name__) + + +# For warmup reqs, we don't kv transfer, we use the fake sender and receiver +class FakeKVSender(BaseKVSender): + def __init__( + self, + mgr: BaseKVManager, + bootstrap_addr: str, + bootstrap_room: int, + dest_tp_ranks: List[int], + pp_rank: int, + ): + self.has_sent = False + + def poll(self) -> KVPoll: + if self.has_sent is False: + # Assume handshake completed instantly + return KVPoll.WaitingForInput + else: + # Assume transfer completed instantly + logger.debug("FakeKVSender poll success") + return KVPoll.Success + + def init( + self, + kv_indices: list[int], + aux_index: Optional[int] = None, + ): + logger.debug( + f"FakeKVSender init with kv_indices: {kv_indices}, aux_index: {aux_index}" + ) + pass + + def send( + self, + kv_indices: npt.NDArray[np.int32], + ): + self.has_sent = True + logger.debug(f"FakeKVSender send with kv_indices: {kv_indices}") + + def failure_exception(self): + raise Exception("Fake KVSender Exception") + + +class FakeKVReceiver(BaseKVReceiver): + def __init__( + self, + mgr: BaseKVManager, + bootstrap_addr: str, + bootstrap_room: Optional[int] = None, + prefill_dp_rank: Optional[int] = None, + ): + self.has_init = False + + def poll(self) -> KVPoll: + if self.has_init is False: + # Assume handshake completed instantly + return KVPoll.WaitingForInput + else: + # Assume transfer completed instantly + logger.debug("FakeKVReceiver poll success") + return KVPoll.Success + + def init(self, kv_indices: list[int], aux_index: Optional[int] = None): + self.has_init = True + logger.debug( + f"FakeKVReceiver init with kv_indices: {kv_indices}, aux_index: {aux_index}" + ) + + def failure_exception(self): + raise Exception("Fake KVReceiver Exception") diff --git a/python/sglang/srt/disaggregation/kv_events.py b/python/sglang/srt/disaggregation/kv_events.py new file mode 100644 index 000000000..f0a3a4357 --- /dev/null +++ b/python/sglang/srt/disaggregation/kv_events.py @@ -0,0 +1,412 @@ +""" +Copyright 2025 SGLang Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +""" +KV caching events +""" + +import atexit +import logging +import queue +import threading +import time +from abc import ABC, abstractmethod +from collections import deque +from itertools import count +from queue import Queue +from typing import Any, Callable, Optional, Union + +import msgspec +import zmq +from pydantic import BaseModel + +logger = logging.getLogger(__name__) + + +class EventBatch( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True, # type: ignore[call-arg] + gc=False, # type: ignore[call-arg] +): + ts: float + events: list[Any] + attn_dp_rank: Optional[int] = None + + +class KVCacheEvent( + msgspec.Struct, + array_like=True, # type: ignore[call-arg] + omit_defaults=True, # type: ignore[call-arg] + gc=False, # type: ignore[call-arg] + tag=True, +): + """Base class for all KV cache-related events""" + + +class BlockStored(KVCacheEvent): + block_hashes: list[int] + parent_block_hash: Optional[int] + token_ids: list[int] + block_size: int + lora_id: Optional[int] + + +class BlockRemoved(KVCacheEvent): + block_hashes: list[int] + + +class AllBlocksCleared(KVCacheEvent): + pass + + +class KVEventBatch(EventBatch): + events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]] + + +class EventPublisher(ABC): + """ + Lightweight publisher for EventBatch batches with + support for DP attention. + + In DP attention - each rank has its own Scheduler and + KV cache instance in order to avoid duplicate events + and ensure proper event attribution. In our implementation + + - Each DP rank has its own EventPublisher + - Publishers annotate events with the dp rank + - This allows consumers to distinguish events from different DP ranks + """ + + def __init__(self, attn_dp_rank: int = 0): + self._attn_dp_rank = attn_dp_rank + + @abstractmethod + def publish(self, events: EventBatch) -> None: + """Emit events in order. + + Implementations should guarantee at-least-once delivery and + monotonic ordering (e.g., via sequence numbers). + """ + + @abstractmethod + def shutdown(self) -> None: + """Shutdown the publisher.""" + + +class NullEventPublisher(EventPublisher): + """No-op implementation (default when disabled).""" + + def publish(self, events) -> None: + return + + def shutdown(self) -> None: + return + + +class ZmqEventPublisher(EventPublisher): + """Reliable PUB/ROUTER publisher with an in-memory replay buffer. + + Spawns a separate thread to handle publishing from a queue. + + Parameters + ---------- + endpoint: + PUB address. Use ``tcp://*:5557`` to bind or ``tcp://host:5557`` to + connect. + replay_endpoint: + Optional ROUTER address for replay requests. When given, subscribers can + request missed batches by sending the starting sequence number as an + 8-byte big-endian integer. + buffer_steps: + Number of past batches to keep for replay. + hwm: + ZeroMQ high-water-mark for PUB socket. + max_queue_size: + Maximum number of events to buffer in memory. + topic: + Topic to publish events to. + """ + + SHUTDOWN_TIMEOUT: float = 1.0 + END_SEQ = (-1).to_bytes(8, "big", signed=True) + + def __init__( + self, + attn_dp_rank: int, + endpoint: str = "tcp://*:5557", + replay_endpoint: Optional[str] = None, + buffer_steps: int = 10_000, + hwm: int = 100_000, + max_queue_size: int = 100_000, + topic: str = "", + ) -> None: + # Storage + super().__init__(attn_dp_rank) + self._event_queue = Queue[Optional[EventBatch]](maxsize=max_queue_size) + self._buffer = deque[tuple[int, bytes]](maxlen=buffer_steps) + + # ZMQ sockets + self._ctx = zmq.Context.instance() + self._pub: Optional[zmq.Socket] = None + self._replay: Optional[zmq.Socket] = None + self._dp_rank = attn_dp_rank + self._endpoint = self.offset_endpoint_port(endpoint, self._dp_rank) + self._replay_endpoint = self.offset_endpoint_port( + replay_endpoint, self._dp_rank + ) + self._hwm = hwm + self._socket_setup() + + # Payload + self._seq_gen = count() + self._topic_bytes = topic.encode("utf-8") + + # Thread + self._running = True + logger.info("Starting ZMQ publisher thread") + + self._thread = threading.Thread( + target=self._publisher_thread, daemon=True, name="zmq-publisher" + ) + self._thread.start() + + atexit.register(self.shutdown) + + def publish(self, events: EventBatch) -> None: + if not self._running: + raise RuntimeError("Publisher is closed") + if events.attn_dp_rank is None: + events.attn_dp_rank = self._dp_rank + self._event_queue.put(events) + + def shutdown(self) -> None: + """Stop the publisher thread and clean up resources.""" + self._running = False + self._event_queue.put_nowait(None) + + start = time.time() + pending_items = True + while pending_items and (time.time() - start < self.SHUTDOWN_TIMEOUT): + pending_items = not self._event_queue.empty() + if pending_items: + time.sleep(0.1) + + if pending_items: + logger.warning( + "Warning: Queue still has %s items after %s seconds timeout", + self._event_queue.qsize(), + self.SHUTDOWN_TIMEOUT, + ) + + if self._thread.is_alive(): + self._thread.join(timeout=self.SHUTDOWN_TIMEOUT) + + # Clean up ZMQ resources + try: + if self._pub is not None: + self._pub.close(linger=0) + if self._replay is not None: + self._replay.close(linger=0) + finally: + pass # Do not terminate context; other sockets may use it + + def _socket_setup(self) -> None: + """Initialize sockets + https://pyzmq.readthedocs.io/en/v19.0.0/morethanbindings.html#thread-safety + """ + if self._pub is None: + self._pub = self._ctx.socket(zmq.PUB) + self._pub.set_hwm(self._hwm) + # Heuristic: bind if wildcard / * present, else connect. + # bind stable, connect volatile convention + if ( + "*" in self._endpoint + or "::" in self._endpoint + or self._endpoint.startswith("ipc://") + or self._endpoint.startswith("inproc://") + ): + self._pub.bind(self._endpoint) + else: + self._pub.connect(self._endpoint) + + # Set up replay socket: use ROUTER + # 1) handles multiple REQ clients (identities) + # 2) lets us send back one request → many replies (streamed events) + # 3) works in our non‑blocking poll loop alongside PUB + if self._replay_endpoint is not None: + self._replay = self._ctx.socket(zmq.ROUTER) + self._replay.bind(self._replay_endpoint) + + def _publisher_thread(self) -> None: + """Background thread that processes the event queue.""" + self._pack = msgspec.msgpack.Encoder() + + assert self._pub is not None # narrows type for mypy + + while self._running or self._event_queue.qsize() > 0: + # --- replay (non-critical) --------------------------------- + if self._replay is not None and self._replay.poll(0): + try: + self._service_replay() + except Exception as e: + logger.exception("Error in replay: %s", e) + + # --- main queue (critical) --------------------------------- + try: + event = self._event_queue.get(timeout=0.1) + if event is None: + break # Sentinel received, exit thread + except queue.Empty: + continue + + try: + seq = next(self._seq_gen) + + payload = self._pack.encode(event) + seq_bytes = seq.to_bytes(8, "big") + self._pub.send_multipart((self._topic_bytes, seq_bytes, payload)) + + self._buffer.append((seq, payload)) + self._event_queue.task_done() + + except Exception as e: + # Publishing failed; back-off a bit to avoid a tight error loop + logger.exception("Error in publisher thread: %s", e) + time.sleep(0.1) + + def _service_replay(self) -> None: + """If a replay request is waiting, send buffered batches.""" + assert self._replay is not None # narrows type for mypy + + frame = self._replay.recv_multipart() + if len(frame) != 3: + logger.warning("Invalid replay request: %s", frame) + return + client_id, _, start_seq_bytes = frame + start_seq = int.from_bytes(start_seq_bytes, "big") + + for seq, buf in self._buffer: + if seq >= start_seq: + # [identity, empty_delim, seq_bytes, payload] + # (identity, empty_delim) are stripped off by the router + # receiving payload is (seq_bytes, payload) + self._replay.send_multipart( + (client_id, b"", seq.to_bytes(8, "big"), buf) + ) + # Send end of sequence marker + # receiving payload is (-1, b""") + self._replay.send_multipart((client_id, b"", self.END_SEQ, b"")) + + @staticmethod + def offset_endpoint_port( + endpoint: Optional[str], data_parallel_rank: int + ) -> Optional[str]: + """Helper function to offset the port in an endpoint by + the data parallel rank. + + Args: + endpoint: The endpoint string + (e.g., "tcp://*:5557" or "inproc://cache") + data_parallel_rank: The data parallel rank to offset by + + Returns: + The endpoint with the port offset by data_parallel_rank + or suffix appended + """ + # Do nothing if input is None or data_parallel_rank is 0 + if not endpoint or data_parallel_rank == 0: + return endpoint + + if "inproc" in endpoint: + return f"{endpoint}_dp{data_parallel_rank}" + if "tcp" in endpoint: + if endpoint and ":" in endpoint: + # Get everything after the last colon (the port) + last_colon_idx = endpoint.rfind(":") + base_addr = endpoint[:last_colon_idx] + base_port = int(endpoint[last_colon_idx + 1 :]) + new_port = base_port + data_parallel_rank + return f"{base_addr}:{new_port}" + return endpoint + raise ValueError("Invalid endpoint: must contain 'inproc' or 'tcp'") + + +class KVEventsConfig(BaseModel): + """Configuration for KV event publishing.""" + + publisher: str = "null" + """The publisher to use for publishing kv events. Can be "null", "zmq". + """ + + endpoint: str = "tcp://*:5557" + """The zmq endpoint to use for publishing kv events. + """ + + replay_endpoint: Optional[str] = None + """The zmq endpoint to use for replaying kv events. + """ + + buffer_steps: int = 10_000 + """The number of steps to cache for replay endpoint. Will only save + events from the last N steps for the replay endpoint. + """ + + hwm: int = 100_000 + """The zmq high water mark for the event publisher. After queueing N events, + events will start dropping if the consumer is not keeping up. + """ + + max_queue_size: int = 100_000 + """The maximum number of events to queue while waiting for publishing. + """ + + topic: str = "" + """The topic to use for the event publisher. Consumers can subscribe to + this topic to receive events. + """ + + @classmethod + def from_cli(cls, cli_value: str) -> "KVEventsConfig": + """Parse the CLI value for the event publisher config.""" + return KVEventsConfig.model_validate_json(cli_value) + + +class EventPublisherFactory: + _registry: dict[str, Callable[..., EventPublisher]] = { + "null": NullEventPublisher, + "zmq": ZmqEventPublisher, + } + + @classmethod + def register_publisher(cls, name: str, ctor: Callable[..., EventPublisher]) -> None: + if name in cls._registry: + raise KeyError(f"publisher '{name}' already registered") + cls._registry[name] = ctor + + @classmethod + def create(cls, config: Optional[str], attn_dp_rank: int = 0) -> EventPublisher: + """Create publisher from a config mapping.""" + if not config: + return NullEventPublisher() + config = KVEventsConfig.from_cli(config) + config_dict = config.model_dump() + + kind = config_dict.pop("publisher", "null") + try: + constructor = cls._registry[kind] + except KeyError as exc: + raise ValueError(f"Unknown event publisher '{kind}'") from exc + return constructor(attn_dp_rank=attn_dp_rank, **config_dict) diff --git a/python/sglang/srt/disaggregation/mini_lb.py b/python/sglang/srt/disaggregation/mini_lb.py new file mode 100644 index 000000000..5aaa2a70e --- /dev/null +++ b/python/sglang/srt/disaggregation/mini_lb.py @@ -0,0 +1,6 @@ +raise RuntimeError( + """The 'mini_lb' module has been relocated to the 'sglang_router' package. + We recommend installing 'sglang-router' with Rust support for optimal performance. + If you encounter issues building the router with Rust, set the environment variable + 'SGLANG_ROUTER_BUILD_NO_RUST=1' and add '--mini-lb' to the command line to use the Python version of 'mini_lb'.""" +) diff --git a/python/sglang/srt/disaggregation/mooncake/__init__.py b/python/sglang/srt/disaggregation/mooncake/__init__.py new file mode 100644 index 000000000..bea967e4e --- /dev/null +++ b/python/sglang/srt/disaggregation/mooncake/__init__.py @@ -0,0 +1,6 @@ +from sglang.srt.disaggregation.mooncake.conn import ( + MooncakeKVBootstrapServer, + MooncakeKVManager, + MooncakeKVReceiver, + MooncakeKVSender, +) diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py new file mode 100644 index 000000000..f69d29622 --- /dev/null +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -0,0 +1,1704 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +import ctypes +import dataclasses +import logging +import os +import queue +import socket +import struct +import threading +import time +from collections import defaultdict +from functools import cache +from typing import Dict, List, Optional, Tuple, Union + +import numpy as np +import numpy.typing as npt +import requests +import zmq +from aiohttp import web + +from sglang.srt.disaggregation.base.conn import ( + BaseKVBootstrapServer, + BaseKVManager, + BaseKVReceiver, + BaseKVSender, + KVArgs, + KVPoll, +) +from sglang.srt.disaggregation.common.utils import ( + FastQueue, + group_concurrent_contiguous, +) +from sglang.srt.disaggregation.mooncake.transfer_engine import MooncakeTransferEngine +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.distributed import get_pp_group +from sglang.srt.layers.dp_attention import ( + get_attention_dp_rank, + get_attention_dp_size, + get_attention_tp_rank, + get_attention_tp_size, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ( + format_tcp_address, + get_bool_env_var, + get_free_port, + get_int_env_var, + get_ip, + get_local_ip_auto, + is_valid_ipv6_address, + maybe_wrap_ipv6_address, +) + +logger = logging.getLogger(__name__) + + +class KVTransferError(Exception): + def __init__(self, bootstrap_room: int, failure_reason: str): + super().__init__(failure_reason) + self.bootstrap_room = bootstrap_room + self.failure_reason = failure_reason + + def __str__(self): + return f"KVTransferError(bootstrap_room={self.bootstrap_room}): {self.failure_reason}" + + +# prefill +@dataclasses.dataclass +class TransferKVChunk: + room: int + prefill_kv_indices: npt.NDArray[np.int32] + index_slice: slice + is_last: bool + prefill_aux_index: Optional[int] + + +# decode +@dataclasses.dataclass +class TransferInfo: + room: int + endpoint: str + dst_port: int + mooncake_session_id: str + dst_kv_indices: npt.NDArray[np.int32] + dst_aux_index: int + required_dst_info_num: int + is_dummy: bool + + @classmethod + def from_zmq(cls, msg: List[bytes]): + if msg[4] == b"" and msg[5] == b"": + is_dummy = True + dst_kv_indices = np.array([], dtype=np.int32) + dst_aux_index = None + else: + dst_kv_indices = np.frombuffer(msg[4], dtype=np.int32) + dst_aux_index = int(msg[5].decode("ascii")) + is_dummy = False + return cls( + room=int(msg[0].decode("ascii")), + endpoint=msg[1].decode("ascii"), + dst_port=int(msg[2].decode("ascii")), + mooncake_session_id=msg[3].decode("ascii"), + dst_kv_indices=dst_kv_indices, + dst_aux_index=dst_aux_index, + required_dst_info_num=int(msg[6].decode("ascii")), + is_dummy=is_dummy, + ) + + +# decode +@dataclasses.dataclass +class KVArgsRegisterInfo: + room: str + endpoint: str + dst_port: int + mooncake_session_id: str + dst_kv_ptrs: list[int] + dst_aux_ptrs: list[int] + dst_tp_rank: int + dst_attn_tp_size: int + dst_kv_item_len: int + + @classmethod + def from_zmq(cls, msg: List[bytes]): + return cls( + room=str(msg[0].decode("ascii")), + endpoint=msg[1].decode("ascii"), + dst_port=int(msg[2].decode("ascii")), + mooncake_session_id=msg[3].decode("ascii"), + dst_kv_ptrs=list(struct.unpack(f"{len(msg[4])//8}Q", msg[4])), + dst_aux_ptrs=list(struct.unpack(f"{len(msg[5])//8}Q", msg[5])), + dst_tp_rank=int(msg[6].decode("ascii")), + dst_attn_tp_size=int(msg[7].decode("ascii")), + dst_kv_item_len=int(msg[8].decode("ascii")), + ) + + +class AuxDataCodec: + """Handles serialization and deserialization of auxiliary data buffers""" + + @staticmethod + def serialize_data_from_buffer(src_addr, data_length): + """Serialize data from memory buffer to bytes""" + buffer = (ctypes.c_byte * data_length).from_address(src_addr) + return bytes(buffer) + + @staticmethod + def deserialize_data_to_buffer(kv_args, buffer_index, aux_index, data): + """Deserialize bytes into target memory buffer""" + dst_aux_ptr = kv_args.aux_data_ptrs[buffer_index] + item_len = kv_args.aux_item_lens[buffer_index] + dst_addr = dst_aux_ptr + item_len * aux_index + buffer = (ctypes.c_byte * len(data)).from_address(dst_addr) + buffer[:] = data + return + + +class MooncakeKVManager(BaseKVManager): + AUX_DATA_HEADER = b"AUX_DATA" + + def __init__( + self, + args: KVArgs, + disaggregation_mode: DisaggregationMode, + server_args: ServerArgs, + is_mla_backend: Optional[bool] = False, + ): + self.kv_args = args + self.local_ip = get_local_ip_auto() + self.is_mla_backend = is_mla_backend + self.disaggregation_mode = disaggregation_mode + self.init_engine() + # for p/d multi node infer + self.bootstrap_host = server_args.host + self.bootstrap_port = server_args.disaggregation_bootstrap_port + self.dist_init_addr = server_args.dist_init_addr + self.attn_tp_size = get_attention_tp_size() + self.attn_tp_rank = get_attention_tp_rank() + self.attn_dp_size = get_attention_dp_size() + self.attn_dp_rank = get_attention_dp_rank() + self.system_dp_size = ( + 1 if server_args.enable_dp_attention else server_args.dp_size + ) + self.system_dp_rank = ( + self.kv_args.system_dp_rank if self.kv_args.system_dp_rank else 0 + ) + self.pp_size = server_args.pp_size + self.pp_rank = self.kv_args.pp_rank + self.request_status: Dict[int, KVPoll] = {} + self.rank_port = None + self.server_socket = zmq.Context().socket(zmq.PULL) + if is_valid_ipv6_address(self.local_ip): + self.server_socket.setsockopt(zmq.IPV6, 1) + + self.register_buffer_to_engine() + if self.disaggregation_mode == DisaggregationMode.PREFILL: + self.transfer_infos: Dict[int, Dict[str, TransferInfo]] = {} + self.decode_kv_args_table: Dict[str, KVArgsRegisterInfo] = {} + self.start_prefill_thread() + self._register_to_bootstrap() + self.session_failures = defaultdict(int) + self.failed_sessions = set() + self.session_lock = threading.Lock() + self.pp_group = get_pp_group() + # Determine the number of threads to use for kv sender + cpu_count = os.cpu_count() + transfer_thread_pool_size = get_int_env_var( + "SGLANG_DISAGGREGATION_THREAD_POOL_SIZE", + min(max(4, int(0.75 * cpu_count) // 8), 12), + ) + transfer_queue_size = get_int_env_var("SGLANG_DISAGGREGATION_QUEUE_SIZE", 4) + self.transfer_queues: List[FastQueue] = [ + FastQueue() for _ in range(transfer_queue_size) + ] + assert transfer_thread_pool_size >= transfer_queue_size, ( + f"The environment variable SGLANG_DISAGGREGATION_THREAD_POOL_SIZE={transfer_thread_pool_size} must be " + f"greater than or equal to SGLANG_DISAGGREGATION_QUEUE_SIZE={transfer_queue_size}." + ) + self.executors = [ + concurrent.futures.ThreadPoolExecutor( + transfer_thread_pool_size // transfer_queue_size + ) + for _ in range(transfer_queue_size) + ] + for queue, executor in zip(self.transfer_queues, self.executors): + threading.Thread( + target=self.transfer_worker, args=(queue, executor), daemon=True + ).start() + # If a timeout happens on the prefill side, it means prefill instances + # fail to receive the KV indices from the decode instance of this request. + # These timeout requests should be aborted to release the tree cache. + self.bootstrap_timeout = get_int_env_var( + "SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT", 300 + ) + + self.enable_custom_mem_pool = get_bool_env_var( + "SGLANG_MOONCAKE_CUSTOM_MEM_POOL", "false" + ) + elif self.disaggregation_mode == DisaggregationMode.DECODE: + self.heartbeat_failures = {} + self.session_pool = defaultdict(requests.Session) + self.session_pool_lock = threading.Lock() + self.addr_to_rooms_tracker = defaultdict(set) + self.connection_lock = threading.Lock() + self.required_prefill_response_num_table: Dict[int, int] = {} + self.prefill_response_tracker: Dict[int, Set[int]] = defaultdict(set) + # Heartbeat interval should be at least 2 seconds + self.heartbeat_interval = max( + float(os.getenv("SGLANG_DISAGGREGATION_HEARTBEAT_INTERVAL", 5.0)), 2.0 + ) + # Heartbeat failure should be at least 1 + self.max_failures = max( + get_int_env_var("SGLANG_DISAGGREGATION_HEARTBEAT_MAX_FAILURE", 2), 1 + ) + self.start_decode_thread() + self.connection_pool: Dict[str, Dict[str, Union[str, int]]] = {} + self.prefill_attn_tp_size_table: Dict[str, int] = {} + self.prefill_dp_size_table: Dict[str, int] = {} + self.prefill_pp_size_table: Dict[str, int] = {} + # If a timeout happens on the decode side, it means decode instances + # fail to receive the KV Cache transfer done signal after bootstrapping. + # These timeout requests should be aborted to release the tree cache. + self.waiting_timeout = get_int_env_var( + "SGLANG_DISAGGREGATION_WAITING_TIMEOUT", 300 + ) + else: + raise ValueError( + f"Unsupported DisaggregationMode: {self.disaggregation_mode}" + ) + + self.failure_records: Dict[int, str] = {} + self.failure_lock = threading.Lock() + + def init_engine(self): + self.engine = MooncakeTransferEngine( + hostname=self.local_ip, + gpu_id=self.kv_args.gpu_id, + ib_device=self.kv_args.ib_device, + ) + + def register_buffer_to_engine(self): + # Batch register KV data buffers + if self.kv_args.kv_data_ptrs and self.kv_args.kv_data_lens: + self.engine.batch_register( + self.kv_args.kv_data_ptrs, self.kv_args.kv_data_lens + ) + + # Batch register auxiliary data buffers + if self.kv_args.aux_data_ptrs and self.kv_args.aux_data_lens: + self.engine.batch_register( + self.kv_args.aux_data_ptrs, self.kv_args.aux_data_lens + ) + + @cache + def _connect(self, endpoint: str, is_ipv6: bool = False): + socket = zmq.Context().socket(zmq.PUSH) + if is_ipv6: + socket.setsockopt(zmq.IPV6, 1) + socket.connect(endpoint) + return socket + + def _transfer_data(self, mooncake_session_id, transfer_blocks): + if not transfer_blocks: + return 0 + + src_addrs, dst_addrs, lengths = zip(*transfer_blocks) + return self.engine.batch_transfer_sync( + mooncake_session_id, list(src_addrs), list(dst_addrs), list(lengths) + ) + + def send_kvcache( + self, + mooncake_session_id: str, + prefill_kv_indices: npt.NDArray[np.int32], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int32], + executor: concurrent.futures.ThreadPoolExecutor, + ): + # Group by indices + prefill_kv_blocks, dst_kv_blocks = group_concurrent_contiguous( + prefill_kv_indices, dst_kv_indices + ) + + layers_params = None + + # pp is not supported on the decode side yet + start_layer = self.kv_args.prefill_start_layer + end_layer = start_layer + len(self.kv_args.kv_data_ptrs) + if self.is_mla_backend: + src_kv_ptrs = self.kv_args.kv_data_ptrs + layers_per_pp_stage = len(src_kv_ptrs) + dst_kv_ptrs = dst_kv_ptrs[start_layer:end_layer] + kv_item_len = self.kv_args.kv_item_lens[0] + layers_params = [ + ( + src_kv_ptrs[layer_id], + dst_kv_ptrs[layer_id], + kv_item_len, + ) + for layer_id in range(layers_per_pp_stage) + ] + else: + num_kv_layers = len(self.kv_args.kv_data_ptrs) // 2 + dst_num_total_layers = num_kv_layers * self.pp_size + src_k_ptrs = self.kv_args.kv_data_ptrs[:num_kv_layers] + src_v_ptrs = self.kv_args.kv_data_ptrs[num_kv_layers:] + layers_per_pp_stage = len(src_k_ptrs) + dst_k_ptrs = dst_kv_ptrs[start_layer:end_layer] + dst_v_ptrs = dst_kv_ptrs[ + dst_num_total_layers + start_layer : dst_num_total_layers + end_layer + ] + kv_item_len = self.kv_args.kv_item_lens[0] + layers_params = [ + ( + src_k_ptrs[layer_id], + dst_k_ptrs[layer_id], + kv_item_len, + ) + for layer_id in range(layers_per_pp_stage) + ] + [ + ( + src_v_ptrs[layer_id], + dst_v_ptrs[layer_id], + kv_item_len, + ) + for layer_id in range(layers_per_pp_stage) + ] + assert layers_params is not None + + def set_transfer_blocks( + src_ptr: int, dst_ptr: int, item_len: int + ) -> List[Tuple[int, int, int]]: + transfer_blocks = [] + for prefill_index, decode_index in zip(prefill_kv_blocks, dst_kv_blocks): + src_addr = src_ptr + int(prefill_index[0]) * item_len + dst_addr = dst_ptr + int(decode_index[0]) * item_len + length = item_len * len(prefill_index) + transfer_blocks.append((src_addr, dst_addr, length)) + return transfer_blocks + + # Worker function for processing a single layer + def process_layer(src_ptr: int, dst_ptr: int, item_len: int) -> int: + transfer_blocks = set_transfer_blocks(src_ptr, dst_ptr, item_len) + return self._transfer_data(mooncake_session_id, transfer_blocks) + + # Worker function for processing all layers in a batch + def process_layers(layers_params: List[Tuple[int, int, int]]) -> int: + transfer_blocks = [] + for src_ptr, dst_ptr, item_len in layers_params: + transfer_blocks.extend(set_transfer_blocks(src_ptr, dst_ptr, item_len)) + return self._transfer_data(mooncake_session_id, transfer_blocks) + + if self.enable_custom_mem_pool: + futures = [ + executor.submit( + process_layer, + src_ptr, + dst_ptr, + item_len, + ) + for (src_ptr, dst_ptr, item_len) in layers_params + ] + for future in concurrent.futures.as_completed(futures): + status = future.result() + if status != 0: + for f in futures: + f.cancel() + return status + else: + # Combining all layers' params in one batch transfer is more efficient + # compared to using multiple threads + return process_layers(layers_params) + + return 0 + + def send_kvcache_slice( + self, + mooncake_session_id: str, + prefill_kv_indices: npt.NDArray[np.int64], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int64], + dst_tp_rank: int, + dst_attn_tp_size: int, + dst_kv_item_len: int, + executor: concurrent.futures.ThreadPoolExecutor, + ): + """ + Sends KV cache slices from this Prefill rank to a target Decode rank, + supporting generic M-to-N TP size configurations. + + NOTE: This implementation calls the transfer engine for each token slot within + each page to ensure correctness for any page_size and head-slicing configuration. + This may introduce performance overhead (increased TTFT) for long sequences. + """ + # Extract configuration + local_tp_rank_in_group = self.kv_args.engine_rank % self.attn_tp_size + src_kv_item_len = self.kv_args.kv_item_lens[0] + dst_tp_rank_in_group = dst_tp_rank % dst_attn_tp_size + num_kv_heads = self.kv_args.kv_head_num + num_layers = len(self.kv_args.kv_data_ptrs) + page_size = self.kv_args.page_size + + # Calculate head distribution + src_heads_per_rank = num_kv_heads + dst_heads_per_rank = num_kv_heads * self.attn_tp_size // dst_attn_tp_size + bytes_per_head_slice_to_send = ( + dst_kv_item_len // page_size // dst_heads_per_rank + ) + + # Determine slicing parameters based on TP configuration + if self.attn_tp_size > dst_attn_tp_size: + # Send KVCache from multiple prefill instances to 1 decode instance + src_head_start_offset = 0 + num_heads_to_send = src_heads_per_rank + dst_head_start_offset = local_tp_rank_in_group * src_heads_per_rank + else: + # Send KVCache from 1 prefill instance to multiple decode instances + src_head_start_offset = ( + dst_tp_rank_in_group * dst_heads_per_rank + ) % src_heads_per_rank + num_heads_to_send = dst_heads_per_rank + dst_head_start_offset = 0 + + # pp is not supported on the decode side yet + num_kv_layers = len(self.kv_args.kv_data_ptrs) // 2 + dst_num_total_layers = num_kv_layers * self.pp_size + src_k_ptrs = self.kv_args.kv_data_ptrs[:num_kv_layers] + src_v_ptrs = self.kv_args.kv_data_ptrs[num_kv_layers:] + layers_per_pp_stage = len(src_k_ptrs) + start_layer = self.pp_rank * layers_per_pp_stage + end_layer = start_layer + layers_per_pp_stage + dst_k_ptrs = dst_kv_ptrs[start_layer:end_layer] + dst_v_ptrs = dst_kv_ptrs[ + dst_num_total_layers + start_layer : dst_num_total_layers + end_layer + ] + + # Calculate precise byte offset and length for the sub-slice within the token + src_head_slice_offset = src_head_start_offset * bytes_per_head_slice_to_send + dst_head_slice_offset = dst_head_start_offset * bytes_per_head_slice_to_send + heads_bytes_per_token_to_send = num_heads_to_send * bytes_per_head_slice_to_send + + # Sanity check: The data sub-slice to be sent should fit into the dst buffer. + # This means heads_bytes_per_token_to_send <= (dst_kv_item_len // page_size) + if heads_bytes_per_token_to_send > (dst_kv_item_len // page_size): + logger.error( + f"[{mooncake_session_id}] slice size ({heads_bytes_per_token_to_send}) exceeds " + f"target token slot size ({dst_kv_item_len // page_size})" + ) + return -1 + + layers_params = [ + ( + src_k_ptrs[layer_id], + dst_k_ptrs[layer_id], + src_kv_item_len, + dst_kv_item_len, + src_head_slice_offset, + dst_head_slice_offset, + heads_bytes_per_token_to_send, + ) + for layer_id in range(layers_per_pp_stage) + ] + [ + ( + src_v_ptrs[layer_id], + dst_v_ptrs[layer_id], + src_kv_item_len, + dst_kv_item_len, + src_head_slice_offset, + dst_head_slice_offset, + heads_bytes_per_token_to_send, + ) + for layer_id in range(layers_per_pp_stage) + ] + + def process_layer_tp_aware(layer_params): + ( + src_ptr, + dst_ptr, + src_item_len, + dst_item_len, + src_head_slice_offset, + dst_head_slice_offset, + heads_bytes_per_token_to_send, + ) = layer_params + src_addr_list = [] + dst_addr_list = [] + length_list = [] + + # Calculate strides for a single token slot + bytes_per_token_on_prefill = src_item_len // page_size + bytes_per_token_on_decode = dst_item_len // page_size + + for i in range(len(prefill_kv_indices)): + prefill_page_idx = int(prefill_kv_indices[i]) + decode_page_idx = int(dst_kv_indices[i]) + + # Get the starting addresses for the current src and dst pages + src_page_start_addr = src_ptr + prefill_page_idx * src_item_len + dst_page_start_addr = dst_ptr + decode_page_idx * dst_item_len + + # Iterate through each valid token slot within the current page + for token_slot_in_page in range(page_size): + # Calculate the start address of the current token slot + src_token_slot_start_addr = ( + src_page_start_addr + + token_slot_in_page * bytes_per_token_on_prefill + ) + dst_token_slot_start_addr = ( + dst_page_start_addr + + token_slot_in_page * bytes_per_token_on_decode + ) + + # Calculate final src and dst addresses by applying head-slice offsets + src_slice_addr = src_token_slot_start_addr + src_head_slice_offset + dst_slice_addr = dst_token_slot_start_addr + dst_head_slice_offset + + src_addr_list.append(src_slice_addr) + dst_addr_list.append(dst_slice_addr) + length_list.append(heads_bytes_per_token_to_send) + + return self.engine.batch_transfer_sync( + mooncake_session_id, src_addr_list, dst_addr_list, length_list + ) + + futures = [ + executor.submit( + process_layer_tp_aware, + layer_params, + ) + for layer_params in layers_params + ] + + for future in concurrent.futures.as_completed(futures): + status = future.result() + if status != 0: + for f in futures: + f.cancel() + return status + + return 0 + + def send_aux( + self, + req: TransferInfo, + prefill_aux_index: int, + dst_aux_ptrs: list[int], + ): + # TODO(shangming): Fix me when nvlink_transport of Mooncake is bug-free + if self.enable_custom_mem_pool: + return self.send_aux_tcp(req, prefill_aux_index, dst_aux_ptrs) + + transfer_blocks = [] + prefill_aux_ptrs = self.kv_args.aux_data_ptrs + prefill_aux_item_lens = self.kv_args.aux_item_lens + + for i, dst_aux_ptr in enumerate(dst_aux_ptrs): + length = prefill_aux_item_lens[i] + src_addr = prefill_aux_ptrs[i] + length * prefill_aux_index + dst_addr = dst_aux_ptrs[i] + length * req.dst_aux_index + transfer_blocks.append((src_addr, dst_addr, length)) + + return self._transfer_data(req.mooncake_session_id, transfer_blocks) + + def send_aux_tcp( + self, + req: TransferInfo, + prefill_aux_index: int, + dst_aux_ptrs: list[int], + ): + prefill_aux_ptrs = self.kv_args.aux_data_ptrs + prefill_aux_item_lens = self.kv_args.aux_item_lens + + for i in range(len(prefill_aux_ptrs)): + length = prefill_aux_item_lens[i] + src_addr = prefill_aux_ptrs[i] + length * prefill_aux_index + data = AuxDataCodec.serialize_data_from_buffer(src_addr, length) + + self.send_aux_data_to_endpoint( + remote=req.endpoint, + dst_port=req.dst_port, + room=req.room, + buffer_index=i, + aux_index=req.dst_aux_index, + data=data, + ) + + return 0 + + def send_aux_data_to_endpoint( + self, + remote: str, + dst_port: int, + room: int, + buffer_index: int, + aux_index: int, + data: bytes, + ): + socket = self._connect( + format_tcp_address(remote, dst_port), is_ipv6=is_valid_ipv6_address(remote) + ) + + socket.send_multipart( + [ + MooncakeKVManager.AUX_DATA_HEADER, + str(room).encode("ascii"), + str(buffer_index).encode("ascii"), + str(aux_index).encode("ascii"), + struct.pack(">I", len(data)), + data, + ] + ) + + def sync_status_to_decode_endpoint( + self, remote: str, dst_port: int, room: int, status: int, prefill_rank: int + ): + self._connect( + format_tcp_address(remote, dst_port), is_ipv6=is_valid_ipv6_address(remote) + ).send_multipart( + [ + str(room).encode("ascii"), + str(status).encode("ascii"), + str(prefill_rank).encode("ascii"), + ] + ) + + def transfer_worker( + self, queue: FastQueue, executor: concurrent.futures.ThreadPoolExecutor + ): + while True: + try: + kv_chunk: TransferKVChunk = queue.get() + reqs_to_be_processed = ( + self.transfer_infos[kv_chunk.room].values() + if kv_chunk.room in self.transfer_infos + else [] + ) + polls = [] + dst_ranks_infos = [] + local_rank = self.attn_tp_rank * self.pp_size + self.pp_rank + for req in reqs_to_be_processed: + if not req.is_dummy: + # Early exit if the request has failed + with self.session_lock: + if req.mooncake_session_id in self.failed_sessions: + self.record_failure( + kv_chunk.room, + f"Decode instance could be dead, remote mooncake session {req.mooncake_session_id} is not alive", + ) + self.update_status(kv_chunk.room, KVPoll.Failed) + self.sync_status_to_decode_endpoint( + req.endpoint, + req.dst_port, + req.room, + KVPoll.Failed, + local_rank, + ) + break + + chunked_dst_kv_indice = req.dst_kv_indices[kv_chunk.index_slice] + + # NOTE: This is temporarily a workaround to deal with the case where the prefill_kv_indices + # is mismatched with the dst_kv_indices when page size > 1, this should never happen. + if len(chunked_dst_kv_indice) < len( + kv_chunk.prefill_kv_indices + ): + logger.warning( + f"len(chunked_dst_kv_indice) = {len(chunked_dst_kv_indice)}, len(kv_chunk.prefill_kv_indices) = {len(kv_chunk.prefill_kv_indices)}" + ) + kv_chunk.prefill_kv_indices = kv_chunk.prefill_kv_indices[ + : len(chunked_dst_kv_indice) + ] + + target_rank_registration_info: KVArgsRegisterInfo = ( + self.decode_kv_args_table[req.mooncake_session_id] + ) + if self.is_mla_backend or ( + self.attn_tp_size + == target_rank_registration_info.dst_attn_tp_size + ): + ret = self.send_kvcache( + req.mooncake_session_id, + kv_chunk.prefill_kv_indices, + target_rank_registration_info.dst_kv_ptrs, + chunked_dst_kv_indice, + executor, + ) + else: + ret = self.send_kvcache_slice( + req.mooncake_session_id, + kv_chunk.prefill_kv_indices, + target_rank_registration_info.dst_kv_ptrs, + chunked_dst_kv_indice, + target_rank_registration_info.dst_tp_rank, + target_rank_registration_info.dst_attn_tp_size, + target_rank_registration_info.dst_kv_item_len, + executor, + ) + if ret != 0: + with self.session_lock: + self.session_failures[req.mooncake_session_id] += 1 + # Failures should never happen if the session is not dead, if the session fails once, mark it as failed + if self.session_failures[req.mooncake_session_id] >= 1: + self.failed_sessions.add(req.mooncake_session_id) + logger.error( + f"Session {req.mooncake_session_id} failed." + ) + self.record_failure( + kv_chunk.room, + f"Failed to send kv chunk of {kv_chunk.room} to {req.endpoint}:{req.dst_port}", + ) + self.update_status(kv_chunk.room, KVPoll.Failed) + self.sync_status_to_decode_endpoint( + req.endpoint, + req.dst_port, + req.room, + KVPoll.Failed, + local_rank, + ) + break + + if kv_chunk.is_last: + if self.pp_group.is_last_rank: + # Only the last chunk we need to send the aux data + ret = self.send_aux( + req, + kv_chunk.prefill_aux_index, + target_rank_registration_info.dst_aux_ptrs, + ) + polls.append(True if ret == 0 else False) + dst_ranks_infos.append( + (req.endpoint, req.dst_port, req.room) + ) + + # Only sync status when all the dst ranks have received the kvcache + if len(polls) == req.required_dst_info_num: + status = KVPoll.Success if all(polls) else KVPoll.Failed + self.update_status(req.room, status) + for endpoint, dst_port, room in dst_ranks_infos: + self.sync_status_to_decode_endpoint( + endpoint, dst_port, room, status, local_rank + ) + else: + # Dummy request means the decode instance is not used, so its status can be marked as success directly + # Dummy request does not need to sync status to decode endpoint + if kv_chunk.is_last and req.room in self.request_status: + self.update_status(req.room, KVPoll.Success) + + if ( + kv_chunk.room not in self.request_status + or self.check_status(kv_chunk.room) == KVPoll.Success + ): + if kv_chunk.room in self.transfer_infos: + self.transfer_infos.pop(kv_chunk.room) + + except Exception as e: + # NOTE(shangming): Remove this when we make sure the transfer thread is bug-free + raise RuntimeError( + f"Transfer thread failed because of {e}. Prefill instance with bootstrap_port={self.bootstrap_port} is dead." + ) + + def _bind_server_socket(self): + self.server_socket.bind(format_tcp_address(self.local_ip, self.rank_port)) + + def start_prefill_thread(self): + self.rank_port = get_free_port() + self._bind_server_socket() + + def bootstrap_thread(): + """This thread recvs pre-alloc notification from the decode engine""" + # KVPoll.Bootstrapping -> KVPoll.WaitingForInput + while True: + waiting_req_bytes = self.server_socket.recv_multipart() + room = waiting_req_bytes[0].decode("ascii") + mooncake_session_id = waiting_req_bytes[3].decode("ascii") + if room == "None": + self.decode_kv_args_table[mooncake_session_id] = ( + KVArgsRegisterInfo.from_zmq(waiting_req_bytes) + ) + with self.session_lock: + if mooncake_session_id in self.failed_sessions: + self.failed_sessions.remove(mooncake_session_id) + if mooncake_session_id in self.session_failures: + del self.session_failures[mooncake_session_id] + logger.debug( + f"Register KVArgs from {mooncake_session_id} successfully" + ) + continue + else: + required_dst_info_num = int(waiting_req_bytes[6].decode("ascii")) + room = int(room) + if room not in self.transfer_infos: + self.transfer_infos[room] = {} + + self.transfer_infos[room][mooncake_session_id] = ( + TransferInfo.from_zmq(waiting_req_bytes) + ) + # NOTE: after bootstrapping we can mark the req as waiting for input + if len(self.transfer_infos[room]) == required_dst_info_num: + self.update_status(room, KVPoll.WaitingForInput) + + threading.Thread(target=bootstrap_thread).start() + + def _handle_aux_data(self, msg: List[bytes]): + """Handle AUX_DATA messages received by the decode thread.""" + room = int(msg[1].decode("ascii")) + buffer_index = int(msg[2].decode("ascii")) + aux_index = int(msg[3].decode("ascii")) + data_length = struct.unpack(">I", msg[4])[0] + data = msg[5] + + if len(data) != data_length: + logger.error(f"AUX_DATA length mismatch for bootstrap_room {room}") + return + + AuxDataCodec.deserialize_data_to_buffer( + self.kv_args, buffer_index, aux_index, data + ) + + logger.debug( + f"Received AUX_DATA for bootstrap_room {room} with length:{len(data)}" + ) + + def start_decode_thread(self): + self.rank_port = get_free_port() + self._bind_server_socket() + + def decode_thread(): + while True: + msg = self.server_socket.recv_multipart() + if msg[0] == MooncakeKVManager.AUX_DATA_HEADER: + self._handle_aux_data(msg) + continue + + (bootstrap_room, status, prefill_rank) = msg + status = int(status.decode("ascii")) + bootstrap_room = int(bootstrap_room.decode("ascii")) + prefill_rank = int(prefill_rank.decode("ascii")) + + if status == KVPoll.Success: + if bootstrap_room in self.request_status: + self.prefill_response_tracker[bootstrap_room].add(prefill_rank) + expected_response_num = ( + self.required_prefill_response_num_table[bootstrap_room] + ) + arrived_response_num = len( + self.prefill_response_tracker[bootstrap_room] + ) + if arrived_response_num == expected_response_num: + self.update_status(bootstrap_room, KVPoll.Success) + elif status == KVPoll.Failed: + self.record_failure( + bootstrap_room, + f"Failed to get kvcache from prefill instance, it might be dead", + ) + self.update_status(bootstrap_room, status) + + def heartbeat_checker(): + while True: + time.sleep(self.heartbeat_interval) + with self.connection_lock: + addresses = list(self.prefill_dp_size_table.keys()) + + for bootstrap_addr in addresses: + session = None + try: + with self.session_pool_lock: + session = self.session_pool[bootstrap_addr] + response = session.get( + f"http://{bootstrap_addr}/health", + timeout=(2, 3), + headers={"Connection": "keep-alive"}, + ) + if response.status_code == 200: + self.heartbeat_failures[bootstrap_addr] = 0 + + current_rooms = self.addr_to_rooms_tracker[ + bootstrap_addr + ].copy() + + for bootstrap_room in current_rooms: + # Remove KVPoll.Success requests from the tracker + if bootstrap_room not in self.request_status: + self.addr_to_rooms_tracker[bootstrap_addr].discard( + bootstrap_room + ) + else: + logger.info( + f"Attempting to reconnect to {bootstrap_addr}..." + ) + self.heartbeat_failures[bootstrap_addr] = ( + self.heartbeat_failures.get(bootstrap_addr, 0) + 1 + ) + with self.session_pool_lock: + if bootstrap_addr in self.session_pool: + del self.session_pool[bootstrap_addr] + except Exception: + logger.info(f"Attempting to reconnect to {bootstrap_addr}...") + self.heartbeat_failures[bootstrap_addr] = ( + self.heartbeat_failures.get(bootstrap_addr, 0) + 1 + ) + + if ( + self.heartbeat_failures.get(bootstrap_addr, 0) + >= self.max_failures + ): + self._handle_node_failure(bootstrap_addr) + with self.session_pool_lock: + if bootstrap_addr in self.session_pool: + del self.session_pool[bootstrap_addr] + + threading.Thread(target=decode_thread).start() + threading.Thread(target=heartbeat_checker).start() + + def add_transfer_request( + self, + bootstrap_room: int, + kv_indices: npt.NDArray[np.int32], + index_slice: slice, + is_last: bool, + aux_index: Optional[int] = None, + ): + assert self.disaggregation_mode == DisaggregationMode.PREFILL + assert not is_last or (is_last and aux_index is not None) + + if ( + bootstrap_room not in self.request_status + or self.check_status(bootstrap_room) == KVPoll.Failed + ): + logger.debug( + "Request with bootstrap_room=%s already failed", bootstrap_room + ) + return + + if bootstrap_room not in self.transfer_infos: + # This means that the current rank is a dummy rank for this request, + # and it has already been marked as success, so there is no need to + # add further chunks into the transfer queue. + return + + # NOTE(shangming): sharding according to the dst_infos to make sure + # requests with the same dst_sessions will be added into the same + # queue, which enables early abort with failed sessions. + dst_infos = self.transfer_infos[bootstrap_room].keys() + session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos) + shard_idx = session_port_sum % len(self.transfer_queues) + + self.transfer_queues[shard_idx].put( + TransferKVChunk( + room=bootstrap_room, + prefill_kv_indices=kv_indices, + index_slice=index_slice, + is_last=is_last, + prefill_aux_index=aux_index, + ) + ) + + def check_status(self, bootstrap_room: int): + return self.request_status[bootstrap_room] + + def update_status(self, bootstrap_room: int, status: KVPoll): + if bootstrap_room not in self.request_status: + self.request_status[bootstrap_room] = status + else: + # NOTE: status is only allowed to be incremented unless it is KVPoll.Failed + if status == KVPoll.Failed: + self.request_status[bootstrap_room] = KVPoll.Failed + else: + self.request_status[bootstrap_room] = max( + self.request_status[bootstrap_room], status + ) + + def record_failure(self, bootstrap_room: int, failure_reason: str): + with self.failure_lock: + self.failure_records[bootstrap_room] = failure_reason + + def get_session_id(self): + return self.engine.get_session_id() + + def _register_to_bootstrap(self): + """Register KVSender to bootstrap server via HTTP POST.""" + if self.dist_init_addr: + # multi node case: bootstrap server's host is dist_init_addr + if self.dist_init_addr.startswith("["): # [ipv6]:port or [ipv6] + if self.dist_init_addr.endswith("]"): + host = self.dist_init_addr + else: + host, _ = self.dist_init_addr.rsplit(":", 1) + else: + host = socket.gethostbyname(self.dist_init_addr.rsplit(":", 1)[0]) + else: + # single node case: bootstrap server's host is same as http server's host + host = self.bootstrap_host + host = maybe_wrap_ipv6_address(host) + + bootstrap_server_url = f"{host}:{self.bootstrap_port}" + url = f"http://{bootstrap_server_url}/route" + payload = { + "role": "Prefill", + "attn_tp_size": self.attn_tp_size, + "attn_tp_rank": self.attn_tp_rank, + "attn_dp_size": self.attn_dp_size, + "attn_dp_rank": self.attn_dp_rank, + "pp_size": self.pp_size, + "pp_rank": self.pp_rank, + "system_dp_size": self.system_dp_size, + "system_dp_rank": self.system_dp_rank, + "rank_ip": self.local_ip, + "rank_port": self.rank_port, + } + + try: + response = requests.put(url, json=payload, timeout=5) + if response.status_code == 200: + logger.debug("Prefill successfully registered to bootstrap server.") + else: + logger.error( + f"Prefill instance failed to connect to bootstrap server: {response.status_code}, {response.text}" + ) + except Exception as e: + logger.error( + f"Prefill instance failed to register to bootstrap server: {e}" + ) + + def _handle_node_failure(self, failed_bootstrap_addr): + with self.connection_lock: + keys_to_remove = [ + k for k in self.connection_pool if k.startswith(failed_bootstrap_addr) + ] + for k in keys_to_remove: + del self.connection_pool[k] + if failed_bootstrap_addr in self.prefill_attn_tp_size_table: + del self.prefill_attn_tp_size_table[failed_bootstrap_addr] + if failed_bootstrap_addr in self.prefill_dp_size_table: + del self.prefill_dp_size_table[failed_bootstrap_addr] + if failed_bootstrap_addr in self.prefill_pp_size_table: + del self.prefill_pp_size_table[failed_bootstrap_addr] + + possible_affected_rooms = self.addr_to_rooms_tracker.get( + failed_bootstrap_addr, [] + ) + if failed_bootstrap_addr in self.addr_to_rooms_tracker: + del self.addr_to_rooms_tracker[failed_bootstrap_addr] + + # Report the requests associated with the failed bootstrap addr and mark their status as KVPoll.Failed + affected_rooms = [] + for room in possible_affected_rooms: + if ( + room in self.request_status + and self.check_status(room) != KVPoll.Success + ): + self.record_failure( + room, + f"Losing connection with prefill instance (bootstrap_addr: {failed_bootstrap_addr})", + ) + self.update_status(room, KVPoll.Failed) + affected_rooms.append(room) + logger.error( + f"Losing connection with prefill instance (bootstrap_addr: {failed_bootstrap_addr}), {len(affected_rooms)} requests affected" + ) + + +class MooncakeKVSender(BaseKVSender): + + def __init__( + self, + mgr: MooncakeKVManager, + bootstrap_addr: str, + bootstrap_room: int, + dest_tp_ranks: List[int], + pp_rank: int, + ): + self.kv_mgr = mgr + self.bootstrap_room = bootstrap_room + self.kv_mgr.update_status(bootstrap_room, KVPoll.Bootstrapping) + self.aux_index = None + self.bootstrap_server_url = bootstrap_addr + self.conclude_state = None + self.init_time = time.time() + # inner state + self.curr_idx = 0 + + def init(self, num_kv_indices: int, aux_index: Optional[int] = None): + self.num_kv_indices = num_kv_indices + self.aux_index = aux_index + + def send( + self, + kv_indices: npt.NDArray[np.int32], + ): + index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) + self.curr_idx += len(kv_indices) + is_last = self.curr_idx == self.num_kv_indices + + if not is_last: + self.kv_mgr.add_transfer_request( + self.bootstrap_room, + kv_indices, + index_slice, + False, + ) + else: + self.kv_mgr.add_transfer_request( + self.bootstrap_room, + kv_indices, + index_slice, + True, + aux_index=self.aux_index, + ) + + def poll(self) -> KVPoll: + if self.conclude_state is None: + status = self.kv_mgr.check_status(self.bootstrap_room) + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + elif status == KVPoll.Bootstrapping: + if self.init_time is not None: + now = time.time() + elapsed = now - self.init_time + if elapsed >= self.kv_mgr.bootstrap_timeout: + logger.warning_once( + "Some requests timed out when bootstrapping, " + "which means prefill instances fail to receive the KV indices from the decode instance of this request. " + "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_BOOTSTRAP_TIMEOUT=600' (10 minutes) to relax the timeout condition. " + ) + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.Bootstrapping", + ) + self.conclude_state = KVPoll.Failed + return KVPoll.Failed + + return status + else: + return self.conclude_state + + def clear(self) -> None: + if self.bootstrap_room in self.kv_mgr.request_status: + self.kv_mgr.request_status.pop(self.bootstrap_room) + + def failure_exception(self): + # Explicitly set the status to failure since this request has failed in another rank + if self.conclude_state is None: + self.conclude_state = KVPoll.Failed + + self.clear() + + with self.kv_mgr.failure_lock: + failure_reason = self.kv_mgr.failure_records.pop( + self.bootstrap_room, "Failed due to an unknown reason from another rank" + ) + raise KVTransferError(self.bootstrap_room, failure_reason) + + def abort(self): + self.kv_mgr.record_failure( + self.bootstrap_room, + "Aborted by AbortReq.", + ) + # Explicitly set the status to failure since this request has been aborted + self.conclude_state = KVPoll.Failed + + +class MooncakeKVReceiver(BaseKVReceiver): + _ctx = zmq.Context() + _socket_cache = {} + _socket_locks = {} + _global_lock = threading.Lock() + + def __init__( + self, + mgr: MooncakeKVManager, + bootstrap_addr: str, + bootstrap_room: Optional[int] = None, + prefill_dp_rank: Optional[int] = None, + ): + self.bootstrap_room = bootstrap_room + self.bootstrap_addr = bootstrap_addr + self.kv_mgr = mgr + self.session_id = self.kv_mgr.get_session_id() + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) + self.conclude_state = None + self.init_time = None + + if self.bootstrap_addr not in self.kv_mgr.prefill_dp_size_table: + ( + self.prefill_attn_tp_size, + self.prefill_dp_size, + self.prefill_pp_size, + ) = self._get_prefill_parallel_info_from_server() + if ( + self.prefill_attn_tp_size is None + or self.prefill_dp_size is None + or self.prefill_pp_size is None + ): + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Could not fetch prefill parallel info from bootstrap_addr: {self.bootstrap_addr}", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return + else: + logger.debug( + f"Fetch prefill parallel info from [{self.bootstrap_addr}]: DP size:{self.prefill_dp_size}, TP size:{self.prefill_attn_tp_size} PP size:{self.prefill_pp_size}" + ) + self.kv_mgr.prefill_attn_tp_size_table[self.bootstrap_addr] = ( + self.prefill_attn_tp_size + ) + self.kv_mgr.prefill_dp_size_table[self.bootstrap_addr] = ( + self.prefill_dp_size + ) + self.kv_mgr.prefill_pp_size_table[self.bootstrap_addr] = ( + self.prefill_pp_size + ) + else: + self.prefill_attn_tp_size = self.kv_mgr.prefill_attn_tp_size_table[ + self.bootstrap_addr + ] + self.prefill_dp_size = self.kv_mgr.prefill_dp_size_table[ + self.bootstrap_addr + ] + self.prefill_pp_size = self.kv_mgr.prefill_pp_size_table[ + self.bootstrap_addr + ] + + # Currently, we don't allow prefill instance and decode instance to + # have different TP sizes per DP rank, except for models using MLA. + if self.kv_mgr.attn_tp_size == self.prefill_attn_tp_size: + self.target_tp_rank = ( + self.kv_mgr.kv_args.engine_rank % self.kv_mgr.attn_tp_size + ) + self.required_dst_info_num = 1 + self.required_prefill_response_num = 1 * ( + self.prefill_pp_size // self.kv_mgr.pp_size + ) + self.target_tp_ranks = [self.target_tp_rank] + elif self.kv_mgr.attn_tp_size > self.prefill_attn_tp_size: + if not self.kv_mgr.is_mla_backend: + logger.warning_once( + "Performance is NOT guaranteed when using different TP sizes for non-MLA models. " + ) + self.target_tp_rank = ( + self.kv_mgr.kv_args.engine_rank % self.kv_mgr.attn_tp_size + ) // (self.kv_mgr.attn_tp_size // self.prefill_attn_tp_size) + self.required_dst_info_num = ( + self.kv_mgr.attn_tp_size // self.prefill_attn_tp_size + ) + self.required_prefill_response_num = 1 * ( + self.prefill_pp_size // self.kv_mgr.pp_size + ) + self.target_tp_ranks = [self.target_tp_rank] + else: + if not self.kv_mgr.is_mla_backend: + logger.warning_once( + "Performance is NOT guaranteed when using different TP sizes for non-MLA models. " + ) + # For non-MLA models, one decode rank needs to retrieve KVCache from multiple prefill ranks for non MLA models; + self.target_tp_ranks = [ + rank + for rank in range( + (self.kv_mgr.kv_args.engine_rank % self.kv_mgr.attn_tp_size) + * (self.prefill_attn_tp_size // self.kv_mgr.attn_tp_size), + (self.kv_mgr.kv_args.engine_rank % self.kv_mgr.attn_tp_size + 1) + * (self.prefill_attn_tp_size // self.kv_mgr.attn_tp_size), + ) + ] + + # For MLA models, we can retrieve KVCache from only one prefill rank, but we still need to maintain + # multiple connections in the connection pool and have to send dummy requests to other prefill ranks, + # or the KVPoll will never be set correctly + self.target_tp_rank = self.target_tp_ranks[0] + self.required_dst_info_num = 1 + if self.kv_mgr.is_mla_backend: + self.required_prefill_response_num = ( + self.prefill_pp_size // self.kv_mgr.pp_size + ) + else: + self.required_prefill_response_num = ( + self.prefill_attn_tp_size // self.kv_mgr.attn_tp_size + ) * (self.prefill_pp_size // self.kv_mgr.pp_size) + + if prefill_dp_rank is not None: + logger.debug(f"Targeting DP rank: {prefill_dp_rank}") + self.prefill_dp_rank = prefill_dp_rank + else: + self.prefill_dp_rank = bootstrap_room % self.prefill_dp_size + + # FIXME: alias here: target_dp_group -> prefill_dp_rank + self.target_dp_group = self.prefill_dp_rank + + self.kv_mgr.required_prefill_response_num_table[self.bootstrap_room] = ( + self.required_prefill_response_num + ) + # NOTE: key distinguished by bootstrap_addr, target_dp_group, and target_tp_rank + bootstrap_key = ( + f"{self.bootstrap_addr}_{self.target_dp_group}_{self.target_tp_rank}" + ) + + if bootstrap_key not in self.kv_mgr.connection_pool: + bootstrap_infos = [] + for target_tp_rank in self.target_tp_ranks: + for target_pp_rank in range(self.prefill_pp_size): + bootstrap_info = self._get_bootstrap_info_from_server( + target_tp_rank, self.target_dp_group, target_pp_rank + ) + if bootstrap_info is not None: + if self.kv_mgr.is_mla_backend: + # For MLA: target_tp_rank is the selected real rank, others are dummy ranks + bootstrap_info["is_dummy"] = not bool( + target_tp_rank == self.target_tp_rank + or self.target_tp_rank is None + ) + else: + # For non-MLA: all target_tp_ranks are selected real ranks + bootstrap_info["is_dummy"] = False + logger.debug( + f"Fetched bootstrap info: {bootstrap_info} for DP {self.target_dp_group} TP {target_tp_rank} PP {target_pp_rank}" + ) + bootstrap_infos.append(bootstrap_info) + else: + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Could not fetch bootstrap info for engine rank: {self.kv_mgr.kv_args.engine_rank} and target_dp_group: {self.target_dp_group} and target_pp_rank {target_pp_rank}", + ) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Failed) + return + + self.bootstrap_infos = bootstrap_infos + self.kv_mgr.connection_pool[bootstrap_key] = self.bootstrap_infos + + # Register kv_args only once to prefill KVManager according to the info fetched from the bootstrap server + self._register_kv_args() + else: + self.bootstrap_infos = self.kv_mgr.connection_pool[bootstrap_key] + + assert len(self.bootstrap_infos) > 0 + self.kv_mgr.addr_to_rooms_tracker[self.bootstrap_addr].add(self.bootstrap_room) + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.WaitingForInput) + + def _get_bootstrap_info_from_server( + self, engine_rank, target_dp_group, target_pp_rank + ): + """Fetch the bootstrap info from the bootstrap server.""" + try: + url = f"http://{self.bootstrap_addr}/route?engine_rank={engine_rank}&target_dp_group={target_dp_group}&target_pp_rank={target_pp_rank}" + response = requests.get(url, timeout=5) + if response.status_code == 200: + bootstrap_info = response.json() + return bootstrap_info + else: + logger.error( + f"Failed to get prefill server info: {response.status_code}, {response.text}" + ) + return None + except Exception as e: + logger.error(f"Error fetching prefill info from bootstrap: {e}") + return None + + def _get_prefill_parallel_info_from_server( + self, + ) -> Tuple[Optional[int], Optional[int], Optional[int]]: + """Fetch the prefill parallel info from the bootstrap server.""" + try: + url = f"http://{self.bootstrap_addr}/route?engine_rank={-1}&target_dp_group={-1}&target_pp_rank={-1}" + response = requests.get(url) + if response.status_code == 200: + prefill_parallel_info = response.json() + return ( + int(prefill_parallel_info["prefill_attn_tp_size"]), + int(prefill_parallel_info["prefill_dp_size"]), + int(prefill_parallel_info["prefill_pp_size"]), + ) + else: + logger.error( + f"Failed to get prefill parallel info: {response.status_code}, {response.text}" + ) + return None, None, None + except Exception as e: + logger.error(f"Error fetching prefill parallel info from bootstrap: {e}") + return None, None, None + + def _register_kv_args(self): + for bootstrap_info in self.bootstrap_infos: + packed_kv_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs + ) + packed_aux_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs + ) + # Note(shangming): No need to add pp rank here since pp is not supported on the decode side yet + tp_rank = self.kv_mgr.kv_args.engine_rank + kv_item_len = self.kv_mgr.kv_args.kv_item_lens[0] + dst_tp_rank = str(tp_rank).encode("ascii") + dst_attn_tp_size = str(self.kv_mgr.attn_tp_size).encode("ascii") + dst_kv_item_len = str(kv_item_len).encode("ascii") + + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + with lock: + sock.send_multipart( + [ + "None".encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + self.session_id.encode("ascii"), + packed_kv_data_ptrs, + packed_aux_data_ptrs, + dst_tp_rank, + dst_attn_tp_size, + dst_kv_item_len, + ] + ) + + @classmethod + def _connect(cls, endpoint: str, is_ipv6: bool = False): + with cls._global_lock: + if endpoint not in cls._socket_cache: + sock = cls._ctx.socket(zmq.PUSH) + if is_ipv6: + sock.setsockopt(zmq.IPV6, 1) + sock.connect(endpoint) + cls._socket_cache[endpoint] = sock + cls._socket_locks[endpoint] = threading.Lock() + return cls._socket_cache[endpoint], cls._socket_locks[endpoint] + + @classmethod + def _connect_to_bootstrap_server(cls, bootstrap_info: dict): + ip_address = bootstrap_info["rank_ip"] + port = bootstrap_info["rank_port"] + is_ipv6_address = is_valid_ipv6_address(ip_address) + sock, lock = cls._connect( + format_tcp_address(ip_address, port), is_ipv6=is_ipv6_address + ) + return sock, lock + + def init(self, kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None): + for bootstrap_info in self.bootstrap_infos: + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + is_dummy = bootstrap_info["is_dummy"] + + with lock: + sock.send_multipart( + [ + str(self.bootstrap_room).encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + self.session_id.encode("ascii"), + kv_indices.tobytes() if not is_dummy else b"", + str(aux_index).encode("ascii") if not is_dummy else b"", + str(self.required_dst_info_num).encode("ascii"), + ] + ) + self.init_time = time.time() + + def poll(self) -> KVPoll: + if self.conclude_state is None: + status = self.kv_mgr.check_status(self.bootstrap_room) + if status in (KVPoll.Success, KVPoll.Failed): + self.conclude_state = status + elif status == KVPoll.WaitingForInput: + if self.init_time is not None: + now = time.time() + elapsed = now - self.init_time + if elapsed >= self.kv_mgr.waiting_timeout: + logger.warning_once( + "Some requests fail to receive KV Cache transfer done signal after bootstrapping. " + "If a greater mean TTFT is acceptable, you can 'export SGLANG_DISAGGREGATION_WAITING_TIMEOUT=600' (10 minutes) to relax the timeout condition. " + ) + self.kv_mgr.record_failure( + self.bootstrap_room, + f"Request {self.bootstrap_room} timed out after {elapsed:.1f}s in KVPoll.WaitingForInput", + ) + self.conclude_state = KVPoll.Failed + return KVPoll.Failed + + return status + + else: + return self.conclude_state + + def clear(self) -> None: + if self.bootstrap_room in self.kv_mgr.request_status: + self.kv_mgr.request_status.pop(self.bootstrap_room) + + if self.bootstrap_room in self.kv_mgr.required_prefill_response_num_table: + self.kv_mgr.required_prefill_response_num_table.pop(self.bootstrap_room) + + if self.bootstrap_room in self.kv_mgr.prefill_response_tracker: + self.kv_mgr.prefill_response_tracker.pop(self.bootstrap_room) + + def failure_exception(self): + # Explicitly set the status to failure since this request has failed in another rank + if self.conclude_state is None: + self.conclude_state = KVPoll.Failed + + self.clear() + + with self.kv_mgr.failure_lock: + failure_reason = self.kv_mgr.failure_records.pop( + self.bootstrap_room, "Failed due to an unknown reason from another rank" + ) + raise KVTransferError(self.bootstrap_room, failure_reason) + + def abort(self): + self.kv_mgr.record_failure( + self.bootstrap_room, + "Aborted by AbortReq.", + ) + # Explicitly set the status to failure since this request has been aborted + self.conclude_state = KVPoll.Failed + + +class MooncakeKVBootstrapServer(BaseKVBootstrapServer): + def __init__(self, host: str, port: int): + self.host = host + self.port = port + self.app = web.Application() + self.store = dict() + self.lock = asyncio.Lock() + self._setup_routes() + self.pp_size = None + self.attn_tp_size = None + self.dp_size = None + self.prefill_port_table: Dict[ + int, Dict[int, Dict[int, Dict[str, Union[str, int]]]] + ] = {} + + # Start bootstrap server + self.thread = threading.Thread(target=self._run_server, daemon=True) + self.run() + + def run(self): + self.thread.start() + + def _setup_routes(self): + self.app.router.add_route("*", "/route", self._handle_route) + self.app.router.add_get("/health", self._handle_health_check) + + async def _handle_health_check(self, request): + return web.Response(text="OK", status=200) + + async def _handle_route(self, request: web.Request): + method = request.method + if method == "PUT": + return await self._handle_route_put(request) + elif method == "GET": + return await self._handle_route_get(request) + else: + return web.Response( + text="Method not allowed", status=405, content_type="application/json" + ) + + async def _handle_route_put(self, request: web.Request): + data = await request.json() + role = data["role"] + attn_tp_size = data["attn_tp_size"] + attn_tp_rank = data["attn_tp_rank"] + attn_dp_size = data["attn_dp_size"] + attn_dp_rank = data["attn_dp_rank"] + pp_size = data["pp_size"] + pp_rank = data["pp_rank"] + system_dp_size = data["system_dp_size"] + system_dp_rank = data["system_dp_rank"] + rank_ip = data["rank_ip"] + rank_port = int(data["rank_port"]) + + if self.attn_tp_size is None: + self.attn_tp_size = attn_tp_size + + if self.dp_size is None: + self.dp_size = attn_dp_size if system_dp_size == 1 else system_dp_size + + if self.pp_size is None: + self.pp_size = pp_size + + if role == "Prefill": + if system_dp_size == 1: + dp_group = attn_dp_rank + else: + dp_group = system_dp_rank + + # Add lock to make sure thread-safe + async with self.lock: + if dp_group not in self.prefill_port_table: + self.prefill_port_table[dp_group] = {} + if attn_tp_rank not in self.prefill_port_table[dp_group]: + self.prefill_port_table[dp_group][attn_tp_rank] = {} + + self.prefill_port_table[dp_group][attn_tp_rank][pp_rank] = { + "rank_ip": rank_ip, + "rank_port": rank_port, + } + logger.debug( + f"Register prefill bootstrap: DP{dp_group} TP{attn_tp_rank} PP{pp_rank} with rank_ip: {rank_ip} and rank_port: {rank_port}" + ) + + return web.Response(text="OK", status=200) + + async def _handle_route_get(self, request: web.Request): + engine_rank = request.query.get("engine_rank") + target_dp_group = request.query.get("target_dp_group") + target_pp_rank = request.query.get("target_pp_rank") + if not engine_rank or not target_dp_group or not target_pp_rank: + return web.Response(text="Missing inputs for bootstrap server.", status=400) + + # Currently we use engine_rank == -1 and target_dp_group == -1 to sync dp size + if ( + int(engine_rank) == -1 + and int(target_dp_group) == -1 + and int(target_pp_rank) == -1 + ): + prefill_parallel_info = { + "prefill_attn_tp_size": self.attn_tp_size, + "prefill_dp_size": self.dp_size, + "prefill_pp_size": self.pp_size, + } + return web.json_response(prefill_parallel_info, status=200) + + # Find corresponding prefill info + async with self.lock: + bootstrap_info = self.prefill_port_table[int(target_dp_group)][ + int(engine_rank) + ][int(target_pp_rank)] + + if bootstrap_info is not None: + return web.json_response(bootstrap_info, status=200) + else: + return web.Response(text="Bootstrap info not Found", status=404) + + def _run_server(self): + try: + # Event Loop + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + + access_log = None + if logging.getLogger(__name__).getEffectiveLevel() <= logging.DEBUG: + access_log = self.app.logger + + self._runner = web.AppRunner(self.app, access_log=access_log) + self._loop.run_until_complete(self._runner.setup()) + + site = web.TCPSite(self._runner, host=self.host, port=self.port) + self._loop.run_until_complete(site.start()) + self._loop.run_forever() + except Exception as e: + logger.error(f"Server error: {str(e)}") + finally: + # Cleanup + self._loop.run_until_complete(self._runner.cleanup()) + self._loop.close() + + def close(self): + """Shutdown""" + if self._loop is not None and self._loop.is_running(): + self._loop.call_soon_threadsafe(self._loop.stop) + logger.info("Stopping server loop...") + + if self.thread.is_alive(): + self.thread.join(timeout=2) + logger.info("Server thread stopped") + + def poll(self) -> KVPoll: ... diff --git a/python/sglang/srt/disaggregation/mooncake/transfer_engine.py b/python/sglang/srt/disaggregation/mooncake/transfer_engine.py new file mode 100644 index 000000000..54657bb46 --- /dev/null +++ b/python/sglang/srt/disaggregation/mooncake/transfer_engine.py @@ -0,0 +1,164 @@ +import logging +from typing import List, Optional + +from sglang.srt.utils import get_bool_env_var, get_free_port, maybe_wrap_ipv6_address + +logger = logging.getLogger(__name__) + + +class MooncakeTransferEngine: + + def __init__(self, hostname: str, gpu_id: int, ib_device: Optional[str] = None): + try: + from mooncake.engine import TransferEngine + except ImportError as e: + raise ImportError( + "Please install mooncake by following the instructions at " + "https://github.com/kvcache-ai/Mooncake/blob/main/doc/en/build.md " # noqa: E501 + "to run SGLang with MooncakeTransferEngine." + ) from e + + self.engine = TransferEngine() + self.hostname = hostname + self.gpu_id = gpu_id + self.ib_device = ib_device + + self.initialize( + hostname=self.hostname, + device_name=self.ib_device, + ) + self.session_id = ( + f"{maybe_wrap_ipv6_address(self.hostname)}:{self.engine.get_rpc_port()}" + ) + + def register(self, ptr, length): + try: + ret_value = self.engine.register_memory(ptr, length) + except Exception: + # Mark register as failed + ret_value = -1 + + if ret_value != 0: + logger.debug("Mooncake memory registration %s failed.", ptr) + + def deregister(self, ptr): + try: + ret_value = self.engine.unregister_memory(ptr) + except Exception: + # Mark deregister as failed + ret_value = -1 + + if ret_value != 0: + logger.debug("Mooncake memory deregistration %s failed.", ptr) + + def batch_register(self, ptrs: List[int], lengths: List[int]) -> int: + """Batch register multiple memory regions.""" + try: + ret_value = self.engine.batch_register_memory(ptrs, lengths) + except Exception: + # Mark batch register as failed + ret_value = -1 + if not hasattr(self.engine, "batch_register_memory"): + raise RuntimeError( + "Mooncake's batch register requires a newer version of mooncake-transfer-engine. " + "Please upgrade Mooncake." + ) + + if ret_value != 0: + logger.debug("Mooncake batch memory registration failed.") + return ret_value + + def batch_deregister(self, ptrs: List[int]) -> int: + """Batch deregister multiple memory regions.""" + try: + ret_value = self.engine.batch_unregister_memory(ptrs) + except Exception: + # Mark batch deregister as failed + ret_value = -1 + + if ret_value != 0: + logger.debug("Mooncake batch memory deregistration failed.") + return ret_value + + def initialize( + self, + hostname: str, + device_name: Optional[str], + ) -> None: + """Initialize the mooncake instance.""" + if get_bool_env_var("ENABLE_ASCEND_TRANSFER_WITH_MOONCAKE", "false"): + hostname += f":{get_free_port()}:npu_{self.gpu_id}" + ret_value = self.engine.initialize( + hostname, + "P2PHANDSHAKE", + "ascend", + device_name if device_name is not None else "", + ) + else: + ret_value = self.engine.initialize( + hostname, + "P2PHANDSHAKE", + "rdma", + device_name if device_name is not None else "", + ) + if ret_value != 0: + logger.error("Mooncake Transfer Engine initialization failed.") + raise RuntimeError("Mooncake Transfer Engine initialization failed.") + + def transfer_sync( + self, session_id: str, buffer: int, peer_buffer_address: int, length: int + ) -> int: + """Synchronously transfer data to the specified address.""" + try: + # the first time: based on session_id (which contains remote_ip) to construct a queue pair, and cache the queue pair + # later: based on the cached queue pair to send data + ret = self.engine.transfer_sync_write( + session_id, buffer, peer_buffer_address, length + ) + except Exception: + # Mark transfer request as failed + ret = -1 + + if ret < 0: + # Do not raise an exception here, since some transfer requests fail should be accepted and the execution thread should not be stopped. + logger.debug( + "Failed to transfer data from %s to %s - %s.", + buffer, + session_id, + peer_buffer_address, + ) + + return ret + + def batch_transfer_sync( + self, + session_id: str, + buffers: List[int], + peer_buffer_addresses: List[int], + lengths: List[int], + ) -> int: + """Synchronously transfer data to the specified addresses in batches.""" + try: + ret = self.engine.batch_transfer_sync_write( + session_id, buffers, peer_buffer_addresses, lengths + ) + except Exception: + ret = -1 + # Inform user to upgrade mooncake-transfer-engine >= 0.3.4.post2 + if not hasattr(self.engine, "batch_transfer_sync_write"): + raise RuntimeError( + "Mooncake's batch transfer requires mooncake-transfer-engine >= 0.3.4.post2. " + "Please upgrade Mooncake by 'pip install mooncake-transfer-engine --upgrade'" + ) + + if ret < 0: + logger.debug( + "Failed to batch transfer data. Buffers: %s, Session: %s, Peer addresses: %s", + buffers, + session_id, + peer_buffer_addresses, + ) + return ret + + def get_session_id(self): + return self.session_id diff --git a/python/sglang/srt/disaggregation/nixl/__init__.py b/python/sglang/srt/disaggregation/nixl/__init__.py new file mode 100644 index 000000000..4df7baba2 --- /dev/null +++ b/python/sglang/srt/disaggregation/nixl/__init__.py @@ -0,0 +1,6 @@ +from sglang.srt.disaggregation.nixl.conn import ( + NixlKVBootstrapServer, + NixlKVManager, + NixlKVReceiver, + NixlKVSender, +) diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py new file mode 100644 index 000000000..c911319ea --- /dev/null +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -0,0 +1,696 @@ +from __future__ import annotations + +import asyncio +import dataclasses +import logging +import queue +import socket +import struct +import threading +import uuid +from collections import defaultdict +from functools import cache +from typing import Dict, List, Optional, Set, Tuple, TypeAlias, Union + +import numpy as np +import numpy.typing as npt +import requests +import zmq +from aiohttp import web + +from sglang.srt.disaggregation.base.conn import BaseKVSender, KVArgs, KVPoll +from sglang.srt.disaggregation.common.conn import ( + CommonKVBootstrapServer, + CommonKVManager, + CommonKVReceiver, +) +from sglang.srt.disaggregation.common.utils import group_concurrent_contiguous +from sglang.srt.disaggregation.utils import DisaggregationMode +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ( + format_tcp_address, + get_local_ip_auto, + is_valid_ipv6_address, +) + +logger = logging.getLogger(__name__) + +GUARD = "NixlMsgGuard".encode("ascii") + + +@dataclasses.dataclass +class TransferInfo: + """Contains indices for a transfer, sent by KVReceiver. Received by prefill bootstrap thread.""" + + room: int + endpoint: str + dst_port: int + agent_name: str + dst_kv_indices: npt.NDArray[np.int32] + dst_aux_index: int + required_dst_info_num: int + + def is_dummy(self): + return self.dst_kv_indices.size == 0 + + @classmethod + def from_zmq(cls, msg: List[bytes]): + return cls( + room=int(msg[0].decode("ascii")), + endpoint=msg[1].decode("ascii"), + dst_port=int(msg[2].decode("ascii")), + agent_name=msg[3].decode("ascii"), + dst_kv_indices=np.frombuffer(msg[4], dtype=np.int32), + dst_aux_index=int(msg[5].decode("ascii")), + required_dst_info_num=int(msg[6].decode("ascii")), + ) + + +@dataclasses.dataclass +class KVArgsRegisterInfo: + """Contains base pointers and other info which only needs to be sent once by KVReceiver. Received by prefill bootstrap thread.""" + + room: str + endpoint: str + dst_port: int + agent_name: str + agent_metadata: bytes + dst_kv_ptrs: list[int] + dst_aux_ptrs: list[int] + gpu_id: int + decode_tp_size: int + decode_tp_rank: int + dst_kv_item_len: int + + @classmethod + def from_zmq(cls, msg: List[bytes]): + return cls( + room=str(msg[0].decode("ascii")), + endpoint=msg[1].decode("ascii"), + dst_port=int(msg[2].decode("ascii")), + agent_name=msg[3].decode("ascii"), + agent_metadata=msg[4], + dst_kv_ptrs=list(struct.unpack(f"{len(msg[5])//8}Q", msg[5])), + dst_aux_ptrs=list(struct.unpack(f"{len(msg[6])//8}Q", msg[6])), + gpu_id=int(msg[7].decode("ascii")), + decode_tp_size=int(msg[8].decode("ascii")), + decode_tp_rank=int(msg[9].decode("ascii")), + dst_kv_item_len=int(msg[10].decode("ascii")), + ) + + +@dataclasses.dataclass +class TransferStatus: + """Used by KV Receiver to know when a transfer is done.""" + + # KV chunk IDs that have been received. + received_kvs: Set[int] = dataclasses.field(default_factory=set) + # Number of kv chunks to expect, will know this after last chunk is received. + num_kvs_expected: Optional[int] = None + # Whether aux data has been received. + received_aux: bool = False + + def is_done(self): + if self.num_kvs_expected is None: + return False + return self.num_kvs_expected == len(self.received_kvs) and self.received_aux + + +class NixlKVManager(CommonKVManager): + def __init__( + self, + args: KVArgs, + disaggregation_mode: DisaggregationMode, + server_args: ServerArgs, + is_mla_backend: Optional[bool] = False, + ): + super().__init__(args, disaggregation_mode, server_args, is_mla_backend) + try: + from nixl._api import nixl_agent + except ImportError as e: + raise ImportError( + "Please install NIXL by following the instructions at " + "https://github.com/ai-dynamo/nixl/blob/main/README.md " + "to run SGLang with NixlTransferEngine." + ) from e + self.agent = nixl_agent(str(uuid.uuid4())) + self.local_ip = get_local_ip_auto() + self.server_socket = zmq.Context().socket(zmq.PULL) + if is_valid_ipv6_address(self.local_ip): + self.server_socket.setsockopt(zmq.IPV6, 1) + self.register_buffer_to_engine() + + if self.disaggregation_mode == DisaggregationMode.PREFILL: + self.request_status: Dict[int, KVPoll] = {} + self.transfer_infos: Dict[int, Dict[str, TransferInfo]] = {} + self.decode_kv_args_table: Dict[str, KVArgsRegisterInfo] = {} + self._start_bootstrap_thread() + elif self.disaggregation_mode == DisaggregationMode.DECODE: + self.transfer_statuses: Dict[int, TransferStatus] = defaultdict( + TransferStatus + ) + else: + raise ValueError( + f"Unsupported DisaggregationMode: {self.disaggregation_mode}" + ) + + def check_status(self, bootstrap_room: int): + return self.request_status[bootstrap_room] + + def update_status(self, bootstrap_room: int, status: KVPoll): + if bootstrap_room not in self.request_status: + self.request_status[bootstrap_room] = status + else: + # NOTE: The prefill engine could recv bootstrapping first + self.request_status[bootstrap_room] = max( + self.request_status[bootstrap_room], status + ) + + def register_buffer_to_engine(self): + kv_addrs = [] + for kv_data_ptr, kv_data_len in zip( + self.kv_args.kv_data_ptrs, self.kv_args.kv_data_lens + ): + kv_addrs.append((kv_data_ptr, kv_data_len, self.kv_args.gpu_id, "")) + self.kv_descs = self.agent.register_memory(kv_addrs, "VRAM") + logger.debug(f"Register kv tensors, len(kv_addr)= {len(kv_addrs)}") + if not self.kv_descs: + raise Exception("NIXL memory registration failed for kv tensors") + aux_addrs = [] + for aux_data_ptr, aux_data_len in zip( + self.kv_args.aux_data_ptrs, self.kv_args.aux_data_lens + ): + aux_addrs.append((aux_data_ptr, aux_data_len, 0, "")) + self.aux_descs = self.agent.register_memory(aux_addrs, "DRAM") + logger.debug(f"Register aux tensors, len(aux_addrs)= {len(aux_addrs)}") + if not self.aux_descs: + raise Exception("NIXL memory registration failed for aux tensors") + + def _add_remote_peer(self, decode_kv_args: KVArgsRegisterInfo): + agent_name = decode_kv_args.agent_name + if agent_name in self.decode_kv_args_table: + logger.info(f"Peer {agent_name} was already registered, ignoring.") + return + self.decode_kv_args_table[agent_name] = decode_kv_args + self.agent.add_remote_agent(decode_kv_args.agent_metadata) + + def send_kvcache( + self, + peer_name: str, + prefill_kv_indices: npt.NDArray[np.int32], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int32], + dst_gpu_id: int, + notif: str, + ): + # group by indices + prefill_kv_blocks, dst_kv_blocks = group_concurrent_contiguous( + prefill_kv_indices, dst_kv_indices + ) + + logger.debug(f"sending kvcache to {peer_name} with notif {notif}") + # Make descs + num_layers = len(self.kv_args.kv_data_ptrs) + src_addrs = [] + dst_addrs = [] + for layer_id in range(num_layers): + src_ptr = self.kv_args.kv_data_ptrs[layer_id] + dst_ptr = dst_kv_ptrs[layer_id] + item_len = self.kv_args.kv_item_lens[layer_id] + + for prefill_index, decode_index in zip(prefill_kv_blocks, dst_kv_blocks): + src_addr = src_ptr + int(prefill_index[0]) * item_len + dst_addr = dst_ptr + int(decode_index[0]) * item_len + length = item_len * len(prefill_index) + src_addrs.append((src_addr, length, self.kv_args.gpu_id)) + dst_addrs.append((dst_addr, length, dst_gpu_id)) + + logger.debug( + f"len(src_addrs): before group: {len(prefill_kv_indices)}, after group: {len(src_addrs)}" + ) + src_descs = self.agent.get_xfer_descs(src_addrs, "VRAM") + dst_descs = self.agent.get_xfer_descs(dst_addrs, "VRAM") + # Transfer data + xfer_handle = self.agent.initialize_xfer( + "WRITE", + src_descs, + dst_descs, + peer_name, + notif.encode("ascii"), # type: ignore + ) + if not xfer_handle: + raise Exception("KVSender failed to create transfer") + state = self.agent.transfer(xfer_handle) + if state == "ERR": + raise Exception("KVSender failed to post transfer") + return xfer_handle + + def send_kvcache_slice( + self, + peer_name: str, + prefill_kv_indices: npt.NDArray[np.int32], + dst_kv_ptrs: list[int], + dst_kv_indices: npt.NDArray[np.int32], + dst_gpu_id: int, + notif: str, + prefill_tp_size: int, + decode_tp_size: int, + decode_tp_rank: int, + dst_kv_item_len: int, + ): + # Get configuration from kv_args + local_tp_rank_in_group = self.kv_args.engine_rank % prefill_tp_size + dst_tp_rank_in_group = decode_tp_rank % decode_tp_size + num_kv_heads = self.kv_args.kv_head_num + + # Calculate head distribution + src_heads_per_rank = num_kv_heads + dst_heads_per_rank = num_kv_heads * prefill_tp_size // decode_tp_size + + src_kv_item_len = self.kv_args.kv_item_lens[0] + page_size = self.kv_args.page_size + + bytes_per_head_slice_to_send = ( + dst_kv_item_len // page_size // dst_heads_per_rank + ) + + # Determine which heads to send + if prefill_tp_size > decode_tp_size: + # Multiple prefill ranks to one decode rank + src_head_start_offset = 0 + num_heads_to_send = src_heads_per_rank + dst_head_start_offset = local_tp_rank_in_group * src_heads_per_rank + else: + # Send KVCache from 1 prefill instance to multiple decode instances + src_head_start_offset = ( + dst_tp_rank_in_group * dst_heads_per_rank + ) % src_heads_per_rank + num_heads_to_send = dst_heads_per_rank + dst_head_start_offset = 0 + + # Create transfer descriptors + src_addrs = [] + dst_addrs = [] + + bytes_per_token_on_prefill = src_kv_item_len // page_size + bytes_per_token_on_decode = dst_kv_item_len // page_size + + num_kv_layers = len(self.kv_args.kv_data_ptrs) // 2 + src_k_ptrs = self.kv_args.kv_data_ptrs[:num_kv_layers] + src_v_ptrs = self.kv_args.kv_data_ptrs[num_kv_layers:] + dst_k_ptrs = dst_kv_ptrs[0 : len(src_k_ptrs)] + dst_v_ptrs = dst_kv_ptrs[num_kv_layers : num_kv_layers + len(src_v_ptrs)] + + # Calculate precise byte offset and length for the sub-slice within the token + src_head_slice_offset = src_head_start_offset * bytes_per_head_slice_to_send + dst_head_slice_offset = dst_head_start_offset * bytes_per_head_slice_to_send + heads_bytes_per_token_to_send = num_heads_to_send * bytes_per_head_slice_to_send + + src_dst_ptr_pairs = [ + ( + src_k_ptrs[layer_id], + dst_k_ptrs[layer_id], + ) + for layer_id in range(len(src_k_ptrs)) + ] + [ + ( + src_v_ptrs[layer_id], + dst_v_ptrs[layer_id], + ) + for layer_id in range(len(src_v_ptrs)) + ] + + src_addrs = [] + dst_addrs = [] + + # Calculate strides for a single token slot + bytes_per_token_on_prefill = src_kv_item_len // page_size + bytes_per_token_on_decode = dst_kv_item_len // page_size + + for src_ptr, dst_ptr in src_dst_ptr_pairs: + for i in range(len(prefill_kv_indices)): + prefill_page_idx = int(prefill_kv_indices[i]) + decode_page_idx = int(dst_kv_indices[i]) + + # Get the starting addresses for the current src and dst pages + src_page_start_addr = src_ptr + prefill_page_idx * src_kv_item_len + dst_page_start_addr = dst_ptr + decode_page_idx * dst_kv_item_len + + # Iterate through each valid token slot within the current page + for token_slot_in_page in range(page_size): + # Calculate the start address of the current token slot + src_token_slot_start_addr = ( + src_page_start_addr + + token_slot_in_page * bytes_per_token_on_prefill + ) + dst_token_slot_start_addr = ( + dst_page_start_addr + + token_slot_in_page * bytes_per_token_on_decode + ) + + # Calculate final src and dst addresses by applying head-slice offsets + src_slice_addr = src_token_slot_start_addr + src_head_slice_offset + dst_slice_addr = dst_token_slot_start_addr + dst_head_slice_offset + + src_addrs.append( + ( + src_slice_addr, + heads_bytes_per_token_to_send, + self.kv_args.gpu_id, + ) + ) + dst_addrs.append( + (dst_slice_addr, heads_bytes_per_token_to_send, dst_gpu_id) + ) + + # Use NIXL agent for transfer + src_descs = self.agent.get_xfer_descs(src_addrs, "VRAM") + dst_descs = self.agent.get_xfer_descs(dst_addrs, "VRAM") + + xfer_handle = self.agent.initialize_xfer( + "WRITE", src_descs, dst_descs, peer_name, notif.encode("ascii") + ) + if not xfer_handle: + raise Exception("Failed to create sliced KV transfer") + + state = self.agent.transfer(xfer_handle) + if state == "ERR": + raise Exception("Failed to post sliced KV transfer") + + return xfer_handle + + def send_aux( + self, + peer_name: str, + prefill_aux_index: int, + dst_aux_ptrs: list[int], + dst_aux_index: int, + notif: str, + ): + # Make descs + aux_item_len = self.kv_args.aux_item_lens[0] + prefill_aux_addr = ( + self.kv_args.aux_data_ptrs[0] + prefill_aux_index * aux_item_len + ) + decode_aux_addr = dst_aux_ptrs[0] + dst_aux_index * aux_item_len + src_addrs = [(prefill_aux_addr, aux_item_len, 0)] + dst_addrs = [(decode_aux_addr, aux_item_len, 0)] + src_descs = self.agent.get_xfer_descs(src_addrs, "DRAM") + dst_descs = self.agent.get_xfer_descs(dst_addrs, "DRAM") + # Transfer data + xfer_handle = self.agent.initialize_xfer( + "WRITE", + src_descs, + dst_descs, + peer_name, + notif.encode("ascii"), # type: ignore + ) + if not xfer_handle: + raise Exception("KVSender failed to create transfer") + state = self.agent.transfer(xfer_handle) + if state == "ERR": + raise Exception("KVSender failed to post transfer") + return xfer_handle + + def add_transfer_request( + self, + bootstrap_room: int, + kv_indices: npt.NDArray[np.int32], + index_slice: slice, + is_last: bool, + chunk_id: int, + aux_index: Optional[int] = None, + ): + assert self.disaggregation_mode == DisaggregationMode.PREFILL + assert not is_last or (is_last and aux_index is not None) + + reqs_to_be_processed = self.transfer_infos[bootstrap_room].values() + handles = [] + for req in reqs_to_be_processed: + assert bootstrap_room == req.room + if req.is_dummy(): + continue + + chunked_dst_kv_indice = req.dst_kv_indices[index_slice] + assert len(chunked_dst_kv_indice) == len(kv_indices) + assert req.agent_name in self.decode_kv_args_table + + notif = "_".join([str(req.room), "kv", str(chunk_id), str(int(is_last))]) + decode_tp_size = self.decode_kv_args_table[req.agent_name].decode_tp_size + + if decode_tp_size == self.tp_size: + kv_xfer_handle = self.send_kvcache( + req.agent_name, + kv_indices, + self.decode_kv_args_table[req.agent_name].dst_kv_ptrs, + chunked_dst_kv_indice, + self.decode_kv_args_table[req.agent_name].gpu_id, + notif, + ) + else: + kv_xfer_handle = self.send_kvcache_slice( + req.agent_name, + kv_indices, + self.decode_kv_args_table[req.agent_name].dst_kv_ptrs, + chunked_dst_kv_indice, + self.decode_kv_args_table[req.agent_name].gpu_id, + notif, + prefill_tp_size=self.tp_size, + decode_tp_size=decode_tp_size, + decode_tp_rank=self.decode_kv_args_table[ + req.agent_name + ].decode_tp_rank, + dst_kv_item_len=self.decode_kv_args_table[ + req.agent_name + ].dst_kv_item_len, + ) + + handles.append(kv_xfer_handle) + # Only the last chunk we need to send the aux data. + if is_last: + assert aux_index is not None + aux_xfer_handle = self.send_aux( + req.agent_name, + aux_index, + self.decode_kv_args_table[req.agent_name].dst_aux_ptrs, + req.dst_aux_index, + str(req.room) + "_aux", + ) + handles.append(aux_xfer_handle) + if is_last: + del self.transfer_infos[bootstrap_room] + return handles + + def update_transfer_status(self): + # Process notifications from received transfers. + notif_map = self.agent.get_new_notifs() + for peer_name, messages in notif_map.items(): + # We could also check that self.bootstrap_info['agent_name'] matches + # the message sender. But the bootstrap room alone should be + # sufficient to map the status. + for msg in messages: + components = msg.decode("ascii").split("_") + room = int(components[0]) + if components[1] == "kv": + chunk_id = int(components[2]) + is_last = bool(int(components[3])) + self.transfer_statuses[room].received_kvs.add(chunk_id) + if is_last: + self.transfer_statuses[room].num_kvs_expected = chunk_id + 1 + elif components[1] == "aux": + self.transfer_statuses[room].received_aux = True + + def check_transfer_done(self, room: int): + if room not in self.transfer_statuses: + return False + return self.transfer_statuses[room].is_done() + + def _bind_server_socket(self): + self.server_socket.bind(format_tcp_address(self.local_ip, self.rank_port)) + + def _start_bootstrap_thread(self): + self._bind_server_socket() + + def bootstrap_thread(): + """This thread recvs transfer info from the decode engine""" + while True: + waiting_req_bytes = self.server_socket.recv_multipart() + logger.debug( + f"Received multipart with total byte size {sum(len(x) for x in waiting_req_bytes)}" + ) + assert ( + waiting_req_bytes[0] == GUARD + ), f"First message should be {GUARD}. Foreign traffic?" + waiting_req_bytes = waiting_req_bytes[1:] + room = waiting_req_bytes[0].decode("ascii") + agent_name = waiting_req_bytes[3].decode("ascii") + if room == "None": + # Register new peer and save KV base pointers. + self._add_remote_peer( + KVArgsRegisterInfo.from_zmq(waiting_req_bytes) + ) + logger.debug(f"Register KVArgs from {agent_name} successfully") + continue + room = int(room) + if room not in self.transfer_infos: + self.transfer_infos[room] = {} + self.transfer_infos[room][agent_name] = TransferInfo.from_zmq( + waiting_req_bytes + ) + required_dst_info_num = self.transfer_infos[room][ + agent_name + ].required_dst_info_num + logger.debug(f"got info {room=} {agent_name=} {required_dst_info_num=}") + if len(self.transfer_infos[room]) == required_dst_info_num: + logger.debug(f"{room=} is bootstrapped") + self.update_status(room, KVPoll.WaitingForInput) + + threading.Thread(target=bootstrap_thread).start() + + +class NixlKVSender(BaseKVSender): + + def __init__( + self, + mgr: NixlKVManager, + bootstrap_addr: str, + bootstrap_room: int, + dest_tp_ranks: List[int], + pp_rank: int, + ): + self.kv_mgr = mgr + self.bootstrap_room = bootstrap_room + self.aux_index = None + self.bootstrap_server_url = bootstrap_addr + self.xfer_handles = [] + self.has_sent = False + self.chunk_id = 0 + self.kv_mgr.update_status(self.bootstrap_room, KVPoll.Bootstrapping) + # inner state + self.curr_idx = 0 + + def init(self, num_kv_indices: int, aux_index: Optional[int] = None): + self.num_kv_indices = num_kv_indices + self.aux_index = aux_index + + def send( + self, + kv_indices: npt.NDArray[np.int32], + ): + index_slice = slice(self.curr_idx, self.curr_idx + len(kv_indices)) + self.curr_idx += len(kv_indices) + is_last = self.curr_idx == self.num_kv_indices + + new_xfer_handles = self.kv_mgr.add_transfer_request( + self.bootstrap_room, + kv_indices, + index_slice, + is_last, + self.chunk_id, + self.aux_index, + ) + self.xfer_handles.extend(new_xfer_handles) + self.chunk_id += 1 + if is_last: + self.has_sent = True + del self.kv_mgr.request_status[self.bootstrap_room] + + def poll(self) -> KVPoll: + if not self.has_sent: + return self.kv_mgr.check_status(self.bootstrap_room) + states = [self.kv_mgr.agent.check_xfer_state(x) for x in self.xfer_handles] + if all([x == "DONE" for x in states]): + return KVPoll.Success # type: ignore + if any([x == "ERR" for x in states]): + raise Exception("KVSender transfer encountered an error.") + return KVPoll.WaitingForInput # type: ignore + + def failure_exception(self): + raise Exception("Fake KVSender Exception") + + +class NixlKVReceiver(CommonKVReceiver): + def __init__( + self, + mgr: NixlKVManager, + bootstrap_addr: str, + bootstrap_room: Optional[int] = None, + prefill_dp_rank: Optional[int] = None, + ): + self.started_transfer = False + self.conclude_state = None + super().__init__(mgr, bootstrap_addr, bootstrap_room, prefill_dp_rank) + + def init(self, kv_indices: npt.NDArray[np.int32], aux_index: Optional[int] = None): + for bootstrap_info in self.bootstrap_infos: + logger.debug( + f"Fetched bootstrap info: {bootstrap_info} for engine rank: {self.kv_mgr.kv_args.engine_rank}" + ) + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + is_dummy = bootstrap_info["is_dummy"] + logger.debug( + f"Sending to prefill server with bootstrap room {self.bootstrap_room} {is_dummy=}" + ) + with lock: + sock.send_multipart( + [ + GUARD, + str(self.bootstrap_room).encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + self.kv_mgr.agent.name.encode("ascii"), + kv_indices.tobytes() if not is_dummy else b"", + str(aux_index).encode("ascii"), + str(self.required_dst_info_num).encode("ascii"), + ] + ) + + self.started_transfer = True + + def poll(self) -> KVPoll: + if self.conclude_state is not None: + return self.conclude_state + if not self.started_transfer: + return KVPoll.WaitingForInput # type: ignore + + self.kv_mgr.update_transfer_status() + if self.kv_mgr.check_transfer_done(self.bootstrap_room): # type: ignore + self.conclude_state = KVPoll.Success + del self.kv_mgr.transfer_statuses[self.bootstrap_room] + return KVPoll.Success # type: ignore + return KVPoll.WaitingForInput # type: ignore + + def _register_kv_args(self): + for bootstrap_info in self.bootstrap_infos: + sock, lock = self._connect_to_bootstrap_server(bootstrap_info) + packed_kv_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.kv_data_ptrs + ) + packed_aux_data_ptrs = b"".join( + struct.pack("Q", ptr) for ptr in self.kv_mgr.kv_args.aux_data_ptrs + ) + + with lock: + sock.send_multipart( + [ + GUARD, + "None".encode("ascii"), + self.kv_mgr.local_ip.encode("ascii"), + str(self.kv_mgr.rank_port).encode("ascii"), + self.kv_mgr.agent.name.encode("ascii"), + self.kv_mgr.agent.get_agent_metadata(), + packed_kv_data_ptrs, + packed_aux_data_ptrs, + str(self.kv_mgr.kv_args.gpu_id).encode("ascii"), + str(self.kv_mgr.kv_args.decode_tp_size).encode("ascii"), + str(self.kv_mgr.kv_args.engine_rank).encode("ascii"), + str(self.kv_mgr.kv_args.kv_item_lens[0]).encode("ascii"), + ] + ) + + def failure_exception(self): + raise Exception("Fake KVReceiver Exception") + + +class NixlKVBootstrapServer(CommonKVBootstrapServer): + pass diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py new file mode 100644 index 000000000..b70748250 --- /dev/null +++ b/python/sglang/srt/disaggregation/prefill.py @@ -0,0 +1,867 @@ +""" +Life cycle of a request in the prefill server + +1. Bootstrap Queue + a. Initialize a sender for each request + b. Use the queue to store requests whose bootstrap (handshake and preallocation) has not finished + c. Poll senders to check bootstrap state + d. Once bootstrap is complete, move request to Waiting Queue + +2. Waiting Queue + a. Use PrefillAdder to pop requests + b. Run forward + c. Add the request to Inflight Queue + +3. Inflight Queue + a. Poll (non-blocking) the sender of the request + b. Once the transfer has finished, return the request +""" + +from __future__ import annotations + +import logging +import threading +from collections import deque +from http import HTTPStatus +from typing import TYPE_CHECKING, List, Optional, Type + +import torch + +from sglang.srt.disaggregation.base import BaseKVManager, KVPoll +from sglang.srt.disaggregation.utils import ( + FAKE_BOOTSTRAP_HOST, + DisaggregationMode, + KVClassType, + MetadataBuffers, + ReqToMetadataIdxAllocator, + TransferBackend, + get_kv_class, + is_mla_backend, + kv_to_page_indices, + kv_to_page_num, + poll_and_all_reduce, + prepare_abort, +) +from sglang.srt.managers.schedule_batch import FINISH_LENGTH, Req, ScheduleBatch +from sglang.srt.model_executor.forward_batch_info import ForwardMode, PPProxyTensors +from sglang.srt.utils import ( + DynamicGradMode, + broadcast_pyobj, + point_to_point_pyobj, + require_mlp_sync, +) + +if TYPE_CHECKING: + from torch.distributed import ProcessGroup + + from sglang.srt.managers.scheduler import GenerationBatchResult, Scheduler + from sglang.srt.mem_cache.memory_pool import KVCache + +logger = logging.getLogger(__name__) + + +class PrefillBootstrapQueue: + """ + Store the requests in bootstrapping + """ + + def __init__( + self, + token_to_kv_pool: KVCache, + draft_token_to_kv_pool: Optional[KVCache], + req_to_metadata_buffer_idx_allocator: ReqToMetadataIdxAllocator, + metadata_buffers: MetadataBuffers, + tp_rank: int, + tp_size: int, + gpu_id: int, + bootstrap_port: int, + gloo_group: ProcessGroup, + max_total_num_tokens: int, + decode_tp_size: int, + decode_dp_size: int, + scheduler: Scheduler, + pp_rank: int, + pp_size: int, + transfer_backend: TransferBackend, + ): + self.token_to_kv_pool = token_to_kv_pool + self.draft_token_to_kv_pool = draft_token_to_kv_pool + self.is_mla_backend = is_mla_backend(token_to_kv_pool) + self.metadata_buffers = metadata_buffers + self.req_to_metadata_buffer_idx_allocator = req_to_metadata_buffer_idx_allocator + self.tp_rank = tp_rank + self.tp_size = tp_size + self.decode_tp_size = decode_tp_size + self.decode_dp_size = decode_dp_size + self.pp_rank = pp_rank + self.pp_size = pp_size + self.gpu_id = gpu_id + self.bootstrap_port = bootstrap_port + self.queue: List[Req] = [] + self.gloo_group = gloo_group + self.max_total_num_tokens = max_total_num_tokens + self.scheduler = scheduler + self.transfer_backend = transfer_backend + self.kv_manager = self._init_kv_manager() + + def _init_kv_manager(self) -> BaseKVManager: + kv_args_class = get_kv_class(self.transfer_backend, KVClassType.KVARGS) + kv_args = kv_args_class() + kv_args.engine_rank = self.tp_rank + kv_args.pp_rank = self.pp_rank + kv_args.system_dp_rank = self.scheduler.dp_rank + kv_args.decode_tp_size = self.decode_tp_size // self.decode_dp_size + kv_args.prefill_pp_size = self.pp_size + kv_args.prefill_start_layer = self.token_to_kv_pool.start_layer + kv_data_ptrs, kv_data_lens, kv_item_lens = ( + self.token_to_kv_pool.get_contiguous_buf_infos() + ) + + if self.draft_token_to_kv_pool is not None: + # We should also transfer draft model kv cache. The indices are + # always shared with a target model. + draft_kv_data_ptrs, draft_kv_data_lens, draft_kv_item_lens = ( + self.draft_token_to_kv_pool.get_contiguous_buf_infos() + ) + kv_data_ptrs += draft_kv_data_ptrs + kv_data_lens += draft_kv_data_lens + kv_item_lens += draft_kv_item_lens + + kv_args.kv_data_ptrs = kv_data_ptrs + kv_args.kv_data_lens = kv_data_lens + kv_args.kv_item_lens = kv_item_lens + if not self.is_mla_backend: + kv_args.kv_head_num = self.token_to_kv_pool.head_num + kv_args.page_size = self.token_to_kv_pool.page_size + + kv_args.aux_data_ptrs, kv_args.aux_data_lens, kv_args.aux_item_lens = ( + self.metadata_buffers.get_buf_infos() + ) + kv_args.ib_device = self.scheduler.server_args.disaggregation_ib_device + kv_args.gpu_id = self.scheduler.gpu_id + + kv_manager_class: Type[BaseKVManager] = get_kv_class( + self.transfer_backend, KVClassType.MANAGER + ) + kv_manager: BaseKVManager = kv_manager_class( + kv_args, + DisaggregationMode.PREFILL, + self.scheduler.server_args, + self.is_mla_backend, + ) + return kv_manager + + def add(self, req: Req, num_kv_heads: int) -> None: + if self._check_if_req_exceed_kv_capacity(req): + return + + if req.bootstrap_host == FAKE_BOOTSTRAP_HOST: + kv_sender_class = get_kv_class(TransferBackend.FAKE, KVClassType.SENDER) + else: + kv_sender_class = get_kv_class(self.transfer_backend, KVClassType.SENDER) + + dest_tp_ranks = [self.tp_rank] + + req.disagg_kv_sender = kv_sender_class( + mgr=self.kv_manager, + bootstrap_addr=f"{req.bootstrap_host}:{self.bootstrap_port}", + bootstrap_room=req.bootstrap_room, + dest_tp_ranks=dest_tp_ranks, + pp_rank=self.pp_rank, + ) + self._process_req(req) + self.queue.append(req) + + def extend(self, reqs: List[Req], num_kv_heads: int) -> None: + for req in reqs: + self.add(req, num_kv_heads) + + def _check_if_req_exceed_kv_capacity(self, req: Req) -> bool: + if len(req.origin_input_ids) > self.max_total_num_tokens: + message = f"Request {req.rid} exceeds the maximum number of tokens: {len(req.origin_input_ids)} > {self.max_total_num_tokens}" + logger.error(message) + prepare_abort(req, message, status_code=HTTPStatus.BAD_REQUEST) + self.scheduler.stream_output([req], req.return_logprob) + return True + return False + + def _process_req(self, req: Req) -> None: + """ + Set max_new_tokens = 1, so PrefillAdder memory estimation is accurate + """ + req.sampling_params.max_new_tokens = 1 + + def pop_bootstrapped( + self, + return_failed_reqs: bool = False, + rids_to_check: Optional[List[str]] = None, + ) -> List[Req]: + """ + pop the reqs which has finished bootstrapping + + return_failed_reqs: For PP, on rank 0, also return the failed reqs to notify the next rank + rids_to_check: For PP, on rank > 0, check the rids from the previous rank has consensus with the current rank. + """ + + bootstrapped_reqs = [] + failed_reqs = [] + indices_to_remove = set() + + if len(self.queue) == 0: + if return_failed_reqs is False: + return [] + else: + return [], [] + + polls = poll_and_all_reduce( + [req.disagg_kv_sender for req in self.queue], self.gloo_group + ) + + for i, (req, poll) in enumerate(zip(self.queue, polls)): + if rids_to_check is not None: + # if req not in reqs_info_to_check, skip + if req.rid not in rids_to_check: + continue + # Either waiting for input or failed + assert poll == KVPoll.WaitingForInput or poll == KVPoll.Failed + + if poll == KVPoll.Bootstrapping: + continue + elif poll == KVPoll.Failed: + error_message = f"Prefill bootstrap failed for request rank={self.tp_rank} {req.rid=} {req.bootstrap_room=}" + try: + req.disagg_kv_sender.failure_exception() + except Exception as e: + error_message += f" with exception {e}" + logger.error(error_message) + prepare_abort( + req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR + ) + self.scheduler.stream_output([req], req.return_logprob) + indices_to_remove.add(i) + failed_reqs.append(req) + if self.scheduler.enable_metrics: + self.scheduler.metrics_collector.increment_bootstrap_failed_reqs() + continue + + # KV.WaitingForInput - init here + num_kv_indices = len(req.origin_input_ids) + if self.req_to_metadata_buffer_idx_allocator.available_size() == 0: + break + + req.metadata_buffer_index = ( + self.req_to_metadata_buffer_idx_allocator.alloc() + ) + assert req.metadata_buffer_index is not None + + num_pages = kv_to_page_num(num_kv_indices, self.token_to_kv_pool.page_size) + req.disagg_kv_sender.init(num_pages, req.metadata_buffer_index) + bootstrapped_reqs.append(req) + indices_to_remove.add(i) + + self.queue = [ + entry for i, entry in enumerate(self.queue) if i not in indices_to_remove + ] + + if return_failed_reqs is False: + return bootstrapped_reqs + else: + return bootstrapped_reqs, failed_reqs + + +class SchedulerDisaggregationPrefillMixin: + """ + Mixin for Scheduler to handle disaggregation prefill + """ + + @torch.no_grad() + def event_loop_normal_disagg_prefill(self: Scheduler) -> None: + """A normal scheduler loop for prefill worker in disaggregation mode.""" + + while True: + recv_reqs = self.recv_requests() + self.process_input_requests(recv_reqs) + self.waiting_queue.extend( + self.disagg_prefill_bootstrap_queue.pop_bootstrapped() + ) + self.process_prefill_chunk() + batch = self.get_new_batch_prefill() + + if require_mlp_sync(self.server_args): + batch = self.prepare_mlp_sync_batch(batch) + self.cur_batch = batch + + if batch: + result = self.run_batch(batch) + self.process_batch_result_disagg_prefill(batch, result) + + if len(self.disagg_prefill_inflight_queue) > 0: + self.process_disagg_prefill_inflight_queue() + + if batch is None and len(self.disagg_prefill_inflight_queue) == 0: + self.self_check_during_idle() + + self.last_batch = batch + # HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it + # Otherwise, it hangs under high concurrency + self.running_batch.batch_is_full = False + + @torch.no_grad() + def event_loop_overlap_disagg_prefill(self: Scheduler) -> None: + self.result_queue = deque() + + while True: + recv_reqs = self.recv_requests() + self.process_input_requests(recv_reqs) + self.waiting_queue.extend( + self.disagg_prefill_bootstrap_queue.pop_bootstrapped() + ) + self.process_prefill_chunk() + batch = self.get_new_batch_prefill() + + if require_mlp_sync(self.server_args): + batch = self.prepare_mlp_sync_batch(batch) + self.cur_batch = batch + if batch: + result = self.run_batch(batch) + self.result_queue.append((batch.copy(), result)) + + if self.last_batch is None: + # Create a dummy first batch to start the pipeline for overlap schedule. + # It is now used for triggering the sampling_info_done event. + tmp_batch = ScheduleBatch( + reqs=None, + forward_mode=ForwardMode.DUMMY_FIRST, + next_batch_sampling_info=self.tp_worker.cur_sampling_info, + ) + self.set_next_batch_sampling_info_done(tmp_batch) + + if self.last_batch: + tmp_batch, tmp_result = self.result_queue.popleft() + tmp_batch.next_batch_sampling_info = ( + self.tp_worker.cur_sampling_info if batch else None + ) + self.process_batch_result_disagg_prefill(tmp_batch, tmp_result) + + if len(self.disagg_prefill_inflight_queue) > 0: + self.process_disagg_prefill_inflight_queue() + + if batch is None and len(self.disagg_prefill_inflight_queue) == 0: + self.self_check_during_idle() + + self.last_batch = batch + # HACK (byronhsu): reset the batch_is_full flag because we never enter update_running_batch which resets it + # Otherwise, it hangs under high concurrency + self.running_batch.batch_is_full = False + + def process_batch_result_disagg_prefill( + self: Scheduler, + batch: ScheduleBatch, + result: GenerationBatchResult, + launch_done: Optional[threading.Event] = None, + ) -> None: + """ + Transfer kv for prefill completed requests and add it into disagg_prefill_inflight_queue + Adapted from process_batch_result_prefill + """ + ( + logits_output, + next_token_ids, + extend_input_len_per_req, + extend_logprob_start_len_per_req, + ) = ( + result.logits_output, + result.next_token_ids, + result.extend_input_len_per_req, + result.extend_logprob_start_len_per_req, + ) + + logprob_pt = 0 + # Transfer kv for prefill completed requests and add it into disagg_prefill_inflight_queue + if self.enable_overlap: + # wait + logits_output, next_token_ids, _ = self.tp_worker.resolve_last_batch_result( + launch_done + ) + else: + next_token_ids = result.next_token_ids.tolist() + if batch.return_logprob: + if logits_output.next_token_logprobs is not None: + logits_output.next_token_logprobs = ( + logits_output.next_token_logprobs.tolist() + ) + if logits_output.input_token_logprobs is not None: + logits_output.input_token_logprobs = tuple( + logits_output.input_token_logprobs.tolist() + ) + + hidden_state_offset = 0 + for i, (req, next_token_id) in enumerate( + zip(batch.reqs, next_token_ids, strict=True) + ): + req: Req + if req.is_chunked <= 0: + # There is no output_ids for prefill + req.output_ids.append(next_token_id) + self.tree_cache.cache_unfinished_req(req) # update the tree and lock + self.disagg_prefill_inflight_queue.append(req) + if ( + logits_output is not None + and logits_output.hidden_states is not None + ): + last_hidden_index = ( + hidden_state_offset + extend_input_len_per_req[i] - 1 + ) + req.hidden_states_tensor = ( + logits_output.hidden_states[last_hidden_index].cpu().clone() + ) + hidden_state_offset += extend_input_len_per_req[i] + else: + req.hidden_states_tensor = None + if req.return_logprob: + assert extend_logprob_start_len_per_req is not None + assert extend_input_len_per_req is not None + extend_logprob_start_len = extend_logprob_start_len_per_req[i] + extend_input_len = extend_input_len_per_req[i] + num_input_logprobs = extend_input_len - extend_logprob_start_len + self.add_logprob_return_values( + i, + req, + logprob_pt, + next_token_ids, + num_input_logprobs, + logits_output, + ) + logprob_pt += num_input_logprobs + self.send_kv_chunk(req, last_chunk=True) + + if req.grammar is not None: + # FIXME: this try-except block is for handling unexpected xgrammar issue. + try: + req.grammar.accept_token(next_token_id) + except ValueError as e: + # Grammar accept_token can raise ValueError if the token is not in the grammar. + # This can happen if the grammar is not set correctly or the token is invalid. + error_message = f"Grammar accept_token failed for req {req.rid} with token {next_token_id}: {e}" + self.tree_cache.cache_finished_req(req) + prepare_abort( + req, + error_message, + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + ) + req.grammar.finished = req.finished() + else: + # being chunked reqs' prefill is not finished + req.is_chunked -= 1 + + if req.return_logprob: + extend_logprob_start_len = extend_logprob_start_len_per_req[i] + extend_input_len = extend_input_len_per_req[i] + if extend_logprob_start_len < extend_input_len: + # Update input logprobs. + num_input_logprobs = extend_input_len - extend_logprob_start_len + self.add_input_logprob_return_values( + i, + req, + logits_output, + logprob_pt, + num_input_logprobs, + last_prefill_chunk=False, + ) + logprob_pt += num_input_logprobs + + if self.enable_overlap: + self.send_kv_chunk(req, last_chunk=False, end_idx=req.tmp_end_idx) + + # We need to remove the sync in the following function for overlap schedule. + self.set_next_batch_sampling_info_done(batch) + self.maybe_send_health_check_signal() + + def process_disagg_prefill_inflight_queue( + self: Scheduler, rids_to_check: Optional[List[str]] = None + ) -> List[Req]: + """ + Poll the requests in the middle of transfer. If done, return the request. + rids_to_check: For PP, on rank > 0, check the rids from the previous rank has consensus with the current rank. + """ + if len(self.disagg_prefill_inflight_queue) == 0: + return [] + + done_reqs = [] + + polls = poll_and_all_reduce( + [req.disagg_kv_sender for req in self.disagg_prefill_inflight_queue], + self.attn_tp_cpu_group, + ) + + undone_reqs: List[Req] = [] + # Check .poll() for the reqs in disagg_prefill_inflight_queue. If Success, respond to the client and remove it from the queue + for req, poll in zip(self.disagg_prefill_inflight_queue, polls): + + if rids_to_check is not None: + if req.rid not in rids_to_check: + undone_reqs.append(req) + continue + + assert poll == KVPoll.Success or poll == KVPoll.Failed + + if poll in [KVPoll.WaitingForInput, KVPoll.Transferring]: + undone_reqs.append(req) + elif poll == KVPoll.Success: # transfer done + self.tree_cache.cache_finished_req(req) # unlock the tree + req.finished_reason = FINISH_LENGTH(length=0) + # FIXME: clean up req's data in transfer engine + if hasattr(req.disagg_kv_sender, "clear"): + req.disagg_kv_sender.clear() + done_reqs.append(req) + elif poll == KVPoll.Failed: + error_message = f"Prefill transfer failed for request rank={self.tp_rank} {req.rid=} {req.bootstrap_room=}" + try: + req.disagg_kv_sender.failure_exception() + except Exception as e: + error_message += f" with exception {e}" + logger.warning(error_message) + self.tree_cache.cache_finished_req(req) # unlock the tree + prepare_abort( + req, error_message, status_code=HTTPStatus.INTERNAL_SERVER_ERROR + ) + done_reqs.append(req) + if self.enable_metrics: + self.metrics_collector.increment_transfer_failed_reqs() + else: + assert False, f"Unexpected polling state {poll=}" + + # Stream requests which have finished transfer + self.stream_output( + done_reqs, + any(req.return_logprob for req in done_reqs), + None, + ) + for req in done_reqs: + req: Req + self.req_to_metadata_buffer_idx_allocator.free(req.metadata_buffer_index) + req.metadata_buffer_index = -1 + + self.disagg_prefill_inflight_queue = undone_reqs + + return done_reqs + + def get_transferred_rids(self: Scheduler) -> List[str]: + """ + Used by PP, get the transferred rids but **do not pop** + """ + polls = poll_and_all_reduce( + [req.disagg_kv_sender for req in self.disagg_prefill_inflight_queue], + self.tp_worker.get_tp_group().cpu_group, + ) + + transferred_rids: List[str] = [] + + for req, poll in zip(self.disagg_prefill_inflight_queue, polls): + if poll == KVPoll.Success or poll == KVPoll.Failed: + transferred_rids.append(req.rid) + + return transferred_rids + + def process_prefill_chunk(self: Scheduler) -> None: + if self.last_batch and self.last_batch.forward_mode.is_extend(): + if self.chunked_req: + # Move the chunked request out of the batch so that we can merge + # only finished requests to running_batch. + self.last_batch.filter_batch(chunked_req_to_exclude=self.chunked_req) + self.tree_cache.cache_unfinished_req(self.chunked_req, chunked=True) + if self.enable_overlap: + # Delay KV transfer to process_batch_result_disagg_prefill when overlap is enabled to ensure results are resolved + self.chunked_req.tmp_end_idx = min( + len(self.chunked_req.fill_ids), + len(self.chunked_req.origin_input_ids), + ) + else: + self.send_kv_chunk(self.chunked_req) + # chunked request keeps its rid but will get a new req_pool_idx + self.req_to_token_pool.free(self.chunked_req.req_pool_idx) + self.running_batch.batch_is_full = False + + def send_kv_chunk( + self: Scheduler, + req: Req, + last_chunk: bool = False, + end_idx: Optional[int] = None, + ) -> None: + """ + Send a prefilled chunk to the decode server + """ + page_size = self.token_to_kv_pool_allocator.page_size + start_idx = req.start_send_idx + end_idx = ( + end_idx + if end_idx is not None + else min(len(req.fill_ids), len(req.origin_input_ids)) + ) + + if not last_chunk: + # if not the last chunk and the last page is partial, delay the last partial page to the next send + end_idx = end_idx - end_idx % page_size + + kv_indices = ( + self.req_to_token_pool.req_to_token[req.req_pool_idx, start_idx:end_idx] + .cpu() + .numpy() + ) + req.start_send_idx = end_idx + if last_chunk: + self.disagg_metadata_buffers.set_buf(req) + page_indices = kv_to_page_indices(kv_indices, page_size) + if len(page_indices) == 0: + logger.info( + f"Skip sending kv chunk for request {req.rid=} {req.bootstrap_room=} because page_indices is empty" + ) + return + req.disagg_kv_sender.send(page_indices) + + # PP + @DynamicGradMode() + def event_loop_pp_disagg_prefill(self: Scheduler): + """ + An event loop for the prefill server in pipeline parallelism. + + Rules: + 1. Each stage runs in the same order and is notified by the previous stage. + 2. Each send/recv operation is blocking and matched by the neighboring stage. + + Regular Schedule: + ==================================================================== + Stage i | Stage i+1 + send ith req | recv ith req + send ith proxy | recv ith proxy + send prev (i+1)th carry | recv prev (i+1)th carry + ==================================================================== + + Prefill Server Schedule: + ==================================================================== + Stage i | Stage i+1 + send ith req | recv ith req + send ith bootstrap req | recv ith bootstrap req + send ith transferred req | recv ith transferred req + send ith proxy | recv ith proxy + send prev (i+1)th carry | recv prev (i+1)th carry + send prev (i+1)th release req | recv prev (i+1)th release req + ==================================================================== + + There are two additional elements compared to the regular schedule: + + 1. Bootstrap Requests: + a. Instead of polling the status on the current workers, we should wait for the previous stage to notify to avoid desynchronization. + b. The first stage polls the status and propagates the bootstrapped requests down to all other stages. + c. If the first stage polls successfully, by nature, other ranks are also successful because they performed a handshake together. + + 2. Transferred Requests + Release Requests: + a. The first stage polls the transfer finished requests, performs an intersection with the next stage's finished requests, and propagates down to the last stage. + b. The last stage receives the requests that have finished transfer on all stages (consensus), then sends them to the first stage to release the memory. + c. The first stage receives the release requests, releases the memory, and then propagates the release requests down to the last stage. + """ + from sglang.srt.managers.scheduler import GenerationBatchResult + + mbs = [None] * self.pp_size + last_mbs = [None] * self.pp_size + self.running_mbs = [ + ScheduleBatch(reqs=[], batch_is_full=False) for _ in range(self.pp_size) + ] + bids = [None] * self.pp_size + pp_outputs: Optional[PPProxyTensors] = None + + # Either success or failed + bootstrapped_rids: List[str] = [] + transferred_rids: List[str] = [] + release_rids: Optional[List[str]] = None + + # transferred microbatch + tmbs = [None] * self.pp_size + + ENABLE_RELEASE = True # For debug + + while True: + server_is_idle = True + + for mb_id in range(self.pp_size): + self.running_batch = self.running_mbs[mb_id] + self.last_batch = last_mbs[mb_id] + + recv_reqs = self.recv_requests() + + self.process_input_requests(recv_reqs) + + if self.pp_group.is_first_rank: + # First rank, pop the bootstrap reqs from the bootstrap queue + bootstrapped_reqs, failed_reqs = ( + self.disagg_prefill_bootstrap_queue.pop_bootstrapped( + return_failed_reqs=True + ) + ) + bootstrapped_rids = [req.rid for req in bootstrapped_reqs] + [ + req.rid for req in failed_reqs + ] + self.waiting_queue.extend(bootstrapped_reqs) + else: + # Other ranks, receive the bootstrap reqs info from the previous rank and ensure the consensus + bootstrapped_rids = self.recv_pyobj_from_prev_stage() + bootstrapped_reqs = ( + self.disagg_prefill_bootstrap_queue.pop_bootstrapped( + rids_to_check=bootstrapped_rids + ) + ) + self.waiting_queue.extend(bootstrapped_reqs) + + if self.pp_group.is_first_rank: + transferred_rids = self.get_transferred_rids() + # if other ranks, + else: + # 1. recv previous stage's transferred reqs info + prev_transferred_rids = self.recv_pyobj_from_prev_stage() + # 2. get the current stage's transferred reqs info + curr_transferred_rids = self.get_transferred_rids() + # 3. new consensus rids = intersection(previous consensus rids, transfer finished rids) + transferred_rids = list( + set(prev_transferred_rids) & set(curr_transferred_rids) + ) + + tmbs[mb_id] = transferred_rids + + self.process_prefill_chunk() + mbs[mb_id] = self.get_new_batch_prefill() + self.running_mbs[mb_id] = self.running_batch + + self.cur_batch = mbs[mb_id] + if self.cur_batch: + server_is_idle = False + result = self.run_batch(self.cur_batch) + + # send the outputs to the next step + if self.pp_group.is_last_rank: + if self.cur_batch: + next_token_ids, bids[mb_id] = ( + result.next_token_ids, + result.bid, + ) + pp_outputs = PPProxyTensors( + { + "next_token_ids": next_token_ids, + } + ) + # send the output from the last round to let the next stage worker run post processing + self.pp_group.send_tensor_dict( + pp_outputs.tensors, + all_gather_group=self.attn_tp_group, + ) + + if ENABLE_RELEASE: + if self.pp_group.is_last_rank: + # At the last stage, all stages has reached the consensus to release memory for transferred_rids + release_rids = transferred_rids + # send to the first rank + self.send_pyobj_to_next_stage(release_rids) + + # receive outputs and post-process (filter finished reqs) the coming microbatch + next_mb_id = (mb_id + 1) % self.pp_size + next_pp_outputs = None + next_release_rids = None + + if mbs[next_mb_id] is not None: + next_pp_outputs: Optional[PPProxyTensors] = PPProxyTensors( + self.pp_group.recv_tensor_dict( + all_gather_group=self.attn_tp_group + ) + ) + mbs[next_mb_id].output_ids = next_pp_outputs["next_token_ids"] + output_result = GenerationBatchResult( + logits_output=None, + pp_hidden_states_proxy_tensors=None, + next_token_ids=next_pp_outputs["next_token_ids"], + extend_input_len_per_req=None, + extend_logprob_start_len_per_req=None, + bid=bids[next_mb_id], + can_run_cuda_graph=result.can_run_cuda_graph, + ) + self.process_batch_result_disagg_prefill( + mbs[next_mb_id], output_result + ) + + last_mbs[next_mb_id] = mbs[next_mb_id] + + if ENABLE_RELEASE: + if tmbs[next_mb_id] is not None: + # recv consensus rids from the previous rank + next_release_rids = self.recv_pyobj_from_prev_stage() + self.process_disagg_prefill_inflight_queue(next_release_rids) + + # carry the outputs to the next stage + if not self.pp_group.is_last_rank: + if self.cur_batch: + bids[mb_id] = result.bid + if pp_outputs: + # send the outputs from the last round to let the next stage worker run post processing + self.pp_group.send_tensor_dict( + pp_outputs.tensors, + all_gather_group=self.attn_tp_group, + ) + if ENABLE_RELEASE: + if release_rids is not None: + self.send_pyobj_to_next_stage(release_rids) + + if not self.pp_group.is_last_rank: + # send out reqs to the next stage + self.send_pyobj_to_next_stage(recv_reqs) + self.send_pyobj_to_next_stage(bootstrapped_rids) + self.send_pyobj_to_next_stage(transferred_rids) + + # send out proxy tensors to the next stage + if self.cur_batch: + self.pp_group.send_tensor_dict( + result.pp_hidden_states_proxy_tensors, + all_gather_group=self.attn_tp_group, + ) + + pp_outputs = next_pp_outputs + release_rids = next_release_rids + + self.running_batch.batch_is_full = False + + if not ENABLE_RELEASE: + if len(self.disagg_prefill_inflight_queue) > 0: + self.process_disagg_prefill_inflight_queue() + + # When the server is idle, self-check and re-init some states + if server_is_idle and len(self.disagg_prefill_inflight_queue) == 0: + self.check_memory() + self.check_tree_cache() + self.new_token_ratio = self.init_new_token_ratio + + def send_pyobj_to_next_stage(self, data): + if self.attn_tp_rank == 0: + dp_offset = self.attn_dp_rank * self.attn_tp_size + point_to_point_pyobj( + data, + self.pp_rank * self.tp_size + dp_offset, + self.world_group.device_group, + self.pp_rank * self.tp_size + dp_offset, + ((self.pp_rank + 1) % self.pp_size) * self.tp_size + dp_offset, + ) + + def recv_pyobj_from_prev_stage(self): + if self.attn_tp_rank == 0: + dp_offset = self.attn_dp_rank * self.attn_tp_size + data = point_to_point_pyobj( + [], + self.pp_rank * self.tp_size + dp_offset, + self.world_group.device_group, + ((self.pp_rank - 1) % self.pp_size) * self.tp_size + dp_offset, + self.pp_rank * self.tp_size + dp_offset, + ) + else: + data = None + + if self.tp_size != 1: + data = broadcast_pyobj( + data, self.tp_group.rank, self.tp_cpu_group, src=self.tp_group.ranks[0] + ) + return data diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py new file mode 100644 index 000000000..43770e3e2 --- /dev/null +++ b/python/sglang/srt/disaggregation/utils.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import os +import random +from collections import deque +from contextlib import nullcontext +from enum import Enum +from typing import TYPE_CHECKING, List, Optional, Type, Union + +import numpy as np +import torch +import torch.distributed as dist + +from sglang.srt.utils import is_npu + +if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import Req + +######################### +# Constants & Enums +######################### +FAKE_BOOTSTRAP_HOST = "2.2.2.2" + + +class DisaggregationMode(Enum): + NULL = "null" + PREFILL = "prefill" + DECODE = "decode" + + +######################### +# Synchronization +######################### + +# env var for testing failure, convert to float explicitly +FAILURE_PROB = float(os.getenv("DISAGGREGATION_TEST_FAILURE_PROB", 0)) + + +def poll_and_all_reduce(pollers, gloo_group): + # at a certain prob, the poll is failed to simulate failure + if FAILURE_PROB > 0: + from sglang.srt.disaggregation.base import KVPoll + + polls = [ + int(KVPoll.Failed) if random.random() < FAILURE_PROB else int(poller.poll()) + for poller in pollers + ] + else: + polls = [int(poller.poll()) for poller in pollers] + tensor_to_reduce = torch.tensor(polls, dtype=torch.uint8, device="cpu") + dist.all_reduce(tensor_to_reduce, op=dist.ReduceOp.MIN, group=gloo_group) + return tensor_to_reduce.tolist() + + +######################### +# Metadata Buffers +######################### + + +class ReqToMetadataIdxAllocator: + """A memory pool that maps a request to its first output token location.""" + + def __init__( + self, + size: int, + ): + self.size = size + self.free_slots = deque(list(range(size))) + + def available_size(self): + return len(self.free_slots) + + def alloc(self) -> Optional[int]: + if len(self.free_slots) == 0: + return None + + return self.free_slots.popleft() + + def free(self, free_index: int): + self.free_slots.append(free_index) + + +class MetadataBuffers: + def __init__( + self, + size: int, + hidden_size: int, + dtype: torch.dtype, + max_top_logprobs_num: int = 128, + custom_mem_pool: torch.cuda.MemPool = None, + ): + self.custom_mem_pool = custom_mem_pool + device = "cpu" + if is_npu(): + # For ascend backend, output tokens are placed in the NPU and will be transferred by D2D channel. + device = "npu" + elif self.custom_mem_pool: + # TODO(shangming): Fix me (use 'cuda') when nvlink_transport of Mooncake is bug-free + device = "cpu" + with ( + torch.cuda.use_mem_pool(self.custom_mem_pool) + if self.custom_mem_pool + else nullcontext() + ): + # TODO: abort top_logprobs_num > 128 in PD + + # We transfer the metadata of first output token to decode + # The minimal size for RDMA is 64Bytes, so we pad it to > 64Bytes + self.output_ids = torch.zeros((size, 16), dtype=torch.int32, device=device) + + self.output_token_logprobs_val = torch.zeros( + (size, 16), dtype=torch.float32, device=device + ) + self.output_token_logprobs_idx = torch.zeros( + (size, 16), dtype=torch.int32, device=device + ) + self.output_top_logprobs_val = torch.zeros( + (size, max_top_logprobs_num), dtype=torch.float32, device=device + ) + self.output_top_logprobs_idx = torch.zeros( + (size, max_top_logprobs_num), dtype=torch.int32, device=device + ) + self.output_hidden_states = torch.zeros( + (size, hidden_size), dtype=dtype, device=device + ) + + def get_buf_infos(self): + ptrs = [ + self.output_ids.data_ptr(), + self.output_token_logprobs_val.data_ptr(), + self.output_token_logprobs_idx.data_ptr(), + self.output_top_logprobs_val.data_ptr(), + self.output_top_logprobs_idx.data_ptr(), + self.output_hidden_states.data_ptr(), + ] + data_lens = [ + self.output_ids.nbytes, + self.output_token_logprobs_val.nbytes, + self.output_token_logprobs_idx.nbytes, + self.output_top_logprobs_val.nbytes, + self.output_top_logprobs_idx.nbytes, + self.output_hidden_states.nbytes, + ] + item_lens = [ + self.output_ids[0].nbytes, + self.output_token_logprobs_val[0].nbytes, + self.output_token_logprobs_idx[0].nbytes, + self.output_top_logprobs_val[0].nbytes, + self.output_top_logprobs_idx[0].nbytes, + self.output_hidden_states[0].nbytes, + ] + return ptrs, data_lens, item_lens + + def get_buf(self, idx: int): + return ( + self.output_ids[idx], + self.output_token_logprobs_val[idx], + self.output_token_logprobs_idx[idx], + self.output_top_logprobs_val[idx], + self.output_top_logprobs_idx[idx], + self.output_hidden_states[idx], + ) + + def set_buf(self, req: Req): + + self.output_ids[req.metadata_buffer_index][0] = req.output_ids[0] + if req.return_logprob: + if req.output_token_logprobs_val: # not none or empty list + self.output_token_logprobs_val[req.metadata_buffer_index][0] = ( + req.output_token_logprobs_val[0] + ) + if req.output_token_logprobs_idx: # not none or empty list + self.output_token_logprobs_idx[req.metadata_buffer_index][0] = ( + req.output_token_logprobs_idx[0] + ) + + if req.output_top_logprobs_val: # not none or empty list + self.output_top_logprobs_val[req.metadata_buffer_index][ + : len(req.output_top_logprobs_val[0]) + ] = torch.tensor( + req.output_top_logprobs_val[0], dtype=torch.float32, device="cpu" + ) + if req.output_top_logprobs_idx: # not none or empty list + self.output_top_logprobs_idx[req.metadata_buffer_index][ + : len(req.output_top_logprobs_idx[0]) + ] = torch.tensor( + req.output_top_logprobs_idx[0], dtype=torch.int32, device="cpu" + ) + # for PD + spec decode + if req.hidden_states_tensor is not None: + self.output_hidden_states[req.metadata_buffer_index].copy_( + req.hidden_states_tensor + ) + + +######################### +# Transfer Backend +######################### + + +class TransferBackend(Enum): + MOONCAKE = "mooncake" + NIXL = "nixl" + ASCEND = "ascend" + FAKE = "fake" + + +class KVClassType(Enum): + KVARGS = "kvargs" + MANAGER = "manager" + SENDER = "sender" + RECEIVER = "receiver" + BOOTSTRAP_SERVER = "bootstrap_server" + + +def get_kv_class( + transfer_backend: TransferBackend, class_type: KVClassType +) -> Optional[Type]: + from sglang.srt.disaggregation.fake import FakeKVReceiver, FakeKVSender + + if transfer_backend == TransferBackend.MOONCAKE: + from sglang.srt.disaggregation.base import KVArgs + from sglang.srt.disaggregation.mooncake import ( + MooncakeKVBootstrapServer, + MooncakeKVManager, + MooncakeKVReceiver, + MooncakeKVSender, + ) + + class_mapping = { + KVClassType.KVARGS: KVArgs, + KVClassType.MANAGER: MooncakeKVManager, + KVClassType.SENDER: MooncakeKVSender, + KVClassType.RECEIVER: (MooncakeKVReceiver), + KVClassType.BOOTSTRAP_SERVER: MooncakeKVBootstrapServer, + } + return class_mapping.get(class_type) + elif transfer_backend == TransferBackend.ASCEND: + from sglang.srt.disaggregation.ascend import ( + AscendKVBootstrapServer, + AscendKVManager, + AscendKVReceiver, + AscendKVSender, + ) + from sglang.srt.disaggregation.base import KVArgs + + class_mapping = { + KVClassType.KVARGS: KVArgs, + KVClassType.MANAGER: AscendKVManager, + KVClassType.SENDER: AscendKVSender, + KVClassType.RECEIVER: (AscendKVReceiver), + KVClassType.BOOTSTRAP_SERVER: AscendKVBootstrapServer, + } + return class_mapping.get(class_type) + elif transfer_backend == TransferBackend.NIXL: + from sglang.srt.disaggregation.base import KVArgs + from sglang.srt.disaggregation.nixl import ( + NixlKVBootstrapServer, + NixlKVManager, + NixlKVReceiver, + NixlKVSender, + ) + + class_mapping = { + KVClassType.KVARGS: KVArgs, + KVClassType.MANAGER: NixlKVManager, + KVClassType.SENDER: NixlKVSender, + KVClassType.RECEIVER: (NixlKVReceiver), + KVClassType.BOOTSTRAP_SERVER: NixlKVBootstrapServer, + } + return class_mapping.get(class_type) + elif transfer_backend == TransferBackend.FAKE: + from sglang.srt.disaggregation.base import KVArgs + from sglang.srt.disaggregation.fake import FakeKVReceiver, FakeKVSender + + class_mapping = { + KVClassType.KVARGS: KVArgs, + KVClassType.SENDER: FakeKVSender, + KVClassType.RECEIVER: (FakeKVReceiver), + } + return class_mapping.get(class_type) + + raise ValueError(f"Unsupported transfer backend: {transfer_backend}") + + +######################### +# KV Pages +######################### + + +def kv_to_page_indices(kv_indices: np.ndarray, page_size: int): + # 1. The page is guaranteed to be full except the last page. + # 2. page index = kv_index // page_size + # The return vector is kv_indices[::page_size] // page_size + if page_size == 1: # shortcut + return kv_indices + + return kv_indices[::page_size] // page_size + + +def kv_to_page_num(num_kv_indices: int, page_size: int): + # ceil(num_kv_indices / page_size) + return (num_kv_indices + page_size - 1) // page_size + + +######################### +# Misc +######################### + + +def is_mla_backend(target_kv_pool) -> bool: + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + return isinstance(target_kv_pool, MLATokenToKVPool) + + +def prepare_abort(req: Req, error_message: str, status_code=None): + from sglang.srt.managers.schedule_batch import FINISH_ABORT + + # populate finish metadata and stream output + req.finished_reason = FINISH_ABORT(error_message, status_code) + + if req.return_logprob: + req.input_token_logprobs_val = [] + req.input_token_logprobs_idx = [] + req.input_top_logprobs_val = [] + req.input_top_logprobs_idx = [] + req.input_token_ids_logprobs_val = [] + req.input_token_ids_logprobs_idx = [] diff --git a/python/sglang/srt/distributed/__init__.py b/python/sglang/srt/distributed/__init__.py new file mode 100644 index 000000000..12f802055 --- /dev/null +++ b/python/sglang/srt/distributed/__init__.py @@ -0,0 +1,3 @@ +from sglang.srt.distributed.communication_op import * +from sglang.srt.distributed.parallel_state import * +from sglang.srt.distributed.utils import * diff --git a/python/sglang/srt/distributed/communication_op.py b/python/sglang/srt/distributed/communication_op.py new file mode 100644 index 000000000..95600edfb --- /dev/null +++ b/python/sglang/srt/distributed/communication_op.py @@ -0,0 +1,35 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/communication_op.py + +from typing import Any, Dict, Optional, Union + +import torch +import torch.distributed + +from .parallel_state import get_tp_group + + +def tensor_model_parallel_all_reduce(input_: torch.Tensor) -> torch.Tensor: + """All-reduce the input tensor across model parallel group.""" + return get_tp_group().all_reduce(input_) + + +def tensor_model_parallel_all_gather( + input_: torch.Tensor, dim: int = -1 +) -> torch.Tensor: + """All-gather the input tensor across model parallel group.""" + return get_tp_group().all_gather(input_, dim) + + +def tensor_model_parallel_gather( + input_: torch.Tensor, dst: int = 0, dim: int = -1 +) -> Optional[torch.Tensor]: + """Gather the input tensor across model parallel group.""" + return get_tp_group().gather(input_, dst, dim) + + +def broadcast_tensor_dict( + tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None, src: int = 0 +): + if not torch.distributed.is_initialized(): + return tensor_dict + return get_tp_group().broadcast_tensor_dict(tensor_dict, src) diff --git a/python/sglang/srt/distributed/device_communicators/cuda_wrapper.py b/python/sglang/srt/distributed/device_communicators/cuda_wrapper.py new file mode 100644 index 000000000..c902f3141 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/cuda_wrapper.py @@ -0,0 +1,183 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/cuda_wrapper.py + +"""This file is a pure Python wrapper for the cudart library. +It avoids the need to compile a separate shared library, and is +convenient for use when we just need to call a few functions. +""" + +import ctypes +import logging +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +# this line makes it possible to directly load `libcudart.so` using `ctypes` +import torch # noqa + +logger = logging.getLogger(__name__) + +# === export types and functions from cudart to Python === +# for the original cudart definition, please check +# https://docs.nvidia.com/cuda/cuda-runtime-api/index.html + +cudaError_t = ctypes.c_int +cudaMemcpyKind = ctypes.c_int + + +class cudaIpcMemHandle_t(ctypes.Structure): + _fields_ = [("internal", ctypes.c_byte * 128)] + + +@dataclass +class Function: + name: str + restype: Any + argtypes: List[Any] + + +def find_loaded_library(lib_name) -> Optional[str]: + """ + According to according to https://man7.org/linux/man-pages/man5/proc_pid_maps.5.html, + the file `/proc/self/maps` contains the memory maps of the process, which includes the + shared libraries loaded by the process. We can use this file to find the path of the + a loaded library. + """ # noqa + found = False + with open("/proc/self/maps") as f: + for line in f: + if lib_name in line: + found = True + break + if not found: + # the library is not loaded in the current process + return None + # if lib_name is libcudart, we need to match a line with: + # address /path/to/libcudart-hash.so.11.0 + start = line.index("/") + path = line[start:].strip() + filename = path.split("/")[-1] + assert filename.rpartition(".so")[0].startswith( + lib_name + ), f"Unexpected filename: {filename} for library {lib_name}" + return path + + +class CudaRTLibrary: + exported_functions = [ + # ​cudaError_t cudaSetDevice ( int device ) + Function("cudaSetDevice", cudaError_t, [ctypes.c_int]), + # cudaError_t cudaDeviceSynchronize ( void ) + Function("cudaDeviceSynchronize", cudaError_t, []), + # ​cudaError_t cudaDeviceReset ( void ) + Function("cudaDeviceReset", cudaError_t, []), + # const char* cudaGetErrorString ( cudaError_t error ) + Function("cudaGetErrorString", ctypes.c_char_p, [cudaError_t]), + # ​cudaError_t cudaMalloc ( void** devPtr, size_t size ) + Function( + "cudaMalloc", + cudaError_t, + [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t], + ), + # ​cudaError_t cudaFree ( void* devPtr ) + Function("cudaFree", cudaError_t, [ctypes.c_void_p]), + # ​cudaError_t cudaMemset ( void* devPtr, int value, size_t count ) + Function( + "cudaMemset", cudaError_t, [ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t] + ), + # ​cudaError_t cudaMemcpy ( void* dst, const void* src, size_t count, cudaMemcpyKind kind ) # noqa + Function( + "cudaMemcpy", + cudaError_t, + [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, cudaMemcpyKind], + ), + # cudaError_t cudaIpcGetMemHandle ( cudaIpcMemHandle_t* handle, void* devPtr ) # noqa + Function( + "cudaIpcGetMemHandle", + cudaError_t, + [ctypes.POINTER(cudaIpcMemHandle_t), ctypes.c_void_p], + ), + # ​cudaError_t cudaIpcOpenMemHandle ( void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags ) # noqa + Function( + "cudaIpcOpenMemHandle", + cudaError_t, + [ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint], + ), + ] + + # class attribute to store the mapping from the path to the library + # to avoid loading the same library multiple times + path_to_library_cache: Dict[str, Any] = {} + + # class attribute to store the mapping from library path + # to the corresponding dictionary + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {} + + def __init__(self, so_file: Optional[str] = None): + if so_file is None: + so_file = find_loaded_library("libcudart") + assert so_file is not None, "libcudart is not loaded in the current process" + if so_file not in CudaRTLibrary.path_to_library_cache: + lib = ctypes.CDLL(so_file) + CudaRTLibrary.path_to_library_cache[so_file] = lib + self.lib = CudaRTLibrary.path_to_library_cache[so_file] + + if so_file not in CudaRTLibrary.path_to_dict_mapping: + _funcs = {} + for func in CudaRTLibrary.exported_functions: + f = getattr(self.lib, func.name) + f.restype = func.restype + f.argtypes = func.argtypes + _funcs[func.name] = f + CudaRTLibrary.path_to_dict_mapping[so_file] = _funcs + self.funcs = CudaRTLibrary.path_to_dict_mapping[so_file] + + def CUDART_CHECK(self, result: cudaError_t) -> None: + if result != 0: + error_str = self.cudaGetErrorString(result) + raise RuntimeError(f"CUDART error: {error_str}") + + def cudaGetErrorString(self, error: cudaError_t) -> str: + return self.funcs["cudaGetErrorString"](error).decode("utf-8") + + def cudaSetDevice(self, device: int) -> None: + self.CUDART_CHECK(self.funcs["cudaSetDevice"](device)) + + def cudaDeviceSynchronize(self) -> None: + self.CUDART_CHECK(self.funcs["cudaDeviceSynchronize"]()) + + def cudaDeviceReset(self) -> None: + self.CUDART_CHECK(self.funcs["cudaDeviceReset"]()) + + def cudaMalloc(self, size: int) -> ctypes.c_void_p: + devPtr = ctypes.c_void_p() + self.CUDART_CHECK(self.funcs["cudaMalloc"](ctypes.byref(devPtr), size)) + return devPtr + + def cudaFree(self, devPtr: ctypes.c_void_p) -> None: + self.CUDART_CHECK(self.funcs["cudaFree"](devPtr)) + + def cudaMemset(self, devPtr: ctypes.c_void_p, value: int, count: int) -> None: + self.CUDART_CHECK(self.funcs["cudaMemset"](devPtr, value, count)) + + def cudaMemcpy( + self, dst: ctypes.c_void_p, src: ctypes.c_void_p, count: int + ) -> None: + cudaMemcpyDefault = 4 + kind = cudaMemcpyDefault + self.CUDART_CHECK(self.funcs["cudaMemcpy"](dst, src, count, kind)) + + def cudaIpcGetMemHandle(self, devPtr: ctypes.c_void_p) -> cudaIpcMemHandle_t: + handle = cudaIpcMemHandle_t() + self.CUDART_CHECK( + self.funcs["cudaIpcGetMemHandle"](ctypes.byref(handle), devPtr) + ) + return handle + + def cudaIpcOpenMemHandle(self, handle: cudaIpcMemHandle_t) -> ctypes.c_void_p: + cudaIpcMemLazyEnablePeerAccess = 1 + devPtr = ctypes.c_void_p() + self.CUDART_CHECK( + self.funcs["cudaIpcOpenMemHandle"]( + ctypes.byref(devPtr), handle, cudaIpcMemLazyEnablePeerAccess + ) + ) + return devPtr diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py new file mode 100644 index 000000000..0a506d35f --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce.py @@ -0,0 +1,421 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/custom_all_reduce.py + +import ctypes +import logging +import os +from contextlib import contextmanager +from typing import Any, List, Optional, Union + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt import _custom_ops as ops +from sglang.srt.distributed.device_communicators.cuda_wrapper import CudaRTLibrary +from sglang.srt.distributed.device_communicators.custom_all_reduce_utils import ( + gpu_p2p_access_check, + is_full_nvlink, + is_weak_contiguous, +) +from sglang.srt.distributed.parallel_state import in_the_same_node_as +from sglang.srt.utils import is_cuda, is_hip + +logger = logging.getLogger(__name__) + +_is_cuda = is_cuda() +_is_hip = is_hip() + + +try: + if ops.use_vllm_custom_allreduce and not _is_hip: + # Use vLLM custom allreduce + ops.meta_size() + else: + # Use custom allreduce from sgl kernel (ROCM and TRT-LLM) + import sgl_kernel + custom_ar = True +except Exception: + # For CPUs + custom_ar = False + +logger = logging.getLogger(__name__) + + +def _can_p2p(rank: int, world_size: int) -> bool: + # SGLANG_SKIP_P2P_CHECK can be set to False in sglang + SGLANG_SKIP_P2P_CHECK = os.getenv("SGLANG_SKIP_P2P_CHECK", "0") == "1" + for i in range(world_size): + if i == rank: + continue + if SGLANG_SKIP_P2P_CHECK: + logger.info("Skipping P2P check and trusting the driver's P2P report.") + return torch.cuda.can_device_access_peer(rank, i) + if not gpu_p2p_access_check(rank, i): + return False + return True + + +class CustomAllreduce: + _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8] + _MAX_CAR_SIZE = 8192 * 1024 + if _is_hip: + # crossover is at 16MB buffer size for ROCm + _MAX_CAR_SIZE = 2 * 8192 * 1024 + + # max_size: max supported allreduce size + def __init__( + self, + group: ProcessGroup, + device: Union[int, str, torch.device], + max_size=_MAX_CAR_SIZE, + ) -> None: + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the CustomAllreduce to. If None, + it will be bind to f"cuda:{local_rank}". + It is the caller's responsibility to make sure each communicator + is bind to a unique device, and all communicators in this group + are in the same node. + """ + self._IS_CAPTURING = False + self.disabled = True + + if not custom_ar: + # disable because of missing custom allreduce library + # e.g. in a non-cuda environment + return + + self.group = group + + assert ( + dist.get_backend(group) != dist.Backend.NCCL + ), "CustomAllreduce should be attached to a non-NCCL group." + + if not all(in_the_same_node_as(group, source_rank=0)): + # No need to initialize custom allreduce for multi-node case. + logger.warning( + "Custom allreduce is disabled because this process group" + " spans across nodes." + ) + return + + rank = dist.get_rank(group=self.group) + world_size = dist.get_world_size(group=self.group) + if world_size == 1: + # No need to initialize custom allreduce for single GPU case. + return + + if world_size not in CustomAllreduce._SUPPORTED_WORLD_SIZES: + logger.warning( + "Custom allreduce is disabled due to an unsupported world" + " size: %d. Supported world sizes: %s. To silence this " + "warning, specify disable_custom_all_reduce=True explicitly.", + world_size, + str(CustomAllreduce._SUPPORTED_WORLD_SIZES), + ) + return + + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None) + if cuda_visible_devices: + device_ids = list(map(int, cuda_visible_devices.split(","))) + else: + device_ids = list(range(torch.cuda.device_count())) + + physical_device_id = device_ids[device.index] + tensor = torch.tensor([physical_device_id], dtype=torch.int, device="cpu") + gather_list = [ + torch.tensor([0], dtype=torch.int, device="cpu") for _ in range(world_size) + ] + dist.all_gather(gather_list, tensor, group=self.group) + physical_device_ids = [t.item() for t in gather_list] + + # test nvlink first, this will filter out most of the cases + # where custom allreduce is not supported + # this checks hardware and driver support for NVLink + if _is_cuda or _is_hip: + full_nvlink = is_full_nvlink(physical_device_ids, world_size) + + if world_size > 2 and not full_nvlink: + logger.warning( + "Custom allreduce is disabled because it's not supported on" + " more than two PCIe-only GPUs. To silence this warning, " + "specify disable_custom_all_reduce=True explicitly." + ) + return + # test P2P capability, this checks software/cudaruntime support + # this is expensive to compute at the first time + # then we cache the result + # On AMD GPU, p2p is always enabled between XGMI connected GPUs + if not _is_hip and not _can_p2p(rank, world_size): + logger.warning( + "Custom allreduce is disabled because your platform lacks " + "GPU P2P capability or P2P test failed. To silence this " + "warning, specify disable_custom_all_reduce=True explicitly." + ) + return + + self.max_size = max_size + self.rank = rank + self.world_size = world_size + self.full_nvlink = full_nvlink + + if not _is_hip: + # Buffers memory are owned by this Python class and passed to C++. + # Meta data composes of two parts: meta data for synchronization and a + # temporary buffer for storing intermediate allreduce results. + self.meta_ptrs = self.create_shared_buffer( + ops.meta_size() + max_size, group=group + ) + # This is a pre-registered IPC buffer. In eager mode, input tensors + # are first copied into this buffer before allreduce is performed + self.buffer_ptrs = self.create_shared_buffer(max_size, group=group) + # This is a buffer for storing the tuples of pointers pointing to + # IPC buffers from all ranks. Each registered tuple has size of + # 8*world_size bytes where world_size is at most 8. Allocating 8MB + # is enough for 131072 such tuples. The largest model I've seen only + # needs less than 10000 of registered tuples. + self.rank_data = torch.empty( + 8 * 1024 * 1024, dtype=torch.uint8, device=self.device + ) + self._ptr = ops.init_custom_ar( + self.meta_ptrs, self.rank_data, rank, self.full_nvlink + ) + ops.register_buffer(self._ptr, self.buffer_ptrs) + else: + # meta data buffers need to be "uncached" for signal on MI200 + self.meta = ops.allocate_meta_buffer(ops.meta_size() + max_size) + self.buffer = torch.empty(max_size, dtype=torch.uint8, device=self.device) + handle = ops.get_meta_buffer_ipc_handle(self.meta) + shard_data = ( + bytes(handle), # ipc handle to base ptr + 0, # offset of base ptr + ) + handles, offsets = self._gather_ipc_meta(shard_data) + self.rank_data = torch.empty( + 8 * 1024 * 1024, dtype=torch.uint8, device=self.device + ) + self._ptr = ops.init_custom_ar( + self.meta, self.rank_data, handles, offsets, rank, self.full_nvlink + ) + self.register_buffer(self.buffer) + + self.disabled = False + + @staticmethod + def create_shared_buffer( + size_in_bytes: int, group: Optional[ProcessGroup] = None + ) -> List[int]: + """ + Creates a shared buffer and returns a list of pointers + representing the buffer on all processes in the group. + """ + lib = CudaRTLibrary() + pointer = lib.cudaMalloc(size_in_bytes) + handle = lib.cudaIpcGetMemHandle(pointer) + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + handles = [None] * world_size + dist.all_gather_object(handles, handle, group=group) + + pointers: List[int] = [] + for i, h in enumerate(handles): + if i == rank: + pointers.append(pointer.value) # type: ignore + else: + pointers.append(lib.cudaIpcOpenMemHandle(h).value) # type: ignore + + return pointers + + @staticmethod + def free_shared_buffer( + pointers: List[int], group: Optional[ProcessGroup] = None + ) -> None: + rank = dist.get_rank(group=group) + lib = CudaRTLibrary() + lib.cudaFree(ctypes.c_void_p(pointers[rank])) + + @contextmanager + def capture(self): + """ + The main responsibility of this context manager is the + `register_graph_buffers` call at the end of the context. + It records all the buffer addresses used in the CUDA graph. + """ + try: + self._IS_CAPTURING = True + yield + finally: + self._IS_CAPTURING = False + if not self.disabled: + self.register_graph_buffers() + + def _get_ipc_meta(self, inp: torch.Tensor): + # _share_cuda_() doesn't accept meta buffer not allocated from + # PyTorch cache allocator, use direct HIP call to get IPC handle + handle = ops.get_meta_buffer_ipc_handle(inp) + shard_data = ( + bytes(handle), # ipc handle to base ptr + 0, # offset of base ptr + ) + return self._gather_ipc_meta(shard_data) + + def _gather_ipc_meta(self, shard_data): + # Note: don't use `[[None]] * self.world_size` here + # because it will create a list of the same reference + all_data: List[Optional[Any]] = [[None] for i in range(self.world_size)] + all_data[self.rank][0] = shard_data + + ranks = dist.get_process_group_ranks(group=self.group) + ranks.sort() + for i, rank in enumerate(ranks): + dist.broadcast_object_list( + all_data[i], src=rank, group=self.group, device="cpu" + ) + + # we cannot directly use `dist.all_gather_object` here + # because it is incompatible with `gloo` backend under inference mode. + # see https://github.com/pytorch/pytorch/issues/126032 for details. + + handles = [] + offsets = [] + for i in range(len(all_data)): + handles.append(all_data[i][0][0]) # type: ignore + offsets.append(all_data[i][0][1]) # type: ignore + return handles, offsets + + def register_buffer(self, inp: torch.Tensor): + handles, offsets = self._get_ipc_meta(inp) + ops.register_buffer(self._ptr, inp, handles, offsets) + + def register_graph_buffers(self): + if _is_hip: + handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr) + handles, offsets = self._gather_ipc_meta((bytes(handle), offset)) + logger.info("Registering %d cuda graph addresses", len(offset)) + ops.register_graph_buffers(self._ptr, handles, offsets) + else: + handle, offset = ops.get_graph_buffer_ipc_meta(self._ptr) + logger.info("Registering %d cuda graph addresses", len(offset)) + # We cannot directly use `dist.all_gather_object` here + # because it is incompatible with `gloo` backend under inference mode. + # see https://github.com/pytorch/pytorch/issues/126032 for details. + all_data = [ + [None, None] for _ in range(dist.get_world_size(group=self.group)) + ] + all_data[self.rank] = [handle, offset] + ranks = sorted(dist.get_process_group_ranks(group=self.group)) + for i, rank in enumerate(ranks): + dist.broadcast_object_list( + all_data[i], src=rank, group=self.group, device="cpu" + ) + # Unpack list of tuples to tuple of lists. + handles = [d[0] for d in all_data] # type: ignore + offsets = [d[1] for d in all_data] # type: ignore + ops.register_graph_buffers(self._ptr, handles, offsets) + + def should_custom_ar(self, inp: torch.Tensor): + if self.disabled: + return False + inp_size = inp.numel() * inp.element_size() + # custom allreduce requires input byte size to be multiples of 16 + if inp_size % 16 != 0: + return False + if not is_weak_contiguous(inp): + return False + # for 4 or more non NVLink-capable GPUs, custom allreduce provides + # little performance improvement over NCCL. + if not _is_hip: + if self.world_size == 2 or self.full_nvlink: + return inp_size < self.max_size + return False + + if _is_hip: + if self.full_nvlink: + return inp_size < self.max_size + return False + + return False + + # all reduce, assuming inp tensor is IPC registered with register_buffer, + # or, in the context of cuda graphs, register_graph_buffers + def all_reduce_reg(self, inp: torch.Tensor, out: torch.Tensor = None): + if out is None: + out = torch.empty_like(inp) + ops.all_reduce_reg(self._ptr, inp, out) + return out + + # all reduce, assuming inp tensor is NOT IPC registered + def all_reduce_unreg(self, inp: torch.Tensor, out: torch.Tensor = None): + if out is None: + out = torch.empty_like(inp) + ops.all_reduce_unreg(self._ptr, inp, self.buffer, out) + return out + + def all_reduce( + self, + inp: torch.Tensor, + *, + out: torch.Tensor = None, + registered: bool = False, + ): + """Performs an out-of-place all reduce. + + If registered is True, this assumes inp's pointer is already + IPC-registered. Otherwise, inp is first copied into a pre-registered + buffer. + """ + if out is None: + out = torch.empty_like(inp) + if registered: + ops.all_reduce(self._ptr, inp, out, 0, 0) + else: + ops.all_reduce( + self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size + ) + return out + + def custom_all_reduce(self, input: torch.Tensor) -> Optional[torch.Tensor]: + """The main allreduce API that provides support for cuda graph.""" + # When custom allreduce is disabled, this will be None. + if self.disabled or not self.should_custom_ar(input): + return None + if self._IS_CAPTURING: + if torch.cuda.is_current_stream_capturing(): + if _is_hip: + return self.all_reduce_reg(input) + else: + return self.all_reduce(input, registered=True) + else: + # If warm up, mimic the allocation pattern since custom + # allreduce is out-of-place. + return torch.zeros_like(input) + else: + if _is_hip: + # note: outside of cuda graph context, + # custom allreduce incurs a cost of cudaMemcpy, which should + # be small(<=1% of overall latency) compared to the performance + # gains of using custom kernels + return self.all_reduce_unreg(input) + else: + return self.all_reduce(input, registered=False) + + def close(self): + if not self.disabled and self._ptr: + ops.dispose(self._ptr) + if _is_cuda: + self.free_shared_buffer(self.meta_ptrs) + self.free_shared_buffer(self.buffer_ptrs) + self._ptr = 0 + + def __del__(self): + self.close() diff --git a/python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py new file mode 100644 index 000000000..c7baac845 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/custom_all_reduce_utils.py @@ -0,0 +1,386 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/custom_all_reduce_utils.py + +import ctypes +import json +import logging +import os +import pickle +import subprocess +import sys +import tempfile +from functools import wraps +from itertools import product +from typing import Callable, Dict, List, Optional, Sequence, TypeVar + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from typing_extensions import ParamSpec + +from sglang.srt.distributed.device_communicators.cuda_wrapper import CudaRTLibrary +from sglang.srt.utils import is_cuda, is_hip + +logger = logging.getLogger(__name__) + +_is_cuda = is_cuda() +_is_hip = is_hip() + +if _is_cuda: + try: + import pynvml + except ImportError as e: + logger.warning("Failed to import pynvml with %r", e) + +if _is_hip: + try: + from amdsmi import ( + AmdSmiException, + amdsmi_get_processor_handles, + amdsmi_init, + amdsmi_shut_down, + amdsmi_topo_get_link_type, + ) + except ImportError as e: + logger.warning("Failed to import amdsmi with %r", e) + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def update_environment_variables(envs: Dict[str, str]): + for k, v in envs.items(): + if k in os.environ and os.environ[k] != v: + logger.warning( + "Overwriting environment variable %s " "from '%s' to '%s'", + k, + os.environ[k], + v, + ) + os.environ[k] = v + + +def producer( + batch_src: Sequence[int], + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices: Optional[str] = None, +): + if cuda_visible_devices is not None: + update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices}) + + lib = CudaRTLibrary() + for i in batch_src: + lib.cudaSetDevice(i) + pointer = lib.cudaMalloc(1024) + lib.cudaMemset(pointer, 1, 1024) + lib.cudaDeviceSynchronize() + handle = lib.cudaIpcGetMemHandle(pointer) + producer_queue.put(handle) + open_success = consumer_queue.get() + if open_success: + # use two queues to simulate barrier + producer_queue.put(0) + consumer_queue.get() + # check if the memory is modified + host_data = (ctypes.c_char * 1024)() + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore + for i in range(1024): + if ord(host_data[i]) != 2: + open_success = False + break + result_queue.put(open_success) + lib.cudaDeviceReset() + + +def consumer( + batch_tgt: Sequence[int], + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices: Optional[str] = None, +): + if cuda_visible_devices is not None: + update_environment_variables({"CUDA_VISIBLE_DEVICES": cuda_visible_devices}) + + lib = CudaRTLibrary() + for j in batch_tgt: + lib.cudaSetDevice(j) + handle = producer_queue.get() + open_success = False + try: + pointer = lib.cudaIpcOpenMemHandle(handle) # type: ignore + open_success = True + except RuntimeError: + # cannot error out here, because the producer process + # is still waiting for the response. + pass + consumer_queue.put(open_success) + if open_success: + # modify the memory + lib.cudaMemset(pointer, 2, 1024) + lib.cudaDeviceSynchronize() + # use two queues to simulate barrier + producer_queue.get() + consumer_queue.put(0) + # check if the memory is modified + host_data = (ctypes.c_char * 1024)() + lib.cudaMemcpy(host_data, pointer, 1024) # type: ignore + for i in range(1024): + if ord(host_data[i]) != 2: + open_success = False + break + result_queue.put(open_success) + lib.cudaDeviceReset() + + +def can_actually_p2p( + batch_src: Sequence[int], + batch_tgt: Sequence[int], +) -> Sequence[bool]: + """ + Usually, checking if P2P access is enabled can be done by + `torch.cuda.can_device_access_peer(src, tgt)`. However, sometimes + the driver might be broken, and `torch.cuda.can_device_access_peer(src, tgt)` + returns `True` even if P2P access is not actually possible. + See https://github.com/vllm-project/vllm/issues/2728 and + https://forums.developer.nvidia.com/t/direct-gpu-gpu-communication-does-not-seem-to-work-properly/283264/10 + Therefore, we have to perform a real P2P access to check if it is actually + possible. + + Note on p2p and cuda IPC: + Usually, one process uses one GPU: + GPU src --> cuda context src --> tensor src --> process src + + We need to combine p2p and cuda IPC, so that: + GPU src --> cuda context src --> tensor src --> process src + |shared| + GPU tgt --> cuda context tgt --> tensor tgt --> process tgt + That is to say, process src creates a tensor in GPU src, passes IPC handle to + process tgt, and process tgt accesses the tensor in GPU tgt. Any operation on the + tensor in process tgt will be reflected in the tensor in process src, because + they are the same memory segment. + It is important to note that process tgt accesses the tensor in GPU tgt, not + GPU src. That's why we need p2p access. + + The most time-consuming part is the process creation. To avoid creating + processes for every pair of GPUs, we use batched testing. We create two + processes for testing all pairs of GPUs in batch. The trick is to reset + the device after each test (which is not available in PyTorch). + """ # noqa + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None) + # pass the CUDA_VISIBLE_DEVICES to the child process + # to make sure they see the same set of GPUs + + # make sure the processes are spawned + smp = mp.get_context("spawn") + producer_queue = smp.Queue() + consumer_queue = smp.Queue() + result_queue = smp.Queue() + p_src = smp.Process( + target=producer, + args=( + batch_src, + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices, + ), + ) + p_tgt = smp.Process( + target=consumer, + args=( + batch_tgt, + producer_queue, + consumer_queue, + result_queue, + cuda_visible_devices, + ), + ) + p_src.start() + p_tgt.start() + p_src.join() + p_tgt.join() + assert p_src.exitcode == 0 and p_tgt.exitcode == 0 + result: List[bool] = [] + for src, tgt in zip(batch_src, batch_tgt): + a = result_queue.get() + b = result_queue.get() + if a != b: + logger.warning( + "Two processes do not agree on the P2P access" + " status on %d -> %d, treat as disabled.", + src, + tgt, + ) + result.append(False) + else: + result.append(a) + return result + + +# why do we need this cache? +# we are testing peer-to-peer (p2p) access between GPUs,across processes. +# if we test it every time, it will be very slow, because we need to create +# N * N * 2 processes, where N is the world size. This is very slow. +# to reduce the time, we use a cache file to store the p2p access status. +# the cache file is generated by the master process if it does not exist. +# then all the processes can read the cache file to check the p2p access status. +# Note that the cache file is suffixed by the CUDA_VISIBLE_DEVICES, so that we +# can have different cache files for different CUDA_VISIBLE_DEVICES settings, +# e.g. used by different vllm engines. The device id in the cache file is a +# **local** device id, i.e. from 0 to num_dev-1, where num_dev is the number +# of visible devices in the vllm engine. +_gpu_p2p_access_cache: Optional[Dict[str, bool]] = None + + +def gpu_p2p_access_check(src: int, tgt: int) -> bool: + """Check if GPU src can access GPU tgt.""" + + # if the cache variable is already calculated, + # read from the cache instead of checking it again + global _gpu_p2p_access_cache + if _gpu_p2p_access_cache is not None: + return _gpu_p2p_access_cache[f"{src}->{tgt}"] + + is_distributed = dist.is_initialized() + + num_dev = torch.cuda.device_count() + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None) + if cuda_visible_devices is None: + cuda_visible_devices = ",".join(str(i) for i in range(num_dev)) + + # VLLM_CACHE_ROOT -> SGLANG_CACHE_ROOT + # "~/.cache/vllm" -> "~/.cache/sglang" + SGLANG_CACHE_ROOT = os.path.expanduser("~/.cache/sglang") + path = os.path.join( + SGLANG_CACHE_ROOT, f"gpu_p2p_access_cache_for_{cuda_visible_devices}.json" + ) + os.makedirs(os.path.dirname(path), exist_ok=True) + from sglang.srt.distributed.parallel_state import get_world_group + + if (not is_distributed or get_world_group().local_rank == 0) and ( + not os.path.exists(path) + ): + # only the local master process (with local_rank == 0) can + # enter this block to calculate the cache + logger.info("generating GPU P2P access cache in %s", path) + cache: Dict[str, bool] = {} + ids = list(range(num_dev)) + # batch of all pairs of GPUs + batch_src, batch_tgt = zip(*list(product(ids, ids))) + # NOTE: we use `subprocess` rather than `multiprocessing` here + # because the caller might not have `if __name__ == "__main__":`, + # in that case we cannot use spawn method in multiprocessing. + # However, `can_actually_p2p` requires spawn method. + # The fix is, we use `subprocess` to call the function, + # where we have `if __name__ == "__main__":` in this file. + + # use a temporary file to store the result + # we don't use the output of the subprocess directly, + # because the subprocess might produce logging output + with tempfile.NamedTemporaryFile() as output_file: + input_bytes = pickle.dumps((batch_src, batch_tgt, output_file.name)) + returned = subprocess.run( + [sys.executable, __file__], input=input_bytes, capture_output=True + ) + # check if the subprocess is successful + try: + returned.check_returncode() + except Exception as e: + # wrap raised exception to provide more information + raise RuntimeError( + f"Error happened when batch testing " + f"peer-to-peer access from {batch_src} to {batch_tgt}:\n" + f"{returned.stderr.decode()}" + ) from e + with open(output_file.name, "rb") as f: + result = pickle.load(f) + for _i, _j, r in zip(batch_src, batch_tgt, result): + cache[f"{_i}->{_j}"] = r + with open(path, "w") as f: + json.dump(cache, f, indent=4) + if is_distributed: + get_world_group().barrier() + logger.info("reading GPU P2P access cache from %s", path) + with open(path) as f: + cache = json.load(f) + _gpu_p2p_access_cache = cache + return _gpu_p2p_access_cache[f"{src}->{tgt}"] + + +def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]: + @wraps(fn) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + if _is_hip: + try: + amdsmi_init() + return fn(*args, **kwargs) + finally: + amdsmi_shut_down() + else: + pynvml.nvmlInit() + try: + return fn(*args, **kwargs) + finally: + pynvml.nvmlShutdown() + + return wrapper + + +@with_nvml_context +def is_full_nvlink(physical_device_ids: List[int], world_size: int) -> bool: + if _is_hip: + """ + query if the set of gpus are fully connected by xgmi (1 hop) + """ + handles = [amdsmi_get_processor_handles()[i] for i in physical_device_ids] + for i, handle in enumerate(handles): + for j, peer_handle in enumerate(handles): + if i < j: + try: + link_type = amdsmi_topo_get_link_type(handle, peer_handle) + # type is 2 for XGMI + if link_type["hops"] != 1 or link_type["type"] != 2: + return False + except AmdSmiException as error: + logger.error("AMD 1 hop XGMI detection failed.", exc_info=error) + return False + return True + else: + """ + query if the set of gpus are fully connected by nvlink (1 hop) + """ + handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in physical_device_ids] + for i, handle in enumerate(handles): + for j, peer_handle in enumerate(handles): + if i < j: + try: + p2p_status = pynvml.nvmlDeviceGetP2PStatus( + handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK + ) + if p2p_status != pynvml.NVML_P2P_STATUS_OK: + return False + except pynvml.NVMLError: + logger.exception( + "NVLink detection failed. This is normal if your" + " machine has no NVLink equipped." + ) + return False + return True + + +def is_weak_contiguous(inp: torch.Tensor): + return inp.is_contiguous() or ( + inp.storage().nbytes() - inp.storage_offset() * inp.element_size() + == inp.numel() * inp.element_size() + ) + + +__all__ = ["gpu_p2p_access_check"] + +if __name__ == "__main__": + batch_src, batch_tgt, output_file = pickle.loads(sys.stdin.buffer.read()) + result = can_actually_p2p(batch_src, batch_tgt) + with open(output_file, "wb") as f: + f.write(pickle.dumps(result)) diff --git a/python/sglang/srt/distributed/device_communicators/hpu_communicator.py b/python/sglang/srt/distributed/device_communicators/hpu_communicator.py new file mode 100644 index 000000000..722e494cf --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/hpu_communicator.py @@ -0,0 +1,49 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/hpu_communicator.py + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.utils import is_hpu + +if is_hpu(): + import habana_frameworks.torch as htorch # noqa: F401 + + +class HpuCommunicator: + + def __init__(self, group: ProcessGroup): + if not is_hpu(): + self.disabled = True + return + self.disabled = False + self.group = group + self.world_size = dist.get_world_size(self.group) + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + # FIXME(kzawora): this is a workaround for a bug in Habana PT bridge + # occurring when PT_HPU_ENABLE_LAZY_COLLECTIVES=true env var is used + # (which is required for tensor parallel HPUGraph inference) + htorch.core.mark_step() + dist.all_reduce(x, group=self.group) + return x + + def all_gather(self, x: torch.Tensor, dim: int = -1) -> torch.Tensor: + world_size = self.world_size + if dim < 0: + # Convert negative dim to positive. + dim += x.dim() + input_size = x.size() + # Allocate output tensor. + output_tensor = torch.empty( + (world_size,) + input_size, dtype=x.dtype, device=x.device + ) + # All-gather. + htorch.core.mark_step() + dist.all_gather_into_tensor(output_tensor, x, group=self.group) + # Reshape + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + (world_size * input_size[dim],) + input_size[dim + 1 :] + ) + return output_tensor diff --git a/python/sglang/srt/distributed/device_communicators/npu_communicator.py b/python/sglang/srt/distributed/device_communicators/npu_communicator.py new file mode 100644 index 000000000..cb6eb88e3 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/npu_communicator.py @@ -0,0 +1,39 @@ +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.utils import is_npu + + +class NpuCommunicator: + + def __init__(self, group: ProcessGroup): + if not is_npu(): + self.disabled = True + return + self.disabled = False + self.group = group + self.world_size = dist.get_world_size(self.group) + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + dist.all_reduce(x, group=self.group) + return x + + def all_gather(self, x: torch.Tensor, dim: int = -1) -> torch.Tensor: + world_size = self.world_size + if dim < 0: + # Convert negative dim to positive. + dim += x.dim() + input_size = x.size() + output_size = (input_size[0] * world_size,) + input_size[1:] + # Allocate output tensor. + output_tensor = torch.empty(output_size, dtype=x.dtype, device=x.device) + # All-gather. + dist.all_gather_into_tensor(output_tensor, x, group=self.group) + # Reshape + output_tensor = output_tensor.reshape((world_size,) + input_size) + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + (world_size * input_size[dim],) + input_size[dim + 1 :] + ) + return output_tensor diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py new file mode 100644 index 000000000..78269ed05 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -0,0 +1,315 @@ +import bisect +import logging +import math +import os +from contextlib import contextmanager +from enum import IntEnum +from typing import Any, Callable, List, Optional, TypeVar, Union + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup, ReduceOp + +from sglang.srt import _custom_ops as ops +from sglang.srt.utils import is_cuda, is_hip + +logger = logging.getLogger(__name__) + +_is_cuda = is_cuda() +_is_hip = is_hip() + +mscclpp_is_available = False +if _is_hip: + # TODO(zyksir): mscclpp is untested on AMD and therefore disabled. + mscclpp_is_available = False +if _is_cuda: + try: + import sgl_kernel + + mscclpp_is_available = True + except: + mscclpp_is_available = False + + +class MscclContextSelection(IntEnum): + MSCCL1SHOT1NODELL = 1 + MSCCL1SHOT2NODELL = 2 + + +def mscclpp_is_weak_contiguous(inp: torch.Tensor): + return inp.is_contiguous() or ( + inp.storage().nbytes() - inp.storage_offset() * inp.element_size() + == inp.numel() * inp.element_size() + ) + + +def mscclpp_convert_to_bytes(size_str): + """ + Converts a human-readable size string (e.g., "1MB", "2.5kb", "3 GB") + into the equivalent number of bytes using binary units. + + Args: + size_str (str): A string representing size with unit (KB, MB, GB). + + Returns: + int: Number of bytes. + """ + size_str = size_str.strip().lower() + + if not size_str: + raise ValueError("Empty input string") + + # Extract numeric part and unit + for i in range(len(size_str)): + if not size_str[i].isdigit() and size_str[i] != ".": + break + num_str = size_str[:i] + unit = size_str[i:].strip() + + try: + num = float(num_str) + except ValueError: + raise ValueError(f"Invalid numeric value in '{size_str}'") + + # Conversion factors + if unit == "b": + return int(num) + elif unit == "kb": + return int(num * 1024) + elif unit == "mb": + return int(num * 1024 * 1024) + elif unit == "gb": + return int(num * 1024 * 1024 * 1024) + else: + raise ValueError(f"Unsupported unit: {unit}, support B, KB, MB, GB only") + + +def mscclpp_bench_time(func, test_niter: int = 10, warmup_niter: int = 2): + # warmup + for _ in range(warmup_niter): + func() + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + dist.barrier() + start_event.record() + for _ in range(test_niter): + func() + end_event.record() + end_event.synchronize() + func_cost_us = start_event.elapsed_time(end_event) / test_niter * 1000 + return func_cost_us + + +class PyMscclppCommunicator: + _SUPPORTED_WORLD_SIZES = [8, 16] + _MAX_BYTES = mscclpp_convert_to_bytes(os.getenv("SGLANG_MSCCLPP_MAX_BYTES", "1MB")) + _SUPPORTED_DTYPE = [torch.float, torch.float16, torch.bfloat16] + + # max_bytes: max supported mscclpp allreduce size + # in A100 mscclpp is faster than nccl only under condition of msg size smaller than1MB + def __init__( + self, + group: ProcessGroup, + device: Union[int, str, torch.device], + max_bytes=_MAX_BYTES, + ) -> None: + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the CustomAllreduce to. If None, + it will be bind to f"cuda:{local_rank}". + It is the caller's responsibility to make sure each communicator + is bind to a unique device, and all communicators in this group + are in the same node. + """ + self._IS_CAPTURING = False + self.disabled = True + + if not mscclpp_is_available: + # disable because of missing mscclpp library + # e.g. in a non-cuda environment + return + + self.group = group + + assert ( + dist.get_backend(group) != dist.Backend.NCCL + ), "CustomAllreduce should be attached to a non-NCCL group." + + rank = dist.get_rank(group=self.group) + world_size = dist.get_world_size(group=self.group) + if world_size == 1: + # No need to initialize mscclpp for single GPU case. + return + + if world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES: + logger.warning( + "PyMscclpp is disabled due to an unsupported world" + " size: %d. Supported world sizes: %s. To silence this " + "warning, specify disable_mscclpp=True explicitly.", + world_size, + str(PyMscclppCommunicator._SUPPORTED_WORLD_SIZES), + ) + return + + self.ranks = torch.distributed.get_process_group_ranks(group) + self.nranks_per_node = torch.cuda.device_count() + # for now mscclpp with stride in the communicator is not tested + if not (abs(self.ranks[-1] - self.ranks[0]) == world_size - 1): + logger.warning( + "PyMscclpp is disabled due to an unsupported group %s." + "Please ensure all ranks in the group are consecutive." + "To silence this warning, specify disable_mscclpp=True explicitly.", + str(self.ranks), + ) + return + + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + + self.max_bytes = max_bytes + self.rank = rank + self.world_size = world_size + + if dist.get_rank(group) == 0: + unique_id = [ops.mscclpp_generate_unique_id()] + else: + unique_id = [None] + dist.broadcast_object_list(unique_id, src=self.ranks[0], group=self.group) + self.unique_id = unique_id[0] + self.rank_to_node, self.rank_to_ib = list(range(world_size)), list( + range(world_size) + ) + for r in range(world_size): + self.rank_to_node[r] = r // 8 + self.rank_to_ib[r] = self.rank % 8 + + self._context = None + self.context_selection = None + self.msg_size_for_finetune = [ + 2**i for i in range(10, math.floor(math.log2(self.max_bytes)) + 1) + ] + self.msg_size2best_config = {} + if world_size == 8: + self.context_selection = MscclContextSelection.MSCCL1SHOT1NODELL + elif world_size == 16: + self.context_selection = MscclContextSelection.MSCCL1SHOT2NODELL + if not _is_hip: + self.scratch = torch.empty( + self.max_bytes * 8, + dtype=torch.uint8, + device=self.device, + ) + self.put_buffer = torch.empty( + self.max_bytes * 8 // self.nranks_per_node, + dtype=torch.uint8, + device=self.device, + ) + self._context = ops.mscclpp_init_context( + self.unique_id, + self.rank, + self.world_size, + self.scratch, + self.put_buffer, + self.nranks_per_node, + self.rank_to_node, + self.rank_to_ib, + int(self.context_selection), + ) + else: + raise NotImplementedError("HIP Mscclpp is not supported yet.") + + self.msg_size2best_config = {} + self.pre_tune_config() + if dist.get_rank(group) == 0: + msg_size2best_config = [self.msg_size2best_config] + else: + msg_size2best_config = [None] + dist.broadcast_object_list( + msg_size2best_config, src=self.ranks[0], group=self.group + ) + self.msg_size2best_config = msg_size2best_config[0] + + # PyMscclpp is enabled only in cuda graph + self.disabled = True + + def pre_tune_config(self, dtype=torch.bfloat16) -> bool: + logger.debug(f"start to pre-tune configs for rank {self.rank}") + nthreads_to_try = [256, 512, 1024] + nblocks_to_try = [21, 42, 84] + inp_randn = torch.ones( + self.msg_size_for_finetune[-1] // dtype.itemsize, dtype=dtype, device="cuda" + ) + oup_randn = torch.empty_like(inp_randn) + for msg_size in self.msg_size_for_finetune: + mock_inp, mock_outp = ( + inp_randn[: msg_size // dtype.itemsize], + oup_randn[: msg_size // dtype.itemsize], + ) + best_config, best_time = None, None + for nthreads in nthreads_to_try: + for nblocks in nblocks_to_try: + cur_cost = mscclpp_bench_time( + lambda: ops.mscclpp_allreduce( + self._context, mock_inp, mock_outp, nthreads, nblocks + ) + ) + if best_time is None or cur_cost < best_time: + best_config = (nthreads, nblocks) + best_time = cur_cost + self.msg_size2best_config[msg_size] = best_config + if self.rank == 0: + logger.debug( + f"for msg_size {msg_size}, best_config: {best_config}, best_time: {best_time}us" + ) + + def should_mscclpp_allreduce( + self, inp: torch.Tensor, op: ReduceOp = ReduceOp.SUM + ) -> bool: + if self.disabled or self._context is None: + return False + if inp.dtype not in PyMscclppCommunicator._SUPPORTED_DTYPE: + return False + if not mscclpp_is_weak_contiguous(inp): + return False + # only support sum op + if op != ReduceOp.SUM: + return False + if inp.numel() * inp.element_size() > self.max_bytes: + return False + return True + + def all_reduce(self, tensor: torch.Tensor, op: ReduceOp = ReduceOp.SUM): + if self._IS_CAPTURING: + if torch.cuda.is_current_stream_capturing(): + self.graph_input_set.add((tensor.dtype, tensor.numel())) + msg_size = tensor.numel() * tensor.itemsize + index = bisect.bisect_left(self.msg_size_for_finetune, msg_size) + msg_size_finetune = self.msg_size_for_finetune[index] + nthreads, nblocks = self.msg_size2best_config[msg_size_finetune] + result = torch.empty_like(tensor) + ops.mscclpp_allreduce(self._context, tensor, result, nthreads, nblocks) + return result + + @contextmanager + def change_state( + self, + enable: Optional[bool] = None, + ): + if enable is None: + # guess a default value when not specified + enable = self.available + + old_disable = self.disabled + self.disabled = not enable + + yield + + self.disabled = old_disable diff --git a/python/sglang/srt/distributed/device_communicators/pynccl.py b/python/sglang/srt/distributed/device_communicators/pynccl.py new file mode 100644 index 000000000..fbb59c477 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/pynccl.py @@ -0,0 +1,341 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/pynccl.py + +import logging +from contextlib import contextmanager +from typing import Optional, Union + +# ===================== import region ===================== +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup, ReduceOp + +from sglang.srt.distributed.device_communicators.pynccl_wrapper import ( + NCCLLibrary, + buffer_type, + cudaStream_t, + ncclComm_t, + ncclDataTypeEnum, + ncclRedOpTypeEnum, + ncclUniqueId, +) +from sglang.srt.distributed.utils import StatelessProcessGroup + +logger = logging.getLogger(__name__) + + +class PyNcclCommunicator: + + def __init__( + self, + group: Union[ProcessGroup, StatelessProcessGroup], + device: Union[int, str, torch.device], + library_path: Optional[str] = None, + ): + """ + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the PyNcclCommunicator to. If None, + it will be bind to f"cuda:{local_rank}". + library_path: the path to the NCCL library. If None, it will + use the default library path. + It is the caller's responsibility to make sure each communicator + is bind to a unique device. + """ + if not isinstance(group, StatelessProcessGroup): + assert dist.is_initialized() + assert ( + dist.get_backend(group) != dist.Backend.NCCL + ), "PyNcclCommunicator should be attached to a non-NCCL group." + # note: this rank is the rank in the group + self.rank = dist.get_rank(group) + self.world_size = dist.get_world_size(group) + else: + self.rank = group.rank + self.world_size = group.world_size + + self.group = group + + # if world_size == 1, no need to create communicator + if self.world_size == 1: + self.available = False + self.disabled = True + self.stream = None + return + try: + self.nccl = NCCLLibrary(library_path) + except Exception: + # disable because of missing NCCL library + # e.g. in a non-GPU environment + self.available = False + self.disabled = True + self.stream = None + return + + self.available = True + self.disabled = False + + self.nccl_version = self.nccl.ncclGetRawVersion() + if self.rank == 0: + logger.info("sglang is using nccl==%s", self.nccl.ncclGetVersion()) + + if self.rank == 0: + # get the unique id from NCCL + self.unique_id = self.nccl.ncclGetUniqueId() + else: + # construct an empty unique id + self.unique_id = ncclUniqueId() + + if not isinstance(group, StatelessProcessGroup): + tensor = torch.ByteTensor(list(self.unique_id.internal)) + ranks = dist.get_process_group_ranks(group) + # arg `src` in `broadcast` is the global rank + dist.broadcast(tensor, src=ranks[0], group=group) + byte_list = tensor.tolist() + for i, byte in enumerate(byte_list): + self.unique_id.internal[i] = byte + else: + self.unique_id = group.broadcast_obj(self.unique_id, src=0) + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + # now `device` is a `torch.device` object + assert isinstance(device, torch.device) + self.device = device + # nccl communicator and stream will use this device + # `torch.cuda.device` is a context manager that changes the + # current cuda device to the specified one + with torch.cuda.device(device): + self.comm: ncclComm_t = self.nccl.ncclCommInitRank( + self.world_size, self.unique_id, self.rank + ) + self.stream = torch.cuda.Stream() + + # A small all_reduce for warmup. + data = torch.zeros(1, device=device) + self.all_reduce(data) + self.stream.synchronize() + del data + + # by default it is disabled, e.g. in profiling models and prefill phase. + # to use it, use under `with obj.change_state(enable=True)`, usually + # when we are using CUDA graph. + self.disabled = True + + def all_reduce( + self, tensor: torch.Tensor, op: ReduceOp = ReduceOp.SUM, stream=None + ): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = self.stream + self.nccl.ncclAllReduce( + buffer_type(tensor.data_ptr()), + buffer_type(tensor.data_ptr()), + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def all_gather( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + stream=None, + sizes: Optional[list[int]] = None, + ): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert input_tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {input_tensor.device}" + ) + if stream is None: + stream = self.stream + + if sizes is not None: + split_offset = 0 + + self.nccl.ncclGroupStart() + for root, split_size in enumerate(sizes): + dst_slice = output_tensor[split_offset : split_offset + split_size] + self.nccl.ncclBroadcast( + buffer_type(input_tensor.data_ptr()), + buffer_type(dst_slice.data_ptr()), + dst_slice.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + root, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + split_offset += split_size + self.nccl.ncclGroupEnd() + else: + self.nccl.ncclAllGather( + buffer_type(input_tensor.data_ptr()), + buffer_type(output_tensor.data_ptr()), + input_tensor.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def reduce_scatter( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + op: ReduceOp = ReduceOp.SUM, + stream=None, + sizes: Optional[list[int]] = None, + ): + if self.disabled: + return + # nccl communicator created on a specific device + # will only work on tensors on the same device + # otherwise it will cause "illegal memory access" + assert input_tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {input_tensor.device}" + ) + if stream is None: + stream = self.stream + + if sizes is not None: + split_offset = 0 + self.nccl.ncclGroupStart() + for root, split_size in enumerate(sizes): + chunk = input_tensor[split_offset : split_offset + split_size, ...] + + self.nccl.ncclReduce( + buffer_type(chunk.data_ptr()), + buffer_type(output_tensor.data_ptr()), + chunk.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), + root, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + split_offset += split_size + self.nccl.ncclGroupEnd() + else: + self.nccl.ncclReduceScatter( + buffer_type(input_tensor.data_ptr()), + buffer_type(output_tensor.data_ptr()), + output_tensor.numel(), + ncclDataTypeEnum.from_torch(input_tensor.dtype), + ncclRedOpTypeEnum.from_torch(op), + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def send(self, tensor: torch.Tensor, dst: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = self.stream + self.nccl.ncclSend( + buffer_type(tensor.data_ptr()), + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + dst, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def recv(self, tensor: torch.Tensor, src: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = self.stream + self.nccl.ncclRecv( + buffer_type(tensor.data_ptr()), + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + src, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def broadcast(self, tensor: torch.Tensor, src: int, stream=None): + if self.disabled: + return + assert tensor.device == self.device, ( + f"this nccl communicator is created to work on {self.device}, " + f"but the input tensor is on {tensor.device}" + ) + if stream is None: + stream = self.stream + if src == self.rank: + sendbuff = buffer_type(tensor.data_ptr()) + # NCCL requires the sender also to have a receive buffer + recvbuff = buffer_type(tensor.data_ptr()) + else: + sendbuff = buffer_type() + recvbuff = buffer_type(tensor.data_ptr()) + self.nccl.ncclBroadcast( + sendbuff, + recvbuff, + tensor.numel(), + ncclDataTypeEnum.from_torch(tensor.dtype), + src, + self.comm, + cudaStream_t(stream.cuda_stream), + ) + + def register_comm_window_raw(self, ptr: int, size: int): + return self.nccl.ncclCommWindowRegister(self.comm, buffer_type(ptr), size, 1) + + def deregister_comm_window(self, window): + return self.nccl.ncclCommWindowDeregister(self.comm, window) + + def group_start(self): + self.nccl.ncclGroupStart() + + def group_end(self): + self.nccl.ncclGroupEnd() + + @contextmanager + def change_state( + self, enable: Optional[bool] = None, stream: Optional[torch.cuda.Stream] = None + ): + """ + A context manager to change the state of the communicator. + """ + if enable is None: + # guess a default value when not specified + enable = self.available + + if stream is None: + stream = self.stream + + old_disable = self.disabled + old_stream = self.stream + + self.stream = stream + self.disabled = not enable + yield + + self.disabled = old_disable + self.stream = old_stream diff --git a/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py new file mode 100644 index 000000000..d7274cf2c --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py @@ -0,0 +1,133 @@ +import tempfile + +import torch +from packaging import version +from torch.cuda.memory import CUDAPluggableAllocator + +from sglang.srt.distributed.parallel_state import GroupCoordinator +from sglang.srt.managers.schedule_batch import global_server_args_dict + +nccl_allocator_source = """ +#include +extern "C" { + +void* nccl_alloc_plug(size_t size, int device, void* stream) { + void* ptr; + ncclResult_t err = ncclMemAlloc(&ptr, size); + return ptr; + +} + +void nccl_free_plug(void* ptr, size_t size, int device, void* stream) { + ncclResult_t err = ncclMemFree(ptr); +} + +} +""" + +_allocator = None +_mem_pool = None +_registered_base_addrs = set() +_graph_pool_id = None + + +def is_symmetric_memory_enabled(): + return global_server_args_dict["enable_symm_mem"] + + +def set_graph_pool_id(graph_pool_id): + global _graph_pool_id + _graph_pool_id = graph_pool_id + + +def get_nccl_mem_pool(): + global _allocator, _mem_pool + if _mem_pool is None: + out_dir = tempfile.gettempdir() + nccl_allocator_libname = "nccl_allocator" + torch.utils.cpp_extension.load_inline( + name=nccl_allocator_libname, + cpp_sources=nccl_allocator_source, + with_cuda=True, + extra_ldflags=["-lnccl"], + verbose=True, + is_python_module=False, + build_directory=out_dir, + ) + _allocator = CUDAPluggableAllocator( + f"{out_dir}/{nccl_allocator_libname}.so", + "nccl_alloc_plug", + "nccl_free_plug", + ).allocator() + _mem_pool = torch.cuda.MemPool(_allocator) + return _mem_pool + + +class use_symmetric_memory: + def __init__(self, group_coordinator: GroupCoordinator): + if not is_symmetric_memory_enabled(): + self.group_coordinator = None + self._mem_pool_ctx = None + self.is_graph_capture = None + self.device = None + self.pre_2_8_0 = None + else: + self.group_coordinator = group_coordinator + self._mem_pool_ctx = torch.cuda.use_mem_pool(get_nccl_mem_pool()) + self.is_graph_capture = torch.cuda.is_current_stream_capturing() + self.device = torch.cuda.current_device() + self.pre_2_8_0 = version.parse(torch.__version__) < version.parse("2.8.0") + + def __enter__(self): + if not is_symmetric_memory_enabled(): + return self + assert ( + self.group_coordinator.pynccl_comm is not None + ), f"Symmetric memory requires pynccl to be enabled in group '{self.group_coordinator.group_name}'" + assert ( + self.group_coordinator.pynccl_comm.nccl_version >= 22703 + ), "NCCL version 2.27.3 or higher is required for NCCL symmetric memory" + if self.is_graph_capture: + assert ( + _graph_pool_id is not None + ), "graph_pool_id is not set under graph capture" + # Pause graph memory pool to use symmetric memory with cuda graph + if self.pre_2_8_0: + torch._C._cuda_endAllocateCurrentStreamToPool( + self.device, _graph_pool_id + ) + else: + torch._C._cuda_endAllocateToPool(self.device, _graph_pool_id) + self._mem_pool_ctx.__enter__() + return self + + def tag(self, tensor: torch.Tensor): + if not is_symmetric_memory_enabled(): + return + tensor.symmetric_memory = True + + def __exit__(self, exc_type, exc_val, exc_tb): + if not is_symmetric_memory_enabled(): + return + global _registered_base_addrs + self._mem_pool_ctx.__exit__(exc_type, exc_val, exc_tb) + for segment in get_nccl_mem_pool().snapshot(): + if segment["address"] not in _registered_base_addrs: + if segment["stream"] == 0 and self.pre_2_8_0: + # PyTorch version < 2.8.0 has a multi-thread MemPool bug + # See https://github.com/pytorch/pytorch/issues/152861 + # Fixed at https://github.com/pytorch/pytorch/commit/f01e628e3b31852983ab30b25bf251f557ba9c0b + # WAR is to skip allocations on the default stream since the forward_pass thread always runs on a custom stream + continue + self.group_coordinator.pynccl_comm.register_comm_window_raw( + segment["address"], segment["total_size"] + ) + _registered_base_addrs.add(segment["address"]) + + if self.is_graph_capture: + if self.pre_2_8_0: + torch._C._cuda_beginAllocateToPool(self.device, _graph_pool_id) + else: + torch._C._cuda_beginAllocateCurrentThreadToPool( + self.device, _graph_pool_id + ) diff --git a/python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py b/python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py new file mode 100644 index 000000000..579811777 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/pynccl_wrapper.py @@ -0,0 +1,563 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/pynccl.py + +# This file is a pure Python wrapper for the NCCL library. +# The main purpose is to use NCCL combined with CUDA graph. +# Before writing this script, we tried the following approach: +# 1. We tried to use `cupy`, it calls NCCL correctly, but `cupy` itself +# often gets stuck when initializing the NCCL communicator. +# 2. We tried to use `torch.distributed`, but `torch.distributed.all_reduce` +# contains many other potential cuda APIs, that are not allowed during +# capturing the CUDA graph. For further details, please check +# https://discuss.pytorch.org/t/pytorch-cudagraph-with-nccl-operation-failed/ . +# +# Another rejected idea is to write a C/C++ binding for NCCL. It is usually +# doable, but we often encounter issues related with nccl versions, and need +# to switch between different versions of NCCL. See +# https://github.com/NVIDIA/nccl/issues/1234 for more details. +# A C/C++ binding is not flexible enough to handle this. It requires +# recompilation of the code every time we want to switch between different +# versions. This current implementation, with a **pure** Python wrapper, is +# more flexible. We can easily switch between different versions of NCCL by +# changing the environment variable `SGLANG_NCCL_SO_PATH`, or the `so_file` +# variable in the code. + +import ctypes +import logging +import os +import platform +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import torch +from torch.distributed import ReduceOp + +logger = logging.getLogger(__name__) + + +def find_nccl_library() -> str: + """ + We either use the library file specified by the `SGLANG_NCCL_SO_PATH` + environment variable, or we find the library file brought by PyTorch. + After importing `torch`, `libnccl.so.2` or `librccl.so.1` can be + found by `ctypes` automatically. + """ + + # so_file can be set to None in sglang + so_file = os.environ.get("SGLANG_NCCL_SO_PATH", None) + + # manually load the nccl library + if so_file: + logger.info( + "Found nccl from environment variable SGLANG_NCCL_SO_PATH=%s", so_file + ) + else: + if torch.version.cuda is not None: + so_file = "libnccl.so.2" + elif torch.version.hip is not None: + so_file = "librccl.so.1" + else: + raise ValueError("NCCL only supports CUDA and ROCm backends.") + logger.debug("Found nccl from library %s", so_file) + return so_file + + +# === export types and functions from nccl to Python === +# for the original nccl definition, please check +# https://github.com/NVIDIA/nccl/blob/master/src/nccl.h.in + +ncclResult_t = ctypes.c_int +ncclComm_t = ctypes.c_void_p +ncclWindow_t = ctypes.c_void_p + + +class ncclUniqueId(ctypes.Structure): + _fields_ = [("internal", ctypes.c_byte * 128)] + + +cudaStream_t = ctypes.c_void_p +buffer_type = ctypes.c_void_p + +ncclDataType_t = ctypes.c_int + + +class ncclDataTypeEnum: + ncclInt8 = 0 + ncclChar = 0 + ncclUint8 = 1 + ncclInt32 = 2 + ncclInt = 2 + ncclUint32 = 3 + ncclInt64 = 4 + ncclUint64 = 5 + ncclFloat16 = 6 + ncclHalf = 6 + ncclFloat32 = 7 + ncclFloat = 7 + ncclFloat64 = 8 + ncclDouble = 8 + ncclBfloat16 = 9 + ncclNumTypes = 10 + + @classmethod + def from_torch(cls, dtype: torch.dtype) -> int: + if dtype == torch.int8: + return cls.ncclInt8 + if dtype == torch.uint8: + return cls.ncclUint8 + if dtype == torch.int32: + return cls.ncclInt32 + if dtype == torch.int64: + return cls.ncclInt64 + if dtype == torch.float16: + return cls.ncclFloat16 + if dtype == torch.float32: + return cls.ncclFloat32 + if dtype == torch.float64: + return cls.ncclFloat64 + if dtype == torch.bfloat16: + return cls.ncclBfloat16 + raise ValueError(f"Unsupported dtype: {dtype}") + + +ncclRedOp_t = ctypes.c_int + + +class ncclRedOpTypeEnum: + ncclSum = 0 + ncclProd = 1 + ncclMax = 2 + ncclMin = 3 + ncclAvg = 4 + ncclNumOps = 5 + + @classmethod + def from_torch(cls, op: ReduceOp) -> int: + if op == ReduceOp.SUM: + return cls.ncclSum + if op == ReduceOp.PRODUCT: + return cls.ncclProd + if op == ReduceOp.MAX: + return cls.ncclMax + if op == ReduceOp.MIN: + return cls.ncclMin + if op == ReduceOp.AVG: + return cls.ncclAvg + raise ValueError(f"Unsupported op: {op}") + + +@dataclass +class Function: + name: str + restype: Any + argtypes: List[Any] + + +class NCCLLibrary: + exported_functions = [ + # const char* ncclGetErrorString(ncclResult_t result) + Function("ncclGetErrorString", ctypes.c_char_p, [ncclResult_t]), + # ncclResult_t ncclGetVersion(int *version); + Function("ncclGetVersion", ncclResult_t, [ctypes.POINTER(ctypes.c_int)]), + # ncclResult_t ncclGetUniqueId(ncclUniqueId* uniqueId); + Function("ncclGetUniqueId", ncclResult_t, [ctypes.POINTER(ncclUniqueId)]), + # ncclResult_t ncclCommInitRank( + # ncclComm_t* comm, int nranks, ncclUniqueId commId, int rank); + # note that ncclComm_t is a pointer type, so the first argument + # is a pointer to a pointer + Function( + "ncclCommInitRank", + ncclResult_t, + [ctypes.POINTER(ncclComm_t), ctypes.c_int, ncclUniqueId, ctypes.c_int], + ), + # ncclResult_t ncclAllReduce( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclAllReduce", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclRedOp_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclAllGather( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclAllGather", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclReduce( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, int root, + # ncclComm_t comm, cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclReduce", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclRedOp_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclReduceScatter( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, + # cudaStream_t stream); + # note that cudaStream_t is a pointer type, so the last argument + # is a pointer + Function( + "ncclReduceScatter", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ncclRedOp_t, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclSend( + # const void* sendbuff, size_t count, ncclDataType_t datatype, + # int dest, ncclComm_t comm, cudaStream_t stream); + Function( + "ncclSend", + ncclResult_t, + [ + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclRecv( + # void* recvbuff, size_t count, ncclDataType_t datatype, + # int src, ncclComm_t comm, cudaStream_t stream); + Function( + "ncclRecv", + ncclResult_t, + [ + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # ncclResult_t ncclBroadcast( + # const void* sendbuff, void* recvbuff, size_t count, + # ncclDataType_t datatype, int root, ncclComm_t comm, + # cudaStream_t stream); + Function( + "ncclBroadcast", + ncclResult_t, + [ + buffer_type, + buffer_type, + ctypes.c_size_t, + ncclDataType_t, + ctypes.c_int, + ncclComm_t, + cudaStream_t, + ], + ), + # be cautious! this is a collective call, it will block until all + # processes in the communicator have called this function. + # because Python object destruction can happen in random order, + # it is better not to call it at all. + # ncclResult_t ncclCommDestroy(ncclComm_t comm); + Function("ncclCommDestroy", ncclResult_t, [ncclComm_t]), + # ncclResult_t ncclGroupStart(); + Function("ncclGroupStart", ncclResult_t, []), + # ncclResult_t ncclGroupEnd(); + Function("ncclGroupEnd", ncclResult_t, []), + ] + + exported_functions_symm_mem = [ + # ncclResult_t ncclCommWindowRegister(ncclComm_t comm, void* buff, size_t size, ncclWindow_t* win, int winFlags); + Function( + "ncclCommWindowRegister", + ncclResult_t, + [ + ncclComm_t, + buffer_type, + ctypes.c_size_t, + ctypes.POINTER(ncclWindow_t), + ctypes.c_int, + ], + ), + # ncclResult_t ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win); + Function("ncclCommWindowDeregister", ncclResult_t, [ncclComm_t, ncclWindow_t]), + ] + + # class attribute to store the mapping from the path to the library + # to avoid loading the same library multiple times + path_to_library_cache: Dict[str, Any] = {} + + # class attribute to store the mapping from library path + # to the corresponding dictionary + path_to_dict_mapping: Dict[str, Dict[str, Any]] = {} + + def __init__(self, so_file: Optional[str] = None): + + so_file = so_file or find_nccl_library() + + try: + if so_file not in NCCLLibrary.path_to_dict_mapping: + lib = ctypes.CDLL(so_file) + NCCLLibrary.path_to_library_cache[so_file] = lib + self.lib = NCCLLibrary.path_to_library_cache[so_file] + except Exception as e: + logger.error( + "Failed to load NCCL library from %s ." + "It is expected if you are not running on NVIDIA/AMD GPUs." + "Otherwise, the nccl library might not exist, be corrupted " + "or it does not support the current platform %s." + "If you already have the library, please set the " + "environment variable SGLANG_NCCL_SO_PATH" + " to point to the correct nccl library path.", + so_file, + platform.platform(), + ) + raise e + + if so_file not in NCCLLibrary.path_to_dict_mapping: + _funcs: Dict[str, Any] = {} + exported_functions = NCCLLibrary.exported_functions + if hasattr(self.lib, "ncclCommWindowRegister"): + exported_functions.extend(NCCLLibrary.exported_functions_symm_mem) + for func in exported_functions: + f = getattr(self.lib, func.name) + f.restype = func.restype + f.argtypes = func.argtypes + _funcs[func.name] = f + NCCLLibrary.path_to_dict_mapping[so_file] = _funcs + self._funcs = NCCLLibrary.path_to_dict_mapping[so_file] + + def ncclGetErrorString(self, result: ncclResult_t) -> str: + return self._funcs["ncclGetErrorString"](result).decode("utf-8") + + def NCCL_CHECK(self, result: ncclResult_t) -> None: + if result != 0: + error_str = self.ncclGetErrorString(result) + raise RuntimeError(f"NCCL error: {error_str}") + + def ncclGetRawVersion(self) -> int: + version = ctypes.c_int() + self.NCCL_CHECK(self._funcs["ncclGetVersion"](ctypes.byref(version))) + # something like 21903 + return version.value + + def ncclGetVersion(self) -> str: + version_str = str(self.ncclGetRawVersion()) + # something like 21903 --> "2.19.3" + major = version_str[0].lstrip("0") + minor = version_str[1:3].lstrip("0") + patch = version_str[3:].lstrip("0") + return f"{major}.{minor}.{patch}" + + def ncclGetUniqueId(self) -> ncclUniqueId: + unique_id = ncclUniqueId() + self.NCCL_CHECK(self._funcs["ncclGetUniqueId"](ctypes.byref(unique_id))) + return unique_id + + def ncclCommInitRank( + self, world_size: int, unique_id: ncclUniqueId, rank: int + ) -> ncclComm_t: + comm = ncclComm_t() + self.NCCL_CHECK( + self._funcs["ncclCommInitRank"]( + ctypes.byref(comm), world_size, unique_id, rank + ) + ) + return comm + + def ncclAllReduce( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + op: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclAllReduce"]( + sendbuff, recvbuff, count, datatype, op, comm, stream + ) + ) + + def ncclReduce( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + op: int, + root: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclReduce"]( + sendbuff, recvbuff, count, datatype, op, root, comm, stream + ) + ) + + def ncclReduceScatter( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + op: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # and `op` should be `ncclRedOp_t` + # both are aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclReduceScatter"]( + sendbuff, recvbuff, count, datatype, op, comm, stream + ) + ) + + def ncclAllGather( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + # `datatype` actually should be `ncclDataType_t` + # which is an aliases of `ctypes.c_int` + # when we pass int to a function, it will be converted to `ctypes.c_int` + # by ctypes automatically + self.NCCL_CHECK( + self._funcs["ncclAllGather"]( + sendbuff, recvbuff, count, datatype, comm, stream + ) + ) + + def ncclSend( + self, + sendbuff: buffer_type, + count: int, + datatype: int, + dest: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclSend"](sendbuff, count, datatype, dest, comm, stream) + ) + + def ncclRecv( + self, + recvbuff: buffer_type, + count: int, + datatype: int, + src: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclRecv"](recvbuff, count, datatype, src, comm, stream) + ) + + def ncclBroadcast( + self, + sendbuff: buffer_type, + recvbuff: buffer_type, + count: int, + datatype: int, + root: int, + comm: ncclComm_t, + stream: cudaStream_t, + ) -> None: + self.NCCL_CHECK( + self._funcs["ncclBroadcast"]( + sendbuff, recvbuff, count, datatype, root, comm, stream + ) + ) + + def ncclCommDestroy(self, comm: ncclComm_t) -> None: + self.NCCL_CHECK(self._funcs["ncclCommDestroy"](comm)) + + def ncclCommWindowRegister( + self, comm: ncclComm_t, buff: buffer_type, size: int, win_flags: int + ) -> ncclWindow_t: + window = ncclWindow_t() + self.NCCL_CHECK( + self._funcs["ncclCommWindowRegister"]( + comm, buff, size, ctypes.byref(window), win_flags + ) + ) + return window + + def ncclCommWindowDeregister(self, comm: ncclComm_t, window: ncclWindow_t) -> None: + self.NCCL_CHECK(self._funcs["ncclCommWindowDeregister"](comm, window)) + + def ncclGroupStart(self) -> None: + self.NCCL_CHECK(self._funcs["ncclGroupStart"]()) + + def ncclGroupEnd(self) -> None: + self.NCCL_CHECK(self._funcs["ncclGroupEnd"]()) + + +__all__ = [ + "NCCLLibrary", + "ncclDataTypeEnum", + "ncclRedOpTypeEnum", + "ncclUniqueId", + "ncclComm_t", + "cudaStream_t", + "buffer_type", +] diff --git a/python/sglang/srt/distributed/device_communicators/quick_all_reduce.py b/python/sglang/srt/distributed/device_communicators/quick_all_reduce.py new file mode 100644 index 000000000..0113c432d --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/quick_all_reduce.py @@ -0,0 +1,273 @@ +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +from enum import Enum +from typing import Union + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt import _custom_ops as ops +from sglang.srt.distributed.device_communicators.custom_all_reduce_utils import ( + is_full_nvlink, + is_weak_contiguous, +) +from sglang.srt.distributed.parallel_state import in_the_same_node_as +from sglang.srt.utils import is_cuda, is_hip + +logger = logging.getLogger(__name__) + +_is_cuda = is_cuda() +_is_hip = is_hip() + + +try: + ops.qr_max_size() + quick_ar = True +except Exception: + # For CPUs and CUDA + quick_ar = False + + +def qr_rocm_arch_available(): + if not _is_hip: + return False + try: + props = torch.cuda.get_device_properties(0) + gcn_arch = getattr(props, "gcnArchName", "") + supported_archs = ["gfx94", "gfx95"] + return any(gfx in gcn_arch for gfx in supported_archs) + except Exception as e: + logger.warning("Failed to determine ROCm for quick allreduce: %s", e) + return False + + +class QuickReduceRegime(Enum): + FP = 0 + INT8 = 1 + INT6 = 2 + INT4 = 3 + NONE = 4 + + +MB = 1024 * 1024 + + +class QuickAllReduce: + + _SUPPORTED_WORLD_SIZES = [2, 4, 8] + _SUPPORTED_DTYPES = [torch.float16, torch.bfloat16] + # The following data is based on kernel tests. + # In this order [FP, INT8, INT6, INT4]. + _QR_MIN_SIZE = { + (torch.float16, 2): [1 * MB, 2 * MB, 2 * MB, 1 * MB], + (torch.float16, 4): [1 * MB, 16 * MB, 4 * MB, 2 * MB], + (torch.float16, 8): [16 * MB, 4 * MB, 4 * MB, 2 * MB], + (torch.bfloat16, 2): [2 * MB, 8 * MB, 8 * MB, 8 * MB], + (torch.bfloat16, 4): [8 * MB, 64 * MB, 64 * MB, 16 * MB], + (torch.bfloat16, 8): [16 * MB, 2048 * MB, 2048 * MB, 2048 * MB], + } + + def __init__( + self, group: ProcessGroup, device: Union[int, str, torch.device] + ) -> None: + """ + Custom allreduce provides non-destructive acceleration and is + available for CUDA and ROCm MI300 series. + Custom quick allreduce leverages quantization for further + acceleration on ROCm. It currently supports Q8, Q6, and Q4 + quantization formats and FP(float16, bfloat16). + Quick allreduce is designed as a complement to custom allreduce. + Its initialization requires even stricter conditions. + Only the ROCm MI300 series is supported for quick allreduce at + this time. + Args: + group: the process group to work on. If None, it will use the + default process group. + device: the device to bind the CustomAllreduce to. If None, + it will be bind to f"cuda:{local_rank}". + It is the caller's responsibility to make sure each communicator + is bind to a unique device, and all communicators in this group + are in the same node. + """ + self.disabled = True + if not qr_rocm_arch_available(): + logger.debug( + "Custom quick allreduce is only supported on ROCm MI300 series." + ) + return + + if not quick_ar: + # disable because of missing quick reduce library + # e.g. in a cuda environment + logger.info( + "Custom quick allreduce is disabled because " + "of missing custom quick allreduce library" + ) + return + + self.group = group + assert ( + dist.get_backend(group) != dist.Backend.NCCL + ), "Custom quick allreduce should be attached to a non-NCCL group." + if not all(in_the_same_node_as(group, source_rank=0)): + # No need to initialize custom quick allreduce for + # multi-node case. + logger.warning( + "Custom quick allreduce is disabled because this " + "process group spans across nodes." + ) + return + rank = dist.get_rank(group=self.group) + world_size = dist.get_world_size(group=self.group) + self.rank = rank + self.world_size = world_size + if world_size == 1: + # No need to initialize QuickReduce for single GPU case. + return + + if world_size not in QuickAllReduce._SUPPORTED_WORLD_SIZES: + logger.warning( + "Custom quick allreduce is disabled due to an " + "unsupported world size: %d. Supported world sizes: %s.", + world_size, + str(QuickAllReduce._SUPPORTED_WORLD_SIZES), + ) + return + + if isinstance(device, int): + device = torch.device(f"cuda:{device}") + elif isinstance(device, str): + device = torch.device(device) + assert isinstance(device, torch.device) + self.device = device + + cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES", None) + if cuda_visible_devices: + device_ids = list(map(int, cuda_visible_devices.split(","))) + else: + device_ids = list(range(torch.cuda.device_count())) + physical_device_id = device_ids[device.index] + tensor = torch.tensor([physical_device_id], dtype=torch.int, device="cpu") + gather_list = [ + torch.tensor([0], dtype=torch.int, device="cpu") + for _ in range(self.world_size) + ] + dist.all_gather(gather_list, tensor, group=self.group) + physical_device_ids = [t.item() for t in gather_list] + + # test nvlink first, this will filter out most of the cases + # where custom quick allreduce is not supported + # this checks hardware and driver support for NVLink + if _is_cuda or _is_hip: + self.fully_connected = is_full_nvlink(physical_device_ids, self.world_size) + if self.world_size > 2 and not self.fully_connected: + logger.debug( + "Custom quick allreduce is disabled because it's not supported " + "on more than two PCIe-only GPUs. " + ) + return + + self.init_quick_all_reduce() + + def init_quick_all_reduce(self): + # On RocM, bfloat16 kernels are slower than fp16 + # due to slower match operations + # If environment variable is set to 1, we convert input to fp16 + self.use_fp16_kernels = int( + os.environ.get("ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16", 1) + ) + regime_str = os.environ.get("ROCM_QUICK_REDUCE_QUANTIZATION", "NONE") + if regime_str not in QuickReduceRegime.__members__: + logger.warning( + "Custom quick allreduce:", + f"Invalid quantization level: {regime_str}. " + "Supported levels: " + f"{list(QuickReduceRegime.__members__.keys())}", + ) + return + + if regime_str == "NONE": + logger.debug( + "Custom quick allreduce is disabled based " + "on env variable " + "ROCM_QUICK_REDUCE_QUANTIZATION='NONE'" + ) + return + self.qr_quant_level = QuickReduceRegime[regime_str] + + # TODO: If the dtype is not bfloat16 or then float16, + # quickallreduce should not be created. + + # ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB is specified in MB + qr_max_size = int(os.environ.get("ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB", 0)) + if qr_max_size > 0: + if qr_max_size < 1: + logger.info( + "You should not set a max_size smaller than 1MB, which can " + "lead to error or degradation to custom allreduce or rccl." + ) + qr_max_size = qr_max_size * MB + # If qr_max_size is None, then 2GB is used by default. + self._ptr = ops.init_custom_qr(self.rank, self.world_size, qr_max_size) + self.qr_max_size = qr_max_size if qr_max_size > 0 else ops.qr_max_size() + self.create_shared_buffer() + self.disabled = False + + def create_shared_buffer(self): + """ + Creates a shared buffer for quickreduce. + Has to be called after init_custom_qr + """ + handle = ops.qr_get_handle(self._ptr) + world_size = dist.get_world_size(group=self.group) + handles = [None] * world_size + dist.all_gather_object(handles, handle, group=self.group) + ops.qr_open_handles(self._ptr, handles) + + def should_quick_allreduce(self, inp: torch.Tensor): + """ + Check if quickreduce is available + """ + if self.disabled: + return False + if inp.dtype not in self._SUPPORTED_DTYPES: + return False + inp_size = inp.numel() * inp.element_size() + # custom quick allreduce requires input byte size to be + # multiples of 16 + if inp_size % 16 != 0: + return False + if not is_weak_contiguous(inp): + return False + dtype = inp.dtype + if self.use_fp16_kernels: + dtype = torch.float16 + return ( + inp_size <= self.qr_max_size + and inp_size + >= self._QR_MIN_SIZE[(dtype, self.world_size)][self.qr_quant_level.value] + ) + + def quick_all_reduce(self, inp: torch.Tensor, *, out: torch.Tensor = None): + """Performs an out-of-place custom quick all reduce.""" + # quick allreduce doesn't require a separate graph mode, + # as QR uses static IPC buffer. + if out is None: + out = torch.empty_like(inp) + ops.qr_all_reduce( + self._ptr, inp, out, self.qr_quant_level.value, self.use_fp16_kernels + ) + return out + + def close(self): + if not self.disabled and getattr(self, "_ptr", None): + if ops is not None: + ops.qr_destroy(self._ptr) + self._ptr = 0 + self.disabled = True + + def __del__(self): + self.close() diff --git a/python/sglang/srt/distributed/device_communicators/shm_broadcast.py b/python/sglang/srt/distributed/device_communicators/shm_broadcast.py new file mode 100644 index 000000000..e5b59e7cc --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/shm_broadcast.py @@ -0,0 +1,509 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/shm_broadcast.py + +import logging +import os +import pickle +import time +from contextlib import contextmanager +from dataclasses import dataclass, field +from multiprocessing import shared_memory +from typing import List, Optional +from unittest.mock import patch + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup +from zmq import IPV6 # type: ignore +from zmq import SUB, SUBSCRIBE, XPUB, XPUB_VERBOSE, Context # type: ignore + +from sglang.srt.utils import ( + format_tcp_address, + get_ip, + get_open_port, + is_valid_ipv6_address, +) + +# SGLANG_RINGBUFFER_WARNING_INTERVAL can be set to 60 +SGLANG_RINGBUFFER_WARNING_INTERVAL = int( + os.environ.get("SGLANG_RINGBUFFER_WARNING_INTERVAL", "60") +) + +logger = logging.getLogger(__name__) + + +class ShmRingBuffer: + + def __init__( + self, + n_reader: int, + max_chunk_bytes: int, + max_chunks: int, + name: Optional[str] = None, + ): + """ + A shared memory ring buffer implementation for broadcast communication. + Essentially, it is a queue where only one will `enqueue` and multiple + will `dequeue`. The max size of each item, together with the max number + of items that can be stored in the buffer are known in advance. + In this case, we don't need to synchronize the access to + the buffer. + + Buffer memory layout: + data metadata + | | + | (current_idx) | (current_idx) + v v + +-------------------------------+----------------------------------------+ + | chunk0 | chunk1 | ... | chunk | metadata0 | metadata1 | ... | metadata | + +-------------------------------+----------------------------------------+ + | max_chunks x max_chunk_bytes | max_chunks x (1 + n_reader) bytes | + + metadata memory layout: each byte is a flag, the first byte is the written + flag, and the rest are reader flags. The flags are set to 0 by default. + +--------------+--------------+--------------+-----+--------------+ + | written_flag | reader0_flag | reader1_flag | ... | readerN_flag | + +--------------+--------------+--------------+-----+--------------+ + + The state of metadata is as follows: + + (case 1) 0???...???: the block is not written yet, cannot read, can write + (case 2) 1000...000: the block is just written, can read, cannot write + (case 3) 1???...???: the block is written and read by some readers, can read if not read, cannot write + (case 4) 1111...111: the block is written and read by all readers, cannot read, can write + + State transition for readers: + + When a reader finds a block that it can read (case 2 or 3), it can yield the block for caller to read. + Only after the caller finishes reading the block, the reader can mark the block as read. + Readers only mark the block as read (from 0 to 1), the writer marks the block as ready to read (from 1 to 0). + + State transition for writer: + + When the writer writes to a block (case 1 or 4), it first resets the written flag to 0, converting either case + to case 1. Then it can yield the block for caller to write. After the caller finishes writing the block, the writer + can reset the reader flags to 0, and mark the block as written (from 0 to 1). + NOTE: the order is important here, first reset the reader flags (so that we are still in case 1), then mark the block as written. The state transition is atomic. If we do it in the reverse order, it will go through case 3 and then back to case 2, and readers might read the intermediate case 3, which is not correct. + + During creation, `name` is None and the buffer is created. We can pass the + created object to other processes by pickling it. The other processes will + get the name of the shared memory and open it, so that they can access the + same shared memory buffer. + """ # noqa + self.n_reader = n_reader + self.metadata_size = 1 + n_reader + self.max_chunk_bytes = max_chunk_bytes + self.max_chunks = max_chunks + self.total_bytes_of_buffer = ( + self.max_chunk_bytes + self.metadata_size + ) * self.max_chunks + self.data_offset = 0 + self.metadata_offset = self.max_chunk_bytes * self.max_chunks + + if name is None: + # we are creating a buffer + self.is_creator = True + self.shared_memory = shared_memory.SharedMemory( + create=True, size=self.total_bytes_of_buffer + ) + # initialize the metadata section to 0 + with memoryview( + self.shared_memory.buf[self.metadata_offset :] + ) as metadata_buffer: + torch.frombuffer(metadata_buffer, dtype=torch.uint8).fill_(0) + else: + # we are opening an existing buffer + self.is_creator = False + # fix to https://stackoverflow.com/q/62748654/9191338 + # Python incorrectly tracks shared memory even if it is not + # created by the process. The following patch is a workaround. + with patch( + "multiprocessing.resource_tracker.register", + lambda *args, **kwargs: None, + ): + try: + self.shared_memory = shared_memory.SharedMemory(name=name) + assert self.shared_memory.size == self.total_bytes_of_buffer + except FileNotFoundError: + # we might deserialize the object in a different node + # in this case, this object is not used, + # and we should suppress the error + pass + + def __reduce__(self): + return ( + self.__class__, + ( + self.n_reader, + self.max_chunk_bytes, + self.max_chunks, + self.shared_memory.name, + ), + ) + + def __del__(self): + if hasattr(self, "shared_memory"): + self.shared_memory.close() + if self.is_creator: + self.shared_memory.unlink() + + @contextmanager + def get_data(self, current_idx: int): + start = self.data_offset + current_idx * self.max_chunk_bytes + end = start + self.max_chunk_bytes + with memoryview(self.shared_memory.buf[start:end]) as buf: + yield buf + + @contextmanager + def get_metadata(self, current_idx: int): + start = self.metadata_offset + current_idx * self.metadata_size + end = start + self.metadata_size + with memoryview(self.shared_memory.buf[start:end]) as buf: + yield buf + + +@dataclass +class Handle: + connect_ip: str + local_reader_ranks: List[int] = field(default_factory=list) + + buffer: Optional[ShmRingBuffer] = None + local_subscribe_port: Optional[int] = None + remote_subscribe_port: Optional[int] = None + + +class MessageQueue: + + def __init__( + self, + n_reader, # number of all readers + n_local_reader, # number of local readers through shared memory + local_reader_ranks: Optional[List[int]] = None, + max_chunk_bytes: int = 1024 * 1024 * 10, + max_chunks: int = 10, + connect_ip: Optional[str] = None, + ): + if local_reader_ranks is None: + local_reader_ranks = list(range(n_local_reader)) + else: + assert len(local_reader_ranks) == n_local_reader + self.n_local_reader = n_local_reader + n_remote_reader = n_reader - n_local_reader + self.n_remote_reader = n_remote_reader + + if connect_ip is None: + connect_ip = get_ip() if n_remote_reader > 0 else "127.0.0.1" + + context = Context() + + if n_local_reader > 0: + # for local readers, we will: + # 1. create a shared memory ring buffer to communicate small data + # 2. create a publish-subscribe socket to communicate large data + self.buffer = ShmRingBuffer(n_local_reader, max_chunk_bytes, max_chunks) + + # XPUB is very similar to PUB, + # except that it can receive subscription messages + # to confirm the number of subscribers + self.local_socket = context.socket(XPUB) + # set the verbose option so that we can receive every subscription + # message. otherwise, we will only receive the first subscription + # see http://api.zeromq.org/3-3:zmq-setsockopt for more details + self.local_socket.setsockopt(XPUB_VERBOSE, True) + local_subscribe_port = get_open_port() + socket_addr = f"tcp://127.0.0.1:{local_subscribe_port}" + logger.debug("Binding to %s", socket_addr) + self.local_socket.bind(socket_addr) + + self.current_idx = 0 + + else: + self.buffer = None # type: ignore + local_subscribe_port = None + self.local_socket = None + self.current_idx = -1 + + if n_remote_reader > 0: + # for remote readers, we will: + # create a publish-subscribe socket to communicate large data + self.remote_socket = context.socket(XPUB) + self.remote_socket.setsockopt(XPUB_VERBOSE, True) + remote_subscribe_port = get_open_port() + if is_valid_ipv6_address(connect_ip): + self.remote_socket.setsockopt(IPV6, 1) + self.remote_socket.bind( + format_tcp_address(connect_ip, remote_subscribe_port) + ) + + else: + remote_subscribe_port = None + self.remote_socket = None + + self._is_writer = True + self._is_local_reader = False + self.local_reader_rank = -1 + # rank does not matter for remote readers + self._is_remote_reader = False + + self.handle = Handle( + connect_ip=connect_ip, + local_reader_ranks=local_reader_ranks, + buffer=self.buffer, + local_subscribe_port=local_subscribe_port, + remote_subscribe_port=remote_subscribe_port, + ) + + logger.debug("Message queue communication handle: %s", self.handle) + + def export_handle(self) -> Handle: + return self.handle + + @staticmethod + def create_from_handle(handle: Handle, rank) -> "MessageQueue": + self = MessageQueue.__new__(MessageQueue) + self.handle = handle + self._is_writer = False + + context = Context() + + if rank in handle.local_reader_ranks: + assert handle.buffer is not None + self.buffer = handle.buffer + self.current_idx = 0 + self.local_reader_rank = handle.local_reader_ranks.index(rank) + self._is_local_reader = True + self._is_remote_reader = False + + self.local_socket = context.socket(SUB) + self.local_socket.setsockopt_string(SUBSCRIBE, "") + socket_addr = f"tcp://127.0.0.1:{handle.local_subscribe_port}" + logger.debug("Connecting to %s", socket_addr) + self.local_socket.connect(socket_addr) + + self.remote_socket = None + else: + self.buffer = None # type: ignore + self.current_idx = -1 + self.local_reader_rank = -1 + self._is_local_reader = False + self._is_remote_reader = True + + self.local_socket = None + + self.remote_socket = context.socket(SUB) + self.remote_socket.setsockopt_string(SUBSCRIBE, "") + if is_valid_ipv6_address(handle.connect_ip): + self.remote_socket.setsockopt(IPV6, 1) + socket_addr = format_tcp_address( + handle.connect_ip, handle.remote_subscribe_port + ) + logger.debug("Connecting to %s", socket_addr) + self.remote_socket.connect(socket_addr) + + return self + + def wait_until_ready(self): + """This is a collective operation. All processes (including the + readers and the writer) should call this function. + """ + if self._is_writer: + # wait for all readers to connect + + # local readers + for i in range(self.n_local_reader): + # wait for subscription messages from all local readers + self.local_socket.recv() + if self.n_local_reader > 0: + # send a message to all local readers + # to make sure the publish channel is working + self.local_socket.send(b"READY") + + # remote readers + for i in range(self.n_remote_reader): + # wait for subscription messages from all remote readers + self.remote_socket.recv() + if self.n_remote_reader > 0: + # send a message to all remote readers + # to make sure the publish channel is working + self.remote_socket.send(b"READY") + elif self._is_local_reader: + # wait for the writer to send a message + recv = self.local_socket.recv() + assert recv == b"READY" + elif self._is_remote_reader: + # wait for the writer to send a message + recv = self.remote_socket.recv() + assert recv == b"READY" + + @contextmanager + def acquire_write(self): + assert self._is_writer, "Only writers can acquire write" + start_time = time.monotonic() + n_warning = 1 + while True: + with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + read_count = sum(metadata_buffer[1:]) + written_flag = metadata_buffer[0] + if written_flag and read_count != self.buffer.n_reader: + # this block is written and not read by all readers + # for writers, `self.current_idx` is the next block to write + # if this block is not ready to write, + # we need to wait until it is read by all readers + + # Release the processor to other threads + os.sched_yield() + + # if we wait for a long time, we should warn the user + if ( + time.monotonic() - start_time + > SGLANG_RINGBUFFER_WARNING_INTERVAL * n_warning + ): + logger.warning( + "No available block found in %s second. ", + SGLANG_RINGBUFFER_WARNING_INTERVAL, + ) + n_warning += 1 + + continue + # found a block that is either + # (1) not written + # (2) read by all readers + + # mark the block as not written + metadata_buffer[0] = 0 + # let caller write to the buffer + with self.buffer.get_data(self.current_idx) as buf: + yield buf + + # caller has written to the buffer + # NOTE: order is important here + # first set the read flags to 0 + # then set the written flag to 1 + # otherwise, the readers may think they already read the block + for i in range(1, self.buffer.n_reader + 1): + # set read flag to 0, meaning it is not read yet + metadata_buffer[i] = 0 + # mark the block as written + metadata_buffer[0] = 1 + self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks + break + + @contextmanager + def acquire_read(self): + assert self._is_local_reader, "Only readers can acquire read" + start_time = time.monotonic() + n_warning = 1 + while True: + with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + read_flag = metadata_buffer[self.local_reader_rank + 1] + written_flag = metadata_buffer[0] + if not written_flag or read_flag: + # this block is either + # (1) not written + # (2) already read by this reader + + # for readers, `self.current_idx` is the next block to read + # if this block is not ready, + # we need to wait until it is written + + # Release the processor to other threads + os.sched_yield() + + # if we wait for a long time, we should warn the user + if ( + time.monotonic() - start_time + > SGLANG_RINGBUFFER_WARNING_INTERVAL * n_warning + ): + logger.warning( + "No available block found in %s second. ", + SGLANG_RINGBUFFER_WARNING_INTERVAL, + ) + n_warning += 1 + + continue + # found a block that is not read by this reader + # let caller read from the buffer + with self.buffer.get_data(self.current_idx) as buf: + yield buf + + # caller has read from the buffer + # set the read flag + metadata_buffer[self.local_reader_rank + 1] = 1 + self.current_idx = (self.current_idx + 1) % self.buffer.max_chunks + break + + def enqueue(self, obj): + assert self._is_writer, "Only writers can enqueue" + serialized_obj = pickle.dumps(obj, protocol=pickle.HIGHEST_PROTOCOL) + if self.n_local_reader > 0: + if len(serialized_obj) >= self.buffer.max_chunk_bytes: + with self.acquire_write() as buf: + buf[0] = 1 # overflow + self.local_socket.send(serialized_obj) + else: + with self.acquire_write() as buf: + buf[0] = 0 # not overflow + buf[1 : len(serialized_obj) + 1] = serialized_obj + if self.n_remote_reader > 0: + self.remote_socket.send(serialized_obj) + + def dequeue(self): + if self._is_local_reader: + with self.acquire_read() as buf: + overflow = buf[0] == 1 + if not overflow: + # no need to know the size of serialized object + # pickle format contains the size information internally + # see https://docs.python.org/3/library/pickle.html + obj = pickle.loads(buf[1:]) + if overflow: + recv = self.local_socket.recv() + obj = pickle.loads(recv) + elif self._is_remote_reader: + recv = self.remote_socket.recv() + obj = pickle.loads(recv) + else: + raise RuntimeError("Only readers can dequeue") + return obj + + def broadcast_object(self, obj=None): + if self._is_writer: + self.enqueue(obj) + return obj + else: + return self.dequeue() + + @staticmethod + def create_from_process_group( + pg: ProcessGroup, max_chunk_bytes, max_chunks, writer_rank=0 + ) -> "MessageQueue": + group_rank = dist.get_rank(pg) + group_world_size = dist.get_world_size(pg) + global_ranks = dist.get_process_group_ranks(pg) + + from sglang.srt.distributed.parallel_state import in_the_same_node_as + + status = in_the_same_node_as(pg, source_rank=writer_rank) + same_node_ranks = [i for i, s in enumerate(status) if s] + n_reader = group_world_size - 1 + n_local_reader = len(same_node_ranks) - 1 + local_reader_ranks = [i for i in same_node_ranks if i != writer_rank] + buffer_io: MessageQueue + if group_rank == writer_rank: + buffer_io = MessageQueue( + n_reader=n_reader, + n_local_reader=n_local_reader, + local_reader_ranks=local_reader_ranks, + max_chunk_bytes=max_chunk_bytes, + max_chunks=max_chunks, + ) + handle = buffer_io.export_handle() + dist.broadcast_object_list( + [handle], src=global_ranks[writer_rank], group=pg + ) + else: + recv = [None] + dist.broadcast_object_list(recv, src=global_ranks[writer_rank], group=pg) + handle = recv[0] # type: ignore + buffer_io = MessageQueue.create_from_handle(handle, group_rank) + buffer_io.wait_until_ready() + return buffer_io diff --git a/python/sglang/srt/distributed/device_communicators/xpu_communicator.py b/python/sglang/srt/distributed/device_communicators/xpu_communicator.py new file mode 100644 index 000000000..532279f70 --- /dev/null +++ b/python/sglang/srt/distributed/device_communicators/xpu_communicator.py @@ -0,0 +1,48 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/device_communicators/xpu_communicator.py + +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.utils import is_xpu + + +class XpuCommunicator: + + def __init__(self, group: ProcessGroup): + if not is_xpu(): + self.disabled = True + return + self.disabled = False + self.group = group + self.world_size = dist.get_world_size(self.group) + + def all_reduce(self, x: torch.Tensor) -> torch.Tensor: + dist.all_reduce(x, group=self.group) + return x + + def gather( + self, input_: torch.Tensor, rank_in_group: int, dst: int = 0, dim: int = -1 + ): + # For xpu path, gather doesn't work properly together with ray + # cluster so we use all_gather instead for now. + input_size = input_.size() + # Allocate output tensor. + output_tensor = torch.empty( + (self.world_size,) + input_size, dtype=input_.dtype, device=input_.device + ) + # All-gather. + torch.distributed.all_gather_into_tensor( + output_tensor, input_, group=self.group + ) + if rank_in_group == dst: + # Reshape + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + + (self.world_size * input_size[dim],) + + input_size[dim + 1 :] + ) + else: + output_tensor = None + return output_tensor diff --git a/python/sglang/srt/distributed/naive_distributed.py b/python/sglang/srt/distributed/naive_distributed.py new file mode 100644 index 000000000..61165d90c --- /dev/null +++ b/python/sglang/srt/distributed/naive_distributed.py @@ -0,0 +1,112 @@ +import base64 +import os +import pickle +import time +from pathlib import Path +from typing import Any, List, Optional + +import torch + +from sglang.srt.utils import MultiprocessingSerializer + + +class NaiveDistributed: + def __init__(self, rank: int, world_size: int, rendezvous: str): + self._rank = rank + self._world_size = world_size + self._operation_index = 0 + self._directory = Path(rendezvous) + self._directory.mkdir(parents=True, exist_ok=True) + assert 0 <= rank < world_size + + # both barrier to be safe, and as a sanity check + self.barrier() + + def get_rank(self): + return self._rank + + def get_world_size(self): + return self._world_size + + def scatter( + self, tensor: torch.Tensor, scatter_list: List[torch.Tensor], src: int = 0 + ): + if self._rank == src: + assert len(scatter_list) == self._world_size + else: + assert scatter_list is None + + gathered_objects = self.all_gather_object( + dict( + serialized_scatter_list=[ + ( + None + if item_rank == src + else MultiprocessingSerializer.serialize(item) + ) + for item_rank, item in enumerate(scatter_list) + ] + ) + if self._rank == src + else dict() + ) + + remote_serialized_tensor = gathered_objects[src]["serialized_scatter_list"][ + self._rank + ] + if self._rank == src: + assert remote_serialized_tensor is None + remote_tensor = scatter_list[self._rank] + else: + remote_tensor = MultiprocessingSerializer.deserialize( + remote_serialized_tensor + ) + tensor.copy_(remote_tensor) + + # avoid src tensor be deleted too early + self.barrier() + + def all_gather_object(self, obj: Any) -> List[Any]: + self._operation_index += 1 + + text_postfix = "\n" + + def _get_path(interesting_rank: int): + return ( + self._directory + / f"rank{interesting_rank}_op{self._operation_index}.txt" + ) + + _get_path(self._rank).write_text( + base64.b64encode(pickle.dumps(obj)).decode("utf-8") + text_postfix + ) + + def _read_one(interesting_rank: int): + p = _get_path(interesting_rank) + while True: + if p.exists() and (text := p.read_text()).endswith(text_postfix): + return pickle.loads(base64.b64decode(text[: -len(text_postfix)])) + time.sleep(0.001) + + return [ + _read_one(interesting_rank) for interesting_rank in range(self._world_size) + ] + + def barrier(self): + actual_objs = self.all_gather_object(self._rank) + assert actual_objs == list(range(self._world_size)), f"{actual_objs=}" + + +# Can have multi instances if needed +_instance: Optional[NaiveDistributed] = None + + +def get_naive_distributed(): + assert _instance is not None + return _instance + + +def set_naive_distributed(instance: NaiveDistributed): + global _instance + assert _instance is None + _instance = instance diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py new file mode 100644 index 000000000..4f410570d --- /dev/null +++ b/python/sglang/srt/distributed/parallel_state.py @@ -0,0 +1,1776 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/parallel_state.py + +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +"""vLLM distributed state. +It takes over the control of the distributed environment from PyTorch. +The typical workflow is: + +- call `init_distributed_environment` to initialize the distributed environment. +- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to + initialize the model parallel groups. + +- any code dealing with the distributed stuff + +- call `destroy_model_parallel` to destroy the model parallel groups. +- call `destroy_distributed_environment` to destroy the distributed environment. + +If you only need to use the distributed environment without model/pipeline + parallelism, you can skip the model parallel initialization and destruction + steps. +""" +import contextlib +import gc +import logging +import os +import pickle +import weakref +from collections import namedtuple +from contextlib import contextmanager, nullcontext +from dataclasses import dataclass +from datetime import timedelta +from multiprocessing import shared_memory +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from unittest.mock import patch + +import torch +import torch.distributed +from torch.distributed import Backend, ProcessGroup + +from sglang.srt.utils import ( + direct_register_custom_op, + get_bool_env_var, + get_int_env_var, + is_cpu, + is_cuda_alike, + is_hip, + is_npu, + is_shm_available, + supports_custom_op, +) + +_is_npu = is_npu() +_is_cpu = is_cpu() + +IS_ONE_DEVICE_PER_PROCESS = get_bool_env_var("SGLANG_ONE_DEVICE_PER_PROCESS") + + +@dataclass +class GraphCaptureContext: + stream: torch.cuda.Stream if not _is_npu else torch.npu.Stream + + +TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"]) + +# use int value instead of ReduceOp.SUM to support torch compile +REDUCE_OP_SUM = int(torch.distributed.ReduceOp.SUM) + + +def _split_tensor_dict( + tensor_dict: Dict[str, Union[torch.Tensor, Any]] +) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]: + """Split the tensor dictionary into two parts: + 1. A list of (key, value) pairs. If the value is a tensor, it is replaced + by its metadata. + 2. A list of tensors. + """ + metadata_list: List[Tuple[str, Any]] = [] + tensor_list: List[torch.Tensor] = [] + for key, value in tensor_dict.items(): + if isinstance(value, torch.Tensor): + # Note: we cannot use `value.device` here, + # because it contains not only the device type but also the device + # index (e.g. "cuda:0"). We only need the device type. + # receiving side will set the device index. + device = value.device.type + metadata_list.append( + (key, TensorMetadata(device, value.dtype, value.size())) + ) + tensor_list.append(value) + else: + metadata_list.append((key, value)) + return metadata_list, tensor_list + + +_group_name_counter: Dict[str, int] = {} + + +def _get_unique_name(name: str) -> str: + """Get a unique name for the group. + Example: + _get_unique_name("tp") -> "tp:0" + _get_unique_name("tp") -> "tp:1" + """ + if name not in _group_name_counter: + _group_name_counter[name] = 0 + newname = f"{name}:{_group_name_counter[name]}" + _group_name_counter[name] += 1 + return newname + + +_groups: Dict[str, Callable[[], Optional["GroupCoordinator"]]] = {} + + +def _register_group(group: "GroupCoordinator") -> None: + _groups[group.unique_name] = weakref.ref(group) + + +if supports_custom_op(): + + def inplace_all_reduce(tensor: torch.Tensor, group_name: str) -> None: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + group._all_reduce_in_place(tensor) + + def inplace_all_reduce_fake(tensor: torch.Tensor, group_name: str) -> None: + return + + direct_register_custom_op( + op_name="inplace_all_reduce", + op_func=inplace_all_reduce, + mutates_args=["tensor"], + fake_impl=inplace_all_reduce_fake, + ) + + def outplace_all_reduce( + tensor: torch.Tensor, group_name: str, outplace_all_reduce_method: str + ) -> torch.Tensor: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + return group._all_reduce_out_place(tensor, outplace_all_reduce_method) + + def outplace_all_reduce_fake( + tensor: torch.Tensor, group_name: str, outplace_all_reduce_method: str + ) -> torch.Tensor: + return torch.empty_like(tensor) + + direct_register_custom_op( + op_name="outplace_all_reduce", + op_func=outplace_all_reduce, + mutates_args=[], + fake_impl=outplace_all_reduce_fake, + ) + + def reg_all_gather_into_tensor( + output: torch.Tensor, input: torch.Tensor, group_name: str + ) -> None: + assert group_name in _groups, f"Group {group_name} is not found." + group = _groups[group_name]() + if group is None: + raise ValueError(f"Group {group_name} is destroyed.") + group._all_gather_into_tensor(output, input) + + def reg_all_gather_into_tensor_fake( + output: torch.Tensor, input: torch.Tensor, group_name: str + ) -> None: + pass + + direct_register_custom_op( + op_name="reg_all_gather_into_tensor", + op_func=reg_all_gather_into_tensor, + mutates_args=["output"], + fake_impl=reg_all_gather_into_tensor_fake, + ) + + +class GroupCoordinator: + """ + PyTorch ProcessGroup wrapper for a group of processes. + PyTorch ProcessGroup is bound to one specific communication backend, + e.g. NCCL, Gloo, MPI, etc. + GroupCoordinator takes charge of all the communication operations among + the processes in the group. It can route the communication to + a specific implementation (e.g. switch allreduce implementation + based on the tensor size and cuda graph mode). + """ + + # available attributes: + rank: int # global rank + ranks: List[int] # global ranks in the group + world_size: int # size of the group + # difference between `local_rank` and `rank_in_group`: + # if we have a group of size 4 across two nodes: + # Process | Node | Rank | Local Rank | Rank in Group + # 0 | 0 | 0 | 0 | 0 + # 1 | 0 | 1 | 1 | 1 + # 2 | 1 | 2 | 0 | 2 + # 3 | 1 | 3 | 1 | 3 + local_rank: int # local rank used to assign devices + rank_in_group: int # rank inside the group + cpu_group: ProcessGroup # group for CPU communication + device_group: ProcessGroup # group for device communication + use_pynccl: bool # a hint of whether to use PyNccl + use_pymscclpp: bool # a hint of whether to use PyMsccl + use_custom_allreduce: bool # a hint of whether to use CustomAllreduce + use_message_queue_broadcaster: ( + bool # a hint of whether to use message queue broadcaster + ) + # communicators are only created for world size > 1 + pynccl_comm: Optional[Any] # PyNccl communicator + ca_comm: Optional[Any] # Custom allreduce communicator + mq_broadcaster: Optional[Any] # shared memory broadcaster + + def __init__( + self, + group_ranks: List[List[int]], + local_rank: int, + torch_distributed_backend: Union[str, Backend], + use_pynccl: bool, + use_pymscclpp: bool, + use_custom_allreduce: bool, + use_hpu_communicator: bool, + use_xpu_communicator: bool, + use_npu_communicator: bool, + use_message_queue_broadcaster: bool = False, + group_name: Optional[str] = None, + ): + # Set group info + group_name = group_name or "anonymous" + self.unique_name = _get_unique_name(group_name) + _register_group(self) + + # Set rank info + self.rank = torch.distributed.get_rank() + self.local_rank = local_rank + self.device_group = None + self.cpu_group = None + self.local_size = get_int_env_var("LOCAL_SIZE", 0) + + for ranks in group_ranks: + device_group = torch.distributed.new_group( + ranks, backend=torch_distributed_backend + ) + # a group with `gloo` backend, to allow direct coordination between + # processes through the CPU. + cpu_group = torch.distributed.new_group(ranks, backend="gloo") + if self.rank in ranks: + self.ranks = ranks + self.world_size = len(ranks) + self.rank_in_group = ranks.index(self.rank) + self.device_group = device_group + self.cpu_group = cpu_group + + assert self.cpu_group is not None + assert self.device_group is not None + + device_id = 0 if IS_ONE_DEVICE_PER_PROCESS else local_rank + if is_cuda_alike(): + self.device = torch.device(f"cuda:{device_id}") + elif _is_npu: + self.device = torch.device(f"npu:{device_id}") + else: + self.device = torch.device("cpu") + self.device_module = torch.get_device_module(self.device) + + # Import communicators + self.use_pynccl = use_pynccl + self.use_pymscclpp = use_pymscclpp + self.use_custom_allreduce = use_custom_allreduce + self.use_hpu_communicator = use_hpu_communicator + self.use_xpu_communicator = use_xpu_communicator + self.use_npu_communicator = use_npu_communicator + self.use_message_queue_broadcaster = use_message_queue_broadcaster + + # lazy import to avoid documentation build error + from sglang.srt.distributed.device_communicators.custom_all_reduce import ( + CustomAllreduce, + ) + from sglang.srt.distributed.device_communicators.pymscclpp import ( + PyMscclppCommunicator, + ) + from sglang.srt.distributed.device_communicators.pynccl import ( + PyNcclCommunicator, + ) + + if is_hip(): + from sglang.srt.distributed.device_communicators.quick_all_reduce import ( + QuickAllReduce, + qr_rocm_arch_available, + ) + + self.pynccl_comm: Optional[PyNcclCommunicator] = None + if use_pynccl and self.world_size > 1: + self.pynccl_comm = PyNcclCommunicator( + group=self.cpu_group, + device=self.device, + ) + + self.pymscclpp_comm: Optional[PyMscclppCommunicator] = None + if use_pymscclpp and self.world_size > 1: + self.pymscclpp_comm = PyMscclppCommunicator( + group=self.cpu_group, + device=self.device, + ) + + self.ca_comm: Optional[CustomAllreduce] = None + self.qr_comm: Optional[QuickAllReduce] = None + if use_custom_allreduce and self.world_size > 1: + # Initialize a custom fast all-reduce implementation. + try: + self.ca_comm = CustomAllreduce( + group=self.cpu_group, + device=self.device, + ) + except Exception as e: + logger.warning( + f"Setup Custom allreduce failed with {e}. To silence this " + "warning, specify --disable-custom-all-reduce explicitly." + ) + if is_hip(): + try: + # Initialize a custom quick all-reduce implementation for AMD + # when rocm >= gfx942. Quick reduce is designed as a + # complement to custom allreduce. + # Based on quickreduce (https://github.com/mk1-project/quickreduce). + if qr_rocm_arch_available(): + self.qr_comm = QuickAllReduce( + group=self.cpu_group, device=self.device + ) + except Exception as e: + logger.warning(f"Failed to initialize QuickAllReduce: {e}") + + # Create communicator for other hardware backends + from sglang.srt.distributed.device_communicators.hpu_communicator import ( + HpuCommunicator, + ) + from sglang.srt.distributed.device_communicators.npu_communicator import ( + NpuCommunicator, + ) + from sglang.srt.distributed.device_communicators.xpu_communicator import ( + XpuCommunicator, + ) + + self.hpu_communicator: Optional[HpuCommunicator] = None + if use_hpu_communicator and self.world_size > 1: + self.hpu_communicator = HpuCommunicator(group=self.device_group) + + self.xpu_communicator: Optional[XpuCommunicator] = None + if use_xpu_communicator and self.world_size > 1: + self.xpu_communicator = XpuCommunicator(group=self.device_group) + + self.npu_communicator: Optional[NpuCommunicator] = None + if use_npu_communicator and self.world_size > 1: + self.npu_communicator = NpuCommunicator(group=self.device_group) + + # Create message queue + from sglang.srt.distributed.device_communicators.shm_broadcast import ( + MessageQueue, + ) + + self.mq_broadcaster: Optional[MessageQueue] = None + if use_message_queue_broadcaster and self.world_size > 1: + self.mq_broadcaster = MessageQueue.create_from_process_group( + self.cpu_group, 1 << 22, 6 + ) + + def __repr__(self): + return ( + f"ranks={self.ranks} rank={self.rank} local_rank={self.local_rank} use_pynccl={self.use_pynccl} " + f"device_group={self.device_group} cpu_group={self.cpu_group} unique_name={self.unique_name} " + f"world_size={self.world_size} rank_in_group={self.rank_in_group}" + ) + + @property + def first_rank(self): + """Return the global rank of the first process in the group""" + return self.ranks[0] + + @property + def last_rank(self): + """Return the global rank of the last process in the group""" + return self.ranks[-1] + + @property + def is_first_rank(self): + """Return whether the caller is the first process in the group""" + return self.rank == self.first_rank + + @property + def is_last_rank(self): + """Return whether the caller is the last process in the group""" + return self.rank == self.last_rank + + @property + def next_rank(self): + """Return the global rank of the process that follows the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group + 1) % world_size] + + @property + def prev_rank(self): + """Return the global rank of the process that precedes the caller""" + rank_in_group = self.rank_in_group + world_size = self.world_size + return self.ranks[(rank_in_group - 1) % world_size] + + @contextmanager + def graph_capture( + self, graph_capture_context: Optional[GraphCaptureContext] = None + ): + if graph_capture_context is None: + stream = self.device_module.Stream() + graph_capture_context = GraphCaptureContext(stream) + else: + stream = graph_capture_context.stream + # We don't need the context of custom quick allreduce because the ipc access + # is already collected in init() and we can capture the quick allreduce directly. + ca_comm = self.ca_comm + maybe_ca_context = nullcontext() if ca_comm is None else ca_comm.capture() + + # ensure all initialization operations complete before attempting to + # capture the graph on another stream + curr_stream = self.device_module.current_stream() + if curr_stream != stream: + stream.wait_stream(curr_stream) + + with self.device_module.stream(stream), maybe_ca_context: + # In graph mode, we have to be very careful about the collective + # operations. The current status is: + # allreduce \ Mode | Eager | Graph | + # -------------------------------------------- + # quick allreduce | enabled | enabled | + # custom allreduce | enabled | enabled | + # PyNccl | disabled| enabled | + # PyMscclpp | disabled| enabled | + # torch.distributed | enabled | disabled| + # + # Note: When custom quick allreduce is enabled, a runtime check + # will be performed. If the tensor size is too small, it will + # automatically fall back to the next available option. + # Note that custom allreduce will have a runtime check, if the + # tensor size is too large, it will fallback to the next + # available option. + # Note that the PyMsccl needs to register the tensor in ahead, + # which will introduce large overhead in the eager case, + # therefore it is only supported in the graph case. + # In summary: We select the appropriate allreduce method for + # each mode based on the algorithm order in the table and + # their usage conditions. + pynccl_comm = self.pynccl_comm + maybe_pynccl_context: Any + if not pynccl_comm: + maybe_pynccl_context = nullcontext() + else: + maybe_pynccl_context = pynccl_comm.change_state( + enable=True, stream=torch.cuda.current_stream() + ) + + pymscclpp_comm = self.pymscclpp_comm + maybe_pymscclpp_context: Any + if not pymscclpp_comm: + maybe_pymscclpp_context = nullcontext() + else: + maybe_pymscclpp_context = pymscclpp_comm.change_state(enable=True) + with maybe_pynccl_context, maybe_pymscclpp_context: + yield graph_capture_context + + def all_reduce(self, input_: torch.Tensor) -> torch.Tensor: + """ + User-facing all-reduce function before we actually call the + all-reduce operation. + + We need this because Dynamo does not support passing an arbitrary + object (`self` in this case) to a custom op. We need to pass the + group name as a string, and then look up the group coordinator from + the group name, dispatch the all-reduce operation to the group + coordinator. + + In addition, PyTorch custom ops do not support mutation or returning + a new tensor in the same op. So we need to figure out if the op is + in-place or out-of-place ahead of time. + """ + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + + if input_.is_cpu: + if is_shm_available(input_.dtype, self.world_size, self.local_size): + torch.ops.sgl_kernel.shm_allreduce(input_, REDUCE_OP_SUM) + else: + torch.distributed.all_reduce(input_, group=self.device_group) + return input_ + + if not supports_custom_op(): + self._all_reduce_in_place(input_) + return input_ + + if self.hpu_communicator is not None and not self.hpu_communicator.disabled: + return self.hpu_communicator.all_reduce(input_) + + if self.xpu_communicator is not None and not self.xpu_communicator.disabled: + return self.xpu_communicator.all_reduce(input_) + + if self.npu_communicator is not None and not self.npu_communicator.disabled: + return self.npu_communicator.all_reduce(input_) + + if ( + self.pynccl_comm is not None + and hasattr(input_, "symmetric_memory") + and input_.symmetric_memory + ): + with self.pynccl_comm.change_state( + enable=True, stream=torch.cuda.current_stream() + ): + self.pynccl_comm.all_reduce(input_) + return input_ + + outplace_all_reduce_method = None + if ( + self.qr_comm is not None + and not self.qr_comm.disabled + and self.qr_comm.should_quick_allreduce(input_) + ): + outplace_all_reduce_method = "qr" + elif ( + self.ca_comm is not None + and not self.ca_comm.disabled + and self.ca_comm.should_custom_ar(input_) + ): + outplace_all_reduce_method = "ca" + elif ( + self.pymscclpp_comm is not None + and not self.pymscclpp_comm.disabled + and self.pymscclpp_comm.should_mscclpp_allreduce(input_) + ): + outplace_all_reduce_method = "pymscclpp" + if outplace_all_reduce_method is not None: + return torch.ops.sglang.outplace_all_reduce( + input_, + group_name=self.unique_name, + outplace_all_reduce_method=outplace_all_reduce_method, + ) + else: + torch.ops.sglang.inplace_all_reduce(input_, group_name=self.unique_name) + return input_ + + def _all_reduce_out_place( + self, input_: torch.Tensor, outplace_all_reduce_method: str + ) -> torch.Tensor: + qr_comm = self.qr_comm + ca_comm = self.ca_comm + pymscclpp_comm = self.pymscclpp_comm + assert any([qr_comm, ca_comm, pymscclpp_comm]) + if outplace_all_reduce_method == "qr": + assert not qr_comm.disabled + out = qr_comm.quick_all_reduce(input_) + elif outplace_all_reduce_method == "ca": + assert not ca_comm.disabled + out = ca_comm.custom_all_reduce(input_) + else: + assert not pymscclpp_comm.disabled + out = pymscclpp_comm.all_reduce(input_) + assert out is not None + return out + + def _all_reduce_in_place(self, input_: torch.Tensor) -> None: + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.all_reduce(input_) + else: + torch.distributed.all_reduce(input_, group=self.device_group) + + def reduce_scatter_tensor( + self, + output: torch.Tensor, + input: torch.Tensor, + ) -> None: + # TODO(ch-wan): support other backends + torch.distributed.reduce_scatter_tensor(output, input, group=self.device_group) + return output + + def reduce_scatter( + self, + output: torch.Tensor, + input_list: List[torch.Tensor], + ) -> None: + # TODO(ch-wan): support other backends + torch.distributed.reduce_scatter(output, input_list, group=self.device_group) + return output + + def reduce_scatterv( + self, + input_: torch.Tensor, + output: Optional[torch.Tensor] = None, + sizes: Optional[List[int]] = None, + ) -> torch.Tensor: + world_size = self.world_size + pynccl_comm = self.pynccl_comm + + with pynccl_comm.change_state(enable=True, stream=torch.cuda.current_stream()): + assert ( + pynccl_comm is not None and not pynccl_comm.disabled + ), "pynccl is required for reduce_scatterv" + + if sizes is not None: + assert len(sizes) == world_size + assert input_.shape[0] == sum(sizes) + chunk_size = sizes[self.rank_in_group] + else: + assert input_.shape[0] % world_size == 0 + chunk_size = input_.shape[0] // world_size + output_shape = (chunk_size,) + input_.shape[1:] + + if output is None: + output = torch.empty( + output_shape, dtype=input_.dtype, device=input_.device + ) + else: + assert output.shape == output_shape + + pynccl_comm.reduce_scatter(output, input_, sizes=sizes) + return output + + def _all_gather_into_tensor(self, output: torch.Tensor, input: torch.Tensor): + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.all_gather(output, input) + else: + torch.distributed.all_gather_into_tensor( + output, input, group=self.device_group + ) + + def all_gather_into_tensor(self, output: torch.Tensor, input: torch.Tensor): + if _is_npu or not supports_custom_op(): + self._all_gather_into_tensor(output, input) + else: + torch.ops.sglang.reg_all_gather_into_tensor( + output, input, group_name=self.unique_name + ) + + def all_gather( + self, + input_: torch.Tensor, + dim: int = -1, + output_tensor_list: Optional[List[torch.Tensor]] = None, + ) -> torch.Tensor: + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + if output_tensor_list is not None: + logger.warning( + "Performing in-place all-gather with a group size of 1. " + "This may be unnecessary; consider bypassing it for better efficiency." + ) + output_tensor_list[0].copy_(input_) + return None + else: + return input_ + + if output_tensor_list is not None: + # TODO(ch-wan): support other backends + return torch.distributed.all_gather( + output_tensor_list, input_, group=self.device_group + ) + + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + + # For HPUs, use HPU communicator. + hpu_comm = self.hpu_communicator + if hpu_comm is not None and not hpu_comm.disabled: + return hpu_comm.all_gather(input_, dim) + + # For NPUs, use NPU communicator. + npu_comm = self.npu_communicator + if npu_comm is not None and not npu_comm.disabled: + return npu_comm.all_gather(input_, dim) + + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + input_size = input_.size() + # NOTE: we have to use concat-style all-gather here, + # stack-style all-gather has compatibility issues with + # torch.compile . see https://github.com/pytorch/pytorch/issues/138795 + output_size = (input_size[0] * world_size,) + input_size[1:] + # Allocate output tensor. + output_tensor = torch.empty( + output_size, dtype=input_.dtype, device=input_.device + ) + + # All-gather. + if input_.is_cpu and is_shm_available( + input_.dtype, self.world_size, self.local_size + ): + return torch.ops.sgl_kernel.shm_allgather(input_, dim) + + if input_.is_cpu: + torch.distributed.all_gather_into_tensor( + output_tensor, input_, group=self.device_group + ) + else: + self.all_gather_into_tensor(output_tensor, input_) + + # Reshape + output_tensor = output_tensor.reshape((world_size,) + input_size) + output_tensor = output_tensor.movedim(0, dim) + output_tensor = output_tensor.reshape( + input_size[:dim] + (world_size * input_size[dim],) + input_size[dim + 1 :] + ) + return output_tensor + + def all_gatherv( + self, + input_: Union[torch.Tensor, List[torch.Tensor]], + sizes: Optional[List[int]] = None, + ) -> Union[torch.Tensor, List[torch.Tensor]]: + """ + Supports varying sizes per rank and input tensor list. + `sizes`: a list of len(world_size) with the number of items per rank to gather. + """ + world_size = self.world_size + pynccl_comm = self.pynccl_comm + + with pynccl_comm.change_state(enable=True, stream=torch.cuda.current_stream()): + assert ( + pynccl_comm is not None and not pynccl_comm.disabled + ), "pynccl is required for all_gatherv" + + def _all_gather_single( + input_: torch.Tensor, sizes: Optional[List[int]] = None + ): + input_size = input_.size() + if sizes is not None: + assert len(sizes) == world_size + assert input_.shape[0] == sizes[self.rank_in_group] + output_size = (sum(sizes),) + input_size[1:] + # 'sizes' is not needed if all inputs in the same group have the same shape + if all(s == sizes[0] for s in sizes): + sizes = None + else: + output_size = (input_size[0] * world_size,) + input_size[1:] + # Allocate output tensor. + output_tensor = torch.empty( + output_size, dtype=input_.dtype, device=input_.device + ) + pynccl_comm.all_gather(output_tensor, input_, sizes=sizes) + return output_tensor + + if isinstance(input_, torch.Tensor): + return _all_gather_single(input_, sizes) + + output_list = [] + pynccl_comm.group_start() + for inp in input_: + output_list.append(_all_gather_single(inp, sizes=sizes)) + pynccl_comm.group_end() + + return output_list + + def gather( + self, input_: torch.Tensor, dst: int = 0, dim: int = -1 + ) -> Optional[torch.Tensor]: + """ + NOTE: We assume that the input tensor is on the same device across + all the ranks. + NOTE: `dst` is the local rank of the destination rank. + """ + world_size = self.world_size + # Bypass the function if we are using only 1 GPU. + if world_size == 1: + return input_ + assert ( + -input_.dim() <= dim < input_.dim() + ), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}" + if dim < 0: + # Convert negative dim to positive. + dim += input_.dim() + if self.xpu_communicator is not None and not self.xpu_communicator.disabled: + return self.xpu_communicator.gather(input_, self.rank_in_group, dst, dim) + # Allocate output tensor. + if self.rank_in_group == dst: + gather_list = [torch.empty_like(input_) for _ in range(world_size)] + else: + gather_list = None + # Gather. + torch.distributed.gather( + input_, gather_list, dst=self.ranks[dst], group=self.device_group + ) + if self.rank_in_group == dst: + output_tensor = torch.cat(gather_list, dim=dim) + else: + output_tensor = None + return output_tensor + + def broadcast(self, input_: torch.Tensor, src: int = 0): + """Broadcast the input tensor. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return input_ + # Broadcast. + torch.distributed.broadcast( + input_, src=self.ranks[src], group=self.device_group + ) + return input_ + + def broadcast_object(self, obj: Optional[Any] = None, src: int = 0): + """Broadcast the input object. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj + if self.mq_broadcaster is not None: + assert src == 0, "Message queue broadcaster only supports src=0" + return self.mq_broadcaster.broadcast_object(obj) + if self.rank_in_group == src: + torch.distributed.broadcast_object_list( + [obj], src=self.ranks[src], group=self.cpu_group + ) + return obj + else: + recv = [None] + torch.distributed.broadcast_object_list( + recv, src=self.ranks[src], group=self.cpu_group + ) + return recv[0] + + def broadcast_object_list( + self, obj_list: List[Any], src: int = 0, group: Optional[ProcessGroup] = None + ): + """Broadcast the input object list. + NOTE: `src` is the local rank of the source rank. + """ + assert src < self.world_size, f"Invalid src rank ({src})" + + # Bypass the function if we are using only 1 GPU. + if self.world_size == 1: + return obj_list + # Broadcast. + torch.distributed.broadcast_object_list( + obj_list, src=self.ranks[src], group=self.device_group + ) + return obj_list + + def all_gather_object(self, obj: Any) -> List[Any]: + objs = [None] * self.world_size + torch.distributed.all_gather_object(objs, obj, group=self.cpu_group) + return objs + + def send_object(self, obj: Any, dst: int) -> None: + """Send the input object list to the destination rank.""" + """NOTE: `dst` is the local rank of the destination rank.""" + + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + assert dst != self.rank_in_group, ( + "Invalid destination rank. Destination rank is the same " + "as the current rank." + ) + + # Serialize object to tensor and get the size as well + object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8).cuda( + device=torch.cuda.current_device() + ) + + size_tensor = torch.tensor( + [object_tensor.numel()], + dtype=torch.long, + device="cpu", + ) + # Send object size + torch.distributed.send(size_tensor, dst=self.ranks[dst], group=self.cpu_group) + + # Send object + torch.distributed.send( + object_tensor, + dst=self.ranks[dst], + group=self.device_group, + ) + + return None + + def recv_object(self, src: int) -> Any: + """Receive the input object list from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + + assert src < self.world_size, f"Invalid src rank ({src})" + + assert ( + src != self.rank_in_group + ), "Invalid source rank. Source rank is the same as the current rank." + + size_tensor = torch.empty(1, dtype=torch.long, device="cpu") + + # Receive object size + rank_size = torch.distributed.recv( + size_tensor, src=self.ranks[src], group=self.cpu_group + ) + + # Tensor to receive serialized objects into. + object_tensor = torch.empty( # type: ignore[call-overload] + size_tensor.item(), # type: ignore[arg-type] + dtype=torch.uint8, + device=torch.cuda.current_device(), + ) + + rank_object = torch.distributed.recv( + object_tensor, src=self.ranks[src], group=self.device_group + ) + + assert ( + rank_object == rank_size + ), "Received object sender rank does not match the size sender rank." + + obj = pickle.loads(object_tensor.cpu().numpy()) + + return obj + + def broadcast_tensor_dict( + self, + tensor_dict: Optional[Dict[str, Union[torch.Tensor, Any]]] = None, + src: int = 0, + group: Optional[ProcessGroup] = None, + metadata_group: Optional[ProcessGroup] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Broadcast the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return tensor_dict + + group = self.device_group + metadata_group = self.cpu_group + assert src < self.world_size, f"Invalid src rank ({src})" + + rank_in_group = self.rank_in_group + if rank_in_group == src: + metadata_list: List[Tuple[Any, Any]] = [] + assert isinstance( + tensor_dict, dict + ), f"Expecting a dictionary, got {type(tensor_dict)}" + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # `metadata_list` lives in CPU memory. + # `broadcast_object_list` has serialization & deserialization, + # all happening on CPU. Therefore, we can use the CPU group. + self.broadcast_object(metadata_list, src=src) + async_handles = [] + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast( + tensor, src=self.ranks[src], group=metadata_group, async_op=True + ) + else: + # use group for GPU tensors + handle = torch.distributed.broadcast( + tensor, src=self.ranks[src], group=group, async_op=True + ) + async_handles.append(handle) + for async_handle in async_handles: + async_handle.wait() + + else: + metadata_list = self.broadcast_object(None, src=src) + tensor_dict = {} + async_handles = [] + for key, value in metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty( + value.size, dtype=value.dtype, device=value.device + ) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + tensor_dict[key] = tensor + continue + if tensor.is_cpu: + # use metadata_group for CPU tensors + handle = torch.distributed.broadcast( + tensor, + src=self.ranks[src], + group=metadata_group, + async_op=True, + ) + else: + # use group for GPU tensors + handle = torch.distributed.broadcast( + tensor, src=self.ranks[src], group=group, async_op=True + ) + async_handles.append(handle) + tensor_dict[key] = tensor + else: + tensor_dict[key] = value + for async_handle in async_handles: + async_handle.wait() + return tensor_dict + + def send_tensor_dict( + self, + tensor_dict: Dict[str, Union[torch.Tensor, Any]], + dst: Optional[int] = None, + all_gather_group: Optional["GroupCoordinator"] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Send the input tensor dictionary. + NOTE: `dst` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return tensor_dict + + all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size + all_gather_rank = ( + 0 if all_gather_group is None else all_gather_group.rank_in_group + ) + + group = self.device_group + metadata_group = self.cpu_group + + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + assert dst < self.world_size, f"Invalid dst rank ({dst})" + + assert isinstance( + tensor_dict, dict + ), f"Expecting a dictionary, got {type(tensor_dict)}" + metadata_list, tensor_list = _split_tensor_dict(tensor_dict) + # Note: While switching to Device-to-Device (D2D) would introduce an extra + # Device-to-Host (D2H) memory copy overhead for serialization, our benchmarks + # show better overall transmission performance with D2D due to: + # 1. Superior D2D transfer bandwidth + # 2. Ability to overlap send and recv operations + # Thus the net performance gain justifies this approach. + self.send_object(metadata_list, dst=dst) + for tensor in tensor_list: + if tensor.numel() == 0: + # Skip sending empty tensors. + continue + + # send-allgather: send only a slice, then do allgather. + if all_gather_group is not None and tensor.numel() % all_gather_size == 0: + tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank] + + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.send( + tensor, dst=self.ranks[dst], group=metadata_group + ) + else: + # use group for GPU tensors + torch.distributed.send(tensor, dst=self.ranks[dst], group=group) + return None + + def recv_tensor_dict( + self, + src: Optional[int] = None, + all_gather_group: Optional["GroupCoordinator"] = None, + ) -> Optional[Dict[str, Union[torch.Tensor, Any]]]: + """Recv the input tensor dictionary. + NOTE: `src` is the local rank of the source rank. + """ + # Bypass the function if we are using only 1 GPU. + if not torch.distributed.is_initialized() or self.world_size == 1: + return None + + all_gather_size = 1 if all_gather_group is None else all_gather_group.world_size + all_gather_rank = ( + 0 if all_gather_group is None else all_gather_group.rank_in_group + ) + + group = self.device_group + metadata_group = self.cpu_group + + if src is None: + src = (self.rank_in_group - 1) % self.world_size + assert src < self.world_size, f"Invalid src rank ({src})" + + recv_metadata_list = self.recv_object(src=src) + tensor_dict: Dict[str, Any] = {} + for key, value in recv_metadata_list: + if isinstance(value, TensorMetadata): + tensor = torch.empty(value.size, dtype=value.dtype, device=value.device) + if tensor.numel() == 0: + # Skip broadcasting empty tensors. + tensor_dict[key] = tensor + continue + + # send-allgather: send only a slice, then do allgather. + use_all_gather = ( + all_gather_group is not None + and tensor.numel() % all_gather_size == 0 + ) + + if use_all_gather: + orig_shape = tensor.shape + tensor = tensor.reshape(all_gather_size, -1)[all_gather_rank] + + if tensor.is_cpu: + # use metadata_group for CPU tensors + torch.distributed.recv( + tensor, src=self.ranks[src], group=metadata_group + ) + else: + # use group for GPU tensors + torch.distributed.recv(tensor, src=self.ranks[src], group=group) + if use_all_gather: + # do the allgather + tensor = all_gather_group.all_gather(tensor, dim=0) # type: ignore + tensor = tensor.reshape(orig_shape) + + tensor_dict[key] = tensor + else: + tensor_dict[key] = value + return tensor_dict + + def barrier(self): + """Barrier synchronization among the group. + NOTE: don't use `device_group` here! `barrier` in NCCL is + terrible because it is internally a broadcast operation with + secretly created GPU tensors. It is easy to mess up the current + device. Use the CPU group instead. + """ + torch.distributed.barrier(group=self.cpu_group) + + def send(self, tensor: torch.Tensor, dst: Optional[int] = None) -> None: + """Sends a tensor to the destination rank in a non-blocking way""" + """NOTE: `dst` is the local rank of the destination rank.""" + if dst is None: + dst = (self.rank_in_group + 1) % self.world_size + + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.send(tensor, dst) + else: + torch.distributed.send(tensor, self.ranks[dst], self.device_group) + + def recv( + self, size: torch.Size, dtype: torch.dtype, src: Optional[int] = None + ) -> torch.Tensor: + """Receives a tensor from the source rank.""" + """NOTE: `src` is the local rank of the source rank.""" + if src is None: + src = (self.rank_in_group - 1) % self.world_size + + tensor = torch.empty(size, dtype=dtype, device=self.device) + pynccl_comm = self.pynccl_comm + if pynccl_comm is not None and not pynccl_comm.disabled: + pynccl_comm.recv(tensor, src) + else: + torch.distributed.recv(tensor, self.ranks[src], self.device_group) + return tensor + + def destroy(self): + if self.device_group is not None: + torch.distributed.destroy_process_group(self.device_group) + self.device_group = None + if self.cpu_group is not None: + torch.distributed.destroy_process_group(self.cpu_group) + self.cpu_group = None + if self.pynccl_comm is not None: + self.pynccl_comm = None + if self.ca_comm is not None: + self.ca_comm = None + if self.mq_broadcaster is not None: + self.mq_broadcaster = None + + +_WORLD: Optional[GroupCoordinator] = None + + +def get_world_group() -> GroupCoordinator: + assert _WORLD is not None, "world group is not initialized" + return _WORLD + + +def init_world_group( + ranks: List[int], local_rank: int, backend: str +) -> GroupCoordinator: + return GroupCoordinator( + group_ranks=[ranks], + local_rank=local_rank, + torch_distributed_backend=backend, + use_pynccl=False, + use_pymscclpp=False, + use_custom_allreduce=False, + use_hpu_communicator=False, + use_xpu_communicator=False, + use_npu_communicator=False, + group_name="world", + ) + + +def init_model_parallel_group( + group_ranks: List[List[int]], + local_rank: int, + backend: str, + use_custom_allreduce: Optional[bool] = None, + use_message_queue_broadcaster: bool = False, + group_name: Optional[str] = None, + use_mscclpp_allreduce: Optional[bool] = None, +) -> GroupCoordinator: + if use_custom_allreduce is None: + use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE + if use_mscclpp_allreduce is None: + use_mscclpp_allreduce = _ENABLE_MSCCLPP_ALL_REDUCE + return GroupCoordinator( + group_ranks=group_ranks, + local_rank=local_rank, + torch_distributed_backend=backend, + use_pynccl=not _is_npu, + use_pymscclpp=use_mscclpp_allreduce, + use_custom_allreduce=use_custom_allreduce, + use_hpu_communicator=True, + use_xpu_communicator=True, + use_npu_communicator=True, + use_message_queue_broadcaster=use_message_queue_broadcaster, + group_name=group_name, + ) + + +_TP: Optional[GroupCoordinator] = None + +# duplicate GroupCoordinator for prefill in PD-Multiplexing +_PDMUX_PREFILL_TP_GROUP: Optional[GroupCoordinator] = None + +_ENABLE_PDMUX_P_TP: bool = False + + +def set_pdmux_status(enable_prefill_multiplexing: bool): + global _ENABLE_PDMUX_P_TP + _ENABLE_PDMUX_P_TP = enable_prefill_multiplexing + + +def get_tp_group() -> GroupCoordinator: + if _ENABLE_PDMUX_P_TP: + assert ( + _PDMUX_PREFILL_TP_GROUP is not None + ), "tensor model parallel group for PD-Multiplexing Prefill is not initialized" + return _PDMUX_PREFILL_TP_GROUP + assert _TP is not None, "tensor model parallel group is not initialized" + return _TP + + +_MOE_EP: Optional[GroupCoordinator] = None +_MOE_TP: Optional[GroupCoordinator] = None + + +def get_moe_ep_group() -> GroupCoordinator: + assert _MOE_EP is not None, "expert model parallel group is not initialized" + return _MOE_EP + + +def get_moe_tp_group() -> GroupCoordinator: + assert _MOE_TP is not None, "expert model parallel group is not initialized" + return _MOE_TP + + +# kept for backward compatibility +get_tensor_model_parallel_group = get_tp_group + +_PP: Optional[GroupCoordinator] = None + + +def get_pp_group() -> GroupCoordinator: + assert _PP is not None, "pipeline model parallel group is not initialized" + return _PP + + +# kept for backward compatibility +get_pipeline_model_parallel_group = get_pp_group + + +@contextmanager +def graph_capture(): + """ + `graph_capture` is a context manager which should surround the code that + is capturing the CUDA graph. Its main purpose is to ensure that the + some operations will be run after the graph is captured, before the graph + is replayed. It returns a `GraphCaptureContext` object which contains the + necessary data for the graph capture. Currently, it only contains the + stream that the graph capture is running on. This stream is set to the + current CUDA stream when the context manager is entered and reset to the + default stream when the context manager is exited. This is to ensure that + the graph capture is running on a separate stream from the default stream, + in order to explicitly distinguish the kernels to capture + from other kernels possibly launched on background in the default stream. + """ + with get_tp_group().graph_capture() as context, get_pp_group().graph_capture( + context + ): + yield context + + +logger = logging.getLogger(__name__) + +_ENABLE_CUSTOM_ALL_REDUCE = True +_ENABLE_MSCCLPP_ALL_REDUCE = False + + +def set_custom_all_reduce(enable: bool): + global _ENABLE_CUSTOM_ALL_REDUCE + _ENABLE_CUSTOM_ALL_REDUCE = enable + + +def set_mscclpp_all_reduce(enable: bool): + global _ENABLE_MSCCLPP_ALL_REDUCE + _ENABLE_MSCCLPP_ALL_REDUCE = enable + + +def init_distributed_environment( + world_size: int = -1, + rank: int = -1, + distributed_init_method: str = "env://", + local_rank: int = -1, + backend: str = "nccl", + timeout: Optional[int] = None, +): + logger.debug( + "world_size=%d rank=%d local_rank=%d " "distributed_init_method=%s backend=%s", + world_size, + rank, + local_rank, + distributed_init_method, + backend, + ) + if not torch.distributed.is_initialized(): + assert distributed_init_method is not None, ( + "distributed_init_method must be provided when initializing " + "distributed environment" + ) + if timeout is not None: + assert isinstance(timeout, (int)), "timeout must be a number" + assert timeout > 0, "timeout must be positive" + timeout = timedelta(seconds=timeout) + + # this backend is used for WORLD + torch.distributed.init_process_group( + backend=backend, + init_method=distributed_init_method, + world_size=world_size, + rank=rank, + timeout=timeout, + ) + + # set the local rank + # local_rank is not available in torch ProcessGroup, + # see https://github.com/pytorch/pytorch/issues/122816 + if local_rank == -1: + # local rank not set, this usually happens in single-node + # setting, where we can use rank as local rank + if distributed_init_method == "env://": + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + else: + local_rank = rank + global _WORLD + if _WORLD is None: + ranks = list(range(torch.distributed.get_world_size())) + _WORLD = init_world_group(ranks, local_rank, backend) + else: + assert ( + _WORLD.world_size == torch.distributed.get_world_size() + ), "world group already initialized with a different world size" + + +def initialize_model_parallel( + tensor_model_parallel_size: int = 1, + expert_model_parallel_size: int = 1, + pipeline_model_parallel_size: int = 1, + backend: Optional[str] = None, + duplicate_tp_group: bool = False, +) -> None: + """ + Initialize model parallel groups. + + Arguments: + tensor_model_parallel_size: number of GPUs used for tensor model + parallelism. + pipeline_model_parallel_size: number of GPUs used for pipeline model + parallelism. + + Let's say we have a total of 8 GPUs denoted by g0 ... g7 and we + use 2 GPUs to parallelize the model tensor, and 4 GPUs to parallelize + the model pipeline. The present function will + create 4 tensor model-parallel groups and 2 pipeline model-parallel groups: + 4 tensor model-parallel groups: + [g0, g1], [g2, g3], [g4, g5], [g6, g7] + 2 pipeline model-parallel groups: + [g0, g2, g4, g6], [g1, g3, g5, g7] + Note that for efficiency, the caller should make sure adjacent ranks + are on the same DGX box. For example if we are using 2 DGX-1 boxes + with a total of 16 GPUs, rank 0 to 7 belong to the first box and + ranks 8 to 15 belong to the second box. + """ + # Get world size and rank. Ensure some consistencies. + assert torch.distributed.is_initialized() + world_size: int = torch.distributed.get_world_size() + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + + if world_size != tensor_model_parallel_size * pipeline_model_parallel_size: + raise RuntimeError( + f"world_size ({world_size}) is not equal to " + f"tensor_model_parallel_size ({tensor_model_parallel_size}) x " + f"pipeline_model_parallel_size ({pipeline_model_parallel_size})" + ) + + # Build the tensor model-parallel groups. + num_tensor_model_parallel_groups: int = world_size // tensor_model_parallel_size + global _TP + assert _TP is None, "tensor model parallel group is already initialized" + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + ranks = list( + range(i * tensor_model_parallel_size, (i + 1) * tensor_model_parallel_size) + ) + group_ranks.append(ranks) + + # message queue broadcaster is only used in tensor model parallel group + _TP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + use_message_queue_broadcaster=get_bool_env_var( + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER", "true" + ), + group_name="tp", + ) + + if duplicate_tp_group: + global _PDMUX_PREFILL_TP_GROUP + assert ( + _PDMUX_PREFILL_TP_GROUP is None + ), "tensor model parallel group for PD-Multiplexing Prefill is already initialized" + _PDMUX_PREFILL_TP_GROUP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + use_message_queue_broadcaster=get_bool_env_var( + "SGLANG_USE_MESSAGE_QUEUE_BROADCASTER", "true" + ), + group_name="pdmux_prefill_tp", + ) + _TP.pynccl_comm.disabled = False + _PDMUX_PREFILL_TP_GROUP.pynccl_comm.disabled = False + + moe_ep_size = expert_model_parallel_size + moe_tp_size = tensor_model_parallel_size // moe_ep_size + + global _MOE_EP + assert _MOE_EP is None, "expert model parallel group is already initialized" + + if moe_ep_size == tensor_model_parallel_size: + _MOE_EP = _TP + else: + # TODO(ch-wan): use split_group to save memory + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + for j in range(moe_tp_size): + st = i * tensor_model_parallel_size + j + en = (i + 1) * tensor_model_parallel_size + j + ranks = list(range(st, en, moe_tp_size)) + group_ranks.append(ranks) + _MOE_EP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + group_name="moe_ep", + ) + + global _MOE_TP + assert _MOE_TP is None, "expert model parallel group is already initialized" + + if moe_tp_size == tensor_model_parallel_size: + _MOE_TP = _TP + else: + # TODO(ch-wan): use split_group to save memory + group_ranks = [] + for i in range(num_tensor_model_parallel_groups): + for j in range(moe_ep_size): + st = i * tensor_model_parallel_size + j * moe_tp_size + en = i * tensor_model_parallel_size + (j + 1) * moe_tp_size + ranks = list(range(st, en)) + group_ranks.append(ranks) + _MOE_TP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + group_name="moe_tp", + ) + + # Build the pipeline model-parallel groups. + num_pipeline_model_parallel_groups: int = world_size // pipeline_model_parallel_size + global _PP + assert _PP is None, "pipeline model parallel group is already initialized" + group_ranks = [] + for i in range(num_pipeline_model_parallel_groups): + ranks = list(range(i, world_size, num_pipeline_model_parallel_groups)) + group_ranks.append(ranks) + # pipeline parallel does not need custom allreduce + _PP = init_model_parallel_group( + group_ranks, + get_world_group().local_rank, + backend, + use_custom_allreduce=False, + group_name="pp", + ) + + +def ensure_model_parallel_initialized( + tensor_model_parallel_size: int, + expert_model_parallel_size: int, + pipeline_model_parallel_size: int, + backend: Optional[str] = None, +) -> None: + """Helper to initialize model parallel groups if they are not initialized, + or ensure tensor-parallel and pipeline-parallel sizes are equal to expected + values if the model parallel groups are initialized. + """ + backend = backend or torch.distributed.get_backend(get_world_group().device_group) + if not model_parallel_is_initialized(): + initialize_model_parallel( + tensor_model_parallel_size, + expert_model_parallel_size, + pipeline_model_parallel_size, + backend, + ) + return + + assert get_tensor_model_parallel_world_size() == tensor_model_parallel_size, ( + "tensor parallel group already initialized, but of unexpected size: " + f"{get_tensor_model_parallel_world_size()=} vs. " + f"{tensor_model_parallel_size=}" + ) + pp_world_size = get_pp_group().world_size + assert pp_world_size == pipeline_model_parallel_size, ( + "pipeline parallel group already initialized, but of unexpected size: " + f"{pp_world_size=} vs. " + f"{pipeline_model_parallel_size=}" + ) + + +def model_parallel_is_initialized(): + """Check if tensor and pipeline parallel groups are initialized.""" + return _TP is not None and _PP is not None + + +_TP_STATE_PATCHED = False + + +@contextmanager +def patch_tensor_parallel_group(tp_group: GroupCoordinator): + """Patch the tp group temporarily until this function ends. + + This method is for draft workers of speculative decoding to run draft model + with different tp degree from that of target model workers. + + Args: + tp_group (GroupCoordinator): the tp group coordinator + """ + global _TP_STATE_PATCHED + assert not _TP_STATE_PATCHED, "Should not call when it's already patched" + + _TP_STATE_PATCHED = True + old_tp_group = get_tp_group() + global _TP + _TP = tp_group + try: + yield + finally: + # restore the original state + _TP_STATE_PATCHED = False + _TP = old_tp_group + + +def get_world_size(): + """Return world size for the world group.""" + return get_world_group().world_size + + +def get_world_rank(): + """Return my rank for the world group.""" + return get_world_group().rank_in_group + + +def get_tensor_model_parallel_world_size(): + """Return world size for the tensor model parallel group.""" + return get_tp_group().world_size + + +def get_tensor_model_parallel_rank(): + """Return my rank for the tensor model parallel group.""" + return get_tp_group().rank_in_group + + +def get_pipeline_model_parallel_world_size(): + """Return world size for the pipeline model parallel group.""" + return get_pp_group().world_size + + +def get_pipeline_model_parallel_rank(): + """Return my rank for the pipeline model parallel group.""" + return get_pp_group().rank_in_group + + +def get_moe_expert_parallel_world_size(): + """Return world size for the moe expert parallel group.""" + return get_moe_ep_group().world_size + + +def get_moe_expert_parallel_rank(): + """Return my rank for the moe expert parallel group.""" + return get_moe_ep_group().rank_in_group + + +def get_moe_tensor_parallel_world_size(): + """Return world size for the moe tensor parallel group.""" + return get_moe_tp_group().world_size + + +def get_moe_tensor_parallel_rank(): + """Return my rank for the moe tensor parallel group.""" + return get_moe_tp_group().rank_in_group + + +def destroy_model_parallel(): + """Set the groups to none and destroy them.""" + global _TP + if _TP: + _TP.destroy() + _TP = None + + global _PP + if _PP: + _PP.destroy() + _PP = None + + +def destroy_distributed_environment(): + global _WORLD + if _WORLD: + _WORLD.destroy() + _WORLD = None + if torch.distributed.is_initialized(): + torch.distributed.destroy_process_group() + + +def cleanup_dist_env_and_memory(shutdown_ray: bool = False): + destroy_model_parallel() + destroy_distributed_environment() + with contextlib.suppress(AssertionError): + torch.distributed.destroy_process_group() + if shutdown_ray: + import ray # Lazy import Ray + + ray.shutdown() + gc.collect() + if not _is_cpu: + if hasattr(torch, "cuda") and torch.cuda.is_available(): + torch.cuda.empty_cache() + if hasattr(torch._C, "_host_emptyCache"): + torch._C._host_emptyCache() + else: + logger.warning( + "torch._C._host_emptyCache() only available in Pytorch >=2.5" + ) + elif hasattr(torch, "xpu") and torch.xpu.is_available(): + torch.xpu.empty_cache() + elif hasattr(torch, "npu") and torch.npu.is_available(): + torch.npu.empty_cache() + + +def in_the_same_node_as(pg: ProcessGroup, source_rank: int = 0) -> List[bool]: + """ + This is a collective operation that returns if each rank is in the same node + as the source rank. It tests if processes are attached to the same + memory system (shared access to shared memory). + """ + assert ( + torch.distributed.get_backend(pg) != torch.distributed.Backend.NCCL + ), "in_the_same_node_as should be tested with a non-NCCL group." + # local rank inside the group + rank = torch.distributed.get_rank(group=pg) + world_size = torch.distributed.get_world_size(group=pg) + + # local tensor in each process to store the result + is_in_the_same_node = torch.tensor([0] * world_size, dtype=torch.int32) + + # global ranks of the processes in the group + ranks = torch.distributed.get_process_group_ranks(pg) + + magic_message = b"magic_message" + shm = None + + try: + with contextlib.suppress(OSError): + if rank == source_rank: + # create a shared memory segment + shm = shared_memory.SharedMemory(create=True, size=128) + shm.buf[: len(magic_message)] = magic_message + torch.distributed.broadcast_object_list( + [shm.name], src=ranks[source_rank], group=pg + ) + is_in_the_same_node[rank] = 1 + else: + # try to open the shared memory segment + recv = [None] + torch.distributed.broadcast_object_list( + recv, src=ranks[source_rank], group=pg + ) + name = recv[0] + # fix to https://stackoverflow.com/q/62748654/9191338 + # Python incorrectly tracks shared memory even if it is not + # created by the process. The following patch is a workaround. + with patch( + "multiprocessing.resource_tracker.register", + lambda *args, **kwargs: None, + ): + shm = shared_memory.SharedMemory(name=name) + if shm.buf[: len(magic_message)] == magic_message: + is_in_the_same_node[rank] = 1 + except Exception as e: + logger.error("Error ignored in is_in_the_same_node: %s", e) + finally: + if shm: + shm.close() + + torch.distributed.barrier(group=pg) + + # clean up the shared memory segment + with contextlib.suppress(OSError): + if rank == source_rank and shm: + shm.unlink() + torch.distributed.all_reduce(is_in_the_same_node, group=pg) + + return [x == 1 for x in is_in_the_same_node.tolist()] + + +vllm_get_pp_group = None +vllm_get_tp_group = None +vllm_get_world_group = None + + +def monkey_patch_vllm_parallel_state(reverse: bool = False): + try: + import vllm.distributed.parallel_state as vllm_parrlel_state + except ImportError: + return + + global vllm_get_pp_group, vllm_get_tp_group, vllm_get_world_group + if vllm_get_pp_group is None: + vllm_get_pp_group = vllm_parrlel_state.get_pp_group + vllm_get_tp_group = vllm_parrlel_state.get_tp_group + vllm_get_world_group = vllm_parrlel_state.get_world_group + if reverse: + setattr(vllm_parrlel_state, "get_pp_group", vllm_get_pp_group) + setattr(vllm_parrlel_state, "get_tp_group", vllm_get_tp_group) + setattr(vllm_parrlel_state, "get_world_group", vllm_get_world_group) + else: + setattr(vllm_parrlel_state, "get_pp_group", get_pp_group) + setattr(vllm_parrlel_state, "get_tp_group", get_tp_group) + setattr(vllm_parrlel_state, "get_world_group", get_world_group) diff --git a/python/sglang/srt/distributed/utils.py b/python/sglang/srt/distributed/utils.py new file mode 100644 index 000000000..bfe54b9d4 --- /dev/null +++ b/python/sglang/srt/distributed/utils.py @@ -0,0 +1,224 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/distributed/utils.py + +# Copyright 2023 The vLLM team. +# Adapted from +# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/tensor_parallel/utils.py +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +import dataclasses +import logging +import os +import pickle +import time +from collections import deque +from typing import Any, Deque, Dict, Optional, Sequence, Tuple + +import torch +from torch.distributed import TCPStore + +logger = logging.getLogger(__name__) + + +def ensure_divisibility(numerator, denominator): + """Ensure that numerator is divisible by the denominator.""" + assert numerator % denominator == 0, "{} is not divisible by {}".format( + numerator, denominator + ) + + +def divide(numerator, denominator): + """Ensure that numerator is divisible by the denominator and return + the division value.""" + ensure_divisibility(numerator, denominator) + return numerator // denominator + + +def split_tensor_along_last_dim( + tensor: torch.Tensor, + num_partitions: int, + contiguous_split_chunks: bool = False, +) -> Sequence[torch.Tensor]: + """Split a tensor along its last dimension. + + Arguments: + tensor: input tensor. + num_partitions: number of partitions to split the tensor + contiguous_split_chunks: If True, make each chunk contiguous + in memory. + + Returns: + A list of Tensors + """ + # Get the size and dimension. + last_dim = tensor.dim() - 1 + last_dim_size = divide(tensor.size()[last_dim], num_partitions) + # Split. + tensor_list = torch.split(tensor, last_dim_size, dim=last_dim) + # NOTE: torch.split does not create contiguous tensors by default. + if contiguous_split_chunks: + return tuple(chunk.contiguous() for chunk in tensor_list) + + return tensor_list + + +def get_pp_indices( + num_hidden_layers: int, pp_rank: int, pp_size: int +) -> Tuple[int, int]: + """Try to evenly distribute layers across partitions. + If the number of layers is not divisible by the number of partitions, + the last partition will have the remaining layers. + """ + # partition_list_str can be set to None in sglang + partition_list_str = os.getenv("SGLANG_PP_LAYER_PARTITION", None) + if partition_list_str is not None: + try: + partitions = [int(layer) for layer in partition_list_str.split(",")] + except ValueError as err: + raise ValueError( + "Invalid partition string: {}".format(partition_list_str) + ) from err + if len(partitions) != pp_size: + raise ValueError(f"{len(partitions)=} does not match {pp_size=}.") + if sum(partitions) != num_hidden_layers: + raise ValueError(f"{sum(partitions)=} does not match {num_hidden_layers=}.") + start_layer = sum(partitions[:pp_rank]) + end_layer = start_layer + partitions[pp_rank] + else: + layers_per_partition = num_hidden_layers // pp_size + start_layer = pp_rank * layers_per_partition + end_layer = start_layer + layers_per_partition + + if pp_rank == pp_size - 1: + end_layer = num_hidden_layers + + return (start_layer, end_layer) + + +@dataclasses.dataclass +class StatelessProcessGroup: + """A dataclass to hold a metadata store, and the rank, world_size of the + group. Only use it to communicate metadata between processes. + For data-plane communication, create NCCL-related objects. + """ + + rank: int + world_size: int + store: torch._C._distributed_c10d.Store + data_expiration_seconds: int = 3600 # 1 hour + + # dst rank -> counter + send_dst_counter: Dict[int, int] = dataclasses.field(default_factory=dict) + # src rank -> counter + recv_src_counter: Dict[int, int] = dataclasses.field(default_factory=dict) + broadcast_send_counter: int = 0 + broadcast_recv_src_counter: Dict[int, int] = dataclasses.field(default_factory=dict) + + # A deque to store the data entries, with key and timestamp. + entries: Deque[Tuple[str, float]] = dataclasses.field(default_factory=deque) + + def __post_init__(self): + assert self.rank < self.world_size + self.send_dst_counter = {i: 0 for i in range(self.world_size)} + self.recv_src_counter = {i: 0 for i in range(self.world_size)} + self.broadcast_recv_src_counter = {i: 0 for i in range(self.world_size)} + + def send_obj(self, obj: Any, dst: int): + """Send an object to a destination rank.""" + self.expire_data() + key = f"send_to/{dst}/{self.send_dst_counter[dst]}" + self.store.set(key, pickle.dumps(obj)) + self.send_dst_counter[dst] += 1 + self.entries.append((key, time.perf_counter())) + + def expire_data(self): + """Expire data that is older than `data_expiration_seconds` seconds.""" + while self.entries: + # check the oldest entry + key, timestamp = self.entries[0] + if time.perf_counter() - timestamp > self.data_expiration_seconds: + self.store.delete_key(key) + self.entries.popleft() + else: + break + + def recv_obj(self, src: int) -> Any: + """Receive an object from a source rank.""" + obj = pickle.loads( + self.store.get(f"send_to/{self.rank}/{self.recv_src_counter[src]}") + ) + self.recv_src_counter[src] += 1 + return obj + + def broadcast_obj(self, obj: Optional[Any], src: int) -> Any: + """Broadcast an object from a source rank to all other ranks. + It does not clean up after all ranks have received the object. + Use it for limited times, e.g., for initialization. + """ + if self.rank == src: + self.expire_data() + key = f"broadcast_from/{src}/" f"{self.broadcast_send_counter}" + self.store.set(key, pickle.dumps(obj)) + self.broadcast_send_counter += 1 + self.entries.append((key, time.perf_counter())) + return obj + else: + key = f"broadcast_from/{src}/" f"{self.broadcast_recv_src_counter[src]}" + recv_obj = pickle.loads(self.store.get(key)) + self.broadcast_recv_src_counter[src] += 1 + return recv_obj + + def all_gather_obj(self, obj: Any) -> list[Any]: + """All gather an object from all ranks.""" + gathered_objs = [] + for i in range(self.world_size): + if i == self.rank: + gathered_objs.append(obj) + self.broadcast_obj(obj, src=self.rank) + else: + recv_obj = self.broadcast_obj(None, src=i) + gathered_objs.append(recv_obj) + return gathered_objs + + def barrier(self): + """A barrier to synchronize all ranks.""" + for i in range(self.world_size): + if i == self.rank: + self.broadcast_obj(None, src=self.rank) + else: + self.broadcast_obj(None, src=i) + + @staticmethod + def create( + host: str, + port: int, + rank: int, + world_size: int, + data_expiration_seconds: int = 3600, + ) -> "StatelessProcessGroup": + """A replacement for `torch.distributed.init_process_group` that does not + pollute the global state. + + If we have process A and process B called `torch.distributed.init_process_group` + to form a group, and then we want to form another group with process A, B, C, + D, it is not possible in PyTorch, because process A and process B have already + formed a group, and process C and process D cannot join that group. This + function is a workaround for this issue. + + `torch.distributed.init_process_group` is a global call, while this function + is a stateless call. It will return a `StatelessProcessGroup` object that can be + used for exchanging metadata. With this function, process A and process B + can call `StatelessProcessGroup.create` to form a group, and then process A, B, + C, and D can call `StatelessProcessGroup.create` to form another group. + """ # noqa + store = TCPStore( + host_name=host, + port=port, + world_size=world_size, + is_master=(rank == 0), + ) + + return StatelessProcessGroup( + rank=rank, + world_size=world_size, + store=store, + data_expiration_seconds=data_expiration_seconds, + ) diff --git a/python/sglang/srt/entrypoints/EngineBase.py b/python/sglang/srt/entrypoints/EngineBase.py new file mode 100644 index 000000000..42ecb12aa --- /dev/null +++ b/python/sglang/srt/entrypoints/EngineBase.py @@ -0,0 +1,72 @@ +from abc import ABC, abstractmethod +from typing import Dict, Iterator, List, Optional, Tuple, Union + +import torch + + +class EngineBase(ABC): + """ + Abstract base class for engine interfaces that support generation, weight updating, and memory control. + This base class provides a unified API for both HTTP-based engines and engines. + """ + + @abstractmethod + def generate( + self, + prompt: Optional[Union[List[str], str]] = None, + sampling_params: Optional[Union[List[Dict], Dict]] = None, + input_ids: Optional[Union[List[List[int]], List[int]]] = None, + image_data: Optional[Union[List[str], str]] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + token_ids_logprob: Optional[Union[List[List[int]], List[int]]] = None, + lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None, + custom_logit_processor: Optional[Union[List[str], str]] = None, + return_hidden_states: Optional[bool] = None, + stream: Optional[bool] = None, + bootstrap_host: Optional[Union[List[str], str]] = None, + bootstrap_port: Optional[Union[List[int], int]] = None, + bootstrap_room: Optional[Union[List[int], int]] = None, + data_parallel_rank: Optional[int] = None, + ) -> Union[Dict, Iterator[Dict]]: + """Generate outputs based on given inputs.""" + pass + + @abstractmethod + def flush_cache(self): + """Flush the cache of the engine.""" + pass + + @abstractmethod + def update_weights_from_tensor( + self, + named_tensors: List[Tuple[str, torch.Tensor]], + load_format: Optional[str] = None, + flush_cache: bool = True, + ): + """Update model weights with in-memory tensor data.""" + pass + + def load_lora_adapter(self, lora_name: str, lora_path: str): + """Load a new LoRA adapter without re-launching the engine.""" + pass + + def unload_lora_adapter(self, lora_name: str): + """Unload a LoRA adapter without re-launching the engine.""" + pass + + @abstractmethod + def release_memory_occupation(self): + """Release GPU memory occupation temporarily.""" + pass + + @abstractmethod + def resume_memory_occupation(self): + """Resume GPU memory occupation which is previously released.""" + pass + + @abstractmethod + def shutdown(self): + """Shutdown the engine and clean up resources.""" + pass diff --git a/python/sglang/srt/entrypoints/context.py b/python/sglang/srt/entrypoints/context.py new file mode 100644 index 000000000..66f58200f --- /dev/null +++ b/python/sglang/srt/entrypoints/context.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copied from vLLM +import json +import logging +from abc import ABC, abstractmethod +from typing import Union + +logger = logging.getLogger(__name__) + +try: + from mcp import ClientSession +except ImportError as e: + mcp = e + +from openai_harmony import Author, Message, Role, StreamState, TextContent + +from sglang.srt.entrypoints.harmony_utils import ( + get_encoding, + get_streamable_parser_for_assistant, + render_for_completion, +) +from sglang.srt.entrypoints.tool import Tool + + +class ConversationContext(ABC): + + @abstractmethod + def append_output(self, output) -> None: + pass + + @abstractmethod + async def call_tool(self) -> list[Message]: + pass + + @abstractmethod + def need_builtin_tool_call(self) -> bool: + pass + + @abstractmethod + def render_for_completion(self) -> list[int]: + pass + + +class SimpleContext(ConversationContext): + + def __init__(self): + self.last_output = None + + def append_output(self, output) -> None: + self.last_output = output + + def need_builtin_tool_call(self) -> bool: + return False + + async def call_tool(self) -> list[Message]: + raise NotImplementedError("Should not be called.") + + def render_for_completion(self) -> list[int]: + raise NotImplementedError("Should not be called.") + + +class HarmonyContext(ConversationContext): + + def __init__( + self, + messages: list, + tool_sessions: dict[str, Union["ClientSession", Tool]], + ): + # TODO: Remove the hack of Union[ClientSession, Tool] by using MCP + # when demo. + self._messages = messages + self.tool_sessions = tool_sessions + + self.parser = get_streamable_parser_for_assistant() + self.num_init_messages = len(messages) + # TODO + self.num_prompt_tokens = 0 + self.num_cached_tokens = 0 + self.num_output_tokens = 0 + self.num_reasoning_tokens = 0 + + def append_output(self, output) -> None: + if isinstance(output, dict) and "output_ids" in output: + output_token_ids = output["output_ids"] + + # TODO: REMOVE here: + # Very hacky, find the first occurrence of token 200006 and cut from there + try: + start_index = output_token_ids.index(200006) + output_token_ids = output_token_ids[start_index:] + except ValueError: + pass + + for token_id in output_token_ids: + self.parser.process(token_id) + output_msgs = self.parser.messages + + meta_info = output["meta_info"] + + if isinstance(meta_info, dict): + if "prompt_token_ids" in meta_info: + self.num_prompt_tokens = meta_info["prompt_tokens"] + if "cached_tokens" in meta_info: + self.num_cached_tokens = meta_info["cached_tokens"] + if "completion_tokens" in meta_info: + self.num_output_tokens += meta_info["completion_tokens"] + + else: + output_msgs = output + + self._messages.extend(output_msgs) + + @property + def messages(self) -> list: + return self._messages + + def need_builtin_tool_call(self) -> bool: + if not self.messages: + return False + last_msg = self.messages[-1] + recipient = last_msg.recipient + return recipient is not None and ( + recipient.startswith("browser.") or recipient.startswith("python") + ) + + async def call_tool(self) -> list[Message]: + if not self.messages: + return [] + last_msg = self.messages[-1] + recipient = last_msg.recipient + if recipient is not None: + if recipient.startswith("browser."): + return await self.call_search_tool( + self.tool_sessions["browser"], last_msg + ) + elif recipient.startswith("python"): + return await self.call_python_tool( + self.tool_sessions["python"], last_msg + ) + raise ValueError("No tool call found") + + def render_for_completion(self) -> list[int]: + return render_for_completion(self.messages) + + async def call_search_tool( + self, tool_session: Union["ClientSession", Tool], last_msg: Message + ) -> list[Message]: + if isinstance(tool_session, Tool): + return await tool_session.get_result(self) + tool_name = last_msg.recipient.split(".")[1] + args = json.loads(last_msg.content[0].text) + result = await tool_session.call_tool(tool_name, args) + result_str = result.content[0].text + content = TextContent(text=result_str) + author = Author(role=Role.TOOL, name=last_msg.recipient) + return [Message(author=author, content=[content], recipient=Role.ASSISTANT)] + + async def call_python_tool( + self, tool_session: Union["ClientSession", Tool], last_msg: Message + ) -> list[Message]: + if isinstance(tool_session, Tool): + return await tool_session.get_result(self) + param = { + "code": last_msg.content[0].text, + } + result = await tool_session.call_tool("python", param) + result_str = result.content[0].text + + content = TextContent(text=result_str) + author = Author(role=Role.TOOL, name="python") + + return [ + Message( + author=author, + content=[content], + channel=last_msg.channel, + recipient=Role.ASSISTANT, + ) + ] + + +class StreamingHarmonyContext(HarmonyContext): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.last_output = None + + self.parser = get_streamable_parser_for_assistant() + self.encoding = get_encoding() + self.last_tok = None + + @property + def messages(self) -> list: + return self.parser.messages + + def append_output(self, output) -> None: + if isinstance(output, dict) and "output_ids" in output: + # RequestOutput from SGLang with outputs + output_token_ids = output["output_ids"] + + # TODO: REMOVE here: + # Very hacky, find the first occurrence of token 200006 and cut from there + # Find the first occurrence of token 200006 and cut from there + try: + start_index = output_token_ids.index(200006) + output_token_ids = output_token_ids[start_index:] + except ValueError: + pass + + for token_id in output_token_ids: + self.parser.process(token_id) + + else: + # Handle the case of tool output in direct message format + assert len(output) == 1, "Tool output should be a single message" + msg = output[0] + # Sometimes the recipient is not set for tool messages, + # so we set it to "assistant" + if msg.author.role == Role.TOOL and msg.recipient is None: + msg.recipient = "assistant" + toks = self.encoding.render(msg) + for tok in toks: + self.parser.process(tok) + self.last_tok = toks[-1] + + def is_expecting_start(self) -> bool: + return self.parser.state == StreamState.EXPECT_START + + def is_assistant_action_turn(self) -> bool: + return self.last_tok in self.encoding.stop_tokens_for_assistant_actions() + + def render_for_completion(self) -> list[int]: + # now this list of tokens as next turn's starting tokens + # `<|start|>assistant``, + # we need to process them in parser. + rendered_tokens = super().render_for_completion() + + last_n = -1 + to_process = [] + while rendered_tokens[last_n] != self.last_tok: + to_process.append(rendered_tokens[last_n]) + last_n -= 1 + for tok in reversed(to_process): + self.parser.process(tok) + + return rendered_tokens diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py new file mode 100644 index 000000000..fbd923d91 --- /dev/null +++ b/python/sglang/srt/entrypoints/engine.py @@ -0,0 +1,872 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The entry point of inference server. (SRT = SGLang Runtime) + +This file implements python APIs for the inference engine. +""" + +import asyncio +import atexit +import dataclasses +import logging +import multiprocessing as mp +import os +import random +import signal +import threading +import time +from typing import AsyncIterator, Dict, Iterator, List, Optional, Tuple, Union + +import zmq +import zmq.asyncio +from PIL.Image import Image + +# Fix a bug of Python threading +setattr(threading, "_register_atexit", lambda *args, **kwargs: None) + +import torch +import uvloop + +from sglang.srt.entrypoints.EngineBase import EngineBase +from sglang.srt.managers.data_parallel_controller import ( + run_data_parallel_controller_process, +) +from sglang.srt.managers.detokenizer_manager import run_detokenizer_process +from sglang.srt.managers.io_struct import ( + EmbeddingReqInput, + GenerateReqInput, + GetWeightsByNameReqInput, + InitWeightsUpdateGroupReqInput, + LoadLoRAAdapterReqInput, + MultimodalDataInputFormat, + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, + RpcReqInput, + RpcReqOutput, + UnloadLoRAAdapterReqInput, + UpdateWeightFromDiskReqInput, + UpdateWeightsFromDistributedReqInput, + UpdateWeightsFromTensorReqInput, +) +from sglang.srt.managers.multi_tokenizer_mixin import MultiTokenizerRouter +from sglang.srt.managers.scheduler import run_scheduler_process +from sglang.srt.managers.template_manager import TemplateManager +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter +from sglang.srt.utils import ( + MultiprocessingSerializer, + assert_pkg_version, + configure_logger, + get_bool_env_var, + get_zmq_socket, + is_cuda, + kill_process_tree, + launch_dummy_health_check_server, + prepare_model_and_tokenizer, + set_prometheus_multiproc_dir, + set_ulimit, +) +from sglang.version import __version__ + +logger = logging.getLogger(__name__) +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + +_is_cuda = is_cuda() + + +class Engine(EngineBase): + """ + The entry point to the inference engine. + + - The engine consists of three components: + 1. TokenizerManager: Tokenizes the requests and sends them to the scheduler. + 2. Scheduler (subprocess): Receives requests from the Tokenizer Manager, schedules batches, forwards them, and sends the output tokens to the Detokenizer Manager. + 3. DetokenizerManager (subprocess): Detokenizes the output tokens and sends the result back to the Tokenizer Manager. + + Note: + 1. The HTTP server, Engine, and TokenizerManager all run in the main process. + 2. Inter-process communication (IPC) is handled via the ZMQ library, with each process using a different port. + """ + + def __init__(self, **kwargs): + """ + The arguments of this function is the same as `sglang/srt/server_args.py::ServerArgs`. + Please refer to `ServerArgs` for the documentation. + """ + if "server_args" in kwargs: + # Directly load server_args + server_args = kwargs["server_args"] + else: + # Construct server_args from kwargs + if "log_level" not in kwargs: + # Do not print logs by default + kwargs["log_level"] = "error" + server_args = ServerArgs(**kwargs) + + # Shutdown the subprocesses automatically when the program exits + atexit.register(self.shutdown) + + # Allocate ports for inter-process communications + self.port_args = PortArgs.init_new(server_args) + logger.info(f"{server_args=}") + + # Launch subprocesses + tokenizer_manager, template_manager, scheduler_info = _launch_subprocesses( + server_args=server_args, + port_args=self.port_args, + ) + self.server_args = server_args + self.tokenizer_manager = tokenizer_manager + self.template_manager = template_manager + self.scheduler_info = scheduler_info + + context = zmq.Context(2) + self.send_to_rpc = get_zmq_socket( + context, zmq.DEALER, self.port_args.rpc_ipc_name, True + ) + + def generate( + self, + # The input prompt. It can be a single prompt or a batch of prompts. + prompt: Optional[Union[List[str], str]] = None, + sampling_params: Optional[Union[List[Dict], Dict]] = None, + # The token ids for text; one can either specify text or input_ids. + input_ids: Optional[Union[List[List[int]], List[int]]] = None, + # The image input. It can be an image instance, file name, URL, or base64 encoded string. + # Can be formatted as: + # - Single image for a single request + # - List of images (one per request in a batch) + # - List of lists of images (multiple images per request) + # See also python/sglang/srt/utils.py:load_image for more details. + image_data: Optional[MultimodalDataInputFormat] = None, + audio_data: Optional[MultimodalDataInputFormat] = None, + video_data: Optional[MultimodalDataInputFormat] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + token_ids_logprob: Optional[Union[List[List[int]], List[int]]] = None, + lora_path: Optional[List[Optional[str]]] = None, + custom_logit_processor: Optional[Union[List[str], str]] = None, + return_hidden_states: bool = False, + stream: bool = False, + bootstrap_host: Optional[Union[List[str], str]] = None, + bootstrap_port: Optional[Union[List[int], int]] = None, + bootstrap_room: Optional[Union[List[int], int]] = None, + data_parallel_rank: Optional[int] = None, + ) -> Union[Dict, Iterator[Dict]]: + """ + The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`. + Please refer to `GenerateReqInput` for the documentation. + """ + if self.server_args.enable_dp_attention: + if data_parallel_rank is None: + logger.debug("data_parallel_rank not provided, using default dispatch") + elif data_parallel_rank < 0: + raise ValueError("data_parallel_rank must be non-negative") + elif data_parallel_rank >= self.server_args.dp_size: + raise ValueError( + f"data_parallel_rank must be less than dp_size: {self.server_args.dp_size}" + ) + + obj = GenerateReqInput( + text=prompt, + input_ids=input_ids, + sampling_params=sampling_params, + image_data=image_data, + audio_data=audio_data, + video_data=video_data, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + token_ids_logprob=token_ids_logprob, + lora_path=lora_path, + custom_logit_processor=custom_logit_processor, + return_hidden_states=return_hidden_states, + stream=stream, + bootstrap_host=bootstrap_host, + bootstrap_port=bootstrap_port, + bootstrap_room=bootstrap_room, + data_parallel_rank=data_parallel_rank, + ) + loop = asyncio.get_event_loop() + generator = self.tokenizer_manager.generate_request(obj, None) + + if stream: + + def generator_wrapper(): + while True: + try: + chunk = loop.run_until_complete(generator.__anext__()) + yield chunk + except StopAsyncIteration: + break + + return generator_wrapper() + else: + ret = loop.run_until_complete(generator.__anext__()) + return ret + + async def async_generate( + self, + # The input prompt. It can be a single prompt or a batch of prompts. + prompt: Optional[Union[List[str], str]] = None, + sampling_params: Optional[Union[List[Dict], Dict]] = None, + # The token ids for text; one can either specify text or input_ids. + input_ids: Optional[Union[List[List[int]], List[int]]] = None, + # The image input. It can be an image instance, file name, URL, or base64 encoded string. + # Can be formatted as: + # - Single image for a single request + # - List of images (one per request in a batch) + # - List of lists of images (multiple images per request) + # See also python/sglang/srt/utils.py:load_image for more details. + image_data: Optional[MultimodalDataInputFormat] = None, + audio_data: Optional[MultimodalDataInputFormat] = None, + video_data: Optional[MultimodalDataInputFormat] = None, + return_logprob: Optional[Union[List[bool], bool]] = False, + logprob_start_len: Optional[Union[List[int], int]] = None, + top_logprobs_num: Optional[Union[List[int], int]] = None, + token_ids_logprob: Optional[Union[List[List[int]], List[int]]] = None, + lora_path: Optional[List[Optional[str]]] = None, + custom_logit_processor: Optional[Union[List[str], str]] = None, + return_hidden_states: bool = False, + stream: bool = False, + bootstrap_host: Optional[Union[List[str], str]] = None, + bootstrap_port: Optional[Union[List[int], int]] = None, + bootstrap_room: Optional[Union[List[int], int]] = None, + data_parallel_rank: Optional[int] = None, + ) -> Union[Dict, AsyncIterator[Dict]]: + """ + The arguments of this function is the same as `sglang/srt/managers/io_struct.py::GenerateReqInput`. + Please refer to `GenerateReqInput` for the documentation. + """ + + if self.server_args.enable_dp_attention: + if data_parallel_rank is None: + logger.debug("data_parallel_rank not provided, using default dispatch") + elif data_parallel_rank < 0: + raise ValueError("data_parallel_rank must be non-negative") + elif data_parallel_rank >= self.server_args.dp_size: + raise ValueError( + f"data_parallel_rank must be in range [0, {self.server_args.dp_size-1}]" + ) + + logger.debug(f"data_parallel_rank: {data_parallel_rank}") + obj = GenerateReqInput( + text=prompt, + input_ids=input_ids, + sampling_params=sampling_params, + image_data=image_data, + audio_data=audio_data, + video_data=video_data, + return_logprob=return_logprob, + logprob_start_len=logprob_start_len, + top_logprobs_num=top_logprobs_num, + token_ids_logprob=token_ids_logprob, + lora_path=lora_path, + return_hidden_states=return_hidden_states, + stream=stream, + custom_logit_processor=custom_logit_processor, + bootstrap_host=bootstrap_host, + bootstrap_port=bootstrap_port, + bootstrap_room=bootstrap_room, + data_parallel_rank=data_parallel_rank, + ) + generator = self.tokenizer_manager.generate_request(obj, None) + + if stream is True: + return generator + else: + return await generator.__anext__() + + def encode( + self, + prompt: Union[str, List[str], List[Dict], List[List[Dict]]], + image_data: Optional[MultimodalDataInputFormat] = None, + audio_data: Optional[MultimodalDataInputFormat] = None, + video_data: Optional[MultimodalDataInputFormat] = None, + ) -> Dict: + """ + The arguments of this function is the same as `sglang/srt/managers/io_struct.py::EmbeddingReqInput`. + Please refer to `EmbeddingReqInput` for the documentation. + """ + obj = EmbeddingReqInput( + text=prompt, + image_data=image_data, + audio_data=audio_data, + video_data=video_data, + ) + loop = asyncio.get_event_loop() + generator = self.tokenizer_manager.generate_request(obj, None) + ret = loop.run_until_complete(generator.__anext__()) + return ret + + async def async_encode( + self, + prompt: Union[str, List[str], List[Dict], List[List[Dict]]], + image_data: Optional[MultimodalDataInputFormat] = None, + audio_data: Optional[MultimodalDataInputFormat] = None, + video_data: Optional[MultimodalDataInputFormat] = None, + ) -> Dict: + """ + Asynchronous version of encode method. + + The arguments of this function is the same as `sglang/srt/managers/io_struct.py::EmbeddingReqInput`. + Please refer to `EmbeddingReqInput` for the documentation. + """ + obj = EmbeddingReqInput( + text=prompt, + image_data=image_data, + audio_data=audio_data, + video_data=video_data, + ) + generator = self.tokenizer_manager.generate_request(obj, None) + return await generator.__anext__() + + def rerank( + self, + prompt: Union[List[List[str]]], + ) -> Dict: + """ + The arguments of this function is the same as `sglang/srt/managers/io_struct.py::EmbeddingReqInput`. + Please refer to `EmbeddingReqInput` for the documentation. + """ + obj = EmbeddingReqInput(text=prompt, is_cross_encoder_request=True) + loop = asyncio.get_event_loop() + generator = self.tokenizer_manager.generate_request(obj, None) + ret = loop.run_until_complete(generator.__anext__()) + return ret + + def shutdown(self): + """Shutdown the engine""" + kill_process_tree(os.getpid(), include_parent=False) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.shutdown() + return False + + def flush_cache(self): + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.tokenizer_manager.flush_cache()) + + def start_profile(self): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.tokenizer_manager.start_profile()) + + def stop_profile(self): + loop = asyncio.get_event_loop() + loop.run_until_complete(self.tokenizer_manager.stop_profile()) + + def start_expert_distribution_record(self): + loop = asyncio.get_event_loop() + loop.run_until_complete( + self.tokenizer_manager.start_expert_distribution_record() + ) + + def stop_expert_distribution_record(self): + loop = asyncio.get_event_loop() + loop.run_until_complete( + self.tokenizer_manager.stop_expert_distribution_record() + ) + + def dump_expert_distribution_record(self): + loop = asyncio.get_event_loop() + loop.run_until_complete( + self.tokenizer_manager.dump_expert_distribution_record() + ) + + def get_server_info(self): + loop = asyncio.get_event_loop() + internal_states = loop.run_until_complete( + self.tokenizer_manager.get_internal_state() + ) + return { + **dataclasses.asdict(self.tokenizer_manager.server_args), + **self.scheduler_info, + "internal_states": internal_states, + "version": __version__, + } + + def init_weights_update_group( + self, + master_address: str, + master_port: int, + rank_offset: int, + world_size: int, + group_name: str, + backend: str = "nccl", + ): + """Initialize parameter update group.""" + obj = InitWeightsUpdateGroupReqInput( + master_address=master_address, + master_port=master_port, + rank_offset=rank_offset, + world_size=world_size, + group_name=group_name, + backend=backend, + ) + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.init_weights_update_group(obj, None) + ) + + def update_weights_from_distributed( + self, + names: list[str], + dtypes: list[str], + shapes: list[list[int]], + group_name: str = "weight_update_group", + flush_cache: bool = True, + ): + """Update weights from distributed source.""" + obj = UpdateWeightsFromDistributedReqInput( + names=names, + dtypes=dtypes, + shapes=shapes, + group_name=group_name, + flush_cache=flush_cache, + ) + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.update_weights_from_distributed(obj, None) + ) + + def update_weights_from_tensor( + self, + named_tensors: List[Tuple[str, torch.Tensor]], + load_format: Optional[str] = None, + flush_cache: bool = True, + ): + """Update weights from distributed source. If there are going to be more updates, set `flush_cache` to be false + to avoid duplicated cache cleaning operation.""" + if load_format == "flattened_bucket": + serialized_named_tensors = named_tensors + else: + serialized_named_tensors = [ + MultiprocessingSerializer.serialize(named_tensors) + for _ in range(self.server_args.tp_size) + ] + obj = UpdateWeightsFromTensorReqInput( + serialized_named_tensors=serialized_named_tensors, + load_format=load_format, + flush_cache=flush_cache, + ) + loop = asyncio.get_event_loop() + + return loop.run_until_complete( + self.tokenizer_manager.update_weights_from_tensor(obj, None) + ) + + def update_weights_from_disk( + self, + model_path: str, + load_format: Optional[str] = None, + ): + """Update the weights from disk inplace without re-launching the engine. + + This method allows updating the model weights from disk without restarting + the engine. It can be used to load a different model or update weights with + new training. + """ + obj = UpdateWeightFromDiskReqInput( + model_path=model_path, + load_format=load_format, + ) + + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.update_weights_from_disk(obj, None) + ) + + def get_weights_by_name(self, name: str, truncate_size: int = 100): + """Get weights by parameter name.""" + obj = GetWeightsByNameReqInput(name=name, truncate_size=truncate_size) + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.get_weights_by_name(obj, None) + ) + + def load_lora_adapter(self, lora_name: str, lora_path: str, pinned: bool = False): + """Load a new LoRA adapter without re-launching the engine.""" + + obj = LoadLoRAAdapterReqInput( + lora_name=lora_name, + lora_path=lora_path, + pinned=pinned, + ) + + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.load_lora_adapter(obj, None) + ) + + def unload_lora_adapter(self, lora_name: str): + """Unload a LoRA adapter without re-launching the engine.""" + + obj = UnloadLoRAAdapterReqInput(lora_name=lora_name) + + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.unload_lora_adapter(obj, None) + ) + + def release_memory_occupation(self, tags: Optional[List[str]] = None): + obj = ReleaseMemoryOccupationReqInput(tags=tags) + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.release_memory_occupation(obj, None) + ) + + def resume_memory_occupation(self, tags: Optional[List[str]] = None): + obj = ResumeMemoryOccupationReqInput(tags=tags) + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.resume_memory_occupation(obj, None) + ) + + def freeze_gc(self): + """ + To maintain a high performance server with low latency, we want to reduce the + stalls caused by the garbage collector scanning through a large number of objects. + + It is usually helpful to start the server and warm it up with real requests to + initialize many of the long-lived objects that do not need to be garbage collected. + + After sufficient warmup, we can call this function to freeze the garbage collector + so that all objects created before this point are considered out of scope for garbage + collection. + """ + + loop = asyncio.get_event_loop() + loop.run_until_complete(self.tokenizer_manager.freeze_gc()) + + """ + Execute an RPC call on all scheduler processes. + """ + + def collective_rpc(self, method: str, **kwargs): + obj = RpcReqInput(method=method, parameters=kwargs) + self.send_to_rpc.send_pyobj(obj) + recv_req = self.send_to_rpc.recv_pyobj(zmq.BLOCKY) + assert isinstance(recv_req, RpcReqOutput) + assert recv_req.success, recv_req.message + + def save_remote_model(self, **kwargs): + self.collective_rpc("save_remote_model", **kwargs) + + def save_sharded_model(self, **kwargs): + self.collective_rpc("save_sharded_model", **kwargs) + + def score( + self, + query: Optional[Union[str, List[int]]] = None, + items: Optional[Union[str, List[str], List[List[int]]]] = None, + label_token_ids: Optional[List[int]] = None, + apply_softmax: bool = False, + item_first: bool = False, + ) -> List[List[float]]: + """ + Score the probability of specified token IDs appearing after the given (query + item) pair. For example: + query = "<|user|>Is the following city the capital of France? " + items = ["Paris <|assistant|>", "London <|assistant|>", "Berlin <|assistant|>"] + label_token_ids = [2332, 1223] # Token IDs for "Yes" and "No" + item_first = False + + This would pass the following prompts to the model: + "<|user|>Is the following city the capital of France? Paris <|assistant|>" + "<|user|>Is the following city the capital of France? London <|assistant|>" + "<|user|>Is the following city the capital of France? Berlin <|assistant|>" + The api would then return the probabilities of the model producing "Yes" and "No" as the next token. + The output would look like: + [[0.9, 0.1], [0.2, 0.8], [0.1, 0.9]] + + + Args: + query: The query text or pre-tokenized query token IDs. Must be provided. + items: The item text(s) or pre-tokenized item token IDs. Must be provided. + label_token_ids: List of token IDs to compute probabilities for. If None, no token probabilities will be computed. + apply_softmax: Whether to normalize probabilities using softmax. + item_first: If True, prepend items to query. Otherwise append items to query. + + Returns: + List of dictionaries mapping token IDs to their probabilities for each item. + Each dictionary in the list corresponds to one item input. + + Raises: + ValueError: If query is not provided, or if items is not provided, + or if token IDs are out of vocabulary, or if logprobs are not available for the specified tokens. + """ + loop = asyncio.get_event_loop() + return loop.run_until_complete( + self.tokenizer_manager.score_request( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=apply_softmax, + item_first=item_first, + request=None, + ) + ) + + async def async_score( + self, + query: Optional[Union[str, List[int]]] = None, + items: Optional[Union[str, List[str], List[List[int]]]] = None, + label_token_ids: Optional[List[int]] = None, + apply_softmax: bool = False, + item_first: bool = False, + ) -> List[List[float]]: + """ + Asynchronous version of score method. + + See score() for detailed documentation. + """ + return await self.tokenizer_manager.score_request( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=apply_softmax, + item_first=item_first, + request=None, + ) + + +def _set_envs_and_config(server_args: ServerArgs): + # Set global environments + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = str(int(server_args.enable_symm_mem)) + if not server_args.enable_symm_mem: + os.environ["NCCL_NVLS_ENABLE"] = str(int(server_args.enable_nccl_nvls)) + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + # flashinfer uses this environment variable for various kernels from MoE to quant kernels + if os.environ.get("TRTLLM_ENABLE_PDL", "1") != "0": + os.environ["TRTLLM_ENABLE_PDL"] = "1" + + # Can also be passed as argument + os.environ["SGLANG_RUN_ID"] = ( + f"sglang-run-{time.time()}-{random.randint(0, 100000000)}" + ) + + # Set prometheus env vars + if server_args.enable_metrics: + set_prometheus_multiproc_dir() + + # Set ulimit + set_ulimit() + + # Check flashinfer version + if server_args.attention_backend == "flashinfer": + assert_pkg_version( + "flashinfer_python", + "0.3.1", + "Please uninstall the old version and " + "reinstall the latest version by following the instructions " + "at https://docs.flashinfer.ai/installation.html.", + ) + if _is_cuda and not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"): + assert_pkg_version( + "sgl-kernel", + "0.3.9.post2", + "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`", + ) + + if True: # Keep this check for internal code compatibility + # Register the signal handler. + # The child processes will send SIGQUIT to this process when any error happens + # This process then clean up the whole process tree + # Note: This sigquit handler is used in the launch phase, and may be replaced by + # the running_phase_sigquit_handler in the tokenizer manager after the grpc server is launched. + def launch_phase_sigquit_handler(signum, frame): + logger.error( + "Received sigquit from a child process. It usually means the child failed." + ) + kill_process_tree(os.getpid()) + + signal.signal(signal.SIGQUIT, launch_phase_sigquit_handler) + + # Set mp start method + mp.set_start_method("spawn", force=True) + + +def _init_tokenizer_manager( + server_args: ServerArgs, port_args: PortArgs +) -> TokenizerManager: + # Launch tokenizer process + tokenizer_manager = TokenizerManager(server_args, port_args) + + # Initialize templates + template_manager = TemplateManager() + template_manager.initialize_templates( + tokenizer_manager=tokenizer_manager, + model_path=server_args.model_path, + chat_template=server_args.chat_template, + completion_template=server_args.completion_template, + ) + + return tokenizer_manager, template_manager + + +def _launch_subprocesses( + server_args: ServerArgs, port_args: Optional[PortArgs] = None +) -> Tuple[TokenizerManager, TemplateManager, Dict]: + """ + Launch the TokenizerManager in the main process, the Scheduler in a subprocess, and the DetokenizerManager in another subprocess. + """ + # Configure global environment + configure_logger(server_args) + server_args.check_server_args() + _set_envs_and_config(server_args) + + # Allocate ports for inter-process communications + if port_args is None: + port_args = PortArgs.init_new(server_args) + logger.info(f"{server_args=}") + + # If using model from www.modelscope.cn, first download the model. + server_args.model_path, server_args.tokenizer_path = prepare_model_and_tokenizer( + server_args.model_path, server_args.tokenizer_path + ) + + scheduler_procs = [] + if server_args.dp_size == 1: + memory_saver_adapter = TorchMemorySaverAdapter.create( + enable=server_args.enable_memory_saver + ) + scheduler_pipe_readers = [] + + nnodes_per_tp_group = max(server_args.nnodes // server_args.pp_size, 1) + tp_size_per_node = server_args.tp_size // nnodes_per_tp_group + tp_rank_range = range( + tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group), + tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group + 1), + ) + + pp_size_per_node = max(server_args.pp_size // server_args.nnodes, 1) + pp_rank_range = range( + pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group), + pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group + 1), + ) + + for pp_rank in pp_rank_range: + for tp_rank in tp_rank_range: + reader, writer = mp.Pipe(duplex=False) + gpu_id = ( + server_args.base_gpu_id + + ((pp_rank % pp_size_per_node) * tp_size_per_node) + + (tp_rank % tp_size_per_node) * server_args.gpu_id_step + ) + moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size) + proc = mp.Process( + target=run_scheduler_process, + args=( + server_args, + port_args, + gpu_id, + tp_rank, + moe_ep_rank, + pp_rank, + None, + writer, + None, + ), + ) + + with memory_saver_adapter.configure_subprocess(): + proc.start() + scheduler_procs.append(proc) + scheduler_pipe_readers.append(reader) + else: + # Launch the data parallel controller + reader, writer = mp.Pipe(duplex=False) + scheduler_pipe_readers = [reader] + proc = mp.Process( + target=run_data_parallel_controller_process, + args=(server_args, port_args, writer), + ) + proc.start() + scheduler_procs.append(proc) + + if server_args.node_rank >= 1: + # In multi-node cases, non-zero rank nodes do not need to run tokenizer or detokenizer, + # so they can just wait here. + + for reader in scheduler_pipe_readers: + data = reader.recv() + assert data["status"] == "ready" + + if os.getenv("SGLANG_BLOCK_NONZERO_RANK_CHILDREN") == "0": + # When using `Engine` as a Python API, we don't want to block here. + return None, None, None + + launch_dummy_health_check_server( + server_args.host, server_args.port, server_args.enable_metrics + ) + + for proc in scheduler_procs: + proc.join() + logger.error( + f"Scheduler or DataParallelController {proc.pid} terminated with {proc.exitcode}" + ) + return None, None, None + + # Launch detokenizer process + detoken_proc = mp.Process( + target=run_detokenizer_process, + args=( + server_args, + port_args, + ), + ) + detoken_proc.start() + + # Init tokenizer manager first, as the bootstrap server is initialized here + if server_args.tokenizer_worker_num > 1: + # Launch multi-tokenizer router + tokenizer_manager = MultiTokenizerRouter(server_args, port_args) + template_manager = None + else: + tokenizer_manager, template_manager = _init_tokenizer_manager( + server_args, port_args + ) + + # Wait for the model to finish loading + scheduler_infos = [] + for i in range(len(scheduler_pipe_readers)): + try: + data = scheduler_pipe_readers[i].recv() + except EOFError: + logger.error( + f"Rank {i} scheduler is dead. Please check if there are relevant logs." + ) + scheduler_procs[i].join() + logger.error(f"Exit code: {scheduler_procs[i].exitcode}") + raise + + if data["status"] != "ready": + raise RuntimeError( + "Initialization failed. Please see the error messages above." + ) + scheduler_infos.append(data) + + # Assume all schedulers have the same scheduler_info + scheduler_info = scheduler_infos[0] + + tokenizer_manager.max_req_input_len = scheduler_info["max_req_input_len"] + + return tokenizer_manager, template_manager, scheduler_info diff --git a/python/sglang/srt/entrypoints/harmony_utils.py b/python/sglang/srt/entrypoints/harmony_utils.py new file mode 100644 index 000000000..5ebb653b3 --- /dev/null +++ b/python/sglang/srt/entrypoints/harmony_utils.py @@ -0,0 +1,372 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM: https://github.com/vllm-project/vllm/blob/1b9902806915040ac9b3029f2ab7522ec505afc3/vllm/entrypoints/harmony_utils.py +# Slight differences in processing chat messages +import datetime +import json +from collections.abc import Iterable +from typing import Literal, Optional, Union + +from openai.types.responses import ( + ResponseOutputItem, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, +) +from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall +from openai.types.responses.response_function_web_search import ( + ActionFind, + ActionOpenPage, + ActionSearch, + ResponseFunctionWebSearch, +) +from openai.types.responses.response_reasoning_item import ( + Content as ResponseReasoningTextContent, +) +from openai.types.responses.tool import Tool +from openai_harmony import ( + Author, + Conversation, + DeveloperContent, + HarmonyEncodingName, + Message, + ReasoningEffort, + Role, + StreamableParser, + SystemContent, + TextContent, + ToolDescription, + load_harmony_encoding, +) + +from sglang.srt.entrypoints.openai.protocol import ResponseInputOutputItem +from sglang.srt.utils import random_uuid + +REASONING_EFFORT = { + "high": ReasoningEffort.HIGH, + "medium": ReasoningEffort.MEDIUM, + "low": ReasoningEffort.LOW, +} + +_harmony_encoding = None + + +def get_encoding(): + global _harmony_encoding + if _harmony_encoding is None: + _harmony_encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) + return _harmony_encoding + + +def get_system_message( + model_identity: Optional[str] = None, + reasoning_effort: Optional[Literal["high", "medium", "low"]] = None, + start_date: Optional[str] = None, + browser_description: Optional[str] = None, + python_description: Optional[str] = None, +) -> Message: + sys_msg_content = SystemContent.new() + if model_identity is not None: + sys_msg_content = sys_msg_content.with_model_identity(model_identity) + if reasoning_effort is not None: + sys_msg_content = sys_msg_content.with_reasoning_effort( + REASONING_EFFORT[reasoning_effort] + ) + if start_date is None: + start_date = datetime.datetime.now().strftime("%Y-%m-%d") + sys_msg_content = sys_msg_content.with_conversation_start_date(start_date) + if browser_description is not None: + sys_msg_content = sys_msg_content.with_tools(browser_description) + if python_description is not None: + sys_msg_content = sys_msg_content.with_tools(python_description) + sys_msg = Message.from_role_and_content(Role.SYSTEM, sys_msg_content) + return sys_msg + + +def get_developer_message( + instructions: Optional[str] = None, tools: Optional[list[Tool]] = None +) -> Message: + dev_msg_content = DeveloperContent.new() + if instructions is not None: + dev_msg_content = dev_msg_content.with_instructions(instructions) + if tools is not None: + function_tools = [] + for tool in tools: + if tool.type in ("web_search_preview", "code_interpreter"): + # These are built-in tools that are added to the system message. + pass + elif tool.type == "function": + function_tools.append(tool) + else: + raise ValueError(f"tool type {tool.type} not supported") + if function_tools: + function_tool_descriptions = [ + ToolDescription.new( + name=tool.name, + description=tool.description, + parameters=tool.parameters, + ) + for tool in function_tools + ] + dev_msg_content = dev_msg_content.with_function_tools( + function_tool_descriptions + ) + dev_msg = Message.from_role_and_content(Role.DEVELOPER, dev_msg_content) + return dev_msg + + +def get_user_message(content: str) -> Message: + return Message.from_role_and_content(Role.USER, content) + + +def parse_response_input( + response_msg: ResponseInputOutputItem, + prev_responses: list[Union[ResponseOutputItem, ResponseReasoningItem]], +) -> Message: + if not isinstance(response_msg, dict): + response_msg = response_msg.model_dump() + if "type" not in response_msg or response_msg["type"] == "message": + role = response_msg["role"] + content = response_msg["content"] + if role == "system": + # User is trying to set a system message. Change it to: + # <|start|>developer<|message|># Instructions + # {instructions}<|end|> + role = "developer" + text_prefix = "Instructions:\n" + else: + text_prefix = "" + if isinstance(content, str): + msg = Message.from_role_and_content(role, text_prefix + content) + else: + contents = [TextContent(text=text_prefix + c["text"]) for c in content] + msg = Message.from_role_and_contents(role, contents) + elif response_msg["type"] == "function_call_output": + call_id = response_msg["call_id"] + call_response: Optional[ResponseFunctionToolCall] = None + for prev_response in reversed(prev_responses): + if ( + isinstance(prev_response, ResponseFunctionToolCall) + and prev_response.call_id == call_id + ): + call_response = prev_response + break + if call_response is None: + raise ValueError(f"No call message found for {call_id}") + msg = Message.from_author_and_content( + Author.new(Role.TOOL, f"functions.{call_response.name}"), + response_msg["output"], + ) + elif response_msg["type"] == "reasoning": + content = response_msg["content"] + assert len(content) == 1 + msg = Message.from_role_and_content(Role.ASSISTANT, content[0]["text"]) + elif response_msg["type"] == "function_call": + msg = Message.from_role_and_content(Role.ASSISTANT, response_msg["arguments"]) + msg = msg.with_channel("commentary") + msg = msg.with_recipient(f"functions.{response_msg['name']}") + msg = msg.with_content_type("json") + else: + raise ValueError(f"Unknown input type: {response_msg['type']}") + return msg + + +def parse_response_output(output: ResponseOutputItem) -> Message: + if isinstance(output, ResponseOutputMessage): + role = output.role + contents = [TextContent(text=c.text) for c in output.content] + msg = Message.from_role_and_contents(role, contents) + return msg + elif isinstance(output, ResponseFunctionToolCall): + msg = Message.from_role_and_content(Role.ASSISTANT, output.arguments) + msg = msg.with_channel("commentary") + msg = msg.with_recipient(output.name) + msg = msg.with_content_type("json") + return msg + else: + raise ValueError(f"Unknown output type: {type(output)}") + + +def parse_chat_input(chat_msg) -> Message: + role = chat_msg.role + content = chat_msg.content + if isinstance(content, str): + contents = [TextContent(text=content)] + else: + # TODO: Support refusal. + contents = [TextContent(text=c.text) for c in content] + msg = Message.from_role_and_contents(role, contents) + return msg + + +def render_for_completion(messages: list[Message]) -> list[int]: + conversation = Conversation.from_messages(messages) + token_ids = get_encoding().render_conversation_for_completion( + conversation, Role.ASSISTANT + ) + return token_ids + + +def get_stop_tokens_for_assistant_actions() -> list[int]: + return get_encoding().stop_tokens_for_assistant_actions() + + +def get_streamable_parser_for_assistant() -> StreamableParser: + return StreamableParser(get_encoding(), role=Role.ASSISTANT) + + +def parse_output_message(message: Message): + if message.author.role != "assistant": + # This is a message from a tool to the assistant (e.g., search result). + # Don't include it in the final output for now. This aligns with + # OpenAI's behavior on models like o4-mini. + return [] + + output_items = [] + recipient = message.recipient + if recipient is not None and recipient.startswith("browser."): + if len(message.content) != 1: + raise ValueError("Invalid number of contents in browser message") + content = message.content[0] + browser_call = json.loads(content.text) + # TODO: translate to url properly! + if recipient == "browser.search": + action = ActionSearch( + query=f"cursor:{browser_call.get('query', '')}", type="search" + ) + elif recipient == "browser.open": + action = ActionOpenPage( + url=f"cursor:{browser_call.get('url', '')}", type="open_page" + ) + elif recipient == "browser.find": + action = ActionFind( + pattern=browser_call["pattern"], + url=f"cursor:{browser_call.get('url', '')}", + type="find", + ) + else: + raise ValueError(f"Unknown browser action: {recipient}") + web_search_item = ResponseFunctionWebSearch( + id=f"ws_{random_uuid()}", + action=action, + status="completed", + type="web_search_call", + ) + output_items.append(web_search_item) + elif message.channel == "analysis": + for content in message.content: + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + type="reasoning", + summary=[], + content=[ + ResponseReasoningTextContent( + text=content.text, type="reasoning_text" + ) + ], + status=None, + ) + output_items.append(reasoning_item) + elif message.channel == "commentary": + if message.recipient.startswith("functions."): + function_name = message.recipient.split(".")[-1] + for content in message.content: + random_id = random_uuid() + response_item = ResponseFunctionToolCall( + arguments=content.text, + call_id=f"call_{random_id}", + type="function_call", + name=function_name, + id=f"ft_{random_id}", + ) + output_items.append(response_item) + elif message.recipient.startswith("python") or message.recipient.startswith( + "browser" + ): + for content in message.content: + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + type="reasoning", + summary=[], + content=[ + ResponseReasoningTextContent( + text=content.text, type="reasoning_text" + ) + ], + status=None, + ) + output_items.append(reasoning_item) + else: + raise ValueError(f"Unknown recipient: {message.recipient}") + elif message.channel == "final": + contents = [] + for content in message.content: + output_text = ResponseOutputText( + text=content.text, + annotations=[], # TODO + type="output_text", + logprobs=None, # TODO + ) + contents.append(output_text) + text_item = ResponseOutputMessage( + id=f"msg_{random_uuid()}", + content=contents, + role=message.author.role, + status="completed", + type="message", + ) + output_items.append(text_item) + else: + raise ValueError(f"Unknown channel: {message.channel}") + return output_items + + +def parse_remaining_state(parser: StreamableParser): + if not parser.current_content: + return [] + if parser.current_role != Role.ASSISTANT: + return [] + current_recipient = parser.current_recipient + if current_recipient is not None and current_recipient.startswith("browser."): + return [] + + if parser.current_channel == "analysis": + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + type="reasoning", + summary=[], + content=[ + ResponseReasoningTextContent( + text=parser.current_content, type="reasoning_text" + ) + ], + status=None, + ) + return [reasoning_item] + elif parser.current_channel == "final": + output_text = ResponseOutputText( + content=[ + ResponseReasoningTextContent( + text=parser.current_content, type="reasoning_text" + ) + ], + annotations=[], # TODO + type="output_text", + logprobs=None, # TODO + ) + text_item = ResponseOutputMessage( + id=f"msg_{random_uuid()}", + content=[output_text], + role="assistant", + status="completed", + type="message", + ) + return [text_item] + return [] + + +def parse_output_into_messages(token_ids: Iterable[int]): + parser = get_streamable_parser_for_assistant() + for token_id in token_ids: + parser.process(token_id) + return parser diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py new file mode 100644 index 000000000..9adac76ce --- /dev/null +++ b/python/sglang/srt/entrypoints/http_server.py @@ -0,0 +1,1398 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The entry point of inference server. (SRT = SGLang Runtime) + +This file implements HTTP APIs for the inference engine via fastapi. +""" + +import asyncio +import dataclasses +import json +import logging +import multiprocessing as multiprocessing +import os +import tempfile +import threading +import time +from http import HTTPStatus +from typing import Any, AsyncIterator, Callable, Dict, List, Optional + +import setproctitle + +# Fix a bug of Python threading +setattr(threading, "_register_atexit", lambda *args, **kwargs: None) + +from contextlib import asynccontextmanager +from typing import AsyncGenerator + +import numpy as np +import orjson +import requests +import uvicorn +import uvloop +from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile +from fastapi.exceptions import RequestValidationError +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import ORJSONResponse, Response, StreamingResponse + +from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST, DisaggregationMode +from sglang.srt.entrypoints.engine import _launch_subprocesses +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + CompletionRequest, + EmbeddingRequest, + ErrorResponse, + ModelCard, + ModelList, + ResponsesRequest, + ScoringRequest, + V1RerankReqInput, +) +from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat +from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion +from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from sglang.srt.entrypoints.openai.serving_rerank import OpenAIServingRerank +from sglang.srt.entrypoints.openai.serving_score import OpenAIServingScore +from sglang.srt.function_call.function_call_parser import FunctionCallParser +from sglang.srt.managers.io_struct import ( + AbortReq, + CloseSessionReqInput, + ConfigureLoggingReq, + EmbeddingReqInput, + GenerateReqInput, + GetWeightsByNameReqInput, + InitWeightsUpdateGroupReqInput, + LoadLoRAAdapterReqInput, + OpenSessionReqInput, + ParseFunctionCallReq, + ProfileReqInput, + ReleaseMemoryOccupationReqInput, + ResumeMemoryOccupationReqInput, + SeparateReasoningReqInput, + SetInternalStateReq, + SlowDownReqInput, + UnloadLoRAAdapterReqInput, + UpdateWeightFromDiskReqInput, + UpdateWeightsFromDistributedReqInput, + UpdateWeightsFromTensorReqInput, + UpdateWeightVersionReqInput, + VertexGenerateReqInput, +) +from sglang.srt.managers.multi_tokenizer_mixin import ( + MultiTokenizerManager, + get_main_process_id, + monkey_patch_uvicorn_multiprocessing, + read_from_shared_memory, + write_data_for_multi_tokenizer, +) +from sglang.srt.managers.template_manager import TemplateManager +from sglang.srt.managers.tokenizer_manager import ServerStatus, TokenizerManager +from sglang.srt.metrics.func_timer import enable_func_timer +from sglang.srt.parser.reasoning_parser import ReasoningParser +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.utils import ( + add_api_key_middleware, + add_prometheus_middleware, + delete_directory, + get_bool_env_var, + kill_process_tree, + set_uvicorn_logging_configs, +) +from sglang.srt.warmup import execute_warmups +from sglang.utils import get_exception_traceback +from sglang.version import __version__ + +logger = logging.getLogger(__name__) +asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) + +HEALTH_CHECK_TIMEOUT = int(os.getenv("SGLANG_HEALTH_CHECK_TIMEOUT", 20)) + + +# Store global states +@dataclasses.dataclass +class _GlobalState: + tokenizer_manager: TokenizerManager + template_manager: TemplateManager + scheduler_info: Dict + + +_global_state: Optional[_GlobalState] = None + + +def set_global_state(global_state: _GlobalState): + global _global_state + _global_state = global_state + + +async def init_multi_tokenizer() -> ServerArgs: + """Read args information from shm and init tokenizer manager for current process""" + pid = os.getpid() + main_pid = get_main_process_id() + logger.info(f"current worker_id: {pid}, main processID: {main_pid}") + + # Read configuration from shared memory + port_args, server_args, scheduler_info = read_from_shared_memory( + f"multi_tokenizer_args_{main_pid}" + ) + server_args: ServerArgs + + # API key authentication is not supported in multi-tokenizer mode + assert ( + server_args.api_key is None + ), "API key is not supported in multi-tokenizer mode" + + port_args.tokenizer_ipc_name = ( + f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}" + ) + + # Launch multi-tokenizer manager process + tokenizer_manager = MultiTokenizerManager(server_args, port_args) + template_manager = TemplateManager() + template_manager.initialize_templates( + tokenizer_manager=tokenizer_manager, + model_path=server_args.model_path, + chat_template=server_args.chat_template, + completion_template=server_args.completion_template, + ) + # Register this tokenizer with the main tokenizer manager + await tokenizer_manager.register_to_main_tokenizer_manager() + + tokenizer_manager.max_req_input_len = scheduler_info["max_req_input_len"] + set_global_state( + _GlobalState( + tokenizer_manager=tokenizer_manager, + template_manager=template_manager, + scheduler_info=scheduler_info, + ) + ) + return server_args + + +@asynccontextmanager +async def lifespan(fast_api_app: FastAPI): + if not getattr(fast_api_app, "is_single_tokenizer_mode", False): + # Initialize multi-tokenizer support for worker processes + fast_api_app.server_args: ServerArgs = await init_multi_tokenizer() + + # only metrics middleware is supported in multi-tokenizer mode + worker_pid = os.getpid() + if fast_api_app.server_args.enable_metrics: + add_prometheus_middleware(app) + enable_func_timer() + + logger.info(f"Worker {worker_pid} added prometheus middleware") + fast_api_app.warmup_thread = threading.Thread( + target=_wait_and_warmup, + args=( + fast_api_app.server_args, + None, # pipe_finish_writer not needed in worker + None, # launch_callback not needed in worker + ), + ) + + # Initialize OpenAI serving handlers + fast_api_app.state.openai_serving_completion = OpenAIServingCompletion( + _global_state.tokenizer_manager, _global_state.template_manager + ) + fast_api_app.state.openai_serving_chat = OpenAIServingChat( + _global_state.tokenizer_manager, _global_state.template_manager + ) + fast_api_app.state.openai_serving_embedding = OpenAIServingEmbedding( + _global_state.tokenizer_manager, _global_state.template_manager + ) + fast_api_app.state.openai_serving_score = OpenAIServingScore( + _global_state.tokenizer_manager + ) + fast_api_app.state.openai_serving_rerank = OpenAIServingRerank( + _global_state.tokenizer_manager + ) + + server_args: ServerArgs = fast_api_app.server_args + + tool_server = None + if server_args.tool_server == "demo": + from sglang.srt.entrypoints.openai.tool_server import DemoToolServer + + tool_server = DemoToolServer() + elif server_args.tool_server: + from sglang.srt.entrypoints.openai.tool_server import MCPToolServer + + tool_server = MCPToolServer() + await tool_server.add_tool_server(server_args.tool_server) + + try: + from sglang.srt.entrypoints.openai.serving_responses import ( + OpenAIServingResponses, + ) + + fast_api_app.state.openai_serving_responses = OpenAIServingResponses( + _global_state.tokenizer_manager, + _global_state.template_manager, + enable_prompt_tokens_details=True, + enable_force_include_usage=True, + tool_server=tool_server, + ) + except Exception as e: + import traceback + + traceback.print_exc() + logger.warning(f"Can not initialize OpenAIServingResponses, error: {e}") + + if server_args.warmups is not None: + await execute_warmups( + server_args.disaggregation_mode, + server_args.warmups.split(","), + _global_state.tokenizer_manager, + ) + logger.info("Warmup ended") + + warmup_thread = getattr(fast_api_app, "warmup_thread", None) + if warmup_thread is not None: + warmup_thread.start() + + try: + yield + finally: + if server_args.tokenizer_worker_num > 1: + pid = os.getpid() + logger.info(f"uvicorn worker {pid} ending...") + warmup_thread.join() + logger.info(f"uvicorn worker {pid} ended.") + + +# Fast API +app = FastAPI( + lifespan=lifespan, + openapi_url=None if get_bool_env_var("DISABLE_OPENAPI_DOC") else "/openapi.json", +) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.exception_handler(HTTPException) +async def validation_exception_handler(request: Request, exc: HTTPException): + """Enrich HTTP exception with status code and other details""" + error = ErrorResponse( + object="error", + message=exc.detail, + type=str(exc.status_code), + code=exc.status_code, + ) + return ORJSONResponse(content=error.model_dump(), status_code=exc.status_code) + + +# Custom exception handlers to change validation error status codes +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + """Override FastAPI's default 422 validation error with 400""" + exc_str = str(exc) + errors_str = str(exc.errors()) + + if errors_str and errors_str != exc_str: + message = f"{exc_str} {errors_str}" + else: + message = exc_str + + err = ErrorResponse( + message=message, + type=HTTPStatus.BAD_REQUEST.phrase, + code=HTTPStatus.BAD_REQUEST.value, + ) + + return ORJSONResponse( + status_code=400, + content=err.model_dump(), + ) + + +async def validate_json_request(raw_request: Request): + """Validate that the request content-type is application/json.""" + content_type = raw_request.headers.get("content-type", "").lower() + media_type = content_type.split(";", maxsplit=1)[0] + if media_type != "application/json": + raise RequestValidationError( + errors=[ + { + "loc": ["header", "content-type"], + "msg": "Unsupported Media Type: Only 'application/json' is allowed", + "type": "value_error", + } + ] + ) + + +##### Native API endpoints ##### + + +@app.get("/health") +@app.get("/health_generate") +async def health_generate(request: Request) -> Response: + """ + Check the health of the inference server by sending a special request to generate one token. + + If the server is running something, this request will be ignored, so it creates zero overhead. + If the server is not running anything, this request will be run, so we know whether the server is healthy. + """ + + if _global_state.tokenizer_manager.gracefully_exit: + logger.info("Health check request received during shutdown. Returning 503.") + return Response(status_code=503) + + if _global_state.tokenizer_manager.server_status == ServerStatus.Starting: + return Response(status_code=503) + + sampling_params = {"max_new_tokens": 1, "temperature": 0.0} + rid = f"HEALTH_CHECK_{time.time()}" + + if _global_state.tokenizer_manager.is_image_gen: + # Keep this branch for some internal use cases. + raise NotImplementedError("Image generation is not supported yet.") + elif _global_state.tokenizer_manager.is_generation: + gri = GenerateReqInput( + rid=rid, + input_ids=[0], + sampling_params=sampling_params, + log_metrics=False, + ) + if ( + _global_state.tokenizer_manager.server_args.disaggregation_mode + != DisaggregationMode.NULL + ): + gri.bootstrap_host = FAKE_BOOTSTRAP_HOST + gri.bootstrap_room = 0 + else: + gri = EmbeddingReqInput( + rid=rid, input_ids=[0], sampling_params=sampling_params, log_metrics=False + ) + + async def gen(): + async for _ in _global_state.tokenizer_manager.generate_request(gri, request): + break + + task = asyncio.create_task(gen()) + + # As long as we receive any response from the detokenizer/scheduler, we consider the server is healthy. + tic = time.time() + while time.time() < tic + HEALTH_CHECK_TIMEOUT: + await asyncio.sleep(1) + if _global_state.tokenizer_manager.last_receive_tstamp > tic: + task.cancel() + _global_state.tokenizer_manager.rid_to_state.pop(rid, None) + _global_state.tokenizer_manager.server_status = ServerStatus.Up + return Response(status_code=200) + + task.cancel() + tic_time = time.strftime("%H:%M:%S", time.localtime(tic)) + last_receive_time = time.strftime( + "%H:%M:%S", time.localtime(_global_state.tokenizer_manager.last_receive_tstamp) + ) + logger.error( + f"Health check failed. Server couldn't get a response from detokenizer for last " + f"{HEALTH_CHECK_TIMEOUT} seconds. tic start time: {tic_time}. " + f"last_heartbeat time: {last_receive_time}" + ) + _global_state.tokenizer_manager.rid_to_state.pop(rid, None) + _global_state.tokenizer_manager.server_status = ServerStatus.UnHealthy + return Response(status_code=503) + + +@app.get("/get_model_info") +async def get_model_info(): + """Get the model information.""" + result = { + "model_path": _global_state.tokenizer_manager.model_path, + "tokenizer_path": _global_state.tokenizer_manager.server_args.tokenizer_path, + "is_generation": _global_state.tokenizer_manager.is_generation, + "preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params, + "weight_version": _global_state.tokenizer_manager.server_args.weight_version, + } + return result + + +@app.get("/get_weight_version") +async def get_weight_version(): + """Get the current weight version.""" + return { + "weight_version": _global_state.tokenizer_manager.server_args.weight_version + } + + +@app.get("/get_server_info") +async def get_server_info(): + # Returns interna states per DP. + internal_states: List[Dict[Any, Any]] = ( + await _global_state.tokenizer_manager.get_internal_state() + ) + return { + **dataclasses.asdict(_global_state.tokenizer_manager.server_args), + **_global_state.scheduler_info, + "internal_states": internal_states, + "version": __version__, + } + + +@app.get("/get_load") +async def get_load(): + return await _global_state.tokenizer_manager.get_load() + + +# example usage: +# curl -s -X POST http://localhost:30000/set_internal_state -H "Content-Type: application/json" -d '{"server_args": {"max_micro_batch_size": 8}}' +@app.api_route("/set_internal_state", methods=["POST", "PUT"]) +async def set_internal_state(obj: SetInternalStateReq, request: Request): + res = await _global_state.tokenizer_manager.set_internal_state(obj) + return res + + +# fastapi implicitly converts json in the request to obj (dataclass) +@app.api_route("/generate", methods=["POST", "PUT"]) +async def generate_request(obj: GenerateReqInput, request: Request): + """Handle a generate request.""" + if obj.stream: + + async def stream_results() -> AsyncIterator[bytes]: + try: + async for out in _global_state.tokenizer_manager.generate_request( + obj, request + ): + yield b"data: " + orjson.dumps( + out, option=orjson.OPT_NON_STR_KEYS + ) + b"\n\n" + except ValueError as e: + out = {"error": {"message": str(e)}} + logger.error(f"[http_server] Error: {e}") + yield b"data: " + orjson.dumps( + out, option=orjson.OPT_NON_STR_KEYS + ) + b"\n\n" + yield b"data: [DONE]\n\n" + + return StreamingResponse( + stream_results(), + media_type="text/event-stream", + background=_global_state.tokenizer_manager.create_abort_task(obj), + ) + else: + try: + ret = await _global_state.tokenizer_manager.generate_request( + obj, request + ).__anext__() + return ret + except ValueError as e: + logger.error(f"[http_server] Error: {e}") + return _create_error_response(e) + + +@app.api_route("/generate_from_file", methods=["POST"]) +async def generate_from_file_request(file: UploadFile, request: Request): + """Handle a generate request, this is purely to work with input_embeds.""" + content = await file.read() + input_embeds = json.loads(content.decode("utf-8")) + + obj = GenerateReqInput( + input_embeds=input_embeds, + sampling_params={ + "temperature": 0.0, + "max_new_tokens": 512, + }, + ) + + try: + ret = await _global_state.tokenizer_manager.generate_request( + obj, request + ).__anext__() + return ret + except ValueError as e: + logger.error(f"Error: {e}") + return _create_error_response(e) + + +@app.api_route("/encode", methods=["POST", "PUT"]) +async def encode_request(obj: EmbeddingReqInput, request: Request): + """Handle an embedding request.""" + try: + ret = await _global_state.tokenizer_manager.generate_request( + obj, request + ).__anext__() + return ret + except ValueError as e: + return _create_error_response(e) + + +@app.api_route("/classify", methods=["POST", "PUT"]) +async def classify_request(obj: EmbeddingReqInput, request: Request): + """Handle a reward model request. Now the arguments and return values are the same as embedding models.""" + try: + ret = await _global_state.tokenizer_manager.generate_request( + obj, request + ).__anext__() + return ret + except ValueError as e: + return _create_error_response(e) + + +@app.api_route("/flush_cache", methods=["GET", "POST"]) +async def flush_cache(): + """Flush the radix cache.""" + ret = await _global_state.tokenizer_manager.flush_cache() + return Response( + content="Cache flushed.\nPlease check backend logs for more details. " + "(When there are running or waiting requests, the operation will not be performed.)\n", + status_code=200 if ret.success else HTTPStatus.BAD_REQUEST, + ) + + +@app.api_route("/clear_hicache_storage_backend", methods=["GET", "POST"]) +async def clear_hicache_storage_backend(): + """Clear the hierarchical cache storage backend.""" + ret = await _global_state.tokenizer_manager.clear_hicache_storage() + return Response( + content="Hierarchical cache storage backend cleared.\n", + status_code=200 if ret.success else HTTPStatus.BAD_REQUEST, + ) + + +@app.api_route("/start_profile", methods=["GET", "POST"]) +async def start_profile_async(obj: Optional[ProfileReqInput] = None): + """Start profiling.""" + if obj is None: + obj = ProfileReqInput() + + await _global_state.tokenizer_manager.start_profile( + output_dir=obj.output_dir, + start_step=obj.start_step, + num_steps=obj.num_steps, + activities=obj.activities, + with_stack=obj.with_stack, + record_shapes=obj.record_shapes, + profile_by_stage=obj.profile_by_stage, + ) + return Response( + content="Start profiling.\n", + status_code=200, + ) + + +@app.api_route("/stop_profile", methods=["GET", "POST"]) +async def stop_profile_async(): + """Stop profiling.""" + await _global_state.tokenizer_manager.stop_profile() + return Response( + content="Stop profiling. This will take some time.\n", + status_code=200, + ) + + +@app.api_route("/freeze_gc", methods=["GET", "POST"]) +async def freeze_gc_async(): + """ + See engine.freeze_gc for more details. + """ + await _global_state.tokenizer_manager.freeze_gc() + return Response( + content="Garbage collection frozen.\n", + status_code=200, + ) + + +@app.api_route("/start_expert_distribution_record", methods=["GET", "POST"]) +async def start_expert_distribution_record_async(): + """Start recording the expert distribution. Clear the previous record if any.""" + await _global_state.tokenizer_manager.start_expert_distribution_record() + return Response( + content="Start recording the expert distribution.\n", + status_code=200, + ) + + +@app.api_route("/stop_expert_distribution_record", methods=["GET", "POST"]) +async def stop_expert_distribution_record_async(): + """Stop recording the expert distribution.""" + await _global_state.tokenizer_manager.stop_expert_distribution_record() + return Response( + content="Stop recording the expert distribution.\n", + status_code=200, + ) + + +@app.api_route("/dump_expert_distribution_record", methods=["GET", "POST"]) +async def dump_expert_distribution_record_async(): + """Dump expert distribution record.""" + await _global_state.tokenizer_manager.dump_expert_distribution_record() + return Response( + content="Dump expert distribution record.\n", + status_code=200, + ) + + +@app.post("/update_weights_from_disk") +async def update_weights_from_disk(obj: UpdateWeightFromDiskReqInput, request: Request): + """Update the weights from disk inplace without re-launching the server.""" + success, message, num_paused_requests = ( + await _global_state.tokenizer_manager.update_weights_from_disk(obj, request) + ) + + # Update weight version if provided and weights update was successful + if success and obj.weight_version is not None: + _update_weight_version_if_provided(obj.weight_version) + message += f" Weight version updated to {obj.weight_version}." + + content = { + "success": success, + "message": message, + "num_paused_requests": num_paused_requests, + } + if success: + return ORJSONResponse( + content, + status_code=HTTPStatus.OK, + ) + else: + return ORJSONResponse( + content, + status_code=HTTPStatus.BAD_REQUEST, + ) + + +@app.post("/init_weights_update_group") +async def init_weights_update_group( + obj: InitWeightsUpdateGroupReqInput, request: Request +): + """Initialize the parameter update group.""" + success, message = await _global_state.tokenizer_manager.init_weights_update_group( + obj, request + ) + content = {"success": success, "message": message} + if success: + return ORJSONResponse(content, status_code=200) + else: + return ORJSONResponse(content, status_code=HTTPStatus.BAD_REQUEST) + + +@app.post("/update_weights_from_tensor") +async def update_weights_from_tensor( + obj: UpdateWeightsFromTensorReqInput, request: Request +): + """Update the weights from tensor inplace without re-launching the server. + Notes: + 1. Ensure that the model is on the correct device (e.g., GPU) before calling this endpoint. If the model is moved to the CPU unexpectedly, it may cause performance issues or runtime errors. + 2. HTTP will transmit only the metadata of the tensor, while the tensor itself will be directly copied to the model. + 3. Any binary data in the named tensors should be base64 encoded. + """ + + success, message = await _global_state.tokenizer_manager.update_weights_from_tensor( + obj, request + ) + + # Update weight version if provided and weights update was successful + if success and obj.weight_version is not None: + _update_weight_version_if_provided(obj.weight_version) + message += f" Weight version updated to {obj.weight_version}." + + content = {"success": success, "message": message} + return ORJSONResponse( + content, status_code=200 if success else HTTPStatus.BAD_REQUEST + ) + + +@app.post("/update_weights_from_distributed") +async def update_weights_from_distributed( + obj: UpdateWeightsFromDistributedReqInput, request: Request +): + """Update model parameter from distributed online.""" + success, message = ( + await _global_state.tokenizer_manager.update_weights_from_distributed( + obj, request + ) + ) + + # Update weight version if provided and weights update was successful + if success and obj.weight_version is not None: + _update_weight_version_if_provided(obj.weight_version) + message += f" Weight version updated to {obj.weight_version}." + + content = {"success": success, "message": message} + if success: + return ORJSONResponse(content, status_code=200) + else: + return ORJSONResponse(content, status_code=HTTPStatus.BAD_REQUEST) + + +@app.post("/update_weight_version") +async def update_weight_version(obj: UpdateWeightVersionReqInput, request: Request): + """Update the weight version. This operation requires no active requests.""" + if obj.abort_all_requests: + _global_state.tokenizer_manager.abort_request(abort_all=True) + + # Use a simple approach without the complex lock mechanism for now + # since weight_version update is a simple operation that doesn't affect model weights + try: + # Update the weight version in server args (the single source of truth) + _global_state.tokenizer_manager.server_args.weight_version = obj.new_version + + return ORJSONResponse( + { + "success": True, + "message": f"Weight version updated to {obj.new_version}", + "new_version": obj.new_version, + }, + status_code=HTTPStatus.OK, + ) + except Exception as e: + return ORJSONResponse( + { + "success": False, + "message": f"Failed to update weight version: {str(e)}", + }, + status_code=HTTPStatus.BAD_REQUEST, + ) + + +@app.api_route("/get_weights_by_name", methods=["GET", "POST"]) +async def get_weights_by_name(obj: GetWeightsByNameReqInput, request: Request): + """Get model parameter by name.""" + try: + ret = await _global_state.tokenizer_manager.get_weights_by_name(obj, request) + if ret is None: + return _create_error_response("Get parameter by name failed") + else: + return ORJSONResponse(ret, status_code=200) + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/release_memory_occupation", methods=["GET", "POST"]) +async def release_memory_occupation( + obj: ReleaseMemoryOccupationReqInput, request: Request +): + """Release GPU memory occupation temporarily.""" + try: + await _global_state.tokenizer_manager.release_memory_occupation(obj, request) + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/resume_memory_occupation", methods=["GET", "POST"]) +async def resume_memory_occupation( + obj: ResumeMemoryOccupationReqInput, request: Request +): + """Resume GPU memory occupation.""" + try: + await _global_state.tokenizer_manager.resume_memory_occupation(obj, request) + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/slow_down", methods=["GET", "POST"]) +async def slow_down(obj: SlowDownReqInput, request: Request): + """Slow down the system deliberately. Only for testing. Example scenario: + when we want to test performance of D in large-scale PD disaggregation and have no enough nodes for P, + we can use this to slow down D to let it have enough running sequences, and then disable slowdown + to let it run in full batch size. + """ + try: + await _global_state.tokenizer_manager.slow_down(obj, request) + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/load_lora_adapter", methods=["POST"]) +async def load_lora_adapter(obj: LoadLoRAAdapterReqInput, request: Request): + """Load a new LoRA adapter without re-launching the server.""" + result = await _global_state.tokenizer_manager.load_lora_adapter(obj, request) + + if result.success: + return ORJSONResponse( + result, + status_code=HTTPStatus.OK, + ) + else: + return ORJSONResponse( + result, + status_code=HTTPStatus.BAD_REQUEST, + ) + + +@app.api_route("/unload_lora_adapter", methods=["POST"]) +async def unload_lora_adapter(obj: UnloadLoRAAdapterReqInput, request: Request): + """Load a new LoRA adapter without re-launching the server.""" + result = await _global_state.tokenizer_manager.unload_lora_adapter(obj, request) + + if result.success: + return ORJSONResponse( + result, + status_code=HTTPStatus.OK, + ) + else: + return ORJSONResponse( + result, + status_code=HTTPStatus.BAD_REQUEST, + ) + + +@app.api_route("/open_session", methods=["GET", "POST"]) +async def open_session(obj: OpenSessionReqInput, request: Request): + """Open a session, and return its unique session id.""" + try: + session_id = await _global_state.tokenizer_manager.open_session(obj, request) + if session_id is None: + raise Exception( + "Failed to open the session. Check if a session with the same id is still open." + ) + return session_id + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/close_session", methods=["GET", "POST"]) +async def close_session(obj: CloseSessionReqInput, request: Request): + """Close the session.""" + try: + await _global_state.tokenizer_manager.close_session(obj, request) + return Response(status_code=200) + except Exception as e: + return _create_error_response(e) + + +@app.api_route("/configure_logging", methods=["GET", "POST"]) +async def configure_logging(obj: ConfigureLoggingReq, request: Request): + """Configure the request logging options.""" + _global_state.tokenizer_manager.configure_logging(obj) + return Response(status_code=200) + + +@app.post("/abort_request") +async def abort_request(obj: AbortReq, request: Request): + """Abort a request.""" + try: + _global_state.tokenizer_manager.abort_request( + rid=obj.rid, abort_all=obj.abort_all + ) + return Response(status_code=200) + except Exception as e: + return _create_error_response(e) + + +@app.post("/parse_function_call") +async def parse_function_call_request(obj: ParseFunctionCallReq, request: Request): + """ + A native API endpoint to parse function calls from a text. + """ + # 1) Initialize the parser based on the request body + parser = FunctionCallParser(tools=obj.tools, tool_call_parser=obj.tool_call_parser) + + # 2) Call the non-stream parsing method (non-stream) + normal_text, calls = parser.parse_non_stream(obj.text) + + # 3) Organize the response content + response_data = { + "normal_text": normal_text, + "calls": [ + call.model_dump() for call in calls + ], # Convert pydantic objects to dictionaries + } + + return ORJSONResponse(content=response_data, status_code=200) + + +@app.post("/separate_reasoning") +async def separate_reasoning_request(obj: SeparateReasoningReqInput, request: Request): + """ + A native API endpoint to separate reasoning from a text. + """ + # 1) Initialize the parser based on the request body + parser = ReasoningParser(model_type=obj.reasoning_parser) + + # 2) Call the non-stream parsing method (non-stream) + reasoning_text, normal_text = parser.parse_non_stream(obj.text) + + # 3) Organize the response content + response_data = { + "reasoning_text": reasoning_text, + "text": normal_text, + } + + return ORJSONResponse(content=response_data, status_code=200) + + +@app.post("/pause_generation") +async def pause_generation(request: Request): + """Pause generation.""" + await _global_state.tokenizer_manager.pause_generation() + return ORJSONResponse( + content={"message": "Generation paused successfully.", "status": "ok"}, + status_code=200, + ) + + +@app.post("/continue_generation") +async def continue_generation(request: Request): + """Continue generation.""" + await _global_state.tokenizer_manager.continue_generation() + return ORJSONResponse( + content={"message": "Generation continued successfully.", "status": "ok"}, + status_code=200, + ) + + +##### OpenAI-compatible API endpoints ##### + + +@app.post("/v1/completions", dependencies=[Depends(validate_json_request)]) +async def openai_v1_completions(request: CompletionRequest, raw_request: Request): + """OpenAI-compatible text completion endpoint.""" + return await raw_request.app.state.openai_serving_completion.handle_request( + request, raw_request + ) + + +@app.post("/v1/chat/completions", dependencies=[Depends(validate_json_request)]) +async def openai_v1_chat_completions( + request: ChatCompletionRequest, raw_request: Request +): + """OpenAI-compatible chat completion endpoint.""" + return await raw_request.app.state.openai_serving_chat.handle_request( + request, raw_request + ) + + +@app.post( + "/v1/embeddings", + response_class=ORJSONResponse, + dependencies=[Depends(validate_json_request)], +) +async def openai_v1_embeddings(request: EmbeddingRequest, raw_request: Request): + """OpenAI-compatible embeddings endpoint.""" + return await raw_request.app.state.openai_serving_embedding.handle_request( + request, raw_request + ) + + +@app.get("/v1/models", response_class=ORJSONResponse) +async def available_models(): + """Show available models. OpenAI-compatible endpoint.""" + served_model_names = [_global_state.tokenizer_manager.served_model_name] + model_cards = [] + for served_model_name in served_model_names: + model_cards.append( + ModelCard( + id=served_model_name, + root=served_model_name, + max_model_len=_global_state.tokenizer_manager.model_config.context_len, + ) + ) + return ModelList(data=model_cards) + + +@app.get("/v1/models/{model:path}", response_class=ORJSONResponse) +async def retrieve_model(model: str): + """Retrieves a model instance, providing basic information about the model.""" + served_model_names = [_global_state.tokenizer_manager.served_model_name] + + if model not in served_model_names: + return ORJSONResponse( + status_code=404, + content={ + "error": { + "message": f"The model '{model}' does not exist", + "type": "invalid_request_error", + "param": "model", + "code": "model_not_found", + } + }, + ) + + return ModelCard( + id=model, + root=model, + max_model_len=_global_state.tokenizer_manager.model_config.context_len, + ) + + +@app.post("/v1/score", dependencies=[Depends(validate_json_request)]) +async def v1_score_request(request: ScoringRequest, raw_request: Request): + """Endpoint for the decoder-only scoring API. See Engine.score() for detailed documentation.""" + return await raw_request.app.state.openai_serving_score.handle_request( + request, raw_request + ) + + +@app.post("/v1/responses", dependencies=[Depends(validate_json_request)]) +async def v1_responses_request(request: dict, raw_request: Request): + """Endpoint for the responses API with reasoning support.""" + + request_obj = ResponsesRequest(**request) + result = await raw_request.app.state.openai_serving_responses.create_responses( + request_obj, raw_request + ) + + # Handle streaming responses + if isinstance(result, AsyncGenerator): + return StreamingResponse( + result, + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "Connection": "keep-alive"}, + ) + + return result + + +@app.get("/v1/responses/{response_id}") +async def v1_retrieve_responses(response_id: str, raw_request: Request): + """Retrieve a response by ID.""" + return await raw_request.app.state.openai_serving_responses.retrieve_responses( + response_id + ) + + +@app.post("/v1/responses/{response_id}/cancel") +async def v1_cancel_responses(response_id: str, raw_request: Request): + """Cancel a background response.""" + return await raw_request.app.state.openai_serving_responses.cancel_responses( + response_id + ) + + +@app.api_route( + "/v1/rerank", methods=["POST", "PUT"], dependencies=[Depends(validate_json_request)] +) +async def v1_rerank_request(request: V1RerankReqInput, raw_request: Request): + """Endpoint for reranking documents based on query relevance.""" + return await raw_request.app.state.openai_serving_rerank.handle_request( + request, raw_request + ) + + +## SageMaker API +@app.get("/ping") +async def sagemaker_health() -> Response: + """Check the health of the http server.""" + return Response(status_code=200) + + +@app.post("/invocations") +async def sagemaker_chat_completions( + request: ChatCompletionRequest, raw_request: Request +): + """OpenAI-compatible chat completion endpoint.""" + return await raw_request.app.state.openai_serving_chat.handle_request( + request, raw_request + ) + + +## Vertex AI API +@app.post(os.environ.get("AIP_PREDICT_ROUTE", "/vertex_generate")) +async def vertex_generate(vertex_req: VertexGenerateReqInput, raw_request: Request): + if not vertex_req.instances: + return [] + inputs = {} + for input_key in ("text", "input_ids", "input_embeds"): + if vertex_req.instances[0].get(input_key): + inputs[input_key] = [ + instance.get(input_key) for instance in vertex_req.instances + ] + break + image_data = [ + instance.get("image_data") + for instance in vertex_req.instances + if instance.get("image_data") is not None + ] or None + req = GenerateReqInput( + **inputs, + image_data=image_data, + **(vertex_req.parameters or {}), + ) + ret = await generate_request(req, raw_request) + if isinstance(ret, Response): + return ret + return ORJSONResponse({"predictions": ret}) + + +def _update_weight_version_if_provided(weight_version: Optional[str]) -> None: + """Update weight version if provided.""" + if weight_version is not None: + _global_state.tokenizer_manager.server_args.weight_version = weight_version + + +def _create_error_response(e): + return ORJSONResponse( + {"error": {"message": str(e)}}, status_code=HTTPStatus.BAD_REQUEST + ) + + +def launch_server( + server_args: ServerArgs, + pipe_finish_writer: Optional[multiprocessing.connection.Connection] = None, + launch_callback: Optional[Callable[[], None]] = None, +): + """ + Launch SRT (SGLang Runtime) Server. + + The SRT server consists of an HTTP server and an SRT engine. + + - HTTP server: A FastAPI server that routes requests to the engine. + - The engine consists of three components: + 1. TokenizerManager: Tokenizes the requests and sends them to the scheduler. + 2. Scheduler (subprocess): Receives requests from the Tokenizer Manager, schedules batches, forwards them, and sends the output tokens to the Detokenizer Manager. + 3. DetokenizerManager (subprocess): Detokenizes the output tokens and sends the result back to the Tokenizer Manager. + + Note: + 1. The HTTP server, Engine, and TokenizerManager both run in the main process. + 2. Inter-process communication is done through IPC (each process uses a different port) via the ZMQ library. + """ + if server_args.tokenizer_worker_num > 1: + setproctitle.setproctitle(f"sglang::http_server/multi_tokenizer_router") + port_args = PortArgs.init_new(server_args) + port_args.tokenizer_worker_ipc_name = ( + f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}" + ) + tokenizer_manager, template_manager, scheduler_info = _launch_subprocesses( + server_args=server_args, port_args=port_args + ) + else: + setproctitle.setproctitle(f"sglang::http_server/tokenizer_manager") + tokenizer_manager, template_manager, scheduler_info = _launch_subprocesses( + server_args=server_args, + ) + + set_global_state( + _GlobalState( + tokenizer_manager=tokenizer_manager, + template_manager=template_manager, + scheduler_info=scheduler_info, + ) + ) + + if server_args.tokenizer_worker_num > 1: + multi_tokenizer_args_shm = write_data_for_multi_tokenizer( + port_args, + server_args, + scheduler_info, + ) + else: + # Add api key authorization + if server_args.api_key: + add_api_key_middleware(app, server_args.api_key) + + # Add prometheus middleware + if server_args.enable_metrics: + add_prometheus_middleware(app) + enable_func_timer() + + # Send a warmup request - we will create the thread launch it + # in the lifespan after all other warmups have fired. + warmup_thread = threading.Thread( + target=_wait_and_warmup, + args=( + server_args, + pipe_finish_writer, + launch_callback, + ), + ) + app.warmup_thread = warmup_thread + + try: + # Update logging configs + set_uvicorn_logging_configs() + app.server_args = server_args + # Listen for HTTP requests + if server_args.tokenizer_worker_num > 1: + from uvicorn.config import LOGGING_CONFIG + + LOGGING_CONFIG["loggers"]["sglang.srt.entrypoints.http_server"] = { + "handlers": ["default"], + "level": "INFO", + "propagate": False, + } + + monkey_patch_uvicorn_multiprocessing() + + uvicorn.run( + "sglang.srt.entrypoints.http_server:app", + host=server_args.host, + port=server_args.port, + log_level=server_args.log_level_http or server_args.log_level, + timeout_keep_alive=5, + loop="uvloop", + workers=server_args.tokenizer_worker_num, + ) + else: + app.is_single_tokenizer_mode = True + uvicorn.run( + app, + host=server_args.host, + port=server_args.port, + log_level=server_args.log_level_http or server_args.log_level, + timeout_keep_alive=5, + loop="uvloop", + ) + finally: + if server_args.tokenizer_worker_num > 1: + multi_tokenizer_args_shm.unlink() + _global_state.tokenizer_manager.socket_mapping.clear_all_sockets() + else: + warmup_thread.join() + + +def _execute_server_warmup( + server_args: ServerArgs, + pipe_finish_writer: Optional[multiprocessing.connection.Connection], +): + headers = {} + url = server_args.url() + if server_args.api_key: + headers["Authorization"] = f"Bearer {server_args.api_key}" + + # Wait until the server is launched + success = False + for _ in range(120): + time.sleep(1) + try: + res = requests.get(url + "/get_model_info", timeout=5, headers=headers) + assert res.status_code == 200, f"{res=}, {res.text=}" + success = True + break + except (AssertionError, requests.exceptions.RequestException): + last_traceback = get_exception_traceback() + pass + + if not success: + if pipe_finish_writer is not None: + pipe_finish_writer.send(last_traceback) + logger.error(f"Initialization failed. warmup error: {last_traceback}") + kill_process_tree(os.getpid()) + return success + + model_info = res.json() + + # Send a warmup request + request_name = "/generate" if model_info["is_generation"] else "/encode" + max_new_tokens = 8 if model_info["is_generation"] else 1 + json_data = { + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + } + if server_args.skip_tokenizer_init: + json_data["input_ids"] = [[10, 11, 12] for _ in range(server_args.dp_size)] + # TODO Workaround the bug that embedding errors for list of size 1 + if server_args.dp_size == 1: + json_data["input_ids"] = json_data["input_ids"][0] + else: + json_data["text"] = ["The capital city of France is"] * server_args.dp_size + # TODO Workaround the bug that embedding errors for list of size 1 + if server_args.dp_size == 1: + json_data["text"] = json_data["text"][0] + + # Debug dumping + if server_args.debug_tensor_dump_input_file: + json_data.pop("text", None) + json_data["input_ids"] = np.load( + server_args.debug_tensor_dump_input_file + ).tolist() + json_data["sampling_params"]["max_new_tokens"] = 0 + + try: + if server_args.disaggregation_mode == "null": + res = requests.post( + url + request_name, + json=json_data, + headers=headers, + timeout=600, + ) + assert res.status_code == 200, f"{res}" + _global_state.tokenizer_manager.server_status = ServerStatus.Up + + else: + logger.info(f"Start of pd disaggregation warmup ...") + json_data = { + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 8, + "ignore_eos": True, + }, + "bootstrap_host": [FAKE_BOOTSTRAP_HOST] * server_args.dp_size, + # This is a hack to ensure fake transfer is enabled during prefill warmup + # ensure each dp rank has a unique bootstrap_room during prefill warmup + "bootstrap_room": [ + i * (2**63 // server_args.dp_size) + (i % server_args.tp_size) + for i in range(server_args.dp_size) + ], + "input_ids": [[0, 1, 2, 3]] * server_args.dp_size, + } + res = requests.post( + url + request_name, + json=json_data, + headers=headers, + timeout=1800, # because of deep gemm precache is very long if not precache. + ) + if res.status_code == 200: + logger.info( + f"End of prefill disaggregation mode warmup with status {res.status_code}, resp: {res.json()}" + ) + _global_state.tokenizer_manager.server_status = ServerStatus.Up + else: + logger.info( + "Prefill disaggregation mode warm Up Failed, status code: {}".format( + res.status_code + ) + ) + _global_state.tokenizer_manager.server_status = ServerStatus.UnHealthy + + except Exception: + last_traceback = get_exception_traceback() + if pipe_finish_writer is not None: + pipe_finish_writer.send(last_traceback) + logger.error(f"Initialization failed. warmup error: {last_traceback}") + kill_process_tree(os.getpid()) + return False + + # Debug print + # logger.info(f"warmup request returns: {res.json()=}") + return success + + +def _wait_and_warmup( + server_args: ServerArgs, + pipe_finish_writer: Optional[multiprocessing.connection.Connection], + launch_callback: Optional[Callable[[], None]] = None, +): + if not server_args.skip_server_warmup: + if not _execute_server_warmup( + server_args, + pipe_finish_writer, + ): + return + else: + _global_state.tokenizer_manager.server_status = ServerStatus.Up + + logger.info("The server is fired up and ready to roll!") + + if pipe_finish_writer is not None: + pipe_finish_writer.send("ready") + + if server_args.delete_ckpt_after_loading: + delete_directory(server_args.model_path) + + if server_args.debug_tensor_dump_input_file: + kill_process_tree(os.getpid()) + + if launch_callback is not None: + launch_callback() diff --git a/python/sglang/srt/entrypoints/http_server_engine.py b/python/sglang/srt/entrypoints/http_server_engine.py new file mode 100644 index 000000000..d1db80d65 --- /dev/null +++ b/python/sglang/srt/entrypoints/http_server_engine.py @@ -0,0 +1,142 @@ +import copy +import dataclasses +import multiprocessing +import pickle +import threading +import time +from typing import Any, Dict, List, Optional, Tuple, Union + +import pybase64 +import requests +import torch +import torch.distributed as dist + +from sglang.srt.entrypoints.EngineBase import EngineBase +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import MultiprocessingSerializer, kill_process_tree + + +def launch_server_process(server_args: ServerArgs) -> multiprocessing.Process: + + p = multiprocessing.Process(target=launch_server, args=(server_args,)) + p.start() + + base_url = server_args.url() + timeout = 300.0 # Increased timeout to 5 minutes for downloading large models + start_time = time.perf_counter() + + with requests.Session() as session: + while time.perf_counter() - start_time < timeout: + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {server_args.api_key}", + } + response = session.get(f"{base_url}/health_generate", headers=headers) + if response.status_code == 200: + return p + except requests.RequestException: + pass + + if not p.is_alive(): + raise Exception("Server process terminated unexpectedly.") + + time.sleep(2) + + p.terminate() + raise TimeoutError("Server failed to start within the timeout period.") + + +class HttpServerEngineAdapter(EngineBase): + """ + You can use this class to launch a server from a VerlEngine instance. + We recommend using this class only you need to use http server. + Otherwise, you can use Engine directly. + """ + + def __init__(self, **kwargs): + self.server_args = ServerArgs(**kwargs) + print( + f"Launch HttpServerEngineAdapter at: {self.server_args.host}:{self.server_args.port}" + ) + self.process = launch_server_process(self.server_args) + + def _make_request(self, endpoint: str, payload: Optional[dict] = None): + """Make a POST request to the specified endpoint with the given payload. + Args: + endpoint: The API endpoint to call + payload: The JSON payload to send (default: empty dict) + Returns: + The JSON response from the server + """ + url = f"http://{self.server_args.host}:{self.server_args.port}/{endpoint}" + response = requests.post(url, json=payload or {}) + response.raise_for_status() + return response.json() + + def update_weights_from_tensor( + self, + named_tensors: List[Tuple[str, torch.Tensor]], + load_format: Optional[str] = None, + flush_cache: bool = False, + ): + """ + Update model weights from tensor data. The HTTP server will only post meta data, and the real weights will be copied directly from GPUs. + Note: The model should be on GPUs rather than CPU for this functionality to work properly. + If you encounter issues, ensure your model is loaded on GPU devices rather than CPU. + """ + + return self._make_request( + "update_weights_from_tensor", + { + "serialized_named_tensors": [ + MultiprocessingSerializer.serialize(named_tensors, output_str=True) + for _ in range(self.server_args.tp_size) + ], + "load_format": load_format, + "flush_cache": flush_cache, + }, + ) + + def shutdown(self): + kill_process_tree(self.process.pid) + + def generate( + self, + prompt=None, + sampling_params=None, + input_ids=None, + image_data=None, + return_logprob=False, + logprob_start_len=None, + top_logprobs_num=None, + token_ids_logprob=None, + lora_path=None, + custom_logit_processor=None, + ): + payload = { + "text": prompt, + "sampling_params": sampling_params, + "input_ids": input_ids, + "image_data": image_data, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + "token_ids_logprob": token_ids_logprob, + "lora_path": lora_path, + "custom_logit_processor": custom_logit_processor, + } + # Filter out None values + payload = {k: v for k, v in payload.items() if v is not None} + + return self._make_request("generate", payload) + + def release_memory_occupation(self): + return self._make_request("release_memory_occupation") + + def resume_memory_occupation(self): + return self._make_request("resume_memory_occupation") + + def flush_cache(self): + return self._make_request("flush_cache") diff --git a/python/sglang/srt/entrypoints/openai/__init__.py b/python/sglang/srt/entrypoints/openai/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py new file mode 100644 index 000000000..f73e67d0b --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -0,0 +1,929 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Pydantic models for OpenAI API protocol""" + +import time +import uuid +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, TypeAlias, Union + +from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseInputItemParam, + ResponseOutputItem, + ResponseReasoningItem, +) +from openai.types.responses.response import ToolChoice +from openai.types.responses.tool import Tool +from pydantic import ( + BaseModel, + Field, + field_validator, + model_serializer, + model_validator, +) +from typing_extensions import Literal + +DEFAULT_MODEL_NAME = "default" + + +class ModelCard(BaseModel): + """Model cards.""" + + id: str + object: str = "model" + created: int = Field(default_factory=lambda: int(time.time())) + owned_by: str = "sglang" + root: Optional[str] = None + max_model_len: Optional[int] = None + + +class ModelList(BaseModel): + """Model list consists of model cards.""" + + object: str = "list" + data: List[ModelCard] = Field(default_factory=list) + + +class ErrorResponse(BaseModel): + object: str = "error" + message: str + type: str + param: Optional[str] = None + code: int + + +class LogProbs(BaseModel): + text_offset: List[int] = Field(default_factory=list) + token_logprobs: List[Optional[float]] = Field(default_factory=list) + tokens: List[str] = Field(default_factory=list) + top_logprobs: List[Optional[Dict[str, float]]] = Field(default_factory=list) + + +class TopLogprob(BaseModel): + token: str + bytes: List[int] + logprob: float + + +class ChatCompletionTokenLogprob(BaseModel): + token: str + bytes: List[int] + logprob: float + top_logprobs: List[TopLogprob] + + +class ChoiceLogprobs(BaseModel): + # build for v1/chat/completions response + content: List[ChatCompletionTokenLogprob] + + +class UsageInfo(BaseModel): + prompt_tokens: int = 0 + total_tokens: int = 0 + completion_tokens: Optional[int] = 0 + # only used to return cached tokens when --enable-cache-report is set + prompt_tokens_details: Optional[Dict[str, int]] = None + reasoning_tokens: Optional[int] = 0 + + +class StreamOptions(BaseModel): + include_usage: Optional[bool] = False + + +class JsonSchemaResponseFormat(BaseModel): + name: str + description: Optional[str] = None + # use alias to workaround pydantic conflict + schema_: Optional[Dict[str, object]] = Field(alias="schema", default=None) + strict: Optional[bool] = False + + +class ResponseFormat(BaseModel): + type: Literal["text", "json_object", "json_schema"] + json_schema: Optional[JsonSchemaResponseFormat] = None + + +class StructuresResponseFormat(BaseModel): + begin: str + schema_: Optional[Dict[str, object]] = Field(alias="schema", default=None) + end: str + + +class StructuralTagResponseFormat(BaseModel): + type: Literal["structural_tag"] + structures: List[StructuresResponseFormat] + triggers: List[str] + + +class FileRequest(BaseModel): + # https://platform.openai.com/docs/api-reference/files/create + file: bytes # The File object (not file name) to be uploaded + purpose: str = ( + "batch" # The intended purpose of the uploaded file, default is "batch" + ) + + +class FileResponse(BaseModel): + id: str + object: str = "file" + bytes: int + created_at: int + filename: str + purpose: str + + +class FileDeleteResponse(BaseModel): + id: str + object: str = "file" + deleted: bool + + +class BatchRequest(BaseModel): + input_file_id: ( + str # The ID of an uploaded file that contains requests for the new batch + ) + endpoint: str # The endpoint to be used for all requests in the batch + completion_window: str # The time frame within which the batch should be processed + metadata: Optional[dict] = None # Optional custom metadata for the batch + + +class BatchResponse(BaseModel): + id: str + object: str = "batch" + endpoint: str + errors: Optional[dict] = None + input_file_id: str + completion_window: str + status: str = "validating" + output_file_id: Optional[str] = None + error_file_id: Optional[str] = None + created_at: int + in_progress_at: Optional[int] = None + expires_at: Optional[int] = None + finalizing_at: Optional[int] = None + completed_at: Optional[int] = None + failed_at: Optional[int] = None + expired_at: Optional[int] = None + cancelling_at: Optional[int] = None + cancelled_at: Optional[int] = None + request_counts: Optional[dict] = None + metadata: Optional[dict] = None + + +class CompletionRequest(BaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/completions/create + model: str = DEFAULT_MODEL_NAME + prompt: Union[List[int], List[List[int]], str, List[str]] + best_of: Optional[int] = None + echo: bool = False + frequency_penalty: float = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: Optional[int] = None + max_tokens: int = 16 + n: int = 1 + presence_penalty: float = 0.0 + seed: Optional[int] = None + stop: Optional[Union[str, List[str]]] = None + stream: bool = False + stream_options: Optional[StreamOptions] = None + suffix: Optional[str] = None + temperature: float = 1.0 + top_p: float = 1.0 + user: Optional[str] = None + return_hidden_states: bool = False + + # Extra parameters for SRT backend only and will be ignored by OpenAI models. + top_k: int = -1 + min_p: float = 0.0 + min_tokens: int = 0 + json_schema: Optional[str] = None + regex: Optional[str] = None + ebnf: Optional[str] = None + repetition_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = None + no_stop_trim: bool = False + ignore_eos: bool = False + skip_special_tokens: bool = True + lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None + session_params: Optional[Dict] = None + response_format: Optional[Union[ResponseFormat, StructuralTagResponseFormat]] = None + + # For PD disaggregation + bootstrap_host: Optional[Union[List[str], str]] = None + bootstrap_port: Optional[Union[List[Optional[int]], int]] = None + bootstrap_room: Optional[Union[List[int], int]] = None + + # For request id + rid: Optional[Union[List[str], str]] = None + + @field_validator("max_tokens") + @classmethod + def validate_max_tokens_positive(cls, v): + if v is not None and v <= 0: + raise ValueError("max_tokens must be positive") + return v + + +class CompletionResponseChoice(BaseModel): + index: int + text: str + logprobs: Optional[LogProbs] = None + finish_reason: Optional[Literal["stop", "length", "content_filter", "abort"]] = None + matched_stop: Union[None, int, str] = None + hidden_states: Optional[object] = None + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.hidden_states is None: + data.pop("hidden_states", None) + return data + + +class CompletionResponse(BaseModel): + id: str + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseChoice] + usage: UsageInfo + metadata: Optional[Dict[str, Any]] = None + + +class CompletionResponseStreamChoice(BaseModel): + index: int + text: str + logprobs: Optional[LogProbs] = None + finish_reason: Optional[Literal["stop", "length", "content_filter", "abort"]] = None + matched_stop: Union[None, int, str] = None + hidden_states: Optional[object] = None + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.hidden_states is None: + data.pop("hidden_states", None) + return data + + +class CompletionStreamResponse(BaseModel): + id: str + object: str = "text_completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[CompletionResponseStreamChoice] + usage: Optional[UsageInfo] = None + + +class ChatCompletionMessageContentTextPart(BaseModel): + type: Literal["text"] + text: str + + +class ChatCompletionMessageContentImageURL(BaseModel): + url: str + detail: Optional[Literal["auto", "low", "high"]] = "auto" + + +class ChatCompletionMessageContentVideoURL(BaseModel): + url: str + + +class ChatCompletionMessageContentAudioURL(BaseModel): + url: str + + +class ChatCompletionMessageContentImagePart(BaseModel): + type: Literal["image_url"] + image_url: ChatCompletionMessageContentImageURL + modalities: Optional[Literal["image", "multi-images", "video"]] = "image" + + +class ChatCompletionMessageContentVideoPart(BaseModel): + type: Literal["video_url"] + video_url: ChatCompletionMessageContentVideoURL + + +class ChatCompletionMessageContentAudioPart(BaseModel): + type: Literal["audio_url"] + audio_url: ChatCompletionMessageContentAudioURL + + +ChatCompletionMessageContentPart = Union[ + ChatCompletionMessageContentTextPart, + ChatCompletionMessageContentImagePart, + ChatCompletionMessageContentVideoPart, + ChatCompletionMessageContentAudioPart, +] + + +class FunctionResponse(BaseModel): + """Function response.""" + + name: Optional[str] = None + arguments: Optional[str] = None + + +class ToolCall(BaseModel): + """Tool call response.""" + + id: Optional[str] = None + index: Optional[int] = None + type: Literal["function"] = "function" + function: FunctionResponse + + +class ChatCompletionMessageGenericParam(BaseModel): + role: Literal["system", "assistant", "tool", "function"] + content: Union[str, List[ChatCompletionMessageContentTextPart], None] = Field( + default=None + ) + tool_call_id: Optional[str] = None + name: Optional[str] = None + reasoning_content: Optional[str] = None + tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) + + @field_validator("role", mode="before") + @classmethod + def _normalize_role(cls, v): + if isinstance(v, str): + v_lower = v.lower() + if v_lower not in {"system", "assistant", "tool", "function"}: + raise ValueError( + "'role' must be one of 'system', 'assistant', 'tool', or 'function' (case-insensitive)." + ) + return v_lower + raise ValueError("'role' must be a string") + + +class ChatCompletionMessageUserParam(BaseModel): + role: Literal["user"] + content: Union[str, List[ChatCompletionMessageContentPart]] + + +ChatCompletionMessageParam = Union[ + ChatCompletionMessageGenericParam, ChatCompletionMessageUserParam +] + + +class Function(BaseModel): + """Function descriptions.""" + + description: Optional[str] = Field(default=None, examples=[None]) + name: Optional[str] = None + parameters: Optional[object] = None + strict: bool = False + + +class Tool(BaseModel): + """Function wrapper.""" + + type: str = Field(default="function", examples=["function"]) + function: Function + + +class ToolChoiceFuncName(BaseModel): + """The name of tool choice function.""" + + name: Optional[str] = None + + +class ToolChoice(BaseModel): + """The tool choice definition.""" + + function: ToolChoiceFuncName + type: Literal["function"] = Field(default="function", examples=["function"]) + + +class ChatCompletionRequest(BaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/chat/create + messages: List[ChatCompletionMessageParam] + model: str = DEFAULT_MODEL_NAME + frequency_penalty: float = 0.0 + logit_bias: Optional[Dict[str, float]] = None + logprobs: bool = False + top_logprobs: Optional[int] = None + max_tokens: Optional[int] = Field( + default=None, + deprecated="max_tokens is deprecated in favor of the max_completion_tokens field", + description="The maximum number of tokens that can be generated in the chat completion. ", + ) + max_completion_tokens: Optional[int] = Field( + default=None, + description="The maximum number of completion tokens for a chat completion request, " + "including visible output tokens and reasoning tokens. Input tokens are not included. ", + ) + n: int = 1 + presence_penalty: float = 0.0 + response_format: Optional[Union[ResponseFormat, StructuralTagResponseFormat]] = None + seed: Optional[int] = None + stop: Optional[Union[str, List[str]]] = None + stream: bool = False + stream_options: Optional[StreamOptions] = None + temperature: float = 0.7 + top_p: float = 1.0 + user: Optional[str] = None + tools: Optional[List[Tool]] = Field(default=None, examples=[None]) + tool_choice: Union[ToolChoice, Literal["auto", "required", "none"]] = Field( + default="auto", examples=["none"] + ) # noqa + return_hidden_states: bool = False + reasoning_effort: Optional[Literal["low", "medium", "high"]] = Field( + default="medium", + description="Constrains effort on reasoning for reasoning models. " + "'low' is the least effort, 'high' is the most effort. Reducing reasoning effort can " + "result in faster responses and fewer tokens used on reasoning in a response. " + "Currently only supported for OpenAI models.", + ) + + @model_validator(mode="before") + @classmethod + def set_tool_choice_default(cls, values): + if values.get("tool_choice") is None: + if values.get("tools") is None: + values["tool_choice"] = "none" + else: + values["tool_choice"] = "auto" + return values + + @model_validator(mode="before") + @classmethod + def normalize_reasoning_inputs(cls, values: Dict): + r = values.get("reasoning") + if r is None: + return values + + if isinstance(r, dict): + effort = r.get("effort") or r.get("reasoning_effort") + if effort in {"low", "medium", "high"}: + values["reasoning_effort"] = effort + + enabled = ( + r.get("enabled") + if r.get("enabled") is not None + else r.get("enable", False) + ) + if isinstance(enabled, str): + enabled = enabled.strip().lower() in {"1", "true", "yes", "y", "on"} + if enabled: + ctk = values.get("chat_template_kwargs") + if not isinstance(ctk, dict): + ctk = {} + ctk.setdefault("thinking", True) + values["chat_template_kwargs"] = ctk + + return values + + @model_validator(mode="before") + @classmethod + def set_json_schema(cls, values): + response_format = values.get("response_format") + if not response_format: + return values + + if response_format.get("type") != "json_schema": + return values + + schema = response_format.pop("schema", None) + json_schema = response_format.get("json_schema") + + if json_schema: + return values + + if schema: + name_ = schema.get("title", "Schema") + strict_ = False + if "properties" in schema and "strict" in schema["properties"]: + item = schema["properties"].pop("strict", None) + if item and item.get("default", False): + strict_ = True + + response_format["json_schema"] = { + "name": name_, + "schema": schema, + "strict": strict_, + } + + return values + + # Extra parameters for SRT backend only and will be ignored by OpenAI models. + top_k: int = -1 + min_p: float = 0.0 + min_tokens: int = 0 + regex: Optional[str] = None + ebnf: Optional[str] = None + repetition_penalty: float = 1.0 + stop_token_ids: Optional[List[int]] = None + no_stop_trim: bool = False + ignore_eos: bool = False + continue_final_message: bool = False + skip_special_tokens: bool = True + lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None + session_params: Optional[Dict] = None + separate_reasoning: bool = True + stream_reasoning: bool = True + chat_template_kwargs: Optional[Dict] = None + + # For request id + rid: Optional[Union[List[str], str]] = None + + # For PD disaggregation + bootstrap_host: Optional[Union[List[str], str]] = None + bootstrap_port: Optional[Union[List[Optional[int]], int]] = None + bootstrap_room: Optional[Union[List[int], int]] = None + + +class ChatMessage(BaseModel): + role: Optional[str] = None + content: Optional[str] = None + reasoning_content: Optional[str] = None + tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) + + +class ChatCompletionResponseChoice(BaseModel): + index: int + message: ChatMessage + logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None + finish_reason: Optional[ + Literal[ + "stop", "length", "tool_calls", "content_filter", "function_call", "abort" + ] + ] = None + matched_stop: Union[None, int, str] = None + hidden_states: Optional[object] = None + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.hidden_states is None: + data.pop("hidden_states", None) + return data + + +class ChatCompletionResponse(BaseModel): + id: str + object: str = "chat.completion" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseChoice] + usage: UsageInfo + metadata: Optional[Dict[str, Any]] = None + + +class DeltaMessage(BaseModel): + role: Optional[str] = None + content: Optional[str] = None + reasoning_content: Optional[str] = None + tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) + hidden_states: Optional[object] = None + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.hidden_states is None: + data.pop("hidden_states", None) + return data + + +class ChatCompletionResponseStreamChoice(BaseModel): + index: int + delta: DeltaMessage + logprobs: Optional[Union[LogProbs, ChoiceLogprobs]] = None + finish_reason: Optional[ + Literal[ + "stop", "length", "tool_calls", "content_filter", "function_call", "abort" + ] + ] = None + matched_stop: Union[None, int, str] = None + + +class ChatCompletionStreamResponse(BaseModel): + id: str + object: str = "chat.completion.chunk" + created: int = Field(default_factory=lambda: int(time.time())) + model: str + choices: List[ChatCompletionResponseStreamChoice] + usage: Optional[UsageInfo] = None + + +class MultimodalEmbeddingInput(BaseModel): + text: Optional[str] = None + image: Optional[str] = None + + +EmbeddingInput = Union[ + List[int], List[List[int]], str, List[str], List[MultimodalEmbeddingInput] +] + + +class EmbeddingRequest(BaseModel): + # Ordered by official OpenAI API documentation + # https://platform.openai.com/docs/api-reference/embeddings/create + input: EmbeddingInput + model: str = DEFAULT_MODEL_NAME + encoding_format: str = "float" + dimensions: Optional[int] = None + user: Optional[str] = None + + # The request id. + rid: Optional[Union[List[str], str]] = None + + +class EmbeddingObject(BaseModel): + embedding: List[float] + index: int + object: str = "embedding" + + +class EmbeddingResponse(BaseModel): + data: List[EmbeddingObject] + model: str + object: str = "list" + usage: Optional[UsageInfo] = None + + +class ScoringRequest(BaseModel): + query: Optional[Union[str, List[int]]] = ( + None # Query text or pre-tokenized token IDs + ) + items: Optional[Union[str, List[str], List[List[int]]]] = ( + None # Item text(s) or pre-tokenized token IDs + ) + label_token_ids: Optional[List[int]] = ( + None # Token IDs to compute probabilities for + ) + apply_softmax: bool = False + item_first: bool = False + model: str = DEFAULT_MODEL_NAME + + +class ScoringResponse(BaseModel): + scores: List[ + List[float] + ] # List of lists of probabilities, each in the order of label_token_ids + model: str + usage: Optional[UsageInfo] = None + object: str = "scoring" + + +class V1RerankReqInput(BaseModel): + query: str + documents: List[str] + + +class RerankResponse(BaseModel): + score: float + document: str + index: int + meta_info: Optional[dict] = None + + +OpenAIServingRequest = Union[ + ChatCompletionRequest, + CompletionRequest, + EmbeddingRequest, + ScoringRequest, + V1RerankReqInput, +] + + +# Response API protocol definitions +class ResponseReasoningParam(BaseModel): + """Reasoning parameters for responses.""" + + effort: Optional[Literal["low", "medium", "high"]] = Field( + default="medium", + description="Constrains effort on reasoning for reasoning models.", + ) + + +class ResponseTool(BaseModel): + """Tool definition for responses.""" + + type: Literal["web_search_preview", "code_interpreter"] = Field( + description="Type of tool to enable" + ) + + +ResponseInputOutputItem: TypeAlias = Union[ + ResponseInputItemParam, + "ResponseReasoningItem", + ResponseFunctionToolCall, +] + + +class ResponsesRequest(BaseModel): + """Request body for v1/responses endpoint.""" + + # Core OpenAI API fields (ordered by official documentation) + background: Optional[bool] = False + include: Optional[ + List[ + Literal[ + "code_interpreter_call.outputs", + "computer_call_output.output.image_url", + "file_search_call.results", + "message.input_image.image_url", + "message.output_text.logprobs", + "reasoning.encrypted_content", + ] + ] + ] = None + input: Union[str, List[ResponseInputOutputItem]] + instructions: Optional[str] = None + max_output_tokens: Optional[int] = None + max_tool_calls: Optional[int] = None + metadata: Optional[Dict[str, Any]] = None + model: Optional[str] = None # Made optional to match vLLM + parallel_tool_calls: Optional[bool] = True + previous_response_id: Optional[str] = None + reasoning: Optional[ResponseReasoningParam] = None + service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto" + store: Optional[bool] = True + stream: Optional[bool] = False + temperature: Optional[float] = None + tool_choice: Literal["auto", "required", "none"] = "auto" + tools: List[ResponseTool] = Field(default_factory=list) + top_logprobs: Optional[int] = 0 + top_p: Optional[float] = None + truncation: Optional[Literal["auto", "disabled"]] = "disabled" + user: Optional[str] = None + + # Extra SGLang parameters + request_id: str = Field( + default_factory=lambda: f"resp_{uuid.uuid4().hex}", + description="The request_id related to this request. If the caller does not set it, a random uuid will be generated.", + ) + priority: int = Field(default=0, description="Request priority") + + # SGLang-specific sampling parameters + frequency_penalty: float = 0.0 + presence_penalty: float = 0.0 + stop: Optional[Union[str, List[str]]] = None + top_k: int = -1 + min_p: float = 0.0 + repetition_penalty: float = 1.0 + + # Default sampling parameters + _DEFAULT_SAMPLING_PARAMS = { + "temperature": 0.7, + "top_p": 1.0, + "top_k": -1, + "min_p": 0.0, + "repetition_penalty": 1.0, + } + + def to_sampling_params( + self, default_max_tokens: int, default_params: Optional[Dict] = None + ) -> Dict[str, Any]: + """Convert to sampling parameters for generation.""" + if default_params is None: + default_params = {} + + # Use max_output_tokens if available, otherwise use max_tokens for backwards compatibility + if self.max_output_tokens is not None: + max_tokens = min(self.max_output_tokens, default_max_tokens) + else: + max_tokens = default_max_tokens + + # Avoid exceed the context length by minus 2 token + max_tokens -= 2 + + # Get parameters with defaults + temperature = self.temperature + if temperature is None: + temperature = default_params.get( + "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"] + ) + + top_p = self.top_p + if top_p is None: + top_p = default_params.get("top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]) + + params = { + "max_new_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "frequency_penalty": self.frequency_penalty, + "presence_penalty": self.presence_penalty, + "stop": self.stop, + "top_k": self.top_k, + "min_p": self.min_p, + "repetition_penalty": self.repetition_penalty, + } + + # Apply any additional default parameters + for key, value in default_params.items(): + if key not in params or params[key] is None: + params[key] = value + + return params + + +class PromptTokenUsageInfo(BaseModel): + """Prompt token usage details.""" + + cached_tokens: int = 0 + + +class ResponsesResponse(BaseModel): + """Response body for v1/responses endpoint.""" + + id: str = Field(default_factory=lambda: f"resp_{time.time()}") + object: Literal["response"] = "response" + created_at: int = Field(default_factory=lambda: int(time.time())) + model: str + + output: List[ + Union[ResponseOutputItem, ResponseReasoningItem, ResponseFunctionToolCall] + ] = Field(default_factory=list) + status: Literal["queued", "in_progress", "completed", "failed", "cancelled"] + usage: Optional[UsageInfo] = None + parallel_tool_calls: bool = True + tool_choice: str = "auto" + tools: List[ResponseTool] = Field(default_factory=list) + + @classmethod + def from_request( + cls, + request: ResponsesRequest, + sampling_params: Any, + model_name: str, + created_time: int, + output: List[ + Union[ResponseOutputItem, ResponseReasoningItem, ResponseFunctionToolCall] + ], + status: str, + usage: Optional[UsageInfo], + ) -> "ResponsesResponse": + """Create a response from a request.""" + return cls( + id=request.request_id, + created_at=created_time, + model=model_name, + output=output, + status=status, + usage=usage, + parallel_tool_calls=request.parallel_tool_calls or True, + tool_choice=request.tool_choice, + tools=request.tools, + ) + + +class RequestResponseMetadata(BaseModel): + """Metadata for request/response tracking.""" + + request_id: str + final_usage_info: Optional[UsageInfo] = None + + +@dataclass +class MessageProcessingResult: + """Result of processing chat messages and applying templates. + + This dataclass encapsulates all the outputs from message processing including + prompt generation, multimodal data extraction, and constraint preparation. + Used internally by OpenAIServingChat to pass processed data between methods. + + Args: + prompt: The final text prompt after applying chat template + prompt_ids: Either the text prompt (str) or tokenized IDs (List[int]) + image_data: Extracted image data from messages, if any + audio_data: Extracted audio data from messages, if any + modalities: List of modality types present in the messages + stop: Combined stop strings from template and request + tool_call_constraint: Optional constraint for structured tool calls + """ + + prompt: str + prompt_ids: Union[str, List[int]] + image_data: Optional[Any] + audio_data: Optional[Any] + video_data: Optional[Any] + modalities: List[str] + stop: List[str] + tool_call_constraint: Optional[Any] = None + + +class ResponseReasoningTextContent(BaseModel): + text: str + type: Literal["reasoning_text"] = "reasoning_text" + + +ResponseInputOutputItem: TypeAlias = Union[ + ResponseInputItemParam, "ResponseReasoningItem", ResponseFunctionToolCall +] diff --git a/python/sglang/srt/entrypoints/openai/serving_base.py b/python/sglang/srt/entrypoints/openai/serving_base.py new file mode 100644 index 000000000..28b317e6d --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_base.py @@ -0,0 +1,156 @@ +from __future__ import annotations + +import json +import logging +import uuid +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any, Optional, Union + +from fastapi import HTTPException, Request +from fastapi.responses import ORJSONResponse, StreamingResponse + +from sglang.srt.entrypoints.openai.protocol import ErrorResponse, OpenAIServingRequest +from sglang.srt.managers.io_struct import GenerateReqInput + +if TYPE_CHECKING: + from sglang.srt.managers.tokenizer_manager import TokenizerManager + +logger = logging.getLogger(__name__) + + +# Base class for specific endpoint handlers +class OpenAIServingBase(ABC): + """Abstract base class for OpenAI endpoint handlers""" + + def __init__(self, tokenizer_manager: TokenizerManager): + self.tokenizer_manager = tokenizer_manager + + async def handle_request( + self, request: OpenAIServingRequest, raw_request: Request + ) -> Union[Any, StreamingResponse, ErrorResponse]: + """Handle the specific request type with common pattern""" + try: + # Validate request + error_msg = self._validate_request(request) + if error_msg: + return self.create_error_response(error_msg) + + # Convert to internal format + adapted_request, processed_request = self._convert_to_internal_request( + request + ) + + # Note(Xinyuan): raw_request below is only used for detecting the connection of the client + if hasattr(request, "stream") and request.stream: + return await self._handle_streaming_request( + adapted_request, processed_request, raw_request + ) + else: + return await self._handle_non_streaming_request( + adapted_request, processed_request, raw_request + ) + except HTTPException as e: + return self.create_error_response( + message=e.detail, err_type=str(e.status_code), status_code=e.status_code + ) + except Exception as e: + logger.exception(f"Error in request: {e}") + return self.create_error_response( + message=f"Internal server error: {str(e)}", + err_type="InternalServerError", + status_code=500, + ) + + @abstractmethod + def _request_id_prefix(self) -> str: + """Generate request ID based on request type""" + pass + + def _generate_request_id_base(self, request: OpenAIServingRequest) -> Optional[str]: + """Generate request ID based on request type""" + return None + + # TODO(chang): the rid is used in io_strcut check and often violates `The rid should be a list` AssertionError + # Temporarily return None in this function until the rid logic is clear. + if rid := getattr(request, "rid", None): + return rid + + return f"{self._request_id_prefix()}{uuid.uuid4().hex}" + + @abstractmethod + def _convert_to_internal_request( + self, + request: OpenAIServingRequest, + ) -> tuple[GenerateReqInput, OpenAIServingRequest]: + """Convert OpenAI request to internal format""" + pass + + async def _handle_streaming_request( + self, + adapted_request: GenerateReqInput, + request: OpenAIServingRequest, + raw_request: Request, + ) -> Union[StreamingResponse, ErrorResponse, ORJSONResponse]: + """Handle streaming request + + Override this method in child classes that support streaming requests. + """ + return self.create_error_response( + message=f"{self.__class__.__name__} does not support streaming requests", + err_type="NotImplementedError", + status_code=501, + ) + + async def _handle_non_streaming_request( + self, + adapted_request: GenerateReqInput, + request: OpenAIServingRequest, + raw_request: Request, + ) -> Union[Any, ErrorResponse, ORJSONResponse]: + """Handle non-streaming request + + Override this method in child classes that support non-streaming requests. + """ + return self.create_error_response( + message=f"{self.__class__.__name__} does not support non-streaming requests", + err_type="NotImplementedError", + status_code=501, + ) + + def _validate_request(self, _: OpenAIServingRequest) -> Optional[str]: + """Validate request""" + pass + + def create_error_response( + self, + message: str, + err_type: str = "BadRequestError", + status_code: int = 400, + param: Optional[str] = None, + ) -> ORJSONResponse: + """Create an error response""" + # TODO: remove fastapi dependency in openai and move response handling to the entrypoint + error = ErrorResponse( + object="error", + message=message, + type=err_type, + param=param, + code=status_code, + ) + return ORJSONResponse(content=error.model_dump(), status_code=status_code) + + def create_streaming_error_response( + self, + message: str, + err_type: str = "BadRequestError", + status_code: int = 400, + ) -> str: + """Create a streaming error response""" + error = ErrorResponse( + object="error", + message=message, + type=err_type, + param=None, + code=status_code, + ) + return json.dumps({"error": error.model_dump()}) diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py new file mode 100644 index 000000000..d67cbfde3 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -0,0 +1,1083 @@ +from __future__ import annotations + +import copy +import json +import logging +import time +import uuid +from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union + +from fastapi import Request +from fastapi.responses import ORJSONResponse, StreamingResponse + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + ChatCompletionTokenLogprob, + ChatMessage, + ChoiceLogprobs, + DeltaMessage, + ErrorResponse, + FunctionResponse, + LogProbs, + MessageProcessingResult, + ToolCall, + TopLogprob, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase +from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor +from sglang.srt.entrypoints.openai.utils import ( + process_hidden_states_from_ret, + to_openai_style_logprobs, +) +from sglang.srt.function_call.function_call_parser import FunctionCallParser +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.parser.conversation import generate_chat_conv +from sglang.srt.parser.jinja_template_utils import process_content_for_template_format +from sglang.srt.parser.reasoning_parser import ReasoningParser +from sglang.utils import convert_json_schema_to_str + +if TYPE_CHECKING: + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.managers.tokenizer_manager import TokenizerManager + +logger = logging.getLogger(__name__) + + +class OpenAIServingChat(OpenAIServingBase): + """Handler for /v1/chat/completions requests""" + + def __init__( + self, + tokenizer_manager: TokenizerManager, + template_manager: TemplateManager, + ): + super().__init__(tokenizer_manager) + self.template_manager = template_manager + self.tool_call_parser = self.tokenizer_manager.server_args.tool_call_parser + + def _request_id_prefix(self) -> str: + return "chatcmpl-" + + def _validate_request(self, request: ChatCompletionRequest) -> Optional[str]: + """Validate that the input is valid.""" + if not request.messages: + return "Messages cannot be empty." + + if ( + isinstance(request.tool_choice, str) + and request.tool_choice.lower() == "required" + and not request.tools + ): + return "Tools cannot be empty if tool choice is set to required." + + max_output_tokens = request.max_completion_tokens or request.max_tokens + server_context_length = self.tokenizer_manager.server_args.context_length + if ( + max_output_tokens + and server_context_length + and max_output_tokens > server_context_length + ): + return ( + f"max_completion_tokens is too large: {max_output_tokens}." + f"This model supports at most {server_context_length} completion tokens." + ) + + if request.response_format and request.response_format.type == "json_schema": + schema = getattr(request.response_format.json_schema, "schema_", None) + if schema is None: + return "schema_ is required for json_schema response format request." + + return None + + def _convert_to_internal_request( + self, + request: ChatCompletionRequest, + ) -> tuple[GenerateReqInput, ChatCompletionRequest]: + reasoning_effort = ( + request.chat_template_kwargs.pop("reasoning_effort", None) + if request.chat_template_kwargs + else None + ) + if reasoning_effort is not None: + request.reasoning_effort = reasoning_effort + + """Convert OpenAI chat completion request to internal format""" + is_multimodal = self.tokenizer_manager.model_config.is_multimodal + + # Process messages and apply chat template + processed_messages = self._process_messages(request, is_multimodal) + + # Build sampling parameters + sampling_params = self._build_sampling_params( + request, + processed_messages.stop, + processed_messages.tool_call_constraint, + ) + + # Handle single vs multiple requests + if is_multimodal: + prompt_kwargs = {"text": processed_messages.prompt} + else: + if isinstance(processed_messages.prompt_ids, str): + prompt_kwargs = {"text": processed_messages.prompt_ids} + else: + prompt_kwargs = {"input_ids": processed_messages.prompt_ids} + + adapted_request = GenerateReqInput( + **prompt_kwargs, + image_data=processed_messages.image_data, + video_data=processed_messages.video_data, + audio_data=processed_messages.audio_data, + sampling_params=sampling_params, + return_logprob=request.logprobs, + logprob_start_len=-1, + top_logprobs_num=request.top_logprobs or 0, + stream=request.stream, + return_text_in_logprobs=True, + modalities=processed_messages.modalities, + lora_path=request.lora_path, + bootstrap_host=request.bootstrap_host, + bootstrap_port=request.bootstrap_port, + bootstrap_room=request.bootstrap_room, + return_hidden_states=request.return_hidden_states, + rid=request.rid, + ) + + return adapted_request, request + + def _process_messages( + self, request: ChatCompletionRequest, is_multimodal: bool + ) -> MessageProcessingResult: + """Process chat messages and apply chat template""" + is_gpt_oss = ( + hasattr(self.tokenizer_manager.model_config, "hf_config") + and hasattr(self.tokenizer_manager.model_config.hf_config, "model_type") + and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss" + ) + + # GptOss model needs to keep special tokens for harmony parsing + if is_gpt_oss: + request.skip_special_tokens = False + + tool_call_constraint = None + + # Apply chat template and its stop strings + tools = None + if request.tools and request.tool_choice != "none": + request.skip_special_tokens = False + if not isinstance(request.tool_choice, str): + tools = [ + item.function.model_dump() + for item in request.tools + if item.function.name == request.tool_choice.function.name + ] + else: + tools = [item.function.model_dump() for item in request.tools] + if self.tool_call_parser: + parser = FunctionCallParser(request.tools, self.tool_call_parser) + tool_call_constraint = parser.get_structure_constraint( + request.tool_choice + ) + + # Use chat template + if self.template_manager.chat_template_name is None: + result = self._apply_jinja_template(request, tools, is_multimodal) + else: + result = self._apply_conversation_template(request, is_multimodal) + + result.tool_call_constraint = tool_call_constraint + return result + + def _apply_jinja_template( + self, + request: ChatCompletionRequest, + tools: Optional[List[Dict]], + is_multimodal: bool, + ) -> MessageProcessingResult: + """Apply Jinja chat template""" + prompt = "" + prompt_ids = [] + openai_compatible_messages = [] + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + template_content_format = self.template_manager.jinja_template_content_format + + for message in request.messages: + if message.content is None: + message.content = "" + msg_dict = message.model_dump() + + # Process content based on detected template format + processed_msg = process_content_for_template_format( + msg_dict, + template_content_format, + image_data, + video_data, + audio_data, + modalities, + ) + + # per the Transformers docs & maintainers, tool call arguments in + # assistant-role messages with tool_calls need to be dicts not JSON str - + # this is how tool-use chat templates will expect them moving forwards + # so, for messages that have tool_calls, parse the string (which we get + # from openAI format) to dict + if ( + processed_msg["role"] == "assistant" + and "tool_calls" in processed_msg + and isinstance(processed_msg["tool_calls"], list) + ): + for item in processed_msg["tool_calls"]: + if "arguments" in item["function"] and isinstance( + item["function"]["arguments"], str + ): + item["function"]["arguments"] = json.loads( + item["function"]["arguments"] + ) + + openai_compatible_messages.append(processed_msg) + + # Handle assistant prefix for continue_final_message + assistant_prefix = None + if ( + openai_compatible_messages + and openai_compatible_messages[-1]["role"] == "assistant" + ): + if request.continue_final_message: + assistant_prefix = openai_compatible_messages[-1]["content"] + openai_compatible_messages = openai_compatible_messages[:-1] + + try: + prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( + openai_compatible_messages, + tokenize=True, + add_generation_prompt=True, + tools=tools, + reasoning_effort=request.reasoning_effort, + **( + request.chat_template_kwargs if request.chat_template_kwargs else {} + ), + ) + except Exception: + # This except branch will be triggered when the chosen model + # has a different tools input format that is not compatible + # with openAI's apply_chat_template tool_call format, like Mistral. + tools = ( + [t if "function" in t else {"function": t} for t in tools] + if tools + else None + ) + prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template( + openai_compatible_messages, + tokenize=True, + add_generation_prompt=True, + tools=tools, + reasoning_effort=request.reasoning_effort, + **( + request.chat_template_kwargs if request.chat_template_kwargs else {} + ), + ) + + if assistant_prefix: + encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix) + if encoded and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id: + encoded = encoded[1:] + prompt_ids += encoded + + if is_multimodal: + prompt = self.tokenizer_manager.tokenizer.decode(prompt_ids) + + stop = request.stop + image_data = image_data if image_data else None + audio_data = audio_data if audio_data else None + video_data = video_data if video_data else None + modalities = modalities if modalities else [] + return MessageProcessingResult( + prompt=prompt, + prompt_ids=prompt_ids, + image_data=image_data, + video_data=video_data, + audio_data=audio_data, + modalities=modalities, + stop=stop, + ) + + def _apply_conversation_template( + self, + request: ChatCompletionRequest, + is_multimodal: bool, + ) -> MessageProcessingResult: + """Apply conversation template""" + prompt = "" + prompt_ids = [] + conv = generate_chat_conv(request, self.template_manager.chat_template_name) + + # If we should continue the final assistant message, adjust the conversation. + if ( + request.continue_final_message + and request.messages + and request.messages[-1].role == "assistant" + ): + # Remove the auto-added blank assistant turn, if present. + if conv.messages and conv.messages[-1][1] is None: + conv.messages.pop() + # Rebuild the prompt from the conversation. + prompt = conv.get_prompt() + # Strip trailing stop tokens or separators that indicate end-of-assistant. + if isinstance(conv.stop_str, list): + for stop_token in conv.stop_str: + if prompt.endswith(stop_token): + prompt = prompt[: -len(stop_token)] + elif isinstance(conv.stop_str, str) and prompt.endswith(conv.stop_str): + prompt = prompt[: -len(conv.stop_str)] + if conv.sep and prompt.endswith(conv.sep): + prompt = prompt[: -len(conv.sep)] + if getattr(conv, "sep2", None) and prompt.endswith(conv.sep2): + prompt = prompt[: -len(conv.sep2)] + else: + prompt = conv.get_prompt() + if self._get_enable_thinking_from_request(request): + prompt += "" # Note(Xinyuan): hard code thinking token + + image_data = conv.image_data if conv.image_data else None + video_data = conv.video_data if conv.video_data else None + audio_data = conv.audio_data if conv.audio_data else None + modalities = conv.modalities if conv.modalities else [] + stop = copy.copy(conv.stop_str or [] if not request.ignore_eos else []) + + if request.stop: + if isinstance(request.stop, str): + stop.append(request.stop) + else: + stop.extend(request.stop) + + if not is_multimodal: + prompt_ids = self.tokenizer_manager.tokenizer.encode(prompt) + + return MessageProcessingResult( + prompt=prompt, + prompt_ids=prompt_ids, + image_data=image_data, + video_data=video_data, + audio_data=audio_data, + modalities=modalities, + stop=stop, + ) + + def _build_sampling_params( + self, + request: ChatCompletionRequest, + stop: List[str], + tool_call_constraint: Optional[Any], + ) -> Dict[str, Any]: + """Build sampling parameters for the request""" + + sampling_params = { + "temperature": request.temperature, + "max_new_tokens": request.max_tokens or request.max_completion_tokens, + "min_new_tokens": request.min_tokens, + "stop": stop, + "stop_token_ids": request.stop_token_ids, + "top_p": request.top_p, + "top_k": request.top_k, + "min_p": request.min_p, + "presence_penalty": request.presence_penalty, + "frequency_penalty": request.frequency_penalty, + "repetition_penalty": request.repetition_penalty, + "regex": request.regex, + "ebnf": request.ebnf, + "n": request.n, + "no_stop_trim": request.no_stop_trim, + "ignore_eos": request.ignore_eos, + "skip_special_tokens": request.skip_special_tokens, + "logit_bias": request.logit_bias, + } + + if request.response_format and request.response_format.type == "json_schema": + sampling_params["json_schema"] = convert_json_schema_to_str( + request.response_format.json_schema.schema_ + ) + elif request.response_format and request.response_format.type == "json_object": + sampling_params["json_schema"] = '{"type": "object"}' + elif ( + request.response_format and request.response_format.type == "structural_tag" + ): + sampling_params["structural_tag"] = convert_json_schema_to_str( + request.response_format.model_dump(by_alias=True) + ) + + # Check if there are already existing output constraints + has_existing_constraints = ( + sampling_params.get("regex") + or sampling_params.get("ebnf") + or sampling_params.get("structural_tag") + or sampling_params.get("json_schema") + ) + + if tool_call_constraint and has_existing_constraints: + logger.warning("Constrained decoding is not compatible with tool calls.") + elif tool_call_constraint: + constraint_type, constraint_value = tool_call_constraint + if constraint_type == "structural_tag": + sampling_params[constraint_type] = convert_json_schema_to_str( + constraint_value.model_dump(by_alias=True) + ) + else: + sampling_params[constraint_type] = constraint_value + return sampling_params + + async def _handle_streaming_request( + self, + adapted_request: GenerateReqInput, + request: ChatCompletionRequest, + raw_request: Request, + ) -> StreamingResponse: + """Handle streaming chat completion request""" + return StreamingResponse( + self._generate_chat_stream(adapted_request, request, raw_request), + media_type="text/event-stream", + background=self.tokenizer_manager.create_abort_task(adapted_request), + ) + + async def _generate_chat_stream( + self, + adapted_request: GenerateReqInput, + request: ChatCompletionRequest, + raw_request: Request, + ) -> AsyncGenerator[str, None]: + """Generate streaming chat completion response""" + # Parsers for tool calls and reasoning + parser_dict = {} + reasoning_parser_dict = {} + + # State tracking for streaming + is_firsts = {} + stream_buffers = {} + n_prev_tokens = {} + has_tool_calls = {} + finish_reasons = {} + + # Usage tracking + prompt_tokens = {} + completion_tokens = {} + cached_tokens = {} + hidden_states = {} + + try: + async for content in self.tokenizer_manager.generate_request( + adapted_request, raw_request + ): + index = content.get("index", 0) + + prompt_tokens[index] = content["meta_info"]["prompt_tokens"] + completion_tokens[index] = content["meta_info"]["completion_tokens"] + cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) + hidden_states[index] = content["meta_info"].get("hidden_states", None) + + # Handle logprobs + choice_logprobs = None + if request.logprobs: + choice_logprobs = self._process_streaming_logprobs( + content, n_prev_tokens.get(index, 0) + ) + n_prev_tokens[index] = len( + content["meta_info"]["output_token_logprobs"] + ) + + finish_reason = content["meta_info"]["finish_reason"] + finish_reason_type = finish_reason["type"] if finish_reason else None + + # Track finish_reason for each index + if finish_reason_type: + finish_reasons[index] = finish_reason + + # First chunk with role + if is_firsts.get(index, True): + is_firsts[index] = False + delta = DeltaMessage(role="assistant", content="") + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=delta, + finish_reason=None, + logprobs=None, + ) + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + + stream_buffer = stream_buffers.get(index, "") + delta = content["text"][len(stream_buffer) :] + stream_buffers[index] = stream_buffer + delta + + # Handle reasoning content + if ( + self.tokenizer_manager.server_args.reasoning_parser + and request.separate_reasoning + ): + reasoning_text, delta = self._process_reasoning_stream( + index, delta, reasoning_parser_dict, content, request + ) + if reasoning_text: + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(reasoning_content=reasoning_text), + finish_reason=None, + ) + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + + # Handle tool calls + if ( + request.tool_choice != "none" + and request.tools + and self.tool_call_parser + ): + async for chunk in self._process_tool_call_stream( + index, + delta, + parser_dict, + content, + request, + has_tool_calls, + ): + if chunk: + yield chunk + + # Send any remaining tool call arguments when generation finishes + if finish_reason_type is not None and index in parser_dict: + parser = parser_dict[index] + remaining_chunk = self._check_for_unstreamed_tool_args( + parser, content, request, index + ) + if remaining_chunk: + yield remaining_chunk + + else: + # Regular content + if delta: + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(content=delta), + finish_reason=None, + matched_stop=None, + logprobs=choice_logprobs, + ) + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + + # Send finish_reason chunks for each index that completed + for idx, finish_reason_data in finish_reasons.items(): + finish_reason_type = finish_reason_data["type"] + + # Change finish_reason to "tool_calls" if we had tool calls and stopped naturally + final_finish_reason = finish_reason_type + if has_tool_calls.get(idx, False) and finish_reason_type == "stop": + final_finish_reason = "tool_calls" + + finish_reason_chunk = ChatCompletionStreamResponse( + id=content["meta_info"][ + "id" + ], # NOTE: openai uses the same chatcmpl-id for all indices + created=int(time.time()), + choices=[ + ChatCompletionResponseStreamChoice( + index=idx, + delta=DeltaMessage(), + finish_reason=final_finish_reason, + matched_stop=( + finish_reason_data["matched"] + if "matched" in finish_reason_data + else None + ), + ) + ], + model=request.model, + usage=None, + ) + yield f"data: {finish_reason_chunk.model_dump_json()}\n\n" + + # Send hidden states if requested + if request.return_hidden_states and hidden_states: + for index, choice_hidden_states in hidden_states.items(): + if choice_hidden_states: + last_token_hidden_states = ( + choice_hidden_states[-1] + if len(choice_hidden_states) > 1 + else [] + ) + hidden_states_chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[ + ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage( + hidden_states=last_token_hidden_states + ), + finish_reason=None, # Hidden states don't need finish_reason + ) + ], + model=request.model, + ) + yield f"data: {hidden_states_chunk.model_dump_json()}\n\n" + + # Additional usage chunk + if request.stream_options and request.stream_options.include_usage: + usage = UsageProcessor.calculate_streaming_usage( + prompt_tokens, + completion_tokens, + cached_tokens, + n_choices=request.n, + enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + ) + usage_chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[], # Empty choices array as per OpenAI spec + model=request.model, + usage=usage, + ) + yield f"data: {usage_chunk.model_dump_json()}\n\n" + + except ValueError as e: + error = self.create_streaming_error_response(str(e)) + yield f"data: {error}\n\n" + + yield "data: [DONE]\n\n" + + async def _handle_non_streaming_request( + self, + adapted_request: GenerateReqInput, + request: ChatCompletionRequest, + raw_request: Request, + ) -> Union[ChatCompletionResponse, ErrorResponse, ORJSONResponse]: + """Handle non-streaming chat completion request""" + try: + ret = await self.tokenizer_manager.generate_request( + adapted_request, raw_request + ).__anext__() + except ValueError as e: + return self.create_error_response(str(e)) + + if not isinstance(ret, list): + ret = [ret] + + response = self._build_chat_response( + request, + ret, + int(time.time()), + ) + + return response + + def _build_chat_response( + self, + request: ChatCompletionRequest, + ret: List[Dict[str, Any]], + created: int, + ) -> Union[ChatCompletionResponse, ORJSONResponse]: + """Build chat completion response from generation results""" + choices = [] + + for idx, ret_item in enumerate(ret): + # Process logprobs + choice_logprobs = None + if request.logprobs: + choice_logprobs = self._process_response_logprobs(ret_item) + + # Handle hidden states + hidden_states = process_hidden_states_from_ret(ret_item, request) + + finish_reason = ret_item["meta_info"]["finish_reason"] + text = ret_item["text"] + + # Handle reasoning content + reasoning_text = None + reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser + if reasoning_parser and request.separate_reasoning: + is_force_reasoning = ( + self.template_manager.force_reasoning + or self._get_enable_thinking_from_request(request) + ) + try: + parser = ReasoningParser( + model_type=reasoning_parser, + stream_reasoning=False, + force_reasoning=is_force_reasoning, + ) + reasoning_text, text = parser.parse_non_stream(text) + except Exception as e: + logger.error(f"Reasoning parsing error: {e}") + return self.create_error_response( + "Failed to parse reasoning content", + err_type="InternalServerError", + status_code=500, + ) + + # Handle tool calls + tool_calls = None + if ( + request.tool_choice != "none" + and request.tools + and self.tool_call_parser + ): + tool_calls, text, finish_reason = self._process_tool_calls( + text, request.tools, finish_reason + ) + + choice_data = ChatCompletionResponseChoice( + index=idx, + message=ChatMessage( + role="assistant", + content=text if text else None, + tool_calls=tool_calls, + reasoning_content=reasoning_text if reasoning_text else None, + ), + logprobs=choice_logprobs, + finish_reason=finish_reason["type"] if finish_reason else None, + matched_stop=( + finish_reason["matched"] + if finish_reason and "matched" in finish_reason + else None + ), + hidden_states=hidden_states, + ) + choices.append(choice_data) + + # Calculate usage + usage = UsageProcessor.calculate_response_usage( + ret, + n_choices=request.n, + enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + ) + + return ChatCompletionResponse( + id=ret[0]["meta_info"]["id"], + created=created, + model=request.model, + choices=choices, + usage=usage, + metadata={"weight_version": ret[0]["meta_info"]["weight_version"]}, + ) + + def _process_logprobs_tokens( + self, logprobs: LogProbs, use_token_index: bool = False + ) -> List[ChatCompletionTokenLogprob]: + """Common helper to process logprobs tokens for both streaming and non-streaming + + Args: + logprobs: LogProbs data from model + use_token_index: True for non-streaming (use token_idx), False for streaming (use index 0) + """ + token_logprobs = [] + + for token_idx, (token, logprob) in enumerate( + zip(logprobs.tokens, logprobs.token_logprobs) + ): + token_bytes = list(token.encode("utf-8")) + top_logprobs = [] + if logprobs.top_logprobs: + # - Non-streaming (use_token_index=True): uses token_idx for full data + # - Streaming (use_token_index=False): uses index 0 for pre-sliced data + top_logprobs_idx = token_idx if use_token_index else 0 + for top_token, top_logprob in logprobs.top_logprobs[ + top_logprobs_idx + ].items(): + top_token_bytes = list(top_token.encode("utf-8")) + top_logprobs.append( + TopLogprob( + token=top_token, + bytes=top_token_bytes, + logprob=top_logprob, + ) + ) + token_logprobs.append( + ChatCompletionTokenLogprob( + token=token, + bytes=token_bytes, + logprob=logprob, + top_logprobs=top_logprobs, + ) + ) + + return token_logprobs + + def _process_response_logprobs(self, ret_item: Dict[str, Any]) -> ChoiceLogprobs: + """Process logprobs for non-streaming response""" + logprobs = to_openai_style_logprobs( + output_token_logprobs=ret_item["meta_info"]["output_token_logprobs"], + output_top_logprobs=ret_item["meta_info"].get("output_top_logprobs", None), + ) + + token_logprobs = self._process_logprobs_tokens(logprobs, use_token_index=True) + return ChoiceLogprobs(content=token_logprobs) + + def _process_tool_calls( + self, + text: str, + tools: List[Any], + finish_reason: Dict[str, Any], + ) -> tuple[Optional[List[ToolCall]], str, Dict[str, Any]]: + """Process tool calls in the response""" + parser = FunctionCallParser(tools, self.tool_call_parser) + if parser.has_tool_call(text): + if finish_reason["type"] == "stop": + finish_reason["type"] = "tool_calls" + finish_reason["matched"] = None + try: + text, call_info_list = parser.parse_non_stream(text) + tool_calls = [] + for call_info in call_info_list: + # For Kimi-K2, align tool_call_id with the model format: functions.{name}:{index} + if ( + self.tool_call_parser == "kimi_k2" + and call_info.name is not None + ): + tool_id = f"functions.{call_info.name}:{call_info.tool_index}" + else: + tool_id = f"call_{uuid.uuid4().hex[:24]}" + + tool_calls.append( + ToolCall( + id=tool_id, + index=getattr(call_info, "tool_index", None), + function=FunctionResponse( + name=call_info.name, arguments=call_info.parameters + ), + ) + ) + return tool_calls, text, finish_reason + except Exception as e: + logger.error(f"Tool call parsing error: {e}") + # Return error but don't fail the whole request + return None, text, finish_reason + + return None, text, finish_reason + + def _process_streaming_logprobs( + self, content: Dict[str, Any], n_prev_token: int + ) -> ChoiceLogprobs: + """Process logprobs for streaming response""" + logprobs = to_openai_style_logprobs( + output_token_logprobs=content["meta_info"]["output_token_logprobs"][ + n_prev_token: + ], + output_top_logprobs=content["meta_info"].get("output_top_logprobs", [])[ + n_prev_token: + ], + ) + + token_logprobs = self._process_logprobs_tokens(logprobs, use_token_index=False) + return ChoiceLogprobs(content=token_logprobs) + + def _process_reasoning_stream( + self, + index: int, + delta: str, + reasoning_parser_dict: Dict[int, ReasoningParser], + content: Dict[str, Any], + request: ChatCompletionRequest, + ) -> tuple[Optional[str], str]: + """Process reasoning content in streaming response""" + if index not in reasoning_parser_dict: + is_force_reasoning = ( + self.template_manager.force_reasoning + or self._get_enable_thinking_from_request(request) + ) + reasoning_parser_dict[index] = ReasoningParser( + self.tokenizer_manager.server_args.reasoning_parser, + request.stream_reasoning, + is_force_reasoning, + ) + reasoning_parser = reasoning_parser_dict[index] + return reasoning_parser.parse_stream_chunk(delta) + + def _get_enable_thinking_from_request(self, request: ChatCompletionRequest) -> bool: + """Extracts the 'enable_thinking' flag from request chat_template_kwargs. + + NOTE: This parameter is only useful for models that support enable_thinking + flag, such as Qwen3. + + Args: + request_obj: The request object (or an item from a list of requests). + Returns: + The boolean value of 'enable_thinking' if found, otherwise False. + """ + if hasattr(request, "chat_template_kwargs") and request.chat_template_kwargs: + # For Qwen3 models, `enable_thinking` is supported. + if request.chat_template_kwargs.get("enable_thinking") is not None: + return request.chat_template_kwargs.get("enable_thinking") + # For DeepSeek-V3.1 models, `thinking` is supported. + elif request.chat_template_kwargs.get("thinking") is not None: + return request.chat_template_kwargs.get("thinking") + else: + return False + return False + + async def _process_tool_call_stream( + self, + index: int, + delta: str, + parser_dict: Dict[int, FunctionCallParser], + content: Dict[str, Any], + request: ChatCompletionRequest, + has_tool_calls: Dict[int, bool], + ): + """Process tool calls in streaming response""" + if index not in parser_dict: + parser_dict[index] = FunctionCallParser( + tools=request.tools, + tool_call_parser=self.tool_call_parser, + ) + parser = parser_dict[index] + + normal_text, calls = parser.parse_stream_chunk(delta) + + # Yield normal text + if normal_text: + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(content=normal_text), + finish_reason=None, + ) + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + + # Yield tool calls + for call_item in calls: + # Mark that this choice has tool calls + has_tool_calls[index] = True + + # Tool call ID should be generated only once per tool call + if call_item.name: + # First chunk: include ID and function name + if self.tool_call_parser == "kimi_k2": + # Align with Kimi-K2 format: functions.{name}:{index} + tool_call_id = f"functions.{call_item.name}:{call_item.tool_index}" + else: + tool_call_id = f"call_{uuid.uuid4().hex[:24]}" + function_name = call_item.name + else: + # Subsequent chunks: null ID and name for argument deltas + tool_call_id = None + function_name = None + + tool_call = ToolCall( + id=tool_call_id, + index=call_item.tool_index, + function=FunctionResponse( + name=function_name, + arguments=call_item.parameters, + ), + ) + + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(tool_calls=[tool_call]), + finish_reason=None, + ) + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + yield f"data: {chunk.model_dump_json()}\n\n" + + def _check_for_unstreamed_tool_args( + self, + parser: FunctionCallParser, + content: Dict[str, Any], + request: ChatCompletionRequest, + index: int, + ) -> Optional[str]: + """ + Check for any remaining tool call arguments that need to be streamed + when generation finishes. This ensures tool calls are properly completed + even if the model generates the final arguments in the last chunk. + """ + # Only check if we have tool calls and the parser has tracked data + if ( + not hasattr(parser.detector, "prev_tool_call_arr") + or not parser.detector.prev_tool_call_arr + ): + return None + + if ( + not hasattr(parser.detector, "streamed_args_for_tool") + or not parser.detector.streamed_args_for_tool + ): + return None + + # Get the last tool call that was being processed + tool_index = len(parser.detector.prev_tool_call_arr) - 1 + if tool_index < 0 or tool_index >= len(parser.detector.streamed_args_for_tool): + return None + + # Get expected vs actual arguments + expected_args = parser.detector.prev_tool_call_arr[tool_index].get( + "arguments", {} + ) + expected_call = json.dumps(expected_args, ensure_ascii=False) + actual_call = parser.detector.streamed_args_for_tool[tool_index] + + # Check if there are remaining arguments to send + remaining_call = ( + expected_call.replace(actual_call, "", 1) + if actual_call in expected_call + else "" + ) + + if remaining_call: + # Create tool call chunk with remaining arguments + tool_call = ToolCall( + id=None, # No ID for argument deltas + index=tool_index, + function=FunctionResponse( + name=None, # No name for argument deltas + arguments=remaining_call, + ), + ) + + choice_data = ChatCompletionResponseStreamChoice( + index=index, + delta=DeltaMessage(tool_calls=[tool_call]), + finish_reason=None, # Don't send finish_reason with this chunk + ) + + chunk = ChatCompletionStreamResponse( + id=content["meta_info"]["id"], + created=int(time.time()), + choices=[choice_data], + model=request.model, + ) + + return f"data: {chunk.model_dump_json()}\n\n" + + return None diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py new file mode 100644 index 000000000..6fe02d325 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -0,0 +1,455 @@ +from __future__ import annotations + +import logging +import time +from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union + +from fastapi import Request +from fastapi.responses import ORJSONResponse, StreamingResponse + +from sglang.srt.entrypoints.openai.protocol import ( + CompletionRequest, + CompletionResponse, + CompletionResponseChoice, + CompletionResponseStreamChoice, + CompletionStreamResponse, + ErrorResponse, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase +from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor +from sglang.srt.entrypoints.openai.utils import ( + process_hidden_states_from_ret, + to_openai_style_logprobs, +) +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.parser.code_completion_parser import ( + generate_completion_prompt_from_request, +) +from sglang.utils import convert_json_schema_to_str + +if TYPE_CHECKING: + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.managers.tokenizer_manager import TokenizerManager + +logger = logging.getLogger(__name__) + + +class OpenAIServingCompletion(OpenAIServingBase): + """Handler for /v1/completion requests""" + + def __init__( + self, + tokenizer_manager: TokenizerManager, + template_manager: TemplateManager, + ): + super().__init__(tokenizer_manager) + self.template_manager = template_manager + + def _request_id_prefix(self) -> str: + return "cmpl-" + + def _validate_request(self, request: CompletionRequest) -> Optional[str]: + """Validate that the input is valid.""" + prompt = request.prompt + if not prompt or (isinstance(prompt, list) and all(not p for p in prompt)): + return "Prompt cannot be empty" + + return None + + def _convert_to_internal_request( + self, + request: CompletionRequest, + ) -> tuple[GenerateReqInput, CompletionRequest]: + """Convert OpenAI completion request to internal format""" + # NOTE: with openai API, the prompt's logprobs are always not computed + if request.echo and request.logprobs: + logger.warning( + "Echo is not compatible with logprobs. " + "To compute logprobs of input prompt, please use the native /generate API." + ) + # Process prompt + prompt = request.prompt + if self.template_manager.completion_template_name is not None: + prompt = generate_completion_prompt_from_request(request) + + # Set logprob start length based on echo and logprobs + if request.echo and request.logprobs: + logprob_start_len = 0 + else: + logprob_start_len = -1 + + # Build sampling parameters + sampling_params = self._build_sampling_params(request) + + # Determine prompt format + if isinstance(prompt, str) or ( + isinstance(prompt, list) and isinstance(prompt[0], str) + ): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + adapted_request = GenerateReqInput( + **prompt_kwargs, + sampling_params=sampling_params, + return_logprob=request.logprobs is not None, + top_logprobs_num=request.logprobs if request.logprobs is not None else 0, + logprob_start_len=logprob_start_len, + return_text_in_logprobs=True, + stream=request.stream, + lora_path=request.lora_path, + bootstrap_host=request.bootstrap_host, + bootstrap_port=request.bootstrap_port, + bootstrap_room=request.bootstrap_room, + return_hidden_states=request.return_hidden_states, + rid=request.rid, + ) + + return adapted_request, request + + def _build_sampling_params(self, request: CompletionRequest) -> Dict[str, Any]: + """Build sampling parameters for the request""" + # Start with common parameters + sampling_params = { + "temperature": request.temperature, + "max_new_tokens": request.max_tokens, + "min_new_tokens": request.min_tokens, + "stop": request.stop, + "stop_token_ids": request.stop_token_ids, + "top_p": request.top_p, + "top_k": request.top_k, + "min_p": request.min_p, + "presence_penalty": request.presence_penalty, + "frequency_penalty": request.frequency_penalty, + "repetition_penalty": request.repetition_penalty, + "regex": request.regex, + "json_schema": request.json_schema, + "ebnf": request.ebnf, + "n": request.n, + "no_stop_trim": request.no_stop_trim, + "ignore_eos": request.ignore_eos, + "skip_special_tokens": request.skip_special_tokens, + "logit_bias": request.logit_bias, + } + + # Handle response_format constraints + if request.response_format and request.response_format.type == "json_schema": + sampling_params["json_schema"] = convert_json_schema_to_str( + request.response_format.json_schema.schema_ + ) + elif request.response_format and request.response_format.type == "json_object": + sampling_params["json_schema"] = '{"type": "object"}' + elif ( + request.response_format and request.response_format.type == "structural_tag" + ): + sampling_params["structural_tag"] = convert_json_schema_to_str( + request.response_format.model_dump(by_alias=True) + ) + + return sampling_params + + async def _handle_streaming_request( + self, + adapted_request: GenerateReqInput, + request: CompletionRequest, + raw_request: Request, + ) -> StreamingResponse: + """Handle streaming completion request""" + return StreamingResponse( + self._generate_completion_stream(adapted_request, request, raw_request), + media_type="text/event-stream", + background=self.tokenizer_manager.create_abort_task(adapted_request), + ) + + async def _generate_completion_stream( + self, + adapted_request: GenerateReqInput, + request: CompletionRequest, + raw_request: Request, + ) -> AsyncGenerator[str, None]: + """Generate streaming completion response""" + created = int(time.time()) + + # State tracking for streaming + stream_buffers = {} + n_prev_tokens = {} + + # Usage tracking + prompt_tokens = {} + completion_tokens = {} + cached_tokens = {} + hidden_states = {} + + try: + async for content in self.tokenizer_manager.generate_request( + adapted_request, raw_request + ): + index = content.get("index", 0) + + text = content["text"] + prompt_tokens[index] = content["meta_info"]["prompt_tokens"] + completion_tokens[index] = content["meta_info"]["completion_tokens"] + cached_tokens[index] = content["meta_info"].get("cached_tokens", 0) + hidden_states[index] = content["meta_info"].get("hidden_states", None) + + stream_buffer = stream_buffers.get(index, "") + # Handle echo for first chunk + if not stream_buffer: # The first chunk + if request.echo: + echo_text = self._get_echo_text(request, index) + text = echo_text + text + + # Handle logprobs + logprobs = None + if request.logprobs is not None: + # The first chunk and echo is enabled. + if not stream_buffer and request.echo: + input_token_logprobs = content["meta_info"][ + "input_token_logprobs" + ] + input_top_logprobs = content["meta_info"]["input_top_logprobs"] + else: + input_token_logprobs = None + input_top_logprobs = None + + n_prev_token = n_prev_tokens.get(index, 0) + logprobs = to_openai_style_logprobs( + input_token_logprobs=input_token_logprobs, + input_top_logprobs=input_top_logprobs, + output_token_logprobs=content["meta_info"][ + "output_token_logprobs" + ][n_prev_token:], + output_top_logprobs=content["meta_info"]["output_top_logprobs"][ + n_prev_token: + ], + ) + n_prev_tokens[index] = len( + content["meta_info"]["output_token_logprobs"] + ) + + # Generate delta + delta = text[len(stream_buffer) :] + stream_buffers[index] = stream_buffer + delta + finish_reason = content["meta_info"]["finish_reason"] + + choice_data = CompletionResponseStreamChoice( + index=index, + text=delta, + logprobs=logprobs, + finish_reason=finish_reason["type"] if finish_reason else None, + matched_stop=( + finish_reason["matched"] + if finish_reason and "matched" in finish_reason + else None + ), + ) + chunk = CompletionStreamResponse( + id=content["meta_info"]["id"], + created=created, + object="text_completion", + choices=[choice_data], + model=request.model, + ) + + yield f"data: {chunk.model_dump_json()}\n\n" + + if request.return_hidden_states and hidden_states: + for index, choice_hidden_states in hidden_states.items(): + if choice_hidden_states: + last_token_hidden_states = ( + choice_hidden_states[-1] + if len(choice_hidden_states) > 1 + else [] + ) + hidden_states_chunk = CompletionStreamResponse( + id=content["meta_info"]["id"], + created=created, + object="text_completion", + choices=[ + CompletionResponseStreamChoice( + index=index, + text="", + hidden_states=last_token_hidden_states, + finish_reason=None, + ) + ], + model=request.model, + ) + yield f"data: {hidden_states_chunk.model_dump_json()}\n\n" + + # Handle final usage chunk + if request.stream_options and request.stream_options.include_usage: + usage = UsageProcessor.calculate_streaming_usage( + prompt_tokens, + completion_tokens, + cached_tokens, + n_choices=request.n, + enable_cache_report=self.tokenizer_manager.server_args.enable_cache_report, + ) + final_usage_chunk = CompletionStreamResponse( + id=content["meta_info"]["id"], + created=created, + choices=[], + model=request.model, + usage=usage, + ) + final_usage_data = final_usage_chunk.model_dump_json(exclude_none=True) + yield f"data: {final_usage_data}\n\n" + + except Exception as e: + error = self.create_streaming_error_response(str(e)) + yield f"data: {error}\n\n" + + yield "data: [DONE]\n\n" + + async def _handle_non_streaming_request( + self, + adapted_request: GenerateReqInput, + request: CompletionRequest, + raw_request: Request, + ) -> Union[CompletionResponse, ErrorResponse, ORJSONResponse]: + """Handle non-streaming completion request""" + try: + generator = self.tokenizer_manager.generate_request( + adapted_request, raw_request + ) + ret = await generator.__anext__() + except ValueError as e: + return self.create_error_response(str(e)) + + if not isinstance(ret, list): + ret = [ret] + + response = self._build_completion_response( + request, + ret, + int(time.time()), + ) + + return response + + def _build_completion_response( + self, + request: CompletionRequest, + ret: List[Dict[str, Any]], + created: int, + ) -> CompletionResponse: + """Build completion response from generation results""" + choices = [] + echo = False + + # Prepare echo prompts if needed + echo_prompts = [] + if request.echo: + echo_prompts = self._prepare_echo_prompts(request) + echo = True + + for idx, ret_item in enumerate(ret): + text = ret_item["text"] + + # Handle echo + if echo: + prompt_index = idx // request.n + text = echo_prompts[prompt_index] + text + + # Handle logprobs + logprobs = None + if request.logprobs is not None: + if echo: + input_token_logprobs = ret_item["meta_info"]["input_token_logprobs"] + input_top_logprobs = ret_item["meta_info"]["input_top_logprobs"] + else: + input_token_logprobs = None + input_top_logprobs = None + + logprobs = to_openai_style_logprobs( + input_token_logprobs=input_token_logprobs, + input_top_logprobs=input_top_logprobs, + output_token_logprobs=ret_item["meta_info"][ + "output_token_logprobs" + ], + output_top_logprobs=ret_item["meta_info"]["output_top_logprobs"], + ) + + # Handle hidden states + hidden_states = process_hidden_states_from_ret(ret_item, request) + + finish_reason = ret_item["meta_info"]["finish_reason"] + + choice_data = CompletionResponseChoice( + index=idx, + text=text, + logprobs=logprobs, + finish_reason=finish_reason["type"] if finish_reason else None, + matched_stop=( + finish_reason["matched"] + if finish_reason and "matched" in finish_reason + else None + ), + hidden_states=hidden_states, + ) + choices.append(choice_data) + + # Calculate usage + cache_report = self.tokenizer_manager.server_args.enable_cache_report + usage = UsageProcessor.calculate_response_usage( + ret, n_choices=request.n, enable_cache_report=cache_report + ) + + return CompletionResponse( + id=ret[0]["meta_info"]["id"], + model=request.model, + created=created, + choices=choices, + usage=usage, + metadata={"weight_version": ret[0]["meta_info"]["weight_version"]}, + ) + + def _get_echo_text(self, request: CompletionRequest, index: int) -> str: + """Get echo text for streaming response""" + if isinstance(request.prompt, str): + # for the case of single str prompts + return request.prompt + elif isinstance(request.prompt, list): + if isinstance(request.prompt[0], str): + # for the case of multiple str prompts + return request.prompt[index // request.n] + elif isinstance(request.prompt[0], int): + # for the case of single token ids prompt + return self.tokenizer_manager.tokenizer.decode( + request.prompt, skip_special_tokens=True + ) + elif isinstance(request.prompt[0], list) and isinstance( + request.prompt[0][0], int + ): + # for the case of multiple token ids prompts + return self.tokenizer_manager.tokenizer.decode( + request.prompt[index // request.n], + skip_special_tokens=True, + ) + return "" + + def _prepare_echo_prompts(self, request: CompletionRequest) -> List[str]: + """Prepare echo prompts for non-streaming response""" + # TODO: handle the case prompt is token ids + if isinstance(request.prompt, list) and isinstance(request.prompt[0], str): + # for the case of multiple str prompts + return request.prompt + elif isinstance(request.prompt, list) and isinstance(request.prompt[0], list): + # for the case of multiple token ids prompts + return [ + self.tokenizer_manager.tokenizer.decode( + prompt, skip_special_tokens=True + ) + for prompt in request.prompt + ] + elif isinstance(request.prompt, list) and isinstance(request.prompt[0], int): + # for the case of single token ids prompt + return [ + self.tokenizer_manager.tokenizer.decode( + request.prompt, skip_special_tokens=True + ) + ] + else: + # for the case of single str prompt + return [request.prompt] diff --git a/python/sglang/srt/entrypoints/openai/serving_embedding.py b/python/sglang/srt/entrypoints/openai/serving_embedding.py new file mode 100644 index 000000000..63c4fc34a --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_embedding.py @@ -0,0 +1,174 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from fastapi import Request +from fastapi.responses import ORJSONResponse + +from sglang.srt.entrypoints.openai.protocol import ( + EmbeddingObject, + EmbeddingRequest, + EmbeddingResponse, + ErrorResponse, + MultimodalEmbeddingInput, + UsageInfo, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase +from sglang.srt.managers.io_struct import EmbeddingReqInput +from sglang.srt.parser.conversation import generate_embedding_convs + +if TYPE_CHECKING: + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.managers.tokenizer_manager import TokenizerManager + + +class OpenAIServingEmbedding(OpenAIServingBase): + """Handler for v1/embeddings requests""" + + def __init__( + self, + tokenizer_manager: TokenizerManager, + template_manager: TemplateManager, + ): + super().__init__(tokenizer_manager) + self.template_manager = template_manager + + def _request_id_prefix(self) -> str: + return "embd-" + + def _validate_request(self, request: EmbeddingRequest) -> Optional[str]: + """Validate that the input is not empty or whitespace only.""" + if not (input := request.input): + return "Input cannot be empty" + + # Handle single string + if isinstance(input, str): + if not input.strip(): + return "Input cannot be empty or whitespace only" + return None + + # Handle list inputs + if isinstance(input, list): + if len(input) == 0: + return "Input cannot be empty" + + # Check first element to determine type + first_item = input[0] + + if isinstance(first_item, str): + # List of strings + for i, item in enumerate(input): + if not isinstance(item, str): + return f"All items in input list must be strings" + if not item.strip(): + return f"Input at index {i} cannot be empty or whitespace only" + elif isinstance(first_item, int): + # List of integers (token IDs) + for i, item in enumerate(input): + if not isinstance(item, int): + return f"All items in input list must be integers" + if item < 0: + return f"Token ID at index {i} must be non-negative" + return None + + def _convert_to_internal_request( + self, + request: EmbeddingRequest, + ) -> tuple[EmbeddingReqInput, EmbeddingRequest]: + """Convert OpenAI embedding request to internal format""" + prompt = request.input + + if isinstance(prompt, str): + # Single string input + prompt_kwargs = {"text": prompt} + elif isinstance(prompt, list): + if len(prompt) > 0 and isinstance(prompt[0], str): + prompt_kwargs = {"text": prompt} + elif len(prompt) > 0 and isinstance(prompt[0], MultimodalEmbeddingInput): + # Handle multimodal embedding inputs + texts = [] + images = [] + for item in prompt: + # Use padding for text if None - this could be improved + texts.append(item.text if item.text is not None else "padding") + images.append(item.image if item.image is not None else None) + + generate_prompts = [] + # Check if we have a chat template for multimodal embeddings + if self.template_manager.chat_template_name is not None: + convs = generate_embedding_convs( + texts, images, self.template_manager.chat_template_name + ) + for conv in convs: + generate_prompts.append(conv.get_prompt()) + else: + generate_prompts = texts + + if len(generate_prompts) == 1: + prompt_kwargs = { + "text": generate_prompts[0], + "image_data": images[0], + } + else: + prompt_kwargs = { + "text": generate_prompts, + "image_data": images, + } + else: + # List of integers (token IDs) or empty list + prompt_kwargs = {"input_ids": prompt} + else: + # Other types (should not happen but handle gracefully) + prompt_kwargs = {"input_ids": prompt} + + adapted_request = EmbeddingReqInput( + **prompt_kwargs, + rid=request.rid, + ) + + return adapted_request, request + + async def _handle_non_streaming_request( + self, + adapted_request: EmbeddingReqInput, + request: EmbeddingRequest, + raw_request: Request, + ) -> Union[EmbeddingResponse, ErrorResponse, ORJSONResponse]: + """Handle the embedding request""" + try: + ret = await self.tokenizer_manager.generate_request( + adapted_request, raw_request + ).__anext__() + except ValueError as e: + return self.create_error_response(str(e)) + + if not isinstance(ret, list): + ret = [ret] + + response = self._build_embedding_response(ret) + return response + + def _build_embedding_response(self, ret: List[Dict[str, Any]]) -> EmbeddingResponse: + """Build the embedding response""" + embedding_objects = [] + prompt_tokens = 0 + + for idx, ret_item in enumerate(ret): + embedding_objects.append( + EmbeddingObject( + embedding=ret_item["embedding"], + index=idx, + ) + ) + # Handle missing prompt_tokens gracefully + meta_info = ret_item.get("meta_info", {}) + prompt_tokens += meta_info.get("prompt_tokens", 0) + + return EmbeddingResponse( + data=embedding_objects, + model=self.tokenizer_manager.model_path, + usage=UsageInfo( + prompt_tokens=prompt_tokens, + total_tokens=prompt_tokens, + ), + ) diff --git a/python/sglang/srt/entrypoints/openai/serving_rerank.py b/python/sglang/srt/entrypoints/openai/serving_rerank.py new file mode 100644 index 000000000..b053c55b3 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_rerank.py @@ -0,0 +1,102 @@ +import logging +from typing import Any, Dict, List, Optional, Union + +from fastapi import Request +from fastapi.responses import ORJSONResponse + +from sglang.srt.entrypoints.openai.protocol import ( + ErrorResponse, + RerankResponse, + V1RerankReqInput, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase +from sglang.srt.managers.io_struct import EmbeddingReqInput + +logger = logging.getLogger(__name__) + + +class OpenAIServingRerank(OpenAIServingBase): + """Handler for /v1/rerank requests""" + + # NOTE: /v1/rerank is not an official OpenAI endpoint. This module may be moved + # to another module in the future. + + def _request_id_prefix(self) -> str: + return "rerank-" + + def _validate_request(self, request: V1RerankReqInput) -> Optional[str]: + """Validate rerank request format and content""" + if not request.query: + return "Query cannot be empty" + + if isinstance(request.query, str): + if not request.query.strip(): + return "Query cannot be empty or whitespace only" + + if not request.documents: + return "Documents cannot be empty" + + for doc in request.documents: + if not doc: + return "Each document must be a non-empty string" + if isinstance(doc, str) and not doc.strip(): + return "Each document cannot be empty or whitespace only" + + return None + + def _convert_to_internal_request( + self, request: V1RerankReqInput + ) -> tuple[EmbeddingReqInput, V1RerankReqInput]: + """Convert OpenAI rerank request to internal embedding format""" + # Create pairs of [query, document] for each document + pairs = [] + for doc in request.documents: + pairs.append([request.query, doc]) + + adapted_request = EmbeddingReqInput( + text=pairs, + is_cross_encoder_request=True, + ) + + return adapted_request, request + + async def _handle_non_streaming_request( + self, + adapted_request: EmbeddingReqInput, + request: V1RerankReqInput, + raw_request: Request, + ) -> Union[List[RerankResponse], ErrorResponse, ORJSONResponse]: + """Handle the rerank request""" + try: + ret = await self.tokenizer_manager.generate_request( + adapted_request, raw_request + ).__anext__() + + except ValueError as e: + return self.create_error_response(str(e)) + + if not isinstance(ret, list): + ret = [ret] + + responses = self._build_rerank_response(ret, request) + return responses + + def _build_rerank_response( + self, ret: List[Dict[str, Any]], request: V1RerankReqInput + ) -> List[RerankResponse]: + """Build the rerank response from generation results""" + responses = [] + for idx, ret_item in enumerate(ret): + responses.append( + RerankResponse( + score=ret_item["embedding"], + document=request.documents[idx], + index=idx, + meta_info=ret_item["meta_info"], + ) + ) + + # Sort by score in descending order (highest relevance first) + responses.sort(key=lambda x: x.score, reverse=True) + + return responses diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py new file mode 100644 index 000000000..3f7619678 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -0,0 +1,1276 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from vLLM's OpenAIServingResponses +"""Handler for /v1/responses requests""" +from __future__ import annotations + +import asyncio +import copy +import json +import logging +import time +from contextlib import AsyncExitStack +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, AsyncGenerator, AsyncIterator, Optional, Union + +import jinja2 +import openai.types.responses as openai_responses_types +from fastapi import Request +from fastapi.responses import ORJSONResponse +from openai.types.responses import ( + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, +) +from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall +from openai.types.responses.response_reasoning_item import ( + Content as ResponseReasoningTextContent, +) +from openai_harmony import Message as OpenAIMessage + +from sglang.srt.entrypoints.context import ( + ConversationContext, + HarmonyContext, + SimpleContext, + StreamingHarmonyContext, +) +from sglang.srt.entrypoints.harmony_utils import ( + get_developer_message, + get_stop_tokens_for_assistant_actions, + get_system_message, + get_user_message, + parse_output_message, + parse_remaining_state, + parse_response_input, + render_for_completion, +) +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionMessageParam, + ChatCompletionRequest, + PromptTokenUsageInfo, + RequestResponseMetadata, + ResponsesRequest, + ResponsesResponse, + UsageInfo, +) +from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat +from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.parser.reasoning_parser import ReasoningParser +from sglang.srt.utils import random_uuid + +if TYPE_CHECKING: + from sglang.srt.managers.template_manager import TemplateManager + from sglang.srt.managers.tokenizer_manager import TokenizerManager + +logger = logging.getLogger(__name__) + + +class OpenAIServingResponses(OpenAIServingChat): + """Handler for /v1/responses requests""" + + def __init__( + self, + tokenizer_manager: TokenizerManager, + template_manager: TemplateManager, + *, + enable_prompt_tokens_details: bool = False, + enable_force_include_usage: bool = False, + tool_server: Optional[ToolServer] = None, + ) -> None: + super().__init__(tokenizer_manager, template_manager) + + # template_manager is already set by parent class + self.reasoning_parser = self.tokenizer_manager.server_args.reasoning_parser + self.enable_prompt_tokens_details = enable_prompt_tokens_details + self.enable_force_include_usage = enable_force_include_usage + + # Get default sampling params from model config if available + self.default_sampling_params = {} + + self.supports_browsing = ( + tool_server.has_tool("browser") if tool_server else False + ) + self.supports_code_interpreter = ( + tool_server.has_tool("python") if tool_server else False + ) + self.tool_server = tool_server + # Get from model config + self.use_harmony = ( + self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss" + ) + + if self.use_harmony: + # OpenAI models have two EOS-like tokens: <|return|> and <|call|>. + # We need to add them to the stop token ids. + if "stop_token_ids" not in self.default_sampling_params: + self.default_sampling_params["stop_token_ids"] = [] + self.default_sampling_params["stop_token_ids"].extend( + get_stop_tokens_for_assistant_actions() + ) + + # Response storage for background and retrieval operations + # Note: In production, this should use a proper storage backend (Redis, database) + # with TTL/expiration to prevent memory leaks + self.response_store: dict[str, ResponsesResponse] = {} + self.response_store_lock = asyncio.Lock() + + # Message storage for conversation continuity + # Note: In production, this should use a proper storage backend (Redis, database) + # with TTL/expiration to prevent memory leaks + self.msg_store: dict[ + str, Union[list[ChatCompletionMessageParam], list["OpenAIMessage"]] + ] = {} + + self.background_tasks: dict[str, asyncio.Task] = {} + + def _request_id_prefix(self) -> str: + return "resp_" + + async def create_responses( + self, + request: ResponsesRequest, + raw_request: Optional[Request] = None, + ) -> Union[AsyncGenerator[str, None], ResponsesResponse, ORJSONResponse]: + # Validate model + if not self.tokenizer_manager: + return self.create_error_response("Model not loaded") + + # FIXME: If the engine is dead, raise an error + # This is required for the streaming case + + # Handle the previous response ID + prev_response_id = request.previous_response_id + if prev_response_id is not None: + if not prev_response_id.startswith("resp_"): + return self._make_invalid_id_error(prev_response_id) + async with self.response_store_lock: + prev_response = self.response_store.get(prev_response_id) + if prev_response is None: + return self._make_not_found_error(prev_response_id) + else: + prev_response = None + + try: + model_name = request.model + tokenizer = self.tokenizer_manager.tokenizer + + if self.use_harmony: + messages, request_prompts, engine_prompts = ( + self._make_request_with_harmony(request, prev_response) + ) + else: + messages, request_prompts, engine_prompts = await self._make_request( + request, prev_response, tokenizer + ) + + except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e: + logger.exception("Error in preprocessing prompt inputs") + return self.create_error_response(f"{e} {e.__cause__}") + + request_metadata = RequestResponseMetadata(request_id=request.request_id) + if raw_request: + raw_request.state.request_metadata = request_metadata + + if ( + self.tool_server is not None + and isinstance(self.tool_server, MCPToolServer) + and (request.background or request.stream) + and request.tools + and any( + tool.type in ["web_search_preview", "code_interpreter"] + for tool in request.tools + ) + ): + return self.create_error_response( + "MCP tool server is not supported in background mode and " + "streaming mode" + ) + + # Schedule the request and get the result generator + generators: list[AsyncGenerator[Any, None]] = [] + tool_list = [] + if self.use_harmony: + if self.supports_browsing: + tool_list.append("browser") + if self.supports_code_interpreter: + tool_list.append("python") + async with AsyncExitStack() as exit_stack: + try: + if self.tool_server is not None: + tool_session_ctxs: dict[str, Any] = { + tool_name: exit_stack.enter_async_context( + self.tool_server.get_tool_session(tool_name) + ) + for tool_name in tool_list + } + tool_sessions = {} + for tool_name in tool_list: + tool_sessions[tool_name] = await tool_session_ctxs[tool_name] + else: + assert len(tool_list) == 0 + tool_sessions = {} + for i, engine_prompt in enumerate(engine_prompts): + # Calculate default max tokens from context length minus prompt length + if hasattr(engine_prompt, "__len__"): + prompt_length = len(engine_prompt) + elif isinstance(engine_prompt, list): + prompt_length = len(engine_prompt) + else: + prompt_length = 0 + + context_len = ( + self.tokenizer_manager.model_config.context_len + if hasattr(self.tokenizer_manager.model_config, "context_len") + else 4096 + ) + default_max_tokens = max( + context_len - prompt_length, 512 + ) # Ensure minimum 512 tokens + sampling_params = request.to_sampling_params( + default_max_tokens, self.default_sampling_params + ) + + context: ConversationContext + if self.use_harmony: + if request.stream: + context = StreamingHarmonyContext(messages, tool_sessions) + else: + context = HarmonyContext(messages, tool_sessions) + else: + context = SimpleContext() + + # Create GenerateReqInput for SGLang + adapted_request = GenerateReqInput( + input_ids=engine_prompt, + sampling_params=sampling_params, + stream=request.stream, + rid=request.request_id, + background=request.background, + ) + + generator = self._generate_with_builtin_tools( + request.request_id, + request_prompts[i], + adapted_request, + sampling_params, + context, + raw_request=raw_request, + priority=request.priority, + ) + generators.append(generator) + except ValueError as e: + return self.create_error_response(str(e)) + + assert len(generators) == 1 + (result_generator,) = generators + + # Store the input messages + if request.store: + self.msg_store[request.request_id] = messages + + if request.background: + created_time = int(time.time()) + response = ResponsesResponse.from_request( + request, + sampling_params, + model_name=model_name, + created_time=created_time, + output=[], + status="queued", + usage=None, + ) + async with self.response_store_lock: + self.response_store[response.id] = response + + # Run the request in the background + task = asyncio.create_task( + self._run_background_request( + request, + sampling_params, + result_generator, + context, + model_name, + tokenizer, + request_metadata, + created_time, + ), + name=f"create_{response.id}", + ) + + # For cleanup + self.background_tasks[response.id] = task + task.add_done_callback( + lambda _: self.background_tasks.pop(response.id, None) + ) + return response + + if request.stream: + return self.responses_stream_generator( + request, + sampling_params, + result_generator, + context, + model_name, + tokenizer, + request_metadata, + ) + try: + result: Union[ORJSONResponse, ResponsesResponse] = ( + await self.responses_full_generator( + request, + sampling_params, + result_generator, + context, + model_name, + tokenizer, + request_metadata, + ) + ) + return result + except Exception as e: + return self.create_error_response(str(e)) + return self.create_error_response("Unknown error") + + async def _make_request( + self, + request: ResponsesRequest, + prev_response: Optional[ResponsesResponse], + tokenizer: Any, + ): + # Construct the input messages + messages = self._construct_input_messages(request, prev_response) + + # Follow SGLang's pattern: create a ChatCompletionRequest and process messages + try: + # Convert ResponsesRequest to ChatCompletionRequest for processing + chat_request = ChatCompletionRequest( + model=request.model, + messages=messages, + stream=request.stream, + ) + + # Follow SGLang's _process_messages pattern + is_multimodal = self.tokenizer_manager.model_config.is_multimodal + processed_messages = self._process_messages(chat_request, is_multimodal) + + # Extract the results + if is_multimodal: + request_prompts = [processed_messages.prompt] + engine_prompts = [processed_messages.prompt] + else: + request_prompts = [processed_messages.prompt_ids] + engine_prompts = [processed_messages.prompt_ids] + + except Exception as e: + logger.warning(f"Chat processing failed, using fallback: {e}") + # Fallback to simple encoding + prompt_text = "" + for msg in messages: + role = msg.get("role", "user") + content = msg.get("content", "") + prompt_text += f"{role}: {content}\n" + prompt_ids = tokenizer.encode(prompt_text) + request_prompts = [prompt_ids] + engine_prompts = [prompt_ids] + + return messages, request_prompts, engine_prompts + + def _make_request_with_harmony( + self, + request: ResponsesRequest, + prev_response: Optional[ResponsesResponse], + ): + if request.tool_choice != "auto": + raise NotImplementedError( + "Only 'auto' tool_choice is supported in " "response API" + ) + messages = self._construct_input_messages_with_harmony(request, prev_response) + prompt_token_ids = render_for_completion(messages) + engine_prompt = prompt_token_ids + return messages, [prompt_token_ids], [engine_prompt] + + async def responses_full_generator( + self, + request: ResponsesRequest, + sampling_params: Any, + result_generator: AsyncIterator[Any], + context: ConversationContext, + model_name: str, + tokenizer: Any, + request_metadata: RequestResponseMetadata, + created_time: Optional[int] = None, + ) -> Union[ResponsesResponse, ORJSONResponse]: + if created_time is None: + created_time = int(time.time()) + + try: + async for _ in result_generator: + pass + except asyncio.CancelledError: + return self.create_error_response("Client disconnected") + except ValueError as e: + return self.create_error_response(str(e)) + + if self.use_harmony: + assert isinstance(context, HarmonyContext) + output = self._make_response_output_items_with_harmony(context) + # TODO: these are all 0 for now! + num_prompt_tokens = context.num_prompt_tokens + num_generated_tokens = context.num_output_tokens + num_cached_tokens = context.num_cached_tokens + num_reasoning_tokens = context.num_reasoning_tokens + else: + assert isinstance(context, SimpleContext) + final_res = context.last_output + assert final_res is not None + + output = self._make_response_output_items( + request, final_res["text"], tokenizer + ) + + # Calculate usage from actual output + if hasattr(final_res, "meta_info"): + num_prompt_tokens = final_res.meta_info.get("prompt_tokens", 0) + num_generated_tokens = final_res.meta_info.get("completion_tokens", 0) + num_cached_tokens = final_res.meta_info.get("cached_tokens", 0) + elif hasattr(final_res, "prompt_token_ids") and hasattr( + final_res, "outputs" + ): + # Fallback calculation if meta_info not available + num_prompt_tokens = ( + len(final_res.prompt_token_ids) if final_res.prompt_token_ids else 0 + ) + num_generated_tokens = ( + len(final_res.outputs[0].token_ids) + if final_res.outputs and final_res.outputs[0].token_ids + else 0 + ) + num_cached_tokens = getattr(final_res, "num_cached_tokens", 0) + num_reasoning_tokens = 0 + else: + # Final fallback + num_prompt_tokens = 0 + num_generated_tokens = 0 + num_cached_tokens = 0 + num_reasoning_tokens = 0 + + usage = UsageInfo( + prompt_tokens=num_prompt_tokens, + completion_tokens=num_generated_tokens, + total_tokens=num_prompt_tokens + num_generated_tokens, + reasoning_tokens=num_reasoning_tokens, + ) + if self.enable_prompt_tokens_details and num_cached_tokens: + usage.prompt_tokens_details = PromptTokenUsageInfo( + cached_tokens=num_cached_tokens + ) + request_metadata.final_usage_info = usage + + response = ResponsesResponse.from_request( + request, + sampling_params, + model_name=model_name, + created_time=created_time, + output=output, + status="completed", + usage=usage, + ) + + if request.store: + async with self.response_store_lock: + stored_response = self.response_store.get(response.id) + # If the response is already cancelled, don't update it + if stored_response is None or stored_response.status != "cancelled": + self.response_store[response.id] = response + + return response + + def _make_response_output_items( + self, + request: ResponsesRequest, + final_output: Any, + tokenizer: Any, + ): + # Handle reasoning parsing if enabled + if self.reasoning_parser: + # Use standard reasoning parser (openai maps to T4Detector internally) + reasoning_parser = ReasoningParser( + model_type=self.reasoning_parser, stream_reasoning=False + ) + reasoning_content, content = reasoning_parser.parse_non_stream(final_output) + else: + reasoning_content = None + content = final_output + + output_items = [] + if reasoning_content: + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + type="reasoning", + summary=[], + content=[ + ResponseReasoningTextContent( + type="reasoning_text", text=reasoning_content + ), + ], + status=None, + ) + output_items.append(reasoning_item) + if content: + output_text = ResponseOutputText( + text=content, + annotations=[], # TODO + type="output_text", + logprobs=None, # TODO + ) + message = ResponseOutputMessage( + id=f"msg_{random_uuid()}", + content=[output_text], + role="assistant", + status="completed", + type="message", + ) + output_items.append(message) + return output_items + + def _make_response_output_items_with_harmony( + self, + context: HarmonyContext, + ): + output_items = [] + num_init_messages = context.num_init_messages + for msg in context.messages[num_init_messages:]: + output_items.extend(parse_output_message(msg)) + # Handle the generation stopped in the middle (if any). + last_items = parse_remaining_state(context.parser) + if last_items: + output_items.extend(last_items) + return output_items + + def _construct_input_messages( + self, + request: ResponsesRequest, + prev_response: Optional[ResponsesResponse] = None, + ) -> list[ChatCompletionMessageParam]: + messages: list[ChatCompletionMessageParam] = [] + if request.instructions: + messages.append( + { + "role": "system", + "content": request.instructions, + } + ) + + # Prepend the conversation history + if prev_response is not None: + # Add the previous messages + prev_msg = self.msg_store[prev_response.id] + messages.extend(prev_msg) + + # Add the previous output + for output_item in prev_response.output: + # NOTE: We skip the reasoning output of the previous response + if isinstance(output_item, ResponseReasoningItem): + continue + for content in output_item.content: + messages.append( + { + "role": "system", + "content": request.instructions, + } + ) + + # Append the new input + # Responses API supports simple text inputs without chat format + if isinstance(request.input, str): + messages.append({"role": "user", "content": request.input}) + else: + messages.extend(request.input) # type: ignore + return messages + + def _construct_input_messages_with_harmony( + self, + request: ResponsesRequest, + prev_response: Optional[ResponsesResponse], + ) -> list["OpenAIMessage"]: + messages: list["OpenAIMessage"] = [] + if prev_response is None: + # New conversation. + reasoning_effort = request.reasoning.effort if request.reasoning else None + tool_types = [tool.type for tool in request.tools] + enable_browser = ( + "web_search_preview" in tool_types and self.tool_server is not None + ) + enable_code_interpreter = ( + "code_interpreter" in tool_types and self.tool_server is not None + ) + sys_msg = get_system_message( + reasoning_effort=reasoning_effort, + browser_description=( + self.tool_server.get_tool_description("browser") + if self.tool_server and enable_browser + else None + ), + python_description=( + self.tool_server.get_tool_description("python") + if self.tool_server and enable_code_interpreter + else None + ), + ) + messages.append(sys_msg) + dev_msg = get_developer_message(request.instructions, request.tools) + messages.append(dev_msg) + else: + # Continue the previous conversation. + # FIXME: Currently, request params like reasoning and + # instructions are ignored. + prev_msgs = self.msg_store[prev_response.id] + # Remove the previous chain-of-thoughts if there is a new "final" + # message. + if ( + len(prev_msgs) > 0 + and hasattr(prev_msgs[-1], "channel") + and prev_msgs[-1].channel == "final" + ): # type: ignore[union-attr] + prev_final_msg_idx = -1 + for i in range(len(prev_msgs) - 2, -1, -1): + if ( + hasattr(prev_msgs[i], "channel") + and prev_msgs[i].channel == "final" + ): # type: ignore[union-attr] + prev_final_msg_idx = i + break + recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :] + del prev_msgs[prev_final_msg_idx + 1 :] + for msg in recent_turn_msgs: + if ( + hasattr(msg, "channel") and msg.channel != "analysis" + ): # type: ignore[union-attr] + prev_msgs.append(msg) + messages.extend(prev_msgs) + # Append the new input. + # Responses API supports simple text inputs without chat format. + if isinstance(request.input, str): + messages.append(get_user_message(request.input)) + else: + if prev_response is not None: + prev_outputs = copy(prev_response.output) + else: + prev_outputs = [] + for response_msg in request.input: + messages.append(parse_response_input(response_msg, prev_outputs)) + if isinstance(response_msg, ResponseFunctionToolCall): + prev_outputs.append(response_msg) + return messages + + async def _run_background_request( + self, + request: ResponsesRequest, + sampling_params: Any, + result_generator: AsyncIterator[Any], + context: ConversationContext, + model_name: str, + tokenizer: Any, + request_metadata: RequestResponseMetadata, + created_time: Optional[int] = None, + *args, + **kwargs, + ): + try: + # Update the status to "in_progress" + async with self.response_store_lock: + stored_response = self.response_store.get(request.request_id) + assert stored_response is not None + stored_response.status = "in_progress" + + response = await self.responses_full_generator( + request, + sampling_params, + result_generator, + context, + model_name, + tokenizer, + request_metadata, + created_time, + *args, + **kwargs, + ) + except Exception as e: + logger.exception("Background request failed for %s", request.request_id) + response = self.create_error_response(str(e)) + + if isinstance(response, ORJSONResponse): + # If the request has failed, update the status to "failed" + response_id = request.request_id + async with self.response_store_lock: + stored_response = self.response_store.get(response_id) + assert stored_response is not None + if stored_response.status not in ("completed", "cancelled"): + stored_response.status = "failed" + + async def retrieve_responses( + self, + response_id: str, + ) -> Union[ResponsesResponse, ORJSONResponse]: + if not response_id.startswith("resp_"): + return self._make_invalid_id_error(response_id) + + async with self.response_store_lock: + response = self.response_store.get(response_id) + + if response is None: + return self._make_not_found_error(response_id) + return response + + async def cancel_responses( + self, + response_id: str, + ) -> Union[ResponsesResponse, ORJSONResponse]: + if not response_id.startswith("resp_"): + return self._make_invalid_id_error(response_id) + + async with self.response_store_lock: + response = self.response_store.get(response_id) + if response is None: + return self._make_not_found_error(response_id) + + prev_status = response.status + if prev_status not in ("queued", "in_progress"): + return self.create_error_response( + err_type="invalid_request_error", + message="Cannot cancel a synchronous response.", + ) + + # Update the status to "cancelled" + response.status = "cancelled" + + # Abort the request + if task := self.background_tasks.get(response_id): + task.cancel() + try: + await task + except asyncio.CancelledError: + logger.exception("Background task for %s was cancelled", response_id) + return response + + def _make_invalid_id_error(self, response_id: str): + return self.create_error_response( + message=( + f"Invalid 'response_id': '{response_id}'. " + "Expected an ID that begins with 'resp'." + ), + err_type="invalid_request_error", + param="response_id", + ) + + def _make_not_found_error(self, response_id: str): + return self.create_error_response( + message=f"Response with id '{response_id}' not found.", + err_type="invalid_request_error", + status_code=HTTPStatus.NOT_FOUND, + param="response_id", + ) + + async def responses_stream_generator( + self, + request: ResponsesRequest, + sampling_params: Any, + result_generator: AsyncIterator[StreamingHarmonyContext], + context: StreamingHarmonyContext, + model_name: str, + tokenizer: Any, + request_metadata: RequestResponseMetadata, + created_time: Optional[int] = None, + ) -> AsyncGenerator[str, None]: + # TODO: + # 1. Handle disconnect + + created_time = created_time or int(time.time()) + + sequence_number = 0 + + def _send_event(event): + nonlocal sequence_number + # Set sequence_number if the event has this attribute + if hasattr(event, "sequence_number"): + event.sequence_number = sequence_number + sequence_number += 1 + # Get event type from the event's type field if it exists + event_type = getattr(event, "type", "unknown") + return ( + f"event: {event_type}\n" + f"data: {event.model_dump_json(indent=None)}\n\n" + ) + + current_content_index = 0 + current_output_index = 0 + current_item_id = f"item_{random_uuid()}" + sent_output_item_added = False + + initial_response = ResponsesResponse.from_request( + request, + sampling_params, + model_name=model_name, + created_time=created_time, + output=[], + status="in_progress", + usage=None, + ).model_dump() + yield _send_event( + openai_responses_types.ResponseCreatedEvent( + type="response.created", + sequence_number=-1, + response=initial_response, + ) + ) + yield _send_event( + openai_responses_types.ResponseInProgressEvent( + type="response.in_progress", + sequence_number=-1, + response=initial_response, + ) + ) + + async for ctx in result_generator: + + if ctx.is_expecting_start(): + current_output_index += 1 + sent_output_item_added = False + + if len(ctx.parser.messages) > 0: + previous_item = ctx.parser.messages[-1] + if previous_item.recipient is not None: + # Deal with tool call here + pass + elif previous_item.channel == "analysis": + reasoning_item = ResponseReasoningItem( + id=f"rs_{random_uuid()}", + type="reasoning", + summary=[], + content=[ + ResponseReasoningTextContent( + text=previous_item.content[0].text, + type="reasoning_text", + ), + ], + status="completed", + ) + yield _send_event( + openai_responses_types.ResponseReasoningTextDoneEvent( + type="response.reasoning_text.done", + item_id=current_item_id, + sequence_number=-1, + output_index=current_output_index, + content_index=current_content_index, + text=previous_item.content[0].text, + ) + ) + yield _send_event( + openai_responses_types.ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=current_output_index, + item=reasoning_item, + ) + ) + elif previous_item.channel == "final": + text_content = openai_responses_types.ResponseOutputText( + type="output_text", + text=previous_item.content[0].text, + annotations=[], + ) + yield _send_event( + openai_responses_types.ResponseTextDoneEvent( + type="response.output_text.done", + sequence_number=-1, + output_index=current_output_index, + content_index=current_content_index, + text=previous_item.content[0].text, + logprobs=[], + item_id=current_item_id, + ) + ) + yield _send_event( + openai_responses_types.ResponseContentPartDoneEvent( + type="response.content_part.done", + sequence_number=-1, + item_id=current_item_id, + output_index=current_output_index, + content_index=current_content_index, + part=text_content, + ) + ) + yield _send_event( + openai_responses_types.ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseOutputMessage( + id=current_item_id, + type="message", + role="assistant", + content=[text_content], + status="completed", + ), + ) + ) + + if ctx.parser.last_content_delta: + if ( + ctx.parser.current_channel == "final" + and ctx.parser.current_recipient is None + ): + if not sent_output_item_added: + sent_output_item_added = True + yield _send_event( + openai_responses_types.ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseOutputMessage( + id=current_item_id, + type="message", + role="assistant", + content=[], + status="in_progress", + ), + ) + ) + yield _send_event( + openai_responses_types.ResponseContentPartAddedEvent( + type="response.content_part.added", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + content_index=current_content_index, + part=openai_responses_types.ResponseOutputText( + type="output_text", + text="", + annotations=[], + logprobs=None, + ), + ) + ) + yield _send_event( + openai_responses_types.ResponseTextDeltaEvent( + type="response.output_text.delta", + sequence_number=-1, + content_index=current_content_index, + output_index=current_output_index, + item_id=current_item_id, + delta=ctx.parser.last_content_delta, + # TODO, use logprobs from ctx.last_request_output + logprobs=[], + ) + ) + elif ( + ctx.parser.current_channel == "analysis" + and ctx.parser.current_recipient is None + ): + if not sent_output_item_added: + sent_output_item_added = True + yield _send_event( + openai_responses_types.ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseReasoningItem( + type="reasoning", + id=current_item_id, + summary=[], + status="in_progress", + ), + ) + ) + yield _send_event( + openai_responses_types.ResponseContentPartAddedEvent( + type="response.content_part.added", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + content_index=current_content_index, + # TODO: migrate this to + # ResponseReasoningTextContent for now + part=openai_responses_types.ResponseOutputText( + type="output_text", + text="", + annotations=[], + logprobs=None, + ), + ) + ) + # TODO: migrate to OpenAI types once updated. + yield _send_event( + openai_responses_types.ResponseReasoningTextDeltaEvent( + type="response.reasoning_text.delta", + item_id=current_item_id, + output_index=current_output_index, + content_index=current_content_index, + delta=ctx.parser.last_content_delta, + sequence_number=-1, + ) + ) + + if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0: + previous_item = ctx.parser.messages[-1] + if ( + self.supports_browsing + and previous_item.recipient is not None + and previous_item.recipient.startswith("browser.") + ): + function_name = previous_item.recipient[len("browser.") :] + action = None + parsed_args = json.loads(previous_item.content[0].text) + if function_name == "search": + action = openai_responses_types.response_function_web_search.ActionSearch( + type="search", + query=parsed_args["query"], + ) + elif function_name == "open": + action = openai_responses_types.response_function_web_search.ActionOpenPage( + type="open_page", + # TODO: translate to url + url=f"cursor:{parsed_args.get('cursor', '')}", + ) + elif function_name == "find": + action = openai_responses_types.response_function_web_search.ActionFind( + type="find", + pattern=parsed_args["pattern"], + # TODO: translate to url + url=f"cursor:{parsed_args.get('cursor', '')}", + ) + else: + raise ValueError(f"Unknown function name: {function_name}") + + yield _send_event( + openai_responses_types.ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.response_function_web_search.ResponseFunctionWebSearch( + # TODO: generate a unique id for web search call + type="web_search_call", + id=current_item_id, + action=action, + status="in_progress", + ), + ) + ) + yield _send_event( + openai_responses_types.ResponseWebSearchCallInProgressEvent( + type="response.web_search_call.in_progress", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + yield _send_event( + openai_responses_types.ResponseWebSearchCallSearchingEvent( + type="response.web_search_call.searching", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + + # enqueue + yield _send_event( + openai_responses_types.ResponseWebSearchCallCompletedEvent( + type="response.web_search_call.completed", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + yield _send_event( + openai_responses_types.ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseFunctionWebSearch( + type="web_search_call", + id=current_item_id, + action=action, + status="completed", + ), + ) + ) + + if ( + self.supports_code_interpreter + and previous_item.recipient is not None + and previous_item.recipient.startswith("python") + ): + yield _send_event( + openai_responses_types.ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseCodeInterpreterToolCallParam( + type="code_interpreter_call", + id=current_item_id, + code="", + container_id="auto", + outputs=[], + status="in_progress", + ), + ) + ) + yield _send_event( + openai_responses_types.ResponseCodeInterpreterCallInProgressEvent( + type="response.code_interpreter_call.in_progress", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + # TODO: do we need to add delta event here? + yield _send_event( + openai_responses_types.ResponseCodeInterpreterCallCodeDoneEvent( + type="response.code_interpreter_call_code.done", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + code=previous_item.content[0].text, + ) + ) + yield _send_event( + openai_responses_types.ResponseCodeInterpreterCallInterpretingEvent( + type="response.code_interpreter_call.interpreting", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + yield _send_event( + openai_responses_types.ResponseCodeInterpreterCallCompletedEvent( + type="response.code_interpreter_call.completed", + sequence_number=-1, + output_index=current_output_index, + item_id=current_item_id, + ) + ) + yield _send_event( + openai_responses_types.ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=current_output_index, + item=openai_responses_types.ResponseCodeInterpreterToolCallParam( + type="code_interpreter_call", + id=current_item_id, + code=previous_item.content[0].text, + container_id="auto", + # TODO: add outputs here + outputs=[], + status="completed", + ), + ) + ) + + async def empty_async_generator(): + if False: + yield + + final_response = await self.responses_full_generator( + request, + sampling_params, + empty_async_generator(), + context, + model_name, + tokenizer, + request_metadata, + created_time=created_time, + ) + # Convert final_response to the format expected by ResponseCompletedEvent + response_dict = final_response.model_dump() + + # Convert UsageInfo to ResponseUsage format + if response_dict.get("usage"): + usage_info = response_dict["usage"] + response_dict["usage"] = { + "input_tokens": usage_info.get("prompt_tokens", 0), + "input_tokens_details": { + "cached_tokens": usage_info.get("cached_tokens", 0) + }, + "output_tokens": usage_info.get("completion_tokens", 0), + "output_tokens_details": { + "reasoning_tokens": usage_info.get("reasoning_tokens", 0) + }, + "total_tokens": usage_info.get("total_tokens", 0), + } + + yield _send_event( + openai_responses_types.ResponseCompletedEvent( + type="response.completed", + sequence_number=-1, + response=response_dict, + ) + ) + + async def _generate_with_builtin_tools( + self, + request_id: str, + request_prompt: Any, + adapted_request: GenerateReqInput, + sampling_params: Any, + context: ConversationContext, + raw_request: Optional[Request] = None, + priority: Optional[int] = None, + **kwargs, + ) -> AsyncGenerator[Any, None]: + """Generate with builtin tool support for harmony-based models.""" + orig_priority = priority or 0 + + while True: + # Generate using SGLang's tokenizer manager + generator = self.tokenizer_manager.generate_request( + adapted_request, raw_request + ) + + async for res in generator: + context.append_output(res) + # NOTE(woosuk): The stop condition is handled by the engine. + yield context + + if not context.need_builtin_tool_call(): + # The model did not ask for a tool call, so we're done. + break + + # Call the tool and update the context with the result. + tool_output = await context.call_tool() + context.append_output(tool_output) + + # Prepare for the next generation turn + # Render the updated conversation for the next completion + prompt_token_ids = context.render_for_completion() + + # Update the adapted request with new prompt + adapted_request = GenerateReqInput( + input_ids=prompt_token_ids, + sampling_params=sampling_params, + stream=adapted_request.stream, + rid=request_id, + return_logprob=adapted_request.return_logprob, + logprob_start_len=adapted_request.logprob_start_len, + top_logprobs_num=adapted_request.top_logprobs_num, + return_text_in_logprobs=adapted_request.return_text_in_logprobs, + return_hidden_states=adapted_request.return_hidden_states, + background=adapted_request.background, + ) + + # Update sampling params with reduced max_tokens + if hasattr(sampling_params, "max_new_tokens") or isinstance( + sampling_params, dict + ): + context_len = getattr( + self.tokenizer_manager.model_config, "context_len", 4096 + ) + remaining_tokens = context_len - len(prompt_token_ids) - 1 + + if isinstance(sampling_params, dict): + sampling_params["max_new_tokens"] = max(remaining_tokens, 1) + else: + sampling_params.max_new_tokens = max(remaining_tokens, 1) + + # Slightly reduce priority for subsequent tool calls + priority = orig_priority - 1 diff --git a/python/sglang/srt/entrypoints/openai/serving_score.py b/python/sglang/srt/entrypoints/openai/serving_score.py new file mode 100644 index 000000000..fc8ce5dca --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_score.py @@ -0,0 +1,61 @@ +import logging +from typing import Union + +from fastapi import Request + +from sglang.srt.entrypoints.openai.protocol import ( + ErrorResponse, + ScoringRequest, + ScoringResponse, +) +from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase + +logger = logging.getLogger(__name__) + + +class OpenAIServingScore(OpenAIServingBase): + """Handler for /v1/score requests""" + + # NOTE: /v1/rerank is not an official OpenAI endpoint. This module may be moved + # to another module in the future. + + def _request_id_prefix(self) -> str: + return "score-" + + def _convert_to_internal_request( + self, + request: ScoringRequest, + ) -> tuple[ScoringRequest, ScoringRequest]: + """Convert OpenAI scoring request to internal format""" + # For scoring, we pass the request directly as the tokenizer_manager + # has a specialized score_request method that doesn't use GenerateReqInput + + return request, request + + async def _handle_non_streaming_request( + self, + adapted_request: ScoringRequest, + request: ScoringRequest, + raw_request: Request, + ) -> Union[ScoringResponse, ErrorResponse]: + """Handle the scoring request""" + try: + # Use tokenizer_manager's score_request method directly + scores = await self.tokenizer_manager.score_request( + query=request.query, + items=request.items, + label_token_ids=request.label_token_ids, + apply_softmax=request.apply_softmax, + item_first=request.item_first, + request=raw_request, + ) + + # Create response with just the scores, without usage info + response = ScoringResponse( + scores=scores, + model=request.model, + ) + return response + + except ValueError as e: + return self.create_error_response(str(e)) diff --git a/python/sglang/srt/entrypoints/openai/tool_server.py b/python/sglang/srt/entrypoints/openai/tool_server.py new file mode 100644 index 000000000..269d9e99e --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/tool_server.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging +from abc import ABC, abstractmethod +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import Any + +try: + from mcp import ClientSession + from mcp.client.sse import sse_client + from mcp.types import ListToolsResult +except ImportError as e: + ClientSession = sse_client = ListToolsResult = e + +from openai_harmony import ToolDescription, ToolNamespaceConfig + +logger = logging.getLogger(__name__) + + +async def list_server_and_tools(server_url: str): + + async with sse_client(url=server_url) as streams, ClientSession( + *streams + ) as session: + initialize_response = await session.initialize() + list_tools_response = await session.list_tools() + return initialize_response, list_tools_response + + +def trim_schema(schema: dict) -> dict: + # Turn JSON Schema from MCP generated into Harmony's variant. + if "title" in schema: + del schema["title"] + if "default" in schema and schema["default"] is None: + del schema["default"] + if "anyOf" in schema: + # Turn "anyOf": [{"type": "type-1"}, {"type": "type-2"}] + # into "type": ["type-1", "type-2"] + # if there's more than 1 types, also remove "null" type as Harmony will + # just ignore it + types = [ + type_dict["type"] + for type_dict in schema["anyOf"] + if type_dict["type"] != "null" + ] + schema["type"] = types + del schema["anyOf"] + if "properties" in schema: + schema["properties"] = { + k: trim_schema(v) for k, v in schema["properties"].items() + } + return schema + + +def post_process_tools_description( + list_tools_result: "ListToolsResult", +) -> "ListToolsResult": + # Adapt the MCP tool result for Harmony + for tool in list_tools_result.tools: + tool.inputSchema = trim_schema(tool.inputSchema) + + # Some tools schema don't need to be part of the prompt (e.g. simple text + # in text out for Python) + list_tools_result.tools = [ + tool + for tool in list_tools_result.tools + if getattr(tool.annotations, "include_in_prompt", True) + ] + + return list_tools_result + + +class ToolServer(ABC): + + @abstractmethod + def has_tool(self, tool_name: str): + pass + + @abstractmethod + def get_tool_description(self, tool_name: str): + pass + + @abstractmethod + def get_tool_session(self, tool_name: str) -> AbstractAsyncContextManager[Any]: ... + + +class MCPToolServer(ToolServer): + + def __init__(self): + self.harmony_tool_descriptions = {} + + async def add_tool_server(self, server_url: str): + tool_urls = server_url.split(",") + self.harmony_tool_descriptions = {} + self.urls: dict[str, str] = {} + for url in tool_urls: + url = f"http://{url}/sse" + initialize_response, list_tools_response = await list_server_and_tools(url) + + list_tools_response = post_process_tools_description(list_tools_response) + + tool_from_mcp = ToolNamespaceConfig( + name=initialize_response.serverInfo.name, + description=initialize_response.instructions, + tools=[ + ToolDescription.new( + name=tool.name, + description=tool.description, + parameters=tool.inputSchema, + ) + for tool in list_tools_response.tools + ], + ) + self.harmony_tool_descriptions[tool_from_mcp.name] = tool_from_mcp + if tool_from_mcp.name not in self.urls: + self.urls[tool_from_mcp.name] = url + else: + logger.warning( + "Tool %s already exists. Ignoring duplicate tool server %s", + tool_from_mcp.name, + url, + ) + + def has_tool(self, tool_name: str): + return tool_name in self.harmony_tool_descriptions + + def get_tool_description(self, tool_name: str): + return self.harmony_tool_descriptions.get(tool_name) + + @asynccontextmanager + async def get_tool_session(self, tool_name: str): + url = self.urls.get(tool_name) + if url: + async with sse_client(url=url) as streams, ClientSession( + *streams + ) as session: + await session.initialize() + yield session + else: + logger.warning("Tool %s not found", tool_name) + + +class DemoToolServer(ToolServer): + + def __init__(self): + from sglang.srt.entrypoints.tool import ( + HarmonyBrowserTool, + HarmonyPythonTool, + Tool, + ) + + self.tools: dict[str, Tool] = {} + browser_tool = HarmonyBrowserTool() + if browser_tool.enabled: + self.tools["browser"] = browser_tool + python_tool = HarmonyPythonTool() + if python_tool.enabled: + self.tools["python"] = python_tool + + def has_tool(self, tool_name: str): + return tool_name in self.tools + + def get_tool_description(self, tool_name: str): + if tool_name not in self.tools: + return None + if tool_name == "browser": + return ToolNamespaceConfig.browser() + elif tool_name == "python": + return ToolNamespaceConfig.python() + else: + raise ValueError(f"Unknown tool {tool_name}") + + @asynccontextmanager + async def get_tool_session(self, tool_name: str): + yield self.tools[tool_name] diff --git a/python/sglang/srt/entrypoints/openai/usage_processor.py b/python/sglang/srt/entrypoints/openai/usage_processor.py new file mode 100644 index 000000000..cdd14d74e --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/usage_processor.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Mapping, Optional, final + +from sglang.srt.entrypoints.openai.protocol import UsageInfo + + +@final +class UsageProcessor: + """Stateless helpers that turn raw token counts into a UsageInfo.""" + + @staticmethod + def _details_if_cached(count: int) -> Optional[Dict[str, int]]: + """Return {"cached_tokens": N} only when N > 0 (keeps JSON slim).""" + return {"cached_tokens": count} if count > 0 else None + + @staticmethod + def calculate_response_usage( + responses: List[Dict[str, Any]], + n_choices: int = 1, + enable_cache_report: bool = False, + ) -> UsageInfo: + completion_tokens = sum(r["meta_info"]["completion_tokens"] for r in responses) + + prompt_tokens = sum( + responses[i]["meta_info"]["prompt_tokens"] + for i in range(0, len(responses), n_choices) + ) + + cached_details = None + if enable_cache_report: + cached_total = sum( + r["meta_info"].get("cached_tokens", 0) for r in responses + ) + cached_details = UsageProcessor._details_if_cached(cached_total) + + return UsageProcessor.calculate_token_usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + cached_tokens=cached_details, + ) + + @staticmethod + def calculate_streaming_usage( + prompt_tokens: Mapping[int, int], + completion_tokens: Mapping[int, int], + cached_tokens: Mapping[int, int], + n_choices: int, + enable_cache_report: bool = False, + ) -> UsageInfo: + # index % n_choices == 0 marks the first choice of a prompt + total_prompt_tokens = sum( + tok for idx, tok in prompt_tokens.items() if idx % n_choices == 0 + ) + total_completion_tokens = sum(completion_tokens.values()) + + cached_details = ( + UsageProcessor._details_if_cached(sum(cached_tokens.values())) + if enable_cache_report + else None + ) + + return UsageProcessor.calculate_token_usage( + prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + cached_tokens=cached_details, + ) + + @staticmethod + def calculate_token_usage( + prompt_tokens: int, + completion_tokens: int, + cached_tokens: Optional[Dict[str, int]] = None, + ) -> UsageInfo: + """Calculate token usage information""" + return UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=cached_tokens, + ) diff --git a/python/sglang/srt/entrypoints/openai/utils.py b/python/sglang/srt/entrypoints/openai/utils.py new file mode 100644 index 000000000..94ac5458d --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/utils.py @@ -0,0 +1,72 @@ +import logging +from typing import Any, Dict, List, Optional, Union + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + CompletionRequest, + LogProbs, +) + +logger = logging.getLogger(__name__) + + +def to_openai_style_logprobs( + input_token_logprobs=None, + output_token_logprobs=None, + input_top_logprobs=None, + output_top_logprobs=None, +): + ret_logprobs = LogProbs() + + def append_token_logprobs(token_logprobs): + for logprob, _, token_text in token_logprobs: + ret_logprobs.tokens.append(token_text) + ret_logprobs.token_logprobs.append(logprob) + + # Not supported yet + ret_logprobs.text_offset.append(-1) + + def append_top_logprobs(top_logprobs): + for tokens in top_logprobs: + if tokens is not None: + ret_logprobs.top_logprobs.append( + {token[2]: token[0] for token in tokens} + ) + else: + ret_logprobs.top_logprobs.append(None) + + if input_token_logprobs is not None: + append_token_logprobs(input_token_logprobs) + if output_token_logprobs is not None: + append_token_logprobs(output_token_logprobs) + if input_top_logprobs is not None: + append_top_logprobs(input_top_logprobs) + if output_top_logprobs is not None: + append_top_logprobs(output_top_logprobs) + + return ret_logprobs + + +def process_hidden_states_from_ret( + ret_item: Dict[str, Any], + request: Union[ + ChatCompletionRequest, + CompletionRequest, + ], +) -> Optional[List]: + """Process hidden states from a ret item in non-streaming response. + + Args: + ret_item: Response item containing meta_info + request: The original request object + + Returns: + Processed hidden states for the last token, or None + """ + if not request.return_hidden_states: + return None + + hidden_states = ret_item["meta_info"].get("hidden_states", None) + if hidden_states is not None: + hidden_states = hidden_states[-1] if len(hidden_states) > 1 else [] + return hidden_states diff --git a/python/sglang/srt/entrypoints/tool.py b/python/sglang/srt/entrypoints/tool.py new file mode 100644 index 000000000..45b87ac3a --- /dev/null +++ b/python/sglang/srt/entrypoints/tool.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +import logging +import os +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Any + +from sglang.srt.utils import print_info_once, print_warning_once + +if TYPE_CHECKING: + # Avoid circular import. + from sglang.srt.entrypoints.context import ConversationContext + +logger = logging.getLogger(__name__) + + +class Tool(ABC): + + @abstractmethod + async def get_result(self, context: "ConversationContext") -> Any: + pass + + +class HarmonyBrowserTool(Tool): + + def __init__(self): + self.enabled = True + exa_api_key = os.getenv("EXA_API_KEY") + if not exa_api_key: + self.enabled = False + print_warning_once("EXA_API_KEY is not set, browsing is disabled") + return + + try: + from gpt_oss.tools.simple_browser import SimpleBrowserTool + from gpt_oss.tools.simple_browser.backend import ExaBackend + except ImportError: + self.enabled = False + print_warning_once("gpt_oss is not installed, browsing is disabled") + return + + browser_backend = ExaBackend(source="web", api_key=exa_api_key) + self.browser_tool = SimpleBrowserTool(backend=browser_backend) + print_info_once("Browser tool initialized") + + async def get_result(self, context: "ConversationContext") -> Any: + from sglang.srt.entrypoints.context import HarmonyContext + + assert isinstance(context, HarmonyContext) + last_msg = context.messages[-1] + tool_output_msgs = [] + async for msg in self.browser_tool.process(last_msg): + tool_output_msgs.append(msg) + return tool_output_msgs + + @property + def tool_config(self) -> Any: + return self.browser_tool.tool_config + + +class HarmonyPythonTool(Tool): + + def __init__(self): + self.enabled = True + + try: + from gpt_oss.tools.python_docker.docker_tool import PythonTool + except ImportError: + self.enabled = False + print_warning_once("gpt_oss is not installed, code interpreter is disabled") + return + + self.python_tool = PythonTool() + print_info_once("Code interpreter tool initialized") + + async def get_result(self, context: "ConversationContext") -> Any: + from sglang.srt.entrypoints.context import HarmonyContext + + assert isinstance(context, HarmonyContext) + last_msg = context.messages[-1] + tool_output_msgs = [] + async for msg in self.python_tool.process(last_msg): + tool_output_msgs.append(msg) + return tool_output_msgs + + @property + def tool_config(self) -> Any: + return self.python_tool.tool_config diff --git a/python/sglang/srt/eplb/__init__.py b/python/sglang/srt/eplb/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/eplb/eplb_algorithms/__init__.py b/python/sglang/srt/eplb/eplb_algorithms/__init__.py new file mode 100644 index 000000000..e2a267810 --- /dev/null +++ b/python/sglang/srt/eplb/eplb_algorithms/__init__.py @@ -0,0 +1,63 @@ +from enum import Enum, auto +from typing import Optional + +import torch + +from sglang.srt.eplb.eplb_algorithms import deepseek, deepseek_vec + + +class EplbAlgorithm(Enum): + deepseek = auto() + deepseek_hierarchical = auto() + deepseek_vec = auto() + deepseek_vec_hierarchical = auto() + # TODO may have more algorithm later + + +def rebalance_experts( + tokens_per_expert: torch.Tensor, + num_physical_experts: int, + num_local_physical_experts: int, + num_groups: Optional[int], + num_nodes: int, + algorithm: EplbAlgorithm, +): + if algorithm in [EplbAlgorithm.deepseek, EplbAlgorithm.deepseek_hierarchical]: + return deepseek.rebalance_experts( + weight=tokens_per_expert.sum(dim=0), + num_replicas=num_physical_experts, + num_groups=num_groups, + num_nodes=num_nodes, + num_gpus=num_physical_experts // num_local_physical_experts, + enable_hierarchical=algorithm == EplbAlgorithm.deepseek_hierarchical, + ) + + if algorithm in [ + EplbAlgorithm.deepseek_vec, + EplbAlgorithm.deepseek_vec_hierarchical, + ]: + return deepseek_vec.rebalance_experts( + tokens_per_expert=tokens_per_expert, + num_physical_experts=num_physical_experts, + num_local_physical_experts=num_local_physical_experts, + num_groups=num_groups, + num_nodes=num_nodes, + enable_hierarchical=algorithm == EplbAlgorithm.deepseek_vec_hierarchical, + ) + + raise NotImplementedError + + +def compute_algorithm( + raw_algorithm: str, + num_groups: Optional[int], + num_nodes: int, +) -> EplbAlgorithm: + if raw_algorithm != "auto": + return EplbAlgorithm[raw_algorithm] + + # TODO test on real scenarios and know which ones perform better + if (num_groups is not None) and (num_groups % num_nodes == 0): + return EplbAlgorithm.deepseek_hierarchical + else: + return EplbAlgorithm.deepseek diff --git a/python/sglang/srt/eplb/eplb_algorithms/deepseek.py b/python/sglang/srt/eplb/eplb_algorithms/deepseek.py new file mode 100644 index 000000000..180ccdee4 --- /dev/null +++ b/python/sglang/srt/eplb/eplb_algorithms/deepseek.py @@ -0,0 +1,223 @@ +# This file is copied from https://github.com/deepseek-ai/EPLB/blob/main/eplb.py since that one is not a pypi package +from typing import Tuple + +import torch + +from sglang.srt.utils import get_bool_env_var + + +def balanced_packing( + weight: torch.Tensor, num_packs: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Pack n weighted objects to m packs, such that each bin contains exactly n/m objects and the weights of all packs + are as balanced as possible. + + Parameters: + weight: [X, n], the weight of each item + num_packs: number of packs + + Returns: + pack_index: [X, n], the pack index of each item + rank_in_pack: [X, n], the rank of the item in the pack + """ + num_layers, num_groups = weight.shape + assert num_groups % num_packs == 0 + groups_per_pack = num_groups // num_packs + + if groups_per_pack == 1: + pack_index = torch.arange( + weight.size(-1), dtype=torch.int64, device=weight.device + ).expand(weight.shape) + rank_in_pack = torch.zeros_like(weight, dtype=torch.int64) + return pack_index, rank_in_pack + + indices = weight.float().sort(-1, descending=True).indices.cpu() + pack_index = torch.full_like(weight, fill_value=-1, dtype=torch.int64, device="cpu") + rank_in_pack = torch.full_like(pack_index, fill_value=-1) + for i in range(num_layers): + pack_weights = [0] * num_packs + pack_items = [0] * num_packs + for group in indices[i]: + pack = min( + (i for i in range(num_packs) if pack_items[i] < groups_per_pack), + key=pack_weights.__getitem__, + ) + assert pack_items[pack] < groups_per_pack + pack_index[i, group] = pack + rank_in_pack[i, group] = pack_items[pack] + pack_weights[pack] += weight[i, group] + pack_items[pack] += 1 + return pack_index, rank_in_pack + + +def replicate_experts( + weight: torch.Tensor, num_phy: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Replicate `num_log` experts to `num_phy` replicas, such that the maximum load of all replicas is minimized. + + Parameters: + weight: [X, num_log] + num_phy: total number of experts after replication + + Returns: + phy2log: [X, num_phy], logical expert id of each physical expert + rank: [X, num_phy], the replica rank + logcnt: [X, num_log], number of replicas for each logical expert + """ + n, num_log = weight.shape + num_redundant = num_phy - num_log + assert num_redundant >= 0 + device = weight.device + phy2log = torch.arange(num_phy, dtype=torch.int64, device=device).repeat(n, 1) + rank = torch.zeros(n, num_phy, dtype=torch.int64, device=device) + logcnt = torch.ones(n, num_log, dtype=torch.int64, device=device) + arangen = torch.arange(n, dtype=torch.int64, device=device) + for i in range(num_log, num_phy): + redundant_indices = (weight / logcnt).max(dim=-1).indices + phy2log[:, i] = redundant_indices + rank[:, i] = logcnt[arangen, redundant_indices] + logcnt[arangen, redundant_indices] += 1 + return phy2log, rank, logcnt + + +def rebalance_experts_hierarchical( + weight: torch.Tensor, + num_physical_experts: int, + num_groups: int, + num_nodes: int, + num_gpus: int, +): + """ + Parameters: + weight: [num_moe_layers, num_logical_experts] + num_physical_experts: number of physical experts after replication + num_groups: number of expert groups + num_nodes: number of server nodes, where the intra-node network (e.g, NVLink) is faster + num_gpus: number of GPUs, must be a multiple of `num_nodes` + + Returns: + physical_to_logical_map: [num_moe_layers, num_physical_experts] + logical_to_physical_map: [num_moe_layers, num_logical_experts, X] + logical_count: [num_moe_layers, num_logical_experts] + """ + num_layers, num_logical_experts = weight.shape + assert num_logical_experts % num_groups == 0 + group_size = num_logical_experts // num_groups + assert num_groups % num_nodes == 0 + groups_per_node = num_groups // num_nodes + assert num_gpus % num_nodes == 0 + assert num_physical_experts % num_gpus == 0 + phy_experts_per_gpu = num_physical_experts // num_gpus + + def inverse(perm: torch.Tensor) -> torch.Tensor: + inv = torch.empty_like(perm) + inv.scatter_( + 1, + perm, + torch.arange(perm.size(1), dtype=torch.int64, device=perm.device).expand( + perm.shape + ), + ) + return inv + + # Step 1: pack groups to nodes + tokens_per_group = weight.unflatten(-1, (num_groups, group_size)).sum(-1) + group_pack_index, group_rank_in_pack = balanced_packing(tokens_per_group, num_nodes) + log2mlog = ( + ( + (group_pack_index * groups_per_node + group_rank_in_pack) * group_size + ).unsqueeze(-1) + + torch.arange(group_size, dtype=torch.int64, device=group_pack_index.device) + ).flatten(-2) + mlog2log = inverse(log2mlog) + + # Step 2: construct redundant experts within nodes + # [num_layers * num_nodes, num_logical_experts // num_nodes] + tokens_per_mlog = weight.gather(-1, mlog2log).view( + -1, num_logical_experts // num_nodes + ) + phy2mlog, phyrank, mlogcnt = replicate_experts( + tokens_per_mlog, num_physical_experts // num_nodes + ) + + # Step 3: pack physical_experts to GPUs + # [num_layers * num_nodes, num_physical_experts // num_nodes] + tokens_per_phy = (tokens_per_mlog / mlogcnt).gather(-1, phy2mlog) + pack_index, rank_in_pack = balanced_packing(tokens_per_phy, num_gpus // num_nodes) + phy2pphy = pack_index * phy_experts_per_gpu + rank_in_pack + pphy2phy = inverse(phy2pphy) + + pphy2mlog = phy2mlog.gather( + -1, pphy2phy + ) # [num_layers * num_nodes, num_log_per_nodes] + pphy2mlog = ( + pphy2mlog.view(num_layers, num_nodes, -1) + + torch.arange( + 0, + num_logical_experts, + num_logical_experts // num_nodes, + device=group_pack_index.device, + ).view(1, -1, 1) + ).flatten(-2) + pphy2log = mlog2log.gather(-1, pphy2mlog) + pphyrank = phyrank.gather(-1, pphy2phy).view(num_layers, -1) + logcnt = mlogcnt.view(num_layers, -1).gather(-1, log2mlog) + return pphy2log, pphyrank, logcnt + + +def rebalance_experts( + weight: torch.Tensor, + num_replicas: int, + num_groups: int, + num_nodes: int, + num_gpus: int, + enable_hierarchical: bool, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Entry point for expert-parallelism load balancer. + + Parameters: + weight: [layers, num_logical_experts], the load statistics for all logical experts + num_replicas: number of physical experts, must be a multiple of `num_gpus` + num_groups: number of expert groups + num_nodes: number of server nodes, where the intra-node network (e.g, NVLink) is faster + num_gpus: number of GPUs, must be a multiple of `num_nodes` + + Returns: + physical_to_logical_map: [layers, num_replicas], the expert index of each replica + logical_to_physical_map: [layers, num_logical_experts, X], the replica indices for each expert + expert_count: [layers, num_logical_experts], number of physical replicas for each logical expert + """ + + num_layers, num_logical_experts = weight.shape + weight = weight.float().cpu() + if enable_hierarchical: + # use hierarchical load-balance policy + phy2log, phyrank, logcnt = rebalance_experts_hierarchical( + weight, num_replicas, num_groups, num_nodes, num_gpus + ) + else: + # use global load-balance policy + phy2log, phyrank, logcnt = rebalance_experts_hierarchical( + weight, num_replicas, 1, 1, num_gpus + ) + maxlogcnt = logcnt.max().item() + log2phy: torch.Tensor = torch.full( + (num_layers, num_logical_experts, maxlogcnt), + -1, + dtype=torch.int64, + device=logcnt.device, + ) + log2phy.view(num_layers, -1).scatter_( + -1, + phy2log * maxlogcnt + phyrank, + torch.arange(num_replicas, dtype=torch.int64, device=log2phy.device).expand( + num_layers, -1 + ), + ) + return phy2log, log2phy, logcnt + + +__all__ = ["rebalance_experts"] diff --git a/python/sglang/srt/eplb/eplb_algorithms/deepseek_vec.py b/python/sglang/srt/eplb/eplb_algorithms/deepseek_vec.py new file mode 100644 index 000000000..cb165448a --- /dev/null +++ b/python/sglang/srt/eplb/eplb_algorithms/deepseek_vec.py @@ -0,0 +1,276 @@ +# This file is copied from https://github.com/deepseek-ai/EPLB/blob/main/eplb.py since that one is not a pypi package +from typing import Optional, Tuple + +import torch + + +def pack_groups(tokens_per_group: torch.Tensor, num_nodes: int) -> torch.Tensor: + num_layers, num_groups = tokens_per_group.shape + assert num_groups % num_nodes == 0 + groups_per_rank = num_groups // num_nodes + + indices = tokens_per_group.float().sort(-1, descending=True).indices.cpu() + ret = torch.full_like( + tokens_per_group, fill_value=-1, dtype=torch.int64, device="cpu" + ) + for layer in range(num_layers): + node_tokens = [0] * num_nodes + node_groups = [0] * num_nodes + for group in indices[layer]: + + def key_func(rank: int) -> int: + if node_groups[rank] >= groups_per_rank: + return 1, 0 + else: + return 0, node_tokens[rank] + + rank = min(range(num_nodes), key=key_func) + assert node_groups[rank] < groups_per_rank + ret[layer, group] = rank * groups_per_rank + node_groups[rank] + node_tokens[rank] += tokens_per_group[layer, group] + node_groups[rank] += 1 + return ret + + +def make_redundant_experts_chunkwise( + tokens_per_expert: torch.Tensor, + num_physical_experts: int, + num_local_physical_experts: int, + num_physical_experts_per_chunk: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + num_steps, num_moe_layers, num_logical_experts = tokens_per_expert.shape + num_redundancy_experts = num_physical_experts - num_logical_experts + + physical_to_logical_map = torch.empty( + num_moe_layers, + num_physical_experts, + dtype=torch.int, + device=tokens_per_expert.device, + ) + logical_to_physical_map = torch.full( + (num_moe_layers, num_logical_experts, num_redundancy_experts + 1), + -1, + dtype=torch.int, + device=tokens_per_expert.device, + ) + logical_count = torch.ones( + num_moe_layers, + num_logical_experts, + dtype=torch.int, + device=tokens_per_expert.device, + ) + + assert num_physical_experts % num_physical_experts_per_chunk == 0 + num_chunks = num_physical_experts // num_physical_experts_per_chunk + assert num_logical_experts % num_chunks == 0 + num_logical_experts_per_group = num_logical_experts // num_chunks + assert num_redundancy_experts % num_chunks == 0 + num_redundancy_experts_per_group = num_redundancy_experts // num_chunks + + arange_num_moe_layers_num_groups = torch.arange( + num_moe_layers * num_chunks, dtype=torch.int, device=tokens_per_expert.device + ) + arange_num_logical_experts = torch.arange( + num_logical_experts, dtype=torch.int, device=tokens_per_expert.device + ) + arange_num_logical_experts_per_group = torch.arange( + num_logical_experts_per_group, dtype=torch.int, device=tokens_per_expert.device + ) + arange_num_groups = torch.arange( + num_chunks, dtype=torch.int, device=tokens_per_expert.device + ) + physical_to_logical_map.view( + num_moe_layers, num_chunks, num_physical_experts_per_chunk + )[:, :, :num_logical_experts_per_group] = arange_num_logical_experts.view( + num_chunks, num_logical_experts_per_group + ) + logical_to_physical_map[:, :, 0] = ( + arange_num_logical_experts_per_group.expand( + num_chunks, num_logical_experts_per_group + ) + + arange_num_groups[:, None] * num_physical_experts_per_chunk + ).view(num_logical_experts) + + tokens_per_expert_all_diff = tokens_per_expert + arange_num_logical_experts * 1e-4 + for i in range(num_redundancy_experts_per_group): + score = ( + tokens_per_expert_all_diff / logical_count + ) # NOTE: Values in score must be different from each other + score1 = tokens_per_expert / (logical_count + 1) + score = score.view( + num_steps, num_moe_layers, num_chunks, num_logical_experts_per_group + ) + score1 = score1.view_as(score) + values, indices = score.max(-1, keepdim=True) + values = values.expand_as(score).contiguous() + score.scatter_(-1, indices, score1.gather(-1, indices)) + values.scatter_(-1, indices, score.max(-1, keepdim=True).values) + redundancy_indices = values.sum(0).argmin(-1) + physical_to_logical_map.view( + num_moe_layers, num_chunks, num_physical_experts_per_chunk + )[:, :, num_logical_experts_per_group + i] = ( + redundancy_indices + arange_num_groups * num_logical_experts_per_group + ) + redundancy_count = ( + logical_count.view( + num_moe_layers * num_chunks, num_logical_experts_per_group + ) + .gather(-1, redundancy_indices.view(num_moe_layers * num_chunks, 1)) + .squeeze(1) + ) + physical_redundancy_indices = ( + ( + arange_num_groups * num_physical_experts_per_chunk + + num_logical_experts_per_group + + i + ) + .expand(num_moe_layers, num_chunks) + .flatten() + ) + logical_to_physical_map.view( + num_moe_layers * num_chunks, + num_logical_experts_per_group, + num_redundancy_experts + 1, + )[ + arange_num_moe_layers_num_groups, + redundancy_indices.view(num_moe_layers * num_chunks), + redundancy_count, + ] = physical_redundancy_indices + logical_count.view(num_moe_layers * num_chunks, num_logical_experts_per_group)[ + arange_num_moe_layers_num_groups, + redundancy_indices.view(num_moe_layers * num_chunks), + ] += 1 + + if num_local_physical_experts > 1: + # Load-balancing between GPUs + physical_to_logical_map_int64 = physical_to_logical_map.to(torch.int64) + counts = logical_count.gather(-1, physical_to_logical_map_int64) + score = tokens_per_expert.sum(0).gather(-1, physical_to_logical_map_int64) + score = score / counts + score = score.view(num_moe_layers, num_chunks, num_physical_experts_per_chunk) + indices = score.argsort(-1, descending=True) + indices += torch.arange( + 0, + num_physical_experts, + num_physical_experts_per_chunk, + dtype=indices.dtype, + device=indices.device, + )[None, :, None] + + assert num_physical_experts_per_chunk % num_local_physical_experts == 0 + num_local_groups = num_physical_experts_per_chunk // num_local_physical_experts + indices = indices.view( + num_moe_layers, num_chunks, num_local_physical_experts, num_local_groups + ) + indices[:, :, 1::2, :] = indices[:, :, 1::2, :].flip(-1) + indices = indices.transpose(2, 3) + indices = indices.reshape(num_moe_layers, num_physical_experts) + physical_to_logical_map = physical_to_logical_map.gather(-1, indices) + mask = logical_to_physical_map == -1 + logical_to_physical_map[mask] = 0 + logical_to_physical_map = ( + indices.argsort(-1) + .gather( + -1, logical_to_physical_map.view(num_moe_layers, -1).to(torch.int64) + ) + .view_as(logical_to_physical_map) + .to(torch.int) + ) + logical_to_physical_map[mask] = -1 + + return physical_to_logical_map, logical_to_physical_map, logical_count + + +def decode_rebalance_experts( + tokens_per_expert: torch.Tensor, + num_physical_experts: int, + num_local_physical_experts: int, +): + return make_redundant_experts_chunkwise( + tokens_per_expert, + num_physical_experts, + num_local_physical_experts, + num_physical_experts, + ) + + +def prefill_rebalance_experts( + tokens_per_expert: torch.Tensor, + num_physical_experts: int, + num_local_physical_experts: int, + num_groups: int, + num_nodes: int, +): + tokens_per_expert = tokens_per_expert.float().cpu() + + num_steps, _, num_logical_experts = tokens_per_expert.shape + assert num_logical_experts % num_groups == 0 + group_size = num_logical_experts // num_groups + assert num_groups % num_nodes == 0, f"{num_groups=} {num_nodes=}" + + tokens_per_group = tokens_per_expert.sum(0).unflatten(-1, (num_groups, -1)).sum(-1) + group_perm = pack_groups( + tokens_per_group, num_nodes + ) # [num_moe_layers, num_groups] => [num_moe_layers, num_nodes] + + # log2mlog [layers, #logexp] -> [layers, #logexp] + log2mlog = ( + (group_perm * group_size).unsqueeze(-1) + + torch.arange(group_size, dtype=torch.int64, device=group_perm.device) + ).flatten(-2) + + # mlog2log [layers, #logexp] -> [layers, #logexp], inverse of log2mlog + mlog2log = torch.empty_like(log2mlog) + arange = torch.arange( + num_logical_experts, dtype=torch.int64, device=mlog2log.device + ) + mlog2log.scatter_(1, log2mlog, arange.expand(log2mlog.size(0), -1)) + + # tokens_per_mlog[i][j][k] = tokens_per_expert[i][j][mlog2log[j][k]] + tokens_per_mlog = tokens_per_expert.gather( + 2, mlog2log.unsqueeze(0).expand(num_steps, -1, -1) + ) + + phy2mlog, mlog2phy, mlog_count = make_redundant_experts_chunkwise( + tokens_per_mlog, + num_physical_experts, + num_local_physical_experts, + num_physical_experts // num_nodes, + ) + + # phy2log[i][j] = mlog2log[i][phy2mlog[i][j]] + phy2log = mlog2log.gather(1, phy2mlog.to(torch.int64)) + + # mlog2phy: [num_moe_layers, num_logical_experts, ...] + # log2phy[i][j][k] = mlog2phy[i][log2mlog[i][j]][k] + log2phy = mlog2phy.gather( + 1, log2mlog.unsqueeze(-1).expand(-1, -1, mlog2phy.size(-1)).to(torch.int64) + ) + + # log_count[i][j] = mlog_count[i][log2mlog[i][j]] + log_count = mlog_count.gather(1, log2mlog) + return phy2log, log2phy, log_count + + +def rebalance_experts( + tokens_per_expert: torch.Tensor, + num_physical_experts: int, + num_local_physical_experts: int, + num_groups: Optional[int], + num_nodes: int, + enable_hierarchical: bool, +): + if enable_hierarchical: + return prefill_rebalance_experts( + tokens_per_expert=tokens_per_expert, + num_physical_experts=num_physical_experts, + num_local_physical_experts=num_local_physical_experts, + num_groups=num_groups, + num_nodes=num_nodes, + ) + else: + return decode_rebalance_experts( + tokens_per_expert=tokens_per_expert, + num_physical_experts=num_physical_experts, + num_local_physical_experts=num_local_physical_experts, + ) diff --git a/python/sglang/srt/eplb/eplb_manager.py b/python/sglang/srt/eplb/eplb_manager.py new file mode 100644 index 000000000..e88a3d28e --- /dev/null +++ b/python/sglang/srt/eplb/eplb_manager.py @@ -0,0 +1,118 @@ +import logging +import time +from typing import TYPE_CHECKING, List + +import torch.cuda + +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location import ExpertLocationMetadata + +if TYPE_CHECKING: + from sglang.srt.model_executor.model_runner import ModelRunner + +logger = logging.getLogger(__name__) + + +class EPLBManager: + def __init__(self, model_runner: "ModelRunner"): + super().__init__() + self._model_runner = model_runner + self._server_args = model_runner.server_args + self._rebalance_layers_per_chunk = ( + self._server_args.eplb_rebalance_layers_per_chunk + ) + self._rebalance_num_iterations = self._server_args.eplb_rebalance_num_iterations + + # Otherwise, the circular buffer will contain stale data. If the case is needed, it can be implemented. + assert ( + self._server_args.eplb_rebalance_num_iterations + >= self._server_args.expert_distribution_recorder_buffer_size + ), "eplb_rebalance_num_iterations must be greater than expert_distribution_recorder_buffer_size" + + if not get_global_expert_distribution_recorder().recording: + get_global_expert_distribution_recorder().start_record() + + logger.info( + f"[EPLBManager] system started, will rebalance per {self._rebalance_num_iterations} iterations." + ) + + self._main_generator = self._entrypoint() + + def on_forward_pass_end(self): + next(self._main_generator) + + # can be more complex if needed + def _entrypoint(self): + while True: + for _ in range(self._rebalance_num_iterations): + yield + + yield from self.rebalance() + + def rebalance(self): + logger.info("[EPLBManager] rebalance start") + + enable_timing = self._rebalance_layers_per_chunk is None + + if enable_timing: + torch.get_device_module().synchronize() + time_start = time.time() + + dump_record_output = get_global_expert_distribution_recorder().dump_record( + output_mode="object" + ) + logical_count = dump_record_output["logical_count"] + average_utilization_rate_over_window = dump_record_output[ + "average_utilization_rate_over_window" + ] + + # Check whether rebalancing is needed + if not self._check_rebalance_needed(average_utilization_rate_over_window): + return + + expert_location_metadata = ExpertLocationMetadata.init_by_eplb( + self._server_args, self._model_runner.model_config, logical_count + ) + + update_layer_ids_chunks = self._compute_update_layer_ids_chunks() + for chunk_index, update_layer_ids in enumerate(update_layer_ids_chunks): + if len(update_layer_ids_chunks) > 1: + yield + self._model_runner.update_expert_location( + expert_location_metadata, + update_layer_ids=update_layer_ids, + ) + + msg = f"[EPLBManager] rebalance end" + if enable_timing: + torch.get_device_module().synchronize() + time_end = time.time() + msg += f" time={time_end - time_start:.3f}s" + logger.info(msg) + + def _check_rebalance_needed(self, average_utilization_rate_over_window): + if average_utilization_rate_over_window is None: + return True + + if ( + average_utilization_rate_over_window + > self._server_args.eplb_min_rebalancing_utilization_threshold + ): + logger.info( + f"[EPLBManager] Skipped ep rebalancing: current GPU utilization {average_utilization_rate_over_window:.2f} > minimum rebalance threshold {self._server_args.eplb_min_rebalancing_utilization_threshold:.2f}" + ) + return False + + return True + + def _compute_update_layer_ids_chunks(self) -> List[List[int]]: + all_layer_ids = sorted( + list(self._model_runner.model.routed_experts_weights_of_layer.keys()) + ) + chunk_size = self._rebalance_layers_per_chunk or 1000000 + return list(_chunk_list(all_layer_ids, chunk_size=chunk_size)) + + +def _chunk_list(items: List, chunk_size): + for start_index in range(0, len(items), chunk_size): + yield items[start_index : start_index + chunk_size] diff --git a/python/sglang/srt/eplb/eplb_simulator/__init__.py b/python/sglang/srt/eplb/eplb_simulator/__init__.py new file mode 100644 index 000000000..a1fcbdf00 --- /dev/null +++ b/python/sglang/srt/eplb/eplb_simulator/__init__.py @@ -0,0 +1 @@ +from . import reader diff --git a/python/sglang/srt/eplb/eplb_simulator/reader.py b/python/sglang/srt/eplb/eplb_simulator/reader.py new file mode 100644 index 000000000..97405c319 --- /dev/null +++ b/python/sglang/srt/eplb/eplb_simulator/reader.py @@ -0,0 +1,51 @@ +from collections import defaultdict +from pathlib import Path + +import torch +from tqdm import tqdm + +from sglang.srt.eplb.expert_distribution import ( + _convert_global_physical_count_to_logical_count, +) + +convert_global_physical_count_to_logical_count = ( + _convert_global_physical_count_to_logical_count +) + + +def read_mode_per_pass(dir_data: Path): + """Read data from ExpertDistributionRecorder when recorded with mode `per_pass`""" + + # gpc := global_physical_count + gpc_of_forward_pass_and_rank = defaultdict(lambda: defaultdict()) + for path in tqdm(list(dir_data.glob("*.pt"))): + data_pack = torch.load(path, weights_only=True) + last_physical_to_logical_map = data_pack["last_physical_to_logical_map"] + for record in data_pack["records"]: + forward_pass_id = record["forward_pass_id"] + rank = record["rank"] + assert ( + gpc_of_forward_pass_and_rank[forward_pass_id].get(rank) is None + ), f"Duplicated {forward_pass_id=} {rank=}" + gpc_of_forward_pass_and_rank[forward_pass_id][rank] = record[ + "global_physical_count" + ] + + forward_pass_ids = sorted(gpc_of_forward_pass_and_rank.keys()) + print(f"Make {forward_pass_ids=} into array") + + items = [] + for forward_pass_id, gpc_of_rank in sorted(gpc_of_forward_pass_and_rank.items()): + gpc_of_rank_tensor = torch.stack( + [gpc for rank, gpc in sorted(gpc_of_rank.items())] + ).sum(dim=0) + items.append(gpc_of_rank_tensor) + + gpc_of_forward_pass = torch.stack(items) + print(f"{gpc_of_forward_pass.shape=}") + + return dict( + global_physical_count_of_forward_pass=gpc_of_forward_pass, + last_physical_to_logical_map=last_physical_to_logical_map, + forward_pass_ids=forward_pass_ids, + ) diff --git a/python/sglang/srt/eplb/expert_distribution.py b/python/sglang/srt/eplb/expert_distribution.py new file mode 100644 index 000000000..3faf981ef --- /dev/null +++ b/python/sglang/srt/eplb/expert_distribution.py @@ -0,0 +1,966 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import annotations + +import logging +import math +import os +import time +from abc import ABC +from collections import deque +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Type + +import einops +import torch +import torch.distributed + +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import Withable, get_bool_env_var, is_npu + +_is_npu = is_npu() + +if TYPE_CHECKING: + from sglang.srt.eplb.expert_location import ExpertLocationMetadata + +logger = logging.getLogger(__name__) + +# --------------------------------------- Entrypoint ----------------------------------------- + +_OutputMode = Literal["file", "object"] + + +class ExpertDistributionRecorder(ABC): + """Global expert distribution recording""" + + @staticmethod + def init_new( + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ): + if server_args.expert_distribution_recorder_mode is not None: + assert ( + expert_location_metadata is not None + ), "ExpertLocationMetadata is required for expert distribution recording. One possible" + "reason is that you are using a model that does not support expert distribution" + "recording. Try setting `get_model_config_for_expert_location` in your model." + return _ExpertDistributionRecorderReal( + server_args, expert_location_metadata, rank + ) + else: + return _ExpertDistributionRecorderNoop() + + @contextmanager + def with_current_layer(self, layer_idx): + yield + + @contextmanager + def with_debug_name(self, debug_name): + yield + + @contextmanager + def disable_this_region(self): + yield + + @contextmanager + def with_forward_pass(self, forward_pass_id: int, forward_batch: ForwardBatch): + yield + + def on_select_experts(self, topk_ids: torch.Tensor): + pass + + def on_deepep_dispatch_normal( + self, + local_physical_count_of_layer: List[int], + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + ): + pass + + def on_deepep_dispatch_low_latency( + self, local_physical_count_of_layer: torch.Tensor + ): + pass + + def start_record(self): + self._on_not_implemented() + + def stop_record(self): + self._on_not_implemented() + + def dump_record(self, output_mode: _OutputMode = "file"): + self._on_not_implemented() + + @property + def recording(self): + return False + + def _on_not_implemented(self): + raise Exception( + "Please set ServerArgs.expert_distribution_recorder_mode to use ExpertDistributionRecorder." + ) + + +class _ExpertDistributionRecorderNoop(ExpertDistributionRecorder): + pass + + +class _ExpertDistributionRecorderReal(ExpertDistributionRecorder): + def __init__( + self, + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ): + self._server_args = server_args + self._expert_location_metadata = expert_location_metadata + + self._recording = False + self._disable_all = False + self._current_forward_pass_id = Withable() + self._current_layer_idx = Withable() + self._current_debug_name = Withable() + self._accumulator = _Accumulator.init_new( + server_args, expert_location_metadata, rank + ) + self._single_pass_gatherers = { + k: _SinglePassGatherer.init_new(server_args, expert_location_metadata, rank) + for k in self._accumulator.get_single_pass_gatherer_keys() + } + + if server_args.enable_expert_distribution_metrics: + logger.info( + "ExpertDistributionRecorder auto start record since enable_expert_distribution_metrics" + ) + self.start_record() + + def with_current_layer(self, layer_idx): + return self._current_layer_idx.with_value(layer_idx) + + def with_debug_name(self, debug_name): + return self._current_debug_name.with_value(debug_name) + + @contextmanager + def with_forward_pass(self, forward_pass_id: int, forward_batch: ForwardBatch): + with self._current_forward_pass_id.with_value(forward_pass_id): + self._on_forward_pass_start(forward_batch) + try: + yield + finally: + self._on_forward_pass_end(forward_pass_id) + + @contextmanager + def disable_this_region(self): + """Context manager to temporarily disable recording.""" + previous_disable_all = self._disable_all + self._disable_all = True + try: + yield + finally: + self._disable_all = previous_disable_all + + def _on_forward_pass_start(self, forward_batch: ForwardBatch): + if not self._recording: + return + for gatherer_key, gatherer in self._single_pass_gatherers.items(): + gatherer.reset() + gatherer.on_forward_pass_start(forward_batch) + + def _on_forward_pass_end(self, forward_pass_id: int): + if not self._recording: + return + for gatherer_key, gatherer in self._single_pass_gatherers.items(): + single_pass_data = gatherer.collect() + self._accumulator.append(forward_pass_id, gatherer_key, single_pass_data) + + def on_select_experts(self, topk_ids: torch.Tensor): + self._on_hook("on_select_experts", topk_ids=topk_ids) + + def on_deepep_dispatch_normal( + self, + local_physical_count_of_layer: List[int], + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + ): + self._on_hook( + "on_deepep_dispatch_normal", + local_physical_count_of_layer=local_physical_count_of_layer, + num_tokens_per_rank=num_tokens_per_rank, + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, + num_tokens_per_expert=num_tokens_per_expert, + ) + + def on_deepep_dispatch_low_latency( + self, local_physical_count_of_layer: torch.Tensor + ): + self._on_hook( + "on_deepep_dispatch_low_latency", + local_physical_count_of_layer=local_physical_count_of_layer, + ) + + def _on_hook(self, hook_name: str, **kwargs): + if self._disable_all: + return + if not ( + self._recording or torch.get_device_module().is_current_stream_capturing() + ): + return + gatherer = self._single_pass_gatherers[ + self._accumulator.get_single_pass_gatherer_key( + self._current_debug_name.value + ) + ] + getattr(gatherer, hook_name)(layer_idx=self._current_layer_idx.value, **kwargs) + + def _reset(self): + """Reset the expert distribution recorder.""" + logger.info("Resetting ExpertDistributionRecorder...") + assert ( + self._current_layer_idx.value is None + ), f"{self._current_layer_idx.value=}" + for gatherer in self._single_pass_gatherers.values(): + gatherer.reset() + self._accumulator.reset() + + def start_record(self): + """Start recording the expert distribution.""" + if self._recording: + logger.warning( + "SGLang server is already recording expert ids. Did you forget to dump the expert ids recorded so far by sending requests to the `/stop_expert_distribution_record` and `/dump_expert_distribution_record` endpoints?" + ) + self._reset() + self._recording = True + + def stop_record(self): + """Stop recording the expert distribution.""" + if not self._recording: + logger.warning( + "SGLang server has not been recording expert ids. Did you forget to start recording by sending request to the `/start_expert_distribution_record` endpoint?" + ) + self._recording = False + + def dump_record(self, output_mode: _OutputMode = "file"): + """Dump the expert distribution record and reset the recorder after dumping.""" + output = self._accumulator.dump(output_mode=output_mode) + self._reset() + return output + + @property + def recording(self): + return self._recording + + +_global_expert_distribution_recorder: Optional[ExpertDistributionRecorder] = ( + _ExpertDistributionRecorderNoop() +) + + +def get_global_expert_distribution_recorder(): + return _global_expert_distribution_recorder + + +def set_global_expert_distribution_recorder(value): + global _global_expert_distribution_recorder + _global_expert_distribution_recorder = value + + +# --------------------------------------- SinglePassGatherer ----------------------------------------- + + +class _SinglePassGatherer(ABC): + @staticmethod + def init_new( + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ) -> "_SinglePassGatherer": + if server_args.expert_distribution_recorder_mode == "per_token": + return _DetailSinglePassGatherer( + server_args, expert_location_metadata, rank + ) + + if server_args.expert_distribution_recorder_mode == "stat_approx": + if server_args.moe_a2a_backend != "none" and ( + server_args.deepep_mode == "normal" + ): + return _DeepepNormalSinglePassGatherer(expert_location_metadata, rank) + else: + raise NotImplementedError + + if server_args.moe_a2a_backend != "none": + if server_args.deepep_mode == "normal": + return _SelectExpertsSinglePassGatherer(expert_location_metadata, rank) + elif server_args.deepep_mode == "low_latency": + return _DeepepLowLatencySinglePassGatherer( + expert_location_metadata, rank + ) + else: + raise NotImplementedError + + return _SelectExpertsSinglePassGatherer(expert_location_metadata, rank) + + def __init__(self, expert_location_metadata: ExpertLocationMetadata, rank: int): + self._expert_location_metadata = expert_location_metadata + self._rank = rank + + def on_forward_pass_start(self, forward_batch: ForwardBatch): + pass + + def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor): + pass + + def on_deepep_dispatch_normal( + self, + layer_idx: int, + local_physical_count_of_layer: List[int], + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + ): + pass + + def on_deepep_dispatch_low_latency( + self, layer_idx: int, local_physical_count_of_layer: torch.Tensor + ): + pass + + def reset(self): + raise NotImplementedError + + def collect(self) -> Dict: + raise NotImplementedError + + +class _DetailSinglePassGatherer(_SinglePassGatherer): + # DeepSeek V3 has this value; should generalize later + _TOP_K_NUM = 8 + + def __init__( + self, + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ): + super().__init__(expert_location_metadata, rank) + self._metadata: Optional[Dict[str, Any]] = None + self._topk_ids_of_layer = torch.zeros( + ( + expert_location_metadata.num_layers, + # TODO determine the max number + server_args.chunked_prefill_size * 8, + self._TOP_K_NUM, + ), + dtype=torch.int32, + device=server_args.device, + ) + self._misc_objects: List[Dict[str, Any]] = [] + assert ( + not server_args.enable_two_batch_overlap + ), "DetailSinglePassGatherer does not support TBO yet" + # TODO assert shared experts fusion is disabled, o/w data is wrong + + def on_forward_pass_start(self, forward_batch: ForwardBatch): + assert self._metadata is None + self._metadata = dict( + # TODO pr-chain + # rids=forward_batch.rids, + input_ids=forward_batch.input_ids.cpu().tolist(), + positions=forward_batch.positions.cpu().tolist(), + extend_seq_lens=forward_batch.extend_seq_lens_cpu, + forward_mode=forward_batch.forward_mode.value, + ) + + def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor): + self._topk_ids_of_layer[layer_idx, : topk_ids.shape[0], : topk_ids.shape[1]] = ( + topk_ids + ) + + def on_deepep_dispatch_normal( + self, + layer_idx: int, + local_physical_count_of_layer: List[int], + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + ): + self._misc_objects.append( + dict( + layer_id=layer_idx, + num_tokens_per_rank=num_tokens_per_rank.cpu().tolist(), + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank.cpu().tolist(), + num_tokens_per_expert=num_tokens_per_expert.cpu().tolist(), + ) + ) + + def reset(self): + self._topk_ids_of_layer[...] = -1 + self._misc_objects.clear() + self._metadata = None + + def collect(self) -> Dict: + num_tokens = len(self._metadata["input_ids"]) + return dict( + **self._metadata, + topk_ids_of_layer=self._topk_ids_of_layer[:, :num_tokens, :].clone().cpu(), + misc_objects=self._misc_objects, + ) + + +class _LayerBasedCpuSinglePassGatherer(_SinglePassGatherer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._objects_of_layer = {} + + def _on_layer_data(self, layer_idx: int, objects: List[int]): + assert 0 <= layer_idx < self._expert_location_metadata.num_layers + if layer_idx in self._objects_of_layer: + self._objects_of_layer[layer_idx] = _list_sum( + self._objects_of_layer[layer_idx], objects + ) + else: + self._objects_of_layer[layer_idx] = objects + + def reset(self): + self._objects_of_layer.clear() + + def _collect_objects(self, pad_len: int) -> torch.Tensor: + data = [ + self._objects_of_layer.get(layer_index) or ([0] * pad_len) + for layer_index in range(self._expert_location_metadata.num_layers) + ] + return torch.tensor(data) + + +def _list_sum(a: List, b: List) -> List: + return [x + y for x, y in zip(a, b, strict=True)] + + +class _LayerBasedGpuSinglePassGatherer(_SinglePassGatherer): + def __init__(self, *args, enable_global_physical_experts: bool, **kwargs): + super().__init__(*args, **kwargs) + if not _is_npu: + device = "cuda" + else: + device = "npu" + self._enable_global_physical_experts = enable_global_physical_experts + self._data = torch.zeros( + ( + self._expert_location_metadata.num_layers, + ( + self._expert_location_metadata.num_physical_experts + if enable_global_physical_experts + else self._expert_location_metadata.num_local_physical_experts + ), + ), + dtype=torch.int, + device=device, + ) + + def reset(self): + self._data[...] = 0 + + def collect(self) -> Dict: + if self._enable_global_physical_experts: + global_physical_count = self._data + else: + # Can optimize if bottleneck + global_physical_count = _convert_local_to_global_physical_count( + self._data, + rank=self._rank, + num_local_physical_experts=self._expert_location_metadata.num_local_physical_experts, + num_physical_experts=self._expert_location_metadata.num_physical_experts, + ) + + return dict(global_physical_count=global_physical_count) + + +class _SelectExpertsSinglePassGatherer(_LayerBasedGpuSinglePassGatherer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, enable_global_physical_experts=True) + + # can optimize (e.g. fuse / compile) + def on_select_experts(self, layer_idx: int, topk_ids: torch.Tensor): + topk_ids = topk_ids.flatten() + mask = topk_ids != -1 + self._data[layer_idx, :].scatter_add_( + dim=0, index=topk_ids.masked_fill(~mask, 0).long(), src=mask.int() + ) + + +class _DeepepNormalSinglePassGatherer(_LayerBasedCpuSinglePassGatherer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if torch.distributed.get_rank() == 0: + logger.info( + "DeepepNormalSinglePassGatherer gathers approximate statistics. " + "If used with small batch size, consider using expert_distribution_recorder_mode=stat." + ) + + def on_deepep_dispatch_normal( + self, + layer_idx: int, + local_physical_count_of_layer: List[int], + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + ): + assert isinstance(local_physical_count_of_layer, list) + self._on_layer_data(layer_idx, local_physical_count_of_layer) + + def collect(self) -> Dict: + local_physical_count = super()._collect_objects( + pad_len=self._expert_location_metadata.num_local_physical_experts + ) + global_physical_count = _convert_local_to_global_physical_count( + local_physical_count, + rank=self._rank, + num_local_physical_experts=self._expert_location_metadata.num_local_physical_experts, + num_physical_experts=self._expert_location_metadata.num_physical_experts, + ) + return dict(global_physical_count=global_physical_count) + + +class _DeepepLowLatencySinglePassGatherer(_LayerBasedGpuSinglePassGatherer): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs, enable_global_physical_experts=False) + + def on_deepep_dispatch_low_latency( + self, layer_idx: int, local_physical_count_of_layer: torch.Tensor + ): + # Most naive implementation, can optimize later + self._data[layer_idx, :] += local_physical_count_of_layer + + +def _convert_local_to_global_physical_count( + local_physical_count: torch.Tensor, + rank: int, + num_local_physical_experts: int, + num_physical_experts: int, +) -> torch.Tensor: + dtype = local_physical_count.dtype + device = local_physical_count.device + num_layers, _ = local_physical_count.shape + + ans = torch.zeros((num_layers, num_physical_experts), dtype=dtype, device=device) + ans[ + :, num_local_physical_experts * rank : num_local_physical_experts * (rank + 1) + ] = local_physical_count + return ans + + +# --------------------------------------- Accumulator ----------------------------------------- + +_SINGLE_PASS_GATHERER_KEY_PRIMARY = "primary" + + +class _Accumulator(ABC): + @staticmethod + def init_new( + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ) -> "_Accumulator": + return _Accumulator.get_class(server_args)( + server_args, expert_location_metadata, rank + ) + + @staticmethod + def get_class(server_args: ServerArgs) -> Type["_Accumulator"]: + return { + "stat": _StatAccumulator, + "stat_approx": _StatAccumulator, + "per_pass": _DetailAccumulator, + "per_token": _DetailAccumulator, + }[server_args.expert_distribution_recorder_mode] + + def __init__( + self, + server_args: ServerArgs, + expert_location_metadata: ExpertLocationMetadata, + rank: int, + ): + self._server_args = server_args + self._expert_location_metadata = expert_location_metadata + self._rank = rank + + def get_single_pass_gatherer_keys(self): + return [_SINGLE_PASS_GATHERER_KEY_PRIMARY] + + def get_single_pass_gatherer_key(self, debug_name: Optional[str]): + return _SINGLE_PASS_GATHERER_KEY_PRIMARY + + def append( + self, + forward_pass_id: int, + gatherer_key: str, + single_pass_data: Dict, + ): + pass + + def reset(self): + pass + + def dump(self, output_mode: _OutputMode): + pass + + +class _UtilizationRateAccumulatorMixin(_Accumulator): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self._enable = self._server_args.enable_expert_distribution_metrics + + if self._enable: + self.window_sizes = [10, 100, 1000] + self._history = _DequeCollection(maxlens=self.window_sizes) + self._rank = torch.distributed.get_rank() + + def append( + self, + forward_pass_id: int, + gatherer_key: str, + single_pass_data: Dict, + ): + super().append(forward_pass_id, gatherer_key, single_pass_data) + if self._enable: + self._append_utilization_rate( + forward_pass_id, single_pass_data["global_physical_count"] + ) + + def reset(self): + super().reset() + if self._enable: + self._history.clear() + + def _append_utilization_rate( + self, forward_pass_id: int, single_pass_global_physical_count: torch.Tensor + ): + gpu_physical_count = compute_gpu_physical_count( + single_pass_global_physical_count, + num_gpu=self._expert_location_metadata.ep_size, + ) + gpu_physical_count = gpu_physical_count.to(self._server_args.device) + torch.distributed.reduce( + gpu_physical_count, dst=0, op=torch.distributed.ReduceOp.SUM + ) + + if self._rank == 0: + utilization_rate_tensor = compute_utilization_rate(gpu_physical_count) + utilization_rate = torch.mean(utilization_rate_tensor).item() + self._history.append(utilization_rate) + + gpu_physical_count_sum = gpu_physical_count.sum().item() + + logger.info( + f"[Expert Balancedness] " + f"forward_pass_id={forward_pass_id} " + f"current_pass_balancedness={utilization_rate:.03f} " + f"{''.join(f'last_{size}_average_balancedness={value:.03f} ' for size, value in self._history.mean().items())} " + f"gpu_physical_count_sum={gpu_physical_count_sum}" + # f"current_pass_per_layer={[round(x, 2) for x in utilization_rate_tensor.cpu().tolist()]}" + ) + + +class _DequeCollection: + def __init__(self, maxlens: List[int]): + self._dequeues = [deque(maxlen=maxlen) for maxlen in maxlens] + + def append(self, value): + for d in self._dequeues: + d.append(value) + + def clear(self): + for d in self._dequeues: + d.clear() + + def mean(self) -> Dict[int, float]: + return {d.maxlen: sum(d) / len(d) for d in self._dequeues} + + +class _DetailAccumulator(_UtilizationRateAccumulatorMixin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._records = [] + + def get_single_pass_gatherer_keys(self): + if False: # TODO `server_args.enable_two_batch_overlap` + return [_SINGLE_PASS_GATHERER_KEY_PRIMARY, "child_a", "child_b"] + return super().get_single_pass_gatherer_keys() + + def get_single_pass_gatherer_key(self, debug_name: Optional[str]): + if False: # TODO `server_args.enable_two_batch_overlap` + return debug_name or _SINGLE_PASS_GATHERER_KEY_PRIMARY + return super().get_single_pass_gatherer_key(debug_name) + + def append( + self, + forward_pass_id: int, + gatherer_key: str, + single_pass_data: Dict, + ): + super().append(forward_pass_id, gatherer_key, single_pass_data) + + def _process_object(obj): + if isinstance(obj, torch.Tensor): + return obj.cpu().clone() + return obj + + single_pass_data_processed = { + k: _process_object(v) for k, v in single_pass_data.items() + } + + self._records.append( + dict( + forward_pass_id=forward_pass_id, + rank=self._rank, + gatherer_key=gatherer_key, + **single_pass_data_processed, + ) + ) + + def reset(self): + super().reset() + self._records.clear() + + def dump(self, output_mode: _OutputMode): + assert output_mode == "file" + output = dict( + records=self._records, + # NOTE: This may change during recording, so here we say it is the "last" one + last_physical_to_logical_map=self._expert_location_metadata.physical_to_logical_map, + ) + _dump_to_file( + f"expert_distribution_recorder_{time.time()}_{self._rank}.pt", output + ) + + +class _StatAccumulator(_UtilizationRateAccumulatorMixin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._global_physical_count_of_buffered_step = _Buffer.init_new( + item_shape=( + self._expert_location_metadata.num_layers, + # Cannot use local_physical_count to support select_experts + self._expert_location_metadata.num_physical_experts, + ), + buffer_size=self._server_args.expert_distribution_recorder_buffer_size, + dtype=torch.int32, + device=self._server_args.device, + ) + self._first_dump = True + + def append( + self, + forward_pass_id: int, + gatherer_key: str, + single_pass_data: Dict, + ): + super().append(forward_pass_id, gatherer_key, single_pass_data) + # Can optimize if overhead here is large + self._global_physical_count_of_buffered_step.append( + single_pass_data["global_physical_count"] + ) + + def reset(self): + super().reset() + self._global_physical_count_of_buffered_step.reset() + + def dump(self, output_mode: _OutputMode): + logical_count_of_buffered_step = _convert_global_physical_count_to_logical_count( + self._global_physical_count_of_buffered_step.get_all(), + num_layers=self._expert_location_metadata.num_layers, + num_logical_experts=self._expert_location_metadata.num_logical_experts, + physical_to_logical_map=self._expert_location_metadata.physical_to_logical_map, + ) + + if self._first_dump: + self._first_dump = False + torch.get_device_module().empty_cache() + + torch.distributed.all_reduce( + logical_count_of_buffered_step, op=torch.distributed.ReduceOp.SUM + ) + + output = dict( + rank=self._rank, + logical_count=logical_count_of_buffered_step, + average_utilization_rate_over_window=self._get_global_average_utilization_rate(), + ) + + if output_mode == "file": + if self._rank == 0: + _dump_to_file(f"expert_distribution_recorder_{time.time()}.pt", output) + elif output_mode == "object": + return output + else: + raise NotImplementedError + + def _get_global_average_utilization_rate(self): + if not self._enable or math.isclose( + self._server_args.eplb_min_rebalancing_utilization_threshold, 1.0 + ): + return None + + if self._rank == 0: + utilization_mean_rates = self._history.mean() + window_index = self.window_sizes[-1] + average_utilization_rate_over_window = ( + utilization_mean_rates[window_index] + if window_index in utilization_mean_rates + else 0 + ) + + avg_rate_tensor = torch.tensor( + [average_utilization_rate_over_window], + dtype=torch.float32, + device="cuda", + ) + else: + avg_rate_tensor = torch.empty(1, dtype=torch.float32, device="cuda") + torch.distributed.broadcast(avg_rate_tensor, src=0) + return avg_rate_tensor.item() + + +def _dump_to_file(name, data): + save_dir = Path(os.environ.get("SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR", "/tmp")) + path_output = save_dir / name + logger.info(f"Write expert distribution to {path_output}") + if not save_dir.exists(): + save_dir.mkdir(parents=True, exist_ok=True) + torch.save(data, str(path_output)) + + +class _Buffer: + @staticmethod + def init_new(item_shape: Tuple, buffer_size: int, dtype, device): + if buffer_size < 0: + return _InfiniteBuffer(item_shape, dtype=dtype, device=device) + else: + return _CircularBuffer(item_shape, buffer_size, dtype=dtype, device=device) + + def append(self, value: torch.Tensor): + raise NotImplementedError + + def get_all(self) -> torch.Tensor: + raise NotImplementedError + + def reset(self): + raise NotImplementedError + + +class _CircularBuffer(_Buffer): + def __init__(self, item_shape: Tuple, buffer_size: int, dtype, device): + self._buffer = torch.zeros( + (buffer_size, *item_shape), dtype=dtype, device=device + ) + self._curr_index = 0 + + def append(self, value: torch.Tensor): + self._buffer[self._curr_index] = value + self._curr_index = (self._curr_index + 1) % len(self._buffer) + + def get_all(self) -> torch.Tensor: + return self._buffer + + def reset(self): + self._buffer[...] = 0 + + +class _InfiniteBuffer(_Buffer): + def __init__(self, item_shape: Tuple, dtype, device): + self._item_shape = item_shape + self._buffer = torch.zeros((128, *item_shape), dtype=dtype, device=device) + self._size = 0 + + def append(self, value: torch.Tensor): + curr_buffer_size = len(self._buffer) + dtype = self._buffer.dtype + device = self._buffer.device + + if self._size == curr_buffer_size: + new_buffer = torch.zeros( + (2 * curr_buffer_size, *self._item_shape), dtype=dtype, device=device + ) + new_buffer[:curr_buffer_size] = self._buffer + self._buffer = new_buffer + + self._buffer[self._size] = value + self._size += 1 + + def get_all(self) -> torch.Tensor: + return self._buffer[: self._size] + + def reset(self): + self._buffer[...] = 0 + self._size = 0 + + +def _convert_global_physical_count_to_logical_count( + # (whatever, num_layers, num_physical_experts) + global_physical_count: torch.Tensor, + num_layers: int, + num_logical_experts: int, + physical_to_logical_map: torch.Tensor, +): + dim_extra, _, _ = global_physical_count.shape + dtype = global_physical_count.dtype + device = global_physical_count.device + logical_count = torch.zeros( + (dim_extra, num_layers, num_logical_experts), dtype=dtype, device=device + ) + logical_count.scatter_add_( + dim=2, + index=physical_to_logical_map.unsqueeze(0) + .expand(dim_extra, -1, -1) + .to(torch.int64), + src=global_physical_count, + ) + return logical_count + + +def compute_gpu_physical_count( + physical_count_of_whatever: torch.Tensor, # (..., num_layer, num_physical_expert) + num_gpu: int, +): + """output: gpu_physical_count_of_batch (..., num_layer, num_gpu)""" + return einops.reduce( + physical_count_of_whatever, + "... num_layer (num_gpu num_expert_per_gpu) -> ... num_layer num_gpu", + "sum", + num_gpu=num_gpu, + ) + + +def compute_utilization_rate( + gpu_physical_count_of_batch: torch.Tensor, # (..., num_layer, num_gpu) +): + """output: utilization_rate (..., num_layer)""" + gpu_physical_count_of_batch = gpu_physical_count_of_batch.float() + max_gpu_physical_count = einops.reduce( + gpu_physical_count_of_batch, + "... num_layer num_gpu -> ... num_layer", + "max", + ) + avg_gpu_physical_count = einops.reduce( + gpu_physical_count_of_batch, + "... num_layer num_gpu -> ... num_layer", + "mean", + ) + return (avg_gpu_physical_count + 1e-5) / (max_gpu_physical_count + 1e-5) diff --git a/python/sglang/srt/eplb/expert_location.py b/python/sglang/srt/eplb/expert_location.py new file mode 100644 index 000000000..ee5f2c7ca --- /dev/null +++ b/python/sglang/srt/eplb/expert_location.py @@ -0,0 +1,468 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import annotations + +import json +import logging +import random +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional + +import torch +import torch.distributed +import torch.nn.functional as F + +from sglang.srt.eplb import eplb_algorithms +from sglang.srt.model_loader import get_model_architecture + +if TYPE_CHECKING: + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + + +@dataclass +class ExpertLocationMetadata: + physical_to_logical_map: torch.Tensor # (layers, num_physical_experts) + physical_to_logical_map_cpu: torch.Tensor + logical_to_all_physical_map: torch.Tensor # (layers, num_logical_experts, X) + logical_to_all_physical_map_cpu: torch.Tensor # CPU copy for performance + logical_to_all_physical_map_num_valid: torch.Tensor # (layers, num_logical_experts) + # (layers, num_logical_experts) + logical_to_rank_dispatch_physical_map: Optional[torch.Tensor] + + # -------------------------------- properties ------------------------------------ + + @property + def num_layers(self) -> int: + return self.physical_to_logical_map.shape[0] + + @property + def num_physical_experts(self) -> int: + return self.physical_to_logical_map.shape[1] + + @property + def num_local_physical_experts(self) -> int: + ans, remainder = divmod(self.num_physical_experts, self.ep_size) + assert remainder == 0 + return ans + + @property + def num_logical_experts(self) -> int: + return self.logical_to_all_physical_map.shape[1] + + @property + def ep_size(self): + # TODO change when EP size != world size + return torch.distributed.get_world_size() + + def __post_init__(self): + num_layers_0, num_physical_experts_0 = self.physical_to_logical_map.shape + num_layers_1, num_logical_experts_0, num_physical_experts_1 = ( + self.logical_to_all_physical_map.shape + ) + num_layers_2, num_logical_experts_1 = ( + self.logical_to_all_physical_map_num_valid.shape + ) + assert num_layers_0 == num_layers_1 == num_layers_2 + assert num_logical_experts_0 == num_logical_experts_1 + assert num_physical_experts_0 == num_physical_experts_1 + + # -------------------------------- construction ------------------------------------ + + @staticmethod + def init_trivial(server_args: ServerArgs, model_config: ModelConfig): + """Trivial location - logical expert i corresponds to physical expert i""" + common = ExpertLocationMetadata._init_common(server_args, model_config) + + if common is None: + return None + + num_physical_experts = common["num_physical_experts"] + model_config_for_expert_location = common["model_config_for_expert_location"] + num_layers = model_config_for_expert_location.num_layers + num_logical_experts = model_config_for_expert_location.num_logical_experts + + physical_to_logical_map = ( + torch.arange(0, num_physical_experts).repeat(num_layers, 1) + % num_logical_experts + ) + + return ExpertLocationMetadata.init_by_mapping( + server_args, + model_config, + physical_to_logical_map=physical_to_logical_map, + ) + + @staticmethod + def init_by_mapping( + server_args: ServerArgs, + model_config: ModelConfig, + physical_to_logical_map, + ): + if not isinstance(physical_to_logical_map, torch.Tensor): + physical_to_logical_map = torch.tensor(physical_to_logical_map) + physical_to_logical_map = physical_to_logical_map.to(server_args.device) + + common = ExpertLocationMetadata._init_common(server_args, model_config) + + if common is None: + return None + + model_config_for_expert_location = common["model_config_for_expert_location"] + logical_to_all_physical_map = _compute_logical_to_all_physical_map( + physical_to_logical_map, + num_logical_experts=model_config_for_expert_location.num_logical_experts, + ) + + return ExpertLocationMetadata._init_raw( + server_args=server_args, + ep_size=common["ep_size"], + physical_to_logical_map=physical_to_logical_map, + logical_to_all_physical_map=logical_to_all_physical_map, + ) + + @staticmethod + def init_by_eplb( + server_args: ServerArgs, model_config: ModelConfig, logical_count: torch.Tensor + ): + if not isinstance(logical_count, torch.Tensor): + logical_count = torch.tensor(logical_count) + if len(logical_count.shape) == 2: + logical_count = logical_count.unsqueeze(0) + logical_count = logical_count.to(server_args.device) + + common = ExpertLocationMetadata._init_common(server_args, model_config) + + if common is None: + return None + + model_config_for_expert_location = common["model_config_for_expert_location"] + num_physical_experts = common["num_physical_experts"] + num_groups = model_config_for_expert_location.num_groups + num_nodes = server_args.nnodes + + physical_to_logical_map, logical_to_all_physical_map, expert_count = ( + eplb_algorithms.rebalance_experts( + tokens_per_expert=logical_count, + num_physical_experts=num_physical_experts, + num_local_physical_experts=num_physical_experts // common["ep_size"], + num_groups=num_groups, + num_nodes=num_nodes, + algorithm=eplb_algorithms.compute_algorithm( + raw_algorithm=server_args.eplb_algorithm, + num_groups=num_groups, + num_nodes=num_nodes, + ), + ) + ) + + return ExpertLocationMetadata._init_raw( + server_args=server_args, + ep_size=common["ep_size"], + physical_to_logical_map=physical_to_logical_map.to(server_args.device), + logical_to_all_physical_map=logical_to_all_physical_map.to( + server_args.device + ), + ) + + @staticmethod + def _init_common(server_args: ServerArgs, model_config: ModelConfig): + model_config_for_expert_location = ( + ModelConfigForExpertLocation.from_model_config(model_config) + ) + + if model_config_for_expert_location is None: + return None + + num_physical_experts = ( + model_config_for_expert_location.num_logical_experts + + server_args.ep_num_redundant_experts + ) + ep_size = server_args.ep_size + assert num_physical_experts % ep_size == 0 + num_local_physical_experts = num_physical_experts // ep_size + + return dict( + model_config_for_expert_location=model_config_for_expert_location, + num_physical_experts=num_physical_experts, + num_local_physical_experts=num_local_physical_experts, + ep_size=ep_size, + ) + + @staticmethod + def _init_raw( + server_args: ServerArgs, + ep_size: int, + physical_to_logical_map: torch.Tensor, + logical_to_all_physical_map: torch.Tensor, + ): + _, num_physical_experts = physical_to_logical_map.shape + + logical_to_all_physical_map_padded = F.pad( + logical_to_all_physical_map, + (0, num_physical_experts - logical_to_all_physical_map.shape[-1]), + value=-1, + ) + + logical_to_all_physical_map_num_valid = torch.count_nonzero( + logical_to_all_physical_map != -1, dim=-1 + ) + + return ExpertLocationMetadata( + physical_to_logical_map=physical_to_logical_map, + physical_to_logical_map_cpu=physical_to_logical_map.cpu(), + logical_to_all_physical_map=logical_to_all_physical_map_padded, + logical_to_all_physical_map_cpu=logical_to_all_physical_map_padded.cpu(), + logical_to_all_physical_map_num_valid=logical_to_all_physical_map_num_valid, + logical_to_rank_dispatch_physical_map=( + compute_logical_to_rank_dispatch_physical_map( + logical_to_all_physical_map=logical_to_all_physical_map, + num_gpus=ep_size, + num_physical_experts=num_physical_experts, + # TODO improve when we have real EP rank + ep_rank=torch.distributed.get_rank() % ep_size, + ) + if server_args.ep_dispatch_algorithm == "static" + else None + ), + ) + + # -------------------------------- mutation ------------------------------------ + + def update( + self, + other: "ExpertLocationMetadata", + update_layer_ids: List[int], + ): + for field in [ + "ep_size", + ]: + assert getattr(self, field) == getattr(other, field) + + for field in [ + "physical_to_logical_map", + "physical_to_logical_map_cpu", + "logical_to_all_physical_map", + "logical_to_all_physical_map_cpu", + "logical_to_all_physical_map_num_valid", + "logical_to_rank_dispatch_physical_map", + ]: + other_field = getattr(other, field) + self_field = getattr(self, field) + assert (other_field is not None) == (self_field is not None) + if self_field is not None: + mask_update = torch.tensor( + [i in update_layer_ids for i in range(self.num_layers)] + ) + mask_update = mask_update.view(*([-1] + [1] * (self_field.dim() - 1))) + mask_update = mask_update.to(self_field.device, non_blocking=True) + self_field[...] = torch.where(mask_update, other_field, self_field) + + # -------------------------------- usage ------------------------------------ + + def logical_to_all_physical( + self, layer_id: int, logical_expert_id: int + ) -> List[int]: + # Use CPU copy to avoid GPU→CPU sync on every call, which is expensive in update weights scenario + return [ + physical_expert_id + for physical_expert_id in self.logical_to_all_physical_map_cpu[ + layer_id, logical_expert_id + ].tolist() + if physical_expert_id != -1 + ] + + +_global_expert_location_metadata: Optional[ExpertLocationMetadata] = None + + +def get_global_expert_location_metadata(): + return _global_expert_location_metadata + + +def set_global_expert_location_metadata(value): + global _global_expert_location_metadata + assert _global_expert_location_metadata is None + _global_expert_location_metadata = value + + +def _compute_logical_to_all_physical_map( + physical_to_logical_map: torch.Tensor, num_logical_experts: int +): + # This is rarely called, so we use for loops for maximum clarity + + num_layers, num_physical_experts = physical_to_logical_map.shape + + logical_to_all_physical_map = [ + [[] for _ in range(num_logical_experts)] for _ in range(num_layers) + ] + for layer_id in range(num_layers): + for physical_expert_id in range(num_physical_experts): + logical_expert_id = physical_to_logical_map[ + layer_id, physical_expert_id + ].item() + logical_to_all_physical_map[layer_id][logical_expert_id].append( + physical_expert_id + ) + + logical_to_all_physical_map = _pad_nested_array( + logical_to_all_physical_map, pad_value=-1 + ) + + return torch.tensor( + logical_to_all_physical_map, device=physical_to_logical_map.device + ) + + +def _pad_nested_array(arr, pad_value): + max_len = max(len(inner) for outer in arr for inner in outer) + padded = [ + [inner + [pad_value] * (max_len - len(inner)) for inner in outer] + for outer in arr + ] + return padded + + +# TODO optimize performance (rewrite and/or run in separate process with overlap) +def compute_logical_to_rank_dispatch_physical_map( + logical_to_all_physical_map: torch.Tensor, + num_gpus: int, + num_physical_experts: int, + ep_rank: int, + seed: int = 42, +): + r = random.Random(seed) + + num_local_physical_experts = num_physical_experts // num_gpus + num_layers, num_logical_experts, _ = logical_to_all_physical_map.shape + dtype = logical_to_all_physical_map.dtype + + logical_to_rank_dispatch_physical_map = torch.full( + size=(num_gpus, num_layers, num_logical_experts), + fill_value=-1, + dtype=dtype, + ) + + for layer_id in range(num_layers): + for logical_expert_id in range(num_logical_experts): + candidate_physical_expert_ids = _logical_to_all_physical_raw( + logical_to_all_physical_map, layer_id, logical_expert_id + ) + output_partial = logical_to_rank_dispatch_physical_map[ + :, layer_id, logical_expert_id + ] + + for gpu_id in range(num_gpus): + same_gpu_physical_expert_ids = [ + physical_expert_id + for physical_expert_id in candidate_physical_expert_ids + if _compute_gpu_id_of_physical_expert( + physical_expert_id, num_local_physical_experts + ) + == gpu_id + ] + if len(same_gpu_physical_expert_ids) > 0: + output_partial[gpu_id] = same_gpu_physical_expert_ids[0] + + num_remain = torch.sum(output_partial == -1).item() + output_partial[output_partial == -1] = torch.tensor( + _fair_choices(candidate_physical_expert_ids, k=num_remain, r=r), + dtype=dtype, + ) + + assert torch.all(logical_to_rank_dispatch_physical_map != -1) + + device = logical_to_all_physical_map.device + return logical_to_rank_dispatch_physical_map[ep_rank, :, :].to(device) + + +def _logical_to_all_physical_raw( + logical_to_all_physical_map, layer_id: int, logical_expert_id: int +) -> List[int]: + return [ + physical_expert_id + for physical_expert_id in logical_to_all_physical_map[ + layer_id, logical_expert_id + ].tolist() + if physical_expert_id != -1 + ] + + +def _compute_gpu_id_of_physical_expert( + physical_expert_id: int, num_local_physical_experts: int +) -> int: + return physical_expert_id // num_local_physical_experts + + +def _fair_choices(arr: List, k: int, r: random.Random) -> List: + quotient, remainder = divmod(k, len(arr)) + ans = arr * quotient + r.sample(arr, k=remainder) + r.shuffle(ans) + return ans + + +@dataclass +class ModelConfigForExpertLocation: + num_layers: int + num_logical_experts: int + num_groups: Optional[int] = None + + @staticmethod + def from_model_config(model_config: ModelConfig): + model_class, _ = get_model_architecture(model_config) + if hasattr(model_class, "get_model_config_for_expert_location"): + return model_class.get_model_config_for_expert_location( + model_config.hf_config + ) + else: + return None + + +def compute_initial_expert_location_metadata( + server_args: ServerArgs, model_config: ModelConfig +) -> Optional[ExpertLocationMetadata]: + data = server_args.init_expert_location + if data == "trivial": + return ExpertLocationMetadata.init_trivial(server_args, model_config) + + # TODO unify with the utils function + if data.endswith(".pt"): + data_dict = torch.load(data, weights_only=True) + elif data.endswith(".json"): + data_dict = json.loads(Path(data).read_text()) + else: + data_dict = json.loads(data) + + if "physical_to_logical_map" in data_dict: + logger.info( + "init_expert_location from init_by_mapping using ServerArgs.init_expert_location" + ) + return ExpertLocationMetadata.init_by_mapping( + server_args, model_config, **data_dict + ) + elif "logical_count" in data_dict: + logger.info( + "init_expert_location from init_by_eplb using ServerArgs.init_expert_location" + ) + return ExpertLocationMetadata.init_by_eplb( + server_args, model_config, logical_count=data_dict["logical_count"] + ) + else: + raise NotImplementedError( + f"Unknown init_expert_location format ({list(data_dict.keys())=})" + ) diff --git a/python/sglang/srt/eplb/expert_location_dispatch.py b/python/sglang/srt/eplb/expert_location_dispatch.py new file mode 100644 index 000000000..624dc3fd1 --- /dev/null +++ b/python/sglang/srt/eplb/expert_location_dispatch.py @@ -0,0 +1,109 @@ +# Copyright 2023-2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from dataclasses import dataclass +from typing import Literal, Optional + +import torch + +from sglang.srt.eplb.expert_location import get_global_expert_location_metadata +from sglang.srt.managers.schedule_batch import global_server_args_dict + + +@dataclass +class ExpertLocationDispatchInfo: + ep_dispatch_algorithm: Literal["static", "random"] + # (num_logical_experts,) + partial_logical_to_rank_dispatch_physical_map: Optional[torch.Tensor] + # (num_logical_experts, X) + partial_logical_to_all_physical_map: torch.Tensor + # (num_logical_experts,) + partial_logical_to_all_physical_map_num_valid: torch.Tensor + num_physical_experts: int + + @classmethod + def init_new(cls, layer_id: int): + ep_dispatch_algorithm = global_server_args_dict["ep_dispatch_algorithm"] + expert_location_metadata = get_global_expert_location_metadata() + assert expert_location_metadata is not None + + if ep_dispatch_algorithm is None: + return None + + return cls( + ep_dispatch_algorithm=ep_dispatch_algorithm, + partial_logical_to_rank_dispatch_physical_map=( + expert_location_metadata.logical_to_rank_dispatch_physical_map[ + layer_id, : + ] + if expert_location_metadata.logical_to_rank_dispatch_physical_map + is not None + else None + ), + partial_logical_to_all_physical_map=expert_location_metadata.logical_to_all_physical_map[ + layer_id, : + ], + partial_logical_to_all_physical_map_num_valid=expert_location_metadata.logical_to_all_physical_map_num_valid[ + layer_id, : + ], + num_physical_experts=expert_location_metadata.num_physical_experts, + ) + + +def transform_select_experts_inputs( + router_logits: torch.Tensor, + correction_bias: Optional[torch.Tensor], + info: Optional[ExpertLocationDispatchInfo], +): + if (info is not None) and (info.ep_dispatch_algorithm == "fake"): + router_logits.uniform_(5, 10) + if correction_bias is not None: + correction_bias = torch.zeros_like(correction_bias) + return router_logits, correction_bias + + +def topk_ids_logical_to_physical( + topk_ids: torch.Tensor, info: Optional[ExpertLocationDispatchInfo] +) -> torch.Tensor: + if info is None: + return topk_ids + + if info.ep_dispatch_algorithm == "static": + return _topk_ids_logical_to_physical_static(topk_ids, info) + if info.ep_dispatch_algorithm in ["dynamic", "fake"]: + return _topk_ids_logical_to_physical_dynamic(topk_ids, info) + raise NotImplementedError(f"Unknown algorithm {info.ep_dispatch_algorithm}") + + +def _topk_ids_logical_to_physical_static( + topk_ids: torch.Tensor, info: Optional[ExpertLocationDispatchInfo] +) -> torch.Tensor: + return info.partial_logical_to_rank_dispatch_physical_map[topk_ids] + + +def _topk_ids_logical_to_physical_dynamic( + topk_ids: torch.Tensor, info: Optional[ExpertLocationDispatchInfo] +) -> torch.Tensor: + topk_ids_original_shape = topk_ids.shape + device = topk_ids.device + topk_ids = topk_ids.flatten() + + chosen_dispatch_index = ( + torch.randint(0, 65536, topk_ids.shape, dtype=torch.int32, device=device) + % info.partial_logical_to_all_physical_map_num_valid[topk_ids] + ) + topk_ids = info.partial_logical_to_all_physical_map[topk_ids, chosen_dispatch_index] + + topk_ids = topk_ids.view(topk_ids_original_shape) + return topk_ids diff --git a/python/sglang/srt/eplb/expert_location_updater.py b/python/sglang/srt/eplb/expert_location_updater.py new file mode 100644 index 000000000..772e65f18 --- /dev/null +++ b/python/sglang/srt/eplb/expert_location_updater.py @@ -0,0 +1,575 @@ +# Copyright 2023-2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import logging +from collections import defaultdict +from typing import Dict, List, Optional, Tuple + +import einops +import torch +import torch.distributed +from torch.distributed import P2POp + +from sglang.srt.eplb.expert_location import ( + ExpertLocationMetadata, + get_global_expert_location_metadata, +) +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.utils import get_bool_env_var + +logger = logging.getLogger(__name__) + + +_LOG_INPUT = get_bool_env_var("SGLANG_EXPERT_LOCATION_UPDATER_LOG_INPUT") + + +class ExpertLocationUpdater: + def __init__(self): + self._first_execution = True + + def update( + self, + routed_experts_weights_of_layer: Dict[int, List[torch.Tensor]], + new_expert_location_metadata: ExpertLocationMetadata, + update_layer_ids: List[int], + nnodes: int, + rank: int, + ): + if self._first_execution: + self._first_execution = False + torch.get_device_module().empty_cache() + + old_expert_location_metadata = get_global_expert_location_metadata() + assert old_expert_location_metadata is not None + + _update_expert_weights( + routed_experts_weights_of_layer=routed_experts_weights_of_layer, + old_expert_location_metadata=old_expert_location_metadata, + new_expert_location_metadata=new_expert_location_metadata, + update_layer_ids=update_layer_ids, + nnodes=nnodes, + rank=rank, + ) + old_expert_location_metadata.update( + new_expert_location_metadata, + update_layer_ids=update_layer_ids, + ) + + +def _update_expert_weights(**kwargs): + if get_bool_env_var("SGLANG_EXPERT_LOCATION_UPDATER_CANARY"): + return _update_expert_weights_with_canary(**kwargs) + else: + return _update_expert_weights_raw(**kwargs) + + +# can add watchdog as well +def _update_expert_weights_with_canary( + routed_experts_weights_of_layer: Dict[int, List[torch.Tensor]], + old_expert_location_metadata: ExpertLocationMetadata, + new_expert_location_metadata: ExpertLocationMetadata, + update_layer_ids: List[int], + nnodes: int, + rank: int, +): + num_local_physical_experts = old_expert_location_metadata.num_local_physical_experts + + def _get_canary_value(meta: ExpertLocationMetadata, layer_id: int): + return meta.physical_to_logical_map_cpu[ + layer_id, + num_local_physical_experts * rank : num_local_physical_experts * (rank + 1), + ] + + routed_experts_weights_of_layer = { + k: [x for x in v] for k, v in routed_experts_weights_of_layer.items() + } + for layer_id in update_layer_ids: + canary_tensor = ( + _get_canary_value(old_expert_location_metadata, layer_id) + .clone() + .to(device=global_server_args_dict["device"], non_blocking=True) + ) + routed_experts_weights_of_layer[layer_id].append(canary_tensor) + + _update_expert_weights_raw( + routed_experts_weights_of_layer=routed_experts_weights_of_layer, + old_expert_location_metadata=old_expert_location_metadata, + new_expert_location_metadata=new_expert_location_metadata, + update_layer_ids=update_layer_ids, + nnodes=nnodes, + rank=rank, + ) + + for layer_id in update_layer_ids: + # can optimize speed if needed + expect_value = _get_canary_value(new_expert_location_metadata, layer_id) + actual_value = routed_experts_weights_of_layer[layer_id][-1].cpu() + assert torch.all(expect_value == actual_value), ( + f"{expect_value=} {actual_value=} {layer_id=} " + f"{old_expert_location_metadata.physical_to_logical_map_cpu.tolist()=} " + f"{new_expert_location_metadata.physical_to_logical_map_cpu.tolist()=} " + ) + + +def _update_expert_weights_raw( + routed_experts_weights_of_layer: Dict[int, List[torch.Tensor]], + old_expert_location_metadata: ExpertLocationMetadata, + new_expert_location_metadata: ExpertLocationMetadata, + update_layer_ids: List[int], + nnodes: int, + rank: int, +): + log_metrics = get_bool_env_var("SGLANG_EXPERT_LOCATION_UPDATER_LOG_METRICS") + + temp_buffers = create_temp_buffers( + routed_experts_weights_of_layer[update_layer_ids[0]] + ) + + world_size = torch.distributed.get_world_size() + num_local_physical_experts = old_expert_location_metadata.num_local_physical_experts + num_gpu_per_node = world_size // nnodes + + for layer_id in update_layer_ids: + update_expert_weights_single_layer( + routed_experts_weights=routed_experts_weights_of_layer[layer_id], + temp_buffers=temp_buffers, + old_physical_to_logical_map=old_expert_location_metadata.physical_to_logical_map_cpu[ + layer_id + ].tolist(), + new_physical_to_logical_map=new_expert_location_metadata.physical_to_logical_map_cpu[ + layer_id + ].tolist(), + num_local_physical_experts=num_local_physical_experts, + num_gpu_per_node=num_gpu_per_node, + rank=rank, + world_size=world_size, + log_metrics=log_metrics, + ) + + +def create_temp_buffers(sample_tensors): + return [torch.empty_like(tensor) for tensor in sample_tensors] + + +def update_expert_weights_single_layer( + routed_experts_weights: List[torch.Tensor], + temp_buffers: List[torch.Tensor], + old_physical_to_logical_map: List[int], # (num_physical_Experts,) + new_physical_to_logical_map: List[int], # (num_physical_Experts,) + num_local_physical_experts: int, + num_gpu_per_node: int, + rank: int, + world_size: Optional[int] = None, + debug: bool = False, + log_metrics: bool = False, +): + assert all( + tensor.shape[0] == num_local_physical_experts + for tensor in routed_experts_weights + ), f"{num_local_physical_experts=} {[x.shape for x in routed_experts_weights]=}" + assert isinstance(old_physical_to_logical_map, list) + assert isinstance(new_physical_to_logical_map, list) + + if _LOG_INPUT: + logger.info( + "update_expert_weights_single_layer " + f"{[x.shape for x in routed_experts_weights]=} " + f"{[x.shape for x in temp_buffers]=} " + f"{old_physical_to_logical_map=} " + f"{new_physical_to_logical_map=} " + f"{num_local_physical_experts=} " + f"{num_gpu_per_node=} " + f"{rank=} " + f"{world_size=} " + ) + + output_logs = [] if debug else None + + num_physical_experts = len(old_physical_to_logical_map) + num_tensors = len(routed_experts_weights) + + self_node_id = rank // num_gpu_per_node + + local_expert_location_range = ( + rank * num_local_physical_experts, + (rank + 1) * num_local_physical_experts, + ) + + def _entrypoint(): + # List[Tuple[logical_expert_id, List[P2POp]]] + p2p_op_infos: List[Tuple[int, List[P2POp]]] = [] + # List[Tuple[temp_buffers_expert_location, routed_experts_weights_expert_location]] + buffer2weight_copy_infos: List[Tuple[int, int]] = [] + + _handle_recv(buffer2weight_copy_infos, p2p_op_infos) + _create_isend_ops(p2p_op_infos) + _execute_p2p_ops(p2p_op_infos) + _execute_buffer2weight_copies(buffer2weight_copy_infos) + + if log_metrics: + _log_p2p_op_metrics( + p2p_op_infos, + world_size=world_size, + num_gpu_per_node=num_gpu_per_node, + self_node_id=self_node_id, + ) + + if debug: + output_logs.append(f"{p2p_op_infos=}") + output_logs.append(f"{buffer2weight_copy_infos=}") + + def _handle_recv(buffer2weight_copy_infos, p2p_op_infos): + for dst_expert_location in range(*local_expert_location_range): + _handle_recv_of_dst_expert_location( + dst_expert_location, buffer2weight_copy_infos, p2p_op_infos + ) + + def _handle_recv_of_dst_expert_location( + dst_expert_location: int, buffer2weight_copy_infos, p2p_op_infos + ): + logical_expert_id = new_physical_to_logical_map[dst_expert_location] + + # case 1: unchanged + if old_physical_to_logical_map[dst_expert_location] == logical_expert_id: + if debug: + output_logs.append( + f"handle_recv_of_dst_expert_location {dst_expert_location=} case=unchanged" + ) + return + + # case 2: same-gpu + for src_expert_location in range(*local_expert_location_range): + if old_physical_to_logical_map[src_expert_location] == logical_expert_id: + for i in range(num_tensors): + _get_tensor(temp_buffers, i, dst_expert_location).copy_( + _get_tensor(routed_experts_weights, i, src_expert_location) + ) + buffer2weight_copy_infos.append( + (dst_expert_location, dst_expert_location) + ) + if debug: + output_logs.append( + f"handle_recv_of_dst_expert_location {dst_expert_location=} case=same-gpu {src_expert_location=}" + ) + return + + # case 3: free-rider + for src_expert_location in range( + rank * num_local_physical_experts, dst_expert_location + ): + if new_physical_to_logical_map[src_expert_location] == logical_expert_id: + buffer2weight_copy_infos.append( + (src_expert_location, dst_expert_location) + ) + if debug: + output_logs.append( + f"handle_recv_of_dst_expert_location {dst_expert_location=} case=free-rider {src_expert_location=}" + ) + return + + same_node_mapping, cross_node_mapping, need_comm_self_node_dst_ranks = ( + _compute_comm_info(logical_expert_id=logical_expert_id) + ) + + # case 4: same-node + if rank in need_comm_self_node_dst_ranks: + chosen_src_rank = same_node_mapping.chunk_value_from_element_value( + element_value=rank + ) + _create_p2p_recv_and_buffer2weight_copy( + buffer2weight_copy_infos, + p2p_op_infos, + src_rank=chosen_src_rank, + logical_expert_id=logical_expert_id, + dst_expert_location=dst_expert_location, + ) + if debug: + output_logs.append( + f"handle_recv_of_dst_expert_location {dst_expert_location=} case=same-node {chosen_src_rank=}" + ) + return + + # case 5: cross-node + # Future work: can optimize when there are multiple ranks in the same dst node that uses the same logical expert + chosen_src_rank = cross_node_mapping.chunk_value_from_element_value( + element_value=rank + ) + _create_p2p_recv_and_buffer2weight_copy( + buffer2weight_copy_infos, + p2p_op_infos, + src_rank=chosen_src_rank, + logical_expert_id=logical_expert_id, + dst_expert_location=dst_expert_location, + ) + if debug: + output_logs.append( + f"handle_recv_of_dst_expert_location {dst_expert_location=} case=cross-node {chosen_src_rank=}" + ) + return + + def _create_p2p_recv_and_buffer2weight_copy( + buffer2weight_copy_infos, + p2p_op_infos, + *, + logical_expert_id: int, + src_rank: int, + dst_expert_location: int, + ): + p2p_op_infos.append( + ( + logical_expert_id, + [ + P2POp( + op=torch.distributed.irecv, + tensor=_get_tensor(temp_buffers, i, dst_expert_location), + peer=src_rank, + ) + for i in range(num_tensors) + ], + ) + ) + buffer2weight_copy_infos.append((dst_expert_location, dst_expert_location)) + + def _create_isend_ops(p2p_op_infos): + handled_logical_expert_ids = set() + for src_expert_location in range(*local_expert_location_range): + logical_expert_id = old_physical_to_logical_map[src_expert_location] + + if logical_expert_id in handled_logical_expert_ids: + continue + handled_logical_expert_ids.add(logical_expert_id) + + _create_isend_ops_of_logical_expert_id( + logical_expert_id, src_expert_location, p2p_op_infos + ) + + def _create_isend_ops_of_logical_expert_id( + logical_expert_id, src_expert_location, p2p_op_infos + ): + same_node_mapping, cross_node_mapping, need_comm_self_node_dst_ranks = ( + _compute_comm_info(logical_expert_id=logical_expert_id) + ) + + same_node_dst_ranks = same_node_mapping.element_values_from_chunk_value( + chunk_value=rank + ) + cross_node_dst_ranks = cross_node_mapping.element_values_from_chunk_value( + chunk_value=rank + ) + all_dst_ranks = same_node_dst_ranks + cross_node_dst_ranks + + if debug: + output_logs.append( + f"create_isend_ops_of_logical_expert_id {logical_expert_id=} {src_expert_location=} {same_node_dst_ranks=} {cross_node_dst_ranks=}" + ) + + p2p_op_infos.append( + ( + logical_expert_id, + [ + P2POp( + op=torch.distributed.isend, + tensor=_get_tensor( + routed_experts_weights, i, src_expert_location + ), + peer=dst_rank, + ) + for dst_rank in all_dst_ranks + for i in range(num_tensors) + ], + ) + ) + + def _compute_comm_info(logical_expert_id: int): + all_src_ranks = _deduplicate_ordered( + [ + x // num_local_physical_experts + for x in range(num_physical_experts) + if old_physical_to_logical_map[x] == logical_expert_id + ] + ) + all_src_nodes = [x // num_gpu_per_node for x in all_src_ranks] + self_node_src_ranks = [ + x for x in all_src_ranks if x // num_gpu_per_node == self_node_id + ] + + need_comm_dst_ranks = _deduplicate_ordered( + [ + x // num_local_physical_experts + for x in range(num_physical_experts) + if new_physical_to_logical_map[x] == logical_expert_id + and x // num_local_physical_experts not in all_src_ranks + ] + ) + need_comm_self_node_dst_ranks = ( + [x for x in need_comm_dst_ranks if x // num_gpu_per_node == self_node_id] + if len(self_node_src_ranks) > 0 + else [] + ) + need_comm_cross_node_dst_ranks = [ + x + for x in need_comm_dst_ranks + if (x // num_gpu_per_node) not in all_src_nodes + ] + + same_node_mapping = _ChunkUtils( + chunk_values=self_node_src_ranks, + element_values=need_comm_self_node_dst_ranks, + ) + + cross_node_mapping = _ChunkUtils( + chunk_values=all_src_ranks, + element_values=need_comm_cross_node_dst_ranks, + ) + + return same_node_mapping, cross_node_mapping, need_comm_self_node_dst_ranks + + def _execute_p2p_ops(p2p_op_infos): + sorted_infos = sorted(p2p_op_infos, key=lambda info: info[0]) + p2p_ops = [op for _, ops in sorted_infos for op in ops] + if len(p2p_ops) == 0: + return + + reqs = torch.distributed.batch_isend_irecv(p2p_ops) + for req in reqs: + req.wait() + + def _execute_buffer2weight_copies(buffer2weight_copy_infos): + for ( + temp_buffers_expert_location, + routed_experts_weights_expert_location, + ) in buffer2weight_copy_infos: + for i in range(num_tensors): + _get_tensor( + routed_experts_weights, i, routed_experts_weights_expert_location + ).copy_(_get_tensor(temp_buffers, i, temp_buffers_expert_location)) + + def _get_tensor(tensors, tensor_index: int, expert_location: int) -> torch.Tensor: + return tensors[tensor_index][_get_local_expert_location(expert_location)] + + def _get_local_expert_location(expert_location: int) -> int: + assert ( + local_expert_location_range[0] + <= expert_location + < local_expert_location_range[1] + ) + return expert_location % num_local_physical_experts + + _entrypoint() + + return output_logs + + +class _ChunkUtils: + def __init__(self, *, chunk_values: List, element_values: List): + self.chunk_values = chunk_values + self.element_values = element_values + + def chunk_value_from_element_value(self, element_value): + chunk_index = self._chunk_index_from_element_index( + num_elements=len(self.element_values), + num_chunks=len(self.chunk_values), + element_index=self.element_values.index(element_value), + ) + return self.chunk_values[chunk_index] + + def element_values_from_chunk_value(self, chunk_value) -> List: + if len(self.element_values) == 0: + return [] + element_slice = self._element_slice_from_chunk_index( + num_elements=len(self.element_values), + num_chunks=len(self.chunk_values), + chunk_index=self.chunk_values.index(chunk_value), + ) + return self.element_values[element_slice] + + @staticmethod + def _chunk_index_from_element_index( + num_elements: int, num_chunks: int, element_index: int + ) -> int: + short_chunk_size, num_long_chunks = divmod(num_elements, num_chunks) + num_elements_for_long_chunks = num_long_chunks * (short_chunk_size + 1) + if element_index < num_elements_for_long_chunks: + return element_index // (short_chunk_size + 1) + else: + return ( + num_long_chunks + + (element_index - num_elements_for_long_chunks) // short_chunk_size + ) + + @staticmethod + def _element_slice_from_chunk_index( + num_elements: int, num_chunks: int, chunk_index: int + ) -> slice: + short_chunk_size, num_long_chunks = divmod(num_elements, num_chunks) + start = chunk_index * short_chunk_size + min(chunk_index, num_long_chunks) + end = start + short_chunk_size + int(chunk_index < num_long_chunks) + return slice(start, end) + + +def _deduplicate_ordered(arr: List[int]): + output = [] + for item in arr: + if len(output) == 0 or item != output[-1]: + output.append(item) + return output + + +def _log_p2p_op_metrics( + p2p_op_infos: List[Tuple[int, List[P2POp]]], + num_gpu_per_node: int, + world_size: int, + self_node_id: int, +): + text = "" + all_ops = [op for _, ops in p2p_op_infos for op in ops] + + for direction, ops in _group_by(all_ops, _get_direction_from_op).items(): + nbytes_of_gpu = [0] * world_size + for op in ops: + nbytes_of_gpu[op.peer] += op.tensor.nbytes + nbytes_of_gpu = torch.tensor(nbytes_of_gpu, dtype=torch.int64) + + nbytes_of_node = einops.reduce( + nbytes_of_gpu, + "(num_nodes num_gpu_per_node) -> num_nodes", + num_gpu_per_node=num_gpu_per_node, + reduction="sum", + ) + + nbytes_curr_node = nbytes_of_node[self_node_id] + nbytes_cross_node = torch.sum(nbytes_of_node) - nbytes_curr_node + + text += ( + f"{direction}_nbytes_of_gpu={nbytes_of_gpu.tolist()} " + f"{direction}_nbytes_of_node={nbytes_of_node.tolist()} " + f"{direction}_nbytes_curr_node={nbytes_curr_node.item()} " + f"{direction}_nbytes_cross_node={nbytes_cross_node.item()} " + ) + + logger.info(f"[ExpertLocationUpdater] {text}") + + +def _get_direction_from_op(op: P2POp): + if op.op == torch.distributed.isend: + return "isend" + if op.op == torch.distributed.irecv: + return "irecv" + raise NotImplementedError + + +def _group_by(items, keyfunc): + ans = defaultdict(list) + for item in items: + ans[keyfunc(item)].append(item) + return dict(ans) diff --git a/python/sglang/srt/function_call/base_format_detector.py b/python/sglang/srt/function_call/base_format_detector.py new file mode 100644 index 000000000..39bb92f5f --- /dev/null +++ b/python/sglang/srt/function_call/base_format_detector.py @@ -0,0 +1,367 @@ +import json +import logging +from abc import ABC, abstractmethod +from typing import Any, Dict, List + +from partial_json_parser.core.exceptions import MalformedJSON +from partial_json_parser.core.options import Allow + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.utils import ( + _find_common_prefix, + _is_complete_json, + _partial_json_loads, +) + +logger = logging.getLogger(__name__) + + +class BaseFormatDetector(ABC): + """Base class providing two sets of interfaces: one-time and streaming incremental.""" + + def __init__(self): + # Streaming state management + # Buffer for accumulating incomplete patterns that arrive across multiple streaming chunks + self._buffer = "" + # Stores complete tool call info (name and arguments) for each tool being parsed. + # Used by serving layer for completion handling when streaming ends. + # Format: [{"name": str, "arguments": dict}, ...] + self.prev_tool_call_arr: List[Dict] = [] + # Index of currently streaming tool call. Starts at -1 (no active tool), + # increments as each tool completes. Tracks which tool's arguments are streaming. + self.current_tool_id: int = -1 + # Flag for whether current tool's name has been sent to client. + # Tool names sent first with empty parameters, then arguments stream incrementally. + self.current_tool_name_sent: bool = False + # Tracks raw JSON string content streamed to client for each tool's arguments. + # Critical for serving layer to calculate remaining content when streaming ends. + # Each index corresponds to a tool_id. Example: ['{"location": "San Francisco"', '{"temp": 72'] + self.streamed_args_for_tool: List[str] = [] + + # Token configuration (override in subclasses) + self.bot_token = "" + self.eot_token = "" + self.tool_call_separator = ", " + + def _get_tool_indices(self, tools: List[Tool]) -> Dict[str, int]: + """ + Get a mapping of tool names to their indices in the tools list. + + This utility method creates a dictionary mapping function names to their + indices in the tools list, which is commonly needed for tool validation + and ToolCallItem creation. + + Args: + tools: List of available tools + + Returns: + Dictionary mapping tool names to their indices + """ + return { + tool.function.name: i for i, tool in enumerate(tools) if tool.function.name + } + + def parse_base_json(self, action: Any, tools: List[Tool]) -> List[ToolCallItem]: + tool_indices = self._get_tool_indices(tools) + if not isinstance(action, list): + action = [action] + + results = [] + for act in action: + name = act.get("name") + if name and name in tool_indices: + results.append( + ToolCallItem( + tool_index=-1, # Caller should update this based on the actual tools array called + name=name, + parameters=json.dumps( + act.get("parameters") or act.get("arguments", {}), + ensure_ascii=False, + ), + ) + ) + else: + logger.warning(f"Model attempted to call undefined function: {name}") + + return results + + @abstractmethod + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + Parses the text in one go. Returns success=True if the format matches, otherwise False. + Note that leftover_text here represents "content that this parser will not consume further". + """ + action = json.loads(text) + return StreamingParseResult(calls=self.parse_base_json(action, tools)) + + def _ends_with_partial_token(self, buffer: str, bot_token: str) -> int: + """ + Check if buffer ends with a partial bot_token. + Return the length of the partial bot_token. + + For some format, the bot_token is not a token in model's vocabulary, such as + `[TOOL_CALLS] [` in Mistral. + """ + for i in range(1, min(len(buffer) + 1, len(bot_token))): + if bot_token.startswith(buffer[-i:]): + return i + return 0 + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing with tool validation. + + This base implementation works best with formats where: + 1. bot_token is followed immediately by JSON (e.g., bot_token + JSON_array) + 2. JSON can be parsed incrementally using partial_json_loads + 3. Multiple tool calls are separated by "; " or ", " + + Examples of incompatible formats (need custom implementation, may reuse some logic from this class): + - Each tool call is wrapped in a separate block: See Qwen25Detector + - Multiple separate blocks: [TOOL_CALLS] [...] \n [TOOL_CALLS] [...] + - Tool call is Pythonic style + + For incompatible formats, detectors should override this method with custom logic. + """ + # Append new text to buffer + self._buffer += new_text + current_text = self._buffer + + # The current_text has tool_call if it is the start of a new tool call sequence + # or it is the start of a new tool call after a tool call separator, when there is a previous tool call + if not ( + self.has_tool_call(current_text) + or ( + self.current_tool_id > 0 + and current_text.startswith(self.tool_call_separator) + ) + ): + # Only clear buffer if we're sure no tool call is starting + if not self._ends_with_partial_token(self._buffer, self.bot_token): + normal_text = self._buffer + self._buffer = "" + if self.eot_token in normal_text: + normal_text = normal_text.replace(self.eot_token, "") + return StreamingParseResult(normal_text=normal_text) + else: + # Might be partial bot_token, keep buffering + return StreamingParseResult() + + # Build tool indices if not already built + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR + + try: + try: + if current_text.startswith(self.bot_token): + start_idx = len(self.bot_token) + elif self.current_tool_id > 0 and current_text.startswith( + self.tool_call_separator + self.bot_token + ): + start_idx = len(self.tool_call_separator + self.bot_token) + elif self.current_tool_id > 0 and current_text.startswith( + self.tool_call_separator + ): + start_idx = len(self.tool_call_separator) + else: + start_idx = 0 + + if start_idx >= len(current_text): + return StreamingParseResult() + + (obj, end_idx) = _partial_json_loads(current_text[start_idx:], flags) + + is_current_complete = _is_complete_json( + current_text[start_idx : start_idx + end_idx] + ) + + # Validate tool name if present + if "name" in obj and obj["name"] not in self._tool_indices: + # Invalid tool name - reset state + self._buffer = "" + self.current_tool_id = -1 + self.current_tool_name_sent = False + if self.streamed_args_for_tool: + self.streamed_args_for_tool.pop() + return StreamingParseResult() + + # Handle parameters/arguments consistency + # NOTE: we assume here that the obj is always partial of a single tool call + if "parameters" in obj: + assert ( + "arguments" not in obj + ), "model generated both parameters and arguments" + obj["arguments"] = obj["parameters"] + + current_tool_call = obj + + except MalformedJSON: + return StreamingParseResult() + + if not current_tool_call: + return StreamingParseResult() + + # Case 1: Handle tool name streaming + # This happens when we encounter a tool but haven't sent its name yet + if not self.current_tool_name_sent: + function_name = current_tool_call.get("name") + + if function_name and function_name in self._tool_indices: + # If this is a new tool (current_tool_id was -1), initialize it + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.streamed_args_for_tool.append("") + # If this is a subsequent tool, ensure streamed_args_for_tool is large enough + elif self.current_tool_id >= len(self.streamed_args_for_tool): + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Send the tool name with empty parameters + res = StreamingParseResult( + calls=[ + ToolCallItem( + tool_index=self.current_tool_id, + name=function_name, + parameters="", + ) + ], + ) + self.current_tool_name_sent = True + else: + res = StreamingParseResult() + + # Case 2: Handle streaming arguments + # This happens when we've already sent the tool name and now need to stream arguments incrementally + else: + cur_arguments = current_tool_call.get("arguments") + res = StreamingParseResult() + + if cur_arguments: + # Calculate how much of the arguments we've already streamed + sent = len(self.streamed_args_for_tool[self.current_tool_id]) + cur_args_json = json.dumps(cur_arguments) + prev_arguments = None + if self.current_tool_id < len(self.prev_tool_call_arr): + prev_arguments = self.prev_tool_call_arr[ + self.current_tool_id + ].get("arguments") + + argument_diff = None + + # If the current tool's JSON is complete, send all remaining arguments + if is_current_complete: + argument_diff = cur_args_json[sent:] + completing_tool_id = ( + self.current_tool_id + ) # Save the ID of the tool that's completing + + # Only remove the processed portion, keep unprocessed content + self._buffer = current_text[start_idx + end_idx :] + + if self.current_tool_id < len(self.prev_tool_call_arr): + self.prev_tool_call_arr[self.current_tool_id].clear() + self.current_tool_name_sent = False + self.streamed_args_for_tool[self.current_tool_id] = "" + self.current_tool_id += 1 + + # If the tool is still being parsed, send incremental changes + elif prev_arguments: + prev_args_json = json.dumps(prev_arguments) + if cur_args_json != prev_args_json: + prefix = _find_common_prefix(prev_args_json, cur_args_json) + argument_diff = prefix[sent:] + + # Send the argument diff if there's something new + if argument_diff is not None: + # Use the correct tool_index: completing_tool_id for completed tools, current_tool_id for ongoing + tool_index_to_use = ( + completing_tool_id + if is_current_complete + else self.current_tool_id + ) + res = StreamingParseResult( + calls=[ + ToolCallItem( + tool_index=tool_index_to_use, + parameters=argument_diff, + ) + ], + ) + if not is_current_complete: + self.streamed_args_for_tool[ + self.current_tool_id + ] += argument_diff + + # Update prev_tool_call_arr with current state + if self.current_tool_id >= 0: + # Ensure prev_tool_call_arr is large enough + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + self.prev_tool_call_arr[self.current_tool_id] = current_tool_call + + return res + + except Exception as e: + logger.error(f"Error in parse_streaming_increment: {e}") + return StreamingParseResult() + + @abstractmethod + def has_tool_call(self, text: str) -> bool: + """ + Check if the given text contains function call markers specific to this format. + """ + raise NotImplementedError() + + def supports_structural_tag(self) -> bool: + """Return True if this detector supports structural tag format.""" + return True + + @abstractmethod + def structure_info(self) -> _GetInfoFunc: + """ + Return a function that creates StructureInfo for constrained generation. + + The returned function takes a tool name and returns a StructureInfo object + containing the begin/end patterns and trigger tokens needed for constrained + generation of function calls in this format. + + Returns: + A function that takes a tool name (str) and returns StructureInfo + """ + raise NotImplementedError() + + @abstractmethod + def build_ebnf(self, tools: List[Tool]) -> str: + """ + Build an EBNF grammar for constrained generation of function calls. + + This method generates an Extended Backus-Naur Form (EBNF) grammar that + constrains the model's output to valid function calls in this format. + The grammar should include all available tools and their parameter schemas. + + Args: + tools: List of available tools/functions that can be called + + Returns: + A string containing the EBNF grammar for this function call format + + The EBNF grammar should: + - Define the overall structure of function calls in this format + - Include all tool names from the provided tools list + - Define valid JSON structures for function arguments + - Handle multiple function calls if the format supports them + + Note: + Most implementations use EBNFComposer.build_ebnf() utility with + format-specific parameters rather than writing EBNF from scratch. + """ + raise NotImplementedError() diff --git a/python/sglang/srt/function_call/core_types.py b/python/sglang/srt/function_call/core_types.py new file mode 100644 index 000000000..1ea87df79 --- /dev/null +++ b/python/sglang/srt/function_call/core_types.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass +from typing import Callable, List, Optional + +from pydantic import BaseModel + + +class ToolCallItem(BaseModel): + """Simple encapsulation of the parsed ToolCall result for easier usage in streaming contexts.""" + + tool_index: int + name: Optional[str] = None + parameters: str # JSON string + + +class StreamingParseResult(BaseModel): + """Result of streaming incremental parsing.""" + + normal_text: str = "" + calls: List[ToolCallItem] = [] + + +@dataclass +class StructureInfo: + begin: str + end: str + trigger: str + + +""" +Helper alias of function +Usually it is a function that takes a name string and returns a StructureInfo object, +which can be used to construct a structural_tag object +""" +_GetInfoFunc = Callable[[str], StructureInfo] diff --git a/python/sglang/srt/function_call/deepseekv31_detector.py b/python/sglang/srt/function_call/deepseekv31_detector.py new file mode 100644 index 000000000..2045d8daa --- /dev/null +++ b/python/sglang/srt/function_call/deepseekv31_detector.py @@ -0,0 +1,222 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer +from sglang.srt.function_call.utils import _is_complete_json + +logger = logging.getLogger(__name__) + + +class DeepSeekV31Detector(BaseFormatDetector): + """ + Detector for DeepSeek V3 model function call format. + + The DeepSeek V3 format uses special Unicode tokens to delimit function calls + with JSON code blocks for arguments. + + Format Structure: + ``` + <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>{function_name}<｜tool▁sep｜>{json_arguments}<｜tool▁calls▁end｜><｜end▁of▁sentence｜> + ``` + Examples: + ``` + <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_current_weather<｜tool▁sep｜>{"location": "Tokyo"}<｜tool▁call▁end｜><｜tool▁call▁begin｜>get_current_weather<｜tool▁sep｜>{"location": "Paris"}<｜tool▁call▁end｜><｜tool▁calls▁end｜><｜end▁of▁sentence｜> + ``` + + Key Components: + - Tool Calls Section: Wrapped between `<｜tool▁calls▁begin｜>` and `<｜tool▁calls▁end｜>` + - Individual Tool Call: Wrapped between `<｜tool▁call▁begin｜>` and `<｜tool▁call▁end｜>` + - Function Declaration: `<｜tool▁call▁begin｜>{function_name}<｜tool▁sep｜>` + - Arguments: JSON code block between `<｜tool▁sep｜>` and `<｜tool▁call▁end｜>` + - Supports multiple tool calls + + Reference: https://www.modelscope.cn/models/deepseek-ai/DeepSeek-V3.1 + """ + + def __init__(self): + super().__init__() + self.bot_token = "<｜tool▁calls▁begin｜>" + self.eot_token = "<｜tool▁calls▁end｜>" + self.func_call_regex = r"<｜tool▁call▁begin｜>.*?<｜tool▁call▁end｜>" + self.func_detail_regex = ( + r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)<｜tool▁call▁end｜>" + ) + self._last_arguments = "" + self.current_tool_id = -1 + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a deepseek format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + match_result_list = re.findall(self.func_call_regex, text, re.DOTALL) + calls = [] + try: + for match_result in match_result_list: + # Get function name + func_detail = re.search(self.func_detail_regex, match_result, re.DOTALL) + func_name = func_detail.group(1) + func_args = func_detail.group(2) + func_args = json.loads(func_args) + # construct match_result for parse_base_json + match_result = {"name": func_name, "parameters": func_args} + calls.extend(self.parse_base_json(match_result, tools)) + return StreamingParseResult(normal_text=normal_text, calls=calls) + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # return the normal text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing tool calls for DeepSeekV3 format. + """ + self._buffer += new_text + current_text = self._buffer + + # Check if we have a tool call (either the start token or individual tool call) + has_tool_call = ( + self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text + ) + + if not has_tool_call: + self._buffer = "" + for e_token in [self.eot_token, "<｜tool▁call▁end｜>"]: + if e_token in new_text: + new_text = new_text.replace(e_token, "") + return StreamingParseResult(normal_text=new_text) + + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + calls: list[ToolCallItem] = [] + try: + partial_match = re.search( + pattern=r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)<｜tool▁call▁end｜>", + string=current_text, + flags=re.DOTALL, + ) + if partial_match: + func_name = partial_match.group(1).strip() + func_args_raw = partial_match.group(2).strip() + + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Ensure we have enough entries in our tracking arrays + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + if not self.current_tool_name_sent: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=func_name, + parameters="", + ) + ) + self.current_tool_name_sent = True + # Store the tool call info for serving layer completions endpoint + self.prev_tool_call_arr[self.current_tool_id] = { + "name": func_name, + "arguments": {}, + } + else: + argument_diff = ( + func_args_raw[len(self._last_arguments) :] + if func_args_raw.startswith(self._last_arguments) + else func_args_raw + ) + + if argument_diff: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=argument_diff, + ) + ) + self._last_arguments += argument_diff + self.streamed_args_for_tool[ + self.current_tool_id + ] += argument_diff + + if _is_complete_json(func_args_raw): + # Update the stored arguments + try: + parsed_args = json.loads(func_args_raw) + self.prev_tool_call_arr[self.current_tool_id][ + "arguments" + ] = parsed_args + except json.JSONDecodeError: + pass + + # Find the end of the current tool call and remove only that part from buffer + tool_call_end_pattern = ( + r"<｜tool▁call▁begin｜>.*?<｜tool▁call▁end｜>" + ) + match = re.search( + tool_call_end_pattern, current_text, re.DOTALL + ) + if match: + # Remove the completed tool call from buffer, keep any remaining content + self._buffer = current_text[match.end() :] + else: + self._buffer = "" + + result = StreamingParseResult(normal_text="", calls=calls) + self.current_tool_id += 1 + self._last_arguments = "" + self.current_tool_name_sent = False + return result + + return StreamingParseResult(normal_text="", calls=calls) + + except Exception as e: + logger.error(f"Error in parse_streaming_increment: {e}") + return StreamingParseResult(normal_text=current_text) + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin="<｜tool▁call▁begin｜>" + name + "<｜tool▁sep｜>", + end="<｜tool▁call▁end｜>", + trigger="<｜tool▁call▁begin｜>" + name + "<｜tool▁sep｜>", + ) + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + sequence_start_token=self.bot_token, + sequence_end_token=self.eot_token, + tool_call_separator="", + call_rule_fmt='"<｜tool▁call▁begin｜>{name}<｜tool▁sep｜>{arguments_rule}<｜tool▁call▁end｜>"', + function_format="json", + ) diff --git a/python/sglang/srt/function_call/deepseekv3_detector.py b/python/sglang/srt/function_call/deepseekv3_detector.py new file mode 100644 index 000000000..33c4dfc44 --- /dev/null +++ b/python/sglang/srt/function_call/deepseekv3_detector.py @@ -0,0 +1,220 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer +from sglang.srt.function_call.utils import _is_complete_json + +logger = logging.getLogger(__name__) + + +class DeepSeekV3Detector(BaseFormatDetector): + """ + Detector for DeepSeek V3 model function call format. + + The DeepSeek V3 format uses special Unicode tokens to delimit function calls + with JSON code blocks for arguments. + + Format Structure: + ``` + <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>{function_name}\n```json\n{json_arguments}\n```<｜tool▁calls▁end｜><｜end▁of▁sentence｜> + ``` + Examples: + ``` + <｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_weather\n```json\n{"location": "Tokyo"}\n```<｜tool▁call▁end｜>\n<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_current_weather\n```json\n{"location": "Paris"}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜><｜end▁of▁sentence｜> + ``` + + Key Components: + - Tool Calls Section: Wrapped between `<｜tool▁calls▁begin｜>` and `<｜tool▁calls▁end｜>` + - Individual Tool Call: Wrapped between `<｜tool▁call▁begin｜>` and `<｜tool▁call▁end｜>` + - Function Declaration: `function<｜tool▁sep｜>{function_name}` + - Arguments: JSON code block between ````json` and ```` + - Supports multiple tool calls + + Reference: https://huggingface.co/deepseek-ai/DeepSeek-V3-0324?chat_template=default + """ + + def __init__(self): + super().__init__() + self.bot_token = "<｜tool▁calls▁begin｜>" + self.eot_token = "<｜tool▁calls▁end｜>" + self.func_call_regex = r"<｜tool▁call▁begin｜>.*?<｜tool▁call▁end｜>" + self.func_detail_regex = r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)\n```json\n(.*)\n```<｜tool▁call▁end｜>" + self._last_arguments = "" + self.current_tool_id = -1 + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a deepseek format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + match_result_list = re.findall(self.func_call_regex, text, re.DOTALL) + calls = [] + try: + for match_result in match_result_list: + # Get function name + func_detail = re.search(self.func_detail_regex, match_result, re.DOTALL) + func_name = func_detail.group(2) + func_args = func_detail.group(3) + func_args = json.loads(func_args) + # construct match_result for parse_base_json + match_result = {"name": func_name, "parameters": func_args} + calls.extend(self.parse_base_json(match_result, tools)) + return StreamingParseResult(normal_text=normal_text, calls=calls) + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # return the normal text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing tool calls for DeepSeekV3 format. + """ + self._buffer += new_text + current_text = self._buffer + + # Check if we have a tool call (either the start token or individual tool call) + has_tool_call = ( + self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text + ) + + if not has_tool_call: + self._buffer = "" + for e_token in [self.eot_token, "```", "<｜tool▁call▁end｜>"]: + if e_token in new_text: + new_text = new_text.replace(e_token, "") + return StreamingParseResult(normal_text=new_text) + + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + calls: list[ToolCallItem] = [] + try: + partial_match = re.search( + pattern=r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)\n```json\n(.*)\n```.*", + string=current_text, + flags=re.DOTALL, + ) + if partial_match: + func_name = partial_match.group(2).strip() + func_args_raw = partial_match.group(3).strip() + + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Ensure we have enough entries in our tracking arrays + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + if not self.current_tool_name_sent: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=func_name, + parameters="", + ) + ) + self.current_tool_name_sent = True + # Store the tool call info for serving layer completions endpoint + self.prev_tool_call_arr[self.current_tool_id] = { + "name": func_name, + "arguments": {}, + } + else: + argument_diff = ( + func_args_raw[len(self._last_arguments) :] + if func_args_raw.startswith(self._last_arguments) + else func_args_raw + ) + + if argument_diff: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=argument_diff, + ) + ) + self._last_arguments += argument_diff + self.streamed_args_for_tool[ + self.current_tool_id + ] += argument_diff + + if _is_complete_json(func_args_raw): + # Update the stored arguments + try: + parsed_args = json.loads(func_args_raw) + self.prev_tool_call_arr[self.current_tool_id][ + "arguments" + ] = parsed_args + except json.JSONDecodeError: + pass + + # Find the end of the current tool call and remove only that part from buffer + tool_call_end_pattern = ( + r"<｜tool▁call▁begin｜>.*?<｜tool▁call▁end｜>" + ) + match = re.search( + tool_call_end_pattern, current_text, re.DOTALL + ) + if match: + # Remove the completed tool call from buffer, keep any remaining content + self._buffer = current_text[match.end() :] + else: + self._buffer = "" + + result = StreamingParseResult(normal_text="", calls=calls) + self.current_tool_id += 1 + self._last_arguments = "" + self.current_tool_name_sent = False + return result + + return StreamingParseResult(normal_text="", calls=calls) + + except Exception as e: + logger.error(f"Error in parse_streaming_increment: {e}") + return StreamingParseResult(normal_text=current_text) + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin=">" + name + "\n```json\n", + end="\n```<", + trigger=">" + name + "\n```json\n", + ) + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + sequence_start_token=self.bot_token, + sequence_end_token=self.eot_token, + tool_call_separator="", + call_rule_fmt='"<｜tool▁call▁begin｜>function<｜tool▁sep｜>{name}\\n```json\\n"{arguments_rule}"\\n```<｜tool▁call▁end｜>"', + function_format="json", + ) diff --git a/python/sglang/srt/function_call/ebnf_composer.py b/python/sglang/srt/function_call/ebnf_composer.py new file mode 100644 index 000000000..21b313982 --- /dev/null +++ b/python/sglang/srt/function_call/ebnf_composer.py @@ -0,0 +1,344 @@ +from typing import Any, Dict, Literal, Optional + + +class EBNFComposer: + # Adapted from https://xgrammar.mlc.ai/docs/how_to/ebnf_guided_generation.html#try-out-via-hf-transformers + # Shared primitive grammar rules used across all formats + BASE_PRIMITIVE_GRAMMAR = r""" + basic_string ::= (([\"] basic_string_1 [\"])) + basic_string_1 ::= "" | [^"\\\x00-\x1F] basic_string_1 | "\\" escape basic_string_1 + escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9]{4} + basic_integer ::= ("0" | "-"? [1-9] [0-9]*) ".0"? + basic_number ::= ("0" | "-"? [1-9] [0-9]*) ("." [0-9]+)? ([eE] [+-]? [0-9]+)? + basic_array ::= "[" ("" | ws basic_any (ws "," ws basic_any)*) ws "]" + basic_object ::= "{" ("" | ws basic_string ws ":" ws basic_any ( ws "," ws basic_string ws ":" ws basic_any)*) ws "}" + ws ::= [ \n\t]* + """ + + # Format-specific extensions + json_grammar_ebnf_str = ( + r""" + json ::= basic_array | basic_object + basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object + basic_boolean ::= "true" | "false" + basic_null ::= "null" + """ + + BASE_PRIMITIVE_GRAMMAR + ) + + pythonic_grammar_ebnf_str = ( + r""" + pythonic ::= basic_number | basic_string | basic_array | "True" | "False" | "None" + basic_any ::= basic_number | basic_string | basic_array | basic_object + basic_boolean ::= "True" | "False" + basic_null ::= "None" + """ + + BASE_PRIMITIVE_GRAMMAR + ) + + xml_grammar_ebnf_str = ( + r""" + xml ::= xml_element | xml_text + xml_element ::= basic_string | basic_number | basic_boolean | basic_null | basic_array | basic_object + xml_text ::= [^<>]* + basic_any ::= basic_number | basic_string | basic_boolean | basic_null | basic_array | basic_object + basic_boolean ::= "true" | "false" + basic_null ::= "null" + """ + + BASE_PRIMITIVE_GRAMMAR + ) + + CALL_RULE_MAP = { + "pythonic": 'call_{name} ::= "{name}" "(" {arguments_rule} ")"', + "json": 'call_{name} ::= "{{" ws "\\"name\\"" ws ":" ws "\\"{name}\\"" ws "," ws "\\"arguments\\"" ws ":" ws {arguments_rule} ws "}}"', + "xml": 'call_{name} ::= "\\n" {arguments_rule} "\\n"', + } + + ARGUMENTS_RULE_MAP = { + "pythonic": "{arg_rules}", + "json": '"{{" ws {arg_rules} ws "}}"', + "xml": "{arg_rules}", + } + + KEY_VALUE_RULE_MAP = { + "pythonic": '"{key}" "=" {valrule}', + "json": '"\\"{key}\\"" ws ":" ws {valrule}', + "xml": '"\\n" {valrule} "\\n"', + } + + # Base type mapping - most types are the same across formats + BASE_TYPE_MAPPING = { + "string": "basic_string", + "number": "basic_number", + "integer": "basic_number", + "boolean": "basic_boolean", + "null": "basic_null", + "array": "basic_array", + "object": "basic_object", + } + + # Format-specific overrides for types that differ + FORMAT_TYPE_OVERRIDES = { + "pythonic": { + "boolean": '"True" | "False"', + "null": '"None"', + }, + "xml": { + "string": "xml_text", + }, + } + + @staticmethod + def get_value_rule( + prop: dict, function_format: Literal["pythonic", "json", "xml"] = "json" + ) -> str: + if "enum" in prop: + return EBNFComposer._handle_enum(prop, function_format) + + if "type" in prop: + return EBNFComposer._handle_type(prop, function_format) + + return function_format + + @staticmethod + def _handle_enum(prop: dict, function_format: str) -> str: + """Handle enum properties by formatting each value according to type and format.""" + enum_values = prop["enum"] + prop_type = prop.get("type", "string") + + def format_enum_val(v: Any) -> str: + if prop_type == "boolean": + if function_format == "json" or function_format == "xml": + return "true" if v else "false" + elif function_format == "pythonic": + return "True" if v else "False" + else: + return str(v) # fallback + + if prop_type == "string": + if function_format == "xml": + return f'"{v}"' + else: # json or pythonic + return f'"\\"{v}\\""' # escape quote-wrapped string + + # All other types (number, integer, etc.) + return str(v) + + formatted_values = [format_enum_val(v) for v in enum_values] + enum_rule = " | ".join(formatted_values) + return f"({enum_rule})" if len(formatted_values) > 1 else enum_rule + + @staticmethod + def get_type_mapping(function_format: str) -> Dict[str, str]: + """Get the complete type mapping for a given format.""" + mapping = EBNFComposer.BASE_TYPE_MAPPING.copy() + overrides = EBNFComposer.FORMAT_TYPE_OVERRIDES.get(function_format, {}) + mapping.update({k: v for k, v in overrides.items() if v is not None}) + return mapping + + @staticmethod + def _handle_type(prop: dict, function_format: str) -> str: + """Handle type properties using the appropriate type mapping.""" + prop_type = prop["type"] + type_mapping = EBNFComposer.get_type_mapping(function_format) + + if isinstance(prop_type, list): + type_rules = [ + type_mapping.get(single_type, function_format) + for single_type in prop_type + ] + return " | ".join(type_rules) if type_rules else function_format + + return type_mapping.get(prop_type, function_format) + + @staticmethod + def build_ebnf( + tools, + function_format: Literal["pythonic", "json", "xml"] = "json", + # Parameters for wrapping the entire sequence of tool calls + sequence_start_token: Optional[str] = None, + sequence_end_token: Optional[str] = None, + # Parameters for wrapping individual tool calls + individual_call_start_token: Optional[str] = None, + individual_call_end_token: Optional[str] = None, + # Parameter for separating multiple tool calls + tool_call_separator: Optional[str] = None, + call_rule_fmt: Optional[str] = None, + key_value_rule_fmt: Optional[str] = None, + key_value_separator: str = 'ws "," ws', + ): + """ + Generalized EBNF builder for all detectors. + Args: + tools: List of Tool objects to generate EBNF grammar for + function_format: The format of function calls, either "pythonic" or "json" + sequence_start_token: Token that wraps the entire sequence of tool calls (start) + sequence_end_token: Token that wraps the entire sequence of tool calls (end) + individual_call_start_token: Token that wraps each individual tool call (start) + individual_call_end_token: Token that wraps each individual tool call (end) + tool_call_separator: The separator between multiple tool calls + call_rule_fmt: Optional custom format string for call_{name} rule. It should define each function call's format, with + the placeholders {name} for the function name and {arguments_rule} for the arguments rule. If None, a default + format based on function_format will be used. + key_value_rule_fmt: Optional custom format string for key-value pairs. It should define how each parameter is formatted, + with placeholders {key} for the parameter name and {valrule} for the value rule. If None, a default format + based on function_format will be used. + key_value_separator: Raw EBNF fragment inserted between key-value pairs. + This string is used verbatim (not auto-quoted). Pass: + - Quoted terminals when you need a literal token (e.g. '","' or '"\\n"'). + - Raw/non-terminals when you need grammar tokens (e.g. 'ws "," ws'). + """ + # ================================================================= + # Step 1: Determine the root tool calls rule + # ================================================================= + # Handle a single function call + if individual_call_start_token and individual_call_end_token: + function_call_unit = f'"{individual_call_start_token}" function_call "{individual_call_end_token}"' + else: + function_call_unit = "function_call" + + # Handle multiple function calls with separators + if tool_call_separator is not None: + base_pattern = f'{function_call_unit} ( "{tool_call_separator}" {function_call_unit} )*' + else: + # Assume only support single function call + base_pattern = function_call_unit + + # Apply sequence-level wrapping if needed + if sequence_start_token and sequence_end_token: + root_rule = ( + f'"{sequence_start_token}" {base_pattern} "{sequence_end_token}"' + ) + else: + root_rule = base_pattern + + # ================================================================= + # Step 2: Build the header rules + # ================================================================= + ebnf_lines = [ + f"root ::= {root_rule}", + "function_call ::= " + + " | ".join([f"call_{tool.function.name}" for tool in tools]), + ] + + # ================================================================= + # Step 3: Set up formatting templates + # ================================================================= + call_template = ( + f"call_{{name}} ::= {call_rule_fmt}" + if call_rule_fmt + else EBNFComposer.CALL_RULE_MAP[function_format] + ) + args_template = EBNFComposer.ARGUMENTS_RULE_MAP[function_format] + key_value_template = ( + key_value_rule_fmt + if key_value_rule_fmt + else EBNFComposer.KEY_VALUE_RULE_MAP[function_format] + ) + + # ================================================================= + # Step 4: Build rules for each tool + # ================================================================= + for tool in tools: + tool_name = tool.function.name + params = tool.function.parameters or {} + properties = params.get("properties", {}) + required_props = set(params.get("required", [])) + + # The generated pattern ensures: + # 1. Required properties appear first, joined by commas + # 2. Optional properties are wrapped with comma included: ( "," ( "prop" : value )? )? + # 3. For multiple optional properties, we allow flexible ordering: + # - Each optional can be skipped entirely + # - They can appear in any combination + # + # Example patterns generated: + # - One required, one optional: + # "{" "location" ":" string ( "," ( "unit" ":" enum ) )? "}" + # Allows: {"location": "Paris"} or {"location": "Paris", "unit": "celsius"} + # + # - Multiple optional properties with flexible ordering: + # "{" "req" ":" string ( "," ( "opt1" ":" value ( "," "opt2" ":" value )? | "opt2" ":" value ) )? "}" + # Allows: {"req": "x"}, {"req": "x", "opt1": "y"}, {"req": "x", "opt2": "z"}, + # {"req": "x", "opt1": "y", "opt2": "z"} + # + # - All optional properties with flexible ordering: + # "{" ( "opt1" ":" value ( "," "opt2" ":" value )? | "opt2" ":" value )? "}" + # Allows: {}, {"opt1": "x"}, {"opt2": "y"}, {"opt1": "x", "opt2": "y"} + + prop_kv_pairs = {} + ordered_props = list(properties.keys()) + + for prop_name, prop_schema in properties.items(): + value_rule = EBNFComposer.get_value_rule(prop_schema, function_format) + # Create key=value pair + pair = key_value_template.format(key=prop_name, valrule=value_rule) + prop_kv_pairs[prop_name] = pair + + # Separate into required and optional while preserving order + required = [p for p in ordered_props if p in required_props] + optional = [p for p in ordered_props if p not in required_props] + + # Build the combined rule + rule_parts = [] + + # Add required properties joined by commas + if required: + rule_parts.append( + f" {key_value_separator} ".join(prop_kv_pairs[k] for k in required) + ) + + # Add optional properties with flexible ordering + if optional: + # Build alternatives where any optional property can appear first + opt_alternatives = [] + for i in range(len(optional)): + # Build pattern for optional[i] appearing first + opt_parts = [] + for j in range(i, len(optional)): + if j == i: + opt_parts.append(prop_kv_pairs[optional[j]]) + else: + opt_parts.append( + f" ( {key_value_separator} {prop_kv_pairs[optional[j]]} )?" + ) + opt_alternatives.append("".join(opt_parts)) + + # Wrap with appropriate comma handling based on whether we have required properties + if required: + # Required properties exist, so optional group needs outer comma + rule_parts.append(f" ( {key_value_separator} ( ") + rule_parts.append(" | ".join(opt_alternatives)) + rule_parts.append(" ) )?") + else: + # All properties are optional + rule_parts.append("( ") + rule_parts.append(" | ".join(opt_alternatives)) + rule_parts.append(" )?") + + combined_args = "".join(rule_parts) + arguments_rule = args_template.format(arg_rules=combined_args) + arguments_rule = arguments_rule or '""' + + # Add the function call rule and its arguments rule + ebnf_lines.append( + call_template.format( + name=tool_name, arguments_rule=f"arguments_{tool_name}" + ) + ) + ebnf_lines.append(f"arguments_{tool_name} ::= {arguments_rule}") + + # ================================================================= + # Step 5: Add base grammar rules + # ================================================================= + grammar_dict = { + "pythonic": EBNFComposer.pythonic_grammar_ebnf_str, + "json": EBNFComposer.json_grammar_ebnf_str, + "xml": EBNFComposer.xml_grammar_ebnf_str, + } + base_grammar = grammar_dict.get( + function_format, EBNFComposer.json_grammar_ebnf_str + ) + ebnf_lines.append(base_grammar) + + return "\n".join(ebnf_lines) diff --git a/python/sglang/srt/function_call/function_call_parser.py b/python/sglang/srt/function_call/function_call_parser.py new file mode 100644 index 000000000..18fe488e4 --- /dev/null +++ b/python/sglang/srt/function_call/function_call_parser.py @@ -0,0 +1,214 @@ +import logging +from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Type, Union + +from sglang.srt.entrypoints.openai.protocol import ( + StructuralTagResponseFormat, + StructuresResponseFormat, + Tool, + ToolChoice, +) +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ToolCallItem +from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector +from sglang.srt.function_call.deepseekv31_detector import DeepSeekV31Detector +from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector +from sglang.srt.function_call.gpt_oss_detector import GptOssDetector +from sglang.srt.function_call.kimik2_detector import KimiK2Detector +from sglang.srt.function_call.llama32_detector import Llama32Detector +from sglang.srt.function_call.mistral_detector import MistralDetector +from sglang.srt.function_call.pythonic_detector import PythonicDetector +from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector +from sglang.srt.function_call.qwen25_detector import Qwen25Detector +from sglang.srt.function_call.step3_detector import Step3Detector + +logger = logging.getLogger(__name__) + + +class FunctionCallParser: + """ + Parser for function/tool calls in model outputs. + + This class handles both streaming and non-streaming parsing of function calls using a detector. + In streaming scenarios, each time new_text is received, it calls detector.parse_streaming_increment + and returns the resulting normal_text and calls to the upper layer (or SSE). + """ + + ToolCallParserEnum: Dict[str, Type[BaseFormatDetector]] = { + "llama3": Llama32Detector, + "qwen25": Qwen25Detector, + "mistral": MistralDetector, + "deepseekv3": DeepSeekV3Detector, + "deepseekv31": DeepSeekV31Detector, + "pythonic": PythonicDetector, + "kimi_k2": KimiK2Detector, + "qwen3_coder": Qwen3CoderDetector, + "glm45": Glm4MoeDetector, + "step3": Step3Detector, + "gpt-oss": GptOssDetector, + } + + def __init__(self, tools: List[Tool], tool_call_parser: str): + detector: Type[BaseFormatDetector] = None + detector_class = self.ToolCallParserEnum.get(tool_call_parser) + if detector_class: + detector = detector_class() + else: + raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}") + + self.detector = detector + self.tools = tools + + def has_tool_call(self, text: str) -> bool: + """ + Check if the given text contains a tool call in the format supported by this parser. + This delegates to the detector's implementation. + + Args: + text: The text to check for tool calls + + Returns: + True if the text contains a tool call, False otherwise + """ + return self.detector.has_tool_call(text) + + def parse_non_stream(self, full_text: str) -> Tuple[str, list[ToolCallItem]]: + """ + One-time parsing of the full text to extract tool calls. + + Args: + full_text: The complete text to parse + + Returns: + A tuple containing: + - The remaining text after parsing that was not consumed by the detector (can be treated as normal text) + - A list of tool calls parsed from the text + """ + parsed_result = self.detector.detect_and_parse(full_text, self.tools) + tool_call_list = parsed_result.calls + if tool_call_list: + return parsed_result.normal_text, tool_call_list + else: + return full_text, [] + + def parse_stream_chunk(self, chunk_text: str) -> Tuple[str, list[ToolCallItem]]: + """ + Streaming incremental parsing of chunks of text as they arrive. + + Args: + chunk_text: The new chunk of text to parse + + Returns: + A tuple containing: + - The normal text that should be displayed to the user + - A list of tool calls parsed from the chunk + """ + final_normal_text = "" + final_calls = [] + + sp_result = self.detector.parse_streaming_increment(chunk_text, self.tools) + if sp_result.normal_text: + final_normal_text = sp_result.normal_text + if sp_result.calls: + final_calls.extend(sp_result.calls) + final_normal_text = sp_result.normal_text + + return final_normal_text, final_calls + + def get_structure_tag(self) -> StructuralTagResponseFormat: + """ + Generate a structural tag response format for all available tools. + + This creates the necessary structural tags that guide the model's output format. + """ + tool_structures: List[StructuresResponseFormat] = list() + tool_trigger_set: Set[str] = set() + + get_structure_info = self.detector.structure_info() + for tool in self.tools: + function = tool.function + name = function.name + assert name is not None + info = get_structure_info(name) + + # accept all if not strict, otherwise only accept the schema + schema = function.parameters if function.strict else {} + + tool_structures.append( + StructuresResponseFormat( + begin=info.begin, + schema=schema, # type: ignore + end=info.end, + ) + ) + tool_trigger_set.add(info.trigger) + + return StructuralTagResponseFormat( + type="structural_tag", + structures=tool_structures, + triggers=list(tool_trigger_set), + ) + + def get_structure_constraint( + self, tool_choice: Union[ToolChoice, Literal["auto", "required"]] + ) -> Optional[Tuple[str, Any]]: + """ + Returns the appropriate structure constraint for tool calls based on the tool_choice. + The constraint is used to guide the model's output format. + + Args: + tool_choice: The tool choice setting from the request + + Returns: + A tuple of (constraint_type, constraint_value) to be added to sampling parameters, + or None if no constraint applies. + """ + # NOTE: structural_tag only supports JSON-compatible content between the begin and end. + # It cannot parse or validate function call Pythonic or XML-ish syntax. + if ( + self.detector.supports_structural_tag() + and tool_choice == "auto" + and any(tool.function.strict for tool in self.tools) + ): + strict_tag = self.get_structure_tag() + return ("structural_tag", strict_tag) + elif tool_choice == "required" or isinstance(tool_choice, ToolChoice): + ebnf = self.get_ebnf(tool_choice) + return ("ebnf", ebnf) if ebnf is not None else None + + def get_ebnf( + self, tool_choice: Union[ToolChoice, Literal["required"]] + ) -> Optional[str]: + """ + Get the EBNF grammar for the specified tool choice. + + Args: + tool_choice: The tool choice specification + + Returns: + EBNF grammar string, or None if no valid tools found + + Note: + If a specific function is requested but not found in available tools, + logs a warning and falls back to using all available tools for backward compatibility. + """ + filtered_tools = [] + if isinstance(tool_choice, ToolChoice): + fn_name = tool_choice.function.name + filtered_tools = [t for t in self.tools if t.function.name == fn_name] + + # Check if the requested function exists in available tools + if not filtered_tools: + available_functions = [t.function.name for t in self.tools] + logger.warning( + f"Function '{fn_name}' not found in available tools. " + f"Available functions: {available_functions}. " + f"Skipping tool choice." + ) + + # TODO: Return a 400 error instead of warning when adapter supports proper error handling + # For now, fall back to return None + return None + else: + filtered_tools = self.tools + + return self.detector.build_ebnf(filtered_tools) diff --git a/python/sglang/srt/function_call/glm4_moe_detector.py b/python/sglang/srt/function_call/glm4_moe_detector.py new file mode 100644 index 000000000..6e89fe0a1 --- /dev/null +++ b/python/sglang/srt/function_call/glm4_moe_detector.py @@ -0,0 +1,164 @@ +import ast +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +def get_argument_type(func_name: str, arg_key: str, defined_tools: list): + name2tool = {tool.function.name: tool for tool in defined_tools} + if func_name not in name2tool: + return None + tool = name2tool[func_name] + if arg_key not in tool.function.parameters["properties"]: + return None + return tool.function.parameters["properties"][arg_key].get("type", None) + + +def parse_arguments(json_value): + try: + try: + parsed_value = json.loads(json_value) + except: + parsed_value = ast.literal_eval(json_value) + return parsed_value, True + except: + return json_value, False + + +class Glm4MoeDetector(BaseFormatDetector): + """ + Detector for GLM-4.5 models. + Assumes function call format: + get_weather\ncity\n北京\ndate\n2024-06-27\n\nget_weather\ncity\n上海\ndate\n2024-06-27\n + """ + + def __init__(self): + super().__init__() + self.bot_token = "" + self.eot_token = "" + self.func_call_regex = r".*?" + self.func_detail_regex = r"([^\n]*)\n(.*)" + self.func_arg_regex = r"(.*?)\s*(.*?)" + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a glm-4.5 format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + match_result_list = re.findall(self.func_call_regex, text, re.DOTALL) + calls = [] + try: + for match_result in match_result_list: + # Get function name + func_detail = re.search(self.func_detail_regex, match_result, re.DOTALL) + func_name = func_detail.group(1) + func_args = func_detail.group(2) + pairs = re.findall( + r"(.*?)\s*(.*?)", + func_args, + re.DOTALL, + ) + arguments = {} + for arg_key, arg_value in pairs: + arg_key = arg_key.strip() + arg_value = arg_value.strip() + arg_type = get_argument_type(func_name, arg_key, tools) + if arg_type != "string": + arg_value, is_good_json = parse_arguments(arg_value) + arguments[arg_key] = arg_value + # construct match_result for parse_base_json + match_result = {"name": func_name, "parameters": arguments} + calls.extend(self.parse_base_json(match_result, tools)) + return StreamingParseResult(normal_text=normal_text, calls=calls) + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # return the normal text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing tool calls for GLM-4.5 format. + """ + self._buffer += new_text + current_text = self._buffer + + start = current_text.find(self.bot_token) + if start == -1: + self._buffer = "" + if self.current_tool_id > 0: + current_text = "" + return StreamingParseResult(normal_text=current_text) + # find ensures we find the first self.eot_token so there will be at most one tool_call in current_text[:end+len(self.eot_token) + end = current_text.find(self.eot_token) + if end != -1: + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + # Ensure we have enough entries in our tracking arrays + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + result = self.detect_and_parse( + current_text[: end + len(self.eot_token)], tools=tools + ) + if result.calls: + self.prev_tool_call_arr[self.current_tool_id] = { + "name": result.calls[0].name, + "arguments": json.loads(result.calls[0].parameters), + } + self.streamed_args_for_tool[self.current_tool_id] = result.calls[ + 0 + ].parameters + result.calls[0].tool_index = self.current_tool_id + self.current_tool_id += 1 + self._buffer = current_text[end + len(self.eot_token) :] + return result + normal_text = current_text[:start] + self._buffer = current_text[start:] + return StreamingParseResult(normal_text=normal_text) + + def supports_structural_tag(self) -> bool: + return False + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError() + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + individual_call_start_token=self.bot_token, + individual_call_end_token=self.eot_token, + tool_call_separator="\\n", + function_format="xml", + call_rule_fmt='"{name}" "\\n" ( {arguments_rule} "\\n" )?', + key_value_rule_fmt='"{key}" "\\n" "" {valrule} ""', + key_value_separator='"\\n"', + ) diff --git a/python/sglang/srt/function_call/gpt_oss_detector.py b/python/sglang/srt/function_call/gpt_oss_detector.py new file mode 100644 index 000000000..7fe0a7dc8 --- /dev/null +++ b/python/sglang/srt/function_call/gpt_oss_detector.py @@ -0,0 +1,219 @@ +import json +import logging +import re +from typing import List, Optional + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.parser.harmony_parser import HarmonyParser + +logger = logging.getLogger(__name__) + + +class GptOssDetector(BaseFormatDetector): + """ + Detector for T4-style function calls using HarmonyParser. + + Handles tool calls in the format: + <|channel|>commentary to={namespace.function}<|constrain|>json<|message|>{args}<|call|> + """ + + def __init__(self): + super().__init__() + self.harmony_parser = HarmonyParser() + self.bot_token = "<|start|>assistant<|channel|>commentary" + self.eot_token = "<|call|>" + + # Pattern to extract function name and JSON from tool_call event content + self.tool_extract_pattern = re.compile( + r"to=([a-zA-Z_][a-zA-Z0-9_.]*)\s*<\|constrain\|>json<\|message\|>(.*?)(?:<\|call\|>|$)", + re.DOTALL, + ) + + def has_tool_call(self, text: str) -> bool: + """Check if text contains TypeScript-style function call markers.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """Parse TypeScript-style function calls from complete text.""" + if not self.has_tool_call(text): + return StreamingParseResult(normal_text=text, calls=[]) + + # Parse with HarmonyParser + events = self.harmony_parser.parse(text) + # Flush buffer for complete parsing + events += self.harmony_parser.parse("") + + tool_indices = self._get_tool_indices(tools) + calls = [] + normal_parts = [] + tool_index = 0 + + for event in events: + if event.event_type == "tool_call": + # Extract tool call from event content + tool_call = self._extract_tool_call_from_event( + event.raw_text if event.raw_text else event.content, + tool_indices, + tool_index, + ) + if tool_call: + calls.append(tool_call) + tool_index += 1 + elif event.event_type == "normal": + normal_parts.append(event.content) + # Ignore reasoning events in function call context + + normal_text = " ".join(normal_parts).strip() + return StreamingParseResult(normal_text=normal_text, calls=calls) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """Parse incremental streaming text for TypeScript-style function calls.""" + self._buffer += new_text + + # Always use HarmonyParser for parsing to ensure proper filtering + events = self.harmony_parser.parse(new_text) + + # Quick check if we might have tool calls + if ( + "<|channel|>commentary to=" not in self._buffer + and not self.current_tool_name_sent + ): + # No tool calls detected, check for final content + if ( + "<|channel|>final" in self._buffer + or "assistantfinal" in self._buffer.lower() + ): + # Extract normal text from events + normal_text = "".join( + [e.content for e in events if e.event_type == "normal"] + ) + if normal_text: + self._buffer = "" + return StreamingParseResult(normal_text=normal_text, calls=[]) + + # For other content, extract normal text from events (with filtering applied) + normal_text = "".join( + [e.content for e in events if e.event_type == "normal"] + ) + if normal_text or events: + self._buffer = "" + return StreamingParseResult(normal_text=normal_text, calls=[]) + else: + # No events processed, continue buffering + return StreamingParseResult(normal_text="", calls=[]) + + if not events: + # No complete events yet + return StreamingParseResult(normal_text="", calls=[]) + + # Initialize state if needed + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + calls = [] + normal_text = "" + + for event in events: + if event.event_type == "tool_call": + # We got a complete tool call from HarmonyParser + tool_call_info = self._extract_tool_call_from_event( + event.raw_text if event.raw_text else event.content, + self._tool_indices, + self.current_tool_id if self.current_tool_id >= 0 else 0, + ) + + if tool_call_info: + # Initialize state if first tool + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Ensure arrays are large enough + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Store tool call info + self.prev_tool_call_arr[self.current_tool_id] = { + "name": tool_call_info.name, + "arguments": json.loads(tool_call_info.parameters), + } + + # Emit the complete tool call at once + # (Could be modified to emit name first, then args, if needed) + calls.append(tool_call_info) + + # Mark as streamed + self.streamed_args_for_tool[self.current_tool_id] = ( + tool_call_info.parameters + ) + + # Move to next tool + self.current_tool_id += 1 + self.current_tool_name_sent = False + + elif event.event_type == "normal": + normal_text += event.content + + # Clear buffer since HarmonyParser handles buffering + self._buffer = "" + + return StreamingParseResult(normal_text=normal_text, calls=calls) + + def _extract_tool_call_from_event( + self, content: str, tool_indices: dict, tool_index: int + ) -> Optional[ToolCallItem]: + """ + Extract tool call information from HarmonyParser event content. + + Content format: "commentary to=functions.get_weather<|constrain|>json<|message|>{...}" + """ + match = self.tool_extract_pattern.search(content) + + if not match: + logger.debug(f"Could not extract tool call from: {content[:100]}") + return None + + full_function_name = match.group(1) + json_content = match.group(2) + + # Extract function name (last part after .) + function_name = ( + full_function_name.split(".")[-1] + if "." in full_function_name + else full_function_name + ) + + # Check if tool exists + if function_name not in tool_indices: + logger.debug(f"Function {function_name} not in available tools") + return None + + # Parse JSON arguments + try: + arguments = json.loads(json_content) if json_content.strip() else {} + except json.JSONDecodeError as e: + logger.debug(f"Failed to parse JSON arguments: {e}") + return None + + return ToolCallItem( + tool_index=tool_index, + name=function_name, + parameters=json.dumps(arguments, ensure_ascii=False), + ) + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError("structure_info not used with HarmonyParser") + + def build_ebnf(self, tools: List[Tool]) -> str: + raise NotImplementedError("build_ebnf not used with HarmonyParser") diff --git a/python/sglang/srt/function_call/kimik2_detector.py b/python/sglang/srt/function_call/kimik2_detector.py new file mode 100644 index 000000000..4f39433a6 --- /dev/null +++ b/python/sglang/srt/function_call/kimik2_detector.py @@ -0,0 +1,245 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer +from sglang.srt.function_call.utils import _is_complete_json + +logger = logging.getLogger(__name__) + + +class KimiK2Detector(BaseFormatDetector): + """ + Detector for Kimi K2 model function call format. + + Format Structure: + ``` + <|tool_calls_section_begin|> + <|tool_call_begin|>functions.{func_name}:{index}<|tool_call_argument_begin|>{json_args}<|tool_call_end|> + <|tool_calls_section_end|> + ``` + + Reference: https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/main/docs/tool_call_guidance.md + """ + + def __init__(self): + super().__init__() + + self.bot_token: str = "<|tool_calls_section_begin|>" + self.eot_token: str = "<|tool_calls_section_end|>" + + self.tool_call_start_token: str = "<|tool_call_begin|>" + self.tool_call_end_token: str = "<|tool_call_end|>" + + self.tool_call_regex = re.compile( + r"<\|tool_call_begin\|>\s*(?P[\w\.]+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P\{.*?\})\s*<\|tool_call_end\|>" + ) + + self.stream_tool_call_portion_regex = re.compile( + r"<\|tool_call_begin\|>\s*(?P[\w\.]+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P\{.*)" + ) + + self._last_arguments = "" + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a KimiK2 format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + if self.bot_token not in text: + return StreamingParseResult(normal_text=text, calls=[]) + try: + # there are two possible captures - between tags, or between a + # tag and end-of-string so the result of + # findall is an array of tuples where one is a function call and + # the other is None + function_call_tuples = self.tool_call_regex.findall(text) + + logger.debug("function_call_tuples: %s", function_call_tuples) + + tool_calls = [] + for match in function_call_tuples: + function_id, function_args = match + function_name = function_id.split(".")[1].split(":")[0] + function_idx = int(function_id.split(".")[1].split(":")[1]) + + logger.info(f"function_name {function_name}") + + tool_calls.append( + ToolCallItem( + tool_index=function_idx, # Use the call index in the response, not tool position + name=function_name, + parameters=function_args, + ) + ) + + content = text[: text.find(self.bot_token)] + return StreamingParseResult(normal_text=content, calls=tool_calls) + + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # return the normal text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing tool calls for KimiK2 format. + """ + self._buffer += new_text + current_text = self._buffer + + # Check if we have a tool call (either the start token or individual tool call) + has_tool_call = ( + self.bot_token in current_text or self.tool_call_start_token in current_text + ) + + if not has_tool_call: + self._buffer = "" + for e_token in [self.eot_token, self.tool_call_end_token]: + if e_token in new_text: + new_text = new_text.replace(e_token, "") + return StreamingParseResult(normal_text=new_text) + + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + calls: list[ToolCallItem] = [] + try: + match = self.stream_tool_call_portion_regex.search(current_text) + if match: + function_id = match.group("tool_call_id") + function_args = match.group("function_arguments") + + function_name = function_id.split(".")[1].split(":")[0] + + # Initialize state if this is the first tool call + if self.current_tool_id == -1: + self.current_tool_id = 0 + self.prev_tool_call_arr = [] + self.streamed_args_for_tool = [""] + + # Ensure we have enough entries in our tracking arrays + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + if not self.current_tool_name_sent: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=function_name, + parameters="", + ) + ) + self.current_tool_name_sent = True + # Store the tool call info for serving layer completions endpoint + self.prev_tool_call_arr[self.current_tool_id] = { + "name": function_name, + "arguments": {}, + } + else: + argument_diff = ( + function_args[len(self._last_arguments) :] + if function_args.startswith(self._last_arguments) + else function_args + ) + + parsed_args_diff = argument_diff.split("<|tool_call_end|>", 1)[0] + + if parsed_args_diff: + + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=parsed_args_diff, + ) + ) + self._last_arguments += argument_diff + self.streamed_args_for_tool[ + self.current_tool_id + ] += parsed_args_diff + + parsed_args = function_args.split("<|tool_call_end|>", 1)[0] + if _is_complete_json(parsed_args): + try: + parsed_args = json.loads(parsed_args) + self.prev_tool_call_arr[self.current_tool_id][ + "arguments" + ] = parsed_args + except json.JSONDecodeError: + pass + + # Find the end of the current tool call and remove only that part from buffer + tool_call_end_pattern = ( + r"<\|tool_call_begin\|>.*?<\|tool_call_end\|>" + ) + match = re.search( + tool_call_end_pattern, current_text, re.DOTALL + ) + if match: + # Remove the completed tool call from buffer, keep any remaining content + self._buffer = current_text[match.end() :] + else: + self._buffer = "" + + result = StreamingParseResult(normal_text="", calls=calls) + self.current_tool_id += 1 + self._last_arguments = "" + self.current_tool_name_sent = False + return result + + return StreamingParseResult(normal_text="", calls=calls) + + except Exception as e: + logger.error(f"Error in parse_streaming_increment: {e}") + return StreamingParseResult(normal_text=current_text) + + def structure_info(self) -> _GetInfoFunc: + """Return function that creates StructureInfo for guided generation.""" + + def get_info(name: str) -> StructureInfo: + return StructureInfo( + begin=f"<|tool_calls_section_begin|><|tool_call_begin|>functions.{name}:0<|tool_call_argument_begin|>", + end="<|tool_call_end|><|tool_calls_section_end|>", + trigger="<|tool_calls_section_begin|>", + ) + + return get_info + + def build_ebnf(self, tools: List[Tool]) -> str: + """ + Build EBNF grammar for KimiK2 tool call format. + + NOTE: The call_rule_fmt uses [0-9]+ for the function index to allow the grammar + to accept any numeric index (0, 1, 2, etc.) for proper sequential indexing in + multiple function call scenarios, while still maintaining the correct KimiK2 + format structure for constrained generation. + """ + return EBNFComposer.build_ebnf( + tools, + sequence_start_token=self.bot_token, + sequence_end_token=self.eot_token, + tool_call_separator="", + call_rule_fmt='"<|tool_call_begin|>functions.{name}:"[0-9]+"<|tool_call_argument_begin|>"{arguments_rule}"<|tool_call_end|>"', + function_format="json", + ) diff --git a/python/sglang/srt/function_call/llama32_detector.py b/python/sglang/srt/function_call/llama32_detector.py new file mode 100644 index 000000000..453bcbc9a --- /dev/null +++ b/python/sglang/srt/function_call/llama32_detector.py @@ -0,0 +1,96 @@ +import json +import logging +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +class Llama32Detector(BaseFormatDetector): + """ + Detector for Llama 3.2 models with json tool call format. + + Format Structure: + ``` + {"name":"xxx", "arguments":{...}} + ``` + """ + + def __init__(self): + super().__init__() + self.bot_token = "<|python_tag|>" + # NOTE: technically Llama3.2 doesn't support well with parallel tool calls + # They need specific prompt engineering to support parallel tool calls + # Here we use ';' as the separator, which might have compatibility issues + # if users define to use a different separator in their prompt + self.tool_call_separator = ";" + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a Llama 3.2 format tool call.""" + # depending on the prompt format the Llama model may or may not + # prefix the output with the <|python_tag|> token + return "<|python_tag|>" in text or text.startswith("{") + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """Parse function calls from text, handling multiple JSON objects.""" + if "<|python_tag|>" not in text and not text.startswith("{"): + return StreamingParseResult(normal_text=text, calls=[]) + + if "<|python_tag|>" in text: + normal_text, action_text = text.split("<|python_tag|>", maxsplit=1) + else: + normal_text, action_text = "", text + + decoder = json.JSONDecoder() + idx = 0 + safe_idx = idx # the index of the last valid JSON object + all_actions = [] + action_text_len = len(action_text) + while idx < action_text_len: + try: + obj, end = decoder.raw_decode(action_text[idx:]) + all_actions.append(obj) + idx += end + len(self.tool_call_separator) + safe_idx = idx + except json.JSONDecodeError as e: + # Find where next `{"name"` appears and try again + logger.warning( + f"Failed to parse JSON part: {action_text[idx:]}, JSON parse error: {str(e)}" + ) + next_obj_start = action_text.find('{"name":', idx + 1) + if next_obj_start == -1: + break + idx = next_obj_start + continue + + # Only process if we found valid JSON objects + calls = self.parse_base_json(all_actions, tools) if all_actions else [] + # Use safe_idx to avoid idx containing the last part of an invalid JSON object + trailing_text = ( + action_text[safe_idx:].strip() if safe_idx < action_text_len else "" + ) + return StreamingParseResult( + normal_text=normal_text + trailing_text, calls=calls + ) + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin='<|python_tag|>{"name":"' + name + '", "arguments":', + end="}", + trigger="<|python_tag|>", + ) + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + function_format="json", + tool_call_separator=self.tool_call_separator, + ) diff --git a/python/sglang/srt/function_call/mistral_detector.py b/python/sglang/srt/function_call/mistral_detector.py new file mode 100644 index 000000000..49767fd53 --- /dev/null +++ b/python/sglang/srt/function_call/mistral_detector.py @@ -0,0 +1,139 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +class MistralDetector(BaseFormatDetector): + """ + Detector for Mistral model function call format. + + The Mistral format uses a simple bracket-delimited structure with JSON arrays + containing function call objects. + + Format Structure: + ``` + [TOOL_CALLS] [{"name": "function_name", "arguments": {json_args}}, ...] + ``` + + Reference: https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3?chat_template=default + """ + + def __init__(self): + """ + Initializes the detector with necessary state variables. + """ + super().__init__() + self.bot_token = "[TOOL_CALLS] [" + self.eot_token = "]" + self.tool_call_regex = re.compile(r"\[{.*}\]", re.DOTALL) + self.tool_call_separator = ", " + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a Mistral format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + + # Extract the JSON array part from [TOOL_CALLS] [...] + # Use bracket counting to properly handle nested brackets in JSON content + json_array_str = self._extract_json_array(text) + if not json_array_str: + return StreamingParseResult(normal_text=normal_text, calls=[]) + + calls = [] + try: + function_call_arr = json.loads(json_array_str) + # Handle both single object and array of objects + if not isinstance(function_call_arr, list): + function_call_arr = [function_call_arr] + calls = self.parse_base_json(function_call_arr, tools) + except json.JSONDecodeError as e: + logger.warning( + f"Failed to parse JSON part: {json_array_str}, JSON parse error: {str(e)}" + ) + + return StreamingParseResult(normal_text=normal_text, calls=calls) + + def _extract_json_array(self, text: str) -> str: + """ + Extract the JSON array part using bracket counting to handle nested brackets. + + :param text: The complete text containing [TOOL_CALLS] [...] + :return: The JSON array string or None if not found + """ + start_idx = text.find(self.bot_token) + if start_idx == -1: + return None + + # Start from the opening bracket after [TOOL_CALLS] + json_start = ( + start_idx + len(self.bot_token) - 1 + ) # -1 to include the opening bracket + bracket_count = 0 + in_string = False + escape_next = False + + for i in range(json_start, len(text)): + char = text[i] + + if escape_next: + escape_next = False + continue + + if char == "\\": + escape_next = True + continue + + if char == '"' and not escape_next: + in_string = not in_string + continue + + if not in_string: + if char == "[": + bracket_count += 1 + elif char == "]": + bracket_count -= 1 + if bracket_count == 0: + return text[json_start : i + 1] + + return None + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin='[TOOL_CALLS] [{"name":"' + name + '", "arguments":', + end="}]", + trigger="[TOOL_CALLS]", + ) + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + sequence_start_token=self.bot_token, + sequence_end_token=self.eot_token, + function_format="json", + tool_call_separator=self.tool_call_separator, + ) diff --git a/python/sglang/srt/function_call/pythonic_detector.py b/python/sglang/srt/function_call/pythonic_detector.py new file mode 100644 index 000000000..be183c6bf --- /dev/null +++ b/python/sglang/srt/function_call/pythonic_detector.py @@ -0,0 +1,231 @@ +import ast +import json +import logging +import re +from typing import List, Optional + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +class PythonicDetector(BaseFormatDetector): + """ + Detector for Llama-4 models with Pythonic tool call format. + + The Pythonic format uses Python function call syntax within square brackets, + with arguments as Python literals rather than JSON. + + Format Structure: + ``` + [tool1(arg1=val1, arg2=val2), tool2(arg1=val3)] + ``` + + Reference: https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct?chat_template=default + """ + + def __init__(self): + super().__init__() + self.tool_call_regex = re.compile( + r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", + re.DOTALL, + ) + + @staticmethod + def _text_strip(text: str) -> str: + # Llama 4 model sometime will output <|python_start|> and <|python_end|> tokens + # remove those tokens + text = text.replace("<|python_start|>", "") + text = text.replace("<|python_end|>", "") + return text + + def has_tool_call(self, text: str) -> bool: + return bool(self.tool_call_regex.search(self._text_strip(text.strip()))) + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + # Try parsing the text as a Python list of function calls + text = text.strip() + + # Remove unexpected <|python_start|> and <|python_end|> for llama4 + text = self._text_strip(text) + + match = self.tool_call_regex.search(text) + if match is None: + return StreamingParseResult(normal_text=text, calls=[]) + + # Extract the tool call part and any text before/after it + tool_call_start = match.start() + tool_call_end = match.end() + + normal_text_before = text[:tool_call_start] if tool_call_start > 0 else "" + tool_call_text = text[tool_call_start:tool_call_end] + normal_text_after = text[tool_call_end:] if tool_call_end < len(text) else "" + + # Combine normal text + normal_text = normal_text_before + normal_text_after + + try: + module = ast.parse(tool_call_text) + parsed = getattr(module.body[0], "value", None) + if not ( + isinstance(parsed, ast.List) + and all(isinstance(e, ast.Call) for e in parsed.elts) + ): + return StreamingParseResult(normal_text=normal_text, calls=[]) + + calls = [] + tool_indices = self._get_tool_indices(tools) + for call_index, call in enumerate(parsed.elts): + if not isinstance(call.func, ast.Name): + continue + function_name = call.func.id + # Validate that the function exists in the tools + if function_name not in tool_indices: + logger.warning( + f"Model attempted to call undefined function: {function_name}" + ) + continue + arguments = {} + for keyword in call.keywords: + arguments[keyword.arg] = self._get_parameter_value(keyword.value) + calls.append( + ToolCallItem( + tool_index=call_index, # Use the call index in the response, not tool position + name=function_name, + parameters=json.dumps(arguments, ensure_ascii=False), + ) + ) + + return StreamingParseResult(normal_text=normal_text, calls=calls) + except Exception: + logger.exception("Error in pythonic tool call parsing.") + return StreamingParseResult(normal_text=normal_text, calls=[]) + + def _find_matching_bracket(self, buffer: str, start: int) -> int: + """ + Find the matching closing bracket for the opening bracket at start position. + Properly handles nested brackets. + + Args: + buffer: The text buffer to search in + start: Position of the opening bracket '[' + + Returns: + Position of the matching closing bracket ']', or -1 if not found + """ + bracket_count = 0 + for i in range(start, len(buffer)): + if buffer[i] == "[": + bracket_count += 1 + elif buffer[i] == "]": + bracket_count -= 1 + if bracket_count == 0: + return i + return -1 # No matching bracket found + + def _strip_and_split_buffer(self, buffer: str) -> tuple[str, str]: + """ + Strip special tokens from buffer and split into safe_text and held_back_text. + + Returns: + tuple of (safe_text_to_output, text_to_hold_in_buffer) + """ + # Check if original buffer ends with a partial token at the end + special_tokens = ["<|python_start|>", "<|python_end|>"] + + for token in special_tokens: + partial_length = self._ends_with_partial_token(buffer, token) + if partial_length > 0: + # Split buffer: safe part + held back partial token + safe_text = buffer[:-partial_length] + held_back = buffer[-partial_length:] + # Strip complete special tokens from safe part only + safe_text = self._text_strip(safe_text) + return safe_text, held_back + + # No partial tokens found, strip complete tokens from entire buffer + safe_text = self._text_strip(buffer) + return safe_text, "" + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing for pythonic tool calls. + Buffers input until a complete pythonic tool call (from [ to ]) is found, + then parses and emits any detected calls. + """ + self._buffer += new_text + + # Strip special tokens from entire buffer and handle partial tokens + stripped_buffer, held_back = self._strip_and_split_buffer(self._buffer) + + start = stripped_buffer.find("[") + + if start == -1: + # No tool call bracket found + self._buffer = held_back + return StreamingParseResult(normal_text=stripped_buffer) + + normal_text = stripped_buffer[:start] if start > 0 else "" + + end = self._find_matching_bracket(stripped_buffer, start) + if end != -1: + # Found complete tool call + call_text = stripped_buffer[start : end + 1] + result = self.detect_and_parse(call_text, tools) + + # Update buffer with remaining text after tool call plus any held back text + remaining_text = stripped_buffer[end + 1 :] + held_back + self._buffer = remaining_text + + # If we had normal text before the tool call, add it to the result + if normal_text: + result.normal_text = normal_text + (result.normal_text or "") + + return result + + # We have an opening bracket but no closing bracket yet + # Put back everything from the bracket onwards plus held back text + self._buffer = stripped_buffer[start:] + held_back + + if normal_text: + return StreamingParseResult(normal_text=normal_text) + + # Otherwise, we're still accumulating a potential tool call + return StreamingParseResult(normal_text="") + + def _get_parameter_value(self, val): + if isinstance(val, ast.Constant): + return val.value + elif isinstance(val, ast.Dict): + return { + k.value: self._get_parameter_value(v) + for k, v in zip(val.keys, val.values) + } + elif isinstance(val, ast.List): + return [self._get_parameter_value(v) for v in val.elts] + else: + raise ValueError("Tool call arguments must be literals") + + def supports_structural_tag(self) -> bool: + return False + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError + + def build_ebnf(self, tools: List[Tool]) -> Optional[str]: + return EBNFComposer.build_ebnf( + tools, + sequence_start_token="[", + sequence_end_token="]", + tool_call_separator=",", + function_format="pythonic", + ) diff --git a/python/sglang/srt/function_call/qwen25_detector.py b/python/sglang/srt/function_call/qwen25_detector.py new file mode 100644 index 000000000..40a65e5df --- /dev/null +++ b/python/sglang/srt/function_call/qwen25_detector.py @@ -0,0 +1,130 @@ +import json +import logging +import re +from typing import List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + StructureInfo, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +class Qwen25Detector(BaseFormatDetector): + """ + Detector for Qwen 2.5 and Qwen 3 model function call format. + + Format Structure: + ``` + \n{"name":"func1", "arguments":{...}}\n\n\n{"name":"func2", "arguments":{...}}\n + ``` + + Key Components: + - Tool Call Tags: `` and `` wrap each individual call + - Function Call Object: JSON object with "name" and "arguments" fields + + Reference: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct?chat_template=default + """ + + def __init__(self): + """ + Initializes the detector with necessary state variables. + """ + super().__init__() + self.bot_token = "\n" + self.eot_token = "\n" + self.tool_call_separator = "\n" + self._normal_text_buffer = "" # Buffer for handling partial end tokens + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a Qwen 2.5 format tool call.""" + return self.bot_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + + :param text: The complete text to parse. + :param tools: List of available tools. + :return: ParseResult indicating success or failure, consumed text, leftover text, and parsed calls. + """ + idx = text.find(self.bot_token) + normal_text = text[:idx].strip() if idx != -1 else text + if self.bot_token not in text: + return StreamingParseResult(normal_text=normal_text, calls=[]) + + # Find all \n...\n blocks + pattern = rf"{re.escape(self.bot_token)}(.*?){re.escape(self.eot_token)}" + match_result_list = re.findall(pattern, text, re.DOTALL) + calls = [] + for match_result in match_result_list: + try: + parsed_call = json.loads(match_result.strip()) + calls.extend(self.parse_base_json(parsed_call, tools)) + except json.JSONDecodeError as e: + logger.warning( + f"Failed to parse JSON part: {match_result}, JSON parse error: {str(e)}" + ) + continue + return StreamingParseResult(normal_text=normal_text, calls=calls) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing for Qwen 2.5 tool calls. + Uses base class implementation with buffering to handle partial end tokens. + """ + result = super().parse_streaming_increment(new_text, tools) + + # Handle partial end tokens that are streamed character by character + if result.normal_text: + self._normal_text_buffer += result.normal_text + + # Check if buffer contains complete end token (without leading newline) + end_token_without_newline = self.eot_token[1:] # "" + if end_token_without_newline in self._normal_text_buffer: + cleaned_text = self._normal_text_buffer.replace( + end_token_without_newline, "" + ) + self._normal_text_buffer = "" + result.normal_text = cleaned_text + else: + # Check if buffer might contain partial end token at the end + partial_match_len = self._ends_with_partial_token( + self._normal_text_buffer, end_token_without_newline + ) + + if partial_match_len: + # Keep potential partial match in buffer, return the rest + result.normal_text = self._normal_text_buffer[:-partial_match_len] + self._normal_text_buffer = self._normal_text_buffer[ + -partial_match_len: + ] + else: + # No partial match, return all buffered text + result.normal_text = self._normal_text_buffer + self._normal_text_buffer = "" + + return result + + def structure_info(self) -> _GetInfoFunc: + return lambda name: StructureInfo( + begin='\n{"name":"' + name + '", "arguments":', + end="}\n", + trigger="", + ) + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + individual_call_start_token=self.bot_token.replace("\n", "\\n"), + individual_call_end_token=self.eot_token.replace("\n", "\\n"), + tool_call_separator="\\n", + function_format="json", + ) diff --git a/python/sglang/srt/function_call/qwen3_coder_detector.py b/python/sglang/srt/function_call/qwen3_coder_detector.py new file mode 100644 index 000000000..9bd3c7c24 --- /dev/null +++ b/python/sglang/srt/function_call/qwen3_coder_detector.py @@ -0,0 +1,362 @@ +import ast +import html +import json +import logging +import re +from typing import Any, Dict, List, Tuple + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +def _safe_val(raw: str) -> Any: + raw = html.unescape(raw.strip()) + try: + return json.loads(raw) + except Exception: + try: + return ast.literal_eval(raw) + except Exception: + return raw + + +class Qwen3CoderDetector(BaseFormatDetector): + """ + Detector for Qwen 3 models. + Assumes function call format: + + + + pwd && ls + + + + """ + + def __init__(self): + super().__init__() + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + self.tool_call_prefix: str = "(.*?)|(.*?)$", re.DOTALL + ) + self.tool_call_function_regex = re.compile( + r"|| bool: + return self.tool_call_start_token in text + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + normal, calls = self._extract(text, tools) + return StreamingParseResult(normal_text=normal, calls=calls) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + self._buf += new_text + normal = "" + calls: List[ToolCallItem] = [] + + # Build tool indices for validation + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + while True: + # If we're not in a tool call and don't see a start token, return normal text + if not self._in_tool_call and self.tool_call_start_token not in self._buf: + normal += self._buf + self._buf = "" + break + + # Look for tool call start + if not self._in_tool_call: + s = self._buf.find(self.tool_call_start_token) + if s == -1: + normal += self._buf + self._buf = "" + break + + normal += self._buf[:s] + self._buf = self._buf[s:] + + self._in_tool_call = True + self._function_name_sent = False + self._current_function_name = "" + self._current_parameters = {} + self._streamed_parameters = {} + + # Remove the start token + self._buf = self._buf[len(self.tool_call_start_token) :] + continue + + # We're in a tool call, try to parse function name if not sent yet + if not self._function_name_sent: + # Look for function name pattern: + function_match = re.search(r"]+)>", self._buf) + if function_match: + function_name = function_match.group(1).strip() + + # Validate function name + if function_name in self._tool_indices: + self._current_function_name = function_name + self._function_name_sent = True + + # Initialize tool call tracking + if self.current_tool_id == -1: + self.current_tool_id = 0 + + # Ensure tracking arrays are large enough + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Store tool call info + self.prev_tool_call_arr[self.current_tool_id] = { + "name": function_name, + "arguments": {}, + } + + # Send tool name with empty parameters + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=function_name, + parameters="", + ) + ) + + # Remove the processed function declaration + self._buf = self._buf[function_match.end() :] + continue + else: + # Invalid function name, reset state + logger.warning(f"Invalid function name: {function_name}") + self._reset_streaming_state() + normal += self._buf + self._buf = "" + break + else: + # Function name not complete yet, wait for more text + break + + # Parse parameters incrementally + if self._function_name_sent: + # Process parameters and get any calls to emit + parameter_calls = self._parse_and_stream_parameters(self._buf) + calls.extend(parameter_calls) + + # Check if tool call is complete + if self.tool_call_end_token in self._buf: + end_pos = self._buf.find(self.tool_call_end_token) + + # Add closing brace to complete the JSON object + current_streamed = self.streamed_args_for_tool[self.current_tool_id] + if current_streamed: + # Count opening and closing braces to check if JSON is complete + open_braces = current_streamed.count("{") + close_braces = current_streamed.count("}") + if open_braces > close_braces: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters="}", + ) + ) + self.streamed_args_for_tool[self.current_tool_id] = ( + current_streamed + "}" + ) + + # Complete the tool call + self._buf = self._buf[end_pos + len(self.tool_call_end_token) :] + self._reset_streaming_state() + self.current_tool_id += 1 + continue + else: + # Tool call not complete yet, wait for more text + break + + return StreamingParseResult(normal_text=normal, calls=calls) + + def _parse_and_stream_parameters(self, text_to_parse: str) -> List[ToolCallItem]: + """ + Parse complete parameter blocks from text and return any tool call items to emit. + + This method: + 1. Finds all complete blocks + 2. Parses them into a dictionary + 3. Compares with current parameters and generates diff if needed + 4. Updates internal state + + Args: + text_to_parse: The text to search for parameter blocks + + Returns: + List of ToolCallItem objects to emit (may be empty) + """ + calls: List[ToolCallItem] = [] + + # Find all complete parameter patterns + param_matches = list( + re.finditer( + r"]+)>(.*?)", text_to_parse, re.DOTALL + ) + ) + + # Build new parameters dictionary + new_params = {} + for match in param_matches: + param_name = match.group(1).strip() + param_value = match.group(2) + new_params[param_name] = _safe_val(param_value) + + # Calculate parameter diff to stream with proper incremental JSON building + if new_params != self._current_parameters: + previous_args_json = self.streamed_args_for_tool[self.current_tool_id] + + # Build incremental JSON properly + if not self._current_parameters: + # First parameter(s) - start JSON object but don't close it yet + items = [] + for key, value in new_params.items(): + items.append( + f"{json.dumps(key, ensure_ascii=False)}: {json.dumps(value, ensure_ascii=False)}" + ) + json_fragment = "{" + ", ".join(items) + + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=json_fragment, + ) + ) + self.streamed_args_for_tool[self.current_tool_id] = json_fragment + + else: + # Additional parameters - add them incrementally + new_keys = set(new_params.keys()) - set(self._current_parameters.keys()) + if new_keys: + # Build the continuation part (no closing brace yet) + continuation_parts = [] + for key in new_keys: + value = new_params[key] + continuation_parts.append( + f"{json.dumps(key, ensure_ascii=False)}: {json.dumps(value, ensure_ascii=False)}" + ) + + json_fragment = ", " + ", ".join(continuation_parts) + + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=None, + parameters=json_fragment, + ) + ) + self.streamed_args_for_tool[self.current_tool_id] = ( + previous_args_json + json_fragment + ) + + # Update current state + self._current_parameters = new_params + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = new_params + + return calls + + def _reset_streaming_state(self): + """Reset streaming state for the next tool call""" + self._in_tool_call = False + self._function_name_sent = False + self._current_function_name = "" + self._current_parameters = {} + self._streamed_parameters = {} + self.current_tool_name_sent = False + + def _extract(self, text: str, tools: List[Tool]) -> Tuple[str, List[ToolCallItem]]: + normal_parts: List[str] = [] + calls: List[ToolCallItem] = [] + cursor = 0 + while True: + s = text.find(self.tool_call_start_token, cursor) + if s == -1: + normal_parts.append(text[cursor:]) + break + normal_parts.append(text[cursor:s]) + e = text.find(self.tool_call_end_token, s) + if e == -1: + normal_parts.append(text[s:]) + break + block = text[s : e + len(self.tool_call_end_token)] + cursor = e + len(self.tool_call_end_token) + calls.extend(self._parse_block(block, tools)) + return "".join(normal_parts), calls + + def _parse_block(self, block: str, tools: List[Tool]) -> List[ToolCallItem]: + res: List[ToolCallItem] = [] + for m in self.tool_call_function_regex.findall(block): + txt = m[0] if m[0] else m[1] + if ">" not in txt: + continue + idx = txt.index(">") + fname = txt[:idx].strip() + body = txt[idx + 1 :] + params: Dict[str, Any] = {} + for pm in self.tool_call_parameter_regex.findall(body): + ptxt = pm[0] if pm[0] else pm[1] + if ">" not in ptxt: + continue + pidx = ptxt.index(">") + pname = ptxt[:pidx].strip() + pval = ptxt[pidx + 1 :].lstrip("\n").rstrip("\n") + params[pname] = _safe_val(pval) + raw = {"name": fname, "arguments": params} + try: + # TODO: fix idx in function call, the index for a function + # call will always be -1 in parse_base_json + res.extend(self.parse_base_json(raw, tools)) + except Exception: + logger.warning("invalid tool call for %s dropped", fname) + return res + + def supports_structural_tag(self) -> bool: + return False + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError + + def build_ebnf(self, tools: List[Tool]): + return EBNFComposer.build_ebnf( + tools, + individual_call_start_token=self.tool_call_start_token.replace("\n", "\\n"), + individual_call_end_token=self.tool_call_end_token.replace("\n", "\\n"), + tool_call_separator="\\n", + function_format="xml", + call_rule_fmt='"\\n" {arguments_rule} "\\n"', + key_value_rule_fmt='"\\n" {valrule} "\\n"', + key_value_separator='"\\n"', + ) diff --git a/python/sglang/srt/function_call/step3_detector.py b/python/sglang/srt/function_call/step3_detector.py new file mode 100644 index 000000000..b46f4544f --- /dev/null +++ b/python/sglang/srt/function_call/step3_detector.py @@ -0,0 +1,436 @@ +import ast +import json +import logging +import re +from typing import Any, Dict, List + +from sglang.srt.entrypoints.openai.protocol import Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.core_types import ( + StreamingParseResult, + ToolCallItem, + _GetInfoFunc, +) +from sglang.srt.function_call.ebnf_composer import EBNFComposer + +logger = logging.getLogger(__name__) + + +def get_argument_type(func_name: str, arg_key: str, defined_tools: List[Tool]) -> str: + """Get the expected type for a function argument from tool schema.""" + name2tool = {tool.function.name: tool for tool in defined_tools} + if func_name not in name2tool: + return None + tool = name2tool[func_name] + parameters = tool.function.parameters or {} + properties = parameters.get("properties", {}) + if arg_key not in properties: + return None + return properties[arg_key].get("type", None) + + +def parse_arguments(value: str) -> tuple[Any, bool]: + """Parse a string value to appropriate type. Returns (parsed_value, success).""" + try: + try: + parsed_value = json.loads(value) + except: + parsed_value = ast.literal_eval(value) + return parsed_value, True + except: + return value, False + + +class Step3Detector(BaseFormatDetector): + """ + Detector for Step3 model function call format. + + The Step3 format uses special Unicode tokens to delimit function calls + with steptml XML format for invocations. + + Format Structure: + ``` + <｜tool_calls_begin｜> + <｜tool_call_begin｜>function<｜tool_sep｜> + value1 + value2 + <｜tool_call_end｜> + <｜tool_calls_end｜> + ``` + """ + + def __init__(self): + super().__init__() + self.bot_token = "<｜tool_calls_begin｜>" + self.eot_token = "<｜tool_calls_end｜>" + self.tool_call_begin = "<｜tool_call_begin｜>" + self.tool_call_end = "<｜tool_call_end｜>" + self.tool_sep = "<｜tool_sep｜>" + + # Regex for parsing steptml invocations + self.invoke_regex = re.compile( + r'(.+?)', re.DOTALL + ) + self.param_regex = re.compile( + r'([^<]*)', re.DOTALL + ) + + # Streaming state variables + self._in_tool_block: bool = False + self._tool_block_finished: bool = False + self._current_function_name: str = "" + self._current_parameters: Dict[str, Any] = {} + self._in_tool_call: bool = False + self._function_name_sent: bool = False + + def has_tool_call(self, text: str) -> bool: + """Check if the text contains a Step3 format tool call.""" + return self.bot_token in text + + def _parse_steptml_invoke( + self, text: str, tools: List[Tool] = None + ) -> tuple[str, dict]: + """Parse steptml invoke format to extract function name and parameters.""" + invoke_match = self.invoke_regex.search(text) + if not invoke_match: + return None, {} + + func_name = invoke_match.group(1) + params_text = invoke_match.group(2) + + params = {} + for param_match in self.param_regex.finditer(params_text): + param_name = param_match.group(1) + param_value = param_match.group(2).strip() + + # If tools provided, use schema-aware parsing + if tools: + arg_type = get_argument_type(func_name, param_name, tools) + if arg_type and arg_type != "string": + parsed_value, _ = parse_arguments(param_value) + params[param_name] = parsed_value + else: + params[param_name] = param_value + else: + # Fallback to generic parsing if no tools provided + parsed_value, _ = parse_arguments(param_value) + params[param_name] = parsed_value + + return func_name, params + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """ + One-time parsing: Detects and parses tool calls in the provided text. + """ + if self.bot_token not in text: + return StreamingParseResult(normal_text=text, calls=[]) + + try: + pre_text, rest = text.split(self.bot_token, 1) + + # If no end token, return everything as normal text + if self.eot_token not in rest: + return StreamingParseResult(normal_text=text, calls=[]) + + tool_section, post_text = rest.split(self.eot_token, 1) + + # Find all individual tool calls using regex + calls = [] + tool_call_pattern = ( + f"{re.escape(self.tool_call_begin)}(.*?){re.escape(self.tool_call_end)}" + ) + + for match in re.finditer(tool_call_pattern, tool_section, re.DOTALL): + call_content = match.group(1) + + # Check if it's a function call + if self.tool_sep not in call_content: + continue + + type_part, invoke_part = call_content.split(self.tool_sep, 1) + if type_part.strip() != "function": + continue + + func_name, params = self._parse_steptml_invoke(invoke_part, tools) + if func_name: + # Use parse_base_json to create the ToolCallItem + action = {"name": func_name, "arguments": params} + calls.extend(self.parse_base_json(action, tools)) + + # Combine pre and post text + normal_text = pre_text + post_text + + return StreamingParseResult(normal_text=normal_text, calls=calls) + + except Exception as e: + logger.error(f"Error in detect_and_parse: {e}") + # Return the original text if parsing fails + return StreamingParseResult(normal_text=text) + + def parse_streaming_increment( + self, new_text: str, tools: List[Tool] + ) -> StreamingParseResult: + """ + Streaming incremental parsing for Step3 format. + """ + self._buffer += new_text + + # Build tool indices for validation + if not hasattr(self, "_tool_indices"): + self._tool_indices = self._get_tool_indices(tools) + + # If we've finished the tool block, everything is normal text + if self._tool_block_finished: + normal_text = self._buffer + self._buffer = "" + return StreamingParseResult(normal_text=normal_text) + + # Check if tool block hasn't started yet + if not self._in_tool_block: + if self.bot_token in self._buffer: + idx = self._buffer.find(self.bot_token) + normal_text = self._buffer[:idx] + self._buffer = self._buffer[idx + len(self.bot_token) :] + self._in_tool_block = True + return StreamingParseResult(normal_text=normal_text) + else: + # Check if we might have a partial bot_token + partial_len = self._ends_with_partial_token( + self._buffer, self.bot_token + ) + if partial_len: + return StreamingParseResult() # Wait for more text + else: + normal_text = self._buffer + self._buffer = "" + return StreamingParseResult(normal_text=normal_text) + + # We're inside the tool block + calls: List[ToolCallItem] = [] + + # Check if tool block is ending + if self.eot_token in self._buffer: + idx = self._buffer.find(self.eot_token) + + # If we're in the middle of a tool call, we need to handle it + if self._in_tool_call: + # The buffer before eot_token might contain the end of the current tool call + before_eot = self._buffer[:idx] + if self.tool_call_end in before_eot: + # Parse this final tool call + result = self._parse_partial_tool_call(tools) + calls.extend(result.calls) + else: + # Incomplete tool call - log warning + logger.warning("Tool block ended with incomplete tool call") + + remaining = self._buffer[idx + len(self.eot_token) :] + self._buffer = "" + self._tool_block_finished = True + + # Reset any partial tool call state + self._reset_streaming_state() + + return StreamingParseResult(normal_text=remaining, calls=calls) + + # Check if we're in a tool call or need to start one + if not self._in_tool_call: + if self.tool_call_begin in self._buffer: + idx = self._buffer.find(self.tool_call_begin) + # Remove any content before tool call begin (shouldn't happen but be safe) + self._buffer = self._buffer[idx + len(self.tool_call_begin) :] + self._in_tool_call = True + self._function_name_sent = False + self._current_function_name = "" + self._current_parameters = {} + # Fall through to parse the partial tool call + else: + # Wait for tool call to begin + return StreamingParseResult() + + # Parse partial tool call + if self._in_tool_call: + return self._parse_partial_tool_call(tools) + + return StreamingParseResult() + + def _parse_partial_tool_call(self, tools: List[Tool]) -> StreamingParseResult: + """Parse partial tool call for streaming scenarios.""" + calls = [] + + # Check if we have tool_sep (means we're past the type declaration) + if self.tool_sep not in self._buffer: + return StreamingParseResult(calls=calls) # Wait for more text + + type_part, invoke_part = self._buffer.split(self.tool_sep, 1) + if type_part.strip() != "function": + # Invalid tool type, skip this tool call + self._reset_streaming_state() + return StreamingParseResult(calls=calls) + + # Try to extract function name if not sent yet + if not self._function_name_sent: + name_match = re.search(r'', invoke_part) + if name_match: + func_name = name_match.group(1) + + # Validate function name + if func_name in self._tool_indices: + self._current_function_name = func_name + self._function_name_sent = True + + # Initialize tool tracking + if self.current_tool_id == -1: + self.current_tool_id = 0 + + # Ensure tracking arrays are large enough + while len(self.prev_tool_call_arr) <= self.current_tool_id: + self.prev_tool_call_arr.append({}) + while len(self.streamed_args_for_tool) <= self.current_tool_id: + self.streamed_args_for_tool.append("") + + # Store tool call info + self.prev_tool_call_arr[self.current_tool_id] = { + "name": func_name, + "arguments": {}, + } + + # Send tool name with empty parameters + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + name=func_name, + parameters="", + ) + ) + else: + # Invalid function name + logger.warning(f"Invalid function name: {func_name}") + self._reset_streaming_state() + return StreamingParseResult(calls=calls) + else: + # Function name not complete yet + return StreamingParseResult(calls=calls) + + # Parse parameters incrementally + if self._function_name_sent: + # Extract all complete parameters + new_params = {} + for param_match in self.param_regex.finditer(invoke_part): + param_name = param_match.group(1) + param_value = param_match.group(2).strip() + + # Use schema-aware parsing + arg_type = get_argument_type( + self._current_function_name, param_name, tools + ) + if arg_type and arg_type != "string": + parsed_value, _ = parse_arguments(param_value) + new_params[param_name] = parsed_value + else: + new_params[param_name] = param_value + + # Check if we have new parameters to stream + if new_params != self._current_parameters: + # Build the JSON content without the closing brace for streaming + if not self._current_parameters: + # First parameters - send opening brace and content + params_content = json.dumps(new_params, ensure_ascii=False) + if len(params_content) > 2: # More than just "{}" + # Send everything except the closing brace + diff = params_content[:-1] + else: + diff = "{" + else: + # Subsequent parameters - calculate the incremental diff + old_json = json.dumps(self._current_parameters, ensure_ascii=False) + new_json = json.dumps(new_params, ensure_ascii=False) + + # Remove closing braces for comparison + old_without_brace = old_json[:-1] + new_without_brace = new_json[:-1] + + # The new content should extend the old content + if new_without_brace.startswith(old_without_brace): + diff = new_without_brace[len(old_without_brace) :] + else: + # Parameters changed in unexpected way - shouldn't happen in normal streaming + diff = "" + + if diff: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + parameters=diff, + ) + ) + self.streamed_args_for_tool[self.current_tool_id] += diff + + # Update current state + self._current_parameters = new_params + self.prev_tool_call_arr[self.current_tool_id]["arguments"] = new_params + + # Check if tool call is complete + if self.tool_call_end in self._buffer: + # Send closing brace if we've sent any parameters + if self.streamed_args_for_tool[self.current_tool_id]: + calls.append( + ToolCallItem( + tool_index=self.current_tool_id, + parameters="}", + ) + ) + self.streamed_args_for_tool[self.current_tool_id] += "}" + + # Find the end position + end_idx = self._buffer.find(self.tool_call_end) + # Remove the processed tool call from buffer + self._buffer = self._buffer[end_idx + len(self.tool_call_end) :] + + # Reset state for next tool call + self._reset_streaming_state() + self.current_tool_id += 1 + + return StreamingParseResult(calls=calls) + + def _reset_streaming_state(self): + """Reset streaming state for the next tool call""" + self._in_tool_call = False + self._function_name_sent = False + self._current_function_name = "" + self._current_parameters = {} + + def supports_structural_tag(self) -> bool: + """Return True if this detector supports structural tag format.""" + return False + + def structure_info(self) -> _GetInfoFunc: + raise NotImplementedError() + + def build_ebnf(self, tools: List[Tool]) -> str: + """ + Build EBNF grammar for Step3 tool call format. + """ + # Custom call rule for steptml format + call_rule_fmt = ( + '"function" "<｜tool_sep｜>" "" ' + '{arguments_rule} ""' + ) + + # Custom key-value rule for steptml parameters + key_value_rule_fmt = ( + '"" {valrule} ""' + ) + + return EBNFComposer.build_ebnf( + tools, + sequence_start_token=self.bot_token, + sequence_end_token=self.eot_token, + individual_call_start_token=self.tool_call_begin, + individual_call_end_token=self.tool_call_end, + tool_call_separator="", + function_format="xml", + call_rule_fmt=call_rule_fmt, + key_value_rule_fmt=key_value_rule_fmt, + key_value_separator="", + ) diff --git a/python/sglang/srt/function_call/utils.py b/python/sglang/srt/function_call/utils.py new file mode 100644 index 000000000..c4da456f3 --- /dev/null +++ b/python/sglang/srt/function_call/utils.py @@ -0,0 +1,52 @@ +import json +from json import JSONDecodeError, JSONDecoder +from typing import Any, Tuple + +import partial_json_parser +from partial_json_parser.core.options import Allow + + +def _find_common_prefix(s1: str, s2: str) -> str: + prefix = "" + min_length = min(len(s1), len(s2)) + for i in range(0, min_length): + if s1[i] == s2[i]: + prefix += s1[i] + else: + break + return prefix + + +def _partial_json_loads(input_str: str, flags: Allow) -> Tuple[Any, int]: + """ + Parse incomplete or partial JSON strings commonly encountered during streaming. + + Args: + input_str (str): The potentially incomplete JSON string to parse. + flags (Allow): Bitwise flags controlling what types of partial data are allowed. + Common flags include: + - Allow.STR: Allow partial strings (e.g., '"hello wo' -> 'hello wo') + - Allow.OBJ: Allow partial objects (e.g., '{"key":' -> {'key': None}) + - Allow.ARR: Allow partial arrays (e.g., '[1, 2,' -> [1, 2]) + - Allow.ALL: Allow all types of partial data + + Returns: + Tuple[Any, int]: A tuple containing: + - parsed_object: The Python object parsed from the JSON + - consumed_length: Number of characters consumed from input_str + """ + try: + return (partial_json_parser.loads(input_str, flags), len(input_str)) + except JSONDecodeError as e: + if "Extra data" in e.msg: + dec = JSONDecoder() + return dec.raw_decode(input_str) + raise + + +def _is_complete_json(input_str: str) -> bool: + try: + json.loads(input_str) + return True + except JSONDecodeError: + return False diff --git a/python/sglang/srt/hf_transformers_utils.py b/python/sglang/srt/hf_transformers_utils.py new file mode 100644 index 000000000..d7dcf8904 --- /dev/null +++ b/python/sglang/srt/hf_transformers_utils.py @@ -0,0 +1,442 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for Huggingface Transformers.""" + +import contextlib +import json +import os +import warnings +from pathlib import Path +from typing import Any, Dict, Optional, Type, Union + +import torch +from huggingface_hub import snapshot_download +from transformers import ( + AutoConfig, + AutoProcessor, + AutoTokenizer, + GenerationConfig, + PretrainedConfig, + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, +) +from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES + +from sglang.srt.configs import ( + ChatGLMConfig, + DbrxConfig, + DeepseekVL2Config, + ExaoneConfig, + KimiVLConfig, + LongcatFlashConfig, + MultiModalityConfig, + Qwen3NextConfig, + Step3VLConfig, +) +from sglang.srt.configs.internvl import InternVLChatConfig +from sglang.srt.connector import create_remote_connector +from sglang.srt.utils import is_remote_url, logger, lru_cache_frozenset + +_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { + ChatGLMConfig.model_type: ChatGLMConfig, + DbrxConfig.model_type: DbrxConfig, + ExaoneConfig.model_type: ExaoneConfig, + DeepseekVL2Config.model_type: DeepseekVL2Config, + MultiModalityConfig.model_type: MultiModalityConfig, + KimiVLConfig.model_type: KimiVLConfig, + InternVLChatConfig.model_type: InternVLChatConfig, + Step3VLConfig.model_type: Step3VLConfig, + LongcatFlashConfig.model_type: LongcatFlashConfig, + Qwen3NextConfig.model_type: Qwen3NextConfig, +} + +for name, cls in _CONFIG_REGISTRY.items(): + with contextlib.suppress(ValueError): + AutoConfig.register(name, cls) + + +def download_from_hf( + model_path: str, + allow_patterns: Optional[Union[str, list]] = None, +): + if os.path.exists(model_path): + return model_path + + if not allow_patterns: + allow_patterns = ["*.json", "*.bin", "*.model"] + + return snapshot_download(model_path, allow_patterns=allow_patterns) + + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + if config.architectures is not None: + class_name = config.architectures[0] + if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"): + # We support non-hf version of llava models, so we do not want to + # read the wrong values from the unused default text_config. + # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as + # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`. + setattr(config, "torch_dtype", torch.float16) + return config + + if hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + return config.text_config + if hasattr(config, "language_config"): + return config.language_config + if hasattr(config, "thinker_config"): + # qwen2.5 omni + thinker_config = config.thinker_config + if hasattr(thinker_config, "text_config"): + setattr( + thinker_config.text_config, + "torch_dtype", + getattr(thinker_config, "torch_dtype", None), + ) + return thinker_config.text_config + return thinker_config + else: + return config + + +@lru_cache_frozenset(maxsize=32) +def get_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + model_override_args: Optional[dict] = None, + **kwargs, +): + is_gguf = check_gguf_file(model) + if is_gguf: + kwargs["gguf_file"] = model + model = Path(model).parent + + if is_remote_url(model): + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(model) + client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) + model = client.get_local_dir() + + config = AutoConfig.from_pretrained( + model, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + if ( + config.architectures is not None + and config.architectures[0] == "Phi4MMForCausalLM" + ): + # Phi4MMForCausalLM uses a hard-coded vision_config. See: + # https://github.com/vllm-project/vllm/blob/6071e989df1531b59ef35568f83f7351afb0b51e/vllm/model_executor/models/phi4mm.py#L71 + # We set it here to support cases where num_attention_heads is not divisible by the TP size. + from transformers import SiglipVisionConfig + + vision_config = { + "hidden_size": 1152, + "image_size": 448, + "intermediate_size": 4304, + "model_type": "siglip_vision_model", + "num_attention_heads": 16, + "num_hidden_layers": 26, # Model is originally 27-layer, we only need the first 26 layers for feature extraction. + "patch_size": 14, + } + config.vision_config = SiglipVisionConfig(**vision_config) + text_config = get_hf_text_config(config=config) + + if isinstance(model, str) and text_config is not None: + for key, val in text_config.__dict__.items(): + if not hasattr(config, key) and getattr(text_config, key, None) is not None: + setattr(config, key, val) + + if config.model_type in _CONFIG_REGISTRY: + config_class = _CONFIG_REGISTRY[config.model_type] + config = config_class.from_pretrained(model, revision=revision) + # NOTE(HandH1998): Qwen2VL requires `_name_or_path` attribute in `config`. + setattr(config, "_name_or_path", model) + + if isinstance(model, str) and config.model_type == "internvl_chat": + for key, val in config.llm_config.__dict__.items(): + if not hasattr(config, key): + setattr(config, key, val) + + if config.model_type == "multi_modality": + config.update({"architectures": ["MultiModalityCausalLM"]}) + + if model_override_args: + config.update(model_override_args) + + # Special architecture mapping check for GGUF models + if is_gguf: + if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: + raise RuntimeError(f"Can't get gguf config for {config.model_type}.") + model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type] + config.update({"architectures": [model_type]}) + + return config + + +@lru_cache_frozenset(maxsize=32) +def get_generation_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + **kwargs, +): + try: + return GenerationConfig.from_pretrained( + model, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + except OSError as e: + return None + + +# Qwen-1M related +def get_sparse_attention_config( + model: str, + sparse_attention_config_filename: str = "sparse_attention_config.json", +) -> Dict[str, Any]: + is_local = os.path.isdir(model) + if not is_local: + # Download the config files. + model = download_from_hf(model, allow_patterns=["*.json"]) + + config_file = os.path.join(model, sparse_attention_config_filename) + if not os.path.exists(config_file): + return {} + + # Load the sparse attention config. + with open(config_file) as f: + config = json.load(f) + return config + + +# Models don't use the same configuration key for determining the maximum +# context length. Store them here so we can sanely check them. +# NOTE: The ordering here is important. Some models have two of these and we +# have a preference for which value gets used. +CONTEXT_LENGTH_KEYS = [ + "max_sequence_length", + "seq_length", + "max_seq_len", + "model_max_length", + "max_position_embeddings", +] + + +def get_context_length(config): + """Get the context length of a model from a huggingface model configs.""" + text_config = config + rope_scaling = getattr(text_config, "rope_scaling", None) + if rope_scaling: + rope_scaling_factor = rope_scaling.get("factor", 1) + if "original_max_position_embeddings" in rope_scaling: + rope_scaling_factor = 1 + if rope_scaling.get("rope_type", None) == "llama3": + rope_scaling_factor = 1 + else: + rope_scaling_factor = 1 + + for key in CONTEXT_LENGTH_KEYS: + val = getattr(text_config, key, None) + if val is not None: + return int(rope_scaling_factor * val) + return 2048 + + +# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file. +_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer" + + +def get_tokenizer( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + **kwargs, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + """Gets a tokenizer for the given model name via Huggingface.""" + if tokenizer_name.endswith(".json"): + from sglang.srt.tokenizer.tiktoken_tokenizer import TiktokenTokenizer + + return TiktokenTokenizer(tokenizer_name) + + if tokenizer_mode == "slow": + if kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + kwargs["use_fast"] = False + + # TODO(Xinyuan): Remove this once we have a proper tokenizer for Devstral + if tokenizer_name == "mistralai/Devstral-Small-2505": + tokenizer_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" + + is_gguf = check_gguf_file(tokenizer_name) + if is_gguf: + kwargs["gguf_file"] = tokenizer_name + tokenizer_name = Path(tokenizer_name).parent + + if is_remote_url(tokenizer_name): + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(tokenizer_name) + client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) + tokenizer_name = client.get_local_dir() + + try: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + tokenizer_revision=tokenizer_revision, + clean_up_tokenization_spaces=False, + **kwargs, + ) + except TypeError as e: + # The LLaMA tokenizer causes a protobuf error in some environments. + err_msg = ( + "Failed to load the tokenizer. If you are using a LLaMA V1 model " + f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the " + "original tokenizer." + ) + raise RuntimeError(err_msg) from e + except ValueError as e: + # If the error pertains to the tokenizer class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + if not trust_remote_code and ( + "does not exist or is not currently imported." in str(e) + or "requires you to execute the tokenizer file" in str(e) + ): + err_msg = ( + "Failed to load the tokenizer. If the tokenizer is a custom " + "tokenizer not yet available in the HuggingFace transformers " + "library, consider setting `trust_remote_code=True` in LLM " + "or using the `--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + + if not isinstance(tokenizer, PreTrainedTokenizerFast): + warnings.warn( + "Using a slow tokenizer. This might cause a significant " + "slowdown. Consider using a fast tokenizer instead." + ) + + attach_additional_stop_token_ids(tokenizer) + return tokenizer + + +# Some models doesn't have an available processor, e.g.: InternVL +def get_tokenizer_from_processor(processor): + if isinstance(processor, PreTrainedTokenizerBase): + return processor + return processor.tokenizer + + +def get_processor( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + use_fast: Optional[bool] = True, + **kwargs, +): + # pop 'revision' from kwargs if present. + revision = kwargs.pop("revision", tokenizer_revision) + + config = AutoConfig.from_pretrained( + tokenizer_name, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + + # fix: for Qwen2-VL model, inject default 'size' if not provided. + if config.model_type in {"qwen2_vl"}: + if "size" not in kwargs: + kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520} + + if config.model_type not in {"llava", "clip"}: + kwargs["use_fast"] = use_fast + try: + if "InternVL3_5" in tokenizer_name: + processor = AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + else: + processor = AutoProcessor.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + + except ValueError as e: + error_message = str(e) + if "does not have a slow version" in error_message: + logger.info( + f"Processor {tokenizer_name} does not have a slow version. Automatically use fast version" + ) + kwargs["use_fast"] = True + processor = AutoProcessor.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + else: + raise e + tokenizer = get_tokenizer_from_processor(processor) + + attach_additional_stop_token_ids(tokenizer) + return processor + + +def attach_additional_stop_token_ids(tokenizer): + # Special handling for stop token <|eom_id|> generated by llama 3 tool use. + if "<|eom_id|>" in tokenizer.get_added_vocab(): + tokenizer.additional_stop_token_ids = set( + [tokenizer.get_added_vocab()["<|eom_id|>"]] + ) + else: + tokenizer.additional_stop_token_ids = None + + +def check_gguf_file(model: Union[str, os.PathLike]) -> bool: + """Check if the file is a GGUF model.""" + model = Path(model) + if not model.is_file(): + return False + elif model.suffix == ".gguf": + return True + + with open(model, "rb") as f: + header = f.read(4) + return header == b"GGUF" diff --git a/python/sglang/srt/host_shared_memory.py b/python/sglang/srt/host_shared_memory.py new file mode 100644 index 000000000..c599527f9 --- /dev/null +++ b/python/sglang/srt/host_shared_memory.py @@ -0,0 +1,83 @@ +import logging +import os +from dataclasses import dataclass +from multiprocessing import shared_memory +from pathlib import Path +from typing import List, Optional + +import numpy as np +import torch + +from sglang.srt.distributed.naive_distributed import get_naive_distributed +from sglang.srt.utils import check_cuda_result + +logger = logging.getLogger(__name__) + + +class HostSharedMemoryManager: + def __init__(self, base_name: str): + self._base_name = Path(base_name) + self._operation_index = 0 + self._records: List[_Record] = [] + + def malloc(self, *, shape, dtype): + meta_tensor = torch.empty(size=shape, dtype=dtype, device="meta") + raw = self._malloc_raw(num_bytes=meta_tensor.nbytes) + return raw.view(dtype).view(*shape) + + def _malloc_raw(self, *, num_bytes: int) -> torch.Tensor: + import cuda.bindings.runtime as cuda_rt + + self._operation_index += 1 + shm_name = f"{self._base_name}_op{self._operation_index}" + + # TODO handle dispose + if get_naive_distributed().get_rank() == 0: + shm = shared_memory.SharedMemory(name=shm_name, create=True, size=num_bytes) + + get_naive_distributed().barrier() + + if get_naive_distributed().get_rank() != 0: + shm = shared_memory.SharedMemory(name=shm_name) + + np_array = np.ndarray((num_bytes,), dtype=np.uint8, buffer=shm.buf) + tensor = torch.from_numpy(np_array) + + check_cuda_result( + cuda_rt.cudaHostRegister( + tensor.data_ptr(), num_bytes, cuda_rt.cudaHostRegisterPortable + ) + ) + + get_naive_distributed().barrier() + + self._records.append( + _Record( + shm=shm, + np_array=np_array, + tensor=tensor, + ) + ) + return tensor + + +@dataclass +class _Record: + shm: shared_memory.SharedMemory + np_array: np.ndarray + tensor: torch.Tensor + + +# Can have multi instances if needed +_instance: Optional[HostSharedMemoryManager] = None + + +def get_host_shared_memory_manager(): + assert _instance is not None + return _instance + + +def set_host_shared_memory_manager(instance: HostSharedMemoryManager): + global _instance + assert _instance is None + _instance = instance diff --git a/python/sglang/srt/layers/activation.py b/python/sglang/srt/layers/activation.py new file mode 100644 index 000000000..3d973393e --- /dev/null +++ b/python/sglang/srt/layers/activation.py @@ -0,0 +1,272 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Fused operators for activation layers.""" + +import logging +import math +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import PretrainedConfig + +from sglang.srt.custom_op import CustomOp +from sglang.srt.distributed import ( + divide, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.utils import ( + cpu_has_amx_support, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_xpu, + set_weight_attrs, +) +from sglang.utils import resolve_obj_by_qualname + +_is_cuda = is_cuda() +_is_npu = is_npu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +_is_hip = is_hip() +_is_xpu = is_xpu() + +if _is_cuda or _is_xpu: + from sgl_kernel import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul +elif _is_hip: + from sgl_kernel import gelu_and_mul, gelu_quick, gelu_tanh_and_mul, silu_and_mul + +if is_npu(): + import torch_npu + +logger = logging.getLogger(__name__) + + +class SiluAndMul(CustomOp): + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + output_shape = x.shape[:-1] + (d,) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + silu_and_mul(x, out) + return out + + def forward_cpu(self, x: torch.Tensor) -> torch.Tensor: + if _is_cpu_amx_available: + out = torch.ops.sgl_kernel.silu_and_mul_cpu(x) + return out + else: + return self.forward_native(x) + + def forward_npu(self, x: torch.Tensor) -> torch.Tensor: + out = torch_npu.npu_swiglu(x) + return out + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + output_shape = x.shape[:-1] + (d,) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + silu_and_mul(x, out) + return out + + +class GeluAndMul(CustomOp): + def __init__(self, approximate="tanh"): + super().__init__() + self.approximate = approximate + + def _forward_impl(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + output_shape = x.shape[:-1] + (d,) + out = torch.empty(output_shape, dtype=x.dtype, device=x.device) + if self.approximate == "tanh": + gelu_tanh_and_mul(x, out) + elif self.approximate == "none": + gelu_and_mul(x, out) + else: + raise RuntimeError("GeluAndMul only support tanh or none") + return out + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:] + + def forward_cpu(self, x: torch.Tensor) -> torch.Tensor: + if _is_cpu_amx_available and self.approximate == "tanh": + return torch.ops.sgl_kernel.gelu_tanh_and_mul_cpu(x) + elif _is_cpu_amx_available and self.approximate == "none": + return torch.ops.sgl_kernel.gelu_and_mul_cpu(x) + else: + return self.forward_native(x) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + return self._forward_impl(x) + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + return self._forward_impl(x) + + def forward_npu(self, x: torch.Tensor) -> torch.Tensor: + y_npu, gelu_npu = torch_npu.npu_geglu( + x, + dim=-1, + approximate=1 if self.approximate == "tanh" else 0, + activate_left=True, + ) + return y_npu + + +class NewGELU(CustomOp): + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + c = math.sqrt(2.0 / math.pi) + return 0.5 * x * (1.0 + torch.tanh(c * (x + 0.044715 * torch.pow(x, 3.0)))) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + # TODO: Implement the CUDA kernel for NewGELU in sgl-kernel + return self.forward_native(x) + + +class ReLU2(nn.Module): + """ + Applies the squared Rectified Linear Unit function. + y = max(0, x)^2 + """ + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = F.relu(x) + return x * x + + +class QuickGELU(CustomOp): + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + return x * torch.sigmoid(1.702 * x) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) + + def forward_hip(self, x: torch.Tensor) -> torch.Tensor: + out = torch.empty(x.shape, dtype=x.dtype, device=x.device) + gelu_quick(x, out) + return out + + def forward_npu(self, x: torch.Tensor) -> torch.Tensor: + return torch_npu.npu_fast_gelu(x) + + +class ScaledActivation(nn.Module): + """An activation function with post-scale parameters. + + This is used for some quantization methods like AWQ. + """ + + def __init__( + self, + act_module: nn.Module, + intermediate_size: int, + input_is_parallel: bool = True, + params_dtype: Optional[torch.dtype] = None, + ): + super().__init__() + self.act = act_module + self.input_is_parallel = input_is_parallel + if input_is_parallel: + tp_size = get_tensor_model_parallel_world_size() + intermediate_size_per_partition = divide(intermediate_size, tp_size) + else: + intermediate_size_per_partition = intermediate_size + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.scales = nn.Parameter( + torch.empty(intermediate_size_per_partition, dtype=params_dtype) + ) + set_weight_attrs(self.scales, {"weight_loader": self.weight_loader}) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.act(x) / self.scales + + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor): + param_data = param.data + if self.input_is_parallel: + tp_rank = get_tensor_model_parallel_rank() + shard_size = param_data.shape[0] + start_idx = tp_rank * shard_size + loaded_weight = loaded_weight.narrow(0, start_idx, shard_size) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +_ACTIVATION_REGISTRY = { + "gelu": nn.GELU(), + "gelu_pytorch_tanh": nn.GELU(approximate="tanh"), + "gelu_new": NewGELU(), + "relu2": ReLU2(), +} + + +def get_act_fn( + act_fn_name: str, + quant_config: Optional[QuantizationConfig] = None, + intermediate_size: Optional[int] = None, + input_is_parallel: bool = True, + params_dtype: Optional[torch.dtype] = None, +) -> nn.Module: + """Get an activation function by name.""" + act_fn_name = act_fn_name.lower() + if act_fn_name not in _ACTIVATION_REGISTRY: + raise ValueError(f"Activation function {act_fn_name!r} is not supported.") + + act_fn = _ACTIVATION_REGISTRY[act_fn_name] + if quant_config is not None and act_fn_name in quant_config.get_scaled_act_names(): + if intermediate_size is None: + raise ValueError( + "intermediate_size must be specified for scaled " + "activation functions." + ) + return ScaledActivation( + act_fn, intermediate_size, input_is_parallel, params_dtype + ) + return act_fn + + +def get_cross_encoder_activation_function(config: PretrainedConfig): + if ( + hasattr(config, "sbert_ce_default_activation_function") + and config.sbert_ce_default_activation_function is not None + ): + + function_name = config.sbert_ce_default_activation_function + assert function_name.startswith("torch.nn.modules."), ( + "Loading of activation functions is restricted to " + "torch.nn.modules for security reasons" + ) + return resolve_obj_by_qualname(function_name)() + else: + # adapt bge-reranker + return nn.Identity() + + +if not ( + _is_cuda or _is_npu or (_is_cpu and _is_cpu_amx_available) or _is_hip or _is_xpu +): + logger.info( + "sgl-kernel is not available on Non-NV, Non-AMD platforms or Non-AMX CPUs. Fallback to other kernel libraries." + ) + from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul diff --git a/python/sglang/srt/layers/amx_utils.py b/python/sglang/srt/layers/amx_utils.py new file mode 100644 index 000000000..df2a05ba5 --- /dev/null +++ b/python/sglang/srt/layers/amx_utils.py @@ -0,0 +1,86 @@ +import logging + +import torch + +from sglang.srt.utils import cpu_has_amx_support + +logger = logging.getLogger(__name__) + + +def amx_process_weight_after_loading(weight): + if weight.device != torch.device("cpu"): + return weight + if not cpu_has_amx_support(): + return weight + + return torch.ops.sgl_kernel.convert_weight_packed(weight) + + +# TODO: currently gemm kernel has the below requirements: +# OC % TILE_N == 0, where TILE_N = 16 +# IC % TILE_K == 0, where TILE_K = 32 +def dim_is_supported(weight): + TILE_N = 16 + TILE_K = 32 + ndim = weight.ndim + OC = weight.size(1) if ndim == 3 else weight.size(0) + IC = weight.size(2) if ndim == 3 else weight.size(1) + return OC % TILE_N == 0 and IC % TILE_K == 0 + + +def _amx_process_weight_after_loading( + module, weight_names, transpose_dims=None +) -> None: + # Pack weight for get better performance on CPU + devices = {getattr(module, weight_name).device for weight_name in weight_names} + assert len(devices) == 1, f"Expects all weights to be on the same device" + device = devices.pop() + + if transpose_dims: + assert len(weight_names) == len( + transpose_dims + ), "len(weight_names) should be equal to len(transpose_dims)" + + for i, weight_name in enumerate(weight_names): + weight_tensor = getattr(module, weight_name) + + if transpose_dims and transpose_dims[i]: + weight_tensor = weight_tensor.transpose(*transpose_dims[i]) + + # We don't pack weight or use intel amx backend if any weight of this module has unsupported dim. + if not dim_is_supported(weight_tensor): + logger.warning( + f"Unsupported dimension for prepacking for weight '{weight_name}' with shape {weight_tensor.shape} in {module}. " + f"The derived (OC, IC) dimensions must be divisible by (16, 32). " + ) + module.use_intel_amx_backend = False + return + + packed_weight = torch.nn.Parameter( + amx_process_weight_after_loading(weight_tensor), + requires_grad=False, + ) + packed_weight.__dict__ = weight_tensor.__dict__ + setattr(module, weight_name, packed_weight) + + module.use_intel_amx_backend = ( + device == torch.device("cpu") and cpu_has_amx_support() + ) + + if ( + module.use_intel_amx_backend + and hasattr(module, "bias") + and module.bias is not None + ): + module.bias = torch.nn.Parameter(module.bias.data.float(), requires_grad=False) + + +class PackWeightMethod: + def __init__(self, weight_names, transpose_dims=None): + self.weight_names = weight_names + self.transpose_dims = transpose_dims + + def process_weights_after_loading(self, module) -> None: + _amx_process_weight_after_loading( + module, self.weight_names, self.transpose_dims + ) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py new file mode 100644 index 000000000..188d772c7 --- /dev/null +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -0,0 +1,1179 @@ +from __future__ import annotations + +""" +end to end attention solution with aiter kernels +""" + +import math +import os +from dataclasses import dataclass +from enum import Enum, auto +from functools import partial +from typing import TYPE_CHECKING, List, Optional, Union + +import torch +import triton +import triton.language as tl + +from sglang.global_config import global_config +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.layers.dp_attention import ( + get_attention_tp_size, + is_dp_attention_enabled, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + +try: + from aiter import ( + flash_attn_varlen_func, + mha_batch_prefill_func, + paged_attention_ragged, + ) + from aiter.mla import mla_decode_fwd, mla_prefill_fwd +except ImportError: + print( + "aiter is AMD specific kernel library. Please make sure aiter is installed on your AMD device." + ) + +from sglang.srt.configs.model_config import AttentionArch + + +class WrapperDispatch(Enum): + SLIDING_WINDOW = auto() + CROSS_ATTENTION = auto() + + +@dataclass +class ForwardMetadata: + kv_indptr: torch.Tensor + kv_indices: torch.Tensor + qo_indptr: torch.Tensor + kv_last_page_len: torch.Tensor + max_q_len: int + max_kv_len: Optional[int] + + +global_workspace_buffer = None + +_AITER_PARTITION_SIZE_ROCM = 256 + + +class AiterAttnBackend(AttentionBackend): + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + ): + super().__init__() + # Lazy import to avoid the initialization of cuda context + from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + ) + + self.extend_attention_fwd = torch.compiler.disable(extend_attention_fwd) + + self.device = model_runner.device + self.is_multimodal = model_runner.model_config.is_multimodal + self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens + self.speculative_num_steps = model_runner.server_args.speculative_num_steps + self.num_head = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.head_dim = model_runner.model_config.head_dim + self.v_head_dim = model_runner.token_to_kv_pool.get_value_buffer(0).shape[-1] + self.num_kv_head = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.kv_cache_dtype = model_runner.kv_cache_dtype + + self.req_to_token = model_runner.req_to_token_pool.req_to_token + + self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA + + # Parse constants + self.max_context_len = model_runner.model_config.context_len + self.skip_prefill = skip_prefill + + max_bs = model_runner.req_to_token_pool.size + + if kv_indptr_buf is None: + self.kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + else: + self.kv_indptr = kv_indptr_buf + + self.kv_last_page_len = torch.ones( + (max_bs,), dtype=torch.int32, device=model_runner.device + ) + self.qo_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + + # Create prefill indices updater + if not skip_prefill: + self.indices_updater_prefill = AiterIndicesUpdaterPrefill( + model_runner, self + ) + if self.use_mla: + self.mla_indices_updater_prefill = AiterMlaIndicesUpdaterPrefill( + model_runner, self + ) + + # aiter kernel related initialization + self.max_num_partitions = ( + self.max_context_len + _AITER_PARTITION_SIZE_ROCM - 1 + ) // _AITER_PARTITION_SIZE_ROCM + + nbyes_per_qo_elem = torch.finfo(torch.float32).bits // 8 + + if not self.use_mla: + self.workspace_buffer = torch.empty( + (max_bs * self.num_head * self.max_num_partitions * self.head_dim) + * nbyes_per_qo_elem + + 2 * (max_bs * self.num_head * self.max_num_partitions) * 4, + dtype=torch.uint8, + device=self.device, + ) + + self.scale = float(1.0 / (self.head_dim**0.5)) + self.k_scale = self.v_scale = torch.tensor([1.0], dtype=torch.float32).to( + self.device + ) + + self.logits_soft_cap = 0.0 + + self.forward_metadata: ForwardMetadata = None + + if self.use_mla: + self.qo_indptr_ = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + + self.enable_dp_attention = is_dp_attention_enabled() + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init auxiliary variables for triton attention backend.""" + + bs = forward_batch.batch_size + kv_indptr = self.kv_indptr + spec_info = forward_batch.spec_info + qo_indptr = None + kv_last_page_len = None + max_q_len = None + + if forward_batch.forward_mode.is_decode_or_idle(): + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + forward_batch.seq_lens_sum, dtype=torch.int32, device=self.device + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + bs = kv_indptr.shape[0] - 1 + + if self.use_mla: + qo_indptr = self.qo_indptr_[: bs + 1] + qo_indptr[1 : bs + 1] = torch.cumsum(self.kv_last_page_len[:bs], dim=0) + kv_last_page_len = self.kv_last_page_len[:bs] + max_q_len = 1 + + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + kv_last_page_len, + max_q_len, + None, + ) + + elif forward_batch.forward_mode.is_draft_extend(): + if self.use_mla: + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + self.req_to_token, + ) + ) + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + # self.mla_indices_updater_prefill.kv_last_page_len, + self.kv_last_page_len[:bs], + max(forward_batch.extend_seq_lens_cpu), + forward_batch.seq_lens_cpu.max().item(), + ) + else: + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens=None, + encoder_lens=forward_batch.encoder_lens, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = ForwardMetadata( + self.indices_updater_prefill.kv_indptr, + self.indices_updater_prefill.kv_indices, + None, + None, + self.indices_updater_prefill.max_q_len, + self.indices_updater_prefill.max_kv_len, + ) + elif forward_batch.forward_mode.is_target_verify(): + if self.use_mla: + draft_num = spec_info.draft_token_num + kv_lens = forward_batch.seq_lens + draft_num + kv_lens_sum = forward_batch.seq_lens_sum + draft_num * bs + device = forward_batch.seq_lens.device + + qo_indptr = torch.arange( + 0, + (1 + bs) * draft_num, + step=draft_num, + dtype=torch.int32, + device=device, + ) + kv_indptr = self.kv_indptr + kv_indptr[1 : bs + 1] = torch.cumsum(kv_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + kv_lens_sum, + dtype=torch.int32, + device=device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + kv_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + # self.mla_indices_updater_prefill.kv_last_page_len, + self.kv_last_page_len[:bs], + draft_num, + None, + ) + else: + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens=None, + encoder_lens=forward_batch.encoder_lens, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = ForwardMetadata( + self.indices_updater_prefill.kv_indptr, + self.indices_updater_prefill.kv_indices, + None, + None, + self.indices_updater_prefill.max_q_len, + self.indices_updater_prefill.max_kv_len, + ) + else: + prefix_lens = forward_batch.extend_prefix_lens + + if self.is_multimodal: + extend_no_prefix = False + else: + extend_no_prefix = not any(forward_batch.extend_prefix_lens_cpu) + if self.use_mla: + self.mla_indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + forward_batch.extend_seq_lens, + forward_batch.extend_seq_lens.max().item(), + forward_batch.seq_lens.max().item(), + spec_info=None, + ) + + kv_indices = self.mla_indices_updater_prefill.kv_indices + + self.forward_metadata = ForwardMetadata( + self.mla_indices_updater_prefill.kv_indptr, + kv_indices, + self.mla_indices_updater_prefill.qo_indptr, + self.kv_last_page_len[:bs], + self.mla_indices_updater_prefill.max_q_len, + self.mla_indices_updater_prefill.max_kv_len, + ) + else: + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens, + encoder_lens=forward_batch.encoder_lens, + spec_info=None, + ) + self.forward_metadata = ForwardMetadata( + self.indices_updater_prefill.kv_indptr, + self.indices_updater_prefill.kv_indices, + None, + None, + self.indices_updater_prefill.max_q_len, + self.indices_updater_prefill.max_kv_len, + ) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + self.cuda_graph_kv_last_page_len = torch.ones(max_bs, dtype=torch.int) + if kv_indices_buf is None: + self.cuda_graph_kv_indices = torch.zeros( + (max_bs * self.max_context_len), + dtype=torch.int32, + device=self.device, + ) + else: + self.cuda_graph_kv_indices = kv_indices_buf + + if not self.skip_prefill: + self.cuda_graph_custom_mask = torch.zeros( + (max_num_tokens * self.max_context_len), + dtype=torch.uint8, + device=self.device, + ) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + if forward_mode.is_decode_or_idle(): + qo_indptr = None + kv_last_page_len = None + max_q_len = None + + if spec_info is None: + kv_indptr = self.kv_indptr + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + + if self.use_mla: + qo_indptr = self.qo_indptr_[: bs + 1] + qo_indptr[1 : bs + 1] = torch.cumsum( + self.cuda_graph_kv_last_page_len[:bs], dim=0 + ) + kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs] + max_q_len = 1 + + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + kv_last_page_len, + max_q_len, + None, + ) + + elif forward_mode.is_target_verify(): + if self.use_mla: + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs] + max_q_len = self.num_draft_tokens + + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + kv_last_page_len, + max_q_len, + None, + ) + else: + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_prefill.update( + req_pool_indices, + seq_lens, + seq_lens_sum, + prefix_lens=None, + encoder_lens=encoder_lens, + spec_info=spec_info, + ) + self.forward_metadata = ForwardMetadata( + self.indices_updater_prefill.kv_indptr, + self.indices_updater_prefill.kv_indices, + None, + None, + self.indices_updater_prefill.max_q_len, + self.indices_updater_prefill.max_kv_len, + ) + elif forward_mode.is_draft_extend(): + num_tokens_per_bs = self.speculative_num_steps + 1 + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + bs * num_tokens_per_bs + 1, + step=num_tokens_per_bs, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + kv_last_page_len = self.cuda_graph_kv_last_page_len[:bs] + max_q_len = num_tokens_per_bs + self.forward_metadata = ForwardMetadata( + kv_indptr, + kv_indices, + qo_indptr, + kv_last_page_len, + max_q_len, + None, + ) + else: + raise ValueError(f"Invalid mode: {forward_mode=}") + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + if forward_mode.is_decode_or_idle(): + kv_indptr = self.kv_indptr + kv_indices = self.cuda_graph_kv_indices + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens[:bs], dim=0) + kv_indptr = kv_indptr[: bs + 1] + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens[:bs], + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + kv_indptr[: spec_info.kv_indptr.shape[0]] = spec_info.kv_indptr + kv_indices[: spec_info.kv_indices.shape[0]] = spec_info.kv_indices + + elif forward_mode.is_target_verify(): + bs = len(req_pool_indices) + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_lens = seq_lens + self.num_draft_tokens + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(kv_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + kv_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + elif forward_mode.is_draft_extend(): + seq_lens = seq_lens[:bs] + accept_lens = spec_info.accept_length[:bs] + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[1 : bs + 1] = torch.cumsum(accept_lens, dim=0) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + raise ValueError("Invalid forward mode") + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + cache_loc = ( + forward_batch.out_cache_loc + if not layer.is_cross_attention + else forward_batch.encoder_out_cache_loc + ) + + self.logits_soft_cap = layer.logit_cap + + if k is not None: + assert v is not None + if save_kv_cache: + if self.use_mla: + forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + else: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + + if self.use_mla: + max_q_len = self.forward_metadata.max_q_len + max_kv_len = self.forward_metadata.max_kv_len + kv_indptr = self.forward_metadata.kv_indptr + kv_indices = self.forward_metadata.kv_indices + qo_indptr = self.forward_metadata.qo_indptr + K_Buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + V_Buffer = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + kv_lora_rank = V_Buffer.shape[-1] + qk_rope_head_dim = K_Buffer.shape[-1] - kv_lora_rank + qk_nope_head_dim = k.shape[-1] - qk_rope_head_dim + assert len(q.shape) == 3 + assert len(k.shape) == 3 + assert len(v.shape) == 3 + + if forward_batch.forward_mode.is_extend(): + if kv_indices.shape[0] == 0: + o = flash_attn_varlen_func( + q, + k, + v, + qo_indptr, + qo_indptr, + max_q_len, + max_q_len, + softmax_scale=layer.scaling, + causal=True, + ) + return o + elif layer.qk_head_dim != (kv_lora_rank + qk_rope_head_dim): + K_Buffer = torch.index_select(K_Buffer, 0, kv_indices) + kvc, k_pe = torch.split( + K_Buffer, [kv_lora_rank, qk_rope_head_dim], dim=-1 + ) + kvprefix = layer.kv_b_proj(kvc.contiguous())[0] + + kvprefix = kvprefix.view( + -1, layer.tp_k_head_num, qk_nope_head_dim + layer.v_head_dim + ) + k_prefix, v_prefix = torch.split( + kvprefix, [qk_nope_head_dim, layer.v_head_dim], dim=-1 + ) + k_prefix = torch.cat( + [ + k_prefix, + torch.broadcast_to( + k_pe, + (k_pe.shape[0], layer.tp_k_head_num, k_pe.shape[2]), + ), + ], + dim=-1, + ) + assert ( + forward_batch.extend_prefix_lens.shape + == forward_batch.extend_seq_lens.shape + ) + + k = k_prefix + v = v_prefix + + o = flash_attn_varlen_func( + q, + k, + v, + qo_indptr, + kv_indptr, + max_q_len, + max_kv_len, + softmax_scale=layer.scaling, + causal=True, + ) + return o + + else: + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty( + (q.shape[0], layer.tp_q_head_num * layer.v_head_dim) + ) + else: + o = torch.empty_like(q) + + mla_prefill_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + K_Buffer.view(-1, 1, 1, layer.qk_head_dim), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + qo_indptr, + kv_indptr, + kv_indices, + self.forward_metadata.kv_last_page_len, + self.forward_metadata.max_q_len, + layer.scaling, + layer.logit_cap, + ) + K_Buffer = K_Buffer.view(-1, layer.tp_k_head_num, layer.qk_head_dim) + return o + elif forward_batch.forward_mode.is_target_verify(): + o = q.new_empty((q.shape[0], layer.tp_q_head_num, layer.v_head_dim)) + mla_decode_fwd( + q, + K_Buffer.view(-1, 1, 1, layer.qk_head_dim), + o, + self.forward_metadata.qo_indptr, + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.forward_metadata.kv_last_page_len, + self.forward_metadata.max_q_len, + layer.scaling, + layer.logit_cap, + ) + K_Buffer = K_Buffer.view(-1, 1, layer.qk_head_dim) + return o + elif forward_batch.forward_mode.is_draft_extend(): + o = q.new_empty((q.shape[0], layer.tp_q_head_num, layer.v_head_dim)) + causal = True + sliding_window_size = -1 + kv_indptr = self.forward_metadata.kv_indptr + kv_indices = self.forward_metadata.kv_indices + mla_prefill_fwd( + q, + K_Buffer.view(-1, 1, 1, layer.qk_head_dim), + o, + self.forward_metadata.qo_indptr, + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.forward_metadata.kv_last_page_len, + self.forward_metadata.max_q_len, + layer.scaling, + layer.logit_cap, + ) + K_Buffer = K_Buffer.view(-1, 1, layer.qk_head_dim) + return o + # self.extend_attention_fwd( + # q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + # k.contiguous(), + # v.contiguous(), + # o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + # forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + # forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + # self.forward_metadata.qo_indptr, + # kv_indptr, + # kv_indices, + # None, + # causal, + # None, + # self.forward_metadata.max_q_len, + # layer.scaling, + # layer.logit_cap, + # sliding_window_size, + # ) + # return o + else: + raise ValueError( + f"Invalid forward mode for MLA prefill: {forward_batch.forward_mode=}" + ) + else: + k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + layer.layer_id + ) + + bs0 = forward_batch.batch_size + 1 + + o = mha_batch_prefill_func( + q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + k_cache, + v_cache, + self.qo_indptr[:bs0], + self.forward_metadata.kv_indptr[:bs0], + self.forward_metadata.kv_indices, + self.forward_metadata.max_q_len, + self.forward_metadata.max_kv_len, + causal=True, + logits_soft_cap=self.logits_soft_cap, + alibi_slopes=None, + return_lse=False, + return_attn_probs=False, + ) + + return o.view(-1, layer.tp_q_head_num * layer.head_dim) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + if self.use_mla: + k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + mla_decode_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + k_buffer.view(-1, 1, 1, layer.qk_head_dim), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + self.forward_metadata.qo_indptr, + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.forward_metadata.kv_last_page_len, + self.forward_metadata.max_q_len, + layer.scaling, + layer.logit_cap, + ) + k_buffer = k_buffer.view(-1, 1, layer.qk_head_dim) + else: + self.logits_soft_cap = layer.logit_cap + paged_attention_ragged( + o.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + self.workspace_buffer, + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).view( + -1, 1, layer.tp_k_head_num, layer.qk_head_dim + ), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id).view( + -1, 1, layer.tp_v_head_num, layer.v_head_dim + ), + self.scale, + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.kv_last_page_len, + 1, + self.max_num_partitions, + None, + "auto", + "NHD", + self.logits_soft_cap, + self.k_scale, + self.v_scale, + None, + _AITER_PARTITION_SIZE_ROCM, + ) + + return o + + +class AiterIndicesUpdaterPrefill: + def __init__(self, model_runner: ModelRunner, attn_backend: AttentionBackend): + # Parse Constants + self.num_qo_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_heads = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.head_dim = model_runner.model_config.head_dim + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.sliding_window_size = model_runner.sliding_window_size + self.attn_backend = attn_backend + + # Buffers and wrappers + self.kv_indptr = attn_backend.kv_indptr + self.kv_last_page_len = attn_backend.kv_last_page_len + self.qo_indptr = attn_backend.qo_indptr + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.update = self.update_single_wrapper + + self.kv_indices = None + self.max_q_len = 0 + self.max_kv_len = 0 + + def update( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + prefix_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[SpecInfo], + ): + # Keep the signature for type checking. It will be assigned during runtime. + raise NotImplementedError() + + def update_single_wrapper( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + prefix_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[SpecInfo], + ): + + kv_start_idx = None + kv_indptr = self.kv_indptr + qo_indptr = self.qo_indptr + paged_kernel_lens = seq_lens + paged_kernel_lens_sum = seq_lens_sum + + bs = len(req_pool_indices) + if spec_info is None: + # Normal extend + kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + + # (TODO: Kk) WA - CI test_moe_eval_accuracy_large.py + # mha_batch_prefill reads 128 data to do computatoin + # if real data is not long enough then original padding value 0 is used + # but the 0 location will be made nan (noqa) in cuda graph capture mode + # this will cause the output tensor value becomes nan + # WA is to assure that last index of pool not changed + kv_indices = torch.empty( + paged_kernel_lens_sum + 256, + dtype=torch.int32, + device=req_pool_indices.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + kv_start_idx, + kv_indices, + self.req_to_token.shape[1], + ) + + token_num = kv_indptr[-1] + kv_indices[token_num:] = kv_indices[0] + + self.max_kv_len = torch.max(paged_kernel_lens).item() + + extend_lens = seq_lens - prefix_lens + self.max_q_len = torch.max(extend_lens).item() + + qo_indptr[1 : bs + 1] = torch.cumsum(extend_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + custom_mask = None + else: + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + self.req_to_token, + ) + ) + + self.kv_indices = kv_indices + + +class AiterMlaIndicesUpdaterPrefill: + def __init__(self, model_runner: ModelRunner, attn_backend: AttentionBackend): + # Parse Constants + self.attn_backend = attn_backend + + # Buffers and wrappers + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.update = self.update_single_wrapper + + self.kv_indptr = None + self.kv_indices = None + self.qo_indptr = None + self.kv_last_page_len = None + self.max_q_len = 0 + self.max_kv_len = 0 + + def update( + self, + req_pool_indices: torch.Tensor, + kv_lens: torch.Tensor, + kv_lens_sum: int, + extend_lens: torch.Tensor, + max_q_len: int, + max_kv_len: int, + spec_info: Optional[SpecInfo], + ): + # Keep the signature for type checking. It will be assigned during runtime. + raise NotImplementedError() + + def update_single_wrapper( + self, + req_pool_indices: torch.Tensor, + kv_lens: torch.Tensor, + kv_lens_sum: int, + extend_lens: torch.Tensor, + max_q_len: int, + max_kv_len: int, + spec_info: Optional[SpecInfo], + ): + bs = len(req_pool_indices) + + kv_indptr = self.attn_backend.kv_indptr + + if spec_info is None: + # Normal extend + kv_indptr[1 : bs + 1] = torch.cumsum(kv_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + kv_lens_sum, + dtype=torch.int32, + device=req_pool_indices.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + kv_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + qo_indptr = self.attn_backend.qo_indptr + qo_indptr[1 : bs + 1] = torch.cumsum(extend_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + else: + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + req_pool_indices, + kv_lens, + kv_lens_sum, + self.req_to_token, + ) + ) + + self.kv_indptr = kv_indptr + self.kv_indices = kv_indices + self.qo_indptr = qo_indptr + self.max_q_len = max_q_len + self.max_kv_len = max_kv_len + + +class AiterMultiStepDraftBackend: + """ + Wrap multiple triton attention backends as one for multiple consecutive + draft decoding steps. + """ + + def __init__( + self, + model_runner: ModelRunner, + topk: int, + speculative_num_steps: int, + ): + from sglang.srt.speculative.eagle_utils import generate_draft_decode_kv_indices + + self.topk = topk + self.speculative_num_steps = speculative_num_steps + self.generate_draft_decode_kv_indices = generate_draft_decode_kv_indices + max_bs = model_runner.req_to_token_pool.size * self.topk + self.kv_indptr = torch.zeros( + ( + self.speculative_num_steps, + max_bs + 1, + ), + dtype=torch.int32, + device=model_runner.device, + ) + self.attn_backends = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + AiterAttnBackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + ) + ) + self.max_context_len = self.attn_backends[0].max_context_len + self.num_head = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.device = model_runner.device + # Cached variables for generate_draft_decode_kv_indices + self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] + self.page_size = model_runner.server_args.page_size + assert self.page_size == 1, "Page size must be 1" + + def common_template( + self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int + ): + num_seqs = forward_batch.batch_size + bs = self.topk * num_seqs + seq_lens_sum = forward_batch.seq_lens_sum + + self.generate_draft_decode_kv_indices[ + (self.speculative_num_steps, num_seqs, self.topk) + ]( + forward_batch.req_pool_indices, + forward_batch.req_to_token_pool.req_to_token, + forward_batch.seq_lens, + kv_indices_buffer, + self.kv_indptr, + forward_batch.positions, + self.pool_len, + kv_indices_buffer.shape[1], + self.kv_indptr.shape[1], + triton.next_power_of_2(num_seqs), + triton.next_power_of_2(self.speculative_num_steps), + triton.next_power_of_2(bs), + self.page_size, + ) + + for i in range(self.speculative_num_steps): + forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] + forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ + : seq_lens_sum * self.topk + bs * (i + 1) + ] + call_fn(i, forward_batch) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + kv_indices = torch.empty( + ( + self.speculative_num_steps, + forward_batch.batch_size * self.topk * self.max_context_len, + ), + dtype=torch.int32, + device=self.device, + ) + + def call_fn(i, forward_batch): + forward_batch.spec_info.kv_indptr = ( + forward_batch.spec_info.kv_indptr.clone() + ) + forward_batch.spec_info.kv_indices = ( + forward_batch.spec_info.kv_indices.clone() + ) + self.attn_backends[i].init_forward_metadata(forward_batch) + + self.common_template(forward_batch, kv_indices, call_fn) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.cuda_graph_kv_indices = torch.zeros( + (self.speculative_num_steps, max_num_tokens * self.max_context_len), + dtype=torch.int32, + device=self.device, + ) + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state( + max_bs, max_num_tokens, kv_indices_buf=self.cuda_graph_kv_indices[i] + ) + + def init_forward_metadata_capture_cuda_graph(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=-1, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=None, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) diff --git a/python/sglang/srt/layers/attention/ascend_backend.py b/python/sglang/srt/layers/attention/ascend_backend.py new file mode 100644 index 000000000..6d5ed0a5c --- /dev/null +++ b/python/sglang/srt/layers/attention/ascend_backend.py @@ -0,0 +1,577 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional + +import torch +import torch_npu +from torch.nn.functional import scaled_dot_product_attention + +from sglang.srt.configs.model_config import AttentionArch +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.torch_native_backend import TorchNativeAttnBackend +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.utils import get_bool_env_var + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + +import os + +import numpy as np + + +@dataclass +class ForwardMetadata: + + # calculated map for kv positions [bs * maxseqlen] + block_tables: Optional[torch.Tensor] = None + + # seq len inputs + extend_seq_lens_cpu_int: Optional[torch.Tensor] = None + seq_lens_cpu_int: Optional[torch.Tensor] = None + seq_lens_cpu_list: Optional[List[int]] = None + seq_lens_list_cumsum: Optional[List[int]] = None + + +class AscendAttnBackend(AttentionBackend): + + def gen_attention_mask(self, max_seq_len: int, dtype=torch.float16): + mask_flag = torch.tril( + torch.ones((max_seq_len, max_seq_len), dtype=torch.bool) + ).view(max_seq_len, max_seq_len) + mask_flag = ~mask_flag + if dtype == torch.float16: + mask_value = torch.finfo(torch.float32).min + else: + mask_value = 1 + self.mask = ( + torch.masked_fill( + torch.zeros(size=(max_seq_len, max_seq_len)), mask_flag, mask_value + ) + .to(dtype) + .to(self.device) + ) + self.mask_len = max_seq_len + + def __init__(self, model_runner: ModelRunner): + super().__init__() + self.forward_metadata = None + self.device = model_runner.device + self.page_size = model_runner.page_size + self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA + if self.use_mla: + self.kv_lora_rank = model_runner.model_config.kv_lora_rank + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.native_attn = TorchNativeAttnBackend(model_runner) + self.graph_metadata = {} + self.max_context_len = model_runner.model_config.context_len + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.graph_mode = False + self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False") + if not self.use_fia: + self.gen_attention_mask(128, model_runner.dtype) + mask_length = 2048 + self.fia_mask = ~torch.tril( + torch.ones( + (mask_length, mask_length), + dtype=torch.bool, + device=model_runner.device, + ) + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init the metadata for a forward pass.""" + tp_size = get_attention_tp_size() + self.forward_metadata = ForwardMetadata() + + self.forward_metadata.block_tables = ( + forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : forward_batch.seq_lens.max() + ][:, :: self.page_size] + // self.page_size + ) + if forward_batch.extend_seq_lens is not None: + self.forward_metadata.extend_seq_lens_cpu_int = ( + forward_batch.extend_seq_lens.cpu().int() + ) + self.forward_metadata.seq_lens_cpu_int = forward_batch.seq_lens_cpu.int() + + seq_lens_list_cumsum = np.cumsum(forward_batch.extend_seq_lens_cpu) + if forward_batch.is_extend_in_batch: + seq_lens_list_cumsum[-1] = ( + (seq_lens_list_cumsum[-1] - 1) // tp_size + 1 + ) * tp_size + self.forward_metadata.seq_lens_list_cumsum = seq_lens_list_cumsum + + self.graph_mode = False + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.graph_metadata = { + "block_tables": torch.empty( + (max_bs, self.max_context_len // self.page_size), + dtype=torch.int32, + device=self.device, + ), + } + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + metadata = ForwardMetadata() + + metadata.block_tables = self.graph_metadata["block_tables"][:bs, :] + metadata.seq_lens_cpu_list = seq_lens.cpu().int().tolist() + + self.graph_metadata[bs] = metadata + self.forward_metadata = metadata + + self.graph_mode = True + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + metadata = self.graph_metadata[bs] + max_len = seq_lens_cpu[:bs].max().item() + max_seq_pages = (max_len + self.page_size - 1) // self.page_size + + metadata.block_tables[:bs, :max_seq_pages].copy_( + self.req_to_token[req_pool_indices[:bs], :max_len][:, :: self.page_size] + // self.page_size + ) + metadata.block_tables[:bs, max_seq_pages:].fill_(0) + metadata.block_tables[bs:, :].fill_(0) + + self.forward_metadata = metadata + + self.graph_mode = True + + def get_cuda_graph_seq_len_fill_value(self): + return 0 + + def forward_extend( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + ): + if not self.use_mla: + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + + if self.use_fia: + """FIA will support multi-bs in the later version of CANN""" + q = q.reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) + attn_output = torch.empty( + (q.size(0), layer.tp_q_head_num, layer.v_head_dim), + device=q.device, + dtype=q.dtype, + ) + q_len_offset = 0 + for q_len in forward_batch.extend_seq_lens_cpu: + attn_output[q_len_offset : q_len_offset + q_len] = ( + torch.ops.npu.npu_fused_infer_attention_score( + q[None, q_len_offset : q_len_offset + q_len], + k[None, q_len_offset : q_len_offset + q_len], + v[None, q_len_offset : q_len_offset + q_len], + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="BSND", # todo, TND not supports q_heads!=k_heads + atten_mask=self.fia_mask.unsqueeze(0), + sparse_mode=3, + scale=layer.scaling, + next_tokens=0, + )[0] + ) + q_len_offset += q_len + attn_output = attn_output.view( + -1, layer.tp_q_head_num * layer.v_head_dim + ) + + else: + if layer.qk_head_dim <= 128: + query = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + attn_output = torch.empty( + (query.shape[0], layer.tp_q_head_num * layer.v_head_dim), + dtype=query.dtype, + device=query.device, + ) + + torch_npu._npu_flash_attention_qlens( + query=query, + key_cache=k_cache, + value_cache=v_cache, + mask=self.mask, + block_table=self.forward_metadata.block_tables, + seq_len=self.forward_metadata.extend_seq_lens_cpu_int, + context_lens=self.forward_metadata.seq_lens_cpu_int, + scale_value=layer.scaling, + num_heads=layer.tp_q_head_num, + num_kv_heads=layer.tp_k_head_num, + out=attn_output, + ) + else: + if layer.qk_head_dim != layer.v_head_dim: + attn_output = q.new_empty( + (q.shape[0], layer.tp_q_head_num * layer.v_head_dim) + ) + else: + attn_output = torch.empty_like(q) + + use_gqa = layer.tp_q_head_num != layer.tp_k_head_num + + q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim) + o_ = attn_output.view(-1, layer.tp_q_head_num, layer.v_head_dim) + + causal = True + if ( + layer.is_cross_attention + or layer.attn_type == AttentionType.ENCODER_ONLY + ): + causal = False + + self.native_attn._run_sdpa_forward_extend( + q_, + o_, + k_cache.view(-1, layer.tp_k_head_num, layer.qk_head_dim), + v_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.extend_prefix_lens, + forward_batch.extend_seq_lens, + scaling=layer.scaling, + enable_gqa=use_gqa, + causal=causal, + ) + else: + assert ( + layer.qk_head_dim != layer.v_head_dim + ), "FIA only supports qk_head_dim != v_head_dim" + q_nope, q_rope = q.split([layer.v_head_dim, self.qk_rope_head_dim], dim=-1) + k_nope, k_rope = k.split([layer.v_head_dim, self.qk_rope_head_dim], dim=-1) + + attn_output, _ = torch.ops.npu.npu_fused_infer_attention_score( + q_nope, + k_nope, + v, + query_rope=q_rope, + key_rope=k_rope, + num_heads=layer.tp_q_head_num, + input_layout="TND", + atten_mask=self.fia_mask, + sparse_mode=3, + actual_seq_lengths=self.forward_metadata.seq_lens_list_cumsum, + actual_seq_lengths_kv=self.forward_metadata.seq_lens_list_cumsum, + scale=layer.scaling, + next_tokens=0, + ) + + return attn_output + + def forward_decode_graph( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ): + if save_kv_cache: + if self.use_mla: + k = k.view(-1, layer.tp_k_head_num, self.kv_lora_rank) + k_rope = k_rope.view(-1, layer.tp_k_head_num, self.qk_rope_head_dim) + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, k_rope + ) + else: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + if not self.use_mla: + k_cache = forward_batch.token_to_kv_pool.get_key_buffer( + layer.layer_id + ).view(-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim) + v_cache = forward_batch.token_to_kv_pool.get_value_buffer( + layer.layer_id + ).view(-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim) + query = q.reshape(-1, 1, layer.tp_q_head_num * layer.qk_head_dim) + if self.forward_metadata.seq_lens_cpu_int is None: + actual_seq_len_kv = self.forward_metadata.seq_lens_cpu_list + else: + actual_seq_len_kv = ( + self.forward_metadata.seq_lens_cpu_int.cpu().int().tolist() + ) + num_tokens = query.shape[0] + workspace = torch_npu._npu_fused_infer_attention_score_get_max_workspace( + query, + k_cache, + v_cache, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="BSH", + scale=layer.scaling, + actual_seq_lengths_kv=actual_seq_len_kv, + ) + output = torch.empty( + (num_tokens, 1, layer.tp_q_head_num * layer.v_head_dim), + dtype=q.dtype, + device=q.device, + ) + softmax_lse = torch.empty(1, dtype=q.dtype, device=q.device) + torch_npu.npu_fused_infer_attention_score.out( + query, + k_cache, + v_cache, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="BSH", + scale=layer.scaling, + actual_seq_lengths_kv=actual_seq_len_kv, + workspace=workspace, + out=[output, softmax_lse], + ) + return output.view(num_tokens, layer.tp_q_head_num * layer.v_head_dim) + else: + c_kv, k_rope = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_rope_cache = k_rope.view( + -1, layer.tp_k_head_num, self.page_size, self.qk_rope_head_dim + ) + c_kv_cache = c_kv.view( + -1, layer.tp_v_head_num, self.page_size, self.kv_lora_rank + ) + + q_nope = q.view(-1, layer.tp_q_head_num, 1, self.kv_lora_rank).contiguous() + q_rope = q_rope.view(-1, layer.tp_q_head_num, 1, self.qk_rope_head_dim) + if self.forward_metadata.seq_lens_cpu_int is None: + actual_seq_len_kv = self.forward_metadata.seq_lens_cpu_list + else: + actual_seq_len_kv = ( + self.forward_metadata.seq_lens_cpu_int.cpu().int().tolist() + ) + + workspace = torch_npu._npu_fused_infer_attention_score_get_max_workspace( + q_nope, + c_kv_cache, + c_kv_cache, + query_rope=q_rope, + key_rope=k_rope_cache, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + input_layout="BNSD", + scale=layer.scaling, + actual_seq_lengths_kv=actual_seq_len_kv, + antiquant_mode=0, + antiquant_scale=None, + sparse_mode=0, + ) + output = torch.zeros_like(q_nope, dtype=q.dtype, device=q.device) + softmax_lse = torch.empty(1, dtype=q.dtype, device=q.device) + + torch_npu.npu_fused_infer_attention_score.out( + q_nope, + c_kv_cache, + c_kv_cache, + query_rope=q_rope, + key_rope=k_rope_cache, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + input_layout="BNSD", + scale=layer.scaling, + actual_seq_lengths_kv=actual_seq_len_kv, + antiquant_mode=0, + antiquant_scale=None, + sparse_mode=0, + workspace=workspace, + out=[output, softmax_lse], + ) + return output.view(-1, layer.tp_q_head_num * self.kv_lora_rank) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + # For multi-head latent attention + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ): + if self.graph_mode: + return self.forward_decode_graph( + q, + k, + v, + layer, + forward_batch, + save_kv_cache, + q_rope=q_rope, + k_rope=k_rope, + ) + + if not self.use_mla: + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + num_tokens = q.shape[0] + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + v_cache = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + if self.use_fia: + attn_output, _ = torch.ops.npu.npu_fused_infer_attention_score( + q.view( + forward_batch.batch_size, + -1, + layer.tp_q_head_num, + layer.qk_head_dim, + ), + k_cache.view( + -1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim + ), + v_cache.view( + -1, self.page_size, layer.tp_v_head_num * layer.qk_head_dim + ), + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="BSND", + atten_mask=None, + block_size=self.page_size, + block_table=self.forward_metadata.block_tables, + actual_seq_lengths_kv=self.forward_metadata.seq_lens_cpu_int, + scale=layer.scaling, + ) + else: + query = q.reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) + num_tokens = query.shape[0] + attn_output = torch.empty( + (num_tokens, layer.tp_q_head_num, layer.v_head_dim), + dtype=query.dtype, + device=query.device, + ) + + torch_npu._npu_paged_attention( + query=query, + key_cache=k_cache, + value_cache=v_cache, + num_heads=layer.tp_q_head_num, + num_kv_heads=layer.tp_k_head_num, + scale_value=layer.scaling, + block_table=self.forward_metadata.block_tables, + context_lens=self.forward_metadata.seq_lens_cpu_int, + out=attn_output, + ) + return attn_output.view(num_tokens, layer.tp_q_head_num * layer.v_head_dim) + else: + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, k_rope + ) + num_tokens = q.shape[0] + kv_c = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + k_pe = forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id) + + if self.use_fia and (layer.tp_q_head_num // layer.tp_k_head_num) >= 8: + """layer.tp_q_head_num // layer.tp_k_head_num < 8 will support in the later version of CANN""" + kv_c = kv_c.view( + -1, self.page_size, layer.tp_k_head_num * self.kv_lora_rank + ) + k_pe = k_pe.view( + -1, self.page_size, layer.tp_k_head_num * self.qk_rope_head_dim + ) + q = q.view( + forward_batch.batch_size, -1, layer.tp_q_head_num, self.kv_lora_rank + ) + q_rope = q_rope.view( + forward_batch.batch_size, + -1, + layer.tp_q_head_num, + self.qk_rope_head_dim, + ) + attn_output, _ = torch.ops.npu.npu_fused_infer_attention_score( + q, + kv_c, + kv_c, + query_rope=q_rope, + key_rope=k_pe, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="BSND", + atten_mask=None, + sparse_mode=0, + scale=layer.scaling, + antiquant_mode=0, + antiquant_scale=None, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + actual_seq_lengths_kv=self.forward_metadata.seq_lens_cpu_int, + ) + else: + assert ( + self.graph_mode == False + ) # _npu_paged_attention_mla not support graph mode + q = torch.cat([q, q_rope], dim=-1) + query = q.view(-1, layer.tp_q_head_num, layer.head_dim) + kv_c_and_k_pe_cache = torch.cat([kv_c, k_pe], dim=-1) + kv_c_and_k_pe_cache = kv_c_and_k_pe_cache.view( + -1, + self.page_size, + layer.tp_k_head_num, + self.kv_lora_rank + self.qk_rope_head_dim, + ) + attn_output = torch.empty( + [num_tokens, layer.tp_q_head_num, self.kv_lora_rank], + dtype=q.dtype, + device=q.device, + ) + torch_npu._npu_paged_attention_mla( + query=query, + key_cache=kv_c_and_k_pe_cache, + num_kv_heads=layer.tp_k_head_num, + num_heads=layer.tp_q_head_num, + scale_value=layer.scaling, + block_table=self.forward_metadata.block_tables, + context_lens=self.forward_metadata.seq_lens_cpu_int, + mla_vheadsize=self.kv_lora_rank, + out=attn_output, + ) + return attn_output.view(num_tokens, layer.tp_q_head_num * self.kv_lora_rank) diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py new file mode 100644 index 000000000..3025d0b11 --- /dev/null +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Optional, Union + +import torch + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + + +class AttentionBackend(ABC): + """The base class of attention backends""" + + @abstractmethod + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init the metadata for a forward pass.""" + raise NotImplementedError() + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + """Init the global shared states for cuda graph.""" + raise NotImplementedError() + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + """Init the metadata for a forward pass for capturing a cuda graph.""" + raise NotImplementedError() + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + """Init the metadata for a forward pass for replaying a cuda graph.""" + raise NotImplementedError() + + def get_cuda_graph_seq_len_fill_value(self): + """Get the fill value for padded seq lens. Typically, it is 0 or 1.""" + raise NotImplementedError() + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + """Run forward on an attention layer.""" + if forward_batch.forward_mode.is_idle(): + return q.new_empty(q.shape[0], layer.tp_q_head_num * layer.v_head_dim) + elif forward_batch.forward_mode.is_decode(): + return self.forward_decode( + q, + k, + v, + layer, + forward_batch, + save_kv_cache=save_kv_cache, + **kwargs, + ) + else: + return self.forward_extend( + q, + k, + v, + layer, + forward_batch, + save_kv_cache=save_kv_cache, + **kwargs, + ) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + ): + """Run a forward for decode.""" + raise NotImplementedError() + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + ): + """Run a forward for extend.""" + raise NotImplementedError() + + def support_triton(self): + """Check if the current backend supports triton.""" + return True diff --git a/python/sglang/srt/layers/attention/cutlass_mla_backend.py b/python/sglang/srt/layers/attention/cutlass_mla_backend.py new file mode 100644 index 000000000..eb0cae262 --- /dev/null +++ b/python/sglang/srt/layers/attention/cutlass_mla_backend.py @@ -0,0 +1,285 @@ +from __future__ import annotations + +""" +Support attention backend for Cutlass MLA. + +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import torch +import triton + +from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend +from sglang.srt.layers.attention.utils import create_flashmla_kv_indices_triton +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import is_cuda + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + +_is_cuda = is_cuda() +if _is_cuda: + from sgl_kernel import cutlass_mla_decode, cutlass_mla_get_workspace_size + + +# Cutlass MLA only supports pagesize=128 +PAGE_SIZE = 128 + + +@dataclass +class CutlassMLADecodeMetadata: + workspace: Optional[torch.Tensor] = None + block_kv_indices: Optional[torch.Tensor] = None + + def __init__( + self, + workspace: Optional[torch.Tensor] = None, + block_kv_indices: Optional[torch.Tensor] = None, + ): + self.workspace = workspace + self.block_kv_indices = block_kv_indices + + +class CutlassMLABackend(FlashInferMLAAttnBackend): + """Cutlass attention kernels.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + kv_last_page_len_buf: Optional[torch.Tensor] = None, + ): + super().__init__( + model_runner, skip_prefill, kv_indptr_buf, kv_last_page_len_buf + ) + + self.num_q_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_heads = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.num_local_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.forward_metadata: Union[CutlassMLADecodeMetadata] = None + self.kv_lora_rank = model_runner.model_config.kv_lora_rank + self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.v_head_dim = model_runner.model_config.v_head_dim + self.scaling = model_runner.model_config.scaling + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.kv_cache_dim = self.kv_lora_rank + self.qk_rope_head_dim + + def init_forward_metadata(self, forward_batch: ForwardBatch): + + bs = forward_batch.batch_size + spec_info = forward_batch.spec_info + if forward_batch.forward_mode.is_decode_or_idle(): + if spec_info is None: + max_seqlen_pad = triton.cdiv( + forward_batch.seq_lens_cpu.max().item(), PAGE_SIZE + ) + block_kv_indices = torch.full( + (bs, max_seqlen_pad), + -1, + dtype=torch.int32, + device=forward_batch.seq_lens.device, + ) + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + max_seqlen_pad, + PAGED_SIZE=PAGE_SIZE, + ) + workspace_size = cutlass_mla_get_workspace_size( + max_seqlen_pad * PAGE_SIZE, bs, num_kv_splits=1 + ) + workspace = torch.empty( + workspace_size, device="cuda", dtype=torch.uint8 + ) + self.forward_metadata = CutlassMLADecodeMetadata( + workspace, + block_kv_indices, + ) + else: + super().init_forward_metadata(forward_batch) + else: + super().init_forward_metadata(forward_batch) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + block_kv_indices: Optional[torch.Tensor] = None, + ): + if block_kv_indices is None: + cuda_graph_kv_indices = torch.full( + (max_bs, (self.max_context_len + PAGE_SIZE) // PAGE_SIZE), + 1, + dtype=torch.int32, + device="cuda", + ) + else: + cuda_graph_kv_indices = block_kv_indices + + workspace_size = cutlass_mla_get_workspace_size( + cuda_graph_kv_indices.shape[1] * PAGE_SIZE, max_bs, num_kv_splits=1 + ) + self.cuda_graph_mla_workspace = torch.empty( + workspace_size, device="cuda", dtype=torch.uint8 + ) + self.cuda_graph_kv_indices = cuda_graph_kv_indices + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + if forward_mode.is_decode_or_idle(): + if spec_info is None: + max_seqlen_pad = self.cuda_graph_kv_indices.shape[1] + + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + PAGED_SIZE=PAGE_SIZE, + ) + self.forward_metadata = CutlassMLADecodeMetadata( + self.cuda_graph_mla_workspace, + self.cuda_graph_kv_indices[:bs, :max_seqlen_pad], + ) + else: + super().init_forward_metadata_capture_cuda_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + encoder_lens, + forward_mode, + spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + + if forward_mode.is_decode_or_idle(): + assert seq_lens_cpu is not None + seq_lens = seq_lens[:bs] + + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + PAGED_SIZE=PAGE_SIZE, + ) + else: + super().init_forward_metadata_replay_cuda_graph( + bs, + req_pool_indices, + seq_lens, + seq_lens_sum, + encoder_lens, + forward_mode, + spec_info, + seq_lens_cpu, + ) + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + # For multi-head latent attention + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ): + cache_loc = forward_batch.out_cache_loc + + if k is not None: + assert v is not None + if save_kv_cache: + if k_rope is not None: + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, + cache_loc, + k, + k_rope, + ) + else: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, + cache_loc, + k, + v, + ) + + # Reshape inputs + if q_rope is not None: + q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + else: + reshaped_q = q.view(-1, layer.tp_q_head_num, layer.head_dim) + q_nope = reshaped_q[:, :, : layer.v_head_dim] + q_rope = reshaped_q[:, :, layer.v_head_dim :] + + q_nope = q_nope.to(self.q_data_type) + q_rope = q_rope.to(self.q_data_type) + + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + + o = cutlass_mla_decode( + q_nope=q_nope, + q_pe=q_rope, + kv_c_and_k_pe_cache=k_cache.view(-1, PAGE_SIZE, self.kv_cache_dim), + seq_lens=forward_batch.seq_lens.to(torch.int32), + page_table=self.forward_metadata.block_kv_indices, + workspace=self.forward_metadata.workspace, + sm_scale=layer.scaling, + num_kv_splits=1, + ) + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) diff --git a/python/sglang/srt/layers/attention/double_sparsity_backend.py b/python/sglang/srt/layers/attention/double_sparsity_backend.py new file mode 100644 index 000000000..47b867f61 --- /dev/null +++ b/python/sglang/srt/layers/attention/double_sparsity_backend.py @@ -0,0 +1,257 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + + +class DoubleSparseAttnBackend(AttentionBackend): + def __init__(self, model_runner: ModelRunner): + # Lazy import to avoid the initialization of cuda context + from sglang.srt.layers.attention.triton_ops.double_sparsity_attention import ( + extend_attention_fwd, + flash_decode_attention_fwd, + flash_decode_sparse_attention_fwd, + ) + + super().__init__() + + self.decode_attention_fwd = flash_decode_attention_fwd + self.decode_sparse_attention_fwd = flash_decode_sparse_attention_fwd + self.extend_attention_fwd = extend_attention_fwd + self.num_head = model_runner.model_config.num_attention_heads + self.head_dim = model_runner.model_config.hidden_size // self.num_head + self.heavy_token_num = model_runner.server_args.ds_heavy_token_num + + self.sorted_channels = model_runner.sorted_channels + self.sparse_decode_thresold = ( + model_runner.server_args.ds_sparse_decode_threshold + ) + self.att_out_approx: torch.Tensor = None + self.mid_out: torch.Tensor = None + self.mid_o_logexpsum: torch.Tensor = None + + # TODO: Change the hard-coded block_seq_num + self.BLOCK_SEQ = 128 + + if global_server_args_dict.get("triton_attention_reduce_in_fp32", False): + self.reduce_dtype = torch.float32 + else: + self.reduce_dtype = torch.float16 + + self.forward_metadata = None + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init auxiliary variables for triton attention backend.""" + + if forward_batch.forward_mode.is_decode(): + start_loc = torch.zeros_like(forward_batch.seq_lens, dtype=torch.int32) + start_loc[1:] = torch.cumsum(forward_batch.seq_lens[:-1], dim=0) + + total_num_tokens = torch.sum(forward_batch.seq_lens).item() + attn_logits = torch.empty( + (self.num_head, total_num_tokens), + dtype=self.reduce_dtype, + device="cuda", + ) + + max_seq_len = torch.max(forward_batch.seq_lens).item() + min_seq_len = torch.min(forward_batch.seq_lens).item() + max_extend_len = None + # NOTE: Align sequence order with req_to_token order + ds_req_to_token = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices + ] + + bsz = forward_batch.seq_lens.shape[0] + + att_out_approx = torch.empty( + [self.num_head, bsz, max_seq_len], + dtype=self.reduce_dtype, + device="cuda", + ) + + block_seq_num = ( + self.heavy_token_num + self.BLOCK_SEQ - 1 + ) // self.BLOCK_SEQ + + mid_out = torch.empty( + [bsz, self.num_head, block_seq_num, self.head_dim], + dtype=torch.float32, + device="cuda", + ) + mid_o_logexpsum = torch.empty( + [bsz, self.num_head, block_seq_num], dtype=torch.float32, device="cuda" + ) + self.att_out_approx = att_out_approx + self.mid_out = mid_out + self.mid_o_logexpsum = mid_o_logexpsum + + else: + start_loc = attn_logits = max_seq_len = min_seq_len = None + prefix_lens = forward_batch.extend_prefix_lens + max_extend_len = torch.max(forward_batch.seq_lens - prefix_lens).item() + ds_req_to_token = None + + self.forward_metadata = ( + start_loc, + attn_logits, + max_seq_len, + min_seq_len, + max_extend_len, + ds_req_to_token, + ) + + def forward_extend( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + k_label = torch.gather( + k, + 2, + self.sorted_channels[layer.layer_id] + .unsqueeze(0) + .expand(k.shape[0], -1, -1), + ) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v, k_label + ) + + ( + start_loc, + attn_logits, + max_seq_len, + min_seq_len, + max_extend_len, + ds_req_to_token, + ) = self.forward_metadata + self.extend_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + k.contiguous(), + v.contiguous(), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.extend_seq_lens, + forward_batch.extend_start_loc, + max_extend_len, + layer.scaling, + layer.logit_cap, + ) + return o + + def forward_decode( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # During torch.compile, there is a bug in rotary_emb that causes the + # output value to have a 3D tensor shape. This reshapes the output correctly. + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + # TODO: Add min seqlen + ( + start_loc, + attn_logits, + max_seq_len, + min_seq_len, + max_extend_len, + ds_req_to_token, + ) = self.forward_metadata + + k_label = torch.gather( + k, + 2, + self.sorted_channels[layer.layer_id] + .unsqueeze(0) + .expand(k.shape[0], -1, -1), + ) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v, k_label + ) + + # NOTE(Andy) shouldn't be used when max_len_in_batch < heavy_token_num + # and set a minimum value for sparse_decode + if ( + min_seq_len < self.heavy_token_num + or max_seq_len < self.sparse_decode_thresold + ): + self.decode_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + start_loc, + forward_batch.seq_lens, + attn_logits, + max_seq_len, + layer.scaling, + layer.logit_cap, + ) + else: + # TODO(Andy): indexing with torch.gather or torch.index_select or customized kernel + q_label = torch.gather( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + 2, + self.sorted_channels[layer.layer_id] + .unsqueeze(0) + .expand(q.shape[0], -1, -1), + ) + self.decode_sparse_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + o.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + q_label, + forward_batch.token_to_kv_pool.get_label_buffer(layer.layer_id), + ds_req_to_token, + forward_batch.seq_lens, + max_seq_len, + layer.scaling, + layer.logit_cap, + self.heavy_token_num, + self.att_out_approx, + self.mid_out, + self.mid_o_logexpsum, + self.BLOCK_SEQ, + ) + + return o diff --git a/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py new file mode 100644 index 000000000..84876b438 --- /dev/null +++ b/python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py @@ -0,0 +1,1700 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Attention layer with Dual chunk flash attention and sparse attention. +""" +import functools +import logging +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple + +import torch +import torch.nn.functional as F +from sgl_kernel.flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache +from sgl_kernel.sparse_flash_attn import ( + convert_vertical_slash_indexes, + convert_vertical_slash_indexes_mergehead, + sparse_attn_func, +) + +from sglang.srt.distributed.parallel_state import get_tensor_model_parallel_rank +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.flashattention_backend import FlashAttentionMetadata +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + + +logger = logging.getLogger(__name__) + + +@dataclass +class DualChunkFlashAttentionMetadata: + """Metadata for FlashAttentionBackend. + + NOTE: Any python object stored here is not updated when it is + cuda-graph replayed. If you have values that need to be changed + dynamically, it should be stored in tensor. The tensor has to be + updated from `CUDAGraphRunner.forward` API. + """ + + # (batch_size,). The sequence length per sequence. Sequence length means + # the computed tokens + new tokens None if it is a decoding. + seq_lens: Optional[List[int]] = None + # seq_lens stored as a tensor. + seq_lens_tensor: Optional[torch.Tensor] = None + # Maximum sequence length among prefill batch. 0 if there are decoding + # requests only. + max_seq_len: int = None + + # (batch_size,). The orig sequence length per sequence. + orig_seq_lens: Optional[List[int]] = None + + # orig_seq_lens stored as a tensor. + orig_seq_lens_tensor: Optional[torch.Tensor] = None + + # Block addresses per sequence. (Seq id -> list of physical block) + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks + # in the kv cache. Each block can contain up to block_size tokens. + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph + # captured. + block_tables: Optional[torch.Tensor] = None + + # (batch_size + 1,). The cumulative subquery lengths of the sequences in + # the batch, used to index into subquery. E.g., if the subquery length + # is [4, 6], it is [0, 4, 10]. + query_start_loc: Optional[torch.Tensor] = None + # (batch_size + 1,). The cumulative sequence lengths of the sequences in + # the batch, used to index into sequence. E.g., if the sequence length is + # [4, 6], it is [0, 4, 10]. + seq_start_loc: Optional[torch.Tensor] = None + + # Length scaling factor + scaling_factor: Optional[torch.Tensor] = None + + # (batch_size,). Sequence lengths for intra attention. + seq_lens_intra: Optional[torch.Tensor] = None + + # Max sequence length for intra attention. + max_seq_len_intra: Optional[int] = None + + # (batch_size, num_blocks). Block table for intra attention. + block_tables_intra: Optional[torch.Tensor] = None + + # (batch_size,). Sequence lengths for succ attention. + seq_lens_succ: Optional[torch.Tensor] = None + + # Max sequence length for succ attention. + max_seq_len_succ: Optional[int] = None + + # (batch_size, num_blocks). Block table for succ attention. + block_tables_succ: Optional[torch.Tensor] = None + + # (batch_size,). Sequence lengths for inter attention. + seq_lens_inter: Optional[torch.Tensor] = None + + # Max sequence length for inter attention. + max_seq_len_inter: Optional[int] = None + + +class DualChunkFlashAttentionBackend(AttentionBackend): + def __init__( + self, + model_runner: "ModelRunner", + ) -> None: + self.forward_metadata: FlashAttentionMetadata = None + self.device = model_runner.device + self.max_context_len = model_runner.model_config.context_len + self.num_heads = model_runner.model_config.get_num_attention_heads( + model_runner.server_args.tp_size + ) + self.num_kv_heads = model_runner.model_config.get_num_kv_heads( + model_runner.server_args.tp_size + ) + self.head_size = model_runner.model_config.head_dim + + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.kv_cache_dtype = model_runner.kv_cache_dtype + self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype + self.page_size = model_runner.page_size + + assert self.num_heads % self.num_kv_heads == 0 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + + dual_chunk_attention_config = getattr( + model_runner.model_config.hf_config, "dual_chunk_attention_config", None + ) + assert dual_chunk_attention_config is not None + self.chunk_size = dual_chunk_attention_config.get("chunk_size", 8192) + self.local_size = dual_chunk_attention_config.get("local_size", 1024) + self.original_max_position_embeddings = dual_chunk_attention_config.get( + "original_max_position_embeddings", 0 + ) + self.sparse_attention_config = dual_chunk_attention_config.get( + "sparse_attention_config", None + ) + if not self.sparse_attention_config: + logger.warning_once( + "Sparse attention will not be enabled as " + "sparse attention config is not provided." + ) + self.sparse_attention_enabled = dual_chunk_attention_config.get( + "sparse_attention_enabled", self.sparse_attention_config is not None + ) + self.sparse_attention_threshold = dual_chunk_attention_config.get( + "sparse_attention_threshold", 32768 + ) + self.sparse_attention_last_q = dual_chunk_attention_config.get( + "sparse_attention_last_q", 64 + ) + self.dual_chunk_attention_config = dual_chunk_attention_config + + if self.sparse_attention_enabled: + self.arange = torch.arange(self.sparse_attention_last_q, device="cuda") + self.last_q_mask = ( + self.arange[None, None, :, None] >= self.arange[None, None, None, :] + ) + + @functools.lru_cache() + def get_sparse_attention_config(self, layer_idx) -> List[Dict[str, Any]]: + layer_sparse_attention_config = { + int(i): j for i, j in self.sparse_attention_config[layer_idx].items() + } + start_head = self.num_heads * get_tensor_model_parallel_rank() + end_head = start_head + self.num_heads + return [layer_sparse_attention_config[i] for i in range(start_head, end_head)] + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Initialize forward metadata hence all layers in the forward pass can reuse it.""" + + forward_mode: ForwardMode = forward_batch.forward_mode + assert forward_mode.is_prefill() or forward_mode.is_decode() + batch_size = forward_batch.batch_size + + metadata = DualChunkFlashAttentionMetadata() + metadata.seq_lens_tensor = forward_batch.seq_lens.to(torch.int32) + metadata.seq_lens = forward_batch.seq_lens.tolist() + metadata.max_seq_len = forward_batch.seq_lens.max().item() + + metadata.orig_seq_lens_tensor = forward_batch.orig_seq_lens + metadata.orig_seq_lens = forward_batch.orig_seq_lens.tolist() + + metadata.block_tables = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len + ] + # Convert the block table to a strided format. + if self.page_size > 1: + strided_indices = torch.arange( + 0, metadata.block_tables.shape[1], self.page_size, device=self.device + ) + metadata.block_tables = ( + metadata.block_tables[:, strided_indices] // self.page_size + ) + + metadata.query_start_loc = torch.zeros( + batch_size + 1, dtype=torch.int32, device=metadata.seq_lens_tensor.device + ) + if forward_mode.is_prefill(): + metadata.query_start_loc[1:] = torch.cumsum( + forward_batch.extend_seq_lens.to(torch.int32), dim=0, dtype=torch.int32 + ) + else: + metadata.query_start_loc[1:] = torch.cumsum( + torch.arange( + batch_size, + dtype=metadata.query_start_loc.dtype, + device=metadata.query_start_loc.device, + ), + dim=0, + dtype=torch.int32, + ) + metadata.seq_start_loc = torch.zeros( + batch_size + 1, dtype=torch.int32, device=metadata.seq_lens_tensor.device + ) + metadata.seq_start_loc[1:] = torch.cumsum( + metadata.seq_lens_tensor, dim=0, dtype=torch.int32 + ) + + if self.original_max_position_embeddings > 0: + if forward_mode.is_prefill(): + metadata.scaling_factor = ( + 0.1 + * torch.log( + metadata.orig_seq_lens_tensor + / self.original_max_position_embeddings + ) + + 1.0 + ).clip(min=1) + else: + metadata.scaling_factor = ( + 0.1 + * torch.log( + metadata.orig_seq_lens_tensor + / self.original_max_position_embeddings + ) + + 1.0 + ).clip(min=1) + + if forward_mode.is_decode(): + cache_seq_lens = metadata.orig_seq_lens_tensor + + chunk_len = self.chunk_size - self.local_size + chunk_num_curr = (cache_seq_lens - 1) // chunk_len + + seq_lens_intra = cache_seq_lens - chunk_num_curr * chunk_len + max_seq_len_intra = seq_lens_intra.max().item() + metadata.seq_lens_intra = seq_lens_intra + metadata.max_seq_len_intra = max_seq_len_intra + + block_tables_intra = torch.zeros( + batch_size, + (max_seq_len_intra - 1) // self.page_size + 1, + dtype=metadata.block_tables.dtype, + device=metadata.block_tables.device, + ) + for i in range(batch_size): + st = chunk_num_curr[i] * chunk_len // self.page_size + ed = min( + st + (max_seq_len_intra - 1) // self.page_size + 1, + (cache_seq_lens[i] - 1) // self.page_size + 1, + ) + block_tables_intra[i, : ed - st] = metadata.block_tables[i, st:ed] + metadata.block_tables_intra = block_tables_intra + + metadata.seq_lens_succ = ( + chunk_num_curr - (chunk_num_curr - 1).clip(min=0) + ) * chunk_len + metadata.max_seq_len_succ = metadata.seq_lens_succ.max().item() + if metadata.max_seq_len_succ: + block_tables_succ = torch.zeros( + batch_size, + (metadata.max_seq_len_succ - 1) // self.page_size + 1, + dtype=metadata.block_tables.dtype, + device=metadata.block_tables.device, + ) + for i in range(batch_size): + start = ( + (chunk_num_curr[i] - 1).clip(min=0) + * chunk_len + // self.page_size + ) + end = min( + start + (metadata.max_seq_len_succ - 1) // self.page_size + 1, + (cache_seq_lens[i] - 1) // self.page_size + 1, + ) + block_tables_succ[i, : end - start] = metadata.block_tables[ + i, start:end + ] + metadata.block_tables_succ = block_tables_succ + + metadata.seq_lens_inter = (chunk_num_curr - 1).clip(min=0) * chunk_len + metadata.max_seq_len_inter = metadata.seq_lens_inter.max().item() + + self.forward_metadata = metadata + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: "RadixAttention", + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # Use precomputed metadata across all layers + metadata = self.forward_metadata + + ( + query, + query_succ, + query_inter, + query_succ_critical, + query_inter_critical, + ) = torch.split(q, q.shape[-1] // 5, dim=-1) + + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + query_succ = query_succ.view(-1, self.num_heads, self.head_size) + query_inter = query_inter.view(-1, self.num_heads, self.head_size) + query_succ_critical = query_succ_critical.view( + -1, self.num_heads, self.head_size + ) + query_inter_critical = query_inter_critical.view( + -1, self.num_heads, self.head_size + ) + key = k.view(-1, self.num_kv_heads, self.head_size) + value = v.view(-1, self.num_kv_heads, self.head_size) + + # apply DCA scaling + if self.original_max_position_embeddings > 0: + assert metadata.scaling_factor is not None + assert metadata.query_start_loc is not None + assert metadata.orig_seq_lens is not None + current_start = 0 + query_start_loc_cpu = metadata.query_start_loc.cpu() + for i in range(len(metadata.orig_seq_lens)): + current_end = ( + current_start + + (query_start_loc_cpu[i + 1] - query_start_loc_cpu[i]).item() + ) + key[current_start:current_end].mul_(metadata.scaling_factor[i]) + current_start = current_end + assert current_end <= self.max_context_len + + # Do multi-head attention + key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + layer.layer_id + ) + key_cache = key_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ) + value_cache = value_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ) + + if key is not None and value is not None: + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, + forward_batch.out_cache_loc, + key, + value, + layer.k_scale, + layer.v_scale, + ) + + if not save_kv_cache: + # profile run + o = flash_attn_varlen_func( + q=query, + k=key, + v=value, + cu_seqlens_q=metadata.seq_start_loc, + cu_seqlens_k=metadata.seq_start_loc, + max_seqlen_q=metadata.max_seq_len, + max_seqlen_k=metadata.max_seq_len, + softmax_scale=layer.scaling, + causal=True, + ) + else: + # prefill/chunked-prefill + # get per layer sparse attention config + if self.sparse_attention_enabled: + self.layer_sparse_attention_config = self.get_sparse_attention_config( + layer.layer_id + ) + assert metadata.orig_seq_lens is not None + o = self._dual_chunk_flash_attn_prefill( + q=query, + q_succ=query_succ, + q_inter=query_inter, + q_succ_critical=query_succ_critical, + q_inter_critical=query_inter_critical, + k=key_cache, + v=value_cache, + cu_seqlens_q=metadata.query_start_loc, + cu_seqlens_k=metadata.seq_start_loc, + orig_seq_lens=metadata.orig_seq_lens, + scaling_factor=metadata.scaling_factor, + softmax_scale=layer.scaling, + causal=True, + window_size=(-1, -1), + block_table=metadata.block_tables, + chunk_size=self.chunk_size, + local_size=self.local_size, + ) + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: "RadixAttention", + forward_batch: ForwardBatch, + save_kv_cache=True, + ) -> torch.Tensor: + # Use precomputed metadata across all layers + metadata = self.forward_metadata + + ( + query, + query_succ, + query_inter, + query_succ_critical, + query_inter_critical, + ) = torch.split(q, q.shape[-1] // 5, dim=-1) + + # Reshape the query, key, and value tensors. + query = query.view(-1, self.num_heads, self.head_size) + query_succ = query_succ.view(-1, self.num_heads, self.head_size) + query_inter = query_inter.view(-1, self.num_heads, self.head_size) + query_succ_critical = query_succ_critical.view( + -1, self.num_heads, self.head_size + ) + query_inter_critical = query_inter_critical.view( + -1, self.num_heads, self.head_size + ) + key = k.view(-1, self.num_kv_heads, self.head_size) + value = v.view(-1, self.num_kv_heads, self.head_size) + + key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + layer.layer_id + ) + key_cache = key_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ) + value_cache = value_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ) + + if key is not None and value is not None: + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, + forward_batch.out_cache_loc, + key, + value, + layer.k_scale, + layer.v_scale, + ) + + # apply DCA scaling + if self.original_max_position_embeddings > 0: + assert metadata.scaling_factor is not None + scaling_factor = metadata.scaling_factor + key.mul_(scaling_factor.unsqueeze(-1).unsqueeze(-1)) + + o = self._dual_chunk_flash_attn_decoding( + query.unsqueeze(1), + query_succ.unsqueeze(1), + query_inter.unsqueeze(1), + key_cache, + value_cache, + block_table=metadata.block_tables, + cache_seqlens=metadata.seq_lens_tensor, + softmax_scale=layer.scaling, + causal=True, + chunk_size=self.chunk_size, + local_size=self.local_size, + original_max_position_embeddings=self.original_max_position_embeddings, + decode_meta=metadata, + ).squeeze(1) + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + """Initialize CUDA graph state for the attention backend. + + Args: + max_bs (int): Maximum batch size to support in CUDA graphs + + This creates fixed-size tensors that will be reused during CUDA graph replay + to avoid memory allocations. + """ + self.decode_metadata = { + "seq_lens_tensor": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "orig_seq_lens_tensor": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "scaling_factor": torch.zeros( + max_bs, dtype=torch.float32, device=self.device + ), + "block_tables": torch.zeros( + max_bs, + (self.max_context_len - 1) // self.page_size + 1, + dtype=torch.int32, + device=self.device, + ), + "block_tables_intra": torch.zeros( + max_bs, + (self.max_context_len - 1) // self.page_size + 1, + dtype=torch.int32, + device=self.device, + ), + "seq_lens_intra": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "block_tables_succ": torch.zeros( + max_bs, + (self.max_context_len - 1) // self.page_size + 1, + dtype=torch.int32, + device=self.device, + ), + "seq_lens_succ": torch.zeros(max_bs, dtype=torch.int32, device=self.device), + "seq_lens_inter": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + } + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[None], + ): + metadata = DualChunkFlashAttentionMetadata() + + if forward_mode.is_decode_or_idle(): + if self.original_max_position_embeddings > 0: + metadata.scaling_factor = self.decode_metadata["scaling_factor"][:bs] + + metadata.seq_lens_tensor = self.decode_metadata["seq_lens_tensor"][:bs] + metadata.orig_seq_lens_tensor = self.decode_metadata[ + "orig_seq_lens_tensor" + ][:bs] + metadata.max_seq_len = self.max_context_len + metadata.block_tables = self.decode_metadata["block_tables"][ + req_pool_indices, : + ] + + # intra + metadata.max_seq_len_intra = self.max_context_len + metadata.seq_lens_intra = self.decode_metadata["seq_lens_intra"][:bs] + + metadata.block_tables_intra = self.decode_metadata["block_tables_intra"][ + :bs, : + ] + + # succ + metadata.seq_lens_succ = self.decode_metadata["seq_lens_succ"][:bs] + metadata.max_seq_len_succ = self.max_context_len + + metadata.block_tables_succ = self.decode_metadata["block_tables_succ"][ + :bs, : + ] + + metadata.seq_lens_inter = self.decode_metadata["seq_lens_inter"][:bs] + metadata.max_seq_len_inter = self.max_context_len + + self.decode_metadata[bs] = metadata + + self.forward_metadata = metadata + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[None], + seq_lens_cpu: Optional[torch.Tensor], + out_cache_loc: torch.Tensor = None, + ): + """Initialize forward metadata for replaying CUDA graph.""" + assert forward_mode.is_decode() + seq_lens = seq_lens[:bs] + req_pool_indices = req_pool_indices[:bs] + metadata = self.decode_metadata[bs] + + metadata.seq_lens_tensor.copy_(seq_lens.to(torch.int32)) + metadata.seq_lens = seq_lens.tolist() + metadata.max_seq_len = seq_lens.max().item() + + metadata.orig_seq_lens_tensor.copy_(seq_lens) + metadata.orig_seq_lens = seq_lens.tolist() + + block_tables = self.req_to_token[req_pool_indices, : metadata.max_seq_len] + # Convert the block table to a strided format. + if self.page_size > 1: + strided_indices = torch.arange( + 0, block_tables.shape[1], self.page_size, device=self.device + ) + block_tables = block_tables[:, strided_indices] // self.page_size + metadata.block_tables.fill_(0) + metadata.block_tables[: block_tables.shape[0], : block_tables.shape[1]].copy_( + block_tables + ) + + if self.original_max_position_embeddings > 0: + scaling_factor = ( + 0.1 + * torch.log( + metadata.orig_seq_lens_tensor + / self.original_max_position_embeddings + ) + + 1.0 + ).clip(min=1) + metadata.scaling_factor.copy_(scaling_factor) + + cache_seq_lens = metadata.orig_seq_lens_tensor + + chunk_len = self.chunk_size - self.local_size + chunk_num_curr = (cache_seq_lens - 1) // chunk_len + + seq_lens_intra = cache_seq_lens - chunk_num_curr * chunk_len + max_seq_len_intra = seq_lens_intra.max().item() + metadata.seq_lens_intra.copy_(seq_lens_intra) + metadata.max_seq_len_intra = max_seq_len_intra + + metadata.block_tables_intra.fill_(0) + for i in range(bs): + st = chunk_num_curr[i] * chunk_len // self.page_size + ed = min( + st + (max_seq_len_intra - 1) // self.page_size + 1, + (cache_seq_lens[i] - 1) // self.page_size + 1, + ) + metadata.block_tables_intra[i, : ed - st] = metadata.block_tables[i, st:ed] + + seq_lens_succ = (chunk_num_curr - (chunk_num_curr - 1).clip(min=0)) * chunk_len + metadata.seq_lens_succ.copy_(seq_lens_succ) + metadata.max_seq_len_succ = metadata.seq_lens_succ.max().item() + if metadata.max_seq_len_succ: + metadata.block_tables_succ.fill_(0) + for i in range(bs): + start = ( + (chunk_num_curr[i] - 1).clip(min=0) * chunk_len // self.page_size + ) + end = min( + start + (metadata.max_seq_len_succ - 1) // self.page_size + 1, + (cache_seq_lens[i] - 1) // self.page_size + 1, + ) + metadata.block_tables_succ[i, : end - start] = metadata.block_tables[ + i, start:end + ] + + seq_lens_inter = (chunk_num_curr - 1).clip(min=0) * chunk_len + metadata.seq_lens_inter.copy_(seq_lens_inter) + metadata.max_seq_len_inter = metadata.seq_lens_inter.max().item() + + self.forward_metadata = metadata + + def get_cuda_graph_seq_len_fill_value(self): + """Get the fill value for sequence length in CUDA graph.""" + return 1 + + def _dual_chunk_flash_attn_prefill( + self, + q, + q_succ, + q_inter, + q_succ_critical, + q_inter_critical, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + orig_seq_lens: List[int], + scaling_factor: torch.Tensor, + softmax_scale: float, + causal: Optional[bool] = True, + window_size: Tuple[int, int] = (-1, -1), + block_table: Optional[torch.Tensor] = None, + chunk_size: int = 8192, + local_size: int = 1024, + ): + if not causal: + raise ValueError("Dual Chunk Attention does not support causal=False") + if window_size != (-1, -1): + raise ValueError("Dual Chunk Attention does not support window_size") + + cu_seqlens_q_cpu = cu_seqlens_q.cpu().tolist() + cu_seqlens_k_cpu = cu_seqlens_k.cpu().tolist() + all_outputs = [] + + for i in range(0, len(cu_seqlens_q_cpu) - 1): + qs = cu_seqlens_q_cpu[i] + qe = cu_seqlens_q_cpu[i : i + 2][-1] + ks = cu_seqlens_k_cpu[i] + ke = cu_seqlens_k_cpu[i : i + 2][-1] + + current_q = q[qs:qe] + current_q_succ = q_succ[qs:qe] + current_q_inter = q_inter[qs:qe] + current_q_succ_critical = q_succ_critical[qs:qe] + current_q_inter_critical = q_inter_critical[qs:qe] + + if block_table is None: + current_k = k[ks:ke] + current_v = v[ks:ke] + current_block_table = None + current_orig_seq_len = orig_seq_lens[i] + else: + current_block_table = block_table[i] + current_orig_seq_len = orig_seq_lens[i] + current_k = k + current_v = v + sparse_attn_enabled = ( + self.sparse_attention_enabled + and current_orig_seq_len > self.sparse_attention_threshold + ) + + if current_q.shape[0] == 0: + continue + + if current_k.shape[0] == 0: + all_outputs.append( + torch.zeros( + (current_q.shape[0], current_q.shape[1], v.shape[2]), + device=q.device, + dtype=q.dtype, + ) + ) + continue + + current_output = torch.empty_like(current_q) + group_size = int(current_q.size(-2) / current_k.size(-2)) + + if sparse_attn_enabled: + num_device_q_heads = current_q.size(-2) + heads_vertical_size = torch.empty( + size=(num_device_q_heads,), dtype=torch.int32 + ) + heads_slash_size = torch.empty( + size=(num_device_q_heads,), dtype=torch.int32 + ) + for head_id in range(current_q.size(-2)): + ( + ty, + vertical_size, + slash_size, + _, + ) = self.layer_sparse_attention_config[head_id] + assert ty == "vertical_and_slash", "only support slash mode" + + if vertical_size == 30: + vertical_size += 100 + heads_vertical_size[head_id] = vertical_size + heads_slash_size[head_id] = slash_size + + current_output = self._dual_chunk_flash_attn_prefill_func( + current_q, # allheads + current_q_succ, + current_q_inter, + current_q_succ_critical, + current_q_inter_critical, + current_k, + current_v, + current_block_table, + softmax_scale, + chunk_size, + local_size, + scaling_factor[i].item(), + ke - ks, + sparse_attn_enabled=sparse_attn_enabled, + heads_vertical_size=heads_vertical_size, + heads_slash_size=heads_slash_size, + group_size=group_size, + ) + else: + for head_id in range(current_q.size(-2)): + # (seq_len, num_heads, head_size) + current_q_head = current_q[:, head_id, :].unsqueeze(1) + current_q_succ_head = current_q_succ[:, head_id, :].unsqueeze(1) + current_q_inter_head = current_q_inter[:, head_id, :].unsqueeze(1) + current_q_succ_head_critical = current_q_succ_critical[ + :, head_id, : + ].unsqueeze(1) + current_q_inter_head_critical = current_q_inter_critical[ + :, head_id, : + ].unsqueeze(1) + if block_table is not None: + current_k_head = current_k[ + ..., head_id // group_size, : + ].unsqueeze(2) + current_v_head = current_v[ + ..., head_id // group_size, : + ].unsqueeze(2) + + else: + current_k_head = current_k[:, head_id, :].unsqueeze(1) + current_v_head = current_v[:, head_id, :].unsqueeze(1) + + current_out = self._dual_chunk_flash_attn_prefill_func( + current_q_head, + current_q_succ_head, + current_q_inter_head, + current_q_succ_head_critical, + current_q_inter_head_critical, + current_k_head, + current_v_head, + current_block_table, + softmax_scale, + chunk_size, + local_size, + scaling_factor[i].item(), + ke - ks, + sparse_attn_enabled=sparse_attn_enabled, + ) + current_output[:, head_id : head_id + 1, :] = current_out + all_outputs.append(current_output) + return torch.cat(all_outputs, dim=0) + + def _dual_chunk_flash_attn_prefill_func( + self, + q, + q_succ, + q_inter, + q_succ_critical, + q_inter_critical, + k, + v, + block_table, + softmax_scale: float, + chunk_size: int, + local_size: int, + scaling_factor: float, + k_length: int, + sparse_attn_enabled: Optional[bool] = True, + heads_vertical_size=None, + heads_slash_size=None, + group_size=None, + ): + flash_results = [] + chunk_len = chunk_size - local_size + + if block_table is not None: + block_size = v.shape[1] + if chunk_len % block_size != 0: + raise ValueError("chunk_len must be divisible by block_size.") + else: + block_size = 1 + + if self.original_max_position_embeddings > 0: + softmax_scale = softmax_scale * scaling_factor + + begin = k_length - q.shape[0] + while begin < k_length: + flash_per_chunk = [] + + prev_chunk_end_pos = (begin // chunk_len) * chunk_len + next_chunk_end_pos = prev_chunk_end_pos + chunk_len + end = min(next_chunk_end_pos, k_length) + qbegin = begin - (k_length - q.shape[0]) + qend = end - (k_length - q.shape[0]) + + qk_chunks = [] + q_states_intra = q[qbegin:qend] + # choose critical token + if block_table is not None: + block_tables_intra = _get_block( + block_table, block_size, prev_chunk_end_pos, end + ) + k_states_intra = k[block_tables_intra].view(-1, *k.shape[-2:])[ + : (end - prev_chunk_end_pos) + ] + v_states_intra = v[block_tables_intra].view(-1, *v.shape[-2:])[ + : (end - prev_chunk_end_pos) + ] + else: + block_tables_intra = None + k_states_intra = k[prev_chunk_end_pos:end] + v_states_intra = v[prev_chunk_end_pos:end] + + if sparse_attn_enabled: + last_q_size = min(qend - qbegin, self.sparse_attention_last_q) + _, num_device_k_heads, head_dim = k_states_intra.shape + k_states_intra = ( + k_states_intra.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + v_states_intra = ( + v_states_intra.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + qk_chunks.append( + (q_states_intra.transpose(0, 1)[:, -last_q_size:] * softmax_scale) + @ k_states_intra.permute(1, 2, 0) + ) + + if prev_chunk_end_pos - chunk_len >= 0: + q_states_succ = q_succ[qbegin:qend] + q_states_succ_critical = q_succ_critical[qbegin:qend] + if block_table is not None: + block_tables_succ = _get_block( + block_table, + block_size, + prev_chunk_end_pos - chunk_len, + prev_chunk_end_pos, + ) + k_states_succ = k[block_tables_succ].view(-1, *k.shape[-2:])[ + :chunk_len + ] + v_states_succ = v[block_tables_succ].view(-1, *v.shape[-2:])[ + :chunk_len + ] + else: + k_states_succ = k[ + prev_chunk_end_pos - chunk_len : prev_chunk_end_pos + ] + v_states_succ = v[ + prev_chunk_end_pos - chunk_len : prev_chunk_end_pos + ] + + if sparse_attn_enabled: + k_states_succ = ( + k_states_succ.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + v_states_succ = ( + v_states_succ.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + qk_chunks.append( + ( + q_states_succ_critical.transpose(0, 1)[:, -last_q_size:] + * softmax_scale + ) + @ k_states_succ.permute(1, 2, 0) + ) + + if prev_chunk_end_pos - chunk_len * 2 >= 0: + q_states_inter = q_inter[qbegin:qend] + q_states_inter_critical = q_inter_critical[qbegin:qend] + if block_table is not None: + block_tables_inter = _get_block( + block_table, block_size, 0, prev_chunk_end_pos - chunk_len + ) + k_states_inter = k[block_tables_inter].view(-1, *k.shape[-2:])[ + : (prev_chunk_end_pos - chunk_len) + ] + v_states_inter = v[block_tables_inter].view(-1, *v.shape[-2:])[ + : (prev_chunk_end_pos - chunk_len) + ] + else: + k_states_inter = k[: prev_chunk_end_pos - chunk_len] + v_states_inter = v[: prev_chunk_end_pos - chunk_len] + + if sparse_attn_enabled: + k_states_inter = ( + k_states_inter.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + v_states_inter = ( + v_states_inter.unsqueeze(2) + .repeat(1, 1, group_size, 1) + .reshape(-1, num_device_k_heads * group_size, head_dim) + ) + qk_chunks.append( + ( + q_states_inter_critical.transpose(0, 1)[:, -last_q_size:] + * softmax_scale + ) + @ k_states_inter.permute(1, 2, 0) + ) + + if sparse_attn_enabled: + reversed_qk = qk_chunks[::-1] + qk = torch.cat(reversed_qk, dim=-1) + + qk[:, :, -last_q_size:] = torch.where( + self.last_q_mask[..., -last_q_size:, -last_q_size:].to(qk.device), + qk[:, :, -last_q_size:], + -torch.inf, + ) + qk = F.softmax(qk, dim=-1, dtype=torch.float32) + + vertical = qk.sum(-2, keepdim=True) + vertical[..., :30] = torch.inf + + # Avoid sorting by using the min/max ints to fill the indexer + # buffers. + int32_max = torch.iinfo(torch.int32).max + int32_min = torch.iinfo(torch.int32).min + n_heads = qk.size()[0] + max_slash_topk = torch.max(heads_slash_size).item() + max_vertical_topk = torch.max(heads_vertical_size).item() + # store each head's slash topk, vertical topk + vertical = vertical.reshape((n_heads, -1)) + # prevent out of range when prompt size < max_vertical_topk + max_vertical_topk = min(vertical.shape[-1], max_vertical_topk) + vertical_topk_buffer = torch.topk( + vertical, max_vertical_topk, -1 + ).indices + slash_topk_buffer = torch.empty( + size=(n_heads, max_slash_topk), dtype=torch.int64, device=qk.device + ) + for head_i in range(n_heads): + # (nqheads=1, lastq, k_len) + head_score = qk[head_i : head_i + 1, :, :] + slash_scores = _sum_all_diagonal_matrix(head_score) + if head_score.size(1) != 1: + # drop right up corner + slash_scores = slash_scores[..., : -last_q_size + 1] + slash_scores[..., -100:] = torch.inf + + head_slash_size = heads_slash_size[head_i] + head_slash_size = min(head_slash_size, vertical.size(-1)) + slash_topk = torch.topk(slash_scores, head_slash_size, -1).indices + # （nheads, max_topk） + slash_topk_buffer[head_i, :head_slash_size] = slash_topk + + # reset heads topk + heads_slash_size[head_i] = head_slash_size + heads_vertical_size[head_i] = min( + heads_vertical_size[head_i], max_vertical_topk + ) + + # store + vertical_buffer = torch.full( + (n_heads, max_vertical_topk), + int32_max, + dtype=torch.int64, + device=q.device, + ) + slash_buffer = torch.full( + (n_heads, max_slash_topk), + int32_min, + dtype=torch.int64, + device=q.device, + ) + succ_vertical_buffer = torch.full( + (n_heads, max_vertical_topk), + int32_max, + dtype=torch.int64, + device=q.device, + ) + succ_slash_buffer = torch.full( + (n_heads, max_slash_topk), + int32_min, + dtype=torch.int64, + device=q.device, + ) + inter_vertical_buffer = torch.full( + (n_heads, max_vertical_topk), + int32_max, + dtype=torch.int64, + device=q.device, + ) + inter_slash_buffer = torch.full( + (n_heads, max_slash_topk), + int32_min, + dtype=torch.int64, + device=q.device, + ) + + vertical_size_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + slash_sizes_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + succ_vertical_size_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + succ_slash_sizes_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + inter_vertical_size_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + inter_slash_sizes_buffer = torch.empty( + size=(n_heads,), dtype=torch.int32, device=q.device + ) + + for head_i in range(n_heads): + vertical_topk = vertical_topk_buffer[ + head_i, : heads_vertical_size[head_i] + ] + # intra + intra_vertical_indices = ( + vertical_topk[vertical_topk >= prev_chunk_end_pos] + - prev_chunk_end_pos + ) + if intra_vertical_indices.nelement() == 0: + intra_vertical_indices = torch.cat( + [ + intra_vertical_indices, + torch.arange( + 0, + k_states_intra.size(0), + max(1, k_states_intra.size(0) / 5), + dtype=torch.int32, + device=intra_vertical_indices.device, + ), + ] + ) + slash_topk = slash_topk_buffer[head_i, : heads_slash_size[head_i]] + intra_slash_indices = (qk.size(-1) - 1) - slash_topk[ + slash_topk >= prev_chunk_end_pos + ] + # fill buffer + v_count = intra_vertical_indices.nelement() + s_count = intra_slash_indices.nelement() + vertical_size_buffer[head_i] = v_count + slash_sizes_buffer[head_i] = s_count + vertical_buffer[head_i, :v_count].copy_(intra_vertical_indices) + slash_buffer[head_i, :s_count].copy_(intra_slash_indices) + # succ + if prev_chunk_end_pos - chunk_len >= 0: + succ_vertical_indices = vertical_topk[ + (vertical_topk < prev_chunk_end_pos) + & (vertical_topk >= prev_chunk_end_pos - chunk_len) + ] - (prev_chunk_end_pos - chunk_len) + # TODO: support no vertical + if succ_vertical_indices.nelement() == 0: + succ_vertical_indices = torch.cat( + [ + succ_vertical_indices, + torch.arange( + 0, + k_states_succ.size(0), + max(1, k_states_succ.size(0) / 5), + dtype=torch.int32, + device=intra_vertical_indices.device, + ), + ] + ) + succ_slash_indices = ( + prev_chunk_end_pos + (qend - qbegin) - 1 + ) - slash_topk[ + ( + (slash_topk >= (prev_chunk_end_pos - chunk_len)) + & (slash_topk < (prev_chunk_end_pos + (qend - qbegin))) + ) + ] + if succ_slash_indices.nelement() == 0: + succ_slash_indices = torch.cat( + [ + succ_slash_indices, + torch.arange( + 0, + k_states_succ.size(0), + max(1, k_states_succ.size(0) / 5), + dtype=torch.int32, + device=intra_vertical_indices.device, + ), + ] + ) + # fill buffer + v_count = succ_vertical_indices.nelement() + s_count = succ_slash_indices.nelement() + succ_vertical_size_buffer[head_i] = v_count + succ_slash_sizes_buffer[head_i] = s_count + succ_vertical_buffer[head_i, :v_count].copy_( + succ_vertical_indices + ) + succ_slash_buffer[head_i, :s_count].copy_(succ_slash_indices) + + if prev_chunk_end_pos - 2 * chunk_len >= 0: + inter_vertical_indices = vertical_topk[ + vertical_topk < prev_chunk_end_pos - chunk_len + ] + + if inter_vertical_indices.nelement() == 0: + inter_vertical_indices = torch.cat( + [ + inter_vertical_indices, + torch.arange( + 0, + k_states_inter.size(0), + max(1, k_states_inter.size(0) / 5), + dtype=torch.int32, + device=intra_vertical_indices.device, + ), + ] + ) + inter_slash_indices = ( + prev_chunk_end_pos - chunk_len + (qend - qbegin) - 1 + ) - slash_topk[ + slash_topk + < (prev_chunk_end_pos - chunk_len + (qend - qbegin)) + ] + if inter_slash_indices.nelement() == 0: + inter_slash_indices = torch.cat( + [ + inter_slash_indices, + torch.arange( + 0, + k_states_inter.size(0), + max(1, k_states_inter.size(0) / 5), + dtype=torch.int32, + device=intra_vertical_indices.device, + ), + ] + ) + # fill buffer + v_count = inter_vertical_indices.nelement() + s_count = inter_slash_indices.nelement() + inter_vertical_size_buffer[head_i] = v_count + inter_slash_sizes_buffer[head_i] = s_count + inter_vertical_buffer[head_i, :v_count].copy_( + inter_vertical_indices + ) + inter_slash_buffer[head_i, :s_count].copy_(inter_slash_indices) + else: + intra_vertical_indices, intra_slash_indices = None, None + succ_vertical_indices, succ_slash_indices = None, None + inter_vertical_indices, inter_slash_indices = None, None + + if sparse_attn_enabled: + flash_result = self._do_flash_attn( + q_states_intra, + k_states_intra, + v_states_intra, + softmax_scale=softmax_scale, + causal=True, + stage="intra", + vertical_indices=vertical_buffer, + slash_indices=slash_buffer, + vertical_indices_count=vertical_size_buffer, + slash_indices_count=slash_sizes_buffer, + mergehead_softmax_scale=softmax_scale, + sparse_attn_enabled=sparse_attn_enabled, + ) + else: + flash_result = self._do_flash_attn( + q_states_intra, + k_states_intra, + v_states_intra, + softmax_scale=softmax_scale, + causal=True, + stage="intra", + vertical_indices=intra_vertical_indices, + slash_indices=intra_slash_indices, + sparse_attn_enabled=sparse_attn_enabled, + ) + flash_per_chunk.append(flash_result) + + if prev_chunk_end_pos - chunk_len >= 0: + if sparse_attn_enabled: + flash_result = self._do_flash_attn( + q_states_succ, + k_states_succ, + v_states_succ, + softmax_scale=softmax_scale, + causal=False, + stage="succ", + vertical_indices=succ_vertical_buffer, + slash_indices=succ_slash_buffer, + vertical_indices_count=succ_vertical_size_buffer, + slash_indices_count=succ_slash_sizes_buffer, + mergehead_softmax_scale=softmax_scale, + sparse_attn_enabled=sparse_attn_enabled, + ) + else: + flash_result = self._do_flash_attn( + q_states_succ, + k_states_succ, + v_states_succ, + softmax_scale=softmax_scale, + causal=False, + stage="succ", + vertical_indices=succ_vertical_indices, + slash_indices=succ_slash_indices, + sparse_attn_enabled=sparse_attn_enabled, + ) + flash_per_chunk.append(flash_result) + + if prev_chunk_end_pos - chunk_len * 2 >= 0: + if sparse_attn_enabled: + flash_result = self._do_flash_attn( + q_states_inter, + k_states_inter, + v_states_inter, + softmax_scale=softmax_scale, + causal=False, + stage="inter", + vertical_indices=inter_vertical_buffer, + slash_indices=inter_slash_buffer, + vertical_indices_count=inter_vertical_size_buffer, + slash_indices_count=inter_slash_sizes_buffer, + mergehead_softmax_scale=softmax_scale, + sparse_attn_enabled=sparse_attn_enabled, + ) + else: + flash_result = self._do_flash_attn( + q_states_inter, + k_states_inter, + v_states_inter, + softmax_scale=softmax_scale, + causal=False, + stage="inter", + vertical_indices=inter_vertical_indices, + slash_indices=inter_slash_indices, + sparse_attn_enabled=sparse_attn_enabled, + ) + flash_per_chunk.append(flash_result) + + flash_results.append(flash_per_chunk) + begin = end + + attn_output = self._merge_attn_outputs(flash_results) + del flash_results + return attn_output + + def _do_flash_attn( + self, + query_states: torch.Tensor, + key_states: torch.Tensor, + value_states: torch.Tensor, + softmax_scale: float, + causal: bool = True, + max_seqlen_k: Optional[int] = None, + stage: str = "intra", + vertical_indices: Optional[torch.Tensor] = None, + slash_indices: Optional[torch.Tensor] = None, + vertical_indices_count: Optional[torch.Tensor] = None, + slash_indices_count: Optional[torch.Tensor] = None, + mergehead_softmax_scale: Optional[float] = None, + sparse_attn_enabled: Optional[bool] = False, + ): + if max_seqlen_k is None: + max_seqlen_k = key_states.shape[0] + + q_len = query_states.shape[0] + q_heads = query_states.shape[1] + h_dim = query_states.shape[-1] + + if sparse_attn_enabled: + assert slash_indices is not None + if stage == "intra": + assert causal + else: + assert not causal + + query_states = query_states.unsqueeze(0).transpose(1, 2) + key_states = key_states.unsqueeze(0).transpose(1, 2) + value_states = value_states.unsqueeze(0).transpose(1, 2) + + q = query_states + k = key_states + v = value_states + + if vertical_indices_count is not None and slash_indices_count is not None: + assert mergehead_softmax_scale is not None + + res, s_lse = _vertical_slash_sparse_attention( + q, + k, + v, + vertical_indices, + slash_indices, + mergehead_softmax_scale, + causal=causal, + stage=stage, + vertical_indices_count=vertical_indices_count, + slash_indices_count=slash_indices_count, + ) + res = res.view(q_heads, q_len, h_dim).transpose( + 0, 1 + ) # (qlen,nhead,h_dim) + s_lse = ( + s_lse.view(q_heads, q_len, 1).squeeze(-1).unsqueeze(0).float() + ) # (1, nhead,qlen) + else: + res, s_lse = _vertical_slash_sparse_attention( + q, + k, + v, + vertical_indices, + slash_indices, + softmax_scale, + causal=causal, + stage=stage, + ) + res = res.view(q_len, q_heads, h_dim) + s_lse = s_lse.view(q_len, q_heads, 1).transpose(0, 2).float() + return res, s_lse + + output, softmax_lse, *rest = flash_attn_varlen_func( + q=query_states, + k=key_states, + v=value_states, + softmax_scale=softmax_scale, + cu_seqlens_q=torch.tensor( + [0, query_states.shape[0]], + dtype=torch.int32, + device=query_states.device, + ), + max_seqlen_q=query_states.shape[0], + cu_seqlens_k=torch.tensor( + [0, max_seqlen_k], dtype=torch.int32, device=query_states.device + ), + max_seqlen_k=max_seqlen_k, + causal=causal, + return_softmax_lse=True, + ) + softmax_lse = softmax_lse.view(q_len, q_heads, 1).transpose(0, 2).float() + return output, softmax_lse + + def _merge_attn_outputs( + self, + flash_results: List[List[Tuple[torch.Tensor, torch.Tensor]]], + return_lse: Optional[bool] = False, + ) -> torch.Tensor: + attn_outputs_all = [] + logits_all = [] + + for flash_per_chunk in flash_results: + if len(flash_per_chunk) == 1: + attn_outputs_all.append(flash_per_chunk[0][0]) + if return_lse: + logits_all.append(flash_per_chunk[0][1]) + continue + + attn_outputs = torch.stack( + [flash_attn_output[0] for flash_attn_output in flash_per_chunk] + ) + logits = torch.stack( + [flash_attn_output[1] for flash_attn_output in flash_per_chunk] + ) + logits = logits.to(torch.float32) + + if return_lse: + max_val = torch.max(logits, dim=0).values + diff = torch.abs(logits[0] - logits[1]) + log_sum_exp = max_val + torch.log1p(torch.exp(-diff)) + logits_all.append(log_sum_exp) + + max_logits = torch.max(logits, dim=0).values + stable_logits = logits - max_logits.unsqueeze(0) + lse_s = torch.exp(stable_logits).detach() + lse_sum = torch.sum(lse_s, dim=0) + lse_s /= lse_sum + attn_outputs *= lse_s.unsqueeze(-1).transpose(2, 3).squeeze(1) + attn_outputs_all.append(attn_outputs.sum(dim=0)) + + if return_lse: + return (torch.cat(attn_outputs_all, dim=0), torch.cat(logits_all, dim=-1)) + else: + return torch.cat(attn_outputs_all, dim=0) + + def _dual_chunk_flash_attn_decoding( + self, + query: torch.Tensor, + query_succ: torch.Tensor, + query_inter: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + softmax_scale: float, + causal: bool, + chunk_size: int, + local_size: int, + original_max_position_embeddings: int, + decode_meta: DualChunkFlashAttentionMetadata, + ): + if not causal: + raise ValueError("Dual Chunk Attention does not support causal=False") + + block_size = value_cache.shape[1] + chunk_len = chunk_size - local_size + if chunk_len % block_size != 0: + raise ValueError("chunk_len must be divisible by block_size.") + if original_max_position_embeddings > 0: + assert decode_meta.scaling_factor is not None + scaling_factor = decode_meta.scaling_factor + query = (query * scaling_factor.view(-1, 1, 1, 1)).to( + query.dtype + ) # possible for numerical issue, need to fused in the kernel + query_succ = (query_succ * scaling_factor.view(-1, 1, 1, 1)).to(query.dtype) + query_inter = (query_inter * scaling_factor.view(-1, 1, 1, 1)).to( + query.dtype + ) + outputs_list = [] + softmax_lses_list = [] + + # intra-attention + intra_output, intra_softmax_lse = ( + self._dual_chunk_flash_attn_decoding_with_exp_sums( + query, + key_cache, + value_cache, + decode_meta.block_tables_intra, + decode_meta.seq_lens_intra, + softmax_scale, + causal=False, + ) + ) + outputs_list.append(intra_output) + softmax_lses_list.append(intra_softmax_lse) + + # succ-attention + if decode_meta.max_seq_len_succ: + succ_output, succ_softmax_lse = ( + self._dual_chunk_flash_attn_decoding_with_exp_sums( + query_succ, + key_cache, + value_cache, + decode_meta.block_tables_succ, + decode_meta.seq_lens_succ, + softmax_scale, + causal=False, + ) + ) + outputs_list.append(succ_output) + softmax_lses_list.append(succ_softmax_lse) + + # inter-attention + if decode_meta.max_seq_len_inter: + inter_output, inter_softmax_lse = ( + self._dual_chunk_flash_attn_decoding_with_exp_sums( + query_inter, + key_cache, + value_cache, + block_table[:, : decode_meta.max_seq_len_inter], + decode_meta.seq_lens_inter, + softmax_scale, + causal=False, + ) + ) + outputs_list.append(inter_output) + softmax_lses_list.append(inter_softmax_lse) + outputs = torch.stack(outputs_list, dim=0) + del outputs_list + softmax_lses = torch.stack(softmax_lses_list, dim=0).to(torch.float32) + del softmax_lses_list + max_logits = torch.max(softmax_lses, dim=0).values + stable_logits = softmax_lses - max_logits.unsqueeze(0) + lse_s = torch.exp(stable_logits).detach() + lse_sum = torch.sum(lse_s, dim=0) + lse_s /= lse_sum + outputs *= lse_s.unsqueeze(-1).transpose(2, 3) + return outputs.sum(0) + + def _dual_chunk_flash_attn_decoding_with_exp_sums( + self, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + block_table: torch.Tensor, + cache_seqlens: torch.Tensor, + softmax_scale: float, + causal: bool, + ): + out, softmax_lse, *rest_expand = flash_attn_with_kvcache( + q=query, + k_cache=key_cache, + v_cache=value_cache, + page_table=block_table, + cache_seqlens=cache_seqlens, + softmax_scale=softmax_scale, + causal=causal, + return_softmax_lse=True, + ) + mask = cache_seqlens == 0 + out[mask] = 0 + softmax_lse[mask] = -float("inf") + return out, softmax_lse + + +def _vertical_slash_sparse_attention( + query: torch.Tensor, # [BATCH, N_HEADS, N_CTX, D_HEAD] + key: torch.Tensor, # [BATCH, N_HEADS, N_KV_CTX, D_HEAD] + value: torch.Tensor, # [BATCH, N_HEADS, N_KV_CTX, D_HEAD] + v_idx: torch.Tensor, # [BATCH, N_HEADS, NNZ_V] + s_idx: torch.Tensor, # [BATCH, N_HEADS, NNZ_S] + softmax_scale: float, + causal: bool = True, + stage: str = "intra", + block_size_M: int = 64, + block_size_N: int = 64, + vertical_indices_count: torch.Tensor = None, # [N_HEADS,] + slash_indices_count: torch.Tensor = None, +): + if stage == "intra": + assert causal + else: + assert not causal + + batch_size, num_heads, context_size, head_dim = query.shape + _, _, kv_seq_len, _ = key.shape + + if head_dim not in [16, 32, 64, 128, 256, 512]: + target_dim = 2 ** math.ceil(math.log2(head_dim)) - head_dim + query = F.pad(query, [0, target_dim, 0, 0, 0, 0, 0, 0]) + key = F.pad(key, [0, target_dim, 0, 0, 0, 0, 0, 0]) + value = F.pad(value, [0, target_dim, 0, 0, 0, 0, 0, 0]) + + v_idx = ( + v_idx.to(torch.int32) + .reshape((batch_size, num_heads, -1)) + .sort(dim=-1, descending=False)[0] + ) + s_idx = ( + s_idx.to(torch.int32) + .reshape((batch_size, num_heads, -1)) + .sort(dim=-1, descending=True)[0] + ) + q_seqlens = torch.tensor([context_size], dtype=torch.int32, device=query.device) + kv_seqlens = torch.tensor([kv_seq_len], dtype=torch.int32, device=query.device) + + if vertical_indices_count is not None and slash_indices_count is not None: + ( + block_count, + block_offset, + column_count, + column_index, + ) = convert_vertical_slash_indexes_mergehead( + q_seqlens, + kv_seqlens, + v_idx, + s_idx, + vertical_indices_count, + slash_indices_count, + context_size, + block_size_M, + block_size_N, + causal, + ) + else: + ( + block_count, + block_offset, + column_count, + column_index, + ) = convert_vertical_slash_indexes( + q_seqlens, + kv_seqlens, + v_idx, + s_idx, + context_size, + block_size_M, + block_size_N, + causal, + ) + + q = query.transpose(1, 2).contiguous() + k = key.transpose(1, 2).contiguous() + v = value.transpose(1, 2).contiguous() + out, lse = sparse_attn_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + causal=causal, + softmax_scale=softmax_scale, + return_softmax_lse=True, + ) + out = out.transpose(1, 2).contiguous() + softmax_lse = lse.reshape(*lse.shape, 1) + return (out[..., :context_size, :head_dim], softmax_lse[..., :context_size, :]) + + +def _sum_all_diagonal_matrix(mat: torch.tensor): + h, n, m = mat.shape + # Zero matrix used for padding + zero_mat = torch.zeros((h, n, n), device=mat.device) + # pads the matrix on left and right + mat_padded = torch.cat((zero_mat, mat, zero_mat), -1) + # Change the strides + mat_strided = mat_padded.as_strided( + (1, n, n + m), (n * (2 * n + m), 2 * n + m + 1, 1) + ) + # Sums the resulting matrix's columns + sum_diags = torch.sum(mat_strided, 1) + return sum_diags[:, 1:] # drop left bottom corner + + +def _get_block(block_table: torch.Tensor, block_size: int, begin: int, end: int): + begin_block = begin // block_size + end_block = (end - 1) // block_size + 1 + return block_table[begin_block:end_block] diff --git a/python/sglang/srt/layers/attention/fla/chunk.py b/python/sglang/srt/layers/attention/fla/chunk.py new file mode 100644 index 000000000..a48a9e649 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/chunk.py @@ -0,0 +1,242 @@ +# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/chunk.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +import warnings +from typing import Optional + +import torch +from einops import rearrange + +from sglang.srt.layers.attention.fla.chunk_delta_h import chunk_gated_delta_rule_fwd_h +from sglang.srt.layers.attention.fla.chunk_o import chunk_fwd_o +from sglang.srt.layers.attention.fla.chunk_scaled_dot_kkt import ( + chunk_scaled_dot_kkt_fwd, +) +from sglang.srt.layers.attention.fla.cumsum import chunk_local_cumsum +from sglang.srt.layers.attention.fla.l2norm import l2norm_fwd +from sglang.srt.layers.attention.fla.solve_tril import solve_tril +from sglang.srt.layers.attention.fla.utils import ( + SUPPRESS_LEVEL, + autocast_custom_fwd, + input_guard, +) +from sglang.srt.layers.attention.fla.wy_fast import recompute_w_u_fwd + + +def chunk_gated_delta_rule_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: Optional[torch.LongTensor] = None, +): + g = chunk_local_cumsum(g, chunk_size=64, cu_seqlens=cu_seqlens) + # obtain WY representation. u is actually the new v. + A = chunk_scaled_dot_kkt_fwd( + k=k, beta=beta, g_cumsum=g, cu_seqlens=cu_seqlens, output_dtype=torch.float32 + ) + A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype) + w, u = recompute_w_u_fwd( + k=k, + v=v, + beta=beta, + A=A, + g_cumsum=g, + cu_seqlens=cu_seqlens, + ) + h, v_new, final_state = chunk_gated_delta_rule_fwd_h( + k=k, + w=w, + u=u, + g=g, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + ) + o = chunk_fwd_o( + q=q, + k=k, + v=v_new, + h=h, + g=g, + scale=scale, + cu_seqlens=cu_seqlens, + ) + if SUPPRESS_LEVEL < 3: + return g, o, A, final_state, None, None, None + elif SUPPRESS_LEVEL >= 3: + return g, o, A, final_state, w, h, v_new + + +class ChunkGatedDeltaRuleFunction(torch.autograd.Function): + + @staticmethod + @input_guard + @autocast_custom_fwd + def forward( + ctx, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: Optional[torch.LongTensor] = None, + use_qk_l2norm_in_kernel: bool = False, + ): + q_orig = q + k_orig = k + + if use_qk_l2norm_in_kernel: + q = l2norm_fwd(q) + k = l2norm_fwd(k) + + g, o, A, final_state, w, h, v_new = chunk_gated_delta_rule_fwd( + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=scale, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + ) + return o.to(q.dtype), final_state + + +@torch.compiler.disable +def chunk_gated_delta_rule( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float = None, + initial_state: torch.Tensor = None, + output_final_state: bool = False, + cu_seqlens: Optional[torch.LongTensor] = None, + head_first: bool = False, + use_qk_l2norm_in_kernel: bool = False, +): + r""" + Args: + q (torch.Tensor): + queries of shape `[B, T, H, K]` if `head_first=False` else `[B, H, T, K]`. + k (torch.Tensor): + keys of shape `[B, T, H, K]` if `head_first=False` else `[B, H, T, K]`. + v (torch.Tensor): + values of shape `[B, T, H, V]` if `head_first=False` else `[B, H, T, V]`. + g (torch.Tensor): + (forget) gating tensor (in log space!) of shape `[B, T, H]` if `head_first=False` else `[B, H, T]`. + beta (torch.Tensor): + betas of shape `[B, T, H]` if `head_first=False` else `[B, H, T]`. + scale (Optional[int]): + Scale factor for the RetNet attention scores. + If not provided, it will default to `1 / sqrt(K)`. Default: `None`. + initial_state (Optional[torch.Tensor]): + Initial state of shape `[N, H, K, V]` for `N` input sequences. + For equal-length input sequences, `N` equals the batch size `B`. + Default: `None`. + output_final_state (Optional[bool]): + Whether to output the final state of shape `[N, H, K, V]`. Default: `False`. + cu_seqlens (torch.LongTensor): + Cumulative sequence lengths of shape `[N+1]` used for variable-length training, + consistent with the FlashAttention API. + head_first (Optional[bool]): + Whether the inputs are in the head-first format, which is not supported for variable-length inputs. + Default: `False`. + + Returns: + o (torch.Tensor): + Outputs of shape `[B, T, H, V]` if `head_first=False` else `[B, H, T, V]`. + final_state (torch.Tensor): + Final state of shape `[N, H, K, V]` if `output_final_state=True` else `None`. + + Examples:: + >>> import torch + >>> import torch.nn.functional as F + >>> from einops import rearrange + >>> from fla.ops.gated_delta_rule import chunk_gated_delta_rule + # inputs with equal lengths + >>> B, T, H, K, V = 4, 2048, 4, 512, 512 + >>> q = torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda') + >>> k = F.normalize(torch.randn(B, T, H, K, dtype=torch.bfloat16, device='cuda'), p=2, dim=-1) + >>> v = torch.randn(B, T, H, V, dtype=torch.bfloat16, device='cuda') + >>> beta = torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda').sigmoid() + >>> g = F.logsigmoid(torch.rand(B, T, H, dtype=torch.bfloat16, device='cuda')) + >>> h0 = torch.randn(B, H, K, V, dtype=torch.bfloat16, device='cuda') + >>> o, ht = chunk_gated_delta_rule( + q, k, v, g, beta, + initial_state=h0, + output_final_state=True + ) + # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required + >>> q, k, v, beta, g = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, beta, g)) + # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected + >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.long) + >>> o_var, ht_var = chunk_gated_delta_rule( + q, k, v, g, beta, + initial_state=h0, + output_final_state=True, + cu_seqlens=cu_seqlens + ) + """ + assert q.dtype == k.dtype == v.dtype + assert ( + q.dtype != torch.float32 + ), "ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16." + assert ( + len(beta.shape) == 3 + ), "beta must be of shape [B, T, H] if head_first=False, or [B, H, T] otherwise." + + if head_first: + raise DeprecationWarning( + "head_first is deprecated and will be removed in a future version. " + "Please use head_first=False for now instead." + ) + q, k, v, beta, g = map( + lambda x: rearrange(x, "b h t ... -> b t h ..."), (q, k, v, beta, g) + ) + # if not head_first and q.shape[1] < q.shape[2]: + # warnings.warn( + # f"Input tensor shape suggests potential format mismatch: seq_len ({q.shape[1]}) < num_heads ({q.shape[2]}). " + # "This may indicate the inputs were passed in head-first format [B, H, T, ...] " + # "when head_first=False was specified. " + # "Please verify your input tensor format matches the expected shape [B, T, H, ...]." + # ) + if cu_seqlens is not None: + if q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1: + raise ValueError( + f"The number of initial states is expected to be equal to the number of input sequences, " + f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + o, final_state = ChunkGatedDeltaRuleFunction.apply( + q, + k, + v, + g, + beta, + scale, + initial_state, + output_final_state, + cu_seqlens, + use_qk_l2norm_in_kernel, + ) + if head_first: + o = rearrange(o, "b t h ... -> b h t ...") + return o, final_state diff --git a/python/sglang/srt/layers/attention/fla/chunk_delta_h.py b/python/sglang/srt/layers/attention/fla/chunk_delta_h.py new file mode 100644 index 000000000..8f4109abd --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/chunk_delta_h.py @@ -0,0 +1,890 @@ +# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_delta_h.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +# from typing import Optional, Tuple + +# import torch +# import triton +# import triton.language as tl + +# from sglang.srt.layers.attention.fla.index import ( +# prepare_chunk_indices, +# prepare_chunk_offsets, +# ) +# from sglang.srt.layers.attention.fla.op import exp, safe_exp +# from sglang.srt.layers.attention.fla.utils import is_nvidia_hopper + +# NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16] + + +# @triton.heuristics( +# { +# "USE_G": lambda args: args["g"] is not None, +# "USE_INITIAL_STATE": lambda args: args["h0"] is not None, +# "STORE_FINAL_STATE": lambda args: args["ht"] is not None, +# "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None, +# "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, +# } +# ) +# # @triton.autotune( +# # configs=[ +# # triton.Config({"BV": BV}, num_warps=num_warps, num_stages=num_stages) +# # for num_warps in [2, 4] +# # for num_stages in [2, 3, 4] +# # for BV in [32, 64] +# # ], +# # key=["H", "K", "V", "BT", "USE_G"], +# # use_cuda_graph=use_cuda_graph, +# # ) +# @triton.jit(do_not_specialize=["T"]) +# def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( +# k, +# v, +# w, +# v_new, +# g, +# h, +# h0, +# ht, +# cu_seqlens, +# chunk_offsets, +# T, +# H: tl.constexpr, +# Hg: tl.constexpr, +# K: tl.constexpr, +# V: tl.constexpr, +# BT: tl.constexpr, +# BV: tl.constexpr, +# USE_G: tl.constexpr, +# USE_INITIAL_STATE: tl.constexpr, +# STORE_FINAL_STATE: tl.constexpr, +# SAVE_NEW_VALUE: tl.constexpr, +# IS_VARLEN: tl.constexpr, +# ): +# i_v, i_nh = tl.program_id(0), tl.program_id(1) +# i_n, i_h = i_nh // H, i_nh % H +# if IS_VARLEN: +# bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( +# cu_seqlens + i_n + 1 +# ).to(tl.int32) +# T = eos - bos +# NT = tl.cdiv(T, BT) +# boh = tl.load(chunk_offsets + i_n).to(tl.int32) +# else: +# bos, eos = i_n * T, i_n * T + T +# NT = tl.cdiv(T, BT) +# boh = i_n * NT + +# # [BK, BV] +# b_h1 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 64: +# b_h2 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 128: +# b_h3 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 192: +# b_h4 = tl.zeros([64, BV], dtype=tl.float32) + +# # calculate offset +# h += (boh * H + i_h) * K * V +# v += (bos * H + i_h) * V +# k += (bos * Hg + i_h // (H // Hg)) * K +# w += (bos * H + i_h) * K +# if SAVE_NEW_VALUE: +# v_new += (bos * H + i_h) * V +# stride_v = H * V +# stride_h = H * K * V +# stride_k = Hg * K +# stride_w = H * K +# if USE_INITIAL_STATE: +# h0 = h0 + i_nh * K * V +# if STORE_FINAL_STATE: +# ht = ht + i_nh * K * V + +# # load initial state +# if USE_INITIAL_STATE: +# p_h0_1 = tl.make_block_ptr(h0, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) +# b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) +# if K > 64: +# p_h0_2 = tl.make_block_ptr( +# h0, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0) +# ) +# b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) +# if K > 128: +# p_h0_3 = tl.make_block_ptr( +# h0, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0) +# ) +# b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) +# if K > 192: +# p_h0_4 = tl.make_block_ptr( +# h0, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0) +# ) +# b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) + +# # main recurrence +# for i_t in range(NT): +# p_h1 = tl.make_block_ptr( +# h + i_t * stride_h, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) +# if K > 64: +# p_h2 = tl.make_block_ptr( +# h + i_t * stride_h, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1)) +# if K > 128: +# p_h3 = tl.make_block_ptr( +# h + i_t * stride_h, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1)) +# if K > 192: +# p_h4 = tl.make_block_ptr( +# h + i_t * stride_h, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) + +# p_v = tl.make_block_ptr( +# v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) +# ) +# p_v_new = ( +# tl.make_block_ptr( +# v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) +# ) +# if SAVE_NEW_VALUE +# else None +# ) +# b_v_new = tl.zeros([BT, BV], dtype=tl.float32) +# p_w = tl.make_block_ptr( +# w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0) +# ) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h1.to(b_w.dtype)) +# if K > 64: +# p_w = tl.make_block_ptr( +# w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0) +# ) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h2.to(b_w.dtype)) +# if K > 128: +# p_w = tl.make_block_ptr( +# w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0) +# ) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h3.to(b_w.dtype)) +# if K > 192: +# p_w = tl.make_block_ptr( +# w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0) +# ) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h4.to(b_w.dtype)) +# b_v_new = -b_v_new + tl.load(p_v, boundary_check=(0, 1)) + +# if SAVE_NEW_VALUE: +# p_v_new = tl.make_block_ptr( +# v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) +# ) +# tl.store( +# p_v_new, b_v_new.to(p_v_new.dtype.element_ty), boundary_check=(0, 1) +# ) + +# if USE_G: +# last_idx = min((i_t + 1) * BT, T) - 1 +# b_g_last = tl.load(g + bos * H + last_idx * H + i_h) +# p_g = tl.make_block_ptr( +# g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) +# ) +# b_g = tl.load(p_g, boundary_check=(0,)) +# b_v_new = b_v_new * safe_exp(b_g_last - b_g)[:, None] +# b_g_last = exp(b_g_last) +# b_h1 = b_h1 * b_g_last +# if K > 64: +# b_h2 = b_h2 * b_g_last +# if K > 128: +# b_h3 = b_h3 * b_g_last +# if K > 192: +# b_h4 = b_h4 * b_g_last +# b_v_new = b_v_new.to(k.dtype.element_ty) +# p_k = tl.make_block_ptr( +# k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1) +# ) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h1 += tl.dot(b_k, b_v_new) +# if K > 64: +# p_k = tl.make_block_ptr( +# k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1) +# ) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h2 += tl.dot(b_k, b_v_new) +# if K > 128: +# p_k = tl.make_block_ptr( +# k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1) +# ) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h3 += tl.dot(b_k, b_v_new) +# if K > 192: +# p_k = tl.make_block_ptr( +# k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1) +# ) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h4 += tl.dot(b_k, b_v_new) + +# # epilogue +# if STORE_FINAL_STATE: +# p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 64: +# p_ht = tl.make_block_ptr( +# ht, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 128: +# p_ht = tl.make_block_ptr( +# ht, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 192: +# p_ht = tl.make_block_ptr( +# ht, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0) +# ) +# tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + + +# def chunk_gated_delta_rule_fwd_h( +# k: torch.Tensor, +# w: torch.Tensor, +# u: torch.Tensor, +# g: Optional[torch.Tensor] = None, +# initial_state: Optional[torch.Tensor] = None, +# output_final_state: bool = False, +# chunk_size: int = 64, # SY: remove this argument and force chunk size 64? +# save_new_value: bool = True, +# cu_seqlens: Optional[torch.LongTensor] = None, +# ) -> Tuple[torch.Tensor, torch.Tensor]: +# B, T, Hg, K, V = *k.shape, u.shape[-1] +# H = u.shape[-2] +# BT = chunk_size + +# chunk_indices = ( +# prepare_chunk_indices(cu_seqlens, chunk_size) +# if cu_seqlens is not None +# else None +# ) +# # N: the actual number of sequences in the batch with either equal or variable lengths +# if cu_seqlens is None: +# N, NT, chunk_offsets = B, triton.cdiv(T, BT), None +# else: +# N, NT, chunk_offsets = ( +# len(cu_seqlens) - 1, +# len(chunk_indices), +# prepare_chunk_offsets(cu_seqlens, BT), +# ) +# assert K <= 256, "current kernel does not support head dimension larger than 256." + +# h = k.new_empty(B, NT, H, K, V) +# final_state = ( +# k.new_empty(N, H, K, V, dtype=torch.float32) if output_final_state else None +# ) + +# v_new = torch.empty_like(u) if save_new_value else None + +# def grid(meta): +# return (triton.cdiv(V, meta["BV"]), N * H) +# # if hasattr(torch.version, 'hip') and torch.version.hip is not None: +# # # HIP环境回退实现 +# # return h, v_new, final_state +# chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid]( +# k=k, +# v=u, +# w=w, +# v_new=v_new, +# g=g, +# h=h, +# h0=initial_state, +# ht=final_state, +# cu_seqlens=cu_seqlens, +# chunk_offsets=chunk_offsets, +# T=T, +# H=H, +# Hg=Hg, +# K=K, +# V=V, +# BT=BT, +# BV=32, +# num_warps=4, +# num_stages=2, +# ) +# return h, v_new, final_state + + +from typing import Optional, Tuple +import math +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import ( + prepare_chunk_indices, + prepare_chunk_offsets, +) +from sglang.srt.layers.attention.fla.op import exp, safe_exp +from sglang.srt.layers.attention.fla.utils import is_nvidia_hopper + +# 保守一些在 HIP/ROCm 上更稳；NVIDIA 可调大 +NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16] + + +@triton.heuristics( + { + "USE_G": lambda args: args["g"] is not None, + "USE_INITIAL_STATE": lambda args: args["h0"] is not None, + "STORE_FINAL_STATE": lambda args: args["ht"] is not None, + "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.jit(do_not_specialize=["T"]) +def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( + k, # [B, T, Hg, K] + v, # [B, T, H, V] + w, # [B, T, H, K] + v_new, # [B, T, H, V] or None + g, # [B, T, H] or None + h, # [B, NT, H, K, V] + h0, # [B, H, K, V] or None + ht, # [B, H, K, V] or None + cu_seqlens, # [N+1] or None + chunk_offsets,# [N] or None + T, # 全局 T（固定长时就是实际长度） + H: tl.constexpr, + Hg: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, # chunk size（通常 64） + BV: tl.constexpr, # V 方向 tile 宽度（通常 32） + NT: tl.constexpr, # host 计算的上界（chunk 数） + USE_G: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, + STORE_FINAL_STATE: tl.constexpr, + SAVE_NEW_VALUE: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + # 程序实例：X 维为 V/BV，Y 维为 N*H + i_v, i_nh = tl.program_id(0), tl.program_id(1) + i_n = i_nh // H + i_h = i_nh % H + + # 计算该序列 runtime 真实长度 TR、时间起点 bos、h 的 chunk 偏移 boh + if IS_VARLEN: + bos = tl.load(cu_seqlens + i_n).to(tl.int32) + eos = tl.load(cu_seqlens + i_n + 1).to(tl.int32) + TR = (eos - bos).to(tl.int32) + TR = tl.maximum(TR, 0) + boh = tl.load(chunk_offsets + i_n).to(tl.int32) + else: + bos = (i_n * T).to(tl.int32) + eos = (i_n * T + T).to(tl.int32) + TR = T.to(tl.int32) + boh = (i_n * NT).to(tl.int32) + + # 预计算步长（以扁平指针计） + stride_h_t = H * K * V # h 时间块步长 + stride_k_hg = Hg * K # k 的 Hg*K 步长 + stride_w_hk = H * K # w 的 H*K 步长 + stride_v_hv = H * V # v/v_new 的 H*V 步长 + + # 移动到当前 sequence/head 的基址 + h = h + (boh * H + i_h) * (K * V) + v = v + (bos * stride_v_hv + i_h * V) + if SAVE_NEW_VALUE: + v_new = v_new + (bos * stride_v_hv + i_h * V) + k = k + (bos * stride_k_hg + (i_h // (H // Hg)) * K) + w = w + (bos * stride_w_hk + i_h * K) + + # hidden state tiles + b_h1 = tl.zeros([64, BV], dtype=tl.float32) + if K > 64: + b_h2 = tl.zeros([64, BV], dtype=tl.float32) + if K > 128: + b_h3 = tl.zeros([64, BV], dtype=tl.float32) + if K > 192: + b_h4 = tl.zeros([64, BV], dtype=tl.float32) + + # 初始状态 + if USE_INITIAL_STATE: + h0 = h0 + i_nh * (K * V) + p_h0_1 = tl.make_block_ptr(h0, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) + b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) + if K > 64: + p_h0_2 = tl.make_block_ptr(h0, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) + b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) + if K > 128: + p_h0_3 = tl.make_block_ptr(h0, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) + b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) + if K > 192: + p_h0_4 = tl.make_block_ptr(h0, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) + b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) + + # 最终状态写回基址 + if STORE_FINAL_STATE: + ht = ht + i_nh * (K * V) + + # 主循环（上界 NT） + for i_t in range(NT): + # 先把当前 hidden state 写到 h（合法，因为 h 的时间维是 NT 上界） + p_h1 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) + tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) + if K > 64: + p_h2 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) + tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1)) + if K > 128: + p_h3 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) + tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1)) + if K > 192: + p_h4 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) + tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) + + # 这个 chunk 是否有效（t0 >= TR 则整个 tile 越界 -> 跳过所有 block_ptr 操作） + t0 = i_t * BT + active = t0 < TR + if active: + # v_new = v - w @ h + # --- w @ h --- + b_v_new = tl.zeros([BT, BV], dtype=tl.float32) + + # w 的 0..63 列 + p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 0), (BT, 64), (1, 0)) + b_w = tl.load(p_w, boundary_check=(0, 1)) # 不能使用 mask + b_v_new += tl.dot(b_w, b_h1.to(b_w.dtype)) + + if K > 64: + p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 64), (BT, 64), (1, 0)) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v_new += tl.dot(b_w, b_h2.to(b_w.dtype)) + if K > 128: + p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 128), (BT, 64), (1, 0)) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v_new += tl.dot(b_w, b_h3.to(b_w.dtype)) + if K > 192: + p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 192), (BT, 64), (1, 0)) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v_new += tl.dot(b_w, b_h4.to(b_w.dtype)) + + # v - (w @ h) + p_v = tl.make_block_ptr(v, (TR, V), (stride_v_hv, 1), (t0, i_v * BV), (BT, BV), (1, 0)) + b_v_chunk = tl.load(p_v, boundary_check=(0, 1)) # 不能使用 mask + b_v_new = -b_v_new + b_v_chunk + + # 保存 v_new + if SAVE_NEW_VALUE: + p_vn = tl.make_block_ptr(v_new, (TR, V), (stride_v_hv, 1), (t0, i_v * BV), (BT, BV), (1, 0)) + tl.store(p_vn, b_v_new.to(p_vn.dtype.element_ty), boundary_check=(0, 1)) + + # gating：对 g 用普通指针 + mask 读取（不是 block_ptr） + if USE_G: + offs_t = t0 + tl.arange(0, BT) # [BT] + t_mask = offs_t < TR # [BT] + g_base = g + bos * H + i_h + b_g = tl.load(g_base + offs_t * H, mask=t_mask, other=0.0) # ✔ 这里不是 block_ptr，可用 mask + + t_end = tl.minimum(t0 + BT, TR) + last_idx = tl.maximum(t_end - 1, 0).to(tl.int32) + b_g_last = tl.load(g_base + last_idx * H, mask=(TR > 0), other=0.0) + + gating_factor = safe_exp(b_g_last - b_g)[:, None] + b_v_new = b_v_new * gating_factor + + g_last_exp = exp(b_g_last) + b_h1 = b_h1 * g_last_exp + if K > 64: + b_h2 = b_h2 * g_last_exp + if K > 128: + b_h3 = b_h3 * g_last_exp + if K > 192: + b_h4 = b_h4 * g_last_exp + + # h += k^T @ v_new + b_v_new_cast = b_v_new.to(k.dtype.element_ty) + p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (0, t0), (64, BT), (0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) # 不能使用 mask + b_h1 += tl.dot(b_k, b_v_new_cast) + + if K > 64: + p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (64, t0), (64, BT), (0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h2 += tl.dot(b_k, b_v_new_cast) + if K > 128: + p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (128, t0), (64, BT), (0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h3 += tl.dot(b_k, b_v_new_cast) + if K > 192: + p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (192, t0), (64, BT), (0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h4 += tl.dot(b_k, b_v_new_cast) + + # 末态写回 + if STORE_FINAL_STATE: + p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) + tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 64: + p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) + tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 128: + p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) + tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 192: + p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) + tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_gated_delta_rule_fwd_h( + k: torch.Tensor, # [B, T, Hg, K] + w: torch.Tensor, # [B, T, H, K] + u: torch.Tensor, # [B, T, H, V] + g: Optional[torch.Tensor] = None, # [B, T, H] + initial_state: Optional[torch.Tensor] = None, # [B, H, K, V] + output_final_state: bool = False, + chunk_size: int = 64, + save_new_value: bool = True, + cu_seqlens: Optional[torch.LongTensor] = None, # [N+1] 或 None +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """ + Host wrapper： + - 计算 NT（以 chunk 为单位的上界，constexpr）； + - kernel 内对块是否有效做 runtime 守卫（active）； + - block_ptr 的读写只用 boundary_check，严禁 mask/other； + - 对 g 使用标量指针 + mask 读取。 + """ + B, T, Hg, K = k.shape + H, V = u.shape[-2], u.shape[-1] + BT = chunk_size + + if cu_seqlens is None: + N = B + NT = math.ceil(T / BT) # 固定长：全局上界 + chunk_offsets = None + else: + N = len(cu_seqlens) - 1 + chunk_indices = prepare_chunk_indices(cu_seqlens, BT) + NT = len(chunk_indices) # 变长：所有序列的总 chunk 数 + chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT) + + assert K <= 256, "current kernel does not support head dimension larger than 256." + + h = k.new_empty(B, NT, H, K, V) + final_state = k.new_empty(N, H, K, V, dtype=torch.float32) if output_final_state else None + v_new = torch.empty_like(u) if save_new_value else None + + def grid(meta): + return (triton.cdiv(V, meta["BV"]), N * H) + + chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid]( + k=k, + v=u, + w=w, + v_new=v_new, + g=g, + h=h, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + chunk_offsets=chunk_offsets, + T=T, + H=H, + Hg=Hg, + K=K, + V=V, + BT=BT, + BV=32, + NT=NT, + num_warps=2, + num_stages=1, + ) + return h, v_new, final_state + + + + +# from typing import Optional, Tuple +# import math +# import torch +# import triton +# import triton.language as tl + +# from sglang.srt.layers.attention.fla.index import ( +# prepare_chunk_indices, +# prepare_chunk_offsets, +# ) +# from sglang.srt.layers.attention.fla.op import exp, safe_exp +# from sglang.srt.layers.attention.fla.utils import is_nvidia_hopper + +# # 保守一些在 HIP/ROCm 上更稳；NVIDIA 可调大 +# NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8, 16] + + +# @triton.heuristics( +# { +# "USE_G": lambda args: args["g"] is not None, +# "USE_INITIAL_STATE": lambda args: args["h0"] is not None, +# "STORE_FINAL_STATE": lambda args: args["ht"] is not None, +# "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None, +# "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, +# } +# ) +# @triton.jit(do_not_specialize=["T"]) +# def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( +# k, # [B, T, Hg, K] +# v, # [B, T, H, V] +# w, # [B, T, H, K] +# v_new, # [B, T, H, V] or None +# g, # [B, T, H] or None +# h, # [B, NT, H, K, V] +# h0, # [B, H, K, V] or None +# ht, # [B, H, K, V] or None +# cu_seqlens, # [N+1] or None +# chunk_offsets,# [N] or None +# T, # 全局 T（固定长时就是实际长度） +# H: tl.constexpr, +# Hg: tl.constexpr, +# K: tl.constexpr, +# V: tl.constexpr, +# BT: tl.constexpr, # chunk size（通常 64） +# BV: tl.constexpr, # V 方向 tile 宽度（通常 32） +# NT: tl.constexpr, # host 计算的上界（chunk 数） +# USE_G: tl.constexpr, +# USE_INITIAL_STATE: tl.constexpr, +# STORE_FINAL_STATE: tl.constexpr, +# SAVE_NEW_VALUE: tl.constexpr, +# IS_VARLEN: tl.constexpr, +# ): +# # 程序实例：X 维为 V/BV，Y 维为 N*H +# i_v, i_nh = tl.program_id(0), tl.program_id(1) +# i_n = i_nh // H +# i_h = i_nh % H + +# # 计算该序列 runtime 真实长度 TR、时间起点 bos、h 的 chunk 偏移 boh +# if IS_VARLEN: +# bos = tl.load(cu_seqlens + i_n).to(tl.int32) +# eos = tl.load(cu_seqlens + i_n + 1).to(tl.int32) +# TR = (eos - bos).to(tl.int32) +# TR = tl.maximum(TR, 0) +# boh = tl.load(chunk_offsets + i_n).to(tl.int32) +# else: +# bos = (i_n * T).to(tl.int32) +# eos = (i_n * T + T).to(tl.int32) +# TR = T.to(tl.int32) +# boh = (i_n * NT).to(tl.int32) + +# # 预计算步长（以扁平指针计） +# stride_h_t = H * K * V # h 时间块步长 +# stride_k_hg = Hg * K # k 的 Hg*K 步长 +# stride_w_hk = H * K # w 的 H*K 步长 +# stride_v_hv = H * V # v/v_new 的 H*V 步长 + +# # 移动到当前 sequence/head 的基址 +# h = h + (boh * H + i_h) * (K * V) +# v = v + (bos * stride_v_hv + i_h * V) +# if SAVE_NEW_VALUE: +# v_new = v_new + (bos * stride_v_hv + i_h * V) +# k = k + (bos * stride_k_hg + (i_h // (H // Hg)) * K) +# w = w + (bos * stride_w_hk + i_h * K) + +# # hidden state tiles +# b_h1 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 64: +# b_h2 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 128: +# b_h3 = tl.zeros([64, BV], dtype=tl.float32) +# if K > 192: +# b_h4 = tl.zeros([64, BV], dtype=tl.float32) + +# # 初始状态 +# if USE_INITIAL_STATE: +# h0 = h0 + i_nh * (K * V) +# p_h0_1 = tl.make_block_ptr(h0, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) +# b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) +# if K > 64: +# p_h0_2 = tl.make_block_ptr(h0, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) +# b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) +# if K > 128: +# p_h0_3 = tl.make_block_ptr(h0, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) +# b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) +# if K > 192: +# p_h0_4 = tl.make_block_ptr(h0, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) +# b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) + +# # 最终状态写回基址 +# if STORE_FINAL_STATE: +# ht = ht + i_nh * (K * V) + +# # 主循环（上界 NT） +# for i_t in range(NT): +# # 先把当前 hidden state 写到 h（合法，因为 h 的时间维是 NT 上界） +# p_h1 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) +# if K > 64: +# p_h2 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1)) +# if K > 128: +# p_h3 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1)) +# if K > 192: +# p_h4 = tl.make_block_ptr(h + i_t * stride_h_t, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) + +# # 这个 chunk 是否有效（t0 >= TR 则整个 tile 越界 -> 跳过所有 block_ptr 操作） +# t0 = i_t * BT +# active = t0 < TR +# if active: +# # v_new = v - w @ h +# # --- w @ h --- +# b_v_new = tl.zeros([BT, BV], dtype=tl.float32) + +# # w 的 0..63 列 +# p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 0), (BT, 64), (1, 0)) +# b_w = tl.load(p_w, boundary_check=(0, 1)) # 不能使用 mask +# b_v_new += tl.dot(b_w, b_h1.to(b_w.dtype)) + +# if K > 64: +# p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 64), (BT, 64), (1, 0)) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h2.to(b_w.dtype)) +# if K > 128: +# p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 128), (BT, 64), (1, 0)) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h3.to(b_w.dtype)) +# if K > 192: +# p_w = tl.make_block_ptr(w, (TR, K), (stride_w_hk, 1), (t0, 192), (BT, 64), (1, 0)) +# b_w = tl.load(p_w, boundary_check=(0, 1)) +# b_v_new += tl.dot(b_w, b_h4.to(b_w.dtype)) + +# # v - (w @ h) +# p_v = tl.make_block_ptr(v, (TR, V), (stride_v_hv, 1), (t0, i_v * BV), (BT, BV), (1, 0)) +# b_v_chunk = tl.load(p_v, boundary_check=(0, 1)) # 不能使用 mask +# b_v_new = -b_v_new + b_v_chunk + +# # 保存 v_new +# if SAVE_NEW_VALUE: +# p_vn = tl.make_block_ptr(v_new, (TR, V), (stride_v_hv, 1), (t0, i_v * BV), (BT, BV), (1, 0)) +# tl.store(p_vn, b_v_new.to(p_vn.dtype.element_ty), boundary_check=(0, 1)) + +# # gating：对 g 用普通指针 + mask 读取（不是 block_ptr） +# if USE_G: +# offs_t = t0 + tl.arange(0, BT) # [BT] +# t_mask = offs_t < TR # [BT] +# g_base = g + bos * H + i_h +# b_g = tl.load(g_base + offs_t * H, mask=t_mask, other=0.0) # ✔ 这里不是 block_ptr，可用 mask + +# t_end = tl.minimum(t0 + BT, TR) +# last_idx = tl.maximum(t_end - 1, 0).to(tl.int32) +# b_g_last = tl.load(g_base + last_idx * H, mask=(TR > 0), other=0.0) + +# gating_factor = safe_exp(b_g_last - b_g)[:, None] +# b_v_new = b_v_new * gating_factor + +# g_last_exp = exp(b_g_last) +# b_h1 = b_h1 * g_last_exp +# if K > 64: +# b_h2 = b_h2 * g_last_exp +# if K > 128: +# b_h3 = b_h3 * g_last_exp +# if K > 192: +# b_h4 = b_h4 * g_last_exp + +# # h += k^T @ v_new +# b_v_new_cast = b_v_new.to(k.dtype.element_ty) +# p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (0, t0), (64, BT), (0, 1)) +# b_k = tl.load(p_k, boundary_check=(0, 1)) # 不能使用 mask +# b_h1 += tl.dot(b_k, b_v_new_cast) + +# if K > 64: +# p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (64, t0), (64, BT), (0, 1)) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h2 += tl.dot(b_k, b_v_new_cast) +# if K > 128: +# p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (128, t0), (64, BT), (0, 1)) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h3 += tl.dot(b_k, b_v_new_cast) +# if K > 192: +# p_k = tl.make_block_ptr(k, (K, TR), (1, stride_k_hg), (192, t0), (64, BT), (0, 1)) +# b_k = tl.load(p_k, boundary_check=(0, 1)) +# b_h4 += tl.dot(b_k, b_v_new_cast) + +# # 末态写回 +# if STORE_FINAL_STATE: +# p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (0, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 64: +# p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (64, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 128: +# p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (128, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) +# if K > 192: +# p_ht = tl.make_block_ptr(ht, (K, V), (V, 1), (192, i_v * BV), (64, BV), (1, 0)) +# tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + + +# def chunk_gated_delta_rule_fwd_h( +# k: torch.Tensor, # [B, T, Hg, K] +# w: torch.Tensor, # [B, T, H, K] +# u: torch.Tensor, # [B, T, H, V] +# g: Optional[torch.Tensor] = None, # [B, T, H] +# initial_state: Optional[torch.Tensor] = None, # [B, H, K, V] +# output_final_state: bool = False, +# chunk_size: int = 64, +# save_new_value: bool = True, +# cu_seqlens: Optional[torch.LongTensor] = None, # [N+1] 或 None +# ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: +# """ +# Host wrapper： +# - 计算 NT（以 chunk 为单位的上界，constexpr）； +# - kernel 内对块是否有效做 runtime 守卫（active）； +# - block_ptr 的读写只用 boundary_check，严禁 mask/other； +# - 对 g 使用标量指针 + mask 读取。 +# """ +# B, T, Hg, K = k.shape +# H, V = u.shape[-2], u.shape[-1] +# BT = chunk_size + +# if cu_seqlens is None: +# N = B +# NT = math.ceil(T / BT) # 固定长：全局上界 +# chunk_offsets = None +# else: +# N = len(cu_seqlens) - 1 +# chunk_indices = prepare_chunk_indices(cu_seqlens, BT) +# NT = len(chunk_indices) # 变长：所有序列的总 chunk 数 +# chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT) + +# assert K <= 256, "current kernel does not support head dimension larger than 256." + +# h = k.new_empty(B, NT, H, K, V) +# final_state = k.new_empty(N, H, K, V, dtype=torch.float32) if output_final_state else None +# v_new = torch.empty_like(u) if save_new_value else None + +# def grid(meta): +# return (triton.cdiv(V, meta["BV"]), N * H) + +# chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid]( +# k=k, +# v=u, +# w=w, +# v_new=v_new, +# g=g, +# h=h, +# h0=initial_state, +# ht=final_state, +# cu_seqlens=cu_seqlens, +# chunk_offsets=chunk_offsets, +# T=T, +# H=H, +# Hg=Hg, +# K=K, +# V=V, +# BT=BT, +# BV=32, +# NT=NT, +# num_warps=2, +# num_stages=1, +# ) +# return h, v_new, final_state diff --git a/python/sglang/srt/layers/attention/fla/chunk_o.py b/python/sglang/srt/layers/attention/fla/chunk_o.py new file mode 100644 index 000000000..d672c646b --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/chunk_o.py @@ -0,0 +1,178 @@ +# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_o.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import prepare_chunk_indices +from sglang.srt.layers.attention.fla.op import exp, safe_exp +from sglang.srt.layers.attention.fla.utils import check_shared_mem, is_nvidia_hopper + +BKV_LIST = [64, 128] if check_shared_mem() else [32, 64] +NUM_WARPS = [2, 4] if is_nvidia_hopper else [2, 4, 8] + + +@triton.heuristics( + { + "USE_G": lambda args: args["g"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +# @triton.autotune( +# configs=[ +# triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages) +# for BK in BKV_LIST +# for BV in BKV_LIST +# for num_warps in NUM_WARPS +# for num_stages in [2, 3, 4] +# ], +# key=["H", "K", "V", "BT"], +# ) +@triton.jit(do_not_specialize=["T"]) +def chunk_fwd_kernel_o( + q, + k, + v, + h, + g, + o, + cu_seqlens, + chunk_indices, + scale, + T, + H: tl.constexpr, + Hg: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + USE_G: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + + if IS_VARLEN: + i_tg = i_t + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + NT = tl.cdiv(T, BT) + else: + NT = tl.cdiv(T, BT) + i_tg = i_b * NT + i_t + bos, eos = i_b * T, i_b * T + T + + # offset calculation + q += (bos * Hg + i_h // (H // Hg)) * K + k += (bos * Hg + i_h // (H // Hg)) * K + v += (bos * H + i_h) * V + o += (bos * H + i_h) * V + h += (i_tg * H + i_h).to(tl.int64) * K * V + + b_o = tl.zeros([BT, BV], dtype=tl.float32) + b_A = tl.zeros([BT, BT], dtype=tl.float32) + + for i_k in range(tl.cdiv(K, BK)): + p_q = tl.make_block_ptr( + q, (T, K), (Hg * K, 1), (i_t * BT, i_k * BK), (BT, BK), (1, 0) + ) + p_k = tl.make_block_ptr( + k, (K, T), (1, Hg * K), (i_k * BK, i_t * BT), (BK, BT), (0, 1) + ) + p_h = tl.make_block_ptr( + h, (K, V), (V, 1), (i_k * BK, i_v * BV), (BK, BV), (1, 0) + ) + # [BT, BK] + b_q = tl.load(p_q, boundary_check=(0, 1)) + # [BK, BT] + b_k = tl.load(p_k, boundary_check=(0, 1)) + # [BK, BV] + b_h = tl.load(p_h, boundary_check=(0, 1)) + + # [BT, BK] @ [BK, BV] -> [BT, BV] + b_o += tl.dot(b_q, b_h) + # [BT, BK] @ [BK, BT] -> [BT, BT] + b_A += tl.dot(b_q, b_k) + + if USE_G: + g += bos * H + i_h + p_g = tl.make_block_ptr(g, (T,), (H,), (i_t * BT,), (BT,), (0,)) + b_g = tl.load(p_g, boundary_check=(0,)) + b_o = b_o * exp(b_g)[:, None] + b_A = b_A * safe_exp(b_g[:, None] - b_g[None, :]) + + o_i = tl.arange(0, BT) + m_A = o_i[:, None] >= o_i[None, :] + b_A = tl.where(m_A, b_A, 0) + + p_v = tl.make_block_ptr( + v, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) + ) + p_o = tl.make_block_ptr( + o, (T, V), (H * V, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) + + # to fix mma -> mma layout conversion + # already solved by triton v3.2 or higher + b_o = b_o * scale + tl.dot(b_A.to(b_v.dtype), b_v) * scale + tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_fwd_o( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + h: torch.Tensor, + g: Optional[torch.Tensor] = None, # cumsum of log decay + scale: Optional[float] = None, + cu_seqlens: Optional[torch.LongTensor] = None, + chunk_size: int = 64, +) -> torch.Tensor: + B, T, Hg, K, V = *q.shape, v.shape[-1] + H = v.shape[-2] + BT = min(chunk_size, max(16, triton.next_power_of_2(T))) + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + if scale is None: + scale = k.shape[-1] ** -0.5 + + o = torch.empty_like(v) + + def grid(meta): + return (triton.cdiv(V, meta["BV"]), NT, B * H) + + chunk_fwd_kernel_o[grid]( + q, + k, + v, + h, + g, + o, + cu_seqlens, + chunk_indices, + scale, + T=T, + H=H, + Hg=Hg, + K=K, + V=V, + BT=BT, + BK=128, + BV=64, + num_warps=4, + num_stages=2, + ) + return o diff --git a/python/sglang/srt/layers/attention/fla/chunk_scaled_dot_kkt.py b/python/sglang/srt/layers/attention/fla/chunk_scaled_dot_kkt.py new file mode 100644 index 000000000..699350d31 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/chunk_scaled_dot_kkt.py @@ -0,0 +1,151 @@ +# Adapted from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/common/chunk_scaled_dot_kkt.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import prepare_chunk_indices +from sglang.srt.layers.attention.fla.op import safe_exp + + +@triton.heuristics( + { + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + "USE_G": lambda args: args["g_cumsum"] is not None, + } +) +# @triton.autotune( +# configs=[ +# triton.Config({"BK": BK}, num_warps=num_warps, num_stages=num_stages) +# for BK in [32, 64, 128] +# for num_warps in [2, 4, 8] +# for num_stages in [2, 3, 4] +# ], +# key=["H", "K", "BT", "IS_VARLEN"], +# ) +@triton.jit(do_not_specialize=["T"]) +def chunk_scaled_dot_kkt_fwd_kernel( + k, + beta, + g_cumsum, + A, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + Hg: tl.constexpr, + K: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + IS_VARLEN: tl.constexpr, + USE_G: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + o_t = tl.arange(0, BT) + + p_beta = tl.make_block_ptr( + beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + ) + b_beta = tl.load(p_beta, boundary_check=(0,)) + + b_A = tl.zeros([BT, BT], dtype=tl.float32) + for i_k in range(tl.cdiv(K, BK)): + p_k = tl.make_block_ptr( + k + (bos * Hg + i_h // (H // Hg)) * K, + (T, K), + (Hg * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_kb = b_k * b_beta[:, None] + b_A += tl.dot(b_kb.to(b_k.dtype), tl.trans(b_k)) + + if USE_G: + p_g = tl.make_block_ptr( + g_cumsum + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + ) + b_g = tl.load(p_g, boundary_check=(0,)) + b_g_diff = b_g[:, None] - b_g[None, :] + b_A = b_A * safe_exp(b_g_diff) + + b_A = tl.where(o_t[:, None] > o_t[None, :], b_A, 0) + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (BT * H, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + tl.store(p_A, b_A.to(p_A.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_scaled_dot_kkt_fwd( + k: torch.Tensor, + beta: torch.Tensor, + g_cumsum: Optional[torch.Tensor] = None, + cu_seqlens: Optional[torch.LongTensor] = None, + chunk_size: int = 64, + output_dtype: torch.dtype = torch.float32, +) -> torch.Tensor: + r""" + Compute beta * K * K^T. + + Args: + k (torch.Tensor): + The key tensor of shape `[B, T, H, K]`. + beta (torch.Tensor): + The beta tensor of shape `[B, T, H]`. + g_cumsum (torch.Tensor): + The cumulative sum of the gate tensor of shape `[B, T, H]`. + Default: None + cu_seqlens (torch.LongTensor): + The cumulative sequence lengths of the input tensor. + Default: None + chunk_size (int): + The chunk size. Default: 64. + output_dtype (torch.dtype): + The dtype of the output tensor. Default: `torch.float32` + + Returns: + beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size. + """ + + B, T, Hg, K = k.shape + + H = beta.shape[-1] + BT = chunk_size + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + A = torch.empty(B, T, H, BT, device=k.device, dtype=output_dtype) + chunk_scaled_dot_kkt_fwd_kernel[(NT, B * H)]( + k=k, + beta=beta, + g_cumsum=g_cumsum, + A=A, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + Hg=Hg, + K=K, + BT=BT, + BK=64, + num_warps=8, + num_stages=3, + ) + return A diff --git a/python/sglang/srt/layers/attention/fla/cumsum.py b/python/sglang/srt/layers/attention/fla/cumsum.py new file mode 100644 index 000000000..b8e3cdde1 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/cumsum.py @@ -0,0 +1,300 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/cumsum.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import prepare_chunk_indices +from sglang.srt.layers.attention.fla.utils import check_shared_mem, input_guard + +BS_LIST = [32, 64] if check_shared_mem() else [16, 32] + + +@triton.heuristics( + { + "HAS_SCALE": lambda args: args["scale"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +# @triton.autotune( +# configs=[triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8]], +# key=["B", "H", "BT", "IS_VARLEN", "REVERSE"], +# ) +@triton.jit(do_not_specialize=["T"]) +def chunk_local_cumsum_scalar_kernel( + s, + o, + scale, + cu_seqlens, + chunk_indices, + T, + B: tl.constexpr, + H: tl.constexpr, + BT: tl.constexpr, + REVERSE: tl.constexpr, + HAS_SCALE: tl.constexpr, + IS_VARLEN: tl.constexpr, + HEAD_FIRST: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + if HEAD_FIRST: + p_s = tl.make_block_ptr( + s + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,) + ) + p_o = tl.make_block_ptr( + o + bos * H + i_h * T, (T,), (1,), (i_t * BT,), (BT,), (0,) + ) + else: + p_s = tl.make_block_ptr(s + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) + p_o = tl.make_block_ptr(o + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) + # [BT] + b_s = tl.load(p_s, boundary_check=(0,)).to(tl.float32) + b_o = tl.cumsum(b_s, axis=0) + if REVERSE: + b_z = tl.sum(b_s, axis=0) + b_o = -b_o + b_z[None] + b_s + if HAS_SCALE: + b_o *= scale + tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0,)) + + +@triton.heuristics( + { + "HAS_SCALE": lambda args: args["scale"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BS": BS}, num_warps=num_warps) + for BS in BS_LIST + for num_warps in [2, 4, 8] + ], + key=["B", "H", "S", "BT", "IS_VARLEN", "REVERSE"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_local_cumsum_vector_kernel( + s, + o, + scale, + cu_seqlens, + chunk_indices, + T, + B: tl.constexpr, + H: tl.constexpr, + S: tl.constexpr, + BT: tl.constexpr, + BS: tl.constexpr, + REVERSE: tl.constexpr, + HAS_SCALE: tl.constexpr, + IS_VARLEN: tl.constexpr, + HEAD_FIRST: tl.constexpr, +): + i_s, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + o_i = tl.arange(0, BT) + if REVERSE: + m_s = tl.where(o_i[:, None] <= o_i[None, :], 1.0, 0.0) + else: + m_s = tl.where(o_i[:, None] >= o_i[None, :], 1.0, 0.0) + + if HEAD_FIRST: + p_s = tl.make_block_ptr( + s + (bos * H + i_h * T) * S, + (T, S), + (S, 1), + (i_t * BT, i_s * BS), + (BT, BS), + (1, 0), + ) + p_o = tl.make_block_ptr( + o + (bos * H + i_h * T) * S, + (T, S), + (S, 1), + (i_t * BT, i_s * BS), + (BT, BS), + (1, 0), + ) + else: + p_s = tl.make_block_ptr( + s + (bos * H + i_h) * S, + (T, S), + (H * S, 1), + (i_t * BT, i_s * BS), + (BT, BS), + (1, 0), + ) + p_o = tl.make_block_ptr( + o + (bos * H + i_h) * S, + (T, S), + (H * S, 1), + (i_t * BT, i_s * BS), + (BT, BS), + (1, 0), + ) + # [BT, BS] + b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32) + b_o = tl.dot(m_s, b_s, allow_tf32=False) + if HAS_SCALE: + b_o *= scale + tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_local_cumsum_scalar( + g: torch.Tensor, + chunk_size: int, + reverse: bool = False, + scale: float = None, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + output_dtype: Optional[torch.dtype] = torch.float, +) -> torch.Tensor: + if head_first: + B, H, T = g.shape + else: + B, T, H = g.shape + assert chunk_size == 2 ** ( + chunk_size.bit_length() - 1 + ), "chunk_size must be a power of 2" + BT = chunk_size + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + g_org, g = g, torch.empty_like(g, dtype=output_dtype or g.dtype) + grid = (NT, B * H) + chunk_local_cumsum_scalar_kernel[grid]( + s=g_org, + o=g, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + B=B, + H=H, + BT=BT, + HEAD_FIRST=head_first, + REVERSE=reverse, + num_warps=8, + num_stages=3, + ) + return g + + +def chunk_local_cumsum_vector( + g: torch.Tensor, + chunk_size: int, + reverse: bool = False, + scale: float = None, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + output_dtype: Optional[torch.dtype] = torch.float, +) -> torch.Tensor: + if head_first: + B, H, T, S = g.shape + else: + B, T, H, S = g.shape + BT = chunk_size + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, chunk_size) + if cu_seqlens is not None + else None + ) + NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + assert chunk_size == 2 ** ( + chunk_size.bit_length() - 1 + ), "chunk_size must be a power of 2" + + g_org, g = g, torch.empty_like(g, dtype=output_dtype or g.dtype) + + def grid(meta): + return (triton.cdiv(meta["S"], meta["BS"]), NT, B * H) + + # keep cumulative normalizer in fp32 + # this kernel is equivalent to + # g = g.view(B, H, NT, BT, -1).cumsum(-2).view(B, H, T, -1) + chunk_local_cumsum_vector_kernel[grid]( + s=g_org, + o=g, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + B=B, + H=H, + S=S, + BT=BT, + HEAD_FIRST=head_first, + REVERSE=reverse, + ) + return g + + +@input_guard +def chunk_local_cumsum( + g: torch.Tensor, + chunk_size: int, + reverse: bool = False, + scale: float = None, + cu_seqlens: Optional[torch.Tensor] = None, + head_first: bool = False, + output_dtype: Optional[torch.dtype] = torch.float, + **kwargs, +) -> torch.Tensor: + if cu_seqlens is not None: + assert ( + g.shape[0] == 1 + ), "Only batch size 1 is supported when cu_seqlens are provided" + if len(g.shape) == 3: + return chunk_local_cumsum_scalar( + g=g, + chunk_size=chunk_size, + reverse=reverse, + scale=scale, + cu_seqlens=cu_seqlens, + head_first=head_first, + output_dtype=output_dtype, + ) + elif len(g.shape) == 4: + return chunk_local_cumsum_vector( + g=g, + chunk_size=chunk_size, + reverse=reverse, + scale=scale, + cu_seqlens=cu_seqlens, + head_first=head_first, + output_dtype=output_dtype, + ) + else: + raise ValueError( + f"Unsupported input shape {g.shape}, " + f"which should be (B, T, H, D) if `head_first=False` " + f"or (B, H, T, D) otherwise" + ) diff --git a/python/sglang/srt/layers/attention/fla/fused_recurrent.py b/python/sglang/srt/layers/attention/fla/fused_recurrent.py new file mode 100644 index 000000000..fa7262ce2 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/fused_recurrent.py @@ -0,0 +1,640 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/fused_recurrent.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.op import exp +from sglang.srt.layers.attention.fla.utils import input_guard + + +@triton.heuristics( + { + "USE_INITIAL_STATE": lambda args: args["h0"] is not None, + "STORE_FINAL_STATE": lambda args: args["ht"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.jit(do_not_specialize=["T"]) +def fused_recurrent_gated_delta_rule_fwd_kernel( + q, + k, + v, + g, + beta, + o, + h0, + ht, + cu_seqlens, + scale, + T, + B: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, # whether to use initial state + STORE_FINAL_STATE: tl.constexpr, # whether to store final state + IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + if IS_VARLEN: + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int64) + all = T + T = eos - bos + else: + bos, eos = i_n * T, i_n * T + T + all = B * T + o_k = i_k * BK + tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + + p_q = q + (bos * H + i_h) * K + o_k + p_k = k + (bos * H + i_h) * K + o_k + p_v = v + (bos * HV + i_hv) * V + o_v + if IS_BETA_HEADWISE: + p_beta = beta + (bos * HV + i_hv) * V + o_v + else: + p_beta = beta + bos * HV + i_hv + p_g = g + bos * HV + i_hv + p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v + + mask_k = o_k < K + mask_v = o_v < V + mask_h = mask_k[:, None] & mask_v[None, :] + + b_h = tl.zeros([BK, BV], dtype=tl.float32) + if USE_INITIAL_STATE: + p_h0 = h0 + i_nh * K * V + o_k[:, None] * V + o_v[None, :] + b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) + + for _ in range(0, T): + b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) + b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) + b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) + b_g = tl.load(p_g).to(tl.float32) + + if USE_QK_L2NORM_IN_KERNEL: + b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q)) + 1e-6) + b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k)) + 1e-6) + b_q = b_q * scale + # [BK, BV] + b_h *= exp(b_g) + # [BV] + b_v -= tl.sum(b_h * b_k[:, None], 0) + if IS_BETA_HEADWISE: + b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32) + else: + b_beta = tl.load(p_beta).to(tl.float32) + b_v *= b_beta + # [BK, BV] + b_h += b_k[:, None] * b_v[None, :] + # [BV] + b_o = tl.sum(b_h * b_q[:, None], 0) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) + + p_q += H * K + p_k += H * K + p_o += HV * V + p_v += HV * V + p_g += HV + p_beta += HV * (V if IS_BETA_HEADWISE else 1) + + if STORE_FINAL_STATE: + p_ht = ht + i_nh * K * V + o_k[:, None] * V + o_v[None, :] + tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) + + +def fused_recurrent_gated_delta_rule_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: Optional[torch.LongTensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 8) + NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + o = q.new_empty(NK, *v.shape) + if output_final_state: + final_state = q.new_empty(N, HV, K, V, dtype=torch.float32) + else: + final_state = None + + grid = (NK, NV, N * HV) + fused_recurrent_gated_delta_rule_fwd_kernel[grid]( + q=q, + k=k, + v=v, + g=g, + beta=beta, + o=o, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + scale=scale, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + IS_BETA_HEADWISE=beta.ndim == v.ndim, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + num_warps=num_warps, + num_stages=num_stages, + ) + o = o.squeeze(0) + return o, final_state + + +class FusedRecurrentFunction(torch.autograd.Function): + + @staticmethod + @input_guard + def forward( + ctx, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: Optional[torch.LongTensor] = None, + use_qk_l2norm_in_kernel: bool = False, + ): + o, final_state = fused_recurrent_gated_delta_rule_fwd( + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=scale, + initial_state=initial_state, + output_final_state=output_final_state, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + cu_seqlens=cu_seqlens, + ) + + return o, final_state + + @staticmethod + @input_guard + def backward(ctx, do, dht): + raise NotImplementedError( + "Backward pass is not implemented yet and we do not have plans to implement it " + "because we haven't figured out how to compute dg without materializing the full " + "hidden states for all time steps." + ) + + +def fused_recurrent_gated_delta_rule( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor = None, + scale: float = None, + initial_state: torch.Tensor = None, + output_final_state: bool = False, + cu_seqlens: Optional[torch.LongTensor] = None, + use_qk_l2norm_in_kernel: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + r""" + Args: + q (torch.Tensor): + queries of shape `[B, T, H, K]`. + k (torch.Tensor): + keys of shape `[B, T, H, K]`. + v (torch.Tensor): + values of shape `[B, T, HV, V]`. + GVA is applied if `HV > H`. + g (torch.Tensor): + g (decays) of shape `[B, T, HV]`. + beta (torch.Tensor): + betas of shape `[B, T, HV]`. + scale (Optional[int]): + Scale factor for the RetNet attention scores. + If not provided, it will default to `1 / sqrt(K)`. Default: `None`. + initial_state (Optional[torch.Tensor]): + Initial state of shape `[N, HV, K, V]` for `N` input sequences. + For equal-length input sequences, `N` equals the batch size `B`. + Default: `None`. + output_final_state (Optional[bool]): + Whether to output the final state of shape `[N, HV, K, V]`. Default: `False`. + cu_seqlens (torch.LongTensor): + Cumulative sequence lengths of shape `[N+1]` used for variable-length training, + consistent with the FlashAttention API. + Returns: + o (torch.Tensor): + Outputs of shape `[B, T, HV, V]`. + final_state (torch.Tensor): + Final state of shape `[N, HV, K, V]` if `output_final_state=True` else `None`. + Examples:: + >>> import torch + >>> import torch.nn.functional as F + >>> from einops import rearrange + >>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule + # inputs with equal lengths + >>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512 + >>> q = torch.randn(B, T, H, K, device='cuda') + >>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1) + >>> v = torch.randn(B, T, HV, V, device='cuda') + >>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda')) + >>> beta = torch.rand(B, T, HV, device='cuda').sigmoid() + >>> h0 = torch.randn(B, HV, K, V, device='cuda') + >>> o, ht = fused_gated_recurrent_delta_rule( + q, k, v, g, beta, + initial_state=h0, + output_final_state=True + ) + # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required + >>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta)) + # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected + >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.long) + >>> o_var, ht_var = fused_gated_recurrent_delta_rule( + q, k, v, g, beta, + initial_state=h0, + output_final_state=True, + cu_seqlens=cu_seqlens + ) + """ + if cu_seqlens is not None: + if q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if initial_state is not None and initial_state.shape[0] != len(cu_seqlens) - 1: + raise ValueError( + f"The number of initial states is expected to be equal to the number of input sequences, " + f"i.e., {len(cu_seqlens) - 1} rather than {initial_state.shape[0]}." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + else: + assert scale > 0, "scale must be positive" + if beta is None: + beta = torch.ones_like(q[..., 0]) + o, final_state = FusedRecurrentFunction.apply( + q, + k, + v, + g, + beta, + scale, + initial_state, + output_final_state, + cu_seqlens, + use_qk_l2norm_in_kernel, + ) + return o, final_state + + +@triton.heuristics( + { + "USE_INITIAL_STATE": lambda args: args["h0_source"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + "CACHE_INTERMEDIATE_STATES": lambda args: args["intermediate_states_buffer"] + is not None, + } +) +@triton.jit(do_not_specialize=["T"]) +def fused_recurrent_gated_delta_rule_update_fwd_kernel( + q, + k, + v, + g, + beta, + o, + h0_source, + h0_indices, + cu_seqlens, + scale, + intermediate_states_buffer, + cache_steps, + T, + B: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, # whether to use initial state + IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + IS_VARLEN: tl.constexpr, + DISABLE_STATE_UPDATE: tl.constexpr, # whether to disable final state update + DISABLE_OUTPUT_CALCULATION: tl.constexpr, # whether to disable output calculation + CACHE_INTERMEDIATE_STATES: tl.constexpr, +): + i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + if IS_VARLEN: + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int64), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int64) + all = T + T = eos - bos + else: + bos, eos = i_n * T, i_n * T + T + all = B * T + o_k = i_k * BK + tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + + p_q = q + (bos * H + i_h) * K + o_k + p_k = k + (bos * H + i_h) * K + o_k + p_v = v + (bos * HV + i_hv) * V + o_v + if IS_BETA_HEADWISE: + p_beta = beta + (bos * HV + i_hv) * V + o_v + else: + p_beta = beta + bos * HV + i_hv + p_g = g + bos * HV + i_hv + p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v + + mask_k = o_k < K + mask_v = o_v < V + mask_h = mask_k[:, None] & mask_v[None, :] + + b_h = tl.zeros([BK, BV], dtype=tl.float32) + if USE_INITIAL_STATE: + idx = tl.load(h0_indices + i_n) + # Add bounds checking for idx + if idx >= 0: # Assuming negative indices are invalid + p_h0 = ( + h0_source + + idx * HV * K * V + + i_hv * K * V + + o_k[:, None] * V + + o_v[None, :] + ) + b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) + + # Prepare intermediate state cache variables if enabled + cache_idx = -1 + if CACHE_INTERMEDIATE_STATES: + cache_idx = tl.load(h0_indices + i_n) + + step_idx = 0 + for _ in range(0, T): + b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) + b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) + b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) + b_g = tl.load(p_g).to(tl.float32) + + if USE_QK_L2NORM_IN_KERNEL: + b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q)) + 1e-6) + b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k)) + 1e-6) + b_q = b_q * scale + # [BK, BV] + b_h *= exp(b_g) + # [BV] + b_v -= tl.sum(b_h * b_k[:, None], 0) + if IS_BETA_HEADWISE: + b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32) + else: + b_beta = tl.load(p_beta).to(tl.float32) + b_v *= b_beta + # [BK, BV] + b_h += b_k[:, None] * b_v[None, :] + # [BV] + if not DISABLE_OUTPUT_CALCULATION: + b_o = tl.sum(b_h * b_q[:, None], 0) + # core attn output + tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) + + # store intermediate states if enabled + if CACHE_INTERMEDIATE_STATES: + if cache_idx >= 0: + # Compute cache pointer for this step + step_offset = step_idx * HV * K * V + cache_ptr = ( + intermediate_states_buffer + + cache_idx * cache_steps * HV * K * V + + step_offset + + i_hv * K * V + + o_k[:, None] * V + + o_v[None, :] + ) + tl.store(cache_ptr, b_h.to(cache_ptr.dtype.element_ty), mask=mask_h) + + step_idx += 1 + + p_q += H * K + p_k += H * K + p_o += HV * V + p_v += HV * V + p_g += HV + p_beta += HV * (V if IS_BETA_HEADWISE else 1) + + # Store final state back to h0_source with bounds checking + # ssm states + if not DISABLE_STATE_UPDATE: + idx = tl.load(h0_indices + i_n) + if idx >= 0: # Add bounds checking + p_h0 = ( + h0_source + + idx * HV * K * V + + i_hv * K * V + + o_k[:, None] * V + + o_v[None, :] + ) + tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) + + +def fused_recurrent_gated_delta_rule_update_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state_source: torch.Tensor, + initial_state_indices: torch.Tensor, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: Optional[torch.LongTensor] = None, + disable_state_update: bool = False, + disable_output_calculation: bool = False, + intermediate_states_buffer: Optional[torch.Tensor] = None, + cache_steps: Optional[int] = None, +) -> torch.Tensor: + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 8) + NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + if disable_output_calculation: + # When output calculation is disabled, allocate minimal tensor + o = q.new_empty(NK, 1, 1, 1, 1) # minimal allocation + else: + o = q.new_empty(NK, *v.shape) + + grid = (NK, NV, N * HV) + + fused_recurrent_gated_delta_rule_update_fwd_kernel[grid]( + q=q, + k=k, + v=v, + g=g, + beta=beta, + o=o, + h0_source=initial_state_source, + h0_indices=initial_state_indices, + cu_seqlens=cu_seqlens, + scale=scale, + intermediate_states_buffer=intermediate_states_buffer, + cache_steps=0 if cache_steps is None else cache_steps, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + IS_BETA_HEADWISE=beta.ndim == v.ndim, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + DISABLE_STATE_UPDATE=disable_state_update, + DISABLE_OUTPUT_CALCULATION=disable_output_calculation, + num_warps=num_warps, + num_stages=num_stages, + ) + o = o.squeeze(0) + return o + + +class FusedRecurrentUpdateFunction(torch.autograd.Function): + + @staticmethod + @input_guard + def forward( + ctx, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state_source: torch.Tensor, + initial_state_indices: torch.Tensor, + cu_seqlens: Optional[torch.LongTensor] = None, + use_qk_l2norm_in_kernel: bool = False, + disable_state_update: bool = False, + disable_output_calculation: bool = False, + intermediate_states_buffer: Optional[torch.Tensor] = None, + cache_steps: Optional[int] = None, + ): + o = fused_recurrent_gated_delta_rule_update_fwd( + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=scale, + initial_state_source=initial_state_source, + initial_state_indices=initial_state_indices, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + cu_seqlens=cu_seqlens, + disable_state_update=disable_state_update, + disable_output_calculation=disable_output_calculation, + intermediate_states_buffer=intermediate_states_buffer, + cache_steps=cache_steps, + ) + + return o + + @staticmethod + @input_guard + def backward(ctx, do, dht): + raise NotImplementedError( + "Backward pass is not implemented yet and we do not have plans to implement it " + "because we haven't figured out how to compute dg without materializing the full " + "hidden states for all time steps." + ) + + +def fused_recurrent_gated_delta_rule_update( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor = None, + scale: float = None, + initial_state_source: torch.Tensor = None, + initial_state_indices: torch.Tensor = None, + cu_seqlens: Optional[torch.LongTensor] = None, + use_qk_l2norm_in_kernel: bool = False, + disable_state_update: bool = False, + disable_output_calculation: bool = False, + intermediate_states_buffer: Optional[torch.Tensor] = None, + cache_steps: Optional[int] = None, +) -> torch.Tensor: + if cu_seqlens is not None: + if q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if ( + initial_state_source is not None + and initial_state_indices.shape[0] != len(cu_seqlens) - 1 + ): + raise ValueError( + f"The number of initial states is expected to be equal to the number of input sequences, " + f"i.e., {len(cu_seqlens) - 1} rather than {initial_state_indices.shape[0]}." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + else: + assert scale > 0, "scale must be positive" + if beta is None: + beta = torch.ones_like(q[..., 0]) + o = FusedRecurrentUpdateFunction.apply( + q, + k, + v, + g, + beta, + scale, + initial_state_source, + initial_state_indices, + cu_seqlens, + use_qk_l2norm_in_kernel, + disable_state_update, + disable_output_calculation, + intermediate_states_buffer, + cache_steps, + ) + return o diff --git a/python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py b/python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py new file mode 100644 index 000000000..41837b980 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/fused_sigmoid_gating_recurrent.py @@ -0,0 +1,232 @@ +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.utils import input_guard + + +@triton.heuristics( + { + "USE_INITIAL_STATE": lambda args: args["h0_source"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.jit(do_not_specialize=["T"]) +def fused_sigmoid_gating_delta_rule_update_kernel( + A_log, + a, + dt_bias, + softplus_beta, + softplus_threshold, + q, + k, + v, + b, + o, + h0_source, + h0_indices, + cu_seqlens, + scale, + T, + B: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + """ + Fused kernel that combines sigmoid gating computation with recurrent delta rule update. + """ + i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + + if IS_VARLEN: + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int64), + tl.load(cu_seqlens + i_n + 1).to(tl.int64), + ) + all = T + T = eos - bos + else: + bos, eos = i_n * T, i_n * T + T + all = B * T + + o_k = i_k * BK + tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + + p_q = q + (bos * H + i_h) * K + o_k + p_k = k + (bos * H + i_h) * K + o_k + p_v = v + (bos * HV + i_hv) * V + o_v + p_b = b + bos * HV + i_hv + p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v + + # Gating computation pointers + p_A_log = A_log + i_hv + p_a = a + bos * HV + i_hv + p_dt_bias = dt_bias + i_hv + + mask_k = o_k < K + mask_v = o_v < V + mask_h = mask_k[:, None] & mask_v[None, :] + + b_h = tl.zeros([BK, BV], dtype=tl.float32) + if USE_INITIAL_STATE: + idx = tl.load(h0_indices + i_n) + if idx >= 0: + p_h0 = ( + h0_source + + idx * HV * K * V + + i_hv * K * V + + o_k[:, None] * V + + o_v[None, :] + ) + b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) + + for _ in range(0, T): + # Load inputs + b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) + b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) + b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) + b_b = tl.load(p_b).to(tl.float32) + + # Compute sigmoid gating + # Load gating parameters + b_A_log = tl.load(p_A_log).to(tl.float32) + b_a = tl.load(p_a).to(tl.float32) + b_dt_bias = tl.load(p_dt_bias).to(tl.float32) + + # Compute g = -exp(A_log) * softplus(a + dt_bias) + x = b_a + b_dt_bias + beta_x = softplus_beta * x + # Apply softplus with numerical stability + softplus_x = tl.where( + beta_x <= softplus_threshold, + (1.0 / softplus_beta) * tl.log(1.0 + tl.exp(beta_x)), + x, + ) + b_g = -tl.exp(b_A_log) * softplus_x + + # Compute beta = sigmoid(b) + b_beta = 1.0 / (1.0 + tl.exp(-b_b)) + + # Apply L2 normalization if enabled + if USE_QK_L2NORM_IN_KERNEL: + b_q = b_q / (tl.sqrt(tl.sum(b_q * b_q)) + 1e-6) + b_k = b_k / (tl.sqrt(tl.sum(b_k * b_k)) + 1e-6) + + b_q = b_q * scale + + # Apply gating to hidden state: h *= exp(g) + b_h *= tl.exp(b_g) + + # Delta rule: v -= sum(h * k, dim=0) + b_v -= tl.sum(b_h * b_k[:, None], 0) + + # Apply beta gating: v *= beta + b_v *= b_beta + + # Update hidden state: h += k[:, None] * v[None, :] + b_h += b_k[:, None] * b_v[None, :] + + # Compute output: o = sum(h * q, dim=0) + b_o = tl.sum(b_h * b_q[:, None], 0) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) + + # Update pointers for next timestep + p_q += H * K + p_k += H * K + p_o += HV * V + p_v += HV * V + p_b += HV + p_a += HV + + # Store final state back to h0_source with bounds checking + if USE_INITIAL_STATE: + idx = tl.load(h0_indices + i_n) + if idx >= 0: + p_h0 = ( + h0_source + + idx * HV * K * V + + i_hv * K * V + + o_k[:, None] * V + + o_v[None, :] + ) + tl.store(p_h0, b_h.to(p_h0.dtype.element_ty), mask=mask_h) + + +@input_guard +def fused_sigmoid_gating_delta_rule_update( + A_log: torch.Tensor, + a: torch.Tensor, + dt_bias: torch.Tensor, + softplus_beta: float, + softplus_threshold: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + b: torch.Tensor, + initial_state_source: torch.Tensor, + initial_state_indices: torch.Tensor, + scale: Optional[float] = None, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: Optional[torch.Tensor] = None, +): + """ + Fused triton implementation of sigmoid gating delta rule update. + This function uses a single fused kernel that combines both sigmoid gating computation + and the recurrent delta rule update for better performance. + """ + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 8) + NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + if scale is None: + scale = k.shape[-1] ** -0.5 + else: + assert scale > 0, "scale must be positive" + + o = q.new_empty(NK, *v.shape) + grid = (NK, NV, N * HV) + + fused_sigmoid_gating_delta_rule_update_kernel[grid]( + A_log=A_log, + a=a, + dt_bias=dt_bias, + softplus_beta=softplus_beta, + softplus_threshold=softplus_threshold, + q=q, + k=k, + v=v, + b=b, + o=o, + h0_source=initial_state_source, + h0_indices=initial_state_indices, + cu_seqlens=cu_seqlens, + scale=scale, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + num_warps=num_warps, + num_stages=num_stages, + ) + o = o.squeeze(0) + return o diff --git a/python/sglang/srt/layers/attention/fla/index.py b/python/sglang/srt/layers/attention/fla/index.py new file mode 100644 index 000000000..754b98714 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/index.py @@ -0,0 +1,37 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/index.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.utils import tensor_cache + + +@tensor_cache +def prepare_lens(cu_seqlens: torch.LongTensor) -> torch.LongTensor: + return cu_seqlens[1:] - cu_seqlens[:-1] + + +@tensor_cache +def prepare_chunk_indices( + cu_seqlens: torch.LongTensor, chunk_size: int +) -> torch.LongTensor: + indices = torch.cat( + [ + torch.arange(n) + for n in triton.cdiv(prepare_lens(cu_seqlens), chunk_size).tolist() + ] + ) + return torch.stack([indices.eq(0).cumsum(0) - 1, indices], 1).to(cu_seqlens) + + +@tensor_cache +def prepare_chunk_offsets( + cu_seqlens: torch.LongTensor, chunk_size: int +) -> torch.LongTensor: + return torch.cat( + [cu_seqlens.new_tensor([0]), triton.cdiv(prepare_lens(cu_seqlens), chunk_size)] + ).cumsum(-1) diff --git a/python/sglang/srt/layers/attention/fla/l2norm.py b/python/sglang/srt/layers/attention/fla/l2norm.py new file mode 100644 index 000000000..d6b6ae7f7 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/l2norm.py @@ -0,0 +1,150 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/modules/l2norm.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional + +import torch +import torch.nn as nn +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.utils import input_guard + +BT_LIST = [8, 16, 32, 64, 128] + + +# @triton.autotune( +# configs=[ +# triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8, 16, 32] +# ], +# key=["D"], +# ) +@triton.jit +def l2norm_fwd_kernel1( + x, + y, + D, + BD: tl.constexpr, + eps, +): + i_t = tl.program_id(0) + x += i_t * D + y += i_t * D + # Compute mean and variance + cols = tl.arange(0, BD) + mask = cols < D + b_x = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32) + b_var = tl.sum(b_x * b_x, axis=0) + b_rstd = 1 / tl.sqrt(b_var + eps) + # tl.store(Rstd + i_t, rstd) + # Normalize and apply linear transformation + b_y = b_x * b_rstd + tl.store(y + cols, b_y, mask=mask) + + +# @triton.autotune( +# configs=[ +# triton.Config({"BT": BT}, num_warps=num_warps) +# for num_warps in [1, 2, 4, 8, 16] +# for BT in BT_LIST +# ], +# key=["D", "NB"], +# ) +@triton.jit +def l2norm_fwd_kernel( + x, + y, + eps, + NB: tl.constexpr, + T: tl.constexpr, + D: tl.constexpr, + BT: tl.constexpr, + BD: tl.constexpr, +): + i_t = tl.program_id(0) + p_x = tl.make_block_ptr(x, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + b_x = tl.load(p_x, boundary_check=(0, 1)).to(tl.float32) + b_var = tl.sum(b_x * b_x, axis=1) + b_y = b_x / tl.sqrt(b_var + eps)[:, None] + p_y = tl.make_block_ptr(y, (T, D), (D, 1), (i_t * BT, 0), (BT, BD), (1, 0)) + tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1)) + + +def l2norm_fwd( + x: torch.Tensor, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None +): + x_shape_og = x.shape + x = x.view(-1, x.shape[-1]) + # allocate output + if output_dtype is None: + y = torch.empty_like(x) + else: + y = torch.empty_like(x, dtype=output_dtype) + assert y.stride(-1) == 1 + T, D = x.shape[0], x.shape[-1] + # rstd = torch.empty((T,), dtype=torch.float32, device=x.device) + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BD = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BD: + raise RuntimeError("This layer doesn't support feature dim >= 64KB.") + + if D <= 512: + NB = triton.cdiv(T, 2048) + + def grid(meta): + return (triton.cdiv(T, meta["BT"]),) + + l2norm_fwd_kernel[grid]( + x, + y, + eps, + NB=NB, + T=T, + D=D, + BD=BD, + BT=16, + num_warps=8, + num_stages=3, + ) + else: + l2norm_fwd_kernel1[(T,)]( + x, + y, + eps=eps, + D=D, + BD=BD, + num_warps=8, + num_stages=3, + ) + + return y.view(x_shape_og) + + +class L2NormFunction(torch.autograd.Function): + + @staticmethod + @input_guard + def forward(ctx, x, eps=1e-6, output_dtype=None): + return l2norm_fwd(x, eps, output_dtype) + + +def l2norm( + x: torch.Tensor, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + return L2NormFunction.apply(x, eps, output_dtype) + + +l2_norm = l2norm + + +class L2Norm(nn.Module): + + def __init__(self, eps: float = 1e-6, output_dtype: Optional[torch.dtype] = None): + super().__init__() + self.eps = eps + self.output_dtype = output_dtype + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return l2norm(x, self.eps, self.output_dtype) diff --git a/python/sglang/srt/layers/attention/fla/layernorm_gated.py b/python/sglang/srt/layers/attention/fla/layernorm_gated.py new file mode 100644 index 000000000..bd53d0d64 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/layernorm_gated.py @@ -0,0 +1,326 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/modules/layernorm_gated.py +# Copyright (c) 2024, Tri Dao. +# Based on the Triton LayerNorm tutorial: https://triton-lang.org/main/getting-started/tutorials/05-layer-norm.html +# For the backward pass, we keep weight_grad and bias_grad in registers and accumulate. +# This backward pass is faster for dimensions up to 8k, but after that it's much slower due to register spilling. +# The models we train have hidden dim up to 8k anyway (e.g. Llama 70B), so this is fine. + +import math + +import torch +import torch.nn.functional as F +import triton +import triton.language as tl +from einops import rearrange + + +def rms_norm_ref( + x, + weight, + bias, + z=None, + eps=1e-6, + group_size=None, + norm_before_gate=True, + upcast=True, +): + dtype = x.dtype + N = x.shape[-1] + weight = weight.float() + bias = bias.float() if bias is not None else None + if upcast: + x = x.float() + z = z.float() if z is not None else z + if z is not None and not norm_before_gate: + x = x * F.silu(z) + if group_size is None: + rstd = 1 / torch.sqrt((x.square()).mean(dim=-1, keepdim=True) + eps) + out = (x * rstd * weight) + bias if bias is not None else (x * rstd * weight) + else: + x_group = rearrange(x, "... (g d) -> ... g d", d=group_size) + rstd = 1 / torch.sqrt((x_group.square()).mean(dim=-1, keepdim=True) + eps) + out = rearrange(x_group * rstd, "... g d -> ... (g d)") * weight + if bias is not None: + out = out + bias + if z is not None and norm_before_gate: + out *= F.silu(z) + return out.to(dtype) + + +@triton.heuristics({"HAS_BIAS": lambda args: args["B"] is not None}) +@triton.heuristics({"HAS_Z": lambda args: args["Z"] is not None}) +@triton.jit +def _layer_norm_fwd_1pass_kernel( + X, # pointer to the input + Y, # pointer to the output + W, # pointer to the weights + B, # pointer to the biases + Z, # pointer to the other branch + Mean, # pointer to the mean + Rstd, # pointer to the 1/std + stride_x_row, # how much to increase the pointer when moving by 1 row + stride_y_row, + stride_z_row, + M, # number of rows in X + N, # number of columns in X + eps, # epsilon to avoid division by zero + BLOCK_N: tl.constexpr, + HAS_BIAS: tl.constexpr, + HAS_Z: tl.constexpr, + NORM_BEFORE_GATE: tl.constexpr, + IS_RMS_NORM: tl.constexpr, +): + # Map the program id to the row of X and Y it should compute. + row = tl.program_id(0) + group = tl.program_id(1) + X += row * stride_x_row + group * N + Y += row * stride_y_row + group * N + if HAS_Z: + Z += row * stride_z_row + group * N + if not IS_RMS_NORM: + Mean += group * M + Rstd += group * M + W += group * N + if HAS_BIAS: + B += group * N + # Compute mean and variance + cols = tl.arange(0, BLOCK_N) + x = tl.load(X + cols, mask=cols < N, other=0.0).to(tl.float32) + if HAS_Z and not NORM_BEFORE_GATE: + z = tl.load(Z + cols, mask=cols < N).to(tl.float32) + x *= z * tl.sigmoid(z) + if not IS_RMS_NORM: + mean = tl.sum(x, axis=0) / N + tl.store(Mean + row, mean) + xbar = tl.where(cols < N, x - mean, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + else: + xbar = tl.where(cols < N, x, 0.0) + var = tl.sum(xbar * xbar, axis=0) / N + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + row, rstd) + # Normalize and apply linear transformation + mask = cols < N + w = tl.load(W + cols, mask=mask).to(tl.float32) + if HAS_BIAS: + b = tl.load(B + cols, mask=mask).to(tl.float32) + x_hat = (x - mean) * rstd if not IS_RMS_NORM else x * rstd + y = x_hat * w + b if HAS_BIAS else x_hat * w + if HAS_Z and NORM_BEFORE_GATE: + z = tl.load(Z + cols, mask=mask).to(tl.float32) + y *= z * tl.sigmoid(z) + # Write output + tl.store(Y + cols, y, mask=mask) + + +def _layer_norm_fwd( + x, + weight, + bias, + eps, + z=None, + out=None, + group_size=None, + norm_before_gate=True, + is_rms_norm=False, +): + M, N = x.shape + if group_size is None: + group_size = N + assert N % group_size == 0 + ngroups = N // group_size + assert x.stride(-1) == 1 + if z is not None: + assert z.stride(-1) == 1 + assert z.shape == (M, N) + assert weight.shape == (N,) + assert weight.stride(-1) == 1 + if bias is not None: + assert bias.stride(-1) == 1 + assert bias.shape == (N,) + # allocate output + if out is not None: + assert out.shape == x.shape + else: + out = torch.empty_like(x) + assert out.stride(-1) == 1 + mean = ( + torch.empty((ngroups * M,), dtype=torch.float32, device=x.device) + if not is_rms_norm + else None + ) + rstd = torch.empty((ngroups * M,), dtype=torch.float32, device=x.device) + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_N = min(MAX_FUSED_SIZE, triton.next_power_of_2(group_size)) + if group_size > BLOCK_N: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + # heuristics for number of warps + num_warps = min(max(BLOCK_N // 256, 1), 8) + grid = (M, ngroups) + with torch.cuda.device(x.device.index): + _layer_norm_fwd_1pass_kernel[grid]( + x, + out, + weight, + bias, + z, + mean, + rstd, + x.stride(0), + out.stride(0), + z.stride(0) if z is not None else 0, + M, + group_size, + eps, + BLOCK_N=BLOCK_N, + NORM_BEFORE_GATE=norm_before_gate, + IS_RMS_NORM=is_rms_norm, + num_warps=num_warps, + ) + return out, mean, rstd + + +class LayerNormFn(torch.autograd.Function): + + @staticmethod + def forward( + ctx, + x, + weight, + bias, + z=None, + eps=1e-6, + group_size=None, + norm_before_gate=True, + is_rms_norm=False, + ): + """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))""" + + x_shape_og = x.shape + # reshape input data into 2D tensor + x = x.reshape(-1, x.shape[-1]) + if x.stride(-1) != 1: + x = x.contiguous() + if z is not None: + assert z.shape == x_shape_og + z = z.reshape(-1, z.shape[-1]) + if z.stride(-1) != 1: + z = z.contiguous() + weight = weight.contiguous() + if bias is not None: + bias = bias.contiguous() + y, mean, rstd = _layer_norm_fwd( + x, + weight, + bias, + eps, + z=z, + group_size=group_size, + norm_before_gate=norm_before_gate, + is_rms_norm=is_rms_norm, + ) + return y.reshape(x_shape_og) + + +def layernorm_fn( + x, + weight, + bias, + z=None, + eps=1e-6, + group_size=None, + norm_before_gate=True, + is_rms_norm=False, +): + return LayerNormFn.apply( + x, weight, bias, z, eps, group_size, norm_before_gate, is_rms_norm + ) + + +def rmsnorm_fn( + x, weight, bias, z=None, eps=1e-6, group_size=None, norm_before_gate=True +): + return LayerNormFn.apply( + x, weight, bias, z, eps, group_size, norm_before_gate, True + ) + + +class LayerNorm(torch.nn.Module): + + def __init__( + self, + hidden_size, + eps=1e-5, + group_size=None, + norm_before_gate=True, + device=None, + dtype=None, + ): + """If group_size is not None, we do GroupNorm with each group having group_size elements. + group_size=None is equivalent to group_size=hidden_size (i.e. there's only 1 group). + """ + + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + self.weight = torch.nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + self.bias = torch.nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + self.group_size = group_size + self.norm_before_gate = norm_before_gate + self.reset_parameters() + + def reset_parameters(self): + torch.nn.init.ones_(self.weight) + torch.nn.init.zeros_(self.bias) + + def forward(self, x, z=None): + """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))""" + return layernorm_fn( + x, + self.weight, + self.bias, + z=z, + group_size=self.group_size, + eps=self.eps, + norm_before_gate=self.norm_before_gate, + ) + + +class RMSNorm(torch.nn.Module): + + def __init__( + self, + hidden_size, + eps=1e-5, + group_size=None, + norm_before_gate=True, + device=None, + dtype=None, + ): + """If group_size is not None, we do GroupNorm with each group having group_size elements. + group_size=None is equivalent to group_size=hidden_size (i.e. there's only 1 group). + """ + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + self.weight = torch.nn.Parameter(torch.empty(hidden_size, **factory_kwargs)) + self.register_parameter("bias", None) + self.group_size = group_size + self.norm_before_gate = norm_before_gate + self.reset_parameters() + + def reset_parameters(self): + torch.nn.init.ones_(self.weight) + + def forward(self, x, z=None): + """If z is not None, we do norm(x) * silu(z) if norm_before_gate, else norm(x * silu(z))""" + return rmsnorm_fn( + x, + self.weight, + self.bias, + z=z, + eps=self.eps, + group_size=self.group_size, + norm_before_gate=self.norm_before_gate, + ) diff --git a/python/sglang/srt/layers/attention/fla/op.py b/python/sglang/srt/layers/attention/fla/op.py new file mode 100644 index 000000000..9b3191075 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/op.py @@ -0,0 +1,66 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/op.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +import os + +import triton +import triton.language as tl +import triton.language.extra.libdevice as tldevice + +from sglang.srt.layers.attention.fla.utils import is_gather_supported + +if os.environ.get("FLA_USE_FAST_OPS", "0") == "1": + exp = tldevice.fast_expf + exp2 = tldevice.exp2 + log = tldevice.fast_logf + log2 = tldevice.fast_log2f +else: + exp = tl.exp + exp2 = tl.math.exp2 + log = tl.log + log2 = tl.log2 + + +@triton.jit +def safe_exp(x): + return exp(tl.where(x <= 0, x, float("-inf"))) + + +if not is_gather_supported: + + @triton.jit + def gather(src, index, axis, _builder=None): + """ + Gather operation that works when tl.gather is not supported. + This is a fallback implementation that returns None. + Just to make triton compiler happy. + """ + return None + +else: + gather = tl.gather + + +if hasattr(triton.language, "_experimental_make_tensor_descriptor"): + # For Triton 3.3.x + make_tensor_descriptor = triton.language._experimental_make_tensor_descriptor +elif hasattr(triton.language, "make_tensor_descriptor"): + # For Triton 3.4.x and later + make_tensor_descriptor = triton.language.make_tensor_descriptor +else: + """ + Fallback implementation when TMA is not supported. + Returns None to indicate TMA descriptors are unavailable. + Just make triton compiler happy. + """ + + @triton.jit + def make_tensor_descriptor( + base, + shape, + strides, + block_shape, + _builder=None, + ): + return None diff --git a/python/sglang/srt/layers/attention/fla/solve_tril.py b/python/sglang/srt/layers/attention/fla/solve_tril.py new file mode 100644 index 000000000..5c519507d --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/solve_tril.py @@ -0,0 +1,465 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/utils/solve_tril.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import prepare_chunk_indices +from sglang.srt.layers.attention.fla.utils import input_guard + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +# @triton.autotune( +# configs=[ +# triton.Config({}, num_warps=num_warps, num_stages=num_stages) +# for num_warps in [1, 2, 4, 8] +# for num_stages in [2, 3, 4, 5] +# ], +# key=["BT"], +# ) +@triton.jit(do_not_specialize=["T"]) +def solve_tril_16x16_kernel( + A, + Ad, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + A = A + (bos * H + i_h) * BT + Ad = Ad + (bos * H + i_h) * 16 + + offset = (i_t * 16) % BT + p_A = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * 16, offset), (16, 16), (1, 0) + ) + p_Ai = tl.make_block_ptr(Ad, (T, 16), (H * 16, 1), (i_t * 16, 0), (16, 16), (1, 0)) + b_A = tl.load(p_A, boundary_check=(0, 1)).to(tl.float32) + b_A = -tl.where(tl.arange(0, 16)[:, None] > tl.arange(0, 16)[None, :], b_A, 0) + + o_i = tl.arange(0, 16) + for i in range(1, min(16, T - i_t * 16)): + b_a = -tl.load(A + (i_t * 16 + i) * H * BT + o_i + offset) + b_a = b_a + tl.sum(b_a[:, None] * b_A, 0) + mask = o_i == i + b_A = tl.where(mask[:, None], b_a, b_A) + b_A += o_i[:, None] == o_i[None, :] + tl.store( + p_Ai, + b_A.to(p_Ai.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +# @triton.autotune( +# configs=[ +# triton.Config({}, num_warps=num_warps, num_stages=num_stages) +# for num_warps in [1, 2, 4, 8] +# for num_stages in [2, 3, 4, 5] +# ], +# key=["H", "BT", "IS_VARLEN"], +# ) +@triton.jit(do_not_specialize=["T"]) +def merge_16x16_to_32x32_inverse_kernel( + A, + Ad, + Ai, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + A += (bos * H + i_h) * 32 + Ad += (bos * H + i_h) * 16 + Ai += (bos * H + i_h) * 32 + + p_A_21 = tl.make_block_ptr( + A, (T, 32), (H * 32, 1), (i_t * 32 + 16, 0), (16, 16), (1, 0) + ) + p_Ad_11 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 32, 0), (16, 16), (1, 0) + ) + p_Ad_22 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 32 + 16, 0), (16, 16), (1, 0) + ) + p_Ai_11 = tl.make_block_ptr( + Ai, (T, 32), (H * 32, 1), (i_t * 32, 0), (16, 16), (1, 0) + ) + p_Ai_22 = tl.make_block_ptr( + Ai, (T, 32), (H * 32, 1), (i_t * 32 + 16, 16), (16, 16), (1, 0) + ) + p_Ai_21 = tl.make_block_ptr( + Ai, (T, 32), (H * 32, 1), (i_t * 32 + 16, 0), (16, 16), (1, 0) + ) + + A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32) + Ai_11 = tl.load(p_Ad_11, boundary_check=(0, 1)).to(tl.float32) + Ai_22 = tl.load(p_Ad_22, boundary_check=(0, 1)).to(tl.float32) + Ai_21 = -tl.dot( + tl.dot(Ai_22, A_21, input_precision="ieee"), Ai_11, input_precision="ieee" + ) + tl.store( + p_Ai_11, + Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_22, + Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_21, + Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +# @triton.autotune( +# configs=[ +# triton.Config({}, num_warps=num_warps, num_stages=num_stages) +# for num_warps in [2, 4, 8] +# for num_stages in [2, 3, 4, 5] +# ], +# key=["H", "BT", "IS_VARLEN"], +# ) +@triton.jit(do_not_specialize=["T"]) +def merge_16x16_to_64x64_inverse_kernel( + A, + Ad, + Ai, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + A += (bos * H + i_h) * 64 + Ad += (bos * H + i_h) * 16 + Ai += (bos * H + i_h) * 64 + + p_A_21 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 16, 0), (16, 16), (1, 0) + ) + p_A_32 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 32, 16), (16, 16), (1, 0) + ) + p_A_31 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 32, 0), (16, 16), (1, 0) + ) + p_A_43 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 48, 32), (16, 16), (1, 0) + ) + p_A_42 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 48, 16), (16, 16), (1, 0) + ) + p_A_41 = tl.make_block_ptr( + A, (T, 64), (H * 64, 1), (i_t * 64 + 48, 0), (16, 16), (1, 0) + ) + p_Ad_11 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 64, 0), (16, 16), (1, 0) + ) + p_Ad_22 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 64 + 16, 0), (16, 16), (1, 0) + ) + p_Ad_33 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 64 + 32, 0), (16, 16), (1, 0) + ) + p_Ad_44 = tl.make_block_ptr( + Ad, (T, 16), (H * 16, 1), (i_t * 64 + 48, 0), (16, 16), (1, 0) + ) + + A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32) + A_32 = tl.load(p_A_32, boundary_check=(0, 1)).to(tl.float32) + A_31 = tl.load(p_A_31, boundary_check=(0, 1)).to(tl.float32) + A_43 = tl.load(p_A_43, boundary_check=(0, 1)).to(tl.float32) + A_42 = tl.load(p_A_42, boundary_check=(0, 1)).to(tl.float32) + A_41 = tl.load(p_A_41, boundary_check=(0, 1)).to(tl.float32) + + Ai_11 = tl.load(p_Ad_11, boundary_check=(0, 1)).to(tl.float32) + Ai_22 = tl.load(p_Ad_22, boundary_check=(0, 1)).to(tl.float32) + Ai_33 = tl.load(p_Ad_33, boundary_check=(0, 1)).to(tl.float32) + Ai_44 = tl.load(p_Ad_44, boundary_check=(0, 1)).to(tl.float32) + + Ai_21 = -tl.dot( + tl.dot(Ai_22, A_21, input_precision="ieee"), Ai_11, input_precision="ieee" + ) + Ai_32 = -tl.dot( + tl.dot(Ai_33, A_32, input_precision="ieee"), Ai_22, input_precision="ieee" + ) + Ai_43 = -tl.dot( + tl.dot(Ai_44, A_43, input_precision="ieee"), Ai_33, input_precision="ieee" + ) + + Ai_31 = -tl.dot( + Ai_33, + tl.dot(A_31, Ai_11, input_precision="ieee") + + tl.dot(A_32, Ai_21, input_precision="ieee"), + input_precision="ieee", + ) + Ai_42 = -tl.dot( + Ai_44, + tl.dot(A_42, Ai_22, input_precision="ieee") + + tl.dot(A_43, Ai_32, input_precision="ieee"), + input_precision="ieee", + ) + Ai_41 = -tl.dot( + Ai_44, + tl.dot(A_41, Ai_11, input_precision="ieee") + + tl.dot(A_42, Ai_21, input_precision="ieee") + + tl.dot(A_43, Ai_31, input_precision="ieee"), + input_precision="ieee", + ) + + p_Ai_11 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64, 0), (16, 16), (1, 0) + ) + p_Ai_22 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 16, 16), (16, 16), (1, 0) + ) + p_Ai_33 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 32, 32), (16, 16), (1, 0) + ) + p_Ai_44 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 48, 48), (16, 16), (1, 0) + ) + p_Ai_21 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 16, 0), (16, 16), (1, 0) + ) + p_Ai_31 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 32, 0), (16, 16), (1, 0) + ) + p_Ai_32 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 32, 16), (16, 16), (1, 0) + ) + p_Ai_41 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 48, 0), (16, 16), (1, 0) + ) + p_Ai_42 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 48, 16), (16, 16), (1, 0) + ) + p_Ai_43 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 48, 32), (16, 16), (1, 0) + ) + tl.store( + p_Ai_11, + Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_22, + Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_33, + Ai_33.to(p_Ai_33.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_44, + Ai_44.to(p_Ai_44.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_21, + Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_31, + Ai_31.to(p_Ai_31.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_32, + Ai_32.to(p_Ai_32.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_41, + Ai_41.to(p_Ai_41.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_42, + Ai_42.to(p_Ai_42.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_43, + Ai_43.to(p_Ai_43.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + + fill_zeros = tl.zeros((16, 16), dtype=tl.float32) + p_Ai_12 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64, 16), (16, 16), (1, 0) + ) + p_Ai_13 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64, 32), (16, 16), (1, 0) + ) + p_Ai_14 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64, 48), (16, 16), (1, 0) + ) + p_Ai_23 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 16, 32), (16, 16), (1, 0) + ) + p_Ai_24 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 16, 48), (16, 16), (1, 0) + ) + p_Ai_34 = tl.make_block_ptr( + Ai, (T, 64), (H * 64, 1), (i_t * 64 + 32, 48), (16, 16), (1, 0) + ) + tl.store( + p_Ai_12, + fill_zeros.to(p_Ai_12.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_13, + fill_zeros.to(p_Ai_13.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_14, + fill_zeros.to(p_Ai_14.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_23, + fill_zeros.to(p_Ai_23.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_24, + fill_zeros.to(p_Ai_24.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_34, + fill_zeros.to(p_Ai_34.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + + +@input_guard +def solve_tril( + A: torch.Tensor, + cu_seqlens: Optional[torch.Tensor] = None, + output_dtype: torch.dtype = torch.float, +) -> torch.Tensor: + """ + Compute the inverse of the lower triangular matrix + A should be strictly lower triangular, i.e., A.triu() == 0. + + Args: + A (torch.Tensor): + [B, T, H, K] + cu_seqlens (torch.Tensor): + The cumulative sequence lengths of the input tensor. + Default: None. + output_dtype (torch.dtype): + The dtype of the output tensor. Default: `torch.float` + + Returns: + (I + A)^-1 with the same shape as A + """ + assert A.shape[-1] in [16, 32, 64] + + B, T, H, BT = A.shape + Ad = torch.empty( + B, T, H, 16, device=A.device, dtype=torch.float if BT != 16 else output_dtype + ) + + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, 16) if cu_seqlens is not None else None + ) + NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, 16) + solve_tril_16x16_kernel[NT, B * H]( + A=A, + Ad=Ad, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + BT=BT, + num_warps=1, + num_stages=4, + ) + if BT == 16: + return Ad + + Ai = torch.empty(B, T, H, BT, device=A.device, dtype=output_dtype) + merge_fn = ( + merge_16x16_to_32x32_inverse_kernel + if BT == 32 + else merge_16x16_to_64x64_inverse_kernel + ) + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT) + merge_fn[NT, B * H]( + A=A, + Ad=Ad, + Ai=Ai, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + BT=BT, + num_warps=4, + num_stages=3, + ) + return Ai diff --git a/python/sglang/srt/layers/attention/fla/utils.py b/python/sglang/srt/layers/attention/fla/utils.py new file mode 100644 index 000000000..3caf70de5 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/utils.py @@ -0,0 +1,331 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/utils.py +# -*- coding: utf-8 -*- + +import contextlib +import functools +import logging +import os +import sys +from enum import Enum +from functools import lru_cache +from typing import Any, Callable, Dict, Literal, Optional, Tuple + +import torch +import triton +from packaging import version + +logger = logging.getLogger(__name__) + +COMPILER_MODE = os.getenv("FLA_COMPILER_MODE") == "1" +FLA_CI_ENV = os.getenv("FLA_CI_ENV") == "1" + + +@lru_cache(maxsize=1) +def check_environments(): + """ + Checks the current operating system, Triton version, and Python version, + issuing warnings if they don't meet recommendations. + This function's body only runs once due to lru_cache. + """ + # Check Operating System + if sys.platform == "win32": + logger.warning( + "Detected Windows operating system. Triton does not have an official Windows release, " + "thus FLA will not be adapted for Windows, and any potential errors will not be fixed. " + "Please consider using a Linux environment for compatibility." + ) + + triton_version = version.parse(triton.__version__) + required_triton_version = version.parse("3.2.0") + + if triton_version < required_triton_version: + logger.warning( + f"Current Triton version {triton_version} is below the recommended 3.2.0 version. " + "Errors may occur and these issues will not be fixed. " + "Please consider upgrading Triton." + ) + + # Check Python version + py_version = version.parse(f"{sys.version_info.major}.{sys.version_info.minor}") + required_py_version = version.parse("3.11") + + if py_version < required_py_version: + logger.warning( + f"Current Python version {py_version} is below the recommended 3.11 version. " + "It is recommended to upgrade to Python 3.11 or higher for the best experience." + ) + + return None + + +check_environments() + + +def get_abs_err(x, y): + return (x.detach() - y.detach()).flatten().abs().max().item() + + +def get_err_ratio(x, y): + err = (x.detach() - y.detach()).flatten().square().mean().sqrt().item() + base = (x.detach()).flatten().square().mean().sqrt().item() + return err / (base + 1e-8) + + +def assert_close(prefix, ref, tri, ratio, warning=False, err_atol=1e-6): + abs_atol = get_abs_err(ref, tri) + msg = f"{prefix} diff: {abs_atol:.6f} ratio: {get_err_ratio(ref, tri):.6f}" + logger.info(msg) + error_rate = get_err_ratio(ref, tri) + if abs_atol <= err_atol: + return + if warning or (FLA_CI_ENV and (error_rate < 0.01 or abs_atol <= 0.3)): + if error_rate > ratio: + import warnings + + warnings.warn(msg) + else: + assert error_rate < ratio, msg + + +SUPPRESS_LEVEL = int(os.getenv("GDN_RECOMPUTE_SUPPRESS_LEVEL", "0")) + + +def tensor_cache(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: + """ + A decorator that caches the most recent results of a function with tensor inputs. + This decorator will store the output of the decorated function for the most recent set of input tensors. + The cache is limited to a fixed size (default is 4). When the cache is full, the oldest entry will be removed. + Args: + fn (Callable[..., torch.Tensor]): + The function to be decorated. It should take tensor inputs and return tensor outputs. + Returns: + Callable[..., torch.Tensor]: + A wrapped version of the input function with single-entry caching. + """ + + cache_entries: Tuple[Optional[Tuple], Optional[Dict], Any] = [] + cache_size = 4 + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + nonlocal cache_entries, cache_size + for i, entry in enumerate(cache_entries): + last_args, last_kwargs, last_result = entry + if len(args) == len(last_args) and len(kwargs) == len(last_kwargs): + if all(a is b for a, b in zip(args, last_args)) and all( + k in last_kwargs and v is last_kwargs[k] for k, v in kwargs.items() + ): + cache_entries = ( + cache_entries[:i] + + cache_entries[i + 1 :] + + [(args, kwargs, last_result)] + ) + return last_result + + result = fn(*args, **kwargs) + + if len(cache_entries) >= cache_size: + cache_entries = cache_entries[1:] + cache_entries.append((args, kwargs, result)) + return result + + return wrapper + + +def input_guard(fn: Callable[..., torch.Tensor]) -> Callable[..., torch.Tensor]: + """ + A decorator to make sure all input tensors are contiguous and set the device based on input tensors. + """ + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + contiguous_args = ( + i if not isinstance(i, torch.Tensor) else i.contiguous() for i in args + ) + contiguous_kwargs = { + k: (v if not isinstance(v, torch.Tensor) else v.contiguous()) + for k, v in kwargs.items() + } + + tensor = None + for arg in args: + if isinstance(arg, torch.Tensor): + tensor = arg + break + if tensor is None: + for value in kwargs.values(): + if isinstance(value, torch.Tensor): + tensor = value + break + + if tensor is not None: + ctx = custom_device_ctx(tensor.device.index) + else: + ctx = contextlib.nullcontext() + + with ctx: + return fn(*contiguous_args, **contiguous_kwargs) + + return wrapper + + +contiguous = input_guard + + +def require_version(version, hint): + """ + Perform a runtime check of the dependency versions, using the exact same syntax used by pip. + """ + + def decorator(fn): + @functools.wraps(fn) + def wrapper(ctx, *args, **kwargs): + from transformers.utils.versions import require_version + + require_version(version, hint) + return fn( + ctx, + *( + i if not isinstance(i, torch.Tensor) else i.contiguous() + for i in args + ), + **{ + k: (v if not isinstance(v, torch.Tensor) else v.contiguous()) + for k, v in kwargs.items() + }, + ) + + return wrapper + + return decorator + + +def checkpoint(fn): + def wrapper(*args, **kwargs): + return torch.utils.checkpoint.checkpoint(fn, *args, **kwargs) + + return wrapper + + +@lru_cache(maxsize=None) +def check_pytorch_version(version_s: str = "2.4") -> bool: + return version.parse(torch.__version__) >= version.parse(version_s) + + +def _cpu_device_warning(): + import warnings + + warnings.warn( + ("Triton is not supported on current platform, roll back to CPU."), stacklevel=1 + ) + + +@lru_cache(maxsize=None) +def get_multiprocessor_count(tensor_idx: int = 0) -> int: + try: + return triton.runtime.driver.active.utils.get_device_properties(tensor_idx)[ + "multiprocessor_count" + ] + except BaseException: + _cpu_device_warning() + return -1 + + +@lru_cache(maxsize=None) +def get_available_device() -> str: + try: + return triton.runtime.driver.active.get_current_target().backend + except BaseException: + _cpu_device_warning() + return "cpu" + + +@lru_cache(maxsize=None) +def _check_platform() -> Literal["nvidia", "amd", "intel", "musa"]: + device = get_available_device() + if device == "cuda": + return "nvidia" + elif device == "hip": + return "amd" + elif device == "xpu": + return "intel" + else: + return device + + +# For AMD GPUs, the triton backend is 'hip', while for Nvidia GPUs, the triton backend is 'cuda'. +# However, the torch backend is 'cuda' for both Nvidia and AMD GPUs. +# Therefore, we need to check the triton backend to determine the actual GPU vendor. +device = get_available_device() if get_available_device() != "hip" else "cuda" +device_torch_lib = getattr(torch, device) +device_platform = _check_platform() + +is_amd = device_platform == "amd" +is_intel = device_platform == "intel" +is_nvidia = device_platform == "nvidia" +is_intel_alchemist = is_intel and "Intel(R) Arc(TM) A" in torch.xpu.get_device_name(0) +is_nvidia_hopper = is_nvidia and ( + "NVIDIA H" in torch.cuda.get_device_name(0) + or torch.cuda.get_device_capability()[0] >= 9 +) +use_cuda_graph = is_nvidia and os.environ.get("FLA_USE_CUDA_GRAPH", "0") == "1" + +# Nvidia Ampere or newer, haven't check AMD and intel yet. +is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8 +is_gather_supported = hasattr(triton.language, "gather") + + +def get_all_max_shared_mem(): + try: + return [ + triton.runtime.driver.active.utils.get_device_properties(i)[ + "max_shared_mem" + ] + for i in range(device_torch_lib.device_count()) + ] + except BaseException: + _cpu_device_warning() + return [-1] + + +class Backend(Enum): + ADA = 101376 # RTX 4090 + AMPERE = 166912 # A100 + HOPPER = 232448 # H100 + DEFAULT = 102400 # Default + + @classmethod + def get_shared_memory(cls, arch: str) -> int: + try: + return cls[arch.upper()].value + except KeyError: + return cls.DEFAULT.value + + +@lru_cache(maxsize=None) +def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool: + try: + device_shared_mem_list = get_all_max_shared_mem() + max_shared_memory = device_shared_mem_list[tensor_idx] + return max_shared_memory >= Backend.get_shared_memory(arch) + except Exception: + return False + + +if check_pytorch_version("2.4"): + device = "cuda" if device == "cpu" else device + autocast_custom_fwd = functools.partial(torch.amp.custom_fwd, device_type=device) + autocast_custom_bwd = functools.partial(torch.amp.custom_bwd, device_type=device) + + def custom_device_ctx(index: int): + return device_torch_lib.device(index) + +else: + assert ( + device == "cuda" + ), "Only cuda device is supported for PyTorch version < 2.4.0." + autocast_custom_fwd = device_torch_lib.amp.custom_fwd + autocast_custom_bwd = device_torch_lib.amp.custom_bwd + + def custom_device_ctx(index: int): + return torch.cuda.device(index) diff --git a/python/sglang/srt/layers/attention/fla/wy_fast.py b/python/sglang/srt/layers/attention/fla/wy_fast.py new file mode 100644 index 000000000..d51500eb4 --- /dev/null +++ b/python/sglang/srt/layers/attention/fla/wy_fast.py @@ -0,0 +1,158 @@ +# Adapt from https://github.com/fla-org/flash-linear-attention/blob/main/fla/ops/gated_delta_rule/wy_fast.py +# -*- coding: utf-8 -*- +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang + +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.fla.index import prepare_chunk_indices +from sglang.srt.layers.attention.fla.op import safe_exp +from sglang.srt.layers.attention.fla.utils import check_shared_mem + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +# @triton.autotune( +# configs=[ +# triton.Config({}, num_warps=num_warps, num_stages=num_stages) +# for num_warps in [2, 4, 8] +# for num_stages in [2, 3, 4] +# ], +# key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"], +# ) +@triton.jit(do_not_specialize=["T"]) +def recompute_w_u_fwd_kernel( + k, + v, + beta, + w, + u, + A, + g, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + Hg: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = tl.load(chunk_indices + i_t * 2).to(tl.int32), tl.load( + chunk_indices + i_t * 2 + 1 + ).to(tl.int32) + bos, eos = tl.load(cu_seqlens + i_n).to(tl.int32), tl.load( + cu_seqlens + i_n + 1 + ).to(tl.int32) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + p_beta = tl.make_block_ptr( + beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + ) + p_g = tl.make_block_ptr(g + (bos * H + i_h), (T,), (H,), (i_t * BT,), (BT,), (0,)) + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + b_beta = tl.load(p_beta, boundary_check=(0,)) + b_A = tl.load(p_A, boundary_check=(0, 1)) + b_g = tl.exp(tl.load(p_g, boundary_check=(0,))) + + for i_v in range(tl.cdiv(V, BV)): + p_v = tl.make_block_ptr( + v + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_u = tl.make_block_ptr( + u + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) + b_vb = (b_v * b_beta[:, None]).to(b_v.dtype) + b_u = tl.dot(b_A, b_vb, allow_tf32=False) + tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) + + for i_k in range(tl.cdiv(K, BK)): + p_k = tl.make_block_ptr( + k + (bos * Hg + i_h // (H // Hg)) * K, + (T, K), + (Hg * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_w = tl.make_block_ptr( + w + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_kb = (b_k * b_beta[:, None] * b_g[:, None]).to(b_k.dtype) + b_w = tl.dot(b_A, b_kb) + tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) + + +def recompute_w_u_fwd( + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + g_cumsum: torch.Tensor, + A: torch.Tensor, + cu_seqlens: Optional[torch.LongTensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + B, T, Hg, K, V = *k.shape, v.shape[-1] + H = v.shape[-2] + BT = A.shape[-1] + + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, BT) if cu_seqlens is not None else None + ) + NT = triton.cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + BK = 64 + BV = 64 + u = torch.empty_like(v) + w = k.new_empty(B, T, H, K) + recompute_w_u_fwd_kernel[(NT, B * H)]( + k=k, + v=v, + beta=beta, + w=w, + u=u, + A=A, + g=g_cumsum, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + Hg=Hg, + K=K, + V=V, + BT=BT, + BK=BK, + BV=BV, + num_warps=4, + num_stages=3, + ) + return w, u + + +fwd_recompute_w_u = recompute_w_u_fwd diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py new file mode 100644 index 000000000..f7ca5e203 --- /dev/null +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -0,0 +1,2365 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import numpy as np +import torch +import triton +import triton.language as tl + +from sglang.srt.configs.model_config import AttentionArch +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.mem_cache.memory_pool import SWAKVPool +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + +from sgl_kernel import merge_state_v2 +from sgl_kernel.flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache + + +@dataclass +class FlashAttentionMetadata: + """Metadata to be init once in the model forward pass, + each layer's forward pass can reuse the metadata. + + For each init metadata function, we will try set up them in below order + """ + + # Sequence lengths for the forward batch + cache_seqlens_int32: torch.Tensor = None + # Maximum sequence length for query + max_seq_len_q: int = 1 + # Maximum sequence length for key + max_seq_len_k: int = 0 + # Cumulative sequence lengths for query + cu_seqlens_q: torch.Tensor = None + # Cumulative sequence lengths for key + cu_seqlens_k: torch.Tensor = None + # Window size (typically used by Gemma) + window_size: tuple = (-1, -1) + # Page table, the index of KV Cache Tables/Blocks + page_table: torch.Tensor = None + + # Encoder metadata + # Cumulative sequence lengths for encoder key + encoder_cu_seqlens_k: torch.Tensor = None + # Maximum sequence length for encoder key + encoder_max_seq_len_k: int = 0 + # Sequence lengths for the forward batch + encoder_lens_int32: torch.Tensor = None + # Page table for the encoder + encoder_page_table: torch.Tensor = None + + @dataclass + class LocalAttentionMetadata: + local_query_start_loc: torch.Tensor = None # cu_seqlens_q for local attention + local_seqused_k: torch.Tensor = None # sequence lengths for local attention + local_block_table: torch.Tensor = None # block table for local attention + local_max_query_len: int = 0 # max query length for local attention + local_max_seq_len: int = 0 # max sequence length for local attention + + local_attn_metadata: Optional[LocalAttentionMetadata] = None + + # For sliding window attention topk>1 spec decoding + swa_spec_metadata: Optional[FlashAttentionMetadata] = None + + +# Copied from: +# https://github.com/houseroad/vllm/blob/4e45bfcaf928bdb9bd952b4ac922a3c205589ae8/vllm/v1/attention/backends/flash_attn.py +# +# Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into +# local attention blocks, where each block is passed to the attention kernel +# as an independent local ("virtual") batch item. +# +# For example, if are performing a chunked prefill a batch of 3 sequences: +# q_seqlens = [4, 10, 5] +# kv_seqlens = [6, 17, 9] +# Then normally for regular attention we would compute with an attention mask +# for batch idx 0 (q_seqlens = 4, kv_seqlens = 6) like: +# batch idx: 0 (q_seqlens = 4, kv_seqlens = 6) +# k_toks > 0 1 2 3 4 5 +# q_toks v _____________ +# 0 | 1 1 1 +# 1 | 1 1 1 1 +# 2 | 1 1 1 1 1 +# 3 | 1 1 1 1 1 1 +# +# for local attention (with attn_chunk_size = 4) we would compute with an +# attention mask like: +# batch idx: 0 (q_seqlens = 4, kv_seqlens = 6, attn_chunk_size = 4) +# k_toks > 0 1 2 3 4 5 +# q_toks v _____________ +# 0 | 1 1 1 +# 1 | 1 1 1 1 +# 2 | 1 +# 3 | 1 1 +# +# We can simulate this mask using standard flash-attention by breaking the +# sequences into local ("virtual") batches, where each local batch item is a +# local attention block, so in this case batch idx 0 would be broken up into: +# +# local-batch idx: 0 (q_seqlens = 2, kv_seqlens = 4) (batch 0) +# k_toks > 0 1 2 3 +# q_toks v _____________ +# 0 | 1 1 1 +# 1 | 1 1 1 1 +# local-batch idx: 1 (q_seqlens = 2, kv_seqlens = 2) (batch 0) +# k_toks > 4 5 +# q_toks v _____________ +# 2 | 1 +# 3 | 1 1 +# +# e.g. if we have: +# attn_chunk_size = 4 +# query_start_loc_np = [0, 4, 14, 19] (q_seqlens = [4, 10, 5]) +# Then this function would return: +# __b0__ ______b1______ __b2__ < orig batch indices +# q_seqlens_local = [ 2, 2, 1, 4, 4, 1, 4, 1] +# cu_seqlens_q_local = [0, 4, 6, 10, 14, 18, 19, 23, 24] +# seqlens_k_local = [ 4, 2, 4, 4, 4, 1, 4, 1] +# block_table_local : shape[local_virtual_batches, pages_per_local_batch] +def make_local_attention_virtual_batches( + attn_chunk_size: int, + query_start_loc_np: np.ndarray, + seq_lens_np: np.ndarray, + block_table: torch.Tensor, + page_size: int = 0, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, torch.Tensor]: + """ + Take in `query_start_loc_np` and `seq_lens_np` and break the sequences into + local attention blocks, where each block is passed to the attention kernel + as an independent local ("virtual") batch item. + + Args: + attn_chunk_size: Size of local attention chunks + query_start_loc_np: Cumulative sum of query lengths (numpy array) + seq_lens_np: Sequence lengths (numpy array) + block_table: Block table for KV cache + page_size: Size of each page in the KV cache + + Returns: + seqlens_q_local: Query sequence lengths for local attention + cu_seqlens_q_local: Cumulative sum of query sequence lengths for local attention + seqlens_k_local: Key sequence lengths for local attention + block_table_local: Block table for local attention + """ + # Adjust attention_chunk_size based on the actual sequence length + # to avoid index out of bounds errors + max_seq_len = seq_lens_np.max() + effective_chunk_size = min(attn_chunk_size, max_seq_len) + # Make sure effective_chunk_size is divisible by page_size + effective_chunk_size = (effective_chunk_size // page_size) * page_size + if effective_chunk_size < page_size: + effective_chunk_size = page_size + attn_chunk_size = effective_chunk_size + + q_seqlens = query_start_loc_np[1:] - query_start_loc_np[:-1] + actual_batch_size = seq_lens_np.shape[0] + + # Handle if we are starting in the middle of a local attention block, + # we assume q_seqlens > 0 (for all elements), for each batch idx we compute + # the number of tokens that are not in the first local attention block and + # then we can simply use a cdiv for the rest. + # For example if we have: + # attn_chunk_size = 4 + # q_seqlens = [4, 10, 5] + # k_seqlens = [6, 17, 9] + # Then we would get: + # new_tokens_in_first_block = [2, 1, 4] + # local_blocks = [2, 4, 2] + q_tokens_in_first_block = np.minimum( + attn_chunk_size - ((seq_lens_np - q_seqlens) % attn_chunk_size), q_seqlens + ).astype(np.int32) + tokens_in_last_block = attn_chunk_size + (seq_lens_np % -attn_chunk_size) + local_blocks = 1 + cdiv(q_seqlens - q_tokens_in_first_block, attn_chunk_size) + + # Once we know the number of local blocks we can compute the request spans + # for each batch idx, we can figure out the number of "virtual" requests we + # have to make, + # For the above example we would get: + # seqlens_q_local = [2, 2, 1, 4, 4, 1, 4, 1] + # + # First Get batched arange. (E.g., [2, 4, 2] -> [0, 1, 0, 1, 2, 3, 0, 1]) + # (TODO: max a utility to share this code with _prepare_inputs) + # arange step 1. [2, 4, 2] -> [2, 6, 8] + cu_num_blocks = np.cumsum(local_blocks) + virtual_batches = cu_num_blocks[-1] + # arange step 2. [2, 6, 8] -> [0, 0, 2, 2, 2, 2, 6, 6] + block_offsets = np.repeat(cu_num_blocks - local_blocks, local_blocks) + # arange step 3. [0, 1, 0, 1, 2, 3, 0, 1] + arange = np.arange(virtual_batches, dtype=np.int32) - block_offsets + # also compute reverse arange (i.e. [1, 0, 3, 2, 1, 0, 1, 0]) + rarange = np.repeat(local_blocks, local_blocks) - arange - 1 + # Then we can compute the seqlens_q_local, handling the fact that the + # first and last blocks could be partial + seqlens_q_local = np.repeat(q_seqlens - q_tokens_in_first_block, local_blocks) + # set the first block since this may be a partial block + seqlens_q_local[arange == 0] = q_tokens_in_first_block + # set the remaining blocks + seqlens_q_local[arange > 0] = np.minimum( + seqlens_q_local - attn_chunk_size * (arange - 1), attn_chunk_size + )[arange > 0] + + # convert from q_seqlens to cu_seqlens_q + cu_seqlens_q_local = np.pad(np.cumsum(seqlens_q_local), (1, 0)).astype(np.int32) + + # compute the seqlens_k_local, + # basically a full local attention block for all but the last block in each + # batch + # For our example this will be: + # seqlens_k_local = [4, 2, 4, 4, 4, 1, 4, 1] + seqlens_k_local = np.full(cu_num_blocks[-1], attn_chunk_size, dtype=np.int32) + seqlens_k_local[cu_num_blocks - 1] = tokens_in_last_block + + k_seqstarts_absolute = np.repeat(seq_lens_np, local_blocks) - ( + rarange * attn_chunk_size + np.repeat(tokens_in_last_block, local_blocks) + ) + # For the example the local attention blocks start at: + # _b0_ _____b1_____ _b2_ + # k_seqstarts_absolute = [0, 4, 4, 8, 12, 16, 4, 8] + block_starts = k_seqstarts_absolute // page_size + + assert attn_chunk_size % page_size == 0, ( + f"attn_chunk_size {attn_chunk_size} is not " + f"divisible by page_size {page_size}" + ) + pages_per_local_batch = attn_chunk_size // page_size + + # Create a block_table for the local attention blocks + # For out example if we have a block-table like (assuming page_size=2): + # block_table = [ + # [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], < batch 0 + # [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], < batch 1 + # [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], < batch 2 + # ] + # Then for the local batches we would want a block-table like + # block_table_local = [ + # [ 0, 1 ], < local-batch 0, (batch 0, starting from k[0]) + # [ 2, 3 ], < local-batch 1, (batch 0, starting from k[4]) + # [ 12, 13 ], < local-batch 2, (batch 1, starting from k[4]) + # [ 14, 15 ], < local-batch 3, (batch 1, starting from k[8]) + # [ 16, 17 ], < local-batch 4, (batch 1, starting from k[12]) + # [ 18, 19 ], < local-batch 5, (batch 1, starting from k[16]) + # [ 22, 23 ], < local-batch 6, (batch 2, starting from k[4]) + # [ 24, 25 ], < local-batch 7, (batch 2, starting from k[8]) + # ] + block_indices = np.broadcast_to( + np.arange(pages_per_local_batch, dtype=np.int32), + (virtual_batches, pages_per_local_batch), + ) + np.expand_dims(block_starts, axis=1) + # Ensure block_indices doesn't exceed block_table dimensions + # This is a critical safety check that prevents index out of bounds errors + # when dealing with large sequences (>8192 tokens) or when the block_table + # dimensions are smaller than what would be needed for the full attention chunk size. + block_indices = block_indices.flatten().clip(max=block_table.shape[1] - 1) + batch_indices = np.repeat( + np.arange(actual_batch_size, dtype=np.int32), + local_blocks * pages_per_local_batch, + ) + block_table_local = block_table[batch_indices, block_indices].view( + virtual_batches, -1 + ) + + return seqlens_q_local, cu_seqlens_q_local, seqlens_k_local, block_table_local + + +def cdiv(a: int, b: int) -> int: + """Ceiling division.""" + return -(a // -b) + + +# TODO(hebiao064): remove this once we have a better way to handle the merge_state_v2 torch.compile issue +@torch._dynamo.disable() +def merge_state_v2_wrapper(o, s_a, o_exp, s_b): + return merge_state_v2(o, s_a, o_exp, s_b) + + +class FlashAttentionBackend(AttentionBackend): + """FlashAttention backend implementation. + + Note about the init: + - If no spec decoding + - FlashAttentionBackend will be init once when the server starts. + - If spec decoding + - FlashAttentionBackend will be init once for the target worker + - FlashAttentionMultiStepBackend will be once for the draft worker + - It will spawn num_steps FlashAttentionBackend for the draft worker + + Note about CUDA Graph: + - We only support CUDA Graph for Decode (Normal Decode and Draft Decode) and Target Verify. + - We don't support CUDA Graph for Extend and Draft Extend. + - When server init, init_cuda_graph_state will be called first and then init_cuda_graph_capture will be called. + - For each forward batch, init_replay_cuda_graph will be called first and then replay the graph. + """ + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + speculative_step_id=0, + topk=0, + speculative_num_steps=0, + ): + super().__init__() + + assert not ( + model_runner.sliding_window_size is not None + and model_runner.model_config.is_encoder_decoder + ), "Sliding window and cross attention are not supported together" + + self.forward_metadata: FlashAttentionMetadata = None + # extra metadata for handling speculative decoding topk > 1, extended draft decode and verify + self.forward_metadata_spec_decode_expand: FlashAttentionMetadata = None + self.max_context_len = model_runner.model_config.context_len + self.device = model_runner.device + self.decode_cuda_graph_metadata = {} + self.target_verify_metadata = {} + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.kv_cache_dtype = model_runner.kv_cache_dtype + self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype + self.page_size = model_runner.page_size + self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA + self.skip_prefill = skip_prefill + self.is_hybrid = model_runner.is_hybrid + if self.is_hybrid: + self.full_to_swa_index_mapping = ( + model_runner.token_to_kv_pool.full_to_swa_index_mapping + ) + self.topk = model_runner.server_args.speculative_eagle_topk or 0 + self.speculative_num_steps = speculative_num_steps + self.speculative_num_draft_tokens = ( + model_runner.server_args.speculative_num_draft_tokens + ) + self.speculative_step_id = speculative_step_id + + # Local attention settings + self.attention_chunk_size = ( + model_runner.attention_chunk_size + if hasattr(model_runner, "attention_chunk_size") + else None + ) + + # For each layer, the sliding_window_size can be different. This is only used for preparing SWA metadata. + # We use `layer.sliding_window_size` to decide whether to use SWA for each layer. + self.sliding_window_size = model_runner.sliding_window_size + self.has_swa = ( + self.sliding_window_size is not None and self.sliding_window_size > -1 + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Initialize forward metadata hence all layers in the forward pass can reuse it.""" + metadata = FlashAttentionMetadata() + seqlens_in_batch = forward_batch.seq_lens + batch_size = forward_batch.batch_size + device = seqlens_in_batch.device + + if forward_batch.forward_mode.is_decode_or_idle(): + # Draft Decode + if forward_batch.spec_info is not None: + if self.topk <= 1: + metadata.cache_seqlens_int32 = ( + seqlens_in_batch + (self.speculative_step_id + 1) + ).to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + ( + self.speculative_step_id + 1 + ) + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + else: + metadata.cache_seqlens_int32 = (seqlens_in_batch).to(torch.int32) + metadata.max_seq_len_q = self.topk + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_q = torch.arange( + 0, + batch_size * self.topk + 1, + step=self.topk, + dtype=torch.int32, + device=device, + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + metadata_expand = FlashAttentionMetadata() + decode_length = self.speculative_step_id + 1 + metadata_expand.cache_seqlens_int32 = torch.full( + (seqlens_in_batch.numel() * self.topk,), + decode_length, + device=device, + dtype=torch.int32, + ) + metadata_expand.max_seq_len_q = 1 + metadata_expand.cu_seqlens_q = torch.arange( + 0, + metadata_expand.cache_seqlens_int32.numel() + 1, + dtype=torch.int32, + device=device, + ) + metadata_expand.cu_seqlens_k = torch.arange( + 0, + metadata_expand.cache_seqlens_int32.numel() * decode_length + 1, + step=decode_length, + dtype=torch.int32, + device=device, + ) + # shape: [bs, num_steps, topk] -> [bs x topk, num_steps] + cache_loc = forward_batch.out_cache_loc.view( + -1, self.speculative_num_steps + ) + metadata_expand.page_table = ( + cache_loc[:, :decode_length].contiguous().to(torch.int32) + ) + self.forward_metadata_spec_decode_expand = metadata_expand + else: + # Normal Decode + metadata.cache_seqlens_int32 = seqlens_in_batch.to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + # TODO: we need to test this part for llama 4 eagle case + self._init_local_attn_metadata(forward_batch, metadata, device) + elif forward_batch.forward_mode.is_target_verify(): + if self.topk <= 1: + metadata.cache_seqlens_int32 = ( + forward_batch.seq_lens + self.speculative_num_draft_tokens + ).to(torch.int32) + metadata.max_seq_len_q = self.speculative_num_draft_tokens + metadata.max_seq_len_k = ( + forward_batch.seq_lens_cpu.max().item() + + self.speculative_num_draft_tokens + ) + metadata.cu_seqlens_q = torch.arange( + 0, + batch_size * self.speculative_num_draft_tokens + 1, + self.speculative_num_draft_tokens, + dtype=torch.int32, + device=device, + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + self._init_local_attn_metadata(forward_batch, metadata, device) + else: + metadata.cache_seqlens_int32 = forward_batch.seq_lens.to(torch.int32) + metadata.max_seq_len_q = self.speculative_num_draft_tokens + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_q = torch.arange( + 0, + batch_size * self.speculative_num_draft_tokens + 1, + step=self.speculative_num_draft_tokens, + dtype=torch.int32, + device=device, + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + metadata_expand = FlashAttentionMetadata() + + metadata_expand.max_seq_len_q = 1 + metadata_expand.cu_seqlens_q = torch.arange( + 0, + forward_batch.seq_lens.numel() * self.speculative_num_draft_tokens + + 1, + dtype=torch.int32, + device=device, + ) + + # create expand page table + offsets = torch.arange( + self.speculative_num_draft_tokens, device=device + ).unsqueeze( + 0 + ) # shape: (1, self.speculative_num_draft_tokens) + cols = offsets.expand( + forward_batch.seq_lens.numel(), -1 + ) + forward_batch.seq_lens.unsqueeze(1) + cum_len = torch.nn.functional.pad( + torch.cumsum( + ( + forward_batch.seq_lens + self.speculative_num_draft_tokens + ).repeat_interleave(self.speculative_num_draft_tokens), + dim=0, + ), + (1, 0), + )[:-1] + mask_extraction_indices = ( + cols.repeat_interleave(self.speculative_num_draft_tokens, dim=0) + + cum_len[:, None] + ).view(1, -1) + mask = forward_batch.spec_info.custom_mask[ + mask_extraction_indices + ].view( + -1, self.speculative_num_draft_tokens + ) # (bsz * draft_num, draft_num) + + # shift table indices to avoid padding + # non_masked_page_table [[8, 9, 10], mask (display with int format) [[1, 0, 0], + # [8, 9, 10], [1, 1, 0], + # [8, 9, 10]] [1, 0, 1]] + # if masked with padding [[8, 0, 0], our mask without padding [[8, 9, 10], + # [8, 9, 0], [8, 9, 10], + # [8, 0, 10]] [8, 10, 9]] + # note here cache_seqlens_int32 is [1, 2, 2] so extra page indices will be ignored in each row + col_indices = offsets.expand( + mask.shape[0], self.speculative_num_draft_tokens + ) + # Build keys: if an entry is valid (mask==True), keep its original index; + # if not, add self.speculative_num_draft_tokens so that it sorts after all valid entries. + keys = torch.where( + mask, col_indices, col_indices + self.speculative_num_draft_tokens + ) + _, sort_order = torch.sort(keys, dim=1) + non_masked_page_table = ( + forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : + ] + .gather(1, cols) + .repeat_interleave(self.speculative_num_draft_tokens, dim=0) + ) # (bsz, draft_num) + metadata_expand.page_table = non_masked_page_table.gather(1, sort_order) + metadata_expand.cache_seqlens_int32 = mask.sum(dim=1).to(torch.int32) + metadata_expand.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata_expand.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + self.forward_metadata_spec_decode_expand = metadata_expand + + if self.has_swa: + self._init_sliding_window_attn_spec_metadata( + metadata, metadata_expand + ) + + elif forward_batch.forward_mode.is_extend_or_draft_extend_or_mixed(): + metadata.cache_seqlens_int32 = seqlens_in_batch.to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + if ( + any(forward_batch.extend_prefix_lens_cpu) + or forward_batch.forward_mode == ForwardMode.DRAFT_EXTEND + ): + extend_seq_lens = forward_batch.extend_seq_lens + metadata.max_seq_len_q = max(forward_batch.extend_seq_lens_cpu) + metadata.cu_seqlens_q = torch.nn.functional.pad( + torch.cumsum(extend_seq_lens, dim=0, dtype=torch.int32), (1, 0) + ) + else: + metadata.max_seq_len_q = metadata.max_seq_len_k + metadata.cu_seqlens_q = metadata.cu_seqlens_k + + # Setup local attention if enabled + if forward_batch.forward_mode == ForwardMode.EXTEND: + self._init_local_attn_metadata(forward_batch, metadata, device) + + # Encoder metadata for cross attention + if forward_batch.encoder_lens is not None: + assert ( + forward_batch.encoder_lens.numel() == 1 + ), "Only encoder size 1 is supported for now" + + metadata.encoder_lens_int32 = forward_batch.encoder_lens.to(torch.int32) + metadata.encoder_cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(metadata.encoder_lens_int32, dim=0, dtype=torch.int32), + (1, 0), + ) + metadata.encoder_max_seq_len_k = metadata.encoder_lens_int32.max().item() + metadata.encoder_page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.encoder_max_seq_len_k + ] + + # Currently only support forward_batch.encoder_lens.numel() == 1 + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, + metadata.encoder_max_seq_len_k : ( + metadata.encoder_max_seq_len_k + metadata.max_seq_len_k + ), + ] + + # Convert the page table to a strided format which is needed by FA3 API + if self.page_size > 1: + self.strided_indices = torch.arange( + 0, metadata.page_table.shape[1], self.page_size, device=self.device + ) + metadata.page_table = ( + metadata.page_table[:, self.strided_indices] // self.page_size + ) + + self.forward_metadata = metadata + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + # For multi-head latent attention + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + sinks: Optional[torch.Tensor] = None, + ): + if k is not None: + assert v is not None + if save_kv_cache: + cache_loc = ( + forward_batch.out_cache_loc + if not layer.is_cross_attention + else forward_batch.encoder_out_cache_loc + ) + if not self.use_mla: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + else: + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, + cache_loc, + k, + k_rope, + ) + + # Use precomputed metadata across all layers + metadata = self.forward_metadata + + # Calculate window size (can be moved to metadata if layer properties don't change) + # we don't do layer.sliding_window_size - 1 since in model.get_attention_sliding_window_size() we already - 1 + # here is two side inclusive + is_swa = ( + layer.sliding_window_size is not None and layer.sliding_window_size > -1 + ) + window_size = (layer.sliding_window_size, 0) if is_swa else (-1, -1) + k_descale, v_descale = None, None + # only use kv scaling if: 1) fp8 kv is explicitly enabled, 2) RadixAttention + # has corresponding quantization method so that layer.k_scale is not None, + # 3) layer.head_dim <= 256 since fa3 kernel require fp16 and bf16 data type in this case. + if self.kv_cache_dtype_str != "auto" and layer.head_dim <= 256: + if layer.k_scale is not None: + descale_shape = (forward_batch.batch_size, layer.tp_k_head_num) + k_descale = layer.k_scale.expand(descale_shape) + v_descale = layer.v_scale.expand(descale_shape) + q = q.to(self.kv_cache_dtype) + q_rope = q_rope.to(self.kv_cache_dtype) if q_rope is not None else None + k_rope = k_rope.to(self.kv_cache_dtype) if k_rope is not None else None + causal = not layer.is_cross_attention + + # Check if we should use local attention + use_local_attn = ( + self.attention_chunk_size is not None + and metadata.local_attn_metadata is not None + and (hasattr(layer, "use_irope") and layer.use_irope) + ) + + # We do cascade attention for Target Verify with topk > 1 + # We don't use cascade attention for Sliding Window Attention: + # - Different window sizes should be passed in for each q in the first stage of cascade attention, but FA3 interface doesn't support pass in a list of window sizes. + # - The overhead of duplicated computation of the common prefix part is small for sliding window layers (seq_len <= window_size), so we can just expand it. + use_cascade_attn = ( + forward_batch.forward_mode.is_target_verify() + and self.topk > 1 + and not is_swa + ) + + # For fa3 interface version compatibility, we put new fields into conditional keyword args + kwargs = {} + if sinks is not None: + kwargs["sinks"] = sinks + + # Get the appropriate page table based on whether we're using local attention + if use_local_attn: + local_metadata = metadata.local_attn_metadata + page_table = local_metadata.local_block_table + cu_seqlens_q = local_metadata.local_query_start_loc + cache_seqlens = local_metadata.local_seqused_k + max_seqlen_q = local_metadata.local_max_query_len + elif is_swa and metadata.swa_spec_metadata is not None: + swa_spec_metadata = metadata.swa_spec_metadata + page_table = swa_spec_metadata.page_table + cu_seqlens_q = swa_spec_metadata.cu_seqlens_q + cache_seqlens = swa_spec_metadata.cache_seqlens_int32 + max_seqlen_q = swa_spec_metadata.max_seq_len_q + cu_seqlens_k = swa_spec_metadata.cu_seqlens_k + else: + page_table = metadata.page_table + cu_seqlens_q = metadata.cu_seqlens_q + cache_seqlens = metadata.cache_seqlens_int32 + max_seqlen_q = metadata.max_seq_len_q + cu_seqlens_k = metadata.cu_seqlens_k + + # Use Flash Attention for prefill + if not self.use_mla: + # Do multi-head attention + key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + layer.layer_id + ) + key_cache = key_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ) + value_cache = value_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ) + if layer.is_cross_attention: + page_table = metadata.encoder_page_table + cache_seqlens = metadata.encoder_lens_int32 + cu_seqlens_k = metadata.encoder_cu_seqlens_k + window_size = (-1, -1) + + result = flash_attn_with_kvcache( + q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + k_cache=key_cache, + v_cache=value_cache, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k if not use_local_attn else None, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False if use_cascade_attn else causal, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=use_cascade_attn, + **kwargs, + ) + + if use_cascade_attn: + o, softmax_lse, *rest = result + o_expand, softmax_lse_expand, *rest_expand = flash_attn_with_kvcache( + q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + k_cache=key_cache, + v_cache=value_cache, + page_table=self.forward_metadata_spec_decode_expand.page_table, + cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, + cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, + cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, + max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + softmax_scale=layer.scaling, + causal=False, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=True, + **kwargs, + ) + o, _ = merge_state_v2_wrapper( + o, + softmax_lse.T.contiguous(), + o_expand, + softmax_lse_expand.T.contiguous(), + ) + else: + o = result + else: + if ( + forward_batch.attn_attend_prefix_cache is not None + and not forward_batch.forward_mode.is_target_verify() + and not forward_batch.forward_mode.is_draft_extend() + ): + # Do multi-head attention with chunked prefix cache + if forward_batch.attn_attend_prefix_cache: + assert not global_server_args_dict["disable_chunked_prefix_cache"] + # MHA for chunked prefix kv cache when running model with MLA + assert forward_batch.prefix_chunk_idx is not None + assert forward_batch.prefix_chunk_cu_seq_lens is not None + assert forward_batch.prefix_chunk_max_seq_lens is not None + + chunk_idx = forward_batch.prefix_chunk_idx + assert chunk_idx >= 0 + + assert forward_batch.mha_return_lse + output = flash_attn_varlen_func( + q=q.view(-1, layer.tp_q_head_num, layer.head_dim), + k=k.view(-1, layer.tp_k_head_num, layer.head_dim).to(q.dtype), + v=v.view(-1, layer.tp_k_head_num, layer.v_head_dim).to(q.dtype), + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k=forward_batch.prefix_chunk_cu_seq_lens[chunk_idx], + max_seqlen_q=metadata.max_seq_len_q, + max_seqlen_k=forward_batch.prefix_chunk_max_seq_lens[chunk_idx], + softmax_scale=layer.scaling, + causal=False, + return_softmax_lse=True, + ) + else: + # MHA for extend part of sequence without attending prefix kv cache + output = flash_attn_varlen_func( + q=q.view(-1, layer.tp_q_head_num, layer.head_dim), + k=k.view(-1, layer.tp_k_head_num, layer.head_dim).to(q.dtype), + v=v.view(-1, layer.tp_k_head_num, layer.v_head_dim).to(q.dtype), + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k=metadata.cu_seqlens_q, + max_seqlen_q=metadata.max_seq_len_q, + max_seqlen_k=metadata.max_seq_len_q, + softmax_scale=layer.scaling, + causal=True, + return_softmax_lse=forward_batch.mha_return_lse, + ) + if forward_batch.mha_return_lse: + output, lse, *rest = output + lse = torch.transpose(lse, 0, 1).contiguous() + return output, lse + return output + else: + # Do absorbed multi-latent attention + kv_cache = forward_batch.token_to_kv_pool.get_key_buffer( + layer.layer_id + ).to(q.dtype) + k_rope = kv_cache[:, :, layer.v_head_dim :] + c_kv = kv_cache[:, :, : layer.v_head_dim] + k_rope_cache = k_rope.view( + -1, + self.page_size, + layer.tp_k_head_num, + layer.head_dim - layer.v_head_dim, + ) + c_kv_cache = c_kv.view( + -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim + ) + if q_rope is not None: + q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + else: + q_all = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + q_nope = q_all[:, :, : layer.v_head_dim] + q_rope = q_all[:, :, layer.v_head_dim :] + + result = flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k if not use_local_attn else None, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False if use_cascade_attn else causal, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=use_cascade_attn, + ) + if use_cascade_attn: + o, softmax_lse, *rest = result + o_expand, softmax_lse_expand, *rest_expand = ( + flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=self.forward_metadata_spec_decode_expand.page_table, + cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, + cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, + cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, + max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + softmax_scale=layer.scaling, + causal=False, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=True, + ) + ) + o, _ = merge_state_v2_wrapper( + o, + softmax_lse.T.contiguous(), + o_expand, + softmax_lse_expand.T.contiguous(), + ) + else: + o = result + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + # For multi-head latent attention + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + sinks: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if k is not None: + assert v is not None + if save_kv_cache: + cache_loc = ( + forward_batch.out_cache_loc + if not layer.is_cross_attention + else forward_batch.encoder_out_cache_loc + ) + if not self.use_mla: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + else: + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, + cache_loc, + k, + k_rope, + ) + + # Use precomputed metadata across all layers + metadata = self.forward_metadata + local_attn_metadata = getattr(metadata, "local_attn_metadata", None) + use_local_attn = ( + self.attention_chunk_size is not None + and local_attn_metadata is not None + and (hasattr(layer, "use_irope") and layer.use_irope) + ) + + # When Spec Decode enabled, forward_decode would be called with two mode: + # 1. DRAFT_DECODE: we enable cascade attention when top_k > 1 + # 2. IDLE: we don’t need cascade attention, spec_info will be none in this case + use_cascade_attn = forward_batch.spec_info is not None and self.topk > 1 + + # Calculate window size (can be moved to metadata if layer properties don't change) + # we don't do layer.sliding_window_size - 1 since in model.get_attention_sliding_window_size() we already - 1 + # here is two side inclusive + window_size = ( + (layer.sliding_window_size, 0) + if layer.sliding_window_size is not None and layer.sliding_window_size > -1 + else (-1, -1) + ) + causal = not layer.is_cross_attention + + # For fa3 interface version compatibility, we put new fields into conditional keyword args + kwargs = {} + if sinks is not None: + kwargs["sinks"] = sinks + + k_descale, v_descale = None, None + # only use kv scaling if: 1) fp8 kv is explicitly enabled, 2) RadixAttention + # has corresponding quantization method so that layer.k_scale is not None, + # 3) layer.head_dim <= 256 since fa3 kernel require fp16 and bf16 data type in this case. + if self.kv_cache_dtype_str != "auto" and layer.head_dim <= 256: + if layer.k_scale is not None: + descale_shape = (forward_batch.batch_size, layer.tp_k_head_num) + k_descale = layer.k_scale.expand(descale_shape) + v_descale = layer.v_scale.expand(descale_shape) + q = q.to(self.kv_cache_dtype) + q_rope = q_rope.to(self.kv_cache_dtype) if q_rope is not None else None + k_rope = k_rope.to(self.kv_cache_dtype) if k_rope is not None else None + if not self.use_mla: + # Do multi-head attention + + key_cache, value_cache = forward_batch.token_to_kv_pool.get_kv_buffer( + layer.layer_id + ) + key_cache = key_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ) + value_cache = value_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ) + + if layer.is_cross_attention: + # Always use non-chunked logic for cross-attention + o = flash_attn_with_kvcache( + q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + k_cache=key_cache, + v_cache=value_cache, + page_table=metadata.encoder_page_table, + cache_seqlens=metadata.encoder_lens_int32, + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k_new=metadata.encoder_cu_seqlens_k, + max_seqlen_q=1, + softmax_scale=layer.scaling, + causal=False, + window_size=(-1, -1), + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + **kwargs, + ) + elif use_local_attn: + # Use chunked (local) attention batching for self-attention + o = flash_attn_with_kvcache( + q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + k_cache=key_cache, + v_cache=value_cache, + page_table=local_attn_metadata.local_block_table, + cache_seqlens=local_attn_metadata.local_seqused_k, + cu_seqlens_q=local_attn_metadata.local_query_start_loc, + cu_seqlens_k_new=None, + max_seqlen_q=local_attn_metadata.local_max_query_len, + softmax_scale=layer.scaling, + causal=True, + window_size=(-1, -1), + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + **kwargs, + ) + else: + page_table = metadata.page_table + cache_seqlens = metadata.cache_seqlens_int32 + cu_seqlens_k = metadata.cu_seqlens_k + max_seqlen_q = metadata.max_seq_len_q + q_reshaped = q.contiguous().view( + -1, layer.tp_q_head_num, layer.head_dim + ) + + # Default: single-token self-attention + result = flash_attn_with_kvcache( + q=q_reshaped, + k_cache=key_cache, + v_cache=value_cache, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False if use_cascade_attn else causal, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=use_cascade_attn, + **kwargs, + ) + if use_cascade_attn: + o, softmax_lse, *rest = result + o_expand, softmax_lse_expand, *rest_expand = ( + flash_attn_with_kvcache( + q=q_reshaped, + k_cache=key_cache, + v_cache=value_cache, + page_table=self.forward_metadata_spec_decode_expand.page_table, + cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, + cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, + cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, + max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + softmax_scale=layer.scaling, + causal=False, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=True, + **kwargs, + ) + ) + o, _ = merge_state_v2( + o, + softmax_lse.T.contiguous(), + o_expand, + softmax_lse_expand.T.contiguous(), + ) + else: + o = result + else: + # Do absorbed multi-latent attention + kv_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) + k_rope = kv_cache[:, :, layer.v_head_dim :] + c_kv = kv_cache[:, :, : layer.v_head_dim] + k_rope_cache = k_rope.view( + -1, + self.page_size, + layer.tp_k_head_num, + layer.head_dim - layer.v_head_dim, + ) + c_kv_cache = c_kv.view( + -1, self.page_size, layer.tp_v_head_num, layer.v_head_dim + ) + + if q_rope is not None: + q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + else: + q_all = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + q_nope = q_all[:, :, : layer.v_head_dim] + q_rope = q_all[:, :, layer.v_head_dim :] + max_seqlen_q = metadata.max_seq_len_q + + result = flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=metadata.page_table, + cache_seqlens=metadata.cache_seqlens_int32, + cu_seqlens_q=metadata.cu_seqlens_q, + cu_seqlens_k_new=metadata.cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + softmax_scale=layer.scaling, + causal=False if use_cascade_attn else causal, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=use_cascade_attn, # softmax_lse is needed for merge states + ) + if use_cascade_attn: + o, softmax_lse, *rest = result + o_expand, softmax_lse_expand, *rest_expand = flash_attn_with_kvcache( + q=q_rope, + k_cache=k_rope_cache, + v_cache=c_kv_cache, + qv=q_nope, + page_table=self.forward_metadata_spec_decode_expand.page_table, + cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32, + cu_seqlens_q=self.forward_metadata_spec_decode_expand.cu_seqlens_q, + cu_seqlens_k_new=self.forward_metadata_spec_decode_expand.cu_seqlens_k, + max_seqlen_q=self.forward_metadata_spec_decode_expand.max_seq_len_q, + softmax_scale=layer.scaling, + causal=False, + window_size=window_size, + softcap=layer.logit_cap, + k_descale=k_descale, + v_descale=v_descale, + return_softmax_lse=True, + ) + o, _ = merge_state_v2( + o, + softmax_lse.T.contiguous(), + o_expand, + softmax_lse_expand.T.contiguous(), + ) + else: + o = result + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + """Initialize CUDA graph state for the attention backend. + + Args: + max_bs (int): Maximum batch size to support in CUDA graphs + + This creates fixed-size tensors that will be reused during CUDA graph replay + to avoid memory allocations. + """ + max_num_pages = (self.max_context_len + self.page_size - 1) // self.page_size + + # This is being used by normal decode and draft decode when topk == 1 + self.decode_cuda_graph_metadata = { + "cache_seqlens": torch.zeros(max_bs, dtype=torch.int32, device=self.device), + "cu_seqlens_q": torch.arange( + 0, max_bs + 1, dtype=torch.int32, device=self.device + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + # Only allocate local attention buffers if local attention is enabled + # This prevents OOM errors when local attention is not being used + if self.attention_chunk_size is not None: + # Estimate maximum sizes for local attention metadata + max_seq_len = self.max_context_len + page_size = self.page_size or 1 + attn_chunk_size = self.attention_chunk_size + max_virtual_batches = max_bs * ( + (max_seq_len + attn_chunk_size - 1) // attn_chunk_size + ) + max_pages_per_block = (attn_chunk_size + page_size - 1) // page_size + + self.decode_cuda_graph_local_attn_metadata = { + "local_query_start_loc": torch.zeros( + max_virtual_batches + 1, dtype=torch.int32, device=self.device + ), + "local_seqused_k": torch.zeros( + max_virtual_batches, dtype=torch.int32, device=self.device + ), + "local_block_table": torch.zeros( + max_virtual_batches, + max_pages_per_block, + dtype=torch.int32, + device=self.device, + ), + } + + # This is used by draft decode's first half of metadata when topk > 1 + if self.topk > 1: + self.draft_decode_metadata_topk_normal = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.topk + 1, + step=self.topk, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + self.max_context_len, + dtype=torch.int32, + device=self.device, + ), + } + + # This is used by draft decode's second half of metadata when topk > 1 + decode_length = self.speculative_step_id + 1 + self.draft_decode_metadata_topk_expand = { + "cache_seqlens": torch.full( + (max_bs * self.topk,), + decode_length, + device=self.device, + dtype=torch.int32, + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.topk + 1, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.arange( + 0, + max_bs * self.topk * decode_length + 1, + step=decode_length, + dtype=torch.int32, + device=self.device, + ), + "page_table": torch.zeros( + max_bs * self.topk, + decode_length, + dtype=torch.int32, + device=self.device, + ), + } + + if ( + self.speculative_num_draft_tokens is not None + and self.speculative_num_draft_tokens > 0 + ): + # "page_table_draft_decode" will be set only when spec decoding enabled to save memory + self.decode_cuda_graph_metadata["page_table_draft_decode"] = torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ) + + self.target_verify_metadata = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.speculative_num_draft_tokens + 1, + step=self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + + self.draft_extend_metadata = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.zeros( + max_bs + 1, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + + if self.topk > 1: + self.target_verify_metadata_topk_normal = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.speculative_num_draft_tokens + 1, + step=self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + self.max_context_len, + dtype=torch.int32, + device=self.device, + ), + } + + self.target_verify_metadata_topk_expand = { + "cache_seqlens": torch.zeros( + max_bs * self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs * self.speculative_num_draft_tokens + 1, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.speculative_num_draft_tokens + 1, + dtype=torch.int32, + device=self.device, + ), + "page_table": torch.zeros( + max_bs * self.speculative_num_draft_tokens, + self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + } + + if self.has_swa: + self.target_verify_metadata_topk_swa = { + "cache_seqlens": torch.zeros( + max_bs * self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs * self.speculative_num_draft_tokens + 1, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.speculative_num_draft_tokens + 1, + dtype=torch.int32, + device=self.device, + ), + "page_table": torch.zeros( + max_bs * self.speculative_num_draft_tokens, + self.max_context_len, + dtype=torch.int32, + device=self.device, + ), + } + + self.encoder_metadata = { + "encoder_page_table": torch.zeros( + max_bs, + self.max_context_len, + dtype=torch.int32, + device=self.device, + ), + "encoder_lens_int32": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "encoder_cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + } + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + """Initialize forward metadata for capturing CUDA graph.""" + metadata = FlashAttentionMetadata() + + # metadata_expand is needed for Spec Decoding when top k > 1 + metadata_expand = FlashAttentionMetadata() + + device = seq_lens.device + if forward_mode.is_decode_or_idle(): + if spec_info is not None: + # Draft Decode + if self.topk <= 1: + # When topk = 1, we use the normal decode metadata + metadata.cache_seqlens_int32 = self.decode_cuda_graph_metadata[ + "cache_seqlens" + ][:bs] + metadata.max_seq_len_k = seq_lens.max().item() + ( + self.speculative_step_id + 1 + ) + metadata.cu_seqlens_q = self.decode_cuda_graph_metadata[ + "cu_seqlens_q" + ][: bs + 1] + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = self.decode_cuda_graph_metadata[ + "page_table_draft_decode" + ][:bs, :] + self.decode_cuda_graph_metadata[bs] = metadata + else: + # When top k > 1, we need two specific draft decode metadata, and then merge states + # 1. The first half of metadata for prefix tokens + metadata.cache_seqlens_int32 = ( + self.draft_decode_metadata_topk_normal["cache_seqlens"][:bs] + ) + metadata.max_seq_len_q = self.topk + metadata.max_seq_len_k = seq_lens.max().item() + metadata.cu_seqlens_q = self.draft_decode_metadata_topk_normal[ + "cu_seqlens_q" + ][: bs + 1] + metadata.cu_seqlens_k = self.draft_decode_metadata_topk_normal[ + "cu_seqlens_k" + ][: bs + 1] + metadata.page_table = self.draft_decode_metadata_topk_normal[ + "page_table" + ][:bs, :] + + # 2. The second half of metadata for draft tokens (per_batch_num_tokens = topk) + metadata_expand.cache_seqlens_int32 = ( + self.draft_decode_metadata_topk_expand["cache_seqlens"][ + : bs * self.topk + ] + ) + metadata_expand.max_seq_len_q = 1 + metadata_expand.cu_seqlens_q = ( + self.draft_decode_metadata_topk_expand["cu_seqlens_q"][ + : bs * self.topk + 1 + ] + ) + metadata_expand.cu_seqlens_k = ( + self.draft_decode_metadata_topk_expand["cu_seqlens_k"][ + : bs * self.topk + 1 + ] + ) + metadata_expand.page_table = self.draft_decode_metadata_topk_expand[ + "page_table" + ][: bs * self.topk] + self.draft_decode_metadata_topk_normal[bs] = metadata + self.draft_decode_metadata_topk_expand[bs] = metadata_expand + else: + # Normal Decode + # Get sequence information + metadata.cache_seqlens_int32 = seq_lens.to(torch.int32) + batch_size = len(seq_lens) + device = seq_lens.device + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seq_lens, dim=0, dtype=torch.int32), (1, 0) + ) + # Precompute maximum sequence length + metadata.max_seq_len_k = seq_lens.max().item() + # Precompute page table + metadata.page_table = self.decode_cuda_graph_metadata["page_table"][ + :bs, : + ] + # Precompute cumulative sequence lengths + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + self.decode_cuda_graph_metadata[bs] = metadata + + if self.attention_chunk_size is not None: + self._update_local_attn_metadata_for_capture(metadata, batch_size) + + elif forward_mode.is_target_verify(): + if self.topk <= 1: + metadata.cache_seqlens_int32 = self.target_verify_metadata[ + "cache_seqlens" + ][:bs] + metadata.cache_seqlens_int32.copy_( + (seq_lens + self.speculative_num_draft_tokens) + ) + + metadata.max_seq_len_q = self.speculative_num_draft_tokens + metadata.max_seq_len_k = ( + seq_lens.max().item() + self.speculative_num_draft_tokens + ) + + metadata.cu_seqlens_q = torch.arange( + 0, + bs * self.speculative_num_draft_tokens + 1, + self.speculative_num_draft_tokens, + dtype=torch.int32, + device=device, + ) + + metadata.cu_seqlens_k = self.target_verify_metadata["cu_seqlens_k"][ + : (bs + 1) + ] + + metadata.page_table = self.target_verify_metadata["page_table"][:bs, :] + + self.target_verify_metadata[bs] = metadata + else: + # When topk > 1, we need two specific target verify metadata, and then merge states + # 1. The first half of metadata for prefix tokens + metadata.cache_seqlens_int32 = self.target_verify_metadata_topk_normal[ + "cache_seqlens" + ][:bs] + metadata.max_seq_len_q = self.speculative_num_draft_tokens + # metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item(), do this in replay + metadata.cu_seqlens_q = self.target_verify_metadata_topk_normal[ + "cu_seqlens_q" + ][: bs + 1] + metadata.cu_seqlens_k = self.target_verify_metadata_topk_normal[ + "cu_seqlens_k" + ][: bs + 1] + metadata.page_table = self.target_verify_metadata_topk_normal[ + "page_table" + ][:bs, :] + + # 2. The second half of metadata for draft tokens (per_batch_num_tokens = topk) + metadata_expand.cache_seqlens_int32 = ( + self.target_verify_metadata_topk_expand["cache_seqlens"][ + : bs * self.speculative_num_draft_tokens + ] + ) + metadata_expand.max_seq_len_q = 1 + metadata_expand.cu_seqlens_q = self.target_verify_metadata_topk_expand[ + "cu_seqlens_q" + ][: bs * self.speculative_num_draft_tokens + 1] + metadata_expand.cu_seqlens_k = self.target_verify_metadata_topk_expand[ + "cu_seqlens_k" + ][: bs * self.speculative_num_draft_tokens + 1] + + metadata_expand.page_table = self.target_verify_metadata_topk_expand[ + "page_table" + ][: bs * self.speculative_num_draft_tokens] + + self.target_verify_metadata_topk_normal[bs] = metadata + self.target_verify_metadata_topk_expand[bs] = metadata_expand + + if self.has_swa: + metadata_swa = FlashAttentionMetadata() + metadata_swa.cache_seqlens_int32 = ( + self.target_verify_metadata_topk_swa["cache_seqlens"][ + : bs * self.speculative_num_draft_tokens + ] + ) + metadata_swa.max_seq_len_q = 1 + metadata_swa.cu_seqlens_q = self.target_verify_metadata_topk_swa[ + "cu_seqlens_q" + ][: bs * self.speculative_num_draft_tokens + 1] + metadata_swa.cu_seqlens_k = self.target_verify_metadata_topk_swa[ + "cu_seqlens_k" + ][: bs * self.speculative_num_draft_tokens + 1] + + metadata_swa.page_table = self.target_verify_metadata_topk_swa[ + "page_table" + ][: bs * self.speculative_num_draft_tokens] + self.target_verify_metadata_topk_swa[bs] = metadata_swa + metadata.swa_spec_metadata = metadata_swa + + elif forward_mode.is_draft_extend(): + metadata.cache_seqlens_int32 = self.draft_extend_metadata["cache_seqlens"][ + :bs + ] + metadata.cache_seqlens_int32.copy_(seq_lens) + + num_tokens_per_bs = num_tokens // bs + metadata.max_seq_len_q = num_tokens_per_bs + metadata.max_seq_len_k = seq_lens.max().item() + + metadata.cu_seqlens_q = torch.arange( + 0, + bs * num_tokens_per_bs + 1, + num_tokens_per_bs, + dtype=torch.int32, + device=device, + ) + + metadata.cu_seqlens_k = self.draft_extend_metadata["cu_seqlens_k"][ + : (bs + 1) + ] + metadata.page_table = self.draft_extend_metadata["page_table"][:bs, :] + + self.draft_extend_metadata[bs] = metadata + + if encoder_lens is not None: + encoder_bs = encoder_lens.numel() + metadata.encoder_lens_int32 = self.encoder_metadata["encoder_lens_int32"][ + :encoder_bs + ] + metadata.encoder_cu_seqlens_k = self.encoder_metadata[ + "encoder_cu_seqlens_k" + ][: (encoder_bs + 1)] + + metadata.encoder_page_table = self.encoder_metadata["encoder_page_table"][ + :bs, : + ] + + self.forward_metadata = metadata + self.forward_metadata_spec_decode_expand = metadata_expand + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + out_cache_loc: Optional[torch.Tensor] = None, + ): + """Initialize forward metadata for replaying CUDA graph.""" + seq_lens = seq_lens[:bs] + seq_lens_cpu = seq_lens_cpu[:bs] + req_pool_indices = req_pool_indices[:bs] + device = seq_lens.device + metadata = None + metadata_expand = None + + if forward_mode.is_decode_or_idle(): + + if spec_info is not None: + # Draft Decode + if self.topk <= 1: + # When topk = 1, we use the normal decode metadata + metadata = self.decode_cuda_graph_metadata[bs] + max_len = seq_lens_cpu.max().item() + metadata.max_seq_len_k = max_len + self.speculative_step_id + 1 + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + + normal_decode_set_metadata( + metadata.cache_seqlens_int32, + metadata.cu_seqlens_k, + metadata.page_table, + self.req_to_token, + req_pool_indices, + self.decode_cuda_graph_metadata["strided_indices"], + max_seq_pages, + seq_lens, + self.speculative_step_id + 1, + self.page_size, + ) + + else: + # When top k > 1, we need two specific draft decode metadata, and then merge states + # 1. The first half of metadata for prefix tokens + metadata = self.draft_decode_metadata_topk_normal[bs] + metadata.cache_seqlens_int32.copy_(seq_lens) + # metadata.max_seq_len_q = self.topk, already set in capture + metadata.max_seq_len_k = seq_lens_cpu.max().item() + # metadata.cu_seqlens_q already set in capture + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ) + ) + + page_table = self.req_to_token[ + req_pool_indices, : metadata.max_seq_len_k + ] + + metadata.page_table[:, : metadata.max_seq_len_k].copy_(page_table) + + # 2. The second half of metadata for draft tokens (per_batch_num_tokens = topk) + metadata_expand = self.draft_decode_metadata_topk_expand[bs] + decode_length = self.speculative_step_id + 1 + # shape: [bs, num_steps, topk] -> [bs x topk, num_steps] + cache_loc = out_cache_loc.view(-1, self.speculative_num_steps) + metadata_expand.page_table[: cache_loc.shape[0]].copy_( + cache_loc[:, :decode_length] + ) + # TODO: Handle local attention metadata for draft decode when llama4 eagle is supported + else: + # Normal Decode + metadata = self.decode_cuda_graph_metadata[bs] + max_len = seq_lens_cpu.max().item() + max_seq_pages = (max_len + self.page_size - 1) // self.page_size + metadata.max_seq_len_k = max_len + + normal_decode_set_metadata( + metadata.cache_seqlens_int32, + metadata.cu_seqlens_k, + metadata.page_table, + self.req_to_token, + req_pool_indices, + self.decode_cuda_graph_metadata["strided_indices"], + max_seq_pages, + seq_lens, + 0, + self.page_size, + ) + + self._update_local_attn_metadata_for_replay( + metadata, + bs, + ) + elif forward_mode.is_target_verify(): + if self.topk <= 1: + metadata = self.target_verify_metadata[bs] + metadata.cache_seqlens_int32.copy_( + (seq_lens + self.speculative_num_draft_tokens) + ) + + metadata.max_seq_len_k = ( + seq_lens_cpu.max().item() + self.speculative_num_draft_tokens + ) + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.decode_cuda_graph_metadata["strided_indices"][:max_seq_pages], + ] + page_indices //= self.page_size + metadata.page_table[:, :max_seq_pages].copy_(page_indices) + else: + # When topk > 1, we need two specific target verify metadata, and then merge states + # 1. The first half of metadata for prefix tokens + metadata = self.target_verify_metadata_topk_normal[bs] + metadata.cache_seqlens_int32.copy_(seq_lens) + # metadata.max_seq_len_q = self.speculative_num_draft_tokens, already set in capture + metadata.max_seq_len_k = seq_lens_cpu.max().item() + # metadata.cu_seqlens_q already set in capture + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + page_table = self.req_to_token[ + req_pool_indices, : metadata.max_seq_len_k + ] + metadata.page_table[:, : metadata.max_seq_len_k].copy_(page_table) + + # 2. The second half of metadata for draft tokens (per_batch_num_tokens = topk) + metadata_expand = self.target_verify_metadata_topk_expand[bs] + + # metadata_expand.max_seq_len_q = 1, already set in capture + # metadata_expand.cu_seqlens_q already set in capture + offsets = torch.arange( + self.speculative_num_draft_tokens, device=device + ).unsqueeze( + 0 + ) # shape: (1, self.speculative_num_draft_tokens) + + cols = offsets.expand(seq_lens.numel(), -1) + seq_lens.unsqueeze(1) + cum_len = torch.nn.functional.pad( + torch.cumsum( + ( + seq_lens + self.speculative_num_draft_tokens + ).repeat_interleave(self.speculative_num_draft_tokens), + dim=0, + ), + (1, 0), + )[:-1] + mask_extraction_indices = ( + cols.repeat_interleave(self.speculative_num_draft_tokens, dim=0) + + cum_len[:, None] + ).view(1, -1) + # avoid extracting padded seq indices which will be out of boundary + mask_extraction_indices[ + :, + spec_info.positions.numel() * self.speculative_num_draft_tokens :, + ].fill_(0) + mask = spec_info.custom_mask[mask_extraction_indices].view( + -1, self.speculative_num_draft_tokens + ) # (bsz * draft_num, draft_num) + + col_indices = offsets.expand( + mask.shape[0], self.speculative_num_draft_tokens + ) + keys = torch.where( + mask, + col_indices, + col_indices + self.speculative_num_draft_tokens, + ) + _, sort_order = torch.sort(keys, dim=1) + + non_masked_page_table = ( + self.req_to_token[req_pool_indices, :] + .gather(1, cols) + .repeat_interleave(self.speculative_num_draft_tokens, dim=0) + ) # (bsz, draft_num) + + metadata_expand.page_table.copy_( + non_masked_page_table.gather(1, sort_order) + ) + metadata_expand.cache_seqlens_int32.copy_(mask.sum(dim=1)) + metadata_expand.cu_seqlens_k[1:].copy_( + torch.cumsum( + metadata_expand.cache_seqlens_int32, + dim=0, + dtype=torch.int32, + ) + ) + + if self.has_swa: + metadata_swa = self.target_verify_metadata_topk_swa[bs] + self._init_sliding_window_attn_spec_metadata( + metadata, metadata_expand, metadata_swa + ) + + elif forward_mode.is_draft_extend(): + metadata = self.draft_extend_metadata[bs] + metadata.cache_seqlens_int32.copy_(seq_lens) + + metadata.max_seq_len_k = seq_lens_cpu.max().item() + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + accept_length = spec_info.accept_length[:bs] + if spec_info.accept_length_cpu: + metadata.max_seq_len_q = max(spec_info.accept_length_cpu) + 1 + else: + metadata.max_seq_len_q = 1 + + metadata.cu_seqlens_q[1:].copy_( + torch.cumsum(accept_length, dim=0, dtype=torch.int32) + ) + + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.draft_extend_metadata["strided_indices"][:max_seq_pages], + ] + metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size) + + if encoder_lens is not None: + # Only support encoder size 1 for now + metadata.encoder_max_seq_len_k = encoder_lens[0] + metadata.encoder_lens_int32.copy_(encoder_lens[:1]) + metadata.encoder_cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.encoder_lens_int32, dim=0, dtype=torch.int32) + ) + + metadata.encoder_page_table[:, : metadata.encoder_max_seq_len_k].copy_( + self.req_to_token[req_pool_indices, : metadata.encoder_max_seq_len_k] + ) + + # Update the regular page table + page_table = self.req_to_token[ + req_pool_indices, + metadata.encoder_max_seq_len_k : ( + metadata.encoder_max_seq_len_k + metadata.max_seq_len_k + ), + ] + metadata.page_table[:, : metadata.max_seq_len_k].copy_(page_table) + + self.forward_metadata = metadata + self.forward_metadata_spec_decode_expand = metadata_expand + + def get_cuda_graph_seq_len_fill_value(self): + """Get the fill value for sequence length in CUDA graph.""" + return 1 + + def _init_local_attn_metadata( + self, forwardbatch: ForwardBatch, metadata: FlashAttentionMetadata, device + ): + """Centralized utility to initialize local_attn_metadata if chunked attention is enabled.""" + if self.attention_chunk_size is None: + metadata.local_attn_metadata = None + return + + cu_seqlens_q = metadata.cu_seqlens_q + cache_seqlens_int32 = metadata.cache_seqlens_int32 + if self.is_hybrid: + page_table = self.full_to_swa_index_mapping[metadata.page_table].to( + torch.int32 + ) + else: + page_table = metadata.page_table + if cu_seqlens_q is None or cache_seqlens_int32 is None or page_table is None: + metadata.local_attn_metadata = None + return + + cu_seqlens_q_np = cu_seqlens_q.cpu().numpy() + seq_lens_np = cache_seqlens_int32.cpu().numpy() + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + cu_seqlens_q_np, + seq_lens_np, + page_table, + self.page_size, + ) + + local_metadata = FlashAttentionMetadata.LocalAttentionMetadata( + local_query_start_loc=torch.from_numpy(cu_seqlens_q_local_np).to(device), + local_seqused_k=torch.from_numpy(seqlens_k_local_np).to(device), + local_block_table=block_table_local.to(device), + local_max_query_len=int(seqlens_q_local_np.max()), + local_max_seq_len=int(seqlens_k_local_np.max()), + ) + metadata.local_attn_metadata = local_metadata + + def _update_local_attn_metadata_for_capture( + self, metadata: FlashAttentionMetadata, bs: int + ): + """Update local attention metadata during CUDA graph capture phase. + + This method calculates the exact buffer sizes needed for local attention metadata + during the CUDA graph capture phase, optimizing memory usage by creating views of + pre-allocated buffers with exactly the sizes needed. + """ + seq_lens_capture = metadata.cache_seqlens_int32 + max_seq_len = int(seq_lens_capture.max().item()) + page_table_capture = metadata.page_table + + cu_seqlens_q_np = metadata.cu_seqlens_q.cpu().numpy() + seqlens_np = seq_lens_capture.cpu().numpy() + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local_np, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + cu_seqlens_q_np, + seqlens_np, + page_table_capture, + self.page_size, + ) + + # Get exact dimensions from the calculation + q_len = len(cu_seqlens_q_local_np) + k_len = len(seqlens_k_local_np) + b0 = block_table_local_np.shape[0] if block_table_local_np.shape[0] > 0 else bs + b1 = block_table_local_np.shape[1] if block_table_local_np.shape[1] > 0 else 1 + + # Create views of the pre-allocated buffers with exactly these sizes + # This is the key optimization - we only use the memory we actually need + local_query_start_loc = self.decode_cuda_graph_local_attn_metadata[ + "local_query_start_loc" + ][:q_len] + + local_seqused_k = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"][ + :k_len + ] + + local_block_table = self.decode_cuda_graph_local_attn_metadata[ + "local_block_table" + ][:b0, :b1] + + metadata.local_attn_metadata = FlashAttentionMetadata.LocalAttentionMetadata( + local_query_start_loc=local_query_start_loc, + local_seqused_k=local_seqused_k, + local_block_table=local_block_table, + local_max_query_len=1, + local_max_seq_len=max_seq_len, + ) + + def _update_local_attn_metadata_for_replay( + self, + metadata: FlashAttentionMetadata, + bs: int, + ): + """Update preallocated local attention metadata in-place before CUDA graph replay.""" + if self.attention_chunk_size is None: + return + + # Access preallocated buffers + local_q_buf = self.decode_cuda_graph_local_attn_metadata[ + "local_query_start_loc" + ] + local_k_buf = self.decode_cuda_graph_local_attn_metadata["local_seqused_k"] + local_block_buf = self.decode_cuda_graph_local_attn_metadata[ + "local_block_table" + ] + cu_seqlens_q = self.decode_cuda_graph_metadata["cu_seqlens_q"] + + # Create a modified version for local attention that only processes the last token + # This mimics the normal decode pattern + cu_seqlens_q = torch.arange( + bs + 1, device=cu_seqlens_q.device, dtype=cu_seqlens_q.dtype + ) + seqlens = metadata.cache_seqlens_int32[:bs] + # Slice the page_table to match the batch size and actual sequence length + # This serves three important purposes: + # 1. Ensures we only process the actual batch size (bs) and not the maximum batch size + # 2. Limits the sequence length to prevent processing padding tokens or garbage values + # 3. Prevents zeros in the block table which can cause garbage output during replay + # + # Without this slicing, the pre-allocated page_table may contain zeros or invalid indices + # beyond the actual sequence length, leading to incorrect attention calculations + max_seq_len = int(seqlens.max().item()) + if self.is_hybrid: + sliced_page_table = self.full_to_swa_index_mapping[ + metadata.page_table[:bs, :max_seq_len] + ].to(torch.int32) + else: + sliced_page_table = metadata.page_table[:bs, :max_seq_len] + + cu_seqlens_q_np = cu_seqlens_q.cpu().numpy() + seqlens_np = seqlens.cpu().numpy() + ( + seqlens_q_local_np, + cu_seqlens_q_local_np, + seqlens_k_local_np, + block_table_local, + ) = make_local_attention_virtual_batches( + self.attention_chunk_size, + cu_seqlens_q_np, + seqlens_np, + sliced_page_table, + self.page_size, + ) + + # Convert back to tensors + device = local_q_buf.device + cu_seqlens_q_local = torch.from_numpy(cu_seqlens_q_local_np).to(device) + seqlens_k_local = torch.from_numpy(seqlens_k_local_np).to(device) + block_table_local = block_table_local.to(device) + # Get sizes + q_len = cu_seqlens_q_local.shape[0] + k_len = seqlens_k_local.shape[0] + b0, b1 = block_table_local.shape + + # In-place updates into preallocated tensors and zero out the unused space + local_q_buf[:q_len].copy_(cu_seqlens_q_local) + local_q_buf[q_len:].fill_(0) + local_k_buf[:k_len].copy_(seqlens_k_local) + local_k_buf[k_len:].fill_(0) + local_block_buf[:b0, :b1].copy_(block_table_local) + local_block_buf[b0:, :].fill_(0) + local_block_buf[:b0, b1:].fill_(0) + + if metadata.local_attn_metadata is not None: + lam = metadata.local_attn_metadata + lam.local_max_query_len = int(seqlens_q_local_np.max()) + lam.local_max_seq_len = int(seqlens_k_local_np.max()) + + def _init_sliding_window_attn_spec_metadata( + self, + metadata: FlashAttentionMetadata, + metadata_expand: FlashAttentionMetadata, + metadata_swa: Optional[FlashAttentionMetadata] = None, + ): + # TODO: support page_size > 1 for swa spec + assert ( + self.page_size == 1 + ), "FlashAttention backend doesn't support topk > 1 speculative decoding with page size > 1 sliding window attention" + + cache_seqlens_int32 = ( + metadata.cache_seqlens_int32.repeat_interleave( + self.speculative_num_draft_tokens + ) + + metadata_expand.cache_seqlens_int32 + ) + cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(cache_seqlens_int32, dim=0, dtype=torch.int32), (1, 0) + ) + bs = cache_seqlens_int32.shape[0] + page_table = ( + metadata.page_table.new_zeros( + (bs, metadata.max_seq_len_k + metadata_expand.page_table.shape[1]) + ) + if metadata_swa is None + else metadata_swa.page_table + ) + + prepare_swa_spec_page_table_triton( + page_table, + metadata.page_table, + metadata_expand.page_table, + metadata.cache_seqlens_int32, + metadata_expand.cache_seqlens_int32, + self.speculative_num_draft_tokens, + ) + + if metadata_swa is None: + metadata_swa = FlashAttentionMetadata() + metadata_swa.max_seq_len_q = 1 + metadata_swa.cu_seqlens_q = metadata_expand.cu_seqlens_q + metadata_swa.cache_seqlens_int32 = cache_seqlens_int32 + metadata_swa.cu_seqlens_k = cu_seqlens_k + metadata_swa.page_table = page_table + else: + metadata_swa.cache_seqlens_int32.copy_(cache_seqlens_int32) + metadata_swa.cu_seqlens_k.copy_(cu_seqlens_k) + + metadata.swa_spec_metadata = metadata_swa + + +@triton.jit +def _prepare_swa_spec_page_table_kernel( + dst_ptr, + src_a_ptr, + src_b_ptr, + seq_len_a_ptr, + seq_len_b_ptr, + dst_stride_m, + dst_stride_n, + a_stride_m, + a_stride_n, + b_stride_m, + b_stride_n, + LEN_A: tl.constexpr, + LEN_B: tl.constexpr, + REPEAT_STEP: tl.constexpr, + BLOCK_N: tl.constexpr, +): + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + + idx_a = pid_m // REPEAT_STEP + idx_b = pid_m + seq_len_a = tl.load(seq_len_a_ptr + idx_a) + seq_len_b = tl.load(seq_len_b_ptr + idx_b) + + offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N) + total_len = seq_len_a + seq_len_b + + if pid_n * BLOCK_N >= total_len: + return + + mask = offs_n < total_len + dst = dst_ptr + pid_m * dst_stride_m + offs_n * dst_stride_n + + if (pid_n + 1) * BLOCK_N < seq_len_a: + a_ptr = src_a_ptr + idx_a * a_stride_m + offs_n * a_stride_n + a_mask = mask & (offs_n < LEN_A) + val = tl.load(a_ptr, mask=a_mask, other=0) + tl.store(dst, val, mask=mask) + elif pid_n * BLOCK_N >= seq_len_a: + offs_b = offs_n - seq_len_a + b_ptr = src_b_ptr + idx_b * b_stride_m + offs_b * b_stride_n + b_mask = mask & (offs_b < LEN_B) + val = tl.load(b_ptr, mask=b_mask, other=0) + tl.store(dst, val, mask=mask) + else: + # mixed part + a_offs = offs_n + a_mask = (a_offs < seq_len_a) & (a_offs < LEN_A) + a_ptr = src_a_ptr + idx_a * a_stride_m + a_offs * a_stride_n + a_val = tl.load(a_ptr, mask=a_mask, other=0) + + b_offs = offs_n - seq_len_a + b_mask = (b_offs >= 0) & (b_offs < seq_len_b) & (b_offs < LEN_B) + b_ptr = src_b_ptr + idx_b * b_stride_m + b_offs * b_stride_n + b_val = tl.load(b_ptr, mask=b_mask, other=0) + + result = tl.where(offs_n < seq_len_a, a_val, b_val) + tl.store(dst, result, mask=mask) + + +def prepare_swa_spec_page_table_triton( + page_table_dst: torch.Tensor, + page_table_a: torch.Tensor, + page_table_b: torch.Tensor, # expand page table + seq_len_a: torch.Tensor, + seq_len_b: torch.Tensor, # expand seq lens + speculative_num_draft_tokens: int, +): + # concat page_table and expand page_table by kv seq length + bs = seq_len_a.numel() + bs_expand = seq_len_b.numel() + assert bs_expand == bs * speculative_num_draft_tokens + + LEN_A = page_table_a.shape[1] + LEN_B = page_table_b.shape[1] + LEN_OUT = LEN_A + LEN_B + REPEAT_STEP = speculative_num_draft_tokens + BLOCK_N = 256 + + grid = (bs_expand, triton.cdiv(LEN_OUT, BLOCK_N)) + _prepare_swa_spec_page_table_kernel[grid]( + page_table_dst, + page_table_a, + page_table_b, + seq_len_a, + seq_len_b, + page_table_dst.stride(0), + page_table_dst.stride(1), + page_table_a.stride(0), + page_table_a.stride(1), + page_table_b.stride(0), + page_table_b.stride(1), + LEN_A=LEN_A, + LEN_B=LEN_B, + REPEAT_STEP=REPEAT_STEP, + BLOCK_N=BLOCK_N, + num_warps=4, + ) + + +class FlashAttentionMultiStepBackend: + + def __init__( + self, model_runner: ModelRunner, topk: int, speculative_num_steps: int + ): + self.model_runner = model_runner + self.topk = topk + self.speculative_num_steps = speculative_num_steps + self.attn_backends = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + FlashAttentionBackend( + model_runner, + speculative_step_id=i, + topk=self.topk, + speculative_num_steps=self.speculative_num_steps, + ) + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata(forward_batch) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state(max_bs, max_num_tokens) + + def init_forward_metadata_capture_cuda_graph( + self, + forward_batch: ForwardBatch, + ): + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=forward_batch.encoder_lens, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + for i in range(self.speculative_num_steps - 1): + # TODO: incrementally update the metadata for the later steps, + # so that they do not need to recompute everything from scratch. + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + encoder_lens=forward_batch.encoder_lens, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + out_cache_loc=forward_batch.out_cache_loc, + ) + + +# @torch.compile(dynamic=True, backend=get_compiler_backend()) +# TODO: fuse these kernels +# NOTE: torch.compile makes it slower in speculative decoding +def normal_decode_set_metadata( + cache_seqlens_int32: torch.Tensor, + cu_seqlens_k: torch.Tensor, + page_table: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + strided_indices: torch.Tensor, + max_seq_pages: torch.Tensor, + seq_lens: torch.Tensor, + seq_len_delta: int, + page_size: int, +): + cache_seqlens_int32.copy_(seq_lens + seq_len_delta) + cu_seqlens_k[1:].copy_(torch.cumsum(cache_seqlens_int32, dim=0, dtype=torch.int32)) + page_indices = req_to_token[ + req_pool_indices[:, None], + strided_indices[:max_seq_pages][None, :], + ] + page_table[:, :max_seq_pages].copy_(page_indices // page_size) diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py new file mode 100644 index 000000000..a5b207c77 --- /dev/null +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -0,0 +1,1441 @@ +from __future__ import annotations + +""" +Support different attention backends. +Now there are two backends: FlashInfer and Triton. +FlashInfer is faster and Triton is easier to customize. +Each backend supports two operators: extend (i.e. prefill with cached prefix) and decode. +""" + +import os +from dataclasses import dataclass +from enum import Enum, auto +from functools import partial +from typing import TYPE_CHECKING, Callable, List, Optional, Union + +import torch + +if os.environ["SGLANG_ENABLE_TORCH_COMPILE"] == "1": + import logging + + torch._logging.set_logs(dynamo=logging.ERROR) + torch._dynamo.config.suppress_errors = True + +from sglang.global_config import global_config +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.mem_cache.allocator import SWATokenToKVPoolAllocator +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput +from sglang.srt.utils import ( + is_flashinfer_available, + is_sm100_supported, + next_power_of_2, +) + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + +if is_flashinfer_available(): + from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + BatchPrefillWithPagedKVCacheWrapper, + BatchPrefillWithRaggedKVCacheWrapper, + ) + from flashinfer.cascade import merge_state + from flashinfer.decode import _get_range_buf, get_seq_lens + + +class WrapperDispatch(Enum): + SLIDING_WINDOW = auto() + CROSS_ATTENTION = auto() + + +@dataclass +class DecodeMetadata: + decode_wrappers: List[BatchDecodeWithPagedKVCacheWrapper] + + +@dataclass +class PrefillMetadata: + prefill_wrappers: List[BatchPrefillWithPagedKVCacheWrapper] + use_ragged: bool + extend_no_prefix: bool + + +# Reuse this workspace buffer across all flashinfer wrappers +global_workspace_buffer = None + +# Use as a fast path to override the indptr in flashinfer's plan function +# This is used to remove some host-to-device copy overhead. +global_override_indptr_cpu = None + + +class FlashInferAttnBackend(AttentionBackend): + """Flashinfer attention kernels.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + kv_last_page_len_buf: Optional[torch.Tensor] = None, + ): + super().__init__() + + # Parse constants + self.decode_use_tensor_cores = should_use_tensor_core( + kv_cache_dtype=model_runner.kv_cache_dtype, + num_attention_heads=model_runner.model_config.num_attention_heads + // get_attention_tp_size(), + num_kv_heads=model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ), + ) + self.max_context_len = model_runner.model_config.context_len + self.skip_prefill = skip_prefill + self.is_multimodal = model_runner.model_config.is_multimodal + + assert not ( + model_runner.sliding_window_size is not None + and model_runner.model_config.is_encoder_decoder + ), "Sliding window and cross attention are not supported together" + + if model_runner.sliding_window_size is not None: + self.num_wrappers = 2 + self.dispatch_reason = WrapperDispatch.SLIDING_WINDOW + elif model_runner.model_config.is_encoder_decoder: + self.num_wrappers = 2 + self.dispatch_reason = WrapperDispatch.CROSS_ATTENTION + else: + self.num_wrappers = 1 + self.dispatch_reason = None + + # Qwen2/Qwen3 models require higher flashinfer workspace size + if ( + "Qwen2ForCausalLM" in model_runner.model_config.hf_config.architectures + or "Qwen3ForCausalLM" in model_runner.model_config.hf_config.architectures + or "MiMoForCausalLM" in model_runner.model_config.hf_config.architectures + ): + global_config.flashinfer_workspace_size = 512 * 1024 * 1024 + + # Allocate buffers + global global_workspace_buffer + if global_workspace_buffer is None: + # different from flashinfer zero_init_global_workspace_buffer + global_workspace_buffer = torch.empty( + global_config.flashinfer_workspace_size, + dtype=torch.uint8, + device=model_runner.device, + ) + self.workspace_buffer = global_workspace_buffer + max_bs = model_runner.req_to_token_pool.size + if kv_indptr_buf is None: + self.kv_indptr = [ + torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + for _ in range(self.num_wrappers) + ] + else: + assert self.num_wrappers == 1 + self.kv_indptr = [kv_indptr_buf] + + if kv_last_page_len_buf is None: + self.kv_last_page_len = torch.ones( + (max_bs,), dtype=torch.int32, device=model_runner.device + ) + else: + assert self.num_wrappers == 1 + self.kv_last_page_len = kv_last_page_len_buf + + if not self.skip_prefill: + self.qo_indptr = [ + torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + for _ in range(self.num_wrappers) + ] + + fmha_backend = "auto" + if is_sm100_supported(): + fmha_backend = "cutlass" + self.prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper( + self.workspace_buffer, "NHD", backend=fmha_backend + ) + + # Two wrappers: one for sliding window attention and one for full attention. + # Using two wrappers is unnecessary in the current PR, but are prepared for future PRs + self.prefill_wrappers_paged = [] + self.prefill_wrappers_verify = [] + self.decode_wrappers = [] + for _ in range(self.num_wrappers): + if not skip_prefill: + self.prefill_wrappers_paged.append( + BatchPrefillWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + backend="fa2", + ) + ) + self.prefill_wrappers_verify.append( + BatchPrefillWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + ) + ) + self.decode_wrappers.append( + BatchDecodeWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + use_tensor_cores=self.decode_use_tensor_cores, + ) + ) + + # Create indices updater + if not skip_prefill: + self.indices_updater_prefill = FlashInferIndicesUpdaterPrefill( + model_runner, self + ) # for verify + self.indices_updater_decode = FlashInferIndicesUpdaterDecode(model_runner, self) + + # Other metadata + self.forward_metadata: Union[PrefillMetadata, DecodeMetadata] = None + self.decode_cuda_graph_metadata = {} + self.prefill_cuda_graph_metadata = {} # For verify + self.draft_extend_cuda_graph_metadata = {} # For draft extend + + def init_forward_metadata(self, forward_batch: ForwardBatch): + if forward_batch.forward_mode.is_decode_or_idle(): + self.indices_updater_decode.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_cpu, + forward_batch.seq_lens_sum, + decode_wrappers=self.decode_wrappers, + encoder_lens=forward_batch.encoder_lens, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = DecodeMetadata(self.decode_wrappers) + elif forward_batch.forward_mode.is_draft_extend(): + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_cpu, + forward_batch.seq_lens_sum, + prefix_lens=None, + prefill_wrappers=self.prefill_wrappers_paged, + use_ragged=False, + encoder_lens=forward_batch.encoder_lens, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = PrefillMetadata( + self.prefill_wrappers_paged, False, False + ) + elif forward_batch.forward_mode.is_target_verify(): + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_cpu, + forward_batch.seq_lens_sum, + prefix_lens=None, + prefill_wrappers=self.prefill_wrappers_verify, + use_ragged=False, + encoder_lens=forward_batch.encoder_lens, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = PrefillMetadata( + self.prefill_wrappers_verify, False, False + ) + else: + prefix_lens = forward_batch.extend_prefix_lens + + if self.is_multimodal: + use_ragged = False + extend_no_prefix = False + else: + use_ragged = True + extend_no_prefix = not any(forward_batch.extend_prefix_lens_cpu) + + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_cpu, + forward_batch.seq_lens_sum, + prefix_lens, + prefill_wrappers=self.prefill_wrappers_paged, + use_ragged=use_ragged, + encoder_lens=forward_batch.encoder_lens, + spec_info=None, + ) + self.forward_metadata = PrefillMetadata( + self.prefill_wrappers_paged, use_ragged, extend_no_prefix + ) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + if kv_indices_buf is None: + cuda_graph_kv_indices = torch.zeros( + (max_num_tokens * self.max_context_len,), + dtype=torch.int32, + device="cuda", + ) + else: + cuda_graph_kv_indices = kv_indices_buf + + self.cuda_graph_kv_indices = [cuda_graph_kv_indices] + [ + cuda_graph_kv_indices.clone() for _ in range(self.num_wrappers - 1) + ] + + # Ensure tensors are properly allocated + for i in range(self.num_wrappers): + # Force allocation by performing a small operation + if len(self.cuda_graph_kv_indices[i]) > 0: + self.cuda_graph_kv_indices[i][0] = 0 + + if not self.skip_prefill: + self.cuda_graph_custom_mask = torch.zeros( + (max_num_tokens * self.max_context_len), + dtype=torch.uint8, + device="cuda", + ) + self.cuda_graph_qk_indptr = [x.clone() for x in self.kv_indptr] + self.cuda_graph_qo_indptr = [x.clone() for x in self.kv_indptr] + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + if forward_mode.is_decode_or_idle(): + decode_wrappers = [] + for i in range(self.num_wrappers): + decode_wrappers.append( + BatchDecodeWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + use_cuda_graph=True, + use_tensor_cores=self.decode_use_tensor_cores, + paged_kv_indptr_buffer=self.kv_indptr[i][: num_tokens + 1], + paged_kv_indices_buffer=self.cuda_graph_kv_indices[i], + paged_kv_last_page_len_buffer=self.kv_last_page_len[ + :num_tokens + ], + ) + ) + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_decode.update( + req_pool_indices, + seq_lens, + seq_lens.cpu(), # may add a little overhead in capture stage + seq_lens_sum, + decode_wrappers=decode_wrappers, + encoder_lens=encoder_lens, + spec_info=spec_info, + ) + self.decode_cuda_graph_metadata[bs] = decode_wrappers + self.forward_metadata = DecodeMetadata(decode_wrappers) + for i in range(self.num_wrappers): + decode_wrappers[i].begin_forward = partial( + fast_decode_plan, decode_wrappers[i] + ) + elif forward_mode.is_target_verify(): + prefill_wrappers = [] + for i in range(self.num_wrappers): + prefill_wrappers.append( + BatchPrefillWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + use_cuda_graph=True, + qo_indptr_buf=self.cuda_graph_qo_indptr[i][: bs + 1], + paged_kv_indptr_buf=self.kv_indptr[i][: bs + 1], + paged_kv_indices_buf=self.cuda_graph_kv_indices[i], + paged_kv_last_page_len_buf=self.kv_last_page_len[:bs], + custom_mask_buf=self.cuda_graph_custom_mask, + mask_indptr_buf=self.cuda_graph_qk_indptr[i][: bs + 1], + ) + ) + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_prefill.update( + req_pool_indices, + seq_lens, + seq_lens.cpu(), # may add a little overhead in capture stage + seq_lens_sum, + prefix_lens=None, + prefill_wrappers=prefill_wrappers, + use_ragged=False, + encoder_lens=encoder_lens, + spec_info=spec_info, + ) + self.prefill_cuda_graph_metadata[bs] = prefill_wrappers + self.forward_metadata = PrefillMetadata(prefill_wrappers, False, False) + elif forward_mode.is_draft_extend(): + prefill_wrappers = [] + for i in range(self.num_wrappers): + prefill_wrappers.append( + BatchPrefillWithPagedKVCacheWrapper( + self.workspace_buffer, + "NHD", + backend="fa2", + use_cuda_graph=True, + qo_indptr_buf=self.cuda_graph_qo_indptr[i][: bs + 1], + paged_kv_indptr_buf=self.kv_indptr[i][: bs + 1], + paged_kv_indices_buf=self.cuda_graph_kv_indices[i], + paged_kv_last_page_len_buf=self.kv_last_page_len[:bs], + ) + ) + + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_prefill.update( + req_pool_indices, + seq_lens, + seq_lens.cpu(), # may add a little overhead in capture stage + seq_lens_sum, + prefix_lens=None, + prefill_wrappers=prefill_wrappers, + use_ragged=False, + encoder_lens=encoder_lens, + spec_info=spec_info, + ) + self.prefill_cuda_graph_metadata[bs] = prefill_wrappers + self.forward_metadata = PrefillMetadata(prefill_wrappers, False, False) + else: + raise ValueError(f"Invalid mode: {forward_mode=}") + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + if forward_mode.is_decode_or_idle(): + self.indices_updater_decode.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_cpu[:bs] if seq_lens_cpu is not None else None, + seq_lens_sum, + decode_wrappers=self.decode_cuda_graph_metadata[bs], + encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, + spec_info=spec_info, + ) + elif forward_mode.is_target_verify(): + self.indices_updater_prefill.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_cpu[:bs] if seq_lens_cpu is not None else None, + seq_lens_sum, + prefix_lens=None, + prefill_wrappers=self.prefill_cuda_graph_metadata[bs], + use_ragged=False, + encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, + spec_info=spec_info, + ) + elif forward_mode.is_draft_extend(): + self.indices_updater_prefill.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_cpu[:bs] if seq_lens_cpu is not None else None, + seq_lens_sum, + prefix_lens=None, + prefill_wrappers=self.prefill_cuda_graph_metadata[bs], + use_ragged=False, + encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None, + spec_info=spec_info, + ) + else: + raise ValueError("Invalid forward mode") + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + prefill_wrapper_paged = self.forward_metadata.prefill_wrappers[ + self._get_wrapper_idx(layer) + ] + cache_loc = ( + forward_batch.out_cache_loc + if not layer.is_cross_attention + else forward_batch.encoder_out_cache_loc + ) + + logits_soft_cap = layer.logit_cap + + q = q.contiguous() + if not self.forward_metadata.use_ragged: + if k is not None: + assert v is not None + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + + o = prefill_wrapper_paged.forward( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + causal=not layer.is_cross_attention, + sm_scale=layer.scaling, + window_left=layer.sliding_window_size, + logits_soft_cap=logits_soft_cap, + # Must use _float to avoid device-to-host copy that breaks cuda graph capture. + k_scale=layer.k_scale_float, + v_scale=layer.v_scale_float, + ) + else: + causal = True + if layer.attn_type == AttentionType.ENCODER_ONLY: + save_kv_cache = False + causal = False + + if self.forward_metadata.extend_no_prefix: + # NOTE: FlashInfer currently has limitations with head_dim = 32 or other dimensions + # The FlashInfer head_dim limitation itself is tracked here: + # https://github.com/flashinfer-ai/flashinfer/issues/1048 + o = self.prefill_wrapper_ragged.forward( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + k.view(-1, layer.tp_k_head_num, layer.head_dim), + v.view(-1, layer.tp_v_head_num, layer.head_dim), + causal=causal, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + + else: + o1, s1 = self.prefill_wrapper_ragged.forward_return_lse( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + k.view(-1, layer.tp_k_head_num, layer.head_dim), + v.view(-1, layer.tp_v_head_num, layer.head_dim), + causal=True, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + o2, s2 = prefill_wrapper_paged.forward_return_lse( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + causal=False, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + + o, _ = merge_state(o1, s1, o2, s2) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + + return o.view(-1, layer.tp_q_head_num * layer.head_dim) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + decode_wrapper = self.forward_metadata.decode_wrappers[ + self._get_wrapper_idx(layer) + ] + cache_loc = ( + forward_batch.out_cache_loc + if not layer.is_cross_attention + else forward_batch.encoder_out_cache_loc + ) + + if k is not None: + assert v is not None + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + + # Call the wrapped function + o = decode_wrapper.forward( + q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim), + forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id), + sm_scale=layer.scaling, + logits_soft_cap=layer.logit_cap, + # Must use _float to avoid device-to-host copy that breaks cuda graph capture. + k_scale=layer.k_scale_float, + v_scale=layer.v_scale_float, + ) + + return o.view(-1, layer.tp_q_head_num * layer.head_dim) + + def _get_wrapper_idx(self, layer: RadixAttention): + if self.num_wrappers == 1: + return 0 + + if self.dispatch_reason == WrapperDispatch.SLIDING_WINDOW: + return layer.sliding_window_size == -1 + if self.dispatch_reason == WrapperDispatch.CROSS_ATTENTION: + return layer.is_cross_attention + + raise ValueError(f"Unknown dispatch reason: {self.dispatch_reason}") + + +class FlashInferIndicesUpdaterDecode: + def __init__(self, model_runner: ModelRunner, attn_backend: FlashInferAttnBackend): + # Parse Constants + self.num_qo_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_heads = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.head_dim = model_runner.model_config.head_dim + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.sliding_window_size = model_runner.sliding_window_size + self.attn_backend = attn_backend + + # Buffers and wrappers + self.kv_indptr = attn_backend.kv_indptr + self.kv_last_page_len = attn_backend.kv_last_page_len + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.token_to_kv_pool_allocator = model_runner.token_to_kv_pool_allocator + + # Dispatch the update function + if self.attn_backend.dispatch_reason == WrapperDispatch.SLIDING_WINDOW: + self.update = self.update_sliding_window + elif self.attn_backend.dispatch_reason == WrapperDispatch.CROSS_ATTENTION: + self.update = self.update_cross_attention + else: + assert self.attn_backend.num_wrappers == 1 + self.update = self.update_single_wrapper + + def update( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + decode_wrappers: List[BatchDecodeWithPagedKVCacheWrapper], + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + # Keep the signature for type checking. It will be assigned during runtime. + raise NotImplementedError() + + def update_single_wrapper( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + decode_wrappers: List[BatchDecodeWithPagedKVCacheWrapper], + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + decode_wrappers = decode_wrappers or self.decode_wrappers + self.call_begin_forward( + decode_wrappers[0], + req_pool_indices, + seq_lens, + seq_lens_sum, + self.kv_indptr[0], + None, + spec_info, + seq_lens_cpu, + ) + + def update_sliding_window( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + decode_wrappers: List[BatchDecodeWithPagedKVCacheWrapper], + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + assert self.sliding_window_size is not None + for wrapper_id in range(2): + if wrapper_id == 0: + # Sliding window attention + paged_kernel_lens_tmp = torch.clamp( + seq_lens, max=self.sliding_window_size + 1 + ) + if seq_lens_cpu is not None: + seq_lens_cpu_tmp = torch.clamp( + seq_lens_cpu, max=self.sliding_window_size + 1 + ) + paged_kernel_lens_sum_tmp = seq_lens_cpu_tmp.sum().item() + else: + paged_kernel_lens_sum_tmp = paged_kernel_lens_tmp.sum().item() + kv_start_idx_tmp = seq_lens - paged_kernel_lens_tmp + else: + # Full attention + paged_kernel_lens_tmp = seq_lens + paged_kernel_lens_sum_tmp = seq_lens_sum + seq_lens_cpu_tmp = seq_lens_cpu + kv_start_idx_tmp = None + + use_sliding_window_kv_pool = wrapper_id == 0 and isinstance( + self.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator + ) + + self.call_begin_forward( + decode_wrappers[wrapper_id], + req_pool_indices, + paged_kernel_lens_tmp, + paged_kernel_lens_sum_tmp, + self.kv_indptr[wrapper_id], + kv_start_idx_tmp, + spec_info, + seq_lens_cpu=seq_lens_cpu_tmp, + use_sliding_window_kv_pool=use_sliding_window_kv_pool, + ) + + def update_cross_attention( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + decode_wrappers: List[BatchDecodeWithPagedKVCacheWrapper], + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + for wrapper_id in range(2): + if wrapper_id == 0: + # Normal attention + paged_kernel_lens = seq_lens + kv_start_idx = encoder_lens + else: + # Cross attention + paged_kernel_lens = encoder_lens + kv_start_idx = torch.zeros_like(encoder_lens) + seq_lens_sum = encoder_lens.sum().item() + + self.call_begin_forward( + decode_wrappers[wrapper_id], + req_pool_indices, + paged_kernel_lens, + seq_lens_sum, + self.kv_indptr[wrapper_id], + kv_start_idx, + spec_info, + seq_lens_cpu=seq_lens_cpu, + ) + + def call_begin_forward( + self, + wrapper: BatchDecodeWithPagedKVCacheWrapper, + req_pool_indices: torch.Tensor, + paged_kernel_lens: torch.Tensor, + paged_kernel_lens_sum: int, + kv_indptr: torch.Tensor, + kv_start_idx: torch.Tensor, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + use_sliding_window_kv_pool: bool = False, + ): + if spec_info is None: + bs = len(req_pool_indices) + kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + + if wrapper.is_cuda_graph_enabled: + # Directly write to the cuda graph input buffer + kv_indices = wrapper._paged_kv_indices_buf + else: + kv_indices = torch.empty( + paged_kernel_lens_sum, dtype=torch.int32, device="cuda" + ) + + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + kv_start_idx, + kv_indices, + self.req_to_token.shape[1], + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + bs = kv_indptr.shape[0] - 1 + + if use_sliding_window_kv_pool: + kv_last_index = kv_indptr[-1] + kv_indices[:kv_last_index] = ( + self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + kv_indices[:kv_last_index] + ) + ) + + global global_override_indptr_cpu + locally_override = False + if seq_lens_cpu is not None and global_override_indptr_cpu is None: + locally_override = True + global_override_indptr_cpu = torch.empty_like(kv_indptr, device="cpu") + global_override_indptr_cpu[0] = 0 + global_override_indptr_cpu[1 : bs + 1] = torch.cumsum(seq_lens_cpu, dim=0) + + wrapper.begin_forward( + kv_indptr, + kv_indices, + self.kv_last_page_len[:bs], + self.num_qo_heads, + self.num_kv_heads, + self.head_dim, + 1, + data_type=self.data_type, + q_data_type=self.q_data_type, + non_blocking=True, + ) + + if locally_override: + global_override_indptr_cpu = None + + +class FlashInferIndicesUpdaterPrefill: + def __init__(self, model_runner: ModelRunner, attn_backend: FlashInferAttnBackend): + # Parse Constants + self.num_qo_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_heads = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.head_dim = model_runner.model_config.head_dim + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.sliding_window_size = model_runner.sliding_window_size + self.attn_backend = attn_backend + + # Buffers and wrappers + self.kv_indptr = attn_backend.kv_indptr + self.kv_last_page_len = attn_backend.kv_last_page_len + self.qo_indptr = attn_backend.qo_indptr + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.token_to_kv_pool_allocator = model_runner.token_to_kv_pool_allocator + self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged + + # Dispatch the update function + if self.attn_backend.dispatch_reason == WrapperDispatch.SLIDING_WINDOW: + self.update = self.update_sliding_window + elif self.attn_backend.dispatch_reason == WrapperDispatch.CROSS_ATTENTION: + self.update = self.update_cross_attention + else: + assert self.attn_backend.num_wrappers == 1 + self.update = self.update_single_wrapper + + def update( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + prefix_lens: torch.Tensor, + prefill_wrappers: List[BatchPrefillWithPagedKVCacheWrapper], + use_ragged: bool, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + # Keep the signature for type checking. It will be assigned during runtime. + raise NotImplementedError() + + def update_single_wrapper( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + prefix_lens: torch.Tensor, + prefill_wrappers: List[BatchPrefillWithPagedKVCacheWrapper], + use_ragged: bool, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + if use_ragged: + # TODO: remove this device sync, we can use forward_batch.extend_prefix_lens_cpu + # and forward_batch.extend_seq_lens_cpu + paged_kernel_lens = prefix_lens + paged_kernel_lens_sum = paged_kernel_lens.sum().item() + else: + paged_kernel_lens = seq_lens + paged_kernel_lens_sum = seq_lens_sum + + self.call_begin_forward( + self.prefill_wrapper_ragged, + prefill_wrappers[0], + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + seq_lens, + prefix_lens, + None, + self.kv_indptr[0], + self.qo_indptr[0], + use_ragged, + spec_info, + ) + + def update_sliding_window( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + prefix_lens: torch.Tensor, + prefill_wrappers: List[BatchPrefillWithPagedKVCacheWrapper], + use_ragged: bool, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + for wrapper_id in range(2): + if wrapper_id == 0: + # window attention use paged only + paged_kernel_lens = torch.minimum( + seq_lens, + torch.tensor(self.sliding_window_size) + seq_lens - prefix_lens, + ) + paged_kernel_lens_sum = paged_kernel_lens.sum().item() + else: + # full attention + paged_kernel_lens = seq_lens + paged_kernel_lens_sum = seq_lens_sum + + kv_start_idx = seq_lens - paged_kernel_lens + use_sliding_window_kv_pool = wrapper_id == 0 and isinstance( + self.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator + ) + + self.call_begin_forward( + self.prefill_wrapper_ragged, + prefill_wrappers[wrapper_id], + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + seq_lens, + prefix_lens, + kv_start_idx, + self.kv_indptr[wrapper_id], + self.qo_indptr[wrapper_id], + use_ragged, + spec_info, + use_sliding_window_kv_pool=use_sliding_window_kv_pool, + ) + + def update_cross_attention( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_cpu: Optional[torch.Tensor], + seq_lens_sum: int, + prefix_lens: torch.Tensor, + prefill_wrappers: List[BatchPrefillWithPagedKVCacheWrapper], + use_ragged: bool, + encoder_lens: Optional[torch.Tensor], + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + for wrapper_id in range(2): + if wrapper_id == 0: + # normal attention + paged_kernel_lens = seq_lens + kv_start_idx = encoder_lens + paged_kernel_lens_sum = seq_lens_sum + else: + # cross attention + paged_kernel_lens = encoder_lens + kv_start_idx = torch.zeros_like(encoder_lens) + paged_kernel_lens_sum = paged_kernel_lens.sum().item() + + self.call_begin_forward( + self.prefill_wrapper_ragged, + prefill_wrappers[wrapper_id], + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + seq_lens, + prefix_lens, + kv_start_idx, + self.kv_indptr[wrapper_id], + self.qo_indptr[wrapper_id], + use_ragged, + spec_info, + ) + + def call_begin_forward( + self, + wrapper_ragged: BatchPrefillWithRaggedKVCacheWrapper, + wrapper_paged: BatchPrefillWithPagedKVCacheWrapper, + req_pool_indices: torch.Tensor, + paged_kernel_lens: torch.Tensor, + paged_kernel_lens_sum: int, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + kv_start_idx: torch.Tensor, + kv_indptr: torch.Tensor, + qo_indptr: torch.Tensor, + use_ragged: bool, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + use_sliding_window_kv_pool: bool = False, + ): + bs = len(seq_lens) + if spec_info is None: + assert len(seq_lens) == len(req_pool_indices) + # Normal extend + kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + paged_kernel_lens_sum + 256, + dtype=torch.int32, + device=req_pool_indices.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + kv_start_idx, + kv_indices, + self.req_to_token.shape[1], + ) + qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + custom_mask = None + else: + assert isinstance(spec_info, EagleDraftInput) or isinstance( + spec_info, EagleVerifyInput + ) + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + self.req_to_token, + ) + ) + + # extend part + if use_ragged: + wrapper_ragged.begin_forward( + qo_indptr, + qo_indptr, + self.num_qo_heads, + self.num_kv_heads, + self.head_dim, + q_data_type=self.q_data_type, + ) + + if use_sliding_window_kv_pool: + kv_last_index = kv_indptr[-1] + kv_indices[:kv_last_index] = ( + self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + kv_indices[:kv_last_index] + ) + ) + + # cached part + wrapper_paged.begin_forward( + qo_indptr, + kv_indptr, + kv_indices, + self.kv_last_page_len[:bs], + self.num_qo_heads, + self.num_kv_heads, + self.head_dim, + 1, + q_data_type=self.q_data_type, + kv_data_type=self.data_type, + custom_mask=custom_mask, + non_blocking=True, + ) + + +class FlashInferMultiStepDraftBackend: + """ + Wrap multiple flashinfer attention backends as one for multiple consecutive + draft decoding steps. + """ + + def __init__( + self, + model_runner: ModelRunner, + topk: int, + speculative_num_steps: int, + ): + from sglang.srt.speculative.eagle_utils import generate_draft_decode_kv_indices + + self.topk = topk + self.speculative_num_steps = speculative_num_steps + self.generate_draft_decode_kv_indices = generate_draft_decode_kv_indices + self.page_size = model_runner.page_size + + max_bs = model_runner.req_to_token_pool.size * self.topk + self.kv_indptr = torch.zeros( + ( + self.speculative_num_steps, + max_bs + 1, + ), + dtype=torch.int32, + device=model_runner.device, + ) + self.kv_last_page_len = torch.ones( + (max_bs,), dtype=torch.int32, device=model_runner.device + ) + self.attn_backends: List[FlashInferAttnBackend] = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + FlashInferAttnBackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + kv_last_page_len_buf=self.kv_last_page_len, + ) + ) + + self.max_context_len = self.attn_backends[0].max_context_len + + # Cached variables for generate_draft_decode_kv_indices + self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] + + def common_template( + self, + forward_batch: ForwardBatch, + kv_indices_buffer: torch.Tensor, + call_fn: Callable, + ): + num_seqs = forward_batch.batch_size + bs = self.topk * num_seqs + seq_lens_sum = forward_batch.seq_lens_sum + + self.generate_draft_decode_kv_indices[ + (self.speculative_num_steps, num_seqs, self.topk) + ]( + forward_batch.req_pool_indices, + forward_batch.req_to_token_pool.req_to_token, + forward_batch.seq_lens, + kv_indices_buffer, + self.kv_indptr, + forward_batch.positions, + self.pool_len, + kv_indices_buffer.shape[1], + self.kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(self.speculative_num_steps), + next_power_of_2(bs), + self.page_size, + ) + + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + # Copy the kv_indptr once to avoid multiple device-to-host copies in flashinfer's plan. + indptr_cpu_whole = self.kv_indptr[:, : bs + 1].cpu() + global global_override_indptr_cpu + + for i in range(self.speculative_num_steps - 1): + forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] + forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ + : seq_lens_sum * self.topk + bs * (i + 1) + ] + global_override_indptr_cpu = indptr_cpu_whole[i] + call_fn(i, forward_batch) + + global_override_indptr_cpu = None + + def init_forward_metadata(self, forward_batch: ForwardBatch): + kv_indices = torch.empty( + ( + self.speculative_num_steps, + forward_batch.batch_size * self.topk * self.max_context_len, + ), + dtype=torch.int32, + device="cuda", + ) + + def call_fn(i, forward_batch): + forward_batch.spec_info.kv_indptr = ( + forward_batch.spec_info.kv_indptr.clone() + ) + forward_batch.spec_info.kv_indices = ( + forward_batch.spec_info.kv_indices.clone() + ) + self.attn_backends[i].init_forward_metadata(forward_batch) + + self.common_template(forward_batch, kv_indices, call_fn) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.cuda_graph_kv_indices = torch.zeros( + (self.speculative_num_steps, max_bs * self.max_context_len), + dtype=torch.int32, + device="cuda", + ) + + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state( + max_bs, max_num_tokens, kv_indices_buf=self.cuda_graph_kv_indices[i] + ) + + def init_forward_metadata_capture_cuda_graph(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=-1, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + +def should_use_tensor_core( + kv_cache_dtype: torch.dtype, + num_attention_heads: int, + num_kv_heads: int, +) -> bool: + """ + Determine whether to use tensor cores for attention computation. + + Args: + kv_cache_dtype: Data type of the KV cache + num_attention_heads: Number of attention heads + num_kv_heads: Number of key/value heads + + Returns: + bool: Whether to use tensor cores + """ + # Try to use environment variable first + env_override = os.environ.get("SGLANG_FLASHINFER_USE_TENSOR_CORE") + if env_override is not None: + return env_override.lower() == "true" + + # Try to use _grouped_size_compiled_for_decode_kernels if available + # This is for flashinfer <=0.1.6. Otherwise, there is an accuracy bug + try: + from flashinfer.decode import _grouped_size_compiled_for_decode_kernels + + if not _grouped_size_compiled_for_decode_kernels( + num_attention_heads, + num_kv_heads, + ): + return True + else: + return False + except (ImportError, AttributeError): + pass + + # Calculate GQA group size + gqa_group_size = num_attention_heads // num_kv_heads + + # For Flashinfer, a GQA group size of at least 4 is needed to efficiently + # use Tensor Cores, as it fuses the head group with the token dimension in MMA. + if kv_cache_dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + return True + elif kv_cache_dtype in (torch.float16, torch.half, torch.bfloat16): + return gqa_group_size >= 4 + else: + return False + + +# Use as a fast path to override the indptr in flashinfer's plan function +# This is used to remove some host-to-device copy overhead. +global_override_indptr_cpu = None + + +def fast_decode_plan( + self, + indptr: torch.Tensor, + indices: torch.Tensor, + last_page_len: torch.Tensor, + num_qo_heads: int, + num_kv_heads: int, + head_dim: int, + page_size: int, + pos_encoding_mode: str = "NONE", + window_left: int = -1, + logits_soft_cap: Optional[float] = None, + q_data_type: Optional[Union[str, torch.dtype]] = None, + kv_data_type: Optional[Union[str, torch.dtype]] = None, + data_type: Optional[Union[str, torch.dtype]] = None, + sm_scale: Optional[float] = None, + rope_scale: Optional[float] = None, + rope_theta: Optional[float] = None, + non_blocking: bool = True, +) -> None: + """ + A faster version of BatchDecodeWithPagedKVCacheWrapper::plan used for FlashInferMultiStepDraftBackend. + Modifications: + - Remove unnecessary device-to-device copy for the cuda graph buffers. + - Remove unnecessary host-to-device copy for the metadata buffers. + """ + batch_size = len(last_page_len) + if logits_soft_cap is None: + logits_soft_cap = 0.0 + + # Handle data types consistently + if data_type is not None: + if q_data_type is None: + q_data_type = data_type + if kv_data_type is None: + kv_data_type = data_type + elif q_data_type is None: + q_data_type = "float16" + + if kv_data_type is None: + kv_data_type = q_data_type + + if self.use_tensor_cores: + qo_indptr_host = _get_range_buf(batch_size + 1, "cpu") + + if self.is_cuda_graph_enabled: + if batch_size != self._fixed_batch_size: + raise ValueError( + "The batch size should be fixed in cudagraph mode, the runtime batch size {} " + " mismatches the batch size set during initialization {}".format( + batch_size, self._fixed_batch_size + ) + ) + if len(indices) > len(self._paged_kv_indices_buf): + raise ValueError( + "The size of indices should be less than or equal to the allocated buffer" + ) + else: + self._paged_kv_indptr_buf = indptr + self._paged_kv_indices_buf = indices + self._paged_kv_last_page_len_buf = last_page_len + if self.use_tensor_cores: + self._qo_indptr_buf = qo_indptr_host.to( + self.device, non_blocking=non_blocking + ) + + # Create empty tensors for dtype info if needed + empty_q_data = torch.empty( + 0, + dtype=( + getattr(torch, q_data_type) if isinstance(q_data_type, str) else q_data_type + ), + device=self.device, + ) + + empty_kv_cache = torch.empty( + 0, + dtype=( + getattr(torch, kv_data_type) + if isinstance(kv_data_type, str) + else kv_data_type + ), + device=self.device, + ) + + indptr_host = ( + global_override_indptr_cpu + if global_override_indptr_cpu is not None + else indptr.cpu() + ) + + with torch.cuda.device(self.device): + + if self.use_tensor_cores: + # ALSO convert last_page_len to CPU + if page_size == 1: + # When page size is 1, last_page_len is always 1. + # Directly construct the host tensor rather than executing a device-to-host copy. + last_page_len_host = torch.ones( + (batch_size,), dtype=torch.int32, device="cpu" + ) + else: + last_page_len_host = last_page_len.cpu() + + kv_lens_arr_host = get_seq_lens(indptr_host, last_page_len_host, page_size) + + try: + # Make sure we pass exactly 15 arguments for tensor core version + self._plan_info = self._cached_module.plan( + self._float_workspace_buffer, + self._int_workspace_buffer, + self._pin_memory_int_workspace_buffer, + qo_indptr_host, + indptr_host, + kv_lens_arr_host, + batch_size, # total_num_rows + batch_size, + num_qo_heads, + num_kv_heads, + page_size, + self.is_cuda_graph_enabled, + head_dim, + head_dim, + False, # causal + ) + except Exception as e: + raise RuntimeError(f"Error in standard plan: {e}") + else: + try: + # Make sure we pass exactly 15 arguments for standard version + self._plan_info = self._cached_module.plan( + self._float_workspace_buffer, + self._int_workspace_buffer, + self._pin_memory_int_workspace_buffer, + indptr_host, + batch_size, + num_qo_heads, + num_kv_heads, + page_size, + self.is_cuda_graph_enabled, + window_left, + logits_soft_cap, + head_dim, + head_dim, + empty_q_data, + empty_kv_cache, + ) + except Exception as e: + raise RuntimeError(f"Error in standard plan: {e}") + + self._pos_encoding_mode = pos_encoding_mode + self._window_left = window_left + self._logits_soft_cap = logits_soft_cap + self._sm_scale = sm_scale + self._rope_scale = rope_scale + self._rope_theta = rope_theta diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py new file mode 100644 index 000000000..05e9bef80 --- /dev/null +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -0,0 +1,1079 @@ +from __future__ import annotations + +""" +Support attention backend for flashinfer MLA. +The flashinfer_mla_disable_ragged flag controls whether to use ragged prefill wrapper and defaults to be false. +When it's set to false, all wrappers are BatchMLAPaged wrapper. +When it's set to true, the backend uses BatchRagged and BatchMLAPaged wrapper for prefilling, +and uses BatchMLAPaged wrapper for decoding. +More details can be found in https://docs.flashinfer.ai/api/mla.html +""" + +import os +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING, Callable, Optional, Union + +import torch + +if os.environ["SGLANG_ENABLE_TORCH_COMPILE"] == "1": + import logging + + torch._logging.set_logs(dynamo=logging.ERROR) + torch._dynamo.config.suppress_errors = True + +from sglang.global_config import global_config +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.flashinfer_backend import ( + create_flashinfer_kv_indices_triton, +) +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput +from sglang.srt.utils import ( + is_flashinfer_available, + is_sm100_supported, + next_power_of_2, +) + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + +if is_flashinfer_available(): + from flashinfer import ( + BatchMLAPagedAttentionWrapper, + BatchPrefillWithRaggedKVCacheWrapper, + ) + + +@dataclass +class DecodeMetadata: + decode_wrapper: BatchMLAPagedAttentionWrapper + + +@dataclass +class PrefillMetadata: + prefill_wrapper: BatchMLAPagedAttentionWrapper + use_ragged: bool + + +# Reuse this workspace buffer across all flashinfer wrappers +global_workspace_buffer = None + + +class FlashInferMhaChunkKVRunner: + def __init__( + self, model_runner: ModelRunner, attn_backend: "FlashInferMlaAttnBackend" + ): + # Parse Constants + self.num_local_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.v_head_dim = model_runner.model_config.v_head_dim + self.data_type = model_runner.dtype + self.q_data_type = model_runner.dtype + + # Buffers and wrappers + self.qo_indptr = attn_backend.qo_indptr + self.workspace_buffer = attn_backend.workspace_buffer + self.fmha_backend = attn_backend.fmha_backend + + self.chunk_ragged_wrappers = [] + self.ragged_wrapper = attn_backend.prefill_wrapper_ragged + + def update_prefix_chunks(self, num_prefix_chunks: int): + while num_prefix_chunks > len(self.chunk_ragged_wrappers): + ragged_wrapper = BatchPrefillWithRaggedKVCacheWrapper( + self.workspace_buffer, "NHD", backend=self.fmha_backend + ) + self.chunk_ragged_wrappers.append(ragged_wrapper) + + def update_wrapper( + self, + forward_batch: ForwardBatch, + disable_flashinfer_ragged: bool = False, + ): + assert forward_batch.num_prefix_chunks is not None + num_prefix_chunks = forward_batch.num_prefix_chunks + self.update_prefix_chunks(num_prefix_chunks) + + prefix_lens = forward_batch.extend_prefix_lens + seq_lens = forward_batch.seq_lens + + bs = len(seq_lens) + qo_indptr = self.qo_indptr + qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + + for chunk_idx in range(forward_batch.num_prefix_chunks): + # MHA for chunked prefix kv cache when running model with MLA + assert forward_batch.prefix_chunk_idx is not None + assert forward_batch.prefix_chunk_cu_seq_lens is not None + assert forward_batch.prefix_chunk_max_seq_lens is not None + + kv_indptr = forward_batch.prefix_chunk_cu_seq_lens[chunk_idx] + wrapper = self.chunk_ragged_wrappers[chunk_idx] + wrapper.begin_forward( + qo_indptr=qo_indptr, + kv_indptr=kv_indptr, + num_qo_heads=self.num_local_heads, + num_kv_heads=self.num_local_heads, + head_dim_qk=self.qk_nope_head_dim + self.qk_rope_head_dim, + head_dim_vo=self.v_head_dim, + q_data_type=self.q_data_type, + causal=False, + ) + # ragged prefill + if not disable_flashinfer_ragged: + self.ragged_wrapper.begin_forward( + qo_indptr=qo_indptr, + kv_indptr=qo_indptr, + num_qo_heads=self.num_local_heads, + num_kv_heads=self.num_local_heads, + head_dim_qk=self.qk_nope_head_dim + self.qk_rope_head_dim, + head_dim_vo=self.v_head_dim, + q_data_type=self.q_data_type, + causal=True, + ) + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + ): + logits_soft_cap = layer.logit_cap + if forward_batch.attn_attend_prefix_cache: + chunk_idx = forward_batch.prefix_chunk_idx + assert chunk_idx >= 0 + wrapper = self.chunk_ragged_wrappers[chunk_idx] + o1, s1 = wrapper.forward_return_lse( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + k.view(-1, layer.tp_k_head_num, layer.head_dim).to(q.dtype), + v.view(-1, layer.tp_v_head_num, layer.v_head_dim).to(q.dtype), + causal=False, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + else: + o1, s1 = self.ragged_wrapper.forward_return_lse( + q.view(-1, layer.tp_q_head_num, layer.head_dim), + k.view(-1, layer.tp_k_head_num, layer.head_dim).to(q.dtype), + v.view(-1, layer.tp_v_head_num, layer.v_head_dim).to(q.dtype), + causal=True, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + + return o1, s1 + + +class FlashInferMLAAttnBackend(AttentionBackend): + """Flashinfer attention kernels.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + q_indptr_decode_buf: Optional[torch.Tensor] = None, + ): + super().__init__() + + # Parse constants + self.max_context_len = model_runner.model_config.context_len + self.device = model_runner.device + self.skip_prefill = skip_prefill + self.enable_chunk_kv = ( + not skip_prefill + and global_server_args_dict["disaggregation_mode"] != "decode" + and not global_server_args_dict["disable_chunked_prefix_cache"] + and not global_server_args_dict["flashinfer_mla_disable_ragged"] + ) + self.page_size = model_runner.page_size + + # Allocate buffers + global global_workspace_buffer + if global_workspace_buffer is None: + # different from flashinfer zero_init_global_workspace_buffer + global_workspace_buffer = torch.empty( + global_config.flashinfer_workspace_size, + dtype=torch.uint8, + device=model_runner.device, + ) + self.workspace_buffer = global_workspace_buffer + + max_bs = model_runner.req_to_token_pool.size + if kv_indptr_buf is None: + self.kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + else: + self.kv_indptr = kv_indptr_buf + + if not self.skip_prefill: + self.qo_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + + if q_indptr_decode_buf is None: + self.q_indptr_decode = torch.arange( + 0, max_bs + 1, dtype=torch.int32, device=model_runner.device + ) + else: + self.q_indptr_decode = q_indptr_decode_buf + + self.fmha_backend = "auto" + if is_sm100_supported(): + self.fmha_backend = "cutlass" + self.prefill_wrapper_ragged = BatchPrefillWithRaggedKVCacheWrapper( + self.workspace_buffer, "NHD", backend=self.fmha_backend + ) + + if not self.skip_prefill: + self.prefill_wrapper_paged = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + backend="auto", + ) + + # FlashinferMLA backend uses mla wrapper for target verify + self.prefill_wrapper_verify = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + backend="auto", + ) + + self.decode_wrapper = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, backend="auto" + ) + + # Create indices updater + if not skip_prefill: + self.indices_updater_prefill = FlashInferMLAIndicesUpdaterPrefill( + model_runner, self + ) + if self.enable_chunk_kv: + self.mha_chunk_kv_cache = FlashInferMhaChunkKVRunner(model_runner, self) + + self.indices_updater_decode = FlashInferMLAIndicesUpdaterDecode( + model_runner, self + ) + + # Other metadata + self.forward_metadata: Union[PrefillMetadata, DecodeMetadata] = None + self.decode_cuda_graph_metadata = {} + self.prefill_cuda_graph_metadata = {} # For verify + + def init_forward_metadata(self, forward_batch: ForwardBatch): + if forward_batch.forward_mode.is_decode_or_idle(): + self.indices_updater_decode.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + decode_wrapper=self.decode_wrapper, + init_metadata_replay=False, + ) + self.forward_metadata = DecodeMetadata(self.decode_wrapper) + elif forward_batch.forward_mode.is_draft_extend(): + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=self.prefill_wrapper_paged, + use_ragged=False, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = PrefillMetadata(self.prefill_wrapper_paged, False) + elif forward_batch.forward_mode.is_target_verify(): + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=self.prefill_wrapper_verify, + use_ragged=False, + spec_info=forward_batch.spec_info, + ) + self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False) + else: + prefix_lens = forward_batch.extend_prefix_lens + extend_no_prefix = not any(forward_batch.extend_prefix_lens_cpu) + use_ragged = ( + not global_server_args_dict["flashinfer_mla_disable_ragged"] + and extend_no_prefix + ) + + self.indices_updater_prefill.update( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + prefix_lens, + prefill_wrapper_paged=self.prefill_wrapper_paged, + use_ragged=use_ragged, + ) + self.forward_metadata = PrefillMetadata( + self.prefill_wrapper_paged, use_ragged + ) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + if kv_indices_buf is None: + cuda_graph_kv_indices = torch.zeros( + (max_bs * self.max_context_len,), + dtype=torch.int32, + device="cuda", + ) + else: + cuda_graph_kv_indices = kv_indices_buf + + self.cuda_graph_kv_indices = cuda_graph_kv_indices + self.cuda_graph_qo_indptr = self.q_indptr_decode.clone() + self.cuda_graph_kv_indptr = self.kv_indptr.clone() + self.cuda_graph_kv_lens = torch.ones( + (max_bs,), dtype=torch.int32, device=self.device + ) + + # For fast decode plan in graph replaying + self.cuda_graph_qo_indptr_cpu = self.cuda_graph_qo_indptr.to("cpu") + self.cuda_graph_kv_indptr_cpu = self.cuda_graph_kv_indptr.to("cpu") + self.fast_decode_kwargs = { + "qo_indptr_cpu": self.cuda_graph_qo_indptr_cpu, + "kv_indptr_cpu": self.cuda_graph_kv_indptr_cpu, + "kv_indices": self.cuda_graph_kv_indices, + } + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + if forward_mode.is_decode_or_idle(): + decode_wrapper = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=True, + qo_indptr=self.cuda_graph_qo_indptr[: num_tokens + 1], + kv_indptr=self.cuda_graph_kv_indptr[: num_tokens + 1], + kv_indices=self.cuda_graph_kv_indices, + kv_len_arr=self.cuda_graph_kv_lens[:num_tokens], + backend="auto", + ) + + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_decode.update( + req_pool_indices, + seq_lens, + seq_lens_sum, + decode_wrapper=decode_wrapper, + init_metadata_replay=False, + spec_info=spec_info, + ) + self.decode_cuda_graph_metadata[bs] = decode_wrapper + self.forward_metadata = DecodeMetadata(decode_wrapper) + decode_wrapper.plan = partial(fast_mla_decode_plan, decode_wrapper) + elif forward_mode.is_target_verify(): + verify_wrapper = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=True, + qo_indptr=self.cuda_graph_qo_indptr[: bs + 1], + kv_indptr=self.cuda_graph_kv_indptr[: bs + 1], + kv_indices=self.cuda_graph_kv_indices, + kv_len_arr=self.cuda_graph_kv_lens[:bs], + backend="auto", + ) + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_prefill.update( + req_pool_indices, + seq_lens, + seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=verify_wrapper, + use_ragged=False, + spec_info=spec_info, + ) + self.prefill_cuda_graph_metadata[bs] = verify_wrapper + self.forward_metadata = PrefillMetadata(verify_wrapper, False) + elif forward_mode.is_draft_extend(): + draft_extend_wrapper = BatchMLAPagedAttentionWrapper( + self.workspace_buffer, + use_cuda_graph=True, + qo_indptr=self.cuda_graph_qo_indptr[: bs + 1], + kv_indptr=self.cuda_graph_kv_indptr[: bs + 1], + kv_indices=self.cuda_graph_kv_indices, + kv_len_arr=self.cuda_graph_kv_lens[:bs], + backend="auto", + ) + seq_lens_sum = seq_lens.sum().item() + self.indices_updater_prefill.update( + req_pool_indices, + seq_lens, + seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=draft_extend_wrapper, + use_ragged=False, + spec_info=spec_info, + ) + self.prefill_cuda_graph_metadata[bs] = draft_extend_wrapper + self.forward_metadata = PrefillMetadata(draft_extend_wrapper, False) + else: + raise ValueError(f"Invalid mode: {forward_mode=}") + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + if forward_mode.is_decode_or_idle(): + assert seq_lens_cpu is not None + kv_len_arr_cpu = seq_lens_cpu[:bs] + self.cuda_graph_kv_indptr_cpu[1 : bs + 1] = torch.cumsum( + kv_len_arr_cpu, dim=0 + ) + self.fast_decode_kwargs.update( + { + "qo_indptr_cpu": self.cuda_graph_qo_indptr_cpu[: bs + 1], + "kv_indptr_cpu": self.cuda_graph_kv_indptr_cpu[: bs + 1], + "kv_len_arr_cpu": kv_len_arr_cpu, + } + ) + + self.indices_updater_decode.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_sum, + decode_wrapper=self.decode_cuda_graph_metadata[bs], + init_metadata_replay=True, + spec_info=spec_info, + **self.fast_decode_kwargs, + ) + elif forward_mode.is_target_verify(): + self.indices_updater_prefill.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=self.prefill_cuda_graph_metadata[bs], + use_ragged=False, + spec_info=spec_info, + ) + elif forward_mode.is_draft_extend(): + self.indices_updater_prefill.update( + req_pool_indices[:bs], + seq_lens[:bs], + seq_lens_sum, + prefix_lens=None, + prefill_wrapper_paged=self.prefill_cuda_graph_metadata[bs], + use_ragged=False, + spec_info=spec_info, + ) + else: + raise ValueError(f"Invalid forward mode: {forward_mode=}") + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def init_mha_chunk_metadata( + self, forward_batch: ForwardBatch, disable_flashinfer_ragged: bool = False + ): + """Init the metadata for a forward pass.""" + self.mha_chunk_kv_cache.update_wrapper(forward_batch, disable_flashinfer_ragged) + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ): + if ( + forward_batch.attn_attend_prefix_cache is not None + and forward_batch.mha_return_lse + ): # MHA Chunk + assert self.enable_chunk_kv + assert q_rope is None + assert k_rope is None + o1, s1 = self.mha_chunk_kv_cache.forward(q, k, v, layer, forward_batch) + return o1, s1 + + cache_loc = forward_batch.out_cache_loc + logits_soft_cap = layer.logit_cap + prefill_wrapper_paged = self.forward_metadata.prefill_wrapper + + # Save kv cache + if save_kv_cache and k is not None: + assert v is not None + if save_kv_cache: + if k_rope is not None: + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, cache_loc, k, k_rope + ) + else: + forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + if q_rope is not None: + q = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + + if self.forward_metadata.use_ragged: + # ragged prefill + if q_rope is not None: + q = torch.cat([q, q_rope], dim=-1) + qall = q.view(-1, layer.tp_q_head_num, layer.head_dim) + if k_rope is not None: + k = torch.cat([k, k_rope], dim=-1) + o = self.prefill_wrapper_ragged.forward( + qall, + k.view(-1, layer.tp_k_head_num, layer.head_dim).to(q.dtype), + v.view(-1, layer.tp_k_head_num, layer.v_head_dim).to(q.dtype), + causal=True, + sm_scale=layer.scaling, + logits_soft_cap=logits_soft_cap, + ) + else: + # mla paged prefill + k_buf = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) + if q_rope is None: + qall = q.view(-1, layer.tp_q_head_num, layer.head_dim) + q, q_rope = ( + qall[:, :, : layer.v_head_dim], + qall[:, :, layer.v_head_dim :], + ) + o = q.new_empty(q.shape) + o = prefill_wrapper_paged.run( + q, + q_rope, + k_buf[:, :, : layer.v_head_dim], + k_buf[:, :, layer.v_head_dim :], + out=o, + ) + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + # For multi-head latent attention + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ): + decode_wrapper = self.forward_metadata.decode_wrapper + cache_loc = forward_batch.out_cache_loc + + if k is not None: + assert v is not None + if save_kv_cache: + if k_rope is not None: + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, + cache_loc, + k, + k_rope, + ) + else: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, + cache_loc, + k, + v, + ) + + # Reshape inputs + if q_rope is not None: + q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + else: + reshaped_q = q.view(-1, layer.tp_q_head_num, layer.head_dim) + q_nope = reshaped_q[:, :, : layer.v_head_dim] + q_rope = reshaped_q[:, :, layer.v_head_dim :] + + k_buffer = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id).to( + q.dtype + ) + + o = q_nope.new_empty(q_nope.shape) + # Direct call to run without the wrapper + o = decode_wrapper.run( + q_nope, + q_rope, + k_buffer[:, :, : layer.v_head_dim], + k_buffer[:, :, layer.v_head_dim :], + out=o, + ) + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + +class FlashInferMLAIndicesUpdaterDecode: + def __init__(self, model_runner: ModelRunner, attn_backend: AttentionBackend): + # Parse Constants + self.num_local_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.kv_lora_rank = model_runner.model_config.kv_lora_rank + self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.scaling = model_runner.model_config.scaling + self.data_type = model_runner.dtype + self.attn_backend = attn_backend + + # Buffers and wrappers + self.kv_indptr = attn_backend.kv_indptr + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.q_indptr = attn_backend.q_indptr_decode + + def update( + self, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + decode_wrapper: BatchMLAPagedAttentionWrapper, + init_metadata_replay: bool = False, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]] = None, + **fast_decode_kwargs, + ): + decode_wrapper = decode_wrapper or self.decode_wrapper + self.call_begin_forward( + decode_wrapper, + req_pool_indices, + seq_lens, + seq_lens_sum, + self.q_indptr, + self.kv_indptr, + init_metadata_replay, + spec_info, + **fast_decode_kwargs, + ) + + def call_begin_forward( + self, + wrapper: BatchMLAPagedAttentionWrapper, + req_pool_indices: torch.Tensor, + paged_kernel_lens: torch.Tensor, + paged_kernel_lens_sum: int, + q_indptr: torch.Tensor, + kv_indptr: torch.Tensor, + init_metadata_replay: bool = False, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]] = None, + **fast_decode_kwargs, + ): + bs = len(req_pool_indices) + q_indptr = q_indptr[: bs + 1] + kv_lens = paged_kernel_lens.to(torch.int32) + sm_scale = self.scaling + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = ( + torch.empty(paged_kernel_lens_sum, dtype=torch.int32, device="cuda") + if not init_metadata_replay + else fast_decode_kwargs["kv_indices"] + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.shape[1], + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + + if not init_metadata_replay: + wrapper.plan( + q_indptr, + kv_indptr, + kv_indices, + kv_lens, + self.num_local_heads, + self.kv_lora_rank, + self.qk_rope_head_dim, + 1, + False, + sm_scale, + self.data_type, + self.data_type, + ) + else: + wrapper.plan( + fast_decode_kwargs["qo_indptr_cpu"], + fast_decode_kwargs["kv_indptr_cpu"], + kv_indices, + fast_decode_kwargs["kv_len_arr_cpu"], + self.num_local_heads, + self.kv_lora_rank, + self.qk_rope_head_dim, + 1, + False, + sm_scale, + self.data_type, + self.data_type, + ) + + +class FlashInferMLAIndicesUpdaterPrefill: + def __init__(self, model_runner: ModelRunner, attn_backend: AttentionBackend): + # Parse Constants + self.num_local_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.kv_lora_rank = model_runner.model_config.kv_lora_rank + self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.v_head_dim = model_runner.model_config.v_head_dim + self.scaling = model_runner.model_config.scaling + self.data_type = model_runner.dtype + self.q_data_type = model_runner.dtype + self.attn_backend = attn_backend + + # Buffers and wrappers + self.kv_indptr = attn_backend.kv_indptr + self.qo_indptr = attn_backend.qo_indptr + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.prefill_wrapper_ragged = attn_backend.prefill_wrapper_ragged + + def update( + self, + req_pool_indices: torch.Tnesor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + prefix_lens: torch.Tensor, + prefill_wrapper_paged: BatchMLAPagedAttentionWrapper, + use_ragged: bool, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]] = None, + ): + if use_ragged: + paged_kernel_lens = prefix_lens + paged_kernel_lens_sum = paged_kernel_lens.sum().item() + else: + paged_kernel_lens = seq_lens + paged_kernel_lens_sum = seq_lens_sum + + self.call_begin_forward( + self.prefill_wrapper_ragged, + prefill_wrapper_paged, + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + seq_lens, + prefix_lens, + self.kv_indptr, + self.qo_indptr, + use_ragged, + spec_info, + ) + + def call_begin_forward( + self, + wrapper_ragged: BatchPrefillWithRaggedKVCacheWrapper, + wrapper_paged: BatchMLAPagedAttentionWrapper, + req_pool_indices: torch.Tensor, + paged_kernel_lens: torch.Tensor, + paged_kernel_lens_sum: int, + seq_lens: torch.Tensor, + prefix_lens: torch.Tensor, + kv_indptr: torch.Tensor, + qo_indptr: torch.Tensor, + use_ragged: bool, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]] = None, + ): + bs = len(seq_lens) + sm_scale = self.scaling + + if spec_info is None: + assert len(seq_lens) == len(req_pool_indices) + kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + paged_kernel_lens_sum, + dtype=torch.int32, + device=req_pool_indices.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.shape[1], + ) + qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + custom_mask = None + else: + assert isinstance(spec_info, EagleDraftInput) or isinstance( + spec_info, EagleVerifyInput + ) + # TODO: Support topk > 1 with custom mask + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + req_pool_indices, + paged_kernel_lens, + paged_kernel_lens_sum, + self.req_to_token, + ) + ) + + if use_ragged: + # ragged prefill + wrapper_ragged.begin_forward( + qo_indptr=qo_indptr, + kv_indptr=qo_indptr, + num_qo_heads=self.num_local_heads, + num_kv_heads=self.num_local_heads, + head_dim_qk=self.qk_nope_head_dim + self.qk_rope_head_dim, + head_dim_vo=self.v_head_dim, + q_data_type=self.q_data_type, + causal=True, + ) + else: + # mla paged prefill + kv_len_arr = kv_indptr[1:] - kv_indptr[:-1] + wrapper_paged.plan( + qo_indptr, + kv_indptr, + kv_indices, + kv_len_arr, + self.num_local_heads, + self.kv_lora_rank, + self.qk_rope_head_dim, + 1, + True, + sm_scale, + self.q_data_type, + self.data_type, + ) + + +class FlashInferMLAMultiStepDraftBackend: + """ + Wrap multiple flashinfer mla attention backends as one for multiple consecutive + draft decoding steps. + """ + + def __init__( + self, + model_runner: ModelRunner, + topk: int, + speculative_num_steps: int, + ): + from sglang.srt.speculative.eagle_utils import generate_draft_decode_kv_indices + + if topk > 1: + raise ValueError( + "Currently Flashinfer MLA only supports topk=1 for speculative decoding" + ) + self.topk = topk + self.speculative_num_steps = speculative_num_steps + self.generate_draft_decode_kv_indices = generate_draft_decode_kv_indices + + max_bs = model_runner.req_to_token_pool.size * self.topk + self.kv_indptr = torch.zeros( + ( + self.speculative_num_steps, + max_bs + 1, + ), + dtype=torch.int32, + device=model_runner.device, + ) + self.q_indptr_decode = torch.arange( + 0, max_bs + 1, dtype=torch.int32, device=model_runner.device + ) + + self.attn_backends = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + FlashInferMLAAttnBackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + q_indptr_decode_buf=self.q_indptr_decode, + ) + ) + + self.max_context_len = self.attn_backends[0].max_context_len + + # Cached variables for generate_draft_decode_kv_indices + self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] + self.page_size = model_runner.server_args.page_size + + def common_template( + self, + forward_batch: ForwardBatch, + kv_indices_buffer: torch.Tensor, + call_fn: Callable, + ): + num_seqs = forward_batch.batch_size + bs = self.topk * num_seqs + seq_lens_sum = forward_batch.seq_lens_sum + + self.generate_draft_decode_kv_indices[ + (self.speculative_num_steps, num_seqs, self.topk) + ]( + forward_batch.req_pool_indices, + forward_batch.req_to_token_pool.req_to_token, + forward_batch.seq_lens, + kv_indices_buffer, + self.kv_indptr, + forward_batch.positions, + self.pool_len, + kv_indices_buffer.shape[1], + self.kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(self.speculative_num_steps), + next_power_of_2(bs), + self.page_size, + ) + + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + for i in range(self.speculative_num_steps - 1): + forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] + forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ + : seq_lens_sum * self.topk + bs * (i + 1) + ] + call_fn(i, forward_batch) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + kv_indices = torch.zeros( + ( + self.speculative_num_steps, + forward_batch.batch_size * self.topk * self.max_context_len, + ), + dtype=torch.int32, + device="cuda", + ) + + def call_fn(i, forward_batch): + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + forward_batch.spec_info.kv_indptr = ( + forward_batch.spec_info.kv_indptr.clone() + ) + forward_batch.spec_info.kv_indices = ( + forward_batch.spec_info.kv_indices.clone() + ) + self.attn_backends[i].init_forward_metadata(forward_batch) + + self.common_template(forward_batch, kv_indices, call_fn) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.cuda_graph_kv_indices = torch.zeros( + (self.speculative_num_steps, max_bs * self.max_context_len), + dtype=torch.int32, + device="cuda", + ) + + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state( + max_bs, max_num_tokens, kv_indices_buf=self.cuda_graph_kv_indices[i] + ) + + def init_forward_metadata_capture_cuda_graph(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=-1, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + +def fast_mla_decode_plan( + self, + qo_indptr_cpu: torch.Tensor, + kv_indptr_cpu: torch.Tensor, + kv_indices: torch.Tensor, + kv_len_arr_cpu: torch.Tensor, + num_heads: int, + head_dim_ckv: int, + head_dim_kpe: int, + page_size: int, + causal: bool, + sm_scale: float, + q_data_type: torch.dtype, + kv_data_type: torch.dtype, +) -> None: + """A faster version of BatchMLAPagedAttentionWrapper::plan, + for skipping the stream synchronization in original plan function during + cuda graph replaying. + """ + self._causal = causal + self._page_size = page_size + self._sm_scale = sm_scale + + try: + # Standard version with just the required arguments (no use_profiler) + self._cached_module.plan.default( + self._float_workspace_buffer, + self._int_workspace_buffer, + self._pin_memory_int_workspace_buffer, + qo_indptr_cpu, + kv_indptr_cpu, + kv_len_arr_cpu, + num_heads, + head_dim_ckv, + causal, + ) + except Exception as e: + raise RuntimeError(f"Error in alternate MLA plan: {e}") diff --git a/python/sglang/srt/layers/attention/flashmla_backend.py b/python/sglang/srt/layers/attention/flashmla_backend.py new file mode 100644 index 000000000..d1acb1a58 --- /dev/null +++ b/python/sglang/srt/layers/attention/flashmla_backend.py @@ -0,0 +1,541 @@ +from __future__ import annotations + +""" +Support attention backend for FlashMLA. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Optional, Tuple, Union + +import torch +import triton +from flash_mla import flash_mla_with_kvcache, get_mla_metadata + +from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend +from sglang.srt.layers.attention.utils import create_flashmla_kv_indices_triton +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + + +# FlashMLA only supports pagesize=64 +PAGE_SIZE = 64 + +# FlashMLA FP8 issue: https://github.com/deepseek-ai/FlashMLA/issues/56 + + +@dataclass +class FlashMLADecodeMetadata: + flashmla_metadata: Optional[Tuple[torch.Tensor, torch.Tensor]] = None + num_splits: Optional[torch.Tensor] = None + block_kv_indices: Optional[torch.Tensor] = None + + def __init__( + self, + flashmla_metadata: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + num_splits: Optional[torch.Tensor] = None, + block_kv_indices: Optional[torch.Tensor] = None, + ): + self.flashmla_metadata = flashmla_metadata + self.num_splits = num_splits + self.block_kv_indices = block_kv_indices + + +class FlashMLABackend(FlashInferMLAAttnBackend): + """Flashmla attention kernels.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + kv_last_page_len_buf: Optional[torch.Tensor] = None, + ): + super().__init__( + model_runner, skip_prefill, kv_indptr_buf, kv_last_page_len_buf + ) + + self.num_q_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.num_local_heads = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.forward_metadata: Union[FlashMLADecodeMetadata] = None + self.kv_lora_rank = model_runner.model_config.kv_lora_rank + self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim + self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + self.v_head_dim = model_runner.model_config.v_head_dim + self.scaling = model_runner.model_config.scaling + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.kv_cache_dim = self.kv_lora_rank + self.qk_rope_head_dim + + self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens + + def init_forward_metadata(self, forward_batch: ForwardBatch): + + bs = forward_batch.batch_size + if forward_batch.forward_mode.is_decode_or_idle(): + max_seqlen_pad = triton.cdiv( + forward_batch.seq_lens_cpu.max().item(), PAGE_SIZE + ) + block_kv_indices = torch.full( + (bs, max_seqlen_pad), + -1, + dtype=torch.int32, + device=forward_batch.seq_lens.device, + ) + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + max_seqlen_pad, + ) + mla_metadata, num_splits = get_mla_metadata( + forward_batch.seq_lens.to(torch.int32), + self.num_q_heads, + 1, + ) + self.forward_metadata = FlashMLADecodeMetadata( + mla_metadata, + num_splits, + block_kv_indices, + ) + elif forward_batch.forward_mode.is_target_verify(): + seq_lens_cpu = forward_batch.seq_lens_cpu + self.num_draft_tokens + seq_lens = forward_batch.seq_lens + self.num_draft_tokens + + max_seqlen_pad = triton.cdiv(seq_lens_cpu.max().item(), PAGE_SIZE) + block_kv_indices = torch.full( + (bs, max_seqlen_pad), + -1, + dtype=torch.int32, + device=seq_lens.device, + ) + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + max_seqlen_pad, + ) + mla_metadata, num_splits = get_mla_metadata( + seq_lens.to(torch.int32), + self.num_draft_tokens * self.num_q_heads, + 1, + ) + + # Use FlashMLADecodeMetadata which has the attributes forward_extend expects + self.forward_metadata = FlashMLADecodeMetadata( + mla_metadata, + num_splits, + block_kv_indices, + ) + else: + super().init_forward_metadata(forward_batch) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + block_kv_indices: Optional[torch.Tensor] = None, + ): + if block_kv_indices is None: + cuda_graph_kv_indices = torch.full( + (max_bs, (self.max_context_len + PAGE_SIZE) // PAGE_SIZE), + 1, + dtype=torch.int32, + device="cuda", + ) + else: + cuda_graph_kv_indices = block_kv_indices + + if self.num_draft_tokens: + self.cuda_graph_mla_metadata, self.cuda_graph_num_splits = get_mla_metadata( + torch.ones( + max_bs, dtype=torch.int32, device=cuda_graph_kv_indices.device + ), + self.num_draft_tokens * self.num_q_heads, + 1, + ) + else: + self.cuda_graph_mla_metadata, self.cuda_graph_num_splits = get_mla_metadata( + torch.ones( + max_bs, dtype=torch.int32, device=cuda_graph_kv_indices.device + ), + self.num_q_heads, + 1, + ) + self.cuda_graph_kv_indices = cuda_graph_kv_indices + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + if forward_mode.is_decode_or_idle(): + max_seqlen_pad = triton.cdiv(seq_lens.max().item(), PAGE_SIZE) + + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + ) + mla_metadata, num_splits = get_mla_metadata( + seq_lens.to(torch.int32), + self.num_q_heads, + 1, + ) + self.cuda_graph_mla_metadata.copy_(mla_metadata) + self.cuda_graph_num_splits[: bs + 1].copy_(num_splits) + self.forward_metadata = FlashMLADecodeMetadata( + self.cuda_graph_mla_metadata, + self.cuda_graph_num_splits[: bs + 1], + self.cuda_graph_kv_indices[:bs, :max_seqlen_pad], + ) + elif forward_mode.is_target_verify(): + seq_lens = seq_lens + self.num_draft_tokens + max_seqlen_pad = triton.cdiv(seq_lens.max().item(), PAGE_SIZE) + + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + ) + mla_metadata, num_splits = get_mla_metadata( + seq_lens.to(torch.int32), + self.num_draft_tokens * self.num_q_heads, + 1, + ) + self.cuda_graph_mla_metadata.copy_(mla_metadata) + self.cuda_graph_num_splits[: bs + 1].copy_(num_splits) + self.forward_metadata = FlashMLADecodeMetadata( + self.cuda_graph_mla_metadata, + self.cuda_graph_num_splits[: bs + 1], + self.cuda_graph_kv_indices[:bs, :max_seqlen_pad], + ) + else: + super().init_forward_metadata_capture_cuda_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + encoder_lens, + forward_mode, + spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + + if forward_mode.is_decode_or_idle(): + assert seq_lens_cpu is not None + seq_lens = seq_lens[:bs] + seq_lens_cpu = seq_lens_cpu[:bs] + max_seqlen_pad = triton.cdiv(seq_lens_cpu.max().item(), PAGE_SIZE) + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + ) + mla_metadata, num_splits = get_mla_metadata( + seq_lens.to(torch.int32), + self.num_q_heads, + 1, + ) + self.cuda_graph_mla_metadata.copy_(mla_metadata) + self.cuda_graph_num_splits[: bs + 1].copy_(num_splits) + self.forward_metadata.mla_metadata = self.cuda_graph_mla_metadata + self.forward_metadata.num_splits = self.cuda_graph_num_splits[: bs + 1] + self.forward_metadata.block_kv_indices = self.cuda_graph_kv_indices[ + :bs, :max_seqlen_pad + ] + elif forward_mode.is_target_verify(): + seq_lens = seq_lens[:bs] + self.num_draft_tokens + seq_lens_cpu = seq_lens_cpu[:bs] + self.num_draft_tokens + max_seqlen_pad = triton.cdiv(seq_lens_cpu.max().item(), PAGE_SIZE) + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens, + None, + self.cuda_graph_kv_indices, + self.req_to_token.stride(0), + self.cuda_graph_kv_indices.stride(0), + ) + mla_metadata, num_splits = get_mla_metadata( + seq_lens.to(torch.int32), + self.num_draft_tokens * self.num_q_heads, + 1, + ) + self.cuda_graph_mla_metadata.copy_(mla_metadata) + self.cuda_graph_num_splits[: bs + 1].copy_(num_splits) + self.forward_metadata.mla_metadata = self.cuda_graph_mla_metadata + self.forward_metadata.num_splits = self.cuda_graph_num_splits[: bs + 1] + self.forward_metadata.block_kv_indices = self.cuda_graph_kv_indices[ + :bs, :max_seqlen_pad + ] + else: + super().init_forward_metadata_replay_cuda_graph( + bs, + req_pool_indices, + seq_lens, + seq_lens_sum, + encoder_lens, + forward_mode, + spec_info, + seq_lens_cpu, + ) + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + ): + cache_loc = forward_batch.out_cache_loc + + if k is not None: + assert v is not None + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, + cache_loc, + k, + v, + ) + bs = forward_batch.batch_size + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + + reshape_q = q.view(bs, -1, layer.tp_q_head_num, layer.head_dim) + if self.data_type == torch.float8_e4m3fn: + reshape_q_fp8 = reshape_q.to(torch.float8_e4m3fn) + o, _ = flash_mla_with_kvcache( + q=reshape_q_fp8, + k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + block_table=self.forward_metadata.block_kv_indices[:bs], + cache_seqlens=forward_batch.seq_lens.to(torch.int32), + head_dim_v=self.kv_lora_rank, # TODO Retrieve from config. + tile_scheduler_metadata=self.forward_metadata.flashmla_metadata, + num_splits=self.forward_metadata.num_splits, + softmax_scale=layer.scaling, + causal=True, + descale_q=torch.ones((1), dtype=torch.float32, device=reshape_q.device), + descale_k=torch.ones((1), dtype=torch.float32, device=reshape_q.device), + ) + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + else: + # todo: need check all causal True or False? + o, _ = flash_mla_with_kvcache( + q=reshape_q, + k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + block_table=self.forward_metadata.block_kv_indices[:bs], + cache_seqlens=forward_batch.seq_lens.to(torch.int32), + head_dim_v=self.kv_lora_rank, # TODO Retrieve from config. + tile_scheduler_metadata=self.forward_metadata.flashmla_metadata, + num_splits=self.forward_metadata.num_splits, + softmax_scale=layer.scaling, + causal=True, + ) + + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + ): + if ( + forward_batch.forward_mode == ForwardMode.EXTEND + or forward_batch.forward_mode == ForwardMode.DRAFT_EXTEND + ): + return super().forward_extend(q, k, v, layer, forward_batch, save_kv_cache) + else: + cache_loc = forward_batch.out_cache_loc + + if k is not None: + assert v is not None + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + + bs = forward_batch.batch_size + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + + reshape_q = q.view(bs, -1, layer.tp_q_head_num, layer.head_dim) + if self.data_type == torch.float8_e4m3fn: + reshape_q_fp8 = reshape_q.to(torch.float8_e4m3fn) + o, _ = flash_mla_with_kvcache( + q=reshape_q_fp8, + k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + block_table=self.forward_metadata.block_kv_indices[:bs], + cache_seqlens=forward_batch.seq_lens.to(torch.int32) + + self.num_draft_tokens, + head_dim_v=self.kv_lora_rank, + tile_scheduler_metadata=self.forward_metadata.flashmla_metadata, + num_splits=self.forward_metadata.num_splits, + softmax_scale=layer.scaling, + causal=True, + descale_q=torch.ones( + (1), dtype=torch.float32, device=reshape_q.device + ), + descale_k=torch.ones( + (1), dtype=torch.float32, device=reshape_q.device + ), + ) + else: + o, _ = flash_mla_with_kvcache( + q=reshape_q, + k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim), + block_table=self.forward_metadata.block_kv_indices[:bs], + cache_seqlens=forward_batch.seq_lens.to(torch.int32) + + self.num_draft_tokens, + head_dim_v=self.kv_lora_rank, + tile_scheduler_metadata=self.forward_metadata.flashmla_metadata, + num_splits=self.forward_metadata.num_splits, + softmax_scale=layer.scaling, + causal=True, + ) + return o.view(-1, layer.tp_q_head_num * layer.v_head_dim) + + +# TODO: multi step kv indices optimization +class FlashMLAMultiStepDraftBackend: + """ + Wrap multiple flashmla attention backends as one for multiple consecutive + draft decoding steps. + """ + + def __init__( + self, + model_runner: ModelRunner, + topk: int, + speculative_num_steps: int, + ): + if topk > 1: + raise ValueError( + "Currently FlashMLA only supports topk=1 for speculative decoding" + ) + self.topk = topk + self.speculative_num_steps = speculative_num_steps + max_bs = model_runner.req_to_token_pool.size * self.topk + self.kv_indptr = torch.zeros( + ( + self.speculative_num_steps, + max_bs + 1, + ), + dtype=torch.int32, + device=model_runner.device, + ) + + self.attn_backends = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + FlashMLABackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + kv_last_page_len_buf=None, + ) + ) + + def common_template( + self, + forward_batch: ForwardBatch, + call_fn: Callable, + ): + assert forward_batch.spec_info is not None + + for i in range(self.speculative_num_steps - 1): + call_fn(i, forward_batch) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + assert forward_batch.spec_info is not None + self.attn_backends[i].init_forward_metadata(forward_batch) + + self.common_template(forward_batch, call_fn) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state( + max_bs, max_num_tokens, block_kv_indices=None + ) + + def init_forward_metadata_capture_cuda_graph(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + self.common_template(forward_batch, call_fn) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=-1, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + ) + + self.common_template(forward_batch, call_fn) diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py new file mode 100644 index 000000000..580a977ec --- /dev/null +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -0,0 +1,139 @@ +from typing import Optional, Union + +import torch + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + + +class HybridAttnBackend(AttentionBackend): + """Support different backends for prefill and decode.""" + + def __init__( + self, + model_runner: ModelRunner, + prefill_backend: AttentionBackend, + decode_backend: AttentionBackend, + ): + self.model_runner = model_runner + self.prefill_backend = prefill_backend + self.decode_backend = decode_backend + + def _select_backend(self, forward_mode: ForwardMode) -> AttentionBackend: + """ + Select the appropriate attention backend based on the forward mode. + + Args: + forward_mode: The current forward mode indicating the operation type + + Returns: + The selected attention backend (prefill or decode) + + Note: + - decode_or_idle: Always uses decode backend + - target_verify or draft_extend: Uses decode backend if speculative_attention_mode is "decode", otherwise prefill backend + - prefill: Always uses prefill backend + """ + if forward_mode.is_decode_or_idle(): + return self.decode_backend + elif forward_mode.is_target_verify() or forward_mode.is_draft_extend(): + return ( + self.decode_backend + if self.model_runner.server_args.speculative_attention_mode == "decode" + else self.prefill_backend + ) + else: + return self.prefill_backend + + def init_forward_metadata(self, forward_batch: ForwardBatch): + backend = self._select_backend(forward_batch.forward_mode) + backend.init_forward_metadata(forward_batch) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.decode_backend.init_cuda_graph_state(max_bs, max_num_tokens) + if ( + self.model_runner.server_args.speculative_algorithm is not None + and self.model_runner.server_args.speculative_attention_mode == "prefill" + ): + # When speculative decoding is enabled, we need to initialize the backend + # that will be used for target_verify. + self.prefill_backend.init_cuda_graph_state(max_bs, max_num_tokens) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + backend = self._select_backend(forward_mode) + backend.init_forward_metadata_capture_cuda_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + encoder_lens, + forward_mode, + spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + backend = self._select_backend(forward_mode) + backend.init_forward_metadata_replay_cuda_graph( + bs, + req_pool_indices, + seq_lens, + seq_lens_sum, + encoder_lens, + forward_mode, + spec_info, + seq_lens_cpu, + ) + + def get_cuda_graph_seq_len_fill_value(self): + return self.decode_backend.get_cuda_graph_seq_len_fill_value() + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + return self.decode_backend.forward_decode( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + backend = self._select_backend(forward_batch.forward_mode) + return backend.forward_extend( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py new file mode 100644 index 000000000..a676573f2 --- /dev/null +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -0,0 +1,584 @@ +from dataclasses import astuple, dataclass +from functools import lru_cache +from typing import Optional, Union + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.fla.chunk import chunk_gated_delta_rule +from sglang.srt.layers.attention.fla.fused_recurrent import ( + fused_recurrent_gated_delta_rule_update, +) +from sglang.srt.layers.attention.fla.fused_sigmoid_gating_recurrent import ( + fused_sigmoid_gating_delta_rule_update, +) +from sglang.srt.layers.attention.mamba.causal_conv1d_triton import ( + causal_conv1d_fn, + causal_conv1d_update, +) +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.models.qwen3_next import Qwen3HybridLinearDecoderLayer, fused_gdn_gating +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + + +@dataclass +class ForwardMetadata: + query_start_loc: Optional[torch.Tensor] + mamba_cache_indices: torch.Tensor + + +class MambaAttnBackend(AttentionBackend): + """Attention backend using Mamba kernel.""" + + def __init__(self, model_runner: ModelRunner): + super().__init__() + self.pad_slot_id = -1 # Default pad slot id + self.device = model_runner.device + self.req_to_token_pool: HybridReqToTokenPool = model_runner.req_to_token_pool + self.forward_metadata: ForwardMetadata = None + self.state_indices_list = [] + self.query_start_loc_list = [] + + @classmethod + @lru_cache(maxsize=128) + def _get_cached_arange(cls, bs: int, device_str: str) -> torch.Tensor: + """Cache torch.arange tensors for common batch sizes to avoid repeated allocation.""" + device = torch.device(device_str) + return torch.arange(0, bs + 1, dtype=torch.int32, device=device) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + bs = forward_batch.batch_size + if forward_batch.forward_mode.is_decode_or_idle(): + query_start_loc = self._get_cached_arange(bs, str(self.device)) + elif forward_batch.forward_mode.is_extend(): + if forward_batch.forward_mode.is_target_verify(): + query_start_loc = torch.arange( + 0, + forward_batch.input_ids.shape[0] + 1, + step=forward_batch.spec_info.draft_token_num, + dtype=torch.int32, + device=forward_batch.input_ids.device, + ) + else: + query_start_loc = torch.empty( + (bs + 1,), dtype=torch.int32, device=self.device + ) + query_start_loc[:bs] = forward_batch.extend_start_loc + query_start_loc[bs] = ( + forward_batch.extend_start_loc[-1] + + forward_batch.extend_seq_lens[-1] + ) + else: + raise ValueError(f"Invalid forward mode: {forward_batch.forward_mode=}") + mamba_cache_indices = self.req_to_token_pool.get_mamba_indices( + forward_batch.req_pool_indices + ) + self.forward_metadata = ForwardMetadata( + query_start_loc=query_start_loc, + mamba_cache_indices=mamba_cache_indices, + ) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + for i in range(max_bs): + self.state_indices_list.append( + torch.full((i + 1,), self.pad_slot_id, dtype=torch.int32, device="cuda") + ) + self.query_start_loc_list.append( + torch.empty((i + 2,), dtype=torch.int32, device="cuda") + ) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + if forward_mode.is_decode_or_idle(): + self.query_start_loc_list[bs - 1].copy_(self._get_cached_arange(bs, "cuda")) + elif forward_mode.is_target_verify(): + self.query_start_loc_list[bs - 1].copy_( + torch.arange( + 0, + bs * spec_info.draft_token_num + 1, + step=spec_info.draft_token_num, + dtype=torch.int32, + device="cuda", + ) + ) + else: + raise ValueError(f"Invalid forward mode: {forward_mode=}") + mamba_indices = self.req_to_token_pool.get_mamba_indices(req_pool_indices) + self.state_indices_list[bs - 1][: len(mamba_indices)].copy_(mamba_indices) + self.forward_metadata = ForwardMetadata( + query_start_loc=self.query_start_loc_list[bs - 1], + mamba_cache_indices=self.state_indices_list[bs - 1], + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + num_padding = torch.count_nonzero( + seq_lens_cpu == self.get_cuda_graph_seq_len_fill_value() + ) + # Make sure forward metadata is correctly handled for padding reqs + req_pool_indices[bs - num_padding :] = 0 + mamba_indices = self.req_to_token_pool.get_mamba_indices(req_pool_indices) + mamba_indices[bs - num_padding :] = -1 + self.state_indices_list[bs - 1][: len(mamba_indices)].copy_(mamba_indices) + if forward_mode.is_decode_or_idle(): + self.query_start_loc_list[bs - 1].copy_(self._get_cached_arange(bs, "cuda")) + if num_padding > 0: + self.query_start_loc_list[bs - 1][bs - num_padding :] = bs - num_padding + elif forward_mode.is_target_verify(): + self.query_start_loc_list[bs - 1].copy_( + torch.arange( + 0, + bs * spec_info.draft_token_num + 1, + step=spec_info.draft_token_num, + dtype=torch.int32, + device="cuda", + ) + ) + if num_padding > 0: + self.query_start_loc_list[bs - 1][bs - num_padding :] = ( + bs - num_padding + ) * spec_info.draft_token_num + else: + raise ValueError(f"Invalid forward mode: {forward_mode=}") + + self.forward_metadata = ForwardMetadata( + query_start_loc=self.query_start_loc_list[bs - 1], + mamba_cache_indices=self.state_indices_list[bs - 1], + ) + + def get_cuda_graph_seq_len_fill_value(self): + return 1 # Mamba attn does not use seq lens to index kv cache + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + mixed_qkv = kwargs["mixed_qkv"] + conv_weights = kwargs["conv_weights"] + bias = kwargs["bias"] + activation = kwargs["activation"] + key_dim = kwargs["key_dim"] + value_dim = kwargs["value_dim"] + attn_tp_size = kwargs["attention_tp_size"] + head_k_dim = kwargs["head_k_dim"] + head_v_dim = kwargs["head_v_dim"] + a = kwargs["a"] + b = kwargs["b"] + A_log = kwargs["A_log"] + dt_bias = kwargs["dt_bias"] + layer_id = kwargs["layer_id"] + + conv_states, ssm_states, *rest = self.req_to_token_pool.get_mamba_params( + layer_id + ) + query_start_loc = self.forward_metadata.query_start_loc + cache_indices = self.forward_metadata.mamba_cache_indices + + mixed_qkv = causal_conv1d_update( + mixed_qkv, + conv_states, + conv_weights, + bias, + activation, + conv_state_indices=cache_indices, + ) + + query, key, value = torch.split( + mixed_qkv, + [ + key_dim // attn_tp_size, + key_dim // attn_tp_size, + value_dim // attn_tp_size, + ], + dim=-1, + ) + # Reshape from [l, h*d] to [1, l, h, d] + seq_len = query.shape[0] + num_heads = query.shape[1] // head_k_dim + query = query.view(1, seq_len, num_heads, head_k_dim) + key = key.view(1, seq_len, num_heads, head_k_dim) + value = value.view(1, seq_len, value.shape[1] // head_v_dim, head_v_dim) + + core_attn_out = fused_sigmoid_gating_delta_rule_update( + A_log=A_log, + dt_bias=dt_bias, + q=query, + k=key, + v=value, + a=a, + b=b, + initial_state_source=ssm_states, + initial_state_indices=cache_indices, + cu_seqlens=query_start_loc, + use_qk_l2norm_in_kernel=True, + softplus_beta=1.0, + softplus_threshold=20.0, + ) + + return core_attn_out + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + mixed_qkv = kwargs["mixed_qkv"] + conv_weights = kwargs["conv_weights"] + bias = kwargs["bias"] + activation = kwargs["activation"] + key_dim = kwargs["key_dim"] + value_dim = kwargs["value_dim"] + attn_tp_size = kwargs["attention_tp_size"] + head_k_dim = kwargs["head_k_dim"] + head_v_dim = kwargs["head_v_dim"] + a = kwargs["a"] + b = kwargs["b"] + A_log = kwargs["A_log"] + dt_bias = kwargs["dt_bias"] + layer_id = kwargs["layer_id"] + seq_len = kwargs["seq_len"] + + is_target_verify = forward_batch.forward_mode.is_target_verify() + + query_start_loc = self.forward_metadata.query_start_loc + cache_indices = self.forward_metadata.mamba_cache_indices + + if is_target_verify: + ( + conv_states, + ssm_states, + intermediate_state_cache, + intermediate_conv_window_cache, + ) = self.req_to_token_pool.get_mamba_params(layer_id) + has_initial_states = torch.ones( + seq_len // forward_batch.spec_info.draft_token_num, + dtype=torch.bool, + device=forward_batch.input_ids.device, + ) + conv_states_to_use = conv_states.clone() + else: + conv_states, ssm_states, *rest = self.req_to_token_pool.get_mamba_params( + layer_id + ) + has_initial_states = forward_batch.extend_prefix_lens > 0 + conv_states_to_use = conv_states + + if is_target_verify: + batch_size = seq_len // forward_batch.spec_info.draft_token_num + draft_token_num = forward_batch.spec_info.draft_token_num + mixed_qkv_reshaped = ( + mixed_qkv.view(batch_size, draft_token_num, -1) + .transpose(1, 2) + .contiguous() + ) + mixed_qkv_processed = causal_conv1d_update( + mixed_qkv_reshaped, + conv_states_to_use, + conv_weights, + bias, + activation, + conv_state_indices=cache_indices[:batch_size], + intermediate_conv_window=intermediate_conv_window_cache, + ) + mixed_qkv = ( + mixed_qkv_processed.transpose(1, 2).contiguous().view(seq_len, -1) + ) + else: + mixed_qkv = causal_conv1d_fn( + mixed_qkv.transpose(0, 1), + conv_weights, + bias, + activation=activation, + conv_states=conv_states_to_use, + has_initial_state=has_initial_states, + cache_indices=cache_indices, + query_start_loc=query_start_loc, + ).transpose(0, 1)[:seq_len] + + key_split_dim = key_dim // attn_tp_size + value_split_dim = value_dim // attn_tp_size + + query, key, value = torch.split( + mixed_qkv, + [key_split_dim, key_split_dim, value_split_dim], + dim=-1, + ) + + actual_seq_len = query.shape[0] + num_heads = query.shape[1] // head_k_dim + num_value_heads = value.shape[1] // head_v_dim + + query = query.view(1, actual_seq_len, num_heads, head_k_dim) + key = key.view(1, actual_seq_len, num_heads, head_k_dim) + value = value.view(1, actual_seq_len, num_value_heads, head_v_dim) + + beta = b.sigmoid() + g = fused_gdn_gating(A_log, a, dt_bias) + + g = g.unsqueeze(0) + beta = beta.unsqueeze(0) + + if is_target_verify: + core_attn_out = fused_recurrent_gated_delta_rule_update( + q=query, + k=key, + v=value, + g=g, + beta=beta, + initial_state_source=ssm_states, + initial_state_indices=cache_indices, + cu_seqlens=query_start_loc, + use_qk_l2norm_in_kernel=True, + disable_state_update=True, + intermediate_states_buffer=intermediate_state_cache, + cache_steps=forward_batch.spec_info.draft_token_num, + ) + else: + recurrent_state = ssm_states[cache_indices] + core_attn_out, last_recurrent_state = chunk_gated_delta_rule( + q=query, + k=key, + v=value, + g=g, + beta=beta, + initial_state=recurrent_state, + output_final_state=True, + cu_seqlens=query_start_loc, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + last_recurrent_state = last_recurrent_state.to(ssm_states.dtype, copy=False) + ssm_states[cache_indices] = last_recurrent_state + + return core_attn_out + + +class HybridLinearAttnBackend(AttentionBackend): + """Support different backends for prefill and decode.""" + + def __init__( + self, + full_attn_backend: AttentionBackend, + linear_attn_backend: AttentionBackend, + full_attn_layers: list[int], + ): + self.full_attn_layers = full_attn_layers + self.attn_backend_list = [full_attn_backend, linear_attn_backend] + + def init_forward_metadata(self, forward_batch: ForwardBatch): + for attn_backend in self.attn_backend_list: + attn_backend.init_forward_metadata(forward_batch) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + for attn_backend in self.attn_backend_list: + attn_backend.init_cuda_graph_state(max_bs, max_num_tokens) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + for attn_backend in self.attn_backend_list: + attn_backend.init_forward_metadata_capture_cuda_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + encoder_lens, + forward_mode, + spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + for attn_backend in self.attn_backend_list: + attn_backend.init_forward_metadata_replay_cuda_graph( + bs, + req_pool_indices, + seq_lens, + seq_lens_sum, + encoder_lens, + forward_mode, + spec_info, + seq_lens_cpu, + ) + + def get_cuda_graph_seq_len_fill_value(self): + return self.attn_backend_list[0].get_cuda_graph_seq_len_fill_value() + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + layer_id = layer.layer_id if layer else kwargs["layer_id"] + if layer_id in self.full_attn_layers: + return self.attn_backend_list[0].forward_decode( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) + return self.attn_backend_list[1].forward_decode( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + layer_id = layer.layer_id if layer else kwargs["layer_id"] + if layer_id in self.full_attn_layers: + return self.attn_backend_list[0].forward_extend( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) + return self.attn_backend_list[1].forward_extend( + q, k, v, layer, forward_batch, save_kv_cache, **kwargs + ) + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + """Run forward on an attention layer.""" + if forward_batch.forward_mode.is_idle(): + if layer is None: + return torch.empty_like(kwargs["z"]) + return q.new_empty(q.shape[0], layer.tp_q_head_num * layer.v_head_dim) + elif forward_batch.forward_mode.is_decode(): + return self.forward_decode( + q, + k, + v, + layer, + forward_batch, + save_kv_cache=save_kv_cache, + **kwargs, + ) + else: + return self.forward_extend( + q, + k, + v, + layer, + forward_batch, + save_kv_cache=save_kv_cache, + **kwargs, + ) + + def update_mamba_state_after_mtp_verify(self, accepted_length, model): + request_number = accepted_length.shape[0] + + state_indices_tensor = self.attn_backend_list[ + 1 + ].forward_metadata.mamba_cache_indices[:request_number] + + mamba_caches = self.attn_backend_list[ + 1 + ].req_to_token_pool.get_mamba_params_all_layers() + + ( + conv_states, + ssm_states, + intermediate_state_cache, + intermediate_conv_window_cache, + ) = mamba_caches + + # SSM state updates (chunked to reduce peak memory) + valid_mask = accepted_length > 0 + + # Compute common indices once to avoid duplication + last_steps_all = (accepted_length - 1).to(torch.int64) + valid_state_indices = state_indices_tensor[valid_mask].to(torch.int64) + last_steps = last_steps_all[valid_mask].to(torch.int64) + + if valid_state_indices.numel() > 0: + chunk = 256 + num_valid = valid_state_indices.numel() + + # SSM state updates + for i in range(0, num_valid, chunk): + idx = valid_state_indices[i : i + chunk] + steps = last_steps[i : i + chunk] + # per (cache line, step) + for j in range(idx.numel()): + ci = idx[j].item() + st = steps[j].item() + ssm_states[:, ci, :].copy_( + intermediate_state_cache[:, ci, st].to( + ssm_states.dtype, copy=False + ) + ) + + # Conv window updates + for i in range(0, num_valid, chunk): + idx = valid_state_indices[i : i + chunk] + steps = last_steps[i : i + chunk] + for j in range(idx.numel()): + ci = idx[j].item() + st = steps[j].item() + conv_states[:, ci, :, :].copy_( + intermediate_conv_window_cache[:, ci, st].to( + conv_states.dtype, copy=False + ) + ) diff --git a/python/sglang/srt/layers/attention/intel_amx_backend.py b/python/sglang/srt/layers/attention/intel_amx_backend.py new file mode 100644 index 000000000..39e5c7428 --- /dev/null +++ b/python/sglang/srt/layers/attention/intel_amx_backend.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + + +class IntelAMXAttnBackend(AttentionBackend): + def __init__(self, model_runner: ModelRunner): + import sgl_kernel + + super().__init__() + self.forward_metadata = None + self.device = model_runner.device + + self.num_head = ( + model_runner.model_config.num_attention_heads // model_runner.tp_size + ) + + self.v_head_dim = model_runner.token_to_kv_pool.get_value_buffer(0).shape[-1] + + self.decode_attention_fwd = torch.ops.sgl_kernel.decode_attention_cpu + self.extend_attention_fwd = torch.ops.sgl_kernel.extend_attention_cpu + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init the metadata for a forward pass.""" + + bs = forward_batch.batch_size + attn_logits = torch.zeros( + ( + bs, + self.num_head, + 8, # self.num_kv_splits, + self.v_head_dim + 1, + ), + dtype=torch.float32, + device=self.device, + ) + if forward_batch.forward_mode.is_decode_or_idle(): + max_extend_len = None + else: + max_extend_len = torch.max(forward_batch.extend_seq_lens).item() + self.forward_metadata = (attn_logits, max_extend_len) + + def get_graph_seq_len_fill_value(self): + return 1 + + def forward_extend( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + _, max_extend_len = self.forward_metadata + + self.extend_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + k, + v, + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.extend_seq_lens, + forward_batch.extend_start_loc, + max_extend_len, + layer.scaling, + layer.logit_cap, + ) + return o + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + attn_logits, _ = self.forward_metadata + + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + self.decode_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + k, + v, + forward_batch.out_cache_loc, + attn_logits, + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + layer.scaling, + layer.logit_cap, + ) + + return o + + def support_triton(self): + return False diff --git a/python/sglang/srt/layers/attention/mamba/causal_conv1d.py b/python/sglang/srt/layers/attention/mamba/causal_conv1d.py new file mode 100644 index 000000000..d004337ff --- /dev/null +++ b/python/sglang/srt/layers/attention/mamba/causal_conv1d.py @@ -0,0 +1,128 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/mamba/ops/causal_conv1d.py +# SPDX-License-Identifier: Apache-2.0 + +# Copyright (c) 2024, Tri Dao. +# Adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/causal_conv1d/causal_conv1d_interface.py + +from typing import Optional + +import torch +from sgl_kernel import causal_conv1d_fwd +from sgl_kernel import causal_conv1d_update as causal_conv1d_update_kernel + +PAD_SLOT_ID = -1 + + +def causal_conv1d_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + query_start_loc: Optional[torch.Tensor] = None, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + conv_states: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen + sequences are concatenated from left to right for varlen + weight: (dim, width) + bias: (dim,) + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended by 0. + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + indicates the corresponding state index, + like so: conv_state = conv_states[cache_indices[batch_id]] + has_initial_state: (batch) bool + indicates whether should the kernel take the current state as initial + state for the calculations + conv_states: (...,dim,width - 1) itype + updated inplace if provided + activation: either None or "silu" or "swish" + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + if x.stride(-1) != 1: + x = x.contiguous() + bias = bias.contiguous() if bias is not None else None + + causal_conv1d_fwd( + x, + weight, + bias, + conv_states, + query_start_loc, + cache_indices, + has_initial_state, + activation in ["silu", "swish"], + pad_slot_id, + ) + return x + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + activation: Optional[str] = None, + cache_seqlens: Optional[torch.Tensor] = None, + conv_state_indices: Optional[torch.Tensor] = None, + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the conv_state + starting at the index + @cache_seqlens % state_len. + conv_state_indices: (batch,), dtype int32 + If not None, the conv_state is a larger tensor along the batch dim, + and we are selecting the batch coords specified by conv_state_indices. + Useful for a continuous batching scenario. + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError( + f"activation must be None, silu, or swish, actual: {activation}" + ) + activation_val = activation in ["silu", "swish"] + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + causal_conv1d_update_kernel( + x, + conv_state, + weight, + bias, + activation_val, + cache_seqlens, + conv_state_indices, + pad_slot_id, + ) + if unsqueeze: + x = x.squeeze(-1) + return x diff --git a/python/sglang/srt/layers/attention/mamba/causal_conv1d_triton.py b/python/sglang/srt/layers/attention/mamba/causal_conv1d_triton.py new file mode 100644 index 000000000..3c1bdec48 --- /dev/null +++ b/python/sglang/srt/layers/attention/mamba/causal_conv1d_triton.py @@ -0,0 +1,1052 @@ +# Copyright (c) 2024, Tri Dao. +# Adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/causal_conv1d/causal_conv1d_interface.py +# and https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/mamba/ops/causal_conv1d.py + +from typing import Optional, Union + +import numpy as np +import torch + +PAD_SLOT_ID = -1 +import triton +import triton.language as tl + + +@triton.jit() +def _causal_conv1d_fwd_kernel( # continuous batching + # Pointers to matrices + x_ptr, # (dim, cu_seqlen) holding `batch` of actual sequences + padded sequences + w_ptr, # (dim, width) + bias_ptr, + initial_states_ptr, # conv_states_ptr + cache_indices_ptr, # conv_state_indices_ptr + has_initial_states_ptr, + query_start_loc_ptr, + batch_ptr, + token_chunk_offset_ptr, + o_ptr, # (dim, seqlen) - actually pointing to x_ptr + # Matrix dimensions + batch: tl.int32, # actually padded_batch + dim: tl.constexpr, + seqlen: tl.int32, # cu_seqlen + num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines + # Strides + stride_x_seq: tl.constexpr, # stride to get to next sequence, + stride_x_dim: tl.constexpr, # stride to get to next feature-value, + stride_x_token: tl.constexpr, # stride to get to next token (same feature-index, same sequence-index) + stride_w_dim: tl.constexpr, # stride to get to next dim-axis value + stride_w_width: tl.constexpr, # stride to get to next width-axis value + stride_istate_seq: tl.constexpr, + stride_istate_dim: tl.constexpr, + stride_istate_token: tl.constexpr, + stride_o_seq: tl.constexpr, + stride_o_dim: tl.constexpr, + stride_o_token: tl.constexpr, + # others + pad_slot_id: tl.constexpr, + # Meta-parameters + HAS_BIAS: tl.constexpr, + KERNEL_WIDTH: tl.constexpr, + SILU_ACTIVATION: tl.constexpr, + HAS_INITIAL_STATES: tl.constexpr, + HAS_CACHE: tl.constexpr, + IS_CONTINUOUS_BATCHING: tl.constexpr, + USE_PAD_SLOT: tl.constexpr, + NP2_STATELEN: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + conv_states_ptr = initial_states_ptr + conv_state_indices_ptr = cache_indices_ptr + stride_conv_state_seq = stride_istate_seq + stride_conv_state_dim = stride_istate_dim + stride_conv_state_tok = stride_istate_token + state_len = ( + KERNEL_WIDTH - 1 + ) # can be passed via argument if it's not the same as this value + + # one program handles one chunk in a single sequence + # rather than mixing sequences - to make updating initial_states across sequences efficiently + + # single-sequence id + idx_seq = tl.load(batch_ptr + tl.program_id(0)) + chunk_offset = tl.load(token_chunk_offset_ptr + tl.program_id(0)) + + # BLOCK_N elements along the feature-dimension (channel) + idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + + if idx_seq == pad_slot_id: + return + + sequence_start_index = tl.load(query_start_loc_ptr + idx_seq) + sequence_end_index = tl.load(query_start_loc_ptr + idx_seq + 1) + # find the actual sequence length + seqlen = sequence_end_index - sequence_start_index + + token_offset = BLOCK_M * chunk_offset + segment_len = min(BLOCK_M, seqlen - token_offset) + + # base of the sequence + x_base = ( + x_ptr + sequence_start_index * stride_x_token + idx_feats * stride_x_dim + ) # [BLOCK_N,] + + if IS_CONTINUOUS_BATCHING: + # cache_idx + conv_state_batch_coord = tl.load(conv_state_indices_ptr + idx_seq).to(tl.int64) + else: + # cache_idx + conv_state_batch_coord = idx_seq + if USE_PAD_SLOT: # noqa + if conv_state_batch_coord == pad_slot_id: + # not processing as this is not the actual sequence + return + conv_states_base = ( + conv_states_ptr + + (conv_state_batch_coord * stride_conv_state_seq) + + (idx_feats * stride_conv_state_dim) + ) # [BLOCK_N,] + + w_base = w_ptr + (idx_feats * stride_w_dim) # [BLOCK_N,] + + # Does 2 things: + # 1. READ prior-block init-state data - [done by every Triton programs] + # 2. update conv_state with new data [only by the Triton program handles chunk_offset=0] + if chunk_offset == 0: + # read from conv_states + load_init_state = False + if HAS_INITIAL_STATES: # the new HAS_INITIAL_STATES + load_init_state = tl.load(has_initial_states_ptr + idx_seq).to(tl.int1) + if load_init_state: + # load from conv_states + prior_tokens = conv_states_base + (state_len - 1) * stride_conv_state_tok + mask_w = idx_feats < dim + if KERNEL_WIDTH == 2: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH == 3: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 1 * stride_conv_state_tok # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH == 4: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col2 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 1 * stride_conv_state_tok # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 2 * stride_conv_state_tok # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH == 5: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col3 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 1 * stride_conv_state_tok # [BLOCK_N] + col2 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 2 * stride_conv_state_tok # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0) + conv_states_ptrs = prior_tokens - 3 * stride_conv_state_tok # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0) + else: + # prior-tokens are zeros + if KERNEL_WIDTH >= 2: # STRATEGY1 + # first chunk and does not have prior-token, so just set to 0 + col0 = tl.zeros((BLOCK_N,), dtype=x_ptr.dtype.element_ty) + if KERNEL_WIDTH >= 3: # STRATEGY1 + col1 = tl.zeros((BLOCK_N,), dtype=x_ptr.dtype.element_ty) + if KERNEL_WIDTH >= 4: # STRATEGY1 + col2 = tl.zeros((BLOCK_N,), dtype=x_ptr.dtype.element_ty) + if KERNEL_WIDTH >= 5: # STRATEGY1 + col3 = tl.zeros((BLOCK_N,), dtype=x_ptr.dtype.element_ty) + + # STEP 2: + # here prepare data for updating conv_state + if ( + state_len <= seqlen + ): # SMALL_CACHE=True (only move part of 'x' into conv_state cache) + # just read from 'x' + # copy 'x' data to conv_state + # load only 'x' data (and set 0 before 'x' if seqlen < state_len) + idx_tokens_last = (seqlen - state_len) + tl.arange( + 0, NP2_STATELEN + ) # [BLOCK_M] + x_ptrs = ( + x_ptr + + ((sequence_start_index + idx_tokens_last) * stride_x_token)[:, None] + + (idx_feats * stride_x_dim)[None, :] + ) # [BLOCK_M,BLOCK_N,] + mask_x = ( + (idx_tokens_last >= 0)[:, None] + & (idx_tokens_last < seqlen)[:, None] + & (idx_feats < dim)[None, :] + ) # token-index # token-index # feature-index + loaded_x = tl.load(x_ptrs, mask_x, 0.0) + new_conv_state = tl.load(x_ptrs, mask_x, 0.0) + idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M] + conv_states_ptrs_target = ( + conv_states_base[None, :] + + (idx_tokens_conv * stride_conv_state_tok)[:, None] + ) + + mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[None, :] + tl.debug_barrier() # NOTE: use this due to bug in Triton compiler + tl.store(conv_states_ptrs_target, new_conv_state, mask) + + else: + if load_init_state: + # update conv_state by shifting left, i.e. take last few cols from conv_state + cols from 'x' + idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M] + + conv_states_ptrs_source = ( + conv_states_ptr + + (conv_state_batch_coord * stride_conv_state_seq) + + (idx_feats * stride_conv_state_dim)[None, :] + + ((idx_tokens_conv + seqlen) * stride_conv_state_tok)[:, None] + ) # [BLOCK_M, BLOCK_N] + mask = ( + (conv_state_batch_coord < num_cache_lines) + & ((idx_tokens_conv + seqlen) < state_len)[:, None] + & (idx_feats < dim)[None, :] + ) + conv_state = tl.load(conv_states_ptrs_source, mask, other=0.0) + + VAL = state_len - seqlen + + x_ptrs = ( + x_base[None, :] + + ((idx_tokens_conv - VAL) * stride_x_token)[:, None] + ) # [BLOCK_M, BLOCK_N] + + mask_x = ( + (idx_tokens_conv - VAL >= 0)[:, None] + & (idx_tokens_conv - VAL < seqlen)[:, None] + & (idx_feats < dim)[None, :] + ) # token-index # token-index # feature-index + loaded_x = tl.load(x_ptrs, mask_x, 0.0) + + tl.debug_barrier() # need this due to the bug in tl.where not enforcing this when data is the result of another tl.load + new_conv_state = tl.where( + mask, conv_state, loaded_x + ) # BUG in 'tl.where' which requires a barrier before this + conv_states_ptrs_target = ( + conv_states_base + + (idx_tokens_conv * stride_conv_state_tok)[:, None] + ) # [BLOCK_M, BLOCK_N] + mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[ + None, : + ] + tl.store(conv_states_ptrs_target, new_conv_state, mask) + else: # load_init_state == False + # update conv_state by shifting left, BUT + # set cols prior to 'x' as zeros + cols from 'x' + idx_tokens_conv = tl.arange(0, NP2_STATELEN) # [BLOCK_M] + + VAL = state_len - seqlen + + x_ptrs = ( + x_base[None, :] + + ((idx_tokens_conv - VAL) * stride_x_token)[:, None] + ) # [BLOCK_M, BLOCK_N] + + mask_x = ( + (idx_tokens_conv - VAL >= 0)[:, None] + & (idx_tokens_conv - VAL < seqlen)[:, None] + & (idx_feats < dim)[None, :] + ) # token-index # token-index # feature-index + new_conv_state = tl.load(x_ptrs, mask_x, 0.0) + + conv_states_ptrs_target = ( + conv_states_base + + (idx_tokens_conv * stride_conv_state_tok)[:, None] + ) # [BLOCK_M, BLOCK_N] + mask = (idx_tokens_conv < state_len)[:, None] & (idx_feats < dim)[ + None, : + ] + tl.store(conv_states_ptrs_target, new_conv_state, mask) + + else: # chunk_offset > 0 + # read prior-token data from `x` + load_init_state = True + prior_tokens = x_base + (token_offset - 1) * stride_x_token + mask_w = idx_feats < dim + if KERNEL_WIDTH == 2: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + if KERNEL_WIDTH == 3: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 1 * stride_x_token # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + if KERNEL_WIDTH == 4: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col2 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 1 * stride_x_token # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 2 * stride_x_token # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + if KERNEL_WIDTH == 5: + # ruff: noqa: F841 + conv_states_ptrs = prior_tokens # [BLOCK_N] + col3 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 1 * stride_x_token # [BLOCK_N] + col2 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 2 * stride_x_token # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + conv_states_ptrs = prior_tokens - 3 * stride_x_token # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0, cache_modifier=".ca") + + if HAS_BIAS: + bias = bias_ptr + idx_feats + mask_bias = idx_feats < dim + acc_preload = tl.load(bias, mask=mask_bias, other=0.0).to( + tl.float32 + ) # [BLOCK_N] + else: + acc_preload = tl.zeros((BLOCK_N,), dtype=tl.float32) + + x_base_1d = x_base + token_offset * stride_x_token # starting of chunk + + # PRE-LOAD WEIGHTS + mask_w = idx_feats < dim + if KERNEL_WIDTH >= 2: + w_ptrs = w_base + (0 * stride_w_width) # [BLOCK_N] tensor + w_col0 = tl.load(w_ptrs, mask_w, other=0.0) + w_ptrs = w_base + (1 * stride_w_width) # [BLOCK_N] tensor + w_col1 = tl.load(w_ptrs, mask_w, other=0.0) + if KERNEL_WIDTH >= 3: + w_ptrs = w_base + (2 * stride_w_width) # [BLOCK_N] tensor + w_col2 = tl.load(w_ptrs, mask_w, other=0.0) + if KERNEL_WIDTH >= 4: + w_ptrs = w_base + (3 * stride_w_width) # [BLOCK_N] tensor + w_col3 = tl.load(w_ptrs, mask_w, other=0.0) + mask_x_1d = idx_feats < dim + for idx_token in range(segment_len): + acc = acc_preload + + matrix_w = w_col0 + matrix_x = col0 + for j in tl.static_range(KERNEL_WIDTH): + + if KERNEL_WIDTH == 2: + if j == 1: # KERNEL_WIDTH-1: + matrix_w = w_col1 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + elif KERNEL_WIDTH == 3: + if j == 1: + matrix_w = w_col1 + matrix_x = col1 + elif j == 2: + matrix_w = w_col2 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + elif KERNEL_WIDTH == 4: + if j == 1: + matrix_w = w_col1 + matrix_x = col1 + elif j == 2: + matrix_w = w_col2 + matrix_x = col2 + elif j == 3: + matrix_w = w_col3 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + + acc += matrix_x * matrix_w # [BLOCK_N] + + if KERNEL_WIDTH == 2: + col0 = matrix_x + elif KERNEL_WIDTH == 3: + col0 = col1 + col1 = matrix_x + elif KERNEL_WIDTH == 4: + col0 = col1 + col1 = col2 + col2 = matrix_x + + if SILU_ACTIVATION: + acc = acc / (1 + tl.exp(-acc)) + mask_1d = (idx_token < segment_len) & ( + idx_feats < dim + ) # token-index # feature-index + o_ptrs = ( + o_ptr + + (sequence_start_index + token_offset + idx_token) * stride_o_token + + (idx_feats * stride_o_dim) + ) + + tl.store(o_ptrs, acc, mask=mask_1d) + + +def causal_conv1d_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Union[torch.Tensor, None], + conv_states: torch.Tensor, + query_start_loc: torch.Tensor, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", + pad_slot_id: int = PAD_SLOT_ID, + metadata=None, + validate_data=False, +): + """support varlen + continuous batching when x is 2D tensor + + x: (dim,cu_seq_len) + cu_seq_len = total tokens of all seqs in that batch + sequences are concatenated from left to right for varlen + weight: (dim, width) + conv_states: (...,dim,width - 1) itype + updated inplace if provided + [it use `cache_indices` to get the index to the cache of conv_state for that sequence + + conv_state[cache_indices[i]] for seq-i - to be used as initial_state when has_initial_state[i] = True + and after that conv_state[cache_indices[i]] need to be shift-left and updated with values from 'x' + ] + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended by 0. + if + x = [5, 1, 1, 1] <- continuous batching (batch=4) + then + query_start_loc = [0, 5, 6, 7, 8] <- the starting index of the next sequence; while the last value is + the ending index of the last sequence + [length(query_start_loc)-1 == batch] + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + indicates the corresponding state index, + like so: conv_state = conv_states[cache_indices[batch_id]] + has_initial_state: (batch) bool + indicates whether should the kernel take the current state as initial + state for the calculations + [single boolean for each sequence in the batch: True or False] + bias: (dim,) + activation: either None or "silu" or "swish" or True + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + + out: same shape as `x` + """ + if isinstance(activation, bool) and activation: + activation = "silu" + + args = None + out = torch.empty_like(x) + if metadata is not None: + cu_seqlen = metadata.cu_seqlen + nums_dict = metadata.nums_dict + # x = metadata.x + args = nums_dict + batch_ptr = metadata.batch_ptr + token_chunk_offset_ptr = metadata.token_chunk_offset_ptr + else: + seqlens = np.diff(query_start_loc.to("cpu")) + args = seqlens + MAX_NUM_PROGRAMS = 1024 + + batch_ptr = torch.full( + (MAX_NUM_PROGRAMS,), PAD_SLOT_ID, dtype=torch.int32, device=x.device + ) # tracking which seq-idx the Triton program is handling + token_chunk_offset_ptr = torch.full( + (MAX_NUM_PROGRAMS,), PAD_SLOT_ID, dtype=torch.int32, device=x.device + ) # tracking BLOCK_M-based index in the sequence the Triton program is handling + + is_channel_last = (x.stride(0) == 1) & (x.stride(1) > 1) + dim, cu_seqlen = x.shape + _, width = weight.shape + state_len = width - 1 + np2_statelen = triton.next_power_of_2(state_len) + + padded_batch = query_start_loc.size(0) - 1 + stride_x_seq = 0 + stride_x_dim = x.stride(0) + stride_x_token = x.stride(1) + stride_w_dim = weight.stride(0) + stride_w_width = weight.stride(1) + stride_istate_seq = 0 + stride_istate_dim = 0 + stride_istate_token = 0 + num_cache_lines = 0 + if conv_states is not None: + # extensions to support vLLM: + # 1. conv_states is used to replaced initial_states + # 2. conv_states serve as a cache with num cache lines can be larger than batch size + # 3. mapping from sequence x[idx] to a cache line at index as specified via cache_indices[idx] + # 4. computation can be skipped if cache_indices[idx] == pad_slot_id + num_cache_lines = conv_states.size(0) + assert ( + num_cache_lines == conv_states.shape[0] + and dim == conv_states.shape[1] + and width - 1 <= conv_states.shape[2] + ) + stride_istate_seq = conv_states.stride(0) + stride_istate_dim = conv_states.stride(1) + stride_istate_token = conv_states.stride(2) + # assert stride_istate_dim == 1 + if out.dim() == 2: + stride_o_seq = 0 + stride_o_dim = out.stride(0) + stride_o_token = out.stride(1) + else: + stride_o_seq = out.stride(0) + stride_o_dim = out.stride(1) + stride_o_token = out.stride(2) + + if validate_data: + assert x.dim() == 2 + assert query_start_loc is not None + assert query_start_loc.dim() == 1 + assert x.stride(0) == 1 or x.stride(1) == 1 + if bias is not None: + assert bias.dim() == 1 + assert dim == bias.size(0) + if cache_indices is not None: + assert cache_indices.dim() == 1 + assert padded_batch == cache_indices.size(0) + if has_initial_state is not None: + assert has_initial_state.size() == (padded_batch,) + assert ( + conv_states is not None + ), "ERROR: `has_initial_state` is used, which needs also `conv_states`" + assert weight.stride(1) == 1 + assert (dim, width) == weight.shape + assert is_channel_last, "Need to run in channel-last layout" + + if metadata is None: + + def num_program(META, seqlens): + tot = 0 + + mlist = [] + offsetlist = [] # type: ignore + + nums = -(-seqlens // META["BLOCK_M"]) + + tot = nums.sum().item() + mlist = np.repeat(np.arange(len(nums)), nums) + for idx, num in enumerate(nums): + offsetlist.extend( + range(num) + ) # chunk-idx if a sequence is split into multiple chunks + + if META["batch_ptr"].nelement() < len(mlist): + newlen = len(mlist) + 1 + META["batch_ptr"].resize_(newlen).fill_(PAD_SLOT_ID) + META["token_chunk_offset_ptr"].resize_(newlen).fill_(PAD_SLOT_ID) + + if META["batch_ptr"].nelement() >= len(mlist): + META["batch_ptr"][0 : len(mlist)].copy_( + torch.from_numpy(np.array(mlist)) + ) + META["token_chunk_offset_ptr"][0 : len(mlist)].copy_( + torch.from_numpy(np.array(offsetlist)) + ) + + META["batch_ptr"] = META["batch_ptr"].to(META["x_ptr"].device) + META["token_chunk_offset_ptr"] = META["token_chunk_offset_ptr"].to( + META["x_ptr"].device + ) + return tot + + else: + + def num_program(META, nums_dict): + tot = nums_dict[META["BLOCK_M"]]["tot"] + + mlist = nums_dict[META["BLOCK_M"]]["mlist"] + mlist_len = nums_dict[META["BLOCK_M"]]["mlist_len"] + + offsetlist = nums_dict[META["BLOCK_M"]]["offsetlist"] + + if nums_dict[META["BLOCK_M"]]["batch_ptr"] is not None: + META["batch_ptr"] = nums_dict[META["BLOCK_M"]]["batch_ptr"] + META["token_chunk_offset_ptr"] = nums_dict[META["BLOCK_M"]][ + "token_chunk_offset_ptr" + ] + else: + if META["batch_ptr"].nelement() < mlist_len: + newlen = mlist_len + 1 + META["batch_ptr"].resize_(newlen).fill_(PAD_SLOT_ID) + META["token_chunk_offset_ptr"].resize_(newlen).fill_(PAD_SLOT_ID) + + if META["batch_ptr"].nelement() >= mlist_len: + META["batch_ptr"][0:mlist_len].copy_(mlist) + META["token_chunk_offset_ptr"][0:mlist_len].copy_(offsetlist) + return tot + + def grid(META): + return ( + num_program(META, args), + triton.cdiv(dim, META["BLOCK_N"]), + ) + + if batch_ptr.device != x.device: + batch_ptr = batch_ptr.to(x.device) + token_chunk_offset_ptr = token_chunk_offset_ptr.to(x.device) + + _causal_conv1d_fwd_kernel[grid]( + # Pointers to matrices + x, + weight, + bias, + conv_states, + cache_indices, + has_initial_state, + query_start_loc, + batch_ptr, + token_chunk_offset_ptr, + out, + # Matrix dimensions + padded_batch, + dim, + cu_seqlen, + num_cache_lines, + # stride + stride_x_seq, + stride_x_dim, + stride_x_token, + stride_w_dim, + stride_w_width, + stride_istate_seq, + stride_istate_dim, + stride_istate_token, + stride_o_seq, + stride_o_dim, + stride_o_token, + # others + pad_slot_id, + # META + HAS_BIAS=bias is not None, + KERNEL_WIDTH=width, + SILU_ACTIVATION=activation in ["silu", "swish"], + HAS_INITIAL_STATES=has_initial_state is not None, + HAS_CACHE=conv_states is not None, + IS_CONTINUOUS_BATCHING=cache_indices is not None, + USE_PAD_SLOT=pad_slot_id is not None, + NP2_STATELEN=np2_statelen, + # launch_cooperative_grid=True + BLOCK_M=8, + BLOCK_N=256, + num_stages=2, + ) + return out + + +@triton.jit() +def _causal_conv1d_update_kernel( + # Pointers to matrices + x_ptr, # (batch, dim, seqlen) + w_ptr, # (dim, width) + bias_ptr, + conv_state_ptr, + cache_seqlens_ptr, # circular buffer + conv_state_indices_ptr, + num_accepted_tokens_ptr, + intermediate_conv_window_ptr, + o_ptr, # (batch, dim, seqlen) + # Matrix dimensions + batch: int, + dim: tl.constexpr, + seqlen: tl.constexpr, + state_len: tl.constexpr, + num_cache_lines: tl.constexpr, # added to support vLLM larger cache lines + # Strides + stride_x_seq: tl.constexpr, + stride_x_dim: tl.constexpr, + stride_x_token: tl.constexpr, + stride_w_dim: tl.constexpr, + stride_w_width: tl.constexpr, + stride_conv_state_seq: tl.constexpr, + stride_conv_state_dim: tl.constexpr, + stride_conv_state_tok: tl.constexpr, + stride_state_indices: tl.constexpr, + stride_inter_seq: tl.constexpr, + stride_inter_step: tl.constexpr, + stride_inter_dim: tl.constexpr, + stride_inter_win: tl.constexpr, + stride_o_seq: tl.constexpr, + stride_o_dim: tl.constexpr, + stride_o_token: tl.constexpr, + # others + pad_slot_id: tl.constexpr, + # Meta-parameters + HAS_BIAS: tl.constexpr, + KERNEL_WIDTH: tl.constexpr, + SILU_ACTIVATION: tl.constexpr, + IS_CONTINUOUS_BATCHING: tl.constexpr, + IS_SPEC_DECODING: tl.constexpr, + NP2_STATELEN: tl.constexpr, + USE_PAD_SLOT: tl.constexpr, + BLOCK_N: tl.constexpr, + SAVE_INTERMEDIATE: tl.constexpr, +): + # ruff: noqa: E501 + idx_seq = tl.program_id(0) + if idx_seq >= batch: + return + + # [BLOCK_N,] elements along the feature-dimension (channel) + idx_feats = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N) + + if IS_CONTINUOUS_BATCHING: + # mask = idx_seq < batch + conv_state_batch_coord = tl.load( + conv_state_indices_ptr + idx_seq * stride_state_indices + ).to(tl.int64) + else: + conv_state_batch_coord = idx_seq + if USE_PAD_SLOT: # noqa + if conv_state_batch_coord == pad_slot_id: + # not processing as this is not the actual sequence + return + + if IS_SPEC_DECODING: + # The rolling of conv state: + # + # Before forward, the conv_state is: + # [history1, history2, ..., historyM]. + # + # After forward, the conv_state becomes: + # [history2, ..., historyM, draft1, draft2, ..., draftN]. + # + # After acceptance, it becomes: + # + # - accept 1 tokens: [history2, ..., historyM, draft1] + # - accept 2 tokens: [history3, ..., historyM, draft1, draft2] + # - and so on. + conv_state_token_offset = tl.load(num_accepted_tokens_ptr + idx_seq) - 1 + else: + conv_state_token_offset = 0 + + # STEP 1: READ init_state data + conv_states_base = ( + conv_state_ptr + + (conv_state_batch_coord * stride_conv_state_seq) + + (idx_feats * stride_conv_state_dim) + ) + mask_w = idx_feats < dim + + prior_tokens = conv_states_base + conv_state_token_offset * stride_conv_state_tok + if KERNEL_WIDTH >= 2: + conv_states_ptrs = prior_tokens # [BLOCK_N] + col0 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH >= 3: + conv_states_ptrs = prior_tokens + 1 * stride_conv_state_tok # [BLOCK_N] + col1 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH >= 4: + conv_states_ptrs = prior_tokens + 2 * stride_conv_state_tok # [BLOCK_N] + col2 = tl.load(conv_states_ptrs, mask_w, 0.0) + if KERNEL_WIDTH == 5: + conv_states_ptrs = prior_tokens + 3 * stride_conv_state_tok # [BLOCK_N] + col3 = tl.load(conv_states_ptrs, mask_w, 0.0) + + # STEP 2: assume state_len > seqlen + idx_tokens = tl.arange(0, NP2_STATELEN) # [BLOCK_M] + + # The conv_state updates works in a sliding window manner, + # at each forward pass, the tokens are shift by 1, so we + # load since idx_tokens + 1. + conv_state_ptrs_source = ( + conv_state_ptr + + (conv_state_batch_coord * stride_conv_state_seq) + + conv_state_token_offset * stride_conv_state_tok + + (idx_feats * stride_conv_state_dim)[None, :] + + ((idx_tokens + 1) * stride_conv_state_tok)[:, None] + ) # [BLOCK_M, BLOCK_N] + mask = ( + (conv_state_batch_coord < num_cache_lines) + & ((idx_tokens + seqlen) < state_len)[:, None] + & (idx_feats < dim)[None, :] + ) + conv_state = tl.load(conv_state_ptrs_source, mask, other=0.0) + + VAL = state_len - seqlen + x_base = x_ptr + (idx_seq * stride_x_seq) + (idx_feats * stride_x_dim) # [BLOCK_N] + + x_ptrs = ( + x_base[None, :] + ((idx_tokens - VAL) * stride_x_token)[:, None] + ) # [BLOCK_M, BLOCK_N] + + mask_x = ( + (idx_tokens - VAL >= 0)[:, None] + & (idx_tokens - VAL < seqlen)[:, None] + & (idx_feats < dim)[None, :] + ) # token-index # token-index # feature-index + loaded_x = tl.load(x_ptrs, mask_x, 0.0) + tl.debug_barrier() + + new_conv_state = tl.where(mask, conv_state, loaded_x) + + conv_state_base = ( + conv_state_ptr + + (conv_state_batch_coord * stride_conv_state_seq) + + (idx_feats * stride_conv_state_dim) + ) # [BLOCK_N,] + conv_state_ptrs_target = ( + conv_state_base + (idx_tokens * stride_conv_state_tok)[:, None] + ) # [BLOCK_M, BLOCK_N] + mask = (idx_tokens < state_len)[:, None] & (idx_feats < dim)[None, :] + tl.store(conv_state_ptrs_target, new_conv_state, mask) + + # STEP 3: init accumulator + if HAS_BIAS: + bias = bias_ptr + idx_feats + mask_bias = idx_feats < dim + acc_preload = tl.load(bias, mask=mask_bias, other=0.0).to( + tl.float32 + ) # [BLOCK_N] + else: + acc_preload = tl.zeros((BLOCK_N,), dtype=tl.float32) + + # STEP 4: + # PRE-LOAD WEIGHTS + # first kernel column, configured for weights to handle BLOCK_N features in range + w_base = w_ptr + (idx_feats * stride_w_dim) # [BLOCK_N,] + mask_w = idx_feats < dim + if KERNEL_WIDTH >= 2: + w_ptrs = w_base + (0 * stride_w_width) # [BLOCK_N] tensor + w_col0 = tl.load(w_ptrs, mask_w, other=0.0) + w_ptrs = w_base + (1 * stride_w_width) # [BLOCK_N] tensor + w_col1 = tl.load(w_ptrs, mask_w, other=0.0) + if KERNEL_WIDTH >= 3: + w_ptrs = w_base + (2 * stride_w_width) # [BLOCK_N] tensor + w_col2 = tl.load(w_ptrs, mask_w, other=0.0) + if KERNEL_WIDTH >= 4: + w_ptrs = w_base + (3 * stride_w_width) # [BLOCK_N] tensor + w_col3 = tl.load(w_ptrs, mask_w, other=0.0) + + x_base_1d = x_base # starting of chunk [BLOCK_N] + mask_x_1d = idx_feats < dim + + # STEP 5: compute each token + for idx_token in tl.static_range(seqlen): + acc = acc_preload + + matrix_w = w_col0 + matrix_x = col0 + for j in tl.static_range(KERNEL_WIDTH): + if KERNEL_WIDTH == 2: + if j == 1: # KERNEL_WIDTH-1: + matrix_w = w_col1 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + elif KERNEL_WIDTH == 3: + if j == 1: + matrix_w = w_col1 + matrix_x = col1 + elif j == 2: + matrix_w = w_col2 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + elif KERNEL_WIDTH == 4: + if j == 1: + matrix_w = w_col1 + matrix_x = col1 + elif j == 2: + matrix_w = w_col2 + matrix_x = col2 + elif j == 3: + matrix_w = w_col3 + x_ptrs_1d = x_base_1d + idx_token * stride_x_token # [BLOCK_N] + matrix_x = tl.load(x_ptrs_1d, mask=mask_x_1d) + + acc += matrix_x * matrix_w # [BLOCK_N] + + if KERNEL_WIDTH == 2: + col0 = matrix_x + elif KERNEL_WIDTH == 3: + col0 = col1 + col1 = matrix_x + elif KERNEL_WIDTH == 4: + col0 = col1 + col1 = col2 + col2 = matrix_x + + if SILU_ACTIVATION: + acc = acc / (1 + tl.exp(-acc)) + mask_1d = (idx_token < seqlen) & ( + idx_feats < dim + ) # token-index # feature-index + o_ptrs = ( + o_ptr + + (idx_seq) * stride_o_seq + + idx_token * stride_o_token + + (idx_feats * stride_o_dim) + ) + + tl.store(o_ptrs, acc, mask=mask_1d) + + if SAVE_INTERMEDIATE: + # Save the window state after consuming this token + # Layout: [seq(cache line), step, dim, win(K-1)] + base_ptr = ( + intermediate_conv_window_ptr + + conv_state_batch_coord * stride_inter_seq + + idx_token * stride_inter_step + + idx_feats * stride_inter_dim + ) + if KERNEL_WIDTH >= 2: + tl.store(base_ptr + 0 * stride_inter_win, col0, mask=mask_w) + if KERNEL_WIDTH >= 3: + tl.store(base_ptr + 1 * stride_inter_win, col1, mask=mask_w) + if KERNEL_WIDTH >= 4: + tl.store(base_ptr + 2 * stride_inter_win, col2, mask=mask_w) + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + activation: Union[bool, str, None] = None, + cache_seqlens: Optional[torch.Tensor] = None, + conv_state_indices: Optional[torch.Tensor] = None, + num_accepted_tokens: Optional[torch.Tensor] = None, + intermediate_conv_window: Optional[torch.Tensor] = None, + pad_slot_id: int = PAD_SLOT_ID, + metadata=None, + validate_data=False, +): + """ + x: (batch, dim) or (batch, dim, seqlen) + [shape=2: single token prediction] + [shape=3: single or multiple tokens prediction] + conv_state: (..., dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the conv_state + starting at the index + @cache_seqlens % state_len. + conv_state_indices: (batch,), dtype int32 + If not None, the conv_state is a larger tensor along the batch dim, + and we are selecting the batch coords specified by conv_state_indices. + Useful for a continuous batching scenario. + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim) or (batch, dim, seqlen) + """ + if validate_data: + assert cache_seqlens is None # not implemented yet - ok for vLLM + assert pad_slot_id is not None + assert x.stride(1) == 1 + if isinstance(activation, bool): + activation = "silu" if activation is True else None + elif activation is not None: + assert activation in ["silu", "swish"] + unsqueeze = x.dim() == 2 + if unsqueeze: + # make it (batch, dim, seqlen) with seqlen == 1 + x = x.unsqueeze(-1) + batch, dim, seqlen = x.shape + _, width = weight.shape + # conv_state: (..., dim, state_len), where state_len >= width - 1 + num_cache_lines, _, state_len = conv_state.size() + + if validate_data: + assert dim == weight.size(0) + assert ( + conv_state.stride(-2) == 1 + ), f"ERROR: expect contiguous along feat-dim of conv_state (currently stride={conv_state.stride()})" + assert state_len >= width - 1 + # when above happens, we don't shift-left to keep any records in conv_state + assert dim == conv_state.size(1) + if conv_state_indices is None: + assert conv_state.size(0) >= batch + else: + assert (batch,) == conv_state_indices.shape + + assert num_cache_lines >= batch + assert weight.stride(1) == 1 # Need this + assert cache_seqlens is None # not needed for vLLM - circular buffer + + # adopt the strategy in vLLM that overwrite on 'x' directly, rather than creating a new tensor 'o' + out = x + stride_w_dim, stride_w_width = weight.stride() + + stride_x_seq, stride_x_dim, stride_x_token = x.stride() # X (batch, dim, seqlen) + + stride_o_seq, stride_o_dim, stride_o_token = out.stride() + stride_istate_seq, stride_istate_dim, stride_istate_token = conv_state.stride() + stride_state_indices = ( + conv_state_indices.stride(0) if conv_state_indices is not None else 0 + ) + state_len = width - 1 + (seqlen - 1) # effective state_len needed + np2_statelen = triton.next_power_of_2(state_len) + + def grid(META): + return ( + batch, + triton.cdiv(dim, META["BLOCK_N"]), + ) + + # prepare intermediate buffer strides if provided + if intermediate_conv_window is not None: + stride_inter_seq, stride_inter_step, stride_inter_dim, stride_inter_win = ( + intermediate_conv_window.stride(0), + intermediate_conv_window.stride(1), + intermediate_conv_window.stride(2), + intermediate_conv_window.stride(3), + ) + else: + stride_inter_seq = stride_inter_step = stride_inter_dim = stride_inter_win = 0 + + _causal_conv1d_update_kernel[grid]( + # Pointers to matrices + x, + weight, + bias, + conv_state, + cache_seqlens, + conv_state_indices, + num_accepted_tokens, + intermediate_conv_window if intermediate_conv_window is not None else x, + out, + # Matrix dimensions + batch, + dim, + seqlen, + state_len, + num_cache_lines, + # stride + stride_x_seq, + stride_x_dim, + stride_x_token, + stride_w_dim, + stride_w_width, + stride_istate_seq, + stride_istate_dim, + stride_istate_token, + stride_state_indices, + stride_inter_seq, + stride_inter_step, + stride_inter_dim, + stride_inter_win, + stride_o_seq, + stride_o_dim, + stride_o_token, + # others + pad_slot_id, + # META + HAS_BIAS=bias is not None, + KERNEL_WIDTH=width, + SILU_ACTIVATION=activation in ["silu", "swish"], + IS_CONTINUOUS_BATCHING=conv_state_indices is not None, + IS_SPEC_DECODING=num_accepted_tokens is not None, + NP2_STATELEN=np2_statelen, + USE_PAD_SLOT=pad_slot_id is not None, + BLOCK_N=256, + SAVE_INTERMEDIATE=intermediate_conv_window is not None, + ) + if unsqueeze: + out = out.squeeze(-1) + return out diff --git a/python/sglang/srt/layers/attention/mamba/mamba.py b/python/sglang/srt/layers/attention/mamba/mamba.py new file mode 100644 index 000000000..045a04048 --- /dev/null +++ b/python/sglang/srt/layers/attention/mamba/mamba.py @@ -0,0 +1,64 @@ +from typing import Callable, List, Tuple + +import torch + +LoaderFunction = Callable[[torch.Tensor, torch.Tensor], None] + + +def mamba_v2_sharded_weight_loader( + shard_spec: List[Tuple[int, int, float]], + tp_size: int, + tp_rank: int, +) -> LoaderFunction: + """Create a weight loader for mamba v2. This ensures that the projections + are correctly sharded so that they can be split into x, B, C. It also + ensures the the all the groups corresponding to a head shard is placed + together with it. + """ + + def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: + + # - track boundary of (sharded) param, and loaded_weight, respectively + boundary, loaded_boundary = 0, 0 + + # - iterate over the shard specs + for full_dim, extra, duplicate_groups in shard_spec: + # - full dim is the model dim (before TP). + # - extra > 0, means there is expected overall increase + # of dimensions. This is so because of replication. + # - ratio is used map the tp_rank to the actual shard + # rank. This is useful when there is replication of + # groups to accompany head shards. + + # - size of the loaded shard + shard_size = full_dim // tp_size + + # - compute the rank into the loaded shard. + # - if there is replication, different TP shards will + # take from the same rank. + # NOTE: currently we only support duplication + # in the case where num_groups == 1 + rank = 0 if duplicate_groups else tp_rank + + # - leftmost boundary index into loaded weight. + loaded_skip = rank * shard_size + loaded_start_idx = loaded_boundary + loaded_skip + + # - take these many dims from the loaded weight. + take = min(shard_size, full_dim - extra - loaded_skip) + + # - always shard on dim 0 + # - the ignore is for a mundane mypy error as it does not + # seem to handle slices well. + # https://github.com/python/mypy/issues/2410 + param.data[ + boundary : (boundary + take), ... # type: ignore[misc] + ] = loaded_weight[ + loaded_start_idx : (loaded_start_idx + take) # type: ignore[misc] + ] # type: ignore[misc] + + # move indexing boundaries + boundary += shard_size + loaded_boundary += full_dim - extra + + return loader diff --git a/python/sglang/srt/layers/attention/merge_state.py b/python/sglang/srt/layers/attention/merge_state.py new file mode 100644 index 000000000..245418a9d --- /dev/null +++ b/python/sglang/srt/layers/attention/merge_state.py @@ -0,0 +1,46 @@ +from typing import Optional, Tuple + +import torch +from sgl_kernel import merge_state_v2 + +from sglang.srt.layers.attention.triton_ops.merge_state import merge_state_triton +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() + + +# Automatically fallback to the Triton kernel in some cases +# (e.g., for AMD GPUs, when the head dimension is not a multiple +# of 4 or 8, and in FP8 precision) +def _supported_dtypes(o: torch.Tensor) -> bool: + return o.dtype in [torch.float32, torch.half, torch.bfloat16] + + +def _supported_headdim(o: torch.Tensor) -> bool: + headdim = o.shape[2] # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + if o.dtype == torch.float32: + return headdim % 4 == 0 + return headdim % 8 == 0 + + +def merge_state( + prefix_output: torch.Tensor, + prefix_lse: torch.Tensor, + suffix_output: torch.Tensor, + suffix_lse: torch.Tensor, + output: Optional[torch.Tensor] = None, + output_lse: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + if ( + _is_cuda + and _supported_dtypes(prefix_output) + and _supported_headdim(prefix_output) + ): + return merge_state_v2( + prefix_output, prefix_lse, suffix_output, suffix_lse, output, output_lse + ) + else: + # Fallback to Triton kernel + return merge_state_triton( + prefix_output, prefix_lse, suffix_output, suffix_lse, output, output_lse + ) diff --git a/python/sglang/srt/layers/attention/tbo_backend.py b/python/sglang/srt/layers/attention/tbo_backend.py new file mode 100644 index 000000000..06cfbd4ef --- /dev/null +++ b/python/sglang/srt/layers/attention/tbo_backend.py @@ -0,0 +1,260 @@ +from typing import TYPE_CHECKING, Callable, List, Optional, Union + +import torch + +from sglang.srt import two_batch_overlap +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode + + +class TboAttnBackend(AttentionBackend): + def __init__(self, primary: AttentionBackend, children: List[AttentionBackend]): + super().__init__() + self.primary = primary + self.children = children + + @classmethod + def init_new(cls, creator: Callable[[], AttentionBackend]): + return cls( + primary=creator(), + children=[creator() for _ in range(2)], + ) + + def init_forward_metadata(self, forward_batch: "ForwardBatch"): + self.primary.init_forward_metadata(forward_batch=forward_batch) + if forward_batch.tbo_children is not None: + for child, forward_batch_child in zip( + self.children, forward_batch.tbo_children, strict=True + ): + if forward_batch_child.batch_size > 0: + child.init_forward_metadata(forward_batch=forward_batch_child) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.primary.init_cuda_graph_state(max_bs=max_bs, max_num_tokens=max_num_tokens) + for item in self.children: + # TODO for children, maybe can provide *smaller* max_bs to optimize + item.init_cuda_graph_state(max_bs=max_bs, max_num_tokens=max_num_tokens) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: "ForwardMode", + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + self.primary.init_forward_metadata_capture_cuda_graph( + bs=bs, + num_tokens=num_tokens, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + encoder_lens=encoder_lens, + forward_mode=forward_mode, + spec_info=spec_info, + ) + + self._init_forward_metadata_cuda_graph_children( + fn_name="init_forward_metadata_capture_cuda_graph", + bs=bs, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + encoder_lens=encoder_lens, + forward_mode=forward_mode, + spec_info=spec_info, + capture_num_tokens=num_tokens, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: "ForwardMode", + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + self.primary.init_forward_metadata_replay_cuda_graph( + bs=bs, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + seq_lens_sum=seq_lens_sum, + encoder_lens=encoder_lens, + forward_mode=forward_mode, + spec_info=spec_info, + seq_lens_cpu=seq_lens_cpu, + ) + + self._init_forward_metadata_cuda_graph_children( + fn_name="init_forward_metadata_replay_cuda_graph", + bs=bs, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + encoder_lens=encoder_lens, + forward_mode=forward_mode, + spec_info=spec_info, + replay_seq_lens_sum=seq_lens_sum, + replay_seq_lens_cpu=seq_lens_cpu, + ) + + def _init_forward_metadata_cuda_graph_children( + self, + fn_name: str, + # common args + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: "ForwardMode", + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + # capture args + capture_num_tokens: int = None, + # replay args + replay_seq_lens_sum: int = None, + replay_seq_lens_cpu: Optional[torch.Tensor] = None, + ): + token_num_per_seq = two_batch_overlap.get_token_num_per_seq( + forward_mode=forward_mode, spec_info=spec_info + ) + if fn_name == "init_forward_metadata_capture_cuda_graph": + assert ( + capture_num_tokens == bs * token_num_per_seq + ), "For target-verify or decode mode, num_tokens should be equal to token_num_per_seq * bs" + num_tokens = bs * token_num_per_seq + + tbo_split_seq_index, tbo_split_token_index = ( + two_batch_overlap.compute_split_indices_for_cuda_graph_replay( + forward_mode=forward_mode, + cuda_graph_num_tokens=num_tokens, + spec_info=spec_info, + ) + ) + + num_tokens_child_left = tbo_split_token_index + num_tokens_child_right = num_tokens - tbo_split_token_index + bs_child_left = tbo_split_seq_index + bs_child_right = bs - bs_child_left + + assert ( + num_tokens_child_left > 0 and num_tokens_child_right > 0 + ), f"{num_tokens_child_left=} {num_tokens_child_right=} {forward_mode=} {num_tokens=}" + + common_pre_split_args = dict( + fn_name=fn_name, + bs=bs, + req_pool_indices=req_pool_indices, + seq_lens=seq_lens, + encoder_lens=encoder_lens, + forward_mode=forward_mode, + spec_info=spec_info, + capture_num_tokens=capture_num_tokens, + replay_seq_lens_sum=replay_seq_lens_sum, + replay_seq_lens_cpu=replay_seq_lens_cpu, + ) + + args_left = _init_forward_metadata_cuda_graph_split( + output_bs=bs_child_left, + seq_slice=slice(None, tbo_split_seq_index), + **common_pre_split_args, + ) + args_right = _init_forward_metadata_cuda_graph_split( + output_bs=bs_child_right, + seq_slice=slice(tbo_split_seq_index, None), + **common_pre_split_args, + ) + + child_left, child_right = self.children + getattr(child_left, fn_name)(**args_left) + getattr(child_right, fn_name)(**args_right) + + def get_cuda_graph_seq_len_fill_value(self): + ans = self.primary.get_cuda_graph_seq_len_fill_value() + for child in self.children: + assert ans == child.get_cuda_graph_seq_len_fill_value() + return ans + + def forward_extend(self, *args, **kwargs): + return self.primary.forward_extend(*args, **kwargs) + + def forward_decode(self, *args, **kwargs): + return self.primary.forward_decode(*args, **kwargs) + + +def _init_forward_metadata_cuda_graph_split( + fn_name: str, + seq_slice: slice, + output_bs: int, + # common args + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: "ForwardMode", + spec_info: Optional[EagleVerifyInput], + # capture args + capture_num_tokens: int = None, + # replay args + replay_seq_lens_sum: int = None, + replay_seq_lens_cpu: Optional[torch.Tensor] = None, +): + token_num_per_seq = two_batch_overlap.get_token_num_per_seq( + forward_mode=forward_mode, spec_info=spec_info + ) + assert encoder_lens is None, "encoder_lens is not supported yet" + if spec_info is not None: + output_spec_info = two_batch_overlap.split_spec_info( + spec_info=spec_info, + start_seq_index=seq_slice.start if seq_slice.start is not None else 0, + end_seq_index=seq_slice.stop if seq_slice.stop is not None else bs, + start_token_index=( + seq_slice.start * token_num_per_seq + if seq_slice.start is not None + else 0 + ), + end_token_index=( + seq_slice.stop * token_num_per_seq + if seq_slice.stop is not None + else bs * token_num_per_seq + ), + ) + + else: + output_spec_info = None + ans = dict( + bs=output_bs, + req_pool_indices=req_pool_indices[seq_slice], + seq_lens=seq_lens[seq_slice], + # directly forward + forward_mode=forward_mode, + # ignore + encoder_lens=None, + spec_info=output_spec_info, + ) + + if fn_name == "init_forward_metadata_capture_cuda_graph": + assert ( + capture_num_tokens == bs * token_num_per_seq + ), "Only support num_tokens==bs * token_num_per_seq for target-verify or decode mode" + ans.update( + dict( + num_tokens=output_bs * token_num_per_seq, + ) + ) + elif fn_name == "init_forward_metadata_replay_cuda_graph": + output_seq_lens_cpu = replay_seq_lens_cpu[seq_slice] + ans.update( + dict( + seq_lens_sum=output_seq_lens_cpu.sum().item(), + seq_lens_cpu=output_seq_lens_cpu, + ) + ) + else: + raise NotImplementedError + + return ans diff --git a/python/sglang/srt/layers/attention/torch_native_backend.py b/python/sglang/srt/layers/attention/torch_native_backend.py new file mode 100644 index 000000000..6a67ea947 --- /dev/null +++ b/python/sglang/srt/layers/attention/torch_native_backend.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from torch.nn.functional import scaled_dot_product_attention + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + + +class TorchNativeAttnBackend(AttentionBackend): + def __init__(self, model_runner: ModelRunner): + super().__init__() + self.forward_metadata = None + self.device = model_runner.device + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init the metadata for a forward pass.""" + pass + + def _run_sdpa_forward_extend( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + extend_prefix_lens: torch.Tensor, + extend_seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + """Run the extend forward by using torch native sdpa op. + + Args: + query: [num_tokens, num_heads, head_size] + output: [num_tokens, num_heads, head_size] + k_cache: [max_total_num_tokens, num_heads, head_size] + v_cache: [max_total_num_tokens, num_heads, head_size] + req_to_token: [max_num_reqs, max_context_len] + req_pool_indices: [num_seqs] + seq_lens: [num_seqs] + extend_prefix_lens: [num_seqs] + extend_seq_lens: [num_seqs] + scaling: float or None + enable_gqa: bool + causal: bool + + Returns: + output: [num_tokens, num_heads, head_size] + """ + + assert seq_lens.shape[0] == extend_prefix_lens.shape[0] + assert seq_lens.shape[0] == extend_seq_lens.shape[0] + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + # TODO: this loop process a sequence per iter, this is inefficient. + # Need optimize the performance later. + + extend_seq_len_q = extend_seq_lens[seq_idx] + prefill_seq_len_q = extend_prefix_lens[seq_idx] + + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + extend_seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + per_req_query_redudant = torch.empty( + (per_req_query.shape[0], seq_len_kv, per_req_query.shape[2]), + dtype=per_req_query.dtype, + device=per_req_query.device, + ) + + per_req_query_redudant[:, prefill_seq_len_q:, :] = per_req_query + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out_redudant = ( + scaled_dot_product_attention( + per_req_query_redudant.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out_redudant[prefill_seq_len_q:, :, :] + start_q, start_kv = end_q, end_kv + return output + + def _run_sdpa_forward_decode( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + """Run the decode forward by using torch native sdpa op. + + Args: + query: [num_tokens, num_heads, head_size] + output: [num_tokens, num_heads, head_size] + k_cache: [max_total_num_tokens, num_heads, head_size] + v_cache: [max_total_num_tokens, num_heads, head_size] + req_to_token: [max_num_reqs, max_context_len] + req_pool_indices: [num_seqs] + seq_lens: [num_seqs] + scaling: float or None + enable_gqa: bool + causal: bool + + Returns: + output: [num_tokens, num_heads, head_size] + """ + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + # TODO: this loop process a sequence per iter, this is inefficient. + # Need optimize the performance later. + + seq_len_q = 1 + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out = ( + scaled_dot_product_attention( + per_req_query.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out + start_q, start_kv = end_q, end_kv + + return output + + def forward_extend( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if layer.is_cross_attention: + cache_loc = forward_batch.encoder_out_cache_loc + else: + cache_loc = forward_batch.out_cache_loc + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + + use_gqa = layer.tp_q_head_num != layer.tp_k_head_num + + q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim) + o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim) + + causal = True + if layer.is_cross_attention or layer.attn_type == AttentionType.ENCODER_ONLY: + causal = False + + self._run_sdpa_forward_extend( + q_, + o_, + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.extend_prefix_lens, + forward_batch.extend_seq_lens, + scaling=layer.scaling, + enable_gqa=use_gqa, + causal=causal, + ) + return o + + def forward_decode( + self, + q, + k, + v, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # During torch.compile, there is a bug in rotary_emb that causes the + # output value to have a 3D tensor shape. This reshapes the output correctly. + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if layer.is_cross_attention: + cache_loc = forward_batch.encoder_out_cache_loc + else: + cache_loc = forward_batch.out_cache_loc + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer(layer, cache_loc, k, v) + + use_gqa = layer.tp_q_head_num != layer.tp_k_head_num + + q_ = q.view(-1, layer.tp_q_head_num, layer.qk_head_dim) + o_ = o.view(-1, layer.tp_q_head_num, layer.v_head_dim) + + self._run_sdpa_forward_decode( + q_, + o_, + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + forward_batch.req_to_token_pool.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + scaling=layer.scaling, + enable_gqa=use_gqa, + causal=False, + ) + + return o + + def support_triton(self): + return False diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py new file mode 100644 index 000000000..26241d849 --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -0,0 +1,1090 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.layers.radix_attention import AttentionType +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import get_bool_env_var, get_device_core_count, next_power_of_2 + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + + +def logit_capping_mod(logit_capping_method, logit_cap): + # positive logit_cap -> tanh cap + if logit_capping_method == "tanh": + return logit_cap + else: + raise ValueError() + + +@dataclass +class ForwardMetadata: + attn_logits: torch.Tensor + attn_lse: torch.Tensor + max_extend_len: int + num_kv_splits: torch.Tensor + kv_indptr: torch.Tensor + kv_indices: torch.Tensor + qo_indptr: torch.Tensor + custom_mask: torch.Tensor + mask_indptr: torch.Tensor + # Sliding window + window_kv_indptr: torch.Tensor + window_kv_indices: torch.Tensor + window_num_kv_splits: torch.Tensor + window_kv_offsets: torch.Tensor + + +class TritonAttnBackend(AttentionBackend): + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + ): + # Lazy import to avoid the initialization of cuda context + from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd, + ) + from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + ) + + super().__init__() + + self.decode_attention_fwd = torch.compiler.disable(decode_attention_fwd) + self.extend_attention_fwd = torch.compiler.disable(extend_attention_fwd) + + # Parse args + self.skip_prefill = skip_prefill + max_bs = model_runner.req_to_token_pool.size + self.sliding_window_size = model_runner.sliding_window_size + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.token_to_kv_pool_allocator = model_runner.token_to_kv_pool_allocator + self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens + self.speculative_num_steps = model_runner.server_args.speculative_num_steps + self.num_head = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_head = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + self.v_head_dim = model_runner.token_to_kv_pool.get_value_buffer(0).shape[-1] + self.max_context_len = model_runner.model_config.context_len + self.device = model_runner.device + self.device_core_count = get_device_core_count(model_runner.gpu_id) + self.static_kv_splits = get_bool_env_var( + "SGLANG_TRITON_DECODE_ATTN_STATIC_KV_SPLITS", "false" + ) + self.max_kv_splits = model_runner.server_args.triton_attention_num_kv_splits + + # Check arguments + assert not ( + model_runner.sliding_window_size is not None + and model_runner.model_config.is_encoder_decoder + ), "Sliding window and cross attention are not supported together" + + # Initialize buffers + # TODO(Jianan Ji): Make sure it behaves as expected when kv_indptr_buf is provided and sliding window is enabled + if kv_indptr_buf is None: + self.kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + else: + self.kv_indptr = kv_indptr_buf + + # If sliding window is enabled, we might need two sets of buffers + # because of interleaved attention types (e.g. for Gemma3) + self.window_kv_indptr = None + if self.sliding_window_size is not None and self.sliding_window_size > 0: + if kv_indptr_buf is None: + self.window_kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + else: + # When provided a buffer, create a clone for the second buffer + self.window_kv_indptr = torch.zeros_like(kv_indptr_buf) + + if not self.skip_prefill: + self.qo_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + + self.mask_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int64, device=model_runner.device + ) + + # Initialize forward metadata + self.forward_metadata: ForwardMetadata = None + + def get_num_kv_splits( + self, + num_kv_splits: torch.Tensor, + seq_lens: torch.Tensor, + ): + num_token, num_seq = num_kv_splits.shape[0], seq_lens.shape[0] + # NOTE(alcanderian): Considering speculative_decodeing, + # num_kv_splits.shape[0] will be topk * real_num_token. + # And the real_num_token is num_seq in decoding phase. + num_group = num_token // num_seq + + assert ( + num_group * num_seq == num_token + ), f"num_seq({num_seq}), num_token({num_token}), something goes wrong!" + + if self.static_kv_splits or self.device_core_count <= 0: + num_kv_splits.fill_(self.max_kv_splits) + return + + if num_seq < 256: + SCHEDULE_SEQ = 256 + else: + SCHEDULE_SEQ = triton.next_power_of_2(num_seq) + + get_num_kv_splits_triton[(1,)]( + num_kv_splits, + seq_lens, + num_seq, + num_group, + self.num_head, + self.num_kv_head, + self.max_kv_splits, + self.device_core_count, + MAX_NUM_SEQ=SCHEDULE_SEQ, + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init auxiliary variables for triton attention backend.""" + + bs = forward_batch.batch_size + kv_indptr = self.kv_indptr + window_kv_indptr = self.window_kv_indptr + window_kv_indices = None + window_num_kv_splits = None + window_kv_offsets = None + spec_info = forward_batch.spec_info + + if forward_batch.forward_mode.is_decode_or_idle(): + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + forward_batch.seq_lens_sum, dtype=torch.int64, device=self.device + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + # Sliding window + if ( + self.sliding_window_size is not None + and self.sliding_window_size > 0 + ): + window_kv_indptr, window_kv_indices, window_kv_lens, _ = ( + update_sliding_window_buffer( + self.window_kv_indptr, + self.req_to_token, + self.sliding_window_size, + forward_batch.seq_lens, + forward_batch.req_pool_indices, + bs, + self.device, + self.token_to_kv_pool_allocator, + ) + ) + window_num_kv_splits = torch.empty( + (bs,), dtype=torch.int32, device=self.device + ) + self.get_num_kv_splits(window_num_kv_splits, window_kv_lens) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + bs = kv_indptr.shape[0] - 1 + + attn_logits = torch.empty( + (bs, self.num_head, self.max_kv_splits, self.v_head_dim), + dtype=torch.float32, + device=self.device, + ) + attn_lse = torch.empty( + (bs, self.num_head, self.max_kv_splits), + dtype=torch.float32, + device=self.device, + ) + num_kv_splits = torch.empty((bs,), dtype=torch.int32, device=self.device) + self.get_num_kv_splits(num_kv_splits, forward_batch.seq_lens) + + qo_indptr = None + custom_mask = None + mask_indptr = None + max_extend_len = None + elif forward_batch.forward_mode.is_target_verify(): + bs = len(forward_batch.req_pool_indices) + qo_indptr = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + # Different with flashinfer kv_indptr and kv_indices construction + kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + kv_indptr[-1], dtype=torch.int64, device=self.device + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + if self.sliding_window_size is not None and self.sliding_window_size > 0: + # window_kv_offsets is used to calculate the start position in custom mask + ( + window_kv_indptr, + window_kv_indices, + window_kv_lens, + window_kv_offsets, + ) = update_sliding_window_buffer( + self.window_kv_indptr, + self.req_to_token, + self.sliding_window_size, + forward_batch.seq_lens, + forward_batch.req_pool_indices, + bs, + self.device, + self.token_to_kv_pool_allocator, + ) + + custom_mask = spec_info.custom_mask + seq_mask_len = self.num_draft_tokens * ( + forward_batch.seq_lens + self.num_draft_tokens + ) + mask_indptr = self.mask_indptr + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len[:bs], dim=0) + mask_indptr = mask_indptr[: bs + 1] + max_extend_len = self.num_draft_tokens + num_kv_splits = None + attn_logits = None + attn_lse = None + + elif forward_batch.forward_mode.is_draft_extend(): + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + None, + self.req_to_token, + ) + ) + kv_indices = kv_indices.to(torch.int64) + mask_indptr = None + # TODO(FIXME): This will trigger an invalid Eagle tree when using + # `max(spec_info.accept_length_cpu)`. + # It might have been forgotten to update somewhere. + max_extend_len = torch.max(spec_info.accept_length).item() + num_kv_splits = None + attn_logits = None + attn_lse = None + else: + kv_indptr[1 : bs + 1] = torch.cumsum( + forward_batch.extend_prefix_lens, dim=0 + ) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + forward_batch.extend_prefix_lens.sum().item(), + dtype=torch.int64, + device=self.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.extend_prefix_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + # Sliding window + if self.sliding_window_size is not None and self.sliding_window_size > 0: + window_kv_indptr, window_kv_indices, _, _ = ( + update_sliding_window_buffer( + self.window_kv_indptr, + self.req_to_token, + self.sliding_window_size, + forward_batch.extend_prefix_lens, + forward_batch.req_pool_indices, + bs, + self.device, + self.token_to_kv_pool_allocator, + ) + ) + + qo_indptr = self.qo_indptr + qo_indptr[1 : bs + 1] = torch.cumsum(forward_batch.extend_seq_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + custom_mask = None + mask_indptr = None + attn_logits = None + attn_lse = None + max_extend_len = max(forward_batch.extend_seq_lens_cpu) + num_kv_splits = None + + self.forward_metadata = ForwardMetadata( + attn_logits, + attn_lse, + max_extend_len, + num_kv_splits, + kv_indptr, + kv_indices, + qo_indptr, + custom_mask, + mask_indptr, + window_kv_indptr, + window_kv_indices, + window_num_kv_splits, + window_kv_offsets, + ) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + self.cuda_graph_attn_logits = torch.zeros( + (max_num_tokens, self.num_head, self.max_kv_splits, self.v_head_dim), + dtype=torch.float32, + device=self.device, + ) + self.cuda_graph_attn_lse = torch.zeros( + (max_num_tokens, self.num_head, self.max_kv_splits), + dtype=torch.float32, + device=self.device, + ) + self.cuda_graph_num_kv_splits = torch.full( + (max_num_tokens,), self.max_kv_splits, dtype=torch.int32, device=self.device + ) + if kv_indices_buf is None: + self.cuda_graph_kv_indices = torch.zeros( + (max_num_tokens * self.max_context_len), + dtype=torch.int64, + device=self.device, + ) + else: + self.cuda_graph_kv_indices = kv_indices_buf + + if not self.skip_prefill: + self.cuda_graph_custom_mask = torch.zeros( + (max_num_tokens * self.max_context_len), + dtype=torch.uint8, + device=self.device, + ) + + if self.sliding_window_size is not None and self.sliding_window_size > 0: + if kv_indices_buf is None: + self.cuda_graph_window_kv_indices = torch.zeros( + (max_num_tokens * self.sliding_window_size), + dtype=torch.int64, + device=self.device, + ) + else: + self.cuda_graph_window_kv_indices = torch.zeros_like(kv_indices_buf) + + self.cuda_graph_window_num_kv_splits = torch.full( + (max_num_tokens,), + self.max_kv_splits, + dtype=torch.int32, + device=self.device, + ) + + self.cuda_graph_window_kv_offsets = torch.zeros( + (max_bs,), + dtype=torch.int32, + device=self.device, + ) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + assert encoder_lens is None, "Not supported" + window_kv_indptr = self.window_kv_indptr + window_kv_indices = None + window_num_kv_splits = None + window_kv_offsets = None + + if forward_mode.is_decode_or_idle(): + if spec_info is None: + kv_indptr = self.kv_indptr + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + if ( + self.sliding_window_size is not None + and self.sliding_window_size > 0 + ): + window_kv_indices = self.cuda_graph_window_kv_indices + window_num_kv_splits = self.cuda_graph_window_num_kv_splits + window_kv_indptr, window_kv_indices, _, _ = ( + update_sliding_window_buffer_cuda_graph( + self.window_kv_indptr, + window_kv_indices, + self.req_to_token, + self.sliding_window_size, + seq_lens[:bs], + req_pool_indices, + bs, + self.token_to_kv_pool_allocator, + ) + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + + attn_logits = self.cuda_graph_attn_logits + attn_lse = self.cuda_graph_attn_lse + max_extend_len = None + num_kv_splits = self.cuda_graph_num_kv_splits + qo_indptr = None + custom_mask = None + mask_indptr = None + elif forward_mode.is_target_verify(): + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + if self.sliding_window_size is not None and self.sliding_window_size > 0: + window_kv_indices = self.cuda_graph_window_kv_indices + window_num_kv_splits = self.cuda_graph_window_num_kv_splits + window_kv_offsets = self.cuda_graph_window_kv_offsets + window_kv_indptr, window_kv_indices, _, window_kv_offsets[:bs] = ( + update_sliding_window_buffer_cuda_graph( + self.window_kv_indptr, + window_kv_indices, + self.req_to_token, + self.sliding_window_size, + seq_lens[:bs], + req_pool_indices, + bs, + self.token_to_kv_pool_allocator, + ) + ) + + custom_mask = self.cuda_graph_custom_mask + custom_mask[: spec_info.custom_mask.shape[0]] = spec_info.custom_mask + seq_mask_len = self.num_draft_tokens * (seq_lens + self.num_draft_tokens) + mask_indptr = self.mask_indptr[: bs + 1] + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len, dim=0) + max_extend_len = self.num_draft_tokens + num_kv_splits = None + attn_logits = None + attn_lse = None + elif forward_mode.is_draft_extend(): + num_tokens_per_bs = self.speculative_num_steps + 1 + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + bs * num_tokens_per_bs + 1, + step=num_tokens_per_bs, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + custom_mask = None + mask_indptr = None + max_extend_len = num_tokens_per_bs + num_kv_splits = None + attn_logits = None + attn_lse = None + else: + raise ValueError( + f"Invalid forward mode: {forward_mode=} for CUDA Graph capture." + ) + + self.forward_metadata = ForwardMetadata( + attn_logits, + attn_lse, + max_extend_len, + num_kv_splits, + kv_indptr, + kv_indices, + qo_indptr, + custom_mask, + mask_indptr, + window_kv_indptr, + window_kv_indices, + window_num_kv_splits, + window_kv_offsets, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + # NOTE: encoder_lens expected to be zeros or None + if forward_mode.is_decode_or_idle(): + # Update kv_indptr, kv_indices + kv_indptr = self.kv_indptr + kv_indices = self.cuda_graph_kv_indices + num_kv_splits = self.cuda_graph_num_kv_splits + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens[:bs], dim=0) + kv_indptr = kv_indptr[: bs + 1] + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens[:bs], + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + num_token = bs + if ( + self.sliding_window_size is not None + and self.sliding_window_size > 0 + ): + window_num_kv_splits = self.cuda_graph_window_num_kv_splits + window_kv_indices = self.cuda_graph_window_kv_indices + _, _, window_kv_lens, _ = update_sliding_window_buffer_cuda_graph( + self.window_kv_indptr, + window_kv_indices, + self.req_to_token, + self.sliding_window_size, + seq_lens[:bs], + req_pool_indices[:bs], + bs, + self.token_to_kv_pool_allocator, + ) + self.get_num_kv_splits( + window_num_kv_splits[:num_token], window_kv_lens[:bs] + ) + + else: + kv_indptr[: spec_info.kv_indptr.shape[0]] = spec_info.kv_indptr + kv_indices[: spec_info.kv_indices.shape[0]] = spec_info.kv_indices + num_token = spec_info.kv_indptr.shape[0] - 1 + self.get_num_kv_splits(num_kv_splits[:num_token], seq_lens[:bs]) + + elif forward_mode.is_target_verify(): + # Update qo_indptr, kv_indptr, kv_indices, custom_mask, mask_indptr + bs = len(req_pool_indices) + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + if self.sliding_window_size is not None and self.sliding_window_size > 0: + window_num_kv_splits = self.cuda_graph_window_num_kv_splits + window_kv_indices = self.cuda_graph_window_kv_indices + window_kv_offsets = self.cuda_graph_window_kv_offsets + _, _, window_kv_lens, window_kv_offsets[:bs] = ( + update_sliding_window_buffer_cuda_graph( + self.window_kv_indptr, + window_kv_indices, + self.req_to_token, + self.sliding_window_size, + seq_lens[:bs], + req_pool_indices, + bs, + self.token_to_kv_pool_allocator, + ) + ) + custom_mask = self.cuda_graph_custom_mask + custom_mask[: spec_info.custom_mask.shape[0]] = spec_info.custom_mask + seq_mask_len = self.num_draft_tokens * (seq_lens + self.num_draft_tokens) + mask_indptr = self.mask_indptr[: bs + 1] + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len, dim=0) + elif forward_mode.is_draft_extend(): + seq_lens = seq_lens[:bs] + accept_lens = spec_info.accept_length[:bs] + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[1 : bs + 1] = torch.cumsum(accept_lens, dim=0) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + raise ValueError( + f"Invalid forward mode: {forward_mode=} for CUDA Graph replay." + ) + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + sinks=None, + ): + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + logits_soft_cap = logit_capping_mod(layer.logit_capping_method, layer.logit_cap) + + causal = True + if layer.attn_type == AttentionType.ENCODER_ONLY: + causal = False + + if layer.sliding_window_size is not None and layer.sliding_window_size > -1: + sliding_window_size = ( + layer.sliding_window_size + ) # Needed for sliding window mask + kv_indptr = self.forward_metadata.window_kv_indptr + kv_indices = self.forward_metadata.window_kv_indices + window_kv_offsets = self.forward_metadata.window_kv_offsets + else: + sliding_window_size = -1 + kv_indptr = self.forward_metadata.kv_indptr + kv_indices = self.forward_metadata.kv_indices + window_kv_offsets = None + + self.extend_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + k.contiguous(), + v.contiguous(), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.forward_metadata.qo_indptr, + kv_indptr, + kv_indices, + self.forward_metadata.custom_mask, + causal, + self.forward_metadata.mask_indptr, + self.forward_metadata.max_extend_len, + layer.scaling, + logit_cap=logits_soft_cap, + sliding_window_size=sliding_window_size, + sinks=sinks, + window_kv_offsets=window_kv_offsets, + xai_temperature_len=layer.xai_temperature_len, + ) + return o + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + sinks=None, + ): + # During torch.compile, there is a bug in rotary_emb that causes the + # output value to have a 3D tensor shape. This reshapes the output correctly. + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + logits_soft_cap = logit_capping_mod(layer.logit_capping_method, layer.logit_cap) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + if layer.sliding_window_size is not None and layer.sliding_window_size > -1: + kv_indptr = self.forward_metadata.window_kv_indptr + kv_indices = self.forward_metadata.window_kv_indices + else: + kv_indptr = self.forward_metadata.kv_indptr + kv_indices = self.forward_metadata.kv_indices + + self.decode_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + kv_indptr, + kv_indices, + self.forward_metadata.attn_logits, + self.forward_metadata.attn_lse, + self.forward_metadata.num_kv_splits, + self.max_kv_splits, + layer.scaling, + logit_cap=logits_soft_cap, + sinks=sinks, + xai_temperature_len=layer.xai_temperature_len, + ) + return o + + +class TritonMultiStepDraftBackend: + """ + Wrap multiple triton attention backends as one for multiple consecutive + draft decoding steps. + """ + + def __init__( + self, + model_runner: ModelRunner, + topk: int, + speculative_num_steps: int, + ): + from sglang.srt.speculative.eagle_utils import generate_draft_decode_kv_indices + + self.topk = topk + self.speculative_num_steps = speculative_num_steps + self.generate_draft_decode_kv_indices = generate_draft_decode_kv_indices + max_bs = model_runner.req_to_token_pool.size * self.topk + self.kv_indptr = torch.zeros( + ( + self.speculative_num_steps, + max_bs + 1, + ), + dtype=torch.int32, + device=model_runner.device, + ) + self.attn_backends = [] + for i in range(self.speculative_num_steps): + self.attn_backends.append( + TritonAttnBackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + ) + ) + self.max_context_len = self.attn_backends[0].max_context_len + self.num_head = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.device = model_runner.device + # Cached variables for generate_draft_decode_kv_indices + self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1] + self.page_size = model_runner.server_args.page_size + + def common_template( + self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int + ): + num_seqs = forward_batch.batch_size + bs = self.topk * num_seqs + seq_lens_sum = forward_batch.seq_lens_sum + + self.generate_draft_decode_kv_indices[ + (self.speculative_num_steps, num_seqs, self.topk) + ]( + forward_batch.req_pool_indices, + forward_batch.req_to_token_pool.req_to_token, + forward_batch.seq_lens, + kv_indices_buffer, + self.kv_indptr, + forward_batch.positions, + self.pool_len, + kv_indices_buffer.shape[1], + self.kv_indptr.shape[1], + next_power_of_2(num_seqs), + next_power_of_2(self.speculative_num_steps), + next_power_of_2(bs), + self.page_size, + ) + + for i in range(self.speculative_num_steps): + forward_batch.spec_info.kv_indptr = self.kv_indptr[i, : bs + 1] + forward_batch.spec_info.kv_indices = kv_indices_buffer[i][ + : seq_lens_sum * self.topk + bs * (i + 1) + ] + call_fn(i, forward_batch) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + kv_indices = torch.empty( + ( + self.speculative_num_steps, + forward_batch.batch_size * self.topk * self.max_context_len, + ), + dtype=torch.int64, + device=self.device, + ) + + def call_fn(i, forward_batch): + forward_batch.spec_info.kv_indptr = ( + forward_batch.spec_info.kv_indptr.clone() + ) + forward_batch.spec_info.kv_indices = ( + forward_batch.spec_info.kv_indices.clone() + ) + self.attn_backends[i].init_forward_metadata(forward_batch) + + self.common_template(forward_batch, kv_indices, call_fn) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + self.cuda_graph_kv_indices = torch.zeros( + (self.speculative_num_steps, max_num_tokens * self.max_context_len), + dtype=torch.int64, + device=self.device, + ) + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state( + max_bs, max_num_tokens, kv_indices_buf=self.cuda_graph_kv_indices[i] + ) + + def init_forward_metadata_capture_cuda_graph(self, forward_batch: ForwardBatch): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + def call_fn(i, forward_batch): + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + seq_lens_sum=-1, + encoder_lens=None, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=None, + ) + + self.common_template(forward_batch, self.cuda_graph_kv_indices, call_fn) + + +@triton.jit +def get_num_kv_splits_triton( + num_kv_splits_ptr, + seq_lens_ptr, + num_seq, + num_group, + num_head, + num_kv_head, + max_kv_splits, + device_core_count, + MAX_NUM_SEQ: tl.constexpr, +): + # TODO: this method is tunable, we need more online serving data to tune it + offs_seq = tl.arange(0, MAX_NUM_SEQ) + mask_seq = offs_seq < num_seq + + seq_lens = tl.load(seq_lens_ptr + offs_seq, mask=mask_seq, other=0) + max_seq_len = tl.max(seq_lens) + seq_lens = tl.load(seq_lens_ptr + offs_seq, mask=mask_seq, other=max_seq_len) + min_seq_len = tl.min(seq_lens) + if max_seq_len * 8 < min_seq_len * 10: + min_seq_len = max_seq_len + max_kv_splits_1 = tl.minimum(tl.cdiv(max_seq_len, min_seq_len), max_kv_splits) + kv_chunk_size_1 = tl.cdiv(max_seq_len, max_kv_splits_1) + + # NOTE: this is a hack to let num_kv_split grows up with seqlen gradually + ext_seq_len = tl.cast(max_seq_len, tl.float32) / 64.0 + ext_device_core_count = tl.cast( + device_core_count * tl.maximum(tl.log2(ext_seq_len), 1.0), tl.int32 + ) + block_h, num_kv_group = 16, num_head // num_kv_head + if num_kv_group == 1: + token_grid = num_seq * num_group * num_head + else: + # from triton_ops/decode_attention.py:_decode_grouped_att_m_fwd + block_h = tl.minimum(block_h, num_kv_group) + token_grid = num_seq * num_group * tl.cdiv(num_head, block_h) + max_kv_splits_2 = tl.minimum( + tl.cdiv(ext_device_core_count, token_grid), max_kv_splits + ) + kv_chunk_size_2 = tl.cdiv(max_seq_len, max_kv_splits_2) + + num_kv_splits = tl.maximum( + tl.cdiv(seq_lens, kv_chunk_size_1), tl.cdiv(seq_lens, kv_chunk_size_2) + ) + + offs_token = offs_seq * num_group + mask_token = offs_token < num_seq * num_group + for i in range(0, num_group): + tl.store(num_kv_splits_ptr + i + offs_token, num_kv_splits, mask=mask_token) + + +def update_sliding_window_buffer( + window_kv_indptr, + req_to_token, + sliding_window_size, + seq_lens, + req_pool_indices, + bs, + device, + token_to_kv_pool_allocator=None, +): + window_kv_lens = torch.minimum( + seq_lens, + torch.tensor(sliding_window_size), + ) + window_kv_indptr[1 : bs + 1] = torch.cumsum(window_kv_lens, dim=0) + window_kv_indptr = window_kv_indptr[: bs + 1] + window_kv_indices = torch.empty( + window_kv_indptr[-1], dtype=torch.int64, device=device + ) + window_kv_start_idx = seq_lens - window_kv_lens + create_flashinfer_kv_indices_triton[(bs,)]( + req_to_token, + req_pool_indices, + window_kv_lens, + window_kv_indptr, + window_kv_start_idx, + window_kv_indices, + req_to_token.stride(0), + ) + # full to swa index mapping + if hasattr(token_to_kv_pool_allocator, "translate_loc_from_full_to_swa"): + kv_last_index = window_kv_indptr[-1] + window_kv_indices[:kv_last_index] = ( + token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + window_kv_indices[:kv_last_index] + ) + ) + return window_kv_indptr, window_kv_indices, window_kv_lens, window_kv_start_idx + + +def update_sliding_window_buffer_cuda_graph( + window_kv_indptr, + window_kv_indices, + req_to_token, + sliding_window_size, + seq_lens, + req_pool_indices, + bs, + token_to_kv_pool_allocator=None, +): + window_kv_lens = torch.minimum( + seq_lens, + torch.tensor(sliding_window_size), + ) + window_kv_indptr[1 : bs + 1] = torch.cumsum(window_kv_lens, dim=0) + window_kv_indptr = window_kv_indptr[: bs + 1] + window_kv_start_idx = seq_lens - window_kv_lens + create_flashinfer_kv_indices_triton[(bs,)]( + req_to_token, + req_pool_indices, + window_kv_lens, + window_kv_indptr, + window_kv_start_idx, + window_kv_indices, + req_to_token.stride(0), + ) + # full to swa index mapping + if hasattr(token_to_kv_pool_allocator, "translate_loc_from_full_to_swa"): + kv_last_index = window_kv_indptr[-1] + window_kv_indices[:kv_last_index] = ( + token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + window_kv_indices[:kv_last_index] + ) + ) + return window_kv_indptr, window_kv_indices, window_kv_lens, window_kv_start_idx diff --git a/python/sglang/srt/layers/attention/triton_ops/decode_attention.py b/python/sglang/srt/layers/attention/triton_ops/decode_attention.py new file mode 100644 index 000000000..1ba5d463d --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/decode_attention.py @@ -0,0 +1,776 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Memory-efficient attention for decoding. +It supports page size = 1. +""" + +# Adapted from +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage1.py +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage2.py + +import logging + +import triton +import triton.language as tl + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + +logger = logging.getLogger(__name__) + + +_MIN_BLOCK_KV = 32 + + +@triton.jit +def tanh(x): + # Tanh is just a scaled sigmoid + return 2 * tl.sigmoid(2 * x) - 1 + + +@triton.jit +def _fwd_kernel_stage1( + Q, + K_Buffer, + V_Buffer, + sm_scale, + kv_indptr, + kv_indices, + Att_Out, + Att_Lse, + num_kv_splits, + stride_qbs, + stride_qh, + stride_buf_kbs, + stride_buf_kh, + stride_buf_vbs, + stride_buf_vh, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + kv_group_num: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_DV: tl.constexpr, + BLOCK_N: tl.constexpr, + MIN_BLOCK_KV: tl.constexpr, + logit_cap: tl.constexpr, + Lk: tl.constexpr, + Lv: tl.constexpr, + xai_temperature_len: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + split_kv_id = tl.program_id(2) + + cur_kv_head = cur_head // kv_group_num + + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_dv = tl.arange(0, BLOCK_DV) + mask_d = offs_d < Lk + mask_dv = offs_dv < Lv + + cur_batch_kv_start_idx = tl.load(kv_indptr + cur_batch) + cur_batch_seq_len = tl.load(kv_indptr + cur_batch + 1) - cur_batch_kv_start_idx + kv_splits = tl.load(num_kv_splits + cur_batch) + + if xai_temperature_len > 0: + offs_qidx = cur_batch_seq_len - 1 + xai_temperature_scale = 1.0 / tl.log2(float(xai_temperature_len)) + _qtemp = tl.log2(offs_qidx.to(tl.float32)) * xai_temperature_scale + xai_temperature_reg = tl.where(offs_qidx > xai_temperature_len, _qtemp, 1.0) + + off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d + + kv_len_per_split = ( + tl.cdiv(tl.cdiv(cur_batch_seq_len, kv_splits), MIN_BLOCK_KV) * MIN_BLOCK_KV + ) + split_kv_start = kv_len_per_split * split_kv_id + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split, cur_batch_seq_len) + + e_max = -float("inf") + e_sum = 0.0 + acc = tl.zeros([BLOCK_DV], dtype=tl.float32) + + if split_kv_end > split_kv_start: + q = tl.load(Q + off_q, mask=mask_d, other=0.0) + for start_n in range(split_kv_start, split_kv_end, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + kv_loc = tl.load( + kv_indices + cur_batch_kv_start_idx + offs_n, + mask=offs_n < split_kv_end, + other=0, + ) + offs_buf_k = ( + kv_loc[:, None] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_d[None, :] + ) + k = tl.load( + K_Buffer + offs_buf_k, + mask=(offs_n[:, None] < split_kv_end) & (mask_d[None, :]), + other=0.0, + ) + qk = tl.sum(q[None, :] * k, 1) + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + if xai_temperature_len > 0: + qk *= xai_temperature_reg + + qk = tl.where(offs_n < split_kv_end, qk, float("-inf")) + + offs_buf_v = ( + kv_loc[:, None] * stride_buf_vbs + + cur_kv_head * stride_buf_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Buffer + offs_buf_v, + mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]), + other=0.0, + ) + + n_e_max = tl.maximum(tl.max(qk, 0), e_max) + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max) + acc *= re_scale + acc += tl.sum(p[:, None] * v, 0) + + e_sum = e_sum * re_scale + tl.sum(p, 0) + e_max = n_e_max + + offs_mid_o = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + split_kv_id * stride_mid_os + + offs_dv + ) + + tl.store( + Att_Out + offs_mid_o, + acc / e_sum, + mask=(mask_dv), + ) + + offs_mid_o_1 = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + split_kv_id * stride_mid_os + ) // Lv + + tl.store( + Att_Lse + offs_mid_o_1, + e_max + tl.log(e_sum), + ) + + +def _decode_att_m_fwd( + q, + k_buffer, + v_buffer, + att_out, + att_lse, + kv_indptr, + kv_indices, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + xai_temperature_len=-1, +): + BLOCK = 64 + # [TODO] work around SGPR limit on MI3xx + if _is_hip: + BLOCK = 8 + MAX_KV_SPLITS = max_kv_splits + Lk = k_buffer.shape[-1] + Lv = v_buffer.shape[-1] + + batch, head_num = q.shape[0], q.shape[1] + + grid = (batch, head_num, MAX_KV_SPLITS) + kv_group_num = q.shape[1] // k_buffer.shape[1] + + if kv_group_num == 1: + num_warps = 4 + else: + num_warps = 2 + if _is_hip: + num_warps = 1 + + BLOCK_DMODEL = triton.next_power_of_2(Lk) + BLOCK_DV = triton.next_power_of_2(Lv) + + _fwd_kernel_stage1[grid]( + q, + k_buffer, + v_buffer, + sm_scale, + kv_indptr, + kv_indices, + att_out, + att_lse, + num_kv_splits, + q.stride(0), + q.stride(1), + k_buffer.stride(0), + k_buffer.stride(1), + v_buffer.stride(0), + v_buffer.stride(1), + att_out.stride(0), + att_out.stride(1), + att_out.stride(2), + kv_group_num=kv_group_num, + BLOCK_DMODEL=BLOCK_DMODEL, + BLOCK_DV=BLOCK_DV, + BLOCK_N=BLOCK, + MIN_BLOCK_KV=_MIN_BLOCK_KV, + logit_cap=logit_cap, + xai_temperature_len=xai_temperature_len, + num_warps=num_warps, + num_stages=2, + Lk=Lk, + Lv=Lv, + ) + + +@triton.jit +def _fwd_grouped_kernel_stage1( + Q, + K_Buffer, + V_Buffer, + sm_scale, + kv_indptr, + kv_indices, + Att_Out, + Att_Lse, + num_kv_splits, + stride_qbs, + stride_qh, + stride_buf_kbs, + stride_buf_kh, + stride_buf_vbs, + stride_buf_vh, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + kv_group_num: tl.constexpr, + q_head_num: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_DPE: tl.constexpr, + BLOCK_DV: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_H: tl.constexpr, + MIN_BLOCK_KV: tl.constexpr, + logit_cap: tl.constexpr, + xai_temperature_len: tl.constexpr, + Lk: tl.constexpr, + Lv: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head_id = tl.program_id(1) + cur_kv_head = cur_head_id // tl.cdiv(kv_group_num, BLOCK_H) + split_kv_id = tl.program_id(2) + + if BLOCK_H < kv_group_num: + VALID_BLOCK_H: tl.constexpr = BLOCK_H + else: + VALID_BLOCK_H: tl.constexpr = kv_group_num + cur_head = cur_head_id * VALID_BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = cur_head < (cur_head_id + 1) * VALID_BLOCK_H + mask_h = mask_h & (cur_head < q_head_num) + + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_dv = tl.arange(0, BLOCK_DV) + mask_d = offs_d < Lk + mask_dv = offs_dv < Lv + + cur_batch_kv_start_idx = tl.load(kv_indptr + cur_batch) + cur_batch_seq_len = tl.load(kv_indptr + cur_batch + 1) - cur_batch_kv_start_idx + kv_splits = tl.load(num_kv_splits + cur_batch) + + if xai_temperature_len > 0: + offs_qidx = cur_batch_seq_len - 1 + xai_temperature_scale = 1.0 / tl.log2(float(xai_temperature_len)) + _qtemp = tl.log2(offs_qidx.to(tl.float32)) * xai_temperature_scale + xai_temperature_reg = tl.where(offs_qidx > xai_temperature_len, _qtemp, 1.0) + + offs_q = cur_batch * stride_qbs + cur_head[:, None] * stride_qh + offs_d[None, :] + + if BLOCK_DPE > 0: + offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE) + mask_dpe = offs_dpe < Lk + off_qpe = ( + cur_batch * stride_qbs + cur_head[:, None] * stride_qh + offs_dpe[None, :] + ) + + kv_len_per_split = ( + tl.cdiv(tl.cdiv(cur_batch_seq_len, kv_splits), MIN_BLOCK_KV) * MIN_BLOCK_KV + ) + split_kv_start = kv_len_per_split * split_kv_id + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split, cur_batch_seq_len) + + e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf") + e_sum = tl.zeros([BLOCK_H], dtype=tl.float32) + acc = tl.zeros([BLOCK_H, BLOCK_DV], dtype=tl.float32) + + if split_kv_end > split_kv_start: + q = tl.load(Q + offs_q, mask=(mask_h[:, None]) & (mask_d[None, :]), other=0.0) + if BLOCK_DPE > 0: + qpe = tl.load( + Q + off_qpe, mask=(mask_h[:, None]) & (mask_dpe[None, :]), other=0.0 + ) + for start_n in range(split_kv_start, split_kv_end, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + kv_loc = tl.load( + kv_indices + cur_batch_kv_start_idx + offs_n, + mask=offs_n < split_kv_end, + other=0, + ) + offs_buf_k = ( + kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_d[:, None] + ) + k = tl.load( + K_Buffer + offs_buf_k, + mask=(offs_n[None, :] < split_kv_end) & (mask_d[:, None]), + other=0.0, + ) + qk = tl.dot(q, k.to(q.dtype)) + if BLOCK_DPE > 0: + offs_buf_kpe = ( + kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_dpe[:, None] + ) + kpe = tl.load( + K_Buffer + offs_buf_kpe, + mask=(offs_n[None, :] < split_kv_end) & (mask_dpe[:, None]), + other=0.0, + ) + qk += tl.dot(qpe, kpe.to(qpe.dtype)) + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + if xai_temperature_len > 0: + qk *= xai_temperature_reg[:, None] + + qk = tl.where( + mask_h[:, None] & (offs_n[None, :] < split_kv_end), qk, float("-inf") + ) + + offs_buf_v = ( + kv_loc[:, None] * stride_buf_vbs + + cur_kv_head * stride_buf_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Buffer + offs_buf_v, + mask=(offs_n[:, None] < split_kv_end) & (mask_dv[None, :]), + other=0.0, + ) + + n_e_max = tl.maximum(tl.max(qk, 1), e_max) + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + acc *= re_scale[:, None] + acc += tl.dot(p.to(v.dtype), v) + + e_sum = e_sum * re_scale + tl.sum(p, 1) + e_max = n_e_max + + offs_mid_o = ( + cur_batch * stride_mid_ob + + cur_head[:, None] * stride_mid_oh + + split_kv_id * stride_mid_os + + offs_dv[None, :] + ) + + tl.store( + Att_Out + offs_mid_o, + acc / e_sum[:, None], + mask=(mask_h[:, None]) & (mask_dv[None, :]), + ) + + offs_mid_o_1 = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + split_kv_id * stride_mid_os + ) // Lv + + tl.store( + Att_Lse + offs_mid_o_1, + e_max + tl.log(e_sum), + mask=mask_h, + ) + + +def _decode_grouped_att_m_fwd( + q, + k_buffer, + v_buffer, + att_out, + att_lse, + kv_indptr, + kv_indices, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + xai_temperature_len=-1, +): + BLOCK = 32 + Lk = k_buffer.shape[-1] + Lv = v_buffer.shape[-1] + + # [TODO] work around shmem limit on MI3xx + if _is_hip and Lk >= 576: + BLOCK = 16 + + if Lk == 576: + BLOCK_DMODEL = 512 + BLOCK_DPE = 64 + elif Lk == 288: + BLOCK_DMODEL = 256 + BLOCK_DPE = 32 + else: + BLOCK_DMODEL = triton.next_power_of_2(Lk) + BLOCK_DPE = 0 + BLOCK_DV = triton.next_power_of_2(Lv) + + batch, head_num = q.shape[0], q.shape[1] + kv_group_num = q.shape[1] // k_buffer.shape[1] + + BLOCK_H = 16 + MAX_KV_SPLITS = max_kv_splits + grid = ( + batch, + triton.cdiv(head_num, min(BLOCK_H, kv_group_num)), + MAX_KV_SPLITS, + ) + + extra_kargs = {} + num_stages = 2 + if _is_hip: + # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html + # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py + extra_kargs = {"waves_per_eu": 1, "matrix_instr_nonkdim": 16, "kpack": 2} + num_stages = 1 + + _fwd_grouped_kernel_stage1[grid]( + q, + k_buffer, + v_buffer, + sm_scale, + kv_indptr, + kv_indices, + att_out, + att_lse, + num_kv_splits, + q.stride(0), + q.stride(1), + k_buffer.stride(0), + k_buffer.stride(1), + v_buffer.stride(0), + v_buffer.stride(1), + att_out.stride(0), + att_out.stride(1), + att_out.stride(2), + kv_group_num=kv_group_num, + q_head_num=head_num, + BLOCK_DMODEL=BLOCK_DMODEL, + BLOCK_DPE=BLOCK_DPE, + BLOCK_DV=BLOCK_DV, + BLOCK_N=BLOCK, + BLOCK_H=BLOCK_H, + MIN_BLOCK_KV=_MIN_BLOCK_KV, + logit_cap=logit_cap, + xai_temperature_len=xai_temperature_len, + num_warps=4, + num_stages=num_stages, + Lk=Lk, + Lv=Lv, + **extra_kargs, + ) + + +@triton.jit +def _fwd_kernel_stage2( + Mid_O, + Mid_O_1, + O, + kv_indptr, + num_kv_splits, + sink_ptr, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_obs, + stride_oh, + MAX_KV_SPLITS: tl.constexpr, + MIN_BLOCK_KV: tl.constexpr, + BLOCK_DV: tl.constexpr, + Lv: tl.constexpr, + HAS_SINK: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + + cur_batch_seq_len = tl.load(kv_indptr + cur_batch + 1) - tl.load( + kv_indptr + cur_batch + ) + kv_splits = tl.load(num_kv_splits + cur_batch) + + offs_d = tl.arange(0, BLOCK_DV) + mask_d = offs_d < Lv + + e_sum = 0.0 + e_max = -float("inf") + acc = tl.zeros([BLOCK_DV], dtype=tl.float32) + + offs_v = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + offs_d + offs_logic = (cur_batch * stride_mid_ob + cur_head * stride_mid_oh) // Lv + kv_len_per_split = ( + tl.cdiv(tl.cdiv(cur_batch_seq_len, kv_splits), MIN_BLOCK_KV) * MIN_BLOCK_KV + ) + + for split_kv_id in range(0, MAX_KV_SPLITS): + split_kv_start = kv_len_per_split * split_kv_id + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split, cur_batch_seq_len) + + if split_kv_end > split_kv_start: + tv = tl.load( + Mid_O + offs_v + split_kv_id * stride_mid_os, mask=mask_d, other=0.0 + ) + tlogic = tl.load(Mid_O_1 + offs_logic + split_kv_id * stride_mid_os // Lv) + n_e_max = tl.maximum(tlogic, e_max) + + old_scale = tl.exp(e_max - n_e_max) + acc *= old_scale + exp_logic = tl.exp(tlogic - n_e_max) + acc += exp_logic * tv + + e_sum = e_sum * old_scale + exp_logic + e_max = n_e_max + + if HAS_SINK: + cur_sink = tl.load(sink_ptr + cur_head) + e_sum += tl.exp(cur_sink - e_max) + + tl.store( + O + cur_batch * stride_obs + cur_head * stride_oh + offs_d, + acc / e_sum, + mask=mask_d, + ) + + +def _decode_softmax_reducev_fwd( + logits, + lse, + q, + o, + v_buffer, + kv_indptr, + num_kv_splits, + max_kv_splits, + sinks=None, +): + batch, head_num = q.shape[0], q.shape[1] + Lv = v_buffer.shape[-1] + BLOCK_DV = triton.next_power_of_2(Lv) + + MAX_KV_SPLITS = max_kv_splits + HAS_SINK = sinks is not None + + extra_kargs = {} + if _is_hip: + # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html + # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py + extra_kargs = {"waves_per_eu": 4, "matrix_instr_nonkdim": 16, "kpack": 2} + + grid = (batch, head_num) + _fwd_kernel_stage2[grid]( + logits, + lse, + o, + kv_indptr, + num_kv_splits, + sinks, + logits.stride(0), + logits.stride(1), + logits.stride(2), + o.stride(0), + o.stride(1), + MAX_KV_SPLITS=MAX_KV_SPLITS, + MIN_BLOCK_KV=_MIN_BLOCK_KV, + BLOCK_DV=BLOCK_DV, + Lv=Lv, + HAS_SINK=HAS_SINK, + num_warps=4, + num_stages=2, + **extra_kargs, + ) + + +def decode_attention_fwd_normal( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=None, + xai_temperature_len=-1, +): + _decode_att_m_fwd( + q, + k_buffer, + v_buffer, + attn_logits, + attn_lse, + kv_indptr, + kv_indices, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + xai_temperature_len, + ) + _decode_softmax_reducev_fwd( + attn_logits, + attn_lse, + q, + o, + v_buffer, + kv_indptr, + num_kv_splits, + max_kv_splits, + sinks, + ) + + +def decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=None, + xai_temperature_len=-1, +): + _decode_grouped_att_m_fwd( + q, + k_buffer, + v_buffer, + attn_logits, + attn_lse, + kv_indptr, + kv_indices, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + xai_temperature_len, + ) + _decode_softmax_reducev_fwd( + attn_logits, + attn_lse, + q, + o, + v_buffer, + kv_indptr, + num_kv_splits, + max_kv_splits, + sinks, + ) + + +def decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, + sinks=None, + xai_temperature_len=-1, +): + assert max_kv_splits == attn_logits.shape[2] + assert q.shape[0] <= kv_indptr.shape[0] - 1 + assert q.shape[0] <= attn_logits.shape[0] + + kv_group_num = q.shape[1] // v_buffer.shape[1] + + if kv_group_num == 1: + # MHA + decode_attention_fwd_normal( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=logit_cap, + sinks=sinks, + xai_temperature_len=xai_temperature_len, + ) + else: + # GQA/MQA/MLA + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=logit_cap, + sinks=sinks, + xai_temperature_len=xai_temperature_len, + ) diff --git a/python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py b/python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py new file mode 100644 index 000000000..72e0bfe78 --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/double_sparsity_attention.py @@ -0,0 +1,1106 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +if _is_cuda: + CUDA_CAPABILITY = torch.cuda.get_device_capability() + +_is_hip = is_hip() + +if global_server_args_dict.get("attention_reduce_in_fp32", False): + REDUCE_TRITON_TYPE = tl.float32 + REDUCE_TORCH_TYPE = torch.float32 +else: + REDUCE_TRITON_TYPE = tl.float16 + REDUCE_TORCH_TYPE = torch.float16 + + +@triton.jit +def tanh(x): + # Tanh is just a scaled sigmoid + return 2 * tl.sigmoid(2 * x) - 1 + + +@triton.jit +def _fwd_kernel_flash_decode_stage1( + Q, + K, + V, + sm_scale, + Req_to_tokens, + B_req_idx, + B_Seqlen, + Mid_O, # [batch, head, seq_block_num, head_dim] + Mid_O_LogExpSum, # [batch, head, seq_block_num] + stride_req_to_tokens_b, + stride_req_to_tokens_s, + stride_qbs, + stride_qh, + stride_qd, + stride_kbs, + stride_kh, + stride_kd, + stride_vbs, + stride_vh, + stride_vd, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_mid_od, + stride_mid_o_eb, + stride_mid_o_eh, + stride_mid_o_es, + gqa_group_size, + BLOCK_SEQ: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + seq_start_block = tl.program_id(2) + cur_kv_head = cur_head // gqa_group_size + + offs_d = tl.arange(0, BLOCK_DMODEL) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_req_idx = tl.load(B_req_idx + cur_batch) + cur_batch_start_index = seq_start_block * BLOCK_SEQ + cur_batch_end_index = tl.minimum( + cur_batch_seq_len, cur_batch_start_index + BLOCK_SEQ + ) + + off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d + + block_n_size = ( + tl.where( + cur_batch_end_index - cur_batch_start_index <= 0, + 0, + cur_batch_end_index - cur_batch_start_index + BLOCK_N - 1, + ) + // BLOCK_N + ) + + offs_n = cur_batch_start_index + tl.arange(0, BLOCK_N) + + q = tl.load(Q + off_q) + + sum_exp = 0.0 + max_logic = -float("inf") + acc = tl.zeros([BLOCK_DMODEL], dtype=tl.float32) + + for start_n in range(0, block_n_size, 1): + offs_n_new = start_n * BLOCK_N + offs_n + k_loc = tl.load( + Req_to_tokens + stride_req_to_tokens_b * cur_batch_req_idx + offs_n_new, + mask=offs_n_new < cur_batch_end_index, + other=0, + ) + off_k = k_loc[:, None] * stride_kbs + cur_kv_head * stride_kh + offs_d[None, :] + k = tl.load( + K + off_k, mask=offs_n_new[:, None] < cur_batch_end_index, other=0.0 + ) + att_value = tl.sum(q[None, :] * k, 1) + att_value *= sm_scale + att_value = tl.where(offs_n_new < cur_batch_end_index, att_value, float("-inf")) + v = tl.load( + V + off_k, mask=offs_n_new[:, None] < cur_batch_end_index, other=0.0 + ) + + cur_max_logic = tl.max(att_value, axis=0) + new_max_logic = tl.maximum(cur_max_logic, max_logic) + + exp_logic = tl.exp(att_value - new_max_logic) + logic_scale = tl.exp(max_logic - new_max_logic) + acc *= logic_scale + acc += tl.sum(exp_logic[:, None] * v, axis=0) + + sum_exp = sum_exp * logic_scale + tl.sum(exp_logic, axis=0) + max_logic = new_max_logic + + need_store = tl.where(block_n_size == 0, 0, 1) + for _ in range(0, need_store, 1): + off_mid_o = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + seq_start_block * stride_mid_os + + offs_d + ) + off_mid_o_logexpsum = ( + cur_batch * stride_mid_o_eb + cur_head * stride_mid_o_eh + seq_start_block + ) + tl.store(Mid_O + off_mid_o, acc / sum_exp) + tl.store(Mid_O_LogExpSum + off_mid_o_logexpsum, max_logic + tl.log(sum_exp)) + return + + +@triton.jit +def _fwd_kernel_flash_decode_stage2( + B_Seqlen, + Mid_O, # [batch, head, seq_block_num, head_dim] + Mid_O_LogExpSum, # [batch, head, seq_block_num] + O, # [batch, head, head_dim] + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_mid_od, + stride_mid_o_eb, + stride_mid_o_eh, + stride_mid_o_es, + stride_obs, + stride_oh, + stride_od, + BLOCK_SEQ: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + + offs_d = tl.arange(0, BLOCK_DMODEL) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + + block_n_size = ( + tl.where(cur_batch_seq_len <= 0, 0, cur_batch_seq_len + BLOCK_SEQ - 1) + // BLOCK_SEQ + ) + + sum_exp = 0.0 + max_logic = -float("inf") + acc = tl.zeros([BLOCK_DMODEL], dtype=tl.float32) + + offs_v = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + offs_d + offs_logic = cur_batch * stride_mid_o_eb + cur_head * stride_mid_o_eh + for block_seq_n in range(0, block_n_size, 1): + tv = tl.load(Mid_O + offs_v + block_seq_n * stride_mid_os) + tlogic = tl.load(Mid_O_LogExpSum + offs_logic + block_seq_n) + new_max_logic = tl.maximum(tlogic, max_logic) + + old_scale = tl.exp(max_logic - new_max_logic) + acc *= old_scale + exp_logic = tl.exp(tlogic - new_max_logic) + acc += exp_logic * tv + sum_exp = sum_exp * old_scale + exp_logic + max_logic = new_max_logic + + tl.store(O + cur_batch * stride_obs + cur_head * stride_oh + offs_d, acc / sum_exp) + return + + +@torch.no_grad() +def flash_decode_stage1( + q, + k, + v, + Req_to_tokens, + B_req_idx, + B_Seqlen, + max_len_in_batch, + mid_out, + mid_out_logsumexp, + block_seq, +): + BLOCK_SEQ = block_seq + BLOCK_N = 16 + assert BLOCK_SEQ % BLOCK_N == 0 + # shape constraints + Lq, Lk = q.shape[-1], k.shape[-1] + assert Lq == Lk + assert Lk in {16, 32, 64, 128} + sm_scale = 1.0 / (Lk**0.5) + batch, head_num = B_req_idx.shape[0], q.shape[1] + grid = (batch, head_num, triton.cdiv(max_len_in_batch, BLOCK_SEQ)) + gqa_group_size = q.shape[1] // k.shape[1] + + _fwd_kernel_flash_decode_stage1[grid]( + q, + k, + v, + sm_scale, + Req_to_tokens, + B_req_idx, + B_Seqlen, + mid_out, + mid_out_logsumexp, + Req_to_tokens.stride(0), + Req_to_tokens.stride(1), + q.stride(0), + q.stride(1), + q.stride(2), + k.stride(0), + k.stride(1), + k.stride(2), + v.stride(0), + v.stride(1), + v.stride(2), + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_out.stride(3), + mid_out_logsumexp.stride(0), + mid_out_logsumexp.stride(1), + mid_out_logsumexp.stride(2), + gqa_group_size, + BLOCK_SEQ=BLOCK_SEQ, + BLOCK_DMODEL=Lk, + BLOCK_N=BLOCK_N, + num_warps=1, + num_stages=2, + ) + return + + +@torch.no_grad() +def flash_decode_stage2(mid_out, mid_out_logexpsum, B_Seqlen, O, block_seq): + Lk = mid_out.shape[-1] + assert Lk in {16, 32, 64, 128} + batch, head_num = mid_out.shape[0], mid_out.shape[1] + grid = (batch, head_num) + + _fwd_kernel_flash_decode_stage2[grid]( + B_Seqlen, + mid_out, + mid_out_logexpsum, + O, + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_out.stride(3), + mid_out_logexpsum.stride(0), + mid_out_logexpsum.stride(1), + mid_out_logexpsum.stride(2), + O.stride(0), + O.stride(1), + O.stride(2), + BLOCK_SEQ=block_seq, + BLOCK_DMODEL=Lk, + num_warps=4, + num_stages=2, + ) + return + + +def flash_decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + req_to_token, + b_req_idx, + b_start_loc, + b_seq_len, + attn_logits, + max_len_in_batch, + sm_scale, + logit_cap=0.0, +): + BLOCK_SEQ = 256 + kv_group_num = q.shape[1] // v_buffer.shape[1] + # batch_size = q.shape[0] + + block_seq_num = (max_len_in_batch + BLOCK_SEQ - 1) // BLOCK_SEQ + + mid_o = torch.empty( + [q.shape[0], q.shape[1], block_seq_num, q.shape[-1]], + dtype=torch.float32, + device="cuda", + ) + mid_o_logexpsum = torch.empty( + [q.shape[0], q.shape[1], block_seq_num], dtype=torch.float32, device="cuda" + ) + + flash_decode_stage1( + q, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + b_seq_len, + max_len_in_batch, + mid_o, + mid_o_logexpsum, + BLOCK_SEQ, + ) + flash_decode_stage2(mid_o, mid_o_logexpsum, b_seq_len, o, BLOCK_SEQ) + + +@triton.jit +def _sparse_fwd_kernel_flash_decode_stage1( # Double Sparsity's approximate attention + Q_Label, + K_Label_Buffer, + sm_scale, + Req_to_tokens, # shape: [B, S] + B_Seqlen, + Att_Out, # shape: [H, B, S] easier for topk + stride_req_to_tokens_b, + stride_qbs, + stride_qh, + stride_buf_kbs, + stride_buf_kh, + att_stride_h, + att_stride_b, + kv_group_num: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + logit_cap: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_n = tl.program_id(2) + + cur_kv_head = cur_head // kv_group_num + + offs_d = tl.arange(0, BLOCK_DMODEL) + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + + cur_batch_start_index = 0 + cur_batch_end_index = cur_batch_seq_len + + min_val = -float("inf") + att_value = tl.full([BLOCK_N], min_val, dtype=tl.float32) + + off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d + + offs_n = start_n * BLOCK_N + tl.arange(0, BLOCK_N) + + block_index = start_n * BLOCK_N + block_mask = tl.where(block_index < cur_batch_seq_len, 1, 0) + + for start_mark in range(0, block_mask, 1): + q = tl.load(Q_Label + off_q + start_mark).to(REDUCE_TRITON_TYPE) + offs_n_new = cur_batch_start_index + offs_n + k_loc = tl.load( + Req_to_tokens + stride_req_to_tokens_b * cur_batch + offs_n_new, + mask=offs_n_new < cur_batch_end_index, + other=0, + ) + offs_buf_k = ( + k_loc[:, None] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_d[None, :] + ) + k = tl.load( + K_Label_Buffer + offs_buf_k, + mask=offs_n_new[:, None] < cur_batch_end_index, + other=0.0, + ).to(REDUCE_TRITON_TYPE) + + att_value = tl.sum(q[None, :] * k, 1) + att_value *= sm_scale + + if logit_cap > 0: + att_value = logit_cap * tanh(att_value / logit_cap) + + att_value = tl.where(offs_n < cur_batch_end_index, att_value, min_val) + off_o = cur_head * att_stride_h + (cur_batch * att_stride_b + offs_n) + tl.store(Att_Out + off_o, att_value) + + +@triton.jit +def _sparse_fwd_kernel_flash_decode_stage2( + Q, + K, + V, + sm_scale, + Req_to_tokens, # shape: [B, S] + Topk_token_indices, # shape: [H, B, k] + Mid_O, # [batch, head, seq_block_num, head_dim] + Mid_O_LogExpSum, # [batch, head, seq_block_num] + Heavy_token_num, # NOTE: This can be used as constexpr but we may support dynamic heavy token number in the future + stride_req_to_tokens_b, + stride_topk_token_indices_h, + stride_topk_token_indices_b, + stride_qbs, + stride_qh, + stride_kbs, + stride_kh, + stride_vbs, + stride_vh, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_mid_o_eb, + stride_mid_o_eh, + gqa_group_size, + BLOCK_SEQ: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + seq_start_block = tl.program_id(2) + cur_kv_head = cur_head // gqa_group_size + + offs_d = tl.arange(0, BLOCK_DMODEL) + cur_batch_start_index = seq_start_block * BLOCK_SEQ + cur_batch_end_index = tl.minimum(Heavy_token_num, cur_batch_start_index + BLOCK_SEQ) + + off_q = cur_batch * stride_qbs + cur_head * stride_qh + offs_d + + block_n_size = ( + tl.where( + cur_batch_end_index - cur_batch_start_index <= 0, + 0, + cur_batch_end_index - cur_batch_start_index + BLOCK_N - 1, + ) + // BLOCK_N + ) + + # offs_n = cur_batch_start_index + tl.arange(0, BLOCK_N) + offs_n = tl.arange(0, BLOCK_N) + + q = tl.load(Q + off_q) + + sum_exp = 0.0 + max_logic = -float("inf") + acc = tl.zeros([BLOCK_DMODEL], dtype=tl.float32) + + for start_n in range(cur_batch_start_index, cur_batch_end_index, BLOCK_N): + # for start_n in range(0, block_n_size, 1): + # offs_n_new = start_n * BLOCK_N + offs_n + offs_n_new = start_n + offs_n + # offs_n_new = cur_batch_start_index + start_n * BLOCK_N + offs_n + topk_token_indices = tl.load( + Topk_token_indices + + stride_topk_token_indices_h * cur_head + + stride_topk_token_indices_b * cur_batch + + offs_n_new, + mask=offs_n_new < cur_batch_end_index, + other=0, + ) + k_loc = tl.load( + Req_to_tokens + stride_req_to_tokens_b * cur_batch + topk_token_indices, + mask=offs_n_new < cur_batch_end_index, + other=0, + ) + off_k = k_loc[:, None] * stride_kbs + cur_kv_head * stride_kh + offs_d[None, :] + k = tl.load( + K + off_k, mask=offs_n_new[:, None] < cur_batch_end_index, other=0.0 + ) + att_value = tl.sum(q[None, :] * k, 1) + att_value *= sm_scale + att_value = tl.where(offs_n_new < cur_batch_end_index, att_value, float("-inf")) + v = tl.load( + V + off_k, mask=offs_n_new[:, None] < cur_batch_end_index, other=0.0 + ) + + cur_max_logic = tl.max(att_value, axis=0) + new_max_logic = tl.maximum(cur_max_logic, max_logic) + + exp_logic = tl.exp(att_value - new_max_logic) + logic_scale = tl.exp(max_logic - new_max_logic) + acc *= logic_scale + acc += tl.sum(exp_logic[:, None] * v, axis=0) + + sum_exp = sum_exp * logic_scale + tl.sum(exp_logic, axis=0) + max_logic = new_max_logic + + # need_store = tl.where(block_n_size == 0, 0, 1) + need_store = 1 + for _ in range(0, need_store, 1): + off_mid_o = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + seq_start_block * stride_mid_os + + offs_d + ) + off_mid_o_logexpsum = ( + cur_batch * stride_mid_o_eb + cur_head * stride_mid_o_eh + seq_start_block + ) + tl.store(Mid_O + off_mid_o, acc / sum_exp) + tl.store(Mid_O_LogExpSum + off_mid_o_logexpsum, max_logic + tl.log(sum_exp)) + return + + +@triton.jit +def _sparse_fwd_kernel_flash_decode_stage3( + Mid_O, # [batch, head, seq_block_num, head_dim] + Mid_O_LogExpSum, # [batch, head, seq_block_num] + O, # [batch, head, head_dim] + seq_len, # NOTE: This can be used as constexpr but we may support dynamic heavy token number in the future + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_mid_o_eb, + stride_mid_o_eh, + stride_obs, + stride_oh, + BLOCK_SEQ: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + + offs_d = tl.arange(0, BLOCK_DMODEL) + + block_n_size = tl.where(seq_len <= 0, 0, seq_len + BLOCK_SEQ - 1) // BLOCK_SEQ + + sum_exp = 0.0 + max_logic = -float("inf") + acc = tl.zeros([BLOCK_DMODEL], dtype=tl.float32) + + offs_v = cur_batch * stride_mid_ob + cur_head * stride_mid_oh + offs_d + offs_logic = cur_batch * stride_mid_o_eb + cur_head * stride_mid_o_eh + for block_seq_n in range(0, block_n_size, 1): + tv = tl.load(Mid_O + offs_v + block_seq_n * stride_mid_os) + tlogic = tl.load(Mid_O_LogExpSum + offs_logic + block_seq_n) + new_max_logic = tl.maximum(tlogic, max_logic) + + old_scale = tl.exp(max_logic - new_max_logic) + acc *= old_scale + exp_logic = tl.exp(tlogic - new_max_logic) + acc += exp_logic * tv + sum_exp = sum_exp * old_scale + exp_logic + max_logic = new_max_logic + + tl.store(O + cur_batch * stride_obs + cur_head * stride_oh + offs_d, acc / sum_exp) + return + + +def sparse_flash_decode_stage1( + q_label, + k_label_buffer, + att_out, + Req_to_tokens, + B_Seqlen, + max_len_in_batch, + sm_scale, + logit_cap, +): + BLOCK = 32 + # shape constraints + Lq, Lk = q_label.shape[-1], k_label_buffer.shape[-1] + assert Lq == Lk + assert Lk in {16, 32, 64, 128, 256, 576} + + BLOCK_DMODEL = Lk + + batch, head_num = q_label.shape[0], q_label.shape[1] + + grid = (batch, head_num, triton.cdiv(max_len_in_batch, BLOCK)) + kv_group_num = q_label.shape[1] // k_label_buffer.shape[1] + + if kv_group_num == 1: + num_warps = 4 + else: + num_warps = 2 + + _sparse_fwd_kernel_flash_decode_stage1[grid]( + q_label, + k_label_buffer, + sm_scale, + Req_to_tokens, + B_Seqlen, + att_out, + Req_to_tokens.stride(0), + q_label.stride(0), + q_label.stride(1), + k_label_buffer.stride(0), + k_label_buffer.stride(1), + att_out.stride(0), + att_out.stride(1), + kv_group_num, + BLOCK_DMODEL, + BLOCK, + logit_cap, + num_warps=num_warps, + num_stages=1, + ) + + +@torch.no_grad() +def sparse_flash_decode_stage2( + q, + k, + v, + Req_to_tokens, + Topk_token_indices, + heavy_token_num, + mid_out, + mid_out_logsumexp, + block_seq, + sm_scale, +): + BLOCK_SEQ = block_seq + BLOCK_N = 16 + assert BLOCK_SEQ % BLOCK_N == 0 + # shape constraints + Lq, Lk = q.shape[-1], k.shape[-1] + assert Lq == Lk + assert Lk in {16, 32, 64, 128} + assert heavy_token_num == Topk_token_indices.shape[-1] + # sm_scale = 1.0 / (Lk ** 0.5) + batch, head_num = q.shape[0], q.shape[1] + grid = (batch, head_num, triton.cdiv(heavy_token_num, BLOCK_SEQ)) + + gqa_group_size = q.shape[1] // k.shape[1] + + _sparse_fwd_kernel_flash_decode_stage2[grid]( + q, + k, + v, + sm_scale, + Req_to_tokens, + Topk_token_indices, + mid_out, + mid_out_logsumexp, + heavy_token_num, + Req_to_tokens.stride(0), + Topk_token_indices.stride(0), + Topk_token_indices.stride(1), + q.stride(0), + q.stride(1), + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_out_logsumexp.stride(0), + mid_out_logsumexp.stride(1), + gqa_group_size, + BLOCK_SEQ=BLOCK_SEQ, + BLOCK_DMODEL=Lk, + BLOCK_N=BLOCK_N, + num_warps=1, + num_stages=2, + ) + return + + +@torch.no_grad() +def sparse_flash_decode_stage3(Seqlen, mid_out, mid_out_logexpsum, O, block_seq): + Lk = mid_out.shape[-1] + assert Lk in {16, 32, 64, 128} + batch, head_num = mid_out.shape[0], mid_out.shape[1] + grid = (batch, head_num) + + _sparse_fwd_kernel_flash_decode_stage3[grid]( + mid_out, + mid_out_logexpsum, + O, + Seqlen, + mid_out.stride(0), + mid_out.stride(1), + mid_out.stride(2), + mid_out_logexpsum.stride(0), + mid_out_logexpsum.stride(1), + O.stride(0), + O.stride(1), + BLOCK_SEQ=block_seq, + BLOCK_DMODEL=Lk, + num_warps=4, + num_stages=2, + ) + return + + +def flash_decode_sparse_attention_fwd( + q, + k_buffer, + v_buffer, + o, + q_label, + k_label_buffer, + req_to_token, + b_seq_len, + max_len_in_batch, + sm_scale, + logit_cap, + heavy_token_num=32, + att_out_approx=None, + mid_out=None, + mid_o_logexpsum=None, + BLOCK_SEQ=256, +): + # TODO(Andy): Tune BLOCK_SEQ & BLOCK_D + kv_group_num = q.shape[1] // v_buffer.shape[1] + # batch_size = q.shape[0] + + # Step 1: BGEMV approximate attention (page implementation) + + if att_out_approx is None: + att_out_approx = torch.empty( + [q.shape[1], q.shape[0], max_len_in_batch], + dtype=REDUCE_TORCH_TYPE, + device=q.device, + ) + + if mid_out is None: + block_seq_num = (heavy_token_num + BLOCK_SEQ - 1) // BLOCK_SEQ + + mid_out = torch.empty( + [q.shape[0], q.shape[1], block_seq_num, q.shape[-1]], + dtype=torch.float32, + device=q.device, + ) + mid_o_logexpsum = torch.empty( + [q.shape[0], q.shape[1], block_seq_num], + dtype=torch.float32, + device=q.device, + ) + + sparse_flash_decode_stage1( + q_label, + k_label_buffer, + att_out_approx, + req_to_token, + b_seq_len, + max_len_in_batch, + sm_scale, + logit_cap, + ) + + # Step 2: TopK token selection + # NOTE(Andy): Apply sparse decoding when min > heavy_token_num and max > sparse decoding threshold + # TODO(Andy): Change a faster topk implementation + topk_token_indices = torch.topk(att_out_approx, heavy_token_num, dim=-1).indices + # topk_token_indices: [H, B, k], Req_to_tokens: [B, S] + # topk_token_indices = torch.arange(0, heavy_token_num, device=q.device).unsqueeze(0).unsqueeze(0).expand(q.shape[1], q.shape[0], -1) + + sparse_flash_decode_stage2( + q, + k_buffer, + v_buffer, + req_to_token, + topk_token_indices, + heavy_token_num, + mid_out, + mid_o_logexpsum, + BLOCK_SEQ, + sm_scale, + ) + + sparse_flash_decode_stage3(heavy_token_num, mid_out, mid_o_logexpsum, o, BLOCK_SEQ) + + +# Extend attention kernel for Double Sparsity +# Moved from https://github.com/sgl-project/sglang/blob/v0.4.2.post1/python/sglang/srt/layers/attention/triton_ops/extend_attention.py +@triton.jit +def _fwd_kernel( + Q_Extend, + K_Extend, + V_Extend, + O_Extend, + K_Buffer, + V_Buffer, + Req_to_tokens, + B_req_idx, + B_Seq_Len, + B_Start_Loc_Extend, + B_Seq_Len_Extend, + sm_scale, + kv_group_num, + stride_qbs, + stride_qh, + stride_kbs, + stride_kh, + stride_vbs, + stride_vh, + stride_obs, + stride_oh, + stride_buf_kbs, + stride_buf_kh, + stride_buf_vbs, + stride_buf_vh, + stride_req_to_tokens_b, + logit_cap: tl.constexpr, + Lq: tl.constexpr, + Lv: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_DPE: tl.constexpr, + BLOCK_DV: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + cur_seq = tl.program_id(0) + cur_head = tl.program_id(1) + cur_block_m = tl.program_id(2) + cur_kv_head = cur_head // kv_group_num + + cur_seq_len = tl.load(B_Seq_Len + cur_seq) + cur_seq_len_extend = tl.load(B_Seq_Len_Extend + cur_seq) + cur_seq_len_prefix = cur_seq_len - cur_seq_len_extend + + cur_seq_prefix_start_in_loc = 0 + cur_seq_extend_start_contiguous = tl.load(B_Start_Loc_Extend + cur_seq) + cur_batch_req_idx = tl.load(B_req_idx + cur_seq) + + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_dv = tl.arange(0, BLOCK_DV) + offs_m = tl.arange(0, BLOCK_M) + mask_m = (cur_block_m * BLOCK_M + offs_m) < cur_seq_len_extend + + mask_d = offs_d < Lq + mask_dv = offs_dv < Lv + + offs_q = ( + (cur_seq_extend_start_contiguous + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_qbs + + cur_head * stride_qh + + offs_d[None, :] + ) + q = tl.load( + Q_Extend + offs_q, mask=(mask_m[:, None]) & (mask_d[None, :]), other=0.0 + ) + + if BLOCK_DPE > 0: + offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE) + offs_qpe = ( + (cur_seq_extend_start_contiguous + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_qbs + + cur_head * stride_qh + + offs_dpe[None, :] + ) + qpe = tl.load(Q_Extend + offs_qpe, mask=mask_m[:, None], other=0.0) + + # stage 1: compute scores with prefix + offs_n = tl.arange(0, BLOCK_N) + + acc = tl.zeros([BLOCK_M, BLOCK_DV], dtype=tl.float32) + deno = tl.zeros([BLOCK_M], dtype=tl.float32) + e_max = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + + for start_n in range(0, cur_seq_len_prefix, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + mask_n = (start_n + offs_n) < cur_seq_len_prefix + offs_b_loc_prefix = cur_batch_req_idx * stride_req_to_tokens_b + ( + cur_seq_prefix_start_in_loc + start_n + offs_n + ) + offs_kv_loc = tl.load(Req_to_tokens + offs_b_loc_prefix, mask=mask_n, other=0) + + # load k in transposed way + offs_buf_k = ( + offs_kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_d[:, None] + ) + k = tl.load( + K_Buffer + offs_buf_k, mask=(mask_n[None, :]) & (mask_d[:, None]), other=0.0 + ) + + qk = tl.dot(q.to(k.dtype), k) + if BLOCK_DPE > 0: + offs_kpe = ( + offs_kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_dpe[:, None] + ) + kpe = tl.load( + K_Buffer + offs_kpe, + mask=mask_n[None, :], + other=0.0, + ) + qk += tl.dot(qpe.to(kpe.dtype), kpe) + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + qk = tl.where(mask_m[:, None] & mask_n[None, :], qk, float("-inf")) + + n_e_max = tl.maximum(tl.max(qk, 1), e_max) + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + deno = deno * re_scale + tl.sum(p, 1) + + offs_buf_v = ( + offs_kv_loc[:, None] * stride_buf_vbs + + cur_kv_head * stride_buf_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Buffer + offs_buf_v, mask=mask_n[:, None] & mask_dv[None, :], other=0.0 + ) + p = p.to(v.dtype) + acc = acc * re_scale[:, None] + tl.dot(p, v) + + e_max = n_e_max + + # stage 2: compute the triangle part + + cur_block_m_end = tl.minimum(cur_seq_len_extend, (cur_block_m + 1) * BLOCK_M) + for start_n in range(0, cur_block_m_end, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + mask_n = (start_n + offs_n) < cur_block_m_end + + # load k in transposed way + offs_k = ( + (cur_seq_extend_start_contiguous + start_n + offs_n[None, :]) * stride_kbs + + cur_kv_head * stride_kh + + offs_d[:, None] + ) + k = tl.load( + K_Extend + offs_k, mask=(mask_n[None, :]) & (mask_d[:, None]), other=0.0 + ) + + qk = tl.dot(q, k, out_dtype=tl.float32) + if BLOCK_DPE > 0: + offs_kpe = ( + (cur_seq_extend_start_contiguous + start_n + offs_n[None, :]) + * stride_kbs + + cur_kv_head * stride_kh + + offs_dpe[:, None] + ) + kpe = tl.load( + K_Extend + offs_kpe, + mask=mask_n[None, :], + other=0.0, + ) + qk += tl.dot(qpe, kpe) + + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + mask_causual = (cur_block_m * BLOCK_M + offs_m[:, None]) >= ( + start_n + offs_n[None, :] + ) + mask_causual &= mask_m[:, None] & mask_n[None, :] + qk = tl.where(mask_causual, qk, float("-inf")) + + n_e_max = tl.maximum(tl.max(qk, 1), e_max) + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + deno = deno * re_scale + tl.sum(p, 1) + + offs_v = ( + (cur_seq_extend_start_contiguous + start_n + offs_n[:, None]) * stride_vbs + + cur_kv_head * stride_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Extend + offs_v, mask=mask_n[:, None] & mask_dv[None, :], other=0.0 + ) + p = p.to(v.dtype) + acc = acc * re_scale[:, None] + tl.dot(p, v) + + e_max = n_e_max + + offs_o = ( + (cur_seq_extend_start_contiguous + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_obs + + cur_head * stride_oh + + offs_dv[None, :] + ) + tl.store( + O_Extend + offs_o, acc / deno[:, None], mask=mask_m[:, None] & mask_dv[None, :] + ) + + +def extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_seq_len_extend, + b_start_loc_extend, + max_len_extend, + sm_scale=None, + logit_cap=0.0, +): + """ + q_extend, k_extend, v_extend, o_extend: contiguous tensors + + k_buffer, v_buffer: (prefix + extend) tensors in mem_manager + """ + Lq, Lk, Lv = ( + q_extend.shape[-1], + k_extend.shape[-1], + v_extend.shape[-1], + ) + + if Lq == 576: + BLOCK_DMODEL = 512 + BLOCK_DPE = 64 + elif Lq == 288: + BLOCK_DMODEL = 256 + BLOCK_DPE = 32 + elif Lq == 192: + BLOCK_DMODEL = 128 + BLOCK_DPE = 64 + else: + BLOCK_DMODEL = triton.next_power_of_2(Lq) + BLOCK_DPE = 0 + BLOCK_DV = triton.next_power_of_2(Lv) + + if _is_hip: + BLOCK_M, BLOCK_N = (64, 64) + num_warps = 4 + + else: + if _is_cuda and CUDA_CAPABILITY[0] >= 9: + if Lq <= 256: + BLOCK_M, BLOCK_N = (128, 64) + else: + BLOCK_M, BLOCK_N = (32, 64) + elif _is_cuda and CUDA_CAPABILITY[0] >= 8: + if Lq <= 128: + BLOCK_M, BLOCK_N = (128, 128) + elif Lq <= 256: + BLOCK_M, BLOCK_N = (64, 64) + else: + BLOCK_M, BLOCK_N = (32, 64) + else: + BLOCK_M, BLOCK_N = (64, 64) if Lq <= 128 else (32, 32) + + num_warps = 4 if Lk <= 64 else 8 + + sm_scale = sm_scale or 1.0 / (Lq**0.5) + batch_size, head_num = b_seq_len.shape[0], q_extend.shape[1] + kv_group_num = q_extend.shape[1] // k_extend.shape[1] + + grid = (batch_size, head_num, triton.cdiv(max_len_extend, BLOCK_M)) + num_stages = 1 + + extra_kargs = {} + if _is_hip: + extra_kargs = {"waves_per_eu": 4, "matrix_instr_nonkdim": 16, "kpack": 2} + + _fwd_kernel[grid]( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_start_loc_extend, + b_seq_len_extend, + sm_scale, + kv_group_num, + q_extend.stride(0), + q_extend.stride(1), + k_extend.stride(0), + k_extend.stride(1), + v_extend.stride(0), + v_extend.stride(1), + o_extend.stride(0), + o_extend.stride(1), + k_buffer.stride(0), + k_buffer.stride(1), + v_buffer.stride(0), + v_buffer.stride(1), + req_to_tokens.stride(0), + logit_cap=logit_cap, + BLOCK_DMODEL=BLOCK_DMODEL, + BLOCK_DPE=BLOCK_DPE, + BLOCK_DV=BLOCK_DV, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + Lq=Lq, + Lv=Lv, + num_warps=num_warps, + num_stages=num_stages, + **extra_kargs, + ) diff --git a/python/sglang/srt/layers/attention/triton_ops/extend_attention.py b/python/sglang/srt/layers/attention/triton_ops/extend_attention.py new file mode 100644 index 000000000..e91467743 --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/extend_attention.py @@ -0,0 +1,550 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Memory-efficient attention for prefill. +It supports page size = 1 and prefill with KV cache (i.e. extend). +""" + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +if _is_cuda: + CUDA_CAPABILITY = torch.cuda.get_device_capability() + +_is_hip = is_hip() + + +@triton.jit +def tanh(x): + # Tanh is just a scaled sigmoid + return 2 * tl.sigmoid(2 * x) - 1 + + +@triton.jit +def _fwd_kernel( + Q_Extend, + K_Extend, + V_Extend, + O_Extend, + K_Buffer, + V_Buffer, + qo_indptr, + kv_indptr, + kv_indices, + mask_ptr, + mask_indptr, + sink_ptr, + window_kv_offset_ptr, + sm_scale, + kv_group_num, + stride_qbs, + stride_qh, + stride_kbs, + stride_kh, + stride_vbs, + stride_vh, + stride_obs, + stride_oh, + stride_buf_kbs, + stride_buf_kh, + stride_buf_vbs, + stride_buf_vh, + SLIDING_WINDOW_SIZE: tl.constexpr, + logit_cap: tl.constexpr, + xai_temperature_len: tl.constexpr, + Lq: tl.constexpr, + Lv: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_DPE: tl.constexpr, + BLOCK_DV: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + USE_CUSTOM_MASK: tl.constexpr, + IS_CAUSAL: tl.constexpr, + SKIP_PREFIX_CUSTOM_MASK: tl.constexpr, + STORE_TRANSPOSE: tl.constexpr, + HAS_SINK: tl.constexpr, +): + cur_seq = tl.program_id(0) + cur_head = tl.program_id(1) + cur_block_m = tl.program_id(2) + cur_kv_head = cur_head // kv_group_num + + cur_seq_extend_start_idx = tl.load(qo_indptr + cur_seq) + cur_seq_len_extend = tl.load(qo_indptr + cur_seq + 1) - cur_seq_extend_start_idx + cur_seq_kv_start_idx = tl.load(kv_indptr + cur_seq) + cur_seq_len_prefix = tl.load(kv_indptr + cur_seq + 1) - cur_seq_kv_start_idx + cur_seq_len = cur_seq_len_prefix + cur_seq_len_extend + + if USE_CUSTOM_MASK: + cur_seq_mask_start_idx = tl.load(mask_indptr + cur_seq) + + # For SWA, we should only load the mask in the sliding window + window_kv_offset = 0 + if USE_CUSTOM_MASK and SLIDING_WINDOW_SIZE > 0: + window_kv_offset = tl.load(window_kv_offset_ptr + cur_seq) + + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_dv = tl.arange(0, BLOCK_DV) + offs_m = tl.arange(0, BLOCK_M) + mask_m = (cur_block_m * BLOCK_M + offs_m) < cur_seq_len_extend + + mask_d = offs_d < Lq + mask_dv = offs_dv < Lv + + if xai_temperature_len > 0: + offs_qidx = cur_seq_len_prefix + cur_block_m * BLOCK_M + offs_m + xai_temperature_scale = 1.0 / tl.log2(float(xai_temperature_len)) + xai_temperature_reg = tl.where( + offs_qidx > xai_temperature_len, + tl.log2(offs_qidx.to(tl.float32)) * xai_temperature_scale, + 1.0, + ) + + offs_q = ( + (cur_seq_extend_start_idx + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_qbs + + cur_head * stride_qh + + offs_d[None, :] + ) + q = tl.load( + Q_Extend + offs_q, mask=(mask_m[:, None]) & (mask_d[None, :]), other=0.0 + ) + + if BLOCK_DPE > 0: + offs_dpe = BLOCK_DMODEL + tl.arange(0, BLOCK_DPE) + offs_qpe = ( + (cur_seq_extend_start_idx + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_qbs + + cur_head * stride_qh + + offs_dpe[None, :] + ) + qpe = tl.load(Q_Extend + offs_qpe, mask=mask_m[:, None], other=0.0) + + # stage 1: compute scores with prefix + offs_n = tl.arange(0, BLOCK_N) + + acc = tl.zeros([BLOCK_M, BLOCK_DV], dtype=tl.float32) + deno = tl.zeros([BLOCK_M], dtype=tl.float32) + e_max = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + + for start_n in range(0, cur_seq_len_prefix, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + mask_n = (start_n + offs_n) < cur_seq_len_prefix + + final_mask = mask_m[:, None] & mask_n[None, :] + if USE_CUSTOM_MASK and not SKIP_PREFIX_CUSTOM_MASK: + custom_mask = tl.load( + mask_ptr + + cur_seq_mask_start_idx + + (cur_block_m * BLOCK_M + offs_m[:, None]) + * (cur_seq_len + window_kv_offset) + + window_kv_offset + + start_n + + offs_n[None, :], + mask=(mask_m[:, None] & mask_n[None, :]), + other=0, + ) + final_mask &= custom_mask + if SLIDING_WINDOW_SIZE > 0: + # Add mask where q_id <= kv_id + sliding_window_size + # q_id = prefix_len + cur_m, kv_id = cur_n + window_mask = ( + cur_seq_len_prefix + cur_block_m * BLOCK_M + offs_m[:, None] + ) <= (start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE) + final_mask &= window_mask + + SKIP_TILE = False + if (USE_CUSTOM_MASK and not SKIP_PREFIX_CUSTOM_MASK) or SLIDING_WINDOW_SIZE > 0: + SKIP_TILE = tl.max(tl.max(final_mask.to(tl.int32), axis=1), axis=0) == 0 + + if not SKIP_TILE: + offs_kv_loc = tl.load( + kv_indices + cur_seq_kv_start_idx + start_n + offs_n, + mask=mask_n, + other=0, + ) + + # load k in transposed way + offs_buf_k = ( + offs_kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_d[:, None] + ) + k = tl.load( + K_Buffer + offs_buf_k, + mask=(mask_n[None, :]) & (mask_d[:, None]), + other=0.0, + ) + + qk = tl.dot(q.to(k.dtype), k) + if BLOCK_DPE > 0: + offs_kpe = ( + offs_kv_loc[None, :] * stride_buf_kbs + + cur_kv_head * stride_buf_kh + + offs_dpe[:, None] + ) + kpe = tl.load( + K_Buffer + offs_kpe, + mask=mask_n[None, :], + other=0.0, + ) + qk += tl.dot(qpe.to(kpe.dtype), kpe) + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + if xai_temperature_len > 0: + qk *= xai_temperature_reg[:, None] + + qk = tl.where(final_mask, qk, float("-inf")) + + row_max = tl.max(qk, 1) + row_max_fixed = tl.where(row_max == float("-inf"), -1e20, row_max) + n_e_max = tl.maximum(row_max_fixed, e_max) + + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + deno = deno * re_scale + tl.sum(p, 1) + + offs_buf_v = ( + offs_kv_loc[:, None] * stride_buf_vbs + + cur_kv_head * stride_buf_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Buffer + offs_buf_v, + mask=mask_n[:, None] & mask_dv[None, :], + other=0.0, + ) + p = p.to(v.dtype) + acc = acc * re_scale[:, None] + tl.dot(p, v) + + e_max = n_e_max + + # stage 2: compute the triangle part + + cur_block_m_end = ( + cur_seq_len_extend + if not IS_CAUSAL + else tl.minimum(cur_seq_len_extend, (cur_block_m + 1) * BLOCK_M) + ) + for start_n in range(0, cur_block_m_end, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + mask_n = (start_n + offs_n) < cur_block_m_end + + final_mask = mask_m[:, None] & mask_n[None, :] + if USE_CUSTOM_MASK: + custom_mask = tl.load( + mask_ptr + + cur_seq_mask_start_idx + + (cur_block_m * BLOCK_M + offs_m[:, None]) + * (cur_seq_len + window_kv_offset) + + window_kv_offset + + cur_seq_len_prefix + + start_n + + offs_n[None, :], + mask=(mask_m[:, None] & mask_n[None, :]), + other=0, + ) + custom_mask &= mask_m[:, None] & mask_n[None, :] + final_mask &= custom_mask + elif IS_CAUSAL: + mask_causual = (cur_block_m * BLOCK_M + offs_m[:, None]) >= ( + start_n + offs_n[None, :] + ) + mask_causual &= mask_m[:, None] & mask_n[None, :] + final_mask &= mask_causual + else: + mask_non_causal = mask_m[:, None] & mask_n[None, :] + final_mask &= mask_non_causal + + if SLIDING_WINDOW_SIZE > 0: + # Add mask where q_id <= kv_id + sliding_window_size + window_mask = (cur_block_m * BLOCK_M + offs_m[:, None]) <= ( + start_n + offs_n[None, :] + SLIDING_WINDOW_SIZE + ) + final_mask &= window_mask + + SKIP_TILE = False + if USE_CUSTOM_MASK or SLIDING_WINDOW_SIZE > 0: + SKIP_TILE = tl.max(tl.max(final_mask.to(tl.int32), axis=1), axis=0) == 0 + + if not SKIP_TILE: + # load k in transposed way + offs_k = ( + (cur_seq_extend_start_idx + start_n + offs_n[None, :]) * stride_kbs + + cur_kv_head * stride_kh + + offs_d[:, None] + ) + k = tl.load( + K_Extend + offs_k, mask=(mask_n[None, :]) & (mask_d[:, None]), other=0.0 + ) + + qk = tl.dot(q, k, out_dtype=tl.float32) + if BLOCK_DPE > 0: + offs_kpe = ( + (cur_seq_extend_start_idx + start_n + offs_n[None, :]) * stride_kbs + + cur_kv_head * stride_kh + + offs_dpe[:, None] + ) + kpe = tl.load( + K_Extend + offs_kpe, + mask=mask_n[None, :], + other=0.0, + ) + qk += tl.dot(qpe, kpe) + + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + if xai_temperature_len > 0: + qk *= xai_temperature_reg[:, None] + + qk = tl.where(final_mask, qk, float("-inf")) + + row_max = tl.max(qk, 1) + row_max_fixed = tl.where(row_max == float("-inf"), -1e20, row_max) + n_e_max = tl.maximum(row_max_fixed, e_max) + + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + deno = deno * re_scale + tl.sum(p, 1) + + offs_v = ( + (cur_seq_extend_start_idx + start_n + offs_n[:, None]) * stride_vbs + + cur_kv_head * stride_vh + + offs_dv[None, :] + ) + v = tl.load( + V_Extend + offs_v, mask=mask_n[:, None] & mask_dv[None, :], other=0.0 + ) + p = p.to(v.dtype) + acc = acc * re_scale[:, None] + tl.dot(p, v) + + e_max = n_e_max + + if HAS_SINK: + cur_sink = tl.load(sink_ptr + cur_head) + deno += tl.exp(cur_sink - e_max) + + offs_o = ( + (cur_seq_extend_start_idx + cur_block_m * BLOCK_M + offs_m[:, None]) + * stride_obs + + cur_head * stride_oh + + offs_dv[None, :] + ) + if STORE_TRANSPOSE: + tl.store( + O_Extend + offs_o.T, + (acc / deno[:, None]).T, + mask=(mask_m[:, None] & mask_dv[None, :]).T, + ) + else: + tl.store( + O_Extend + offs_o, + acc / deno[:, None], + mask=mask_m[:, None] & mask_dv[None, :], + ) + + +def extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + is_causal, + mask_indptr, + max_len_extend, + sm_scale=None, + logit_cap=0.0, + skip_prefix_custom_mask=True, + sliding_window_size=-1, + sinks=None, + window_kv_offsets=None, + xai_temperature_len=-1, +): + """ + q_extend, k_extend, v_extend, o_extend: contiguous tensors + + k_buffer, v_buffer: (prefix + extend) tensors in mem_manager + """ + Lq, Lk, Lv = ( + q_extend.shape[-1], + k_extend.shape[-1], + v_extend.shape[-1], + ) + + if Lq == 576: + BLOCK_DMODEL = 512 + BLOCK_DPE = 64 + elif Lq == 288: + BLOCK_DMODEL = 256 + BLOCK_DPE = 32 + elif Lq == 192: + BLOCK_DMODEL = 128 + BLOCK_DPE = 64 + else: + BLOCK_DMODEL = triton.next_power_of_2(Lq) + BLOCK_DPE = 0 + BLOCK_DV = triton.next_power_of_2(Lv) + + if _is_hip: + BLOCK_M, BLOCK_N = (64, 64) + num_warps = 4 + + else: + if _is_cuda and CUDA_CAPABILITY[0] >= 9: + if Lq <= 256: + BLOCK_M, BLOCK_N = (128, 64) + else: + BLOCK_M, BLOCK_N = (32, 64) + elif _is_cuda and CUDA_CAPABILITY[0] >= 8: + # sm86/sm89 has a much smaller shared memory size (100K) than sm80 (160K) + if CUDA_CAPABILITY[1] == 9 or CUDA_CAPABILITY[1] == 6: + if Lq <= 128: + BLOCK_M, BLOCK_N = (64, 128) + elif Lq <= 256: + BLOCK_M, BLOCK_N = (64, 64) + else: + BLOCK_M, BLOCK_N = (32, 32) + else: + if Lq <= 128: + BLOCK_M, BLOCK_N = (128, 128) + elif Lq <= 256: + BLOCK_M, BLOCK_N = (64, 64) + else: + BLOCK_M, BLOCK_N = (32, 64) + else: + BLOCK_M, BLOCK_N = (64, 64) if Lq <= 128 else (32, 32) + + num_warps = 4 if Lk <= 64 else 8 + + sm_scale = sm_scale or 1.0 / (Lq**0.5) + batch_size, head_num = qo_indptr.shape[0] - 1, q_extend.shape[1] + kv_group_num = q_extend.shape[1] // k_extend.shape[1] + + USE_CUSTOM_MASK = custom_mask is not None + # Skip custom mask for prefix part + SKIP_PREFIX_CUSTOM_MASK = skip_prefix_custom_mask + + HAS_SINK = sinks is not None + + grid = (batch_size, head_num, triton.cdiv(max_len_extend, BLOCK_M)) + num_stages = 1 + + extra_kargs = {} + if _is_hip: + extra_kargs = {"waves_per_eu": 1, "matrix_instr_nonkdim": 16, "kpack": 2} + + _fwd_kernel[grid]( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + sinks, + window_kv_offsets, + sm_scale, + kv_group_num, + q_extend.stride(0), + q_extend.stride(1), + k_extend.stride(0), + k_extend.stride(1), + v_extend.stride(0), + v_extend.stride(1), + o_extend.stride(0), + o_extend.stride(1), + k_buffer.stride(0), + k_buffer.stride(1), + v_buffer.stride(0), + v_buffer.stride(1), + SLIDING_WINDOW_SIZE=sliding_window_size, + logit_cap=logit_cap, + xai_temperature_len=xai_temperature_len, + BLOCK_DMODEL=BLOCK_DMODEL, + BLOCK_DPE=BLOCK_DPE, + BLOCK_DV=BLOCK_DV, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + Lq=Lq, + Lv=Lv, + USE_CUSTOM_MASK=USE_CUSTOM_MASK, + IS_CAUSAL=is_causal, + SKIP_PREFIX_CUSTOM_MASK=SKIP_PREFIX_CUSTOM_MASK, + HAS_SINK=HAS_SINK, + STORE_TRANSPOSE=_is_hip, + num_warps=num_warps, + num_stages=num_stages, + **extra_kargs, + ) + + +def redundant_attention( + q_extend, + o_extend, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, +): + total_token_num = k_buffer.shape[0] + B, H_Q, D = b_req_idx.shape[0], q_extend.shape[-2], q_extend.shape[-1] + q_buffer = torch.empty( + (total_token_num, H_Q, D), dtype=q_extend.dtype, device=q_extend.device + ) + + pt = 0 + for i in range(B): + cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] + pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] + q_buffer[pl:pr] = q_extend[pt : pt + cur_seq_len_extend] + pt += cur_seq_len_extend + + o_buffer = torch.empty_like(q_buffer) + context_attention_fwd( + q_buffer, k_buffer, v_buffer, o_buffer, b_start_loc, b_seq_len, max_len_in_batch + ) + + pt = 0 + for i in range(B): + cur_seq_len_extend = b_seq_len[i] - b_seq_len_prefix[i] + pl, pr = b_start_loc[i] + b_seq_len_prefix[i], b_start_loc[i] + b_seq_len[i] + o_extend[pt : pt + cur_seq_len_extend] = o_buffer[pl:pr] + pt += cur_seq_len_extend diff --git a/python/sglang/srt/layers/attention/triton_ops/merge_state.py b/python/sglang/srt/layers/attention/triton_ops/merge_state.py new file mode 100644 index 000000000..955097fc9 --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/merge_state.py @@ -0,0 +1,96 @@ +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + + +@triton.jit +def merge_state_kernel( + output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged + output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged + prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a + prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a + suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b + suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b + HEAD_SIZE: tl.constexpr, + PADDED_HEAD_SIZE: tl.constexpr, + OUTPUT_LSE: tl.constexpr, +): + token_idx = tl.program_id(0) + num_tokens = tl.num_programs(0) + head_idx = tl.program_id(1) + num_heads = tl.num_programs(1) + + p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx) + s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx) + p_lse = float("-inf") if p_lse == float("inf") else p_lse + s_lse = float("-inf") if s_lse == float("inf") else s_lse + + max_lse = tl.maximum(p_lse, s_lse) + p_lse = p_lse - max_lse + s_lse = s_lse - max_lse + out_se = tl.exp(p_lse) + tl.exp(s_lse) + + if OUTPUT_LSE: + out_lse = tl.log(out_se) + max_lse + tl.store(output_lse + token_idx * num_heads + head_idx, out_lse) + + head_arange = tl.arange(0, PADDED_HEAD_SIZE) + head_mask = head_arange < HEAD_SIZE + p_out = tl.load( + prefix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + s_out = tl.load( + suffix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + + p_scale = tl.exp(p_lse) / out_se + s_scale = tl.exp(s_lse) / out_se + out = p_out * p_scale + s_out * s_scale + tl.store( + output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange, + out, + mask=head_mask, + ) + + +def merge_state_triton( + prefix_output: torch.Tensor, + prefix_lse: torch.Tensor, + suffix_output: torch.Tensor, + suffix_lse: torch.Tensor, + output: Optional[torch.Tensor] = None, + output_lse: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + # Avoid creating new tensors if they are already provided + if output is None: + output = torch.empty_like(prefix_output) + if output_lse is None: + output_lse = torch.empty_like(prefix_lse) + + num_tokens = output.shape[0] + num_query_heads = output.shape[1] + head_size = output.shape[2] + padded_head_size = triton.next_power_of_2(head_size) + + merge_state_kernel[(num_tokens, num_query_heads)]( + output, + output_lse, + prefix_output, + prefix_lse, + suffix_output, + suffix_lse, + head_size, + padded_head_size, + output_lse is not None, + ) + return output, output_lse diff --git a/python/sglang/srt/layers/attention/triton_ops/prefill_attention.py b/python/sglang/srt/layers/attention/triton_ops/prefill_attention.py new file mode 100644 index 000000000..ac0fc72af --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/prefill_attention.py @@ -0,0 +1,217 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Memory-efficient attention for prefill. +It supporst page size = 1. +""" + +# Adapted from +# https://github.com/ModelTC/lightllm/blob/f2a54f0912293f683bf1d1695fd12c4098a5bf82/lightllm/models/llama/triton_kernel/context_flashattention_nopad.py#L1 +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() + +if _is_cuda or _is_hip: + CUDA_CAPABILITY = torch.cuda.get_device_capability() + + +@triton.jit +def _fwd_kernel( + Q, + K, + V, + sm_scale, + B_Start_Loc, + B_Seqlen, + Out, + stride_qbs, + stride_qh, + stride_kbs, + stride_kh, + stride_vbs, + stride_vh, + stride_obs, + stride_oh, + kv_group_num: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DMODEL: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_CAUSAL: tl.constexpr, + Lk: tl.constexpr, +): + cur_batch = tl.program_id(0) + cur_head = tl.program_id(1) + start_m = tl.program_id(2) + + cur_kv_head = cur_head // kv_group_num + + cur_batch_seq_len = tl.load(B_Seqlen + cur_batch) + cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch) + + block_start_loc = BLOCK_M * start_m + + # initialize offsets + offs_n = tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_DMODEL) + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M) + off_q = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs + + cur_head * stride_qh + + offs_d[None, :] + ) + off_k = offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh + offs_d[:, None] + off_v = offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh + offs_d[None, :] + + mask_d = offs_d < Lk + + q = tl.load( + Q + off_q, + mask=(offs_m[:, None] < cur_batch_seq_len) & (mask_d[None, :]), + other=0.0, + ) + + k_ptrs = K + off_k + v_ptrs = V + off_v + + # initialize pointer to m and l + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32) + + block_mask = tl.where(block_start_loc < cur_batch_seq_len, 1, 0) + + end_n = ( + cur_batch_seq_len + if not IS_CAUSAL + else tl.minimum((start_m + 1) * BLOCK_M, cur_batch_seq_len) + ) + for start_n in range(0, block_mask * end_n, BLOCK_N): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = tl.load( + k_ptrs + (cur_batch_in_all_start_index + start_n) * stride_kbs, + mask=((start_n + offs_n[None, :]) < cur_batch_seq_len) & (mask_d[:, None]), + other=0.0, + ) + # mask = tl.load(mask_ptrs + start_n, mask=start_n + offs_n < cur_batch_end_loc, other=0.0) + + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) + qk += tl.dot(q, k) + qk *= sm_scale + + if IS_CAUSAL: + qk += tl.where( + (start_n + offs_n[None, :] < cur_batch_seq_len) + & (offs_m[:, None] >= (start_n + offs_n[None, :])), + 0, + float("-inf"), + ) + else: + qk += tl.where( + (start_n + offs_n[None, :]) < cur_batch_seq_len, 0, float("-inf") + ) + + # -- compute m_ij, p, l_ij + m_ij = tl.max(qk, 1) + p = tl.exp(qk - m_ij[:, None]) + l_ij = tl.sum(p, 1) + # -- update m_i and l_i + m_i_new = tl.maximum(m_i, m_ij) + alpha = tl.exp(m_i - m_i_new) + beta = tl.exp(m_ij - m_i_new) + l_i_new = alpha * l_i + beta * l_ij + # -- update output accumulator -- + # scale p + p_scale = beta / l_i_new + p = p * p_scale[:, None] + # scale acc + acc_scale = l_i / l_i_new * alpha + acc = acc * acc_scale[:, None] + # update acc + v = tl.load( + v_ptrs + (cur_batch_in_all_start_index + start_n) * stride_vbs, + mask=((start_n + offs_n[:, None]) < cur_batch_seq_len) & (mask_d[None, :]), + other=0.0, + ) + + p = p.to(v.dtype) + acc += tl.dot(p, v) + # update m_i and l_i + l_i = l_i_new + m_i = m_i_new + # initialize pointers to output + off_o = ( + (cur_batch_in_all_start_index + offs_m[:, None]) * stride_obs + + cur_head * stride_oh + + offs_d[None, :] + ) + out_ptrs = Out + off_o + tl.store( + out_ptrs, acc, mask=(offs_m[:, None] < cur_batch_seq_len) & (mask_d[None, :]) + ) + + +def context_attention_fwd( + q, k, v, o, b_start_loc, b_seq_len, max_input_len, is_causal=True +): + """ + q, k, v: [b * s, head, head_dim] + b_start_loc: [b] + b_seq_len: [b] + out: [b * s, head, head_dim] + """ + if (_is_cuda or _is_hip) and CUDA_CAPABILITY[0] > 8: + BLOCK = 128 + else: + BLOCK = 64 + + Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1] + + sm_scale = 1.0 / (Lq**0.5) + batch, head = b_seq_len.shape[0], q.shape[1] + kv_group_num = q.shape[1] // k.shape[1] + + grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) + num_warps = 4 if Lk <= 64 else 8 + + _fwd_kernel[grid]( + q, + k, + v, + sm_scale, + b_start_loc, + b_seq_len, + o, + q.stride(0), + q.stride(1), + k.stride(0), + k.stride(1), + v.stride(0), + v.stride(1), + o.stride(0), + o.stride(1), + kv_group_num=kv_group_num, + BLOCK_M=BLOCK, + BLOCK_DMODEL=triton.next_power_of_2(Lk), + BLOCK_N=BLOCK, + IS_CAUSAL=is_causal, + num_warps=num_warps, + num_stages=1, + Lk=Lk, + ) diff --git a/python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py b/python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py new file mode 100644 index 000000000..89b17ce2d --- /dev/null +++ b/python/sglang/srt/layers/attention/triton_ops/rocm_mla_decode_rope.py @@ -0,0 +1,439 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Memory-efficient attention for decoding. +It supports page size = 1. +""" + +# Adapted from +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage1.py +# https://github.com/ModelTC/lightllm/blob/96353e868a840db4d103138caf15ed9dbea8c186/lightllm/models/deepseek2/triton_kernel/gqa_flash_decoding_stage2.py + +import triton +import triton.language as tl + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + _decode_softmax_reducev_fwd, +) + + +def is_hip(): + return triton.runtime.driver.active.get_current_target().backend == "hip" + + +_is_hip = is_hip() + + +@triton.jit +def tanh(x): + # Tanh is just a scaled sigmoid + return 2 * tl.sigmoid(2 * x) - 1 + + +@triton.jit +def _fwd_grouped_kernel_stage1_rope( + Q, # Holds [Q_NOPE; Q_PE], b x h x (d+r) + K_Buffer, # Holds [KV; K_PE], b*s x (c+r) + V_buffer, # Holds [KV], b*s x (c) + cos_sin_cache, # max_seq_len x (rotary_dim * 2) + positions, # sequence positions + sm_scale, + kv_indptr, + kv_indices, + Att_Out, # b x h x NUM_KV_SPLITS x (kv_lora_rank + 1) + k_pe_t_out, + stride_qb, + stride_qh, + stride_buf_kbs, + stride_buf_vbs, + stride_mid_ob, + stride_mid_oh, + stride_mid_os, + stride_kpe_tokens_out_b, + stride_cos_sin_cache_s, + stride_positions_b, + rotary_dim: tl.constexpr, + kv_lora_rank: tl.constexpr, + qk_rope_head_dim: tl.constexpr, + kv_group_num: tl.constexpr, + q_head_num: tl.constexpr, + BLOCK_C: tl.constexpr, + BLOCK_R: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_H: tl.constexpr, + NUM_KV_SPLITS: tl.constexpr, + logit_cap: tl.constexpr, + USE_ROPE: tl.constexpr, + IS_NEOX_STYLE: tl.constexpr, +): + + cur_batch = tl.program_id(0) + cur_head_id = tl.program_id(1) + split_kv_id = tl.program_id(2) + + if BLOCK_H < kv_group_num: + VALID_BLOCK_H: tl.constexpr = BLOCK_H + else: + VALID_BLOCK_H: tl.constexpr = kv_group_num + cur_head = cur_head_id * VALID_BLOCK_H + tl.arange(0, BLOCK_H) + mask_h = cur_head < (cur_head_id + 1) * VALID_BLOCK_H + mask_h = mask_h & (cur_head < q_head_num) + + offs_c = tl.arange(0, BLOCK_C) + offs_qk_r = tl.arange(kv_lora_rank, kv_lora_rank + BLOCK_R) # to get the k_pe + + off_q_pe = ( + cur_batch * stride_qb + cur_head[:, None] * stride_qh + offs_qk_r[None, :] + ) + offs_q = cur_batch * stride_qb + cur_head[:, None] * stride_qh + offs_c[None, :] + + mask_c = offs_c < kv_lora_rank + mask_qk_r = offs_qk_r < (kv_lora_rank + qk_rope_head_dim) + + cur_batch_kv_start_idx = tl.load(kv_indptr + cur_batch) + cur_batch_seq_len = tl.load(kv_indptr + cur_batch + 1) - cur_batch_kv_start_idx + + q = tl.load(Q + offs_q, mask=(mask_h[:, None]) & (mask_c[None, :]), other=0.0) + q_pe = tl.load( + Q + off_q_pe, mask=(mask_h[:, None]) & (mask_qk_r[None, :]), other=0.0 + ) + + kv_len_per_split = tl.cdiv(cur_batch_seq_len, NUM_KV_SPLITS) + split_kv_start = kv_len_per_split * split_kv_id + split_kv_end = tl.minimum(split_kv_start + kv_len_per_split, cur_batch_seq_len) + + # apply rotary embedding for q_pe, and k_pe (last token per batch of K_PE) + LAST_SPLIT = split_kv_end == cur_batch_seq_len + k_pe_last_token = tl.zeros([BLOCK_R], dtype=q.dtype) + + if USE_ROPE: + if IS_NEOX_STYLE: + # [BLOCK_ROTARY // 2, BLOCK_ROTARY // 2 + 1, BLOCK_ROTARY // 2 + 2, ..., 0, 1, 2, ..., BLOCK_ROTARY // 2 - 1, pass:] + offs_qk_rot_r = kv_lora_rank + ( + (tl.arange(0, BLOCK_R) + (rotary_dim // 2)) % rotary_dim + ) + # Which elements to flip + mask_rotate = tl.arange(0, BLOCK_R) < (rotary_dim // 2) + # [0 , 1, 2, ..., rotary_dim // 2 - 1, 0 , 1, 2, ..., rotary_dim // 2 - 1] + offs_rotary = tl.arange(0, BLOCK_R) % (rotary_dim // 2) + else: + # [1, 0, 3, 2, 5, 4, ..., BLOCK_R, BLOCK_R - 1] + offs_qk_rot_r = ( + kv_lora_rank + + (((tl.arange(0, BLOCK_R) + 1) % 2) * 2) + - 1 + + tl.arange(0, BLOCK_R) + ) + mask_rotate = tl.arange(0, BLOCK_R) % 2 < 1 + # [0, 0, 1, 1, ..., rotary_dim // 2 - 1, rotary_dim // 2 - 1] + offs_rotary = tl.arange(0, BLOCK_R) // 2 + + if qk_rope_head_dim > rotary_dim: + offs_qk_rot_r = tl.where( + tl.arange(0, BLOCK_R) < rotary_dim, offs_qk_rot_r, tl.arange(0, BLOCK_R) + ) + offs_rotary = tl.where( + tl.arange(0, BLOCK_R) < rotary_dim, offs_rotary, tl.arange(0, BLOCK_R) + ) + + mask_rotary = tl.arange(0, BLOCK_R) < rotary_dim + + pos = tl.load(positions + cur_batch * stride_positions_b) + cos = tl.load( + cos_sin_cache + pos * stride_cos_sin_cache_s + offs_rotary, + mask=mask_rotary, + other=1.0, + ) + sin = tl.load( + cos_sin_cache + + pos * stride_cos_sin_cache_s + + offs_rotary + + rotary_dim // 2, + mask_rotary, + other=0.0, + ) + + off_q_pe_rot = ( + cur_batch * stride_qb + + cur_head[:, None] * stride_qh + + offs_qk_rot_r[None, :] + ) + mask_qk_rot_r = offs_qk_rot_r < (kv_lora_rank + qk_rope_head_dim) + + # 0, 2, 4,.... 1, 3, 5... + q_pe_rot = tl.load( + Q + off_q_pe_rot, + mask=(mask_h[:, None]) & (mask_qk_rot_r[None, :]), + other=0.0, + ) + q_pe_rot = tl.where(mask_rotate[None, :], -q_pe_rot, q_pe_rot) + + q_pe = q_pe * cos + q_pe_rot * sin + + # we only apply to the last token in the K_PE + if LAST_SPLIT: + # debug assert + if (cur_batch == 0 and cur_head == 0) and split_kv_id < NUM_KV_SPLITS - 1: + tl.device_assert(False, "Only last split should compute k_pe") + + kv_loc = tl.load( + kv_indices + cur_batch_kv_start_idx + cur_batch_seq_len - 1 + ) + offs_buf_k_pe_last_token = kv_loc * stride_buf_kbs + offs_qk_r + offs_buf_k_pe_rot_last_token = kv_loc * stride_buf_kbs + offs_qk_rot_r + k_pe_last_token = tl.load(K_Buffer + offs_buf_k_pe_last_token) + + k_pe_rot_last_token = tl.load(K_Buffer + offs_buf_k_pe_rot_last_token) + k_pe_rot_last_token = tl.where( + mask_rotate, -k_pe_rot_last_token, k_pe_rot_last_token + ) + + k_pe_last_token = k_pe_last_token * cos + k_pe_rot_last_token * sin + + e_max = tl.zeros([BLOCK_H], dtype=tl.float32) - float("inf") + e_sum = tl.zeros([BLOCK_H], dtype=tl.float32) + acc = tl.zeros([BLOCK_H, BLOCK_C], dtype=tl.float32) + + if split_kv_end > split_kv_start: + for start_n in range(split_kv_start, split_kv_end, BLOCK_N): + offs_n = start_n + tl.arange(0, BLOCK_N) + kv_loc = tl.load( + kv_indices + cur_batch_kv_start_idx + offs_n, + mask=offs_n < split_kv_end, + other=0, + ) + + offs_buf_kv = kv_loc[None, :] * stride_buf_kbs + offs_c[:, None] + offs_buf_k_pe = kv_loc[None, :] * stride_buf_kbs + offs_qk_r[:, None] + + k_pe = tl.load( + K_Buffer + offs_buf_k_pe, + mask=(offs_n[None, :] < split_kv_end) & (mask_qk_r[:, None]), + other=0.0, + ) # positional embedding part of keys + + if (USE_ROPE and LAST_SPLIT) and start_n >= cur_batch_seq_len - BLOCK_N: + k_pe = tl.where( + offs_n[None, :] != (split_kv_end - 1), + k_pe, + k_pe_last_token[:, None], + ) + + # (16, 64) x (64, 32) + # dot product of rope parts + qk = tl.dot(q_pe, k_pe.to(q_pe.dtype)) + + kv = tl.load( + K_Buffer + offs_buf_kv, + mask=(offs_n[None, :] < split_kv_end) & (mask_c[:, None]), + other=0.0, + ) # the shared latent tensor for keys and values + + # (16, 512) x (512, 32) + # dot product of nope parts + qk += tl.dot(q, kv) + + qk *= sm_scale + + if logit_cap > 0: + qk = logit_cap * tanh(qk / logit_cap) + + qk = tl.where( + mask_h[:, None] & (offs_n[None, :] < split_kv_end), qk, float("-inf") + ) + + offs_buf_v = kv_loc[:, None] * stride_buf_vbs + offs_c[None, :] + v = tl.load( + V_buffer + offs_buf_v, + mask=(offs_n[:, None] < split_kv_end) & (mask_c[None, :]), + other=0.0, + ) + + n_e_max = tl.maximum(tl.max(qk, 1), e_max) + re_scale = tl.exp(e_max - n_e_max) + p = tl.exp(qk - n_e_max[:, None]) + acc *= re_scale[:, None] + # (16, 32) x (32, 512) + acc += tl.dot(p.to(v.dtype), v) + + e_sum = e_sum * re_scale + tl.sum(p, 1) + e_max = n_e_max + + offs_mid_o = ( + cur_batch * stride_mid_ob + + cur_head[:, None] * stride_mid_oh + + split_kv_id * stride_mid_os + + offs_c[None, :] + ) + + if USE_ROPE: + if LAST_SPLIT: + k_pe_last_token_ptrs = ( + k_pe_t_out + + cur_batch * stride_kpe_tokens_out_b + + tl.arange(0, BLOCK_R) + ) + tl.store(k_pe_last_token_ptrs, k_pe_last_token, mask=mask_qk_r) + + tl.store( + Att_Out + offs_mid_o, + acc / e_sum[:, None], + mask=(mask_h[:, None]) & (mask_c[None, :]), + ) + + offs_mid_o_1 = ( + cur_batch * stride_mid_ob + + cur_head * stride_mid_oh + + split_kv_id * stride_mid_os + + kv_lora_rank + ) + + tl.store( + Att_Out + offs_mid_o_1, + e_max + tl.log(e_sum), + mask=mask_h, + ) + + +# TODO rope offset +def _decode_grouped_att_m_fwd_rope( + q, + k_buffer, + v_buffer, + att_out, + k_pe_tokens_out, + kv_lora_rank, # c + cos_sin_cache, + positions, + rotary_dim, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + use_rope, + is_neox_style=True, +): + if use_rope: + assert ( + k_pe_tokens_out is not None + ), "We must output the k_pe tokens with rope applied if rope fusion enabled." + + BLOCK = 32 + + # # [TODO] work around shmem limit on MI3xx + # if _is_hip and kv_lora_rank >= 576: + # BLOCK = 16 + + qk_rope_head_dim = k_buffer.shape[-1] - kv_lora_rank + batch, head_num = kv_indptr.shape[0] - 1, q.shape[1] + kv_group_num = q.shape[1] // k_buffer.shape[1] + + BLOCK_C = triton.next_power_of_2(kv_lora_rank) + BLOCK_R = triton.next_power_of_2(qk_rope_head_dim) + + BLOCK_H = 16 + NUM_KV_SPLITS = num_kv_splits + grid = ( + batch, + triton.cdiv(head_num, min(BLOCK_H, kv_group_num)), + NUM_KV_SPLITS, + ) + + extra_kargs = {} + num_stages = 2 + if _is_hip: + # https://rocm.docs.amd.com/en/docs-6.2.0/how-to/llm-fine-tuning-optimization/optimizing-triton-kernel.html + # https://github.com/triton-lang/triton/blob/main/third_party/amd/backend/compiler.py + extra_kargs = {"waves_per_eu": 1, "matrix_instr_nonkdim": 16, "kpack": 2} + num_stages = 1 + + _fwd_grouped_kernel_stage1_rope[grid]( + q, + k_buffer, + v_buffer, + cos_sin_cache, + positions, + sm_scale, + kv_indptr, + kv_indices, + att_out, + k_pe_tokens_out, + q.stride(0), + q.stride(1), + k_buffer.stride(0), + v_buffer.stride(0), + att_out.stride(0), + att_out.stride(1), + att_out.stride(2), + k_pe_tokens_out.stride(0) if use_rope else 0, + cos_sin_cache.stride(0) if use_rope else 0, + positions.stride(0) if use_rope else 0, + rotary_dim, + kv_lora_rank, + qk_rope_head_dim, + kv_group_num=kv_group_num, + q_head_num=head_num, + BLOCK_C=BLOCK_C, + BLOCK_R=BLOCK_R, + BLOCK_N=BLOCK, + BLOCK_H=BLOCK_H, + NUM_KV_SPLITS=NUM_KV_SPLITS, + logit_cap=logit_cap, + USE_ROPE=use_rope, + IS_NEOX_STYLE=is_neox_style, + num_warps=4, + num_stages=num_stages, + **extra_kargs + ) + + +def decode_attention_fwd_grouped_rope( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + k_pe_tokens, + kv_lora_rank, + rotary_dim, + cos_sin_cache, + positions, + attn_logits, + num_kv_splits, + sm_scale, + logit_cap=0.0, + use_rope=False, + is_neox_style=False, +): + _decode_grouped_att_m_fwd_rope( + q, + k_buffer, + v_buffer, + attn_logits, + k_pe_tokens, + kv_lora_rank, + cos_sin_cache, + positions, + rotary_dim, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + use_rope, + is_neox_style, + ) + _decode_softmax_reducev_fwd(attn_logits, q, o, v_buffer, kv_indptr, num_kv_splits) diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py new file mode 100644 index 000000000..a48cc9794 --- /dev/null +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -0,0 +1,694 @@ +from __future__ import annotations + +""" +Support attention backend for TRTLLM MHA kernels from flashinfer. +The kernel supports sm100 only, with sliding window and attention sink features. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +import torch + +from sglang.srt.layers.attention.flashinfer_backend import ( + FlashInferAttnBackend, + FlashInferMultiStepDraftBackend, +) +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import is_flashinfer_available + +if is_flashinfer_available(): + import flashinfer + +from sglang.srt.speculative.eagle_utils import EagleDraftInput + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + +# Constants +DEFAULT_WORKSPACE_SIZE_MB = ( + 512 # Memory workspace size in MB, todo(Yingyi): read from config +) + +# Reuse this workspace buffer across all TRTLLM MHA wrappers +global_zero_init_workspace_buffer = None + + +@dataclass +class TRTLLMMHAMetadata: + # Sequence lengths for the forward batch + cache_seqlens_int32: torch.Tensor = None + # Maximum sequence length for query + max_seq_len_q: int = 1 + # Maximum sequence length for key + max_seq_len_k: int = 0 + # Cumulative sequence lengths for `query + cu_seqlens_q: torch.Tensor = None + # Cumulative sequence lengths for key + cu_seqlens_k: torch.Tensor = None + # Page table, the index of KV Cache Tables/Blocks + page_table: torch.Tensor = None + + +class TRTLLMHAAttnBackend(FlashInferAttnBackend): + """TRTLLM MHA attention kernel from flashinfer.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + kv_last_page_len_buf: Optional[torch.Tensor] = None, + speculative_step_id: int = 0, + ): + super().__init__( + model_runner, skip_prefill, kv_indptr_buf, kv_last_page_len_buf + ) + + config = model_runner.model_config + + # MHA-specific dimensions + self.max_context_len = model_runner.model_config.context_len + self.hidden_size = config.hidden_size + + # Runtime parameters + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.page_size = model_runner.page_size + self.req_to_token = model_runner.req_to_token_pool.req_to_token + self.device = model_runner.device + + # Workspace allocation + self.workspace_size = DEFAULT_WORKSPACE_SIZE_MB * 1024 * 1024 + # Allocate buffers + global global_zero_init_workspace_buffer + if global_zero_init_workspace_buffer is None: + global_zero_init_workspace_buffer = torch.zeros( + self.workspace_size, + dtype=torch.uint8, + device=model_runner.device, + ) + self.workspace_buffer = global_zero_init_workspace_buffer + + # CUDA graph state + self.decode_cuda_graph_metadata = {} + + # Speculative decoding + # Only support topk <= 1 for now. + self.topk = model_runner.server_args.speculative_eagle_topk or 0 + self.speculative_step_id = speculative_step_id + self.target_verify_metadata = {} + + self.speculative_num_draft_tokens = ( + model_runner.server_args.speculative_num_draft_tokens + ) + + # Forward metadata + self.forward_metadata: Optional[TRTLLMMHAMetadata] = None + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + """Initialize CUDA graph state for TRTLLM MHA.""" + max_num_pages = (self.max_context_len + self.page_size - 1) // self.page_size + self.decode_cuda_graph_metadata = { + "cache_seqlens": torch.zeros(max_bs, dtype=torch.int32, device=self.device), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + + if ( + self.speculative_num_draft_tokens is not None + and self.speculative_num_draft_tokens > 0 + ): + self.decode_cuda_graph_metadata["cu_seqlens_q"] = torch.arange( + 0, max_bs + 1, dtype=torch.int32, device=self.device + ) + self.decode_cuda_graph_metadata["cu_seqlens_k"] = torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ) + self.decode_cuda_graph_metadata["page_table_draft_decode"] = torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ) + self.target_verify_metadata = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.arange( + 0, + max_bs * self.speculative_num_draft_tokens + 1, + step=self.speculative_num_draft_tokens, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + + self.draft_extend_metadata = { + "cache_seqlens": torch.zeros( + max_bs, dtype=torch.int32, device=self.device + ), + "cu_seqlens_q": torch.zeros( + max_bs + 1, + dtype=torch.int32, + device=self.device, + ), + "cu_seqlens_k": torch.zeros( + max_bs + 1, dtype=torch.int32, device=self.device + ), + "page_table": torch.zeros( + max_bs, + max_num_pages, + dtype=torch.int32, + device=self.device, + ), + "strided_indices": torch.arange( + 0, self.max_context_len, self.page_size, device=self.device + ), + } + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + """Initialize metadata for CUDA graph capture.""" + metadata = TRTLLMMHAMetadata() + device = seq_lens.device + + if forward_mode.is_decode_or_idle(): + if spec_info is not None: + # Draft Decode + # Here we only support topk = 1 for now. + metadata.cache_seqlens_int32 = self.decode_cuda_graph_metadata[ + "cache_seqlens" + ][:bs] + metadata.max_seq_len_k = seq_lens.max().item() + ( + self.speculative_step_id + 1 + ) + metadata.cu_seqlens_q = self.decode_cuda_graph_metadata["cu_seqlens_q"][ + : bs + 1 + ] + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = self.decode_cuda_graph_metadata[ + "page_table_draft_decode" + ][:bs, :] + self.decode_cuda_graph_metadata[bs] = metadata + else: + # Normal Decode + # Get sequence information + metadata.cache_seqlens_int32 = seq_lens[:bs].to(torch.int32) + batch_size = len(seq_lens) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seq_lens, dim=0, dtype=torch.int32), (1, 0) + ) + + # Precompute maximum sequence length + metadata.max_seq_len_k = seq_lens.max().item() + # Precompute cumulative sequence lengths + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + # Precompute page table + metadata.page_table = self.decode_cuda_graph_metadata["page_table"][ + :bs, : + ] + self.decode_cuda_graph_metadata[bs] = metadata + elif forward_mode.is_target_verify(): + # Target Verify + # Here we only support topk = 1 for now. + metadata.cache_seqlens_int32 = self.target_verify_metadata["cache_seqlens"][ + :bs + ] + metadata.cache_seqlens_int32.copy_( + (seq_lens + self.speculative_num_draft_tokens) + ) + + metadata.cu_seqlens_q = torch.arange( + 0, + bs * self.speculative_num_draft_tokens + 1, + self.speculative_num_draft_tokens, + dtype=torch.int32, + device=device, + ) + + metadata.cu_seqlens_k = self.target_verify_metadata["cu_seqlens_k"][ + : (bs + 1) + ] + + metadata.max_seq_len_q = self.speculative_num_draft_tokens + metadata.max_seq_len_k = ( + seq_lens.max().item() + self.speculative_num_draft_tokens + ) + + metadata.page_table = self.target_verify_metadata["page_table"][:bs, :] + + self.target_verify_metadata[bs] = metadata + elif forward_mode.is_draft_extend(): + metadata.cache_seqlens_int32 = self.draft_extend_metadata["cache_seqlens"][ + :bs + ] + metadata.cache_seqlens_int32.copy_(seq_lens) + num_tokens_per_bs = num_tokens // bs + metadata.cu_seqlens_q = torch.arange( + 0, + bs * num_tokens_per_bs + 1, + num_tokens_per_bs, + dtype=torch.int32, + device=device, + ) + + metadata.cu_seqlens_k = self.draft_extend_metadata["cu_seqlens_k"][ + : (bs + 1) + ] + num_tokens_per_bs = num_tokens // bs + metadata.max_seq_len_q = num_tokens_per_bs + metadata.max_seq_len_k = seq_lens.max().item() + + metadata.page_table = self.draft_extend_metadata["page_table"][:bs, :] + + self.draft_extend_metadata[bs] = metadata + self.forward_metadata = metadata + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + """Replay CUDA graph with new inputs.""" + seq_lens = seq_lens[:bs] + seq_lens_cpu = seq_lens_cpu[:bs] + req_pool_indices = req_pool_indices[:bs] + metadata = None + if forward_mode.is_decode_or_idle(): + if spec_info is not None: + # Draft Decode + # Here we only support topk = 1 for now. + metadata = self.decode_cuda_graph_metadata[bs] + max_len = seq_lens_cpu.max().item() + metadata.max_seq_len_k = max_len + self.speculative_step_id + 1 + + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + + metadata.cache_seqlens_int32.copy_( + seq_lens + self.speculative_step_id + 1 + ) + else: + # Normal Decode + metadata = self.decode_cuda_graph_metadata[bs] + max_len = seq_lens_cpu.max().item() + max_seq_pages = (max_len + self.page_size - 1) // self.page_size + metadata.max_seq_len_k = max_len + + metadata.cache_seqlens_int32.copy_(seq_lens) + + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.decode_cuda_graph_metadata["strided_indices"][:max_seq_pages][ + None, : + ], + ] + metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size) + elif forward_mode.is_target_verify(): + # Here we only support topk = 1 for now. + metadata = self.target_verify_metadata[bs] + metadata.cache_seqlens_int32.copy_( + (seq_lens + self.speculative_num_draft_tokens) + ) + + metadata.max_seq_len_k = ( + seq_lens_cpu.max().item() + self.speculative_num_draft_tokens + ) + max_len = seq_lens_cpu.max().item() + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.decode_cuda_graph_metadata["strided_indices"][:max_seq_pages], + ] + page_indices //= self.page_size + metadata.page_table[:, :max_seq_pages].copy_(page_indices) + elif forward_mode.is_draft_extend(): + metadata = self.draft_extend_metadata[bs] + metadata.cache_seqlens_int32.copy_(seq_lens) + + metadata.max_seq_len_k = seq_lens_cpu.max().item() + max_len = seq_lens_cpu.max().item() + metadata.cu_seqlens_k[1:].copy_( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32) + ) + accept_length = spec_info.accept_length[:bs] + if spec_info.accept_length_cpu: + metadata.max_seq_len_q = max(spec_info.accept_length_cpu) + 1 + else: + metadata.max_seq_len_q = 1 + + metadata.cu_seqlens_q[1:].copy_( + torch.cumsum(accept_length, dim=0, dtype=torch.int32) + ) + + max_seq_pages = ( + metadata.max_seq_len_k + self.page_size - 1 + ) // self.page_size + page_indices = self.req_to_token[ + req_pool_indices[:, None], + self.draft_extend_metadata["strided_indices"][:max_seq_pages], + ] + metadata.page_table[:, :max_seq_pages].copy_(page_indices // self.page_size) + self.forward_metadata = metadata + + def get_cuda_graph_seq_len_fill_value(self) -> int: + """Get the fill value for sequence lengths in CUDA graph.""" + return 1 + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Initialize the metadata for a forward pass.""" + + metadata = TRTLLMMHAMetadata() + seqlens_in_batch = forward_batch.seq_lens + batch_size = forward_batch.batch_size + device = seqlens_in_batch.device + + if forward_batch.forward_mode.is_decode_or_idle(): + if forward_batch.spec_info is not None: + # Draft Decode + # Here we only support topk = 1 for now. + metadata.cache_seqlens_int32 = ( + seqlens_in_batch + (self.speculative_step_id + 1) + ).to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + ( + self.speculative_step_id + 1 + ) + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum( + metadata.cache_seqlens_int32, dim=0, dtype=torch.int32 + ), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + else: + # Normal Decode + metadata.cache_seqlens_int32 = seqlens_in_batch.to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_q = torch.arange( + 0, batch_size + 1, dtype=torch.int32, device=device + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + elif forward_batch.forward_mode.is_target_verify(): + # Only support topk = 1 for now. + metadata.cache_seqlens_int32 = ( + forward_batch.seq_lens + self.speculative_num_draft_tokens + ).to(torch.int32) + metadata.max_seq_len_q = self.speculative_num_draft_tokens + metadata.max_seq_len_k = ( + forward_batch.seq_lens_cpu.max().item() + + self.speculative_num_draft_tokens + ) + metadata.cu_seqlens_q = torch.arange( + 0, + batch_size * self.speculative_num_draft_tokens + 1, + self.speculative_num_draft_tokens, + dtype=torch.int32, + device=device, + ) + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(metadata.cache_seqlens_int32, dim=0, dtype=torch.int32), + (1, 0), + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + else: + metadata.cache_seqlens_int32 = seqlens_in_batch.to(torch.int32) + metadata.max_seq_len_k = forward_batch.seq_lens_cpu.max().item() + metadata.cu_seqlens_k = torch.nn.functional.pad( + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0) + ) + metadata.page_table = forward_batch.req_to_token_pool.req_to_token[ + forward_batch.req_pool_indices, : metadata.max_seq_len_k + ] + + if ( + any(forward_batch.extend_prefix_lens_cpu) + or forward_batch.forward_mode == ForwardMode.DRAFT_EXTEND + ): + extend_seq_lens = forward_batch.extend_seq_lens + metadata.max_seq_len_q = max(forward_batch.extend_seq_lens_cpu) + metadata.cu_seqlens_q = torch.nn.functional.pad( + torch.cumsum(extend_seq_lens, dim=0, dtype=torch.int32), (1, 0) + ) + else: + metadata.max_seq_len_q = metadata.max_seq_len_k + metadata.cu_seqlens_q = metadata.cu_seqlens_k + + # Convert the page table to a strided format + if self.page_size > 1: + self.strided_indices = torch.arange( + 0, metadata.page_table.shape[1], self.page_size, device=self.device + ) + metadata.page_table = ( + metadata.page_table[:, self.strided_indices] // self.page_size + ) + + self.forward_metadata = metadata + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ) -> torch.Tensor: + """Run forward for decode using TRTLLM MHA kernel.""" + cache_loc = forward_batch.out_cache_loc + if save_kv_cache and k is not None: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + + q = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + # shape conversion: + # [num_pages, page_size, num_kv_heads, head_dim] -> [num_pages, num_kv_heads, page_size, head_dim] + k_cache = k_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ).permute(0, 2, 1, 3) + v_cache = v_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ).permute(0, 2, 1, 3) + kv_cache = (k_cache, v_cache) + + # TODO: add support for quantization + q_scale = 1.0 + k_scale = ( + layer.k_scale_float + if getattr(layer, "k_scale_float", None) is not None + else 1.0 + ) + bmm1_scale = q_scale * k_scale * layer.scaling + bmm2_scale = 1.0 + # sink: additional value per head in the denominator of the softmax. + attention_sink = kwargs.get("sinks", None) + + # Call TRT-LLM kernel + # raw_out: like q, [bs, acc_q_len, num_q_heads, head_dim] but with output dtype + o = flashinfer.decode.trtllm_batch_decode_with_kv_cache( + query=q, + kv_cache=kv_cache, + workspace_buffer=self.workspace_buffer, + block_tables=self.forward_metadata.page_table, + seq_lens=self.forward_metadata.cache_seqlens_int32, + max_seq_len=self.max_context_len, + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + window_left=layer.sliding_window_size, + # TODO: add attention_sink operation or nvfp4 scale factor if needed + sinks=attention_sink, + ) + + return o.view(-1, layer.tp_q_head_num * layer.head_dim) + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + **kwargs, + ): + cache_loc = forward_batch.out_cache_loc + if save_kv_cache and k is not None: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, cache_loc, k, v, layer.k_scale, layer.v_scale + ) + q = q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim) + # [num_pages, page_size, num_kv_heads, head_dim] -> [num_pages, num_kv_heads, page_size, head_dim] + k_cache, v_cache = forward_batch.token_to_kv_pool.get_kv_buffer(layer.layer_id) + k_cache = k_cache.view( + -1, self.page_size, layer.tp_k_head_num, layer.head_dim + ).permute(0, 2, 1, 3) + v_cache = v_cache.view( + -1, self.page_size, layer.tp_v_head_num, layer.head_dim + ).permute(0, 2, 1, 3) + kv_cache = (k_cache, v_cache) + + # sink: additional value per head in the denominator of the softmax. + attention_sink = kwargs.get("sinks", None) + # TODO: add support for quantization + q_scale = 1.0 + k_scale = ( + layer.k_scale_float + if getattr(layer, "k_scale_float", None) is not None + else 1.0 + ) + bmm1_scale = q_scale * k_scale * layer.scaling + bmm2_scale = 1.0 + + o = flashinfer.prefill.trtllm_batch_context_with_kv_cache( + query=q, + kv_cache=kv_cache, + workspace_buffer=self.workspace_buffer, + block_tables=self.forward_metadata.page_table, + seq_lens=self.forward_metadata.cache_seqlens_int32, + max_q_len=self.forward_metadata.max_seq_len_q, + max_kv_len=self.max_context_len, + bmm1_scale=bmm1_scale, + bmm2_scale=bmm2_scale, + batch_size=forward_batch.batch_size, + cum_seq_lens_q=self.forward_metadata.cu_seqlens_q, + cum_seq_lens_kv=self.forward_metadata.cu_seqlens_k, + window_left=layer.sliding_window_size, + # TODO: add attention_sink operation or nvfp4 scale factor if needed + sinks=attention_sink, + ) + + return o.view(-1, layer.tp_q_head_num * layer.head_dim) + + +class TRTLLMHAAttnMultiStepDraftBackend(FlashInferMultiStepDraftBackend): + """Multi-step TRTLLM MHA attention kernel used by EAGLE.""" + + def __init__( + self, model_runner: ModelRunner, topk: int, speculative_num_steps: int + ): + super().__init__(model_runner, topk, speculative_num_steps) + for i in range(speculative_num_steps): + self.attn_backends[i] = TRTLLMHAAttnBackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + kv_last_page_len_buf=self.kv_last_page_len, + speculative_step_id=i, + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata(forward_batch) + + def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int): + for i in range(self.speculative_num_steps): + self.attn_backends[i].init_cuda_graph_state(max_bs, max_num_tokens) + + def init_forward_metadata_capture_cuda_graph( + self, + forward_batch: ForwardBatch, + ): + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + for i in range(self.speculative_num_steps - 1): + self.attn_backends[i].init_forward_metadata_capture_cuda_graph( + forward_batch.batch_size, + forward_batch.batch_size * self.topk, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + encoder_lens=forward_batch.encoder_lens, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + ) + + def init_forward_metadata_replay_cuda_graph( + self, forward_batch: ForwardBatch, bs: int + ): + assert forward_batch.spec_info is not None + assert isinstance(forward_batch.spec_info, EagleDraftInput) + + for i in range(self.speculative_num_steps - 1): + + self.attn_backends[i].init_forward_metadata_replay_cuda_graph( + bs, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens_sum, + encoder_lens=forward_batch.encoder_lens, + forward_mode=ForwardMode.DECODE, + spec_info=forward_batch.spec_info, + seq_lens_cpu=forward_batch.seq_lens_cpu, + ) diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py new file mode 100755 index 000000000..b8d62c3fa --- /dev/null +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -0,0 +1,589 @@ +from __future__ import annotations + +""" +Support attention backend for TRTLLM MLA kernels from flashinfer. +""" + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import torch +import triton + +from sglang.srt.layers.attention.flashinfer_mla_backend import ( + FlashInferMLAAttnBackend, + FlashInferMLAMultiStepDraftBackend, +) +from sglang.srt.layers.attention.utils import ( + TRITON_PAD_NUM_PAGE_PER_BLOCK, + create_flashmla_kv_indices_triton, +) +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import is_flashinfer_available + +if is_flashinfer_available(): + import flashinfer + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.spec_info import SpecInfo + +# Constants +DEFAULT_WORKSPACE_SIZE_MB = 128 # Memory workspace size in MB + +# Block constraint from flashinfer requirements +# From flashinfer.decode._check_trtllm_gen_mla_shape: +# block_num % (128 / block_size) == 0 +# This imposes that the total number of blocks must be divisible by +# (128 / block_size). We capture the 128 constant here so we can +# compute the LCM with other padding constraints. +TRTLLM_BLOCK_CONSTRAINT = 128 + +global_zero_init_workspace_buffer = None + + +@dataclass +class TRTLLMMLAPrefillMetadata: + """Metadata for TRTLLM MLA prefill operations.""" + + max_seq_len: int + cum_seq_lens: torch.Tensor + seq_lens: torch.Tensor + + +@dataclass +class TRTLLMMLADecodeMetadata: + """Metadata for TRTLLM MLA decode operations.""" + + block_kv_indices: Optional[torch.Tensor] = None + max_seq_len: Optional[int] = None + + +class TRTLLMMLABackend(FlashInferMLAAttnBackend): + """TRTLLM MLA attention kernel from flashinfer.""" + + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + q_indptr_decode_buf: Optional[torch.Tensor] = None, + ): + super().__init__(model_runner, skip_prefill, kv_indptr_buf, q_indptr_decode_buf) + + config = model_runner.model_config + + # Model parameters + self.num_q_heads = config.num_attention_heads // get_attention_tp_size() + self.num_kv_heads = config.get_num_kv_heads(get_attention_tp_size()) + self.num_local_heads = config.num_attention_heads // get_attention_tp_size() + + # MLA-specific dimensions + self.kv_lora_rank = config.kv_lora_rank + self.qk_nope_head_dim = config.qk_nope_head_dim + self.qk_rope_head_dim = config.qk_rope_head_dim + self.v_head_dim = config.v_head_dim + self.kv_cache_dim = self.kv_lora_rank + self.qk_rope_head_dim + + # Runtime parameters + self.scaling = config.scaling + self.data_type = model_runner.kv_cache_dtype + self.q_data_type = model_runner.dtype + self.page_size = model_runner.page_size + self.req_to_token = model_runner.req_to_token_pool.req_to_token + + # Workspace allocation + self.workspace_size = DEFAULT_WORKSPACE_SIZE_MB * 1024 * 1024 + global global_zero_init_workspace_buffer + if global_zero_init_workspace_buffer is None: + global_zero_init_workspace_buffer = torch.zeros( + self.workspace_size, + dtype=torch.uint8, + device=model_runner.device, + ) + self.workspace_buffer = global_zero_init_workspace_buffer + + # CUDA graph state + self.decode_cuda_graph_metadata = {} + self.decode_cuda_graph_kv_indices = None + self.forward_prefill_metadata: Optional[TRTLLMMLAPrefillMetadata] = None + self.forward_decode_metadata: Union[TRTLLMMLADecodeMetadata, None] = None + + def _calc_padded_blocks(self, max_seq_len: int) -> int: + """ + Calculate padded block count that satisfies both TRT-LLM and Triton constraints. + + Args: + max_seq_len: Maximum sequence length in tokens + + Returns: + Number of blocks padded to satisfy all constraints + """ + blocks = triton.cdiv(max_seq_len, self.page_size) + + # Apply dual constraints (take LCM to satisfy both): + # 1. TRT-LLM: block_num % (128 / page_size) == 0 + # 2. Triton: page table builder uses 64-index bursts, needs multiple of 64 + trtllm_constraint = TRTLLM_BLOCK_CONSTRAINT // self.page_size + constraint_lcm = math.lcm(trtllm_constraint, TRITON_PAD_NUM_PAGE_PER_BLOCK) + + if blocks % constraint_lcm != 0: + blocks = triton.cdiv(blocks, constraint_lcm) * constraint_lcm + return blocks + + def _create_block_kv_indices( + self, + batch_size: int, + max_blocks: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + device: torch.device, + ) -> torch.Tensor: + """ + Create block KV indices tensor using Triton kernel. + + Args: + batch_size: Batch size + max_blocks: Maximum number of blocks per sequence + req_pool_indices: Request pool indices + seq_lens: Sequence lengths + device: Target device + + Returns: + Block KV indices tensor + """ + block_kv_indices = torch.full( + (batch_size, max_blocks), -1, dtype=torch.int32, device=device + ) + + create_flashmla_kv_indices_triton[(batch_size,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + max_blocks, + NUM_PAGE_PER_BLOCK=TRITON_PAD_NUM_PAGE_PER_BLOCK, + PAGED_SIZE=self.page_size, + ) + + return block_kv_indices + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + """Initialize CUDA graph state for TRTLLM MLA.""" + + max_blocks_per_seq = self._calc_padded_blocks(self.max_context_len) + + self.decode_cuda_graph_kv_indices = torch.full( + (max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device + ) + + super().init_cuda_graph_state(max_bs, max_num_tokens, kv_indices_buf) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + ): + """Initialize metadata for CUDA graph capture.""" + + # Delegate to parent for non-decode modes. + if not forward_mode.is_decode_or_idle(): + return super().init_forward_metadata_capture_cuda_graph( + bs, + num_tokens, + req_pool_indices, + seq_lens, + encoder_lens, + forward_mode, + spec_info, + ) + + # Custom fast-path for decode/idle. + # Capture with full width so future longer sequences are safe during replay + max_blocks_per_seq = self._calc_padded_blocks(self.max_context_len) + block_kv_indices = self.decode_cuda_graph_kv_indices[:bs, :max_blocks_per_seq] + + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + None, + block_kv_indices, + self.req_to_token.stride(0), + max_blocks_per_seq, + NUM_PAGE_PER_BLOCK=TRITON_PAD_NUM_PAGE_PER_BLOCK, + PAGED_SIZE=self.page_size, + ) + + # Record the true maximum sequence length for this capture batch so that + # the kernel launch path (which requires an int not a tensor) can reuse + # it safely during both capture and replay. + max_seq_len_val = int(seq_lens.max().item()) + + metadata = TRTLLMMLADecodeMetadata( + block_kv_indices, + max_seq_len_val, + ) + self.decode_cuda_graph_metadata[bs] = metadata + self.forward_decode_metadata = metadata + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[SpecInfo], + seq_lens_cpu: Optional[torch.Tensor], + ): + """Replay CUDA graph with new inputs.""" + # Delegate to parent for non-decode modes. + if not forward_mode.is_decode_or_idle(): + return super().init_forward_metadata_replay_cuda_graph( + bs, + req_pool_indices, + seq_lens, + seq_lens_sum, + encoder_lens, + forward_mode, + spec_info, + seq_lens_cpu, + ) + + metadata = self.decode_cuda_graph_metadata[bs] + + # Update block indices for new sequences. + create_flashmla_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens[:bs], + None, + metadata.block_kv_indices, + self.req_to_token.stride(0), + metadata.block_kv_indices.shape[1], + NUM_PAGE_PER_BLOCK=TRITON_PAD_NUM_PAGE_PER_BLOCK, + PAGED_SIZE=self.page_size, + ) + + # Update stored max_seq_len so subsequent kernel calls use the correct value + # Prefer CPU tensor to avoid GPU synchronization when available. + if seq_lens_cpu is not None: + metadata.max_seq_len = int(seq_lens_cpu.max().item()) + else: + metadata.max_seq_len = int(seq_lens.max().item()) + + def get_cuda_graph_seq_len_fill_value(self) -> int: + """Get the fill value for sequence lengths in CUDA graph.""" + return 1 + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Initialize the metadata for a forward pass.""" + # Delegate to parent for non-decode modes. + if ( + forward_batch.forward_mode.is_extend() + and not forward_batch.forward_mode.is_target_verify() + and not forward_batch.forward_mode.is_draft_extend() + ): + seq_lens = forward_batch.seq_lens - forward_batch.extend_prefix_lens + cum_seq_lens_q = torch.cat( + ( + torch.tensor([0], device=forward_batch.seq_lens.device), + torch.cumsum(seq_lens, dim=0), + ) + ).int() + max_seq_len = max(forward_batch.extend_seq_lens_cpu) + self.forward_prefill_metadata = TRTLLMMLAPrefillMetadata( + max_seq_len, + cum_seq_lens_q, + seq_lens, + ) + elif forward_batch.forward_mode.is_decode_or_idle(): + bs = forward_batch.batch_size + + # Get maximum sequence length. + if getattr(forward_batch, "seq_lens_cpu", None) is not None: + max_seq = forward_batch.seq_lens_cpu.max().item() + else: + max_seq = forward_batch.seq_lens.max().item() + + max_seqlen_pad = self._calc_padded_blocks(max_seq) + block_kv_indices = self._create_block_kv_indices( + bs, + max_seqlen_pad, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + forward_batch.seq_lens.device, + ) + + max_seq_len_val = int(max_seq) + self.forward_decode_metadata = TRTLLMMLADecodeMetadata( + block_kv_indices, max_seq_len_val + ) + forward_batch.decode_trtllm_mla_metadata = self.forward_decode_metadata + else: + return super().init_forward_metadata(forward_batch) + + def init_mha_chunk_metadata(self, forward_batch: ForwardBatch): + super().init_mha_chunk_metadata(forward_batch, disable_flashinfer_ragged=True) + + def quantize_and_rope_for_fp8( + self, + q_nope: torch.Tensor, + q_rope: torch.Tensor, + k_nope: torch.Tensor, + k_rope: torch.Tensor, + forward_batch: ForwardBatch, + cos_sin_cache: torch.Tensor, + is_neox: bool, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Quantize and apply RoPE for FP8 attention path. + + This function handles the FP8 quantization and RoPE application for MLA attention. + It takes separate query/key nope and rope components, applies RoPE to the rope parts, + quantizes all components to FP8, and merges the query components into a single tensor. + + Args: + q_nope: Query no-position-encoding component [seq_len, num_heads, kv_lora_rank] + - expected dtype: torch.bfloat16 + q_rope: Query RoPE component [seq_len, num_heads, qk_rope_head_dim] + - expected dtype: torch.bfloat16 + k_nope: Key no-position-encoding component [seq_len, num_heads, kv_lora_rank] + - expected dtype: torch.bfloat16 + k_rope: Key RoPE component [seq_len, num_heads, qk_rope_head_dim] + - expected dtype: torch.bfloat16 + forward_batch: Forward batch containing position information + cos_sin_cache: Precomputed cosine/sine cache for RoPE + - expected dtype: matches q_/k_ input dtype (torch.bfloat16) + is_neox: Whether to use NeoX-style RoPE (interleaved) or GPT-style (half rotation) + + Returns: + tuple: (merged_q_out, k_nope_out, k_rope_out) quantized to FP8 + - merged_q_out: [seq_len, num_heads, kv_lora_rank + qk_rope_head_dim], dtype=torch.float8_e4m3fn + - k_nope_out: [seq_len, num_heads, kv_lora_rank], dtype=torch.float8_e4m3fn + - k_rope_out: [seq_len, num_heads, qk_rope_head_dim], dtype=torch.float8_e4m3fn + """ + attn_dtype = torch.float8_e4m3fn + q_len, num_heads = q_rope.shape[0], q_rope.shape[1] + + # Allocate output tensors with FP8 dtype + # Query output will contain merged nope + rope components + q_out = q_rope.new_empty( + q_len, + num_heads, + self.kv_lora_rank + self.qk_rope_head_dim, + dtype=attn_dtype, + ) + + # Key outputs maintain original shapes but with FP8 dtype + k_rope_out = k_rope.new_empty(k_rope.shape, dtype=attn_dtype) + k_nope_out = k_nope.new_empty(k_nope.shape, dtype=attn_dtype) + + # Apply RoPE and quantize all components in a single fused kernel call + # This kernel handles: + # 1. RoPE application to q_rope and k_rope using cos_sin_cache and positions + # 2. Quantization of all components to FP8 format + # 3. Output placement into pre-allocated tensors + flashinfer.rope.mla_rope_quantize_fp8( + q_rope=q_rope, + k_rope=k_rope, + q_nope=q_nope, + k_nope=k_nope, + cos_sin_cache=cos_sin_cache, + pos_ids=forward_batch.positions, + is_neox=is_neox, + quantize_dtype=attn_dtype, + # Output tensor slicing: q_out contains [nope_part, rope_part] + q_rope_out=q_out[..., self.kv_lora_rank :], # RoPE part goes to end + k_rope_out=k_rope_out, + q_nope_out=q_out[..., : self.kv_lora_rank], # Nope part goes to beginning + k_nope_out=k_nope_out, + # Quantization scales (set to 1.0 for no additional scaling) + quant_scale_q=1.0, + quant_scale_kv=1.0, + ) + + return q_out, k_nope_out, k_rope_out + + def forward_decode( + self, + q: torch.Tensor, # q_nope + k: torch.Tensor, # k_nope + v: torch.Tensor, # not used in this backend + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + cos_sin_cache: Optional[torch.Tensor] = None, + is_neox: Optional[bool] = False, + ) -> torch.Tensor: + """Run forward for decode using TRTLLM MLA kernel.""" + merge_query = q_rope is not None + if self.data_type == torch.float8_e4m3fn: + # For FP8 path, we quantize the query and rope parts and merge them into a single tensor + # Note: rope application in deepseek_v2.py:forward_absorb_prepare is skipped for FP8 decode path of this trtllm_mla backend + assert all( + x is not None for x in [q_rope, k_rope, cos_sin_cache] + ), "For FP8 path and using flashinfer.rope.mla_rope_quantize we need all of q_rope, k_rope and cos_sin_cache to be not None." + q, k, k_rope = self.quantize_and_rope_for_fp8( + q, + q_rope, + k.squeeze(1), + k_rope.squeeze(1), + forward_batch, + cos_sin_cache, + is_neox, + ) + merge_query = False + + # Save KV cache if requested + if save_kv_cache: + assert ( + k is not None and k_rope is not None + ), "For populating trtllm_mla kv cache, both k_nope and k_rope should be not None." + forward_batch.token_to_kv_pool.set_mla_kv_buffer( + layer, forward_batch.out_cache_loc, k, k_rope + ) + + # Prepare query tensor inline + if merge_query: + # For FP16 path, we merge the query and rope parts into a single tensor + q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim) + q_rope_reshaped = q_rope.view( + -1, layer.tp_q_head_num, layer.head_dim - layer.v_head_dim + ) + query = torch.cat([q_nope, q_rope_reshaped], dim=-1) + else: + # For FP8 path, we already have the query and rope parts merged because of the quantize_and_rope_for_fp8 function + query = q.view(-1, layer.tp_q_head_num, layer.head_dim) + + # Ensure query has shape [bs, acc_q_len, num_q_heads, head_dim] when seq_len 1 + if query.dim() == 3: + query = query.unsqueeze(1) + + # Prepare KV cache inline + k_cache = forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id) + kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1) + + # Get metadata + metadata = ( + getattr(forward_batch, "decode_trtllm_mla_metadata", None) + or self.forward_decode_metadata + ) + + # Scale computation for TRTLLM MLA kernel BMM1 operation: + # The final BMM1 scale is computed as: q_scale * k_scale * softmax_scale + # Scale components: + # - q_scale: Query scaling factor (set to 1.0 for both FP16/FP8 paths) + # - k_scale: Key scaling factor from model checkpoint (defaults to 1.0 if not available) + # - softmax_scale: Attention softmax scaling = 1/sqrt(head_dim), pre-computed as layer.scaling + # This unified approach works for both FP16 and FP8 quantized attention paths. + q_scale = 1.0 + k_scale = ( + layer.k_scale_float + if getattr(layer, "k_scale_float", None) is not None + else 1.0 + ) + + bmm1_scale = q_scale * k_scale * layer.scaling + + # Call TRT-LLM kernel + raw_out = flashinfer.decode.trtllm_batch_decode_with_kv_cache_mla( + query=query, + kv_cache=kv_cache, + workspace_buffer=self.workspace_buffer, + qk_nope_head_dim=self.qk_nope_head_dim, + kv_lora_rank=self.kv_lora_rank, + qk_rope_head_dim=self.qk_rope_head_dim, + block_tables=metadata.block_kv_indices, + seq_lens=forward_batch.seq_lens.to(torch.int32), + max_seq_len=metadata.max_seq_len, + bmm1_scale=bmm1_scale, + ) + + # Reshape output directly without slicing + output = raw_out.view(-1, layer.tp_q_head_num * layer.v_head_dim) + return output + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + q_rope: Optional[torch.Tensor] = None, + k_rope: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if ( + forward_batch.forward_mode.is_target_verify() + or forward_batch.forward_mode.is_draft_extend() + ): + return super().forward_extend( + q, k, v, layer, forward_batch, save_kv_cache, q_rope, k_rope + ) + + if not forward_batch.attn_attend_prefix_cache: + q = q.view(-1, layer.tp_q_head_num, layer.head_dim) + k = k.view(-1, layer.tp_k_head_num, layer.head_dim) + v = v.view(-1, layer.tp_k_head_num, layer.v_head_dim) + output = flashinfer.prefill.trtllm_ragged_attention_deepseek( + query=q, + key=k, + value=v, + workspace_buffer=self.workspace_buffer, + seq_lens=self.forward_prefill_metadata.seq_lens, + max_q_len=self.forward_prefill_metadata.max_seq_len, + max_kv_len=self.forward_prefill_metadata.max_seq_len, + bmm1_scale=layer.scaling, + bmm2_scale=1.0, + o_sf_scale=1.0, + batch_size=forward_batch.batch_size, + window_left=-1, + cum_seq_lens_q=self.forward_prefill_metadata.cum_seq_lens, + cum_seq_lens_kv=self.forward_prefill_metadata.cum_seq_lens, + enable_pdl=False, + is_causal=True, + return_lse=forward_batch.mha_return_lse, + ) + else: + # replace with trtllm ragged attention once accuracy is resolved. + output = super().forward_extend( + q, k, v, layer, forward_batch, save_kv_cache, q_rope, k_rope + ) + return output + + +class TRTLLMMLAMultiStepDraftBackend(FlashInferMLAMultiStepDraftBackend): + """Multi-step draft backend for TRT-LLM MLA used by EAGLE.""" + + def __init__( + self, model_runner: "ModelRunner", topk: int, speculative_num_steps: int + ): + super().__init__(model_runner, topk, speculative_num_steps) + + for i in range(self.speculative_num_steps): + self.attn_backends[i] = TRTLLMMLABackend( + model_runner, + skip_prefill=True, + kv_indptr_buf=self.kv_indptr[i], + q_indptr_decode_buf=self.q_indptr_decode, + ) diff --git a/python/sglang/srt/layers/attention/utils.py b/python/sglang/srt/layers/attention/utils.py new file mode 100644 index 000000000..e8cd2e158 --- /dev/null +++ b/python/sglang/srt/layers/attention/utils.py @@ -0,0 +1,99 @@ +import triton +import triton.language as tl + +# Keep this in sync with the Triton kernel inside `create_flashmla_kv_indices_triton`. +# Number of pages that the kernel writes per iteration. +# Exposed here so other Python modules can import it instead of hard-coding 64. +TRITON_PAD_NUM_PAGE_PER_BLOCK = 64 + + +@triton.jit +def create_flashinfer_kv_indices_triton( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices_ptr, + page_kernel_lens_ptr, + kv_indptr, + kv_start_idx, + kv_indices_ptr, + req_to_token_ptr_stride: tl.constexpr, +): + BLOCK_SIZE: tl.constexpr = 512 + pid = tl.program_id(axis=0) + + # find the req pool idx, this is for batch to token + req_pool_index = tl.load(req_pool_indices_ptr + pid) + kv_indices_offset = tl.load(kv_indptr + pid) + + kv_start = 0 + kv_end = 0 + if kv_start_idx: + kv_start = tl.load(kv_start_idx + pid).to(tl.int32) + kv_end = kv_start + kv_end += tl.load(page_kernel_lens_ptr + pid).to(tl.int32) + + num_loop = tl.cdiv(kv_end - kv_start, BLOCK_SIZE) + for i in range(num_loop): + # index into req_to_token_ptr needs to be int64 + offset = tl.arange(0, BLOCK_SIZE).to(tl.int64) + i * BLOCK_SIZE + mask = offset < kv_end - kv_start + data = tl.load( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + kv_start + + offset, + mask=mask, + ) + tl.store(kv_indices_ptr + kv_indices_offset + offset, data, mask=mask) + + +@triton.jit +def create_flashmla_kv_indices_triton( + req_to_token_ptr, # [max_batch, max_context_len] + req_pool_indices_ptr, + page_kernel_lens_ptr, + kv_start_idx, + kv_indices_ptr, + req_to_token_ptr_stride: tl.constexpr, + kv_indices_ptr_stride: tl.constexpr, + NUM_PAGE_PER_BLOCK: tl.constexpr = TRITON_PAD_NUM_PAGE_PER_BLOCK, + PAGED_SIZE: tl.constexpr = 64, +): + BLOCK_SIZE: tl.constexpr = 4096 + pid = tl.program_id(axis=0) + + # find the req pool idx, this is for batch to token + req_pool_index = tl.load(req_pool_indices_ptr + pid) + + kv_start = 0 + kv_end = 0 + if kv_start_idx: + kv_start = tl.load(kv_start_idx + pid).to(tl.int32) + kv_end = kv_start + + kv_end += tl.load(page_kernel_lens_ptr + pid).to(tl.int32) + + num_paged = tl.cdiv(kv_end - kv_start, PAGED_SIZE) + num_pages_loop = tl.cdiv(kv_end - kv_start, BLOCK_SIZE) + + for i in range(num_pages_loop): + # index into req_to_token_ptr needs to be int64 + paged_offset = ( + tl.arange(0, NUM_PAGE_PER_BLOCK).to(tl.int64) + i * NUM_PAGE_PER_BLOCK + ) * PAGED_SIZE + paged_offset_out = tl.arange(0, NUM_PAGE_PER_BLOCK) + i * NUM_PAGE_PER_BLOCK + + mask = paged_offset < num_paged * PAGED_SIZE + mask_out = paged_offset_out < num_paged + + data = tl.load( + req_to_token_ptr + + req_pool_index * req_to_token_ptr_stride + + kv_start + + paged_offset, + mask=mask, + ) + tl.store( + kv_indices_ptr + pid * kv_indices_ptr_stride + paged_offset_out, + data // PAGED_SIZE, + mask=mask_out, + ) diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py new file mode 100644 index 000000000..2be3e450b --- /dev/null +++ b/python/sglang/srt/layers/attention/vision.py @@ -0,0 +1,630 @@ +from __future__ import annotations + +import dataclasses +import functools +import math +from functools import lru_cache, partial +from typing import Any, Callable, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from einops import rearrange + +from sglang.srt.layers.dp_attention import get_attention_tp_rank, get_attention_tp_size +from sglang.srt.utils import ( + get_device_capability, + is_blackwell, + is_cuda, + print_info_once, +) + +_is_cuda = is_cuda() + +if _is_cuda: + from sgl_kernel.flash_attn import flash_attn_varlen_func + +from sglang.srt.distributed import ( + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, +) +from sglang.srt.distributed import utils as dist_utils +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.srt.layers.layernorm import RMSNorm +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.srt.layers.quantization import QuantizationConfig +from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.utils import add_prefix + +ROTARY_EMBED_CLASSES = { + "normal": apply_rotary_pos_emb, +} + + +@dataclasses.dataclass +class SingletonCache: + data: Any = None + + def set_data(self, value: Any) -> None: + self.data = value + + def get_data(self) -> Optional[Any]: + return self.data + + def empty(self) -> bool: + return self.get_data() is None + + +# TODO: requires real seqlens from images +@functools.lru_cache(maxsize=128) +def _get_cu_seqlens_for_shape(batch_size: int, seqlen: int, device) -> torch.Tensor: + """ + Generates cumulative sequence lengths (cu_seqlens) for a given batch_size, seqlen, and device. + Caches the result based on these parameters. + """ + cu_seqlens = torch.arange( + 0, + (batch_size + 1) * seqlen, + step=seqlen, + dtype=torch.int32, + device=device, + ) + return cu_seqlens + + +class VisionSdpaAttention(nn.Module): + r""" + Scaled Dot Product Attention inner product + + """ + + def __init__( + self, + head_dim: int, + num_heads: int, + num_kv_heads: int, + dropout: float = 0.0, + flatten_batch: bool = False, + softmax_in_single_precision: bool = False, + **kwargs, + ): + super().__init__() + self.head_size = head_dim + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.flatten_batch = flatten_batch + self.softmax_in_single_precision = softmax_in_single_precision + self.dropout = dropout + self.scale = 1.0 / math.sqrt(self.head_size) + + @staticmethod + @lru_cache(maxsize=128) + def _generate_mask_cache( + s: int, flatten_batch: bool, cu_seqlens: tuple + ) -> torch.BoolTensor: + """ + Generate a boolean attention mask with caching mechanism. + Args: + s: sequence length + flatten_batch: whether to flatten batch dimension + cu_seqlens: tuple of cumulative sequence lengths + Returns: + attention mask tensor of shape [b, 1, s, s] or [1, s, s] + """ + if flatten_batch: + mask = torch.zeros([1, s, s], dtype=torch.bool) + for i in range(1, len(cu_seqlens)): + start = cu_seqlens[i - 1] + end = cu_seqlens[i] + mask[..., start:end, start:end] = True + else: + # [1, 1, 1, s] + row_indices = torch.arange(s).view(1, 1, 1, s) + # [1, 1, s, 1] + col_indices = torch.arange(s).view(1, 1, s, 1) + # [b, 1, 1, 1] + seq_lens = torch.tensor( + [end - start for start, end in zip(cu_seqlens[:-1], cu_seqlens[1:])], + ).view(-1, 1, 1, 1) + + mask = (row_indices < seq_lens) & (col_indices < seq_lens) + + return mask + + def generate_patch_attention_mask( + self, + s: int, + cu_seqlens: Optional[torch.Tensor], + flatten_batch: bool = False, + ) -> Optional[torch.Tensor]: + r""" + Creates a non-causal 4D mask of shape `(b, 1, s, s)` or `(1, 1, s, s)`. + Args: + s: sequence length + cu_seqlens: cumulative sequence lengths tensor. If not, returns an empty mask + flatten_batch: whether to flatten batch dimension + Returns: + attention mask tensor or None + """ + if cu_seqlens is None: + return None + + cu_seqlens_tuple = tuple(cu_seqlens.cpu().tolist()) + + return self._generate_mask_cache(s, flatten_batch, cu_seqlens_tuple) + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + bsz: int, + cu_seqlens: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + r""" + Args: + cu_seqlens: [b] + Returns: + [b * s, h, head_size] + """ + if self.flatten_batch: + assert bsz == 1, "flatten_batch is True, bsz must be 1" + + assert q.dim() == 3, q.shape + + s = q.shape[0] // bsz + + # [b, 1, s, s] + if attention_mask is None: + attention_mask = self.generate_patch_attention_mask( + s, cu_seqlens, flatten_batch=self.flatten_batch + ) + + if attention_mask is None: + if self.softmax_in_single_precision: + raise RuntimeError("Empty attention mask") + else: + attention_mask = attention_mask.to(device=q.device) + + q, k, v = [rearrange(x, "(b s) h d -> b h s d", b=bsz) for x in [q, k, v]] + + if self.softmax_in_single_precision: + k = rearrange(k, "b h s d -> b h d s") + attn_weights = torch.matmul(q, k) * self.scale + del k + # masking + attention_mask = (~attention_mask) * torch.finfo(q.dtype).min + attn_weights = attn_weights + attention_mask + del attention_mask + # full-precision + attn_weights = nn.functional.softmax( + attn_weights, dim=-1, dtype=torch.float32 + ).to(q.dtype) + attn_weights = nn.functional.dropout( + attn_weights, p=self.dropout, training=False + ) + output = torch.matmul(attn_weights, v) + del attn_weights, v + else: + # SDPA + # [b, h, s, head_size] + output = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=attention_mask, + dropout_p=self.dropout, + is_causal=False, + ) + + # [b, h, s, head_size] --> [b * s, h, head_size] + output = rearrange(output, "b h s d -> (b s) h d") + + return output + + +class VisionTritonAttention(nn.Module): + """ + Triton-implemented attention without a causal mask + """ + + def __init__( + self, + **kwargs, + ): + super().__init__() + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: Optional[torch.Tensor], + bsz: int, + seq_len: int, + **kwargs, + ) -> torch.Tensor: + r""" + Args: + cu_seqlens: [b] + Returns: + [b * s, h, head_size] + """ + if cu_seqlens is None: + cu_seqlens = _get_cu_seqlens_for_shape(bsz, seq_len, device=q.device) + + # [b * s, head, head_size] + output = torch.empty_like(q) + seq_lens = cu_seqlens[1:] - cu_seqlens[:-1] + max_seqlen = seq_lens.max().item() + context_attention_fwd( + q, + k, + v, + output, + cu_seqlens.cuda(), + seq_lens.cuda(), + max_seqlen, + is_causal=False, + ) + + return output + + +class VisionFlash3Attention(nn.Module): + def __init__( + self, + **kwargs, + ): + if not _is_cuda: + raise Exception("VisionFlash3Attention is only available for cuda") + super().__init__() + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens: Optional[Union[SingletonCache, torch.Tensor]], + bsz: int, + seq_len: int, + **kwargs, + ) -> torch.Tensor: + r""" + Args: + cu_seqlens: [b] + Returns: + [b * s, h, head_size] + """ + if cu_seqlens is None: + cu_seqlens = _get_cu_seqlens_for_shape(bsz, seq_len, device=q.device) + elif isinstance(cu_seqlens, SingletonCache): + if cu_seqlens.empty(): + cu_seqlens.set_data( + _get_cu_seqlens_for_shape(bsz, seq_len, device=q.device) + ) + cu_seqlens = cu_seqlens.get_data() + + cu_seqlens = cu_seqlens.to(dtype=torch.int32).to(q.device) + seq_lens = cu_seqlens[1:] - cu_seqlens[:-1] + max_seqlen = seq_lens.max().item() + + output = flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q=cu_seqlens, + cu_seqlens_k=cu_seqlens, + max_seqlen_q=max_seqlen, + max_seqlen_k=max_seqlen, + ) + + return output + + +QKV_BACKEND_IMPL = { + "triton_attn": VisionTritonAttention, + "sdpa": VisionSdpaAttention, + "fa3": VisionFlash3Attention, +} + + +class VisionAttention(nn.Module): + r""" + Multi-headed attention without any cache, mostly used for multimodal transformers. + + + Args: + use_qkv_parallel (bool, optional): If True, use QKV-parallel attention. + softmax_in_single_precision (bool, default to False): + if ``True``, the softmax will be performed in single-precision + Otherwise, it will be performed in half-precision + + """ + + def __init__( + self, + embed_dim: int, + num_heads: int, + projection_size: int, + use_qkv_parallel: bool, + qkv_backend: Optional[str] = None, + quant_config: Optional[QuantizationConfig] = None, + dropout: float = 0.0, + softmax_in_single_precision: bool = False, + flatten_batch: bool = False, + prefix: str = "", + proj_bias: bool = True, + num_dummy_heads: int = 0, + qkv_bias: bool = True, + qk_normalization: bool = False, + layer_norm_eps: float = 1e-06, + customized_position_embedding_applier: Callable[ + [torch.Tensor, torch.Tensor, Any, Any], Tuple[torch.Tensor, torch.Tensor] + ] = None, + **kwargs, + ): + super().__init__() + attn_tp_rank = get_attention_tp_rank() + attn_tp_size = get_attention_tp_size() + self.tp_size = attn_tp_size + self.tp_rank = attn_tp_rank + self.dropout = dropout + self.head_size = embed_dim // num_heads + self.hidden_size_per_attention_head = dist_utils.divide( + projection_size, num_heads + ) + self.num_attention_heads_per_partition = dist_utils.divide( + num_dummy_heads + num_heads, self.tp_size + ) + self.num_attention_kv_heads_per_partition = dist_utils.divide( + num_dummy_heads + num_heads, self.tp_size + ) + + self.q_size = self.num_attention_heads_per_partition * self.head_size + self.kv_size = self.num_attention_kv_heads_per_partition * self.head_size + + self.qk_normalization = qk_normalization + + # Additional dummy heads are used to enable TP for common GPU counts. + self.dummy_dim = (num_dummy_heads + num_heads) * self.head_size + + if self.qk_normalization: + self.q_norm = RMSNorm( + self.dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim + ) + self.k_norm = RMSNorm( + self.dummy_dim, eps=layer_norm_eps, var_hidden_size=embed_dim + ) + + # Select attention backend via a unified method + _passed_backend = qkv_backend + qkv_backend = self._determine_attention_backend(_passed_backend) + if ( + global_server_args_dict["mm_attention_backend"] is None + and _passed_backend is None + ): + print_info_once(f"Multimodal attention backend not set. Use {qkv_backend}.") + print_info_once(f"Using {qkv_backend} as multimodal attention backend.") + + self.customized_position_embedding_applier = ( + customized_position_embedding_applier + ) + self.qkv_backend = QKV_BACKEND_IMPL[qkv_backend]( + head_dim=self.head_size, + num_heads=self.num_attention_heads_per_partition, + num_kv_heads=self.num_attention_kv_heads_per_partition, + dropout=dropout, + flatten_batch=flatten_batch, + softmax_in_single_precision=softmax_in_single_precision, + ) + + self.use_qkv_parallel = use_qkv_parallel + if use_qkv_parallel: + self.qkv_proj = QKVParallelLinear( + hidden_size=embed_dim, + head_size=self.head_size, + total_num_heads=num_dummy_heads + num_heads, + total_num_kv_heads=num_dummy_heads + num_heads, + bias=qkv_bias, + quant_config=quant_config, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + prefix=add_prefix("qkv_proj", prefix), + ) + else: + self.qkv_proj = ColumnParallelLinear( + input_size=embed_dim, + output_size=3 * self.dummy_dim, + bias=qkv_bias, + quant_config=quant_config, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + prefix=add_prefix("qkv_proj", prefix), + ) + self.proj = RowParallelLinear( + input_size=self.dummy_dim, + output_size=embed_dim, + bias=proj_bias, + quant_config=quant_config, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + prefix=add_prefix("proj", prefix), + ) + + def _determine_attention_backend(self, passed_backend: Optional[str]) -> str: + """Decide the multimodal attention backend string. + + Priority: server args override > constructor arg > platform default. + + Platform defaults: + - CUDA: "triton_attn" + - Non-CUDA: "sdpa" + """ + override_backend = global_server_args_dict["mm_attention_backend"] + if override_backend is not None: + backend = override_backend + elif passed_backend is not None: + backend = passed_backend + elif is_cuda(): + major, minor = get_device_capability() + if major == 9: + backend = "fa3" + else: + backend = "triton_attn" + else: + backend = "sdpa" + if backend == "fa3" and is_blackwell(): + raise ValueError("The 'fa3' backend is not supported on Blackwell GPUs") + + return backend + + def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor): + """apply qk norm for internvl vit attn""" + q = q.flatten(1, 2) + k = k.flatten(1, 2) + + if self.tp_size > 1: + q = tensor_model_parallel_all_gather(q.contiguous()) + k = tensor_model_parallel_all_gather(k.contiguous()) + q = self.q_norm(q) + k = self.k_norm(k) + if self.tp_size > 1: + splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size) + q = splitter(q)[self.tp_rank] + k = splitter(k)[self.tp_rank] + q = q.unflatten(-1, (-1, self.head_size)) + k = k.unflatten(-1, (-1, self.head_size)) + return q, k + + def forward( + self, + x: torch.Tensor, + cu_seqlens: Optional[torch.Tensor] = None, + position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + r""" + Args: + x: [b, s, embed_dim] + cu_seqlens: [b] + Returns: + [s, b, head * head_size] + """ + if x.dim() == 2: + x = x.unsqueeze(0) + assert x.dim() == 3, x.shape + x_shape = x.shape + bsz, s, _ = x_shape + head = self.num_attention_heads_per_partition + kv_head = self.num_attention_kv_heads_per_partition + if self.use_qkv_parallel: + # [b, s, embed_dim] --> [b, s, embed_dim] + qkv, _ = self.qkv_proj(x) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + # [b, s, embed_dim] --> [b * s, head, head_size] + q = q.reshape(bsz * s, head, -1).contiguous() + k = k.reshape(bsz * s, kv_head, -1).contiguous() + v = v.reshape(bsz * s, kv_head, -1).contiguous() + else: + # [b, s, embed_dim] --> [s, b, embed_dim] + x = rearrange(x, "b s ... -> s b ...") + # [s, b, embed_dim] --> [s, b, head * 3 * head_size] + qkv, _ = self.qkv_proj(x) + + # [s, b, head, head_dim_sum] + new_x_shape = qkv.size()[:-1] + ( + head, + self.q_size + 2 * self.kv_size, + ) + qkv = qkv.view(*new_x_shape) + + # [s, b, head, 3 * head_size] --> 3 [s, b, head, head_size] + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + # [s, b, head, head_size] --> [b, s, head, head_size] + q, k, v = [ + rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v) + ] + + if position_embeddings is not None: + original_shape = q.shape + + if self.customized_position_embedding_applier is not None: + q, k = self.customized_position_embedding_applier( + q, k, position_embeddings, x_shape + ) + q = q.view(original_shape) + k = k.view(original_shape) + else: + cos, sin = position_embeddings + + # [total_tokens, head, head_size] + q = q.view(-1, head, self.head_size) + k = k.view(-1, head, self.head_size) + + q, k = apply_rotary_pos_emb(q, k, cos, sin) + + q = q.view(original_shape) + k = k.view(original_shape) + + if q.dim() == 4: + # [b, s, head, head_size] --> [b * s, head, head_size] + q = rearrange(q, "b s ... -> (b s) ...") + if k.dim() == 4: + # [b, s, head, head_size] --> [b * s, head, head_size] + k = rearrange(k, "b s ... -> (b s) ...") + if v.dim() == 4: + # [b, s, head, head_size] --> [b * s, head, head_size] + v = rearrange(v, "b s ... -> (b s) ...") + + assert q.dim() == 3, q.dim() + assert k.dim() == 3, k.dim() + assert v.dim() == 3, v.dim() + + # internvl + if self.qk_normalization: + q, k = self._apply_qk_norm(q, k) + + output = self.qkv_backend.forward( + q=q, + k=k, + v=v, + bsz=bsz, + seq_len=s, + cu_seqlens=cu_seqlens, + attention_mask=attention_mask, + ) + + assert output.dim() == 3, output.shape + + if self.use_qkv_parallel: + # [b * s, h, head_size] --> [b, s, h * head_size] + output = rearrange(output, "(b s) ... h d -> b s ... (h d)", b=bsz) + + # [b, s, h * head_size] --> [b, s, h * head_size] + output, _ = self.proj(output) + else: + # [b * s, h, head_size] --> [s, b, h * head_size] + context_layer = rearrange( + output, "(b s) h d -> s b (h d)", b=bsz, s=s + ).contiguous() + + # [s, b, h * head_size] --> [s, b, h * head_size] + output, _ = self.proj(context_layer) + + # [s, b, h * head_size] --> [b, s, h * head_size] + output = output.view(bsz, s, -1) + + return output diff --git a/python/sglang/srt/layers/attention/vision_utils.py b/python/sglang/srt/layers/attention/vision_utils.py new file mode 100644 index 000000000..ecccb1f85 --- /dev/null +++ b/python/sglang/srt/layers/attention/vision_utils.py @@ -0,0 +1,65 @@ +"""Utility functions for vision attention layers.""" + +import torch + +from sglang.srt.layers.dp_attention import get_attention_tp_size + + +def update_vit_attn_dummy_heads_config(config): + """Update HF config to ensure vision attention num_attention_heads is divisible by tp_size""" + tp_size = get_attention_tp_size() + num_heads = getattr( + config.vision_config, + "num_heads", + getattr(config.vision_config, "num_attention_heads", None), + ) + head_dim = config.vision_config.hidden_size // num_heads + num_dummy_heads = 0 + + if num_heads % tp_size != 0: + num_dummy_heads = ((num_heads + tp_size - 1) // tp_size) * tp_size - num_heads + + setattr(config.vision_config, "head_dim", head_dim) + setattr(config.vision_config, "num_dummy_heads", num_dummy_heads) + + +def pad_vit_attn_dummy_heads(config, name: str, loaded_weight: torch.Tensor): + """Pad attention qkv weights for dummy heads""" + num_dummy_heads = config.vision_config.num_dummy_heads + if num_dummy_heads == 0: + return loaded_weight + head_dim = config.vision_config.head_dim + + if "attn.qkv_proj" in name: + wq, wk, wv = loaded_weight.chunk(3, dim=0) + if name.endswith(".weight"): + dummy_shape = [num_dummy_heads, head_dim, wq.shape[-1]] + elif name.endswith(".bias"): + dummy_shape = [num_dummy_heads, head_dim] + else: + raise RuntimeError(f"Unsupported weight with name={name}") + pad_func = lambda x: torch.cat( + [x.unflatten(0, (-1, head_dim)), x.new_zeros(dummy_shape)], dim=0 + ).flatten(0, 1) + wq, wk, wv = pad_func(wq), pad_func(wk), pad_func(wv) + loaded_weight = torch.cat([wq, wk, wv], dim=0) + elif any([_ in name for _ in ["attn.q_proj", "attn.k_proj", "attn.v_proj"]]): + if name.endswith(".weight"): + dummy_shape = [num_dummy_heads, head_dim, loaded_weight.shape[-1]] + elif name.endswith(".bias"): + dummy_shape = [num_dummy_heads, head_dim] + else: + raise RuntimeError(f"Unsupported weight with name={name}") + padded_weight = loaded_weight.new_zeros(dummy_shape) + loaded_weight = torch.cat( + [loaded_weight.unflatten(0, (-1, head_dim)), padded_weight], dim=0 + ).flatten(0, 1) + elif "attn.proj.weight" in name: + padded_weight = loaded_weight.new_zeros( + loaded_weight.shape[0], head_dim * num_dummy_heads + ) + loaded_weight = torch.cat([loaded_weight, padded_weight], dim=-1) + elif "attn.q_norm.weight" in name or "attn.k_norm.weight" in name: + padded_weight = loaded_weight.new_zeros(head_dim * num_dummy_heads) + loaded_weight = torch.cat([loaded_weight, padded_weight], dim=0) + return loaded_weight diff --git a/python/sglang/srt/layers/attention/wave_backend.py b/python/sglang/srt/layers/attention/wave_backend.py new file mode 100644 index 000000000..eb6e061ac --- /dev/null +++ b/python/sglang/srt/layers/attention/wave_backend.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional, Union + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.attention.base_attn_backend import AttentionBackend +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.srt.layers.dp_attention import get_attention_tp_size +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.utils import get_bool_env_var, get_device_core_count + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.speculative.eagle_utils import EagleDraftInput, EagleVerifyInput + +logger = logging.getLogger(__name__) + + +@triton.jit +def get_num_kv_splits_triton( + num_kv_splits_ptr, + seq_lens_ptr, + num_seq, + num_group, + num_head, + num_kv_head, + max_kv_splits, + device_core_count, + MAX_NUM_SEQ: tl.constexpr, +): + # TODO: this method is tunable, we need more online serving data to tune it + offs_seq = tl.arange(0, MAX_NUM_SEQ) + mask_seq = offs_seq < num_seq + + seq_lens = tl.load(seq_lens_ptr + offs_seq, mask=mask_seq, other=0) + max_seq_len = tl.max(seq_lens) + seq_lens = tl.load(seq_lens_ptr + offs_seq, mask=mask_seq, other=max_seq_len) + min_seq_len = tl.min(seq_lens) + if max_seq_len * 8 < min_seq_len * 10: + min_seq_len = max_seq_len + max_kv_splits_1 = tl.minimum(tl.cdiv(max_seq_len, min_seq_len), max_kv_splits) + kv_chunk_size_1 = tl.cdiv(max_seq_len, max_kv_splits_1) + + # NOTE: this is a hack to let num_kv_split grows up with seqlen gradually + ext_seq_len = tl.cast(max_seq_len, tl.float32) / 64.0 + ext_device_core_count = tl.cast( + device_core_count * tl.maximum(tl.log2(ext_seq_len), 1.0), tl.int32 + ) + block_h, num_kv_group = 16, num_head // num_kv_head + if num_kv_group == 1: + token_grid = num_seq * num_group * num_head + else: + # from triton_ops/decode_attention.py:_decode_grouped_att_m_fwd + block_h = tl.minimum(block_h, num_kv_group) + token_grid = num_seq * num_group * tl.cdiv(num_head, block_h) + max_kv_splits_2 = tl.minimum( + tl.cdiv(ext_device_core_count, token_grid), max_kv_splits + ) + kv_chunk_size_2 = tl.cdiv(max_seq_len, max_kv_splits_2) + + num_kv_splits = tl.maximum( + tl.cdiv(seq_lens, kv_chunk_size_1), tl.cdiv(seq_lens, kv_chunk_size_2) + ) + + offs_token = offs_seq * num_group + mask_token = offs_token < num_seq * num_group + for i in range(0, num_group): + tl.store(num_kv_splits_ptr + i + offs_token, num_kv_splits, mask=mask_token) + + +@dataclass +class ForwardMetadata: + attn_logits: torch.Tensor + attn_lse: torch.Tensor + max_extend_len: int + num_kv_splits: torch.Tensor + kv_indptr: torch.Tensor + kv_indices: torch.Tensor + qo_indptr: torch.Tensor + custom_mask: torch.Tensor + mask_indptr: torch.Tensor + + +class WaveAttnBackend(AttentionBackend): + def __init__( + self, + model_runner: ModelRunner, + skip_prefill: bool = False, + kv_indptr_buf: Optional[torch.Tensor] = None, + ): + # Lazy import to avoid the initialization of cuda context + from sglang.srt.layers.attention.wave_ops.decode_attention import ( + decode_attention_fwd, + ) + from sglang.srt.layers.attention.wave_ops.extend_attention import ( + extend_attention_wave, + ) + + super().__init__() + + # Set unique cache dir for each process to avoid cache write races + import wave_lang.kernel.wave.cache as cache + + base_cache_dir = cache.CACHE_BASE_DIR + new_dir = base_cache_dir / f"worker_{model_runner.tp_rank}" + logger.info(f"Setting Wave cache dir: {new_dir}") + cache.CACHE_BASE_DIR = new_dir + + self.decode_attention_fwd = decode_attention_fwd + self.extend_attention_fwd = extend_attention_wave + + self.skip_prefill = skip_prefill + + max_bs = model_runner.req_to_token_pool.size + + if kv_indptr_buf is None: + self.kv_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + else: + self.kv_indptr = kv_indptr_buf + + self.req_to_token = model_runner.req_to_token_pool.req_to_token + + if not self.skip_prefill: + self.qo_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int32, device=model_runner.device + ) + + self.mask_indptr = torch.zeros( + (max_bs + 1,), dtype=torch.int64, device=model_runner.device + ) + + self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens + + self.num_head = ( + model_runner.model_config.num_attention_heads // get_attention_tp_size() + ) + self.num_kv_head = model_runner.model_config.get_num_kv_heads( + get_attention_tp_size() + ) + + self.static_kv_splits = get_bool_env_var( + "SGLANG_TRITON_DECODE_ATTN_STATIC_KV_SPLITS", "false" + ) + self.max_kv_splits = model_runner.server_args.triton_attention_num_kv_splits + self.v_head_dim = model_runner.token_to_kv_pool.get_value_buffer(0).shape[-1] + + self.forward_metadata: ForwardMetadata = None + + self.max_context_len = model_runner.model_config.context_len + + self.device = model_runner.device + self.device_core_count = get_device_core_count(model_runner.gpu_id) + + def get_num_kv_splits( + self, + num_kv_splits: torch.Tensor, + seq_lens: torch.Tensor, + ): + num_token, num_seq = num_kv_splits.shape[0], seq_lens.shape[0] + num_group = num_token // num_seq + + assert ( + num_group * num_seq == num_token + ), f"num_seq({num_seq}), num_token({num_token}), something goes wrong!" + + if self.static_kv_splits or self.device_core_count <= 0: + num_kv_splits.fill_(self.max_kv_splits) + return + + if num_seq < 256: + SCHEDULE_SEQ = 256 + else: + SCHEDULE_SEQ = triton.next_power_of_2(num_seq) + + get_num_kv_splits_triton[(1,)]( + num_kv_splits, + seq_lens, + num_seq, + num_group, + self.num_head, + self.num_kv_head, + self.max_kv_splits, + self.device_core_count, + MAX_NUM_SEQ=SCHEDULE_SEQ, + ) + + def init_forward_metadata(self, forward_batch: ForwardBatch): + """Init auxiliary variables for wave attention backend.""" + + bs = forward_batch.batch_size + kv_indptr = self.kv_indptr + spec_info = forward_batch.spec_info + + if forward_batch.forward_mode.is_decode_or_idle(): + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + forward_batch.seq_lens_sum, dtype=torch.int32, device=self.device + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + bs = kv_indptr.shape[0] - 1 + + from sglang.srt.layers.attention.wave_ops.decode_attention import ( + decode_attention_intermediate_arrays_shapes, + ) + + attn_logits_shape, attn_logits_max_shape = ( + decode_attention_intermediate_arrays_shapes( + bs, self.v_head_dim, self.num_head, self.max_kv_splits + ) + ) + attn_logits = torch.empty( + attn_logits_shape, + dtype=torch.float32, + device=self.device, + ) + attn_lse = torch.empty( + attn_logits_max_shape, + dtype=torch.float32, + device=self.device, + ) + num_kv_splits = torch.empty((bs,), dtype=torch.int32, device=self.device) + + self.get_num_kv_splits(num_kv_splits, forward_batch.seq_lens) + + qo_indptr = None + custom_mask = None + mask_indptr = None + max_extend_len = None + elif forward_batch.forward_mode.is_target_verify(): + bs = len(forward_batch.req_pool_indices) + qo_indptr = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + # Different with flashinfer kv_indptr and kv_indices construction + kv_indptr[1 : bs + 1] = torch.cumsum(forward_batch.seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + kv_indptr[-1], dtype=torch.int32, device=self.device + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + custom_mask = spec_info.custom_mask + seq_mask_len = self.num_draft_tokens * ( + forward_batch.seq_lens + self.num_draft_tokens + ) + mask_indptr = self.mask_indptr + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len[:bs], dim=0) + mask_indptr = mask_indptr[: bs + 1] + max_extend_len = self.num_draft_tokens + num_kv_splits = None + attn_logits = None + attn_lse = None + elif forward_batch.forward_mode.is_draft_extend(): + kv_indices, kv_indptr, qo_indptr, custom_mask = ( + spec_info.generate_attn_arg_prefill( + forward_batch.req_pool_indices, + forward_batch.seq_lens, + None, + self.req_to_token, + ) + ) + mask_indptr = None + # TODO(FIXME): This will trigger an invalid Eagle tree when using + # `max(spec_info.accept_length_cpu)`. + # It might have been forgotten to update somewhere. + max_extend_len = torch.max(spec_info.accept_length).item() + num_kv_splits = None + attn_logits = None + attn_lse = None + else: + kv_indptr[1 : bs + 1] = torch.cumsum( + forward_batch.extend_prefix_lens, dim=0 + ) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = torch.empty( + forward_batch.extend_prefix_lens.sum().item(), + dtype=torch.int32, + device=self.device, + ) + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + forward_batch.req_pool_indices, + forward_batch.extend_prefix_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + qo_indptr = self.qo_indptr + qo_indptr[1 : bs + 1] = torch.cumsum(forward_batch.extend_seq_lens, dim=0) + qo_indptr = qo_indptr[: bs + 1] + custom_mask = None + mask_indptr = None + attn_logits = None + attn_lse = None + max_extend_len = torch.max(forward_batch.extend_seq_lens).item() + num_kv_splits = None + + self.forward_metadata = ForwardMetadata( + attn_logits, + attn_lse, + max_extend_len, + num_kv_splits, + kv_indptr, + kv_indices, + qo_indptr, + custom_mask, + mask_indptr, + ) + + def init_cuda_graph_state( + self, + max_bs: int, + max_num_tokens: int, + kv_indices_buf: Optional[torch.Tensor] = None, + ): + from sglang.srt.layers.attention.wave_ops.decode_attention import ( + decode_attention_intermediate_arrays_shapes, + ) + + attn_logits_shape, attn_logits_max_shape = ( + decode_attention_intermediate_arrays_shapes( + max_bs, self.v_head_dim, self.num_head, self.max_kv_splits + ) + ) + self.cuda_graph_attn_logits = torch.zeros( + attn_logits_shape, + dtype=torch.float32, + device=self.device, + ) + self.cuda_graph_attn_lse = torch.zeros( + attn_logits_max_shape, + dtype=torch.float32, + device=self.device, + ) + self.cuda_graph_num_kv_splits = torch.full( + (max_bs,), self.max_kv_splits, dtype=torch.int32, device=self.device + ) + if kv_indices_buf is None: + self.cuda_graph_kv_indices = torch.zeros( + (max_bs * self.max_context_len), + dtype=torch.int32, + device=self.device, + ) + else: + self.cuda_graph_kv_indices = kv_indices_buf + + if not self.skip_prefill: + self.cuda_graph_custom_mask = torch.zeros( + (max_bs * self.max_context_len), + dtype=torch.uint8, + device=self.device, + ) + + def init_forward_metadata_capture_cuda_graph( + self, + bs: int, + num_tokens: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + ): + assert encoder_lens is None, "Not supported" + + if forward_mode.is_decode_or_idle(): + if spec_info is None: + kv_indptr = self.kv_indptr + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indptr = kv_indptr[: bs + 1] + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + else: + kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices + + attn_logits = self.cuda_graph_attn_logits + attn_lse = self.cuda_graph_attn_lse + max_extend_len = None + num_kv_splits = self.cuda_graph_num_kv_splits + qo_indptr = None + custom_mask = None + mask_indptr = None + elif forward_mode.is_target_verify(): + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + + custom_mask = self.cuda_graph_custom_mask + seq_mask_len = self.num_draft_tokens * (seq_lens + self.num_draft_tokens) + mask_indptr = self.mask_indptr[: bs + 1] + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len, dim=0) + max_extend_len = self.num_draft_tokens + num_kv_splits = None + attn_logits = None + attn_lse = None + else: + raise ValueError( + f"Invalid forward mode: {forward_mode=} for CUDA Graph capture." + ) + + self.forward_metadata = ForwardMetadata( + attn_logits, + attn_lse, + max_extend_len, + num_kv_splits, + kv_indptr, + kv_indices, + qo_indptr, + custom_mask, + mask_indptr, + ) + + def init_forward_metadata_replay_cuda_graph( + self, + bs: int, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + seq_lens_sum: int, + encoder_lens: Optional[torch.Tensor], + forward_mode: ForwardMode, + spec_info: Optional[Union[EagleDraftInput, EagleVerifyInput]], + seq_lens_cpu: Optional[torch.Tensor], + ): + # NOTE: encoder_lens expected to be zeros or None + if forward_mode.is_decode_or_idle(): + # Update kv_indptr, kv_indices + kv_indptr = self.kv_indptr + kv_indices = self.cuda_graph_kv_indices + num_kv_splits = self.cuda_graph_num_kv_splits + if spec_info is None: + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens[:bs], dim=0) + kv_indptr = kv_indptr[: bs + 1] + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices[:bs], + seq_lens[:bs], + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + num_token = bs + else: + kv_indptr[: spec_info.kv_indptr.shape[0]] = spec_info.kv_indptr + kv_indices[: spec_info.kv_indices.shape[0]] = spec_info.kv_indices + num_token = spec_info.kv_indptr.shape[0] - 1 + self.get_num_kv_splits(num_kv_splits[:num_token], seq_lens[:bs]) + elif forward_mode.is_target_verify(): + # Update qo_indptr, kv_indptr, kv_indices, custom_mask, mask_indptr + bs = len(req_pool_indices) + qo_indptr = self.qo_indptr[: bs + 1] + qo_indptr[: bs + 1] = torch.arange( + 0, + (1 + bs) * self.num_draft_tokens, + step=self.num_draft_tokens, + dtype=torch.int32, + device=self.device, + ) + kv_indptr = self.kv_indptr[: bs + 1] + kv_indptr[1 : bs + 1] = torch.cumsum(seq_lens, dim=0) + kv_indices = self.cuda_graph_kv_indices + create_flashinfer_kv_indices_triton[(bs,)]( + self.req_to_token, + req_pool_indices, + seq_lens, + kv_indptr, + None, + kv_indices, + self.req_to_token.stride(0), + ) + custom_mask = self.cuda_graph_custom_mask + custom_mask[: spec_info.custom_mask.shape[0]] = spec_info.custom_mask + seq_mask_len = self.num_draft_tokens * (seq_lens + self.num_draft_tokens) + mask_indptr = self.mask_indptr[: bs + 1] + mask_indptr[1 : bs + 1] = torch.cumsum(seq_mask_len, dim=0) + else: + raise ValueError( + f"Invalid forward mode: {forward_mode=} for CUDA Graph replay." + ) + + def get_cuda_graph_seq_len_fill_value(self): + return 1 + + def forward_extend( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + max_extend_len = self.forward_metadata.max_extend_len + computed_max_ext_seq_len = torch.max(forward_batch.extend_seq_lens) + if computed_max_ext_seq_len != max_extend_len: + assert len(forward_batch.extend_seq_lens) == 1 + forward_batch.extend_seq_lens[0] = max_extend_len + forward_batch.seq_lens = max_extend_len + + self.extend_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + k.contiguous(), + v.contiguous(), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + self.forward_metadata.qo_indptr, + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.forward_metadata.custom_mask, + self.forward_metadata.mask_indptr, + self.forward_metadata.max_extend_len, + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + is_causal=True, + layer_scaling=layer.scaling, + logit_cap=layer.logit_cap, + ) + return o + + def forward_decode( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer: RadixAttention, + forward_batch: ForwardBatch, + save_kv_cache=True, + ): + # During torch.compile, there is a bug in rotary_emb that causes the + # output value to have a 3D tensor shape. This reshapes the output correctly. + q = q.reshape(-1, layer.tp_q_head_num * layer.qk_head_dim) + + # TODO: reuse the buffer across layers + if layer.qk_head_dim != layer.v_head_dim: + o = q.new_empty((q.shape[0], layer.tp_q_head_num * layer.v_head_dim)) + else: + o = torch.empty_like(q) + + if save_kv_cache: + forward_batch.token_to_kv_pool.set_kv_buffer( + layer, forward_batch.out_cache_loc, k, v + ) + + self.decode_attention_fwd( + q.view(-1, layer.tp_q_head_num, layer.qk_head_dim), + forward_batch.token_to_kv_pool.get_key_buffer(layer.layer_id), + forward_batch.token_to_kv_pool.get_value_buffer(layer.layer_id), + o.view(-1, layer.tp_q_head_num, layer.v_head_dim), + self.forward_metadata.kv_indptr, + self.forward_metadata.kv_indices, + self.forward_metadata.attn_logits, + self.forward_metadata.attn_lse, + self.forward_metadata.num_kv_splits, + self.max_kv_splits, + layer.scaling, + layer.logit_cap, + ) + return o diff --git a/python/sglang/srt/layers/attention/wave_ops/decode_attention.py b/python/sglang/srt/layers/attention/wave_ops/decode_attention.py new file mode 100644 index 000000000..c76bee9af --- /dev/null +++ b/python/sglang/srt/layers/attention/wave_ops/decode_attention.py @@ -0,0 +1,184 @@ +""" +Memory-efficient attention for decoding. +It supports page size = 1. +""" + +import functools +import logging + +from wave_lang.kernel.lang.global_symbols import * +from wave_lang.kernel.wave.compile import WaveCompileOptions, wave_compile +from wave_lang.kernel.wave.constraints import GenericDot, MMAOperand, MMAType +from wave_lang.kernel.wave.templates.paged_decode_attention import ( + get_paged_decode_attention_kernels, + get_paged_decode_intermediate_arrays_shapes, + paged_decode_attention_shape, +) +from wave_lang.kernel.wave.utils.general_utils import get_default_scheduling_params +from wave_lang.kernel.wave.utils.run_utils import set_default_run_config + +logger = logging.getLogger(__name__) +import os + +dump_generated_mlir = int(os.environ.get("WAVE_DUMP_MLIR", 0)) + + +@functools.lru_cache(maxsize=4096) +def get_wave_kernel( + shape: paged_decode_attention_shape, + max_kv_splits, + input_dtype, + output_dtype, + logit_cap, +): + mha = (shape.num_query_heads // shape.num_kv_heads) == 1 + + # Get the kernels (either compile or load from cache). + if mha: + mfma_variant = ( + GenericDot(along_dim=MMAOperand.M, k_vec_size=4, k_mult=1), + GenericDot(along_dim=MMAOperand.M, k_vec_size=1, k_mult=64), + ) + else: + mfma_variant = (MMAType.F32_16x16x16_F16, MMAType.F32_16x16x16_F16) + + ( + phase_0, + phase_1, + hyperparams_0, + hyperparams_1, + dynamic_symbols_0, + dynamic_symbols_1, + ) = get_paged_decode_attention_kernels( + shape, + mfma_variant, + max_kv_splits, + input_dtype=input_dtype, + output_dtype=output_dtype, + logit_cap=logit_cap, + ) + hyperparams_0.update(get_default_scheduling_params()) + hyperparams_1.update(get_default_scheduling_params()) + + options = WaveCompileOptions( + subs=hyperparams_0, + canonicalize=True, + run_bench=False, + use_buffer_ops=True, + waves_per_eu=2, + dynamic_symbols=dynamic_symbols_0, + wave_runtime=True, + ) + options = set_default_run_config(options) + phase_0 = wave_compile(options, phase_0) + + options = WaveCompileOptions( + subs=hyperparams_1, + canonicalize=True, + run_bench=False, + use_buffer_ops=False, + waves_per_eu=4, + dynamic_symbols=dynamic_symbols_1, + wave_runtime=True, + ) + options = set_default_run_config(options) + phase_1 = wave_compile(options, phase_1) + + return phase_0, phase_1 + + +def decode_attention_intermediate_arrays_shapes( + num_seqs, head_size_kv, num_query_heads, max_kv_splits +): + # Not all fields are used, but we need to pass them to the function + shape = paged_decode_attention_shape( + num_query_heads=num_query_heads, + num_kv_heads=0, + head_size=0, + head_size_kv=head_size_kv, + block_size=0, + num_seqs=num_seqs, + ) + return get_paged_decode_intermediate_arrays_shapes(shape, max_kv_splits) + + +def decode_attention_wave( + q, + k_buffer, + v_buffer, + o, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, +): + num_seqs, num_query_heads, head_size = q.shape + _, num_kv_heads, _ = k_buffer.shape + _, _, head_size_kv = v_buffer.shape + block_size = 32 + shape = paged_decode_attention_shape( + num_query_heads, + num_kv_heads, + head_size, + head_size_kv, + block_size, + num_seqs, + ) + + phase_0, phase_1 = get_wave_kernel( + shape, max_kv_splits, q.dtype, o.dtype, logit_cap + ) + + mb_qk = phase_0( + q, + k_buffer, + v_buffer, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + ) + if dump_generated_mlir: + filename = f"wave_decode_attention_phase0_{'x'.join(map(str, shape))}.mlir" + with open(filename, "w") as f: + f.write(mb_qk.module_op.get_asm()) + + mb_sv = phase_1(attn_logits, attn_logits_max, b_req_idx, o) + if dump_generated_mlir: + filename = f"wave_decode_attention_phase1_{'x'.join(map(str, shape))}.mlir" + with open(filename, "w") as f: + f.write(mb_sv.module_op.get_asm()) + + +def decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap=0.0, +): + decode_attention_wave( + q, + k_buffer, + v_buffer, + o, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + ) diff --git a/python/sglang/srt/layers/attention/wave_ops/extend_attention.py b/python/sglang/srt/layers/attention/wave_ops/extend_attention.py new file mode 100644 index 000000000..27e674db2 --- /dev/null +++ b/python/sglang/srt/layers/attention/wave_ops/extend_attention.py @@ -0,0 +1,147 @@ +""" +Memory-efficient attention for prefill. +It support page size = 1. +""" + +import functools +import os + +import torch +from wave_lang.kernel.lang.global_symbols import * +from wave_lang.kernel.wave.compile import WaveCompileOptions, wave_compile +from wave_lang.kernel.wave.constraints import MMAType +from wave_lang.kernel.wave.scheduling.schedule import SchedulingType +from wave_lang.kernel.wave.templates.attention_common import AttentionShape +from wave_lang.kernel.wave.templates.extend_attention import get_extend_attention_kernel +from wave_lang.kernel.wave.utils.general_utils import get_default_scheduling_params +from wave_lang.kernel.wave.utils.run_utils import set_default_run_config + +dump_generated_mlir = int(os.environ.get("WAVE_DUMP_MLIR", 0)) + + +@functools.lru_cache +def get_wave_kernel( + shape: AttentionShape, + q_shape: tuple[int], + k_shape: tuple[int], + v_shape: tuple[int], + k_cache_shape: tuple[int], + v_cache_shape: tuple[int], + o_shape: tuple[int], + input_dtype: torch.dtype, + output_dtype: torch.dtype, + size_dtype: torch.dtype, + is_causal: bool, + logit_cap: float, + layer_scaling: float, +): + assert shape.num_query_heads % shape.num_kv_heads == 0 + + mfma_variant = (MMAType.F32_16x16x32_K8_F16, MMAType.F32_16x16x16_F16) + ( + extend_attention, + hyperparams, + dynamic_symbols, + ) = get_extend_attention_kernel( + shape, + mfma_variant, + q_shape, + k_shape, + v_shape, + k_cache_shape, + v_cache_shape, + o_shape, + input_dtype=input_dtype, + output_dtype=output_dtype, + size_dtype=size_dtype, + is_causal=is_causal, + layer_scaling=layer_scaling, + logit_cap=logit_cap, + ) + + hyperparams.update(get_default_scheduling_params()) + options = WaveCompileOptions( + subs=hyperparams, + canonicalize=True, + run_bench=False, + schedule=SchedulingType.NONE, + use_scheduling_barriers=False, + dynamic_symbols=dynamic_symbols, + use_buffer_ops=True, + waves_per_eu=2, + denorm_fp_math_f32="preserve-sign", + wave_runtime=True, + ) + options = set_default_run_config(options) + extend_attention = wave_compile(options, extend_attention) + + return extend_attention + + +def extend_attention_wave( + q_extend, + k_extend, + v_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + max_seq_len, + output, + is_causal=True, + layer_scaling=None, + logit_cap=0, +): + shape = AttentionShape( + num_query_heads=q_extend.shape[1], + num_kv_heads=k_extend.shape[1], + head_size=q_extend.shape[2], + head_size_kv=k_extend.shape[2], + num_seqs=kv_indptr.shape[0] - 1, + max_seq_len=max_seq_len, + ) + + # Run the wave kernel. + extend_attention = get_wave_kernel( + shape, + q_extend.shape, + k_extend.shape, + v_extend.shape, + k_buffer.shape, + v_buffer.shape, + output.shape, + input_dtype=q_extend.dtype, + output_dtype=output.dtype, + size_dtype=qo_indptr.dtype, + is_causal=is_causal, + layer_scaling=layer_scaling, + logit_cap=logit_cap, + ) + + mb = extend_attention( + q_extend, + k_extend, + v_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + max_seq_len, + output, + ) + + if dump_generated_mlir: + shape_list = [ + q_extend.shape[0], + q_extend.shape[1], + k_extend.shape[1], + q_extend.shape[2], + k_extend.shape[2], + ] + filename = f"wave_prefill_attention_{'x'.join(map(str, shape_list))}.mlir" + with open(filename, "w") as f: + f.write(mb.module_op.get_asm()) diff --git a/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py b/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py new file mode 100644 index 000000000..2d8aa4678 --- /dev/null +++ b/python/sglang/srt/layers/attention/wave_ops/prefill_attention.py @@ -0,0 +1,79 @@ +""" +Memory-efficient attention for prefill. +It support page size = 1. +""" + +import math +import os + +from wave_lang.kernel.lang.global_symbols import * +from wave_lang.kernel.wave.compile import WaveCompileOptions, wave_compile +from wave_lang.kernel.wave.constraints import MMAType +from wave_lang.kernel.wave.templates.attention_common import AttentionShape +from wave_lang.kernel.wave.templates.prefill_attention import ( + get_prefill_attention_kernel, +) +from wave_lang.kernel.wave.utils.general_utils import get_default_scheduling_params +from wave_lang.kernel.wave.utils.run_utils import set_default_run_config + +dump_generated_mlir = int(os.environ.get("WAVE_DUMP_MLIR", 0)) + + +def prefill_attention_wave( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=True +): + + shape = AttentionShape( + num_query_heads=q.shape[1], + num_kv_heads=k.shape[1], + head_size=q.shape[2], + head_size_kv=k.shape[2], + num_seqs=b_seq_len.shape[0], + max_seq_len=max_seq_len, + total_seq_len=q.shape[0], + ) + + assert shape.num_query_heads % shape.num_kv_heads == 0 + + output_shape = (shape.total_seq_len, shape.num_query_heads, shape.head_size_kv) + # Run the wave kernel. + mfma_variant = (MMAType.F32_16x16x16_F16, MMAType.F32_16x16x16_F16) + (prefill, hyperparams) = get_prefill_attention_kernel( + shape, + mfma_variant, + q.shape, + k.shape, + v.shape, + output_shape, + input_dtype=q.dtype, + output_dtype=o.dtype, + size_dtype=b_seq_len.dtype, + ) + + hyperparams.update(get_default_scheduling_params()) + + log2e = 1.44269504089 + dk_sqrt = math.sqrt(1.0 / shape.head_size) + + options = WaveCompileOptions( + subs=hyperparams, + canonicalize=True, + run_bench=False, + use_scheduling_barriers=False, + ) + options = set_default_run_config(options) + prefill = wave_compile(options, prefill) + + mb = prefill( + q * dk_sqrt * log2e, + k, + v, + b_start_loc, + b_seq_len, + o, + ) + if dump_generated_mlir: + shape_list = [q.shape[0], q.shape[1], k.shape[1], q.shape[2], k.shape[2]] + filename = f"wave_prefill_attention_{'x'.join(map(str, shape_list))}.mlir" + with open(filename, "w") as f: + f.write(mb.module_op.get_asm()) diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py new file mode 100644 index 000000000..fba8d8f18 --- /dev/null +++ b/python/sglang/srt/layers/communicator.py @@ -0,0 +1,676 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from dataclasses import dataclass +from enum import Enum, auto +from functools import partial +from typing import Dict, Optional + +import torch + +from sglang.srt.distributed import ( + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_reduce, +) +from sglang.srt.layers.dp_attention import ( + attn_tp_all_gather_into_tensor, + attn_tp_reduce_scatter_tensor, + dp_gather_partial, + dp_reduce_scatter_tensor, + dp_scatter, + get_attention_dp_size, + get_attention_tp_rank, + get_attention_tp_size, + get_global_dp_buffer, + get_local_dp_buffer, + is_dp_attention_enabled, +) +from sglang.srt.layers.moe import ( + get_moe_a2a_backend, + should_use_flashinfer_cutlass_moe_fp4_allgather, +) +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.utils import ( + get_bool_env_var, + is_cuda, + is_flashinfer_available, + is_gfx95_supported, + is_hip, + is_sm90_supported, + is_sm100_supported, +) + +_is_flashinfer_available = is_flashinfer_available() +_is_sm90_supported = is_cuda() and is_sm90_supported() +_is_sm100_supported = is_cuda() and is_sm100_supported() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() +_is_gfx95_supported = is_gfx95_supported() + +if _use_aiter and _is_gfx95_supported: + from sglang.srt.layers.quantization.rocm_mxfp4_utils import fused_rms_mxfp4_quant + +FUSE_ALLREDUCE_MAX_BATCH_SIZE = 2048 + + +class ScatterMode(Enum): + """ + Suppose we have TP=4, DP=2, enable-dp-attention, and the system handles seq a,b,c,d + Model input/output: [ab, ab, cd, cd] for four ranks respectively + SCATTERED: [a, b, c, d] + TP_ATTN_FULL: [ab, ab, cd, cd], i.e. all ranks inside a TP attn group have full data of the group + FULL: [abcd, abcd, abcd, abcd] + """ + + SCATTERED = auto() + TP_ATTN_FULL = auto() + FULL = auto() + + @staticmethod + def model_input_output(): + """The scatter mode for model forward pass input and output data""" + return ScatterMode.TP_ATTN_FULL + + +@dataclass +class _LayerModeComputationContext: + num_layers: int + layer_id: int + is_layer_sparse: bool + is_previous_layer_sparse: Optional[bool] + + def previous_layer(self): + assert self.is_previous_layer_sparse is not None + return _LayerModeComputationContext( + layer_id=self.layer_id - 1, + is_layer_sparse=self.is_previous_layer_sparse, + is_previous_layer_sparse=None, + num_layers=self.num_layers, + ) + + +@dataclass +class LayerScatterModes: + layer_input_mode: ScatterMode + attn_mode: ScatterMode + # Can be further split into e.g. mlp_input_mode and mlp_output_mode if needed + mlp_mode: ScatterMode + middle_residual_mode: ScatterMode + layer_output_mode: ScatterMode + + @classmethod + def init_new(cls, **kwargs): + context = _LayerModeComputationContext(**kwargs) + return cls( + layer_input_mode=cls._compute_layer_input_mode(context), + attn_mode=ScatterMode.TP_ATTN_FULL, + mlp_mode=cls._compute_mlp_mode(context), + middle_residual_mode=cls._compute_middle_residual_mode(context), + layer_output_mode=cls._compute_layer_output_mode(context), + ) + + @classmethod + def _compute_layer_input_mode(cls, context: _LayerModeComputationContext): + if context.layer_id == 0: + return ScatterMode.model_input_output() + return cls._compute_layer_output_mode(context.previous_layer()) + + @classmethod + def _compute_mlp_mode(cls, context: _LayerModeComputationContext): + if context.is_layer_sparse: + return ( + ScatterMode.SCATTERED + if ( + # Token dispatch/combine will be handled outside of LayerCommunicator for these modes. + not get_moe_a2a_backend().is_none() + or should_use_flashinfer_cutlass_moe_fp4_allgather() + ) + else ScatterMode.FULL + ) + else: + return ( + ScatterMode.SCATTERED + if enable_moe_dense_fully_dp() + else ScatterMode.FULL + ) + + @classmethod + def _compute_middle_residual_mode(cls, context: _LayerModeComputationContext): + mlp_mode = cls._compute_mlp_mode(context) + if mlp_mode == ScatterMode.SCATTERED: + return ScatterMode.SCATTERED + if mlp_mode == ScatterMode.FULL: + return ScatterMode.TP_ATTN_FULL + raise NotImplementedError + + @classmethod + def _compute_layer_output_mode(cls, context: _LayerModeComputationContext): + mlp_mode = cls._compute_mlp_mode(context) + if context.layer_id == context.num_layers - 1: + return ScatterMode.model_input_output() + if mlp_mode == ScatterMode.SCATTERED: + return ScatterMode.SCATTERED + if mlp_mode == ScatterMode.FULL: + return ScatterMode.TP_ATTN_FULL + raise NotImplementedError + + +def enable_moe_dense_fully_dp(): + return global_server_args_dict["moe_dense_tp_size"] == 1 + + +class LayerCommunicator: + def __init__( + self, + layer_scatter_modes: LayerScatterModes, + input_layernorm: torch.nn.Module, + post_attention_layernorm: torch.nn.Module, + # Reduce scatter requires skipping all-reduce in model code after MoE/MLP, so only enable for models which have that implemented. Remove flag once done for all models that use LayerCommunicator. + allow_reduce_scatter: bool = False, + is_last_layer: bool = False, + ): + self.layer_scatter_modes = layer_scatter_modes + self.input_layernorm = input_layernorm + self.post_attention_layernorm = post_attention_layernorm + self.allow_reduce_scatter = allow_reduce_scatter + self.is_last_layer = is_last_layer + + self._context = CommunicateContext.init_new() + self._communicate_simple_fn = CommunicateSimpleFn.get_fn( + input_mode=self.layer_scatter_modes.layer_input_mode, + output_mode=self.layer_scatter_modes.attn_mode, + context=self._context, + ) + self._communicate_with_all_reduce_and_layer_norm_fn = ( + CommunicateWithAllReduceAndLayerNormFn.get_fn( + hidden_states_input_mode=self.layer_scatter_modes.attn_mode, + residual_input_mode=self.layer_scatter_modes.layer_input_mode, + hidden_states_output_mode=self.layer_scatter_modes.mlp_mode, + residual_output_mode=self.layer_scatter_modes.middle_residual_mode, + context=self._context, + ) + ) + self._communicate_summable_tensor_pair_fn = ( + CommunicateSummableTensorPairFn.get_fn( + hidden_states_input_mode=self.layer_scatter_modes.mlp_mode, + residual_input_mode=self.layer_scatter_modes.middle_residual_mode, + output_mode=self.layer_scatter_modes.layer_output_mode, + context=self._context, + ) + ) + + def prepare_attn( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + qaunt_format: str = "", + ): + if hidden_states.shape[0] == 0: + residual = hidden_states + else: + if ( + residual is not None + and hasattr(hidden_states, "_sglang_needs_allreduce_fusion") + and hidden_states._sglang_needs_allreduce_fusion + ): + hidden_states, residual = ( + self.input_layernorm.forward_with_allreduce_fusion( + hidden_states, residual + ) + ) + else: + if residual is None: + residual = hidden_states + + if _use_aiter and _is_gfx95_supported and ("mxfp4" in qaunt_format): + hidden_states = fused_rms_mxfp4_quant( + hidden_states, + self.input_layernorm.weight, + self.input_layernorm.variance_epsilon, + None, + None, + None, + None, + ) + else: + hidden_states = self.input_layernorm(hidden_states) + else: + if _use_aiter and _is_gfx95_supported and ("mxfp4" in qaunt_format): + hidden_states, residual = fused_rms_mxfp4_quant( + hidden_states, + self.input_layernorm.weight, + self.input_layernorm.variance_epsilon, + None, + None, + None, + residual, + ) + else: + hidden_states, residual = self.input_layernorm( + hidden_states, residual + ) + + hidden_states = self._communicate_simple_fn( + hidden_states=hidden_states, + forward_batch=forward_batch, + context=self._context, + ) + + return hidden_states, residual + + def prepare_mlp( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ): + return self._communicate_with_all_reduce_and_layer_norm_fn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + layernorm=self.post_attention_layernorm, + context=self._context, + ) + + def postprocess_layer( + self, + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + ): + return self._communicate_summable_tensor_pair_fn( + hidden_states=hidden_states, + residual=residual, + forward_batch=forward_batch, + context=self._context, + allow_reduce_scatter=self.allow_reduce_scatter, + ) + + def should_use_reduce_scatter(self, forward_batch: ForwardBatch): + return ( + self.allow_reduce_scatter + and self._communicate_summable_tensor_pair_fn + is CommunicateSummableTensorPairFn._scatter_hidden_states + and forward_batch.dp_padding_mode.is_max_len() + ) + + def should_fuse_mlp_allreduce_with_next_layer( + self, forward_batch: ForwardBatch + ) -> bool: + speculative_algo = global_server_args_dict.get("speculative_algorithm", None) + if ( + is_dp_attention_enabled() + and speculative_algo is not None + and speculative_algo.is_eagle() + ): + return False + + batch_size = ( + forward_batch.input_ids.shape[0] + if hasattr(forward_batch, "input_ids") + else 0 + ) + if batch_size > FUSE_ALLREDUCE_MAX_BATCH_SIZE: + return False + + static_conditions_met = ( + (not self.is_last_layer) + and (self._context.tp_size > 1) + and global_server_args_dict.get("enable_flashinfer_allreduce_fusion", False) + and _is_flashinfer_available + ) + + if not static_conditions_met: + return False + + return ( + batch_size > 0 + and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE + and (not self.is_last_layer) + ) + + +@dataclass +class CommunicateContext: + process_group_sizes: Dict[ScatterMode, int] + attn_tp_rank: int + attn_tp_size: int + attn_dp_size: int + tp_size: int + + def is_same_group_size(self, a: ScatterMode, b: ScatterMode): + return self.process_group_sizes[a] == self.process_group_sizes[b] + + @classmethod + def init_new(cls): + attn_tp_rank = get_attention_tp_rank() + attn_tp_size = get_attention_tp_size() + attn_dp_size = get_attention_dp_size() + tp_size = get_tensor_model_parallel_world_size() + process_group_sizes = { + ScatterMode.SCATTERED: 1, + ScatterMode.TP_ATTN_FULL: attn_tp_size, + # TODO: support --moe-dense-tp-size > 1 + ScatterMode.FULL: tp_size, + } + return cls( + process_group_sizes=process_group_sizes, + attn_tp_rank=attn_tp_rank, + attn_tp_size=attn_tp_size, + attn_dp_size=attn_dp_size, + tp_size=tp_size, + ) + + +class CommunicateSimpleFn: + @staticmethod + def get_fn( + input_mode: ScatterMode, + output_mode: ScatterMode, + context: CommunicateContext, + ): + if context.is_same_group_size(input_mode, output_mode): + return CommunicateSimpleFn._trivial + + if (input_mode == ScatterMode.SCATTERED) and ( + output_mode == ScatterMode.TP_ATTN_FULL + ): + return CommunicateSimpleFn._scattered_to_tp_attn_full + + raise NotImplementedError(f"{input_mode=} {output_mode=}") + + @staticmethod + def _trivial( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + ) -> torch.Tensor: + return hidden_states + + @staticmethod + def _scattered_to_tp_attn_full( + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + ) -> torch.Tensor: + hidden_states, local_hidden_states = ( + get_local_dp_buffer(), + hidden_states, + ) + attn_tp_all_gather_into_tensor( + hidden_states, + local_hidden_states, + ) + return hidden_states + + +class CommunicateWithAllReduceAndLayerNormFn: + """Besides communication, needs to + 1. All reduce in tp_attn_group on hidden_states + 2. Apply layer norm + """ + + @staticmethod + def get_fn( + hidden_states_input_mode: ScatterMode, + residual_input_mode: ScatterMode, + hidden_states_output_mode: ScatterMode, + residual_output_mode: ScatterMode, + context: CommunicateContext, + ): + + if ( + context.is_same_group_size( + hidden_states_input_mode, hidden_states_output_mode + ) + and context.is_same_group_size(residual_input_mode, residual_output_mode) + and context.attn_tp_size == 1 + ): + return CommunicateWithAllReduceAndLayerNormFn._simple + + if ( + (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) + and ( + residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] + ) + and (hidden_states_output_mode == ScatterMode.FULL) + and (residual_output_mode == ScatterMode.TP_ATTN_FULL) + ): + return partial( + CommunicateWithAllReduceAndLayerNormFn._gather_hidden_states_and_residual, + residual_input_mode=residual_input_mode, + ) + + if ( + (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) + and ( + residual_input_mode in [ScatterMode.SCATTERED, ScatterMode.TP_ATTN_FULL] + ) + and (hidden_states_output_mode == ScatterMode.SCATTERED) + and (residual_output_mode == ScatterMode.SCATTERED) + ): + return partial( + CommunicateWithAllReduceAndLayerNormFn._scatter_hidden_states_and_residual, + residual_input_mode=residual_input_mode, + ) + + raise NotImplementedError( + f"{hidden_states_input_mode=} {residual_input_mode=} {hidden_states_output_mode=} {residual_output_mode=}" + ) + + @staticmethod + def _simple( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + ): + # TODO move these `if shape != 0` into LayerNorm itself + if hidden_states.shape[0] != 0: + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual + + @staticmethod + def _gather_hidden_states_and_residual( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + residual_input_mode, + ): + if residual_input_mode == ScatterMode.SCATTERED and context.attn_tp_size > 1: + residual, local_residual = ( + get_local_dp_buffer(), + residual, + ) + attn_tp_all_gather_into_tensor(residual, local_residual) + if context.attn_dp_size != 1: + if context.attn_tp_rank == 0: + hidden_states += residual + + # Perform layernorm on smaller data before comm. Only valid when attn_tp_size is 1 (tp_size == dp_size) + use_layer_norm_before_gather = context.attn_tp_size == 1 + if use_layer_norm_before_gather and hidden_states.shape[0] != 0: + residual = hidden_states + hidden_states = layernorm(hidden_states) + hidden_states, local_hidden_states = ( + get_global_dp_buffer(), + hidden_states, + ) + dp_gather_partial(hidden_states, local_hidden_states, forward_batch) + + if not use_layer_norm_before_gather: + dp_scatter(residual, hidden_states, forward_batch) + if hidden_states.shape[0] != 0: + hidden_states = layernorm(hidden_states) + else: + # According to the discussion in https://github.com/flashinfer-ai/flashinfer/issues/1223#issuecomment-3047256465 + # We set the max token num to 128 for allreduce fusion with min-latency case(use_oneshot=True). + if ( + (_is_sm100_supported or _is_sm90_supported) + and _is_flashinfer_available + and hasattr(layernorm, "forward_with_allreduce_fusion") + and global_server_args_dict["enable_flashinfer_allreduce_fusion"] + and hidden_states.shape[0] <= 4096 + ): + hidden_states, residual = layernorm.forward_with_allreduce_fusion( + hidden_states, residual + ) + else: + hidden_states = tensor_model_parallel_all_reduce(hidden_states) + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual + + @staticmethod + def _scatter_hidden_states_and_residual( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + layernorm: torch.nn.Module, + context: CommunicateContext, + *, + residual_input_mode, + ): + input_hidden_states = hidden_states + hidden_states = hidden_states.tensor_split(context.attn_tp_size)[ + context.attn_tp_rank + ] + attn_tp_reduce_scatter_tensor(hidden_states, input_hidden_states) + if residual_input_mode == ScatterMode.TP_ATTN_FULL: + residual = residual.tensor_split(context.attn_tp_size)[context.attn_tp_rank] + if hidden_states.shape[0] != 0: + hidden_states, residual = layernorm(hidden_states, residual) + return hidden_states, residual + + +class CommunicateSummableTensorPairFn: + """It is allowed to make (hidden_states, residual) := (hidden_states + residual, None) if needed.""" + + @classmethod + def execute( + cls, + hidden_states_input_mode, + residual_input_mode, + output_mode, + context, + **kwargs, + ): + return cls.get_fn( + hidden_states_input_mode=hidden_states_input_mode, + residual_input_mode=residual_input_mode, + output_mode=output_mode, + context=context, + )(context=context, **kwargs) + + @staticmethod + def get_fn( + hidden_states_input_mode: ScatterMode, + residual_input_mode: ScatterMode, + output_mode: ScatterMode, + context: CommunicateContext, + ): + if context.is_same_group_size( + hidden_states_input_mode, output_mode + ) and context.is_same_group_size(residual_input_mode, output_mode): + return CommunicateSummableTensorPairFn._trivial + + if ( + (hidden_states_input_mode == ScatterMode.FULL) + and (residual_input_mode == ScatterMode.TP_ATTN_FULL) + and (output_mode == ScatterMode.TP_ATTN_FULL) + ): + return CommunicateSummableTensorPairFn._scatter_hidden_states + + if ( + (hidden_states_input_mode == ScatterMode.SCATTERED) + and (residual_input_mode == ScatterMode.SCATTERED) + and (output_mode == ScatterMode.TP_ATTN_FULL) + ): + return CommunicateSummableTensorPairFn._gather + + if ( + (hidden_states_input_mode == ScatterMode.TP_ATTN_FULL) + and (residual_input_mode == ScatterMode.TP_ATTN_FULL) + and (output_mode == ScatterMode.SCATTERED) + ): + return CommunicateSummableTensorPairFn._scatter + + raise NotImplementedError( + f"{hidden_states_input_mode=} {residual_input_mode=} {output_mode=}" + ) + + @staticmethod + def _trivial( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + return hidden_states, residual + + @staticmethod + def _scatter_hidden_states( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + allow_reduce_scatter: bool = False, + ): + hidden_states, global_hidden_states = ( + get_local_dp_buffer(), + hidden_states, + ) + if allow_reduce_scatter and forward_batch.dp_padding_mode.is_max_len(): + # When using padding, all_reduce is skipped after MLP and MOE and reduce scatter is used here instead. + dp_reduce_scatter_tensor(hidden_states, global_hidden_states) + else: + dp_scatter(hidden_states, global_hidden_states, forward_batch) + return hidden_states, residual + + @staticmethod + def _gather( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + **kwargs, + ): + hidden_states += residual + residual = None + hidden_states, local_hidden_states = ( + get_local_dp_buffer(), + hidden_states, + ) + attn_tp_all_gather_into_tensor( + hidden_states, + local_hidden_states, + ) + return hidden_states, residual + + @staticmethod + def _scatter( + hidden_states: torch.Tensor, + residual: torch.Tensor, + forward_batch: ForwardBatch, + context: CommunicateContext, + ): + assert residual is None, "not yet handled residual!=None" + tensor_list = list(hidden_states.tensor_split(context.attn_tp_size)) + hidden_states = tensor_list[context.attn_tp_rank] + return hidden_states, residual diff --git a/python/sglang/srt/layers/dp_attention.py b/python/sglang/srt/layers/dp_attention.py new file mode 100644 index 000000000..1250636eb --- /dev/null +++ b/python/sglang/srt/layers/dp_attention.py @@ -0,0 +1,489 @@ +from __future__ import annotations + +import functools +import logging +from contextlib import contextmanager +from enum import IntEnum, auto +from typing import TYPE_CHECKING, List, Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.distributed import ( + GroupCoordinator, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + get_tp_group, + tensor_model_parallel_all_reduce, +) + +if TYPE_CHECKING: + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +_ATTN_TP_GROUP: Optional[GroupCoordinator] = None +_ATTN_TP_RANK: Optional[int] = None +_ATTN_TP_SIZE: Optional[int] = None +_ATTN_DP_RANK: Optional[int] = None +_ATTN_DP_SIZE: Optional[int] = None +_LOCAL_ATTN_DP_SIZE: Optional[int] = None +_LOCAL_ATTN_DP_RANK: Optional[int] = None +_ENABLE_DP_ATTENTION_FLAG: bool = False + + +class DpPaddingMode(IntEnum): + + # Padding tokens to max length and then gather tokens using `all_gather_into_tensor` + MAX_LEN = auto() + # Padding tokens to sum length and then gather tokens using `all_reduce` + SUM_LEN = auto() + + def is_max_len(self): + return self == DpPaddingMode.MAX_LEN + + def is_sum_len(self): + return self == DpPaddingMode.SUM_LEN + + @classmethod + def get_dp_padding_mode(cls, global_num_tokens: List[int]) -> DpPaddingMode: + # we choose the mode that minimizes the communication cost + max_len = max(global_num_tokens) + sum_len = sum(global_num_tokens) + if sum_len * 2 > max_len * get_attention_dp_size(): + return cls.MAX_LEN + else: + return cls.SUM_LEN + + @classmethod + def get_default_mode_in_cuda_graph(cls) -> DpPaddingMode: + return cls.MAX_LEN + + +class _DpGatheredBufferWrapper: + + _hidden_size: int + _dtype: torch.dtype + _device: torch.device + _global_dp_buffer_len: int + _local_dp_buffer_len: int + _global_num_tokens: Optional[List[int]] + + @classmethod + def set_metadata(cls, hidden_size: int, dtype: torch.dtype, device: torch.device): + cls._hidden_size = hidden_size + cls._dtype = dtype + cls._device = device + + @classmethod + def set_dp_buffer_len( + cls, + global_dp_buffer_len: int, + local_dp_buffer_len: int, + global_num_tokens: Optional[List[int]] = None, + ): + cls._global_dp_buffer_len = global_dp_buffer_len + cls._local_dp_buffer_len = local_dp_buffer_len + cls._global_num_tokens = global_num_tokens + + @classmethod + def get_global_dp_buffer(cls) -> torch.Tensor: + return torch.empty( + (cls._global_dp_buffer_len, cls._hidden_size), + dtype=cls._dtype, + device=cls._device, + ) + + @classmethod + def get_local_dp_buffer(cls) -> torch.Tensor: + return torch.empty( + (cls._local_dp_buffer_len, cls._hidden_size), + dtype=cls._dtype, + device=cls._device, + ) + + @classmethod + def get_global_dp_buffer_len(cls) -> int: + return cls._global_dp_buffer_len + + @classmethod + def get_local_dp_buffer_len(cls) -> int: + return cls._local_dp_buffer_len + + @classmethod + def get_dp_global_num_tokens(cls) -> List[int]: + return cls._global_num_tokens + + +def set_dp_buffer_len( + global_dp_buffer_len: int, + local_dp_buffer_len: int, + global_num_tokens: Optional[List[int]] = None, +): + _DpGatheredBufferWrapper.set_dp_buffer_len( + global_dp_buffer_len, local_dp_buffer_len, global_num_tokens + ) + + +def get_global_dp_buffer() -> torch.Tensor: + return _DpGatheredBufferWrapper.get_global_dp_buffer() + + +def get_local_dp_buffer() -> torch.Tensor: + return _DpGatheredBufferWrapper.get_local_dp_buffer() + + +def get_global_dp_buffer_len() -> int: + return _DpGatheredBufferWrapper.get_global_dp_buffer_len() + + +def get_local_dp_buffer_len() -> int: + return _DpGatheredBufferWrapper.get_local_dp_buffer_len() + + +def get_dp_global_num_tokens() -> List[int]: + return _DpGatheredBufferWrapper.get_dp_global_num_tokens() + + +def compute_dp_attention_world_info(enable_dp_attention, tp_rank, tp_size, dp_size): + if not enable_dp_attention: + return tp_rank, tp_size, 0 + + attn_tp_size = tp_size // dp_size + attn_dp_rank = tp_rank // attn_tp_size + attn_tp_rank = tp_rank % attn_tp_size + + return attn_tp_rank, attn_tp_size, attn_dp_rank + + +def compute_dp_attention_local_info( + enable_dp_attention, tp_rank, tp_size, dp_size, moe_dense_tp_size +): + if not enable_dp_attention: + return tp_rank, tp_size, 0 + + local_tp_size = moe_dense_tp_size if moe_dense_tp_size else tp_size + local_tp_rank = tp_rank % local_tp_size + local_dp_size = max(1, dp_size // (tp_size // local_tp_size)) + + local_attn_tp_size = local_tp_size // local_dp_size + local_attn_dp_rank = local_tp_rank // local_attn_tp_size + local_attn_tp_rank = local_tp_rank % local_attn_tp_size + + return local_attn_tp_rank, local_attn_tp_size, local_attn_dp_rank + + +def initialize_dp_attention( + server_args: ServerArgs, + model_config: ModelConfig, +): + global _ATTN_TP_GROUP, _ATTN_TP_RANK, _ATTN_TP_SIZE, _ATTN_DP_RANK, _ATTN_DP_SIZE + global _LOCAL_ATTN_DP_SIZE, _LOCAL_ATTN_DP_RANK, _ENABLE_DP_ATTENTION_FLAG + + from sglang.srt.layers.sampler import SYNC_TOKEN_IDS_ACROSS_TP + + enable_dp_attention = server_args.enable_dp_attention + tp_size = server_args.tp_size + dp_size = server_args.dp_size + moe_dense_tp_size = server_args.moe_dense_tp_size + pp_size = server_args.pp_size + + tp_rank = get_tensor_model_parallel_rank() + + _ENABLE_DP_ATTENTION_FLAG = enable_dp_attention + + _ATTN_TP_RANK, _ATTN_TP_SIZE, _ATTN_DP_RANK = compute_dp_attention_world_info( + enable_dp_attention, tp_rank, tp_size, dp_size + ) + _, _, _LOCAL_ATTN_DP_RANK = compute_dp_attention_local_info( + enable_dp_attention, tp_rank, tp_size, dp_size, moe_dense_tp_size + ) + + if enable_dp_attention: + _ATTN_DP_SIZE = dp_size + if moe_dense_tp_size is None: + _LOCAL_ATTN_DP_SIZE = _ATTN_DP_SIZE + else: + _LOCAL_ATTN_DP_SIZE = max(1, dp_size // (tp_size // moe_dense_tp_size)) + else: + _ATTN_DP_SIZE = 1 + _LOCAL_ATTN_DP_SIZE = 1 + + tp_group = get_tp_group() + _ATTN_TP_GROUP = GroupCoordinator( + [ + list(range(head, head + _ATTN_TP_SIZE)) + for head in range(0, pp_size * tp_size, _ATTN_TP_SIZE) + ], + tp_group.local_rank, + torch.distributed.get_backend(tp_group.device_group), + use_pynccl=SYNC_TOKEN_IDS_ACROSS_TP, + use_pymscclpp=False, + use_custom_allreduce=False, + use_hpu_communicator=False, + use_xpu_communicator=False, + use_npu_communicator=False, + group_name="attention_tp", + ) + + _DpGatheredBufferWrapper.set_metadata( + hidden_size=model_config.hidden_size, + dtype=model_config.dtype, + device=torch.device(server_args.device), + ) + + +def is_dp_attention_enabled() -> bool: + return _ENABLE_DP_ATTENTION_FLAG + + +def get_attention_tp_group() -> GroupCoordinator: + assert _ATTN_TP_GROUP is not None, "dp attention not initialized!" + return _ATTN_TP_GROUP + + +def get_attention_tp_rank() -> int: + assert _ATTN_TP_RANK is not None, "dp attention not initialized!" + return _ATTN_TP_RANK + + +def get_attention_tp_size() -> int: + assert _ATTN_TP_SIZE is not None, "dp attention not initialized!" + return _ATTN_TP_SIZE + + +def get_attention_dp_rank() -> int: + assert _ATTN_DP_RANK is not None, "dp attention not initialized!" + return _ATTN_DP_RANK + + +def get_attention_dp_size() -> int: + assert _ATTN_DP_SIZE is not None, "dp attention not initialized!" + return _ATTN_DP_SIZE + + +def get_local_attention_dp_rank() -> int: + assert _LOCAL_ATTN_DP_RANK is not None, "dp attention not initialized!" + return _LOCAL_ATTN_DP_RANK + + +def get_local_attention_dp_size() -> int: + assert _LOCAL_ATTN_DP_SIZE is not None, "dp attention not initialized!" + return _LOCAL_ATTN_DP_SIZE + + +@contextmanager +def disable_dp_size(): + """Patch the tp group temporarily until this function ends. + + This method is for draft workers of speculative decoding to run draft model + with different tp degree from that of target model workers. + + Args: + tp_group (GroupCoordinator): the tp group coordinator + """ + global _ATTN_DP_SIZE + assert _ATTN_DP_SIZE is not None, "dp attention not initialized!" + + old_dp_size = _ATTN_DP_SIZE + _ATTN_DP_SIZE = 1 + try: + yield + finally: + _ATTN_DP_SIZE = old_dp_size + + +def get_dp_local_info(forward_batch: ForwardBatch) -> Tuple[torch.Tensor, torch.Tensor]: + # `get_dp_local_info` is only called in global DP gather and scatter. We use global DP rank here. + dp_rank = get_attention_dp_rank() + + if forward_batch.dp_local_start_pos is None: + cumtokens = torch.cumsum(forward_batch.global_num_tokens_gpu, dim=0) + if dp_rank == 0: + local_start_pos = torch.zeros_like(cumtokens[0]) + else: + local_start_pos = cumtokens[dp_rank - 1] + local_num_tokens = forward_batch.global_num_tokens_gpu[dp_rank] + + forward_batch.dp_local_start_pos = local_start_pos + forward_batch.dp_local_num_tokens = local_num_tokens + + return forward_batch.dp_local_start_pos, forward_batch.dp_local_num_tokens + + +@triton.jit +def memcpy_triton_kernel( + dst_ptr, + src_ptr, + offset_ptr, + sz_ptr, + offset_src: tl.constexpr, + chunk_size, # multiplied for offset and sz + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0).to(tl.int64) + offset = tl.load(offset_ptr).to(tl.int64) * chunk_size + sz = tl.load(sz_ptr).to(tl.int64) * chunk_size + + start_index = pid * BLOCK_SIZE + offs = tl.arange(0, BLOCK_SIZE) + mask = start_index + offs < sz + + if offset_src: + data = tl.load(src_ptr + offset + start_index + offs, mask=mask) + tl.store(dst_ptr + start_index + offs, data, mask=mask) + else: + data = tl.load(src_ptr + start_index + offs, mask=mask) + tl.store(dst_ptr + offset + start_index + offs, data, mask=mask) + + +def prod(x): + return functools.reduce(lambda a, b: a * b, x, 1) + + +def memcpy_triton(dst, src, dim, offset, sz, offset_src): + max_size = min(src.numel(), dst.numel()) + assert dim == 0, "dim != 0 unsupported" + assert src.shape[1:] == dst.shape[1:], "src and dst must have same shape" + chunk_size = prod(src.shape[1:]) + BLOCK_SIZE = 8192 + grid = (triton.cdiv(max_size, BLOCK_SIZE),) + + memcpy_triton_kernel[grid](dst, src, offset, sz, offset_src, chunk_size, BLOCK_SIZE) + + +def _dp_gather_via_all_reduce( + global_tokens: torch.Tensor, + local_tokens: torch.Tensor, + forward_batch: ForwardBatch, + is_partial: bool, +): + local_start_pos, local_num_tokens = get_dp_local_info(forward_batch) + + global_tokens.fill_(0) + assert local_tokens.is_contiguous() + assert global_tokens.is_contiguous() + + if local_tokens.shape[0] > 0 and (is_partial or get_attention_tp_rank() == 0): + assert ( + local_tokens.untyped_storage() is not global_tokens.untyped_storage() + ), "aliasing between global_tokens and local_tokens not allowed" + + memcpy_triton( + global_tokens, local_tokens, 0, local_start_pos, local_num_tokens, False + ) + + # Input IDs are in int 32. We should use inplace_all_reduce for local case because of custom all reduce. + NUM_GPUS_PER_NODE = 8 + if ( + not local_tokens.dtype.is_floating_point + and get_tensor_model_parallel_world_size() <= NUM_GPUS_PER_NODE + ): + torch.ops.sglang.inplace_all_reduce( + global_tokens, group_name=get_tp_group().unique_name + ) + + else: + global_tokens[:] = tensor_model_parallel_all_reduce(global_tokens) + + +def _dp_gather_via_all_gather( + global_tokens: torch.Tensor, + local_tokens: torch.Tensor, + forward_batch: ForwardBatch, + is_partial: bool, +): + if get_attention_tp_size() == 1: + get_tp_group().all_gather_into_tensor(global_tokens, local_tokens) + return + + if not is_partial: + if get_attention_tp_rank() != 0: + local_tokens.fill_(0) + scattered_local_tokens = local_tokens.tensor_split(get_attention_tp_size())[ + get_attention_tp_rank() + ] + get_attention_tp_group().reduce_scatter_tensor(scattered_local_tokens, local_tokens) + get_tp_group().all_gather_into_tensor(global_tokens, scattered_local_tokens) + + +def _dp_gather( + global_tokens: torch.Tensor, + local_tokens: torch.Tensor, + forward_batch: ForwardBatch, + is_partial: bool, +): + if forward_batch.dp_padding_mode.is_max_len(): + _dp_gather_via_all_gather( + global_tokens, local_tokens, forward_batch, is_partial + ) + else: + _dp_gather_via_all_reduce( + global_tokens, local_tokens, forward_batch, is_partial + ) + + +def dp_gather_partial( + global_tokens: torch.Tensor, + local_tokens: torch.Tensor, + forward_batch: ForwardBatch, +): + _dp_gather(global_tokens, local_tokens, forward_batch, is_partial=True) + + +def dp_gather_replicate( + global_tokens: torch.Tensor, + local_tokens: torch.Tensor, + forward_batch: ForwardBatch, +): + _dp_gather(global_tokens, local_tokens, forward_batch, is_partial=False) + + +def dp_scatter( + local_tokens: torch.Tensor, # output + global_tokens: torch.Tensor, # input + forward_batch: ForwardBatch, +): + # local_num_tokens is not necessarily the same as local_tokens.shape[0], + # since local_tokens may be padded for cuda graph + local_start_pos, local_num_tokens = get_dp_local_info(forward_batch) + + local_tokens.fill_(0) + assert local_tokens.is_contiguous() + assert global_tokens.is_contiguous() + if local_tokens.shape[0] > 0: + assert ( + local_tokens.untyped_storage() is not global_tokens.untyped_storage() + ), "aliasing between local_tokens and global_tokens not allowed" + + memcpy_triton( + local_tokens, global_tokens, 0, local_start_pos, local_num_tokens, True + ) + + +def dp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): + if get_tensor_model_parallel_world_size() == get_attention_dp_size(): + get_tp_group().reduce_scatter_tensor(output, input) + else: + scattered_local_tokens = input.tensor_split( + get_tensor_model_parallel_world_size() + )[get_tensor_model_parallel_rank()] + get_tp_group().reduce_scatter_tensor(scattered_local_tokens, input) + get_attention_tp_group().all_gather_into_tensor(output, scattered_local_tokens) + + +def attn_tp_reduce_scatter_tensor(output: torch.Tensor, input: torch.Tensor): + return get_attention_tp_group().reduce_scatter_tensor(output, input) + + +def attn_tp_all_gather_into_tensor(output: torch.Tensor, input: torch.Tensor): + return get_attention_tp_group().all_gather_into_tensor(output, input) + + +def attn_tp_all_gather(output_list: List[torch.Tensor], input: torch.Tensor): + return get_attention_tp_group().all_gather(input, output_tensor_list=output_list) diff --git a/python/sglang/srt/layers/elementwise.py b/python/sglang/srt/layers/elementwise.py new file mode 100644 index 000000000..e05d88b32 --- /dev/null +++ b/python/sglang/srt/layers/elementwise.py @@ -0,0 +1,582 @@ +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +fused_softcap_autotune = triton.autotune( + configs=[ + triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=4), + triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 128}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=4), + triton.Config(kwargs={"BLOCK_SIZE": 256}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=4), + triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 512}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 32768}, num_warps=32), + ], + key=["n_ele"], +) + + +@triton.jit +def fused_softcap_kernel( + output_ptr, + input_ptr, + n_ele, + softcap_const: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + block_start = pid * BLOCK_SIZE + offsets = block_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < n_ele + x = tl.load(input_ptr + offsets, mask=mask) + fx = x.to(tl.float32) + fxs = fx / softcap_const + exped = tl.exp(2 * fxs) + top = exped - 1 + bottom = exped + 1 + output = top / bottom * softcap_const + tl.store(output_ptr + offsets, output, mask=mask) + + +fused_softcap_kernel_autotuned = fused_softcap_autotune(fused_softcap_kernel) + + +def fused_softcap(x, softcap_const, autotune=False): + output = torch.empty_like(x, dtype=torch.float32) + n_elements = output.numel() + if autotune: + grid = lambda meta: (triton.cdiv(n_elements, meta["BLOCK_SIZE"]),) + fused_softcap_kernel_autotuned[grid](output, x, n_elements, softcap_const) + else: + fused_softcap_kernel[(triton.cdiv(n_elements, 128),)]( + output, x, n_elements, softcap_const, BLOCK_SIZE=128, num_warps=8 + ) + return output + + +# cast to float + softcap +class Softcap: + def __init__(self, softcap_const: float): + self.softcap_const = softcap_const + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.is_cuda: + return self.forward_cuda(x) + else: + return self.forward_native(x) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + return torch.tanh(x.float() / self.softcap_const) * self.softcap_const + + def forward_cuda(self, x: torch.Tensor, autotune=False) -> torch.Tensor: + return fused_softcap(x, self.softcap_const, autotune=autotune) + + +rmsnorm_autotune = triton.autotune( + configs=[ + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=4, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=8, num_stages=8), + triton.Config(kwargs={"BLOCK_SIZE": 1024}, num_warps=16, num_stages=8), + triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=8, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 2048}, num_warps=16, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 4096}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=8, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=16, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 8192}, num_warps=32, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32, num_stages=1), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=8, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=16, num_stages=4), + triton.Config(kwargs={"BLOCK_SIZE": 16384}, num_warps=32, num_stages=4), + ], + key=["hidden_dim"], +) + + +@triton.jit +def fused_dual_residual_rmsnorm_kernel( + output_ptr, + mid_ptr, + activ_ptr, + residual_ptr, + weight1_ptr, + weight2_ptr, + eps: tl.constexpr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + input_start = pid * hidden_dim + + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + + a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) + a = a_.to(tl.float32) + rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) + + r = tl.load(residual_ptr + input_start + offsets, mask=mask, other=0.0) + w1_ = tl.load(weight1_ptr + offsets, mask=mask, other=0.0) + w1 = w1_.to(tl.float32) + + a2r = r + (a / rms * w1).to(r.dtype) + tl.store( + mid_ptr + input_start + offsets, + a2r, + mask=mask, + ) + + a2r = a2r.to(tl.float32) + rms2 = tl.sqrt(tl.sum(a2r * a2r, axis=0) / hidden_dim + eps) + + w2_ = tl.load(weight2_ptr + offsets, mask=mask, other=0.0) + w2 = w2_.to(tl.float32) + + tl.store( + output_ptr + input_start + offsets, + a2r / rms2 * w2, # implicitly casts to output dtype here + mask=mask, + ) + + +fused_dual_residual_rmsnorm_kernel_autotune = rmsnorm_autotune( + fused_dual_residual_rmsnorm_kernel +) + + +def fused_dual_residual_rmsnorm(x, residual, weight1, weight2, eps, autotune=False): + assert len(x.shape) == 2 + assert x.shape == residual.shape and x.dtype == residual.dtype + output, mid = torch.empty_like(x), torch.empty_like(x) + bs, hidden_dim = x.shape + if autotune: + fused_dual_residual_rmsnorm_kernel_autotune[(bs,)]( + output, mid, x, residual, weight1, weight2, eps=eps, hidden_dim=hidden_dim + ) + else: + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } + + fused_dual_residual_rmsnorm_kernel[(bs,)]( + output, + mid, + x, + residual, + weight1, + weight2, + eps=eps, + hidden_dim=hidden_dim, + **config, + ) + + return output, mid + + +@triton.jit +def fused_rmsnorm_kernel( + output_ptr, + activ_ptr, + weight_ptr, + eps: tl.constexpr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + input_start = pid * hidden_dim + + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + + a_ = tl.load(activ_ptr + input_start + offsets, mask=mask, other=0.0) + a = a_.to(tl.float32) + rms = tl.sqrt(tl.sum(a * a, axis=0) / hidden_dim + eps) + + w1_ = tl.load(weight_ptr + offsets, mask=mask, other=0.0) + w1 = w1_.to(tl.float32) + + a_rms = a / rms * w1 + + tl.store( + output_ptr + input_start + offsets, + a_rms, # implicitly casts to output dtype here + mask=mask, + ) + + +def fused_rmsnorm(x, weight, eps, autotune=False, inplace=False): + assert len(x.shape) == 2 + if inplace: + output = x + else: + output = torch.empty_like(x) + bs, hidden_dim = x.shape + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } + + fused_rmsnorm_kernel[(bs,)]( + output, x, weight, eps=eps, hidden_dim=hidden_dim, **config + ) + return output + + +class FusedDualResidualRMSNorm: + """ + Fused implementation of + y = RMSNorm2(RMSNorm1(x) + residual)) + """ + + def __init__(self, rmsnorm1, rmsnorm2) -> None: # the one after rmsnorm1 + self.rmsnorm1 = rmsnorm1 + self.rmsnorm2 = rmsnorm2 + self.variance_epsilon = self.rmsnorm1.variance_epsilon + assert self.rmsnorm1.variance_epsilon == self.rmsnorm2.variance_epsilon + assert self.rmsnorm1.weight.shape == self.rmsnorm2.weight.shape + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + def forward( + self, x: torch.Tensor, residual: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + if x.is_cuda: + return self.forward_cuda(x, residual) + else: + return self.forward_flashinfer(x, residual) + + def forward_cuda( + self, x: torch.Tensor, residual: torch.Tensor, autotune=False + ) -> Tuple[torch.Tensor, torch.Tensor]: + return fused_dual_residual_rmsnorm( + x, + residual, + self.rmsnorm1.weight, + self.rmsnorm2.weight, + self.variance_epsilon, + autotune=autotune, + ) + + def forward_flashinfer( + self, + x: torch.Tensor, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + normed1 = self.rmsnorm1(x) + residual = normed1 + residual + return self.rmsnorm2(residual), residual + + def forward_native( + self, + x: torch.Tensor, + residual: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + normed1 = self.rmsnorm1.forward_native(x) + residual = normed1 + residual + return self.rmsnorm2.forward_native(residual), residual + + +@triton.jit +def experts_combine_kernel( + out_hidden_states, + moe_hidden_states, + mlp_hidden_states, + combine_k: tl.constexpr, + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(0) + start_index_mlp = pid * hidden_dim + start_index_rmoe = pid * hidden_dim * combine_k + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + combine_k_offsets = tl.arange(0, combine_k) + + moe_x = tl.load( + moe_hidden_states + + start_index_rmoe + + combine_k_offsets[:, None] * hidden_dim + + offsets[None, :], + mask=mask[None, :], + other=0.0, + ) + moe_x = tl.sum(moe_x, axis=0) + mlp_x = tl.load(mlp_hidden_states + start_index_mlp + offsets, mask=mask, other=0.0) + combined_x = (moe_x + mlp_x) / 1.4142135623730951 + + tl.store(out_hidden_states + start_index_mlp + offsets, combined_x, mask=mask) + + +def experts_combine_triton(moe_hidden_states, mlp_hidden_states, output_buffer=None): + assert moe_hidden_states.is_contiguous() + assert mlp_hidden_states.is_contiguous() + + if len(moe_hidden_states.shape) == 2: + combine_k = 1 # pre-combined + else: + combine_k = moe_hidden_states.shape[1] + + if output_buffer is None: + out_hidden_states = torch.empty_like(mlp_hidden_states) + else: + flat_output_buffer = output_buffer.view(mlp_hidden_states.dtype).reshape(-1) + assert flat_output_buffer.numel() >= mlp_hidden_states.numel() + out_hidden_states = flat_output_buffer[: mlp_hidden_states.numel()].reshape( + mlp_hidden_states.shape + ) + + bs, hidden_dim = mlp_hidden_states.shape + + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 1024)), 8), 4 + ), + } + + experts_combine_kernel[(bs,)]( + out_hidden_states, + moe_hidden_states, + mlp_hidden_states, + combine_k, + hidden_dim, + **config, + ) + return out_hidden_states + + +# gelu on first half of vector +@triton.jit +def gelu_and_mul_kernel( + out_hidden_states_ptr, # (bs, hidden_dim) + out_scales_ptr, # (bs,) + hidden_states_ptr, # (bs, hidden_dim * 2) + quant_max: tl.constexpr, + static_scale: tl.constexpr, + hidden_dim: tl.constexpr, # the output hidden_dim + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + + input_start = pid * hidden_dim * 2 + output_start = pid * hidden_dim + + input1_offs = tl.arange(0, BLOCK_SIZE) + mask = tl.arange(0, BLOCK_SIZE) < hidden_dim # shared for input1, input3, output + input3_offs = hidden_dim + tl.arange(0, BLOCK_SIZE) + output_offs = tl.arange(0, BLOCK_SIZE) + + x1 = tl.load( + hidden_states_ptr + input_start + input1_offs, mask=mask, other=0.0 + ).to(tl.float32) + x3 = tl.load( + hidden_states_ptr + input_start + input3_offs, mask=mask, other=0.0 + ).to(tl.float32) + + # gelu + # cast down before mul to better match training? + gelu_x1 = 0.5 * (1.0 + tl.erf(x1 * 0.7071067811865475)) * x1 + out = x3 * gelu_x1.to(hidden_states_ptr.dtype.element_ty) + + if quant_max is not None: + raise NotImplementedError() + + tl.store(out_hidden_states_ptr + output_start + output_offs, out, mask=mask) + + +def gelu_and_mul_triton( + hidden_states, + scales=None, + quantize=None, # dtype to quantize to + out=None, +): + bs, in_hidden_dim = hidden_states.shape + hidden_dim = in_hidden_dim // 2 + + if out is None: + out_hidden_states = torch.empty( + (bs, hidden_dim), + dtype=quantize or hidden_states.dtype, + device=hidden_states.device, + ) + else: + assert out.shape == (bs, hidden_dim) + assert out.dtype == (quantize or hidden_states.dtype) + out_hidden_states = out + out_scales = None + static_scale = False + if quantize is not None: + if scales is None: + out_scales = torch.empty( + (bs,), dtype=torch.float32, device=hidden_states.device + ) + else: + out_scales = scales + static_scale = True + + max_warps = 16 if _is_hip else 32 + config = { + # 8 ele per thread (not tuned) + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 + ), + } + + gelu_and_mul_kernel[(bs,)]( + out_hidden_states, + out_scales, + hidden_states, + quant_max=torch.finfo(quantize).max if quantize is not None else None, + static_scale=static_scale, + hidden_dim=hidden_dim, + BLOCK_SIZE=triton.next_power_of_2(hidden_dim), + **config, + ) + + if quantize is not None: + return out_hidden_states, out_scales + else: + return out_hidden_states, None + + +# silu on first half of vector +@triton.jit +def silu_and_mul_kernel( + out_hidden_states_ptr, # (bs, hidden_dim) + out_scales_ptr, # (bs,) + hidden_states_ptr, # (bs, hidden_dim * 2) + quant_max: tl.constexpr, + static_scale: tl.constexpr, + hidden_dim: tl.constexpr, # the output hidden_dim + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + + input_start = pid * hidden_dim * 2 + output_start = pid * hidden_dim + + input1_offs = tl.arange(0, BLOCK_SIZE) + mask = tl.arange(0, BLOCK_SIZE) < hidden_dim # shared for input1, input3, output + input3_offs = hidden_dim + tl.arange(0, BLOCK_SIZE) + output_offs = tl.arange(0, BLOCK_SIZE) + + x1 = tl.load( + hidden_states_ptr + input_start + input1_offs, mask=mask, other=0.0 + ).to(tl.float32) + x3 = tl.load( + hidden_states_ptr + input_start + input3_offs, mask=mask, other=0.0 + ).to(tl.float32) + + # silu + # cast down before mul to better match training? + silu_x1 = x1 * tl.sigmoid(x1) + out = x3 * silu_x1.to(hidden_states_ptr.dtype.element_ty) + + if quant_max is not None: + raise NotImplementedError() + + tl.store(out_hidden_states_ptr + output_start + output_offs, out, mask=mask) + + +def silu_and_mul_triton( + hidden_states, + scales=None, + quantize=None, # dtype to quantize to + out=None, +): + bs, in_hidden_dim = hidden_states.shape + hidden_dim = in_hidden_dim // 2 + + if out is None: + out_hidden_states = torch.empty( + (bs, hidden_dim), + dtype=quantize or hidden_states.dtype, + device=hidden_states.device, + ) + else: + assert out.shape == (bs, hidden_dim) + assert out.dtype == (quantize or hidden_states.dtype) + out_hidden_states = out + out_scales = None + static_scale = False + if quantize is not None: + if scales is None: + out_scales = torch.empty( + (bs,), dtype=torch.float32, device=hidden_states.device + ) + else: + out_scales = scales + static_scale = True + + max_warps = 16 if _is_hip else 32 + config = { + # 8 ele per thread (not tuned) + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 8 * 32)), max_warps), 4 + ), + } + + silu_and_mul_kernel[(bs,)]( + out_hidden_states, + out_scales, + hidden_states, + quant_max=torch.finfo(quantize).max if quantize is not None else None, + static_scale=static_scale, + hidden_dim=hidden_dim, + BLOCK_SIZE=triton.next_power_of_2(hidden_dim), + **config, + ) + + if quantize is not None: + return out_hidden_states, out_scales + else: + return out_hidden_states, None diff --git a/python/sglang/srt/layers/flashinfer_comm_fusion.py b/python/sglang/srt/layers/flashinfer_comm_fusion.py new file mode 100644 index 000000000..81280db0a --- /dev/null +++ b/python/sglang/srt/layers/flashinfer_comm_fusion.py @@ -0,0 +1,230 @@ +import logging +from typing import Optional, Tuple + +import torch +import torch.distributed as dist + +from sglang.srt.distributed import get_tensor_model_parallel_world_size +from sglang.srt.utils import ( + direct_register_custom_op, + is_flashinfer_available, + supports_custom_op, +) + +logger = logging.getLogger(__name__) + +_flashinfer_comm = None +_workspace_manager = None + +if is_flashinfer_available(): + try: + import flashinfer.comm as comm + + _flashinfer_comm = comm + except ImportError: + logger.warning( + "flashinfer.comm is not available, falling back to standard " + "implementation" + ) + + +class FlashInferWorkspaceManager: + def __init__(self): + self.workspace_tensor = None + self.ipc_handles = None + self.world_size = None + self.rank = None + self.initialized = False + + def initialize( + self, + world_size: int, + rank: int, + max_token_num: int, + hidden_dim: int, + group=None, + use_fp32_lamport: bool = False, + ): + """Initialize workspace""" + if self.initialized and self.world_size == world_size: + return + + if _flashinfer_comm is None: + logger.warning( + "FlashInfer comm not available, skipping workspace " "initialization" + ) + return + + self.cleanup() + + self.ipc_handles, self.workspace_tensor = ( + comm.trtllm_create_ipc_workspace_for_all_reduce_fusion( + rank, + world_size, + max_token_num, + hidden_dim, + group=group, + use_fp32_lamport=use_fp32_lamport, + ) + ) + + self.world_size = world_size + self.rank = rank + self.initialized = True + + logger.info( + f"FlashInfer workspace initialized for rank {rank}, " + f"world_size {world_size}" + ) + + def cleanup(self): + """Clean up workspace""" + if self.initialized and self.ipc_handles is not None: + try: + _flashinfer_comm.trtllm_destroy_ipc_workspace_for_all_reduce( + self.ipc_handles, group=dist.group.WORLD + ) + except Exception as e: + logger.warning(f"Failed to cleanup FlashInfer workspace: {e}") + finally: + self.workspace_tensor = None + self.ipc_handles = None + self.initialized = False + + +_workspace_manager = FlashInferWorkspaceManager() + + +def ensure_workspace_initialized( + max_token_num: int = 2048, hidden_dim: int = 4096, use_fp32_lamport: bool = False +): + """Ensure workspace is initialized""" + if not is_flashinfer_available() or _flashinfer_comm is None: + return False + + world_size = get_tensor_model_parallel_world_size() + if world_size <= 1: + return False + + rank = dist.get_rank() + + if ( + not _workspace_manager.initialized + or _workspace_manager.world_size != world_size + ): + _workspace_manager.initialize( + world_size=world_size, + rank=rank, + max_token_num=max_token_num, + hidden_dim=hidden_dim, + use_fp32_lamport=use_fp32_lamport, + ) + + return _workspace_manager.initialized + + +def flashinfer_allreduce_residual_rmsnorm( + input_tensor: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + max_token_num: int = 2048, + use_oneshot: Optional[bool] = None, + trigger_completion_at_end: bool = False, + fp32_acc: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Use FlashInfer's fused allreduce + residual + RMS norm operation + + Args: + input_tensor: Input tensor that needs allreduce + residual: Residual tensor + weight: RMS norm weight + eps: RMS norm epsilon + max_token_num: Maximum token number + use_oneshot: Whether to use oneshot mode + trigger_completion_at_end: Whether to trigger completion at end + fp32_acc: Whether to use fp32 precision + + Returns: + Tuple[torch.Tensor, torch.Tensor]: (norm_output, residual_output) + """ + if not is_flashinfer_available() or _flashinfer_comm is None: + logger.debug( + "FlashInfer not available, falling back to standard " "implementation" + ) + return None, None + + world_size = get_tensor_model_parallel_world_size() + if world_size <= 1: + logger.debug("Single GPU, no need for allreduce fusion") + return None, None + + if not ensure_workspace_initialized( + max_token_num=max_token_num, + hidden_dim=input_tensor.shape[-1], + use_fp32_lamport=(input_tensor.dtype == torch.float32), + ): + logger.debug("FlashInfer workspace not available") + return None, None + + token_num, hidden_dim = input_tensor.shape + + residual_out = torch.empty_like(residual) + norm_out = torch.empty_like(input_tensor) + + _flashinfer_comm.trtllm_allreduce_fusion( + allreduce_in=input_tensor, + world_size=world_size, + world_rank=dist.get_rank(), + token_num=token_num, + hidden_dim=hidden_dim, + workspace_ptrs=_workspace_manager.workspace_tensor, + launch_with_pdl=True, + use_oneshot=use_oneshot, + trigger_completion_at_end=trigger_completion_at_end, + fp32_acc=fp32_acc, + pattern_code=(_flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm), + allreduce_out=None, + residual_in=residual, + residual_out=residual_out, + norm_out=norm_out, + quant_out=None, + scale_out=None, + rms_gamma=weight, + rms_eps=eps, + scale_factor=None, + layout_code=None, + ) + + return norm_out, residual_out + + +def fake_flashinfer_allreduce_residual_rmsnorm( + input_tensor: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + max_token_num: int = 2048, + use_oneshot: Optional[bool] = None, + trigger_completion_at_end: bool = False, + fp32_acc: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + residual_out = torch.empty_like(residual) + norm_out = torch.empty_like(input_tensor) + return norm_out, residual_out + + +if supports_custom_op(): + direct_register_custom_op( + "flashinfer_allreduce_residual_rmsnorm", + flashinfer_allreduce_residual_rmsnorm, + mutates_args=["input_tensor", "residual", "weight"], + fake_impl=fake_flashinfer_allreduce_residual_rmsnorm, + ) + + +def cleanup_flashinfer_workspace(): + global _workspace_manager + if _workspace_manager is not None: + _workspace_manager.cleanup() diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py new file mode 100644 index 000000000..12cd49adf --- /dev/null +++ b/python/sglang/srt/layers/layernorm.py @@ -0,0 +1,343 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Fused operators for normalization layers.""" + +import logging +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +from packaging.version import Version + +from sglang.srt.custom_op import CustomOp +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_cuda, + is_flashinfer_available, + is_hip, + is_npu, + is_xpu, + supports_custom_op, +) + +_is_cuda = is_cuda() +_is_flashinfer_available = is_flashinfer_available() +_is_hip = is_hip() +_is_npu = is_npu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +_is_xpu = is_xpu() + +if _is_cuda: + if _is_flashinfer_available: + from flashinfer.norm import fused_add_rmsnorm + else: + from sgl_kernel import fused_add_rmsnorm + from sgl_kernel import gemma_fused_add_rmsnorm, gemma_rmsnorm, rmsnorm + +if _use_aiter: + from aiter import rmsnorm2d_fwd as rms_norm + from aiter import rmsnorm2d_fwd_with_add as fused_add_rms_norm +elif _is_hip: + import vllm + from vllm._custom_ops import fused_add_rms_norm, rms_norm + + _vllm_version = Version(vllm.__version__) + +logger = logging.getLogger(__name__) + +if _is_npu: + import torch_npu + + +class RMSNorm(CustomOp): + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + var_hidden_size: Optional[int] = None, + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + self.hidden_size = hidden_size + self.variance_size_override = ( + None if var_hidden_size == hidden_size else var_hidden_size + ) + if _use_aiter: + self._forward_method = self.forward_aiter + + def forward_cuda( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if self.variance_size_override is not None: + return self.forward_native(x, residual) + if residual is not None: + fused_add_rmsnorm(x, residual, self.weight.data, self.variance_epsilon) + return x, residual + out = rmsnorm(x, self.weight.data, self.variance_epsilon) + return out + + def forward_npu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if residual is not None: + out, _, residual_out = torch_npu.npu_add_rms_norm( + residual, x, self.weight.data, self.variance_epsilon + ) + return out, residual_out + return torch_npu.npu_rms_norm(x, self.weight.data, self.variance_epsilon)[0] + + def forward_aiter( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if residual is not None: + residual_out = torch.empty_like(x) + output = torch.empty_like(x) + fused_add_rms_norm( + output, + x, + residual, + residual_out, + self.weight.data, + self.variance_epsilon, + ) + return output, residual_out + return rms_norm(x, self.weight.data, self.variance_epsilon) + + def forward_hip( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ): + if not x.is_contiguous(): + x = x.contiguous() + + if residual is not None: + try: + output = torch.empty_like(x) + residual_out = torch.empty_like(x) + fused_add_rms_norm( + output, + x, + residual_out, + residual, + self.weight.data, + self.variance_epsilon, + ) + return output, residual_out + except TypeError: + fused_add_rms_norm( + x, + residual, + self.weight.data, + self.variance_epsilon, + ) + return x, residual + + out = torch.empty_like(x) + rms_norm(out, x, self.weight.data, self.variance_epsilon) + return out + + + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if not x.is_contiguous(): + x = x.contiguous() + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + hidden_size = x.shape[-1] + if hidden_size != self.hidden_size: + raise ValueError( + "Expected hidden_size to be " + f"{self.hidden_size}, but found: {hidden_size}" + ) + + if self.variance_size_override is None: + x_var = x + else: + if hidden_size < self.variance_size_override: + raise ValueError( + "Expected hidden_size to be at least " + f"{self.variance_size_override}, but found: {hidden_size}" + ) + + x_var = x[..., : self.variance_size_override] + + variance = x_var.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = (x * self.weight).to(orig_dtype) + if residual is None: + return x + else: + return x, residual + + def forward_cpu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if _is_cpu_amx_available: + if residual is not None: + torch.ops.sgl_kernel.fused_add_rmsnorm_cpu( + x, residual, self.weight.data, self.variance_epsilon + ) + return x, residual + return torch.ops.sgl_kernel.rmsnorm_cpu( + x, self.weight.data, self.variance_epsilon + ) + else: + return self.forward_native(x, residual) + + def forward_with_allreduce_fusion( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + Forward method with allreduce fusion, prioritizing flashinfer fused operations + """ + if residual is not None: + from sglang.srt.distributed import get_tensor_model_parallel_world_size + from sglang.srt.layers.flashinfer_comm_fusion import ( + flashinfer_allreduce_residual_rmsnorm, + ) + + fused_op = ( + torch.ops.sglang.flashinfer_allreduce_residual_rmsnorm + if supports_custom_op() + else flashinfer_allreduce_residual_rmsnorm + ) + + if get_tensor_model_parallel_world_size() > 1: + fused_result = fused_op( + input_tensor=x, + residual=residual, + weight=self.weight, + eps=self.variance_epsilon, + ) + if fused_result[0] is not None: + return fused_result + + return self.forward(x, residual) + + +class GemmaRMSNorm(CustomOp): + def __init__( + self, + hidden_size: int, + eps: float = 1e-6, + ) -> None: + super().__init__() + self.weight = nn.Parameter(torch.zeros(hidden_size)) + self.variance_epsilon = eps + + # Re-dispatch + if _is_hip: + self._forward_method = self.forward_native + + def forward_native( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + if residual is not None: + x = x + residual + residual = x + + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + self.variance_epsilon) + x = x * (1.0 + self.weight.float()) + x = x.to(orig_dtype) + return x if residual is None else (x, residual) + + def forward_cuda( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if residual is not None: + gemma_fused_add_rmsnorm( + x, residual, self.weight.data, self.variance_epsilon + ) + return x, residual + out = gemma_rmsnorm(x, self.weight.data, self.variance_epsilon) + return out + + def forward_npu( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if residual is not None: + x = x + residual + residual = x + + x, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.variance_epsilon) + return x if residual is None else (x, residual) + + +class Gemma3RMSNorm(CustomOp): + def __init__(self, dim: int, eps: float = 1e-6): + super().__init__() + self.eps = eps + self.weight = nn.Parameter(torch.zeros(dim)) + # Re-dispatch + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward_native(self, x): + output = self._norm(x.float()) + # Llama does x.to(float16) * w whilst Gemma3 is (x * w).to(float16) + # See https://github.com/huggingface/transformers/pull/29402 + output = output * (1.0 + self.weight.float()) + return output.type_as(x) + + def forward_cuda(self, x): + return self.forward_native(x) + + def forward_npu(self, x): + output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps) + return output + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.eps}" + + +if not ( + _is_cuda or _is_hip or _is_npu or (_is_cpu and _is_cpu_amx_available) or _is_xpu +): + logger.info( + "sgl-kernel layernorm implementation is not available on current platform. Fallback to other kernel libraries." + ) + from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py new file mode 100644 index 000000000..df2b77e08 --- /dev/null +++ b/python/sglang/srt/layers/linear.py @@ -0,0 +1,1337 @@ +"""Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/layers/linear.py""" + +from __future__ import annotations + +import itertools +import logging +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple + +import torch +from torch.nn.parameter import Parameter, UninitializedParameter + +from sglang.srt.distributed import ( + divide, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + parallel_state, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.layers.parameter import ( + BasevLLMParameter, + BlockQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + PerTensorScaleParameter, + RowvLLMParameter, + _ColumnvLLMParameter, +) +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.utils import is_cpu, is_npu, set_weight_attrs + +if TYPE_CHECKING: + from sglang.srt.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, + ) + +logger = logging.getLogger(__name__) + +WEIGHT_LOADER_V2_SUPPORTED = [ + "CompressedTensorsLinearMethod", + "AWQMarlinLinearMethod", + "AWQLinearMethod", + "GPTQMarlinLinearMethod", + "Fp8LinearMethod", + "BlockInt8LinearMethod", + "MarlinLinearMethod", + "QQQLinearMethod", + "GPTQMarlin24LinearMethod", + "TPUInt8LinearMethod", + "GPTQLinearMethod", + "FBGEMMFp8LinearMethod", + "ModelOptFp8LinearMethod", + "ModelOptFp4LinearMethod", + "IPEXAWQLinearMethod", + "PetitNvFp4LinearMethod", +] + +_is_cpu = is_cpu() +_is_npu = is_npu() + + +def adjust_marlin_shard(param, shard_size, shard_offset): + marlin_tile_size = getattr(param, "marlin_tile_size", None) + if marlin_tile_size is None: + return shard_size, shard_offset + + return shard_size * marlin_tile_size, shard_offset * marlin_tile_size + + +def adjust_bitsandbytes_4bit_shard( + param: Parameter, shard_offsets: Dict[str, Tuple[int, int]], loaded_shard_id: str +) -> Tuple[int, int]: + """Adjust the quantization offsets and sizes for BitsAndBytes sharding.""" + + total, _ = shard_offsets["total"] + orig_offset, orig_size = shard_offsets[loaded_shard_id] + + quantized_total = param.data.shape[0] + quantized_offset = orig_offset * quantized_total // total + quantized_size = orig_size * quantized_total // total + + return quantized_size, quantized_offset + + +def adjust_scalar_to_fused_array(param, loaded_weight, shard_id): + """For fused modules (QKV and MLP) we have an array of length + N that holds 1 scale for each "logical" matrix. So the param + is an array of length N. The loaded_weight corresponds to + one of the shards on disk. Here, we slice the param based on + the shard_id for loading. + """ + qkv_idxs = {"q": 0, "k": 1, "v": 2} + + if isinstance(shard_id, str): + shard_id = qkv_idxs[shard_id] + elif not isinstance(shard_id, int): + raise ValueError(f"Unknown Shard Id {shard_id}") + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + return param[shard_id], loaded_weight + + +def adjust_shard_offsets(shard_offsets, loaded_weight, dim): + actual_weight_size = loaded_weight.size(dim) + target_weight_size = shard_offsets[-1][-1] + shard_offsets[-1][-2] + if actual_weight_size != target_weight_size: + new_shard_offsets = [] + new_offset = 0 + for shard_id, shard_offset, shard_size in shard_offsets: + actual_shard_size = actual_weight_size * shard_size // target_weight_size + new_shard_offsets.append((shard_id, new_offset, actual_shard_size)) + new_offset += actual_shard_size + return new_shard_offsets + return shard_offsets + + +class LinearBase(torch.nn.Module): + """Base linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__( + self, + input_size: int, + output_size: int, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__() + + # Keep input parameters + self.input_size = input_size + self.output_size = output_size + self.skip_bias_add = skip_bias_add + if params_dtype is None: + params_dtype = torch.get_default_dtype() + self.params_dtype = params_dtype + if quant_config is None: + self.quant_method: Optional[QuantizeMethodBase] = UnquantizedLinearMethod() + else: + self.quant_method = quant_config.get_quant_method(self, prefix=prefix) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + raise NotImplementedError + + +class ReplicatedLinear(LinearBase): + """Replicated linear layer. + + Args: + input_size: input dimension of the linear layer. + output_size: output dimension of the linear layer. + bias: If true, add bias. + skip_bias_add: If true, skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__( + input_size, + output_size, + skip_bias_add, + params_dtype, + quant_config, + prefix=prefix, + ) + + # All the linear layer supports quant method. + assert self.quant_method is not None + self.quant_method.create_weights( + self, + self.input_size, + [self.output_size], + self.input_size, + self.output_size, + self.params_dtype, + weight_loader=self.weight_loader, + ) + + if bias: + self.bias = Parameter( + torch.empty(self.output_size, dtype=self.params_dtype) + ) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + # If the weight on disk does not have a shape, give it one + # (such scales for AutoFp8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + # The per-tensor quant-scale must be 1 dimension + if _is_npu: + if param.size() != loaded_weight.size() and param.size(0) == 1: + if torch.allclose(loaded_weight, loaded_weight[0]): + loaded_weight = loaded_weight[:1] + else: + raise ValueError(f"{loaded_weight} are not all equal") + + assert param.size() == loaded_weight.size() + param.data.copy_(loaded_weight) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + bias = self.bias if not self.skip_bias_add else None + assert self.quant_method is not None + output = self.quant_method.apply(self, x, bias) + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + return s + + +class ColumnParallelLinear(LinearBase): + """Linear layer with column parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its second dimension as A = [A_1, ..., A_p]. + + Args: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make Y available + to all GPUs, otherwise, every GPU will have its output + which is Y_i = XA_i + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + output_sizes: list of output sizes packed into one output, like for QKV + the list would be size 3. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + output_sizes: Optional[List[int]] = None, + prefix: str = "", + tp_rank: Optional[int] = None, + tp_size: Optional[int] = None, + use_presharded_weights: bool = False, + ): + super().__init__( + input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix + ) + + self.gather_output = gather_output + self.use_presharded_weights = use_presharded_weights + + # Divide the weight matrix along the last dimension. + if tp_rank is None: + tp_rank = get_tensor_model_parallel_rank() + if tp_size is None: + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank, self.tp_size = tp_rank, tp_size + assert self.quant_method is not None + self.output_size_per_partition = divide(self.output_size, tp_size) + self.output_partition_sizes = [self.output_size_per_partition] + # If QKV or MergedColumn, use output size of each partition. + if hasattr(self, "output_sizes"): + self.output_partition_sizes = [ + divide(output_size, tp_size) for output_size in self.output_sizes + ] + + if output_sizes is None: + output_sizes = [output_size] + + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size, + output_partition_sizes=self.output_partition_sizes, + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 + if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED + else self.weight_loader + ), + ) + if bias: + self.bias = Parameter( + torch.empty(self.output_size_per_partition, dtype=params_dtype) + ) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + output_dim = getattr(param, "output_dim", None) + + # Special case for GGUF + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.weight_type = loaded_weight.item() + + # Materialize GGUF UninitializedParameter + if is_gguf_weight and isinstance(param, UninitializedParameter): + param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + + param_data = param.data + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if output_dim is not None and not use_bitsandbytes_4bit: + shard_size = param_data.shape[output_dim] + start_idx = self.tp_rank * shard_size + + if _is_cpu: + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + start_idx, + output_dim, + shard_size, + not self.use_presharded_weights, + ) + else: + if not self.use_presharded_weights: + loaded_weight = loaded_weight.narrow( + output_dim, start_idx, shard_size + ) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: Parameter, loaded_weight: torch.Tensor): + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + + if isinstance(param, _ColumnvLLMParameter): + param.load_column_parallel_weight( + loaded_weight, + tp_rank=self.tp_rank, + use_presharded_weights=self.use_presharded_weights, + ) + else: + # FIXME: This branch is needed to load deepseek v3 awq. + # However, we should fix this and avoid the branching here. + param.load_column_parallel_weight(loaded_weight) + + def forward(self, input_): + bias = self.bias if not self.skip_bias_add else None + + # Matrix multiply. + assert self.quant_method is not None + output_parallel = self.quant_method.apply(self, input_, bias) + if self.gather_output: + # All-gather across the partitions. + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = self.bias if self.skip_bias_add else None + return output, output_bias + + def extra_repr(self) -> str: + s = f"in_features={self.input_size}" + s += f", output_features={self.output_size_per_partition}" + s += f", bias={self.bias is not None}" + s += f", tp_size={self.tp_size}" + s += f", gather_output={self.gather_output}" + return s + + +class MergedColumnParallelLinear(ColumnParallelLinear): + """Packed linear layers with column parallelism. + + Similar to ColumnParallelLinear, but the weight matrix is concatenated + along the output dimension. When the weight matrix is loaded, the + different partitions are sharded separately. + + Args: + input_size: input dimension of the linear layer. + output_sizes: list of output dimensions of the linear layer. + bias: If true, add bias. + gather_output: If true, call all-gather on output and make the output + available to all GPUs, otherwise, every GPU will have + its own output. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + input_size: int, + output_sizes: List[int], + bias: bool = True, + gather_output: bool = False, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + tp_rank: Optional[int] = None, + tp_size: Optional[int] = None, + use_presharded_weights: bool = False, + ): + self.output_sizes = output_sizes + if tp_rank is None: + tp_rank = get_tensor_model_parallel_rank() + if tp_size is None: + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank, self.tp_size = tp_rank, tp_size + assert all(output_size % tp_size == 0 for output_size in output_sizes) + self.use_presharded_weights = use_presharded_weights + super().__init__( + input_size=input_size, + output_size=sum(output_sizes), + bias=bias, + gather_output=gather_output, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + tp_rank=tp_rank, + tp_size=tp_size, + use_presharded_weights=use_presharded_weights, + ) + self.prefix = prefix + + def weight_loader( + self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[int] = None, + ): + + # Special case for GGUF + # initialize GGUF param after we know the quantize type + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.data[loaded_shard_id].copy_(loaded_weight) + param.shard_weight_type[loaded_shard_id] = loaded_weight.item() + return + + if is_gguf_weight: + output_dim = getattr(param, "output_dim", None) + shard_size = loaded_weight.size(output_dim) // self.tp_size + start_idx = self.tp_rank * shard_size + + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + + param.shard_id.append(loaded_shard_id) + param.shard_id_map[loaded_shard_id] = len(param.data_container) + param.data_container.append(loaded_weight) + return + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + # Special case for per-tensor scale to load scalar into fused array. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (qkv/mlp). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0 + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + current_shard_offset = 0 + shard_offsets: List[Tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + packed_dim = getattr(param, "packed_dim", None) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + if _is_cpu: + shard_offsets = adjust_shard_offsets( + shard_offsets, loaded_weight, output_dim + ) + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset + ) + + if use_bitsandbytes_4bit: + index = list(itertools.accumulate([0] + self.output_sizes)) + orig_offsets = { + str(i): (index[i], size) + for i, size in enumerate(self.output_sizes) + } + orig_offsets["total"] = (self.output_size, 0) + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( + param, orig_offsets, str(shard_id) + ) + + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size + ) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + assert loaded_shard_id < len(self.output_sizes) + if output_dim is not None: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size + shard_size = self.output_sizes[loaded_shard_id] // self.tp_size + # Special case for quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + packed_dim = getattr(param, "packed_dim", None) + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset + ) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + if use_bitsandbytes_4bit: + shard_size = loaded_weight.shape[output_dim] + shard_offset = loaded_weight.shape[output_dim] * loaded_shard_id + + param_data = param_data.narrow(output_dim, shard_offset, shard_size) + start_idx = self.tp_rank * shard_size + + if _is_cpu: + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + start_idx, + output_dim, + shard_size, + not use_bitsandbytes_4bit and not self.use_presharded_weights, + ) + else: + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if not use_bitsandbytes_4bit and not self.use_presharded_weights: + loaded_weight = loaded_weight.narrow( + output_dim, start_idx, shard_size + ) + + # Special case for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_offset = loaded_shard_id * shard_size + param_data = param_data.narrow(0, shard_offset, shard_size) + + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id + ) + + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "MergedColumnParallelLinear, assume the weight is " + "the same for all partitions." + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def _load_fused_module_from_checkpoint( + self, param: BasevLLMParameter, loaded_weight: torch.Tensor + ): + """ + Handle special case for models where MLP layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + + current_shard_offset = 0 + shard_offsets: List[Tuple[int, int, int]] = [] + for i, output_size in enumerate(self.output_sizes): + shard_offsets.append((i, current_shard_offset, output_size)) + current_shard_offset += output_size + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if ( + isinstance(param, (PackedColumnParameter, PackedvLLMParameter)) + and param.packed_dim == param.output_dim + ): + shard_size, shard_offset = param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset + ) + + loaded_weight_shard = loaded_weight.narrow( + param.output_dim, shard_offset, shard_size + ) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2( + self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[int] = None, + ): + if loaded_shard_id is None: + if isinstance(param, PerTensorScaleParameter): + param.load_merged_column_weight( + loaded_weight=loaded_weight, + shard_id=0, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + ) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_merged_column_weight( + loaded_weight=loaded_weight, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + ) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id < len(self.output_sizes) + + if isinstance(param, BlockQuantScaleParameter): + weight_block_size = self.quant_method.quant_config.weight_block_size + block_n, _ = weight_block_size[0], weight_block_size[1] + shard_offset = ( + (sum(self.output_sizes[:loaded_shard_id]) + block_n - 1) // block_n + ) // self.tp_size + shard_size = ( + (self.output_sizes[loaded_shard_id] + block_n - 1) + // block_n + // self.tp_size + ) + else: + shard_offset = sum(self.output_sizes[:loaded_shard_id]) // self.tp_size + shard_size = self.output_sizes[loaded_shard_id] // self.tp_size + + param.load_merged_column_weight( + loaded_weight=loaded_weight, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size, + use_presharded_weights=self.use_presharded_weights, + tp_rank=self.tp_rank, + tp_size=self.tp_size, + ) + + +class QKVParallelLinear(ColumnParallelLinear): + """Linear layers for the attention's QKV transformation. + + Linear layers for the linear transformation of the query, key, and value + vectors in the attention layer. The weight matrix is concatenated along + the output dimension. The layer is parallelized along the head dimension. + When the number of key/value heads is smaller than the number of query + heads (e.g., multi-query/grouped-query attention), the key/value head may + be replicated while the query heads are partitioned. + + Args: + hidden_size: input hidden state size of the transformer. + head_size: size of each attention head. + total_num_heads: total number of attention query heads. + total_num_kv_heads: total number of attention key/value heads. If + None, assume total_num_kv_heads = total_num_heads. + bias: If true, add bias. + skip_bias_add: This was added to enable performance optimizations where + bias can be fused with other element-wise operations. we + skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + prefix: The name of the layer in the state dict, including all parents + (e.g. model.layers.0.qkv_proj) + """ + + def __init__( + self, + hidden_size: int, + head_size: int, + total_num_heads: int, + total_num_kv_heads: Optional[int] = None, + bias: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + tp_rank: Optional[int] = None, + tp_size: Optional[int] = None, + load_presharded_attn: bool = False, + ): + self.hidden_size = hidden_size + self.head_size = head_size + self.total_num_heads = total_num_heads + if total_num_kv_heads is None: + total_num_kv_heads = total_num_heads + self.total_num_kv_heads = total_num_kv_heads + # Divide the weight matrix along the last dimension. + if tp_rank is None: + tp_rank = get_tensor_model_parallel_rank() + if tp_size is None: + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank, self.tp_size = tp_rank, tp_size + self.num_heads = divide(self.total_num_heads, tp_size) + if tp_size >= self.total_num_kv_heads: + self.num_kv_heads = 1 + self.num_kv_head_replicas = divide(tp_size, self.total_num_kv_heads) + else: + self.num_kv_heads = divide(self.total_num_kv_heads, tp_size) + self.num_kv_head_replicas = 1 + self.q_proj_shard_size = self.num_heads * self.head_size + self.kv_proj_shard_size = self.num_kv_heads * self.head_size + input_size = self.hidden_size + output_size = ( + (self.num_heads + 2 * self.num_kv_heads) * tp_size * self.head_size + ) + self.output_sizes = [ + self.num_heads * self.head_size * tp_size, # q_proj + self.num_kv_heads * self.head_size * tp_size, # k_proj + self.num_kv_heads * self.head_size * tp_size, # v_proj + ] + self.use_presharded_weights = load_presharded_attn + + super().__init__( + input_size=input_size, + output_size=output_size, + bias=bias, + gather_output=False, + skip_bias_add=skip_bias_add, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + tp_rank=tp_rank, + tp_size=tp_size, + use_presharded_weights=self.use_presharded_weights, + ) + + def _get_shard_offset_mapping(self, loaded_shard_id: str): + shard_offset_mapping = { + "q": 0, + "k": self.num_heads * self.head_size, + "v": (self.num_heads + self.num_kv_heads) * self.head_size, + "total": (self.num_heads + 2 * self.num_kv_heads) * self.head_size, + } + return shard_offset_mapping.get(loaded_shard_id) + + def _get_shard_size_mapping(self, loaded_shard_id: str): + shard_size_mapping = { + "q": self.num_heads * self.head_size, + "k": self.num_kv_heads * self.head_size, + "v": self.num_kv_heads * self.head_size, + } + return shard_size_mapping.get(loaded_shard_id) + + def _load_fused_module_from_checkpoint( + self, param: BasevLLMParameter, loaded_weight: torch.Tensor + ): + """ + Handle special case for models where QKV layers are already + fused on disk. In this case, we have no shard id. This function + determmines the shard id by splitting these layers and then calls + the weight loader using the shard id. + + An example of a model with these fused layers: + https://huggingface.co/microsoft/Phi-3-mini-4k-instruct + """ + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ( + "k", + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ( + "v", + (self.total_num_heads + self.total_num_kv_heads) * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ] + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantization. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if ( + isinstance(param, (PackedColumnParameter, PackedvLLMParameter)) + and param.packed_dim == param.output_dim + ): + shard_size, shard_offset = param.adjust_shard_indexes_for_packing( + shard_size=shard_size, shard_offset=shard_offset + ) + + if not self.use_presharded_weights: + loaded_weight_shard = loaded_weight.narrow( + param.output_dim, shard_offset, shard_size + ) + self.weight_loader_v2(param, loaded_weight_shard, shard_id) + + def weight_loader_v2( + self, + param: BasevLLMParameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[str] = None, + ): + if loaded_shard_id is None: # special case for certain models + if isinstance(param, PerTensorScaleParameter): + param.load_qkv_weight(loaded_weight=loaded_weight, shard_id=0) + return + elif type(param) in (RowvLLMParameter, BasevLLMParameter): + param.load_qkv_weight(loaded_weight=loaded_weight) + return + # TODO: @dsikka - move to parameter.py + self._load_fused_module_from_checkpoint(param, loaded_weight) + return + + assert loaded_shard_id in ["q", "k", "v"] + + shard_offset = self._get_shard_offset_mapping(loaded_shard_id) + shard_size = self._get_shard_size_mapping(loaded_shard_id) + + if isinstance(param, BlockQuantScaleParameter): + weight_block_size = self.quant_method.quant_config.weight_block_size + block_n, _ = weight_block_size[0], weight_block_size[1] + shard_offset = (shard_offset + block_n - 1) // block_n + shard_size = (shard_size + block_n - 1) // block_n + + param.load_qkv_weight( + loaded_weight=loaded_weight, + num_heads=self.num_kv_head_replicas, + shard_id=loaded_shard_id, + shard_offset=shard_offset, + shard_size=shard_size, + tp_rank=self.tp_rank, + use_presharded_weights=self.use_presharded_weights, + ) + + def weight_loader( + self, + param: Parameter, + loaded_weight: torch.Tensor, + loaded_shard_id: Optional[str] = None, + ): + + # Special case for GGUF + # initialize GGUF param after we know the quantize type + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type and loaded_shard_id is not None: + idx_map = {"q": 0, "k": 1, "v": 2} + param.data[idx_map[loaded_shard_id]].copy_(loaded_weight) + param.shard_weight_type[loaded_shard_id] = loaded_weight.item() + return + + if is_gguf_weight: + output_dim = getattr(param, "output_dim", None) + shard_size = loaded_weight.size(output_dim) // self.tp_size + start_idx = self.tp_rank * shard_size + + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + + param.shard_id.append(loaded_shard_id) + param.shard_id_map[loaded_shard_id] = len(param.data_container) + param.data_container.append(loaded_weight) + return + + param_data = param.data + output_dim = getattr(param, "output_dim", None) + # Special case for AQLM codebooks. + is_metadata = getattr(param, "is_metadata", False) + + # Special case for per-tensor scales in fused case. + needs_scalar_to_array = getattr(param, "needs_scalar_to_array", False) + + if loaded_shard_id is None: + # Loaded weight is already fused on disk (qkv/mlp). + if output_dim is None: + if needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, 0 + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + shard_offsets = [ + # (shard_id, shard_offset, shard_size) + ("q", 0, self.total_num_heads * self.head_size), + ( + "k", + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ( + "v", + (self.total_num_heads + self.total_num_kv_heads) * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + ] + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + + packed_dim = getattr(param, "packed_dim", None) + if _is_cpu: + shard_offsets = adjust_shard_offsets( + shard_offsets, loaded_weight, output_dim + ) + + for shard_id, shard_offset, shard_size in shard_offsets: + # Special case for Quantized Weights. + # If quantized, we need to adjust the offset and size to account + # for the packing. + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset + ) + + if use_bitsandbytes_4bit: + orig_qkv_offsets = { + "q": (0, self.total_num_heads * self.head_size), + "k": ( + self.total_num_heads * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + "v": ( + (self.total_num_heads + self.total_num_kv_heads) + * self.head_size, + self.total_num_kv_heads * self.head_size, + ), + "total": ( + (self.total_num_heads + 2 * self.total_num_kv_heads) + * self.head_size, + 0, + ), + } + + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( + param, orig_qkv_offsets, shard_id + ) + + if not self.use_presharded_weights: + loaded_weight_shard = loaded_weight.narrow( + output_dim, shard_offset, shard_size + ) + self.weight_loader(param, loaded_weight_shard, shard_id) + return + + assert loaded_shard_id in ["q", "k", "v"] + + # If output dim is defined, use the default loading process. + if output_dim is not None: + if loaded_shard_id == "q": + shard_offset = 0 + shard_size = self.num_heads * self.head_size + elif loaded_shard_id == "k": + shard_offset = self.num_heads * self.head_size + shard_size = self.num_kv_heads * self.head_size + elif loaded_shard_id == "v": + shard_offset = (self.num_heads + self.num_kv_heads) * self.head_size + shard_size = self.num_kv_heads * self.head_size + # Special case for Quantized Weights. + # If quantized, we need to adjust the offset and size to account + # for the packing. + packed_dim = getattr(param, "packed_dim", None) + if packed_dim == output_dim: + shard_size = shard_size // param.pack_factor + shard_offset = shard_offset // param.pack_factor + + # Special case for Marlin. + shard_size, shard_offset = adjust_marlin_shard( + param, shard_size, shard_offset + ) + + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + if use_bitsandbytes_4bit: + orig_qkv_offsets = { + "q": (0, self.num_heads * self.head_size), + "k": ( + self.num_heads * self.head_size, + self.num_kv_heads * self.head_size, + ), + "v": ( + (self.num_heads + self.num_kv_heads) * self.head_size, + self.num_kv_heads * self.head_size, + ), + "total": ( + (self.num_heads + 2 * self.num_kv_heads) * self.head_size, + 0, + ), + } + shard_size, shard_offset = adjust_bitsandbytes_4bit_shard( + param, orig_qkv_offsets, loaded_shard_id + ) + + param_data = param_data.narrow(output_dim, shard_offset, shard_size) + if loaded_shard_id == "q": + shard_id = self.tp_rank + else: + shard_id = self.tp_rank // self.num_kv_head_replicas + start_idx = shard_id * shard_size + + if _is_cpu: + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + start_idx, + output_dim, + shard_size, + not use_bitsandbytes_4bit and not self.use_presharded_weights, + ) + else: + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if not use_bitsandbytes_4bit and not self.use_presharded_weights: + loaded_weight = loaded_weight.narrow( + output_dim, start_idx, shard_size + ) + + # Special case for for AQLM codebooks. + elif is_metadata: + # metadata indicates fixed size concatenated along dim 0 + shard_size = loaded_weight.shape[0] + shard_index = ["q", "k", "v"].index(loaded_shard_id) + param_data = param_data.narrow(0, shard_index * shard_size, shard_size) + # Special case for per-tensor scales in fused case. + elif needs_scalar_to_array: + param_data, loaded_weight = adjust_scalar_to_fused_array( + param_data, loaded_weight, loaded_shard_id + ) + else: + ignore_warning = getattr(param, "ignore_warning", False) + if not ignore_warning: + logger.warning( + "Loading a weight without `output_dim` attribute in " + "QKVParallelLinear, assume the weight is the same " + "for all partitions." + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class RowParallelLinear(LinearBase): + """Linear layer with row parallelism. + + The linear layer is defined as Y = XA + b. A is parallelized along + its first dimension and X along its second dimension as: + - - + | A_1 | + | . | + A = | . | X = [X_1, ..., X_p] + | . | + | A_p | + - - + Arguments: + input_size: first dimension of matrix A. + output_size: second dimension of matrix A. + bias: If true, add bias. Note that bias is not parallelized. + input_is_parallel: If true, we assume that the input is already + split across the GPUs and we do not split + again. + skip_bias_add: This was added to enable performance optimization where + bias can be fused with other element-wise operations. + We skip adding bias but instead return it. + params_dtype: Data type for the parameters. + quant_config: Quantization configure. + """ + + def __init__( + self, + input_size: int, + output_size: int, + bias: bool = True, + input_is_parallel: bool = True, + skip_bias_add: bool = False, + params_dtype: Optional[torch.dtype] = None, + reduce_results: bool = True, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + tp_rank: Optional[int] = None, + tp_size: Optional[int] = None, + use_presharded_weights: bool = False, + ): + super().__init__( + input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix + ) + + self.input_is_parallel = input_is_parallel + self.reduce_results = reduce_results + + # Divide the weight matrix along the last dimension. + if tp_rank is None: + tp_rank = get_tensor_model_parallel_rank() + if tp_size is None: + tp_size = get_tensor_model_parallel_world_size() + self.tp_rank, self.tp_size = tp_rank, tp_size + self.input_size_per_partition = divide(input_size, self.tp_size) + assert self.quant_method is not None + self.use_presharded_weights = use_presharded_weights + + self.quant_method.create_weights( + layer=self, + input_size_per_partition=self.input_size_per_partition, + output_partition_sizes=[self.output_size], + input_size=self.input_size, + output_size=self.output_size, + params_dtype=self.params_dtype, + weight_loader=( + self.weight_loader_v2 + if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED + else self.weight_loader + ), + ) + + if bias: + self.bias = Parameter(torch.empty(self.output_size, dtype=params_dtype)) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + input_dim = getattr(param, "input_dim", None) + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False) + + # Special case for GGUF + is_gguf_weight = getattr(param, "is_gguf_weight", False) + is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False) + if is_gguf_weight_type: + param.weight_type = loaded_weight.item() + + # Materialize GGUF UninitializedParameter + if is_gguf_weight and isinstance(param, UninitializedParameter): + weight_shape = list(loaded_weight.shape) + if input_dim: + weight_shape[input_dim] = weight_shape[input_dim] // self.tp_size + param.materialize(tuple(weight_shape), dtype=loaded_weight.dtype) + + param_data = param.data + # bitsandbytes loads the weights of the specific portion + # no need to narrow here + if ( + input_dim is not None + and not use_bitsandbytes_4bit + and not self.use_presharded_weights + ): + shard_size = param_data.shape[input_dim] + start_idx = self.tp_rank * shard_size + + if _is_cpu: + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + start_idx, + input_dim, + shard_size, + ) + else: + loaded_weight = loaded_weight.narrow(input_dim, start_idx, shard_size) + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def weight_loader_v2(self, param: BasevLLMParameter, loaded_weight: torch.Tensor): + + # Special case for loading scales off disk, which often do not + # have a shape (such as in the case of AutoFP8). + if len(loaded_weight.shape) == 0: + assert loaded_weight.numel() == 1 + loaded_weight = loaded_weight.reshape(1) + + if isinstance(param, RowvLLMParameter): + # This `BasevLLMParameter` is defined in sglang/srt/layers/parameter.py, + # It supports additional parameters like tp_rank and use_presharded_weights. + param.load_row_parallel_weight( + loaded_weight, + tp_rank=self.tp_rank, + use_presharded_weights=self.use_presharded_weights, + ) + else: + # `params` is defined in `vllm/model_executor/parameter.py`, + # It does not support additional parameters. + param.load_row_parallel_weight(loaded_weight) + + def forward(self, input_, skip_all_reduce=False): + if self.input_is_parallel: + input_parallel = input_ + else: + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.tp_size + ) + input_parallel = splitted_input[self.tp_rank].contiguous() + + # Matrix multiply. + assert self.quant_method is not None + # Only fuse bias add into GEMM for rank 0 (this ensures that + # bias will not get added more than once in TP>1 case) + bias_ = None if (self.tp_rank > 0 or self.skip_bias_add) else self.bias + with use_symmetric_memory(parallel_state.get_tp_group()) as sm: + output_parallel = self.quant_method.apply(self, input_parallel, bias=bias_) + sm.tag(output_parallel) + + if self.reduce_results and self.tp_size > 1 and not skip_all_reduce: + output = tensor_model_parallel_all_reduce(output_parallel) + else: + output = output_parallel + + output_bias = self.bias if self.skip_bias_add else None + + return output, output_bias + + def extra_repr(self) -> str: + s = f"input_features={self.input_size_per_partition}" + s += f", output_features={self.output_size}" + s += f", bias={self.bias is not None}" + s += f", tp_size={self.tp_size}" + s += f", reduce_results={self.reduce_results}" + return s diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py new file mode 100644 index 000000000..f6603907a --- /dev/null +++ b/python/sglang/srt/layers/logits_processor.py @@ -0,0 +1,652 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Logits processing.""" + +import dataclasses +import logging +from typing import List, Optional, Union + +import torch +import triton +import triton.language as tl +from torch import nn + +from sglang.srt.distributed import ( + get_tensor_model_parallel_world_size, + tensor_model_parallel_all_gather, +) +from sglang.srt.layers.dp_attention import ( + DpPaddingMode, + attn_tp_all_gather, + attn_tp_all_gather_into_tensor, + dp_gather_replicate, + dp_scatter, + get_attention_dp_rank, + get_attention_dp_size, + get_attention_tp_size, + get_global_dp_buffer, + get_local_attention_dp_size, + set_dp_buffer_len, +) +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, +) +from sglang.srt.utils import dump_to_file, is_npu, use_intel_amx_backend + +logger = logging.getLogger(__name__) + +_is_npu = is_npu() + + +@dataclasses.dataclass +class LogitsProcessorOutput: + ## Part 1: This part will be assigned in python/sglang/srt/layers/logits_processor.py::LogitsProcessor + # The logits of the next tokens. shape: [#seq, vocab_size] + next_token_logits: torch.Tensor + # Used by speculative decoding (EAGLE) + # The last hidden layers + hidden_states: Optional[torch.Tensor] = None + + ## Part 2: This part will be assigned in python/sglang/srt/layers/sampler.py::Sampler + # he log probs of output tokens, if RETURN_ORIGINAL_LOGPROB = True, will get the log probs before applying temperature. If False, will get the log probs before applying temperature. + next_token_logprobs: Optional[torch.Tensor] = None + # The logprobs and ids of the top-k tokens in output positions. shape: [#seq, k] + next_token_top_logprobs_val: Optional[List] = None + next_token_top_logprobs_idx: Optional[List] = None + # The logprobs and ids of the requested token ids in output positions. shape: [#seq, n] (n is the number of requested token ids) + next_token_token_ids_logprobs_val: Optional[List] = None + next_token_token_ids_logprobs_idx: Optional[List] = None + + ## Part 3: Prefill-only. This part will be assigned in python/sglang/srt/layers/logits_processor.py::LogitsProcessor + # The logprobs of input tokens. shape: [#token] + input_token_logprobs: Optional[torch.Tensor] = None + # The logprobs and ids of the top-k tokens in input positions. shape: [#seq, #token, k] + input_top_logprobs_val: List = None + input_top_logprobs_idx: List = None + # The logprobs and ids of the requested token ids in input positions. shape: [#seq, n] (n is the number of requested token ids) + input_token_ids_logprobs_val: Optional[List] = None + input_token_ids_logprobs_idx: Optional[List] = None + + +@dataclasses.dataclass +class LogitsMetadata: + forward_mode: ForwardMode + capture_hidden_mode: CaptureHiddenMode = CaptureHiddenMode.NULL + next_token_logits_buffer: Optional[torch.Tensor] = None + + extend_return_logprob: bool = False + extend_return_top_logprob: bool = False + extend_token_ids_logprob: bool = False + extend_seq_lens: Optional[torch.Tensor] = None + extend_seq_lens_cpu: Optional[List[int]] = None + extend_logprob_start_lens_cpu: Optional[List[int]] = None + extend_logprob_pruned_lens_cpu: Optional[List[int]] = None + top_logprobs_nums: Optional[List[int]] = None + extend_input_logprob_token_ids_gpu: Optional[torch.Tensor] = None + token_ids_logprobs: Optional[List[List[int]]] = None + + # logits and logprobs post processing + temp_scaled_logprobs: bool = False + temperature: torch.Tensor = None + top_p_normalized_logprobs: bool = False + top_p: torch.Tensor = None + + # DP attention metadata. Not needed when DP attention is not used. + # Number of tokens in the request. + global_num_tokens_gpu: Optional[torch.Tensor] = None + # The start position of local hidden states. + dp_local_start_pos: Optional[torch.Tensor] = None + dp_local_num_tokens: Optional[torch.Tensor] = None + global_dp_buffer_len: Optional[int] = None + # Number of tokens to sample per DP rank + global_num_tokens_for_logprob_cpu: Optional[torch.Tensor] = None + global_num_tokens_for_logprob_gpu: Optional[torch.Tensor] = None + # The gather mode for DP attention + dp_padding_mode: Optional[DpPaddingMode] = None + # for padding + padded_static_len: int = -1 + + @classmethod + def from_forward_batch(cls, forward_batch: ForwardBatch): + if ( + forward_batch.forward_mode.is_extend() + and forward_batch.return_logprob + and not forward_batch.forward_mode.is_target_verify() + ): + extend_return_top_logprob = any( + x > 0 for x in forward_batch.top_logprobs_nums + ) + extend_token_ids_logprob = any( + x is not None for x in forward_batch.token_ids_logprobs + ) + extend_return_logprob = False + extend_logprob_pruned_lens_cpu = [] + for extend_len, start_len in zip( + forward_batch.extend_seq_lens_cpu, + forward_batch.extend_logprob_start_lens_cpu, + ): + if extend_len - start_len > 0: + extend_return_logprob = True + extend_logprob_pruned_lens_cpu.append(extend_len - start_len) + else: + extend_return_logprob = extend_return_top_logprob = ( + extend_token_ids_logprob + ) = extend_logprob_pruned_lens_cpu = False + + return cls( + forward_mode=forward_batch.forward_mode, + capture_hidden_mode=forward_batch.capture_hidden_mode, + next_token_logits_buffer=forward_batch.next_token_logits_buffer, + extend_return_logprob=extend_return_logprob, + extend_return_top_logprob=extend_return_top_logprob, + extend_token_ids_logprob=extend_token_ids_logprob, + extend_seq_lens=forward_batch.extend_seq_lens, + extend_seq_lens_cpu=forward_batch.extend_seq_lens_cpu, + extend_logprob_start_lens_cpu=forward_batch.extend_logprob_start_lens_cpu, + extend_logprob_pruned_lens_cpu=extend_logprob_pruned_lens_cpu, + top_logprobs_nums=forward_batch.top_logprobs_nums, + token_ids_logprobs=forward_batch.token_ids_logprobs, + extend_input_logprob_token_ids_gpu=forward_batch.extend_input_logprob_token_ids_gpu, + padded_static_len=forward_batch.padded_static_len, + global_num_tokens_gpu=forward_batch.global_num_tokens_gpu, + dp_local_start_pos=forward_batch.dp_local_start_pos, + dp_local_num_tokens=forward_batch.dp_local_num_tokens, + global_dp_buffer_len=forward_batch.global_dp_buffer_len, + global_num_tokens_for_logprob_cpu=forward_batch.global_num_tokens_for_logprob_cpu, + global_num_tokens_for_logprob_gpu=forward_batch.global_num_tokens_for_logprob_gpu, + dp_padding_mode=DpPaddingMode.SUM_LEN, + ) + + def compute_dp_attention_metadata(self): + + cumtokens = torch.cumsum(self.global_num_tokens_for_logprob_gpu, dim=0) + dp_rank = get_attention_dp_rank() + if dp_rank == 0: + dp_local_start_pos = torch.zeros_like( + self.global_num_tokens_for_logprob_gpu[0] + ) + else: + dp_local_start_pos = cumtokens[dp_rank - 1] + dp_local_num_tokens = self.global_num_tokens_for_logprob_gpu[dp_rank] + + self.dp_local_start_pos = dp_local_start_pos + self.dp_local_num_tokens = dp_local_num_tokens + + if self.global_num_tokens_for_logprob_cpu is not None: + # create a smaller buffer to reduce peak memory usage + self.global_dp_buffer_len = sum(self.global_num_tokens_for_logprob_cpu) + else: + self.global_dp_buffer_len = self.global_dp_buffer_len + + set_dp_buffer_len( + self.global_dp_buffer_len, + self.dp_local_num_tokens, + self.global_num_tokens_for_logprob_cpu, + ) + + +class LogitsProcessor(nn.Module): + def __init__( + self, config, skip_all_gather: bool = False, logit_scale: Optional[float] = None + ): + super().__init__() + self.config = config + self.logit_scale = logit_scale + self.use_attn_tp_group = global_server_args_dict["enable_dp_lm_head"] + if self.use_attn_tp_group: + self.attn_tp_size = get_attention_tp_size() + self.do_tensor_parallel_all_gather = ( + not skip_all_gather and self.attn_tp_size > 1 + ) + self.do_tensor_parallel_all_gather_dp_attn = False + else: + self.do_tensor_parallel_all_gather = ( + not skip_all_gather and get_tensor_model_parallel_world_size() > 1 + ) + self.do_tensor_parallel_all_gather_dp_attn = ( + self.do_tensor_parallel_all_gather and get_attention_dp_size() != 1 + ) + self.final_logit_softcapping = getattr( + self.config, "final_logit_softcapping", None + ) + if ( + self.final_logit_softcapping is not None + and self.final_logit_softcapping < 0 + ): + self.final_logit_softcapping = None + + self.debug_tensor_dump_output_folder = global_server_args_dict.get( + "debug_tensor_dump_output_folder", None + ) + + def forward( + self, + input_ids, + hidden_states, + lm_head: VocabParallelEmbedding, + logits_metadata: Union[LogitsMetadata, ForwardBatch], + aux_hidden_states: Optional[torch.Tensor] = None, + ) -> LogitsProcessorOutput: + if isinstance(logits_metadata, ForwardBatch): + logits_metadata = LogitsMetadata.from_forward_batch(logits_metadata) + # Get the last hidden states and last logits for the next token prediction + if ( + logits_metadata.forward_mode.is_decode_or_idle() + or logits_metadata.forward_mode.is_target_verify() + ): + pruned_states = hidden_states + if aux_hidden_states is not None: + aux_pruned_states = [hidden for hidden in aux_hidden_states] + sample_indices = None + input_logprob_indices = None + elif ( + logits_metadata.forward_mode.is_extend() + and not logits_metadata.extend_return_logprob + ): + # Prefill without input logprobs. + if logits_metadata.padded_static_len < 0: + last_index = torch.cumsum(logits_metadata.extend_seq_lens, dim=0) - 1 + else: + # If padding_static length is 5 and extended_seq_lens is [2, 3], + # then our batch looks like [t00, t01, p, p, p, t10, t11, t12, p, p] + # and this retrieves t01 and t12, which are the valid last tokens + idx = torch.arange( + len(logits_metadata.extend_seq_lens), + device=logits_metadata.extend_seq_lens.device, + ) + last_index = ( + idx * logits_metadata.padded_static_len + + logits_metadata.extend_seq_lens + - 1 + ) + pruned_states = hidden_states[last_index] + if aux_hidden_states is not None: + aux_pruned_states = [hidden[last_index] for hidden in aux_hidden_states] + sample_indices = None + input_logprob_indices = None + else: + # Input logprobs are required. + # Find 3 different indices. + # 1. pruned_states: hidden states that we want logprobs from. + # 2. sample_indices: Indices that have sampled tokens. + # 3. input_logprob_indices: Indices that have input logprob tokens. + sample_index_pt = -1 + sample_indices = [] + input_logprob_indices_pt = 0 + input_logprob_indices = [] + pt, pruned_states = 0, [] + for extend_logprob_start_len, extend_len in zip( + logits_metadata.extend_logprob_start_lens_cpu, + logits_metadata.extend_seq_lens_cpu, + ): + # It can happen in chunked prefill. We still need to sample 1 token, + # But we don't want to include it in input logprob. + if extend_len == extend_logprob_start_len: + start_len = extend_logprob_start_len - 1 + else: + start_len = extend_logprob_start_len + + # We always need at least 1 token to sample because that's required + # by a caller. + assert extend_len > start_len + pruned_states.append(hidden_states[pt + start_len : pt + extend_len]) + pt += extend_len + sample_index_pt += extend_len - start_len + sample_indices.append(sample_index_pt) + input_logprob_indices.extend( + [ + input_logprob_indices_pt + i + for i in range(extend_len - extend_logprob_start_len) + ] + ) + input_logprob_indices_pt += extend_len - start_len + + pruned_states = torch.cat(pruned_states) + sample_indices = torch.tensor( + sample_indices, device=pruned_states.device, dtype=torch.int64 + ) + input_logprob_indices = torch.tensor( + input_logprob_indices, device=pruned_states.device, dtype=torch.int64 + ) + + # Compute logits for both input and sampled tokens. + logits = self._get_logits(pruned_states, lm_head, logits_metadata) + sampled_logits = ( + logits[sample_indices] if sample_indices is not None else logits + ) + + if self.debug_tensor_dump_output_folder: + assert ( + not self.do_tensor_parallel_all_gather + or get_local_attention_dp_size() == 1 + ), "dp attention + sharded lm_head doesn't support full logits" + full_logits = self._get_logits(hidden_states, lm_head, logits_metadata) + dump_to_file(self.debug_tensor_dump_output_folder, "logits", full_logits) + + hidden_states_to_store: Optional[torch.Tensor] = None + if logits_metadata.capture_hidden_mode.need_capture(): + if logits_metadata.capture_hidden_mode.is_full(): + if aux_hidden_states is not None: + aux_hidden_states = torch.cat(aux_hidden_states, dim=-1) + hidden_states_to_store = aux_hidden_states + else: + hidden_states_to_store = hidden_states + elif logits_metadata.capture_hidden_mode.is_last(): + # Get the last token hidden states. If sample_indices is None, + # pruned states only contain the last tokens already. + if aux_hidden_states is not None: + aux_pruned_states = torch.cat(aux_pruned_states, dim=-1) + hidden_states_to_store = ( + aux_pruned_states[sample_indices] + if sample_indices is not None + else aux_pruned_states + ) + else: + hidden_states_to_store = ( + pruned_states[sample_indices] + if sample_indices is not None + else pruned_states + ) + else: + assert False, "Should never reach" + + if not logits_metadata.extend_return_logprob: + # Decode mode or extend mode without return_logprob. + return LogitsProcessorOutput( + next_token_logits=sampled_logits, + hidden_states=hidden_states_to_store, + ) + else: + input_logprobs = logits[input_logprob_indices] + del hidden_states, logits + + # Normalize the logprob w/o temperature, top-p + pruned_lens = torch.tensor( + logits_metadata.extend_logprob_pruned_lens_cpu, + device=input_logprobs.device, + ) + if logits_metadata.temp_scaled_logprobs: + logits_metadata.temperature = torch.repeat_interleave( + logits_metadata.temperature.view(-1), + pruned_lens, + ).view(-1, 1) + if logits_metadata.top_p_normalized_logprobs: + logits_metadata.top_p = torch.repeat_interleave( + logits_metadata.top_p, + pruned_lens, + ) + input_logprobs = self.compute_temp_top_p_normalized_logprobs( + input_logprobs, logits_metadata + ) + + # Get the logprob of top-k tokens + if logits_metadata.extend_return_top_logprob: + ( + input_top_logprobs_val, + input_top_logprobs_idx, + ) = self.get_top_logprobs(input_logprobs, logits_metadata) + else: + input_top_logprobs_val = input_top_logprobs_idx = None + + # Get the logprob of given token id + if logits_metadata.extend_token_ids_logprob: + ( + input_token_ids_logprobs_val, + input_token_ids_logprobs_idx, + ) = self.get_token_ids_logprobs(input_logprobs, logits_metadata) + else: + input_token_ids_logprobs_val = input_token_ids_logprobs_idx = None + + input_token_logprobs = input_logprobs[ + torch.arange(input_logprobs.shape[0], device=input_logprobs.device), + logits_metadata.extend_input_logprob_token_ids_gpu, + ] + + return LogitsProcessorOutput( + next_token_logits=sampled_logits, + input_token_logprobs=input_token_logprobs, + input_top_logprobs_val=input_top_logprobs_val, + input_top_logprobs_idx=input_top_logprobs_idx, + hidden_states=hidden_states_to_store, + input_token_ids_logprobs_val=input_token_ids_logprobs_val, + input_token_ids_logprobs_idx=input_token_ids_logprobs_idx, + ) + + def _get_logits( + self, + hidden_states: torch.Tensor, + lm_head: VocabParallelEmbedding, + logits_metadata: LogitsMetadata, + embedding_bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Get logits from hidden_states. + + If sampled_logits_only is True, it means hidden_states only contain the + last position (e.g., extend without input logprobs). The caller should + guarantee the given hidden_states follow this constraint. + """ + if self.do_tensor_parallel_all_gather_dp_attn: + logits_metadata.compute_dp_attention_metadata() + hidden_states, local_hidden_states = ( + get_global_dp_buffer(), + hidden_states, + ) + dp_gather_replicate(hidden_states, local_hidden_states, logits_metadata) + + if hasattr(lm_head, "weight"): + if use_intel_amx_backend(lm_head): + logits = torch.ops.sgl_kernel.weight_packed_linear( + hidden_states.to(lm_head.weight.dtype), + lm_head.weight, + None, # bias + True, # is_vnni + ) + else: + logits = torch.matmul( + hidden_states.to(lm_head.weight.dtype), lm_head.weight.T + ) + else: + # GGUF models + # TODO: use weight_packed_linear for GGUF models + logits = lm_head.quant_method.apply(lm_head, hidden_states, embedding_bias) + + if self.logit_scale is not None: + logits.mul_(self.logit_scale) + + if self.do_tensor_parallel_all_gather: + if self.use_attn_tp_group: + if self.config.vocab_size % self.attn_tp_size == 0: + global_logits = torch.empty( + ( + self.attn_tp_size, + logits.shape[0], + self.config.vocab_size // self.attn_tp_size, + ), + device=logits.device, + dtype=logits.dtype, + ) + attn_tp_all_gather_into_tensor(global_logits, logits) + global_logits = global_logits.permute(1, 0, 2).reshape( + logits.shape[0], self.config.vocab_size + ) + else: + global_logits = torch.empty( + (self.config.vocab_size, logits.shape[0]), + device=logits.device, + dtype=logits.dtype, + ) + global_logits = global_logits.T + attn_tp_all_gather( + list(global_logits.tensor_split(self.attn_tp_size, dim=-1)), + logits, + ) + logits = global_logits + else: + logits = tensor_model_parallel_all_gather(logits) + + if self.do_tensor_parallel_all_gather_dp_attn: + logits, global_logits = ( + torch.empty( + (local_hidden_states.shape[0], logits.shape[1]), + device=logits.device, + dtype=logits.dtype, + ), + logits, + ) + dp_scatter(logits, global_logits, logits_metadata) + + if logits_metadata.next_token_logits_buffer is not None: + logits_buffer = logits_metadata.next_token_logits_buffer + assert logits_buffer.dtype == torch.float + logits_buffer.copy_(logits[:, : self.config.vocab_size]) + logits = logits_buffer + else: + logits = logits[:, : self.config.vocab_size].float() + + if self.final_logit_softcapping: + if not _is_npu: + fused_softcap(logits, self.final_logit_softcapping) + else: + logits = self.final_logit_softcapping * torch.tanh( + logits / self.final_logit_softcapping + ) + + return logits + + @staticmethod + def get_top_logprobs(all_logprobs: torch.Tensor, logits_metadata: LogitsMetadata): + max_k = max(logits_metadata.top_logprobs_nums) + ret = all_logprobs.topk(max_k, dim=1) + values = ret.values.tolist() + indices = ret.indices.tolist() + + input_top_logprobs_val, input_top_logprobs_idx = [], [] + + pt = 0 + for k, pruned_len in zip( + logits_metadata.top_logprobs_nums, + logits_metadata.extend_logprob_pruned_lens_cpu, + ): + if pruned_len <= 0: + input_top_logprobs_val.append([]) + input_top_logprobs_idx.append([]) + continue + + input_top_logprobs_val.append( + [values[pt + j][:k] for j in range(pruned_len)] + ) + input_top_logprobs_idx.append( + [indices[pt + j][:k] for j in range(pruned_len)] + ) + pt += pruned_len + + return input_top_logprobs_val, input_top_logprobs_idx + + @staticmethod + def get_token_ids_logprobs( + all_logprobs: torch.Tensor, logits_metadata: LogitsMetadata + ): + input_token_ids_logprobs_val, input_token_ids_logprobs_idx = [], [] + pt = 0 + for token_ids, pruned_len in zip( + logits_metadata.token_ids_logprobs, + logits_metadata.extend_logprob_pruned_lens_cpu, + ): + if pruned_len <= 0: + input_token_ids_logprobs_val.append([]) + input_token_ids_logprobs_idx.append([]) + continue + + input_token_ids_logprobs_val.append( + [all_logprobs[pt + j, token_ids].tolist() for j in range(pruned_len)] + ) + input_token_ids_logprobs_idx.append([token_ids for _ in range(pruned_len)]) + pt += pruned_len + + return input_token_ids_logprobs_val, input_token_ids_logprobs_idx + + @staticmethod + def compute_temp_top_p_normalized_logprobs( + last_logits: torch.Tensor, logits_metadata: LogitsMetadata + ) -> torch.Tensor: + """ + compute logprobs for the output token from the given logits. + + Returns: + torch.Tensor: logprobs from logits + """ + # Scale logits if temperature scaling is enabled + if logits_metadata.temp_scaled_logprobs: + last_logits = last_logits / logits_metadata.temperature + + # Normalize logprobs if top_p normalization is enabled + # NOTE: only normalize logprobs when top_p is set and not equal to 1.0 + if ( + logits_metadata.top_p_normalized_logprobs + and (logits_metadata.top_p != 1.0).any() + ): + from sglang.srt.layers.sampler import top_p_normalize_probs_torch + + probs = torch.softmax(last_logits, dim=-1) + del last_logits + probs = top_p_normalize_probs_torch(probs, logits_metadata.top_p) + return torch.log(probs) + else: + return torch.nn.functional.log_softmax(last_logits, dim=-1) + + +@triton.jit +def fused_softcap_kernel( + full_logits_ptr, + softcapping_value, + n_elements, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + block_start = pid * BLOCK_SIZE + offsets = block_start + tl.arange(0, BLOCK_SIZE) + mask = offsets < n_elements + + # Load values + x = tl.load(full_logits_ptr + offsets, mask=mask) + + # Perform operations in-place + x = x / softcapping_value + + # Manual tanh implementation using exp + exp2x = tl.exp(2 * x) + x = (exp2x - 1) / (exp2x + 1) + + x = x * softcapping_value + + # Store result + tl.store(full_logits_ptr + offsets, x, mask=mask) + + +def fused_softcap(full_logits, final_logit_softcapping): + n_elements = full_logits.numel() + BLOCK_SIZE = 1024 + grid = ((n_elements + BLOCK_SIZE - 1) // BLOCK_SIZE, 1, 1) + + fused_softcap_kernel[grid]( + full_logits_ptr=full_logits, + softcapping_value=final_logit_softcapping, + n_elements=n_elements, + BLOCK_SIZE=BLOCK_SIZE, + ) + return full_logits diff --git a/python/sglang/srt/layers/model_parallel.py b/python/sglang/srt/layers/model_parallel.py new file mode 100644 index 000000000..53c8b622e --- /dev/null +++ b/python/sglang/srt/layers/model_parallel.py @@ -0,0 +1,155 @@ +""" +Common utilities for torch model parallelism. +""" + +from typing import Optional, Sequence + +import torch +import torch.nn as nn +from torch.distributed.device_mesh import DeviceMesh + +try: + import torch.distributed.tensor as dt +except ImportError: + # torch 2.4 or older + import torch.distributed._tensor as dt + +from torch.distributed.tensor.parallel import ( + ColwiseParallel, + RowwiseParallel, + parallelize_module, +) + + +def _shard_tensor( + full_tensor: torch.Tensor, + device_mesh: DeviceMesh, + placements: Sequence[dt.Shard], +) -> "dt.DTensor": + """ + Locally shards a full tensor based on indicated sharding arrangement, and + returns a DTensor containing the local shard. + + .. warning:: This is a private API that is subject to change. It skips the + communication otherwise required by `distribute_tensor`. It is only + applicable to cases where all ranks have the same `full_tensor`. For + example, in distributed inference all ranks load from the same + checkpoint. This API will not check for data equality between ranks, it + is thus user's responsibility to ensure the `full_tensor` is the same + across ranks. + + Args: + full_tensor (torch.Tensor): the full tensor to be sharded. + device_mesh (:class:`DeviceMesh`): DeviceMesh to place the + DTensor. Must have same dimension as the number of placements. + placements (Sequence[:class:`Shard`]): the placements that + describes how to place the local tensor on DeviceMesh. + + Returns: + A :class:`DTensor` object with the shard as its local tensor. + + Examples: + >>> # xdoctest: +SKIP("need world_size and rank") + >>> device_mesh = dist.init_device_mesh("cuda", (world_size,)) + >>> full_tensor = torch.arange(world_size, device=f"cuda:{rank}") + >>> dtensor = _shard_tensor(full_tensor, device_mesh, [Shard(1)]) + """ + shape, offset = dt._utils.compute_local_shape_and_global_offset( + full_tensor.shape, device_mesh, placements + ) + slices = [ + slice(cur_offset, cur_offset + cur_shape) + for cur_shape, cur_offset in zip(shape, offset) + ] + local_tensor = full_tensor[slices] + return dt.DTensor.from_local(local_tensor, device_mesh, placements) + + +class ColwiseParallelSharded(ColwiseParallel): + """ + A version of ColwiseParallel where the local weight has been already + sharded. This is used for the fused wqkv case, where during loading, we + already sharded wq, wk, wv before fusing them. + """ + + # Override the _partition_linear_fn in ColwiseParallel + def _partition_linear_fn(self, name, module, device_mesh): + # colwise shard weight/bias to Shard(0), weight be Shard(0) + # means Colwise as Linear is input * weight^T + bias, where + # weight would become Shard(1) + for name, param in module.named_parameters(): + dtensor = dt.DTensor.from_local(param, device_mesh, [dt.Shard(0)]) + dist_param = torch.nn.Parameter(dtensor, requires_grad=False) + module.register_parameter(name, dist_param) + + +class RowwiseParallelMaybeWait(RowwiseParallel): + """ + A version of RowwiseParallel that waits for the output (establish dependency + between comm stream and compute stream in CUDA sense) before going into the + next op. This is needed to workaround the current interaction between + AsyncCollectiveTensor and custom ops, such as `class RMSNorm(CustomOp)`. + """ + + def _partition_linear_fn(self, name, module, device_mesh): + # Rowwise shard weight to Shard(1), bias to Replicate(), weight be Shard(1) + # means Rowwise as nn.Linear is input * weight^T + bias, where + # weight would become Shard(0) + module.register_parameter( + "weight", + nn.Parameter(_shard_tensor(module.weight, device_mesh, [dt.Shard(1)])), + ) + if getattr(module, "bias", None) is not None: + # The Linear module has bias + module.register_parameter( + "bias", + nn.Parameter( + dt.distribute_tensor(module.bias, device_mesh, [dt.Replicate()]) + ), + ) + + @staticmethod + def _prepare_output_fn(output_layouts, use_local_output, mod, outputs, device_mesh): + outputs = super( + RowwiseParallelMaybeWait, RowwiseParallelMaybeWait + )._prepare_output_fn( + output_layouts, use_local_output, mod, outputs, device_mesh + ) + return torch.distributed._functional_collectives.wait_tensor(outputs) + + +def tensor_parallel( + module: torch.nn.Module, + device_mesh: Optional[DeviceMesh] = None, +): + """ + Tensor parallelize the model across the given device mesh. + Args: + module (`torch.nn.Module`): + The module to tensor parallelize. + device_mesh (`torch.distributed.DeviceMesh`): + The device mesh to use for tensor parallelism. + """ + + # Tensor parallelize a nn.Module based on the `_tp_plan` attribute of the module. + # No op if `_tp_plan` attribute does not exist under the module. + # This is a helper function to be used with `model.apply` to recursively + # parallelize a model. + def tplize(mod: torch.nn.Module) -> None: + tp_plan = getattr(mod, "_tp_plan", None) + if tp_plan is None: + return + for child_name, tp_style in tp_plan.items(): + submod = mod.get_submodule(child_name) + if tp_style == "Colwise": + parallelize_module(submod, device_mesh, ColwiseParallel()) + elif tp_style == "Rowwise": + parallelize_module(submod, device_mesh, RowwiseParallelMaybeWait()) + elif tp_style == "Colwise_Sharded": + parallelize_module(submod, device_mesh, ColwiseParallelSharded()) + else: + raise ValueError(f"Unknown TP style {tp_style}") + + # `apply` is a native method of `nn.Module` that recursively applies a + # function to every submodule. + module.apply(tplize) diff --git a/python/sglang/srt/layers/moe/__init__.py b/python/sglang/srt/layers/moe/__init__.py new file mode 100644 index 000000000..5c75a3682 --- /dev/null +++ b/python/sglang/srt/layers/moe/__init__.py @@ -0,0 +1,32 @@ +from sglang.srt.layers.moe.moe_runner import MoeRunner, MoeRunnerConfig +from sglang.srt.layers.moe.utils import ( + DeepEPMode, + MoeA2ABackend, + MoeRunnerBackend, + get_deepep_config, + get_deepep_mode, + get_moe_a2a_backend, + get_moe_runner_backend, + get_tbo_token_distribution_threshold, + initialize_moe_config, + is_tbo_enabled, + should_use_flashinfer_cutlass_moe_fp4_allgather, + should_use_flashinfer_trtllm_moe, +) + +__all__ = [ + "DeepEPMode", + "MoeA2ABackend", + "MoeRunner", + "MoeRunnerConfig", + "MoeRunnerBackend", + "initialize_moe_config", + "get_moe_a2a_backend", + "get_moe_runner_backend", + "get_deepep_mode", + "should_use_flashinfer_trtllm_moe", + "should_use_flashinfer_cutlass_moe_fp4_allgather", + "is_tbo_enabled", + "get_tbo_token_distribution_threshold", + "get_deepep_config", +] diff --git a/python/sglang/srt/layers/moe/cutlass_moe.py b/python/sglang/srt/layers/moe/cutlass_moe.py new file mode 100755 index 000000000..d0fb4e3ef --- /dev/null +++ b/python/sglang/srt/layers/moe/cutlass_moe.py @@ -0,0 +1,365 @@ +"""CUTLASS based Fused MoE kernels.""" + +import torch + +from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() +if _is_cuda: + from sgl_kernel import ( + apply_shuffle_mul_sum, + cutlass_fp4_group_mm, + fp8_blockwise_scaled_grouped_mm, + prepare_moe_input, + scaled_fp4_experts_quant, + shuffle_rows, + silu_and_mul, + ) + + +def cutlass_fused_experts_fp8( + a: torch.Tensor, + w1_q: torch.Tensor, + w2_q: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + a1_strides: torch.Tensor, + c1_strides: torch.Tensor, + a2_strides: torch.Tensor, + c2_strides: torch.Tensor, + workspace: torch.Tensor, + a_ptrs: torch.Tensor, + b_ptrs: torch.Tensor, + out_ptrs: torch.Tensor, + a_scales_ptrs: torch.Tensor, + b_scales_ptrs: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, + problem_sizes2: torch.Tensor, + use_fp8_blockscale: bool = True, +) -> torch.Tensor: + """Performs Fused MoE computation using CUTLASS-like kernels with FP8 weights and activations. + + This function implements a Mixture of Experts (MoE) layer with a SwiGLU/SiLU + activation, leveraging custom kernels likely derived from CUTLASS principles + for grouped matrix multiplication (`fp8_blockwise_scaled_grouped_mm`) and + data preparation (`prepare_moe_input`, `silu_and_mul`). + + It handles per-token routing, quantizes input activations to FP8 with + per-token scales, performs the expert computations using FP8 GEMMs with + pre-quantized FP8 weights (per-block scales), applies the SiLU activation, + and combines the results weighted by the router scores. + + Args: + a (torch.Tensor): Input activations. Shape: `(m, k)`, where `m` is the total + number of tokens and `k` is the hidden size. Expected dtype: `torch.half` + or `torch.bfloat16`. + w1_q (torch.Tensor): Pre-quantized FP8 weight tensor for the first GEMM + (up-projection part of SwiGLU). Expected shape: `(E, k, n*2)`, where + `E` is the number of experts, `k` is the hidden size, and `n*2` is the + intermediate size (`I`). Expected dtype: `torch.float8_e4m3fn`. + Note: This shape implies weights are stored as (num_experts, hidden_size, intermediate_size). + w2_q (torch.Tensor): Pre-quantized FP8 weight tensor for the second GEMM + (down-projection). Expected shape: `(E, n, k)`, where `n` is half the + intermediate size (`I // 2`). Expected dtype: `torch.float8_e4m3fn`. + Note: This shape implies weights are stored as (num_experts, intermediate_size // 2, hidden_size). + w1_scale (torch.Tensor): Scales corresponding to `w1_q` (per-block scales). + Shape: `(E, num_blocks_n, num_blocks_k)`. Dtype: `torch.float32`. + w2_scale (torch.Tensor): Scales corresponding to `w2_q` (per-block scales). + Shape: `(E, num_blocks_k, num_blocks_n)`. Dtype: `torch.float32`. + topk_weights (torch.Tensor): Router weights for the selected top-k experts + for each token. Shape: `(m, topk)`. Dtype should ideally match `a`. + topk_ids (torch.Tensor): Indices of the selected top-k experts for each token. + Shape: `(m, topk)`. Dtype: `torch.int32`. + a1_strides (torch.Tensor): Stride information for the first GEMM's 'a' input. + Passed directly to the underlying kernel. Expected shape `(E,)`, dtype `torch.int64`. + Note: Its exact usage within `fp8_blockwise_scaled_grouped_mm` needs clarification + as it's passed as both a_stride and b_stride in the first call. + c1_strides (torch.Tensor): Stride information for the first GEMM's 'c' output. + Passed directly to the underlying kernel. Expected shape `(E,)`, dtype `torch.int64`. + a2_strides (torch.Tensor): Stride information for the second GEMM's 'a' input. + Passed directly to the underlying kernel. Expected shape `(E,)`, dtype `torch.int64`. + Note: Its exact usage within `fp8_blockwise_scaled_grouped_mm` needs clarification + as it's passed as both a_stride and b_stride in the second call. + c2_strides (torch.Tensor): Stride information for the second GEMM's 'c' output. + Passed directly to the underlying kernel. Expected shape `(E,)`, dtype `torch.int64`. + workspace (torch.Tensor): Reusable workspace for the underlying kernel. + a_ptrs (torch.Tensor): Pointers container for calculating offsets of the input activations for each expert. + b_ptrs (torch.Tensor): Pointers container for calculating offsets of the input weights for each expert. + out_ptrs (torch.Tensor): Pointers container for calculating offsets of the output activations for each expert. + a_scales_ptrs (torch.Tensor): Pointers container for calculating offsets of the input scales for each expert. + b_scales_ptrs (torch.Tensor): Pointers container for calculating offsets of the input scales for each expert. + use_fp8_blockscale (bool, optional): Flag indicating usage of FP8 with + block scaling. Currently, only `True` is supported. Defaults to `True`. + + Returns: + torch.Tensor: The computed MoE layer output. Shape: `(m, k)`, dtype matches `a`. + + Raises: + AssertionError: If input shapes, dtypes, or flags are inconsistent or unsupported. + NotImplementedError: If CUDA is not available or `sgl_kernel` is not properly installed. + """ + assert use_fp8_blockscale, "Only support fp8 blockscale for now" + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert w1_q.dtype == torch.float8_e4m3fn + assert w2_q.dtype == torch.float8_e4m3fn + assert a.shape[1] == w1_q.shape[1], "Hidden size mismatch w1" + assert w1_q.shape[2] == w2_q.shape[1] * 2, "Hidden size mismatch w2" + assert w1_q.shape[0] == w2_q.shape[0], "Expert number mismatch" + assert w1_q.shape[0] == w2_q.shape[0], "Weights expert number mismatch" + assert w1_q.shape[0] == w1_scale.shape[0], "w1 scales expert number mismatch" + assert w1_q.shape[0] == w2_scale.shape[0], "w2 scales expert number mismatch" + assert a.dtype in [torch.half, torch.bfloat16], "Invalid output dtype" + + if is_cuda: + from sglang.srt.layers.quantization.fp8_kernel import ( + per_group_transpose, + per_token_group_quant_fp8_hopper_moe_mn_major, + sglang_per_token_group_quant_fp8, + ) + + out_dtype = a.dtype + num_experts = w1_q.size(0) + m = a.size(0) + k = w1_q.size(1) + n = w2_q.size(1) + + topk = topk_ids.size(1) + device = a.device + + a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + + prepare_moe_input( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + a_map, + c_map, + num_experts, + n, + k, + ) + + a_q, a1_scale = sglang_per_token_group_quant_fp8(a, 128) + rep_a_q = shuffle_rows(a_q, a_map, (m * topk, k)) + rep_a1_scales = shuffle_rows(a1_scale, a_map, (m * topk, int(k / 128))) + + c1 = torch.empty((m * topk, n * 2), device=device, dtype=out_dtype) + c2 = torch.empty((m * topk, k), device=device, dtype=out_dtype) + + a_sf_layout = torch.empty((num_experts, 5), device=device, dtype=torch.int) + w_sf_layout = torch.empty((num_experts, 5), device=device, dtype=torch.int) + + fp8_blockwise_scaled_grouped_mm( + c1, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + rep_a_q, + w1_q, + rep_a1_scales, + w1_scale, + a1_strides, + a1_strides, + c1_strides, + a_sf_layout, + w_sf_layout, + problem_sizes1, + expert_offsets[:-1], + workspace, + ) + + intermediate = torch.empty((m * topk, n), device=device, dtype=out_dtype) + silu_and_mul(c1, intermediate) + + intemediate_q, a2_scale = sglang_per_token_group_quant_fp8(intermediate, 128) + + fp8_blockwise_scaled_grouped_mm( + c2, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + intemediate_q, + w2_q, + a2_scale, + w2_scale, + a2_strides, + a2_strides, + c2_strides, + a_sf_layout, + w_sf_layout, + problem_sizes2, + expert_offsets[:-1], + workspace, + ) + + result = torch.empty((m, k), device=device, dtype=out_dtype) + apply_shuffle_mul_sum(c2, result, c_map, topk_weights.to(out_dtype)) + return result + + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = 448.0 + + +def cutlass_moe_fp4( + a: torch.Tensor, + a1_gscale: torch.Tensor, + w1_fp4: torch.Tensor, + w1_blockscale: torch.Tensor, + w1_alphas: torch.Tensor, + a2_gscale: torch.Tensor, + w2_fp4: torch.Tensor, + w2_blockscale: torch.Tensor, + w2_alphas: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + params: CutlassMoEParams, + apply_router_weight_on_input: bool = False, +): + """ + MoE implementation for FP4 Inputs + + # Gemm 1 + a: Input tensor: [m, k] (half/bfloat16) + a1_gscale: Activation scale per expert: [e] (float32) + w1(gate up) (not an argument to cutlass_moe_fp4): [e, 2 * n, k] + w1_fp4: [e, 2 * n, k // 2], dtype: torch.uint8 (stacked fp4: E2M1) + (Note: `n` is the up projection output dim, `k` is the input dim in + full precision) + w1_blockscale: [e, 2 * n, k // block_size] (float8_e4m3) + (Block size = 16 for NVFP4) + + # Gemm 2 + a2_gscale: Activation scale per expert: [e] + w2(down projection) (not an argument to cutlass_moe_fp4): [e, k, n] + w2_fp4: [e, k, n // 2], dtype: torch.uint8 (stacked E2M1) + w2_blockscale: [e, k, n // block_size], dtype: float8_e4m3 + + Strides for activations, weights and output in logical number of elements. + The activations & output stride is the number of elements to the next row. + The weights stride is the number of elements to the next row per expert. + For example, if the weight is [e, n, k], then the b_stride is a tensor of + shape [e] with each element being k. Similarly for activations, if the + shape is [m, k], then the a_stride has shape [e] with each value k. + Similarly for output, if the output is [m, n], then the c_stride is a + tensor of shape [e] with each element being k. + + Note: cutlass_fp4_group_mm is designed to accept the strides of + activations and weights to be the same, so it is passed in as a single + tensor. + ab_strides_13: [e] dtype: int64 [Gemm 1: Activation / Weight strides] + ab_strides_2: [e] dtype: int64 [Gemm 2: Activation / Weight strides] + c_strides_13: [e] dtype: int64 [Gemm 1: Output Strides] + c_strides_2: [e] dtype: int64 [Gemm 1: Output Strides] + + topk_weights: [m, topk] dtype: float8 + topk_ids: [m, topk] dtype: float8 + + m, n, k: Unquantized weight shapes, dtype: int + e: number of experts for the current rank, dtype: int + assumes that topk < k < n to satisfy - up/down projection expectations. + """ + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert w1_fp4.dtype == torch.uint8, "weight 1 must be uint8" + assert w2_fp4.dtype == torch.uint8, "weight 2 must be uint8" + assert ( + w1_fp4.ndim == 3 + and w2_fp4.ndim == 3 + and w1_blockscale.ndim == 3 + and w2_blockscale.ndim == 3 + ), "All Weights must be of rank 3 for cutlass_moe_fp4" + m_a, k_a = a.shape + e_w1, nx2_w1, half_k_w1 = w1_fp4.shape + e_w2, k_w2, half_n_w2 = w2_fp4.shape + + assert e_w1 == e_w2 and e_w1 == params.num_experts, ( + "Number of experts must match", + " between weights.", + ) + assert ( + k_a // 2 == half_k_w1 and params.hidden_size == k_w2 + ), "Hidden size mismatch between a, w1 and w2" + assert ( + nx2_w1 == params.intermediate_size_per_partition * 2 + and half_n_w2 == params.intermediate_size_per_partition // 2 + ), ("mismatch in " "expected `n`") + assert 2 * half_k_w1 == k_w2, "Hidden size mismatch w2 and w1" + assert a.dtype in [torch.half, torch.bfloat16], "Invalid input dtype" + + out_dtype = a.dtype + num_topk = topk_ids.shape[1] + device = a.device + a_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + c_map = torch.empty((topk_ids.numel()), dtype=torch.int32, device=device) + prepare_moe_input( + topk_ids, + params.expert_offsets, + params.problem_sizes1, + params.problem_sizes2, + a_map, + c_map, + params.num_experts, + params.intermediate_size_per_partition, + params.hidden_size, + params.blockscale_offsets, + ) + + rep_a_fp4, rep_a_blockscale = scaled_fp4_experts_quant( + a, + a1_gscale, + params.expert_offsets, + params.blockscale_offsets, + num_topk, + expert_map=a_map, + ) + c1 = cutlass_fp4_group_mm( + rep_a_fp4, + w1_fp4, + rep_a_blockscale, + w1_blockscale, + w1_alphas, + out_dtype, + device, + params.to_gemm1_args(), + ) + del rep_a_fp4, rep_a_blockscale + + # hidden size dimension is split to one halfpytho sized tensor. + intermediate = torch.empty( + (m_a * num_topk, w1_fp4.shape[1] // 2), device=device, dtype=out_dtype + ) + silu_and_mul(c1, intermediate) + + int_fp4, int_blockscale = scaled_fp4_experts_quant( + intermediate, + a2_gscale, + params.expert_offsets, + params.blockscale_offsets, + num_topk, + ) + c2 = cutlass_fp4_group_mm( + int_fp4, + w2_fp4, + int_blockscale, + w2_blockscale, + w2_alphas, + out_dtype, + device, + params.to_gemm2_args(), + ) + del int_fp4, int_blockscale + c2 = shuffle_rows(c2, c_map, (m_a * num_topk, params.hidden_size)) + c2 = c2.view(m_a, num_topk, params.hidden_size) + if not apply_router_weight_on_input: + c2 = c2 * topk_weights.view(m_a, num_topk, 1).to(out_dtype) + return c2.sum(dim=1).to(out_dtype) diff --git a/python/sglang/srt/layers/moe/cutlass_moe_params.py b/python/sglang/srt/layers/moe/cutlass_moe_params.py new file mode 100644 index 000000000..f3de60e04 --- /dev/null +++ b/python/sglang/srt/layers/moe/cutlass_moe_params.py @@ -0,0 +1,169 @@ +from dataclasses import dataclass +from enum import Enum, auto +from typing import Optional + +import torch + + +class CutlassMoEType(Enum): + """ + Enum for the different types of cutlass moe operations + that are currently supported in SGLang. + """ + + BlockscaledFP8 = auto() + BlockscaledFP4 = auto() + + +@dataclass +class CutlassMoEParams: + """ + Parameters for the cutlass moe operation. + """ + + # Type as defined above + cutlass_moe_type: CutlassMoEType + + # Strides for activations, weights and output in logical number of elements. + # The activations & output stride is the number of elements to the next row. + # The weights stride is the number of elements to the next row per expert. + # For example, if the weight is [e, n, k], then the b_stride is a tensor of + # shape [e] with each element being k. Similarly for activations, if the + # shape is [m, k], then the a_stride has shape [e] with each value k. + # Similarly for output, if the output is [m, n], then the c_stride is a + # tensor of shape [e] with each element being k. + + # Note: cutlass_fp4_group_mm is designed to accept the strides of + # activations and weights to be the same, so it is passed in as a single + # tensor. + # ab_strides_13: [e] dtype: int64 [Gemm 1: Activation / Weight strides] + # ab_strides_2: [e] dtype: int64 [Gemm 2: Activation / Weight strides] + # c_strides_13: [e] dtype: int64 [Gemm 1: Output Strides] + # c_strides_2: [e] dtype: int64 [Gemm 2: Output Strides] + ab_strides_13: torch.Tensor + ab_strides_2: torch.Tensor + c_strides_13: torch.Tensor + c_strides_2: torch.Tensor + + # m: Total number of tokens + # n: intermediate size per partition + # k: hidden size per expert + # e: Number of experts + # device: Device to run computation on and store tensors + m: int + intermediate_size_per_partition: int + hidden_size: int + num_experts: int + device: torch.device + + # Pointers container for calculating offsets of the input activations for each expert + # a_ptrs: [e] dtype: int64 + a_ptrs: torch.Tensor + + # Pointers container for calculating offsets of the input weights for each expert + # b_ptrs: [e] dtype: int64 + b_ptrs: torch.Tensor + + # Pointers container for calculating offsets of the output activations for each expert + # out_ptrs: [e] dtype: int64 + out_ptrs: torch.Tensor + # Pointers container for calculating offsets of the input scales for each expert + # a_scales_ptrs: [e] dtype: int64 + # b_scales_ptrs: [e] dtype: int64 + a_scales_ptrs: torch.Tensor + b_scales_ptrs: torch.Tensor + + # Offsets that mark at which token index each expert begins its computation + # The number of tokens computed with expert E is expert_offsets[E + 1] - expert_offsets[E] + # expert_offsets: [e+1] dtype: int32 + expert_offsets: torch.Tensor + + # Problem size: (num_experts, (m,2n,k)) for first GEMM + # problem_sizes1: [e, 3] dtype: int32 + # Problem size: (num_experts, (m,n,k)) for second GEMM + # problem_sizes2: [e, 3] dtype: int32 + problem_sizes1: torch.Tensor + problem_sizes2: torch.Tensor + # Similar to expert_offsets, but for blockscales for FP4 blockscaled Group GEMM + blockscale_offsets: Optional[torch.Tensor] = None + + def __init__( + self, + cutlass_moe_type: CutlassMoEType, + device: torch.device, + num_experts: int, + intermediate_size_per_partition: int, + hidden_size: int, + ): + self.cutlass_moe_type = cutlass_moe_type + self.device = device + self.num_experts = num_experts + self.intermediate_size_per_partition = intermediate_size_per_partition + self.hidden_size = hidden_size + self.n = self.intermediate_size_per_partition + self.k = self.hidden_size + self.e = self.num_experts + self.ab_strides_13 = torch.full( + (self.e,), self.k, dtype=torch.int64, device=self.device + ) + self.ab_strides_2 = torch.full( + (self.e,), self.n, dtype=torch.int64, device=self.device + ) + self.c_strides_13 = torch.full( + (self.e,), 2 * self.n, dtype=torch.int64, device=self.device + ) + self.c_strides_2 = torch.full( + (self.e,), self.k, dtype=torch.int64, device=self.device + ) + self.expert_offsets = torch.empty( + (self.e + 1,), dtype=torch.int32, device=self.device + ) + self.problem_sizes1 = torch.empty( + (self.e, 3), dtype=torch.int32, device=self.device + ) + self.problem_sizes2 = torch.empty( + (self.e, 3), dtype=torch.int32, device=self.device + ) + if self.cutlass_moe_type == CutlassMoEType.BlockscaledFP4: + self.blockscale_offsets = torch.empty( + (self.e + 1,), dtype=torch.int32, device=self.device + ) + else: + self.blockscale_offsets = None + self.a_ptrs = torch.empty((self.e,), dtype=torch.int64, device=self.device) + self.b_ptrs = torch.empty((self.e,), dtype=torch.int64, device=self.device) + self.out_ptrs = torch.empty((self.e,), dtype=torch.int64, device=self.device) + self.a_scales_ptrs = torch.empty( + (self.e,), dtype=torch.int64, device=self.device + ) + self.b_scales_ptrs = torch.empty( + (self.e,), dtype=torch.int64, device=self.device + ) + + def to_gemm1_args(self) -> dict: + return { + "ab_strides": self.ab_strides_13, + "c_strides": self.c_strides_13, + "problem_sizes": self.problem_sizes1, + "expert_offsets": self.expert_offsets[:-1], + "blockscale_offsets": self.blockscale_offsets[:-1], + # "a_ptrs": self.a_ptrs, + # "b_ptrs": self.b_ptrs, + # "out_ptrs": self.out_ptrs, + # "a_scales_ptrs": self.a_scales_ptrs, + # "b_scales_ptrs": self.b_scales_ptrs, + } + + def to_gemm2_args(self) -> dict: + return { + "ab_strides": self.ab_strides_2, + "c_strides": self.c_strides_2, + "problem_sizes": self.problem_sizes2, + "expert_offsets": self.expert_offsets[:-1], + "blockscale_offsets": self.blockscale_offsets[:-1], + # "a_ptrs": self.a_ptrs, + # "b_ptrs": self.b_ptrs, + # "out_ptrs": self.out_ptrs, + # "a_scales_ptrs": self.a_scales_ptrs, + # "b_scales_ptrs": self.b_scales_ptrs, + } diff --git a/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py new file mode 100644 index 000000000..216424eea --- /dev/null +++ b/python/sglang/srt/layers/moe/cutlass_w4a8_moe.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Cutlass W4A8 MoE kernel.""" +from typing import Optional + +import torch +from sgl_kernel import ( + cutlass_w4a8_moe_mm, + get_cutlass_w4a8_moe_mm_data, + sgl_per_tensor_quant_fp8, + silu_and_mul, +) + +from sglang.srt.layers.moe.ep_moe.kernels import ( + post_reorder_triton_kernel_for_cutlass_moe, + pre_reorder_triton_kernel_for_cutlass_moe, + run_cutlass_moe_ep_preproess, +) + + +def cutlass_w4a8_moe( + start_expert_id: int, + end_expert_id: int, + total_num_experts: int, + a: torch.Tensor, + w1_q: torch.Tensor, + w2_q: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids_: torch.Tensor, + local_topk_ids: torch.Tensor, + a_strides1: torch.Tensor, + b_strides1: torch.Tensor, + c_strides1: torch.Tensor, + a_strides2: torch.Tensor, + b_strides2: torch.Tensor, + c_strides2: torch.Tensor, + s_strides13: torch.Tensor, + s_strides2: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, + problem_sizes2: torch.Tensor, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, +) -> torch.Tensor: + """ + This function computes a w4a8-quantized Mixture of Experts (MoE) layer + using two sets of quantized weights, w1_q and w2_q, and top-k gating + mechanism. The matrix multiplications are implemented with CUTLASS + grouped gemm. + + Parameters: + - a (torch.Tensor): The input tensor to the MoE layer. + Shape: [M, K] + - w1_q (torch.Tensor): The first set of int4-quantized expert weights. + Shape: [num_experts, N * 2, K // 2] + (the weights are passed transposed and int4-packed) + - w2_q (torch.Tensor): The second set of int4-quantized expert weights. + Shape: [num_experts, K, N // 2] + (the weights are passed transposed and int4-packed) + - w1_scale (torch.Tensor): The fp32 scale to dequantize w1_q. + Shape: [num_experts, K // 512, N * 8] + - w2_scale (torch.Tensor): The fp32 scale to dequantize w2_q. + Shape: [num_experts, N // 512, K * 4] + - topk_weights (torch.Tensor): The weights of each token->expert mapping. + - a_strides1 (torch.Tensor): The input strides of the first grouped gemm. + - b_strides1 (torch.Tensor): The weights strides of the first grouped gemm. + - c_strides1 (torch.Tensor): The output strides of the first grouped gemm. + - a_strides2 (torch.Tensor): The input strides of the second grouped gemm. + - b_strides2 (torch.Tensor): The weights strides of the second grouped gemm. + - c_strides2 (torch.Tensor): The output strides of the second grouped gemm. + - s_strides13 (torch.Tensor): The input and scale strides of the first grouped gemm. + - s_strides2 (torch.Tensor): The scale strides of the second grouped gemm. + - a1_scale (Optional[torch.Tensor]): The optional fp32 scale to quantize a. + Shape: scalar or [1, K] + - a2_scale (Optional[torch.Tensor]): The optional fp32 scale to + quantize the intermediate result between the gemms. + Shape: scalar or [1, N] + - apply_router_weight_on_input (bool): When true, the topk weights are + applied directly on the inputs. This is only applicable when topk is 1. + + Returns: + - torch.Tensor: The fp8 output tensor after applying the MoE layer. + """ + assert topk_weights.shape == topk_ids_.shape, "topk shape mismatch" + assert w1_q.dtype == torch.int8 + assert w2_q.dtype == torch.int8 + assert a.shape[1] // 2 == w1_q.shape[2], "Hidden size mismatch w1" + assert w1_q.shape[2] * 2 == w2_q.shape[1], "Hidden size mismatch w2" + assert w1_q.shape[0] == w2_q.shape[0], "Expert number mismatch" + assert w1_q.shape[0] == w1_scale.shape[0], "w1 scales expert number mismatch" + assert w1_q.shape[0] == w2_scale.shape[0], "w2 scales expert number mismatch" + + assert a_strides1.shape[0] == w1_q.shape[0], "A Strides 1 expert number mismatch" + assert b_strides1.shape[0] == w1_q.shape[0], "B Strides 1 expert number mismatch" + assert a_strides2.shape[0] == w2_q.shape[0], "A Strides 2 expert number mismatch" + assert b_strides2.shape[0] == w2_q.shape[0], "B Strides 2 expert number mismatch" + num_experts = w1_q.size(0) + m = a.size(0) + k = w1_q.size(2) * 2 # w1_q is transposed and packed + n = w2_q.size(2) * 2 # w2_q is transposed and packed + topk = topk_ids_.size(1) + + if apply_router_weight_on_input: + assert topk == 1, "apply_router_weight_on_input is only implemented for topk=1" + + device = a.device + + _, src2dst, _ = run_cutlass_moe_ep_preproess( + local_topk_ids, + num_experts, + ) + + gateup_input = torch.empty( + (m * topk, k), + device=device, + dtype=torch.float8_e4m3fn, + ) + + pre_reorder_triton_kernel_for_cutlass_moe[(m,)]( + a, + gateup_input, + src2dst, + local_topk_ids, + a1_scale, + total_num_experts, + topk, + k, + BLOCK_SIZE=512, + ) + + # NOTE: a_map and c_map are not used in the get_cutlass_w4a8_moe_mm_data kernel, + # they are kept to allow for a quick switch of the permutation logic + # from the current triton kernel implementation to the cutlass-based one if needed. + a_map = torch.empty((local_topk_ids.numel()), dtype=torch.int32, device=device) + c_map = torch.empty((local_topk_ids.numel()), dtype=torch.int32, device=device) + get_cutlass_w4a8_moe_mm_data( + local_topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + a_map, + c_map, + num_experts, + n, + k, + ) + + c1 = torch.empty((m * topk, n * 2), device=device, dtype=torch.bfloat16) + c2 = torch.zeros((m * topk, k), device=device, dtype=torch.bfloat16) + + cutlass_w4a8_moe_mm( + c1, + gateup_input, + w1_q, + a1_scale.float(), + w1_scale, + expert_offsets[:-1], + problem_sizes1, + a_strides1, + b_strides1, + c_strides1, + s_strides13, + 128, + topk, + ) + + intermediate = torch.empty((m * topk, n), device=device, dtype=torch.bfloat16) + silu_and_mul(c1, intermediate) + + intermediate_q = torch.empty( + intermediate.shape, dtype=torch.float8_e4m3fn, device=device + ) + sgl_per_tensor_quant_fp8(intermediate, intermediate_q, a2_scale.float(), True) + + cutlass_w4a8_moe_mm( + c2, + intermediate_q, + w2_q, + a2_scale.float(), + w2_scale, + expert_offsets[:-1], + problem_sizes2, + a_strides2, + b_strides2, + c_strides2, + s_strides2, + 128, + topk, + ) + + output = torch.empty_like(a) + post_reorder_triton_kernel_for_cutlass_moe[(m,)]( + c2, + output, + src2dst, + local_topk_ids, + topk_weights, + num_experts, + topk, + k, + 0, + BLOCK_SIZE=512, + ) + return output diff --git a/python/sglang/srt/layers/moe/ep_moe/__init__.py b/python/sglang/srt/layers/moe/ep_moe/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/layers/moe/ep_moe/kernels.py b/python/sglang/srt/layers/moe/ep_moe/kernels.py new file mode 100644 index 000000000..08660812d --- /dev/null +++ b/python/sglang/srt/layers/moe/ep_moe/kernels.py @@ -0,0 +1,1438 @@ +import logging +from typing import List, Optional + +import torch +import triton + +from sglang.srt.layers.quantization.fp8_kernel import per_token_group_quant_fp8 +from sglang.srt.utils import ceil_div, dispose_tensor, is_cuda +from sglang.utils import is_in_ci + +logger = logging.getLogger(__name__) + +_is_cuda = is_cuda() +if _is_cuda: + from sglang.srt.layers.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8 as per_token_group_quant_fp8, + ) + +import triton.language as tl + + +@triton.jit +def deepep_permute_triton_kernel( + input_ptr, + gateup_input_ptr, + src2dst_ptr, + topk_ids_ptr, + a1_scales_ptr, + topk, + hidden_size, + BLOCK_SIZE: tl.constexpr, +): + OutDtype = gateup_input_ptr.dtype.element_ty + + src_idx = tl.program_id(0) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + + src_ptr = input_ptr + src_idx * hidden_size + + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + tl.arange(0, BLOCK_SIZE) + mask = offset < hidden_size + in_data = tl.load(src_ptr + offset, mask=mask).to(OutDtype) + + for idx in range(topk): + dst_idx = tl.load(src2dst_ptr + idx) + if dst_idx >= 0: + dst_ptr = gateup_input_ptr + dst_idx * hidden_size + tl.store(dst_ptr + offset, in_data, mask=mask) + + +@triton.jit +def deepep_post_reorder_triton_kernel( + down_output_ptr, + output_ptr, + src2dst_ptr, + topk_ids_ptr, + topk_weights_ptr, + topk, + hidden_size, + BLOCK_SIZE: tl.constexpr, +): + InDtype = down_output_ptr.dtype.element_ty + + src_idx = tl.program_id(0) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + topk_weights_ptr = topk_weights_ptr + src_idx * topk + + store_ptr = output_ptr + src_idx * hidden_size + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + tl.arange(0, BLOCK_SIZE) + mask = offset < hidden_size + sum_vec = tl.zeros([BLOCK_SIZE], dtype=InDtype) + for idx in range(topk): + dst_idx = tl.load(src2dst_ptr + idx) + if dst_idx >= 0: + weigh_scale = tl.load(topk_weights_ptr + idx).to(InDtype) + load_ptr = down_output_ptr + dst_idx * hidden_size + in_data = tl.load(load_ptr + offset, mask=mask) + sum_vec += in_data * weigh_scale + tl.store(store_ptr + offset, sum_vec, mask=mask) + + +@triton.jit +def compute_src2dst_triton_kernel( + reorder_ids, src2dst, num_toks, BLOCK_SIZE: tl.constexpr +): + pid = tl.program_id(axis=0) + dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = dst_id < num_toks + src_id = tl.load(reorder_ids + dst_id, mask=mask) + tl.store(src2dst + src_id, dst_id, mask=mask) + + +@triton.jit +def deepep_compute_src2dst_triton_kernel( + reorder_ids, src2dst, num_toks, num_minus_one, BLOCK_SIZE: tl.constexpr +): + pid = tl.program_id(axis=0) + dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = dst_id < num_toks + src_id = tl.load(reorder_ids + dst_id, mask=mask) + num_invalid = tl.load(num_minus_one) + tl.store(src2dst + src_id, dst_id - num_invalid, mask=mask) + + +def deepep_run_moe_deep_preprocess(topk_ids: torch.Tensor, num_experts: int): + reorder_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True) + seg_indptr = torch.empty(num_experts + 1, device=topk_ids.device, dtype=torch.int64) + src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int64) + + # Find offset + expert_ids = torch.arange( + num_experts + 1, device=topk_ids.device, dtype=reorder_topk_ids.dtype + ) + torch.searchsorted(reorder_topk_ids, expert_ids, out=seg_indptr) + num_minus_one = seg_indptr[0] + seg_indptr = seg_indptr - num_minus_one + + BLOCK_SIZE = 512 + grid = (triton.cdiv(topk_ids.numel(), BLOCK_SIZE),) + deepep_compute_src2dst_triton_kernel[grid]( + reorder_ids, src2dst, topk_ids.numel(), num_minus_one, BLOCK_SIZE + ) + reorder_topk_ids = reorder_topk_ids[num_minus_one:] + return reorder_topk_ids, src2dst, seg_indptr + + +@triton.jit +def compute_seg_indptr_triton_kernel(reorder_topk_ids, seg_indptr, num_toks): + expert = tl.program_id(0) + low = 0 + high = num_toks - 1 + target_location = -1 + while low <= high: + mid = (low + high) // 2 + + if tl.load(reorder_topk_ids + mid) > expert: + high = mid - 1 + else: + low = mid + 1 + target_location = mid + tl.store(seg_indptr + expert + 1, target_location + 1) + + +def run_moe_ep_preproess(topk_ids: torch.Tensor, num_experts: int): + reorder_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True) + + seg_indptr = torch.zeros(num_experts + 1, device=topk_ids.device, dtype=torch.int64) + src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32) + + compute_seg_indptr_triton_kernel[(num_experts,)]( + reorder_topk_ids, seg_indptr, topk_ids.numel() + ) + + BLOCK_SIZE = 512 + grid = (triton.cdiv(topk_ids.numel(), BLOCK_SIZE),) + compute_src2dst_triton_kernel[grid]( + reorder_ids, src2dst, topk_ids.numel(), BLOCK_SIZE + ) + + return reorder_topk_ids, src2dst, seg_indptr + + +def run_cutlass_moe_ep_preproess(local_topk_ids: torch.Tensor, local_num_experts: int): + reorder_topk_ids, reorder_ids = torch.sort(local_topk_ids.view(-1), stable=True) + + seg_indptr = torch.zeros( + local_num_experts + 1, device=local_topk_ids.device, dtype=torch.int64 + ) + src2dst = torch.empty( + local_topk_ids.numel(), device=local_topk_ids.device, dtype=torch.int32 + ) + + BLOCK_SIZE = 512 + grid = (triton.cdiv(local_topk_ids.numel(), BLOCK_SIZE),) + compute_src2dst_triton_kernel[grid]( + reorder_ids, src2dst, local_topk_ids.numel(), BLOCK_SIZE + ) + + return reorder_topk_ids, src2dst, seg_indptr + + +@triton.jit +def pre_reorder_triton_kernel_for_cutlass_moe( + input_ptr, + gateup_input_ptr, + src2dst_ptr, + topk_ids_ptr, + a1_scales_ptr, + num_experts, + topk, + hidden_size, + BLOCK_SIZE: tl.constexpr, +): + OutDtype = gateup_input_ptr.dtype.element_ty + + src_idx = tl.program_id(0) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + + src_ptr = input_ptr + src_idx * hidden_size + for idx in range(topk): + expert_id = tl.load(topk_ids_ptr + idx) + if expert_id != num_experts: + if a1_scales_ptr is not None: + scale = 1.0 / tl.load(a1_scales_ptr) + else: + scale = 1.0 + + dst_idx = tl.load(src2dst_ptr + idx) + dst_ptr = gateup_input_ptr + dst_idx * hidden_size + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + tl.arange(0, BLOCK_SIZE) + mask = offset < hidden_size + in_data = tl.load(src_ptr + offset, mask=mask).to(tl.float32) + out_data = (in_data * scale).to(OutDtype) + tl.store(dst_ptr + offset, out_data, mask=mask) + + +@triton.jit +def pre_reorder_triton_kernel( + input_ptr, + gateup_input_ptr, + src2dst_ptr, + topk_ids_ptr, + a1_scales_ptr, + start_expert_id, + end_expert_id, + topk, + hidden_size, + BLOCK_SIZE: tl.constexpr, + use_per_token_if_dynamic: tl.constexpr, +): + OutDtype = gateup_input_ptr.dtype.element_ty + + src_idx_int32 = tl.program_id(0) + src_idx = src_idx_int32.to(tl.int64) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + src_ptr = input_ptr + src_idx * hidden_size + + vec = tl.arange(0, BLOCK_SIZE) + + if a1_scales_ptr is not None and use_per_token_if_dynamic: + scale = 1.0 / tl.load(a1_scales_ptr + src_idx) + + for idx in range(topk): + expert_id = tl.load(topk_ids_ptr + idx) + if expert_id >= start_expert_id and expert_id <= end_expert_id: + if a1_scales_ptr is not None: + if not use_per_token_if_dynamic: + scale = 1.0 / tl.load(a1_scales_ptr + expert_id - start_expert_id) + else: + scale = 1.0 + + dst_idx_int32 = tl.load(src2dst_ptr + idx) + dst_idx = dst_idx_int32.to(tl.int64) + dst_ptr = gateup_input_ptr + dst_idx * hidden_size + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < hidden_size + in_data = tl.load(src_ptr + offset, mask=mask).to(tl.float32) + out_data = (in_data * scale).to(OutDtype) + tl.store(dst_ptr + offset, out_data, mask=mask) + + +@triton.jit +def silu_and_mul_triton_kernel( + gateup_output, + down_input, + hidden_size, + reorder_topk_ids, + scales, + start_expert_id, + end_expert_id, + BLOCK_SIZE: tl.constexpr, +): + InDtype = gateup_output.dtype.element_ty + OutDtype = down_input.dtype.element_ty + + half_hidden_size = hidden_size // 2 + + pid = tl.program_id(0) + expert_id = tl.load(reorder_topk_ids + pid) + if expert_id >= start_expert_id and expert_id <= end_expert_id: + gateup_output_ptr = gateup_output + pid * hidden_size + gate_output_ptr = gateup_output_ptr + up_output_ptr = gateup_output_ptr + half_hidden_size + down_input_ptr = down_input + pid * half_hidden_size + + if scales is not None: + scale = tl.load(scales + expert_id - start_expert_id) + scale = (1 / scale).to(InDtype) + else: + scale = 1 + + for start_offset in tl.range(0, half_hidden_size, BLOCK_SIZE): + offset = start_offset + tl.arange(0, BLOCK_SIZE) + mask = offset < half_hidden_size + + gate_output = tl.load(gate_output_ptr + offset, mask=mask).to(tl.float32) + up_output = tl.load(up_output_ptr + offset, mask=mask) + + # silu & mul & quantize + gate_output = gate_output * tl.sigmoid(gate_output) + gate_output = gate_output.to(InDtype) + + silu_mul_output = gate_output * up_output * scale + silu_mul_output = silu_mul_output.to(OutDtype) + tl.store(down_input_ptr + offset, silu_mul_output, mask=mask) + + +# copy from https://github.com/ModelTC/lightllm/blob/a000ab69098654df4731f5b12587dd4e7f0a4f41/lightllm/common/fused_moe/moe_silu_and_mul_mix_quant_ep.py +@triton.jit +def _silu_and_mul_post_quant_kernel( + input_ptr, + stride_input_0, + stride_input_1, + stride_input_2, + output_ptr, + stride_output_0, + stride_output_1, + stride_output_2, + output_scale_ptr, + stride_output_scale_0, + stride_output_scale_1, + stride_output_scale_2, + masked_m_ptr, + size_n, + fp8_max, + fp8_min, + BLOCK_N: tl.constexpr, + NUM_STAGE: tl.constexpr, + SCALE_UE8M0: tl.constexpr, +): + expert_id = tl.program_id(2) + token_id = tl.program_id(1) + hidden_dim_block_index = tl.program_id(0) + + block_num_per_expert = tl.num_programs(1) + + token_num_cur_expert = tl.load(masked_m_ptr + expert_id) + + stride_input_0 = tl.cast(stride_input_0, dtype=tl.int64) + stride_output_0 = tl.cast(stride_output_0, dtype=tl.int64) + stride_input_1 = tl.cast(stride_input_1, dtype=tl.int64) + stride_output_1 = tl.cast(stride_output_1, dtype=tl.int64) + + offs_in_d = hidden_dim_block_index * BLOCK_N + tl.arange(0, BLOCK_N) + input_ptr_offs = input_ptr + expert_id * stride_input_0 + offs_in_d + output_ptr_offs = output_ptr + expert_id * stride_output_0 + offs_in_d + output_scale_offs = ( + output_scale_ptr + + expert_id * stride_output_scale_0 + + hidden_dim_block_index * stride_output_scale_2 + ) + + for token_index in tl.range( + token_id, token_num_cur_expert, block_num_per_expert, num_stages=NUM_STAGE + ): + gate = tl.load( + input_ptr_offs + token_index * stride_input_1, + mask=offs_in_d < size_n, + other=0.0, + ).to(tl.float32) + up = tl.load( + input_ptr_offs + token_index * stride_input_1 + size_n, + mask=offs_in_d < size_n, + other=0.0, + ) + gate = gate / (1 + tl.exp(-gate)) + gate = gate.to(input_ptr.dtype.element_ty) + gate_up = up * gate + _absmax = tl.maximum(tl.max(tl.abs(gate_up)), 1e-10) + output_s = _absmax / fp8_max + if SCALE_UE8M0: + output_s = tl.exp2(tl.ceil(tl.log2(tl.abs(output_s)))) + output_q = tl.clamp(gate_up / output_s, fp8_min, fp8_max).to( + output_ptr.dtype.element_ty + ) + tl.store( + output_ptr_offs + token_index * stride_output_1, + output_q, + mask=offs_in_d < size_n, + ) + tl.store( + output_scale_offs + token_index * stride_output_scale_1, + output_s, + ) + + +def silu_and_mul_masked_post_quant_fwd( + input: torch.Tensor, + output: torch.Tensor, + output_scale: torch.Tensor, + quant_group_size: int, + masked_m: torch.Tensor, + scale_ue8m0: bool = False, +): + """ + input shape [expert_num, token_num_padded, hidden_dim] + output shape [expert_num, token_num_padded, hidden_dim // 2], dtype fp8 + output_scale [expert_num token_num_paddded, hidden_dim // 2 // 128] dtype float32 + quant_group_size int, + masked_m shape [expert_num], + """ + + assert input.is_contiguous() + assert output.dtype == torch.float8_e4m3fn + assert output.is_contiguous() + assert len(input.shape) == 3 + assert input.shape[0] == masked_m.shape[0] + assert input.shape[-1] % 2 == 0 + + size_n = input.shape[-1] // 2 + assert size_n % quant_group_size == 0 + + expert_num = len(masked_m) + + if expert_num < 4: + BLOCK_NUM_PER_EXPERT = 64 + else: + BLOCK_NUM_PER_EXPERT = 32 + + BLOCK_N = quant_group_size + num_warps = 1 + NUM_STAGES = 6 + hidden_dim_split_block_num = triton.cdiv(size_n, BLOCK_N) + assert BLOCK_N % quant_group_size == 0 + + grid = ( + hidden_dim_split_block_num, + BLOCK_NUM_PER_EXPERT, + expert_num, + ) + + finfo = torch.finfo(torch.float8_e4m3fn) + fp8_max = finfo.max + fp8_min = -fp8_max + + _silu_and_mul_post_quant_kernel[grid]( + input, + *input.stride(), + output, + *output.stride(), + output_scale, + *output_scale.stride(), + masked_m, + size_n, + fp8_max, + fp8_min, + BLOCK_N=BLOCK_N, + NUM_STAGE=NUM_STAGES, + num_warps=num_warps, + SCALE_UE8M0=scale_ue8m0, + ) + return + + +@triton.jit +def tanh(x): + return 2 * tl.sigmoid(2 * x) - 1 + + +@triton.jit +def gelu_and_mul_triton_kernel( + gateup_output, + down_input, + hidden_size, + reorder_topk_ids, + scales, + start_expert_id, + end_expert_id, + BLOCK_SIZE: tl.constexpr, +): + InDtype = gateup_output.dtype.element_ty + OutDtype = down_input.dtype.element_ty + + half_hidden_size = hidden_size // 2 + + pid = tl.program_id(0) + expert_id = tl.load(reorder_topk_ids + pid) + if expert_id >= start_expert_id and expert_id <= end_expert_id: + gateup_output_ptr = gateup_output + pid * hidden_size + gate_output_ptr = gateup_output_ptr + up_output_ptr = gateup_output_ptr + half_hidden_size + down_input_ptr = down_input + pid * half_hidden_size + + if scales is not None: + scale = tl.load(scales + expert_id - start_expert_id) + scale = (1 / scale).to(InDtype) + else: + scale = 1 + + for start_offset in tl.range(0, half_hidden_size, BLOCK_SIZE): + offset = start_offset + tl.arange(0, BLOCK_SIZE) + mask = offset < half_hidden_size + + gate_output = tl.load(gate_output_ptr + offset, mask=mask).to(tl.float32) + up_output = tl.load(up_output_ptr + offset, mask=mask) + + # gelu & mul & quantize + # https://pytorch.org/docs/stable/generated/torch.nn.GELU.html + # sqrt(2/pi) + kAlpha = 0.7978845608028654 + gate_output = ( + 0.5 + * gate_output + * ( + 1 + + tanh( + kAlpha + * ( + gate_output + + 0.044715 * gate_output * gate_output * gate_output + ) + ) + ) + ) + gate_output = gate_output.to(InDtype) + + gelu_mul_output = gate_output * up_output * scale + gelu_mul_output = gelu_mul_output.to(OutDtype) + tl.store(down_input_ptr + offset, gelu_mul_output, mask=mask) + + +@triton.jit +def post_reorder_triton_kernel( + down_output_ptr, + output_ptr, + src2dst_ptr, + topk_ids_ptr, + topk_weights_ptr, + start_expert_id, + end_expert_id, + topk, + hidden_size, + dst_start, + BLOCK_SIZE: tl.constexpr, +): + InDtype = down_output_ptr.dtype.element_ty + + src_idx_int32 = tl.program_id(0) + src_idx = src_idx_int32.to(tl.int64) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + topk_weights_ptr = topk_weights_ptr + src_idx * topk + + computed = False + store_ptr = output_ptr + src_idx * hidden_size + + vec = tl.arange(0, BLOCK_SIZE) + + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < hidden_size + + sum_vec = tl.zeros([BLOCK_SIZE], dtype=InDtype) + for idx in range(topk): + expert_id = tl.load(topk_ids_ptr + idx) + if expert_id >= start_expert_id and expert_id <= end_expert_id: + computed = True + dst_idx_int32 = tl.load(src2dst_ptr + idx) + dst_idx = dst_idx_int32.to(tl.int64) + dst_idx = dst_idx - dst_start + weigh_scale = tl.load(topk_weights_ptr + idx).to(InDtype) + load_ptr = down_output_ptr + dst_idx * hidden_size + in_data = tl.load(load_ptr + offset, mask=mask) + sum_vec += in_data * weigh_scale + tl.store(store_ptr + offset, sum_vec, mask=mask) + + if computed == False: + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < hidden_size + tl.store( + store_ptr + offset, tl.zeros([BLOCK_SIZE], dtype=InDtype), mask=mask + ) + + +@triton.jit +def post_reorder_triton_kernel_for_cutlass_moe( + down_output_ptr, + output_ptr, + src2dst_ptr, + topk_ids_ptr, + topk_weights_ptr, + num_experts, + topk, + hidden_size, + dst_start, + BLOCK_SIZE: tl.constexpr, +): + InDtype = down_output_ptr.dtype.element_ty + + src_idx_int32 = tl.program_id(0) + src_idx = src_idx_int32.to(tl.int64) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + topk_weights_ptr = topk_weights_ptr + src_idx * topk + + store_ptr = output_ptr + src_idx * hidden_size + + vec = tl.arange(0, BLOCK_SIZE) + + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < hidden_size + + sum_vec = tl.zeros([BLOCK_SIZE], dtype=InDtype) + for idx in range(topk): + expert_id = tl.load(topk_ids_ptr + idx) + if expert_id != num_experts: + dst_idx_int32 = tl.load(src2dst_ptr + idx) + dst_idx = dst_idx_int32.to(tl.int64) + dst_idx = dst_idx - dst_start + weigh_scale = tl.load(topk_weights_ptr + idx).to(InDtype) + load_ptr = down_output_ptr + dst_idx * hidden_size + in_data = tl.load(load_ptr + offset, mask=mask) + sum_vec += in_data * weigh_scale + tl.store(store_ptr + offset, sum_vec, mask=mask) + + +@triton.jit +def compute_m_range( + pid, + batch_size, + seg_indptr, + weight_indices, + m_num_tiles_indptr, + BLOCK_SIZE_M: tl.constexpr, +): + idx = 0 + for bs in range(batch_size): + tiles = tl.load(m_num_tiles_indptr + bs) + if pid >= tiles: + idx = bs + + idx_start = tl.load(m_num_tiles_indptr + idx) + + m_range_start = tl.load(seg_indptr + idx) + (pid - idx_start) * BLOCK_SIZE_M + m_range_end = min(tl.load(seg_indptr + idx + 1), m_range_start + BLOCK_SIZE_M) + expert_id = tl.load(weight_indices + idx) + return m_range_start, m_range_end, expert_id + + +@triton.jit +def grouped_gemm_triton_kernel( + a, + b, + c, + batch_size, + N, + K, + seg_indptr, + weight_indices, + m_num_tiles_indptr, + scale_a, + scale_b, + use_fp8_w8a8: tl.constexpr, + group_n: tl.constexpr, + group_k: tl.constexpr, + a_stride_0: tl.constexpr, + b_stride_0: tl.constexpr, + b_stride_1: tl.constexpr, + as_stride_0: tl.constexpr, + as_stride_1: tl.constexpr, + bs_stride_0: tl.constexpr, + bs_stride_2: tl.constexpr, + bs_stride_1: tl.constexpr, + use_per_token_if_dynamic: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, +): + c_dtype = c.dtype.element_ty + + pid_m = tl.program_id(0) + pid_n = tl.program_id(1) + total_m_block = tl.load(m_num_tiles_indptr + batch_size) + if pid_m >= total_m_block: + return + + m_range_start, m_range_end, expert_id = compute_m_range( + pid_m, batch_size, seg_indptr, weight_indices, m_num_tiles_indptr, BLOCK_SIZE_M + ) + if m_range_end - m_range_start == 0: + return + + n_range_start = pid_n * BLOCK_SIZE_N + n_range_end = min(n_range_start + BLOCK_SIZE_N, N) + + offs_am = tl.arange(0, BLOCK_SIZE_M) + offs_bn = tl.arange(0, BLOCK_SIZE_N) + + offs_am = tl.where(offs_am < m_range_end - m_range_start, offs_am, 0) + offs_bn = tl.where(offs_bn < n_range_end - n_range_start, offs_bn, 0) + offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) + offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) + offs_k = tl.arange(0, BLOCK_SIZE_K) + + a_ptr = a + (m_range_start + offs_am[:, None]) * a_stride_0 + offs_k[None, :] + b_ptr = b + ( + (expert_id * b_stride_0) + + (n_range_start + offs_bn[:, None]) * b_stride_1 + + offs_k[None, :] + ) + + if group_k > 0 and group_n > 0: + a_scale_ptrs = scale_a + (m_range_start + offs_am[:, None]) * as_stride_0 + offs_bsn = (n_range_start + offs_bn) // group_n + b_scale_ptrs = scale_b + (expert_id * bs_stride_0) + offs_bsn * bs_stride_1 + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a_tile = tl.load( + a_ptr, mask=offs_k[None, :] < (K - k * BLOCK_SIZE_K), other=0.0 + ) + b_tile = tl.load( + b_ptr, mask=offs_k[None, :] < (K - k * BLOCK_SIZE_K), other=0.0 + ) + + if group_k > 0 and group_n > 0: + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_scale = tl.load(a_scale_ptrs + offs_ks * as_stride_1) + b_scale = tl.load(b_scale_ptrs + offs_ks * bs_stride_2) + accumulator += tl.dot(a_tile, b_tile.T) * a_scale * b_scale[None, :] + else: + accumulator = tl.dot(a_tile, b_tile.T, accumulator) + a_ptr += BLOCK_SIZE_K + b_ptr += BLOCK_SIZE_K + + if use_fp8_w8a8 and not (group_k > 0 and group_n > 0): + if use_per_token_if_dynamic: + scale_a_value = tl.load(scale_a + (m_range_start + offs_am[:, None])) + else: + scale_a_value = tl.load(scale_a + expert_id) + scale_b_value = tl.load(scale_b + expert_id) + accumulator *= scale_a_value * scale_b_value + + c_tile = accumulator.to(c_dtype) + + offs_cm = m_range_start + tl.arange(0, BLOCK_SIZE_M) + offs_cn = n_range_start + tl.arange(0, BLOCK_SIZE_N) + c_ptr = c + offs_cm[:, None] * N + offs_cn[None, :] + c_mask = (offs_cm[:, None] < m_range_end) & (offs_cn[None, :] < n_range_end) + tl.store(c_ptr, c_tile, mask=c_mask) + + +@triton.jit +def compute_m_num_tiles_indptr( + m_num_tiles_indptr, seg_indptr, batch_size: tl.constexpr, BLOCK_SIZE_M: tl.constexpr +): + for bs in range(batch_size): + m = tl.load(seg_indptr + bs + 1) - tl.load(seg_indptr + bs) + cur_num_tiles = tl.cdiv(m, BLOCK_SIZE_M) + pre_num_tiles = tl.load(m_num_tiles_indptr + bs) + tl.store(m_num_tiles_indptr + bs + 1, pre_num_tiles + cur_num_tiles) + + +def grouped_gemm_triton( + a: torch.Tensor, + b: torch.Tensor, + c: torch.Tensor, + batch_size: int, + weight_column_major: bool, + seg_indptr: Optional[torch.Tensor] = None, + weight_indices: Optional[torch.Tensor] = None, + use_fp8_w8a8: bool = False, + scale_a: torch.Tensor = None, + scale_b: torch.Tensor = None, + block_shape: Optional[List[int]] = None, + c_dtype=None, + use_per_token_if_dynamic: bool = True, +): + assert weight_column_major == True # TODO: more + if use_fp8_w8a8 and block_shape is None: + assert scale_a is not None and scale_b is not None + + if block_shape is not None: + a_original = a + + assert len(block_shape) == 2 + block_n, block_k = block_shape[0], block_shape[1] + a, scale_a = per_token_group_quant_fp8(a, block_k) + + assert triton.cdiv(a.shape[-1], block_k) == scale_a.shape[-1] + assert triton.cdiv(b.shape[-2], block_n) == scale_b.shape[-2] + assert triton.cdiv(b.shape[-1], block_k) == scale_b.shape[-1] + + dispose_tensor(a_original) + + # TODO: adjust config or tune kernel + # Reduce block size to prevent L40 shared memory overflow. + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + } + + m_num_tiles_indptr = torch.zeros(batch_size + 1, device=a.device, dtype=torch.int64) + compute_m_num_tiles_indptr[(1,)]( + m_num_tiles_indptr, seg_indptr, batch_size, config["BLOCK_SIZE_M"] + ) + + if c is None: + assert c_dtype is not None + c = torch.empty(a.shape[0], b.shape[1], device=a.device, dtype=c_dtype) + + grid = lambda META: ( + triton.cdiv(a.size(0), META["BLOCK_SIZE_M"]) + batch_size, + triton.cdiv(b.size(1), META["BLOCK_SIZE_N"]), + ) + + if use_fp8_w8a8 and block_shape is None and use_per_token_if_dynamic: + assert ( + scale_a.shape[0] == a.shape[0] + ), f"scale_a.shape: {scale_a.shape}, a.shape: {a.shape}" + + grouped_gemm_triton_kernel[grid]( + a, + b, + c, + batch_size, + b.size(1), + b.size(2), + seg_indptr, + weight_indices, + m_num_tiles_indptr, + scale_a, + scale_b, + use_fp8_w8a8, + 0 if block_shape is None else block_shape[0], + 0 if block_shape is None else block_shape[1], + a.stride(0), + b.stride(0), + b.stride(1), + scale_a.stride(0) if scale_a is not None and scale_a.ndim == 2 else 0, + scale_a.stride(1) if scale_a is not None and scale_a.ndim == 2 else 0, + scale_b.stride(0) if scale_b is not None and scale_b.ndim >= 2 else 0, + scale_b.stride(2) if scale_b is not None and scale_b.ndim == 3 else 0, + scale_b.stride(1) if scale_b is not None and scale_b.ndim >= 2 else 0, + use_per_token_if_dynamic, + **config, + ) + return c + + +@triton.jit +def _fwd_kernel_ep_scatter_1( + num_recv_tokens_per_expert, + expert_start_loc, + m_indices, + num_experts: tl.constexpr, + BLOCK_E: tl.constexpr, + BLOCK_EXPERT_NUM: tl.constexpr, +): + cur_expert = tl.program_id(0) + + offset_cumsum = tl.arange(0, BLOCK_EXPERT_NUM) + tokens_per_expert = tl.load( + num_recv_tokens_per_expert + offset_cumsum, + mask=offset_cumsum < num_experts, + other=0, + ) + cumsum = tl.cumsum(tokens_per_expert) - tokens_per_expert + tl.store(expert_start_loc + offset_cumsum, cumsum, mask=offset_cumsum < num_experts) + + cur_expert_start = tl.load(expert_start_loc + cur_expert) + cur_expert_token_num = tl.load(num_recv_tokens_per_expert + cur_expert) + + m_indices_start_ptr = m_indices + cur_expert_start + off_expert = tl.arange(0, BLOCK_E) + + for start_m in tl.range(0, cur_expert_token_num, BLOCK_E, num_stages=4): + tl.store( + m_indices_start_ptr + start_m + off_expert, + cur_expert, + ) + + +@triton.jit +def _fwd_kernel_ep_scatter_2( + total_token_num, + expert_start_loc, + recv_x, + recv_x_stride0, + recv_x_stride1, + recv_x_scale, + recv_x_scale_stride0, + recv_x_scale_stride1, + recv_topk, + recv_topk_stride0, + recv_topk_stride1, + output_tensor, + output_tensor_stride0, + output_tensor_stride1, + output_tensor_scale, + output_tensor_scale_stride0, + output_tensor_scale_stride1, + output_index, + output_index_stride0, + output_index_stride1, + topk_num: tl.constexpr, + HIDDEN_SIZE: tl.constexpr, + HIDDEN_SIZE_PAD: tl.constexpr, + SCALE_HIDDEN_SIZE: tl.constexpr, + SCALE_HIDDEN_SIZE_PAD: tl.constexpr, +): + start_token_id = tl.program_id(0) + grid_num = tl.num_programs(0) + + offset_in = tl.arange(0, HIDDEN_SIZE_PAD) + mask = offset_in < HIDDEN_SIZE + + index_in_s = tl.arange(0, SCALE_HIDDEN_SIZE_PAD) + mask_s = index_in_s < SCALE_HIDDEN_SIZE + + for token_id_int32 in range(start_token_id, total_token_num, grid_num): + token_id = token_id_int32.to(tl.int64) + to_copy = tl.load(recv_x + token_id * recv_x_stride0 + offset_in, mask=mask) + to_copy_s = tl.load( + recv_x_scale + + token_id * recv_x_scale_stride0 + + index_in_s * recv_x_scale_stride1, + mask=mask_s, + ) + + for topk_idx_int32 in tl.range(0, topk_num, 1, num_stages=4): + topk_index = topk_idx_int32.to(tl.int64) + expert_id = tl.load(recv_topk + token_id * recv_topk_stride0 + topk_index) + if expert_id >= 0: + dest_token_index_int32 = tl.atomic_add(expert_start_loc + expert_id, 1) + dest_token_index = dest_token_index_int32.to(tl.int64) + + tl.store( + output_index + token_id * output_index_stride0 + topk_index, + dest_token_index_int32, + ) + output_tensor_ptr = ( + output_tensor + dest_token_index * output_tensor_stride0 + ) + output_tensor_scale_ptr = ( + output_tensor_scale + dest_token_index * output_tensor_scale_stride0 + ) + tl.store(output_tensor_ptr + offset_in, to_copy, mask=mask) + tl.store( + output_tensor_scale_ptr + index_in_s * output_tensor_scale_stride1, + to_copy_s, + mask=mask_s, + ) + + +# copy from https://github.com/ModelTC/lightllm/blob/main/lightllm/common/fused_moe/deepep_scatter_gather.py +@torch.no_grad() +def ep_scatter( + recv_x: torch.Tensor, + recv_x_scale: torch.Tensor, + recv_topk: torch.Tensor, + num_recv_tokens_per_expert: torch.Tensor, + expert_start_loc: torch.Tensor, + output_tensor: torch.Tensor, + output_tensor_scale: torch.Tensor, + m_indices: torch.Tensor, + output_index: torch.Tensor, + scale_ue8m0: bool = False, +): + BLOCK_E = 128 # token num of per expert is aligned to 128 + BLOCK_D = 128 # block size of quantization + num_warps = 8 + num_experts = num_recv_tokens_per_expert.shape[0] + hidden_size = recv_x.shape[1] + # grid = (triton.cdiv(hidden_size, BLOCK_D), num_experts) + grid = num_experts + + scale_hidden_size = hidden_size // BLOCK_D + if scale_ue8m0: + # ue8m0 scales are packed here (4 scales per int32), + # hence the effective size of this dimension is divided by 4. + scale_hidden_size = ceil_div(scale_hidden_size, 4) + + assert m_indices.shape[0] % BLOCK_E == 0 + assert recv_x_scale.dtype == output_tensor_scale.dtype + assert recv_x_scale.shape[1] == output_tensor_scale.shape[1] == scale_hidden_size + + _fwd_kernel_ep_scatter_1[(grid,)]( + num_recv_tokens_per_expert, + expert_start_loc, + m_indices, + num_experts=num_experts, + num_warps=num_warps, + BLOCK_E=BLOCK_E, + BLOCK_EXPERT_NUM=triton.next_power_of_2(num_experts), + ) + + grid = min(recv_topk.shape[0], 1024 * 8) + + _fwd_kernel_ep_scatter_2[(grid,)]( + recv_topk.shape[0], + expert_start_loc, + recv_x, + recv_x.stride(0), + recv_x.stride(1), + recv_x_scale, + recv_x_scale.stride(0), + recv_x_scale.stride(1), + recv_topk, + recv_topk.stride(0), + recv_topk.stride(1), + output_tensor, + output_tensor.stride(0), + output_tensor.stride(1), + output_tensor_scale, + output_tensor_scale.stride(0), + output_tensor_scale.stride(1), + output_index, + output_index.stride(0), + output_index.stride(1), + topk_num=recv_topk.shape[1], + num_warps=num_warps, + HIDDEN_SIZE=hidden_size, + HIDDEN_SIZE_PAD=triton.next_power_of_2(hidden_size), + SCALE_HIDDEN_SIZE=scale_hidden_size, + SCALE_HIDDEN_SIZE_PAD=triton.next_power_of_2(scale_hidden_size), + ) + return + + +@triton.jit +def _fwd_kernel_ep_gather( + total_token_num, + input_tensor, + input_tensor_stride0, + input_tensor_stride1, + recv_topk_ids, + recv_topk_ids_stride0, + recv_topk_ids_stride1, + recv_topk_weight, + recv_topk_weight_stride0, + recv_topk_weight_stride1, + input_index, + input_index_stride0, + input_index_stride1, + output_tensor, + output_tensor_stride0, + output_tensor_stride1, + topk_num: tl.constexpr, + BLOCK_D: tl.constexpr, +): + cur_block_int32 = tl.program_id(0) + cur_block = cur_block_int32.to(tl.int64) + + start_cur_token_int32 = tl.program_id(1) + + grid_num = tl.num_programs(1) + + for cur_token_int32 in range(start_cur_token_int32, total_token_num, grid_num): + cur_token = cur_token_int32.to(tl.int64) + + off_d = tl.arange(0, BLOCK_D) + accumulator = tl.zeros([BLOCK_D], dtype=tl.float32) + + for topk_index_int32 in range(0, topk_num): + topk_index = topk_index_int32.to(tl.int64) + + expert_id = tl.load( + recv_topk_ids + cur_token * recv_topk_ids_stride0 + topk_index + ) + if expert_id >= 0: + source_token_index_int32 = tl.load( + input_index + cur_token * input_index_stride0 + topk_index + ) + source_token_index = source_token_index_int32.to(tl.int64) + + acc_weight = tl.load( + recv_topk_weight + cur_token * recv_topk_weight_stride0 + topk_index + ) + tmp = tl.load( + input_tensor + + source_token_index * input_tensor_stride0 + + cur_block * BLOCK_D + + off_d + ) + accumulator += tmp.to(tl.float32) * acc_weight + + tl.store( + output_tensor + + cur_token * output_tensor_stride0 + + cur_block * BLOCK_D + + off_d, + accumulator.to(output_tensor.dtype.element_ty), + ) + + +@torch.no_grad() +def ep_gather( + input_tensor: torch.Tensor, + recv_topk_ids: torch.Tensor, + recv_topk_weight: torch.Tensor, + input_index: torch.Tensor, + output_tensor: torch.Tensor, +): + BLOCK_D = 1024 if not is_in_ci() else 128 # block size of quantization + num_warps = 2 + num_tokens = output_tensor.shape[0] + hidden_size = input_tensor.shape[1] + assert hidden_size % BLOCK_D == 0 + grid = (triton.cdiv(hidden_size, BLOCK_D), min(num_tokens, 1024)) + _fwd_kernel_ep_gather[grid]( + num_tokens, + input_tensor, + input_tensor.stride(0), + input_tensor.stride(1), + recv_topk_ids, + recv_topk_ids.stride(0), + recv_topk_ids.stride(1), + recv_topk_weight, + recv_topk_weight.stride(0), + recv_topk_weight.stride(1), + input_index, + input_index.stride(0), + input_index.stride(1), + output_tensor, + output_tensor.stride(0), + output_tensor.stride(1), + topk_num=recv_topk_ids.shape[1], + num_warps=num_warps, + BLOCK_D=BLOCK_D, + ) + return + + +# copy from +# https://github.com/deepseek-ai/DeepGEMM/blob/bd2a77552886b98c205af12f8d7d2d61247c4b27/deep_gemm/jit_kernels/utils.py#L58 +def get_tma_aligned_size(x: int, element_size: int) -> int: + """ + Global memory address of TMA must be 16-byte aligned. + Since we use column-major layout for the LHS scaling tensor, + the M-axis of the LHS scaling tensor needs to be padded to a multiple of 16 bytes. + + Arguments: + x: original M-axis shape of the LHS scaling tensor. + element_size: element size of the LHS scaling tensor. + + Returns: + M-axis shape of the LHS scaling tensor after padding. + """ + tma_alignment_bytes = 16 + assert tma_alignment_bytes % element_size == 0 + alignment = tma_alignment_bytes // element_size + return ceil_div(x, alignment) * alignment + + +@triton.jit +def _tma_align_input_scale_kernel( + input_scale_ptr, + output_ptr, + m, + k_div_block_size, + input_scale_stride_m, + input_scale_stride_k, + output_stride_m, + output_stride_k, + BLOCK_SIZE_K: tl.constexpr, +): + pid_m = tl.program_id(axis=0) + grid_m = tl.num_programs(0) + k_offsets = tl.arange(0, BLOCK_SIZE_K) + + for m_base in range(pid_m, m, grid_m): + input_offset = ( + input_scale_ptr + + m_base * input_scale_stride_m + + k_offsets * input_scale_stride_k + ) + input_data = tl.load(input_offset, mask=k_offsets < k_div_block_size) + + output_offset = ( + output_ptr + k_offsets * output_stride_k + m_base * output_stride_m + ) + tl.store(output_offset, input_data, mask=k_offsets < k_div_block_size) + + +# copy from https://github.com/ModelTC/lightllm/blob/main/lightllm/common/quantization/triton_quant/fp8/fp8act_quant_kernel.py +def tma_align_input_scale(input_scale: torch.Tensor): + assert input_scale.dim() == 2 + m, k_div_block_size = input_scale.shape + padd_m = get_tma_aligned_size(m, input_scale.element_size()) + output = torch.empty( + (k_div_block_size, padd_m), dtype=input_scale.dtype, device=input_scale.device + ) + + grid_m = min(m, 8192) + BLOCK_SIZE_K = triton.next_power_of_2(k_div_block_size) + + _tma_align_input_scale_kernel[(grid_m,)]( + input_scale_ptr=input_scale, + output_ptr=output, + m=m, + k_div_block_size=k_div_block_size, + input_scale_stride_m=input_scale.stride(0), + input_scale_stride_k=input_scale.stride(1), + output_stride_m=output.stride(1), # Note: these are swapped + output_stride_k=output.stride(0), # for column-major + BLOCK_SIZE_K=BLOCK_SIZE_K, + ) + return output.t()[:m] + + +@triton.jit +def compute_masked_m_triton_kernel(seg_indptr, masked_m): + expert_id = tl.program_id(0) + start = tl.load(seg_indptr + expert_id) + end = tl.load(seg_indptr + expert_id + 1) + tl.store(masked_m + expert_id, (end - start)) + + +@triton.jit +def deepgemm_compute_src2dst_triton_kernel( + topk_ids, + reorder_ids, + seg_indptr, + src2dst, + m_max, + num_toks, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + dst_id = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = dst_id < num_toks + src_id = tl.load(reorder_ids + dst_id, mask=mask) + expert_id = tl.load(topk_ids + src_id, mask=(src_id < num_toks)) + expert_dst_start = tl.load(seg_indptr + expert_id) + expert_dst_offset = dst_id - expert_dst_start + dst_id = expert_id * m_max + expert_dst_offset + tl.store(src2dst + src_id, dst_id, mask=mask) + + +@triton.jit +def fill_gateup_input_triton_kernel( + input_ptr, + scale_ptr, + gateup_input_ptr, + gateup_input_scale_ptr, + src2dst_ptr, + topk_ids_ptr, + start_expert_id, + end_expert_id, + topk, + m_max, + hidden_size, + scale_size, + BLOCK_SIZE: tl.constexpr, +): + + src_idx_int32 = tl.program_id(0) + src_idx = src_idx_int32.to(tl.int64) + src2dst_ptr = src2dst_ptr + src_idx * topk + topk_ids_ptr = topk_ids_ptr + src_idx * topk + src_ptr = input_ptr + src_idx * hidden_size + scale_src_ptr = scale_ptr + src_idx * scale_size + + vec = tl.arange(0, BLOCK_SIZE) + for idx in range(topk): + expert_id = tl.load(topk_ids_ptr + idx) + if expert_id >= start_expert_id and expert_id <= end_expert_id: + dst_idx_int32 = tl.load(src2dst_ptr + idx) + dst_idx = dst_idx_int32.to(tl.int64) + dst_idx = dst_idx - start_expert_id * m_max + dst_ptr = gateup_input_ptr + dst_idx * hidden_size + for start_offset in tl.range(0, hidden_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < hidden_size + in_data = tl.load(src_ptr + offset, mask=mask) + tl.store(dst_ptr + offset, in_data, mask=mask) + scale_dst_ptr = gateup_input_scale_ptr + dst_idx * scale_size + for start_offset in tl.range(0, scale_size, BLOCK_SIZE): + offset = start_offset + vec + mask = offset < scale_size + in_scale = tl.load(scale_src_ptr + offset, mask=mask) + tl.store(scale_dst_ptr + offset, in_scale, mask=mask) + + +def moe_ep_deepgemm_preprocess( + topk_ids: torch.Tensor, + num_experts: int, + hidden_states: torch.Tensor, + top_k: int, + start_expert_id, + end_expert_id, + block_shape, + output_dtype: torch.dtype = torch.float8_e4m3fn, +): + reorder_topk_ids, reorder_ids = torch.sort(topk_ids.view(-1), stable=True) + seg_indptr = torch.zeros(num_experts + 1, device=topk_ids.device, dtype=torch.int64) + src2dst = torch.empty(topk_ids.numel(), device=topk_ids.device, dtype=torch.int32) + masked_m = torch.zeros(num_experts, device=topk_ids.device, dtype=torch.int32) + + compute_seg_indptr_triton_kernel[(num_experts,)]( + reorder_topk_ids, seg_indptr, topk_ids.numel() + ) + + grid = lambda meta: (triton.cdiv(topk_ids.numel(), meta["BLOCK_SIZE"]),) + compute_masked_m_triton_kernel[(num_experts,)](seg_indptr, masked_m) + + # For masked grouped GEMM, shape M should be multiple of the block M (current block M: {block_m}) https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/jit_kernels/m_grouped_gemm.py#L165 + m_max = (hidden_states.size(0) + 255) // 256 * 256 + expected_m = (topk_ids.numel() + num_experts - 1) // num_experts + gateup_input = torch.empty( + (int(end_expert_id - start_expert_id + 1), m_max, hidden_states.size(1)), + device=hidden_states.device, + dtype=output_dtype, + ) + + deepgemm_compute_src2dst_triton_kernel[grid]( + topk_ids, + reorder_ids, + seg_indptr, + src2dst, + m_max, + topk_ids.numel(), + BLOCK_SIZE=256, + ) + + if block_shape is None: + block_shape = [128, 128] + assert len(block_shape) == 2 + block_n, block_k = block_shape[0], block_shape[1] + hidden_states, scale = per_token_group_quant_fp8(hidden_states, block_k) + + gateup_input_scale = torch.empty( + (gateup_input.size(0), gateup_input.size(1), scale.size(1)), + device=hidden_states.device, + dtype=scale.dtype, + ) + + fill_gateup_input_triton_kernel[(hidden_states.shape[0],)]( + hidden_states, + scale, + gateup_input, + gateup_input_scale, + src2dst, + topk_ids, + start_expert_id, + end_expert_id, + top_k, + m_max, + hidden_states.size(1), + scale.size(1), + BLOCK_SIZE=1024, + ) + + return ( + m_max, + masked_m[start_expert_id : (end_expert_id + 1)], + expected_m, + src2dst, + gateup_input, + gateup_input_scale, + ) + + +@triton.jit +def compute_identity_kernel( + top_k, + hidden_states_ptr, + expert_scales_ptr, + num_tokens, + output_ptr, + hidden_dim, + scales_stride, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(0) + + batch_id = pid // (hidden_dim // BLOCK_SIZE) + dim_offset = pid % (hidden_dim // BLOCK_SIZE) * BLOCK_SIZE + + if batch_id >= num_tokens or dim_offset >= hidden_dim: + return + + h = tl.load( + hidden_states_ptr + + batch_id * hidden_dim + + dim_offset + + tl.arange(0, BLOCK_SIZE), + mask=(dim_offset + tl.arange(0, BLOCK_SIZE)) < hidden_dim, + ) + + result = tl.zeros([BLOCK_SIZE], dtype=tl.float32) + for i in range(top_k): + scale = tl.load(expert_scales_ptr + batch_id * scales_stride + i) + result += h * scale + + tl.store( + output_ptr + batch_id * hidden_dim + dim_offset + tl.arange(0, BLOCK_SIZE), + result, + mask=(dim_offset + tl.arange(0, BLOCK_SIZE)) < hidden_dim, + ) + + +def zero_experts_compute_triton( + expert_indices, expert_scales, num_experts, zero_expert_type, hidden_states +): + N = expert_indices.numel() + top_k = expert_indices.size(-1) + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_SIZE"]),) + + if zero_expert_type == "identity": + zero_expert_mask = expert_indices < num_experts + zero_expert_scales = expert_scales.clone() + zero_expert_scales[zero_expert_mask] = 0.0 + + normal_expert_mask = expert_indices >= num_experts + expert_indices[normal_expert_mask] = -1 + expert_scales[normal_expert_mask] = 0.0 + + output = torch.zeros_like(hidden_states).to(hidden_states.device) + hidden_dim = hidden_states.size(-1) + num_tokens = hidden_states.size(0) + + grid = lambda meta: (num_tokens * (hidden_dim // meta["BLOCK_SIZE"]),) + compute_identity_kernel[grid]( + top_k, + hidden_states, + zero_expert_scales, + num_tokens, + output, + hidden_dim, + zero_expert_scales.stride(0), + BLOCK_SIZE=256, + ) + + return output diff --git a/python/sglang/srt/layers/moe/ep_moe/layer.py b/python/sglang/srt/layers/moe/ep_moe/layer.py new file mode 100644 index 000000000..ef33665c3 --- /dev/null +++ b/python/sglang/srt/layers/moe/ep_moe/layer.py @@ -0,0 +1,869 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Optional, Union + +import torch + +from sglang.srt.distributed.parallel_state import get_moe_expert_parallel_world_size +from sglang.srt.layers.moe import ( + get_deepep_mode, + get_moe_a2a_backend, + get_moe_runner_backend, + should_use_flashinfer_trtllm_moe, +) +from sglang.srt.layers.moe.ep_moe.kernels import ( + ep_gather, + ep_scatter, + moe_ep_deepgemm_preprocess, + post_reorder_triton_kernel, + silu_and_mul_masked_post_quant_fwd, + tma_align_input_scale, +) +from sglang.srt.layers.moe.fused_moe_triton.layer import FlashInferFusedMoE, FusedMoE +from sglang.srt.layers.moe.topk import TopKOutput +from sglang.srt.layers.quantization import deep_gemm_wrapper +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.layers.quantization.fp8_kernel import ( + is_fp8_fnuz, + sglang_per_token_group_quant_fp8, +) +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.utils import ceil_div, dispose_tensor, get_bool_env_var, is_hip, is_npu + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + DeepEPLLOutput, + DeepEPNormalOutput, + DispatchOutput, + ) + +_is_hip = is_hip() +_is_npu = is_npu() +_is_fp8_fnuz = is_fp8_fnuz() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if not (_is_npu or _is_hip): + from sgl_kernel import silu_and_mul + +if _use_aiter: + from aiter import ActivationType, QuantType + from aiter.fused_moe import fused_moe + +logger = logging.getLogger(__name__) + + +# TODO(kaixih@nvidia): ideally we should merge this logic into +# `fill_gateup_input_triton_kernel` to directly generate e8m0 scale. +@torch.compile +def _cast_to_e8m0_with_rounding_up(x: torch.Tensor) -> torch.Tensor: + temp = x.to(torch.float32).view(torch.int32) + exp = torch.bitwise_right_shift(temp, 23) + mant = torch.bitwise_and(temp, 0x7FFFFF) + is_ru = torch.logical_and( + torch.logical_and((mant > 0), (exp != 0xFE)), + ~torch.logical_and((exp == 0), (mant <= 0x400000)), + ) + exp = torch.where(is_ru, exp + 1, exp) + new_x = exp.to(torch.uint8).view(torch.int) + return new_x.transpose(1, 2).contiguous().transpose(1, 2) + + +class EPMoE(FusedMoE): + """ + MoE Expert Parallel Impl + + + """ + + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + layer_id: int, + num_fused_shared_experts: int = 0, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + activation: str = "silu", + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_clamp_limit: Optional[float] = None, + with_bias: bool = False, + ): + super().__init__( + num_experts=num_experts, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + num_fused_shared_experts=num_fused_shared_experts, + layer_id=layer_id, + top_k=top_k, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + activation=activation, + # apply_router_weight_on_input=apply_router_weight_on_input, + routed_scaling_factor=routed_scaling_factor, + gemm1_alpha=gemm1_alpha, + gemm1_clamp_limit=gemm1_clamp_limit, + with_bias=with_bias, + ) + + self.intermediate_size = intermediate_size + + if isinstance(quant_config, Fp8Config): + self.use_block_quant = getattr(self.quant_method, "block_quant", False) + self.block_shape = ( + self.quant_method.quant_config.weight_block_size + if self.use_block_quant + else None + ) + self.use_fp8_w8a8 = True + self.fp8_dtype = torch.float8_e4m3fn + self.activation_scheme = quant_config.activation_scheme + else: + self.use_fp8_w8a8 = False + self.use_block_quant = False + self.block_shape = None + self.activation_scheme = None + + def forward(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + if deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and self.use_fp8_w8a8: + return self.forward_deepgemm(hidden_states, topk_output) + else: + return super().forward(hidden_states, topk_output) + + def forward_deepgemm( + self, + hidden_states: torch.Tensor, + topk_output: TopKOutput, + ): + + self.w13_weight_fp8 = ( + self.w13_weight, + ( + self.w13_weight_scale_inv + if self.use_block_quant + else self.w13_weight_scale + ), + ) + self.w2_weight_fp8 = ( + self.w2_weight, + self.w2_weight_scale_inv if self.use_block_quant else self.w2_weight_scale, + ) + + assert self.quant_method is not None + assert self.moe_runner_config.activation == "silu" + + hidden_states_shape = hidden_states.shape + hidden_states_dtype = hidden_states.dtype + hidden_states_device = hidden_states.device + + topk_weights, topk_ids, _ = topk_output + + if not self.use_block_quant: + # Convert per-tensor quant to per-block quant by repeating scales for forward_deepgemm + scale_block_size = 128 + w13_weight_scale_n = 2 * ( + (self.intermediate_size + scale_block_size - 1) // scale_block_size + ) + w13_weight_scale_k = ( + hidden_states_shape[-1] + scale_block_size - 1 + ) // scale_block_size + w13_weight_scale = ( + self.w13_weight_scale.unsqueeze(1) + .repeat_interleave(w13_weight_scale_n, dim=1) + .unsqueeze(2) + .repeat_interleave(w13_weight_scale_k, dim=2) + ) + self.w13_weight_fp8 = ( + self.w13_weight, + w13_weight_scale, + ) + w2_weight_scale_n = ( + hidden_states_shape[-1] + scale_block_size - 1 + ) // scale_block_size + w2_weight_scale_k = ( + self.intermediate_size + scale_block_size - 1 + ) // scale_block_size + w2_weight_scale = ( + self.w2_weight_scale.unsqueeze(1) + .repeat_interleave(w2_weight_scale_n, dim=1) + .unsqueeze(2) + .repeat_interleave(w2_weight_scale_k, dim=2) + ) + self.w2_weight_fp8 = ( + self.w2_weight, + w2_weight_scale, + ) + + # PreReorder + m_max, masked_m, expected_m, src2dst, gateup_input, gateup_input_scale = ( + moe_ep_deepgemm_preprocess( + topk_ids, + self.num_experts, + hidden_states, + self.top_k, + self.start_expert_id, + self.end_expert_id, + self.block_shape, + ) + ) + + dispose_tensor(hidden_states) + + if deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0: + b, s_mn, s_k = gateup_input_scale.shape + assert ( + s_mn % 4 == 0 and s_k % 4 == 0 + ), f"scales must be aligned to 4, but got ({b}, {s_mn}, {s_k})" + + # GroupGemm-0 + gateup_input_fp8 = ( + gateup_input, + ( + _cast_to_e8m0_with_rounding_up(gateup_input_scale) + if deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + else deep_gemm_wrapper.get_mn_major_tma_aligned_tensor( + gateup_input_scale + ) + ), + ) + num_groups, m, k = gateup_input_fp8[0].size() + n = self.w13_weight.size(1) + gateup_output = torch.empty( + (num_groups, m, n), device=hidden_states_device, dtype=torch.bfloat16 + ) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_masked( + gateup_input_fp8, + self.w13_weight_fp8, + gateup_output, + masked_m, + expected_m, + ) + del gateup_input + del gateup_input_fp8 + + # Act + down_input = torch.empty( + ( + gateup_output.shape[0], + gateup_output.shape[1], + gateup_output.shape[2] // 2, + ), + device=hidden_states_device, + dtype=self.fp8_dtype, + ) + scale_block_size = 128 + down_input_scale = torch.empty( + ( + gateup_output.shape[0], + gateup_output.shape[1], + gateup_output.shape[2] // 2 // scale_block_size, + ), + device=hidden_states_device, + dtype=torch.float32, + ) + silu_and_mul_masked_post_quant_fwd( + gateup_output, + down_input, + down_input_scale, + scale_block_size, + masked_m, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + del gateup_output + + # GroupGemm-1 + n = self.w2_weight.size(1) + down_input_fp8 = ( + down_input, + ( + down_input_scale + if deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + else deep_gemm_wrapper.get_mn_major_tma_aligned_tensor(down_input_scale) + ), + ) + down_output = torch.empty( + (num_groups, m, n), device=hidden_states_device, dtype=torch.bfloat16 + ) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_masked( + down_input_fp8, + self.w2_weight_fp8, + down_output, + masked_m, + expected_m, + ) + del down_input + del down_input_fp8 + + # PostReorder + output = torch.empty( + hidden_states_shape, dtype=hidden_states_dtype, device=hidden_states_device + ) + post_reorder_triton_kernel[(hidden_states_shape[0],)]( + down_output, + output, + src2dst, + topk_ids, + topk_weights, + self.start_expert_id, + self.end_expert_id, + self.top_k, + hidden_states_shape[1], + m_max * self.start_expert_id, + BLOCK_SIZE=512, + ) + if self.moe_runner_config.routed_scaling_factor is not None: + output *= self.moe_runner_config.routed_scaling_factor + return output + + +class DeepEPMoE(EPMoE): + """ + MoE Expert Parallel Impl based on DeepEP (https://github.com/deepseek-ai/DeepEP/tree/main) + """ + + _has_printed = False + + def __init__( + self, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + layer_id: int, + num_fused_shared_experts: int = 0, + params_dtype: Optional[torch.dtype] = None, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + activation: str = "silu", + routed_scaling_factor: Optional[float] = None, + ): + super().__init__( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + layer_id=layer_id, + num_fused_shared_experts=num_fused_shared_experts, + params_dtype=params_dtype, + quant_config=quant_config, + prefix=prefix, + activation=activation, + routed_scaling_factor=routed_scaling_factor, + ) + self.deepep_mode = get_deepep_mode() + + # TODO: move to the beginning of the file + from sglang.srt.distributed.parallel_state import get_tp_group + from sglang.srt.two_batch_overlap import MaybeTboDeepEPDispatcher + + self.deepep_dispatcher = MaybeTboDeepEPDispatcher( + group=get_tp_group().device_group, + router_topk=self.top_k, + permute_fusion=True, + num_experts=self.num_experts, + num_local_experts=self.num_local_experts, + hidden_size=hidden_size, + params_dtype=params_dtype, + deepep_mode=self.deepep_mode, + async_finish=True, # TODO + return_recv_hook=True, + ) + + if self.deepep_mode.enable_low_latency() and not _is_npu: + # NPU supports low_latency deepep without deepgemm + assert ( + deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + ), f"DeepEP {self.deepep_mode} mode requires deep_gemm" + if _use_aiter: + # expert_mask is of size (self.num_local_experts + 1), + # the extra 1 is for invalid rank_id (in original deepep, the invalid rank_id is -1, but aiter does not allow -1, we use a mask to make those ids invalid) + # for instance, if we have 4 experts on this rank, we would have a expert_mask like: + # self.expert_mask = [1, 1, 1, 1, 0] + # idx from 0-3 is valid and will be processed, while idx == 4 will be masked out + self.expert_mask = torch.zeros( + (self.num_local_experts + 1), + device=torch.cuda.current_device(), + dtype=torch.int, + ) + # the last one is invalid rank_id + self.expert_mask[:-1] = 1 + elif not _is_npu: + self.w13_weight_fp8 = ( + self.w13_weight, + ( + self.w13_weight_scale_inv + if self.use_block_quant + else self.w13_weight_scale + ), + ) + self.w2_weight_fp8 = ( + self.w2_weight, + ( + self.w2_weight_scale_inv + if self.use_block_quant + else self.w2_weight_scale + ), + ) + + def forward( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + forward_batch: ForwardBatch, + ): + dispatch_output = self.dispatch( + hidden_states, topk_idx, topk_weights, forward_batch + ) + hidden_states = self.moe_impl(dispatch_output) + hidden_states = self.combine( + hidden_states, + dispatch_output.topk_idx, + dispatch_output.topk_weights, + forward_batch, + ) + return hidden_states + + def dispatch( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + forward_batch: ForwardBatch, + ): + return self.deepep_dispatcher.dispatch( + hidden_states=hidden_states, + topk_idx=topk_idx, + topk_weights=topk_weights, + forward_batch=forward_batch, + ) + + def moe_impl(self, dispatch_output: DispatchOutput): + from sglang.srt.layers.moe.token_dispatcher import DispatchOutputChecker + + if _use_aiter: + assert DispatchOutputChecker.format_is_deepep(dispatch_output) + # in forward_aiter, we skip token permutation and unpermutation, which have been fused inside aiter kernel + return self.forward_aiter(dispatch_output) + if _is_npu: + assert DispatchOutputChecker.format_is_deepep(dispatch_output) + return self.forward_npu(dispatch_output) + if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): + assert deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and self.use_fp8_w8a8 + return self.forward_deepgemm_contiguous(dispatch_output) + elif DispatchOutputChecker.format_is_deepep_ll(dispatch_output): + assert deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM and self.use_fp8_w8a8 + return self.forward_deepgemm_masked(dispatch_output) + else: + raise ValueError( + f"Dispatch output format {dispatch_output.format} is not supported" + ) + + def combine( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + forward_batch: ForwardBatch, + ): + return self.deepep_dispatcher.combine( + hidden_states=hidden_states, + topk_idx=topk_idx, + topk_weights=topk_weights, + forward_batch=forward_batch, + ) + + def forward_aiter( + self, + dispatch_output: Union[DeepEPNormalOutput, DeepEPLLOutput], + ): + hidden_states, topk_idx, topk_weights = ( + dispatch_output.hidden_states, + dispatch_output.topk_idx, + dispatch_output.topk_weights, + ) + if hidden_states.shape[0] == 0: + return hidden_states + # in original deepep, idx == -1 meaning invalid and will not be processed. + # aiter does not accept -1, we use a expert mask to make these idx invalid + # (idx == num_local_experts) meaning not used in aiter fused_moe + topk_idx_copy = topk_idx.to(torch.int32) + topk_idx_copy[topk_idx_copy == -1] = self.num_local_experts + + return fused_moe( + hidden_states, + self.w13_weight, + self.w2_weight, + topk_weights, + topk_idx_copy, + w1_scale=self.w13_weight_scale_inv, + w2_scale=self.w2_weight_scale_inv, + quant_type=QuantType.per_128x128, + activation=( + ActivationType.Silu + if self.moe_runner_config.activation == "silu" + else ActivationType.Gelu + ), + expert_mask=self.expert_mask, + ) + + def forward_deepgemm_contiguous( + self, + dispatch_output: DeepEPNormalOutput, + ): + hidden_states_fp8, topk_idx, topk_weights, num_recv_tokens_per_expert = ( + dispatch_output + ) + hidden_states_fp8, hidden_states_scale = hidden_states_fp8 + assert self.quant_method is not None + assert self.moe_runner_config.activation == "silu" + if num_recv_tokens_per_expert is None: + return hidden_states_fp8.bfloat16() + all_tokens = sum(num_recv_tokens_per_expert) + if all_tokens <= 0: + return hidden_states_fp8.bfloat16() + M, K = hidden_states_fp8.size() + N = self.w13_weight.size(1) + scale_block_size = 128 + + hidden_states_fp8_shape = hidden_states_fp8.shape + hidden_states_fp8_device = hidden_states_fp8.device + hidden_states_fp8_dtype = hidden_states_fp8.dtype + + input_tensor = [ + torch.empty( + (all_tokens, K), + device=hidden_states_fp8.device, + dtype=hidden_states_fp8.dtype, + ), + ( + # TODO check whether need `zeros` + torch.zeros( + (ceil_div(K // 128, 4), all_tokens), + device=hidden_states_fp8.device, + dtype=torch.int, + ).transpose(0, 1) + if deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + else torch.empty( + (all_tokens, K // 128), + device=hidden_states_fp8.device, + dtype=torch.float32, + ) + ), + ] + m_indices = torch.empty( + all_tokens, device=hidden_states_fp8.device, dtype=torch.int32 + ) + output_index = torch.empty_like(topk_idx) + + num_recv_tokens_per_expert_gpu = torch.tensor( + num_recv_tokens_per_expert, + dtype=torch.int32, + pin_memory=True, + device="cpu", + ).cuda(non_blocking=True) + expert_start_loc = torch.empty_like(num_recv_tokens_per_expert_gpu) + + ep_scatter( + hidden_states_fp8, + hidden_states_scale, + topk_idx, + num_recv_tokens_per_expert_gpu, + expert_start_loc, + input_tensor[0], + input_tensor[1], + m_indices, + output_index, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + dispose_tensor(hidden_states_fp8) + + gateup_output = torch.empty( + (all_tokens, N), + device=hidden_states_fp8_device, + dtype=torch.bfloat16, + ) + if not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0: + input_tensor[1] = tma_align_input_scale(input_tensor[1]) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_contig( + input_tensor, self.w13_weight_fp8, gateup_output, m_indices + ) + del input_tensor + down_input = torch.empty( + ( + all_tokens, + N // 2, + ), + device=gateup_output.device, + dtype=torch.bfloat16, + ) + silu_and_mul(gateup_output.view(-1, N), down_input) + del gateup_output + down_output = torch.empty( + (all_tokens, K), + device=hidden_states_fp8_device, + dtype=torch.bfloat16, + ) + down_input_fp8, down_input_scale = sglang_per_token_group_quant_fp8( + down_input, + scale_block_size, + column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + del down_input + if not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0: + down_input_scale = tma_align_input_scale(down_input_scale) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_contig( + (down_input_fp8, down_input_scale), + self.w2_weight_fp8, + down_output, + m_indices, + ) + del down_input_fp8, down_input_scale + + gather_out = torch.empty( + hidden_states_fp8_shape, + device=hidden_states_fp8_device, + dtype=torch.bfloat16, + ) + ep_gather(down_output, topk_idx, topk_weights, output_index, gather_out) + + return gather_out + + def forward_deepgemm_masked( + self, + dispatch_output: DeepEPLLOutput, + ): + hidden_states_fp8, _, _, masked_m, expected_m = dispatch_output + assert self.quant_method is not None + assert self.moe_runner_config.activation == "silu" + + # GroupGemm-0 + num_groups, m, k = hidden_states_fp8[0].size() + n = self.w13_weight.size(1) + expected_m = min(expected_m, m) + gateup_output = torch.empty( + (num_groups, m, n), device=hidden_states_fp8[0].device, dtype=torch.bfloat16 + ) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_masked( + hidden_states_fp8, + self.w13_weight_fp8, + gateup_output, + masked_m, + expected_m, + ) + dispose_tensor(hidden_states_fp8[0]) + + # Act + down_input = torch.empty( + ( + gateup_output.shape[0], + gateup_output.shape[1], + gateup_output.shape[2] // 2, + ), + device=gateup_output.device, + dtype=self.fp8_dtype, + ) + scale_block_size = 128 + down_input_scale = torch.empty( + ( + gateup_output.shape[0], + gateup_output.shape[1], + gateup_output.shape[2] // 2 // scale_block_size, + ), + device=gateup_output.device, + dtype=torch.float32, + ) + silu_and_mul_masked_post_quant_fwd( + gateup_output, + down_input, + down_input_scale, + scale_block_size, + masked_m, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + del gateup_output + + # GroupGemm-1 + n = self.w2_weight.size(1) + down_input_fp8 = ( + down_input, + ( + down_input_scale + if deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + else deep_gemm_wrapper.get_mn_major_tma_aligned_tensor(down_input_scale) + ), + ) + down_output = torch.empty( + (num_groups, m, n), device=down_input.device, dtype=torch.bfloat16 + ) + deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_masked( + down_input_fp8, + self.w2_weight_fp8, + down_output, + masked_m, + expected_m, + ) + + return down_output + + def forward_npu( + self, + dispatch_output: Union[DeepEPNormalOutput, DeepEPLLOutput], + ): + assert self.quant_method is not None + assert self.moe_runner_config.activation == "silu" + + import torch_npu + + from sglang.srt.layers.moe.token_dispatcher import DispatchOutputChecker + + # NOTE: Ascend's Dispatch & Combine does not support FP16 + output_dtype = torch.bfloat16 + group_list_type = 1 + + def _forward_normal(dispatch_output: DeepEPNormalOutput): + if TYPE_CHECKING: + assert isinstance(dispatch_output, DeepEPNormalOutput) + hidden_states, _, _, num_recv_tokens_per_expert = dispatch_output + + if isinstance(hidden_states, tuple): + per_token_scale = hidden_states[1] + hidden_states = hidden_states[0] + else: + # dynamic quant + hidden_states, per_token_scale = torch_npu.npu_dynamic_quant( + hidden_states + ) + + group_list = torch.tensor(num_recv_tokens_per_expert, dtype=torch.int64).to( + hidden_states.device + ) + + # gmm1: gate_up_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[self.w13_weight], + scale=[self.w13_weight_scale.to(output_dtype)], + per_token_scale=[per_token_scale], + split_item=2, + group_list_type=group_list_type, + group_type=0, + group_list=group_list, + output_dtype=output_dtype, + )[0] + + # act_fn: swiglu + hidden_states = torch_npu.npu_swiglu(hidden_states) + hidden_states, swiglu_out_scale = torch_npu.npu_dynamic_quant(hidden_states) + + # gmm2: down_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[self.w2_weight], + scale=[self.w2_weight_scale.to(output_dtype)], + per_token_scale=[swiglu_out_scale], + split_item=2, + group_list_type=group_list_type, + group_type=0, + group_list=group_list, + output_dtype=output_dtype, + )[0] + + return hidden_states + + def _forward_ll(dispatch_output: DeepEPLLOutput): + if TYPE_CHECKING: + assert isinstance(dispatch_output, DeepEPLLOutput) + hidden_states, topk_idx, topk_weights, group_list, _ = dispatch_output + + per_token_scale = hidden_states[1] + hidden_states = hidden_states[0] + + group_list = group_list.to(torch.int64) + + # gmm1: gate_up_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[self.w13_weight], + split_item=2, + group_list_type=group_list_type, + group_type=0, + group_list=group_list, + output_dtype=torch.int32, + )[0] + + # act_fn: swiglu + hidden_states, swiglu_out_scale = torch_npu.npu_dequant_swiglu_quant( + x=hidden_states, + weight_scale=self.w13_weight_scale.to(torch.float32), + activation_scale=per_token_scale, + bias=None, + quant_scale=None, + quant_offset=None, + group_index=group_list, + activate_left=True, + quant_mode=1, + ) + + # gmm2: down_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[self.w2_weight], + scale=[self.w2_weight_scale.to(output_dtype)], + per_token_scale=[swiglu_out_scale], + split_item=2, + group_list_type=group_list_type, + group_type=0, + group_list=group_list, + output_dtype=output_dtype, + )[0] + + return hidden_states + + if DispatchOutputChecker.format_is_deepep_normal(dispatch_output): + return _forward_normal(dispatch_output) + elif DispatchOutputChecker.format_is_deepep_ll(dispatch_output): + return _forward_ll(dispatch_output) + else: + raise ValueError(f"Not Supported DeepEP format {dispatch_output.format}") + + +def get_moe_impl_class(quant_config: Optional[QuantizationConfig] = None): + if get_moe_a2a_backend().is_deepep(): + return DeepEPMoE + + # NEW: Direct FP4 detection (bypasses EP requirements) + # Check for FP4 quantization with TRTLLM flag, regardless of EP + if get_moe_runner_backend().is_flashinfer_trtllm(): + # FlashInferFP4MoE must be paired with ModelOptNvFp4FusedMoEMethod. + # If UnquantizedFusedMoEMethod is detected, fall back to FusedMoE instead. + if quant_config is None: + return FusedMoE + try: + # Check the quantization argument directly + quantization = global_server_args_dict.get("quantization") + if quantization == "modelopt_fp4": + from sglang.srt.layers.moe.fused_moe_triton.layer import ( + FlashInferFP4MoE, + ) + + return FlashInferFP4MoE + except: + pass + + if should_use_flashinfer_trtllm_moe(): + return FlashInferFusedMoE + if get_moe_runner_backend().is_flashinfer_cutlass(): + return FusedMoE + if get_moe_expert_parallel_world_size() > 1: + return EPMoE + return FusedMoE diff --git a/python/sglang/srt/layers/moe/fused_moe_native.py b/python/sglang/srt/layers/moe/fused_moe_native.py new file mode 100644 index 000000000..a3d3a09bf --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_native.py @@ -0,0 +1,101 @@ +""" +Torch-native implementation for FusedMoE. This is used for torch.compile. +It is based on https://github.com/pytorch-labs/gpt-fast/blob/32971d3129541c5bfb4f715abc33d1c5f408d204/mixtral-moe/model.py#L204 +""" + +import torch +from torch.nn import functional as F + +from sglang.srt.layers.activation import GeluAndMul, SiluAndMul +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.token_dispatcher import StandardDispatchOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput + + +def fused_moe_forward_native( + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, +) -> torch.Tensor: + + x, topk_output = dispatch_output + moe_runner_config = layer.moe_runner_config + + if moe_runner_config.apply_router_weight_on_input: + raise NotImplementedError() + + topk_weights, topk_ids, _ = topk_output + + w13_weights = layer.w13_weight[topk_ids] + w1_weights, w3_weights = torch.chunk(w13_weights, 2, dim=2) + w2_weights = layer.w2_weight[topk_ids] + x1 = torch.einsum("ti,taoi -> tao", x, w1_weights) + if moe_runner_config.activation == "silu": + x1 = F.silu(x1) + elif moe_runner_config.activation == "gelu": + x1 = F.gelu(x1) + else: + raise ValueError(f"Unsupported activation: {moe_runner_config.activation=}") + x3 = torch.einsum("ti, taoi -> tao", x, w3_weights) + expert_outs = torch.einsum("tao, taio -> tai", (x1 * x3), w2_weights) + return torch.einsum("tai,ta -> ti", expert_outs, topk_weights.to(expert_outs.dtype)) + + +def moe_forward_native( + layer: torch.nn.Module, + x: torch.Tensor, + topk_output: StandardTopKOutput, + moe_runner_config: MoeRunnerConfig, +) -> torch.Tensor: + + if moe_runner_config.apply_router_weight_on_input: + raise NotImplementedError() + + topk_weights, topk_ids, _ = topk_output + + # Ref code from https://huggingface.co/deepseek-ai/DeepSeek-V2/blob/e0828e3cc0a03408724b80c3cc92c8e072db8d01/modeling_deepseek.py#L589 + len_experts = layer.num_experts + + cnts = topk_ids.new_zeros((topk_ids.shape[0], len_experts)) + cnts.scatter_(1, topk_ids.to(torch.int64), 1) + tokens_per_expert = cnts.sum(dim=0) + idxs = topk_ids.view(-1).argsort() + + sorted_tokens = x[idxs // topk_ids.shape[1]] + tokens_per_expert = tokens_per_expert.cpu().numpy() + + if moe_runner_config.activation == "silu": + act = SiluAndMul() + elif moe_runner_config.activation == "gelu": + act = GeluAndMul() + else: + raise ValueError(f"Unsupported activation: {moe_runner_config.activation=}") + + outputs = [] + start_idx = 0 + for i, num_tokens in enumerate(tokens_per_expert): + end_idx = start_idx + num_tokens + if num_tokens == 0: + continue + tokens_for_this_expert = sorted_tokens[start_idx:end_idx] + + layer_w13_weight = layer.w13_weight[i] + layer_w2_weight = layer.w2_weight[i] + + gate_up = F.linear(tokens_for_this_expert, layer_w13_weight) + gate_up = act(gate_up) + expert_out = F.linear(gate_up, layer_w2_weight) + outputs.append(expert_out) + start_idx = end_idx + + outs = torch.cat(outputs, dim=0) if len(outputs) else sorted_tokens.new_empty(0) + new_x = torch.empty_like(outs) + + new_x[idxs] = outs + final_out = ( + new_x.view(*topk_ids.shape, -1) + .type(topk_weights.dtype) + .mul_(topk_weights.unsqueeze(dim=-1)) + .sum(dim=1) + .type(new_x.dtype) + ) + return final_out diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/__init__.py b/python/sglang/srt/layers/moe/fused_moe_triton/__init__.py new file mode 100644 index 000000000..be3ed3af4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/__init__.py @@ -0,0 +1,42 @@ +from contextlib import contextmanager +from typing import Any, Dict, Optional + +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_experts +from sglang.srt.layers.moe.fused_moe_triton.fused_moe_triton_config import ( + get_config_file_name, + try_get_optimal_moe_config, +) +from sglang.srt.layers.moe.fused_moe_triton.layer import ( + FusedMoE, + FusedMoeWeightScaleSupported, +) +from sglang.srt.layers.moe.fused_moe_triton.moe_align_block_size import ( + moe_align_block_size, +) + +_config: Optional[Dict[str, Any]] = None + + +@contextmanager +def override_config(config): + global _config + old_config = _config + _config = config + yield + _config = old_config + + +def get_config() -> Optional[Dict[str, Any]]: + return _config + + +__all__ = [ + "FusedMoE", + "FusedMoeWeightScaleSupported", + "override_config", + "get_config", + "fused_experts", + "get_config_file_name", + "moe_align_block_size", + "try_get_optimal_moe_config", +] diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/README.md b/python/sglang/srt/layers/moe/fused_moe_triton/configs/README.md new file mode 100644 index 000000000..3679e698a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/README.md @@ -0,0 +1,15 @@ +This directory contains tuned configurations for different settings of the fused_moe kernel. +For different settings of +- E (number of experts) +- N (intermediate size) +- device_name (torch.cuda.get_device_name()) +- dtype: The data type used by the fused MoE kernel for computation. Supported types include fp8_w8a8, int8_w8a8, int8_w8a16, int4_w4a16, etc. This determines the precision and quantization scheme for both weights and activations. +- block_shape: The block quantization shape introduced starting from DeepSeek V3/R1 models. This parameter defines the granularity for block-wise quantization, typically specified as `[block_n, block_k]` where `block_n` and `block_k` represent the block dimensions. For example, DeepSeek V3 commonly uses `[128, 128]` block shapes for efficient block-wise FP8 quantization. + +the JSON file contains a mapping from M (batch size) to the chosen configuration. + +The example configurations provided are for the Mixtral model for TP2 on H100 +and TP4 on A100. Mixtral has intermediate size N = 14336, i.e. for TP2 we have +N = 7168 and for TP4 we have N = 3584. + +See `benchmark/kernels/fused_moe_triton/README.md` on how to generate these config files. diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..56c1a4e3a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..21924c7e7 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..598993c61 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "5120": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "9216": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "13312": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "17408": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "41984": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "50176": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "58368": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..a7c626e9b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..14bf13935 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "5120": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "9216": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "17408": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "41984": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "50176": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "58368": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json new file mode 100644 index 000000000..29972c5f3 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..082fba99b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3072,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..dbed86769 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "5120": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "9216": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "13312": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "17408": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "41984": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "50176": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "58368": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..b89336a26 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..39bc5b9e5 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "5120": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "9216": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "17408": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "41984": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "50176": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "58368": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..35d67f837 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=1,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=144,N=512,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=144,N=512,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..587fb2f2e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=144,N=512,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..089894816 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..9814a3819 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1024,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json new file mode 100644 index 000000000..9262a74a4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-40GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..d251f9b5a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..0ecf814a2 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1344,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..6443c5cd0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..1fe9aad96 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=14336,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..9b48fe26d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "17408": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "33792": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "41984": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "50176": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "58368": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..5ff4a1600 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..7e09be1ff --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..039a10ed1 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..3793fcafe --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=2688,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..f1f626667 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json new file mode 100644 index 000000000..6c809a096 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3072,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "1536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..3a9e2ef0f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3200,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,130 @@ +{ + "3328": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "768": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "1792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2560": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2816": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3584": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "3840": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1280": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "2304": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..1bfd02a41 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..0bb90b7e4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,218 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "5120": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "9216": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "13312": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "17408": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "25600": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "33792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "41984": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "50176": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "58368": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..033ba506d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=6400,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,130 @@ +{ + "3840": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "1792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "3584": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2816": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1280": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "768": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "3328": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2560": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "2304": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json new file mode 100644 index 000000000..14723c576 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..9e9d27b1c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json new file mode 100644 index 000000000..6ac5ba746 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=int8_w8a16.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "1536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..c87ca0de8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=16,N=800,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,130 @@ +{ + "2048": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "1792": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "3328": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2560": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 4 + }, + "768": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2816": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2304": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_stages": 2 + }, + "1280": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3840": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3584": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..09d812937 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=160,N=192,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=20,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=20,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..d471f9821 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=20,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=24,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=24,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..fe8dab46e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=24,N=1024,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1cd253d11 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..9acb73b02 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..ddd55abe0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..4ce62139b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..2e692a158 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,block_shape=[128, 128].json new file mode 100644 index 000000000..4e7a2a1a0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..857d11e48 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..90facc4a8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e25d0492d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e25d0492d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5f32e830a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..2840e9f47 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c6c98c28e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6fcf40875 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..4dcfefa5b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json new file mode 100644 index 000000000..ac9349fef --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L20,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json new file mode 100644 index 000000000..b99349fb8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=256,N=64,device_name=NVIDIA_L40S,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e9c5b9251 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..8cc6c643f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..283ffd8ff --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..e2f8164cc --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..ef6a0479c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..67bf2b720 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..da71451a7 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=1280,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..14c6f8c1a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..2b974a78d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..054873f38 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=2560,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..869f75127 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..40d85e43e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..4ebd7a781 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..90416662e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=320,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3483f05a2 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..b2799ed3a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..8a18afe7d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json new file mode 100644 index 000000000..a1ec0b3ce --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..96437b5ba --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..6e6050694 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..824009a59 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..bf2aa0d5f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=64,N=640,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI300X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI300X.json new file mode 100644 index 000000000..4d4b752fa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI300X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI325X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI325X.json new file mode 100644 index 000000000..4d4b752fa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Instinct_MI325X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Radeon_Graphics.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Radeon_Graphics.json new file mode 100644 index 000000000..4d4b752fa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=AMD_Radeon_Graphics.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 1, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..3f3ccdafa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,138 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..62fd21136 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..da40db132 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=14336,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI300X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI300X.json new file mode 100644 index 000000000..a218fc406 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI300X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI325X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI325X.json new file mode 100644 index 000000000..a218fc406 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Instinct_MI325X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Radeon_Graphics.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Radeon_Graphics.json new file mode 100644 index 000000000..a218fc406 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=AMD_Radeon_Graphics.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json new file mode 100644 index 000000000..f4c0f8417 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-40GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..5c8185cfd --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..97c9f4445 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..9d0db1cdc --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..1c906494a --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=1792,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..0bb423b28 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..555718733 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..26bcbf269 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..430f50090 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..c51565f67 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=2048,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI300X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI300X.json new file mode 100644 index 000000000..3682cc548 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI300X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI325X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI325X.json new file mode 100644 index 000000000..3682cc548 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Instinct_MI325X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Radeon_Graphics.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Radeon_Graphics.json new file mode 100644 index 000000000..3682cc548 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=AMD_Radeon_Graphics.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json new file mode 100644 index 000000000..b41f9d443 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-40GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..edf2a38d1 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json new file mode 100644 index 000000000..7532826a6 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_GeForce_RTX_4090,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..673bae2ba --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..b2100cebb --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..449a13384 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..d317994cb --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_L40S.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_L40S.json new file mode 100644 index 000000000..6499c8586 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=3584,device_name=NVIDIA_L40S.json @@ -0,0 +1,173 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 2, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 7 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 128, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 2, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 2, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 2, + "num_warps": 4, + "num_ctas": 1, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 2 + }, + "192": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 128, + "num_warps": 2, + "num_ctas": 1, + "num_stages": 8 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 2 + }, + "6144": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_ctas": 1, + "num_stages": 2 + }, + "8192": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 16, + "num_ctas": 1, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d7f14d665 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json @@ -0,0 +1,178 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d7f14d665 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json @@ -0,0 +1,178 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d7f14d665 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json @@ -0,0 +1,178 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..dbc624731 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..cc614e635 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..32c0c9da4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..72c3f560b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..dd07b3f6e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=4096,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI300X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI300X.json new file mode 100644 index 000000000..21742854c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI300X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI325X.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI325X.json new file mode 100644 index 000000000..21742854c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Instinct_MI325X.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Radeon_Graphics.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Radeon_Graphics.json new file mode 100644 index 000000000..21742854c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=AMD_Radeon_Graphics.json @@ -0,0 +1,200 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 2, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 4, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 32, + "kpack": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..f578c8d01 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..918f68396 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..e341a6791 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..a841518ca --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..13cc2cee1 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=7168,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d9d2f5eac --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8.json @@ -0,0 +1,175 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d9d2f5eac --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8.json @@ -0,0 +1,175 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json new file mode 100644 index 000000000..d9d2f5eac --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8.json @@ -0,0 +1,175 @@ +{ + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 4, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + }, + "8192": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "16384": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "32768": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 0, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "65536": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2, + "waves_per_eu": 1, + "matrix_instr_nonkdim": 16, + "kpack": 1 + }, + "131072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 2, + "matrix_instr_nonkdim": 16, + "kpack": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json new file mode 100644 index 000000000..34b916e57 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..b50cfc13d --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_1_0/E=8,N=8192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..952b750aa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..5de5605d4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H20.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H20.json new file mode 100644 index 000000000..2221e99cd --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H20.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..74374c573 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=192,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..663f32539 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b34b6e4e8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20.json new file mode 100644 index 000000000..ab169a018 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H20.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b11acb1ee --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..ab6e15552 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=384,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..249359fb9 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=512,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..4d3fcf75c --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..9259aa780 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b4efc9b7e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20.json new file mode 100644 index 000000000..03dfc73b6 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H20.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e7f9143f5 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200.json new file mode 100644 index 000000000..beaac7f64 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=768,device_name=NVIDIA_H200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=96,device_name=NVIDIA_H20.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=96,device_name=NVIDIA_H20.json new file mode 100644 index 000000000..ebff99e26 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=128,N=96,device_name=NVIDIA_H20.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=129,N=352,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=129,N=352,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..ff3fc2d01 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=129,N=352,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=160,N=320,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=160,N=320,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..550d314f0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=160,N=320,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=161,N=192,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=161,N=192,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..763a1657f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=161,N=192,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..63ba315b0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..42d00fe60 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..739979fad --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..0762b9084 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..29b5aab37 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..eb5b9b741 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..741060e32 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..f3f8f7198 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..52b459d2e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=264,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json new file mode 100644 index 000000000..da6cf2a04 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e078fd083 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5ed7ad20f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=64,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=64,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..b00455054 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=272,N=64,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=288,N=64,device_name=NVIDIA_A800-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=288,N=64,device_name=NVIDIA_A800-SXM4-80GB.json new file mode 100644 index 000000000..f8f336e26 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=288,N=64,device_name=NVIDIA_A800-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..e341a6791 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_2_0/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_0/E=16,N=1024,device_name=NVIDIA_B200.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_0/E=16,N=1024,device_name=NVIDIA_B200.json new file mode 100644 index 000000000..1fa444bca --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_0/E=16,N=1024,device_name=NVIDIA_B200.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=352,device_name=NVIDIA_RTX_6000_Ada_Generation,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=352,device_name=NVIDIA_RTX_6000_Ada_Generation,dtype=fp8_w8a8.json new file mode 100644 index 000000000..f8fd97b5e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=352,device_name=NVIDIA_RTX_6000_Ada_Generation,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c4751c00f --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6d6af0808 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..379708af4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20.json new file mode 100644 index 000000000..1f3648ca8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=128,N=768,device_name=NVIDIA_H20.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..adeaacb0e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=192,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=320,device_name=NVIDIA_H20-3e.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=320,device_name=NVIDIA_H20-3e.json new file mode 100644 index 000000000..a3022a054 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=320,device_name=NVIDIA_H20-3e.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..c3c6e0ac0 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=384,device_name=NVIDIA_H200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1eba1e4b8 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=160,N=640,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..8e9668889 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3bac45f89 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c9c8bb66e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..68f6fb5ac --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..8ddf8beae --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..532c16e89 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=257,N=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5ee3d4d35 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..358873315 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..02a119508 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=384,N=256,device_name=NVIDIA_H20-3e,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..faf1aa4d4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b5c45dd72 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=385,N=128,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..6f586d1fa --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_3_1/E=8,N=7168,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c848ea577 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=128,N=384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_B200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_B200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..41d97b17b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_B200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json new file mode 100644 index 000000000..f8fd97b5e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=352,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json new file mode 100644 index 000000000..b962d1950 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=129,N=704,device_name=NVIDIA_B200,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json new file mode 100644 index 000000000..f8fd97b5e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=161,N=384,device_name=NVIDIA_RTX_PRO_6000_Blackwell_Max-Q_Workstation_Edition,dtype=fp8_w8a8.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 256, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4e36c1544 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=256,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a6c635be4 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=64,device_name=NVIDIA_A100-SXM4-80GB.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=64,device_name=NVIDIA_A100-SXM4-80GB.json new file mode 100644 index 000000000..dc8d6d68b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=257,N=64,device_name=NVIDIA_A100-SXM4-80GB.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..486b94720 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=384,N=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..b8f35b62e --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H20-3e.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H20-3e.json new file mode 100644 index 000000000..039d5ade7 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=128,device_name=NVIDIA_H20-3e.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H20-3e.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H20-3e.json new file mode 100644 index 000000000..991b315f7 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=256,device_name=NVIDIA_H20-3e.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H100_80GB_HBM3.json b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H100_80GB_HBM3.json new file mode 100644 index 000000000..64861b390 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/configs/triton_3_4_0/E=512,N=64,device_name=NVIDIA_H100_80GB_HBM3.json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py new file mode 100644 index 000000000..3a64c1457 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe.py @@ -0,0 +1,701 @@ +# NOTE: this file will be separated into sglang/srt/layers/moe/moe_runner/triton_utils.py +# Adapted from https://github.com/vllm-project/vllm/blob/a6221a144af772fd1a68fe7e627935dc53e81738/vllm/model_executor/layers/fused_moe/fused_moe.py + +"""Fused MoE kernel.""" + +from __future__ import annotations + +import functools +import os +from typing import TYPE_CHECKING, List, Optional + +import torch +import triton.language as tl + +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.utils import ( + cpu_has_amx_support, + direct_register_custom_op, + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, +) + +from .fused_moe_triton_config import get_config_dtype_str, try_get_optimal_moe_config +from .fused_moe_triton_kernels import invoke_fused_moe_kernel, moe_sum_reduce_triton +from .moe_align_block_size import moe_align_block_size + +if TYPE_CHECKING: + from sglang.srt.layers.moe.topk import StandardTopKOutput + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _is_cuda: + from sgl_kernel import gelu_and_mul, silu_and_mul +elif _is_cpu and _is_cpu_amx_available: + pass +elif _is_hip: + from sgl_kernel import gelu_and_mul, silu_and_mul + + if _use_aiter: + try: + from aiter import moe_sum + except ImportError: + raise ImportError("aiter is required when SGLANG_USE_AITER is set to True") + else: + from vllm import _custom_ops as vllm_ops + +padding_size = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0 + + +def inplace_fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + activation: int = 0,#0 silu 1 gelu + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_limit: Optional[float] = None, +) -> None: + if isinstance(activation, int): + activation = "silu" if activation == 0 else "gelu" + fused_experts_impl( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + b1, + b2, + True, + activation, + apply_router_weight_on_input, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + use_int4_w4a16, + per_channel_quant, + w1_scale, + w2_scale, + w1_zp, + w2_zp, + a1_scale, + a2_scale, + block_shape, + False, + routed_scaling_factor, + gemm1_alpha, + gemm1_limit, + ) + + +def inplace_fused_experts_fake( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + activation: int = 0,#0 silu 1 gelu + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_limit: Optional[float] = None, +) -> None: + pass + + +direct_register_custom_op( + op_name="inplace_fused_experts", + op_func=inplace_fused_experts, + mutates_args=["hidden_states"], + fake_impl=inplace_fused_experts_fake, +) + + +def outplace_fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + activation: int = 0,#0 silu 1 gelu + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_limit: Optional[float] = None, +) -> torch.Tensor: + if isinstance(activation, int): + activation = "silu" if activation == 0 else "gelu" + return fused_experts_impl( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + b1, + b2, + False, + activation, + apply_router_weight_on_input, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + use_int4_w4a16, + per_channel_quant, + w1_scale, + w2_scale, + w1_zp, + w2_zp, + a1_scale, + a2_scale, + block_shape, + no_combine=no_combine, + routed_scaling_factor=routed_scaling_factor, + gemm1_alpha=gemm1_alpha, + gemm1_limit=gemm1_limit, + ) + + +def outplace_fused_experts_fake( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + activation: int = 0,#0 silu 1 gelu + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_limit: Optional[float] = None, +) -> torch.Tensor: + return torch.empty_like(hidden_states) + + +direct_register_custom_op( + op_name="outplace_fused_experts", + op_func=outplace_fused_experts, + mutates_args=[], + fake_impl=outplace_fused_experts_fake, +) + + +def fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_output: StandardTopKOutput, + moe_runner_config: MoeRunnerConfig, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, +): + topk_weights, topk_ids, _ = topk_output + # 统一把 activation -> int（0=silu, 1=gelu） + act_id = ( + 0 if ( + moe_runner_config.activation == 0 + or (isinstance(moe_runner_config.activation, str) + and moe_runner_config.activation.lower() == "silu") + ) else 1 + ) + if moe_runner_config.inplace: + assert not moe_runner_config.no_combine, "no combine + inplace makes no sense" + torch.ops.sglang.inplace_fused_experts( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + b1, + b2, + # moe_runner_config.activation, + act_id, + moe_runner_config.apply_router_weight_on_input, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + use_int4_w4a16, + per_channel_quant, + w1_scale, + w2_scale, + w1_zp, + w2_zp, + a1_scale, + a2_scale, + block_shape, + moe_runner_config.routed_scaling_factor, + moe_runner_config.gemm1_alpha, + moe_runner_config.gemm1_clamp_limit, + ) + return hidden_states + else: + return torch.ops.sglang.outplace_fused_experts( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + b1, + b2, + # moe_runner_config.activation, + act_id, + moe_runner_config.apply_router_weight_on_input, + use_fp8_w8a8, + use_int8_w8a8, + use_int8_w8a16, + use_int4_w4a16, + per_channel_quant, + w1_scale, + w2_scale, + w1_zp, + w2_zp, + a1_scale, + a2_scale, + block_shape, + no_combine=moe_runner_config.no_combine, + routed_scaling_factor=moe_runner_config.routed_scaling_factor, + gemm1_alpha=moe_runner_config.gemm1_alpha, + gemm1_limit=moe_runner_config.gemm1_clamp_limit, + ) + + +@torch.compile +def moe_sum_reduce_torch_compile(x, out, routed_scaling_factor): + torch.sum(x, dim=1, out=out) + out.mul_(routed_scaling_factor) + + +@torch.compile +def swiglu_with_alpha_and_limit(x, gemm1_alpha, gemm1_limit): + gate, up = x[..., ::2], x[..., 1::2] + gate = gate.clamp(min=None, max=gemm1_limit) + up = up.clamp(min=-gemm1_limit, max=gemm1_limit) + return gate * torch.sigmoid(gate * gemm1_alpha) * (up + 1) + + +def fused_experts_impl( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + inplace: bool = False, + activation: int = 0,#0 silu 1 gelu + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_limit: Optional[float] = None, +): + padded_size = padding_size + if not (use_fp8_w8a8 or use_int8_w8a8) or block_shape is not None or _use_aiter: + padded_size = 0 + + # Check constraints. + if use_int4_w4a16: + assert hidden_states.shape[1] // 2 == w1.shape[2], "Hidden size mismatch" + else: + assert ( + hidden_states.shape[1] == w1.shape[2] - padded_size + ), f"Hidden size mismatch" + assert topk_weights.shape == topk_ids.shape, "topk shape mismatch" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.is_contiguous(), "Expert weights1 must be contiguous" + assert w2.is_contiguous(), "Expert weights2 must be contiguous" + assert hidden_states.dtype in [torch.float32, torch.float16, torch.bfloat16] + + num_tokens, _ = hidden_states.shape + E, N, _ = w1.shape + # We execute the fused_moe kernel in chunks to circumvent this issue: + # https://github.com/vllm-project/vllm/issues/5938 + CHUNK_SIZE = 64 * 1024 + M = min(num_tokens, CHUNK_SIZE) + config_dtype = get_config_dtype_str( + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + dtype=hidden_states.dtype, + ) + + get_config_func = functools.partial( + try_get_optimal_moe_config, + w1.shape, + (w2.shape[0], w2.shape[1], w2.shape[2] - padded_size), + topk_ids.shape[1], + config_dtype, + block_shape=block_shape, + ) + + config = get_config_func(M) + + cache = torch.empty( + M * topk_ids.shape[1] * max(N, w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache1 = cache[: M * topk_ids.shape[1] * N].view( + (M, topk_ids.shape[1], N), + ) + intermediate_cache2 = torch.empty( + (M * topk_ids.shape[1], N // 2), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache3 = cache[: M * topk_ids.shape[1] * w2.shape[1]].view( + (M, topk_ids.shape[1], w2.shape[1]), + ) + + compute_type = tl.bfloat16 if hidden_states.dtype == torch.bfloat16 else tl.float16 + + if no_combine: + assert not inplace + out_hidden_states = torch.empty( + (num_tokens, topk_ids.shape[1], w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + elif inplace: + out_hidden_states = hidden_states + else: + out_hidden_states = torch.empty_like(hidden_states) + + for chunk in range((num_tokens // CHUNK_SIZE) + 1): + begin_chunk_idx, end_chunk_idx = ( + chunk * CHUNK_SIZE, + min((chunk + 1) * CHUNK_SIZE, num_tokens), + ) + curr_hidden_states = hidden_states[begin_chunk_idx:end_chunk_idx] + tokens_in_chunk, _ = curr_hidden_states.shape + + if tokens_in_chunk == 0: + break + + if tokens_in_chunk < CHUNK_SIZE and chunk > 0: + # Adjust the intermediate cache size and config for the last + # chunk. Note that in most cases we only have one chunk + # so the cache size and config are already set correctly and + # do not need to be adjusted. + intermediate_cache1 = intermediate_cache1[:tokens_in_chunk] + intermediate_cache2 = intermediate_cache2[ + : tokens_in_chunk * topk_ids.shape[1] + ] + intermediate_cache3 = intermediate_cache3[:tokens_in_chunk] + config = get_config_func(tokens_in_chunk) + + curr_topk_ids = topk_ids[begin_chunk_idx:end_chunk_idx] + curr_topk_weights = topk_weights[begin_chunk_idx:end_chunk_idx] + + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + curr_topk_ids, config["BLOCK_SIZE_M"], E + ) + + invoke_fused_moe_kernel( + curr_hidden_states, + w1, + b1, + intermediate_cache1, + a1_scale, + w1_scale, + w1_zp, + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + apply_router_weight_on_input, + topk_ids.shape[1], + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + if activation == "silu": + if gemm1_alpha is not None: + assert gemm1_limit is not None + intermediate_cache2 = swiglu_with_alpha_and_limit( + intermediate_cache1.view(-1, N), + gemm1_alpha, + gemm1_limit, + ) + elif _is_cuda or _is_hip: + silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + else: + vllm_ops.silu_and_mul( + intermediate_cache2, intermediate_cache1.view(-1, N) + ) + elif activation == "gelu": + assert gemm1_alpha is None, "gemm1_alpha is not supported for gelu" + assert gemm1_limit is None, "gemm1_limit is not supported for gelu" + if _is_cuda or _is_hip: + gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + else: + vllm_ops.gelu_and_mul( + intermediate_cache2, intermediate_cache1.view(-1, N) + ) + else: + raise ValueError(f"Unsupported activation: {activation=}") + + invoke_fused_moe_kernel( + intermediate_cache2, + w2, + b2, + ( + intermediate_cache3 + if not no_combine and topk_ids.shape[1] != 1 + else out_hidden_states[begin_chunk_idx:end_chunk_idx].unsqueeze(0) + ), + a2_scale, + w2_scale, + w2_zp, + curr_topk_weights, + curr_topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + config, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + if routed_scaling_factor is None: + routed_scaling_factor = 1.0 + + if no_combine: + pass + elif _is_cuda: + if topk_ids.shape[1] == 1 and routed_scaling_factor == 1.0: + pass # we write directly into out_hidden_states + elif topk_ids.shape[1] == 2 and routed_scaling_factor == 1.0: + torch.add( + intermediate_cache3[:, 0], + intermediate_cache3[:, 1], + out=out_hidden_states[begin_chunk_idx:end_chunk_idx], + ).squeeze(dim=1) + else: + # According to micro benchmark results, torch.compile can get better performance for small token. + if tokens_in_chunk <= 32: + moe_sum_reduce_torch_compile( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + else: + moe_sum_reduce_triton( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + elif _is_hip: + if _use_aiter: + moe_sum( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + ) + else: + # According to micro benchmark results, torch.compile can get better performance for small token. + if tokens_in_chunk <= 32: + moe_sum_reduce_torch_compile( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + else: + moe_sum_reduce_triton( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + routed_scaling_factor, + ) + else: + vllm_ops.moe_sum( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states[begin_chunk_idx:end_chunk_idx], + ) + + return out_hidden_states + + +def fused_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_output: StandardTopKOutput, + moe_runner_config: MoeRunnerConfig = MoeRunnerConfig(), + b1: Optional[torch.Tensor] = None, + b2: Optional[torch.Tensor] = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + use_int4_w4a16: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + w1_zp: Optional[torch.Tensor] = None, + w2_zp: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, +) -> torch.Tensor: + """ + This function computes a Mixture of Experts (MoE) layer using two sets of + weights, w1 and w2, and top-k gating mechanism. + + Parameters: + - hidden_states (torch.Tensor): The input tensor to the MoE layer. + - w1 (torch.Tensor): The first set of expert weights. + - w2 (torch.Tensor): The second set of expert weights. + - topk_output (StandardTopKOutput): The top-k output of the experts. + - moe_runner_config (MoeRunnerConfig): The configuration for the MoE runner. + - b1 (Optional[torch.Tensor]): Optional bias for w1. + - b2 (Optional[torch.Tensor]): Optional bias for w2. + - use_fp8_w8a8 (bool): If True, use fp8 arithmetic to compute the inner + products for w1 and w2. Defaults to False. + - use_int8_w8a8 (bool): If True, use int8 arithmetic to compute the inner + products for w1 and w2. Defaults to False. + - use_int8_w8a16 (bool): If True, use fp8 arithmetic to compute the inner + products for w1 and w2. Defaults to False. + - use_int4_w4a16 (bool): If True, use matmul of int4 weight and bf16/fp16 + activation to compute the inner products for w1 and w2. + Defaults to False. + - w1_scale (Optional[torch.Tensor]): Optional scale to be used for + w1. + - w2_scale (Optional[torch.Tensor]): Optional scale to be used for + w2. + - a1_scale (Optional[torch.Tensor]): Optional scale to be used for + a1. + - a2_scale (Optional[torch.Tensor]): Optional scale to be used for + a2. + - block_shape: (Optional[List[int]]): Optional block size for block-wise + quantization. + - gemm1_alpha (Optional[float]): Optional gemm1_alpha for the activation + function. + - gemm1_limit (Optional[float]): Optional gemm1_limit for the swiglu activation + function. + + Returns: + - torch.Tensor: The output tensor after applying the MoE layer. + """ + + return fused_experts( + hidden_states, + w1, + w2, + topk_output, + moe_runner_config=moe_runner_config, + b1=b1, + b2=b2, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + per_channel_quant=per_channel_quant, + w1_scale=w1_scale, + w2_scale=w2_scale, + w1_zp=w1_zp, + w2_zp=w2_zp, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_config.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_config.py new file mode 100644 index 000000000..51114aade --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_config.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import functools +import json +import logging +import os +from typing import Any, Dict, List, Optional, Tuple + +import torch +import triton + +from sglang.srt.utils import get_device_name, is_hip + +logger = logging.getLogger(__name__) +_is_hip = is_hip() + + +def get_config_file_name( + E: int, N: int, dtype: Optional[str], block_shape: Optional[int] = None +) -> str: + device_name = get_device_name().replace(" ", "_") + dtype_selector = "" if not dtype else f",dtype={dtype}" + block_shape_selector = ( + "" if not block_shape or not all(block_shape) else f",block_shape={block_shape}" + ) + return f"E={E},N={N},device_name={device_name}{dtype_selector}{block_shape_selector}.json" + + +@functools.lru_cache +def get_moe_configs( + E: int, + N: int, + dtype: Optional[str], + block_n: Optional[int] = 0, + block_k: Optional[int] = 0, +) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the fused MoE kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the fused_moe kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + # Supported Triton versions, should be sorted from the newest to the oldest + supported_triton_versions = ["3.3.1", "3.2.0", "3.1.0"] + + # First look up if an optimized configuration is available in the configs + # directory + json_file_name = get_config_file_name(E, N, dtype, [block_n, block_k]) + + # We found that using the fused_moe_kernel config from Triton 3.1.0 with Triton 3.2.0 results in negative performance gains, + # so we also include the Triton version as a key for finding the fused_moe_kernel config to achieve the best performance. + triton_version = triton.__version__ + version_dir = f"triton_{triton_version.replace('.', '_')}" + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "configs", + version_dir, + json_file_name, + ) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + # Please note that although we find the config files, performance might still be suboptimal. + # This is because the tuning environment might differ from your current environment. + # For example, updating the Triton version might cause all old configs to become suboptimal. + # To achieve the best performance, consider re-tuning the Triton fused MOE kernel in your environment. + # For the tuning method, refer to: https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton + logger.info(f"Using MoE kernel config from {config_file_path}.") + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # Searching for other triton versions that supports the same config + for try_triton_version in supported_triton_versions: + if try_triton_version == triton_version: + continue + try_config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "configs", + f"triton_{try_triton_version.replace('.', '_')}", + json_file_name, + ) + if os.path.exists(try_config_file_path): + with open(try_config_file_path) as f: + logger.warning( + f"Config file not found at {config_file_path}. Fallback to triton version {try_triton_version} and use MoE kernel config from {try_config_file_path}. Performance might be sub-optimal!", + ) + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ( + "Using default MoE kernel config. Performance might be sub-optimal! " + "Config file not found at %s, you can create them with https://github.com/sgl-project/sglang/tree/main/benchmark/kernels/fused_moe_triton" + ), + config_file_path, + ) + return None + + +def get_default_config( + M: int, + E: int, + N: int, + K: int, + topk: int, + dtype: Optional[str], + is_marlin: bool, + block_shape: Optional[List[int]] = None, +) -> Dict[str, int]: + if dtype == "fp8_w8a8": + if block_shape is None: + config = { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 if _is_hip else 4, + } + if M <= E: + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 if _is_hip else 4, + } + else: + # Block-wise quant: BLOCK_SIZE_K must be divisible by block_shape[1] + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": block_shape[0], + "BLOCK_SIZE_K": block_shape[1], + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 if _is_hip else 3, + } + else: + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 32, + "GROUP_SIZE_M": 8, + } + # A heuristic: fused marlin works faster with this config for small M + if M <= E or (is_marlin and M <= 32): + config = { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + } + return config + + +def try_get_optimal_moe_config( + w1_shape: Tuple[int, ...], + w2_shape: Tuple[int, ...], + top_k: int, + dtype: Optional[str], + M: int, + is_marlin: bool = False, + block_shape: Optional[List[int]] = None, +): + from sglang.srt.layers.moe.fused_moe_triton import get_config + + override_config = get_config() + if override_config: + config = override_config + else: + # First try to load optimal config from the file + E, _, N = w2_shape + block_n = block_shape[0] if block_shape else 0 + block_k = block_shape[1] if block_shape else 0 + configs = get_moe_configs(E, N, dtype, block_n, block_k) + + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Else use the default config + config = get_default_config( + M, E, N, w1_shape[2], top_k, dtype, is_marlin, block_shape + ) + return config + + +def get_config_dtype_str( + dtype: torch.dtype, + use_int8_w8a16: Optional[bool] = False, + use_int4_w4a16: Optional[bool] = False, + use_fp8_w8a8: Optional[bool] = False, + use_int8_w8a8: Optional[bool] = False, +): + if use_fp8_w8a8: + return "fp8_w8a8" + elif use_int8_w8a8: + return "int8_w8a8" + elif use_int4_w4a16: + return "int4_w4a16" + elif use_int8_w8a16: + return "int8_w8a16" + elif dtype == torch.float: + # avoiding cases where kernel fails when float32 MoE + # use fp16/bfloat16 configs + return "float32" + return None diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py new file mode 100644 index 000000000..6a7229a9b --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/fused_moe_triton_kernels.py @@ -0,0 +1,799 @@ +from __future__ import annotations + +import os +from typing import Any, Dict, List, Optional + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + scaled_fp8_quant, + sglang_per_token_group_quant_fp8, +) +from sglang.srt.layers.quantization.int8_kernel import ( + per_token_group_quant_int8, + per_token_quant_int8, + sglang_per_token_group_quant_int8, +) +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, +) + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _is_cuda: + pass +elif _is_cpu and _is_cpu_amx_available: + pass +elif _is_hip: + pass + +padding_size = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0 + + +@triton.jit +def write_zeros_to_output( + c_ptr, + stride_cm, + stride_cn, + pid_n, + N, + offs_token, + token_mask, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + compute_type, +): + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=compute_type) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +@triton.jit +def fused_moe_kernel_gptq_awq( + # Pointers to matrices + a_ptr, + b_ptr, + c_ptr, + b_scale_ptr, + b_zp_ptr, + topk_weights_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + num_tokens_post_padded_ptr, + # Matrix dimensions + N: tl.constexpr, + K: tl.constexpr, + EM, + num_valid_tokens, + # The stride variables represent how much to increase the ptr by when + # moving by 1 element in a particular dimension. E.g. `stride_am` is + # how much to increase `a_ptr` by to get the element one row down + # (A has M rows). + stride_am, + stride_ak, + stride_be, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_bse, + stride_bsk, + stride_bsn, + stride_bze, + stride_bzk, + stride_bzn, + group_size: tl.constexpr, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + MUL_ROUTED_WEIGHT: tl.constexpr, + top_k: tl.constexpr, + compute_type: tl.constexpr, + has_zp: tl.constexpr, + use_int4_w4a16: tl.constexpr, + use_int8_w8a16: tl.constexpr, + even_Ks: tl.constexpr, +): + """ + Implements the fused computation for a Mixture of Experts (MOE) using + token and expert matrices. + Key Parameters: + - A: The input tensor representing tokens with shape (*, K), where '*' can + be any shape representing batches and K is the feature dimension of + each token. + - B: The stacked MOE weight tensor with shape (E, N, K), where E is + the number of experts, K is the input feature dimension, and N is + the output feature dimension. + - C: The output cache tensor with shape (M, topk, N), where M is the + total number of tokens post padding, topk is the number of times + each token is repeated, and N is the output feature dimension. + - sorted_token_ids: A tensor containing the sorted indices of tokens, + repeated topk times and arranged by the expert index they are + assigned to. + - expert_ids: A tensor containing the indices of the expert for each + block. It determines which expert matrix from B should be used for + each block in A. + This kernel performs the multiplication of a token by its corresponding + expert matrix as determined by `expert_ids`. The sorting of + `sorted_token_ids` by expert index and padding ensures divisibility by + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix + multiplication across different blocks processed by the same expert. + """ + # ----------------------------------------------------------- + # Map program ids `pid` to the block of C it should compute. + # This is done in a grouped ordering to promote L2 data reuse. + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + # ---------------------------------------------------------- + # Create pointers for the first blocks of A and B. + # We will advance this pointer as we move in the K direction + # and accumulate + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr) + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: + return + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id) + token_mask = offs_token < num_valid_tokens + + off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64) + if off_experts == -1: + # ----------------------------------------------------------- + # Write back zeros to the output when the expert is not + # in the current expert parallel rank. + write_zeros_to_output( + c_ptr, + stride_cm, + stride_cn, + pid_n, + N, + offs_token, + token_mask, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + compute_type, + ) + return + + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + ( + offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak + ) + + if use_int4_w4a16: + b_ptrs = ( + b_ptr + + off_experts * stride_be + + (offs_k[:, None] // 2) * stride_bk + + offs_bn[None, :] * stride_bn + ) + b_shifter = (offs_k[:, None] % 2) * 4 + elif use_int8_w8a16: + b_ptrs = ( + b_ptr + + off_experts * stride_be + + offs_k[:, None] * stride_bk + + offs_bn[None, :] * stride_bn + ) + + if not has_zp and use_int4_w4a16: + b_zp_num = 8 + if not has_zp and use_int8_w8a16: + b_zp_num = 128 + elif has_zp and use_int4_w4a16: + b_zp_shifter = (offs_bn[None, :] % 2) * 4 + + # ----------------------------------------------------------- + # Iterate to compute a block of the C matrix. + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block + # of fp32 values for higher accuracy. + # `accumulator` will be converted back to fp16 after the loop. + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + # Load the next block of A and B, generate a mask by checking the + # K dimension. + + if not even_Ks: + k_mask = offs_k[:, None] < K - k * BLOCK_SIZE_K + k_other = 0.0 + else: + k_mask = None + k_other = None + + a = tl.load( + a_ptrs, + mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), + other=0.0, + ) + b = tl.load(b_ptrs) + if use_int4_w4a16: + b = (b >> b_shifter) & 0xF + + b_scale_ptrs = ( + b_scale_ptr + + off_experts * stride_bse + + offs_bn[None, :] * stride_bsn + + ((offs_k[:, None] + BLOCK_SIZE_K * k) // group_size) * stride_bsk + ) + b_scale = tl.load(b_scale_ptrs, mask=k_mask, other=k_other) + b_scale = b_scale.to(tl.float32) + + if has_zp and use_int4_w4a16: + offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size + b_zp_ptrs = ( + b_zp_ptr + + off_experts * stride_bze + + (offs_bn[None, :] // 2) * stride_bzn + + offs_k_true * stride_bzk + ) + b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other) + b_zp = (b_zp >> b_zp_shifter) & 0xF + b_zp = b_zp.to(tl.float32) + elif has_zp and use_int8_w8a16: + offs_k_true = (offs_k[:, None] + BLOCK_SIZE_K * k) // group_size + b_zp_ptrs = ( + b_zp_ptr + + off_experts * stride_bze + + offs_bn[None, :] * stride_bzn + + offs_k_true * stride_bzk + ) + b_zp = tl.load(b_zp_ptrs, mask=k_mask, other=k_other) + b_zp = b_zp.to(tl.float32) + + # We accumulate along the K dimension. + if has_zp: + b = ((b.to(tl.float32) - b_zp) * b_scale).to(compute_type) + else: + b = ((b.to(tl.float32) - b_zp_num) * b_scale).to(compute_type) + accumulator = tl.dot(a, b, acc=accumulator) + + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K * stride_ak + if use_int4_w4a16: + b_ptrs += (BLOCK_SIZE_K // 2) * stride_bk + else: + b_ptrs += BLOCK_SIZE_K * stride_bk + + if MUL_ROUTED_WEIGHT: + moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0) + accumulator = accumulator * moe_weight[:, None] + + accumulator = accumulator.to(compute_type) + # ----------------------------------------------------------- + # Write back the block of the output + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +@triton.jit +def fused_moe_kernel( + # Pointers to matrices + a_ptr, + b_ptr, + bias_ptr, + c_ptr, + a_scale_ptr, + b_scale_ptr, + topk_weights_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + num_tokens_post_padded_ptr, + # Matrix dimensions + N, + K, + EM, + num_valid_tokens, + # The stride variables represent how much to increase the ptr by when + # moving by 1 element in a particular dimension. E.g. `stride_am` is + # how much to increase `a_ptr` by to get the element one row down + # (A has M rows). + stride_am, + stride_ak, + stride_be, + stride_bk, + stride_bn, + stride_bias_e, + stride_bias_n, + stride_cm, + stride_cn, + stride_asm, + stride_ask, + stride_bse, + stride_bsk, + stride_bsn, + # Block size for block-wise quantization + group_n: tl.constexpr, + group_k: tl.constexpr, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + MUL_ROUTED_WEIGHT: tl.constexpr, + top_k: tl.constexpr, + compute_type: tl.constexpr, + use_fp8_w8a8: tl.constexpr, + use_int8_w8a8: tl.constexpr, + use_int8_w8a16: tl.constexpr, + per_channel_quant: tl.constexpr, + even_Ks: tl.constexpr, +): + """ + Implements the fused computation for a Mixture of Experts (MOE) using + token and expert matrices. + + Key Parameters: + - A: The input tensor representing tokens with shape (*, K), where '*' can + be any shape representing batches and K is the feature dimension of + each token. + - B: The stacked MOE weight tensor with shape (E, N, K), where E is + the number of experts, K is the input feature dimension, and N is + the output feature dimension. + - C: The output cache tensor with shape (M, topk, N), where M is the + total number of tokens post padding, topk is the number of times + each token is repeated, and N is the output feature dimension. + - sorted_token_ids: A tensor containing the sorted indices of tokens, + repeated topk times and arranged by the expert index they are + assigned to. + - expert_ids: A tensor containing the indices of the expert for each + block. It determines which expert matrix from B should be used for + each block in A. + + This kernel performs the multiplication of a token by its corresponding + expert matrix as determined by `expert_ids`. The sorting of + `sorted_token_ids` by expert index and padding ensures divisibility by + BLOCK_SIZE_M, which is necessary to maintain consistency in block matrix + multiplication across different blocks processed by the same expert. + """ + # ----------------------------------------------------------- + # Map program ids `pid` to the block of C it should compute. + # This is done in a grouped ordering to promote L2 data reuse. + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + # ---------------------------------------------------------- + # Create pointers for the first blocks of A and B. + # We will advance this pointer as we move in the K direction + # and accumulate + # `a_ptrs` is a block of [BLOCK_SIZE_M, BLOCK_SIZE_K] pointers + # `b_ptrs` is a block of [BLOCK_SIZE_K, BLOCK_SIZE_N] pointers + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr) + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: + return + offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + offs_token = tl.load(sorted_token_ids_ptr + offs_token_id) + offs_token = offs_token.to(tl.int64) + token_mask = offs_token < num_valid_tokens + + off_experts = tl.load(expert_ids_ptr + pid_m).to(tl.int64) + + if off_experts == -1: + # ----------------------------------------------------------- + # Write back zeros to the output when the expert is not + # in the current expert parallel rank. + write_zeros_to_output( + c_ptr, + stride_cm, + stride_cn, + pid_n, + N, + offs_token, + token_mask, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + compute_type, + ) + return + + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = a_ptr + ( + offs_token[:, None] // top_k * stride_am + offs_k[None, :] * stride_ak + ) + + b_ptrs = ( + b_ptr + + off_experts * stride_be + + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + ) + if bias_ptr is not None: + bias = tl.load( + bias_ptr + off_experts * stride_bias_e + offs_bn[None, :] * stride_bias_n + ) + if use_int8_w8a16: + b_scale_ptrs = ( + b_scale_ptr + off_experts * stride_bse + offs_bn[None, :] * stride_bsn + ) + b_scale = tl.load(b_scale_ptrs) + + if use_fp8_w8a8 or use_int8_w8a8: + # block-wise + if group_k > 0 and group_n > 0: + a_scale_ptrs = a_scale_ptr + (offs_token // top_k) * stride_asm + offs_bsn = offs_bn // group_n + b_scale_ptrs = ( + b_scale_ptr + off_experts * stride_bse + offs_bsn * stride_bsn + ) + # channel-wise + elif per_channel_quant: + b_scale_ptrs = ( + b_scale_ptr + off_experts * stride_bse + offs_bn[None, :] * stride_bsn + ) + b_scale = tl.load(b_scale_ptrs) + # Load per-token scale for activations + a_scale_ptrs = a_scale_ptr + (offs_token // top_k) * stride_asm + a_scale = tl.load(a_scale_ptrs, mask=token_mask, other=0.0)[:, None] + # tensor-wise + else: + a_scale = tl.load(a_scale_ptr) + b_scale = tl.load(b_scale_ptr + off_experts) + + # ----------------------------------------------------------- + # Iterate to compute a block of the C matrix. + # We accumulate into a `[BLOCK_SIZE_M, BLOCK_SIZE_N]` block + # of fp32 values for higher accuracy. + # `accumulator` will be converted back to fp16 after the loop. + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + # Load the next block of A and B, generate a mask by checking the + # K dimension. + if even_Ks: + a = tl.load( + a_ptrs, + mask=token_mask[:, None], + other=0.0, + ) + b = tl.load(b_ptrs) + else: + a = tl.load( + a_ptrs, + mask=token_mask[:, None] & (offs_k[None, :] < K - k * BLOCK_SIZE_K), + other=0.0, + ) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + # We accumulate along the K dimension. + if use_int8_w8a16: + accumulator = tl.dot(a, b.to(compute_type), acc=accumulator) + elif use_fp8_w8a8 or use_int8_w8a8: + if group_k > 0 and group_n > 0: + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_scale = tl.load( + a_scale_ptrs + offs_ks * stride_ask, mask=token_mask, other=0.0 + ) + b_scale = tl.load(b_scale_ptrs + offs_ks * stride_bsk) + + accumulator += tl.dot(a, b) * a_scale[:, None] * b_scale[None, :] + else: + if use_fp8_w8a8: + accumulator = tl.dot(a, b, acc=accumulator) + else: + accumulator += tl.dot(a, b) + else: + accumulator += tl.dot(a, b) + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if use_int8_w8a16: + accumulator *= b_scale + elif use_fp8_w8a8 or use_int8_w8a8: + if group_k == 0 or group_n == 0: + accumulator *= a_scale * b_scale + + if bias_ptr is not None: + accumulator += bias + + if MUL_ROUTED_WEIGHT: + moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0) + accumulator *= moe_weight[:, None] + + accumulator = accumulator.to(compute_type) + # ----------------------------------------------------------- + # Write back the block of the output + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + tl.store(c_ptrs, accumulator, mask=c_mask) + + +def invoke_fused_moe_kernel( + A: torch.Tensor, + B: torch.Tensor, + bias: Optional[torch.Tensor], + C: torch.Tensor, + A_scale: Optional[torch.Tensor], + B_scale: Optional[torch.Tensor], + B_zp: Optional[torch.Tensor], + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor, + mul_routed_weight: bool, + top_k: int, + config: Dict[str, Any], + compute_type: tl.dtype, + use_fp8_w8a8: bool, + use_int8_w8a8: bool, + use_int8_w8a16: bool, + use_int4_w4a16: bool, + per_channel_quant: bool, + block_shape: Optional[List[int]] = None, + no_combine: bool = False, +) -> None: + assert topk_weights.stride(1) == 1 + assert sorted_token_ids.stride(0) == 1 + + padded_size = 0 + if use_fp8_w8a8: + assert B_scale is not None + if block_shape is None: + # activation tensor-wise fp8 quantization, dynamic or static + padded_size = padding_size + # activations apply per-token quantization when weights apply per-channel quantization by default + A, A_scale = scaled_fp8_quant( + A, A_scale, use_per_token_if_dynamic=per_channel_quant + ) + else: + # activation block-wise fp8 quantization + assert len(block_shape) == 2 + block_n, block_k = block_shape[0], block_shape[1] + if _is_cuda: + A, A_scale = sglang_per_token_group_quant_fp8(A, block_k) + else: + A, A_scale = per_token_group_quant_fp8(A, block_k) + assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1] + assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2] + assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1] + elif use_int8_w8a8: + assert B_scale is not None + if block_shape is None: + # activation channel-wise int8 quantization + assert ( + per_channel_quant + ), "int8 quantization only supports channel-wise quantization except for block-wise quantization" + A, A_scale = per_token_quant_int8(A) + else: + # activation block-wise int8 quantization + assert len(block_shape) == 2 + block_n, block_k = block_shape[0], block_shape[1] + if _is_cuda: + A, A_scale = sglang_per_token_group_quant_int8(A, block_k) + else: + A, A_scale = per_token_group_quant_int8(A, block_k) + assert triton.cdiv(A.shape[-1], block_k) == A_scale.shape[-1] + assert triton.cdiv(B.shape[-2], block_n) == B_scale.shape[-2] + assert triton.cdiv(B.shape[-1], block_k) == B_scale.shape[-1] + elif use_int8_w8a16 or use_int4_w4a16: + assert B_scale is not None + assert block_shape is None or block_shape[0] == 0 + else: + assert A_scale is None + assert B_scale is None + + grid = lambda META: ( + triton.cdiv(sorted_token_ids.shape[0], META["BLOCK_SIZE_M"]) + * triton.cdiv(B.shape[1], META["BLOCK_SIZE_N"]), + ) + + K = B.shape[2] - padded_size + if K % config["BLOCK_SIZE_K"] == 0: + even_Ks = True + else: + even_Ks = False + + if ( + (use_int8_w8a16 or use_int4_w4a16) + and block_shape is not None + and block_shape[1] > 0 + ): + assert B_scale is not None and B_scale.ndim == 3 + assert B_zp is None or B_zp.ndim == 3 + assert bias is None + fused_moe_kernel_gptq_awq[grid]( + A, + B, + C, + B_scale, + B_zp, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + B.shape[1], + A.shape[1], + sorted_token_ids.shape[0], + topk_ids.numel(), + A.stride(0), + A.stride(1), + B.stride(0), + B.stride(2), + B.stride(1), + C.stride(1), + C.stride(2), + B_scale.stride(0), + B_scale.stride(2), + B_scale.stride(1), + B_zp.stride(0) if B_zp is not None else 0, + B_zp.stride(2) if B_zp is not None else 0, + B_zp.stride(1) if B_zp is not None else 0, + group_size=block_shape[1], + MUL_ROUTED_WEIGHT=mul_routed_weight, + top_k=top_k, + compute_type=compute_type, + has_zp=B_zp is not None, + use_int4_w4a16=use_int4_w4a16, + use_int8_w8a16=use_int8_w8a16, + even_Ks=even_Ks, + **config, + ) + + else: + + fused_moe_kernel[grid]( + A, + B, + bias, + C, + A_scale, + B_scale, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + B.shape[1], + B.shape[2] - padded_size, + sorted_token_ids.shape[0], + topk_ids.numel(), + A.stride(0), + A.stride(1), + B.stride(0), + B.stride(2), + B.stride(1), + bias.stride(0) if bias is not None else 0, + bias.stride(1) if bias is not None else 0, + C.stride(1), + C.stride(2), + A_scale.stride(0) if A_scale is not None and A_scale.ndim == 2 else 0, + A_scale.stride(1) if A_scale is not None and A_scale.ndim == 2 else 0, + B_scale.stride(0) if B_scale is not None and B_scale.ndim >= 2 else 0, + B_scale.stride(2) if B_scale is not None and B_scale.ndim == 3 else 0, + B_scale.stride(1) if B_scale is not None and B_scale.ndim >= 2 else 0, + 0 if block_shape is None else block_shape[0], + 0 if block_shape is None else block_shape[1], + MUL_ROUTED_WEIGHT=mul_routed_weight, + top_k=top_k, + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + per_channel_quant=per_channel_quant, + even_Ks=even_Ks, + **config, + ) + + +# _moe_sum_reduce_kernel kernel modified from https://github.com/ModelTC/lightllm/blob/main/lightllm/common/fused_moe/moe_sum_reduce.py +@triton.jit +def _moe_sum_reduce_kernel( + input_ptr, + input_stride_0, + input_stride_1, + input_stride_2, + output_ptr, + output_stride_0, + output_stride_1, + token_num: int, + topk_num: int, + hidden_dim: int, + routed_scaling_factor: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_DIM: tl.constexpr, + NUM_STAGE: tl.constexpr, +): + input_stride_0 = tl.cast(input_stride_0, dtype=tl.int64) + input_stride_1 = tl.cast(input_stride_1, dtype=tl.int64) + output_stride_0 = tl.cast(output_stride_0, dtype=tl.int64) + + token_block_id = tl.program_id(0) + dim_block_id = tl.program_id(1) + + offs_token = token_block_id * BLOCK_M + tl.arange(0, BLOCK_M) + offs_dim = dim_block_id * BLOCK_DIM + tl.arange(0, BLOCK_DIM) + + mask_token = offs_token < token_num + mask_dim = offs_dim < hidden_dim + + base_ptrs = input_ptr + offs_token[:, None] * input_stride_0 + offs_dim[None, :] + + accumulator = tl.zeros((BLOCK_M, BLOCK_DIM), dtype=tl.float32) + + for i in tl.range(0, topk_num, num_stages=NUM_STAGE): + tile = tl.load( + base_ptrs + i * input_stride_1, + mask=mask_token[:, None] & mask_dim[None, :], + other=0.0, + ) + accumulator += tile.to(tl.float32) + accumulator *= routed_scaling_factor + + # -------- Write back -------- + store_ptrs = output_ptr + offs_token[:, None] * output_stride_0 + offs_dim[None, :] + tl.store( + store_ptrs, + accumulator.to(input_ptr.dtype.element_ty), + mask=mask_token[:, None] & mask_dim[None, :], + ) + + +def moe_sum_reduce_triton( + input: torch.Tensor, output: torch.Tensor, routed_scaling_factor: float +): + assert input.is_contiguous() + assert output.is_contiguous() + + token_num, topk_num, hidden_dim = input.shape + assert output.shape[0] == token_num and output.shape[1] == hidden_dim + + BLOCK_M = 1 + BLOCK_DIM = 2048 + NUM_STAGE = 1 + num_warps = 16 + + grid = ( + triton.cdiv(token_num, BLOCK_M), + triton.cdiv(hidden_dim, BLOCK_DIM), + ) + + _moe_sum_reduce_kernel[grid]( + input, + *input.stride(), + output, + *output.stride(), + token_num=token_num, + topk_num=topk_num, + hidden_dim=hidden_dim, + routed_scaling_factor=routed_scaling_factor, + BLOCK_M=BLOCK_M, + BLOCK_DIM=BLOCK_DIM, + NUM_STAGE=NUM_STAGE, + num_warps=num_warps, + ) + return diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py new file mode 100644 index 000000000..0ea1fa1eb --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -0,0 +1,1078 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/a6221a144af772fd1a68fe7e627935dc53e81738/vllm/model_executor/layers/fused_moe/layer.py + +import logging +from enum import Enum +from typing import List, Optional, Tuple + +import torch + +from sglang.srt.distributed import ( + get_moe_expert_parallel_rank, + get_moe_expert_parallel_world_size, + get_moe_tensor_parallel_rank, + get_moe_tensor_parallel_world_size, + get_tp_group, + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.eplb.expert_location import get_global_expert_location_metadata +from sglang.srt.layers.moe import ( + MoeRunnerConfig, + get_moe_runner_backend, + should_use_flashinfer_trtllm_moe, +) +from sglang.srt.layers.moe.token_dispatcher.standard import ( + CombineInput, + StandardDispatcher, +) +from sglang.srt.layers.moe.topk import TopKOutput, TopKOutputChecker +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod +from sglang.srt.layers.quantization.modelopt_quant import ModelOptNvFp4FusedMoEMethod +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.model_loader.weight_utils import narrow_padded_param_and_loaded_weight +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_flashinfer_available, + is_hip, + next_power_of_2, + round_up, +) + +if is_flashinfer_available(): + from flashinfer import ( + RoutingMethodType, + fp4_quantize, + reorder_rows_for_gated_act_gemm, + shuffle_matrix_a, + shuffle_matrix_sf_a, + ) + +_is_hip = is_hip() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() + + +# Try to import FP4 TRTLLM function if flashinfer is available +trtllm_fp4_block_scale_moe = None +if should_use_flashinfer_trtllm_moe(): + try: + from flashinfer.fused_moe import trtllm_fp4_block_scale_moe + except ImportError: + trtllm_fp4_block_scale_moe = None + +logger = logging.getLogger(__name__) + + +def _is_fp4_quantization_enabled(): + """Check if ModelOpt FP4 quantization is enabled.""" + try: + # Use the same simple check that works for class selection + quantization = global_server_args_dict.get("quantization") + return quantization == "modelopt_fp4" + except: + return False + + +def _get_tile_tokens_dim(num_tokens, top_k, num_experts): + # Guess tokens per expert assuming perfect expert distribution first. + num_tokens_per_expert = (num_tokens * top_k) // num_experts + # And pad the number to the next power of 2. + tile_tokens_dim = next_power_of_2(num_tokens_per_expert) + # Cap to 8-64 tokens per CTA tile as it's the range supported by the kernel. + tile_tokens_dim = min(max(tile_tokens_dim, 8), 64) + return tile_tokens_dim + + +class FusedMoeWeightScaleSupported(Enum): + TENSOR = "tensor" + CHANNEL = "channel" + GROUP = "group" + BLOCK = "block" + + +class FusedMoE(torch.nn.Module): + """FusedMoE layer for MoE models. + + This layer contains both MergedColumnParallel weights (gate_up_proj / + w13) and RowParallelLinear weights (down_proj/ w2). + + Note: Mixtral uses w1, w2, and w3 for gate, up, and down_proj. We + copy that naming convention here and handle any remapping in the + load_weights function in each model implementation. + + Args: + num_experts: Number of experts in the model + top_k: Number of experts selected for each token + hidden_size: Input hidden state size of the transformer + intermediate_size: Intermediate size of the experts + params_dtype: Data type for the parameters. + reduce_results: Whether to apply all_reduce on the output of the layer + quant_config: Quantization configuration. + inplace: suggestion to compute inplace (modify input activation). + """ + + def __init__( + self, + num_experts: int, + hidden_size: int, + intermediate_size: int, + layer_id: int, + top_k: Optional[int] = None, + num_fused_shared_experts: int = 0, + params_dtype: Optional[torch.dtype] = None, + reduce_results: bool = False, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + activation: str = "silu", + apply_router_weight_on_input: bool = False, + use_presharded_weights: bool = False, + inplace: bool = True, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, + gemm1_alpha: Optional[float] = None, + gemm1_clamp_limit: Optional[float] = None, + use_weight_loader_fused: bool = False, + with_bias=False, + ): + super().__init__() + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + + self.layer_id = layer_id + self.top_k = top_k + self.hidden_size = hidden_size + self.num_experts = num_experts + self.num_fused_shared_experts = num_fused_shared_experts + self.expert_map_cpu = None + self.expert_map_gpu = None + + enable_flashinfer_cutlass_moe = get_moe_runner_backend().is_flashinfer_cutlass() + + if enable_flashinfer_cutlass_moe and quant_config is None: + logger.warning("Disable flashinfer MoE when quantization config is None.") + enable_flashinfer_cutlass_moe = False + + self.enable_flashinfer_cutlass_moe = enable_flashinfer_cutlass_moe + self.moe_ep_size = get_moe_expert_parallel_world_size() + self.moe_ep_rank = get_moe_expert_parallel_rank() + self.moe_tp_size = get_moe_tensor_parallel_world_size() + self.moe_tp_rank = get_moe_tensor_parallel_rank() + assert num_experts % self.moe_ep_size == 0 + self.num_local_experts = num_experts // self.moe_ep_size + self.start_expert_id = self.moe_ep_rank * self.num_local_experts + self.end_expert_id = self.start_expert_id + self.num_local_experts - 1 + if self.moe_ep_size > 1: + # TODO(ch-wan): support shared experts fusion + # Create a tensor of size num_experts filled with -1 + self.expert_map_cpu = torch.full( + (self.num_experts,), -1, dtype=torch.int32, device="cpu" + ) + # Create a expert map for the local experts + self.expert_map_cpu[ + self.moe_ep_rank + * self.num_local_experts : (self.moe_ep_rank + 1) + * self.num_local_experts + ] = torch.arange(0, self.num_local_experts, dtype=torch.int32, device="cpu") + + assert intermediate_size % self.moe_tp_size == 0 + self.intermediate_size_per_partition = intermediate_size // self.moe_tp_size + self.reduce_results = reduce_results + self.use_presharded_weights = use_presharded_weights + + self.use_triton_kernels = get_moe_runner_backend().is_triton_kernel() + + self.quant_config = quant_config + self.use_flashinfer_mxfp4_moe = get_moe_runner_backend().is_flashinfer_mxfp4() + # TODO maybe we should remove this `if`, since `Mxfp4MoEMethod` does another round-up logic + if ( + self.quant_config is not None + and self.quant_config.get_name() == "mxfp4" + and self.use_flashinfer_mxfp4_moe + ): + hidden_size = round_up(hidden_size, 256) + self.hidden_size = hidden_size + + self.moe_runner_config = MoeRunnerConfig( + num_experts=num_experts, + num_local_experts=self.num_local_experts, + hidden_size=hidden_size, + intermediate_size_per_partition=self.intermediate_size_per_partition, + layer_id=layer_id, + top_k=top_k, + num_fused_shared_experts=num_fused_shared_experts, + params_dtype=params_dtype, + activation=activation, + apply_router_weight_on_input=apply_router_weight_on_input, + inplace=inplace, + no_combine=no_combine, + routed_scaling_factor=routed_scaling_factor, + gemm1_alpha=gemm1_alpha, + gemm1_clamp_limit=gemm1_clamp_limit, + ) + + if quant_config is None: + self.quant_method: FusedMoEMethodBase = UnquantizedFusedMoEMethod( + self.use_triton_kernels + ) + else: + self.quant_method: FusedMoEMethodBase = quant_config.get_quant_method( + self, prefix + ) + assert self.quant_method is not None + + self.quant_method.create_weights( + layer=self, + num_experts=self.num_local_experts, + hidden_size=hidden_size, + intermediate_size_per_partition=self.intermediate_size_per_partition, + params_dtype=params_dtype, + weight_loader=( + self.weight_loader + if not use_weight_loader_fused + else self.weight_loader_fused + ), + with_bias=with_bias, + ) + + self.quant_method.create_moe_runner(self, self.moe_runner_config) + self.dispatcher = StandardDispatcher() + + def _load_per_tensor_weight_scale( + self, + shard_id: str, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + expert_id: int, + ): + param_data = param.data + # for per tensor weight quantization + if shard_id in ("w1", "w3"): + # We have to keep the weight scales of w1 and w3 because + # we need to re-quantize w1/w3 weights after weight loading. + idx = 0 if shard_id == "w1" else 1 + param_data[expert_id][idx] = loaded_weight + # If we are in the row parallel case (down_proj) + elif shard_id == "w2": + param_data[expert_id] = loaded_weight + + def _load_model_weight_or_group_weight_scale( + self, + shard_dim: int, + expert_data: torch.Tensor, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + is_bias: bool = False, + ): + # Load grouped weight scales for group quantization + # or model weights + if shard_id == "w2": + self._load_w2( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + is_bias=is_bias, + ) + elif shard_id in ("w1", "w3", "w13"): + self._load_w13( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + is_bias=is_bias, + ) + + def _load_per_channel_weight_scale( + self, + expert_data: torch.Tensor, + shard_dim: int, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + ): + # for per channel weight quantization + if shard_id == "w2": + expert_data.copy_(loaded_weight) + elif shard_id in ("w1", "w3"): + self._load_w13( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + + def _load_w13( + self, + expert_data: torch.Tensor, + shard_dim: int, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + is_bias: bool = False, + ): + + # Index the loaded weight for tp sharding. + # gate_up_proj: "MergedColumnParallel", so tp sharding on output_dim + assert shard_id in {"w1", "w3", "w13"} + + if is_bias: + # if this weight is a bias, the last dimension must be the sharded dimension + shard_dim = -1 + + if shard_id in {"w1", "w3"}: + # non-fused version + shard_size = expert_data.shape[shard_dim] // 2 + elif shard_id in {"w13"}: + # fused version + shard_size = expert_data.shape[shard_dim] + else: + raise NotImplementedError + + # Narrow parameter and load. + # w1, gate_proj: Load into first logical weight of w13. + # w3, up_proj: Load into second logical weight of w13. + # trtllm cutlass kernel assumes differently + switch_w13 = getattr(self.quant_method, "load_up_proj_weight_first", False) + if (switch_w13 and shard_id == "w1") or (not switch_w13 and shard_id == "w3"): + start = shard_size + else: + start = 0 + + if _is_cpu: + expert_data, loaded_weight = narrow_padded_param_and_loaded_weight( + expert_data, + loaded_weight, + start, + shard_size * tp_rank, + shard_dim, + shard_size, + not self.use_presharded_weights, + ) + else: + if not self.use_presharded_weights: + if not is_bias and self.use_triton_kernels: + # do not transpose for bias + loaded_weight = loaded_weight.transpose(-2, -1) + loaded_weight = loaded_weight.narrow( + shard_dim, shard_size * tp_rank, shard_size + ) + + expert_data = expert_data.narrow(shard_dim, start, shard_size) + expert_data.copy_(loaded_weight) + + def _load_w2( + self, + expert_data: torch.Tensor, + shard_dim: int, + shard_id: str, + loaded_weight: torch.Tensor, + tp_rank: int, + is_bias: bool = False, + ): + """Load w2 weights for down projection. + + Args: + expert_data: The expert data tensor to load into + shard_dim: The dimension to shard along + shard_id: The shard ID (must be "w2") + loaded_weight: The weight tensor to load from + tp_rank: The tensor parallel rank + """ + if not isinstance(expert_data, torch.Tensor) or not isinstance( + loaded_weight, torch.Tensor + ): + raise ValueError("expert_data and loaded_weight must be torch.Tensor") + + if ( + self.quant_config is not None + and "modelopt" in self.quant_config.get_name() + and (expert_data.dim() != 2 or loaded_weight.dim() != 2) + ): + raise ValueError( + f"Expected 2D tensors, got expert_data shape {expert_data.shape} and loaded_weight shape {loaded_weight.shape}" + ) + + if shard_id != "w2": + raise ValueError(f"shard_id must be 'w2', got {shard_id}") + + # Index the loaded weight for tp sharding. + # down_proj: "RowParallel" so tp sharding on input_dim + # Narrow parameter and load. + if is_bias: + # this expert_data is a bias, not weight, + # for w2_weight_bias in TP, it does not need to be sharded + shard_size = expert_data.shape[-1] + else: + # this parameter is a weight matrix + # for w2 in TP, it shards the input_features, i.e., shard_dim=2 + shard_size = expert_data.shape[shard_dim] + + if _is_cpu: + expert_data, loaded_weight = narrow_padded_param_and_loaded_weight( + expert_data, + loaded_weight, + 0, # param_data_start + shard_size * tp_rank, + shard_dim, + shard_size, + not self.use_presharded_weights, + ) + else: + if not is_bias and not self.use_presharded_weights: + if self.use_triton_kernels: + loaded_weight = loaded_weight.transpose(-2, -1) + loaded_weight = loaded_weight.narrow( + shard_dim, shard_size * tp_rank, shard_size + ) + + # w2, down_proj: Load into only logical weight of w2. + expert_data.copy_(loaded_weight) + + def _load_single_value( + self, param: torch.nn.Parameter, loaded_weight: torch.Tensor, expert_id: int + ): + param_data = param.data + + # Input scales can be loaded directly and should be equal. + param_data[expert_id] = loaded_weight + + def _load_g_idx( + self, + shard_id: str, + expert_data: torch.Tensor, + shard_dim: int, + loaded_weight: torch.Tensor, + tp_rank: int, + ): + + if shard_id == "w2": + self._load_w2( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + else: + assert shard_id in ("w1", "w3") + expert_data.copy_(loaded_weight) + + def _map_global_expert_id_to_local_expert_id(self, expert_id: int) -> int: + if self.expert_map_cpu is None: + return expert_id + return self.expert_map_cpu[expert_id].item() + + def weight_loader( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: Optional[int], + ) -> None: + + # if expert_id is None, then + # all the experts are loaded at the same time + if ( + not expert_id + and self.quant_config is not None + and self.quant_config.get_name() == "mxfp4" + and self.quant_config.is_static_cfg() + ): + if "bias" in weight_name: + dim1 = loaded_weight.shape[1] + param.data[:, :dim1].copy_(loaded_weight) + else: + dim1 = loaded_weight.shape[1] + dim2 = loaded_weight.shape[2] + param.data[:, :dim1, :dim2].copy_(loaded_weight) + return + + global_expert_location_metadata = get_global_expert_location_metadata() + if global_expert_location_metadata is None: + self._weight_loader_impl( + param=param, + loaded_weight=loaded_weight, + weight_name=weight_name, + shard_id=shard_id, + expert_id=expert_id, + ) + return + + if expert_id >= self.num_experts - self.num_fused_shared_experts: + # This is a shared expert. + physical_expert_ids = [expert_id] + else: + physical_expert_ids = ( + global_expert_location_metadata.logical_to_all_physical( + self.layer_id, expert_id + ) + ) + + for physical_expert_id in physical_expert_ids: + self._weight_loader_physical( + param=param, + loaded_weight=loaded_weight, + weight_name=weight_name, + shard_id=shard_id, + expert_id=physical_expert_id, + ) + + def _weight_loader_physical( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + ) -> None: + + expert_id = self._map_global_expert_id_to_local_expert_id(expert_id) + if expert_id == -1: + return + self._weight_loader_impl( + param=param, + loaded_weight=loaded_weight, + weight_name=weight_name, + shard_id=shard_id, + expert_id=expert_id, + ) + + def _weight_loader_impl( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + ) -> None: + + tp_rank = self.moe_tp_rank + + # compressed-tensors checkpoints with packed weights are stored flipped + # TODO (mgoin): check self.quant_method.quant_config.quant_format + # against known CompressionFormat enum values that have this quality + loaded_weight = ( + loaded_weight.t().contiguous() + if ( + self.quant_method.__class__.__name__ + == "CompressedTensorsWNA16MoEMethod" + ) + else loaded_weight + ) + + if shard_id not in ("w1", "w2", "w3"): + raise ValueError( + f"shard_id must be ['w1','w2','w3'] but " f"got {shard_id}." + ) + + # Flashinfer assumes w31 format for w13_weight. Same for the scales. + if should_use_flashinfer_trtllm_moe(): + shard_id = {"w1": "w3", "w3": "w1", "w2": "w2"}[shard_id] + + WEIGHT_SCALE_SUPPORTED = [e.value for e in FusedMoeWeightScaleSupported] + # Fetch the dim to shard the parameter/loaded weight + # based on the shard id. This will be whatever + # dimension intermediate_size is used. + SHARD_ID_TO_SHARDED_DIM = {"w1": 0, "w2": 1, "w3": 0} + + expert_data = param.data[expert_id] + + # is_transposed: if the dim to shard the weight + # should be flipped. Required by GPTQ, compressed-tensors + # should be whatever dimension intermediate_size is + is_transposed = getattr(param, "is_transposed", False) + shard_dim = SHARD_ID_TO_SHARDED_DIM[shard_id] + if self.use_triton_kernels: + is_transposed = True + if is_transposed: + shard_dim = int(not shard_dim) + + # Case input scale: input_scale loading is only supported for fp8 + if "input_scale" in weight_name: + # INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust input_scale for e4m3fnuz (AMD) + if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"): + loaded_weight = loaded_weight * 2.0 + + # this is needed for compressed-tensors only + loaded_weight = loaded_weight.to(param.data.device) + + if ( + ( + "compressed" in self.quant_method.__class__.__name__.lower() + or "w4afp8" in self.quant_config.get_name() + ) + and (param.data[expert_id] != 1).any() + and ((param.data[expert_id] - loaded_weight).abs() > 1e-5).any() + ): + raise ValueError( + "input_scales of w1 and w3 of a layer " + f"must be equal. But got {param.data[expert_id]} " + f"vs. {loaded_weight}" + ) + + self._load_single_value( + param=param, loaded_weight=loaded_weight, expert_id=expert_id + ) + return + + # Case g_idx + if "g_idx" in weight_name: + self._load_g_idx( + shard_dim=0, + shard_id=shard_id, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + return + + if "ModelOpt" in self.quant_method.__class__.__name__: + # Determine per-tensor weight scale patterns based on variant + is_fp4_variant = isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod) + + # FP4 uses "weight_scale_2" for per-tensor, FP8 uses "weight_scale" for per-tensor + per_tensor_conditions = ( + "weight_scale_2" in weight_name + if is_fp4_variant + else "weight_scale" in weight_name + ) or "input_scale" in weight_name + + if per_tensor_conditions: + self._load_per_tensor_weight_scale( + shard_id=shard_id, + param=param, + loaded_weight=loaded_weight, + expert_id=expert_id, + ) + elif "weight" in weight_name: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + return + + # Case weight scales and zero_points + if "scale" in weight_name or "zero" in weight_name or "offset" in weight_name: + # load the weight scales and zp based on the quantization scheme + # supported weight scales/zp can be found in + # FusedMoeWeightScaleSupported + # TODO @dsikka: once hardened, refactor to use vLLM Parameters + # specific to each case + quant_method = getattr(param, "quant_method", None) + if quant_method == FusedMoeWeightScaleSupported.CHANNEL.value: + # INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust INT4 column-wise scaling number to e4m3fnuz (AMD) + if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"): + loaded_weight = loaded_weight * 0.5 + + self._load_per_channel_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + elif quant_method in [ + FusedMoeWeightScaleSupported.GROUP.value, + FusedMoeWeightScaleSupported.BLOCK.value, + ]: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + elif quant_method == FusedMoeWeightScaleSupported.TENSOR.value: + # INT4-FP8 (INT4 MoE Weight, FP8 Compute): Adjust FP8 per-tensor scaling number for e4m3fnuz (AMD) + if _is_hip and get_bool_env_var("SGLANG_INT4_WEIGHT"): + loaded_weight = loaded_weight * 2.0 + + self._load_per_tensor_weight_scale( + shard_id=shard_id, + param=param, + loaded_weight=loaded_weight, + expert_id=expert_id, + ) + else: + raise ValueError( + f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}" + ) + return + + # Case weight_shape + if "weight_shape" in weight_name: + # only required by compressed-tensors + self._load_single_value( + param=param, loaded_weight=loaded_weight, expert_id=expert_id + ) + return + + # Case model weights + if "weight" in weight_name: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + ) + return + + def weight_loader_fused( + self, + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + ) -> None: + tp_rank = self.moe_tp_rank + + if ( + self.quant_config is not None + and self.quant_config.get_name() == "mxfp4" + and self.quant_config.is_static_cfg() + ): + if "bias" in weight_name: + dim1 = loaded_weight.shape[1] + param.data[:, :dim1].copy_(loaded_weight) + elif "scale" in weight_name: + param.data.copy_(loaded_weight) + else: + dim1 = loaded_weight.shape[1] + dim2 = loaded_weight.shape[2] + param.data[:, :dim1, :dim2].copy_(loaded_weight) + return + + # compressed-tensors checkpoints with packed weights are stored flipped + # TODO: check self.quant_method.quant_config.quant_format + # against known CompressionFormat enum values that have this quality + loaded_weight = ( + loaded_weight.t().contiguous() + if ( + self.quant_method.__class__.__name__ + == "CompressedTensorsWNA16MoEMethod" + ) + else loaded_weight + ) + + if shard_id not in ("w13", "w2"): + raise ValueError(f"shard_id must be ['w13','w2'] but " f"got {shard_id}.") + + # Fetch the dim to shard the parameter/loaded weight + # based on the shard id. This will be whatever + # dimension intermediate_size is used. + SHARD_ID_TO_SHARDED_DIM = {"w13": 1, "w2": 2} + SHARD_ID_TO_SHARDED_DIM_TRANSPOSE = {"w13": 2, "w2": 1} + + expert_data = param.data + is_bias = expert_data.dim() == 2 + + # is_transposed: if the dim to shard the weight + # should be flipped. Required by GPTQ, compressed-tensors + # should be whatever dimension intermediate_size is + is_transposed = getattr(param, "is_transposed", False) + + if self.use_triton_kernels: + is_transposed = True + shard_dim = ( + SHARD_ID_TO_SHARDED_DIM[shard_id] + if not is_transposed + else SHARD_ID_TO_SHARDED_DIM_TRANSPOSE[shard_id] + ) + + # Case model weights + if "weight" in weight_name: + self._load_model_weight_or_group_weight_scale( + shard_id=shard_id, + shard_dim=shard_dim, + loaded_weight=loaded_weight, + expert_data=expert_data, + tp_rank=tp_rank, + is_bias=is_bias, + ) + return + else: + logging.warning( + f"Unsupported weight_name {weight_name} for FusedMoE weight_loader_fused. Nothing is loaded." + ) + + def forward(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + origin_hidden_states_dim = hidden_states.shape[-1] + assert self.quant_method is not None + + if self.moe_ep_size > 1 and not self.enable_flashinfer_cutlass_moe: + if self.expert_map_cpu is not None and self.expert_map_gpu is None: + # If we are in EP mode, we need to move the expert map to GPU. + self.expert_map_gpu = self.expert_map_cpu.to(device="cuda") + + if self.expert_map_gpu is not None: + if TopKOutputChecker.format_is_standard(topk_output): + topk_output = topk_output._replace( + topk_ids=self.expert_map_gpu[topk_output.topk_ids] + ) + elif TopKOutputChecker.format_is_triton_kernel(topk_output): + raise NotImplementedError() + + dispatch_output = self.dispatcher.dispatch( + hidden_states=hidden_states, topk_output=topk_output + ) + + # TODO: consider using symmetric memory + combine_input = self.quant_method.apply( + layer=self, + dispatch_output=dispatch_output, + ) + + final_hidden_states = self.dispatcher.combine(combine_input) + + final_hidden_states = final_hidden_states[ + ..., :origin_hidden_states_dim + ].contiguous() + + if self.reduce_results and (self.moe_tp_size > 1 or self.moe_ep_size > 1): + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + + return final_hidden_states + + @classmethod + def make_expert_params_mapping( + cls, + ckpt_gate_proj_name: str, + ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int, + ) -> List[Tuple[str, str, int, str]]: + + return [ + # (param_name, weight_name, expert_id, shard_id) + ( + ( + "experts.w13_" + if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] + else "experts.w2_" + ), + f"experts.{expert_id}.{weight_name}.", + expert_id, + shard_id, + ) + for expert_id in range(num_experts) + for shard_id, weight_name in [ + ("w1", ckpt_gate_proj_name), + ("w2", ckpt_down_proj_name), + ("w3", ckpt_up_proj_name), + ] + ] + + @classmethod + def make_expert_params_mapping_fused( + cls, + ckpt_gate_up_proj_name: str, + ckpt_down_proj_name: str, + ckpt_gate_up_proj_bias_name: str, + ckpt_down_proj_bias_name: str, + ): + return [ + ("experts.w13_weight", f"experts.{ckpt_gate_up_proj_name}", "w13"), + ( + "experts.w13_weight_bias", + f"experts.{ckpt_gate_up_proj_bias_name}", + "w13", + ), + ("experts.w2_weight", f"experts.{ckpt_down_proj_name}", "w2"), + ("experts.w2_weight_bias", f"experts.{ckpt_down_proj_bias_name}", "w2"), + ] + + @classmethod + def make_expert_params_mapping_fused_mxfp4( + cls, + ckpt_gate_up_proj_name: str, + ckpt_down_proj_name: str, + ckpt_gate_up_proj_bias_name: str, + ckpt_down_proj_bias_name: str, + ckpt_gate_up_proj_scale_name: str, + ckpt_down_proj_scale_name: str, + ): + return [ + ("experts.w13_weight", f"experts.{ckpt_gate_up_proj_name}", "w13"), + ( + "experts.w13_weight_bias", + f"experts.{ckpt_gate_up_proj_bias_name}", + "w13", + ), + ("experts.w2_weight", f"experts.{ckpt_down_proj_name}", "w2"), + ("experts.w2_weight_bias", f"experts.{ckpt_down_proj_bias_name}", "w2"), + ( + "experts.w13_weight_scale", + f"experts.{ckpt_gate_up_proj_scale_name}", + "w13", + ), + ("experts.w2_weight_scale", f"experts.{ckpt_down_proj_scale_name}", "w2"), + ] + + @classmethod + def make_expert_input_scale_params_mapping( + cls, + num_experts: int, + ) -> List[Tuple[str, str, int, str]]: + # (param_name, weight_name, expert_id, shard_id) + return [ + ( + "experts.w13_" if shard_id in ["w1", "w3"] else "experts.w2_", + f"experts.{expert_id}.{shard_id}.", + expert_id, + shard_id, + ) + for expert_id in range(num_experts) + for shard_id in ["w1", "w2", "w3"] + ] + + def should_fuse_routed_scaling_factor_in_topk(self): + return isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod) or ( + isinstance(self.quant_method, Fp8MoEMethod) + and self.quant_method.use_cutlass_fused_experts_fp8 + ) + + +class FlashInferFusedMoE(FusedMoE): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.use_flashinfer_trtllm_moe = should_use_flashinfer_trtllm_moe() + + def forward(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + assert self.use_flashinfer_trtllm_moe + assert ( + self.moe_runner_config.activation == "silu" + ), "Only silu is supported for flashinfer blockscale fp8 moe" + assert self.quant_method is not None + assert ( + topk_output.topk_config.renormalize + ), "Renormalize is required for flashinfer blockscale fp8 moe" + assert ( + self.num_fused_shared_experts == 0 + ), "Fused shared experts are not supported for flashinfer blockscale fp8 moe" + + assert TopKOutputChecker.format_is_bypassed(topk_output) + + # Matrix multiply. + final_hidden_states = self.quant_method.apply_with_router_logits( + layer=self, + x=hidden_states, + topk_output=topk_output, + ) + + if self.reduce_results and (self.moe_tp_size > 1 or self.moe_ep_size > 1): + final_hidden_states = tensor_model_parallel_all_reduce(final_hidden_states) + + return final_hidden_states + + +class FlashInferFP4MoE(FusedMoE): + """FP4 TRTLLM MoE implementation using FlashInfer.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # --------------------------------------------------------------------- + # Helper: quantize hidden states to FP4 each forward pass + # --------------------------------------------------------------------- + def _quantize_hidden_states_fp4(self, hidden_states: torch.Tensor): + """ + Quantize hidden states using global scale factor from quantization method. + + Global scale factor is set by ModelOptNvFp4FusedMoEMethod during weight loading. + Only block scales are computed at runtime for efficiency. + + Returns (packed_fp4_uint8, scale_float8_e4m3fn_runtime, global_scale_float32) + """ + + # flashinfer.fp4_quantize returns (packed_uint8, scale_fp8) + # Only the block scales are computed at runtime + hs_fp4_bytes, hs_sf_bytes = fp4_quantize( + hidden_states, + self.w13_input_scale_quant, + 16, # sf_vec_size + False, # use_ue8m0 + False, # is_sf_swizzled_layout + ) + + hs_fp4 = hs_fp4_bytes.reshape( + hidden_states.shape[0], hidden_states.shape[1] // 2 + ) + hs_sf = hs_sf_bytes.view(torch.float8_e4m3fn).reshape(-1) + + return hs_fp4, hs_sf + + def forward(self, hidden_states: torch.Tensor, topk_output: TopKOutput): + """Forward pass using FP4 TRTLLM kernel. + + Args: + hidden_states: Input tensor + topk_output: TopKOutput object with Bypassed format + """ + assert isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod) + + assert TopKOutputChecker.format_is_bypassed(topk_output) + + router_logits = topk_output.router_logits + topk_config = topk_output.topk_config + + hs_fp4, hs_scale_linear = self._quantize_hidden_states_fp4(hidden_states) + + router_logits = router_logits.to(torch.float32) + + result = trtllm_fp4_block_scale_moe( + routing_logits=router_logits, + routing_bias=topk_config.correction_bias.to(hidden_states.dtype), + hidden_states=hs_fp4, + hidden_states_scale=hs_scale_linear.view(torch.float8_e4m3fn).flatten(), + gemm1_weights=self.gemm1_weights_fp4_shuffled.data, + gemm1_weights_scale=self.gemm1_scales_fp4_shuffled.data.view( + torch.float8_e4m3fn + ), + gemm1_bias=None, + gemm1_alpha=None, + gemm1_beta=None, + gemm1_clamp_limit=None, + gemm2_weights=self.gemm2_weights_fp4_shuffled.data, + gemm2_weights_scale=self.gemm2_scales_fp4_shuffled.data.view( + torch.float8_e4m3fn + ), + gemm2_bias=None, + output1_scale_scalar=self.g1_scale_c.data, + output1_scale_gate_scalar=self.g1_alphas.data, + output2_scale_scalar=self.g2_alphas.data, + num_experts=self.num_experts, + top_k=topk_config.top_k, + n_group=topk_config.num_expert_group, + topk_group=topk_config.topk_group, + intermediate_size=self.intermediate_size_per_partition, + local_expert_offset=self.moe_ep_rank * self.num_local_experts, + local_num_experts=self.num_local_experts, + routed_scaling_factor=self.moe_runner_config.routed_scaling_factor, + tile_tokens_dim=_get_tile_tokens_dim( + hidden_states.shape[0], topk_config.top_k, self.num_local_experts + ), + routing_method_type=RoutingMethodType.DeepSeekV3, + do_finalize=True, + )[0] + + return result diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/moe_align_block_size.py b/python/sglang/srt/layers/moe/fused_moe_triton/moe_align_block_size.py new file mode 100644 index 000000000..64d0126d6 --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/moe_align_block_size.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from typing import Tuple + +import torch +import triton + +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() + +if _is_cuda or _is_hip: + from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size + + +def moe_align_block_size( + topk_ids: torch.Tensor, block_size: int, num_experts: int +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Aligns the token distribution across experts to be compatible with block + size for matrix multiplication. + + Parameters: + - topk_ids: A tensor of shape [total_tokens, top_k] representing the + top-k expert indices for each token. + - block_size: The block size used in block matrix multiplication. + - num_experts: The total number of experts. + + Returns: + - sorted_token_ids: A tensor containing the sorted token indices according + to their allocated expert. + - expert_ids: A tensor indicating the assigned expert index for each block. + - num_tokens_post_padded: The total number of tokens after padding, + ensuring divisibility by block_size. + + This function pads the number of tokens that each expert needs to process + so that it is divisible by block_size. + Padding ensures that during block matrix multiplication, the dimensions + align correctly. + + Example: + Given topk_ids = [[2, 3, 4], [1, 2, 4], [1, 3, 4], [1, 2, 3]], + block_size = 4, and num_experts = 4: + - We initially have 12 tokens (after repeating 'top_k' times) and 4 experts, + with each expert needing to process 3 tokens. + - As block_size is 4, we pad 1 token for each expert. + - First, flatten topk_ids to [2, 3, 4, 1, 2, 4, 1, 3, 4, 1, 2, 3]. + - Then append padding tokens [12, 12, 12, 12] for each block. + - After sorting by expert index, we obtain token_ids + [3, 6, 9, 12, 0, 4, 10, 12, 1, 7, 11, 12, 2, 5, 8, 12]. + Tokens 12 are non-existent (padding) and are ignored in + the subsequent matrix multiplication. + - The padding ensures that the total number of tokens is now divisible + by block_size for proper block matrix operations. + """ + max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + sorted_ids = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + max_num_m_blocks = triton.cdiv(max_num_tokens_padded, block_size) + expert_ids = torch.empty( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad = torch.empty((1), dtype=torch.int32, device=topk_ids.device) + + # In EP, expert_ids for filtered experts are -1. We have num_experts + 1 ids in total. + cumsum_buffer = torch.empty( + (num_experts + 2,), dtype=torch.int32, device=topk_ids.device + ) + + # Threshold based on benchmark results + fuse_sorted_ids_padding = sorted_ids.shape[0] <= 4096 + if not fuse_sorted_ids_padding: + sorted_ids.fill_(topk_ids.numel()) + + sgl_moe_align_block_size( + topk_ids, + num_experts + 1, + block_size, + sorted_ids, + expert_ids, + num_tokens_post_pad, + cumsum_buffer, + fuse_sorted_ids_padding, + ) + return sorted_ids, expert_ids, num_tokens_post_pad diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/triton_kernels_moe.py b/python/sglang/srt/layers/moe/fused_moe_triton/triton_kernels_moe.py new file mode 100644 index 000000000..5d39b8bbc --- /dev/null +++ b/python/sglang/srt/layers/moe/fused_moe_triton/triton_kernels_moe.py @@ -0,0 +1,335 @@ +# Adapted from https://github.com/vllm-project/vllm/pull/18595/files#diff-f426a6de78c82ffec568eff6811bfbf0043dab5f87f1a8c0cffdbdcb8a81e035 + +from __future__ import annotations + +from typing import TYPE_CHECKING, Optional + +import torch +from sgl_kernel import gelu_and_mul, silu_and_mul +from triton_kernels.matmul_ogs import ( + FlexCtx, + FnSpecs, + FusedActivation, + PrecisionConfig, + matmul_ogs, +) +from triton_kernels.numerics import InFlexData +from triton_kernels.routing import GatherIndx, RoutingData, ScatterIndx +from triton_kernels.swiglu import swiglu_fn + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig + from sglang.srt.layers.moe.topk import TopKOutput + + +def quantize(w, dtype, dev, **opt): + if dtype == "bf16": + return w.to(torch.bfloat16), InFlexData() + elif dtype == "fp8": + wq = w.to(torch.float8_e4m3fn).transpose(-1, -2).contiguous().transpose(-1, -2) + return ( + wq, + InFlexData(dtype=wq.dtype, scale=w.abs().max().unsqueeze(0)), + MicroscalingCtx(), + ) + else: + assert dtype == "mx4", f"{dtype=}" + swizzle_mx_scale = opt["swizzle_mx_scale"] + swizzle_axis = 2 if swizzle_mx_scale else None + w = w.to(torch.bfloat16) + w, mx_scales, weight_scale_shape = downcast_to_mxfp( + w, torch.uint8, axis=1, swizzle_axis=swizzle_axis + ) + return ( + w, + InFlexData(), + MicroscalingCtx( + weight_scale=mx_scales, + swizzle_mx=swizzle_mx_scale, + actual_weight_scale_shape=weight_scale_shape, + ), + ) + + +def triton_kernel_moe_forward( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_output: TopKOutput, + moe_runner_config: MoeRunnerConfig, + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + per_channel_quant: bool = False, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[list[int]] = None, +) -> torch.Tensor: + + from sglang.srt.layers.moe.topk import TopKOutputChecker + + assert TopKOutputChecker.format_is_triton_kernel(topk_output) + + routing_data, gather_idx, scatter_idx = topk_output + + return triton_kernel_fused_experts( + hidden_states, + w1, + w2, + routing_data, + gather_idx, + scatter_idx, + inplace=False, # triton kernel doesn't support inplace + activation=moe_runner_config.activation, + apply_router_weight_on_input=apply_router_weight_on_input, + use_fp8_w8a8=use_fp8_w8a8, + per_channel_quant=per_channel_quant, + global_num_experts=global_num_experts, + expert_map=expert_map, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + ) + + +# This is a triton implementation of the fused_experts function +def triton_kernel_fused_experts( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + routing_data: RoutingData, + gather_indx: GatherIndx, + scatter_indx: ScatterIndx, + inplace: bool = False, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + per_channel_quant: bool = False, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[list[int]] = None, +) -> torch.Tensor: + + assert use_fp8_w8a8 == False, "use_fp8_w8a8 is not supported" + assert per_channel_quant == False, "per_channel_quant is not supported" + assert expert_map == None, "expert_map is not supported" + assert w1_scale == None, "w1_scale is not supported" + assert w2_scale == None, "w2_scale is not supported" + assert a1_scale == None, "a1_scale is not supported" + assert a2_scale == None, "a2_scale is not supported" + assert block_shape == None, "block_shape is not supported" + + # type check + assert hidden_states.dtype == torch.bfloat16, "hidden_states must be bfloat16" + assert w1.dtype == torch.bfloat16, "w1 must be bfloat16" + assert w2.dtype == torch.bfloat16, "w2 must be bfloat16" + + # Shape check + assert hidden_states.ndim == 2, "hidden_states must be 2D" + assert ( + hidden_states.shape[-1] == w1.shape[-2] + ), f"hidden_states shape[-1] {hidden_states.shape} must be equal to w1 shape[-2] {w1.shape}" + assert ( + w2.shape[-1] == w1.shape[1] + ), f"w2 shape[-1] {w2.shape[-1]} must be equal to w1 shape[1] {w1.shape[1]}" + + # feature check + assert inplace == False, "Inplace is not supported in new triton MoE kernel" + + M, K = hidden_states.shape + E, _, N = w1.shape + n_expts_act = routing_data.n_expts_act + dtype = hidden_states.dtype + + if global_num_experts == -1: + global_num_experts = E + + # consistent with default implementation + intermediate_cache2 = torch.empty( + (M * n_expts_act, N // 2), device="cuda", dtype=dtype + ) + + intermediate_cache1 = matmul_ogs( + hidden_states, + w1, + None, + routing_data, + gather_indx=gather_indx, + gammas=routing_data.gate_scal if apply_router_weight_on_input else None, + ) + + if activation == "silu": + silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + elif activation == "gelu": + gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + else: + raise ValueError(f"Unsupported FusedMoe activation: {activation}") + + intermediate_cache3 = matmul_ogs( + intermediate_cache2, + w2, + None, + routing_data, + scatter_indx=scatter_indx, + gammas=None if apply_router_weight_on_input else routing_data.gate_scal, + ) + + return intermediate_cache3 + + +def triton_kernel_moe_with_bias_forward( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w1_pcg, + b1: torch.Tensor, + w2: torch.Tensor, + w2_pcg, + b2: torch.Tensor, + topk_output: TopKOutput, + moe_runner_config: MoeRunnerConfig, + use_fp8_w8a8: bool = False, + per_channel_quant: bool = False, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[list[int]] = None, +) -> torch.Tensor: + from sglang.srt.layers.moe.topk import TopKOutputChecker + + assert TopKOutputChecker.format_is_triton_kernel(topk_output) + + routing_data, gather_idx, scatter_idx = topk_output + + return triton_kernel_fused_experts_with_bias( + hidden_states, + w1=w1, + w1_pcg=w1_pcg, + b1=b1, + w2=w2, + w2_pcg=w2_pcg, + b2=b2, + routing_data=routing_data, + gather_indx=gather_idx, + scatter_indx=scatter_idx, + inplace=False, # triton kernel doesn't support inplace + activation=moe_runner_config.activation, + use_fp8_w8a8=use_fp8_w8a8, + per_channel_quant=per_channel_quant, + global_num_experts=global_num_experts, + expert_map=expert_map, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + block_shape=block_shape, + gemm1_alpha=moe_runner_config.gemm1_alpha, + gemm1_clamp_limit=moe_runner_config.gemm1_clamp_limit, + ) + + +def triton_kernel_fused_experts_with_bias( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w1_pcg, + b1: torch.Tensor, + w2: torch.Tensor, + w2_pcg, + b2: torch.Tensor, + routing_data: RoutingData, + gather_indx: GatherIndx, + scatter_indx: ScatterIndx, + inplace: bool = False, + activation: str = "silu", + use_fp8_w8a8: bool = False, + per_channel_quant: bool = False, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[list[int]] = None, + gemm1_alpha: Optional[float] = None, + gemm1_clamp_limit: Optional[float] = None, +) -> torch.Tensor: + assert use_fp8_w8a8 == False, "use_fp8_w8a8 is not supported" + assert per_channel_quant == False, "per_channel_quant is not supported" + assert expert_map == None, "expert_map is not supported" + assert w1_scale == None, "w1_scale is not supported" + assert w2_scale == None, "w2_scale is not supported" + assert a1_scale == None, "a1_scale is not supported" + assert a2_scale == None, "a2_scale is not supported" + assert block_shape == None, "block_shape is not supported" + + # type check + assert hidden_states.dtype == torch.bfloat16, "hidden_states must be bfloat16" + for w in (w1, w2): + # TODO assert bf16 or mxfp4 + # assert (w.dtype == torch.bfloat16) or check-is-mxfp4, f"w must be bfloat16 or mxfp4 {w1.dtype=}" + pass + + # Shape check + assert hidden_states.ndim == 2, "hidden_states must be 2D" + assert ( + hidden_states.shape[-1] == w1.shape[-2] + ), f"hidden_states shape[-1] {hidden_states.shape} must be equal to w1 shape[-2] {w1.shape}" + assert ( + w2.shape[-1] == w1.shape[1] + ), f"w2 shape[-1] {w2.shape[-1]} must be equal to w1 shape[1] {w1.shape[1]}" + + # feature check + assert inplace == False, "Inplace is not supported in new triton MoE kernel" + + E, _, _ = w1.shape + + if global_num_experts == -1: + global_num_experts = E + + # TODO maybe completely remove this branch + if w1.dtype == torch.bfloat16: + device = "cuda" + optg = dict() + w1, w1_flex = quantize(w1, "bf16", device, **optg) + w1_pcg = PrecisionConfig(flex_ctx=FlexCtx(rhs_data=w1_flex)) + + w2, w2_flex = quantize(w2, "bf16", device, **optg) + w2_pcg = PrecisionConfig(flex_ctx=FlexCtx(rhs_data=w2_flex)) + + act = FusedActivation( + FnSpecs("swiglu", swiglu_fn, ("alpha", "limit")), + (gemm1_alpha, gemm1_clamp_limit), + 2, + ) + + intermediate_cache = matmul_ogs( + hidden_states, + w1, + b1, + routing_data, + gather_indx=gather_indx, + precision_config=w1_pcg, + gammas=None, + fused_activation=act, + ) + + return matmul_ogs( + intermediate_cache, + w2, + b2, + routing_data, + scatter_indx=scatter_indx, + precision_config=w2_pcg, + gammas=routing_data.gate_scal, + ) diff --git a/python/sglang/srt/layers/moe/moe_runner/__init__.py b/python/sglang/srt/layers/moe/moe_runner/__init__.py new file mode 100644 index 000000000..3320a7875 --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/__init__.py @@ -0,0 +1,4 @@ +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.runner import MoeRunner + +__all__ = ["MoeRunnerConfig", "MoeRunner"] diff --git a/python/sglang/srt/layers/moe/moe_runner/base.py b/python/sglang/srt/layers/moe/moe_runner/base.py new file mode 100644 index 000000000..4d95540e6 --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/base.py @@ -0,0 +1,286 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, Optional, Tuple, TypeGuard + +import torch + +from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner.triton import ( + TritonRunnerCore, + TritonRunnerInput, + TritonRunnerOutput, + ) + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + CombineInputFormat, + DispatchOutput, + DispatchOutputFormat, + ) + + +@dataclass +class MoeRunnerConfig: + + # MoE parameters + num_experts: Optional[int] = None + num_local_experts: Optional[int] = None + hidden_size: Optional[int] = None + intermediate_size_per_partition: Optional[int] = None + layer_id: Optional[int] = None + top_k: Optional[int] = None + num_fused_shared_experts: Optional[int] = None + params_dtype: Optional[torch.dtype] = None + + # Runner configuration + activation: str = "silu" + apply_router_weight_on_input: bool = False + inplace: bool = True + no_combine: bool = False + routed_scaling_factor: Optional[float] = None + gemm1_alpha: Optional[float] = None + gemm1_clamp_limit: Optional[float] = None + + +@dataclass +class RunnerInput(ABC): + + @property + @abstractmethod + def runner_backend(self) -> MoeRunnerBackend: ... + + def runner_backend_is_triton(self) -> TypeGuard[TritonRunnerInput]: + return self.runner_backend == MoeRunnerBackend.TRITON + + +class RunnerOutput(ABC): + + @property + @abstractmethod + def runner_backend(self) -> MoeRunnerBackend: ... + + def runner_backend_is_triton(self) -> TypeGuard[TritonRunnerOutput]: + return self.runner_backend == MoeRunnerBackend.TRITON + + +@dataclass +class MoeQuantInfo(ABC): + """Moe quantization data.""" + + pass + + +class MoeRunnerCore(ABC): + + def __init__(self, config: MoeRunnerConfig): + self.config = config + + @abstractmethod + def run( + self, runner_input: RunnerInput, quant_info: MoeQuantInfo, running_state: dict + ) -> RunnerOutput: + pass + + @property + @abstractmethod + def runner_backend(self) -> MoeRunnerBackend: ... + + def runner_backend_is_triton(self) -> TypeGuard[TritonRunnerCore]: + return self.runner_backend == MoeRunnerBackend.TRITON + + +class FusedOpPool: + + _fused_funcs: dict[str, Callable] = {} + + @classmethod + def register_fused_func( + cls, a2a_backend_name: str, runner_backend_name: str, fused_func: Callable + ): + key = (a2a_backend_name, runner_backend_name) + if key in cls._fused_funcs: + raise ValueError( + f"Fused function for {a2a_backend_name} to {runner_backend_name} is already registered." + ) + assert MoeA2ABackend( + a2a_backend_name + ), f"Invalid dispatch name: {a2a_backend_name}" + assert MoeRunnerBackend( + runner_backend_name + ), f"Invalid runner name: {runner_backend_name}" + cls._fused_funcs[key] = fused_func + + @classmethod + def get_fused_func(cls, dispatch_name: str, runner_name: str) -> Optional[Callable]: + key = (dispatch_name, runner_name) + fused_func = cls._fused_funcs.get(key) + return fused_func + + +class PermuteMethodPool: + + _pre_permute_methods: dict[ + Tuple[DispatchOutputFormat, MoeRunnerBackend], Callable + ] = {} + _post_permute_methods: dict[ + Tuple[MoeRunnerBackend, CombineInputFormat], Callable + ] = {} + + @classmethod + def register_pre_permute( + cls, + dispatch_output_name: str, + runner_backend_name: str, + permute_func: Callable, + ): + """ + Register a customized pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. + + :param dispatch_output_name: The DispatchOutputFormat name. + :param runner_backend_name: The MoeRunnerBackend name. + :param permute_func: The permute function to register. + """ + # TODO: check if registration is valid + key = (dispatch_output_name, runner_backend_name) + if key in cls._pre_permute_methods: + raise ValueError( + f"Pre-permute method for {dispatch_output_name} to {runner_backend_name} is already registered." + ) + cls._pre_permute_methods[key] = permute_func + + @classmethod + def register_post_permute( + cls, + runner_backend_name: str, + combine_input_name: str, + permute_func: Callable, + ): + """ + Register a customized post-permute function for the given MoeRunnerBackend and CombineInputFormat. + + :param runner_backend_name: The MoeRunnerBackend name. + :param combine_input_name: The CombineInputFormat name. + :param permute_func: The permute function to register. + """ + # TODO: check if registration is valid + key = (runner_backend_name, combine_input_name) + if key in cls._post_permute_methods: + raise ValueError( + f"Post-permute method for {runner_backend_name} to {combine_input_name} is already registered." + ) + cls._post_permute_methods[key] = permute_func + + @classmethod + def get_pre_permute( + cls, + dispatch_output_format: DispatchOutputFormat, + runner_input_format: MoeRunnerBackend, + ) -> Callable: + """ + Retrieve the pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. + + :param dispatch_output_format: The DispatchOutputFormat type. + :param runner_input_format: The MoeRunnerBackend type. + :return: The registered permute function or None if not found. + """ + key = (dispatch_output_format, runner_input_format) + pre_permute_func = cls._pre_permute_methods.get(key) + assert ( + pre_permute_func is not None + ), f"Pre-permute function for {dispatch_output_format} to {runner_input_format} is not registered" + return pre_permute_func + + @classmethod + def get_post_permute( + cls, + runner_output_format: MoeRunnerBackend, + combine_input_format: CombineInputFormat, + ) -> Callable: + """ + Retrieve the post-permute function for the given MoeRunnerBackend and CombineInputFormat. + + :param runner_output_format: The MoeRunnerBackend type. + :param combine_input_format: The CombineInputFormat type. + :return: The registered permute function or None if not found. + """ + key = (runner_output_format, combine_input_format) + post_permute_func = cls._post_permute_methods.get(key) + assert ( + post_permute_func is not None + ), f"Post-permute function for {runner_output_format} to {combine_input_format} is not registered" + return post_permute_func + + +def register_fused_func( + a2a_backend_name: str, + runner_backend_name: str, +) -> Callable: + """ + Decorator to register a fused function for the given DispatchOutputFormat and MoeRunnerBackend. + + :param a2a_backend_name: The A2A backend name. + :param runner_backend_name: The MoeRunnerBackend name. + :return: The decorator function. + """ + + def decorator(fused_func: Callable): + FusedOpPool.register_fused_func( + a2a_backend_name, runner_backend_name, fused_func + ) + return fused_func + + return decorator + + +def register_pre_permute( + dispatch_output_name: str, + runner_backend_name: str, +) -> Callable: + """ + Decorator to register a pre-permute function for the given DispatchOutputFormat and MoeRunnerBackend. + + :param dispatch_output_name: The DispatchOutputFormat name. + :param runner_backend_name: The MoeRunnerBackend name. + :return: The decorator function. + """ + + def decorator( + permute_func: Callable[ + [DispatchOutput, MoeQuantInfo, MoeRunnerConfig, dict], RunnerInput + ] + ) -> Callable: + + PermuteMethodPool.register_pre_permute( + dispatch_output_name, runner_backend_name, permute_func + ) + return permute_func + + return decorator + + +def register_post_permute( + runner_backend_name: str, + combine_input_name: str, +) -> Callable: + """ + Decorator to register a post-permute function for the given MoeRunnerBackend and CombineInputFormat. + + :param runner_backend_name: The MoeRunnerBackend name. + :param combine_input_name: The CombineInputFormat name. + :return: The decorator function. + """ + + def decorator( + permute_func: Callable[ + [RunnerOutput, MoeQuantInfo, MoeRunnerConfig, dict], CombineInput + ] + ) -> Callable: + PermuteMethodPool.register_post_permute( + runner_backend_name, combine_input_name, permute_func + ) + return permute_func + + return decorator diff --git a/python/sglang/srt/layers/moe/moe_runner/runner.py b/python/sglang/srt/layers/moe/moe_runner/runner.py new file mode 100644 index 000000000..3b6fcd980 --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/runner.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING + +from sglang.srt.layers.moe.moe_runner.base import ( + FusedOpPool, + MoeRunnerConfig, + PermuteMethodPool, +) +from sglang.srt.layers.moe.moe_runner.triton import TritonRunnerCore +from sglang.srt.layers.moe.utils import get_moe_a2a_backend + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner.base import MoeQuantInfo + from sglang.srt.layers.moe.token_dispatcher.base import CombineInput, DispatchOutput + from sglang.srt.layers.moe.utils import MoeRunnerBackend + +logger = logging.getLogger(__name__) + + +class MoeRunner: + + def __init__(self, runner_backend: MoeRunnerBackend, config: MoeRunnerConfig): + self.runner_backend = runner_backend + self.config = config + + self.fused_func = None + + if runner_backend.is_triton(): + self.runner_core = TritonRunnerCore(config) + else: + raise NotImplementedError(f"Unsupported runner backend: {runner_backend}") + + a2a_backend_name = get_moe_a2a_backend().value + runner_backend_name = runner_backend.value + + self.fused_func = FusedOpPool.get_fused_func( + a2a_backend_name, runner_backend_name + ) + + SGLANG_CI_DISABLE_MOE_FUSED_FUNC = os.environ.get( + "SGLANG_CI_DISABLE_MOE_FUSED_FUNC", "0" + ) + if SGLANG_CI_DISABLE_MOE_FUSED_FUNC == "1": + logger.info( + "SGLANG_CI_DISABLE_MOE_FUSED_FUNC is set to 1, disabling fused func" + ) + self.fused_func = None + + def run( + self, dispatch_output: DispatchOutput, quant_info: MoeQuantInfo + ) -> CombineInput: + + if self.fused_func is not None: + return self.fused_func(dispatch_output, quant_info, self.config) + + dispatch_format = dispatch_output.format.value + runner_format = self.runner_core.runner_backend.value + self.pre_permute_func = PermuteMethodPool.get_pre_permute( + dispatch_format, runner_format + ) + + running_state = {} + runner_input = self.pre_permute_func( + dispatch_output, quant_info, self.config, running_state + ) + runner_output = self.runner_core.run(runner_input, quant_info, running_state) + + runner_format = self.runner_core.runner_backend.value + combine_format = dispatch_output.format.value + self.post_permute_func = PermuteMethodPool.get_post_permute( + runner_format, combine_format + ) + combine_input = self.post_permute_func( + runner_output, quant_info, self.config, running_state + ) + + return combine_input diff --git a/python/sglang/srt/layers/moe/moe_runner/triton.py b/python/sglang/srt/layers/moe/moe_runner/triton.py new file mode 100644 index 000000000..116fdcaa0 --- /dev/null +++ b/python/sglang/srt/layers/moe/moe_runner/triton.py @@ -0,0 +1,448 @@ +from __future__ import annotations + +import functools +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional + +import torch +import triton.language as tl + +from sglang.srt.layers.moe.moe_runner.base import ( + MoeQuantInfo, + MoeRunnerConfig, + MoeRunnerCore, + RunnerInput, + RunnerOutput, + register_fused_func, + register_post_permute, + register_pre_permute, +) +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.utils import cpu_has_amx_support, is_cpu, is_cuda, is_hip + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardCombineInput, + StandardDispatchOutput, + ) + + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +_use_aiter = bool(int(os.getenv("SGLANG_MOE_USE_AITER", "0"))) +_MOE_PADDING_SIZE = 128 if bool(int(os.getenv("SGLANG_MOE_PADDING", "0"))) else 0 + + +if _is_cuda: + from sgl_kernel import gelu_and_mul, silu_and_mul +elif _is_cpu and _is_cpu_amx_available: + pass +elif _is_hip: + from vllm import _custom_ops as vllm_ops # gelu_and_mul, silu_and_mul + + if _use_aiter: + try: + from aiter import moe_sum + except ImportError: + raise ImportError("aiter is required when SGLANG_USE_AITER is set to True") + + +if _is_cuda or _is_hip: + from sgl_kernel import moe_align_block_size as sgl_moe_align_block_size + + +@dataclass +class TritonRunnerInput(RunnerInput): + + hidden_states: torch.Tensor + topk_weights: torch.Tensor + topk_ids: torch.Tensor + sorted_token_ids: torch.Tensor + expert_ids: torch.Tensor + num_tokens_post_padded: torch.Tensor + + @property + def runner_backend(self) -> MoeRunnerBackend: + return MoeRunnerBackend.TRITON + + +@dataclass +class TritonRunnerOutput(RunnerOutput): + + hidden_states: torch.Tensor + + @property + def runner_backend(self) -> MoeRunnerBackend: + return MoeRunnerBackend.TRITON + + +@dataclass +class TritonMoeQuantInfo(MoeQuantInfo): + w13_weight: torch.Tensor + w2_weight: torch.Tensor + b13: Optional[torch.Tensor] = None + b2: Optional[torch.Tensor] = None + use_fp8_w8a8: bool = False + use_int8_w8a8: bool = False + use_int8_w8a16: bool = False + use_int4_w4a16: bool = False + per_channel_quant: bool = False + w13_scale: Optional[torch.Tensor] = None + w2_scale: Optional[torch.Tensor] = None + w13_zp: Optional[torch.Tensor] = None + w2_zp: Optional[torch.Tensor] = None + a13_scale: Optional[torch.Tensor] = None + a2_scale: Optional[torch.Tensor] = None + block_shape: Optional[List[int]] = None + + +class TritonRunnerCore(MoeRunnerCore): + + def __init__(self, config: MoeRunnerConfig): + super().__init__(config) + + def run( + self, + runner_input: TritonRunnerInput, + quant_info: TritonMoeQuantInfo, + running_state: dict, + ) -> TritonRunnerOutput: + + # TODO: move these functions to the triton runner + from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + invoke_fused_moe_kernel, + moe_sum_reduce_torch_compile, + moe_sum_reduce_triton, + swiglu_with_alpha_and_limit, + ) + + hidden_states = runner_input.hidden_states + topk_weights = runner_input.topk_weights + topk_ids = runner_input.topk_ids + sorted_token_ids = runner_input.sorted_token_ids + expert_ids = runner_input.expert_ids + num_tokens_post_padded = runner_input.num_tokens_post_padded + + w13 = quant_info.w13_weight + w2 = quant_info.w2_weight + b13 = quant_info.b13 + b2 = quant_info.b2 + a13_scale = quant_info.a13_scale + a2_scale = quant_info.a2_scale + w13_scale = quant_info.w13_scale + w2_scale = quant_info.w2_scale + w13_zp = quant_info.w13_zp + w2_zp = quant_info.w2_zp + block_shape = quant_info.block_shape + per_channel_quant = quant_info.per_channel_quant + use_fp8_w8a8 = quant_info.use_fp8_w8a8 + use_int8_w8a8 = quant_info.use_int8_w8a8 + use_int8_w8a16 = quant_info.use_int8_w8a16 + use_int4_w4a16 = quant_info.use_int4_w4a16 + + activation = self.config.activation + no_combine = self.config.no_combine + inplace = self.config.inplace + gemm1_alpha = self.config.gemm1_alpha + gemm1_limit = self.config.gemm1_clamp_limit + routed_scaling_factor = self.config.routed_scaling_factor + apply_router_weight_on_input = self.config.apply_router_weight_on_input + + M = hidden_states.shape[0] + E, N, _ = w13.shape + compute_type = ( + tl.bfloat16 if hidden_states.dtype == torch.bfloat16 else tl.float16 + ) + + intermediate_cache1 = torch.empty( + (M, topk_ids.shape[1], N), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + invoke_fused_moe_kernel( + hidden_states, + w13, + b13, + intermediate_cache1, + a13_scale, + w13_scale, + w13_zp, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + apply_router_weight_on_input, + topk_ids.shape[1], + running_state["config"], + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + intermediate_cache2 = torch.empty( + (M * topk_ids.shape[1], N // 2), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + if activation == "silu": + if gemm1_alpha is not None: + assert gemm1_limit is not None + intermediate_cache2 = swiglu_with_alpha_and_limit( + intermediate_cache1.view(-1, N), + gemm1_alpha, + gemm1_limit, + ) + elif _is_cuda: + silu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + else: + vllm_ops.silu_and_mul( + intermediate_cache2, intermediate_cache1.view(-1, N) + ) + elif activation == "gelu": + assert gemm1_alpha is None, "gemm1_alpha is not supported for gelu" + assert gemm1_limit is None, "gemm1_limit is not supported for gelu" + if _is_cuda: + gelu_and_mul(intermediate_cache1.view(-1, N), intermediate_cache2) + else: + vllm_ops.gelu_and_mul( + intermediate_cache2, intermediate_cache1.view(-1, N) + ) + else: + raise ValueError(f"Unsupported activation: {activation=}") + + intermediate_cache3 = torch.empty( + (M, topk_ids.shape[1], w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + if no_combine: + assert not inplace + out_hidden_states = torch.empty( + (M, topk_ids.shape[1], w2.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + elif inplace: + out_hidden_states = hidden_states + else: + out_hidden_states = torch.empty_like(hidden_states) + + invoke_fused_moe_kernel( + intermediate_cache2, + w2, + b2, + ( + intermediate_cache3 + if not no_combine and topk_ids.shape[1] != 1 + else out_hidden_states.unsqueeze(0) + ), + a2_scale, + w2_scale, + w2_zp, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + not apply_router_weight_on_input, + 1, + running_state["config"], + compute_type=compute_type, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + use_int4_w4a16=use_int4_w4a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + if routed_scaling_factor is None: + routed_scaling_factor = 1.0 + + if no_combine: + pass + elif _is_cuda: + if topk_ids.shape[1] == 1 and routed_scaling_factor == 1.0: + pass # we write directly into out_hidden_states + elif topk_ids.shape[1] == 2 and routed_scaling_factor == 1.0: + torch.add( + intermediate_cache3[:, 0], + intermediate_cache3[:, 1], + out=out_hidden_states, + ).squeeze(dim=1) + else: + # According to micro benchmark results, torch.compile can get better performance for small token. + if M <= 32: + moe_sum_reduce_torch_compile( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states, + routed_scaling_factor, + ) + else: + moe_sum_reduce_triton( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states, + routed_scaling_factor, + ) + elif _is_hip: + if _use_aiter: + moe_sum( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states, + ) + else: + vllm_ops.moe_sum( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states, + ) + else: + vllm_ops.moe_sum( + intermediate_cache3.view(*intermediate_cache3.shape), + out_hidden_states, + ) + + return TritonRunnerOutput( + hidden_states=out_hidden_states, + ) + + @property + def runner_backend(self) -> MoeRunnerBackend: + return MoeRunnerBackend.TRITON + + +@register_fused_func("none", "triton") +def fused_experts_none_to_triton( + dispatch_output: StandardDispatchOutput, + quant_info: TritonMoeQuantInfo, + runner_config: MoeRunnerConfig, +) -> StandardCombineInput: + from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_experts + from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput + + output = fused_experts( + hidden_states=dispatch_output.hidden_states, + w1=quant_info.w13_weight, + w2=quant_info.w2_weight, + topk_output=dispatch_output.topk_output, + moe_runner_config=runner_config, + b1=quant_info.b13, + b2=quant_info.b2, + use_fp8_w8a8=quant_info.use_fp8_w8a8, + use_int8_w8a8=quant_info.use_int8_w8a8, + use_int8_w8a16=quant_info.use_int8_w8a16, + use_int4_w4a16=quant_info.use_int4_w4a16, + per_channel_quant=quant_info.per_channel_quant, + w1_scale=quant_info.w13_scale, + w2_scale=quant_info.w2_scale, + w1_zp=quant_info.w13_zp, + w2_zp=quant_info.w2_zp, + a1_scale=quant_info.a13_scale, + a2_scale=quant_info.a2_scale, + block_shape=quant_info.block_shape, + ) + + return StandardCombineInput( + hidden_states=output, + ) + + +@register_pre_permute("standard", "triton") +def pre_permute_standard_to_triton( + dispatch_output: StandardDispatchOutput, + quant_info: TritonMoeQuantInfo, + runner_config: MoeRunnerConfig, + running_state: dict, +) -> TritonRunnerInput: + + # NOTE: this is dead code as a fused func for standard format is registered. + # This is left here for testing and examples. + + from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + get_config_dtype_str, + moe_align_block_size, + try_get_optimal_moe_config, + ) + from sglang.srt.layers.moe.topk import TopKOutputChecker + + hidden_states, topk_output = dispatch_output + + assert TopKOutputChecker.format_is_standard(topk_output) + + num_tokens = hidden_states.shape[0] + num_local_experts = runner_config.num_local_experts + + if ( + not (quant_info.use_fp8_w8a8 or quant_info.use_int8_w8a8) + or quant_info.block_shape is not None + or _use_aiter + ): + padding_size = 0 + else: + padding_size = _MOE_PADDING_SIZE + + config_dtype = get_config_dtype_str( + use_fp8_w8a8=quant_info.use_fp8_w8a8, + use_int8_w8a8=quant_info.use_int8_w8a8, + use_int8_w8a16=quant_info.use_int8_w8a16, + use_int4_w4a16=quant_info.use_int4_w4a16, + dtype=hidden_states.dtype, + ) + + get_config_func = functools.partial( + try_get_optimal_moe_config, + quant_info.w13_weight.shape, + ( + num_local_experts, + quant_info.w2_weight.shape[1], + quant_info.w2_weight.shape[2] - padding_size, + ), + topk_output.topk_ids.shape[1], + config_dtype, + block_shape=quant_info.block_shape, + ) + + config = get_config_func(num_tokens) + + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_output.topk_ids, config["BLOCK_SIZE_M"], num_local_experts + ) + + running_state["config"] = config + + return TritonRunnerInput( + hidden_states=hidden_states, + topk_weights=topk_output.topk_weights, + topk_ids=topk_output.topk_ids, + sorted_token_ids=sorted_token_ids, + expert_ids=expert_ids, + num_tokens_post_padded=num_tokens_post_padded, + ) + + +@register_post_permute("triton", "standard") +def post_permute_triton_to_standard( + runner_output: TritonRunnerOutput, + quant_info: TritonMoeQuantInfo, + runner_config: MoeRunnerConfig, + running_state: dict, +) -> StandardCombineInput: + + # NOTE: this is dead code as a fused func for standard format is registered. + # This is left here for testing and examples. + + from sglang.srt.layers.moe.token_dispatcher.standard import StandardCombineInput + + return StandardCombineInput( + hidden_states=runner_output.hidden_states, + ) diff --git a/python/sglang/srt/layers/moe/rocm_moe_utils.py b/python/sglang/srt/layers/moe/rocm_moe_utils.py new file mode 100644 index 000000000..5fe2de1e5 --- /dev/null +++ b/python/sglang/srt/layers/moe/rocm_moe_utils.py @@ -0,0 +1,141 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.9.1rc2/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from enum import IntEnum +from functools import cache +from typing import Optional + +import torch + +from sglang.srt.utils import direct_register_custom_op, get_bool_env_var, is_hip + +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + + +class ActivationMethod(IntEnum): + # This allows interfacing with AITER ActivationType enum + # without importing the ActivationType enum from AITER globally. + SILU = 0 + GELU = 1 + + +def rocm_aiter_asm_moe_tkw1_impl( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + fc1_scale: Optional[torch.Tensor] = None, + fc2_scale: Optional[torch.Tensor] = None, + fc1_smooth_scale: Optional[torch.Tensor] = None, + fc2_smooth_scale: Optional[torch.Tensor] = None, + a16: bool = False, + per_tensor_quant_scale: Optional[torch.Tensor] = None, + expert_mask: Optional[torch.Tensor] = None, + activation_method: int = ActivationMethod.SILU.value, +) -> torch.Tensor: + + from aiter import ActivationType + from aiter.fused_moe_bf16_asm import asm_moe_tkw1 + + activation = ActivationType(activation_method) + + return asm_moe_tkw1( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + fc1_scale=fc1_scale, + fc2_scale=fc2_scale, + fc1_smooth_scale=fc1_smooth_scale, + fc2_smooth_scale=fc2_smooth_scale, + a16=a16, + per_tensor_quant_scale=per_tensor_quant_scale, + expert_mask=expert_mask, + activation=activation, + ) + + +def rocm_aiter_asm_moe_tkw1_fake( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + fc1_scale: Optional[torch.Tensor] = None, + fc2_scale: Optional[torch.Tensor] = None, + fc1_smooth_scale: Optional[torch.Tensor] = None, + fc2_smooth_scale: Optional[torch.Tensor] = None, + a16: bool = False, + per_tensor_quant_scale: Optional[torch.Tensor] = None, + expert_mask: Optional[torch.Tensor] = None, + activation_method: int = ActivationMethod.SILU.value, +) -> torch.Tensor: + return torch.empty_like(hidden_states) + + +if _use_aiter: + + direct_register_custom_op( + op_name="rocm_aiter_asm_moe_tkw1", + op_func=rocm_aiter_asm_moe_tkw1_impl, + mutates_args=[], + fake_impl=rocm_aiter_asm_moe_tkw1_fake, + ) + + +def rocm_fused_experts_tkw1( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + use_fp8_w8a8: bool = False, + per_channel_quant: bool = False, + w1_scale: Optional[torch.Tensor] = None, + w2_scale: Optional[torch.Tensor] = None, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + block_shape: Optional[list[int]] = None, +) -> torch.Tensor: + + activation_method = ( + ActivationMethod.SILU if activation == "silu" else ActivationMethod.GELU + ) + # All AITER Fused MoE kernels are expecting the following datatypes + topk_weights = topk_weights.to(torch.float32) + topk_ids = topk_ids.to(torch.int32) + + # w8a8 per-channel quantization + if per_channel_quant and apply_router_weight_on_input and use_fp8_w8a8: + # AITER tkw1 kernel for FP8 models with `apply_router_weight_on_input` + # This applies topk_weights on the GEMM output of the first FC layer + # rather than the second FC. + assert ( + topk_weights.dim() == 2 + ), "`topk_weights` should be in shape (num_tokens, topk)" + assert topk_weights.shape[-1] == 1, ( + "Only support topk=1 when" " `apply_router_weight_on_input` is True" + ) + + return torch.ops.sglang.rocm_aiter_asm_moe_tkw1( + hidden_states, + w1, + w2, + topk_weights, + topk_ids, + fc1_scale=w1_scale, + fc2_scale=w2_scale, + fc1_smooth_scale=None, + fc2_smooth_scale=None, + a16=False, + per_tensor_quant_scale=None, + expert_mask=None, + activation_method=activation_method, + ) + else: + assert False, "This should not be called." diff --git a/python/sglang/srt/layers/moe/router.py b/python/sglang/srt/layers/moe/router.py new file mode 100644 index 000000000..0138dcdad --- /dev/null +++ b/python/sglang/srt/layers/moe/router.py @@ -0,0 +1,392 @@ +from typing import Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.moe.topk import fused_topk +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + + +@triton.jit +def fused_moe_router_kernel( + input_ptr, # input (bs, hidden_dim) + moe_router_weight_ptr, # input (num_experts, hidden_dim) + topk_weights_ptr, # output (bs, topk) + topk_ids_ptr, # output (bs, topk) + correction_bias_ptr, + is_correction_bias: tl.constexpr, + num_experts: tl.constexpr, + topk: tl.constexpr, + moe_softcapping: tl.constexpr, + moe_renormalize: tl.constexpr, # not supported + hidden_dim: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + pid = tl.program_id(axis=0) + + offsets = tl.arange(0, BLOCK_SIZE) + mask = offsets < hidden_dim + + # moe_router_weight is k major + expert_offsets = tl.arange(0, num_experts)[:, None] + router_mask = mask[None, :] + w_router = tl.load( + moe_router_weight_ptr + expert_offsets * hidden_dim + offsets[None, :], + mask=router_mask, + other=0.0, + ) + + x = tl.load(input_ptr + pid * hidden_dim + offsets, mask=mask, other=0.0) + + # todo: tl.dot? + logits = tl.sum((w_router.to(tl.float32) * x[None, :].to(tl.float32)), axis=-1) + + # logit softcap + if moe_softcapping == 0: + logits_softcapped = logits + else: + logits_scaled = logits / moe_softcapping + exped = tl.exp(2 * logits_scaled) + top = exped - 1 + bottom = exped + 1 + logits_softcapped = top / bottom * moe_softcapping + + # Add bias after softcapping + if is_correction_bias: + bias = tl.load(correction_bias_ptr + tl.arange(0, num_experts)) + logits_softcapped = logits_softcapped + bias + + # topk + # assert 1 <= topk <= num_experts + + # 5.38 us + + top1 = tl.argmax(logits_softcapped, axis=0) + tl.store(topk_ids_ptr + pid * topk + 0, top1) # 5.63 us + + top1_v = tl.max(logits_softcapped, axis=0) + invsumexp = 1.0 / tl.sum(tl.exp(logits_softcapped - top1_v), axis=0) + + tl.store( + topk_weights_ptr + pid * topk + 0, + invsumexp, + ) # 5.73 us + + if topk >= 2: + top2 = tl.argmax( + tl.where( + tl.arange(0, num_experts) != top1, logits_softcapped, float("-inf") + ), + axis=0, + ) + tl.store(topk_ids_ptr + pid * topk + 1, top2) + top2_v = tl.sum(logits_softcapped * (tl.arange(0, num_experts) == top2), axis=0) + tl.store( + topk_weights_ptr + pid * topk + 1, + tl.exp(top2_v - top1_v) * invsumexp, + ) # 5.95us + + # probably slow + if topk > 2: + topk_mask = tl.full(logits_softcapped.shape, 1.0, dtype=logits_softcapped.dtype) + topk_mask = tl.where( + tl.arange(0, num_experts) != top1, topk_mask, float("-inf") + ) + topk_mask = tl.where( + tl.arange(0, num_experts) != top2, topk_mask, float("-inf") + ) + for i in range(2, topk): + topi = tl.argmax(logits_softcapped + topk_mask, axis=0) + topk_mask = tl.where( + tl.arange(0, num_experts) != topi, topk_mask, float("-inf") + ) + tl.store(topk_ids_ptr + pid * topk + i, topi) + topi_v = tl.sum( + logits_softcapped * (tl.arange(0, num_experts) == topi), axis=0 + ) + tl.store( + topk_weights_ptr + pid * topk + i, + tl.exp(topi_v - top1_v) * invsumexp, + ) + # assert not moe_renormalize, "moe weight renormalization not implemented" + + +def fused_moe_router_impl( + x: torch.Tensor, + router_weight: torch.Tensor, + topk: int, + moe_softcapping: float, + correction_bias: Optional[torch.Tensor] = None, +): + assert len(x.shape) == 2 and x.shape[1] == router_weight.shape[1] + bs, hidden_dim = x.shape + num_experts = router_weight.shape[0] + + # router_logits = torch.empty((bs, num_experts), dtype=torch.float32, device=x.device) + topk_weights = torch.empty((bs, topk), dtype=torch.float32, device=x.device) + topk_ids = torch.empty((bs, topk), dtype=torch.int32, device=x.device) + is_correction_bias = correction_bias is not None + + max_warps = 16 if _is_hip else 32 + config = { + "BLOCK_SIZE": triton.next_power_of_2(hidden_dim), + "num_warps": max( + min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), max_warps), 4 + ), + } + + fused_moe_router_kernel[(bs,)]( + x, + router_weight, + topk_weights, + topk_ids, + correction_bias, + is_correction_bias=is_correction_bias, + num_experts=num_experts, + topk=topk, + moe_softcapping=moe_softcapping, + moe_renormalize=False, + hidden_dim=hidden_dim, + **config, + ) + + return topk_weights, topk_ids + + +@triton.jit +def fused_moe_router_large_bs_kernel( + a_ptr, # input (bs, hidden_dim) + b_ptr, # input (num_experts, hidden_dim) + topk_weights_ptr, # output (bs, topk) + topk_ids_ptr, # output (bs, topk) + bs, + num_experts: tl.constexpr, + topk: tl.constexpr, # only support topk <= 2 + moe_softcapping: tl.constexpr, + moe_renormalize: tl.constexpr, # not supported + K: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + stride_am: tl.constexpr, + stride_bn: tl.constexpr, +): + + # 1. get block id + pid = tl.program_id(axis=0) + + # 2. create pointers for the first block of A and B + # 2.1. setup a_ptrs with offsets in m and k + offs_m = pid * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)[:, None] + bs_mask = offs_m < bs + offs_k = tl.arange(0, BLOCK_SIZE_K)[None, :] + a_ptrs = a_ptr + (offs_m * stride_am + offs_k) + + # 2.2. setup b_ptrs with offsets in k and n. + # Note: b matrix is k-major. + offs_k = tl.arange(0, BLOCK_SIZE_K)[None, :] + offs_n = tl.arange(0, BLOCK_SIZE_N)[:, None] + expert_mask = offs_n < num_experts + b_ptrs = b_ptr + (offs_n * stride_bn + offs_k) + + # 3. Create an accumulator of float32 of size [BLOCK_SIZE_M, BLOCK_SIZE_N] + # 3.1. iterate in K dimension + # 3.2. transpose tile B + acc = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, K // BLOCK_SIZE_K): # hidden_dim % BLOCK_SIZE_K == 0 + a = tl.load( + a_ptrs, + mask=bs_mask, + other=0.0, + ).to(tl.float32) + b = tl.load(b_ptrs, mask=expert_mask, other=0.0).to(tl.float32).T + acc += tl.dot(a, b) + + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K + b_ptrs += BLOCK_SIZE_K + + # 4. logit softcap + if moe_softcapping == 0: + logits_softcapped = acc + else: + logits_scaled = acc / moe_softcapping + exped = tl.exp(2 * logits_scaled) + logits_softcapped = (exped - 1) / (exped + 1) * moe_softcapping + + # 5. top1 + arange_block_size_n = tl.arange(0, BLOCK_SIZE_N)[None, :] + cond_top1 = arange_block_size_n < num_experts + top1 = tl.argmax(tl.where(cond_top1, logits_softcapped, float("-inf")), axis=1) + top1_v = tl.max( + tl.where(cond_top1, logits_softcapped, float("-inf")), axis=1, keep_dims=True + ) + top1_invsumexp = 1.0 / tl.sum( + tl.where(cond_top1, tl.exp(logits_softcapped - top1_v), 0.0), axis=1 + ) + + # 6. store top1 to output + offs_top1 = pid * topk * BLOCK_SIZE_M + topk * tl.arange(0, BLOCK_SIZE_M) + top1_mask = offs_top1 < bs * topk + tl.store(topk_ids_ptr + offs_top1, top1, mask=top1_mask) + tl.store( + topk_weights_ptr + offs_top1, + top1_invsumexp, + mask=top1_mask, + ) + + # 7. handle topk == 2 + if topk == 2: + cond_top2 = (arange_block_size_n < num_experts) & ( + arange_block_size_n != top1[:, None] + ) + top2 = tl.argmax( + tl.where(cond_top2, logits_softcapped, float("-inf")), + axis=1, + keep_dims=True, + ) + top2_v = tl.sum( + logits_softcapped * (arange_block_size_n == top2), axis=1, keep_dims=True + ) + top2_invsumexp = tl.exp(top2_v - top1_v) * top1_invsumexp[:, None] + + # store top2 + offs_top2 = ( + pid * topk * BLOCK_SIZE_M + topk * tl.arange(0, BLOCK_SIZE_M)[:, None] + 1 + ) + top2_mask = offs_top2 < bs * topk + tl.store(topk_ids_ptr + offs_top2, top2, mask=top2_mask) + tl.store( + topk_weights_ptr + offs_top2, + top2_invsumexp, + mask=top2_mask, + ) + + +def fused_moe_router_large_bs_impl( + x: torch.Tensor, + router_weight: torch.Tensor, + topk: int, + moe_softcapping: float, + BLOCK_SIZE_M: int, + BLOCK_SIZE_N: int, + BLOCK_SIZE_K: int, +): + assert len(x.shape) == 2 and x.shape[1] == router_weight.shape[1] + bs, hidden_dim = x.shape + num_experts = router_weight.shape[0] + + assert num_experts <= BLOCK_SIZE_N + assert hidden_dim % BLOCK_SIZE_K == 0 + assert topk <= 2 + + topk_weights = torch.empty((bs, topk), dtype=torch.float32, device=x.device) + topk_ids = torch.empty((bs, topk), dtype=torch.int32, device=x.device) + + grid = (triton.cdiv(bs, BLOCK_SIZE_M) * triton.cdiv(num_experts, BLOCK_SIZE_N),) + + fused_moe_router_large_bs_kernel[grid]( + a_ptr=x, + b_ptr=router_weight, + topk_weights_ptr=topk_weights, + topk_ids_ptr=topk_ids, + bs=bs, + num_experts=num_experts, + topk=topk, + moe_softcapping=moe_softcapping, + moe_renormalize=False, + K=hidden_dim, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + stride_am=hidden_dim, + stride_bn=hidden_dim, + ) + + return topk_weights, topk_ids + + +def fused_moe_router_shim( + moe_softcapping, + hidden_states, + gating_output, + topk, + renormalize, + correction_bias: Optional[torch.Tensor] = None, +): + assert not renormalize + assert ( + len(hidden_states.shape) == 2 + and hidden_states.shape[1] == gating_output.shape[1] + ) + bs, hidden_dim = hidden_states.shape + num_experts = gating_output.shape[0] + BLOCK_SIZE_M = 32 + BLOCK_SIZE_N = 16 + BLOCK_SIZE_K = 256 + if ( + bs >= 512 + and topk <= 2 + and num_experts <= BLOCK_SIZE_N + and hidden_dim % BLOCK_SIZE_K == 0 + ): + return fused_moe_router_large_bs_impl( + x=hidden_states, + router_weight=gating_output, + topk=topk, + moe_softcapping=moe_softcapping, + BLOCK_SIZE_M=BLOCK_SIZE_M, + BLOCK_SIZE_N=BLOCK_SIZE_N, + BLOCK_SIZE_K=BLOCK_SIZE_K, + ) + else: + return fused_moe_router_impl( + x=hidden_states, + router_weight=gating_output, + topk=topk, + moe_softcapping=moe_softcapping, + correction_bias=correction_bias, + ) + + +class FusedMoeRouter: + def __init__(self, router_linear, topk, moe_softcapping) -> None: + self.router_linear = router_linear + self.topk = topk + self.moe_softcapping = moe_softcapping + + def __call__(self, *args, **kwargs): + return self.forward(*args, **kwargs) + + def forward( + self, x: torch.Tensor, residual: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + if x.is_cuda: + return self.forward_cuda(x, residual) + else: + return self.forward_vllm(x, residual) + + def forward_cuda( + self, x: torch.Tensor, autotune=False + ) -> Tuple[torch.Tensor, torch.Tensor]: + return fused_moe_router_shim( + moe_softcapping=self.moe_softcapping, + hidden_states=x, + gating_output=self.router_linear.weight, + topk=self.topk, + renormalize=False, + ) + + def forward_vllm( + self, + x: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # g, _ = self.router_linear.forward(x) + g = x.float() @ self.router_linear.weight.T.float() + + g = torch.tanh(g.float() / self.moe_softcapping) * self.moe_softcapping + + return fused_topk(x, g, self.topk, False) diff --git a/python/sglang/srt/layers/moe/token_dispatcher/__init__.py b/python/sglang/srt/layers/moe/token_dispatcher/__init__.py new file mode 100644 index 000000000..e1dbcdd44 --- /dev/null +++ b/python/sglang/srt/layers/moe/token_dispatcher/__init__.py @@ -0,0 +1,41 @@ +from sglang.srt.layers.moe.token_dispatcher.base import ( + BaseDispatcher, + BaseDispatcherConfig, + CombineInput, + CombineInputChecker, + CombineInputFormat, + DispatchOutput, + DispatchOutputChecker, + DispatchOutputFormat, +) +from sglang.srt.layers.moe.token_dispatcher.deepep import ( + DeepEPConfig, + DeepEPDispatcher, + DeepEPLLCombineInput, + DeepEPLLOutput, + DeepEPNormalCombineInput, + DeepEPNormalOutput, +) +from sglang.srt.layers.moe.token_dispatcher.standard import ( + StandardCombineInput, + StandardDispatchOutput, +) + +__all__ = [ + "BaseDispatcher", + "BaseDispatcherConfig", + "CombineInput", + "CombineInputChecker", + "CombineInputFormat", + "DispatchOutput", + "DispatchOutputFormat", + "DispatchOutputChecker", + "StandardDispatchOutput", + "StandardCombineInput", + "DeepEPConfig", + "DeepEPDispatcher", + "DeepEPNormalOutput", + "DeepEPLLOutput", + "DeepEPLLCombineInput", + "DeepEPNormalCombineInput", +] diff --git a/python/sglang/srt/layers/moe/token_dispatcher/base.py b/python/sglang/srt/layers/moe/token_dispatcher/base.py new file mode 100644 index 000000000..155860886 --- /dev/null +++ b/python/sglang/srt/layers/moe/token_dispatcher/base.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Protocol, TypeGuard, Union, runtime_checkable + +import torch + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + DeepEPLLCombineInput, + DeepEPLLOutput, + DeepEPNormalCombineInput, + DeepEPNormalOutput, + StandardCombineInput, + StandardDispatchOutput, + ) + from sglang.srt.layers.moe.topk import TopKOutput + +# ------------------------------ Dispatch Output ------------------------------------- + + +class DispatchOutputChecker: + + @staticmethod + def format_is_standard( + dispatch_output: DispatchOutput, + ) -> TypeGuard[StandardDispatchOutput]: + return dispatch_output.format.is_standard() + + @staticmethod + def format_is_deepep_normal( + dispatch_output: DispatchOutput, + ) -> TypeGuard[DeepEPNormalOutput]: + return dispatch_output.format.is_deepep_normal() + + @staticmethod + def format_is_deepep_ll( + dispatch_output: DispatchOutput, + ) -> TypeGuard[DeepEPLLOutput]: + return dispatch_output.format.is_deepep_ll() + + @staticmethod + def format_is_deepep( + dispatch_output: DispatchOutput, + ) -> TypeGuard[Union[DeepEPNormalOutput, DeepEPLLOutput]]: + return dispatch_output.format.is_deepep() + + +class DispatchOutputFormat(Enum): + + STANDARD = "standard" + DEEPEP_NORMAL = "deepep_normal" + DEEPEP_LL = "deepep_ll" + + def is_standard(self) -> bool: + return self == DispatchOutputFormat.STANDARD + + def is_deepep_normal(self) -> bool: + return self == DispatchOutputFormat.DEEPEP_NORMAL + + def is_deepep_ll(self) -> bool: + return self == DispatchOutputFormat.DEEPEP_LL + + def is_deepep(self) -> bool: + return self in [ + DispatchOutputFormat.DEEPEP_NORMAL, + DispatchOutputFormat.DEEPEP_LL, + ] + + +@runtime_checkable +class DispatchOutput(Protocol): + """Protocol for dispatch outputs in different formats.""" + + # TODO: add hidden_states to the protocol + + @property + def format(self) -> DispatchOutputFormat: ... + + +# ------------------------------ Combine Input ------------------------------------- + + +class CombineInputChecker: + @staticmethod + def format_is_standard( + combine_input: CombineInput, + ) -> TypeGuard[StandardCombineInput]: + return combine_input.format == CombineInputFormat.STANDARD + + @staticmethod + def format_is_deepep_normal( + combine_input: CombineInput, + ) -> TypeGuard[DeepEPNormalCombineInput]: + return combine_input.format == CombineInputFormat.DEEPEP_NORMAL + + @staticmethod + def format_is_deepep_ll( + combine_input: CombineInput, + ) -> TypeGuard[DeepEPLLCombineInput]: + return combine_input.format == CombineInputFormat.DEEPEP_LL + + @staticmethod + def format_is_deepep( + combine_input: CombineInput, + ) -> TypeGuard[Union[DeepEPNormalCombineInput, DeepEPLLCombineInput]]: + return combine_input.format in [ + CombineInputFormat.DEEPEP_NORMAL, + CombineInputFormat.DEEPEP_LL, + ] + + +class CombineInputFormat(Enum): + STANDARD = "standard" + DEEPEP_NORMAL = "deepep_normal" + DEEPEP_LL = "deepep_ll" + + +@runtime_checkable +class CombineInput(Protocol): + """Protocol for combine inputs in different formats.""" + + # TODO: add hidden_states to the protocol + + @property + def format(self) -> CombineInputFormat: ... + + +# ------------------------------ Base Dispatcher ------------------------------------- + + +class BaseDispatcherConfig(ABC): + """Base class for dispatcher configs.""" + + pass + + +class BaseDispatcher(ABC): + """Base class for dispatchers.""" + + @abstractmethod + def dispatch( + self, hidden_states: torch.Tensor, topk_output: TopKOutput, **kwargs + ) -> DispatchOutput: + pass + + @abstractmethod + def combine(self, combine_input: CombineInput, **kwargs) -> torch.Tensor: + pass diff --git a/python/sglang/srt/layers/moe/token_dispatcher/deepep.py b/python/sglang/srt/layers/moe/token_dispatcher/deepep.py new file mode 100644 index 000000000..450cff0cb --- /dev/null +++ b/python/sglang/srt/layers/moe/token_dispatcher/deepep.py @@ -0,0 +1,732 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, NamedTuple, Optional, Tuple, Union + +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.layers.moe.token_dispatcher.base import ( + BaseDispatcher, + BaseDispatcherConfig, + CombineInput, + CombineInputFormat, + DispatchOutput, + DispatchOutputFormat, +) +from sglang.srt.layers.moe.utils import DeepEPMode, get_deepep_config, is_tbo_enabled +from sglang.srt.layers.quantization import deep_gemm_wrapper +from sglang.srt.utils import ( + get_bool_env_var, + get_int_env_var, + is_hip, + is_npu, + load_json_config, +) + +_is_npu = is_npu() + +try: + from deep_ep import Buffer, Config + + if not _is_npu: + from sglang.srt.layers.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8, + ) + + use_deepep = True +except ImportError: + use_deepep = False + +from enum import Enum, IntEnum, auto + +import torch +import torch.distributed as dist + +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and is_hip() + +logger = logging.getLogger(__name__) + + +class DeepEPNormalOutput(NamedTuple): + """DeepEP normal dispatch output.""" + + hidden_states: torch.Tensor | Tuple[torch.Tensor, torch.Tensor] + # hidden_states_scale + topk_idx: torch.Tensor + topk_weights: torch.Tensor + num_recv_tokens_per_expert: List[int] + + @property + def format(self) -> DispatchOutputFormat: + return DispatchOutputFormat.DEEPEP_NORMAL + + +class DeepEPLLOutput(NamedTuple): + """DeepEP low latency dispatch output.""" + + hidden_states_fp8: Tuple[torch.Tensor, torch.Tensor] + topk_idx: torch.Tensor + topk_weights: torch.Tensor + masked_m: torch.Tensor + expected_m: int + + @property + def format(self) -> DispatchOutputFormat: + return DispatchOutputFormat.DEEPEP_LL + + +assert isinstance(DeepEPNormalOutput, DispatchOutput) +assert isinstance(DeepEPLLOutput, DispatchOutput) + + +class DeepEPNormalCombineInput(NamedTuple): + """DeepEP normal combine input.""" + + pass + + @property + def format(self) -> CombineInputFormat: + return CombineInputFormat.DEEPEP_NORMAL + + +class DeepEPLLCombineInput(NamedTuple): + """DeepEP low latency combine input.""" + + pass + + @property + def format(self) -> CombineInputFormat: + return CombineInputFormat.DEEPEP_LL + + +assert isinstance(DeepEPNormalCombineInput, CombineInput) +assert isinstance(DeepEPLLCombineInput, CombineInput) + + +class DeepEPDispatchMode(IntEnum): + NORMAL = auto() + LOW_LATENCY = auto() + + +class DeepEPBuffer: + _buffer = None + _dispatch_mode: Optional[DeepEPDispatchMode] = None + _hidden_size: Optional[int] = None + _num_max_dispatch_tokens_per_rank: Optional[int] = None + _num_experts: Optional[int] = None + + @classmethod + def get_deepep_buffer( + cls, + group: dist.ProcessGroup, + hidden_size: int, + param_bytes: int, + deepep_mode: DeepEPMode, + num_max_dispatch_tokens_per_rank: int = -1, + num_experts: int = -1, + ): + if cls._buffer is not None: + return cls._buffer + + cls._hidden_size = hidden_size + cls._num_max_dispatch_tokens_per_rank = num_max_dispatch_tokens_per_rank + cls._num_experts = num_experts + + num_nvl_bytes, num_rdma_bytes = 0, 0 + if deepep_mode.enable_normal(): + hidden_bytes = hidden_size * param_bytes + for config in ( + DeepEPConfig.get_instance().normal_dispatch_config + or Buffer.get_dispatch_config(group.size()), + DeepEPConfig.get_instance().normal_combine_config + or Buffer.get_combine_config(group.size()), + ): + num_nvl_bytes = max( + config.get_nvl_buffer_size_hint(hidden_bytes, group.size()), + num_nvl_bytes, + ) + num_rdma_bytes = max( + config.get_rdma_buffer_size_hint(hidden_bytes, group.size()), + num_rdma_bytes, + ) + if deepep_mode.enable_low_latency(): + assert num_max_dispatch_tokens_per_rank != -1 + assert num_experts != -1 and num_experts % group.size() == 0 + num_rdma_bytes = max( + Buffer.get_low_latency_rdma_size_hint( + num_max_dispatch_tokens_per_rank, + hidden_size, + group.size(), + num_experts, + ), + num_rdma_bytes, + ) + + if deepep_mode == DeepEPMode.NORMAL: + num_qps_per_rank = DeepEPConfig.get_instance().num_sms // 2 + elif deepep_mode in [DeepEPMode.LOW_LATENCY, DeepEPMode.AUTO]: + num_qps_per_rank = num_experts // group.size() + else: + raise NotImplementedError + + if not _is_npu: + total_num_sms = torch.cuda.get_device_properties( + device="cuda" + ).multi_processor_count + if ( + (deepep_mode != DeepEPMode.LOW_LATENCY) + and not is_tbo_enabled() + and (DeepEPConfig.get_instance().num_sms < total_num_sms // 2) + ): + logger.warning( + f"Only use {DeepEPConfig.get_instance().num_sms} SMs for DeepEP communication. " + f"This may result in highly suboptimal performance. " + f"Consider using --deepep-config to change the behavior." + ) + + cls._buffer = Buffer( + group, + num_nvl_bytes, + num_rdma_bytes, + low_latency_mode=deepep_mode.enable_low_latency(), + num_qps_per_rank=num_qps_per_rank, + # TODO can be false when unneeded + allow_mnnvl=True, + ) + return cls._buffer + + @classmethod + def clean_buffer(cls): + if not cls._buffer.low_latency_mode: + return + cls._buffer.clean_low_latency_buffer( + cls._num_max_dispatch_tokens_per_rank, + cls._hidden_size, + cls._num_experts, + ) + + @classmethod + def set_dispatch_mode_as_normal(cls): + cls._dispatch_mode = DeepEPDispatchMode.NORMAL + + @classmethod + def set_dispatch_mode_as_low_latency(cls): + if cls._dispatch_mode == DeepEPDispatchMode.NORMAL: + cls.clean_buffer() + cls._dispatch_mode = DeepEPDispatchMode.LOW_LATENCY + + +class DeepEPConfig(BaseDispatcherConfig): + _instance = None + + def __init__(self): + config_str = get_deepep_config() + if config_str: + config_parsed = load_json_config(config_str) + if torch.distributed.get_rank() == 0: + logger.info(f"Use DeepEP Config: {config_parsed}") + config_dispatch = config_parsed["normal_dispatch"] + config_combine = config_parsed["normal_combine"] + + self.normal_dispatch_config = Config(**config_dispatch) + self.normal_combine_config = Config(**config_combine) + + assert config_dispatch["num_sms"] == config_combine["num_sms"] + self.num_sms = config_dispatch["num_sms"] + else: + self.normal_dispatch_config = None + self.normal_combine_config = None + self.num_sms = Buffer.num_sms + + @classmethod + def get_instance(cls): + if cls._instance is None: + cls._instance = DeepEPConfig() + return cls._instance + + +class _DeepEPDispatcherImplBase: + def __init__( + self, + group: torch.distributed.ProcessGroup, + router_topk: int, + permute_fusion: bool, + num_experts: int, + num_local_experts: int, + hidden_size: int, + params_dtype: torch.dtype, + deepep_mode: DeepEPMode, + ): + if not use_deepep: + raise ImportError( + "DeepEP is not installed. Please install DeepEP package from " + "https://github.com/deepseek-ai/deepep." + ) + + self.group = group + self.router_topk = router_topk + self.permute_fusion = permute_fusion + self.num_experts = num_experts + self.num_local_experts = num_local_experts + self.hidden_size = hidden_size + self.params_dtype = params_dtype + self.deepep_mode = deepep_mode + + self.params_bytes = 2 + self.num_max_dispatch_tokens_per_rank = get_int_env_var( + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK", 128 + ) + # DeepEP internode_ll dispatch uses FINISHED_SUM_TAG=1024 + # and the logic requires num-tokens-sent-from-one-rank-to-another-rank less than it + assert self.num_max_dispatch_tokens_per_rank <= 1024 + + self.handle = None + + def dispatch_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + raise NotImplementedError + + def dispatch_b(self, *args, **kwargs): + raise NotImplementedError + + def combine_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + raise NotImplementedError + + def combine_b(self, *args, **kwargs): + raise NotImplementedError + + def _get_buffer(self): + raise NotImplementedError + + +class _DeepEPDispatcherImplNormal(_DeepEPDispatcherImplBase): + def __init__(self, async_finish: bool, **kwargs): + super().__init__(**kwargs) + + self.async_finish = async_finish + self.src2dst = None + + def dispatch_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + topk_idx = topk_idx.to(torch.int64) + if deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: + # TODO hard code 128 block quant,use fp8 communication + hidden_states = sglang_per_token_group_quant_fp8( + hidden_states, + 128, + column_major_scales=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_tma_aligned=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + previous_event = Buffer.capture() if self.async_finish else None + return hidden_states, topk_idx, topk_weights, previous_event + + def dispatch_b(self, hidden_states, topk_idx, topk_weights, previous_event): + ( + hidden_states, + topk_idx, + topk_weights, + num_recv_tokens_per_expert, + event, + ) = self._dispatch_core(hidden_states, topk_idx, topk_weights, previous_event) + event.current_stream_wait() if self.async_finish else () + return DeepEPNormalOutput( + hidden_states, topk_idx, topk_weights, num_recv_tokens_per_expert + ) + + def _dispatch_core( + self, + x: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + previous_event, + ): + buffer = self._get_buffer() + ( + num_tokens_per_rank, + num_tokens_per_rdma_rank, + num_tokens_per_expert, + is_token_in_rank, + previous_event, + ) = buffer.get_dispatch_layout( + topk_idx, + self.num_experts, + previous_event=previous_event, + async_finish=self.async_finish, + allocate_on_comm_stream=previous_event is not None, + ) + + # FIXME: `handle` should be transmitted with tokens from dispatch to combine. + # However, doing this would incur an unknown synchronization error, but keeping + # `handle` as a member variable works. + + ( + recv_x, + recv_topk_idx, + recv_topk_weights, + num_recv_tokens_per_expert, + self.handle, + event, + ) = buffer.dispatch( + x, + topk_idx=topk_idx, + topk_weights=topk_weights, + num_tokens_per_rank=num_tokens_per_rank, + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, + is_token_in_rank=is_token_in_rank, + num_tokens_per_expert=num_tokens_per_expert, + previous_event=previous_event, + async_finish=self.async_finish, + allocate_on_comm_stream=(previous_event is not None) and self.async_finish, + expert_alignment=128 if deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM else 1, + config=DeepEPConfig.get_instance().normal_dispatch_config, + ) + + get_global_expert_distribution_recorder().on_deepep_dispatch_normal( + num_recv_tokens_per_expert, + num_tokens_per_rank=num_tokens_per_rank, + num_tokens_per_rdma_rank=num_tokens_per_rdma_rank, + num_tokens_per_expert=num_tokens_per_expert, + ) + + return ( + recv_x, + recv_topk_idx, + recv_topk_weights, + num_recv_tokens_per_expert, + event, + ) + + def combine_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + from sglang.srt.layers.moe.ep_moe.kernels import ( + deepep_post_reorder_triton_kernel, + ) + + if deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM or _use_aiter or _is_npu: + output = hidden_states + else: + if hidden_states.shape[0] > 0: + num_tokens = self.src2dst.shape[0] // self.router_topk + output = torch.empty( + (num_tokens, hidden_states.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + deepep_post_reorder_triton_kernel[(num_tokens,)]( + hidden_states, + output, + self.src2dst, + topk_idx, + topk_weights, + self.router_topk, + hidden_states.shape[1], + BLOCK_SIZE=512, + ) + else: + output = torch.zeros( + (0, hidden_states.shape[1]), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + previous_event = Buffer.capture() if self.async_finish else None + return output, previous_event + + def combine_b(self, output, previous_event): + hidden_states, event = self._combine_core(output, previous_event) + event.current_stream_wait() if self.async_finish else () + self.handle = None + self.src2dst = None + return hidden_states + + def _combine_core(self, x: torch.Tensor, previous_event): + buffer = self._get_buffer() + combined_x, _, event = buffer.combine( + x, + self.handle, + async_finish=self.async_finish, + previous_event=previous_event, + allocate_on_comm_stream=previous_event is not None, + config=DeepEPConfig.get_instance().normal_combine_config, + ) + return combined_x, event + + def _get_buffer(self): + DeepEPBuffer.set_dispatch_mode_as_normal() + + return DeepEPBuffer.get_deepep_buffer( + self.group, + self.hidden_size, + self.params_bytes, + self.deepep_mode, + self.num_max_dispatch_tokens_per_rank, + self.num_experts, + ) + + +class _DeepEPDispatcherImplLowLatency(_DeepEPDispatcherImplBase): + def __init__(self, return_recv_hook: bool, **kwargs): + super().__init__(**kwargs) + + """ + num_max_dispatch_tokens_per_rank: the actual batch size in the decoding engine should be less than 256 + https://github.com/deepseek-ai/DeepEP?tab=readme-ov-file#example-use-in-inference-decoding + """ + self.return_recv_hook = return_recv_hook + + def dispatch_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + buffer = self._get_buffer() + topk_idx = topk_idx.to(torch.int64) + expected_m = ( + hidden_states.shape[0] * buffer.group_size * topk_idx.shape[1] + + self.num_experts + ) // self.num_experts + hidden_states, masked_m, event, hook = self._dispatch_core( + hidden_states, + topk_idx, + use_fp8=True, + ) + return ( + hidden_states, + topk_idx, + topk_weights, + masked_m, + expected_m, + event, + hook, + ) + + def dispatch_b( + self, + hidden_states, + topk_idx, + topk_weights, + masked_m, + expected_m, + event, + hook, + ): + hook() if self.return_recv_hook else event.current_stream_wait() + + get_global_expert_distribution_recorder().on_deepep_dispatch_low_latency( + masked_m + ) + + deepep_output = DeepEPLLOutput( + hidden_states, + topk_idx, + topk_weights, + masked_m, + expected_m, + ) + return deepep_output + + def _dispatch_core( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + use_fp8: bool = False, + ): + buffer = self._get_buffer() + packed_recv_hidden, packed_recv_count, self.handle, event, hook = ( + buffer.low_latency_dispatch( + hidden_states, + topk_idx, + self.num_max_dispatch_tokens_per_rank, + self.num_experts, + use_fp8=use_fp8, + async_finish=not self.return_recv_hook, + return_recv_hook=self.return_recv_hook, + round_scale=deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + and deep_gemm_wrapper.DEEPGEMM_BLACKWELL, + use_ue8m0=deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + and deep_gemm_wrapper.DEEPGEMM_BLACKWELL, + ) + ) + return packed_recv_hidden, packed_recv_count, event, hook + + def combine_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + hidden_states, event, hook = self._combine_core( + hidden_states, + topk_idx, + topk_weights, + ) + return hidden_states, event, hook + + def combine_b(self, hidden_states, event, hook): + hook() if self.return_recv_hook else event.current_stream_wait() + return hidden_states + + def _combine_core( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + ): + buffer = self._get_buffer() + combined_hidden_states, event, hook = buffer.low_latency_combine( + hidden_states, + topk_idx, + topk_weights, + self.handle, + async_finish=not self.return_recv_hook, + return_recv_hook=self.return_recv_hook, + ) + self.handle = None + return combined_hidden_states, event, hook + + def _get_buffer(self): + DeepEPBuffer.set_dispatch_mode_as_low_latency() + return DeepEPBuffer.get_deepep_buffer( + self.group, + self.hidden_size, + self.params_bytes, + self.deepep_mode, + self.num_max_dispatch_tokens_per_rank, + self.num_experts, + ) + + +@dataclass +class _Stage(Enum): + INITIAL = auto() + AFTER_DISPATCH_A = auto() + AFTER_DISPATCH_B = auto() + AFTER_COMBINE_A = auto() + + +class DeepEPDispatcher(BaseDispatcher): + def __init__( + self, + group: torch.distributed.ProcessGroup, + router_topk: int, + permute_fusion: bool = False, + num_experts: int = None, + num_local_experts: int = None, + hidden_size: int = None, + params_dtype: torch.dtype = None, + deepep_mode: DeepEPMode = DeepEPMode.AUTO, + async_finish: bool = False, + return_recv_hook: bool = False, + ): + self.deepep_mode = deepep_mode + + common_kwargs = dict( + group=group, + router_topk=router_topk, + permute_fusion=permute_fusion, + num_experts=num_experts, + num_local_experts=num_local_experts, + hidden_size=hidden_size, + params_dtype=params_dtype, + deepep_mode=deepep_mode, + ) + + if self.deepep_mode.enable_low_latency(): + self._low_latency_dispatcher = _DeepEPDispatcherImplLowLatency( + return_recv_hook=return_recv_hook, + **common_kwargs, + ) + if self.deepep_mode.enable_normal(): + self._normal_dispatcher = _DeepEPDispatcherImplNormal( + async_finish=async_finish, + **common_kwargs, + ) + + self._stage = _Stage.INITIAL + + def dispatch(self, *args, **kwargs) -> DispatchOutput: + self.dispatch_a(*args, **kwargs) + ret = self.dispatch_b() + return ret + + def dispatch_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + forward_batch: ForwardBatch, + ): + self._update_stage(_Stage.INITIAL, _Stage.AFTER_DISPATCH_A) + inner_state = self._get_impl(forward_batch).dispatch_a( + hidden_states=hidden_states, + topk_idx=topk_idx, + topk_weights=topk_weights, + ) + self._dispatch_intermediate_state = forward_batch, inner_state + + def dispatch_b(self): + self._update_stage(_Stage.AFTER_DISPATCH_A, _Stage.AFTER_DISPATCH_B) + forward_batch, inner_state = self._dispatch_intermediate_state + del self._dispatch_intermediate_state + return self._get_impl(forward_batch).dispatch_b(*inner_state) + + def combine(self, *args, **kwargs) -> Tuple: + self.combine_a(*args, **kwargs) + ret = self.combine_b() + return ret + + def combine_a( + self, + hidden_states: torch.Tensor, + topk_idx: torch.Tensor, + topk_weights: torch.Tensor, + forward_batch: ForwardBatch, + ): + self._update_stage(_Stage.AFTER_DISPATCH_B, _Stage.AFTER_COMBINE_A) + inner_state = self._get_impl(forward_batch).combine_a( + hidden_states=hidden_states, + topk_idx=topk_idx, + topk_weights=topk_weights, + ) + self._combine_intermediate_state = forward_batch, inner_state + + def combine_b(self): + self._update_stage(_Stage.AFTER_COMBINE_A, _Stage.INITIAL) + forward_batch, inner_state = self._combine_intermediate_state + del self._combine_intermediate_state + return self._get_impl(forward_batch).combine_b(*inner_state) + + def _get_impl(self, forward_batch: ForwardBatch) -> _DeepEPDispatcherImplBase: + resolved_deepep_mode = self.deepep_mode.resolve( + forward_batch.is_extend_in_batch + ) + if resolved_deepep_mode == DeepEPMode.NORMAL: + return self._normal_dispatcher + elif resolved_deepep_mode == DeepEPMode.LOW_LATENCY: + return self._low_latency_dispatcher + else: + raise ValueError(f"Invalid deepep_mode: {self.deepep_mode}") + + def _update_stage(self, old_stage, new_stage): + assert self._stage == old_stage + self._stage = new_stage diff --git a/python/sglang/srt/layers/moe/token_dispatcher/standard.py b/python/sglang/srt/layers/moe/token_dispatcher/standard.py new file mode 100644 index 000000000..f984104f6 --- /dev/null +++ b/python/sglang/srt/layers/moe/token_dispatcher/standard.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, NamedTuple + +import torch + +from sglang.srt.layers.moe.token_dispatcher.base import ( + BaseDispatcher, + CombineInput, + CombineInputFormat, + DispatchOutput, + DispatchOutputFormat, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.topk import TopKOutput + + +class StandardDispatchOutput(NamedTuple): + """Standard dispatch output.""" + + hidden_states: torch.Tensor + topk_output: TopKOutput + + @property + def format(self) -> DispatchOutputFormat: + return DispatchOutputFormat.STANDARD + + +assert isinstance(StandardDispatchOutput, DispatchOutput) + + +class StandardCombineInput(NamedTuple): + """Standard combine input.""" + + hidden_states: torch.Tensor + + @property + def format(self) -> CombineInputFormat: + return CombineInputFormat.STANDARD + + +assert isinstance(StandardCombineInput, CombineInput) + + +class StandardDispatcher(BaseDispatcher): + + def dispatch( + self, hidden_states: torch.Tensor, topk_output: TopKOutput + ) -> DispatchOutput: + return StandardDispatchOutput( + hidden_states=hidden_states, topk_output=topk_output + ) + + def combine(self, combine_input: CombineInput) -> torch.Tensor: + if isinstance(combine_input, StandardCombineInput): + return combine_input.hidden_states + else: + # TODO: this branch should be removed in the future + assert isinstance(combine_input, torch.Tensor) + return combine_input diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py new file mode 100644 index 000000000..b8f73473c --- /dev/null +++ b/python/sglang/srt/layers/moe/topk.py @@ -0,0 +1,876 @@ +# Copyright 2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from enum import Enum, auto +from typing import ( + Callable, + NamedTuple, + Optional, + Protocol, + TypeGuard, + runtime_checkable, +) + +import torch +import torch.nn.functional as F + +from sglang.srt.custom_op import CustomOp +from sglang.srt.eplb import expert_location_dispatch +from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_location_dispatch import ( + ExpertLocationDispatchInfo, + topk_ids_logical_to_physical, +) +from sglang.srt.layers.moe import ( + get_moe_runner_backend, + should_use_flashinfer_trtllm_moe, +) +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + get_compiler_backend, + is_cpu, + is_cuda, + is_hip, + is_npu, +) + +try: + from triton_kernels.routing import GatherIndx, RoutingData, ScatterIndx, routing +except ImportError: + pass +logger = logging.getLogger(__name__) + + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_cpu = is_cpu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_npu = is_npu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _is_cuda: + from sgl_kernel import moe_fused_gate + +if _is_cuda or _is_hip: + from sgl_kernel import topk_softmax +if _use_aiter: + try: + from aiter import biased_grouped_topk as aiter_biased_grouped_topk + except ImportError: + raise ImportError("aiter is required when SGLANG_USE_AITER is set to True") +if _is_npu: + import torch_npu + +# -------------------------------- TopKConfig --------------------------------------- + + +@dataclass +class TopKConfig: + top_k: int + use_grouped_topk: bool = False + topk_group: Optional[int] = None + num_expert_group: Optional[int] = None + renormalize: bool = True + num_fused_shared_experts: int = 0 + custom_routing_function: Optional[Callable] = None + correction_bias: Optional[torch.Tensor] = None + torch_native: bool = False + routed_scaling_factor: Optional[float] = None + apply_routed_scaling_factor_on_output: bool = False + + +# -------------------------------- TopKOutput --------------------------------------- + + +class TopKOutputChecker: + + @staticmethod + def format_is_standard(topk_output: TopKOutput) -> TypeGuard[StandardTopKOutput]: + return topk_output.format.is_standard() + + @staticmethod + def format_is_triton_kernel( + topk_output: TopKOutput, + ) -> TypeGuard[TritonKernelTopKOutput]: + return topk_output.format.is_triton_kernel() + + @staticmethod + def format_is_bypassed(topk_output: TopKOutput) -> TypeGuard[BypassedTopKOutput]: + return topk_output.format.is_bypassed() + + +class TopKOutputFormat(Enum): + STANDARD = auto() + TRITON_KERNEL = auto() + BYPASSED = auto() + + def is_standard(self) -> bool: + return self == TopKOutputFormat.STANDARD + + def is_triton_kernel(self) -> bool: + return self == TopKOutputFormat.TRITON_KERNEL + + def is_bypassed(self) -> bool: + return self == TopKOutputFormat.BYPASSED + + +@runtime_checkable +class TopKOutput(Protocol): + """Protocol for top-k outputs in different formats.""" + + @property + def format(self) -> TopKOutputFormat: + """The format of the output.""" + ... + + +class StandardTopKOutput(NamedTuple): + """Standard top-k output format.""" + + topk_weights: torch.Tensor + topk_ids: torch.Tensor + router_logits: torch.Tensor + + @property + def format(self) -> TopKOutputFormat: + return TopKOutputFormat.STANDARD + + +class TritonKernelTopKOutput(NamedTuple): + """Triton kernel top-k output format.""" + + routing_data: RoutingData + gather_indx: GatherIndx + scatter_indx: ScatterIndx + + @property + def format(self) -> TopKOutputFormat: + return TopKOutputFormat.TRITON_KERNEL + + +class BypassedTopKOutput(NamedTuple): + """Bypassed top-k output format.""" + + hidden_states: torch.Tensor + router_logits: torch.Tensor + topk_config: TopKConfig + num_token_non_padded: Optional[torch.Tensor] = None + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None + + @property + def format(self) -> TopKOutputFormat: + return TopKOutputFormat.BYPASSED + + +# -------------------------------- TopK --------------------------------------- + + +class TopK(CustomOp): + + def __init__( + self, + top_k: int, + *, + use_grouped_topk: bool = False, + topk_group: Optional[int] = None, + num_expert_group: Optional[int] = None, + renormalize: bool = True, + num_fused_shared_experts: int = 0, + custom_routing_function: Optional[Callable] = None, + scoring_func: str = "softmax", + correction_bias: Optional[torch.Tensor] = None, + routed_scaling_factor: Optional[float] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, + force_topk: bool = False, + ): + # NOTE: scoring_func is not used for now, but we keep it for future use + # see https://github.com/sgl-project/sglang/pull/4505 for more details + super().__init__() + + if use_grouped_topk: + assert num_expert_group is not None and topk_group is not None + + self.topk_config = TopKConfig( + top_k=top_k, + use_grouped_topk=use_grouped_topk, + renormalize=renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + num_fused_shared_experts=num_fused_shared_experts, + custom_routing_function=custom_routing_function, + correction_bias=correction_bias, + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + self.use_triton_kernels = get_moe_runner_backend().is_triton_kernel() + self.force_topk = force_topk + + def forward_native( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + *, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + ) -> TopKOutput: + self.topk_config.torch_native = True + return select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=self.topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + + def forward_cuda( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + *, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + ) -> TopKOutput: + if self.use_triton_kernels: + # renormalize=True is equivalent to sm_first=False + routing_data, gather_idx, scatter_idx = routing( + router_logits, + self.topk_config.top_k, + sm_first=not self.topk_config.renormalize, + ) + return TritonKernelTopKOutput(routing_data, gather_idx, scatter_idx) + elif not self.force_topk and ( + should_use_flashinfer_trtllm_moe() + or get_moe_runner_backend().is_flashinfer_mxfp4() + ): + return BypassedTopKOutput( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=self.topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + else: + self.topk_config.torch_native = False + return select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=self.topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + + def forward_cpu( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + *, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + ) -> TopKOutput: + return select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=self.topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + + def forward_npu( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + *, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + ) -> TopKOutput: + global_num_experts = router_logits.shape[-1] + + # NOTE: now npu_moe_gating_top_k can only support `group_count=256` pattern + if global_num_experts == 256: + + routed_scaling_factor = self.topk_config.routed_scaling_factor or 1 + router_logits = router_logits.to(torch.float32) + + topk_weights, topk_ids, _ = torch_npu.npu_moe_gating_top_k( + router_logits, + k=self.topk_config.top_k, + bias=self.topk_config.correction_bias.to(torch.float32), + k_group=self.topk_config.topk_group, + group_count=self.topk_config.num_expert_group, + group_select_mode=1, + renorm=0, + norm_type=1, + routed_scaling_factor=routed_scaling_factor, + eps=float(1e-20), + ) + + if self.topk_config.renormalize: + topk_weights_sum = ( + topk_weights.sum(dim=-1, keepdim=True) + if self.topk_config.num_fused_shared_experts == 0 + else topk_weights[:, :-1].sum(dim=-1, keepdim=True) + ) + topk_weights = topk_weights / topk_weights_sum + + if expert_location_dispatch_info is not None: + topk_ids = topk_ids_logical_to_physical( + topk_ids, expert_location_dispatch_info + ) + get_global_expert_distribution_recorder().on_select_experts( + topk_ids=topk_ids + ) + + return StandardTopKOutput(topk_weights, topk_ids, _) + else: + self.topk_config.torch_native = True + return select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=self.topk_config, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + + def empty_topk_output(self, device: torch.device) -> TopKOutput: + topk = self.topk_config.top_k - self.topk_config.num_fused_shared_experts + topk_weights = torch.empty((0, topk), dtype=torch.float32, device=device) + topk_idx = torch.full((0, topk), -1, dtype=torch.int32, device=device) + router_logits = torch.empty((0, topk), dtype=torch.float32, device=device) + return StandardTopKOutput(topk_weights, topk_idx, router_logits) + + +# ------------------------------- TopK implementation ------------------------------------- + + +def fused_topk_torch_native( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + correction_bias: torch.Tensor = None, +): + if correction_bias is not None: + n_routed_experts = gating_output.shape[-1] + scores = gating_output.softmax(dim=-1) + scores_for_choice = scores.view( + -1, n_routed_experts + ) + correction_bias.unsqueeze(0) + topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=False)[1] + topk_weights = scores.gather(1, topk_ids) + else: + assert ( + hidden_states.shape[0] == gating_output.shape[0] + ), f"Number of tokens mismatch, {hidden_states.shape=} vs {gating_output.shape=}" + M, _ = hidden_states.shape + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + topk_weights = F.softmax(gating_output.float(), dim=-1) + topk_weights, topk_ids = torch.topk(topk_weights, topk, dim=-1) + + if renormalize: + topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True) + return topk_weights, topk_ids + + +def fused_topk_cpu( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + correction_bias: torch.Tensor = None, +): + topk_weights, topk_ids = torch.ops.sgl_kernel.topk_softmax_cpu( + hidden_states=hidden_states, + gating_output=gating_output, + topk=topk, + renormalize=renormalize, + ) + topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_weights, topk_ids + + +def apply_topk_weights_cpu(need_apply, topk_weights, inputs): + if not need_apply: + return inputs, topk_weights + + # TODO: fuse below processing in fused_experts_cpu kernel + inputs = inputs * topk_weights.to(inputs.dtype) + topk_weights = torch.ones_like( + topk_weights, dtype=torch.float32 + ) # clear topk_weights as already applied + + return inputs, topk_weights + + +def fused_topk( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + + M, _ = hidden_states.shape + + topk_weights = torch.empty( + M, topk, dtype=torch.float32, device=hidden_states.device + ) + topk_ids = torch.empty(M, topk, dtype=torch.int32, device=hidden_states.device) + + topk_softmax( + topk_weights, + topk_ids, + gating_output, + renormalize, + ) + + topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_weights, topk_ids + + +# This is used by the Deepseek V2/V3/R1 series models +@torch.compile(dynamic=True, backend=get_compiler_backend()) +def grouped_topk_gpu( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: Optional[float] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + + scores = torch.softmax(gating_output, dim=-1) + # NPU compiler limitation + if _is_npu and scores.dtype == torch.bfloat16: + scores = scores.to(torch.float16) + num_token = scores.shape[0] + num_experts = scores.shape[1] + group_scores = ( + scores.view(num_token, num_expert_group, -1).max(dim=-1).values + ) # [n, n_group] + group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[ + 1 + ] # [n, top_k_group] + group_mask = torch.zeros_like(group_scores) # [n, n_group] + group_mask.scatter_(1, group_idx, 1) # [n, n_group] + score_mask = ( + group_mask.unsqueeze(-1) + .expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group) + .reshape(num_token, -1) + ) # [n, e] + tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0) # [n, e] + # TODO: NPU can't support directly evaluating a comparison for now + topk_weights, topk_ids = torch.topk( + tmp_scores, + k=topk, + dim=-1, + sorted=(True if num_fused_shared_experts > 0 else False), + ) + if num_fused_shared_experts: + topk_ids[:, -1] = torch.randint( + low=num_experts, + high=num_experts + num_fused_shared_experts, + size=(topk_ids.size(0),), + dtype=topk_ids.dtype, + device=topk_ids.device, + ) + topk_weights[:, -1] = topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor + + if renormalize: + topk_weights_sum = ( + topk_weights.sum(dim=-1, keepdim=True) + if num_fused_shared_experts == 0 + else topk_weights[:, :-1].sum(dim=-1, keepdim=True) + ) + topk_weights = topk_weights / topk_weights_sum + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor + + topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) + topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_weights, topk_ids + + +def grouped_topk_cpu( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: Optional[float] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, +): + assert not apply_routed_scaling_factor_on_output + assert expert_location_dispatch_info is None + return torch.ops.sgl_kernel.grouped_topk_cpu( + hidden_states, + gating_output, + topk, + renormalize, + num_expert_group, + topk_group, + num_fused_shared_experts, + routed_scaling_factor, + num_token_non_padded, + ) + + +@torch.compile(dynamic=True, backend=get_compiler_backend(), disable=_is_npu) +def biased_grouped_topk_impl( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + correction_bias: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: Optional[float] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, +): + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + + scores = gating_output.sigmoid() + num_token = scores.shape[0] + num_experts = scores.shape[1] + scores_for_choice = scores.view(num_token, -1) + correction_bias.unsqueeze(0) + group_scores = ( + scores_for_choice.view(num_token, num_expert_group, -1) + .topk(2, dim=-1)[0] + .sum(dim=-1) + ) # [n, n_group] + group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[ + 1 + ] # [n, top_k_group] + group_mask = torch.zeros_like(group_scores) # [n, n_group] + group_mask.scatter_(1, group_idx, 1) # [n, n_group] + score_mask = ( + group_mask.unsqueeze(-1) + .expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group) + .reshape(num_token, -1) + ) # [n, e] + tmp_scores = scores_for_choice.masked_fill( + ~score_mask.bool(), float("-inf") + ) # [n, e] + # TODO: NPU can't support directly evaluating a comparison for now + _, topk_ids = torch.topk( + tmp_scores, + k=topk, + dim=-1, + sorted=(True if num_fused_shared_experts > 0 else False), + ) + topk_weights = scores.gather(1, topk_ids) + + if num_fused_shared_experts: + topk_ids[:, -1] = torch.randint( + low=num_experts, + high=num_experts + num_fused_shared_experts, + size=(topk_ids.size(0),), + dtype=topk_ids.dtype, + device=topk_ids.device, + ) + topk_weights[:, -1] = topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor + + if renormalize: + topk_weights_sum = ( + topk_weights.sum(dim=-1, keepdim=True) + if num_fused_shared_experts == 0 + else topk_weights[:, :-1].sum(dim=-1, keepdim=True) + ) + topk_weights = topk_weights / topk_weights_sum + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor + + topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) + topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_weights, topk_ids + + +def is_power_of_two(n): + return n > 0 and math.log2(n).is_integer() + + +def _mask_topk_ids_padded_region( + topk_ids: torch.Tensor, + num_token_non_padded: Optional[torch.Tensor] = None, +): + if num_token_non_padded is None: + return + indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device) + topk_ids[indices >= num_token_non_padded, :] = -1 + + +@torch.compile(dynamic=True, backend=get_compiler_backend()) +def _biased_grouped_topk_postprocess( + topk_ids, expert_location_dispatch_info, num_token_non_padded +): + topk_ids = topk_ids_logical_to_physical(topk_ids, expert_location_dispatch_info) + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) + return topk_ids + + +def biased_grouped_topk_gpu( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + correction_bias: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + num_fused_shared_experts: int = 0, + routed_scaling_factor: Optional[float] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, +): + assert ( + routed_scaling_factor is not None + ), "routed_scaling_factor is required for biased_grouped_topk" + # TODO: moe_fused_gate kernel is not supported for num_fused_shared_experts > 0 now. + if ( + _is_cuda + and gating_output.shape[1] // num_expert_group + <= 32 # moe_fused_gate kernel ensure that num_experts/num_expert_group does not exceed MAX_VPT=32 now. And when kernel can handle MAX_VPT > 32, we can remove this assertion. + and is_power_of_two(correction_bias.shape[0]) + ): + topk_weights, topk_ids = moe_fused_gate( + gating_output.to(dtype=torch.float32), + correction_bias, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + ) + # TODO merge into kernel + if (expert_location_dispatch_info is not None) or ( + num_token_non_padded is not None + ): + topk_ids = _biased_grouped_topk_postprocess( + topk_ids, expert_location_dispatch_info, num_token_non_padded + ) + return topk_weights, topk_ids + elif _use_aiter: + assert not apply_routed_scaling_factor_on_output, "Not implemented" + token = gating_output.shape[0] + device = gating_output.device + assert ( + hidden_states.shape[0] == gating_output.shape[0] + ), f"Number of tokens mismatch: hidden_states.shape[0] = {hidden_states.shape[0]}, gating_output.shape[0] = {gating_output.shape[0]}" + topk_weights = torch.empty((token, topk), dtype=torch.float32, device=device) + topk_ids = torch.empty((token, topk), dtype=torch.int32, device=device) + aiter_biased_grouped_topk( + gating_output.to(dtype=torch.float32), + correction_bias, + topk_weights, + topk_ids, + num_expert_group, + topk_group, + renormalize, + routed_scaling_factor, + ) + return topk_weights, topk_ids + else: + return biased_grouped_topk_impl( + hidden_states, + gating_output, + correction_bias, + topk, + renormalize, + num_expert_group, + topk_group, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=routed_scaling_factor, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + +def biased_grouped_topk_cpu( + hidden_states: torch.Tensor, + gating_output: torch.Tensor, + correction_bias: torch.Tensor, + topk: int, + renormalize: bool, + num_expert_group: Optional[int] = None, + topk_group: Optional[int] = None, + compiled: bool = True, + num_fused_shared_experts: int = 0, + routed_scaling_factor: Optional[float] = None, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, + apply_routed_scaling_factor_on_output: Optional[bool] = False, +): + assert expert_location_dispatch_info is None + assert not apply_routed_scaling_factor_on_output, "Not implemented" + return torch.ops.sgl_kernel.biased_grouped_topk_cpu( + hidden_states, + gating_output, + correction_bias, + topk, + renormalize, + num_expert_group, + topk_group, + num_fused_shared_experts, + routed_scaling_factor, + num_token_non_padded, + ) + + +if _is_cpu and _is_cpu_amx_available: + biased_grouped_topk = biased_grouped_topk_cpu + grouped_topk = grouped_topk_cpu + fused_topk_native = fused_topk_cpu + fused_topk = fused_topk_cpu +else: + biased_grouped_topk = biased_grouped_topk_gpu + grouped_topk = grouped_topk_gpu + fused_topk_native = fused_topk_torch_native + + +def select_experts( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + topk_config: TopKConfig, + *, + num_token_non_padded: Optional[torch.Tensor] = None, + expert_location_dispatch_info: Optional[ExpertLocationDispatchInfo] = None, +) -> StandardTopKOutput: + + top_k = topk_config.top_k + use_grouped_topk = topk_config.use_grouped_topk + topk_group = topk_config.topk_group + num_expert_group = topk_config.num_expert_group + renormalize = topk_config.renormalize + num_fused_shared_experts = topk_config.num_fused_shared_experts + custom_routing_function = topk_config.custom_routing_function + correction_bias = topk_config.correction_bias + torch_native = topk_config.torch_native + routed_scaling_factor = topk_config.routed_scaling_factor + apply_routed_scaling_factor_on_output = ( + topk_config.apply_routed_scaling_factor_on_output + ) + + router_logits, correction_bias = ( + expert_location_dispatch.transform_select_experts_inputs( + router_logits=router_logits, + correction_bias=correction_bias, + info=expert_location_dispatch_info, + ) + ) + + # DeepSeek V2/V3/R1 series models use grouped_top_k + if use_grouped_topk: + assert topk_group is not None + assert num_expert_group is not None + if correction_bias is None: + topk_weights, topk_ids = grouped_topk( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + num_expert_group=num_expert_group, + topk_group=topk_group, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=routed_scaling_factor, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + else: + topk_weights, topk_ids = biased_grouped_topk( + hidden_states=hidden_states, + gating_output=router_logits, + correction_bias=correction_bias, + topk=top_k, + renormalize=renormalize, + num_expert_group=num_expert_group, + topk_group=topk_group, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=routed_scaling_factor, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + elif torch_native and custom_routing_function is None: + assert ( + num_token_non_padded is None + ), "num_token_non_padded is not yet supported in fused_topk_native" + assert expert_location_dispatch_info is None + assert not apply_routed_scaling_factor_on_output, "Not implemented" + topk_weights, topk_ids = fused_topk_native( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + correction_bias=correction_bias, + ) + elif custom_routing_function is None: + assert not apply_routed_scaling_factor_on_output, "Not implemented" + # Qwen3MOE uses fused_topk + topk_weights, topk_ids = fused_topk( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + num_token_non_padded=num_token_non_padded, + expert_location_dispatch_info=expert_location_dispatch_info, + ) + else: + assert ( + num_token_non_padded is None + ), "num_token_non_padded is not yet supported in custom_routing_function" + assert expert_location_dispatch_info is None + assert not apply_routed_scaling_factor_on_output, "Not implemented" + topk_weights, topk_ids = custom_routing_function( + hidden_states=hidden_states, + gating_output=router_logits, + topk=top_k, + renormalize=renormalize, + ) + + get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids) + + return StandardTopKOutput(topk_weights, topk_ids, router_logits) diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py new file mode 100644 index 000000000..b4e4ec424 --- /dev/null +++ b/python/sglang/srt/layers/moe/utils.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import importlib.util +import logging +from enum import Enum +from functools import lru_cache +from typing import TYPE_CHECKING, Optional + +from packaging import version as pkg_version + +from sglang.srt.distributed.parallel_state import get_moe_expert_parallel_world_size +from sglang.srt.layers.dp_attention import ( + get_attention_dp_size, + is_dp_attention_enabled, +) + +if TYPE_CHECKING: + from sglang.srt.server_args import ServerArgs + +logger = logging.getLogger(__name__) + + +class MoeA2ABackend(Enum): + + NONE = "none" + DEEPEP = "deepep" + + @classmethod + def _missing_(cls, value): + if value is None: + return cls.NONE + for member in cls: + if value == member.value: + return member + raise ValueError(f"No {cls.__name__} member for value {value}") + + def is_none(self): + return self == MoeA2ABackend.NONE + + def is_deepep(self): + return self == MoeA2ABackend.DEEPEP + + +class MoeRunnerBackend(Enum): + + AUTO = "auto" + TRITON = "triton" + TRITON_KERNEL = "triton_kernel" + FLASHINFER = "flashinfer_trtllm" + FLASHINFER_CUTLASS = "flashinfer_cutlass" + FLASHINFER_MXFP4 = "flashinfer_mxfp4" + + def is_auto(self): + return self == MoeRunnerBackend.AUTO + + def is_triton(self): + return self == MoeRunnerBackend.TRITON + + def is_triton_kernel(self): + return self == MoeRunnerBackend.TRITON_KERNEL + + def is_flashinfer_trtllm(self): + return self == MoeRunnerBackend.FLASHINFER + + def is_flashinfer_cutlass(self): + return self == MoeRunnerBackend.FLASHINFER_CUTLASS + + def is_flashinfer_mxfp4(self): + return self == MoeRunnerBackend.FLASHINFER_MXFP4 + + +class DeepEPMode(Enum): + + NORMAL = "normal" + LOW_LATENCY = "low_latency" + AUTO = "auto" + + def enable_normal(self) -> bool: + return self in [DeepEPMode.NORMAL, DeepEPMode.AUTO] + + def enable_low_latency(self) -> bool: + return self in [DeepEPMode.LOW_LATENCY, DeepEPMode.AUTO] + + def resolve(self, is_extend_in_batch: bool) -> DeepEPMode: + if self != DeepEPMode.AUTO: + return self + + if is_extend_in_batch: + return DeepEPMode.NORMAL + else: + return DeepEPMode.LOW_LATENCY + + def is_normal(self) -> bool: + return self == DeepEPMode.NORMAL + + def is_low_latency(self) -> bool: + return self == DeepEPMode.LOW_LATENCY + + def is_auto(self) -> bool: + return self == DeepEPMode.AUTO + + +MOE_A2A_BACKEND: Optional[MoeA2ABackend] = None +MOE_RUNNER_BACKEND: Optional[MoeRunnerBackend] = None +DEEPEP_MODE: Optional[DeepEPMode] = None +IS_TBO_ENABLED: Optional[bool] = None +TBO_TOKEN_DISTRIBUTION_THRESHOLD: Optional[float] = None +DEEPEP_CONFIG: Optional[str] = None +DISABLE_FLASHINFER_CUTLASS_MOE_FP4_ALLGATHER: Optional[bool] = None + + +def initialize_moe_config(server_args: ServerArgs): + global MOE_A2A_BACKEND + global MOE_RUNNER_BACKEND + global DEEPEP_MODE + global DEEPEP_CONFIG + global IS_TBO_ENABLED + global TBO_TOKEN_DISTRIBUTION_THRESHOLD + global DISABLE_FLASHINFER_CUTLASS_MOE_FP4_ALLGATHER + + MOE_A2A_BACKEND = MoeA2ABackend(server_args.moe_a2a_backend) + MOE_RUNNER_BACKEND = MoeRunnerBackend(server_args.moe_runner_backend) + DEEPEP_MODE = DeepEPMode(server_args.deepep_mode) + DEEPEP_CONFIG = server_args.deepep_config or "" + IS_TBO_ENABLED = server_args.enable_two_batch_overlap + TBO_TOKEN_DISTRIBUTION_THRESHOLD = server_args.tbo_token_distribution_threshold + DISABLE_FLASHINFER_CUTLASS_MOE_FP4_ALLGATHER = ( + server_args.disable_flashinfer_cutlass_moe_fp4_allgather + ) + + +def get_moe_a2a_backend() -> MoeA2ABackend: + global MOE_A2A_BACKEND + if MOE_A2A_BACKEND is None: + logger.warning("MOE_A2A_BACKEND is not initialized, using default backend") + MOE_A2A_BACKEND = MoeA2ABackend.NONE + return MOE_A2A_BACKEND + + +def get_moe_runner_backend() -> MoeRunnerBackend: + global MOE_RUNNER_BACKEND + if MOE_RUNNER_BACKEND is None: + logger.warning("MOE_RUNNER_BACKEND is not initialized, using triton backend") + MOE_RUNNER_BACKEND = MoeRunnerBackend.AUTO + return MOE_RUNNER_BACKEND + + +def get_deepep_mode() -> DeepEPMode: + global DEEPEP_MODE + if DEEPEP_MODE is None: + logger.warning("DEEPEP_MODE is not initialized, using auto mode") + DEEPEP_MODE = DeepEPMode.AUTO + return DEEPEP_MODE + + +def get_deepep_config() -> str: + global DEEPEP_CONFIG + if DEEPEP_CONFIG is None: + logger.warning("DEEPEP_CONFIG is not initialized, using default config") + DEEPEP_CONFIG = "" + return DEEPEP_CONFIG + + +def is_tbo_enabled() -> bool: + global IS_TBO_ENABLED + if IS_TBO_ENABLED is None: + IS_TBO_ENABLED = False + return IS_TBO_ENABLED + + +def get_tbo_token_distribution_threshold() -> float: + global TBO_TOKEN_DISTRIBUTION_THRESHOLD + if TBO_TOKEN_DISTRIBUTION_THRESHOLD is None: + logger.warning( + "TBO_TOKEN_DISTRIBUTION_THRESHOLD is not initialized, using 0.48" + ) + TBO_TOKEN_DISTRIBUTION_THRESHOLD = 0.48 + return TBO_TOKEN_DISTRIBUTION_THRESHOLD + + +@lru_cache(maxsize=1) +def should_use_flashinfer_trtllm_moe(): + result = get_moe_runner_backend().is_flashinfer_trtllm() and ( + not importlib.util.find_spec("flashinfer") + or pkg_version.parse(__import__("flashinfer").__version__) + >= pkg_version.parse("0.2.9rc1") + ) + return result + + +@lru_cache(maxsize=1) +def should_use_flashinfer_cutlass_moe_fp4_allgather(): + """ + Perform FP4 quantize before all-gather for flashinfer cutlass moe to reduce communication cost for high-throughput serving. + """ + return ( + not DISABLE_FLASHINFER_CUTLASS_MOE_FP4_ALLGATHER + and get_moe_runner_backend().is_flashinfer_cutlass() + and is_dp_attention_enabled() + and get_moe_expert_parallel_world_size() == get_attention_dp_size() + ) diff --git a/python/sglang/srt/layers/multimodal.py b/python/sglang/srt/layers/multimodal.py new file mode 100644 index 000000000..738c65830 --- /dev/null +++ b/python/sglang/srt/layers/multimodal.py @@ -0,0 +1,189 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Logits processing.""" + +import torch +import triton +import triton.language as tl + +FMIX32_C1 = 0x85EBCA6B +FMIX32_C2 = 0xC2B2AE35 +POS_C1 = 0x27D4EB2D +POS_C2 = 0x165667B1 + + +@triton.jit +def _rotl32(x, r: tl.constexpr): + return (x << r) | (x >> (32 - r)) + + +@triton.jit +def _fmix32(x, C1: tl.constexpr, C2: tl.constexpr): + c1 = tl.full((), C1, tl.uint32) + c2 = tl.full((), C2, tl.uint32) + x ^= x >> 16 + x = x * c1 + x ^= x >> 13 + x = x * c2 + x ^= x >> 16 + return x + + +@triton.jit +def hash_tiles32_kernel_blocked( + in_ptr, + out_ptr, + n_u32, + seed1, + seed2, + FM_C1: tl.constexpr, + FM_C2: tl.constexpr, + POS_A: tl.constexpr, + POS_B: tl.constexpr, + TILE: tl.constexpr, + BLOCK: tl.constexpr, + USE_CG: tl.constexpr, +): + pid = tl.program_id(axis=0) + base = pid * TILE + + s1 = tl.full((), seed1, tl.uint32) + s2 = tl.full((), seed2, tl.uint32) + posA = tl.full((), POS_A, tl.uint32) + posB = tl.full((), POS_B, tl.uint32) + + h1 = tl.zeros((), dtype=tl.uint32) + h2 = tl.zeros((), dtype=tl.uint32) + + for off in tl.static_range(0, TILE, BLOCK): + idx = base + off + tl.arange(0, BLOCK) + m = idx < n_u32 + + if USE_CG: + v = tl.load(in_ptr + idx, mask=m, other=0, cache_modifier=".cg") + else: + v = tl.load(in_ptr + idx, mask=m, other=0) + v = v.to(tl.uint32) + + iu = idx.to(tl.uint32) + p1 = (iu * posA + s1) ^ _rotl32(iu, 15) + p2 = (iu * posB + s2) ^ _rotl32(iu, 13) + + k1 = _fmix32(v ^ p1, C1=FM_C1, C2=FM_C2) + k2 = _fmix32(v ^ p2, C1=FM_C1, C2=FM_C2) + + zero32 = tl.zeros_like(k1) + k1 = tl.where(m, k1, zero32) + k2 = tl.where(m, k2, zero32) + + h1 += tl.sum(k1, axis=0).to(tl.uint32) + h2 += tl.sum(k2, axis=0).to(tl.uint32) + + nbytes = tl.full((), n_u32 * 4, tl.uint32) + h1 ^= nbytes + h2 ^= nbytes + h1 = _fmix32(h1, C1=FM_C1, C2=FM_C2) + h2 = ( + _fmix32(h2, C1=FMIX32_C1, C2=FMIX32_C2) + if False + else _fmix32(h2, C1=FM_C1, C2=FM_C2) + ) + + out = (h1.to(tl.uint64) << 32) | h2.to(tl.uint64) + tl.store(out_ptr + pid, out) + + +@triton.jit +def add_tree_reduce_u64_kernel(in_ptr, out_ptr, n_elems, CHUNK: tl.constexpr): + pid = tl.program_id(axis=0) + start = pid * CHUNK + h = tl.zeros((), dtype=tl.uint64) + for i in tl.static_range(0, CHUNK): + idx = start + i + m = idx < n_elems + v = tl.load(in_ptr + idx, mask=m, other=0).to(tl.uint64) + h += v + tl.store(out_ptr + pid, h) + + +def _as_uint32_words(t: torch.Tensor) -> torch.Tensor: + assert t.is_cuda, "Use .cuda() first" + tb = t.contiguous().view(torch.uint8) + nbytes = tb.numel() + pad = (4 - (nbytes & 3)) & 3 + if pad: + tb_p = torch.empty(nbytes + pad, dtype=torch.uint8, device=tb.device) + tb_p[:nbytes].copy_(tb) + tb_p[nbytes:].zero_() + tb = tb_p + return tb.view(torch.uint32) + + +def _final_splitmix64(x: int) -> int: + mask = (1 << 64) - 1 + x &= mask + x ^= x >> 30 + x = (x * 0xBF58476D1CE4E5B9) & mask + x ^= x >> 27 + x = (x * 0x94D049BB133111EB) & mask + x ^= x >> 31 + return x + + +@torch.inference_mode() +def gpu_tensor_hash( + tensor: torch.Tensor, + *, + seed: int = 0x243F6A88, + tile_words: int = 8192, + block_words: int = 256, + reduce_chunk: int = 1024, + num_warps: int = 4, + num_stages: int = 4, + use_cg: bool = True, +) -> int: + assert tensor.is_cuda, "Use .cuda() first" + u32 = _as_uint32_words(tensor) + n = u32.numel() + if n == 0: + return 0 + + grid1 = (triton.cdiv(n, tile_words),) + partials = torch.empty(grid1[0], dtype=torch.uint64, device=u32.device) + hash_tiles32_kernel_blocked[grid1]( + u32, + partials, + n, + seed1=seed & 0xFFFFFFFF, + seed2=((seed * 0x9E3779B1) ^ 0xDEADBEEF) & 0xFFFFFFFF, + FM_C1=FMIX32_C1, + FM_C2=FMIX32_C2, + POS_A=POS_C1, + POS_B=POS_C2, + TILE=tile_words, + BLOCK=block_words, + USE_CG=use_cg, + num_warps=num_warps, + num_stages=num_stages, + ) + + cur = partials + while cur.numel() > 1: + n_elems = cur.numel() + grid2 = (triton.cdiv(n_elems, reduce_chunk),) + nxt = torch.empty(grid2[0], dtype=torch.uint64, device=cur.device) + add_tree_reduce_u64_kernel[grid2](cur, nxt, n_elems, CHUNK=reduce_chunk) + cur = nxt + + return _final_splitmix64(int(cur.item())) diff --git a/python/sglang/srt/layers/parameter.py b/python/sglang/srt/layers/parameter.py new file mode 100644 index 000000000..1ea75d70c --- /dev/null +++ b/python/sglang/srt/layers/parameter.py @@ -0,0 +1,533 @@ +"""Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/parameter.py""" + +import logging +from fractions import Fraction +from typing import Callable, Optional, Union + +import torch +from torch.nn import Parameter + +from sglang.srt.utils import is_cpu + +__all__ = [ + "BasevLLMParameter", + "PackedvLLMParameter", + "PerTensorScaleParameter", + "ModelWeightParameter", + "ChannelQuantScaleParameter", + "GroupQuantScaleParameter", + "BlockQuantScaleParameter", + "PackedColumnParameter", + "RowvLLMParameter", +] + +logger = logging.getLogger(__name__) + +_is_cpu = is_cpu() + + +class BasevLLMParameter(Parameter): + """ + Base parameter for vLLM linear layers. Extends the torch.nn.parameter + by taking in a linear weight loader. Will copy the loaded weight + into the parameter when the provided weight loader is called. + """ + + def __new__(cls, data: torch.Tensor, **kwargs): + + return super().__new__(cls, data=data, requires_grad=False) + + def __init__(self, data: torch.Tensor, weight_loader: Callable): + """ + Initialize the BasevLLMParameter + + :param data: torch tensor with the parameter data + :param weight_loader: weight loader callable + + :returns: a torch.nn.parameter + """ + + self._weight_loader = weight_loader + + @property + def weight_loader(self): + return self._weight_loader + + def _assert_and_load(self, loaded_weight: torch.Tensor): + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + def load_column_parallel_weight(self, loaded_weight: torch.Tensor): + self._assert_and_load(loaded_weight) + + def load_row_parallel_weight(self, loaded_weight: torch.Tensor): + self._assert_and_load(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs): + self._assert_and_load(loaded_weight) + + def load_qkv_weight(self, loaded_weight: torch.Tensor, **kwargs): + self._assert_and_load(loaded_weight) + + +class _ColumnvLLMParameter(BasevLLMParameter): + """ + Private class defining weight loading functionality + (load_merged_column_weight, load_qkv_weight) + for parameters being loaded into linear layers with column + parallelism. This includes QKV and MLP layers which are + not already fused on disk. Requires an output dimension + to be defined. Called within the weight loader of + each of the column parallel linear layers. + """ + + def __init__(self, output_dim: int, **kwargs): + self._output_dim = output_dim + super().__init__(**kwargs) + + @property + def output_dim(self): + return self._output_dim + + def load_column_parallel_weight( + self, + loaded_weight: torch.Tensor, + tp_rank: int, + use_presharded_weights: bool = False, + ): + if not use_presharded_weights: + shard_size = self.data.shape[self.output_dim] + + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + if _is_cpu: + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + self.data, + loaded_weight, + 0, # param_data_start + tp_rank * shard_size, + self.output_dim, + shard_size, + ) + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + return + else: + loaded_weight = loaded_weight.narrow( + self.output_dim, tp_rank * shard_size, shard_size + ) + + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + def load_merged_column_weight(self, loaded_weight: torch.Tensor, **kwargs): + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + tp_rank = kwargs.get("tp_rank") + use_presharded_weights = kwargs.get("use_presharded_weights") + if ( + isinstance(self, (PackedColumnParameter, PackedvLLMParameter)) + and self.packed_dim == self.output_dim + ): + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size + ) + + param_data = self.data + + param_data = param_data.narrow(self.output_dim, shard_offset, shard_size) + + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + if _is_cpu: + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + tp_rank * shard_size, + self.output_dim, + shard_size, + not use_presharded_weights, + ) + else: + if not use_presharded_weights: + loaded_weight = loaded_weight.narrow( + self.output_dim, tp_rank * shard_size, shard_size + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + def load_qkv_weight( + self, + loaded_weight: torch.Tensor, + tp_rank: int, + use_presharded_weights: bool = False, + **kwargs, + ): + + shard_offset = kwargs.get("shard_offset") + shard_size = kwargs.get("shard_size") + shard_id = kwargs.get("shard_id") + num_heads = kwargs.get("num_heads") + + if ( + isinstance(self, (PackedColumnParameter, PackedvLLMParameter)) + and self.output_dim == self.packed_dim + ): + shard_size, shard_offset = self.adjust_shard_indexes_for_packing( + shard_offset=shard_offset, shard_size=shard_size + ) + + param_data = self.data + shard_id = tp_rank if shard_id == "q" else tp_rank // num_heads + param_data = param_data.narrow(self.output_dim, shard_offset, shard_size) + + if _is_cpu: + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + param_data, + loaded_weight, + 0, # param_data_start + shard_id * shard_size, + self.output_dim, + shard_size, + not use_presharded_weights, + ) + else: + if not use_presharded_weights: + loaded_weight = loaded_weight.narrow( + self.output_dim, shard_id * shard_size, shard_size + ) + + assert ( + param_data.shape == loaded_weight.shape + ), f"{param_data.shape=}, {loaded_weight.shape=}" + param_data.copy_(loaded_weight) + + +class RowvLLMParameter(BasevLLMParameter): + """ + Parameter class defining weight_loading functionality + (load_row_parallel_weight) for parameters being loaded + into linear layers with row parallel functionality. + Requires an input_dim to be defined. + """ + + def __init__(self, input_dim: int, **kwargs): + self._input_dim = input_dim + super().__init__(**kwargs) + + @property + def input_dim(self): + return self._input_dim + + def load_row_parallel_weight( + self, + loaded_weight: torch.Tensor, + tp_rank: int, + use_presharded_weights: bool = False, + ): + if not use_presharded_weights: + shard_size = self.data.shape[self.input_dim] + + from sglang.srt.model_loader.weight_utils import ( + narrow_padded_param_and_loaded_weight, + ) + + if _is_cpu: + param_data, loaded_weight = narrow_padded_param_and_loaded_weight( + self.data, + loaded_weight, + 0, # param_data_start + tp_rank * shard_size, + self.input_dim, + shard_size, + ) + + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + return + else: + loaded_weight = loaded_weight.narrow( + self.input_dim, tp_rank * shard_size, shard_size + ) + + if len(loaded_weight.shape) == 0: + loaded_weight = loaded_weight.reshape(1) + + assert self.data.shape == loaded_weight.shape + self.data.copy_(loaded_weight) + + +class ModelWeightParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for linear layer weights. Uses both column and + row parallelism. + """ + + pass + + +class GroupQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + grouped quantization. Uses both column and row parallelism. + """ + + pass + + +class ChannelQuantScaleParameter(_ColumnvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + channel-wise quantization. Equivalent to _ColumnvLLMParameter. + """ + + pass + + +class BlockQuantScaleParameter(_ColumnvLLMParameter, RowvLLMParameter): + """ + Parameter class for weight scales loaded for weights with + block-wise quantization. Uses both column and row parallelism. + """ + + pass + + +class PerTensorScaleParameter(BasevLLMParameter): + """ + Parameter class for scales where the number of scales is + equivalent to the number of logical matrices in fused linear + layers (e.g. for QKV, there are 3 scales loaded from disk). + This is relevant to weights with per-tensor quantization. + Adds functionality to map the scalers to a shard during + weight loading. + + Note: additional parameter manipulation may be handled + for each quantization config specifically, within + process_weights_after_loading + """ + + def __init__(self, **kwargs): + self.qkv_idxs = {"q": 0, "k": 1, "v": 2} + super().__init__(**kwargs) + + def _shard_id_as_int(self, shard_id: Union[str, int]) -> int: + if isinstance(shard_id, int): + return shard_id + + # if not int, assume shard_id for qkv + # map to int and return + assert isinstance(shard_id, str) + assert shard_id in self.qkv_idxs + return self.qkv_idxs[shard_id] + + # For row parallel layers, no sharding needed + # load weight into parameter as is + def load_row_parallel_weight(self, *args, **kwargs): + kwargs.pop("tp_rank", None) + kwargs.pop("use_presharded_weights", None) + super().load_row_parallel_weight(*args, **kwargs) + + def load_merged_column_weight(self, *args, **kwargs): + self._load_into_shard_id(*args, **kwargs) + + def load_qkv_weight(self, *args, **kwargs): + self._load_into_shard_id(*args, **kwargs) + + def load_column_parallel_weight(self, *args, **kwargs): + kwargs.pop("tp_rank", None) + kwargs.pop("use_presharded_weights", None) + super().load_row_parallel_weight(*args, **kwargs) + + def _load_into_shard_id( + self, loaded_weight: torch.Tensor, shard_id: Union[str, int], **kwargs + ): + """ + Slice the parameter data based on the shard id for + loading. + """ + + param_data = self.data + shard_id = self._shard_id_as_int(shard_id) + + # AutoFP8 scales do not have a shape + # compressed-tensors scales do have a shape + if len(loaded_weight.shape) != 0: + assert loaded_weight.shape[0] == 1 + loaded_weight = loaded_weight[0] + + param_data = param_data[shard_id] + assert param_data.shape == loaded_weight.shape + param_data.copy_(loaded_weight) + + +class PackedColumnParameter(_ColumnvLLMParameter): + """ + Parameter for model parameters which are packed on disk + and support column parallelism only. See PackedvLLMParameter + for more details on the packed properties. + """ + + def __init__( + self, + packed_factor: Union[int, Fraction], + packed_dim: int, + marlin_tile_size: Optional[int] = None, + **kwargs, + ): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + self._marlin_tile_size = marlin_tile_size + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + @property + def marlin_tile_size(self): + return self._marlin_tile_size + + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset): + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + marlin_tile_size=self.marlin_tile_size, + ) + + +class PackedvLLMParameter(ModelWeightParameter): + """ + Parameter for model weights which are packed on disk. + Example: GPTQ Marlin weights are int4 or int8, packed into int32. + Extends the ModelWeightParameter to take in the + packed factor, the packed dimension, and optionally, marlin + tile size for marlin kernels. Adjusts the shard_size and + shard_offset for fused linear layers model weight loading + by accounting for packing and optionally, marlin tile size. + """ + + def __init__( + self, + packed_factor: Union[int, Fraction], + packed_dim: int, + marlin_tile_size: Optional[int] = None, + **kwargs, + ): + self._packed_factor = packed_factor + self._packed_dim = packed_dim + self._marlin_tile_size = marlin_tile_size + super().__init__(**kwargs) + + @property + def packed_dim(self): + return self._packed_dim + + @property + def packed_factor(self): + return self._packed_factor + + @property + def marlin_tile_size(self): + return self._marlin_tile_size + + def adjust_shard_indexes_for_packing(self, shard_size, shard_offset): + return _adjust_shard_indexes_for_packing( + shard_size=shard_size, + shard_offset=shard_offset, + packed_factor=self.packed_factor, + marlin_tile_size=self.marlin_tile_size, + ) + + +def permute_param_layout_( + param: BasevLLMParameter, input_dim: int, output_dim: int, **kwargs +) -> BasevLLMParameter: + """ + Permute a parameter's layout to the specified input and output dimensions, + useful for forcing the parameter into a known layout, for example, if I need + a packed (quantized) weight matrix to be in the layout + {input_dim = 0, output_dim = 1, packed_dim = 0} + then I can call: + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + to ensure x is in the correct layout (permuting it to the correct layout if + required, asserting if it cannot get it to the correct layout) + """ + + curr_input_dim = getattr(param, "input_dim", None) + curr_output_dim = getattr(param, "output_dim", None) + + if curr_input_dim is None or curr_output_dim is None: + assert param.data.dim() == 2, ( + "permute_param_layout_ only supports 2D parameters when either " + "input_dim or output_dim is not set" + ) + + # if one of the dimensions is not set, set it to the opposite of the other + # we can only do this since we asserted the parameter is 2D above + if curr_input_dim is None: + assert curr_output_dim is not None, "either input or output dim must be set" + curr_input_dim = (curr_output_dim + 1) % 2 + if curr_output_dim is None: + assert curr_input_dim is not None, "either input or output dim must be set" + curr_output_dim = (curr_input_dim + 1) % 2 + + # create permutation from the current layout to the layout with + # self.input_dim at input_dim and self.output_dim at output_dim preserving + # other dimensions + perm = [ + i for i in range(param.data.dim()) if i not in [curr_input_dim, curr_output_dim] + ] + perm.insert(input_dim, curr_input_dim) + perm.insert(output_dim, curr_output_dim) + + if "packed_dim" in kwargs: + assert ( + hasattr(param, "packed_dim") + and param.packed_dim == perm[kwargs["packed_dim"]] + ), "permute_param_layout_ currently doesn't support repacking" + + param.data = param.data.permute(*perm) + if hasattr(param, "_input_dim"): + param._input_dim = input_dim + if hasattr(param, "_output_dim"): + param._output_dim = output_dim + if "packed_dim" in kwargs and hasattr(param, "_packed_dim"): + param._packed_dim = kwargs["packed_dim"] + + return param + + +def _adjust_shard_indexes_for_marlin(shard_size, shard_offset, marlin_tile_size): + return shard_size * marlin_tile_size, shard_offset * marlin_tile_size + + +def _adjust_shard_indexes_for_packing( + shard_size, shard_offset, packed_factor, marlin_tile_size +): + shard_size = shard_size // packed_factor + shard_offset = shard_offset // packed_factor + if marlin_tile_size is not None: + return _adjust_shard_indexes_for_marlin( + shard_size=shard_size, + shard_offset=shard_offset, + marlin_tile_size=marlin_tile_size, + ) + return shard_size, shard_offset diff --git a/python/sglang/srt/layers/pooler.py b/python/sglang/srt/layers/pooler.py new file mode 100644 index 000000000..26bc5899e --- /dev/null +++ b/python/sglang/srt/layers/pooler.py @@ -0,0 +1,112 @@ +# adapted from +# https://github.com/vllm-project/vllm/blob/82a1b1a82b1fbb454c82a9ef95730b929c9b270c/vllm/model_executor/layers/pooler.py + +from dataclasses import dataclass +from enum import IntEnum +from typing import Optional + +import torch +import torch.nn as nn +from transformers import PretrainedConfig + +from sglang.srt.layers.activation import get_cross_encoder_activation_function +from sglang.srt.model_executor.model_runner import ForwardBatch + + +class PoolingType(IntEnum): + LAST = 0 + CLS = 1 + + +@dataclass +class EmbeddingPoolerOutput: + embeddings: torch.Tensor + + +class Pooler(nn.Module): + """A layer that pools specific information from hidden states. + This layer does the following: + 1. Extracts specific tokens or aggregates data based on pooling method. + 2. Normalizes output if specified. + 3. Returns structured results as `PoolerOutput`. + Attributes: + pooling_type: The type of pooling to use (LAST, AVERAGE, MAX). + normalize: Whether to normalize the pooled data. + """ + + def __init__(self, pooling_type: PoolingType, normalize: bool): + super().__init__() + self.pooling_type = pooling_type + self.normalize = normalize + + def forward( + self, hidden_states: torch.Tensor, forward_batch: ForwardBatch + ) -> EmbeddingPoolerOutput: + if self.pooling_type == PoolingType.LAST: + last_token_indices = torch.cumsum(forward_batch.extend_seq_lens, dim=0) - 1 + pooled_data = hidden_states[last_token_indices] + elif self.pooling_type == PoolingType.CLS: + prompt_lens = forward_batch.extend_seq_lens + first_token_flat_indices = torch.zeros_like(prompt_lens) + first_token_flat_indices[1:] += torch.cumsum(prompt_lens, dim=0)[:-1] + pooled_data = hidden_states[first_token_flat_indices] + else: + raise ValueError(f"Invalid pooling type: {self.pooling_type}") + + if self.normalize: + pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1) + + return EmbeddingPoolerOutput(embeddings=pooled_data) + + +class CrossEncodingPooler(nn.Module): + """A layer that pools specific information from hidden states. + + This layer does the following: + 1. Extracts specific tokens or aggregates data based on pooling method. + 2. Normalizes output if specified. + 3. Returns structured results as `EmbeddingPoolerOutput`. + """ + + def __init__( + self, + config: PretrainedConfig, + classifier: nn.Module, + pooler: Optional[nn.Module] = None, + ): + super().__init__() + self.classifier = classifier + self.pooler = pooler + self.default_activation_function = get_cross_encoder_activation_function(config) + + def forward( + self, + hidden_states: torch.Tensor, + forward_batch: ForwardBatch, + ) -> EmbeddingPoolerOutput: + """Pools sentence pair scores from the hidden_states.""" + + prompt_lens = forward_batch.extend_seq_lens + + offset = 0 + pooled_data_lst = [] + for prompt_len in prompt_lens: + pooled_data_i = hidden_states[offset : offset + prompt_len] + + if self.pooler is not None: + final_shape_tensor = self.pooler(pooled_data_i, forward_batch) + else: + final_shape_tensor = self.classifier(pooled_data_i) + + pooled_data_lst.append(final_shape_tensor) + offset += prompt_len + + pooled_output = torch.stack(pooled_data_lst) + + if self.pooler is not None: + # apply classifier once on the full batch if possible + pooled_output = self.classifier(pooled_output) + + scores = self.default_activation_function(pooled_output).squeeze(-1) + + return EmbeddingPoolerOutput(embeddings=scores) diff --git a/python/sglang/srt/layers/quantization/__init__.py b/python/sglang/srt/layers/quantization/__init__.py new file mode 100644 index 000000000..ff3c2b148 --- /dev/null +++ b/python/sglang/srt/layers/quantization/__init__.py @@ -0,0 +1,224 @@ +# Adapted from https://raw.githubusercontent.com/vllm-project/vllm/v0.5.5/vllm/model_executor/layers/quantization/__init__.py +from __future__ import annotations + +import builtins +import inspect +from typing import TYPE_CHECKING, Dict, Optional, Type + +import torch + +try: + from vllm.model_executor.layers.quantization.aqlm import AQLMConfig + from vllm.model_executor.layers.quantization.bitsandbytes import BitsAndBytesConfig + from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( + CompressedTensorsW8A8Fp8MoEMethod, + CompressedTensorsWNA16MoEMethod, + ) + from vllm.model_executor.layers.quantization.deepspeedfp import DeepSpeedFPConfig + from vllm.model_executor.layers.quantization.experts_int8 import ExpertsInt8Config + from vllm.model_executor.layers.quantization.gguf import GGUFConfig + from vllm.model_executor.layers.quantization.gptq_marlin_24 import ( + GPTQMarlin24Config, + ) + from vllm.model_executor.layers.quantization.marlin import MarlinConfig + from vllm.model_executor.layers.quantization.qqq import QQQConfig + from vllm.model_executor.layers.quantization.tpu_int8 import Int8TpuConfig + + VLLM_AVAILABLE = True +except ImportError as e: + VLLM_AVAILABLE = False + VLLM_IMPORT_ERROR = e + + # Define empty classes as placeholders when vllm is not available + class DummyConfig: + def override_quantization_method(self, *args, **kwargs): + return None + + AQLMConfig = BitsAndBytesConfig = CompressedTensorsConfig = DeepSpeedFPConfig = ( + ExpertsInt8Config + ) = GGUFConfig = GPTQMarlin24Config = MarlinConfig = QQQConfig = Int8TpuConfig = ( + DummyConfig + ) + + +from sglang.srt.layers.quantization.awq import AWQConfig, AWQMarlinConfig +from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.blockwise_int8 import BlockInt8Config +from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsConfig, +) +from sglang.srt.layers.quantization.fp8 import Fp8Config +from sglang.srt.layers.quantization.fpgemm_fp8 import FBGEMMFp8Config +from sglang.srt.layers.quantization.gptq import GPTQConfig, GPTQMarlinConfig +from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptFp4Config, + ModelOptFp8Config, +) +from sglang.srt.layers.quantization.moe_wna16 import MoeWNA16Config +from sglang.srt.layers.quantization.mxfp4 import Mxfp4Config +from sglang.srt.layers.quantization.petit import PetitNvFp4Config +from sglang.srt.layers.quantization.qoq import QoQConfig +from sglang.srt.layers.quantization.w4afp8 import W4AFp8Config +from sglang.srt.layers.quantization.w8a8_fp8 import W8A8Fp8Config +from sglang.srt.layers.quantization.w8a8_int8 import W8A8Int8Config +from sglang.srt.utils import is_cuda, is_hip, mxfp_supported + +_is_mxfp_supported = mxfp_supported() + +if TYPE_CHECKING: + from sglang.srt.layers.moe.topk import TopKOutput + +# Base quantization methods that don't depend on vllm +BASE_QUANTIZATION_METHODS: Dict[str, Type[QuantizationConfig]] = { + "fp8": Fp8Config, + "blockwise_int8": BlockInt8Config, + "modelopt": ModelOptFp8Config, + "modelopt_fp4": ModelOptFp4Config, + "w8a8_int8": W8A8Int8Config, + "w8a8_fp8": W8A8Fp8Config, + "awq": AWQConfig, + "awq_marlin": AWQMarlinConfig, + "gptq": GPTQConfig, + "gptq_marlin": GPTQMarlinConfig, + "moe_wna16": MoeWNA16Config, + "compressed-tensors": CompressedTensorsConfig, + "qoq": QoQConfig, + "w4afp8": W4AFp8Config, + "petit_nvfp4": PetitNvFp4Config, + "fbgemm_fp8": FBGEMMFp8Config, +} + + +if is_cuda(): + BASE_QUANTIZATION_METHODS.update( + { + "quark": Mxfp4Config, + "mxfp4": Mxfp4Config, + } + ) +elif _is_mxfp_supported and is_hip(): + from sglang.srt.layers.quantization.quark.quark import QuarkConfig + + BASE_QUANTIZATION_METHODS.update( + { + "quark": QuarkConfig, + "mxfp4": Mxfp4Config, + } + ) +# VLLM-dependent quantization methods +VLLM_QUANTIZATION_METHODS = { + "aqlm": AQLMConfig, + "deepspeedfp": DeepSpeedFPConfig, + "tpu_int8": Int8TpuConfig, + "marlin": MarlinConfig, + "gguf": GGUFConfig, + "gptq_marlin_24": GPTQMarlin24Config, + "bitsandbytes": BitsAndBytesConfig, + "qqq": QQQConfig, + "experts_int8": ExpertsInt8Config, +} + +QUANTIZATION_METHODS = {**BASE_QUANTIZATION_METHODS, **VLLM_QUANTIZATION_METHODS} + + +def get_quantization_config(quantization: str) -> Type[QuantizationConfig]: + if quantization not in QUANTIZATION_METHODS: + raise ValueError( + f"Invalid quantization method: {quantization}. " + f"Available methods: {list(QUANTIZATION_METHODS.keys())}" + ) + if quantization in VLLM_QUANTIZATION_METHODS and not VLLM_AVAILABLE: + raise ValueError( + f"{quantization} quantization requires some operators from vllm. " + f"Please install vllm by `pip install vllm==0.9.0.1`\n" + f"Import error: {VLLM_IMPORT_ERROR}" + ) + + return QUANTIZATION_METHODS[quantization] + + +original_isinstance = builtins.isinstance + + +def monkey_patch_isinstance_for_vllm_base_layer(reverse: bool = False): + """ + Patch isinstance so that the `get_quant_method` in vllm's QuantizationConfig + can recognize sglang layers + """ + if not VLLM_AVAILABLE: + return + + if reverse: + builtins.isinstance = original_isinstance + return + + from vllm.model_executor.layers.fused_moe import FusedMoE + from vllm.model_executor.layers.linear import LinearBase + from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, + ) + + from sglang.srt.layers.linear import LinearBase as PatchedLinearBase + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE as PatchedFusedMoE + from sglang.srt.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding as PatchedVocabParallelEmbedding, + ) + + def patched_isinstance(obj, classinfo): + if classinfo is LinearBase: + return original_isinstance(obj, PatchedLinearBase) + if classinfo is FusedMoE: + return original_isinstance(obj, PatchedFusedMoE) + if classinfo is VocabParallelEmbedding: + return original_isinstance(obj, PatchedVocabParallelEmbedding) + return original_isinstance(obj, classinfo) + + builtins.isinstance = patched_isinstance + + +def monkey_patch_moe_apply(class_obj: "FusedMoEMethodBase"): + """ + Monkey patch the apply function of vllm's FusedMoEMethodBase. + Convert sglang arguments to vllm arguments. + """ + original_apply = class_obj.apply + sig = inspect.signature(original_apply) + param_names = list(sig.parameters.keys()) + has_correction_bias = "e_score_correction_bias" in param_names + + def new_apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + topk_output: TopKOutput, + *, + activation: str = "silu", + apply_router_weight_on_input: bool = False, + inplace: bool = True, + no_combine: bool = False, + routed_scaling_factor: Optional[float] = None, + ): + assert activation == "silu" + assert inplace and not no_combine + + kwargs = { + "self": self, + "layer": layer, + "x": x, + "topk_output": topk_output, + } + return original_apply(**kwargs) + + setattr(class_obj, "apply", new_apply) + + +def monkey_patch_quant_configs(): + """Apply all monkey patches in one place.""" + + monkey_patch_moe_apply(CompressedTensorsW8A8Fp8MoEMethod) + monkey_patch_moe_apply(CompressedTensorsWNA16MoEMethod) + + +# Only apply monkey patches if vllm is available +if VLLM_AVAILABLE: + monkey_patch_quant_configs() diff --git a/python/sglang/srt/layers/quantization/awq.py b/python/sglang/srt/layers/quantization/awq.py new file mode 100644 index 000000000..9cba60c2b --- /dev/null +++ b/python/sglang/srt/layers/quantization/awq.py @@ -0,0 +1,782 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import logging +import warnings +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional + +import torch + +from sglang.srt.layers.linear import LinearBase, set_weight_attrs +from sglang.srt.layers.parameter import GroupQuantScaleParameter, PackedvLLMParameter +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.marlin_utils import ( + apply_awq_marlin_linear, + awq_to_marlin_zero_points, + check_marlin_supported, + check_marlin_supports_layer, + check_moe_marlin_supports_layer, + marlin_make_empty_g_idx, + marlin_make_workspace, + marlin_moe_permute_scales, + marlin_permute_scales, + moe_awq_to_marlin_zero_points, + verify_marlin_supported, + verify_marlin_supports_shape, +) +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import get_scalar_types, replace_parameter + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher import ( + StandardDispatchOutput, + CombineInput, + ) + +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() +if _is_cuda: + from sgl_kernel import ( + awq_dequantize, + awq_marlin_moe_repack, + awq_marlin_repack, + fused_marlin_moe, + ) + + +elif _is_hip: + from sglang.srt.layers.quantization.awq_triton import ( + awq_dequantize_triton as awq_dequantize, + ) + + warnings.warn(f"HIP does not support fused_marlin_moe currently.") +else: + warnings.warn(f"Only CUDA and HIP support AWQ currently.") + +logger = logging.getLogger(__name__) + + +ScalarType, scalar_types = get_scalar_types() + + +def is_layer_skipped_awq(prefix: str, modules_to_not_convert: List[str]): + return any(module_name in prefix for module_name in modules_to_not_convert) + + +class AWQConfig(QuantizationConfig): + """Config class for AWQ. + + Reference: https://arxiv.org/abs/2306.00978 + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + zero_point: bool, + modules_to_not_convert: Optional[List[str]] = None, + ) -> None: + super().__init__() + self.weight_bits = weight_bits + self.group_size = group_size + self.zero_point = zero_point + self.modules_to_not_convert = modules_to_not_convert or [] + + if self.weight_bits != 4: + raise ValueError( + "Currently, only 4-bit weight quantization is supported for " + f"AWQ, but got {self.weight_bits} bits." + ) + self.pack_factor = 32 // self.weight_bits + + def __repr__(self) -> str: + return ( + f"AWQConfig(weight_bits={self.weight_bits}, " + f"group_size={self.group_size}, " + f"zero_point={self.zero_point}, " + f"modules_to_not_convert={self.modules_to_not_convert})" + ) + + def get_scaled_act_names(self) -> List[str]: + return [] + + def get_name(self) -> str: + return "awq" + + def get_supported_act_dtypes(self) -> List[torch.dtype]: + return [torch.half] + + @classmethod + def get_min_capability(cls) -> int: + # The AWQ kernel only supports Turing or newer GPUs. + return 75 + + @staticmethod + def get_config_filenames() -> List[str]: + return [ + "quant_config.json", # E.g., casperhansen/vicuna-7b-v1.5-awq + # E.g., abhinavkulkarni/mosaicml-mpt-7b-instruct-w4-g128-awq + "quantize_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> AWQConfig: + weight_bits = cls.get_from_keys(config, ["w_bit", "bits"]) + group_size = cls.get_from_keys(config, ["q_group_size", "group_size"]) + zero_point = cls.get_from_keys(config, ["zero_point"]) + modules_to_not_convert = cls.get_from_keys_or( + config, ["modules_to_not_convert"], None + ) + return cls(weight_bits, group_size, zero_point, modules_to_not_convert) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[LinearMethodBase]: + from sglang.srt.layers.linear import LinearBase + + if isinstance(layer, LinearBase): + if is_layer_skipped_awq(prefix, self.modules_to_not_convert): + return UnquantizedLinearMethod() + return AWQLinearMethod(self) + return None + + +class AWQMarlinConfig(QuantizationConfig): + """Config class for AWQ Marlin""" + + # num_bits -> type + TYPE_MAP = { + 4: scalar_types.uint4, + 8: scalar_types.uint8, + } + + def __init__( + self, + weight_bits: int, + group_size: int, + zero_point: bool, + lm_head_quantized: bool, + modules_to_not_convert: Optional[list[str]], + full_config: dict[str, Any], + ) -> None: + super().__init__() + self.pack_factor = 32 // weight_bits # packed into int32 + self.group_size = group_size + self.zero_point = zero_point + self.lm_head_quantized = lm_head_quantized + self.weight_bits = weight_bits + self.modules_to_not_convert = modules_to_not_convert or [] + self.full_config = full_config + + if self.weight_bits not in self.TYPE_MAP: + raise ValueError( + f"Unsupported num_bits = {self.weight_bits}. " + f"Supported num_bits = {self.TYPE_MAP.keys()}" + ) + + self.quant_type = self.TYPE_MAP[self.weight_bits] + + verify_marlin_supported( + self.quant_type, group_size=self.group_size, has_zp=self.zero_point + ) + + def __repr__(self) -> str: + return ( + f"AWQMarlinConfig(quant_type={self.quant_type}, " + f"group_size={self.group_size}, " + f"zero_point={self.zero_point}, " + f"lm_head_quantized={self.lm_head_quantized}, " + f"modules_to_not_convert={self.modules_to_not_convert})" + ) + + def get_scaled_act_names(self) -> List[str]: + return [] + + @classmethod + def get_name(cls) -> str: + return "awq_marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> list[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> AWQMarlinConfig: + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + zero_point = cls.get_from_keys(config, ["zero_point"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) + modules_to_not_convert = cls.get_from_keys_or( + config, ["modules_to_not_convert"], None + ) + return cls( + weight_bits, + group_size, + zero_point, + lm_head_quantized, + modules_to_not_convert, + config, + ) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg) + is_valid_user_quant = ( + user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin" + ) + + if can_convert and is_valid_user_quant: + msg = ( + "The model is convertible to {} during runtime." + " Using {} kernel.".format(cls.get_name(), cls.get_name()) + ) + logger.info(msg) + return cls.get_name() + + if can_convert and user_quant == "awq": + logger.info( + "Detected that the model can run with awq_marlin" + ", however you specified quantization=awq explicitly," + " so forcing awq. Use quantization=awq_marlin for" + " faster inference" + ) + return None + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead + + if isinstance(layer, LinearBase) or ( + isinstance(layer, ParallelLMHead) and self.lm_head_quantized + ): + if is_layer_skipped_awq(prefix, self.modules_to_not_convert): + return UnquantizedLinearMethod() + # Check if the layer is supported by AWQMarlin. + if not check_marlin_supports_layer(layer, self.group_size): + logger.warning_once( + "Layer '%s' is not supported by AWQMarlin. Falling back to unoptimized AWQ kernels.", # noqa: E501 + prefix, + ) + return AWQConfig.from_config(self.full_config).get_quant_method( + layer, prefix + ) + return AWQMarlinLinearMethod(self) + elif isinstance(layer, FusedMoE): + from sglang.srt.layers.quantization.moe_wna16 import MoeWNA16Config + + if not check_moe_marlin_supports_layer(layer, self.group_size): + logger.warning_once( + f"Layer '{prefix}' is not supported by AWQMoeMarlin. " + "Falling back to Moe WNA16 kernels." + ) + return MoeWNA16Config.from_config(self.full_config).get_quant_method( + layer, prefix + ) + return AWQMoEMethod(self) + return None + + @classmethod + def is_awq_marlin_compatible(cls, quant_config: dict[str, Any]): + # Extract data from quant config. + quant_method = quant_config.get("quant_method", "").lower() + num_bits = quant_config.get("bits") + group_size = quant_config.get("group_size") + zero_point = quant_config.get("zero_point") + + if not _is_cuda: + return False + + if quant_method != "awq": + return False + + # If we cannot find the info needed in the config, cannot convert. + if num_bits is None or group_size is None or zero_point is None: + return False + + if num_bits not in cls.TYPE_MAP: + return False + + return check_marlin_supported( + quant_type=cls.TYPE_MAP[num_bits], group_size=group_size, has_zp=zero_point + ) + + +class AWQLinearMethod(LinearMethodBase): + """Linear method for AWQ. + + Args: + quant_config: The AWQ quantization config. + """ + + def __init__(self, quant_config: AWQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + if input_size_per_partition % self.quant_config.group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + + weight_loader = extra_weight_attrs.get("weight_loader") + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + qzeros = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + scales = GroupQuantScaleParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader, + ) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qweight = layer.qweight + scales = layer.scales + qzeros = layer.qzeros + pack_factor = self.quant_config.pack_factor + out_shape = x.shape[:-1] + (qweight.shape[-1] * pack_factor,) + reshaped_x = x.reshape(-1, x.shape[-1]) + out = awq_dequantize(qweight, scales, qzeros) + out = torch.matmul(reshaped_x, out) + + if bias is not None: + out.add_(bias) + return out.reshape(out_shape) + + +class AWQMarlinLinearMethod(LinearMethodBase): + """Linear method for AWQ Marlin. + + Args: + quant_config: The AWQ Marlin quantization config. + """ + + def __init__(self, quant_config: AWQMarlinConfig) -> None: + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + del output_size + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + # Normalize group_size + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + verify_marlin_supports_shape( + output_size_per_partition=output_size_per_partition, + input_size_per_partition=input_size_per_partition, + input_size=input_size, + group_size=group_size, + ) + + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + num_groups = input_size_per_partition // group_size + + qzeros = PackedvLLMParameter( + data=torch.empty( + num_groups, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + scales = GroupQuantScaleParameter( + data=torch.empty( + num_groups, + output_size_per_partition, + dtype=params_dtype, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader, + ) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.num_groups = num_groups + + # TODO: Update this docs + # Checkpoints are serialized in AutoAWQ format, which is different from the + # marlin format. This function is called after the weights are loaded. + # Here, we handle the repacking + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + device = layer.qweight.device + layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) + + # Allocate marlin workspace + layer.workspace = marlin_make_workspace(device) + + # Repack weights from AWQ format to marlin format. + marlin_qweight = awq_marlin_repack( + layer.qweight, + size_k=layer.input_size_per_partition, + size_n=layer.output_size_per_partition, + num_bits=self.quant_config.quant_type.size_bits, + ) + replace_parameter(layer, "qweight", marlin_qweight) + + # Permute scales from AWQ format to marlin format. + marlin_scales = marlin_permute_scales( + layer.scales, + size_k=layer.input_size_per_partition, + size_n=layer.output_size_per_partition, + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "scales", marlin_scales) + + # Permute zero-points from AWQ format to marlin format. + marlin_zp = awq_to_marlin_zero_points( + layer.qzeros, + size_k=layer.num_groups, + size_n=layer.output_size_per_partition, + num_bits=self.quant_config.quant_type.size_bits, + ) + replace_parameter(layer, "qzeros", marlin_zp) + + # Not-used + layer.g_idx = marlin_make_empty_g_idx(device) + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_awq_marlin_linear( + input=x, + weight=layer.qweight, + weight_scale=layer.scales, + weight_zp=layer.qzeros, + g_idx=layer.g_idx, + g_idx_sort_indices=layer.g_idx_sort_indices, + workspace=layer.workspace, + quant_type=self.quant_config.quant_type, + output_size_per_partition=layer.output_size_per_partition, + input_size_per_partition=layer.input_size_per_partition, + bias=bias, + ) + + +class AWQMoEMethod(FusedMoEMethodBase): + + def __init__(self, quant_config: AWQMarlinConfig): + self.quant_config = quant_config + if self.quant_config.weight_bits != 4: + raise ValueError("AWQMoEMethod only supports 4bit now.") + self.quant_type = scalar_types.uint4 + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + # Delay the import to avoid circular dependency + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + extra_weight_attrs.update( + { + "is_transposed": True, + "quant_method": FusedMoeWeightScaleSupported.GROUP.value, + } + ) + + w13_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qweight", w13_qweight) + set_weight_attrs(w13_qweight, extra_weight_attrs) + + w2_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition, + hidden_size // self.quant_config.pack_factor, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qweight", w2_qweight) + set_weight_attrs(w2_qweight, extra_weight_attrs) + + num_groups_w13 = hidden_size // self.quant_config.group_size + num_groups_w2 = intermediate_size_per_partition // self.quant_config.group_size + + # WEIGHT_SCALES + # Allocate 2 scales for w1 and w3 respectively. + w13_scales = torch.nn.Parameter( + torch.empty( + num_experts, + num_groups_w13, + intermediate_size_per_partition * 2, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_scales", w13_scales) + set_weight_attrs(w13_scales, extra_weight_attrs) + + w2_scales = torch.nn.Parameter( + torch.empty(num_experts, num_groups_w2, hidden_size, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_scales", w2_scales) + set_weight_attrs(w2_scales, extra_weight_attrs) + + # WEIGHT_ZERO_POINT + # Allocate 2 zero points for w1 and w3 respectively. + w13_qzeros = torch.nn.Parameter( + torch.empty( + num_experts, + num_groups_w13, + 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qzeros", w13_qzeros) + set_weight_attrs(w13_qzeros, extra_weight_attrs) + + w2_qzeros = torch.nn.Parameter( + torch.empty( + num_experts, + num_groups_w2, + hidden_size // self.quant_config.pack_factor, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qzeros", w2_qzeros) + set_weight_attrs(w2_qzeros, extra_weight_attrs) + + device = layer.w13_qweight.device + layer.workspace = marlin_make_workspace(device, 4) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + num_experts = layer.w13_qweight.shape[0] + device = layer.w13_qweight.device + + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + + marlin_w13_qweight = awq_marlin_moe_repack( + layer.w13_qweight, + layer.w13_g_idx_sort_indices, + size_k=layer.w13_qweight.shape[1], + size_n=layer.w13_qweight.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w13_qweight", marlin_w13_qweight) + + marlin_w2_qweight = awq_marlin_moe_repack( + layer.w2_qweight, + layer.w2_g_idx_sort_indices, + size_k=layer.w2_qweight.shape[1], + size_n=layer.w2_qweight.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w2_qweight", marlin_w2_qweight) + + # hidden_size->intermediate_size + marlin_w13_scales = marlin_moe_permute_scales( + s=layer.w13_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w13_scales.shape[2], + group_size=self.quant_config.group_size, + ) + + replace_parameter(layer, "w13_scales", marlin_w13_scales) + + marlin_w2_scales = marlin_moe_permute_scales( + s=layer.w2_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w2_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w2_scales", marlin_w2_scales) + + marlin_w13_zp = moe_awq_to_marlin_zero_points( + layer.w13_qzeros, + size_k=layer.w13_qzeros.shape[1], + size_n=layer.w13_qzeros.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w13_qzeros", marlin_w13_zp) + + marlin_w2_zp = moe_awq_to_marlin_zero_points( + layer.w2_qzeros, + size_k=layer.w2_qzeros.shape[1], + size_n=layer.w2_qzeros.shape[2] * self.quant_config.pack_factor, + num_bits=self.quant_config.weight_bits, + ) + replace_parameter(layer, "w2_qzeros", marlin_w2_zp) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + assert ( + self.moe_runner_config.activation == "silu" + ), "Only SiLU activation is supported." + + # The input must currently be float16 + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + orig_dtype = x.dtype + x = x.half() + + topk_weights, topk_ids, router_logits = topk_output + + output = fused_marlin_moe( + x, + layer.w13_qweight, + layer.w2_qweight, + layer.w13_scales, + layer.w2_scales, + router_logits, + topk_weights, + topk_ids, + sort_indices1=layer.w13_g_idx_sort_indices, + sort_indices2=layer.w2_g_idx_sort_indices, + w1_zeros=layer.w13_qzeros, + w2_zeros=layer.w2_qzeros, + num_bits=self.quant_config.weight_bits, + ).to(orig_dtype) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/awq_triton.py b/python/sglang/srt/layers/quantization/awq_triton.py new file mode 100644 index 000000000..13352efdb --- /dev/null +++ b/python/sglang/srt/layers/quantization/awq_triton.py @@ -0,0 +1,339 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/awq_triton.py + +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch +import triton +import triton.language as tl + +AWQ_TRITON_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + + +@triton.jit +def awq_dequantize_kernel( + qweight_ptr, # quantized matrix + scales_ptr, # scales, per group + zeros_ptr, # zeros, per group + group_size, # Should always be one of the supported group sizes + result_ptr, # Output matrix + num_cols, # input num cols in qweight + num_rows, # input num rows in qweight + BLOCK_SIZE_X: tl.constexpr, + BLOCK_SIZE_Y: tl.constexpr, +): + # Setup the pids. + pid_x = tl.program_id(axis=0) + pid_y = tl.program_id(axis=1) + + # Compute offsets and masks for qweight_ptr. + offsets_y = pid_y * BLOCK_SIZE_Y + tl.arange(0, BLOCK_SIZE_Y) + offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X) + offsets = num_cols * offsets_y[:, None] + offsets_x[None, :] + + masks_y = offsets_y < num_rows + masks_x = offsets_x < num_cols + + masks = masks_y[:, None] & masks_x[None, :] + + # Compute offsets and masks for result output ptr. + result_offsets_y = pid_y * BLOCK_SIZE_Y + tl.arange(0, BLOCK_SIZE_Y) + result_offsets_x = pid_x * BLOCK_SIZE_X * 8 + tl.arange(0, BLOCK_SIZE_X * 8) + result_offsets = ( + 8 * num_cols * result_offsets_y[:, None] + result_offsets_x[None, :] + ) + + result_masks_y = result_offsets_y < num_rows + result_masks_x = result_offsets_x < num_cols * 8 + result_masks = result_masks_y[:, None] & result_masks_x[None, :] + + # Load the weights. + iweights = tl.load(qweight_ptr + offsets, masks, 0.0) + iweights = tl.interleave(iweights, iweights) + iweights = tl.interleave(iweights, iweights) + iweights = tl.interleave(iweights, iweights) + + # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7] + # that will map given indices to the correct order. + reverse_awq_order_tensor = ( + (tl.arange(0, 2) * 4)[None, :] + tl.arange(0, 4)[:, None] + ).reshape(8) + + # Use this to compute a set of shifts that can be used to unpack and + # reorder the values in iweights and zeros. + shifts = reverse_awq_order_tensor * 4 + shifts = tl.broadcast_to(shifts[None, :], (BLOCK_SIZE_Y * BLOCK_SIZE_X, 8)) + shifts = tl.reshape(shifts, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Unpack and reorder: shift out the correct 4-bit value and mask. + iweights = (iweights >> shifts) & 0xF + + # Compute zero offsets and masks. + zero_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1) + zero_offsets_x = pid_x * BLOCK_SIZE_X + tl.arange(0, BLOCK_SIZE_X) + zero_offsets = num_cols * zero_offsets_y[:, None] + zero_offsets_x[None, :] + + zero_masks_y = zero_offsets_y < num_rows // group_size + zero_masks_x = zero_offsets_x < num_cols + zero_masks = zero_masks_y[:, None] & zero_masks_x[None, :] + + # Load the zeros. + zeros = tl.load(zeros_ptr + zero_offsets, zero_masks, 0.0) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Unpack and reorder: shift out the correct 4-bit value and mask. + zeros = (zeros >> shifts) & 0xF + + # Compute scale offsets and masks. + scale_offsets_y = pid_y * BLOCK_SIZE_Y // group_size + tl.arange(0, 1) + scale_offsets_x = pid_x * BLOCK_SIZE_X * 8 + tl.arange(0, BLOCK_SIZE_X * 8) + scale_offsets = num_cols * 8 * scale_offsets_y[:, None] + scale_offsets_x[None, :] + scale_masks_y = scale_offsets_y < num_rows // group_size + scale_masks_x = scale_offsets_x < num_cols * 8 + scale_masks = scale_masks_y[:, None] & scale_masks_x[None, :] + + # Load the scales. + scales = tl.load(scales_ptr + scale_offsets, scale_masks, 0.0) + scales = tl.broadcast_to(scales, (BLOCK_SIZE_Y, BLOCK_SIZE_X * 8)) + + # Dequantize. + iweights = (iweights - zeros) * scales + iweights = iweights.to(result_ptr.type.element_ty) + + # Finally, store. + tl.store(result_ptr + result_offsets, iweights, result_masks) + + +@triton.jit +def awq_gemm_kernel( + a_ptr, + b_ptr, + c_ptr, + zeros_ptr, + scales_ptr, + M, + N, + K, + group_size, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + SPLIT_K: tl.constexpr, +): + pid = tl.program_id(axis=0) + pid_z = tl.program_id(1) + + # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. + # num_pid_n = (N + BLOCK_SIZE_N - 1) // BLOCK_SIZE_N + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + + pid_m = pid // num_pid_n + pid_n = pid % num_pid_n + + accumulator_dtype = c_ptr.type.element_ty + + # NOTE: This doesn't work in TRITON_INTERPRET=1 mode. Use below instead. + # accumulator = tl.arange(0, BLOCK_SIZE_N) + # accumulator = tl.broadcast_to(accumulator[None, :], + # (BLOCK_SIZE_M, BLOCK_SIZE_N)) + # accumulator = accumulator & 0x0 + # accumulator = accumulator.to(accumulator_dtype) + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=accumulator_dtype) + + # Create reverse AWQ order as tensor: [0, 4, 1, 5, 2, 6, 3, 7] + # that will map given indices to the correct order. + reverse_awq_order_tensor = ( + (tl.arange(0, 2) * 4)[None, :] + tl.arange(0, 4)[:, None] + ).reshape(8) + + # Create the necessary shifts to use to unpack. + shifts = reverse_awq_order_tensor * 4 + shifts = tl.broadcast_to(shifts[None, :], (BLOCK_SIZE_K * (BLOCK_SIZE_N // 8), 8)) + shifts = tl.reshape(shifts, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + # Offsets and masks. + offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + masks_am = offsets_am < M + + offsets_bn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) + masks_bn = offsets_bn < N // 8 + + offsets_zn = pid_n * (BLOCK_SIZE_N // 8) + tl.arange(0, BLOCK_SIZE_N // 8) + masks_zn = offsets_zn < N // 8 + + offsets_sn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + masks_sn = offsets_sn < N + + offsets_k = pid_z * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + offsets_a = K * offsets_am[:, None] + offsets_k[None, :] + offsets_b = (N // 8) * offsets_k[:, None] + offsets_bn[None, :] + + a_ptrs = a_ptr + offsets_a + b_ptrs = b_ptr + offsets_b + + # NOTE: Use this in TRITON_INTERPRET=1 mode instead of tl.cdiv + # block_offset = BLOCK_SIZE_K * SPLIT_K + # for k in range(0, (K + block_offset - 1) // (block_offset)): + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)): + masks_k = offsets_k < K + masks_a = masks_am[:, None] & masks_k[None, :] + a = tl.load(a_ptrs, mask=masks_a, other=0.0) + + masks_b = masks_k[:, None] & masks_bn[None, :] + b = tl.load(b_ptrs, mask=masks_b, other=0.0) + b = tl.interleave(b, b) + b = tl.interleave(b, b) + b = tl.interleave(b, b) + + # Dequantize b. + offsets_szk = ( + BLOCK_SIZE_K * SPLIT_K * k + pid_z * BLOCK_SIZE_K + ) // group_size + tl.arange(0, 1) + offsets_z = (N // 8) * offsets_szk[:, None] + offsets_zn[None, :] + masks_zk = offsets_szk < K // group_size + masks_z = masks_zk[:, None] & masks_zn[None, :] + zeros_ptrs = zeros_ptr + offsets_z + zeros = tl.load(zeros_ptrs, mask=masks_z, other=0.0) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.interleave(zeros, zeros) + zeros = tl.broadcast_to(zeros, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + offsets_s = N * offsets_szk[:, None] + offsets_sn[None, :] + masks_sk = offsets_szk < K // group_size + masks_s = masks_sk[:, None] & masks_sn[None, :] + scales_ptrs = scales_ptr + offsets_s + scales = tl.load(scales_ptrs, mask=masks_s, other=0.0) + scales = tl.broadcast_to(scales, (BLOCK_SIZE_K, BLOCK_SIZE_N)) + + b = (b >> shifts) & 0xF + zeros = (zeros >> shifts) & 0xF + b = (b - zeros) * scales + b = b.to(c_ptr.type.element_ty) + + # Accumulate results. + accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) + + offsets_k += BLOCK_SIZE_K * SPLIT_K + a_ptrs += BLOCK_SIZE_K * SPLIT_K + b_ptrs += BLOCK_SIZE_K * SPLIT_K * (N // 8) + + c = accumulator.to(c_ptr.type.element_ty) + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = c_ptr + pid_z * N * M + N * offs_cm[:, None] + offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +# qweights - [K , M // 8], int32 +# scales - [K // G, M ], float16 +# zeros - [K // G, M // 8], int32 +def awq_dequantize_triton( + qweight: torch.Tensor, + scales: torch.Tensor, + zeros: torch.Tensor, + block_size_x: int = 32, + block_size_y: int = 32, +) -> torch.Tensor: + K = qweight.shape[0] + M = scales.shape[1] + group_size = qweight.shape[0] // scales.shape[0] + + assert K > 0 and M > 0 + assert scales.shape[0] == K // group_size and scales.shape[1] == M + assert zeros.shape[0] == K // group_size and zeros.shape[1] == M // 8 + assert group_size <= K + assert group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES or group_size == K + + # Result tensor: + # number of rows = same as input tensor + # number of cols = 8 x input tensor num cols + result = torch.empty( + qweight.shape[0], + qweight.shape[1] * 8, + device=qweight.device, + dtype=scales.dtype, + ) + + Y = qweight.shape[0] # num rows + X = qweight.shape[1] # num cols + + grid = lambda META: ( + triton.cdiv(X, META["BLOCK_SIZE_X"]), + triton.cdiv(Y, META["BLOCK_SIZE_Y"]), + ) + awq_dequantize_kernel[grid]( + qweight, + scales, + zeros, + group_size, + result, + X, + Y, + BLOCK_SIZE_X=block_size_x, + BLOCK_SIZE_Y=block_size_y, + ) + + return result + + +# input - [M, K] +# qweight - [K, N // 8] +# qzeros - [K // G, N // 8] +# scales - [K // G, N] +# split_k_iters - parallelism along K-dimension, int, power of 2. +def awq_gemm_triton( + input: torch.Tensor, + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + split_k_iters: int, + block_size_m: int = 32, + block_size_n: int = 32, + block_size_k: int = 32, +) -> torch.Tensor: + M, K = input.shape + N = qweight.shape[1] * 8 + group_size = qweight.shape[0] // qzeros.shape[0] + + assert N > 0 and K > 0 and M > 0 + assert qweight.shape[0] == K and qweight.shape[1] == N // 8 + assert qzeros.shape[0] == K // group_size and qzeros.shape[1] == N // 8 + assert scales.shape[0] == K // group_size and scales.shape[1] == N + assert split_k_iters & (split_k_iters - 1) == 0 and split_k_iters != 0 + assert split_k_iters <= 32 + assert group_size <= K + assert group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES or group_size == K + + grid = lambda META: ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + split_k_iters, + ) + + result = torch.zeros((split_k_iters, M, N), dtype=scales.dtype, device=input.device) + + # A = input, B = qweight, C = result + # A = M x K, B = K x N, C = M x N + awq_gemm_kernel[grid]( + input, + qweight, + result, + qzeros, + scales, + M, + N, + K, + group_size, + BLOCK_SIZE_M=block_size_m, + BLOCK_SIZE_N=block_size_n, + BLOCK_SIZE_K=block_size_k, + SPLIT_K=split_k_iters, + ) + + result = result.sum(0) + + return result diff --git a/python/sglang/srt/layers/quantization/base_config.py b/python/sglang/srt/layers/quantization/base_config.py new file mode 100644 index 000000000..4a5b7905e --- /dev/null +++ b/python/sglang/srt/layers/quantization/base_config.py @@ -0,0 +1,216 @@ +# Adapted from https://raw.githubusercontent.com/vllm-project/vllm/v0.5.5/vllm/model_executor/layers/quantization/base_config.py +from __future__ import annotations + +import inspect +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type + +import torch +from torch import nn + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput + + +class QuantizeMethodBase(ABC): + """Base class for different quantized methods.""" + + @abstractmethod + def create_weights( + self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs + ): + """Create weights for a layer. + + The weights will be set as attributes of the layer.""" + raise NotImplementedError() + + @abstractmethod + def apply(self, layer: torch.nn.Module, *args, **kwargs) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError() + + def process_weights_after_loading(self, layer: nn.Module) -> None: + """Process the weight after loading. + + This can be used for example, to transpose weights for computation. + """ + return + + +class LinearMethodBase(QuantizeMethodBase): + """Base class for different (maybe quantized) linear methods.""" + + @abstractmethod + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + """Create weights for a linear layer. + The weights will be set as attributes of the layer. + + Args: + layer: The layer that is using the LinearMethodBase factory. + input_size_per_partition: Size of the weight input dim on rank X. + output_partition_sizes: Sizes of the output dim of each logical + weight on rank X. E.g., output_partition_sizes for QKVLinear + is a list contains the width of Wq, Wk, Wv on rank X. + input_size: Size of the input dim of the weight across all ranks. + output_size: Size of the output dim of the weight across all ranks. + params_dtype: Datatype of the parameters. + """ + raise NotImplementedError() + + @abstractmethod + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Apply the weights in layer to the input tensor. + Expects create_weights to have been called before on the layer.""" + raise NotImplementedError() + + +class FusedMoEMethodBase(QuantizeMethodBase): + + @abstractmethod + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + raise NotImplementedError + + @abstractmethod + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + raise NotImplementedError + + @abstractmethod + def apply( + self, + layer: torch.nn.Module, + dispatch_output: DispatchOutput, + ) -> CombineInput: + raise NotImplementedError + + +class QuantizationConfig(ABC): + """Base class for quantization configs.""" + + def __init__(self): + super().__init__() + # mapping is updated by models as they initialize + self.packed_modules_mapping: Dict[str, List[str]] = dict() + + @abstractmethod + def get_name(self) -> str: + """Name of the quantization method.""" + raise NotImplementedError() + + @abstractmethod + def get_supported_act_dtypes(self) -> List[torch.dtype]: + """List of supported activation dtypes.""" + raise NotImplementedError() + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + """Minimum GPU capability to support the quantization method. + + E.g., 70 for Volta, 75 for Turing, 80 for Ampere. + This requirement is due to the custom CUDA kernels used by the + quantization method. + """ + raise NotImplementedError() + + @staticmethod + @abstractmethod + def get_config_filenames() -> List[str]: + """List of filenames to search for in the model directory.""" + raise NotImplementedError() + + @classmethod + @abstractmethod + def from_config(cls, config: Dict[str, Any]) -> "QuantizationConfig": + """Create a config class from the model's quantization config.""" + raise NotImplementedError() + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + """ + Detects if this quantization method can support a given checkpoint + format by overriding the user specified quantization method -- + this method should only be overwritten by subclasses in exceptional + circumstances + """ + return None + + @staticmethod + def get_from_keys(config: Dict[str, Any], keys: List[str]) -> Any: + """Get a value from the model's quantization config.""" + for key in keys: + if key in config: + return config[key] + raise ValueError( + f"Cannot find any of {keys} in the model's " "quantization config." + ) + + @staticmethod + def get_from_keys_or(config: Dict[str, Any], keys: List[str], default: Any) -> Any: + """Get a optional value from the model's quantization config.""" + try: + return QuantizationConfig.get_from_keys(config, keys) + except ValueError: + return default + + @abstractmethod + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + """Get the quantize method to use for the quantized layer. + + Args: + layer: The layer for the quant method. + prefix: The full name of the layer in the state dict + Returns: + The quantize method. None if the given layer doesn't support quant + method. + """ + raise NotImplementedError() + + @abstractmethod + def get_scaled_act_names(self) -> List[str]: + """Returns the activation function names that should be post-scaled. + + For now, this is only used by AWQ. + """ + raise NotImplementedError() + + +def method_has_implemented_embedding(method_class: Type[QuantizeMethodBase]) -> bool: + """ + Not all quant methods have embedding implemented, so we need to check that + it exists for our given method. We check this by making sure the function + has been changed from the base implementation. + """ + base_embedding = inspect.getattr_static(QuantizeMethodBase, "embedding", None) + class_embedding = inspect.getattr_static(method_class, "embedding", None) + + return class_embedding is not None and class_embedding is not base_embedding diff --git a/python/sglang/srt/layers/quantization/blockwise_int8.py b/python/sglang/srt/layers/quantization/blockwise_int8.py new file mode 100644 index 000000000..60d4e3929 --- /dev/null +++ b/python/sglang/srt/layers/quantization/blockwise_int8.py @@ -0,0 +1,380 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/layers/quantization/fp8.py + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import torch +from torch.nn import Module + +from sglang.srt.distributed import get_tensor_model_parallel_world_size +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.parameter import BlockQuantScaleParameter, ModelWeightParameter +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.int8_utils import apply_w8a8_block_int8_linear +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import is_layer_skipped +from sglang.srt.utils import set_weight_attrs + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +ACTIVATION_SCHEMES = ["static", "dynamic"] + +logger = logging.getLogger(__name__) + + +class BlockInt8Config(QuantizationConfig): + """Config class for INT8.""" + + def __init__( + self, + is_checkpoint_int8_serialized: bool = False, + activation_scheme: str = "dynamic", + ignored_layers: Optional[List[str]] = None, + weight_block_size: List[int] = None, + ) -> None: + self.is_checkpoint_int8_serialized = is_checkpoint_int8_serialized + if is_checkpoint_int8_serialized: + logger.warning( + "Detected int8 checkpoint. Please note that the " + "format is experimental and subject to change." + ) + if activation_scheme not in ACTIVATION_SCHEMES: + raise ValueError(f"Unsupported activation scheme {activation_scheme}") + self.activation_scheme = activation_scheme + self.ignored_layers = ignored_layers or [] + if weight_block_size is not None: + if not is_checkpoint_int8_serialized: + raise ValueError( + f"The block-wise quantization only supports int8-serialized checkpoint for now." + ) + if len(weight_block_size) != 2: + raise ValueError( + f"The quantization block size of weight must have 2 dimensions, but got {len(weight_block_size)} dimensions." + ) + if activation_scheme != "dynamic": + raise ValueError( + f"The block-wise quantization only supports dynamic activation scheme for now, but got {activation_scheme} activation scheme." + ) + self.weight_block_size = weight_block_size + + @classmethod + def get_name(cls) -> str: + return "blockwise_int8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> BlockInt8Config: + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_int8_serialized = "int8" in quant_method + activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) + ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) + weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None) + return cls( + is_checkpoint_int8_serialized=is_checkpoint_int8_serialized, + activation_scheme=activation_scheme, + ignored_layers=ignored_layers, + weight_block_size=weight_block_size, + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.ignored_layers): + return UnquantizedLinearMethod() + return BlockInt8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return BlockInt8MoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class BlockInt8LinearMethod(LinearMethodBase): + """Linear method for INT8. + Supports loading INT8 checkpoints with static weight scale and + dynamic activation scale. + + Limitations: + Only support block-wise int8 quantization and int8 checkpoint + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: BlockInt8Config): + self.quant_config = quant_config + assert self.quant_config.weight_block_size is not None + assert self.quant_config.is_checkpoint_int8_serialized + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + tp_size = get_tensor_model_parallel_world_size() + + block_n, block_k = ( + self.quant_config.weight_block_size[0], + self.quant_config.weight_block_size[1], + ) + # Required by row parallel + if tp_size > 1 and input_size // input_size_per_partition == tp_size: + if input_size_per_partition % block_k != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + # Required by column parallel or enabling merged weights + if (tp_size > 1 and output_size // output_size_per_partition == tp_size) or len( + output_partition_sizes + ) > 1: + for output_partition_size in output_partition_sizes: + if output_partition_size % block_n != 0: + raise ValueError( + f"Weight output_partition_size = " + f"{output_partition_size} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight_dtype = ( + torch.int8 + if self.quant_config.is_checkpoint_int8_serialized + else params_dtype + ) + + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, input_size_per_partition, dtype=weight_dtype + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + + scale = BlockQuantScaleParameter( + data=torch.empty( + (output_size_per_partition + block_n - 1) // block_n, + (input_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale_inv", scale) + + # INPUT ACTIVATION SCALE + assert self.quant_config.activation_scheme == "dynamic" + layer.register_parameter("input_scale", None) + + def process_weights_after_loading(self, layer: Module) -> None: + # Block quant doesn't need to process weights after loading + # Use torch Parameter to avoid cuda graph capturing issue + layer.weight = torch.nn.Parameter(layer.weight.data, requires_grad=False) + layer.weight_scale_inv = torch.nn.Parameter( + layer.weight_scale_inv.data, requires_grad=False + ) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_w8a8_block_int8_linear( + input=x, + weight=layer.weight, + block_size=self.quant_config.weight_block_size, + weight_scale=layer.weight_scale_inv, + input_scale=None, + bias=bias, + ) + + +class BlockInt8MoEMethod(FusedMoEMethodBase): + """MoE method for INT8. + Supports loading INT8 checkpoints with static weight scale and + dynamic activation scale. + + Limitations: + Only support block-wise int8 quantization and int8 checkpoint + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: BlockInt8Config): + self.quant_config = quant_config + assert self.quant_config.weight_block_size is not None + assert self.quant_config.is_checkpoint_int8_serialized + + def create_weights( + self, + layer: Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + if self.quant_config.is_checkpoint_int8_serialized: + params_dtype = torch.int8 + tp_size = get_tensor_model_parallel_world_size() + + block_n, block_k = ( + self.quant_config.weight_block_size[0], + self.quant_config.weight_block_size[1], + ) + # NOTE(HandH1998): To ensure proper alignment of the block-wise quantization scales, the output_size of the weights for both the gate and up layers must be divisible by block_n. + # Required by column parallel or enabling merged weights + if intermediate_size_per_partition % block_n != 0: + raise ValueError( + f"The output_size of gate's and up's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + if tp_size > 1: + # Required by row parallel + if intermediate_size_per_partition % block_k != 0: + raise ValueError( + f"The input_size of down's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + + # WEIGHTS + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * ((intermediate_size_per_partition + block_n - 1) // block_n), + (hidden_size + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + (hidden_size + block_n - 1) // block_n, + (intermediate_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale_inv", w13_weight_scale) + layer.register_parameter("w2_weight_scale_inv", w2_weight_scale) + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value} + ) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + assert self.quant_config.activation_scheme == "dynamic" + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: Module) -> None: + # Block quant doesn't need to process weights after loading + return + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_int8_w8a8=True, + w13_scale=layer.w13_weight_scale_inv, + w2_scale=layer.w2_weight_scale_inv, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + block_shape=self.quant_config.weight_block_size, + ) + + return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/README.md b/python/sglang/srt/layers/quantization/compressed_tensors/README.md new file mode 100644 index 000000000..e8caf538b --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/README.md @@ -0,0 +1,6 @@ +# quantization compressed_tensors module + +To support compressed_tensors format quantization models, we adapted https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors into SGLang. + + +For practical purposes, we have only applied the compressed_tensors format of `w8a8_fp8`. If you have requirements for other formats, you can submit an issue through this [link](https://github.com/sgl-project/sglang/issues). diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/__init__.py b/python/sglang/srt/layers/quantization/compressed_tensors/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py new file mode 100644 index 000000000..8afc15a73 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -0,0 +1,666 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/v0.8.2/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import logging +from contextlib import suppress +from typing import Any, Dict, List, Literal, NamedTuple, Optional, Tuple, cast + +import torch +from compressed_tensors.config import ( + CompressionFormat, + SparsityCompressionConfig, + SparsityStructure, +) +from compressed_tensors.quantization import ( + QuantizationArgs, + QuantizationStrategy, + QuantizationType, +) +from pydantic import BaseModel + +from sglang.srt.layers.quantization.base_config import ( + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa: E501 + CompressedTensorsMoEMethod, +) +from sglang.srt.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, + CompressedTensorsW8A8Fp8, + CompressedTensorsW8A16Fp8, +) +from sglang.srt.layers.quantization.compressed_tensors.utils import ( + find_matched_target, + is_activation_quantization_format, + should_ignore_layer, +) +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod + +try: + from vllm.model_executor.layers.quantization.compressed_tensors.schemes.compressed_tensors_wNa16 import ( + WNA16_SUPPORTED_BITS, + CompressedTensorsWNA16, + ) + + VLLM_AVAILABLE = True +except ImportError: + VLLM_AVAILABLE = False + +logger = logging.getLogger(__name__) + +__all__ = ["CompressedTensorsLinearMethod"] + +SPARSITY_CONFIG_NAME: Literal["sparsity_config"] = "sparsity_config" +QUANTIZATION_SCHEME_MAP_TYPE = Dict[str, Optional[Dict[str, QuantizationArgs]]] + + +class DeviceCapability(NamedTuple): + major: int + minor: int + + def as_version_str(self) -> str: + return f"{self.major}.{self.minor}" + + def to_int(self) -> int: + """ + Express device capability as an integer ````. + + It is assumed that the minor version is always a single digit. + """ + assert 0 <= self.minor < 10 + return self.major * 10 + self.minor + + +class CompressedTensorsConfig(QuantizationConfig): + + def __init__( + self, + target_scheme_map: Dict[str, Any], + ignore: List[str], + quant_format: str, + sparsity_scheme_map: Dict[str, SparsityCompressionConfig], + sparsity_ignore_list: List[str], + kv_cache_scheme: Optional[Dict[str, Any]] = None, + config: Optional[Dict[str, Any]] = None, + packed_modules_mapping: Dict[str, List[str]] = {}, + ): + super().__init__() + self.ignore = ignore + self.quant_format = quant_format + # Map from [target -> scheme] + self.target_scheme_map = target_scheme_map + self.kv_cache_scheme = kv_cache_scheme + self.sparsity_scheme_map = sparsity_scheme_map + self.sparsity_ignore_list = sparsity_ignore_list + self.config = config + self.packed_modules_mapping = packed_modules_mapping + + def get_linear_method(self) -> CompressedTensorsLinearMethod: + return CompressedTensorsLinearMethod(self) + + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def get_name(self) -> str: + return "compressed_tensors" + + def get_scaled_act_names(self) -> List[str]: + return [] + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + + # Check if the layer is skipped for quantization. + # TODO (@robertgshaw2): support module names + if should_ignore_layer( + prefix, ignore=self.ignore, fused_mapping=self.packed_modules_mapping + ): + return UnquantizedLinearMethod() + if isinstance(layer, LinearBase): + scheme = self.get_scheme(layer=layer, layer_name=prefix) + if scheme is None: + return UnquantizedLinearMethod() + layer.scheme = scheme + return CompressedTensorsLinearMethod(self) + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, FusedMoE): + return CompressedTensorsMoEMethod.get_moe_method(self) + return None + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> CompressedTensorsConfig: + ignore: List[str] = cast(List[str], config.get("ignore", [])) + quant_format = cast(str, config.get("format")) + target_scheme_map = cls._quantization_scheme_map_from_config(config=config) + sparsity_scheme_map, sparsity_ignore_list = cls._parse_sparsity_config( + config=config + ) + packed_modules_mapping = config.get("packed_modules_mapping", {}) + + return cls( + target_scheme_map=target_scheme_map, + ignore=ignore, + quant_format=quant_format, + sparsity_scheme_map=sparsity_scheme_map, + sparsity_ignore_list=sparsity_ignore_list, + config=config, + packed_modules_mapping=packed_modules_mapping, + ) + + @classmethod + def _parse_sparsity_config( + cls, config: Dict[str, Any] + ) -> Tuple[Dict[str, SparsityCompressionConfig], List[str]]: + """ + :param config: The `quantization_config` dictionary from config.json + :return: A tuple with two elements + 1. A dictionary mapping target layer names to their corresponding + sparsity_config + 2. A list of layer names to ignore for sparsity + """ + if not (sparsity_config := config.get(SPARSITY_CONFIG_NAME)): + return dict(), [] + + sparsity_config = SparsityCompressionConfig.model_validate(sparsity_config) + sparse_scheme_map: Dict[str, SparsityCompressionConfig] = { + target: sparsity_config for target in sparsity_config.targets or list() + } + sparsity_ignore_list = sparsity_config.ignore or list() + return sparse_scheme_map, sparsity_ignore_list + + @classmethod + def _quantization_scheme_map_from_config( + cls, config: Dict[str, Any] + ) -> QUANTIZATION_SCHEME_MAP_TYPE: + """ + :param config: The `quantization_config` dictionary from config.json + :return: A dictionary mapping target layer names to their corresponding + quantization_args for weights and input activations + """ + target_scheme_map: Dict[str, Any] = dict() + quant_format = cast(str, config.get("format")) + + # The quant_config has multiple config_groups, each containing + # an input_activations key with details about how the activations are + # quantized, a weights key indicating how the weights are quantized, + # and a list of targets under the `targets` key, dictating which + # layers are impacted by the quantization details. The quantization + # details follow the structure defined by the QuantizationArgs + # pydantic model, which is used to verify the structure of the + # quant_config and also store the details for later use. + + config_groups = config.get("config_groups", dict()) + for _, quant_config in config_groups.items(): + targets = quant_config.get("targets") + for target in targets: + target_scheme_map[target] = {} + target_scheme_map[target]["weights"] = QuantizationArgs.model_validate( + quant_config.get("weights") + ) + + target_scheme_map[target]["input_activations"] = None + if is_activation_quantization_format(quant_format): + input_activations = quant_config.get("input_activations") + # The only case where we have activation quant supported + # but no input_activations provided in the config + # should be w8a16fp8 w8a16fp8 can also run for cases where + # there is an input_quant but it is ignored + if not input_activations: + assert ( + target_scheme_map[target]["weights"].type + == QuantizationType.FLOAT + ) + else: + target_scheme_map[target]["input_activations"] = ( + QuantizationArgs.model_validate( # noqa: E501 + quant_config.get("input_activations") + ) + ) + return target_scheme_map + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + def _check_scheme_supported(self, min_capability: int, error: bool = True) -> bool: + capability_tuple = DeviceCapability(*torch.cuda.get_device_capability()) + + if capability_tuple is not None: + capability = capability_tuple.to_int() + supported = capability >= min_capability + if error and not supported: + raise RuntimeError( + "Quantization scheme is not supported for ", + f"the current GPU. Min capability: {min_capability}. ", + f"Current capability: {capability}.", + ) + return supported + else: + return False + + def _is_static_tensor_w8a8( + self, weight_quant: BaseModel, input_quant: BaseModel + ) -> bool: + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8 + weight_strategy = ( + weight_quant.strategy == QuantizationStrategy.TENSOR.value + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value + ) + is_tensor = ( + weight_strategy + and input_quant.strategy == QuantizationStrategy.TENSOR.value + ) + is_static = not weight_quant.dynamic and not input_quant.dynamic + + # Both symmetric and asymmetric input quantization supported. + # Only symmetric weight quantization supported. + return is_8_bits and is_tensor and weight_quant.symmetric and is_static + + def _is_dynamic_token_w8a8( + self, weight_quant: BaseModel, input_quant: BaseModel + ) -> bool: + is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8 + weight_strategy = ( + weight_quant.strategy == QuantizationStrategy.TENSOR.value + or weight_quant.strategy == QuantizationStrategy.CHANNEL.value + ) + is_token = ( + weight_strategy and input_quant.strategy == QuantizationStrategy.TOKEN.value + ) + is_dynamic = not weight_quant.dynamic and input_quant.dynamic + + # Both symmetric and asymmetric input quantization supported. + # Only symmetric weight quantization supported. + return is_8_bits and is_token and weight_quant.symmetric and is_dynamic + + def _is_fp8_w8a8(self, weight_quant: BaseModel, input_quant: BaseModel) -> bool: + # Confirm weights and activations quantized. + if weight_quant is None or input_quant is None: + return False + + # Confirm weight scheme is supported. + is_floating_point = ( + weight_quant.type == QuantizationType.FLOAT + and input_quant.type == QuantizationType.FLOAT + ) + is_symmetric_weight = weight_quant.symmetric + is_static_weight = not weight_quant.dynamic + is_per_tensor_or_channel_weight = weight_quant.strategy in [ + QuantizationStrategy.TENSOR, + QuantizationStrategy.CHANNEL, + ] + if not ( + is_floating_point + and is_symmetric_weight + and is_static_weight + and is_per_tensor_or_channel_weight + ): + return False + + # Dynamic quantization is always supported if weights supported. + if input_quant.dynamic: + return True + + # Confirm activation scheme is supported. + is_symmetric_activation = input_quant.symmetric + is_per_tensor_activation = input_quant.strategy == QuantizationStrategy.TENSOR + return is_symmetric_activation and is_per_tensor_activation + + def _is_fp8_w8a16(self, weight_quant: BaseModel, input_quant: BaseModel) -> bool: + # Confirm weights quantized. + if weight_quant is None: + return False + + # Confirm we have floating points. + if weight_quant.type != QuantizationType.FLOAT: + return False + + # Confirm weight scheme is supported. + is_symmetric_weight = weight_quant.symmetric + is_static_weight = not weight_quant.dynamic + is_per_tensor_or_channel_weight = weight_quant.strategy in [ + QuantizationStrategy.TENSOR, + QuantizationStrategy.CHANNEL, + ] + if not ( + is_symmetric_weight + and is_static_weight # noqa: SIM103 + and is_per_tensor_or_channel_weight + ): + return False + + # All conditions satisfied. + return True + + def _is_wNa16_group_channel( + self, weight_quant: BaseModel, input_quant: BaseModel + ) -> bool: + input_quant_none = input_quant is None + is_symmetric = weight_quant.symmetric + is_channel_group = ( + weight_quant.strategy == QuantizationStrategy.CHANNEL.value + or weight_quant.strategy == QuantizationStrategy.GROUP.value + ) + is_static = not weight_quant.dynamic + + return is_channel_group and input_quant_none and is_symmetric and is_static + + def _get_scheme_from_parts( + self, weight_quant: BaseModel, input_quant: BaseModel + ) -> CompressedTensorsScheme: + + # Detect If Mixed Precision + if self._is_wNa16_group_channel(weight_quant, input_quant): + if not VLLM_AVAILABLE: + raise ImportError( + "vllm is not installed, to use CompressedTensorsW4A16Sparse24 and CompressedTensorsWNA16, please install vllm" + ) + if ( + self.quant_format == CompressionFormat.marlin_24.value + and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS + ): + return CompressedTensorsW4A16Sparse24( + strategy=weight_quant.strategy, + num_bits=weight_quant.num_bits, + group_size=weight_quant.group_size, + ) + if ( + self.quant_format == CompressionFormat.pack_quantized.value + and weight_quant.num_bits in WNA16_SUPPORTED_BITS + ): + return CompressedTensorsWNA16( + num_bits=weight_quant.num_bits, + strategy=weight_quant.strategy, + group_size=weight_quant.group_size, + actorder=weight_quant.actorder, + ) + + if is_activation_quantization_format(self.quant_format): + if self._is_fp8_w8a8(weight_quant, input_quant): + is_fp8_w8a8_supported = self._check_scheme_supported( + CompressedTensorsW8A8Fp8.get_min_capability(), error=False + ) + if is_fp8_w8a8_supported: + return CompressedTensorsW8A8Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=( + input_quant and not input_quant.dynamic + ), + ) + else: + # note: input_quant will be present for converted models; + # will be ignored during inference post loading + return CompressedTensorsW8A16Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=not input_quant.dynamic, + ) + + # note: input_quant can be None + if self._is_fp8_w8a16(weight_quant, input_quant): + if not VLLM_AVAILABLE: + raise ImportError( + "vllm is not installed, to use CompressedTensorsW8A16Fp8, please install vllm" + ) + is_static_input_scheme = input_quant and not input_quant.dynamic + return CompressedTensorsW8A16Fp8( + strategy=weight_quant.strategy, + is_static_input_scheme=is_static_input_scheme, + ) + + if self._is_static_tensor_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Int8( + strategy=weight_quant.strategy, + is_static_input_scheme=True, + input_symmetric=input_quant.symmetric, + ) + + if self._is_dynamic_token_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Int8( + strategy=weight_quant.strategy, + is_static_input_scheme=False, + input_symmetric=input_quant.symmetric, + ) + + raise NotImplementedError("No compressed-tensors compatible scheme was found.") + + def get_scheme( + self, layer: torch.nn.Module, layer_name: Optional[str] = None + ) -> Optional[CompressedTensorsScheme]: + """ + compressed-tensors supports non uniform in the following way: + + targets of config_groups: There can be N config_groups which each + have a quantization scheme. Each config_group has a list of targets + which can be a full layer_name, a regex for a layer_name, or + an nn.Module name. + + Detect whether a layer_name is found in any target and + use the quantization scheme corresponding to the matched target + to select the CompressedTensorsScheme used for infernece. + """ + + # Find the "target" in the compressed-tensors config + # that our layer conforms to. + # TODO (@robertgshaw): add compressed-tensors as dep + # so we do not have to re-write these functions + # need to make accelerate optional in ct to do this + + # Will be empty for models with only sparsity + weight_quant = input_quant = None + if self.target_scheme_map: + matched_target = find_matched_target( + layer_name=layer_name, + module=layer, + targets=self.target_scheme_map.keys(), + fused_mapping=self.packed_modules_mapping, + ) + + scheme_dict = self.target_scheme_map[matched_target] + weight_quant = scheme_dict.get("weights") + input_quant = scheme_dict.get("input_activations") + + # Find the sparsity scheme of the layer + # assume that fused layers inerhit first component's sparsity scheme + sparsity_targets = self.sparsity_scheme_map.keys() - set( + self.sparsity_ignore_list + ) + sparsity_scheme: Optional[SparsityCompressionConfig] = None + with suppress(ValueError): + matched_target = find_matched_target( + layer_name=layer_name, + module=layer, + targets=sparsity_targets, + fused_mapping=self.packed_modules_mapping, + ) + sparsity_scheme = self.sparsity_scheme_map[matched_target] + + if self.supports_cutlass_24( + weight_quant=weight_quant, + input_quant=input_quant, + sparsity_scheme=sparsity_scheme, + ): + if not VLLM_AVAILABLE: + raise ImportError( + "vllm is not installed, to use CompressedTensors24, please install vllm" + ) + # Have a valid sparsity scheme + # Validate layer is supported by Cutlass 2:4 Kernel + model_compression_config = ( + None + if sparsity_scheme is None or sparsity_scheme.format == "dense" + else self.config + ) + + scheme = CompressedTensors24( + quantized=weight_quant is not None or input_quant is not None, + weight_quant=weight_quant, + input_quant=input_quant, + model_compression_config=model_compression_config, + ) + elif weight_quant is None: + logger.warning_once( + "Acceleration for non-quantized schemes is " + "not supported by Compressed Tensors. " + "Falling back to UnquantizedLinearMethod" + ) + return None + + else: + # Find the quant_scheme + scheme = self._get_scheme_from_parts( # type: ignore + weight_quant=weight_quant, + input_quant=input_quant, + ) + + # Raise error if device does not support the scheme + # (e.g. fp8 needs ada lovelace) + self._check_scheme_supported(scheme.get_min_capability()) + logger.debug("Using scheme: %s for %s", scheme.__class__.__name__, layer_name) + return scheme + + def get_cache_scale(self, name: str) -> Optional[str]: + """ + Check whether the param name matches the format for k/v cache scales + in compressed-tensors. If this is the case, return its equivalent + param name expected by vLLM + + :param name: param name + :return: matching param name for KV cache scale in vLLM + """ + if name.endswith(".output_scale") and ".k_proj" in name: + return name.replace(".k_proj.output_scale", ".attn.k_scale") + if name.endswith(".output_scale") and ".v_proj" in name: + return name.replace(".v_proj.output_scale", ".attn.v_scale") + # If no matches, return None + return None + + @staticmethod + def supports_cutlass_24( + weight_quant: Optional[QuantizationArgs], + input_quant: Optional[QuantizationArgs], + sparsity_scheme: Optional[SparsityCompressionConfig] = None, + ) -> bool: + """ + Check if the layer is supported by the Cutlass 2:4 Kernel + Conditions: + - Overarching condition: Sparsity Structure is 2:4 + - Unquantized cases are supported + - Weight only quantization is not-supported + - Supported weight quantization strategies are TENSOR and CHANNEL + - Supported input quantization strategies are TENSOR and TOKEN + - Only 8 bit quantization is supported + + :return: True if the layer is supported by the Cutlass 2:4 Kernel + False otherwise + """ + if sparsity_scheme is None: + return False + + is_valid_sparsity_structure: bool = ( + sparsity_scheme.sparsity_structure == SparsityStructure.TWO_FOUR.value + ) + + valid_compressors = { + CompressionFormat.dense.value, + CompressionFormat.sparse_24_bitmask.value, + } + + is_valid_sparsity = ( + is_valid_sparsity_structure and sparsity_scheme.format in valid_compressors + ) + + if not is_valid_sparsity: + return False + + # Unquantized cases are supported + if weight_quant is None and input_quant is None: + return True + + # Weight only quantization is not-supported + if weight_quant is not None and input_quant is None: + return False + + supported_weight_quant_strategies = [ + QuantizationStrategy.TENSOR.value, + QuantizationStrategy.CHANNEL.value, + ] + + assert weight_quant is not None + assert input_quant is not None + if weight_quant.strategy not in supported_weight_quant_strategies: + return False + + supported_input_quant_strategies = [ + QuantizationStrategy.TENSOR.value, + QuantizationStrategy.TOKEN.value, + ] + + if input_quant.strategy not in supported_input_quant_strategies: + return False + + return weight_quant.num_bits == input_quant.num_bits == 8 + + +class CompressedTensorsLinearMethod(LinearMethodBase): + + def __init__(self, quantization_config: CompressedTensorsConfig): + self.quantization_config = quantization_config + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.scheme.process_weights_after_loading(layer) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + """ + Use the CompressedTensorsScheme associated with each layer to create + the necessary parameters for the layer. See LinearMethodBase for param + details + """ + weight_loader = extra_weight_attrs.get("weight_loader") + layer.scheme.create_weights( + layer=layer, + input_size=input_size, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + output_size=output_size, + params_dtype=params_dtype, + weight_loader=weight_loader, + ) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ): + """ + Use the output of create_weights and the CompressedTensorsScheme + associated with the layer to apply the forward pass with the + layer input. See LinearMethodBase for param details + + """ + + scheme = layer.scheme + if scheme is None: + raise ValueError("A scheme must be defined for each layer") + return scheme.apply_weights(layer, x, bias=bias) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py new file mode 100644 index 000000000..e2ff25e68 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors_moe.py @@ -0,0 +1,692 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/v0.8.2/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import enum +import logging +from enum import Enum +from typing import TYPE_CHECKING, List, Optional + +import torch +from compressed_tensors import CompressionFormat +from compressed_tensors.quantization import QuantizationStrategy + +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.quantization.base_config import FusedMoEMethodBase +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz, scaled_fp8_quant +from sglang.srt.layers.quantization.fp8_utils import normalize_e4m3fn_to_e4m3fnuz +from sglang.srt.layers.quantization.utils import ( + all_close_1d, + per_tensor_dequantize, + replace_parameter, +) +from sglang.srt.utils import ( + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, + is_npu, + set_weight_attrs, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsConfig, + ) + +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _use_aiter: + from aiter.ops.shuffle import shuffle_weight + + from sglang.srt.layers.moe.rocm_moe_utils import rocm_fused_experts_tkw1 + +try: + import vllm + + VLLM_AVAILABLE = True +except ImportError: + VLLM_AVAILABLE = False + +logger = logging.getLogger(__name__) + + +class GPTQMarlinState(Enum): + REPACK = enum.auto() + READY = enum.auto() + + +__all__ = [ + "CompressedTensorsMoEMethod", + "CompressedTensorsW8A8Fp8MoEMethod", + "CompressedTensorsWNA16MoEMethod", +] + + +class CompressedTensorsMoEMethod(FusedMoEMethodBase): + def __new__(cls, *args, **kwargs): + if cls is CompressedTensorsMoEMethod: + return super().__new__(cls) + return super().__new__(cls) + + @staticmethod + def get_moe_method( + quant_config: CompressedTensorsConfig, + ) -> "CompressedTensorsMoEMethod": + # TODO: @dsikka: refactor this to use schemes as other kernels + # are supported + check if the layer is being ignored. + weight_quant = quant_config.target_scheme_map["Linear"].get("weights") + input_quant = quant_config.target_scheme_map["Linear"].get("input_activations") + + if quant_config._is_wNa16_group_channel(weight_quant, input_quant): + if not VLLM_AVAILABLE: + raise ImportError( + "vllm is not installed, to use CompressedTensorsWNA16MoEMethod, please install vllm." + ) + return CompressedTensorsWNA16MoEMethod(quant_config) + elif quant_config._is_fp8_w8a8(weight_quant, input_quant): + return CompressedTensorsW8A8Fp8MoEMethod(quant_config) + else: + raise RuntimeError( + f"Unsupported FusedMoe scheme: {weight_quant}, {input_quant}" + ) + + +class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): + + def __init__(self, quant_config: CompressedTensorsConfig): + self.quant_config = quant_config + self.weight_quant = self.quant_config.target_scheme_map["Linear"].get("weights") + self.input_quant = self.quant_config.target_scheme_map["Linear"].get( + "input_activations" + ) + + self.static_input_scales = not self.input_quant.dynamic + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + params_dtype = torch.float8_e4m3fn + + # WEIGHTS + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + # per-tensor quantization + if self.weight_quant.strategy == QuantizationStrategy.TENSOR: + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + weight_quant_method = FusedMoeWeightScaleSupported.TENSOR.value + elif self.weight_quant.strategy == QuantizationStrategy.CHANNEL: + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * intermediate_size_per_partition, + 1, + dtype=torch.float32, + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False, + ) + weight_quant_method = FusedMoeWeightScaleSupported.CHANNEL.value + else: + raise ValueError( + f"Unsupported weight quantization strategy: {self.weight_quant.strategy}" + ) + + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update({"quant_method": weight_quant_method}) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # INPUT_SCALES + if self.static_input_scales: + assert ( + self.input_quant.strategy == QuantizationStrategy.TENSOR + ), "Only per-tensor quantization is supported for static input scales" + w13_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: FusedMoE) -> None: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if self.static_input_scales: + if layer.w13_input_scale is None or layer.w2_input_scale is None: + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None." + ) + if not all_close_1d(layer.w13_input_scale) or not all_close_1d( + layer.w2_input_scale + ): + logger.warning( + "Found input_scales that are not equal for " + "fp8 MoE layer. Using the maximum across experts " + "for each layer." + ) + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False + ) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False + ) + + if is_fp8_fnuz(): + # Normalize the weights and scales + w13_weight, w13_weight_scale, w13_input_scale = ( + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale + ) + ) + w2_weight, w2_weight_scale, w2_input_scale = normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale + ) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter( + w13_weight_scale, requires_grad=False + ) + if w13_input_scale is not None: + layer.w13_input_scale = torch.nn.Parameter( + w13_input_scale, requires_grad=False + ) + layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter( + w2_weight_scale, requires_grad=False + ) + if w2_input_scale is not None: + layer.w2_input_scale = torch.nn.Parameter( + w2_input_scale, requires_grad=False + ) + if self.weight_quant.strategy == QuantizationStrategy.TENSOR: + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.num_local_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start : start + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id], + ) + ( + layer.w13_weight[expert_id][start : start + shard_size, :], + _, + ) = scaled_fp8_quant(dq_weight, max_w13_scales[expert_id]) + + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter( + max_w13_scales, requires_grad=False + ) + + if _use_aiter: + with torch.no_grad(): + # Pre-shuffle weights + layer.w13_weight = torch.nn.Parameter( + shuffle_weight(layer.w13_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + layer.w2_weight = torch.nn.Parameter( + shuffle_weight(layer.w2_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + moe_runner_config = self.moe_runner_config + + if ( + _use_aiter + and self.weight_quant.strategy == QuantizationStrategy.CHANNEL + and moe_runner_config.apply_router_weight_on_input + ): + topk_weights, topk_ids, _ = topk_output + output = rocm_fused_experts_tkw1( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=moe_runner_config.activation, + apply_router_weight_on_input=moe_runner_config.apply_router_weight_on_input, + use_fp8_w8a8=True, + per_channel_quant=self.weight_quant.strategy + == QuantizationStrategy.CHANNEL, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + ) + return StandardCombineInput(hidden_states=output) + else: + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_fp8_w8a8=True, + per_channel_quant=self.weight_quant.strategy + == QuantizationStrategy.CHANNEL, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + ) + return self.runner.run(dispatch_output, quant_info) + + +class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): + + def __init__(self, quant_config: CompressedTensorsConfig): + self.quant_config = quant_config + # TODO: @dsikka: refactor this to use schemes as other kernels + # are supported + check if the layer is being ignored. + config = self.quant_config.target_scheme_map["Linear"].get("weights") + self.num_bits = config.num_bits + self.packed_factor = 32 // config.num_bits + self.strategy = config.strategy + self.group_size = config.group_size + self.actorder = config.actorder + assert config.symmetric, "Only symmetric quantization is supported for MoE" + + if not ( + self.quant_config.quant_format == CompressionFormat.pack_quantized.value + and self.num_bits in WNA16_SUPPORTED_BITS + ): + raise ValueError( + "For Fused MoE layers, only ", + f"{CompressionFormat.pack_quantized.value} ", + "is supported for the following bits: ", + f"{WNA16_SUPPORTED_BITS}", + ) + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + + assert ( + params_dtype == torch.float16 + ), "float16 is required for MoE compressed models. Set dtype=torch.float16" # noqa: E501 + + # Will transpose the loaded weight along the + # intermediate and hidden dim sizes. Will + # shard for TP along the transposed dims + extra_weight_attrs.update( + {"is_transposed": True, "quant_method": self.strategy} + ) + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size // self.packed_factor, + 2 * intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_packed", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition // self.packed_factor, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_packed", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # In the case where we have actorder/g_idx, + # we do not partition the w2 scales + load_full_w2 = self.actorder and self.group_size != -1 + + if load_full_w2: + w2_scales_size = intermediate_size_per_partition * layer.moe_tp_size + else: + w2_scales_size = intermediate_size_per_partition + + self.is_k_full = (not self.actorder) or layer.moe_tp_size == 1 + + if self.strategy == "channel": + num_groups_w2 = num_groups_w13 = 1 + self.group_size = -1 + else: + num_groups_w2 = w2_scales_size // self.group_size + num_groups_w13 = hidden_size // self.group_size + + w13_scale = torch.nn.Parameter( + torch.ones( + num_experts, + num_groups_w13, + 2 * intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_scale) + set_weight_attrs(w13_scale, extra_weight_attrs) + + w2_scale = torch.nn.Parameter( + torch.ones(num_experts, num_groups_w2, hidden_size, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_scale) + set_weight_attrs(w2_scale, extra_weight_attrs) + set_weight_attrs(w2_scale, {"load_full_w2": load_full_w2}) + + w2_weight_shape = torch.nn.Parameter( + torch.empty(num_experts, 2), requires_grad=False + ) + layer.register_parameter("w2_weight_shape", w2_weight_shape) + set_weight_attrs(w2_weight_shape, extra_weight_attrs) + w13_weight_shape = torch.nn.Parameter( + torch.empty(num_experts, 2), requires_grad=False + ) + + layer.register_parameter("w13_weight_shape", w13_weight_shape) + set_weight_attrs(w13_weight_shape, extra_weight_attrs) + + w13_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_g_idx", w13_g_idx) + set_weight_attrs(w13_g_idx, extra_weight_attrs) + + w2_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_g_idx", w2_g_idx) + set_weight_attrs(w2_g_idx, extra_weight_attrs) + + w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices) + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs) + + w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices) + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs) + + layer.a13_scale = None + layer.a2_scale = None + layer.marlin_state = GPTQMarlinState.REPACK + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + + def replace_tensor(name, new_t): + # It is important to use resize_() here since it ensures + # the same buffer is reused + getattr(layer, name).resize_(new_t.shape) + getattr(layer, name).copy_(new_t) + del new_t + + def get_scale_perms(num_bits: int): + scale_perm: List[int] = [] + for i in range(8): + scale_perm.extend([i + 8 * j for j in range(8)]) + scale_perm_single: List[int] = [] + for i in range(4): + scale_perm_single.extend( + [2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]] + ) + return scale_perm, scale_perm_single + + def marlin_permute_scales( + s: torch.Tensor, size_k: int, size_n: int, group_size: int, num_bits: int + ): + scale_perm, scale_perm_single = get_scale_perms(num_bits) + if group_size < size_k and group_size != -1: + s = s.reshape((-1, len(scale_perm)))[:, scale_perm] + else: + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] + s = s.reshape((-1, size_n)).contiguous() + return s + + def marlin_moe_permute_scales( + s: torch.Tensor, size_k: int, size_n: int, group_size: int, num_bits: int + ): + num_experts = s.shape[0] + output = torch.empty( + (num_experts, s.shape[1], s.shape[2]), device=s.device, dtype=s.dtype + ) + for e in range(num_experts): + output[e] = marlin_permute_scales( + s[e], size_k, size_n, group_size, num_bits + ) + return output + + size_k2 = layer.w2_weight_packed.shape[2] + size_k13 = layer.w13_weight_packed.shape[2] + + num_experts = layer.w13_weight_g_idx.shape[0] + device = layer.w13_weight_g_idx.device + + # when running models with grouped act order, + # resort to g_idx values provided in checkpoint + if self.actorder == "group": + w13_g_idx_sort_indices = torch.empty_like(layer.w13_weight_g_idx) + w2_g_idx_sort_indices = torch.empty_like(layer.w2_weight_g_idx) + w13_sorted_g_idx = torch.empty_like(layer.w13_weight_g_idx) + w2_sorted_g_idx = torch.empty_like(layer.w2_weight_g_idx) + + for e in range(num_experts): + w13_g_idx_sort_indices[e] = torch.argsort(layer.w13_weight_g_idx[e]).to( + torch.int32 + ) + w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_weight_g_idx[e]).to( + torch.int32 + ) + w13_sorted_g_idx[e] = layer.w13_weight_g_idx[e][ + w13_g_idx_sort_indices[e] + ] + w2_sorted_g_idx[e] = layer.w2_weight_g_idx[e][w2_g_idx_sort_indices[e]] + + replace_parameter(layer, "w13_weight_g_idx", w13_sorted_g_idx) + replace_parameter(layer, "w2_weight_g_idx", w2_sorted_g_idx) + replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices) + replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) + + else: + layer.w13_weight_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_weight_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + + from vllm import _custom_ops as vllm_ops + + marlin_w13_qweight = vllm_ops.gptq_marlin_moe_repack( + layer.w13_weight_packed, + layer.w13_g_idx_sort_indices, + layer.w13_weight_packed.shape[1] * self.packed_factor, + layer.w13_weight_packed.shape[2], + self.num_bits, + ) + replace_tensor("w13_weight_packed", marlin_w13_qweight) + marlin_w2_qweight = vllm_ops.gptq_marlin_moe_repack( + layer.w2_weight_packed, + layer.w2_g_idx_sort_indices, + layer.w2_weight_packed.shape[1] * self.packed_factor, + layer.w2_weight_packed.shape[2], + self.num_bits, + ) + replace_tensor("w2_weight_packed", marlin_w2_qweight) + # Repack scales + marlin_w13_scales = marlin_moe_permute_scales( + layer.w13_weight_scale, + size_k13, + layer.w13_weight_scale.shape[2], + self.group_size, + self.num_bits, + ) + replace_tensor("w13_weight_scale", marlin_w13_scales) + marlin_w2_scales = marlin_moe_permute_scales( + layer.w2_weight_scale, + layer.w2_weight_scale.shape[1] + * (self.group_size if self.group_size != -1 else self.packed_factor), + size_k2, + self.group_size, + self.num_bits, + ) + replace_tensor("w2_weight_scale", marlin_w2_scales) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + assert ( + self.moe_runner_config.activation == "silu" + ), "Only SiLU activation is supported." + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + topk_weights, topk_ids, router_logits = topk_output + + output = torch.ops.vllm.fused_marlin_moe( + x, + layer.w13_weight_packed, + layer.w2_weight_packed, + layer.w13_weight_scale, + layer.w2_weight_scale, + router_logits, + topk_weights, + topk_ids, + g_idx1=layer.w13_weight_g_idx, + g_idx2=layer.w2_weight_g_idx, + sort_indices1=layer.w13_g_idx_sort_indices, + sort_indices2=layer.w2_g_idx_sort_indices, + num_bits=self.num_bits, + is_k_full=self.is_k_full, + ) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py new file mode 100644 index 000000000..c94575316 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/__init__.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: Apache-2.0 + +from .compressed_tensors_scheme import CompressedTensorsScheme +from .compressed_tensors_w8a8_fp8 import CompressedTensorsW8A8Fp8 +from .compressed_tensors_w8a16_fp8 import CompressedTensorsW8A16Fp8 + +__all__ = [ + "CompressedTensorsScheme", + "CompressedTensorsW8A8Fp8", + "CompressedTensorsW8A16Fp8", +] diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py new file mode 100644 index 000000000..3795d0a54 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py @@ -0,0 +1,56 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod +from typing import Optional + +import torch + +__all__ = ["CompressedTensorsScheme"] + + +class CompressedTensorsScheme(ABC): + """ + Abstract class used to describe the weight creation and forward pass + of different quantization schemes supported by CompressedTensors. + """ + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + """ + Get minimum device capability. + """ + raise NotImplementedError + + @abstractmethod + def create_weights(self, *args, **kwargs): + """ + Weight creation for the particular scheme. Inputs to this function + + """ + raise NotImplementedError + + @abstractmethod + def apply_weights( + self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] + ): + """ + Run the forward pass for the particular scheme. This is where + scheme-specific dequant/quant steps/kernels should be applied. + + :param layer: torch.nn.Module with the registered weights and + other parameters relevant to the particular scheme. + :param x: input to the layer + :param bias: bias parameter + + """ + raise NotImplementedError + + @abstractmethod + def process_weights_after_loading(self, layer: torch.nn.Module): + """ + Called after weight loading is complete for any cleanup that + needs to occur. + """ + raise NotImplementedError diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py new file mode 100644 index 000000000..af4f1a0e0 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a16_fp8.py @@ -0,0 +1,153 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 + +from typing import Callable, List, Optional + +import torch +from compressed_tensors.quantization import QuantizationStrategy + +from sglang.srt.layers.parameter import ( + ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter, +) +from sglang.srt.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) +from sglang.srt.layers.quantization.utils import convert_to_channelwise + +try: + from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, + prepare_fp8_layer_for_marlin, + ) + + MARLIN_FP8_AVAILABLE = True +except ImportError: + MARLIN_FP8_AVAILABLE = False + + def apply_fp8_marlin_linear(*args, **kwargs): + raise ImportError("vllm is not installed") + + def prepare_fp8_layer_for_marlin(*args, **kwargs): + raise ImportError("vllm is not installed") + + +__all__ = ["CompressedTensorsW8A16Fp8"] + +SUPPORTED_STRATEGIES = [QuantizationStrategy.CHANNEL, QuantizationStrategy.TENSOR] + + +class CompressedTensorsW8A16Fp8(CompressedTensorsScheme): + def __init__(self, strategy: str, is_static_input_scheme: bool): + self.strategy = strategy + self.is_static_input_scheme = is_static_input_scheme + + if not MARLIN_FP8_AVAILABLE: + raise ImportError( + "vllm is not installed. To use CompressedTensorsW8A16Fp8, please install vllm" + ) + + @classmethod + def get_min_capability(cls) -> int: + # ampere and up + return 80 + + # W8A8-Fp8 kernels support only per-tensor and per-channel cases. + # So if we have a fused module (QKV, MLP) with per tensor scales, + # we expand each scale to its shard's channels. + def process_weights_after_loading(self, layer) -> None: + if self.strategy == QuantizationStrategy.TENSOR: + ws_channelwise = convert_to_channelwise( + layer.weight_scale, layer.logical_widths + ) + layer.weight_scale = torch.nn.Parameter(ws_channelwise, requires_grad=False) + else: + # required by torch.compile to be torch.nn.Parameter + layer.weight_scale = torch.nn.Parameter( + layer.weight_scale.data, requires_grad=False + ) + + # Weights must be transposed for marlin + layer.weight = torch.nn.Parameter(layer.weight.t(), requires_grad=False) + + if self.is_static_input_scheme: + # required by torch.compile to be torch.nn.Parameter + layer.input_scale = torch.nn.Parameter( + layer.input_scale.data, requires_grad=False + ) + prepare_fp8_layer_for_marlin(layer, size_k_first=True) + + def create_weights( + self, + layer: torch.nn.Module, + input_size: int, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + if self.strategy == QuantizationStrategy.CHANNEL: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader, + ) + elif self.strategy == QuantizationStrategy.TENSOR: + weight_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + else: + raise ValueError( + f"Unsupported weight strategy={self.strategy}, " + f"supported strategies are {SUPPORTED_STRATEGIES}" + ) + + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE (to deal with converted checkpoints) + if self.is_static_input_scheme: + input_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("input_scale", input_scale) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_fp8_marlin_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias, + ) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py new file mode 100644 index 000000000..a157ebc3e --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py @@ -0,0 +1,172 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 + +from typing import Callable, List, Optional + +import torch +from compressed_tensors.quantization import QuantizationStrategy +from torch.nn import Parameter + +from sglang.srt.layers.parameter import ( + ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter, +) +from sglang.srt.layers.quantization.compressed_tensors.schemes import ( + CompressedTensorsScheme, +) +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, + normalize_e4m3fn_to_e4m3fnuz, +) +from sglang.srt.layers.quantization.utils import requantize_with_max_scale +from sglang.srt.utils import get_bool_env_var, is_hip + +__all__ = ["CompressedTensorsW8A8Fp8"] + +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +if _use_aiter: + from aiter.ops.shuffle import shuffle_weight + + +class CompressedTensorsW8A8Fp8(CompressedTensorsScheme): + + def __init__(self, strategy: str, is_static_input_scheme: bool): + self.strategy = strategy + self.is_static_input_scheme = is_static_input_scheme + + @classmethod + def get_min_capability(cls) -> int: + # lovelace and up + return 89 + + def process_weights_after_loading(self, layer) -> None: + # If per tensor, when we have a fused module (e.g. QKV) with per + # tensor scales (thus N scales being passed to the kernel), + # requantize so we can always run per tensor + if self.strategy == QuantizationStrategy.TENSOR: + max_w_scale, weight = requantize_with_max_scale( + weight=layer.weight, + weight_scale=layer.weight_scale, + logical_widths=layer.logical_widths, + ) + + if is_fp8_fnuz(): + input_scale = getattr(layer, "input_scale", None) + + weight, max_w_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz( + weight=weight, weight_scale=max_w_scale, input_scale=input_scale + ) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, requires_grad=False) + + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(max_w_scale, requires_grad=False) + + # If channelwise, scales are already lined up, so just transpose. + elif self.strategy == QuantizationStrategy.CHANNEL: + weight = layer.weight + + if is_fp8_fnuz(): + input_scale = getattr(layer, "input_scale", None) + + weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=layer.weight_scale, + input_scale=input_scale, + ) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, requires_grad=False) + else: + weight_scale = layer.weight_scale.data + + if _use_aiter: + layer.weight = Parameter( + shuffle_weight(weight, (16, 16)), requires_grad=False + ) + else: + layer.weight = Parameter(weight.t(), requires_grad=False) + + # required by torch.compile to be torch.nn.Parameter + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + + else: + raise ValueError(f"Unknown quantization strategy {self.strategy}") + + # INPUT SCALE + if self.is_static_input_scheme and hasattr(layer, "input_scale"): + layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False) + else: + layer.input_scale = None + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: List[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs, + ): + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + + # WEIGHT + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + # TODO: update create_xxx_parameter functions to return + # the newly added parameters + if self.strategy == QuantizationStrategy.CHANNEL: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader, + ) + else: + assert self.strategy == QuantizationStrategy.TENSOR + weight_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + + # min requirement for fp8 kernels + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE + if self.is_static_input_scheme: + input_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + input_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", input_scale) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + use_per_token_if_dynamic=True, + compressed_tensor_quant=True, + ) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/utils.py b/python/sglang/srt/layers/quantization/compressed_tensors/utils.py new file mode 100644 index 000000000..ddefa5ea3 --- /dev/null +++ b/python/sglang/srt/layers/quantization/compressed_tensors/utils.py @@ -0,0 +1,218 @@ +# Adapted from https://github.com/vllm-project/vllm/tree/main/vllm/model_executor/layers/quantization/compressed_tensors +# SPDX-License-Identifier: Apache-2.0 + +import re +from types import MappingProxyType +from typing import Iterable, List, Mapping, Optional + +from compressed_tensors import CompressionFormat +from torch.nn import Module + + +def is_activation_quantization_format(format: str) -> bool: + _ACTIVATION_QUANTIZATION_FORMATS = [ + CompressionFormat.naive_quantized.value, + CompressionFormat.int_quantized.value, + CompressionFormat.float_quantized.value, + ] + return format in _ACTIVATION_QUANTIZATION_FORMATS + + +def should_ignore_layer( + layer_name: Optional[str], + ignore: Iterable[str] = tuple(), + fused_mapping: Mapping[str, List[str]] = MappingProxyType({}), +) -> bool: + if layer_name is None: + return False + + # layer_name = model.layers.0.self_attn.qkv_proj + # proj_name = qkv_proj + proj_name = layer_name.split(".")[-1] + + # Fused layers like gate_up_proj or qkv_proj will not be fused + # in the safetensors checkpoint. So, we convert the name + # from the fused version to unfused + check to make sure that + # each shard of the fused layer has the same scheme. + if proj_name in fused_mapping and layer_name not in ignore: + shard_proj_names = fused_mapping[proj_name] + + # Convert fused_name --> [shard_names] + shard_names = [ + layer_name.replace(proj_name, shard_proj_name) + for shard_proj_name in shard_proj_names + ] + + # Layer should be ignored if shards are ignored. + should_ignore_layer = None + for shard_name in shard_names: + should_ignore_shard = check_equal_or_regex_match( + layer_name=shard_name, targets=ignore + ) + + # If shard_idx=0, set layer ignore to match shard. + if should_ignore_layer is None: + should_ignore_layer = should_ignore_shard + + # If shard_idx=1+ confirm scheme matches prior shards. + elif should_ignore_shard != should_ignore_layer: + raise ValueError( + f"Found a different quantization schemes for " + f"{shard_proj_names} in {layer_name}. vLLM " + "requires all to use the same scheme." + ) + + # Unfused layers like down_proj and o_proj will match + # the safetensors checkpoint already. + else: + should_ignore_layer = check_equal_or_regex_match( + layer_name=layer_name, targets=ignore + ) + + assert should_ignore_layer is not None + return should_ignore_layer + + +def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool: + """ + Checks whether a layer_name is exactly equal or a regex match for + if target starts with 're:' to any target in list. + """ + for target in targets: + if _is_equal_or_regex_match(layer_name, target): + return True + return False + + +def find_matched_target( + layer_name: Optional[str], + module: Module, + targets: Iterable[str], + fused_mapping: Mapping[str, List[str]] = MappingProxyType({}), +) -> str: + """ + Helper function to look up which "target" in the compressed-tensors + config that a layer corresponds to. + + Recall that a compressed-tensors configs has a concept of + config_groups, where each layer can be quantized with with a different + scheme. + + targets in each config_group will be a list of either layer names + (or regexes corresponding to layer names) or names of torch Modules. + + First, we try to match the layer_name with a target + Second, we try to match the module's name with a target + Third, we try to map the layer_name to a list of fused module names. + *All* component module names must match in order for a match to be + successful. A successful match returns the first component target + + :param layer_name: layer name + :param module: torch.nn.Module + :param targets: list of targets to match the layer against + :param fused_mapping: map from fused layer names to its components + :param fused_strategy: either "all" or "any". If using "all", fused + layers match if "all" of its components match + """ + + if layer_name is None: + layer_name = "" + + matched_target = ( + _find_first_match(layer_name, targets) + or _find_first_match(module.__class__.__name__, targets, True) + or _match_fused_layer(layer_name, targets, fused_mapping) + ) + + if matched_target is None: + raise ValueError( + f"Unable to find matching target for {layer_name} in the " + "compressed-tensors config." + ) + + return matched_target + + +def _find_first_match( + value: str, targets: Iterable[str], check_contains: bool = False +) -> Optional[str]: + """ + Returns first element of target that matches value either + exactly or as a regex after 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + + :param value: string to compare the list of targets against + :param targets: list of targets to match the layer against + :param check_contains: whether or not to do a substring match + """ + + for target in targets: + if _is_equal_or_regex_match(value, target, check_contains=check_contains): + return target + return None + + +def _is_equal_or_regex_match( + value: str, target: str, check_contains: bool = False +) -> bool: + """ + Checks whether a value is exactly equal or a regex match for target + if target starts with 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + """ + + if target.startswith("re:"): + pattern = target[3:] + if re.match(pattern, value): + return True + elif check_contains: + if target.lower() in value.lower(): + return True + elif target == value: + return True + return False + + +def _match_fused_layer( + layer_name: str, + target_layers: Iterable[str], + fused_mapping: Mapping[str, List[str]], +) -> Optional[str]: + """ + Match a fused layer name to its corresponding individual layer in + target_layers. Returns first value in fused_mapping which matches targets + + Implements an "all" matching strategy where a fused layer matches iff + "all" of its components match + + :param layer_name: layer name + :param target_layers: list of targets to match the layer against + :param fused_mapping: map from fused layer names to its components + + Examples: + layer_name = "model.layers.0.self_attn.qkv_proj" + target_layers = ["model.layers.0.self_attn.q_proj", + "model.layers.0.self_attn.k_proj", + "model.layers.0.self_attn.v_proj"] + """ + # find layer_name in mapping + fused = next((key for key in fused_mapping if layer_name.endswith(key)), None) + if fused is None: + return None + + # expand path of unfused components + unfused_paths = [ + layer_name.replace(fused, unfused) for unfused in fused_mapping[fused] + ] + + # for each unfused component, find a match in targets + unfused_matches: List[Optional[str]] = [] + for unfused in unfused_paths: + for target in target_layers: + if _is_equal_or_regex_match(unfused, target): + unfused_matches.append(target) + break + else: + unfused_matches.append(None) + + return unfused_matches[0] if all(unfused_matches) else None diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6ae89c757 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..ae4c397ef --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6496a38fb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..effb7bda7 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..eb1848894 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=1536,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c098ef2db --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c098ef2db --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3633e363f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..653bb997e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..31d29e10c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3618053b6 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..cb573ccdf --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..46a982f5e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a5679be13 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=1536,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..7d8044772 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..eb11c1696 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..035ec027f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..62ccf8e6f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..8b49f2781 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dd66019cc --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2048,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5011fd572 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1957a7f32 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..851bc9f9f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6c2eebe0f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..d1227c215 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..448daf360 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=2304,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6f5adbb93 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6f5adbb93 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6f5adbb93 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..7b10d6d78 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e76d41dfd --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..77ba0d747 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1c61451fb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..66ca4ee16 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..48b64a87d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..63e661c80 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..d521e2bd0 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=24576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..d72288781 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c65a16f3d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..cf3540379 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a8ab31bfe --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1477e063d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=256,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6f5adbb93 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6f5adbb93 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..80d23e969 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..0a5d7bfdb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..970345270 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..eccb86a76 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=1536,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..cb91a279d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..88af48431 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a794fc821 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dd069726d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=3072,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4225c78eb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4225c78eb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4225c78eb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..933170b7b --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..797ee3d45 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..7febe3d27 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..56b939e52 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1c36fe951 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..30932a1cc --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..63d9a0bf5 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..65989f012 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=32768,K=512,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..7fa398c15 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..f15d8f64c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=36864,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4225c78eb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4225c78eb --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a5518d979 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..9d7658bfc --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..cd3e07804 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..932321cca --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..9d5a329d7 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4096,K=512,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5e6789d00 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5e6789d00 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..44c67c008 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..03dba5ad1 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..96e1594a3 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..017dca003 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5ffd367df --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=4608,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..49ac14d2a --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..49ac14d2a --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..09502f05c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..9a5ff48b8 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b57588f9b --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..eabc42394 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=512,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dcbb0efc5 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dcbb0efc5 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..7fd942e3e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..13e3c20b7 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..ecbca9d8d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..386928de1 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 256, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..51e237b91 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a8ab31bfe --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..22ca71e2c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6280219c9 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..78db73078 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=576,K=7168,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5e40bbe64 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..044868d2d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..40c01c0b9 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b96981dec --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c6fd36597 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..f4e026497 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1024,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..246062a22 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..0b24efa37 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..160f12ed3 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a3a434e12 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e5c4a1d2c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..2f3962f68 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=1152,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..bfdff8adc --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..d694716c3 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..2bf5eb27e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..871a0cf2e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 2 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..f211a5d38 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=128,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dfe5c1e43 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dfe5c1e43 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dfe5c1e43 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b2cab0831 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..9de2ba9c8 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..9c908e804 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..0a1e14cff --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c4b1bd435 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..c87cb4f95 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..15b1c93f6 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5bf208ce4 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=16384,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..6ae00329e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A100-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..59b79432f --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_A800-SXM4-80GB,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..f78e7060e --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..8ff12e64c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..e997126f9 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b474e6265 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..4532f9368 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a3f5371f1 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=18432,device_name=NVIDIA_L20Y,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,26 @@ +{ + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dfe5c1e43 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..dfe5c1e43 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..72ab3dff2 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..1d3ce5c94 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..ca7f32b95 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..12e305853 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..5acea242c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2048,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a87f5de1b --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a87f5de1b --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..37208577d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3ab5796ee --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 4 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..58cdd93e9 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H100_80GB_HBM3,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..a5ce5e49b --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..b72e0371d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=2304,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..468f9e78d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI300X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..468f9e78d --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Instinct_MI325X,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 16, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..d8cc0f896 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=AMD_Radeon_Graphics,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,164 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "8": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "24": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "32": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "96": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1024": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 8, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "2048": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 4, + "num_warps": 4, + "num_stages": 2, + "waves_per_eu": 0 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..3cb7eaa07 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_B200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 1, + "num_warps": 8, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 2 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 8, + "num_stages": 5 + }, + "48": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "96": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 3 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..129f59e0c --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H20,dtype=int8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "2": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 8, + "num_stages": 5 + }, + "4": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "8": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 5 + }, + "16": { + "BLOCK_SIZE_M": 32, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "32": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "64": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 256, + "BLOCK_SIZE_K": 64, + "GROUP_SIZE_M": 64, + "num_warps": 8, + "num_stages": 2 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "128": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 32, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "256": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 2 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + } +} diff --git a/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json new file mode 100644 index 000000000..293adce38 --- /dev/null +++ b/python/sglang/srt/layers/quantization/configs/N=7168,K=256,device_name=NVIDIA_H200,dtype=fp8_w8a8,block_shape=[128, 128].json @@ -0,0 +1,146 @@ +{ + "1": { + "BLOCK_SIZE_M": 16, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "2": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "4": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "8": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "16": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "24": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "32": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "48": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "64": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 4 + }, + "96": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 5 + }, + "128": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 5 + }, + "256": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 64, + "num_warps": 4, + "num_stages": 4 + }, + "512": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 128, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 16, + "num_warps": 4, + "num_stages": 3 + }, + "1024": { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 3 + }, + "1536": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "2048": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "3072": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + }, + "4096": { + "BLOCK_SIZE_M": 128, + "BLOCK_SIZE_N": 64, + "BLOCK_SIZE_K": 128, + "GROUP_SIZE_M": 1, + "num_warps": 4, + "num_stages": 2 + } +} diff --git a/python/sglang/srt/layers/quantization/deep_gemm_wrapper/__init__.py b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/__init__.py new file mode 100644 index 000000000..32f3ebe04 --- /dev/null +++ b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/__init__.py @@ -0,0 +1 @@ +from .entrypoint import * diff --git a/python/sglang/srt/layers/quantization/deep_gemm_wrapper/compile_utils.py b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/compile_utils.py new file mode 100644 index 000000000..e374759c4 --- /dev/null +++ b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/compile_utils.py @@ -0,0 +1,241 @@ +import logging +import os +from contextlib import contextmanager +from enum import IntEnum, auto +from typing import Dict, List, Tuple + +import torch +from tqdm import tqdm + +from sglang.srt.layers.quantization.deep_gemm_wrapper.configurer import ( + ENABLE_JIT_DEEPGEMM, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import ceil_div, get_bool_env_var, get_int_env_var + +logger = logging.getLogger(__name__) + +if ENABLE_JIT_DEEPGEMM: + import deep_gemm + + +_BUILTIN_M_LIST = list(range(1, 1024 * 16 + 1)) +_ENABLE_JIT_DEEPGEMM_PRECOMPILE = get_bool_env_var( + "SGL_JIT_DEEPGEMM_PRECOMPILE", "true" +) +_DO_COMPILE_ALL = True +_IS_FIRST_RANK_ON_NODE = get_bool_env_var("SGL_IS_FIRST_RANK_ON_NODE", "true") +_COMPILE_WORKERS = get_int_env_var("SGL_JIT_DEEPGEMM_COMPILE_WORKERS", 4) +_IN_PRECOMPILE_STAGE = get_bool_env_var("SGL_IN_DEEPGEMM_PRECOMPILE_STAGE", "false") + +# Force redirect deep_gemm cache_dir +os.environ["DG_JIT_CACHE_DIR"] = os.getenv( + "SGL_DG_CACHE_DIR", os.path.join(os.path.expanduser("~"), ".cache", "deep_gemm") +) + +# Refer to https://github.com/deepseek-ai/DeepGEMM/commit/d75b218b7b8f4a5dd5406ac87905039ead3ae42f +# NVRTC may have performance loss with some cases. +# And NVCC JIT speed is also 9x faster in the ref commit +os.environ["DG_JIT_USE_NVRTC"] = os.getenv("SGL_DG_USE_NVRTC", "0") + + +def update_deep_gemm_config(gpu_id: int, server_args: ServerArgs): + global _BUILTIN_M_LIST + global _DO_COMPILE_ALL + global _IS_FIRST_RANK_ON_NODE + + # Generate m_max + m_max = 1024 * 16 + if server_args.chunked_prefill_size < 1: + m_max = 1024 * 64 + elif server_args.chunked_prefill_size > 8192: + m_max = server_args.chunked_prefill_size * 2 + m_max = min(1024 * 128, m_max) + _BUILTIN_M_LIST = list(range(1, m_max + 1)) + + _IS_FIRST_RANK_ON_NODE = ServerArgs.base_gpu_id == gpu_id + + # Check if is the first rank on node. + # Default each rank will try compile all Ms to + # load all symbols at the launch stages. + # Avoid loading symbols at the serving stages. + _DO_COMPILE_ALL = _IS_FIRST_RANK_ON_NODE + + +class DeepGemmKernelType(IntEnum): + GROUPED_GEMM_NT_F8F8BF16_MASKED = auto() + GROUPED_GEMM_NT_F8F8BF16_CONTIG = auto() + GEMM_NT_F8F8BF16 = auto() + + +_INITIALIZATION_DICT: Dict[Tuple[DeepGemmKernelType, int, int, int], bool] = dict() + + +# TODO improve code +def _maybe_compile_deep_gemm_one_type_all( + kernel_type: DeepGemmKernelType, + n: int, + k: int, + num_groups: int, +) -> None: + global _INITIALIZATION_DICT + global _BUILTIN_M_LIST + + query_key = (kernel_type, n, k, num_groups) + if ( + _ENABLE_JIT_DEEPGEMM_PRECOMPILE + and _DO_COMPILE_ALL + and _INITIALIZATION_DICT.get(query_key) is None + ): + _INITIALIZATION_DICT[query_key] = True + + # TODO maybe improve logs + if not _IN_PRECOMPILE_STAGE and _IS_FIRST_RANK_ON_NODE: + logger.warning( + "Entering DeepGEMM JIT Pre-Compile session. " + "It may take a long time (typically 10-20 mins) " + "if you have not run `sglang.compile_deep_gemm`. " + "It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server`" + " for pre-compilation to reduce the overhead if you have not run it before. " + "For example: " + "`python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`" + ) + + logger.info( + f"Try DeepGEMM JIT Compiling for " + f"<{kernel_type.name}> N={n}, K={k}, num_groups={num_groups} with all Ms." + f"{' It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. ' if not _IN_PRECOMPILE_STAGE else ''}" + ) + + _compile_deep_gemm_one_type_all( + kernel_type=kernel_type, + n=n, + k=k, + num_groups=num_groups, + m_list=_BUILTIN_M_LIST, + ) + + +# NOTE(alcanderian): get_num_sms should be change when 2-batch-overlap is introduced +def _compile_deep_gemm_one_type_all( + kernel_type: DeepGemmKernelType, + n: int, + k: int, + num_groups: int, + m_list: List[int], +) -> None: + if kernel_type == DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: + m_alignment = deep_gemm.get_mk_alignment_for_contiguous_layout() + m_list = sorted(list(set(m for m in m_list if m % m_alignment == 0))) + + executor = _BaseWarmupExecutor.create( + kernel_type, max_m=max(m_list), n=n, k=k, num_groups=num_groups + ) + + old_compile_mode = deep_gemm.get_compile_mode() + deep_gemm.set_compile_mode(1) + # TODO can use multi thread + for m in tqdm(m_list, desc=f"DeepGEMM warmup"): + executor.execute(m=m) + deep_gemm.set_compile_mode(old_compile_mode) + + # clean up input buffers + torch.cuda.current_stream().synchronize() + del executor + torch.cuda.empty_cache() + + +class _BaseWarmupExecutor: + @staticmethod + def create(kernel_type: DeepGemmKernelType, **kwargs): + return { + DeepGemmKernelType.GEMM_NT_F8F8BF16: _NormalWarmupExecutor, + DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG: _GroupedContWarmupExecutor, + DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED: _GroupedMaskedWarmupExecutor, + }[kernel_type](**kwargs) + + def execute(self, m): + raise NotImplementedError + + +def _empty_token_fp8(size): + *dims, k = size + return ( + torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn), + torch.empty( + (*dims, ceil_div(k, _BLOCK_SIZE)), device="cuda", dtype=torch.float32 + ), + ) + + +def _empty_block_fp8(size): + *dims, n, k = size + return ( + torch.empty(size, device="cuda", dtype=torch.float8_e4m3fn), + torch.empty( + (*dims, ceil_div(n, _BLOCK_SIZE), ceil_div(k, _BLOCK_SIZE)), + device="cuda", + dtype=torch.float32, + ), + ) + + +_BLOCK_SIZE = 128 + + +class _NormalWarmupExecutor(_BaseWarmupExecutor): + def __init__(self, max_m: int, n: int, k: int, num_groups: int): + self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k)) + self.rhs_q, self.rhs_s = _empty_block_fp8((n, k)) + self.out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16) + + def execute(self, m): + deep_gemm.fp8_gemm_nt( + (self.lhs_q[:m], self.lhs_s[:m]), + (self.rhs_q, self.rhs_s), + self.out[:m], + ) + + +class _GroupedContWarmupExecutor(_BaseWarmupExecutor): + def __init__(self, max_m: int, n: int, k: int, num_groups: int): + self.lhs_q, self.lhs_s = _empty_token_fp8((max_m, k)) + self.rhs_q, self.rhs_s = _empty_block_fp8((num_groups, n, k)) + self.m_indices = torch.zeros((max_m,), device="cuda", dtype=torch.int32) + self.out = torch.empty((max_m, n), device="cuda", dtype=torch.bfloat16) + + def execute(self, m): + deep_gemm.m_grouped_fp8_gemm_nt_contiguous( + (self.lhs_q[:m], self.lhs_s[:m]), + (self.rhs_q, self.rhs_s), + self.out[:m], + m_indices=self.m_indices[:m], + ) + + +class _GroupedMaskedWarmupExecutor(_BaseWarmupExecutor): + def __init__(self, max_m: int, n: int, k: int, num_groups: int): + self.lhs_q, self.lhs_s = _empty_token_fp8((num_groups, max_m, k)) + self.rhs_q, self.rhs_s = _empty_block_fp8((num_groups, n, k)) + self.masked_m = torch.zeros((num_groups,), device="cuda", dtype=torch.int32) + self.out = torch.empty( + (num_groups, max_m, n), device="cuda", dtype=torch.bfloat16 + ) + + def execute(self, m): + deep_gemm.fp8_m_grouped_gemm_nt_masked( + (self.lhs_q, self.lhs_s), + (self.rhs_q, self.rhs_s), + self.out, + masked_m=self.masked_m, + # DeepGEMM uses `expect_m` instead of input shape for `get_best_config` + expected_m=m, + ) + + +@contextmanager +def deep_gemm_execution_hook( + m: int, n: int, k: int, num_groups: int, kernel_type: DeepGemmKernelType +): + _maybe_compile_deep_gemm_one_type_all(kernel_type, n, k, num_groups) + yield diff --git a/python/sglang/srt/layers/quantization/deep_gemm_wrapper/configurer.py b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/configurer.py new file mode 100644 index 000000000..ecf7d1647 --- /dev/null +++ b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/configurer.py @@ -0,0 +1,32 @@ +import logging + +import torch + +from sglang.srt.utils import get_bool_env_var, get_device_sm + +logger = logging.getLogger(__name__) + + +def _compute_enable_deep_gemm(): + sm_version = get_device_sm() + if sm_version < 90: + return False + + try: + import deep_gemm + except ImportError: + logger.warning("Failed to import deep_gemm, disable ENABLE_JIT_DEEPGEMM.") + return False + + return get_bool_env_var("SGL_ENABLE_JIT_DEEPGEMM", default="true") + + +def _is_blackwell_arch() -> bool: + major, minor = torch.cuda.get_device_capability(torch.cuda.current_device()) + return major == 10 + + +ENABLE_JIT_DEEPGEMM = _compute_enable_deep_gemm() + +DEEPGEMM_BLACKWELL = ENABLE_JIT_DEEPGEMM and _is_blackwell_arch() +DEEPGEMM_SCALE_UE8M0 = DEEPGEMM_BLACKWELL diff --git a/python/sglang/srt/layers/quantization/deep_gemm_wrapper/entrypoint.py b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/entrypoint.py new file mode 100644 index 000000000..02945f449 --- /dev/null +++ b/python/sglang/srt/layers/quantization/deep_gemm_wrapper/entrypoint.py @@ -0,0 +1,119 @@ +import logging +from contextlib import contextmanager +from typing import Tuple + +import torch + +from sglang.srt.layers.quantization.deep_gemm_wrapper import compile_utils +from sglang.srt.layers.quantization.deep_gemm_wrapper.configurer import ( + DEEPGEMM_BLACKWELL, + DEEPGEMM_SCALE_UE8M0, + ENABLE_JIT_DEEPGEMM, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import get_bool_env_var + +logger = logging.getLogger(__name__) + +if ENABLE_JIT_DEEPGEMM: + import deep_gemm + from deep_gemm.utils.layout import get_mn_major_tma_aligned_tensor + +_SANITY_CHECK = get_bool_env_var("SGLANG_DEEPGEMM_SANITY_CHECK") + + +# TODO maybe rename these functions +def grouped_gemm_nt_f8f8bf16_masked( + lhs: Tuple[torch.Tensor, torch.Tensor], + rhs: Tuple[torch.Tensor, torch.Tensor], + out: torch.Tensor, + masked_m: torch.Tensor, + expected_m: int, +): + num_groups, _, k = lhs[0].shape + _, n, _ = rhs[0].shape + kernel_type = compile_utils.DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_MASKED + + _sanity_check_input(lhs) + _sanity_check_input(rhs) + + with compile_utils.deep_gemm_execution_hook( + expected_m, n, k, num_groups, kernel_type + ): + deep_gemm.fp8_m_grouped_gemm_nt_masked( + lhs, + rhs, + out, + masked_m, + expected_m, + ) + + +def grouped_gemm_nt_f8f8bf16_contig( + lhs: Tuple[torch.Tensor, torch.Tensor], + rhs: Tuple[torch.Tensor, torch.Tensor], + out: torch.Tensor, + m_indices: torch.Tensor, +): + m, k = lhs[0].shape + num_groups, n, _ = rhs[0].shape + kernel_type = compile_utils.DeepGemmKernelType.GROUPED_GEMM_NT_F8F8BF16_CONTIG + + _sanity_check_input(lhs) + _sanity_check_input(rhs) + + with compile_utils.deep_gemm_execution_hook(m, n, k, num_groups, kernel_type): + deep_gemm.m_grouped_fp8_gemm_nt_contiguous(lhs, rhs, out, m_indices) + + +def gemm_nt_f8f8bf16( + lhs: Tuple[torch.Tensor, torch.Tensor], + rhs: Tuple[torch.Tensor, torch.Tensor], + out: torch.Tensor, +): + m, k = lhs[0].shape + n, _ = rhs[0].shape + num_groups = 1 + kernel_type = compile_utils.DeepGemmKernelType.GEMM_NT_F8F8BF16 + + _sanity_check_input(lhs) + _sanity_check_input(rhs) + + with compile_utils.deep_gemm_execution_hook(m, n, k, num_groups, kernel_type): + deep_gemm.fp8_gemm_nt( + lhs, + rhs, + out, + ) + + +def update_deep_gemm_config(gpu_id: int, server_args: ServerArgs): + compile_utils.update_deep_gemm_config(gpu_id, server_args) + + +@contextmanager +def configure_deep_gemm_num_sms(num_sms): + if num_sms is None: + yield + else: + original_num_sms = deep_gemm.get_num_sms() + deep_gemm.set_num_sms(num_sms) + try: + yield + finally: + deep_gemm.set_num_sms(original_num_sms) + + +def _sanity_check_input(x_fp8: Tuple[torch.Tensor, torch.Tensor]): + if not _SANITY_CHECK: + return + + x, x_scale = x_fp8 + + if x_scale.dtype == torch.int: + return + + from sglang.srt.layers.quantization.fp8_utils import ceil_to_ue8m0 + + x_scale_ceil = ceil_to_ue8m0(x_scale) + assert torch.all(x_scale == x_scale_ceil), f"{x_scale=} {x_scale_ceil=}" diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py new file mode 100644 index 000000000..b020e4188 --- /dev/null +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -0,0 +1,1240 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/layers/quantization/fp8.py + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +import torch +import torch.nn.functional as F +from torch.nn import Module +from torch.nn.parameter import Parameter + +try: + from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, + prepare_fp8_layer_for_marlin, + ) + + MARLIN_FP8_AVAILABLE = True +except ImportError: + MARLIN_FP8_AVAILABLE = False + + def dummy_func(*args, **kwargs): + raise ImportError( + "marlin FP8 requires some operators from vllm. Please install vllm." + ) + + apply_fp8_marlin_linear = prepare_fp8_layer_for_marlin = dummy_func + + +from sglang.srt.distributed import get_tensor_model_parallel_world_size +from sglang.srt.layers.amx_utils import _amx_process_weight_after_loading +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.moe.token_dispatcher.base import DispatchOutputChecker +from sglang.srt.layers.parameter import ( + BlockQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter, +) +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8_kernel import ( + fp8_dtype, + is_fp8_fnuz, + per_token_group_quant_fp8, + scaled_fp8_quant, +) +from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, + can_auto_enable_marlin_fp8, + cutlass_fp8_supported, + dispatch_w8a8_block_fp8_linear, + input_to_float8, + normalize_e4m3fn_to_e4m3fnuz, +) +from sglang.srt.layers.quantization.kv_cache import BaseKVCacheMethod +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import ( + all_close_1d, + convert_to_channelwise, + is_layer_skipped, + per_tensor_dequantize, + requantize_with_max_scale, +) +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, + is_npu, + is_sm90_supported, + is_sm100_supported, + log_info_on_rank0, + next_power_of_2, + print_warning_once, + set_weight_attrs, + use_intel_amx_backend, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + DispatchOutput, + StandardDispatchOutput, + ) + from sglang.srt.layers.moe.topk import TopKOutput + from sglang.srt.layers.quantization.w4afp8 import W4AFp8Config + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_npu = is_npu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() + +_is_fp8_fnuz = is_fp8_fnuz() + +_use_hip_int4 = get_bool_env_var("SGLANG_INT4_WEIGHT") +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _is_hip and (_use_aiter or _use_hip_int4): + from aiter import ActivationType, QuantType + from aiter.fused_moe import fused_moe + from aiter.ops.shuffle import shuffle_weight + + +ACTIVATION_SCHEMES = ["static", "dynamic"] + +logger = logging.getLogger(__name__) + + +class Fp8Config(QuantizationConfig): + """Config class for FP8.""" + + def __init__( + self, + is_checkpoint_fp8_serialized: bool = False, + activation_scheme: str = "dynamic", + ignored_layers: Optional[List[str]] = None, + weight_block_size: List[int] = None, + ) -> None: + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + if is_checkpoint_fp8_serialized: + log_info_on_rank0(logger, "Detected fp8 checkpoint.") + if activation_scheme not in ACTIVATION_SCHEMES: + raise ValueError(f"Unsupported activation scheme {activation_scheme}") + self.activation_scheme = activation_scheme + self.ignored_layers = ignored_layers or [] + if weight_block_size is not None: + if not is_checkpoint_fp8_serialized: + raise ValueError( + f"The block-wise quantization only supports fp8-serialized checkpoint for now." + ) + if len(weight_block_size) != 2: + raise ValueError( + f"The quantization block size of weight must have 2 dimensions, but got {len(weight_block_size)} dimensions." + ) + if activation_scheme != "dynamic": + raise ValueError( + f"The block-wise quantization only supports dynamic activation scheme for now, but got {activation_scheme} activation scheme." + ) + self.weight_block_size = weight_block_size + + @classmethod + def get_name(cls) -> str: + return "fp8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> Fp8Config: + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_fp8_serialized = "fp8" in quant_method + activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) + ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) + weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None) + return cls( + is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, + activation_scheme=activation_scheme, + ignored_layers=ignored_layers, + weight_block_size=weight_block_size, + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.ignored_layers): + return UnquantizedLinearMethod() + return Fp8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return Fp8MoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class Fp8LinearMethod(LinearMethodBase): + """Linear method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + + Limitations: + 1. Only support per-tensor quantization due to torch._scaled_mm support. + 2. Only support float8_e4m3fn data type due to the limitation of + torch._scaled_mm (https://github.com/pytorch/pytorch/blob/2e48b39603411a41c5025efbe52f89560b827825/aten/src/ATen/native/cuda/Blas.cpp#L854-L856) + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: Union[Fp8Config, W4AFp8Config]): + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + # For GPUs that lack FP8 hardware support, we can leverage the Marlin + # kernel for fast weight-only FP8 quantization + self.use_marlin = False + if _is_cuda and MARLIN_FP8_AVAILABLE: + force_marlin = get_bool_env_var("SGLANG_FORCE_FP8_MARLIN") + auto_enable = can_auto_enable_marlin_fp8() + self.use_marlin = force_marlin or auto_enable + + self.block_quant = self.quant_config.weight_block_size is not None + + self.w8a8_block_fp8_linear = dispatch_w8a8_block_fp8_linear() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + tp_size = get_tensor_model_parallel_world_size() + if self.block_quant: + block_n, block_k = ( + self.quant_config.weight_block_size[0], + self.quant_config.weight_block_size[1], + ) + # Required by row parallel + if tp_size > 1 and input_size // input_size_per_partition == tp_size: + if input_size_per_partition % block_k != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + # Required by column parallel or enabling merged weights + if ( + tp_size > 1 and output_size // output_size_per_partition == tp_size + ) or len(output_partition_sizes) > 1: + for output_partition_size in output_partition_sizes: + if output_partition_size % block_n != 0: + raise ValueError( + f"Weight output_partition_size = " + f"{output_partition_size} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight_dtype = ( + torch.float8_e4m3fn + if self.quant_config.is_checkpoint_fp8_serialized + else params_dtype + ) + + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, input_size_per_partition, dtype=weight_dtype + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # If checkpoint is serialized fp8, load them. + # Otherwise, wait until process_weights_after_loading. + if self.quant_config.is_checkpoint_fp8_serialized: + # WEIGHT SCALE + if self.block_quant: + if hasattr(self.quant_config, "activation_scheme"): + assert self.quant_config.activation_scheme == "dynamic" + elif hasattr(self.quant_config, "linear_activation_scheme"): + assert self.quant_config.linear_activation_scheme == "dynamic" + scale = BlockQuantScaleParameter( + data=torch.empty( + (output_size_per_partition + block_n - 1) // block_n, + (input_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale_inv", scale) + else: + scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", scale) + + # INPUT ACTIVATION SCALE + if ( + hasattr(self.quant_config, "activation_scheme") + and self.quant_config.activation_scheme == "static" + ) or ( + hasattr(self.quant_config, "linear_activation_scheme") + and self.quant_config.linear_activation_scheme == "static" + ): + scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + + scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", scale) + else: + layer.register_parameter("input_scale", None) + + def process_weights_after_loading(self, layer: Module) -> None: + if self.block_quant: + # If ROCm, normalize the weights and scales to e4m3fnuz + if _is_fp8_fnuz: + # activation_scheme: dynamic + weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=layer.weight, + weight_scale=layer.weight_scale_inv, + input_scale=None, + ) + layer.input_scale = None + elif _is_cpu: + assert ( + _is_cpu_amx_available + ), "Fp8LinearMethod on CPU requires that CPU has AMX support" + _amx_process_weight_after_loading(layer, ["weight"]) + layer.weight_scale_inv = torch.nn.Parameter( + layer.weight_scale_inv.data, requires_grad=False + ) + return + else: + weight, weight_scale = layer.weight.data, layer.weight_scale_inv.data + layer.weight = Parameter(weight, requires_grad=False) + layer.weight_scale_inv = Parameter(weight_scale, requires_grad=False) + else: + layer.weight = Parameter(layer.weight.data, requires_grad=False) + + # If checkpoint not serialized fp8, quantize the weights. + if not self.quant_config.is_checkpoint_fp8_serialized: + if self.cutlass_fp8_supported or self.use_marlin: + # apply per-channel quantization default as + # cutlass sgl-kernel and marlin only support per-channel scale + qweight, weight_scale = per_token_group_quant_fp8( + layer.weight, layer.weight.shape[-1] + ) + weight_scale = weight_scale.t().contiguous() + else: + # per-tensor quantization + qweight, weight_scale = input_to_float8(layer.weight) + + # Update the layer with the new values. + layer.weight = Parameter(qweight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + layer.input_scale = None + + # If checkpoint is fp8, handle that there are N scales for N + # shards in a fused module + else: + layer.weight_scale = Parameter( + layer.weight_scale.data, requires_grad=False + ) + if ( + hasattr(self.quant_config, "activation_scheme") + and self.quant_config.activation_scheme == "static" + ) or ( + hasattr(self.quant_config, "linear_activation_scheme") + and self.quant_config.linear_activation_scheme == "static" + ): + layer.input_scale = Parameter( + layer.input_scale.data, requires_grad=False + ) + + # cutlass sgl-kernel and marlin only support per-channel scale + if self.cutlass_fp8_supported or self.use_marlin: + weight = layer.weight + weight_scale = convert_to_channelwise( + layer.weight_scale, layer.logical_widths + ) + else: + # Dequant -> Quant with max scale so we can run per tensor. + weight = layer.weight + weight_scale = layer.weight_scale + # If ROCm, normalize the weights and scales to e4m3fnuz + if _is_fp8_fnuz: + weight, weight_scale, input_scale = ( + normalize_e4m3fn_to_e4m3fnuz( + weight=weight, + weight_scale=weight_scale, + input_scale=layer.input_scale, + ) + ) + if input_scale is not None: + layer.input_scale = Parameter( + input_scale, requires_grad=False + ) + + weight_scale, weight = requantize_with_max_scale( + weight=weight, + weight_scale=weight_scale, + logical_widths=layer.logical_widths, + ) + + # Update layer with new values. + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + if ( + hasattr(self.quant_config, "activation_scheme") + and self.quant_config.activation_scheme == "static" + ) or ( + hasattr(self.quant_config, "linear_activation_scheme") + and self.quant_config.linear_activation_scheme == "static" + ): + layer.input_scale = Parameter( + layer.input_scale.max(), requires_grad=False + ) + + if self.use_marlin: + if self.block_quant: + layer.weight_block_size = self.quant_config.weight_block_size + prepare_fp8_layer_for_marlin(layer, not self.block_quant) + # Activations not quantized for marlin. + del layer.input_scale + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if self.use_marlin: + return apply_fp8_marlin_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias, + ) + + if self.block_quant: + if use_intel_amx_backend(layer): + return torch.ops.sgl_kernel.fp8_scaled_mm_cpu( + x, + layer.weight, + layer.weight_scale_inv, + self.quant_config.weight_block_size, + bias, + x.dtype, + True, # is_vnni + ) + + return self.w8a8_block_fp8_linear( + input=x, + weight=layer.weight, + block_size=self.quant_config.weight_block_size, + weight_scale=layer.weight_scale_inv, + input_scale=None, + bias=bias, + ) + + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + use_per_token_if_dynamic=False, + ) + + +def get_tile_tokens_dim(num_tokens, top_k, num_experts): + # Guess tokens per expert assuming perfect expert distribution first. + num_tokens_per_expert = (num_tokens * top_k) // num_experts + # And pad the number to the next power of 2. + tile_tokens_dim = next_power_of_2(num_tokens_per_expert) + # Cap to 8-64 tokens per CTA tile as it's the range supported by the kernel. + tile_tokens_dim = min(max(tile_tokens_dim, 8), 64) + return tile_tokens_dim + + +class Fp8MoEMethod(FusedMoEMethodBase): + """MoE method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: Fp8Config): + self.quant_config = quant_config + self.block_quant = self.quant_config.weight_block_size is not None + self.cutlass_fp8_supported = cutlass_fp8_supported() + self.use_cutlass_fused_experts_fp8 = ( + get_bool_env_var("SGLANG_CUTLASS_MOE") + and self.cutlass_fp8_supported + and self.block_quant + and (is_sm100_supported() or is_sm90_supported()) + ) + + def create_weights( + self, + layer: Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + if self.quant_config.is_checkpoint_fp8_serialized: + params_dtype = torch.uint32 if _use_hip_int4 else torch.float8_e4m3fn + tp_size = get_tensor_model_parallel_world_size() + if self.block_quant: + block_n, block_k = ( + self.quant_config.weight_block_size[0], + self.quant_config.weight_block_size[1], + ) + # NOTE(HandH1998): To ensure proper alignment of the block-wise quantization scales, the output_size of the weights for both the gate and up layers must be divisible by block_n. + # Required by column parallel or enabling merged weights + if intermediate_size_per_partition % block_n != 0: + raise ValueError( + f"The output_size of gate's and up's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + if tp_size > 1: + # Required by row parallel + if intermediate_size_per_partition % block_k != 0: + raise ValueError( + f"The input_size of down's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + + # WEIGHTS + if _is_hip and _use_hip_int4: + # INT4 MoE weight - INT32 packed + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // 8, + dtype=params_dtype, + ), + requires_grad=False, + ) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // 8, + dtype=params_dtype, + ), + requires_grad=False, + ) + else: + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + if self.block_quant: + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * ((intermediate_size_per_partition + block_n - 1) // block_n), + (hidden_size + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + (hidden_size + block_n - 1) // block_n, + (intermediate_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale_inv", w13_weight_scale) + layer.register_parameter("w2_weight_scale_inv", w2_weight_scale) + assert self.quant_config.activation_scheme == "dynamic" + if self.use_cutlass_fused_experts_fp8: + self.ab_strides1 = torch.full( + (num_experts,), + hidden_size, + device=w13_weight.device, + dtype=torch.int64, + ) + self.c_strides1 = torch.full( + (num_experts,), + 2 * intermediate_size_per_partition, + device=w13_weight.device, + dtype=torch.int64, + ) + self.ab_strides2 = torch.full( + (num_experts,), + intermediate_size_per_partition, + device=w2_weight.device, + dtype=torch.int64, + ) + self.c_strides2 = torch.full( + (num_experts,), + hidden_size, + device=w2_weight.device, + dtype=torch.int64, + ) + self.workspace = torch.empty( + 90000, device=w13_weight.device, dtype=torch.uint8 + ) + self.a_ptr = torch.empty( + num_experts, device=w13_weight.device, dtype=torch.int64 + ) + self.b_ptr = torch.empty( + num_experts, device=w13_weight.device, dtype=torch.int64 + ) + self.out_ptr = torch.empty( + num_experts, device=w13_weight.device, dtype=torch.int64 + ) + self.a_scales_ptr = torch.empty( + num_experts, device=w13_weight.device, dtype=torch.int64 + ) + self.b_scales_ptr = torch.empty( + num_experts, device=w13_weight.device, dtype=torch.int64 + ) + self.expert_offsets = torch.empty( + num_experts + 1, device=w13_weight.device, dtype=torch.int32 + ) + self.problem_sizes1 = torch.empty( + num_experts, 3, device=w13_weight.device, dtype=torch.int32 + ) + self.problem_sizes2 = torch.empty( + num_experts, 3, device=w13_weight.device, dtype=torch.int32 + ) + + else: + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + if _is_hip: # _use_aiter: TODO: add check back after triton kernel + # ROCm - using column scaling, duplicate scaling numbers in case per tensor scaling + w13_weight_scale1 = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * intermediate_size_per_partition, + dtype=torch.float32, + ), + requires_grad=False, + ) + w2_weight_scale1 = torch.nn.Parameter( + torch.ones(num_experts, hidden_size, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale1", w13_weight_scale1) + layer.register_parameter("w2_weight_scale1", w2_weight_scale1) + + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value} + if self.block_quant + else {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value} + ) + # If loading fp8 checkpoint, pass the weight loaders. + # If loading an fp16 checkpoint, do not (we will quantize in + # process_weights_after_loading() + if self.quant_config.is_checkpoint_fp8_serialized: + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + if _is_hip and _use_hip_int4: + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value} + ) + set_weight_attrs(w13_weight_scale1, extra_weight_attrs) + set_weight_attrs(w2_weight_scale1, extra_weight_attrs) + + # INPUT_SCALES + if self.quant_config.activation_scheme == "static": + if not self.quant_config.is_checkpoint_fp8_serialized: + raise ValueError( + "Found static activation scheme for checkpoint that " + "was not serialized fp8." + ) + + w13_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + + else: + layer.w13_input_scale = None + layer.w2_input_scale = None + + def process_weights_after_loading(self, layer: Module) -> None: + if _is_hip and _use_hip_int4: + self.process_weights_hip_int4(layer) + return + + # Block quant doesn't need to process weights after loading + if self.block_quant: + # If ROCm, normalize the weights and scales to e4m3fnuz + if _is_fp8_fnuz: + # activation_scheme: dynamic + w13_weight, w13_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=layer.w13_weight, + weight_scale=layer.w13_weight_scale_inv, + input_scale=None, + ) + w2_weight, w2_weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=layer.w2_weight, + weight_scale=layer.w2_weight_scale_inv, + input_scale=None, + ) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale_inv = torch.nn.Parameter( + w13_weight_scale, requires_grad=False + ) + layer.w13_input_scale = None + layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale_inv = torch.nn.Parameter( + w2_weight_scale, requires_grad=False + ) + layer.w2_input_scale = None + + if _use_aiter: + # Pre-shuffle weights + layer.w13_weight.data = shuffle_weight( + layer.w13_weight.contiguous(), (16, 16) + ) + layer.w2_weight.data = shuffle_weight( + layer.w2_weight.contiguous(), (16, 16) + ) + + if _is_cpu: + assert ( + _is_cpu_amx_available + ), "Fp8MoEMethod on CPU requires that CPU has AMX support" + _amx_process_weight_after_loading(layer, ["w13_weight", "w2_weight"]) + + return + + # If checkpoint is fp16 or bfloat16, quantize in place. + if not self.quant_config.is_checkpoint_fp8_serialized: + # If ROCm, fp8_dtype will be float8_e4m3fnuz (MI300x HW) + w13_weight = torch.empty_like(layer.w13_weight.data, dtype=fp8_dtype) + w2_weight = torch.empty_like(layer.w2_weight.data, dtype=fp8_dtype) + + # Re-initialize w13_scale because we directly quantize + # merged w13 weights and generate a single scaling factor. + layer.w13_weight_scale = torch.nn.Parameter( + torch.ones( + layer.num_local_experts, + dtype=torch.float32, + device=w13_weight.device, + ), + requires_grad=False, + ) + for expert in range(layer.num_local_experts): + w13_weight[expert, :, :], layer.w13_weight_scale[expert] = ( + scaled_fp8_quant(layer.w13_weight.data[expert, :, :]) + ) + w2_weight[expert, :, :], layer.w2_weight_scale[expert] = ( + scaled_fp8_quant(layer.w2_weight.data[expert, :, :]) + ) + layer.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False) + layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + + if _is_hip: + self.process_weights_hip_scale_padding(layer) + return + + # If checkpoint is fp8, we need to handle that the + # MoE kernels require single activation scale and single weight + # scale for w13 per expert. + else: + # Fp8 moe kernels require a single activation scale. + # We take the max of all the scales in case they differ. + if self.quant_config.activation_scheme == "static": + if layer.w13_input_scale is None or layer.w2_input_scale is None: + raise ValueError( + "QuantConfig has static quantization, but found " + "activation scales are None." + ) + if not all_close_1d(layer.w13_input_scale) or not all_close_1d( + layer.w2_input_scale + ): + print_warning_once( + "Found input_scales that are not equal for " + "fp8 MoE layer. Using the maximum across experts " + "for each layer. " + ) + layer.w13_input_scale = torch.nn.Parameter( + layer.w13_input_scale.max(), requires_grad=False + ) + layer.w2_input_scale = torch.nn.Parameter( + layer.w2_input_scale.max(), requires_grad=False + ) + + # If ROCm, normalize the weights and scales to e4m3fnuz + if _is_fp8_fnuz: + # Normalize the weights and scales + w13_weight, w13_weight_scale, w13_input_scale = ( + normalize_e4m3fn_to_e4m3fnuz( + layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale + ) + ) + w2_weight, w2_weight_scale, w2_input_scale = ( + normalize_e4m3fn_to_e4m3fnuz( + layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale + ) + ) + # Reset the parameter + layer.w13_weight = torch.nn.Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter( + w13_weight_scale, requires_grad=False + ) + if w13_input_scale is not None: + layer.w13_input_scale = torch.nn.Parameter( + w13_input_scale, requires_grad=False + ) + layer.w2_weight = torch.nn.Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter( + w2_weight_scale, requires_grad=False + ) + if w2_input_scale is not None: + layer.w2_input_scale = torch.nn.Parameter( + w2_input_scale, requires_grad=False + ) + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max then dequant and requant each expert. + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.num_local_experts): + start = 0 + for shard_id in range(2): + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][start : start + shard_size, :], + layer.w13_weight_scale[expert_id][shard_id], + ) + ( + layer.w13_weight[expert_id][start : start + shard_size, :], + _, + ) = scaled_fp8_quant(dq_weight, max_w13_scales[expert_id]) + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter( + max_w13_scales, requires_grad=False + ) + + if _is_hip: + self.process_weights_hip_scale_padding(layer) + return + + def process_weights_hip_int4(self, layer: Module): + # TODO: _use_aiter: add after triton kernel added + # INT4-FP8 (INT4 MoE Weight, FP8 Compute) + # Weight Permutation + layer.w13_weight = torch.nn.Parameter( + shuffle_weight(layer.w13_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + layer.w2_weight = torch.nn.Parameter( + shuffle_weight(layer.w2_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + + # INT4-FP8 : offset INT4 w13_weight_scale1 to single w13_weight_scale + # Fp8 moe kernel needs single fp8 w13_weight_scale for w13 per expert. + # We won't do requant each expert's fp8 weight (not direct available), + # instead we adjust half of INT4 w13_weight_scale1 numbers + assert layer.w13_weight_scale is not None + shard_size = layer.intermediate_size_per_partition + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + for expert_id in range(layer.num_local_experts): + start = 0 + max_w13_scale_fp8 = max_w13_scales[expert_id] + for shard_id in range(2): + if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8: + int4_rescale = ( + layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8 + ) + layer.w13_weight_scale1[expert_id][ + start : start + shard_size + ] *= int4_rescale + start += shard_size + + layer.w13_weight_scale = torch.nn.Parameter(max_w13_scales, requires_grad=False) + + # special hack to asm_moe, which takes (weight_scale1 * weight_scale) as post GEMM scaling + # optimal design - shall apply per-column weight_scale1 before GEMM, and weight_scale post + for expert_id in range(layer.num_local_experts): + layer.w13_weight_scale1[expert_id] *= max_w13_scales[expert_id] + layer.w2_weight_scale1[expert_id] *= layer.w2_weight_scale[expert_id] + + def process_weights_hip_scale_padding(self, layer: Module): + from sglang.srt.layers.moe.fused_moe_triton.fused_moe import ( + padding_size, # Avoid circular import + ) + + if _use_aiter: + layer.w13_weight = torch.nn.Parameter( + shuffle_weight(layer.w13_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + layer.w2_weight = torch.nn.Parameter( + shuffle_weight(layer.w2_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + + # ROCm (_use_aiter): using column-wise scaling + layer.w13_weight_scale1 *= layer.w13_weight_scale.unsqueeze(-1) + layer.w2_weight_scale1 *= layer.w2_weight_scale.unsqueeze(-1) + elif get_bool_env_var("SGLANG_MOE_PADDING"): + # If ROCm, apply weight padding (min. Mem channel contention) only if set + layer.w13_weight = torch.nn.Parameter( + F.pad(layer.w13_weight.data, (0, padding_size), "constant", 0), + requires_grad=False, + ) + torch.cuda.empty_cache() + layer.w2_weight = torch.nn.Parameter( + F.pad(layer.w2_weight.data, (0, padding_size), "constant", 0), + requires_grad=False, + ) + torch.cuda.empty_cache() + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: DispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + moe_runner_config = self.moe_runner_config + + if use_intel_amx_backend(layer): + from sglang.srt.layers.moe.topk import apply_topk_weights_cpu + + topk_weights, topk_ids, _ = topk_output + x, topk_weights = apply_topk_weights_cpu( + moe_runner_config.apply_router_weight_on_input, topk_weights, x + ) + + output = torch.ops.sgl_kernel.fused_experts_cpu( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + False, # inplace See [Note] inplace should be False in fused_experts. + False, # use_int8_w8a8 + True, # use_fp8_w8a16 + layer.w13_weight_scale_inv, # w1_scale + layer.w2_weight_scale_inv, # w2_scale + self.quant_config.weight_block_size, # block_size + None, # a1_scale + None, # a2_scale + True, # is_vnni + ) + return StandardCombineInput(hidden_states=output) + + if _is_hip: + ret = self.maybe_apply_hip_fused_experts( + layer, + x, + topk_output, + moe_runner_config.activation, + moe_runner_config.no_combine, + ) + if ret is not None: + return StandardCombineInput(hidden_states=ret) + + if self.use_cutlass_fused_experts_fp8: + from sglang.srt.layers.moe.cutlass_moe import cutlass_fused_experts_fp8 + + topk_weights, topk_ids, _ = topk_output + output = cutlass_fused_experts_fp8( + x, + layer.w13_weight.transpose(1, 2), + layer.w2_weight.transpose(1, 2), + layer.w13_weight_scale_inv.transpose(1, 2), + layer.w2_weight_scale_inv.transpose(1, 2), + topk_weights, + topk_ids, + self.ab_strides1, + self.c_strides1, + self.ab_strides2, + self.c_strides2, + self.workspace, + self.a_ptr, + self.b_ptr, + self.out_ptr, + self.a_scales_ptr, + self.b_scales_ptr, + self.expert_offsets, + self.problem_sizes1, + self.problem_sizes2, + use_fp8_blockscale=True, + ) + return StandardCombineInput(hidden_states=output) + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_fp8_w8a8=True, + w13_scale=( + layer.w13_weight_scale_inv + if self.block_quant + else layer.w13_weight_scale + ), + w2_scale=( + layer.w2_weight_scale_inv if self.block_quant else layer.w2_weight_scale + ), + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + block_shape=self.quant_config.weight_block_size, + ) + return self.runner.run(dispatch_output, quant_info) + + def apply_with_router_logits( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> torch.Tensor: + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + activation = self.moe_runner_config.activation + routed_scaling_factor = self.moe_runner_config.routed_scaling_factor + + from flashinfer.fused_moe import trtllm_fp8_block_scale_moe + + from sglang.srt.layers.moe.topk import TopKOutputChecker + + assert TopKOutputChecker.format_is_bypassed(topk_output) + router_logits = topk_output.router_logits + topk_config = topk_output.topk_config + assert ( + activation == "silu" + ), "Only silu is supported for flashinfer blockscale fp8 moe" + a_q, a_sf = per_token_group_quant_fp8(x, self.quant_config.weight_block_size[1]) + # NOTE: scales of hidden states have to be transposed! + a_sf_t = a_sf.t().contiguous() + + assert ( + topk_config.num_expert_group is not None + and topk_config.topk_group is not None + ), "Current trtllm_fp8_block_scale_moe kernel does not support these two arguments as None" + + correction_bias = ( + None + if topk_config.correction_bias is None + else topk_config.correction_bias.to(x.dtype) + ) + + return trtllm_fp8_block_scale_moe( + routing_logits=router_logits.to(torch.float32), + routing_bias=correction_bias, + hidden_states=a_q, + hidden_states_scale=a_sf_t, + gemm1_weights=layer.w13_weight, + gemm1_weights_scale=layer.w13_weight_scale_inv, + gemm2_weights=layer.w2_weight, + gemm2_weights_scale=layer.w2_weight_scale_inv, + num_experts=layer.num_experts, + top_k=topk_config.top_k, + n_group=topk_config.num_expert_group, + topk_group=topk_config.topk_group, + intermediate_size=layer.w2_weight.shape[2], + local_expert_offset=layer.moe_ep_rank * layer.num_local_experts, + local_num_experts=layer.num_local_experts, + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + tile_tokens_dim=get_tile_tokens_dim( + x.shape[0], topk_config.top_k, layer.num_experts + ), + routing_method_type=2, # DeepSeek-styled routing method + use_shuffled_weight=False, + ) + + def maybe_apply_hip_fused_experts( + self, + layer: torch.nn.Module, + x: torch.Tensor, + topk_output: TopKOutput, + activation: str = "silu", + no_combine: bool = False, + ) -> Optional[torch.Tensor]: + topk_weights, topk_ids, _ = topk_output + if _use_hip_int4: + # TODO: add triton kernel and add check _use_aiter + assert not no_combine, f"{no_combine=} is not supported." + return fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + quant_type=QuantType.per_Token, + w1_scale=layer.w13_weight_scale1, + w2_scale=layer.w2_weight_scale1, + activation=( + ActivationType.Silu if activation == "silu" else ActivationType.Gelu + ), + ) + + if _use_aiter: + assert not no_combine, f"{no_combine=} is not supported." + if self.block_quant: + return fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + w1_scale=layer.w13_weight_scale_inv, + w2_scale=layer.w2_weight_scale_inv, + quant_type=QuantType.per_128x128, + activation=( + ActivationType.Silu + if activation == "silu" + else ActivationType.Gelu + ), + expert_mask=None, + ) + else: + return fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + quant_type=QuantType.per_Token, + w1_scale=layer.w13_weight_scale1, + w2_scale=layer.w2_weight_scale1, + activation=( + ActivationType.Silu + if activation == "silu" + else ActivationType.Gelu + ), + ) + return None + + +class Fp8KVCacheMethod(BaseKVCacheMethod): + """ + Supports loading kv-cache scaling factors from FP8 checkpoints. + """ + + def __init__(self, quant_config: Fp8Config): + super().__init__(quant_config) diff --git a/python/sglang/srt/layers/quantization/fp8_kernel.py b/python/sglang/srt/layers/quantization/fp8_kernel.py new file mode 100644 index 000000000..f0512365b --- /dev/null +++ b/python/sglang/srt/layers/quantization/fp8_kernel.py @@ -0,0 +1,1806 @@ +# Copyright 2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import functools +import json +import logging +import os +from functools import lru_cache +from typing import Any, Dict, List, Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.layers.quantization import deep_gemm_wrapper +from sglang.srt.utils import ( + align, + direct_register_custom_op, + get_bool_env_var, + get_device_core_count, + get_device_name, + is_cpu, + is_cuda, + is_hip, + log_info_on_rank0, + supports_custom_op, +) + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_cpu = is_cpu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _is_cuda: + from sgl_kernel import ( + sgl_per_tensor_quant_fp8, + sgl_per_token_group_quant_fp8, + sgl_per_token_quant_fp8, + ) + +if _is_hip: + if _use_aiter: + try: + from aiter import ( # v0.1.3 + dynamic_per_tensor_quant, + dynamic_per_token_scaled_quant, + static_per_tensor_quant, + ) + except ImportError: + raise ImportError("aiter is required when SGLANG_USE_AITER is set to True") + else: + try: + import vllm._C + except ImportError: + raise ImportError("vllm is required when SGLANG_USE_AITER is set to False") + +logger = logging.getLogger(__name__) + + +@lru_cache() +def is_fp8_fnuz() -> bool: + if _is_hip: + # only device 0 is checked, this assumes MI300 platforms are homogeneous + return "gfx94" in torch.cuda.get_device_properties(0).gcnArchName + return False + + +if is_fp8_fnuz(): + fp8_dtype = torch.float8_e4m3fnuz + fp8_max = 224.0 +else: + fp8_dtype = torch.float8_e4m3fn + fp8_max = torch.finfo(fp8_dtype).max +fp8_min = -fp8_max + +if supports_custom_op(): + + def deep_gemm_fp8_fp8_bf16_nt( + A: torch.Tensor, + As: torch.Tensor, + B: torch.Tensor, + Bs: torch.Tensor, + C: torch.Tensor, + ) -> None: + deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C) + + def deep_gemm_fp8_fp8_bf16_nt_fake( + A: torch.Tensor, + As: torch.Tensor, + B: torch.Tensor, + Bs: torch.Tensor, + C: torch.Tensor, + ) -> None: + return + + direct_register_custom_op( + op_name="deep_gemm_fp8_fp8_bf16_nt", + op_func=deep_gemm_fp8_fp8_bf16_nt, + mutates_args=["C"], + fake_impl=deep_gemm_fp8_fp8_bf16_nt_fake, + ) + + +@triton.jit +def _per_token_group_quant_8bit( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + # Stride of input + y_stride, + # Columns of input + N, + # Avoid to divide zero + eps, + # Information for float8 + bit8_min, + bit8_max, + # Meta-parameters + BLOCK: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group quantization on a + tensor. + + This function converts the tensor values into float8 values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id * y_stride + y_q_ptr += g_id * y_stride + y_s_ptr += g_id + + cols = tl.arange(0, BLOCK) # N <= BLOCK + mask = cols < N + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Quant + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / bit8_max + y_s_inv = 1.0 / y_s + y_q = tl.clamp(y * y_s_inv, bit8_min, bit8_max).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + tl.store(y_s_ptr, y_s) + + +@triton.jit +def _per_token_group_quant_8bit_colmajor( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + group_size, + # Num columns of y + y_num_columns, + # Stride from one column to the next of y_s + y_s_col_stride, + # Avoid to divide zero + eps, + # Information for float8 + bit8_min, + bit8_max, + # Meta-parameters + BLOCK: tl.constexpr, + SCALE_UE8M0: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group + quantization on a tensor. + This function converts the tensor values into float8 values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id.to(tl.int64) * group_size + y_q_ptr += g_id.to(tl.int64) * group_size + + # Convert g_id the flattened block coordinate to 2D so we can index + # into the output y_scales matrix + blocks_per_row = y_num_columns // group_size + scale_col = g_id % blocks_per_row + scale_row = g_id // blocks_per_row + y_s_ptr += scale_col * y_s_col_stride + scale_row + + cols = tl.arange(0, BLOCK) # group_size <= BLOCK + mask = cols < group_size + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Quant + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / bit8_max + if SCALE_UE8M0: + y_s = tl.exp2(tl.ceil(tl.log2(tl.abs(y_s)))) + y_q = tl.clamp(y / y_s, bit8_min, bit8_max).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + tl.store(y_s_ptr, y_s) + + +def _per_token_group_quant_8bit_raw( + x: torch.Tensor, + group_size: int, + eps: float = 1e-10, + dtype: torch.dtype = fp8_dtype, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Function to perform per-token-group quantization on an input tensor `x`. + + It converts the tensor values into signed float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + + Args: + x: The input tenosr with ndim >= 2. + group_size: The group size used for quantization. + eps: The minimum to avoid dividing zero. + dtype: The dype of output tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the scaling factor for quantization. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + if _is_hip: + if dtype == torch.int8: + bit8_max = 127.0 + else: + bit8_max = 224.0 + bit8_min = -bit8_max # TODO incorrect for int8 + else: + if dtype == torch.int8: + info = torch.iinfo(dtype) + else: + info = torch.finfo(dtype) + bit8_max = info.max + bit8_min = info.min + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + x_s = create_per_token_group_quant_fp8_output_scale( + x_shape=x.shape, + device=x.device, + group_size=group_size, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=False, + ) + + M = x.numel() // group_size + N = group_size + + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + if column_major_scales: + _per_token_group_quant_8bit_colmajor[(M,)]( + x, + x_q, + x_s, + group_size, + x.shape[1], + x_s.stride(1), + eps, + bit8_min=bit8_min, + bit8_max=bit8_max, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + SCALE_UE8M0=scale_ue8m0, + ) + else: + assert not scale_ue8m0 + _per_token_group_quant_8bit[(M,)]( + x, + x_q, + x_s, + group_size, + N, + eps, + bit8_min=bit8_min, + bit8_max=bit8_max, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + + if scale_ue8m0: + from deep_gemm import transform_sf_into_required_layout + + assert group_size == 128 + x_s = transform_sf_into_required_layout( + x_s, + num_groups=None, + mn=x_q.shape[0], + k=x_q.shape[1], + recipe=(1, group_size, group_size), + is_sfa=True, + ) + + return x_q, x_s + + +# backward compatibility +per_token_group_quant_fp8 = _per_token_group_quant_8bit_raw + + +def _per_token_group_quant_8bit_fuse_silu_and_mul( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + column_major_scales: bool, + scale_tma_aligned: bool, + scale_ue8m0: bool, + masked_m: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + # Another way to implement (can be used in e.g. comparison tests) + # from sgl_kernel import silu_and_mul + # x_after_silu_and_mul = silu_and_mul(x) + # return per_token_group_quant_fp8( + # x_after_silu_and_mul, + # group_size=group_size, + # eps=eps, + # column_major_scales=column_major_scales, + # scale_tma_aligned=scale_tma_aligned, + # scale_ue8m0=scale_ue8m0, + # ) + + from deep_gemm import transform_sf_into_required_layout + + from sglang.srt.layers.moe.ep_moe.kernels import silu_and_mul_masked_post_quant_fwd + + assert column_major_scales + assert scale_tma_aligned + assert scale_ue8m0 + + needs_unsqueeze = x.dim() == 2 + if needs_unsqueeze: + num_tokens, _ = x.shape + x = x.unsqueeze(0) + assert masked_m is None + masked_m = torch.tensor([num_tokens], device=x.device, dtype=torch.int32) + + # Use `zeros` for easier testing + output = torch.zeros( + (*x.shape[:-1], x.shape[-1] // 2), + device=x.device, + dtype=dst_dtype, + ) + # Use `zeros` for easier testing + output_scale_for_kernel = torch.zeros( + (*x.shape[:-1], x.shape[-1] // 2 // group_size), + device=x.device, + dtype=torch.float32, + ) + silu_and_mul_masked_post_quant_fwd( + input=x, + output=output, + output_scale=output_scale_for_kernel, + quant_group_size=group_size, + masked_m=masked_m, + scale_ue8m0=scale_ue8m0, + ) + + assert group_size == 128 + output_scale = transform_sf_into_required_layout( + output_scale_for_kernel, + num_groups=output.shape[0], + mn=output.shape[-2], + k=output.shape[-1], + recipe=(1, group_size, group_size), + is_sfa=True, + ) + + if needs_unsqueeze: + output = output.squeeze(0) + output_scale = output_scale.squeeze(0) + + return output, output_scale + + +def per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + if fuse_silu_and_mul: + return _per_token_group_quant_8bit_fuse_silu_and_mul( + x=x, + group_size=group_size, + dst_dtype=dst_dtype, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + masked_m=masked_m, + ) + else: + return _per_token_group_quant_8bit_raw( + x=x, + group_size=group_size, + eps=eps, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + dtype=dst_dtype, + ) + + +def create_per_token_group_quant_fp8_output_scale( + x_shape, + device, + group_size, + column_major_scales: bool, + scale_tma_aligned: bool, + scale_ue8m0: bool, +): + if scale_ue8m0: + assert column_major_scales and scale_tma_aligned + *x_batch, x_q_mn, x_q_k = x_shape + x_s_mn, x_s_k = x_q_mn, x_q_k // 128 + aligned_mn = align(x_s_mn, 4) + aligned_k = align(x_s_k, 4) + # TODO(FIXME): Fix cuda kernel and recover here to empty. + return torch.empty( + (*x_batch, aligned_k // 4, aligned_mn), + device=device, + dtype=torch.int, + ).transpose(-1, -2)[..., :x_s_mn, :] + elif column_major_scales: + if scale_tma_aligned: + # TODO extract "align" function + # aligned to 4 * sizeof(float) + aligned_size = (x_shape[-2] + 3) // 4 * 4 + return torch.empty( + x_shape[:-2] + (x_shape[-1] // group_size, aligned_size), + device=device, + dtype=torch.float32, + ).permute(-1, -2)[: x_shape[-2], :] + else: + return torch.empty( + (x_shape[-1] // group_size,) + x_shape[:-1], + device=device, + dtype=torch.float32, + ).permute(-1, -2) + else: + return torch.empty( + x_shape[:-1] + (x_shape[-1] // group_size,), + device=device, + dtype=torch.float32, + ) + + +def sglang_per_token_group_quant_fp8( + x: torch.Tensor, + group_size: int, + eps: float = 1e-10, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, +): + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + out_shape = (*x.shape[:-1], x.shape[-1] // (2 if fuse_silu_and_mul else 1)) + + x_q = torch.empty(out_shape, device=x.device, dtype=fp8_dtype) + x_s = create_per_token_group_quant_fp8_output_scale( + x_shape=out_shape, + device=x.device, + group_size=group_size, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + ) + + if x.shape[0] > 0: + sgl_per_token_group_quant_fp8( + x, x_q, x_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0 + ) + + return x_q, x_s + + +# TODO maybe unify int8 and fp8 code later +def sglang_per_token_group_quant_8bit( + x: torch.Tensor, + group_size: int, + dst_dtype: torch.dtype, + eps: float = 1e-10, + column_major_scales: bool = False, + scale_tma_aligned: bool = False, + scale_ue8m0: bool = False, + fuse_silu_and_mul: bool = False, + masked_m: Optional[torch.Tensor] = None, +): + from sglang.srt.layers.quantization.int8_kernel import ( + sglang_per_token_group_quant_int8, + ) + + if dst_dtype == torch.int8: + assert not column_major_scales + assert not scale_tma_aligned + assert not fuse_silu_and_mul + assert masked_m is None + return sglang_per_token_group_quant_int8( + x=x, + group_size=group_size, + eps=eps, + dtype=dst_dtype, + ) + + return sglang_per_token_group_quant_fp8( + x=x, + group_size=group_size, + eps=eps, + column_major_scales=column_major_scales, + scale_tma_aligned=scale_tma_aligned, + scale_ue8m0=scale_ue8m0, + fuse_silu_and_mul=fuse_silu_and_mul, + masked_m=masked_m, + ) + + +def sglang_per_token_quant_fp8( + x: torch.Tensor, + dtype: torch.dtype = fp8_dtype, +): + assert x.is_contiguous(), "`x` is not contiguous" + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + x_s = torch.empty( + x.shape[0], + 1, + device=x.device, + dtype=torch.float32, + ) + + sgl_per_token_quant_fp8(x, x_q, x_s) + + return x_q, x_s + + +@triton.jit +def _static_quant_fp8( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + y_s_repeat_ptr, + # Stride of input + y_stride, + # Columns of input + N, + # Information for float8 + fp8_min, + fp8_max, + # Meta-parameters + BLOCK: tl.constexpr, + REPEAT_SCALE: tl.constexpr, +): + """A Triton-accelerated function to perform quantization using the given scale on a + tensor + + This function converts the tensor values into float8 values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id * y_stride + y_q_ptr += g_id * y_stride + if REPEAT_SCALE: + y_s_repeat_ptr += g_id + + cols = tl.arange(0, BLOCK) # N <= BLOCK + mask = cols < N + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + y_s = tl.load(y_s_ptr).to(tl.float32) + y_s_inv = 1.0 / y_s + y_q = tl.clamp(y * y_s_inv, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + if REPEAT_SCALE: + tl.store(y_s_repeat_ptr, y_s) + + +def static_quant_fp8( + x: torch.Tensor, + x_s: torch.Tensor, + repeat_scale: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Function to perform static quantization using the given scale on an input tensor `x`. + + It converts the tensor values into signed float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + + Args: + x: The input tenosr with ndim >= 2. + x_s: The quantization scale. + repeat_scale: Whether to broadcast per-tensor scale to per-channel scale. + dtype: The dype of output tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the scaling factor for quantization. + """ + assert x.is_contiguous(), "`x` is not contiguous" + assert x_s.numel() == 1, "only supports per-tensor scale" + + x_q = torch.empty_like(x, device=x.device, dtype=fp8_dtype) + M = x.numel() // x.shape[-1] + N = x.shape[-1] + if repeat_scale: + x_s_repeat = torch.empty( + (M, 1), + device=x.device, + dtype=torch.float32, + ) + else: + x_s_repeat = None + + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + _static_quant_fp8[(M,)]( + x, + x_q, + x_s, + x_s_repeat, + N, + N, + fp8_min=fp8_min, + fp8_max=fp8_max, + BLOCK=BLOCK, + REPEAT_SCALE=repeat_scale, + num_warps=num_warps, + num_stages=num_stages, + ) + x_s = x_s_repeat if repeat_scale else x_s + return x_q, x_s + + +@triton.jit +def _w8a8_block_fp8_matmul( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@triton.jit +def _w8a8_block_fp8_matmul_unrolledx4( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + # manually unroll to 4 iterations + UNROLL_FACTOR = 4 + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K * UNROLL_FACTOR)): + # 1st iteration + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR) * BLOCK_SIZE_K, + other=0.0, + ) + + k_start = (k * UNROLL_FACTOR) * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # 2nd iteration + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 1) * BLOCK_SIZE_K, + other=0.0, + ) + + k_start = k_start + BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # 3rd iteration + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 2) * BLOCK_SIZE_K, + other=0.0, + ) + + k_start = k_start + BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # 4th iteration + a = tl.load( + a_ptrs, + mask=offs_k[None, :] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, + other=0.0, + ) + b = tl.load( + b_ptrs, + mask=offs_k[:, None] < K - (k * UNROLL_FACTOR + 3) * BLOCK_SIZE_K, + other=0.0, + ) + + k_start = k_start + BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@functools.lru_cache +def get_w8a8_block_fp8_configs( + N: int, K: int, block_n: int, block_k: int +) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the w8a8 block fp8 kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + + # First look up if an optimized configuration is available in the configs + # directory + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype=fp8_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name + ) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + log_info_on_rank0( + logger, + f"Using configuration from {config_file_path} for W8A8 Block FP8 kernel.", + ) + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ( + "Using default W8A8 Block FP8 kernel config. Performance might be sub-optimal! " + "Config file not found at %s" + ), + config_file_path, + ) + return None + + +def select_w8a8_block_fp8_matmul_kernel(M, N, META): + return _w8a8_block_fp8_matmul + + +if _is_hip: + + def use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): + # Use manually unrolledx4 kernel on AMD GPU when the grid size is small. + # Empirical testing shows the sweet spot lies when it's less than the # of + # compute units available on the device. + num_workgroups = triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv( + N, META["BLOCK_SIZE_N"] + ) + num_workgroups <= get_device_core_count() + + def select_w8a8_block_fp8_matmul_kernel(M, N, META): + if use_w8a8_block_fp8_matmul_unrolledx4(M, N, META): + return _w8a8_block_fp8_matmul_unrolledx4 + else: + return _w8a8_block_fp8_matmul + + +def prepare_block_fp8_matmul_inputs( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> Tuple[int, int, int]: + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + assert A.is_contiguous() + + if As.dtype == torch.float: + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + elif As.dtype == torch.int: + assert ( + triton.cdiv(triton.cdiv(A.shape[-1], block_k), 4) == As.shape[-1] + ), f"{A.shape=} {As.shape=} {block_size=}" + else: + raise NotImplementedError + + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 + assert B.is_contiguous() + assert Bs.ndim == 2 + N, K = B.shape + + if Bs.dtype == torch.float: + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + elif Bs.dtype == torch.int: + assert N == Bs.shape[0], f"{B.shape=} {Bs.shape=} {block_size=}" + assert ( + triton.cdiv(triton.cdiv(K, block_k), 4) == Bs.shape[1] + ), f"{B.shape=} {Bs.shape=} {block_size=}" + else: + raise NotImplementedError + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + return M, N, K, C + + +def w8a8_block_fp8_matmul_deepgemm( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype, +) -> torch.Tensor: + M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) + + # Deepgemm only supports output tensor type as bfloat16 + assert C.dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM + + if supports_custom_op(): + torch.ops.sglang.deep_gemm_fp8_fp8_bf16_nt(A, As, B, Bs, C) + else: + deep_gemm_wrapper.gemm_nt_f8f8bf16((A, As), (B, Bs), C) + + return C + + +def w8a8_block_fp8_matmul_triton( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + + M, N, K, C = prepare_block_fp8_matmul_inputs(A, B, As, Bs, block_size, output_dtype) + + block_n, block_k = block_size + + configs = get_w8a8_block_fp8_configs(N, K, block_size[0], block_size[1]) + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Default config + # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": block_size[0], + "BLOCK_SIZE_K": block_size[1], + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + } + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + kernel = select_w8a8_block_fp8_matmul_kernel(M, N, config) + + kernel[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C + + +# universal entry point, for testing purposes +def w8a8_block_fp8_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + if output_dtype == torch.bfloat16 and deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: + return w8a8_block_fp8_matmul_deepgemm( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + + return w8a8_block_fp8_matmul_triton( + A, B, As, Bs, block_size, output_dtype=output_dtype + ) + + +@triton.jit +def _per_tensor_quant_mla_fp8_stage1( + x_ptr, + x_s_ptr, + head_size, + x_stride_h, + x_stride_s, + eps, + fp8_max, + BLOCK_SIZE: tl.constexpr, +): + seq_id = tl.program_id(0) + head_id = tl.program_id(1) + offset = tl.arange(0, BLOCK_SIZE) + mask = offset < head_size + + x_ptr += head_id * x_stride_h + seq_id * x_stride_s + x = tl.load(x_ptr + offset, mask=mask, other=0.0).to(tl.float32) + _absmax = tl.maximum(tl.max(tl.abs(x)), eps) + + tl.atomic_max(x_s_ptr, _absmax / fp8_max) + + +@triton.jit +def _per_tensor_quant_mla_fp8_stage2( + x_ptr, + x_s_ptr, + x_q_ptr, + num_seq, + head_size, + x_stride_h, + x_stride_s, + fp8_min, + fp8_max, + BLOCK_SIZE: tl.constexpr, +): + seq_id = tl.program_id(0) + head_id = tl.program_id(1) + offset = tl.arange(0, BLOCK_SIZE) + mask = offset < head_size + + x_s = tl.load(x_s_ptr) + x_s_inv = 1.0 / x_s + + x_ptr += head_id * x_stride_h + seq_id * x_stride_s + x_q_ptr += head_id * num_seq * head_size + seq_id * head_size + + x = tl.load(x_ptr + offset, mask=mask, other=0.0).to(tl.float32) + x_q = tl.clamp(x * x_s_inv, fp8_min, fp8_max).to(x_q_ptr.dtype.element_ty) + tl.store(x_q_ptr + offset, x_q, mask=mask) + + +def per_tensor_quant_mla_fp8( + x: torch.Tensor, x_s_out: torch.Tensor, eps: float = 1e-12 +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + This function quantizes input values to float8 values with tensor-wise quantization + and specialized for mla absorbed case. + """ + assert x.dim() == 3, "`x` is not a 3d-tensor" + assert ( + x_s_out.shape == (1,) + and x_s_out.dtype == torch.float32 + and x_s_out.device == x.device + ) + + x_q = x.new_empty(x.size(), dtype=fp8_dtype) + + num_head, num_seq, head_size = x.shape + BLOCK_SIZE = triton.next_power_of_2(head_size) + grid = (num_seq, num_head) + + _per_tensor_quant_mla_fp8_stage1[grid]( + x, + x_s_out, + head_size, + x.stride(0), + x.stride(1), + eps, + fp8_max, + BLOCK_SIZE, + ) + _per_tensor_quant_mla_fp8_stage2[grid]( + x, + x_s_out, + x_q, + num_seq, + head_size, + x.stride(0), + x.stride(1), + fp8_min, + fp8_max, + BLOCK_SIZE, + ) + + return x_q, x_s_out + + +@triton.jit +def _per_token_group_quant_mla_deep_gemm_masked_fp8( + y_ptr, + y_q_ptr, + y_s_ptr, + masked_m_ptr, + group_size, + y_stride_b, + y_stride_t, + y_q_stride_b, + y_q_stride_t, + y_s_stride_b, + y_s_stride_g, + eps, + fp8_min, + fp8_max, + NUM_GROUP: tl.constexpr, + BLOCK: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group + quantization on a tensor for deep_gemm grouped_gemm_masked. + This function converts the tensor values into float8 values. + y and y_q: (b, t, k) + y_s: (b, k//group_size, t) + """ + t_id = tl.program_id(0) + b_id = tl.program_id(1) + + y_ptr += b_id * y_stride_b + t_id * y_stride_t + y_q_ptr += b_id * y_q_stride_b + t_id * y_q_stride_t + y_s_ptr += b_id * y_s_stride_b + t_id + + if t_id == 0: + tl.store(masked_m_ptr + b_id, tl.num_programs(0)) + + cols = tl.arange(0, BLOCK) # group_size <= BLOCK + mask = cols < group_size + + for gid in range(NUM_GROUP): + y = tl.load(y_ptr + gid * group_size + cols, mask=mask, other=0.0).to( + tl.float32 + ) + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / fp8_max + y_q = tl.clamp(y / y_s, fp8_min, fp8_max).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + gid * group_size + cols, y_q, mask=mask) + tl.store(y_s_ptr + gid * y_s_stride_g, y_s) + + +def per_token_group_quant_mla_deep_gemm_masked_fp8( + x: torch.Tensor, + group_size: int = 128, + eps: float = 1e-12, + dtype: torch.dtype = fp8_dtype, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + This function quantizes input values to float8 values with per-token-group-quantization + for deep_gemm grouped_gemm_masked and specialized for mla absorbed case. + """ + assert x.dim() == 3, "`x` is not a 3d-tensor" + + b, m, k = x.shape + aligned_m = (m + 255) // 256 * 256 # 256 is the max block_m of the gemm kernel + num_tiles_k = k // group_size + assert num_tiles_k * group_size == k, f"k % {group_size} must be zero" + + x_q = x.new_empty((b, aligned_m, k), dtype=dtype) + x_s = x.new_empty((b, num_tiles_k, aligned_m), dtype=torch.float32) + masked_m = x.new_empty((b,), dtype=torch.int32) + + BLOCK_SIZE = triton.next_power_of_2(group_size) + grid = (m, b) + + _per_token_group_quant_mla_deep_gemm_masked_fp8[grid]( + x, + x_q, + x_s, + masked_m, + group_size, + x.stride(0), + x.stride(1), + x_q.stride(0), + x_q.stride(1), + x_s.stride(0), + x_s.stride(1), + eps, + -fp8_max, + fp8_max, + num_tiles_k, + BLOCK_SIZE, + ) + + return x_q, x_s.transpose(1, 2), masked_m, m, aligned_m + + +""" +Quantize input tensor to FP8 (8-bit floating point) format. + +Args: + input (torch.Tensor): Input tensor to be quantized + scale (Optional[torch.Tensor]): Pre-computed scaling factor for static quantization. + If None, scales will be computed dynamically. + num_token_padding (Optional[int]): If specified, pad the first dimension + of the output to at least this value. + use_per_token_if_dynamic (bool): When using dynamic scaling (scale=None), + determines the quantization granularity: + - True: compute scale per token + - False: compute single scale per tensor + +Returns: + Tuple[torch.Tensor, torch.Tensor]: A tuple containing: + - quantized_tensor: The FP8 quantized version of input + - scale_tensor: The scaling factors used for quantization + +Raises: + AssertionError: If input is not 2D or if static scale's numel != 1 +""" +if _is_hip: + + def scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, + num_token_padding: Optional[int] = None, + use_per_token_if_dynamic: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + assert input.ndim == 2, f"Expected 2D input tensor, got {input.ndim}D" + shape = input.shape + if num_token_padding: + shape = (max(num_token_padding, input.shape[0]), shape[1]) + output = torch.empty(shape, device=input.device, dtype=fp8_dtype) + + if scale is None: + # Dynamic scaling + if use_per_token_if_dynamic: + scale = torch.empty( + (shape[0], 1), device=input.device, dtype=torch.float32 + ) + if _use_aiter: + dynamic_per_token_scaled_quant(output, input, scale) + else: + torch.ops._C.dynamic_per_token_scaled_fp8_quant( + output, input.contiguous(), scale, None + ) + else: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + if _use_aiter: + dynamic_per_tensor_quant(output, input, scale) + else: + torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale) + else: + # Static scaling + assert ( + scale.numel() == 1 + ), f"Expected scalar scale, got numel={scale.numel()}" + if _use_aiter: + static_per_tensor_quant(output, input, scale) + else: + torch.ops._C.static_scaled_fp8_quant(output, input, scale) + + return output, scale + +else: + + def scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, + num_token_padding: Optional[int] = None, + use_per_token_if_dynamic: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + + assert input.ndim == 2, f"Expected 2D input tensor, got {input.ndim}D" + shape = input.shape + if num_token_padding: + shape = (max(num_token_padding, input.shape[0]), shape[1]) + output = torch.empty(shape, device=input.device, dtype=fp8_dtype) + + if scale is None: + # Dynamic scaling + if use_per_token_if_dynamic: + scale = torch.empty( + (shape[0], 1), device=input.device, dtype=torch.float32 + ) + sgl_per_token_quant_fp8(input, output, scale) + else: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + sgl_per_tensor_quant_fp8( + input, output, scale, is_static=False + ) # False for dynamic + else: + # Static scaling + assert ( + scale.numel() == 1 + ), f"Expected scalar scale, got numel={scale.numel()}" + sgl_per_tensor_quant_fp8( + input, output, scale, is_static=True + ) # True for static + + return output, scale + + +fp8_autotune = triton.autotune( + configs=[ + triton.Config({"BLOCK_M": block_m}, num_warps=num_warps) + for block_m in [16, 32, 64, 128] + for num_warps in [2, 4, 8] + ], + key=["K", "BLOCK_K", "M_ALIGNMENT"], +) + + +@triton.jit +def _per_token_group_quant_fp8_hopper_moe_mn_major( + a, # (M, K):(K, 1) + expert_offsets, # (num_experts,) + problem_sizes, # (num_experts, 3) + a_fp8, # (M, K):(K, 1) + sfa, # (M, k) + K: tl.constexpr, + BLOCK_K: tl.constexpr, + M_ALIGNMENT: tl.constexpr, + BLOCK_M: tl.constexpr, # tune +): + k_offset = tl.program_id(0) + expert_id = tl.program_id(1) + + m = tl.load(problem_sizes + expert_id * 3) + current_expert_offset = tl.load(expert_offsets + expert_id).to(tl.int64) + tl.multiple_of(m, M_ALIGNMENT) + tl.multiple_of(current_expert_offset, M_ALIGNMENT) + + coord_k = k_offset * BLOCK_K + tl.arange(0, BLOCK_K) + for i in tl.range(tl.cdiv(m, BLOCK_M)): + coord_m = i * BLOCK_M + tl.arange(0, BLOCK_M) + a_ptrs = a + current_expert_offset * K + coord_m[:, None] * K + coord_k[None, :] + a_mask = (coord_m < m)[:, None] & (coord_k < K)[None, :] + + inp = tl.load(a_ptrs, mask=a_mask).to(tl.float32) # [BLOCK_M, BLOCK_K] + inp_amax = tl.max(tl.abs(inp), axis=1) # [BLOCK_M,] + inp_amax = tl.clamp(inp_amax, min=1e-4, max=float("inf")) + inp_fp8 = (inp * (448.0 / inp_amax[:, None])).to(tl.float8e4nv) + + # Store fp8 + a_fp8_ptrs = ( + a_fp8 + current_expert_offset * K + coord_m[:, None] * K + coord_k[None, :] + ) + tl.store(a_fp8_ptrs, inp_fp8, mask=a_mask) + + # Store sfa + k = tl.cdiv(K, BLOCK_K) + sfa_ptrs = ( + sfa + current_expert_offset * k + k_offset * m + coord_m + ) # MN-Major with sfa + tl.store(sfa_ptrs, inp_amax / 448.0, mask=coord_m < m) + + +if not _is_cpu: + _per_token_group_quant_fp8_hopper_moe_mn_major = fp8_autotune( + _per_token_group_quant_fp8_hopper_moe_mn_major + ) + + +def per_token_group_quant_fp8_hopper_moe_mn_major( + A: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes: torch.Tensor, + group_size: int, + expert_tokens_alignment: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + assert A.dim() == 2 + assert A.is_contiguous(), "`A` is not contiguous" + assert ( + A.shape[-1] % group_size == 0 + ), "the last dimension of `A` cannot be divisible by `group_size`" + + a_q = torch.empty_like(A, device=A.device, dtype=fp8_dtype) + M, K = A.shape[0], A.shape[1] + k = K // group_size + sfa = torch.empty((M, k), device=A.device, dtype=torch.float32) + num_experts = problem_sizes.shape[0] + grid = (k, num_experts) + _per_token_group_quant_fp8_hopper_moe_mn_major[grid]( + A, + expert_offsets, + problem_sizes, + a_q, + sfa, + K, + group_size, + expert_tokens_alignment, + ) + return a_q, sfa + + +@triton.jit +def _per_group_transpose( + data_ptr: torch.Tensor, + trans_data_ptr: torch.Tensor, + expert_offsets: torch.Tensor, + k: int, + M_ALIGNMENT: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, +): + expert_id = tl.program_id(0) + m_id = tl.program_id(1) + k_id = tl.program_id(2) + + curr_expert_offset = tl.load(expert_offsets + expert_id) + next_expert_offset = tl.load(expert_offsets + expert_id + 1) + num_tokens_of_expert = next_expert_offset - curr_expert_offset + tl.multiple_of(curr_expert_offset, M_ALIGNMENT) + tl.multiple_of(next_expert_offset, M_ALIGNMENT) + + data_start_ptr = data_ptr + curr_expert_offset * k + trans_data_start_ptr = trans_data_ptr + curr_expert_offset * k + + k_coord = k_id * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + k_mask = k_coord < k + for start_m in tl.range(0, num_tokens_of_expert, BLOCK_SIZE_M * tl.num_programs(1)): + m_coord = start_m + m_id * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + m_mask = m_coord < num_tokens_of_expert + off = m_coord[:, None] * k + k_coord[None, :] + trans_off = m_coord[:, None] + k_coord[None, :] * num_tokens_of_expert + mask = m_mask[:, None] & k_mask[None, :] + + data = tl.load(data_start_ptr + off, mask=mask) + tl.store(trans_data_start_ptr + trans_off, data, mask=mask) + + +def per_group_transpose( + a: torch.Tensor, + expert_offsets: torch.Tensor, + M_ALIGNMENT: int = 1, +) -> torch.Tensor: + assert a.dim() == 2 + assert a.is_contiguous(), "`a` is not contiguous" + + m, k = a.size() + trans_a = torch.empty_like(a) + num_experts = expert_offsets.size(0) - 1 + + grid = lambda META: ( + num_experts, + triton.cdiv((m + num_experts - 1) // num_experts, META["BLOCK_SIZE_M"]), + triton.cdiv(k, META["BLOCK_SIZE_K"]), + ) + _per_group_transpose[grid]( + a, trans_a, expert_offsets, k, M_ALIGNMENT, BLOCK_SIZE_M=16, BLOCK_SIZE_K=8 + ) + return trans_a + + +def is_weak_contiguous(x: torch.Tensor): + strides = x.stride() + sizes = x.shape + is_not_transpose = strides[0] == 1 and (strides[1] >= max(1, sizes[0])) + is_transpose = strides[1] == 1 and (strides[0] >= max(1, sizes[1])) + return is_transpose or is_not_transpose + + +@triton.jit +def scaled_mm_kernel( + a_ptr, + b_ptr, + scale_a_ptr, + scale_b_ptr, + c_ptr, + bias_ptr, + M, + N, + K, + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + ACCUMULATOR_DTYPE: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + BLOCK_SIZE_SCALE_A: tl.constexpr, + BLOCK_SIZE_SCALE_B: tl.constexpr, +): + pid = tl.program_id(axis=0) + + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + + pid_m = pid // num_pid_n + pid_n = pid % num_pid_n + + accumulator_dtype = ACCUMULATOR_DTYPE + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=accumulator_dtype) + + # NOTE: Some tensor inputs are so large, they will cause int32 overflow + # so it is necessary to use tl.int64 for all the offsets, else SEGV will + # eventually occur. + + # Offsets and masks. + offsets_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + masks_am = offsets_am < M + + offsets_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) + masks_bn = offsets_bn < N + + offsets_k = tl.arange(0, BLOCK_SIZE_K).to(tl.int64) + offsets_a = stride_am * offsets_am[:, None] + stride_ak * offsets_k[None, :] + offsets_b = stride_bk * offsets_k[:, None] + stride_bn * offsets_bn[None, :] + + # NOTE: BLOCK_SIZE_SCALE_A could be 1 or BLOCK_SIZE_M, so need to create + # appropriate offsets and masks for each case. Same goes for + # BLOCK_SIZE_SCALE_B. + offsets_scale_am = ( + tl.arange(0, BLOCK_SIZE_SCALE_A) + + (BLOCK_SIZE_SCALE_A > 1) * pid_m * BLOCK_SIZE_M + ) + masks_scale_am = offsets_scale_am < M + + offsets_scale_bn = ( + tl.arange(0, BLOCK_SIZE_SCALE_B) + + (BLOCK_SIZE_SCALE_B > 1) * pid_n * BLOCK_SIZE_N + ) + masks_scale_bn = offsets_scale_bn < N + + a_ptrs = a_ptr + offsets_a + b_ptrs = b_ptr + offsets_b + + scale_a_ptrs = scale_a_ptr + offsets_scale_am + scale_b_ptrs = scale_b_ptr + offsets_scale_bn + + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + masks_k = offsets_k < K + masks_a = masks_am[:, None] & masks_k[None, :] + a = tl.load(a_ptrs, mask=masks_a) + + masks_b = masks_k[:, None] & masks_bn[None, :] + b = tl.load(b_ptrs, mask=masks_b) + + # Accumulate results. + accumulator = tl.dot(a, b, accumulator, out_dtype=accumulator_dtype) + + offsets_k += BLOCK_SIZE_K + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + # Apply scale at end. + masks_scale_a = masks_scale_am[:, None] & (tl.arange(0, 1) < 1)[:, None] + scale_a = tl.load(scale_a_ptrs[:, None], masks_scale_a) + # Need to broadcast to the appropriate size, if scale_a is already + # (BLOCK_SIZE_M, 1) then it will broadcast to its own shape. Same goes + # for scale_b below. + scale_a = scale_a.broadcast_to((BLOCK_SIZE_M, 1)) + accumulator = scale_a * accumulator.to(tl.float32) + + masks_scale_b = masks_scale_bn[:, None] & (tl.arange(0, 1) < 1)[None, :] + scale_b = tl.load(scale_b_ptrs[:, None], masks_scale_b) + scale_b = scale_b.broadcast_to((BLOCK_SIZE_N, 1)) + accumulator = scale_b.T * accumulator.to(tl.float32) + + # Convert to output format. + c = accumulator.to(c_ptr.type.element_ty) + + # Add bias, it's already in output format, so add it after conversion. + if bias_ptr: + offsets_bias = offsets_bn + bias_ptrs = bias_ptr + offsets_bias + bias_mask = offsets_bias < N + bias = tl.load(bias_ptrs, bias_mask) + c += bias + + # Save output + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int64) + offs_cm = offs_cm.to(tl.int64) + offs_cn = offs_cn.to(tl.int64) + c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + + tl.store(c_ptrs, c, mask=c_mask) + + +# input - [M, K] +# weight - [K, N] +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py +def triton_scaled_mm( + input: torch.Tensor, + weight: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: type[torch.dtype], + bias: Optional[torch.Tensor] = None, + block_size_m: int = 32, + block_size_n: int = 32, + block_size_k: int = 32, + use_heuristic=True, +) -> torch.Tensor: + M, K = input.shape + N = weight.shape[1] + + assert N > 0 and K > 0 and M > 0 + assert weight.shape[0] == K + assert input.dtype == weight.dtype + + scale_a = scale_a.reshape(-1, 1) if scale_a.dim() <= 1 else scale_a + scale_b = scale_b.reshape(-1, 1) if scale_b.dim() <= 1 else scale_b + + assert scale_a.dtype == scale_b.dtype and scale_a.is_floating_point() + assert scale_a.shape[1] == 1 and (scale_a.shape[0] == 1 or scale_a.shape[0] == M) + assert scale_b.shape[1] == 1 and (scale_b.shape[0] == 1 or scale_b.shape[0] == N) + assert out_dtype.is_floating_point + assert bias is None or bias.is_floating_point() + assert is_weak_contiguous(input) + assert is_weak_contiguous(weight) + + grid = lambda META: ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + result = torch.empty((M, N), dtype=out_dtype, device=input.device) + + has_scalar = lambda x: x.shape[0] == 1 and x.shape[1] == 1 + + if use_heuristic: + is_small_N = N < 8192 + next_power_of_2_M = max(32, triton.next_power_of_2(M)) + if next_power_of_2_M <= 32: + tile_shape = (64, 64, 256) if is_small_N else (64, 128, 256) + elif next_power_of_2_M <= 64: + tile_shape = (64, 64, 256) + elif next_power_of_2_M <= 128: + tile_shape = (64, 128, 128) + else: + tile_shape = (128, 128, 128) + + block_size_m, block_size_n, block_size_k = tile_shape + + block_size_sa = 1 if has_scalar(scale_a) else block_size_m + block_size_sb = 1 if has_scalar(scale_b) else block_size_n + + accumulator_dtype = tl.float32 if input.is_floating_point() else tl.int32 + + # A = input, B = weight, C = result + # A = M x K, B = K x N, C = M x N + scaled_mm_kernel[grid]( + input, + weight, + scale_a, + scale_b, + result, + bias, + M, + N, + K, + input.stride(0), + input.stride(1), + weight.stride(0), + weight.stride(1), + result.stride(0), + result.stride(1), + accumulator_dtype, + BLOCK_SIZE_M=block_size_m, + BLOCK_SIZE_N=block_size_n, + BLOCK_SIZE_K=block_size_k, + BLOCK_SIZE_SCALE_A=block_size_sa, + BLOCK_SIZE_SCALE_B=block_size_sb, + ) + + return result.to(out_dtype) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py new file mode 100644 index 000000000..998423b86 --- /dev/null +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -0,0 +1,829 @@ +from typing import Callable, List, Optional, Tuple + +import torch + +from sglang.srt.layers.quantization import deep_gemm_wrapper +from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 +from sglang.srt.layers.quantization.mxfp4_tensor import MXFP4QuantizeUtil +from sglang.srt.utils import is_sm100_supported + +try: + from vllm import _custom_ops as ops + + VLLM_AVAILABLE = True +except ImportError: + VLLM_AVAILABLE = False + +from sglang.srt.layers.quantization.fp8_kernel import ( + fp8_dtype, + fp8_max, + is_fp8_fnuz, + per_token_group_quant_fp8, + scaled_fp8_quant, + sglang_per_token_quant_fp8, + static_quant_fp8, + triton_scaled_mm, + w8a8_block_fp8_matmul_deepgemm, + w8a8_block_fp8_matmul_triton, +) +from sglang.srt.utils import ( + align, + ceil_div, + get_bool_env_var, + get_cuda_version, + get_device_capability, + is_cuda, + is_flashinfer_available, + is_hip, +) + +_is_hip = is_hip() +_is_cuda = is_cuda() +_is_fp8_fnuz = is_fp8_fnuz() + +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _use_aiter: + import aiter + from aiter import gemm_a8w8_blockscale, gemm_a8w8_bpreshuffle, get_hip_quant + + aiter_per1x128_quant = get_hip_quant(aiter.QuantType.per_1x128) + +if _is_cuda: + from sgl_kernel import fp8_blockwise_scaled_mm, fp8_scaled_mm + +use_vllm_cutlass_w8a8_fp8_kernel = get_bool_env_var("USE_VLLM_CUTLASS_W8A8_FP8_KERNEL") +use_triton_w8a8_fp8_kernel = get_bool_env_var("USE_TRITON_W8A8_FP8_KERNEL") + +# Input scaling factors are no longer optional in _scaled_mm starting +# from pytorch 2.5. Allocating a dummy tensor to pass as input_scale +TORCH_DEVICE_IDENTITY = None + + +def use_rowwise_torch_scaled_mm(): + _TORCH_VERSION = torch.__version__.split("+")[0] + try: + _TORCH_VERSION_TUPLE = tuple(map(int, _TORCH_VERSION.split(".")[:3])) + except ValueError: + _TORCH_VERSION_TUPLE = (0, 0, 0) + if _is_hip: + # The condition to determine if it is on a platform that supports + # torch._scaled_mm rowwise feature. + # The condition is determined once as the operations + # are time consuming. + return get_device_capability() >= (9, 4) and _TORCH_VERSION_TUPLE >= (2, 7, 0) + return False + + +USE_ROWWISE_TORCH_SCALED_MM = use_rowwise_torch_scaled_mm() + + +def cutlass_fp8_supported(): + if not _is_cuda: + return False + major, minor = get_device_capability() + cuda_version = get_cuda_version() + if major >= 9: + return cuda_version >= (12, 0) + elif major == 8 and minor == 9: + return cuda_version >= (12, 4) + return False + + +def normalize_e4m3fn_to_e4m3fnuz( + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + assert weight.dtype == torch.float8_e4m3fn + # The bits pattern 10000000(-128) represents zero in e4m3fn + # but NaN in e4m3fnuz. So here we set it to 0. + # https://onnx.ai/onnx/technical/float8.html + weight_as_int8 = weight.view(torch.int8) + ROCM_FP8_NAN_AS_INT = -128 + weight_as_int8[weight_as_int8 == ROCM_FP8_NAN_AS_INT] = 0 + weight = weight_as_int8.view(torch.float8_e4m3fnuz) + + # For the same bits representation, e4m3fnuz value is half of + # the e4m3fn value, so we should double the scaling factor to + # get the same dequantized value. + # https://onnx.ai/onnx/technical/float8.html + weight_scale = weight_scale * 2.0 + if input_scale is not None: + input_scale = input_scale * 2.0 + return weight, weight_scale, input_scale + + +# TODO(ch-wan): define these backends in --moe-runner-backend +def cutlass_block_fp8_supported() -> bool: + if not get_bool_env_var("SGLANG_SUPPORT_CUTLASS_BLOCK_FP8"): + return False + if _is_cuda: + major, minor = torch.cuda.get_device_capability() + sm_version = major * 10 + minor + cuda_version = tuple(map(int, torch.version.cuda.split("."))) + if cuda_version >= (12, 0) and sm_version >= 90: + return True + return False + + +CUTLASS_BLOCK_FP8_SUPPORTED = cutlass_block_fp8_supported() +ENABLE_FLASHINFER_GEMM = ( + get_bool_env_var("SGLANG_ENABLE_FLASHINFER_GEMM") + and is_sm100_supported() + and is_flashinfer_available() +) +if ENABLE_FLASHINFER_GEMM: + from flashinfer.gemm import gemm_fp8_nt_groupwise + + +def dispatch_w8a8_block_fp8_linear() -> Callable: + if ENABLE_FLASHINFER_GEMM: + return flashinfer_gemm_w8a8_block_fp8_linear + elif CUTLASS_BLOCK_FP8_SUPPORTED: + return cutlass_w8a8_block_fp8_linear_with_fallback + elif _use_aiter: + return aiter_w8a8_block_fp8_linear + elif deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM: + return deepgemm_w8a8_block_fp8_linear_with_fallback + else: + return triton_w8a8_block_fp8_linear + + +def flashinfer_gemm_w8a8_block_fp8_linear( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = sglang_per_token_group_quant_fp8( + input_2d, block_size[1], column_major_scales=True + ) + # TRTLLM requires column-major scaling factors + output = gemm_fp8_nt_groupwise( + q_input, + weight, + x_scale, + weight_scale, + out_dtype=input_2d.dtype, + backend="trtllm", + ) + + if bias is not None: + output += bias + + return output.to(dtype=input_2d.dtype).view(*output_shape) + + +def cutlass_w8a8_block_fp8_linear_with_fallback( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + + # TODO: add more robust shape check here + shape_supported = weight.shape[0] % 128 == 0 and weight.shape[1] % 128 == 0 + + if not shape_supported: + # fallback to triton + return triton_w8a8_block_fp8_linear( + input, weight, block_size, weight_scale, input_scale, bias + ) + + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = per_token_group_quant_fp8( + input_2d, block_size[1], column_major_scales=True + ) + output = fp8_blockwise_scaled_mm( + q_input, weight.T, x_scale, weight_scale.T, out_dtype=input_2d.dtype + ) + if bias is not None: + output += bias + return output.to(dtype=input_2d.dtype).view(*output_shape) + + +def deepgemm_w8a8_block_fp8_linear_with_fallback( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + + output_dtype = input.dtype + dtype_supported = output_dtype == torch.bfloat16 + + # TODO: https://github.com/sgl-project/sglang/pull/6890#issuecomment-2943395737 + shape_supported = weight.shape[0] % 64 == 0 and weight.shape[1] % 128 == 0 + + if not (shape_supported and dtype_supported): + # fall back to triton + return triton_w8a8_block_fp8_linear( + input, weight, block_size, weight_scale, input_scale, bias + ) + + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = sglang_per_token_group_quant_fp8( + input_2d, + block_size[1], + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0, + ) + + output = w8a8_block_fp8_matmul_deepgemm( + q_input, weight, x_scale, weight_scale, block_size, output_dtype=output_dtype + ) + if bias is not None: + output += bias + return output.to(dtype=output_dtype).view(*output_shape) + + +def aiter_w8a8_block_fp8_linear( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = aiter_per1x128_quant(input_2d, quant_dtype=aiter.dtypes.fp8) + output = gemm_a8w8_blockscale( + q_input, weight, x_scale, weight_scale, dtype=input.dtype + ) + + if bias is not None: + output += bias + + return output.to(dtype=input_2d.dtype).view(*output_shape) + + +def triton_w8a8_block_fp8_linear( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = per_token_group_quant_fp8( + input_2d, block_size[1], column_major_scales=False + ) + output = w8a8_block_fp8_matmul_triton( + q_input, weight, x_scale, weight_scale, block_size, output_dtype=input_2d.dtype + ) + if bias is not None: + output += bias + return output.to(dtype=input_2d.dtype).view(*output_shape) + + +def dequant_mxfp4( + w_block: torch.Tensor, + w_scale: torch.Tensor, + out_dtype, +) -> torch.Tensor: + """ + :param w_block: (batch, n, k, 16), uint8, pack two mxfp4 into one byte + :param w_scale: (batch, n, k), uint8 + :return: (batch, n, k * 32), float32 + """ + + assert w_block.dtype == torch.uint8 + assert w_scale.dtype == torch.uint8 + + batch, n, k, pack_dim = w_block.shape + batch_, n_, k_ = w_scale.shape + assert pack_dim == 16 + assert batch == batch_ + assert n == n_ + assert k == k_ + + out_raw = MXFP4QuantizeUtil.dequantize( + quantized_data=w_block, scale=w_scale, dtype=out_dtype, block_sizes=[32] + ) + return out_raw.reshape(batch, n, k * 32) + + +def input_to_float8( + x: torch.Tensor, dtype: torch.dtype = fp8_dtype +) -> Tuple[torch.Tensor, torch.Tensor]: + """This function quantizes input values to float8 values with tensor-wise quantization.""" + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).float().clamp(min=1e-12) + + if _is_fp8_fnuz: + dtype = fp8_dtype + fp_max = fp8_max + else: + finfo = torch.finfo(dtype) + fp_max = finfo.max + + scale = fp_max / amax + x_scl_sat = (x.float() * scale).clamp(min=-fp_max, max=fp_max) + return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() + + +def block_quant_to_tensor_quant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], +) -> Tuple[torch.Tensor, torch.Tensor]: + """This function converts block-wise quantization to tensor-wise quantization. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The outputs are tensor-wise quantization tensor and tensor-wise quantization scale. + Note only float8 is supported for now. + """ + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = x_q_block.to(torch.float32) + + x_dq_block_tiles = [ + [ + x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + + for i in range(k_tiles): + for j in range(n_tiles): + x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i] + + x_q_tensor, scale = ( + scaled_fp8_quant(x_dq_block) + if _is_cuda + else input_to_float8(x_dq_block, dtype=x_q_block.dtype) + ) + return x_q_tensor, scale + + +def block_quant_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], + dtype: torch.dtype, +) -> torch.Tensor: + """This function converts block-wise quantization to unquantized. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The output is an unquantized tensor with dtype. + """ + block_n, block_k = block_size[0], block_size[1] + *_, n, k = x_q_block.shape + + # ... n_scale k_scale -> ... (n_scale block_n) (k_scale block_k) + x_scale_repeat = x_s.repeat_interleave(block_n, dim=-2).repeat_interleave( + block_k, dim=-1 + ) + x_scale_repeat = x_scale_repeat[..., :n, :k] + + return (x_q_block.to(torch.float32) * x_scale_repeat).to(dtype) + + +def requant_weight_ue8m0_inplace(weight, weight_scale_inv, weight_block_size): + assert isinstance(weight, torch.nn.Parameter) + assert isinstance(weight_scale_inv, torch.nn.Parameter) + weight.data, weight_scale_inv.data = _requant_weight_ue8m0( + weight, weight_scale_inv, weight_block_size + ) + + +def _requant_weight_ue8m0( + weight: torch.Tensor, + weight_scale_inv: torch.Tensor, + weight_block_size: List[int], +): + assert weight_block_size == [128, 128] + + *_, n, k = weight.shape + + weight_dequant = block_quant_dequant( + weight, + weight_scale_inv, + weight_block_size, + torch.bfloat16, + ) + + weight_dequant_flat = weight_dequant.view((-1, k)) + out_w_flat, out_s_flat = per_block_cast_to_fp8(weight_dequant_flat) + + out_w = out_w_flat.view(weight.shape) + out_s = out_s_flat.view(weight_scale_inv.shape) + + # NOTE copy and modified from DeepGEMM + def _transform_scale(sf, mn: int): + import deep_gemm.utils.layout + + sf = sf.index_select(-2, torch.arange(mn, device=sf.device) // 128) + sf = deep_gemm.utils.layout.get_mn_major_tma_aligned_packed_ue8m0_tensor(sf) + return sf + + out_s = _transform_scale(out_s, mn=out_w.shape[-2]) + + return out_w, out_s + + +# COPIED FROM DeepGEMM +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (align(m, 128), align(n, 128)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + sf = ceil_to_ue8m0(x_amax / 448.0) + x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view( + x_view.size(0), x_view.size(2) + ) + + +# COPIED FROM DeepGEMM +def ceil_to_ue8m0(x: torch.Tensor): + return torch.pow(2.0, torch.ceil(torch.log2(x.abs()))) + + +def channel_quant_to_tensor_quant( + x_q_channel: torch.Tensor, + x_s: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + x_dq_channel = x_q_channel.to(torch.float32) * x_s + x_q_tensor, scale = ( + scaled_fp8_quant(x_dq_channel) + if _is_cuda + else input_to_float8(x_dq_channel, dtype=x_q_channel.dtype) + ) + return x_q_tensor, scale + + +def _process_scaled_mm_output(output, input_2d_shape, output_shape): + if type(output) is tuple and len(output) == 2: + output = output[0] + return torch.narrow(output, 0, 0, input_2d_shape[0]).view(*output_shape) + + +def _apply_fallback_scaled_mm( + qinput, + weight, + x_scale, + weight_scale, + input_2d_shape, + output_shape, + bias, + input_dtype, +): + global TORCH_DEVICE_IDENTITY + if TORCH_DEVICE_IDENTITY is None: + TORCH_DEVICE_IDENTITY = torch.ones(1, dtype=torch.float32, device=weight.device) + + output = torch._scaled_mm( + qinput, + weight, + scale_a=TORCH_DEVICE_IDENTITY, + scale_b=TORCH_DEVICE_IDENTITY, + out_dtype=torch.float32, + ) + + output = _process_scaled_mm_output(output, input_2d_shape, output_shape) + x_scale = torch.narrow(x_scale, 0, 0, input_2d_shape[0]) + + output = output * x_scale * weight_scale.t() + if bias is not None: + output = output + bias + return output.to(dtype=input_dtype) + + +def apply_fp8_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + input_scale_ub: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, + cutlass_fp8_supported: bool = cutlass_fp8_supported(), + use_per_token_if_dynamic: bool = False, + pad_output: Optional[bool] = None, + compressed_tensor_quant: bool = False, +) -> torch.Tensor: + # Note: we pad the input because torch._scaled_mm is more performant + # for matrices with batch dimension > 16. + # This could change in the future. + # We also don't pad when using torch.compile, + # as it breaks with dynamic shapes. + if pad_output is None: + pad_output = ( + not get_bool_env_var("SGLANG_ENABLE_TORCH_COMPILE") + and not cutlass_fp8_supported + ) + output_padding = 17 if pad_output else None + + # View input as 2D matrix for fp8 methods + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[1]] + + if compressed_tensor_quant: + # cutlass_scaled_mm supports per tensor/channel W and per tensor/token A + # for sgl-kernel fp8_scaled_mm, it support per channel W now + if cutlass_fp8_supported and weight_scale.numel() == weight.shape[1]: + qinput, x_scale = scaled_fp8_quant( + input_2d, + input_scale, + use_per_token_if_dynamic=use_per_token_if_dynamic, + ) + + # Fused GEMM_DQ + if VLLM_AVAILABLE and use_vllm_cutlass_w8a8_fp8_kernel: + # Fall back to vllm cutlass w8a8 fp8 kernel + output = ops.cutlass_scaled_mm( + qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias, + ) + else: + assert ( + weight_scale.numel() == weight.shape[1] + ), "cutlass w8a8 fp8 sgl-kernel only supports per-channel scale" + + cutlass_compatible_b = ( + weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0 + ) + if not cutlass_compatible_b or use_triton_w8a8_fp8_kernel: + # Massage the input to be 2D + qinput = qinput.view(-1, qinput.shape[-1]) + output = triton_scaled_mm( + qinput, weight, x_scale, weight_scale, input.dtype, bias + ) + else: + output = fp8_scaled_mm( + qinput, + weight, + x_scale, + weight_scale, + out_dtype=input.dtype, + bias=bias, + ) + return output.view(*output_shape) + + # torch.scaled_mm supports per tensor weights + activations only + # so fallback to naive if per channel or per token + else: + # Maybe apply padding to output, see comment in __init__ + qinput, x_scale = ( + scaled_fp8_quant( + input_2d, + input_scale, + num_token_padding=output_padding, + use_per_token_if_dynamic=use_per_token_if_dynamic, + ) + if _is_cuda + else ops.scaled_fp8_quant( + input_2d, + input_scale, + num_token_padding=output_padding, + use_per_token_if_dynamic=use_per_token_if_dynamic, + ) + ) + + per_tensor_weights = weight_scale.numel() == 1 + per_tensor_activations = x_scale.numel() == 1 + + if per_tensor_weights and per_tensor_activations: + # Fused GEMM_DQ + output = torch._scaled_mm( + qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias, + ) + return _process_scaled_mm_output(output, input_2d.shape, output_shape) + + elif ( + use_per_token_if_dynamic + and not per_tensor_weights + and not per_tensor_activations + and (USE_ROWWISE_TORCH_SCALED_MM or _use_aiter) + ): + # into this sector means use dynamic per-token-per-channel quant + # per-token scale quant for input matrix, every row(one token) have one scale factor + # per-channel scale quant for weight matrix, every col(one channel) have one scale factor + if _use_aiter: + # gemm_a8w8_bpreshuffle(XQ, WQ, x_scale, w_scale, dtype) + # XQ -> input tensor, shape = (m, k) + # WQ -> weight tensor, shape = (n, k), with preshuffe get better perf + # x_scale -> input scale tensor, shape = (m, 1) + # w_scale -> weight scale tensor, shape = (n ,1) + # dtype -> output dtype + output = gemm_a8w8_bpreshuffle( + XQ=qinput, + WQ=weight, + x_scale=x_scale, + w_scale=weight_scale, + dtype=input.dtype, + ) + if bias is not None: + output += bias + return _process_scaled_mm_output( + output, input_2d.shape, [*input.shape[:-1], weight.shape[0]] + ) + else: + # For now validated on ROCm platform + # fp8 rowwise scaling in torch._scaled_mm is introduced in + # https://github.com/pytorch/pytorch/pull/144432 using hipBLASLt + # and ROCm 6.3, which only exists in torch 2.7 and above. + # For CUDA platform please validate if the + # torch._scaled_mm support rowwise scaled GEMM + # Fused GEMM_DQ Rowwise GEMM + output = torch._scaled_mm( + qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale.t(), + bias=bias, + ) + return _process_scaled_mm_output( + output, input_2d.shape, output_shape + ) + else: + # Fallback for channelwise case, where we use unfused DQ + # due to limitations with scaled_mm + + # Symmetric quantized GEMM by definition computes the following: + # C = (s_x * X) (s_w * W) + bias + # This is equivalent to dequantizing the weights and activations + # before applying a GEMM. + # + # In order to compute quantized operands, a quantized kernel + # will rewrite the above like so: + # C = s_w * s_x * (X * W) + bias + # + # For the scaled_mm fallback case, we break this down, since it + # does not support s_w being a vector. + return _apply_fallback_scaled_mm( + qinput, + weight, + x_scale, + weight_scale, + input_2d.shape, + output_shape, + bias, + input.dtype, + ) + else: + # cutlass w8a8 fp8 sgl-kernel only supports per-token scale + if input_scale is not None: + assert input_scale.numel() == 1 + # broadcast per-tensor scale to per-token scale when supporting cutlass + qinput, x_scale = static_quant_fp8( + input_2d, input_scale, repeat_scale=cutlass_fp8_supported + ) + else: + # default use per-token quantization if dynamic + if _is_cuda: + qinput, x_scale = sglang_per_token_quant_fp8(input_2d) + else: + # TODO(kkhuang): temporarily enforce per-tensor activation scaling if weight is per-tensor scaling + # final solution should be: 1. add support to per-tensor activation scaling. + # 2. solve the torch.compile error from weight_scale.numel() == 1 and x_scale.numel() > 1 (below line#308) + if _is_hip and weight_scale.numel() == 1: + qinput, x_scale = ops.scaled_fp8_quant( + input_2d, + input_scale, + use_per_token_if_dynamic=use_per_token_if_dynamic, + ) + else: + qinput, x_scale = per_token_group_quant_fp8( + input_2d, group_size=input_2d.shape[1] + ) + + if cutlass_fp8_supported: + try: + if VLLM_AVAILABLE and use_vllm_cutlass_w8a8_fp8_kernel: + # Fall back to vllm cutlass w8a8 fp8 kernel + output = ops.cutlass_scaled_mm( + qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias, + ) + else: + assert ( + weight_scale.numel() == weight.shape[1] + ), "cutlass w8a8 fp8 sgl-kernel only supports per-channel scale" + + cutlass_compatible_b = ( + weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0 + ) + if not cutlass_compatible_b or use_triton_w8a8_fp8_kernel: + # Massage the input to be 2D + qinput = qinput.view(-1, qinput.shape[-1]) + output = triton_scaled_mm( + qinput, weight, x_scale, weight_scale, input.dtype, bias + ) + else: + output = fp8_scaled_mm( + qinput, + weight, + x_scale, + weight_scale, + out_dtype=input.dtype, + bias=bias, + ) + return output.view(*output_shape) + except (ImportError, NameError, AttributeError): + pass + + # torch.scaled_mm supports per tensor weights + activations only + # so fallback to naive if per channel or per token + per_tensor_weights = weight_scale.numel() == 1 + per_tensor_activations = x_scale.numel() == 1 + + if per_tensor_weights and per_tensor_activations: + # Fused GEMM_DQ + output = torch._scaled_mm( + qinput, + weight, + out_dtype=input.dtype, + scale_a=x_scale, + scale_b=weight_scale, + bias=bias, + ) + return _process_scaled_mm_output(output, input_2d.shape, output_shape) + + else: + # Fallback for channelwise case, where we use unfused DQ + # due to limitations with scaled_mm + + # Symmetric quantized GEMM by definition computes the following: + # C = (s_x * X) (s_w * W) + bias + # This is equivalent to dequantizing the weights and activations + # before applying a GEMM. + # + # In order to compute quantized operands, a quantized kernel + # will rewrite the above like so: + # C = s_w * s_x * (X * W) + bias + # + # For the scaled_mm fallback case, we break this down, since it + # does not support s_w being a vector. + return _apply_fallback_scaled_mm( + qinput, + weight, + x_scale, + weight_scale, + input_2d.shape, + output_shape, + bias, + input.dtype, + ) + + +def can_auto_enable_marlin_fp8() -> bool: + try: + major, minor = get_device_capability() + sm = major * 10 + minor + return 80 <= sm < 89 + except Exception: + return False diff --git a/python/sglang/srt/layers/quantization/fpgemm_fp8.py b/python/sglang/srt/layers/quantization/fpgemm_fp8.py new file mode 100644 index 000000000..5a78626ff --- /dev/null +++ b/python/sglang/srt/layers/quantization/fpgemm_fp8.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import logging +from typing import Any, Optional + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +from sglang.srt.layers.linear import LinearBase +from sglang.srt.layers.parameter import ChannelQuantScaleParameter, ModelWeightParameter +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, + can_auto_enable_marlin_fp8, + cutlass_fp8_supported, + normalize_e4m3fn_to_e4m3fnuz, +) +from sglang.srt.layers.quantization.marlin_utils_fp8 import ( + apply_fp8_marlin_linear, + prepare_fp8_layer_for_marlin, +) +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import is_layer_skipped, replace_parameter +from sglang.srt.utils import get_bool_env_var, is_cuda + +_is_cuda = is_cuda() +_is_fp8_fnuz = is_fp8_fnuz() + +logger = logging.getLogger(__name__) + + +class FBGEMMFp8Config(QuantizationConfig): + """Config class for FBGEMM Fp8.""" + + def __init__(self, ignore_list: list[str], input_scale_ub: float): + super().__init__() + self.ignore_list = ignore_list if ignore_list else [] + self.input_scale_ub = input_scale_ub + + # For GPUs that lack FP8 hardware suspport, we can leverage the Marlin + # kernel for fast weight-only FP8 quantization + # self.use_marlin = not marlin_fp8_supported() + self.use_marlin = False + if _is_cuda: + force_marlin = get_bool_env_var("SGLANG_FORCE_FP8_MARLIN") + auto_enable = can_auto_enable_marlin_fp8() + self.use_marlin = force_marlin or auto_enable + + @classmethod + def get_name(cls) -> str: + return "fbgemm_fp8" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.bfloat16, torch.float16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> list[str]: + return [] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> FBGEMMFp8Config: + ignore_list = cls.get_from_keys(config, ["modules_to_not_convert"]) + input_scale_ub = cls.get_from_keys(config, ["activation_scale_ub"]) + return cls(ignore_list=ignore_list, input_scale_ub=input_scale_ub) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + if isinstance(layer, LinearBase): + if is_layer_skipped( + prefix=prefix, + ignored_layers=self.ignore_list, + fused_mapping=self.packed_modules_mapping, + ): + return UnquantizedLinearMethod() + return FBGEMMFp8LinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class FBGEMMFp8LinearMethod(LinearMethodBase): + + def __init__(self, quant_config: FBGEMMFp8Config): + self.quant_config = quant_config + # self.fp8_linear = Fp8LinearOp( + # act_quant_static=False, act_quant_group_shape=GroupShape.PER_TOKEN) + self.out_dtype = torch.get_default_dtype() + self.cutlass_fp8_supported = cutlass_fp8_supported() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + # maybe_create_device_identity() + weight_loader = extra_weight_attrs.get("weight_loader") + del input_size, output_size + output_size_per_partition = sum(output_partition_sizes) + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + layer.orig_dtype = params_dtype + + # WEIGHT + weight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=torch.float8_e4m3fn, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader, + ) + weight_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("weight_scale", weight_scale) + + # INPUT SCALE UPPER BOUND + input_scale_ub = torch.nn.Parameter( + torch.tensor((self.quant_config.input_scale_ub), dtype=torch.float32), + requires_grad=False, + ) + layer.input_scale_ub = input_scale_ub + + def process_weights_after_loading(self, layer: Module) -> None: + # required by torch.compile + layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False) + layer.weight = Parameter(layer.weight.data, requires_grad=False) + + weight = layer.weight + + if _is_fp8_fnuz: + weight, weight_scale, input_scale = normalize_e4m3fn_to_e4m3fnuz( + weight=weight, weight_scale=layer.weight_scale, input_scale=None + ) + if input_scale is not None: + layer.input_scale = Parameter(input_scale, requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + + layer.weight = Parameter(weight.t(), requires_grad=False) + if self.quant_config.use_marlin: + prepare_fp8_layer_for_marlin(layer) + # Activations not quantized for marlin. + del layer.input_scale_ub + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + + if self.quant_config.use_marlin: + return apply_fp8_marlin_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + workspace=layer.workspace, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias, + ) + + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=None, + input_scale_ub=layer.input_scale_ub, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + use_per_token_if_dynamic=False, + ) diff --git a/python/sglang/srt/layers/quantization/gptq.py b/python/sglang/srt/layers/quantization/gptq.py new file mode 100644 index 000000000..ccd3d46f7 --- /dev/null +++ b/python/sglang/srt/layers/quantization/gptq.py @@ -0,0 +1,1097 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from fractions import Fraction +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union + +import torch + +from sglang.srt.layers.parameter import ( + BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedColumnParameter, + PackedvLLMParameter, + RowvLLMParameter, + permute_param_layout_, +) +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.marlin_utils import ( + apply_gptq_marlin_linear, + check_marlin_supported, + check_marlin_supports_shape, + marlin_is_k_full, + marlin_make_empty_g_idx, + marlin_make_workspace, + marlin_moe_permute_scales, + marlin_permute_scales, + marlin_repeat_scales_on_all_ranks, + marlin_sort_g_idx, + marlin_zero_points, + verify_marlin_supported, +) +from sglang.srt.layers.quantization.utils import ( + get_linear_quant_method, + get_scalar_types, + replace_parameter, + unpack_cols, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher import ( + StandardDispatchOutput, + CombineInput, + ) + +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() + +if _is_cuda: + from sgl_kernel import fused_marlin_moe, gptq_gemm, gptq_marlin_repack, gptq_shuffle + + +logger = logging.getLogger(__name__) +ScalarType, scalar_types = get_scalar_types() + + +def check_marlin_format(hf_quant_cfg: Dict[str, Any]) -> bool: + # compat: gptqmodel and autogptq (eol) main use checkpoint_format: str + # compat: autogptq <=0.7.1 is_marlin_format: bool + return hf_quant_cfg.get("checkpoint_format") == "marlin" or hf_quant_cfg.get( + "is_marlin_format", False + ) + + +def gptq_marlin_moe_repack( + b_q_weight: torch.Tensor, + perm: torch.Tensor, + size_k: int, + size_n: int, + num_bits: int, +) -> torch.Tensor: + num_experts = b_q_weight.shape[0] + assert size_k % 16 == 0 + output = torch.empty( + (num_experts, size_k // 16, size_n * (num_bits // 2)), + device=b_q_weight.device, + dtype=b_q_weight.dtype, + ) + for e in range(num_experts): + output[e] = gptq_marlin_repack(b_q_weight[e], perm[e], size_k, size_n, num_bits) + return output + + +@dataclass +class MarlinLinearLayerConfig: + full_weight_shape: tuple[int, int] # [in, out] + partition_weight_shape: tuple[int, int] + weight_type: ScalarType + act_type: torch.dtype + group_size: int + zero_points: bool + has_g_idx: bool + + +class GPTQConfig(QuantizationConfig): + """Config class for GPTQ. + + Reference: https://arxiv.org/abs/2210.17323 + """ + + def __init__( + self, + weight_bits: int, + group_size: int, + desc_act: bool, + lm_head_quantized: bool, + dynamic: Dict[str, Dict[str, Union[int, bool]]], + ) -> None: + # GPTQModel use `dynamic` config property to allow per module + # quantization config so each module can be individually optimized. + # Format is Dict[str, Dict] where key is a regex string that can + # perform both positive ("+:" prefixed) or negative ("-:" prefixed) + # matching of a module. + # Default to positive match, override base quant config mode, if no + # prefix is used. Value is in dict format of field key and override + # value. + # Negative matching will skip quantization init for this module + # entirely: + # non-quantized inference. More details and quantization examples can be + # found at: https://github.com/ModelCloud/GPTQModel + # Example: + # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9 + # # last 1/4 of the layers 16-21 has 8bit and group_size 64 + # dynamic = { + # #`.*\.` matches the layers_node prefix + # # positive match layer 10-15 + # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,}, + # # positive match layer 16-21 + # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,}, + # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers + # } + super().__init__() + self.dynamic = dynamic + + self.weight_bits = weight_bits + self.group_size = group_size + self.desc_act = desc_act + self.lm_head_quantized = lm_head_quantized + self.pack_factor = Fraction(32, self.weight_bits) + if self.weight_bits not in [2, 3, 4, 8]: + raise ValueError( + "Currently, only 2/3/4/8-bit weight quantization is " + f"supported for GPTQ, but got {self.weight_bits} bits." + ) + + def __repr__(self) -> str: + return ( + f"GPTQConfig(weight_bits={self.weight_bits}, " + f"group_size={self.group_size}, " + f"desc_act={self.desc_act})," + f"lm_head_quantized={self.lm_head_quantized}), " + f"dynamic={self.dynamic}" + ) + + def get_scaled_act_names(self) -> List[str]: + """Returns the activation function names that should be post-scaled. + + For now, this is only used by AWQ. + """ + raise NotImplementedError + + @classmethod + def get_name(cls) -> str: + return "gptq" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.half] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 60 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> GPTQConfig: + dynamic = cls.get_from_keys_or(config, ["dynamic"], default={}) + dynamic = {} if dynamic is None else dynamic + + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + desc_act = cls.get_from_keys(config, ["desc_act"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) + return cls(weight_bits, group_size, desc_act, lm_head_quantized, dynamic) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[LinearMethodBase]: + # Delay the import to avoid circular dependency + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, FusedMoE): + raise TypeError("GPTQ Method does not support MoE, please use gptq_marlin") + else: + return get_linear_quant_method( + self, layer, prefix=prefix, linear_method_cls=GPTQLinearMethod + ) + + +class GPTQMarlinConfig(QuantizationConfig): + """Config class for GPTQ Marlin""" + + # (num_bits, is_sym) -> quant_type + TYPE_MAP = { + (4, True): scalar_types.uint4b8, + (8, True): scalar_types.uint8b128, + } + + def __init__( + self, + weight_bits: int, + group_size: int, + desc_act: bool, + is_sym: bool, + lm_head_quantized: bool, + dynamic: Dict[str, Dict[str, Union[int, bool]]], + full_config: Dict[str, Any], + ) -> None: + super().__init__() + if desc_act and group_size == -1: + # In this case, act_order == True is the same as act_order == False + # (since we have only one group per output channel) + desc_act = False + + # GPTQModel use `dynamic` config property to allow per module + # quantization config so each module can be individually optimized. + # Format is Dict[str, Dict] where key is a regex string that can + # perform both positive ("+:" prefixed) or negative ("-:" prefixed) + # matching of a module. + # Default to positive match, override base quant config mode, if no + # prefix is used. Value is in dict format of field key and override + # value. + # Negative matching will skip quantization init for this module + # entirely: + # non-quantized inference. More details and quantization examples can be + # found at: https://github.com/ModelCloud/GPTQModel + # Example: + # # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9 + # # last 1/4 of the layers 16-21 has 8bit and group_size 64 + # dynamic = { + # #`.*\.` matches the layers_node prefix + # # positive match layer 10-15 + # r"+:.*\.(?:1[0-5])\..*": {"bits": 8,}, + # # positive match layer 16-21 + # r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,}, + # r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers + # } + self.dynamic = dynamic + + self.weight_bits = weight_bits + self.is_sym = is_sym + + self.pack_factor = 32 // weight_bits # packed into int32 + self.group_size = group_size + self.desc_act = desc_act + self.lm_head_quantized = lm_head_quantized + self.full_config = full_config + + if (weight_bits, is_sym) not in self.TYPE_MAP: + raise ValueError( + "Unsupported quantization config: " f"bits={weight_bits}, sym={is_sym}" + ) + + # (num_bits, is_sym) -> quant_type + self.quant_type = self.TYPE_MAP[(weight_bits, is_sym)] + + def __repr__(self) -> str: + return ( + f"GPTQMarlinConfig(quant_type={self.quant_type}, " + f"group_size={self.group_size}, " + f"desc_act={self.desc_act}, " + f"lm_head_quantized={self.lm_head_quantized}), " + f"dynamic={self.dynamic}" + ) + + def get_scaled_act_names(self) -> List[str]: + """Returns the activation function names that should be post-scaled. + + For now, this is only used by AWQ. + """ + raise NotImplementedError + + @classmethod + def get_name(cls) -> str: + return "gptq_marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.half, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> GPTQMarlinConfig: + dynamic = cls.get_from_keys_or(config, ["dynamic"], default={}) + dynamic = {} if dynamic is None else dynamic + + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + desc_act = cls.get_from_keys(config, ["desc_act"]) + is_sym = cls.get_from_keys(config, ["sym"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) + return cls( + weight_bits, + group_size, + desc_act, + is_sym, + lm_head_quantized, + dynamic, + config, + ) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + is_marlin_format = check_marlin_format(hf_quant_cfg) + + can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg) + + is_valid_user_quant = ( + user_quant is None or user_quant == "marlin" or user_quant == "gptq_marlin" + ) + + if not is_marlin_format and can_convert and is_valid_user_quant: + msg = ( + "The model is convertible to {} during runtime." + " Using {} kernel.".format(cls.get_name(), cls.get_name()) + ) + logger.info(msg) + return cls.get_name() + + if not is_marlin_format and can_convert and user_quant == "gptq": + logger.info( + "Detected that the model can run with gptq_marlin" + ", however you specified quantization=gptq explicitly," + " so forcing gptq. Use quantization=gptq_marlin for" + " faster inference" + ) + return None + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + # Delay the import to avoid circular dependency + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, FusedMoE): + return GPTQMarlinMoEMethod(self) + return get_linear_quant_method(self, layer, prefix, GPTQMarlinLinearMethod) + + @classmethod + def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]): + quant_method = quant_config.get("quant_method", "").lower() + num_bits = quant_config.get("bits") + group_size = quant_config.get("group_size") + sym = quant_config.get("sym") + desc_act = quant_config.get("desc_act") + + if not _is_cuda: + return False + + if quant_method != "gptq": + return False + + # Marlin conversion is only valid if required properties are found + if num_bits is None or group_size is None or sym is None or desc_act is None: + return False + + if (num_bits, sym) not in cls.TYPE_MAP: + return False + + return check_marlin_supported( + quant_type=cls.TYPE_MAP[(num_bits, sym)], group_size=group_size + ) + + +class GPTQLinearMethod(LinearMethodBase): + """Linear method for GPTQ. + + Args: + quant_config: The GPTQ quantization config. + """ + + def __init__(self, quant_config: GPTQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del output_size # Unused. + weight_loader = extra_weight_attrs.get("weight_loader") + if input_size_per_partition % self.quant_config.group_size != 0: + raise ValueError( + "The input size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.pack_factor.numerator != 0: + raise ValueError( + "The output size is not aligned with the quantized " + "weight shape. This can be caused by too large " + "tensor parallel size." + ) + + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + self.use_shuffle = True + scale_and_zero_size = input_size // group_size + scale_and_zero_input_dim = None + if ( + input_size != input_size_per_partition + and self.quant_config.group_size != -1 + ): + if self.quant_config.desc_act: + self.use_shuffle = False + else: + # we need to partition qzeros and scales for exllama kernel + scale_and_zero_size = input_size_per_partition // group_size + scale_and_zero_input_dim = 0 + + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.pack_factor, + output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=0, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + g_idx = RowvLLMParameter( + data=torch.tensor( + [ + i // self.quant_config.group_size + for i in range(input_size_per_partition) + ], + dtype=torch.int32, + ), + input_dim=0, + weight_loader=weight_loader, + ) + qzeros_args = { + "data": torch.empty( + scale_and_zero_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + "weight_loader": weight_loader, + } + weight_scale_args = { + "data": torch.empty( + scale_and_zero_size, + output_size_per_partition, + dtype=params_dtype, + ), + "weight_loader": weight_loader, + } + if scale_and_zero_input_dim is None: + scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args) + qzeros = PackedColumnParameter( + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args, + ) + + else: + scales = GroupQuantScaleParameter( + output_dim=1, input_dim=0, **weight_scale_args + ) + qzeros = PackedvLLMParameter( + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args, + ) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("g_idx", g_idx) + layer.register_parameter("qzeros", qzeros) + layer.register_parameter("scales", scales) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # for torch.compile + layer.qzeros = torch.nn.Parameter(layer.qzeros.data, requires_grad=False) + layer.qweight = torch.nn.Parameter(layer.qweight.data, requires_grad=False) + layer.g_idx = torch.nn.Parameter(layer.g_idx.data, requires_grad=False) + layer.scales = torch.nn.Parameter(layer.scales.data, requires_grad=False) + + # exllama needs to shuffle the weight after the weight is loaded + # here we do the shuffle on first forward pass + if self.use_shuffle: + if self.quant_config.desc_act: + layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int) + else: + layer.g_idx.data = torch.empty( + (0,), dtype=torch.int, device=layer.g_idx.device + ) + gptq_shuffle(layer.qweight, layer.g_idx, self.quant_config.weight_bits) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + out_shape = x.shape[:-1] + (layer.qweight.shape[-1],) + reshaped_x = x.reshape(-1, x.shape[-1]) + + output = gptq_gemm( + reshaped_x, + layer.qweight, + layer.qzeros, + layer.scales, + layer.g_idx, + self.use_shuffle, + self.quant_config.weight_bits, + ) + if bias is not None: + output.add_(bias) + return output.reshape(out_shape) + + +class GPTQMarlinLinearMethod(LinearMethodBase): + """Linear method for GPTQ Marlin. + + Args: + quant_config: The GPTQ Marlin quantization config. + """ + + _kernel_backends_being_used: set[str] = set() + + def __init__(self, quant_config: GPTQMarlinConfig) -> None: + self.quant_config = quant_config + + # Verify supported on platform. + verify_marlin_supported( + quant_type=self.quant_config.quant_type, + group_size=self.quant_config.group_size, + ) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + is_row_parallel = input_size != input_size_per_partition + weight_loader = extra_weight_attrs.get("weight_loader") + + self.kernel_config = MarlinLinearLayerConfig( + full_weight_shape=(input_size, output_size), + partition_weight_shape=( + input_size_per_partition, + output_size_per_partition, + ), + weight_type=self.quant_config.quant_type, + act_type=params_dtype, + group_size=self.quant_config.group_size, + zero_points=False, + has_g_idx=self.quant_config.desc_act, + ) + # Normalize group_size + if self.quant_config.group_size != -1: + group_size = self.quant_config.group_size + else: + group_size = input_size + + # Determine sharding + if marlin_repeat_scales_on_all_ranks( + self.quant_config.desc_act, self.quant_config.group_size, is_row_parallel + ): + # By setting scale_dim == None, weight_loader will + # repeat the scales on each GPU in TP>1 case. + scales_and_zp_input_dim = None + scales_and_zp_size = input_size // group_size + else: + # By setting scale_dim == 0, weight_loader will + # shard the scales in TP>1 case. + scales_and_zp_input_dim = 0 + scales_and_zp_size = input_size_per_partition // group_size + + # Quantized weights + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.pack_factor, + output_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=0, + packed_factor=self.quant_config.pack_factor, + weight_loader=weight_loader, + ) + + # Activation order + g_idx = RowvLLMParameter( + data=torch.empty( + input_size_per_partition, + dtype=torch.int32, + ), + input_dim=0, + weight_loader=weight_loader, + ) + + qzeros_args = { + "data": torch.empty( + scales_and_zp_size, + output_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int32, + ), + "weight_loader": weight_loader, + } + weight_scale_args = { + "data": torch.empty( + scales_and_zp_size, + output_size_per_partition, + dtype=params_dtype, + ), + "weight_loader": weight_loader, + } + + if scales_and_zp_input_dim is None: + scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args) + qzeros = PackedColumnParameter( + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args, + ) + + else: + scales = GroupQuantScaleParameter( + output_dim=1, input_dim=0, **weight_scale_args + ) + qzeros = PackedvLLMParameter( + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + **qzeros_args, + ) + + layer.register_parameter("qweight", qweight) + layer.register_parameter("g_idx", g_idx) + layer.register_parameter("scales", scales) + layer.register_parameter("qzeros", qzeros) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + device = getattr(layer, "qweight").device + c = self.kernel_config + + check_marlin_supports_shape( + c.partition_weight_shape[1], # out_features + c.partition_weight_shape[0], # in_features + c.full_weight_shape[0], # in_features + c.group_size, + ) + + row_parallel = c.partition_weight_shape[0] != c.full_weight_shape[0] + self.is_k_full = marlin_is_k_full(c.has_g_idx, row_parallel) + + # Allocate marlin workspace. + self.workspace = marlin_make_workspace(device) + + # Default names since marlin requires empty parameters for these, + # TODO: remove this requirement from marlin (allow optional tensors) + self.w_q_name = "qweight" + self.w_s_name = "scales" + self.w_zp_name = "qzeros" + self.w_gidx_name = "g_idx" + + def _transform_param( + layer: torch.nn.Module, name: Optional[str], fn: Callable + ) -> None: + if name is not None and getattr(layer, name, None) is not None: + + old_param = getattr(layer, name) + new_param = fn(old_param) + # replace the parameter with torch.nn.Parameter for TorchDynamo + # compatibility + replace_parameter( + layer, name, torch.nn.Parameter(new_param.data, requires_grad=False) + ) + + def transform_w_q(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1, packed_dim=0) + x.data = gptq_marlin_repack( + x.data.contiguous(), + perm=layer.g_idx_sort_indices, + size_k=c.partition_weight_shape[0], + size_n=c.partition_weight_shape[1], + num_bits=c.weight_type.size_bits, + ) + return x + + def transform_w_s(x): + assert isinstance(x, BasevLLMParameter) + permute_param_layout_(x, input_dim=0, output_dim=1) + x.data = marlin_permute_scales( + x.data.contiguous(), + size_k=c.partition_weight_shape[0], + size_n=c.partition_weight_shape[1], + group_size=c.group_size, + ) + return x + + if c.has_g_idx: + g_idx, g_idx_sort_indices = marlin_sort_g_idx( + getattr(layer, self.w_gidx_name) + ) + _transform_param(layer, self.w_gidx_name, lambda _: g_idx) + layer.g_idx_sort_indices = g_idx_sort_indices + else: + setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device)) + layer.g_idx_sort_indices = marlin_make_empty_g_idx(device) + + if c.zero_points: + grouped_k = ( + c.partition_weight_shape[0] // c.group_size if c.group_size != -1 else 1 + ) + _transform_param( + layer, + self.w_zp_name, + lambda x: marlin_zero_points( + unpack_cols( + x.t(), + c.weight_type.size_bits, + grouped_k, + c.partition_weight_shape[1], + ), + size_k=grouped_k, + size_n=c.partition_weight_shape[1], + num_bits=c.weight_type.size_bits, + ), + ) + else: + setattr(layer, self.w_zp_name, marlin_make_empty_g_idx(device)) + _transform_param(layer, self.w_q_name, transform_w_q) + _transform_param(layer, self.w_s_name, transform_w_s) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + c = self.kernel_config + + def _get_weight_params( + layer: torch.nn.Module, + ) -> tuple[ + torch.Tensor, # w_q + torch.Tensor, # w_s + Optional[torch.Tensor], # w_zp, + Optional[torch.Tensor], # w_gidx + ]: + return ( + getattr(layer, self.w_q_name), + getattr(layer, self.w_s_name), + getattr(layer, self.w_zp_name or "", None), + getattr(layer, self.w_gidx_name or "", None), + ) + + w_q, w_s, w_zp, w_gidx = _get_weight_params(layer) + + # `process_weights_after_loading` will ensure w_zp and w_gidx are not + # None for marlin + return apply_gptq_marlin_linear( + input=x, + weight=w_q, + weight_scale=w_s, + weight_zp=w_zp, # type: ignore + g_idx=w_gidx, # type: ignore + g_idx_sort_indices=layer.g_idx_sort_indices, + workspace=self.workspace, + wtype=c.weight_type, + input_size_per_partition=c.partition_weight_shape[0], + output_size_per_partition=c.partition_weight_shape[1], + is_k_full=self.is_k_full, + bias=bias, + ) + + +class GPTQMarlinMoEMethod(FusedMoEMethodBase): + """MoE Marlin method with quantization.""" + + def __init__(self, quant_config: GPTQMarlinConfig) -> None: + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + # Delay the import to avoid circular dependency + from sglang.srt.layers.linear import set_weight_attrs + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + self.is_k_full = (not self.quant_config.desc_act) or layer.moe_tp_size == 1 + + if self.quant_config.group_size != -1: + scales_size13 = hidden_size // self.quant_config.group_size + if self.quant_config.desc_act: + w2_scales_size = intermediate_size_per_partition + else: + w2_scales_size = intermediate_size_per_partition * layer.moe_tp_size + scales_size2 = w2_scales_size // self.quant_config.group_size + strategy = FusedMoeWeightScaleSupported.GROUP.value + else: + scales_size13 = 1 + scales_size2 = 1 + strategy = FusedMoeWeightScaleSupported.CHANNEL.value + + extra_weight_attrs.update({"quant_method": strategy, "is_transposed": True}) + # Fused gate_up_proj (column parallel) + w13_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size // self.quant_config.pack_factor, + 2 * intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qweight", w13_qweight) + set_weight_attrs(w13_qweight, extra_weight_attrs) + # down_proj (row parallel) + w2_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition // self.quant_config.pack_factor, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qweight", w2_qweight) + set_weight_attrs(w2_qweight, extra_weight_attrs) + # up_proj scales + w13_scales = torch.nn.Parameter( + torch.empty( + num_experts, + scales_size13, + 2 * intermediate_size_per_partition, + dtype=torch.half, + ), + requires_grad=False, + ) + layer.register_parameter("w13_scales", w13_scales) + set_weight_attrs(w13_scales, extra_weight_attrs) + # down_proj scales + w2_scales = torch.nn.Parameter( + torch.empty(num_experts, scales_size2, hidden_size, dtype=torch.half), + requires_grad=False, + ) + layer.register_parameter("w2_scales", w2_scales) + set_weight_attrs(w2_scales, extra_weight_attrs) + # dont shard the w2 scales when running act order + set_weight_attrs(w2_scales, {"load_full_w2": self.quant_config.desc_act}) + # up_proj scales + w13_qzeros = torch.nn.Parameter( + torch.empty( + num_experts, + scales_size13, + 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qzeros", w13_qzeros) + set_weight_attrs(w13_qzeros, extra_weight_attrs) + # down_proj scales + w2_qzeros = torch.nn.Parameter( + torch.empty( + num_experts, + scales_size2, + hidden_size // self.quant_config.pack_factor, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qzeros", w2_qzeros) + set_weight_attrs(w2_qzeros, extra_weight_attrs) + # dont shard the w2 scales when running act order + set_weight_attrs(w2_qzeros, {"load_full_w2": self.quant_config.desc_act}) + w13_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx", w13_g_idx) + set_weight_attrs(w13_g_idx, extra_weight_attrs) + w2_g_idx = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx", w2_g_idx) + set_weight_attrs(w2_g_idx, extra_weight_attrs) + w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_g_idx_sort_indices", w13_g_idx_sort_indices) + set_weight_attrs(w13_g_idx_sort_indices, extra_weight_attrs) + w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition, + dtype=torch.int32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices) + set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + + # Process act_order + if self.quant_config.desc_act: + # Get sorting based on g_idx + num_experts = layer.w13_g_idx.shape[0] + w13_g_idx_sort_indices = torch.empty_like(layer.w13_g_idx) + w2_g_idx_sort_indices = torch.empty_like(layer.w2_g_idx) + w13_sorted_g_idx = torch.empty_like(layer.w13_g_idx) + w2_sorted_g_idx = torch.empty_like(layer.w2_g_idx) + for e in range(num_experts): + w13_g_idx_sort_indices[e] = torch.argsort(layer.w13_g_idx[e]).to( + torch.int32 + ) + w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_g_idx[e]).to( + torch.int32 + ) + w13_sorted_g_idx[e] = layer.w13_g_idx[e][w13_g_idx_sort_indices[e]] + w2_sorted_g_idx[e] = layer.w2_g_idx[e][w2_g_idx_sort_indices[e]] + replace_parameter(layer, "w13_g_idx", w13_sorted_g_idx) + replace_parameter(layer, "w2_g_idx", w2_sorted_g_idx) + replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices) + replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) + else: + # Reset g_idx related tensors + num_experts = layer.w13_g_idx.shape[0] + device = layer.w13_g_idx.device + layer.w13_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + layer.w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + # Repack weights + marlin_w13_qweight = gptq_marlin_moe_repack( + layer.w13_qweight, + layer.w13_g_idx_sort_indices, + layer.w13_qweight.shape[1] * self.quant_config.pack_factor, + layer.w13_qweight.shape[2], + self.quant_config.weight_bits, + ) + replace_parameter(layer, "w13_qweight", marlin_w13_qweight) + marlin_w2_qweight = gptq_marlin_moe_repack( + layer.w2_qweight, + layer.w2_g_idx_sort_indices, + layer.w2_qweight.shape[1] * self.quant_config.pack_factor, + layer.w2_qweight.shape[2], + self.quant_config.weight_bits, + ) + replace_parameter(layer, "w2_qweight", marlin_w2_qweight) + # Repack scales + marlin_w13_scales = marlin_moe_permute_scales( + s=layer.w13_scales, + size_k=layer.intermediate_size_per_partition, + size_n=layer.w13_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w13_scales", marlin_w13_scales) + marlin_w2_scales = marlin_moe_permute_scales( + s=layer.w2_scales, + size_k=layer.w2_scales.shape[1] + * ( + self.quant_config.group_size + if self.quant_config.group_size != -1 + else self.quant_config.pack_factor + ), + size_n=layer.w2_scales.shape[2], + group_size=self.quant_config.group_size, + ) + replace_parameter(layer, "w2_scales", marlin_w2_scales) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + # Delay the import to avoid circular dependency + + assert ( + self.moe_runner_config.activation == "silu" + ), "Only SiLU activation is supported." + + # The input must currently be float16 + orig_dtype = x.dtype + x = x.half() + + topk_weights, topk_ids, router_logits = topk_output + + output = fused_marlin_moe( + x, + layer.w13_qweight, + layer.w2_qweight, + layer.w13_scales, + layer.w2_scales, + router_logits, + topk_weights, + topk_ids, + g_idx1=layer.w13_g_idx, + g_idx2=layer.w2_g_idx, + sort_indices1=layer.w13_g_idx_sort_indices, + sort_indices2=layer.w2_g_idx_sort_indices, + num_bits=self.quant_config.weight_bits, + is_k_full=self.is_k_full, + ).to(orig_dtype) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/int8_kernel.py b/python/sglang/srt/layers/quantization/int8_kernel.py new file mode 100644 index 000000000..7c6c3dbd4 --- /dev/null +++ b/python/sglang/srt/layers/quantization/int8_kernel.py @@ -0,0 +1,425 @@ +import functools +import json +import logging +import os +from typing import Any, Dict, List, Optional, Tuple + +import torch +import triton +import triton.language as tl + +from sglang.srt.utils import get_device_name, is_cuda + +_is_cuda = is_cuda() +if _is_cuda: + from sgl_kernel import sgl_per_token_group_quant_int8 + +logger = logging.getLogger(__name__) + + +@triton.jit +def _per_token_quant_int8( + x_ptr, + xq_ptr, + scale_ptr, + x_sum_ptr, + stride_x, + stride_xq, + N, + CAL_SUM: tl.constexpr, + BLOCK: tl.constexpr, +): + # Adapted from https://github.com/InternLM/lmdeploy/blob/086481ed84b59bee3b8e4274e5fc69620040c048/lmdeploy/pytorch/kernels/cuda/w8a8_triton_kernels.py#L282 + row_id = tl.program_id(0) + + cols = tl.arange(0, BLOCK) + mask = cols < N + + x = tl.load(x_ptr + row_id * stride_x + cols, mask=mask, other=0.0).to(tl.float32) + absmax = tl.maximum(tl.max(tl.abs(x)), 1e-10) + scale_x = absmax / 127 + x_q = x * (127 / absmax) + x_q = tl.extra.cuda.libdevice.round(x_q).to(tl.int8) + if CAL_SUM: + x_sum = tl.sum(x, axis=0) + tl.store(x_sum_ptr + row_id, x_sum.to(x_sum_ptr.dtype.element_ty)) + + tl.store(xq_ptr + row_id * stride_xq + cols, x_q, mask=mask) + tl.store(scale_ptr + row_id, scale_x.to(scale_ptr.dtype.element_ty)) + + +def per_token_quant_int8(x, scale_dtype=torch.float32, cal_sum=False): + M = x.numel() // x.shape[-1] + N = x.shape[-1] + x_q = torch.empty_like(x, device=x.device, dtype=torch.int8) + scales = torch.empty(x.shape[:-1] + (1,), device=x.device, dtype=scale_dtype) + if cal_sum: + x_sum = torch.empty(x.shape[:-1], device=x.device, dtype=x.dtype) + else: + x_sum = None + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + + assert x.is_contiguous() + _per_token_quant_int8[(M,)]( + x, + x_q, + scales, + x_sum, + stride_x=x.stride(-2), + stride_xq=x_q.stride(-2), + N=N, + CAL_SUM=cal_sum, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=1, + ) + if cal_sum: + return x_q, scales, x_sum + else: + return x_q, scales + + +@triton.jit +def _per_token_group_quant_int8( + # Pointers to inputs and output + y_ptr, + y_q_ptr, + y_s_ptr, + # Stride of input + y_stride, + # Columns of input + N, + # Avoid to divide zero + eps, + # Information for int8 + int8_min, + int8_max, + # Meta-parameters + BLOCK: tl.constexpr, +): + """A Triton-accelerated function to perform per-token-group quantization on a + tensor. + + This function converts the tensor values into int8 values. + """ + # Map the program id to the row of X and Y it should compute. + g_id = tl.program_id(0) + y_ptr += g_id * y_stride + y_q_ptr += g_id * y_stride + y_s_ptr += g_id + + cols = tl.arange(0, BLOCK) # N <= BLOCK + mask = cols < N + + y = tl.load(y_ptr + cols, mask=mask, other=0.0).to(tl.float32) + # Quant + _absmax = tl.maximum(tl.max(tl.abs(y)), eps) + y_s = _absmax / int8_max + y_q = tl.clamp(y / y_s, int8_min, int8_max).to(y_q_ptr.dtype.element_ty) + + tl.store(y_q_ptr + cols, y_q, mask=mask) + tl.store(y_s_ptr, y_s) + + +def per_token_group_quant_int8( + x: torch.Tensor, + group_size: int, + eps: float = 1e-10, + dtype: torch.dtype = torch.int8, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Function to perform per-token-group quantization on an input tensor `x`. + + It converts the tensor values into signed int8 values and returns the + quantized tensor along with the scaling factor used for quantization. + + Args: + x: The input tenosr with ndim >= 2. + group_size: The group size used for quantization. + eps: The minimum to avoid dividing zero. + dtype: The dype of output tensor. Note that only `torch.int8` is supported for now. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The quantized tensor and the scaling factor for quantization. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_max = iinfo.max + int8_min = iinfo.min + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + M = x.numel() // group_size + N = group_size + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + BLOCK = triton.next_power_of_2(N) + # heuristics for number of warps + num_warps = min(max(BLOCK // 256, 1), 8) + num_stages = 1 + _per_token_group_quant_int8[(M,)]( + x, + x_q, + x_s, + group_size, + N, + eps, + int8_min=int8_min, + int8_max=int8_max, + BLOCK=BLOCK, + num_warps=num_warps, + num_stages=num_stages, + ) + + return x_q, x_s + + +def sglang_per_token_group_quant_int8( + x: torch.Tensor, + group_size: int, + eps: float = 1e-10, + dtype: torch.dtype = torch.int8, +): + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_max = iinfo.max + int8_min = iinfo.min + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + x_s = torch.empty( + x.shape[:-1] + (x.shape[-1] // group_size,), + device=x.device, + dtype=torch.float32, + ) + + sgl_per_token_group_quant_int8(x, x_q, x_s, group_size, eps, int8_min, int8_max) + + return x_q, x_s + + +@triton.jit +def _w8a8_block_int8_matmul( + # Pointers to inputs and output + A, + B, + C, + As, + Bs, + # Shape for matmul + M, + N, + K, + # Block size for block-wise quantization + group_n, + group_k, + # Stride for inputs and output + stride_am, + stride_ak, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_As_m, + stride_As_k, + stride_Bs_k, + stride_Bs_n, + # Meta-parameters + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, +): + """Triton-accelerated function used to perform linear operations (dot + product) on input tensors `A` and `B` with block-wise quantization, and store the result in output + tensor `C`. + """ + + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M + offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N + offs_k = tl.arange(0, BLOCK_SIZE_K) + a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) + b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) + + As_ptrs = As + offs_am * stride_As_m + offs_bsn = offs_bn // group_n + Bs_ptrs = Bs + offs_bsn * stride_Bs_n + + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_SIZE_K)): + a = tl.load(a_ptrs, mask=offs_k[None, :] < K - k * BLOCK_SIZE_K, other=0.0) + b = tl.load(b_ptrs, mask=offs_k[:, None] < K - k * BLOCK_SIZE_K, other=0.0) + + k_start = k * BLOCK_SIZE_K + offs_ks = k_start // group_k + a_s = tl.load(As_ptrs + offs_ks * stride_As_k) + b_s = tl.load(Bs_ptrs + offs_ks * stride_Bs_k) + + accumulator += tl.dot(a, b).to(tl.float32) * a_s[:, None] * b_s[None, :] + a_ptrs += BLOCK_SIZE_K * stride_ak + b_ptrs += BLOCK_SIZE_K * stride_bk + + if C.dtype.element_ty == tl.bfloat16: + c = accumulator.to(tl.bfloat16) + elif C.dtype.element_ty == tl.float16: + c = accumulator.to(tl.float16) + else: + c = accumulator.to(tl.float32) + + offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] + c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) + tl.store(c_ptrs, c, mask=c_mask) + + +@functools.lru_cache +def get_w8a8_block_int8_configs( + N: int, K: int, block_n: int, block_k: int +) -> Optional[Dict[int, Any]]: + """ + Return optimized configurations for the w8a8 block fp8 kernel. + + The return value will be a dictionary that maps an irregular grid of + batch sizes to configurations of the w8a8 block fp8 kernel. To evaluate the + kernel on a given batch size bs, the closest batch size in the grid should + be picked and the associated configuration chosen to invoke the kernel. + """ + + # First look up if an optimized configuration is available in the configs + # directory + device_name = get_device_name().replace(" ", "_") + json_file_name = f"N={N},K={K},device_name={device_name},dtype=int8_w8a8,block_shape=[{block_n}, {block_k}].json" + + config_file_path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "configs", json_file_name + ) + if os.path.exists(config_file_path): + with open(config_file_path) as f: + logger.info( + "Using configuration from %s for W8A8 Block INT8 kernel.", + config_file_path, + ) + # If a configuration has been found, return it + return {int(key): val for key, val in json.load(f).items()} + + # If no optimized configuration is available, we will use the default + # configuration + logger.warning( + ( + "Using default W8A8 Block INT8 kernel config. Performance might be sub-optimal! " + "Config file not found at %s" + ), + config_file_path, + ) + return None + + +def w8a8_block_int8_matmul( + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + block_size: List[int], + output_dtype: torch.dtype = torch.float16, +) -> torch.Tensor: + """This function performs matrix multiplication with block-wise quantization. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + + Args: + A: The input tensor, e.g., activation. + B: The input tensor, e.g., weight. + As: The per-token-group quantization scale for `A`. + Bs: The per-block quantization scale for `B`. + block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + output_dytpe: The dtype of the returned tensor. + + Returns: + torch.Tensor: The result of matmul. + """ + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + + assert A.shape[-1] == B.shape[-1] + assert A.shape[:-1] == As.shape[:-1] and A.is_contiguous() + assert triton.cdiv(A.shape[-1], block_k) == As.shape[-1] + M = A.numel() // A.shape[-1] + + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + N, K = B.shape + assert triton.cdiv(N, block_n) == Bs.shape[0] + assert triton.cdiv(K, block_k) == Bs.shape[1] + + C_shape = A.shape[:-1] + (N,) + C = A.new_empty(C_shape, dtype=output_dtype) + + configs = get_w8a8_block_int8_configs(N, K, block_size[0], block_size[1]) + if configs: + # If an optimal configuration map has been found, look up the + # optimal config + config = configs[min(configs.keys(), key=lambda x: abs(x - M))] + else: + # Default config + # Block-wise quant: BLOCK_SIZE_K must be divisible by block_size[1] + config = { + "BLOCK_SIZE_M": 64, + "BLOCK_SIZE_N": block_size[0], + "BLOCK_SIZE_K": block_size[1], + "GROUP_SIZE_M": 32, + "num_warps": 4, + "num_stages": 3, + } + + def grid(META): + return ( + triton.cdiv(M, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + ) + + _w8a8_block_int8_matmul[grid]( + A, + B, + C, + As, + Bs, + M, + N, + K, + block_n, + block_k, + A.stride(-2), + A.stride(-1), + B.stride(1), + B.stride(0), + C.stride(-2), + C.stride(-1), + As.stride(-2), + As.stride(-1), + Bs.stride(1), + Bs.stride(0), + **config, + ) + + return C diff --git a/python/sglang/srt/layers/quantization/int8_utils.py b/python/sglang/srt/layers/quantization/int8_utils.py new file mode 100644 index 000000000..9a7aa6d82 --- /dev/null +++ b/python/sglang/srt/layers/quantization/int8_utils.py @@ -0,0 +1,73 @@ +from typing import List, Optional, Tuple + +import torch + +from sglang.srt.layers.quantization.int8_kernel import ( + per_token_group_quant_int8, + w8a8_block_int8_matmul, +) + + +def apply_w8a8_block_int8_linear( + input: torch.Tensor, + weight: torch.Tensor, + block_size: List[int], + weight_scale: torch.Tensor, + input_scale: Optional[torch.Tensor] = None, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + assert input_scale is None + # View input as 2D matrix for fp8 methods + input_2d = input.view(-1, input.shape[-1]) + output_shape = [*input.shape[:-1], weight.shape[0]] + + q_input, x_scale = per_token_group_quant_int8(input_2d, block_size[1]) + output = w8a8_block_int8_matmul( + q_input, weight, x_scale, weight_scale, block_size, output_dtype=input.dtype + ) + + if bias is not None: + output = output + bias + return output.to(dtype=input.dtype).view(*output_shape) + + +def input_to_int8( + x: torch.Tensor, dtype: torch.dtype = torch.int8 +) -> Tuple[torch.Tensor, torch.Tensor]: + """This function quantizes input values to int8 values with tensor-wise quantization.""" + iinfo = torch.iinfo(dtype) + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) + int8_min, int8_max = iinfo.min, iinfo.max + scale = int8_max / amax + x_scl_sat = (x * scale).clamp(min=int8_min, max=int8_max) + return x_scl_sat.to(dtype).contiguous(), scale.float().reciprocal() + + +def block_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], +) -> torch.Tensor: + """This function conducts block-wise dequantization. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The outputs are dequantized tensor. + """ + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = x_q_block.to(torch.float32) + + for i in range(k_tiles): + for j in range(n_tiles): + x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] *= x_s[j][i] + + return x_dq_block diff --git a/python/sglang/srt/layers/quantization/kv_cache.py b/python/sglang/srt/layers/quantization/kv_cache.py new file mode 100644 index 000000000..7da2dac17 --- /dev/null +++ b/python/sglang/srt/layers/quantization/kv_cache.py @@ -0,0 +1,82 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/kv_cache.py + +import logging + +import torch + +from sglang.srt.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.layers.radix_attention import RadixAttention + +logger = logging.getLogger(__name__) + + +class BaseKVCacheMethod(QuantizeMethodBase): + """ + Quant method that adds `k_scale` and `v_scale` attributes to the + Attention layer to support loading those scaling factors from checkpoints. + The k/v_scale will be used to: + - quantize k/v_cache entries before saving them to the cache + - dequantize k/v_cache entries before fetching them from the cache + + :param quant_config: the appropriate QuantizationConfig + """ + + def __init__(self, quant_config: QuantizationConfig): + self.quant_config = quant_config + + def create_weights(self, layer: torch.nn.Module): + """ + Create "weight" (aka k_scale and v_scale) for an attention layer. + """ + # Initialize the KV cache scales to -1.0, which is an invalid value. + # If the k/v_scale appears in the checkpoint, it will be + # overwritten when loading weights. + layer.k_scale = torch.nn.Parameter( + torch.tensor(-1.0, dtype=torch.float32), requires_grad=False + ) + layer.v_scale = torch.nn.Parameter( + torch.tensor(-1.0, dtype=torch.float32), requires_grad=False + ) + + def apply(self, layer: torch.nn.Module) -> torch.Tensor: + raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.") + + def process_weights_after_loading(self, layer: RadixAttention) -> None: + if layer.k_scale > 0.0 and layer.v_scale > 0.0: + # We prefer to use separate k_scale and v_scale if present + k_scale = layer.k_scale.to("cpu").tolist() + v_scale = layer.v_scale.to("cpu").tolist() + if is_fp8_fnuz(): + k_scale *= 2 + v_scale *= 2 + elif layer.k_scale < 0.0 and layer.v_scale < 0.0: + # If no scales were loaded (both scales are invalid negative + # values), use the default value of 1.0 + k_scale = 1.0 + v_scale = 1.0 + else: + # If we find a single kv_scale in the checkpoint, we remap + # kv_scale to k_scale during weight loading, and duplicate + # k_scale to v_scale here + assert layer.k_scale > 0.0 + scale_to_duplicate = max(layer.k_scale, layer.v_scale) + k_scale = scale_to_duplicate.to("cpu").tolist() + v_scale = scale_to_duplicate.to("cpu").tolist() + if is_fp8_fnuz(): + k_scale *= 2 + v_scale *= 2 + + if not isinstance(k_scale, float) or not isinstance(v_scale, float): + raise ValueError( + "Only support per-tensor scaling factor " "for fp8 KV cache" + ) + + # These are used in the final Attention.forward() + layer.k_scale.copy_(k_scale) + layer.v_scale.copy_(v_scale) + layer.k_scale_float = k_scale + layer.v_scale_float = v_scale diff --git a/python/sglang/srt/layers/quantization/marlin_utils.py b/python/sglang/srt/layers/quantization/marlin_utils.py new file mode 100644 index 000000000..e0b398c25 --- /dev/null +++ b/python/sglang/srt/layers/quantization/marlin_utils.py @@ -0,0 +1,808 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/utils/marlin_utils.py + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Optional + +import numpy +import torch + +from sglang.srt.layers.parameter import ( + BasevLLMParameter, + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + PackedvLLMParameter, +) +from sglang.srt.layers.quantization.base_config import ( + LinearMethodBase, + QuantizationConfig, +) +from sglang.srt.layers.quantization.utils import ( + get_scalar_types, + pack_cols, + unpack_cols, +) +from sglang.srt.utils import get_device_capability, is_cuda + +if TYPE_CHECKING: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + +try: + from vllm import _custom_ops as ops +except ImportError: + ops = None + +_is_cuda = is_cuda() + +if _is_cuda: + from sgl_kernel import gptq_marlin_gemm + +logger = logging.getLogger(__name__) + +ScalarType, scalar_types = get_scalar_types() + +GPTQ_MARLIN_TILE = 16 +GPTQ_MARLIN_MIN_THREAD_N = 64 +GPTQ_MARLIN_MIN_THREAD_K = 128 +GPTQ_MARLIN_MAX_PARALLEL = 16 + +MARLIN_SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + +# In case there is a performance issue with Marlin, the variable below can be +# changed to False, which allows Marlin to perform global reductions in fp16 +# precision (instead of fp32), and therefore, save on some memory movements. +USE_FP32_REDUCE_DEFAULT = True + + +# For binary size and compile time, we don't support the same types for with and +# without runtime zero-point. We support common cases, i.e. AWQ and GPTQ. +# TODO: we may want to move this into the C++ so its closer to the actual impl +def query_marlin_supported_quant_types( + has_zp: Optional[bool] = None, + include_fp_type: bool = True, + device_capability: Optional[int] = None, +): + if device_capability is None: + major, minor = get_device_capability() + capability = major * 10 + minor + device_capability = -1 if capability is None else capability + + if device_capability < 80: + return [] + + # - has_zp is True: return quant_types that has zero points + # - has_zp is False: return quant_types that has not zero points + # - has_zp is None: both + if has_zp is None: + types0 = query_marlin_supported_quant_types( + False, include_fp_type, device_capability + ) + types1 = query_marlin_supported_quant_types( + True, include_fp_type, device_capability + ) + return types0 + types1 + + if has_zp: + # AWQ style, unsigned + runtime zero-point + return [scalar_types.uint4] + else: + # GPTQ style, unsigned + symmetric bias + res = [scalar_types.uint4b8, scalar_types.uint8b128] + if include_fp_type: + res += [scalar_types.float8_e4m3fn, scalar_types.float4_e2m1f] + return res + + +def _check_marlin_supported( + quant_type: ScalarType, + group_size: Optional[int], + has_zp: bool, + device_capability: Optional[int] = None, +) -> tuple[bool, Optional[str]]: + + if device_capability is None: + major, minor = get_device_capability() + capability = major * 10 + minor + device_capability = -1 if capability is None else capability + + supported_types = query_marlin_supported_quant_types( + has_zp, True, device_capability + ) + + if quant_type not in supported_types: + return ( + False, + f"Marlin does not support weight_bits = {quant_type}. " + f"Only types = {supported_types} " + f"are supported (for group_size = {group_size}, " + f"device_capability = {device_capability}, zp = {has_zp}).", + ) + if group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES: + return ( + False, + f"Marlin does not support group_size = {group_size}. " + f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} " + "are supported.", + ) + + return True, None + + +def check_marlin_supported( + quant_type: ScalarType, + group_size: int, + has_zp: bool = False, + device_capability: Optional[int] = None, +) -> bool: + cond, _ = _check_marlin_supported(quant_type, group_size, has_zp, device_capability) + return cond + + +def verify_marlin_supported( + quant_type: ScalarType, group_size: int, has_zp: bool = False +) -> None: + cond, err_msg = _check_marlin_supported(quant_type, group_size, has_zp) + if not cond: + assert err_msg is not None + raise ValueError(err_msg) + + +def verify_marlin_supports_shape( + output_size_per_partition: int, + input_size_per_partition: int, + input_size: int, + group_size: int, +) -> None: + + # Validate output_size_per_partition + if output_size_per_partition % GPTQ_MARLIN_MIN_THREAD_N != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f" min_thread_n = {GPTQ_MARLIN_MIN_THREAD_N}. " + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq." + ) + + # Validate input_size_per_partition + if input_size_per_partition % GPTQ_MARLIN_MIN_THREAD_K != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible " + f"by min_thread_k = {GPTQ_MARLIN_MIN_THREAD_K}. " + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq." + ) + + if group_size < input_size and input_size_per_partition % group_size != 0: + raise ValueError( + f"Weight input_size_per_partition = {input_size_per_partition}" + f" is not divisible by group_size = {group_size}. " + "Consider reducing tensor_parallel_size or running " + "with --quantization gptq." + ) + + +def check_marlin_supports_shape( + output_size_per_partition: int, + input_size_per_partition: int, + input_size: int, + group_size: int, +) -> tuple[bool, Optional[str]]: + try: + verify_marlin_supports_shape( + output_size_per_partition, input_size_per_partition, input_size, group_size + ) + except ValueError as e: + return False, e.__str__() + return True, None + + +def check_marlin_supports_layer(layer: LinearBase, group_size: int) -> bool: + output_size_per_partition = ( + getattr(layer, "output_size_per_partition", None) or layer.output_size + ) + input_size_per_partition = ( + getattr(layer, "input_size_per_partition", None) or layer.input_size + ) + + return check_marlin_supports_shape( + output_size_per_partition=output_size_per_partition, + input_size_per_partition=input_size_per_partition, + input_size=layer.input_size, + group_size=group_size, + )[0] + + +def check_moe_marlin_supports_layer(layer: FusedMoE, group_size: int) -> bool: + hidden_size = layer.hidden_size + intermediate_size_per_partition = layer.intermediate_size_per_partition + # apply_router_weight_on_input is not supported for moe marlin + supports_router_weight = not layer.moe_runner_config.apply_router_weight_on_input + # moe marlin requires the activation to be silu + supports_activation = layer.moe_runner_config.activation == "silu" + + # gate-up: (n, k) = (intermediate_size_per_partition * 2, hidden_size) + # down: (n, k) = (hidden_size, intermediate_size_per_partition) + # moe marlin requires n % 128 == 0 and k % 64 == 0 + supports_shape = ( + hidden_size % 128 == 0 + and intermediate_size_per_partition % max(64, group_size) == 0 + ) + supports_group_size = group_size in [-1, 32, 64, 128] + return ( + supports_shape + and supports_group_size + and supports_router_weight + and supports_activation + ) + + +def marlin_make_workspace( + device: torch.device, max_blocks_per_sm: int = 1 +) -> torch.Tensor: + # In the new marlin kernel, we use the num of threadblocks as workspace + # size. The num of threadblocks is is sms_count * max_blocks_per_sm. + sms = torch.cuda.get_device_properties(device).multi_processor_count + return torch.zeros( + sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False + ) + + +def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool: + return (not act_order) or (act_order and not is_row_parallel) + + +def marlin_repeat_scales_on_all_ranks( + act_order: bool, group_size: int, is_row_parallel: bool +) -> bool: + # Need to repeat scales on every rank if act_ordering or + # channelwise and RowParallelLinear + is_channelwise = group_size == -1 + return act_order or (is_channelwise and is_row_parallel) + + +def marlin_make_empty_g_idx(device: torch.device) -> torch.Tensor: + return torch.nn.Parameter( + torch.empty(0, dtype=torch.int, device=device), requires_grad=False + ) + + +def marlin_make_empty_zp(device: torch.device) -> torch.Tensor: + return torch.nn.Parameter( + torch.empty(0, dtype=torch.int, device=device), requires_grad=False + ) + + +def marlin_sort_g_idx(g_idx: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + g_idx_sort_indices = torch.argsort(g_idx).to(torch.int) + return g_idx[g_idx_sort_indices], g_idx_sort_indices + + +def get_scale_perms(): + scale_perm: list[int] = [] + for i in range(8): + scale_perm.extend([i + 8 * j for j in range(8)]) + scale_perm_single: list[int] = [] + for i in range(4): + scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]]) + return scale_perm, scale_perm_single + + +def marlin_permute_scales( + s: torch.Tensor, size_k: int, size_n: int, group_size: int +) -> torch.Tensor: + + scale_perm, scale_perm_single = get_scale_perms() + if group_size < size_k and group_size != -1: + s = s.reshape((-1, len(scale_perm)))[:, scale_perm] + else: + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] + s = s.reshape((-1, size_n)).contiguous() + + return s + + +def marlin_permute_bias(s: torch.Tensor) -> torch.Tensor: + origin_shape = s.shape + _, scale_perm_single = get_scale_perms() + s = s.reshape((-1, len(scale_perm_single)))[:, scale_perm_single] + return s.reshape(*origin_shape).contiguous() + + +def marlin_moe_permute_scales( + s: torch.Tensor, + size_k: int, + size_n: int, + group_size: int, +): + num_experts = s.shape[0] + output = torch.empty( + (num_experts, s.shape[1], s.shape[2]), + device=s.device, + dtype=s.dtype, + ) + + for e in range(num_experts): + output[e] = marlin_permute_scales(s[e], size_k, size_n, group_size) + return output + + +def marlin_zero_points( + zp: torch.Tensor, size_k: int, size_n: int, num_bits: int +) -> torch.Tensor: + # Permute zero-points in a similar way to scales, but do not use the + # "single" permutation, since zero-points are applied on every MMA + scale_perm, _ = get_scale_perms() + zp = zp.reshape((-1, len(scale_perm)))[:, scale_perm] + + # Interleave column dim (for the dequantize code) and pack it to int32 + if num_bits == 4: + interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = numpy.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + zp = zp.reshape((-1, len(interleave)))[:, interleave].ravel() + zp = zp.reshape((-1, size_n)).contiguous() + zp = pack_cols(zp, num_bits, size_k, size_n) + + return zp + + +def awq_to_marlin_zero_points( + q_zp_packed: torch.Tensor, size_k: int, size_n: int, num_bits: int +) -> torch.Tensor: + # AWQ zero-points are quantized and packed on the column dim. + # In addition, the values are permuted based on dequantizer. + # Here we undo both of these, and then apply marlin permutation + # and pack it back. + q_zp = unpack_cols(q_zp_packed, num_bits, size_k, size_n) + + # Undo interleaving (use argsort(..) to get inverse perm) + if num_bits == 4: + undo_interleave = numpy.argsort(numpy.array([0, 2, 4, 6, 1, 3, 5, 7])) + elif num_bits == 8: + undo_interleave = numpy.argsort(numpy.array([0, 2, 1, 3])) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + q_zp = q_zp.reshape((-1, len(undo_interleave)))[:, undo_interleave].ravel() + q_zp = q_zp.reshape((-1, size_n)).contiguous() + + marlin_zp = marlin_zero_points(q_zp, size_k, size_n, num_bits) + return marlin_zp + + +def moe_awq_to_marlin_zero_points( + q_zp_packed: torch.Tensor, size_k: int, size_n: int, num_bits: int +): + num_experts = q_zp_packed.shape[0] + output = torch.empty( + (num_experts, q_zp_packed.shape[1], q_zp_packed.shape[2]), + device=q_zp_packed.device, + dtype=q_zp_packed.dtype, + ) + for e in range(num_experts): + output[e] = awq_to_marlin_zero_points(q_zp_packed[e], size_k, size_n, num_bits) + return output + + +def maybe_warn_marlin_atomic_add(device, dtype): + if torch.compiler.is_dynamo_compiling(): + return + device_capability = torch.cuda.get_device_capability(device) + if device_capability[0] < 9 and dtype == torch.bfloat16: + logger.info_once( + "You are running Marlin kernel with bf16 on GPUs before SM90. " + "You can consider change to fp16 to achieve better performance " + "if possible." + ) + + +def maybe_warn_marlin_atomic_add_env(): + if torch.compiler.is_dynamo_compiling(): + return + # TODO(yiyun): Need to add sglang's MARLIN_USE_ATOMIC_ADD: bool = False + if True: + return + # if envs.VLLM_MARLIN_USE_ATOMIC_ADD: + # return + logger.info_once( + "Marlin kernel can achieve better performance for small size_n " + "with experimental use_atomic_add feature. " + "You can consider set environment variable " + "VLLM_MARLIN_USE_ATOMIC_ADD to 1 if possible." + ) + + +def should_use_atomic_add_reduce( + m: int, n: int, k: int, device: torch.device, dtype: torch.dtype +) -> bool: + + # the performance of atomicAdd is better than global reduce + # only when m*n is small and k is large + if n >= 2048 or k < 2048 or device.type != "cuda": + return False + + # disable atomicAdd reduce by default, + # one can enable it with VLLM_MARLIN_USE_ATOMIC_ADD=1 + # TODO: Need to add sglang's MARLIN_USE_ATOMIC_ADD: bool = False + if not True: + maybe_warn_marlin_atomic_add_env() + return False + + # sm8x doesn't support atomicAdd + bfloat16 natively + device_capability = torch.cuda.get_device_capability(device) + if device_capability[0] < 9 and dtype == torch.bfloat16: + maybe_warn_marlin_atomic_add(device, dtype) + return False + + return True + + +def apply_gptq_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_zp: torch.Tensor, + g_idx: torch.Tensor, + g_idx_sort_indices: torch.Tensor, + workspace: torch.Tensor, + wtype: ScalarType, + output_size_per_partition: int, + input_size_per_partition: int, + is_k_full: bool, + bias: Optional[torch.Tensor] = None, + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT, +) -> torch.Tensor: + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (output_size_per_partition,) + + use_atomic_add = should_use_atomic_add_reduce( + m=reshaped_x.size(0), + n=output_size_per_partition, + k=reshaped_x.size(1), + device=input.device, + dtype=input.dtype, + ) + + output = gptq_marlin_gemm( + reshaped_x, + None, + weight, + weight_scale, + None, + weight_zp, + g_idx, + g_idx_sort_indices, + workspace, + wtype, + size_m=reshaped_x.shape[0], + size_n=output_size_per_partition, + size_k=input_size_per_partition, + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + is_zp_float=False, + ) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) + + +def apply_awq_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_zp: torch.Tensor, + g_idx: torch.Tensor, + g_idx_sort_indices: torch.Tensor, + workspace: torch.Tensor, + quant_type: ScalarType, + output_size_per_partition: int, + input_size_per_partition: int, + bias: Optional[torch.Tensor] = None, + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT, +) -> torch.Tensor: + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (output_size_per_partition,) + + use_atomic_add = should_use_atomic_add_reduce( + m=reshaped_x.size(0), + n=output_size_per_partition, + k=reshaped_x.size(1), + device=input.device, + dtype=input.dtype, + ) + + output = gptq_marlin_gemm( + reshaped_x, + None, + weight, + weight_scale, + None, + weight_zp, + g_idx, + g_idx_sort_indices, + workspace, + quant_type, + size_m=reshaped_x.shape[0], + size_n=output_size_per_partition, + size_k=input_size_per_partition, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + is_zp_float=False, + ) + + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) + + +class MarlinConfig(QuantizationConfig): + """Config class for Marlin. + + Reference: https://github.com/IST-DASLab/marlin/tree/master + """ + + def __init__( + self, + group_size: int, + lm_head_quantized: bool, + ) -> None: + super().__init__() + + # Group size for the quantization. + self.group_size = group_size + self.lm_head_quantized = lm_head_quantized + if self.group_size != 128 and self.group_size != -1: + raise ValueError( + "Currently, only group size 128 and -1 (channelwise) " + "is supported for Marlin, but got group_size of " + f"{self.group_size}" + ) + + # 4 Bits packed into 32 bit datatype. + self.pack_factor = 32 // 4 + + # Tile size used by marlin kernels. + self.tile_size = 16 + + # Min out_features dim + self.min_n_threads = 64 + + # Min in_features dim + self.min_k_threads = 128 + + # Max parallel problems to solve at once (improves large + # batch performance) + self.max_parallel = 16 + + # Permutation length used by the marlin kernels. + self.perm_len = 1024 + + def __repr__(self) -> str: + return ( + f"MarlinConfig(group_size={self.group_size}, " + f"lm_head_quantized={self.lm_head_quantized})" + ) + + @classmethod + def get_name(cls) -> str: + return "marlin" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.half] + + @classmethod + # Need to figure it out + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_config_filenames(cls) -> list[str]: + return ["quantize_config.json"] + + @classmethod + def from_config(cls, config: dict[str, Any]) -> "MarlinConfig": + group_size = cls.get_from_keys(config, ["group_size"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) + return cls(group_size, lm_head_quantized) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + # compat: autogptq >=0.8.0 use checkpoint_format: str + # compat: autogptq <=0.7.1 is_marlin_format: bool + is_marlin_format = hf_quant_cfg.get( + "checkpoint_format" + ) == "marlin" or hf_quant_cfg.get("is_marlin_format", False) + + is_valid_user_quant = ( + user_quant is None or user_quant == "gptq" or user_quant == "marlin" + ) + + if is_marlin_format and is_valid_user_quant: + msg = "The model is serialized in {} format. Using {} kernel.".format( + cls.get_name(), cls.get_name() + ) + logger.info(msg) + return cls.get_name() + + return None + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[MarlinLinearMethod]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead + + if isinstance(layer, LinearBase) or ( + isinstance(layer, ParallelLMHead) and self.lm_head_quantized + ): + return MarlinLinearMethod(self) + return None + + +class MarlinLinearMethod(LinearMethodBase): + """Linear method for Marlin. + + Args: + quant_config: The Marlin quantization config. + """ + + def __init__(self, quant_config: MarlinConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del output_size # Unused. + weight_loader = extra_weight_attrs["weight_loader"] + + if params_dtype != torch.float16: + raise ValueError( + f"The params dtype must be float16, but got {params_dtype}" + ) + + # Validate output_size_per_partition + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % self.quant_config.min_n_threads != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"min_n_threads = {self.quant_config.min_n_threads}." + ) + if output_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by " + f"pack_factor = {self.quant_config.pack_factor}." + ) + + # Validate input_size_per_partition + if input_size_per_partition % self.quant_config.min_k_threads != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"min_k_threads = {self.quant_config.min_k_threads}." + ) + if ( + self.quant_config.group_size != -1 + and input_size_per_partition % self.quant_config.group_size != 0 + ): + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"group_size = {self.quant_config.group_size}." + ) + + # Check that we have at least 4 tiles horizontally in the shard + num_tiles_per_perm = self.quant_config.perm_len // ( + self.quant_config.tile_size**2 + ) + if output_size_per_partition % num_tiles_per_perm != 0: + raise ValueError("Each permutation group must reside on the same gpu") + + # Quantized 4Bit weights packed into Int32. + qweight = PackedvLLMParameter( + data=torch.empty( + input_size_per_partition // self.quant_config.tile_size, + output_size_per_partition + * self.quant_config.tile_size + // self.quant_config.pack_factor, + device="cuda", + dtype=torch.int32, + ), + input_dim=0, + output_dim=1, + packed_dim=1, + packed_factor=self.quant_config.pack_factor, + marlin_tile_size=self.quant_config.tile_size, + weight_loader=weight_loader, + ) + + # Determine if channelwise or not + input_groups = ( + 1 + if self.quant_config.group_size == -1 + else input_size_per_partition // self.quant_config.group_size + ) + + weight_scale_args = { + "data": torch.empty( + input_groups, + output_size_per_partition, + device="cuda", + dtype=params_dtype, + ), + "weight_loader": weight_loader, + } + if input_groups == 1: + scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args) + else: + scales = GroupQuantScaleParameter( + output_dim=1, input_dim=0, **weight_scale_args + ) + + # Allocate workspace (Used for internal locking mechanism) + max_workspace_size = ( + output_size_per_partition // self.quant_config.min_n_threads + ) * self.quant_config.max_parallel + + workspace = BasevLLMParameter( + data=torch.zeros(max_workspace_size, device="cuda", dtype=torch.int), + weight_loader=weight_loader, + ) + + layer.register_parameter("B", qweight) + layer.register_parameter("s", scales) + layer.register_parameter("workspace", workspace) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # required by torch.compile + layer.B = torch.nn.Parameter(layer.B.data, requires_grad=False) + layer.s = torch.nn.Parameter(layer.s.data, requires_grad=False) + layer.workspace = torch.nn.Parameter(layer.workspace.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + qweight = layer.B + scales = layer.s + workspace = layer.workspace + + x_2d = x.view(-1, x.shape[-1]) + + size_m = x_2d.shape[0] + size_k = x_2d.shape[1] + size_n = scales.shape[1] + + output_2d = ops.marlin_gemm( + x_2d, qweight, scales, workspace, size_m, size_n, size_k + ) + + output = output_2d.view(x.shape[:-1] + (output_2d.shape[1],)) + + if bias is not None: + output.add_(bias) # In-place add + + return output diff --git a/python/sglang/srt/layers/quantization/marlin_utils_fp8.py b/python/sglang/srt/layers/quantization/marlin_utils_fp8.py new file mode 100644 index 000000000..94326d71e --- /dev/null +++ b/python/sglang/srt/layers/quantization/marlin_utils_fp8.py @@ -0,0 +1,352 @@ +# SPDX-License-Identifier: Apache-2.0 + +import logging +from typing import Optional + +import torch + +from sglang.srt.layers.quantization.marlin_utils import ( + USE_FP32_REDUCE_DEFAULT, + marlin_make_workspace, + marlin_permute_bias, + marlin_permute_scales, + should_use_atomic_add_reduce, +) +from sglang.srt.layers.quantization.utils import get_scalar_types +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() +if _is_cuda: + from sgl_kernel import gptq_marlin_gemm, gptq_marlin_repack + +ScalarType, scalar_types = get_scalar_types() + +logger = logging.getLogger(__name__) + + +def fp8_fused_exponent_bias_into_scales(scales): + fp8_exponent = 4 + if scales.dtype == torch.half: + target_exponent = 5 + elif scales.dtype == torch.bfloat16: + target_exponent = 8 + # exponent_bias_fp16 = 2 ** 4 - 2 ** 3 = 8 + # exponent_bias_bf16 = 2 ** 7 - 2 ** 3 = 120 + exponent_bias = 2 ** (target_exponent - 1) - 2 ** (fp8_exponent - 1) + s = torch.ones_like(scales) * 2 + s = s**exponent_bias + return scales * s + + +def apply_fp8_marlin_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + workspace: torch.Tensor, + size_n: int, + size_k: int, + bias: Optional[torch.Tensor], + use_fp32_reduce: bool = USE_FP32_REDUCE_DEFAULT, +) -> torch.Tensor: + # For GPUs that lack FP8 hardware support, we can leverage the + # Marlin kernel for fast weight-only FP8 quantization + + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (size_n,) + + use_atomic_add = should_use_atomic_add_reduce( + m=reshaped_x.size(0), n=size_n, k=size_k, device=input.device, dtype=input.dtype + ) + + output = gptq_marlin_gemm( + a=reshaped_x, + c=None, + b_q_weight=weight, + b_bias=bias, + b_scales=weight_scale, + global_scale=None, + b_zeros=None, + g_idx=None, + perm=None, + workspace=workspace, + b_q_type=scalar_types.float8_e4m3fn, + size_m=reshaped_x.size(0), + size_n=size_n, + size_k=size_k, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + ) + + return output.reshape(out_shape) + + +def prepare_fp8_layer_for_marlin( + layer: torch.nn.Module, size_k_first: bool = True +) -> None: + logger.warning_once( + "Your GPU does not have native support for FP8 computation but " + "FP8 quantization is being used. Weight-only FP8 compression will " + "be used leveraging the Marlin kernel. This may degrade " + "performance for compute-heavy workloads." + ) + + part_size_n = layer.output_size_per_partition + part_size_k = layer.input_size_per_partition + weight_block_size = getattr(layer, "weight_block_size", None) + + if size_k_first: + assert layer.weight.shape == (part_size_k, part_size_n) + else: + assert layer.weight.shape == (part_size_n, part_size_k) + + device = layer.weight.device + + # WORKSPACE + layer.workspace = marlin_make_workspace(device) + + # WEIGHT + # Repack weights to marlin format + perm = torch.empty(0, dtype=torch.int, device=device) + qweight = pack_fp8_to_int32(layer.weight, size_k_first) + if not size_k_first: + qweight = qweight.T.contiguous() + + marlin_qweight = gptq_marlin_repack( + b_q_weight=qweight, + perm=perm, + size_k=part_size_k, + size_n=part_size_n, + num_bits=8, + ) + layer.weight = torch.nn.Parameter(marlin_qweight, requires_grad=False) + + # WEIGHT SCALES + # Permute scales + if "weight_scale" in dir(layer): + scales = layer.weight_scale.to(layer.orig_dtype) + elif "weight_scale_inv" in dir(layer): + scales = layer.weight_scale_inv.to(layer.orig_dtype) + del layer.weight_scale_inv + + group_size = -1 if weight_block_size is None else weight_block_size[1] + + # marlin kernel only support channel-wise and group-wise quantization + # we need to convert the scales + if weight_block_size is None: + if scales.nelement() == 1: + # tensor-wise quantization -> channel-wise quantization + # (1, 1) =>(repeat)=> (1, size_n) + scales = scales.view(1, 1).repeat_interleave(part_size_n, 1) + elif scales.nelement() > 1 and scales.nelement() != part_size_n: + assert part_size_n % scales.nelement() == 0 + s_size = scales.nelement() + # tensor-wise quantization (for gate-up proj) + # -> channel-wise quantization + # (1, s_size) =>(repeat)=> (1, size_n) + scales = scales.view(1, s_size) + scales = scales.repeat_interleave(part_size_n // s_size, 1) + else: + # channel-wise quantization + # (1, size_n) + scales = scales.view(1, part_size_n) + else: + # block-wise quantization -> group-wise quantization + # (size_k // block_size[1], ceil(size_n / block_size[0])) + # =>(repeat)=> (size_k // block_size[1], size_n) + if not size_k_first: + scales = scales.T.contiguous() + block_n = weight_block_size[0] + scales = scales.repeat_interleave(block_n, 1) + # size_n may not divisible by block_size[0] + scales = scales[:, :part_size_n] + + marlin_scales = marlin_permute_scales( + s=scales, size_k=part_size_k, size_n=part_size_n, group_size=group_size + ) + marlin_scales = fp8_fused_exponent_bias_into_scales(marlin_scales) + layer.weight_scale = torch.nn.Parameter(marlin_scales, requires_grad=False) + + if hasattr(layer, "bias") and layer.bias is not None: + assert layer.bias.shape == (part_size_n,) + bias = marlin_permute_bias(layer.bias) + layer.bias = torch.nn.Parameter(bias, requires_grad=False) + + +def prepare_moe_fp8_layer_for_marlin( + layer: torch.nn.Module, size_k_first: bool = True +) -> None: + logger.warning_once( + "Your GPU does not have native support for FP8 computation but " + "FP8 quantization is being used. Weight-only FP8 compression will " + "be used leveraging the Marlin kernel. This may degrade " + "performance for compute-heavy workloads." + ) + + e = layer.num_experts + k = layer.hidden_size + n = layer.intermediate_size_per_partition + weight_block_size = getattr(layer, "weight_block_size", None) + + # WORKSPACE + device = layer.w13_weight.device + layer.workspace = marlin_make_workspace(device, 4) + perm = torch.empty(0, dtype=torch.int, device=device) + + # WEIGHT + # Repack weights to marlin format + for name in ["w13_weight", "w2_weight"]: + weight = getattr(layer, name) + tensor_list = [] + if "w13" in name: + size_n, size_k = n * 2, k + else: + size_n, size_k = k, n + + if size_k_first: + assert weight.shape == (e, size_k, size_n) + else: + assert weight.shape == (e, size_n, size_k) + + for i in range(e): + qweight = pack_fp8_to_int32(weight[i], size_k_first) + if not size_k_first: + qweight = qweight.T.contiguous() + + marlin_qweight = gptq_marlin_repack( + b_q_weight=qweight, perm=perm, size_k=size_k, size_n=size_n, num_bits=8 + ) + tensor_list.append(marlin_qweight) + + weight = torch.cat([x.unsqueeze(0) for x in tensor_list], 0) + weight = torch.nn.Parameter(weight, requires_grad=False) + + setattr(layer, name, weight) + + # WEIGHT SCALES + # Permute scales + group_size = -1 if weight_block_size is None else weight_block_size[1] + + for name in ["w13", "w2"]: + if name + "_weight_scale" in dir(layer): + new_name = name + "_weight_scale" + scales = getattr(layer, new_name).to(layer.orig_dtype) + delattr(layer, new_name) + elif name + "_weight_scale_inv" in dir(layer): + new_name = name + "_weight_scale_inv" + scales = getattr(layer, new_name).to(layer.orig_dtype) + delattr(layer, new_name) + + tensor_list = [] + if "w13" in name: + size_n, size_k = n * 2, k + else: + size_n, size_k = k, n + + # marlin kernel only support channel-wise and group-wise quantization + # we need to convert the scales + if weight_block_size is None: + if scales.nelement() == e: + # tensor-wise quantization -> channel-wise quantization + # (e, 1, 1) =>(repeat)=> (e, 1, size_n) + scales = scales.view(e, 1, 1).repeat_interleave(size_n, 2) + elif scales.nelement() > e and scales.nelement() != e * size_n: + assert (e * size_n) % scales.nelement() == 0 + s_size = scales.nelement() // e + # tensor-wise quantization (for gate-up proj) + # -> channel-wise quantization + # (e, 1, s_size) =>(repeat)=> (e, 1, size_n) + scales = scales.view(e, 1, s_size) + scales = scales.repeat_interleave(size_n // s_size, 2) + else: + # channel-wise quantization + # (e, 1, size_n) + scales = scales.view(e, 1, size_n) + else: + # block-wise quantization -> group-wise quantization + # (e, size_k // block_size[1], ceil(size_n / block_size[0])) + # =>(repeat)=> (e, size_k // block_size[1], size_n) + if not size_k_first: + scales = scales.permute(0, 2, 1) + block_n = weight_block_size[0] + scales = scales.repeat_interleave(block_n, 2) + # size_n may not divisible by block_size[0] + scales = scales[..., :size_n].contiguous() + + for i in range(e): + marlin_scales = marlin_permute_scales( + s=scales[i], size_k=size_k, size_n=size_n, group_size=group_size + ) + tensor_list.append(marlin_scales) + + scales = torch.cat([x.unsqueeze(0) for x in tensor_list], 0) + scales = fp8_fused_exponent_bias_into_scales(scales) + scales = torch.nn.Parameter(scales, requires_grad=False) + + setattr(layer, name + "_weight_scale", scales) + + # BIAS + # Permute bias + for name in ["w13_bias", "w2_bias"]: + if not hasattr(layer, name): + continue + bias = getattr(layer, name).to(layer.orig_dtype) + + tensor_list = [] + for i in range(e): + expert_bias = bias[i] + + tensor_list.append(marlin_permute_bias(expert_bias)) + + bias = torch.cat([x.unsqueeze(0) for x in tensor_list], 0) + bias = torch.nn.Parameter(bias, requires_grad=False) + setattr(layer, name, bias) + + +def pack_fp8_to_int32( + fp8_tensor: torch.Tensor, size_k_first: bool = True +) -> torch.Tensor: + """ + Repack FP8 weights to gptq format (packed int32 elements) + """ + assert fp8_tensor.dtype == torch.float8_e4m3fn + assert fp8_tensor.ndim == 2 + + fp8_tensor = fp8_tensor.T if size_k_first else fp8_tensor + fp8_tensor = fp8_tensor.contiguous() + # fp8_tensor is contiguous and have shape (N, K) now + # with `.view(torch.int32)`, it become (N, K // 4) + int32_tensor = fp8_tensor.view(torch.int32) + return int32_tensor.T.contiguous() if size_k_first else int32_tensor + + +def marlin_quant_fp8_torch(weight, group_size): + size_n, size_k = weight.shape + device = weight.device + + if group_size != -1: + scales = weight.view(size_n, -1, group_size).abs().max(-1)[0] / 448 + repeated_scales = scales.repeat_interleave(group_size, 1) + fp8_weight = (weight / repeated_scales).to(torch.float8_e4m3fn) + weight_ref = fp8_weight.to(weight.dtype) * repeated_scales + else: + scales = weight.view(size_n, 1, group_size).abs().max(-1)[0] / 448 + repeated_scales = scales.repeat_interleave(size_k, 1) + fp8_weight = (weight / repeated_scales).to(torch.float8_e4m3fn) + weight_ref = fp8_weight.to(weight.dtype) * repeated_scales + + packed_weight = pack_fp8_to_int32(fp8_weight, False).T.contiguous() + marlin_qweight = gptq_marlin_repack( + b_q_weight=packed_weight, + perm=torch.empty(0, dtype=torch.int, device=device), + size_k=size_k, + size_n=size_n, + num_bits=8, + ) + + marlin_scales = marlin_permute_scales( + s=scales.T, size_k=size_k, size_n=size_n, group_size=group_size + ) + + marlin_scales = fp8_fused_exponent_bias_into_scales(marlin_scales) + + return weight_ref.T, marlin_qweight, marlin_scales diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py new file mode 100755 index 000000000..89ecd44f5 --- /dev/null +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -0,0 +1,1402 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.distributed import get_tp_group +from sglang.srt.layers.dp_attention import get_dp_global_num_tokens, get_local_dp_buffer +from sglang.srt.layers.moe import ( + MoeRunner, + MoeRunnerBackend, + MoeRunnerConfig, + should_use_flashinfer_cutlass_moe_fp4_allgather, + should_use_flashinfer_trtllm_moe, +) +from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.parameter import ModelWeightParameter, PerTensorScaleParameter +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, + cutlass_fp8_supported, + is_sm100_supported, +) +from sglang.srt.layers.quantization.kv_cache import BaseKVCacheMethod +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import ( + convert_to_channelwise, + is_layer_skipped, + per_tensor_dequantize, + requantize_with_max_scale, +) +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.utils import is_cuda, next_power_of_2 + +if TYPE_CHECKING: + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +if is_cuda(): + from sgl_kernel import scaled_fp4_quant + +try: + from flashinfer import mm_fp4 as fp4_gemm + from flashinfer import reorder_rows_for_gated_act_gemm, shuffle_matrix_sf_a + + enable_flashinfer_fp4_gemm = True +except ImportError: + if is_cuda(): + from sgl_kernel import cutlass_scaled_fp4_mm as fp4_gemm + else: + fp4_gemm = None + enable_flashinfer_fp4_gemm = False + reorder_rows_for_gated_act_gemm = None + shuffle_matrix_a = None + shuffle_matrix_sf_a = None + +try: + from flashinfer.fused_moe import cutlass_fused_moe as flashinfer_cutlass_fused_moe +except ImportError: + flashinfer_cutlass_fused_moe = None + +# Initialize logger for the module +logger = logging.getLogger(__name__) + +# Supported activation schemes for the current configuration +ACTIVATION_SCHEMES = ["static"] + + +class ModelOptFp8Config(QuantizationConfig): + """Configuration for ModelOpt FP8 quantization, including serialization and compatibility checks.""" + + def __init__( + self, + is_checkpoint_fp8_serialized: bool = False, + kv_cache_quant_method: Optional[str] = None, + exclude_modules: Optional[List[str]] = None, + ) -> None: + """ + Args: + is_checkpoint_fp8_serialized (bool): Indicates if the checkpoint uses serialized FP8 format. + """ + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + self.kv_cache_quant_method = kv_cache_quant_method + self.exclude_modules = exclude_modules + if is_checkpoint_fp8_serialized: + logger.warning( + "Detected ModelOpt FP8 checkpoint. The format is experimental and subject to change." + ) + + @classmethod + def get_name(cls) -> str: + return "modelopt" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 89 # Minimum hardware capability (e.g., Hopper GPUs). + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["hf_quant_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> ModelOptFp8Config: + # Handle two different config formats: + # 1. hf_quant_config.json format: {"quantization": {"quant_algo": "FP8", ...}} + # 2. config.json quantization_config format: {"quant_algo": "FP8", ...} + # In future modelopt will deprecate hf_quant_config.json, and only keep config.json. + # For legacy reasons, we keep hf_quant_config.json for now. + + # Initialize variables + kv_cache_quant_method = None + exclude_modules = None + + # Try flat format first (config.json quantization_config - preferred format) + quant_method = config.get("quant_algo") + if quant_method is not None: + # Flat format (config.json quantization_config) + # For kv_cache, check if kv_cache_scheme exists and extract algo + kv_cache_scheme = config.get("kv_cache_scheme") + if ( + kv_cache_scheme + and kv_cache_scheme.get("type") == "float" + and kv_cache_scheme.get("num_bits") == 8 + ): + kv_cache_quant_method = "FP8" + + # Map 'ignore' field to 'exclude_modules' + exclude_modules = config.get("ignore") + else: + # Fall back to nested format (hf_quant_config.json - legacy format) + try: + quantization_section = cls.get_from_keys(config, ["quantization"]) + quant_method = quantization_section.get("quant_algo") + kv_cache_quant_method = quantization_section.get("kv_cache_quant_algo") + exclude_modules = quantization_section.get("exclude_modules") + except ValueError: + raise ValueError( + "Cannot find 'quant_algo' in the model's quantization config. " + "Expected either flat format (config.json) or nested format (hf_quant_config.json)." + ) + if quant_method is None: + raise ValueError( + "Cannot find 'quant_algo' in the model's quantization config. " + ) + if "FP8" not in quant_method: + raise ValueError( + "ModelOptFp8Config only supports static FP8 quantization in SGLang. " + "For FP4 quantization, use ModelOptFp4Config. " + "Check the quantization config for your model's configuration." + ) + + return cls( + is_checkpoint_fp8_serialized=True, + kv_cache_quant_method=kv_cache_quant_method, + exclude_modules=exclude_modules, + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if self.exclude_modules and any( + module in prefix + or ( + prefix.startswith("language_model.") + and module in prefix.removeprefix("language_model.") + ) + for module in self.exclude_modules + ): + return None + + if isinstance(layer, LinearBase): + return ModelOptFp8LinearMethod(self) + if self.kv_cache_quant_method and isinstance(layer, RadixAttention): + return ModelOptFp8KVCacheMethod(self) + + if isinstance(layer, FusedMoE): + return ModelOptFp8MoEMethod(self) + + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class ModelOptFp8LinearMethod(LinearMethodBase): + """Linear method for ModelOpt static FP8 quantization. + + Supports loading FP8 checkpoints with static weight and activation scales. + Future support may include dynamic scales. + + **Limitations**: + 1. Only supports per-tensor quantization due to `torch._scaled_mm` limitations. + 2. Only supports the `float8_e4m3fn` data type. + + Args: + quant_config (ModelOptFp8Config): The ModelOpt quantization configuration. + """ + + def __init__(self, quant_config: ModelOptFp8Config): + super().__init__() + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + """Creates and registers weights, weight scales, and input scales for FP8 quantization.""" + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + weight_dtype = ( + torch.float8_e4m3fn + if self.quant_config.is_checkpoint_fp8_serialized + else params_dtype + ) + + # Set layer attributes + layer.logical_widths = output_partition_sizes + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + + # Register weight + layer.register_parameter( + "weight", + ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ), + ) + + if self.quant_config.is_checkpoint_fp8_serialized: + # Register weight and input scales + for scale_name in ["weight_scale", "input_scale"]: + layer.register_parameter( + scale_name, + PerTensorScaleParameter( + data=torch.full( + (len(output_partition_sizes),), + torch.finfo(torch.float32).min, + dtype=torch.float32, + ), + weight_loader=weight_loader, + ), + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Requantizes weights after loading using the maximum scale.""" + max_w_scale, quantized_weight = requantize_with_max_scale( + layer.weight, layer.weight_scale, layer.logical_widths + ) + layer.weight = Parameter(quantized_weight.t(), requires_grad=False) + # cutlass sgl-kernel only supports per-channel scale + if self.cutlass_fp8_supported: + max_w_scale = convert_to_channelwise(max_w_scale, layer.logical_widths) + layer.weight_scale = Parameter(max_w_scale, requires_grad=False) + layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + """Applies FP8 linear transformation.""" + return apply_fp8_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + input_scale=layer.input_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + ) + + +class ModelOptFp8KVCacheMethod(BaseKVCacheMethod): + """ + Handles loading FP8 kv-cache scaling factors from modelopt quantized checkpoints. + """ + + def __init__(self, quant_config: ModelOptFp8Config): + super().__init__(quant_config) + + +class ModelOptFp8MoEMethod(FusedMoEMethodBase): + """MoE method for ModelOpt FP8. + Supports loading FP8 checkpoints with static weight scale and activation scale. + + Args: + quant_config: The ModelOpt quantization config. + """ + + def __init__(self, quant_config: ModelOptFp8Config): + self.quant_config = quant_config + self.cutlass_fp8_supported = cutlass_fp8_supported() + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + # Use FP8 dtype if checkpoint is serialized, otherwise use the default dtype + weight_dtype = ( + torch.float8_e4m3fn + if self.quant_config.is_checkpoint_fp8_serialized + else params_dtype + ) + weight_loader = extra_weight_attrs.get("weight_loader") + + w13_weight = ModelWeightParameter( + data=torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=weight_dtype, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight", w13_weight) + + w2_weight = ModelWeightParameter( + data=torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=weight_dtype, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight", w2_weight) + + if self.quant_config.is_checkpoint_fp8_serialized: + # WEIGHT SCALES - Per-tensor scaling for ModelOpts + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = PerTensorScaleParameter( + data=torch.full( + (num_experts, 2), + torch.finfo(torch.float32).min, + dtype=torch.float32, + ), + weight_loader=weight_loader, + ) + w2_weight_scale = PerTensorScaleParameter( + data=torch.full( + (num_experts,), torch.finfo(torch.float32).min, dtype=torch.float32 + ), + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + # Set weight loader attributes for scales + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value} + ) + + # INPUT SCALES - Per-tensor scaling for ModelOpt + w13_input_scale = PerTensorScaleParameter( + data=torch.full((num_experts,), 1.0, dtype=torch.float32), + weight_loader=weight_loader, + ) + w2_input_scale = PerTensorScaleParameter( + data=torch.full((num_experts,), 1.0, dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("w13_input_scale", w13_input_scale) + layer.register_parameter("w2_input_scale", w2_input_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Process FP8 MoE weights after loading from serialized checkpoint. + + Only supports pre-quantized checkpoints with FP8 weights and scales. + """ + + layer.w13_weight = Parameter(layer.w13_weight.data, requires_grad=False) + layer.w2_weight = Parameter(layer.w2_weight.data, requires_grad=False) + + # Handle scale parameters + if hasattr(layer, "w13_weight_scale") and layer.w13_weight_scale is not None: + # Fp8 moe kernel needs single weight scale for w13 per expert. + # We take the max of the w1 and w3 scales then dequant and requant each expert. + if layer.w13_weight_scale.dim() == 2: # Shape: (num_experts, 2) + from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant + + # Get the maximum scale across w1 and w3 for each expert + max_w13_scales = layer.w13_weight_scale.max(dim=1).values + + # Requantize each expert's weights using the combined scale + # w13_weight has shape (num_experts, 2 * intermediate_size_per_partition, hidden_size) + # where the first intermediate_size_per_partition rows are w1, the next are w3 + intermediate_size_per_partition = layer.w13_weight.shape[1] // 2 + for expert_id in range(layer.w13_weight.shape[0]): + start = 0 + for shard_id in range(2): # w1 and w3 + # Dequantize using the original scale for this shard + dq_weight = per_tensor_dequantize( + layer.w13_weight[expert_id][ + start : start + intermediate_size_per_partition, : + ], + layer.w13_weight_scale[expert_id][shard_id], + ) + # Requantize using the combined max scale + ( + layer.w13_weight[expert_id][ + start : start + intermediate_size_per_partition, : + ], + _, + ) = scaled_fp8_quant(dq_weight, max_w13_scales[expert_id]) + + start += intermediate_size_per_partition + + # Update the scale parameter to be per-expert instead of per-shard + layer.w13_weight_scale = Parameter(max_w13_scales, requires_grad=False) + else: + layer.w13_weight_scale = Parameter( + layer.w13_weight_scale.data, requires_grad=False + ) + + if hasattr(layer, "w2_weight_scale") and layer.w2_weight_scale is not None: + layer.w2_weight_scale = Parameter( + layer.w2_weight_scale.data, requires_grad=False + ) + if hasattr(layer, "w13_input_scale") and layer.w13_input_scale is not None: + layer.w13_input_scale = Parameter( + layer.w13_input_scale.max(), requires_grad=False + ) + if hasattr(layer, "w2_input_scale") and layer.w2_input_scale is not None: + layer.w2_input_scale = Parameter( + layer.w2_input_scale.max(), requires_grad=False + ) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_fp8_w8a8=True, + per_channel_quant=False, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + ) + + return self.runner.run(dispatch_output, quant_info) + + +class ModelOptFp4Config(QuantizationConfig): + """Config class for FP4.""" + + def __init__( + self, + is_checkpoint_nvfp4_serialized: bool = False, + kv_cache_quant_algo: str = None, + group_size: int = None, + exclude_modules: List[str] = None, + ) -> None: + self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized + if is_checkpoint_nvfp4_serialized: + logger.warning( + "Detected nvfp4 checkpoint. Please note that the " + "format is experimental and subject to change." + ) + self.group_size = group_size + self.kv_cache_quant_algo = kv_cache_quant_algo + self.exclude_modules = exclude_modules + + @classmethod + def get_name(cls) -> str: + return "modelopt_fp4" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half, torch.float8_e4m3fn] + + @classmethod + def get_min_capability(cls) -> int: + return 100 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["hf_quant_config.json"] + + @staticmethod + def common_group_size(cfg: dict) -> int: + """Return the unique group_size across the config; raise if missing/mismatched.""" + sizes = set() + + # Top-level and 'quantization' block + v = cfg.get("group_size") + if isinstance(v, int): + sizes.add(v) + q = cfg.get("quantization") + if isinstance(q, dict): + v = q.get("group_size") + if isinstance(v, int): + sizes.add(v) + + # config_groups: accept group-level or nested dicts (e.g., weights/input_activations) + for g in (cfg.get("config_groups") or {}).values(): + if isinstance(g, dict): + v = g.get("group_size") + if isinstance(v, int): + sizes.add(v) + for sub in g.values(): + if isinstance(sub, dict): + v = sub.get("group_size") + if isinstance(v, int): + sizes.add(v) + + if not sizes: + raise ValueError("No group_size found in config.") + if len(sizes) > 1: + raise ValueError(f"Inconsistent group_size values: {sorted(sizes)}") + return next(iter(sizes)) + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> ModelOptFp4Config: + # Handle two different config formats: + # 1. hf_quant_config.json format: {"quantization": {"quant_algo": "NVFP4", ...}} + # 2. config.json quantization_config format: {"quant_algo": "NVFP4", ...} + # In future modelopt will deprecate hf_quant_config.json, and only keep config.json. + # For legacy reasons, we keep hf_quant_config.json for now. + + # Initialize variables + kv_cache_quant_algo = None + group_size = None + exclude_modules = [] + + # Try flat format first (config.json quantization_config - preferred format) + quant_method = config.get("quant_algo") + if quant_method is not None: + # Flat format (config.json quantization_config) + # Note: FP4 models in config.json format may not have all the detailed fields + # that are present in hf_quant_config.json, so we need to handle defaults + kv_cache_quant_algo = config.get("kv_cache_quant_algo") + if not kv_cache_quant_algo: + # For config.json format, derive from kv_cache_scheme if available + kv_cache_scheme = config.get("kv_cache_scheme") + if ( + kv_cache_scheme + and kv_cache_scheme.get("type") == "float" + and kv_cache_scheme.get("num_bits") == 8 + ): + kv_cache_quant_algo = "FP8" + else: + kv_cache_quant_algo = "auto" + + group_size = ModelOptFp4Config.common_group_size(config) + exclude_modules = config.get("ignore", []) + else: + # Fall back to nested format (hf_quant_config.json - legacy format) + try: + quant_config = cls.get_from_keys(config, ["quantization"]) + quant_method = quant_config["quant_algo"] + kv_cache_quant_algo = quant_config.get("kv_cache_quant_algo") + if not kv_cache_quant_algo: + kv_cache_quant_algo = "auto" + group_size = ModelOptFp4Config.common_group_size(config) + exclude_modules = quant_config.get("exclude_modules", []) + except (ValueError, KeyError): + raise ValueError( + "Cannot find 'quant_algo' in the model's quantization config. " + "Expected either flat format (config.json) or nested format (hf_quant_config.json)." + ) + + if not quant_method in ["FP8", "NVFP4"]: + raise ValueError( + f"ModelOpt currently only supports: FP8, NVFP4" + " quantizations in sglang. Please check the " + "quantization config for your model's configuration." + ) + is_checkpoint_nvfp4_serialized = "NVFP4" in quant_method + + if not (group_size and kv_cache_quant_algo) or exclude_modules is None: + logger.warning( + f"group_size: {group_size}," + f"kv_cache_quant_algo: {kv_cache_quant_algo}," + f"exclude_modules: {exclude_modules}" + ) + raise ValueError( + "NVFP4 quantization requires group size and " + "kv_cache_quant_algo specified in the quantization config" + ) + return cls( + is_checkpoint_nvfp4_serialized, + kv_cache_quant_algo, + group_size, + exclude_modules, + ) + + def is_layer_excluded(self, prefix: str, exclude_modules: list): + import regex as re + + fused_patterns = ["q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj"] + prefix_split = prefix.split(".") + for pattern in exclude_modules: + regex_str = pattern.replace(".", r"\.").replace("*", r".*") + pattern_split = pattern.split(".") + if re.fullmatch(regex_str, prefix): + return True + elif ( + pattern_split[-1] in fused_patterns + and pattern_split[-1] in prefix_split[-1] + ): + # Check if the last part of the excluded pattern is contained in the last part of the prefix + # This handles fused modules like fused_qkv_a_proj_with_mqa that contain q_a_proj and kv_a_proj_with_mqa + # e.g., model.layers.{i}.self_attn.{fused_weight_name} + assert len(prefix_split) == 5 and len(pattern_split) == 5 + return True + return False + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.layers.moe.fused_moe_triton.layer import FlashInferFP4MoE + + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.exclude_modules) or self.is_layer_excluded( + prefix, self.exclude_modules + ): + return UnquantizedLinearMethod() + return ModelOptFp4LinearMethod(self) + if self.kv_cache_quant_algo and isinstance(layer, RadixAttention): + return ModelOptFp8KVCacheMethod(self) + elif isinstance(layer, FlashInferFP4MoE): + # FlashInferFP4MoE needs the same quantization method but with compatible attribute handling + return ModelOptNvFp4FusedMoEMethod(self) + elif isinstance(layer, FusedMoE): + return ModelOptNvFp4FusedMoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class ModelOptFp4LinearMethod(LinearMethodBase): + """Linear method for NVFP4. + Supports loading NVFP4 checkpoints with the following structure: + + |Tensor Name | datatype | shape | + |----------------------------------------------------| + |input_scale | torch.float32 | scalar | + |weight | NVFP4(SE2M1) | [1, X, y/2] | + |weight_scale | FP8-E4M3 | [X, Y] | + |weight_scale_2 | torch.float32 | scalar | + + The weights are quantized per block of 16 elements. + Args: quant_config: The ModelOpt quantization config. + """ + + def __init__(self, quant_config: ModelOptFp4Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del input_size, output_size + if not self.quant_config.is_checkpoint_nvfp4_serialized: + raise ValueError( + "NVFP4 quantization was selected, " + " dynamic quantization is not supported." + ) + + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + if input_size_per_partition % 16 != 0: + raise ValueError( + "Unsupported model when in features size is " "not multiple of 16" + ) + + weight_dtype = ( + torch.float8_e4m3fn + if self.quant_config.is_checkpoint_nvfp4_serialized + else params_dtype + ) + + weight = ModelWeightParameter( + data=torch.empty( + # 2 fp4 data is packed in one uint8 in the input dimension + output_size_per_partition, + input_size_per_partition // 2, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + input_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + + layer.register_parameter("input_scale", input_scale) + + weight_scale_2 = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale_2", weight_scale_2) + + weight_scale = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // self.quant_config.group_size, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + input_scale_2 = layer.input_scale.max().to(torch.float32) + weight_scale_2 = layer.weight_scale_2.max().to(torch.float32) + layer.input_scale = Parameter(input_scale_2, requires_grad=False) + layer.weight_scale_2 = Parameter(weight_scale_2, requires_grad=False) + layer.alpha = Parameter( + layer.input_scale * layer.weight_scale_2, requires_grad=False + ) + layer.input_scale_inv = Parameter( + (1 / input_scale_2).to(torch.float32), requires_grad=False + ) + + # Pad and blockwise interleave weight_scale + scales = layer.weight_scale + scale_ndim = scales.ndim + if scale_ndim == 2: + scales = scales.unsqueeze(0) + assert scales.ndim == 3 + B, M, K = scales.shape + round_up_multiple = lambda x, m: (x + m - 1) // m * m + M_padded = round_up_multiple(M, 128) + K_padded = round_up_multiple(K, 4) + padded_scales = torch.zeros((B, M_padded, K_padded), dtype=scales.dtype) + padded_scales[:B, :M, :K] = scales + batches, rows, cols = padded_scales.shape + assert rows % 128 == 0 + assert cols % 4 == 0 + padded_scales = padded_scales.reshape(batches, rows // 128, 4, 32, cols // 4, 4) + padded_scales = padded_scales.permute((0, 1, 4, 3, 2, 5)) + padded_scales = padded_scales.contiguous().cuda() + padded_scales = ( + padded_scales.reshape(M_padded, K_padded) + if scale_ndim == 2 + else padded_scales.reshape(B, M_padded, K_padded) + ) + layer.weight_scale_interleaved = Parameter(padded_scales, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + output_dtype = x.dtype + x_m, _ = x.shape + w_n, _ = layer.weight.shape + output_shape = [x_m, w_n] + + # Quantize BF16 or FP16 to (FP4 and interleaved block scale) + x_fp4, x_scale_interleaved = scaled_fp4_quant(x, layer.input_scale_inv) + + assert x_fp4.dtype == torch.uint8 + assert x_scale_interleaved.dtype == torch.float8_e4m3fn + assert layer.weight.dtype == torch.uint8 + assert layer.weight_scale_interleaved.dtype == torch.float8_e4m3fn + assert layer.alpha.dtype == torch.float32 + + w = layer.weight + w_scale_interleaved = layer.weight_scale_interleaved + if enable_flashinfer_fp4_gemm: + w = layer.weight.T + w_scale_interleaved = layer.weight_scale_interleaved.T + out = fp4_gemm( + x_fp4, + w, + x_scale_interleaved, + w_scale_interleaved, + layer.alpha, + output_dtype, + ) + if bias is not None: + out = out + bias + return out.view(*output_shape) + + +class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): + """ + MoE Method for FP4 Quantization with Blockscales and PerTensorScales + Args: + quant_config: NVFP4 Quant Config + """ + + def __init__(self, quant_config: ModelOptFp4Config): + self.quant_config = quant_config + if not is_sm100_supported(): + raise ValueError( + "Current platform does not support NVFP4" + " quantization. Please use Blackwell and" + " above." + ) + self.enable_flashinfer_trtllm_moe = should_use_flashinfer_trtllm_moe() + self._cache_permute_indices = {} + + @property + def enable_flashinfer_cutlass_moe(self) -> bool: + from sglang.srt.layers.moe import get_moe_runner_backend + + """Access the global enable_flashinfer_cutlass_moe setting.""" + return get_moe_runner_backend().is_flashinfer_cutlass() + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + if not self.quant_config.is_checkpoint_nvfp4_serialized: + raise ValueError( + "NVFP4 quantization was selected, " + " dynamic quantization is not supported." + ) + + # TODO(ch-wan): check if this is needed + layer.intermediate_size_per_partition = intermediate_size_per_partition + layer.params_dtype = params_dtype + layer.quant_config = self.quant_config + + weight_dtype = torch.uint8 + weight_scale_dtype = torch.float8_e4m3fn + weight_loader = extra_weight_attrs.get("weight_loader") + # GEMM 1 + w13_weight = ModelWeightParameter( + data=torch.empty( + layer.num_local_experts, + 2 * intermediate_size_per_partition, + # 2 fp4 items are packed in the input dimension + hidden_size // 2, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=2, + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight", w13_weight) + + # GEMM 2 + w2_weight = ModelWeightParameter( + data=torch.empty( + layer.num_local_experts, + hidden_size, + # 2 fp4 items are packed in the input dimension + intermediate_size_per_partition // 2, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=2, + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight", w2_weight) + + w13_weight_scale = ModelWeightParameter( + data=torch.empty( + layer.num_local_experts, + 2 * intermediate_size_per_partition, + hidden_size // self.quant_config.group_size, + dtype=weight_scale_dtype, + ), + input_dim=1, + output_dim=2, + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + + # Only use `swizzle_blockscale` for shapes, not for real content + layer.w13_blockscale_swizzled = Parameter( + self.swizzle_blockscale(layer.w13_weight_scale), requires_grad=False + ) + + w2_weight_scale = ModelWeightParameter( + data=torch.empty( + layer.num_local_experts, + hidden_size, + intermediate_size_per_partition // self.quant_config.group_size, + dtype=weight_scale_dtype, + ), + input_dim=1, + output_dim=2, + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + layer.w2_blockscale_swizzled = Parameter( + self.swizzle_blockscale(layer.w2_weight_scale), requires_grad=False + ) + + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value} + ) + + w13_weight_scale_2 = PerTensorScaleParameter( + data=torch.empty(layer.num_local_experts, 2, dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2) + + w2_weight_scale_2 = PerTensorScaleParameter( + data=torch.empty(layer.num_local_experts, dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight_scale_2", w2_weight_scale_2) + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value} + ) + + w13_input_scale = PerTensorScaleParameter( + data=torch.empty(layer.num_local_experts, 2, dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("w13_input_scale", w13_input_scale) + + w2_input_scale = PerTensorScaleParameter( + data=torch.empty(layer.num_local_experts, dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("w2_input_scale", w2_input_scale) + + def swizzle_blockscale(self, scale: torch.Tensor): + assert scale.dtype == torch.float8_e4m3fn + # Pad and blockwise interleave weight_scale + scale_ndim = scale.ndim + if scale.ndim == 2: + scale = scale.unsqueeze(0) + assert scale.ndim == 3 + B, M, K = scale.shape + round_up_multiple = lambda x, m: (x + m - 1) // m * m + M_padded = round_up_multiple(M, 128) + K_padded = round_up_multiple(K, 4) + padded_scale = torch.zeros((B, M_padded, K_padded), dtype=scale.dtype) + padded_scale[:B, :M, :K] = scale + batches, rows, cols = padded_scale.shape + assert rows % 128 == 0 + assert cols % 4 == 0 + padded_scale = padded_scale.reshape(batches, rows // 128, 4, 32, cols // 4, 4) + swizzled_scale = padded_scale.permute((0, 1, 4, 3, 2, 5)) + swizzled_scale = swizzled_scale.contiguous().cuda() + return ( + swizzled_scale.reshape(M_padded, K_padded) + if scale_ndim == 2 + else swizzled_scale.reshape(B, M_padded, K_padded) + ) + + def prepare_static_weights_for_kernel( + self, + # args_dequant, + # args, + gemm1_weights, + gemm2_weights, + gemm1_scales_linear_fp4_bytes, + gemm2_scales_linear_fp4_bytes, + hidden_size, + intermediate_size, + num_experts, + ): + from flashinfer import ( + RoutingMethodType, + e2m1_and_ufp8sf_scale_to_float, + fp4_quantize, + next_positive_power_of_2, + nvfp4_block_scale_interleave, + reorder_rows_for_gated_act_gemm, + shuffle_matrix_a, + shuffle_matrix_sf_a, + ) + from flashinfer.fused_moe.core import ( + _maybe_get_cached_w2_permute_indices, + _maybe_get_cached_w3_w1_permute_indices, + ) + + """Prepare quantized weights for kernel (done offline with weights).""" + epilogue_tile_m = 128 # FIXME: this depends on the kernel internals + + # Convert quantized weights to proper formats + gemm1_weights_fp4 = gemm1_weights.view(torch.float8_e4m3fn).reshape( + num_experts, 2 * intermediate_size, hidden_size // 2 + ) # packed fp4 + gemm1_scales_linear_fp4 = gemm1_scales_linear_fp4_bytes.view( + torch.float8_e4m3fn + ).reshape( + num_experts, 2 * intermediate_size, hidden_size // 16 + ) # fp8 scaling factors + + gemm2_weights_fp4 = gemm2_weights.view(torch.float8_e4m3fn).reshape( + num_experts, hidden_size, intermediate_size // 2 + ) # packed fp4 + gemm2_scales_linear_fp4 = gemm2_scales_linear_fp4_bytes.view( + torch.float8_e4m3fn + ).reshape( + num_experts, hidden_size, intermediate_size // 16 + ) # fp8 scaling factors + + gemm1_weights_fp4_shuffled = [] + gemm1_scales_fp4_shuffled = [] + gemm2_weights_fp4_shuffled = [] + gemm2_scales_fp4_shuffled = [] + for i in range(num_experts): + # Calculate the permute indices for the following: + # 1. Reorder rows of W1 and scales for fused gated activation + # 2. Shuffle weights and scaling factors for transposed mma output + # for both w3_w1 and w2 weights and scale factors + permute_indices = _maybe_get_cached_w3_w1_permute_indices( + self._cache_permute_indices, + gemm1_weights_fp4[i].view(torch.uint8), + epilogue_tile_m, + ) + gemm1_weights_fp4_shuffled.append( + gemm1_weights_fp4[i] + .view(torch.uint8)[permute_indices.to(gemm1_weights_fp4.device)] + .contiguous() + ) + + permute_sf_indices = _maybe_get_cached_w3_w1_permute_indices( + self._cache_permute_indices, + gemm1_scales_linear_fp4[i].view(torch.uint8), + epilogue_tile_m, + num_elts_per_sf=16, + ) + gemm1_scales_fp4_shuffled.append( + nvfp4_block_scale_interleave( + gemm1_scales_linear_fp4[i] + .view(torch.uint8)[ + permute_sf_indices.to(gemm1_scales_linear_fp4.device) + ] + .contiguous() + ) + ) + + permute_indices = _maybe_get_cached_w2_permute_indices( + self._cache_permute_indices, + gemm2_weights_fp4[i].view(torch.uint8), + epilogue_tile_m, + ) + gemm2_weights_fp4_shuffled.append( + gemm2_weights_fp4[i] + .view(torch.uint8)[permute_indices.to(gemm2_weights_fp4.device)] + .contiguous() + ) + + permute_sf_indices = _maybe_get_cached_w2_permute_indices( + self._cache_permute_indices, + gemm2_scales_linear_fp4[i].view(torch.uint8), + epilogue_tile_m, + num_elts_per_sf=16, + ) + gemm2_scales_fp4_shuffled.append( + nvfp4_block_scale_interleave( + gemm2_scales_linear_fp4[i] + .view(torch.uint8)[ + permute_sf_indices.to(gemm2_scales_linear_fp4.device) + ] + .contiguous() + ) + ) + + # Stack weights for all experts + gemm1_weights_fp4_shuffled = torch.stack(gemm1_weights_fp4_shuffled) + gemm1_scales_fp4_shuffled = ( + torch.stack(gemm1_scales_fp4_shuffled) + .view(torch.float8_e4m3fn) + .reshape(num_experts, 2 * intermediate_size, hidden_size // 16) + ) + + gemm2_weights_fp4_shuffled = torch.stack(gemm2_weights_fp4_shuffled) + gemm2_scales_fp4_shuffled = ( + torch.stack(gemm2_scales_fp4_shuffled) + .view(torch.float8_e4m3fn) + .reshape(num_experts, hidden_size, intermediate_size // 16) + ) + return ( + gemm1_weights_fp4_shuffled, + gemm1_scales_fp4_shuffled, + gemm2_weights_fp4_shuffled, + gemm2_scales_fp4_shuffled, + ) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Process FP4 MoE weights after loading from serialized checkpoint. + + Only supports pre-quantized checkpoints with FP8 weights and scales. + """ + + # GEMM 1 scale processing + if not torch.allclose( + layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1] + ): + logger.warning_once( + "w1_weight_scale_2 must match w3_weight_scale_2. " + "Accuracy may be affected." + ) + + w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0] + layer.w13_weight_scale_2 = Parameter(w13_weight_scale_2, requires_grad=False) + + # Calculate input scales based on strategy + if self.enable_flashinfer_cutlass_moe or self.enable_flashinfer_trtllm_moe: + w13_input_scale = layer.w13_input_scale.max().to(torch.float32) + w2_input_scale = layer.w2_input_scale.max().to(torch.float32) + else: + w13_input_scale = layer.w13_input_scale.max(dim=1).values.to(torch.float32) + w2_input_scale = layer.w2_input_scale + + # Create shared parameters + layer.g1_alphas = Parameter( + (w13_input_scale * w13_weight_scale_2).to(torch.float32), + requires_grad=False, + ) + layer.g2_alphas = Parameter( + (w2_input_scale * layer.w2_weight_scale_2).to(torch.float32), + requires_grad=False, + ) + layer.w13_input_scale_quant = Parameter( + (1 / w13_input_scale).to(torch.float32), requires_grad=False + ) + layer.w2_input_scale_quant = Parameter( + (1 / w2_input_scale).to(torch.float32), requires_grad=False + ) + + # Validate weight scales + for name, weight_scale in [ + ("w13", layer.w13_weight_scale), + ("w2", layer.w2_weight_scale), + ]: + assert ( + weight_scale.shape[2] % 16 == 0 + ), f"Expected {name}_weight_scale.dim(2) to be divisible by 16" + assert ( + weight_scale.dtype == torch.float8_e4m3fn + ), f"{name} Weight Blockscale must be represented as FP8-E4M3" + + # Weight processing based on strategy + if ( + self.enable_flashinfer_trtllm_moe + and reorder_rows_for_gated_act_gemm is not None + and shuffle_matrix_sf_a is not None + ): + # FlashInfer TRTLLM processing - handles both w13 and w2 + ( + gemm1_weights_fp4_shuffled, + gemm1_scales_fp4_shuffled, + gemm2_weights_fp4_shuffled, + gemm2_scales_fp4_shuffled, + ) = self.prepare_static_weights_for_kernel( + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale, + layer.w2_weight_scale, + layer.w2_weight.size(-2), # hidden_size + layer.w13_weight.size(-2) // 2, # intermediate_size + layer.w13_weight.size(0), # num_experts + ) + + # Set flashinfer parameters + layer.gemm1_weights_fp4_shuffled = Parameter( + gemm1_weights_fp4_shuffled, requires_grad=False + ) + layer.gemm2_weights_fp4_shuffled = Parameter( + gemm2_weights_fp4_shuffled, requires_grad=False + ) + layer.gemm1_scales_fp4_shuffled = Parameter( + gemm1_scales_fp4_shuffled, requires_grad=False + ) + layer.gemm2_scales_fp4_shuffled = Parameter( + gemm2_scales_fp4_shuffled, requires_grad=False + ) + + # Additional parameter needed for TRT-LLM + layer.g1_scale_c = Parameter( + (layer.w2_input_scale_quant * layer.g1_alphas).to(torch.float32), + requires_grad=False, + ) + + # Clean up weights that won't be used by TRT-LLM + del ( + layer.w2_weight, + layer.w2_weight_scale, + layer.w13_weight, + layer.w13_weight_scale, + ) + + else: + # CUTLASS processing - handle w13 and w2 separately + + # Process w13 weights + w13_blockscale_swizzled = self.swizzle_blockscale(layer.w13_weight_scale) + del layer.w13_weight_scale + layer.w13_blockscale_swizzled.data.copy_(w13_blockscale_swizzled) + layer.w13_weight = Parameter(layer.w13_weight.data, requires_grad=False) + + # Process w2 weights + w2_blockscale_swizzled = self.swizzle_blockscale(layer.w2_weight_scale) + del layer.w2_weight_scale + layer.w2_blockscale_swizzled.data.copy_(w2_blockscale_swizzled) + layer.w2_weight = Parameter(layer.w2_weight.data, requires_grad=False) + + # Both flashinfer cutlass and regular cutlass use same processing for w2 + + # Set up CUTLASS MoE parameters + device = layer.w13_weight.device + layer.cutlass_moe_params = CutlassMoEParams( + CutlassMoEType.BlockscaledFP4, + device, + num_experts=layer.num_experts, # global num experts + intermediate_size_per_partition=layer.w2_weight.shape[2] * 2, # n + hidden_size=layer.w13_weight.shape[2] * 2, + ) # k + + @property + def load_up_proj_weight_first(self) -> bool: + # FlashInfer CUTLASS kernel assumes [Up, Gate] Proj as W13 + return self.enable_flashinfer_cutlass_moe + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: FusedMoE, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + assert ( + self.moe_runner_config.activation == "silu" + ), "Only SiLU activation is supported." + + moe_runner_config = self.moe_runner_config + + # Check if this is a FlashInferFP4MoE layer that should handle its own forward + if hasattr(layer, "gemm1_weights_fp4_shuffled"): + # This layer was processed with flashinfer TRTLLM - delegate to its own forward + return StandardCombineInput(hidden_states=layer.forward(x, topk_output)) + + if self.enable_flashinfer_cutlass_moe: + assert ( + not moe_runner_config.apply_router_weight_on_input + ), "apply_router_weight_on_input is not supported for Flashinfer" + # TRTLLM Cutlass moe takes in activations in BF16/Half/nvfp4 precision + # and fp4 quantized weights loaded from the checkpoint + topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids + + output_dtype = x.dtype + x_sf = None + if should_use_flashinfer_cutlass_moe_fp4_allgather(): + from flashinfer import fp4_quantize, nvfp4_block_scale_interleave + + # Quantize before comm, swizzle after. + if x.shape[0] > 0: + x, x_sf = fp4_quantize( + x, layer.w13_input_scale_quant, is_sf_swizzled_layout=False + ) + else: + x_col = x.shape[1] + x = torch.zeros(0, x_col // 2, dtype=torch.uint8, device=x.device) + x_sf = torch.zeros( + 0, x_col // 16, dtype=torch.uint8, device=x.device + ) + topk_weights, topk_ids, x, x_sf = get_tp_group().all_gatherv( + [topk_weights, topk_ids, x, x_sf], sizes=get_dp_global_num_tokens() + ) + x_sf = nvfp4_block_scale_interleave(x_sf) + + output = flashinfer_cutlass_fused_moe( + input=x, + token_selected_experts=topk_ids.to(torch.int), + token_final_scales=topk_weights, + fc1_expert_weights=layer.w13_weight.view(torch.long), + fc2_expert_weights=layer.w2_weight.view(torch.long), + output_dtype=output_dtype, + input_sf=x_sf, + quant_scales=[ + layer.w13_input_scale_quant, + layer.w13_blockscale_swizzled.view(torch.int32), + layer.g1_alphas, + layer.w2_input_scale_quant, + layer.w2_blockscale_swizzled.view(torch.int32), + layer.g2_alphas, + ], + ep_size=layer.moe_ep_size, + ep_rank=layer.moe_ep_rank, + tp_size=layer.moe_tp_size, + tp_rank=layer.moe_tp_rank, + tune_max_num_tokens=next_power_of_2(x.shape[0]), + )[0] + if should_use_flashinfer_cutlass_moe_fp4_allgather(): + output, global_output = get_local_dp_buffer(), output + get_tp_group().reduce_scatterv( + global_output, output=output, sizes=get_dp_global_num_tokens() + ) + return StandardCombineInput(hidden_states=output) + + from sglang.srt.layers.moe.cutlass_moe import cutlass_moe_fp4 + + topk_weights, topk_ids = topk_output.topk_weights, topk_output.topk_ids + output = cutlass_moe_fp4( + a=x, + a1_gscale=layer.w13_input_scale_quant, + w1_fp4=layer.w13_weight, + w1_blockscale=layer.w13_blockscale_swizzled, + w1_alphas=layer.g1_alphas, + a2_gscale=layer.w2_input_scale_quant, + w2_fp4=layer.w2_weight, + w2_blockscale=layer.w2_blockscale_swizzled, + w2_alphas=layer.g2_alphas, + topk_weights=topk_weights, + topk_ids=topk_ids, + params=layer.cutlass_moe_params, + apply_router_weight_on_input=moe_runner_config.apply_router_weight_on_input, + ).to(x.dtype) + # Scale by routed_scaling_factor is fused into select_experts. + + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/moe_wna16.py b/python/sglang/srt/layers/quantization/moe_wna16.py new file mode 100644 index 000000000..531e4271f --- /dev/null +++ b/python/sglang/srt/layers/quantization/moe_wna16.py @@ -0,0 +1,501 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/moe_wna16.py +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import numpy as np +import torch + +from sglang.srt.distributed import get_tensor_model_parallel_rank +from sglang.srt.distributed.parallel_state import get_tp_group +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.quantization.awq import AWQConfig +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.gptq import GPTQConfig, GPTQMarlinConfig +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.utils import get_device_capability, set_weight_attrs + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + + +def get_weight_perm(num_bits: int): + perm_list: List[int] = [] + for i in range(32): + perm1: List[int] = [] + col = i // 4 + for block in [0, 1]: + for row in [ + 2 * (i % 4), + 2 * (i % 4) + 1, + 2 * (i % 4 + 4), + 2 * (i % 4 + 4) + 1, + ]: + perm1.append(16 * row + col + 8 * block) + for j in range(4): + perm_list.extend([p + 256 * j for p in perm1]) + + perm = np.array(perm_list) + + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +class MoeWNA16Config(QuantizationConfig): + """Config class for MOE WNA16 (W8A16/W4A16) quantization.""" + + def __init__( + self, + linear_quant_method: str, + weight_bits: int, + group_size: int, + has_zp: bool, + lm_head_quantized: bool, + modules_to_not_convert: Optional[List[str]], + full_config: Dict[str, Any], + ) -> None: + super().__init__() + self.weight_bits = weight_bits + self.group_size = group_size + self.has_zp = has_zp + self.bit8_pack_factor = 8 // self.weight_bits + self.lm_head_quantized = lm_head_quantized + self.linear_quant_method = linear_quant_method + self.full_config = full_config + self.use_marlin = False + # Avoid circular import + + if self.linear_quant_method == "gptq": + self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(full_config) + elif self.linear_quant_method == "awq": + capability_tuple = get_device_capability() + device_capability = ( + -1 + if capability_tuple is None + else capability_tuple[0] * 10 + capability_tuple[1] + ) + awq_min_capability = AWQConfig.get_min_capability() + if device_capability < awq_min_capability: + raise ValueError( + "The quantization method moe_wna16 + awq is not supported " + "for the current GPU. " + f"Minimum capability: {awq_min_capability}. " + f"Current capability: {device_capability}." + ) + else: + raise ValueError("moe_wna16 only support gptq and awq.") + + if modules_to_not_convert is None: + self.modules_to_not_convert = [] + else: + self.modules_to_not_convert = modules_to_not_convert + + @classmethod + def get_name(cls) -> str: + return "moe_wna16" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["quantize_config.json"] + + def get_scaled_act_names(self) -> List[str]: + raise NotImplementedError + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> MoeWNA16Config: + quant_method = cls.get_from_keys(config, ["quant_method"]) + weight_bits = cls.get_from_keys(config, ["bits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False) + if quant_method == "gptq": + has_zp = not cls.get_from_keys(config, ["sym"]) + modules_to_not_convert = [] + elif quant_method == "awq": + has_zp = cls.get_from_keys(config, ["zero_point"]) + modules_to_not_convert = cls.get_from_keys_or( + config, ["modules_to_not_convert"], None + ) + else: + raise ValueError("moe_wna16 only support gptq and awq.") + + return cls( + quant_method, + weight_bits, + group_size, + has_zp, + lm_head_quantized, + modules_to_not_convert, + config, + ) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + if user_quant == "moe_wna16" and cls.is_moe_wna16_compatible(hf_quant_cfg): + return cls.get_name() + return None + + @classmethod + def is_moe_wna16_compatible(cls, quant_config: Dict[str, Any]): + # Extract data from quant config. + quant_method = quant_config.get("quant_method", "").lower() + num_bits = quant_config.get("bits") + desc_act = quant_config.get("desc_act") + + capability_tuple = get_device_capability() + device_capability = ( + -1 + if all(capability is None for capability in capability_tuple) + else capability_tuple[0] * 10 + capability_tuple[1] + ) + # Avoid circular import + awq_min_capability = AWQConfig.get_min_capability() + + gptq_compatible = quant_method == "gptq" and not desc_act and num_bits in [4, 8] + awq_compatible = ( + quant_method == "awq" + and num_bits == 4 + and device_capability >= awq_min_capability + ) + + return gptq_compatible or awq_compatible + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + # avoid circular import + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + + if is_layer_skipped_quant(prefix, self.modules_to_not_convert): + return UnquantizedLinearMethod() + elif isinstance(layer, LinearBase): + + if self.linear_quant_method == "gptq": + if self.use_marlin: + return GPTQMarlinConfig.from_config( + self.full_config + ).get_quant_method(layer, prefix) + else: + return GPTQConfig.from_config(self.full_config).get_quant_method( + layer, prefix + ) + elif self.linear_quant_method == "awq": + return AWQConfig.from_config(self.full_config).get_quant_method( + layer, prefix + ) + else: + raise ValueError("moe_wna16 only support gptq and awq.") + elif isinstance(layer, FusedMoE): + return MoeWNA16Method(self) + return None + + +def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]): + return any(module_name in prefix for module_name in modules_to_not_convert) + + +class MoeWNA16Method(FusedMoEMethodBase): + """Linear method for MOE WNA16 (W8A16/W4A16) quantization. + + Args: + quant_config: The MOE WNA16 (W8A16/W4A16) quantization config. + """ + + def __init__(self, quant_config: MoeWNA16Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + layer.quant_config = self.quant_config + bit8_pack_factor = self.quant_config.bit8_pack_factor + group_size = self.quant_config.group_size + group_size_div_factor = 1 + + # make intermediate_size and hidden_size diviable by group_size + # we reduce the group size to ensure that + # and we would repeat the loaded_weight later + while intermediate_size_per_partition % group_size or hidden_size % group_size: + group_size = group_size // 2 + group_size_div_factor *= 2 + assert group_size >= 32 + layer.group_size = group_size + layer.group_size_div_factor = group_size_div_factor + + strategy = FusedMoeWeightScaleSupported.GROUP.value + extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False}) + + assert "weight_loader" in extra_weight_attrs + weight_loader = extra_weight_attrs["weight_loader"] + wrapped_weight_loader = MoeWNA16Method.get_weight_loader(layer, weight_loader) + extra_weight_attrs["weight_loader"] = wrapped_weight_loader + + # Fused gate_up_proj (column parallel) + w13_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qweight", w13_qweight) + set_weight_attrs(w13_qweight, extra_weight_attrs) + + # down_proj (row parallel) + w2_qweight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // bit8_pack_factor, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qweight", w2_qweight) + set_weight_attrs(w2_qweight, extra_weight_attrs) + + w13_scales = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_scales", w13_scales) + set_weight_attrs(w13_scales, extra_weight_attrs) + + w2_scales = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size, + intermediate_size_per_partition // group_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_scales", w2_scales) + set_weight_attrs(w2_scales, extra_weight_attrs) + + if self.quant_config.has_zp: + w13_qzeros = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition // bit8_pack_factor, + hidden_size // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_qzeros", w13_qzeros) + set_weight_attrs(w13_qzeros, extra_weight_attrs) + + w2_qzeros = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size // bit8_pack_factor, + intermediate_size_per_partition // group_size, + dtype=torch.uint8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_qzeros", w2_qzeros) + set_weight_attrs(w2_qzeros, extra_weight_attrs) + + if self.quant_config.linear_quant_method == "gptq": + # some param are unused, but we need to init them in order to + # load weights + invalid_param_keys = ["w13_g_idx", "w2_g_idx"] + if not self.quant_config.has_zp: + invalid_param_keys += ["w13_qzeros", "w2_qzeros"] + for key in invalid_param_keys: + param = torch.nn.Parameter( + torch.empty((0,), dtype=torch.int32), requires_grad=False + ) + layer.register_parameter(key, param) + set_weight_attrs(param, extra_weight_attrs) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + assert ( + self.moe_runner_config.activation == "silu" + ), "Only SiLU activation is supported." + + weight_bits = self.quant_config.weight_bits + has_zp = self.quant_config.has_zp + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_qweight, + w2_weight=layer.w2_qweight, + use_int4_w4a16=weight_bits == 4, + use_int8_w8a16=weight_bits == 8, + w13_scale=layer.w13_scales, + w2_scale=layer.w2_scales, + w13_zp=layer.w13_qzeros if has_zp else None, + w2_zp=layer.w2_qzeros if has_zp else None, + block_shape=[0, layer.group_size], + ) + return self.runner.run(dispatch_output, quant_info) + + @staticmethod + def get_weight_loader(layer, weight_loader): + + def convert_awq_tensor(tensor, tensor_type): + # convert awq qweight/qzeros to a standard format (assume int4) + # qweight: (k, n // pack_factor_bit32) -> (n, k // pack_factor_bit8) + # qzeros: (k // group_size, n // pack_factor_bit32) -> + # (n // pack_factor_bit8, k // group_size) + # pack_factor_bit32 = 32 // weight_bits + # pack_factor_bit8 = 8 // weight_bits + + # 0. suppose origin shape (a, b), dtype int32 + # 1. convert to uint8, shape (a, b) -> (a, 4 * b) + size0 = tensor.size(0) + tensor = tensor.view(torch.uint8) + + # 2. unpack to uint4 (only when weight_bits == 4) + # shape (a, 4 * b) -> (a, 4 * b, 2) + shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device) + tensor = (tensor[:, :, None] >> shifter) & 0xF + + # 3. change order, see + # https://github.com/casper-hansen/AutoAWQ/blob/v0.2.8/awq/utils/quant_utils.py + # shape -> (a, 4 * b * pack_factor_bit8) + reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7] + tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order] + tensor = tensor.view(size0, -1) + + # 4. transpose, shape -> (4 * b * pack_factor_bit8, a) + tensor = tensor.T.contiguous() + + # 5. repack (only when weight_bits == 4) + # qweight shape -> (4 * b * pack_factor_bit8, a // pack_factor_bit8) + # qzeros shape -> (4 * b, a) + + if tensor_type == "qweight": + tensor = tensor[:, 1::2] * 16 + tensor[:, ::2] + elif tensor_type == "qzeros": + tensor = tensor[1::2, :] * 16 + tensor[::2, :] + return tensor + + def convert_gptq_int4_qzeros(tensor): + tensor = tensor.view(torch.uint8) + shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device) + tensor = (tensor[:, :, None] >> shifter) & 0xF + tensor = tensor + 1 + tensor = tensor[:, :, 0] + tensor[:, :, 1] * 16 + return tensor + + def moe_wna16_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + ): + if "g_idx" in weight_name: + return + if not layer.quant_config.has_zp and "qzeros" in weight_name: + return + + device = get_tp_group().device + tp_rank = get_tensor_model_parallel_rank() + loaded_weight = loaded_weight.to(device) + shard_size = layer.intermediate_size_per_partition + + # convert gptq and awq weight to a standard format + if layer.quant_config.linear_quant_method == "awq": + assert layer.quant_config.weight_bits == 4 + if "weight" in weight_name: + loaded_weight = convert_awq_tensor(loaded_weight, "qweight") + elif "zeros" in weight_name: + loaded_weight = convert_awq_tensor(loaded_weight, "qzeros") + else: + loaded_weight = loaded_weight.T + elif layer.quant_config.linear_quant_method == "gptq": + assert layer.quant_config.weight_bits in [4, 8] + if "weight" in weight_name: + loaded_weight = loaded_weight.T.contiguous().view(torch.uint8) + elif "zeros" in weight_name: + # add 1 to gptq qzeros to align with awq + loaded_weight = loaded_weight.view(torch.uint8) + if layer.quant_config.weight_bits == 4: + loaded_weight = convert_gptq_int4_qzeros(loaded_weight).T + else: + loaded_weight = loaded_weight.T + 1 + else: + loaded_weight = loaded_weight.T + + # repeat the qzeros/scales to fit new group size + if ( + layer.group_size_div_factor > 1 + and "qzeros" in weight_name + or "scales" in weight_name + ): + loaded_weight = loaded_weight.repeat_interleave( + layer.group_size_div_factor, 1 + ) + + if "w13_qzeros" in weight_name: + tensor = loaded_weight.view( + layer.moe_tp_size, -1, loaded_weight.size(1) + )[tp_rank] + if shard_id == "w1": + param.data[expert_id, : shard_size // 2] = tensor + else: + param.data[expert_id, shard_size // 2 :] = tensor + elif "w2_qzeros" in weight_name: + param.data[expert_id] = loaded_weight.view( + loaded_weight.size(0), layer.moe_tp_size, -1 + )[:, tp_rank] + else: + weight_loader(param, loaded_weight, weight_name, shard_id, expert_id) + + return moe_wna16_weight_loader diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py new file mode 100644 index 000000000..0d98d00d6 --- /dev/null +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -0,0 +1,862 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/mxfp4.py + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.moe.utils import get_moe_runner_backend +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.utils import is_layer_skipped +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.utils import ( + direct_register_custom_op, + is_cuda, + is_flashinfer_available, + is_hip, + is_sm100_supported, + is_triton_kernels_available, + log_info_on_rank0, + mxfp_supported, + next_power_of_2, + round_up, + set_weight_attrs, +) + +_is_sm100_supported = is_cuda() and is_sm100_supported() +has_triton_kernels = is_triton_kernels_available() + + +if is_flashinfer_available(): + from flashinfer import ( + mxfp8_quantize, + shuffle_matrix_a, + shuffle_matrix_sf_a, + trtllm_fp4_block_scale_moe, + ) + +logger = logging.getLogger(__name__) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +_is_hip = is_hip() + +if _is_hip: + # import aiter + try: + from aiter import ActivationType, QuantType, dtypes + from aiter.fused_moe import fused_moe + from aiter.ops.triton.quant import dynamic_mxfp4_quant + from aiter.utility.fp4_utils import e8m0_shuffle + except ImportError as err: + ActivationType = QuantType = dtypes = fused_moe = dynamic_mxfp4_quant = ( + e8m0_shuffle + ) = err + + +def _swizzle_mxfp4(quant_tensor, scale, num_warps): + """weight swizzle for mxfp4 moe, used for OAI mxfp4 kernel""" + import triton_kernels.matmul_ogs_details.opt_flags as opt_flags + from triton_kernels.numerics import InFlexData + from triton_kernels.tensor import FP4, convert_layout, wrap_torch_tensor + from triton_kernels.tensor_details import layout + + value_layout, value_layout_opts = layout.make_default_matmul_mxfp4_w_layout( + mx_axis=1 + ) + scale_layout, scale_layout_opts = layout.make_default_matmul_mxfp4_w_scale_layout( + mx_axis=1, num_warps=num_warps + ) + if _is_sm100_supported: + constraints = { + "is_persistent": True, + "epilogue_subtile": 1, + } + opt_flags.update_opt_flags_constraints(constraints) + # transpose the tensor so that the quantization axis is on dim1 + quant_tensor = quant_tensor.transpose(-2, -1) + scale = scale.transpose(-2, -1) + quant_tensor = convert_layout( + wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts + ) + scale = convert_layout(wrap_torch_tensor(scale), scale_layout, **scale_layout_opts) + return quant_tensor, InFlexData(), scale + + +def _dequant_mxfp4( + x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype +) -> torch.Tensor: + try: + from quark.torch.kernel import mx + except ImportError as err: + raise ImportError( + "The package `amd-quark` is required to use " + "MX-FP4 models. Please install it with `pip install " + "amd-quark`." + ) from err + + return mx.dq_mxfp4(x, scale, float_dtype) + + +def _dequant_mxfp4_fake( + x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype +) -> torch.Tensor: + return torch.empty( + (*x.shape[:-1], x.shape[-1] * 2), dtype=float_dtype, device=x.device + ) + + +def _quant_dequant_mxfp4( + x: torch.Tensor, scale_calculation_mode: str = "even" +) -> torch.Tensor: + try: + from quark.torch.kernel import mx + except ImportError as err: + raise ImportError( + "The package `amd-quark` is required to use " + "MX-FP4 models. Please install it with `pip install " + "amd-quark`." + ) from err + + return mx.qdq_mxfp4(x, scale_calculation_mode) + + +def _quant_dequant_mxfp4_fake( + x: torch.Tensor, scale_calculation_mode: str = "even" +) -> torch.Tensor: + return torch.empty_like(x) + + +direct_register_custom_op( + op_name="dequant_mxfp4", + op_func=_dequant_mxfp4, + mutates_args=[], + fake_impl=_dequant_mxfp4_fake, +) +dequant_mxfp4 = torch.ops.sglang.dequant_mxfp4 + +direct_register_custom_op( + op_name="quant_dequant_mxfp4", + op_func=_quant_dequant_mxfp4, + mutates_args=[], + fake_impl=_quant_dequant_mxfp4_fake, +) +quant_dequant_mxfp4 = torch.ops.sglang.quant_dequant_mxfp4 + + +class Mxfp4Config(QuantizationConfig): + + def __init__( + self, + ignored_layers: Optional[list[str]] = None, + is_checkpoint_mxfp4_serialized: bool = False, + ): + super().__init__() + self.is_checkpoint_mxfp4_serialized = is_checkpoint_mxfp4_serialized + self.ignored_layers = ignored_layers + + @classmethod + def from_config(cls, config): + + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_mxfp4_serialized = "mxfp4" in quant_method + + if _is_hip: + if mxfp_supported(): + return cls( + is_checkpoint_mxfp4_serialized=is_checkpoint_mxfp4_serialized + ) + else: + + platform = torch.cuda.get_device_properties(0).gcnArchName + raise ValueError( + f"Current platform {platform} not support mxfp4 computation" + ) + + return cls(is_checkpoint_mxfp4_serialized=is_checkpoint_mxfp4_serialized) + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_name(cls) -> str: + return "mxfp4" + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.bfloat16, torch.float16] + + @classmethod + def get_config_filenames(cls) -> list[str]: + return [] + + def is_static_cfg(self): + return self.is_checkpoint_mxfp4_serialized + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional["QuantizeMethodBase"]: + + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod + + if isinstance(layer, LinearBase): + if self.ignored_layers and is_layer_skipped( + prefix=prefix, + ignored_layers=self.ignored_layers, + fused_mapping=self.packed_modules_mapping, + ): + return UnquantizedLinearMethod() + elif _is_hip: + return UnquantizedLinearMethod() + elif isinstance(layer, FusedMoE): + if self.is_checkpoint_mxfp4_serialized: + return Mxfp4MoEMethod(prefix=prefix) + else: + return Mxfp4DynamicQuantMoEMethod() + else: + if self.is_checkpoint_mxfp4_serialized: + raise NotImplementedError("Mxfp4 attention layer is not implemented") + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class Mxfp4MoEMethod(FusedMoEMethodBase): + + def __init__( + self, + prefix: str, + ): + super().__init__() + + self.prefix = prefix + self.topk_indices_dtype = None + self.use_triton_kernels = get_moe_runner_backend().is_triton_kernel() + self.with_bias = False + self.use_flashinfer = get_moe_runner_backend().is_flashinfer_mxfp4() + self.flashinfer_mxfp4_moe_precision = global_server_args_dict[ + "flashinfer_mxfp4_moe_precision" + ] + + self.triton_kernel_moe_forward = None + self.triton_kernel_moe_with_bias_forward = None + if torch.cuda.is_available() and has_triton_kernels: + from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_forward as _tk_forward, + ) + from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_with_bias_forward as _tk_with_bias_forward, + ) + + self.triton_kernel_moe_forward = _tk_forward + self.triton_kernel_moe_with_bias_forward = _tk_with_bias_forward + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + with_bias: bool = False, + **extra_weight_attrs, + ): + self.num_experts = num_experts + weight_dtype = torch.uint8 + scale_dtype = torch.uint8 + self.with_bias = with_bias + mxfp4_block = 32 + + # pad the intermediate size to be a multiple of 2 * mxfp4_block + # for to hold non-uniform sharded tensor as well as swizzling + intermediate_size_per_partition_after_pad = intermediate_size_per_partition + if _is_sm100_supported: + if self.use_flashinfer: + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, 256 + ) + hidden_size = round_up(hidden_size, 256) + else: + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, 64 + ) + elif has_triton_kernels: + # TODO: this is a hack to make + # intermediate_size_per_partition_after_pad the same as the + # per_rank_intermediate_size during weight loading + intermediate_size_per_partition_after_pad = round_up( + intermediate_size_per_partition, mxfp4_block + ) + + self.intermediate_size_per_partition = intermediate_size_per_partition_after_pad + + self.hidden_size = hidden_size + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter( + torch.zeros( + layer.num_local_experts, + 2 * intermediate_size_per_partition_after_pad, + hidden_size // 2, + dtype=weight_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.zeros( + layer.num_local_experts, + 2 * intermediate_size_per_partition_after_pad, + hidden_size // mxfp4_block, + dtype=scale_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + w13_weight_bias = torch.nn.Parameter( + torch.zeros( + layer.num_local_experts, + 2 * intermediate_size_per_partition_after_pad, + dtype=torch.bfloat16, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_bias", w13_weight_bias) + set_weight_attrs(w13_weight_bias, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter( + torch.zeros( + layer.num_local_experts, + hidden_size, + intermediate_size_per_partition_after_pad // 2, + dtype=weight_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w2_weight_scale = torch.nn.Parameter( + torch.zeros( + layer.num_local_experts, + hidden_size, + intermediate_size_per_partition_after_pad // mxfp4_block, + dtype=scale_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w2_weight_bias = torch.nn.Parameter( + torch.zeros(layer.num_local_experts, hidden_size, dtype=torch.bfloat16), + requires_grad=False, + ) + layer.register_parameter("w2_weight_bias", w2_weight_bias) + set_weight_attrs(w2_weight_bias, extra_weight_attrs) + + def process_weights_after_loading(self, layer): + if self.use_flashinfer: + log_info_on_rank0( + logger, + f"Shuffling MoE weights for FlashInfer MXFP4 moe kernel (layer: {self.prefix}), it might take a while...", + ) + # TODO: these values are hardcoded for now, we need to get them from the model + layer.gemm1_alpha = Parameter( + torch.tensor([1.702] * self.num_experts, dtype=torch.float32).cuda(), + requires_grad=False, + ) + layer.gemm1_beta = Parameter( + torch.tensor([1.0] * self.num_experts, dtype=torch.float32).cuda(), + requires_grad=False, + ) + layer.gemm1_clamp_limit = Parameter( + torch.tensor([7.0] * self.num_experts, dtype=torch.float32).cuda(), + requires_grad=False, + ) + sf_block_size = 32 # mxfp4 block size + + assert ( + layer.w13_weight.dim() == 3 + and layer.w13_weight.shape[0] == self.num_experts + and layer.w13_weight.shape[1] + == self.intermediate_size_per_partition * 2 + and layer.w13_weight.shape[2] == self.hidden_size // 2 + ) + assert ( + layer.w13_weight_scale.dim() == 3 + and layer.w13_weight_scale.shape[0] == self.num_experts + and layer.w13_weight_scale.shape[1] + == self.intermediate_size_per_partition * 2 + and layer.w13_weight_scale.shape[2] == self.hidden_size // sf_block_size + ) + assert ( + layer.w2_weight.dim() == 3 + and layer.w2_weight.shape[0] == self.num_experts + and layer.w2_weight.shape[1] == self.hidden_size + and layer.w2_weight.shape[2] + == self.intermediate_size_per_partition // 2 + ) + assert ( + layer.w2_weight_scale.dim() == 3 + and layer.w2_weight_scale.shape[1] == self.hidden_size + and layer.w2_weight_scale.shape[2] + == self.intermediate_size_per_partition // sf_block_size + ) + assert ( + layer.w13_weight_bias.dim() == 2 + and layer.w13_weight_bias.shape[0] == self.num_experts + and layer.w13_weight_bias.shape[1] + == self.intermediate_size_per_partition * 2 + ) + assert ( + layer.w2_weight_bias.dim() == 2 + and layer.w2_weight_bias.shape[0] == self.num_experts + and layer.w2_weight_bias.shape[1] == self.hidden_size + ) + + w13_weight_scale = layer.w13_weight_scale.data + w2_weight_scale = layer.w2_weight_scale.data + w13_weight = layer.w13_weight.data + w2_weight = layer.w2_weight.data + w13_bias = layer.w13_weight_bias.data.to(torch.float32) + w2_bias = layer.w2_weight_bias.data.to(torch.float32) + + # Swap w1 and w3 as the definition of + # swiglu is different in the trtllm-gen + def swap_every_two_rows(x, axis=-1): + shape = x.shape + if axis < 0: + axis = len(shape) + axis + + # Create a new shape with pairs swapped along specified axis + new_shape = list(shape) + new_shape[axis] = shape[axis] // 2 + new_shape.insert(axis + 1, 2) + + # Reshape to expose pairs, swap them, and reshape back + x = x.reshape(*new_shape) + x = x.flip(axis + 1) + new_shape = list(shape) + return x.reshape(*new_shape) + + w13_weight_scale = swap_every_two_rows(w13_weight_scale, -2) + w13_weight = swap_every_two_rows(w13_weight, -2) + w13_bias = swap_every_two_rows(w13_bias, -1) + + # Shuffle weights and scaling factors for transposed mma output + gemm1_weights_mxfp4_shuffled = [] + gemm1_scales_mxfp4_shuffled = [] + gemm2_weights_mxfp4_shuffled = [] + gemm2_scales_mxfp4_shuffled = [] + gemm1_bias_shuffled = [] + gemm2_bias_shuffled = [] + epilogue_tile_m = 128 # FIXME: this depends on the kernel internals + for i in range(self.num_experts): + gemm1_weights_mxfp4_shuffled.append( + shuffle_matrix_a(w13_weight[i].view(torch.uint8), epilogue_tile_m) + ) + gemm1_scales_mxfp4_shuffled.append( + shuffle_matrix_sf_a( + w13_weight_scale[i].view(torch.uint8), epilogue_tile_m + ) + ) + gemm1_bias_shuffled.append( + shuffle_matrix_a( + w13_bias[i].clone().reshape(-1, 1), epilogue_tile_m + ) + ) + + gemm2_weights_mxfp4_shuffled.append( + shuffle_matrix_a(w2_weight[i].view(torch.uint8), epilogue_tile_m) + ) + gemm2_scales_mxfp4_shuffled.append( + shuffle_matrix_sf_a( + w2_weight_scale[i].view(torch.uint8), epilogue_tile_m + ) + ) + gemm2_bias_shuffled.append( + shuffle_matrix_a(w2_bias[i].clone().reshape(-1, 1), epilogue_tile_m) + ) + + w13_weight = torch.stack(gemm1_weights_mxfp4_shuffled) + w13_weight_scale = ( + torch.stack(gemm1_scales_mxfp4_shuffled) + .reshape( + self.num_experts, + 2 * self.intermediate_size_per_partition, + self.hidden_size // sf_block_size, + ) + .view(torch.float8_e4m3fn) + ) + + w2_weight = torch.stack(gemm2_weights_mxfp4_shuffled) + w2_weight_scale = ( + torch.stack(gemm2_scales_mxfp4_shuffled) + .reshape( + self.num_experts, + self.hidden_size, + self.intermediate_size_per_partition // sf_block_size, + ) + .view(torch.float8_e4m3fn) + ) + + layer.w13_weight = Parameter(w13_weight, requires_grad=False) + layer.w13_weight_scale = Parameter(w13_weight_scale, requires_grad=False) + layer.w2_weight = Parameter(w2_weight, requires_grad=False) + layer.w2_weight_scale = Parameter(w2_weight_scale, requires_grad=False) + layer.w13_weight_bias = Parameter( + torch.stack(gemm1_bias_shuffled).reshape(self.num_experts, -1), + requires_grad=False, + ) + layer.w2_weight_bias = Parameter( + torch.stack(gemm2_bias_shuffled).reshape(self.num_experts, -1), + requires_grad=False, + ) + return + + if self.use_triton_kernels: + + from triton_kernels.matmul_ogs import FlexCtx, PrecisionConfig + + w13_weight_bias = layer.w13_weight_bias.to(torch.float32) + w2_weight_bias = layer.w2_weight_bias.to(torch.float32) + + layer.w13_weight_bias = Parameter(w13_weight_bias, requires_grad=False) + layer.w2_weight_bias = Parameter(w2_weight_bias, requires_grad=False) + + num_warps = 8 + + w13_weight, w13_flex, w13_scale = _swizzle_mxfp4( + layer.w13_weight, layer.w13_weight_scale, num_warps + ) + w2_weight, w2_flex, w2_scale = _swizzle_mxfp4( + layer.w2_weight, layer.w2_weight_scale, num_warps + ) + + self.w13_precision_config = PrecisionConfig( + weight_scale=w13_scale, flex_ctx=FlexCtx(rhs_data=w13_flex) + ) + self.w2_precision_config = PrecisionConfig( + weight_scale=w2_scale, flex_ctx=FlexCtx(rhs_data=w2_flex) + ) + + self.w13_weight_triton_tensor = w13_weight + self.w2_weight_triton_tensor = w2_weight + del layer.w13_weight + del layer.w2_weight + else: + from triton_kernels.numerics_details.mxfp import upcast_from_mxfp + + w13_weight = upcast_from_mxfp( + layer.w13_weight, layer.w13_weight_scale, dtype=torch.bfloat16, axis=-1 + ) + w2_weight = upcast_from_mxfp( + layer.w2_weight, layer.w2_weight_scale, dtype=torch.bfloat16, axis=-1 + ) + del layer.w13_weight + del layer.w2_weight + del layer.w13_weight_scale + del layer.w2_weight_scale + layer.w13_weight = Parameter(w13_weight.data, requires_grad=False) + layer.w2_weight = Parameter(w2_weight.data, requires_grad=False) + torch.cuda.empty_cache() + + def _get_tile_tokens_dim(self, x: torch.Tensor, top_k: int): + # Number of tokens in the input tensor. + num_tokens = x.shape[0] + # Factor to account for the imbalance of the experts. + # factor equals to the + # max_real_num_tokens_per_expert / perfect_num_tokens_per_expert + # - 1.0 means perfect expert distribution. + # - > 1.0 means some experts have more + # tokens than the perfect distribution. + # - < 1.0 does not make sense. + imbalance_factor = 1.3 + # Calculate the number of tokens per expert + # assuming perfect distribution. + num_tokens_per_expert = (num_tokens * top_k) // self.num_experts + # Apply the imbalance factor. + num_tokens_per_expert = int(num_tokens_per_expert * imbalance_factor) + # And pad the number to the next power of 2. + tile_tokens_dim = next_power_of_2(num_tokens_per_expert) + # Cap to 8-64 tokens per CTA tile + # as it's the range supported by the kernel. + tile_tokens_dim = min(max(tile_tokens_dim, 8), 64) + + return tile_tokens_dim + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + from sglang.srt.layers.moe.topk import TopKOutputChecker + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + moe_runner_config = self.moe_runner_config + + if self.use_flashinfer: + # When bf16 mode is enabled, we don't need to quantize the input, + # TRT-LLM automatically handles quantization in the kernel implementation and pipelines it with GEMM operations, + # which can theoretically improve performance + if self.flashinfer_mxfp4_moe_precision == "bf16": + assert x.dtype == torch.bfloat16 + x_quant = x + x_scale = None + + # May be fused later if this code branch is frequently needed + origin_hidden_states_dim = x_quant.shape[-1] + if self.hidden_size != origin_hidden_states_dim: + x_quant = torch.nn.functional.pad( + x_quant, + (0, self.hidden_size - origin_hidden_states_dim), + mode="constant", + value=0.0, + ) + elif self.flashinfer_mxfp4_moe_precision == "default": + x_quant, x_scale = mxfp8_quantize(x, False, alignment=self.hidden_size) + x_scale = x_scale.view(torch.float8_e4m3fn).reshape(-1) + else: + raise NotImplementedError + + assert x_quant.shape[-1] == self.hidden_size + assert TopKOutputChecker.format_is_bypassed(topk_output) + + top_k = topk_output.topk_config.top_k + router_logits = topk_output.router_logits + + trtllm_gen_output = trtllm_fp4_block_scale_moe( + router_logits.to(torch.bfloat16), + None, # routing_bias + x_quant, + x_scale, + layer.w13_weight, # uint8 (e2m1 x 2) + layer.w13_weight_scale, # uint8 (e4m3 x 2) + layer.w13_weight_bias, # fp32 per expert per channel + layer.gemm1_alpha, # fp32 per expert + layer.gemm1_beta, # fp32 per expert + layer.gemm1_clamp_limit, # fp32 per expert + layer.w2_weight, # uint8 (e2m1 x 2) + layer.w2_weight_scale, # ue8m0 + layer.w2_weight_bias, # fp32 per expert per channel + None, # output1_scale_scalar + None, # output1_scale_gate_scalar + None, # output2_scale_scalar + layer.num_experts, + top_k, + None, # n_group # TODO: support n_group + None, # topk_group # TODO: support topk_group + self.intermediate_size_per_partition, # padded to multiple of 256 + layer.moe_ep_rank * layer.num_local_experts, # local_expert_offset + layer.num_local_experts, # local num experts + None, + self._get_tile_tokens_dim(x, top_k), + 1, # routing_method_type, renormalize + True, # do finalize + )[0] + return StandardCombineInput(hidden_states=trtllm_gen_output) + + if self.use_triton_kernels: + assert ( + layer.moe_ep_size == 1 + ), "Expert parallel is not supported when using triton kernels" + if self.with_bias: + output = self.triton_kernel_moe_with_bias_forward( + hidden_states=x, + w1=self.w13_weight_triton_tensor, + w1_pcg=self.w13_precision_config, + w2=self.w2_weight_triton_tensor, + w2_pcg=self.w2_precision_config, + b1=layer.w13_weight_bias, + b2=layer.w2_weight_bias, + topk_output=topk_output, + moe_runner_config=moe_runner_config, + ) + else: + output = self.triton_kernel_moe_forward( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_output=topk_output, + moe_runner_config=moe_runner_config, + ) + return StandardCombineInput(hidden_states=output) + else: + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + w13_weight_bias=layer.w13_weight_bias, + w2_weight_bias=layer.w2_weight_bias, + ) + return self.runner.run(dispatch_output, quant_info) + + +class Mxfp4DynamicQuantMoEMethod(FusedMoEMethodBase): + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=params_dtype, + ), + requires_grad=False, + ) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + # Allocate 2 scales for w1 and w3 respectively. + # They will be combined to a single scale after weight loading. + w13_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.float32), requires_grad=False + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value} + ) + + layer.w13_input_scale = None + layer.w2_input_scale = None + + def mxfp4_quantize(self, w): + w_shape = w.shape + w_need_reshape = True if w.dim() != 2 else False + + if w_need_reshape: + w_last_dim_size = w_shape[-1] + w = w.view(-1, w_last_dim_size) + + w, mx_scales = dynamic_mxfp4_quant(w) + + if w_need_reshape: + w_new_shape = w_shape[:-1] + (w.shape[-1],) + w = w.view(w_new_shape) + + mx_scales = e8m0_shuffle(mx_scales) + + return w, mx_scales + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + w13, w13_mx_scales = self.mxfp4_quantize(layer.w13_weight.data) + w2, w2_mx_scales = self.mxfp4_quantize(layer.w2_weight.data) + + layer.w13_weight = torch.nn.Parameter(w13, requires_grad=False) + layer.w13_weight_scale = torch.nn.Parameter(w13_mx_scales, requires_grad=False) + + layer.w2_weight = torch.nn.Parameter(w2, requires_grad=False) + layer.w2_weight_scale = torch.nn.Parameter(w2_mx_scales, requires_grad=False) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + topk_weights, topk_ids, _ = topk_output + if _is_hip: + topk_weights = topk_weights.to( + torch.float32 + ) # aiter's moe_sorting requires topk_weights to be FP32 + output = fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + quant_type=QuantType.per_1x32, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + activation=( + ActivationType.Silu + if self.moe_runner_config.activation == "silu" + else ActivationType.Gelu + ), + doweight_stage1=False, + ) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/mxfp4_tensor.py b/python/sglang/srt/layers/quantization/mxfp4_tensor.py new file mode 100644 index 000000000..76cb92c54 --- /dev/null +++ b/python/sglang/srt/layers/quantization/mxfp4_tensor.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +import torch + + +# https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/main/modelopt/torch/quantization/qtensor/mxfp4_tensor.py +class MXFP4QuantizeUtil: + E2M1_max = 6.0 + + E2M1_values = [0, 0.5, 1, 1.5, 2, 3, 4, 6] + E2M1_bounds = torch.tensor([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5]) + + @classmethod + def quantize(cls, input: torch.Tensor, block_size: Optional[int]) -> tuple: + """Converting a tensor to a quantized format based on MXFP4 quantization. Only E4M3 is supported. + Args: + input (torch.Tensor): The input tensor to be quantized. + block_sizes (dict | None): The block sizes for quantization. + """ + + def cast_fp4(x): + sign = torch.sign(x) + sign_bit = (2 - sign) // 2 + ord_ = torch.sum( + (x.abs().unsqueeze(-1) - cls.E2M1_bounds.to(x.device)) > 0, dim=-1 + ) + fp4_val = (sign_bit * 0b1000 + ord_).to(torch.uint8) + return fp4_val + + def fuse_uint4_to_uint8(x): + # If the last dimension is odd, pad with zeros + # If this behavior is not desired, please modify the code accordingly + left_side = x[..., 0::2] # Even indices (0, 2, 4...) + right_side = x[..., 1::2] # Odd indices (1, 3, 5...) + new_data = ( + right_side.clone() << 4 + ) # Put odd indices (higher addresses) in high bits + new_data[ + ..., : left_side.shape[-1] + ] += left_side # Put even indices in low bits + return new_data + + if block_size is None: + block_size = 32 + + original_shape = input.shape + original_dtype = input.dtype + input = input.view(-1, block_size) + # get scales + input_amax = input.abs().max(dim=-1, keepdim=True).values + descale = input_amax / cls.E2M1_max + min_value = torch.tensor(-127.0, device=descale.device) + e8m0_scale = torch.ceil(torch.maximum(torch.log2(descale), min_value)) + + input = (input / torch.exp2(e8m0_scale)).view(original_shape) + input_q = cast_fp4(input) + input_q = fuse_uint4_to_uint8(input_q) + e8m0_scale = (e8m0_scale + 127).to(torch.uint8) + return cls(original_shape, original_dtype, input_q), e8m0_scale + + @classmethod + def dequantize(cls, quantized_data, dtype: torch.dtype, scale, block_sizes): + """Dequantze MXFP4 packed tensor to a target dtype.""" + + def unfuse_uint8_to_uint4(x): + """Unfuse uint8 values back to uint4 values. + This is the inverse operation of fuse_uint4_to_uint8. + """ + # Extract the lower 4 bits (even indices) + left_side = x & 0x0F + + # Extract the upper 4 bits (odd indices) + right_side = (x >> 4) & 0x0F + + # Create a new tensor with alternating values + shape = list(x.shape) + shape[-1] = shape[-1] * 2 + result = torch.zeros(shape, dtype=torch.uint8, device=x.device) + + # Fill in the values - even indices get low bits, odd indices get high bits + result[..., 0::2] = left_side # Even indices from low bits + result[..., 1::2] = right_side # Odd indices from high bits + + return result + + e8m0_scale = scale + block_size = block_sizes[-1] + + # Unfuse the uint8 values back to uint4 + x_unfused = unfuse_uint8_to_uint4(quantized_data) + # Extract sign and magnitude + sign = 1 - 2 * ((x_unfused & 0b1000) >> 3).to( + torch.float32 + ) # Extract sign bit and convert to +1/-1 + magnitude = x_unfused & 0b0111 # Extract magnitude bits + magnitude = magnitude.to(torch.long) + + # Create a tensor with the E2M1 values + values = torch.tensor(cls.E2M1_values, device=quantized_data.device) + + # Use gather to index the values tensor properly + # We need to reshape magnitude to match the dimensions we want to gather along + original_shape = magnitude.shape + x_float = values[magnitude.reshape(-1)].reshape(original_shape) + + # Apply sign and scale + x_float = sign.float() * x_float + + # Reshape to apply block-wise scaling + x_float = x_float.reshape(-1, block_size) + + # Apply the E8M0 scale + scale_factor = torch.exp2(e8m0_scale.float() - 127) + scale_factor = scale_factor.reshape(-1, 1) # Reshape for proper broadcasting + + # Apply scaling and reshape back to original shape + x_float = x_float * scale_factor + + # Reshape back to the original shape + return x_float.reshape(original_shape).to(dtype) diff --git a/python/sglang/srt/layers/quantization/petit.py b/python/sglang/srt/layers/quantization/petit.py new file mode 100644 index 000000000..2c608507c --- /dev/null +++ b/python/sglang/srt/layers/quantization/petit.py @@ -0,0 +1,252 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py + + +import logging +from typing import Any, Callable, Dict, List, Optional + +import regex as re +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.layers.linear import LinearBase, UnquantizedLinearMethod +from sglang.srt.layers.parameter import ModelWeightParameter, PerTensorScaleParameter +from sglang.srt.layers.quantization.base_config import ( + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.petit_utils import ( + apply_petit_nvfp4_linear, + prepare_nvfp4_layer_for_petit, + verify_petit_nvfp4_supported, +) +from sglang.srt.layers.quantization.utils import is_layer_skipped +from sglang.srt.utils import is_hip + +_is_hip = is_hip() + +# Initialize logger for the module +logger = logging.getLogger(__name__) + + +# Configuration class to support the NVFP4 quantized model generated by the ModelOpt quantization tool +class PetitNvFp4Config(QuantizationConfig): + """Config class for Petit FP4.""" + + def __init__( + self, + is_checkpoint_nvfp4_serialized: bool = False, + kv_cache_quant_algo: str = None, + group_size: int = None, + exclude_modules: List[str] = None, + ) -> None: + self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized + if is_checkpoint_nvfp4_serialized: + logger.warning( + "Detected nvfp4 checkpoint. Please note that the " + "format is experimental and subject to change." + ) + self.group_size = group_size + self.kv_cache_quant_algo = kv_cache_quant_algo + self.exclude_modules = exclude_modules + + @classmethod + def get_name(cls) -> str: + return "petit_nvfp4" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.half] + + @classmethod + def get_min_capability(cls) -> int: + # Petit supports the gfx90a and gfx942 GPUs + return 90 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return ["hf_quant_config.json"] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> "PetitNvFp4Config": + quant_config = cls.get_from_keys(config, ["quantization"]) + quant_method = quant_config["quant_algo"] + group_size = quant_config.get("group_size", None) + verify_petit_nvfp4_supported(quant_method, group_size) + + is_checkpoint_nvfp4_serialized = "NVFP4" in quant_method + kv_cache_quant_algo = quant_config["kv_cache_quant_algo"] + if not kv_cache_quant_algo: + kv_cache_quant_algo = "auto" + exclude_modules = quant_config.get("exclude_modules", None) + if not (group_size and kv_cache_quant_algo and (exclude_modules is not None)): + logger.warning( + f"group_size: {group_size}," + f"kv_cache_quant_algo: {kv_cache_quant_algo}," + f"exclude_modules: {exclude_modules}" + ) + raise ValueError( + "NVFP4 quantization requires group size and " + "kv_cache_quant_algo specified in " + "hf_quant_config.json" + ) + return cls( + is_checkpoint_nvfp4_serialized, + kv_cache_quant_algo, + group_size, + exclude_modules, + ) + + @classmethod + def override_quantization_method(cls, hf_quant_cfg, user_quant) -> Optional[str]: + can_convert = cls.is_petit_nvfp4_compatible(hf_quant_cfg) + if can_convert: + return cls.get_name() + return None + + @classmethod + def is_petit_nvfp4_compatible(cls, quant_config: Dict[str, Any]) -> bool: + quant_method = quant_config.get("quant_method", "").lower() + return _is_hip and quant_method == "modelopt" + + def is_layer_excluded(self, prefix: str, exclude_modules: list): + for pattern in exclude_modules: + regex_str = pattern.replace(".", r"\.").replace("*", r".*") + if re.fullmatch(regex_str, prefix): + return True + return False + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional["QuantizeMethodBase"]: + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.exclude_modules) or self.is_layer_excluded( + prefix, self.exclude_modules + ): + return UnquantizedLinearMethod() + return PetitNvFp4LinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class PetitNvFp4LinearMethod(LinearMethodBase): + """Linear method for NVFP4. + Supports loading NVFP4 checkpoints with the following structure: + + |Tensor Name | datatype | shape | + |----------------------------------------------------| + |input_scale | torch.float32 | scalar | + |weight | NVFP4(SE2M1) | [1, X, y/2] | + |weight_scale | FP8-E4M3 | [X, Y] | + |weight_scale_2 | torch.float32 | scalar | + + The weights are quantized per block of 16 elements. + Args: quant_config: The ModelOpt quantization config. + """ + + def __init__(self, quant_config: PetitNvFp4Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + del input_size, output_size + if not self.quant_config.is_checkpoint_nvfp4_serialized: + raise ValueError( + "NVFP4 quantization was selected, " + " dynamic quantization is not supported." + ) + + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + layer.logical_widths = output_partition_sizes + + layer.input_size_per_partition = input_size_per_partition + layer.output_size_per_partition = output_size_per_partition + if input_size_per_partition % 16 != 0: + raise ValueError( + "Unsupported model when in features size is " "not multiple of 16" + ) + + weight_dtype = ( + torch.float8_e4m3fn + if self.quant_config.is_checkpoint_nvfp4_serialized + else params_dtype + ) + + weight = ModelWeightParameter( + data=torch.empty( + # 2 fp4 data is packed in one uint8 in the input dimension + output_size_per_partition, + input_size_per_partition // 2, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + input_scale = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + + layer.register_parameter("input_scale", input_scale) + + weight_scale_2 = PerTensorScaleParameter( + data=torch.empty(len(output_partition_sizes), dtype=torch.float32), + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale_2", weight_scale_2) + + weight_scale = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // self.quant_config.group_size, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + + layer.register_parameter("weight_scale", weight_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + input_scale_2 = layer.input_scale.max().to(torch.float32) + weight_scale_2 = layer.weight_scale_2.max().to(torch.float32) + layer.input_scale = Parameter(input_scale_2, requires_grad=False) + layer.weight_scale_2 = Parameter(weight_scale_2, requires_grad=False) + layer.alpha = Parameter( + layer.input_scale * layer.weight_scale_2, requires_grad=False + ) + + prepare_nvfp4_layer_for_petit(layer) + del layer.input_scale + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return apply_petit_nvfp4_linear( + input=x, + weight=layer.weight, + weight_scale=layer.weight_scale, + weight_scale_2=layer.weight_scale_2, + size_n=layer.output_size_per_partition, + size_k=layer.input_size_per_partition, + bias=bias, + ) diff --git a/python/sglang/srt/layers/quantization/petit_utils.py b/python/sglang/srt/layers/quantization/petit_utils.py new file mode 100644 index 000000000..529869f24 --- /dev/null +++ b/python/sglang/srt/layers/quantization/petit_utils.py @@ -0,0 +1,104 @@ +from typing import Optional + +import torch + +try: + from petit_kernel import mul_nvfp4_a16, process_nvfp4_scales, repack_nvfp4 +except ImportError: + + def _check_petit_nvfp4_supported( + quant_method: str, group_size: Optional[int] + ) -> tuple[bool, Optional[str]]: + return ( + False, + "Petit is not installed. Please install it with `pip install petit-kernel`.", + ) + + def prepare_nvfp4_layer_for_petit(layer: torch.nn.Module) -> None: + raise ValueError( + "Petit is not installed. Please install it with `pip install petit-kernel`." + ) + + def apply_petit_nvfp4_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_scale_2: torch.Tensor, + size_n: int, + size_k: int, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise ValueError( + "Petit is not installed. Please install it with `pip install petit-kernel`." + ) + + +def _check_petit_nvfp4_supported( + quant_method: str, group_size: Optional[int] +) -> tuple[bool, Optional[str]]: + if quant_method != "NVFP4": + return ( + False, + "Petit currently only supports: NVFP4" + " quantizations in sglang. Please check the " + "`hf_quant_config.json` file for your model's " + "quant configuration.", + ) + if group_size is not None and group_size != 16: + return ( + False, + "Petit currently only supports: group_size=16" " quantizations.", + ) + return (True, None) + + +def verify_petit_nvfp4_supported(quant_method: str, group_size: Optional[int]) -> None: + supported, error_msg = _check_petit_nvfp4_supported(quant_method, group_size) + if not supported: + raise ValueError(error_msg) + + +def prepare_nvfp4_layer_for_petit(layer: torch.nn.Module) -> None: + # Repack weights to petit format + part_size_n = layer.output_size_per_partition + part_size_k = layer.input_size_per_partition + qweight = layer.weight.view(torch.int32).contiguous() + petit_qweight = repack_nvfp4(qweight, size_n=part_size_n, size_k=part_size_k) + layer.weight = torch.nn.Parameter(petit_qweight, requires_grad=False) + + # Permute scales + weight_scale = process_nvfp4_scales( + scales=layer.weight_scale, size_k=part_size_k, size_n=part_size_n + ) + layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False) + + return + + +def apply_petit_nvfp4_linear( + input: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + weight_scale_2: torch.Tensor, + size_n: int, + size_k: int, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + reshaped_x = input.reshape(-1, input.shape[-1]) + out_shape = input.shape[:-1] + (size_n,) + + # TODO: Use auto-tuning to find the performant solution_id + output = mul_nvfp4_a16( + a=reshaped_x, + b=weight, + s=weight_scale, + global_scale=weight_scale_2, + size_m=reshaped_x.size(0), + size_n=size_n, + size_k=size_k, + solution_id=-1, + ) + if bias is not None: + output.add_(bias) # In-place add + + return output.reshape(out_shape) diff --git a/python/sglang/srt/layers/quantization/qoq.py b/python/sglang/srt/layers/quantization/qoq.py new file mode 100644 index 000000000..ec0fda482 --- /dev/null +++ b/python/sglang/srt/layers/quantization/qoq.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.layers.parameter import ( + ChannelQuantScaleParameter, + GroupQuantScaleParameter, + ModelWeightParameter, +) +from sglang.srt.layers.quantization.base_config import ( + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 +from sglang.srt.utils import is_cuda + +_is_cuda = is_cuda() +if _is_cuda: + from sgl_kernel import qserve_w4a8_per_chn_gemm, qserve_w4a8_per_group_gemm + + +QoQ_SUPPORTED_WEIGHT_BITS = [4] +QoQ_SUPPORTED_GROUP_SIZES = [-1, 128] + + +class QoQConfig(QuantizationConfig): + """Config class for QoQ Quantization. + + - Weight: static, per-channel/group, asymmetric + - Activation: dynamic, per-token, symmetric + + Reference: https://arxiv.org/abs/2405.04532 + https://github.com/mit-han-lab/omniserve + """ + + def __init__(self, weight_bits: int, group_size: int) -> None: + self.weight_bits = weight_bits + self.group_size = group_size + + # Verify + if self.weight_bits not in QoQ_SUPPORTED_WEIGHT_BITS: + raise ValueError( + f"QoQ does not support weight_bits = {self.weight_bits}. " + f"Only weight_bits = {QoQ_SUPPORTED_WEIGHT_BITS} " + "are supported." + ) + if self.group_size not in QoQ_SUPPORTED_GROUP_SIZES: + raise ValueError( + f"QoQ does not support group_size = {self.group_size}. " + f"Only group_sizes = {QoQ_SUPPORTED_GROUP_SIZES} " + "are supported." + ) + + # 4 bits packed into 8 bit datatype. + self.pack_factor = 8 // self.weight_bits + + def __repr__(self) -> str: + return "QoQConfig(weight_bits={}, group_size={})".format( + self.weight_bits, self.group_size + ) + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.float16] + + @classmethod + def get_min_capability(cls) -> int: + return 80 + + @classmethod + def get_name(cls) -> str: + return "qoq" + + @classmethod + def get_config_filenames(cls) -> List[str]: + """List of filenames to search for in the model directory.""" + return [ + "quant_config.json", + "quantize_config.json", + ] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> QoQConfig: + weight_bits = cls.get_from_keys(config, ["wbits"]) + group_size = cls.get_from_keys(config, ["group_size"]) + return cls(weight_bits, group_size) + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + + if isinstance(layer, LinearBase): + return QoQLinearMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class QoQLinearMethod(LinearMethodBase): + """Linear method for QoQ. + + Args: + quant_config: The QoQ quantization config. + """ + + def __init__(self, quant_config: QoQConfig): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + + weight_loader = extra_weight_attrs.get("weight_loader") + + # Validate output_size_per_partition + output_size_per_partition = sum(output_partition_sizes) + if output_size_per_partition % 32 != 0: + raise ValueError( + f"Weight output_size_per_partition = " + f"{output_size_per_partition} is not divisible by 32." + ) + + # Validate input_size_per_partition + if input_size_per_partition % self.quant_config.pack_factor != 0: + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"pack_factor = {self.quant_config.pack_factor}." + ) + if ( + self.quant_config.group_size != -1 + and input_size_per_partition % self.quant_config.group_size != 0 + ): + raise ValueError( + f"Weight input_size_per_partition = " + f"{input_size_per_partition} is not divisible by " + f"group_size = {self.quant_config.group_size}." + ) + + qweight = ModelWeightParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // self.quant_config.pack_factor, + dtype=torch.int8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("qweight", qweight) + + s1_scales = ChannelQuantScaleParameter( + data=torch.empty(output_size_per_partition, dtype=torch.float16), + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("s1_scales", s1_scales) + + if self.quant_config.group_size == -1: + s1_szeros = ChannelQuantScaleParameter( + data=torch.empty(output_size_per_partition, dtype=torch.float16), + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("s1_szeros", s1_szeros) + else: + s2_scales = GroupQuantScaleParameter( + data=torch.empty( + ( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition, + ), + dtype=torch.int8, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("s2_scales", s2_scales) + + s2_zeros = GroupQuantScaleParameter( + data=torch.empty( + ( + input_size_per_partition // self.quant_config.group_size, + output_size_per_partition, + ), + dtype=torch.int8, + ), + input_dim=0, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("s2_zeros", s2_zeros) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.qweight = Parameter(layer.qweight.data, requires_grad=False) + layer.s1_scales = Parameter(layer.s1_scales.data, requires_grad=False) + if self.quant_config.group_size == -1: + layer.s1_szeros = Parameter(layer.s1_szeros.data, requires_grad=False) + else: + layer.s2_scales = Parameter(layer.s2_scales.data, requires_grad=False) + layer.s2_zeros = Parameter(layer.s2_zeros.data, requires_grad=False) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ): + assert x.dtype == torch.float16, "QoQ only supports float16 input now" + if self.quant_config.group_size == -1: + x_q, x_scale, x_sum = per_token_quant_int8( + x, scale_dtype=x.dtype, cal_sum=True + ) + out = qserve_w4a8_per_chn_gemm( + x_q, layer.qweight, layer.s1_scales, x_scale, layer.s1_szeros, x_sum + ) + else: + x_q, x_scale = per_token_quant_int8(x, scale_dtype=x.dtype) + out = qserve_w4a8_per_group_gemm( + x_q, + layer.qweight, + layer.s2_zeros, + layer.s2_scales, + layer.s1_scales, + x_scale, + ) + if bias is not None: + out = out + bias + return out diff --git a/python/sglang/srt/layers/quantization/quark/__init__.py b/python/sglang/srt/layers/quantization/quark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py new file mode 100644 index 000000000..6d5a66544 --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -0,0 +1,390 @@ +# SPDX-License-Identifier: Apache-2.0 + +import fnmatch +import logging +from typing import Any, List, Optional, cast + +import torch + +from sglang.srt.layers.linear import LinearBase, UnquantizedLinearMethod +from sglang.srt.layers.quantization.base_config import ( # noqa: E501 + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.kv_cache import BaseKVCacheMethod +from sglang.srt.layers.quantization.quark.quark_moe import QuarkMoEMethod +from sglang.srt.layers.quantization.quark.schemes import QuarkScheme, QuarkW4A4MXFP4 +from sglang.srt.layers.quantization.quark.utils import deep_compare, should_ignore_layer +from sglang.srt.layers.radix_attention import RadixAttention +from sglang.srt.utils import get_device_capability + +__all__ = ["QuarkLinearMethod"] + +logger = logging.getLogger(__name__) + + +class QuarkConfig(QuantizationConfig): + + def __init__( + self, + quant_config: dict[str, Any], + kv_cache_group: Optional[list[str]] = None, + kv_cache_config: Optional[dict[str, Any]] = None, + pack_method: str = "reorder", + ): + super().__init__() + if kv_cache_group is None: + kv_cache_group = [] + self.quant_config = quant_config + self.kv_cache_group = kv_cache_group + self.kv_cache_config = kv_cache_config + self.pack_method = pack_method + + self.packed_modules_mapping = self.quant_config["packed_modules_mapping"] + + def get_linear_method(self) -> "QuarkLinearMethod": + return QuarkLinearMethod(self) + + @classmethod + def get_supported_act_dtypes(cls) -> list[torch.dtype]: + return [torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def get_name(self) -> str: + return "quark" + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional["QuantizeMethodBase"]: + # Check if the layer is skipped for quantization. + exclude_layers = cast(list[str], self.quant_config.get("exclude")) + if should_ignore_layer( + prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping + ): + return UnquantizedLinearMethod() + + if isinstance(layer, LinearBase): + scheme = self.get_scheme(layer=layer, layer_name=prefix) + layer.scheme = scheme + return QuarkLinearMethod(self) + + if isinstance(layer, RadixAttention): + return QuarkKVCacheMethod(self) + + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + + if isinstance(layer, FusedMoE): + return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix) + + return None + + @classmethod + def from_config(cls, config: dict[str, Any]) -> "QuarkConfig": + export_config = config.get("export") + if export_config is None: + raise ValueError( + "The export key should be included in " + "the configurations of Quark quantized model" + ) + + kv_cache_group = cast(list[str], export_config.get("kv_cache_group")) + pack_method = cast(str, export_config.get("pack_method")) + + # In the export model of quark, the quantization configuration + # of kv_cache is stored in layer_quant_config. First, it is + # judged whether kv_cache_group exists, and then it is judged + # whether layer_quant_config has a quantization configuration + # that matches kv_cache. + if len(kv_cache_group) == 0: + kv_cache_config = None + else: + kv_cache_set = set(kv_cache_group) + layer_quant_config = cast(dict[str, Any], config.get("layer_quant_config")) + layer_quant_names = list(layer_quant_config.keys()) + layer_quant_set = set(layer_quant_names) + + if not kv_cache_set.issubset(layer_quant_set): + raise ValueError( + "The Quark quantized model has the " + "kv_cache_group parameter setting, " + "but no kv_cache quantization settings " + "were found in the quantization " + "configuration." + ) + + q_configs = [ + cast(dict[str, Any], layer_quant_config.get(name)) + for name in kv_cache_group + ] + if not all(deep_compare(q_config, q_configs[0]) for q_config in q_configs): + raise ValueError( + "The quantization method used for kv_cache should " + "be the same, but the quantization method for the " + "kv_cache layer in the config is different." + ) + kv_cache_config = q_configs[0].get("output_tensors") + if kv_cache_config is None: + raise ValueError("The kv_cache quantization configuration is empty.") + + # Since we have already set kv_cache quantization configurations, + # we will remove the quantization configuration for the + # output_tensors corresponding to the kv_cache layer. + for q_config in q_configs: + q_config["output_tensors"] = None + + # In case q_proj output is also quantized, remove the configuration + # to keep qkv consistency. + q_proj_q_config = cast(dict[str, Any], layer_quant_config.get("*q_proj")) + if q_proj_q_config is not None: + q_proj_q_config["output_tensors"] = None + + return cls( + quant_config=config, + kv_cache_group=kv_cache_group, + kv_cache_config=kv_cache_config, + pack_method=pack_method, + ) + + @classmethod + def get_config_filenames(cls) -> list[str]: + return [] + + def _check_scheme_supported(self, min_capability: int, error: bool = True) -> bool: + capability_tuple = get_device_capability() + + if capability_tuple is not None: + assert 0 <= capability_tuple[1] < 10 + capability = capability_tuple[0] * 10 + capability_tuple[1] + + supported = capability >= min_capability + if error and not supported: + raise RuntimeError( + "Quantization scheme is not supported for ", + f"the current GPU. Min capability: {min_capability}. ", + f"Current capability: {capability}.", + ) + return supported + else: + return False + + def _is_mx_fp4( + self, + weight_quant: Optional[dict[str, Any]], + input_quant: Optional[dict[str, Any]], + ) -> bool: + # Confirm weights and input quantized. + if weight_quant is None or input_quant is None: + logger.debug( + "Quark model is not in MX-FP4 format: " + "weight_quant or input_quant not set" + ) + return False + + # Input and weight dtype needs to be fp4. + if weight_quant.get("dtype") != "fp4" or input_quant.get("dtype") != "fp4": + logger.debug("Quark model is not in MX-FP4 format: dtype not fp4") + return False + + # Input and weight qscheme needs to be per group. + if ( + weight_quant.get("qscheme") != "per_group" + or input_quant.get("qscheme") != "per_group" + ): + logger.debug("Quark model is not in MX-FP4 format: not per_group") + return False + + # Input and weight group size needs to be 32. + if weight_quant.get("group_size") != 32 or input_quant.get("group_size") != 32: + logger.debug("Quark model is not in MX-FP4 format: not group_size=32") + return False + + # Weights need to use static quantization. + if weight_quant.get("is_dynamic") is True: + logger.debug("Quark model is not in MX-FP4 format: not weight static") + return False + + # Activations need to use dynamic quantization. + if input_quant.get("is_dynamic") is False: + logger.debug("Quark model is not in MX-FP4 format: not activation dynamic") + return False + + # Activations and weight scales need to be in e8m0 format. + if ( + weight_quant.get("scale_format") != "e8m0" + or input_quant.get("scale_format") != "e8m0" + ): + logger.debug("Quark model is not in MX-FP4 format: not scale_format e8m0") + return False + + return True + + def _find_matched_config( + self, layer_name: str, module: torch.nn.Module + ) -> dict[str, Any]: + + proj_name = layer_name.split(".")[-1] + if proj_name in self.packed_modules_mapping: + shard_proj_names = self.packed_modules_mapping[proj_name] + + # Convert fused_name --> [shard_names] + shard_names = [ + layer_name.replace(proj_name, shard_proj_name) + for shard_proj_name in shard_proj_names + ] + shard_configs = [ + self._find_matched_config(shard_name, module) + for shard_name in shard_names + ] + if not all( + deep_compare(q_config, shard_configs[0]) for q_config in shard_configs + ): + raise ValueError( + f"Found a different quantization configuration for " + f"{shard_proj_names} in {layer_name}. vLLM " + "requires all to use the same scheme." + ) + return shard_configs[0] + else: + layer_quant_config = cast( + dict[str, Any], self.quant_config.get("layer_quant_config") + ) + for name_pattern in layer_quant_config: + if fnmatch.fnmatch(layer_name, name_pattern): + return layer_quant_config[name_pattern] + + layer_type = type(module).__name__ + layer_type_quant_config = cast( + dict[str, Any], self.quant_config.get("layer_type_quant_config") + ) + if layer_type in layer_type_quant_config: + return layer_type_quant_config[layer_type] + + global_quant_config = cast( + dict[str, Any], self.quant_config.get("global_quant_config") + ) + return global_quant_config + + def _get_scheme_from_config(self, config: dict[str, Any]) -> "QuarkScheme": + if config.get("output_tensors") or config.get("bias"): + raise NotImplementedError( + "Currently, Quark models with output_tensors " + "and bias quantized are not supported" + ) + weight_config = cast(dict[str, Any], config.get("weight")) + input_config = cast(dict[str, Any], config.get("input_tensors")) + + if self._is_mx_fp4(weight_config, input_config): + return QuarkW4A4MXFP4(weight_config, input_config) + + raise NotImplementedError( + "No quark compatible scheme was found. " + f"Weight config: {weight_config}, " + f"Input config: {input_config}" + ) + + def get_scheme(self, layer: torch.nn.Module, layer_name: str) -> "QuarkScheme": + + layer_quant_config = self._find_matched_config(layer_name, layer) + + # Find the quant_scheme + scheme = self._get_scheme_from_config(layer_quant_config) + + # Raise error if device does not support the scheme + # (e.g. fp8 needs ada lovelace) + self._check_scheme_supported(scheme.get_min_capability()) + + return scheme + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class QuarkLinearMethod(LinearMethodBase): + + def __init__(self, quantization_config: QuarkConfig): + self.quantization_config = quantization_config + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.scheme.process_weights_after_loading(layer) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: list[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + """ + Use the CompressedTensorsScheme associated with each layer to create + the necessary parameters for the layer. See LinearMethodBase for param + details + """ + weight_loader = extra_weight_attrs.get("weight_loader") + layer.scheme.create_weights( + layer=layer, + input_size=input_size, + input_size_per_partition=input_size_per_partition, + output_partition_sizes=output_partition_sizes, + output_size=output_size, + params_dtype=params_dtype, + weight_loader=weight_loader, + ) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ): + """ + Use the output of create_weights and the CompressedTensorsScheme + associated with the layer to apply the forward pass with the + layer input. See LinearMethodBase for param details + + """ + scheme = layer.scheme + if scheme is None: + raise ValueError("A scheme must be defined for each layer") + return scheme.apply_weights(layer, x, bias=bias) + + +class QuarkKVCacheMethod(BaseKVCacheMethod): + """ + Supports loading kv-cache scaling factors from quark checkpoints. + """ + + def __init__(self, quant_config: QuarkConfig): + self.validate_kv_cache_config(quant_config.kv_cache_config) + super().__init__(quant_config) + + @staticmethod + def validate_kv_cache_config(kv_cache_config: Optional[dict[str, Any]]): + """ + Validator for the kv cache configuration. Useful for controlling the + kv cache quantization schemes, that are being supported in vLLM + :param kv_cache_config: the quark kv cache scheme + """ + if kv_cache_config is None: + return + + dtype = kv_cache_config.get("dtype") + if dtype != "fp8_e4m3": + raise NotImplementedError( + "Currently supported kv cache quantization is " + f"dtype=fp8_e4m3, however received {dtype}" + ) + + qscheme = kv_cache_config.get("qscheme") + if qscheme != "per_tensor": + raise NotImplementedError( + "Only support per-tensor scaling factor " + "for quark KV cache. " + f"Expected qscheme: per_tensor, found qscheme: {qscheme}" + ) diff --git a/python/sglang/srt/layers/quantization/quark/quark_moe.py b/python/sglang/srt/layers/quantization/quark/quark_moe.py new file mode 100644 index 000000000..f6e750a2c --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/quark_moe.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Callable, Optional + +import torch +from aiter import ActivationType, QuantType, biased_grouped_topk +from aiter.fused_moe import fused_moe +from aiter.utility.fp4_utils import e8m0_shuffle + +from sglang.srt.layers.moe import MoeRunnerConfig +from sglang.srt.layers.quantization.base_config import FusedMoEMethodBase +from sglang.srt.utils import get_bool_env_var, mxfp_supported, set_weight_attrs + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + from sglang.srt.layers.quantization.quark.quark import QuarkConfig + +logger = logging.getLogger(__name__) + +__all__ = ["QuarkMoEMethod", "QuarkW4A4MXFp4MoEMethod"] + +OCP_MX_BLOCK_SIZE = 32 + +if TYPE_CHECKING: + from sglang.srt.layers.quantization import QuarkConfig + + +class QuarkMoEMethod(FusedMoEMethodBase): + + def __init__(self, quant_config: QuarkConfig): + self.quant_config = quant_config + + @staticmethod + def get_moe_method( + quant_config: QuarkConfig, # type: ignore # noqa E501 # noqa F821 + module: torch.nn.Module, + layer_name: str, + ) -> "QuarkMoEMethod": + layer_quant_config = quant_config._find_matched_config(layer_name, module) + + if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"): + raise NotImplementedError( + "Currently, Quark models with " + "output_tensors and bias " + "quantized are not supported" + ) + weight_config = layer_quant_config.get("weight") + input_config = layer_quant_config.get("input_tensors") + + if quant_config._is_mx_fp4(weight_config, input_config): + return QuarkW4A4MXFp4MoEMethod(weight_config, input_config) + else: + raise RuntimeError("Unsupported FusedMoe scheme") + + +class QuarkW4A4MXFp4MoEMethod(QuarkMoEMethod): + + def __init__(self, weight_config: dict[str, Any], input_config: dict[str, Any]): + self.weight_quant = weight_config + self.input_quant = input_config + + weight_qscheme = self.weight_quant.get("qscheme") + input_qscheme = self.input_quant.get("qscheme") + if not (weight_qscheme == "per_group" and input_qscheme == "per_group"): + raise ValueError( + "For MX(FP4) Fused MoE layers, only per-group scales " + "for weights and activations are supported. Found " + f"{weight_qscheme}, {input_qscheme}" + ) # noqa E501 + + self.static_input_scales = not self.input_quant.get("is_dynamic") + self.with_bias = False + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + # Add the quantization method used (per tensor/grouped/channel) + # to ensure the weight scales are loaded in properly + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value} + ) + + params_dtype = torch.uint8 + + # WEIGHTS + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // 2, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // 2, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + + set_weight_attrs(w2_weight, extra_weight_attrs) + + # WEIGHT_SCALES + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // OCP_MX_BLOCK_SIZE, + dtype=params_dtype, + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + hidden_size, + intermediate_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + float_dtype = torch.get_default_dtype() + + # Pre-shuffle weight scales + s0, s1, _ = layer.w13_weight_scale.shape + w13_weight_scale = layer.w13_weight_scale.view(s0 * s1, -1) + w13_weight_scale = e8m0_shuffle(w13_weight_scale) + # layer.w13_weight_scale = torch.nn.Parameter(w13_weight_scale, requires_grad=False) + layer.w13_weight_scale.data = w13_weight_scale.view(s0, s1, -1) + + s0, s1, _ = layer.w2_weight_scale.shape + w2_weight_scale = layer.w2_weight_scale.view(s0 * s1, -1) + w2_weight_scale = e8m0_shuffle(w2_weight_scale) + # layer.w2_weight_scale = torch.nn.Parameter(w2_weight_scale, requires_grad=False) + layer.w2_weight_scale.data = w2_weight_scale.view(s0, s1, -1) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + moe_runner_config = self.moe_runner_config + topk_weights, topk_ids, _ = topk_output + + output = fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + quant_type=QuantType.per_1x32, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + activation=( + ActivationType.Silu + if moe_runner_config.activation == "silu" + else ActivationType.Gelu + ), + doweight_stage1=False, + ) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/quark/schemes/__init__.py b/python/sglang/srt/layers/quantization/quark/schemes/__init__.py new file mode 100644 index 000000000..91b08c512 --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/schemes/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 + +from .quark_scheme import QuarkScheme +from .quark_w4a4_mxfp4 import QuarkW4A4MXFP4 + +__all__ = ["QuarkScheme", "QuarkW4A4MXFP4"] diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_scheme.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_scheme.py new file mode 100644 index 000000000..aab5c9c1e --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_scheme.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 + +from abc import ABC, abstractmethod +from typing import Optional + +import torch + +__all__ = ["QuarkScheme"] + + +class QuarkScheme(ABC): + """ + Abstract class used to describe the weight creation and forward pass + of different quantization schemes supported by Quark. + """ + + @classmethod + @abstractmethod + def get_min_capability(cls) -> int: + """ + Get minimum device capability. + """ + raise NotImplementedError + + @abstractmethod + def create_weights(self, *args, **kwargs): + """ + Weight creation for the particular scheme. Inputs to this function + + """ + raise NotImplementedError + + @abstractmethod + def apply_weights( + self, layer: torch.nn.Module, x: torch.Tensor, bias: Optional[torch.Tensor] + ): + """ + Run the forward pass for the particular scheme. This is where + scheme-specific dequant/quant steps/kernels should be applied. + + :param layer: torch.nn.Module with the registered weights and + other parameters relevant to the particular scheme. + :param x: input to the layer + :param bias: bias parameter + + """ + raise NotImplementedError + + @abstractmethod + def process_weights_after_loading(self, layer: torch.nn.Module): + """ + Called after weight loading is complete for any cleanup that + needs to occur. + """ + raise NotImplementedError diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py new file mode 100644 index 000000000..a0787baaf --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Callable, Optional + +import aiter +import torch +import torch.nn.functional as F +from aiter.ops.gemm_op_a4w4 import gemm_a4w4 +from aiter.ops.shuffle import shuffle_weight +from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4 +from aiter.ops.triton.gemm_afp4wfp4_pre_quant_atomic import gemm_afp4wfp4_pre_quant +from aiter.ops.triton.quant import dynamic_mxfp4_quant +from aiter.utility import dtypes +from aiter.utility.fp4_utils import e8m0_shuffle + +from sglang.srt.layers.parameter import GroupQuantScaleParameter, PackedvLLMParameter +from sglang.srt.layers.quantization.quark.schemes import QuarkScheme +from sglang.srt.utils import get_bool_env_var + +__all__ = ["QuarkW4A4MXFP4"] + +OCP_MX_BLOCK_SIZE = 32 + + +class QuarkW4A4MXFP4(QuarkScheme): + + def __init__( + self, weight_quant_spec: dict[str, Any], input_quant_spec: dict[str, Any] + ): + self.out_dtype = torch.get_default_dtype() + self.qscheme = "per_group" + self.weight_quant_spec = weight_quant_spec + self.input_quant_spec = input_quant_spec + + @classmethod + def get_min_capability(cls) -> int: + return 70 + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + return + + def create_weights( + self, + layer: torch.nn.Module, + output_partition_sizes: list[int], + input_size_per_partition: int, + params_dtype: torch.dtype, + weight_loader: Callable, + **kwargs + ): + output_size_per_partition = sum(output_partition_sizes) + layer.logical_widths = output_partition_sizes + + # WEIGHT + weight = PackedvLLMParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // 2, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + packed_dim=1, + packed_factor=2, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + # WEIGHT SCALE + weight_scale = GroupQuantScaleParameter( + data=torch.empty( + output_size_per_partition, + input_size_per_partition // OCP_MX_BLOCK_SIZE, + dtype=torch.uint8, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # This path does not have support for bias currently + assert bias is None, "bias is not supported" + + three_d = False + x_s = None + y = None + if isinstance(x, tuple): + assert len(x) in [ + 2, + 3, + ], "For tuple input, only (x, x_s) or (x, x_s, y) formats are accepted" + if len(x) == 2: + x, x_s = x + elif len(x) == 3: + x, x_s, y = x + + use_fused_quant_gemm = ( + x_s is None and y is not None and layer.weight.shape[0] == y.shape[1] + ) + + if x.dim() == 3: + three_d = True + x = x.view(-1, x.shape[-1]) + output_shape = [*x.shape[:-1], layer.weight.shape[0]] + + # use_fused_quant_gemm = true, x_q is a bf16/fp16 num + # x_s is not None = true, x_q is uint8 num + if use_fused_quant_gemm or x_s is not None: + x_q = x + else: + x_q, x_s = dynamic_mxfp4_quant(x) + + if y is None: + y = torch.empty( + x_q.shape[0], + layer.weight.shape[0], + device=x_q.device, + dtype=self.out_dtype, + ) + + if use_fused_quant_gemm: + gemm_afp4wfp4_pre_quant(x_q, layer.weight, layer.weight_scale, y.dtype, y) + y = y.to(x.dtype) + else: + gemm_afp4wfp4(x_q, layer.weight, x_s, layer.weight_scale, self.out_dtype, y) + + if three_d: + return y.view(*output_shape) + + return y diff --git a/python/sglang/srt/layers/quantization/quark/utils.py b/python/sglang/srt/layers/quantization/quark/utils.py new file mode 100644 index 000000000..eacbf3ba9 --- /dev/null +++ b/python/sglang/srt/layers/quantization/quark/utils.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 + +import re +from collections.abc import Iterable, Mapping +from types import MappingProxyType +from typing import Any, Optional + +import torch +from aiter.ops.triton.quant import dynamic_mxfp4_quant +from torch import nn + + +def deep_compare(dict1: Any, dict2: Any) -> bool: + if type(dict1) is not type(dict2): + return False + if isinstance(dict1, dict): + if dict1.keys() != dict2.keys(): + return False + return all(deep_compare(dict1[k], dict2[k]) for k in dict1) + elif isinstance(dict1, list): + return set(dict1) == set(dict2) + else: + return dict1 == dict2 + + +def should_ignore_layer( + layer_name: Optional[str], + ignore: Iterable[str], + fused_mapping: Mapping[str, list[str]] = MappingProxyType({}), +) -> bool: + if layer_name is None: + return False + + # layer_name = model.layers.0.self_attn.qkv_proj + # proj_name = qkv_proj + proj_name = layer_name.split(".")[-1] + + # Fused layers like gate_up_proj or qkv_proj will not be fused + # in the safetensors checkpoint. So, we convert the name + # from the fused version to unfused + check to make sure that + # each shard of the fused layer has the same scheme. + if proj_name in fused_mapping: + shard_proj_names = fused_mapping[proj_name] + + # Convert fused_name --> [shard_names] + shard_names = [ + layer_name.replace(proj_name, shard_proj_name) + for shard_proj_name in shard_proj_names + ] + + # Layer should be ignored if shards are ignored. + should_ignore_layer = None + for shard_name in shard_names: + should_ignore_shard = check_equal_or_regex_match( + layer_name=shard_name, targets=ignore + ) + + # If shard_idx=0, set layer ignore to match shard. + if should_ignore_layer is None: + should_ignore_layer = should_ignore_shard + + # If shard_idx=1+ confirm scheme matches prior shards. + elif should_ignore_shard != should_ignore_layer: + raise ValueError( + f"Found a different quantization schemes for " + f"{shard_proj_names} in {layer_name}. vLLM " + "requires all to use the same scheme." + ) + + # Unfused layers like down_proj and o_proj will match + # the safetensors checkpoint already. + else: + should_ignore_layer = check_equal_or_regex_match( + layer_name=layer_name, targets=ignore + ) + + assert should_ignore_layer is not None + + return should_ignore_layer + + +def check_equal_or_regex_match(layer_name: str, targets: Iterable[str]) -> bool: + """ + Checks whether a layer_name is exactly equal or a regex match for + if target starts with 're:' to any target in list. + """ + for target in targets: + if _is_equal_or_regex_match(layer_name, target): + return True + return False + + +def _is_equal_or_regex_match( + value: str, target: str, check_contains: bool = False +) -> bool: + """ + Checks whether a value is exactly equal or a regex match for target + if target starts with 're:'. If check_contains is set to True, + additionally checks if the target string is contained within the value. + """ + + if target.startswith("re:"): + pattern = target[3:] + if re.match(pattern, value): + return True + elif check_contains: + if target.lower() in value.lower(): + return True + elif target == value: + return True + return False + + +# utility for tensor dims > 2 cases +def b_dynamic_mxfp4_quant(x): + h, b, d = x.shape + x, x_scales = dynamic_mxfp4_quant(x.reshape(-1, d)) + return x.view(h, b, d // 2), x_scales.view(h, b, d // 32) + + +def mxfp4_to_f32(x, is_threed): + # 2 because we pack fp4 in uint8. + x = x.repeat_interleave(2, dim=-1) + if is_threed: + x[..., ::2] = x[..., ::2] & 0xF + x[..., 1::2] = x[..., 1::2] >> 4 + else: + x[:, ::2] = x[:, ::2] & 0xF + x[:, 1::2] = x[:, 1::2] >> 4 + + mxfp4_list = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + -0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, + ] + mxfp4_in_f32 = torch.tensor(mxfp4_list, dtype=torch.float32, device="cuda") + return mxfp4_in_f32[x.long()] + + +def e8m0_to_f32(x): + # Convert the input tensor `x` (assumed to be in e8m0 format) to float32. + # e8m0 is a custom 8-bit floating point format with 8 bits for exponent, 0 for mantissa. + # This means the value is essentially 2^(exponent - 127), similar to how IEEE-754 stores floats. + + # Convert x to float32 for computation, and compute the power of 2 by subtracting the bias (127). + x_f32 = 2 ** ((x.to(torch.float32)) - 127) + + # If the exponent value was 255 (i.e., 2^(128)), this is a special case usually used to represent NaN or Inf. + # Since this custom format has no mantissa, treat 2^128 as NaN. + x_f32[x_f32 == 128] = float("nan") + return x_f32 + + +def quark_post_load_weights(self_attn: nn.Module, w: torch.Tensor, quant_format: str): + if "mxfp4" in quant_format: + # when dtype is bf16, the processing flow is to dynamic quantize bf16 tensor to uint8 tensor + # do w_kc (bf16) first to get the w_kc(uint8) w_s_kc(uint8) + # and w_vc repeating the same procedure of w_kc to get w_vc(uint8) w_s_vc(uint8) + if w.dtype == torch.bfloat16: + w_kc, w_vc = w.unflatten( + 0, (-1, self_attn.qk_nope_head_dim + self_attn.v_head_dim) + ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) + w_kc, w_s_kc = b_dynamic_mxfp4_quant(w_kc.transpose(-2, -1)) + w_kc = w_kc.transpose(-2, -1) + w_s_kc = w_s_kc.transpose(-2, -1) + w_vc, w_s_vc = b_dynamic_mxfp4_quant(w_vc) + w_s_kc = w_s_kc.transpose(1, 2).contiguous().transpose(1, 2) + w_s_vc = w_s_vc.contiguous().transpose(1, 2) + elif w.dtype == torch.uint8: # static quant for mxfp4 + # when dtype is uint8, it means the w has been quantized to mxfp4 format + # but we must separate it to w_kc and w_vc. + # The quantized tensor size is only half of original tensor size + # and the scaling factor is 1/32, the transpose behavior will be not correct + # need to upcast it to fp32 to separate w to w_kc and w_vc + # to ensure the following transpose behavior is correct + # and then do mxfp4 quant again + w = mxfp4_to_f32(w, True).to(torch.bfloat16) + w_scales = self_attn.kv_b_proj.weight_scale.repeat_interleave(32, dim=-1) + w_scales = e8m0_to_f32(w_scales).to(torch.bfloat16) + w = w * w_scales + w_kc, w_vc = w.unflatten( + 0, (-1, (self_attn.qk_nope_head_dim + self_attn.v_head_dim)) + ).split([self_attn.qk_nope_head_dim, self_attn.v_head_dim], dim=1) + w_kc, w_s_kc = b_dynamic_mxfp4_quant(w_kc.transpose(-2, -1)) + w_kc = w_kc.transpose(-2, -1) + w_s_kc = w_s_kc.transpose(-2, -1) + w_vc, w_s_vc = b_dynamic_mxfp4_quant(w_vc) + w_s_kc = w_s_kc.transpose(1, 2).contiguous().transpose(1, 2) + w_s_vc = w_s_vc.contiguous().transpose(1, 2) + + return w_kc, w_s_kc, w_vc, w_s_vc diff --git a/python/sglang/srt/layers/quantization/rocm_mxfp4_utils.py b/python/sglang/srt/layers/quantization/rocm_mxfp4_utils.py new file mode 100644 index 000000000..4659f76bd --- /dev/null +++ b/python/sglang/srt/layers/quantization/rocm_mxfp4_utils.py @@ -0,0 +1,13 @@ +from aiter.ops.triton.batched_gemm_afp4wfp4_pre_quant import ( + batched_gemm_afp4wfp4_pre_quant, +) +from aiter.ops.triton.fused_mxfp4_quant import ( + fused_flatten_mxfp4_quant, + fused_rms_mxfp4_quant, +) + +__all__ = [ + "fused_rms_mxfp4_quant", + "fused_flatten_mxfp4_quant", + "batched_gemm_afp4wfp4_pre_quant", +] diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py new file mode 100644 index 000000000..495beb009 --- /dev/null +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +import importlib.util +from typing import TYPE_CHECKING, List, Optional + +import torch +import torch.nn.functional as F +from torch.nn.parameter import Parameter + +from sglang.srt.custom_op import CustomOp +from sglang.srt.layers.amx_utils import _amx_process_weight_after_loading +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizeMethodBase, +) +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_hip, + set_weight_attrs, + use_intel_amx_backend, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +has_triton_kernels = importlib.util.find_spec("triton_kernels") is not None + + +_is_cpu_amx_available = cpu_has_amx_support() +_is_hip = is_hip() +_is_cpu = is_cpu() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + +if _use_aiter: + from aiter import ActivationType + from aiter.fused_moe import fused_moe + from aiter.ops.shuffle import shuffle_weight + + +class UnquantizedEmbeddingMethod(QuantizeMethodBase): + """Unquantized method for embeddings.""" + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + """Create weights for embedding layer.""" + weight = Parameter( + torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return F.linear(x, layer.weight, bias) + + def embedding(self, layer: torch.nn.Module, input_: torch.Tensor) -> torch.Tensor: + return F.embedding(input_, layer.weight) + + +class UnquantizedLinearMethod(LinearMethodBase): + """Linear method without quantization.""" + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + weight = Parameter( + torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter("weight", weight) + set_weight_attrs(weight, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if _is_cpu and _is_cpu_amx_available: + _amx_process_weight_after_loading(layer, ["weight"]) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + + if use_intel_amx_backend(layer): + x_shapes = x.shape + if len(x_shapes) == 3: + x = x.view(-1, x.shape[-1]) + output = torch.ops.sgl_kernel.weight_packed_linear( + x, layer.weight, bias, True # is_vnni + ) + if len(x_shapes) == 3: + output = output.view(x_shapes[0], x_shapes[1], -1) + return output + + return F.linear(x, layer.weight, bias) + + +class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): + """MoE method without quantization.""" + + def __init__(self, use_triton_kernels: bool = False): + super().__init__() + self.use_triton_kernels = use_triton_kernels + self.with_bias = False + + self.triton_kernel_moe_forward = None + self.triton_kernel_moe_with_bias_forward = None + if torch.cuda.is_available() and has_triton_kernels: + from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_forward as _tk_forward, + ) + from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_with_bias_forward as _tk_with_bias_forward, + ) + + self.triton_kernel_moe_forward = _tk_forward + self.triton_kernel_moe_with_bias_forward = _tk_with_bias_forward + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + with_bias: bool = False, + **extra_weight_attrs, + ): + self.with_bias = with_bias + + # Fused gate_up_proj (column parallel) + w13_weight_n, w13_weight_k = 2 * intermediate_size_per_partition, hidden_size + if self.use_triton_kernels: + w13_weight_n, w13_weight_k = w13_weight_k, w13_weight_n + w13_weight = torch.nn.Parameter( + torch.empty(num_experts, w13_weight_n, w13_weight_k, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + if self.with_bias: + w13_weight_bias = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_bias", w13_weight_bias) + set_weight_attrs(w13_weight_bias, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight_n, w2_weight_k = ( + hidden_size, + intermediate_size_per_partition, + ) + if self.use_triton_kernels: + w2_weight_n, w2_weight_k = w2_weight_k, w2_weight_n + w2_weight = torch.nn.Parameter( + torch.empty(num_experts, w2_weight_n, w2_weight_k, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + if self.with_bias: + w2_weight_bias = torch.nn.Parameter( + torch.empty(num_experts, hidden_size, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w2_weight_bias", w2_weight_bias) + set_weight_attrs(w2_weight_bias, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if _use_aiter: + layer.w13_weight = torch.nn.Parameter( + shuffle_weight(layer.w13_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + layer.w2_weight = torch.nn.Parameter( + shuffle_weight(layer.w2_weight.data, (16, 16)), + requires_grad=False, + ) + torch.cuda.empty_cache() + + # Pack weight for get better performance on CPU + if _is_cpu and _is_cpu_amx_available: + _amx_process_weight_after_loading(layer, ["w13_weight", "w2_weight"]) + + return + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + return self.forward( + layer=layer, + dispatch_output=dispatch_output, + ) + + def forward_cuda( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + moe_runner_config = self.moe_runner_config + + if self.use_triton_kernels: + if self.with_bias: + assert self.triton_kernel_moe_with_bias_forward is not None + output = self.triton_kernel_moe_with_bias_forward( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + b1=layer.w13_weight_bias, + b2=layer.w2_weight_bias, + topk_output=topk_output, + moe_runner_config=moe_runner_config, + w1_pcg=None, + w2_pcg=None, + ) + else: + assert self.triton_kernel_moe_forward is not None + output = self.triton_kernel_moe_forward( + hidden_states=x, + w1=layer.w13_weight, + w2=layer.w2_weight, + topk_output=topk_output, + moe_runner_config=moe_runner_config, + ) + return StandardCombineInput(hidden_states=output) + else: + if _use_aiter: + assert not moe_runner_config.no_combine, "unsupported" + topk_weights, topk_ids, _ = topk_output + if moe_runner_config.apply_router_weight_on_input: + assert ( + topk_weights.dim() == 2 + ), "`topk_weights` should be in shape (num_tokens, topk)" + _, topk = topk_weights.shape + assert ( + topk == 1 + ), "Only support topk=1 when `apply_router_weight_on_input` is True" + x = x * topk_weights.to(x.dtype) + topk_weights = torch.ones_like( + topk_weights, dtype=torch.float32 + ) # topk_weights must be FP32 (float32) + output = fused_moe( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + activation=( + ActivationType.Silu + if moe_runner_config.activation == "silu" + else ActivationType.Gelu + ), + ) + return StandardCombineInput(hidden_states=output) + else: + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + b13=getattr(layer, "w13_weight_bias", None), + b2=getattr(layer, "w2_weight_bias", None), + ) + return self.runner.run(dispatch_output, quant_info) + + def forward_cpu( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + moe_runner_config = self.moe_runner_config + + assert ( + moe_runner_config.activation == "silu" + ), f"activation = {moe_runner_config.activation} is not supported." + + if ( + use_intel_amx_backend(layer) + and not moe_runner_config.apply_router_weight_on_input + ): + from sglang.srt.layers.moe.topk import apply_topk_weights_cpu + + topk_weights, topk_ids, _ = topk_output + x, topk_weights = apply_topk_weights_cpu( + moe_runner_config.apply_router_weight_on_input, topk_weights, x + ) + output = torch.ops.sgl_kernel.fused_experts_cpu( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + False, # inplace # See [Note] inplace should be False in fused_experts. + False, # use_int8_w8a8 + False, # use_fp8_w8a16 + None, # w1_scale + None, # w2_scale + None, # block_size + None, # a1_scale + None, # a2_scale + True, # is_vnni + ) + return StandardCombineInput(hidden_states=output) + else: + from sglang.srt.layers.moe.fused_moe_native import moe_forward_native + + output = moe_forward_native( + layer, + x, + topk_output, + moe_runner_config, + ) + return StandardCombineInput(hidden_states=output) + + def forward_npu( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + import torch_npu + + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_weights, topk_ids, _ = dispatch_output.topk_output + + original_dtype = x.dtype + num_tokens = x.shape[0] + topk_weights = topk_weights.to(x.dtype) + topk_ids = topk_ids.to(torch.int32) + num_experts = layer.num_experts + top_k = layer.top_k + row_idx_len = num_tokens * top_k + row_idx = ( + torch.arange(0, row_idx_len, dtype=torch.int32, device=topk_weights.device) + .view(top_k, -1) + .permute(1, 0) + .contiguous() + ) + + hidden_states, expanded_row_idx, expanded_expert_idx = ( + torch_npu.npu_moe_init_routing( + x, row_idx=row_idx, expert_idx=topk_ids, active_num=num_tokens + ) + ) + + expert_tokens = torch_npu.npu_moe_compute_expert_tokens( + expanded_expert_idx, num_experts + ) + + expert_tokens = expert_tokens.to(torch.int64) + if layer.w13_weight.shape[-1] == layer.hidden_size: + w13 = layer.w13_weight.transpose(1, 2) + w2 = layer.w2_weight.transpose(1, 2) + + # gmm1: gate_up_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[w13], + split_item=2, + group_list_type=0, + group_type=0, + group_list=expert_tokens, + output_dtype=original_dtype, + )[0] + + # act_fn: + if self.moe_runner_config.activation == "silu": + hidden_states = torch_npu.npu_swiglu(hidden_states) + else: + from sglang.srt.layers.activation import GeluAndMul + + hidden_states = GeluAndMul()(hidden_states) + + # gmm2: down_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[w2], + split_item=2, + group_list_type=0, + group_type=0, + group_list=expert_tokens, + output_dtype=original_dtype, + )[0] + + final_hidden_states = torch_npu.npu_moe_finalize_routing( + hidden_states, + skip1=None, + skip2=None, + bias=None, + scales=topk_weights, + expanded_src_to_dst_row=expanded_row_idx, + export_for_source_row=topk_ids, + ) + + return StandardCombineInput(hidden_states=final_hidden_states) + + def forward_tpu(self, *args, **kwargs) -> CombineInput: + raise NotImplementedError("The TPU backend currently does not support MoE.") + + forward_native = forward_cpu diff --git a/python/sglang/srt/layers/quantization/utils.py b/python/sglang/srt/layers/quantization/utils.py new file mode 100644 index 000000000..63b8b6eb7 --- /dev/null +++ b/python/sglang/srt/layers/quantization/utils.py @@ -0,0 +1,564 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/utils/quant_utils.py + +from __future__ import annotations + +import re +from copy import deepcopy +from types import MappingProxyType +from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union + +import numpy +import torch + +from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant +from sglang.srt.utils import is_cuda + +if TYPE_CHECKING: + from sglang.srt.layers.quantization.base_config import QuantizationConfig + + +def get_scalar_types(): + """ + Returns: + tuple: (ScalarType, scalar_types) + """ + try: + from sgl_kernel.scalar_type import ScalarType, scalar_types + + return ScalarType, scalar_types + except ImportError: + + class MockScalarType: + pass + + class MockScalarTypes: + uint4b8 = "uint4b8" + uint8b128 = "uint8b128" + + def __getattr__(self, name): + return f"mock_{name}" + + return MockScalarType, MockScalarTypes() + + +ScalarType, scalar_types = get_scalar_types() + + +def is_layer_skipped( + prefix: str, + ignored_layers: List[str], + fused_mapping: Mapping[str, List[str]] = MappingProxyType({}), +) -> bool: + # prefix: model.layers.0.self_attn.q_proj + # proj_name: q_proj + proj_name = prefix.split(".")[-1] + + # Fused layers like gate_up_proj or qkv_proj will not be fused + # in the safetensors checkpoint. So, we convert the name + # from the fused version to unfused + check to make sure that + # each shard of the fused layer has the same scheme. + if proj_name in fused_mapping: + shard_prefixes = [ + prefix.replace(proj_name, shard_proj_name) + for shard_proj_name in fused_mapping[proj_name] + ] + + is_skipped = None + for shard_prefix in shard_prefixes: + is_shard_skipped = shard_prefix in ignored_layers + + if is_skipped is None: + is_skipped = is_shard_skipped + elif is_shard_skipped != is_skipped: + raise ValueError( + f"Detected some but not all shards of {prefix} " + "are quantized. All shards of fused layers " + "to have the same precision." + ) + else: + is_skipped = prefix in ignored_layers + if "gate_up_proj" in prefix: + prefix_gate = prefix.replace("gate_up_proj", "gate_proj") + prefix_up = prefix.replace("gate_up_proj", "up_proj") + if prefix_gate in ignored_layers and prefix_up in ignored_layers: + is_skipped = True + elif "experts" in prefix: + is_skipped = any( + [ + prefix in layer_name + for layer_name in ignored_layers + if "experts" in layer_name + ] + ) + + assert is_skipped is not None + return is_skipped + + +def per_tensor_dequantize( + tensor: torch.Tensor, inv_scale: Union[float, torch.Tensor] +) -> torch.Tensor: + fake_qweight = tensor.to(torch.float16) + dq_weight = fake_qweight * inv_scale + return dq_weight + + +def all_close_1d(x: torch.Tensor) -> bool: + assert len(x.shape) == 1 + return all(torch.allclose(x[0], x[i]) for i in range(x.shape[0])) + + +def convert_to_channelwise( + weight_scale: torch.Tensor, logical_widths: List[int] +) -> Tuple[torch.Tensor, torch.Tensor]: + # Create channelwise buffer + weight_scale_channel = torch.empty( + (sum(logical_widths), 1), dtype=torch.float32, device=weight_scale.device + ) + + # Handle scalar tensor case: broadcast same scale to all channels + if weight_scale.dim() == 0: + weight_scale_channel.fill_(weight_scale.item()) + return weight_scale_channel + + # Expand each scale to match the size of each logical matrix. + start = 0 + for idx, logical_width in enumerate(logical_widths): + end = start + logical_width + weight_scale_channel[start:end, :] = weight_scale[idx] + start = end + + return weight_scale_channel + + +def requantize_with_max_scale( + weight: torch.Tensor, weight_scale: torch.Tensor, logical_widths: List[int] +) -> Tuple[torch.Tensor, torch.Tensor]: + # Max scale to be used for requanitzation. + max_w_scale = weight_scale.max() + + # QKV / MLP is fused in the on disk checkpoint if any of the + # weight scales are still set to the default since we initialize + # N weight scales for N shards but we only load 1 weight scale + # from disk in this case. Skip requantization in this case (since) + # we already are quantized with the single scale. + # * Sample Model: nm-testing/Phi-3-mini-128k-instruct-FP8 + unfused_module_in_checkpoint = ( + weight_scale[-1] > torch.finfo(torch.float8_e4m3fn).min + ) + + # If unfused checkpoint, need requanize with the single scale. + if unfused_module_in_checkpoint: + start = 0 + for idx, logical_width in enumerate(logical_widths): + end = start + logical_width + weight_dq = per_tensor_dequantize(weight[start:end, :], weight_scale[idx]) + weight[start:end, :], _ = scaled_fp8_quant(weight_dq, max_w_scale) + start = end + + return max_w_scale, weight + + +def update_tensor_inplace(old: torch.Tensor, new: torch.Tensor) -> None: + old.copy_(new) + + +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/utils/layer_utils.py +# Newly generated tensors need to replace existing tensors that are +# already registered as parameters by vLLM (and won't be freed) +def replace_parameter( + mod: torch.nn.Module, name: str, new: Union[torch.Tensor, torch.nn.Parameter] +) -> None: + + old = getattr(mod, name) + if ( + type(old) is type(new) + and old.dtype == new.dtype + and old.untyped_storage().nbytes() == new.untyped_storage().nbytes() + ): + # If we can just update in-place to avoid re-registering + # can be faster if the underlying storage is the same + update_tensor_inplace(old, new) + else: + # Fallback re-register parameter, convert to Parameter if necessary + # this not only ensures we don't register a tensor as a parameter, but + # also ensures that all parameter subclasses get re-registered as + # parameters for `torch.compile` compatibility + if not isinstance(new, torch.nn.Parameter): + new = torch.nn.Parameter(new, requires_grad=False) + mod.register_parameter(name, torch.nn.Parameter(new, requires_grad=False)) + + +def assert_fp8_all_close(a: torch.Tensor, b: torch.Tensor): + assert a.shape == b.shape + assert a.dtype == b.dtype == torch.float8_e4m3fn + + a_u8 = a.view(torch.uint8) + b_u8 = b.view(torch.uint8) + diff_u8 = (a_u8.to(torch.int16) - b_u8.to(torch.int16)).abs() + + numel = a.numel() + + count_diff_sign = ((a_u8 >= 0) & (b_u8 < 0)).sum().item() + count_tiny_diff = (diff_u8 >= 1).sum().item() + count_large_diff = (diff_u8 >= 2).sum().item() + + assert ( + (count_diff_sign == 0) + and (count_tiny_diff / numel < 0.005) + and (count_large_diff == 0) + ), f"{count_diff_sign=} {count_tiny_diff=} {count_large_diff=} {numel=}" + + +# Match dynamic rules with module name (prefix) and override quantize +# config if module (prefix) matches a rule +def override_config(config: QuantizationConfig, prefix: str): + weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits) + if isinstance(weight_bits, int): + config.weight_bits = weight_bits + group_size = get_dynamic_override(config, prefix, "group_size", config.group_size) + if isinstance(group_size, int): + config.group_size = group_size + desc_act = get_dynamic_override(config, prefix, "desc_act", config.desc_act) + if isinstance(desc_act, bool): + config.desc_act = desc_act + + config.pack_factor = 32 // config.weight_bits # packed into int32 + if config.get_name() == "gptq_marlin": + is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym) + if isinstance(is_sym, bool): + config.is_sym = is_sym + + if (config.weight_bits, config.is_sym) not in config.TYPE_MAP: + raise ValueError( + "Unsupported quantization config: " + f"bits={config.weight_bits}, sym={config.is_sym}" + ) + + config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)] + elif config.get_name() == "gptq": + if config.weight_bits not in [2, 3, 4, 8]: + raise ValueError( + "Currently, only 2/3/4/8-bit weight quantization is " + f"supported for GPTQ, but got {config.weight_bits} bits." + ) + + +def get_dynamic_override( + config: QuantizationConfig, + layer_name: str, + key: Optional[str] = None, + default_value: Union[int, bool, None] = None, +) -> Union[Dict, int, bool, None]: + for pattern, pattern_dict in config.dynamic.items(): + # Negative match: matched modules are excluded from quantized init + if pattern.startswith("-:"): + if re.match(pattern.removeprefix("-:"), layer_name): + return False + # Positive match: matched modules have quant properties overrides + # base quant config + elif re.match(pattern.removeprefix("+:"), layer_name): + if key is None: + return pattern_dict + else: + return pattern_dict.get(key, default_value) + return default_value + + +def get_linear_quant_method( + config: QuantizationConfig, + layer: torch.nn.Module, + prefix: str, + linear_method_cls: type, +): + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.quantization.unquant import ( + UnquantizedEmbeddingMethod, + UnquantizedLinearMethod, + ) + from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead + + cloned_config = deepcopy(config) + parallel_lm_head_quantized = ( + isinstance(layer, ParallelLMHead) and cloned_config.lm_head_quantized + ) + + if isinstance(layer, LinearBase) or parallel_lm_head_quantized: + # False = skip module, None = no override, else = Positive match + if get_dynamic_override(cloned_config, layer_name=prefix) is False: + if parallel_lm_head_quantized: + return UnquantizedEmbeddingMethod() + return UnquantizedLinearMethod() + + if prefix: + # Dynamic per module/layer rules may override base config + override_config(cloned_config, prefix=prefix) + + return linear_method_cls(cloned_config) + return None + + +def get_pack_factor(num_bits): + assert 32 % num_bits == 0, f"Unsupported num_bits = {num_bits}" + return 32 // num_bits + + +def permute_rows( + q_w: torch.Tensor, + w_ref: torch.Tensor, + group_size: int, + test_perm: Optional[torch.Tensor] = None, +): + assert q_w.shape == w_ref.shape + + orig_device = q_w.device + k_size, _ = q_w.shape + + g_idx = torch.zeros((k_size,), dtype=torch.int32) + for i in range(k_size): + g_idx[i] = i // group_size + + # Simulate act_order by doing a random permutation on K + rand_perm = test_perm if test_perm is not None else torch.randperm(k_size) + + g_idx = g_idx[rand_perm].contiguous() + q_w = q_w[rand_perm, :].contiguous() + w_ref = w_ref[rand_perm, :].contiguous() + + return ( + w_ref.to(device=orig_device), + q_w.to(device=orig_device), + g_idx.to(device=orig_device), + rand_perm.to(device=orig_device), + ) + + +def pack_cols( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + pack_factor = get_pack_factor(num_bits) + assert size_n % pack_factor == 0 + + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(numpy.uint32) + + q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32) + + for i in range(pack_factor): + q_res |= q_w[:, i::pack_factor] << num_bits * i + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + q_res = q_res.contiguous() + + return q_res + + +def pack_rows( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + pack_factor = get_pack_factor(num_bits) + assert size_k % pack_factor == 0 + + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(numpy.uint32) + + q_res = numpy.zeros((size_k // pack_factor, size_n), dtype=numpy.uint32) + + for i in range(pack_factor): + q_res |= q_w[i::pack_factor, :] << num_bits * i + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + return q_res + + +def unpack_cols( + packed_q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + pack_factor = get_pack_factor(num_bits) + assert size_n % pack_factor == 0 + assert packed_q_w.shape == ( + size_k, + size_n // pack_factor, + ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format( + packed_q_w.shape, size_k, size_n, pack_factor + ) + + orig_device = packed_q_w.device + + packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32) + q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32) + + mask = (1 << num_bits) - 1 + for i in range(pack_factor): + vals = packed_q_w_cpu & mask + packed_q_w_cpu >>= num_bits + q_res[:, i::pack_factor] = vals + + q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device) + q_res = q_res.contiguous() + + return q_res + + +# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/utils/quant_utils.py +def quantize_weights( + w: torch.Tensor, + quant_type: ScalarType, + group_size: Optional[int], + zero_points: bool = False, + ref_zero_points_after_scales: bool = False, +): + assert ( + quant_type.is_integer() + ), "Floating point quantization may work but has not been tested" + assert not zero_points or group_size is not None, ( + "to have group zero points, group_size must be provided " + "(-1 group_size is channelwise)" + ) + + orig_device = w.device + orig_type = w.dtype + size_k, size_n = w.shape + + assert w.is_floating_point(), "w must be float" + + if group_size == -1: + group_size = size_k + + # Reshape to [groupsize, -1] + if group_size is not None and group_size < size_k: + w = w.reshape((-1, group_size, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((group_size, -1)) + + # Compute scale for each group + max_val = torch.max(w, 0, keepdim=True).values + min_val = torch.min(w, 0, keepdim=True).values + + max_q_val = quant_type.max() + min_q_val = quant_type.min() + + w_s = torch.Tensor([1.0]).to(w.device) # unscaled case + maybe_w_zp = None + if group_size is not None: + if zero_points: + assert not quant_type.is_signed() and quant_type.max() > 0 + w_s = (max_val - min_val).clamp(min=1e-5) / quant_type.max() + maybe_w_zp = ( + torch.round(torch.abs(min_val / w_s)).clamp(min_q_val, max_q_val).int() + ) + else: + # If the bias is such that there are no possible negative/positive + # values, set the max value to inf to avoid divide by 0 + w_s = torch.max( + abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)), + abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)), + ) + + # Quantize + w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0) + w_q = torch.clamp(w_q, min_q_val, max_q_val) + + # Compute ref (dequantized) + # For some kernels (namely Machete) the zero-points are applied after the + # scales are applied, for this case computing the reference in similar way + # allows us to use tighter error tolerances in our unit tests. + if ref_zero_points_after_scales and maybe_w_zp is not None: + w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s + else: + w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s + + if quant_type.has_bias(): + w_q += quant_type.bias + + # Restore original shapes + if group_size is not None and group_size < size_k: + + def reshape_w(w): + w = w.reshape((group_size, -1, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((size_k, size_n)).contiguous() + return w + + w_q = reshape_w(w_q) + w_ref = reshape_w(w_ref) + w_s = w_s.reshape((-1, size_n)).contiguous() + + if maybe_w_zp is not None: + maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous() + maybe_w_zp = maybe_w_zp.to(device=orig_device) + + return ( + w_ref.to(device=orig_device), + w_q.to(device=orig_device), + w_s if group_size is not None else None, + maybe_w_zp, + ) + + +SUPPORTED_GPTQ_QUANT_TYPES = [scalar_types.uint4b8, scalar_types.uint8b128] +SUPPORTED_GROUP_SIZES = [-1, 32, 64, 128] + + +def gptq_quantize_weights( + w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + act_order: bool, + test_perm: Optional[torch.Tensor] = None, +): + size_k, _ = w.shape + + assert w.is_floating_point(), "w must be float" + assert ( + quant_type in SUPPORTED_GPTQ_QUANT_TYPES + ), f"Unsupported gptq type = {quant_type}" + assert group_size in SUPPORTED_GROUP_SIZES + [ + size_k + ], f"Unsupported groupsize = {group_size}" + + w_ref, w_q, w_s, _ = quantize_weights(w, quant_type, group_size) + + # Apply act_order + g_idx = torch.empty(0, dtype=torch.int, device=w.device) + rand_perm = torch.empty(0, dtype=torch.int, device=w.device) + if act_order: + assert ( + group_size < size_k + ), "For act_order, groupsize = {} must be less than size_k = {}".format( + group_size, size_k + ) + + w_ref, w_q, g_idx, rand_perm = permute_rows(w_q, w_ref, group_size, test_perm) + + return w_ref, w_q, w_s, g_idx, rand_perm + + +def sort_weights(q_w: torch.Tensor, g_idx: torch.Tensor): + orig_device = q_w.device + + sort_indices = torch.argsort(g_idx).to(dtype=torch.int32) # Sort based on g_idx + + g_idx = g_idx[sort_indices].contiguous() + q_w = q_w[sort_indices, :].contiguous() + + return ( + q_w.to(device=orig_device), + g_idx.to(device=orig_device), + sort_indices.to(device=orig_device), + ) diff --git a/python/sglang/srt/layers/quantization/w4afp8.py b/python/sglang/srt/layers/quantization/w4afp8.py new file mode 100644 index 000000000..e95247041 --- /dev/null +++ b/python/sglang/srt/layers/quantization/w4afp8.py @@ -0,0 +1,346 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional + +import torch +from torch.nn import Module +from torch.nn.parameter import Parameter + +from sglang.srt.distributed.parallel_state import get_moe_expert_parallel_world_size +from sglang.srt.layers.linear import LinearBase, UnquantizedLinearMethod +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.layers.quantization.utils import is_layer_skipped +from sglang.srt.utils import is_npu, set_weight_attrs + +_is_npu = is_npu() +if not _is_npu: + from sglang.srt.layers.moe.cutlass_w4a8_moe import cutlass_w4a8_moe + +if TYPE_CHECKING: + from sglang.srt.layers.moe import MoeRunnerConfig + from sglang.srt.layers.moe.ep_moe.layer import EPMoE + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +ACTIVATION_SCHEMES = ["static", "dynamic"] + +logger = logging.getLogger(__name__) + + +class W4AFp8Config(QuantizationConfig): + """Config class for MIXED_PRECISION W4AFp8.""" + + def __init__( + self, + is_checkpoint_fp8_serialized: bool = True, + is_checkpoint_w4afp8_serialized: bool = True, + linear_activation_scheme: str = "dynamic", + moe_activation_scheme: str = "static", + ignored_layers: Optional[List[str]] = None, + weight_block_size: Optional[List[int]] = None, + group_size: int = 128, + ) -> None: + super().__init__() + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + self.is_checkpoint_w4afp8_serialized = is_checkpoint_w4afp8_serialized + if is_checkpoint_w4afp8_serialized: + logger.warning("Detected w4afp8 checkpoint. Please note that") + if moe_activation_scheme not in ACTIVATION_SCHEMES: + raise ValueError(f"Unsupported activation scheme {moe_activation_scheme}") + self.linear_activation_scheme = linear_activation_scheme + self.moe_activation_scheme = moe_activation_scheme + self.ignored_layers = ignored_layers or [] + self.weight_block_size = [128, 128] + self.group_size = group_size + + @classmethod + def get_name(cls) -> str: + return "w4afp8" + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.bfloat16, torch.float8_e4m3fn] + + @classmethod + def get_min_capability(cls) -> int: + return 90 + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> W4AFp8Config: + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_fp8_serialized = "fp8" in quant_method + is_checkpoint_w4afp8_serialized = "w4afp8" in quant_method + linear_activation_scheme = "dynamic" + moe_activation_scheme = "static" + weight_block_size = [128, 128] + return cls( + is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized, + is_checkpoint_w4afp8_serialized=is_checkpoint_w4afp8_serialized, + linear_activation_scheme=linear_activation_scheme, + moe_activation_scheme=moe_activation_scheme, + weight_block_size=weight_block_size, + ) + + def get_quant_method( + self, layer: torch.nn.Module, prefix: str + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.ep_moe.layer import EPMoE + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + from sglang.srt.managers.schedule_batch import global_server_args_dict + + if isinstance(layer, LinearBase): + if is_layer_skipped(prefix, self.ignored_layers): + return UnquantizedLinearMethod() + return Fp8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return W4AFp8MoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +def interleave_scales(scales: torch.Tensor) -> torch.Tensor: + """Interleave scales in groups of 4 similar to TRT-LLM implementation.""" + s_shape = scales.shape + # Reshape to separate groups of 4 + alignment = 4 if s_shape[2] % 4 == 0 else 1 + scales_interleaved = scales.reshape( + s_shape[0], s_shape[1], (s_shape[2] // alignment), alignment + ) + # Permute dimensions to interleave + scales_interleaved = scales_interleaved.permute(0, 2, 1, 3) + # Reshape back to original dimensions but with interleaved values + scales_interleaved = scales_interleaved.reshape( + s_shape[0], s_shape[2] // alignment, s_shape[1] * alignment + ) + return scales_interleaved.contiguous() + + +class W4AFp8MoEMethod(FusedMoEMethodBase): + def __init__(self, quant_config: W4AFp8Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: EPMoE, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + assert "weight_loader" in extra_weight_attrs + + # Fused gate_up_proj (column parallel) + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + intermediate_size_per_partition * 2, + hidden_size // 2, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + # down_proj (row parallel) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // 2, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.GROUP.value} + ) + w13_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size // self.quant_config.group_size, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale_inv", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + + w2_weight_scale = torch.nn.Parameter( + torch.zeros( + num_experts, + hidden_size, + intermediate_size_per_partition // self.quant_config.group_size, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale_inv", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + # Input scales + w13_input_scale = torch.nn.Parameter( + torch.ones((num_experts, 2), dtype=torch.bfloat16), + requires_grad=False, + ) + layer.register_parameter("w13_input_scale", w13_input_scale) + set_weight_attrs(w13_input_scale, extra_weight_attrs) + + w2_input_scale = torch.nn.Parameter( + torch.ones(num_experts, dtype=torch.bfloat16), + requires_grad=False, + ) + layer.register_parameter("w2_input_scale", w2_input_scale) + set_weight_attrs(w2_input_scale, extra_weight_attrs) + + # Pre-populate the strides + device = layer.w13_weight.device + + self.a_strides1 = torch.full( + (num_experts, 3), + hidden_size, + device=device, + dtype=torch.int64, + ) + self.c_strides1 = torch.full( + (num_experts, 3), + 2 * intermediate_size_per_partition, + device=device, + dtype=torch.int64, + ) + self.a_strides2 = torch.full( + (num_experts, 3), + intermediate_size_per_partition, + device=device, + dtype=torch.int64, + ) + self.c_strides2 = torch.full( + (num_experts, 3), + hidden_size, + device=device, + dtype=torch.int64, + ) + self.b_strides1 = self.a_strides1 + self.s_strides13 = self.c_strides1 + self.b_strides2 = self.a_strides2 + self.s_strides2 = self.c_strides2 + + self.expert_offsets = torch.empty( + (num_experts + 1), dtype=torch.int32, device=device + ) + self.problem_sizes1 = torch.empty( + (num_experts, 3), dtype=torch.int32, device=device + ) + self.problem_sizes2 = torch.empty( + (num_experts, 3), dtype=torch.int32, device=device + ) + + return + + def process_weights_after_loading(self, layer: Module) -> None: + dtype = torch.bfloat16 + device = layer.w2_weight.device + + # Interleave w13_weight_scale (gate_up_proj) + w13_weight_scale = layer.w13_weight_scale_inv.to(dtype) + w13_weight_scale = interleave_scales(w13_weight_scale) + layer.w13_weight_scale_inv = Parameter(w13_weight_scale, requires_grad=False) + + # Interleave w2_weight_scale (down_proj) + w2_weight_scale = layer.w2_weight_scale_inv.to(dtype) + w2_weight_scale = interleave_scales(w2_weight_scale) + layer.w2_weight_scale_inv = Parameter(w2_weight_scale, requires_grad=False) + + # Process input scales + w13_input_scale_max = layer.w13_input_scale.max().to(dtype).item() + new_w13_input_scale = torch.tensor( + [w13_input_scale_max], + dtype=dtype, + device=device, + ) + layer.w13_input_scale = Parameter(new_w13_input_scale, requires_grad=False) + + w2_input_scale_max = layer.w2_input_scale.max().to(dtype).item() + new_w2_input_scale = torch.tensor( + [w2_input_scale_max], dtype=dtype, device=device + ) + layer.w2_input_scale = Parameter(new_w2_input_scale, requires_grad=False) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer: EPMoE, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + from sglang.srt.layers.moe.cutlass_w4a8_moe import cutlass_w4a8_moe + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + topk_weights, topk_ids, _ = topk_output + local_topk_ids = topk_ids + if get_moe_expert_parallel_world_size() > 1: + local_topk_ids = torch.where( + topk_ids == -1, + layer.num_experts, + topk_ids, + ) + + output = cutlass_w4a8_moe( + layer.start_expert_id, + layer.end_expert_id, + layer.num_experts, + x, + layer.w13_weight, + layer.w2_weight, + layer.w13_weight_scale_inv, + layer.w2_weight_scale_inv, + topk_weights, + topk_ids, + local_topk_ids, + self.a_strides1, + self.b_strides1, + self.c_strides1, + self.a_strides2, + self.b_strides2, + self.c_strides2, + self.s_strides13, + self.s_strides2, + self.expert_offsets, + self.problem_sizes1, + self.problem_sizes2, + layer.w13_input_scale, + layer.w2_input_scale, + ) + if self.moe_runner_config.routed_scaling_factor is not None: + output *= self.moe_runner_config.routed_scaling_factor + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/quantization/w8a8_fp8.py b/python/sglang/srt/layers/quantization/w8a8_fp8.py new file mode 100644 index 000000000..c68591fc6 --- /dev/null +++ b/python/sglang/srt/layers/quantization/w8a8_fp8.py @@ -0,0 +1,305 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.parameter import ChannelQuantScaleParameter, ModelWeightParameter +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.fp8_kernel import ( + fp8_dtype, + is_fp8_fnuz, + per_token_group_quant_fp8, +) +from sglang.srt.layers.quantization.fp8_utils import ( + apply_fp8_linear, + cutlass_fp8_supported, + input_to_float8, + normalize_e4m3fn_to_e4m3fnuz, +) +from sglang.srt.utils import set_weight_attrs + +if TYPE_CHECKING: + from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +_is_fp8_fnuz = is_fp8_fnuz() + + +class W8A8Fp8Config(QuantizationConfig): + """Config class for W8A8 FP8 Quantization. + + Weight Quantization: + - Method: Static quantization + - Granularity: Per-channel + - Type: Symmetric + + Activation Quantization: + - Method: Dynamic quantization + - Granularity: Per-token + - Type: Symmetric + + Note: + - For models without offline quantization, weights will be quantized during model loading + - If CUTLASS is supported: Per-channel weight quantization is used + - If CUTLASS is not supported: Falls back to per-tensor weight quantization + """ + + def __init__(self, is_checkpoint_fp8_serialized: bool = False): + self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return [torch.float16, torch.bfloat16] + + @classmethod + def get_min_capability(cls) -> int: + return 89 + + @classmethod + def get_name(self) -> str: + return "w8a8_fp8" + + @classmethod + def get_config_filenames(cls) -> List[str]: + return [] + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> W8A8Fp8Config: + quant_method = cls.get_from_keys(config, ["quant_method"]) + is_checkpoint_fp8_serialized = ( + "compressed-tensors" in quant_method or "w8a8_fp8" in quant_method + ) + return cls(is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized) + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if isinstance(layer, LinearBase): + return W8A8Fp8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return W8A8FP8MoEMethod(self) + return None + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class W8A8Fp8LinearMethod(LinearMethodBase): + + def __init__(self, quantization_config: W8A8Fp8Config): + self.cutlass_fp8_supported = cutlass_fp8_supported() + self.quantization_config = quantization_config + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + weight = layer.weight + + if self.quantization_config.is_checkpoint_fp8_serialized: + weight_scale = layer.weight_scale.detach() + # If checkpoint offline quantized with w8a8_fp8, load the weight and weight_scale directly. + if _is_fp8_fnuz: + weight, weight_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=weight, weight_scale=weight_scale + ) + + layer.weight = Parameter(weight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + else: + # If checkpoint not offline quantized, quantize the weights with per-channel quantization. + if self.cutlass_fp8_supported: + # if cutlass supported, we use cutlass_scaled_mm + # which requires per-channel quantization on weight + qweight, weight_scale = per_token_group_quant_fp8( + layer.weight, layer.weight.shape[-1] + ) + weight_scale = weight_scale.t().contiguous() + else: + # if cutlass not supported, we fall back to use torch._scaled_mm + # which requires per tensor quantization on weight + qweight, weight_scale = input_to_float8(layer.weight, dtype=fp8_dtype) + + # Update the layer with the new values. + layer.weight = Parameter(qweight.t(), requires_grad=False) + layer.weight_scale = Parameter(weight_scale, requires_grad=False) + layer.input_scale = None + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + weight_dtype = ( + torch.float8_e4m3fn + if self.quantization_config.is_checkpoint_fp8_serialized + else params_dtype + ) + + weight_loader = extra_weight_attrs.get("weight_loader") + self.logical_widths = output_partition_sizes + + weight = ModelWeightParameter( + data=torch.empty( + sum(output_partition_sizes), + input_size_per_partition, + dtype=weight_dtype, + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + if self.quantization_config.is_checkpoint_fp8_serialized: + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) + else: + layer.weight_scale = None + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ): + return apply_fp8_linear( + x, + layer.weight, + layer.weight_scale, + bias=bias, + cutlass_fp8_supported=self.cutlass_fp8_supported, + ) + + +class W8A8FP8MoEMethod(FusedMoEMethodBase): + """MoE method for FP8. + Supports loading FP8 checkpoints with static weight scale and + dynamic/static activation scale. + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: W8A8Fp8Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + # WEIGHTS + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=fp8_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=fp8_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, 2 * intermediate_size_per_partition, 1, dtype=torch.float32 + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value} + ) + + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w13_input_scale = None + layer.register_parameter("w13_input_scale", w13_input_scale) + + w2_input_scale = None + layer.register_parameter("w2_input_scale", w2_input_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.w13_weight = Parameter(layer.w13_weight, requires_grad=False) + layer.w2_weight = Parameter(layer.w2_weight, requires_grad=False) + layer.w13_weight_scale = Parameter( + layer.w13_weight_scale.data, requires_grad=False + ) + layer.w2_weight_scale = Parameter( + layer.w2_weight_scale.data, requires_grad=False + ) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_fp8_w8a8=True, + per_channel_quant=True, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + ) + return self.runner.run(dispatch_output, quant_info) diff --git a/python/sglang/srt/layers/quantization/w8a8_int8.py b/python/sglang/srt/layers/quantization/w8a8_int8.py new file mode 100644 index 000000000..5ccb0259d --- /dev/null +++ b/python/sglang/srt/layers/quantization/w8a8_int8.py @@ -0,0 +1,1034 @@ +from __future__ import annotations + +import importlib +import sys +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Mapping, + Optional, + Tuple, + Union, + cast, +) + +import torch +from torch.nn.parameter import Parameter + +from sglang.srt.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from sglang.srt.layers.amx_utils import _amx_process_weight_after_loading +from sglang.srt.layers.moe import MoeRunner, MoeRunnerBackend, MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo +from sglang.srt.layers.parameter import ( + ChannelQuantScaleParameter, + ModelWeightParameter, + PerTensorScaleParameter, +) +from sglang.srt.layers.quantization.base_config import ( + FusedMoEMethodBase, + LinearMethodBase, + QuantizationConfig, + QuantizeMethodBase, +) +from sglang.srt.layers.quantization.compressed_tensors.utils import should_ignore_layer +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 +from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod +from sglang.srt.utils import ( + apply_module_patch, + cpu_has_amx_support, + is_cpu, + is_cuda, + is_npu, + set_weight_attrs, + use_intel_amx_backend, +) + +if TYPE_CHECKING: + from sglang.srt.layers.moe.token_dispatcher import ( + CombineInput, + StandardDispatchOutput, + ) + +_is_cuda = is_cuda() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() +if _is_cuda: + from sgl_kernel import int8_scaled_mm +_is_npu = is_npu() + +if _is_npu: + import torch_npu + + try: + from mindie_turbo import _ops as ops + from mindie_turbo.quantize.quant_utils import quant_per_tensor + except ImportError: + useMindIETurbo = False + else: + useMindIETurbo = True + + +# func refers to RMSNorm.__init__ +def npu_wrapper_rmsnorm_init(func): + def init(self, hidden_size: int, **extra_args) -> None: + func(self, hidden_size, **extra_args) + self.ignore_anti = True + # The Ascend w8a8_int8 quantization requires adding a bias in rmsnorm + self.bias = torch.nn.Parameter(torch.zeros(hidden_size), requires_grad=False) + + return init + + +# func refers to RMSNorm.forward_oot +def npu_wrapper_rmsnorm_forward(func): + def _rmsnorm_forward_oot( + self, + x: torch.Tensor, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + if not x.is_contiguous(): + x = x.contiguous() + if residual is not None: + out, _, residual_out = torch_npu.npu_add_rms_norm( + residual, x, self.weight.data, self.variance_epsilon + ) + out = out + self.bias + return out.to(x.dtype), residual_out + + out = torch_npu.npu_rms_norm(x, self.weight.data, self.variance_epsilon)[0] + out = out + self.bias + return out.to(x.dtype) + + return _rmsnorm_forward_oot + + +def npu_fused_experts( + hidden_states: torch.Tensor, + w13: torch.Tensor, + w13_scale: torch.Tensor, + w2: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + top_k: int, +): + original_shape = hidden_states.shape + original_dtype = hidden_states.dtype + scale_dtype = original_dtype if original_dtype == torch.bfloat16 else torch.float32 + if len(original_shape) == 3: + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + num_tokens = hidden_states.shape[0] + num_experts = w13.shape[0] + row_idx_len = num_tokens * top_k + row_idx = ( + torch.arange(0, row_idx_len, dtype=torch.int32, device=topk_weights.device) + .view(top_k, -1) + .permute(1, 0) + .contiguous() + ) + hidden_states, expanded_row_idx, expanded_expert_idx = ( + torch_npu.npu_moe_init_routing( + hidden_states, row_idx=row_idx, expert_idx=topk_ids, active_num=num_tokens + ) + ) + expert_tokens = torch_npu.npu_moe_compute_expert_tokens( + expanded_expert_idx, num_experts + ) + expert_tokens = expert_tokens.to(torch.int64) + # gmm1: gate_up_proj + hidden_states, pertoken_scale = torch_npu.npu_dynamic_quant(hidden_states) + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[w13], + scale=[w13_scale.to(scale_dtype)], + per_token_scale=[pertoken_scale], + split_item=2, + group_list_type=0, + group_type=0, + group_list=expert_tokens, + output_dtype=original_dtype, + )[0] + # act_fn: swiglu + hidden_states = torch_npu.npu_swiglu(hidden_states) + hidden_states, pertoken_scale = torch_npu.npu_dynamic_quant(hidden_states) + # gmm2: down_proj + hidden_states = torch_npu.npu_grouped_matmul( + x=[hidden_states], + weight=[w2], + scale=[w2_scale.to(scale_dtype)], + per_token_scale=[pertoken_scale], + split_item=2, + group_list_type=0, + group_type=0, + group_list=expert_tokens, + output_dtype=original_dtype, + )[0] + + final_hidden_states = torch_npu.npu_moe_finalize_routing( + hidden_states, + skip1=None, + skip2=None, + bias=None, + scales=topk_weights, + expanded_src_to_dst_row=expanded_row_idx, + export_for_source_row=topk_ids, + ) + if len(original_shape) == 3: + final_hidden_states = final_hidden_states.view(original_shape) + return final_hidden_states + + +class W8A8Int8Config(QuantizationConfig): + """Config class for W8A8 Int8 Quantization. + + - Weight: static, per-channel, symmetric + - Activation: dynamic, per-token, symmetric + """ + + def __init__(self, quant_config: Dict[str, Any] = {}): + super().__init__() + self.quant_description = quant_config + self.is_dynamic = quant_config.get("is_dynamic", False) + ignore = cast(List[str], quant_config.get("ignore", [])) + self.ignore = ignore if ignore is not None else [] + packed_modules_mapping = quant_config.get("packed_modules_mapping", {}) + self.packed_modules_mapping = ( + packed_modules_mapping if packed_modules_mapping is not None else {} + ) + + if _is_npu: + # Ascend w8a8_int8 quantization with bias, use wrappers to isolate the effects between models + for name in self.quant_description.keys(): + if "norm.bias" in name: + apply_module_patch( + "sglang.srt.layers.layernorm.RMSNorm", + "__init__", + [npu_wrapper_rmsnorm_init], + ) + apply_module_patch( + "sglang.srt.layers.layernorm.RMSNorm", + "forward_npu", + [npu_wrapper_rmsnorm_forward], + ) + + @classmethod + def get_supported_act_dtypes(cls) -> List[torch.dtype]: + return ( + [torch.float16, torch.bfloat16] + if not _is_npu + else [torch.int8, torch.float16, torch.bfloat16] + ) + + @classmethod + def get_min_capability(cls) -> int: + if _is_npu: + raise NotImplementedError( + 'NPU hardware does not support "get_min_capability" feature.' + ) + else: + return 75 + + @classmethod + def get_name(self) -> str: + return "w8a8_int8" + + @classmethod + def get_config_filenames(cls) -> List[str]: + filenames = [] + if _is_npu: + filenames.append("quant_model_description.json") + return filenames + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> W8A8Int8Config: + return cls(config) + + def get_quant_method( + self, + layer: torch.nn.Module, + prefix: str, + ) -> Optional[QuantizeMethodBase]: + from sglang.srt.layers.linear import LinearBase + from sglang.srt.layers.moe.fused_moe_triton import FusedMoE + + if _is_npu: + if isinstance(layer, LinearBase): + key = "model" + if "vision_model" in prefix: + key = "vision_model" + elif "visual" in prefix: + key = "visual" + packed_modules_mapping_subset = self.packed_modules_mapping.get(key, {}) + prefix_in_quant_config = prefix + proj_name = prefix.split(".")[-1] + if proj_name in packed_modules_mapping_subset: + prefix_in_quant_config = prefix.replace( + proj_name, packed_modules_mapping_subset[proj_name][0] + ) + self.is_dynamic = ( + self.quant_description[prefix_in_quant_config + ".weight"] + == "W8A8_DYNAMIC" + ) + if self.is_layer_skipped(prefix, packed_modules_mapping_subset): + return UnquantizedLinearMethod() + return ( + NPU_W8A8DynamicLinearMethod(self) + if self.is_dynamic + else NPU_W8A8LinearMethod(self) + ) + elif isinstance(layer, FusedMoE): + return NPU_W8A8MoEMethod(self) + return None + + if should_ignore_layer( + prefix, ignore=self.ignore, fused_mapping=self.packed_modules_mapping + ): + return UnquantizedLinearMethod() + if isinstance(layer, LinearBase): + return W8A8Int8LinearMethod(self) + elif isinstance(layer, FusedMoE): + return W8A8Int8MoEMethod(self) + return None + + def is_layer_skipped( + self, prefix: str, fused_mapping: Mapping[str, List[str]] = MappingProxyType({}) + ): + # adapted from vllm.model_executor.layers.quantization.utils.quant_utils.is_layer_skipped + proj_name = prefix.split(".")[-1] + if proj_name in fused_mapping: + shard_prefixes = [ + prefix.replace(proj_name, shard_proj_name) + for shard_proj_name in fused_mapping[proj_name] + ] + + is_skipped = None + for shard_prefix in shard_prefixes: + is_shard_skipped = ( + self.quant_description[shard_prefix + ".weight"] == "FLOAT" + ) + + if is_skipped is None: + is_skipped = is_shard_skipped + elif is_shard_skipped != is_skipped: + raise ValueError( + f"Detected some but not all shards of {prefix} " + "are quantized. All shards of fused layers " + "to have the same precision." + ) + else: + is_skipped = self.quant_description[prefix + ".weight"] == "FLOAT" + + assert is_skipped is not None + return is_skipped + + def get_scaled_act_names(self) -> List[str]: + return [] + + +class W8A8Int8LinearMethod(LinearMethodBase): + + def __init__(self, quantization_config: W8A8Int8Config): + self.quantization_config = quantization_config + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if _is_cpu: + assert ( + _is_cpu_amx_available + ), "W8A8Int8LinearMethod on CPU requires that CPU has AMX support" + _amx_process_weight_after_loading(layer, ["weight"]) + else: + layer.weight = Parameter(layer.weight.t(), requires_grad=False) + layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + + weight_loader = extra_weight_attrs.get("weight_loader") + self.logical_widths = output_partition_sizes + + weight = ModelWeightParameter( + data=torch.empty( + sum(output_partition_sizes), input_size_per_partition, dtype=torch.int8 + ), + input_dim=1, + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight", weight) + + weight_scale = ChannelQuantScaleParameter( + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), + output_dim=0, + weight_loader=weight_loader, + ) + layer.register_parameter("weight_scale", weight_scale) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ): + if use_intel_amx_backend(layer): + return torch.ops.sgl_kernel.int8_scaled_mm_with_quant( + x, + layer.weight, + layer.weight_scale, + bias, + x.dtype, + True, # is_vnni + ) + + x_q, x_scale = per_token_quant_int8(x) + + return int8_scaled_mm( + x_q, layer.weight, x_scale, layer.weight_scale, out_dtype=x.dtype, bias=bias + ) + + +class W8A8Int8MoEMethod(FusedMoEMethodBase): + """MoE method for INT8. + Supports loading INT8 checkpoints with static weight scale and + dynamic/static activation scale. + Also supports loading quantized FP16/BF16 model checkpoints with dynamic + activation scaling. The weight scaling factor will be initialized after + the model weights are loaded. + Args: + quant_config: The quantization config. + """ + + def __init__(self, quant_config: W8A8Int8Config): + self.quant_config = quant_config + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ): + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + tp_size = get_tensor_model_parallel_world_size() + + # WEIGHTS + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, 2 * intermediate_size_per_partition, 1, dtype=torch.float32 + ), + requires_grad=False, + ) + w2_weight_scale = torch.nn.Parameter( + torch.ones(num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value} + ) + + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + + w13_input_scale = None + layer.register_parameter("w13_input_scale", w13_input_scale) + + w2_input_scale = None + layer.register_parameter("w2_input_scale", w2_input_scale) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if _is_cpu: + assert ( + _is_cpu_amx_available + ), "W8A8Int8MoEMethod on CPU requires that CPU has AMX support" + _amx_process_weight_after_loading(layer, ["w13_weight", "w2_weight"]) + else: + layer.w13_weight = Parameter(layer.w13_weight, requires_grad=False) + layer.w2_weight = Parameter(layer.w2_weight, requires_grad=False) + layer.w13_weight_scale = Parameter( + layer.w13_weight_scale.data, requires_grad=False + ) + layer.w2_weight_scale = Parameter( + layer.w2_weight_scale.data, requires_grad=False + ) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + + def apply( + self, + layer: torch.nn.Module, + dispatch_output: StandardDispatchOutput, + ) -> torch.Tensor: + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + if use_intel_amx_backend(layer): + from sglang.srt.layers.moe.topk import apply_topk_weights_cpu + + topk_weights, topk_ids, _ = topk_output + x, topk_weights = apply_topk_weights_cpu( + self.moe_runner_config.apply_router_weight_on_input, topk_weights, x + ) + output = torch.ops.sgl_kernel.fused_experts_cpu( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + False, # inplace See [Note] inplace should be False in fused_experts. + True, # use_int8_w8a8 + False, # use_fp8_w8a16 + layer.w13_weight_scale, # w1_scale + layer.w2_weight_scale, # w2_scale + None, # block_size + layer.w13_input_scale, # a1_scale + layer.w2_input_scale, # a2_scale + True, # is_vnni + ) + return StandardCombineInput(hidden_states=output) + + quant_info = TritonMoeQuantInfo( + w13_weight=layer.w13_weight, + w2_weight=layer.w2_weight, + use_int8_w8a8=True, + per_channel_quant=True, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a13_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + ) + return self.runner.run(dispatch_output, quant_info) + + +class NPU_W8A8LinearMethodImpl: + """Linear method for NPU W8A8.""" + + def __init__(self) -> None: + # aclnn quant matmul requires to transpose matrix B, set to true by default. + self.transpose_weight = True + + @staticmethod + def get_weight( + input_size: int, + output_size: int, + params_dtype: torch.dtype = torch.bfloat16, + ) -> Dict[str, Any]: + params_dict = {"weight": torch.empty(output_size, input_size, dtype=torch.int8)} + return params_dict + + @staticmethod + def get_pertensor_param(params_dtype: torch.dtype) -> Dict[str, Any]: + params_dict = {} + params_dict["input_scale"] = torch.empty(1, dtype=params_dtype) + params_dict["input_offset"] = torch.empty(1, dtype=params_dtype) + return params_dict + + @staticmethod + def get_perchannel_param( + output_size: int, + params_dtype: torch.dtype, + ) -> Dict[str, Any]: + params_dict = {} + params_dict["quant_bias"] = torch.empty(output_size, dtype=torch.int32) + if params_dtype == torch.bfloat16: + params_dict["deq_scale"] = torch.empty(output_size, dtype=torch.float32) + elif params_dtype == torch.float16: + params_dict["deq_scale"] = torch.empty(output_size, dtype=torch.int64) + params_dict["weight_scale"] = torch.empty(output_size, 1, dtype=params_dtype) + params_dict["weight_offset"] = torch.empty(output_size, 1, dtype=params_dtype) + return params_dict + + @staticmethod + def apply( + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # To prevent import loops + from sglang.srt.layers.linear import RowParallelLinear + + original_dtype = x.dtype + if original_dtype != torch.int8: + x = torch_npu.npu_quantize( + x, + layer.aclnn_input_scale_reciprocal, + layer.aclnn_input_offset, + torch.qint8, + -1, + False, + ) + # Only fuse bias add into GEMM for rank 0 (this ensures that + # bias will not get added more than once in Attention TP>1 case) + if isinstance(layer, RowParallelLinear) and layer.tp_rank > 0: + quant_bias = None + else: + quant_bias = layer.quant_bias + return torch_npu.npu_quant_matmul( + x, + layer.weight, + layer.deq_scale, + bias=quant_bias, + output_dtype=original_dtype, + ) + + def process_weights_after_loading(self, layer): + expanding_factor = layer.weight.data.shape[1] + layer.aclnn_input_scale = torch.nn.Parameter( + layer.input_scale.data.repeat(expanding_factor).to(device="npu"), + requires_grad=False, + ) + layer.aclnn_input_scale_reciprocal = 1 / torch.nn.Parameter( + layer.input_scale.data.repeat(expanding_factor).to(device="npu"), + requires_grad=False, + ) + layer.aclnn_input_offset = torch.nn.Parameter( + layer.input_offset.data.repeat(expanding_factor).to(device="npu"), + requires_grad=False, + ) + if self.transpose_weight: + layer.weight.data = layer.weight.data.transpose(0, 1).contiguous() + layer.weight_scale.data = torch.flatten(layer.weight_scale.data) + layer.weight_offset.data = torch.flatten(layer.weight_offset.data) + + +class NPU_W8A8LinearMethodMTImpl: + """Linear method for NPU W8A8.""" + + def __init__(self) -> None: + self.transpose_weight = True + + @staticmethod + def get_weight( + input_size: int, + output_size: int, + params_dtype: torch.dtype = torch.bfloat16, + ) -> Dict[str, Any]: + params_dict = {"weight": torch.empty(output_size, input_size, dtype=torch.int8)} + return params_dict + + @staticmethod + def get_pertensor_param(params_dtype: torch.dtype) -> Dict[str, Any]: + params_dict = {} + params_dict["input_scale"] = torch.empty(1, dtype=params_dtype) + params_dict["input_offset"] = torch.empty(1, dtype=torch.int8) + return params_dict + + @staticmethod + def get_perchannel_param( + output_size: int, + params_dtype: torch.dtype, + ) -> Dict[str, Any]: + params_dict = {} + params_dict["quant_bias"] = torch.empty(output_size, dtype=torch.int32) + if params_dtype == torch.bfloat16: + params_dict["deq_scale"] = torch.empty(output_size, dtype=torch.float32) + elif params_dtype == torch.float16: + params_dict["deq_scale"] = torch.empty(output_size, dtype=torch.int64) + params_dict["weight_scale"] = torch.empty(output_size, 1, dtype=params_dtype) + params_dict["weight_offset"] = torch.empty(output_size, 1, dtype=params_dtype) + return params_dict + + @staticmethod + def apply( + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + # To prevent import loops + from sglang.srt.layers.linear import RowParallelLinear + + original_dtype = x.dtype + if original_dtype != torch.int8: + x = quant_per_tensor(x, layer.input_scale, layer.input_offset) + + # Only fuse bias add into GEMM for rank 0 (this ensures that + # bias will not get added more than once in Attention TP>1 case) + if isinstance(layer, RowParallelLinear) and layer.tp_rank > 0: + quant_bias = None + else: + quant_bias = layer.quant_bias + + return ops.quant_matmul( + x=x, weight=layer.weight, deq_scale=layer.deq_scale, deq_bias=quant_bias + ) + + def process_weights_after_loading(self, layer): + layer.aclnn_deq_scale = torch.nn.Parameter( + torch_npu.npu_trans_quant_param(layer.deq_scale.npu()).to(device="npu"), + requires_grad=False, + ) + + +class NPU_W8A8LinearMethod(LinearMethodBase): + """Linear method for NPU quantization. + + This class search for specific quantization + implementation supported on NPU hardware for linear methods. + + Args: + quant_config: The NPU quantization config. + """ + + def __init__(self, quantization_config: W8A8Int8Config) -> None: + self.quantization_config = quantization_config + self.quant_method = ( + NPU_W8A8LinearMethodMTImpl() + if useMindIETurbo + else NPU_W8A8LinearMethodImpl() + ) + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + weight_dict = self.quant_method.get_weight( + input_size_per_partition, output_size_per_partition, params_dtype + ) + for weight_name, weight_param in weight_dict.items(): + param = torch.nn.Parameter(weight_param, requires_grad=False) + set_weight_attrs(param, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter(weight_name, param) + set_weight_attrs(param, extra_weight_attrs) + + pertensor_dict = self.quant_method.get_pertensor_param(params_dtype) + for pertensor_name, pertensor_param in pertensor_dict.items(): + param = PerTensorScaleParameter( + data=pertensor_param, weight_loader=weight_loader + ) + # disable warning + param.ignore_warning = True + layer.register_parameter(pertensor_name, param) + + perchannel_dict = self.quant_method.get_perchannel_param( + output_size_per_partition, params_dtype + ) + for perchannel_name, perchannel_param in perchannel_dict.items(): + param = torch.nn.Parameter(perchannel_param, requires_grad=False) + set_weight_attrs(param, {"output_dim": 0}) + layer.register_parameter(perchannel_name, param) + set_weight_attrs(param, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if hasattr(self.quant_method, "process_weights_after_loading"): + self.quant_method.process_weights_after_loading(layer) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return self.quant_method.apply(layer, x, bias) + + +class NPU_W8A8DynamicLinearMethodImpl: + """Linear method for NPU W8A8_DYNAMIC.""" + + def __init__(self): + self.transpose_weight = True + + @staticmethod + def get_weight( + input_size: int, output_size: int, params_dtype: torch.dtype + ) -> Dict[str, Any]: + params_dict = {"weight": torch.empty(output_size, input_size, dtype=torch.int8)} + return params_dict + + @staticmethod + def get_pertensor_param(params_dtype: torch.dtype) -> Dict[str, Any]: + return {} + + @staticmethod + def get_perchannel_param( + output_size: int, + params_dtype: torch.dtype, + ) -> Dict[str, Any]: + params_dict = {} + params_dict["weight_scale"] = torch.empty(output_size, 1, dtype=params_dtype) + params_dict["weight_offset"] = torch.empty(output_size, 1, dtype=params_dtype) + return params_dict + + @staticmethod + def apply( + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + tp_rank: Optional[int] = 0, + ) -> torch.Tensor: + original_dtype = x.dtype + quant_out, dynamic_scale = torch_npu.npu_dynamic_quant(x) + return torch_npu.npu_quant_matmul( + quant_out, + layer.weight, + layer.weight_scale, + pertoken_scale=dynamic_scale, + bias=bias, + output_dtype=original_dtype, + ) + + def process_weights_after_loading(self, layer): + if self.transpose_weight: + layer.weight.data = layer.weight.data.transpose(0, 1).contiguous() + layer.weight_scale.data = layer.weight_scale.data.flatten() + layer.weight_scale_fp32 = layer.weight_scale.data.to(torch.float32) + layer.weight_offset.data = layer.weight_offset.data.flatten() + + +class NPU_W8A8DynamicLinearMethod(LinearMethodBase): + """Linear method for NPU quantization. + + This class search for specific quantization + implementations supported on NPU hardware for linear methods. + + Args: + quant_config: The NPU quantization config. + """ + + def __init__(self, quantization_config: W8A8Int8Config) -> None: + self.quantization_config = quantization_config + self.quant_method = NPU_W8A8DynamicLinearMethodImpl() + + def create_weights( + self, + layer: torch.nn.Module, + input_size_per_partition: int, + output_partition_sizes: List[int], + input_size: int, + output_size: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + output_size_per_partition = sum(output_partition_sizes) + weight_loader = extra_weight_attrs.get("weight_loader") + + weight_dict = self.quant_method.get_weight( + input_size_per_partition, output_size_per_partition, params_dtype + ) + for weight_name, weight_param in weight_dict.items(): + param = torch.nn.Parameter(weight_param, requires_grad=False) + set_weight_attrs(param, {"input_dim": 1, "output_dim": 0}) + layer.register_parameter(weight_name, param) + set_weight_attrs(param, extra_weight_attrs) + + pertensor_dict = self.quant_method.get_pertensor_param(params_dtype) + for pertensor_name, pertensor_param in pertensor_dict.items(): + param = PerTensorScaleParameter( + data=pertensor_param, weight_loader=weight_loader + ) + # disable warning + param.ignore_warning = True + layer.register_parameter(pertensor_name, param) + + perchannel_dict = self.quant_method.get_perchannel_param( + output_size_per_partition, params_dtype + ) + for perchannel_name, perchannel_param in perchannel_dict.items(): + param = torch.nn.Parameter(perchannel_param, requires_grad=False) + set_weight_attrs(param, {"output_dim": 0}) + layer.register_parameter(perchannel_name, param) + set_weight_attrs(param, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + if hasattr(self.quant_method, "process_weights_after_loading"): + self.quant_method.process_weights_after_loading(layer) + + def apply( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + return self.quant_method.apply(layer, x, bias) + + +class NPU_W8A8MoEMethod(FusedMoEMethodBase): + """MoE method for NPU quantization. + + This class search for specific quantization + implementations supported on NPU hardware for moe methods. + + Args: + quant_config: The NPU quantization config. + """ + + def __init__(self, quantization_config: W8A8Int8Config) -> None: + self.quantization_config = quantization_config + self.quant_method = self + + def create_weights( + self, + layer: torch.nn.Module, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + from sglang.srt.layers.moe.fused_moe_triton import FusedMoeWeightScaleSupported + + self.num_experts = num_experts + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value} + ) + + # weight + w13_weight = torch.nn.Parameter( + torch.empty( + num_experts, + 2 * intermediate_size_per_partition, + hidden_size, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight", w13_weight) + set_weight_attrs(w13_weight, extra_weight_attrs) + w2_weight = torch.nn.Parameter( + torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=torch.int8, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight", w2_weight) + set_weight_attrs(w2_weight, extra_weight_attrs) + # scale + w13_weight_scale = torch.nn.Parameter( + torch.empty( + num_experts, 2 * intermediate_size_per_partition, 1, dtype=torch.float32 + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + w2_weight_scale = torch.nn.Parameter( + torch.empty(num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) + # offset + w13_weight_offset = torch.nn.Parameter( + torch.empty( + num_experts, 2 * intermediate_size_per_partition, 1, dtype=torch.float32 + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_offset", w13_weight_offset) + set_weight_attrs(w13_weight_offset, extra_weight_attrs) + w2_weight_offset = torch.nn.Parameter( + torch.empty(num_experts, hidden_size, 1, dtype=torch.float32), + requires_grad=False, + ) + layer.register_parameter("w2_weight_offset", w2_weight_offset) + set_weight_attrs(w2_weight_offset, extra_weight_attrs) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + layer.w13_weight = Parameter( + layer.w13_weight.data.transpose(1, 2).contiguous(), requires_grad=False + ) + layer.w2_weight = Parameter( + layer.w2_weight.data.transpose(1, 2).contiguous(), requires_grad=False + ) + layer.w13_weight_scale = Parameter( + layer.w13_weight_scale.data.squeeze(-1).contiguous(), requires_grad=False + ) + layer.w2_weight_scale = Parameter( + layer.w2_weight_scale.data.squeeze(-1).contiguous(), requires_grad=False + ) + layer.w13_weight_offset = Parameter( + layer.w13_weight_offset.data.squeeze(-1).contiguous(), requires_grad=False + ) + layer.w2_weight_offset = Parameter( + layer.w2_weight_offset.data.squeeze(-1).contiguous(), requires_grad=False + ) + + def create_moe_runner( + self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig + ): + self.moe_runner_config = moe_runner_config + + def apply( + self, + layer, + dispatch_output: StandardDispatchOutput, + ) -> CombineInput: + from sglang.srt.layers.moe.token_dispatcher import StandardCombineInput + + x = dispatch_output.hidden_states + topk_output = dispatch_output.topk_output + + topk_weights, topk_ids, _ = topk_output + topk_ids = topk_ids.to(torch.int32) + topk_weights = topk_weights.to(x.dtype) + output = npu_fused_experts( + hidden_states=x, + w13=layer.w13_weight, + w13_scale=layer.w13_weight_scale, + w2=layer.w2_weight, + w2_scale=layer.w2_weight_scale, + topk_weights=topk_weights, + topk_ids=topk_ids, + top_k=topk_ids.shape[1], + ) + return StandardCombineInput(hidden_states=output) diff --git a/python/sglang/srt/layers/radix_attention.py b/python/sglang/srt/layers/radix_attention.py new file mode 100644 index 000000000..0719cdd29 --- /dev/null +++ b/python/sglang/srt/layers/radix_attention.py @@ -0,0 +1,116 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Radix attention.""" +from __future__ import annotations + +from enum import Enum +from typing import TYPE_CHECKING, Optional + +from torch import nn + +if TYPE_CHECKING: + from sglang.srt.layers.quantization.base_config import QuantizationConfig + from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +class AttentionType(Enum): + """ + Attention type. + Use string to be compatible with `torch.compile`. + """ + + # Decoder attention between previous layer Q/K/V + DECODER = "decoder" + # Encoder attention between previous layer Q/K/V + ENCODER_ONLY = "encoder_only" + + +class RadixAttention(nn.Module): + """ + The attention layer implementation. + """ + + def __init__( + self, + num_heads: int, + head_dim: int, + scaling: float, + num_kv_heads: int, + layer_id: int, + logit_cap: float = 0.0, + v_head_dim: int = -1, + sliding_window_size: int = -1, + is_cross_attention: bool = False, + pos_encoding_mode: str = "NONE", + logit_capping_method: str = "tanh", + quant_config: Optional[QuantizationConfig] = None, + attn_type: AttentionType = AttentionType.DECODER, + use_irope: bool = False, + prefix: str = "", + ): + super().__init__() + self.tp_q_head_num = num_heads + self.tp_k_head_num = num_kv_heads + self.tp_v_head_num = num_kv_heads + self.head_dim = head_dim + self.qk_head_dim = head_dim + self.v_head_dim = v_head_dim if v_head_dim != -1 else head_dim + self.scaling = scaling + self.layer_id = layer_id + self.logit_cap = logit_cap + self.sliding_window_size = sliding_window_size or -1 + self.is_cross_attention = is_cross_attention + self.use_irope = use_irope + self.k_scale = None + self.v_scale = None + self.k_scale_float = None + self.v_scale_float = None + self.quant_method = None + if quant_config is not None: + self.quant_method = quant_config.get_quant_method(self, prefix=prefix) + if self.quant_method is not None: + self.quant_method.create_weights(self) + self.attn_type = attn_type + + self.pos_encoding_mode = pos_encoding_mode + self.logit_capping_method = logit_capping_method + self.xai_temperature_len = -1 + + def forward( + self, + q, + k, + v, + forward_batch: ForwardBatch, + save_kv_cache: bool = True, + **kwargs, + ): + if k is not None: + # For cross-layer sharing, kv can be None + assert v is not None + if "k_rope" not in kwargs: + k = k.view(-1, self.tp_k_head_num, self.qk_head_dim) + v = v.view(-1, self.tp_v_head_num, self.v_head_dim) + else: + k = k.view(-1, self.tp_k_head_num, self.v_head_dim) + + return forward_batch.attn_backend.forward( + q, + k, + v, + self, + forward_batch, + save_kv_cache, + **kwargs, + ) diff --git a/python/sglang/srt/layers/rocm_linear_utils.py b/python/sglang/srt/layers/rocm_linear_utils.py new file mode 100644 index 000000000..ee7dd1f59 --- /dev/null +++ b/python/sglang/srt/layers/rocm_linear_utils.py @@ -0,0 +1,44 @@ +import torch +from aiter.ops.triton.fused_qk_concat import fused_qk_rope_cat +from aiter.ops.triton.gemm_a16w16 import gemm_a16w16 +from aiter.ops.triton.gemm_a16w16_atomic import gemm_a16w16_atomic + +from sglang.srt.utils import BumpAllocator + +__all__ = ["fused_qk_rope_cat"] + + +def aiter_dsv3_router_gemm( + hidden_states: torch.Tensor, + weight: torch.Tensor, + gemm_output_zero_allocator: BumpAllocator = None, +): + M = hidden_states.shape[0] + N = weight.shape[0] + y = None + + if M <= 256: + # TODO (cagri): convert to bfloat16 as part of another kernel to save time + # for now it is also coupled with zero allocator. + if gemm_output_zero_allocator != None: + y = gemm_output_zero_allocator.allocate(M * N).view(M, N) + else: + y = torch.zeros((M, N), dtype=torch.float32, device=hidden_states.device) + + if y is not None: + logits = gemm_a16w16_atomic(hidden_states, weight, y=y).to(hidden_states.dtype) + else: + logits = gemm_a16w16(hidden_states, weight) + + return logits + + +def get_dsv3_gemm_output_zero_allocator_size( + n_routed_experts: int, num_moe_layers: int, allocate_size: int, embedding_dim: int +): + if embedding_dim != 7168 or n_routed_experts != 256: + return 0 + + per_layer_size = 256 * (allocate_size + n_routed_experts) + + return num_moe_layers * per_layer_size diff --git a/python/sglang/srt/layers/rotary_embedding.py b/python/sglang/srt/layers/rotary_embedding.py new file mode 100644 index 000000000..05f068557 --- /dev/null +++ b/python/sglang/srt/layers/rotary_embedding.py @@ -0,0 +1,2017 @@ +# Adapted from https://raw.githubusercontent.com/vllm-project/vllm/refs/tags/v0.6.6.post1/vllm/model_executor/layers/rotary_embedding.py + +"""Rotary Positional Embeddings.""" +import itertools +import math +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from sglang.srt.custom_op import CustomOp +from sglang.srt.utils import ( + cpu_has_amx_support, + get_bool_env_var, + is_cpu, + is_cuda, + is_hip, + is_npu, +) + +_is_cuda = is_cuda() +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +_is_npu = is_npu() +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() + +if _is_cuda: + from sgl_kernel import apply_rope_with_cos_sin_cache_inplace +if _use_aiter: + from aiter.rotary_embedding import get_rope as aiter_get_rope + +if is_npu(): + import torch_npu + + +def _rotate_neox(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def _rotate_gptj(x: torch.Tensor) -> torch.Tensor: + x1 = x[..., ::2] + x2 = x[..., 1::2] + x = torch.stack((-x2, x1), dim=-1) + return x.flatten(-2) + + +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + cos = cos.unsqueeze(-2).to(x.dtype) + sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_neox_style: + return torch.cat((o1, o2), dim=-1) + else: + return torch.stack((o1, o2), dim=-1).flatten(-2) + + +class RotaryEmbedding(CustomOp): + """Original rotary positional embedding.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + # NOTE(ByronHsu): cache needs to be in FP32 for numerical stability + if not _is_cuda: + cache = cache.to(dtype) + + if ( + not (_is_cuda or _is_npu) or self.head_size not in [64, 128, 256, 512] + ) and not (_is_cpu and _is_cpu_amx_available): + from vllm._custom_ops import rotary_embedding + + self.vllm_rotary_embedding = rotary_embedding + + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + """Compute the inverse frequency.""" + # NOTE(woosuk): To exactly match the HF implementation, we need to + # use CPU to compute the cache and then move it to GPU. However, we + # create the cache on GPU for faster initialization. This may cause + # a slight numerical difference between the HF implementation and ours. + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + if offsets is not None: + positions = positions + offsets + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + return query, key + + def forward_npu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-npu implementation of forward().""" + import os + + if get_bool_env_var("SGLANG_ENABLE_TORCH_COMPILE"): + return self.forward_native(positions, query, key, offsets) + else: + rotary_mode = "half" + if self.is_neox_style: + rotary_mode = "half" + else: + rotary_mode = "interleave" + mrope_section = [0, 0, 0] + query_out, key_out = torch_npu.npu_mrope( + positions, + query, + key, + self.cos_sin_cache, + self.head_size, + mrope_section=mrope_section, + rotary_mode=rotary_mode, + ) + return query_out, key_out + + def forward_cpu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + positions = torch.add(positions, offsets) if offsets is not None else positions + if _is_cpu_amx_available: + return torch.ops.sgl_kernel.rotary_embedding_cpu( + positions, + query, + key, + self.head_size, + self.cos_sin_cache, + self.is_neox_style, + ) + else: + return self.forward_native(positions, query, key, offsets) + + def forward_cuda( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + fused_set_kv_buffer_arg=None, # Optional[FusedSetKVBufferArg] + ) -> Tuple[torch.Tensor, torch.Tensor]: + if _is_cuda and (self.head_size in [64, 128, 256, 512]): + apply_rope_with_cos_sin_cache_inplace( + positions=positions, + query=query, + key=key, + head_size=self.head_size, + cos_sin_cache=self.cos_sin_cache, + is_neox=self.is_neox_style, + # Compatible with old sgl-kernel + **( + dict(fused_set_kv_buffer_arg=fused_set_kv_buffer_arg) + if fused_set_kv_buffer_arg is not None + else {} + ), + ) + else: + assert ( + fused_set_kv_buffer_arg is None + ), "save kv cache is not supported for vllm_rotary_embedding." + self.cos_sin_cache = self.cos_sin_cache.to(query.device, dtype=query.dtype) + self.vllm_rotary_embedding( + positions, + query, + key, + self.head_size, + self.cos_sin_cache, + self.is_neox_style, + ) + return query, key + + def extra_repr(self) -> str: + s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}" + s += f", max_position_embeddings={self.max_position_embeddings}" + s += f", base={self.base}, is_neox_style={self.is_neox_style}" + return s + + +class LinearScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with linear scaling. + + It supports multiple scaling factors. Since multiple LoRA adapters may have + different scaling factors, we need multiple cos/sin caches. In this way, + instead of running rotary embedding kernel per lora, we can run multiple + lora in a batched way. + + In addition to that, we also keep the cos/sin cache for the scaling factor + of 1 (default) at all times. + + Exemplary for two scaling factors x=1, y and z with embeddings + [[x11, x12, ... x1m], ..., [xn1, xn2, ..., xnm]] and + [[y11, y12, ... y1o], ..., [yn1, yn2, ..., yno]], and + [[z11, z12, ... z1p], ..., [zn1, zn2, ..., znp]], + + we construct the cos/sin cache as follows: + [[x11, x12, ... x1m, y11, y12, ... y1o, z11, z12, ... z1p], + ... + [xn1, xn2, ... xnm, yn1, yn2, ... yno, zn1, zn2, ... znp]] + + We then use offsets to index into the cos/sin cache for + the respective scaling factors. + + The offset to cache can be accessed via `scaling_factor_to_offset` API. + + Credits to the Reddit user /u/kaiokendev + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factors: Union[List[float], float], + dtype: torch.dtype, + ) -> None: + if isinstance(scaling_factors, float): + scaling_factors = [scaling_factors] + self.scaling_factors: List[float] = scaling_factors # noqa + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + # Lazy initialized. + self._scaling_factor_to_offset: Dict[float, int] + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.base) + cache_list: List[torch.Tensor] = [] + # offsets to the next cache in a tensor. + # Each offset corresponds to the same index in scaling_factors. + offsets: List[int] = [] + for scaling_factor in self.scaling_factors: + # NOTE(woosuk): self.max_position_embeddings is the original + # maximum length before applying the rope scaling. + # Thus, the maximum length after applying the rope scaling is + # self.max_position_embeddings * self.scaling_factor. + max_len = self.max_position_embeddings * scaling_factor + t = torch.arange(max_len, dtype=torch.float) + t = t / scaling_factor + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + if not cache_list: + offset = 0 + else: + last_offset = offsets[-1] + next_max_len = cache_list[-1].shape[0] + offset = last_offset + next_max_len + offsets.append(offset) + cache_list.append(cache) + self._scaling_factor_to_offset = { + float(scaling_factor): offsets[i] + for i, scaling_factor in enumerate(self.scaling_factors) + } + assert len(self.scaling_factors) == len(offsets) + return torch.cat(cache_list, dim=0) + + @property + def scaling_factor_to_offset(self) -> Dict[float, int]: + return self._scaling_factor_to_offset + + +class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with Dynamic NTK scaling. + + Credits to the Reddit users /u/bloc97 and /u/emozilla + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + ) -> None: + self.scaling_factor = scaling_factor + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + def _compute_cos_sin_cache(self) -> torch.Tensor: + # NOTE(woosuk): self.max_position_embeddings is the original + # maximum length before applying the rope scaling. + # Thus, the maximum length after applying the rope scaling is + # self.max_position_embeddings * self.scaling_factor. + max_len = self.max_position_embeddings * self.scaling_factor + base = self.base * ( + (self.scaling_factor * max_len / self.max_position_embeddings) + - (self.scaling_factor - 1) + ) ** (self.rotary_dim / (self.rotary_dim - 2)) + inv_freq = self._compute_inv_freq(base) + t = torch.arange(max_len, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + +# Inverse dim formula to find dim based on number of rotations +def _yarn_find_correction_dim( + num_rotations: int, + dim: int, + base: float = 10000, + max_position_embeddings: int = 2048, +) -> float: + return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / ( + 2 * math.log(base) + ) + + +# Find dim range bounds based on rotations +def _yarn_find_correction_range( + low_rot: int, + high_rot: int, + dim: int, + base: float = 10000, + max_position_embeddings: int = 2048, +) -> Tuple[int, int]: + low = math.floor( + _yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings) + ) + high = math.ceil( + _yarn_find_correction_dim(high_rot, dim, base, max_position_embeddings) + ) + return max(low, 0), min(high, dim - 1) # Clamp values just in case + + +def _yarn_linear_ramp_mask( + low: float, high: float, dim: int, dtype: torch.dtype, device: torch.device = None +) -> torch.Tensor: + if low == high: + high += 0.001 # Prevent singularity + + linear_func = (torch.arange(dim, dtype=dtype, device=device) - low) / (high - low) + ramp_func = torch.clamp(linear_func, 0, 1) + return ramp_func + + +def _yarn_get_mscale(scale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * math.log(scale) + 1.0 + + +class YaRNScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with YaRN method. + + Credits to Peng et al. github.com/jquesnelle/yarn + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + *, + extrapolation_factor: float = 1, + attn_factor: float = 1, + beta_fast: int = 32, + beta_slow: int = 1, + ) -> None: + self.scaling_factor = scaling_factor + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + # Get n-d magnitude scaling corrected for interpolation + self.mscale = float(_yarn_get_mscale(self.scaling_factor) * attn_factor) + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor: + pos_freqs = self.base ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + inv_freq_extrapolation = 1.0 / pos_freqs + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs) + + low, high = _yarn_find_correction_range( + self.beta_fast, + self.beta_slow, + self.rotary_dim, + self.base, + self.max_position_embeddings, + ) + # Get n-d rotational scaling corrected for extrapolation + inv_freq_mask = ( + 1 + - _yarn_linear_ramp_mask(low, high, self.rotary_dim // 2, dtype=torch.float) + ) * self.extrapolation_factor + inv_freq = ( + inv_freq_interpolation * (1 - inv_freq_mask) + + inv_freq_extrapolation * inv_freq_mask + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.scaling_factor) + t = torch.arange( + self.max_position_embeddings * self.scaling_factor, dtype=torch.float32 + ) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() * self.mscale + sin = freqs.sin() * self.mscale + cache = torch.cat((cos, sin), dim=-1) + return cache + + +class Phi3LongRoPEScaledRotaryEmbedding(nn.Module): + """Phi3 family of models scaled rotary embedding. + + Based on the original RotaryEmbedding implementation. + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + original_max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + short_factor: List[float], + long_factor: List[float], + short_mscale: Optional[float] = None, + long_mscale: Optional[float] = None, + ): + super().__init__() + + if is_neox_style is False: + raise ValueError( + "`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style." + ) + + self.rotary_dim = rotary_dim + self.head_size = head_size + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.base = base + self.short_factor = short_factor + self.long_factor = long_factor + + scale = self.max_position_embeddings / self.original_max_position_embeddings + if scale <= 1.0: + scaling_factor = 1.0 + else: + scaling_factor = math.sqrt( + 1 + math.log(scale) / math.log(self.original_max_position_embeddings) + ) + if short_mscale is None: + short_mscale = scaling_factor + if long_mscale is None: + long_mscale = scaling_factor + + self.short_mscale = short_mscale + self.long_mscale = long_mscale + + short_cache = self._compute_cos_sin_cache( + original_max_position_embeddings, short_factor, short_mscale + ) + short_cache = short_cache.to(dtype) + self.register_buffer("short_cos_sin_cache", short_cache, persistent=False) + + long_cache = self._compute_cos_sin_cache( + max_position_embeddings, long_factor, long_mscale + ) + long_cache = long_cache.to(dtype) + self.register_buffer("long_cos_sin_cache", long_cache, persistent=False) + + long_short_cache = torch.cat( + [self.short_cos_sin_cache, self.long_cos_sin_cache], dim=0 + ) + self.register_buffer( + "long_short_cos_sin_cache", long_short_cache, persistent=False + ) + + def _compute_inv_freq(self, rescale_factors: List[float]) -> torch.Tensor: + rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32) + inv_freq = 1.0 / ( + rescale_factors + * ( + self.base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) + / self.rotary_dim + ) + ) + ) + return inv_freq + + def _compute_cos_sin_cache( + self, + max_position_embeddings: int, + rescale_factors: List[float], + mscale: float, + ) -> torch.Tensor: + inv_freq = self._compute_inv_freq(rescale_factors) + t = torch.arange(max_position_embeddings, dtype=torch.float) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() * mscale + sin = freqs.sin() * mscale + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + query = query.view(*query.shape[:-1], -1, self.head_size) + key = key.view(*key.shape[:-1], -1, self.head_size) + + k = self.original_max_position_embeddings + long_prompt_offset = ( + torch.any(positions > k).float() * torch.full_like(positions, k) + ).long() + idx = ( + torch.add(positions, long_prompt_offset) + if long_prompt_offset is not None + else positions + ) + self.long_short_cos_sin_cache: torch.Tensor = self.long_short_cos_sin_cache.to( + idx.device + ) + idx = torch.add(idx, offsets) if offsets is not None else idx + cos_sin = torch.index_select(self.long_short_cos_sin_cache, 0, idx) + + cos, sin = cos_sin.chunk(2, dim=-1) + cos = cos.repeat(1, 2).unsqueeze(-2) + sin = sin.repeat(1, 2).unsqueeze(-2) + + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = query_rot * cos + _rotate_neox(query_rot) * sin + query = torch.cat((query_rot, query_pass), dim=-1) + + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = key_rot * cos + _rotate_neox(key_rot) * sin + key = torch.cat((key_rot, key_pass), dim=-1) + + return query.flatten(-2), key.flatten(-2) + + +def yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: + if scale <= 1: + return 1.0 + return 0.1 * mscale * math.log(scale) + 1.0 + + +class DeepseekScalingRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with YaRN method. + + Credits to Peng et al. github.com/jquesnelle/yarn + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_factor: float, + dtype: torch.dtype, + *, + extrapolation_factor: float = 1, + attn_factor: float = 1, + beta_fast: int = 32, + beta_slow: int = 1, + mscale: float = 1, + mscale_all_dim: float = 0, + device: Optional[str] = "cuda" if not _is_npu else "npu", + ) -> None: + self.scaling_factor = scaling_factor + self.extrapolation_factor = extrapolation_factor + self.attn_factor = attn_factor + self.beta_fast = beta_fast + self.beta_slow = beta_slow + # Get n-d magnitude scaling corrected for interpolation. + self.mscale = float( + yarn_get_mscale(self.scaling_factor, float(mscale)) + / yarn_get_mscale(self.scaling_factor, float(mscale_all_dim)) + * attn_factor + ) + self.device = device + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + # Re-dispatch + if _is_hip: + self._forward_method = self.forward_native + + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor: + pos_freqs = self.base ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float, device=self.device) + / self.rotary_dim + ) + inv_freq_extrapolation = 1.0 / pos_freqs + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs) + + low, high = _yarn_find_correction_range( + self.beta_fast, + self.beta_slow, + self.rotary_dim, + self.base, + self.max_position_embeddings, + ) + # Get n-d rotational scaling corrected for extrapolation + inv_freq_mask = ( + 1 + - _yarn_linear_ramp_mask( + low, high, self.rotary_dim // 2, dtype=torch.float, device=self.device + ) + ) * self.extrapolation_factor + inv_freq = ( + inv_freq_interpolation * (1 - inv_freq_mask) + + inv_freq_extrapolation * inv_freq_mask + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.scaling_factor) + t = torch.arange( + self.max_position_embeddings * self.scaling_factor, + device=self.device, + dtype=torch.float32, + ) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() * self.mscale + sin = freqs.sin() * self.mscale + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """PyTorch-native implementation equivalent to forward().""" + dtype = query.dtype + query_rot = query[..., : self.rotary_dim] + key_rot = key[..., : self.rotary_dim] + if self.rotary_dim < self.head_size: + query_pass = query[..., self.rotary_dim :] + key_pass = key[..., self.rotary_dim :] + + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(positions.device) + cos_sin = self.cos_sin_cache[ + torch.add(positions, offsets) if offsets is not None else positions + ] + cos, sin = cos_sin.chunk(2, dim=-1) + if self.is_neox_style: + # NOTE(woosuk): Here we assume that the positions tensor has the + # shape [batch_size, seq_len]. + cos = cos.repeat(1, 1, 2).unsqueeze(-2) + sin = sin.repeat(1, 1, 2).unsqueeze(-2) + else: + cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2) + sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2) + + rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj + query_rot = query_rot * cos + rotate_fn(query_rot) * sin + key_rot = key_rot * cos + rotate_fn(key_rot) * sin + + if self.rotary_dim < self.head_size: + query = torch.cat((query_rot, query_pass), dim=-1) + key = torch.cat((key_rot, key_pass), dim=-1) + else: + query = query_rot + key = key_rot + return query.to(dtype), key.to(dtype) + + def forward_npu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # NOTE: now npu_mrope can only support `numQHeads*headSize <= 4096` pattern, + # and generalization to more scenarios will be supported in the future. + if query.shape[1] * query.shape[2] > 4096: + return self.forward_native(positions, query, key, offsets) + num_tokens = query.shape[0] + rotary_mode = "half" if self.is_neox_style else "interleave" + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(positions.device) + query_rot = query[..., : self.rotary_dim] + key_rot = key[..., : self.rotary_dim] + if self.rotary_dim < self.head_size: + query_pass = query[..., self.rotary_dim :] + key_pass = key[..., self.rotary_dim :] + + query_rot, key_rot = torch_npu.npu_mrope( + torch.add(positions, offsets) if offsets is not None else positions, + query_rot.reshape(num_tokens, -1), + key_rot.reshape(num_tokens, -1), + self.cos_sin_cache, + self.rotary_dim, + mrope_section=[0, 0, 0], + rotary_mode=rotary_mode, + ) + query_rot = query_rot.reshape(num_tokens, -1, self.rotary_dim) + key_rot = key_rot.reshape(num_tokens, -1, self.rotary_dim) + + if self.rotary_dim < self.head_size: + query = torch.cat((query_rot, query_pass), dim=-1) + key = torch.cat((key_rot, key_pass), dim=-1) + else: + query = query_rot + key = key_rot + return query, key + + def forward_cpu( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + positions = torch.add(positions, offsets) if offsets is not None else positions + if _is_cpu_amx_available: + return torch.ops.sgl_kernel.rotary_embedding_cpu( + positions, query, key, self.head_size, self.cos_sin_cache, False + ) + else: + return self.forward_native(positions, query, key, offsets) + + +class Llama3RotaryEmbedding(RotaryEmbedding): + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + scaling_factor: float, + low_freq_factor: float, + high_freq_factor: float, + orig_max_position: int, + ) -> None: + self.scaling_factor = scaling_factor + self.low_freq_factor = low_freq_factor + self.high_freq_factor = high_freq_factor + self.orig_max_position = orig_max_position + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freqs = super()._compute_inv_freq(base) + low_freq_wavelen = self.orig_max_position / self.low_freq_factor + high_freq_wavelen = self.orig_max_position / self.high_freq_factor + + wave_len = 2 * math.pi / inv_freqs + if self.low_freq_factor != self.high_freq_factor: + smooth = (self.orig_max_position / wave_len - self.low_freq_factor) / ( + self.high_freq_factor - self.low_freq_factor + ) + else: + smooth = 0 + new_freqs = torch.where( + wave_len < high_freq_wavelen, + inv_freqs, + torch.where( + wave_len > low_freq_wavelen, + inv_freqs / self.scaling_factor, + (1 - smooth) * inv_freqs / self.scaling_factor + smooth * inv_freqs, + ), + ) + return new_freqs + + +class Llama4VisionRotaryEmbedding(RotaryEmbedding): + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ): + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freqs = super()._compute_inv_freq(base) + inv_freqs = inv_freqs[: (self.rotary_dim // 2)] + return inv_freqs + + def _compute_cos_sin_cache(self) -> torch.Tensor: + inv_freq = self._compute_inv_freq(self.base) + + # self.max_position_embeddings here is number of image patches + # i.e. (image_size // patch_size) ** 2 + num_patches = self.max_position_embeddings + img_idx = torch.arange(num_patches, dtype=torch.int32).reshape(num_patches, 1) + img_idx = torch.cat([img_idx, img_idx[:1]], dim=0) + img_idx[-1, -1] = -2 # set to ID_CLS_TOKEN + num_patches_single_dim = int(math.sqrt(num_patches)) + frequencies_x = img_idx % num_patches_single_dim + frequencies_y = img_idx // num_patches_single_dim + freqs_x = ( + (frequencies_x + 1)[..., None] * inv_freq[None, None, :] + ).repeat_interleave(2, dim=-1) + freqs_y = ( + (frequencies_y + 1)[..., None] * inv_freq[None, None, :] + ).repeat_interleave(2, dim=-1) + freqs = torch.cat([freqs_x, freqs_y], dim=-1).float().contiguous()[..., ::2] + freqs = freqs.masked_fill(img_idx.reshape(-1, 1, 1) < 0, 0) + cache = torch.view_as_complex( + torch.stack([torch.cos(freqs), torch.sin(freqs)], dim=-1) + ) + return cache + + def forward( + self, + query: torch.Tensor, + key: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(query.device) + query_ = torch.view_as_complex(query.float().reshape(*query.shape[:-1], -1, 2)) + key_ = torch.view_as_complex(key.float().reshape(*key.shape[:-1], -1, 2)) + broadcast_shape = [ + d if i == 1 or i == (query_.ndim - 1) else 1 + for i, d in enumerate(query_.shape) + ] + freqs_ci = self.cos_sin_cache.view(*broadcast_shape) + query_out = torch.view_as_real(query_ * freqs_ci).flatten(3) + key_out = torch.view_as_real(key_ * freqs_ci).flatten(3) + return query_out.type_as(query), key_out.type_as(key) + + +class DynamicNTKAlphaRotaryEmbedding(RotaryEmbedding): + """RotaryEmbedding extended with Dynamic NTK scaling. + + Credits to the Reddit users /u/bloc97 and /u/emozilla + """ + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + scaling_alpha: float, + dtype: torch.dtype, + ) -> None: + self.scaling_alpha = scaling_alpha + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + def _compute_cos_sin_cache(self) -> torch.Tensor: + max_len = self.max_position_embeddings + base = self.base * self.scaling_alpha ** ( + self.rotary_dim / (self.rotary_dim - 2) + ) + + inv_freq = self._compute_inv_freq(base) + t = torch.arange(max_len, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + +class MRotaryEmbedding(RotaryEmbedding): + """Rotary Embedding with Multimodal Sections.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + mrope_section: Optional[List[int]] = None, + ) -> None: + super().__init__( + head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype + ) + + self.mrope_section = mrope_section + if self.mrope_section: + expected_sum = rotary_dim // 2 + actual_sum = sum(self.mrope_section) + if actual_sum != expected_sum: + print( + f"MRoPE section sum mismatch: expected {expected_sum}, got {actual_sum}. " + f"Adjusting mrope_section to match rotary_dim // 2 = {expected_sum}" + ) + # Auto-correct by scaling the mrope_section proportionally + if actual_sum > 0: + scale_factor = expected_sum / actual_sum + self.mrope_section = [ + max(1, int(section * scale_factor)) + for section in self.mrope_section + ] + # Ensure the sum exactly matches by adjusting the last element + current_sum = sum(self.mrope_section) + if current_sum != expected_sum: + self.mrope_section[-1] += expected_sum - current_sum + else: + # If all sections are 0, create a default distribution + self.mrope_section = [ + expected_sum // len(self.mrope_section) + ] * len(self.mrope_section) + # Handle remainder + remainder = expected_sum % len(self.mrope_section) + for i in range(remainder): + self.mrope_section[i] += 1 + + print( + f"Corrected mrope_section: {self.mrope_section} (sum={sum(self.mrope_section)})" + ) + + @torch.compile(dynamic=True) + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """PyTorch-native implementation equivalent to forward(). + + Args: + positions: + [num_tokens,] (text only) or + [3, num_tokens] (T/H/W positions with multimodal inputs) + query: [num_tokens, num_heads * head_size] + key: [num_tokens, num_kv_heads * head_size] + """ + assert positions.ndim == 1 or positions.ndim == 2 + + num_tokens = positions.shape[-1] + cos_sin = self.cos_sin_cache[positions] + cos, sin = cos_sin.chunk(2, dim=-1) + if positions.ndim == 2: + assert self.mrope_section + + cos = torch.cat( + [m[i] for i, m in enumerate(cos.split(self.mrope_section, dim=-1))], + dim=-1, + ) + sin = torch.cat( + [m[i] for i, m in enumerate(sin.split(self.mrope_section, dim=-1))], + dim=-1, + ) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + return query, key + + # Copied from https://github.com/huggingface/transformers/blob/c8e0e603de9b3d49161a15fe6e8ea84badfb5d02/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py#L1439 + @staticmethod + def get_rope_index( + spatial_merge_size: int, + image_token_id: int, + video_token_id: int, + vision_start_token_id: int, + model_type: str, + tokens_per_second: Optional[int] = None, + input_ids: Optional[torch.LongTensor] = None, + image_grid_thw: Optional[torch.LongTensor] = None, + video_grid_thw: Optional[torch.LongTensor] = None, + second_per_grid_ts: Optional[torch.Tensor] = None, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + mrope_position_deltas = [] + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + for i, input_ids in enumerate(total_input_ids): + image_nums, video_nums = 0, 0 + vision_start_indices = torch.argwhere( + input_ids == vision_start_token_id + ).squeeze(1) + vision_tokens = input_ids[vision_start_indices + 1] + image_nums = (vision_tokens == image_token_id).sum() + video_nums = (vision_tokens == video_token_id).sum() + input_tokens = input_ids.tolist() + llm_pos_ids_list: list = [] + st = 0 + remain_images, remain_videos = image_nums, video_nums + for _ in range(image_nums + video_nums): + if image_token_id in input_tokens and remain_images > 0: + ed_image = input_tokens.index(image_token_id, st) + else: + ed_image = len(input_tokens) + 1 + if video_token_id in input_tokens and remain_videos > 0: + ed_video = input_tokens.index(video_token_id, st) + else: + ed_video = len(input_tokens) + 1 + if ed_image < ed_video: + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + second_per_grid_t = 0 + image_index += 1 + remain_images -= 1 + ed = ed_image + else: + t, h, w = ( + video_grid_thw[video_index][0], + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + if second_per_grid_ts is not None: + second_per_grid_t = second_per_grid_ts[video_index] + else: + second_per_grid_t = 1.0 + video_index += 1 + remain_videos -= 1 + ed = ed_video + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + text_len = ed - st + + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if len(llm_pos_ids_list) > 0 + else 0 + ) + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx + ) + + if model_type == "qwen2_5_vl": + range_tensor = torch.arange(llm_grid_t).view(-1, 1) + expanded_range = range_tensor.expand( + -1, llm_grid_h * llm_grid_w + ) + + time_tensor = ( + expanded_range * second_per_grid_t * tokens_per_second + ) + + time_tensor_long = time_tensor.long() + t_index = time_tensor_long.flatten() + elif model_type == "qwen2_vl": + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + else: + raise RuntimeError("Unimplemented") + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + text_len + st_idx + ) + st = ed + llm_grid_t * llm_grid_h * llm_grid_w + + if st < len(input_tokens): + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if len(llm_pos_ids_list) > 0 + else 0 + ) + text_len = len(input_tokens) - st + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx + ) + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, :] = llm_positions.to(position_ids.device) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]) + ) + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=input_ids.device + ).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + s = input_ids.shape[1] + position_ids = torch.arange(s) + position_ids = ( + position_ids.unsqueeze(0).expand(3, -1, -1).to(input_ids.device) + ) + max_position_ids = position_ids.max(0, keepdim=False)[0].max( + -1, keepdim=True + )[0] + mrope_position_deltas = max_position_ids + 1 - s + return position_ids, mrope_position_deltas + + # Adapted from https://github.com/vllm-project/vllm/blob/3779eb8c81449b924a23457fc77e45a0e6171178/vllm/model_executor/layers/rotary_embedding.py#L1120 + @staticmethod + def get_rope_index_glm4v( + input_ids: torch.Tensor, + hf_config: Any, + image_grid_thw: Union[list[list[int]], torch.Tensor], + video_grid_thw: Union[list[list[int]], torch.Tensor], + attention_mask: torch.Tensor, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Get mrope input positions and delta value for GLM4V.""" + image_token_id = hf_config.image_token_id + video_start_token_id = hf_config.video_start_token_id + video_end_token_id = hf_config.video_end_token_id + spatial_merge_size = hf_config.vision_config.spatial_merge_size + + mrope_position_deltas = [] + if input_ids is not None and ( + image_grid_thw is not None or video_grid_thw is not None + ): + total_input_ids = input_ids + if attention_mask is None: + attention_mask = torch.ones_like(total_input_ids) + position_ids = torch.ones( + 3, + input_ids.shape[0], + input_ids.shape[1], + dtype=input_ids.dtype, + device=input_ids.device, + ) + image_index, video_index = 0, 0 + video_group_index = 0 + attention_mask = attention_mask.to(total_input_ids.device) + for i, input_ids in enumerate(total_input_ids): + input_ids = input_ids[attention_mask[i] == 1] + input_tokens = input_ids.tolist() + + input_token_type = [] + video_check_flg = False + for token in input_tokens: + if token == video_start_token_id: + video_check_flg = True + elif token == video_end_token_id: + video_check_flg = False + + if token == image_token_id and not video_check_flg: + input_token_type.append("image") + elif token == image_token_id and video_check_flg: + input_token_type.append("video") + else: + input_token_type.append("text") + + input_type_group = [] + for key, group in itertools.groupby( + enumerate(input_token_type), lambda x: x[1] + ): + group = list(group) + start_index = group[0][0] + end_index = group[-1][0] + 1 + input_type_group.append((key, start_index, end_index)) + + llm_pos_ids_list = [] + video_frame_num = 1 + for modality_type, start_idx, end_idx in input_type_group: + st_idx = ( + llm_pos_ids_list[-1].max() + 1 + if len(llm_pos_ids_list) > 0 + else 0 + ) + + if modality_type == "image": + t, h, w = ( + image_grid_thw[image_index][0], + image_grid_thw[image_index][1], + image_grid_thw[image_index][2], + ) + llm_grid_t, llm_grid_h, llm_grid_w = ( + t.item(), + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + t_index = ( + torch.arange(llm_grid_t) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(llm_grid_t, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(llm_grid_t, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + st_idx + ) + + image_index += 1 + video_frame_num = 1 + + elif modality_type == "video": + t, h, w = ( + video_frame_num, + video_grid_thw[video_index][1], + video_grid_thw[video_index][2], + ) + + llm_grid_t, llm_grid_h, llm_grid_w = ( + t, + h.item() // spatial_merge_size, + w.item() // spatial_merge_size, + ) + + for t_idx in range(llm_grid_t): + t_index = ( + torch.tensor(t_idx) + .view(-1, 1) + .expand(-1, llm_grid_h * llm_grid_w) + .flatten() + ) + + h_index = ( + torch.arange(llm_grid_h) + .view(1, -1, 1) + .expand(1, -1, llm_grid_w) + .flatten() + ) + w_index = ( + torch.arange(llm_grid_w) + .view(1, 1, -1) + .expand(1, llm_grid_h, -1) + .flatten() + ) + llm_pos_ids_list.append( + torch.stack([t_index, h_index, w_index]) + st_idx + ) + + video_group_index += 1 + + if video_group_index >= video_grid_thw[video_index][0]: + video_index += 1 + video_group_index = 0 + + video_frame_num += 1 + + else: + text_len = end_idx - start_idx + llm_pos_ids_list.append( + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx + ) + + video_frame_num = 1 + + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1) + position_ids[..., i, attention_mask[i] == 1] = llm_positions.to( + position_ids.device + ) + mrope_position_deltas.append( + llm_positions.max() + 1 - len(total_input_ids[i]) + ) + mrope_position_deltas = torch.tensor( + mrope_position_deltas, device=input_ids.device + ).unsqueeze(1) + return position_ids, mrope_position_deltas + else: + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0) + .expand(3, -1, -1) + .to(attention_mask.device) + ) + max_position_ids = position_ids.max(0, keepdim=False)[0].max( + -1, keepdim=True + )[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + +class DualChunkRotaryEmbedding(CustomOp): + """Rotary positional embedding for Dual Chunk Attention.""" + + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + chunk_size: int, + local_size: int, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.chunk_size = chunk_size + self.local_size = local_size + self.dtype = dtype + self.device = torch.device(f"cuda:{torch.cuda.current_device()}") + (q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache) = ( + self._compute_cos_sin_cache() + ) + + self.register_buffer("cos_sin_q_cache", q_cache, persistent=False) + self.register_buffer("cos_sin_qc_cache", qc_cache, persistent=False) + self.register_buffer("cos_sin_k_cache", k_cache, persistent=False) + self.register_buffer( + "cos_sin_qc_no_clamp_cache", qc_no_clamp_cache, persistent=False + ) + self.register_buffer("cos_sin_q_inter_cache", q_inter_cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + """Compute the inverse frequency.""" + # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`. + # However, we use `torch.arange(..., dtype=torch.float)` instead to + # avoid numerical issues with large base values (e.g., 10000000). + # This may cause a slight numerical difference between the HF + # implementation and ours. + # NOTE(woosuk): To exactly match the HF implementation, we need to + # use CPU to compute the cache and then move it to GPU. However, we + # create the cache on GPU for faster initialization. This may cause + # a slight numerical difference between the HF implementation and ours. + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + chunk_len = self.chunk_size - self.local_size + q_t = torch.arange(chunk_len, dtype=torch.float) + qc_t = (torch.arange(chunk_len, dtype=torch.float) + chunk_len).clamp( + max=self.chunk_size + ) + k_t = torch.arange(self.max_position_embeddings, dtype=torch.float) % chunk_len + + # count from chunk_len, no clamp(self.chunk_size) restriction + qc_no_clamp_t = torch.arange(chunk_len, dtype=torch.float) + chunk_len + # count from self.chunk_size for q_inter's rope + q_inter_t = torch.arange(chunk_len, dtype=torch.float) + self.chunk_size + + q_freqs = torch.outer(q_t, inv_freq) + qc_freqs = torch.outer(qc_t, inv_freq) + k_freqs = torch.outer(k_t, inv_freq) + qc_no_clamp_freqs = torch.outer(qc_no_clamp_t, inv_freq) + q_inter_freqs = torch.outer(q_inter_t, inv_freq) + + q_cos = q_freqs.cos() + q_sin = q_freqs.sin() + qc_cos = qc_freqs.cos() + qc_sin = qc_freqs.sin() + k_cos = k_freqs.cos() + k_sin = k_freqs.sin() + + qc_no_clamp_cos = qc_no_clamp_freqs.cos() + qc_no_clamp_sin = qc_no_clamp_freqs.sin() + q_inter_cos = q_inter_freqs.cos() + q_inter_sin = q_inter_freqs.sin() + + q_cache = torch.cat((q_cos, q_sin), dim=-1).to( + dtype=self.dtype, device=self.device + ) + qc_cache = torch.cat((qc_cos, qc_sin), dim=-1).to( + dtype=self.dtype, device=self.device + ) + k_cache = torch.cat((k_cos, k_sin), dim=-1).to( + dtype=self.dtype, device=self.device + ) + qc_no_clamp_cache = torch.cat((qc_no_clamp_cos, qc_no_clamp_sin), dim=-1).to( + dtype=self.dtype, device=self.device + ) + q_inter_cache = torch.cat((q_inter_cos, q_inter_sin), dim=-1).to( + dtype=self.dtype, device=self.device + ) + return q_cache, qc_cache, k_cache, qc_no_clamp_cache, q_inter_cache + + def forward( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + query = query.view(*query.shape[:-1], -1, self.head_size) + key = key.view(*key.shape[:-1], -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + key_rot = key[..., : self.rotary_dim] + if self.rotary_dim < self.head_size: + query_pass = query[..., self.rotary_dim :] + key_pass = key[..., self.rotary_dim :] + else: + query_pass = None + key_pass = None + + positions_with_offsets = ( + torch.add(positions, offsets) if offsets is not None else positions + ) + key = self._apply_rotary_embedding( + self.cos_sin_k_cache[positions_with_offsets], key_rot, key_pass + ) + chunk_len = self.chunk_size - self.local_size + query = self._apply_rotary_embedding( + self.cos_sin_q_cache[positions_with_offsets % chunk_len], + query_rot, + query_pass, + ) + query_succ = self._apply_rotary_embedding( + self.cos_sin_qc_cache[positions_with_offsets % chunk_len], + query_rot, + query_pass, + ) + query_inter = self._apply_rotary_embedding( + self.cos_sin_qc_cache[chunk_len - 1].repeat(positions.shape[0], 1), + query_rot, + query_pass, + ) + query_succ_critical = self._apply_rotary_embedding( + self.cos_sin_qc_no_clamp_cache[positions_with_offsets % chunk_len], + query_rot, + query_pass, + ) + query_inter_critical = self._apply_rotary_embedding( + self.cos_sin_q_inter_cache[positions_with_offsets % chunk_len], + query_rot, + query_pass, + ) + + # merge query into one tensor to simplify the interfaces + query = torch.cat( + ( + query, + query_succ, + query_inter, + query_succ_critical, + query_inter_critical, + ), + dim=-1, + ) + return query, key + + def _apply_rotary_embedding(self, cos_sin, hidden_rot, hidden_pass): + cos, sin = cos_sin.chunk(2, dim=-1) + if self.is_neox_style: + # NOTE(woosuk): Here we assume that the positions tensor has the + # shape [batch_size, seq_len]. + cos = cos.repeat(1, 1, 2).unsqueeze(-2) + sin = sin.repeat(1, 1, 2).unsqueeze(-2) + else: + cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2) + sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2) + rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj + hidden_rot = hidden_rot * cos + rotate_fn(hidden_rot) * sin + + if self.rotary_dim < self.head_size: + hidden = torch.cat((hidden_rot, hidden_pass), dim=-1) + else: + hidden = hidden_rot + return hidden.flatten(-2).squeeze(0) + + def extra_repr(self) -> str: + s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}" + s += f", max_position_embeddings={self.max_position_embeddings}" + s += f", base={self.base}, is_neox_style={self.is_neox_style}" + s += f", chunk_size={self.chunk_size}, local_size={self.local_size}" + return s + + +_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {} + + +def get_rope( + head_size: int, + rotary_dim: int, + max_position: int, + base: int, + is_neox_style: bool = True, + rope_scaling: Optional[Dict[str, Any]] = None, + dtype: Optional[torch.dtype] = None, + partial_rotary_factor: float = 1.0, + dual_chunk_attention_config: Optional[Dict[str, Any]] = None, +) -> RotaryEmbedding: + if dtype is None: + dtype = torch.get_default_dtype() + if rope_scaling is not None: + # Transforms every value that is a list into a tuple for caching calls + rope_scaling_tuple = { + k: tuple(v) if isinstance(v, list) else v for k, v in rope_scaling.items() + } + rope_scaling_args = tuple(rope_scaling_tuple.items()) + else: + rope_scaling_args = None + + if dual_chunk_attention_config is not None: + dual_chunk_attention_tuple = { + k: tuple(v) if isinstance(v, list) else v + for k, v in dual_chunk_attention_config.items() + if k != "sparse_attention_config" + } + dual_chunk_attention_args = tuple(dual_chunk_attention_tuple.items()) + else: + dual_chunk_attention_args = None + + if partial_rotary_factor < 1.0: + rotary_dim = int(rotary_dim * partial_rotary_factor) + key = ( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling_args, + dual_chunk_attention_args, + dtype, + ) + if key in _ROPE_DICT: + return _ROPE_DICT[key] + + if dual_chunk_attention_config is not None: + extra_kwargs = { + k: v + for k, v in dual_chunk_attention_config.items() + if k in ("chunk_size", "local_size") + } + rotary_emb = DualChunkRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + dtype, + **extra_kwargs, + ) + elif rope_scaling is None: + rotary_emb = RotaryEmbedding( + head_size, rotary_dim, max_position, base, is_neox_style, dtype + ) + else: + if "rope_type" in rope_scaling: + scaling_type = rope_scaling["rope_type"] + elif "type" in rope_scaling: + scaling_type = rope_scaling["type"] + else: + raise ValueError("Unknown RoPE scaling type") + + if scaling_type == "llama3": + scaling_factor = rope_scaling["factor"] + low_freq_factor = rope_scaling["low_freq_factor"] + high_freq_factor = rope_scaling["high_freq_factor"] + original_max_position = rope_scaling["original_max_position_embeddings"] + rotary_emb = Llama3RotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + dtype, + scaling_factor, + low_freq_factor, + high_freq_factor, + original_max_position, + ) + elif scaling_type == "default": + if "mrope_section" in rope_scaling: + rotary_emb = MRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + dtype, + mrope_section=rope_scaling["mrope_section"], + ) + else: + rotary_emb = RotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + dtype, + ) + elif scaling_type == "linear": + scaling_factor = rope_scaling["factor"] + rotary_emb = LinearScalingRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + scaling_factor, + dtype, + ) + elif scaling_type == "dynamic": + scaling_factor = rope_scaling["factor"] + if "alpha" in rope_scaling: + rotary_emb = DynamicNTKAlphaRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling["alpha"], + dtype, + ) + else: + rotary_emb = DynamicNTKScalingRotaryEmbedding( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + scaling_factor, + dtype, + ) + elif scaling_type == "yarn": + scaling_factor = rope_scaling["factor"] + original_max_position = rope_scaling["original_max_position_embeddings"] + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k + in ("extrapolation_factor", "attn_factor", "beta_fast", "beta_slow") + } + rotary_emb = YaRNScalingRotaryEmbedding( + head_size, + rotary_dim, + original_max_position, + base, + is_neox_style, + scaling_factor, + dtype, + **extra_kwargs, + ) + elif scaling_type == "deepseek_yarn": + scaling_factor = rope_scaling["factor"] + original_max_position = rope_scaling["original_max_position_embeddings"] + # assert max_position == original_max_position * scaling_factor + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k + in ( + "extrapolation_factor", + "attn_factor", + "beta_fast", + "beta_slow", + "mscale", + "mscale_all_dim", + ) + } + rotary_emb = DeepseekScalingRotaryEmbedding( + head_size, + rotary_dim, + original_max_position, + base, + is_neox_style, + scaling_factor, + dtype, + **extra_kwargs, + ) + elif scaling_type == "longrope": + short_factor = rope_scaling["short_factor"] + long_factor = rope_scaling["long_factor"] + original_max_position = rope_scaling["original_max_position_embeddings"] + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k in ("short_mscale", "long_mscale") + } + rotary_emb = Phi3LongRoPEScaledRotaryEmbedding( + head_size, + rotary_dim, + max_position, + original_max_position, + base, + is_neox_style, + dtype, + short_factor, + long_factor, + **extra_kwargs, + ) + else: + raise ValueError(f"Unknown RoPE scaling type {scaling_type}") + _ROPE_DICT[key] = rotary_emb + return rotary_emb + + +# Copied from transformers +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb_native( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim=1, +) -> Tuple[torch.Tensor, torch.Tensor]: + orig_q_dtype = q.dtype + orig_k_dtype = k.dtype + q, k = q.float(), k.float() + + # embedding is performed in float + cos = cos.unsqueeze(unsqueeze_dim).float() + sin = sin.unsqueeze(unsqueeze_dim).float() + q_embed = (q * cos) + (rotate_half(q) * sin) + k_embed = (k * cos) + (rotate_half(k) * sin) + + q_embed = q_embed.to(orig_q_dtype) + k_embed = k_embed.to(orig_k_dtype) + + return q_embed, k_embed + + +def apply_rotary_pos_emb_npu( + q: torch.Tensor, + k: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim=1, +) -> Tuple[torch.Tensor, torch.Tensor]: + if q.shape[1] != 128: + return apply_rotary_pos_emb_native(q, k, cos, sin, unsqueeze_dim) + cos = cos.unsqueeze(unsqueeze_dim) + cos = torch.transpose(cos, 1, 2) + sin = sin.unsqueeze(unsqueeze_dim) + sin = torch.transpose(sin, 1, 2) + q = torch.transpose(q, 1, 2) + k = torch.transpose(k, 1, 2) + q_embed, k_embed = torch_npu.npu_apply_rotary_pos_emb(q, k, cos, sin) + q_embed = torch.transpose(q_embed, 1, 2) + k_embed = torch.transpose(k_embed, 1, 2) + return q_embed, k_embed + + +if _is_npu: + apply_rotary_pos_emb = apply_rotary_pos_emb_npu +else: + apply_rotary_pos_emb = apply_rotary_pos_emb_native + + +def get_rope_cpu( + head_size: int, + rotary_dim: int, + max_position: int, + base: int, + is_neox_style: bool = True, + rope_scaling: Optional[Dict[str, Any]] = None, + dtype: Optional[torch.dtype] = None, + partial_rotary_factor: float = 1.0, + device: Optional[str] = None, +) -> RotaryEmbedding: + if dtype is None: + dtype = torch.get_default_dtype() + if rope_scaling is not None: + # Transforms every value that is a list into a tuple for caching calls + rope_scaling_tuple = { + k: tuple(v) if isinstance(v, list) else v for k, v in rope_scaling.items() + } + rope_scaling_args = tuple(rope_scaling_tuple.items()) + else: + rope_scaling_args = None + if partial_rotary_factor < 1.0: + rotary_dim = int(rotary_dim * partial_rotary_factor) + key = ( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling_args, + dtype, + ) + if key in _ROPE_DICT: + return _ROPE_DICT[key] + + assert rope_scaling is not None + scaling_type = rope_scaling["rope_type"] + assert ( + scaling_type == "deepseek_yarn" + ), "Only deepseek_yarn is supported for CPU for now" + + scaling_factor = rope_scaling["factor"] + original_max_position = rope_scaling["original_max_position_embeddings"] + extra_kwargs = { + k: v + for k, v in rope_scaling.items() + if k + in ( + "extrapolation_factor", + "attn_factor", + "beta_fast", + "beta_slow", + "mscale", + "mscale_all_dim", + ) + } + extra_kwargs["device"] = device + rotary_emb = DeepseekScalingRotaryEmbedding( + head_size, + rotary_dim, + original_max_position, + base, + is_neox_style, + scaling_factor, + dtype, + **extra_kwargs, + ) + + _ROPE_DICT[key] = rotary_emb + return rotary_emb + + +def get_rope_wrapper( + head_size: int, + rotary_dim: int, + max_position: int, + base: int, + is_neox_style: bool = True, + rope_scaling: Optional[Dict[str, Any]] = None, + dtype: Optional[torch.dtype] = None, + partial_rotary_factor: float = 1.0, + device: Optional[str] = None, +): + if device != "cpu": + wrapper = aiter_get_rope if _use_aiter else get_rope + return wrapper( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling, + dtype, + partial_rotary_factor, + ) + + return get_rope_cpu( + head_size, + rotary_dim, + max_position, + base, + is_neox_style, + rope_scaling, + dtype, + partial_rotary_factor, + device, + ) diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py new file mode 100644 index 000000000..56a831f2d --- /dev/null +++ b/python/sglang/srt/layers/sampler.py @@ -0,0 +1,294 @@ +import logging +from typing import List + +import torch +import torch.distributed as dist +from torch import nn + +from sglang.srt.distributed import get_tp_group +from sglang.srt.layers.dp_attention import ( + get_attention_tp_group, + is_dp_attention_enabled, +) +from sglang.srt.layers.logits_processor import LogitsProcessorOutput +from sglang.srt.managers.schedule_batch import global_server_args_dict +from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo +from sglang.srt.utils import crash_on_warnings, get_bool_env_var, is_cuda + +if is_cuda(): + from sgl_kernel import ( + min_p_sampling_from_probs, + top_k_renorm_prob, + top_k_top_p_sampling_from_probs, + top_p_renorm_prob, + ) + + +logger = logging.getLogger(__name__) + +SYNC_TOKEN_IDS_ACROSS_TP = get_bool_env_var("SYNC_TOKEN_IDS_ACROSS_TP") +RETURN_ORIGINAL_LOGPROB = get_bool_env_var("RETURN_ORIGINAL_LOGPROB") + + +class Sampler(nn.Module): + def __init__(self): + super().__init__() + self.use_nan_detection = global_server_args_dict["enable_nan_detection"] + self.tp_sync_group = get_tp_group().device_group + + if is_dp_attention_enabled(): + self.tp_sync_group = get_attention_tp_group().device_group + + def forward( + self, + logits_output: LogitsProcessorOutput, + sampling_info: SamplingBatchInfo, + return_logprob: bool, + top_logprobs_nums: List[int], + token_ids_logprobs: List[List[int]], + ): + """Run a sampler & compute logprobs and update logits_output accordingly. + + Args: + logits_output: The logits from the model forward + sampling_info: Metadata for sampling + return_logprob: If set, store the output logprob information to + logits_output + top_logprobs_nums: Number of top lobprobs per sequence in a batch + batch_next_token_ids: next token IDs. If set, skip sampling and only + compute output logprobs It is used for speculative decoding which + performs sampling in draft workers. + """ + logits = logits_output.next_token_logits + + # Apply the custom logit processors if registered in the sampling info. + if sampling_info.has_custom_logit_processor: + apply_custom_logit_processor(logits, sampling_info) + + if self.use_nan_detection and torch.any(torch.isnan(logits)): + logger.warning("Detected errors during sampling! NaN in the logits.") + logits = torch.where( + torch.isnan(logits), torch.full_like(logits, -1e5), logits + ) + if crash_on_warnings(): + raise ValueError("Detected errors during sampling! NaN in the logits.") + + if sampling_info.is_all_greedy: + # Use torch.argmax if all requests use greedy sampling + batch_next_token_ids = torch.argmax(logits, -1) + if return_logprob: + logprobs = torch.nn.functional.log_softmax(logits, dim=-1) + + else: + # Post process original logits. if temperatures are all 1.0, no need to rescale + if return_logprob and RETURN_ORIGINAL_LOGPROB: + logprobs = torch.softmax(logits, dim=-1) + + # Post process logits + logits.div_(sampling_info.temperatures) + logits[:] = torch.softmax(logits, dim=-1) + probs = logits + del logits + + if True: # Keep this redundant check to simplify some internal code sync + if global_server_args_dict["sampling_backend"] == "flashinfer": + if sampling_info.need_min_p_sampling: + probs = top_k_renorm_prob(probs, sampling_info.top_ks) + probs = top_p_renorm_prob(probs, sampling_info.top_ps) + batch_next_token_ids = min_p_sampling_from_probs( + probs, sampling_info.min_ps + ) + else: + batch_next_token_ids = top_k_top_p_sampling_from_probs( + probs.contiguous(), + sampling_info.top_ks, + sampling_info.top_ps, + filter_apply_order="joint", + check_nan=self.use_nan_detection, + ) + elif global_server_args_dict["sampling_backend"] == "pytorch": + # A slower fallback implementation with torch native operations. + batch_next_token_ids = top_k_top_p_min_p_sampling_from_probs_torch( + probs, + sampling_info.top_ks, + sampling_info.top_ps, + sampling_info.min_ps, + sampling_info.need_min_p_sampling, + ) + else: + raise ValueError( + f"Invalid sampling backend: {global_server_args_dict['sampling_backend']}" + ) + + if return_logprob: + # clamp to avoid -inf + if RETURN_ORIGINAL_LOGPROB: + logprobs = torch.log(logprobs).clamp( + min=torch.finfo(logprobs.dtype).min + ) + else: + logprobs = torch.log(probs).clamp(min=torch.finfo(probs.dtype).min) + + # Attach logprobs to logits_output (in-place modification) + if return_logprob: + if any(x > 0 for x in top_logprobs_nums): + ( + logits_output.next_token_top_logprobs_val, + logits_output.next_token_top_logprobs_idx, + ) = get_top_logprobs(logprobs, top_logprobs_nums) + + if any(x is not None for x in token_ids_logprobs): + ( + logits_output.next_token_token_ids_logprobs_val, + logits_output.next_token_token_ids_logprobs_idx, + ) = get_token_ids_logprobs(logprobs, token_ids_logprobs) + + logits_output.next_token_logprobs = logprobs[ + torch.arange(len(batch_next_token_ids), device=sampling_info.device), + batch_next_token_ids, + ] + + if SYNC_TOKEN_IDS_ACROSS_TP or sampling_info.grammars: + # For performance reasons, SGLang does not sync the final token IDs across TP ranks by default. + # This saves one all-reduce, but the correctness of this approach depends on the determinism of several operators: + # the last all-reduce, the last lm_head matmul, and all sampling kernels. + # These kernels are deterministic in most cases, but there are some rare instances where they are not deterministic. + # In such cases, enable this env variable to prevent hanging due to TP ranks becoming desynchronized. + # When using xgrammar, this becomes more likely so we also do the sync when grammar is used. + + torch.distributed.all_reduce( + batch_next_token_ids, + op=dist.ReduceOp.MIN, + group=self.tp_sync_group, + ) + + return batch_next_token_ids + + +def top_k_top_p_min_p_sampling_from_probs_torch( + probs: torch.Tensor, + top_ks: torch.Tensor, + top_ps: torch.Tensor, + min_ps: torch.Tensor, + need_min_p_sampling: bool, +): + """A top-k, top-p and min-p sampling implementation with native pytorch operations.""" + probs_sort, probs_idx = probs.sort(dim=-1, descending=True) + probs_sum = torch.cumsum(probs_sort, dim=-1) + probs_sort[ + torch.arange(0, probs.shape[-1], device=probs.device).view(1, -1) + >= top_ks.view(-1, 1) + ] = 0.0 + probs_sort[(probs_sum - probs_sort) > top_ps.view(-1, 1)] = 0.0 + + if need_min_p_sampling: + min_p_thresholds = probs_sort[:, 0] * min_ps + probs_sort[probs_sort < min_p_thresholds.view(-1, 1)] = 0.0 + + sampled_index = torch.multinomial(probs_sort, num_samples=1) + # int32 range is enough to represent the token ids + probs_idx = probs_idx.to(torch.int32) + batch_next_token_ids = torch.gather(probs_idx, dim=1, index=sampled_index).view(-1) + return batch_next_token_ids + + +def sampling_from_probs_torch(probs: torch.Tensor): + """A sampling implementation with native pytorch operations, without + top-k, top-p, or min-p filtering.""" + sampled_index = torch.multinomial(probs, num_samples=1) + batch_next_token_ids = sampled_index.view(-1).to(torch.int32) + return batch_next_token_ids + + +def top_p_normalize_probs_torch( + probs: torch.Tensor, + top_ps: torch.Tensor, +): + # See also top_k_top_p_min_p_sampling_from_probs_torch + probs_sort, probs_idx = probs.sort(dim=-1, descending=True) + probs_sum = torch.cumsum(probs_sort, dim=-1) + probs_sort[(probs_sum - probs_sort) > top_ps.view(-1, 1)] = 0.0 + probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True)) + return torch.zeros_like(probs_sort).scatter_(-1, probs_idx, probs_sort) + + +def get_top_logprobs( + logprobs: torch.Tensor, + top_logprobs_nums: List[int], +): + max_k = max(top_logprobs_nums) + ret = logprobs.topk(max_k, dim=1) + values = ret.values.tolist() + indices = ret.indices.tolist() + + output_top_logprobs_val = [] + output_top_logprobs_idx = [] + for i, k in enumerate(top_logprobs_nums): + output_top_logprobs_val.append(values[i][:k]) + output_top_logprobs_idx.append(indices[i][:k]) + + return ( + output_top_logprobs_val, + output_top_logprobs_idx, + ) + + +def get_token_ids_logprobs( + logprobs: torch.Tensor, + token_ids_logprobs: List[List[int]], +): + output_token_ids_logprobs_val = [] + output_token_ids_logprobs_idx = [] + for i, token_ids in enumerate(token_ids_logprobs): + if token_ids is not None: + output_token_ids_logprobs_val.append(logprobs[i, token_ids].tolist()) + output_token_ids_logprobs_idx.append(token_ids) + else: + output_token_ids_logprobs_val.append([]) + output_token_ids_logprobs_idx.append([]) + + return ( + output_token_ids_logprobs_val, + output_token_ids_logprobs_idx, + ) + + +def apply_custom_logit_processor( + logits: torch.Tensor, + sampling_batch_info: SamplingBatchInfo, + num_tokens_in_batch: int = 1, +): + """Apply custom logit processors to the logits. + This function will modify the logits in-place. + num_tokens_in_batch is needed to support spec decoding, where each batch can contain multiple + tokens. By default, we assume each batch contains only 1 token. + """ + + assert logits.shape[0] == len(sampling_batch_info) * num_tokens_in_batch, ( + f"The batch size of logits ({logits.shape[0]}) does not match the batch size of " + f"sampling_batch_info ({len(sampling_batch_info)}) x num_tokens_in_batch " + f"({num_tokens_in_batch})" + ) + + for _, ( + processor, + batch_mask, + ) in sampling_batch_info.custom_logit_processor.items(): + # Get the batch indices that need to be processed + batch_indices = batch_mask.nonzero(as_tuple=True)[0] + + assert batch_mask.shape[0] == len(sampling_batch_info), ( + f"The number of batch mask ({batch_mask.shape[0]}) does not match the number of " + f"sampling_batch_info ({len(sampling_batch_info)})" + ) + batch_mask = torch.repeat_interleave(batch_mask, num_tokens_in_batch) + + # Apply the processor to the logits + logits[batch_mask] = processor( + logits[batch_mask], + [sampling_batch_info.custom_params[i] for i in batch_indices], + ) + + logger.debug( + f"Custom logit processor {processor.__class__.__name__} is applied." + ) diff --git a/python/sglang/srt/layers/torchao_utils.py b/python/sglang/srt/layers/torchao_utils.py new file mode 100644 index 000000000..e08abd5ae --- /dev/null +++ b/python/sglang/srt/layers/torchao_utils.py @@ -0,0 +1,124 @@ +""" +Common utilities for torchao. +""" + +import logging +import os +import pwd +from typing import Callable, Optional + +import torch + +logger = logging.getLogger(__name__) + + +def get_gemlite_cache_path() -> str: + return f"/tmp/{pwd.getpwuid(os.getuid()).pw_gecos}_gemlite.json" + + +def save_gemlite_cache(print_error: bool = False) -> bool: + try: + from gemlite.core import GemLiteLinearTriton + + GemLiteLinearTriton.cache_config(get_gemlite_cache_path()) + except Exception: + if print_error: + logger.error("Failed to save the GemLite cache.") + return False + return True + + +def proj_filter( + module: torch.nn.Module, + fqn: str, +): + """Filter function for quantizing projection layers.""" + return "proj" in fqn + + +def apply_torchao_config_to_model( + model: torch.nn.Module, + torchao_config: str, + filter_fn: Optional[Callable] = proj_filter, +): + """Quantize a modelwith torchao quantization specified by torchao_config + + Args: + `model`: a model to be quantized based on torchao_config + `torchao_config` (str): type of quantization and their arguments we want to use to + quantize the model, e.g. int4wo-128 means int4 weight only quantization with group_size + 128 + """ + # Lazy import to suppress some warnings + from torchao.quantization import ( + float8_dynamic_activation_float8_weight, + float8_weight_only, + int4_weight_only, + int8_dynamic_activation_int8_weight, + int8_weight_only, + quantize_, + ) + from torchao.quantization.observer import PerRow, PerTensor + + if torchao_config == "" or torchao_config is None: + return model + elif "int8wo" in torchao_config: + quantize_(model, int8_weight_only(), filter_fn=filter_fn) + elif "int8dq" in torchao_config: + quantize_(model, int8_dynamic_activation_int8_weight(), filter_fn=filter_fn) + elif "int4wo" in torchao_config: + group_size = int(torchao_config.split("-")[-1]) + assert group_size in [ + 32, + 64, + 128, + 256, + ], f"int4wo groupsize needs to be one of [32, 64, 128, 256] but got {group_size}" + quantize_(model, int4_weight_only(group_size=group_size), filter_fn=filter_fn) + elif "gemlite" in torchao_config: + # gemlite--- or + # gemlite-- (packing_bitwidth defaults to 32) + from gemlite.core import GemLiteLinearTriton + from torchao.quantization import gemlite_uintx_weight_only + + _quant_args = torchao_config.split("-") + bit_width = int(_quant_args[-2]) + group_size = None if _quant_args[-1] == "None" else int(_quant_args[-1]) + + try: + packing_bitwidth = int(_quant_args[-3]) + except (ValueError, IndexError): + # if only 2 inputs found or conversion fails, use default value + packing_bitwidth = 32 + + quantize_( + model, gemlite_uintx_weight_only(group_size, bit_width, packing_bitwidth) + ) + + # try to load gemlite kernel config + GemLiteLinearTriton.load_config(get_gemlite_cache_path()) + + elif "fp8wo" in torchao_config: + # this requires newer hardware + # [rank0]: AssertionError: fp8e4nv data type is not supported on CUDA arch < 89 + quantize_(model, float8_weight_only(), filter_fn=filter_fn) + elif "fp8dq" in torchao_config: + granularity = torchao_config.split("-")[-1] + GRANULARITY_MAP = { + "per_row": PerRow(), + "per_tensor": PerTensor(), + } + assert ( + granularity in GRANULARITY_MAP + ), f"Supported granularity are: {GRANULARITY_MAP.keys()}, got {granularity}" + quantize_( + model, + float8_dynamic_activation_float8_weight( + granularity=GRANULARITY_MAP[granularity] + ), + filter_fn=filter_fn, + ) + else: + raise ValueError(f"Unexpected config: {torchao_config}") + + return model diff --git a/python/sglang/srt/layers/utils.py b/python/sglang/srt/layers/utils.py new file mode 100644 index 000000000..d79ccc663 --- /dev/null +++ b/python/sglang/srt/layers/utils.py @@ -0,0 +1,36 @@ +import logging +import re +from functools import lru_cache + +import torch + +logger = logging.getLogger(__name__) + + +def get_layer_id(weight_name): + # example weight name: model.layers.10.self_attn.qkv_proj.weight + match = re.search(r"layers\.(\d+)\.", weight_name) + if match: + return int(match.group(1)) + return None + + +class PPMissingLayer(torch.nn.Identity): + # Adapted from + # https://github.com/vllm-project/vllm/blob/18ed3132d2bfe1df9a74729457b69243955221e8/vllm/model_executor/models/utils.py#L468C1-L486C1 + """ + A placeholder layer for missing layers in a pipeline parallel model. + """ + + def __init__(self, *args, **kwargs): + super().__init__() + self.return_tuple = kwargs.get("return_tuple", False) + + def forward(self, *args, **kwargs): + """ + Return the first arg from args or the first value from kwargs. + + Wraps the input in a tuple if `self.return_tuple` is True. + """ + input = args[0] if args else next(iter(kwargs.values())) + return (input,) if self.return_tuple else input diff --git a/python/sglang/srt/layers/vocab_parallel_embedding.py b/python/sglang/srt/layers/vocab_parallel_embedding.py new file mode 100644 index 000000000..66abb7541 --- /dev/null +++ b/python/sglang/srt/layers/vocab_parallel_embedding.py @@ -0,0 +1,571 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/v0.6.3.post1/vllm/model_executor/layers/vocab_parallel_embedding.py + +import logging +from dataclasses import dataclass +from typing import List, Optional, Sequence, Tuple + +import torch +from torch.nn.parameter import Parameter, UninitializedParameter + +from sglang.srt.distributed import ( + divide, + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, + parallel_state, + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) +from sglang.srt.layers.amx_utils import PackWeightMethod +from sglang.srt.layers.dp_attention import get_attention_tp_rank, get_attention_tp_size +from sglang.srt.layers.parameter import BasevLLMParameter +from sglang.srt.layers.quantization.base_config import ( + QuantizationConfig, + QuantizeMethodBase, + method_has_implemented_embedding, +) +from sglang.srt.layers.quantization.unquant import UnquantizedEmbeddingMethod +from sglang.srt.utils import ( + cpu_has_amx_support, + get_compiler_backend, + is_cpu, + set_weight_attrs, +) + +DEFAULT_VOCAB_PADDING_SIZE = 64 + +_is_cpu_amx_available = cpu_has_amx_support() +_is_cpu = is_cpu() + +logger = logging.getLogger(__name__) + + +def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int: + """Pad the vocab size to the given value.""" + return ((vocab_size + pad_to - 1) // pad_to) * pad_to + + +def vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size: int, rank: int, offset: int = 0 +) -> Sequence[int]: + index_f = rank * per_partition_vocab_size + index_l = index_f + per_partition_vocab_size + return index_f + offset, index_l + offset + + +def vocab_range_from_global_vocab_size( + global_vocab_size: int, rank: int, world_size: int, offset: int = 0 +) -> Sequence[int]: + per_partition_vocab_size = divide(global_vocab_size, world_size) + return vocab_range_from_per_partition_vocab_size( + per_partition_vocab_size, rank, offset=offset + ) + + +@dataclass +class VocabParallelEmbeddingShardIndices: + """Indices for a shard of a vocab parallel embedding.""" + + padded_org_vocab_start_index: int + padded_org_vocab_end_index: int + padded_added_vocab_start_index: int + padded_added_vocab_end_index: int + + org_vocab_start_index: int + org_vocab_end_index: int + added_vocab_start_index: int + added_vocab_end_index: int + + @property + def num_org_elements(self) -> int: + return self.org_vocab_end_index - self.org_vocab_start_index + + @property + def num_added_elements(self) -> int: + return self.added_vocab_end_index - self.added_vocab_start_index + + @property + def num_org_elements_padded(self) -> int: + return self.padded_org_vocab_end_index - self.padded_org_vocab_start_index + + @property + def num_added_elements_padded(self) -> int: + return self.padded_added_vocab_end_index - self.padded_added_vocab_start_index + + @property + def num_org_vocab_padding(self) -> int: + return self.num_org_elements_padded - self.num_org_elements + + @property + def num_added_vocab_padding(self) -> int: + return self.num_added_elements_padded - self.num_added_elements + + @property + def num_elements_padded(self) -> int: + return self.num_org_elements_padded + self.num_added_elements_padded + + def __post_init__(self): + # sanity checks + assert self.padded_org_vocab_start_index <= self.padded_org_vocab_end_index + assert self.padded_added_vocab_start_index <= self.padded_added_vocab_end_index + + assert self.org_vocab_start_index <= self.org_vocab_end_index + assert self.added_vocab_start_index <= self.added_vocab_end_index + + assert self.org_vocab_start_index <= self.padded_org_vocab_start_index + assert self.added_vocab_start_index <= self.padded_added_vocab_start_index + assert self.org_vocab_end_index <= self.padded_org_vocab_end_index + assert self.added_vocab_end_index <= self.padded_added_vocab_end_index + + assert self.num_org_elements <= self.num_org_elements_padded + assert self.num_added_elements <= self.num_added_elements_padded + + +@torch.compile(dynamic=True, backend=get_compiler_backend()) +def get_masked_input_and_mask( + input_: torch.Tensor, + org_vocab_start_index: int, + org_vocab_end_index: int, + num_org_vocab_padding: int, + added_vocab_start_index: int, + added_vocab_end_index: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + # torch.compile will fuse all of the pointwise ops below + # into a single kernel, making it very fast + org_vocab_mask = (input_ >= org_vocab_start_index) & (input_ < org_vocab_end_index) + added_vocab_mask = (input_ >= added_vocab_start_index) & ( + input_ < added_vocab_end_index + ) + added_offset = ( + added_vocab_start_index + - (org_vocab_end_index - org_vocab_start_index) + - num_org_vocab_padding + ) + valid_offset = (org_vocab_start_index * org_vocab_mask) + ( + added_offset * added_vocab_mask + ) + vocab_mask = org_vocab_mask | added_vocab_mask + input_ = vocab_mask * (input_ - valid_offset) + return input_, ~vocab_mask + + +class VocabParallelEmbedding(torch.nn.Module): + """Embedding parallelized in the vocabulary dimension. + + Adapted from torch.nn.Embedding, note that we pad the vocabulary size to + make sure it is divisible by the number of model parallel GPUs. + + In order to support various loading methods, we ensure that LoRA-added + embeddings are always at the end of TP-sharded tensors. In other words, + we shard base embeddings and LoRA embeddings separately (both padded), + and place them in the same tensor. + In this example, we will have the original vocab size = 1010, + added vocab size = 16 and padding to 64. Therefore, the total + vocab size with padding will be 1088 (because we first pad 1010 to + 1024, add 16, and then pad to 1088). + Therefore, the tensor format looks like the following: + TP1, rank 0 (no sharding): + |< --------BASE-------- >|< -BASE PADDING-- >|< -----LORA------ >|< -LORA PADDING-- >| + corresponding token_id: | 0 | 1 | ... | 1009 | -1 | ... | -1 | 1010 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | ... | 1009 | 1010 | ... | 1023 | 1024 | ... | 1039 | 1040 | ... | 1087 | + + TP2, rank 0: + |< --------------------BASE--------------------- >|< -----LORA------ >|< -LORA PADDING- >| + corresponding token_id: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 1000 | ... | 1015 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 527 | 520 | ... | 543 | + TP2, rank 1: + |< -----------BASE----------- >|< -BASE PADDING- >|< -----------LORA PADDING----------- >| + corresponding token_id: | 512 | 513 | 514 | ... | 1009 | -1 | ... | -1 | -1 | ... | -1 | -1 | ... | -1 | + index: | 0 | 1 | 2 | ... | 497 | 498 | ... | 511 | 512 | ... | 519 | 520 | ... | 543 | + + Args: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + params_dtype: type of the parameters. + org_num_embeddings: original vocabulary size (without LoRA). + padding_size: padding size for the vocabulary. + quant_config: quant config for the layer + prefix: full name of the layer in the state dict + """ # noqa: E501 + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + params_dtype: Optional[torch.dtype] = None, + org_num_embeddings: Optional[int] = None, + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + enable_tp: bool = True, + use_attn_tp_group: bool = False, + use_presharded_weights: bool = False, + ): + super().__init__() + self.quant_config = quant_config + + self.enable_tp = enable_tp + if self.enable_tp: + if use_attn_tp_group: + tp_rank = get_attention_tp_rank() + self.tp_size = get_attention_tp_size() + else: + tp_rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + else: + assert use_attn_tp_group is False + tp_rank = 0 + self.tp_size = 1 + + self.num_embeddings = num_embeddings + self.org_vocab_size = org_num_embeddings or num_embeddings + + # Support the case where the vocab size is not divisible by the TP size. + if ( + _is_cpu + and pad_vocab_size(self.org_vocab_size, padding_size) % self.tp_size != 0 + ): + padding_size *= self.tp_size + self.padding_size = padding_size + + num_added_embeddings = num_embeddings - self.org_vocab_size + self.use_presharded_weights = use_presharded_weights + if use_presharded_weights: + assert ( + num_added_embeddings == 0 + ), "Lora is not supported with presharded weights." + + self.org_vocab_size_padded = pad_vocab_size( + self.org_vocab_size, self.padding_size + ) + self.num_embeddings_padded = pad_vocab_size( + self.org_vocab_size_padded + num_added_embeddings, self.padding_size + ) + assert self.org_vocab_size_padded <= self.num_embeddings_padded + + self.shard_indices = self._get_indices( + self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, + tp_rank, + self.tp_size, + ) + self.embedding_dim = embedding_dim + + quant_method = None + if quant_config is not None: + quant_method = quant_config.get_quant_method(self, prefix=prefix) + if quant_method is None: + quant_method = UnquantizedEmbeddingMethod() + + # If we are making an embedding layer, then our quantization linear + # method must implement the embedding operation. If we are another + # layer type like ParallelLMHead, this is not important. + is_embedding_layer = type(self.__class__) is VocabParallelEmbedding + quant_method_implements_embedding = method_has_implemented_embedding( + type(quant_method) + ) + if is_embedding_layer and not quant_method_implements_embedding: + raise NotImplementedError( + f"The class {type(quant_method).__name__} must implement " + "the 'embedding' method, see UnquantizedEmbeddingMethod." + ) + + self.quant_method: QuantizeMethodBase = quant_method + + if params_dtype is None: + params_dtype = torch.get_default_dtype() + # Divide the weight matrix along the vocaburaly dimension. + self.num_added_embeddings = self.num_embeddings - self.org_vocab_size + self.num_embeddings_per_partition = divide( + self.num_embeddings_padded, self.tp_size + ) + assert ( + self.shard_indices.num_elements_padded == self.num_embeddings_per_partition + ) + self.num_org_embeddings_per_partition = ( + self.shard_indices.org_vocab_end_index + - self.shard_indices.org_vocab_start_index + ) + self.num_added_embeddings_per_partition = ( + self.shard_indices.added_vocab_end_index + - self.shard_indices.added_vocab_start_index + ) + + self.quant_method.create_weights( + self, + self.embedding_dim, + [self.num_embeddings_per_partition], + self.embedding_dim, + self.num_embeddings_padded, + params_dtype=params_dtype, + weight_loader=self.weight_loader, + ) + + @classmethod + def _get_indices( + cls, + vocab_size_padded: int, + org_vocab_size_padded: int, + vocab_size: int, + org_vocab_size: int, + tp_rank: int, + tp_size: int, + ) -> VocabParallelEmbeddingShardIndices: + """Get start and end indices for vocab parallel embedding, following the + layout outlined in the class docstring, based on the given tp_rank and + tp_size.""" + num_added_embeddings_padded = vocab_size_padded - org_vocab_size_padded + padded_org_vocab_start_index, padded_org_vocab_end_index = ( + vocab_range_from_global_vocab_size(org_vocab_size_padded, tp_rank, tp_size) + ) + padded_added_vocab_start_index, padded_added_vocab_end_index = ( + vocab_range_from_global_vocab_size( + num_added_embeddings_padded, tp_rank, tp_size, offset=org_vocab_size + ) + ) + # remove padding + org_vocab_start_index = min(padded_org_vocab_start_index, org_vocab_size) + org_vocab_end_index = min(padded_org_vocab_end_index, org_vocab_size) + added_vocab_start_index = min(padded_added_vocab_start_index, vocab_size) + added_vocab_end_index = min(padded_added_vocab_end_index, vocab_size) + return VocabParallelEmbeddingShardIndices( + padded_org_vocab_start_index, + padded_org_vocab_end_index, + padded_added_vocab_start_index, + padded_added_vocab_end_index, + org_vocab_start_index, + org_vocab_end_index, + added_vocab_start_index, + added_vocab_end_index, + ) + + def get_sharded_to_full_mapping(self) -> Optional[List[int]]: + """Get a mapping that can be used to reindex the gathered + logits for sampling. + + During sampling, we gather logits from all ranks. The relationship + of index->token_id will follow the same format as outlined in the class + docstring. However, after the gather, we want to reindex the final + logits tensor to map index->token_id one-to-one (the index is always + equal the token_id it corresponds to). The indices returned by this + method allow us to do that. + """ + if self.tp_size < 2: + return None + + base_embeddings: List[int] = [] + added_embeddings: List[int] = [] + padding: List[int] = [] + for tp_rank in range(self.tp_size): + shard_indices = self._get_indices( + self.num_embeddings_padded, + self.org_vocab_size_padded, + self.num_embeddings, + self.org_vocab_size, + tp_rank, + self.tp_size, + ) + range_start = self.num_embeddings_per_partition * tp_rank + range_end = self.num_embeddings_per_partition * (tp_rank + 1) + base_embeddings.extend( + range(range_start, range_start + shard_indices.num_org_elements) + ) + padding.extend( + range( + range_start + shard_indices.num_org_elements, + range_start + shard_indices.num_org_elements_padded, + ) + ) + added_embeddings.extend( + range( + range_start + shard_indices.num_org_elements_padded, + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements, + ) + ) + padding.extend( + range( + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements, + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded, + ) + ) + assert ( + range_start + + shard_indices.num_org_elements_padded + + shard_indices.num_added_elements_padded + == range_end + ) + ret = base_embeddings + added_embeddings + padding + assert len(ret) == self.num_embeddings_padded + return ret + + def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor): + output_dim = getattr(param, "output_dim", None) + packed_dim = getattr(param, "packed_dim", None) + + # If the parameter is a gguf weight, then load it directly. + if getattr(param, "is_gguf_weight_type", None): + param.data.copy_(loaded_weight) + param.weight_type = loaded_weight.item() + return + elif isinstance(param, UninitializedParameter): + shape = list(loaded_weight.shape) + if output_dim is not None: + shape[output_dim] = shape[output_dim] // self.tp_size + param.materialize(tuple(shape), dtype=loaded_weight.dtype) + + # If parameter does not have output dim, then it should + # be copied onto all gpus (e.g. g_idx for act_order gptq). + if output_dim is None: + assert param.data.shape == loaded_weight.shape + param.data.copy_(loaded_weight) + return + + # Shard indexes for loading the weight + start_idx = self.shard_indices.org_vocab_start_index + shard_size = self.shard_indices.org_vocab_end_index - start_idx + + # If param packed on the same dim we are sharding on, then + # need to adjust offsets of loaded weight by pack_factor. + if packed_dim is not None and packed_dim == output_dim: + packed_factor = ( + param.packed_factor + if isinstance(param, BasevLLMParameter) + else param.packed_factor + ) + assert loaded_weight.shape[output_dim] == ( + self.org_vocab_size // param.packed_factor + ) + start_idx = start_idx // packed_factor + shard_size = shard_size // packed_factor + else: + assert loaded_weight.shape[output_dim] == ( + self.org_vocab_size + // (self.tp_size if self.use_presharded_weights else 1) + ), f"{self.org_vocab_size=} {self.use_presharded_weights=} {loaded_weight.shape[output_dim]=}" + + # Copy the data. + if not self.use_presharded_weights: + loaded_weight = loaded_weight.narrow(output_dim, start_idx, shard_size) + param[: loaded_weight.shape[0]].data.copy_(loaded_weight) + param[loaded_weight.shape[0] :].data.fill_(0) + + def forward(self, input_): + if self.tp_size > 1: + # Build the mask. + masked_input, input_mask = get_masked_input_and_mask( + input_, + self.shard_indices.org_vocab_start_index, + self.shard_indices.org_vocab_end_index, + self.shard_indices.num_org_vocab_padding, + self.shard_indices.added_vocab_start_index, + self.shard_indices.added_vocab_end_index, + ) + else: + masked_input = input_ + # Get the embeddings. + with use_symmetric_memory(parallel_state.get_tp_group()) as sm: + output_parallel = self.quant_method.embedding(self, masked_input.long()) + sm.tag(output_parallel) + # Mask the output embedding. + if self.tp_size > 1: + output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0) + # Reduce across all the model parallel GPUs. + output = tensor_model_parallel_all_reduce(output_parallel) + else: + output = output_parallel + return output + + def extra_repr(self) -> str: + s = f"num_embeddings={self.num_embeddings_per_partition}" + s += f", embedding_dim={self.embedding_dim}" + s += f", org_vocab_size={self.org_vocab_size}" + s += f", num_embeddings_padded={self.num_embeddings_padded}" + if self.enable_tp: + s += f", tp_size={self.tp_size}" + return s + + +class ParallelLMHead(VocabParallelEmbedding): + """Parallelized LM head. + + Output logits weight matrices used in the Sampler. The weight and bias + tensors are padded to make sure they are divisible by the number of + model parallel GPUs. + + Args: + num_embeddings: vocabulary size. + embedding_dim: size of hidden state. + bias: whether to use bias. + params_dtype: type of the parameters. + org_num_embeddings: original vocabulary size (without LoRA). + padding_size: padding size for the vocabulary. + """ + + def __init__( + self, + num_embeddings: int, + embedding_dim: int, + *, + bias: bool = False, + params_dtype: Optional[torch.dtype] = None, + org_num_embeddings: Optional[int] = None, + padding_size: int = DEFAULT_VOCAB_PADDING_SIZE, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + use_attn_tp_group: bool = False, + use_presharded_weights: bool = False, + ): + super().__init__( + num_embeddings, + embedding_dim, + params_dtype=params_dtype, + org_num_embeddings=org_num_embeddings, + padding_size=padding_size, + quant_config=quant_config, + prefix=prefix, + use_attn_tp_group=use_attn_tp_group, + use_presharded_weights=use_presharded_weights, + ) + self.quant_config = quant_config + + # We only support pack LMHead if it's not quantized. + if _is_cpu and _is_cpu_amx_available: + if hasattr(self, "weight") and self.weight.dtype == torch.bfloat16: + self.quant_method = PackWeightMethod(weight_names=["weight"]) + + if bias: + self.bias = Parameter( + torch.empty(self.num_embeddings_per_partition, dtype=params_dtype) + ) + set_weight_attrs( + self.bias, + { + "output_dim": 0, + "weight_loader": self.weight_loader, + }, + ) + else: + self.register_parameter("bias", None) + + def tie_weights(self, embed_tokens: VocabParallelEmbedding): + """Tie the weights with word embeddings.""" + # GGUF quantized embed_tokens. + if self.quant_config and self.quant_config.get_name() == "gguf": + return embed_tokens + else: + self.weight = embed_tokens.weight + return self + + def forward(self, input_): + del input_ + raise RuntimeError("LMHead's weights should be used in the sampler.") diff --git a/python/sglang/srt/lora/backend/base_backend.py b/python/sglang/srt/lora/backend/base_backend.py new file mode 100644 index 000000000..7c2c232d5 --- /dev/null +++ b/python/sglang/srt/lora/backend/base_backend.py @@ -0,0 +1,155 @@ +from typing import Optional, Tuple, Union + +import torch + +from sglang.srt.lora.utils import LoRABatchInfo +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +class BaseLoRABackend: + """Base class for different Lora backends. + Each backend has its own implementation of Lora kernels. + + Args: + max_loras_per_batch: maximum number of different lora weights + that can be applied in a single forward batch. + device: the device where the backend runs. + """ + + def __init__(self, max_loras_per_batch: int, device: torch.device): + self.max_loras_per_batch = max_loras_per_batch + self.device = device + + def run_lora_a_sgemm( + self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs + ) -> torch.Tensor: + """Run segment Gemm of lora a modules with current backend. + The definition of segment Gemm can be referred to https://docs.flashinfer.ai/api/gemm.html. + + Args: + x: input matrix with shape (s, input_dim), here s is the sum of all sequence lengths + weights: a set of lora weights with shape (num_lora, c * r, input_dim), + here r is lora rank, c is a multiplier for stacked modules (e.g., c=3 for qkv_proj, c=2 for gate_up_proj) + usually input_dim is much larger than r + Returns: + result with shape (s, c * r) + """ + pass + + def run_lora_b_sgemm( + self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs + ) -> torch.Tensor: + """Run segment Gemm of lora b modules with current backend. + The definition of segment Gemm can be referred to https://docs.flashinfer.ai/api/gemm.html. + + Args: + x: input matrix with shape (s, r), here s is the sum of all sequence lengths, r is lora rank + weights: a set of lora weights with shape (num_lora, output_dim, r) + usually output_dim is much larger than r + Returns: + result with shape (s, output_dim) + """ + pass + + def run_qkv_lora( + self, + x: torch.Tensor, + qkv_lora_a: torch.Tensor, + qkv_lora_b: Union[torch.Tensor, Tuple[torch.Tensor]], + *args, + **kwargs, + ) -> torch.Tensor: + """Run the lora pass for QKV Layer. + + Args: + x: input matrix with shape (s, input_dim), here s is the sum of all sequence lengths + qkv_lora_a: lora_a module for qkv, with shape (num_lora, 3 * r, input_dim) + qkv_lora_b: lora_b module for qkv. + If passed in as a tensor, its shape should be (num_lora,output_dim_q + 2 * output_dim_kv, r) + If passed in as a tuple of two tensors, it should contain: + a lora_b module for q, with shape (1, num_lora, output_dim_q, r) + and a combined lora_b module for kv, with shape (2, num_lora, output_dim_kv, r) + Returns: + result with shape (s, output_dim_q + 2 * output_dim_kv) + """ + pass + + def run_gate_up_lora( + self, + x: torch.Tensor, + gate_up_lora_a: torch.Tensor, + gate_up_lora_b: Union[torch.Tensor, Tuple[torch.Tensor]], + *args, + **kwargs, + ) -> torch.Tensor: + """Run the lora pass for gate_up_proj, usually attached to MergedColumnParallelLayer. + + Args: + x: input matrix with shape (s, input_dim), here s is the sum of all sequence lengths + gate_up_lora_a: lora_a module for gate_up_proj, with shape (num_lora, 2 * r, input_dim) + gate_up_lora_b: lora_b module for qkv. + If passed in as a tensor, its shape should be (num_lora, 2 * output_dim, r) + If passed in as a tuple, it should contain two tensors with shape (num_lora, output_dim, r) + Returns: + result with shape (s, 2 * output_dim) + """ + pass + + def init_cuda_graph_batch_info( + self, + cuda_graph_batch_info: LoRABatchInfo, + max_bs_in_cuda_graph: int, + ): + """Initialize the batch info for CUDA Graph mode. + + This method provides a hook for each backend to conduct its own initialization + logic for CUDA Graph mode. + + Args: + cuda_graph_batch_info: the LoRABatchInfo object created in LoraManager + max_bs_in_cuda_graph: maximum batch size for CUDA Graph mode + """ + pass + + def prepare_lora_batch( + self, + forward_batch: ForwardBatch, + weight_indices: list[int], + lora_ranks: list[int], + scalings: list[float], + batch_info: Optional[LoRABatchInfo] = None, + ): + """Prepare the lora weights and batch info for current forward batch. + + This method provides a hook for each backend to conduct its own preparation + logic for each forward batch. + + Args: + forward_batch: the ForwardBatch object for current forward pass + weight_indices: list of indices of lora weights to be applied for current batch + lora_ranks: list of lora ranks corresponding to weight_indices + scalings: list of scaling factors corresponding to weight_indices + batch_info: optional LoRABatchInfo object, if not provided, the backend should use its own + internal batch info (e.g., self.cuda_graph_batch_info for CUDA Graph mode) + """ + pass + + +def get_backend_from_name(name: str) -> BaseLoRABackend: + """ + Get corresponding backend class from backend's name + """ + if name == "triton": + from sglang.srt.lora.backend.triton_backend import TritonLoRABackend + + return TritonLoRABackend + # elif name == "csgmv": + # from sglang.srt.lora.backend.chunked_backend import ChunkedSgmvLoRABackend + + # return ChunkedSgmvLoRABackend + elif name == "flashinfer": + raise ValueError( + "FlashInfer LoRA backend has been deprecated, please use `triton` instead." + ) + else: + raise ValueError(f"Invalid backend: {name}") diff --git a/python/sglang/srt/lora/backend/triton_backend.py b/python/sglang/srt/lora/backend/triton_backend.py new file mode 100644 index 000000000..7abeef770 --- /dev/null +++ b/python/sglang/srt/lora/backend/triton_backend.py @@ -0,0 +1,176 @@ +from typing import Optional + +import torch + +from sglang.srt.lora.backend.base_backend import BaseLoRABackend +from sglang.srt.lora.triton_ops import ( + gate_up_lora_b_fwd, + qkv_lora_b_fwd, + sgemm_lora_a_fwd, + sgemm_lora_b_fwd, +) +from sglang.srt.lora.utils import LoRABatchInfo +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + + +class TritonLoRABackend(BaseLoRABackend): + name = "triton" + + def __init__(self, max_loras_per_batch: int, device: torch.device): + super().__init__(max_loras_per_batch, device) + + def run_lora_a_sgemm( + self, x: torch.Tensor, weights: torch.Tensor, *args, **kwargs + ) -> torch.Tensor: + return sgemm_lora_a_fwd(x, weights, self.batch_info) + + def run_lora_b_sgemm( + self, + x: torch.Tensor, + weights: torch.Tensor, + base_output: torch.Tensor = None, + *args, + **kwargs + ) -> torch.Tensor: + return sgemm_lora_b_fwd(x, weights, self.batch_info, base_output) + + def run_qkv_lora( + self, + x: torch.Tensor, + qkv_lora_a: torch.Tensor, + qkv_lora_b: torch.Tensor, + output_offset: torch.Tensor, + max_qkv_out_dim: int, + base_output: torch.Tensor = None, + *args, + **kwargs + ) -> torch.Tensor: + + # x: (s, input_dim) + # qkv_lora_a: (num_lora, 3 * r, input_dim) + # qkv_lora_b: (num_lora, output_dim_q + 2 * output_dim_kv, r) + assert isinstance(qkv_lora_b, torch.Tensor) + + lora_a_output = sgemm_lora_a_fwd(x, qkv_lora_a, self.batch_info, stack_num=3) + lora_output = qkv_lora_b_fwd( + lora_a_output, + qkv_lora_b, + self.batch_info, + output_offset, + max_qkv_out_dim, + base_output, + ) + return lora_output + + def run_gate_up_lora( + self, + x: torch.Tensor, + gate_up_lora_a: torch.Tensor, + gate_up_lora_b: torch.Tensor, + base_output: torch.Tensor = None, + *args, + **kwargs + ) -> torch.Tensor: + + # x: (s, input_dim) + # gate_up_lora_a: (num_lora, 2 * r, input_dim) + # gate_up_lora_b: (num_lora, 2 * output_dim, r) + assert isinstance(gate_up_lora_b, torch.Tensor) + output_dim = gate_up_lora_b.shape[-2] // 2 + + # lora_a_output: (s, 2 * r) + lora_a_output = sgemm_lora_a_fwd( + x, gate_up_lora_a, self.batch_info, stack_num=2 + ) + lora_output = gate_up_lora_b_fwd( + lora_a_output, + gate_up_lora_b, + self.batch_info, + output_dim, + base_output, + ) + return lora_output + + def init_cuda_graph_batch_info( + self, cuda_graph_batch_info: LoRABatchInfo, max_bs_in_cuda_graph: int + ): + # Initialize seg_lens and seg_indptr for CUDA graph as they remain constant + # across batches. + cuda_graph_batch_info.seg_lens[:max_bs_in_cuda_graph].fill_(1) + torch.cumsum( + cuda_graph_batch_info.seg_lens[:max_bs_in_cuda_graph], + dim=0, + out=cuda_graph_batch_info.seg_indptr[1 : max_bs_in_cuda_graph + 1], + ) + + def prepare_lora_batch( + self, + forward_batch: ForwardBatch, + weight_indices: list[int], + lora_ranks: list[int], + scalings: list[float], + batch_info: Optional[LoRABatchInfo] = None, + ): + # Use pinned memory to avoid synchronizations during host-to-device transfer + weight_indices_tensor = torch.tensor( + weight_indices, dtype=torch.int32, pin_memory=True, device="cpu" + ) + lora_ranks_tensor = torch.tensor( + lora_ranks, dtype=torch.int32, pin_memory=True, device="cpu" + ) + scalings_tensor = torch.tensor( + scalings, dtype=torch.float, pin_memory=True, device="cpu" + ) + + bs = forward_batch.batch_size + + if batch_info is not None: + assert ( + batch_info.use_cuda_graph + ), "batch_info.use_cuda_graph must be True when batch_info is provided" + batch_info.bs = forward_batch.batch_size + batch_info.num_segments = forward_batch.batch_size + else: + max_len = ( + # Calculate max_len from the CPU copy to avoid D2H transfer. + max(forward_batch.extend_seq_lens_cpu) + if forward_batch.forward_mode.is_extend() + else 1 + ) + seg_lens = ( + forward_batch.extend_seq_lens + if forward_batch.forward_mode.is_extend() + else torch.ones(bs, device=self.device) + ) + seg_indptr = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device) + seg_indptr[1:] = torch.cumsum(seg_lens, dim=0) + + batch_info = LoRABatchInfo( + bs=forward_batch.batch_size, + num_segments=forward_batch.batch_size, + max_len=max_len, + use_cuda_graph=False, + seg_lens=seg_lens, + seg_indptr=seg_indptr, + weight_indices=torch.empty( + (bs,), dtype=torch.int32, device=self.device + ), + lora_ranks=torch.empty( + (self.max_loras_per_batch,), dtype=torch.int64, device=self.device + ), + scalings=torch.empty( + (self.max_loras_per_batch,), dtype=torch.float, device=self.device + ), + permutation=None, + ) + + # Copy to device asynchronously + batch_info.lora_ranks[: self.max_loras_per_batch].copy_( + lora_ranks_tensor, non_blocking=True + ) + batch_info.scalings[: self.max_loras_per_batch].copy_( + scalings_tensor, non_blocking=True + ) + batch_info.weight_indices[:bs].copy_(weight_indices_tensor, non_blocking=True) + + self.batch_info = batch_info diff --git a/python/sglang/srt/lora/layers.py b/python/sglang/srt/lora/layers.py new file mode 100644 index 000000000..4426faccb --- /dev/null +++ b/python/sglang/srt/lora/layers.py @@ -0,0 +1,352 @@ +import torch +from torch import nn + +from sglang.srt.distributed import ( + get_tensor_model_parallel_rank, + split_tensor_along_last_dim, + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding +from sglang.srt.lora.backend.base_backend import BaseLoRABackend + + +class BaseLayerWithLoRA(nn.Module): + def __init__( + self, + base_layer: nn.Module, + lora_backend: BaseLoRABackend, + ): + super().__init__() + self.base_layer: nn.Module = base_layer + self.set_lora: bool = False + self.lora_backend: BaseLoRABackend = lora_backend + + def forward(self, x: torch.Tensor): + return self.base_layer.forward(x) + + def set_lora_info(self, *args): + pass + + def slice_lora_a_weights(self, A: torch.Tensor, tp_rank: int): + pass + + def slice_lora_b_weights(self, B: torch.Tensor, tp_rank: int): + pass + + +class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): + """ + Vocab parallel embedding layer with support for LoRA (Low-Rank Adaptation). + + Note: The current version does not yet implement the LoRA functionality. + This class behaves exactly the same as the base VocabParallelEmbedding. + Future versions will integrate LoRA functionality to support efficient parameter fine-tuning. + """ + + def __init__( + self, + base_layer: VocabParallelEmbedding, + lora_backend: BaseLoRABackend, + ) -> None: + super().__init__(base_layer, lora_backend) + self.weight = base_layer.weight + + +class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA): + def __init__( + self, + base_layer: ColumnParallelLinear, + lora_backend: BaseLoRABackend, + ) -> None: + super().__init__(base_layer, lora_backend) + shard_size = self.base_layer.output_partition_sizes[0] + self.output_offset = torch.tensor( + [ + 0, + shard_size, + ], + dtype=torch.int32, + device=next(self.base_layer.parameters()).device, + ) + + def set_lora_info( + self, + A_buffer: torch.Tensor, + B_buffer: torch.Tensor, + ): + self.set_lora = True + self.A_buffer = A_buffer + self.B_buffer = B_buffer + + def apply_lora(self, base_output: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + lora_a_output = self.lora_backend.run_lora_a_sgemm(x, self.A_buffer) + lora_output = self.lora_backend.run_lora_b_sgemm( + x=lora_a_output, + weights=self.B_buffer, + output_offset=self.output_offset, + base_output=base_output, + ) + return lora_output + + def forward(self, input_: torch.Tensor): + # duplicate the logic in ColumnParallelLinear + bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None + output_parallel = self.base_layer.quant_method.apply( + self.base_layer, input_, bias + ) + + if self.set_lora: + output_parallel = self.apply_lora(output_parallel, input_) + + if self.base_layer.gather_output: + output = tensor_model_parallel_all_gather(output_parallel) + else: + output = output_parallel + output_bias = self.base_layer.bias if self.base_layer.skip_bias_add else None + return output, output_bias + + def slice_lora_a_weights(self, A: torch.Tensor, tp_rank: int): + return A + + def slice_lora_b_weights(self, B: torch.Tensor, tp_rank: int): + shard_size = self.base_layer.output_partition_sizes[0] + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + B = B[start_idx:end_idx, :] + return B + + +class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): + def __init__( + self, + base_layer: MergedColumnParallelLinear, + lora_backend: BaseLoRABackend, + ) -> None: + super().__init__(base_layer, lora_backend) + + def set_lora_info( + self, + A_buffer: torch.Tensor, + B_buffer: torch.Tensor, + ): + self.set_lora = True + self.A_buffer_gate_up = A_buffer + self.B_buffer_gate_up = B_buffer + + shard_size = self.base_layer.output_partition_sizes[0] + self.output_offset = torch.tensor( + [ + 0, + shard_size, + 2 * shard_size, + ], + dtype=torch.int32, + device=next(self.base_layer.parameters()).device, + ) + + def apply_lora(self, base_output: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + lora_output = self.lora_backend.run_gate_up_lora( + x=x, + gate_up_lora_a=self.A_buffer_gate_up, + gate_up_lora_b=self.B_buffer_gate_up, + output_offset=self.output_offset, + base_output=base_output, + ) + return lora_output + + def slice_lora_a_weights(self, A: torch.Tensor, tp_rank: int): + return A + + def slice_lora_b_weights(self, B: torch.Tensor, tp_rank: int): + # Since the outputs for both gate and up are identical, we use a random one. + shard_size = self.base_layer.output_partition_sizes[0] + gate_size = self.base_layer.output_sizes[0] + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + return torch.concat( + ( + B[start_idx:end_idx, :], + B[gate_size + start_idx : gate_size + end_idx], + ), + dim=0, + ) + + +class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): + def __init__( + self, + base_layer: QKVParallelLinear, + lora_backend: BaseLoRABackend, + ) -> None: + super().__init__(base_layer, lora_backend) + q_proj_shard_size = self.base_layer.q_proj_shard_size + kv_proj_shard_size = self.base_layer.kv_proj_shard_size + self.output_offset = torch.tensor( + [ + 0, + q_proj_shard_size, + q_proj_shard_size + kv_proj_shard_size, + q_proj_shard_size + 2 * kv_proj_shard_size, + ], + dtype=torch.int32, + device=next(self.base_layer.parameters()).device, + ) + + # For computing number of launched blocks + self.max_qkv_out_dim = max(q_proj_shard_size, kv_proj_shard_size) + + def set_lora_info( + self, + A_buffer_qkv: torch.Tensor, + B_buffer_qkv: torch.Tensor, + ): + self.set_lora = True + self.A_buffer_qkv = A_buffer_qkv + self.B_buffer_qkv = B_buffer_qkv + + def apply_lora(self, base_output: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + lora_output = self.lora_backend.run_qkv_lora( + x=x, + qkv_lora_a=self.A_buffer_qkv, + qkv_lora_b=self.B_buffer_qkv, + base_output=base_output, + output_offset=self.output_offset, + max_qkv_out_dim=self.max_qkv_out_dim, + ) + return lora_output + + def slice_lora_a_weights(self, A: torch.Tensor, tp_rank: int): + return A + + def slice_lora_b_weights(self, B: torch.Tensor, tp_rank: int) -> torch.Tensor: + base_layer = self.base_layer + q_proj_shard_size = base_layer.q_proj_shard_size + kv_proj_shard_size = base_layer.kv_proj_shard_size + num_kv_head_replicas = base_layer.num_kv_head_replicas + + q_start_idx = q_proj_shard_size * tp_rank + q_end_idx = q_start_idx + q_proj_shard_size + + kv_shard_id = tp_rank // num_kv_head_replicas + kv_start_idx = kv_proj_shard_size * kv_shard_id + kv_end_idx = kv_start_idx + kv_proj_shard_size + + q_size, k_size, _ = base_layer.output_sizes + B_q_shard = B[q_start_idx:q_end_idx, :] + B_k_shard = B[q_size + kv_start_idx : q_size + kv_end_idx, :] + B_v_shard = B[q_size + k_size + kv_start_idx : q_size + k_size + kv_end_idx, :] + + return torch.concat( + ( + B_q_shard, + B_k_shard, + B_v_shard, + ), + dim=0, + ) + + +class RowParallelLinearWithLoRA(BaseLayerWithLoRA): + def __init__( + self, + base_layer: RowParallelLinear, + lora_backend: BaseLoRABackend, + ) -> None: + super().__init__(base_layer, lora_backend) + + def set_lora_info(self, A_buffer: torch.Tensor, B_buffer: torch.Tensor): + self.set_lora = True + self.A_buffer = A_buffer + self.B_buffer = B_buffer + output_size = self.base_layer.output_size + self.output_offset = torch.tensor( + [ + 0, + output_size, + ], + dtype=torch.int32, + device=next(self.base_layer.parameters()).device, + ) + + def apply_lora(self, base_output: torch.Tensor, x: torch.Tensor) -> torch.Tensor: + lora_a_output = self.lora_backend.run_lora_a_sgemm(x, self.A_buffer) + lora_output = self.lora_backend.run_lora_b_sgemm( + x=lora_a_output, + weights=self.B_buffer, + output_offset=self.output_offset, + base_output=base_output, + ) + return lora_output + + def forward(self, input_: torch.Tensor, skip_all_reduce=False): + # duplicate the logic in RowParallelLinear + if self.base_layer.input_is_parallel: + input_parallel = input_ + else: + tp_rank = get_tensor_model_parallel_rank() + splitted_input = split_tensor_along_last_dim( + input_, num_partitions=self.base_layer.tp_size + ) + input_parallel = splitted_input[tp_rank].contiguous() + output_parallel = self.base_layer.quant_method.apply( + self.base_layer, input_parallel + ) + + if self.set_lora: + output_parallel = self.apply_lora(output_parallel, input_parallel) + + if ( + self.base_layer.reduce_results + and self.base_layer.tp_size > 1 + and not skip_all_reduce + ): + output_ = tensor_model_parallel_all_reduce(output_parallel) + else: + output_ = output_parallel + + if not self.base_layer.skip_bias_add: + output = ( + output_ + self.base_layer.bias + if self.base_layer.bias is not None + else output_ + ) + output_bias = None + else: + output = output_ + output_bias = self.base_layer.bias + return output, output_bias + + def slice_lora_a_weights(self, A: torch.Tensor, tp_rank: int): + shard_size = self.base_layer.input_size_per_partition + start_idx = tp_rank * shard_size + end_idx = (tp_rank + 1) * shard_size + A = A[:, start_idx:end_idx].contiguous() + return A + + def slice_lora_b_weights(self, B: torch.Tensor, tp_rank: int): + return B + + +def get_lora_layer( + layer: nn.Module, lora_backend: BaseLoRABackend +) -> BaseLayerWithLoRA: + supported_layer_types = { + # the order matters + VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA, + QKVParallelLinear: QKVParallelLinearWithLoRA, + MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA, + ColumnParallelLinear: ColumnParallelLinearWithLoRA, + RowParallelLinear: RowParallelLinearWithLoRA, + } + for src_layer_type, lora_layer_type in supported_layer_types.items(): + if isinstance(layer, src_layer_type): # pylint: disable=unidiomatic-typecheck + ret = lora_layer_type(layer, lora_backend) + return ret + raise Exception(f"No corresponding LoRA layer supported for {type(layer)}.") diff --git a/python/sglang/srt/lora/lora.py b/python/sglang/srt/lora/lora.py new file mode 100644 index 000000000..e7569624c --- /dev/null +++ b/python/sglang/srt/lora/lora.py @@ -0,0 +1,178 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters" +# and "Punica: Multi-Tenant LoRA Serving" + +# LoRA layers class inheritance adapted from: +# https://github.com/vllm-project/vllm/blob/4abf6336ec65c270343eb895e7b18786e9274176/vllm/lora/layers.py + +import logging +import re +from typing import Dict, List + +import torch +from torch import nn + +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.hf_transformers_utils import AutoConfig +from sglang.srt.lora.backend.base_backend import BaseLoRABackend + +# from sglang.srt.lora.backend.chunked_backend import ChunkedSgmvLoRABackend +from sglang.srt.lora.backend.triton_backend import TritonLoRABackend +from sglang.srt.lora.lora_config import LoRAConfig +from sglang.srt.model_loader.loader import DefaultModelLoader + +logger = logging.getLogger(__name__) + + +class LoRALayer(nn.Module): + def __init__(self, config: LoRAConfig, base_hf_config: AutoConfig): + super().__init__() + self.config: LoRAConfig = config + self.base_hf_config: AutoConfig = base_hf_config + + # lora weights in cpu. The weights are loaded from checkpoint. + self.weights: Dict[str, torch.Tensor] = {} + + +class LoRAAdapter(nn.Module): + def __init__( + self, + uid: str, + config: LoRAConfig, + base_hf_config: AutoConfig, + load_config: LoadConfig, + lora_backend: BaseLoRABackend, + ): + super().__init__() + self.uid: str = uid + self.config: LoRAConfig = config + assert self.config.hf_config["peft_type"].lower() == "lora" + self.base_hf_config: AutoConfig = base_hf_config + self.load_config: LoadConfig = load_config + self.lora_backend: BaseLoRABackend = lora_backend + self.scaling: float = self.config.lora_alpha / self.config.r + + self.layers: List[LoRALayer] = nn.ModuleList( + [ + LoRALayer(config, base_hf_config) + for _ in range(base_hf_config.num_hidden_layers) + ] + ) + + self.weights: Dict[str, torch.Tensor] = {} + + # initialize the LoRA weights to cpu + def initialize_weights(self): + model_path = self.config.path + loader = DefaultModelLoader(self.load_config) + revision = getattr(self.config.hf_config, "revision", None) + for name, loaded_weight in loader._get_weights_iterator( + DefaultModelLoader.Source( + model_path, revision=revision, fall_back_to_pt=True + ) + ): + match = re.search(r"layers\.(\d+)\.", name) + if match is not None: + layer_id = int(match.group(1)) + self.layers[layer_id].weights[name] = loaded_weight.cpu() + else: + self.weights[name] = loaded_weight.cpu() + + # normalize kv_proj and gate_up_proj + for layer in self.layers: + weight_names = list(layer.weights.keys()) + self.normalize_qkv_proj(weight_names, layer.weights) + self.normalize_gate_up_proj(weight_names, layer.weights) + + def normalize_qkv_proj( + self, weight_names: List[str], weights: Dict[str, torch.Tensor] + ): + # Collect target q/k/v modules. This process is necessary since there might be no lora attached to k_proj + target_module = set() + for weight_name in weight_names: + if "k_proj" in weight_name: + target_module.add("k_proj") + if "q_proj" in weight_name: + target_module.add("q_proj") + if "v_proj" in weight_name: + target_module.add("v_proj") + if "qkv_proj" in weight_name: + target_module.add("qkv_proj") + if len(target_module) == 0: + return + + for weight_name in weight_names: + # We assume every lora adaptor should contain lora modules for q_proj + if "q_proj" in weight_name: + q_name = weight_name + k_name = weight_name.replace("q_proj", "k_proj") + v_name = weight_name.replace("q_proj", "v_proj") + qkv_name = weight_name.replace("q_proj", "qkv_proj") + + # If k_proj doesn't have lora, initialize it to zero + k_proj_weight = ( + weights[k_name] + if "k_proj" in target_module + else torch.zeros_like(weights[v_name]) + ) + weights[qkv_name] = torch.cat( + ( + weights[q_name], + k_proj_weight, + weights[v_name], + ), + 0, + ) + weights.pop(q_name) + if "k_proj" in target_module: + weights.pop(k_name) + weights.pop(v_name) + elif "qkv_proj" in weight_name: + # If qkv_proj is already stacked, we normalize it following the SGL convention. + qkv_name = weight_name + q_name = weight_name.replace("qkv_proj", "q_proj") + k_name = weight_name.replace("qkv_proj", "k_proj") + v_name = weight_name.replace("qkv_proj", "v_proj") + if "lora_A" in weight_name: + weights[qkv_name] = weights[qkv_name].repeat(3, 1) + # else: no-op as LoRA B weight is already stacked. + + def normalize_gate_up_proj( + self, weight_names: List[str], weights: Dict[str, torch.Tensor] + ): + for weight_name in weight_names: + if "gate_proj" in weight_name: + up_name = weight_name.replace("gate_proj", "up_proj") + gate_up_name = weight_name.replace("gate_proj", "gate_up_proj") + if up_name not in weights: + weights[up_name] = torch.zeros_like(weights[weight_name]) + assert isinstance(self.lora_backend, TritonLoRABackend), ( + f"LoRA weight initialization currently only supported for 'triton' backend. " + f"Received backend: {self.lora_backend.name}. Please verify your backend configuration " + f"or consider implementing custom initialization logic for other backends." + ) + weights[gate_up_name] = torch.cat( + (weights[weight_name], weights[up_name]), 0 + ) + weights.pop(weight_name) + if up_name in weights: + weights.pop(up_name) + elif "gate_up_proj" in weight_name: + # If gate_up_proj is already stacked, we normalize it following the SGL convention + gate_up_name = weight_name + if "lora_A" in weight_name: + weights[gate_up_name] = weights[gate_up_name].repeat(2, 1) + # else: no-op as LoRA B weight is already stacked. diff --git a/python/sglang/srt/lora/lora_config.py b/python/sglang/srt/lora/lora_config.py new file mode 100644 index 000000000..185b7b824 --- /dev/null +++ b/python/sglang/srt/lora/lora_config.py @@ -0,0 +1,47 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import os + +from huggingface_hub import snapshot_download + + +class LoRAConfig: + def __init__( + self, + path: str, + ) -> None: + self.path = path + self.hf_config = self.get_lora_config() + self.target_modules = self.hf_config["target_modules"] + + # TODO: Support more modules + if any(module in self.target_modules for module in ["embed_tokens", "lm_head"]): + raise ValueError("Not supported yet") + + self.r = self.hf_config["r"] + self.lora_alpha = self.hf_config["lora_alpha"] + + def get_lora_config(self, dummy=False): + if dummy: + raise NotImplementedError() + else: + if not os.path.isdir(self.path): + weights_dir = snapshot_download(self.path, allow_patterns=["*.json"]) + else: + weights_dir = self.path + config_name = "adapter_config.json" + with open(os.path.join(weights_dir, config_name), "r") as f: + return json.load(f) diff --git a/python/sglang/srt/lora/lora_manager.py b/python/sglang/srt/lora/lora_manager.py new file mode 100644 index 000000000..baf120ca2 --- /dev/null +++ b/python/sglang/srt/lora/lora_manager.py @@ -0,0 +1,440 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# Integrates "S-LoRA: Serving Thousands of Concurrent LoRA Adapters" +# and "Punica: Multi-Tenant LoRA Serving" + +import logging +from typing import Dict, Iterable, List, Optional, Set, Tuple + +import torch + +from sglang.srt.configs.load_config import LoadConfig +from sglang.srt.hf_transformers_utils import AutoConfig +from sglang.srt.lora.backend.base_backend import BaseLoRABackend, get_backend_from_name +from sglang.srt.lora.layers import BaseLayerWithLoRA, get_lora_layer +from sglang.srt.lora.lora import LoRAAdapter +from sglang.srt.lora.lora_config import LoRAConfig +from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.lora.mem_pool import LoRAMemoryPool +from sglang.srt.lora.utils import ( + LoRABatchInfo, + LoRAType, + get_layer_id, + get_normalized_target_modules, + get_target_module_name, +) +from sglang.srt.managers.io_struct import LoRAUpdateResult +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.utils import replace_submodule + +logger = logging.getLogger(__name__) + + +class LoRAManager: + def __init__( + self, + base_model: torch.nn.Module, + base_hf_config: AutoConfig, + max_loras_per_batch: int, + load_config: LoadConfig, + dtype: torch.dtype, + lora_backend: str = "triton", + tp_size: int = 1, + tp_rank: int = 0, + max_lora_rank: Optional[int] = None, + target_modules: Optional[Iterable[str]] = None, + lora_paths: Optional[List[LoRARef]] = None, + ): + self.base_model: torch.nn.Module = base_model + self.base_hf_config: AutoConfig = base_hf_config + self.max_loras_per_batch: int = max_loras_per_batch + self.load_config: LoadConfig = load_config + self.dtype: torch.dtype = dtype + self.device: torch.device = next(self.base_model.parameters()).device + self.tp_size: int = tp_size + self.tp_rank: int = tp_rank + + # LoRA backend for running sgemm kernels + logger.info(f"Using {lora_backend} as backend of LoRA kernels.") + backend_type = get_backend_from_name(lora_backend) + self.lora_backend: BaseLoRABackend = backend_type( + max_loras_per_batch=max_loras_per_batch, + device=self.device, + ) + + # Initialize mutable internal state of the LoRAManager. + self.init_state( + max_lora_rank=max_lora_rank, + target_modules=target_modules, + lora_paths=lora_paths, + ) + + def init_cuda_graph_batch_info(self, max_bs_in_cuda_graph: int): + self.max_bs_in_cuda_graph = max_bs_in_cuda_graph + with torch.device("cuda"): + self.cuda_graph_batch_info = LoRABatchInfo( + bs=max_bs_in_cuda_graph, + use_cuda_graph=True, + num_segments=None, + seg_lens=torch.zeros(max_bs_in_cuda_graph, dtype=torch.int32), + seg_indptr=torch.zeros(max_bs_in_cuda_graph + 1, dtype=torch.int32), + max_len=1, + weight_indices=torch.zeros(max_bs_in_cuda_graph, dtype=torch.int32), + permutation=torch.zeros(max_bs_in_cuda_graph, dtype=torch.int32), + lora_ranks=torch.zeros(self.max_loras_per_batch, dtype=torch.int32), + scalings=torch.zeros(self.max_loras_per_batch, dtype=torch.float), + ) + + self.lora_backend.init_cuda_graph_batch_info( + cuda_graph_batch_info=self.cuda_graph_batch_info, + max_bs_in_cuda_graph=max_bs_in_cuda_graph, + ) + + def create_lora_update_result( + self, success: bool, error_message: str = "" + ) -> LoRAUpdateResult: + return LoRAUpdateResult( + success=success, + error_message=error_message, + loaded_adapters={ + lora_ref.lora_name: lora_ref.lora_path + for lora_ref in self.lora_refs.values() + }, + ) + + def load_lora_adapter(self, lora_ref: LoRARef) -> LoRAUpdateResult: + """ + Load a single LoRA adapter from the specified path. + + Args: + lora_ref (LoRARef): The LoRARef object containing the LoRA name, path, and ID. + """ + assert ( + lora_ref.lora_name is not None and lora_ref.lora_path is not None + ), "LoRARef must have both lora_name and lora_path set for loading." + assert ( + lora_ref.lora_id not in self.loras + ), f"LoRA adapter with ID {lora_ref.lora_id} is already loaded. This should have been verified before request is sent to the backend." + + try: + # load configs + new_adapter = LoRAConfig(lora_ref.lora_path) + self.validate_new_adapter(new_adapter, lora_ref) + self.configs[lora_ref.lora_id] = new_adapter + + # load weights + self.load_lora_weights(lora_ref) + + # keep metadata for displayed messages + self.lora_refs[lora_ref.lora_id] = lora_ref + self.num_pinned_loras += int(lora_ref.pinned) + except Exception as e: + return self.create_lora_update_result( + success=False, + error_message=str(e), + ) + + return self.create_lora_update_result(success=True) + + def validate_new_adapter(self, lora_config: LoRAConfig, lora_ref: LoRARef): + """ + Validate if an adapter can be loaded into the current LoRA memory pool and generate error if it is incompatible. + """ + + # Check if the LoRA adapter shape is compatible with the current LoRA memory pool configuration. + memory_pool = getattr(self, "memory_pool", None) + incompatible = memory_pool and not memory_pool.can_support(lora_config) + if incompatible: + raise ValueError( + f"LoRA adapter {lora_ref.lora_name} with rank {lora_config.r} is incompatible with the current " + "LoRA memory pool configuration. Please ensure that the LoRA adapter's rank is within the configured " + "`--max-lora-rank` and that the target modules are included in `--lora-target-modules`." + ) + + # Ensure pinned LoRA adapters does not exceed maximal limit or cause starvation. + if lora_ref.pinned and self.num_pinned_loras >= self.max_loras_per_batch - 1: + raise ValueError( + f"Failed to load LoRA adapter {lora_ref.lora_name} as a pinned adapter. It is not allowed to pin all slots " + "in the LoRA memory pool to avoid starvation for unpinned adapters and base models. Please increase your " + "`--max-loras-per-batch` or load it as unpinned LoRA adapters." + ) + + def unload_lora_adapter(self, lora_ref: LoRARef) -> LoRAUpdateResult: + """ + Unload LoRA adapters by their names. This will remove the adapters from the memory pool and + delete the corresponding LoRA modules. + """ + + adapter = self.configs.get(lora_ref.lora_id) + lora_ref = self.lora_refs.get(lora_ref.lora_id) + assert ( + adapter is not None and lora_ref is not None + ), f"LoRA adapter with ID {lora_ref.lora_id} is not loaded. This should have been verified before request is sent to the backend." + + try: + del self.configs[lora_ref.lora_id] + del self.loras[lora_ref.lora_id] + del self.lora_refs[lora_ref.lora_id] + self.num_pinned_loras -= int(lora_ref.pinned) + except Exception as e: + return self.create_lora_update_result( + success=False, + error_message=str(e), + ) + + return self.create_lora_update_result(success=True) + + def validate_lora_batch(self, lora_ids: set[str]) -> bool: + """ + Validate if the LoRA IDs in the batch can be loaded into the current LoRA memory pool. + """ + if len(lora_ids) > self.max_loras_per_batch: + return False + + # skip pinned LoRA check if no pinned LoRA adapters are loaded. + if self.num_pinned_loras == 0: + return True + + # counting the number of pinned LoRA adapters in the batch. + pinned_loras_in_batch = 0 + for lora_id in lora_ids: + if lora_id is not None: + lora_ref = self.lora_refs.get(lora_id) + assert ( + lora_ref is not None + ), f"LoRA ID {lora_id} not found in lora_refs." + pinned_loras_in_batch += int(lora_ref.pinned) + + assert pinned_loras_in_batch <= self.num_pinned_loras, ( + f"Number of pinned LoRA adapters in the batch ({pinned_loras_in_batch}) exceeds the total number of pinned adapters " + f"({self.num_pinned_loras}). This indicates a bug in the LoRA loading logic." + ) + + required_slots = len(lora_ids) - pinned_loras_in_batch + mem_pool_vacancy = self.memory_pool.max_loras_per_batch - self.num_pinned_loras + + return required_slots <= mem_pool_vacancy + + def prepare_lora_batch(self, forward_batch: ForwardBatch): + # Load active loras into lora memory pool + cur_uids = set(forward_batch.lora_ids) + + assert len(cur_uids) <= self.max_loras_per_batch + self.memory_pool.prepare_lora_batch( + cur_uids=cur_uids, + lora_adapters=self.loras, + lora_modules=self.lora_modules, + lora_refs=self.lora_refs.copy(), # copy snapshot of current lora_refs to avoid mutation during the batch preparation. + ) + + # set up batch info shared by all lora modules + bs = forward_batch.batch_size + + use_cuda_graph = ( + hasattr(self, "max_bs_in_cuda_graph") + and bs <= self.max_bs_in_cuda_graph + and forward_batch.forward_mode.is_cuda_graph() + ) + + weight_indices = [0] * len(forward_batch.lora_ids) + lora_ranks = [0] * self.max_loras_per_batch + scalings = [0] * self.max_loras_per_batch + for i, uid in enumerate(forward_batch.lora_ids): + weight_indices[i] = self.memory_pool.get_buffer_id(uid) + if uid is not None: + lora = self.loras[uid] + lora_ranks[weight_indices[i]] = lora.config.r + scalings[weight_indices[i]] = lora.scaling + # Do in-place updates when CUDA graph is enabled and the batch forward mode + # could use CUDA graph. + self.lora_backend.prepare_lora_batch( + forward_batch=forward_batch, + weight_indices=weight_indices, + lora_ranks=lora_ranks, + scalings=scalings, + batch_info=self.cuda_graph_batch_info if use_cuda_graph else None, + ) + + def update_lora_info(self): + """ + Update all LoRA modules to associate them with the latest memory buffer. + """ + for layer_id, layer_modules in enumerate(self.lora_modules): + for module_name, module in layer_modules.items(): + target_module = get_target_module_name( + module_name, self.memory_pool.target_modules + ) + module.set_lora_info( + self.memory_pool.get_tensor( + target_module=target_module, + layer_id=layer_id, + lora_type=LoRAType.LORA_A, + ), + self.memory_pool.get_tensor( + target_module=target_module, + layer_id=layer_id, + lora_type=LoRAType.LORA_B, + ), + ) + + def init_state( + self, + max_lora_rank: Optional[int] = None, + target_modules: Optional[Iterable[str]] = None, + lora_paths: Optional[List[LoRARef]] = None, + ): + """ + Initialize the internal (mutable) state of the LoRAManager. + + When `lora_paths` is provided and not empty, it might be used for inferring LoRA shape info such as + the target modules and max_lora_rank. + """ + + assert lora_paths or ( + max_lora_rank is not None and target_modules is not None + ), "When no initial --lora-paths is provided, you need to specify both --max-lora-rank and --lora-target-modules for LoRA initialization." + + self.init_lora_adapters(lora_paths) + self.init_lora_shapes( + max_lora_rank=max_lora_rank, + target_modules=target_modules, + ) + self.init_lora_modules() + self.init_memory_pool() + self.update_lora_info() + + def init_lora_adapters(self, lora_paths: Optional[List[LoRARef]] = None): + # Configs of all active LoRA adapters, indexed by LoRA ID. + self.configs: Dict[str, LoRAConfig] = {} + + # LoRA adapter weights cached in CPU memory, indexed by LoRA ID. + self.loras: Dict[str, LoRAAdapter] = {} + + # Mapping from LoRA ID to LoRARef object. + self.lora_refs: Dict[str, LoRARef] = {} + + # Count of pinned LoRA adapters. + self.num_pinned_loras: int = 0 + + if lora_paths: + for lora_ref in lora_paths: + result = self.load_lora_adapter(lora_ref) + if not result.success: + raise RuntimeError( + f"Failed to load LoRA adapter {lora_ref.lora_name}: {result.error_message}" + ) + + def init_lora_shapes( + self, + max_lora_rank: Optional[int] = None, + target_modules: Optional[Iterable[str]] = None, + ): + """Infer LoRA target modules and max_lora_rank from loaded adapters if not provided.""" + + self.target_modules = ( + get_normalized_target_modules(target_modules) if target_modules else set() + ) + + for lora_id, config in self.configs.items(): + if not isinstance(config.target_modules, list): + raise ValueError( + f"SGLang currently only supports inferring LoRA target modules when a list of " + "suffixes is provided in `target_modules` field of PEFT config. Please explicitly " + "specify `--lora-target-modules` during server startup. You can specify `all` to " + "enable all support modules types. " + ) + + adapter_target_modules = get_normalized_target_modules( + config.target_modules + ) + + if target_modules is not None: + # When `--lora-target-modules` is provided, validate adapter target modules is a subset of the specified target modules. + if not adapter_target_modules.issubset(self.target_modules): + unsupported_modules = adapter_target_modules - self.target_modules + lora_name = self.lora_refs[lora_id].lora_name + raise ValueError( + f"LoRA adapter '{lora_name}' contains target modules {sorted(unsupported_modules)} " + f"that are not included in the specified --lora-target-modules {sorted(self.target_modules)}. " + f"Please update --lora-target-modules to include all required modules: " + f"{sorted(self.target_modules | adapter_target_modules)}, or use 'all' to enable all supported modules." + ) + else: + # Otherwise, infer target_modules from adapter configs. + self.target_modules.update(adapter_target_modules) + + if max_lora_rank is not None: + self.max_lora_rank = max_lora_rank + else: + self.max_lora_rank = max( + [x.r for x in self.configs.values()], + default=0, + ) + + def load_lora_weights(self, lora_ref: LoRARef): + """ + Load the weights of a LoRA adapter to CPU memory and conducts post-loading validation. + """ + lora_adapter = LoRAAdapter( + lora_ref.lora_id, + self.configs[lora_ref.lora_id], + self.base_hf_config, + self.load_config, + self.lora_backend, + ) + lora_adapter.initialize_weights() + self.loras[lora_ref.lora_id] = lora_adapter + + def init_memory_pool(self): + """(Re)initialize the LoRA memory pool based on the current configurations.""" + self.memory_pool = LoRAMemoryPool( + base_hf_config=self.base_hf_config, + max_loras_per_batch=self.max_loras_per_batch, + dtype=self.dtype, + tp_size=self.tp_size, + tp_rank=self.tp_rank, + max_lora_rank=self.max_lora_rank, + target_modules=self.target_modules, + base_model=self.base_model, + ) + + def set_lora_module(self, module_name, module): + lora_module = get_lora_layer(module, self.lora_backend) + replace_submodule(self.base_model, module_name, lora_module) + return lora_module + + def init_lora_modules(self): + # Look-up table that essentially maps (layer_index, module_name) to the corresponding LoRA module. + self.lora_modules: List[Dict[str, BaseLayerWithLoRA]] = [ + {} for _ in range(self.base_hf_config.num_hidden_layers) + ] + + for module_name, module in self.base_model.named_modules(): + # TODO (lifuhuang): in the future, we should consider generalizing the + # should_apply_lora function to support mapping by full module name instead + # of just the last part (e.g., "qkv_proj") to support scenarios with multiple + # attention stacks (e.g., multimodal models). + # See: https://github.com/sgl-project/sglang/issues/6608 + if getattr( + self.base_model, "should_apply_lora", None + ) and not self.base_model.should_apply_lora(module_name): + continue + + # The module should be converted if it is included in target_names + if module_name.split(".")[-1] in self.target_modules: + layer_id = get_layer_id(module_name) + self.lora_modules[layer_id][module_name] = self.set_lora_module( + module_name, module + ) diff --git a/python/sglang/srt/lora/lora_registry.py b/python/sglang/srt/lora/lora_registry.py new file mode 100644 index 000000000..51d2b0e66 --- /dev/null +++ b/python/sglang/srt/lora/lora_registry.py @@ -0,0 +1,207 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + + +import asyncio +from dataclasses import dataclass, field, fields +from typing import Dict, List, Optional, Union +from uuid import uuid4 + +from sglang.srt.aio_rwlock import RWLock +from sglang.srt.utils import ConcurrentCounter + + +@dataclass(frozen=True) +class LoRARef: + """ + Reference record for a LoRA model. + + This object guarantees a unique ``lora_id`` and may include ``lora_name``, ``lora_path``, and ``pinned``. + The ID eliminates conflicts from reused LoRA names or paths and can be used to generate deterministic cache + keys (e.g., radix cache). + """ + + lora_id: str = field(default_factory=lambda: uuid4().hex) + lora_name: Optional[str] = None + lora_path: Optional[str] = None + pinned: Optional[bool] = None + + def __post_init__(self): + if self.lora_id is None: + raise ValueError("lora_id cannot be None") + + def __str__(self) -> str: + parts = [ + f"{f.name}={value}" + for f in fields(self) + if (value := getattr(self, f.name)) is not None + ] + return f"{self.__class__.__name__}({', '.join(parts)})" + + +class LoRARegistry: + """ + The central registry to keep track of available LoRA adapters and ongoing LoRA requests. + + The `LoRARegistry` resides in the tokenizer manager process and acts as the single source of truth for all + available LoRA adapters. It supports concurrent inference and dynamic adapter updates through a two-phase + update / eventual consistency model between the tokenizer manager process and the scheduler processes. + """ + + def __init__(self, lora_paths: Optional[List[LoRARef]] = None): + assert lora_paths is None or all( + isinstance(lora, LoRARef) for lora in lora_paths + ), ( + "server_args.lora_paths should have been normalized to LoRARef objects during server initialization. " + "Please file an issue if you see this error." + ) + + # A read-write lock to ensure adapters loading / unloading operations are exclusive. + # Please note that the counter increment/decrement operations are not synchronized through this + # lock, as they are designed to be non-blocking and can be performed concurrently. + self._registry_lock = RWLock() + # A dictionary to hold LoRARef objects, mapping from LoRA name to LoRARef. + self._registry: Dict[str, LoRARef] = {} + # Counters for ongoing requests, mapping from LoRA ID to ConcurrentCounter. + self._counters: Dict[str, ConcurrentCounter] = {} + + # Initialize the registry with provided LoRA paths, if present. + if lora_paths: + for lora_ref in lora_paths: + self._register_adapter(lora_ref) + + async def register(self, lora_ref: LoRARef): + """ + Register a new LoRARef object in the registry. + + Args: + lora_ref (LoRARef): The LoRARef object to register. + """ + async with self._registry_lock.writer_lock: + self._register_adapter(lora_ref) + + async def unregister(self, lora_name: str) -> str: + """ + Unregister a LoRARef object from the registry and returns the removed LoRA ID. + + Args: + lora_name (str): The name of the LoRA model to unregister. + """ + async with self._registry_lock.writer_lock: + lora_ref = self._registry.get(lora_name, None) + if lora_ref is None: + raise ValueError( + f"LoRA with name {lora_name} does not exist. Loaded LoRAs: {self._registry.keys()}" + ) + del self._registry[lora_name] + + return lora_ref.lora_id + + async def acquire(self, lora_name: Union[str, List[str]]) -> Union[str, List[str]]: + """ + Queries registry for LoRA IDs based on LoRA names and start tracking the usage of the corresponding LoRA adapters + by incrementing its counter. + """ + + def _lookup(name: str) -> str: + if name is None: + return None + + lora_ref = self._registry.get(name, None) + if lora_ref is None: + raise ValueError( + f"The following requested LoRA adapters are not loaded: {name}\n" + f"Loaded adapters: {self._registry.keys()}." + ) + return lora_ref.lora_id + + async with self._registry_lock.reader_lock: + if isinstance(lora_name, str): + lora_id = _lookup(lora_name) + await self._counters[lora_id].increment(notify_all=False) + return lora_id + elif isinstance(lora_name, list): + lora_ids = [_lookup(name) for name in lora_name] + + # Increment the counters only after all IDs are looked up. + await asyncio.gather( + *[ + self._counters[id].increment(notify_all=False) + for id in lora_ids + if id is not None + ] + ) + return lora_ids + else: + raise TypeError( + "lora_name must be either a string or a list of strings." + ) + + async def release(self, lora_id: Union[str, List[str]]): + """ + Decrements the usage counter for a LoRA adapter, indicating that it is no longer in use. + """ + + async with self._registry_lock.reader_lock: + if isinstance(lora_id, str): + await self._counters[lora_id].decrement() + elif isinstance(lora_id, list): + await asyncio.gather( + *[ + self._counters[id].decrement() + for id in lora_id + if id is not None + ] + ) + else: + raise TypeError("lora_id must be either a string or a list of strings.") + + async def wait_for_unload(self, lora_id: str): + """ + Waits until the usage counter for a LoRA adapter reaches zero, indicating that it is no longer in use. + This is useful for ensuring that a LoRA adapter can be safely unloaded. + + This method itself is not synchronized, which is safe because it should only be called during LoRA unloading, + which itself is guaranteed to be sequential. + """ + assert ( + lora_id not in self._registry + ), "wait_for_unload should only be called after the LoRA adapter has been unregistered. " + assert ( + lora_id in self._counters + ), "The LoRA ID should still have a counter if it has been registered before." + + # Wait until no requests are using this LoRA adapter. + await self._counters[lora_id].wait_for_zero() + del self._counters[lora_id] + + def _register_adapter(self, lora_ref: LoRARef): + """ + Internal helper method to register a LoRA adapter. + """ + + if lora_ref.lora_name in self._registry: + raise ValueError( + f"LoRA with name {lora_ref.lora_name} already exists. Loaded LoRAs: {self._registry.keys()}" + ) + self._registry[lora_ref.lora_name] = lora_ref + self._counters[lora_ref.lora_id] = ConcurrentCounter() + return lora_ref + + @property + def num_registered_loras(self) -> int: + """ + Returns the total number of LoRA adapters currently registered. + """ + return len(self._registry) diff --git a/python/sglang/srt/lora/mem_pool.py b/python/sglang/srt/lora/mem_pool.py new file mode 100644 index 000000000..cdf1707e8 --- /dev/null +++ b/python/sglang/srt/lora/mem_pool.py @@ -0,0 +1,309 @@ +import logging +from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union + +import torch + +from sglang.srt.distributed import divide +from sglang.srt.hf_transformers_utils import AutoConfig +from sglang.srt.lora.layers import BaseLayerWithLoRA +from sglang.srt.lora.lora import LoRAAdapter +from sglang.srt.lora.lora_config import LoRAConfig +from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.lora.utils import ( + ROW_PARALLELISM_LINEAR_LORA_NAMES, + LoRAType, + get_hidden_dim, + get_normalized_target_modules, + get_stacked_multiply, + get_target_module_name, +) + +logger = logging.getLogger(__name__) + + +class EmptySlot: + """ + Singleton class to represent an empty slot in the memory pool. + This is used to improve readability by not using special str as a placeholder. + """ + + __slots__ = () + + def __repr__(self): + return "|EMPTY|" + + def __new__(cls): + if not hasattr(cls, "_instance"): + cls._instance = super().__new__(cls) + return cls._instance + + +EMPTY_SLOT = EmptySlot() + + +class LoRAMemoryPool: + """Class for memory pool management of lora modules""" + + def __init__( + self, + base_hf_config: AutoConfig, + max_loras_per_batch: int, + dtype: torch.dtype, + tp_size: int, + tp_rank: int, + max_lora_rank: int, + target_modules: Set[str], + base_model: torch.nn.Module, + ): + self.base_hf_config: AutoConfig = base_hf_config + self.num_layer: int = base_hf_config.num_hidden_layers + self.max_loras_per_batch: int = max_loras_per_batch + self.dtype: torch.dtype = dtype + self.tp_size: int = tp_size + self.tp_rank: int = tp_rank + self.max_lora_rank: int = max_lora_rank + self.target_modules: Set[str] = target_modules + + # Both A_buffer and B_buffer maps lora weight names to its buffer space. + # A_buffer contains num_layer number of row-major tensors with shape + # (max_loras_per_batch, stacked_num * max_lora_dim, input_dim) + # B_buffer contains num_layer number of column-major tensors with shape + # (stacked_num, max_loras_per_batch, output_dim, max_lora_dim) + self.A_buffer: Dict[str, List[torch.Tensor]] = {} + self.B_buffer: Dict[str, List[torch.Tensor]] = {} + + # Lora uid -> buffer idx in memory pool + self.uid_to_buffer_id: Dict[Optional[str], int] = {} + + # Buffer idx -> lora uid in memory pool + # All uids are initialized as `EmptySlot` for empty buffer slots + # Here we don't initialize to None since None is a valid uid + self.buffer_id_to_uid: List[Union[str, None, EmptySlot]] = [ + EMPTY_SLOT + ] * self.max_loras_per_batch + + self.init_buffers(base_model) + + def can_support(self, config: Union[LoRAConfig, Iterable[LoRAConfig]]) -> bool: + """ + Check if the memory pool can support the given LoRA adapters. + """ + + def _can_support(config: LoRAConfig) -> bool: + """ + Check if the memory pool can support a single LoRA adapter. + """ + if config.r > self.max_lora_rank: + return False + target_module_names = get_normalized_target_modules(config.target_modules) + return target_module_names.issubset(self.target_modules) + + if isinstance(config, LoRAConfig): + return _can_support(config) + else: + return all(_can_support(x) for x in config) + + def get_lora_A_shape( + self, + module_name: str, + base_model: torch.nn.Module, + max_lora_dim: int, + layer_idx: int, + ) -> Tuple[int]: + """ + Given a module_name (might be a stacked name), return the hidden dims of modules' input and output. + """ + input_dim, _ = get_hidden_dim( + module_name, self.base_hf_config, base_model, layer_idx + ) + c = get_stacked_multiply(module_name) + if self.tp_size > 1 and module_name in ROW_PARALLELISM_LINEAR_LORA_NAMES: + input_dim = divide(input_dim, self.tp_size) + return ( + self.max_loras_per_batch, + max_lora_dim * c, + input_dim, + ) + + def get_lora_B_shape( + self, + module_name: str, + base_model: torch.nn.Module, + max_lora_dim: int, + layer_idx: int, + ) -> Tuple[int]: + """ + Given a module_name (might be a stacked name), return the hidden dims of modules' input and output. + """ + _, output_dim = get_hidden_dim( + module_name, self.base_hf_config, base_model, layer_idx + ) + if self.tp_size > 1 and module_name not in ROW_PARALLELISM_LINEAR_LORA_NAMES: + output_dim = divide(output_dim, self.tp_size) + return ( + self.max_loras_per_batch, + output_dim, + max_lora_dim, + ) + + def init_buffers(self, base_model: torch.nn.Module): + device = next(base_model.parameters()).device + + def init_buffer( + buffer: Dict[str, List[torch.Tensor]], + target_modules: Set[str], + get_lora_shape_fn: Callable[[str, torch.nn.Module, int, int], Tuple[int]], + ): + for module_name in target_modules: + buffer[module_name] = [ + torch.empty( + get_lora_shape_fn( + module_name, + base_model, + self.max_lora_rank, + idx, + ), + dtype=self.dtype, + device=device, + ) + for idx in range(self.num_layer) + ] + + init_buffer( + self.A_buffer, + self.target_modules, + self.get_lora_A_shape, + ) + + init_buffer( + self.B_buffer, + self.target_modules, + self.get_lora_B_shape, + ) + + def prepare_lora_batch( + self, + cur_uids: Set[Optional[str]], + lora_adapters: Dict[str, LoRAAdapter], + lora_modules: List[Dict[str, BaseLayerWithLoRA]], + lora_refs: Dict[str, LoRARef], + ): + def get_available_buffer_slot(): + for buffer_id in range(self.max_loras_per_batch): + # Prioritize empty slots + if self.buffer_id_to_uid[buffer_id] == EMPTY_SLOT: + return buffer_id + + for buffer_id in range(self.max_loras_per_batch): + uid = self.buffer_id_to_uid[buffer_id] + + # Evict unneeded lora + if uid not in cur_uids: + # Skip pinned LoRAs + # TODO (lifuhuang): we might consider supporting pinning base model (uid == None) in the future. + if uid is not None: + lora_ref = lora_refs.get(uid) + if lora_ref is not None and lora_ref.pinned: + continue + + self.uid_to_buffer_id.pop(uid) + logger.debug(f"Evicting LoRA {uid} from buffer slot {buffer_id}.") + self.buffer_id_to_uid[buffer_id] = EMPTY_SLOT + return buffer_id + + raise ValueError( + "No available buffer slots found. Please ensure the number of active loras is less than max_loras_per_batch." + ) + + for uid in cur_uids: + if uid not in self.uid_to_buffer_id: + buffer_id = get_available_buffer_slot() + lora_adapter = lora_adapters.get(uid, None) + self.load_lora_weight_to_buffer( + uid, buffer_id, lora_adapter, lora_modules + ) + self.uid_to_buffer_id[uid] = buffer_id + self.buffer_id_to_uid[buffer_id] = uid + + def load_lora_weight_to_buffer( + self, + uid: str, + buffer_id: int, + lora_adapter: LoRAAdapter, + lora_modules: List[Dict[str, BaseLayerWithLoRA]], + ): + def load_lora_weight_tensor( + buffer_view: torch.Tensor, weight: Optional[torch.Tensor] + ): + if weight is None: + # If the particular weight is not present in the adapter, we initialize the buffer to zero + # to avoid contamination from the residual weight of the evicted adapters. + buffer_view.zero_() + else: + assert ( + buffer_view.shape == weight.shape + ), f"LoRA buffer shape {buffer_view.shape} does not match weight shape {weight.shape}." + buffer_view.copy_(weight) + + if uid is None: + for i in range(self.num_layer): + for k in self.A_buffer.keys(): + self.A_buffer[k][i][buffer_id] = 0 + return + + assert lora_adapter is not None + lora_rank = lora_adapter.config.r + for layer_id in range(self.num_layer): + layer_weights = lora_adapter.layers[layer_id].weights + temp_A_buffer: Dict[str, Optional[torch.Tensor]] = { + target_module: None for target_module in self.A_buffer + } + temp_B_buffer: Dict[str, Optional[torch.Tensor]] = { + target_module: None for target_module in self.B_buffer + } + for name, weights in layer_weights.items(): + target_module = get_target_module_name(name, self.target_modules) + if "lora_A" in name: + temp_A_buffer[target_module] = weights + else: + temp_B_buffer[target_module] = weights + + if self.tp_size > 1: + cur_layer_modules = lora_modules[layer_id] + for module_name, module in cur_layer_modules.items(): + target_module = get_target_module_name( + module_name, self.target_modules + ) + + if temp_A_buffer[target_module] is None: + # Skip weight slicing if the weight is not present in the adapter + continue + + temp_A_buffer[target_module] = module.slice_lora_a_weights( + temp_A_buffer[target_module], self.tp_rank + ) + temp_B_buffer[target_module] = module.slice_lora_b_weights( + temp_B_buffer[target_module], self.tp_rank + ) + + for name, weights in temp_A_buffer.items(): + c = get_stacked_multiply(name) + target_buffer = self.A_buffer[name][layer_id] + buffer_view = target_buffer[buffer_id, : lora_rank * c, :] + load_lora_weight_tensor(buffer_view, weights) + + for name, weights in temp_B_buffer.items(): + target_buffer = self.B_buffer[name][layer_id] + buffer_view = target_buffer[buffer_id, :, :lora_rank] + load_lora_weight_tensor(buffer_view, weights) + + def get_tensor( + self, target_module: str, layer_id: int, lora_type: LoRAType + ) -> torch.Tensor: + if lora_type == LoRAType.LORA_A: + return self.A_buffer[target_module][layer_id] + + return self.B_buffer[target_module][layer_id] + + def get_buffer_id(self, lora_uid: str): + return self.uid_to_buffer_id[lora_uid] diff --git a/python/sglang/srt/lora/triton_ops/__init__.py b/python/sglang/srt/lora/triton_ops/__init__.py new file mode 100644 index 000000000..da55e8fd5 --- /dev/null +++ b/python/sglang/srt/lora/triton_ops/__init__.py @@ -0,0 +1,11 @@ +from .gate_up_lora_b import gate_up_lora_b_fwd +from .qkv_lora_b import qkv_lora_b_fwd +from .sgemm_lora_a import sgemm_lora_a_fwd +from .sgemm_lora_b import sgemm_lora_b_fwd + +__all__ = [ + "gate_up_lora_b_fwd", + "qkv_lora_b_fwd", + "sgemm_lora_a_fwd", + "sgemm_lora_b_fwd", +] diff --git a/python/sglang/srt/lora/triton_ops/gate_up_lora_b.py b/python/sglang/srt/lora/triton_ops/gate_up_lora_b.py new file mode 100644 index 000000000..fc4574dd3 --- /dev/null +++ b/python/sglang/srt/lora/triton_ops/gate_up_lora_b.py @@ -0,0 +1,187 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.lora.utils import LoRABatchInfo + + +@triton.jit +def _gate_up_lora_b_kernel( + # Pointers to matrices + x, + weights, + output, + # Parameters of size + K, # K = R + output_dim, + # Strides + x_stride_0, + x_stride_1, + w_stride_0, + w_stride_1, + w_stride_2, + output_stride_0, + output_stride_1, + # Information on sequence lengths,ranks and weight id + seg_lens, + seg_indptr, + weight_indices, + lora_ranks, + # Meta parameters + BLOCK_S: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + # For fused output scaling + scalings, +): + """ + This kernel packs 2 sgemms (gate/up) into a single kernel. The multiplication + results are accumulated into the output tensor. + + When a sequence's rank is 0, the kernel is essentially a no-op, following + the convention in pytorch where the product of two matrices of shape (m, 0) + and (0, n) is an all-zero matrix of shape (m, n). + + Args: + x (Tensor): The input tensor, which is the result of the LoRA A projection. + Shape: (s, 2 * K), where s is the sum of all sequence lengths in the + batch and K is the maximum LoRA rank. + weights (Tensor): The LoRA B weights for all adapters. + Shape: (num_lora, 2 * output_dim, K). + output (Tensor): The output tensor where the result is stored. + Shape: (s, 2 * output_dim). + """ + # output_dim >> K + + # Current block computes sequence with batch_id, + # which starts from row seg_start of x with length seg_len. + # gate_up_id decides which of gate or up (0: gate, 1: up) + batch_id = tl.program_id(axis=2) + w_index = tl.load(weight_indices + batch_id) + rank = tl.load(lora_ranks + w_index) + + # If rank is 0, this kernel is a no-op. + if rank == 0: + return + + gate_up_id = tl.program_id(axis=1) + pid = tl.program_id(axis=0) + seg_len = tl.load(seg_lens + batch_id) + seg_start = tl.load(seg_indptr + batch_id) + n_start = gate_up_id * output_dim # offset on output dim + scaling = tl.load(scalings + w_index) + + # Adjust K (rank) according to the specific LoRA adapter + K = tl.minimum(K, rank) + + # The tile in output matrix will have (pid_s, pid_n) as id + num_pid_n = tl.cdiv(output_dim, BLOCK_N) + pid_s = pid // num_pid_n + pid_n = pid % num_pid_n + + # Create pointers for the first block of x and weights + # The pointers will be advanced as we move in the K direction + # and accumulate + s_offset = tl.arange(0, BLOCK_S) + pid_s * BLOCK_S + n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + k_offset = tl.arange(0, BLOCK_K) + + x_ptrs = (x + seg_start * x_stride_0 + (gate_up_id * K) * x_stride_1) + ( + s_offset[:, None] * x_stride_0 + k_offset[None, :] * x_stride_1 + ) + w_ptrs = (weights + w_index * w_stride_0 + n_start * w_stride_1) + ( + k_offset[:, None] * w_stride_2 + n_offset[None, :] * w_stride_1 + ) + + # Iterate to compute the block in output matrix + partial_sum = tl.zeros((BLOCK_S, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + x_tile = tl.load( + x_ptrs, + mask=(s_offset[:, None] < seg_len) & (k_offset[None, :] < K - k * BLOCK_K), + other=0.0, + ) + w_tile = tl.load( + w_ptrs, + mask=(k_offset[:, None] < K - k * BLOCK_K) + & (n_offset[None, :] < output_dim), + other=0.0, + ) + partial_sum += tl.dot(x_tile, w_tile) + + x_ptrs += BLOCK_K * x_stride_1 + w_ptrs += BLOCK_K * w_stride_2 + + # Store result to output matrix + partial_sum *= scaling + partial_sum = partial_sum.to(x.dtype.element_ty) + output_ptr = (output + seg_start * output_stride_0 + n_start * output_stride_1) + ( + s_offset[:, None] * output_stride_0 + n_offset[None, :] * output_stride_1 + ) + output_mask = (s_offset[:, None] < seg_len) & (n_offset[None, :] < output_dim) + partial_sum += tl.load(output_ptr, mask=output_mask) + tl.store(output_ptr, partial_sum, mask=output_mask) + + +def gate_up_lora_b_fwd( + x: torch.Tensor, + gate_up_lora_b: torch.Tensor, + batch_info: LoRABatchInfo, + output_dim: int, + base_output: torch.Tensor = None, +) -> torch.Tensor: + + # x: (s, 2 * r) + # gate_up_lora_b: (num_lora, 2 * output_dim, r) + # output: (s, 2 * output_dim) + + # Compute lora_output with shape (s, output_dim) as follows: + # lora_output[:, :output_dim] = sgemm(x[:, :r], gate_up_lora_b[:, :output_dim, :]) + # lora_output[:, output_dim:] + # = sgemm(x[:, r:], gate_up_lora_b[:, output_dim:, :]) + + # Get dims + s = x.shape[0] + input_dim = x.shape[1] + r = gate_up_lora_b.shape[-1] + assert input_dim == 2 * r + + BLOCK_S = 16 + BLOCK_R = 16 + BLOCK_OUT = 64 + + grid_b = ( + triton.cdiv(batch_info.max_len, BLOCK_S) * triton.cdiv(output_dim, BLOCK_OUT), + 2, # this dimension decides current block computes on gate or up proj + batch_info.bs, + ) + + if base_output is None: + output = torch.zeros((s, 2 * output_dim), device=x.device, dtype=x.dtype) + else: + output = base_output + + _gate_up_lora_b_kernel[grid_b]( + x, + gate_up_lora_b, + output, + r, + output_dim, + x.stride(0), + x.stride(1), + gate_up_lora_b.stride(0), + gate_up_lora_b.stride(1), + gate_up_lora_b.stride(2), + output.stride(0), + output.stride(1), + batch_info.seg_lens, + batch_info.seg_indptr, + batch_info.weight_indices, + batch_info.lora_ranks, + BLOCK_S, + BLOCK_OUT, + BLOCK_R, + batch_info.scalings, + ) + + return output diff --git a/python/sglang/srt/lora/triton_ops/qkv_lora_b.py b/python/sglang/srt/lora/triton_ops/qkv_lora_b.py new file mode 100644 index 000000000..1d6663dbe --- /dev/null +++ b/python/sglang/srt/lora/triton_ops/qkv_lora_b.py @@ -0,0 +1,198 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.lora.utils import LoRABatchInfo + + +@triton.jit +def _qkv_lora_b_kernel( + # Pointers to matrices + x, + weights, + output, + # Parameters of size + K, # K = R + max_qkv_out_dim, # max(output_q_dim, output_kv_dim) + # Strides + x_stride_0, + x_stride_1, + w_stride_0, + w_stride_1, + w_stride_2, + output_stride_0, + output_stride_1, + # Information on sequence lengths and weight id + seg_lens, + seg_indptr, + weight_indices, + lora_ranks, + # Offsets of q/k/v slice on output dimension + n_offs, + # Meta parameters + BLOCK_S: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + # For fused output scaling + scalings, +): + """ + This kernel packs 3 sgemms (q/k/v) into a single kernel. The multiplication + results are accumulated into the output tensor. + + When a sequence's rank is 0, the kernel is essentially a no-op, following + the convention in pytorch where the product of two matrices of shape (m, 0) + and (0, n) is an all-zero matrix of shape (m, n). + + Args: + x (Tensor): The input tensor, which is the result of the LoRA A projection. + Shape: (s, 3 * K), where s is the sum of all sequence lengths in the + batch and K is the maximum LoRA rank. The second dimension is partitioned + for Q, K, and V. + weights (Tensor): The LoRA B weights for all adapters. + Shape: (num_lora, N_Q + 2 * N_KV, K). + output (Tensor): The output tensor where the result is stored. + Shape: (s, N_Q + 2 * N_KV). + """ + + # Current block computes sequence with batch_id, + # which starts from row seg_start of x with length seg_len. + # qkv_id decides which of q,k,v to compute (0: q, 1: k, 2: v) + batch_id = tl.program_id(axis=2) + w_index = tl.load(weight_indices + batch_id) + rank = tl.load(lora_ranks + w_index) + + # If rank is 0, this kernel is a no-op. + if rank == 0: + return + + qkv_id = tl.program_id(axis=1) + pid = tl.program_id(axis=0) + seg_len = tl.load(seg_lens + batch_id) + seg_start = tl.load(seg_indptr + batch_id) + n_start = tl.load(n_offs + qkv_id) + n_size = tl.load(n_offs + qkv_id + 1) - n_start + scaling = tl.load(scalings + w_index) + # Adjust K (rank) according to the specific LoRA adapter + K = tl.minimum(K, rank) + + # The tile in output matrix will have (pid_s, pid_n) as id + num_pid_n = tl.cdiv(max_qkv_out_dim, BLOCK_N) + pid_s = pid // num_pid_n + pid_n = pid % num_pid_n + + # Create pointers for the first block of x and weights[batch_id][n_start: n_end][:] + # The pointers will be advanced as we move in the K direction + # and accumulate + s_offset = tl.arange(0, BLOCK_S) + pid_s * BLOCK_S + n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + k_offset = tl.arange(0, BLOCK_K) + + x_ptrs = (x + seg_start * x_stride_0 + (qkv_id * K) * x_stride_1) + ( + s_offset[:, None] * x_stride_0 + k_offset[None, :] * x_stride_1 + ) + w_ptrs = (weights + w_index * w_stride_0 + n_start * w_stride_1) + ( + k_offset[:, None] * w_stride_2 + n_offset[None, :] * w_stride_1 + ) + + # Iterate to compute the block in output matrix + partial_sum = tl.zeros((BLOCK_S, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + x_tile = tl.load( + x_ptrs, + mask=(s_offset[:, None] < seg_len) & (k_offset[None, :] < K - k * BLOCK_K), + other=0.0, + ) + w_tile = tl.load( + w_ptrs, + mask=(k_offset[:, None] < K - k * BLOCK_K) & (n_offset[None, :] < n_size), + other=0.0, + ) + partial_sum += tl.dot(x_tile, w_tile) + + x_ptrs += BLOCK_K * x_stride_1 + w_ptrs += BLOCK_K * w_stride_2 + + # Store result to output matrix + partial_sum *= scaling + partial_sum = partial_sum.to(x.dtype.element_ty) + output_ptr = (output + seg_start * output_stride_0 + n_start * output_stride_1) + ( + s_offset[:, None] * output_stride_0 + n_offset[None, :] * output_stride_1 + ) + output_mask = (s_offset[:, None] < seg_len) & (n_offset[None, :] < n_size) + partial_sum += tl.load(output_ptr, mask=output_mask) + tl.store(output_ptr, partial_sum, mask=output_mask) + + +def qkv_lora_b_fwd( + x: torch.Tensor, + qkv_lora_b: torch.Tensor, + batch_info: LoRABatchInfo, + output_offset: torch.Tensor, + max_qkv_out_dim: int, + base_output: torch.Tensor = None, +) -> torch.Tensor: + + # x: (s, 3 * r) + # qkv_lora_b: (num_lora, output_dim_q + 2 * output_dim_kv, r) + # output_offset = [0, output_dim_q, output_dim_q + output_dim_kv, + # output_dim_q + 2 * output_dim_kv] + # max_qkv_out_dim = max(output_dim_q, output_dim_kv) + # output: (s, output_dim_q + 2 * output_dim_kv) + + # Compute lora_output with shape (s, output_dim) as follows: + # lora_output[:, :output_dim_q] = sgemm(x[:, :r], qkv_lora_b[:, :outptu_dim_q, :]) + # lora_output[:, output_dim_q: output_dim_q + output_dim_kv] + # = sgemm(x[:, r: 2 * r], qkv_lora_b[:, outptu_dim_q: output_dim_q + output_dim_kv, :]) + # lora_output[:, output_dim_q + output_dim_kv: ] + # = sgemm(x[:, 2 * r: , qkv_lora_b[:, output_dim_q + output_dim_kv: , :]) + + # Get dims + s = x.shape[0] + input_dim = x.shape[1] + r = qkv_lora_b.shape[-1] + output_dim = qkv_lora_b.shape[-2] + assert input_dim == 3 * r + assert output_offset.shape[0] == 4 + + BLOCK_S = 16 + BLOCK_R = 16 + BLOCK_OUT = 64 + + grid_b = ( + triton.cdiv(batch_info.max_len, BLOCK_S) + * triton.cdiv(max_qkv_out_dim, BLOCK_OUT), + 3, # this dimension decides current block computes on q, k or v + batch_info.bs, + ) + + if base_output is None: + output = torch.zeros((s, output_dim), device=x.device, dtype=x.dtype) + else: + output = base_output + + _qkv_lora_b_kernel[grid_b]( + x, + qkv_lora_b, + output, + r, + max_qkv_out_dim, + x.stride(0), + x.stride(1), + qkv_lora_b.stride(0), + qkv_lora_b.stride(1), + qkv_lora_b.stride(2), + output.stride(0), + output.stride(1), + batch_info.seg_lens, + batch_info.seg_indptr, + batch_info.weight_indices, + batch_info.lora_ranks, + output_offset, + BLOCK_S, + BLOCK_OUT, + BLOCK_R, + batch_info.scalings, + ) + + return output diff --git a/python/sglang/srt/lora/triton_ops/sgemm_lora_a.py b/python/sglang/srt/lora/triton_ops/sgemm_lora_a.py new file mode 100644 index 000000000..dded64bcf --- /dev/null +++ b/python/sglang/srt/lora/triton_ops/sgemm_lora_a.py @@ -0,0 +1,170 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.lora.utils import LoRABatchInfo + + +@triton.jit +def _sgemm_lora_a_kernel( + # Pointers to matrices + x, + weights, + output, + # Matrix dimensions + N, # stack_num * r + K, # input_dim + stack_num, + # Strides + x_stride_0, + x_stride_1, + w_stride_0, + w_stride_1, + w_stride_2, + output_stride_0, + output_stride_1, + # Information on sequence lengths,ranks and weight id + seg_lens, + seg_indptr, + weight_indices, + lora_ranks, + # Meta parameters + BLOCK_S: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """ + Computes a segmented batched matrix multiplication for the LoRA A matrix. + + The kernel ensures that output[seg_start:seg_start + seg_len, :rank * stack_num] + stores the product of the input `x` and the LoRA weights for the corresponding + sequence. This implies that when rank is 0, the kernel is essentially a no-op, + as output[seg_start:seg_start + seg_len, :0] is trivially correct (empty). + + Args: + x (torch.Tensor): The input activations tensor of shape `(s, K)`, where `s` + is the sum of all sequence lengths in the batch. + weights (torch.Tensor): The LoRA 'A' weights for all available adapters, + with shape `(num_lora, N, K)`. + output (torch.Tensor): The output tensor of shape `(s, N)`. + """ + + # Current block computes sequence with batch_id, + # which starts from row seg_start of x with length seg_len + batch_id = tl.program_id(axis=1) + w_index = tl.load(weight_indices + batch_id) + rank = tl.load(lora_ranks + w_index) + + # If rank is 0, this kernel becomes a no-op as the output is always trivially correct. + if rank == 0: + return + + pid = tl.program_id(axis=0) + seg_start = tl.load(seg_indptr + batch_id) + seg_len = tl.load(seg_lens + batch_id) + + # Adjust N (stack_num * max_rank) according to the specific LoRA adapter + N = tl.minimum(N, rank * stack_num) + + # The tile in output matrix will have (pid_s, pid_n) as id + num_pid_n = tl.cdiv(N, BLOCK_N) + pid_s = pid // num_pid_n + pid_n = pid % num_pid_n + + # Create pointers for the first block of x and weights[batch_id] + # The pointers will be advanced as we move in the K direction + # and accumulate + s_offset = tl.arange(0, BLOCK_S) + pid_s * BLOCK_S + n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + k_offset = tl.arange(0, BLOCK_K) + x_ptrs = (x + seg_start * x_stride_0) + ( + s_offset[:, None] * x_stride_0 + k_offset[None, :] * x_stride_1 + ) + w_ptrs = (weights + w_index * w_stride_0) + ( + k_offset[:, None] * w_stride_2 + n_offset[None, :] * w_stride_1 + ) + + # Iterate to compute the block in output matrix + partial_sum = tl.zeros((BLOCK_S, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + x_tile = tl.load( + x_ptrs, + mask=(s_offset[:, None] < seg_len) & (k_offset[None, :] < K - k * BLOCK_K), + other=0.0, + ) + w_tile = tl.load( + w_ptrs, + mask=(k_offset[:, None] < K - k * BLOCK_K) & (n_offset[None, :] < N), + other=0.0, + ) + partial_sum += tl.dot(x_tile, w_tile) + + x_ptrs += BLOCK_K * x_stride_1 + w_ptrs += BLOCK_K * w_stride_2 + + # Store result to output matrix + partial_sum = partial_sum.to(x.dtype.element_ty) + output_ptr = (output + seg_start * output_stride_0) + ( + s_offset[:, None] * output_stride_0 + n_offset[None, :] * output_stride_1 + ) + output_mask = (s_offset[:, None] < seg_len) & (n_offset[None, :] < N) + tl.store(output_ptr, partial_sum, mask=output_mask) + + +def sgemm_lora_a_fwd( + x: torch.Tensor, + weights: torch.Tensor, + batch_info: LoRABatchInfo, + stack_num: int = 1, +) -> torch.Tensor: + # x: (s, input_dim) + # weights: (num_lora, stack_num * r, input_dim) + # output: (s, stack_num * r) + # stack_num: run_qkv_lora: 3, run_gate_up_lora: 2 + # when called by run_qkv_lora, the weights.shape[-2] will be 3 * r + # input_dim is much larger than r + + assert x.is_contiguous() + assert weights.is_contiguous() + assert len(x.shape) == 2 + assert len(weights.shape) == 3 + + S = x.shape[0] + R = weights.shape[-2] + K = weights.shape[-1] + assert x.shape[-1] == K + + # Block shapes + BLOCK_S = 16 + BLOCK_K = 256 + BLOCK_R = 16 + + grid = ( + triton.cdiv(batch_info.max_len, BLOCK_S) * triton.cdiv(R, BLOCK_R), + batch_info.bs, + ) + + output = torch.empty((S, R), device=x.device, dtype=x.dtype) + _sgemm_lora_a_kernel[grid]( + x, + weights, + output, + R, + K, + stack_num, + x.stride(0), + x.stride(1), + weights.stride(0), + weights.stride(1), + weights.stride(2), + output.stride(0), + output.stride(1), + batch_info.seg_lens, + batch_info.seg_indptr, + batch_info.weight_indices, + batch_info.lora_ranks, + BLOCK_S, + BLOCK_R, + BLOCK_K, + ) + return output diff --git a/python/sglang/srt/lora/triton_ops/sgemm_lora_b.py b/python/sglang/srt/lora/triton_ops/sgemm_lora_b.py new file mode 100644 index 000000000..357d32805 --- /dev/null +++ b/python/sglang/srt/lora/triton_ops/sgemm_lora_b.py @@ -0,0 +1,176 @@ +import torch +import triton +import triton.language as tl + +from sglang.srt.lora.utils import LoRABatchInfo + + +@triton.jit +def _sgemm_lora_b_kernel( + # Pointers to matrices + x, + weights, + output, + # Matrix dimensions + N, # output_dim + K, # r + # Strides + x_stride_0, + x_stride_1, + w_stride_0, + w_stride_1, + w_stride_2, + output_stride_0, + output_stride_1, + # Information on sequence lengths and weight id + seg_lens, + seg_indptr, + weight_indices, + lora_ranks, + # Meta parameters + BLOCK_S: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + # For fused output scaling + scalings, +): + """ + Computes a segmented batched matrix multiplication for the LoRA B matrix + and adds the result to the output in-place. + + When a sequence's rank is 0, the kernel is essentially a no-op, following + the convention in pytorch where the product of two matrices of shape (m, 0) + and (0, n) is an all-zero matrix of shape (m, n). + + Args: + x (torch.Tensor): The intermediate tensor from the LoRA 'A' multiplication, + of shape `(s, K)`, where `s` is the total number of tokens. + weights (torch.Tensor): The LoRA 'B' weights for all available adapters, + with shape `(num_lora, N, K)`. + output (torch.Tensor): The output tensor of shape `(s, N)`. This can be + the base model's output for a fused add operation. + """ + + # Current block computes sequence with batch_id, + # which starts from row seg_start of x with length seg_len + batch_id = tl.program_id(axis=1) + w_index = tl.load(weight_indices + batch_id) + rank = tl.load(lora_ranks + w_index) + + # If rank is 0, this kernel is a no-op. + if rank == 0: + return + + pid = tl.program_id(axis=0) + seg_len = tl.load(seg_lens + batch_id) + seg_start = tl.load(seg_indptr + batch_id) + scaling = tl.load(scalings + w_index) + # Adjust K (rank) according to the specific LoRA adapter + K = tl.minimum(K, rank) + + # The tile in output matrix will have (pid_s, pid_n) as id + num_pid_n = tl.cdiv(N, BLOCK_N) + pid_s = pid // num_pid_n + pid_n = pid % num_pid_n + + # Create pointers for the first block of x and weights[batch_id] + # The pointers will be advanced as we move in the K direction + # and accumulate + s_offset = tl.arange(0, BLOCK_S) + pid_s * BLOCK_S + n_offset = tl.arange(0, BLOCK_N) + pid_n * BLOCK_N + k_offset = tl.arange(0, BLOCK_K) + x_ptrs = (x + seg_start * x_stride_0) + ( + s_offset[:, None] * x_stride_0 + k_offset[None, :] * x_stride_1 + ) + w_ptrs = (weights + w_index * w_stride_0) + ( + k_offset[:, None] * w_stride_2 + n_offset[None, :] * w_stride_1 + ) + + # Iterate to compute the block in output matrix + partial_sum = tl.zeros((BLOCK_S, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + x_tile = tl.load( + x_ptrs, + mask=(s_offset[:, None] < seg_len) & (k_offset[None, :] < K - k * BLOCK_K), + other=0.0, + ) + w_tile = tl.load( + w_ptrs, + mask=(k_offset[:, None] < K - k * BLOCK_K), + other=0.0, + ) + partial_sum += tl.dot(x_tile, w_tile) + + x_ptrs += BLOCK_K * x_stride_1 + w_ptrs += BLOCK_K * w_stride_2 + + # Store result to output matrix + partial_sum *= scaling + partial_sum = partial_sum.to(x.dtype.element_ty) + output_ptr = (output + seg_start * output_stride_0) + ( + s_offset[:, None] * output_stride_0 + n_offset[None, :] * output_stride_1 + ) + output_mask = s_offset[:, None] < seg_len + partial_sum += tl.load(output_ptr, mask=output_mask) + tl.store(output_ptr, partial_sum, mask=output_mask) + + +def sgemm_lora_b_fwd( + x: torch.Tensor, + weights: torch.Tensor, + batch_info: LoRABatchInfo, + base_output: torch.Tensor = None, +) -> torch.Tensor: + # x: (s, max_r) + # weights: (num_lora, output_dim, max_r) + # output: (s, output_dim) + # output_dim is much larger than max_r + + assert x.is_contiguous() + assert weights.is_contiguous() + assert len(x.shape) == 2 + assert len(weights.shape) == 3 + + S = x.shape[0] + N = weights.shape[-2] + R = weights.shape[-1] + assert x.shape[-1] == R + + # Block shapes + BLOCK_S = 16 + BLOCK_R = 16 + BLOCK_N = 256 + + grid = ( + triton.cdiv(batch_info.max_len, BLOCK_S) * triton.cdiv(N, BLOCK_N), + batch_info.bs, + ) + + if base_output is None: + output = torch.zeros((S, N), device=x.device, dtype=x.dtype) + else: + output = base_output + + _sgemm_lora_b_kernel[grid]( + x, + weights, + output, + N, + R, + x.stride(0), + x.stride(1), + weights.stride(0), + weights.stride(1), + weights.stride(2), + output.stride(0), + output.stride(1), + batch_info.seg_lens, + batch_info.seg_indptr, + batch_info.weight_indices, + batch_info.lora_ranks, + BLOCK_S, + BLOCK_N, + BLOCK_R, + batch_info.scalings, + ) + return output diff --git a/python/sglang/srt/lora/utils.py b/python/sglang/srt/lora/utils.py new file mode 100644 index 000000000..459c943b7 --- /dev/null +++ b/python/sglang/srt/lora/utils.py @@ -0,0 +1,143 @@ +import re +from dataclasses import dataclass +from enum import Enum +from typing import Iterable, Optional, Set, Tuple + +import torch + +from sglang.srt.hf_transformers_utils import AutoConfig + + +@dataclass +class LoRABatchInfo: + # The forward mode is using CUDA Graph. + use_cuda_graph: bool + + # Batch size + bs: int + + # Number of segments. For triton backend, it is equal to batch size. + num_segments: int + + # Indice pointers of each segment in shape (num_segments + 1, ) + seg_indptr: torch.Tensor + + # The index of lora adapter used by each segment, in shape (num_segments,) + weight_indices: torch.Tensor + + # ranks of each lora adapter, in shape (lora_num,) + lora_ranks: torch.Tensor + + # scaling of each lora adapter, in shape (lora_num,) + scalings: torch.Tensor + + # Lengths of each segments in shape (num_segments,) + seg_lens: Optional[torch.Tensor] + + # Maximum segment length of current batch + max_len: Optional[int] + + # The logical (re)ordering of input rows (tokens), in shape (num_tokens,) + permutation: Optional[torch.Tensor] + + +class LoRAType(Enum): + LORA_A = 0 + LORA_B = 1 + + +def get_layer_id(name: str) -> int: + """ + Extract integer id of layer from its name in string. + """ + match = re.search(r"layers\.(\d+)\.", name) + if match is None: + return None + return int(match.group(1)) + + +def get_hidden_dim( + module_name: str, config: AutoConfig, base_model: torch.nn.Module, layer_idx: int +) -> Tuple[int]: + """ + Given a module_name (might be a stacked name), return the hidden dims of modules' input and output. + """ + + if hasattr(base_model, "get_hidden_dim"): + return base_model.get_hidden_dim(module_name, layer_idx) + else: + """ + WARNING: get_hidden_dim() is not defined, + which is used to get the hidden dim for different lora modules + Use the default one, but please check if it is correct for your model. + Please implement the function in the model class if it is not. + You can reference this function in llama.py. + """ + head_dim = getattr( + config, "head_dim", config.hidden_size // config.num_attention_heads + ) + if module_name == "qkv_proj": + return config.hidden_size, head_dim * ( + config.num_attention_heads + config.num_key_value_heads * 2 + ) + elif module_name == "o_proj": + return ( + head_dim * config.num_attention_heads, + config.hidden_size, + ) + elif module_name == "gate_up_proj": + return config.hidden_size, config.intermediate_size * 2 + elif module_name == "down_proj": + return config.intermediate_size, config.hidden_size + else: + raise NotImplementedError() + + +def get_normalized_target_modules( + target_modules: Iterable[str], +) -> set[str]: + """ + Mapping a list of target module name to names of the normalized LoRA weights. + """ + params_mapping = { + "q_proj": "qkv_proj", + "k_proj": "qkv_proj", + "v_proj": "qkv_proj", + "gate_proj": "gate_up_proj", + "up_proj": "gate_up_proj", + } + + result = set() + for name in target_modules: + normalized_name = params_mapping.get(name, name) + result.add(normalized_name) + return result + + +def get_stacked_multiply(module_name: str) -> int: + """ + Mapping a lora module name to its magnification at output dimension + """ + stacked_rank = { + "qkv_proj": 3, + "gate_up_proj": 2, + } + return stacked_rank[module_name] if module_name in stacked_rank else 1 + + +def get_target_module_name(full_module_name: str, target_modules: Set[str]) -> str: + """ + Get the target module name in target_modules that can match full_module_name. + + If there is a target module name in target_modules that can match full_module_name, return this name + Else raise ValueError. + """ + for target_module in target_modules: + if target_module in full_module_name: + return target_module + raise ValueError( + f"Cannot find target module name for {full_module_name} in {target_modules}" + ) + + +ROW_PARALLELISM_LINEAR_LORA_NAMES = ["o_proj", "down_proj"] diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py new file mode 100644 index 000000000..f9d45b2f7 --- /dev/null +++ b/python/sglang/srt/managers/cache_controller.py @@ -0,0 +1,886 @@ +from __future__ import annotations + +""" +Copyright 2023-2025 SGLang Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import math +import threading +import time +from queue import Empty, Full, PriorityQueue, Queue +from typing import TYPE_CHECKING, List, NamedTuple, Optional, Set, Tuple + +import torch + +from sglang.srt.mem_cache.hicache_storage import HiCacheStorageConfig + +if TYPE_CHECKING: + from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator + from sglang.srt.mem_cache.memory_pool_host import HostKVCache + +from sglang.srt.distributed import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from sglang.srt.layers.dp_attention import ( + get_attention_dp_rank, + get_attention_tp_rank, + get_attention_tp_size, + is_dp_attention_enabled, +) +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool + +logger = logging.getLogger(__name__) + + +class LayerLoadingEvent: + def __init__(self, num_layers: int): + self._num_layers = num_layers + self.load_events = [torch.cuda.Event() for _ in range(num_layers)] + self.start_event = torch.cuda.Event() # start event on controller stream + + def complete(self, layer_index: int): + assert 0 <= layer_index < self._num_layers + self.load_events[layer_index].record() + + def wait(self, layer_index: int): + torch.cuda.current_stream().wait_event(self.load_events[layer_index]) + + @property + def finish_event(self): + return self.load_events[-1] + + +class LayerDoneCounter: + def __init__(self, num_layers: int): + self.num_layers = num_layers + # extra producer and consumer counters for overlap mode + self.num_counters = 3 + self.events = [LayerLoadingEvent(num_layers) for _ in range(self.num_counters)] + self.producer_index = -1 + self.consumer_index = -1 + + def update_producer(self): + self.producer_index = (self.producer_index + 1) % self.num_counters + assert self.events[ + self.producer_index + ].finish_event.query(), ( + "Producer finish event should be ready before being reused." + ) + return self.producer_index + + def set_consumer(self, index: int): + self.consumer_index = index + + def wait_until(self, threshold: int): + if self.consumer_index < 0: + return + self.events[self.consumer_index].wait(threshold) + + def reset(self): + self.producer_index = -1 + self.consumer_index = -1 + + +class CacheOperation: + + counter = 0 + + def __init__( + self, + host_indices: torch.Tensor, + device_indices: torch.Tensor, + node_id: int, + priority: Optional[int] = None, + ): + self.host_indices = host_indices + self.device_indices = device_indices + self.node_ids = [node_id] + self.data = None + + self.id = CacheOperation.counter + CacheOperation.counter += 1 + # default priority is the order of creation + self.priority = priority if priority is not None else self.id + + @staticmethod + def merge_ops(ops: List[CacheOperation]) -> CacheOperation: + assert len(ops) > 0 + if len(ops) == 1: + return ops[0] + + host_indices = torch.cat([op.host_indices for op in ops]) + device_indices = torch.cat([op.device_indices for op in ops]) + node_ids = [] + priority = min(op.priority for op in ops) + for op in ops: + node_ids.extend(op.node_ids) + merged_op = CacheOperation(host_indices, device_indices, -1, priority) + merged_op.node_ids = node_ids + return merged_op + + def __lt__(self, other: CacheOperation): + return self.priority < other.priority + + +class HiCacheAck(NamedTuple): + start_event: torch.cuda.Event + finish_event: torch.cuda.Event + node_ids: List[int] + + +class TransferBuffer: + """ + Overlapping buffer preparation and transfer operations to improve throughput. + """ + + def __init__( + self, stop_event, buffer_count: int = 3, max_buffer_size: int = 1024 + ) -> None: + self.stop_event = stop_event + self.buffers = Queue(maxsize=buffer_count) + # todo: adjust the buffer size based on throughput profile of the system + self.max_buffer_size = max_buffer_size + + def full(self) -> bool: + return self.buffers.full() + + def empty(self) -> bool: + return self.buffers.empty() + + def put(self, item, block=True, timeout=1) -> None: + while not self.stop_event.is_set(): + try: + self.buffers.put(item, block=block, timeout=timeout) + break + except Full: + if not block: + break + continue + except Exception as e: + logger.error(e) + + def get(self, block=True, timeout=1) -> Optional[CacheOperation]: + try: + return self.buffers.get(block=block, timeout=timeout) + except Empty: + return None + except Exception as e: + logger.error(e) + + def clear(self): + self.buffers.queue.clear() + + +class StorageOperation: + counter = 0 + + def __init__( + self, + host_indices: torch.Tensor, + token_ids: List[int], + last_hash: Optional[str] = None, + hash_value: Optional[List[str]] = None, + ): + self.host_indices = host_indices + self.token_ids = token_ids + self.last_hash = last_hash + self.completed_tokens = 0 + self.hash_value = hash_value if hash_value is not None else [] + + self.id = StorageOperation.counter + StorageOperation.counter += 1 + + def __lt__(self, other: "StorageOperation"): + return self.id < other.id + + +class PrefetchOperation(StorageOperation): + def __init__( + self, + request_id: str, + host_indices: torch.Tensor, + token_ids: List[int], + last_hash: Optional[str] = None, + ): + self.request_id = request_id + + self._lock = threading.Lock() + self._terminated_flag = False + self.start_time = time.monotonic() + + super().__init__(host_indices, token_ids, last_hash) + + def increment(self, num_tokens: int): + with self._lock: + if self._terminated_flag: + return False + self.completed_tokens += num_tokens + return True + + def mark_terminate(self): + with self._lock: + self._terminated_flag = True + + def is_terminated(self) -> bool: + return self._terminated_flag + + +class HiCacheController: + + def __init__( + self, + token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, + mem_pool_host: HostKVCache, + page_size: int, + tp_group: torch.distributed.ProcessGroup, + load_cache_event: threading.Event, + write_policy: str = "write_through_selective", + io_backend: str = "", + storage_backend: Optional[str] = None, + prefetch_threshold: int = 256, + model_name: Optional[str] = None, + storage_backend_extra_config: Optional[str] = None, + ): + self.mem_pool_device_allocator = token_to_kv_pool_allocator + self.mem_pool_device = token_to_kv_pool_allocator.get_kvcache() + self.mem_pool_host = mem_pool_host + self.write_policy = write_policy + self.page_size = page_size + self.io_backend = io_backend + self.enable_storage = False + + if storage_backend is not None: + self.storage_backend_type = storage_backend + from sglang.srt.mem_cache.hicache_storage import get_hash_str + + self.get_hash_str = get_hash_str + self.storage_config = self._generate_storage_config( + model_name, storage_backend_extra_config + ) + # for MLA models, only one rank needs to backup the KV cache + self.backup_skip = ( + self.storage_config.is_mla_model + # todo: load balancing + and self.storage_config.tp_rank != 0 + ) + + if storage_backend == "file": + from sglang.srt.mem_cache.hicache_storage import HiCacheFile + + self.storage_backend = HiCacheFile(self.storage_config) + elif storage_backend == "nixl": + from sglang.srt.mem_cache.storage.nixl.hicache_nixl import HiCacheNixl + + self.storage_backend = HiCacheNixl() + elif storage_backend == "mooncake": + from sglang.srt.mem_cache.storage.mooncake_store.mooncake_store import ( + MooncakeStore, + ) + + self.storage_backend = MooncakeStore(self.storage_config) + self.storage_backend.register_buffer(self.mem_pool_host.kv_buffer) + assert self.mem_pool_host.layout == "page_first" + elif storage_backend == "hf3fs": + from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import ( + HiCacheHF3FS, + ) + + if self.mem_pool_host.layout == "page_first": + bytes_per_page = ( + mem_pool_host.get_ksize_per_token() * mem_pool_host.page_size + ) + elif self.mem_pool_host.layout == "layer_first": + bytes_per_page = ( + mem_pool_host.get_size_per_token() * mem_pool_host.page_size + ) + dtype = mem_pool_host.dtype + self.storage_backend = HiCacheHF3FS.from_env_config( + bytes_per_page, dtype, self.storage_config + ) + else: + raise NotImplementedError( + f"Unsupported storage backend: {storage_backend}" + ) + + self.enable_storage = True + # todo: threshold policy for prefetching + self.prefetch_threshold = max(prefetch_threshold, self.page_size) + self.prefetch_capacity_limit = int( + 0.8 * (self.mem_pool_host.size - self.mem_pool_device.size) + ) + # granularity of batch storage IO operations, in number of pages + self.storage_batch_size = 128 + # tracking the number of tokens locked in prefetching, updated by the main scheduler thread + self.prefetch_tokens_occupied = 0 + + # create a new communication group for synchronizing storage operations across TP workers + self.tp_world_size = torch.distributed.get_world_size(group=tp_group) + if self.tp_world_size > 1: + group_ranks = torch.distributed.get_process_group_ranks(tp_group) + self.prefetch_tp_group = torch.distributed.new_group( + group_ranks, backend="gloo" + ) + + # Select the get and set functions + self.page_get_func = self._generic_page_get + self.page_set_func = self._generic_page_set + self.batch_exists_func = self.storage_backend.batch_exists + self.is_3fs_zerocopy = ( + self.storage_backend_type == "hf3fs" + and self.mem_pool_host.layout == "page_first" + ) + if self.storage_backend_type == "mooncake": + self.page_get_func = self._mooncake_page_get + self.page_set_func = self._mooncake_page_set + elif self.is_3fs_zerocopy: + self.page_get_func = self._3fs_zero_copy_page_get + self.page_set_func = self._3fs_zero_copy_page_set + self.batch_exists_func = self._3fs_zero_copy_batch_exists + + self.device = self.mem_pool_device.device + self.layer_num = self.mem_pool_device.layer_num + self.layer_done_counter = LayerDoneCounter(self.layer_num) + self.mem_pool_device.register_layer_transfer_counter(self.layer_done_counter) + + if write_policy not in [ + "write_through", + "write_through_selective", + "write_back", + ]: + raise ValueError(f"Invalid write policy: {write_policy}") + + # self.write_queue = PriorityQueue[CacheOperation]() + self.load_queue: List[CacheOperation] = [] + self.write_queue: List[CacheOperation] = [] + self.ack_load_queue: List[HiCacheAck] = [] + self.ack_write_queue: List[HiCacheAck] = [] + + self.stop_event = threading.Event() + self.write_buffer = TransferBuffer(self.stop_event) + self.load_buffer = TransferBuffer( + self.stop_event, buffer_count=10, max_buffer_size=100 + ) + + self.write_stream = torch.cuda.Stream() + self.load_stream = torch.cuda.Stream() + + if self.enable_storage: + self.prefetch_thread = threading.Thread( + target=self.prefetch_thread_func, daemon=True + ) + self.backup_thread = threading.Thread( + target=self.backup_thread_func, daemon=True + ) + self.prefetch_queue = Queue() + self.backup_queue = Queue() + + self.prefetch_revoke_queue = Queue() + self.ack_backup_queue = Queue() + self.host_mem_release_queue = Queue() + + self.prefetch_thread.start() + self.backup_thread.start() + + def _generate_storage_config( + self, + model_name: Optional[str] = None, + storage_backend_extra_config: Optional[str] = None, + ): + + if is_dp_attention_enabled(): + self.tp_rank = get_attention_tp_rank() + self.tp_size = get_attention_tp_size() + self.dp_rank = get_attention_dp_rank() + else: + self.tp_rank = get_tensor_model_parallel_rank() + self.tp_size = get_tensor_model_parallel_world_size() + self.dp_rank = 0 + + # Currently, AscendMLAPagedTokenToKVPool is the subclass of MLATokenToKVPool. + is_mla_backend = isinstance(self.mem_pool_device, MLATokenToKVPool) + + # Parse extra config JSON if provided + extra_config = None + if storage_backend_extra_config: + try: + import json + + extra_config = json.loads(storage_backend_extra_config) + except Exception as e: + logger.error(f"Invalid backend extra config JSON: {e}") + + return HiCacheStorageConfig( + tp_rank=self.tp_rank, + tp_size=self.tp_size, + is_mla_model=is_mla_backend, + is_page_first_layout=self.mem_pool_host.layout == "page_first", + model_name=model_name, + extra_config=extra_config, + ) + + def reset(self): + self.stop_event.set() + + self.write_queue.clear() + self.load_queue.clear() + self.write_buffer.clear() + self.load_buffer.clear() + self.ack_write_queue.clear() + self.ack_load_queue.clear() + if self.enable_storage: + self.prefetch_thread.join() + self.backup_thread.join() + self.prefetch_queue.queue.clear() + self.backup_queue.queue.clear() + self.prefetch_revoke_queue.queue.clear() + self.ack_backup_queue.queue.clear() + + self.stop_event.clear() + + if self.enable_storage: + self.prefetch_thread = threading.Thread( + target=self.prefetch_thread_func, daemon=True + ) + self.backup_thread = threading.Thread( + target=self.backup_thread_func, daemon=True + ) + self.prefetch_thread.start() + self.backup_thread.start() + + def write( + self, + device_indices: torch.Tensor, + priority: Optional[int] = None, + node_id: int = -1, + ) -> Optional[torch.Tensor]: + """ + Back up KV caches from device memory to host memory. + """ + host_indices = self.mem_pool_host.alloc(len(device_indices)) + if host_indices is None: + return None + self.mem_pool_host.protect_write(host_indices) + self.write_queue.append( + CacheOperation(host_indices, device_indices, node_id, priority) + ) + self.start_writing() + return host_indices + + def start_writing(self) -> None: + if len(self.write_queue) == 0: + return + + op = CacheOperation.merge_ops(self.write_queue) + host_indices, device_indices = self.move_indices(op) + self.write_queue.clear() + + start_event = torch.cuda.Event() + finish_event = torch.cuda.Event() + + start_event.record() + with torch.cuda.stream(self.write_stream): + start_event.wait(self.write_stream) + self.mem_pool_host.backup_from_device_all_layer( + self.mem_pool_device, host_indices, device_indices, self.io_backend + ) + self.mem_pool_host.complete_io(op.host_indices) + finish_event.record() + # NOTE: We must save the host indices and device indices here, + # this is because we need to guarantee that these tensors are + # still alive when the write stream is executing. + if host_indices.is_cuda: + host_indices.record_stream(self.write_stream) + if device_indices.is_cuda: + device_indices.record_stream(self.write_stream) + + self.ack_write_queue.append(HiCacheAck(start_event, finish_event, op.node_ids)) + + def load( + self, + host_indices: torch.Tensor, + priority: Optional[int] = None, + node_id: int = -1, + ) -> Optional[torch.Tensor]: + """ + Load KV caches from host memory to device memory. + """ + device_indices = self.mem_pool_device_allocator.alloc(len(host_indices)) + if device_indices is None: + return None + self.mem_pool_host.protect_load(host_indices) + self.load_queue.append( + CacheOperation(host_indices, device_indices, node_id, priority) + ) + return device_indices + + def move_indices(self, op: CacheOperation): + host_indices, device_indices = op.host_indices, op.device_indices + # move indices to GPU if using kernels, to host if using direct indexing + if self.io_backend == "kernel": + if not host_indices.is_cuda: + host_indices = host_indices.to(self.device, non_blocking=True) + return host_indices, device_indices + elif self.io_backend == "direct": + device_indices = device_indices.cpu() + host_indices, idx = host_indices.sort() + return host_indices, device_indices.index_select(0, idx) + else: + raise ValueError(f"Unsupported io backend") + + def start_loading(self) -> int: + if len(self.load_queue) == 0: + return -1 + + producer_id = self.layer_done_counter.update_producer() + op = CacheOperation.merge_ops(self.load_queue) + host_indices, device_indices = self.move_indices(op) + self.load_queue.clear() + producer_event = self.layer_done_counter.events[producer_id] + producer_event.start_event.record() + + with torch.cuda.stream(self.load_stream): + producer_event.start_event.wait(self.load_stream) + for i in range(self.layer_num): + self.mem_pool_host.load_to_device_per_layer( + self.mem_pool_device, + host_indices, + device_indices, + i, + self.io_backend, + ) + producer_event.complete(i) + self.mem_pool_host.complete_io(op.host_indices) + # NOTE: We must save the host indices and device indices here, + # this is because we need to guarantee that these tensors are + # still alive when the load stream is executing. + if host_indices.is_cuda: + host_indices.record_stream(self.load_stream) + if device_indices.is_cuda: + device_indices.record_stream(self.load_stream) + + self.ack_load_queue.append( + HiCacheAck( + start_event=producer_event.start_event, + finish_event=producer_event.finish_event, + node_ids=op.node_ids, + ) + ) + return producer_id + + def evict_device( + self, device_indices: torch.Tensor, host_indices: torch.Tensor + ) -> int: + if self.mem_pool_host.is_synced(host_indices): + self.mem_pool_device_allocator.free(device_indices) + self.mem_pool_host.update_backup(host_indices) + return len(device_indices) + else: + raise ValueError( + f"Inconsistent states: {self.mem_pool_host.get_state(host_indices)}" + ) + + def evict_host(self, host_indices: torch.Tensor, backup_only: bool = True) -> int: + if not backup_only: + raise ValueError("Other eviction policies are not supported yet.") + + if self.mem_pool_host.is_backup(host_indices): + self.mem_pool_host.free(host_indices) + return len(host_indices) + else: + raise ValueError( + f"Inconsistent states: {self.mem_pool_host.get_state(host_indices)}" + ) + + def prefetch( + self, + request_id: str, + host_indices: torch.Tensor, + new_input_tokens: List[int], + last_hash: Optional[str] = None, + ) -> PrefetchOperation: + """ + Prefetch KV caches from storage backend to host memory. + """ + operation = PrefetchOperation( + request_id, host_indices, new_input_tokens, last_hash + ) + self.prefetch_queue.put(operation) + return operation + + def terminate_prefetch(self, operation): + operation.mark_terminate() + return operation.completed_tokens, operation.hash_value + + def append_host_mem_release(self, host_indices: torch.Tensor): + chunks = host_indices.split(self.mem_pool_host.page_size) + for chunk in chunks: + self.host_mem_release_queue.put(chunk) + + def _3fs_zero_copy_batch_exists(self, batch_hashes): + _batch_hashes, _, factor = self.mem_pool_host.get_buffer_with_hash(batch_hashes) + hit_page_num = self.storage_backend.batch_exists(_batch_hashes) // factor + return hit_page_num + + def _3fs_zero_copy_page_get(self, operation, hash_values, host_indices): + hashes, dsts, factor = self.mem_pool_host.get_buffer_with_hash( + hash_values, host_indices + ) + page_data = self.storage_backend.batch_get(hashes, dsts) + if page_data: + inc = self.page_size * len(hashes) // factor + operation.increment(inc) + else: + logger.warning( + f"Prefetch operation {operation.request_id} failed to retrieve page {hashes}." + ) + + def _mooncake_page_get(self, operation, hash_values, host_indices): + key_strs, buffer_ptrs, buffer_sizes = self.mem_pool_host.get_buffer_meta( + hash_values, + host_indices, + self.storage_config.tp_rank, + ) + get_result = self.storage_backend.batch_get( + key_strs, + target_locations=buffer_ptrs, + target_sizes=buffer_sizes, + ) + if get_result != len(hash_values): + logger.warning( + f"Prefetch operation {operation.request_id} failed or partially failed." + ) + if get_result != 0: + operation.increment(get_result * self.page_size) + + def _generic_page_get(self, operation, hash_values, host_indices): + dummy_page_dst = [ + self.mem_pool_host.get_dummy_flat_data_page() for _ in hash_values + ] + page_data = self.storage_backend.batch_get(hash_values, dummy_page_dst) + if page_data is None: + return + for i in range(len(hash_values)): + if page_data[i] is None: + logger.warning( + f"Prefetch operation {operation.request_id} failed to retrieve page {hash_values[i]}." + ) + break + # Must set the data before increasing the completed tokens. + # Otherwise this page may be read before being set. + self.mem_pool_host.set_from_flat_data_page( + host_indices[i * self.page_size], + page_data[i], + ) + if not operation.increment(self.page_size): + break # Operation terminated by controller + + def _page_transfer(self, operation): + # Transfer batch by batch + for i in range(0, len(operation.hash_value), self.storage_batch_size): + batch_hashes = operation.hash_value[i : i + self.storage_batch_size] + batch_host_indices = operation.host_indices[ + i * self.page_size : (i + len(batch_hashes)) * self.page_size + ] + prev_completed_tokens = operation.completed_tokens + # Get one batch token, and update the completed_tokens if succeed + self.page_get_func(operation, batch_hashes, batch_host_indices) + # Check termination + if ( + operation.completed_tokens + != prev_completed_tokens + len(batch_hashes) * self.page_size + ): + operation.mark_terminate() + break # Some operations fail or operation terminated by controller + # release pre-allocated memory + self.append_host_mem_release( + operation.host_indices[operation.completed_tokens :] + ) + + def prefetch_io_aux_func(self): + """ + Auxiliary function conducting IO operations for prefetching. + """ + while not self.stop_event.is_set(): + try: + operation = self.prefetch_buffer.get(block=True, timeout=1) + self._page_transfer(operation) + # operation terminated by controller, release pre-allocated memory + self.append_host_mem_release( + operation.host_indices[operation.completed_tokens :] + ) + except Empty: + continue + + def prefetch_rate_limited(self) -> bool: + """ + Rate limit the prefetching operations to avoid overwhelming the storage backend. + """ + # cancel prefetch if too much memory is occupied + if self.prefetch_tokens_occupied >= self.prefetch_capacity_limit: + return True + # todo: more sophisticated rate limiting based on storage backend performance + return False + + def _storage_hit_query(self, operation) -> tuple[list[str], int]: + last_hash = operation.last_hash + tokens_to_fetch = operation.token_ids + + storage_query_count = 0 + hash_value = [] + + for start in range( + 0, len(tokens_to_fetch), self.page_size * self.storage_batch_size + ): + end = min( + start + self.page_size * self.storage_batch_size, len(tokens_to_fetch) + ) + batch_tokens = tokens_to_fetch[start:end] + batch_hashes = [] + for i in range(0, len(batch_tokens), self.page_size): + last_hash = self.get_hash_str( + batch_tokens[i : i + self.page_size], last_hash + ) + batch_hashes.append(last_hash) + hit_page_num = self.batch_exists_func(batch_hashes) + hash_value.extend(batch_hashes[:hit_page_num]) + storage_query_count += hit_page_num * self.page_size + if hit_page_num < len(batch_hashes): + break + return hash_value, storage_query_count + + def prefetch_thread_func(self): + """ + Manage prefetching operations from storage backend to host memory. + """ + self.prefetch_buffer = Queue() + aux_thread = threading.Thread(target=self.prefetch_io_aux_func, daemon=True) + aux_thread.start() + while (not self.stop_event.is_set()) or not self.prefetch_queue.empty(): + try: + operation = self.prefetch_queue.get(block=True, timeout=1) + if operation is None: + continue + + hash_value, storage_hit_count = self._storage_hit_query(operation) + if self.tp_world_size > 1: + storage_hit_count_tensor = torch.tensor( + storage_hit_count, dtype=torch.int + ) + torch.distributed.all_reduce( + storage_hit_count_tensor, + op=torch.distributed.ReduceOp.MIN, + group=self.prefetch_tp_group, + ) + storage_hit_count = storage_hit_count_tensor.item() + + if storage_hit_count < self.prefetch_threshold: + # not to prefetch if not enough benefits + self.prefetch_revoke_queue.put(operation.request_id) + self.append_host_mem_release(operation.host_indices) + logger.debug( + f"Revoking prefetch for request {operation.request_id} due to insufficient hits ({storage_hit_count})." + ) + else: + operation.hash_value = hash_value[ + : (storage_hit_count // self.page_size) + ] + # free the pre-allocated memory for pages that are not hit + self.append_host_mem_release( + operation.host_indices[storage_hit_count:] + ) + operation.host_indices = operation.host_indices[:storage_hit_count] + logger.debug( + f"Prefetching {len(operation.hash_value)} pages for request {operation.request_id}." + ) + self.prefetch_buffer.put(operation) + + except Empty: + continue + + def write_storage( + self, + host_indices: torch.Tensor, + token_ids: List[int], + hash_value: Optional[List[str]] = None, + ) -> int: + """ + Write KV caches from host memory to storage backend. + """ + operation = StorageOperation(host_indices, token_ids, hash_value=hash_value) + self.backup_queue.put(operation) + return operation.id + + # non-zero copy + def _generic_page_set(self, hash_values, host_indices) -> bool: + data = [ + self.mem_pool_host.get_flat_data_page(host_indices[i * self.page_size]) + for i in range(len(hash_values)) + ] + return self.storage_backend.batch_set(hash_values, data) + + # zero copy + def _mooncake_page_set(self, hash_values, host_indices) -> bool: + key_strs, buffer_ptrs, buffer_sizes = self.mem_pool_host.get_buffer_meta( + hash_values, + host_indices, + self.storage_config.tp_rank, + ) + success = self.storage_backend.batch_set( + key_strs, + target_locations=buffer_ptrs, + target_sizes=buffer_sizes, + ) + return success + + # zero copy + def _3fs_zero_copy_page_set(self, hash_values, host_indices) -> bool: + hashes, dsts, _ = self.mem_pool_host.get_buffer_with_hash( + hash_values, host_indices + ) + return self.storage_backend.batch_set(hashes, dsts) + + # Backup batch by batch + def _page_backup(self, operation): + # Backup batch by batch + for i in range(0, len(operation.hash_value), self.storage_batch_size): + batch_hashes = operation.hash_value[i : i + self.storage_batch_size] + batch_host_indices = operation.host_indices[ + i * self.page_size : (i + len(batch_hashes)) * self.page_size + ] + # Set one batch token, and record if success. + # todo: allow partial success + success = self.page_set_func(batch_hashes, batch_host_indices) + if not success: + logger.warning( + f"Write page to storage: {len(batch_hashes)} pages failed." + ) + break + operation.completed_tokens += self.page_size * len(batch_hashes) + + def backup_thread_func(self): + """ + Manage backup operations from host memory to storage backend. + """ + while not self.stop_event.is_set(): + try: + operation = self.backup_queue.get(block=True, timeout=1) + if operation is None: + continue + + if not self.backup_skip: + self._page_backup(operation) + self.ack_backup_queue.put(operation) + + except Empty: + continue diff --git a/python/sglang/srt/managers/configure_logging.py b/python/sglang/srt/managers/configure_logging.py new file mode 100644 index 000000000..0dc78edfa --- /dev/null +++ b/python/sglang/srt/managers/configure_logging.py @@ -0,0 +1,47 @@ +""" +Copyright 2023-2025 SGLang Team +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +""" +Configure the logging settings of a server. + +Usage: +python3 -m sglang.srt.managers.configure_logging --url http://localhost:30000 +""" + +import argparse + +import requests + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--url", type=str, default="http://localhost:30000") + parser.add_argument("--log-requests", action="store_true") + parser.add_argument("--log-requests-level", type=int, default=3) + parser.add_argument( + "--dump-requests-folder", type=str, default="/tmp/sglang_request_dump" + ) + parser.add_argument("--dump-requests-threshold", type=int, default=1000) + args = parser.parse_args() + + response = requests.post( + args.url + "/configure_logging", + json={ + "log_requests": args.log_requests, + "log_requests_level": args.log_requests_level, # Log full requests + "dump_requests_folder": args.dump_requests_folder, + "dump_requests_threshold": args.dump_requests_threshold, + }, + ) + assert response.status_code == 200 diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py new file mode 100644 index 000000000..a7bb6d13a --- /dev/null +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -0,0 +1,390 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A controller that dispatches requests to multiple data parallel workers.""" + +import faulthandler +import logging +import multiprocessing as mp +import signal +import struct +import sys +import threading +import time +from enum import Enum, auto +from multiprocessing import shared_memory +from typing import Dict, List + +import psutil +import setproctitle +import zmq + +from sglang.srt.layers.dp_attention import compute_dp_attention_world_info +from sglang.srt.managers.io_struct import ( + BlockReqInput, + TokenizedEmbeddingReqInput, + TokenizedGenerateReqInput, +) +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.scheduler import run_scheduler_process +from sglang.srt.managers.utils import DPBalanceMeta +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.torch_memory_saver_adapter import TorchMemorySaverAdapter +from sglang.srt.utils import ( + bind_port, + configure_logger, + get_zmq_socket, + kill_itself_when_parent_died, +) +from sglang.utils import get_exception_traceback + +logger = logging.getLogger(__name__) + + +class LoadBalanceMethod(Enum): + """Load balance method.""" + + ROUND_ROBIN = auto() + SHORTEST_QUEUE = auto() + MINIMUM_TOKENS = auto() + + @classmethod + def from_str(cls, method: str): + method = method.upper() + try: + return cls[method] + except KeyError as exc: + raise ValueError(f"Invalid load balance method: {method}") from exc + + +class DataParallelController: + """A controller that dispatches requests to multiple data parallel workers.""" + + def __init__( + self, + server_args: ServerArgs, + port_args: PortArgs, + dp_balance_meta: DPBalanceMeta, + ) -> None: + # for dp balance + self.global_balance_id = 0 + self.balance_meta = dp_balance_meta + + # Parse args + self.max_total_num_tokens = None + self.server_args = server_args + self.port_args = port_args + self.load_balance_method = LoadBalanceMethod.from_str( + server_args.load_balance_method + ) + + # Init inter-process communication + self.context = zmq.Context(1 + server_args.dp_size) + if server_args.node_rank == 0: + self.recv_from_tokenizer = get_zmq_socket( + self.context, zmq.PULL, port_args.scheduler_input_ipc_name, False + ) + + # Dispatch method + self.round_robin_counter = 0 + dispatch_lookup = { + LoadBalanceMethod.ROUND_ROBIN: self.round_robin_scheduler, + LoadBalanceMethod.SHORTEST_QUEUE: self.shortest_queue_scheduler, + LoadBalanceMethod.MINIMUM_TOKENS: self.minimum_tokens_scheduler, + } + self.dispatching = dispatch_lookup[self.load_balance_method] + + # Launch data parallel workers + self.scheduler_procs = [] + self.workers: List[zmq.Socket] = [None] * server_args.dp_size + + if server_args.enable_dp_attention: + dp_port_args = self.launch_dp_attention_schedulers(server_args, port_args) + self.control_message_step = server_args.tp_size + else: + dp_port_args = self.launch_dp_schedulers(server_args, port_args) + self.control_message_step = 1 + + # Only node rank 0 runs the real data parallel controller that dispatches the requests. + if server_args.node_rank == 0: + for dp_rank in range(server_args.dp_size): + self.workers[dp_rank] = get_zmq_socket( + self.context, + zmq.PUSH, + dp_port_args[dp_rank].scheduler_input_ipc_name, + True, + ) + + self.max_req_input_len = None + + def launch_dp_schedulers(self, server_args, port_args): + base_gpu_id = 0 + + threads = [] + sockets = [] + dp_port_args = [] + ready_events = [] + for dp_rank in range(server_args.dp_size): + tmp_port_args = PortArgs.init_new(server_args) + tmp_port_args.tokenizer_ipc_name = port_args.tokenizer_ipc_name + tmp_port_args.detokenizer_ipc_name = port_args.detokenizer_ipc_name + dp_port_args.append(tmp_port_args) + + # This port is checked free in PortArgs.init_new. + # We hold it first so that the next dp worker gets a different port + sockets.append(bind_port(tmp_port_args.nccl_port)) + + ready_event = threading.Event() + ready_events.append(ready_event) + + # Create a thread for each worker + thread = threading.Thread( + target=self.launch_tensor_parallel_group_thread, + args=(server_args, tmp_port_args, base_gpu_id, dp_rank, ready_event), + ) + threads.append(thread) + base_gpu_id += server_args.tp_size * server_args.gpu_id_step + + # Free all sockets before starting the threads to launch TP workers + for sock in sockets: + sock.close() + + # Start all threads + for thread in threads: + thread.start() + for event in ready_events: + event.wait() + + return dp_port_args + + def launch_tensor_parallel_group_thread( + self, + server_args: ServerArgs, + port_args: PortArgs, + base_gpu_id: int, + dp_rank: int, + ready_event: threading.Event, + ): + self.launch_tensor_parallel_group(server_args, port_args, base_gpu_id, dp_rank) + ready_event.set() + + # This thread cannot be closed because otherwise the `kill_itself_when_parent_died` + # function in scheduler.py will kill the scheduler. + while True: + time.sleep(30 * 24 * 3600) + + def launch_dp_attention_schedulers(self, server_args, port_args): + self.launch_tensor_parallel_group(server_args, port_args, 0, None) + dp_port_args = [] + for dp_rank in range(server_args.dp_size): + dp_port_args.append(PortArgs.init_new(server_args, dp_rank)) + return dp_port_args + + def launch_tensor_parallel_group( + self, + server_args: ServerArgs, + port_args: PortArgs, + base_gpu_id: int, + dp_rank: int, + ): + if not server_args.enable_dp_attention: + logger.info(f"Launch DP{dp_rank} starting at GPU #{base_gpu_id}.") + + memory_saver_adapter = TorchMemorySaverAdapter.create( + enable=server_args.enable_memory_saver + ) + + scheduler_pipe_readers = [] + + nnodes_per_tp_group = max(server_args.nnodes // server_args.pp_size, 1) + tp_size_per_node = server_args.tp_size // nnodes_per_tp_group + tp_rank_range = range( + tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group), + tp_size_per_node * (server_args.node_rank % nnodes_per_tp_group + 1), + ) + + pp_size_per_node = max(server_args.pp_size // server_args.nnodes, 1) + pp_rank_range = range( + pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group), + pp_size_per_node * (server_args.node_rank // nnodes_per_tp_group + 1), + ) + + for pp_rank in pp_rank_range: + for tp_rank in tp_rank_range: + rank_port_args = port_args + + if server_args.enable_dp_attention: + # dp attention has different sharding logic + _, _, dp_rank = compute_dp_attention_world_info( + server_args.enable_dp_attention, + tp_rank, + server_args.tp_size, + server_args.dp_size, + ) + # compute zmq ports for this dp rank + rank_port_args = PortArgs.init_new(server_args, dp_rank) + # Data parallelism reuses the tensor parallelism group, + # so all dp ranks should use the same nccl port. + rank_port_args.nccl_port = port_args.nccl_port + + reader, writer = mp.Pipe(duplex=False) + gpu_id = ( + server_args.base_gpu_id + + base_gpu_id + + ((pp_rank % pp_size_per_node) * tp_size_per_node) + + (tp_rank % tp_size_per_node) * server_args.gpu_id_step + ) + moe_ep_rank = tp_rank // (server_args.tp_size // server_args.ep_size) + proc = mp.Process( + target=run_scheduler_process, + args=( + server_args, + rank_port_args, + gpu_id, + tp_rank, + moe_ep_rank, + pp_rank, + dp_rank, + writer, + self.balance_meta, + ), + ) + with memory_saver_adapter.configure_subprocess(): + proc.start() + self.scheduler_procs.append(proc) + scheduler_pipe_readers.append(reader) + + # Wait for model to finish loading + scheduler_info = [] + for i in range(len(scheduler_pipe_readers)): + scheduler_info.append(scheduler_pipe_readers[i].recv()) + + self.max_total_num_tokens = scheduler_info[0]["max_total_num_tokens"] + self.max_req_input_len = scheduler_info[0]["max_req_input_len"] + + def maybe_external_dp_rank_routing(self, req: Req): + if req.data_parallel_rank is not None: + logger.debug(f"Direct routing to DP rank {req.data_parallel_rank}") + self.workers[req.data_parallel_rank].send_pyobj(req) + return True + return False + + def round_robin_scheduler(self, req: Req): + if self.maybe_external_dp_rank_routing(req): + return + + if self.server_args.disaggregation_mode == "null": + self.workers[self.round_robin_counter].send_pyobj(req) + self.round_robin_counter = (self.round_robin_counter + 1) % len( + self.workers + ) + else: + self.workers[req.bootstrap_room % len(self.workers)].send_pyobj(req) + + def shortest_queue_scheduler(self, input_requests): + if self.maybe_external_dp_rank_routing(req): + return + raise NotImplementedError() + + def minimum_tokens_scheduler(self, req): + if self.maybe_external_dp_rank_routing(req): + return + + # This variable corresponds to the balance_id in TokenizedGenerateReqInput. + # We use it to to control the number of onfly tokens (requests dispatched to workers but not yet received). + def get_next_global_balance_id() -> int: + INT32_MAX = 2147483647 + current_id = self.global_balance_id + self.global_balance_id = (self.global_balance_id + 1) % INT32_MAX + return current_id + + req.dp_balance_id = get_next_global_balance_id() + with self.balance_meta.mutex: + # 1. local_tokens represents the tokens currently inferring on the worker, + # while onfly refers to the requests dispatched by the dispatcher but not yet received by the scheduler. + onfly_info = self.balance_meta.get_shared_onfly() + local_tokens = self.balance_meta.get_shared_local_tokens() + total_tokens = [ + local_token + sum(onfly_dict.values()) + for local_token, onfly_dict in zip(local_tokens, onfly_info) + ] + target_worker = total_tokens.index(min(total_tokens)) + onfly_info[target_worker][req.dp_balance_id] = len(req.input_ids) + # 2. write the new onfly info to the shm + self.balance_meta.set_shared_onfly_info(onfly_info) + + # logger.info(f"dp workers {local_tokens=}, {onfly_info=}, {target_worker=}") + self.workers[target_worker].send_pyobj(req) + + def event_loop(self): + while True: + while True: + try: + recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK) + except zmq.ZMQError: + break + + if isinstance( + recv_req, + ( + TokenizedGenerateReqInput, + TokenizedEmbeddingReqInput, + ), + ): + self.dispatching(recv_req) + elif isinstance(recv_req, BlockReqInput): + for worker in self.workers: + worker.send_pyobj(recv_req) + else: + # Send other control messages to first worker of tp group + for worker in self.workers[:: self.control_message_step]: + worker.send_pyobj(recv_req) + + +def run_data_parallel_controller_process( + server_args: ServerArgs, + port_args: PortArgs, + pipe_writer, +): + kill_itself_when_parent_died() + setproctitle.setproctitle("sglang::data_parallel_controller") + faulthandler.enable() + configure_logger(server_args) + parent_process = psutil.Process().parent() + balance_meta = DPBalanceMeta(server_args.dp_size) + + try: + controller = DataParallelController( + server_args, port_args, dp_balance_meta=balance_meta + ) + pipe_writer.send( + { + "status": "ready", + "max_total_num_tokens": controller.max_total_num_tokens, + "max_req_input_len": controller.max_req_input_len, + } + ) + if server_args.node_rank == 0: + controller.event_loop() + for proc in controller.scheduler_procs: + proc.join() + logger.error( + f"Scheduler or DataParallelController {proc.pid} terminated with {proc.exitcode}" + ) + except Exception: + traceback = get_exception_traceback() + logger.error(f"DataParallelController hit an exception: {traceback}") + parent_process.send_signal(signal.SIGQUIT) + finally: + # we need to destruct mp.Manager() in balance_meta + balance_meta.destructor() diff --git a/python/sglang/srt/managers/detokenizer_manager.py b/python/sglang/srt/managers/detokenizer_manager.py new file mode 100644 index 000000000..bc58f4ee5 --- /dev/null +++ b/python/sglang/srt/managers/detokenizer_manager.py @@ -0,0 +1,303 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""DetokenizerManager is a process that detokenizes the token ids.""" + +import dataclasses +import logging +import os +import signal +from collections import OrderedDict +from typing import Dict, List, Union + +import psutil +import setproctitle +import zmq + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.managers.io_struct import ( + BatchEmbeddingOut, + BatchMultimodalDecodeReq, + BatchMultimodalOut, + BatchStrOut, + BatchTokenIDOut, + FreezeGCReq, + MultiTokenizerRegisterReq, +) +from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.utils import ( + configure_logger, + freeze_gc, + get_zmq_socket, + kill_itself_when_parent_died, +) +from sglang.utils import ( + TypeBasedDispatcher, + find_printable_text, + get_exception_traceback, +) + +logger = logging.getLogger(__name__) + +# Maximum number of request states that detokenizer can hold. When exceeded, +# oldest request states will be evicted. Default: 65536 (1<<16). +# For more details, see: https://github.com/sgl-project/sglang/issues/2812 +# Use power of 2 values for better memory allocation. +DETOKENIZER_MAX_STATES = int(os.environ.get("SGLANG_DETOKENIZER_MAX_STATES", 1 << 16)) + + +@dataclasses.dataclass +class DecodeStatus: + """Store the status of incremental decoding.""" + + decoded_text: str + decode_ids: List[int] + surr_offset: int + read_offset: int + # Offset that's sent to tokenizer for incremental update. + sent_offset: int = 0 + + +class DetokenizerManager(MultiHttpWorkerDetokenizerMixin): + """DetokenizerManager is a process that detokenizes the token ids.""" + + def __init__( + self, + server_args: ServerArgs, + port_args: PortArgs, + ): + # Init inter-process communication + context = zmq.Context(2) + self.recv_from_scheduler = get_zmq_socket( + context, zmq.PULL, port_args.detokenizer_ipc_name, True + ) + self.send_to_tokenizer = get_zmq_socket( + context, zmq.PUSH, port_args.tokenizer_ipc_name, False + ) + + if server_args.skip_tokenizer_init: + self.tokenizer = None + else: + self.tokenizer = get_tokenizer( + server_args.tokenizer_path, + tokenizer_mode=server_args.tokenizer_mode, + trust_remote_code=server_args.trust_remote_code, + revision=server_args.revision, + ) + + self.decode_status = LimitedCapacityDict(capacity=DETOKENIZER_MAX_STATES) + self.is_dummy = server_args.load_format == "dummy" + + self._request_dispatcher = TypeBasedDispatcher( + [ + (BatchEmbeddingOut, self.handle_batch_embedding_out), + (BatchTokenIDOut, self.handle_batch_token_id_out), + (BatchMultimodalDecodeReq, self.handle_multimodal_decode_req), + (MultiTokenizerRegisterReq, lambda x: x), + (FreezeGCReq, self.handle_freeze_gc_req), + ] + ) + + self.is_tool_call_parser_gpt_oss = server_args.tool_call_parser == "gpt-oss" + + def event_loop(self): + """The event loop that handles requests""" + while True: + recv_obj = self.recv_from_scheduler.recv_pyobj() + output = self._request_dispatcher(recv_obj) + if output is not None: + self.send_to_tokenizer.send_pyobj(output) + + def trim_matched_stop( + self, output: Union[str, List[int]], finished_reason: Dict, no_stop_trim: bool + ): + if no_stop_trim or not finished_reason: + return output + + matched = finished_reason.get("matched", None) + if not matched: + return output + + # TODO(lmzheng): handle the case where multiple stop strs are hit + + # Trim stop str. + if isinstance(matched, str) and isinstance(output, str): + pos = output.find(matched) + return output[:pos] if pos != -1 else output + + # Trim stop token. + if isinstance(matched, int) and isinstance(output, list): + # 200012 <|call|> is the tool call token and one of eos tokens for gpt-oss model + if output[-1] == 200012 and self.is_tool_call_parser_gpt_oss: + return output + assert len(output) > 0 + return output[:-1] + return output + + def handle_batch_embedding_out(self, recv_obj: BatchEmbeddingOut): + # If it is embedding model, no detokenization is needed. + return recv_obj + + def handle_batch_token_id_out(self, recv_obj: BatchTokenIDOut): + bs = len(recv_obj.rids) + + # Initialize decode status + read_ids, surr_ids = [], [] + for i in range(bs): + rid = recv_obj.rids[i] + if rid not in self.decode_status: + s = DecodeStatus( + decoded_text=recv_obj.decoded_texts[i], + decode_ids=recv_obj.decode_ids[i], + surr_offset=0, + read_offset=recv_obj.read_offsets[i], + ) + self.decode_status[rid] = s + else: + s = self.decode_status[rid] + s.decode_ids.extend(recv_obj.decode_ids[i]) + + read_ids.append( + self.trim_matched_stop( + s.decode_ids[s.surr_offset :], + recv_obj.finished_reasons[i], + recv_obj.no_stop_trim[i], + ) + ) + surr_ids.append(s.decode_ids[s.surr_offset : s.read_offset]) + + # TODO(lmzheng): handle skip_special_tokens/spaces_between_special_tokens per request + surr_texts = self.tokenizer.batch_decode( + surr_ids, + skip_special_tokens=recv_obj.skip_special_tokens[0], + spaces_between_special_tokens=recv_obj.spaces_between_special_tokens[0], + ) + read_texts = self.tokenizer.batch_decode( + read_ids, + skip_special_tokens=recv_obj.skip_special_tokens[0], + spaces_between_special_tokens=recv_obj.spaces_between_special_tokens[0], + ) + + # Incremental decoding + output_strs = [] + for i in range(bs): + try: + s = self.decode_status[recv_obj.rids[i]] + except KeyError: + raise RuntimeError( + f"Decode status not found for request {recv_obj.rids[i]}. " + "It may be due to the request being evicted from the decode status due to memory pressure. " + "Please increase the maximum number of requests by setting " + "the SGLANG_DETOKENIZER_MAX_STATES environment variable to a bigger value than the default value. " + f"The current value is {DETOKENIZER_MAX_STATES}. " + "For more details, see: https://github.com/sgl-project/sglang/issues/2812" + ) + new_text = read_texts[i][len(surr_texts[i]) :] + if recv_obj.finished_reasons[i] is None: + # Streaming chunk: update the decode status + if len(new_text) > 0 and not new_text.endswith("�"): + s.decoded_text = s.decoded_text + new_text + s.surr_offset = s.read_offset + s.read_offset = len(s.decode_ids) + new_text = "" + else: + new_text = find_printable_text(new_text) + + output_str = self.trim_matched_stop( + s.decoded_text + new_text, + recv_obj.finished_reasons[i], + recv_obj.no_stop_trim[i], + ) + # Incrementally send text. + incremental_output = output_str[s.sent_offset :] + s.sent_offset = len(output_str) + output_strs.append(incremental_output) + + return BatchStrOut( + rids=recv_obj.rids, + finished_reasons=recv_obj.finished_reasons, + output_strs=output_strs, + output_ids=recv_obj.decode_ids, + prompt_tokens=recv_obj.prompt_tokens, + completion_tokens=recv_obj.completion_tokens, + cached_tokens=recv_obj.cached_tokens, + spec_verify_ct=recv_obj.spec_verify_ct, + input_token_logprobs_val=recv_obj.input_token_logprobs_val, + input_token_logprobs_idx=recv_obj.input_token_logprobs_idx, + output_token_logprobs_val=recv_obj.output_token_logprobs_val, + output_token_logprobs_idx=recv_obj.output_token_logprobs_idx, + input_top_logprobs_val=recv_obj.input_top_logprobs_val, + input_top_logprobs_idx=recv_obj.input_top_logprobs_idx, + output_top_logprobs_val=recv_obj.output_top_logprobs_val, + output_top_logprobs_idx=recv_obj.output_top_logprobs_idx, + input_token_ids_logprobs_val=recv_obj.input_token_ids_logprobs_val, + input_token_ids_logprobs_idx=recv_obj.input_token_ids_logprobs_idx, + output_token_ids_logprobs_val=recv_obj.output_token_ids_logprobs_val, + output_token_ids_logprobs_idx=recv_obj.output_token_ids_logprobs_idx, + output_hidden_states=recv_obj.output_hidden_states, + placeholder_tokens_idx=None, + placeholder_tokens_val=None, + ) + + def handle_multimodal_decode_req(self, recv_obj: BatchMultimodalDecodeReq): + outputs = self.tokenizer.detokenize(recv_obj) + return BatchMultimodalOut( + rids=recv_obj.rids, + finished_reasons=recv_obj.finished_reasons, + outputs=outputs, + prompt_tokens=recv_obj.prompt_tokens, + completion_tokens=recv_obj.completion_tokens, + cached_tokens=recv_obj.cached_tokens, + placeholder_tokens_idx=None, + placeholder_tokens_val=None, + ) + + def handle_freeze_gc_req(self, recv_req: FreezeGCReq): + freeze_gc("Detokenizer Manager") + return None + + +class LimitedCapacityDict(OrderedDict): + def __init__(self, capacity: int, *args, **kwargs): + super().__init__(*args, **kwargs) + self.capacity = capacity + + def __setitem__(self, key, value): + if len(self) >= self.capacity: + # Remove the oldest element (first item in the dict) + self.popitem(last=False) + # Set the new item + super().__setitem__(key, value) + + +def run_detokenizer_process( + server_args: ServerArgs, + port_args: PortArgs, +): + kill_itself_when_parent_died() + setproctitle.setproctitle("sglang::detokenizer") + configure_logger(server_args) + parent_process = psutil.Process().parent() + + try: + manager = DetokenizerManager(server_args, port_args) + if server_args.tokenizer_worker_num > 1: + manager.multi_http_worker_event_loop() + else: + manager.event_loop() + except Exception: + manager.socket_mapping.clear_all_sockets() + traceback = get_exception_traceback() + logger.error(f"DetokenizerManager hit an exception: {traceback}") + parent_process.send_signal(signal.SIGQUIT) diff --git a/python/sglang/srt/managers/disagg_service.py b/python/sglang/srt/managers/disagg_service.py new file mode 100644 index 000000000..df0eac48b --- /dev/null +++ b/python/sglang/srt/managers/disagg_service.py @@ -0,0 +1,46 @@ +"""Start bootstrap/kv-store-related server""" + +import os +from typing import Type + +from sglang.srt.disaggregation.base import BaseKVBootstrapServer +from sglang.srt.disaggregation.utils import ( + DisaggregationMode, + KVClassType, + TransferBackend, + get_kv_class, +) +from sglang.srt.server_args import ServerArgs + + +def start_disagg_service( + server_args: ServerArgs, +): + # Start kv boostrap server on prefill + disagg_mode = DisaggregationMode(server_args.disaggregation_mode) + transfer_backend = TransferBackend(server_args.disaggregation_transfer_backend) + + if disagg_mode == DisaggregationMode.PREFILL: + # only start bootstrap server on prefill tm + kv_bootstrap_server_class: Type[BaseKVBootstrapServer] = get_kv_class( + transfer_backend, KVClassType.BOOTSTRAP_SERVER + ) + bootstrap_server: BaseKVBootstrapServer = kv_bootstrap_server_class( + host=server_args.host, + port=server_args.disaggregation_bootstrap_port, + ) + is_create_store = ( + server_args.node_rank == 0 and transfer_backend == TransferBackend.ASCEND + ) + if is_create_store: + try: + from mf_adapter import create_config_store + + ascend_url = os.getenv("ASCEND_MF_STORE_URL") + create_config_store(ascend_url) + except Exception as e: + error_message = f"Failed create mf store, invalid ascend_url." + error_message += f" With exception {e}" + raise error_message + + return bootstrap_server diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py new file mode 100644 index 000000000..6237cd383 --- /dev/null +++ b/python/sglang/srt/managers/io_struct.py @@ -0,0 +1,1332 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The definition of objects transferred between different +processes (TokenizerManager, DetokenizerManager, Scheduler). +""" + +import copy +import uuid +from dataclasses import dataclass, field +from enum import Enum +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from sglang.srt.lora.lora_registry import LoRARef +from sglang.srt.managers.schedule_batch import BaseFinishReason +from sglang.srt.multimodal.mm_utils import has_valid_data +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.utils import ImageData + +# Handle serialization of Image for pydantic +if TYPE_CHECKING: + from PIL.Image import Image +else: + Image = Any + + +@dataclass +class SessionParams: + id: Optional[str] = None + rid: Optional[str] = None + offset: Optional[int] = None + replace: Optional[bool] = None + drop_previous_output: Optional[bool] = None + + +# Type definitions for multimodal input data +# Individual data item types for each modality +ImageDataInputItem = Union[Image, str, ImageData, Dict] +AudioDataInputItem = Union[str, Dict] +VideoDataInputItem = Union[str, Dict] +# Union type for any multimodal data item +MultimodalDataInputItem = Union[ + ImageDataInputItem, VideoDataInputItem, AudioDataInputItem +] +# Format types supporting single items, lists, or nested lists for batch processing +MultimodalDataInputFormat = Union[ + List[List[MultimodalDataInputItem]], + List[MultimodalDataInputItem], + MultimodalDataInputItem, +] + + +@dataclass +class GenerateReqInput: + # The input prompt. It can be a single prompt or a batch of prompts. + text: Optional[Union[List[str], str]] = None + # The token ids for text; one can specify either text or input_ids + input_ids: Optional[Union[List[List[int]], List[int]]] = None + # The embeddings for input_ids; one can specify either text or input_ids or input_embeds. + input_embeds: Optional[Union[List[List[List[float]]], List[List[float]]]] = None + # The image input. It can be an image instance, file name, URL, or base64 encoded string. + # Can be formatted as: + # - Single image for a single request + # - List of images (one per request in a batch) + # - List of lists of images (multiple images per request) + # See also python/sglang/srt/utils.py:load_image for more details. + image_data: Optional[MultimodalDataInputFormat] = None + # The video input. Like image data, it can be a file name, a url, or base64 encoded string. + video_data: Optional[MultimodalDataInputFormat] = None + # The audio input. Like image data, it can be a file name, a url, or base64 encoded string. + audio_data: Optional[MultimodalDataInputFormat] = None + # The sampling_params. See descriptions below. + sampling_params: Optional[Union[List[Dict], Dict]] = None + # The request id. + rid: Optional[Union[List[str], str]] = None + # Whether to return logprobs. + return_logprob: Optional[Union[List[bool], bool]] = None + # If return logprobs, the start location in the prompt for returning logprobs. + # By default, this value is "-1", which means it will only return logprobs for output tokens. + logprob_start_len: Optional[Union[List[int], int]] = None + # If return logprobs, the number of top logprobs to return at each position. + top_logprobs_num: Optional[Union[List[int], int]] = None + # If return logprobs, the token ids to return logprob for. + token_ids_logprob: Optional[Union[List[List[int]], List[int]]] = None + # Whether to detokenize tokens in text in the returned logprobs. + return_text_in_logprobs: bool = False + # Whether to stream output. + stream: bool = False + # Whether to log metrics for this request (e.g. health_generate calls do not log metrics) + log_metrics: bool = True + # Whether to return hidden states + return_hidden_states: Union[List[bool], bool] = False + + # The modalities of the image data [image, multi-images, video] + modalities: Optional[List[str]] = None + # Session info for continual prompting + session_params: Optional[Union[List[Dict], Dict]] = None + + # The path to the LoRA adaptors + lora_path: Optional[Union[List[Optional[str]], Optional[str]]] = None + # The uid of LoRA adaptors, should be initialized by tokenizer manager + lora_id: Optional[Union[List[Optional[str]], Optional[str]]] = None + + # Custom logit processor for advanced sampling control. Must be a serialized instance + # of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py + # Use the processor's `to_str()` method to generate the serialized string. + custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None + + # For disaggregated inference + bootstrap_host: Optional[Union[List[str], str]] = None + bootstrap_port: Optional[Union[List[Optional[int]], int]] = None + bootstrap_room: Optional[Union[List[int], int]] = None + bootstrap_pair_key: Optional[Union[List[str], str]] = None + + # For data parallel rank routing + data_parallel_rank: Optional[int] = None + + # For background responses (OpenAI responses API) + background: bool = False + + # Conversation id used for tracking requests + conversation_id: Optional[str] = None + + # Label for the request + label: Optional[str] = None + + # Priority for the request + priority: Optional[int] = None + + # Image gen grpc migration + return_bytes: bool = False + + def contains_mm_input(self) -> bool: + return ( + has_valid_data(self.image_data) + or has_valid_data(self.video_data) + or has_valid_data(self.audio_data) + ) + + def normalize_batch_and_arguments(self): + """ + Normalize the batch size and arguments for the request. + + This method resolves various input formats and ensures all parameters + are properly formatted as either single values or batches depending on the input. + It also handles parallel sampling expansion and sets default values for + unspecified parameters. + + Raises: + ValueError: If inputs are not properly specified (e.g., none or all of + text, input_ids, input_embeds are provided) + """ + self._validate_inputs() + self._determine_batch_size() + self._handle_parallel_sampling() + + if self.is_single: + self._normalize_single_inputs() + else: + self._normalize_batch_inputs() + + def _validate_inputs(self): + """Validate that the input configuration is valid.""" + if ( + self.text is None and self.input_ids is None and self.input_embeds is None + ) or ( + self.text is not None + and self.input_ids is not None + and self.input_embeds is not None + ): + raise ValueError( + "Either text, input_ids or input_embeds should be provided." + ) + + def _determine_batch_size(self): + """Determine if this is a single example or a batch and the batch size.""" + if self.text is not None: + if isinstance(self.text, str): + self.is_single = True + self.batch_size = 1 + else: + self.is_single = False + self.batch_size = len(self.text) + self.input_embeds = None + elif self.input_ids is not None: + if len(self.input_ids) == 0: + raise ValueError("input_ids cannot be empty.") + if isinstance(self.input_ids[0], int): + self.is_single = True + self.batch_size = 1 + else: + self.is_single = False + self.batch_size = len(self.input_ids) + self.input_embeds = None + else: + if isinstance(self.input_embeds[0][0], float): + self.is_single = True + self.batch_size = 1 + else: + self.is_single = False + self.batch_size = len(self.input_embeds) + + def _handle_parallel_sampling(self): + """Handle parallel sampling parameters and adjust batch size if needed.""" + # Determine parallel sample count + if self.sampling_params is None: + self.parallel_sample_num = 1 + return + elif isinstance(self.sampling_params, dict): + self.parallel_sample_num = self.sampling_params.get("n", 1) + else: # isinstance(self.sampling_params, list): + self.parallel_sample_num = self.sampling_params[0].get("n", 1) + for sampling_params in self.sampling_params: + if self.parallel_sample_num != sampling_params.get("n", 1): + raise ValueError( + "The parallel_sample_num should be the same for all samples in sample params." + ) + + # If using parallel sampling with a single example, convert to batch + if self.parallel_sample_num > 1 and self.is_single: + self.is_single = False + if self.text is not None: + self.text = [self.text] + if self.input_ids is not None: + self.input_ids = [self.input_ids] + if self.input_embeds is not None: + self.input_embeds = [self.input_embeds] + + def _normalize_single_inputs(self): + """Normalize inputs for a single example.""" + if self.sampling_params is None: + self.sampling_params = {} + if self.rid is None: + self.rid = uuid.uuid4().hex + if self.return_logprob is None: + self.return_logprob = False + if self.logprob_start_len is None: + self.logprob_start_len = -1 + if self.top_logprobs_num is None: + self.top_logprobs_num = 0 + if not self.token_ids_logprob: # covers both None and [] + self.token_ids_logprob = None + + def _normalize_batch_inputs(self): + """Normalize inputs for a batch of examples, including parallel sampling expansion.""" + # Calculate expanded batch size + if self.parallel_sample_num == 1: + num = self.batch_size + else: + # Expand parallel_sample_num + num = self.batch_size * self.parallel_sample_num + + # Expand input based on type + self._expand_inputs(num) + self._normalize_rid(num) + self._normalize_lora_paths(num) + self._normalize_image_data(num) + self._normalize_video_data(num) + self._normalize_audio_data(num) + self._normalize_sampling_params(num) + self._normalize_logprob_params(num) + self._normalize_custom_logit_processor(num) + self._normalize_bootstrap_params(num) + + def _expand_inputs(self, num): + """Expand the main inputs (text, input_ids, input_embeds) for parallel sampling.""" + if self.text is not None: + if not isinstance(self.text, list): + raise ValueError("Text should be a list for batch processing.") + self.text = self.text * self.parallel_sample_num + elif self.input_ids is not None: + if not isinstance(self.input_ids, list) or not isinstance( + self.input_ids[0], list + ): + raise ValueError( + "input_ids should be a list of lists for batch processing." + ) + self.input_ids = self.input_ids * self.parallel_sample_num + elif self.input_embeds is not None: + if not isinstance(self.input_embeds, list): + raise ValueError("input_embeds should be a list for batch processing.") + self.input_embeds = self.input_embeds * self.parallel_sample_num + + def _normalize_lora_paths(self, num): + """Normalize LoRA paths for batch processing.""" + if self.lora_path is not None: + if isinstance(self.lora_path, str): + self.lora_path = [self.lora_path] * num + elif isinstance(self.lora_path, list): + self.lora_path = self.lora_path * self.parallel_sample_num + else: + raise ValueError("lora_path should be a list or a string.") + + def _normalize_image_data(self, num): + """Normalize image data for batch processing.""" + if self.image_data is None: + self.image_data = [None] * num + elif not isinstance(self.image_data, list): + # Single image, convert to list of single-image lists + self.image_data = [[self.image_data]] * num + self.modalities = ["image"] * num + elif isinstance(self.image_data, list): + # Handle empty list case - treat as no images + if len(self.image_data) == 0: + self.image_data = [None] * num + return + + if len(self.image_data) != self.batch_size: + raise ValueError( + "The length of image_data should be equal to the batch size." + ) + + self.modalities = [] + if len(self.image_data) > 0 and isinstance(self.image_data[0], list): + # Already a list of lists, keep as is + for i in range(len(self.image_data)): + if self.image_data[i] is None or self.image_data[i] == [None]: + self.modalities.append(None) + elif len(self.image_data[i]) == 1: + self.modalities.append("image") + elif len(self.image_data[i]) > 1: + self.modalities.append("multi-images") + else: + # Ensure len(self.modalities) == len(self.image_data) + self.modalities.append(None) + # Expand parallel_sample_num + self.image_data = self.image_data * self.parallel_sample_num + self.modalities = self.modalities * self.parallel_sample_num + else: + # List of images for a batch, wrap each in a list + wrapped_images = [[img] for img in self.image_data] + # Expand for parallel sampling + self.image_data = wrapped_images * self.parallel_sample_num + self.modalities = ["image"] * num + + def _normalize_video_data(self, num): + """Normalize video data for batch processing.""" + if self.video_data is None: + self.video_data = [None] * num + elif not isinstance(self.video_data, list): + self.video_data = [self.video_data] * num + elif isinstance(self.video_data, list): + self.video_data = self.video_data * self.parallel_sample_num + + def _normalize_audio_data(self, num): + """Normalize audio data for batch processing.""" + if self.audio_data is None: + self.audio_data = [None] * num + elif not isinstance(self.audio_data, list): + self.audio_data = [self.audio_data] * num + elif isinstance(self.audio_data, list): + self.audio_data = self.audio_data * self.parallel_sample_num + + def _normalize_sampling_params(self, num): + """Normalize sampling parameters for batch processing.""" + if self.sampling_params is None: + self.sampling_params = [{}] * num + elif isinstance(self.sampling_params, dict): + self.sampling_params = [self.sampling_params] * num + else: # Already a list + self.sampling_params = self.sampling_params * self.parallel_sample_num + + def _normalize_rid(self, num): + """Normalize request IDs for batch processing.""" + if self.rid is None: + self.rid = [uuid.uuid4().hex for _ in range(num)] + elif isinstance(self.rid, str): + new_rids = [f"{self.rid}_{i}" for i in range(num)] + self.rid = new_rids + elif isinstance(self.rid, list): + # Note: the length of rid shall be the same as the batch_size, + # as the rid would be expanded for parallel sampling in tokenizer_manager + if len(self.rid) != self.batch_size: + raise ValueError( + "The specified rids length mismatch with the batch_size for batch processing." + ) + else: + raise ValueError("The rid should be a string or a list of strings.") + + def _normalize_logprob_params(self, num): + """Normalize logprob-related parameters for batch processing.""" + + # Helper function to normalize a parameter + def normalize_param(param, default_value, param_name): + if param is None: + return [default_value] * num + elif not isinstance(param, list): + return [param] * num + else: + if self.parallel_sample_num > 1: + raise ValueError( + f"Cannot use list {param_name} with parallel_sample_num > 1" + ) + return param + + # Normalize each logprob parameter + self.return_logprob = normalize_param( + self.return_logprob, False, "return_logprob" + ) + self.logprob_start_len = normalize_param( + self.logprob_start_len, -1, "logprob_start_len" + ) + self.top_logprobs_num = normalize_param( + self.top_logprobs_num, 0, "top_logprobs_num" + ) + + # Handle token_ids_logprob specially due to its nested structure + if not self.token_ids_logprob: # covers both None and [] + self.token_ids_logprob = [None] * num + elif not isinstance(self.token_ids_logprob, list): + self.token_ids_logprob = [[self.token_ids_logprob] for _ in range(num)] + elif not isinstance(self.token_ids_logprob[0], list): + self.token_ids_logprob = [ + copy.deepcopy(self.token_ids_logprob) for _ in range(num) + ] + elif self.parallel_sample_num > 1: + raise ValueError( + "Cannot use list token_ids_logprob with parallel_sample_num > 1" + ) + + def _normalize_custom_logit_processor(self, num): + """Normalize custom logit processor for batch processing.""" + if self.custom_logit_processor is None: + self.custom_logit_processor = [None] * num + elif not isinstance(self.custom_logit_processor, list): + self.custom_logit_processor = [self.custom_logit_processor] * num + elif self.parallel_sample_num > 1: + raise ValueError( + "Cannot use list custom_logit_processor with parallel_sample_num > 1" + ) + + def _normalize_bootstrap_params(self, num): + """Normalize bootstrap parameters for batch processing.""" + # Normalize bootstrap_host + if self.bootstrap_host is None: + self.bootstrap_host = [None] * num + elif not isinstance(self.bootstrap_host, list): + self.bootstrap_host = [self.bootstrap_host] * num + elif isinstance(self.bootstrap_host, list): + self.bootstrap_host = self.bootstrap_host * self.parallel_sample_num + + # Normalize bootstrap_port + if self.bootstrap_port is None: + self.bootstrap_port = [None] * num + elif not isinstance(self.bootstrap_port, list): + self.bootstrap_port = [self.bootstrap_port] * num + elif isinstance(self.bootstrap_port, list): + self.bootstrap_port = self.bootstrap_port * self.parallel_sample_num + + # Normalize bootstrap_room + if self.bootstrap_room is None: + self.bootstrap_room = [None] * num + elif not isinstance(self.bootstrap_room, list): + self.bootstrap_room = [self.bootstrap_room + i for i in range(num)] + elif isinstance(self.bootstrap_room, list): + self.bootstrap_room = self.bootstrap_room * self.parallel_sample_num + + # Normalize bootstrap_pair_key + if self.bootstrap_pair_key is None: + self.bootstrap_pair_key = [None] * num + elif not isinstance(self.bootstrap_pair_key, list): + self.bootstrap_pair_key = [self.bootstrap_pair_key] * num + elif isinstance(self.bootstrap_pair_key, list): + self.bootstrap_pair_key = self.bootstrap_pair_key * self.parallel_sample_num + + def _validate_session_params(self): + """Validate that session parameters are properly formatted.""" + if self.session_params is not None: + if not isinstance(self.session_params, dict) and not isinstance( + self.session_params[0], dict + ): + raise ValueError("Session params must be a dict or a list of dicts.") + + def regenerate_rid(self): + """Generate a new request ID and return it.""" + self.rid = uuid.uuid4().hex + return self.rid + + def __getitem__(self, i): + return GenerateReqInput( + text=self.text[i] if self.text is not None else None, + input_ids=self.input_ids[i] if self.input_ids is not None else None, + input_embeds=( + self.input_embeds[i] if self.input_embeds is not None else None + ), + image_data=self.image_data[i], + video_data=self.video_data[i], + audio_data=self.audio_data[i], + sampling_params=self.sampling_params[i], + rid=self.rid[i], + return_logprob=self.return_logprob[i], + logprob_start_len=self.logprob_start_len[i], + top_logprobs_num=self.top_logprobs_num[i], + token_ids_logprob=self.token_ids_logprob[i], + return_text_in_logprobs=self.return_text_in_logprobs, + stream=self.stream, + log_metrics=self.log_metrics, + return_hidden_states=( + self.return_hidden_states[i] + if isinstance(self.return_hidden_states, list) + else self.return_hidden_states + ), + modalities=self.modalities[i] if self.modalities else None, + session_params=self.session_params, + lora_path=self.lora_path[i] if self.lora_path is not None else None, + lora_id=self.lora_id[i] if self.lora_id is not None else None, + custom_logit_processor=( + self.custom_logit_processor[i] + if self.custom_logit_processor is not None + else None + ), + # if `__getitem__` is called, the bootstrap_host, bootstrap_port, bootstrap_room must be a list + bootstrap_host=( + self.bootstrap_host[i] if self.bootstrap_host is not None else None + ), + bootstrap_port=( + self.bootstrap_port[i] if self.bootstrap_port is not None else None + ), + bootstrap_room=( + self.bootstrap_room[i] if self.bootstrap_room is not None else None + ), + bootstrap_pair_key=( + self.bootstrap_pair_key[i] + if self.bootstrap_pair_key is not None + else None + ), + data_parallel_rank=( + self.data_parallel_rank if self.data_parallel_rank is not None else None + ), + conversation_id=self.conversation_id, + label=self.label, + priority=self.priority, + return_bytes=self.return_bytes, + ) + + +@dataclass +class TokenizedGenerateReqInput: + # The request id + rid: str + # The input text + input_text: str + # The input token ids + input_ids: List[int] + # The multimodal inputs + mm_inputs: dict + # The sampling parameters + sampling_params: SamplingParams + # Whether to return the logprobs + return_logprob: bool + # If return logprobs, the start location in the prompt for returning logprobs. + logprob_start_len: int + # If return logprobs, the number of top logprobs to return at each position. + top_logprobs_num: int + # If return logprobs, the token id to return logprob for + token_ids_logprob: List[int] + # Whether to stream output + stream: bool + # Whether to return hidden states + return_hidden_states: bool = False + + # The input embeds + input_embeds: Optional[Union[List[List[List[float]]], List[List[float]]]] = None + + # Session info for continual prompting + session_params: Optional[SessionParams] = None + + # LoRA related + lora_id: Optional[str] = None # None means just use the base model + + # Custom logit processor for advanced sampling control. Must be a serialized instance + # of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py + # Use the processor's `to_str()` method to generate the serialized string. + custom_logit_processor: Optional[str] = None + + # For disaggregated inference + bootstrap_host: Optional[str] = None + bootstrap_port: Optional[int] = None + bootstrap_room: Optional[int] = None + bootstrap_pair_key: Optional[str] = None + + # For data parallel rank routing + data_parallel_rank: Optional[int] = None + + # For dp balance + dp_balance_id: int = -1 + + # Label for the request + label: Optional[str] = None + + # Priority for the request + priority: Optional[int] = None + + # Image gen grpc migration + return_bytes: bool = False + + +@dataclass +class BatchTokenizedGenerateReqInput: + # The batch of tokenized requests + batch: List[TokenizedGenerateReqInput] + + def __len__(self): + return len(self.batch) + + def __getitem__(self, i): + return self.batch[i] + + def __iter__(self): + return iter(self.batch) + + +@dataclass +class EmbeddingReqInput: + # The input prompt. It can be a single prompt or a batch of prompts. + text: Optional[Union[List[List[str]], List[str], str]] = None + # The image input. It can be an image instance, file name, URL, or base64 encoded string. + # Can be formatted as: + # - Single image for a single request + # - List of images (one per request in a batch) + # - List of lists of images (multiple images per request) + # See also python/sglang/srt/utils.py:load_image for more details. + image_data: Optional[MultimodalDataInputFormat] = None + # The video input. Like image data, it can be a file name, a url, or base64 encoded string. + video_data: Optional[MultimodalDataInputFormat] = None + # The audio input. Like image data, it can be a file name, a url, or base64 encoded string. + audio_data: Optional[MultimodalDataInputFormat] = None + # The token ids for text; one can either specify text or input_ids. + input_ids: Optional[Union[List[List[int]], List[int]]] = None + # The request id. + rid: Optional[Union[List[str], str]] = None + # Dummy sampling params for compatibility + sampling_params: Optional[Union[List[Dict], Dict]] = None + # Dummy input embeds for compatibility + input_embeds: Optional[Union[List[List[List[float]]], List[List[float]]]] = None + # Whether to log metrics for this request (e.g. health_generate calls do not log metrics) + log_metrics: bool = True + # The modalities of the image data [image, multi-images, video] + modalities: Optional[List[str]] = None + # For cross-encoder requests + is_cross_encoder_request: bool = False + + # For background responses (OpenAI responses API) + background: bool = False + + def normalize_batch_and_arguments(self): + # at least one of text, input_ids, or image should be provided + if self.text is None and self.input_ids is None and self.image_data is None: + raise ValueError( + "At least one of text, input_ids, or image should be provided" + ) + + # text and input_ids cannot be provided at the same time + if self.text is not None and self.input_ids is not None: + raise ValueError("text and input_ids cannot be provided at the same time") + + # Derive the batch size + self.batch_size = 0 + self.is_single = True + + # check the batch size of text + if self.text is not None: + if isinstance(self.text, list): + self.batch_size += len(self.text) + self.is_single = False + else: + self.batch_size += 1 + + # check the batch size of input_ids + if self.input_ids is not None: + if isinstance(self.input_ids[0], list): + self.batch_size += len(self.input_ids) + self.is_single = False + else: + self.batch_size += 1 + + # Fill in default arguments + if self.is_single: + if self.rid is None: + self.rid = uuid.uuid4().hex + if self.sampling_params is None: + self.sampling_params = {} + self.sampling_params["max_new_tokens"] = 0 + else: + if self.rid is None: + self.rid = [uuid.uuid4().hex for _ in range(self.batch_size)] + else: + assert isinstance(self.rid, list), "The rid should be a list." + + if self.sampling_params is None: + self.sampling_params = [{}] * self.batch_size + elif isinstance(self.sampling_params, dict): + self.sampling_params = [self.sampling_params] * self.batch_size + for i in range(self.batch_size): + self.sampling_params[i]["max_new_tokens"] = 0 + + def regenerate_rid(self): + self.rid = uuid.uuid4().hex + return self.rid + + def contains_mm_input(self) -> bool: + return ( + has_valid_data(self.image_data) + or has_valid_data(self.video_data) + or has_valid_data(self.audio_data) + ) + + def __getitem__(self, i): + if self.is_cross_encoder_request: + return EmbeddingReqInput( + text=[self.text[i]] if self.text is not None else None, + sampling_params=self.sampling_params[i], + rid=self.rid[i], + is_cross_encoder_request=True, + ) + + return EmbeddingReqInput( + text=self.text[i] if self.text is not None else None, + input_ids=self.input_ids[i] if self.input_ids is not None else None, + image_data=self.image_data[i] if self.image_data is not None else None, + audio_data=self.audio_data[i] if self.audio_data is not None else None, + video_data=self.video_data[i] if self.video_data is not None else None, + sampling_params=self.sampling_params[i], + rid=self.rid[i], + ) + + +@dataclass +class TokenizedEmbeddingReqInput: + # The request id + rid: str + # The input text + input_text: str + # The input token ids + input_ids: List[int] + # The image inputs + image_inputs: dict + # The token type ids + token_type_ids: List[int] + # Dummy sampling params for compatibility + sampling_params: SamplingParams + # For data parallel rank routing + data_parallel_rank: Optional[int] = None + # For dp balance + dp_balance_id: int = -1 + + +@dataclass +class BatchTokenizedEmbeddingReqInput: + # The batch of tokenized embedding requests + batch: List[TokenizedEmbeddingReqInput] + + def __len__(self): + return len(self.batch) + + def __getitem__(self, i): + return self.batch[i] + + def __iter__(self): + return iter(self.batch) + + +@dataclass +class BatchTokenIDOut: + # The request id + rids: List[str] + # The finish reason + finished_reasons: List[BaseFinishReason] + # For incremental decoding + decoded_texts: List[str] + decode_ids: List[int] + read_offsets: List[int] + # Only used when `--skip-tokenizer-init` is on + output_ids: Optional[List[int]] + # Detokenization configs + skip_special_tokens: List[bool] + spaces_between_special_tokens: List[bool] + no_stop_trim: List[bool] + + # Token counts + prompt_tokens: List[int] + completion_tokens: List[int] + cached_tokens: List[int] + spec_verify_ct: List[int] + + # Logprobs + input_token_logprobs_val: List[float] + input_token_logprobs_idx: List[int] + output_token_logprobs_val: List[float] + output_token_logprobs_idx: List[int] + input_top_logprobs_val: List[List] + input_top_logprobs_idx: List[List] + output_top_logprobs_val: List[List] + output_top_logprobs_idx: List[List] + input_token_ids_logprobs_val: List[List] + input_token_ids_logprobs_idx: List[List] + output_token_ids_logprobs_val: List[List] + output_token_ids_logprobs_idx: List[List] + + # Hidden states + output_hidden_states: List[List[float]] + + # The information of placeholder tokens (e.g., image token) + # idx is the index of the token in the prompt after expansion. + # val is the length of padded tokens after expansion. + placeholder_tokens_idx: List[Optional[List[int]]] + placeholder_tokens_val: List[Optional[List[int]]] + + +@dataclass +class BatchMultimodalDecodeReq: + decoded_ids: List[int] + input_token_logprobs_val: List[float] + input_token_logprobs_idx: List[int] + output_token_logprobs_val: List[float] + output_token_logprobs_idx: List[int] + read_offsets: List[int] + skip_special_tokens: List[bool] + spaces_between_special_tokens: List[bool] + image_resolutions: List[List[int]] + resize_image_resolutions: List[List[int]] + + # The request id + rids: List[str] + finished_reasons: List[BaseFinishReason] + + # Token counts + prompt_tokens: List[int] + completion_tokens: List[int] + cached_tokens: List[int] + + # Placeholder token info + placeholder_tokens_idx: List[Optional[List[int]]] + placeholder_tokens_val: List[Optional[List[int]]] + + return_bytes: bool = False + + +@dataclass +class BatchStrOut: + # The request id + rids: List[str] + # The finish reason + finished_reasons: List[dict] + # The output decoded strings + output_strs: List[str] + # The token ids + output_ids: Optional[List[int]] + + # Token counts + prompt_tokens: List[int] + completion_tokens: List[int] + cached_tokens: List[int] + spec_verify_ct: List[int] + + # Logprobs + input_token_logprobs_val: List[float] + input_token_logprobs_idx: List[int] + output_token_logprobs_val: List[float] + output_token_logprobs_idx: List[int] + input_top_logprobs_val: List[List] + input_top_logprobs_idx: List[List] + output_top_logprobs_val: List[List] + output_top_logprobs_idx: List[List] + input_token_ids_logprobs_val: List[List] + input_token_ids_logprobs_idx: List[List] + output_token_ids_logprobs_val: List[List] + output_token_ids_logprobs_idx: List[List] + + # Hidden states + output_hidden_states: List[List[float]] + + placeholder_tokens_idx: List[Optional[List[int]]] + placeholder_tokens_val: List[Optional[List[int]]] + + +@dataclass +class BatchMultimodalOut: + # The request id + rids: List[str] + # The finish reason + finished_reasons: List[dict] + decoded_ids: List[List[int]] + # The outputs + outputs: Union[List[str | bytes], List[List[Dict]]] + + # probability values for input tokens and output tokens + input_token_logprobs_val: List[List[float]] + input_token_logprobs_idx: List[List[int]] + output_token_logprobs_val: List[List[float]] + output_token_logprobs_idx: List[List[int]] + + # Token counts + prompt_tokens: List[int] + completion_tokens: List[int] + cached_tokens: List[int] + + placeholder_tokens_idx: List[Optional[List[int]]] + placeholder_tokens_val: List[Optional[List[int]]] + + return_bytes: List[bool] + + +@dataclass +class BatchEmbeddingOut: + # The request id + rids: List[str] + # The finish reason + finished_reasons: List[BaseFinishReason] + # The output embedding + embeddings: List[List[float]] + # Token counts + prompt_tokens: List[int] + cached_tokens: List[int] + # Placeholder token info + placeholder_tokens_idx: List[Optional[List[int]]] + placeholder_tokens_val: List[Optional[List[int]]] + + +@dataclass +class ClearHiCacheReqInput: + pass + + +@dataclass +class ClearHiCacheReqOutput: + success: bool + + +@dataclass +class FlushCacheReqInput: + pass + + +@dataclass +class FlushCacheReqOutput: + success: bool + + +@dataclass +class UpdateWeightFromDiskReqInput: + # The model path with the new weights + model_path: str + # The format to load the weights + load_format: Optional[str] = None + # Whether to abort all requests before updating weights + abort_all_requests: bool = False + # Optional: Update weight version along with weights + weight_version: Optional[str] = None + # Whether to update weights asynchronously + is_async: bool = False + # Whether to empty torch cache + torch_empty_cache: bool = False + # Whether to keep the scheduler paused after weight update + keep_pause: bool = False + + +@dataclass +class UpdateWeightFromDiskReqOutput: + success: bool + message: str + # Number of paused requests during weight sync. + num_paused_requests: Optional[int] = 0 + + +@dataclass +class UpdateWeightsFromDistributedReqInput: + names: List[str] + dtypes: List[str] + shapes: List[List[int]] + # The group name + group_name: str = "weight_update_group" + # Whether to flush the cache after updating weights + flush_cache: bool = True + # Whether to abort all requests before updating weights + abort_all_requests: bool = False + # Optional: Update weight version along with weights + weight_version: Optional[str] = None + + +@dataclass +class UpdateWeightsFromDistributedReqOutput: + success: bool + message: str + + +@dataclass +class UpdateWeightsFromTensorReqInput: + """Update model weights from tensor input. + + - Tensors are serialized for transmission + - Data is structured in JSON for easy transmission over HTTP + """ + + serialized_named_tensors: List[Union[str, bytes]] + # Optional format specification for loading + load_format: Optional[str] = None + # Whether to flush the cache after updating weights + flush_cache: bool = True + # Whether to abort all requests before updating weights + abort_all_requests: bool = False + # Optional: Update weight version along with weights + weight_version: Optional[str] = None + + +@dataclass +class UpdateWeightsFromTensorReqOutput: + success: bool + message: str + + +@dataclass +class InitWeightsUpdateGroupReqInput: + # The master address + master_address: str + # The master port + master_port: int + # The rank offset + rank_offset: int + # The world size + world_size: int + # The group name + group_name: str = "weight_update_group" + # The backend + backend: str = "nccl" + + +@dataclass +class InitWeightsUpdateGroupReqOutput: + success: bool + message: str + + +@dataclass +class UpdateWeightVersionReqInput: + # The new weight version + new_version: str + # Whether to abort all running requests before updating + abort_all_requests: bool = True + + +@dataclass +class GetWeightsByNameReqInput: + name: str + truncate_size: int = 100 + + +@dataclass +class GetWeightsByNameReqOutput: + parameter: list + + +@dataclass +class ReleaseMemoryOccupationReqInput: + # Optional tags to identify the memory region, which is primarily used for RL + # Currently we only support `weights` and `kv_cache` + tags: Optional[List[str]] = None + + +@dataclass +class ReleaseMemoryOccupationReqOutput: + pass + + +@dataclass +class ResumeMemoryOccupationReqInput: + # Optional tags to identify the memory region, which is primarily used for RL + # Currently we only support `weights` and `kv_cache` + tags: Optional[List[str]] = None + + +@dataclass +class ResumeMemoryOccupationReqOutput: + pass + + +@dataclass +class SlowDownReqInput: + forward_sleep_time: Optional[float] + + +@dataclass +class SlowDownReqOutput: + pass + + +@dataclass +class AbortReq: + # The request id + rid: str = "" + # Whether to abort all requests + abort_all: bool = False + # The finished reason data + finished_reason: Optional[Dict[str, Any]] = None + abort_reason: Optional[str] = None + # used in MultiTokenzierManager mode + rids: Optional[Union[List[str], str]] = None + + def __post_init__(self): + self.rids = self.rid + + +@dataclass +class GetInternalStateReq: + pass + + +@dataclass +class GetInternalStateReqOutput: + internal_state: Dict[Any, Any] + + +@dataclass +class SetInternalStateReq: + server_args: Dict[str, Any] + + +@dataclass +class SetInternalStateReqOutput: + updated: bool + server_args: Dict[str, Any] + + +@dataclass +class ProfileReqInput: + # The output directory + output_dir: Optional[str] = None + # If set, it profile as many as this number of steps. + # If it is set, profiling is automatically stopped after this step, and + # the caller doesn't need to run stop_profile. + start_step: Optional[int] = None + num_steps: Optional[int] = None + activities: Optional[List[str]] = None + profile_by_stage: bool = False + with_stack: Optional[bool] = None + record_shapes: Optional[bool] = None + + +class ProfileReqType(Enum): + START_PROFILE = 1 + STOP_PROFILE = 2 + + +@dataclass +class ProfileReq: + type: ProfileReqType + output_dir: Optional[str] = None + start_step: Optional[int] = None + num_steps: Optional[int] = None + activities: Optional[List[str]] = None + profile_by_stage: bool = False + with_stack: Optional[bool] = None + record_shapes: Optional[bool] = None + profile_id: Optional[str] = None + + +@dataclass +class ProfileReqOutput: + success: bool + message: str + + +@dataclass +class FreezeGCReq: + pass + + +@dataclass +class ConfigureLoggingReq: + log_requests: Optional[bool] = None + log_requests_level: Optional[int] = None + dump_requests_folder: Optional[str] = None + dump_requests_threshold: Optional[int] = None + crash_dump_folder: Optional[str] = None + + +@dataclass +class OpenSessionReqInput: + capacity_of_str_len: int + session_id: Optional[str] = None + + +@dataclass +class CloseSessionReqInput: + session_id: str + + +@dataclass +class OpenSessionReqOutput: + session_id: Optional[str] + success: bool + + +@dataclass +class HealthCheckOutput: + pass + + +class ExpertDistributionReq(Enum): + START_RECORD = 1 + STOP_RECORD = 2 + DUMP_RECORD = 3 + + +@dataclass +class ExpertDistributionReqOutput: + pass + + +@dataclass +class Function: + description: Optional[str] = None + name: Optional[str] = None + parameters: Optional[object] = None + + +@dataclass +class Tool: + function: Function + type: Optional[str] = "function" + + +@dataclass +class ParseFunctionCallReq: + text: str # The text to parse. + tools: List[Tool] = field( + default_factory=list + ) # A list of available function tools (name, parameters, etc.). + tool_call_parser: Optional[str] = ( + None # Specify the parser type, e.g. 'llama3', 'qwen25', or 'mistral'. If not specified, tries all. + ) + + +@dataclass +class SeparateReasoningReqInput: + text: str # The text to parse. + reasoning_parser: str # Specify the parser type, e.g., "deepseek-r1". + + +@dataclass +class VertexGenerateReqInput: + instances: List[dict] + parameters: Optional[dict] = None + + +@dataclass +class RpcReqInput: + method: str + parameters: Optional[Dict] = None + + +@dataclass +class RpcReqOutput: + success: bool + message: str + + +@dataclass +class LoadLoRAAdapterReqInput: + # The name of the lora module to newly loaded. + lora_name: str + # The path of loading. + lora_path: str + # Whether to pin the LoRA adapter in memory. + pinned: bool = False + # The unique identifier for the LoRA adapter, which automatically generated in the `TokenizerManager`. + lora_id: Optional[str] = None + + def to_ref(self) -> LoRARef: + return LoRARef( + lora_id=self.lora_id, + lora_name=self.lora_name, + lora_path=self.lora_path, + pinned=self.pinned, + ) + + +@dataclass +class UnloadLoRAAdapterReqInput: + # The name of lora module to unload. + lora_name: str + # The unique identifier for the LoRA adapter, which automatically generated in the `TokenizerManager`. + lora_id: Optional[str] = None + + def to_ref(self) -> LoRARef: + return LoRARef( + lora_id=self.lora_id, + lora_name=self.lora_name, + ) + + +@dataclass +class LoRAUpdateResult: + success: bool + error_message: Optional[str] = None + loaded_adapters: Optional[Dict[str, LoRARef]] = None + + +LoadLoRAAdapterReqOutput = UnloadLoRAAdapterReqOutput = LoRAUpdateResult + + +@dataclass +class MultiTokenizerRegisterReq: + rids: Optional[Union[List[str], str]] = None + ipc_name: Optional[str] = None + + +@dataclass +class MultiTokenizerWrapper: + worker_id: int + obj: Optional[Any] = None + + +class BlockReqType(Enum): + BLOCK = 1 + UNBLOCK = 2 + + +@dataclass +class BlockReqInput: + type: BlockReqType diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py new file mode 100644 index 000000000..f495904d5 --- /dev/null +++ b/python/sglang/srt/managers/mm_utils.py @@ -0,0 +1,775 @@ +""" +Multi-modality utils +""" + +import hashlib +import pickle +from abc import abstractmethod +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple + +import numpy as np +import torch +from torch import nn + +from sglang.srt.layers.multimodal import gpu_tensor_hash +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, + global_server_args_dict, +) +from sglang.srt.mem_cache.multimodal_cache import MultiModalCache +from sglang.srt.model_executor.forward_batch_info import ForwardBatch +from sglang.srt.utils import flatten_nested_list, is_npu, print_warning_once +from sglang.utils import logger + +_is_npu = is_npu() + +# NOTE: Using the shared logger from sglang.utils instead of creating a module-specific logger +# to ensure consistent logging behavior across the codebase. This prevents issues with log +# propagation that can cause some log messages (like 'server is fired up') to not appear +# in the console when multimodal support is enabled. + +# TODO(mick): nccl +# cuda_ipc: for intranode tensor sharing +TensorTransportMode = Literal["cuda_ipc", "auto", "default"] + + +class TransportProxyTensor(torch.Tensor): + """ + A convenient torch.Tensor subclass that carries extra metadata and supports + efficient inter-process communications + """ + + @staticmethod + def __new__( + cls, + data: torch.Tensor, + name: Optional[str] = None, + fields: Optional[Dict[str, Any]] = None, + transport_mode: TensorTransportMode = "default", + *args, + **kwargs, + ): + + if not isinstance(data, torch.Tensor): + raise TypeError( + f"Input 'data' must be a torch.Tensor, but got {type(data)}" + ) + + instance = data.as_subclass(cls) + + instance._metadata = { + "name": name, + "fields": fields if fields is not None else {}, + "transport_mode": transport_mode, + } + + return instance + + def __getstate__(self): + """ + Called during pickling. Implements the serialization logic. + """ + # acquire all serialize metadata from _metadata + state = { + "metadata": self._metadata, + "tensor_data": None, + "ipc_extra": None, + } + + transport_mode = self._metadata.get("transport_mode", "default") + + if transport_mode == "cuda_ipc" and self.is_cuda: + try: + storage = self.untyped_storage() + handle = storage._share_cuda_() + + state["ipc_extra"] = { + "handle": handle, + "shape": self.shape, + "dtype": self.dtype, + "stride": self.stride(), + "device_index": self.device.index, + } + state["tensor_data"] = None + except Exception as e: + # Failed to get CUDA IPC handle (possibly tp). Falling back to default transport. + state["metadata"]["transport_mode"] = "default" + state["tensor_data"] = self.as_subclass(torch.Tensor) + else: + state["metadata"]["transport_mode"] = "default" + state["tensor_data"] = self.as_subclass(torch.Tensor) + + return state + + def __setstate__(self, state: Dict[str, Any]): + """ + Called during unpickling. Implements the deserialization logic. + """ + self._metadata = state["metadata"] + + transport_mode = self._metadata.get("transport_mode", "default") + + if transport_mode == "cuda_ipc" and state["ipc_extra"] is not None: + ipc_extra = state["ipc_extra"] + handle, shape, dtype, stride, source_device_index = ( + ipc_extra["handle"], + ipc_extra["shape"], + ipc_extra["dtype"], + ipc_extra["stride"], + ipc_extra["device_index"], + ) + + try: + target_device = torch.device(f"cuda:{source_device_index}") + with torch.cuda.device(target_device): + storage = torch.UntypedStorage._new_shared_cuda(*handle) + reconstructed_tensor = torch.empty( + 0, dtype=dtype, device=target_device + ).set_(storage, storage_offset=0, size=shape, stride=stride) + self.set_(reconstructed_tensor) + except Exception as e: + print(f"Error: Failed to deserialize from CUDA IPC handle ({e}).") + raise e + + elif state["tensor_data"] is not None: + self.set_(state["tensor_data"]) + else: + raise pickle.UnpicklingError( + "Invalid state for TransportProxyTensor: no tensor data found." + ) + + @property + def name(self) -> Optional[str]: + return self._metadata.get("name") + + @property + def fields(self) -> Dict[str, Any]: + return self._metadata.get("fields", {}) + + @property + def transport_mode(self) -> TensorTransportMode: + return self._metadata.get("transport_mode", "default") + + +class MultiModalityDataPaddingPattern: + """ + Data tokens (like image tokens) often need special handling during padding + to maintain model compatibility. This class provides the interface for + implementing different padding strategies for data tokens + """ + + @abstractmethod + def pad_input_tokens( + self, input_ids: List[int], mm_inputs: MultimodalInputs + ) -> List[int]: + """ + Pad the input ids sequence containing data tokens, and replace them with pad_values + """ + pass + + +class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern): + """In this pattern, data tokens should be enclosed by special token pairs (e.g. ..., data_token_pairs) + + The padded value in a region enclosed by a token pair with be the same one, as the MultimodalDataItem's pad value + + This strategy should be applied when data content is marked by start/end token pairs in the input sequence. + """ + + def __init__( + self, + data_token_pairs: Optional[List[Tuple[int, int]]], + data_start_token_ids: Optional[List[int]] = None, + ) -> None: + """ + + Args: + data_start_token_ids marks the start of a single multimodal data + See Minicpmo's slice_start_id for example + """ + self.data_token_id_pairs = data_token_pairs + self.data_start_token_ids = data_start_token_ids or [ + s for s, _e in data_token_pairs + ] + + def pad_input_tokens( + self, input_ids: List[int], mm_inputs: MultimodalInputs + ) -> List[int]: + """ + This function will replace the data-tokens in between with pad_values accordingly + """ + pad_values = [item.pad_value for item in mm_inputs.mm_items] + data_token_pairs = self.data_token_id_pairs + mm_inputs.data_offsets = [] + if data_token_pairs is None: + data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id] + if data_token_pairs is None: + print_warning_once( + "No data_token_pairs provided, RadixAttention might be influenced." + ) + return input_ids + start_token_ids = {s for s, _e in data_token_pairs} + end_tokens_ids = {e for _s, e in data_token_pairs} + + padded_ids = [] + last_idx = 0 + data_idx = -1 + + start_indices = [i for i, x in enumerate(input_ids) if x in start_token_ids] + end_indices = [i for i, x in enumerate(input_ids) if x in end_tokens_ids] + + if len(start_indices) != len(end_indices): + return input_ids + + for start_idx, end_idx in zip(start_indices, end_indices): + padded_ids.extend(input_ids[last_idx : start_idx + 1]) + + if input_ids[start_idx] in self.data_start_token_ids: + data_idx += 1 + mm_inputs.data_offsets += [start_idx] + + if data_idx >= len(pad_values): + data_idx = len(pad_values) - 1 + + num_tokens = end_idx - start_idx - 1 + pad_value = pad_values[data_idx] + padded_ids.extend([pad_value] * num_tokens) + + last_idx = end_idx + + padded_ids.extend(input_ids[last_idx:]) + + assert len(input_ids) == len(padded_ids), "Length validation fails" + return padded_ids + + +class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPattern): + """In this pattern, data tokens should be represented as repetitions of a single token + e.g. ...., or

Prompt conversation

+{% for message in prompt_messages %} +{{ message_to_html(message) | safe }} +{% endfor %} +

Sampled message

+{{ message_to_html(next_message) | safe }} +

Results

+

Correct Answer: {{ correct_answer }}

+

Extracted Answer: {{ extracted_answer }}

+

Score: {{ score }}

+""" + + +def format_multichoice_question(row): + return QUERY_TEMPLATE_MULTICHOICE.format(**row) + + +def check_equality(sampler: SamplerBase, expr1: str, expr2: str): + prompt = EQUALITY_TEMPLATE % {"expression1": expr1, "expression2": expr2} + response = sampler([dict(content=prompt, role="user")]) + return response.lower().strip() == "yes" + + +def _compute_stat(values: list, stat: str): + if stat == "mean": + return np.mean(values) + elif stat == "std": + return np.std(values) + elif stat == "min": + return np.min(values) + elif stat == "max": + return np.max(values) + else: + raise ValueError(f"Unknown {stat =}") + + +def aggregate_results( + single_eval_results: List[SingleEvalResult], + default_stats: Tuple[str] = ("mean", "std"), + name2stats: Optional[Dict[str, Tuple[str]]] = None, +) -> EvalResult: + """ + Aggregate results from multiple evaluations into a single EvalResult. + """ + name2stats = name2stats or {} + name2values = defaultdict(list) + htmls = [] + convos = [] + for single_eval_result in single_eval_results: + for name, value in single_eval_result.metrics.items(): + name2values[name].append(value) + if single_eval_result.score is not None: + name2values["score"].append(single_eval_result.score) + htmls.append(single_eval_result.html) + convos.append(single_eval_result.convo) + final_metrics = {} + for name, values in name2values.items(): + stats = name2stats.get(name, default_stats) + for stat in stats: + key = name if stat == "mean" else f"{name}:{stat}" + final_metrics[key] = _compute_stat(values, stat) + return EvalResult( + score=final_metrics.pop("score", None), + metrics=final_metrics, + htmls=htmls, + convos=convos, + ) + + +def map_with_progress(f: callable, xs: List[Any], num_threads: int): + """ + Apply f to each element of xs, using a ThreadPool, and show progress. + """ + if os.getenv("debug"): + return list(map(f, tqdm(xs, total=len(xs)))) + else: + with ThreadPool(min(num_threads, len(xs))) as pool: + return list(tqdm(pool.imap(f, xs), total=len(xs))) + + +jinja_env = jinja2.Environment( + loader=jinja2.BaseLoader(), + undefined=jinja2.StrictUndefined, + autoescape=jinja2.select_autoescape(["html", "xml"]), +) +_message_template = """ +
+
+ {{ role }} + {% if variant %}({{ variant }}){% endif %} +
+
+ 
{{ content }}
+
+
+""" + + +def message_to_html(message: Message) -> str: + """ + Generate HTML snippet (inside a
) for a message. + """ + return jinja_env.from_string(_message_template).render( + role=message["role"], + content=message["content"], + variant=message.get("variant", None), + ) + + +jinja_env.globals["message_to_html"] = message_to_html + + +_report_template = """ + + + + + + {% if metrics %} +

Metrics

+ + + + + + + + + + {% for name, value in metrics.items() %} + + + + + {% endfor %} +
MetricValue
Score{{ score | float | round(3) }}
{{ name }}{{ value }}
+ {% endif %} +

Examples

+ {% for html in htmls %} + {{ html | safe }} +
+ {% endfor %} + + +""" + + +def make_report(eval_result: EvalResult) -> str: + """ + Create a standalone HTML report from an EvalResult. + """ + return jinja_env.from_string(_report_template).render( + score=eval_result.score, + metrics=eval_result.metrics, + htmls=eval_result.htmls, + ) + + +def make_report_from_example_htmls(htmls: List[str]): + """ + Create a standalone HTML report from a list of example htmls + """ + return jinja_env.from_string(_report_template).render( + score=None, metrics={}, htmls=htmls + ) + + +def download_dataset(path, url): + print(f"Downloading dataset {path} from {url}") + try: + response = requests.get(url, stream=True) + response.raise_for_status() + + total_size = int(response.headers.get("content-length", 0)) + block_size = 8192 + + with open(path, "wb") as f, tqdm( + desc="Downloading", + total=total_size, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as progress_bar: + for data in response.iter_content(block_size): + size = f.write(data) + progress_bar.update(size) + + print(f"Dataset downloaded and saved to {path}") + except requests.RequestException as e: + raise Exception(f"Failed to download dataset: {e}") + + +def set_ulimit(target_soft_limit=65535): + resource_type = resource.RLIMIT_NOFILE + current_soft, current_hard = resource.getrlimit(resource_type) + + if current_soft < target_soft_limit: + try: + resource.setrlimit(resource_type, (target_soft_limit, current_hard)) + except ValueError as e: + print(f"Fail to set RLIMIT_NOFILE: {e}") diff --git a/python/sglang/test/simple_eval_gpqa.py b/python/sglang/test/simple_eval_gpqa.py new file mode 100644 index 000000000..b77ca773e --- /dev/null +++ b/python/sglang/test/simple_eval_gpqa.py @@ -0,0 +1,95 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +GPQA: A Graduate-Level Google-Proof Q&A Benchmark +David Rein, Betty Li Hou, Asa Cooper Stickland, Jackson Petty, Richard Yuanzhe Pang, Julien Dirani, Julian Michael, Samuel R. Bowman +https://arxiv.org/abs/2311.12022 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + MessageList, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + + +class GPQAEval(Eval): + def __init__( + self, + filename: str, + num_examples: Optional[int], + num_threads: int, + n_repeats: int = 1, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + rng = random.Random(0) + if num_examples: + assert n_repeats == 1, "n_repeats only supported for num_examples" + examples = rng.sample(examples, num_examples) + examples = examples * n_repeats + examples = [ + example | {"permutation": rng.sample(range(4), 4)} for example in examples + ] + self.examples = examples + self.n_repeats = n_repeats + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + choices = [ + row["Correct Answer"], + row["Incorrect Answer 1"], + row["Incorrect Answer 2"], + row["Incorrect Answer 3"], + ] + choices = [choices[i] for i in row["permutation"]] + correct_index = choices.index(row["Correct Answer"]) + correct_answer = "ABCD"[correct_index] + choices_dict = dict( + A=choices[0], + B=choices[1], + C=choices[2], + D=choices[3], + Question=row["Question"], + ) + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(choices_dict), role="user" + ) + ] + response_text = sampler(prompt_messages) + if response_text is None: + response_text = "" + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == correct_answer else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={"chars": len(response_text)}, + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_humaneval.py b/python/sglang/test/simple_eval_humaneval.py new file mode 100644 index 000000000..25dcdd53a --- /dev/null +++ b/python/sglang/test/simple_eval_humaneval.py @@ -0,0 +1,133 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +HumanEval: Evaluating Large Language Models Trained on Code +Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba +https://arxiv.org/abs/2107.03374 https://github.com/openai/human-eval/ +""" + +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, List, Optional + +import tqdm + +try: + from human_eval.data import read_problems + from human_eval.evaluation import estimate_pass_at_k + from human_eval.execution import check_correctness # , unsafe_execute +except (ImportError, ModuleNotFoundError): + print("\nPlease install human-eval at https://github.com/openai/human-eval.\n") + raise + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + + +def evaluate_functional_correctness( + sample: Dict[str, str], + completions: List[str], + n_workers: int = 4, + timeout: float = 3.0, +): + """ + Evaluates the functional correctness of generated samples, and writes + results to f"{sample_file}_results.jsonl.gz" + """ + import copy + + # Check the generated samples against test suites. + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = [] + for i, completion in enumerate(completions): + args = (sample, completion, timeout, i) + future = executor.submit(check_correctness, *args) + futures.append(future) + results = [] + for future in as_completed(futures): + result = future.result() + results.append(result) + passed = [int(r["passed"]) for r in results] + return passed + + +class HumanEval(Eval): + def __init__( + self, + num_examples: Optional[int], + num_threads: int, + num_samples_per_task: int = 5, + ks_passes: List[int] = [1, 2, 5], + timeout: int = 120, + ): + self.seed = 0 + self.examples = read_problems() + self.examples = list(self.examples.values()) + + self._num_examples = num_examples + if self._num_examples: + self.examples = random.Random(self.seed).sample(self.examples, num_examples) + self._num_samples_per_task = num_samples_per_task + self._ks_passes = ks_passes + self._timeout = timeout + self._num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + instruction = "Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n" + + def find_code(completion): + pattern = re.compile(r"```python\n(.*?)```", re.DOTALL) + matches = pattern.findall(completion) + extracted_answer = matches[0] if len(matches) >= 1 else completion + extracted_answer = extracted_answer[ + extracted_answer.find(":\n ") + 2 : + ] # remove signature + return extracted_answer + + def fn(sample: Dict[str, str]): + prompt_messages = [ + sampler._pack_message( + role="user", content=instruction + sample["prompt"] + ) + ] + completions = [ + find_code(sampler(prompt_messages)) + for _ in range(self._num_samples_per_task) + ] + results = evaluate_functional_correctness(sample, completions) + total = len(results) + correct = sum(results) + score = sum(results) / len(results) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=completions[0], role="assistant"), + score=score, + correct_answer=[1] * len(results), + extracted_answer=results, + ) + convo = prompt_messages + [ + dict(content=completion, role="assistant") for completion in completions + ] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={ + f"pass@{k}": estimate_pass_at_k([total], [correct], k) + # this will be aggregated so no need of .mean() + for k in self._ks_passes + if total >= k + }, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_math.py b/python/sglang/test/simple_eval_math.py new file mode 100644 index 000000000..74c49abe5 --- /dev/null +++ b/python/sglang/test/simple_eval_math.py @@ -0,0 +1,73 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Mathematical Problem Solving With the MATH Dataset +Dan Hendrycks, Collin Burns, Saurav Kadavath, Akul Arora, Steven Basart, Eric Tang, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2103.03874 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + check_equality, +) + +QUERY_TEMPLATE = """ +Solve the following math problem step by step. The last line of your response should be of the form Answer: $ANSWER (without quotes) where $ANSWER is the answer to the problem. + +{Question} + +Remember to put your answer on its own line after "Answer:", and you do not need to use a \\boxed command. +""".strip() + + +class MathEval(Eval): + def __init__( + self, + filename: str, + equality_checker: SamplerBase, + num_examples: Optional[int], + num_threads: int, + ): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.equality_checker = equality_checker + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message(content=QUERY_TEMPLATE.format(**row), role="user") + ] + response_text = sampler(prompt_messages) + match = re.search(ANSWER_PATTERN, response_text) + extracted_answer = match.group(1) if match else None + score = float( + check_equality(self.equality_checker, row["Answer"], extracted_answer) + ) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult(html=html, score=score, convo=convo) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/simple_eval_mgsm.py b/python/sglang/test/simple_eval_mgsm.py new file mode 100644 index 000000000..0b0b72a20 --- /dev/null +++ b/python/sglang/test/simple_eval_mgsm.py @@ -0,0 +1,203 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +MGSM: Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems. +Language Models are Multilingual Chain-of-Thought Reasoners +Freda Shi, Mirac Suzgun, Markus Freitag, Xuezhi Wang, Suraj Srivats, Soroush Vosoughi, Hyung Won Chung, Yi Tay, Sebastian Ruder, Denny Zhou, Dipanjan Das, Jason Wei +https://arxiv.org/abs/2210.03057 reference: https://github.com/google-research/url-nlp +""" + +import re +import urllib +from typing import Optional + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, +) + +ALL_LANGUAGES = ["bn", "de", "en", "es", "fr", "ja", "ru", "sw", "te", "th", "zh"] +LATIN_LANGUAGES = ["de", "en", "es", "fr", "sw"] +NON_LATIN_LANGUAGES = ["bn", "ja", "ru", "te", "th", "zh"] + +LANG_TO_FPATH = { + "bn": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_bn.tsv", + "de": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_de.tsv", + "en": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_en.tsv", + "es": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_es.tsv", + "fr": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_fr.tsv", + "ja": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ja.tsv", + "ru": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_ru.tsv", + "sw": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_sw.tsv", + "te": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_te.tsv", + "th": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_th.tsv", + "zh": "https://openaipublic.blob.core.windows.net/simple-evals/mgsm_zh.tsv", +} +LANG_TO_INSTRUCTIONS = { + "en": """Solve this math problem. Give the reasoning steps before giving the final answer on the last line by itself in the format of "Answer:". Do not add anything other than the integer answer after "Answer:". + +{input}""", + "bn": """এই গণিতের সমস্যাটি সমাধান করুন। চূড়ান্ত উত্তর দেওয়ার আগে যুক্তিসম্পন্ন পদক্ষেপ প্রদান করুন। চূড়ান্ত উত্তরটি একক সংখ্যা হিসাবে "উত্তর:" এর পরে শেষ লাইনে দিন। "উত্তর:" এর পরে অন্য কিছু যুক্ত করবেন না।. + +{input}""", + "de": """Löse dieses Mathematikproblem. Gib die Schritte zur Begründung an, bevor du die endgültige Antwort in der letzten Zeile alleine im Format "Antwort:" gibst. Füge nichts anderes als die ganzzahlige Antwort nach "Antwort:" hinzu. + +{input}""", + "es": """Resuelve este problema matemático. Proporciona los pasos de razonamiento antes de dar la respuesta final en la última línea por sí misma en el formato de "Respuesta:". No añadas nada más que la respuesta entera después de "Respuesta:". + +{input}""", + "fr": """Résolvez ce problème de mathématiques. Donnez les étapes de raisonnement avant de fournir la réponse finale sur la dernière ligne elle-même dans le format de "Réponse:". N'ajoutez rien d'autre que la réponse entière après "Réponse:". + +{input}""", + "ja": """の数学の問題を解いてください。最終的な答えを出す前に、解答の推論過程を記述してください。そして最後の行には "答え:" の形式で答えを記述し、その後には整数の答え以外何も追加しないでください。 + +{input}""", + "ru": """Решите эту математическую задачу. Объясните шаги рассуждения перед тем, как дать окончательный ответ в последней строке сам по себе в формате "Ответ:". Не добавляйте ничего, кроме целочисленного ответа после "Ответ:". + +{input}""", + "sw": """Suluhisha tatizo hili la hesabu. Toa hatua za mantiki kabla ya kutoa jibu la mwisho kwenye mstari wa mwisho peke yake katika muundo wa "Jibu:". Usiongeze chochote kingine isipokuwa jibu la integer baada ya "Jibu:". + +{input}""", + "te": """ఈ గణిత సమస్యను పరిష్కరించండి. చివరి సమాధానాన్ని ఇవ్వదానికి ముందు తర్కాత్మక అదుగులను ఇవ్వండి. చివరి పంక్తిలో మాత్రమే 'సమాధానం:' అనే ఆకారంలో చివరి సమాధానాద్ని ఇవ్వండి సమాధానం: తర్వాత పూర్ణాంక సమాధానానికి తప్పించి ఎదేనా చేర్చవద్దు. + +{input}""", + "th": """แก้ปัญหาคณิตศาสตร์นี้ ให้ให้ขั้นตอนการใช้เหตุผลก่อนที่จะให้คำตอบสุดท้ายในบรรทัดสุดท้ายโดยอยู่ในรูปแบบ "คำตอบ:" ไม่ควรเพิ่มอะไรนอกจากคำตอบที่เป็นจำนวนเต็มหลังจาก "คำตอบ:" + +{input}""", + "zh": """解决这个数学问题。在最后一行给出答案前，请提供推理步骤。最后一行应该以 "答案: " 的形式独立给出答案。在 "答案：" 后不要添加除整数答案之外的任何内容。 + +{input}""", +} + +LANG_TO_ANSWER_PREFIX = { + "en": "Answer", + "bn": "উত্তর", + "de": "Antwort", + "es": "Respuesta", + "fr": "Réponse", + "ja": "答え", + "ru": "Ответ", + "sw": "Jibu", + "te": "సమాధానం", + "th": "คำตอบ", + "zh": "答案", +} + + +def parse_answer(answer: str, answer_prefix: str) -> str: + if answer_prefix not in answer: + return "" + + answer_text = answer.split(answer_prefix)[-1].strip() + + # find all the numbers (including decimals) in the string + numbers = re.findall(r"\d+\.?\d*", answer_text.replace(",", "")) + + # return the first number (removing trailing decimal point if present), + # or an empty string if there were no numbers + return numbers[-1].rstrip(".") if numbers else "" + + +def score_mgsm(target: str, prediction: str) -> bool: + if "." in prediction: + prediction = prediction.rstrip("0").rstrip(".") + + target = target.replace(",", "") + prediction = prediction.replace(",", "") + + return target == prediction + + +def get_lang_examples(lang: str) -> list[dict[str, str]]: + fpath = LANG_TO_FPATH[lang] + examples = [] + with urllib.request.urlopen(fpath) as f: + for line in f.read().decode("utf-8").splitlines(): + inputs, targets = line.strip().split("\t") + if "." in targets: + raise ValueError(f"targets {targets} contains a decimal point.") + # targets = int(targets.replace(",", "")) + examples.append({"inputs": inputs, "targets": targets, "lang": lang}) + return examples + + +def get_all_examples() -> list[dict[str, str]]: + examples = [] + for lang in ALL_LANGUAGES: + if lang != "en": + continue + examples += get_lang_examples(lang) + return examples + + +class MGSMEval(Eval): + def __init__( + self, + num_examples_per_lang: int = 250, # restrict to a subset of the data for debugging + num_threads: int = 64, + languages: Optional[list[str]] = ALL_LANGUAGES, + ): + if languages is None: + languages = ALL_LANGUAGES + else: + for language in languages: + if language not in ALL_LANGUAGES: + raise ValueError( + f"language {language} is not a valid language. " + f"It should be one in {ALL_LANGUAGES}" + ) + self._languages = languages + self._num_examples_per_lang = num_examples_per_lang + self._num_threads = num_threads + + examples = [] + for lang in self._languages: + lang_examples = get_lang_examples(lang) + examples.extend(lang_examples[: self._num_examples_per_lang]) + self.examples = examples + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(example: dict[str, str]): + language = example["lang"] + latin_language = ( + "group_latin" if language in LATIN_LANGUAGES else "group_non_latin" + ) + correct_answer = example["targets"] + instructoin = LANG_TO_INSTRUCTIONS[language] + prompt_messages = [ + sampler._pack_message( + content=instructoin.format(input=example["inputs"]), role="user" + ) + ] + try: + response_text = sampler(prompt_messages) + except Exception as e: + response_text = "" + + answer_prefix = LANG_TO_ANSWER_PREFIX[language] + extracted_answer = parse_answer(response_text, answer_prefix) + + score = score_mgsm(correct_answer, extracted_answer) + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=correct_answer, + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + return SingleEvalResult( + html=html, + score=score, + convo=convo, + metrics={language: score, latin_language: score}, + ) + + results = common.map_with_progress( + fn, self.examples, num_threads=self._num_threads + ) + return common.aggregate_results(results, default_stats=("mean", "std")) diff --git a/python/sglang/test/simple_eval_mmlu.py b/python/sglang/test/simple_eval_mmlu.py new file mode 100644 index 000000000..36a5c7fe3 --- /dev/null +++ b/python/sglang/test/simple_eval_mmlu.py @@ -0,0 +1,121 @@ +# Adapted from https://github.com/openai/simple-evals/ + +""" +Measuring Massive Multitask Language Understanding +Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, Jacob Steinhardt +https://arxiv.org/abs/2009.03300 +""" + +import random +import re +from typing import Optional + +import pandas + +from sglang.test import simple_eval_common as common +from sglang.test.simple_eval_common import ( + ANSWER_PATTERN_MULTICHOICE, + HTML_JINJA, + Eval, + EvalResult, + SamplerBase, + SingleEvalResult, + format_multichoice_question, +) + +subject2category = { + "abstract_algebra": "stem", + "anatomy": "other", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + + +class MMLUEval(Eval): + def __init__(self, filename: str, num_examples: Optional[int], num_threads: int): + df = pandas.read_csv(filename) + examples = [row.to_dict() for _, row in df.iterrows()] + if num_examples: + examples = random.Random(0).sample(examples, num_examples) + self.examples = examples + self.num_threads = num_threads + + def __call__(self, sampler: SamplerBase) -> EvalResult: + def fn(row: dict): + prompt_messages = [ + sampler._pack_message( + content=format_multichoice_question(row), role="user" + ) + ] + response_text = sampler(prompt_messages) + match = re.search(ANSWER_PATTERN_MULTICHOICE, response_text) + extracted_answer = match.group(1) if match else None + score = 1.0 if extracted_answer == row["Answer"] else 0.0 + html = common.jinja_env.from_string(HTML_JINJA).render( + prompt_messages=prompt_messages, + next_message=dict(content=response_text, role="assistant"), + score=score, + correct_answer=row["Answer"], + extracted_answer=extracted_answer, + ) + convo = prompt_messages + [dict(content=response_text, role="assistant")] + category = subject2category.get(row["Subject"], "other") + return SingleEvalResult( + html=html, score=score, metrics={category: score}, convo=convo + ) + + results = common.map_with_progress(fn, self.examples, self.num_threads) + return common.aggregate_results(results) diff --git a/python/sglang/test/test_activation.py b/python/sglang/test/test_activation.py new file mode 100644 index 000000000..dd5c668cf --- /dev/null +++ b/python/sglang/test/test_activation.py @@ -0,0 +1,105 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import GeluAndMul, QuickGELU +from sglang.srt.utils import is_hip +from sglang.test.test_utils import CustomTestCase + +_is_hip = is_hip() + + +class TestGeluAndMul(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gelu_and_mul_test(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + layer = GeluAndMul().to(dtype=dtype) + x = torch.randn(num_tokens, 2 * d, dtype=dtype) + + with torch.inference_mode(): + ref_out = layer.forward_native(x) + out = layer.forward_cuda(x) + + if dtype == torch.bfloat16: + atol = rtol = 1e-2 + else: + atol = rtol = 1e-3 + + self.assertTrue(torch.allclose(out, ref_out, atol=atol, rtol=rtol)) + + def test_gelu_and_mul(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._run_gelu_and_mul_test(*params) + + +class TestQuickGELU(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 2048] # batch = sequence length + DIMS = [512, 4096, 5120, 13824] # all multiples of 16 bytes + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gelu_quick_test(self, n_tok: int, dim: int, dtype: torch.dtype, seed: int): + torch.manual_seed(seed) + + layer = QuickGELU().to(dtype=dtype) + + x = torch.randn(n_tok, dim, dtype=dtype, device="cuda") + + with torch.inference_mode(): + ref = layer.forward_native(x) # x * sigmoid(1.702 * x), fp32 math + if _is_hip: + out = layer.forward_hip(x) # 128-bit vectorised kernel from sgl-kernel + else: + out = layer.forward_cuda(x) + + tol = 1e-2 if dtype is torch.bfloat16 else 1e-3 + self.assertTrue( + torch.allclose(out, ref, atol=tol, rtol=tol), + msg=f"Mismatch @ B={n_tok}, D={dim}, dtype={dtype}", + ) + print(f"Match @ B={n_tok}, D={dim}, dtype={dtype}") + + def test_quick_gelu(self): + for params in itertools.product( + self.NUM_TOKENS, self.DIMS, self.DTYPES, self.SEEDS + ): + with self.subTest( + num_tokens=params[0], + dim=params[1], + dtype=params[2], + seed=params[3], + ): + self._run_gelu_quick_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_block_fp8.py b/python/sglang/test/test_block_fp8.py new file mode 100644 index 000000000..45271e116 --- /dev/null +++ b/python/sglang/test/test_block_fp8.py @@ -0,0 +1,661 @@ +import itertools +import os +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import ( + per_tensor_quant_mla_fp8, + per_token_group_quant_fp8, + per_token_group_quant_mla_deep_gemm_masked_fp8, + static_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.srt.layers.quantization.fp8_utils import input_to_float8 +from sglang.test.test_utils import CustomTestCase + +_is_cuda = torch.cuda.is_available() and torch.version.cuda + + +# For test +def native_per_token_group_quant_fp8( + x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn +): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / fp8_max + x_q = (x_ / x_s).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +class TestPerTokenGroupQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + GROUP_SIZE = [64, 128, 256, 512] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_token_group_quant_fp8(self, num_tokens, d, dtype, group_size, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + + with torch.inference_mode(): + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size) + out, scale = per_token_group_quant_fp8(x, group_size) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.20) + ) + self.assertTrue(torch.allclose(scale, ref_scale)) + + def test_per_token_group_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.GROUP_SIZE, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + group_size=params[3], + seed=params[4], + ): + self._per_token_group_quant_fp8(*params) + + +# For test +def native_static_quant_fp8(x, x_s, dtype=torch.float8_e4m3fn): + """Function to perform static quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + """ + assert x.is_contiguous(), "`x` is not contiguous" + assert x_s.numel() == 1, "only supports per-tensor scale" + + finfo = torch.finfo(dtype) + fp8_min = finfo.min + fp8_max = finfo.max + + x_ = x.reshape(x.numel() // x.shape[-1], x.shape[-1]) + x_s_inv = 1.0 / x_s + x_q = (x_ * x_s_inv).clamp(min=fp8_min, max=fp8_max).to(dtype) + x_q = x_q.reshape(x.shape) + + return x_q, x_s + + +class TestStaticQuantFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _static_quant_fp8(self, num_tokens, d, dtype, seed): + torch.manual_seed(seed) + + x = torch.rand(num_tokens, d, dtype=dtype) + fp8_max = torch.finfo(torch.float8_e4m3fn).max + x_s = x.max() / fp8_max + + with torch.inference_mode(): + ref_out, _ = native_static_quant_fp8(x, x_s) + out, _ = static_quant_fp8(x, x_s, repeat_scale=True) + + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.50) + ) + + def test_static_quant_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + dtype=params[2], + seed=params[3], + ): + self._static_quant_fp8(*params) + + +class TestPerTensorQuantMlaFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + NUM_TOKENS = [7, 83, 2048] + D = [512, 4096, 5120, 13824] + LAST_D_EXT = [1024, 0] + LAST_D = [512] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_tensor_quant_mla_fp8(self, num_tokens, d, last_d_ext, last_d, dtype, seed): + torch.manual_seed(seed) + + x = torch.rand( + (num_tokens, d // last_d, last_d + last_d_ext), + dtype=dtype, + ) + x_sub, _ = x.split([last_d, last_d_ext], dim=-1) + + with torch.inference_mode(): + ref_out, ref_s = input_to_float8(x_sub.transpose(0, 1)) + out, out_s = per_tensor_quant_mla_fp8(x_sub.transpose(0, 1)) + + self.assertTrue(out.is_contiguous()) + self.assertTrue( + torch.allclose(out.to(torch.float32), ref_out.to(torch.float32), rtol=0.50) + ) + self.assertTrue( + torch.allclose(out_s.to(torch.float32), ref_s.to(torch.float32)) + ) + + def test_per_tensor_quant_mla_fp8(self): + for params in itertools.product( + self.NUM_TOKENS, + self.D, + self.LAST_D_EXT, + self.LAST_D, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + d=params[1], + last_d_ext=params[2], + last_d=params[3], + dtype=params[4], + seed=params[5], + ): + self._per_tensor_quant_mla_fp8(*params) + + +class TestPerTokenGroupQuantMlaDeepGemmMaskedFP8(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16, torch.float32] + B = [128] + NUM_TOKENS = [7, 83, 2048, 1024 * 16] + D = [512, 128] + GROUP_SIZE = [128] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _per_token_group_quant_mla_deep_gemm_masked_fp8( + self, b, num_tokens, d, dtype, group_size, seed + ): + torch.manual_seed(seed) + + x = torch.rand(b, num_tokens, d, dtype=dtype) + + with torch.inference_mode(): + ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size, 1e-12) + out, scale, _, _, _ = per_token_group_quant_mla_deep_gemm_masked_fp8( + x, group_size + ) + out = out[:, :num_tokens, :] + scale = scale[:, :num_tokens, :] + + self.assertTrue( + torch.allclose( + out.to(torch.float32), ref_out.to(torch.float32), rtol=0.20, atol=1e-2 + ) + ) + self.assertTrue(torch.allclose(scale, ref_scale)) + + def test_per_token_group_quant_mla_deep_gemm_masked_fp8(self): + for params in itertools.product( + self.B, + self.NUM_TOKENS, + self.D, + self.DTYPES, + self.GROUP_SIZE, + self.SEEDS, + ): + with self.subTest( + b=params[0], + num_tokens=params[1], + d=params[2], + dtype=params[3], + group_size=params[4], + seed=params[5], + ): + self._per_token_group_quant_mla_deep_gemm_masked_fp8(*params) + + +# For test +def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +class TestW8A8BlockFP8Matmul(CustomTestCase): + + if not _is_cuda: + OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 7, 83, 512, 2048] + NKs = [ + (N, K) + for N in [128, 512, 1024, 4096, 7748, 13824] + for K in [256, 4096, 5120, 3884, 13824] + ] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + else: + # use practical shape in DeepSeek V3 for test + OUT_DTYPES = [torch.bfloat16] + M = [64, 128, 512, 1024, 4096] + NKs = [ + (2112, 7168), + (1536, 7168), + (3072, 1536), + (24576, 7168), + (4096, 512), + (7168, 2048), + (4608, 7168), + (512, 7168), + (7168, 2304), + (7168, 512), + ] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_matmul(self, M, NK, block_size, out_dtype, seed): + N, K = NK + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max + B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + + As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale + Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale + + with torch.inference_mode(): + ref_out = native_w8a8_block_fp8_matmul( + A_fp8, B_fp8, As, Bs, block_size, out_dtype + ) + out = w8a8_block_fp8_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.001 + ) + + def test_w8a8_block_fp8_matmul(self): + for params in itertools.product( + self.M, + self.NKs, + self.BLOCK_SIZE, + self.OUT_DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + NKs=params[1], + block_size=params[2], + out_dtype=params[3], + seed=params[4], + ): + self._w8a8_block_fp8_matmul(*params) + + +# For test +def torch_w8a8_block_fp8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_fp8(a, block_k) + # NOTE(HandH1998): Since "index_cuda" not implemented for 'Float8_e4m3fn', we need to cast `float8`` to `float32``. + a_q = a_q.to(torch.float32) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_fp8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_fp8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_fp8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockFP8FusedMoE(CustomTestCase): + DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 33, 64, 222, 1024 * 128] + N = [128, 1024, 2048] + K = [256, 4096, 5120] + E = [8, 24] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + # BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * fp8_max + w1 = w1_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * fp8_max + w2 = w2_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_block_fp8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_fp8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_fp8_fused_moe(*params) + + +# For test +def torch_w8a8_block_fp8_bmm(a, a_s, w, w_s, block_shape, out_dtype): + """This function performs bmm with block-wise quantization using native torch.""" + + B, N, _ = w.shape + _, M, _ = a.shape + out = torch.empty((B, M, N), dtype=out_dtype, device=a.device) + + for i in range(B): + out[i] = native_w8a8_block_fp8_matmul( + a[i], w[i], a_s[i], w_s[i], block_shape, output_dtype=out_dtype + ) + + return out + + +class TestW8A8BlockFP8BatchedDeepGemm(CustomTestCase): + DTYPES = [torch.bfloat16] + M = [1, 33, 64, 222, 8192] + N = [128, 512] + K = [128, 512] + BATCH = [128] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + try: + import deep_gemm + except ImportError: + raise unittest.SkipTest("DeepGEMM is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_batched_deep_gemm(self, M, N, K, B, block_size, dtype, seed): + torch.manual_seed(seed) + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a_fp32 = torch.randn((B, M, K), dtype=torch.float32) / 10 + a = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w_fp32 = (torch.rand((B, N, K), dtype=torch.float32) - 0.5) * 2 * fp8_max + w = w_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w = (N + block_n - 1) // block_n + k_tiles_w = (K + block_k - 1) // block_k + + w_s = ( + torch.rand((B, n_tiles_w, k_tiles_w), dtype=torch.float32) + * factor_for_scale + ) + a_s = torch.rand((B, M, k_tiles_w), dtype=torch.float32) * factor_for_scale + + ae = a.new_empty(B, (M + 255) // 256 * 256, K) + ae_s = a_s.new_empty(B, (M + 255) // 256 * 256, k_tiles_w) + oe = torch.empty((B, (M + 255) // 256 * 256, N), dtype=dtype) + ae[:, :M, :] = a + ae_s[:, :M, :] = a_s + + masked_m = torch.full((B,), M, dtype=torch.int) + expected_m = M + lhs = ( + ae, + ae_s, + ) + rhs = ( + w, + w_s, + ) + + from deep_gemm import m_grouped_gemm_fp8_fp8_bf16_nt_masked + + with torch.inference_mode(): + ref_out = torch_w8a8_block_fp8_bmm(a, a_s, w, w_s, block_size, dtype) + m_grouped_gemm_fp8_fp8_bf16_nt_masked(lhs, rhs, oe, masked_m, expected_m) + out = oe[:, :M, :] + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.0001 + ) + + def test_w8a8_block_fp8_batched_deep_gemm(self): + + for params in itertools.product( + self.M, + self.N, + self.K, + self.BATCH, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + B=params[3], + block_size=params[4], + dtype=params[5], + seed=params[6], + ): + self._w8a8_block_fp8_batched_deep_gemm(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py b/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py new file mode 100644 index 000000000..36d7acddb --- /dev/null +++ b/python/sglang/test/test_block_fp8_deep_gemm_blackwell.py @@ -0,0 +1,252 @@ +import itertools +import os +import unittest +from typing import List, Tuple + +import torch +from deep_gemm import fp8_gemm_nt + +from sglang.test.test_utils import CustomTestCase + +_is_cuda = torch.cuda.is_available() and torch.version.cuda + + +# Modify form DeepGEMM Blackwell +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def align(x: int, y: int) -> int: + return ceil_div(x, y) * y + + +def per_token_group_quant_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + sf = x_amax / 448.0 + return (x_view * (1.0 / sf.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n), sf + + +def per_block_quant_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (align(m, 128), align(n, 128)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + sf = x_amax / 448.0 + x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view( + x_view.size(0), x_view.size(2) + ) + + +def ceil_to_ue8m0(x: torch.Tensor): + assert x.view(-1).amax().item() > 0 + return torch.pow(2.0, torch.ceil(torch.log2(x.abs()))) + + +def per_token_group_quant_mxfp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + sf = ceil_to_ue8m0(x_amax / 448.0) + return (x_view * (1.0 / sf.unsqueeze(2))).to(torch.float8_e4m3fn).view(m, n), sf + + +def per_block_quant_mxfp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (align(m, 128), align(n, 128)), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + sf = ceil_to_ue8m0(x_amax / 448.0) + x_scaled = (x_view * (1.0 / sf)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), sf.view( + x_view.size(0), x_view.size(2) + ) + + +# For test +def native_w8a8_block_fp8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +def block_quant_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], + dtype: torch.dtype, +) -> torch.Tensor: + """This function converts block-wise quantization to unquantized. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The output is an unquantized tensor with dtype. + """ + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = torch.empty_like(x_q_block, dtype=dtype) + + for j in range(n_tiles): + for i in range(k_tiles): + x_q_block_tile = x_q_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + x_dq_block_tile = x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + x_dq_block_tile[:, :] = x_q_block_tile.to(torch.float32) * x_s[j][i] + + return x_dq_block + + +class TestDeepGemmBlackwell(CustomTestCase): + + if not _is_cuda: + OUT_DTYPES = [torch.float32, torch.half, torch.bfloat16] + M = [1, 7, 83, 512, 2048] + NKs = [ + (N, K) + for N in [128, 512, 1024, 4096, 7748, 13824] + for K in [256, 4096, 5120, 3884, 13824] + ] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + else: + # use practical shape in DeepSeek V3 for test + OUT_DTYPES = [torch.bfloat16] + M = [64, 128, 512, 1024, 4096] + NKs = [ + (2112, 7168), + (1536, 7168), + # (3072, 1536), + # (24576, 7168), + # (4096, 512), + # (7168, 2048), + # (4608, 7168), + # (512, 7168), + # (7168, 2304), + # (7168, 512), + ] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _test_deep_gemm_blackwell(self, M, NK, block_size, out_dtype, seed): + N, K = NK + torch.manual_seed(seed) + + A = torch.empty((M, K), dtype=torch.bfloat16).normal_(0, 0.2) + B = torch.empty((N, K), dtype=torch.bfloat16).normal_(0, 0.2) + + A_q, A_s = per_token_group_quant_fp8(A) + B_q, B_s = per_block_quant_fp8(B) + + A_dq = block_quant_dequant(A_q, A_s, [1, block_size[1]], out_dtype) + B_dq = block_quant_dequant(B_q, B_s, block_size, out_dtype) + + A_qu = per_token_group_quant_mxfp8(A_dq) + B_qu = per_block_quant_mxfp8(B_dq) + out = None + + with torch.inference_mode(): + ref_out = native_w8a8_block_fp8_matmul( + A_q, B_q, A_s, B_s, block_size, out_dtype + ) + out = torch.empty_like(ref_out) + fp8_gemm_nt(A_qu, B_qu, out) + + torch.testing.assert_close(out, ref_out, atol=1e-1, rtol=1e-2) + + def test_deep_gemm_blackwell(self): + for params in itertools.product( + self.M, + self.NKs, + self.BLOCK_SIZE, + self.OUT_DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + NKs=params[1], + block_size=params[2], + out_dtype=params[3], + seed=params[4], + ): + self._test_deep_gemm_blackwell(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_block_fp8_ep.py b/python/sglang/test/test_block_fp8_ep.py new file mode 100644 index 000000000..670f2e0f8 --- /dev/null +++ b/python/sglang/test/test_block_fp8_ep.py @@ -0,0 +1,358 @@ +import itertools +import random +import unittest +from typing import Any, Callable, Dict, List, Optional, Tuple + +import torch + +from sglang.srt.layers.moe.ep_moe.kernels import ( + grouped_gemm_triton, + post_reorder_triton_kernel, + pre_reorder_triton_kernel, + run_moe_ep_preproess, + silu_and_mul_triton_kernel, +) +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.test.test_utils import CustomTestCase + + +# For test +def ep_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + topk_config: TopKConfig, + # ep config + num_experts: int = 256, + fp8_dtype: torch.types = torch.float8_e4m3fn, + num_experts_per_partition: int = 128, + start_expert_id: int = 0, + end_expert_id: int = 127, + use_fp8_w8a8: bool = False, + w1_scale_inv: Optional[torch.Tensor] = None, + w2_scale_inv: Optional[torch.Tensor] = None, + block_shape: Optional[List[int]] = None, +): + use_blockwise_fp8 = block_shape is not None + top_k = topk_config.top_k + topk_output = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=topk_config, + ) + topk_weights, topk_ids, _ = topk_output + + reorder_topk_ids, src2dst, seg_indptr = run_moe_ep_preproess(topk_ids, num_experts) + + gateup_input = torch.empty( + (int(hidden_states.shape[0] * top_k), hidden_states.shape[1]), + device=hidden_states.device, + dtype=( + fp8_dtype + if (use_fp8_w8a8 and not use_blockwise_fp8) + else hidden_states.dtype + ), + ) + + if use_fp8_w8a8 and not use_blockwise_fp8: + max_value = ( + torch.max(hidden_states).repeat(num_experts_per_partition).to(torch.float32) + ) + w1_input_scale = max_value / torch.finfo(fp8_dtype).max + else: + w1_input_scale = None + + # PreReorder + pre_reorder_triton_kernel[(hidden_states.shape[0],)]( + hidden_states, + gateup_input, + src2dst, + topk_ids, + w1_input_scale, + start_expert_id, + end_expert_id, + top_k, + hidden_states.shape[1], + BLOCK_SIZE=512, + use_per_token_if_dynamic=True, + ) + + seg_indptr_cur_rank = seg_indptr[start_expert_id : end_expert_id + 2] + weight_indices_cur_rank = torch.arange( + 0, + num_experts_per_partition, + device=hidden_states.device, + dtype=torch.int64, + ) + + # GroupGemm-0 + gateup_output = torch.empty( + gateup_input.shape[0], + w1.shape[1], + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + gateup_output = grouped_gemm_triton( + a=gateup_input, + b=w1, + c=gateup_output, + batch_size=num_experts_per_partition, + weight_column_major=True, + seg_indptr=seg_indptr_cur_rank, + weight_indices=weight_indices_cur_rank, + use_fp8_w8a8=use_fp8_w8a8, + scale_a=w1_input_scale, + scale_b=w1_scale_inv, + block_shape=block_shape, + ) + + # Act + down_input = torch.empty( + gateup_output.shape[0], + gateup_output.shape[1] // 2, + device=gateup_output.device, + dtype=( + fp8_dtype + if (use_fp8_w8a8 and not use_blockwise_fp8) + else hidden_states.dtype + ), + ) + if use_fp8_w8a8 and not use_blockwise_fp8: + w2_input_scale = torch.ones( + num_experts_per_partition, + dtype=torch.float32, + device=hidden_states.device, + ) + else: + w2_input_scale = None + + silu_and_mul_triton_kernel[(gateup_output.shape[0],)]( + gateup_output, + down_input, + gateup_output.shape[1], + reorder_topk_ids, + w2_input_scale, + start_expert_id, + end_expert_id, + BLOCK_SIZE=512, + ) + + # GroupGemm-1 + down_output = torch.empty( + down_input.shape[0], + w2.shape[1], + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + + down_output = grouped_gemm_triton( + a=down_input, + b=w2, + c=down_output, + batch_size=num_experts_per_partition, + weight_column_major=True, + seg_indptr=seg_indptr_cur_rank, + weight_indices=weight_indices_cur_rank, + use_fp8_w8a8=use_fp8_w8a8, + scale_a=w2_input_scale, + scale_b=w2_scale_inv, + block_shape=block_shape, + ) + + # PostReorder + output = torch.empty_like(hidden_states) + post_reorder_triton_kernel[(hidden_states.size(0),)]( + down_output, + output, + src2dst, + topk_ids, + topk_weights, + start_expert_id, + end_expert_id, + top_k, + hidden_states.size(1), + 0, + BLOCK_SIZE=512, + ) + return output + + +# test util +def block_dequant( + x_q_block: torch.Tensor, + x_s: torch.Tensor, + block_size: List[int], +) -> Tuple[torch.Tensor, torch.Tensor]: + """This function converts block-wise quantization to tensor-wise quantization. + The inputs are block-wise quantization tensor `x_q_block`, block-wise quantization scale + and the block size. + The outputs are tensor-wise quantization tensor and tensor-wise quantization scale. + Note only float8 is supported for now. + """ + + # process 3D tensor + if x_q_block.dim() == 3: + batch_size = x_q_block.size(0) + return torch.stack( + [block_dequant(x_q_block[b], x_s[b], block_size) for b in range(batch_size)] + ) + + block_n, block_k = block_size[0], block_size[1] + n, k = x_q_block.shape + n_tiles = (n + block_n - 1) // block_n + k_tiles = (k + block_k - 1) // block_k + assert n_tiles == x_s.shape[0] + assert k_tiles == x_s.shape[1] + + x_dq_block = x_q_block.to(torch.float32) + + x_dq_block_tiles = [ + [ + x_dq_block[ + j * block_n : min((j + 1) * block_n, n), + i * block_k : min((i + 1) * block_k, k), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + + for i in range(k_tiles): + for j in range(n_tiles): + x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i] + + return x_dq_block + + +class TestW8A8BlockFP8EPMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 222, 1024, 2048] + N = [128, 1024, 2048] + K = [256, 4096, 5120] + E = [8, 16] + ep_size = [2, 4] + TOP_KS = [2, 4] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_fp8_ep_moe( + self, M, N, K, E, ep_size, topk, block_size, dtype, seed + ): + torch.manual_seed(seed) + random.seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=dtype) - 0.5) * 2 * fp8_max + w1 = w1_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=dtype) - 0.5) * 2 * fp8_max + w2 = w2_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + w1_ref = block_dequant(w1, w1_s, block_size).to(dtype) + w2_ref = block_dequant(w2, w2_s, block_size).to(dtype) + + score = torch.randn((M, E), dtype=dtype) + num_experts_per_partition = E // ep_size + cur_rank = random.randint(0, ep_size - 1) + start_id = cur_rank * num_experts_per_partition + end_id = start_id + num_experts_per_partition - 1 + + topk_config = TopKConfig( + top_k=topk, + renormalize=False, + ) + + with torch.inference_mode(): + out = ep_moe( + hidden_states=a, + w1=w1, + w2=w2, + router_logits=score, + topk_config=topk_config, + use_fp8_w8a8=True, + w1_scale_inv=w1_s, + w2_scale_inv=w2_s, + block_shape=block_size, + num_experts=E, + num_experts_per_partition=num_experts_per_partition, + start_expert_id=start_id, + end_expert_id=end_id, + ) + ref_out = ep_moe( + hidden_states=a, + w1=w1_ref, + w2=w2_ref, + router_logits=score, + topk_config=topk_config, + use_fp8_w8a8=False, + w1_scale_inv=None, + w2_scale_inv=None, + block_shape=None, + num_experts=E, + num_experts_per_partition=num_experts_per_partition, + start_expert_id=start_id, + end_expert_id=end_id, + ) + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / (torch.mean(torch.abs(ref_out.to(torch.float32))) + 1e-6) + < 0.06 + ) + + def test_w8a8_block_fp8_ep_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.ep_size, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + ep_size=params[4], + topk=params[5], + block_size=params[6], + dtype=params[7], + seed=params[8], + ): + self._w8a8_block_fp8_ep_moe(*params) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_custom_ops.py b/python/sglang/test/test_custom_ops.py new file mode 100644 index 000000000..c07c95db6 --- /dev/null +++ b/python/sglang/test/test_custom_ops.py @@ -0,0 +1,148 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/8ca7a71df787ad711ad3ac70a5bd2eb2bb398938/tests/quantization/test_fp8.py + +import pytest +import torch + +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz, scaled_fp8_quant +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() +_is_fp8_fnuz = is_fp8_fnuz() +fp8_dtype = torch.float8_e4m3fnuz if _is_fp8_fnuz else torch.float8_e4m3fn + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +def test_scaled_fp8_quant_per_tensor(dtype) -> None: + + def quantize_ref_per_tensor(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(fp8_dtype) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(fp8_dtype) + return qweight + + def dequantize_per_tensor(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 != 0 to cover edge cases, + # because scaled_fp8_quant is vectorized by 8. + x = (torch.randn(size=(11, 11), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + # Test Per Tensor Static quantization + y, _ = scaled_fp8_quant(x, scale) + ref_y = quantize_ref_per_tensor(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_tensor(y, scale, dtype), + dequantize_per_tensor(ref_y, scale, dtype), + ) + + +if _is_cuda or _is_hip: + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) + def test_scaled_fp8_quant_per_token_dynamic(dtype) -> None: + def quantize_ref_per_token(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(fp8_dtype) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp( + min=finfo.min, max=finfo.max + ) + qweight = qweight.to(fp8_dtype) + return qweight + + def dequantize_per_token(tensor, inv_scale, dtype): + fake_qweight = tensor.to(dtype) + dq_weight = fake_qweight * inv_scale + return dq_weight + + # Note that we use a shape % 8 = 0, + # because per_token_quant_fp8 is vectorized by 8 elements. + x = (torch.randn(size=(11, 16), device="cuda") * 13).to(dtype) + + # Test Per Tensor Dynamic quantization + # scale = max(abs(x)) / FP8_E4M3_MAX + y, scale = scaled_fp8_quant(x, None, use_per_token_if_dynamic=True) + ref_y = quantize_ref_per_token(x, scale) + torch.testing.assert_close(y, ref_y) + torch.testing.assert_close( + dequantize_per_token(y, scale, dtype), + dequantize_per_token(ref_y, scale, dtype), + ) + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) + def test_scaled_fp8_quant_with_padding(dtype) -> None: + original_rows = 5 + x = (torch.randn(size=(original_rows, 16), device="cuda") * 13).to(dtype) + + padding_size = 10 + + # Test with dynamic quantization + y_dynamic, scale_dynamic = scaled_fp8_quant( + x, None, num_token_padding=padding_size + ) + + # Verify output shape has the padded size + assert y_dynamic.shape[0] == padding_size + assert y_dynamic.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_without_padding, scale_without_padding = scaled_fp8_quant(x, None) + torch.testing.assert_close(y_dynamic[:original_rows], y_without_padding) + + # Test with static quantization + # First get a scale + _, scale = scaled_fp8_quant(x, None) + + # Then use it for static quantization with padding + y_static, _ = scaled_fp8_quant(x, scale, num_token_padding=padding_size) + + # Verify output shape has the padded size + assert y_static.shape[0] == padding_size + assert y_static.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_static_without_padding, _ = scaled_fp8_quant(x, scale) + torch.testing.assert_close(y_static[:original_rows], y_static_without_padding) + + # Test with per-token dynamic quantization + y_per_token, scale_per_token = scaled_fp8_quant( + x, None, num_token_padding=padding_size, use_per_token_if_dynamic=True + ) + + # Verify output shape has the padded size + assert y_per_token.shape[0] == padding_size + assert y_per_token.shape[1] == x.shape[1] + + # Verify that the actual data in the non-padded region is correctly quantized + y_per_token_without_padding, scale_per_token_without_padding = scaled_fp8_quant( + x, None, use_per_token_if_dynamic=True + ) + torch.testing.assert_close( + y_per_token[:original_rows], y_per_token_without_padding + ) + torch.testing.assert_close( + scale_per_token[:original_rows], scale_per_token_without_padding + ) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/python/sglang/test/test_cutlass_moe.py b/python/sglang/test/test_cutlass_moe.py new file mode 100755 index 000000000..56f276c81 --- /dev/null +++ b/python/sglang/test/test_cutlass_moe.py @@ -0,0 +1,299 @@ +import argparse +import time + +import torch +import triton # Added import +import triton.testing # Added import +from transformers import AutoConfig + +from sglang.srt.layers.moe.cutlass_moe import cutlass_fused_experts_fp8 +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_experts +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.topk import StandardTopKOutput + + +# Copy from: https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/utils.py +def calc_diff(x, y): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def get_model_config(tp_size: int): + config = AutoConfig.from_pretrained( + "deepseek-ai/Deepseek-R1", trust_remote_code=True + ) + E = config.n_routed_experts + topk = config.num_experts_per_tok + intermediate_size = config.moe_intermediate_size + shard_intermediate_size = 2 * intermediate_size // tp_size + + return { + "num_experts": E, + "topk": topk, + "hidden_size": config.hidden_size, + "shard_intermediate_size": shard_intermediate_size, + "dtype": config.torch_dtype, + "block_shape": config.quantization_config["weight_block_size"], + } + + +def to_fp8(tensor: torch.Tensor) -> torch.Tensor: + """Converts tensor to FP8 E4M3, scaling values to fit the range.""" + finfo = torch.finfo(torch.float8_e4m3fn) + # Calculate max absolute value safely + max_val = torch.max(torch.abs(tensor)) + # Avoid division by zero if tensor is all zeros + if max_val == 0: + scale_factor = 1.0 + else: + # Scale factor to bring the max value to finfo.max + scale_factor = finfo.max / max_val + + # Apply scaling + scaled_tensor = tensor * scale_factor + + # Clamp and convert + fp8_tensor = scaled_tensor.clamp(min=finfo.min, max=finfo.max).to( + dtype=torch.float8_e4m3fn + ) + return fp8_tensor + + +def run_test(tp_size, batch_size, model_config, check=False): + print(f"\n--- Batch Size: {batch_size} ---") + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(42) # For reproducible random numbers + + E = model_config["num_experts"] + topk = model_config["topk"] + H = model_config["hidden_size"] + I = model_config["shard_intermediate_size"] + block_shape = model_config["block_shape"] # Tuple (BLOCK_N, BLOCK_K) + dtype = model_config["dtype"] # e.g., torch.bfloat16 + + print( + f"Config: E={E}, topk={topk}, H={H}, I_shard={I}, dtype={dtype}, block_shape={block_shape}" + ) + + # --- Input Data --- + # Use bf16/fp16 for input activation based on model config + x = torch.randn((batch_size, H), device="cuda", dtype=dtype) + # --- Weights (Generate in higher precision, then convert to FP8) --- + # Generate weights suitable for FP8 conversion (e.g., scaled appropriately) + w1_hp = torch.randn((E, I, H), device="cuda", dtype=torch.float32) + w2_hp = torch.randn((E, H, I // 2), device="cuda", dtype=torch.float32) + + w1 = to_fp8(w1_hp) + w2 = to_fp8(w2_hp) + + # --- Scales for FP8 Weights --- + block_n, block_k = block_shape + # Calculate number of blocks needed + w1_blocks_dim1 = (I + block_n - 1) // block_n + w1_blocks_dim2 = (H + block_k - 1) // block_k + w2_blocks_dim1 = (H + block_n - 1) // block_n + w2_blocks_dim2 = (I // 2 + block_k - 1) // block_k + + # Scales are typically float32 or float16/bfloat16 + scale_dtype = torch.float32 # Or dtype if scales match model dtype + w1_scale = torch.full( + (E, w1_blocks_dim1, w1_blocks_dim2), 1, device="cuda", dtype=scale_dtype + ) # Avoid zero scales + w2_scale = torch.full( + (E, w2_blocks_dim1, w2_blocks_dim2), 1, device="cuda", dtype=scale_dtype + ) # Avoid zero scales + + # --- Routing Information --- + topk_weights = torch.softmax( + torch.rand(batch_size, topk, device="cuda", dtype=dtype), dim=-1 + ) + topk_ids = torch.randint(0, E, (batch_size, topk), dtype=torch.int32, device="cuda") + + a1_strides = torch.full((E,), H, dtype=torch.int64, device="cuda") + c1_strides = torch.full((E,), I, dtype=torch.int64, device="cuda") + a2_strides = torch.full((E,), I // 2, dtype=torch.int64, device="cuda") + c2_strides = torch.full((E,), H, dtype=torch.int64, device="cuda") + + workspace = torch.empty( + (7182 * 1024), device="cuda", dtype=torch.uint8 + ) # Allocate sufficient workspace + # Pointer arrays (often filled by the kernel or a prep step, but needed as args) + a_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + b_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + out_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + a_scales_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + b_scales_ptrs = torch.empty((E,), dtype=torch.int64, device="cuda") + expert_offsets = torch.empty((E + 1,), dtype=torch.int32, device="cuda") + problem_sizes1 = torch.empty((E, 3), dtype=torch.int32, device="cuda") + problem_sizes2 = torch.empty((E, 3), dtype=torch.int32, device="cuda") + + # --- Lambdas for Benchmarking --- + cutlass_lambda = lambda: cutlass_fused_experts_fp8( + x, + w1.transpose(1, 2), # Transposed + w2.transpose(1, 2), # Transposed + w1_scale.transpose(1, 2), + w2_scale.transpose(1, 2), + topk_weights, + topk_ids, + a1_strides, + c1_strides, + a2_strides, + c2_strides, + workspace, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + expert_offsets, + problem_sizes1, + problem_sizes2, + ) + + topk_output = StandardTopKOutput( + topk_weights=topk_weights, + topk_ids=topk_ids, + router_logits=torch.randn( + (batch_size, topk), device=topk_weights.device, dtype=dtype + ), + ) + + moe_runner_config = MoeRunnerConfig( + num_experts=E, + top_k=topk, + hidden_size=H, + intermediate_size_per_partition=I, + params_dtype=dtype, + activation="silu", + inplace=False, + ) + + # Note: Triton expects non-transposed weights + triton_lambda = lambda: fused_experts( + x, + w1, + w2, + topk_output, + moe_runner_config, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + + # --- Warmup --- + print("Warming up...") + for _ in range(10): + _ = cutlass_lambda() + _ = triton_lambda() + torch.cuda.synchronize() + + # --- Benchmarking --- + quantiles = [0.5, 0.2, 0.8] + print(f"Benchmarking Cutlass fused_experts...") + cutlass_ms, cutlass_min, cutlass_max = triton.testing.do_bench_cudagraph( + cutlass_lambda, rep=1000, quantiles=quantiles + ) + + print(f"Benchmarking Triton fused_experts...") + triton_ms, triton_min, triton_max = triton.testing.do_bench_cudagraph( + triton_lambda, rep=1000, quantiles=quantiles + ) + print( + f"Cutlass fused_experts time: {cutlass_ms:.3f} ms (median) [{cutlass_min:.3f} - {cutlass_max:.3f}]" + ) + print( + f"Triton fused_experts time: {triton_ms:.3f} ms (median) [{triton_min:.3f} - {triton_max:.3f}]" + ) + + # --- Correctness Check --- + if check: + print("Running correctness check...") + with torch.no_grad(): + # Run CUTLASS version (requires transposed weights) + y_cutlass = cutlass_fused_experts_fp8( + x, + w1.transpose(1, 2), # Transposed + w2.transpose(1, 2), # Transposed + w1_scale.transpose(1, 2), + w2_scale.transpose(1, 2), + topk_weights, + topk_ids, + a1_strides, + c1_strides, + a2_strides, + c2_strides, + workspace, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + expert_offsets, + problem_sizes1, + problem_sizes2, + ) + + # Run Triton version (requires original shape weights, use inplace=False) + y_triton = fused_experts( + x, + w1, # Original shape + w2, # Original shape + topk_output, + moe_runner_config, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=block_shape, + ) + + diff = calc_diff(y_cutlass, y_triton) + print(f"Diff: {diff:.6f}") + + # Tolerance might need adjustment based on FP8 specifics and kernel differences + # FP8 comparisons often require higher tolerance than FP16/BF16 + assert diff < 1e-4, f"Diff too high! {diff}" + print("Correctness check passed.") + + +def main(tp_size=8, batch_sizes=[1, 4, 8, 16, 32, 64, 128, 256, 512], check=False): + model_config = get_model_config(tp_size) + print("Model Config:", model_config) + for batch_size in batch_sizes: + run_test(tp_size, batch_size, model_config, check) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--tp-size", type=int, default=8, help="Tensor Parallel size") + parser.add_argument( + "--batch-sizes", + type=int, + nargs="+", + default=[ + 1, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 2048, + 4096, + 8192, + ], # Adjusted default + help="List of batch sizes to test", + ) + parser.add_argument("--check", action="store_true", help="Enable check mode") + args = parser.parse_args() + + print(f"Running benchmarks with TP size: {args.tp_size}") + print(f"Testing batch sizes: {args.batch_sizes}") + + main(tp_size=args.tp_size, batch_sizes=args.batch_sizes, check=args.check) diff --git a/python/sglang/test/test_cutlass_w4a8_moe.py b/python/sglang/test/test_cutlass_w4a8_moe.py new file mode 100644 index 000000000..6706fc962 --- /dev/null +++ b/python/sglang/test/test_cutlass_w4a8_moe.py @@ -0,0 +1,295 @@ +# SPDX-License-Identifier: Apache-2.0 + +from typing import Literal, Optional + +import pytest +import torch + +from sglang.srt.layers.moe.cutlass_w4a8_moe import cutlass_w4a8_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts + + +def pack_int4_values_to_int8(int4_values_interleaved: torch.Tensor) -> torch.Tensor: + if int4_values_interleaved.shape[-1] % 2 != 0: + raise ValueError( + "the last dim size of int4_values_interleaved tensor must be even." + ) + + input_tensor_int8 = int4_values_interleaved.to(torch.int8) + + low_nibbles = input_tensor_int8[..., 0::2] + high_nibbles = input_tensor_int8[..., 1::2] + + packed_tensor = (high_nibbles << 4) | (low_nibbles & 0x0F) + + return packed_tensor.to(torch.int8) + + +def pack_interleave(num_experts, ref_weight, ref_scale, alignment=4): + n, k = ref_weight.shape[1], ref_weight.shape[2] + + weight = pack_int4_values_to_int8(ref_weight.cpu()).cuda() + w_q = weight.view((num_experts, n, k // 2)).view(torch.int8) + w_q = w_q.contiguous() + + scale_interleaved = ref_scale.reshape( + ref_scale.shape[0], + ref_scale.shape[1], + (ref_scale.shape[2] // alignment), + alignment, + ) # [E, N, K/4, 4] + scale_interleaved = scale_interleaved.permute(0, 2, 1, 3) # [E, K/4, N, 4] + scale_interleaved = scale_interleaved.reshape( + ref_scale.shape[0], + ref_scale.shape[2] // alignment, + ref_scale.shape[1] * alignment, + ) # [E, K/4, N*4] + w_scale = scale_interleaved.contiguous() + + return w_q, w_scale + + +@pytest.mark.parametrize("M", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("N", [2048]) +@pytest.mark.parametrize("K", [7168]) +@pytest.mark.parametrize("E", [256]) +@pytest.mark.parametrize("tp_size", [8]) +@pytest.mark.parametrize("use_ep_moe", [True, False]) +@pytest.mark.parametrize("topk", [8]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +def test_cutlass_w4a8_moe(M, N, K, E, tp_size, use_ep_moe, topk, group_size, dtype): + if use_ep_moe: + local_e = E // tp_size + else: # tp mode + local_e = E + N = N // tp_size + + debug = False + if debug: + a = torch.ones((M, K), dtype=dtype, device="cuda") * 0.001 + ref_weight_1 = torch.ones((local_e, N * 2, K), dtype=torch.int8, device="cuda") + ref_weight_2 = torch.ones((local_e, K, N), dtype=torch.int8, device="cuda") + a1_scale = torch.ones(1, dtype=torch.float32, device="cuda") + a2_scale = torch.ones(1, dtype=torch.float32, device="cuda") + scale_1 = torch.ones( + (local_e, N * 2, K // group_size), dtype=dtype, device="cuda" + ) + scale_2 = torch.ones((local_e, K, N // group_size), dtype=dtype, device="cuda") + else: + a = torch.randn(M, K, dtype=dtype, device="cuda") + ref_weight_1 = torch.randint( + -8, 8, (local_e, N * 2, K), dtype=torch.int8, device="cuda" + ) + ref_weight_2 = torch.randint( + -8, 8, (local_e, K, N), dtype=torch.int8, device="cuda" + ) + affine_coeff = 0.005 + a1_scale = torch.randn(1, dtype=torch.float32, device="cuda") + a2_scale = torch.randn(1, dtype=torch.float32, device="cuda") + scale_1 = ( + torch.randn(local_e, N * 2, K // group_size, dtype=dtype, device="cuda") + * affine_coeff + ) + scale_2 = ( + torch.randn(local_e, K, N // group_size, dtype=dtype, device="cuda") + * affine_coeff + ) + + w1_q, w1_scale = pack_interleave(local_e, ref_weight_1, scale_1) + if use_ep_moe: + w2_q, w2_scale = pack_interleave(local_e, ref_weight_2, scale_2) + else: + w2_q, w2_scale = pack_interleave(local_e, ref_weight_2, scale_2, 1) + + device = "cuda" + a_strides1 = torch.full((local_e, 3), K, device=device, dtype=torch.int64) + c_strides1 = torch.full((local_e, 3), 2 * N, device=device, dtype=torch.int64) + a_strides2 = torch.full((local_e, 3), N, device=device, dtype=torch.int64) + c_strides2 = torch.full((local_e, 3), K, device=device, dtype=torch.int64) + b_strides1 = a_strides1 + s_strides13 = c_strides1 + b_strides2 = a_strides2 + s_strides2 = c_strides2 + + score = torch.randn((M, E), dtype=dtype, device=device) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + topk_weights, topk_ids, _ = topk_output + expert_map = torch.arange(E, dtype=torch.int32, device=device) + expert_map[local_e:] = E + + output = cutlass_moe( + a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids, + a_strides1, + b_strides1, + c_strides1, + a_strides2, + b_strides2, + c_strides2, + s_strides13, + s_strides2, + 0, + local_e - 1, + E, + a1_scale, + a2_scale, + expert_map, + ) + + ref_output = ref( + a, + local_e, + topk_weights, + topk_ids, + ref_weight_1, + ref_weight_2, + scale_1, + scale_2, + has_pre_quant=True, + has_alpha=True, + pre_quant_scale_1=a1_scale, + pre_quant_scale_2=a2_scale, + alpha_1=a1_scale, + alpha_2=a2_scale, + ) + + # compare + torch.cuda.synchronize() + + # compare final output + torch.testing.assert_close(output, ref_output, rtol=1e-2, atol=0.1) + print("SUCCESS: Final output tensors are close.") + + +def cutlass_moe( + a: torch.Tensor, + w1_q: torch.Tensor, + w2_q: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids_: torch.Tensor, + a_strides1: torch.Tensor, + b_strides1: torch.Tensor, + c_strides1: torch.Tensor, + a_strides2: torch.Tensor, + b_strides2: torch.Tensor, + c_strides2: torch.Tensor, + s_strides13: torch.Tensor, + s_strides2: torch.Tensor, + start_expert_id: int, + end_expert_id: int, + E: int, + a1_scale: Optional[torch.Tensor] = None, + a2_scale: Optional[torch.Tensor] = None, + expert_map: Optional[torch.Tensor] = None, + apply_router_weight_on_input: bool = False, +): + local_topk_ids = topk_ids_ + local_topk_ids = torch.where(expert_map[topk_ids_] != E, expert_map[topk_ids_], E) + device = a.device + + local_num_experts = end_expert_id - start_expert_id + 1 + expert_offsets = torch.empty( + (local_num_experts + 1), dtype=torch.int32, device=device + ) + problem_sizes1 = torch.empty( + (local_num_experts, 3), dtype=torch.int32, device=device + ) + problem_sizes2 = torch.empty( + (local_num_experts, 3), dtype=torch.int32, device=device + ) + return cutlass_w4a8_moe( + start_expert_id, + end_expert_id, + E, + a, + w1_q, + w2_q, + w1_scale, + w2_scale, + topk_weights, + topk_ids_, + local_topk_ids, + a_strides1, + b_strides1, + c_strides1, + a_strides2, + b_strides2, + c_strides2, + s_strides13, + s_strides2, + expert_offsets, + problem_sizes1, + problem_sizes2, + a1_scale, + a2_scale, + apply_router_weight_on_input, + ) + + +def ref( + x: torch.Tensor, + num_experts: int, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ref_weight_1: torch.Tensor, + ref_weight_2: torch.Tensor, + ref_weight_scale_1: torch.Tensor, + ref_weight_scale_2: torch.Tensor, + has_pre_quant: bool = False, + has_alpha: bool = False, + pre_quant_scale_1: Optional[torch.Tensor] = None, + pre_quant_scale_2: Optional[torch.Tensor] = None, + alpha_1: Optional[torch.Tensor] = None, + alpha_2: Optional[torch.Tensor] = None, +): + results = torch.zeros_like(x) + dtype = x.dtype + for e_idx in range(num_experts): + mask = topk_ids == e_idx + activated_tokens = mask.sum(1).bool() + act = x[activated_tokens, :] + if act.shape[0] == 0: + continue + final_scale = (topk_weights * mask).sum(1)[activated_tokens].unsqueeze(1) + + act = ( + torch.clamp((act / pre_quant_scale_1.float()), -448.0, 448.0) + .to(torch.float8_e4m3fn) + .to(dtype) + ) + w3_w1 = ref_weight_1[e_idx] + ref_w_scale_repeat = ( + ref_weight_scale_1[e_idx].repeat_interleave(128, dim=1).to(float) + ) + w3_w1 = (w3_w1.to(float) * ref_w_scale_repeat).to(dtype) + fc1 = ((torch.matmul(act, w3_w1.T)) * alpha_1).to(torch.float16) + + gate, fc1 = fc1.chunk(2, dim=-1) + fc1 = fc1 * torch.nn.functional.silu(gate) + act = torch.clamp((fc1 / pre_quant_scale_2.float()), -448.0, 448.0).to( + torch.float8_e4m3fn + ) + act = act.to(dtype) + + w2 = ref_weight_2[e_idx] + ref_w_scale_repeat = ( + ref_weight_scale_2[e_idx].repeat_interleave(128, dim=1).to(float) + ) + w2 = (w2.to(float) * ref_w_scale_repeat).to(dtype) + fc2 = (torch.matmul(act, w2.T) * alpha_2).to(torch.float16) + + results[activated_tokens, :] += (fc2 * final_scale).to(results.dtype) + + return results diff --git a/python/sglang/test/test_deepep_utils.py b/python/sglang/test/test_deepep_utils.py new file mode 100644 index 000000000..aa15b5a0b --- /dev/null +++ b/python/sglang/test/test_deepep_utils.py @@ -0,0 +1,219 @@ +# Copy from deepseek-ai/DeepEP/tests/test_utils.py + +import os +import sys +from typing import Optional + +import numpy as np +import torch +import torch.distributed as dist + + +def init_dist(local_rank: int, num_local_ranks: int): + # NOTES: you may rewrite this function with your own cluster settings + ip = os.getenv("MASTER_ADDR", "127.0.0.1") + port = int(os.getenv("MASTER_PORT", "8361")) + num_nodes = int(os.getenv("WORLD_SIZE", 1)) + node_rank = int(os.getenv("RANK", 0)) + assert (num_local_ranks < 8 and num_nodes == 1) or num_local_ranks == 8 + + dist.init_process_group( + backend="nccl", + init_method=f"tcp://{ip}:{port}", + world_size=num_nodes * num_local_ranks, + rank=node_rank * num_local_ranks + local_rank, + ) + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device("cuda") + torch.cuda.set_device(local_rank) + + return ( + dist.get_rank(), + dist.get_world_size(), + dist.new_group(list(range(num_local_ranks * num_nodes))), + ) + + +def calc_diff(x: torch.Tensor, y: torch.Tensor): + x, y = x.double() + 1, y.double() + 1 + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return (1 - sim).item() + + +def per_token_cast_to_fp8(x: torch.Tensor): + assert x.dim() == 2 and x.size(1) % 128 == 0 + m, n = x.shape + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + return (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn).view( + m, n + ), (x_amax / 448.0).view(m, -1) + + +def per_token_cast_back(x_fp8: torch.Tensor, x_scales: torch.Tensor): + x_fp32 = x_fp8.to(torch.float32).view(x_fp8.size(0), -1, 128) + x_scales = x_scales.view(x_fp8.size(0), -1, 1) + return (x_fp32 * x_scales).view(x_fp8.shape).to(torch.bfloat16) + + +def inplace_unique(x: torch.Tensor, num_slots: int): + assert x.dim() == 2 + mask = x < 0 + x_padded = x.masked_fill(mask, num_slots) + bin_count = torch.zeros((x.size(0), num_slots + 1), dtype=x.dtype, device=x.device) + bin_count.scatter_add_(1, x_padded, torch.ones_like(x_padded)) + bin_count = bin_count[:, :num_slots] + sorted_bin_count, sorted_bin_idx = torch.sort(bin_count, dim=-1, descending=True) + sorted_bin_idx.masked_fill_(sorted_bin_count == 0, -1) + sorted_bin_idx = torch.sort(sorted_bin_idx, descending=True, dim=-1).values + x[:, :].fill_(-1) + valid_len = min(num_slots, x.size(1)) + x[:, :valid_len] = sorted_bin_idx[:, :valid_len] + + +def create_grouped_scores( + scores: torch.Tensor, group_idx: torch.Tensor, num_groups: int +): + num_tokens, num_experts = scores.shape + scores = scores.view(num_tokens, num_groups, -1) + mask = torch.zeros((num_tokens, num_groups), dtype=torch.bool, device=scores.device) + mask = mask.scatter_(1, group_idx, True).unsqueeze(-1).expand_as(scores) + return (scores * mask).view(num_tokens, num_experts) + + +def bench(fn, num_warmups: int = 20, num_tests: int = 30, post_fn=None): + # Flush L2 cache with 256 MB data + torch.cuda.synchronize() + cache = torch.empty(int(256e6 // 4), dtype=torch.int, device="cuda") + + # Warmup + for _ in range(num_warmups): + fn() + + # Flush L2 + cache.zero_() + + # Testing + start_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + end_events = [torch.cuda.Event(enable_timing=True) for _ in range(num_tests)] + for i in range(num_tests): + # Record + start_events[i].record() + fn() + end_events[i].record() + if post_fn is not None: + post_fn() + torch.cuda.synchronize() + + times = np.array( + [s.elapsed_time(e) / 1e3 for s, e in zip(start_events, end_events)] + )[1:] + return np.average(times), np.min(times), np.max(times) + + +class empty_suppress: + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + +class suppress_stdout_stderr: + def __enter__(self): + self.outnull_file = open(os.devnull, "w") + self.errnull_file = open(os.devnull, "w") + + self.old_stdout_fileno_undup = sys.stdout.fileno() + self.old_stderr_fileno_undup = sys.stderr.fileno() + + self.old_stdout_fileno = os.dup(sys.stdout.fileno()) + self.old_stderr_fileno = os.dup(sys.stderr.fileno()) + + self.old_stdout = sys.stdout + self.old_stderr = sys.stderr + + os.dup2(self.outnull_file.fileno(), self.old_stdout_fileno_undup) + os.dup2(self.errnull_file.fileno(), self.old_stderr_fileno_undup) + + sys.stdout = self.outnull_file + sys.stderr = self.errnull_file + return self + + def __exit__(self, *_): + sys.stdout = self.old_stdout + sys.stderr = self.old_stderr + + os.dup2(self.old_stdout_fileno, self.old_stdout_fileno_undup) + os.dup2(self.old_stderr_fileno, self.old_stderr_fileno_undup) + + os.close(self.old_stdout_fileno) + os.close(self.old_stderr_fileno) + + self.outnull_file.close() + self.errnull_file.close() + + +def bench_kineto( + fn, + kernel_names, + num_tests: int = 30, + suppress_kineto_output: bool = False, + trace_path: Optional[str] = None, + barrier_comm_profiling: bool = False, +): + # Profile + suppress = suppress_stdout_stderr if suppress_kineto_output else empty_suppress + with suppress(): + schedule = torch.profiler.schedule(wait=0, warmup=1, active=1, repeat=1) + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CUDA], schedule=schedule + ) as prof: + for i in range(2): + # NOTES: use a large kernel and a barrier to eliminate the unbalanced CPU launch overhead + if barrier_comm_profiling: + lhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + rhs = torch.randn((8192, 8192), dtype=torch.float, device="cuda") + lhs @ rhs + dist.all_reduce(torch.ones(1, dtype=torch.float, device="cuda")) + for _ in range(num_tests): + fn() + prof.step() + + # Parse the profiling table + assert isinstance(kernel_names, str) or isinstance(kernel_names, tuple) + is_tupled = isinstance(kernel_names, tuple) + prof_lines = ( + prof.key_averages() + .table(sort_by="cuda_time_total", max_name_column_width=100) + .split("\n") + ) + kernel_names = (kernel_names,) if isinstance(kernel_names, str) else kernel_names + assert all([isinstance(name, str) for name in kernel_names]) + for name in kernel_names: + assert ( + sum([name in line for line in prof_lines]) == 1 + ), f"Errors of the kernel {name} in the profiling table" + + # Save chrome traces + if trace_path is not None: + prof.export_chrome_trace(trace_path) + + # Return average kernel times + units = {"ms": 1e3, "us": 1e6} + kernel_times = [] + for name in kernel_names: + for line in prof_lines: + if name in line: + time_str = line.split()[-2] + for unit, scale in units.items(): + if unit in time_str: + kernel_times.append(float(time_str.replace(unit, "")) / scale) + break + break + return tuple(kernel_times) if is_tupled else kernel_times[0] + + +def hash_tensor(t: torch.Tensor): + return t.view(torch.int64).sum().item() diff --git a/python/sglang/test/test_disaggregation_utils.py b/python/sglang/test/test_disaggregation_utils.py new file mode 100644 index 000000000..f61b71a9d --- /dev/null +++ b/python/sglang/test/test_disaggregation_utils.py @@ -0,0 +1,66 @@ +import time + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + CustomTestCase, + popen_with_error_check, +) + + +class TestDisaggregationBase(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.process_lb, cls.process_decode, cls.process_prefill = None, None, None + pass + + @classmethod + def launch_lb(cls): + lb_command = [ + "python3", + "-m", + "sglang_router.launch_router", + "--pd-disaggregation", + "--mini-lb", # FIXME: remove this + "--prefill", + cls.prefill_url, + "--decode", + cls.decode_url, + "--host", + cls.base_host, + "--port", + cls.lb_port, + ] + print("Starting load balancer:", " ".join(lb_command)) + cls.process_lb = popen_with_error_check(lb_command) + cls.wait_server_ready(cls.lb_url + "/health") + + @classmethod + def wait_server_ready(cls, url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH): + start_time = time.perf_counter() + while True: + try: + response = requests.get(url) + if response.status_code == 200: + print(f"Server {url} is ready") + return + except Exception: + pass + + if time.perf_counter() - start_time > timeout: + raise RuntimeError(f"Server {url} failed to start in {timeout}s") + time.sleep(1) + + @classmethod + def tearDownClass(cls): + for process in [cls.process_lb, cls.process_decode, cls.process_prefill]: + if process: + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process {process.pid}: {e}") + + # wait for 5 seconds + time.sleep(5) diff --git a/python/sglang/test/test_dynamic_grad_mode.py b/python/sglang/test/test_dynamic_grad_mode.py new file mode 100644 index 000000000..078c3139d --- /dev/null +++ b/python/sglang/test/test_dynamic_grad_mode.py @@ -0,0 +1,58 @@ +import unittest + +import torch + +from sglang.srt.utils import DynamicGradMode +from sglang.test.test_utils import CustomTestCase + + +class TestDynamicGradMode(CustomTestCase): + def test_inference(self): + # Test inference_mode + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_x(): + return torch.empty(0) + + X = create_tensor_x() + self.assertTrue(not X.requires_grad and X.is_inference()) + + def test_no_grad(self): + # Test no_grad + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_y(): + return torch.empty(0) + + Y = create_tensor_y() + self.assertTrue(not Y.requires_grad and not Y.is_inference()) + + def test_nested_inference(self): + # Test no_grad nested inference_mode, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(False) + + @DynamicGradMode() + def create_tensor_z(): + with torch.inference_mode(): + return torch.empty(0) + + Z = create_tensor_z() + self.assertTrue(not Z.requires_grad and Z.is_inference()) + + def test_nested_no_grad(self): + # Test inference_mode nested no_grad, inference_mode should has higher priority + DynamicGradMode.set_inference_mode(True) + + @DynamicGradMode() + def create_tensor_w(): + with torch.no_grad(): + return torch.empty(0) + + W = create_tensor_w() + self.assertTrue(not W.requires_grad and W.is_inference()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_fp4_moe.py b/python/sglang/test/test_fp4_moe.py new file mode 100644 index 000000000..8f8c8e8a7 --- /dev/null +++ b/python/sglang/test/test_fp4_moe.py @@ -0,0 +1,329 @@ +# SPDX-License-Identifier: Apache-2.0 +from typing import Callable + +import pytest +import torch +from flashinfer.fused_moe import cutlass_fused_moe as flashinfer_cutlass_fused_moe +from sgl_kernel import scaled_fp4_quant + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.cutlass_moe import cutlass_moe_fp4 +from sglang.srt.layers.moe.cutlass_moe_params import CutlassMoEParams, CutlassMoEType +from sglang.srt.layers.moe.topk import TopKConfig, select_experts + +if torch.cuda.get_device_capability() < (10, 0): + pytest.skip( + reason="Nvfp4 Requires compute capability of 10 or above.", + allow_module_level=True, + ) + +kE2M1ToFloat = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], dtype=torch.float32 +) + +FLOAT8_E4M3_MAX = 448.0 +FLOAT4_E2M1_MAX = 6.0 + + +def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): + m_tiles = (m + 128 - 1) // 128 + f = block_size * 4 + k_tiles = (k + f - 1) // f + tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size) + return out[0:m, 0:k] + + +def dequantize_nvfp4_to_dtype( + tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16 +): + """Dequantize the fp4 tensor back to high precision.""" + # Two fp4 values are packed into one uint8. + assert tensor_fp4.dtype == torch.uint8 + m, packed_k = tensor_fp4.shape + k = packed_k * 2 + tensor_f32 = break_fp4_bytes(tensor_fp4, dtype) + tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_sf = tensor_sf.view(torch.float8_e4m3fn) + tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale + + # scale the tensor + out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + return out.to(dtype=dtype) + + +def break_fp4_bytes(a, dtype): + assert a.dtype == torch.uint8 + m, n = a.shape + + # Vectorized nibble processing + a_flat = a.flatten() + high = (a_flat & 0xF0) >> 4 # Upper nibbles + low = a_flat & 0x0F # Lower nibbles + + # Combine nibbles for batch processing + combined = torch.stack((low, high), dim=1).flatten() + + # Vectorized sign and magnitude extraction + signs = (combined & 0x08).to(torch.bool) # Sign bits + abs_vals = (combined & 0x07).to(torch.long) # Magnitude indices + + # Device-aware lookup and sign application + kE2M1 = kE2M1ToFloat.to(device=a.device) + values = kE2M1[abs_vals] * torch.where(signs, -1.0, 1.0) + + # Reshape to final form + return values.reshape(m, n * 2).to(dtype=dtype) + + +MNK_FACTORS = [ + (2, 1024, 1024), + (2, 1024, 1536), + (2, 3072, 1024), + (2, 3072, 1536), + (64, 1024, 1024), + (64, 1024, 1536), + (64, 3072, 1024), + (64, 2048, 1024), + (224, 1024, 1024), + (224, 1024, 1536), +] + + +# Reference implementation of torch_moe +def torch_moe(a, w1, w2, score, topk, expert_map): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + if expert_map is not None: + topk_ids = expert_map[topk_ids] + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose( + 0, 1 + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +def check_moe( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + moe_impl: Callable, + flip_w13: bool, +): + torch.manual_seed(7) + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + quant_blocksize = 16 + round_up = lambda x, y: (x + y - 1) // y * y + sf_w1_2n = round_up(2 * n, 128) + sf_w1_k = round_up(k // quant_blocksize, 4) + w1_blockscale = torch.empty( + (e, sf_w1_2n, sf_w1_k), device="cuda", dtype=torch.float8_e4m3fn + ) + + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + sf_w2_k = round_up(k, 128) + sf_w2_n = round_up(n // quant_blocksize, 4) + w2_blockscale = torch.empty( + (e, sf_w2_k, sf_w2_n), device="cuda", dtype=torch.float8_e4m3fn + ) + + w1_q = torch.empty((e, 2 * n, k // 2), device="cuda", dtype=torch.uint8) + w2_q = torch.empty((e, k, n // 2), device="cuda", dtype=torch.uint8) + w1_gs = torch.empty((e,), device="cuda", dtype=torch.float32) + w2_gs = torch.empty((e,), device="cuda", dtype=torch.float32) + + for expert in range(e): + w1_amax = torch.abs(w1).max().to(torch.float32) + w2_amax = torch.abs(w2).max().to(torch.float32) + w1_gs[expert] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w1_amax + w2_gs[expert] = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / w2_amax + + w1_q[expert], w1_blockscale[expert] = scaled_fp4_quant( + w1[expert], w1_gs[expert] + ) + + w2_q[expert], w2_blockscale[expert] = scaled_fp4_quant( + w2[expert], w2_gs[expert] + ) + + score = torch.randn((m, e), device="cuda", dtype=dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + topk_weights, topk_ids, _ = topk_output + + a1_gs = torch.ones((e,), device="cuda", dtype=torch.float32) + a2_gs = torch.ones((e,), device="cuda", dtype=torch.float32) + test_output = moe_impl( + a=a, + topk_weights=topk_weights, + topk_ids=topk_ids, + w1_q=w1_q, + w2_q=w2_q, + a1_gs=a1_gs, + w1_blockscale=w1_blockscale, + w1_alphas=(1 / w1_gs), + a2_gs=a2_gs, + w2_blockscale=w2_blockscale, + w2_alphas=(1 / w2_gs), + ) + + # Reference check: + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a.flatten(), dim=-1) + ).to(torch.float32) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a, a_global_scale) + _, m_k = a_fp4.shape + a_in_dtype = dequantize_nvfp4_to_dtype( + a_fp4, + a_scale_interleaved, + a_global_scale, + dtype=a.dtype, + device=a.device, + block_size=quant_blocksize, + ) + + w1_d = torch.empty((e, 2 * n, k), device="cuda", dtype=dtype) + w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype) + + for idx in range(0, e): + w1_d[idx] = dequantize_nvfp4_to_dtype( + w1_q[idx], + w1_blockscale[idx], + w1_gs[idx], + dtype=w1.dtype, + device=w1.device, + block_size=quant_blocksize, + ) + w2_d[idx] = dequantize_nvfp4_to_dtype( + w2_q[idx], + w2_blockscale[idx], + w2_gs[idx], + dtype=w2.dtype, + device=w2.device, + block_size=quant_blocksize, + ) + + if flip_w13: + dim = -2 + size = w1_d.size(dim) + assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}" + half = size // 2 + # Reorder weight + w1, w3 = w1_d.split(half, dim=dim) + w1_d = torch.cat([w3, w1], dim=dim).contiguous() + + torch_output = torch_moe(a_in_dtype, w1_d, w2_d, score, topk, None) + + torch.testing.assert_close(torch_output, test_output, atol=1e-1, rtol=1e-1) + + +@pytest.mark.parametrize("m,n,k", MNK_FACTORS) +@pytest.mark.parametrize("e", [40, 64, 256]) +@pytest.mark.parametrize("topk", [1, 6, 8]) +@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16]) +@torch.inference_mode() +def test_cutlass_fp4_moe_no_graph( + m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype +): + def cutlass_moe_impl( + a, + topk_weights, + topk_ids, + w1_q, + w2_q, + a1_gs, + w1_blockscale, + w1_alphas, + a2_gs, + w2_blockscale, + w2_alphas, + ): + params = CutlassMoEParams( + CutlassMoEType.BlockscaledFP4, + device=a.device, + num_experts=e, + intermediate_size_per_partition=n, # n + hidden_size=k, + ) # k + return cutlass_moe_fp4( + a=a, + a1_gscale=a1_gs, + w1_fp4=w1_q, + w1_blockscale=w1_blockscale, + w1_alphas=w1_alphas, + a2_gscale=a2_gs, + w2_fp4=w2_q, + w2_blockscale=w2_blockscale, + w2_alphas=w2_alphas, + topk_weights=topk_weights, + topk_ids=topk_ids, + params=params, + apply_router_weight_on_input=False, + ) + + check_moe(m, n, k, e, topk, dtype, cutlass_moe_impl, flip_w13=False) + + +@pytest.mark.parametrize("m,n,k", MNK_FACTORS) +@pytest.mark.parametrize("e", [40, 64, 256]) +@pytest.mark.parametrize("topk", [1, 6, 8]) +@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16]) +@torch.inference_mode() +def test_flashinfer_fp4_moe_no_graph( + m: int, n: int, k: int, e: int, topk: int, dtype: torch.dtype +): + def flashinfer_moe_impl( + a, + topk_weights, + topk_ids, + w1_q, + w2_q, + a1_gs, + w1_blockscale, + w1_alphas, + a2_gs, + w2_blockscale, + w2_alphas, + ): + return flashinfer_cutlass_fused_moe( + a, + topk_ids.to(torch.int), + topk_weights, + w1_q.view(torch.long), + w2_q.view(torch.long), + a.dtype, + quant_scales=[ + a1_gs, + w1_blockscale.view(torch.int32), + w1_alphas, + a2_gs, + w2_blockscale.view(torch.int32), + w2_alphas, + ], + )[0] + + check_moe(m, n, k, e, topk, dtype, flashinfer_moe_impl, flip_w13=True) + + +if __name__ == "__main__": + test_cutlass_fp4_moe_no_graph(224, 1024, 1024, 256, 8, torch.half) + test_flashinfer_fp4_moe_no_graph(224, 1024, 1024, 256, 8, torch.half) diff --git a/python/sglang/test/test_layernorm.py b/python/sglang/test/test_layernorm.py new file mode 100644 index 000000000..05b6593eb --- /dev/null +++ b/python/sglang/test/test_layernorm.py @@ -0,0 +1,113 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.layernorm import GemmaRMSNorm, RMSNorm +from sglang.test.test_utils import CustomTestCase + + +class TestRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_rms_norm_test(self, num_tokens, hidden_size, add_residual, dtype, seed): + torch.manual_seed(seed) + + layer = RMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-2, rtol=1e-2)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-2, rtol=1e-2)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-2, rtol=1e-2)) + + def test_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_rms_norm_test(*params) + + +class TestGemmaRMSNorm(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + NUM_TOKENS = [7, 83, 4096] + HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192, 8199] + ADD_RESIDUAL = [False, True] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_gemma_rms_norm_test( + self, num_tokens, hidden_size, add_residual, dtype, seed + ): + torch.manual_seed(seed) + + layer = GemmaRMSNorm(hidden_size).to(dtype=dtype) + layer.weight.data.normal_(mean=1.0, std=0.1) + scale = 1 / (2 * hidden_size) + x = torch.randn(num_tokens, hidden_size, dtype=dtype) * scale + residual = torch.randn_like(x) * scale if add_residual else None + + with torch.inference_mode(): + ref_out = layer.forward_native(x, residual) + out = layer(x, residual) + + if add_residual: + self.assertTrue(torch.allclose(out[0], ref_out[0], atol=1e-3, rtol=1e-3)) + self.assertTrue(torch.allclose(out[1], ref_out[1], atol=1e-3, rtol=1e-3)) + else: + self.assertTrue(torch.allclose(out, ref_out, atol=1e-3, rtol=1e-3)) + + def test_gemma_rms_norm(self): + for params in itertools.product( + self.NUM_TOKENS, + self.HIDDEN_SIZES, + self.ADD_RESIDUAL, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + num_tokens=params[0], + hidden_size=params[1], + add_residual=params[2], + dtype=params[3], + seed=params[4], + ): + self._run_gemma_rms_norm_test(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/python/sglang/test/test_marlin_moe.py b/python/sglang/test/test_marlin_moe.py new file mode 100644 index 000000000..77b0109df --- /dev/null +++ b/python/sglang/test/test_marlin_moe.py @@ -0,0 +1,286 @@ +import types +from typing import Optional + +import pytest +import torch +from sgl_kernel import fused_marlin_moe +from sgl_kernel.scalar_type import ScalarType, scalar_types + +from sglang.srt.layers.activation import SiluAndMul +from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize + + +def stack_and_dev(tensors: list[torch.Tensor]): + dev = tensors[0].device + return torch.stack(tensors, dim=0).to(dev) + + +def torch_experts( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weight: torch.Tensor, + topk_ids: torch.Tensor, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + quant_dtype: Optional[torch.dtype] = None, + apply_router_weights_on_input: bool = False, +) -> torch.Tensor: + assert ( + global_num_experts == -1 + or (global_num_experts == w1.shape[0] and expert_map is None) + or (expert_map is not None and global_num_experts == expert_map.shape[0]) + ) + + M, K = a.shape + topk = topk_ids.shape[1] + print("quant_dtype", quant_dtype) + # exit(0) + if apply_router_weights_on_input: + assert topk == 1 + a = a * topk_weight.to(a.dtype) + + a = a.view(M, -1, K).repeat(1, topk, 1).reshape(-1, K) + + out = torch.zeros(M * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + num_experts = w1.shape[0] + + topk_ids = topk_ids.view(-1) + if expert_map is not None: + topk_ids = expert_map[topk_ids] + + f32 = torch.float32 + + for i in range(num_experts): + mask = topk_ids == i + if mask.sum(): + if quant_dtype is None: + tmp1 = a[mask] @ w1[i].transpose(0, 1) + tmp2 = SiluAndMul()(tmp1) + out[mask] = tmp2 @ w2[i].transpose(0, 1) + + if apply_router_weights_on_input: + return out + else: + return ( + (out.view(M, -1, w2.shape[1]).to(f32) * topk_weight.view(M, -1, 1)) + .sum(dim=1) + .to(out.dtype) + ) + + +def torch_moe( + a: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + score: torch.Tensor, + topk: int, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, +) -> torch.Tensor: + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + return torch_experts( + a, w1, w2, topk_weight, topk_ids, global_num_experts, expert_map + ) + + +def marlin_moe_generate_valid_test_cases(): + import itertools + + m_list = [1, 123, 666] + n_list = [128, 1024] + k_list = [256, 2048] + e_list = [4, 12] + topk_list = [2, 3] + dtype_list = [torch.half, torch.bfloat16] + group_size_list = [128] + act_order_list = [True, False] + quant_type_list = [ + scalar_types.uint4, + scalar_types.uint4b8, + ] + is_k_full_list = [True, False] + + all_combinations = itertools.product( + m_list, + n_list, + k_list, + e_list, + topk_list, + dtype_list, + group_size_list, + act_order_list, + quant_type_list, + is_k_full_list, + ) + + def is_invalid( + m, n, k, e, topk, dtype, group_size, act_order, quant_type, is_k_full + ): + + # Filter act_order + if act_order: + if group_size in (-1, k, n): + return False + if quant_type not in [scalar_types.uint4b8]: + return False + elif not is_k_full: + return False + + return True + + cases = [] + for case in all_combinations: + if is_invalid(*case): + cases.append(case) + return cases + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize( + ("m, n, k, e, topk, dtype, group_size," "act_order, quant_type, is_k_full"), + marlin_moe_generate_valid_test_cases(), +) +def test_fused_marlin_moe( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + group_size: int, + act_order: bool, + quant_type: ScalarType, + is_k_full: bool, +): + if not torch.cuda.is_available(): + pytest.skip("CUDA device not available") + + torch.manual_seed(0) + + has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8] + + # Filter act_order + if act_order: + if group_size == -1: + return + if group_size in (k, n): + return + if has_zp: + return + else: + if not is_k_full: + return + + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 20 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 20 + + e_map = None + + w_ref1_l = [] + qweight1_l = [] + scales1_l = [] + zeros1_l = [] + g_idx1_l = [] + sort_indices1_l = [] + + for i in range(w1.shape[0]): + if has_zp: + w_ref1, qweight1, scales1, zeros1 = awq_marlin_quantize( + w1[i].transpose(1, 0), quant_type, group_size + ) + + w_ref1_l.append(w_ref1.T) + qweight1_l.append(qweight1) + scales1_l.append(scales1) + zeros1_l.append(zeros1) + else: + test_perm = torch.randperm(k) + w_ref1, qweight1, scales1, g_idx1, sort_indices1, _ = marlin_quantize( + w1[i].transpose(1, 0), quant_type, group_size, act_order, test_perm + ) + + w_ref1_l.append(w_ref1.T) + qweight1_l.append(qweight1) + scales1_l.append(scales1) + g_idx1_l.append(g_idx1) + sort_indices1_l.append(sort_indices1) + + w_ref1 = stack_and_dev(w_ref1_l) + qweight1 = stack_and_dev(qweight1_l).contiguous() + scales1 = stack_and_dev(scales1_l) + g_idx1 = stack_and_dev(g_idx1_l) if g_idx1_l else None + zeros1 = stack_and_dev(zeros1_l) if zeros1_l else None + sort_indices1 = stack_and_dev(sort_indices1_l) if sort_indices1_l else None + + w_ref2_l = [] + qweight2_l = [] + scales2_l = [] + zeros2_l = [] + g_idx2_l = [] + sort_indices2_l = [] + + for i in range(w2.shape[0]): + if has_zp: + w_ref2, qweight2, scales2, zeros2 = awq_marlin_quantize( + w2[i].transpose(1, 0), quant_type, group_size + ) + + w_ref2_l.append(w_ref2.T) + qweight2_l.append(qweight2) + scales2_l.append(scales2) + zeros2_l.append(zeros2) + else: + test_perm = torch.randperm(n) + w_ref2, qweight2, scales2, g_idx2, sort_indices2, _ = marlin_quantize( + w2[i].transpose(1, 0), quant_type, group_size, act_order, test_perm + ) + + w_ref2_l.append(w_ref2.T) + qweight2_l.append(qweight2) + scales2_l.append(scales2) + g_idx2_l.append(g_idx2) + sort_indices2_l.append(sort_indices2) + + w_ref2 = stack_and_dev(w_ref2_l) + qweight2 = stack_and_dev(qweight2_l).contiguous() + scales2 = stack_and_dev(scales2_l) + g_idx2 = stack_and_dev(g_idx2_l) if g_idx2_l else None + zeros2 = stack_and_dev(zeros2_l) if zeros2_l else None + sort_indices2 = stack_and_dev(sort_indices2_l) if sort_indices2_l else None + + score = torch.randn((m, e), device="cuda", dtype=dtype) + from sglang.srt.layers.moe.topk import fused_topk_torch_native + + topk_weights, topk_ids = fused_topk_torch_native(a, score, topk, False) + + torch_output = torch_moe(a, w_ref1, w_ref2, score, topk, expert_map=e_map) + + marlin_output = fused_marlin_moe( + a, + qweight1, + qweight2, + scales1, + scales2, + score, + topk_weights, + topk_ids, + g_idx1=g_idx1, + g_idx2=g_idx2, + sort_indices1=sort_indices1, + sort_indices2=sort_indices2, + w1_zeros=zeros1, + w2_zeros=zeros2, + num_bits=4, + is_k_full=is_k_full, + ) + + torch.testing.assert_close(marlin_output, torch_output, atol=5e-2, rtol=0) + + +if __name__ == "__main__": + # Run the specific test function directly + pytest.main([__file__]) diff --git a/python/sglang/test/test_marlin_utils.py b/python/sglang/test/test_marlin_utils.py new file mode 100644 index 000000000..0c0590077 --- /dev/null +++ b/python/sglang/test/test_marlin_utils.py @@ -0,0 +1,171 @@ +""" +Adapted from +https://github.com/vllm-project/vllm/blob/020f58abcdea65302225663130d08fd8f4dd755a/vllm/model_executor/layers/quantization/utils/marlin_utils_test.py +""" + +# SPDX-License-Identifier: Apache-2.0 +"""Utility functions used for tests and benchmarks""" + +from typing import Optional + +import numpy as np +import torch +from sgl_kernel.scalar_type import ScalarType + +from sglang.srt.layers.quantization.marlin_utils import ( + GPTQ_MARLIN_TILE, + marlin_permute_scales, + marlin_zero_points, +) +from sglang.srt.layers.quantization.utils import ( + get_pack_factor, + gptq_quantize_weights, + quantize_weights, + sort_weights, +) + + +class MarlinWorkspace: + + def __init__(self, out_features, min_thread_n, max_parallel): + assert ( + out_features % min_thread_n == 0 + ), "out_features = {} is undivisible by min_thread_n = {}".format( + out_features, min_thread_n + ) + + max_workspace_size = (out_features // min_thread_n) * max_parallel + + self.scratch = torch.zeros(max_workspace_size, dtype=torch.int, device="cuda") + + +def marlin_permute_weights(q_w, size_k, size_n, perm, tile=GPTQ_MARLIN_TILE): + assert q_w.shape == (size_k, size_n) + assert size_k % tile == 0, f"size_k = {size_k}, tile = {tile}" + assert size_n % tile == 0, f"size_k = {size_n}, tile = {tile}" + + # Permute weights to 16x64 marlin tiles + q_w = q_w.reshape((size_k // tile, tile, size_n // tile, tile)) + q_w = q_w.permute((0, 2, 1, 3)) + q_w = q_w.reshape((size_k // tile, size_n * tile)) + + q_w = q_w.reshape((-1, perm.numel()))[:, perm].reshape(q_w.shape) + + return q_w + + +def marlin_weights(q_w, size_k, size_n, num_bits, perm): + # Permute + q_w = marlin_permute_weights(q_w, size_k, size_n, perm) + + # Pack + pack_factor = get_pack_factor(num_bits) + orig_device = q_w.device + + q_w = q_w.cpu().numpy().astype(np.uint32) + + q_packed = np.zeros((q_w.shape[0], q_w.shape[1] // pack_factor), dtype=np.uint32) + for i in range(pack_factor): + q_packed |= q_w[:, i::pack_factor] << num_bits * i + + q_packed = torch.from_numpy(q_packed.astype(np.int32)).to(orig_device) + + return q_packed + + +def get_weight_perm(num_bits: int): + perm_list: list[int] = [] + for i in range(32): + perm1: list[int] = [] + col = i // 4 + for block in [0, 1]: + for row in [ + 2 * (i % 4), + 2 * (i % 4) + 1, + 2 * (i % 4 + 4), + 2 * (i % 4 + 4) + 1, + ]: + perm1.append(16 * row + col + 8 * block) + for j in range(4): + perm_list.extend([p + 256 * j for p in perm1]) + + perm = np.array(perm_list) + + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + perm = perm.reshape((-1, len(interleave)))[:, interleave].ravel() + perm = torch.from_numpy(perm) + return perm + + +def marlin_quantize( + w: torch.Tensor, + quant_type: ScalarType, + group_size: int, + act_order: bool, + test_perm: Optional[torch.Tensor] = None, +): + size_k, size_n = w.shape + num_bits = quant_type.size_bits + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Quantize (and apply act_order if provided) + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights( + w, quant_type, group_size, act_order, test_perm + ) + + # For act_order, sort the "weights" and "g_idx" so that group ids are + # increasing + sort_indices = torch.empty(0, dtype=torch.int, device=w.device) + if act_order: + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx) + + # Reformat to marlin + weight_perm = get_weight_perm(num_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, rand_perm] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list + + +def awq_marlin_quantize(w: torch.Tensor, quant_type: ScalarType, group_size: int): + size_k, size_n = w.shape + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + # Detect num groups + assert size_k % group_size == 0 + num_groups = size_k // group_size + + # Quantize with zp + w_ref, q_w, s, zp = quantize_weights(w, quant_type, group_size, zero_points=True) + + # Reformat to marlin + weight_perm = get_weight_perm(quant_type.size_bits) + marlin_q_w = marlin_weights(q_w, size_k, size_n, quant_type.size_bits, weight_perm) + marlin_s = marlin_permute_scales(s, size_k, size_n, group_size) + marlin_zp = marlin_zero_points(zp, num_groups, size_n, quant_type.size_bits) + + # Create result + res_list = [w_ref, marlin_q_w, marlin_s, marlin_zp] + for i in range(len(res_list)): + res_list[i] = res_list[i].to(w.device) + + return res_list diff --git a/python/sglang/test/test_programs.py b/python/sglang/test/test_programs.py new file mode 100644 index 000000000..6756f2dd7 --- /dev/null +++ b/python/sglang/test/test_programs.py @@ -0,0 +1,575 @@ +"""This file contains the SGL programs used for unit testing.""" + +import json +import re +import time + +import numpy as np + +import sglang as sgl +from sglang.utils import download_and_cache_file, read_jsonl + + +def test_few_shot_qa(): + @sgl.function + def few_shot_qa(s, question): + s += "The following are questions with answers.\n\n" + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: What is the capital of Germany?\n" + s += "A: Berlin\n" + s += "Q: What is the capital of Italy?\n" + s += "A: Rome\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n", temperature=0) + + ret = few_shot_qa.run(question="What is the capital of the United States?") + assert "washington" in ret["answer"].strip().lower(), f"answer: {ret['answer']}" + + rets = few_shot_qa.run_batch( + [ + {"question": "What is the capital of Japan?"}, + {"question": "What is the capital of the United Kingdom?"}, + {"question": "What is the capital city of China?"}, + ], + temperature=0.1, + ) + answers = [x["answer"].strip().lower() for x in rets] + assert answers == ["tokyo", "london", "beijing"], f"answers: {answers}" + + +def test_mt_bench(): + @sgl.function + def answer_mt_bench(s, question_1, question_2): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question_1) + s += sgl.assistant(sgl.gen("answer_1")) + with s.user(): + s += question_2 + with s.assistant(): + s += sgl.gen("answer_2") + + question_1 = "Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions." + question_2 = ( + "Rewrite your previous response. Start every sentence with the letter A." + ) + ret = answer_mt_bench.run( + question_1=question_1, question_2=question_2, temperature=0.7, max_new_tokens=64 + ) + assert len(ret.messages()) in [4, 5] + + +def test_select(check_answer): + @sgl.function + def true_or_false(s, statement): + s += "Determine whether the statement below is True, False, or Unknown.\n" + s += "Statement: The capital of France is Pairs.\n" + s += "Answer: True\n" + s += "Statement: " + statement + "\n" + s += "Answer:" + sgl.select("answer", ["True", "False", "Unknown"]) + + ret = true_or_false.run( + statement="The capital of Germany is Berlin.", + ) + if check_answer: + assert ret["answer"] == "True", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="The capital of Canada is Tokyo.", + ) + if check_answer: + assert ret["answer"] == "False", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + ret = true_or_false.run( + statement="Purple is a better color than green.", + ) + if check_answer: + assert ret["answer"] == "Unknown", ret.text() + else: + assert ret["answer"] in ["True", "False", "Unknown"] + + +def test_decode_int(): + @sgl.function + def decode_int(s): + s += "The number of hours in a day is " + sgl.gen_int("hours") + "\n" + s += "The number of days in a year is " + sgl.gen_int("days") + "\n" + + ret = decode_int.run(temperature=0.1) + assert int(ret["hours"]) == 24, ret.text() + assert int(ret["days"]) == 365, ret.text() + + +def test_decode_json_regex(): + @sgl.function + def decode_json(s): + from sglang.lang.ir import REGEX_FLOAT, REGEX_INT, REGEX_STR + + s += "Generate a JSON object to describe the basic city information of Paris.\n" + s += "Here are the JSON object:\n" + + # NOTE: we recommend using dtype gen or whole regex string to control the output + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen(regex=REGEX_STR) + ",\n" + s += ' "population": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "area": ' + sgl.gen(regex=REGEX_INT, stop=[" ", "\n"]) + ",\n" + s += ' "latitude": ' + sgl.gen(regex=REGEX_FLOAT, stop=[" ", "\n"]) + "\n" + s += "}" + + ret = decode_json.run(temperature=0.0) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_decode_json(): + @sgl.function + def decode_json(s): + s += "Generate a JSON object to describe the basic city information of Paris.\n" + + with s.var_scope("json_output"): + s += "{\n" + s += ' "name": ' + sgl.gen_string() + ",\n" + s += ' "population": ' + sgl.gen_int() + ",\n" + s += ' "area": ' + sgl.gen(dtype=int) + ",\n" + s += ' "country": ' + sgl.gen_string() + ",\n" + s += ' "timezone": ' + sgl.gen(dtype=str) + "\n" + s += "}" + + ret = decode_json.run(max_new_tokens=64) + try: + js_obj = json.loads(ret["json_output"]) + except json.decoder.JSONDecodeError: + print("JSONDecodeError", ret["json_output"]) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + +def test_expert_answer(check_answer=True): + @sgl.function + def expert_answer(s, question): + s += "Question: " + question + "\n" + s += ( + "A good person to answer this question is" + + sgl.gen("expert", stop=[".", "\n"]) + + ".\n" + ) + s += ( + "For example," + + s["expert"] + + " would answer that " + + sgl.gen("answer", stop=".") + + "." + ) + + ret = expert_answer.run(question="What is the capital of France?", temperature=0.1) + + if check_answer: + assert "paris" in ret.text().lower(), f"Answer: {ret.text()}" + + +def test_tool_use(): + def calculate(expression): + return f"{eval(expression)}" + + @sgl.function + def tool_use(s, lhs, rhs): + s += "Please perform computations using a calculator. You can use calculate(expression) to get the results.\n" + s += "For example,\ncalculate(1+2)=3\ncalculate(3*4)=12\n" + s += "Question: What is the product of " + str(lhs) + " and " + str(rhs) + "?\n" + s += ( + "Answer: The answer is calculate(" + + sgl.gen("expression", stop=")") + + ") = " + ) + with s.var_scope("answer"): + s += calculate(s["expression"]) + + lhs, rhs = 257, 983 + ret = tool_use(lhs=lhs, rhs=rhs, temperature=0) + assert int(ret["answer"]) == lhs * rhs + + +def test_react(): + @sgl.function + def react(s, question): + s += """ +Question: Which country does the founder of Microsoft live in? +Thought 1: I need to search for the founder of Microsoft. +Action 1: Search [Founder of Microsoft]. +Observation 1: The founder of Microsoft is Bill Gates. +Thought 2: I need to search for the country where Bill Gates lives in. +Action 2: Search [Where does Bill Gates live]. +Observation 2: Bill Gates lives in the United States. +Thought 3: The answer is the United States. +Action 3: Finish [United States].\n +""" + + s += "Question: " + question + "\n" + + for i in range(1, 5): + s += f"Thought {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + s += f"Action {i}: " + sgl.select(f"action_{i}", ["Search", "Finish"]) + + if s[f"action_{i}"] == "Search": + s += " [" + sgl.gen(stop="]") + "].\n" + s += f"Observation {i}:" + sgl.gen(stop=[".", "\n"]) + ".\n" + else: + s += " [" + sgl.gen("answer", stop="]") + "].\n" + break + + ret = react.run( + question="What country does the creator of Linux live in?", + temperature=0.1, + ) + answer = ret["answer"].lower() + assert "finland" in answer or "states" in answer + + +def test_parallel_decoding(): + max_tokens = 64 + fork_size = 5 + + @sgl.function + def parallel_decoding(s, topic): + s += "Act as a helpful assistant.\n" + s += "USER: Give some tips for " + topic + ".\n" + s += ( + "ASSISTANT: Okay. Here are " + + str(fork_size) + + " concise tips, each under 8 words:\n" + ) + + # Generate skeleton + for i in range(1, 1 + fork_size): + s += f"{i}." + sgl.gen(max_tokens=16, stop=[".", "\n"]) + ".\n" + + # Generate detailed tips + forks = s.fork(fork_size) + for i in range(fork_size): + forks[ + i + ] += f"Now, I expand tip {i+1} into a detailed paragraph:\nTip {i+1}:" + forks[i] += sgl.gen("detailed_tip", max_tokens, stop=["\n\n"]) + forks.join() + + # Concatenate tips and summarize + s += "Here are these tips with detailed explanation:\n" + for i in range(fork_size): + s += f"Tip {i+1}:" + forks[i]["detailed_tip"] + "\n" + + s += "\nIn summary," + sgl.gen("summary", max_tokens=512) + + ret = parallel_decoding.run(topic="writing a good blog post", temperature=0.3) + assert isinstance(ret["summary"], str) + + +def test_parallel_encoding(check_answer=True): + max_tokens = 64 + + @sgl.function + def parallel_encoding(s, question, context_0, context_1, context_2): + s += "USER: I will ask a question based on some statements.\n" + s += "ASSISTANT: Sure. I will give the answer.\n" + s += "USER: Please memorize these statements.\n" + + contexts = [context_0, context_1, context_2] + + forks = s.fork(len(contexts)) + forks += lambda i: f"Statement {i}: " + contexts[i] + "\n" + forks.join(mode="concate_and_append") + + s += "Now, please answer the following question. " "Do not list options." + s += "\nQuestion: " + question + "\n" + s += "ASSISTANT:" + sgl.gen("answer", max_tokens=max_tokens) + + ret = parallel_encoding.run( + question="Who is the father of Julian?", + context_0="Ethan is the father of Liam.", + context_1="Noah is the father of Julian.", + context_2="Oliver is the father of Carlos.", + temperature=0, + ) + answer = ret["answer"] + + if check_answer: + assert "Noah" in answer + + +def test_image_qa(): + @sgl.function + def image_qa(s, question): + s += sgl.user(sgl.image("example_image.png") + question) + s += sgl.assistant(sgl.gen("answer")) + + state = image_qa.run( + question="Please describe this image in simple words.", + temperature=0, + max_new_tokens=64, + ) + + assert ( + "taxi" in state.messages()[-1]["content"] + or "car" in state.messages()[-1]["content"] + ), f"{state.messages()[-1]['content']}" + + +def test_stream(): + @sgl.function + def qa(s, question): + s += sgl.system("You are a helpful assistant.") + s += sgl.user(question) + s += sgl.assistant(sgl.gen("answer")) + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter(): + out += chunk + + ret = qa( + question="Compose an engaging travel blog post about a recent trip to Hawaii, highlighting cultural experiences and must-see attractions.", + stream=True, + ) + out = "" + for chunk in ret.text_iter("answer"): + out += chunk + + +def test_regex(): + regex = r"((25[0-5]|2[0-4]\d|[01]?\d\d?).){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + + @sgl.function + def regex_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + temperature=0, + regex=regex, + ) + + state = regex_gen.run() + answer = state["answer"] + assert re.match(regex, answer) + + +def test_dtype_gen(): + @sgl.function + def dtype_gen(s): + s += "Q: What is the full name of DNS?\n" + s += "A: The full names is " + sgl.gen("str_res", dtype=str, stop="\n") + "\n" + s += "Q: Which year was DNS invented?\n" + s += "A: " + sgl.gen("int_res", dtype=int) + "\n" + s += "Q: What is the value of pi?\n" + s += "A: " + sgl.gen("float_res", dtype=float) + "\n" + s += "Q: Is the sky blue?\n" + s += "A: " + sgl.gen("bool_res", dtype=bool) + "\n" + + state = dtype_gen.run() + + try: + state["int_res"] = int(state["int_res"]) + state["float_res"] = float(state["float_res"]) + state["bool_res"] = bool(state["bool_res"]) + # assert state["str_res"].startswith('"') and state["str_res"].endswith('"') + except ValueError: + print(state) + raise + + +def test_completion_speculative(): + @sgl.function(num_api_spec_tokens=64) + def gen_character_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + @sgl.function + def gen_character_no_spec(s): + s += "Construct a character within the following format:\n" + s += "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + s += "\nPlease generate new Name, Birthday and Job.\n" + s += ( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + ) + s += "\nJob:" + sgl.gen("job", stop="\n") + "\n" + + token_usage = sgl.global_config.default_backend.token_usage + + token_usage.reset() + gen_character_spec().sync() + usage_with_spec = token_usage.prompt_tokens + + token_usage.reset() + gen_character_no_spec().sync() + usage_with_no_spec = token_usage.prompt_tokens + + assert ( + usage_with_spec < usage_with_no_spec + ), f"{usage_with_spec} vs {usage_with_no_spec}" + + +def test_chat_completion_speculative(): + @sgl.function(num_api_spec_tokens=256) + def gen_character_spec(s): + s += sgl.system("You are a helpful assistant.") + s += sgl.user("Construct a character within the following format:") + s += sgl.assistant( + "Name: Steve Jobs.\nBirthday: February 24, 1955.\nJob: Apple CEO.\n" + ) + s += sgl.user("Please generate new Name, Birthday and Job.\n") + s += sgl.assistant( + "Name:" + + sgl.gen("name", stop="\n") + + "\nBirthday:" + + sgl.gen("birthday", stop="\n") + + "\nJob:" + + sgl.gen("job", stop="\n") + ) + + gen_character_spec().sync() + + +def test_hellaswag_select(): + """Benchmark the accuracy of sgl.select on the HellaSwag dataset.""" + + def get_one_example(lines, i, include_answer): + ret = lines[i]["activity_label"] + ": " + lines[i]["ctx"] + " " + if include_answer: + ret += lines[i]["endings"][lines[i]["label"]] + return ret + + def get_few_shot_examples(lines, k): + ret = "" + for i in range(k): + ret += get_one_example(lines, i, True) + "\n\n" + return ret + + # Read data + url = "https://raw.githubusercontent.com/rowanz/hellaswag/master/data/hellaswag_val.jsonl" + filename = download_and_cache_file(url) + lines = list(read_jsonl(filename)) + + # Construct prompts + num_questions = 200 + num_shots = 20 + few_shot_examples = get_few_shot_examples(lines, num_shots) + + questions = [] + choices = [] + labels = [] + for i in range(len(lines[:num_questions])): + questions.append(get_one_example(lines, i, False)) + choices.append(lines[i]["endings"]) + labels.append(lines[i]["label"]) + arguments = [{"question": q, "choices": c} for q, c in zip(questions, choices)] + + ##################################### + ######### SGL Program Begin ######### + ##################################### + + import sglang as sgl + + @sgl.function + def few_shot_hellaswag(s, question, choices): + s += few_shot_examples + question + s += sgl.select("answer", choices=choices) + + ##################################### + ########## SGL Program End ########## + ##################################### + + # Run requests + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=False, + ) + preds = [] + for i, ret in enumerate(rets): + preds.append(choices[i].index(ret["answer"])) + latency = time.perf_counter() - tic + + # Compute accuracy + accuracy = np.mean(np.array(preds) == np.array(labels)) + + # Test generator style of run_batch + tic = time.perf_counter() + rets = few_shot_hellaswag.run_batch( + arguments, + temperature=0, + num_threads=64, + progress_bar=True, + generator_style=True, + ) + preds_gen = [] + for i, ret in enumerate(rets): + preds_gen.append(choices[i].index(ret["answer"])) + latency_gen = time.perf_counter() - tic + + # Compute accuracy + accuracy_gen = np.mean(np.array(preds_gen) == np.array(labels)) + print(f"{accuracy=}, {accuracy_gen=}") + assert np.abs(accuracy_gen - accuracy) < 0.1 + assert np.abs(latency_gen - latency) < 1 + + return accuracy, latency + + +def test_gen_min_new_tokens(): + """ + Validate sgl.gen(min_tokens) functionality. + + The test asks a question where, without a min_tokens constraint, the generated answer is expected to be short. + By enforcing the min_tokens parameter, we ensure the generated answer has at least the specified number of tokens. + We verify that the number of tokens in the answer is >= the min_tokens threshold. + """ + import sglang as sgl + from sglang.srt.hf_transformers_utils import get_tokenizer + + model_path = sgl.global_config.default_backend.endpoint.get_model_name() + MIN_TOKENS, MAX_TOKENS = 64, 128 + + @sgl.function + def convo_1(s): + s += sgl.user("What is the capital of the United States?") + s += sgl.assistant( + sgl.gen("answer", min_tokens=MIN_TOKENS, max_tokens=MAX_TOKENS) + ) + + def assert_min_tokens(tokenizer, text): + token_ids = tokenizer.encode(text) + assert ( + len(token_ids) >= MIN_TOKENS + ), f"Generated {len(token_ids)} tokens, min required: {MIN_TOKENS}. Text: {text}" + + tokenizer = get_tokenizer(model_path) + + state = convo_1.run() + assert_min_tokens(tokenizer, state["answer"]) diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py new file mode 100644 index 000000000..0d3d769f4 --- /dev/null +++ b/python/sglang/test/test_utils.py @@ -0,0 +1,1430 @@ +"""Common utilities for testing and benchmarking""" + +import argparse +import asyncio +import copy +import json +import logging +import os +import random +import re +import subprocess +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import SimpleNamespace +from typing import Awaitable, Callable, List, Optional, Tuple + +import aiohttp +import numpy as np +import requests +import torch +import torch.nn.functional as F + +from sglang.bench_serving import run_benchmark +from sglang.global_config import global_config +from sglang.srt.utils import ( + get_bool_env_var, + get_device, + is_port_available, + kill_process_tree, + retry, +) +from sglang.test.run_eval import run_eval +from sglang.utils import get_exception_traceback + +# General test models +DEFAULT_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.1-8B-Instruct" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE = "meta-llama/Llama-3.2-1B" +DEFAULT_MOE_MODEL_NAME_FOR_TEST = "mistralai/Mixtral-8x7B-Instruct-v0.1" +DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE = "Qwen/Qwen1.5-MoE-A2.7B" +DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT = "Qwen/Qwen1.5-MoE-A2.7B-Chat" + +# MLA test models +DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" +DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST = "cross-encoder/ms-marco-MiniLM-L6-v2" +DEFAULT_MLA_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct" +DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST = "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_TEST_MLA = "lmsys/sglang-ci-dsv3-test" +DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = "lmsys/sglang-ci-dsv3-test-NextN" + +# FP8 models +DEFAULT_MODEL_NAME_FOR_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8" +DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8 = ( + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8-dynamic" +) +DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8 = ( + "nvidia/Llama-3.1-8B-Instruct-FP8" +) +DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8 = "Qwen/Qwen3-1.7B-FP8" +DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE = "gaunernst/DeepSeek-V2-Lite-Chat-FP8" + +# W8A8 models +DEFAULT_MODEL_NAME_FOR_TEST_W8A8 = "RedHatAI/Llama-3.2-3B-quantized.w8a8" +DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE = "nytopop/Qwen3-30B-A3B.w8a8" + +# EAGLE +DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST = "meta-llama/Llama-2-7b-chat-hf" +DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST = "lmsys/sglang-EAGLE-llama2-chat-7B" +DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B" +DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST = ( + "meta-llama/Llama-3.1-8B-Instruct" +) +DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST = "meta-llama/Llama-3.2-1B-Instruct" + +# Other use cases +DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION = ( + "meta-llama/Llama-4-Scout-17B-16E-Instruct" +) +DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST = "Alibaba-NLP/gte-Qwen2-1.5B-instruct" +DEFAULT_REASONING_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B" +DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST = "deepseek-ai/DeepSeek-V3-0324" +DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST = ( + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +) +DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B" +DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8" + +# Nightly tests +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = "meta-llama/Llama-3.1-8B-Instruct,mistralai/Mistral-7B-Instruct-v0.3,deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct,google/gemma-2-27b-it" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" +DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct" +DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST = "Qwen/Qwen2.5-VL-3B-Instruct" + +DEFAULT_IMAGE_URL = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" +DEFAULT_VIDEO_URL = "https://raw.githubusercontent.com/EvolvingLMMs-Lab/sglang/dev/onevision_local/assets/jobs.mp4" + +DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 600 + + +def is_in_ci(): + """Return whether it is in CI runner.""" + return get_bool_env_var("SGLANG_IS_IN_CI") + + +def is_in_amd_ci(): + """Return whether it is in an AMD CI runner.""" + return get_bool_env_var("SGLANG_AMD_CI") + + +def _use_cached_default_models(model_repo: str): + cache_dir = os.getenv("DEFAULT_MODEL_CACHE_DIR") + if cache_dir and model_repo: + model_path = os.path.join(cache_dir, model_repo) + if os.path.isdir(model_path): + return os.path.abspath(model_path) + return "" + + +if is_in_ci(): + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( + 5000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + ) +else: + DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( + 7000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + ) +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" + +if is_in_amd_ci(): + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH = 3000 + + +def call_generate_lightllm(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "inputs": prompt, + "parameters": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop_sequences": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + pred = res.json()["generated_text"][0] + return pred + + +def find_available_port(base_port: int): + port = base_port + random.randint(100, 1000) + while True: + if is_port_available(port): + return port + if port < 60000: + port += 42 + else: + port -= 43 + + +def call_generate_vllm(prompt, temperature, max_tokens, stop=None, n=1, url=None): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_outlines( + prompt, temperature, max_tokens, stop=None, regex=None, n=1, url=None +): + assert url is not None + + data = { + "prompt": prompt, + "temperature": temperature, + "max_tokens": max_tokens, + "stop": stop, + "regex": regex, + "n": n, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + if n == 1: + pred = res.json()["text"][0][len(prompt) :] + else: + pred = [x[len(prompt) :] for x in res.json()["text"]] + return pred + + +def call_generate_srt_raw(prompt, temperature, max_tokens, stop=None, url=None): + assert url is not None + + data = { + "text": prompt, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "stop": stop, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + obj = res.json() + pred = obj["text"] + return pred + + +def call_generate_guidance( + prompt, temperature, max_tokens, stop=None, n=1, regex=None, model=None +): + assert model is not None + from guidance import gen + + rets = [] + for _ in range(n): + out = ( + model + + prompt + + gen( + name="answer", + max_tokens=max_tokens, + temperature=temperature, + stop=stop, + regex=regex, + ) + ) + rets.append(out["answer"]) + return rets if n > 1 else rets[0] + + +def call_select_lightllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "inputs": context + choices[i], + "parameters": { + "max_new_tokens": 1, + }, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(0) + return np.argmax(scores) + + +def call_select_vllm(context, choices, url=None): + assert url is not None + + scores = [] + for i in range(len(choices)): + data = { + "prompt": context + choices[i], + "max_tokens": 1, + "prompt_logprobs": 1, + } + res = requests.post(url, json=data) + assert res.status_code == 200 + scores.append(res.json().get("prompt_score", 0)) + return np.argmax(scores) + + """ + Modify vllm/entrypoints/api_server.py + + if final_output.prompt_logprobs is not None: + score = np.mean([prob[t_id] for t_id, prob in zip(final_output.prompt_token_ids[1:], final_output.prompt_logprobs[1:])]) + ret["prompt_score"] = score + """ + + +def call_select_guidance(context, choices, model=None): + assert model is not None + from guidance import select + + out = model + context + select(choices, name="answer") + return choices.index(out["answer"]) + + +def add_common_other_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=None) + parser.add_argument( + "--backend", + type=str, + required=True, + choices=[ + "vllm", + "outlines", + "lightllm", + "gserver", + "guidance", + "srt-raw", + "llama.cpp", + ], + ) + parser.add_argument("--n-ctx", type=int, default=4096) + parser.add_argument( + "--model-path", type=str, default="meta-llama/Llama-2-7b-chat-hf" + ) + parser.add_argument("--result-file", type=str, default="result.jsonl") + args = parser.parse_args() + + if args.port is None: + default_port = { + "vllm": 21000, + "outlines": 21000, + "lightllm": 22000, + "srt-raw": 30000, + "gserver": 9988, + } + args.port = default_port.get(args.backend, None) + return args + + +def auto_config_device() -> str: + """Auto-config available device platform""" + + try: + device = get_device() + except (RuntimeError, ImportError) as e: + print(f"Warning: {e} - Falling back to CPU") + device = "cpu" + + return device + + +def add_common_sglang_args_and_parse(parser: argparse.ArgumentParser): + parser.add_argument("--parallel", type=int, default=64) + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument("--backend", type=str, default="srt") + parser.add_argument( + "--device", + type=str, + default="auto", + choices=["auto", "cuda", "rocm", "cpu"], + help="Device type (auto/cuda/rocm/cpu). Auto will detect available platforms", + ) + parser.add_argument("--result-file", type=str, default="result.jsonl") + parser.add_argument("--raw-result-file", type=str) + args = parser.parse_args() + + return args + + +def select_sglang_backend(args: argparse.Namespace): + from sglang.lang.backend.openai import OpenAI + from sglang.lang.backend.runtime_endpoint import RuntimeEndpoint + + if args.backend.startswith("srt"): + if args.backend == "srt-no-parallel": + global_config.enable_parallel_encoding = False + backend = RuntimeEndpoint(f"{args.host}:{args.port}") + elif args.backend.startswith("gpt-"): + backend = OpenAI(args.backend) + else: + raise ValueError(f"Invalid backend: {args.backend}") + return backend + + +def _get_call_generate(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_generate_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_generate_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "srt-raw": + return partial(call_generate_srt_raw, url=f"{args.host}:{args.port}/generate") + elif args.backend == "gserver": + return partial(call_generate_gserver, url=f"{args.host}:{args.port}") + elif args.backend == "outlines": + return partial(call_generate_outlines, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_generate = partial(call_generate_guidance, model=model) + call_generate("Hello,", 1.0, 8, ".") + return call_generate + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def _get_call_select(args: argparse.Namespace): + if args.backend == "lightllm": + return partial(call_select_lightllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "vllm": + return partial(call_select_vllm, url=f"{args.host}:{args.port}/generate") + elif args.backend == "guidance": + from guidance import models + + model = models.LlamaCpp(args.model_path, n_gpu_layers=-1, n_ctx=args.n_ctx) + call_select = partial(call_select_guidance, model=model) + + call_select("Hello,", ["world", "earth"]) + return call_select + else: + raise ValueError(f"Invalid backend: {args.backend}") + + +def get_call_generate(args: argparse.Namespace): + call_generate = _get_call_generate(args) + + def func(*args, **kwargs): + try: + return call_generate(*args, **kwargs) + except Exception: + print("Exception in call_generate:\n" + get_exception_traceback()) + raise + + return func + + +def get_call_select(args: argparse.Namespace): + call_select = _get_call_select(args) + + def func(*args, **kwargs): + try: + return call_select(*args, **kwargs) + except Exception: + print("Exception in call_select:\n" + get_exception_traceback()) + raise + + return func + + +def _get_default_models(): + import inspect + + current_module = inspect.getmodule(_get_default_models) + default_models = set() + for name, value in current_module.__dict__.items(): + if ( + isinstance(name, str) + and "DEFAULT_" in name + and "MODEL_" in name + and isinstance(value, str) + ): + if "," in value: + parts = [part.strip() for part in value.split(",")] + default_models.update(parts) + else: + default_models.add(value.strip()) + return json.dumps(list(default_models)) + + +def try_cached_model(model_repo: str): + model_dir = _use_cached_default_models(model_repo) + return model_dir if model_dir else model_repo + + +def popen_with_error_check(command: list[str], allow_exit: bool = False): + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def _run_and_check(): + stdout, stderr = process.communicate() + + while process.poll() is None: + time.sleep(5) + + if not allow_exit or process.returncode != 0: + raise Exception( + f"{command} exited with code {process.returncode}\n{stdout=}\n{stderr=}" + ) + + t = threading.Thread(target=_run_and_check) + t.start() + return process + + +def popen_launch_server( + model: str, + base_url: str, + timeout: float, + api_key: Optional[str] = None, + other_args: list[str] = [], + env: Optional[dict] = None, + return_stdout_stderr: Optional[tuple] = None, + device: str = "auto", + pd_separated: bool = False, +): + """Launch a server process with automatic device detection. + + Args: + device: Device type ("auto", "cuda", "rocm" or "cpu"). + If "auto", will detect available platforms automatically. + """ + # Auto-detect device if needed + if device == "auto": + device = auto_config_device() + print(f"Auto-configed device: {device}", flush=True) + other_args = list(other_args) + other_args += ["--device", str(device)] + + _, host, port = base_url.split(":") + host = host[2:] + + if pd_separated: + command = "sglang.launch_pd_server" + else: + command = "sglang.launch_server" + + command = [ + "python3", + "-m", + command, + "--model-path", + model, + *[str(x) for x in other_args], + ] + + if pd_separated: + command.extend( + [ + "--lb-host", + host, + "--lb-port", + port, + ] + ) + else: + command.extend( + [ + "--host", + host, + "--port", + port, + ] + ) + + if api_key: + command += ["--api-key", api_key] + + print(f"command={' '.join(command)}") + + if return_stdout_stderr: + process = subprocess.Popen( + command, + stdout=return_stdout_stderr[0], + stderr=return_stdout_stderr[1], + env=env, + text=True, + ) + else: + process = subprocess.Popen(command, stdout=None, stderr=None, env=env) + + start_time = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start_time < timeout: + + return_code = process.poll() + if return_code is not None: + # Server failed to start (non-zero exit code) or crashed + raise Exception( + f"Server process exited with code {return_code}. " + "Check server logs for errors." + ) + + try: + headers = { + "Content-Type": "application/json; charset=utf-8", + "Authorization": f"Bearer {api_key}", + } + response = session.get( + f"{base_url}/health_generate", + headers=headers, + ) + if response.status_code == 200: + return process + except requests.RequestException: + pass + + return_code = process.poll() + if return_code is not None: + raise Exception( + f"Server unexpectedly exits ({return_code=}). Usually there will be error logs describing the cause far above this line." + ) + + time.sleep(10) + + kill_process_tree(process.pid) + raise TimeoutError("Server failed to start within the timeout period.") + + +def popen_launch_pd_server( + model: str, + base_url: str, + timeout: float, + api_key: Optional[str] = None, + other_args: list[str] = (), + env: Optional[dict] = None, +): + _, host, port = base_url.split(":") + host = host[2:] + + command = "sglang.launch_server" + + command = [ + "python3", + "-m", + command, + "--model-path", + model, + *[str(x) for x in other_args], + ] + + command.extend( + [ + "--host", + host, + "--port", + port, + ] + ) + + if api_key: + command += ["--api-key", api_key] + + print(f"command={' '.join(command)}") + + process = subprocess.Popen(command, stdout=None, stderr=None, env=env) + + return process + + +def run_with_timeout( + func: Callable, + args: tuple = (), + kwargs: Optional[dict] = None, + timeout: float = None, +): + """Run a function with timeout.""" + ret_value = [] + + def _target_func(): + ret_value.append(func(*args, **(kwargs or {}))) + + t = threading.Thread(target=_target_func) + t.start() + t.join(timeout=timeout) + if t.is_alive(): + raise TimeoutError() + + if not ret_value: + raise RuntimeError() + + return ret_value[0] + + +@dataclass +class TestFile: + name: str + estimated_time: float = 60 + + +def run_unittest_files(files: List[TestFile], timeout_per_file: float): + tic = time.perf_counter() + success = True + + for i, file in enumerate(files): + filename, estimated_time = file.name, file.estimated_time + process = None + + def run_one_file(filename): + nonlocal process + + filename = os.path.join(os.getcwd(), filename) + print( + f".\n.\nBegin ({i}/{len(files) - 1}):\npython3 {filename}\n.\n.\n", + flush=True, + ) + tic = time.perf_counter() + + process = subprocess.Popen( + ["python3", filename], stdout=None, stderr=None, env=os.environ + ) + process.wait() + elapsed = time.perf_counter() - tic + + print( + f".\n.\nEnd ({i}/{len(files) - 1}):\n{filename=}, {elapsed=:.0f}, {estimated_time=}\n.\n.\n", + flush=True, + ) + return process.returncode + + try: + ret_code = run_with_timeout( + run_one_file, args=(filename,), timeout=timeout_per_file + ) + assert ( + ret_code == 0 + ), f"expected return code 0, but {filename} returned {ret_code}" + except TimeoutError: + kill_process_tree(process.pid) + time.sleep(5) + print( + f"\nTimeout after {timeout_per_file} seconds when running {filename}\n", + flush=True, + ) + success = False + break + + if success: + print(f"Success. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) + else: + print(f"Fail. Time elapsed: {time.perf_counter() - tic:.2f}s", flush=True) + + return 0 if success else -1 + + +def get_similarities(vec1, vec2): + return F.cosine_similarity(torch.tensor(vec1), torch.tensor(vec2), dim=0) + + +def get_benchmark_args( + base_url="", + dataset_name="", + dataset_path="", + tokenizer="", + num_prompts=500, + sharegpt_output_len=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + request_rate=float("inf"), + disable_stream=False, + disable_ignore_eos=False, + seed: int = 0, + device="auto", + pd_separated: bool = False, + lora_name=None, +): + return SimpleNamespace( + backend="sglang", + base_url=base_url, + host=None, + port=None, + dataset_name=dataset_name, + dataset_path=dataset_path, + model=None, + tokenizer=tokenizer, + num_prompts=num_prompts, + sharegpt_output_len=sharegpt_output_len, + sharegpt_context_len=sharegpt_context_len, + random_input_len=random_input_len, + random_output_len=random_output_len, + random_range_ratio=0.0, + request_rate=request_rate, + multi=None, + output_file=None, + disable_tqdm=False, + disable_stream=disable_stream, + return_logprob=False, + seed=seed, + disable_ignore_eos=disable_ignore_eos, + extra_request_body=None, + apply_chat_template=False, + profile=None, + lora_name=lora_name, + prompt_suffix="", + device=device, + pd_separated=pd_separated, + ) + + +def run_bench_serving( + model, + num_prompts, + request_rate, + other_server_args, + dataset_name="random", + dataset_path="", + tokenizer=None, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + disable_stream=False, + disable_ignore_eos=False, + need_warmup=False, + seed: int = 0, + device="auto", + background_task: Optional[Callable[[str, asyncio.Event], Awaitable[None]]] = None, + lora_name: Optional[str] = None, +): + if device == "auto": + device = auto_config_device() + # Launch the server + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + ) + + # Run benchmark + args = get_benchmark_args( + base_url=base_url, + dataset_name=dataset_name, + dataset_path=dataset_path, + tokenizer=tokenizer, + num_prompts=num_prompts, + random_input_len=random_input_len, + random_output_len=random_output_len, + sharegpt_context_len=sharegpt_context_len, + request_rate=request_rate, + disable_stream=disable_stream, + disable_ignore_eos=disable_ignore_eos, + seed=seed, + device=device, + lora_name=lora_name, + ) + + async def _run(): + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + await asyncio.to_thread(run_benchmark, warmup_args) + + start_event = asyncio.Event() + stop_event = asyncio.Event() + task_handle = ( + asyncio.create_task(background_task(base_url, start_event, stop_event)) + if background_task + else None + ) + + try: + start_event.set() + result = await asyncio.to_thread(run_benchmark, args) + finally: + if task_handle: + stop_event.set() + await task_handle + + return result + + try: + res = asyncio.run(_run()) + finally: + kill_process_tree(process.pid) + + assert res["completed"] == num_prompts + return res + + +def run_bench_serving_multi( + model, + base_url, + other_server_args, + benchmark_args, + need_warmup=False, + pd_separated=False, +): + # Launch the server + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + pd_separated=pd_separated, + ) + + # run benchmark for all + res_l = [] + try: + for args in benchmark_args: + if need_warmup: + warmup_args = copy.deepcopy(args) + warmup_args.num_prompts = 16 + run_benchmark(warmup_args) + + res = run_benchmark(args) + res_l.append((args, res)) + finally: + kill_process_tree(process.pid) + + return res_l + + +def run_bench_one_batch(model, other_args): + """Launch a offline process with automatic device detection. + + Args: + device: Device type ("auto", "cuda", "rocm" or "cpu"). + If "auto", will detect available platforms automatically. + """ + # Auto-detect device if needed + + device = auto_config_device() + print(f"Auto-configed device: {device}", flush=True) + other_args += ["--device", str(device)] + + command = [ + "python3", + "-m", + "sglang.bench_one_batch", + "--batch-size", + "1", + "--input", + "128", + "--output", + "8", + *[str(x) for x in other_args], + ] + if model is not None: + command += ["--model-path", model] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + try: + stdout, stderr = process.communicate() + output = stdout.decode() + error = stderr.decode() + print(f"Output: {output}", flush=True) + print(f"Error: {error}", flush=True) + + # Return prefill_latency, decode_throughput, decode_latency + prefill_line = output.split("\n")[-9] + decode_line = output.split("\n")[-3] + pattern = ( + r"latency: (?P\d+\.\d+).*?throughput:\s*(?P\d+\.\d+)" + ) + match = re.search(pattern, prefill_line) + if match: + prefill_latency = float(match.group("latency")) + match = re.search(pattern, decode_line) + if match: + decode_latency = float(match.group("latency")) + decode_throughput = float(match.group("throughput")) + finally: + kill_process_tree(process.pid) + + return prefill_latency, decode_throughput, decode_latency + + +def run_bench_offline_throughput(model, other_args): + command = [ + "python3", + "-m", + "sglang.bench_offline_throughput", + "--num-prompts", + "1", + "--dataset-name", + "random", + "--random-input-len", + "256", + "--random-output-len", + "256", + "--model-path", + model, + *[str(x) for x in other_args], + ] + + print(f"{command=}") + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + try: + stdout, stderr = process.communicate() + output = stdout.decode() + error = stderr.decode() + print(f"Output: {output}", flush=True) + print(f"Error: {error}", flush=True) + + output_throughput = -1 + for line in output.split("\n"): + if "Last generation throughput (tok/s):" in line: + output_throughput = float(line.split(":")[-1]) + finally: + kill_process_tree(process.pid) + + return output_throughput + + +def run_bench_one_batch_server( + model, + base_url, + server_args, + bench_args, + other_server_args, + simulate_spec_acc_lens=None, +): + from sglang.bench_one_batch_server import run_benchmark + + if simulate_spec_acc_lens is not None: + env = {**os.environ, "SIMULATE_ACC_LEN": str(simulate_spec_acc_lens)} + else: + env = None + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_server_args, + env=env, + ) + try: + run_benchmark(server_args=server_args, bench_args=bench_args) + finally: + kill_process_tree(process.pid) + + +def lcs(X, Y): + m = len(X) + n = len(Y) + L = [[0] * (n + 1) for _ in range(m + 1)] + + for i in range(m + 1): + for j in range(n + 1): + if i == 0 or j == 0: + L[i][j] = 0 + elif X[i - 1] == Y[j - 1]: + L[i][j] = L[i - 1][j - 1] + 1 + else: + L[i][j] = max(L[i - 1][j], L[i][j - 1]) + + return L[m][n] + + +def calculate_rouge_l(output_strs_list1, output_strs_list2): + """calculate the ROUGE-L score""" + rouge_l_scores = [] + + for s1, s2 in zip(output_strs_list1, output_strs_list2): + lcs_len = lcs(s1, s2) + precision = lcs_len / len(s1) if len(s1) > 0 else 0 + recall = lcs_len / len(s2) if len(s2) > 0 else 0 + if precision + recall > 0: + fmeasure = (2 * precision * recall) / (precision + recall) + else: + fmeasure = 0.0 + rouge_l_scores.append(fmeasure) + + return rouge_l_scores + + +STDERR_FILENAME = "/tmp/stderr.txt" +STDOUT_FILENAME = "/tmp/stdout.txt" + + +def read_output(output_lines: List[str], filename: str = STDERR_FILENAME): + """Print the output in real time with another thread.""" + while not os.path.exists(filename): + time.sleep(0.01) + + pt = 0 + while pt >= 0: + if pt > 0 and not os.path.exists(filename): + break + try: + lines = open(filename).readlines() + except FileNotFoundError: + print(f"{pt=}, {os.path.exists(filename)=}") + raise + for line in lines[pt:]: + print(line, end="", flush=True) + output_lines.append(line) + pt += 1 + time.sleep(0.1) + + +def run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort, +): + other_args = [ + "--chunked-prefill-size", + str(chunked_prefill_size), + "--log-level", + "debug", + ] + if disable_radix_cache: + other_args += ["--disable-radix-cache"] + if enable_mixed_chunk: + other_args += ["--enable-mixed-chunk"] + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + model = DEFAULT_MODEL_NAME_FOR_TEST + port = random.randint(4000, 5000) + base_url = f"http://127.0.0.1:{port}" + + # Create files and launch the server + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + return_stdout_stderr=(stdout, stderr), + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines,)) + t.start() + + # Run the workload + workload_func(base_url, model) + + # Clean up everything + kill_process_tree(process.pid) + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + # Assert success + has_new_server = False + has_leak = False + has_abort = False + for line in output_lines: + if "Uvicorn running" in line: + has_new_server = True + if "leak" in line: + has_leak = True + if "Abort" in line: + has_abort = True + + assert has_new_server + assert not has_leak + if assert_has_abort: + assert has_abort + + +def run_command_and_capture_output(command, env: Optional[dict] = None): + stdout = open(STDOUT_FILENAME, "w") + stderr = open(STDERR_FILENAME, "w") + process = subprocess.Popen( + command, stdout=stdout, stderr=stdout, env=env, text=True + ) + + # Launch a thread to stream the output + output_lines = [] + t = threading.Thread(target=read_output, args=(output_lines, STDOUT_FILENAME)) + t.start() + + # Join the process + process.wait() + + stdout.close() + stderr.close() + if os.path.exists(STDOUT_FILENAME): + os.remove(STDOUT_FILENAME) + if os.path.exists(STDERR_FILENAME): + os.remove(STDERR_FILENAME) + kill_process_tree(process.pid) + t.join() + + return output_lines + + +def run_mmlu_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + # Run the eval + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=128, + num_threads=128, + ) + + try: + metrics = run_eval(args) + assert metrics["score"] >= 0.65, f"{metrics=}" + finally: + pass + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + disable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def run_mulit_request_test( + disable_radix_cache=False, + enable_mixed_chunk=False, + enable_overlap=False, + chunked_prefill_size=32, +): + def workload_func(base_url, model): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 8, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(2) as executor: + list(executor.map(run_one, list(range(4)))) + + run_and_check_memory_leak( + workload_func, + disable_radix_cache, + enable_mixed_chunk, + enable_overlap, + chunked_prefill_size, + assert_has_abort=False, + ) + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def run_logprob_check(self: unittest.TestCase, arg: Tuple): + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) = arg + input_ids = list(range(input_len)) + + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": output_len, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "logprob_start_len": logprob_start_len, + "top_logprobs_num": top_logprobs_num, + }, + ) + response_json = response.json() + + res = response_json + self.assertEqual(res["meta_info"]["prompt_tokens"], input_len) + self.assertEqual(res["meta_info"]["completion_tokens"], output_len) + + # Test the number of tokens are correct + if return_logprob: + self.assertEqual( + len(res["meta_info"]["input_token_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), output_len) + + if top_logprobs_num: + self.assertEqual( + len(res["meta_info"]["input_top_logprobs"]) + logprob_start_len, + res["meta_info"]["prompt_tokens"], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), output_len) + + for i in range(output_len): + self.assertEqual( + len(res["meta_info"]["output_top_logprobs"][i]), + top_logprobs_num, + ) + + # Test the top-1 tokens are the same as output tokens if temperature == 0 + if temperature == 0: + rank = 0 + while rank < len(res["meta_info"]["output_top_logprobs"][i]): + try: + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][rank], + ) + break + except AssertionError: + # There's a tie. Allow the second item in this case. + if ( + res["meta_info"]["output_top_logprobs"][i][rank][0] + == res["meta_info"]["output_top_logprobs"][i][rank + 1][ + 0 + ] + ): + rank += 1 + else: + raise + + +def send_generate_requests(base_url: str, num_requests: int) -> List[str]: + """Sends generate request serially and returns status codes. Max concurrency is 1.""" + + def generate(): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 50, + }, + }, + ) + return response.status_code + + return [generate() for _ in range(num_requests)] + + +async def send_concurrent_generate_requests( + base_url: str, num_requests: int +) -> List[str]: + """Sends generate request concurrently and returns status codes. Max concurrency is num_requests.""" + + async def async_generate(): + async with aiohttp.ClientSession() as session: + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + async with session.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 50, + }, + }, + ) as response: + return response.status + + tasks = [asyncio.create_task(async_generate()) for _ in range(num_requests)] + return await asyncio.gather(*tasks) + + +class CustomTestCase(unittest.TestCase): + def _callTestMethod(self, method): + max_retry = int( + os.environ.get("SGLANG_TEST_MAX_RETRY", "1" if is_in_ci() else "0") + ) + retry( + lambda: super(CustomTestCase, self)._callTestMethod(method), + max_retry=max_retry, + ) + + +def dump_bench_raw_result( + path: str, + states, + preds, + labels, +): + if not path: + return + + rows = [] + for i in range(len(states)): + state = states[i] + output = state["answer"] + prompt = _ensure_remove_suffix(state.text(), output) + rows.append( + dict( + prompt_id=i, + prompt=prompt, + output=output, + correct=bool(preds[i] == labels[i]), + ) + ) + + print(f"BenchRawResultDumper save results to {path}") + Path(path).write_text("\n".join(json.dumps(row) for row in rows)) + + +def _ensure_remove_suffix(text: str, suffix: str): + assert text.endswith(suffix) + return text.removesuffix(suffix) diff --git a/python/sglang/utils.py b/python/sglang/utils.py new file mode 100644 index 000000000..f6bf20c42 --- /dev/null +++ b/python/sglang/utils.py @@ -0,0 +1,535 @@ +"""Common utilities""" + +import importlib +import json +import logging +import os +import random +import socket +import subprocess +import sys +import time +import traceback +import urllib.request +import weakref +from concurrent.futures import ThreadPoolExecutor +from functools import wraps +from io import BytesIO +from json import dumps +from typing import Any, Callable, List, Optional, Tuple, Type, Union + +import numpy as np +import pybase64 +import requests +from IPython.display import HTML, display +from pydantic import BaseModel +from tqdm import tqdm + +logger = logging.getLogger(__name__) + + +def execute_once(func): + has_run = None + + @wraps(func) + def wrapper(*args, **kwargs): + nonlocal has_run + if not has_run: + func(*args, **kwargs) + has_run = True + + return wrapper + + +@execute_once +def info_once(message: str): + logger.info(message) + + +def convert_json_schema_to_str(json_schema: Union[dict, str, Type[BaseModel]]) -> str: + """Convert a JSON schema to a string. + Parameters + ---------- + json_schema + The JSON schema. + Returns + ------- + str + The JSON schema converted to a string. + Raises + ------ + ValueError + If the schema is not a dictionary, a string or a Pydantic class. + """ + if isinstance(json_schema, dict): + schema_str = json.dumps(json_schema) + elif isinstance(json_schema, str): + schema_str = json_schema + elif issubclass(json_schema, BaseModel): + schema_str = json.dumps(json_schema.model_json_schema()) + else: + raise ValueError( + f"Cannot parse schema {json_schema}. The schema must be either " + + "a Pydantic class, a dictionary or a string that contains the JSON " + + "schema specification" + ) + return schema_str + + +def get_exception_traceback(): + etype, value, tb = sys.exc_info() + err_str = "".join(traceback.format_exception(etype, value, tb)) + return err_str + + +def is_same_type(values: list): + """Return whether the elements in values are of the same type.""" + if len(values) <= 1: + return True + else: + t = type(values[0]) + return all(isinstance(v, t) for v in values[1:]) + + +def read_jsonl(filename: str): + """Read a JSONL file.""" + with open(filename) as fin: + for line in fin: + if line.startswith("#"): + continue + yield json.loads(line) + + +def dump_state_text(filename: str, states: list, mode: str = "w"): + """Dump program state in a text file.""" + from sglang.lang.interpreter import ProgramState + + with open(filename, mode) as fout: + for i, s in enumerate(states): + if isinstance(s, str): + pass + elif isinstance(s, ProgramState): + s = s.text() + else: + s = str(s) + + fout.write( + "=" * 40 + f" {i} " + "=" * 40 + "\n" + s + "\n" + "=" * 80 + "\n\n" + ) + + +class HttpResponse: + def __init__(self, resp): + self.resp = resp + + def json(self): + return json.loads(self.resp.read()) + + @property + def status_code(self): + return self.resp.status + + +def http_request( + url, + json=None, + stream=False, + api_key=None, + verify=None, + method: Optional[str] = None, +): + """A faster version of requests.post with low-level urllib API.""" + headers = {"Content-Type": "application/json; charset=utf-8"} + + # add the Authorization header if an api key is provided + if api_key is not None: + headers["Authorization"] = f"Bearer {api_key}" + + if stream: + return requests.post(url, json=json, stream=True, headers=headers) + else: + req = urllib.request.Request(url, headers=headers, method=method) + if json is None: + data = None + else: + data = bytes(dumps(json), encoding="utf-8") + + try: + resp = urllib.request.urlopen(req, data=data, cafile=verify) + return HttpResponse(resp) + except urllib.error.HTTPError as e: + return HttpResponse(e) + + +def encode_image_base64(image_path: Union[str, bytes]): + """Encode an image in base64.""" + if isinstance(image_path, str): + with open(image_path, "rb") as image_file: + data = image_file.read() + return pybase64.b64encode(data).decode("utf-8") + elif isinstance(image_path, bytes): + return pybase64.b64encode(image_path).decode("utf-8") + else: + # image_path is PIL.WebPImagePlugin.WebPImageFile + image = image_path + buffered = BytesIO() + image.save(buffered, format="PNG") + return pybase64.b64encode(buffered.getvalue()).decode("utf-8") + + +def encode_frame(frame): + import cv2 # pip install opencv-python-headless + from PIL import Image + + # Convert the frame to RGB (OpenCV uses BGR by default) + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + + # Convert the frame to PIL Image to easily convert to bytes + im_pil = Image.fromarray(frame) + + # Convert to bytes + buffered = BytesIO() + + # frame_format = str(os.getenv('FRAME_FORMAT', "JPEG")) + + im_pil.save(buffered, format="PNG") + + frame_bytes = buffered.getvalue() + + # Return the bytes of the frame + return frame_bytes + + +def encode_video_base64(video_path: str, num_frames: int = 16): + import cv2 # pip install opencv-python-headless + + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + raise IOError(f"Could not open video file:{video_path}") + + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + print(f"target_frames: {num_frames}") + + frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) + + frames = [] + for _ in range(total_frames): + ret, frame = cap.read() + if ret: + frames.append(frame) + else: + # Handle the case where the frame could not be read + # print(f"Warning: Could not read frame at index {i}.") + pass + + cap.release() + + # Safely select frames based on frame_indices, avoiding IndexError + frames = [frames[i] for i in frame_indices if i < len(frames)] + + # If there are not enough frames, duplicate the last frame until we reach the target + while len(frames) < num_frames: + frames.append(frames[-1]) + + # Use ThreadPoolExecutor to process and encode frames in parallel + with ThreadPoolExecutor() as executor: + encoded_frames = list(executor.map(encode_frame, frames)) + + # encoded_frames = list(map(encode_frame, frames)) + + # Concatenate all frames bytes + video_bytes = b"".join(encoded_frames) + + # Encode the concatenated bytes to base64 + video_base64 = "video:" + pybase64.b64encode(video_bytes).decode("utf-8") + + return video_base64 + + +def _is_chinese_char(cp: int): + """Checks whether CP is the codepoint of a CJK character.""" + # This defines a "chinese character" as anything in the CJK Unicode block: + # https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) + # + # Note that the CJK Unicode block is NOT all Japanese and Korean characters, + # despite its name. The modern Korean Hangul alphabet is a different block, + # as is Japanese Hiragana and Katakana. Those alphabets are used to write + # space-separated words, so they are not treated specially and handled + # like the all of the other languages. + if ( + (cp >= 0x4E00 and cp <= 0x9FFF) + or (cp >= 0x3400 and cp <= 0x4DBF) # + or (cp >= 0x20000 and cp <= 0x2A6DF) # + or (cp >= 0x2A700 and cp <= 0x2B73F) # + or (cp >= 0x2B740 and cp <= 0x2B81F) # + or (cp >= 0x2B820 and cp <= 0x2CEAF) # + or (cp >= 0xF900 and cp <= 0xFAFF) + or (cp >= 0x2F800 and cp <= 0x2FA1F) # + ): # + return True + + return False + + +def find_printable_text(text: str): + """Returns the longest printable substring of text that contains only entire words.""" + # Borrowed from https://github.com/huggingface/transformers/blob/061580c82c2db1de9139528243e105953793f7a2/src/transformers/generation/streamers.py#L99 + + # After the symbol for a new line, we flush the cache. + if text.endswith("\n"): + return text + # If the last token is a CJK character, we print the characters. + elif len(text) > 0 and _is_chinese_char(ord(text[-1])): + return text + # Otherwise if the penultimate token is a CJK character, we print the characters except for the last one. + elif len(text) > 1 and _is_chinese_char(ord(text[-2])): + return text[:-1] + # Otherwise, prints until the last space char (simple heuristic to avoid printing incomplete words, + # which may change with the subsequent token -- there are probably smarter ways to do this!) + else: + return text[: text.rfind(" ") + 1] + + +class LazyImport: + """Lazy import to make `import sglang` run faster.""" + + def __init__(self, module_name: str, class_name: str): + self.module_name = module_name + self.class_name = class_name + self._module = None + + def _load(self): + if self._module is None: + module = importlib.import_module(self.module_name) + self._module = getattr(module, self.class_name) + return self._module + + def __getattr__(self, name: str): + module = self._load() + return getattr(module, name) + + def __call__(self, *args, **kwargs): + module = self._load() + return module(*args, **kwargs) + + +def download_and_cache_file(url: str, filename: Optional[str] = None): + """Read and cache a file from a url.""" + if filename is None: + filename = os.path.join("/tmp", url.split("/")[-1]) + + # Check if the cache file already exists + if os.path.exists(filename): + return filename + + print(f"Downloading from {url} to {filename}") + + # Stream the response to show the progress bar + response = requests.get(url, stream=True) + response.raise_for_status() # Check for request errors + + # Total size of the file in bytes + total_size = int(response.headers.get("content-length", 0)) + chunk_size = 1024 # Download in chunks of 1KB + + # Use tqdm to display the progress bar + with open(filename, "wb") as f, tqdm( + desc=filename, + total=total_size, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as bar: + for chunk in response.iter_content(chunk_size=chunk_size): + f.write(chunk) + bar.update(len(chunk)) + + return filename + + +def is_in_ci(): + from sglang.test.test_utils import is_in_ci + + return is_in_ci() + + +def print_highlight(html_content: str): + if is_in_ci(): + html_content = str(html_content).replace("\n", "
") + display(HTML(f"{html_content}")) + else: + print(html_content) + + +process_socket_map = weakref.WeakKeyDictionary() + + +def reserve_port(host, start=30000, end=40000): + """ + Reserve an available port by trying to bind a socket. + Returns a tuple (port, lock_socket) where `lock_socket` is kept open to hold the lock. + """ + candidates = list(range(start, end)) + random.shuffle(candidates) + + for port in candidates: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + # Attempt to bind to the port on localhost + sock.bind((host, port)) + return port, sock + except socket.error: + sock.close() # Failed to bind, try next port + continue + raise RuntimeError("No free port available.") + + +def release_port(lock_socket): + """ + Release the reserved port by closing the lock socket. + """ + try: + lock_socket.close() + except Exception as e: + print(f"Error closing socket: {e}") + + +def execute_shell_command(command: str) -> subprocess.Popen: + """ + Execute a shell command and return its process handle. + """ + command = command.replace("\\\n", " ").replace("\\", " ") + parts = command.split() + return subprocess.Popen(parts, text=True, stderr=subprocess.STDOUT) + + +def launch_server_cmd(command: str, host: str = "0.0.0.0", port: int = None): + """ + Launch the server using the given command. + If no port is specified, a free port is reserved. + """ + if port is None: + port, lock_socket = reserve_port(host) + else: + lock_socket = None + + full_command = f"{command} --port {port}" + process = execute_shell_command(full_command) + + if lock_socket is not None: + process_socket_map[process] = lock_socket + + return process, port + + +def terminate_process(process): + """ + Terminate the process and automatically release the reserved port. + """ + from sglang.srt.utils import kill_process_tree + + kill_process_tree(process.pid) + + lock_socket = process_socket_map.pop(process, None) + if lock_socket is not None: + release_port(lock_socket) + + +def wait_for_server(base_url: str, timeout: int = None) -> None: + """Wait for the server to be ready by polling the /v1/models endpoint. + + Args: + base_url: The base URL of the server + timeout: Maximum time to wait in seconds. None means wait forever. + """ + start_time = time.perf_counter() + while True: + try: + response = requests.get( + f"{base_url}/v1/models", + headers={"Authorization": "Bearer None"}, + ) + if response.status_code == 200: + time.sleep(5) + print_highlight( + """\n + NOTE: Typically, the server runs in a separate terminal. + In this notebook, we run the server and notebook code together, so their outputs are combined. + To improve clarity, the server logs are displayed in the original black color, while the notebook outputs are highlighted in blue. + To reduce the log length, we set the log level to warning for the server, the default log level is info. + We are running those notebooks in a CI environment, so the throughput is not representative of the actual performance. + """ + ) + break + + if timeout and time.perf_counter() - start_time > timeout: + raise TimeoutError("Server did not become ready within timeout period") + except requests.exceptions.RequestException: + time.sleep(1) + + +class TypeBasedDispatcher: + def __init__(self, mapping: List[Tuple[Type, Callable]]): + self._mapping = mapping + + def __iadd__(self, other: "TypeBasedDispatcher"): + self._mapping.extend(other._mapping) + return self + + def __call__(self, obj: Any): + for ty, fn in self._mapping: + if isinstance(obj, ty): + return fn(obj) + raise ValueError(f"Invalid object: {obj}") + + +def trim_overlap(existing_text, new_chunk): + """ + Finds the largest suffix of 'existing_text' that is a prefix of 'new_chunk' + and removes that overlap from the start of 'new_chunk'. + """ + max_overlap = 0 + max_possible = min(len(existing_text), len(new_chunk)) + for i in range(max_possible, 0, -1): + if existing_text.endswith(new_chunk[:i]): + max_overlap = i + break + return new_chunk[max_overlap:] + + +def stream_and_merge(llm, prompt, sampling_params): + """ + 1) Streams the text, + 2) Removes chunk overlaps, + 3) Returns the merged text. + """ + final_text = "" + for chunk in llm.generate(prompt, sampling_params, stream=True): + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + return final_text + + +async def async_stream_and_merge(llm, prompt, sampling_params): + """ + Streams tokens asynchronously, removes chunk overlaps, + and yields the cleaned chunk in real time for printing. + """ + final_text = "" + generator = await llm.async_generate(prompt, sampling_params, stream=True) + async for chunk in generator: + chunk_text = chunk["text"] + cleaned_chunk = trim_overlap(final_text, chunk_text) + final_text += cleaned_chunk + yield cleaned_chunk # yield the non-overlapping portion + + +def resolve_obj_by_qualname(qualname: str) -> Any: + """ + Resolve an object by its fully qualified name. + """ + module_name, obj_name = qualname.rsplit(".", 1) + module = importlib.import_module(module_name) + return getattr(module, obj_name) diff --git a/python/sglang/version.py b/python/sglang/version.py new file mode 100644 index 000000000..722515271 --- /dev/null +++ b/python/sglang/version.py @@ -0,0 +1 @@ +__version__ = "0.5.2" diff --git a/scripts/ci/amd_ci_exec.sh b/scripts/ci/amd_ci_exec.sh new file mode 100755 index 000000000..3bd940eb1 --- /dev/null +++ b/scripts/ci/amd_ci_exec.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -euo pipefail + +# Detect GPU family from hostname (e.g., linux-mi35x-gpu-1-xxxxx-runner-zzzzz) +HOSTNAME_VALUE=$(hostname) +GPU_FAMILY="" + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_FAMILY="${BASH_REMATCH[1]}" + echo "Detected GPU family from hostname: ${GPU_FAMILY}" +else + echo "Warning: could not parse GPU family from '${HOSTNAME_VALUE}'" +fi + +WORKDIR="/sglang-checkout/test/srt" +declare -A ENV_MAP=( + [SGLANG_AMD_CI]=1 + [SGLANG_IS_IN_CI]=1 + [SGLANG_USE_AITER]=1 +) + +# Conditionally add GPU_ARCHS only for mi35x +if [[ "${GPU_FAMILY}" == "mi35x" ]]; then + ENV_MAP[GPU_ARCHS]="gfx950" +fi + +# Parse -w/--workdir and -e ENV=VAL +while [[ $# -gt 0 ]]; do + case "$1" in + -w|--workdir) + WORKDIR="$2" + shift 2 + ;; + -e) + IFS="=" read -r key val <<< "$2" + ENV_MAP["$key"]="$val" + shift 2 + ;; + --) + shift + break + ;; + *) + break + ;; + esac +done + +# Build final ENV_ARGS +ENV_ARGS=() +for key in "${!ENV_MAP[@]}"; do + ENV_ARGS+=("-e" "$key=${ENV_MAP[$key]}") +done + +# Run docker exec +docker exec \ + -w "$WORKDIR" \ + "${ENV_ARGS[@]}" \ + ci_sglang "$@" diff --git a/scripts/ci/amd_ci_install_dependency.sh b/scripts/ci/amd_ci_install_dependency.sh new file mode 100755 index 000000000..518f0dde9 --- /dev/null +++ b/scripts/ci/amd_ci_install_dependency.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -euo pipefail +HOSTNAME_VALUE=$(hostname) +GPU_ARCH="mi30x" # default + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_ARCH="${BASH_REMATCH[1]}" + echo "Detected GPU architecture from hostname: ${GPU_ARCH}" +else + echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}" +fi + +# Install the required dependencies in CI. +docker exec ci_sglang pip install --upgrade pip +docker exec ci_sglang pip uninstall sgl-kernel -y || true +docker exec -w /sglang-checkout/sgl-kernel ci_sglang bash -c "rm -f pyproject.toml && mv pyproject_rocm.toml pyproject.toml && python3 setup_rocm.py install" + +case "${GPU_ARCH}" in + mi35x) + echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." + docker exec ci_sglang pip install -e "python[dev_hip]" --no-deps # TODO: only for mi35x + # For lmms_evals evaluating MMMU + docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + docker exec -w /lmms-eval ci_sglang pip install -e . --no-deps # TODO: only for mi35x + ;; + mi30x|mi300|mi325) + echo "Runner uses ${GPU_ARCH}; will fetch mi30x image." + docker exec ci_sglang pip install -e "python[dev_hip]" + # For lmms_evals evaluating MMMU + docker exec -w / ci_sglang git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + docker exec -w /lmms-eval ci_sglang pip install -e . + ;; + *) + echo "Runner architecture '${GPU_ARCH}' unrecognised;" >&2 + ;; +esac + +docker exec -w / ci_sglang git clone https://github.com/merrymercy/human-eval.git +docker exec -w /human-eval ci_sglang pip install -e . + +docker exec -w / ci_sglang mkdir -p /dummy-grok +mkdir -p dummy-grok && wget https://sharkpublic.blob.core.windows.net/sharkpublic/sglang/dummy_grok.json -O dummy-grok/config.json +docker cp ./dummy-grok ci_sglang:/ + +docker exec ci_sglang pip install huggingface_hub[hf_xet] +docker exec ci_sglang pip install pytest diff --git a/scripts/ci/amd_ci_start_container.sh b/scripts/ci/amd_ci_start_container.sh new file mode 100755 index 000000000..a1f281c8d --- /dev/null +++ b/scripts/ci/amd_ci_start_container.sh @@ -0,0 +1,132 @@ +#!/bin/bash +set -euo pipefail + +# Get version from SGLang version.py file +SGLANG_VERSION_FILE="$(dirname "$0")/../../python/sglang/version.py" +SGLANG_VERSION="v0.5.0rc0" # Default version, will be overridden if version.py is found + +if [ -f "$SGLANG_VERSION_FILE" ]; then + VERSION_FROM_FILE=$(python3 -c ' +import re, sys +with open(sys.argv[1], "r") as f: + content = f.read() + match = re.search(r"__version__\s*=\s*[\"'"'"'](.*?)[\"'"'"']", content) + if match: + print("v" + match.group(1)) +' "$SGLANG_VERSION_FILE" 2>/dev/null || echo "") + + if [ -n "$VERSION_FROM_FILE" ]; then + SGLANG_VERSION="$VERSION_FROM_FILE" + echo "Using SGLang version from version.py: $SGLANG_VERSION" + else + echo "Warning: Could not parse version from $SGLANG_VERSION_FILE, using default: $SGLANG_VERSION" >&2 + fi +else + echo "Warning: version.py not found, using default version: $SGLANG_VERSION" >&2 +fi + + +# Default base tags (can be overridden by command line arguments) +DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-rocm630-mi30x" +DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-rocm700-mi35x" + +# Parse command line arguments +MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}" +MI35X_BASE_TAG="${DEFAULT_MI35X_BASE_TAG}" + +while [[ $# -gt 0 ]]; do + case $1 in + --mi30x-base-tag) MI30X_BASE_TAG="$2"; shift 2;; + --mi35x-base-tag) MI35X_BASE_TAG="$2"; shift 2;; + -h|--help) + echo "Usage: $0 [--mi30x-base-tag TAG] [--mi35x-base-tag TAG]" + exit 0 + ;; + *) echo "Unknown option $1"; exit 1;; + esac +done + + + +# Detect GPU architecture from the Kubernetes runner hostname +HOSTNAME_VALUE=$(hostname) +GPU_ARCH="mi30x" # default + +# Host names look like: linux-mi35x-gpu-1-xxxxx-runner-zzzzz +if [[ "${HOSTNAME_VALUE}" =~ ^linux-(mi[0-9]+[a-z]*)-gpu-[0-9]+ ]]; then + GPU_ARCH="${BASH_REMATCH[1]}" + echo "Detected GPU architecture from hostname: ${GPU_ARCH}" +else + echo "Warning: could not parse GPU architecture from '${HOSTNAME_VALUE}', defaulting to ${GPU_ARCH}" +fi + +# Normalise / collapse architectures we don’t yet build specifically for +case "${GPU_ARCH}" in + mi35x) + echo "Runner uses ${GPU_ARCH}; will fetch mi35x image." + ;; + mi30x|mi300|mi325) + echo "Runner uses ${GPU_ARCH}; will fetch mi30x image." + GPU_ARCH="mi30x" + ;; + *) + echo "Runner architecture '${GPU_ARCH}' unrecognised; defaulting to mi30x image." >&2 + GPU_ARCH="mi30x" + ;; +esac + + +# Set up DEVICE_FLAG based on Kubernetes pod info +if [[ -f /etc/podinfo/gha-render-devices ]]; then + DEVICE_FLAG=$(cat /etc/podinfo/gha-render-devices) +else + DEVICE_FLAG="--device /dev/dri" +fi + + +# Find the latest image +find_latest_image() { + local gpu_arch=$1 + local base_tag days_back image_tag + + case "${gpu_arch}" in + mi30x) base_tag="${MI30X_BASE_TAG}" ;; + mi35x) base_tag="${MI35X_BASE_TAG}" ;; + *) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;; + esac + + for days_back in {0..6}; do + image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)" + echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2 + if docker manifest inspect "rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then + echo "Found available image: rocm/sgl-dev:${image_tag}" >&2 + echo "rocm/sgl-dev:${image_tag}" + return 0 + fi + done + + echo "Error: no ${gpu_arch} image found in the last 7 days for base ${base_tag}" >&2 + echo "Using hard-coded fallback…" >&2 + if [[ "${gpu_arch}" == "mi35x" ]]; then + echo "rocm/sgl-dev:v0.5.0rc0-rocm700-mi35x-20250812" + else + echo "rocm/sgl-dev:v0.5.0rc0-rocm630-mi30x-20250812" + fi +} + +# Pull and run the latest image +IMAGE=$(find_latest_image "${GPU_ARCH}") +echo "Pulling Docker image: ${IMAGE}" +docker pull "${IMAGE}" + +echo "Launching container: ci_sglang" +docker run -dt --user root --device=/dev/kfd ${DEVICE_FLAG} \ + -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \ + --ipc=host --group-add video \ + --shm-size 32g \ + --cap-add=SYS_PTRACE \ + -e HF_TOKEN="${HF_TOKEN:-}" \ + --security-opt seccomp=unconfined \ + -w /sglang-checkout \ + --name ci_sglang \ + "${IMAGE}" diff --git a/scripts/ci/ci_install_deepep.sh b/scripts/ci/ci_install_deepep.sh new file mode 100755 index 000000000..4da81eec1 --- /dev/null +++ b/scripts/ci/ci_install_deepep.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Install the dependency in CI. +set -euxo pipefail + +bash scripts/ci/ci_install_dependency.sh + +export GDRCOPY_HOME=/usr/src/gdrdrv-2.4.4/ +export NVSHMEM_DIR=/opt/nvshmem/install +export LD_LIBRARY_PATH="${NVSHMEM_DIR}/lib:$LD_LIBRARY_PATH" +export PATH="${NVSHMEM_DIR}/bin:$PATH" +export CUDA_HOME=/usr/local/cuda + +if python3 -c "import deep_ep" >/dev/null 2>&1; then + echo "deep_ep is already installed or importable. Skipping installation." + exit 0 +fi + +# Install system dependencies +apt install -y curl wget git sudo libibverbs-dev rdma-core infiniband-diags openssh-server perftest ibverbs-providers libibumad3 libibverbs1 libnl-3-200 libnl-route-3-200 librdmacm1 build-essential cmake + +# Install GDRCopy +rm -rf /opt/gdrcopy && mkdir -p /opt/gdrcopy +rm -rf /opt/nvshmem && mkdir -p /opt/nvshmem +cd /opt/gdrcopy +git clone https://github.com/NVIDIA/gdrcopy.git . +git checkout v2.4.4 +apt update +apt install -y nvidia-dkms-535 +apt install -y build-essential devscripts debhelper fakeroot pkg-config dkms +apt install -y check libsubunit0 libsubunit-dev python3-venv +cd packages +CUDA=/usr/local/cuda ./build-deb-packages.sh +dpkg -i gdrdrv-dkms_*.deb +dpkg -i libgdrapi_*.deb +dpkg -i gdrcopy-tests_*.deb +dpkg -i gdrcopy_*.deb + +if [ ! -e "/usr/lib/x86_64-linux-gnu/libmlx5.so" ]; then + ln -s /usr/lib/x86_64-linux-gnu/libmlx5.so.1 /usr/lib/x86_64-linux-gnu/libmlx5.so +fi +apt-get update && apt-get install -y libfabric-dev + +# Install NVSHMEM +cd /opt/nvshmem +wget https://developer.download.nvidia.com/compute/redist/nvshmem/3.3.9/source/nvshmem_src_cuda12-all-all-3.3.9.tar.gz +tar -xf nvshmem_src_cuda12-all-all-3.3.9.tar.gz +mv nvshmem_src nvshmem && cd nvshmem +NVSHMEM_SHMEM_SUPPORT=0 \ +NVSHMEM_UCX_SUPPORT=0 \ +NVSHMEM_USE_NCCL=0 \ +NVSHMEM_MPI_SUPPORT=0 \ +NVSHMEM_IBGDA_SUPPORT=1 \ +NVSHMEM_PMIX_SUPPORT=0 \ +NVSHMEM_TIMEOUT_DEVICE_POLLING=0 \ +NVSHMEM_USE_GDRCOPY=1 \ +cmake -S . -B build/ -DCMAKE_INSTALL_PREFIX=/opt/nvshmem/install -DCMAKE_CUDA_ARCHITECTURES=90 +cd build +make -j$(nproc) install + +# Install DeepEP +rm -rf /root/.cache/deepep && git clone https://github.com/deepseek-ai/DeepEP.git /root/.cache/deepep && cd /root/.cache/deepep && git checkout b92d0d4860ce6866cd6d31bfbae937f9a7a3772b +cd /root/.cache/deepep && python3 setup.py install + +# Verify configuration +echo "=== Verify GDRCOPY ===" +gdrcopy_copybw +echo "=== Verify NVSHMEM ===" +nvshmem-info -a diff --git a/scripts/ci/ci_install_dependency.sh b/scripts/ci/ci_install_dependency.sh new file mode 100755 index 000000000..e007121a3 --- /dev/null +++ b/scripts/ci/ci_install_dependency.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Install the dependency in CI. +set -euxo pipefail + +IS_BLACKWELL=${IS_BLACKWELL:-0} + +if [ "$IS_BLACKWELL" = "1" ]; then + CU_VERSION="cu129" +else + CU_VERSION="cu126" +fi + +# Clear torch compilation cache +python3 -c 'import os, shutil, tempfile, getpass; cache_dir = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join(tempfile.gettempdir(), "torchinductor_" + getpass.getuser()); shutil.rmtree(cache_dir, ignore_errors=True)' + +# Kill existing processes +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +bash "${SCRIPT_DIR}/../killall_sglang.sh" + +# Install apt packages +apt install -y git libnuma-dev + +# Install uv +if [ "$IS_BLACKWELL" = "1" ]; then + # The blackwell CI runner has some issues with pip and uv, + # so we can only use pip with `--break-system-packages` + PIP_CMD="pip" + PIP_INSTALL_SUFFIX="--break-system-packages" + + # Clean up existing installations + $PIP_CMD uninstall -y flashinfer_python sgl-kernel sglang vllm $PIP_INSTALL_SUFFIX || true +else + # In normal cases, we use uv, which is much faster than pip. + pip install --upgrade pip + pip install uv + export UV_SYSTEM_PYTHON=true + + PIP_CMD="uv pip" + PIP_INSTALL_SUFFIX="--index-strategy unsafe-best-match" + + # Clean up existing installations + $PIP_CMD uninstall flashinfer_python sgl-kernel sglang vllm || true +fi + +# Install the main package +$PIP_CMD install -e "python[dev]" --extra-index-url https://download.pytorch.org/whl/${CU_VERSION} $PIP_INSTALL_SUFFIX + +# Install router for pd-disagg test +SGLANG_ROUTER_BUILD_NO_RUST=1 $PIP_CMD install -e "sgl-router" $PIP_INSTALL_SUFFIX + + +SGL_KERNEL_VERSION=0.3.9.post2 +if [ "$IS_BLACKWELL" = "1" ]; then + # TODO auto determine sgl-kernel version + $PIP_CMD install https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu128-cp310-abi3-manylinux2014_x86_64.whl --force-reinstall $PIP_INSTALL_SUFFIX +else + $PIP_CMD install https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu124-cp310-abi3-manylinux2014_x86_64.whl --force-reinstall $PIP_INSTALL_SUFFIX +fi + +# Show current packages +$PIP_CMD list + +# Install additional dependencies +$PIP_CMD install mooncake-transfer-engine==0.3.5 nvidia-cuda-nvrtc-cu12 py-spy huggingface_hub[hf_xet] $PIP_INSTALL_SUFFIX + +if [ "$IS_BLACKWELL" != "1" ]; then + # For lmms_evals evaluating MMMU + git clone --branch v0.3.3 --depth 1 https://github.com/EvolvingLMMs-Lab/lmms-eval.git + $PIP_CMD install -e lmms-eval/ $PIP_INSTALL_SUFFIX + + # Install xformers + $PIP_CMD install xformers --index-url https://download.pytorch.org/whl/${CU_VERSION} --no-deps $PIP_INSTALL_SUFFIX +fi + +# Install FlashMLA for attention backend tests +# $PIP_CMD install git+https://github.com/deepseek-ai/FlashMLA.git $PIP_INSTALL_SUFFIX + +# Show current packages +$PIP_CMD list + +echo "CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-}" diff --git a/scripts/ci/ci_install_rust.sh b/scripts/ci/ci_install_rust.sh new file mode 100755 index 000000000..ac042fc9a --- /dev/null +++ b/scripts/ci/ci_install_rust.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euxo pipefail + +# Check if sudo is available +if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y libssl-dev pkg-config protobuf-compiler +else + apt-get update + apt-get install -y libssl-dev pkg-config protobuf-compiler +fi + +# Install rustup (Rust installer and version manager) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + + +# Follow the installation prompts, then reload your shell +. "$HOME/.cargo/env" +source $HOME/.cargo/env + +# Verify installation +rustc --version +cargo --version +protoc --version diff --git a/scripts/ci/ci_start_disaggregation_servers.sh b/scripts/ci/ci_start_disaggregation_servers.sh new file mode 100755 index 000000000..56490bb06 --- /dev/null +++ b/scripts/ci/ci_start_disaggregation_servers.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +MODEL_PATH="/raid/models/meta-llama/Llama-3.1-8B-Instruct" + +# Function to find the first available active IB device +find_active_ib_device() { + for device in mlx5_{0..11}; do + if ibv_devinfo $device >/dev/null 2>&1; then + state=$(ibv_devinfo $device | grep "state:" | head -1 | awk '{print $2}') + if [[ "$state" == "PORT_ACTIVE" ]]; then + echo "$device" + return 0 + fi + fi + done + echo "No active IB device found" >&2 + return 1 +} + +# Get the first available active IB device +DEVICE=$(find_active_ib_device) +echo "Using IB device: $DEVICE" + +# Launch prefill servers on GPU 0–3 +for i in {0..3}; do + PORT=$((30001 + i)) + BOOTSTRAP_PORT=$((9001 + i)) + HOST="127.0.0.$((i + 1))" + echo "Launching PREFILL server on GPU $i at $HOST:$PORT (bootstrap: $BOOTSTRAP_PORT)" + CUDA_VISIBLE_DEVICES=$i \ + python3 -m sglang.launch_server \ + --model-path "$MODEL_PATH" \ + --disaggregation-mode prefill \ + --host "$HOST" \ + --port "$PORT" \ + --disaggregation-ib-device "$DEVICE" \ + --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" & +done + +# Launch decode servers on GPU 4–7 +for i in {4..7}; do + PORT=$((30001 + i)) + HOST="127.0.0.$((i + 1))" + echo "Launching DECODE server on GPU $i at $HOST:$PORT" + CUDA_VISIBLE_DEVICES=$i \ + python3 -m sglang.launch_server \ + --model-path "$MODEL_PATH" \ + --disaggregation-mode decode \ + --host "$HOST" \ + --port "$PORT" \ + --disaggregation-ib-device "$DEVICE" \ + --base-gpu-id 0 & +done + +# Wait for disaggregation servers to initialize +echo "Waiting for disaggregation servers to initialize..." + +# Health check with 5-minute timeout +TIMEOUT=300 +START_TIME=$(date +%s) + +echo "Checking health of all 8 servers..." +while true; do + CURRENT_TIME=$(date +%s) + ELAPSED=$((CURRENT_TIME - START_TIME)) + + if [ $ELAPSED -ge $TIMEOUT ]; then + echo "❌ Timeout: Servers did not become healthy within 5 minutes" + exit 1 + fi + + HEALTHY_COUNT=0 + # Check all 8 servers (127.0.0.1-8:30001-30008) + for i in {1..8}; do + if curl -s -f "http://127.0.0.$i:$((30000 + i))/health" >/dev/null 2>&1; then + HEALTHY_COUNT=$((HEALTHY_COUNT + 1)) + fi + done + + echo "Healthy servers: $HEALTHY_COUNT/8 (elapsed: ${ELAPSED}s)" + + if [ $HEALTHY_COUNT -eq 8 ]; then + echo "✅ All 8 servers are healthy!" + break + else + sleep 10 # Wait 10 seconds before next check + fi +done + +# Don't launch router here - just keep servers running +echo "✅ All disaggregation servers are ready and waiting for router connections" + +# Keep the script running +wait diff --git a/scripts/ci/npu_ci_install_dependency.sh b/scripts/ci/npu_ci_install_dependency.sh new file mode 100755 index 000000000..71cf46f7f --- /dev/null +++ b/scripts/ci/npu_ci_install_dependency.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -euo pipefail + +PIP_INSTALL="pip install --no-cache-dir" + + +# Install the required dependencies in CI. +apt update -y && apt install -y \ + build-essential \ + cmake \ + wget \ + curl \ + net-tools \ + zlib1g-dev \ + lld \ + clang \ + locales \ + ccache \ + ca-certificates +update-ca-certificates +python3 -m ${PIP_INSTALL} --upgrade pip + + +### Download MemFabricV2 +MF_WHL_NAME="mf_adapter-1.0.0-cp311-cp311-linux_aarch64.whl" +MEMFABRIC_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/${MF_WHL_NAME}" +wget -O "${MF_WHL_NAME}" "${MEMFABRIC_URL}" && ${PIP_INSTALL} "./${MF_WHL_NAME}" + + +### Install vLLM +VLLM_TAG=v0.8.5 +git clone --depth 1 https://github.com/vllm-project/vllm.git --branch $VLLM_TAG +(cd vllm && VLLM_TARGET_DEVICE="empty" ${PIP_INSTALL} -v -e .) + + +### Install PyTorch and PTA +PYTORCH_VERSION=2.6.0 +TORCHVISION_VERSION=0.21.0 +${PIP_INSTALL} torch==$PYTORCH_VERSION torchvision==$TORCHVISION_VERSION --index-url https://download.pytorch.org/whl/cpu + +PTA_VERSION="v7.1.0.1-pytorch2.6.0" +PTA_NAME="torch_npu-2.6.0.post1-cp311-cp311-manylinux_2_28_aarch64.whl" +PTA_URL="https://gitee.com/ascend/pytorch/releases/download/${PTA_VERSION}/${PTA_NAME}" +wget -O "${PTA_NAME}" "${PTA_URL}" && ${PIP_INSTALL} "./${PTA_NAME}" + + +### Install Triton-Ascend +TRITON_ASCEND_NAME="triton_ascend-3.2.0.dev20250729-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" +TRITON_ASCEND_URL="https://sglang-ascend.obs.cn-east-3.myhuaweicloud.com/sglang/${TRITON_ASCEND_NAME}" +${PIP_INSTALL} attrs==24.2.0 numpy==1.26.4 scipy==1.13.1 decorator==5.1.1 psutil==6.0.0 pytest==8.3.2 pytest-xdist==3.6.1 pyyaml pybind11 +wget -O "${TRITON_ASCEND_NAME}" "${TRITON_ASCEND_URL}" && ${PIP_INSTALL} "./${TRITON_ASCEND_NAME}" + + +### Install sgl-kernel-npu +SGL_KERNEL_NPU_TAG="20250901" +git clone --depth 1 https://github.com/sgl-project/sgl-kernel-npu.git --branch ${SGL_KERNEL_NPU_TAG} +(cd sgl-kernel-npu && bash ./build.sh -a deepep && pip install output/deep_ep*.whl && cd "$(pip show deep-ep | grep -E '^Location:' | awk '{print $2}')" && ln -s deep_ep/deep_ep_cpp*.so) + + +### Install SGLang +${PIP_INSTALL} -v -e "python[srt_npu]" diff --git a/scripts/code_sync/copy_from_oss.py b/scripts/code_sync/copy_from_oss.py new file mode 100644 index 000000000..5590a73a0 --- /dev/null +++ b/scripts/code_sync/copy_from_oss.py @@ -0,0 +1,293 @@ +""" +Sync code from OSS repo to the local repo and open a PR if changes exist. + +NOTE: +1. You need to execute this script in the git root folder. +2. A GH_TOKEN environment variable is required to create the pull request. + - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +This script will: +1. Clone the sgl-project/sglang repository (or use a local copy). +2. Sync specified files and directories using rsync. +3. Check if the sync operation resulted in any changes. +4. If there are changes: + a. Create a new branch. + b. Commit and push the changes. + c. Open a pull request using the GitHub CLI (gh). + +Usage: +# Run the full sync and PR creation process +python3 scripts/copy_from_oss.py + +# Perform a dry run without making any actual changes +python3 scripts/copy_from_oss.py --dry-run + +# Use a local directory as the source instead of cloning +python3 scripts/copy_from_oss.py --local-dir ~/projects/sglang +""" + +import argparse +import datetime +import os +import shutil +import subprocess +import tempfile + +# --- Configuration Begin --- +# List of folders and files to copy from the OSS repo. +# Changes outside these paths will be ignored. +folder_names = [ + "3rdparty", + "assets", + "benchmark", + "docker", + "docs", + "examples", + "sgl-kernel", + "README.md", + "python/sglang/lang", + "python/sglang/srt", + "python/sglang/test", + "test/lang", + "test/srt", +] + +private_repo = "your-org/sglang-private-repo" +# --- Configuration End --- + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def check_dependencies(): + """Check for required command-line tools.""" + if not shutil.which("git"): + raise EnvironmentError("git is not installed or not in PATH.") + if not shutil.which("gh"): + raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.") + print("✅ All dependencies (git, gh) are available.") + + +def checkout_main(dry_run): + """Checkout to the main branch.""" + commands = [ + "git checkout main", + "git reset --hard", + ] + for cmd in commands: + print(f"Run: {cmd}") + if not dry_run: + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr.decode()}") + raise + print("✅ Checkout the main branch.") + + +def get_source_folder(args): + """ + Prepare the source repository, either by cloning from GitHub or using a local directory. + Returns the path to the source repo root, a temporary directory path (if created), + and the short commit hash. + """ + temp_dir = None + if args.local_dir: + oss_root = os.path.expanduser(args.local_dir) + if not os.path.exists(oss_root): + raise FileNotFoundError( + f"Specified local directory {oss_root} does not exist." + ) + print(f"Using local directory as the source: {oss_root}") + else: + temp_dir = tempfile.mkdtemp() + oss_root = temp_dir + print(f"Created temporary directory: {oss_root}") + + repo_url = "https://github.com/sgl-project/sglang.git" + try: + subprocess.run( + [ + "git", + "clone", + "--single-branch", + "--branch", + "main", + repo_url, + temp_dir, + ], + check=True, + capture_output=True, + ) + print(f"Successfully cloned repository to {temp_dir}") + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e.stderr.decode()}") + raise + + commit_hash = subprocess.run( + ["git", "-C", oss_root, "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip()[:8] + print(f"✅ Get source OSS code at commit: {commit_hash}") + return oss_root, temp_dir, commit_hash + + +def sync_directories(oss_root, folder_names, dry_run): + """Sync specified directories from oss_root to current working directory.""" + rsync_commands = [] + for folder_name in folder_names: + target_name = f"{oss_root}/{folder_name}" + src_name = "./" + "/".join(folder_name.split("/")[:-1]) + cmd = f"rsync -r --delete {target_name} {src_name}" + rsync_commands.append(cmd) + + for cmd in rsync_commands: + try: + print(f"Run: {cmd}") + if not dry_run: + subprocess.run(cmd, shell=True, check=True) + except subprocess.CalledProcessError as e: + print(f"Error executing command '{cmd}': {e}") + raise + print(f"✅ Sync all folders.") + + +def check_for_changes(): + """Check if there are any uncommitted git changes.""" + # This command exits with 1 if there are changes, 0 otherwise. + result = subprocess.run(["git", "diff", "--quiet"]) + return result.returncode != 0 + + +def create_and_push_branch(branch_name, commit_message, dry_run): + """Create a new branch, commit all changes, and push to origin.""" + commands = [ + f"git checkout -b {branch_name}", + "git config user.name 'github-actions[bot]'", + "git config user.email 'github-actions[bot]@users.noreply.github.com'", + "git add .", + f"git commit -m '{commit_message}'", + f"git push origin {branch_name} --force", + ] + print("\nCreating and pushing git branch...") + for cmd in commands: + print(f"Run: {cmd}") + if not dry_run: + try: + subprocess.run(cmd, shell=True, check=True, capture_output=True) + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr.decode()}") + raise + + +def create_pull_request(branch_name, title, body, dry_run): + """Create a pull request using the GitHub CLI.""" + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print( + "\n⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation." + ) + if not dry_run: + return + + print("\nCreating pull request...") + command = [ + "gh", + "pr", + "create", + "--base", + "main", + "--head", + branch_name, + "--repo", + private_repo, + "--title", + title, + "--body", + body, + ] + print(f"Run: {' '.join(command)}") + if not dry_run: + env = os.environ.copy() + env["GH_TOKEN"] = gh_token + try: + result = subprocess.run( + command, check=True, capture_output=True, text=True, env=env + ) + pr_url = result.stdout.strip() + msg = f"✅ Successfully created pull request: {pr_url}" + print(msg) + write_github_step_summary(msg) + except subprocess.CalledProcessError as e: + print(f"Error creating pull request: {e.stderr}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Copy code from OSS and open a PR if changes are detected." + ) + parser.add_argument( + "--local-dir", + type=str, + help="Path to local SGLang directory to use instead of cloning from GitHub.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run the script without executing git, rsync, or gh commands.", + ) + args = parser.parse_args() + + check_dependencies() + checkout_main(args.dry_run) + + oss_root, temp_dir, oss_commit = get_source_folder(args) + + try: + # Sync directories + sync_directories(oss_root, folder_names, args.dry_run) + + # Check for changes and create PR if necessary + if not check_for_changes(): + msg = "😴 No changes detected. The code is already in sync." + print(msg) + write_github_step_summary(msg) + return + + print("✅ Changes detected. Proceeding to create a PR.") + + current_date = datetime.datetime.now().strftime("%Y%m%d") + branch_name = f"copy-from-oss-{oss_commit}-{current_date}" + commit_message = f"Copy OSS code from {oss_commit} on {current_date}" + pr_title = ( + f"[Automated PR] Copy OSS code from commit {oss_commit} on {current_date}" + ) + pr_body = ( + f"Copy OSS code from https://github.com/sgl-project/sglang/commit/{oss_commit} on {current_date}." + "\n\n---\n\n" + "*This is an automated PR created by scripts/copy_from_oss.py.*" + ) + + create_and_push_branch(branch_name, commit_message, args.dry_run) + create_pull_request(branch_name, pr_title, pr_body, args.dry_run) + + finally: + # Remove temporary directory if it was created + if temp_dir: + try: + shutil.rmtree(temp_dir) + print(f"\nRemoved temporary directory: {temp_dir}") + except OSError as e: + print(f"Error removing temporary directory {temp_dir}: {e}") + + +if __name__ == "__main__": + main() diff --git a/scripts/code_sync/copy_to_oss.py b/scripts/code_sync/copy_to_oss.py new file mode 100644 index 000000000..cd931ffd6 --- /dev/null +++ b/scripts/code_sync/copy_to_oss.py @@ -0,0 +1,425 @@ +""" +Sync a specific commit from the local private repo to the OSS upstream and open a PR. + +NOTE: +1. You need to execute this script in the git root folder. +2. A GH_TOKEN environment variable is required to create the pull request. + - see also https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + +This script will: +1. Take a commit hash as an argument (or use the latest commit by default). +2. Create a patch for that commit. +3. Filter the patch to only include changes in specified directories. +4. Clone the sgl-project/sglang repository. +5. Create a new branch in the OSS repo. +6. Apply the filtered patch, commit, and force push. +7. Open a pull request to the OSS repo using the GitHub CLI (gh). + +Usage: +# Sync the latest commit from the current branch +python3 scripts/copy_to_oss.py + +# Run the full sync and PR creation process for a given commit +python3 scripts/copy_to_oss.py --commit + +# Perform a dry run without making any actual changes +python3 scripts/copy_to_oss.py --commit --dry-run +""" + +import argparse +import datetime +import os +import shutil +import subprocess +import tempfile + +# --- Configuration Begin --- +# List of folders and files to copy to the OSS repo. +# Changes outside these paths will be ignored. +folder_names = [ + "3rdparty", + "assets", + "benchmark", + "docker", + "docs", + "examples", + "sgl-kernel", + "README.md", + "python/sglang/lang", + "python/sglang/srt", + "python/sglang/test", + "test/lang", + "test/srt", +] + +# --- Configuration End --- + + +def write_github_step_summary(content): + if not os.environ.get("GITHUB_STEP_SUMMARY"): + return + + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(content) + + +def get_commit_info(commit_ref): + """ + Retrieves the hash and message of a specific commit. + + Args: + commit_ref (str): The commit hash, tag, or branch to inspect (e.g., 'HEAD'). + + Returns: + A tuple containing the (commit_hash, commit_message), + or (None, None) if an error occurs. + """ + try: + # Use a custom format to get the hash (%H) and the full message (%B) + # separated by a null character for safe parsing. + command = ["git", "log", "-1", f"--pretty=%H%x00%B", commit_ref] + result = subprocess.run( + command, capture_output=True, text=True, check=True, encoding="utf-8" + ) + + # Split the output by the null character separator + commit_hash, commit_message = result.stdout.strip().split("\x00", 1) + return commit_hash, commit_message + + except FileNotFoundError: + print("❌ Error: 'git' command not found. Is Git installed and in your PATH?") + except subprocess.CalledProcessError as e: + print(f"❌ Error getting commit info for '{commit_ref}': {e.stderr.strip()}") + print( + "Hint: Make sure you are running this from within a Git repository and the commit exists." + ) + + return None, None + + +def check_dependencies(): + """Check for required command-line tools.""" + if not shutil.which("git"): + raise EnvironmentError("git is not installed or not in PATH.") + if not shutil.which("gh"): + raise EnvironmentError("GitHub CLI (gh) is not installed or not in PATH.") + print("✅ All dependencies (git, gh) are available.") + + +def create_filtered_patch(commit_hash, dry_run): + """ + Create a patch file for the given commit, containing only changes + to files and directories specified in `folder_names`. + """ + print(f"Creating a filtered patch for commit {commit_hash}") + + try: + # Get the list of all files changed in the commit + changed_files_raw = subprocess.run( + ["git", "diff-tree", "--no-commit-id", "--name-only", "-r", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout + changed_files = changed_files_raw.strip().split("\n") + + # Filter the list of files + relevant_files = [ + f for f in changed_files if any(f.startswith(path) for path in folder_names) + ] + + if not relevant_files: + msg = "\n😴 No relevant file changes found in this commit. Exiting." + print(msg) + write_github_step_summary(msg) + return None, None + + print("Found relevant changes in the following files:") + for f in relevant_files: + print(f" - {f}") + + # Create a patch containing only the changes for the relevant files + patch_command = [ + "git", + "format-patch", + "--stdout", + f"{commit_hash}^..{commit_hash}", + "--", + ] + relevant_files + + print(f"Run: {' '.join(patch_command)}") + + patch_content = subprocess.run( + patch_command, capture_output=True, text=True, check=True + ).stdout + + # Save the patch to a temporary file + patch_file = tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".patch", encoding="utf-8" + ) + patch_file.write(patch_content) + patch_file.close() + + print(f"✅ Filtered patch created successfully at: {patch_file.name}") + return patch_file.name, relevant_files + + except subprocess.CalledProcessError as e: + print(f"Error creating patch: {e.stderr}") + raise + + +def get_oss_repo(dry_run): + """ + Clones the OSS repository into a temporary directory. + Returns the path to the repo root and the temp directory itself. + """ + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + if not dry_run: + return + + temp_dir = tempfile.mkdtemp() + oss_root = os.path.join(temp_dir, "sglang") + print(f"\nCreated temporary directory for OSS repo: {temp_dir}") + + repo_url = f"https://{gh_token}@github.com/sgl-project/sglang.git" + command = ["git", "clone", "--branch", "main", repo_url, oss_root] + + print(f"Run: {' '.join(command)}") + if not dry_run: + try: + subprocess.run(command, check=True, capture_output=True) + print(f"✅ Successfully cloned repository to {oss_root}") + except subprocess.CalledProcessError as e: + print(f"Error cloning repository: {e.stderr.decode()}") + shutil.rmtree(temp_dir) + raise + + return oss_root, temp_dir + + +def apply_patch_and_push(oss_root, patch_file, branch_name, commit_message, dry_run): + """ + In the OSS repo, create a branch, apply the patch, commit, and push. + """ + print("\nApplying patch and pushing to OSS repo...") + + original_cwd = os.getcwd() + if not dry_run: + os.chdir(oss_root) + + try: + # Define commands as lists to avoid shell injection issues + commands_to_run = [ + ["git", "checkout", "-b", branch_name], + ["git", "apply", patch_file], + ["git", "config", "user.name", "github-actions[bot]"], + [ + "git", + "config", + "user.email", + "github-actions[bot]@users.noreply.github.com", + ], + ["git", "add", "."], + ] + + for cmd_list in commands_to_run: + print(f"Run: {' '.join(cmd_list)}") + if not dry_run: + subprocess.run(cmd_list, check=True, capture_output=True, text=True) + + # Handle commit separately to pass multi-line message safely via stdin + commit_cmd = ["git", "commit", "-F", "-"] + print(f"Run: {' '.join(commit_cmd)}") + if not dry_run: + print(f"Commit Message:\n---\n{commit_message}\n---") + subprocess.run( + commit_cmd, + input=commit_message, + text=True, + check=True, + capture_output=True, + ) + + # Push the changes + push_cmd = ["git", "push", "origin", branch_name, "--force"] + print(f"Run: {' '.join(push_cmd)}") + if not dry_run: + subprocess.run(push_cmd, check=True, capture_output=True, text=True) + + except subprocess.CalledProcessError as e: + print(f"Git command failed: {e.stderr}") + raise + finally: + if not dry_run: + os.chdir(original_cwd) + + print("✅ Branch created, patch applied, and pushed successfully.") + + +def create_pull_request(oss_root, branch_name, title, body, dry_run): + """Create a pull request in the OSS repo using the GitHub CLI.""" + gh_token = os.getenv("GH_TOKEN") + if not gh_token: + print("⚠️ Warning: GH_TOKEN environment variable not set. Skipping PR creation.") + if not dry_run: + return + + print("\nCreating pull request...") + command = [ + "gh", + "pr", + "create", + "--base", + "main", + "--head", + branch_name, + "--repo", + "sgl-project/sglang", + "--title", + title, + "--body", + body, + ] + + print(f"Run: {' '.join(command)}") + if not dry_run: + env = os.environ.copy() + env["GH_TOKEN"] = gh_token + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + env=env, + cwd=oss_root, + ) + msg = f"✅ Successfully created pull request: {result.stdout.strip()}" + print(msg) + write_github_step_summary(msg) + except subprocess.CalledProcessError as e: + print(f"Error creating pull request: {e.stderr}") + # Check if a PR already exists + if "A pull request for" in e.stderr and "already exists" in e.stderr: + print("ℹ️ A PR for this branch likely already exists.") + else: + raise + + +def get_commit_author(commit_hash): + """Get the author name and email of a commit.""" + try: + author_name = subprocess.run( + ["git", "show", "-s", "--format=%an", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + author_email = subprocess.run( + ["git", "show", "-s", "--format=%ae", commit_hash], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return author_name, author_email + except subprocess.CalledProcessError as e: + print(f"Error getting commit author for {commit_hash}: {e.stderr}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description="Copy a commit from the private repo to OSS and open a PR." + ) + parser.add_argument( + "--commit", + type=str, + default="LAST", + help="The commit hash to sync. Defaults to 'LAST' to use the latest commit.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Dry run the script without executing git, rsync, or gh commands.", + ) + args = parser.parse_args() + + check_dependencies() + + commit_ref = "HEAD" if args.commit == "LAST" else args.commit + commit_hash, original_commit_message = get_commit_info(commit_ref) + + if not commit_hash: + return # Exit if we couldn't get commit info + + # Display the details of the commit being processed + if args.commit == "LAST": + summary = ( + f"\nℹ️ No commit specified. Using the last commit:\n" + f" - **Hash:** `{commit_hash}`\n" + f" - **Message:** {original_commit_message}\n\n" + ) + else: + summary = ( + f"\nℹ️ Using specified commit:\n" + f" - **Hash:** `{commit_hash}`\n" + f" - **Message:** {original_commit_message}\n\n" + ) + print(summary) + write_github_step_summary(summary) + + short_hash = commit_hash[:8] + + patch_file = None + temp_dir = None + try: + # 1. Create a filtered patch from the local repo + patch_file, relevant_files = create_filtered_patch(commit_hash, args.dry_run) + if not patch_file: + return + + # 2. Get the OSS repo + oss_root, temp_dir = get_oss_repo(args.dry_run) + + # 3. Get original commit author for the co-author line + author_name, author_email = get_commit_author(commit_hash) + + # 4. Prepare content for the commit and PR based on changed files + file_list_str = "\n".join([f"- {f}" for f in relevant_files]) + filename_list_str = ", ".join([f.split("/")[-1] for f in relevant_files]) + if len(filename_list_str) > 40: + filename_list_str = filename_list_str[:40] + "..." + current_date = datetime.datetime.now().strftime("%Y%m%d") + pr_title = f"[Auto Sync] Update {filename_list_str} ({current_date})" + pr_body = ( + f"Sync changes from commit `{short_hash}`.\n\n" + f"**Relevant Files Changed:**\n{file_list_str}" + "\n\n---\n\n" + "*This is an automated PR created by a script.*" + ) + + # 5. Create branch, apply patch, and push + branch_name = f"sync-{short_hash}-{current_date}" + co_author_line = f"Co-authored-by: {author_name} <{author_email}>" + commit_message = f"{pr_title}\n\n{co_author_line}" + apply_patch_and_push( + oss_root, patch_file, branch_name, commit_message, args.dry_run + ) + + # 6. Create Pull Request + create_pull_request(oss_root, branch_name, pr_title, pr_body, args.dry_run) + + finally: + # Cleanup temporary files + if patch_file and os.path.exists(patch_file): + os.remove(patch_file) + print(f"\nRemoved temporary patch file: {patch_file}") + if temp_dir and os.path.exists(temp_dir): + shutil.rmtree(temp_dir) + print(f"Removed temporary directory: {temp_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/code_sync/guideline.md b/scripts/code_sync/guideline.md new file mode 100644 index 000000000..52f08eb4b --- /dev/null +++ b/scripts/code_sync/guideline.md @@ -0,0 +1,27 @@ +### Sync Code Between OSS and Private Fork + +You can use the following principles and tools to sync the code between a private fork and the OSS repo [sgl-project/sglang](https://github.com/sgl-project/sglang/tree/main). +It learns from [Copybara](https://github.com/google/copybara), a tool used at Google for maintaining open-source code synchronization. + +## Principals + +- The core folders (e.g., `python/sglang/srt`) are 100% mirrored between the private fork and OSS repo. +- The OSS repo is the single source of truth. If one commit changes `python/sglang/srt` in the private repo, the change should be synced to the OSS repo as soon as possible with the action B below. +- The common code (e.g., base classes, well-known techniques in the industry without private secrets) goes to `python/sglang/srt`. The private-specific code (e.g., with private-specific features, confidential info) goes to `python/sglang/private` . +- Anytime you want to make private changes to a file or class under `python/sglang/srt`, duplicate the file and move it under `python/sglang/private`. You can achieve code reuse by importing and inheriting. + +## How to sync the code bidirectionally +### Action A: Copy code from OSS to private + +- We can run this action: [Open A PR to Copy Code From OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-from-oss.yml) + - It opens a PR to copy all files under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) from the OSS main branch to the private fork. + - Since the OSS repo is the single source of truth, this action copies files and overwrites any changes in the private fork. To prevent the private changes from being overwritten, you need to ensure all private changes are merged into the OSS repo before running this action. +- This action will be run automatically every day and can also be triggered manually. + +### Action B: Copy diff from private to OSS + +- We can run this action: [Open A PR to Copy Code To OSS](https://github.com/sgl-project/sglang/tree/main/.github/workflows/open-pr-copy-to-oss.yml) + - It opens a PR to apply the diff of one specific commit of the private fork to the OSS main branch. It will only pick the changes under certain folders (e.g., `python/sglang/srt` , `test/srt` , `sgl-kernel` ) and ignore changes under private folders (e.g., `python/sglang/private` ) + - For example, you can have a PR that changes both `python/sglang/srt` and `python/sglang/private/srt`. Once you merge the PR into the private repo, `python/sglang/srt` becomes desynced between the two repos. You need to run this action on your merge commit immediately to open a PR to send your diff to the OSS repo. Then, we need to merge the OSS PR as soon as possible. Once your OSS PR is merged, we can run action A again. + - Action A copies files directly, but Action B applies diff. This is because OSS is the source of truth; action A can just copy files. Action B cannot copy, so it uses diff instead. +- This action currently needs a manual trigger in order to prevent incidental code leaks. One can also consider making it automatic. diff --git a/scripts/code_sync/install_github_cli.sh b/scripts/code_sync/install_github_cli.sh new file mode 100755 index 000000000..2ef1db023 --- /dev/null +++ b/scripts/code_sync/install_github_cli.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Check if gh is installed before attempting to install it +if ! command -v gh &> /dev/null +then +echo "GitHub CLI not found. Installing now..." +(type -p wget >/dev/null || ( apt update && apt install wget -y)) \ +&& mkdir -p -m 755 /etc/apt/keyrings \ +&& out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ +&& cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ +&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ +&& mkdir -p -m 755 /etc/apt/sources.list.d \ +&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ +&& apt update \ +&& apt install gh -y +else +echo "GitHub CLI is already installed. Skipping installation." +fi diff --git a/scripts/deprecated/convert_yi_vl.py b/scripts/deprecated/convert_yi_vl.py new file mode 100644 index 000000000..cdd8aebeb --- /dev/null +++ b/scripts/deprecated/convert_yi_vl.py @@ -0,0 +1,38 @@ +""" +Convert Yi-VL config into a format usable with SGLang + +Usage: python3 scripts/convert_yi_vl.py --model-path +""" + +import argparse +import json +import os + +from transformers import AutoConfig, AutoTokenizer + + +def add_image_token(model_path: str): + tokenizer = AutoTokenizer.from_pretrained(model_path) + tokenizer.add_tokens([""], special_tokens=True) + + print(tokenizer) + tokenizer.save_pretrained(model_path) + + +def edit_model_config(model_path): + config = AutoConfig.from_pretrained(model_path) + + setattr(config, "architectures", ["YiVLForCausalLM"]) + setattr(config, "image_token_index", 64002) + + print(config) + config.save_pretrained(model_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model-path", type=str) + args = parser.parse_args() + + add_image_token(args.model_path) + edit_model_config(args.model_path) diff --git a/scripts/deprecated/convert_yi_vl.sh b/scripts/deprecated/convert_yi_vl.sh new file mode 100644 index 000000000..07a7aeba8 --- /dev/null +++ b/scripts/deprecated/convert_yi_vl.sh @@ -0,0 +1,13 @@ +# For 34B Model +mkdir ~/model_weights +cd ~/model_weights +git clone https://huggingface.co/01-ai/Yi-VL-34B +cp ~/model_weights/Yi-VL-34B/vit/clip-vit-H-14-laion2B-s32B-b79K-yi-vl-34B-448/preprocessor_config.json ~/model_weights/Yi-VL-34B +python3 convert_yi_vl.py --model-path ~/model_weights/Yi-VL-34B + +# For 6B Model +mkdir ~/model_weights +cd ~/model_weights +git clone https://huggingface.co/01-ai/Yi-VL-6B +cp ~/model_weights/Yi-VL-6B/vit/clip-vit-H-14-laion2B-s32B-b79K-yi-vl-6B-448/preprocessor_config.json ~/model_weights/Yi-VL-6B +python3 convert_yi_vl.py --model-path ~/model_weights/Yi-VL-6B diff --git a/scripts/deprecated/test_curl.sh b/scripts/deprecated/test_curl.sh new file mode 100644 index 000000000..1c83208a7 --- /dev/null +++ b/scripts/deprecated/test_curl.sh @@ -0,0 +1,9 @@ +curl http://localhost:30000/generate \ + -H "Content-Type: application/json" \ + -d '{ + "text": "Once upon a time,", + "sampling_params": { + "max_new_tokens": 64, + "temperature": 0 + } + }' diff --git a/scripts/deprecated/test_flashinfer.py b/scripts/deprecated/test_flashinfer.py new file mode 100644 index 000000000..d017d114e --- /dev/null +++ b/scripts/deprecated/test_flashinfer.py @@ -0,0 +1,217 @@ +import pytest +import torch +from flashinfer import ( + BatchDecodeWithPagedKVCacheWrapper, + BatchPrefillWithPagedKVCacheWrapper, +) + +from sglang.srt.layers.attention.triton_ops.decode_attention import decode_attention_fwd +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.utils import should_use_tensor_core + +flashinfer_prefill_wrapper = None +flashinfer_decode_wrapper = None + + +@pytest.mark.parametrize("batch_size", [12, 37, 67]) +@pytest.mark.parametrize("kv_len", [54, 97]) +@pytest.mark.parametrize("qo_len", [37, 17]) +@pytest.mark.parametrize("num_kv_heads", [4]) +@pytest.mark.parametrize("num_qo_heads", [32, 4]) +@pytest.mark.parametrize("head_dim", [128]) +def test_batch_prefill_with_paged_kv_cache( + batch_size, + kv_len, + qo_len, + num_kv_heads, + num_qo_heads, + head_dim, +): + init_flashinfer(num_qo_heads, num_kv_heads) + + q = torch.randn(batch_size * qo_len, num_qo_heads, head_dim).to(0).half() + qo_indptr = torch.arange(0, batch_size + 1).to(0).int() * qo_len + total_tokens = kv_len * batch_size + kv_data = torch.randn(total_tokens, 2, num_kv_heads, head_dim).to(0).half() + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full((batch_size,), 1, dtype=torch.int32).to(0) + + # init args for triton kernel + k_extend = ( + kv_data.view(batch_size, kv_len, 2, -1)[:, -qo_len:, 0] + .contiguous() + .view(-1, num_kv_heads, head_dim) + ) + v_extend = ( + kv_data.view(batch_size, kv_len, 2, -1)[:, -qo_len:, 1] + .contiguous() + .view(-1, num_kv_heads, head_dim) + ) + o_triton = torch.empty_like(q) + k_buffer = kv_data[:, 0].view(-1, num_kv_heads, head_dim).contiguous() + v_buffer = kv_data[:, 1].view(-1, num_kv_heads, head_dim).contiguous() + req_to_token = torch.arange(0, total_tokens).to(0).int().view(batch_size, kv_len) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32).to(0) + b_start_loc_extend = torch.arange(0, batch_size).to(0).int() * qo_len + b_seq_len_extend = torch.full((batch_size,), qo_len, dtype=torch.int32).to(0) + max_len_in_batch = kv_len + max_len_extend = qo_len + + extend_attention_fwd( + q, + k_extend, + v_extend, + o_triton, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + None, # b_start_loc = None + b_seq_len, + None, # b_seq_len_prefix = None + b_start_loc_extend, + b_seq_len_extend, + max_len_in_batch, + max_len_extend, + ) + + o_redundant = torch.empty_like(q) + b_start_loc = torch.zeros((batch_size,), dtype=torch.int32).to(0) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], dim=0) + b_seq_len_prefix = b_seq_len - b_seq_len_extend + + redundant_attention( + q, + k_extend, + v_extend, + o_redundant, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + print("Mean: ", torch.mean(torch.abs(o_redundant - o_triton))) + print("Max: ", torch.max(torch.abs(o_redundant - o_triton))) + assert torch.allclose(o_redundant, o_triton, rtol=1e-2, atol=1e-3) + + flashinfer_prefill_wrapper.end_forward() + + flashinfer_prefill_wrapper.begin_forward( + qo_indptr, + kv_indptr, + kv_indices, + kv_last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + 1, + ) + o = flashinfer_prefill_wrapper.forward( + q.contiguous().view(-1, num_qo_heads, head_dim), kv_data + ) + + print("Mean: ", torch.mean(torch.abs(o - o_triton))) + print("Max: ", torch.max(torch.abs(o - o_triton))) + assert torch.allclose(o, o_triton, rtol=1e-2, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [12, 17, 37]) +@pytest.mark.parametrize("kv_len", [54, 127, 537]) +@pytest.mark.parametrize("num_kv_heads", [32]) +@pytest.mark.parametrize("num_qo_heads", [32]) +@pytest.mark.parametrize("head_dim", [128]) +def test_batch_decode_with_paged_kv_cache( + batch_size, + kv_len, + num_kv_heads, + num_qo_heads, + head_dim, +): + # note(lsyin): when pytest, the number of heads cannot change, because triton kernel has a cache + # to test different shape of decode, change the parameters in the __main__, and run decode only once + init_flashinfer(num_qo_heads, num_kv_heads) + + q = torch.randn(batch_size, num_qo_heads, head_dim).to(0).half() + total_tokens = kv_len * batch_size + kv_data = torch.randn(total_tokens, 2, num_kv_heads, head_dim).to(0).half() + kv_indptr = torch.arange(0, batch_size + 1).to(0).int() * kv_len + kv_indices = torch.arange(0, total_tokens).to(0).int() + kv_last_page_len = torch.full((batch_size,), 1, dtype=torch.int32).to(0) + + # init args for triton kernel + k_buffer = kv_data[:, 0].view(-1, num_kv_heads, head_dim).contiguous() + v_buffer = kv_data[:, 1].view(-1, num_kv_heads, head_dim).contiguous() + o_triton = torch.empty_like(q) + req_to_token = ( + torch.arange(0, kv_len * batch_size).to(0).int().view(batch_size, kv_len) + ) + b_req_idx = torch.arange(0, batch_size).to(0).int() + b_start_loc = torch.arange(0, batch_size).to(0).int() * kv_len + b_seq_len = torch.full((batch_size,), kv_len, dtype=torch.int32).to(0) + max_len_in_batch = kv_len + other_kv_index = 0 + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o_triton, + req_to_token, + b_req_idx, + b_start_loc, + b_seq_len, + max_len_in_batch, + other_kv_index, + total_tokens, + ) + + flashinfer_decode_wrapper.end_forward() + flashinfer_decode_wrapper.begin_forward( + kv_indptr, + kv_indices, + kv_last_page_len, + num_qo_heads, + num_kv_heads, + head_dim, + 1, + pos_encoding_mode="NONE", + data_type="float16", + ) + o = flashinfer_decode_wrapper.forward( + q.contiguous().view(-1, num_qo_heads, head_dim), kv_data + ) + + print("Mean: ", torch.mean(torch.abs(o - o_triton))) + print("Max: ", torch.max(torch.abs(o - o_triton))) + assert torch.allclose(o, o_triton, rtol=1e-2, atol=2e-3) + + +def init_flashinfer(num_attention_heads, num_kv_heads): + use_tensor_cores = should_use_tensor_core( + torch.half, num_attention_heads, num_kv_heads + ) + + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8, device="cuda") + + global flashinfer_prefill_wrapper, flashinfer_decode_wrapper + + flashinfer_prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper( + workspace_buffer, "NHD" + ) + flashinfer_decode_wrapper = BatchDecodeWithPagedKVCacheWrapper( + workspace_buffer, "NHD", use_tensor_cores=use_tensor_cores + ) + + +if __name__ == "__main__": + test_batch_prefill_with_paged_kv_cache(12, 54, 37, 8, 8, 128) + test_batch_prefill_with_paged_kv_cache(37, 1111, 456, 32, 32, 128) + test_batch_decode_with_paged_kv_cache(12, 54, 4, 32, 128) diff --git a/scripts/deprecated/test_httpserver_concurrent.py b/scripts/deprecated/test_httpserver_concurrent.py new file mode 100644 index 000000000..855e51f33 --- /dev/null +++ b/scripts/deprecated/test_httpserver_concurrent.py @@ -0,0 +1,56 @@ +""" +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C. + +The capital of the United Kindom is London.\nThe capital of the United Kingdom is London.\nThe capital of +""" + +import argparse +import asyncio +import json +import time + +import aiohttp +import requests + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def main(args): + url = f"{args.host}:{args.port}" + task1 = send_request( + url + "/generate", + { + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 128}, + }, + delay=1, + ) + + task2 = send_request( + url + "/generate", + { + "text": "The capital of the United Kindom is", + "sampling_params": {"temperature": 0, "max_new_tokens": 128}, + }, + ) + + rets = await asyncio.gather(task1, task2) + print(rets) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + asyncio.run(main(args)) diff --git a/scripts/deprecated/test_httpserver_decode.py b/scripts/deprecated/test_httpserver_decode.py new file mode 100644 index 000000000..57517a15b --- /dev/null +++ b/scripts/deprecated/test_httpserver_decode.py @@ -0,0 +1,55 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 +python3 test_httpserver_decode.py + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse +import json + +import requests + + +def test_decode(url, return_logprob=False, top_logprobs_num=0, return_text=False, n=1): + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 32, + "n": n, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + print(json.dumps(response.json())) + print("=" * 100) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + test_decode(url) + test_decode(url, n=3) + + for top_logprobs_num in [0, 3]: + for return_text in [True, False]: + test_decode( + url, + return_logprob=True, + top_logprobs_num=top_logprobs_num, + return_text=return_text, + ) diff --git a/scripts/deprecated/test_httpserver_decode_stream.py b/scripts/deprecated/test_httpserver_decode_stream.py new file mode 100644 index 000000000..616eaf6c4 --- /dev/null +++ b/scripts/deprecated/test_httpserver_decode_stream.py @@ -0,0 +1,68 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 +python3 test_httpserver_decode_stream.py + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse +import json + +import requests + + +def test_decode_stream(url, return_logprob, top_logprobs_num): + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + }, + "stream": True, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": True, + "logprob_start_len": 0, + }, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + + if return_logprob: + assert data["meta_info"]["input_token_logprobs"] is not None + assert data["meta_info"]["output_token_logprobs"] is not None + for logprob, token_id, token_text in data["meta_info"][ + "output_token_logprobs" + ][prev:]: + print(f"{token_text:12s}\t{logprob}\t{token_id}", flush=True) + prev = len(data["meta_info"]["output_token_logprobs"]) + else: + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + + print("=" * 100) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + test_decode_stream(url, False, 0) + test_decode_stream(url, True, 0) + test_decode_stream(url, True, 3) diff --git a/scripts/deprecated/test_httpserver_llava.py b/scripts/deprecated/test_httpserver_llava.py new file mode 100644 index 000000000..791fc6deb --- /dev/null +++ b/scripts/deprecated/test_httpserver_llava.py @@ -0,0 +1,88 @@ +""" +Usage: +python3 -m sglang.launch_server --model-path liuhaotian/llava-v1.5-7b --tokenizer-path llava-hf/llava-1.5-7b-hf --port 30000 +python3 test_httpserver_llava.py + +Output: +The image features a man standing on the back of a yellow taxi cab, holding +""" + +import argparse +import asyncio +import json + +import aiohttp +import requests + + +async def send_request(url, data, delay=0): + await asyncio.sleep(delay) + async with aiohttp.ClientSession() as session: + async with session.post(url, json=data) as resp: + output = await resp.json() + return output + + +async def test_concurrent(args): + url = f"{args.host}:{args.port}" + + response = [] + for i in range(8): + response.append( + send_request( + url + "/generate", + { + "text": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. USER: \nDescribe this picture ASSISTANT:", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + }, + }, + ) + ) + + rets = await asyncio.gather(*response) + for ret in rets: + print(ret["text"]) + + +def test_streaming(args): + url = f"{args.host}:{args.port}" + + response = requests.post( + url + "/generate", + json={ + "text": "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions. USER: \nDescribe this picture ASSISTANT:", + "image_data": "example_image.png", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 128, + }, + "stream": True, + }, + stream=True, + ) + + prev = 0 + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + data = json.loads(chunk[5:].strip("\n")) + output = data["text"].strip() + print(output[prev:], end="", flush=True) + prev = len(output) + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + asyncio.run(test_concurrent(args)) + + test_streaming(args) diff --git a/scripts/deprecated/test_httpserver_reuse.py b/scripts/deprecated/test_httpserver_reuse.py new file mode 100644 index 000000000..ef866afc6 --- /dev/null +++ b/scripts/deprecated/test_httpserver_reuse.py @@ -0,0 +1,42 @@ +""" +python3 -m sglang.launch_server --model-path TinyLlama/TinyLlama-1.1B-Chat-v0.4 --port 30000 + +Output: +The capital of France is Paris.\nThe capital of the United States is Washington, D.C.\nThe capital of Canada is Ottawa.\nThe capital of Japan is Tokyo +""" + +import argparse + +import requests + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="http://127.0.0.1") + parser.add_argument("--port", type=int, default=30000) + args = parser.parse_args() + + url = f"{args.host}:{args.port}" + + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + print(response.json()) + + response = requests.post( + url + "/generate", + json={ + "text": "The capital of France is Paris.\nThe capital of the United States is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + print(response.json()) diff --git a/scripts/deprecated/test_jump_forward.py b/scripts/deprecated/test_jump_forward.py new file mode 100644 index 000000000..315a50b5b --- /dev/null +++ b/scripts/deprecated/test_jump_forward.py @@ -0,0 +1,138 @@ +import argparse +from enum import Enum + +from pydantic import BaseModel, constr + +import sglang as sgl +from sglang.srt.constrained.outlines_backend import build_regex_from_object +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +IP_REGEX = r"((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)" + +ip_jump_forward = ( + r"The google's DNS sever address is " + + IP_REGEX + + r" and " + + IP_REGEX + + r". " + + r"The google's website domain name is " + + r"www\.(\w)+\.(\w)+" + + r"." +) + + +# fmt: off +@sgl.function +def regex_gen(s): + s += "Q: What is the IP address of the Google DNS servers?\n" + s += "A: " + sgl.gen( + "answer", + max_tokens=128, + temperature=0, + regex=ip_jump_forward, + ) +# fmt: on + +json_jump_forward = ( + r"""The information about Hogwarts is in the following JSON format\.\n""" + + r"""\n\{\n""" + + r""" "name": "[\w\d\s]*",\n""" + + r""" "country": "[\w\d\s]*",\n""" + + r""" "latitude": [-+]?[0-9]*\.?[0-9]+,\n""" + + r""" "population": [-+]?[0-9]+,\n""" + + r""" "top 3 landmarks": \["[\w\d\s]*", "[\w\d\s]*", "[\w\d\s]*"\],\n""" + + r"""\}\n""" +) + +# fmt: off +@sgl.function +def json_gen(s): + s += sgl.gen( + "json", + max_tokens=128, + temperature=0, + regex=json_jump_forward, + ) +# fmt: on + + +class Weapon(str, Enum): + sword = "sword" + axe = "axe" + mace = "mace" + spear = "spear" + bow = "bow" + crossbow = "crossbow" + + +class Armor(str, Enum): + leather = "leather" + chainmail = "chainmail" + plate = "plate" + + +class Character(BaseModel): + name: constr(max_length=10) + age: int + armor: Armor + weapon: Weapon + strength: int + + +@sgl.function +def character_gen(s): + s += "Give me a character description who is a wizard.\n" + s += sgl.gen( + "character", + max_tokens=128, + temperature=0, + regex=build_regex_from_object(Character), + ) + + +def main(args): + # Select backend + backend = select_sglang_backend(args) + sgl.set_default_backend(backend) + + state = regex_gen.run(temperature=0) + + print("=" * 20, "IP TEST", "=" * 20) + print(state.text()) + + state = json_gen.run(temperature=0) + + print("=" * 20, "JSON TEST", "=" * 20) + print(state.text()) + + state = character_gen.run(temperature=0) + + print("=" * 20, "CHARACTER TEST", "=" * 20) + print(state.text()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + args = add_common_sglang_args_and_parse(parser) + main(args) + +# ==================== IP TEST ==================== +# Q: What is the IP address of the Google DNS servers? +# A: The google's DNS sever address is 8.8.8.8 and 8.8.4.4. The google's website domain name is www.google.com. +# ==================== JSON TEST ==================== +# The information about Hogwarts is in the following JSON format. + +# { +# "name": "Hogwarts School of Witchcraft and Wizardry", +# "country": "Scotland", +# "latitude": 55.566667, +# "population": 1000, +# "top 3 landmarks": ["Hogwarts Castle", "The Great Hall", "The Forbidden Forest"], +# } + +# ==================== CHARACTER TEST ==================== +# Give me a character description who is a wizard. +# { "name" : "Merlin", "age" : 500, "armor" : "chainmail" , "weapon" : "sword" , "strength" : 10 } diff --git a/scripts/deprecated/test_robust.py b/scripts/deprecated/test_robust.py new file mode 100644 index 000000000..633e2e649 --- /dev/null +++ b/scripts/deprecated/test_robust.py @@ -0,0 +1,132 @@ +import argparse +import random +import string + +from vllm.transformers_utils.tokenizer import get_tokenizer + +import sglang as sgl +from sglang.test.test_utils import ( + add_common_sglang_args_and_parse, + select_sglang_backend, +) + +TOKENIZER = None +RANDOM_PREFILL_LEN = None +RANDOM_DECODE_LEN = None + + +def gen_prompt(token_num): + if RANDOM_PREFILL_LEN: + token_num = random.randint(1, token_num) + + cha_set = string.ascii_letters + string.digits + ret = "".join(random.choices(cha_set, k=token_num)) + while len(TOKENIZER(ret).input_ids) < token_num: + ret += random.choice(cha_set) + + return ret + + +def robust_test_dfs(s, d, args, leaf_states): + if d == 0: + s += "END" + leaf_states.append(s) + return + + s += gen_prompt(args.len_prefill) + forks = s.fork(args.num_fork) + for fork_s in forks: + fork_s += gen_prompt(args.len_prefill) + new_tokens = ( + args.len_decode + if not RANDOM_DECODE_LEN + else random.randint(1, args.len_decode) + ) + fork_s += sgl.gen( + max_tokens=new_tokens, + ignore_eos=True, + ) + + for fork_s in forks: + robust_test_dfs(fork_s, d - 1, args, leaf_states) + + +def robust_test_bfs(s, args, leaf_states): + old_forks = [s] + new_forks = [] + for _ in range(args.depth): + for old_fork in old_forks: + old_fork += gen_prompt(args.len_prefill) + forks = old_fork.fork(args.num_fork) + for fork_s in forks: + fork_s += gen_prompt(args.len_prefill) + new_tokens = ( + args.len_decode + if not RANDOM_DECODE_LEN + else random.randint(1, args.len_decode) + ) + fork_s += sgl.gen( + max_tokens=new_tokens, + ignore_eos=True, + ) + new_forks.extend(forks) + + old_forks = new_forks + new_forks = [] + + for old_fork in old_forks: + old_fork += "END" + leaf_states.append(old_fork) + + +@sgl.function +def robust_test(s, args): + leaf_states = [] + if args.mode == "bfs": + robust_test_bfs(s, args, leaf_states) + else: + robust_test_dfs(s, args.depth, args, leaf_states) + return leaf_states + + +def main(args): + backend = select_sglang_backend(args) + + arguments = [{"args": args} for _ in range(args.num_req)] + + states = robust_test.run_batch( + arguments, temperature=0, backend=backend, num_threads=args.parallel + ) + + with open(f"tmp_robust_{args.mode}.txt", "w") as f: + for state in states: + leaf_states = state.ret_value + for leaf_state in leaf_states: + assert leaf_state.text()[-3:] == "END" + f.write(leaf_state.text()[:-3] + "\n") + + +if __name__ == "__main__": + # fmt: off + parser = argparse.ArgumentParser() + parser.add_argument("--num-req", type=int, default=2) + parser.add_argument("--depth", type=int, default=3) + parser.add_argument("--num-fork", type=int, default=2) + parser.add_argument("--len-prefill", type=int, default=128) + parser.add_argument("--len-decode", type=int, default=128) + parser.add_argument("--random-prefill-len", action="store_true") + parser.add_argument("--random-decode-len", action="store_true") + parser.add_argument("--mode", type=str, default="bfs", choices=["dfs", "bfs"]) + parser.add_argument("--tokenizer", type=str, default = "meta-llama/Llama-2-7b-chat-hf") + parser.add_argument("--trust-remote-code", action="store_true") + parser.add_argument("--seed", type=int, default=42) + args = add_common_sglang_args_and_parse(parser) + # fmt: on + + RANDOM_PREFILL_LEN = args.random_prefill_len + RANDOM_DECODE_LEN = args.random_decode_len + TOKENIZER = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code) + + random.seed(args.seed) + + main(args) diff --git a/scripts/export_deepseek_nextn.py b/scripts/export_deepseek_nextn.py new file mode 100644 index 000000000..5da0e4bc3 --- /dev/null +++ b/scripts/export_deepseek_nextn.py @@ -0,0 +1,115 @@ +""" +Export NextN layer for DeepSeek-V3/R1 model. The exported model can be used for speculative decoding. + +Usage: +python3 export_deepseek_nextn.py --input-dir /path/to/DeepSeek-V3 --output-dir /path/to/DeepSeek-V3-NextN +""" + +import argparse +import json +import os +import shutil + +from safetensors import safe_open +from safetensors.torch import save_file +from transformers import AutoConfig + + +def get_nextn_layer_id(config): + if not hasattr(config, "num_hidden_layers"): + raise ValueError("'num_hidden_layers' not found in model config.") + return config.num_hidden_layers + + +def update_and_save_config(config, output_dir): + new_config = config.to_dict() + new_config.update( + { + "num_hidden_layers": 1, + "architectures": ["DeepseekV3ForCausalLMNextN"], + } + ) + with open(os.path.join(output_dir, "config.json"), "w") as f: + json.dump(new_config, f, indent=2, ensure_ascii=False, sort_keys=True) + + +def copy_non_safetensors_files(input_dir, output_dir): + for filename in os.listdir(input_dir): + src_file_path = os.path.join(input_dir, filename) + if os.path.isfile(src_file_path) and not filename.endswith(".safetensors"): + dst_file_path = os.path.join(output_dir, filename) + shutil.copy2(src_file_path, dst_file_path) + print(f"All non-safetensors files have been copied to {output_dir}") + + +def export_nextn_layer_parameters(input_dir, output_dir, nextn_layer_id): + prefix = f"model.layers.{nextn_layer_id}" + output_path = os.path.join(output_dir, "nextn_layer_parameters.safetensors") + params = {} + for filename in os.listdir(input_dir): + if not filename.endswith(".safetensors"): + continue + + file_path = os.path.join(input_dir, filename) + print(f"Processing: {filename}") + + try: + with safe_open(file_path, framework="pt") as f: + matching_keys = [k for k in f.keys() if k.startswith(prefix)] + + if not matching_keys: + print(f" No parameters starting with '{prefix}' found") + continue + + for key in matching_keys: + if "embed_tokens" in key or "shared_head.head" in key: + continue + new_key = key.replace(prefix, "model.layers.0") + params[new_key] = f.get_tensor(key) + + except Exception as e: + print(f" Error processing {filename}: {str(e)}") + + if params: + print(f"Saving {len(params)} parameters to {output_path}") + save_file(params, output_path) + else: + print("No matching parameters found.") + + # Update safetensors index + index_path = os.path.join(output_dir, "model.safetensors.index.json") + print(f"Updating safetensors index to {index_path}") + index_data = {"weight_map": {}} + for key in params: + index_data["weight_map"][key] = "nextn_layer_parameters.safetensors" + with open(index_path, "w") as f: + json.dump(index_data, f, indent=4) + + print("All done.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Export NextN layer parameters for DeepSeek-V3/R1" + ) + parser.add_argument( + "--input-dir", + type=str, + required=True, + help="Input HF model directory.", + ) + parser.add_argument( + "--output-dir", + type=str, + required=True, + help="Output nextn model directory.", + ) + args = parser.parse_args() + + config = AutoConfig.from_pretrained(args.input_dir, trust_remote_code=True) + assert config.num_nextn_predict_layers == 1, "Only 1 nextn layer is supported." + nextn_layer_id = get_nextn_layer_id(config) + os.makedirs(args.output_dir, exist_ok=True) + copy_non_safetensors_files(args.input_dir, args.output_dir) + update_and_save_config(config, args.output_dir) + export_nextn_layer_parameters(args.input_dir, args.output_dir, nextn_layer_id) diff --git a/scripts/killall_sglang.sh b/scripts/killall_sglang.sh new file mode 100755 index 000000000..7d0fe8bca --- /dev/null +++ b/scripts/killall_sglang.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ "$1" = "rocm" ]; then + echo "Running in ROCm mode" + + # Clean SGLang processes + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' | xargs -r kill -9 + +else + # Show current GPU status + nvidia-smi + + # Clean SGLang processes + pgrep -f 'sglang::|sglang\.launch_server|sglang\.bench|sglang\.data_parallel|sglang\.srt' | xargs -r kill -9 + + # Clean all GPU processes if any argument is provided + if [ $# -gt 0 ]; then + # Check if sudo is available + if command -v sudo >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y lsof + else + apt-get update + apt-get install -y lsof + fi + kill -9 $(nvidia-smi | sed -n '/Processes:/,$p' | grep " [0-9]" | awk '{print $5}') 2>/dev/null + lsof /dev/nvidia* | awk '{print $2}' | xargs kill -9 2>/dev/null + fi + + # Show GPU status after clean up + nvidia-smi +fi diff --git a/scripts/playground/bench_speculative.py b/scripts/playground/bench_speculative.py new file mode 100644 index 000000000..c89e99242 --- /dev/null +++ b/scripts/playground/bench_speculative.py @@ -0,0 +1,308 @@ +""" +Usage: +# single GPU +python3 bench_speculative.py --model-path meta-llama/Llama-2-7b-chat-hf --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B + +# multiple GPU +python3 bench_speculative.py --model-path deepseek-ai/DeepSeek-V3 --speculative-draft-model-path lmsys/DeepSeek-V3-NextN --tp-size 8 --trust-remote-code --batch-size 1 4 8 16 32 --steps 0 1 2 --topk 0 1 2 4 --num_draft_tokens 0 2 4 8 +""" + +import argparse +import asyncio +import json +import os +import time +from types import SimpleNamespace + +import numpy as np +import requests +from transformers import AutoTokenizer + +from sglang.bench_serving import ( + DatasetRow, + benchmark, + sample_mmmu_requests, + set_global_args, +) +from sglang.srt.server_args import ServerArgs +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + kill_process_tree, + popen_launch_server, +) + + +def node0_print(msg): + if server_args.node_rank == 0: + print(msg) + + +prompts = [ + "Human: Give me a fully functional FastAPI server. Show the full, long python code without stop.\n\nAssistant:", + "Human: Imagine you are an experienced Ethereum developer tasked with creating a smart contract for a blockchain messenger. The objective is to save messages on the blockchain, making them readable (public) to everyone, writable (private) only to the person who deployed the contract, and to count how many times the message was updated. Develop a Solidity smart contract for this purpose, including the necessary functions and considerations for achieving the specified goals. Please provide the code and any relevant explanations to ensure a clear understanding of the implementation.\n\nAssistant:", + "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "Human: I want you to act as an English translator, spelling corrector and improver. I will speak to you in any language and you will detect the language, translate it and answer in the corrected and improved version of my text, in English. I want you to replace my simplified A0-level words and sentences with more beautiful and elegant, upper level English words and sentences. Keep the meaning same, but make them more literary. My first sentence is 'istanbulu cok seviyom burada olmak cok guzel'. Answer in more than 5000 words.\n\nAssistant:", + "Human: I want you to act as a storyteller. You will come up with entertaining stories that are engaging, imaginative and captivating for the audience. It can be fairy tales, educational stories or any other type of stories which has the potential to capture people's attention and imagination. Depending on the target audience, you may choose specific themes or topics for your storytelling session e.g., if it’s children then you can talk about animals; If it’s adults then history-based tales might engage them better etc. Answer in more than 5000 words. My first request is 'I need an interesting story on perseverance.'\n\nAssistant:", + "Human: Solve x^2 = -1. Think step-by-step. Give me a long detailed explanation. \n\nAssistant:", + "Human: Tell me about the president of the USA in wikipedia style.\n\nAssistant:", + "Human: Hello? Who are you? Write code, math, and poem to explanin yourself.\n\nAssistant:", +] + + +class FakeTokenizer: + def encode(self, text: str, add_special_tokens: bool = False): + return [] + + +def send_one_batch(base_url, num_prompts, batch_size, tokenizer, is_multimodal): + # format: (prompt, input_len, output len). We set input_len as a dummy value 0. + if is_multimodal: + input_requests = sample_mmmu_requests( + num_prompts, + tokenizer, + 512, + apply_chat_template=False, + ) + backend = "sglang-oai-chat" + api_url = f"{base_url}/v1/chat/completions" + else: + padded_prompts = (prompts * ((num_prompts + len(prompts) - 1) // len(prompts)))[ + :num_prompts + ] + input_requests: List[DatasetRow] = [ + DatasetRow(p, 0, 512) for p in padded_prompts + ] + backend = "sglang" + api_url = f"{base_url}/generate" + + # We need to set some dummy values in order to call `benchmark` below. + args = SimpleNamespace( + disable_ignore_eos=False, + disable_stream=False, + return_logprob=False, + backend=backend, + dataset_name="custom", + num_prompts=None, + sharegpt_output_len=None, + random_input_len=None, + random_output_len=None, + random_range_ratio=None, + output_file=None, + warmup_requests=1, + output_details=False, + ) + set_global_args(args) + + # Run benchmark + results = asyncio.run( + benchmark( + backend=backend, + api_url=api_url, + base_url=base_url, + model_id="default", + tokenizer=tokenizer, + input_requests=input_requests, + request_rate=float("inf"), + max_concurrency=batch_size, + disable_tqdm=False, + lora_names=None, + extra_request_body={}, + profile=None, + ) + ) + + assert results["completed"] == len(input_requests) + acc_length = results["accept_length"] or 1.0 + avg_output_token = results["total_output_tokens"] / results["completed"] + + server_info = requests.get(base_url + "/get_server_info").json() + # We use 20% percentile instead of median on purpose + step_time = np.percentile( + server_info["internal_states"][0]["step_time_dict"][str(batch_size)], 20 + ) + speed = 1 / step_time * acc_length + + return ( + round(acc_length, 3), + round(step_time, 5), + round(speed, 3), + avg_output_token, + ) + + +def main(args, server_args): + base_url = "http://127.0.0.1:20000" + + configs = [] + for batch_size in args.batch_size: + for steps in args.steps: + for topk in args.topk: + for num_draft_tokens in args.num_draft_tokens: + if steps * topk + 1 < num_draft_tokens: + continue + + if (steps == 0 or topk == 0 or num_draft_tokens == 0) and ( + steps + topk + num_draft_tokens != 0 + ): + # steps == 0 and topk == 0 and num_draft_tokens == 0 is a special case for non-speculative decoding. + continue + + configs.append((batch_size, steps, topk, num_draft_tokens)) + + for i in range(args.start, args.end or len(configs)): + batch_size, steps, topk, num_draft_tokens = configs[i] + + node0_print( + f"Start {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}" + ) + + # Create an LLM. + if steps == 0: + other_args = [] + else: + other_args = [ + "--speculative-num-steps", + steps, + "--speculative-eagle-topk", + topk, + "--speculative-num-draft-tokens", + num_draft_tokens, + ] + if server_args.speculative_draft_model_path is not None: + other_args.extend( + [ + "--speculative-draft-model-path", + server_args.speculative_draft_model_path, + "--speculative-algorithm", + server_args.speculative_algorithm, + ] + ) + + other_args.extend( + [ + "--cuda-graph-max-bs", + batch_size, + "--mem-fraction-static", + server_args.mem_fraction_static, + "--tp-size", + server_args.tp_size, + "--max-running-requests", + batch_size, + ] + ) + + if server_args.trust_remote_code: + other_args.extend( + [ + "--trust-remote-code", + ] + ) + + if server_args.attention_backend: + other_args.extend( + [ + "--attention-backend", + server_args.attention_backend, + ] + ) + + if server_args.quantization: + other_args.extend( + [ + "--quantization", + server_args.quantization, + ] + ) + + process = popen_launch_server( + args.model_path, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + env={ + "SGLANG_RECORD_STEP_TIME": "1", + **os.environ, + }, + ) + + tokenizer = AutoTokenizer.from_pretrained( + args.model_path, trust_remote_code=server_args.trust_remote_code + ) + + try: + # Warmup + send_one_batch( + base_url, batch_size, batch_size, tokenizer, args.is_multimodal + ) + + # Benchmark + acc_length, step_time, speed, completion_tokens = send_one_batch( + base_url, + max(args.num_prompts, batch_size), + batch_size, + tokenizer, + args.is_multimodal, + ) + finally: + kill_process_tree(process.pid) + + node0_print( + f"Finish {i=}: {batch_size=}, {steps=}, {topk=}, {num_draft_tokens=}, {speed=:.2f} token/s, step_time={step_time * 1000:.2f} ms" + ) + + record = { + "batch_size": batch_size, + "steps": steps, + "topk": topk, + "num_draft_tokens": num_draft_tokens, + "acc_length": acc_length, + "step_time": step_time, + "speed": speed, + "completion_tokens": completion_tokens, + } + + with open(args.output, "a") as fout: + fout.write(json.dumps(record) + "\n") + + # Wait for the server to shutdown + time.sleep(5) + + +# The __main__ condition is necessary here because we use "spawn" to create subprocesses +# Spawn starts a fresh program every time, if there is no __main__, it will run into infinite loop to keep spawning processes from sgl.Engine +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ServerArgs.add_cli_args(parser) + parser.add_argument( + "--batch-size", + type=int, + nargs="+", + default=(1, 2, 4, 8, 16), + ) + parser.add_argument( + "--steps", + type=int, + nargs="+", + default=(0, 1, 3, 5, 7), # use (0, 1, 2, 3, 4) for large batch size + ) + parser.add_argument( + "--topk", + type=int, + nargs="+", + default=(0, 1, 2, 4, 8), + ) + parser.add_argument( + "--num_draft_tokens", + type=int, + nargs="+", + default=(0, 2, 4, 8, 16, 32), # use (0, 2, 4, 8) for large batch size + ) + parser.add_argument("--num-prompts", type=int, default=16) + parser.add_argument("--start", type=int, default=0) + parser.add_argument("--end", type=int) + parser.add_argument("--output", type=str, default="output.jsonl") + parser.add_argument("--is-multimodal", action="store_true", default=False) + args = parser.parse_args() + server_args: ServerArgs = ServerArgs.from_cli_args(args) + + main(args, server_args) diff --git a/scripts/playground/disaggregation/cli-logprob.py b/scripts/playground/disaggregation/cli-logprob.py new file mode 100644 index 000000000..4c69a055b --- /dev/null +++ b/scripts/playground/disaggregation/cli-logprob.py @@ -0,0 +1,22 @@ +prompt = "The capital of france is " + +import json + +import requests + +response = requests.post( + "http://0.0.0.0:8000/generate", + json={ + "text": prompt, + "sampling_params": {"temperature": 0}, + "return_logprob": True, + "return_input_logprob": True, + "logprob_start_len": 0, + }, +) + +j = response.json() +input_logprobs = j["meta_info"]["input_token_logprobs"] +output_logprobs = j["meta_info"]["output_token_logprobs"] + +print(len(input_logprobs), len(output_logprobs)) diff --git a/scripts/playground/disaggregation/cli-so.py b/scripts/playground/disaggregation/cli-so.py new file mode 100644 index 000000000..7ccafc7ed --- /dev/null +++ b/scripts/playground/disaggregation/cli-so.py @@ -0,0 +1,34 @@ +import json + +import requests + +port = 8000 + +json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + } +) + +# JSON +response = requests.post( + f"http://localhost:{port}/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, +) + +print(response.json()) + + +# python3 -m sglang.launch_server --model-path meta-llama/Llama-2-7b-chat-hf --trust-remote-code --disaggregation-mode prefill --tp 2 --disaggregation-ib-device mlx5_roce0,mlx5_roce1 --speculative-algorithm EAGLE --speculative-draft-model-path lmsys/sglang-EAGLE-llama2-chat-7B --speculative-num-steps 3 --speculative-eagle-topk 4 --speculative-num-draft-tokens 16 --cuda-graph-max-bs 8 --host 127.0.0.1 --port 8100 diff --git a/scripts/playground/disaggregation/cli.py b/scripts/playground/disaggregation/cli.py new file mode 100644 index 000000000..721a6dd5e --- /dev/null +++ b/scripts/playground/disaggregation/cli.py @@ -0,0 +1,29 @@ +import json + +import requests + +prompt = """ +According to CNBC's Faber, the investors present on the call interpreted this statement as an indication of an upcoming funding round. While speculative, Faber believes the funding round could be as large as $25 billion, and bestow a valuation of between $150 billion and $200 billion on xAI. + +For the benefit of those who might not be aware, xAI recently acquired the social media platform X in an all-stock deal that valued the former at $80 billion and the latter at $33 billion, inclusive of $12 billion in liabilities. This meant that the deal bestowed a gross valuation of $45 billion on X before factoring in its debt load of $12 billion. + +Bear in mind that Elon Musk took X (then called Twitter) private back in 2022 in a $44 billion deal. Since then, Musk has managed to stem X's cash bleed, with the company reportedly generating $1.2 billion in adjusted EBITDA in 2024. + +According to the investors present on the call, xAI is currently generating around $1 billion in annual revenue. This contrasts sharply with the erstwhile muted expectations of many investors, who did not expect the startup to generate any material revenue this year. + +Elsewhere, Faber also alludes to the fact that xAI is already working on its next big training supercluster, officially dubbed the Colossus 2, which is expected to eventually house as many as 1 million NVIDIA GPUs at a cost of between $35 billion and $40 billion. + + +Even though xAI's Grok LLM is already largely comparable with OpenAI's cutting-edge models, the Colossus 2 would significantly up the ante, and could feasibly challenge OpenAI's apex position in the AI sphere. + +Give your honest take on the above text: +""" + +response = requests.post( + "http://0.0.0.0:8000/generate", + json={"text": prompt, "sampling_params": {"temperature": 0}}, +) + + +response_json = response.json() +print(response_json["text"]) diff --git a/scripts/playground/frontend_reasoning.ipynb b/scripts/playground/frontend_reasoning.ipynb new file mode 100644 index 000000000..fcdce25ab --- /dev/null +++ b/scripts/playground/frontend_reasoning.ipynb @@ -0,0 +1,241 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Launch A Server\n", + "\n", + "Launch the server with a reasoning model (Qwen 3.5-4B) and reasoning parser." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sglang import separate_reasoning, assistant_begin, assistant_end\n", + "from sglang import assistant, function, gen, system, user\n", + "from sglang import image\n", + "from sglang import RuntimeEndpoint, set_default_backend\n", + "from sglang.srt.utils import load_image\n", + "from sglang.test.test_utils import is_in_ci\n", + "from sglang.utils import print_highlight, terminate_process, wait_for_server\n", + "\n", + "\n", + "if is_in_ci():\n", + " from patch import launch_server_cmd\n", + "else:\n", + " from sglang.utils import launch_server_cmd\n", + "\n", + "\n", + "server_process, port = launch_server_cmd(\n", + " \"python3 -m sglang.launch_server --model-path Qwen/Qwen3-4B --reasoning-parser qwen3 --host 0.0.0.0\"\n", + ")\n", + "\n", + "wait_for_server(f\"http://localhost:{port}\")\n", + "print(f\"Server started on http://localhost:{port}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the default backend. Note: you can set chat_template_name in RontimeEndpoint. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "set_default_backend(\n", + " RuntimeEndpoint(f\"http://localhost:{port}\", chat_template_name=\"qwen\")\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start with a basic question-answering task. And see how the reasoning content is generated." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant_begin()\n", + " s += gen(\"answer\", max_tokens=512)\n", + " s += assistant_end()\n", + "\n", + "\n", + "state = basic_qa(\"List 3 countries and their capitals.\")\n", + "print_highlight(state[\"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With `separate_reasoning`, you can move the reasoning content to `{param_name}_reasoning_content` in the state." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def basic_qa_separate_reasoning(s, question):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(question)\n", + " s += assistant_begin()\n", + " s += separate_reasoning(gen(\"answer\", max_tokens=512), model_type=\"qwen3\")\n", + " s += assistant_end()\n", + "\n", + "\n", + "reasoning_state = basic_qa_separate_reasoning(\"List 3 countries and their capitals.\")\n", + "print_highlight(reasoning_state.stream_executor.variable_event.keys())\n", + "print_highlight(\n", + " f\"\\nSeparated Reasoning Content:\\n{reasoning_state['answer_reasoning_content']}\"\n", + ")\n", + "\n", + "print_highlight(f\"\\n\\nContent:\\n{reasoning_state['answer']}\")\n", + "print_highlight(f\"\\n\\nMessages:\\n{reasoning_state.messages()[-1]}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`separate_reasoning` can also be used in multi-turn conversations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def multi_turn_qa(s):\n", + " s += system(f\"You are a helpful assistant than can answer questions.\")\n", + " s += user(\"Please give me a list of 3 countries and their capitals.\")\n", + " s += assistant(\n", + " separate_reasoning(gen(\"first_answer\", max_tokens=512), model_type=\"qwen3\")\n", + " )\n", + " s += user(\"Please give me another list of 3 countries and their capitals.\")\n", + " s += assistant(\n", + " separate_reasoning(gen(\"second_answer\", max_tokens=512), model_type=\"qwen3\")\n", + " )\n", + " return s\n", + "\n", + "\n", + "reasoning_state = multi_turn_qa()\n", + "print_highlight(f\"\\n\\nfirst_answer:\\n{reasoning_state['first_answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nfirst_answer_reasoning_content:\\n{reasoning_state['first_answer_reasoning_content']}\"\n", + ")\n", + "print_highlight(f\"\\n\\nsecond_answer:\\n{reasoning_state['second_answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nsecond_answer_reasoning_content:\\n{reasoning_state['second_answer_reasoning_content']}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using No thinking as Qwen 3's advanced feature \n", + "\n", + "sglang separate_reasoning is particularly useful when combined with Qwen 3's advanced feature.\n", + "\n", + "[Qwen 3's advanced usages](https://qwenlm.github.io/blog/qwen3/#advanced-usages)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "reasoning_state = basic_qa_separate_reasoning(\n", + " \"List 3 countries and their capitals. /no_think\"\n", + ")\n", + "print_highlight(f\"Reasoning Content:\\n{reasoning_state['answer_reasoning_content']}\")\n", + "print_highlight(f\"Content:\\n{reasoning_state['answer']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`separate_reasoning` can also be used in regular expression generation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@function\n", + "def regular_expression_gen(s):\n", + " s += user(\n", + " \"What is the IP address of the Google DNS servers? just provide the answer\"\n", + " )\n", + " s += assistant(\n", + " separate_reasoning(\n", + " gen(\n", + " \"answer\",\n", + " temperature=0,\n", + " regex=r\"((25[0-5]|2[0-4]\\d|[01]?\\d\\d?).){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\",\n", + " max_tokens=512,\n", + " ),\n", + " model_type=\"qwen3\",\n", + " ),\n", + " )\n", + "\n", + "\n", + "reasoning_state = regular_expression_gen()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print_highlight(f\"Answer:\\n{reasoning_state['answer']}\")\n", + "print_highlight(\n", + " f\"\\n\\nReasoning Content:\\n{reasoning_state['answer_reasoning_content']}\"\n", + ")" + ] + } + ], + "metadata": { + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scripts/playground/launch_tgi.sh b/scripts/playground/launch_tgi.sh new file mode 100644 index 000000000..a32405cdd --- /dev/null +++ b/scripts/playground/launch_tgi.sh @@ -0,0 +1,7 @@ +# Assuming the model is downdloaded at /home/ubuntu/model_weights/Llama-2-7b-chat-hf +docker run --name tgi --rm -ti --gpus all --network host \ + -v /home/ubuntu/model_weights/Llama-2-7b-chat-hf:/Llama-2-7b-chat-hf \ + ghcr.io/huggingface/text-generation-inference:1.1.0 \ + --model-id /Llama-2-7b-chat-hf --num-shard 1 --trust-remote-code \ + --max-input-length 2048 --max-total-tokens 4096 \ + --port 24000 diff --git a/scripts/playground/load_tokenizer.py b/scripts/playground/load_tokenizer.py new file mode 100644 index 000000000..94cf34bc7 --- /dev/null +++ b/scripts/playground/load_tokenizer.py @@ -0,0 +1,14 @@ +import argparse +import code + +from sglang.srt.hf_transformers_utils import get_tokenizer + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--name", type=str, default="meta-llama/Meta-Llama-3-8B-Instruct" + ) + args = parser.parse_args() + + t = get_tokenizer(args.name) + code.interact(local=locals()) diff --git a/scripts/playground/long_context_example.py b/scripts/playground/long_context_example.py new file mode 100644 index 000000000..c5e035d29 --- /dev/null +++ b/scripts/playground/long_context_example.py @@ -0,0 +1,36 @@ +from urllib.request import urlopen + +from openai import OpenAI + +test_cases = { + "64k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/64k.txt", + "200k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/200k.txt", + "600k": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/600k.txt", + "1m": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-1M/test-data/1m.txt", +} + +client = OpenAI(api_key="EMPTY", base_url="http://127.0.0.1:30000/v1") + +for name, url in test_cases.items(): + print(f"\n==== Running test case: {name} ====") + try: + with urlopen(url, timeout=10) as response: + prompt = response.read().decode("utf-8") + except Exception as e: + print(f"Failed to load prompt for {name}: {e}") + continue + + try: + response = client.chat.completions.create( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + messages=[{"role": "user", "content": prompt}], + stream=True, + max_tokens=128, + temperature=0, + ) + + for chunk in response: + if chunk.choices and chunk.choices[0].delta.content is not None: + print(chunk.choices[0].delta.content, end="", flush=True) + except Exception as e: + print(f"\nError during completion for {name}: {e}") diff --git a/scripts/playground/lora/analyzer.py b/scripts/playground/lora/analyzer.py new file mode 100644 index 000000000..15568fc18 --- /dev/null +++ b/scripts/playground/lora/analyzer.py @@ -0,0 +1,77 @@ +import glob +import json +import os +import re +import sys + +from tqdm import tqdm + +sys.path.append("../../") +from fix_corrupted_json import clean_json_file + +dirpath = "/Users/ying" +output_file_prefix = "analyzed_log" + +time = {} +tot_time = {} +size = {} + +os.system(f"rm {output_file_prefix}*") + +for dirname in glob.glob(os.path.join(dirpath, "trace*")): + print(dirname) + trace_name = dirname.split("/")[-1] + time[trace_name] = {} + size[trace_name] = {} + total_time = 0 + for filename in tqdm(glob.glob(os.path.join(dirname, "*.json"))): + step_name = filename.split("/")[-1].split(".")[0] + step_name = "_".join(step_name.split("_")[1:]) + if "prefill" not in filename and "decode" not in filename: + continue + + match = re.search(r"(prefill|decode)_step_(\d+)\.json", filename) + if match: + phase = match.group(1) + step = match.group(2) + else: + raise Exception(f"Cannot parse {filename}") + + try: + with open(filename, "r") as f: + trace = json.load(f) + except: + clean_json_file(filename, filename) + with open(filename, "r") as f: + trace = json.load(f) + + for event in trace["traceEvents"]: + name = event["name"] + if name in ["profile_prefill_step", "profile_decode_step"]: + dur = event["dur"] / 1e3 + time[trace_name][step_name] = dur + break + total_time += dur + + step = int(step_name.split("_")[-1]) + with open(os.path.join(dirname, f"size_{step}.json"), "r") as f: + size_info = json.load(f) + size[trace_name][step_name] = size_info["size"] + + tot_time[trace_name] = total_time + time[trace_name] = dict( + sorted(time[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + size[trace_name] = dict( + sorted(size[trace_name].items(), key=lambda x: int(x[0].split("_")[-1])) + ) + + with open(f"{output_file_prefix}_{trace_name}", "a") as f: + for k, v in time[trace_name].items(): + size_v = size[trace_name][k] + print(f"{k:>15}{v:10.2f}\t{size_v}") + f.write(f"{k:>15}{v:10.2f}\t{size_v}\n") + +with open(f"{output_file_prefix}_total_time", "w") as f: + print(tot_time) + json.dump(tot_time, f) diff --git a/scripts/playground/lora/lora_hf_play.py b/scripts/playground/lora/lora_hf_play.py new file mode 100644 index 000000000..0abddd2c1 --- /dev/null +++ b/scripts/playground/lora/lora_hf_play.py @@ -0,0 +1,62 @@ +import torch +from peft import PeftModel +from transformers import LlamaForCausalLM, LlamaTokenizer + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +# ADAPTER = "winddude/wizardLM-LlaMA-LoRA-7B" +ADAPTER = "/home/ying/test_lora" +HF_TOKEN = "..." + + +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +tokenizer = LlamaTokenizer.from_pretrained(MODEL) + +base_model = LlamaForCausalLM.from_pretrained( + MODEL, + device_map="auto", + # load_in_8bit=True, + torch_dtype=torch.float16, + # use_auth_token=HF_TOKEN, +).cuda() + + +# base model generate +with torch.no_grad(): + output_tensors = base_model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= base output ========") +print(output) + + +# peft model generate +model = PeftModel.from_pretrained( + base_model, + ADAPTER, + torch_dtype=torch.float16, + is_trainable=False, +) + +with torch.no_grad(): + output_tensors = model.generate( + input_ids=tokenizer(prompt, return_tensors="pt").input_ids.cuda(), + max_new_tokens=32, + do_sample=False, + )[0] + +output = tokenizer.decode(output_tensors, skip_special_tokens=True) +print("======= peft output ========") +print(output) diff --git a/scripts/playground/lora/lora_vllm_play.py b/scripts/playground/lora/lora_vllm_play.py new file mode 100644 index 000000000..4f77d8bea --- /dev/null +++ b/scripts/playground/lora/lora_vllm_play.py @@ -0,0 +1,30 @@ +from vllm import LLM, SamplingParams +from vllm.lora.request import LoRARequest + +MODEL = "mistralai/Mistral-7B-Instruct-v0.3" +ADAPTER = "/home/ying/test_lora" +prompt = """ +### Instruction: +Write a poem about the transformers Python library. +Mention the word "large language models" in that poem. +### Response: +The Transformers are large language models, +They're used to make predictions on text. +""" + + +llm = LLM(model=MODEL, enable_lora=True) + +sampling_params = SamplingParams( + temperature=0, + max_tokens=32, +) + +prompts = [prompt] + +outputs = llm.generate( + prompts, sampling_params, lora_request=LoRARequest("test_lora", 1, ADAPTER) +) + +print(outputs[0].prompt) +print(outputs[0].outputs[0].text) diff --git a/scripts/playground/reference_hf.py b/scripts/playground/reference_hf.py new file mode 100644 index 000000000..14d23fb76 --- /dev/null +++ b/scripts/playground/reference_hf.py @@ -0,0 +1,197 @@ +""" +Usage: python3 scripts/playground/reference_hf.py --model-path MODEL_PATH --model-type {text,vlm} [--max-new-tokens NUM] [--dtype DTYPE] + --model-path MODEL_PATH: Path to model (default: TinyLlama/TinyLlama-1.1B-Chat-v0.4) + --model-type {text,vlm}: Model type, text or vlm (default: text) + --max-new-tokens NUM: Max new tokens to generate (default: 16) + --dtype DTYPE: Data type for computation (default: float16) +Note: '--model' is deprecated; use '--model-path'. Runs normal_text() for text, vlm_text_with_image() for vlm. + +Reference output: +========== Prompt 0 ========== +prefill logits (final) tensor([-8.3125, -7.1172, 3.3398, ..., -4.9531, -4.1328, -3.4141], + device='cuda:0') + The capital of France is Paris. +The capital of the United States is Washington, D.C. + +========== Prompt 1 ========== +prefill logits (final) tensor([-8.9062, -9.0156, 4.1484, ..., -4.9922, -4.4961, -4.0742], + device='cuda:0') + The capital of the United Kindom is London. +The capital of the United Kingdom is London. +The capital of + +========== Prompt 2 ========== +prefill logits (final) tensor([-9.6328, -9.0547, 4.0234, ..., -5.3047, -4.7148, -4.4609], + device='cuda:0') + Today is a sunny day and I like to go for a walk in the park. +I'm going to the +""" + +import argparse + +import requests +import torch +from PIL import Image +from transformers import ( + AutoModelForCausalLM, + AutoModelForImageTextToText, + AutoProcessor, +) + +from sglang.srt.hf_transformers_utils import get_tokenizer + + +@torch.no_grad() +def vlm_text_with_image(args): + # Load the processor and model for ImageTextToText tasks + processor = AutoProcessor.from_pretrained(args.model_path, trust_remote_code=True) + model = AutoModelForImageTextToText.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + torch.cuda.set_device(0) + + # List of image URLs to process + image_urls = [ + "https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/images/llava_v1_5_radar.jpg?raw=true" + ] + + # Conversation template for the processor + conversation = [ + { + "role": "user", + "content": [ + { + "type": "image", + }, + {"type": "text", "text": "Describe this image."}, + ], + } + ] + + max_new_tokens = args.max_new_tokens + + for i, url in enumerate(image_urls): + # Load the image from the URL + image = Image.open(requests.get(url, stream=True).raw) + + # Apply the chat template to the text prompt + # Notice that not all processors support chat templates. + # LLaVA and QWen are two processors that support chat templates. + if not hasattr(processor, "apply_chat_template"): + raise ValueError("The processor does not support chat templates.") + text_prompt = processor.apply_chat_template( + conversation, add_generation_prompt=True + ) + + # Prepare inputs for the model + inputs = processor(text=[text_prompt], images=[image], return_tensors="pt").to( + "cuda:0" + ) + + # Generate output from the model + output_ids = model.generate( + **inputs, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = processor.decode(output_ids[0]) + + # Get the logits from the model's forward pass + outputs = model.forward(**inputs) + logits = outputs.logits[0, -1, :] + + print(f"\n========== Image {i} ==========") + print("prefill logits (final)", logits) + # TODO(gaocegege): The output contains numerous <|image_pad|> tokens, + # making it cluttered and difficult to read. + # These tokens should be removed or cleaned up for better readability. + print(output_str) + + +@torch.no_grad() +def normal_text(args): + t = get_tokenizer(args.model_path, trust_remote_code=True) + m = AutoModelForCausalLM.from_pretrained( + args.model_path, + torch_dtype=args.dtype, + low_cpu_mem_usage=True, + device_map="auto", + trust_remote_code=True, + ) + + prompts = [ + "The capital of France is", + "The capital of the United Kindom is", + "Today is a sunny day and I like", + ] + max_new_tokens = args.max_new_tokens + + torch.cuda.set_device(0) + + for i, p in enumerate(prompts): + if isinstance(p, str): + input_ids = t.encode(p, return_tensors="pt").to("cuda:0") + else: + input_ids = torch.tensor([p], device="cuda:0") + + output_ids = m.generate( + input_ids, do_sample=False, max_new_tokens=max_new_tokens + ) + output_str = t.decode(output_ids[0]) + + prefill_logits = m.forward(input_ids).logits[0][-1] + + print(f"\n========== Prompt {i} ==========") + print("prefill logits (final)", prefill_logits) + print(output_str) + + +@torch.no_grad() +def synthetic_tokens(args): + m = AutoModelForCausalLM.from_pretrained( + args.model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True + ) + m.cuda() + print(m) + + input_len = 256 + output_len = 8 + prompts = [list(range(5, 5 + input_len))] + + for p in prompts: + input_ids = p + for i in range(output_len + 1): + prefill_logits = m.forward(torch.tensor([input_ids], device="cuda")).logits[ + 0 + ][-1] + + if i == 0: + print("prefill logits", prefill_logits) + else: + print("decode", i - 1, prefill_logits) + + input_ids.append(torch.argmax(prefill_logits).item()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--model-path", + type=str, + default="TinyLlama/TinyLlama-1.1B-Chat-v0.4", + ) + parser.add_argument("--max-new-tokens", type=int, default=16) + + parser.add_argument("--dtype", type=str, default="float16") + + parser.add_argument("--model-type", type=str, default="text") + + args = parser.parse_args() + + if args.model_type == "vlm": + vlm_text_with_image(args) + else: + normal_text(args) diff --git a/scripts/playground/replay_request_dump.py b/scripts/playground/replay_request_dump.py new file mode 100644 index 000000000..301cf948e --- /dev/null +++ b/scripts/playground/replay_request_dump.py @@ -0,0 +1,151 @@ +""" +Usage: +# replay from a folder +python3 replay_request_dump.py --file-number 100 --parallel 512 --input-folder /data/lianmin/sglang_request_dump/grok-mini-0220-engine-5756f8f94-28bm6/ + +# replay from a single file +python3 replay_request_dump.py --parallel 512 --input-file /data/sglang_crash_dump/memx-cti-34-sr1.xpop.twttr.net/crash_dump_2025-06-04_20-13-18.pkl +""" + +import argparse +import glob +import json +import pickle +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from datetime import datetime + +import requests + +from sglang.bench_serving import set_ulimit +from sglang.utils import get_exception_traceback + + +def read_records(files): + records = [] + for f in files: + tmp = pickle.load(open(f, "rb")) + if isinstance(tmp, dict) and "requests" in tmp: + records.extend(tmp["requests"]) + else: + records.extend(tmp) + + return records + + +def run_one_request_internal(record): + (req, output, replay_init_time, start_time, end_time, idx) = record + time.sleep(max(0, (start_time - (time.time() - replay_init_time)) / args.speed)) + + if "completion_tokens" in output.get("meta_info", {}): + recorded_completion_tokens = output["meta_info"]["completion_tokens"] + else: + recorded_completion_tokens = "" + + json_data = asdict(req) + stream = json_data["stream"] + + if args.ignore_eos: + json_data["sampling_params"]["ignore_eos"] = True + if recorded_completion_tokens: + json_data["sampling_params"]["max_new_tokens"] = recorded_completion_tokens + + response = requests.post( + f"http://{args.host}:{args.port}/generate", + json=json_data, + stream=stream, + ) + + if stream: + for chunk in response.iter_lines(decode_unicode=False): + chunk = chunk.decode("utf-8") + if chunk and chunk.startswith("data:"): + if chunk == "data: [DONE]": + break + ret = json.loads(chunk[5:].strip("\n")) + else: + ret = response.json() + + prompt_tokens = ret["meta_info"]["prompt_tokens"] + completion_tokens = ret["meta_info"]["completion_tokens"] + print( + f"{idx=}, {start_time=:.2f}, {prompt_tokens=}, " + f"{completion_tokens=}, {recorded_completion_tokens=}" + ) + + +def run_one_request(record): + # global success_ct, error_ct + + try: + run_one_request_internal(record) + # success_ct += 1 + except Exception: + # error_ct += 1 + traceback = get_exception_traceback() + print(f"Hit an exception: {traceback}") + + +def main(records): + if len(records) == 0: + return + + base_time = records[0][-2] + base_time_str = datetime.fromtimestamp(base_time).strftime("%y-%m-%d %H:%M:%S") + print(f"{base_time_str=}") + replay_init_time = time.time() + + for i in range(len(records)): + req, output, start_time, end_time = records[i] + start_time -= base_time + records[i] = (req, output, replay_init_time, start_time, end_time, i) + + with ThreadPoolExecutor(args.parallel) as executor: + executor.map(run_one_request, records) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=30000) + parser.add_argument( + "--input-folder", type=str, default=None, help="Folder containing pickle files" + ) + parser.add_argument( + "--input-file", type=str, default=None, help="Single pickle file to process" + ) + parser.add_argument("--file-number", type=int, default=1) + parser.add_argument("--req-number", type=int, default=1000000) + parser.add_argument("--req-start", type=int, default=0) + parser.add_argument("--parallel", type=int, default=512) + parser.add_argument("--idx", type=int, default=None) + parser.add_argument("--ignore-eos", action="store_true") + parser.add_argument("--speed", type=float, default=1) + args = parser.parse_args() + + set_ulimit() + + files = [] + if args.input_file: + files = [args.input_file] + if args.file_number > 1: + print("Warning: --file-number is ignored when --input-file is provided.") + elif args.input_folder: + files = glob.glob(f"{args.input_folder}/*.pkl") + files = files[: args.file_number] + else: + print("Error: Either --input-folder or --input-file must be provided.") + exit(1) + print(f"{files=}") + + records = read_records(files) + # Sort by the receive time, before filtering + records.sort(key=lambda x: x[-2]) + records = records[args.req_start :] + if args.idx: + records = [records[args.idx]] + print(f"testing {args.idx=}") + print(f"{records[0]}") + print(f"{len(records)=}") + main(records) diff --git a/scripts/playground/router/test_tree.py b/scripts/playground/router/test_tree.py new file mode 100644 index 000000000..af41c738e --- /dev/null +++ b/scripts/playground/router/test_tree.py @@ -0,0 +1,207 @@ +import random +import string +import time +import unittest +from typing import Dict, List, Tuple + +from tree import MultiTenantRadixTree + + +class TestMultiTenantRadixTree(unittest.TestCase): + def setUp(self): + self.tree = MultiTenantRadixTree() + + def test_insert_exact_match(self): + """Test 1: Basic insert and exact match operations""" + # Insert a single string for one tenant + self.tree.insert("hello", "tenant1") + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + # Insert same string for different tenant + self.tree.insert("hello", "tenant2") + matched, tenant = self.tree.prefix_match("hello") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Insert different string for same tenant + self.tree.insert("world", "tenant1") + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant1") + + print(self.tree.pretty_print()) + + def test_insert_partial_match(self): + """Test 2: Insert with partial matching scenarios""" + # Test partial matches with common prefixes + self.tree.insert("hello", "tenant1") + print(self.tree.pretty_print()) + self.tree.insert("help", "tenant2") + print(self.tree.pretty_print()) + + # Match exact strings + matched, tenant = self.tree.prefix_match("hello") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + matched, tenant = self.tree.prefix_match("help") + self.assertEqual(matched, "help") + self.assertEqual(tenant, "tenant2") + + # Match partial string + matched, tenant = self.tree.prefix_match("hel") + self.assertEqual(matched, "hel") + self.assertIn(tenant, ["tenant1", "tenant2"]) + + # Match longer string + matched, tenant = self.tree.prefix_match("hello_world") + self.assertEqual(matched, "hello") + self.assertEqual(tenant, "tenant1") + + def test_insert_edge_cases(self): + """Test 3: Edge cases for insert and match operations""" + # Empty string + self.tree.insert("", "tenant1") + matched, tenant = self.tree.prefix_match("") + self.assertEqual(matched, "") + self.assertEqual(tenant, "tenant1") + + # Single character + self.tree.insert("a", "tenant1") + matched, tenant = self.tree.prefix_match("a") + self.assertEqual(matched, "a") + self.assertEqual(tenant, "tenant1") + + # Very long string + long_str = "a" * 1000 + self.tree.insert(long_str, "tenant1") + matched, tenant = self.tree.prefix_match(long_str) + self.assertEqual(matched, long_str) + self.assertEqual(tenant, "tenant1") + + # Unicode characters + self.tree.insert("你好", "tenant1") + matched, tenant = self.tree.prefix_match("你好") + self.assertEqual(matched, "你好") + self.assertEqual(tenant, "tenant1") + + def test_simple_eviction(self): + """Test 4: Simple eviction scenarios + Tenant1: limit 10 chars + Tenant2: limit 5 chars + + Should demonstrate: + 1. Basic eviction when size limit exceeded + 2. Proper eviction based on last access time + 3. Verification that shared nodes remain intact for other tenants + """ + # Set up size limits + max_size = {"tenant1": 10, "tenant2": 5} + + # Insert strings for both tenants + self.tree.insert("hello", "tenant1") # size 5 + self.tree.insert("hello", "tenant2") # size 5 + self.tree.insert("world", "tenant2") # size 5, total for tenant2 = 10 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 5) # "hello" = 5 + self.assertEqual(sizes_before["tenant2"], 10) # "hello" + "world" = 10 + + # Evict - should remove "hello" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 5) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 5) # Only "world" remains + + # Verify "world" remains for tenant2 (was accessed more recently) + matched, tenant = self.tree.prefix_match("world") + self.assertEqual(matched, "world") + self.assertEqual(tenant, "tenant2") + + def test_medium_eviction(self): + """Test 5: Medium complexity eviction scenarios with shared prefixes + Tenant1: limit 10 chars + Tenant2: limit 7 chars (forces one string to be evicted) + + Tree structure after inserts: + └── 'h' [t1, t2] + ├── 'i' [t1, t2] # Oldest for t2 + └── 'e' [t1, t2] + ├── 'llo' [t1, t2] + └── 'y' [t2] # Newest for t2 + + Size calculations: + tenant1: "h"(1) + "i"(1) + "e"(1) + "llo"(3) = 6 chars + tenant2: "h"(1) + "i"(1) + "e"(1) + "llo"(3) + "y"(1) = 7 chars + + After eviction (tenant2 exceeds limit by 1 char): + "hi" should be removed from tenant2 as it's the oldest access + """ + max_size = { + "tenant1": 10, + "tenant2": 6, + } # tenant2 will need to evict one string + + # Create a tree with overlapping prefixes + self.tree.insert("hi", "tenant1") + self.tree.insert("hi", "tenant2") # OLDEST for t2 + + self.tree.insert("hello", "tenant1") + self.tree.insert("hello", "tenant2") + + self.tree.insert("hey", "tenant2") # NEWEST for t2 + + # Verify initial sizes + sizes_before = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_before["tenant1"], 6) # h(1) + i(1) + e(1) + llo(3) = 6 + self.assertEqual( + sizes_before["tenant2"], 7 + ) # h(1) + i(1) + e(1) + llo(3) + y(1) = 7 + + print("\nTree before eviction:") + print(self.tree.pretty_print()) + + # Evict - should remove "hi" from tenant2 as it's the oldest + self.tree.evict_tenant_data(max_size) + + print("\nTree after eviction:") + print(self.tree.pretty_print()) + + # Verify sizes after eviction + sizes_after = self.tree.get_used_size_per_tenant() + self.assertEqual(sizes_after["tenant1"], 6) # Should be unchanged + self.assertEqual(sizes_after["tenant2"], 6) # h(1) + e(1) + llo(3) + y(1) = 6 + + def test_advanced_eviction(self): + ... + # Create 4 tenants + # Each tenants keeps adding strings with shared prefixes to thousands usage + # Set a strict limit for each tenant to only 100 + # At the end, check whether all of the tenant is under 100 after eviction + + max_size = {"tenant1": 100, "tenant2": 100, "tenant3": 100, "tenant4": 100} + + prefixes = ["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"] + for i in range(100): + for j, prefix in enumerate(prefixes): + random_suffix = "".join(random.choices(string.ascii_letters, k=10)) + self.tree.insert(prefix + random_suffix, f"tenant{j+1}") + + sizes_before = self.tree.get_used_size_per_tenant() + print(sizes_before) + + self.tree.evict_tenant_data(max_size) + + sizes_after = self.tree.get_used_size_per_tenant() + print(sizes_after) + # ensure size_after is below max_size + for tenant, size in sizes_after.items(): + self.assertLessEqual(size, max_size[tenant]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/playground/router/tree.py b/scripts/playground/router/tree.py new file mode 100644 index 000000000..9cbfa7cfe --- /dev/null +++ b/scripts/playground/router/tree.py @@ -0,0 +1,292 @@ +import time +from collections import defaultdict +from typing import Dict, List + + +class Node: + def __init__(self): + self.children: Dict[str, Node] = dict() + # We choose to use text because most of the use cases are text-to-text, + # so we can save the tokenizing overhead. + self.text: str = "" + # Maps tenant_id to their last access timestamp + self.tenant_last_access_time: Dict[str, float] = dict() + self.parent = None + + +def shared_prefix_length(s1, s2): + min_length = min(len(s1), len(s2)) + for i in range(min_length): + if s1[i] != s2[i]: + return i + return min_length + + +class MultiTenantRadixTree: + """ + Python Reference of Rust implementation of MultiTenantRadixTree + + MultiTenantRadixTree is the overlap of multiple radix trees by different tenant + Each node in the tree can be owned by multiple tenants, allowing for efficient storage of common prefixes + while maintaining tenant isolation. + + Key concepts: + - Tenant: An entity that owns a subset of the stored strings + - Each node tracks which tenants have access to it via tenant_last_access_time + - The tree structure is shared, but queries can be filtered by tenant_id + """ + + def __init__(self): + self.root = Node() + + def insert(self, s: str, tenant_id: str) -> None: + """ + Insert string 's' and associate it with the given tenant_id. + + Args: + s: The string to insert + tenant_id: The identifier of the tenant who owns this string + """ + curr = self.root + curr_idx = 0 + curr.tenant_last_access_time[tenant_id] = time.time() + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + # No match => create a new node + new_node = Node() + new_node.text = s[curr_idx:] + new_node.parent = curr + + curr.children[s[curr_idx]] = new_node + curr_idx = len(s) + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + else: + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + + # 1. If the matched text is shorter than the node text => split the node + if shared_len < len(matched_node.text): + # Split structure: [matched_node] => [new_node] -> [contracted_matched_node] + + matched_text = matched_node.text[:shared_len] + unmatched_text = matched_node.text[shared_len:] + + new_node = Node() + new_node.text = matched_text + new_node.children = {unmatched_text[0]: matched_node} + new_node.parent = curr + new_node.parent.children[matched_text[0]] = new_node + new_node.tenant_last_access_time = ( + matched_node.tenant_last_access_time.copy() + ) + + # Contract matched node + matched_node.text = unmatched_text + matched_node.parent = new_node + + curr_idx += shared_len + curr = new_node + curr.tenant_last_access_time[tenant_id] = time.time() + # 2. If the matched text is longer or equal to the node text => walk down the node + else: + curr_idx += shared_len + curr = matched_node + curr.tenant_last_access_time[tenant_id] = time.time() + + def prefix_match(self, s: str) -> tuple[str, int]: + """ + Match string 's' with multiple tenants' trees in one operation. + + Args: + s: The string to match + + Returns: + Tuple(str, int): The longest prefix of 's' that matches the tree and the first tenant_id that own the matched prefix + """ + curr = self.root + curr_idx = 0 + + ret_text = "" + ret_tenant = None + + while curr_idx < len(s): + matched_node = None + if s[curr_idx] in curr.children: + matched_node = curr.children[s[curr_idx]] + + if matched_node is None: + break + + shared_len = shared_prefix_length(s[curr_idx:], matched_node.text) + if shared_len == len(matched_node.text): + curr_idx += shared_len + curr = matched_node + else: + curr_idx += shared_len + curr = matched_node + break + + selected_tenant = list(curr.tenant_last_access_time.keys())[0] + + # traverse back to the root to update last access time for the selected tenant + while curr != self.root: + curr.tenant_last_access_time[selected_tenant] = time.time() + curr = curr.parent + + return s[:curr_idx], selected_tenant + + def evict_tenant_data(self, max_size_per_tenant: Dict[str, int]) -> None: + """ + Evict data for tenants that have exceeded their storage limits. + + Args: + max_size_per_tenant: Dictionary mapping tenant_id to their maximum allowed storage size + """ + + def leaf_of(node): + """ + If the node is a leaf for a tenant, add tenant_id to the return list + This will return list of tenant ids + If not a leaf for all tenants, return [] + """ + candidates = dict([(k, True) for k in node.tenant_last_access_time.keys()]) + + for n in node.children.values(): + for c in n.tenant_last_access_time.keys(): + candidates[c] = False + + return [k for k, v in candidates.items() if v] + + # maintain a heap with (time, tenant, node) as the value + import heapq + + # 1. traverse the tree to + # a. add all the leaves into a heap (a node with N tenants will be added N times into the heap) + # b. calculate the used size for each tenant + # do a dfs with stack + stack = [self.root] + pq = [] + used_size_per_tenant = defaultdict(int) + + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + # if the node is a leaf for a tenant, add the tenant to the heap + tenants = leaf_of(curr) + for t in tenants: + heapq.heappush(pq, (curr.tenant_last_access_time[t], t, curr)) + + # 2. pop the heap + # a. if the tenant's used size is less than the limit, continue + # b. if the tenant's used size is greater than the limit, remove the leaf and update the used size, and add its parent to the heap + while len(pq) > 0: + time, tenant, node = heapq.heappop(pq) + if used_size_per_tenant[tenant] <= max_size_per_tenant[tenant]: + continue + + # remove the leaf + used_size_per_tenant[tenant] -= len(node.text) + del node.tenant_last_access_time[tenant] + # if no children and no tenants, remove the node + if len(node.children) == 0 and len(node.tenant_last_access_time) == 0: + del node.parent.children[node.text[0]] + + # add its parent to the heap + if tenant in leaf_of(node.parent): + heapq.heappush( + pq, + (node.parent.tenant_last_access_time[tenant], tenant, node.parent), + ) + + def get_used_size_per_tenant(self) -> Dict[str, int]: + """ + Calculate the used storage size for each tenant. + + Returns: + Dict[str, int]: A dictionary mapping tenant_id to their used storage size + """ + used_size_per_tenant = defaultdict(int) + + stack = [self.root] + while stack: + curr = stack.pop() + for t in curr.tenant_last_access_time.keys(): + used_size_per_tenant[t] += len(curr.text) + + for c in curr.children.values(): + stack.append(c) + + return used_size_per_tenant + + def remove_tenant(self, tenant_id: str) -> None: + """ + Remove all data associated with a specific tenant from the tree. + This operation maintains the integrity of the shared tree structure while + removing only the specified tenant's access information. + + Args: + tenant_id: The identifier of the tenant whose data should be removed + """ + # TODO: Implementation needed + pass + + def pretty_print(self) -> str: + """ + Returns a string representation of the tree showing the structure, tenant ownership, + and leaf status for each node. + + Returns: + str: A formatted string showing the tree hierarchy with tenant information + """ + + def _node_to_str(node: Node, prefix: str = "", is_last: bool = True) -> str: + # Current node representation + node_str = prefix + node_str += "└── " if is_last else "├── " + + # Add node text + node_str += f"'{node.text}' [" + + # Add tenant information including both timestamp and leaf status + tenant_info = [] + for tid, ts in node.tenant_last_access_time.items(): + time_str = ( + time.strftime("%H:%M:%S.", time.localtime(ts)) + + f"{(ts % 1):0.3f}"[2:] + ) + tenant_info.append(f"{tid} | {time_str}") + + node_str += ", ".join(tenant_info) + node_str += "]\n" + + # Handle children + children = list(node.children.items()) + for i, (char, child) in enumerate(children): + is_last_child = i == len(children) - 1 + # Adjust prefix for children based on whether this is the last child + new_prefix = prefix + (" " if is_last else "│ ") + node_str += _node_to_str(child, new_prefix, is_last_child) + + return node_str + + if not self.root.children: + return "Empty tree" + + # Start with root's children since root itself is just an empty node + result = "" + children = list(self.root.children.items()) + for i, (char, child) in enumerate(children): + is_last = i == len(children) - 1 + result += _node_to_str(child, "", is_last) + + return result diff --git a/scripts/update_kernel_whl_index.py b/scripts/update_kernel_whl_index.py new file mode 100644 index 000000000..e6b2279b1 --- /dev/null +++ b/scripts/update_kernel_whl_index.py @@ -0,0 +1,33 @@ +# Reference: https://github.com/flashinfer-ai/flashinfer/blob/v0.2.0/scripts/update_whl_index.py + +import argparse +import hashlib +import pathlib +import re + + +def update_wheel_index(cuda_version="118"): + index_dir = pathlib.Path(f"sgl-whl/cu{cuda_version}/sgl-kernel") + index_dir.mkdir(exist_ok=True) + base_url = "https://github.com/sgl-project/whl/releases/download" + + for path in sorted(pathlib.Path("sgl-kernel/dist").glob("*.whl")): + with open(path, "rb") as f: + sha256 = hashlib.sha256(f.read()).hexdigest() + ver = re.findall( + r"sgl_kernel-([0-9.]+(?:\.post[0-9]+)?)(?:\+cu[0-9]+)?-", path.name + )[0] + full_url = f"{base_url}/v{ver}/{path.name}#sha256={sha256}" + with (index_dir / "index.html").open("a") as f: + f.write(f'{path.name}
\n') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--cuda", type=str, default="118") + args = parser.parse_args() + update_wheel_index(args.cuda) + + +if __name__ == "__main__": + main() diff --git a/scripts/version_branch_to_tag.sh b/scripts/version_branch_to_tag.sh new file mode 100755 index 000000000..9f587fb0b --- /dev/null +++ b/scripts/version_branch_to_tag.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euxo pipefail + +# This script is used for release. +# It tags all remote branches starting with 'v' with the same name as the branch, +# deletes the corresponding branches from the remote, and pushes the tags to the remote repository. + +git fetch origin --prune + +# List all branches starting with 'v' +branches=$(git branch -r | grep 'origin/v' | sed 's/origin\///') + +# Loop through each branch +for branch in $branches; do + echo "Processing branch: $branch" + + # Get the commit hash for the branch + commit_hash=$(git rev-parse origin/$branch) + + # Create a tag with the same name as the branch using the commit hash + git tag $branch $commit_hash + + # Delete the branch from the remote + git push origin --delete $branch +done + +# Push all tags to the remote repository +git push --tags + +echo "All branches starting with 'v' have been tagged, deleted from remote, and pushed to the remote repository." diff --git a/sgl-kernel/.clang-format b/sgl-kernel/.clang-format new file mode 100644 index 000000000..afbd654a7 --- /dev/null +++ b/sgl-kernel/.clang-format @@ -0,0 +1,15 @@ +BasedOnStyle: Google +IndentWidth: 2 +ColumnLimit: 120 +AllowShortFunctionsOnASingleLine: Empty +DerivePointerAlignment: false +PointerAlignment: Left +NamespaceIndentation: None +SortIncludes: true +AllowShortLoopsOnASingleLine: false +BinPackParameters: false # Prevents packing parameters in declarations +BinPackArguments: false # Prevents packing arguments in function calls +AlignAfterOpenBracket: AlwaysBreak # Forces a break after the opening parenthesis +AlignOperands: Align # Aligns arguments vertically +PenaltyBreakBeforeFirstCallParameter: 1 # Encourages breaking before the first argument +PenaltyReturnTypeOnItsOwnLine: 100 # Keeps return type with function name diff --git a/sgl-kernel/CMakeLists.txt b/sgl-kernel/CMakeLists.txt new file mode 100644 index 000000000..80f29921f --- /dev/null +++ b/sgl-kernel/CMakeLists.txt @@ -0,0 +1,534 @@ +cmake_minimum_required(VERSION 3.26 FATAL_ERROR) +project(sgl-kernel LANGUAGES CXX CUDA) + +# CMake +cmake_policy(SET CMP0169 OLD) +cmake_policy(SET CMP0177 NEW) +include(${CMAKE_CURRENT_LIST_DIR}/cmake/utils.cmake) +set(CMAKE_COLOR_DIAGNOSTICS ON) +set(CMAKE_VERBOSE_MAKEFILE ON CACHE BOOL "ON") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +# Python +find_package(Python COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} REQUIRED) + +# CXX +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3") + +# CUDA +enable_language(CUDA) +find_package(CUDAToolkit REQUIRED) +set_property(GLOBAL PROPERTY CUDA_SEPARABLE_COMPILATION ON) + +message(STATUS "Detected CUDA_VERSION=${CUDA_VERSION}") +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "13.0") + message("CUDA_VERSION ${CUDA_VERSION} >= 13.0") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.8") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.4") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.1") + message("CUDA_VERSION ${CUDA_VERSION} >= 12.1") +elseif ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "11.8") + message("CUDA_VERSION ${CUDA_VERSION} >= 11.8") +endif() + +# Torch +find_package(Torch REQUIRED) +# clean Torch Flag +clear_cuda_arches(CMAKE_FLAG) + +include(FetchContent) + +# cutlass +FetchContent_Declare( + repo-cutlass + GIT_REPOSITORY https://github.com/NVIDIA/cutlass + GIT_TAG a49a78ffefc86a87160dfe0ccc3a3a2d1622c918 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-cutlass) + +# DeepGEMM +FetchContent_Declare( + repo-deepgemm + GIT_REPOSITORY https://github.com/sgl-project/DeepGEMM + GIT_TAG sgl + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-deepgemm) + +FetchContent_Declare( + repo-fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt + GIT_TAG 553ec11ec06fbe0beebfbb45f9dc3c9eabd83d28 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-fmt) + +# Triton +FetchContent_Declare( + repo-triton + GIT_REPOSITORY "https://github.com/triton-lang/triton" + GIT_TAG 8f9f695ea8fde23a0c7c88e4ab256634ca27789f + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-triton) + +# flashinfer +FetchContent_Declare( + repo-flashinfer + GIT_REPOSITORY https://github.com/flashinfer-ai/flashinfer.git + GIT_TAG 018b551825c8e5579206e6eb9d3229fa679202b3 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flashinfer) + +# flash-attention +FetchContent_Declare( + repo-flash-attention + GIT_REPOSITORY https://github.com/sgl-project/sgl-attn + GIT_TAG sgl-kernel + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flash-attention) + +# flash-attention origin +FetchContent_Declare( + repo-flash-attention-origin + GIT_REPOSITORY https://github.com/Dao-AILab/flash-attention.git + GIT_TAG 203b9b3dba39d5d08dffb49c09aa622984dff07d + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-flash-attention-origin) + +# mscclpp +FetchContent_Declare( + repo-mscclpp + GIT_REPOSITORY https://github.com/microsoft/mscclpp.git + GIT_TAG 51eca89d20f0cfb3764ccd764338d7b22cd486a6 + GIT_SHALLOW OFF +) +FetchContent_Populate(repo-mscclpp) + +# ccache option +option(ENABLE_CCACHE "Whether to use ccache" ON) +find_program(CCACHE_FOUND ccache) +if(CCACHE_FOUND AND ENABLE_CCACHE AND DEFINED ENV{CCACHE_DIR}) + message(STATUS "Building with CCACHE enabled") + set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "ccache") + set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "ccache") +endif() + +# Enable gencode below SM90 +option(ENABLE_BELOW_SM90 "Enable below SM90" ON) + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + set(ENABLE_BELOW_SM90 OFF) + message(STATUS "For aarch64, disable gencode below SM90 by default") +endif() + +include_directories( + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/csrc + ${repo-cutlass_SOURCE_DIR}/include + ${repo-cutlass_SOURCE_DIR}/tools/util/include + ${repo-flashinfer_SOURCE_DIR}/include + ${repo-flashinfer_SOURCE_DIR}/csrc + ${repo-mscclpp_SOURCE_DIR}/include +) + +set(SGL_KERNEL_CUDA_FLAGS + "-DNDEBUG" + "-DOPERATOR_NAMESPACE=sgl-kernel" + "-O3" + "-Xcompiler" + "-fPIC" + "-gencode=arch=compute_90,code=sm_90" + "-std=c++17" + "-DFLASHINFER_ENABLE_F16" + "-DCUTE_USE_PACKED_TUPLE=1" + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1" + "-DCUTLASS_VERSIONS_GENERATED" + "-DCUTLASS_TEST_LEVEL=0" + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1" + "-DCUTLASS_DEBUG_TRACE_LEVEL=0" + "--expt-relaxed-constexpr" + "--expt-extended-lambda" + # The following flag leads to the CMAKE_BUILD_PARALLEL_LEVEL breaking, + # it triggers OOM with low memory host. Extract the threads number to + # option named SGL_KERNEL_COMPILE_THREADS, default value 32. + # "--threads=32" + + # Supress warnings + "-Xcompiler=-Wno-clang-format-violations" + "-Xcompiler=-Wno-conversion" + "-Xcompiler=-Wno-deprecated-declarations" + "-Xcompiler=-Wno-terminate" + "-Xcompiler=-Wfatal-errors" + "-Xcompiler=-ftemplate-backtrace-limit=1" + "-Xcudafe=--diag_suppress=177" # variable was declared but never referenced + + # uncomment to debug + # "--ptxas-options=-v" + # "--ptxas-options=--verbose,--register-usage-level=10,--warn-on-local-memory-usage" +) + +set(SGL_KERNEL_COMPILE_THREADS 32 CACHE STRING "Set compilation threads, default 32") + +# When SGL_KERNEL_COMPILE_THREADS value is less than 1, set it to 1 +if (NOT SGL_KERNEL_COMPILE_THREADS MATCHES "^[0-9]+$") + message(FATAL_ERROR "SGL_KERNEL_COMPILE_THREADS must be an integer, but was set to '${SGL_KERNEL_COMPILE_THREADS}'.") +elseif (SGL_KERNEL_COMPILE_THREADS LESS 1) + message(STATUS "SGL_KERNEL_COMPILE_THREADS was set to a value less than 1. Using 1 instead.") + set(SGL_KERNEL_COMPILE_THREADS 1) +endif() + +list(APPEND SGL_KERNEL_CUDA_FLAGS + "--threads=${SGL_KERNEL_COMPILE_THREADS}" +) + +option(SGL_KERNEL_ENABLE_BF16 "Enable BF16" ON) +option(SGL_KERNEL_ENABLE_FP8 "Enable FP8" ON) +option(SGL_KERNEL_ENABLE_FP4 "Enable FP4" OFF) +option(SGL_KERNEL_ENABLE_FA3 "Enable FA3" OFF) +option(SGL_KERNEL_ENABLE_SM90A "Enable SM90A" OFF) +option(SGL_KERNEL_ENABLE_SM100A "Enable SM100A" OFF) + +if (SGL_KERNEL_ENABLE_BF16) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_BF16" + ) +endif() + +if (SGL_KERNEL_ENABLE_FP8) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DFLASHINFER_ENABLE_FP8" + "-DFLASHINFER_ENABLE_FP8_E4M3" + "-DFLASHINFER_ENABLE_FP8_E5M2" + ) +endif() + +if (ENABLE_BELOW_SM90) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_80,code=sm_80" + "-gencode=arch=compute_89,code=sm_89" + ) +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_SM100A) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_100a,code=sm_100a" + "-gencode=arch=compute_120a,code=sm_120a" + ) + + # refer sm_121, sm_110 and sm_101 description https://github.com/pytorch/pytorch/pull/156176 + if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "13.0") + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_103a,code=sm_103a" + "-gencode=arch=compute_110a,code=sm_110a" + "-gencode=arch=compute_121a,code=sm_121a" + "--compress-mode=size" + ) + else() + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_101a,code=sm_101a" + ) + endif() + +else() + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-use_fast_math" + ) +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.4" OR SGL_KERNEL_ENABLE_SM90A) + set(SGL_KERNEL_ENABLE_FA3 ON) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_90a,code=sm_90a" + ) +endif() + +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_FP4) + list(APPEND SGL_KERNEL_CUDA_FLAGS + "-DENABLE_NVFP4=1" + ) +endif() + +set(SOURCES + "csrc/allreduce/custom_all_reduce.cu" + "csrc/allreduce/mscclpp_allreduce.cu" + "csrc/attention/cascade.cu" + "csrc/attention/cutlass_mla_kernel.cu" + "csrc/attention/lightning_attention_decode_kernel.cu" + "csrc/attention/merge_attn_states.cu" + "csrc/attention/vertical_slash_index.cu" + "csrc/elementwise/activation.cu" + "csrc/elementwise/cast.cu" + "csrc/elementwise/copy.cu" + "csrc/elementwise/concat_mla.cu" + "csrc/elementwise/fused_add_rms_norm_kernel.cu" + "csrc/elementwise/rope.cu" + "csrc/common_extension.cc" + + "csrc/gemm/awq_kernel.cu" + "csrc/gemm/bmm_fp8.cu" + "csrc/gemm/dsv3_fused_a_gemm.cu" + "csrc/gemm/dsv3_router_gemm_bf16_out.cu" + "csrc/gemm/dsv3_router_gemm_entry.cu" + "csrc/gemm/dsv3_router_gemm_float_out.cu" + "csrc/gemm/fp8_blockwise_gemm_kernel.cu" + "csrc/gemm/fp8_gemm_kernel.cu" + "csrc/gemm/int8_gemm_kernel.cu" + "csrc/gemm/nvfp4_expert_quant.cu" + "csrc/gemm/nvfp4_quant_entry.cu" + "csrc/gemm/nvfp4_quant_kernels.cu" + "csrc/gemm/nvfp4_scaled_mm_entry.cu" + "csrc/gemm/nvfp4_scaled_mm_kernels.cu" + "csrc/gemm/per_tensor_quant_fp8.cu" + "csrc/gemm/per_token_group_quant_8bit.cu" + "csrc/gemm/per_token_quant_fp8.cu" + "csrc/gemm/qserve_w4a8_per_chn_gemm.cu" + "csrc/gemm/qserve_w4a8_per_group_gemm.cu" + "csrc/gemm/marlin/gptq_marlin.cu" + "csrc/gemm/marlin/gptq_marlin_repack.cu" + "csrc/gemm/marlin/awq_marlin_repack.cu" + "csrc/gemm/gptq/gptq_kernel.cu" + + "csrc/grammar/apply_token_bitmask_inplace_cuda.cu" + + "csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu" + "csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu" + "csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu" + "csrc/moe/marlin_moe_wna16/ops.cu" + "csrc/mamba/causal_conv1d.cu" + "csrc/moe/moe_align_kernel.cu" + "csrc/moe/moe_fused_gate.cu" + "csrc/moe/moe_topk_softmax_kernels.cu" + "csrc/moe/nvfp4_blockwise_moe.cu" + "csrc/moe/fp8_blockwise_moe_kernel.cu" + "csrc/moe/prepare_moe_input.cu" + + "csrc/memory/store.cu" + "csrc/kvcacheio/transfer.cu" + + "csrc/speculative/eagle_utils.cu" + "csrc/speculative/packbit.cu" + "csrc/speculative/speculative_sampling.cu" + + "${repo-flashinfer_SOURCE_DIR}/csrc/norm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/renorm.cu" + "${repo-flashinfer_SOURCE_DIR}/csrc/sampling.cu" + + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_bf16_causal_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_bf16_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_fp16_causal_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src/flash_fwd_sparse_hdim128_fp16_sm80.cu" + "${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/flash_sparse_api.cpp" +) + +Python_add_library(common_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) + +target_compile_options(common_ops PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) +target_include_directories(common_ops PRIVATE + ${repo-cutlass_SOURCE_DIR}/examples/77_blackwell_fmha + ${repo-cutlass_SOURCE_DIR}/examples/common + ${repo-flash-attention_SOURCE_DIR}/csrc/flash_attn/src +) + +find_package(Python3 COMPONENTS Interpreter REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import torch; print(int(torch._C._GLIBCXX_USE_CXX11_ABI))" + OUTPUT_VARIABLE TORCH_CXX11_ABI + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(TORCH_CXX11_ABI STREQUAL "0") + message(STATUS "Using old C++ ABI (-D_GLIBCXX_USE_CXX11_ABI=0)") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +else() + message(STATUS "Using new C++11 ABI (-D_GLIBCXX_USE_CXX11_ABI=1)") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=1") +endif() + +# mscclpp +set(MSCCLPP_USE_CUDA ON) +set(MSCCLPP_BYPASS_GPU_CHECK ON) +set(MSCCLPP_BUILD_TESTS OFF) +add_subdirectory( + ${repo-mscclpp_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/mscclpp-build +) +target_link_libraries(common_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda cublas cublasLt mscclpp_static) + +# flash attention +target_compile_definitions(common_ops PRIVATE + FLASHATTENTION_DISABLE_BACKWARD + FLASHATTENTION_DISABLE_DROPOUT + FLASHATTENTION_DISABLE_UNEVEN_K +) + +install(TARGETS common_ops LIBRARY DESTINATION sgl_kernel) + +# ============================ Optional Install ============================= # +# set flash-attention sources file +# Now FA3 support sm80/sm86/sm90 +if (SGL_KERNEL_ENABLE_FA3) + set(SGL_FLASH_KERNEL_CUDA_FLAGS + "-DNDEBUG" + "-DOPERATOR_NAMESPACE=sgl-kernel" + "-O3" + "-Xcompiler" + "-fPIC" + "-gencode=arch=compute_90a,code=sm_90a" + "-std=c++17" + "-DCUTE_USE_PACKED_TUPLE=1" + "-DCUTLASS_ENABLE_TENSOR_CORE_MMA=1" + "-DCUTLASS_VERSIONS_GENERATED" + "-DCUTLASS_TEST_LEVEL=0" + "-DCUTLASS_TEST_ENABLE_CACHED_RESULTS=1" + "-DCUTLASS_DEBUG_TRACE_LEVEL=0" + "--expt-relaxed-constexpr" + "--expt-extended-lambda" + "--use_fast_math" + "-Xcompiler=-Wconversion" + "-Xcompiler=-fno-strict-aliasing" + ) + + if (ENABLE_BELOW_SM90) + list(APPEND SGL_FLASH_KERNEL_CUDA_FLAGS + "-gencode=arch=compute_80,code=sm_80" + "-gencode=arch=compute_86,code=sm_86" + ) + # SM8X Logic + file(GLOB FA3_SM8X_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdim*_sm80.cu") + endif() + + file(GLOB FA3_BF16_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_bf16*_sm90.cu") + file(GLOB FA3_BF16_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_bf16*_sm90.cu") + list(APPEND FA3_BF16_GEN_SRCS ${FA3_BF16_GEN_SRCS_}) + + # FP16 source files + file(GLOB FA3_FP16_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_fp16*_sm90.cu") + file(GLOB FA3_FP16_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_fp16*_sm90.cu") + list(APPEND FA3_FP16_GEN_SRCS ${FA3_FP16_GEN_SRCS_}) + + # FP8 source files + file(GLOB FA3_FP8_GEN_SRCS + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimall_e4m3*_sm90.cu") + file(GLOB FA3_FP8_GEN_SRCS_ + "${repo-flash-attention_SOURCE_DIR}/hopper/instantiations/flash_fwd_hdimdiff_e4m3*_sm90.cu") + list(APPEND FA3_FP8_GEN_SRCS ${FA3_FP8_GEN_SRCS_}) + + set(FA3_GEN_SRCS ${FA3_BF16_GEN_SRCS} ${FA3_FP16_GEN_SRCS} ${FA3_FP8_GEN_SRCS} ${FA3_SM8X_GEN_SRCS}) + + set(FLASH_SOURCES + "csrc/flash_extension.cc" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_prepare_scheduler.cu" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_api.cpp" + "${repo-flash-attention_SOURCE_DIR}/hopper/flash_fwd_combine.cu" + "${FA3_GEN_SRCS}" + ) + + Python_add_library(flash_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${FLASH_SOURCES}) + + target_compile_options(flash_ops PRIVATE $<$:${SGL_FLASH_KERNEL_CUDA_FLAGS}>) + target_include_directories(flash_ops PRIVATE + ${repo-flash-attention_SOURCE_DIR}/hopper + ) + target_link_libraries(flash_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda) + + install(TARGETS flash_ops LIBRARY DESTINATION "sgl_kernel") + set(FLASH_OPS_COMPILE_DEFS + FLASHATTENTION_DISABLE_BACKWARD + FLASHATTENTION_DISABLE_DROPOUT + FLASHATTENTION_DISABLE_UNEVEN_K + FLASHATTENTION_VARLEN_ONLY + ) + + if(NOT ENABLE_BELOW_SM90) + list(APPEND FLASH_OPS_COMPILE_DEFS FLASHATTENTION_DISABLE_SM8x) + endif() + target_compile_definitions(flash_ops PRIVATE ${FLASH_OPS_COMPILE_DEFS}) +endif() + +# Build spatial_ops as a separate, optional extension for green contexts +set(SPATIAL_SOURCES + "csrc/spatial/greenctx_stream.cu" + "csrc/spatial_extension.cc" +) + +Python_add_library(spatial_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SPATIAL_SOURCES}) +target_compile_options(spatial_ops PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) +target_link_libraries(spatial_ops PRIVATE ${TORCH_LIBRARIES} c10 cuda) +install(TARGETS spatial_ops LIBRARY DESTINATION sgl_kernel) + + +# ============================ DeepGEMM (JIT) ============================= # +# Create a separate library for DeepGEMM's Python API. +# This keeps its compilation isolated from the main common_ops. +set(DEEPGEMM_SOURCES + "${repo-deepgemm_SOURCE_DIR}/csrc/python_api.cpp" +) + +Python_add_library(deep_gemm_cpp MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${DEEPGEMM_SOURCES}) + +# Link against necessary libraries, including nvrtc for JIT compilation. +target_link_libraries(deep_gemm_cpp PRIVATE ${TORCH_LIBRARIES} c10 cuda nvrtc mscclpp_static) + +# Add include directories needed by DeepGEMM. +target_include_directories(deep_gemm_cpp PRIVATE + ${repo-deepgemm_SOURCE_DIR}/deep_gemm/include + ${repo-cutlass_SOURCE_DIR}/include + ${repo-fmt_SOURCE_DIR}/include +) + +# Apply the same compile options as common_ops. +target_compile_options(deep_gemm_cpp PRIVATE $<$:${SGL_KERNEL_CUDA_FLAGS}>) + +# Create an empty __init__.py to make `deepgemm` a Python package. +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/deepgemm_pkg_init.py "") +install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/deepgemm_pkg_init.py + DESTINATION deep_gemm + RENAME __init__.py +) + +# Install the compiled DeepGEMM API library. +install(TARGETS deep_gemm_cpp LIBRARY DESTINATION deep_gemm) + +# Install the source files required by DeepGEMM for runtime JIT compilation. +install( + DIRECTORY ${repo-deepgemm_SOURCE_DIR}/deep_gemm/ + DESTINATION deep_gemm +) + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cute/" + DESTINATION "deep_gemm/include/cute") + +install(DIRECTORY "${repo-cutlass_SOURCE_DIR}/include/cutlass/" + DESTINATION "deep_gemm/include/cutlass") + +# triton_kernels +install(DIRECTORY "${repo-triton_SOURCE_DIR}/python/triton_kernels/triton_kernels/" + DESTINATION "triton_kernels" + PATTERN ".git*" EXCLUDE + PATTERN "__pycache__" EXCLUDE) + +# flash attention 4 +# TODO: find a better install condition. +if ("${CUDA_VERSION}" VERSION_GREATER_EQUAL "12.8" OR SGL_KERNEL_ENABLE_SM100A) + # flash_attn/cute + install(DIRECTORY "${repo-flash-attention-origin_SOURCE_DIR}/flash_attn/cute/" + DESTINATION "flash_attn/cute" + PATTERN ".git*" EXCLUDE + PATTERN "__pycache__" EXCLUDE) + endif() diff --git a/sgl-kernel/LICENSE b/sgl-kernel/LICENSE new file mode 100644 index 000000000..9c422689c --- /dev/null +++ b/sgl-kernel/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2024 SGLang Team + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sgl-kernel/Makefile b/sgl-kernel/Makefile new file mode 100644 index 000000000..c40489800 --- /dev/null +++ b/sgl-kernel/Makefile @@ -0,0 +1,71 @@ +.PHONY: help check-deps install-deps tree ln submodule install build clean rebuild test format update + +# Show help for each target +help: ## Show this help message + @echo "Available targets:" + @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +check-deps: ## Check and install required Python formatting dependencies + @command -v isort >/dev/null 2>&1 || (echo "Installing isort..." && pip install isort) + @command -v black >/dev/null 2>&1 || (echo "Installing black..." && pip install black) + +install-deps: ## Install Python formatting tools (isort and black) + pip install scikit-build-core isort black + +tree: ## Show project directory structure + @tree --prune -I "__pycache__|*.egg-info|*.so|build|3rdparty|dist" + +submodule: ## Initialize and update git submodules + @git submodule update --init --recursive + +ln: submodule ## Create compilation database + @rm -rf build && mkdir build && cd build && cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=YES -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + +install: submodule ## Install package in development mode + @pip install -e . --no-build-isolation + +build: install-deps submodule ## Build and install wheel package + @rm -rf dist/* || true && CMAKE_POLICY_VERSION_MINIMUM=3.5 MAX_JOBS=$(nproc) CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) uv build --wheel -Cbuild-dir=build . --verbose --color=always --no-build-isolation && pip3 install dist/*whl --force-reinstall --no-deps + +clean: ## Remove build artifacts + @rm -rf build dist *.egg-info + +rebuild: clean submodule build ## Clean and rebuild the project + @echo "Succeed to rebuild" + +test: ## Run all tests + @find tests -name "test_*.py" | xargs -n 1 python3 + +format: check-deps ## Format all source files + @echo "Formatting source files..." + @find csrc tests -name '*.cc' -o -name '*.cu' -o -name '*.cuh' -o -name '*.h' -o -name '*.hpp' | xargs clang-format -i + @find python tests -name '*.py' | xargs isort + @find python tests -name '*.py' | xargs black + @pre-commit run --all-files + +FILES_TO_UPDATE = python/sgl_kernel/version.py \ + pyproject.toml \ + pyproject_rocm.toml \ + pyproject_cpu.toml \ + ../docker/Dockerfile \ + ../.github/workflows/pr-test-pd-router.yml + +update: ## Update version numbers across project files. Usage: make update + @if [ -z "$(filter-out $@,$(MAKECMDGOALS))" ]; then \ + echo "Version required. Usage: make update "; \ + exit 1; \ + fi + @OLD_VERSION=$$(grep "version" python/sgl_kernel/version.py | cut -d '"' -f2); \ + NEW_VERSION=$(filter-out $@,$(MAKECMDGOALS)); \ + echo "Updating version from $$OLD_VERSION to $$NEW_VERSION"; \ + for file in $(FILES_TO_UPDATE); do \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + else \ + sed -i -e "s/$$OLD_VERSION/$$NEW_VERSION/g" $$file; \ + fi \ + done; \ + echo "Version update complete" + +%: + @: diff --git a/sgl-kernel/README.md b/sgl-kernel/README.md new file mode 100644 index 000000000..47f3dea54 --- /dev/null +++ b/sgl-kernel/README.md @@ -0,0 +1,263 @@ +# SGL Kernel + +[Kernel Library](https://github.com/sgl-project/sglang/tree/main/sgl-kernel) for SGLang + +[![PyPI](https://img.shields.io/pypi/v/sgl-kernel)](https://pypi.org/project/sgl-kernel) + +## Installation +For CUDA 12.1 and above: + +```bash +pip3 install sgl-kernel +``` + +For CUDA 11.8: + +```bash +pip3 install sgl-kernel -i https://docs.sglang.ai/whl/cu118 +``` + +## Build from source + +Development build: + +```bash +make build +``` + +Note: + +The `sgl-kernel` is rapidly evolving. If you experience a compilation failure, try using `make rebuild`. + +### Build with [ccache](https://github.com/ccache/ccache) +```bash +# or `yum install -y ccache`. +apt-get install -y ccache +# Building with ccache is enabled when ccache is installed and CCACHE_DIR is set. +export CCACHE_DIR=/path/to/your/ccache/dir +export CCACHE_BACKEND="" +export CCACHE_KEEP_LOCAL_STORAGE="TRUE" +unset CCACHE_READONLY +python -m uv build --wheel -Cbuild-dir=build --color=always . +``` + +### Configuring CMake Build Options +Cmake options can be configuring by adding `-Ccmake.define.

(sg.signals[target]); + } + auto tmp_out = tmps[0]; + start_sync(sg, self_sg, rank); + // stage 1: reduce scatter + for (int idx = start + tid; idx < end; idx += stride) { + tmp_out[idx - start] = packed_reduce(ptrs, idx); + } + end_sync(sg, self_sg, rank); + + // stage 2: allgather. Note: it's important to match the tid between + // the two stages, because visibility across devices is only guaranteed + // between threads that have the same tid. If thread i computes the sum of + // start + i in the first stage, then thread i also gathers start + i from all + // ranks. + for (int idx = tid; idx < largest_part; idx += stride) { +#pragma unroll + for (int i = 0; i < ngpus; i++) { + int gather_from_rank = ((rank + i) % ngpus); + if (gather_from_rank == ngpus - 1 || idx < part) { + int dst_idx = gather_from_rank * part + idx; + ((P*)result)[dst_idx] = tmps[i][idx]; + } + } + } +} + +using IPC_KEY = std::array; +static_assert(sizeof(IPC_KEY) == sizeof(hipIpcMemHandle_t)); +static_assert(alignof(IPC_KEY) == alignof(hipIpcMemHandle_t)); + +class CustomAllreduce { + public: + int rank_; + int world_size_; + bool full_nvlink_; + + // below are device pointers + RankSignals sg_; + std::unordered_map buffers_; + Signal* self_sg_; + + // stores the registered device pointers from all ranks + RankData *d_rank_data_base_, *d_rank_data_end_; + std::vector graph_unreg_buffers_; + // a map from IPC handles to opened IPC pointers + std::map ipc_handles_; + + /** + * meta is a pointer to device metadata and temporary buffer for allreduce. + * + * There's a total of sizeof(Signal) of prefix before the actual data, + * so meta + 1 points to actual temporary buffer. + * + * note: this class does not own any device memory. Any required buffers + * are passed in from the constructor + */ + CustomAllreduce( + Signal* meta, + void* rank_data, + size_t rank_data_sz, + const hipIpcMemHandle_t* handles, + const std::vector& offsets, + int rank, + bool full_nvlink = true) + : rank_(rank), + world_size_(offsets.size()), + full_nvlink_(full_nvlink), + self_sg_(meta), + d_rank_data_base_(reinterpret_cast(rank_data)), + d_rank_data_end_(d_rank_data_base_ + rank_data_sz / sizeof(RankData)) { + for (int i = 0; i < world_size_; i++) { + Signal* rank_sg; + if (i != rank_) { + char* handle = open_ipc_handle(&handles[i]); + handle += offsets[i]; + rank_sg = (Signal*)handle; + } else { + rank_sg = self_sg_; + } + sg_.signals[i] = rank_sg; + } + } + + char* open_ipc_handle(const void* ipc_handle) { + auto [it, new_handle] = ipc_handles_.insert({*((IPC_KEY*)ipc_handle), nullptr}); + if (new_handle) { + char* ipc_ptr; + CUDACHECK(hipIpcOpenMemHandle( + (void**)&ipc_ptr, *((const hipIpcMemHandle_t*)ipc_handle), hipIpcMemLazyEnablePeerAccess)); + it->second = ipc_ptr; + } + return it->second; + } + + std::pair, std::vector> get_graph_buffer_ipc_meta() { + auto num_buffers = graph_unreg_buffers_.size(); + auto handle_sz = sizeof(hipIpcMemHandle_t); + std::vector handles(handle_sz * num_buffers, 0); + std::vector offsets(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto ptr = graph_unreg_buffers_[i]; + void* base_ptr; + // note: must share the base address of each allocation, or we get wrong + // address + if (hipPointerGetAttribute( + &base_ptr, +#ifdef USE_ROCM + HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#else + CU_POINTER_ATTRIBUTE_RANGE_START_ADDR, +#endif + (hipDeviceptr_t)ptr) != hipSuccess) + throw std::runtime_error("failed to get pointer attr"); + CUDACHECK(hipIpcGetMemHandle((hipIpcMemHandle_t*)&handles[i * handle_sz], base_ptr)); + offsets[i] = ((char*)ptr) - ((char*)base_ptr); + } + return std::make_pair(handles, offsets); + } + + void check_rank_data_capacity(size_t num = 1) { + if (d_rank_data_base_ + num > d_rank_data_end_) + throw std::runtime_error( + "Rank data buffer is overflowed by " + std::to_string(d_rank_data_base_ + num - d_rank_data_end_)); + } + + void register_buffer(const std::vector& handles, const std::vector& offsets, void* self) { + check_rank_data_capacity(); + RankData data; + for (int i = 0; i < world_size_; i++) { + if (i != rank_) { + char* handle = open_ipc_handle(handles[i].data()); + handle += offsets[i]; + data.ptrs[i] = handle; + } else { + data.ptrs[i] = self; + } + } + auto d_data = d_rank_data_base_++; + CUDACHECK(hipMemcpy(d_data, &data, sizeof(RankData), hipMemcpyHostToDevice)); + buffers_[self] = d_data; + } + + // note: when registering graph buffers, we intentionally choose to not + // deduplicate the addresses. That means if the allocator reuses some + // addresses, they will be registered again. This is to account for the remote + // possibility of different allocation patterns between ranks. For example, + // rank 1 may get the same input address for the second allreduce, but rank 2 + // got a different address. IPC handles have internal reference counting + // mechanism so overhead should be small. + void + register_graph_buffers(const std::vector& handles, const std::vector>& offsets) { + auto num_buffers = graph_unreg_buffers_.size(); + check_rank_data_capacity(num_buffers); + std::vector rank_data(num_buffers); + for (int i = 0; i < num_buffers; i++) { + auto self_ptr = graph_unreg_buffers_[i]; + auto& rd = rank_data[i]; + for (int j = 0; j < world_size_; j++) { + if (j != rank_) { + char* handle = open_ipc_handle(&handles[j][i * sizeof(hipIpcMemHandle_t)]); + handle += offsets[j][i]; + rd.ptrs[j] = handle; + } else { + rd.ptrs[j] = self_ptr; + } + } + } + CUDACHECK(hipMemcpy(d_rank_data_base_, rank_data.data(), sizeof(RankData) * num_buffers, hipMemcpyHostToDevice)); + d_rank_data_base_ += num_buffers; + graph_unreg_buffers_.clear(); + } + + /** + * This is the result after careful grid search. Using 36 blocks give the best + * or close to the best runtime on the devices I tried: A100, A10, A30, T4, + * V100. You'll notice that NCCL kernels also only take a small amount of SMs. + * Not quite sure the underlying reason, but my guess is that too many SMs + * will cause contention on NVLink bus. + */ + template + void allreduce( + hipStream_t stream, + T* input, + T* output, + int size, +#ifndef USE_ROCM + int threads = 512, + int block_limit = 36){ +#else + int threads = 512, + int block_limit = 16) { +#endif + auto d = packed_t::P::size; + if (size % d != 0) + throw std::runtime_error( + "custom allreduce currently requires input length to be multiple " + "of " + + std::to_string(d)); + if (block_limit > kMaxBlocks) + throw std::runtime_error( + "max supported block limit is " + std::to_string(kMaxBlocks) + ". Got " + std::to_string(block_limit)); + + RankData* ptrs; + hipStreamCaptureStatus status; + CUDACHECK(hipStreamIsCapturing(stream, &status)); + if (status == hipStreamCaptureStatusActive) { + ptrs = d_rank_data_base_ + graph_unreg_buffers_.size(); + graph_unreg_buffers_.push_back(input); + } else { + auto it = buffers_.find(input); + if (it == buffers_.end()) + throw std::runtime_error( + "buffer address " + std::to_string(reinterpret_cast(input)) + " is not registered!"); + ptrs = it->second; + } + + size /= d; + auto bytes = size * sizeof(typename packed_t::P); + int blocks = ::min(block_limit, (size + threads - 1) / threads); +#define KL(ngpus, name) \ + hipLaunchKernelGGL( \ + (name), dim3(blocks), dim3(threads), 0, stream, ptrs, sg_, self_sg_, output, rank_, size); +#define REDUCE_CASE(ngpus) \ + case ngpus: { \ + if (world_size_ == 2) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else if (full_nvlink_) { \ + if ((world_size_ <= 4 && bytes < 512 * 1024) || (world_size_ <= 8 && bytes < 256 * 1024)) { \ + KL(ngpus, cross_device_reduce_1stage); \ + } else { \ + KL(ngpus, cross_device_reduce_2stage); \ + } \ + } \ + break; \ + } + + switch (world_size_) { + REDUCE_CASE(2) + REDUCE_CASE(4) + REDUCE_CASE(6) + REDUCE_CASE(8) + default: + throw std::runtime_error( + "custom allreduce only supports num gpus in (2,4,6,8). Actual num " + "gpus = " + + std::to_string(world_size_)); + } +#undef REDUCE_CASE +#undef KL +} + +~CustomAllreduce() { + for (auto [_, ptr] : ipc_handles_) { + CUDACHECK(hipIpcCloseMemHandle(ptr)); + } +} +}; // namespace sglang +/** + * To inspect PTX/SASS, copy paste this header file to compiler explorer and add + a template instantiation: + * template void sglang::CustomAllreduce::allreduce(hipStream_t, half *, + half *, int, int, int); +*/ +} // namespace sglang diff --git a/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu b/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu new file mode 100644 index 000000000..d9cda22f0 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cu @@ -0,0 +1,140 @@ +#include +#include +#include +#include + +#include "mscclpp_allreduce.cuh" + +enum MscclContextSelection { + MSCCL1NODELL = 1, + MSCCL2NODELL = 2, +}; + +class MscclContext { + public: + MscclContextSelection selection_; + std::shared_ptr msccl_1nodeLL_context; + std::shared_ptr msccl_2nodeLL_context; + MscclContext(MscclContextSelection selection) : selection_(selection) {} + template + void allreduce( + cudaStream_t stream, T* input, T* output, const size_t input_numel, int threads = 512, int block_limit = 21) { + if (selection_ == MSCCL1NODELL) { + msccl_1nodeLL_context->allreduce(stream, input, output, input_numel, threads, block_limit); + } else if (selection_ == MSCCL2NODELL) { + msccl_2nodeLL_context->allreduce(stream, input, output, input_numel, threads, block_limit); + } + } +}; + +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +torch::Tensor _unique_id2tensor(const mscclpp::UniqueId& unique_id) { + auto options = torch::TensorOptions().dtype(torch::kByte).device(torch::kCPU); + auto tensor = torch::empty({static_cast(unique_id.size())}, options); + std::memcpy(tensor.data_ptr(), unique_id.data(), unique_id.size()); + return tensor; +} + +// Function to convert vector of int32_t back to array of uint8_t +mscclpp::UniqueId _tensor2unique_id(const torch::Tensor& tensor) { + mscclpp::UniqueId unique_id; + std::memcpy(unique_id.data(), tensor.data_ptr(), unique_id.size()); + return unique_id; +} + +torch::Tensor mscclpp_generate_unique_id() { + mscclpp::UniqueId unique_id = mscclpp::TcpBootstrap::createUniqueId(); + return _unique_id2tensor(unique_id); +} + +fptr_t mscclpp_init_context( + const torch::Tensor& unique_id, + const int64_t rank, + const int64_t world_size, + torch::Tensor& scratch, + torch::Tensor& put_buffer, + const int64_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib, + const int64_t context_selection) { + MscclContext* context_ptr = new MscclContext(static_cast(context_selection)); + mscclpp::UniqueId uid = _tensor2unique_id(unique_id); + if (context_selection == MSCCL1NODELL) { + void* scratch_ptr = reinterpret_cast(scratch.data_ptr()); + const size_t scratch_bytes = scratch.numel() * scratch.element_size(); + context_ptr->msccl_1nodeLL_context = std::make_shared( + uid, rank, world_size, scratch_ptr, scratch_bytes, nranks_per_node, rank_to_node, rank_to_ib); + } else if (context_selection == MSCCL2NODELL) { + void* scratch_ptr = reinterpret_cast(scratch.data_ptr()); + const size_t scratch_bytes = scratch.numel() * scratch.element_size(); + void* put_buffer_ptr = reinterpret_cast(put_buffer.data_ptr()); + const size_t put_buffer_bytes = put_buffer.numel() * put_buffer.element_size(); + context_ptr->msccl_2nodeLL_context = std::make_shared( + uid, + rank, + world_size, + scratch_ptr, + scratch_bytes, + put_buffer_ptr, + put_buffer_bytes, + nranks_per_node, + rank_to_node, + rank_to_ib); + } else { + throw std::runtime_error("invalid context selection"); + } + return (fptr_t)context_ptr; +} + +bool _mscclpp_is_weak_contiguous(torch::Tensor& t) { + return t.is_contiguous() || + (t.storage().nbytes() - t.storage_offset() * t.element_size() == t.numel() * t.element_size()); +} +void mscclpp_allreduce(fptr_t _context, torch::Tensor& inp, torch::Tensor& out, int64_t nthreads, int64_t nblocks) { + MscclContext* context = reinterpret_cast(_context); + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp)); + auto stream = c10::cuda::getCurrentCUDAStream().stream(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK(_mscclpp_is_weak_contiguous(out)); + TORCH_CHECK(_mscclpp_is_weak_contiguous(inp)); + switch (out.scalar_type()) { + case at::ScalarType::Float: { + context->allreduce( + stream, + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } + case at::ScalarType::Half: { + context->allreduce( + stream, + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) + case at::ScalarType::BFloat16: { + context->allreduce<__nv_bfloat16>( + stream, + reinterpret_cast<__nv_bfloat16*>(inp.data_ptr()), + reinterpret_cast<__nv_bfloat16*>(out.data_ptr()), + inp.numel(), + nthreads, + nblocks); + break; + } +#endif + default: + throw std::runtime_error("custom allreduce only supports float32, float16 and bfloat16"); + } +} diff --git a/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh b/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh new file mode 100644 index 000000000..ba0bc33fd --- /dev/null +++ b/sgl-kernel/csrc/allreduce/mscclpp_allreduce.cuh @@ -0,0 +1,779 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +#pragma once +#ifdef USE_ROCM +#include +#else +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include + +// comment this for test_mscclpp_allreduce.cu +#include "utils.h" + +namespace sglang { + +__device__ mscclpp::DeviceSyncer deviceSyncer; +__device__ mscclpp::DeviceSyncer allGatherDeviceSyncer; +__device__ mscclpp::DeviceSyncer reduceScatterDeviceSyncer; +__device__ mscclpp::DeviceSyncer ibDeviceSyncer; + +template +__forceinline__ __device__ To bit_cast(const From& src) { + static_assert(sizeof(To) == sizeof(From), "Size mismatch for bit_cast"); + + union { + From f; + To t; + } u; + u.f = src; + return u.t; +} + +template +__forceinline__ __device__ T add_elements(T a, T b) { + return a + b; +} + +template <> +__forceinline__ __device__ __half2 add_elements(__half2 a, __half2 b) { + return __hadd2(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ __nv_bfloat162 add_elements(__nv_bfloat162 a, __nv_bfloat162 b) { + return __hadd2(a, b); +} +#endif + +template +__forceinline__ __device__ int4 add_vectors_helper(int4 a, int4 b) { + int4 ret; + ret.w = bit_cast(add_elements(bit_cast(a.w), bit_cast(b.w))); + ret.x = bit_cast(add_elements(bit_cast(a.x), bit_cast(b.x))); + ret.y = bit_cast(add_elements(bit_cast(a.y), bit_cast(b.y))); + ret.z = bit_cast(add_elements(bit_cast(a.z), bit_cast(b.z))); + return ret; +} + +template +__forceinline__ __device__ int4 add_vectors(int4 a, int4 b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ int4 add_vectors<__nv_bfloat16>(int4 a, int4 b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ int4 add_vectors<__half>(int4 a, int4 b) { + return add_vectors_helper<__half2>(a, b); +} + +template +__forceinline__ __device__ uint2 add_vectors_helper(uint2 a, uint2 b) { + uint2 ret; + ret.x = bit_cast(add_elements(bit_cast(a.x), bit_cast(b.x))); + ret.y = bit_cast(add_elements(bit_cast(a.y), bit_cast(b.y))); + return ret; +} + +template +__forceinline__ __device__ uint2 add_vectors(uint2 a, uint2 b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ uint2 add_vectors<__nv_bfloat16>(uint2 a, uint2 b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ uint2 add_vectors<__half>(uint2 a, uint2 b) { + return add_vectors_helper<__half2>(a, b); +} + +template +__forceinline__ __device__ int add_vectors_helper(int a, int b) { + return bit_cast(add_elements(bit_cast(a), bit_cast(b))); +} + +template +__forceinline__ __device__ int add_vectors(int a, int b) { + return add_vectors_helper(a, b); +} + +#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +template <> +__forceinline__ __device__ int add_vectors<__nv_bfloat16>(int a, int b) { + return add_vectors_helper<__nv_bfloat162>(a, b); +} +#endif + +template <> +__forceinline__ __device__ int add_vectors<__half>(int a, int b) { + return add_vectors_helper<__half2>(a, b); +} + +// ------------------------------------------------------- +// allreduce_LL_1node using LLPacket, origin allreduce2 +// ------------------------------------------------------- + +__device__ uint64_t globalFlag = 1; + +template +__global__ void __launch_bounds__(1024, 1) allreduce_LL_1node( + mscclpp::MemoryChannelDeviceHandle* memChans, + TYPE* buff, + TYPE* scratch, + void* resultBuff, + int rank, + int worldSize, + size_t nelems) { + nelems = nelems / (sizeof(int) / sizeof(TYPE)); + // This version of allreduce only works for single nodes + const int nPeers = worldSize - 1; + const size_t nPkts = nelems / 2; + const int nelemsPerRank = nelems / worldSize; + const int nPktsPerRank = nelemsPerRank / 2; + // flag for packets. Initially 1 + const uint32_t flag = (uint32_t)globalFlag; + // thread block & channel info + const int nBlocksPerPeer = gridDim.x / nPeers; + const int localBlockIdx = blockIdx.x % nBlocksPerPeer; + const int peerIdx = blockIdx.x / nBlocksPerPeer; + const int remoteRank = peerIdx < rank ? peerIdx : peerIdx + 1; + mscclpp::MemoryChannelDeviceHandle memChan = memChans[peerIdx]; + const int tid = threadIdx.x + localBlockIdx * blockDim.x; + // double buffering + size_t scratchBaseOffset = (flag & 1) ? 0 : nPkts * sizeof(mscclpp::LLPacket); + void* scratchBuff = (void*)((char*)scratch + scratchBaseOffset); + size_t scratchOffset = scratchBaseOffset + rank * nPktsPerRank * sizeof(mscclpp::LLPacket); + size_t scratchResultOffset = + (flag & 1) ? 2 * nPkts * sizeof(mscclpp::LLPacket) : 3 * nPkts * sizeof(mscclpp::LLPacket); + size_t srcOffset = remoteRank * nelemsPerRank * sizeof(int); + uint2* src = (uint2*)((char*)buff + rank * nelemsPerRank * sizeof(int)); + uint2* dst = (uint2*)((char*)resultBuff + rank * nelemsPerRank * sizeof(int)); + + // step 1: write to scratch buffer + memChan.putPackets(scratchOffset, srcOffset, nelemsPerRank * sizeof(int), tid, blockDim.x * nBlocksPerPeer, flag); + // step 2: get data from scratch buffer, reduce data and write result to remote scratch buffer + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerRank; idx += blockDim.x * gridDim.x) { + uint2 data = make_uint2(0, 0); + for (int index = 0; index < nPeers; index++) { + const int remoteRank = index < rank ? index : index + 1; + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + remoteRank * nPktsPerRank; + uint2 val = dstPkt[idx].read(flag); + data = add_vectors(val, data); + } + data = add_vectors(data, src[idx]); + dst[idx] = data; + + mscclpp::LLPacket packet; + packet.data1 = data.x; + packet.flag1 = flag; + packet.data2 = data.y; + packet.flag2 = flag; + size_t offset = scratchResultOffset / sizeof(mscclpp::LLPacket) + (idx + rank * nPktsPerRank); + for (int index = 0; index < nPeers; index++) { + memChans[index].write(offset, packet); + } + } + // step 3: get data result from scratch buffer + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)((char*)scratch + scratchResultOffset); + const int dstOffset = remoteRank * nPktsPerRank; + uint2* result = (uint2*)((char*)resultBuff + remoteRank * nelemsPerRank * sizeof(int)); + for (int idx = threadIdx.x + localBlockIdx * blockDim.x; idx < nPktsPerRank; idx += blockDim.x * nBlocksPerPeer) { + uint2 data = dstPkt[idx + dstOffset].read(flag); + result[idx].x = data.x; + result[idx].y = data.y; + } + if (threadIdx.x == 0 && blockIdx.x == 0) { + globalFlag += 1; + } +} + +// ------------------------------------------------------- +// allreduce_LL_2node using LLPacket, origin allreduce5 +// ------------------------------------------------------- + +template +__global__ void __launch_bounds__(1024, 1) allreduce_LL_2node( + mscclpp::MemoryChannelDeviceHandle* memChans, + mscclpp::PortChannelDeviceHandle* portChans, + TYPE* buff, + TYPE* scratch, + TYPE* putBuff, + TYPE* resultBuff, + int rank, + int nRanksPerNode, + int worldSize, + size_t nelems) { + nelems = nelems / (sizeof(int) / sizeof(TYPE)); + // This version of allreduce only works for single nodes + const int nPeersInNode = nRanksPerNode - 1; + const int nPkts = nelems / 2; + const int nelemsPerLocalRank = nelems / nRanksPerNode; + const int nPktsPerLocalRank = nelemsPerLocalRank / 2; + const int localRankId = rank % nRanksPerNode; + // flag for packets. Initially 1 + const uint32_t flag = (uint32_t)globalFlag; + // thread block & channel info + const int nBlocksPerPeer = gridDim.x / nPeersInNode; + const int localBlockIdx = blockIdx.x % nBlocksPerPeer; + const int peerIdx = blockIdx.x / nBlocksPerPeer; + const int remoteRankIdx = peerIdx < localRankId ? peerIdx : peerIdx + 1; + mscclpp::MemoryChannelDeviceHandle memChan = memChans[peerIdx]; + mscclpp::PortChannelDeviceHandle portChan = portChans[localRankId]; + const int tid = threadIdx.x + localBlockIdx * blockDim.x; + // double buffering + size_t scratchBaseOffset = (flag & 1) ? 0 : nPkts * sizeof(mscclpp::LLPacket); + size_t putBaseOffset = (flag & 1) ? 0 : nPktsPerLocalRank * sizeof(mscclpp::LLPacket); + void* scratchBuff = (void*)((char*)scratch + scratchBaseOffset); + size_t scratchOffset = scratchBaseOffset + localRankId * nPktsPerLocalRank * sizeof(mscclpp::LLPacket); + size_t scratchResultOffset = + (flag & 1) ? 2 * nPkts * sizeof(mscclpp::LLPacket) : 3 * nPkts * sizeof(mscclpp::LLPacket); + size_t srcOffset = remoteRankIdx * nelemsPerLocalRank * sizeof(int); + uint2* src = (uint2*)((char*)buff + localRankId * nelemsPerLocalRank * sizeof(int)); + uint2* dst = (uint2*)((char*)resultBuff + localRankId * nelemsPerLocalRank * sizeof(int)); + + // step 1: write to scratch buffer + if (nRanksPerNode > 1) { + memChan.putPackets( + scratchOffset, srcOffset, nelemsPerLocalRank * sizeof(int), tid, blockDim.x * nBlocksPerPeer, flag); + } + // step 2: get data from scratch buffer, do local reduce-scatter in each node. + mscclpp::LLPacket* putPkt = (mscclpp::LLPacket*)((char*)putBuff + putBaseOffset); + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerLocalRank; idx += blockDim.x * gridDim.x) { + uint2 data = make_uint2(0, 0); + for (int index = 0; index < nPeersInNode; index++) { + const int remoteRank = index < localRankId ? index : index + 1; + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + remoteRank * nPktsPerLocalRank; + uint2 val = dstPkt[idx].read(flag); + data = add_vectors(val, data); + } + data = add_vectors(data, src[idx]); + putPkt[idx].write(data.x, data.y, flag); + dst[idx] = data; + } + deviceSyncer.sync(gridDim.x); + // step 3. send local reduced data to remote node. + if (threadIdx.x == 0 && blockIdx.x == 0) { + portChan.put(scratchOffset, putBaseOffset, nPktsPerLocalRank * sizeof(mscclpp::LLPacket)); + if ((flag & 63) == 0) { + portChan.flush(); + } + } + // step 4. try to read the data from scratch buffer and write to local peers + mscclpp::LLPacket* dstPkt = (mscclpp::LLPacket*)scratchBuff + localRankId * nPktsPerLocalRank; + for (int idx = threadIdx.x + blockIdx.x * blockDim.x; idx < nPktsPerLocalRank; idx += blockDim.x * gridDim.x) { + uint2 res = dst[idx]; + uint2 val = dstPkt[idx].read(flag); + res = add_vectors(res, val); + + mscclpp::LLPacket packet; + packet.data1 = res.x; + packet.flag1 = flag; + packet.data2 = res.y; + packet.flag2 = flag; + size_t offset = scratchResultOffset / sizeof(mscclpp::LLPacket) + (idx + localRankId * nPktsPerLocalRank); + for (int index = 0; index < nPeersInNode; index++) { + memChans[index].write(offset, packet); + } + dst[idx] = res; + } + + // step 5: get data result from scratch buffer + dstPkt = (mscclpp::LLPacket*)((char*)scratch + scratchResultOffset); + const int dstOffset = remoteRankIdx * nPktsPerLocalRank; + uint2* result = (uint2*)((char*)resultBuff + remoteRankIdx * nelemsPerLocalRank * sizeof(int)); + if (nRanksPerNode > 1) { + for (int idx = threadIdx.x + localBlockIdx * blockDim.x; idx < nPktsPerLocalRank; + idx += blockDim.x * nBlocksPerPeer) { + uint2 data = dstPkt[idx + dstOffset].read(flag); + result[idx] = data; + } + } + if (threadIdx.x == 0 && blockIdx.x == 0) { + globalFlag += 1; + } +} + +static const mscclpp::Transport IBs[] = { + mscclpp::Transport::IB0, + mscclpp::Transport::IB1, + mscclpp::Transport::IB2, + mscclpp::Transport::IB3, + mscclpp::Transport::IB4, + mscclpp::Transport::IB5, + mscclpp::Transport::IB6, + mscclpp::Transport::IB7}; + +class MscclCommGroup { + public: + std::shared_ptr comm_; + const size_t rank_; + const size_t world_size_; + const std::vector rank_to_node_; + const std::vector rank_to_ib_; + MscclCommGroup( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : rank_(rank), world_size_(world_size), rank_to_node_(rank_to_node), rank_to_ib_(rank_to_ib) { + auto bootstrap = std::make_shared(rank, world_size); + bootstrap->initialize(unique_id); + comm_ = std::make_shared(bootstrap); + } + template + void allreduce(cudaStream_t stream, T* output, size_t input_numel, int threads = 512, int block_limit = 21) { + throw std::runtime_error("you should not call allreduce of a base context"); + } + bool is_same_node(int r1, int r2) { + return rank_to_node_[r1] == rank_to_node_[r2]; + } + + void make_connection( + std::unordered_map>& same_node_connections, + std::unordered_map>& cross_node_connections) { + same_node_connections.clear(); + cross_node_connections.clear(); + std::unordered_map>> conn_futures; + for (int r = 0; r < world_size_; ++r) { + if (r == rank_) continue; + mscclpp::Transport transport = is_same_node(r, rank_) ? mscclpp::Transport::CudaIpc : IBs[rank_to_ib_[r]]; + conn_futures.emplace(r, comm_->connectOnSetup(r, 0, transport)); + } + comm_->setup(); + for (int r = 0; r < world_size_; ++r) { + if (r == rank_) continue; + if (is_same_node(r, rank_)) { + same_node_connections.emplace(r, conn_futures[r].get()); + } else { + cross_node_connections.emplace(r, conn_futures[r].get()); + } + } + } + + void make_memory_channels_with_scratch( + void* tensor_ptr, + const size_t tensor_bytes, + void* scratch_ptr, + const size_t scratch_bytes, + const std::unordered_map>& connections, + std::unordered_map>& semaphores, + std::unordered_map& registered_memories, + std::unordered_map& channels) { + channels.clear(); + make_semaphores(connections, semaphores); + register_tensor_with_connections(scratch_ptr, scratch_bytes, connections, registered_memories); + for (const auto& [peer, _] : connections) { + channels.emplace( + peer, mscclpp::MemoryChannel(semaphores[peer], registered_memories[peer], tensor_ptr, scratch_ptr)); + } + } + void make_port_channels_with_scratch( + std::shared_ptr proxyService, + void* tensor_ptr, + const size_t tensor_bytes, + void* scratch_ptr, + const size_t scratch_bytes, + const std::unordered_map>& connections, + std::unordered_map>& semaphores, + std::unordered_map& registered_memories, + std::unordered_map& channels) { + channels.clear(); + make_semaphores(connections, semaphores); + + mscclpp::TransportFlags flags; + for (const auto& [_, conn] : connections) { + flags |= conn->transport(); + } + auto local_reg_memory = comm_->registerMemory(tensor_ptr, tensor_bytes, flags); + + register_tensor_with_connections(scratch_ptr, scratch_bytes, connections, registered_memories); + std::unordered_map semaphore_ids; + std::unordered_map memory_ids; + memory_ids[rank_] = proxyService->addMemory(local_reg_memory); + for (const auto& [peer, memory] : registered_memories) { + if (peer == rank_) continue; + memory_ids[peer] = proxyService->addMemory(memory); + } + for (const auto& [peer, semaphore] : semaphores) { + semaphore_ids[peer] = proxyService->addSemaphore(semaphore); + } + + for (const auto& [peer, _] : connections) { + channels.emplace(peer, proxyService->portChannel(semaphore_ids[peer], memory_ids[peer], memory_ids[rank_])); + } + } + + template + void make_semaphores( + const std::unordered_map>& connections, + std::unordered_map>& semaphores) { + semaphores.clear(); + for (const auto& [peer, conn] : connections) { + semaphores[peer] = std::make_shared(*comm_, conn); + } + comm_->setup(); + } + + void register_tensor_with_connections( + void* tensor_ptr, + size_t tensor_bytes, + const std::unordered_map>& connections, + std::unordered_map& registered_memories) { + registered_memories.clear(); + mscclpp::TransportFlags all_transports; + for (const auto& [_, connection] : connections) { + all_transports |= connection->transport(); + } + mscclpp::RegisteredMemory buf_reg_mem = comm_->registerMemory(tensor_ptr, tensor_bytes, all_transports); + registered_memories[rank_] = buf_reg_mem; + + std::unordered_map> remote_mem_futures; + for (const auto& [r, connection] : connections) { + comm_->sendMemoryOnSetup(buf_reg_mem, r, 0); + auto remoteMemory = comm_->recvMemoryOnSetup(r, 0); + remote_mem_futures.emplace(r, remoteMemory); + } + comm_->setup(); + for (auto& [r, mem_feature] : remote_mem_futures) { + registered_memories.emplace(r, mem_feature.get()); + } + } + + void make_device_memory_handle_base_on_new_ptr( + const std::unordered_map& old_memory_channels, + std::unordered_map& registered_sm_memories, + std::unordered_map>& memory_semaphores, + std::unordered_map& memory_channels, + mscclpp::GpuBuffer& device_memory_handle, + void* input, + void* scratch, + const cudaStream_t stream) { + memory_channels.clear(); + for (const auto& [peer, channel] : old_memory_channels) { + memory_channels.emplace( + peer, mscclpp::MemoryChannel(memory_semaphores[peer], registered_sm_memories[peer], input, scratch)); + } + std::vector memory_channels_list; + for (int r = 0; r < world_size_; r++) { + if (r == rank_) continue; + if (is_same_node(r, rank_)) { + memory_channels_list.push_back(memory_channels[r]); + } + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpyAsync( + device_memory_handle.data(), + memory_channel_handlers.data(), + memory_channel_handlers.size(), + stream, + cudaMemcpyHostToDevice); + } +}; + +class Msccl1NodeLLcontext { + private: + std::shared_ptr comm_group_ = nullptr; + void* scratch_; + const size_t scratch_bytes_; + std::unordered_map> same_node_connections_; + std::unordered_map> cross_node_connections_; + + std::unordered_map registered_sm_memories_; + std::unordered_map> memory_semaphores_; + std::unordered_map memory_channels_; + mscclpp::GpuBuffer d_memHandles_; + std::unordered_map> input_ptr2memory_channels_; + std::unordered_map> input_ptr2d_memHandles_; + cudaStream_t h2d_stream; + const size_t nranks_per_node_; + + public: + Msccl1NodeLLcontext( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + void* scratch, + const size_t scratch_bytes, + const size_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : scratch_(scratch), + scratch_bytes_(scratch_bytes), + nranks_per_node_(nranks_per_node), + d_memHandles_(nranks_per_node - 1) { + CHECK_CUDA_SUCCESS(cudaStreamCreateWithFlags(&h2d_stream, cudaStreamNonBlocking)); + comm_group_ = std::make_shared(unique_id, rank, world_size, rank_to_node, rank_to_ib); + comm_group_->make_connection(same_node_connections_, cross_node_connections_); + comm_group_->make_memory_channels_with_scratch( + scratch_, + scratch_bytes_, + scratch_, + scratch_bytes_, + same_node_connections_, + memory_semaphores_, + registered_sm_memories_, + memory_channels_); + std::vector memory_channels_list; + for (int r = 0; r < comm_group_->world_size_; r++) { + if (r == comm_group_->rank_) continue; + memory_channels_list.push_back(memory_channels_[r]); + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_memHandles_.data(), memory_channel_handlers.data(), memory_channel_handlers.size(), cudaMemcpyHostToDevice); + } + + ~Msccl1NodeLLcontext() { + CHECK_CUDA_SUCCESS(cudaStreamDestroy(h2d_stream)); + } + + template + void allreduce(cudaStream_t stream, T* input, T* output, size_t input_numel, int nthreads = 512, int nblocks = 21) { + dim3 nthrs(nthreads); + dim3 nblks(nblocks); + cudaStreamCaptureStatus capturing_status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &capturing_status)); + mscclpp::MemoryChannelDeviceHandle* memChans; + if (capturing_status != cudaStreamCaptureStatusActive) { + std::unordered_map memory_channels; + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + d_memHandles_, + input, + scratch_, + h2d_stream); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(h2d_stream)); + memChans = d_memHandles_.data(); + } else { + void* input_void_ptr = reinterpret_cast(input); + if (input_ptr2d_memHandles_.find(input_void_ptr) == input_ptr2d_memHandles_.end()) { + std::unordered_map memory_channels; + mscclpp::GpuBuffer device_memory_handle(comm_group_->world_size_ - 1); + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + device_memory_handle, + input, + scratch_, + h2d_stream); + input_ptr2memory_channels_.emplace(input_void_ptr, memory_channels); + input_ptr2d_memHandles_.emplace(input_void_ptr, device_memory_handle); + } + auto it = input_ptr2d_memHandles_.find(input_void_ptr); + memChans = it->second.data(); + } + allreduce_LL_1node<<>>( + memChans, (T*)input, (T*)scratch_, output, comm_group_->rank_, comm_group_->world_size_, input_numel); + + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) { + printf("rank: %lu failed to launch allreduce_LL_1node: %s\n", comm_group_->rank_, cudaGetErrorString(status)); + } + } +}; + +class Msccl2NodeLLcontext { + private: + std::shared_ptr comm_group_ = nullptr; + void* scratch_; + const size_t scratch_bytes_; + void* put_buffer_; + const size_t put_buffer_bytes_; + std::unordered_map> same_node_connections_; + std::unordered_map> cross_node_connections_; + + std::unordered_map registered_sm_memories_; + std::unordered_map registered_port_memories_; + + std::unordered_map> memory_semaphores_; + std::unordered_map> port_semaphores_; + + std::unordered_map memory_channels_; + std::unordered_map port_channels_; + + mscclpp::GpuBuffer d_memHandles_; + mscclpp::GpuBuffer d_portHandles_; + + std::shared_ptr proxyService; + cudaStream_t h2d_stream; + const size_t nranks_per_node_; + + std::unordered_map> input_ptr2memory_channels_; + std::unordered_map> input_ptr2d_memHandles_; + + public: + Msccl2NodeLLcontext( + mscclpp::UniqueId unique_id, + const size_t rank, + const size_t world_size, + void* scratch, + const size_t scratch_bytes, + void* put_buffer, + const size_t put_buffer_bytes, + const size_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib) + : scratch_(scratch), + scratch_bytes_(scratch_bytes), + put_buffer_(put_buffer), + put_buffer_bytes_(put_buffer_bytes), + nranks_per_node_(nranks_per_node), + d_memHandles_(nranks_per_node - 1), + d_portHandles_(world_size - nranks_per_node) { + CHECK_CUDA_SUCCESS(cudaStreamCreateWithFlags(&h2d_stream, cudaStreamNonBlocking)); + comm_group_ = std::make_shared(unique_id, rank, world_size, rank_to_node, rank_to_ib); + proxyService = std::make_shared(); + proxyService->startProxy(); + comm_group_->make_connection(same_node_connections_, cross_node_connections_); + comm_group_->make_memory_channels_with_scratch( + scratch_, + scratch_bytes_, + scratch_, + scratch_bytes_, + same_node_connections_, + memory_semaphores_, + registered_sm_memories_, + memory_channels_); + comm_group_->make_port_channels_with_scratch( + proxyService, + put_buffer_, + put_buffer_bytes_, + scratch_, + scratch_bytes_, + cross_node_connections_, + port_semaphores_, + registered_port_memories_, + port_channels_); + std::vector memory_channels_list; + std::vector port_channels_list; + for (int r = 0; r < comm_group_->world_size_; r++) { + if (r == comm_group_->rank_) continue; + if (comm_group_->is_same_node(r, comm_group_->rank_)) { + memory_channels_list.push_back(memory_channels_[r]); + } else { + port_channels_list.push_back(port_channels_[r]); + } + } + std::vector memory_channel_handlers(memory_channels_list.size()); + std::transform( + memory_channels_list.begin(), + memory_channels_list.end(), + memory_channel_handlers.begin(), + [](const mscclpp::MemoryChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_memHandles_.data(), memory_channel_handlers.data(), memory_channel_handlers.size(), cudaMemcpyHostToDevice); + + std::vector port_channel_handlers(port_channels_list.size()); + std::transform( + port_channels_list.begin(), + port_channels_list.end(), + port_channel_handlers.begin(), + [](const mscclpp::PortChannel& channel) { return channel.deviceHandle(); }); + mscclpp::gpuMemcpy( + d_portHandles_.data(), port_channel_handlers.data(), port_channel_handlers.size(), cudaMemcpyHostToDevice); + } + + ~Msccl2NodeLLcontext() { + CHECK_CUDA_SUCCESS(cudaStreamDestroy(h2d_stream)); + if (proxyService) { + proxyService->stopProxy(); + } + } + + template + void + allreduce(cudaStream_t stream, T* input, T* output, const size_t input_numel, int nthreads = 512, int nblocks = 21) { + dim3 nthrs(nthreads); + dim3 nblks(nblocks); + cudaStreamCaptureStatus capturing_status; + CHECK_CUDA_SUCCESS(cudaStreamIsCapturing(stream, &capturing_status)); + mscclpp::MemoryChannelDeviceHandle* memChans; + if (capturing_status != cudaStreamCaptureStatusActive) { + std::unordered_map memory_channels; + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + d_memHandles_, + input, + scratch_, + h2d_stream); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(h2d_stream)); + memChans = d_memHandles_.data(); + } else { + void* input_void_ptr = reinterpret_cast(input); + if (input_ptr2d_memHandles_.find(input_void_ptr) == input_ptr2d_memHandles_.end()) { + std::unordered_map memory_channels; + mscclpp::GpuBuffer device_memory_handle(7); + comm_group_->make_device_memory_handle_base_on_new_ptr( + memory_channels_, + registered_sm_memories_, + memory_semaphores_, + memory_channels, + device_memory_handle, + input, + scratch_, + h2d_stream); + input_ptr2memory_channels_.emplace(input_void_ptr, memory_channels); + input_ptr2d_memHandles_.emplace(input_void_ptr, device_memory_handle); + } + auto it = input_ptr2d_memHandles_.find(input_void_ptr); + memChans = it->second.data(); + } + allreduce_LL_2node<<>>( + memChans, + d_portHandles_.data(), + (T*)input, + (T*)scratch_, + (T*)put_buffer_, + output, + comm_group_->rank_, + nranks_per_node_, + comm_group_->world_size_, + input_numel); + + cudaError_t status = cudaGetLastError(); + if (status != cudaSuccess) { + printf("rank: %lu failed to launch allreduce_LL_2node: %s\n", comm_group_->rank_, cudaGetErrorString(status)); + } + } +}; + +} // namespace sglang diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce.cu b/sgl-kernel/csrc/allreduce/quick_all_reduce.cu new file mode 100644 index 000000000..757c05d2b --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce.cu @@ -0,0 +1,111 @@ +#include +#include +#include +#include + +#ifdef USE_ROCM + +#include "quick_all_reduce.h" + +quickreduce::fptr_t init_custom_qr(int64_t rank, int64_t world_size, std::optional qr_max_size) { + if (world_size > 8) throw std::invalid_argument("world size > 8 is not supported"); + if (world_size == 6) throw std::invalid_argument("world size == 6 is not supported"); + if (world_size % 2 != 0) throw std::invalid_argument("Odd num gpus is not supported for now"); + if (rank < 0 || rank >= world_size) throw std::invalid_argument("invalid rank passed in"); + quickreduce::DeviceComms* fptr = new quickreduce::DeviceComms(); + fptr->init(world_size, rank, qr_max_size); + return (quickreduce::fptr_t)fptr; +} + +void qr_destroy(quickreduce::fptr_t _fa) { + if (_fa) { + auto fa = reinterpret_cast(_fa); + fa->destroy(); + delete fa; + } +} + +torch::Tensor qr_get_handle(quickreduce::fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + hipIpcMemHandle_t handle = fa->get_handle(); + auto options = torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto data_handle = torch::empty({static_cast(sizeof(hipIpcMemHandle_t))}, options); + std::memcpy(data_handle.data_ptr(), &handle, sizeof(hipIpcMemHandle_t)); + return data_handle; +} + +void qr_open_handles(quickreduce::fptr_t _fa, const std::vector& handles) { + auto fa = reinterpret_cast(_fa); + std::vector ipc_handles; + ipc_handles.reserve(handles.size()); + for (auto& handle : handles) { + // Ensure the tensor is on the same device as the current device. + hipIpcMemHandle_t ipc_handle; + std::memcpy(&ipc_handle, handle.data_ptr(), sizeof(hipIpcMemHandle_t)); + ipc_handles.push_back(ipc_handle); + } + fa->open_ipc_handles(ipc_handles); +} + +void qr_all_reduce( + quickreduce::fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, int64_t quant_level, bool cast_bf2half) { + auto fa = reinterpret_cast(_fa); + const at::cuda::OptionalCUDAGuard device_guard(device_of(inp)); + auto stream = at::cuda::getCurrentHIPStreamMasqueradingAsCUDA(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK_LE(out.numel(), fa->kMaxProblemSize); + if (out.scalar_type() == at::ScalarType::Half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else if (out.scalar_type() == at::ScalarType::BFloat16) { + if (cast_bf2half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } + } else { + throw std::runtime_error("quick allreduce only supports float16 and bfloat16"); + } +} + +int64_t qr_max_size() { + // The default is 2GB (2,147,483,648 bytes) + return static_cast(std::numeric_limits::max()) + 1; +} + +#define INSTANTIATE_FOR_WORLDSIZE(T, Codec, cast_bf2half) \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; + +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, true) + +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ8, false) + +#endif // USE_ROCM diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh b/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh new file mode 100644 index 000000000..bd9e7b10f --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce.cuh @@ -0,0 +1,633 @@ +#pragma once + +#include + +#include "quick_all_reduce_base.h" + +namespace quickreduce { + +struct CodecBase { + const int thread; + const int rank; + const int group_leader; + __quickreduce_device_inline__ CodecBase(int thread, int rank) + : thread(thread), rank(rank), group_leader((threadIdx.x / kThreadGroupSize) * kThreadGroupSize) { + set_fp16_ovfl(true); + } +}; + +// Default full precision codec. +template +struct CodecFP : public CodecBase { + static constexpr int kWorldSize = world_size; + static constexpr int kRankAtoms = kAtoms / kWorldSize; + + // Codec tile size process by this workgroup. + // Each thread processes atoms of f16x8_t (16B). + static constexpr int kRankTransmittedTileSize = kBlockSize * kRankAtoms * sizeof(int32x4_t); + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + __quickreduce_device_inline__ CodecFP(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int i = 0; i < kRankAtoms; i++) { + __builtin_nontemporal_store(data[i], send_buffer + thread); + send_buffer += kAtomStride; + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int i = 0; i < kRankAtoms; i++) { + data[i] = __builtin_nontemporal_load(*recv_buffer + thread); + *recv_buffer += kAtomStride; + } + } +}; + +// Int4 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int4 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ4 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of fp16x8_t (16B), + // into a int4x8_t (4B) and a fp16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 1152; + static constexpr int kRankTileScaleOffset = 1024; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/8.0h, -1/8.0h}, f16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xB000B000 : 0xBE00BE00; + + // {1e-7, 1e-7}, f16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-8, -8}, f16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xC800C800 : 0xC100C100; + + // {+7, +7}, f16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x47004700 : 0x40E040E0; + + // {+8, +8}, int16x2_t + static constexpr int kRangeBias = 0x00080008; + + __quickreduce_device_inline__ CodecQ4(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q4 into int32_t + int qw = q[0] | (q[1] << 4) | (q[2] << 8) | (q[3] << 12); + + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + int32_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(qw, qw_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + int32_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + int32_t qw = __builtin_nontemporal_load(qw_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q4 into f16x8_t + int32x4_t w; + { + static constexpr uint kMask000F = 0x000F000F; + static constexpr uint kHalf2_1024 = 0x64006400; // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1032 = 0xE408E408; // {-1032.0, -1032.0}, fp16x2_t + + for (int i = 0; i < 4; i++) { + if constexpr (std::is_same::value) { + int32_t q4 = ((qw >> (i * 4)) & kMask000F) | kHalf2_1024; + w[i] = packed_add(q4, kHalf2_1032); + } else { + int32_t int16_2 = (qw >> (i * 4)) & kMask000F; + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + data[k] = w; + } + } +}; + +// Int6 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int6 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ6 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of fp16x8_t (16B), + // into a int6x8_t (4B + 2B) and a fp16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 1664; + static constexpr int kRankTileQ2Offset = 1024; + static constexpr int kRankTileScaleOffset = 1536; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTransmittedTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/32.0h, -1/32.0h}, fp16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xA800A800 : 0xBD00BD00; + + // {1e-7, 1e-7}, fp16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-32, -32}, fp16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xD000D000 : 0xC200C200; + + // {+31, +31}, fp16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x4FC04FC0 : 0x41F841F8; + + // {+32, +32}, int16x2_t + static constexpr int kRangeBias = 0x00200020; + + __quickreduce_device_inline__ CodecQ6(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, const int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q6 into int32_t + int16_t + uint32_t q4w; + uint16_t q2w = 0; + q4w = (q[0] & 0x000F000F) | ((q[1] & 0x000F000F) << 4) | ((q[2] & 0x000F000F) << 8) | ((q[3] & 0x000F000F) << 12); + { + int16_t* tw = reinterpret_cast(&q); +#pragma unroll + for (int i = 0; i < 8; i++) { + q2w |= (tw[i] >> 4) << (i * 2); + } + } + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + uint32_t* q4w_ptr = reinterpret_cast(atom_ptr) + thread; + uint16_t* q2w_ptr = reinterpret_cast(atom_ptr + kRankTileQ2Offset) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(q4w, q4w_ptr); + __builtin_nontemporal_store(q2w, q2w_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + uint32_t* q4w_ptr = reinterpret_cast(atom_ptr) + thread; + uint16_t* q2w_ptr = reinterpret_cast(atom_ptr + kRankTileQ2Offset) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + uint32_t q4w = __builtin_nontemporal_load(q4w_ptr); + uint16_t q2w = __builtin_nontemporal_load(q2w_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q6 into fp16x8_t + int32x4_t w; + { + static uint constexpr kMask000F = 0x000F000F; + static uint constexpr kHalf2_1024 = 0x64006400; // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1056 = 0xE420E420; // {-1056.0, -1056.0}, fp16x2_t + +#pragma unroll + for (int i = 0; i < 4; i++) { + int32_t q4 = q4w & kMask000F; + int32_t q2 = (q2w & 0x3) | ((q2w & 0xC) << 14); + q4w >>= 4; + q2w >>= 4; + if constexpr (std::is_same::value) { + int32_t q6 = q4 | (q2 << 4) | kHalf2_1024; + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(w[i]) : "v"(q6), "v"(kHalf2_1056)); + } else { + int32_t int16_2 = q4 | (q2 << 4); + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + // That's pretty much it... + data[k] = w; + } + } +}; + +// Int8 symmetric quantization codec. +// We quantize the FP16 data to block-scaled Int8 in blocks of 4 * +// kThreadGroupSize. +template +struct CodecQ8 : public CodecBase { + static constexpr int kWorldSize = world_size; + + // Codec tile size process by this workgroup. + // Each threads processes a fragment of f16x8_t (16B), + // into a int8x8_t (8B) and a f16 scale shared among 32 values. + static constexpr int kRankAtoms = kAtoms / kWorldSize; + static constexpr int kRankTileStride = 2176; + static constexpr int kRankTileScaleOffset = 2048; + static constexpr int kRankTransmittedTileSize = kRankTileStride * kRankAtoms; + static_assert(kRankTransmittedTileSize % 16 == 0, "kRankTileSize must be 16B aligned."); + + static constexpr int kRankBufferTileStride = kRankTileStride / sizeof(int32x4_t); + + // Total tile size for the collective communication. + static constexpr int kTransmittedTileSize = kRankTransmittedTileSize * kWorldSize; + + // Constants configuration + + // {-1/128.0h, -1/128.0h}, f16x2_t + static constexpr int kScaleFactor = std::is_same::value ? 0xA000A000 : 0xBC00BC00; + + // {1e-7, 1e-7}, f16x2_t + static constexpr int kScaleEpsilon = std::is_same::value ? 0x00010001 : 0x33D733D7; + + // {-128, -128}, f16x2_t + static constexpr int kRangeMin = std::is_same::value ? 0xD800D800 : 0xC300C300; + // {+127, +127}, f16x2_t + static constexpr int kRangeMax = std::is_same::value ? 0x57F057F0 : 0x42FE42FE; + + // {+128, +128}, int16x2_t + static constexpr int kRangeBias = 0x00800080; + + __quickreduce_device_inline__ CodecQ8(int thread, int rank) : CodecBase(thread, rank) {} + + __quickreduce_device_inline__ void send(int32x4_t* __restrict__ send_buffer, int32x4_t const* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + int32x4_t const atom = data[k]; + // Compute the absolute maximum of the atom in the thread group + // In 2 blocks of values, upper/lower halves of the f16x2_t + int wblockmax = group_abs_max(atom); + + // Derive scales + int decoding_scale; + int encoding_scale; + decoding_scale = packed_mul(wblockmax, kScaleFactor); + encoding_scale = packed_add(decoding_scale, kScaleEpsilon); + encoding_scale = packed_rcp(encoding_scale); + + // Apply scales to get quantized values + int32x4_t w; + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(atom[i], encoding_scale); + w[i] = packed_max(w[i], kRangeMin); + w[i] = packed_min(w[i], kRangeMax); + } + + // Convert from f16x2_t to uint16x2_t + int32x4_t q; + { + int16_t* qi = reinterpret_cast(&q); + T* wh = reinterpret_cast(&w); + for (int i = 0; i < 8; i++) + qi[i] = (int16_t)rintf(T2float_cast(wh[i])); + + for (int i = 0; i < 4; i++) { + q[i] = packed_add(q[i], kRangeBias); + } + } + + // Pack 8 x q8 into int32x2_t + int32x2_t qw; + qw[0] = q[0] | (q[1] << 8); + qw[1] = q[2] | (q[3] << 8); + + // Write quantized atom to send_buffer + // note: only the group leader stores the scale + uint8_t* atom_ptr = reinterpret_cast(send_buffer + k * kRankBufferTileStride); + int32x2_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + __builtin_nontemporal_store(qw, qw_ptr); + if (threadIdx.x == group_leader) { + __builtin_nontemporal_store(decoding_scale, qs_ptr); + } + } + } + + __quickreduce_device_inline__ void recv(int32x4_t** __restrict__ recv_buffer, int32x4_t* __restrict__ data) { + for (int k = 0; k < kRankAtoms; k++) { + // Directly read quantized atom from recv_buffer + uint8_t* atom_ptr = reinterpret_cast(*recv_buffer); + int32x2_t* qw_ptr = reinterpret_cast(atom_ptr) + thread; + int* qs_ptr = reinterpret_cast(atom_ptr + kRankTileScaleOffset) + (thread / 8); + + int32x2_t qw = __builtin_nontemporal_load(qw_ptr); + int qs = __builtin_nontemporal_load(qs_ptr); + + *recv_buffer += kRankBufferTileStride; + + // Unpack q8 into fp16x8_t + int32x4_t w; + { + static uint constexpr kMask00FF = 0x00FF00FF; + + // {1024.0, 1024.0}, fp16x2_t + static uint constexpr kHalf2_1024 = 0x64006400; + + // {-1152.0, -1152.0}, fp16x2_t + static uint constexpr kHalf2_1152 = 0xE480E480; + +#pragma unroll + for (int i = 0; i < 4; i++) { + if constexpr (std::is_same::value) { + int32_t q8 = ((qw[i / 2] >> ((i % 2) * 8)) & kMask00FF) | kHalf2_1024; + w[i] = packed_add(q8, kHalf2_1152); + } else { + int32_t int16_2 = (qw[i / 2] >> ((i % 2) * 8)) & kMask00FF; + int16_t low = static_cast(int16_2 & 0xFFFF); + int16_t high = static_cast((int16_2 >> 16) & 0xFFFF); + nv_bfloat16 bf_low = __float2bfloat16(static_cast(low)); + nv_bfloat16 bf_high = __float2bfloat16(static_cast(high)); + nv_bfloat162 bf2 = __halves2bfloat162(bf_low, bf_high); + int32_t packed_bf16 = *reinterpret_cast(&bf2); + w[i] = packed_add(packed_bf16, kRangeMin); + } + } + } + + // Apply decoding scales + for (int i = 0; i < 4; i++) { + w[i] = packed_mul(w[i], qs); + } + + data[k] = w; + } + } +}; + +// Twoshot All Reduce +template +struct AllReduceTwoshot { + static_assert(sizeof(T) == 2); + + static constexpr int kWorldSize = Codec::kWorldSize; + + __device__ static void + run(T const* __restrict__ input, + T* __restrict__ output, + uint32_t const N, // number of elements + int const block, // block index + int const rank, // rank index + uint8_t** __restrict__ buffer_list, // communication buffers + uint32_t const data_offset, // offset to start of the data buffer + uint32_t flag_color) { + // Topology + int thread = threadIdx.x + threadIdx.y * kWavefront; + uint8_t* rank_buffer = buffer_list[rank]; + Codec codec(thread, rank); + int block_id = blockIdx.x; + int grid_size = gridDim.x; + // -------------------------------------------------------- + // Read input into registers + int32x4_t tA[kAtoms]; + + BufferResource src_buffer(const_cast(input), N * sizeof(T)); + uint32_t src_offset = block * kTileSize + thread * sizeof(int32x4_t); + + for (int i = 0; i < kAtoms; i++) { + tA[i] = buffer_load_dwordx4(src_buffer.descriptor, src_offset, 0, 0); + src_offset += kAtomStride * sizeof(int32x4_t); + if constexpr (cast_bf2half) { + const nv_bfloat162* bf_buf = reinterpret_cast(&tA[i]); + half2 half_buf[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + float2 f = __bfloat1622float2(bf_buf[j]); + half_buf[j] = __float22half2_rn(f); + } + tA[i] = *reinterpret_cast(half_buf); + } + } + + // -------------------------------------------------------- + // Phase-1A: Write segment data into the communication buffer of the target + // rank responsible for this segment. + uint32_t comm_data0_offset = data_offset + block_id * Codec::kTransmittedTileSize; + uint32_t comm_data1_offset = grid_size * Codec::kTransmittedTileSize + comm_data0_offset; + + uint32_t comm_flags0_offset = block_id * (kWorldSize * sizeof(uint32_t)); + uint32_t comm_flags1_offset = grid_size * (kWorldSize * sizeof(uint32_t)) + comm_flags0_offset; + + for (int r = 0; r < kWorldSize; r++) { + int32x4_t* send_buffer = + reinterpret_cast(buffer_list[r] + comm_data0_offset + rank * Codec::kRankTransmittedTileSize); + codec.send(send_buffer, &tA[r * Codec::kRankAtoms]); + } + + __syncthreads(); + if (thread < kWorldSize) { + int r = thread; + uint32_t* flag_ptr = reinterpret_cast(buffer_list[r] + comm_flags0_offset + rank * sizeof(uint32_t)); + set_sync_flag(flag_ptr, flag_color); + } + // -------------------------------------------------------- + // Phase-1B: Reduce the segment data from the communication buffers. + int32x4_t tR[Codec::kRankAtoms] = {}; + { + // Read the data from the communication buffer. + int32x4_t* recv_buffer = reinterpret_cast(rank_buffer + comm_data0_offset); + uint32_t* flag_ptr = reinterpret_cast(rank_buffer + comm_flags0_offset); + + for (int r = 0; r < kWorldSize; r++) { + // Wait for the flags to be set. + if (thread == 0) { + wait_sync_flag(&flag_ptr[r], flag_color); + } + __syncthreads(); + + // note: we reuse tA as temp buffer here + codec.recv(&recv_buffer, tA); + + for (int i = 0; i < Codec::kRankAtoms; i++) { + packed_assign_add(&tR[i], &tA[i]); + } + } + } + + // Phase-2: Write the reduced segment to every other rank + for (int r = 0; r < kWorldSize; r++) { + int32x4_t* send_buffer = + reinterpret_cast(buffer_list[r] + comm_data1_offset + rank * Codec::kRankTransmittedTileSize); + codec.send(send_buffer, tR); + } + + __syncthreads(); + if (thread < kWorldSize) { + int r = thread; + uint32_t* flag_ptr = reinterpret_cast(buffer_list[r] + comm_flags1_offset + rank * sizeof(uint32_t)); + set_sync_flag(flag_ptr, flag_color); + } + + // Phase-2: Read the gather segments from the rank's communication buffer. + { + // Read the data from the communication buffer. + int32x4_t* recv_buffer = reinterpret_cast(rank_buffer + comm_data1_offset); + uint32_t* flag_ptr = reinterpret_cast(rank_buffer + comm_flags1_offset); + + for (int r = 0; r < kWorldSize; r++) { + // Wait for the flags to be set. + if (thread == 0) { + wait_sync_flag(&flag_ptr[r], flag_color); + } + __syncthreads(); + + // Gather all reduced and final rank segments into tA. + codec.recv(&recv_buffer, &tA[r * Codec::kRankAtoms]); + } + } + + // -------------------------------------------------------- + // Write the result to output. + BufferResource dst_buffer(output, N * sizeof(T)); + uint32_t dst_offset = block * kTileSize + thread * sizeof(int32x4_t); + + for (int i = 0; i < kAtoms; i++) { + if constexpr (cast_bf2half) { + const half2* half_buf = reinterpret_cast(&tA[i]); + nv_bfloat162 bf16_buf[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + float2 f = __half22float2(half_buf[j]); + bf16_buf[j] = __float22bfloat162_rn(f); + } + buffer_store_dwordx4(*reinterpret_cast(bf16_buf), dst_buffer.descriptor, dst_offset, 0, 0); + } else { + buffer_store_dwordx4(tA[i], dst_buffer.descriptor, dst_offset, 0, 0); + } + dst_offset += kAtomStride * sizeof(int32x4_t); + } + } +}; + +} // namespace quickreduce diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce.h b/sgl-kernel/csrc/allreduce/quick_all_reduce.h new file mode 100644 index 000000000..1d629e018 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce.h @@ -0,0 +1,233 @@ +#pragma once + +#include + +#include + +#include "quick_all_reduce.cuh" + +#define HIP_CHECK(err) \ + do { \ + hipError_t err_ = (err); \ + if (err_ != hipSuccess) { \ + std::printf("HIP error %d at %s:%d. %s\n", err_, __FILE__, __LINE__, hipGetErrorString(err_)); \ + throw std::runtime_error("HIP error"); \ + } \ + } while (0) + +namespace quickreduce { +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +template +__global__ __quickreduce_launch_bounds_two_shot__ static void allreduce_prototype_twoshot( + T const* A, + T* B, + uint32_t N, + uint32_t num_blocks, + int rank, + uint8_t** dbuffer_list, + uint32_t data_offset, + uint32_t flag_color) { + int block = blockIdx.x; + int grid = gridDim.x; + + while (block < num_blocks) { + AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset, flag_color); + block += grid; + flag_color++; + } +} + +#define TWOSHOT_DISPATCH(__codec) \ + if (world_size == 2) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } else if (world_size == 4) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } else if (world_size == 8) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } + +enum QuickReduceQuantLevel { + F16 = 0, + INT8 = 1, + INT6 = 2, + INT4 = 3, +}; + +struct DeviceComms { + // Max problem size is 2GB (in bytes) or half of uint32_t max value. + int64_t kMaxProblemSize = static_cast(std::numeric_limits::max()) + 1; + + // Max TP-8 + static int constexpr kMaxWorldSize = 8; + + bool initialized = false; + uint32_t flag_color = 1; + int world_size; + int rank; + + uint8_t* dbuffer; + uint8_t** dbuffer_list; + hipIpcMemHandle_t buffer_ipc_handle; + std::vector all_buffer_ipc_handles; + std::vector buffer_list; + uint32_t data_offset; + + DeviceComms() : initialized(false), world_size(1), rank(0) {} + ~DeviceComms() { + destroy(); + } + + void init(int world_size, int rank, std::optional max_problem_size = std::nullopt) { + destroy(); + this->world_size = world_size; + this->rank = rank; + if (max_problem_size.has_value() && max_problem_size.value() > 0) { + this->kMaxProblemSize = max_problem_size.value(); + } + // Allocate buffer size for worst case: F16 2-stage buffer. + uint32_t flags_buffer_size = 2 * world_size * kMaxNumBlocks * sizeof(uint32_t); + static int64_t data_buffer_size = 2 * this->kMaxProblemSize; + int64_t total_buffer_size = flags_buffer_size + data_buffer_size; + data_offset = flags_buffer_size; + HIP_CHECK(hipExtMallocWithFlags((void**)&dbuffer, total_buffer_size, hipDeviceMallocUncached)); + + // Clear the flags buffer. + HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size)); + + // Device-side list of IPC buffers. + buffer_list.resize(world_size); + HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*))); + + // Create IPC handles for rank's communication buffer. + all_buffer_ipc_handles.resize(world_size); + HIP_CHECK(hipIpcGetMemHandle(&buffer_ipc_handle, dbuffer)); + + initialized = true; + } + int get_world_size() { + return world_size; + } + int get_rank() { + return rank; + } + bool status() { + return initialized; + } + hipIpcMemHandle_t const get_handle() { + return buffer_ipc_handle; + } + + void destroy() { + if (initialized) { + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK(hipIpcCloseMemHandle(dbuffer_list[i])); + } + } + + HIP_CHECK(hipFree(dbuffer)); + HIP_CHECK(hipFree(dbuffer_list)); + + initialized = false; + } + } + + void open_ipc_handles(std::vector const& ipc_handles) { + assert(ipc_handles.size() == all_buffer_ipc_handles.size()); + for (int i = 0; i < world_size; i++) { + all_buffer_ipc_handles[i] = ipc_handles[i]; + } + + // Open device memory access to the IPC communication buffers. + // Note: For our own rank, we do not need to open a handle. + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK( + hipIpcOpenMemHandle((void**)&buffer_list[i], all_buffer_ipc_handles[i], hipIpcMemLazyEnablePeerAccess)); + } else { + buffer_list[i] = dbuffer; + } + } + + HIP_CHECK(hipMemcpy(dbuffer_list, buffer_list.data(), world_size * sizeof(uint8_t*), hipMemcpyHostToDevice)); + } + + template + void allreduce(T const* A, T* B, uint32_t N, int quant_level, hipStream_t stream) { + if (world_size != 2 && world_size != 4 && world_size != 8) { + throw std::runtime_error("All Reduce not supported for world_size = " + std::to_string(world_size)); + } + + // Configuration. + uint32_t msg_size = N * sizeof(T); + uint32_t num_blocks = divceil(msg_size, kTileSize); + uint32_t grid = min(kMaxNumBlocks, num_blocks); + auto quant_level_ = static_cast(quant_level); + switch (quant_level_) { + case QuickReduceQuantLevel::INT8: + TWOSHOT_DISPATCH(CodecQ8) + break; + case QuickReduceQuantLevel::INT6: + TWOSHOT_DISPATCH(CodecQ6) + break; + case QuickReduceQuantLevel::INT4: + TWOSHOT_DISPATCH(CodecQ4) + break; + default: + TWOSHOT_DISPATCH(CodecFP) + break; + } + HIP_CHECK(cudaGetLastError()); + // Rotate the flag color. + flag_color += divceil(N, grid); + } +}; + +} // namespace quickreduce diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce.hip b/sgl-kernel/csrc/allreduce/quick_all_reduce.hip new file mode 100644 index 000000000..9934383e1 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce.hip @@ -0,0 +1,113 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include +#include +#include +#include + +#ifdef USE_ROCM + +#include "quick_all_reduce_hip.h" + +quickreduce::fptr_t init_custom_qr(int64_t rank, int64_t world_size, std::optional qr_max_size) { + if (world_size > 8) throw std::invalid_argument("world size > 8 is not supported"); + if (world_size == 6) throw std::invalid_argument("world size == 6 is not supported"); + if (world_size % 2 != 0) throw std::invalid_argument("Odd num gpus is not supported for now"); + if (rank < 0 || rank >= world_size) throw std::invalid_argument("invalid rank passed in"); + quickreduce::DeviceComms* fptr = new quickreduce::DeviceComms(); + fptr->init(world_size, rank, qr_max_size); + return (quickreduce::fptr_t)fptr; +} + +void qr_destroy(quickreduce::fptr_t _fa) { + if (_fa) { + auto fa = reinterpret_cast(_fa); + fa->destroy(); + delete fa; + } +} + +torch::Tensor qr_get_handle(quickreduce::fptr_t _fa) { + auto fa = reinterpret_cast(_fa); + hipIpcMemHandle_t handle = fa->get_handle(); + auto options = torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCPU); + auto data_handle = torch::empty({static_cast(sizeof(hipIpcMemHandle_t))}, options); + std::memcpy(data_handle.data_ptr(), &handle, sizeof(hipIpcMemHandle_t)); + return data_handle; +} + +void qr_open_handles(quickreduce::fptr_t _fa, const std::vector& handles) { + auto fa = reinterpret_cast(_fa); + std::vector ipc_handles; + ipc_handles.reserve(handles.size()); + for (auto& handle : handles) { + // Ensure the tensor is on the same device as the current device. + hipIpcMemHandle_t ipc_handle; + std::memcpy(&ipc_handle, handle.data_ptr(), sizeof(hipIpcMemHandle_t)); + ipc_handles.push_back(ipc_handle); + } + fa->open_ipc_handles(ipc_handles); +} + +void qr_all_reduce( + quickreduce::fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, int64_t quant_level, bool cast_bf2half) { + auto fa = reinterpret_cast(_fa); + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(inp)); + auto stream = at::cuda::getCurrentHIPStreamMasqueradingAsCUDA(); + + TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type()); + TORCH_CHECK_EQ(inp.numel(), out.numel()); + TORCH_CHECK_LE(out.numel(), fa->kMaxProblemSize); + if (out.scalar_type() == at::ScalarType::Half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else if (out.scalar_type() == at::ScalarType::BFloat16) { + if (cast_bf2half) { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } else { + fa->allreduce( + reinterpret_cast(inp.data_ptr()), + reinterpret_cast(out.data_ptr()), + out.numel(), + quant_level, + stream); + } + } else { + throw std::runtime_error("quick allreduce only supports float16 and bfloat16"); + } +} + +int64_t qr_max_size() { + // The default is 2GB (2,147,483,648 bytes) + return static_cast(std::numeric_limits::max()) + 1; +} + +#define INSTANTIATE_FOR_WORLDSIZE(T, Codec, cast_bf2half) \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; \ + template struct quickreduce::AllReduceTwoshot, cast_bf2half>; + +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, false) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, true) +INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, true) + +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecFP, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ4, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ6, false) +INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ8, false) + +#endif // USE_ROCM diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h b/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h new file mode 100644 index 000000000..759b28f38 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce_base.h @@ -0,0 +1,318 @@ +#pragma once + +#include +#include +#include + +#include + +#define __quickreduce_device_inline__ __device__ __forceinline__ +#define __quickreduce_launch_bounds_two_shot__ __launch_bounds__(256, 4) +#define __quickreduce_launch_bounds_one_shot__ __launch_bounds__(512, 4) + +namespace quickreduce { + +typedef __hip_bfloat16 nv_bfloat16; +typedef __hip_bfloat162 nv_bfloat162; + +using int32x2_t = __attribute__((__vector_size__(2 * sizeof(int)))) int; +using int32x4_t = __attribute__((__vector_size__(4 * sizeof(int)))) int; + +// Setup acquire-release semantics for vector memory reads (mubuf instruction) +// as per architecture. +#if defined(__gfx942__) +// CDNA3: Scope bits sc0, sc1 +#define MUBUF_ACQUIRE 16 +#define MUBUF_RELEASE 16 +#elif (defined(__gfx908__) || defined(__gfx90a__)) +// CDNA1 and CDNA2 - glc bit +#define MUBUF_ACQUIRE 1 +#define MUBUF_RELEASE 0 +#endif + +static constexpr int kNegOne = 0xBC00BC00; // {-1, -1}, fp16x2_t + +// Number of atoms (4xf16x2_t) processed by a single thread +static constexpr int kAtoms = 8; + +// We use a workgroup of 256 threads +static constexpr int kBlockSize = 256; +static constexpr int kAtomStride = kBlockSize; + +// Size and atom stride of source/destination data that the block will +// process. +// Workgroup scope = Tile = (256 threads x 8 atoms x 16B) +static constexpr int kTileSize = kBlockSize * kAtoms * sizeof(int32x4_t); + +// Max number of blocks. 304 CUs on MI300 +static constexpr int kMaxNumBlocks = 304 * 4; + +// Standard CDNA wavefront size. +static constexpr int kWavefront = 64; + +// 256 thread, 4 wavefronts. +static dim3 constexpr kBlockTwoShot = {kWavefront, kBlockSize / kWavefront, 1}; + +// Number of threads in a group for quantization +// It corresponds to 32 F16 elements in quantization block +static constexpr int kThreadGroupSize = 8; + +// Methods +__quickreduce_device_inline__ __host__ unsigned long divceil(unsigned long x, unsigned long y) { + return ((x + y - 1) / y); +} + +union BufferResource { + __quickreduce_device_inline__ constexpr BufferResource() : config(0x00020000U) {} + + __quickreduce_device_inline__ constexpr BufferResource(void* buffer_address, uint32_t buffer_size) + : address(buffer_address), range(buffer_size), config(0x00020000U) {} + + int32x4_t descriptor; + struct { + void* address; // 8B, out of which first 48b is address, and 16b is stride + // (unused) + uint32_t range; // Byte range for the buffer resource + uint32_t config; // Constant, DFMT=32b + }; +}; + +__quickreduce_device_inline__ static int32x4_t buffer_load_dwordx4( + int32x4_t srsrc, int32_t voffset, int32_t soffset, int32_t aux) __asm("llvm.amdgcn.raw.buffer.load.v4i32"); + +__quickreduce_device_inline__ static void +buffer_store_dwordx4(int32x4_t data, int32x4_t srsrc, int32_t voffset, int32_t soffset, int32_t aux) __asm( + "llvm.amdgcn.raw.buffer.store.v4i32"); + +__quickreduce_device_inline__ static void set_fp16_ovfl(bool const value) { +#if defined(__gfx942__) + if (value) { + asm volatile("s_setreg_imm32_b32 0xdc1, 1;" ::); + } else { + asm volatile("s_setreg_imm32_b32 0xdc1, 0;" ::); + } +#endif +} +union bf162_int_union { + int i; + nv_bfloat162 bf2; +}; + +template +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B); + +template <> +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B) { + int32x4_t& tR_fragment = A[0]; + int32x4_t& tA_fragment = B[0]; + + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[0]) : "v"(tR_fragment[0]), "v"(tA_fragment[0])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[1]) : "v"(tR_fragment[1]), "v"(tA_fragment[1])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[2]) : "v"(tR_fragment[2]), "v"(tA_fragment[2])); + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(tR_fragment[3]) : "v"(tR_fragment[3]), "v"(tA_fragment[3])); +} + +template <> +__quickreduce_device_inline__ void packed_assign_add(int32x4_t* A, int32x4_t* B) { + nv_bfloat162* tA = reinterpret_cast(A); + nv_bfloat162* tB = reinterpret_cast(B); +#pragma unroll + for (int i = 0; i < 4; i++) { + tA[i] = __hadd2(tA[i], tB[i]); + } +} + +template +__quickreduce_device_inline__ int packed_max(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_max(int a, int b) { + int result; + asm volatile("v_pk_max_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_max(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hmax2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_min(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_min(int a, int b) { + int result; + asm volatile("v_pk_min_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_min(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hmin2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_abs_max(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_abs_max(int a, int b) { + half2 wmaxh2 = __builtin_bit_cast(half2, a); + half2 wminh2 = __builtin_bit_cast(half2, b); + half2 wblockmaxh2; + + wblockmaxh2.x = __hgt(__habs(wmaxh2.x), __habs(wminh2.x)) ? wmaxh2.x : wminh2.x; + wblockmaxh2.y = __hgt(__habs(wmaxh2.y), __habs(wminh2.y)) ? wmaxh2.y : wminh2.y; + return __builtin_bit_cast(int, wblockmaxh2); +} + +template <> +__quickreduce_device_inline__ int packed_abs_max(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2.x = __hgt(__habs(A.bf2.x), __habs(B.bf2.x)) ? A.bf2.x : B.bf2.x; + R.bf2.y = __hgt(__habs(A.bf2.y), __habs(B.bf2.y)) ? A.bf2.y : B.bf2.y; + return R.i; +} + +template +__quickreduce_device_inline__ int packed_add(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + int result; + asm volatile("v_pk_add_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hadd2(A.bf2, B.bf2); + return R.i; +} + +template <> +__quickreduce_device_inline__ int packed_add(int a, int b) { + int result; + asm volatile("v_pk_add_i16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template +__quickreduce_device_inline__ int packed_sub(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_sub(int a, int b) { + int result; + + // MI300 lacks packed fp16 sub instruction. So we do -1 * min + max + asm volatile("v_pk_fma_f16 %0, %1, %2 %3" : "=v"(result) : "v"(kNegOne), "v"(b), "v"(a)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_sub(int a, int b) { + bf162_int_union A, B, R; + A.i = a; + B.i = b; + R.bf2 = __hsub2(A.bf2, B.bf2); + return R.i; +} + +template +__quickreduce_device_inline__ int packed_mul(int a, int b); + +template <> +__quickreduce_device_inline__ int packed_mul(int a, int b) { + int result; + asm volatile("v_pk_mul_f16 %0, %1, %2" : "=v"(result) : "v"(a), "v"(b)); + return result; +} + +template <> +__quickreduce_device_inline__ int packed_mul(int a, int b) { + nv_bfloat162* tA = reinterpret_cast(&a); + nv_bfloat162* tB = reinterpret_cast(&b); + nv_bfloat162 tR = __hmul2(*tA, *tB); + return *(reinterpret_cast(&tR)); +} + +template +__quickreduce_device_inline__ int packed_rcp(int a); + +template <> +__quickreduce_device_inline__ int packed_rcp(int a) { + return __builtin_bit_cast(int, h2rcp(__builtin_bit_cast(half2, a))); +} + +template <> +__quickreduce_device_inline__ int packed_rcp(int a) { + bf162_int_union A, R; + A.i = a; + R.bf2 = h2rcp(A.bf2); + return R.i; +} + +// changes dtype +__quickreduce_device_inline__ float T2float_cast(half a) { + return __half2float(a); +} + +__quickreduce_device_inline__ float T2float_cast(nv_bfloat16 a) { + return __bfloat162float(a); +} + +template +__quickreduce_device_inline__ int group_abs_max(int32x4_t atom) { + const int group_leader = (threadIdx.x / kThreadGroupSize) * kThreadGroupSize; + + int wmax, wmin, wblockmax; + int a, b; + a = packed_max(atom[0], atom[1]); + b = packed_max(atom[2], atom[3]); + + wmax = packed_max(a, b); + + a = packed_min(atom[0], atom[1]); + b = packed_min(atom[2], atom[3]); + + wmin = packed_min(a, b); + + // Reduce the max among a group of threads + // Note: This is basically 2 blocks of values setup as the + // upper/lower halves of the f16x2_t + for (int i = 1; i < kThreadGroupSize; i <<= 1) { + int x = __shfl_down(wmax, i); + wmax = packed_max(wmax, x); + + int y = __shfl_down(wmin, i); + wmin = packed_min(wmin, y); + } + wblockmax = packed_abs_max(wmax, wmin); + // Share with the cohort + wblockmax = __shfl(wblockmax, group_leader); + return wblockmax; +} + +__quickreduce_device_inline__ void set_sync_flag(uint32_t* flag_ptr, uint32_t flag) { + __atomic_store_n(flag_ptr, flag, __ATOMIC_RELEASE); +} + +__quickreduce_device_inline__ void wait_sync_flag(uint32_t* flag_ptr, uint32_t flag) { + while (__atomic_load_n(flag_ptr, __ATOMIC_RELAXED) != flag) { + } +} + +} // namespace quickreduce diff --git a/sgl-kernel/csrc/allreduce/quick_all_reduce_hip.h b/sgl-kernel/csrc/allreduce/quick_all_reduce_hip.h new file mode 100644 index 000000000..a0a5e3e54 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/quick_all_reduce_hip.h @@ -0,0 +1,235 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#pragma once + +#include + +#include + +#include "quick_all_reduce.cuh" + +#define HIP_CHECK(err) \ + do { \ + hipError_t err_ = (err); \ + if (err_ != hipSuccess) { \ + std::printf("HIP error %d at %s:%d. %s\n", err_, __FILE__, __LINE__, hipGetErrorString(err_)); \ + throw std::runtime_error("HIP error"); \ + } \ + } while (0) + +namespace quickreduce { +using fptr_t = int64_t; +static_assert(sizeof(void*) == sizeof(fptr_t)); + +template +__global__ __quickreduce_launch_bounds_two_shot__ static void allreduce_prototype_twoshot( + T const* A, + T* B, + uint32_t N, + uint32_t num_blocks, + int rank, + uint8_t** dbuffer_list, + uint32_t data_offset, + uint32_t flag_color) { + int block = blockIdx.x; + int grid = gridDim.x; + + while (block < num_blocks) { + AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset, flag_color); + block += grid; + flag_color++; + } +} + +#define TWOSHOT_DISPATCH(__codec) \ + if (world_size == 2) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } else if (world_size == 4) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } else if (world_size == 8) { \ + using LineCodec = __codec; \ + using AllReduceKernel = AllReduceTwoshot; \ + hipLaunchKernelGGL( \ + (allreduce_prototype_twoshot), \ + dim3(grid), \ + dim3(kBlockTwoShot), \ + 0, \ + stream, \ + A, \ + B, \ + N, \ + num_blocks, \ + rank, \ + dbuffer_list, \ + data_offset, \ + flag_color); \ + } + +enum QuickReduceQuantLevel { + F16 = 0, + INT8 = 1, + INT6 = 2, + INT4 = 3, +}; + +struct DeviceComms { + // Max problem size is 2GB (in bytes) or half of uint32_t max value. + int64_t kMaxProblemSize = static_cast(std::numeric_limits::max()) + 1; + + // Max TP-8 + static int constexpr kMaxWorldSize = 8; + + bool initialized = false; + uint32_t flag_color = 1; + int world_size; + int rank; + + uint8_t* dbuffer; + uint8_t** dbuffer_list; + hipIpcMemHandle_t buffer_ipc_handle; + std::vector all_buffer_ipc_handles; + std::vector buffer_list; + uint32_t data_offset; + + DeviceComms() : initialized(false), world_size(1), rank(0) {} + ~DeviceComms() { + destroy(); + } + + void init(int world_size, int rank, std::optional max_problem_size = std::nullopt) { + destroy(); + this->world_size = world_size; + this->rank = rank; + if (max_problem_size.has_value() && max_problem_size.value() > 0) { + this->kMaxProblemSize = max_problem_size.value(); + } + // Allocate buffer size for worst case: F16 2-stage buffer. + uint32_t flags_buffer_size = 2 * world_size * kMaxNumBlocks * sizeof(uint32_t); + static int64_t data_buffer_size = 2 * this->kMaxProblemSize; + int64_t total_buffer_size = flags_buffer_size + data_buffer_size; + data_offset = flags_buffer_size; + HIP_CHECK(hipExtMallocWithFlags((void**)&dbuffer, total_buffer_size, hipDeviceMallocUncached)); + + // Clear the flags buffer. + HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size)); + + // Device-side list of IPC buffers. + buffer_list.resize(world_size); + HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*))); + + // Create IPC handles for rank's communication buffer. + all_buffer_ipc_handles.resize(world_size); + HIP_CHECK(hipIpcGetMemHandle(&buffer_ipc_handle, dbuffer)); + + initialized = true; + } + int get_world_size() { + return world_size; + } + int get_rank() { + return rank; + } + bool status() { + return initialized; + } + hipIpcMemHandle_t const get_handle() { + return buffer_ipc_handle; + } + + void destroy() { + if (initialized) { + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK(hipIpcCloseMemHandle(dbuffer_list[i])); + } + } + + HIP_CHECK(hipFree(dbuffer)); + HIP_CHECK(hipFree(dbuffer_list)); + + initialized = false; + } + } + + void open_ipc_handles(std::vector const& ipc_handles) { + assert(ipc_handles.size() == all_buffer_ipc_handles.size()); + for (int i = 0; i < world_size; i++) { + all_buffer_ipc_handles[i] = ipc_handles[i]; + } + + // Open device memory access to the IPC communication buffers. + // Note: For our own rank, we do not need to open a handle. + for (int i = 0; i < world_size; i++) { + if (i != rank) { + HIP_CHECK( + hipIpcOpenMemHandle((void**)&buffer_list[i], all_buffer_ipc_handles[i], hipIpcMemLazyEnablePeerAccess)); + } else { + buffer_list[i] = dbuffer; + } + } + + HIP_CHECK(hipMemcpy(dbuffer_list, buffer_list.data(), world_size * sizeof(uint8_t*), hipMemcpyHostToDevice)); + } + + template + void allreduce(T const* A, T* B, uint32_t N, int quant_level, hipStream_t stream) { + if (world_size != 2 && world_size != 4 && world_size != 8) { + throw std::runtime_error("All Reduce not supported for world_size = " + std::to_string(world_size)); + } + + // Configuration. + uint32_t msg_size = N * sizeof(T); + uint32_t num_blocks = divceil(msg_size, kTileSize); + uint32_t grid = min(kMaxNumBlocks, num_blocks); + auto quant_level_ = static_cast(quant_level); + switch (quant_level_) { + case QuickReduceQuantLevel::INT8: + TWOSHOT_DISPATCH(CodecQ8) + break; + case QuickReduceQuantLevel::INT6: + TWOSHOT_DISPATCH(CodecQ6) + break; + case QuickReduceQuantLevel::INT4: + TWOSHOT_DISPATCH(CodecQ4) + break; + default: + TWOSHOT_DISPATCH(CodecFP) + break; + } + HIP_CHECK(hipGetLastError()); + // Rotate the flag color. + flag_color += divceil(N, grid); + } +}; + +} // namespace quickreduce diff --git a/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu b/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu new file mode 100644 index 000000000..4ca0c5739 --- /dev/null +++ b/sgl-kernel/csrc/allreduce/test_mscclpp_allreduce.cu @@ -0,0 +1,153 @@ +/* + * this file is used to test mscclpp_allreduce.cu using mpirun + * this file is adapted from https://github.com/flashinfer-ai/flashinfer/blob/v0.2.5/src/test_sum_all_reduce.cu +usage: +cd PATH-TO-THIS-FILE +export MPI_HOME=/usr/local/mpi +# export MPI_HOME=/opt/hpcx/ompi/ +export MSCCLPP_HOME=/workspace/test/mscclpp +nvcc -O2 -arch=native -std=c++17 test_mscclpp_allreduce.cu \ + -o test_mscclpp_allreduce -D_GLIBCXX_USE_CXX11_ABI=0 \ + -I${MSCCLPP_HOME}/include -L${MSCCLPP_HOME}/build -lmscclpp \ + -lnccl -I${MPI_HOME}/include -L${MPI_HOME}/lib -lmpi + +/opt/hpcx/ompi/bin/ +mpirun --allow-run-as-root -H 127.0.0.1:8 -np 8 \ + --map-by ppr:8:node \ + --mca btl_openib_warn_no_device_params_found 0 \ + --mca btl_tcp_if_include bond0 \ + --allow-run-as-root -np 8 \ + -x NCCL_RUNTIME_CONNECT=0 -x NCCL_IB_GID_INDEX=3 -x NCCL_DEBUG=WARN \ + -x LD_PRELOAD=${MSCCLPP_HOME}/build/libmscclpp.so ./test_mscclpp_allreduce + */ +#include +#include +#include +#include + +#ifndef CHECK_CUDA_SUCCESS +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, cudaGetErrorString(e)); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) +#endif + +#include + +#include "mscclpp_allreduce.cuh" + +template +bool isclose(T a, T b, float rtol = 1e-5, float atol = 1e-8) { + return fabs(a - b) <= (atol + rtol * fabs(b)); +} + +int main(int argc, char* argv[]) { + // init mpi + MPI_Init(&argc, &argv); + printf("MPI Initialized.\n"); + int nranks, rank; + + // get work size and rank id + MPI_Comm_size(MPI_COMM_WORLD, &nranks); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + cudaSetDevice(rank); + printf("nranks: %d, rank: %d\n", nranks, rank); + + // init host and device buffers + using T = float; + using ReduceT = float; + const size_t num_elems = 2 * 1024 * 1024; + std::vector host_buf(num_elems); + for (uint32_t i = 0; i < num_elems; ++i) { + host_buf[i] = T(i + rank); + } + thrust::device_vector device_buf(host_buf); + const size_t buf_size_in_bytes = num_elems * sizeof(T); + std::vector host_result_buf(num_elems); + thrust::device_vector device_result_buf(host_result_buf); + + std::vector host_scratch_buf(num_elems * 8); + for (uint32_t i = 0; i < num_elems; ++i) { + host_scratch_buf[i] = 1; + } + thrust::device_vector device_scratch_buf(host_scratch_buf); + std::vector host_put_buf(num_elems); + thrust::device_vector device_put_buf(host_put_buf); + + mscclpp::UniqueId unique_id; + if (rank == 0) unique_id = mscclpp::TcpBootstrap::createUniqueId(); + MPI_Bcast(&unique_id, sizeof(unique_id), MPI_BYTE, 0, MPI_COMM_WORLD); + + std::vector rank_to_node(nranks); + std::vector rank_to_ib(nranks); + for (int i = 0; i < nranks; i++) { + rank_to_node[i] = i / 8; + rank_to_ib[i] = i % 8; + } + + cudaStream_t s; + CHECK_CUDA_SUCCESS(cudaStreamCreate(&s)); + CHECK_CUDA_SUCCESS(cudaStreamSynchronize(s)); + if (nranks == 8) { + auto context = std::make_shared( + unique_id, + rank, + nranks, + thrust::raw_pointer_cast(device_scratch_buf.data()), + buf_size_in_bytes * 8, + rank_to_node, + rank_to_ib); + printf("rank: %d, Msccl1NodeLLcontext setup.\n", rank); + MPI_Barrier(MPI_COMM_WORLD); + context->allreduce( + s, + thrust::raw_pointer_cast(device_buf.data()), + thrust::raw_pointer_cast(device_result_buf.data()), + device_buf.size()); + } else if (nranks == 16) { + // TODO: this branch is untested since there is something wrong with mpirun in my test machince + auto context = std::make_shared( + unique_id, + rank, + nranks, + thrust::raw_pointer_cast(device_scratch_buf.data()), + buf_size_in_bytes * 8, + thrust::raw_pointer_cast(device_put_buf.data()), + buf_size_in_bytes, + rank_to_node, + rank_to_ib); + printf("rank: %d, Msccl2NodeLLcontext setup.\n", rank); + MPI_Barrier(MPI_COMM_WORLD); + context->allreduce( + s, + thrust::raw_pointer_cast(device_buf.data()), + thrust::raw_pointer_cast(device_result_buf.data()), + device_buf.size()); + } + + // check result correctness + thrust::host_vector host_buf_result = device_result_buf; + size_t num_results_error_atol_1e_3_rtol_1e_3 = 0; + bool nan_detected = false; + + for (uint32_t i = 0; i < num_elems; ++i) { + T expected = T(i * nranks + (nranks - 1) * nranks / 2); + if (std::isnan(float(host_buf_result[i]))) { + nan_detected = true; + } + if (!isclose(float(host_buf_result[i]), float(expected), 1e-3, 1e-3)) { + num_results_error_atol_1e_3_rtol_1e_3++; + } + } + float result_accuracy = 1. - float(num_results_error_atol_1e_3_rtol_1e_3) / float(num_elems); + + printf("rank: %d, nan_detected: %d accuracy: %f\n", rank, nan_detected, result_accuracy); + + CHECK_CUDA_SUCCESS(cudaStreamDestroy(s)); + MPI_Finalize(); + return 0; +} diff --git a/sgl-kernel/csrc/attention/cascade.cu b/sgl-kernel/csrc/attention/cascade.cu new file mode 100644 index 000000000..9d49360dd --- /dev/null +++ b/sgl-kernel/csrc/attention/cascade.cu @@ -0,0 +1,55 @@ +// Adapted from +// https://github.com/flashinfer-ai/flashinfer/blob/55576c626421b5ee7e7ebe74afd26465c8ae863f/csrc/cascade.cu + +#include +#include + +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +void merge_state( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged) { + CHECK_INPUT(v_a); + CHECK_INPUT(s_a); + CHECK_INPUT(v_b); + CHECK_INPUT(s_b); + auto device = v_a.device(); + CHECK_EQ(s_a.device(), device); + CHECK_EQ(v_b.device(), device); + CHECK_EQ(s_b.device(), device); + CHECK_DIM(3, v_a); + CHECK_DIM(2, s_a); + CHECK_DIM(3, v_b); + CHECK_DIM(2, s_b); + CHECK_SHAPE(v_a, v_b); + CHECK_SHAPE(s_a, s_b); + CHECK_EQ(v_a.size(0), s_a.size(0)); + CHECK_EQ(v_a.size(1), s_b.size(1)); + unsigned int seq_len = v_a.size(0); + unsigned int num_heads = v_a.size(1); + unsigned int head_dim = v_a.size(2); + + const c10::cuda::OptionalCUDAGuard device_guard(v_a.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + + bool success = DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(v_a.scalar_type(), c_type, [&] { + cudaError_t status = MergeState( + static_cast(v_a.data_ptr()), + static_cast(s_a.data_ptr()), + static_cast(v_b.data_ptr()), + static_cast(s_b.data_ptr()), + static_cast(v_merged.data_ptr()), + static_cast(s_merged.data_ptr()), + seq_len, + num_heads, + head_dim, + stream); + TORCH_CHECK(status == cudaSuccess, "MergeState kernel launch failed: ", cudaGetErrorString(status)); + return true; + }); + + TORCH_CHECK(success, "MergeState kernel launch failed: unsupported data type"); +} diff --git a/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu b/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu new file mode 100644 index 000000000..a41779c1b --- /dev/null +++ b/sgl-kernel/csrc/attention/cutlass_mla_kernel.cu @@ -0,0 +1,274 @@ +/* +Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include "cutlass_sm100_mla/device/sm100_mla.hpp" +#include "cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp" +#include "utils.h" + +// clang-format off +#if !defined(CUDA_VERSION) || CUDA_VERSION < 12040 +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + int64_t num_kv_splits) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for cutlass_mla_decode"); +} +int64_t cutlass_mla_get_workspace_size(int64_t max_seq_len, int64_t num_batches, int64_t sm_count, int64_t num_kv_splits) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for cutlass_mla_get_workspace_size"); +} +#else + +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +using namespace cute; +using namespace cutlass::fmha::kernel; + +template +struct IsPersistent { + static const bool value = v; +}; + +template > +struct MlaSm100 { + using Element = T; + using ElementAcc = float; + using ElementOut = T; + + using TileShape = Shape<_128, _128, Shape<_512, _64>>; + using TileShapeH = cute::tuple_element_t<0, TileShape>; + using TileShapeD = cute::tuple_element_t<2, TileShape>; + + // H K (D_latent D_rope) B + using ProblemShape = cute::tuple; + + using StrideQ = cute::tuple; // H D B + using StrideK = cute::tuple; // K D B + using StrideO = StrideK; // H D B + using StrideLSE = cute::tuple<_1, int>; // H B + + using TileScheduler = + std::conditional_t; + + using FmhaKernel = cutlass::fmha::kernel::Sm100FmhaMlaKernelTmaWarpspecialized< + TileShape, + Element, + ElementAcc, + ElementOut, + ElementAcc, + TileScheduler, + /*kIsCpAsync=*/!IsPaged128>; + using Fmha = cutlass::fmha::device::MLA; +}; + +template +typename T::Fmha::Arguments args_from_options( + at::Tensor const& out, + at::Tensor const& q_nope, + at::Tensor const& q_pe, + at::Tensor const& kv_c_and_k_pe_cache, + at::Tensor const& seq_lens, + at::Tensor const& page_table, + double sm_scale, + int64_t num_kv_splits) { + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = q_nope.device().index(); + hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + + int batches = q_nope.size(0); + int page_count_per_seq = page_table.size(1); + int page_count_total = kv_c_and_k_pe_cache.size(0); + int page_size = kv_c_and_k_pe_cache.size(1); + int max_seq_len = page_size * page_count_per_seq; + using TileShapeH = typename T::TileShapeH; + using TileShapeD = typename T::TileShapeD; + auto problem_shape = cute::make_tuple(TileShapeH{}, max_seq_len, TileShapeD{}, batches); + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + float scale = float(sm_scale); + + using StrideQ = typename T::StrideQ; + using StrideK = typename T::StrideK; + using StrideO = typename T::StrideO; + using StrideLSE = typename T::StrideLSE; + + StrideQ stride_Q_nope = cute::make_tuple( + static_cast(q_nope.stride(1)), _1{}, static_cast(q_nope.stride(0))); + StrideQ stride_Q_pe = cute::make_tuple( + static_cast(q_pe.stride(1)), _1{}, static_cast(q_pe.stride(0))); + + StrideK stride_C = cute::make_tuple( + static_cast(0 + D_latent + D_rope), _1{}, static_cast(page_size * (D_latent + D_rope))); + StrideLSE stride_PT = cute::make_stride(_1{}, page_count_per_seq); + StrideLSE stride_LSE = cute::make_tuple(_1{}, 0 + H); + StrideO stride_O = cute::make_tuple(static_cast(0 + D_latent), _1{}, static_cast(0 + H * D_latent)); + + using Element = typename T::Element; + using ElementOut = typename T::ElementOut; + using ElementAcc = typename T::ElementAcc; + auto Q_nope_ptr = static_cast(q_nope.data_ptr()); + auto Q_pe_ptr = static_cast(q_pe.data_ptr()); + auto C_ptr = static_cast(kv_c_and_k_pe_cache.data_ptr()); + typename T::Fmha::Arguments arguments{ + problem_shape, + {scale, + Q_nope_ptr, + stride_Q_nope, + Q_pe_ptr, + stride_Q_pe, + C_ptr, + stride_C, + C_ptr + D_latent, + stride_C, + static_cast(seq_lens.data_ptr()), + static_cast(page_table.data_ptr()), + stride_PT, + page_count_total, + page_size}, + {static_cast(out.data_ptr()), stride_O, static_cast(nullptr), stride_LSE}, + hw_info, + // TODO(trevor-m): Change split_kv back to -1 when + // https://github.com/NVIDIA/cutlass/issues/2274 is fixed. Split_kv=1 will + // perform worse with larger context length and smaller batch sizes. + static_cast(num_kv_splits), // split_kv + nullptr, // is_var_split_kv + }; + // TODO(kaixih@nvidia): When split_kv=-1 and is_var_split_kv=false, we compute + // split_kv automatically based on batch size and sequence length to balance + // workload across available SMs. Consider using var_split_kv for manual + // control if needed. + T::Fmha::set_split_kv(arguments); + return arguments; +} + +template +void runMla( + at::Tensor const& out, + at::Tensor const& q_nope, + at::Tensor const& q_pe, + at::Tensor const& kv_c_and_k_pe_cache, + at::Tensor const& seq_lens, + at::Tensor const& page_table, + at::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits, + cudaStream_t stream) { + using MlaSm100Type = MlaSm100; + typename MlaSm100Type::Fmha fmha; + auto arguments = args_from_options(out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, sm_scale, num_kv_splits); + + CUTLASS_CHECK(fmha.can_implement(arguments)); + + CUTLASS_CHECK(fmha.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(fmha.run(arguments, workspace.data_ptr(), stream)); +} + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits) { + auto sm_version = getSMVersion(); + // On SM103a, half of the accuracy tests are failing. + TORCH_CHECK(sm_version == 100, "cutlass_mla_decode is only supported on compute capability 10.0, but found sm version ", sm_version); + + auto in_dtype = q_nope.dtype(); + at::cuda::CUDAGuard device_guard{(char)q_nope.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(q_nope.get_device()); + const int page_size = kv_c_and_k_pe_cache.size(1); + + // NOTE(alcanderian): IsPersistent has bug with manual split_kv. + // Kernel will hang if batch is too large with large num_kv_splits. (for example bs=8, num_kv_splits=8) + // Maybe per batch split kv will fix this. + DISPATCH_BOOL(page_size == 128, IsPaged128, [&] { + DISPATCH_BOOL(num_kv_splits <= 1, NotManualSplitKV, [&] { + if (in_dtype == at::ScalarType::Half) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else if (in_dtype == at::ScalarType::Float8_e4m3fn) { + runMla>( + out, q_nope, q_pe, kv_c_and_k_pe_cache, seq_lens, page_table, workspace, sm_scale, num_kv_splits, stream); + } else { + TORCH_CHECK(false, "Unsupported input data type of MLA"); + } + return true; + }); + return true; + }); +} + +int64_t cutlass_mla_get_workspace_size(int64_t max_seq_len, int64_t num_batches, int64_t sm_count, int64_t num_kv_splits) { + // Workspace size depends on ElementAcc and ElementLSE (same as ElementAcc) + // which are float, so Element type here doesn't matter. + using MlaSm100Type = MlaSm100; + + // Get split kv. Requires problem shape and sm_count only. + typename MlaSm100Type::Fmha::Arguments arguments; + using TileShapeH = typename MlaSm100Type::TileShapeH; + using TileShapeD = typename MlaSm100Type::TileShapeD; + arguments.problem_shape = + cute::make_tuple(TileShapeH{}, static_cast(max_seq_len), TileShapeD{}, static_cast(num_batches)); + // Assumes device 0 when getting sm_count. + arguments.hw_info.sm_count = + sm_count <= 0 ? cutlass::KernelHardwareInfo::query_device_multiprocessor_count(/*device_id=*/0) : sm_count; + arguments.split_kv = static_cast(num_kv_splits); + MlaSm100Type::Fmha::set_split_kv(arguments); + + return MlaSm100Type::Fmha::get_workspace_size(arguments); +} + +#endif +// clang-format on diff --git a/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp b/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp new file mode 100644 index 000000000..dd4ed231b --- /dev/null +++ b/sgl-kernel/csrc/attention/cutlass_sm100_mla/device/sm100_mla.hpp @@ -0,0 +1,358 @@ +/*************************************************************************************************** + * Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +/*! + \file + \brief An universal device layer for cutlass 3.x-style kernels. +*/ + +// clang-format off +#pragma once + +// common +#include "cutlass/cutlass.h" +#include "cutlass/device_kernel.h" + +#if !defined(__CUDACC_RTC__) +#include "cutlass/cluster_launch.hpp" +#include "cutlass/trace.h" +#endif // !defined(__CUDACC_RTC__) + +#include "../kernel/sm100_fmha_mla_tma_warpspecialized.hpp" +#include "../kernel/sm100_fmha_mla_reduction.hpp" + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::fmha::device { + +using namespace cute; +using namespace cutlass::fmha::kernel; + + +//////////////////////////////////////////////////////////////////////////////// +////////////////////////////// CUTLASS 3.x API ///////////////////////////////// +//////////////////////////////////////////////////////////////////////////////// + +template< + class Kernel_ +> +class MLA { +public: + + using Kernel = Kernel_; + + using ReductionKernel = cutlass::fmha::kernel::Sm100FmhaMlaReductionKernel< + typename Kernel::ElementOut, + typename Kernel::ElementAcc, + typename Kernel::ElementAcc, + Kernel::TileShapeH::value, + Kernel::TileShapeL::value, + 256 /*Max split*/ + >; + + /// Argument structure: User API + using KernelArguments = typename Kernel::Arguments; + using ReductionArguments = typename ReductionKernel::Arguments; + + using Arguments = KernelArguments; + + /// Argument structure: Kernel API + using KernelParams = typename Kernel::Params; + using ReductionParams = typename ReductionKernel::Params; + struct Params { + KernelParams fmha_params; + ReductionParams reduction_params; + }; + +private: + + /// Kernel API parameters object + Params params_; + + bool is_initialized(bool set = false) { + static bool initialized = false; + if (set) initialized = true; + return initialized; + } + + static ReductionArguments to_reduction_args(Arguments const& args) { + auto [H, K, D, B] = args.problem_shape; + return ReductionArguments{ + nullptr, args.epilogue.ptr_o, nullptr, args.epilogue.ptr_lse, + args.mainloop.softmax_scale, B, args.split_kv, K, args.mainloop.ptr_seq, + args.ptr_split_kv, Kernel::TileShapeS::value + }; + } + +public: + + /// Access the Params structure + Params const& params() const { + return params_; + } + + static void set_split_kv (KernelArguments& args) { + if (args.split_kv >= 1) return; + auto [H, K, D, B] = args.problem_shape; + int sm_count = args.hw_info.sm_count; + int max_splits = ceil_div(K, 128); + int sms_per_batch = max(1, sm_count / B); + int split_heur = min(max_splits, sms_per_batch); + int waves = ceil_div(B * split_heur, sm_count); + int k_waves = ceil_div(max_splits, split_heur); + int split_wave_aware = ceil_div(max_splits, k_waves); + args.split_kv = split_wave_aware; + } + + /// Determines whether the GEMM can execute the given problem. + static Status + can_implement(Arguments const& args) { + if (! Kernel::can_implement(args)) { + return Status::kInvalid; + } + if (! ReductionKernel::can_implement(to_reduction_args(args))) { + return Status::kInvalid; + } + return Status::kSuccess; + } + + /// Gets the workspace size + static size_t + get_workspace_size(Arguments const& args) { + size_t workspace_bytes = 0; + workspace_bytes += Kernel::get_workspace_size(args); + workspace_bytes += ReductionKernel::get_workspace_size(to_reduction_args(args)); + return workspace_bytes; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int /* smem_capacity */ = -1) { + CUTLASS_TRACE_HOST("MLA::maximum_active_blocks()"); + int max_active_blocks = -1; + int smem_size = Kernel::SharedStorageSize; + + // first, account for dynamic smem capacity if needed + cudaError_t result; + if (smem_size >= (48 << 10)) { + CUTLASS_TRACE_HOST(" Setting smem size to " << smem_size); + result = cudaFuncSetAttribute( + device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size); + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST( + " cudaFuncSetAttribute() returned error: " + << cudaGetErrorString(result)); + return -1; + } + } + + // query occupancy after setting smem size + result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, + device_kernel, + Kernel::MaxThreadsPerBlock, + smem_size); + + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error: " + << cudaGetErrorString(result)); + return -1; + } + + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + + /// Initializes GEMM state from arguments. + Status + initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("MLA::initialize() - workspace " + << workspace << ", stream: " << (stream ? "non-null" : "null")); + + // Initialize the workspace + Status status = Kernel::initialize_workspace(args, workspace, stream); + if (status != Status::kSuccess) { + return status; + } + status = ReductionKernel::initialize_workspace(to_reduction_args(args), workspace, stream); + if (status != Status::kSuccess) { + return status; + } + KernelParams kernel_params = Kernel::to_underlying_arguments(args, workspace); + + ReductionArguments reduction_args = to_reduction_args(args); + if (reduction_args.split_kv > 1) { + reduction_args.ptr_oaccum = kernel_params.epilogue.ptr_o_acc; + reduction_args.ptr_lseaccum = kernel_params.epilogue.ptr_lse_acc; + } + ReductionParams reduction_params = ReductionKernel::to_underlying_arguments(reduction_args, workspace); + // Initialize the Params structure + params_ = Params {kernel_params, reduction_params}; + + if (is_initialized()) return Status::kSuccess; + + // account for dynamic smem capacity if needed + // no dynamic smem is needed for reduction kernel + int smem_size = Kernel::SharedStorageSize; + if (smem_size >= (48 << 10)) { + CUTLASS_TRACE_HOST(" Setting smem size to " << smem_size); + cudaError_t result = cudaFuncSetAttribute( + device_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_size); + if (cudaSuccess != result) { + result = cudaGetLastError(); // to clear the error bit + CUTLASS_TRACE_HOST(" cudaFuncSetAttribute() returned error: " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + } + + is_initialized(true); + + return Status::kSuccess; + } + + /// Update API is preserved in 3.0, but does not guarantee a lightweight update of params. + Status + update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("MLA()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + if (workspace_bytes > 0 && nullptr == workspace) { + return Status::kErrorWorkspaceNull; + } + + auto fmha_params = Kernel::to_underlying_arguments(args, workspace); + + ReductionArguments reduction_args = to_reduction_args(args); + if (reduction_args.split_kv > 1) { + reduction_args.ptr_oaccum = fmha_params.epilogue.ptr_o_acc; + reduction_args.ptr_lseaccum = fmha_params.epilogue.ptr_lse_acc; + } + ReductionParams reduction_params = ReductionKernel::to_underlying_arguments(reduction_args, workspace); + // Initialize the Params structure + params_ = Params {fmha_params, reduction_params}; + + return Status::kSuccess; + } + + /// Primary run() entry point API that is static allowing users to create and manage their own params. + /// Supplied params struct must be construct by calling Kernel::to_underling_arguments() + static Status + run(Params& params, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("MLA::run()"); + dim3 const block = Kernel::get_block_shape(); + dim3 const grid = Kernel::get_grid_shape(params.fmha_params); + + // configure smem size and carveout + int smem_size = Kernel::SharedStorageSize; + + Status launch_result; + // Use extended launch API only for mainloops that use it + if constexpr(Kernel::ArchTag::kMinComputeCapability >= 90) { + dim3 cluster(cute::size<0>(typename Kernel::ClusterShape{}), + cute::size<1>(typename Kernel::ClusterShape{}), + cute::size<2>(typename Kernel::ClusterShape{})); + void const* kernel = (void const*) device_kernel; + void* kernel_params[] = {¶ms.fmha_params}; + launch_result = ClusterLauncher::launch(grid, cluster, block, smem_size, stream, kernel, kernel_params); + } + else { + launch_result = Status::kSuccess; + device_kernel<<>>(params.fmha_params); + } + + cudaError_t result = cudaGetLastError(); + if (cudaSuccess != result or Status::kSuccess != launch_result) { + //return Status::kSuccess; + CUTLASS_TRACE_HOST(" Kernel launch failed. Reason: " << result); + return Status::kErrorInternal; + } + if (params.reduction_params.split_kv > 1) { + // launch reduction kernel + dim3 const block = ReductionKernel::get_block_shape(); + dim3 const grid = ReductionKernel::get_grid_shape(params.reduction_params); + device_kernel<<>>(params.reduction_params); + cudaError_t result = cudaGetLastError(); + if (cudaSuccess == result) { + return Status::kSuccess; + } + else { + CUTLASS_TRACE_HOST(" Kernel launch failed. Reason: " << result); + return Status::kErrorInternal; + } + } + else { + return Status::kSuccess; + } + } + + // + // Non-static launch overloads that first create and set the internal params struct of this kernel handle. + // + + /// Launches the kernel after first constructing Params internal state from supplied arguments. + Status + run(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + if (Status::kSuccess == status) { + status = run(params_, stream); + } + return status; + } + + /// Launches the kernel after first constructing Params internal state from supplied arguments. + Status + operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + return run(args, workspace, stream); + } + + /// Overload that allows a user to re-launch the same kernel without updating internal params struct. + Status + run(cudaStream_t stream = nullptr) { + return run(params_, stream); + } + + /// Overload that allows a user to re-launch the same kernel without updating internal params struct. + Status + operator()(cudaStream_t stream = nullptr) { + return run(params_, stream); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::device + +//////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp new file mode 100644 index 000000000..b75870d0c --- /dev/null +++ b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_reduction.hpp @@ -0,0 +1,198 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" +#include "cutlass/arch/arch.h" +#include "cute/tensor.hpp" + +namespace cutlass::fmha::kernel { + +using namespace cute; +template< + class ElementOut, + class ElementAcc, + class ElementScale, + size_t kNumHeads, + size_t kHeadDimLatent, + int kMaxSplits +> +struct Sm100FmhaMlaReductionKernel { + + static const int SharedStorageSize = 0; + static const int MaxThreadsPerBlock = 128; + static const int MinBlocksPerMultiprocessor = 1; + + using ArchTag = cutlass::arch::Sm100; + + static_assert(kHeadDimLatent % MaxThreadsPerBlock == 0); + struct Arguments { + ElementAcc* ptr_oaccum = nullptr; + ElementOut* ptr_o = nullptr; + ElementAcc* ptr_lseaccum = nullptr; + ElementAcc* ptr_lse = nullptr; + ElementScale scale = 1.f; + int num_batches = 0; + int split_kv = -1; + int dim_k = -1; + int* ptr_seq = nullptr; + int* ptr_split_kv = nullptr; + int tile_shape_s = 128; + }; + using Params = Arguments; + + static Params to_underlying_arguments(Arguments const& args, void* workspace) { + return {args.ptr_oaccum, args.ptr_o, args.ptr_lseaccum, args.ptr_lse, + args.scale, args.num_batches, args.split_kv, args.dim_k, args.ptr_seq, + args.ptr_split_kv, args.tile_shape_s}; + } + + static size_t get_workspace_size(Arguments const& /*args*/) { + return 0; + } + + static Status initialize_workspace( + Arguments const& /*args*/, void* /*ws*/, cudaStream_t /*stream*/) { + return Status::kSuccess; + } + + static dim3 get_grid_shape(Params const& params) { + return dim3(kNumHeads, 1, params.num_batches); + } + + static dim3 get_block_shape() { + return dim3(MaxThreadsPerBlock, 1, 1); + } + + static bool can_implement(Arguments const& args) { + if (args.num_batches <= 0) return false; + if (args.split_kv <= 0) return false; + return true; + } + + CUTLASS_DEVICE void operator() (Params const& params, char* smem_raw) { + if (params.split_kv <= 1) return; + auto blk_coord = make_coord(blockIdx.x, _0{}, blockIdx.z); + + __shared__ ElementAcc sLseScale[kMaxSplits]; + const size_t offset_lseaccum = get<0>(blk_coord) + kNumHeads * params.split_kv * get<2>(blk_coord); + const size_t offset_lse = get<0>(blk_coord) + kNumHeads * get<2>(blk_coord); + + Tensor gLSEaccum = make_tensor(make_gmem_ptr(params.ptr_lseaccum + offset_lseaccum), + make_shape(params.split_kv), Stride>{}); + + Tensor gLSE = make_tensor(make_gmem_ptr(params.ptr_lse + offset_lse), + Shape<_1>{}, Stride<_1>{}); + + auto dim_k = params.ptr_seq == nullptr ? params.dim_k : params.ptr_seq[get<2>(blk_coord)]; + auto local_split_kv = params.ptr_split_kv == nullptr ? params.split_kv : params.ptr_split_kv[get<2>(blk_coord)]; + auto k_tile_total = ceil_div(dim_k, params.tile_shape_s); + auto k_tile_per_cta = ceil_div(k_tile_total, local_split_kv); + local_split_kv = ceil_div(k_tile_total, k_tile_per_cta); + + int warp_idx = cutlass::canonical_warp_idx_sync(); + if (warp_idx == 0) { + constexpr int kNLsePerThread = cute::ceil_div(kMaxSplits, 32); + + ElementAcc local_lse[kNLsePerThread]; + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + const int split = i * 32 + threadIdx.x; + local_lse[i] = split < local_split_kv ? gLSEaccum(split) : -std::numeric_limits::infinity(); + } + + ElementAcc lse_max = -std::numeric_limits::infinity(); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + lse_max = max(lse_max, local_lse[i]); + } + CUTLASS_PRAGMA_UNROLL + for (int offset = 16; offset >= 1; offset /= 2) { + lse_max = max(lse_max, __shfl_xor_sync(0xffffffff, lse_max, offset)); + } + lse_max = lse_max == -std::numeric_limits::infinity() ? 0.0f : lse_max; // In case all local LSEs are -inf + lse_max = __shfl_sync(0xffffffff, lse_max, 0); + + ElementAcc sum_lse = 0; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + sum_lse = sum_lse + expf(local_lse[i] - lse_max); + } + + CUTLASS_PRAGMA_UNROLL + for (int offset = 16; offset >= 1; offset /= 2) { + sum_lse = sum_lse + __shfl_xor_sync(0xffffffff, sum_lse, offset); + } + + sum_lse = __shfl_sync(0xffffffff, sum_lse, 0); + + ElementAcc global_lse = (sum_lse == 0.f || sum_lse != sum_lse) ? std::numeric_limits::infinity() : logf(sum_lse) + lse_max; + if (threadIdx.x == 0 and params.ptr_lse != nullptr) { + gLSE(0) = global_lse; + } + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kNLsePerThread; ++i) { + const int split = i * 32 + threadIdx.x; + if (split < local_split_kv) { + sLseScale[split] = expf(local_lse[i] - global_lse); + } + } + } + __syncthreads(); + + constexpr int Elements = kHeadDimLatent / MaxThreadsPerBlock; + const size_t offset_oaccum = kHeadDimLatent * params.split_kv * (get<0>(blk_coord) + kNumHeads * get<2>(blk_coord)); + Tensor gOaccum = make_tensor(make_gmem_ptr(params.ptr_oaccum + offset_oaccum), + Shape>{}, Stride<_1>{}); + ElementAcc local_val[Elements] = {0}; + for (int split = 0; split < local_split_kv; ++split) { + ElementAcc lse_scale = sLseScale[split]; + CUTLASS_PRAGMA_UNROLL + for(int i = 0; i < Elements; ++i) { + local_val[i] += lse_scale * gOaccum(threadIdx.x + MaxThreadsPerBlock * i); + } + gOaccum.data() = gOaccum.data() + kHeadDimLatent; + } + auto ptr_o_local = params.ptr_o + (get<0>(blk_coord) + get<2>(blk_coord) * kNumHeads) * kHeadDimLatent; + Tensor gO = make_tensor(make_gmem_ptr(ptr_o_local), Shape>{}, Stride<_1>{}); + + CUTLASS_PRAGMA_UNROLL + for(int i = 0; i < Elements; ++i) { + gO(threadIdx.x + MaxThreadsPerBlock * i) = static_cast(local_val[i]); + } + } +}; + +} // namespace cutlass::fmha::kernel diff --git a/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp new file mode 100644 index 000000000..9809db84e --- /dev/null +++ b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_fmha_mla_tma_warpspecialized.hpp @@ -0,0 +1,2018 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cute/arch/simd_sm100.hpp" + +#include "cutlass/arch/arch.h" +#include "cutlass/arch/memory_sm80.h" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" + +#include "gather_tensor.hpp" // from examples/common +#include "common/pow_2.hpp" + +namespace cutlass::fmha::kernel { + +using namespace cute; + +template< + class TileShape, + class Element_, + class ElementAcc_, + class ElementOut_, + class ElementLSE_, + class TileScheduler, +#ifdef CPASYNC + bool kIsCpAsync = true +#else + bool kIsCpAsync = false +#endif +> +struct Sm100FmhaMlaKernelTmaWarpspecialized { + + using Element = Element_; + using ElementAcc = ElementAcc_; + using ElementOut = ElementOut_; + using ElementLSE = ElementLSE_; + + // only 2Sm mode is supported + static const bool kIs2Sm = true; + static const int MaxThreadsPerBlock = 256; + static const int MinBlocksPerMultiprocessor = 1; + static const int TotalSNum = 2; + static const int TotalPNum = 2; + using ArchTag = cutlass::arch::Sm100; + + using ClusterShape = cute::conditional_t, Shape<_1, _1, _1>>; + + using TileShapeH = tuple_element_t<0, TileShape>; + using TileShapeS = tuple_element_t<1, TileShape>; + using TileShapeD = tuple_element_t<2, TileShape>; + + using TileShapeL = tuple_element_t<0, TileShapeD>; + using TileShapeR = tuple_element_t<1, TileShapeD>; + static_assert(TileShapeL{} % TileShapeR{} == 0, "Rope head dim must divide latent head dim"); + + using ProblemShape = Shape; + using TensorStride = Stride; + using TmemAllocator = cute::conditional_t; + + static_assert(TileShapeH{} == 128); + static const int kWarpsInN = kIs2Sm ? 2 : 1; + + static const int kNumComputeWarps = 4; + static const int kNumLoadWarps = kIsCpAsync ? 2 : 1; + + enum class WarpRole { + kMma = 0x1, kLoad = 0x2, kCompute = 0x3, kLoadPageTable = 0x4, kEmpty=0x0 + }; + + static const long long unsigned int kWarpAssignment = kIsCpAsync ? 0x4221'3333ull : 0x0021'3333ull; + + static CUTLASS_DEVICE WarpRole warp_idx_to_role(int warp_idx) { + return static_cast((kWarpAssignment >> (4 * warp_idx)) & 0xF); + } + + static const int Alignment = 128 / sizeof_bits_v; + static const int AlignmentOut = 128 / sizeof_bits_v; + + using TileShapeQK = Shape; + static const int StagesQK = 24 / sizeof(Element); // free parameter + static const int IterationsQKLatent = decltype(TileShapeL{} / get<2>(TileShapeQK{}))::value; + static const int IterationsQKRope = decltype(TileShapeR{} / get<2>(TileShapeQK{}))::value; + static const int IterationsQK = IterationsQKLatent + IterationsQKRope; + + using Schedule = cute::conditional_t; + using CollectiveMmaQK = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, + Element, TensorStride, Alignment, + Element, TensorStride, Alignment, + ElementAcc, + TileShapeQK, ClusterShape, cutlass::gemm::collective::StageCount, + Schedule>::CollectiveOp; + using TiledMmaQK = typename CollectiveMmaQK::TiledMma; + using CtaShapeQK = typename CollectiveMmaQK::CtaShape_MNK; + + // chosen for unified smem staging between K and V + using TileShapePV = Shape; + using TransposeTensorStride = decltype(select<1,0,2>(TensorStride{})); + static const int StagesPV = StagesQK; // not sure why, but must be at least two. check pipes + static const int IterationsPV_K = decltype(TileShapeS{} / get<2>(TileShapePV{}))::value; + static const int IterationsPV_N = decltype(TileShapeL{} / get<1>(TileShapePV{}))::value; + + using CollectiveMmaPV = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp, + Element, TensorStride, Alignment, + Element, TransposeTensorStride, Alignment, + ElementAcc, + TileShapePV, ClusterShape, cutlass::gemm::collective::StageCount, + Schedule>::CollectiveOp; + using CtaShapePV = typename CollectiveMmaPV::CtaShape_MNK; + static_assert(std::is_same_v); + + using TiledMmaPV = typename CollectiveMmaPV::TiledMma; + + using AtomThrShapeMNK = typename CollectiveMmaQK::AtomThrShapeMNK; + static_assert(typename CollectiveMmaQK::AtomThrShapeMNK{} == typename CollectiveMmaPV::AtomThrShapeMNK{}, "schedule must match"); + + static const int StagesPageTable = kIsCpAsync ? StagesPV : 1; + + // pipelines from load to mma, PipelineTmaUmmaAsync, stages tbd + // use expect_tx for Q load + using PipelineLoadQK = cute::conditional_t, PipelineTmaUmmaAsync>; + using PipelineLoadPV = PipelineLoadQK; + // pipeline from mma (Q@K) to softmax, PipelineUmmaAsync, 2 stages + using PipelineS = PipelineUmmaAsync; + // pipeline from softmax (P) to mma (bmm2), PipelineUmmaAsync, 2 stages + using PipelineP = PipelineUmmaConsumerAsync; + // pipeline from mma to softmax (for rescale), PipelineUmmaAsync, 1 stage + using PipelineO = PipelineUmmaAsync<1, AtomThrShapeMNK>; + + using PipelinePT = PipelineAsync; + + struct PipelineStorage { + alignas(16) typename PipelineLoadQK::SharedStorage load_qk; + alignas(16) typename PipelineS::SharedStorage mma_s; + alignas(16) typename PipelineP::SharedStorage p_mma; + alignas(16) typename PipelineO::SharedStorage mma_o; + alignas(16) typename PipelinePT::SharedStorage load_page_table; + }; + + template + static CUTE_DEVICE constexpr auto unstageSmemLayout(Layout const& layout, Stages stages = {}) { + return composition(layout, make_tuple(_, _, _, make_layout(stages))); + } + + using SmemLayoutQ = decltype(unstageSmemLayout(typename CollectiveMmaQK::SmemLayoutA{}, Int{})); + using SmemLayoutKC = typename CollectiveMmaQK::SmemLayoutB; + using SmemLayoutVC = typename CollectiveMmaPV::SmemLayoutB; + using SmemLayoutP = decltype(unstageSmemLayout(typename CollectiveMmaPV::SmemLayoutA{}, make_shape(Int{}, _2{}))); + + static const int kBytesLoadQ = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutQ{})) * cute::sizeof_bits_v); + static const int kBytesLoadKC = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutKC{})) * cute::sizeof_bits_v); + static const int kBytesLoadVC = size(AtomThrShapeMNK{}) * cutlass::bits_to_bytes(cosize(take<0,3>(SmemLayoutVC{})) * cute::sizeof_bits_v); + // pre-condition for overlapped smem staging + static_assert(kBytesLoadKC == kBytesLoadVC); + static_assert(StagesQK == StagesPV); + + static const int kTransactionsBytesLoadQK = kBytesLoadKC; + static const int kTransactionsBytesLoadExtraQ = kBytesLoadQ; + static const int kTransactionsBytesLoadPV = kBytesLoadVC; + + static const int kNamedBarrierExchange = (int) cutlass::arch::ReservedNamedBarriers::TransformBarrier; + // This Named Barrier is introduced to solve Q tile loading overwritten issue when enable persistent + // tile scheduler for FP8 MLA. + static const int kNamedBarrierEpilogue = (int) cutlass::arch::ReservedNamedBarriers::EpilogueBarrier; + // + static const int kNamedBarrierTmemDealloc = (int) cutlass::arch::ReservedNamedBarriers::TmemAllocBarrier; + + enum class TmemAllocation : uint32_t { + kSizeS = TileShapeS::value / kWarpsInN, + // Overall + kSizeO = TileShapeL::value / kWarpsInN, + // Between accumulators we loop over + kSizeAccO = decltype(get<1>(TileShapePV{}))::value / kWarpsInN, + kNumS = TotalSNum, + kNumP = TotalPNum, + kNumO = 1, + kS0 = 0, + kS1 = kS0 + kSizeS, + kO0 = kS1 + kSizeS, + kTotal = kO0 + kSizeO + }; + + static_assert(static_cast(TmemAllocation::kTotal) <= TmemAllocator::Sm100TmemCapacityColumns, "using too much tmem"); + + struct TensorStorage { + // to communicate max and row_sum + cute::array smem_exchange; + cute::array smem_page_table; + alignas(2048) cute::array> smem_q; + union { + alignas(2048) cute::array> smem_kc; + alignas(2048) cute::array> smem_vc; + }; + alignas(2048) cute::array> smem_p; + }; + + struct SharedStorage { + PipelineStorage pipelines; + TensorStorage tensors; + uint32_t tmem_base_ptr; + }; + + static const int SharedStorageSize = sizeof(SharedStorage); + static_assert(SharedStorageSize <= cutlass::arch::sm100_smem_capacity_bytes, "using too much smem"); + + struct MainloopArguments { + ElementAcc softmax_scale; + + // all tensors strides are (num_heads or seqlen, head_dim, batch) + // head_dim stride is always 1 + Element* ptr_q_latent; + TensorStride stride_q_latent; + Element* ptr_q_rope; + TensorStride stride_q_rope; + + Element* ptr_c_latent; + TensorStride stride_c_latent; + Element* ptr_k_rope; + TensorStride stride_k_rope; + + // for paged attention, we interpret what was previously [batch, seqlen] + // as [page_count, page_size], and index according to page_table + int* ptr_seq = nullptr; + int* ptr_page_table = nullptr; + // page table is [batch, seqlen or similar] + Stride<_1, int> stride_page_table = {}; + int page_count = 0; + int page_size = TileShapeS{}; // powers of two if kIsCpAsync, otherwise TileShapeS + }; + + struct EpilogueArguments { + ElementOut* ptr_o = nullptr; + TensorStride stride_o; + ElementLSE* ptr_lse = nullptr; + Stride<_1, int> stride_lse; + ElementAcc output_scale = 1.0f; + }; + + struct Arguments { + // (num_heads=128, seqlen, (d_latent=512, d_rope=64), batch_count) + // for paged attention, seqlen is max seqlen + ProblemShape problem_shape; + MainloopArguments mainloop; + EpilogueArguments epilogue; + KernelHardwareInfo hw_info; + int split_kv = -1; + int* ptr_split_kv = nullptr; + }; + + using TmaLoadQLatent = typename CollectiveMmaQK::Params::TMA_A; + using TmaLoadQRope = typename CollectiveMmaQK::Params::TMA_A; + using TmaLoadCLatent = typename CollectiveMmaQK::Params::TMA_B; + using TmaLoadKRope = typename CollectiveMmaQK::Params::TMA_B; + using TmaLoadCLatentTranspose = typename CollectiveMmaPV::Params::TMA_B; + + struct MainloopParams { + TmaLoadQLatent tma_load_q_latent; + TmaLoadQRope tma_load_q_rope; + TmaLoadCLatent tma_load_c_latent; + TmaLoadKRope tma_load_k_rope; + TmaLoadCLatentTranspose tma_load_c_latent_transpose; + }; + + struct EpilogueParams { + ElementOut* ptr_o = nullptr; + ElementAcc* ptr_o_acc = nullptr; + TensorStride stride_o; + TensorStride stride_o_acc; + ElementLSE* ptr_lse = nullptr; + ElementLSE* ptr_lse_acc = nullptr; + Stride<_1, int> stride_lse; + Stride<_1, int> stride_lse_acc; + ElementAcc output_scale = 1.0f; + }; + + struct Params { + ProblemShape problem_shape; + MainloopArguments mainloop; + EpilogueParams epilogue; + MainloopParams mainloop_params; + typename TileScheduler::Params tile_scheduler; + int split_kv = -1; + int* ptr_split_kv = nullptr; + }; + + static Params to_underlying_arguments(Arguments const& args, void* workspace) { + //workspace = nullptr; // let's get an error if one of these needs workspace + + auto [H, K, D, B] = args.problem_shape; + auto [L, R] = D; + + int paged_B = B; + int paged_K = K; + if (args.mainloop.ptr_page_table != nullptr) { + paged_B = args.mainloop.page_count; + paged_K = args.mainloop.page_size; + } + + auto params_qk_latent = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, K, L, B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, + args.mainloop.ptr_c_latent, args.mainloop.stride_c_latent, + }, nullptr); + + auto params_qk_latent_paged = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, paged_K, L, paged_B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, + args.mainloop.ptr_c_latent, args.mainloop.stride_c_latent, + }, nullptr); + + auto params_qk_rope = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, K, R, B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_rope, args.mainloop.stride_q_rope, + args.mainloop.ptr_k_rope, args.mainloop.stride_k_rope, + }, nullptr); + + auto params_qk_rope_paged = CollectiveMmaQK::to_underlying_arguments( + make_shape(H, paged_K, R, paged_B), + typename CollectiveMmaQK::Arguments { + args.mainloop.ptr_q_rope, args.mainloop.stride_q_rope, + args.mainloop.ptr_k_rope, args.mainloop.stride_k_rope, + }, nullptr); + + + auto stride_c_latent_transpose = select<1,0,2>(args.mainloop.stride_c_latent); + auto params_pv_latent = CollectiveMmaPV::to_underlying_arguments( + make_shape(H, L, paged_K, paged_B), + typename CollectiveMmaPV::Arguments { + args.mainloop.ptr_q_latent, args.mainloop.stride_q_latent, // dummy, never used + args.mainloop.ptr_c_latent, stride_c_latent_transpose, + }, nullptr); + + MainloopParams mainloop_params { + params_qk_latent.tma_load_a, + params_qk_rope.tma_load_a, + params_qk_latent_paged.tma_load_b, + params_qk_rope_paged.tma_load_b, + params_pv_latent.tma_load_b + }; + + EpilogueParams epilogue_params; + + epilogue_params.ptr_o = args.epilogue.ptr_o; + epilogue_params.stride_o = args.epilogue.stride_o; + epilogue_params.ptr_lse = args.epilogue.ptr_lse; + epilogue_params.stride_lse = args.epilogue.stride_lse; + epilogue_params.output_scale = args.epilogue.output_scale; + + if (args.split_kv > 1) { + ElementAcc* ptr_o_acc = reinterpret_cast(workspace); + ElementLSE* ptr_lse_acc = reinterpret_cast(ptr_o_acc + H * L * args.split_kv * B); + epilogue_params.ptr_o_acc = ptr_o_acc; + epilogue_params.ptr_lse_acc = ptr_lse_acc; + + epilogue_params.stride_o_acc = make_tuple(static_cast(0 + L) * args.split_kv, _1{}, static_cast(0 + H * L) * args.split_kv); + epilogue_params.stride_lse_acc = make_tuple(_1{}, (0 + H) * args.split_kv); + } + + return {args.problem_shape, args.mainloop, epilogue_params, mainloop_params, + TileScheduler::to_underlying_arguments(args.problem_shape, args.hw_info, ClusterShape{}, args.split_kv), args.split_kv, args.ptr_split_kv}; + } + + static size_t get_workspace_size(Arguments const& args) { + ProblemShape problem_shape = args.problem_shape; + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + auto split_kv = args.split_kv; + return (sizeof(ElementAcc) * D_latent + sizeof(ElementLSE)) * H * split_kv * B; + } + static Status initialize_workspace( + Arguments const& /*args*/, void* /*ws*/, cudaStream_t /*stream*/) { + return Status::kSuccess; + } + + static dim3 get_grid_shape(Params const& params) { + return TileScheduler::get_grid_shape(params.tile_scheduler); + } + + static dim3 get_block_shape() { + dim3 block(MaxThreadsPerBlock, 1, 1); + return block; + } + + static bool can_implement(Arguments const& args) { + if (kIsCpAsync) { + if ((args.mainloop.page_size & (args.mainloop.page_size - 1)) != 0) { + return false; + } + if (args.mainloop.page_size > TileShapeS{}) { + return false; + } + } + else { + if (args.mainloop.ptr_page_table != nullptr && args.mainloop.page_size != TileShapeS{}) { + return false; + } + } + if (get<0>(args.problem_shape) != 128) { + return false; + } + if (get<1>(args.problem_shape) <= 0) { + return false; + } + if (args.split_kv <= 0) { + return false; + } + return true; + } + + + CUTLASS_DEVICE void operator()(Params const& params, char* smem_raw) { + + TileScheduler tile_scheduler(params.tile_scheduler); + + int warp_idx = cutlass::canonical_warp_idx_sync(); + auto role = warp_idx_to_role(warp_idx); + uint32_t lane_predicate = cute::elect_one_sync(); + + uint32_t cta_rank_in_cluster = cute::block_rank_in_cluster(); + int cta_coord_v = cta_rank_in_cluster % size<0>(AtomThrShapeMNK{}); + bool is_mma_leader_cta = cta_coord_v == 0; + + if (role == WarpRole::kLoad && lane_predicate && ! kIsCpAsync) { + prefetch_tma_descriptor(params.mainloop_params.tma_load_q_latent.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_c_latent.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_q_rope.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_k_rope.get_tma_descriptor()); + prefetch_tma_descriptor(params.mainloop_params.tma_load_c_latent_transpose.get_tma_descriptor()); + } + SharedStorage& shared_storage = *reinterpret_cast(smem_raw); + + typename PipelineLoadQK::Params pipeline_load_qk_params; + if (role == WarpRole::kLoad) { + pipeline_load_qk_params.role = PipelineLoadQK::ThreadCategory::Producer; + } + if (role == WarpRole::kMma) { + pipeline_load_qk_params.role = PipelineLoadQK::ThreadCategory::Consumer; + } + if constexpr (kIsCpAsync) { + // we can make our life easier by unconditionally loading blocks + // since we know it'll always be legal + pipeline_load_qk_params.producer_arv_count = kNumLoadWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + } + else { + pipeline_load_qk_params.is_leader = lane_predicate && (role == WarpRole::kLoad) && is_mma_leader_cta; + pipeline_load_qk_params.transaction_bytes = kTransactionsBytesLoadQK; + } + pipeline_load_qk_params.initializing_warp = 0; + PipelineLoadQK pipeline_load_qk(shared_storage.pipelines.load_qk, pipeline_load_qk_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineS::Params pipeline_mma_s_params; + if (role == WarpRole::kMma) { + pipeline_mma_s_params.role = PipelineS::ThreadCategory::Producer; + } + if (role == WarpRole::kCompute) { + pipeline_mma_s_params.role = PipelineS::ThreadCategory::Consumer; + } + pipeline_mma_s_params.consumer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_mma_s_params.initializing_warp = 1; + PipelineS pipeline_mma_s( + shared_storage.pipelines.mma_s, + pipeline_mma_s_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineP::Params pipeline_p_mma_params; + if (role == WarpRole::kMma) { + pipeline_p_mma_params.role = PipelineP::ThreadCategory::Consumer; + } + if (role == WarpRole::kCompute) { + pipeline_p_mma_params.role = PipelineP::ThreadCategory::Producer; + } + pipeline_p_mma_params.producer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_p_mma_params.consumer_arv_count = 1; + pipeline_p_mma_params.initializing_warp = 2; + PipelineP pipeline_p_mma( + shared_storage.pipelines.p_mma, + pipeline_p_mma_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelineO::Params pipeline_mma_o_params; + if (role == WarpRole::kMma) { + pipeline_mma_o_params.role = PipelineO::ThreadCategory::Producer; + } + if (role == WarpRole::kCompute) { + pipeline_mma_o_params.role = PipelineO::ThreadCategory::Consumer; + } + pipeline_mma_o_params.consumer_arv_count = kNumComputeWarps * cutlass::NumThreadsPerWarp * size(AtomThrShapeMNK{}); + pipeline_mma_o_params.initializing_warp = 3; + PipelineO pipeline_mma_o( + shared_storage.pipelines.mma_o, + pipeline_mma_o_params, + ClusterShape{}, /*barrier init*/ cute::true_type{}, /*mask calc*/cute::false_type{}); + + typename PipelinePT::Params pipeline_pt_params; + if (role == WarpRole::kLoad) { + pipeline_pt_params.role = PipelinePT::ThreadCategory::Consumer; + } + if (role == WarpRole::kLoadPageTable) { + pipeline_pt_params.role = PipelinePT::ThreadCategory::Producer; + } + pipeline_pt_params.consumer_arv_count = kNumLoadWarps * cutlass::NumThreadsPerWarp; + pipeline_pt_params.producer_arv_count = cutlass::NumThreadsPerWarp; + pipeline_pt_params.initializing_warp = 4; + PipelinePT pipeline_page_table( + shared_storage.pipelines.load_page_table, + pipeline_pt_params); + + TmemAllocator tmem_allocator; + + pipeline_init_arrive_relaxed(size(ClusterShape{})); + + pipeline_load_qk.init_masks(ClusterShape{}); // do we need an update here for 2Sm? + pipeline_mma_s.init_masks(ClusterShape{}); + pipeline_p_mma.init_masks(ClusterShape{}); + pipeline_mma_o.init_masks(ClusterShape{}); + + typename PipelineLoadQK::PipelineState pipeline_load_qk_consumer_state; + typename PipelineLoadQK::PipelineState pipeline_load_qk_producer_state = cutlass::make_producer_start_state(); + + typename PipelineS::PipelineState pipeline_mma_s_consumer_state; + typename PipelineS::PipelineState pipeline_mma_s_producer_state = cutlass::make_producer_start_state(); + + typename PipelineP::PipelineState pipeline_p_mma_consumer_state; + typename PipelineP::PipelineState pipeline_p_mma_producer_state = cutlass::make_producer_start_state(); + + typename PipelineO::PipelineState pipeline_mma_o_consumer_state; + typename PipelineO::PipelineState pipeline_mma_o_producer_state = cutlass::make_producer_start_state(); + + typename PipelinePT::PipelineState pipeline_pt_consumer_state; + typename PipelinePT::PipelineState pipeline_pt_producer_state = cutlass::make_producer_start_state(); + + pipeline_init_wait(size(ClusterShape{})); + + if (role == WarpRole::kLoadPageTable) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_page_table( + blk_coord, + problem_shape, + params.mainloop, + shared_storage.tensors, + pipeline_page_table, pipeline_pt_producer_state, + local_split_kv + ); + } + } + else if (role == WarpRole::kLoad) { + if constexpr (kIsCpAsync) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_cpasync( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv, + /* must be shared pipe */ + pipeline_page_table, pipeline_pt_consumer_state + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + else { + if (params.mainloop.ptr_page_table != nullptr) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_tma( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + else { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + load_tma( + blk_coord, + problem_shape, + params.mainloop, + params.mainloop_params, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_producer_state, + pipeline_load_qk, pipeline_load_qk_producer_state, + local_split_kv + ); + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive_and_wait(); + } + } + } + } + else if (role == WarpRole::kMma) { + tmem_allocator.allocate(TmemAllocator::Sm100TmemCapacityColumns, &shared_storage.tmem_base_ptr); + __syncwarp(); + + if (is_mma_leader_cta) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto local_split_kv = params.split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + mma(blk_coord, + problem_shape, + shared_storage.tensors, + pipeline_load_qk, pipeline_load_qk_consumer_state, + pipeline_load_qk, pipeline_load_qk_consumer_state, + pipeline_mma_s, pipeline_mma_s_producer_state, + pipeline_p_mma, pipeline_p_mma_consumer_state, + pipeline_mma_o, pipeline_mma_o_producer_state, + local_split_kv + ); + } + } + + //cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive_and_wait(); + + //uint32_t free_stage_ptr = shared_storage.tmem_base_ptr; + //tmem_allocator.free(free_stage_ptr, TmemAllocator::Sm100TmemCapacityColumns); + } + else if (role == WarpRole::kCompute) { + CUTLASS_PRAGMA_NO_UNROLL + for (; tile_scheduler.is_valid(); ++tile_scheduler) { + auto blk_coord = tile_scheduler.get_block_coord(); + auto problem_shape = params.problem_shape; + auto split_kv = params.split_kv; + auto local_split_kv = split_kv; + if (params.mainloop.ptr_seq != nullptr) { + get<1>(problem_shape) = params.mainloop.ptr_seq[get<2>(blk_coord)]; + if (params.ptr_split_kv != nullptr) { + local_split_kv = params.ptr_split_kv[get<2>(blk_coord)]; + } + } + if (local_split_kv <= get<3>(blk_coord)) + continue; + compute( + blk_coord, + problem_shape, + params.mainloop, // for softmax_scale + params.epilogue, + shared_storage.tensors, // for smem_comm + pipeline_mma_s, pipeline_mma_s_consumer_state, + pipeline_p_mma, pipeline_p_mma_producer_state, + pipeline_mma_o, pipeline_mma_o_consumer_state, + local_split_kv + ); + } + + //cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive(); + } + + cute::cluster_sync(); + cutlass::arch::NamedBarrier((kNumComputeWarps + 1) * NumThreadsPerWarp, kNamedBarrierTmemDealloc).arrive(); + if (role == WarpRole::kMma) { + uint32_t free_stage_ptr = shared_storage.tmem_base_ptr; + tmem_allocator.free(free_stage_ptr, TmemAllocator::Sm100TmemCapacityColumns); + } + } + + template + CUTLASS_DEVICE void load_page_table( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + TensorStorage& shared_tensors, + PipelinePT& pipeline_page_table, + typename PipelinePT::PipelineState& pipeline_pt_producer_state, int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + int batch_coord = get<2>(blk_coord); + + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), + make_shape(mainloop_args.page_count, B), + mainloop_args.stride_page_table); + auto mPT = mPT_l(_, batch_coord); + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + auto page_size = Pow2{mainloop_args.page_size}; + auto pages_per_tile = Pow2{TileShapeS{} / page_size}; + int thread_idx = threadIdx.x % cutlass::NumThreadsPerWarp; + +#if 1 + for (; k_tile_count > 0; ++k_index, --k_tile_count) { + pipeline_page_table.producer_acquire(pipeline_pt_producer_state); + + // assume a single warp + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < TileShapeS{}; i += cutlass::NumThreadsPerWarp) { + int idx = i + thread_idx; + bool guard = idx < pages_per_tile; + int smem_idx = pipeline_pt_producer_state.index() * TileShapeS::value + idx; + int pt_idx = pages_per_tile * k_index + idx; + + cutlass::arch::cp_async_zfill( + &shared_tensors.smem_page_table[smem_idx], &mPT(pt_idx), guard + ); + } + + pipeline_page_table.producer_commit(pipeline_pt_producer_state, cutlass::arch::cpasync_barrier_arrive); + ++pipeline_pt_producer_state; + } +#endif + } + + + struct Gather { + int& page_table_stage; + Pow2 pages_per_tile; + const int * __restrict__ smem_page_table; + + CUTLASS_DEVICE int operator()(int idx) const { + return smem_page_table[page_table_stage * TileShapeS::value + idx % pages_per_tile]; + } + + CUTLASS_DEVICE friend void print(Gather const&) { + printf(""); + } + + }; + + + template + CUTLASS_DEVICE void load_cpasync( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + MainloopParams const& mainloop_params, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load, + typename PipelineLoadQK::PipelineState& pipeline_load_producer_state, + int const& split_kv, + PipelinePT& pipeline_page_table, + typename PipelinePT::PipelineState& pipeline_pt_consumer_state) { + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + using X = Underscore; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + // partition all tensors + auto mQL = make_tensor(make_gmem_ptr(mainloop_args.ptr_q_latent), make_shape(H, D_latent, B), mainloop_args.stride_q_latent); + auto mQR = make_tensor(make_gmem_ptr(mainloop_args.ptr_q_rope), make_shape(H, D_rope, B), mainloop_args.stride_q_rope); + + int paged_B = mainloop_args.page_count; + auto paged_K = Pow2{mainloop_args.page_size}; + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), make_shape(paged_B, B), mainloop_args.stride_page_table); + + int batch_coord = get<2>(blk_coord); + auto mPT = mPT_l(_, batch_coord); + + auto gQL = local_tile(mQL, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + auto gQR = local_tile(mQR, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + + ThrMMA cta_mma_qk = TiledMmaQK{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + ThrMMA cta_mma_pv = TiledMmaPV{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + + auto tSgQL = cta_mma_qk.partition_A(gQL); + auto tSgQR = cta_mma_qk.partition_A(gQR); + + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + + auto make_copy_for = [](auto sT) { + auto rT_a = sT.layout()(_, _, _, _0{}); + auto rT = make_ordered_layout(shape(rT_a), stride(rT_a)); + auto threads = Int{}; + auto values = Int{}; + return make_cotiled_copy( + Copy_Atom, Element>{}, + make_ordered_layout( + make_shape(threads, values), + make_stride(_1{}, _0{})), + rT); + }; + + // like cute::copy, but makes sure we do all page table lookups first + auto copy_split = [](auto atom, auto src, auto dst) { + auto src_v = group_modes<1, rank_v>(src); + auto dst_v = group_modes<1, rank_v>(dst); + + auto src_v_ptrs = make_tensor(size<1>(src_v)); + for (int i = 0; i < size<1>(src_v); i++) { + src_v_ptrs(i) = &src_v(_0{}, i); + } + + + for (int i = 0; i < size<1>(src_v); i++) { + auto src_v_i = make_tensor( + make_gmem_ptr(src_v_ptrs(i)), + make_shape(shape<0>(src_v)), + make_stride(make_stride(_1{}, _0{})) + ); + atom.call(src_v_i, dst_v(_, i)); + } + }; + + auto tiled_copy_q = make_copy_for(sQ); + auto tiled_copy_kc = make_copy_for(sKC); + auto tiled_copy_vc = make_copy_for(sVC); + + auto thr_copy_q = tiled_copy_q.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + auto thr_copy_kc = tiled_copy_kc.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + auto thr_copy_vc = tiled_copy_vc.get_thread_slice(threadIdx.x % (kNumLoadWarps * cutlass::NumThreadsPerWarp)); + + auto tQsQ = thr_copy_q.partition_D(sQ); + auto tQgQL = thr_copy_q.partition_S(tSgQL); + auto tQgQR = thr_copy_q.partition_S(tSgQR); + + auto tKCsKC = thr_copy_kc.partition_D(sKC); + auto tVCsVC = thr_copy_vc.partition_D(sVC); + + auto pipeline_pt_release_state = pipeline_pt_consumer_state; + + int page_table_stage = -1; + Pow2 pages_per_tile{TileShapeS{} / paged_K}; + const int * __restrict__ smem_page_table = shared_tensors.smem_page_table.begin(); + Gather gather{page_table_stage, pages_per_tile, smem_page_table}; + + auto mCL = make_tensor( + make_gmem_ptr(mainloop_args.ptr_c_latent), + ComposedLayout{ + make_layout( + make_shape(make_shape(paged_K, paged_B), _1{}), + make_stride(make_stride(get<0>(mainloop_args.stride_c_latent), example::CustomStride(gather, get<2>(mainloop_args.stride_c_latent))), get<1>(mainloop_args.stride_c_latent))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(paged_K * paged_B, D_latent))}); + + auto mKR = make_tensor( + make_gmem_ptr(mainloop_args.ptr_k_rope), + ComposedLayout{ + make_layout( + make_shape(make_shape(paged_K, paged_B), _1{}), + make_stride(make_stride(get<0>(mainloop_args.stride_k_rope), example::CustomStride(gather, get<2>(mainloop_args.stride_k_rope))), get<1>(mainloop_args.stride_k_rope))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(paged_K * paged_B, D_latent))}); + + auto mCLT = make_tensor( + make_gmem_ptr(mainloop_args.ptr_c_latent), + ComposedLayout{ + make_layout( + make_shape(_1{}, make_shape(paged_K, paged_B)), + make_stride(get<1>(mainloop_args.stride_c_latent), make_stride(get<0>(mainloop_args.stride_c_latent), example::CustomStride(gather, get<2>(mainloop_args.stride_c_latent))))), + make_coord(_0{}, _0{}), + make_identity_layout(make_shape(D_latent, paged_K * paged_B))}); + + auto gCL = local_tile(mCL, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gKR = local_tile(mKR, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gCLT = local_tile(mCLT, TileShapePV{}, make_coord(_,_,_), Step{}); + + auto tSgCL = cta_mma_qk.partition_B(gCL); + auto tSgKR = cta_mma_qk.partition_B(gKR); + auto tOgCLT = cta_mma_pv.partition_B(gCLT); + + auto tKCgCL = thr_copy_kc.partition_S(tSgCL); + auto tKCgKR = thr_copy_kc.partition_S(tSgKR); + auto tVCgCLT = thr_copy_vc.partition_S(tOgCLT); + + // latent is first in memory, so let's load it first always + // startup: alternate Q and K, set tx count appropriately, for k_idx = 0 + auto& pipeline_acquire_state = pipeline_load_producer_state; + auto pipeline_commit_state = pipeline_acquire_state; + int pipeline_offset = 0; + + for (int i = 0; i < StagesPV; i++) { + cutlass::arch::cp_async_fence(); + } + + auto load_stage = [&](auto fn) { + pipeline_load.producer_acquire(pipeline_acquire_state); + fn(pipeline_acquire_state.index()); + cutlass::arch::cp_async_fence(); + + ++pipeline_acquire_state; + ++pipeline_offset; + + if (pipeline_offset == StagesPV - 1) { + cutlass::arch::cp_async_wait(); + pipeline_load.producer_commit(pipeline_commit_state); + ++pipeline_commit_state; + --pipeline_offset; + } + }; + + pipeline_page_table.consumer_wait(pipeline_pt_consumer_state); + page_table_stage = pipeline_pt_consumer_state.index(); + ++pipeline_pt_consumer_state; + + // each Q/K tile consists of rope and latent + for (int i = 0; i < IterationsQKLatent; i++) { + load_stage([&](int index) { + cute::copy(tiled_copy_q, tQgQL(_, _, _, _, _0{}, i, batch_coord), tQsQ(_, _, _, _, i)); + copy_split(tiled_copy_kc, tKCgCL(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + for (int i = 0; i < IterationsQKRope; i++) { + load_stage([&](int index) { + cute::copy(tiled_copy_q, tQgQR(_, _, _, _, _0{}, i, batch_coord), tQsQ(_, _, _, _, IterationsQKLatent + i)); + copy_split(tiled_copy_kc, tKCgKR(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + k_index += 1; + k_tile_count -= 1; + + // assume k_tile_count >= 1 + // perform K+Q load here + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + pipeline_page_table.consumer_wait(pipeline_pt_consumer_state); + page_table_stage = pipeline_pt_consumer_state.index(); + ++pipeline_pt_consumer_state; + + for (int i = 0; i < IterationsQKLatent; i++) { + load_stage([&](int index) { + copy_split(tiled_copy_kc, tKCgCL(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + for (int i = 0; i < IterationsQKRope; i++) { + load_stage([&](int index) { + copy_split(tiled_copy_kc, tKCgKR(_, _, _, _, k_index, i), tKCsKC(_, _, _, _, index)); + }); + } + + page_table_stage = pipeline_pt_release_state.index(); + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + load_stage([&](int index) { + copy_split(tiled_copy_vc, tVCgCLT(_, _, _, _, j, IterationsPV_K * (k_index - 1) + i), tVCsVC(_, _, _, _, index)); + }); + } + } + + pipeline_page_table.consumer_release(pipeline_pt_release_state); + ++pipeline_pt_release_state; + + k_index += 1; + k_tile_count -= 1; + } + + page_table_stage = pipeline_pt_release_state.index(); + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + load_stage([&](int index) { + copy_split(tiled_copy_vc, tVCgCLT(_, _, _, _, j, IterationsPV_K * (k_index - 1) + i), tVCsVC(_, _, _, _, index)); + }); + } + } + + pipeline_page_table.consumer_release(pipeline_pt_release_state); + ++pipeline_pt_release_state; + + while (pipeline_offset > 0) { + cutlass::arch::cp_async_fence(); + + cutlass::arch::cp_async_wait(); + pipeline_load.producer_commit(pipeline_commit_state); + ++pipeline_commit_state; + --pipeline_offset; + } + + cutlass::arch::cp_async_wait<0>(); + + } + + + template + CUTLASS_DEVICE void load_tma( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + MainloopParams const& mainloop_params, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load_qk, + typename PipelineLoadQK::PipelineState& pipeline_load_qk_producer_state, + PipelineLoadPV& pipeline_load_pv, + typename PipelineLoadPV::PipelineState& pipeline_load_pv_producer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + using X = Underscore; + + // partition all tensors + auto mQL = mainloop_params.tma_load_q_latent.get_tma_tensor(make_shape(H, D_latent, B)); + auto mQR = mainloop_params.tma_load_q_rope.get_tma_tensor(make_shape(H, D_rope, B)); + + int paged_B = B; + int paged_K = K; + if constexpr (kIsPaged) { + paged_B = mainloop_args.page_count; + paged_K = mainloop_args.page_size; + } + auto mPT_l = make_tensor(make_gmem_ptr(mainloop_args.ptr_page_table), make_shape(paged_B, B), mainloop_args.stride_page_table); + + auto mCL = mainloop_params.tma_load_c_latent.get_tma_tensor(make_shape(paged_K, D_latent, paged_B)); + auto mKR = mainloop_params.tma_load_k_rope.get_tma_tensor(make_shape(paged_K, D_rope, paged_B)); + + auto mCLT = mainloop_params.tma_load_c_latent_transpose.get_tma_tensor(make_shape(D_latent, paged_K, paged_B)); + + auto gQL = local_tile(mQL, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + auto gQR = local_tile(mQR, TileShapeQK{}, make_coord(_,_,_), Step<_1, X, _1>{}); + + auto gCL = local_tile(mCL, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gKR = local_tile(mKR, TileShapeQK{}, make_coord(_,_,_), Step{}); + auto gCLT = local_tile(mCLT, TileShapePV{}, make_coord(_,_,_), Step{}); + + ThrMMA cta_mma_qk = TiledMmaQK{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + ThrMMA cta_mma_pv = TiledMmaPV{}.get_slice(get<0>(blk_coord) % size(AtomThrShapeMNK{})); + + auto tSgQL = cta_mma_qk.partition_A(gQL); + auto tSgQR = cta_mma_qk.partition_A(gQR); + + auto tSgCL = cta_mma_qk.partition_B(gCL); + auto tSgKR = cta_mma_qk.partition_B(gKR); + + auto tOgCLT = cta_mma_pv.partition_B(gCLT); + + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + + auto [tQLgQL_mkl, tQsQ] = tma_partition( + mainloop_params.tma_load_q_latent, _0{}, make_layout(_1{}), + group_modes<0,3>(sQ), group_modes<0,3>(tSgQL)); + + auto [tQRgQR_mkl, tQsQ_ignore] = tma_partition( + mainloop_params.tma_load_q_rope, _0{}, make_layout(_1{}), + group_modes<0,3>(sQ), group_modes<0,3>(tSgQR)); + + auto [tCLgCL_nkl, tKCsKC] = tma_partition( + mainloop_params.tma_load_c_latent, _0{}, make_layout(_1{}), + group_modes<0,3>(sKC), group_modes<0,3>(tSgCL)); + + auto [tKRgKR_nkl, tKCsKC_ignore] = tma_partition( + mainloop_params.tma_load_k_rope, _0{}, make_layout(_1{}), + group_modes<0,3>(sKC), group_modes<0,3>(tSgKR)); + + auto [tCLTgCLT_nkl, tVCsVC] = tma_partition( + mainloop_params.tma_load_c_latent_transpose, _0{}, make_layout(_1{}), + group_modes<0,3>(sVC), group_modes<0,3>(tOgCLT)); + + uint16_t mcast_mask = 0; + + int batch_coord = get<2>(blk_coord); + Tensor tQLgQL = tQLgQL_mkl(_, _, _, batch_coord); + Tensor tQRgQR = tQRgQR_mkl(_, _, _, batch_coord); + + auto mPT = mPT_l(_, batch_coord); + + Tensor tCLgCL = tCLgCL_nkl(_, _, _, _); + Tensor tKRgKR = tKRgKR_nkl(_, _, _, _); + + // careful: stage and k are swapped here! + Tensor tCLTgCLT = tCLTgCLT_nkl(_, _, _, _); + + // latent is first in memory, so let's load it first always + // startup: alternate Q and K, set tx count appropriately, for k_idx = 0 + + // each Q/K tile consists of rope and latent + for (int i = 0; i < IterationsQKLatent; i++) { + pipeline_load_qk.producer_expect_transaction(pipeline_load_qk_producer_state, kTransactionsBytesLoadExtraQ); + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // expect the extra bytes + // load_qk ql + cute::copy(mainloop_params.tma_load_q_latent.with(*tma_barrier, mcast_mask), tQLgQL(_, _0{}, i), tQsQ(_, i)); + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + for (int i = 0; i < IterationsQKRope; i++) { + pipeline_load_qk.producer_expect_transaction(pipeline_load_qk_producer_state, kTransactionsBytesLoadExtraQ); + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // expect the extra bytes + // load_qk ql + cute::copy(mainloop_params.tma_load_q_rope.with(*tma_barrier, mcast_mask), tQRgQR(_, _0{}, i), tQsQ(_, i + IterationsQKLatent)); + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + k_index += 1; + k_tile_count -= 1; + + // assume k_tile_count >= 1 + // perform K+Q load here + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + // perform K load + for (int i = 0; i < IterationsQKLatent; i++) { + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent.with(*tma_barrier, mcast_mask), + tCLgCL(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + for (int i = 0; i < IterationsQKRope; i++) { + pipeline_load_qk.producer_acquire(pipeline_load_qk_producer_state); + auto tma_barrier = pipeline_load_qk.producer_get_barrier(pipeline_load_qk_producer_state); + + if (cute::elect_one_sync()) { + // load_qk cl + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, _0{}, i, mPT(k_index)), + tKCsKC(_, pipeline_load_qk_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_k_rope.with(*tma_barrier, mcast_mask), + tKRgKR(_, k_index, i, batch_coord), + tKCsKC(_, pipeline_load_qk_producer_state.index())); + } + } + ++pipeline_load_qk_producer_state; + } + + // prefetch next K load to keep busy while we transpose-load from cache + const int kPrefetchDistance = 1; + for (int i = 0; i < IterationsQKLatent; i++) { + if (cute::elect_one_sync()) { + if constexpr (kIsPaged) { + if (k_tile_count > kPrefetchDistance) { + cute::prefetch( + mainloop_params.tma_load_c_latent, + tCLgCL(_, _0{}, i, mPT(k_index + kPrefetchDistance)) + ); + } + } + else { + cute::prefetch( + mainloop_params.tma_load_c_latent, + tCLgCL(_, k_index + kPrefetchDistance, i, batch_coord) + ); + } + } + } + + for (int i = 0; i < IterationsQKRope; i++) { + if (cute::elect_one_sync()) { + if constexpr (kIsPaged) { + if (k_tile_count > kPrefetchDistance) { + cute::prefetch( + mainloop_params.tma_load_k_rope, + tKRgKR(_, _0{}, i, mPT(k_index + kPrefetchDistance)) + ); + } + } + else { + cute::prefetch( + mainloop_params.tma_load_k_rope, + tKRgKR(_, k_index + kPrefetchDistance, i, batch_coord) + ); + } + } + } + + // perform V load (k_idx - 1) + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.producer_acquire(pipeline_load_pv_producer_state); + auto tma_barrier = pipeline_load_pv.producer_get_barrier(pipeline_load_pv_producer_state); + + if (cute::elect_one_sync()) { + // load_pv cl + // note the transpose in indices! + // note we are off-by-one on k_index + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, i, mPT(k_index - 1)), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, IterationsPV_K * (k_index - 1) + i, batch_coord), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + } + ++pipeline_load_pv_producer_state; + } + } + + k_index += 1; + k_tile_count -= 1; + } + + for (int i = 0; i < IterationsPV_K; i++) { + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.producer_acquire(pipeline_load_pv_producer_state); + auto tma_barrier = pipeline_load_pv.producer_get_barrier(pipeline_load_pv_producer_state); + + if (cute::elect_one_sync()) { + // load_pv cl + // note the transpose in indices + // note we are off-by-one on k_index + + if constexpr (kIsPaged) { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, i, mPT(k_index - 1)), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + else { + cute::copy( + mainloop_params.tma_load_c_latent_transpose.with(*tma_barrier, mcast_mask, cute::TMA::CacheHintSm100::EVICT_FIRST), + tCLTgCLT(_, j, IterationsPV_K * (k_index - 1) + i, batch_coord), + tVCsVC(_, pipeline_load_pv_producer_state.index()) + ); + } + } + ++pipeline_load_pv_producer_state; + } + } + } + + template + CUTLASS_DEVICE void mma( + BlkCoord const& blk_coord, + ProblemShape const& problem_shape, + TensorStorage& shared_tensors, + PipelineLoadQK& pipeline_load_qk, + typename PipelineLoadQK::PipelineState& pipeline_load_qk_consumer_state, + PipelineLoadPV& pipeline_load_pv, + typename PipelineLoadPV::PipelineState& pipeline_load_pv_consumer_state, + PipelineS& pipeline_mma_s, + typename PipelineS::PipelineState& pipeline_mma_s_producer_state, + PipelineP& pipeline_p_mma, + typename PipelineP::PipelineState& pipeline_p_mma_consumer_state, + PipelineO& pipeline_mma_o, + typename PipelineO::PipelineState& pipeline_mma_o_producer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(blk_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + return; + } + + // mma init + Tensor sQ = make_tensor(make_smem_ptr(shared_tensors.smem_q.begin()), SmemLayoutQ{}); + Tensor sKC = make_tensor(make_smem_ptr(shared_tensors.smem_kc.begin()), SmemLayoutKC{}); + Tensor sVC = make_tensor(make_smem_ptr(shared_tensors.smem_vc.begin()), SmemLayoutVC{}); + Tensor sP = make_tensor(make_smem_ptr((Element*) shared_tensors.smem_p.begin()), SmemLayoutP{}); + + Tensor tSrQ = TiledMmaQK::make_fragment_A(sQ); + Tensor tSrKC = TiledMmaQK::make_fragment_B(sKC); + Tensor tOrP = TiledMmaPV::make_fragment_A(sP); + Tensor tOrVC = TiledMmaPV::make_fragment_B(sVC); + + TiledMmaQK tiled_mma_qk; + TiledMmaPV tiled_mma_pv; + + Tensor tStS = partition_fragment_C(tiled_mma_qk, select<0,1>(TileShapeQK{})); + Tensor tOtO = partition_fragment_C(tiled_mma_pv, select<0,1>(TileShapePV{})); + + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::Zero; + + pipeline_mma_s.producer_acquire(pipeline_mma_s_producer_state); + + // Mma S0 S1 O0 S2 O1 ... Sn On-1 On + // S0 ownership -- ----- -- -- + // S1 ownership -- ----- ---- + // O ownership -- -- ---- -- + + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::Zero; + for (int i = 0; i < IterationsQK; i++) { + pipeline_load_qk.consumer_wait(pipeline_load_qk_consumer_state); + int read_stage = pipeline_load_qk_consumer_state.index(); + + tStS.data() = uint32_t(pipeline_mma_s_producer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1); + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tSrQ); ++k_block) { + cute::gemm(tiled_mma_qk, + tSrQ(_,_,k_block,i), + tSrKC(_,_,k_block,read_stage), + tStS); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_qk.consumer_release(pipeline_load_qk_consumer_state); + ++pipeline_load_qk_consumer_state; + } + + pipeline_mma_s.producer_commit(pipeline_mma_s_producer_state); + ++pipeline_mma_s_producer_state; + + k_tile_count -= 1; + + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + + pipeline_mma_s.producer_acquire(pipeline_mma_s_producer_state); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::Zero; + for (int i = 0; i < IterationsQK; i++) { + pipeline_load_qk.consumer_wait(pipeline_load_qk_consumer_state); + int read_stage = pipeline_load_qk_consumer_state.index(); + + tStS.data() = uint32_t(pipeline_mma_s_producer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1); + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tSrQ); ++k_block) { + cute::gemm(tiled_mma_qk, + tSrQ(_,_,k_block,i), + tSrKC(_,_,k_block,read_stage), + tStS); + tiled_mma_qk.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_qk.consumer_release(pipeline_load_qk_consumer_state); + ++pipeline_load_qk_consumer_state; + } + + pipeline_mma_s.producer_commit(pipeline_mma_s_producer_state); + ++pipeline_mma_s_producer_state; + + pipeline_mma_o.producer_acquire(pipeline_mma_o_producer_state); + pipeline_p_mma.consumer_wait(pipeline_p_mma_consumer_state); + + for (int i = 0; i < IterationsPV_K; i++) { + auto acc_flag = tiled_mma_pv.accumulate_; + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.consumer_wait(pipeline_load_pv_consumer_state); + + int read_stage = pipeline_load_pv_consumer_state.index(); + + tOtO.data() = uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO); + tiled_mma_pv.accumulate_ = acc_flag; + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tOrP); ++k_block) { + cute::gemm(tiled_mma_pv, + tOrP(_,_,k_block, make_coord(i, pipeline_p_mma_consumer_state.index())), + tOrVC(_,_,k_block,read_stage), + tOtO); + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_pv.consumer_release(pipeline_load_pv_consumer_state); + ++pipeline_load_pv_consumer_state; + } + } + + pipeline_p_mma.consumer_release(pipeline_p_mma_consumer_state); + ++pipeline_p_mma_consumer_state; + pipeline_mma_o.producer_commit(pipeline_mma_o_producer_state); + ++pipeline_mma_o_producer_state; + + --k_tile_count; + } + + pipeline_mma_o.producer_acquire(pipeline_mma_o_producer_state); + pipeline_p_mma.consumer_wait(pipeline_p_mma_consumer_state); + + for (int i = 0; i < IterationsPV_K; i++) { + auto acc_flag = tiled_mma_pv.accumulate_; + for (int j = 0; j < IterationsPV_N; j++) { + pipeline_load_pv.consumer_wait(pipeline_load_pv_consumer_state); + + int read_stage = pipeline_load_pv_consumer_state.index(); + + tOtO.data() = uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO); + tiled_mma_pv.accumulate_ = acc_flag; + + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < size<2>(tOrP); ++k_block) { + cute::gemm(tiled_mma_pv, + tOrP(_,_,k_block, make_coord(i, pipeline_p_mma_consumer_state.index())), + tOrVC(_,_,k_block,read_stage), + tOtO); + tiled_mma_pv.accumulate_ = UMMA::ScaleOut::One; + } + + pipeline_load_pv.consumer_release(pipeline_load_pv_consumer_state); + ++pipeline_load_pv_consumer_state; + } + } + + pipeline_p_mma.consumer_release(pipeline_p_mma_consumer_state); + ++pipeline_p_mma_consumer_state; + pipeline_mma_o.producer_commit(pipeline_mma_o_producer_state); + ++pipeline_mma_o_producer_state; + } + + + template + CUTLASS_DEVICE void softmax( + IsLastTile const& is_last_tile, + ElementAcc& row_max, + ElementAcc& row_sum, + ElementAcc& correction_factor, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + TensorStorage& shared_tensors, + int k_index, + uint32_t tmem_s, + int smem_p_index) { + + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + + TiledMmaQK tiled_mma_qk; + + Tensor tStS = partition_fragment_C(tiled_mma_qk, select<0,1>(TileShapeQK{})); + tStS.data() = tmem_s; + + CUTE_STATIC_ASSERT_V(shape<1>(tStS) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tStS) == _1{}); + Tensor tAcc = tStS(make_coord(_,_),_0{},_0{}); + + Tensor cS = make_identity_tensor(take<0,2>(CtaShapeQK{})); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_cS = thread_t2r.partition_D(cS); + Tensor tTR_rAcc = make_tensor(shape(tTR_cS)); + + Tensor tTR_rS_frag = make_tensor(shape(tTR_rAcc)); + const int AlignmentS = 4; + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + Tensor tTR_rAcc_vec = recast>(tTR_rAcc); + Tensor tTR_rS_vec = recast>(tTR_rS_frag); + + // load s + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + if (is_last_tile) { + for (int i = 0; i < size(tTR_rAcc); i++) { + if (get<1>(tTR_cS(i)) + TileShapeS{} * k_index >= get<1>(problem_shape)) { + tTR_rAcc(i) = -std::numeric_limits::infinity(); + } + } + } + + // max + ElementAcc row_max_new = row_max; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i += 1) { + row_max_new = ::fmax(row_max_new, tTR_rAcc(i)); + } + + // for 2x2 dp, reduce here + if constexpr (kWarpsInN > 1) { + shared_tensors.smem_exchange[threadIdx.x] = row_max_new; + cutlass::arch::NamedBarrier(kNumComputeWarps*NumThreadsPerWarp, kNamedBarrierExchange).sync(); + // (64, 2) shape + int peer_index = (threadIdx.x + 64) % 128; + row_max_new = cutlass::max(row_max_new, shared_tensors.smem_exchange[peer_index]); + } + +#ifndef B2B + // find correction factor + ElementAcc softmax_scale_log2 = mainloop_args.softmax_scale * static_cast(M_LOG2E); + correction_factor = ::exp2f(softmax_scale_log2 * (row_max - row_max_new)); + row_max = row_max_new; + + // softmax + ElementAcc row_max_scale_log2 = row_max * softmax_scale_log2; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rAcc(i) = ::exp2f(softmax_scale_log2 * tTR_rAcc(i) - row_max_scale_log2); + } +#endif + + // quantize + cutlass::NumericArrayConverter epilogue_op; + + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc_vec); i++) { + tTR_rS_vec(i) = epilogue_op(tTR_rAcc_vec(i)); + } + + Tensor sP = make_tensor(make_smem_ptr((Element*) shared_tensors.smem_p.begin()), SmemLayoutP{})(_, _, _, make_coord(_, smem_p_index)); + + Tensor tOcP = TiledMmaPV{}.get_slice(_0{}).partition_A(cS); + + // have a mapping for each thread to coord + // find identical mapping to coords for the MMA + auto l = make_ordered_layout(make_shape(make_shape(_64{}, _2{}), make_shape(_16{}, TileShapeS{} / _32{})), make_stride(make_stride(_0{}, _3{}), make_stride(_1{}, _2{}))); + auto sP_ = as_position_independent_swizzle_tensor(sP); + copy_aligned(tTR_rS_frag, sP_.compose(l)(threadIdx.x, _)); + + // sum + row_sum *= correction_factor; + + static_assert(cute::is_same_v); + auto tTR_rAcc_float2 = recast(tTR_rAcc); + auto sums = make_tensor(_4{}); + static_assert(size(tTR_rAcc_float2) % size(sums) == 0); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(sums); i++) { + sums(i) = tTR_rAcc_float2(i); + } + CUTLASS_PRAGMA_UNROLL + for (int i = size(sums); i < size(tTR_rAcc_float2); i += size(sums)) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size(sums); j++) { + cute::add(sums(j), sums(j), tTR_rAcc_float2(i + j)); + } + } + CUTLASS_PRAGMA_UNROLL + for (int i = 1; i < size(sums); i *= 2) { + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size(sums); j += 2*i) { + cute::add(sums(j), sums(j), sums(j+i)); + } + } + row_sum += sums(0).x + sums(0).y; + } + + + CUTLASS_DEVICE void rescale( + ElementAcc correction_factor, + uint32_t tmem_o) { + + // for b2b gemm, do nothing +#ifndef B2B + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + auto store_op = TMEM::tmem_load_to_store(load_op); + + TiledMmaPV tiled_mma_pv; + + Tensor tOtO = partition_fragment_C(tiled_mma_pv, select<0,1>(TileShapePV{})); + tOtO.data() = tmem_o; + + CUTE_STATIC_ASSERT_V(shape<1>(tOtO) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tOtO) == _1{}); + Tensor tAcc = tOtO(make_coord(_,_),_0{},_0{}); + + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = make_tensor(make_gmem_ptr((ElementAcc*) nullptr), cta_tiler_pv, make_stride(0, 0)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto tiled_r2t = make_tmem_copy(store_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + auto thread_r2t = tiled_r2t.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + // load o + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + // multiply by correction factor + float2 correction_factor_vec = make_float2(correction_factor, correction_factor); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i += 2) { + float2 in = make_float2(tTR_rAcc(i + 0), tTR_rAcc(i + 1)); + float2 out; + cute::mul(out, in, correction_factor_vec); + tTR_rAcc(i + 0) = out.x; + tTR_rAcc(i + 1) = out.y; + } + + // store o + copy(tiled_r2t, tTR_rAcc, tTR_tAcc); +#endif + } + + + template + CUTLASS_DEVICE void epilogue( + ElementAcc& row_max, + ElementAcc& row_sum, + BlkCoord const& cta_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + EpilogueParams const& epilogue_args, + TensorStorage& shared_tensors, + uint32_t tmem_o, + int const& split_kv) { + + auto load_op = cute::SM100_TMEM_LOAD_32dp32b32x{}; + + TiledMmaPV tiled_mma_pv; + + Tensor tOtO = TiledMmaPV::make_fragment_C(partition_shape_C(TiledMmaPV{}, take<0, 2>(TileShapePV{}))); + tOtO.data() = tmem_o; + + CUTE_STATIC_ASSERT_V(shape<1>(tOtO) == _1{}); + CUTE_STATIC_ASSERT_V(shape<2>(tOtO) == _1{}); + Tensor tAcc = tOtO(make_coord(_,_),_0{},_0{}); + + auto [H, K, D, B] = problem_shape; + auto [D_latent, D_rope] = D; + if (epilogue_args.ptr_o_acc != nullptr) { + using ElementOutAcc = ElementAcc; + constexpr auto AlignmentOutAcc = 128 / cute::sizeof_bits_v; + Tensor mO = make_tensor(make_gmem_ptr(epilogue_args.ptr_o_acc + get<3>(cta_coord) * D_latent), make_shape(H, D_latent, B), epilogue_args.stride_o_acc); + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = local_tile(mO, cta_tiler_pv, take<0,3>(cta_coord)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_rO_frag = make_tensor(shape(tTR_rAcc)); + Tensor tTR_rO_src = recast>(coalesce(tTR_rO_frag)); + Tensor tR2G_rO_dst = recast>(coalesce(tTR_gO)); + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + cutlass::epilogue::thread::LinearCombination epilogue_op({epilogue_args.output_scale / row_sum}); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rO_frag(i) = epilogue_op(tTR_rAcc(i)); + } + + copy(tTR_rO_src, tR2G_rO_dst); + +#ifndef B2B + + // compute LSE + ElementAcc lse = cutlass::fast_log(row_sum) + mainloop_args.softmax_scale * row_max; + + // store LSE + Tensor mLSE = make_tensor(make_gmem_ptr(epilogue_args.ptr_lse_acc + H * get<3>(cta_coord)), make_shape(H, B), epilogue_args.stride_lse_acc); + Tensor gLSE = local_tile(mLSE, append<3>(cta_tiler_pv, _1{}), take<0,3>(cta_coord), Step<_1, Underscore, _1>{}); + // for 2x2 dp, this must be conditional and the index is wrong + if (! kIs2Sm || (threadIdx.x < 64)) + { + gLSE(threadIdx.x) = lse; + } + #endif + } + else { + Tensor mO = make_tensor(make_gmem_ptr(epilogue_args.ptr_o), make_shape(H, D_latent, B), epilogue_args.stride_o); + auto cta_tiler_pv = take<0,2>(typename CollectiveMmaPV::CtaShape_MNK{}); + Tensor gO = local_tile(mO, cta_tiler_pv, take<0,3>(cta_coord)); + + auto tiled_t2r = make_tmem_copy(load_op, tAcc); + auto thread_idx = threadIdx.x % size(tiled_t2r); + + auto thread_t2r = tiled_t2r.get_slice(thread_idx); + Tensor tTR_gO = thread_t2r.partition_D(gO); + Tensor tTR_rAcc = make_tensor(shape(tTR_gO)); + + Tensor tTR_rO_frag = make_tensor(shape(tTR_rAcc)); + Tensor tTR_rO_src = recast>(coalesce(tTR_rO_frag)); + Tensor tR2G_rO_dst = recast>(coalesce(tTR_gO)); + Tensor tTR_tAcc = thread_t2r.partition_S(tAcc); + + copy(tiled_t2r, tTR_tAcc, tTR_rAcc); + + cutlass::epilogue::thread::LinearCombination epilogue_op({epilogue_args.output_scale / row_sum}); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(tTR_rAcc); i++) { + tTR_rO_frag(i) = epilogue_op(tTR_rAcc(i)); + } + + copy(tTR_rO_src, tR2G_rO_dst); + +#ifndef B2B + if (epilogue_args.ptr_lse != nullptr) { + // compute LSE + ElementAcc lse = cutlass::fast_log(row_sum) + mainloop_args.softmax_scale * row_max; + + // store LSE + Tensor mLSE = make_tensor(make_gmem_ptr(epilogue_args.ptr_lse), make_shape(H, B), epilogue_args.stride_lse); + Tensor gLSE = local_tile(mLSE, append<3>(cta_tiler_pv, _1{}), take<0,3>(cta_coord), Step<_1, Underscore, _1>{}); + + // for 2x2 dp, this must be conditional and the index is wrong + if (! kIs2Sm || (threadIdx.x < 64)) + { + gLSE(threadIdx.x) = lse; + } + } +#endif + } + } + + + template + CUTLASS_DEVICE void compute( + CtaCoord const& cta_coord, + ProblemShape const& problem_shape, + MainloopArguments const& mainloop_args, + EpilogueParams const& epilogue_args, + TensorStorage& shared_tensors, + PipelineS& pipeline_mma_s, + typename PipelineS::PipelineState& pipeline_mma_s_consumer_state, + PipelineP& pipeline_p_mma, + typename PipelineP::PipelineState& pipeline_p_mma_producer_state, + PipelineO& pipeline_mma_o, + typename PipelineO::PipelineState& pipeline_mma_o_consumer_state, + int const& split_kv) { + + auto [H, K, D, B] = problem_shape; + + int k_tile_total = ceil_div(K, TileShapeS{}); + int k_tile_per_cta = ceil_div(k_tile_total, split_kv); + int k_index = get<3>(cta_coord) * k_tile_per_cta; // lower limit + int k_tile_count = max(0, min(k_tile_total, k_index + k_tile_per_cta) - k_index); + if (k_tile_count == 0) { + + // if we return early, we have to make sure we release the load warp + cutlass::arch::NamedBarrier( + (kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, + kNamedBarrierEpilogue + ).arrive(); + + return; + } + int k_index_final = k_tile_total - 1; + + ElementAcc row_max = -std::numeric_limits::infinity(); + ElementAcc row_sum = 0; + ElementAcc correction_factor = 1; + + pipeline_p_mma.producer_acquire(pipeline_p_mma_producer_state); + pipeline_mma_s.consumer_wait(pipeline_mma_s_consumer_state); + + auto dispatch_bool = [](bool b, auto fn) { + if (b) { + fn(cute::true_type{}); + } + else { + fn(cute::false_type{}); + } + }; + + // softmax s0 -> p0 + dispatch_bool(k_index == k_index_final, [&](auto is_last_tile) { + softmax( + is_last_tile, + row_max, row_sum, correction_factor, + problem_shape, mainloop_args, shared_tensors, k_index, + uint32_t(pipeline_mma_s_consumer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1), + pipeline_p_mma_producer_state.index() + ); + }); + + k_index += 1; + + cutlass::arch::fence_view_async_tmem_load(); + cutlass::arch::fence_view_async_shared(); + pipeline_mma_s.consumer_release(pipeline_mma_s_consumer_state); + ++pipeline_mma_s_consumer_state; + pipeline_p_mma.producer_commit(pipeline_p_mma_producer_state); + ++pipeline_p_mma_producer_state; + + k_tile_count -= 1; + + CUTLASS_PRAGMA_NO_UNROLL + while (k_tile_count > 0) { + pipeline_p_mma.producer_acquire(pipeline_p_mma_producer_state); + pipeline_mma_s.consumer_wait(pipeline_mma_s_consumer_state); + + // softmax s1 -> p1 + dispatch_bool(k_index == k_index_final, [&](auto is_last_tile) { + softmax( + is_last_tile, + row_max, row_sum, correction_factor, + problem_shape, mainloop_args, shared_tensors, k_index, + uint32_t(pipeline_mma_s_consumer_state.index() == 0 ? TmemAllocation::kS0 : TmemAllocation::kS1), + pipeline_p_mma_producer_state.index() + ); + }); + + cutlass::arch::fence_view_async_tmem_load(); + cutlass::arch::fence_view_async_shared(); + pipeline_mma_s.consumer_release(pipeline_mma_s_consumer_state); + ++pipeline_mma_s_consumer_state; + pipeline_p_mma.producer_commit(pipeline_p_mma_producer_state); + ++pipeline_p_mma_producer_state; + + pipeline_mma_o.consumer_wait(pipeline_mma_o_consumer_state); + + // rescale + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < IterationsPV_N; j++) { + rescale(correction_factor, uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO)); + } + + cutlass::arch::fence_view_async_tmem_store(); + pipeline_mma_o.consumer_release(pipeline_mma_o_consumer_state); + ++pipeline_mma_o_consumer_state; + + --k_tile_count; + k_index += 1; + } + + pipeline_mma_o.consumer_wait(pipeline_mma_o_consumer_state); + +#ifdef B2B + row_sum = 1; +#else + if constexpr (kWarpsInN > 1) { + // reduce row_sum if needed (for 2x2 dp) + shared_tensors.smem_exchange[threadIdx.x] = row_sum; + cutlass::arch::NamedBarrier(kNumComputeWarps*NumThreadsPerWarp, kNamedBarrierExchange).sync(); + // (64, 2) shape + int peer_index = (threadIdx.x + 64) % 128; + row_sum += shared_tensors.smem_exchange[peer_index]; + } +#endif + + cutlass::arch::NamedBarrier((kNumComputeWarps + kNumLoadWarps) * NumThreadsPerWarp, kNamedBarrierEpilogue).arrive(); + + // epilogue + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < IterationsPV_N; j++) { + epilogue( + row_max, row_sum, + replace<1>(cta_coord, j), problem_shape, + mainloop_args, epilogue_args, shared_tensors, + uint32_t(TmemAllocation::kO0) + j * uint32_t(TmemAllocation::kSizeAccO), split_kv + ); + } + + cutlass::arch::fence_view_async_tmem_load(); + pipeline_mma_o.consumer_release(pipeline_mma_o_consumer_state); + ++pipeline_mma_o_consumer_state; + } + +}; + +/////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::kernel diff --git a/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp new file mode 100644 index 000000000..30389e79f --- /dev/null +++ b/sgl-kernel/csrc/attention/cutlass_sm100_mla/kernel/sm100_mla_tile_scheduler.hpp @@ -0,0 +1,160 @@ +/*************************************************************************************************** + * Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ + +// clang-format off +#pragma once + +#include "cutlass/cutlass.h" +#include "cutlass/fast_math.h" +#include "cutlass/kernel_hardware_info.h" + +namespace cutlass::fmha::kernel { + +//////////////////////////////////////////////////////////////////////////////// + +struct Sm100MlaIndividualTileScheduler { + + struct Params { + dim3 grid; + }; + + bool valid_ = true; + + CUTLASS_DEVICE + Sm100MlaIndividualTileScheduler(Params const&) {} + + template + static Params to_underlying_arguments( + ProblemShape const& problem_shape, KernelHardwareInfo hw_info, + ClusterShape const& cluster_shape, int const& split_kv) { + using namespace cute; + dim3 grid(get<0>(cluster_shape), get<3>(problem_shape) /* Batch */, split_kv /*Maximum Split KV*/); + return Params{ grid }; + } + + static dim3 get_grid_shape(Params const& params) { + return params.grid; + } + + CUTLASS_DEVICE + bool is_valid() { + return valid_; + } + + CUTLASS_DEVICE + auto get_block_coord() { + using namespace cute; + return make_coord(blockIdx.x, _0{}, blockIdx.y, blockIdx.z); + } + + CUTLASS_DEVICE + Sm100MlaIndividualTileScheduler& operator++() { + valid_ = false; + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct Sm100MlaPersistentTileScheduler { + + struct Params { + int num_blocks; + FastDivmod divmod_m_block; + FastDivmod divmod_b; + FastDivmod divmod_split_kv; + KernelHardwareInfo hw_info; + }; + + int block_idx = 0; + Params params; + + CUTLASS_DEVICE + Sm100MlaPersistentTileScheduler(Params const& params) : block_idx(blockIdx.x), params(params) {} + + template + static Params to_underlying_arguments( + ProblemShape const& problem_shape, KernelHardwareInfo hw_info, + ClusterShape const& cluster_shape, int const& split_kv) { + using namespace cute; + // Get SM count if needed, otherwise use user supplied SM count + int sm_count = hw_info.sm_count; + if (sm_count <= 1 || sm_count % size<0>(cluster_shape) != 0) { + CUTLASS_TRACE_HOST(" WARNING: Arguments do not include a valid SM count.\n" + " For optimal performance, populate the arguments KernelHardwareInfo struct with the SM count."); + sm_count = KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + } + + CUTLASS_TRACE_HOST("to_underlying_arguments(): Setting persistent grid SM count to " << sm_count); + hw_info.sm_count = sm_count; + + int num_m_blocks = size<0>(cluster_shape); + int num_blocks = num_m_blocks * get<3>(problem_shape) /* Batch */; + num_blocks *= split_kv; /* Maximum Split KV*/ + + return Params { + num_blocks, + { num_m_blocks}, { get<3>(problem_shape) }, {split_kv}, + hw_info + }; + } + + static dim3 get_grid_shape(Params const& params) { + dim3 grid(std::min(params.num_blocks, params.hw_info.sm_count), 1, 1); + return grid; + } + + CUTLASS_DEVICE + bool is_valid() { + return block_idx < params.num_blocks; + } + + CUTLASS_DEVICE + auto get_block_coord() { + using namespace cute; + int block_decode = block_idx; + int m_block, bidb, n_split_kv; + params.divmod_m_block(block_decode, m_block, block_decode); + params.divmod_b(block_decode, bidb, block_decode); + params.divmod_split_kv(block_decode, n_split_kv, block_decode); + return make_coord(m_block, _0{}, bidb, n_split_kv); + } + + CUTLASS_DEVICE + Sm100MlaPersistentTileScheduler& operator++() { + block_idx += gridDim.x; + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::fmha::kernel diff --git a/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu b/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu new file mode 100644 index 000000000..01bd4797c --- /dev/null +++ b/sgl-kernel/csrc/attention/lightning_attention_decode_kernel.cu @@ -0,0 +1,154 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#define THREADS_PER_BLOCK 128 + +template +__global__ void lightning_attention_decode_kernel( + const T* __restrict__ q, // [b, h, 1, d] + const T* __restrict__ k, // [b, h, 1, d] + const T* __restrict__ v, // [b, h, 1, e] + const float* __restrict__ past_kv, // [b, h, d, e] + const float* __restrict__ slope, // [h, 1, 1] + T* __restrict__ output, // [b, h, 1, e] + float* __restrict__ new_kv, // [b, h, d, e] + const int batch_size, + const int num_heads, + const int qk_dim, + const int v_dim) { + extern __shared__ char smem[]; + T* __restrict__ q_shared = reinterpret_cast(smem); + T* __restrict__ k_shared = reinterpret_cast(smem + qk_dim * sizeof(T)); + T* __restrict__ v_shared = reinterpret_cast(smem + 2 * qk_dim * sizeof(T)); + float* __restrict__ new_kv_shared = reinterpret_cast(smem + (2 * qk_dim + v_dim) * sizeof(T)); + T* __restrict__ output_shared = + reinterpret_cast(smem + (2 * qk_dim + v_dim) * sizeof(T) + qk_dim * (v_dim + 1) * sizeof(float)); + + const int32_t tid = threadIdx.x; + const int32_t current_head = blockIdx.x; + const int32_t b = current_head / num_heads; + const int32_t h = current_head % num_heads; + + if (b >= batch_size) return; + + const int32_t qk_offset = b * num_heads * qk_dim + h * qk_dim; + const int32_t v_offset = b * num_heads * v_dim + h * v_dim; + const int32_t kv_offset = b * num_heads * qk_dim * v_dim + h * qk_dim * v_dim; + + // Load q, k, v into shared memory + for (int d = tid; d < qk_dim; d += blockDim.x) { + q_shared[d] = q[qk_offset + d]; + k_shared[d] = k[qk_offset + d]; + } + for (int e = tid; e < v_dim; e += blockDim.x) { + v_shared[e] = v[v_offset + e]; + } + + __syncthreads(); + + const float ratio = expf(-1.0f * slope[h]); + + // Compute new_kv + for (int d = tid; d < qk_dim; d += blockDim.x) { + const T k_val = k_shared[d]; + for (int e = 0; e < v_dim; ++e) { + const int past_kv_idx = kv_offset + d * v_dim + e; + const T v_val = v_shared[e]; + const float new_val = ratio * past_kv[past_kv_idx] + k_val * v_val; + const int shared_idx = d * (v_dim + 1) + e; + new_kv_shared[shared_idx] = new_val; + } + } + + __syncthreads(); + + // Store new_kv to global memory + for (int idx = tid; idx < qk_dim * v_dim; idx += blockDim.x) { + const int d = idx / v_dim; + const int e = idx % v_dim; + const int shared_idx = d * (v_dim + 1) + e; + const int global_idx = kv_offset + idx; + new_kv[global_idx] = new_kv_shared[shared_idx]; + } + + __syncthreads(); + + // Compute output + for (int e = tid; e < v_dim; e += blockDim.x) { + float sum = 0.0f; + for (int d = 0; d < qk_dim; ++d) { + const int shared_idx = d * (v_dim + 1) + e; + sum += q_shared[d] * new_kv_shared[shared_idx]; + } + output_shared[e] = static_cast(sum); + } + + __syncthreads(); + + // Store output to global memory + if (tid == 0) { + for (int e = 0; e < v_dim; ++e) { + output[v_offset + e] = output_shared[e]; + } + } +} + +void lightning_attention_decode( + const torch::Tensor& q, + const torch::Tensor& k, + const torch::Tensor& v, + const torch::Tensor& past_kv, + const torch::Tensor& slope, + torch::Tensor output, + torch::Tensor new_kv) { + TORCH_CHECK(q.is_contiguous(), "q must be contiguous"); + TORCH_CHECK(k.is_contiguous(), "k must be contiguous"); + TORCH_CHECK(v.is_contiguous(), "v must be contiguous"); + TORCH_CHECK(past_kv.is_contiguous(), "past_kv must be contiguous"); + + auto batch_size = q.size(0); + auto num_heads = q.size(1); + auto qk_dim = q.size(3); + auto v_dim = v.size(3); + + dim3 block(THREADS_PER_BLOCK); + dim3 grid(batch_size * num_heads); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND2( + at::ScalarType::Half, at::ScalarType::BFloat16, q.scalar_type(), "lightning_attention_decode_kernel", ([&] { + size_t smem_size = (2 * qk_dim + 2 * v_dim) * sizeof(scalar_t) + qk_dim * (v_dim + 1) * sizeof(float); + lightning_attention_decode_kernel<<>>( + q.data_ptr(), + k.data_ptr(), + v.data_ptr(), + past_kv.data_ptr(), + slope.data_ptr(), + output.data_ptr(), + new_kv.data_ptr(), + batch_size, + num_heads, + qk_dim, + v_dim); + })); +} diff --git a/sgl-kernel/csrc/attention/merge_attn_states.cu b/sgl-kernel/csrc/attention/merge_attn_states.cu new file mode 100644 index 000000000..c719498b8 --- /dev/null +++ b/sgl-kernel/csrc/attention/merge_attn_states.cu @@ -0,0 +1,204 @@ +#include +#include + +#include +#include + +#include "pytorch_extension_utils.h" + +// Helper functions to convert between different data types +// (float, half, bfloat16) for the merge attention states kernel. +inline __device__ float to_float(float u) { + return u; +} +inline __device__ float to_float(half u) { + return __half2float(u); +} +inline __device__ float to_float(__nv_bfloat16 u) { + return __bfloat162float(u); +} +inline __device__ void from_float(float& d, float s) { + d = s; +} +inline __device__ void from_float(half& d, float s) { + d = __float2half(s); +} +inline __device__ void from_float(__nv_bfloat16& d, float s) { + d = __float2bfloat16(s); +} + +// Implements section 2.2 of https://www.arxiv.org/pdf/2501.01005 +template +__global__ void merge_attn_states_kernel( + scalar_t* output, + float* output_lse, + const scalar_t* prefix_output, + const float* prefix_lse, + const scalar_t* suffix_output, + const float* suffix_lse, + const uint num_tokens, + const uint num_heads, + const uint head_size) { + using pack_128b_t = uint4; + const uint pack_size = 16 / sizeof(scalar_t); + const uint threads_per_head = head_size / pack_size; + + const uint global_idx = blockIdx.x * NUM_THREADS + threadIdx.x; + const uint token_head_threads = num_tokens * num_heads * threads_per_head; + + if (global_idx >= token_head_threads) return; + + // global_idx -> token_idx + head_idx + pack_idx + const uint token_head_idx = global_idx / threads_per_head; + const uint pack_idx = global_idx % threads_per_head; + + const uint token_idx = token_head_idx / num_heads; + const uint head_idx = token_head_idx % num_heads; + + const uint pack_offset = pack_idx * pack_size; // (0~15)*8, etc. + const uint head_offset = token_idx * num_heads * head_size + head_idx * head_size; + const scalar_t* prefix_head_ptr = prefix_output + head_offset; + const scalar_t* suffix_head_ptr = suffix_output + head_offset; + scalar_t* output_head_ptr = output + head_offset; + + // float p_lse = prefix_lse[head_idx * num_tokens + token_idx]; + // float s_lse = suffix_lse[head_idx * num_tokens + token_idx]; + float p_lse = prefix_lse[token_idx * num_heads + head_idx]; + float s_lse = suffix_lse[token_idx * num_heads + head_idx]; + p_lse = std::isinf(p_lse) ? -std::numeric_limits::infinity() : p_lse; + s_lse = std::isinf(s_lse) ? -std::numeric_limits::infinity() : s_lse; + + const float max_lse = fmaxf(p_lse, s_lse); + p_lse = p_lse - max_lse; + s_lse = s_lse - max_lse; + const float p_se = expf(p_lse); + const float s_se = expf(s_lse); + const float out_se = p_se + s_se; + const float p_scale = p_se / out_se; + const float s_scale = s_se / out_se; + + if (pack_offset < head_size) { + // Pack 128b load + pack_128b_t p_out_pack = reinterpret_cast(prefix_head_ptr)[pack_offset / pack_size]; + pack_128b_t s_out_pack = reinterpret_cast(suffix_head_ptr)[pack_offset / pack_size]; + pack_128b_t o_out_pack; + +#pragma unroll + for (uint i = 0; i < pack_size; ++i) { + // Always use float for FMA to keep high precision. + // half(uint16_t), bfloat16, float -> float. + const float p_out_f = to_float(reinterpret_cast(&p_out_pack)[i]); + const float s_out_f = to_float(reinterpret_cast(&s_out_pack)[i]); + // fma: a * b + c = p_out_f * p_scale + (s_out_f * s_scale) + const float o_out_f = p_out_f * p_scale + (s_out_f * s_scale); + // float -> half(uint16_t), bfloat16, float. + from_float(reinterpret_cast(&o_out_pack)[i], o_out_f); + } + + // Pack 128b storage + reinterpret_cast(output_head_ptr)[pack_offset / pack_size] = o_out_pack; + } + // We only need to write to output_lse once per head. + if (output_lse != nullptr && pack_idx == 0) { + float out_lse = logf(out_se) + max_lse; + output_lse[token_idx * num_heads + head_idx] = out_lse; + } +} + +// The following macro is used to dispatch the conversion function based on +// the output data type. The FN is a macro that calls a function with +// template. +#define DISPATCH_BY_SCALAR_DTYPE(scalar_dtype, fn) \ + { \ + if (scalar_dtype == at::ScalarType::Float) { \ + fn(float); \ + } else if (scalar_dtype == at::ScalarType::Half) { \ + fn(half); \ + } else if (scalar_dtype == at::ScalarType::BFloat16) { \ + fn(__nv_bfloat16); \ + } else { \ + TORCH_CHECK(false, "Unsupported data type of O: ", scalar_dtype); \ + } \ + } + +#define LAUNCH_MERGE_ATTN_STATES(scalar_t, NUM_THREADS) \ + { \ + merge_attn_states_kernel<<>>( \ + reinterpret_cast(output.data_ptr()), \ + reinterpret_cast(output_lse.data_ptr()), \ + reinterpret_cast(prefix_output.data_ptr()), \ + reinterpret_cast(prefix_lse.data_ptr()), \ + reinterpret_cast(suffix_output.data_ptr()), \ + reinterpret_cast(suffix_lse.data_ptr()), \ + num_tokens, \ + num_heads, \ + head_size); \ + } + +/*@brief Merges the attention states from prefix and suffix + * into the output tensor. NUM_TOKENS: n, NUM_HEADS: h, HEAD_SIZE: d + * + * @param output [n,h,d] The output tensor to store the merged attention states. + * @param output_lse [h,d] Optional tensor to store the log-sum-exp values. + * @param prefix_output [n,h,d] The prefix attention states. + * @param prefix_lse [n,h] The log-sum-exp values for the prefix attention + * states. + * @param suffix_output [n,h,d] The suffix attention states. + * @param suffix_lse [n,h] The log-sum-exp values for the suffix attention + * states. + */ +template +void merge_attn_states_launcher( + const at::Tensor& prefix_output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + const at::Tensor& prefix_lse, // [NUM_TOKENS, NUM_HEADS] + const at::Tensor& suffix_output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + const at::Tensor& suffix_lse, // [NUM_TOKENS, NUM_HEADS] + at::Tensor& output, // [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + at::Tensor& output_lse // [NUM_TOKENS, NUM_HEADS] +) { + constexpr uint NUM_THREADS = 128; + const uint num_tokens = output.size(0); + const uint num_heads = output.size(1); + const uint head_size = output.size(2); + const uint pack_size = 16 / sizeof(scalar_t); + TORCH_CHECK(head_size % pack_size == 0, "headsize must be multiple of pack_size:", pack_size); + // Process one pack elements per thread. for float, the + // pack_size is 4 for half/bf16, the pack_size is 8. + const uint threads_per_head = head_size / pack_size; + const uint total_threads = num_tokens * num_heads * threads_per_head; + + dim3 block(NUM_THREADS); + dim3 grid((total_threads + NUM_THREADS - 1) / NUM_THREADS); + + const c10::cuda::OptionalCUDAGuard device_guard(prefix_output.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + + LAUNCH_MERGE_ATTN_STATES(scalar_t, NUM_THREADS); +} + +#define CALL_MERGE_ATTN_STATES_LAUNCHER(scalar_t) \ + { merge_attn_states_launcher(v_a, s_a, v_b, s_b, v_merged, s_merged); } + +void merge_state_v2( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged) { + // Input tensors must be contiguous + CHECK_INPUT(v_a); // v_a prefix_output (seq_len, num_heads, head_dim) + CHECK_INPUT(s_a); // s_a prefix_lse (seq_len, num_heads) + CHECK_INPUT(v_b); // v_b suffix_output (seq_len, num_heads, head_dim) + CHECK_INPUT(s_b); // s_b suffix_lse (seq_len, num_heads) + // v_merged output (seq_len, num_heads, head_dim) + // s_merged output_lse (seq_len, num_heads) + auto device = v_a.device(); + CHECK_EQ(s_a.device(), device); + CHECK_EQ(v_b.device(), device); + CHECK_EQ(s_b.device(), device); + CHECK_DIM(3, v_a); + CHECK_DIM(2, s_a); + CHECK_DIM(3, v_b); + CHECK_DIM(2, s_b); + CHECK_SHAPE(v_a, v_b); + CHECK_SHAPE(s_a, s_b); + CHECK_EQ(v_a.size(0), s_a.size(0)); + CHECK_EQ(v_a.size(1), s_b.size(1)); + DISPATCH_BY_SCALAR_DTYPE(v_merged.dtype(), CALL_MERGE_ATTN_STATES_LAUNCHER); +} diff --git a/sgl-kernel/csrc/attention/vertical_slash_index.cu b/sgl-kernel/csrc/attention/vertical_slash_index.cu new file mode 100644 index 000000000..118f780dd --- /dev/null +++ b/sgl-kernel/csrc/attention/vertical_slash_index.cu @@ -0,0 +1,462 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// This file is for blocksparse attention utils cuda kernel. + +#include +#include +#include +#include + +// Save the start index of each block in the given range into block_offset. +// Returns the updated block count. +__device__ int64_t save_blocks( + int* block_offset, + int64_t range_start, + int64_t range_end, + int64_t block_size, + int64_t input_block_count, + int64_t kv_seqlen) { + if (range_start >= kv_seqlen) { + return input_block_count; + } + if (range_end > kv_seqlen) { + range_end = kv_seqlen; + } + int64_t current_block_count = input_block_count; + for (int idx = range_start; idx < range_end; idx += block_size) { + block_offset[current_block_count++] = idx; + } + return current_block_count; +} + +// CUDA kernel: convert sparse vertical/slash indices to block/column offsets. +__global__ void convert_vertical_slash_indexes_kernel( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal // True for intra, False for succ +) { + const int batch_idx = blockIdx.y; + const int head_idx = blockIdx.x; + const int group_idx = blockIdx.z; + + int64_t q_seqlen = q_seqlens[batch_idx]; + int64_t kv_seqlen = kv_seqlens[batch_idx]; + int64_t block_idx_m = group_idx * blockDim.x + threadIdx.x; + int64_t start_m = block_idx_m * BLOCK_SIZE_M; + if (start_m >= q_seqlen) { + return; + } + int64_t end_m = start_m + BLOCK_SIZE_M; + vertical_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_V; + slash_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_S; + int64_t row_offset = (batch_idx * N_HEADS + head_idx) * N_ROWS + block_idx_m; + block_count += row_offset; + block_offset += row_offset * NNZ_S; + column_count += row_offset; + column_index += row_offset * NNZ_V; + + bool has_slash = true; + int64_t tmp_col_cnt = 0, tmp_blk_cnt = 0; + int64_t s = 0, v = 0; + int64_t v_idx = vertical_indexes[v++]; + int64_t s_idx = slash_indexes[s++]; + if (causal) { + while (s_idx >= end_m + (kv_seqlen - q_seqlen) && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + (kv_seqlen - q_seqlen)) has_slash = false; + s_idx = max((kv_seqlen - q_seqlen) + end_m - s_idx, BLOCK_SIZE_M); + } else { + while (s_idx >= end_m + kv_seqlen && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + kv_seqlen) has_slash = false; + s_idx = max(kv_seqlen + end_m - s_idx, BLOCK_SIZE_M); + } + + int64_t range_start = s_idx - BLOCK_SIZE_M, range_end = s_idx; + if (!has_slash) { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + } + + bool slash_finished = false; + while (1) { + if (v_idx < range_end) { + if (v_idx < range_start) { + column_index[tmp_col_cnt++] = v_idx; + } + if (v < NNZ_V) { + v_idx = vertical_indexes[v++]; + } else { + if (causal) + v_idx = end_m + BLOCK_SIZE_N + (kv_seqlen - q_seqlen); + else + v_idx = end_m + BLOCK_SIZE_N + kv_seqlen; + } + } else { + if ((s < NNZ_S && causal) || (s < NNZ_S && !causal && slash_indexes[s] >= start_m)) { + if (causal) + s_idx = max((kv_seqlen - q_seqlen) + end_m - slash_indexes[s++], BLOCK_SIZE_M); + else + s_idx = max(kv_seqlen + end_m - slash_indexes[s++], BLOCK_SIZE_M); + } else { + if (v == NNZ_V || (v_idx > range_start && causal)) { + // add the last vertical if no more slash + if (v == NNZ_V && !causal && v_idx < kv_seqlen) { + column_index[tmp_col_cnt++] = v_idx; + } + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + break; + } else { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + // if slash_finished but there are vertical left, save current + // blocks + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + slash_finished = true; + } + } + if (!slash_finished) { + if (s_idx > range_end + BLOCK_SIZE_M) { + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = s_idx - BLOCK_SIZE_M; + range_end = s_idx; + } else if (s_idx > range_end) { + range_end += BLOCK_SIZE_M; + } + } + } + } + + block_count[0] = tmp_blk_cnt; + column_count[0] = tmp_col_cnt; +} + +// Host function: launches the kernel with 64 threads per block. +void convert_vertical_slash_indexes_64x64( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t BATCH_SIZE, + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal) { + const int N_THREADS = 64; + const dim3 dimBlock((int32_t)N_THREADS); + const dim3 dimGrid( + (int32_t)N_HEADS, (int32_t)BATCH_SIZE, ((int32_t)N_ROWS + (int32_t)N_THREADS - 1) / (int32_t)N_THREADS); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + convert_vertical_slash_indexes_kernel<<>>( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + block_count, + block_offset, + column_count, + column_index, + N_HEADS, + N_ROWS, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + NNZ_V, + NNZ_S, + causal); +} + +// Host function: prepares tensor pointers and launches the CUDA kernel. +void convert_vertical_slash_indexes( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal) { + cudaSetDevice(q_seqlens.get_device()); + + int64_t batch_size = slash_indexes.size(0); + int64_t num_heads = slash_indexes.size(1); + int64_t nnz_slash = slash_indexes.size(2); + int64_t nnz_vertical = vertical_indexes.size(2); + int64_t num_rows = (context_size + block_size_M - 1) / block_size_M; + + convert_vertical_slash_indexes_64x64( + q_seqlens.data_ptr(), + kv_seqlens.data_ptr(), + vertical_indexes.data_ptr(), + slash_indexes.data_ptr(), + block_count.data_ptr(), + block_offset.data_ptr(), + column_count.data_ptr(), + column_index.data_ptr(), + batch_size, + num_heads, + num_rows, + block_size_M, + block_size_N, + nnz_vertical, + nnz_slash, + causal); +} + +// --- mergehead kernels --- // + +// Kernel: like above, but supports per-head variable NNZ_V/NNZ_S. +__global__ void convert_vertical_slash_indexes_kernel_mergehead( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + const int* per_head_vertical_topkv, + const int* per_head_slash_topkv, + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal // True for intra, False for succ +) { + const int batch_idx = blockIdx.y; + const int head_idx = blockIdx.x; + const int group_idx = blockIdx.z; + + int64_t q_seqlen = q_seqlens[batch_idx]; + int64_t kv_seqlen = kv_seqlens[batch_idx]; + int64_t block_idx_m = group_idx * blockDim.x + threadIdx.x; + int64_t start_m = block_idx_m * BLOCK_SIZE_M; + if (start_m >= q_seqlen) { + return; + } + int64_t end_m = start_m + BLOCK_SIZE_M; + vertical_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_V; + slash_indexes += (batch_idx * N_HEADS + head_idx) * NNZ_S; + int64_t row_offset = (batch_idx * N_HEADS + head_idx) * N_ROWS + block_idx_m; + block_count += row_offset; + block_offset += row_offset * NNZ_S; + column_count += row_offset; + column_index += row_offset * NNZ_V; + + // MergeHead: each head has it's unique max topk NNZ_V，NNZ_S. (NNZ_V，NNZ_S + // above is buffer size, use to compute offset) + NNZ_S = per_head_slash_topkv[head_idx]; + NNZ_V = per_head_vertical_topkv[head_idx]; + + bool has_slash = true; + int64_t tmp_col_cnt = 0, tmp_blk_cnt = 0; + int64_t s = 0, v = 0; + int64_t v_idx = vertical_indexes[v++]; + int64_t s_idx = slash_indexes[s++]; + if (causal) { + while (s_idx >= end_m + (kv_seqlen - q_seqlen) && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + (kv_seqlen - q_seqlen)) has_slash = false; + s_idx = max((kv_seqlen - q_seqlen) + end_m - s_idx, BLOCK_SIZE_M); + } else { + while (s_idx >= end_m + kv_seqlen && s < NNZ_S) { + s_idx = slash_indexes[s++]; + } + if (s_idx > end_m + kv_seqlen) has_slash = false; + s_idx = max(kv_seqlen + end_m - s_idx, BLOCK_SIZE_M); + } + + int64_t range_start = s_idx - BLOCK_SIZE_M, range_end = s_idx; + if (!has_slash) { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + } + + bool slash_finished = false; + while (1) { + if (v_idx < range_end) { + if (v_idx < range_start) { + column_index[tmp_col_cnt++] = v_idx; + } + if (v < NNZ_V) { + v_idx = vertical_indexes[v++]; + } else { + if (causal) + v_idx = end_m + BLOCK_SIZE_N + (kv_seqlen - q_seqlen); + else + v_idx = end_m + BLOCK_SIZE_N + kv_seqlen; + } + } else { + if ((s < NNZ_S && causal) || (s < NNZ_S && !causal && slash_indexes[s] >= start_m)) { + if (causal) + s_idx = max((kv_seqlen - q_seqlen) + end_m - slash_indexes[s++], BLOCK_SIZE_M); + else + s_idx = max(kv_seqlen + end_m - slash_indexes[s++], BLOCK_SIZE_M); + } else { + if (v == NNZ_V || (v_idx > range_start && causal)) { + // add the last vertical if no more slash + if (v == NNZ_V && !causal && v_idx < kv_seqlen) { + column_index[tmp_col_cnt++] = v_idx; + } + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + break; + } else { + if (causal) { + range_start = (kv_seqlen - q_seqlen) + end_m; + range_end = (kv_seqlen - q_seqlen) + end_m + BLOCK_SIZE_N; + } else { + // if slash_finished but there are vertical left, save current + // blocks + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = kv_seqlen; + range_end = kv_seqlen + BLOCK_SIZE_N; + } + slash_finished = true; + } + } + if (!slash_finished) { + if (s_idx > range_end + BLOCK_SIZE_M) { + tmp_blk_cnt = save_blocks(block_offset, range_start, range_end, BLOCK_SIZE_N, tmp_blk_cnt, kv_seqlen); + range_start = s_idx - BLOCK_SIZE_M; + range_end = s_idx; + } else if (s_idx > range_end) { + range_end += BLOCK_SIZE_M; + } + } + } + } + + block_count[0] = tmp_blk_cnt; + column_count[0] = tmp_col_cnt; +} + +// Launch the mergehead kernel with 64 threads per block. +void convert_vertical_slash_indexes_64x64_mergehead( + const int* q_seqlens, // [BATCH, ] + const int* kv_seqlens, // [BATCH, ] + const int* vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + const int* slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int* per_head_vertical_topkv, + int* per_head_slash_topkv, + int* block_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* block_offset, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_S] + int* column_count, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M)] + int* column_index, // [BATCH, N_HEADS, cdiv(N_CTX, BLOCK_SIZE_M), NNZ_V] + int64_t BATCH_SIZE, + int64_t N_HEADS, + int64_t N_ROWS, + int64_t BLOCK_SIZE_M, + int64_t BLOCK_SIZE_N, + int64_t NNZ_V, + int64_t NNZ_S, + bool causal) { + const int N_THREADS = 64; + const dim3 dimBlock(N_THREADS); + const dim3 dimGrid(N_HEADS, BATCH_SIZE, (N_ROWS + N_THREADS - 1) / N_THREADS); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + convert_vertical_slash_indexes_kernel_mergehead<<>>( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + per_head_vertical_topkv, + per_head_slash_topkv, + block_count, + block_offset, + column_count, + column_index, + N_HEADS, + N_ROWS, + BLOCK_SIZE_M, + BLOCK_SIZE_N, + NNZ_V, + NNZ_S, + causal); +} + +// Host wrapper for mergehead kernel. +void convert_vertical_slash_indexes_mergehead( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + torch::Tensor vertical_indices_count, // [N_HEADS, ] + torch::Tensor slash_indices_count, + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal) { + cudaSetDevice(q_seqlens.get_device()); + + int batch_size = slash_indexes.size(0); + int num_heads = slash_indexes.size(1); + int nnz_slash = slash_indexes.size(2); + int nnz_vertical = vertical_indexes.size(2); + int num_rows = (context_size + block_size_M - 1) / block_size_M; + + convert_vertical_slash_indexes_64x64_mergehead( + q_seqlens.data_ptr(), + kv_seqlens.data_ptr(), + vertical_indexes.data_ptr(), + slash_indexes.data_ptr(), + vertical_indices_count.data_ptr(), + slash_indices_count.data_ptr(), + block_count.data_ptr(), + block_offset.data_ptr(), + column_count.data_ptr(), + column_index.data_ptr(), + batch_size, + num_heads, + num_rows, + block_size_M, + block_size_N, + nnz_vertical, + nnz_slash, + causal); +} diff --git a/sgl-kernel/csrc/common_extension.cc b/sgl-kernel/csrc/common_extension.cc new file mode 100644 index 000000000..4f95c9138 --- /dev/null +++ b/sgl-kernel/csrc/common_extension.cc @@ -0,0 +1,481 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From csrc/allreduce + */ + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + m.def("dispose", &dispose); + m.def("meta_size", &meta_size); + m.def("register_buffer", ®ister_buffer); + + m.def( + "init_custom_ar(int[] ipc_tensors, Tensor rank_data, " + "int rank, bool full_nvlink) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def( + "all_reduce(int fa, Tensor inp, Tensor! out, int reg_buffer, " + "int reg_buffer_sz_bytes) -> ()"); + m.impl("all_reduce", torch::kCUDA, &all_reduce); + + m.def("mscclpp_generate_unique_id", &mscclpp_generate_unique_id); + m.def( + "mscclpp_init_context(Tensor unique_id, int rank, int world_size, Tensor scratch, Tensor put_buffer, " + "int nranks_per_node, int[] rank_to_node, int[] rank_to_ib, int context_selection) -> int"); + m.impl("mscclpp_init_context", torch::kCUDA, &mscclpp_init_context); + + m.def("mscclpp_allreduce(int context, Tensor inp, Tensor! out, int nthreads, int nblocks) -> ()"); + m.impl("mscclpp_allreduce", torch::kCUDA, &mscclpp_allreduce); + + /* + * From csrc/attention + */ + m.def( + "lightning_attention_decode(Tensor q, Tensor k, Tensor v, Tensor past_kv, Tensor slope, Tensor! output, Tensor! " + "new_kv) -> ()"); + m.impl("lightning_attention_decode", torch::kCUDA, &lightning_attention_decode); + m.def("merge_state(Tensor v_a, Tensor s_a, Tensor v_b, Tensor s_b, Tensor! v_merged, Tensor! s_merged) -> ()"); + m.impl("merge_state", torch::kCUDA, &merge_state); + m.def("merge_state_v2(Tensor v_a, Tensor s_a, Tensor v_b, Tensor s_b, Tensor! v_merged, Tensor! s_merged) -> ()"); + m.impl("merge_state_v2", torch::kCUDA, &merge_state_v2); + m.def( + "cutlass_mla_decode(Tensor! out, Tensor q_nope, Tensor q_pe, Tensor kv_c_and_k_pe_cache, Tensor seq_lens, Tensor " + "page_table, Tensor! workspace, float sm_scale, int num_kv_splits) -> ()"); + m.impl("cutlass_mla_decode", torch::kCUDA, &cutlass_mla_decode); + m.def("cutlass_mla_get_workspace_size", &cutlass_mla_get_workspace_size); + + /* + * From csrc/elementwise + */ + m.def("rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("rmsnorm", torch::kCUDA, &rmsnorm); + + m.def("fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("fused_add_rmsnorm", torch::kCUDA, &sgl_fused_add_rmsnorm); + + m.def("gemma_rmsnorm(Tensor! output, Tensor input, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("gemma_rmsnorm", torch::kCUDA, &gemma_rmsnorm); + + m.def("gemma_fused_add_rmsnorm(Tensor! input, Tensor! residual, Tensor weight, float eps, bool enable_pdl) -> ()"); + m.impl("gemma_fused_add_rmsnorm", torch::kCUDA, &gemma_fused_add_rmsnorm); + + m.def("silu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("silu_and_mul", torch::kCUDA, &silu_and_mul); + + m.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul); + + m.def("gelu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul); + + m.def( + "apply_rope_pos_ids_cos_sin_cache(Tensor q, Tensor k, Tensor! q_rope, Tensor! k_rope, Tensor cos_sin_cache, " + "Tensor pos_ids, bool interleave, bool enable_pdl, int cuda_stream, " + "Tensor? v, Tensor!? k_buffer, Tensor!? v_buffer, Tensor? kv_cache_loc) -> ()"); + m.impl("apply_rope_pos_ids_cos_sin_cache", torch::kCUDA, &apply_rope_pos_ids_cos_sin_cache); + + m.def( + "downcast_fp8(Tensor k, Tensor v, Tensor k_out, Tensor v_out, Tensor k_scale, Tensor v_scale, Tensor loc, int " + "mult, int offset, int cuda_stream) -> ()"); + m.impl("downcast_fp8", torch::kCUDA, &downcast_fp8); + + /* + * From csrc/gemm + */ + m.def("awq_dequantize(Tensor qweight, Tensor scales, Tensor qzeros) -> Tensor"); + m.impl("awq_dequantize", torch::kCUDA, &awq_dequantize); + + m.def( + "int8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("int8_scaled_mm", torch::kCUDA, &int8_scaled_mm); + + m.def( + "fp8_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype, Tensor? " + "bias) -> Tensor"); + m.impl("fp8_scaled_mm", torch::kCUDA, &fp8_scaled_mm); + + m.def( + "fp8_blockwise_scaled_mm(Tensor mat_a, Tensor mat_b, Tensor scales_a, Tensor scales_b, ScalarType out_dtype) -> " + "Tensor"); + m.impl("fp8_blockwise_scaled_mm", torch::kCUDA, &fp8_blockwise_scaled_mm); + + m.def( + "sgl_per_token_group_quant_fp8(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float fp8_min, float fp8_max, bool scale_ue8m0) -> ()"); + m.impl("sgl_per_token_group_quant_fp8", torch::kCUDA, &sgl_per_token_group_quant_fp8); + + m.def( + "sgl_per_token_group_quant_int8(Tensor input, Tensor output_q, Tensor output_s, int group_size," + " float eps, float int8_min, float int8_max) -> ()"); + m.impl("sgl_per_token_group_quant_int8", torch::kCUDA, &sgl_per_token_group_quant_int8); + + m.def("sgl_per_tensor_quant_fp8(Tensor input, Tensor output_q, Tensor output_s, bool is_static) -> ()"); + m.impl("sgl_per_tensor_quant_fp8", torch::kCUDA, &sgl_per_tensor_quant_fp8); + + m.def("sgl_per_token_quant_fp8(Tensor input, Tensor output_q, Tensor output_s) -> ()"); + m.impl("sgl_per_token_quant_fp8", torch::kCUDA, &sgl_per_token_quant_fp8); + + m.def( + "cutlass_scaled_fp4_mm(Tensor! out, Tensor a, Tensor b," + " Tensor block_scale_a, Tensor block_scale_b," + " Tensor alpha) -> ()"); + m.impl("cutlass_scaled_fp4_mm", torch::kCUDA, &cutlass_scaled_fp4_mm); + + m.def( + "scaled_fp4_quant(Tensor! output, Tensor! input," + " Tensor! output_scale, Tensor! input_scale) -> ()"); + m.impl("scaled_fp4_quant", torch::kCUDA, &scaled_fp4_quant); + + m.def("dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + m.impl("dsv3_fused_a_gemm", torch::kCUDA, &dsv3_fused_a_gemm); + + // Compute NVFP4 experts quantization. + m.def( + "scaled_fp4_experts_quant(Tensor! output, Tensor! output_scale," + "Tensor input, Tensor input_global_scale, Tensor input_offset_by_experts," + "Tensor output_scale_offset_by_experts) -> ()"); + m.impl("scaled_fp4_experts_quant", torch::kCUDA, &scaled_fp4_experts_quant); + + m.def( + "silu_and_mul_scaled_fp4_experts_quant(Tensor! output, Tensor! output_scale," + "Tensor input, Tensor input_global_scale, Tensor mask, bool use_silu_and_mul) -> ()"); + m.impl("silu_and_mul_scaled_fp4_experts_quant", torch::kCUDA, &silu_and_mul_scaled_fp4_experts_quant); + + m.def( + "cutlass_fp4_group_mm(Tensor! output, Tensor a, Tensor b," + "Tensor a_blockscale, Tensor b_blockscale, Tensor alphas," + "Tensor ab_strides, Tensor c_strides, Tensor problem_sizes," + " Tensor expert_offsets, Tensor sf_offsets) -> ()"); + m.impl("cutlass_fp4_group_mm", torch::kCUDA, &cutlass_fp4_group_mm); + + m.def("dsv3_router_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()"); + m.impl("dsv3_router_gemm", torch::kCUDA, &dsv3_router_gemm); + + /* + * From csrc/gemm/gptq + */ + m.def( + "gptq_marlin_gemm(Tensor! a, Tensor? c_or_none," + "Tensor! b_q_weight, Tensor! b_scales, Tensor? global_scale_or_none," + "Tensor? b_zeros_or_none, Tensor? g_idx_or_none, Tensor? perm_or_none," + "Tensor! workspace, int b_q_type_id, int size_m, int size_n, int size_k," + "bool is_k_full, bool use_atomic_add, bool use_fp32_reduce, bool is_zp_float) -> Tensor"); + m.impl("gptq_marlin_gemm", torch::kCUDA, &gptq_marlin_gemm); + + m.def( + "gptq_gemm(Tensor a, Tensor b_q_weight, Tensor b_gptq_qzeros, Tensor b_gptq_scales, Tensor b_g_idx, bool " + "use_shuffle, int bit) -> Tensor"); + m.impl("gptq_gemm", torch::kCUDA, &gptq_gemm); + + m.def("gptq_shuffle(Tensor! q_weight, Tensor q_perm, int bit) -> ()"); + m.impl("gptq_shuffle", torch::kCUDA, &gptq_shuffle); + + m.def("gptq_marlin_repack(Tensor! b_q_weight, Tensor! perm, int size_k, int size_n, int num_bits) -> Tensor"); + m.impl("gptq_marlin_repack", torch::kCUDA, &gptq_marlin_repack); + + m.def("awq_marlin_repack(Tensor! b_q_weight, int size_k, int size_n, int num_bits) -> Tensor"); + m.impl("awq_marlin_repack", torch::kCUDA, &awq_marlin_repack); + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! cumsum_buffer, bool " + "pad_sorted_token_ids) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + + m.def("topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + m.def( + "moe_fused_gate(Tensor input, Tensor bias, int num_expert_group, int topk_group, int topk, int " + "num_fused_shared_experts, float routed_scaling_factor, bool apply_routed_scaling_factor_on_output) -> " + "(Tensor[])"); + m.impl("moe_fused_gate", torch::kCUDA, &moe_fused_gate); + m.def( + "fp8_blockwise_scaled_grouped_mm(Tensor output, Tensor a_ptrs, Tensor b_ptrs, Tensor out_ptrs, Tensor " + "a_scales_ptrs, Tensor b_scales_ptrs, Tensor a, Tensor b, Tensor scales_a, Tensor scales_b, Tensor " + "stride_a, Tensor stride_b, Tensor stride_c, Tensor layout_sfa, Tensor layout_sfb, Tensor problem_sizes, Tensor " + "expert_offsets, Tensor workspace) -> ()"); + m.impl("fp8_blockwise_scaled_grouped_mm", torch::kCUDA, &fp8_blockwise_scaled_grouped_mm); + m.def( + "prepare_moe_input(Tensor topk_ids, Tensor expert_offsets, Tensor? blockscale_offsets, Tensor problem_sizes1," + " Tensor problem_sizes2, Tensor input_permutation, Tensor output_permutation, int num_experts, int n, int k) -> " + "()"); + m.impl("prepare_moe_input", torch::kCUDA, &prepare_moe_input); + + m.def("shuffle_rows(Tensor input, Tensor dst2src_map, Tensor output) -> ()"); + m.impl("shuffle_rows", torch::kCUDA, &shuffle_rows); + m.def("apply_shuffle_mul_sum(Tensor input, Tensor output, Tensor permutation, Tensor? factors) -> ()"); + m.impl("apply_shuffle_mul_sum", torch::kCUDA, &apply_shuffle_mul_sum); + + /* + * From csrc/moe/marlin_moe_wna16 + */ + m.def( + "moe_wna16_marlin_gemm(Tensor! a, Tensor? c_or_none," + "Tensor! b_q_weight, Tensor! b_scales, Tensor? b_zeros_or_none," + "Tensor? g_idx_or_none, Tensor? perm_or_none, Tensor! workspace," + "Tensor sorted_token_ids," + "Tensor! expert_ids, Tensor! num_tokens_past_padded," + "Tensor! topk_weights, int moe_block_size, int top_k, " + "bool mul_topk_weights, bool is_ep, int b_q_type_id," + "int size_m, int size_n, int size_k," + "bool is_k_full, bool use_atomic_add," + "bool use_fp32_reduce, bool is_zp_float) -> Tensor"); + m.impl("moe_wna16_marlin_gemm", torch::kCUDA, &moe_wna16_marlin_gemm); + + /* + * From csrc/moe/cutlass_moe/w4a8 + */ + m.def( + "get_cutlass_w4a8_moe_mm_data(Tensor topk_ids, Tensor! expert_offsets, " + " Tensor! problem_sizes1, Tensor! problem_sizes2, " + " Tensor! input_permutation, " + " Tensor! output_permutation, int num_experts, " + " int n, int k) -> ()"); + m.impl("get_cutlass_w4a8_moe_mm_data", torch::kCUDA, &get_cutlass_w4a8_moe_mm_data); + + m.def( + "cutlass_w4a8_moe_mm(Tensor! d, Tensor a, Tensor b, " + " Tensor a_scales, Tensor b_scales, Tensor expert_offsets, " + " Tensor problem_sizes, Tensor a_strides, " + " Tensor b_strides, Tensor d_strides, Tensor s_strides," + " int chunk_size, int topk) -> ()"); + m.impl("cutlass_w4a8_moe_mm", torch::kCUDA, &cutlass_w4a8_moe_mm); + + /* + * From csrc/speculative + */ + m.def( + "tree_speculative_sampling_target_only(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor uniform_samples, Tensor uniform_samples_for_final_sampling, Tensor target_probs, Tensor draft_probs, " + "float threshold_single, float threshold_acc, " + "bool deterministic, int cuda_stream) -> ()"); + m.impl("tree_speculative_sampling_target_only", torch::kCUDA, &tree_speculative_sampling_target_only); + + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict, int cuda_stream) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num, int tree_mask_mode) -> " + "()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); + + m.def( + "segment_packbits(Tensor x, Tensor input_indptr, Tensor output_indptr, Tensor! y, int batch_size, " + "int cuda_stream) -> ()"); + m.impl("segment_packbits", torch::kCUDA, &segment_packbits); + + /* + * From csrc/kvcacheio + */ + m.def( + "transfer_kv_per_layer(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int item_size, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer", torch::kCUDA, &transfer_kv_per_layer); + m.def( + "transfer_kv_per_layer_pf_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_pf_lf", torch::kCUDA, &transfer_kv_per_layer_pf_lf); + m.def( + "transfer_kv_all_layer(Tensor src_k_layers, Tensor dst_k_layers, Tensor src_v_layers, Tensor dst_v_layers, " + "Tensor src_indices, Tensor dst_indices, int item_size, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer", torch::kCUDA, &transfer_kv_all_layer); + m.def( + "transfer_kv_all_layer_lf_pf(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_pf", torch::kCUDA, &transfer_kv_all_layer_lf_pf); + m.def( + "transfer_kv_per_layer_mla(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int item_size, int " + "block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla", torch::kCUDA, &transfer_kv_per_layer_mla); + m.def( + "transfer_kv_per_layer_mla_pf_lf(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int layer_id, " + "int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla_pf_lf", torch::kCUDA, &transfer_kv_per_layer_mla_pf_lf); + m.def( + "transfer_kv_all_layer_mla(Tensor src_layers, Tensor dst_layers, Tensor src_indices, Tensor dst_indices, int " + "item_size, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla", torch::kCUDA, &transfer_kv_all_layer_mla); + m.def( + "transfer_kv_all_layer_mla_lf_pf(Tensor src_layers, Tensor dst, Tensor src_indices, Tensor dst_indices, " + "int item_size, int dst_layout_dim, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla_lf_pf", torch::kCUDA, &transfer_kv_all_layer_mla_lf_pf); + m.def( + "transfer_kv_direct(Tensor[] src_layers, Tensor[] dst_layers, Tensor src_indices, Tensor dst_indices, int " + "page_size) -> ()"); + m.impl("transfer_kv_direct", torch::kCUDA, &transfer_kv_direct); + m.def( + "transfer_kv_per_layer_direct_pf_lf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int layer_id, int page_size)->() "); + m.impl("transfer_kv_per_layer_direct_pf_lf", torch::kCUDA, &transfer_kv_per_layer_direct_pf_lf); + m.def( + "transfer_kv_all_layer_direct_lf_pf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int page_size) ->() "); + m.impl("transfer_kv_all_layer_direct_lf_pf", torch::kCUDA, &transfer_kv_all_layer_direct_lf_pf); + + /* + * From csrc/memory + */ + m.def("store_kv_cache(Tensor k_cache, Tensor v_cache, Tensor out_loc, Tensor k, Tensor v) -> ()"); + m.impl("store_kv_cache", &store_kv_cache); + + /* + * From FlashInfer + */ + m.def( + "bmm_fp8(Tensor A, Tensor B, Tensor! D, Tensor A_scale, Tensor B_scale, Tensor workspace_buffer, int " + "cublas_handle, int cuda_stream) -> ()", + {at::Tag::needs_fixed_stride_order}); + m.impl("bmm_fp8", torch::kCUDA, &bmm_fp8); + + m.def( + "min_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? maybe_min_p_arr, float " + "min_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("min_p_sampling_from_probs", torch::kCUDA, &min_p_sampling_from_probs); + + m.def("top_k_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_k_arr, int top_k_val) -> ()"); + m.impl("top_k_renorm_probs", torch::kCUDA, &top_k_renorm_probs); + + m.def("top_p_renorm_probs(Tensor probs, Tensor! renorm_probs, Tensor? maybe_top_p_arr, float top_p_val) -> ()"); + m.impl("top_p_renorm_probs", torch::kCUDA, &top_p_renorm_probs); + + m.def( + "top_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? " + "maybe_top_p_arr, float top_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("top_p_sampling_from_probs", torch::kCUDA, &top_p_sampling_from_probs); + + m.def( + "top_k_top_p_sampling_from_probs(Tensor probs, Tensor output, Tensor? maybe_indices, Tensor? maybe_top_k_arr, " + "float top_k_val, Tensor? maybe_top_p_arr, float top_p_val, bool deterministic, Generator? gen) -> ()"); + m.impl("top_k_top_p_sampling_from_probs", torch::kCUDA, &top_k_top_p_sampling_from_probs); + + m.def("top_k_mask_logits(Tensor logits, Tensor mask_logits, Tensor? maybe_top_k_arr, int top_k_val) -> ()"); + m.impl("top_k_mask_logits", torch::kCUDA, &top_k_mask_logits); + + /* + * From Sparse Flash Attention + */ + m.def( + "fwd_sparse(Tensor! q, Tensor k, Tensor v, " + "Tensor block_count, Tensor block_offset, Tensor column_count, Tensor column_index, " + "Tensor!? out, Tensor? alibi_slopes, " + "float p_dropout, float softmax_scale, bool is_causal, " + "float softcap, bool return_softmax, Generator? gen)" + "-> Tensor[]"); + m.impl("fwd_sparse", torch::kCUDA, &flash::mha_fwd_sparse); + + m.def( + "varlen_fwd_sparse(Tensor! q, Tensor k, Tensor v, " + "Tensor block_count, Tensor block_offset, Tensor column_count, Tensor column_index, " + "Tensor!? out, Tensor cu_seqlens_q, " + "Tensor cu_seqlens_k, Tensor? seqused_k, Tensor? alibi_slopes, " + "int max_seqlen_q, int max_seqlen_k, float p_dropout, float softmax_scale, bool zero_tensors, " + "bool is_causal, float softcap, bool return_softmax, " + "Generator? gen) -> Tensor[]"); + m.impl("varlen_fwd_sparse", torch::kCUDA, &flash::mha_varlen_fwd_sparse); + + // Sparse Attention utils + m.def( + "convert_vertical_slash_indexes(" + " Tensor! block_count, Tensor! block_offset, " + " Tensor! column_count, Tensor! column_index, " + " Tensor q_seqlens, Tensor q_seqlens, " + " Tensor vertical_indexes, Tensor slash_indexes, " + " int context_size, int block_size_M, int block_size_N, " + " bool causal) -> ()"); + m.impl("convert_vertical_slash_indexes", torch::kCUDA, &convert_vertical_slash_indexes); + + m.def( + "convert_vertical_slash_indexes_mergehead(" + " Tensor! block_count, Tensor! block_offset, " + " Tensor! column_count, Tensor! column_index, " + " Tensor q_seqlens, Tensor q_seqlens, " + " Tensor vertical_indexes, Tensor slash_indexes, " + " Tensor vertical_indices_count, Tensor slash_indices_count, " + " int context_size, int block_size_M, int block_size_N, " + " bool causal) -> ()"); + m.impl("convert_vertical_slash_indexes_mergehead", torch::kCUDA, &convert_vertical_slash_indexes_mergehead); + + /* + * From csrc/grammar + */ + m.def("apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()"); + m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); + + /* + * From csrc/gemm (QServe) + */ + m.def( + "qserve_w4a8_per_chn_gemm(Tensor _in_feats, Tensor _kernel, Tensor _wscales, Tensor _ascales, Tensor _w_szs, " + "Tensor _a_ssums, Tensor! _out_feats) -> ()"); + m.impl("qserve_w4a8_per_chn_gemm", torch::kCUDA, &qserve_w4a8_per_chn_gemm); + + m.def( + "qserve_w4a8_per_group_gemm(Tensor _in_feats, Tensor _kernel, Tensor _zeros, Tensor _scales_i8, Tensor _wscales, " + "Tensor _ascales, Tensor! _out_feats) -> ()"); + m.impl("qserve_w4a8_per_group_gemm", torch::kCUDA, &qserve_w4a8_per_group_gemm); + + m.def("copy_to_gpu_no_ce(Tensor input, Tensor! output) -> ()"); + m.impl("copy_to_gpu_no_ce", torch::kCUDA, ©_to_gpu_no_ce); + m.def("concat_mla_k(Tensor! k, Tensor k_nope, Tensor k_rope) -> ()"); + m.impl("concat_mla_k", torch::kCUDA, &concat_mla_k); + + /* + * From csrc/mamba + */ + m.def( + "causal_conv1d_update(Tensor! x," + "Tensor! conv_state," + "Tensor! weight," + "Tensor? bias_," + "bool silu_activation," + "Tensor? cache_seqlens_," + "Tensor? conv_state_indices," + "int pad_slot_id) -> ()"); + m.impl("causal_conv1d_update", torch::kCUDA, &causal_conv1d_update); + + m.def( + "causal_conv1d_fwd(Tensor! x, Tensor! weight," + "Tensor? bias_," + "Tensor!? conv_states," + "Tensor? query_start_loc," + "Tensor? cache_indices," + "Tensor? has_initial_state," + "bool silu_activation," + "int pad_slot_id) -> ()"); + m.impl("causal_conv1d_fwd", torch::kCUDA, &causal_conv1d_fwd); +} + +REGISTER_EXTENSION(common_ops) diff --git a/sgl-kernel/csrc/common_extension_rocm.cc b/sgl-kernel/csrc/common_extension_rocm.cc new file mode 100644 index 000000000..f4e14d0d5 --- /dev/null +++ b/sgl-kernel/csrc/common_extension_rocm.cc @@ -0,0 +1,176 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_EXPAND(sgl_kernel, m) { + /* + * From csrc/activation + */ + m.def("silu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("silu_and_mul", torch::kCUDA, &silu_and_mul); + + m.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul); + + m.def("gelu_and_mul(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_and_mul", torch::kCUDA, &gelu_and_mul); + + m.def("gelu_quick(Tensor! out, Tensor input) -> ()"); + m.impl("gelu_quick", torch::kCUDA, &gelu_quick); + + /* + * From csrc/allreduce + */ + m.def( + "init_custom_ar(Tensor meta, Tensor rank_data, " + "str[] handles, int[] offsets, int rank, " + "bool full_nvlink) -> int"); + m.impl("init_custom_ar", torch::kCUDA, &init_custom_ar); + + m.def("all_reduce_reg(int fa, Tensor inp, Tensor! out) -> ()"); + m.impl("all_reduce_reg", torch::kCUDA, &all_reduce_reg); + + m.def( + "all_reduce_unreg(int fa, Tensor inp, Tensor reg_buffer, Tensor! out) -> " + "()"); + m.impl("all_reduce_unreg", torch::kCUDA, &all_reduce_unreg); + + m.def("dispose", &dispose); + + m.def("meta_size", &meta_size); + + m.def( + "register_buffer(int fa, Tensor t, str[] handles, " + "int[] offsets) -> ()"); + m.impl("register_buffer", torch::kCUDA, ®ister_buffer); + + m.def("get_graph_buffer_ipc_meta", &get_graph_buffer_ipc_meta); + m.def("register_graph_buffers", ®ister_graph_buffers); + + m.def("allocate_meta_buffer", &allocate_meta_buffer); + m.impl("allocate_meta_buffer", torch::kCUDA, &allocate_meta_buffer); + + m.def("get_meta_buffer_ipc_handle", &get_meta_buffer_ipc_handle); + m.impl("get_meta_buffer_ipc_handle", torch::kCPU, &get_meta_buffer_ipc_handle); + + // quick allreduce +#ifdef USE_ROCM + m.def( + "qr_all_reduce(int fa, Tensor inp, Tensor out, int quant_level, bool " + "cast_bf2half) -> ()"); + m.impl("qr_all_reduce", torch::kCUDA, &qr_all_reduce); + + m.def("init_custom_qr", &init_custom_qr); + m.def("qr_destroy", &qr_destroy); + + m.def("qr_get_handle", &qr_get_handle); + + m.def("qr_open_handles(int _fa, Tensor[](b!) handles) -> ()"); + m.impl("qr_open_handles", torch::kCPU, &qr_open_handles); + + // Max input size in bytes + m.def("qr_max_size", &qr_max_size); +#endif + + /* + * From csrc/moe + */ + m.def( + "moe_align_block_size(Tensor topk_ids, int num_experts, int block_size, Tensor! sorted_token_ids, Tensor! " + "experts_ids, Tensor! num_tokens_post_pad, Tensor! cumsum_buffer, bool " + "pad_sorted_token_ids) -> ()"); + m.impl("moe_align_block_size", torch::kCUDA, &moe_align_block_size); + + m.def("topk_softmax(Tensor! topk_weights, Tensor! topk_indices, Tensor gating_output, bool renormalize) -> ()"); + m.impl("topk_softmax", torch::kCUDA, &topk_softmax); + + /* + * From csrc/speculative + */ + m.def( + "verify_tree_greedy(Tensor! predicts, Tensor! accept_index, Tensor! accept_token_num, " + "Tensor candidates, Tensor retrive_index, Tensor retrive_next_token, Tensor retrive_next_sibling, " + "Tensor target_predict, int cuda_stream) -> ()"); + m.impl("verify_tree_greedy", torch::kCUDA, &verify_tree_greedy); + + m.def( + "build_tree_kernel_efficient(Tensor parent_list, Tensor selected_index, Tensor verified_seq_len, " + "Tensor! tree_mask, Tensor! positions, Tensor! retrive_index, Tensor! retrive_next_token, " + "Tensor! retrive_next_sibling, int topk, int depth, int draft_token_num, int tree_mask_mode) -> " + "()"); + m.impl("build_tree_kernel_efficient", torch::kCUDA, &build_tree_kernel_efficient); + + /* + * From XGrammar + */ + m.def("apply_token_bitmask_inplace_cuda(Tensor logits, Tensor bitmask, Tensor? indices=None) -> ()"); + m.impl("apply_token_bitmask_inplace_cuda", &ApplyTokenBitmaskInplace); + + /* + * From csrc/kvcacheio + */ + m.def( + "transfer_kv_per_layer(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int item_size, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer", torch::kCUDA, &transfer_kv_per_layer); + m.def( + "transfer_kv_per_layer_pf_lf(Tensor src_k, Tensor dst_k, Tensor src_v, Tensor dst_v, Tensor src_indices, Tensor " + "dst_indices, int layer_id, int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_pf_lf", torch::kCUDA, &transfer_kv_per_layer_pf_lf); + m.def( + "transfer_kv_all_layer(Tensor src_k_layers, Tensor dst_k_layers, Tensor src_v_layers, Tensor dst_v_layers, " + "Tensor src_indices, Tensor dst_indices, int item_size, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer", torch::kCUDA, &transfer_kv_all_layer); + m.def( + "transfer_kv_all_layer_lf_pf(Tensor src_k_layers, Tensor dst_k, Tensor src_v_layers, Tensor dst_v, " + "Tensor src_indices, Tensor dst_indices, int item_size, int dst_layout_dim, int num_layers, int block_quota, int " + "num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_lf_pf", torch::kCUDA, &transfer_kv_all_layer_lf_pf); + m.def( + "transfer_kv_per_layer_mla(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int item_size, int " + "block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla", torch::kCUDA, &transfer_kv_per_layer_mla); + m.def( + "transfer_kv_per_layer_mla_pf_lf(Tensor src, Tensor dst, Tensor src_indices, Tensor dst_indices, int layer_id, " + "int item_size, int src_layout_dim, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_per_layer_mla_pf_lf", torch::kCUDA, &transfer_kv_per_layer_mla_pf_lf); + m.def( + "transfer_kv_all_layer_mla(Tensor src_layers, Tensor dst_layers, Tensor src_indices, Tensor dst_indices, int " + "item_size, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla", torch::kCUDA, &transfer_kv_all_layer_mla); + m.def( + "transfer_kv_all_layer_mla_lf_pf(Tensor src_layers, Tensor dst, Tensor src_indices, Tensor dst_indices, " + "int item_size, int dst_layout_dim, int num_layers, int block_quota, int num_warps_per_block) -> ()"); + m.impl("transfer_kv_all_layer_mla_lf_pf", torch::kCUDA, &transfer_kv_all_layer_mla_lf_pf); + m.def( + "transfer_kv_direct(Tensor[] src_layers, Tensor[] dst_layers, Tensor src_indices, Tensor dst_indices, int " + "page_size) -> ()"); + m.impl("transfer_kv_direct", torch::kCUDA, &transfer_kv_direct); + m.def( + "transfer_kv_per_layer_direct_pf_lf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int layer_id, int page_size)->() "); + m.impl("transfer_kv_per_layer_direct_pf_lf", torch::kCUDA, &transfer_kv_per_layer_direct_pf_lf); + m.def( + "transfer_kv_all_layer_direct_lf_pf(Tensor[] src_ptrs, Tensor[] dst_ptrs, Tensor src_indices, " + "Tensor dst_indices, int page_size) ->() "); + m.impl("transfer_kv_all_layer_direct_lf_pf", torch::kCUDA, &transfer_kv_all_layer_direct_lf_pf); +} + +REGISTER_EXTENSION(common_ops) diff --git a/sgl-kernel/csrc/cpu/CMakeLists.txt b/sgl-kernel/csrc/cpu/CMakeLists.txt new file mode 100755 index 000000000..421fbedab --- /dev/null +++ b/sgl-kernel/csrc/cpu/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) +project(sgl_kernel) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(Python COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} REQUIRED) + +execute_process( + COMMAND ${Python_EXECUTABLE} + -c "import torch; print(torch.utils.cmake_prefix_path)" + OUTPUT_VARIABLE TORCH_PY_PREFIX + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +message(STATUS ${TORCH_PY_PREFIX}) +list(APPEND CMAKE_PREFIX_PATH ${TORCH_PY_PREFIX}/Torch) +find_package(Torch REQUIRED) + +include_directories( + ${TORCH_INCLUDE_DIRS} + ${TORCH_INSTALL_PREFIX}/include + ${Python_INCLUDE_DIRS} + ${CMAKE_CURRENT_SOURCE_DIR}/../../csrc + ${CMAKE_CURRENT_SOURCE_DIR}/../../include +) + +# Platform-specific library directory +if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") + set(PLAT_LIB_DIR "/usr/lib/x86_64-linux-gnu") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(PLAT_LIB_DIR "/usr/lib/aarch64-linux-gnu") + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le|ppc64") + set(PLAT_LIB_DIR "/usr/lib/powerpc64le-linux-gnu") +else() + set(PLAT_LIB_DIR "/usr/lib/${CMAKE_SYSTEM_PROCESSOR}-linux-gnu") +endif() +link_directories(${PLAT_LIB_DIR}) + +# Conda library path support +if(DEFINED ENV{CONDA_PREFIX}) + set(CONDA_LIB_DIR "$ENV{CONDA_PREFIX}/lib") + message(STATUS "Using Conda lib dir: ${CONDA_LIB_DIR}") + link_directories(${CONDA_LIB_DIR}) + set(CONDA_INCLUDE_DIR "$ENV{CONDA_PREFIX}/include") + include_directories(${CONDA_INCLUDE_DIR}) + + # Look for libnuma in Conda's lib directory + find_library(NUMA_LIB numa HINTS "${CONDA_LIB_DIR}") + if(NUMA_LIB) + message(STATUS "Found libnuma: ${NUMA_LIB}") + else() + message(FATAL_ERROR "libnuma not found in Conda environment at ${CONDA_LIB_DIR}\n" + "Please install it using: conda install libnuma numactl\n") + endif() +endif() + +file(GLOB SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +if(NOT DEFINED ENV{SGLANG_CPU_FP8_CVT_FTZ}) + set(ENV{SGLANG_CPU_FP8_CVT_FTZ} "1") +endif() + +if("$ENV{SGLANG_CPU_FP8_CVT_FTZ}" STREQUAL "1") + message(STATUS "Enabling macro: SGLANG_CPU_FP8_CVT_FTZ") + add_compile_definitions(SGLANG_CPU_FP8_CVT_FTZ) +endif() + +add_compile_options( + -O3 + -Wno-unknown-pragmas + -march=native + -fopenmp +) + +Python_add_library(common_ops MODULE USE_SABI ${SKBUILD_SABI_VERSION} WITH_SOABI ${SOURCES}) +target_link_libraries(common_ops PRIVATE ${TORCH_LIBRARIES} ${NUMA_LIB}) +target_include_directories(common_ops PRIVATE ${TORCH_INCLUDE_DIRS}) + +install(TARGETS common_ops + LIBRARY DESTINATION sgl_kernel +) diff --git a/sgl-kernel/csrc/cpu/activation.cpp b/sgl-kernel/csrc/cpu/activation.cpp new file mode 100644 index 000000000..70756776b --- /dev/null +++ b/sgl-kernel/csrc/cpu/activation.cpp @@ -0,0 +1,135 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +void act_and_mul_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + int64_t num_tokens, + int64_t dim, + const func_t& f, + const vec_func_t& vf) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int64_t kVecSize = bVec::size(); + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + const scalar_t* __restrict__ input_ptr = input + i * 2 * dim; + const scalar_t* __restrict__ input_other_ptr = input_ptr + dim; + scalar_t* __restrict__ output_ptr = output + i * dim; + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= dim - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec y_bvec = bVec::loadu(input_other_ptr + d); + fVec y_fvec0, y_fvec1; + std::tie(y_fvec0, y_fvec1) = at::vec::convert_to_float(y_bvec); + + x_fvec0 = vf(x_fvec0); + x_fvec1 = vf(x_fvec1); + + x_fvec0 = x_fvec0 * y_fvec0; + x_fvec1 = x_fvec1 * y_fvec1; + + x_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + x_bvec.store(output_ptr + d); + } +#pragma GCC unroll 4 + for (; d < dim; ++d) { + float x_val = static_cast(input_ptr[d]); + float y_val = static_cast(input_other_ptr[d]); + output_ptr[d] = f(x_val) * y_val; + } + } + }); +} + +} // anonymous namespace + +// input : {num_tokens, 2 * d} +// output : {num_tokens, d} +at::Tensor silu_and_mul_cpu(at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::silu_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "silu_and_mul", [&] { + using Vec = at::vec::Vectorized; + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [](float x) { return x / (1.f + std::exp(-x)); }, + [](Vec x) { return x / (Vec(1.f) + x.neg().exp()); }); + }); + return out; +} + +at::Tensor gelu_tanh_and_mul_cpu(const at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::gelu_tanh_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + const float sqrt_2_div_pi = std::sqrt(2.f / M_PI); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_and_mul", [&] { + using Vec = at::vec::Vectorized; + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [sqrt_2_div_pi](float x) { + float x3 = x * x * x; + float tanh_arg = sqrt_2_div_pi * (x + 0.044715f * x3); + return 0.5f * x * (1.f + std::tanh(tanh_arg)); + }, + [sqrt_2_div_pi](Vec x) { + Vec x3 = x * x * x; + Vec tanh_arg = Vec(sqrt_2_div_pi) * (x + Vec(0.044715f) * x3); + return Vec(0.5f) * x * (Vec(1.f) + tanh_arg.tanh()); + }); + }); + + return out; +} + +at::Tensor gelu_and_mul_cpu(const at::Tensor& input) { + RECORD_FUNCTION("sgl-kernel::gelu_and_mul_cpu", std::vector({input})); + auto sizes = input.sizes().vec(); + int64_t last_dim = input.ndimension() - 1; + int64_t d = sizes[last_dim] / 2; + sizes[last_dim] = d; + int64_t num_tokens = input.numel() / input.size(-1); + at::Tensor out = at::empty(sizes, input.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "gelu_and_mul", [&] { + using Vec = at::vec::Vectorized; + const float inv_sqrt2 = 1.0f / std::sqrt(2.0f); + act_and_mul_kernel_impl( + out.data_ptr(), + input.data_ptr(), + num_tokens, + d, + [inv_sqrt2](float x) { return 0.5f * x * (1.f + std::erf(x * inv_sqrt2)); }, + [inv_sqrt2](Vec x) { return Vec(0.5f) * x * (Vec(1.f) + (x * Vec(inv_sqrt2)).erf()); }); + }); + + return out; +} diff --git a/sgl-kernel/csrc/cpu/bmm.cpp b/sgl-kernel/csrc/cpu/bmm.cpp new file mode 100644 index 000000000..9e809a464 --- /dev/null +++ b/sgl-kernel/csrc/cpu/bmm.cpp @@ -0,0 +1,123 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +void bmm_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const scalar_t* __restrict__ mat2, + int64_t B, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideB, + int64_t mat1_strideM, + int64_t out_strideB, + int64_t out_strideM, + float scale = 0.f) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // mat2 contiguous in [B, N, K] + int64_t mat2_strideB = N * K; + int64_t mat2_strideN = K; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [B, MB, NB] + at::parallel_for(0, B * MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, mb{0}, nb{0}; + data_index_init(begin, bs, B, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + bs * mat1_strideB + mb_start * mat1_strideM, + /* B */ mat2 + bs * mat2_strideB + nb_start * mat2_strideN /* nb * BLOCK_N * K */, + /* C */ out + bs * out_strideB + mb_start * out_strideM + nb_start, + /* Ctmp*/ Ctmp, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(bs, B, mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +// mat1 : [B, M, K] +// mat2 : [B, N, K] or [B, OC, IC] +// out : [B, M, N] +// scale: [] 0-dim tensor for per tensor quant +// +void bmm_cpu( + at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale) { + RECORD_FUNCTION("sgl-kernel::bmm_cpu", std::vector({out, mat1, mat2})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + // input and out could be non-contiguous + // weight needs to be contiguous in [OC, IC] order + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(out); + CHECK_INPUT(mat2); + CHECK_DIM(3, out); + CHECK_DIM(3, mat1); + CHECK_DIM(3, mat2); + + int64_t B = mat1.size(0); + int64_t M = mat1.size(1); + int64_t N = mat2.size(1); + int64_t K = mat1.size(2); + + TORCH_CHECK(!scale.has_value(), "bmm: do not support fp8 weight for now.") + TORCH_CHECK(N % 32 == 0, "tinygemm requires N to be 32x."); + + int64_t mat1_strideB = mat1.stride(0); + int64_t mat1_strideM = mat1.stride(1); + int64_t out_strideB = out.stride(0); + int64_t out_strideM = out.stride(1); + + // check shapes + TORCH_CHECK(mat2.size(0) == B && mat2.size(2) == K, "bmm: mat2 shape mismatch!"); + TORCH_CHECK(out.size(0) == B && out.size(1) == M, "bmm: out shape mismatch!"); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "bmm_kernel_impl", [&] { + bmm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + B, + M, + N, + K, + mat1_strideB, + mat1_strideM, + out_strideB, + out_strideM); + }); +} diff --git a/sgl-kernel/csrc/cpu/common.h b/sgl-kernel/csrc/cpu/common.h new file mode 100644 index 000000000..0fb132607 --- /dev/null +++ b/sgl-kernel/csrc/cpu/common.h @@ -0,0 +1,324 @@ +#pragma once + +#include +#include +#include + +#if defined(_OPENMP) +#include +#endif + +namespace { + +// dispatch bool +#define AT_DISPATCH_BOOL(BOOL_V, BOOL_NAME, ...) \ + [&] { \ + if (BOOL_V) { \ + constexpr bool BOOL_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool BOOL_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() + +// dispatch: bfloat16, float16, int8_t, fp8_e4m3 +#define CPU_DISPATCH_PACKED_TYPES(TYPE, ...) \ + [&] { \ + switch (TYPE) { \ + case at::ScalarType::BFloat16: { \ + using packed_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using packed_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Char: { \ + using packed_t = int8_t; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Float8_e4m3fn: { \ + using packed_t = at::Float8_e4m3fn; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + }() + +// dispatch with mixed dtypes (TYPE1, TYPE2): +// TYPE1: the primary dtype (input, output, weight); +// TYPE2: the secondary dtype (bias, etc.). +#define CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(TYPE1, TYPE2, ...) \ + [&] { \ + if (TYPE2 == at::kFloat) { \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = float; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = float; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + } else { \ + TORCH_CHECK(TYPE1 == TYPE2); \ + switch (TYPE1) { \ + case at::ScalarType::BFloat16: { \ + using scalar_t = at::BFloat16; \ + using param_t = at::BFloat16; \ + return __VA_ARGS__(); \ + } \ + case at::ScalarType::Half: { \ + using scalar_t = at::Half; \ + using param_t = at::Half; \ + return __VA_ARGS__(); \ + } \ + default: \ + TORCH_CHECK(false, "Unsupported floating data type.\n"); \ + } \ + } \ + }() + +#define UNUSED(x) (void)(x) + +#define CHECK_CPU(x) TORCH_CHECK(x.device().type() == at::kCPU, #x " must be a CPU tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CPU(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CPU(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +// [NB] Parallel Routines +// +// * at::parallel_for - applies for most of generic use cases, this will be compiled +// against openmp in default torch release. +// +// * parallel_for - same function as above, can choose payload partition scheme in +// balance211. +// +// * parallel_2d - parallel for 2 dimensions, used in GEMM, etc. +// this one will do payload balance across 2 dimensions. +// + +// grain size for each thread +constexpr int GRAIN_SIZE = 1024; + +template ::value, int>::type = 0> +inline T div_up(T x, T y) { + return (x + y - 1) / y; +} + +// you can only use at::get_thread_num() with at::parallel_for() +// as it is lazy initialized, otherwise it will always return 0. +inline int get_thread_num() { +#if defined(_OPENMP) + return omp_get_thread_num(); +#else + return 0; +#endif +} + +// balance payload across each thread +template +inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) { +#if 0 + // onednn partition pattern + T& n_my = n_end; + if (nth <= 1 || n == 0) { + n_start = 0; + n_my = n; + } else { + T n1 = div_up(n, nth); + T n2 = n1 - 1; + T T1 = n - n2 * nth; + n_my = ith < T1 ? n1 : n2; + n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2; + } + n_end += n_start; +#else + // pytorch aten partition pattern + T n_my = div_up(n, nth); + n_start = ith * n_my; + n_end = std::min(n_start + n_my, n); +#endif +} + +template +inline void parallel_for(int n, const func_t& f) { +#if defined(_OPENMP) +#pragma omp parallel + { + int nth = omp_get_num_threads(); + int ith = omp_get_thread_num(); + int tbegin, tend; + balance211(n, nth, ith, tbegin, tend); + f(tbegin, tend); + } +#else + f(0, n); +#endif +} + +// for 1d parallel, use `actual_nth` +// for 2d parallel, use even nths, e.g. 43->42 +int inline adjust_num_threads(int m) { + int actual_nth = at::get_num_threads(); + if (m == 1) { + return actual_nth; + } + return std::max(1, (actual_nth >> 1) * 2); +} + +template +inline void parallel_2d(int m, int n, const func_t& f) { + // make sure we have even num_threads + int nth = adjust_num_threads(m); + + // [NOTE] thread blocking: + // + // 1) prefer square block per thread + // 2) use even number of CPU cores + // 3) use all `num_threads` cores + // + // we have: + // TM * TN = T + // BM / TM = BN / TN + // then: + // TM = ((BM / BN) * T) ^ 0.5 + // + float r = float(m) / n; + int nth_m = std::ceil(std::sqrt(r * nth)); + int nth_n = 1; + for (; nth_m > 0; --nth_m) { + nth_n = nth / nth_m; + if (nth_m * nth_n == nth) { + break; + } + } + +#if defined(_OPENMP) +#pragma omp parallel num_threads(nth) + { + int ith = omp_get_thread_num(); + int ith_m = ith / nth_n; + int ith_n = ith % nth_n; + + int thread_block_m = div_up(m, nth_m); + int thread_block_n = div_up(n, nth_n); + + int begin_m = ith_m * thread_block_m; + int end_m = std::min(m, begin_m + thread_block_m); + int begin_n = ith_n * thread_block_n; + int end_n = std::min(n, begin_n + thread_block_n); + + f(begin_m, end_m, begin_n, end_n); + } +#else + f(0, m, 0, n); +#endif +} + +// limit max cache blocks +// when we need to do pre-unpack for weights, e.g. fp8 +#define MAX_CACHE_BLOCK_SIZE 4 + +template +inline int get_cache_blocks(int chunk_size) { + // L2 2MB and ratio of 50% + const int L2_size = 2048 * 1024 >> 1; + return std::max(1, int(L2_size / (chunk_size * sizeof(T)))); +} + +template <> +inline int get_cache_blocks(int chunk_size) { + // fp8 uses bf16 as accumulate type + int cache_block_size = get_cache_blocks(chunk_size); + return std::min(MAX_CACHE_BLOCK_SIZE, cache_block_size); +} + +// 2d sequential loop in range : [mb0, mb1), [nb0, nb1) +template +inline void loop_2d(int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1, int64_t chunk_size, const func_t& f) { + // get number of blocks for L2 in most inner loop + int64_t cache_blocks_nb = get_cache_blocks(chunk_size); + + // loop order: [NB / cache_blocks_nb, MB, cache_blocks_nb] + // TODO: implement reverse order of [MB / cache_blocks_mb, NB, cache_blocks_mb] + for (int64_t nbb = nb0; nbb < nb1; nbb += cache_blocks_nb) { + for (int64_t mb = mb0; mb < mb1; ++mb) { + for (int64_t nb = nbb; nb < std::min(nbb + cache_blocks_nb, nb1); ++nb) { + f(mb, nb, nb - nbb); + } + } + } +} + +// data indexing for dimension collapse +template +inline T data_index_init(T offset) { + return offset; +} + +template +inline T data_index_init(T offset, T& x, const T& X, Args&&... args) { + offset = data_index_init(offset, std::forward(args)...); + x = offset % X; + return offset / X; +} + +inline bool data_index_step() { + return true; +} + +template +inline bool data_index_step(T& x, const T& X, Args&&... args) { + if (data_index_step(std::forward(args)...)) { + x = ((x + 1) == X) ? 0 : (x + 1); + return x == 0; + } + return false; +} + +// forced unroll for perf critical path + +#if __has_attribute(always_inline) +#define ALWAYS_INLINE __attribute__((__always_inline__)) inline +#else +#define ALWAYS_INLINE inline +#endif + +template +struct Unroll { + template + ALWAYS_INLINE void operator()(const Func& f, Args... args) const { + Unroll{}(f, args...); + f(std::integral_constant{}, args...); + } +}; + +template <> +struct Unroll<1> { + template + ALWAYS_INLINE void operator()(const Func& f, Args... args) const { + f(std::integral_constant{}, args...); + } +}; + +} // anonymous namespace diff --git a/sgl-kernel/csrc/cpu/decode.cpp b/sgl-kernel/csrc/cpu/decode.cpp new file mode 100644 index 000000000..ae5ac51c8 --- /dev/null +++ b/sgl-kernel/csrc/cpu/decode.cpp @@ -0,0 +1,1575 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE] TODO list for this kernel: +// 1. tune the value for BLOCK_N +// 2. planning for {batches, num_heads, num_kv_splits} +// and use actual num_kv_splits for small seq length +// 3. try fast impl of `.tanh()` +// 4. provide amx kernel for index_gemm_kernel_nn when M = 16 +// + +#if defined(CPU_CAPABILITY_AVX512) +// key: from [N, 32] to [32/2, N, 2] +// val: from [N, 32] to [N/2, 32, 2] +template +inline void pack_vnni_Nx32( + scalar_t* __restrict__ dst0, + scalar_t* __restrict__ dst1, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int ld_src, + int ld_dst0, + int ld_dst1, + bool convert_v) { + __m512i vinputs[16]; + int n = 0; + for (; n < N; ++n) { + vinputs[n] = _mm512_loadu_si512(src + ind[n] * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack value, skip 64 elems for deepseek + // handle 2 vectors at a time from [2, 32] to [32, 2] + if (convert_v) { + for (int n = 0; n < 16; n += 2) { + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[n], vinputs[n + 1]); + _mm512_storeu_si512(dst1 + (n >> 1) * ld_dst1 * 2, d0); + _mm512_storeu_si512(dst1 + (n >> 1) * ld_dst1 * 2 + 32, d1); + } + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask = (1 << N) - 1; + for (int k = 0; k < 16; ++k) { + _mm512_mask_storeu_epi32(dst0 + k * ld_dst0 * 2, vmask, vinputs[k]); + } +} +#endif + +// [NOTE]: MLA vnni format conversion +// +// here we apply same strategy as `FlashMLA`: +// each kv_cache is loaded once and packed twice (L2 cache hit) +// +// * for key: from [N, K/2, 2] to [K/2, N, 2] +// * for value: from [N/2, 2, Kv] to [N/2, Kv, 2] +// +template +void pack_vnni( + scalar_t* __restrict__ dst0, + scalar_t* __restrict__ dst1, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int Kv, + int ld_src, + int ld_dst0, + int ld_dst1) { +#if defined(CPU_CAPABILITY_AVX512) + const int NB = div_up(N, 16); + const int KB = K / 32; // no remainder + const int KBv = Kv / 32; // no remainder + + for (int nb = 0; nb < NB; ++nb) { + for (int kb = 0; kb < KB; ++kb) { + // handle 16x512bits each block + int nb_size = std::min(N - nb * 16, 16); + pack_vnni_Nx32( + /* dst0 */ dst0 + ((kb * 32) >> 1) * ld_dst0 * 2 + nb * 16 * 2, + /* dst1 */ dst1 + ((nb * 16) >> 1) * ld_dst1 * 2 + kb * 32 * 2, + /* src */ src + kb * 32, + /* ind */ ind + nb * 16, + /* N */ nb_size, + /* ld_src */ ld_src, + /* ld_dst0 */ ld_dst0, + /* ld_dst1 */ ld_dst1, + /* cvt_v */ kb < KBv); + } + } +#else + for (int n = 0; n < N; ++n) { + index_t index = ind[n]; + for (int k = 0; k < K / 2; ++k) { + for (int d = 0; d < 2; ++d) { + dst0[k * ld_dst0 * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d]; + } + } + } + // from [N/2, 2, K] to [N/2, K, 2] + for (int n = 0; n < (N >> 1) * 2; n += 2) { + index_t index0 = ind[n + 0]; + index_t index1 = ind[n + 1]; + for (int k = 0; k < Kv; ++k) { + dst1[(n >> 1) * ld_dst1 * 2 + k * 2 + 0] = src[index0 * ld_src + k]; + dst1[(n >> 1) * ld_dst1 * 2 + k * 2 + 1] = src[index1 * ld_src + k]; + } + } + if (N % 2 != 0) { + index_t index = ind[N - 1]; + for (int k = 0; k < Kv; ++k) { + dst1[(N >> 1) * ld_dst1 * 2 + k * 2 + 0] = src[index * ld_src + k]; + dst1[(N >> 1) * ld_dst1 * 2 + k * 2 + 1] = 0; + } + } +#endif +} + +template +inline void fill_stub(scalar_t* __restrict__ out, float val, int64_t size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + const Vec data_vec = Vec(static_cast(val)); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + data_vec.store(out + d); + } + if (size - d > 0) { + data_vec.store(out + d, size - d); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ acc, float s, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_fvec = fVec(s); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + fVec a_fvec0 = fVec::loadu(acc + d) * s_fvec; + fVec a_fvec1 = fVec::loadu(acc + d + fVec::size()) * s_fvec; + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(acc[d] * s); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ src, int64_t size) { + using bVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + int64_t d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + bVec out_bvec = bVec::loadu(src + d); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = src[d]; + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) { + static_assert(BLOCK_N % 32 == 0); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int COLS = BLOCK_N / 16; + auto store = [&](auto i) { + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + fVec a_fvec0 = fVec::loadu(input + col * 16); + fVec a_fvec1 = fVec::loadu(input + col * 16 + 16); + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + col * 16); + } + }; + Unroll{}(store); +} + +// GEMM handles query @ key (indexed) x scale +// A : [M, K] +// B : [N, K] indexed +// C : [M, N] +// +template +struct tinygemm_kernel_nt { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + for (int64_t m = 0; m < BLOCK_M; ++m) { + for (int64_t n = 0; n < BLOCK_N; ++n) { + float sum = 0.f; + int64_t b_idx = indices[n]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + for (int64_t k = 0; k < K; ++k) { + sum += scale * static_cast(A[m * lda + k]) * static_cast(B[b_idx * ldb + k]); + } + C[m * ldc + n] = sum; + } + } + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nt { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale = _mm512_set1_ps(scale); + + auto loadc = [&](auto i) { vc[i] = _mm512_setzero_ps(); }; + Unroll{}(loadc); + + // for main loop + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_loadu_si512(A + row * lda + k)); + } + if constexpr (row == 0) { + if constexpr (col + 1 < COLS) { + int64_t b_idx_prefetch = indices[col + 1]; + _mm_prefetch(B + b_idx_prefetch * ldb + k, _MM_HINT_T0); + } + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + vb[col] = (__m512bh)(_mm512_loadu_si512(B + b_idx * ldb + k)); + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + + // for remainder + auto compute2 = [&](auto i, int64_t k, __mmask32 mask) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_maskz_loadu_epi16(mask, A + row * lda + k)); + } + if constexpr (row == 0) { + int64_t b_idx = indices[col]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + vb[col] = (__m512bh)(_mm512_maskz_loadu_epi16(mask, B + b_idx * ldb + k)); + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + + int64_t k = 0; + for (; k <= K - 32; k += 32) { + Unroll{}(compute, k); + } + int64_t count = K - k; + if (count > 0) { + __mmask32 mask = (1ULL << count) - 1; + Unroll{}(compute2, k, mask); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + C[row * ldc + col] = _mm512_reduce_add_ps(_mm512_mul_ps(vc[i], vscale)); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NT(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nt::apply( \ + A + mb_start * lda, B, C + mb_start * ldc + nb_start, indices + nb_start, scale, lda, ldb, ldc, K, max_tokens); + +// this is used when N isn't multiple of 16, +// N corresponds to `head_size_v` which should be 16x +template +inline void tinygemm_kernel_nn_scalar( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + for (int64_t m = 0; m < M; ++m) { + for (int64_t n = 0; n < N; ++n) { + C[m * ldc + n] *= scale[m]; + for (int64_t k = 0; k < K; ++k) { + int64_t b_idx = indices[k]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + C[m * ldc + n] += A[m * lda + k] * static_cast(B[b_idx * ldb + n]); + } + } + } +} + +// GEMM handles v' * scale + attn @ value (indexed) +// A : [M, K] +// B : [K, N] indexed +// C ：[M, N] +// +template +struct tinygemm_kernel_nn { + static inline void apply( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + tinygemm_kernel_nn_scalar(A, B, C, indices, scale, BLOCK_M, BLOCK_N, K, lda, ldb, ldc, max_tokens); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const float* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + const float* __restrict__ scale, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t K, + int64_t max_tokens) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + __m512 va; + __m512 vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vscale; + + auto loadc = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" + if constexpr (col == 0) { + vscale = _mm512_set1_ps(scale[row]); + } +#pragma GCC diagnostic pop + vc[i] = _mm512_loadu_ps(C + row * ldc + col * 16); + vc[i] = _mm512_mul_ps(vc[i], vscale); + }; + Unroll{}(loadc); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_ps(A[row * lda + k]); + } + if constexpr (row == 0) { + if (k + 1 < K) { + int64_t b_idx_prefetch = indices[k + 1]; + _mm_prefetch(B + b_idx_prefetch * ldb + col * 16, _MM_HINT_T0); + } + int64_t b_idx = indices[k]; + TORCH_CHECK(b_idx < max_tokens, "token index out of scope!"); + + // for COLS = 2, 4, 6, 8 use 512 bit load + // for COLS = 1, 3, 5, 7 use 256 bit load + if constexpr (COLS % 2 == 0) { + if constexpr (col % 2 == 0) { + __m512i b16 = _mm512_loadu_si512(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col + 0] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 0)); + vb[col + 1] = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(b16, 1)); + } + } else { + __m256i b16 = _mm256_loadu_si256(reinterpret_cast(B + b_idx * ldb + col * 16)); + vb[col] = CVT_BF16_TO_FP32(b16); + } + } + vc[i] = _mm512_fmadd_ps(va, vb[col], vc[i]); + }; + + for (int64_t k = 0; k < K; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + _mm512_storeu_ps(C + row * ldc + col * 16, vc[i]); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start, \ + C + mb_start * ldc + nb_start, \ + indices, \ + scale + mb_start, \ + lda, \ + ldb, \ + ldc, \ + K, \ + max_tokens); + +template +void index_gemm_kernel_nt( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + // pattern: 1-8-8 + if (M == 1) { + constexpr int64_t BLOCK_N = 8; + const int64_t NB = div_up(N, BLOCK_N); + int64_t mb_start = 0, lda = 1, ldc = 1; + + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (nb_size) { + case 1: + LAUNCH_TINYGEMM_KERNEL_NT(1, 1); + break; + case 2: + LAUNCH_TINYGEMM_KERNEL_NT(1, 2); + break; + case 3: + LAUNCH_TINYGEMM_KERNEL_NT(1, 3); + break; + case 4: + LAUNCH_TINYGEMM_KERNEL_NT(1, 4); + break; + case 5: + LAUNCH_TINYGEMM_KERNEL_NT(1, 5); + break; + case 6: + LAUNCH_TINYGEMM_KERNEL_NT(1, 6); + break; + case 7: + LAUNCH_TINYGEMM_KERNEL_NT(1, 7); + break; + case 8: + LAUNCH_TINYGEMM_KERNEL_NT(1, 8); + break; + default: + TORCH_CHECK(false, "Unexpected block size, 1x", "nb_size"); + } + } + return; + } + + // pattern: 1-6-24 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 6; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NT(1, 1); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NT(1, 2); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NT(1, 3); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NT(1, 4); + break; + case 0x15: + LAUNCH_TINYGEMM_KERNEL_NT(1, 5); + break; + case 0x16: + LAUNCH_TINYGEMM_KERNEL_NT(1, 6); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NT(2, 1); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NT(2, 2); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NT(2, 3); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NT(2, 4); + break; + case 0x25: + LAUNCH_TINYGEMM_KERNEL_NT(2, 5); + break; + case 0x26: + LAUNCH_TINYGEMM_KERNEL_NT(2, 6); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NT(3, 1); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NT(3, 2); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NT(3, 3); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NT(3, 4); + break; + case 0x35: + LAUNCH_TINYGEMM_KERNEL_NT(3, 5); + break; + case 0x36: + LAUNCH_TINYGEMM_KERNEL_NT(3, 6); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NT(4, 1); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NT(4, 2); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NT(4, 3); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NT(4, 4); + break; + case 0x45: + LAUNCH_TINYGEMM_KERNEL_NT(4, 5); + break; + case 0x46: + LAUNCH_TINYGEMM_KERNEL_NT(4, 6); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void index_gemm_kernel_nn( + const float* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + const index_t* __restrict__ indices, + float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + int64_t max_tokens) { + constexpr int kVecSize = 16; + if ((N & (kVecSize - 1)) != 0) { + tinygemm_kernel_nn_scalar(A, B, C, indices, scale, M, N, K, lda, ldb, ldc, max_tokens); + return; + } + + // pattern: 1-8-8 + if (M == 1) { + constexpr int64_t BLOCK_N = 8 * kVecSize; + const int64_t NB = div_up(N, BLOCK_N); + int64_t mb_start = 0, lda = 1, ldc = 1; + + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (nb_size >> 4) { + case 1: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 2: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 3: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 4: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + case 5: + LAUNCH_TINYGEMM_KERNEL_NN(1, 80); + break; + case 6: + LAUNCH_TINYGEMM_KERNEL_NN(1, 96); + break; + case 7: + LAUNCH_TINYGEMM_KERNEL_NN(1, 112); + break; + case 8: + LAUNCH_TINYGEMM_KERNEL_NN(1, 128); + break; + default: + TORCH_CHECK(false, "Unexpected block size, 1x", "nb_size"); + } + } + return; + } + + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 6 * kVecSize; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + case 0x15: + LAUNCH_TINYGEMM_KERNEL_NN(1, 80); + break; + case 0x16: + LAUNCH_TINYGEMM_KERNEL_NN(1, 96); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NN(2, 16); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NN(2, 48); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + case 0x25: + LAUNCH_TINYGEMM_KERNEL_NN(2, 80); + break; + case 0x26: + LAUNCH_TINYGEMM_KERNEL_NN(2, 96); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NN(3, 16); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NN(3, 48); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + case 0x35: + LAUNCH_TINYGEMM_KERNEL_NN(3, 80); + break; + case 0x36: + LAUNCH_TINYGEMM_KERNEL_NN(3, 96); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NN(4, 16); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NN(4, 48); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + case 0x45: + LAUNCH_TINYGEMM_KERNEL_NN(4, 80); + break; + case 0x46: + LAUNCH_TINYGEMM_KERNEL_NN(4, 96); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void decode_set_kv_buffer( + scalar_t* __restrict__ k_buffer, + scalar_t* __restrict__ v_buffer, + const scalar_t* __restrict__ key, + const scalar_t* __restrict__ value, + const int64_t* __restrict__ loc, + int64_t batches, + int64_t num_heads_kv, + int64_t head_size, + int64_t head_size_v, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + int64_t nk_strideN, + int64_t nk_strideH, + int64_t nv_strideN, + int64_t nv_strideH, + bool is_mla) { + at::parallel_for(0, batches * num_heads_kv, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_kv_id{0}; + data_index_init(begin, bs, batches, head_kv_id, num_heads_kv); + + for (int64_t i = begin; i < end; i++) { + int64_t loc_val = loc[bs]; + scalar_t* k_buffer_ptr = k_buffer + loc_val * k_strideN + head_kv_id * k_strideH; + const scalar_t* new_key_ptr = key + bs * nk_strideN + head_kv_id * nk_strideH; + copy_stub(k_buffer_ptr, new_key_ptr, head_size); + if (!is_mla) { + scalar_t* v_buffer_ptr = v_buffer + loc_val * v_strideN + head_kv_id * v_strideH; + const scalar_t* new_value_ptr = value + bs * nv_strideN + head_kv_id * nv_strideH; + copy_stub(v_buffer_ptr, new_value_ptr, head_size_v); + } + + // move to the next index + data_index_step(bs, batches, head_kv_id, num_heads_kv); + } + }); +} + +template +void decode_accumulate_kv_splits( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + int64_t batches, + int64_t num_heads, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t l_stride1, + int64_t l_stride2) { + using Vec = at::vec::Vectorized; + + // parallel on [batches, num_heads] + at::parallel_for(0, batches * num_heads, 0, [&](int64_t begin, int64_t end) { + // NB: here we use logits[b][h][0] as acc, since + // for the first kv split (kv_id == 0): + // m_delta = std::exp(-inf) = 0 + // e_logic = std::exp(0) = 1 + // acc = acc * m_delta + tv * e_logic = tv + for (int64_t i = begin; i < end; ++i) { + float* __restrict__ acc = attn_logits + i * l_stride1; + + float s_prime = 0.f; + float m_prime = -std::numeric_limits::infinity(); + + // update acc with from each kv_split + for (int64_t kv_id = 0; kv_id < num_kv_splits; ++kv_id) { + float* __restrict__ tv = acc + kv_id * l_stride2; + const float tlogic = (acc + kv_id * l_stride2)[head_size_v]; + + float m_i = std::max(tlogic, m_prime); + float m_delta = std::exp(m_prime - m_i); + float e_logic = std::exp(tlogic - m_i); + if (kv_id != 0) { + at::vec::map2( + [m_delta, e_logic](Vec x, Vec y) { return x * Vec(m_delta) + y * Vec(e_logic); }, + acc, + acc, + tv, + head_size_v); + } + + s_prime = s_prime * m_delta + e_logic; + m_prime = m_i; + } + + copy_stub(output + i * head_size_v, acc, 1 / s_prime, head_size_v); + } + }); +} + +template +void decode_attention_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + int64_t batches, + int64_t num_heads, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens) { + using Vec = at::vec::Vectorized; + + // strides + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + // parallel on [batches, num_heads, num_kv_splits] + at::parallel_for(0, batches * num_heads * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, head_id, num_heads, kv_id, num_kv_splits); + + // s_prime and s_delta + alignas(64) float s_i[BLOCK_N]; + float* __restrict__ s_delta = s_i; + + for (int64_t i = begin; i < end; ++i) { + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + head_id * q_strideH; + + // get key/value + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + float m_prime = -std::numeric_limits::infinity(); + float s_prime = 0.f; + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + i * (head_size_v + 1); + fill_stub(v_prime, 0.f, head_size_v); + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + + // calculate s_i <- scale * Q @ K + index_gemm_kernel_nt( + /* A */ q_ptr, + /* B */ k_buffer + head_id * k_strideH, + /* C */ s_i, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ scaling, + /* M */ 1, + /* N */ n_size, + /* K */ head_size, + /* lda */ 1, + /* ldb */ k_strideN, + /* ldc */ 1, + /* mtt */ max_total_num_tokens); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i, + s_i, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all([](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i, n_size); + m_i = std::max(m_i, m_prime); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map([m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta, s_i, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime *= m_delta; + s_prime += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta, n_size); + + m_prime = m_i; + + // calculate V' <- s_delta @ V + V' * m_delta + index_gemm_kernel_nn( + /* A */ s_delta, + /* B */ v_buffer + head_id * v_strideH, + /* C */ v_prime, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ &m_delta, + /* M */ 1, + /* N */ head_size_v, + /* K */ n_size, + /* lda */ 1, + /* ldb */ v_strideN, + /* ldc */ 1, + /* mtt */ max_total_num_tokens); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + float s = 1 / s_prime; + at::vec::map([s](Vec out) { return out * Vec(s); }, v_prime, v_prime, head_size_v); + + v_prime[head_size_v] = m_prime + std::log(s_prime); + } + + // move to the next index + data_index_step(bs, batches, head_id, num_heads, kv_id, num_kv_splits); + } + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // MHA + +template +void decode_attention_mla_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + scalar_t* __restrict__ buffer, + int64_t batches, + int64_t num_heads, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens, + int64_t buffer_size_per_thread) { + using Vec = at::vec::Vectorized; + + // block length for heads + const int64_t BLOCK_H = batches == 1 ? 6 : (batches > 16 ? 22 : 11); + + // strides + const int64_t l_stride0 = num_heads * num_kv_splits * (head_size_v + 1); + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + TORCH_CHECK(logit_cap == 0.f, "decode MLA: expect no logit_cap."); + + // partition the heads into blocks for parallel + const int64_t num_blocks = div_up(num_heads, BLOCK_H); + + // parallel on [batches, num_blocks, num_kv_splits] + at::parallel_for(0, batches * num_blocks * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, block_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, block_id, num_blocks, kv_id, num_kv_splits); + + int tid = at::get_thread_num(); + scalar_t* __restrict__ Btmp0 = buffer + tid * buffer_size_per_thread; + scalar_t* __restrict__ Btmp1 = Btmp0 + BLOCK_N * head_size; + + // init Btmp1 just once for each thread to prevent NaN + // Btmp0 is not needed as it computes full K every single time + fill_stub(Btmp1, 0.f, BLOCK_N * head_size_v); + + alignas(64) float s_i[BLOCK_H * BLOCK_N]; + float* __restrict__ s_delta = s_i; + alignas(64) scalar_t s_delta2[BLOCK_H * BLOCK_N]; + + alignas(64) float s_prime[BLOCK_H]; + alignas(64) float m_prime[BLOCK_H]; + alignas(64) float m_delta[BLOCK_H]; + + for (int64_t i = begin; i < end; ++i) { + const int64_t h_start = block_id * BLOCK_H; + const int64_t h_end = std::min(block_id * BLOCK_H + BLOCK_H, num_heads); + const int64_t h_size = h_end - h_start; + + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + h_start * q_strideH; + + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + fill_stub(s_prime, 0.f, BLOCK_H); + fill_stub(m_prime, -std::numeric_limits::infinity(), BLOCK_H); + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + bs * l_stride0 + h_start * l_stride1 + kv_id * l_stride2; + for (int64_t h = 0; h < h_size; ++h) { + fill_stub(v_prime + h * l_stride1, 0.f, head_size_v); + } + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + const int64_t padded_n_size = div_up(int(n_size), TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst0 */ Btmp0, + /* dst1 */ Btmp1, + /* src */ k_buffer + /* head_kv_id */ 0 * k_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* N */ n_size, + /* K */ head_size, + /* Kv */ head_size_v, + /* ld_src */ k_strideN, + /* ld_dst0 */ BLOCK_N, + /* ld_dst1 */ head_size_v); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ h_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideH, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp0, + /* C */ s_i); + + const Vec scale_vec = Vec(scaling); + for (int64_t h = 0; h < h_size; ++h) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + h * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[h]); + + // m_delta <- exp(m' - m_i) + m_delta[h] = std::exp(m_prime[h] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[h] *= m_delta[h]; + s_prime[h] += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size); + + m_prime[h] = m_i; + + // v' <- v' * m_delta + float scale_m = m_delta[h]; + at::vec::map( + [scale_m](Vec x) { return x * Vec(scale_m); }, + v_prime + h * l_stride1, + v_prime + h * l_stride1, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + h * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + h * BLOCK_N, s_delta + h * BLOCK_N); + } + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ h_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ l_stride1, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp1, + /* C */ v_prime); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + for (int64_t h = 0; h < h_size; ++h) { + float s = 1 / s_prime[h]; + at::vec::map( + [s](Vec out) { return out * Vec(s); }, v_prime + h * l_stride1, v_prime + h * l_stride1, head_size_v); + (v_prime + h * l_stride1)[head_size_v] = m_prime[h] + std::log(s_prime[h]); + } + } + + // move to the next index + data_index_step(bs, batches, block_id, num_blocks, kv_id, num_kv_splits); + } + at::native::cpublas::brgemm_release(); + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // MLA + +template +void decode_attention_grouped_kernel_impl( + scalar_t* __restrict__ output, + float* __restrict__ attn_logits, + const scalar_t* __restrict__ query, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + int64_t batches, + int64_t num_heads, + int64_t num_heads_kv, + int64_t head_size, + int64_t head_size_v, + int64_t num_kv_splits, + int64_t q_strideM, + int64_t q_strideH, + int64_t k_strideN, + int64_t k_strideH, + int64_t v_strideN, + int64_t v_strideH, + float scaling, + float logit_cap, + int64_t max_num_reqs, + int64_t max_context_len, + int64_t max_total_num_tokens) { + using Vec = at::vec::Vectorized; + + // block length for heads + // we parallel on [batches, divup(num_heads, BLOCK_H), num_kv_splits] + // use smaller BLOCK_H when batches is small to utilize all cores + constexpr int64_t kBLOCK_H = 16; + const int64_t BLOCK_H = std::min(4 * batches, kBLOCK_H); + + // strides + const int64_t l_stride0 = num_heads * num_kv_splits * (head_size_v + 1); + const int64_t l_stride1 = num_kv_splits * (head_size_v + 1); + const int64_t l_stride2 = head_size_v + 1; + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + // partition the heads into blocks for parallel + const int64_t num_groups = num_heads / num_heads_kv; + const int64_t num_blocks = div_up(num_groups, BLOCK_H); + + // parallel on [batches, num_heads_kv, num_blocks, num_kv_splits] + at::parallel_for(0, batches * num_heads_kv * num_blocks * num_kv_splits, 0, [&](int64_t begin, int64_t end) { + int64_t bs{0}, head_kv_id{0}, block_id{0}, kv_id{0}; + data_index_init(begin, bs, batches, head_kv_id, num_heads_kv, block_id, num_blocks, kv_id, num_kv_splits); + + alignas(64) float s_i[BLOCK_H * BLOCK_N]; + float* __restrict__ s_delta = s_i; + + alignas(64) float s_prime[BLOCK_H]; + alignas(64) float m_prime[BLOCK_H]; + alignas(64) float m_delta[BLOCK_H]; + + for (int64_t i = begin; i < end; ++i) { + const int64_t h_start = head_kv_id * num_groups + block_id * BLOCK_H; + const int64_t h_end = head_kv_id * num_groups + std::min(block_id * BLOCK_H + BLOCK_H, num_groups); + const int64_t h_size = h_end - h_start; + + // get query + const scalar_t* __restrict__ q_ptr = query + bs * q_strideM + h_start * q_strideH; + + int64_t seq_len_kv = seq_lens[bs]; + int64_t req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_kv <= max_context_len, "seq_len_kv out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + const int64_t SPLIT_SIZE = div_up(seq_len_kv, num_kv_splits); + const int64_t kv_start = kv_id * SPLIT_SIZE; + const int64_t kv_end = std::min(kv_start + SPLIT_SIZE, seq_len_kv); + + fill_stub(s_prime, 0.f, BLOCK_H); + fill_stub(m_prime, -std::numeric_limits::infinity(), BLOCK_H); + + // get v_prime, and init to zero + float* __restrict__ v_prime = attn_logits + bs * l_stride0 + h_start * l_stride1 + kv_id * l_stride2; + for (int64_t h = 0; h < h_size; ++h) { + fill_stub(v_prime + h * l_stride1, 0.f, head_size_v); + } + + // loop over K and V sequence with BLOCK_N + for (int64_t n = kv_start; n < kv_end; n += BLOCK_N) { + int64_t n_size = std::min(BLOCK_N, kv_end - n); + + // calculate Q @ K + index_gemm_kernel_nt( + /* A */ q_ptr, + /* B */ k_buffer + head_kv_id * k_strideH, + /* C */ s_i, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ scaling, + /* M */ h_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideH, + /* ldb */ k_strideN, + /* ldc */ BLOCK_N, + /* mtt */ max_total_num_tokens); + + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i, + s_i, + BLOCK_H * BLOCK_N); + } + + // update the scaling coefficients + for (int64_t h = 0; h < h_size; ++h) { + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + h * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[h]); + + // m_delta <- exp(m' - m_i) + m_delta[h] = std::exp(m_prime[h] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + h * BLOCK_N, s_i + h * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[h] *= m_delta[h]; + s_prime[h] += at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + h * BLOCK_N, n_size); + + m_prime[h] = m_i; + } + + // calculate V' <- s_delta @ V + V' * m_delta + index_gemm_kernel_nn( + /* A */ s_delta, + /* B */ v_buffer + head_kv_id * v_strideH, + /* C */ v_prime, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* scl */ m_delta, + /* M */ h_size, + /* N */ head_size_v, + /* K */ n_size, + /* lda */ BLOCK_N, + /* ldb */ v_strideN, + /* ldc */ l_stride1, + /* mtt */ max_total_num_tokens); + } // loop with KV blocks + + // only update v' when kv_split_size > 0 + if (kv_end > kv_start) { + for (int64_t h = 0; h < h_size; ++h) { + float s = 1 / s_prime[h]; + at::vec::map( + [s](Vec out) { return out * Vec(s); }, v_prime + h * l_stride1, v_prime + h * l_stride1, head_size_v); + (v_prime + h * l_stride1)[head_size_v] = m_prime[h] + std::log(s_prime[h]); + } + } + + // move to the next index + data_index_step(bs, batches, head_kv_id, num_heads_kv, block_id, num_blocks, kv_id, num_kv_splits); + } + }); + + decode_accumulate_kv_splits( + output, attn_logits, batches, num_heads, head_size_v, num_kv_splits, l_stride1, l_stride2); +} // GQA/MQA + +} // anonymous namespace + +// query: [num_tokens, num_heads, head_size] +// output: [num_tokens, num_heads, head_size] +// k_buffer: [max_total_num_tokens, num_heads, head_size] +// v_buffer: [max_total_num_tokens, num_heads, head_size_v] +// attn_logits: [num_seqs, num_heads, num_kv_splits, head_size_v + 1] +// req_to_token: [max_num_reqs, max_context_len] int32 or int64 +// req_pool_indices: [num_seqs] int64 +// seq_lens: [num_seqs] int64 +// +void decode_attention_cpu( + at::Tensor& query, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& output, + at::Tensor& key, + at::Tensor& value, + at::Tensor& loc, + at::Tensor& attn_logits, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + double sm_scale, + double logit_cap) { + RECORD_FUNCTION( + "sgl-kernel::decode_attention_cpu", + std::vector( + {query, output, k_buffer, v_buffer, attn_logits, req_to_token, req_pool_indices, seq_lens})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(query); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_buffer); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_buffer); + // for MLA, key and value shares the same storage and value could be non-contiguous + CHECK_LAST_DIM_CONTIGUOUS_INPUT(key); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(value); + CHECK_DIM(3, query); + CHECK_DIM(3, k_buffer); + CHECK_DIM(3, v_buffer); + CHECK_DIM(3, key); + CHECK_DIM(3, value); + CHECK_DIM(1, loc); + + int64_t num_seqs = seq_lens.size(0); + int64_t max_num_reqs = req_to_token.size(0); + int64_t max_context_len = req_to_token.size(1); + int64_t max_total_num_tokens = k_buffer.size(0); + + int64_t num_heads = query.size(1); + int64_t num_heads_kv = k_buffer.size(1); + int64_t head_size = query.size(2); + int64_t head_size_v = v_buffer.size(2); + + int64_t num_kv_splits = attn_logits.size(2); + + CHECK_EQ(loc.numel(), num_seqs); + CHECK_EQ(attn_logits.size(0), num_seqs); + CHECK_EQ(attn_logits.size(1), num_heads); + CHECK_EQ(attn_logits.size(3), head_size_v + 1); + CHECK_EQ(attn_logits.scalar_type(), at::kFloat); + + // strides for query + int64_t q_strideM = query.stride(0); + int64_t q_strideH = query.stride(1); + + // strides for k_buffer and v_buffer + int64_t k_strideN = k_buffer.stride(0); + int64_t k_strideH = k_buffer.stride(1); + int64_t v_strideN = v_buffer.stride(0); + int64_t v_strideH = v_buffer.stride(1); + // strides for new key and value + int64_t nk_strideN = key.stride(0); + int64_t nk_strideH = key.stride(1); + int64_t nv_strideN = value.stride(0); + int64_t nv_strideH = value.stride(1); + + // check index data types + const auto index_dtype = req_to_token.scalar_type(); + TORCH_CHECK( + index_dtype == at::kInt || index_dtype == at::kLong, + "decode: expect req_to_token to be int32 or int64, got ", + index_dtype); + TORCH_CHECK(seq_lens.scalar_type() == at::kLong, "decode: expect req_lens to be int64, got ", seq_lens.scalar_type()); + TORCH_CHECK( + req_pool_indices.scalar_type() == at::kLong, + "decode: expect req_pool_indices to be int64, got ", + req_pool_indices.scalar_type()); + + // check if we have MLA here + void* k_buffer_data = k_buffer.data_ptr(); + void* v_buffer_data = v_buffer.data_ptr(); + const bool is_mla = (k_buffer_data == v_buffer_data) && (num_heads_kv == 1) && (head_size == head_size_v + 64); + + // block length for k_buffer and v_buffer + constexpr int BLOCK_N = 256; + + // buffer for packing k_cache and v_cache + int num_threads = at::get_num_threads(); + int64_t size_per_thread = is_mla ? BLOCK_N * head_size + BLOCK_N * head_size_v : 0; + auto buffer = at::empty({num_threads, size_per_thread}, k_buffer.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(query.scalar_type(), "decode_attention_kernel", [&] { + AT_DISPATCH_INDEX_TYPES(index_dtype, "decode_attention_indices", [&] { + // update the kv buffer + decode_set_kv_buffer( + (scalar_t*)k_buffer_data, + (scalar_t*)v_buffer_data, + key.data_ptr(), + value.data_ptr(), + loc.data_ptr(), + num_seqs, + num_heads_kv, + head_size, + head_size_v, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + nk_strideN, + nk_strideH, + nv_strideN, + nv_strideH, + is_mla); + + if (num_heads == num_heads_kv) { + // MHA + decode_attention_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + num_seqs, + num_heads, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens); + } else if (is_mla) { + // MLA + decode_attention_mla_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + buffer.data_ptr(), + num_seqs, + num_heads, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens, + size_per_thread); + } else { + // GQA/MQA + decode_attention_grouped_kernel_impl( + output.data_ptr(), + attn_logits.data_ptr(), + query.data_ptr(), + (const scalar_t*)k_buffer_data, + (const scalar_t*)v_buffer_data, + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + num_seqs, + num_heads, + num_heads_kv, + head_size, + head_size_v, + num_kv_splits, + q_strideM, + q_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens); + } + }); + }); +} diff --git a/sgl-kernel/csrc/cpu/extend.cpp b/sgl-kernel/csrc/cpu/extend.cpp new file mode 100644 index 000000000..3162ccea8 --- /dev/null +++ b/sgl-kernel/csrc/cpu/extend.cpp @@ -0,0 +1,723 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: extend attention for CPU +// 1. tune BLOCK_M and BLOCK_N +// 2. can handle non-contiguous k_exttend and v_extend +// 3. computes attention for prefix and extend separately +// 4. TODO: vectorize `pack_vnni` and `pack_vnni2` +// + +template +inline index_t get_index(index_t* ind, int i) { + return (ind == nullptr) ? (index_t)i : ind[i]; +} + +#if defined(CPU_CAPABILITY_AVX512) +// key: from [N, 32] to [32/2, N, 2] +template +inline void pack_vnni_Nx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int ld_src, + int ld_dst) { + __m512i vinputs[16]; + + int n = 0; + for (; n < N; ++n) { + index_t index = get_index(ind, n); + vinputs[n] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; n < 16; ++n) { + vinputs[n] = _mm512_set1_epi32(0); + } + + // pack key + transpose_16x16_32bit(vinputs); + + const __mmask16 vmask = (1 << N) - 1; + for (int k = 0; k < 16; ++k) { + _mm512_mask_storeu_epi32(dst + k * ld_dst * 2, vmask, vinputs[k]); + } +} + +// value: from [K, 32] to [K/2, 32, 2] +template +inline void pack_vnni_Kx32( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int ld_src, + int ld_dst) { + __m512i vinputs[2]; + + int k = 0; + for (; k < K; ++k) { + index_t index = get_index(ind, k); + vinputs[k] = _mm512_loadu_si512(src + index * ld_src); + } + // padding with zero to avoid uninitialized vectors + for (; k < 2; ++k) { + vinputs[k] = _mm512_set1_epi32(0); + } + + // pack value + __m512i d0, d1; + std::tie(d0, d1) = transpose_2x32_16bit(vinputs[0], vinputs[1]); + _mm512_storeu_si512(dst + 0 * ld_dst * 2, d0); + _mm512_storeu_si512(dst + 0 * ld_dst * 2 + 32, d1); +} +#endif + +// convert to vnni format +// from [N, K/2, 2] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int N, + int K, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int NB = div_up(N, 16); + const int KB = K / 32; // no remainder + const bool is_indexed = ind != nullptr; + + for (int nb = 0; nb < NB; ++nb) { + for (int kb = 0; kb < KB; ++kb) { + // handle 16x512bits each block + int nb_size = std::min(N - nb * 16, 16); + pack_vnni_Nx32( + /* dst */ dst + ((kb * 32) >> 1) * ld_dst * 2 + nb * 16 * 2, + /* src */ src + kb * 32 + (is_indexed ? 0 : nb * 16 * ld_src), + /* ind */ is_indexed ? ind + nb * 16 : nullptr, + /* N */ nb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + for (int n = 0; n < N; ++n) { + index_t index = get_index(ind, n); + for (int k = 0; k < K / 2; ++k) { + for (int d = 0; d < 2; ++d) { + dst[k * ld_dst * 2 + n * 2 + d] = src[index * ld_src + k * 2 + d]; + } + } + } +#endif +} + +// convert to vnni format +// from [K/2, 2, N] to [K/2, N, 2] for bfloat16 and float16 +template +void pack_vnni2( + scalar_t* __restrict__ dst, + const scalar_t* __restrict__ src, + const index_t* __restrict__ ind, + int K, + int N, + int ld_src, + int ld_dst) { +#if defined(CPU_CAPABILITY_AVX512) + const int KB = div_up(K, 2); + const int NB = N / 32; // no remainder + const bool is_indexed = ind != nullptr; + + for (int kb = 0; kb < KB; ++kb) { + for (int nb = 0; nb < NB; ++nb) { + // handle 2x512bits each block + int kb_size = std::min(K - kb * 2, 2); + pack_vnni_Kx32( + /* dst */ dst + ((kb * 2) >> 1) * ld_dst * 2 + nb * 32 * 2, + /* src */ src + (is_indexed ? 0 : kb * 2 * ld_src) + nb * 32, + /* ind */ is_indexed ? ind + kb * 2 : nullptr, + /* K */ kb_size, + /* ld_src */ ld_src, + /* ld_dst */ ld_dst); + } + } +#else + int k = 0; + for (; k < (K >> 1) * 2; k += 2) { + index_t index0 = get_index(ind, k + 0); + index_t index1 = get_index(ind, k + 1); + for (int n = 0; n < N; ++n) { + dst[(k >> 1) * ld_dst * 2 + n * 2 + 0] = src[index0 * ld_src + n]; + dst[(k >> 1) * ld_dst * 2 + n * 2 + 1] = src[index1 * ld_src + n]; + } + } + if (K % 2 != 0) { + index_t index = get_index(ind, K - 1); + for (int n = 0; n < N; ++n) { + dst[(K >> 1) * ld_dst * 2 + n * 2 + 0] = src[index * ld_src + n]; + dst[(K >> 1) * ld_dst * 2 + n * 2 + 1] = 0; + } + k += 2; + } +#endif +} + +template +inline void fill_stub(scalar_t* __restrict__ out, float val, int size) { + using Vec = at::vec::Vectorized; + constexpr int kVecSize = Vec::size(); + const Vec data_vec = Vec(static_cast(val)); + int d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + data_vec.store(out + d); + } + if (size - d > 0) { + data_vec.store(out + d, size - d); + } +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input) { + static_assert(BLOCK_N % 32 == 0); + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int COLS = BLOCK_N / 16; + auto store = [&](auto i) { + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + fVec a_fvec0 = fVec::loadu(input + col * 16); + fVec a_fvec1 = fVec::loadu(input + col * 16 + 16); + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + col * 16); + } + }; + Unroll{}(store); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ acc, float s, int size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_fvec = fVec(s); + int d = 0; +#pragma GCC unroll 4 + for (; d <= size - kVecSize; d += kVecSize) { + fVec a_fvec0 = fVec::loadu(acc + d) * s_fvec; + fVec a_fvec1 = fVec::loadu(acc + d + fVec::size()) * s_fvec; + bVec out_bvec = convert_from_float_ext(a_fvec0, a_fvec1); + out_bvec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(acc[d] * s); + } +} + +template +void extend_attention_kernel_impl( + scalar_t* __restrict__ o_extend, + const scalar_t* __restrict__ q_extend, + const scalar_t* __restrict__ k_extend, + const scalar_t* __restrict__ v_extend, + const scalar_t* __restrict__ k_buffer, + const scalar_t* __restrict__ v_buffer, + const index_t* __restrict__ req_to_token, + const int64_t* __restrict__ req_pool_indices, + const int64_t* __restrict__ seq_lens, + const index_t* __restrict__ extend_seq_lens, + const index_t* __restrict__ extend_start_loc, + const void* __restrict__ buffer, + int batches, + int num_heads, + int num_heads_kv, + int head_size, + int head_size_v, + int q_strideM, + int q_strideH, + int ke_strideN, + int ke_strideH, + int ve_strideN, + int ve_strideH, + int k_strideN, + int k_strideH, + int v_strideN, + int v_strideH, + float scaling, + float logit_cap, + int max_num_reqs, + int max_context_len, + int max_total_num_tokens, + int max_len_extend, + int buffer_size_per_thread, + bool is_prefix_skipped) { + using Vec = at::vec::Vectorized; + + // strides + const int o_strideM = num_heads * head_size_v; + const int o_strideH = head_size_v; + + // we use same buffer for packed key and value + const int ldb_tmp = std::max(head_size, head_size_v); + + const bool has_logit_cap = logit_cap > 0; + float rlogit_cap = has_logit_cap ? 1 / logit_cap : 0.f; + + const int num_groups = num_heads / num_heads_kv; + TORCH_CHECK(num_groups * num_heads_kv == num_heads); + + // number of blocks along M + int MB = div_up(max_len_extend, BLOCK_M); + + // parallel on [batches, num_heads, BM] + at::parallel_for(0, batches * num_heads * MB, 0, [&](int begin, int end) { + int bs{0}, head_id{0}, mb{0}; + data_index_init(begin, bs, batches, head_id, num_heads, mb, MB); + + int tid = at::get_thread_num(); + // s_i and s_delta: [BLOCK_M, BLOCK_N] + float* __restrict__ s_i = reinterpret_cast((char*)(buffer) + tid * buffer_size_per_thread); + float* __restrict__ s_delta = s_i; + + // v_prime: [BLOCK_M, head_size_v] + float* __restrict__ v_prime = s_i + BLOCK_M * BLOCK_N; + + // s_delta2: [BLOCK_M, BLOCK_N]; copy of s_delta in scalar_t + scalar_t* __restrict__ s_delta2 = reinterpret_cast(v_prime + BLOCK_N * head_size_v); + + // Btmp: [BLOCK_N, max(head_size, head_size_v)] + scalar_t* __restrict__ Btmp = s_delta2 + BLOCK_M * BLOCK_N; + + // init Btmp just once for each thread to prevent NaN + fill_stub(Btmp, 0.f, BLOCK_N * ldb_tmp); + + alignas(64) float s_prime[BLOCK_M]; + alignas(64) float m_prime[BLOCK_M]; + + for (int i = begin; i < end; ++i) { + // seq_len = prefix + extend + int head_kv_id = head_id / num_groups; + int seq_len = seq_lens[bs]; + int seq_len_extend = extend_seq_lens[bs]; + int seq_len_prefix = seq_len - seq_len_extend; + int seq_extend_start_loc = extend_start_loc[bs]; + + int req_pool_id = req_pool_indices[bs]; + TORCH_CHECK(seq_len_prefix >= 0, "prefix len < 0!"); + TORCH_CHECK(seq_len <= max_context_len, "seq_len out of scope!"); + TORCH_CHECK(req_pool_id < max_num_reqs, "req_pool_id out of scope!"); + + if (is_prefix_skipped) { + TORCH_CHECK(seq_len_prefix == 0, "extend attention: expect seq_len_prefix to be 0, got ", seq_len_prefix); + } + + // offset and size in MB + int m = mb * BLOCK_N; + int m_size = std::min(BLOCK_M, seq_len_extend - m); + + if (m_size <= 0) { + data_index_step(bs, batches, head_id, num_heads, mb, MB); + continue; + } + + // get query + const scalar_t* __restrict__ q_ptr = q_extend + (seq_extend_start_loc + m) * q_strideM + head_id * q_strideH; + + // init v', s' and m' + fill_stub(v_prime, 0.f, m_size * head_size_v); + fill_stub(s_prime, 0.f, m_size); + fill_stub(m_prime, -std::numeric_limits::infinity(), m_size); + + // stage 1: compute scores with prefix + for (int n = 0; n < seq_len_prefix; n += BLOCK_N) { + int n_size = std::min(BLOCK_N, seq_len_prefix - n); + + // `n_size` is K in 2nd gemm, pad to TILE_K; + const int padded_n_size = div_up(n_size, TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst */ Btmp, + /* src */ k_buffer + head_kv_id * k_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* N */ n_size, + /* K */ head_size, + /* ld_src */ k_strideN, + /* ld_dst */ BLOCK_N); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideM, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp, + /* C */ s_i); + + const Vec scale_vec = Vec(scaling); + for (int row = 0; row < m_size; ++row) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i + row * BLOCK_N, + s_i + row * BLOCK_N, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + row * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[row]); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime[row] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[row] *= m_delta; + s_prime[row] += + at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + row * BLOCK_N, n_size); + + m_prime[row] = m_i; + + // v' <- v' * m_delta + at::vec::map( + [m_delta](Vec x) { return x * Vec(m_delta); }, + v_prime + row * head_size_v, + v_prime + row * head_size_v, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + row * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + row * BLOCK_N, s_delta + row * BLOCK_N); + } + + // get value and pack + pack_vnni2( + /* dst */ Btmp, + /* src */ v_buffer + head_kv_id * v_strideH, + /* ind */ req_to_token + req_pool_id * max_context_len + n, + /* K */ n_size, + /* N */ head_size_v, + /* ld_src */ v_strideN, + /* ld_dst */ head_size_v); + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ head_size_v, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp, + /* C */ v_prime); + } // loop with seq_len_prefix + + // stage 2: compute the triangle part + int num_keys = std::min(seq_len_extend, m + BLOCK_M); + for (int n = 0; n < num_keys; n += BLOCK_N) { + int n_size = std::min(BLOCK_N, num_keys - n); + + // `n_size` is K in 2nd gemm, pad to TILE_K; + const int padded_n_size = div_up(n_size, TILE_K) * TILE_K; + + // get key and pack + pack_vnni( + /* dst */ Btmp, + /* src */ k_extend + (seq_extend_start_loc + n) * ke_strideN + head_kv_id * ke_strideH, + /* ind */ nullptr, + /* N */ n_size, + /* K */ head_size, + /* ld_src */ ke_strideN, + /* ld_dst */ BLOCK_N); + + // calculate s_i <- Q @ K + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ head_size, + /* lda */ q_strideM, + /* ldb */ BLOCK_N, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ q_ptr, + /* B */ Btmp, + /* C */ s_i); + + // apply causal mask + if (num_keys - n <= BLOCK_N) { + for (int row = 0; row < m_size; ++row) { + int last_col = m + row - n; + // fill [last_col + 1, n_size) to -inf + float* row_ptr = s_i + row * BLOCK_N; + fill_stub(row_ptr + last_col + 1, -std::numeric_limits::infinity(), n_size - last_col - 1); + } + } + + const Vec scale_vec = Vec(scaling); + for (int row = 0; row < m_size; ++row) { + // s_i <- s_i * scale + at::vec::map( + [scale_vec](Vec x) { return x * scale_vec; }, s_i + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // TODO: `tanh` from torch uses sleef u10, going to be slow + if (has_logit_cap) { + at::vec::map( + [logit_cap, rlogit_cap](Vec x) { return Vec(logit_cap) * (x * Vec(rlogit_cap)).tanh(); }, + s_i + row * BLOCK_N, + s_i + row * BLOCK_N, + n_size); + } + + // m_i: max value per row + float m_i = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, s_i + row * BLOCK_N, n_size); + m_i = std::max(m_i, m_prime[row]); + + // m_delta <- exp(m' - m_i) + float m_delta = std::exp(m_prime[row] - m_i); + + // s_delta <- exp(s_i - m_i) + at::vec::map( + [m_i](Vec x) { return (x - Vec(m_i)).exp_u20(); }, s_delta + row * BLOCK_N, s_i + row * BLOCK_N, n_size); + + // s' <- s' * m_delta + sum(s_delta) + s_prime[row] *= m_delta; + s_prime[row] += + at::vec::reduce_all([](Vec& x, Vec& y) { return x + y; }, s_delta + row * BLOCK_N, n_size); + + m_prime[row] = m_i; + + // v' <- v' * m_delta + at::vec::map( + [m_delta](Vec x) { return x * Vec(m_delta); }, + v_prime + row * head_size_v, + v_prime + row * head_size_v, + head_size_v); + + // pad s_delta with 0 first and then convert to scalar_t + fill_stub(s_delta + row * BLOCK_N + n_size, 0.f, padded_n_size - n_size); + copy_stub(s_delta2 + row * BLOCK_N, s_delta + row * BLOCK_N); + } + + // get value and pack + pack_vnni2( + /* dst */ Btmp, + /* src */ v_extend + (seq_extend_start_loc + n) * ve_strideN + head_kv_id * ve_strideH, + /* ind */ nullptr, + /* K */ n_size, + /* N */ head_size_v, + /* ld_src */ ve_strideN, + /* ld_dst */ head_size_v); + + // calculate V' <- s_delta @ V + V' + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ head_size_v, + /* K */ padded_n_size, // n_size + /* lda */ BLOCK_N, + /* ldb */ head_size_v, + /* ldc */ head_size_v, + /* add_C */ true, + /* A */ s_delta2, + /* B */ Btmp, + /* C */ v_prime); + } // loop with seq_len_extend + + scalar_t* __restrict__ out_ptr = o_extend + (seq_extend_start_loc + m) * o_strideM + head_id * o_strideH; + for (int row = 0; row < m_size; ++row) { + float s = 1 / s_prime[row]; + copy_stub(out_ptr + row * o_strideM, v_prime + row * head_size_v, s, head_size_v); + } + + // move to the next index + data_index_step(bs, batches, head_id, num_heads, mb, MB); + } + at::native::cpublas::brgemm_release(); + }); +} + +} // anonymous namespace + +// q_extend, k_extend, v_extend, o_extend: contiguous tensors +// k_buffer, v_buffer: (prefix + extend) tensors in mem_manager +// +// q_extend: [num_tokens, num_heads, head_size] +// k_extend: [num_extend_tokens, num_heads, head_size] +// v_extend: [num_extend_tokens, num_heads, head_size] +// o_extend: [num_tokens, num_heads, head_size] +// k_buffer: [max_total_num_tokens, num_heads, head_size] +// v_buffer: [max_total_num_tokens, num_heads, head_size] +// req_to_token: [max_num_reqs, max_context_len] int32 or int64 +// req_pool_indices: [num_seqs] int64 +// seq_lens: [num_seqs] int64 +// extend_seq_lens: [num_seqs] +// extend_start_loc: [num_seqs] +// +void extend_attention_cpu( + at::Tensor& q_extend, + at::Tensor& k_extend, + at::Tensor& v_extend, + at::Tensor& o_extend, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + at::Tensor& extend_seq_lens, + at::Tensor& extend_start_loc, + int64_t max_len_extend, + double sm_scale, + double logit_cap) { + RECORD_FUNCTION( + "sgl-kernel::extend_attention_cpu", + std::vector( + {q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_token, + req_pool_indices, + seq_lens, + extend_seq_lens, + extend_start_loc})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(q_extend); + CHECK_INPUT(o_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_extend); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(k_buffer); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(v_buffer); + + int num_seqs = seq_lens.size(0); + int max_num_reqs = req_to_token.size(0); + int max_context_len = req_to_token.size(1); + int max_total_num_tokens = k_buffer.size(0); + + int num_heads = q_extend.size(1); + int num_heads_kv = k_extend.size(1); + int head_size = q_extend.size(2); + int head_size_v = v_extend.size(2); + + // strides for q_extend, k_extend and v_extend + int q_strideM = q_extend.stride(0); + int q_strideH = q_extend.stride(1); + int ke_strideN = k_extend.stride(0); + int ke_strideH = k_extend.stride(1); + int ve_strideN = v_extend.stride(0); + int ve_strideH = v_extend.stride(1); + + // strides for k_buffer and v_buffer + int k_strideN = k_buffer.stride(0); + int k_strideH = k_buffer.stride(1); + int v_strideN = v_buffer.stride(0); + int v_strideH = v_buffer.stride(1); + + // check sizes + CHECK_EQ(req_pool_indices.size(0), num_seqs); + CHECK_EQ(extend_seq_lens.size(0), num_seqs); + CHECK_EQ(extend_start_loc.size(0), num_seqs); + CHECK_EQ(v_extend.size(1), num_heads_kv); + CHECK_EQ(k_buffer.size(1), v_buffer.size(1)); + + // MLA will skip prefix part + const bool is_prefix_skipped = k_buffer.size(1) != num_heads_kv; + + // check index data types + const auto index_dtype = req_to_token.scalar_type(); + TORCH_CHECK( + index_dtype == at::kInt || index_dtype == at::kLong, + "extend: expect req_to_token to be int32 or int64, got ", + index_dtype); + TORCH_CHECK(seq_lens.scalar_type() == at::kLong, "extend: expect req_lens to be int64, got ", seq_lens.scalar_type()); + TORCH_CHECK( + req_pool_indices.scalar_type() == at::kLong, + "extend: expect req_pool_indices to be int64, got ", + req_pool_indices.scalar_type()); + TORCH_CHECK( + extend_seq_lens.scalar_type() == index_dtype && extend_start_loc.scalar_type() == index_dtype, + "extend: expect extend_seq_lens and extend_start_loc to have same dtype as req_to_token."); + + // D and DV need to be 32x as we transpose by 512-bit + TORCH_CHECK(head_size % 32 == 0, "invalid head_size ", head_size); + TORCH_CHECK(head_size_v % 32 == 0, "invalid head_size_v ", head_size_v); + + // block size for query seq length + constexpr int BLOCK_M = 32; + // block size for key/value seq length + constexpr int BLOCK_N = 32; + + const int size_per_thread = + /* s_i */ BLOCK_M * BLOCK_N * sizeof(float) + + /* v_prime */ BLOCK_M * head_size_v * sizeof(float) + + /* s_delta */ BLOCK_M * BLOCK_N * sizeof(uint16_t) + + /* Btmp */ BLOCK_N * std::max(head_size, head_size_v) * sizeof(uint16_t); + + int num_threads = at::get_num_threads(); + auto buffer = at::empty({num_threads, size_per_thread}, q_extend.options().dtype(at::kChar)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(q_extend.scalar_type(), "extend_attention_kernel", [&] { + AT_DISPATCH_INDEX_TYPES(index_dtype, "extend_attention_indices", [&] { + extend_attention_kernel_impl( + o_extend.data_ptr(), + q_extend.data_ptr(), + k_extend.data_ptr(), + v_extend.data_ptr(), + k_buffer.data_ptr(), + v_buffer.data_ptr(), + req_to_token.data_ptr(), + req_pool_indices.data_ptr(), + seq_lens.data_ptr(), + extend_seq_lens.data_ptr(), + extend_start_loc.data_ptr(), + buffer.data_ptr(), + num_seqs, + num_heads, + num_heads_kv, + head_size, + head_size_v, + q_strideM, + q_strideH, + ke_strideN, + ke_strideH, + ve_strideN, + ve_strideH, + k_strideN, + k_strideH, + v_strideN, + v_strideH, + sm_scale, + logit_cap, + max_num_reqs, + max_context_len, + max_total_num_tokens, + max_len_extend, + size_per_thread, + is_prefix_skipped); + }); + }); +} diff --git a/sgl-kernel/csrc/cpu/gemm.cpp b/sgl-kernel/csrc/cpu/gemm.cpp new file mode 100644 index 000000000..48655b9f7 --- /dev/null +++ b/sgl-kernel/csrc/cpu/gemm.cpp @@ -0,0 +1,525 @@ +#include "gemm.h" + +#include "common.h" +#include "vec.h" + +namespace { + +// packed layout: +// quants {N, K} int8_t +// comp {N} int32_t +template +inline void s8s8_compensation(int8_t* __restrict__ packed, int K) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + __m512i vcomp[COLS]; + + for (int col = 0; col < COLS; ++col) { + vcomp[col] = _mm512_setzero_si512(); + } + + const int64_t offset = BLOCK_N * K; + const __m512i off = _mm512_set1_epi8(static_cast(0x80)); + for (int k = 0; k < K / 4; ++k) { + for (int col = 0; col < COLS; ++col) { + __m512i vb = _mm512_loadu_si512((const __m512i*)(packed + k * BLOCK_N * 4 + col * 64)); + vcomp[col] = _mm512_dpbusd_epi32(vcomp[col], off, vb); + } + } + + for (int col = 0; col < COLS; ++col) { + _mm512_storeu_si512((__m512i*)(packed + offset + col * 64), vcomp[col]); + } +#else + TORCH_CHECK(false, "s8s8_compensation not implemented!"); +#endif +} + +// convert to vnni format +// from [N, K] to [K/2, N, 2] for bfloat16 and float16 +template +inline void pack_vnni(packed_t* __restrict__ packed, const packed_t* __restrict__ weight, int N, int K) { + const int VNNI_BLK = 2; + for (int n = 0; n < N; ++n) { + for (int k = 0; k < K / VNNI_BLK; ++k) { + for (int d = 0; d < VNNI_BLK; ++d) { + packed[k * N * VNNI_BLK + n * VNNI_BLK + d] = weight[n * K + k * VNNI_BLK + d]; + } + } + } +} + +template <> +inline void pack_vnni(int8_t* __restrict__ packed, const int8_t* __restrict__ weight, int N, int K) { + constexpr int BLOCK_N = block_size_n(); + TORCH_CHECK(N == BLOCK_N); + + const int VNNI_BLK = 4; + for (int n = 0; n < N; ++n) { + for (int k = 0; k < K / VNNI_BLK; ++k) { + for (int d = 0; d < VNNI_BLK; ++d) { + packed[k * N * VNNI_BLK + n * VNNI_BLK + d] = weight[n * K + k * VNNI_BLK + d]; + } + } + } + s8s8_compensation(packed, K); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d); + fVec data1 = fVec::loadu(input + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); + fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + bias[d]); + } +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + auto loadc = [&](auto i) { + constexpr int col = i % COLS; + if constexpr (has_bias) { + vc[i] = _mm512_loadu_ps(bias + col * 16); + } else { + vc[i] = _mm512_set1_ps(0.f); + } + }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb[col] = (__m512bh)(_mm512_loadu_si512(b_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + // for COLS = 1, 3 use 256bit store + if constexpr (COLS % 2 == 0) { + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); + } + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i*>(C + row * ldc + col * 16), (__m256i)(_mm512_cvtneps_pbh(vc[i]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +struct brgemm { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + + // copy from Ctmp to C + for (int64_t m = 0; m < M; ++m) { + if constexpr (has_bias) { + copy_add_stub(C + m * ldc, Ctmp + m * BLOCK_N, bias, N); + } else { + copy_stub(C + m * ldc, Ctmp + m * BLOCK_N, N); + } + } + } +}; + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + if (brg) { + brgemm::apply(A, B, C, Ctmp, bias, M, N, K, lda, ldb, ldc); + return; + } + + // pattern: 1-4-16, N = 16, 32, 48, 64 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x11: + LAUNCH_TINYGEMM_KERNEL_NN(1, 16); + break; + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x13: + LAUNCH_TINYGEMM_KERNEL_NN(1, 48); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + // mb_size = 2 + case 0x21: + LAUNCH_TINYGEMM_KERNEL_NN(2, 16); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x23: + LAUNCH_TINYGEMM_KERNEL_NN(2, 48); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + // mb_size = 3 + case 0x31: + LAUNCH_TINYGEMM_KERNEL_NN(3, 16); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x33: + LAUNCH_TINYGEMM_KERNEL_NN(3, 48); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + // mb_size = 4 + case 0x41: + LAUNCH_TINYGEMM_KERNEL_NN(4, 16); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x43: + LAUNCH_TINYGEMM_KERNEL_NN(4, 48); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, " x ", nb_size); + } + } + } +} + +template +void weight_packed_linear_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const scalar_t* __restrict__ mat2, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideM, + int64_t out_strideM) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB] + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(M - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + mb_start * mat1_strideM, + /* B */ mat2 + nb_start * K /* nb * BLOCK_N * K */, + /* C */ out + mb_start * out_strideM + nb_start, + /* Ctmp*/ Ctmp, + /* bias*/ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + tinygemm_kernel(A, B, C, Ctmp, nullptr, M, N, K, lda, ldb, ldc, brg); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const TYPE* __restrict__ A, \ + const TYPE* __restrict__ B, \ + TYPE* __restrict__ C, \ + float* __restrict__ Ctmp, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +at::Tensor convert_weight_packed(at::Tensor& weight) { + // for 3d moe weights + // weight : [E, OC, IC] + // w1 : [E, 2N, K] + // w2 : [E, K, N] + CHECK_INPUT(weight); + + const int64_t ndim = weight.ndimension(); + TORCH_CHECK(ndim == 2 || ndim == 3, "expect weight to be 2d or 3d, got ", ndim, "d tensor."); + const auto st = weight.scalar_type(); + const int64_t E = ndim == 3 ? weight.size(0) : 1; + const int64_t OC = ndim == 3 ? weight.size(1) : weight.size(0); + const int64_t IC = ndim == 3 ? weight.size(2) : weight.size(1); + + // we handle 2 TILE_N at a time. + TORCH_CHECK(OC % TILE_N == 0, "invalid weight out features ", OC); + TORCH_CHECK(IC % TILE_K == 0, "invalid weight input features ", IC); + + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t NB = div_up(OC, BLOCK_N); + + // use phony sizes here [E, OC, IC], for each [E], [OC, IC] -> [IC / 2, OC, 2] + auto packed_weight = at::empty({}, weight.options()); + const int64_t stride = OC * IC; + + TORCH_CHECK( + st == at::kBFloat16 || st == at::kHalf || st == at::kChar || st == at::kFloat8_e4m3fn, + "expect weight to be bfloat16, float16, int8 or fp8_e4m3."); + + CPU_DISPATCH_PACKED_TYPES(st, [&] { + // adjust most inner dimension size + const int packed_row_size = get_row_size(IC); + auto sizes = weight.sizes().vec(); + sizes[ndim - 1] = packed_row_size; + packed_weight.resize_(sizes); + + const packed_t* w_data = weight.data_ptr(); + packed_t* packed_data = packed_weight.data_ptr(); + + // parallel on {E, NB} + at::parallel_for(0, E * NB, 0, [&](int64_t begin, int64_t end) { + int64_t e{0}, nb{0}; + data_index_init(begin, e, E, nb, NB); + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + + int64_t n = nb * BLOCK_N; + int64_t n_size = std::min(BLOCK_N, OC - n); + pack_vnni( + packed_data + e * OC * packed_row_size + n * packed_row_size, w_data + e * stride + n * IC, n_size, IC); + + // move to the next index + data_index_step(e, E, nb, NB); + } + }); + }); + return packed_weight; +} + +// mat1 : [M, K] +// mat2 : [N, K] +// bias : [N] +// out : [M, N] +// +at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::weight_packed_linear", std::vector({mat1, mat2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat2.size(1); + CHECK_EQ(mat1.size(1), K); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + auto out = at::empty({M, N}, mat1.options()); + + // strides + int64_t mat1_strideM = mat1.stride(0); + int64_t out_strideM = out.stride(0); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(mat1.scalar_type(), "weight_packed_linear_kernel_impl", [&] { + weight_packed_linear_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + bias_data, + M, + N, + K, + mat1_strideM, + out_strideM); + }); + + return out; +} diff --git a/sgl-kernel/csrc/cpu/gemm.h b/sgl-kernel/csrc/cpu/gemm.h new file mode 100644 index 000000000..6a16a2985 --- /dev/null +++ b/sgl-kernel/csrc/cpu/gemm.h @@ -0,0 +1,202 @@ +#pragma once +#include + +#include "common.h" + +// amx-bf16 +#define TILE_M 16 +#define TILE_N 16 +#define TILE_K 32 + +// block size for AMX gemm +constexpr int block_size_m() { + return 2 * TILE_M; +} +constexpr int block_size_n() { + return 2 * TILE_N; +} + +// define threshold using brgemm (intel AMX) +template +inline bool can_use_brgemm(int M); +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} +template <> +inline bool can_use_brgemm(int M) { + return true; +} +// this requires PyTorch 2.7 or above +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +template <> +inline bool can_use_brgemm(int M) { + return M > 4; +} + +// work around compiler internal error +#define BLOCK_K 128 // 4 * TILE_K + +// adjust leading dimension size for K +template +inline int64_t get_row_size(int64_t K) { + return K; +} + +template <> +inline int64_t get_row_size(int64_t K) { + return K + sizeof(int32_t); +} + +inline int64_t get_row_size(int64_t K, bool use_int8_w8a8) { + return use_int8_w8a8 ? K + sizeof(int32_t) : K; +} + +// pack weight to vnni format +at::Tensor convert_weight_packed(at::Tensor& weight); + +// moe implementations for int8 w8a8 +template +void fused_experts_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); + +// moe implementations for fp8 w8a16 +template +void fused_experts_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad); + +// shared expert implementation for int8 w8a8 +template +void shared_expert_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K); + +template +void shared_expert_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K); + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + scalar_t* __restrict__ C, + float* __restrict__ Ctmp, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true); diff --git a/sgl-kernel/csrc/cpu/gemm_fp8.cpp b/sgl-kernel/csrc/cpu/gemm_fp8.cpp new file mode 100644 index 000000000..008f83298 --- /dev/null +++ b/sgl-kernel/csrc/cpu/gemm_fp8.cpp @@ -0,0 +1,551 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const float* __restrict__ input, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d); + fVec data1 = fVec::loadu(input + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d]); + } +} + +template +inline void copy_add_stub( + scalar_t* __restrict__ out, const float* __restrict__ input, const float* __restrict__ bias, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) + fVec::loadu(bias + d); + fVec data1 = fVec::loadu(input + d + fVec::size()) + fVec::loadu(bias + d + fVec::size()); + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + bias[d]); + } +} + +inline void unpack_B( + at::BFloat16* __restrict__ Btmp, + const at::Float8_e4m3fn* __restrict__ packed_B, + int N, + int K, + int ldb, + int ldb_tmp, + float scale) { +#if defined(CPU_CAPABILITY_AVX512) + // [K/2, N, 2] + const int K2 = K >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const uint16_t* b_ptr = reinterpret_cast(packed_B); + const __m512 vd = _mm512_set1_ps(scale); + + constexpr int BLOCK_N = block_size_n(); + static_assert(BLOCK_N == 32); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + +#pragma GCC unroll 4 + for (int k = 0; k < K2; ++k) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2, _MM_HINT_T0); + } + + __m256i b8_0 = _mm512_extracti32x8_epi32(b8, 0); + __m256i b8_1 = _mm512_extracti32x8_epi32(b8, 1); + + __m512bh bf16_0 = CVT_FP8_TO_BF16(b8_0); + __m512bh bf16_1 = CVT_FP8_TO_BF16(b8_1); + + // Apply scale + __m512 f0_lo = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_0, 0)); + __m512 f0_hi = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_0, 1)); + __m512 f1_lo = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_1, 0)); + __m512 f1_hi = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32((__m512i)bf16_1, 1)); + + f0_lo = _mm512_mul_ps(f0_lo, vd); + f0_hi = _mm512_mul_ps(f0_hi, vd); + f1_lo = _mm512_mul_ps(f1_lo, vd); + f1_hi = _mm512_mul_ps(f1_hi, vd); + + bf16_0 = _mm512_cvtne2ps_pbh(f0_hi, f0_lo); + bf16_1 = _mm512_cvtne2ps_pbh(f1_hi, f1_lo); + + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 0, (__m512i)bf16_0); + _mm512_storeu_si512(Btmp + k * ldb_tmp * 2 + 32, (__m512i)bf16_1); + } +#else + TORCH_CHECK(false, "unpack_B: scalar path not implemented!"); +#endif +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const packed_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ bias, + const float* __restrict__ scale, + int K, + int lda, + int ldb, + int ldc, + int64_t block_size_K) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ bias, + const float* __restrict__ scale, + int K, + int lda, + int ldb, + int ldc, + int64_t block_size_K) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + const int KB = div_up(K, BLOCK_K); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 64; + constexpr int PREFETCH_SIZE_KB = 1; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + __m512 vsum[ROWS * COLS]; + + // block quant scale + __m512 vscale; + + auto loadc = [&](auto i) { + constexpr int col = i % COLS; + if constexpr (has_bias) { + vc[i] = _mm512_loadu_ps(bias + col * 16); + } else { + vc[i] = _mm512_setzero_ps(); + } + }; + Unroll{}(loadc); + + const int lda2 = lda >> 1; + const int ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const uint16_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(a_ptr + row * lda2 + k + PREFETCH_SIZE_K, _MM_HINT_T0); + } + } + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + __m512i b8 = _mm512_loadu_si512(b_ptr + k * ldb2 + col * 16); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + vb[col + 0] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 0)); + vb[col + 1] = CVT_FP8_TO_BF16(_mm512_extracti32x8_epi32(b8, 1)); + } + } + vsum[i] = _mm512_dpbf16_ps(vsum[i], va, vb[col]); + }; + + constexpr int BLOCK_K2 = BLOCK_K >> 1; + for (int kb = 0; kb < KB; ++kb) { + int kb_start = kb * BLOCK_K2; + int kb_end = std::min(K >> 1, kb_start + BLOCK_K2); + // 1. load scale vector + vscale = _mm512_set1_ps(scale[kb]); + if constexpr (PREFETCH_SIZE_KB > 0) { + _mm_prefetch(scale + kb + PREFETCH_SIZE_KB, _MM_HINT_T0); + } + // 2. zero vsum for each block + Unroll{}([&](auto i) { vsum[i] = _mm512_setzero_ps(); }); + // 3. accumulate across each block + for (int k = kb_start; k < kb_end; ++k) { + Unroll{}(compute, k); + } + // 4. apply scale + Unroll{}([&](auto i) { vc[i] = _mm512_fmadd_ps(vsum[i], vscale, vc[i]); }); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2,4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(vc[row * COLS + col + 1], vc[row * COLS + col]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 2, \ + C + mb_start * ldc + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + scale, \ + K, \ + lda, \ + ldb, \ + ldc, \ + block_size_K); + +template +struct brgemm { + static inline void apply( + const scalar_t* __restrict__ A, + const packed_t* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + const float* __restrict__ scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc, + bool do_unpack = true) { + TORCH_CHECK(false, "struct brgemm: primary template not implemented!"); + } +}; + +template +struct brgemm { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + at::BFloat16* __restrict__ C, + at::BFloat16* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ bias, + const float* __restrict__ scale, + int M, + int N, + int K, + int lda, + int ldb, + int ldc, + bool do_unpack = true) { + constexpr int BLOCK_N = block_size_n(); + + // [K, BLOCK_N] -> [K / 2, BLOCK_N * 2] + const int ldb_tmp = BLOCK_N; + + if (do_unpack) { + for (int k = 0; k < K; k += BLOCK_K) { + int kb_size = std::min(BLOCK_K, K - k); + + int idx = k >> 7; // k / BLOCK_K where BLOCK_K = 128 + unpack_B(Btmp + k * ldb_tmp, B + k * ldb, N, kb_size, ldb, ldb_tmp, scale[idx]); + } + } + + at::native::cpublas::brgemm(M, N, K, lda, ldb_tmp, BLOCK_N, /* add_C */ false, A, Btmp, Ctmp); + + // copy from Ctmp to C + for (int m = 0; m < M; ++m) { + if constexpr (has_bias) { + copy_add_stub(C + m * ldc, Ctmp + m * BLOCK_N, bias, N); + } else { + copy_stub(C + m * ldc, Ctmp + m * BLOCK_N, N); + } + } + } +}; + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack = true) { + if (brg) { + brgemm::apply( + A, B, C, Btmp, Ctmp, bias, scale, M, N, K, lda, ldb, ldc, do_unpack); + return; + } + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void fp8_scaled_mm_kernel_impl( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ mat1, + const at::Float8_e4m3fn* __restrict__ mat2, + const float* __restrict__ scales2, + const float* __restrict__ bias, + scalar_t* __restrict__ buffer, + int64_t M, + int64_t N, + int64_t K, + int64_t mat1_strideM, + int64_t out_strideM, + int64_t block_size_N, + int64_t block_size_K, + int64_t buffer_size_per_thread) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const int64_t scale_size_K = div_up(K, block_size_K); + const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB] + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + scalar_t* __restrict__ Btmp = buffer + tid * buffer_size_per_thread; + float* __restrict__ Ctmp = (float*)((void*)(Btmp + MAX_CACHE_BLOCK_SIZE * BLOCK_N * K)); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + const float* scale_ptr = scales2 + (nb / blocks_n_per_group) * scale_size_K; + + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(M - mb_start, BLOCK_M); + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(N - nb_start, BLOCK_N); + + // only do unpacking for the first row + bool do_unpack = (mb == mb0); + + tinygemm_kernel( + /* A */ mat1 + mb_start * mat1_strideM, + /* B */ mat2 + nb_start * K, // nb * BLOCK_N * K + /* C */ out + mb_start * out_strideM + nb_start, + /* Btmp */ Btmp + nb_offset * BLOCK_N * K, + /* Ctmp */ Ctmp, + /* scale */ scale_ptr, + /* bias */ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ mat1_strideM, + /* ldb */ nb_size, + /* ldc */ out_strideM, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B, + scalar_t* __restrict__ C, + scalar_t* __restrict__ Btmp, + float* __restrict__ Ctmp, + const float* __restrict__ scale, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg, + int64_t block_size_K, + bool do_unpack) { + tinygemm_kernel( + A, B, C, Btmp, Ctmp, scale, nullptr, M, N, K, lda, ldb, ldc, brg, block_size_K, do_unpack); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const TYPE* __restrict__ A, \ + const at::Float8_e4m3fn* __restrict__ B, \ + TYPE* __restrict__ C, \ + TYPE* __restrict__ Btmp, \ + float* __restrict__ Ctmp, \ + const float* __restrict__ scale, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg, \ + int64_t block_size_K, \ + bool do_unpack) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::fp8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, block_size, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales2); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales2 to be float32."); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat2.size(1); + + CHECK_EQ(mat1.size(1), K); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + TORCH_CHECK(block_size.size() == 2, "fp8_scaled_mm_cpu: expect block_size.size() to be 2."); + + int64_t block_size_N = block_size[0]; + int64_t block_size_K = block_size[1]; + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + TORCH_CHECK(block_size_N % BLOCK_N == 0, "fp8_scaled_mm_cpu: expect block_size_N to be multiples of BLOCK_N"); + TORCH_CHECK(block_size_K == BLOCK_K, "fp8_scaled_mm_cpu: expect block_size_K equals to BLOCK_K"); + CHECK_EQ(scales2.size(0), div_up(N, block_size_N)); + CHECK_EQ(scales2.size(1), div_up(K, block_size_K)); + + const auto st = mat1.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "fp8_scaled_mm_cpu: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "fp8_scaled_mm_cpu: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kFloat8_e4m3fn, "fp8_scaled_mm_cpu: expect mat2 to be fp8_e4m3."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "fp8_scaled_mm_cpu: expect scales to be float32."); + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + // strides + int64_t mat1_strideM = mat1.stride(0); + int64_t out_strideM = out.stride(0); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + // Btmp : [T, BLOCK_N * K] + // Ctmp : [T, BLOCK_M * BLOCK_N] + int num_threads = at::get_num_threads(); + int64_t size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * K + BLOCK_M * BLOCK_N * 2; + auto buffer = at::empty({num_threads, size_per_thread}, mat1.options()); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "fp8_scaled_mm_kernel_impl", [&] { + fp8_scaled_mm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + scales2.data_ptr(), + bias_data, + buffer.data_ptr(), + M, + N, + K, + mat1_strideM, + out_strideM, + block_size_N, + block_size_K, + size_per_thread); + }); + + return out; +} diff --git a/sgl-kernel/csrc/cpu/gemm_int8.cpp b/sgl-kernel/csrc/cpu/gemm_int8.cpp new file mode 100644 index 000000000..cb6146607 --- /dev/null +++ b/sgl-kernel/csrc/cpu/gemm_int8.cpp @@ -0,0 +1,547 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +struct scale_C { + static inline void apply( + scalar_t* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct scale_C { + static inline void apply( + at::BFloat16* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + float As, + const float* __restrict__ Bs) { + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc[COLS]; + __m512 vd0 = _mm512_set1_ps(As); + + auto compute = [&](auto col) { + __m512 vd1 = _mm512_loadu_ps(Bs + col * 16); + __m512i vcomp = _mm512_loadu_si512(Bcomp + col * 16); + __m512i vc32 = _mm512_loadu_si512(Ctmp + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp)); + if constexpr (has_bias) { + __m512 vbias = _mm512_loadu_ps(bias + col * 16); + vc[col] = _mm512_fmadd_ps(_mm512_mul_ps(vc[col], vd0), vd1, vbias); + } else { + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vd0), vd1); + } + }; + Unroll{}(compute); + + auto storec = [&](auto col) { + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc[col + 1], vc[col + 0]))); + } + }; + Unroll{}(storec); + } +}; +#endif + +template +struct tinygemm_kernel_nn { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + const float* __restrict__ bias, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512i va; + __m512i vb[COLS]; + __m512i vc[ROWS * COLS]; + __m512i vcomp[COLS]; + __m512 vd0; + __m512 vd1[COLS]; + + // oops! 4x4 spills but we use 4x2 + __m512 vbias[COLS]; + + // [NOTE]: s8s8 igemm compensation in avx512-vnni + // + // avx512-vnni has no s8s8, so we need to change s8s8 to u8s8 with compensate: + // + // a * b = (a + 128) * b - 128 * b + // s s u s u s + // + // 1) 128 * b is pre-computed when packing B to vnni formats + // 2) a + 128 is fused when dynamically quantize A + // + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb[col] = _mm512_loadu_si512(b_ptr + k * ldb4 + col * 16); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb4 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbusd_epi32(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vd0 = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp per 2 vectors + // also load bias if any + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + vd1[col + 0] = _mm512_loadu_ps(Bs + col * 16); + vd1[col + 1] = _mm512_loadu_ps(Bs + col * 16 + 16); + vcomp[col + 0] = _mm512_loadu_si512(Bcomp + col * 16); + vcomp[col + 1] = _mm512_loadu_si512(Bcomp + col * 16 + 16); + if constexpr (has_bias) { + vbias[col + 0] = _mm512_loadu_ps(bias + col * 16); + vbias[col + 1] = _mm512_loadu_ps(bias + col * 16 + 16); + } + } + } + + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + __m512 vc0 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[row * COLS + col + 0], vcomp[col + 0])); + __m512 vc1 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[row * COLS + col + 1], vcomp[col + 1])); + if constexpr (has_bias) { + vc0 = _mm512_fmadd_ps(_mm512_mul_ps(vc0, vd0), vd1[col + 0], vbias[col + 0]); + vc1 = _mm512_fmadd_ps(_mm512_mul_ps(vc1, vd0), vd1[col + 1], vbias[col + 1]); + } else { + vc0 = _mm512_mul_ps(_mm512_mul_ps(vc0, vd0), vd1[col + 0]); + vc1 = _mm512_mul_ps(_mm512_mul_ps(vc1, vd0), vd1[col + 1]); + } + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), (__m512i)(_mm512_cvtne2ps_pbh(vc1, vc0))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + has_bias ? bias + nb_start : nullptr, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + // B compensation + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); + + if (brg) { + constexpr int BLOCK_N = block_size_n(); + at::native::cpublas::brgemm(M, N, K, lda, ldb, BLOCK_N, /* add_C */ false, A, B, Ctmp); + + // apply compensation and scale + for (int64_t m = 0; m < M; ++m) { + scale_C::apply(C + m * ldc, Ctmp + m * BLOCK_N, Bcomp, bias, As[m], Bs); + } + return; + } + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + case 0x14: + LAUNCH_TINYGEMM_KERNEL_NN(1, 64); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + case 0x24: + LAUNCH_TINYGEMM_KERNEL_NN(2, 64); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + case 0x34: + LAUNCH_TINYGEMM_KERNEL_NN(3, 64); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + case 0x44: + LAUNCH_TINYGEMM_KERNEL_NN(4, 64); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void int8_scaled_mm_kernel_impl( + scalar_t* __restrict__ out, + const uint8_t* __restrict__ mat1, + const int8_t* __restrict__ mat2, + const float* __restrict__ scales1, + const float* __restrict__ scales2, + const float* __restrict__ bias, + int64_t M, + int64_t N, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + const bool use_brgemm = can_use_brgemm(M); + + // K + 4 after compensation + const int64_t packed_row_size = get_row_size(K); + + AT_DISPATCH_BOOL(bias != nullptr, has_bias, [&] { + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // for brgemm, use int32_t for accumulate + alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = std::min(N - nb_start, BLOCK_N); + + tinygemm_kernel( + /* A */ mat1 + mb_start * K, + /* B */ mat2 + nb_start * packed_row_size /* nb * BLOCK_N * (K + 4) */, + /* C */ out + mb_start * N + nb_start, + /* Ctmp*/ Ctmp, + /* As */ scales1 + mb_start, + /* Bs */ scales2 + nb_start, + /* bias*/ bias + nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ N, + /* brg */ use_brgemm); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + }); +} + +} // anonymous namespace + +// tinygemm interface +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + scalar_t* __restrict__ C, + int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc, + bool brg) { + tinygemm_kernel(A, B, C, Ctmp, As, Bs, nullptr, M, N, K, lda, ldb, ldc, brg); +} + +#define INSTANTIATE_TINYGEMM_TEMPLATE(TYPE) \ + template void tinygemm_kernel( \ + const uint8_t* __restrict__ A, \ + const int8_t* __restrict__ B, \ + TYPE* __restrict__ C, \ + int32_t* __restrict__ Ctmp, \ + const float* __restrict__ As, \ + const float* __restrict__ Bs, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t lda, \ + int64_t ldb, \ + int64_t ldc, \ + bool brg) + +INSTANTIATE_TINYGEMM_TEMPLATE(at::BFloat16); +INSTANTIATE_TINYGEMM_TEMPLATE(at::Half); + +std::tuple per_token_quant_int8_cpu(at::Tensor& A) { + RECORD_FUNCTION("sgl-kernel::per_token_quant_int8_cpu", std::vector({A})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(A); + CHECK_DIM(2, A); + + int64_t M = A.size(0); + int64_t K = A.size(1); + int64_t lda = A.stride(0); + + const auto st = A.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "per_token_quant_int8: expect A to be bfloat16 or half."); + + auto Aq = at::empty({M, K}, A.options().dtype(at::kByte)); + auto As = at::empty({M}, A.options().dtype(at::kFloat)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "per_token_quant_int8", [&] { + uint8_t* __restrict__ Aq_data = Aq.data_ptr(); + float* __restrict__ As_data = As.data_ptr(); + const scalar_t* __restrict__ A_data = A.data_ptr(); + + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); + } + }); + }); + return std::make_tuple(Aq, As); +} + +// weight : static, per-channel, symmetric +// activation : dynamic, per-token, symmetric +// +// mat1 : [M, K] +// mat2 : [N, K] +// scales1 : [M] +// scales2 : [N] +// bias : [N] +// out : [M, N] +// +at::Tensor int8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales1, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales1, scales2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales1); + CHECK_INPUT(scales2); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat1.size(1); + + // see [NOTE]: s8s8 igemm compensation in avx512-vnni + CHECK_EQ(mat2.size(1), (int64_t)(is_vnni ? K + sizeof(int32_t) : K)); + CHECK_EQ(scales1.numel(), M); + CHECK_EQ(scales2.numel(), N); + + TORCH_CHECK(mat1.scalar_type() == at::kByte, "int8_scaled_mm: expect mat1 to be uint8."); + TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm: expect mat2 to be int8."); + TORCH_CHECK( + scales1.scalar_type() == at::kFloat && scales2.scalar_type() == at::kFloat, + "int8_scaled_mm: expect scales to be float32."); + + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "int8_scaled_mm_kernel_impl", [&] { + int8_scaled_mm_kernel_impl( + out.data_ptr(), + mat1.data_ptr(), + packed_w.data_ptr(), + scales1.data_ptr(), + scales2.data_ptr(), + bias_data, + M, + N, + K); + }); + return out; +} + +// fused `per_token_quant_int8_cpu` and `int8_scaled_mm_cpu` +at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::int8_scaled_mm_cpu", std::vector({mat1, mat2, scales2, bias})); + + auto packed_w = is_vnni ? mat2 : convert_weight_packed(mat2); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(mat1); + CHECK_INPUT(mat2); + CHECK_INPUT(scales2); + CHECK_DIM(2, mat1); + CHECK_DIM(2, mat2); + + int64_t M = mat1.size(0); + int64_t N = mat2.size(0); + int64_t K = mat1.size(1); + int64_t lda = mat1.stride(0); + + // see [NOTE]: s8s8 igemm compensation in avx512-vnni + CHECK_EQ(mat2.size(1), (int64_t)(is_vnni ? K + sizeof(int32_t) : K)); + CHECK_EQ(scales2.numel(), N); + + const auto st = mat1.scalar_type(); + TORCH_CHECK(st == at::kBFloat16 || st == at::kHalf, "int8_scaled_mm_with_quant: expect A to be bfloat16 or half."); + TORCH_CHECK(st == out_dtype, "int8_scaled_mm_with_quant: expect A has same dtype with out_dtype."); + TORCH_CHECK(mat2.scalar_type() == at::kChar, "int8_scaled_mm_with_quant: expect mat2 to be int8."); + TORCH_CHECK(scales2.scalar_type() == at::kFloat, "int8_scaled_mm_with_quant: expect scales to be float32."); + + const int64_t buffer_size = M * K + M * sizeof(float); + auto buffer = at::empty({buffer_size}, mat1.options().dtype(at::kByte)); + auto out = at::empty({M, N}, mat1.options().dtype(out_dtype)); + + const bool has_bias = bias.has_value(); + const float* bias_data = nullptr; + if (has_bias) { + CHECK_EQ(bias.value().size(0), N); + bias_data = bias.value().data_ptr(); + } + + AT_DISPATCH_REDUCED_FLOATING_TYPES(out_dtype, "int8_scaled_mm_with_quant_kernel_impl", [&] { + uint8_t* __restrict__ Aq_data = buffer.data_ptr(); + float* __restrict__ As_data = (float*)((void*)(Aq_data + M * K)); + const scalar_t* __restrict__ A_data = mat1.data_ptr(); + + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * K, As_data[m], A_data + m * lda, K); + } + }); + + int8_scaled_mm_kernel_impl( + out.data_ptr(), + Aq_data, + packed_w.data_ptr(), + As_data, + scales2.data_ptr(), + bias_data, + M, + N, + K); + }); + return out; +} diff --git a/sgl-kernel/csrc/cpu/interface.cpp b/sgl-kernel/csrc/cpu/interface.cpp new file mode 100644 index 000000000..9057a50f4 --- /dev/null +++ b/sgl-kernel/csrc/cpu/interface.cpp @@ -0,0 +1,74 @@ +#include +#include + +#include "shm.h" + +// Communication settings +static int world_rank = -1; +static int world_size = -1; + +static bool is_initialized = false; + +static bool all_ranks_local_p = false; + +void initialize(int64_t size, int64_t rank) { + if (is_initialized) { + return; + } + + // Check whether all ranks is on the same physical machine. + // If true, we will use an SHM based low latency allreduce + + auto ls_string = std::getenv("LOCAL_SIZE"); + int ls = 0; + if (ls_string != NULL) { + ls = std::stoi(std::getenv("LOCAL_SIZE")); + } + + if (size >= 1 && size == ls) { + all_ranks_local_p = true; + } + + world_size = size; + world_rank = rank; + is_initialized = true; + + const char* addr_string = std::getenv("MASTER_ADDR"); + if (addr_string == NULL) { + addr_string = ""; + } + const char* port_string = std::getenv("MASTER_PORT"); + if (port_string == NULL) { + port_string = ""; + } + + if (all_ranks_local_p) { + shm_initialize(size, rank, addr_string, port_string); + } +} + +void shm_allreduce(torch::Tensor& data, int64_t op) { + RECORD_FUNCTION("sgl-kernel::shm_allreduce", std::vector({data})); + + TORCH_CHECK(op == c10d::ReduceOp::SUM, "Only torch.distributed.ReduceOp.SUM is supported"); + + auto numel = data.numel(); + int data_size = numel * data.element_size(); + all_reduce_outer_loop(data, numel, data_size); + + return; +} + +torch::Tensor shm_allgather(torch::Tensor& data, int64_t dim) { + RECORD_FUNCTION("sgl-kernel::shm_allgather", std::vector({data})); + + auto numel = data.numel(); + int data_size = numel * data.element_size(); + if (dim < 0) { + dim += data.dim(); + } + std::vector result_shape = data.sizes().vec(); + result_shape[dim] *= world_size; + torch::Tensor result_tensor = torch::empty(result_shape, data.options()); + return all_gather(result_tensor, data, dim, numel, data_size); +} diff --git a/sgl-kernel/csrc/cpu/moe.cpp b/sgl-kernel/csrc/cpu/moe.cpp new file mode 100644 index 000000000..c3d66cec7 --- /dev/null +++ b/sgl-kernel/csrc/cpu/moe.cpp @@ -0,0 +1,1322 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: Fused MoE kernel with AMX +// +// This file contains implementations for +// * `moe_align_block_size` +// * `fused_moe` +// +// The functionality is identical to triton kernel, excepts: +// * fuse silu_and_mul with gemm1, therefore this kernel +// allocates 2 intermediate_caches instead of 3 +// * add `offsets` in `moe_align_block_size` which keeps track +// of starting offset for each M block. this is for keeping +// output of silu_and_mul in sorted order, thus load_A for +// the 2nd gemm would be contiguous, therefore we can directly +// load A from intermediate_cache1. +// +// TODO: +// 1. tune BLOCK_M and BLOCK_N (BLOCK_N * K fit L2) +// 2. add prefetch for load A which is indexed access +// 3. abstract at::native::cpublas::brgemm with WoQ gemm (M = 1 & M != 1) +// + +template +inline void fill_stub(scalar_t* __restrict__ out, scalar_t val, int64_t size) { + using Vec = at::vec::Vectorized; + const Vec data_vec(val); + at::vec::map([data_vec](Vec out) { return out = data_vec; }, out, out, size); +} + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * weight_vec; + fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const float* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec x0 = fVec::loadu(input + d); + fVec x1 = fVec::loadu(input + d + fVec::size()); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +int moe_align_block_size( + int32_t* __restrict__ sorted_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ topk_ids, + int32_t* __restrict__ total_cnts, + int32_t* __restrict__ cumsums, + int32_t* __restrict__ offsets, + int num_experts, + int numel, + int num_threads) { +#define T_INDEX(tt) total_cnts + (tt) * num_experts + + // accumulate count of expert ids locally + at::parallel_for(0, numel, 0, [&](int begin, int end) { + int tid = at::get_thread_num(); + int32_t* __restrict__ local_cnts = T_INDEX(tid + 1); + + for (int i = begin; i < end; ++i) { + local_cnts[topk_ids[i]]++; + } + }); + + using iVec = at::vec::Vectorized; + for (int t = 0; t < num_threads; ++t) { + at::vec::map2( + [](iVec x, iVec y) { return x + y; }, T_INDEX(t + 1), T_INDEX(t + 1), T_INDEX(t), num_experts); + } + + // the last row holds sums of each experts + int32_t* total_cnts_t_1 = T_INDEX(num_threads); + + cumsums[0] = 0; + for (int e = 0; e < num_experts; ++e) { + // accumulate `num_tokens_post_pad`, also as the expert offset + cumsums[e + 1] = cumsums[e] + div_up(total_cnts_t_1[e], BLOCK_M) * BLOCK_M; + + for (int k = cumsums[e]; k < cumsums[e + 1]; k += BLOCK_M) { + expert_ids[k / BLOCK_M] = e; + } + } + int num_tokens_post_pad = cumsums[num_experts]; + + at::parallel_for(0, numel, 0, [&](int begin, int end) { + int tid = at::get_thread_num(); + // thread tid offsets in `total_cnts` + int32_t* __restrict__ offsets = T_INDEX(tid); + + for (int i = begin; i < end; ++i) { + int32_t expert_id = topk_ids[i]; + int32_t b_offset = cumsums[expert_id]; + int32_t t_offset = offsets[expert_id]; + sorted_ids[b_offset + t_offset] = i; + offsets[expert_id]++; + } + }); + + // debug: the offset for thread t_1 should be identical to t_2 + int32_t* total_cnts_t_2 = T_INDEX(num_threads - 1); + for (int e = 0; e < num_experts; ++e) { + TORCH_CHECK(total_cnts_t_1[e] == total_cnts_t_2[e]); + } + + // padding value for sorted_ids: numel + auto sorted_id_size = [=](const int32_t* sorted_ids_ptr) { + for (int d = 0; d < BLOCK_M; ++d) { + if (sorted_ids_ptr[d] == numel) { + return d; + } + } + return BLOCK_M; + }; + + // offsets holds starting offset for each valida M blocks + // shape : [num_token_blocks + 1] + offsets[0] = 0; + const int num_token_blocks = num_tokens_post_pad / BLOCK_M; + at::parallel_for(0, num_token_blocks, GRAIN_SIZE / BLOCK_M, [&](int begin, int end) { + for (int mb = begin; mb < end; ++mb) { + offsets[mb + 1] = sorted_id_size(sorted_ids + mb * BLOCK_M); + } + }); + // TODO: do we need to vecterize this ? + for (int mb = 0; mb < num_token_blocks; ++mb) { + offsets[mb + 1] += offsets[mb]; + } + // debug: the last value of offsets should be `numel` + TORCH_CHECK(offsets[num_token_blocks] == numel); + + return num_tokens_post_pad; +} + +// silu : shape leading dimension +// input0 [m_size, BLOCK_N] BLOCK_N +// input1 [m_size, BLOCK_N] BLOCK_N +// output [M * topk, N] N +template +inline void silu_and_mul( + scalar_t* __restrict__ output, + const float* __restrict__ input0, // x: x0, x1 + const float* __restrict__ input1, // y: y0, y1 + int64_t m_size, + int64_t N) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + const fVec one = fVec(1.f); + + // no remainder + for (int64_t m = 0; m < m_size; ++m) { + scalar_t* __restrict__ out = output + m * N; + const float* __restrict__ x = input0 + m * BLOCK_N; + const float* __restrict__ y = input1 + m * BLOCK_N; + + for (int64_t d = 0; d < BLOCK_N; d += bVec::size()) { + fVec x0 = fVec::loadu(x + d); + fVec x1 = fVec::loadu(x + d + fVec::size()); + fVec y0 = fVec::loadu(y + d); + fVec y1 = fVec::loadu(y + d + fVec::size()); + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + // convert + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + } +} + +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + scalar_t* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn2 { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B0, + const at::BFloat16* __restrict__ B1, + at::BFloat16* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb0[COLS]; + __m512bh vb1[COLS]; + __m512 vc0[ROWS * COLS]; + __m512 vc1[ROWS * COLS]; + + auto loadc = [&](auto i) { + vc0[i] = _mm512_set1_ps(0.f); + vc1[i] = _mm512_set1_ps(0.f); + }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b0_ptr = reinterpret_cast(B0); + const float* b1_ptr = reinterpret_cast(B1); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb0[col] = (__m512bh)(_mm512_loadu_si512(b0_ptr + k * ldb2 + col * 16)); + vb1[col] = (__m512bh)(_mm512_loadu_si512(b1_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b0_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + _mm_prefetch(b1_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc0[i] = _mm512_dpbf16_ps(vc0[i], va, vb0[col]); + vc1[i] = _mm512_dpbf16_ps(vc1[i], va, vb1[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + using Vec = at::vec::Vectorized; + const Vec one = Vec(1.f); + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + Vec x0 = vc0[row * COLS + col + 0]; + Vec x1 = vc0[row * COLS + col + 1]; + Vec y0 = vc1[row * COLS + col + 0]; + Vec y1 = vc1[row * COLS + col + 1]; + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn2::apply( \ + A + mb_start * lda, B0 + nb_start * 2, B1 + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + scalar_t* __restrict__ C, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // pattern: 1-(2+2)-(8+8) + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN(1, 32); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN(2, 32); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN(3, 32); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +struct tinygemm_kernel_nn { + static inline void apply( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_nn { + static inline void apply( + const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + float* __restrict__ C, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + + static_assert(COLS % 2 == 0); + + // prefetch distance + constexpr int PREFETCH_SIZE_K = 0; + + __m512bh va; + __m512bh vb[COLS]; + __m512 vc[ROWS * COLS]; + + auto loadc = [&](auto i) { vc[i] = _mm512_set1_ps(0.f); }; + Unroll{}(loadc); + + const int64_t K2 = K >> 1; + const int64_t lda2 = lda >> 1; + const int64_t ldb2 = ldb; // ldb * 2 >> 1; + const float* a_ptr = reinterpret_cast(A); + const float* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = (__m512bh)(_mm512_set1_ps(a_ptr[row * lda2 + k])); + } + if constexpr (row == 0) { + vb[col] = (__m512bh)(_mm512_loadu_si512(b_ptr + k * ldb2 + col * 16)); + if constexpr (PREFETCH_SIZE_K > 0) { + _mm_prefetch(b_ptr + (k + PREFETCH_SIZE_K) * ldb2 + col * 16, _MM_HINT_T0); + } + } + vc[i] = _mm512_dpbf16_ps(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K2; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), vc[i]); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_NN2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_nn::apply( \ + A + mb_start * lda, B + nb_start * 2, C + mb_start * ldc + nb_start, K, lda, ldb, ldc); + +template +void tinygemm_kernel( + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B, + float* __restrict__ C, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // pattern: 1-2-8 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + // mb_size = 1 + case 0x12: + LAUNCH_TINYGEMM_KERNEL_NN2(1, 32); + break; + // mb_size = 2 + case 0x22: + LAUNCH_TINYGEMM_KERNEL_NN2(2, 32); + break; + // mb_size = 3 + case 0x32: + LAUNCH_TINYGEMM_KERNEL_NN2(3, 32); + break; + // mb_size = 4 + case 0x42: + LAUNCH_TINYGEMM_KERNEL_NN2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +template +void fused_experts_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ packed_w1, + const scalar_t* __restrict__ packed_w2, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // strides for w1: [E, 2N, K] + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + const int64_t stride_e = 2 * N * K; + const int64_t stride_n = K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const scalar_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + // 1.d silu and mul + const int64_t offset = offsets[mb]; + silu_and_mul(ic1 + offset * N + nb * BLOCK_N, C0, C1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + offset * N + nb * BLOCK_N, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_e2 = OC * IC; + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + // we won't be using C1 for gemm2 + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const scalar_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +template +void shared_expert_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + scalar_t* __restrict__ input, + const scalar_t* __restrict__ packed_w1, + const scalar_t* __restrict__ packed_w2, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + const int64_t stride_n = K; + + const bool use_brgemm = can_use_brgemm(M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C0 = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + float* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + + // A shape [m_size, K] + const scalar_t* A = input + mb * BLOCK_M * K; + + // B shape [K, n_size] in vnni format + const scalar_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const scalar_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + // 1.d silu and mul + silu_and_mul(ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 2: output = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + // we won't be using C1 for gemm2 + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A shape [m_size, IC] + const scalar_t* __restrict__ A = ic1 + mb * BLOCK_M * N; + + // B shape [IC, n_size] in vnni format + const scalar_t* __restrict__ B = packed_w2 + nb * BLOCK_N * stride_oc; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +} // anonymous namespace + +// common checks +static inline void check_moe_scales( + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale) { + if (use_int8_w8a8) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for int8 w8a8."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for int8 w8a8."); + TORCH_CHECK(!a1_scale.has_value(), "static quantization for activation not supported."); + TORCH_CHECK(!a2_scale.has_value(), "static quantization for activation not supported."); + } + if (use_fp8_w8a16) { + TORCH_CHECK(w1_scale.has_value(), "missing w1_scale for fp8 w8a16."); + TORCH_CHECK(w2_scale.has_value(), "missing w2_scale for fp8 w8a16."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16."); + TORCH_CHECK(block_size.value().size() == 2, "expect block_size.size() to be 2."); + } +} + +#define CHECK_MOE_SCALES_FP8(DIM0, DIM1) \ + auto w1s = w1_scale.value(); \ + auto w2s = w2_scale.value(); \ + auto block_size_val = block_size.value(); \ + int64_t block_size_N = block_size_val[0]; \ + int64_t block_size_K = block_size_val[1]; \ + TORCH_CHECK(w1s.size(DIM0) == div_up(2 * N, block_size_N)); \ + TORCH_CHECK(w1s.size(DIM1) == div_up(K, block_size_K)); \ + TORCH_CHECK(w2s.size(DIM0) == div_up(K, block_size_N)); \ + TORCH_CHECK(w2s.size(DIM1) == div_up(N, block_size_K)) + +// hidden_states: [M, K] +// w1: [E, 2N, K] +// w2: [E, K, N] +// topk_weights: [M, topk] +// topk_ids: [M, topk] (int32_t) +// +at::Tensor fused_experts_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& topk_weights, + at::Tensor& topk_ids, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni) { + RECORD_FUNCTION( + "sgl-kernel::fused_experts_cpu", std::vector({hidden_states, w1, w2, topk_weights, topk_ids})); + + auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); + auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(w1); + CHECK_INPUT(w2); + CHECK_EQ(topk_weights.sizes(), topk_ids.sizes()); + CHECK_DIM(2, hidden_states); + CHECK_DIM(3, w1); + CHECK_DIM(3, w2); + CHECK_DIM(2, topk_weights); + CHECK_DIM(2, topk_ids); + + CHECK_EQ(topk_ids.scalar_type(), at::kInt); + + // TODO: support topk_weights to be bf16 or fp16 in the kernel. + // The topk_weights of llama4 is computed via Llama4MoE:custom_routing_function and is bf16/fp16 + // while the kernel currently only supports it to be float32 + auto topk_weights_ = topk_weights.to(at::kFloat); + CHECK_EQ(topk_weights_.scalar_type(), at::kFloat); + + int64_t M = hidden_states.size(0); + int64_t K = hidden_states.size(1); + int64_t N = w1.size(1) / 2; + int64_t E = w1.size(0); + int64_t topk = topk_weights_.size(1); + + // we use int32_t compensation for int8 w8a8 + int64_t packed_K = get_row_size(K, use_int8_w8a8); + int64_t packed_N = get_row_size(N, use_int8_w8a8); + + // check weight shapes + CHECK_EQ(w2.size(0), E); + CHECK_EQ(w2.size(1), K); + CHECK_EQ(packed_w1.size(2), packed_K); + CHECK_EQ(packed_w2.size(2), packed_N); + + // check scales + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + + at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); + + // NB: worst case is each expert holds a block with remainder of 1 + // 1. sorted_ids : [M * topk + E * (BLOCK_M - 1)] + // 2. expert_ids : [max_num_blocks] + // 3. total_cnts : [T + 1, E] + // 4. cumsums : [E + 1] + // 5. offsets : [max_num_blocks + 1] + // + int num_threads = at::get_num_threads(); + int64_t max_num_tokens_padded = M * topk + E * (BLOCK_M - 1); + int64_t max_num_blocks = div_up(max_num_tokens_padded, BLOCK_M); + auto buffer = at::empty( + {max_num_tokens_padded + max_num_blocks + (num_threads + 1) * E + (E + 1) + (max_num_blocks + 1)}, + topk_ids.options()); + + int32_t* __restrict__ sorted_ids = buffer.data_ptr(); + int32_t* __restrict__ expert_ids = sorted_ids + max_num_tokens_padded; + int32_t* __restrict__ total_cnts = expert_ids + max_num_blocks; + int32_t* __restrict__ cumsums = total_cnts + (num_threads + 1) * E; + int32_t* __restrict__ offsets = cumsums + (E + 1); + + // init sorted_ids with `numel` as the padding number + // init expert_ids with `num_experts` + int64_t numel = M * topk; + at::parallel_for(0, max_num_blocks, GRAIN_SIZE / BLOCK_M, [&](int64_t begin, int64_t end) { + int64_t m_start = begin * BLOCK_M; + int64_t m_size = std::min((end - begin) * BLOCK_M, max_num_tokens_padded - m_start); + fill_stub(sorted_ids + m_start, (int32_t)numel, m_size); + fill_stub(expert_ids + begin, (int32_t)E, end - begin); + }); + // zero total_cnts and cumsums + at::parallel_for(0, (num_threads + 1) * E + (E + 1), GRAIN_SIZE, [&](int64_t begin, int64_t end) { + fill_stub(total_cnts + begin, 0, end - begin); + }); + + // align experts index + int64_t num_tokens_post_pad = moe_align_block_size( + sorted_ids, expert_ids, topk_ids.data_ptr(), total_cnts, cumsums, offsets, E, numel, num_threads); + + // unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches: + // 1. intermediate_cache1 : [M * topk, N] + // 2. intermediate_cache2 : [M * topk, K] + // 3. A_tmp : [T, BLOCK_M * K] + // 4. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] + // + // for int8 w8a8: + // 5. Aq_tmp : [M, K] or [M * topk, N] + // 6. As_tmp : [M * topk] + // + // for fp8 w8a16: + // 7. intermediate_cache0 : [M * topk, 2N] + // 8. B_tmp : [T, MAX_CACHE_BLOCK_SIZE, BLOCK_N, std::max(K, N)] + // + int64_t buffer_size_nbytes = M * topk * N * 2 + M * topk * K * 2 + + num_threads * BLOCK_M * K * (use_int8_w8a8 ? 1 : 2) + + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); + + if (use_int8_w8a8) { + buffer_size_nbytes += std::max(M * K, M * topk * N) + M * topk * sizeof(float); + } + if (use_fp8_w8a16) { + buffer_size_nbytes += M * topk * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N) * 2; + } + + auto buffer2 = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "fused_experts_kernel_impl", [&] { + scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer2.data_ptr())); + scalar_t* __restrict__ intermediate_cache2 = intermediate_cache1 + M * topk * N; + + if (use_int8_w8a8) { + uint8_t* __restrict__ A_tmp = (uint8_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * topk * N))); + + auto w1s = w1_scale.value(); + auto w2s = w2_scale.value(); + TORCH_CHECK(w1s.numel() == E * 2 * N); + TORCH_CHECK(w2s.numel() == E * K); + + fused_experts_int8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + intermediate_cache2, + A_tmp, + C_tmp, + Aq_tmp, + As_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } else if (use_fp8_w8a16) { + // here we just ignore C_tmp as it is not used + scalar_t* __restrict__ A_tmp = (scalar_t*)((void*)(intermediate_cache2 + M * topk * K)); + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * topk * 2 * N)); + + CHECK_MOE_SCALES_FP8(1, 2); + fused_experts_fp8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + intermediate_cache2, + A_tmp, + B_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } else { + scalar_t* __restrict__ A_tmp = intermediate_cache2 + M * topk * K; + float* __restrict__ C_tmp = (float*)((void*)(A_tmp + num_threads * BLOCK_M * K)); + + fused_experts_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + intermediate_cache2, + A_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + topk_weights_.data_ptr(), + sorted_ids, + expert_ids, + offsets, + M, + N, + K, + E, + topk, + num_tokens_post_pad); + } + }); + return out_hidden_states; +} + +// shared expert kernel +// +// hidden_states: [M, K] +// w1: [2N, K] +// w2: [K, N] +// fused_experts_out +at::Tensor shared_expert_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& fused_experts_out, + double routed_scaling_factor, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni) { + RECORD_FUNCTION("sgl-kernel::shared_expert_cpu", std::vector({hidden_states, w1, w2})); + + auto packed_w1 = is_vnni ? w1 : convert_weight_packed(w1); + auto packed_w2 = is_vnni ? w2 : convert_weight_packed(w2); + + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(fused_experts_out); + CHECK_INPUT(w1); + CHECK_INPUT(w2); + CHECK_DIM(2, hidden_states); + CHECK_DIM(2, w1); + CHECK_DIM(2, w2); + CHECK_EQ(hidden_states.sizes(), fused_experts_out.sizes()); + CHECK_EQ(hidden_states.scalar_type(), st); + + int64_t M = hidden_states.size(0); + int64_t K = hidden_states.size(1); + int64_t N = w1.size(0) / 2; + + // we use int32_t compensation for int8 w8a8 + int64_t packed_K = get_row_size(K, use_int8_w8a8); + int64_t packed_N = get_row_size(N, use_int8_w8a8); + + // check weight shapes + CHECK_EQ(w2.size(0), K); + CHECK_EQ(packed_w1.size(1), packed_K); + CHECK_EQ(packed_w2.size(1), packed_N); + + // check scales + check_moe_scales(use_int8_w8a8, use_fp8_w8a16, w1_scale, w2_scale, block_size, a1_scale, a2_scale); + + at::Tensor out_hidden_states = inplace ? hidden_states : at::empty_like(hidden_states); + + // unlike triton kernel, we fuse silu with gemm1 so only need 2 intermediate_caches: + // 1. intermediate_cache1 : [M, N] + // 2. C_tmp : [T, 2 * BLOCK_M * BLOCK_N] + // + // for int8 w8a8: + // 3. Aq_tmp : [M, K] or [M, N] + // 4. As_tmp : [M] + // + // for fp8 w8a16: + // 5. intermediate_cache0 : [M, 2N] + // 6. B_tmp: [T, MAX_CACHE_BLOCK_SIZE, BLOCK_M, max(K, N)] + // + int num_threads = at::get_num_threads(); + int64_t buffer_size_nbytes = M * N * 2 + num_threads * 2 * BLOCK_M * BLOCK_N * sizeof(float); + + if (use_int8_w8a8) { + buffer_size_nbytes += std::max(M * K, M * N) + M * sizeof(float); + } + if (use_fp8_w8a16) { + buffer_size_nbytes += M * 2 * N * 2 + num_threads * MAX_CACHE_BLOCK_SIZE * BLOCK_M * std::max(K, N) * 2; + } + + auto buffer = at::empty({buffer_size_nbytes}, hidden_states.options().dtype(at::kChar)); + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "share_experts_kernel_impl", [&] { + scalar_t* __restrict__ intermediate_cache1 = (scalar_t*)((void*)(buffer.data_ptr())); + float* __restrict__ C_tmp = (float*)((void*)(intermediate_cache1 + M * N)); + + if (use_int8_w8a8) { + uint8_t* __restrict__ Aq_tmp = (uint8_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + float* __restrict__ As_tmp = (float*)((void*)(Aq_tmp + std::max(M * K, M * N))); + + auto w1s = w1_scale.value(); + auto w2s = w2_scale.value(); + TORCH_CHECK(w1s.numel() == 2 * N); + TORCH_CHECK(w2s.numel() == K); + + shared_expert_int8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + C_tmp, + Aq_tmp, + As_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } else if (use_fp8_w8a16) { + scalar_t* __restrict__ intermediate_cache0 = (scalar_t*)((void*)(C_tmp + num_threads * 2 * BLOCK_M * BLOCK_N)); + scalar_t* __restrict__ B_tmp = (scalar_t*)((void*)(intermediate_cache0 + M * 2 * N)); + + CHECK_MOE_SCALES_FP8(0, 1); + shared_expert_fp8_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache0, + intermediate_cache1, + B_tmp, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + w1s.data_ptr(), + w2s.data_ptr(), + block_size_N, + block_size_K, + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } else { + shared_expert_kernel_impl( + out_hidden_states.data_ptr(), + intermediate_cache1, + C_tmp, + hidden_states.data_ptr(), + packed_w1.data_ptr(), + packed_w2.data_ptr(), + fused_experts_out.data_ptr(), + routed_scaling_factor, + M, + N, + K); + } + }); + return out_hidden_states; +} diff --git a/sgl-kernel/csrc/cpu/moe_fp8.cpp b/sgl-kernel/csrc/cpu/moe_fp8.cpp new file mode 100644 index 000000000..281c00897 --- /dev/null +++ b/sgl-kernel/csrc/cpu/moe_fp8.cpp @@ -0,0 +1,491 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + x0 = x0 * weight_vec; + x1 = x1 * weight_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x_bvec); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +inline void silu_and_mul_stub( + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const scalar_t* __restrict__ input2, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + + // no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x = bVec::loadu(input + d); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x); + bVec y = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y); + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + x0 = x0 * y0; + x1 = x1 * y1; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } +} + +} // anonymous namespace + +template +void fused_experts_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + scalar_t* __restrict__ A_tmp, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(2 * N, BLOCK_N); + int64_t scale_size_N = div_up(2 * N, block_size_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const int64_t stride_e = 2 * N * K; + const int64_t stride_n = K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + scalar_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w1 + expert_id * stride_e + nb * BLOCK_N * stride_n; + const float* __restrict__ Bs = + w1s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, input + index * K, K); + } + + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ ic0 + offset * 2 * N + nb * BLOCK_N, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ 2 * N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + scale_size_N = div_up(K, block_size_N); + scale_size_K = div_up(N, block_size_K); + const int64_t stride_e2 = OC * IC; + const int64_t stride_oc = IC; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const scalar_t* __restrict__ A = ic1 + offsets[mb] * N; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const at::Float8_e4m3fn* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = + w2s + expert_id * scale_size_N * scale_size_K + (nb / blocks_n_per_group) * scale_size_K; + + // do unpacking for the first row or a new expert + int32_t pre_expert_id = mb == 0 ? -1 : expert_ids[mb - 1]; + bool do_unpack = (mb == mb0) || (expert_id != pre_expert_id); + + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_FP8_TEMPLATE(TYPE) \ + template void fused_experts_fp8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + TYPE* __restrict__ A_tmp, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + const TYPE* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_FP8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_FP8_TEMPLATE(at::Half); + +template +void shared_expert_fp8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic0, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ B_tmp, + float* __restrict__ C_tmp, + const scalar_t* __restrict__ input, + const at::Float8_e4m3fn* __restrict__ packed_w1, + const at::Float8_e4m3fn* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + int64_t block_size_N, + int64_t block_size_K, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 1: intermediate_cache0 = hidden_states @ w1 + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(2 * N, BLOCK_N); + int64_t scale_size_K = div_up(K, block_size_K); + int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + int64_t B_tmp_size_per_thread = MAX_CACHE_BLOCK_SIZE * BLOCK_N * std::max(K, N); + + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(2 * N - nb * BLOCK_N, BLOCK_N); + + // do unpacking for the first row + bool do_unpack = (mb == mb0); + + tinygemm_kernel( + /* A */ input + mb * BLOCK_M * K, + /* B */ packed_w1 + nb * BLOCK_N * K, + /* C */ ic0 + mb * BLOCK_M * 2 * N + nb * BLOCK_N, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * K, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ w1s + (nb / blocks_n_per_group) * scale_size_K, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ 2 * N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: intermediate_cache1 = silu(intermediate_cache0) + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + silu_and_mul_stub(ic1 + m * N, ic0 + m * 2 * N, ic0 + m * 2 * N + N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(K, BLOCK_N); + scale_size_K = div_up(N, block_size_K); + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + int tid = get_thread_num(); + alignas(64) scalar_t C[BLOCK_M * BLOCK_K]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // do unpacking for the first row + bool do_unpack = (mb == mb0); + + // 2.a gemm: C = A @ B + tinygemm_kernel( + /* A */ ic1 + mb * BLOCK_M * N, + /* B */ packed_w2 + nb * BLOCK_N * N, + /* C */ C, + /* Btmp */ B_tmp + tid * B_tmp_size_per_thread + nb_offset * BLOCK_N * IC, + /* Ctmp */ C_tmp + tid * 2 * BLOCK_M * BLOCK_N, + /* scale */ w2s + (nb / blocks_n_per_group) * scale_size_K, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K, + /* do_unpack */ do_unpack); + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } +} + +#define INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(TYPE) \ + template void shared_expert_fp8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic0, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ B_tmp, \ + float* __restrict__ C_tmp, \ + const TYPE* __restrict__ input, \ + const at::Float8_e4m3fn* __restrict__ packed_w1, \ + const at::Float8_e4m3fn* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + int64_t block_size_N, \ + int64_t block_size_K, \ + const TYPE* __restrict__ fused_experts_out, \ + float routed_scaling_factor, \ + int64_t M, \ + int64_t N, \ + int64_t K) + +INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(at::BFloat16); +INSTANTIATE_SHARED_EXPERT_FP8_TEMPLATE(at::Half); diff --git a/sgl-kernel/csrc/cpu/moe_int8.cpp b/sgl-kernel/csrc/cpu/moe_int8.cpp new file mode 100644 index 000000000..8fbac902f --- /dev/null +++ b/sgl-kernel/csrc/cpu/moe_int8.cpp @@ -0,0 +1,1068 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +template +inline void copy_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t size) { + using Vec = at::vec::Vectorized; +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += Vec::size()) { + Vec data = Vec::loadu(input + d); + data.store(out + d); + } +} + +template <> +inline void copy_stub(uint8_t* __restrict__ out, const uint8_t* __restrict__ input, int64_t size) { + // size might be 64x + 32 + std::memcpy(out, input, size * sizeof(uint8_t)); +} + +template +inline void copy_mul_stub(scalar_t* __restrict__ out, const float* __restrict__ input, float weight, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec weight_vec = fVec(weight); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec data0 = fVec::loadu(input + d) * weight_vec; + fVec data1 = fVec::loadu(input + d + fVec::size()) * weight_vec; + bVec out_vec = convert_from_float_ext(data0, data1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] * weight); + } +} + +// acc from [topk, K] to [K] +template +inline void sum_stub(scalar_t* __restrict__ out, const scalar_t* __restrict__ input, int64_t topk, int64_t K) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + if (topk == 1) { + // do copy for topk = 1 + copy_stub(out, input, K); + } else { + // do sum for topk != 1 + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= K - kVecSize; d += kVecSize) { + fVec sum_fvec0 = fVec(0.f); + fVec sum_fvec1 = fVec(0.f); + for (int t = 0; t < topk; ++t) { + bVec x_bvec = bVec::loadu(input + t * K + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec0 += x_fvec0; + sum_fvec1 += x_fvec1; + } + bVec out_bvec = convert_from_float_ext(sum_fvec0, sum_fvec1); + out_bvec.store(out + d); + } + for (; d < K; ++d) { + float sum_val = 0.f; + for (int t = 0; t < topk; ++t) { + sum_val += static_cast(input[t * K + d]); + } + out[d] = static_cast(sum_val); + } + } +} + +// out = input + input2 * scale +template +inline void add_mul_stub( + scalar_t* __restrict__ out, + const float* __restrict__ input, + const scalar_t* __restrict__ input2, + float scale, + int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int kVecSize = bVec::size(); + const fVec s_vec = fVec(scale); + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= size - kVecSize; d += kVecSize) { + fVec x0 = fVec::loadu(input + d); + fVec x1 = fVec::loadu(input + d + fVec::size()); + + bVec y_bvec = bVec::loadu(input2 + d); + fVec y0, y1; + std::tie(y0, y1) = at::vec::convert_to_float(y_bvec); + + x0 = x0 + y0 * s_vec; + x1 = x1 + y1 * s_vec; + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(out + d); + } + for (; d < size; ++d) { + out[d] = static_cast(input[d] + float(input2[d]) * scale); + } +} + +template +inline void silu_and_mul( + scalar_t* __restrict__ C, + const int32_t* __restrict__ C0, // x: x0, x1 + const int32_t* __restrict__ C1, // y: y0, y1 + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t m_size, + int64_t N) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc0[COLS]; + __m512 vc1[COLS]; + __m512i vcomp0[COLS]; + __m512i vcomp1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; + + auto load_scale_and_comp = [&](auto col) { + vcomp0[col] = _mm512_loadu_si512(Bcomp0 + col * 16); + vcomp1[col] = _mm512_loadu_si512(Bcomp1 + col * 16); + vbs0[col] = _mm512_loadu_ps(Bs0 + col * 16); + vbs1[col] = _mm512_loadu_ps(Bs1 + col * 16); + }; + Unroll{}(load_scale_and_comp); + + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32_0 = _mm512_loadu_si512(C0 + m * BLOCK_N + col * 16); + __m512i vc32_1 = _mm512_loadu_si512(C1 + m * BLOCK_N + col * 16); + vc0[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_0, vcomp0[col])); + vc1[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32_1, vcomp1[col])); + vc0[col] = _mm512_mul_ps(_mm512_mul_ps(vc0[col], vas), vbs0[col]); + vc1[col] = _mm512_mul_ps(_mm512_mul_ps(vc1[col], vas), vbs1[col]); + }; + + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + const fVec one = fVec(1.f); + auto silu_and_mul = [&](auto col) { + fVec x = fVec(vc0[col]); + fVec y = fVec(vc1[col]); + x = x / (one + x.neg().exp_u20()); + vc0[col] = x * y; + }; + + auto storec = [&](auto col, int64_t m) { + if constexpr (col % 2 == 0) { + fVec x0 = fVec(vc0[col + 0]); + fVec x1 = fVec(vc0[col + 1]); + bVec out_vec = convert_from_float_ext(x0, x1); + out_vec.store(C + m * N + col * 16); + } + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); + Unroll{}(silu_and_mul); + Unroll{}(storec, m); + } +#else + TORCH_CHECK(false, "silu_and_mul: scalar path not implemented!"); +#endif +} + +template +inline void scale_C( + float* __restrict__ C, + const int32_t* __restrict__ Ctmp, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t m_size) { +#if defined(CPU_CAPABILITY_AVX512) + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512 vc[COLS]; + __m512i vcomp[COLS]; + __m512 vas; + __m512 vbs[COLS]; + + auto load_scale_and_comp = [&](auto col) { + vcomp[col] = _mm512_loadu_si512(Bcomp + col * 16); + vbs[col] = _mm512_loadu_ps(Bs + col * 16); + }; + Unroll{}(load_scale_and_comp); + + auto scalec = [&](auto col, int64_t m) { + // update As + vas = _mm512_set1_ps(As[m]); + // C = As * (C - Bcomp) * Bs + __m512i vc32 = _mm512_loadu_si512(Ctmp + m * BLOCK_N + col * 16); + vc[col] = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc32, vcomp[col])); + vc[col] = _mm512_mul_ps(_mm512_mul_ps(vc[col], vas), vbs[col]); + _mm512_storeu_ps(C + m * BLOCK_N + col * 16, vc[col]); + }; + + for (int64_t m = 0; m < m_size; ++m) { + Unroll{}(scalec, m); + } +#else + TORCH_CHECK(false, "scale_C: scalar path not implemented!"); +#endif +} + +/// gemm for w13 +template +struct tinygemm_kernel_vnni { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_vnni { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + at::BFloat16* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + const int32_t* __restrict__ Bcomp0, + const int32_t* __restrict__ Bcomp1, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512i va; + __m512i vb0[COLS]; + __m512i vb1[COLS]; + __m512i vc0[ROWS * COLS]; + __m512i vc1[ROWS * COLS]; + __m512i vcomp0[COLS]; + __m512i vcomp1[COLS]; + __m512 vas; + __m512 vbs0[COLS]; + __m512 vbs1[COLS]; + + auto loadc = [&](auto i) { + vc0[i] = _mm512_set1_epi32(0); + vc1[i] = _mm512_set1_epi32(0); + }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b0_ptr = reinterpret_cast(B0); + const int32_t* b1_ptr = reinterpret_cast(B1); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb0[col] = _mm512_loadu_si512(b0_ptr + k * ldb4 + col * 16); + vb1[col] = _mm512_loadu_si512(b1_ptr + k * ldb4 + col * 16); + } + vc0[i] = _mm512_dpbusd_epi32(vc0[i], va, vb0[col]); + vc1[i] = _mm512_dpbusd_epi32(vc1[i], va, vb1[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto scalec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp + if constexpr (row == 0) { + vbs0[col] = _mm512_loadu_ps(Bs0 + col * 16); + vbs1[col] = _mm512_loadu_ps(Bs1 + col * 16); + vcomp0[col] = _mm512_loadu_si512(Bcomp0 + col * 16); + vcomp1[col] = _mm512_loadu_si512(Bcomp1 + col * 16); + } + __m512 c0 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc0[i], vcomp0[col])); + __m512 c1 = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc1[i], vcomp1[col])); + vc0[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c0, vas), vbs0[col])); + vc1[i] = _mm512_castps_si512(_mm512_mul_ps(_mm512_mul_ps(c1, vas), vbs1[col])); + }; + Unroll{}(scalec); + + using Vec = at::vec::Vectorized; + const Vec one = Vec(1.f); + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + // for COLS = 2, 4 use 512bit store + if constexpr (col % 2 == 0) { + Vec x0 = _mm512_castsi512_ps(vc0[row * COLS + col + 0]); + Vec x1 = _mm512_castsi512_ps(vc0[row * COLS + col + 1]); + Vec y0 = _mm512_castsi512_ps(vc1[row * COLS + col + 0]); + Vec y1 = _mm512_castsi512_ps(vc1[row * COLS + col + 1]); + // silu + x0 = x0 / (one + x0.neg().exp_u20()); + x1 = x1 / (one + x1.neg().exp_u20()); + // mul + x0 = x0 * y0; + x1 = x1 * y1; + + _mm512_storeu_si512( + reinterpret_cast<__m512i*>((C + row * ldc + col * 16)), + (__m512i)(_mm512_cvtne2ps_pbh(__m512(x1), __m512(x0)))); + } + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_VNNI(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni::apply( \ + A + mb_start * lda, \ + B0 + nb_start * 4, \ + B1 + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs0 + nb_start, \ + Bs1 + nb_start, \ + Bcomp0 + nb_start, \ + Bcomp1 + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + scalar_t* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // pattern: 1-(2+2)-(8+8) + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 32; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +/// gemm for w2 +template +struct tinygemm_kernel_vnni2 { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + TORCH_CHECK(false, "tinygemm_kernel_nn: scalar path not implemented!"); + } +}; + +#if defined(CPU_CAPABILITY_AVX512) +template +struct tinygemm_kernel_vnni2 { + static inline void apply( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + const int32_t* __restrict__ Bcomp, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + constexpr int ROWS = BLOCK_M; + constexpr int COLS = BLOCK_N / 16; + static_assert(COLS % 2 == 0); + + __m512i va; + __m512i vb[COLS]; + __m512i vc[ROWS * COLS]; + __m512i vcomp[COLS]; + __m512 vas; + __m512 vbs[COLS]; + + auto loadc = [&](auto i) { vc[i] = _mm512_set1_epi32(0); }; + Unroll{}(loadc); + + const int64_t K4 = K >> 2; + const int64_t lda4 = lda >> 2; + const int64_t ldb4 = ldb; // ldb * 4 >> 2; + const int32_t* a_ptr = reinterpret_cast(A); + const int32_t* b_ptr = reinterpret_cast(B); + + auto compute = [&](auto i, int64_t k) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + if constexpr (col == 0) { + va = _mm512_set1_epi32(a_ptr[row * lda4 + k]); + } + if constexpr (row == 0) { + vb[col] = _mm512_loadu_si512(b_ptr + k * ldb4 + col * 16); + } + vc[i] = _mm512_dpbusd_epi32(vc[i], va, vb[col]); + }; + for (int64_t k = 0; k < K4; ++k) { + Unroll{}(compute, k); + } + + auto storec = [&](auto i) { + constexpr int row = i / COLS; + constexpr int col = i % COLS; + + // load a scale + if constexpr (col == 0) { + vas = _mm512_set1_ps(As[row]); + } + // load b scale and vcomp per 2 vectors + // also load bias if any + if constexpr (row == 0) { + if constexpr (col % 2 == 0) { + vbs[col + 0] = _mm512_loadu_ps(Bs + col * 16); + vbs[col + 1] = _mm512_loadu_ps(Bs + col * 16 + 16); + vcomp[col + 0] = _mm512_loadu_si512(Bcomp + col * 16); + vcomp[col + 1] = _mm512_loadu_si512(Bcomp + col * 16 + 16); + } + } + __m512 x = _mm512_cvtepi32_ps(_mm512_sub_epi32(vc[i], vcomp[col])); + x = _mm512_mul_ps(_mm512_mul_ps(x, vas), vbs[col]); + _mm512_storeu_ps(reinterpret_cast<__m512*>(C + row * ldc + col * 16), x); + }; + Unroll{}(storec); + } +}; +#endif + +#define LAUNCH_TINYGEMM_KERNEL_VNNI2(MB_SIZE, NB_SIZE) \ + tinygemm_kernel_vnni2::apply( \ + A + mb_start * lda, \ + B + nb_start * 4, \ + C + mb_start * ldc + nb_start, \ + As + mb_start, \ + Bs + nb_start, \ + Bcomp + nb_start, \ + K, \ + lda, \ + ldb, \ + ldc); + +template +void tinygemm_kernel( + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B, + float* __restrict__ C, + const float* __restrict__ As, + const float* __restrict__ Bs, + int64_t M, + int64_t N, + int64_t K, + int64_t lda, + int64_t ldb, + int64_t ldc) { + // B compensation + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * K); + + // pattern: 1-4-16 + constexpr int64_t BLOCK_M = 4; + constexpr int64_t BLOCK_N = 64; + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + for (int64_t mb = 0; mb < MB; ++mb) { + int64_t mb_start = mb * BLOCK_M; + int64_t mb_size = std::min(BLOCK_M, M - mb_start); + for (int64_t nb = 0; nb < NB; ++nb) { + int64_t nb_start = nb * BLOCK_N; + int64_t nb_size = std::min(BLOCK_N, N - nb_start); + + switch (mb_size << 4 | nb_size >> 4) { + case 0x12: + LAUNCH_TINYGEMM_KERNEL_VNNI2(1, 32); + break; + case 0x22: + LAUNCH_TINYGEMM_KERNEL_VNNI2(2, 32); + break; + case 0x32: + LAUNCH_TINYGEMM_KERNEL_VNNI2(3, 32); + break; + case 0x42: + LAUNCH_TINYGEMM_KERNEL_VNNI2(4, 32); + break; + default: + TORCH_CHECK(false, "Unexpected block size, ", mb_size, "x", "nb_size"); + } + } + } +} + +} // anonymous namespace + +template +void fused_experts_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + scalar_t* __restrict__ ic2, + uint8_t* __restrict__ A_tmp, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const float* __restrict__ topk_weights, + const int32_t* __restrict__ sorted_ids, + const int32_t* __restrict__ expert_ids, + const int32_t* __restrict__ offsets, + int64_t M, + int64_t N, + int64_t K, + int64_t E, + int64_t topk, + int64_t num_tokens_post_pad) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 0: quantize input to uint8, [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); + } + }); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(num_tokens_post_pad, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + // strides for w1: [E, 2N, K] + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + // K and N are packed for int8 + const int64_t packed_K = get_row_size(K); + const int64_t packed_N = get_row_size(N); + + const int64_t stride_e = 2 * N * packed_K; + const int64_t stride_n = packed_K; + + int64_t avg_M = std::max(int64_t(1), M * topk / E); + const bool use_brgemm = can_use_brgemm(avg_M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + uint8_t* __restrict__ A = A_tmp + tid * BLOCK_M * K; + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + alignas(64) float As[BLOCK_M]; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + + // B shape [K, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const int8_t* __restrict__ B0 = packed_w1 + expert_id * stride_e + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + expert_id * stride_e + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + expert_id * 2 * N + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + expert_id * 2 * N + nb_lower * BLOCK_N; + + // 1.a load A + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + int64_t m_size = offsets[mb + 1] - offsets[mb]; + + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m] / topk; + copy_stub(A + m * K, Aq_tmp + index * K, K); + As[m] = As_tmp[index]; + } + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + const int64_t offset = offsets[mb]; + silu_and_mul( + ic1 + offset * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + const int64_t offset = offsets[mb]; + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + offset * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: quantize ic1 to uint8, [M * topk, N] + at::parallel_for(0, M * topk, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [E, K, N] as [E, OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_e2 = OC * packed_N; + const int64_t stride_oc = packed_N; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = offsets[mb + 1] - offsets[mb]; + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A ptr from ic1 of [M * topk, N] in sorted order + // so as to avoid copy A to tmp buffer again + const uint8_t* __restrict__ A = Aq_tmp + offsets[mb] * N; + const float* __restrict__ As = As_tmp + offsets[mb]; + const int32_t* A_ids = sorted_ids + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + int32_t expert_id = expert_ids[mb]; + const int8_t* __restrict__ B = packed_w2 + expert_id * stride_e2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = w2s + expert_id * K + nb * BLOCK_N; + + // 2.a gemm: C = A @ B + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to ic2 in original order + // and also mul topk_weights in float32 + for (int64_t m = 0; m < m_size; ++m) { + int32_t index = A_ids[m]; + float weight = topk_weights[index]; + copy_mul_stub(ic2 + index * K + nb * BLOCK_N, C + m * BLOCK_N, weight, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 3: out = intermediate_cache2.sum(dim=1) + // from [M, topk, K] to [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + sum_stub(output + m * K, ic2 + m * topk * K, topk, K); + } + }); +} + +#define INSTANTIATE_MOE_INT8_TEMPLATE(TYPE) \ + template void fused_experts_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + TYPE* __restrict__ ic2, \ + uint8_t* __restrict__ A_tmp, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const float* __restrict__ topk_weights, \ + const int32_t* __restrict__ sorted_ids, \ + const int32_t* __restrict__ expert_ids, \ + const int32_t* __restrict__ offsets, \ + int64_t M, \ + int64_t N, \ + int64_t K, \ + int64_t E, \ + int64_t topk, \ + int64_t num_tokens_post_pad) + +INSTANTIATE_MOE_INT8_TEMPLATE(at::BFloat16); +INSTANTIATE_MOE_INT8_TEMPLATE(at::Half); + +template +void shared_expert_int8_kernel_impl( + scalar_t* __restrict__ output, + scalar_t* __restrict__ ic1, + float* __restrict__ C_tmp, + uint8_t* __restrict__ Aq_tmp, + float* __restrict__ As_tmp, + const scalar_t* __restrict__ input, + const int8_t* __restrict__ packed_w1, + const int8_t* __restrict__ packed_w2, + const float* __restrict__ w1s, + const float* __restrict__ w2s, + const scalar_t* __restrict__ fused_experts_out, + float routed_scaling_factor, + int64_t M, + int64_t N, + int64_t K) { + // handle 2 tiles per block + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + + // stage 0: quantize input to uint8, [M, K] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * K, As_tmp[m], input + m * K, K); + } + }); + + // stage 1: intermediate_cache1 = silu(hidden_states @ w1) + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB = div_up(N, BLOCK_N); + + TORCH_CHECK(N % BLOCK_N == 0, "Fixme when N is not multiples of ", BLOCK_N); + + // K and N are packed for int8 + const int64_t packed_K = get_row_size(K); + const int64_t packed_N = get_row_size(N); + const int64_t stride_n = packed_K; + + const bool use_brgemm = can_use_brgemm(M); + + // here we only parallel on half of 2N to fuse silu_and_mul with gemm + parallel_2d(MB, NB, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + int32_t* __restrict__ C0 = reinterpret_cast(C_tmp) + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C1 = C0 + BLOCK_M * BLOCK_N; + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * K * 2, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + // nb_upper from top half and nb_lower from bottom half + int64_t nb_upper = nb, nb_lower = nb + NB; + int64_t n_size = std::min(N - nb * BLOCK_N, BLOCK_N); + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + + // A shape [m_size, K] + const uint8_t* A = Aq_tmp + mb * BLOCK_M * K; + const float* As = As_tmp + mb * BLOCK_M; + + // B shape [K, n_size] in vnni format + const int8_t* __restrict__ B0 = packed_w1 + nb_upper * BLOCK_N * stride_n; + const int8_t* __restrict__ B1 = packed_w1 + nb_lower * BLOCK_N * stride_n; + const float* __restrict__ Bs0 = w1s + nb_upper * BLOCK_N; + const float* __restrict__ Bs1 = w1s + nb_lower * BLOCK_N; + + if (use_brgemm) { + // 1.b gemm: C0 = A @ B0 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B0, + /* C */ C0); + + // 1.c gemm: C1 = A @ B1 + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B1, + /* C */ C1); + + const int32_t* Bcomp0 = reinterpret_cast(B0 + block_size_n() * K); + const int32_t* Bcomp1 = reinterpret_cast(B1 + block_size_n() * K); + + // 1.d silu and mul + silu_and_mul( + ic1 + mb * BLOCK_M * N + nb * BLOCK_N, C0, C1, As, Bs0, Bs1, Bcomp0, Bcomp1, m_size, N); + } else { + // fused 1.bcd: silu_and_mul(A @ B0, A @ B1) + tinygemm_kernel( + /* A */ A, + /* B0 */ B0, + /* B1 */ B1, + /* C */ ic1 + mb * BLOCK_M * N + nb * BLOCK_N, + /* As */ As, + /* Bs0 */ Bs0, + /* Bs1 */ Bs1, + /* M */ m_size, + /* N */ n_size, + /* K */ K, + /* lda */ K, + /* ldb */ n_size, + /* ldc */ N); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); + + // stage 1.5: quantize ic1 to uint8, [M * topk, N] + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_tmp + m * N, As_tmp[m], ic1 + m * N, N); + } + }); + + // stage 2: intermediate_cache2 = intermediate_cache1 @ w2 + // w2 : [K, N] as [OC, IC] + const int64_t OC = K; // rename K as OC + const int64_t IC = N; // rename N as IC + const int64_t MB2 = MB; + const int64_t NB2 = div_up(OC, BLOCK_N); + const int64_t stride_oc = packed_N; + + // parallel on [MB2, NB2] + parallel_2d(MB2, NB2, [&](int64_t mb0, int64_t mb1, int64_t nb0, int64_t nb1) { + // get local pointers + int tid = get_thread_num(); + float* __restrict__ C = C_tmp + tid * 2 * BLOCK_M * BLOCK_N; + int32_t* __restrict__ C32 = reinterpret_cast(C + BLOCK_M * BLOCK_N); + + loop_2d(mb0, mb1, nb0, nb1, BLOCK_N * IC, [&](int64_t mb, int64_t nb, int64_t nb_offset) { + int64_t m_size = std::min(M - mb * BLOCK_M, BLOCK_M); + int64_t n_size = std::min(OC - nb * BLOCK_N, BLOCK_N); + + // A shape [m_size, IC] + const uint8_t* __restrict__ A = Aq_tmp + mb * BLOCK_M * N; + const float* __restrict__ As = As_tmp + mb * BLOCK_M; + + // B shape [IC, n_size] in vnni format + const int8_t* __restrict__ B = packed_w2 + nb * BLOCK_N * stride_oc; + const float* __restrict__ Bs = w2s + nb * BLOCK_N; + + if (use_brgemm) { + at::native::cpublas::brgemm( + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N, + /* add_C */ false, + /* A */ A, + /* B */ B, + /* C */ C32); + + // apply scales + const int32_t* Bcomp = reinterpret_cast(B + block_size_n() * IC); + scale_C(C, C32, As, Bs, Bcomp, m_size); + } else { + // 2.a gemm: C = A @ B + tinygemm_kernel( + /* A */ A, + /* B */ B, + /* C */ C, + /* As */ As, + /* Bs */ Bs, + /* M */ m_size, + /* N */ n_size, + /* K */ IC, + /* lda */ IC, + /* ldb */ n_size, + /* ldc */ BLOCK_N); + } + + // 2.b copy from C to output and add fused_experts_out + scalar_t* __restrict__ out = output + mb * BLOCK_M * K + nb * BLOCK_N; + const scalar_t* __restrict__ fused_out = fused_experts_out + mb * BLOCK_M * K + nb * BLOCK_N; + for (int64_t m = 0; m < m_size; ++m) { + add_mul_stub(out + m * K, C + m * BLOCK_N, fused_out + m * K, routed_scaling_factor, n_size); + } + }); + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +#define INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(TYPE) \ + template void shared_expert_int8_kernel_impl( \ + TYPE* __restrict__ output, \ + TYPE* __restrict__ ic1, \ + float* __restrict__ C_tmp, \ + uint8_t* __restrict__ Aq_tmp, \ + float* __restrict__ As_tmp, \ + const TYPE* __restrict__ input, \ + const int8_t* __restrict__ packed_w1, \ + const int8_t* __restrict__ packed_w2, \ + const float* __restrict__ w1s, \ + const float* __restrict__ w2s, \ + const TYPE* __restrict__ fused_experts_out, \ + float routed_scaling_factor, \ + int64_t M, \ + int64_t N, \ + int64_t K) + +INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::BFloat16); +INSTANTIATE_SHARED_EXPERT_INT8_TEMPLATE(at::Half); diff --git a/sgl-kernel/csrc/cpu/norm.cpp b/sgl-kernel/csrc/cpu/norm.cpp new file mode 100644 index 000000000..2c4e1f38d --- /dev/null +++ b/sgl-kernel/csrc/cpu/norm.cpp @@ -0,0 +1,304 @@ +#include "common.h" +#include "vec.h" + +namespace { + +// NB: avoid using `at::vec::map<>` on bfloat16 or half +// Llama4TextL2Norm +template +void l2norm_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + int64_t batch_size, + int64_t hidden_size, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ out_ptr = output + i * hidden_size; + const scalar_t* __restrict__ input_ptr = input + i * hidden_size; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + sum_val += x_val * x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + x_fvec0 = x_fvec0 * scale_fvec; + x_fvec1 = x_fvec1 * scale_fvec; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(out_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + out_ptr[d] = static_cast(x_val * rsqrt_var); + } + } + }); +} +template +void rmsnorm_kernel_impl( + scalar_t* __restrict__ output, + const scalar_t* __restrict__ input, + const scalar_t* __restrict__ weight, + int64_t batch_size, + int64_t hidden_size, + int64_t input_strideN, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ out_ptr = output + i * hidden_size; + const scalar_t* __restrict__ input_ptr = input + i * input_strideN; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + sum_val += x_val * x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec w_bvec = bVec::loadu(weight + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(out_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + float w_val = static_cast(weight[d]); + out_ptr[d] = static_cast(x_val * rsqrt_var * w_val); + } + } + }); +} + +template +void fused_add_rmsnorm_kernel_impl( + scalar_t* __restrict__ input, + scalar_t* __restrict__ residual, + const scalar_t* __restrict__ weight, + float* __restrict__ buffer, + int64_t batch_size, + int64_t hidden_size, + int64_t input_strideN, + float eps = 1e-5) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + at::parallel_for(0, batch_size, 0, [&](int64_t begin, int64_t end) { + int tid = at::get_thread_num(); + float* __restrict__ buffer_ptr = buffer + tid * hidden_size; + + for (int64_t i = begin; i < end; ++i) { + // local ptrs + scalar_t* __restrict__ input_ptr = input + i * input_strideN; + scalar_t* __restrict__ residual_ptr = residual + i * hidden_size; + + fVec sum_fvec = fVec(float(0)); + float sum_val = float(0); + + int64_t d; +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + bVec x_bvec = bVec::loadu(input_ptr + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + bVec r_bvec = bVec::loadu(residual_ptr + d); + fVec r_fvec0, r_fvec1; + std::tie(r_fvec0, r_fvec1) = at::vec::convert_to_float(r_bvec); + + x_fvec0 += r_fvec0; + x_fvec1 += r_fvec1; + + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(residual_ptr + d); + + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + + x_fvec0.store(buffer_ptr + d); + x_fvec1.store(buffer_ptr + d + fVec::size()); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = static_cast(input_ptr[d]); + float r_val = static_cast(residual_ptr[d]); + + x_val += r_val; + residual_ptr[d] = static_cast(x_val); + + sum_val += x_val * x_val; + buffer_ptr[d] = x_val; + } + + sum_val += vec_reduce_sum(sum_fvec); + float rsqrt_var = float(1) / std::sqrt(sum_val / hidden_size + eps); + const fVec scale_fvec = fVec(rsqrt_var); + +#pragma GCC unroll 4 + for (d = 0; d <= hidden_size - kVecSize; d += kVecSize) { + fVec x_fvec0 = fVec::loadu(buffer_ptr + d); + fVec x_fvec1 = fVec::loadu(buffer_ptr + d + fVec::size()); + + bVec w_bvec = bVec::loadu(weight + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + bVec x_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + x_bvec.store(input_ptr + d); + } +#pragma GCC unroll 4 + for (; d < hidden_size; ++d) { + float x_val = buffer_ptr[d] * rsqrt_var * static_cast(weight[d]); + input_ptr[d] = x_val; + } + } + }); +} + +} // anonymous namespace + +// input : {batch_size, hidden_size} +at::Tensor l2norm_cpu(at::Tensor& input, double eps) { + RECORD_FUNCTION("sgl-kernel::l2norm_cpu", std::vector({input})); + + CHECK_INPUT(input); + CHECK_DIM(2, input); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + at::Tensor output = at::empty_like(input); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "l2norm_kernel", [&] { + l2norm_kernel_impl(output.data_ptr(), input.data_ptr(), batch_size, hidden_size, eps); + }); + return output; +} + +// input : {batch_size, hidden_size} +// weight: {hidden_size} +at::Tensor rmsnorm_cpu(at::Tensor& input, at::Tensor& weight, double eps) { + RECORD_FUNCTION("sgl-kernel::rmsnorm_cpu", std::vector({input, weight})); + + CHECK_LAST_DIM_CONTIGUOUS_INPUT(input); + CHECK_INPUT(weight); + CHECK_DIM(2, input); + CHECK_DIM(1, weight); + CHECK_EQ(input.size(1), weight.size(0)); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + at::Tensor output = at::empty_like(input); + int64_t input_strideN = input.stride(0); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "rmsnorm_kernel", [&] { + rmsnorm_kernel_impl( + output.data_ptr(), + input.data_ptr(), + weight.data_ptr(), + batch_size, + hidden_size, + input_strideN, + eps); + }); + return output; +} + +// input : {batch_size, hidden_size} +// residual: {batch_size, hidden_size} +// weight : {hidden_size} +void fused_add_rmsnorm_cpu(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps) { + RECORD_FUNCTION("sgl-kernel::fused_add_rmsnorm_cpu", std::vector({input, residual, weight})); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(input); + CHECK_INPUT(residual); + CHECK_INPUT(weight); + CHECK_DIM(2, input); + CHECK_DIM(2, residual); + CHECK_DIM(1, weight); + CHECK_EQ(input.size(0), residual.size(0)); + CHECK_EQ(input.size(1), residual.size(1)); + CHECK_EQ(input.size(1), weight.size(0)); + int64_t batch_size = input.size(0); + int64_t hidden_size = input.size(1); + int64_t input_strideN = input.stride(0); + + // allocate temp buffer to store x in float32 per thread + // TODO: implement a singleton for context + int64_t num_threads = at::get_num_threads(); + at::Tensor buffer = at::empty({num_threads, hidden_size}, input.options().dtype(at::kFloat)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input.scalar_type(), "fused_add_rmsnorm_kernel", [&] { + fused_add_rmsnorm_kernel_impl( + input.data_ptr(), + residual.data_ptr(), + weight.data_ptr(), + buffer.data_ptr(), + batch_size, + hidden_size, + input_strideN, + eps); + }); +} diff --git a/sgl-kernel/csrc/cpu/numa_utils.cpp b/sgl-kernel/csrc/cpu/numa_utils.cpp new file mode 100644 index 000000000..2699d0e23 --- /dev/null +++ b/sgl-kernel/csrc/cpu/numa_utils.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include + +#include + +#include "common.h" + +std::string init_cpu_threads_env(const std::string& cpu_ids) { + bitmask* omp_cpu_mask = numa_parse_cpustring(cpu_ids.c_str()); + TORCH_CHECK(omp_cpu_mask->size > 0); + std::vector omp_cpu_ids; + omp_cpu_ids.reserve(omp_cpu_mask->size); + + constexpr int group_size = 8 * sizeof(*omp_cpu_mask->maskp); + + for (int offset = 0; offset < omp_cpu_mask->size; offset += group_size) { + unsigned long group_mask = omp_cpu_mask->maskp[offset / group_size]; + int i = 0; + while (group_mask) { + if (group_mask & 1) { + omp_cpu_ids.emplace_back(offset + i); + } + ++i; + group_mask >>= 1; + } + } + + // Memory node binding + if (numa_available() != -1) { + int mem_node_id = numa_node_of_cpu(omp_cpu_ids.front()); + bitmask* mask = numa_parse_nodestring(std::to_string(mem_node_id).c_str()); + bitmask* src_mask = numa_get_membind(); + + int pid = getpid(); + + // move all existing pages to the specified numa node. + *(src_mask->maskp) = *(src_mask->maskp) ^ *(mask->maskp); + int page_num = numa_migrate_pages(pid, src_mask, mask); + if (page_num == -1) { + TORCH_WARN(false, "numa_migrate_pages failed. errno: " + std::to_string(errno)); + } + + // restrict memory allocation node. + numa_set_membind(mask); + numa_set_strict(1); + } + + // OMP threads binding + omp_set_num_threads((int)omp_cpu_ids.size()); + at::set_num_threads((int)omp_cpu_ids.size()); + TORCH_CHECK_EQ(omp_cpu_ids.size(), at::get_num_threads()); + TORCH_CHECK_EQ(omp_cpu_ids.size(), omp_get_max_threads()); + + std::vector> thread_core_mapping; + thread_core_mapping.reserve(omp_cpu_ids.size()); + omp_lock_t writelock; + omp_init_lock(&writelock); + +#pragma omp parallel for schedule(static, 1) + for (size_t i = 0; i < omp_cpu_ids.size(); ++i) { + cpu_set_t mask; + CPU_ZERO(&mask); + CPU_SET(omp_cpu_ids[i], &mask); + int ret = sched_setaffinity(0, sizeof(cpu_set_t), &mask); + if (ret == -1) { + TORCH_CHECK(false, "sched_setaffinity failed. errno: " + std::to_string(errno)); + } + + omp_set_lock(&writelock); + thread_core_mapping.emplace_back(syscall(SYS_gettid), omp_cpu_ids[i]); + omp_unset_lock(&writelock); + } + + omp_destroy_lock(&writelock); + + numa_free_nodemask(omp_cpu_mask); + + std::stringstream ss; + ss << "OMP threads binding of Process " << getpid() << ":\n"; + std::sort( + thread_core_mapping.begin(), thread_core_mapping.end(), [](auto&& a, auto&& b) { return a.second < b.second; }); + for (auto&& item : thread_core_mapping) { + ss << "\t" + << "OMP tid: " << item.first << ", core " << item.second << "\n"; + } + + return ss.str(); +} diff --git a/sgl-kernel/csrc/cpu/qkv_proj.cpp b/sgl-kernel/csrc/cpu/qkv_proj.cpp new file mode 100644 index 000000000..b3e2072e8 --- /dev/null +++ b/sgl-kernel/csrc/cpu/qkv_proj.cpp @@ -0,0 +1,701 @@ +#include "common.h" +#include "gemm.h" +#include "vec.h" + +namespace { + +// [NOTE]: Fused kernel for QKV projection with weight absorption and RoPE +// +// 1. `q_a_proj` and `kv_a_proj_with_mqa` fused into one gemm, +// otherwise we need to split IC for the 2nd gemm. +// 2. `q_a_layernorm` and `kv_a_layernorm` fused into one parallel loop. +// 3. k_input and v_input share the same storage, the torch API did +// this in `set_kv_buffer`. No additional memory movement. +// + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const scalar_t* __restrict__ A, + const scalar_t* __restrict__ B0, + const scalar_t* __restrict__ B1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K) { + // convert_weight_packed make sure N0 and N1 are 32x + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const scalar_t* __restrict__ B = nb < NB0 ? B0 : B1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * K /* nb * BLOCK_N * K */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Ctmp*/ Ctmp, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const uint8_t* __restrict__ A, + const int8_t* __restrict__ B0, + const int8_t* __restrict__ B1, + const float* __restrict__ As, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const bool use_brgemm = can_use_brgemm(M); + + // K + 4 after compensation + const int64_t packed_row_size = get_row_size(K); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + // for brgemm, use float32 for accumulate + alignas(64) int32_t Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const int8_t* __restrict__ B = nb < NB0 ? B0 : B1; + const float* __restrict__ Bs = nb < NB0 ? Bs0 : Bs1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * packed_row_size /* nb * BLOCK_N * (K + 4) */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Ctmp*/ Ctmp, + /* As */ As + mb_start, + /* Bs */ Bs + local_nb_start, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +// [C0, C1] = A @ [B0, B1] +template +void segment_gemm_kernel_impl( + scalar_t* __restrict__ C0, + scalar_t* __restrict__ C1, + const scalar_t* __restrict__ A, + const at::Float8_e4m3fn* __restrict__ B0, + const at::Float8_e4m3fn* __restrict__ B1, + const float* __restrict__ Bs0, + const float* __restrict__ Bs1, + scalar_t* __restrict__ Btmp, + int64_t M, + int64_t N0, + int64_t N1, + int64_t K, + int64_t block_size_N, + int64_t block_size_K) { + constexpr int64_t BLOCK_M = block_size_m(); + constexpr int64_t BLOCK_N = block_size_n(); + const int64_t MB = div_up(M, BLOCK_M); + const int64_t NB0 = div_up(N0, BLOCK_N); + const int64_t NB1 = div_up(N1, BLOCK_N); + const int64_t NB = NB0 + NB1; + + const int64_t scale_size_K = div_up(K, block_size_K); + const int64_t blocks_n_per_group = block_size_N / BLOCK_N; + + const bool use_brgemm = can_use_brgemm(M); + + // parallel on [MB, NB0 + NB1] + at::parallel_for(0, MB * NB, 0, [&](int64_t begin, int64_t end) { + int64_t mb{0}, nb{0}; + data_index_init(begin, mb, MB, nb, NB); + + int tid = at::get_thread_num(); + // for brgemm, use float32 for accumulate + alignas(64) float Ctmp[BLOCK_M * BLOCK_N]; + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + + int mb_start = mb * BLOCK_M; + int mb_size = std::min(M - mb_start, BLOCK_M); + int nb_start = nb * BLOCK_N; + int nb_size = BLOCK_N; + + const at::Float8_e4m3fn* __restrict__ B = nb < NB0 ? B0 : B1; + const float* __restrict__ Bs = nb < NB0 ? Bs0 : Bs1; + scalar_t* __restrict__ C = nb < NB0 ? C0 : C1; + int64_t ldc = nb < NB0 ? N0 : N1; + int64_t local_nb_start = nb < NB0 ? nb_start : nb_start - N0; + int64_t new_nb = nb < NB0 ? nb : nb - NB0; + + tinygemm_kernel( + /* A */ A + mb_start * K, + /* B */ B + local_nb_start * K /* nb * BLOCK_N * K */, + /* C */ C + mb_start * ldc + local_nb_start, + /* Btmp*/ Btmp + tid * BLOCK_N * K, + /* Ctmp*/ Ctmp, + /* Bs */ Bs + (new_nb / blocks_n_per_group) * scale_size_K, + /* M */ mb_size, + /* N */ nb_size, + /* K */ K, + /* lda */ K, + /* ldb */ nb_size, + /* ldc */ ldc, + /* brg */ use_brgemm, + /* block_size_K */ block_size_K); + + // move to the next index + data_index_step(mb, MB, nb, NB); + } + + if (use_brgemm) { + at::native::cpublas::brgemm_release(); + } + }); +} + +template +inline float reduce(const scalar_t* __restrict__ x, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + fVec sum_fvec = fVec(float(0)); + +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x_bvec = bVec::loadu(x + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + sum_fvec += x_fvec0 * x_fvec0; + sum_fvec += x_fvec1 * x_fvec1; + } + return vec_reduce_sum(sum_fvec); +} + +// map2 from aten functional doesn't have fast bf16->fp32 conversion +template +inline void map2(scalar_t* y, const scalar_t* x, const scalar_t* __restrict__ w, float scale, int64_t size) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + fVec scale_fvec = fVec(scale); + +// no remainder +#pragma GCC unroll 4 + for (int64_t d = 0; d < size; d += bVec::size()) { + bVec x_bvec = bVec::loadu(x + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + bVec w_bvec = bVec::loadu(w + d); + fVec w_fvec0, w_fvec1; + std::tie(w_fvec0, w_fvec1) = at::vec::convert_to_float(w_bvec); + x_fvec0 = x_fvec0 * scale_fvec * w_fvec0; + x_fvec1 = x_fvec1 * scale_fvec * w_fvec1; + bVec out_bvec = convert_from_float_ext(x_fvec0, x_fvec1); + out_bvec.store(y + d); + } +} + +template +void rms_norm_kernel_impl( + scalar_t* __restrict__ input0, + scalar_t* __restrict__ input1, + const scalar_t* __restrict__ weight0, + const scalar_t* __restrict__ weight1, + int64_t M, + int64_t N0, + int64_t N1, + int64_t stride1, + float eps = 1e-5) { + at::parallel_for(0, M, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + scalar_t* x0 = input0 + m * N0; + scalar_t* x1 = input1 + m * stride1; + float scale0 = reduce(x0, N0); + float scale1 = reduce(x1, N1); + scale0 = float(1) / std::sqrt(scale0 / N0 + eps); + scale1 = float(1) / std::sqrt(scale1 / N1 + eps); + map2(x0, x0, weight0, scale0, N0); + map2(x1, x1, weight1, scale1, N1); + } + }); +} + +template +inline void rotary(const scalar_t* input, scalar_t* out, const scalar_t* cos, const scalar_t* sin, int64_t size) { + TORCH_CHECK(false, "rotary scalar path not implemented."); +} + +#if defined(CPU_CAPABILITY_AVX512) +template <> +inline void rotary( + const at::BFloat16* input, at::BFloat16* out, const at::BFloat16* cos, const at::BFloat16* sin, int64_t size) { + // permute indices + const __m512i idx1 = _mm512_set_epi32(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0); + const __m512i idx2 = _mm512_set_epi32(31, 29, 27, 25, 23, 21, 19, 17, 15, 13, 11, 9, 7, 5, 3, 1); + const __m512i idy1 = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0); + const __m512i idy2 = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8); + +// rotary dim is 64, just 2 iters +#pragma GCC unroll 2 + for (int64_t d = 0; d < size; d += 32) { + int64_t d2 = d >> 1; + // load coefs + __m512 vcos = CVT_BF16_TO_FP32(_mm256_loadu_si256(reinterpret_cast(cos + d2))); + __m512 vsin = CVT_BF16_TO_FP32(_mm256_loadu_si256(reinterpret_cast(sin + d2))); + // load input + __m512i a16 = _mm512_loadu_si512(reinterpret_cast(input + d)); + __m512 a = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(a16, 0)); + __m512 b = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(a16, 1)); + // from [16, 2] to [2, 16] + __m512 in1 = _mm512_mask_permutex2var_ps(a, 0xffff, idx1, b); + __m512 in2 = _mm512_mask_permutex2var_ps(a, 0xffff, idx2, b); + // out1 = in1 * cos - in2 * sin; + // out2 = in2 * cos + in1 * sin + __m512 out1 = _mm512_sub_ps(_mm512_mul_ps(in1, vcos), _mm512_mul_ps(in2, vsin)); + __m512 out2 = _mm512_add_ps(_mm512_mul_ps(in2, vcos), _mm512_mul_ps(in1, vsin)); + // from [2, 16] to [16, 2] + a = _mm512_mask_permutex2var_ps(out1, 0xffff, idy1, out2); + b = _mm512_mask_permutex2var_ps(out1, 0xffff, idy2, out2); + + _mm512_storeu_si512(reinterpret_cast<__m512i*>((out + d)), (__m512i)(_mm512_cvtne2ps_pbh(b, a))); + } +} +#endif + +template +void rotary_emb_kernel_impl( + scalar_t* q_pe_out, + scalar_t* k_pe_out, + const scalar_t* q_pe, + const scalar_t* k_pe, + const int64_t* pos, + const scalar_t* cos_sin, + int64_t num_seqs, + int64_t num_heads, + int64_t rotary_dim, + int64_t q_strideB, + int64_t q_strideH, + int64_t k_strideB, + int64_t oq_strideB, + int64_t oq_strideH, + int64_t ok_strideB) { + TORCH_CHECK(rotary_dim % 32 == 0, "rotary_dim is not 32x."); + const int64_t rotary_offset = rotary_dim / 2; + + // parallel on [num_seqs, num_heads + 1] + // top [num_heads] handle q_pe and bottom [1] handle k_pe + at::parallel_for(0, num_seqs * (num_heads + 1), GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t seq{0}, head_id{0}; + data_index_init(begin, seq, num_seqs, head_id, num_heads + 1); + + for (int64_t i = begin; i < end; ++i) { + UNUSED(i); + // get cos and sin cache ptr + int64_t index = pos[seq]; + const scalar_t* cos = cos_sin + index * rotary_dim; + const scalar_t* sin = cos + rotary_offset; + + const scalar_t* input = + (head_id < num_heads) ? q_pe + seq * q_strideB + head_id * q_strideH : k_pe + seq * k_strideB; + scalar_t* out = + (head_id < num_heads) ? q_pe_out + seq * oq_strideB + head_id * oq_strideH : k_pe_out + seq * ok_strideB; + rotary(input, out, cos, sin, rotary_dim); + + // move to the next index + data_index_step(seq, num_seqs, head_id, num_heads + 1); + } + }); +} + +} // anonymous namespace + +extern at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni); + +extern at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +extern void +bmm_cpu(at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale); + +extern at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// NB: shapes in DeepDeek R1 +// +// hidden_states : [num_seqs, hidden_size] [1, 7168] +// q_a_proj_weight : [q_lora_rank, hidden_size] [1536, 7168] +// q_b_proj_weight : [num_heads * qk_head_dim, q_lora_rank] [4224, 1536] +// kv_a_proj_weight : [kv_lora_rank + qk_rope_head_dim, hidden_size] [576, 7168] +// w_kc : [num_heads, kv_lora_rank, qk_nope_head_dim] [22, 512, 128] +// q_a_layernorm_weight : [q_lora_rank] [1536] +// kv_a_layernorm_weight : [kv_lora_rank] [512] +// +std::tuple qkv_proj_with_rope( + at::Tensor& hidden_states, + at::Tensor& q_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& kv_a_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional q_a_proj_scale, + std::optional q_b_proj_scale, + std::optional kv_a_proj_scale, + bool is_vnni, + std::optional> block_size) { + RECORD_FUNCTION( + "sgl-kernel::qkv_proj_with_rope", + std::vector({hidden_states, q_a_proj_weight, q_b_proj_weight, kv_a_proj_weight, w_kc})); + + const auto st = hidden_states.scalar_type(); + CHECK_INPUT(hidden_states); + CHECK_INPUT(positions); + CHECK_INPUT(cos_sin_cache); + CHECK_EQ(q_a_layernorm_weight.scalar_type(), st); + CHECK_EQ(kv_a_layernorm_weight.scalar_type(), st); + CHECK_EQ(positions.scalar_type(), at::kLong); + CHECK_EQ(cos_sin_cache.scalar_type(), st); + CHECK_DIM(2, hidden_states); + CHECK_DIM(3, w_kc); + CHECK_DIM(1, q_a_layernorm_weight); + CHECK_DIM(1, kv_a_layernorm_weight); + CHECK_DIM(1, positions); + CHECK_DIM(2, cos_sin_cache); + + // skip contiguous checks for weights, expect prepacked + TORCH_CHECK(is_vnni, "qkv_proj_with_rope: expect weights are prepacked!"); + + int64_t num_seqs = hidden_states.size(0); + int64_t hidden_size = hidden_states.size(1); + int64_t q_lora_rank = q_a_proj_weight.size(0); + int64_t num_heads = w_kc.size(0); + int64_t kv_lora_rank = w_kc.size(1); + int64_t qk_head_dim = q_b_proj_weight.size(0) / num_heads; + int64_t qk_nope_head_dim = w_kc.size(2); + int64_t qk_rope_head_dim = kv_a_proj_weight.size(0) - kv_lora_rank; + int64_t rotary_dim = cos_sin_cache.size(1); + + CHECK_EQ(positions.numel(), num_seqs); + CHECK_EQ(rotary_dim, qk_rope_head_dim); + CHECK_EQ(q_a_layernorm_weight.numel(), q_lora_rank); + CHECK_EQ(kv_a_layernorm_weight.numel(), kv_lora_rank); + + // check the packed dimension + CHECK_EQ(q_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + CHECK_EQ(q_b_proj_weight.size(1), get_row_size(q_lora_rank, use_int8_w8a8)); + CHECK_EQ(kv_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + + if (use_int8_w8a8) { + TORCH_CHECK(q_a_proj_scale.has_value(), "missing q_a_proj_scale for int8 w8a8."); + TORCH_CHECK(q_b_proj_scale.has_value(), "missing q_b_proj_scale for int8 w8a8."); + TORCH_CHECK(kv_a_proj_scale.has_value(), "missing kv_a_proj_scale for int8 w8a8."); + } + if (use_fp8_w8a16) { + TORCH_CHECK(q_a_proj_scale.has_value(), "missing q_a_proj_scale for fp8 w8a16."); + TORCH_CHECK(q_b_proj_scale.has_value(), "missing q_b_proj_scale for fp8 w8a16."); + TORCH_CHECK(kv_a_proj_scale.has_value(), "missing kv_a_proj_scale for fp8 w8a16."); + TORCH_CHECK(block_size.has_value(), "missing block_size for fp8 w8a16."); + TORCH_CHECK(block_size.value().size() == 2, "block_size should be 2D for fp8 w8a16."); + } + // outputs and temp buffer + const auto options = hidden_states.options(); + auto q_input = at::empty({num_seqs, num_heads, kv_lora_rank + qk_rope_head_dim}, options); + auto k_input = at::empty({num_seqs, 1, kv_lora_rank + qk_rope_head_dim}, options); + auto v_input = k_input.narrow(-1, 0, kv_lora_rank); + + // outputs of q_a_proj and q_b_proj + auto qa = at::empty({num_seqs, q_lora_rank}, options); + + // stage 1: q_a_proj and kv_a_proj + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "qkv_proj_kernel_impl", [&] { + if (use_int8_w8a8) { + auto q_a_proj_s = q_a_proj_scale.value(); + auto kv_a_proj_s = kv_a_proj_scale.value(); + TORCH_CHECK(q_a_proj_s.numel() == q_lora_rank); + TORCH_CHECK(kv_a_proj_s.numel() == kv_lora_rank + qk_rope_head_dim); + + auto buffer = at::empty({num_seqs * hidden_size + num_seqs * 4}, options.dtype(at::kByte)); + uint8_t* __restrict__ Aq_data = buffer.data_ptr(); + float* __restrict__ As_data = (float*)((void*)(Aq_data + num_seqs * hidden_size)); + const scalar_t* __restrict__ A_data = hidden_states.data_ptr(); + + at::parallel_for(0, num_seqs, 0, [&](int64_t begin, int64_t end) { + for (int64_t m = begin; m < end; ++m) { + quantize_row_int8(Aq_data + m * hidden_size, As_data[m], A_data + m * hidden_size, hidden_size); + } + }); + + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + Aq_data, + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + As_data, + q_a_proj_s.data_ptr(), + kv_a_proj_s.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size); + } else if (use_fp8_w8a16) { + int64_t block_size_N = block_size.value()[0]; + int64_t block_size_K = block_size.value()[1]; + auto q_a_proj_s = q_a_proj_scale.value(); + auto kv_a_proj_s = kv_a_proj_scale.value(); + CHECK_EQ(q_a_proj_s.size(0), div_up(q_lora_rank, block_size_N)); + CHECK_EQ(q_a_proj_s.size(1), div_up(hidden_size, block_size_K)); + CHECK_EQ(kv_a_proj_s.size(0), div_up(kv_lora_rank + qk_rope_head_dim, block_size_N)); + CHECK_EQ(kv_a_proj_s.size(1), div_up(hidden_size, block_size_K)); + + const int BLOCK_N = block_size_n(); + const int num_threads = at::get_num_threads(); + auto buffer = at::empty({num_threads, BLOCK_N * hidden_size}, options); + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + hidden_states.data_ptr(), + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + q_a_proj_s.data_ptr(), + kv_a_proj_s.data_ptr(), + buffer.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size, + block_size_N, + block_size_K); + } else { + segment_gemm_kernel_impl( + qa.data_ptr(), + k_input.data_ptr(), + hidden_states.data_ptr(), + q_a_proj_weight.data_ptr(), + kv_a_proj_weight.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank + qk_rope_head_dim, + hidden_size); + } + }); + + // stage 2: apply rmsnorm inplace + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "rms_norm_kernel_impl", [&] { + rms_norm_kernel_impl( + qa.data_ptr(), + v_input.data_ptr(), + q_a_layernorm_weight.data_ptr(), + kv_a_layernorm_weight.data_ptr(), + num_seqs, + q_lora_rank, + kv_lora_rank, + kv_lora_rank + qk_rope_head_dim, + eps); + }); + + // stage 3: q_b_proj + at::Tensor qb; + std::optional bias; + if (use_int8_w8a8) { + qb = int8_scaled_mm_with_quant(qa, q_b_proj_weight, q_b_proj_scale.value(), bias, at::kBFloat16, is_vnni); + } else if (use_fp8_w8a16) { + qb = fp8_scaled_mm_cpu( + qa, q_b_proj_weight, q_b_proj_scale.value(), block_size.value(), bias, at::kBFloat16, is_vnni); + } else { + qb = weight_packed_linear(qa, q_b_proj_weight, bias, is_vnni); + } + qb.as_strided_({num_seqs, num_heads, qk_head_dim}, {num_heads * qk_head_dim, qk_head_dim, 1}); + + // stage 4: bmm + std::optional scale; + auto q_nope = qb.narrow(2, 0, qk_nope_head_dim).transpose_(0, 1); + auto q_nope_out = q_input.narrow(2, 0, kv_lora_rank).transpose_(0, 1); + bmm_cpu(q_nope_out, q_nope, w_kc, is_vnni, scale); + + // stage 5: rope + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "rotary_emb_kernel_impl", [&] { + rotary_emb_kernel_impl( + q_input.data_ptr() + kv_lora_rank, + k_input.data_ptr() + kv_lora_rank, + qb.data_ptr() + qk_nope_head_dim, + k_input.data_ptr() + kv_lora_rank, + positions.data_ptr(), + cos_sin_cache.data_ptr(), + num_seqs, + num_heads, + rotary_dim, + num_heads * qk_head_dim, + qk_head_dim, + kv_lora_rank + qk_rope_head_dim, + num_heads * (kv_lora_rank + qk_rope_head_dim), + kv_lora_rank + qk_rope_head_dim, + kv_lora_rank + qk_rope_head_dim); + }); + + return std::make_tuple(q_input, k_input, v_input); +} + +std::tuple qkv_proj_with_rope_fused_weight( + at::Tensor& hidden_states, + at::Tensor& qkv_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional qkv_a_proj_scale, + std::optional q_b_proj_scale, + bool is_vnni, + std::optional> block_size, + int64_t q_lora_rank, + int64_t kv_lora_rank, + int64_t qk_rope_head_dim) { + RECORD_FUNCTION( + "sgl-kernel::qkv_proj_with_rope_fused_weight", + std::vector({hidden_states, qkv_a_proj_weight, q_b_proj_weight, w_kc})); + + int64_t hidden_size = hidden_states.size(1); + CHECK_EQ(qkv_a_proj_weight.size(0), q_lora_rank + kv_lora_rank + qk_rope_head_dim); + CHECK_EQ(qkv_a_proj_weight.size(1), get_row_size(hidden_size, use_int8_w8a8)); + + std::vector weight_chunks = + at::split(qkv_a_proj_weight, {q_lora_rank, kv_lora_rank + qk_rope_head_dim}, 0); + at::Tensor q_a_proj_weight = weight_chunks[0]; + at::Tensor kv_a_proj_weight = weight_chunks[1]; + at::Tensor q_a_proj_s; + at::Tensor kv_a_proj_s; + + if (use_int8_w8a8) { + TORCH_CHECK(qkv_a_proj_scale.has_value(), "missing qkv_a_proj_scale for int8 w8a8."); + std::vector scale_chunks = + at::split(qkv_a_proj_scale.value(), {q_lora_rank, kv_lora_rank + qk_rope_head_dim}, 0); + q_a_proj_s = scale_chunks[0]; + kv_a_proj_s = scale_chunks[1]; + } + if (use_fp8_w8a16) { + TORCH_CHECK(qkv_a_proj_scale.has_value(), "missing qkv_a_proj_scale for fp8 w8a16."); + int64_t block_size_N = block_size.value()[0]; + int64_t q_a_proj_s_dim0 = div_up(q_lora_rank, block_size_N); + int64_t kv_a_proj_s_dim0 = div_up(kv_lora_rank + qk_rope_head_dim, block_size_N); + std::vector scale_chunks = at::split(qkv_a_proj_scale.value(), {q_a_proj_s_dim0, kv_a_proj_s_dim0}, 0); + q_a_proj_s = scale_chunks[0]; + kv_a_proj_s = scale_chunks[1]; + } + + return qkv_proj_with_rope( + hidden_states, + q_a_proj_weight, + q_b_proj_weight, + kv_a_proj_weight, + w_kc, + q_a_layernorm_weight, + kv_a_layernorm_weight, + positions, + cos_sin_cache, + eps, + use_int8_w8a8, + use_fp8_w8a16, + q_a_proj_s, + q_b_proj_scale, + kv_a_proj_s, + is_vnni, + block_size); +} diff --git a/sgl-kernel/csrc/cpu/rope.cpp b/sgl-kernel/csrc/cpu/rope.cpp new file mode 100644 index 000000000..1c6249466 --- /dev/null +++ b/sgl-kernel/csrc/cpu/rope.cpp @@ -0,0 +1,346 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +void rotary_embedding_3D_kernel_impl( + scalar_t* __restrict__ query_out, + scalar_t* __restrict__ key_out, + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t num_tokens, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t query_out_stride_s, + int64_t key_out_stride_s, + int64_t key_stride_s, + int64_t query_stride_h, + int64_t query_out_stride_h) { + int64_t HR = rotary_dim; + int64_t HK = rotary_dim; + int64_t COFF = HR / 2; + at::parallel_for(0, num_tokens * num_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t seq{0}, head_id{0}; + data_index_init(begin, seq, num_tokens, head_id, num_heads); + for (int64_t i = begin; i < end; ++i) { + int64_t in_offset_q = seq * query_stride_s + head_id * query_stride_h; + int64_t out_offset_q = seq * query_out_stride_s + head_id * query_out_stride_h; + int64_t out_offset_k = seq * key_out_stride_s; + int64_t p = 0; + scalar_t* sin_start = nullptr; + scalar_t* cos_start = nullptr; + // step 0) get the rotary position embedding for the current position + p = positions[seq]; + sin_start = cos_sin_cache + p * HR + COFF; + cos_start = cos_sin_cache + p * HR; + // step 1) apply_rotary_pos_emb for the rotary_dim elements in every + // head of query/key + for (int64_t h = 0; h < rotary_dim; h += 2) { + scalar_t cos = cos_start[h >> 1]; + scalar_t sin = sin_start[h >> 1]; + scalar_t in1 = query[in_offset_q + h]; + scalar_t in2 = query[in_offset_q + h + 1]; + scalar_t out1 = in1 * cos - in2 * sin; + scalar_t out2 = in2 * cos + in1 * sin; + query_out[out_offset_q + h] = out1; + query_out[out_offset_q + h + 1] = out2; + } + for (int64_t h = 0; h < HK; h += 2) { + scalar_t cos = cos_start[h >> 1]; + scalar_t sin = sin_start[h >> 1]; + int64_t k_pe_offset = seq * key_stride_s; + scalar_t in1_k = key[k_pe_offset + h]; + scalar_t in2_k = key[k_pe_offset + h + 1]; + scalar_t out1_k = in1_k * cos - in2_k * sin; + scalar_t out2_k = in2_k * cos + in1_k * sin; + key_out[out_offset_k + h] = out1_k; + key_out[out_offset_k + h + 1] = out2_k; + } + // move to the next index + data_index_step(seq, num_tokens, head_id, num_heads); + } + }); +} + +template +void rotary_embedding_neox_2D_kernel_impl( + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t key_stride_s, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t num_tokens) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + constexpr int64_t bVecSize = bVec::size(); + + int64_t embed_dim = rotary_dim / 2; + bool flag = (embed_dim % bVecSize == 0); + int64_t loop_upper = flag ? embed_dim : embed_dim - bVecSize; + + auto compute_loop = [&](int64_t token_head, scalar_t* cache_ptr, scalar_t* qk) { + int64_t j = 0; + for (; j < loop_upper; j += bVecSize) { + int64_t rot_offset = j; + int64_t x_index = rot_offset; + int64_t y_index = embed_dim + rot_offset; + + int64_t out_x = token_head + x_index; + int64_t out_y = token_head + y_index; + + bVec _cos = bVec::loadu(cache_ptr + x_index); + bVec _sin = bVec::loadu(cache_ptr + y_index); + + bVec _q_x = bVec::loadu(qk + out_x); + bVec _q_y = bVec::loadu(qk + out_y); + fVec _cos_0, _cos_1; + std::tie(_cos_0, _cos_1) = at::vec::convert_to_float(_cos); + fVec _sin_0, _sin_1; + std::tie(_sin_0, _sin_1) = at::vec::convert_to_float(_sin); + fVec _q_x_0, _q_x_1; + std::tie(_q_x_0, _q_x_1) = at::vec::convert_to_float(_q_x); + fVec _q_y_0, _q_y_1; + std::tie(_q_y_0, _q_y_1) = at::vec::convert_to_float(_q_y); + + auto out1_0 = _q_x_0 * _cos_0 - _q_y_0 * _sin_0; + auto out1_1 = _q_x_1 * _cos_1 - _q_y_1 * _sin_1; + auto out1 = convert_from_float_ext(out1_0, out1_1); + out1.store(qk + out_x); + + auto out2_0 = _q_y_0 * _cos_0 + _q_x_0 * _sin_0; + auto out2_1 = _q_y_1 * _cos_1 + _q_x_1 * _sin_1; + auto out2 = convert_from_float_ext(out2_0, out2_1); + out2.store(qk + out_y); + } + if (!flag) { + for (; j < embed_dim; ++j) { + int64_t x_index = j; + int64_t y_index = embed_dim + j; + + int64_t out_x = token_head + x_index; + int64_t out_y = token_head + y_index; + + float _cos = cache_ptr[x_index]; + float _sin = cache_ptr[y_index]; + + float _q_x = qk[out_x]; + float _q_y = qk[out_y]; + + qk[out_x] = _q_x * _cos - _q_y * _sin; + qk[out_y] = _q_y * _cos + _q_x * _sin; + } + } + }; + +#pragma omp parallel for + for (int64_t token_idx = 0; token_idx < num_tokens; ++token_idx) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + + for (int64_t i = 0; i < num_heads; ++i) { + int64_t head_idx = i; + int64_t token_head = token_idx * query_stride_s + head_idx * head_size; + compute_loop(token_head, cache_ptr, query); + } + + for (int64_t i = 0; i < num_kv_heads; ++i) { + int64_t head_idx = i; + int64_t token_head = token_idx * key_stride_s + head_idx * head_size; + compute_loop(token_head, cache_ptr, key); + } + } +} + +template +void rotary_embedding_2D_kernel_impl( + int64_t* __restrict__ positions, + scalar_t* __restrict__ query, + scalar_t* __restrict__ key, + scalar_t* __restrict__ cos_sin_cache, + int64_t rotary_dim, + int64_t query_stride_s, + int64_t key_stride_s, + int64_t num_heads, + int64_t num_kv_heads, + int64_t head_size, + int64_t num_tokens) { + int64_t embed_dim = rotary_dim / 2; + + at::parallel_for(0, num_tokens * num_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t token_idx = {0}, i = {0}; + data_index_init(begin, token_idx, num_tokens, i, num_heads); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + scalar_t* cos_cache_ptr = cache_ptr; + scalar_t* sin_cache_ptr = cache_ptr + embed_dim; + int64_t head_idx = i; + int64_t token_head = token_idx * query_stride_s + head_idx * head_size; + scalar_t* head_query = token_head + query; + for (int64_t j = 0; j < embed_dim; j += 1) { + int64_t rot_offset = j; + int64_t x_index = 2 * rot_offset; + int64_t y_index = 2 * rot_offset + 1; + + float cos = cos_cache_ptr[rot_offset]; + float sin = sin_cache_ptr[rot_offset]; + + float x = head_query[x_index]; + float y = head_query[y_index]; + + head_query[x_index] = x * cos - y * sin; + head_query[y_index] = y * cos + x * sin; + } + data_index_step(token_idx, num_tokens, i, num_heads); + } + }); + + at::parallel_for(0, num_tokens * num_kv_heads, GRAIN_SIZE / rotary_dim, [&](int64_t begin, int64_t end) { + int64_t token_idx{0}, i = {0}; + data_index_init(begin, token_idx, num_tokens, i, num_kv_heads); + for ([[maybe_unused]] auto z : c10::irange(begin, end)) { + int64_t pos = positions[token_idx]; + scalar_t* cache_ptr = cos_sin_cache + pos * rotary_dim; + scalar_t* cos_cache_ptr = cache_ptr; + scalar_t* sin_cache_ptr = cache_ptr + embed_dim; + int64_t head_idx = i; + int64_t token_head = token_idx * key_stride_s + head_idx * head_size; + scalar_t* head_key = key + token_head; + for (int64_t j = 0; j < embed_dim; j += 1) { + int64_t rot_offset = j; + int64_t x_index = 2 * rot_offset; + int64_t y_index = 2 * rot_offset + 1; + + float cos = cos_cache_ptr[rot_offset]; + float sin = sin_cache_ptr[rot_offset]; + + float x = head_key[x_index]; + float y = head_key[y_index]; + + head_key[x_index] = x * cos - y * sin; + head_key[y_index] = y * cos + x * sin; + } + data_index_step(token_idx, num_tokens, i, num_kv_heads); + } + }); +} + +} // namespace + +std::tuple rotary_embedding_cpu( + at::Tensor& positions, + at::Tensor& query, + at::Tensor& key, + int64_t head_size, + at::Tensor& cos_sin_cache, + bool is_neox) { + RECORD_FUNCTION("sgl-kernel::rotary_embedding_cpu", std::vector({query, key})); + CHECK_DIM(1, positions); + const auto input_dim = query.dim(); + const auto input_dtype = query.scalar_type(); + TORCH_CHECK( + input_dim == 2 || input_dim == 3, + " Query/Key must be 2D [num_tokens, num_heads*head_size] or 3D [num_tokens, num_heads, head_size] tensor"); + CHECK_DIM(2, cos_sin_cache); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(query); + CHECK_LAST_DIM_CONTIGUOUS_INPUT(key); + + int64_t rotary_dim = cos_sin_cache.size(1); + if (input_dim == 3) { + // TODO: add support for head_dim != rotary_dim case when input_dim=3 + CHECK_EQ(query.size(-1), rotary_dim); + // TODO: add support for kv_head != 1 + CHECK_EQ(key.size(1), 1); + } + + int64_t num_tokens = positions.numel(); + CHECK_EQ(key.size(0), num_tokens); + CHECK_EQ(query.size(0), num_tokens); + + TORCH_CHECK(positions.scalar_type() == at::kLong, "expect positions to be int64, got ", positions.scalar_type()); + TORCH_CHECK(input_dtype == key.scalar_type(), "query and key must have the same data type"); + TORCH_CHECK(input_dtype == cos_sin_cache.scalar_type(), "query and cos_sin_cache must have the same data type"); + + int64_t num_heads = input_dim == 2 ? query.size(-1) / head_size : query.size(1); + int64_t num_kv_heads = input_dim == 2 ? key.size(-1) / head_size : key.size(1); + int64_t key_stride_s = key.stride(0); + int64_t query_stride_s = query.stride(0); + + // input stride of num head dim is meaningful only when input dim = 3 + int64_t query_stride_h = input_dim == 3 ? query.stride(1) : -1; + at::Tensor query_out = at::empty_like(query); + at::Tensor key_out = at::empty_like(key); + int64_t query_out_stride_s = query_out.stride(0); + int64_t key_out_stride_s = key_out.stride(0); + // output stride of num head dim is meaningful only when input dim = 3 + int64_t query_out_stride_h = input_dim == 3 ? query_out.stride(1) : -1; + + AT_DISPATCH_REDUCED_FLOATING_TYPES(input_dtype, "rotary_embedding_cpu", [&] { + if (input_dim == 2) { + if (is_neox) { + rotary_embedding_neox_2D_kernel_impl( + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + rotary_dim, + query_stride_s, + key_stride_s, + num_heads, + num_kv_heads, + head_size, + num_tokens); + } else { + rotary_embedding_2D_kernel_impl( + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + rotary_dim, + query_stride_s, + key_stride_s, + num_heads, + num_kv_heads, + head_size, + num_tokens); + } + query_out = query; + key_out = key; + + } else { + TORCH_CHECK( + is_neox == false, " Query/Key with 3D [num_tokens, num_heads, head_size] does not support neox rope yet"); + // TODO: add neox style support for rope impl with 3D inputs + rotary_embedding_3D_kernel_impl( + query_out.data_ptr(), + key_out.data_ptr(), + positions.data_ptr(), + query.data_ptr(), + key.data_ptr(), + cos_sin_cache.data_ptr(), + num_tokens, + num_heads, + num_kv_heads, + head_size, + rotary_dim, + query_stride_s, + query_out_stride_s, + key_out_stride_s, + key_stride_s, + query_stride_h, + query_out_stride_h); + } + }); + return std::make_tuple(query_out, key_out); +} diff --git a/sgl-kernel/csrc/cpu/shm.cpp b/sgl-kernel/csrc/cpu/shm.cpp new file mode 100644 index 000000000..1bf65a9b2 --- /dev/null +++ b/sgl-kernel/csrc/cpu/shm.cpp @@ -0,0 +1,666 @@ +#include "shm.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// states for collectives +enum coll_state { + coll_begin = 0, + coll_allreduce_naive__copy_in_done, + coll_allreduce_naive__reduce_done, + // alternative state when allreduce is working on alternative buffer + // of the double buffer. + coll_alt1_allreduce_naive__copy_in_done, + coll_alt2_allreduce_naive__copy_in_done, + coll_alt1_allreduce_naive__reduce_done, + coll_allgather_naive__copy_in_done, + coll_alt1_allgather_naive__copy_in_done, + coll_alt2_allgather_naive__copy_in_done, +}; + +// SHM building blocks +struct SharedData { + const char* name; + int descriptor; + void* bytes; + size_t nbytes; +}; + +void shared_open(SharedData* data, const char* name, size_t nbytes) { + int d = shm_open(name, O_RDWR, S_IRUSR | S_IWUSR); + if (d != -1) { + void* bytes = mmap(NULL, nbytes, PROT_READ | PROT_WRITE, MAP_SHARED, d, 0); + data->name = name; + data->descriptor = d; + data->bytes = bytes; + data->nbytes = nbytes; + } else { + if (errno != ENOENT) { + // don't print if shm can not be found because we want to loop over from + // caller again until the other ranks created the shm + printf("shared_open %s failed, errno=%d\n", name, errno); + } + data->descriptor = -1; + } +} + +void shared_create(SharedData* data, const char* name, void* bytes, size_t nbytes) { + int d = shm_open(name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + if (d != -1) { + nbytes = write(d, bytes, nbytes); + if (nbytes > 0) { + shared_open(data, name, nbytes); + } + } else { + printf("shared_create %s failed\n", name); + } +} + +static int world_size; + +// SHM based allreduce helper functions +// buffer that holds shm name +#define NAME_BUF_SIZE 1000 +#define MAX_BUF_SIZE 1048576 * 32 +#define NAIVE_ALLREDUCE_THRESHOLD 1048576 +#define SHM_BUFFER_NAME "deepspeed_allreduce_buffer" +struct allreduce_workspace { + enum coll_state states[2]; // idx=0 -- state for symmetric_naive_all_reduce + // idx=1 -- state for distributed_naive_all_reduce + // double buffer to avoid syncing between rounds + // offset=0 -- 2*NAIVE_ALLREDUCE_THRESHOLD : buffer for + // symmetric_naive_all_reduce after that : buffer for + // distributed_naive_all_reduce + char buffer[2 * NAIVE_ALLREDUCE_THRESHOLD + 2 * MAX_BUF_SIZE]; +}; + +#define BUFFER0_OFFSET(current_buffer) current_buffer* NAIVE_ALLREDUCE_THRESHOLD +#define BUFFER1_OFFSET(current_buffer) 2 * NAIVE_ALLREDUCE_THRESHOLD + current_buffer* MAX_BUF_SIZE + +struct allreduce_workspace** workspace; + +// buffer for small messages, double buffer +char** symmetric_buffer[2]; +// buffer for large messages, double buffer +char** distributed_buffer[2]; + +void wait_buffer_state_until_2(int index, enum coll_state state0, enum coll_state state1, int state_group) { + volatile enum coll_state* state_ptr = &(workspace[index]->states[state_group]); + + while (1) { + volatile enum coll_state cur_state = *state_ptr; + if (cur_state == state0 || cur_state == state1) break; + } +} + +__m512 cvt_bf16_to_fp32(const __m256i src) __attribute__((target("avx512bw"))); +inline __m512 cvt_bf16_to_fp32(const __m256i src) { + auto y = _mm512_cvtepu16_epi32(src); + return _mm512_castsi512_ps(_mm512_bslli_epi128(y, 2)); +} + +inline __m256i cvt_fp32_to_bf16(const __m512 src) __attribute__((target("avx512bw"))); +inline __m256i cvt_fp32_to_bf16(const __m512 src) { + __m512i value = _mm512_castps_si512(src); + __m512i nan = _mm512_set1_epi32(0xffff); + auto mask_value = _mm512_cmp_ps_mask(src, src, _CMP_ORD_Q); + __m512i ones = _mm512_set1_epi32(0x1); + __m512i vec_bias = _mm512_set1_epi32(0x7fff); + // uint32_t lsb = (input >> 16) & 1; + auto t_value = _mm512_and_si512(_mm512_srli_epi32(value, 16), ones); + // uint32_t rounding_bias = 0x7fff + lsb; + t_value = _mm512_add_epi32(t_value, vec_bias); + // input += rounding_bias; + t_value = _mm512_add_epi32(t_value, value); + // input = input >> 16; + t_value = _mm512_srli_epi32(t_value, 16); + // Check NaN before converting back to bf16 + t_value = _mm512_mask_blend_epi32(mask_value, nan, t_value); + return _mm512_cvtusepi32_epi16(t_value); +} + +__m512 cvt_fp16_to_fp32(const __m256i src) __attribute__((target("avx512bw"))); +inline __m512 cvt_fp16_to_fp32(const __m256i src) { + return _mm512_cvtph_ps(src); +} + +inline __m256i cvt_fp32_to_fp16(const __m512 src) __attribute__((target("avx512bw"))); +inline __m256i cvt_fp32_to_fp16(const __m512 src) { + return _mm512_cvtps_ph(src, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); +} + +void reduce_bf16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_fp16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_fp32_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) + __attribute__((target("avx512bw"))); + +void reduce_all_buffers( + int start_elements, + int num_elements, + c10::ScalarType scalar_type, + int to_buffer_idx, + char* to_buffer, + char** buffers) { + switch (scalar_type) { + case c10::ScalarType::BFloat16: + reduce_bf16_buffers(start_elements, num_elements, to_buffer, buffers); + break; + case c10::ScalarType::Half: + reduce_fp16_buffers(start_elements, num_elements, to_buffer, buffers); + break; + case c10::ScalarType::Float: + reduce_fp32_buffers(start_elements, num_elements, to_buffer, buffers); + break; + default: + assert(!"Should not get here"); + } +} + +#define CVT_ADD_BF16(x) \ + do { \ + auto in##x##_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[x] + i))); \ + inout_val = _mm512_add_ps(inout_val, in##x##_val); \ + } while (0) + +// Reduce functions down below use vectorized algorithm, the number of bytes +// processed each iteration depends on vector length. 256bit vector ==> 32 +// bytes, 512bit vector ==> 64 bytes If you change implementation of +// reduce_bf16_buffers, etc. , check whether this number needs to be changed +#define VECTOR_LENGTH_IN_BYTES 32 + +void reduce_bf16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 2; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[0] + i))); + switch (world_size) { + case 16: + CVT_ADD_BF16(15); + case 15: + CVT_ADD_BF16(14); + case 14: + CVT_ADD_BF16(13); + case 13: + CVT_ADD_BF16(12); + case 12: + CVT_ADD_BF16(11); + case 11: + CVT_ADD_BF16(10); + case 10: + CVT_ADD_BF16(9); + case 9: + CVT_ADD_BF16(8); + case 8: + CVT_ADD_BF16(7); + case 7: + CVT_ADD_BF16(6); + case 6: + CVT_ADD_BF16(5); + case 5: + CVT_ADD_BF16(4); + case 4: + CVT_ADD_BF16(3); + case 3: + CVT_ADD_BF16(2); + case 2: + CVT_ADD_BF16(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = cvt_bf16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[j] + i))); + inout_val = _mm512_add_ps(inout_val, in_val); + } + } + _mm256_storeu_si256((__m256i*)(to_buffer + i), cvt_fp32_to_bf16(inout_val)); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(at::BFloat16*)(buffers[j] + i); + } + *(at::BFloat16*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +#define CVT_ADD_FP16(x) \ + do { \ + auto in##x##_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[x] + i))); \ + inout_val = _mm512_add_ps(inout_val, in##x##_val); \ + } while (0) + +void reduce_fp16_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 2; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[0] + i))); + switch (world_size) { + case 16: + CVT_ADD_FP16(15); + case 15: + CVT_ADD_FP16(14); + case 14: + CVT_ADD_FP16(13); + case 13: + CVT_ADD_FP16(12); + case 12: + CVT_ADD_FP16(11); + case 11: + CVT_ADD_FP16(10); + case 10: + CVT_ADD_FP16(9); + case 9: + CVT_ADD_FP16(8); + case 8: + CVT_ADD_FP16(7); + case 7: + CVT_ADD_FP16(6); + case 6: + CVT_ADD_FP16(5); + case 5: + CVT_ADD_FP16(4); + case 4: + CVT_ADD_FP16(3); + case 3: + CVT_ADD_FP16(2); + case 2: + CVT_ADD_FP16(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = cvt_fp16_to_fp32(_mm256_loadu_si256((__m256i*)(buffers[j] + i))); + inout_val = _mm512_add_ps(inout_val, in_val); + } + } + _mm256_storeu_si256((__m256i*)(to_buffer + i), cvt_fp32_to_fp16(inout_val)); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(at::Half*)(buffers[j] + i); + } + *(at::Half*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +#define CVT_ADD_F32(x) \ + do { \ + auto in##x##_val = _mm256_loadu_ps((float*)(buffers[x] + i)); \ + inout_val = _mm256_add_ps(inout_val, in##x##_val); \ + } while (0) + +void reduce_fp32_buffers(int start_elements, int num_elements, char* to_buffer, char** buffers) { + const int element_size = 4; + const int vector_length = VECTOR_LENGTH_IN_BYTES / element_size; + int main_elements = num_elements - (num_elements % vector_length); + int remain_elements = num_elements % vector_length; + + // process aligned part +#pragma omp parallel for + for (int i = start_elements * element_size; i < (start_elements + main_elements) * element_size; + i += VECTOR_LENGTH_IN_BYTES) { + auto inout_val = _mm256_loadu_ps((float*)(buffers[0] + i)); + switch (world_size) { + case 16: + CVT_ADD_F32(15); + case 15: + CVT_ADD_F32(14); + case 14: + CVT_ADD_F32(13); + case 13: + CVT_ADD_F32(12); + case 12: + CVT_ADD_F32(11); + case 11: + CVT_ADD_F32(10); + case 10: + CVT_ADD_F32(9); + case 9: + CVT_ADD_F32(8); + case 8: + CVT_ADD_F32(7); + case 7: + CVT_ADD_F32(6); + case 6: + CVT_ADD_F32(5); + case 5: + CVT_ADD_F32(4); + case 4: + CVT_ADD_F32(3); + case 3: + CVT_ADD_F32(2); + case 2: + CVT_ADD_F32(1); + case 1: + break; + default: + for (int j = 1; j < world_size; j++) { + auto in_val = _mm256_loadu_ps((float*)(buffers[j] + i)); + inout_val = _mm256_add_ps(inout_val, in_val); + } + } + _mm256_storeu_ps((float*)(to_buffer + i), inout_val); + } + + // process remaining part + int i = (start_elements + main_elements) * element_size; + while (remain_elements > 0) { + float val = 0.0f; + for (int j = 0; j < world_size; j++) { + val += *(float*)(buffers[j] + i); + } + *(float*)(to_buffer + i) = val; + remain_elements--; + i += element_size; + } +} + +static bool is_initialized = false; +static int world_rank; + +void shm_initialize(int size, int rank, const char* addr_string, const char* port_string) { + if (is_initialized) { + return; + } + is_initialized = true; + + world_size = size; + world_rank = rank; + + char shm_name_prefix[NAME_BUF_SIZE]; + char shm_name[NAME_BUF_SIZE]; + snprintf(shm_name_prefix, NAME_BUF_SIZE, "%s_%d_%s_%s", SHM_BUFFER_NAME, getuid(), addr_string, port_string); + // create shared workspace for SHM based allreduce + SharedData allreduce_buffer; + // allocate workspace_buf for current rank + struct allreduce_workspace* workspace_buf; + struct allreduce_workspace* workspace_buf_other; + workspace_buf = (struct allreduce_workspace*)malloc(sizeof(struct allreduce_workspace)); + snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, rank); + shared_create(&allreduce_buffer, shm_name, workspace_buf, sizeof(struct allreduce_workspace)); + workspace_buf = (struct allreduce_workspace*)allreduce_buffer.bytes; + workspace_buf->states[0] = coll_alt2_allreduce_naive__copy_in_done; + workspace_buf->states[1] = coll_begin; + + // create the workspace pointer list + workspace = (struct allreduce_workspace**)malloc(size * sizeof(struct allreduce_workspace*)); + symmetric_buffer[0] = (char**)malloc(size * sizeof(char**)); + symmetric_buffer[1] = (char**)malloc(size * sizeof(char**)); + distributed_buffer[0] = (char**)malloc(size * sizeof(char**)); + distributed_buffer[1] = (char**)malloc(size * sizeof(char**)); + + // map shm of all ranks + for (int i = 0; i < size; i++) { + if (i != rank) { + snprintf(shm_name, NAME_BUF_SIZE, "%.900s_%d", shm_name_prefix, i); + // printf("open %s, %d\n", shm_name, rank); + do { + shared_open(&allreduce_buffer, shm_name, sizeof(struct allreduce_workspace)); + } while (allreduce_buffer.descriptor == -1 && errno == ENOENT); + workspace_buf_other = (struct allreduce_workspace*)allreduce_buffer.bytes; + workspace[i] = workspace_buf_other; + } else { + workspace[i] = workspace_buf; + } + symmetric_buffer[0][i] = workspace[i]->buffer + BUFFER0_OFFSET(0); + symmetric_buffer[1][i] = workspace[i]->buffer + BUFFER0_OFFSET(1); + distributed_buffer[0][i] = workspace[i]->buffer + BUFFER1_OFFSET(0); + distributed_buffer[1][i] = workspace[i]->buffer + BUFFER1_OFFSET(1); + } +} + +static void parallel_memcpy(void* to, void* from, size_t n_bytes) __attribute__((target("avx512bw"))); +static void parallel_memcpy(void* to, void* from, size_t n_bytes) { + auto aligned_bytes = n_bytes - (n_bytes % VECTOR_LENGTH_IN_BYTES); + // process aligned part +#pragma omp parallel for + for (size_t i = 0; i < aligned_bytes; i += VECTOR_LENGTH_IN_BYTES) { + auto val = _mm256_loadu_si256((__m256i*)((char*)from + i)); + _mm256_storeu_si256((__m256i*)((char*)to + i), val); + } + + // process remaining part + for (size_t i = aligned_bytes; i < n_bytes; i++) { + *((char*)to + i) = *((char*)from + i); + } +} + +#define positive_mod(num, mod) ((((num) % (mod)) + (mod)) % (mod)) +#define rank_mod(rank) positive_mod(rank, world_size) +size_t slice_size(size_t chunk_el, int slice_idx) { + size_t slice_size = chunk_el / world_size; + return slice_idx == world_size - 1 ? slice_size + (chunk_el % world_size) : slice_size; +} + +char* slice_data(char* data_ptr, size_t chunk_el, int el_size, int slice_idx) { + size_t slice_size = chunk_el / world_size; + size_t el_offset = slice_size * slice_idx; + return data_ptr + el_offset * el_size; +} + +size_t slice_el_start(size_t chunk_el, int slice_idx) { + size_t slice_size = chunk_el / world_size; + return slice_size * slice_idx; +} + +void symmetric_naive_all_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { + const int state_group = 0; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allreduce_naive__copy_in_done; + enum coll_state copy_next = coll_alt1_allreduce_naive__copy_in_done; + + switch (state_idx) { + case 0: + copy_current = coll_allreduce_naive__copy_in_done; + copy_next = coll_alt1_allreduce_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allreduce_naive__copy_in_done; + copy_next = coll_alt2_allreduce_naive__copy_in_done; + break; + case 2: + copy_current = coll_alt2_allreduce_naive__copy_in_done; + copy_next = coll_allreduce_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 3; + + parallel_memcpy(symmetric_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until the other rank copy the buffer + if (i != world_rank) { + wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + } + } + + // each rank reduce the buffer independently so therre is no need for + // synchronization afterward + reduce_all_buffers(0, chunk_el, scalar_type, world_rank, data_ptr, symmetric_buffer[current_buffer]); + + // switch buffer + current_buffer = 1 - current_buffer; +} + +// naive allreduce distributed, each rank do naive reduce on its slice +void distributed_naive_reduce(char* data_ptr, c10::ScalarType scalar_type, size_t chunk_size, size_t chunk_el) { + const int state_group = 1; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allreduce_naive__copy_in_done; + enum coll_state reduce_current = coll_allreduce_naive__reduce_done; + enum coll_state copy_next = coll_alt1_allreduce_naive__copy_in_done; + + // similar to symmetric_naive_allreduce, but here we only need two sets of + // states, because distributed naive reduce has two barriers in the algorithm + switch (state_idx) { + case 0: + copy_current = coll_allreduce_naive__copy_in_done; + reduce_current = coll_allreduce_naive__reduce_done; + copy_next = coll_alt1_allreduce_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allreduce_naive__copy_in_done; + reduce_current = coll_alt1_allreduce_naive__reduce_done; + copy_next = coll_allreduce_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 2; + + int data_size = chunk_size / chunk_el; + parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks copy the buffer + if (i != world_rank) wait_buffer_state_until_2(i, copy_current, reduce_current, state_group); + } + + // reduce scatter + reduce_all_buffers( + slice_el_start(chunk_el, world_rank), + slice_size(chunk_el, world_rank), + scalar_type, + world_rank, + distributed_buffer[current_buffer][world_rank], + distributed_buffer[current_buffer]); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = reduce_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks reduce the buffer + if (i != world_rank) wait_buffer_state_until_2(i, reduce_current, copy_next, state_group); + } + + for (int i = 0; i < world_size; i++) { + int rank = (i + world_rank) % world_size; + parallel_memcpy( + slice_data(data_ptr, chunk_el, data_size, rank), + slice_data(distributed_buffer[current_buffer][rank], chunk_el, chunk_size / chunk_el, rank), + slice_size(chunk_el, rank) * data_size); + } + + current_buffer = 1 - current_buffer; +} + +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size) { + for (int offset = 0; offset < data_size; offset += MAX_BUF_SIZE) { + auto data_ptr = ((char*)(data.data_ptr()) + offset); + size_t chunk_size = data_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : data_size - offset; + size_t chunk_el = chunk_size / (data_size / numel); + if (chunk_size < NAIVE_ALLREDUCE_THRESHOLD) { + symmetric_naive_all_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + } else { + distributed_naive_reduce(data_ptr, data.scalar_type(), chunk_size, chunk_el); + } + } +} + +void naive_all_gather(char* result_ptr, char* data_ptr, size_t res_stride, size_t chunk_size, size_t chunk_el) { + const int state_group = 1; + static int current_buffer = 0; + static int state_idx = 0; + + // init states to case 0 to get rid of "maybe-uninitialized" warning. + enum coll_state copy_current = coll_allgather_naive__copy_in_done; + enum coll_state copy_next = coll_alt1_allgather_naive__copy_in_done; + + switch (state_idx) { + case 0: + copy_current = coll_allgather_naive__copy_in_done; + copy_next = coll_alt1_allgather_naive__copy_in_done; + break; + case 1: + copy_current = coll_alt1_allgather_naive__copy_in_done; + copy_next = coll_alt2_allgather_naive__copy_in_done; + break; + case 2: + copy_current = coll_alt2_allgather_naive__copy_in_done; + copy_next = coll_allgather_naive__copy_in_done; + break; + default: + assert(!"Should not get here."); + } + state_idx = (state_idx + 1) % 3; + + parallel_memcpy(distributed_buffer[current_buffer][world_rank], data_ptr, chunk_size); + std::atomic_thread_fence(std::memory_order_release); + workspace[world_rank]->states[state_group] = copy_current; + + for (int i = 0; i < world_size; i++) { + // wait until all the other ranks copy the buffer + if (i != world_rank) wait_buffer_state_until_2(i, copy_current, copy_next, state_group); + } + for (int i = 0; i < world_size; i++) { + parallel_memcpy(result_ptr + i * res_stride, distributed_buffer[current_buffer][i], chunk_size); + } + current_buffer = 1 - current_buffer; +} + +torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size) { + size_t dim_el = data.stride(dim) * data.size(dim); + int dtype_size = data_size / numel; + size_t dim_size = dim_el * dtype_size; + int dim_count = data_size / dim_size; + auto data_ptr = (char*)(data.data_ptr()); + auto result_ptr = (char*)(result.data_ptr()); + for (int i = 0; i < dim_count; i++) { + for (size_t offset = 0; offset < dim_size; offset += MAX_BUF_SIZE) { + size_t chunk_size = dim_size - offset > MAX_BUF_SIZE ? MAX_BUF_SIZE : dim_size - offset; + size_t chunk_el = chunk_size / dtype_size; + naive_all_gather( + result_ptr + i * dim_size * world_size + offset, + data_ptr + i * dim_size + offset, + dim_size, + chunk_size, + chunk_el); + } + } + return result; +} diff --git a/sgl-kernel/csrc/cpu/shm.h b/sgl-kernel/csrc/cpu/shm.h new file mode 100644 index 000000000..3e903972c --- /dev/null +++ b/sgl-kernel/csrc/cpu/shm.h @@ -0,0 +1,11 @@ +#include + +#include + +#ifndef __SHM_COLLECTIVES__ +#define __SHM_COLLECTIVES__ +#define VECTOR_LENGTH_IN_BYTES 32 +void shm_initialize(int size, int rank, const char* addr_string, const char* port_string); +void all_reduce_outer_loop(torch::Tensor& data, size_t numel, int data_size); +torch::Tensor& all_gather(torch::Tensor& result, torch::Tensor& data, int dim, size_t numel, int data_size); +#endif diff --git a/sgl-kernel/csrc/cpu/topk.cpp b/sgl-kernel/csrc/cpu/topk.cpp new file mode 100644 index 000000000..abc5a34fa --- /dev/null +++ b/sgl-kernel/csrc/cpu/topk.cpp @@ -0,0 +1,662 @@ +#include "common.h" +#include "vec.h" + +namespace { + +template +inline void softmax(float* __restrict__ out, const scalar_t* __restrict__ input) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + constexpr int kVecSize = bVec::size(); + + // step 1: get max + fVec max_fvec = fVec(-std::numeric_limits::infinity()); + if constexpr (SIZE < kVecSize) { + // SIZE = 1, 2, 4, 8, 16; only the top half is used + bVec x_bvec = bVec::loadu(input, SIZE); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + x_fvec0 = fVec::set(max_fvec, x_fvec0, SIZE); + max_fvec = at::vec::maximum(max_fvec, x_fvec0); + x_fvec0.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + max_fvec = at::vec::maximum(max_fvec, x_fvec0); + max_fvec = at::vec::maximum(max_fvec, x_fvec1); + x_fvec0.store(out + d); + x_fvec1.store(out + d + fVec::size()); + } + } + float max_val = vec_reduce_max(max_fvec); + max_fvec = fVec(max_val); + + // step 2: sum of (x - max).exp() + fVec sum_fvec = fVec(float(0)); + if constexpr (SIZE < fVec::size()) { + // SIZE = 1, 2, 4, 8 + fVec x_fvec = (fVec::loadu(out, SIZE) - max_fvec).exp_u20(); + x_fvec = fVec::set(sum_fvec, x_fvec, SIZE); + sum_fvec += x_fvec; + x_fvec.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += fVec::size()) { + fVec x_fvec = (fVec::loadu(out + d) - max_fvec).exp_u20(); + sum_fvec += x_fvec; + x_fvec.store(out + d); + } + } + float sum_val = vec_reduce_sum(sum_fvec); + + // step 3: x * (1 / sum) + sum_fvec = fVec(1.f / sum_val); + if constexpr (SIZE < fVec::size()) { + // SIZE = 1, 2, 4, 8 + fVec out_fvec = fVec::loadu(out, SIZE) * sum_fvec; + out_fvec.store(out, SIZE); + } else { + for (int d = 0; d < SIZE; d += fVec::size()) { + fVec out_fvec = fVec::loadu(out + d) * sum_fvec; + out_fvec.store(out + d); + } + } +} + +template +void grouped_topk_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + int64_t num_groups, + int64_t topk_group, + bool renormalize) { + const int64_t num_experts_per_group = NUM_EXPERTS / num_groups; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + + using elem_t = std::pair; + std::vector queue(num_groups); + std::vector queue2(topk_group * num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + // do softmax to get scores + softmax(scores, gating_output + i * NUM_EXPERTS); + + // find max score per group + for (int64_t g = 0; g < num_groups; ++g) { + float gmax = -std::numeric_limits::infinity(); + for (int64_t e = 0; e < num_experts_per_group; ++e) { + gmax = std::max(gmax, scores[g * num_experts_per_group + e]); + } + queue[g] = {gmax, g}; + } + + // find group topk + std::partial_sort( + queue.begin(), queue.begin() + topk_group, queue.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t g = 0; g < topk_group; ++g) { + int32_t group_idx = queue[g].second; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + int32_t expert_idx = group_idx * num_experts_per_group + e; + queue2[g * num_experts_per_group + e] = {scores[expert_idx], expert_idx}; + } + } + + // find global topk + std::partial_sort( + queue2.begin(), queue2.begin() + topk, queue2.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] = queue2[j].first; + topk_ids[i * topk + j] = queue2[j].second; + } + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +inline void sigmoid(float* __restrict__ out, const scalar_t* __restrict__ input) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + + const fVec one = fVec(1.f); + + constexpr int kVecSize = bVec::size(); + for (int d = 0; d < SIZE; d += kVecSize) { + bVec x_bvec = bVec::loadu(input + d); + fVec x_fvec0, x_fvec1; + std::tie(x_fvec0, x_fvec1) = at::vec::convert_to_float(x_bvec); + + x_fvec0 = one / (one + x_fvec0.neg().exp_u20()); + x_fvec1 = one / (one + x_fvec1.neg().exp_u20()); + + x_fvec0.store(out + d); + x_fvec1.store(out + d + fVec::size()); + } +} + +template +void topk_sigmoid_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + bool renormalize) { + using Vec = at::vec::Vectorized; + const int64_t num_experts_per_group = NUM_EXPERTS; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + using elem_t = std::pair; + std::vector queue(num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + at::vec::convert(gating_output + i * NUM_EXPERTS, scores, NUM_EXPERTS); + + float gmax = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, scores, num_experts_per_group); + + // find position of first max, + // note that we may have multiple max values. + int first_max_idx = -1; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + if (scores[e] == gmax) { + first_max_idx = e; + break; + } + } + + // scalar sigmoid + topk_weights[i] = 1.0 / (1.0 + exp(0.0 - gmax)); + topk_ids[i] = first_max_idx; + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +void topk_softmax_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + int64_t num_tokens, + int64_t topk, + bool renormalize) { + const int64_t num_experts_per_group = NUM_EXPERTS; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + alignas(64) float scores[NUM_EXPERTS]; + using elem_t = std::pair; + std::vector queue(num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + softmax(scores, gating_output + i * NUM_EXPERTS); + + for (int64_t e = 0; e < num_experts_per_group; ++e) { + queue[e] = {scores[e], e}; + } + + std::partial_sort( + queue.begin(), + queue.begin() + num_experts_per_group, + queue.end(), + [](const elem_t& x, const elem_t& y) -> bool { return x.first > y.first; }); + + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] = queue[j].first; + topk_ids[i * topk + j] = queue[j].second; + } + + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < topk; ++j) { + sum += topk_weights[i * topk + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < topk; ++j) { + topk_weights[i * topk + j] *= scale; + } + } + } + }); +} + +template +inline void +apply_bias(float* __restrict__ scores2, const float* __restrict__ scores, const param_t* __restrict__ bias) { + using fVec = at::vec::Vectorized; + using bVec = at::vec::Vectorized; + auto vec_size = bVec::size(); + int d = 0; + for (; d <= SIZE - vec_size; d += vec_size) { + fVec bias0, bias1, x0, x1; + std::tie(bias0, bias1) = load_float_vec2(bias + d); + std::tie(x0, x1) = load_float_vec2(scores + d); + x0 = x0 + bias0; + x1 = x1 + bias1; + x0.store(scores2 + d); + x1.store(scores2 + d + fVec::size()); + } + for (; d < SIZE; d++) { + scores2[d] = scores[d] + (float)bias[d]; + } +} + +template +void biased_grouped_topk_kernel_impl( + float* __restrict__ topk_weights, + int32_t* __restrict__ topk_ids, + const scalar_t* __restrict__ gating_output, + const param_t* __restrict__ bias, + int64_t num_tokens, + int64_t num_groups, + int64_t topk_group, + bool renormalize) { + using Vec = at::vec::Vectorized; + + const int64_t num_experts_per_group = NUM_EXPERTS / num_groups; + at::parallel_for(0, num_tokens, 0, [&](int64_t begin, int64_t end) { + // scores: sigmoid + alignas(64) float scores[NUM_EXPERTS]; + // scores for choice: sigmoid + bias + alignas(64) float scores2[NUM_EXPERTS]; + + using elem_t = std::pair; + std::vector queue(num_groups); + std::vector queue2(topk_group * num_experts_per_group); + + for (int64_t i = begin; i < end; ++i) { + // do sigmoid to get scores + sigmoid(scores, gating_output + i * NUM_EXPERTS); + + apply_bias(scores2, scores, bias); + + for (int64_t g = 0; g < num_groups; ++g) { + // find the max + float gmax = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, + scores2 + g * num_experts_per_group, + num_experts_per_group); + + // find position of first max, + // note that we may have multiple max values. + int first_max_idx = -1; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + if (scores2[g * num_experts_per_group + e] == gmax) { + first_max_idx = g * num_experts_per_group + e; + break; + } + } + + // find the 2nd max + scores2[first_max_idx] = -std::numeric_limits::infinity(); + float gmax2 = at::vec::reduce_all( + [](Vec& x, Vec& y) { return at::vec::maximum(x, y); }, + scores2 + g * num_experts_per_group, + num_experts_per_group); + // restore scores for choice + scores2[first_max_idx] = gmax; + + queue[g] = {gmax + gmax2, g}; + } + + // find group topk + std::partial_sort( + queue.begin(), queue.begin() + topk_group, queue.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int64_t g = 0; g < topk_group; ++g) { + int32_t group_idx = queue[g].second; + for (int64_t e = 0; e < num_experts_per_group; ++e) { + int32_t expert_idx = group_idx * num_experts_per_group + e; + queue2[g * num_experts_per_group + e] = {scores2[expert_idx], expert_idx}; + } + } + + // find global topk + std::partial_sort( + queue2.begin(), queue2.begin() + TOPK, queue2.end(), [](const elem_t& x, const elem_t& y) -> bool { + return x.first > y.first; + }); + + for (int j = 0; j < TOPK; ++j) { + int32_t index = queue2[j].second; + topk_ids[i * TOPK + j] = index; + topk_weights[i * TOPK + j] = scores[index]; + } + +#if defined(CPU_CAPABILITY_AVX512) + if (renormalize) { + __mmask16 mask = (1ULL << TOPK) - 1; + __m512 x = _mm512_maskz_loadu_ps(mask, topk_weights + i * TOPK); + float sum = _mm512_reduce_add_ps(x); + __m512 vscale = _mm512_set1_ps(1.f / sum); + __m512 y = _mm512_mul_ps(x, vscale); + _mm512_mask_storeu_ps(topk_weights + i * TOPK, mask, y); + } +#else + if (renormalize) { + float sum = 0.f; + for (int64_t j = 0; j < TOPK; ++j) { + sum += topk_weights[i * TOPK + j]; + } + float scale = 1.f / sum; + for (int64_t j = 0; j < TOPK; ++j) { + topk_weights[i * TOPK + j] *= scale; + } + } +#endif + } + }); +} + +#define LAUNCH_GROUPED_TOPK_KERNEL(NE) \ + grouped_topk_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + num_expert_group, \ + topk_group, \ + renormalize); + +#define LAUNCH_TOPK_SIGMOID_KERNEL(NE) \ + topk_sigmoid_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + renormalize); + +#define LAUNCH_TOPK_SOFTMAX_KERNEL(NE) \ + topk_softmax_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + num_tokens, \ + topk, \ + renormalize); + +#define LAUNCH_BIASED_GROUPED_TOPK_KERNEL(NE, NTOPK) \ + biased_grouped_topk_kernel_impl( \ + topk_weights.data_ptr(), \ + topk_ids.data_ptr(), \ + gating_output.data_ptr(), \ + correction_bias.data_ptr(), \ + num_tokens, \ + num_expert_group, \ + topk_group, \ + renormalize); + +} // anonymous namespace + +std::tuple +topk_sigmoid_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize) { + RECORD_FUNCTION("sgl-kernel::topk_sigmoid_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + TORCH_CHECK(topk == 1, "topk_sigmoid only supports topk=1 case"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "topk_sigmoid_kernel", [&] { + switch (num_experts) { + case 1: + LAUNCH_TOPK_SIGMOID_KERNEL(1); + break; + case 2: + LAUNCH_TOPK_SIGMOID_KERNEL(2); + break; + case 4: + LAUNCH_TOPK_SIGMOID_KERNEL(4); + break; + case 8: + LAUNCH_TOPK_SIGMOID_KERNEL(8); + break; + case 16: + LAUNCH_TOPK_SIGMOID_KERNEL(16); + break; + case 32: + LAUNCH_TOPK_SIGMOID_KERNEL(32); + break; + case 64: + LAUNCH_TOPK_SIGMOID_KERNEL(64); + break; + case 128: + LAUNCH_TOPK_SIGMOID_KERNEL(128); + break; + case 160: + LAUNCH_TOPK_SIGMOID_KERNEL(160); + break; + case 256: + LAUNCH_TOPK_SIGMOID_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +std::tuple +topk_softmax_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize) { + RECORD_FUNCTION("sgl-kernel::topk_softmax_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "topk_softmax_cpu", [&] { + switch (num_experts) { + case 1: + LAUNCH_TOPK_SOFTMAX_KERNEL(1); + break; + case 2: + LAUNCH_TOPK_SOFTMAX_KERNEL(2); + break; + case 4: + LAUNCH_TOPK_SOFTMAX_KERNEL(4); + break; + case 8: + LAUNCH_TOPK_SOFTMAX_KERNEL(8); + break; + case 16: + LAUNCH_TOPK_SOFTMAX_KERNEL(16); + break; + case 32: + LAUNCH_TOPK_SOFTMAX_KERNEL(32); + break; + case 64: + LAUNCH_TOPK_SOFTMAX_KERNEL(64); + break; + case 128: + LAUNCH_TOPK_SOFTMAX_KERNEL(128); + break; + case 160: + LAUNCH_TOPK_SOFTMAX_KERNEL(160); + break; + case 256: + LAUNCH_TOPK_SOFTMAX_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +// grouped topk for DeepSeek V2 +std::tuple grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded) { + // TODO: Will support num_fused_shared_experts, routed_scaling_factor and num_token_non_padded. + // For now, we just check them as default value. + TORCH_CHECK( + num_fused_shared_experts == 0, + "num_fused_shared_experts must be 0 default value, got: ", + num_fused_shared_experts); + TORCH_CHECK( + !routed_scaling_factor.has_value() || routed_scaling_factor.value() == 1.0f, + "routed_scaling_factor must be None or 1.0f default value, got: ", + routed_scaling_factor.value()); + TORCH_CHECK( + !num_token_non_padded.has_value(), + "num_token_non_padded must be None default value, got: ", + num_token_non_padded.value()); + + RECORD_FUNCTION("sgl-kernel::grouped_topk_cpu", std::vector({hidden_states, gating_output})); + CHECK_INPUT(gating_output); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + AT_DISPATCH_REDUCED_FLOATING_TYPES(st, "grouped_topk_kernel", [&] { + switch (num_experts) { + case 1: + LAUNCH_GROUPED_TOPK_KERNEL(1); + break; + case 2: + LAUNCH_GROUPED_TOPK_KERNEL(2); + break; + case 4: + LAUNCH_GROUPED_TOPK_KERNEL(4); + break; + case 8: + LAUNCH_GROUPED_TOPK_KERNEL(8); + break; + case 16: + LAUNCH_GROUPED_TOPK_KERNEL(16); + break; + case 32: + LAUNCH_GROUPED_TOPK_KERNEL(32); + break; + case 64: + LAUNCH_GROUPED_TOPK_KERNEL(64); + break; + case 128: + LAUNCH_GROUPED_TOPK_KERNEL(128); + break; + case 160: + LAUNCH_GROUPED_TOPK_KERNEL(160); + break; + case 256: + LAUNCH_GROUPED_TOPK_KERNEL(256); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} + +// biased grouped topk DeepSeek V3/R1 +std::tuple biased_grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + at::Tensor& correction_bias, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded) { + // TODO: Will support num_fused_shared_experts, routed_scaling_factor and num_token_non_padded. + // For now, we just check them as default value. + TORCH_CHECK( + num_fused_shared_experts == 0, + "num_fused_shared_experts must be 0 default value, got: ", + num_fused_shared_experts); + TORCH_CHECK( + !num_token_non_padded.has_value(), + "num_token_non_padded must be None default value, got: ", + num_token_non_padded.value()); + + RECORD_FUNCTION( + "sgl-kernel::biased_grouped_topk_cpu", std::vector({hidden_states, gating_output, correction_bias})); + + CHECK_INPUT(gating_output); + CHECK_INPUT(correction_bias); + + const auto st = hidden_states.scalar_type(); + CHECK_EQ(gating_output.scalar_type(), st); + + int64_t num_tokens = hidden_states.size(0); + int64_t num_experts = gating_output.size(1); + TORCH_CHECK(gating_output.size(0) == num_tokens, "Number of tokens mismatch"); + TORCH_CHECK(correction_bias.numel() == num_experts, "Bias shape mismatch"); + at::Tensor topk_weights = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kFloat)); + at::Tensor topk_ids = at::empty({num_tokens, topk}, hidden_states.options().dtype(at::kInt)); + + CPU_DISPATCH_REDUCED_FLOATING_TYPES_EXT(st, correction_bias.scalar_type(), "biased_grouped_topk_kernel", [&] { + TORCH_CHECK(topk == 8, "Unexpected topk: ", topk); + switch (num_experts) { + case 256: + LAUNCH_BIASED_GROUPED_TOPK_KERNEL(256, 8); + break; + default: + TORCH_CHECK(false, "Unexpected num_experts: ", num_experts); + } + }); + return std::make_tuple(topk_weights, topk_ids); +} diff --git a/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp b/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp new file mode 100644 index 000000000..2c8d9e3ec --- /dev/null +++ b/sgl-kernel/csrc/cpu/torch_extension_cpu.cpp @@ -0,0 +1,373 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "sgl_kernel_ops.h" +#include "shm.h" + +// silu_and_mul +at::Tensor silu_and_mul_cpu(at::Tensor& input); + +// gelu_and_mul +at::Tensor gelu_tanh_and_mul_cpu(const at::Tensor& input); +at::Tensor gelu_and_mul_cpu(const at::Tensor& input); + +// l2norm +at::Tensor l2norm_cpu(at::Tensor& input, double eps); + +// rmsnorm +at::Tensor rmsnorm_cpu(at::Tensor& input, at::Tensor& weight, double eps); + +// fused_add_rmsnorm +void fused_add_rmsnorm_cpu(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps); + +// topk +std::tuple +topk_sigmoid_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize); +std::tuple +topk_softmax_cpu(at::Tensor& hidden_states, at::Tensor& gating_output, int64_t topk, bool renormalize); + +std::tuple grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded); + +std::tuple biased_grouped_topk_cpu( + at::Tensor& hidden_states, + at::Tensor& gating_output, + at::Tensor& correction_bias, + int64_t topk, + bool renormalize, + int64_t num_expert_group, + int64_t topk_group, + int64_t num_fused_shared_experts, + std::optional routed_scaling_factor, + std::optional num_token_non_padded); + +// attention +void decode_attention_cpu( + at::Tensor& query, + at::Tensor& k_cache, + at::Tensor& v_cache, + at::Tensor& output, + at::Tensor& key, + at::Tensor& value, + at::Tensor& loc, + at::Tensor& attn_logits, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + double sm_scale, + double logit_cap); + +void extend_attention_cpu( + at::Tensor& q_extend, + at::Tensor& k_extend, + at::Tensor& v_extend, + at::Tensor& o_extend, + at::Tensor& k_buffer, + at::Tensor& v_buffer, + at::Tensor& req_to_token, + at::Tensor& req_pool_indices, + at::Tensor& seq_lens, + at::Tensor& extend_seq_lens, + at::Tensor& extend_start_loc, + int64_t max_len_extend, + double sm_scale, + double logit_cap); + +// weight prepack +at::Tensor convert_weight_packed(at::Tensor& weight); + +// quant +std::tuple per_token_quant_int8_cpu(at::Tensor& A); + +// gemm +at::Tensor +weight_packed_linear(at::Tensor& mat1, at::Tensor& mat2, const std::optional& bias, bool is_vnni); + +// igemm +at::Tensor int8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales1, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// fp8 gemm +at::Tensor fp8_scaled_mm_cpu( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + std::vector block_size, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// quant + igemm +at::Tensor int8_scaled_mm_with_quant( + at::Tensor& mat1, + at::Tensor& mat2, + at::Tensor& scales2, + const std::optional& bias, + at::ScalarType out_dtype, + bool is_vnni); + +// bmm +void bmm_cpu(at::Tensor& out, at::Tensor& mat1, at::Tensor& mat2, bool is_vnni, const std::optional& scale); + +// fused moe +at::Tensor fused_experts_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& topk_weights, + at::Tensor& topk_ids, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni); + +at::Tensor shared_expert_cpu( + at::Tensor& hidden_states, + at::Tensor& w1, + at::Tensor& w2, + at::Tensor& fused_experts_out, + double routed_scaling_factor, + bool inplace, + bool use_int8_w8a8, + bool use_fp8_w8a16, + const std::optional& w1_scale, + const std::optional& w2_scale, + const std::optional> block_size, + const std::optional& a1_scale, + const std::optional& a2_scale, + bool is_vnni); + +// weight absorption +std::tuple qkv_proj_with_rope( + at::Tensor& hidden_states, + at::Tensor& q_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& kv_a_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional q_a_proj_scale, + std::optional q_b_proj_scale, + std::optional kv_a_proj_scale, + bool is_vnni, + std::optional> block_size); + +std::tuple qkv_proj_with_rope_fused_weight( + at::Tensor& hidden_states, + at::Tensor& qkv_a_proj_weight, + at::Tensor& q_b_proj_weight, + at::Tensor& w_kc, + at::Tensor& q_a_layernorm_weight, + at::Tensor& kv_a_layernorm_weight, + at::Tensor& positions, + at::Tensor& cos_sin_cache, + double eps, + bool use_int8_w8a8, + bool use_fp8_w8a16, + std::optional qkv_a_proj_scale, + std::optional q_b_proj_scale, + bool is_vnni, + std::optional> block_size, + int64_t q_lora_rank, + int64_t kv_lora_rank, + int64_t qk_rope_head_dim); + +// shared memory init +void initialize(int64_t size, int64_t rank); + +// shared mmeory all_reduce +void shm_allreduce(at::Tensor& data, int64_t op); + +// shared memory all_gather +at::Tensor shm_allgather(at::Tensor& data, int64_t dim); + +// rope +std::tuple rotary_embedding_cpu( + at::Tensor& positions, + at::Tensor& query, + at::Tensor& key, + int64_t head_size, + at::Tensor& cos_sin_cache, + bool is_neox); + +// CPU and memory binding +std::string init_cpu_threads_env(const std::string& cpu_ids); + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + // activation + m.def("silu_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("silu_and_mul_cpu", torch::kCPU, &silu_and_mul_cpu); + m.def("gelu_tanh_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("gelu_tanh_and_mul_cpu", torch::kCPU, &gelu_tanh_and_mul_cpu); + m.def("gelu_and_mul_cpu(Tensor input) -> Tensor"); + m.impl("gelu_and_mul_cpu", torch::kCPU, &gelu_and_mul_cpu); + + // norm + m.def("rmsnorm_cpu(Tensor input, Tensor weight, float eps) -> Tensor"); + m.impl("rmsnorm_cpu", torch::kCPU, &rmsnorm_cpu); + m.def("l2norm_cpu(Tensor input, float eps) -> Tensor"); + m.impl("l2norm_cpu", torch::kCPU, &l2norm_cpu); + m.def("fused_add_rmsnorm_cpu(Tensor(a!) input, Tensor residual, Tensor weight, float eps) -> ()"); + m.impl("fused_add_rmsnorm_cpu", torch::kCPU, &fused_add_rmsnorm_cpu); + + // topk + m.def("topk_sigmoid_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize) -> (Tensor, Tensor)"); + m.impl("topk_sigmoid_cpu", torch::kCPU, &topk_sigmoid_cpu); + m.def("topk_softmax_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize) -> (Tensor, Tensor)"); + m.impl("topk_softmax_cpu", torch::kCPU, &topk_softmax_cpu); + m.def( + "grouped_topk_cpu(Tensor hidden_states, Tensor gating_output, int topk, bool renormalize, int num_expert_group, " + "int topk_group, int num_fused_shared_experts, float? routed_scaling_factor, Tensor? num_token_non_padded) -> " + "(Tensor, Tensor)"); + m.impl("grouped_topk_cpu", torch::kCPU, &grouped_topk_cpu); + + // biased group topk + m.def( + "biased_grouped_topk_cpu(Tensor hidden_states, Tensor gating_output, Tensor correction_bias, int topk, bool " + "renormalize, int num_expert_group, int topk_group, int num_fused_shared_experts, float? routed_scaling_factor, " + "Tensor? num_token_non_padded) -> (Tensor, Tensor)"); + m.impl("biased_grouped_topk_cpu", torch::kCPU, &biased_grouped_topk_cpu); + + // decode + m.def( + "decode_attention_cpu(Tensor query, Tensor k_cache, Tensor v_cahce, Tensor(a!) output, Tensor key, Tensor value, " + "Tensor loc, Tensor attn_logits, Tensor req_to_token, Tensor req_pool_indices, Tensor seq_lens, float sm_scale, " + "float logit_cap) -> ()"); + m.impl("decode_attention_cpu", torch::kCPU, &decode_attention_cpu); + + // extend + m.def( + "extend_attention_cpu(Tensor q_extend, Tensor k_extend, Tensor v_extend, Tensor(a!) o_extend, Tensor k_buffer, " + "Tensor v_buffer, Tensor req_to_token, Tensor req_pool_indices, Tensor seq_lens, Tensor extend_seq_lens, Tensor " + "extend_start_loc, int max_len_extend, float sm_scale, float logit_cap) -> ()"); + m.impl("extend_attention_cpu", torch::kCPU, &extend_attention_cpu); + + // weight prepack + m.def("convert_weight_packed(Tensor weight) -> Tensor"); + m.impl("convert_weight_packed", torch::kCPU, &convert_weight_packed); + + // quant + m.def("per_token_quant_int8_cpu(Tensor A) -> (Tensor, Tensor)"); + m.impl("per_token_quant_int8_cpu", torch::kCPU, &per_token_quant_int8_cpu); + + // gemm + m.def("weight_packed_linear(Tensor mat1, Tensor mat2, Tensor? bias, bool is_vnni) -> Tensor"); + m.impl("weight_packed_linear", torch::kCPU, &weight_packed_linear); + + // igemm + m.def( + "int8_scaled_mm_cpu(Tensor mat1, Tensor mat2, Tensor scales1, Tensor scales2, Tensor? bias, ScalarType " + "out_dtype, bool is_vnni) -> Tensor"); + m.impl("int8_scaled_mm_cpu", torch::kCPU, &int8_scaled_mm_cpu); + + // fp8 gemm + m.def( + "fp8_scaled_mm_cpu(Tensor mat1, Tensor mat2, Tensor scales2, int[] block_size, Tensor? bias, ScalarType " + "out_dtype, bool is_vnni) -> Tensor"); + m.impl("fp8_scaled_mm_cpu", torch::kCPU, &fp8_scaled_mm_cpu); + + // quant + igemm + m.def( + "int8_scaled_mm_with_quant(Tensor mat1, Tensor mat2, Tensor scales2, Tensor? bias, ScalarType out_dtype, bool " + "is_vnni) -> Tensor"); + m.impl("int8_scaled_mm_with_quant", torch::kCPU, &int8_scaled_mm_with_quant); + + // bmm + m.def("bmm_cpu(Tensor(a!) out, Tensor mat1, Tensor mat2, bool is_vnni, Tensor? scale) -> ()"); + m.impl("bmm_cpu", torch::kCPU, &bmm_cpu); + + // moe + m.def( + "fused_experts_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor topk_weights, Tensor topk_ids, bool " + "inplace, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? w1_scale, Tensor? w2_scale, int[]? block_size, Tensor? " + "a1_scale, Tensor? a2_scale, bool " + "is_vnni) -> Tensor"); + m.impl("fused_experts_cpu", torch::kCPU, &fused_experts_cpu); + + // weight absorption + m.def( + "qkv_proj_with_rope(Tensor hidden_states, Tensor q_a_proj_weight, Tensor q_b_proj_weight, Tensor " + "kv_a_proj_weight, Tensor w_kc, Tensor q_a_layernorm_weight, Tensor kv_a_layernorm_weight, Tensor positions, " + "Tensor cos_sin_cache, float eps, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? q_a_proj_scale, Tensor? " + "q_b_proj_scale, Tensor? " + "kv_a_proj_scale, bool is_vnni, int[]? block_size) -> (Tensor, Tensor, Tensor)"); + m.impl("qkv_proj_with_rope", torch::kCPU, &qkv_proj_with_rope); + m.def( + "qkv_proj_with_rope_fused_weight(Tensor hidden_states, Tensor qkv_a_proj_weight, Tensor q_b_proj_weight, " + "Tensor w_kc, Tensor q_a_layernorm_weight, Tensor kv_a_layernorm_weight, Tensor positions, " + "Tensor cos_sin_cache, float eps, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? qkv_a_proj_scale, Tensor? " + "q_b_proj_scale," + "bool is_vnni, int[]? block_size, int q_lora_rank, int kv_lora_rank," + "int qk_rope_head_dim) -> (Tensor, Tensor, Tensor)"); + m.impl("qkv_proj_with_rope_fused_weight", torch::kCPU, &qkv_proj_with_rope_fused_weight); + + // shared expert + m.def( + "shared_expert_cpu(Tensor hidden_states, Tensor w1, Tensor w2, Tensor fused_experts_out, float " + "routed_scaling_factor, bool inplace, bool use_int8_w8a8, bool use_fp8_w8a16, Tensor? w1_scale, Tensor? " + "w2_scale, int[]? block_size, Tensor? a1_scale, Tensor? a2_scale, bool is_vnni) -> Tensor"); + m.impl("shared_expert_cpu", torch::kCPU, &shared_expert_cpu); + + // all reduce + m.def("initialize(int size, int rank) -> ()"); + m.def("shm_allreduce(Tensor(a!) data, int reduce_op) -> ()"); + m.impl("shm_allreduce", torch::kCPU, &shm_allreduce); + m.def("shm_allgather(Tensor data, int dim) -> Tensor"); + m.impl("shm_allgather", torch::kCPU, &shm_allgather); + + // rope + m.def( + "rotary_embedding_cpu(Tensor positions, Tensor query, Tensor key, int head_size, Tensor cos_sin_cache, " + "bool is_neox) -> (Tensor, Tensor)"); + m.impl("rotary_embedding_cpu", torch::kCPU, &rotary_embedding_cpu); + + // CPU and memory binding + m.def("init_cpu_threads_env(str cpu_ids) -> str"); +} + +TORCH_LIBRARY_IMPL(sgl_kernel, CatchAll, m) { + m.impl("init_cpu_threads_env", init_cpu_threads_env); + m.impl("initialize", &initialize); +} + +REGISTER_EXTENSION(common_ops) diff --git a/sgl-kernel/csrc/cpu/vec.h b/sgl-kernel/csrc/cpu/vec.h new file mode 100644 index 000000000..d28124c1d --- /dev/null +++ b/sgl-kernel/csrc/cpu/vec.h @@ -0,0 +1,308 @@ +#pragma once + +#if defined(__AVX512F__) && defined(__AVX512BF16__) && defined(__AMX_BF16__) +#define CPU_CAPABILITY_AVX512 +#endif + +#include +#include + +namespace { + +using namespace at::vec; + +template , int> = 0> +inline Vectorized convert_from_float_ext(const Vectorized& a, const Vectorized& b) { + return at::vec::convert_from_float(a, b); +} + +// allow f16, bf16 +template , int> = 1> +inline std::tuple, Vectorized> load_float_vec2(const scalar_t* __restrict__ data) { + using bVec = at::vec::Vectorized; + using fVec = at::vec::Vectorized; + bVec x_vec = bVec::loadu(data); + fVec x0, x1; + std::tie(x0, x1) = at::vec::convert_to_float(x_vec); + return std::make_tuple(x0, x1); +} + +// allow f32 +inline std::tuple, Vectorized> load_float_vec2(const float* __restrict__ data) { + using fVec = at::vec::Vectorized; + fVec x0 = fVec::loadu(data); + fVec x1 = fVec::loadu(data + fVec::size()); + return std::make_tuple(x0, x1); +} + +#if defined(CPU_CAPABILITY_AVX512) + +// `at::vec::convert_from_float<>` from PyTorch doesn't have avx512-bf16 intrinsics +// use native instruction for bfloat16->float32 conversion +template <> +inline Vectorized +convert_from_float_ext(const Vectorized& a, const Vectorized& b) { + return (__m512i)(_mm512_cvtne2ps_pbh(__m512(b), __m512(a))); +} + +#define CVT_BF16_TO_FP32(a) _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16)) + +#define CVT_FP16_TO_FP32(a) _mm512_cvtps_ph(a, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)) + +// this doesn't handle NaN. +inline __m512bh cvt_e4m3_bf16_intrinsic_no_nan(__m256i fp8_vec) { + const __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + + const __m512i mant = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x07)), 4); + const __m512i raw_exp = _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x78)), 3); + const __m512i exp = _mm512_slli_epi16(_mm512_add_epi16(raw_exp, _mm512_set1_epi16(120)), 7); + const __m512i nonsign = _mm512_or_si512(exp, mant); + + const __m512i sign = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(0x80)), 8); + const __m512i combined = _mm512_or_si512(nonsign, sign); + + const __mmask32 is_nonzero = _mm512_cmpneq_epi16_mask(x, _mm512_setzero_si512()); + return (__m512bh)_mm512_maskz_mov_epi16(is_nonzero, combined); +} + +inline __m512bh cvt_e4m3_bf16_intrinsic_without_denorm(__m256i fp8_vec) { + // The following conversion is without denorm behavior, that is to say, + // Max subnorm : S.0000.111 = 0.875 ∗ 2**(−6) + // Min subnorm : S.0000.001 = 2**(−9) + // 0.0019 ~ 0.0137 cannot be converted correctly. + __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + auto mask = _mm512_cmpneq_epi16_mask( + _mm512_and_si512(x, _mm512_set1_epi16(127)), + _mm512_setzero_si512()); // mask = x & 0x7f + auto mask_nan = _mm512_cmpneq_epi16_mask( + _mm512_and_si512(x, _mm512_set1_epi16(127)), + _mm512_set1_epi16(127)); // mask_nan = x & 0x7f + auto mantissa = _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(7)), 4); // mantissa = (x & 7) << 4 + auto exponent = _mm512_add_epi16( + _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(120)), 3), + _mm512_set1_epi16(120)); // exponent = (((x >> 3) & 15) + 120) + auto nonsign = _mm512_maskz_mov_epi16(mask, _mm512_or_si512(mantissa, _mm512_slli_epi16(exponent, 7))); + nonsign = _mm512_mask_mov_epi16(_mm512_set1_epi16(0x7fff), mask_nan, nonsign); // deal with Nan + return (__m512bh)(_mm512_or_si512( + nonsign, + _mm512_slli_epi16( + _mm512_and_si512(x, _mm512_set1_epi16(128)), + 8))); // add sign (x & 128) << 8 +} + +inline __m512bh cvt_e4m3_bf16_intrinsic_with_denorm(__m256i fp8_vec) { + __m512i x = _mm512_cvtepu8_epi16(fp8_vec); + __m512i lg2mant = _mm512_mask_mov_epi16( + _mm512_mask_mov_epi16( + _mm512_setzero_si512(), _mm512_test_epi16_mask(x, _mm512_set1_epi16(2)), _mm512_set1_epi16(1)), + _mm512_test_epi16_mask(x, _mm512_set1_epi16(4)), + _mm512_set1_epi16(2)); + return (__m512bh)(_mm512_or_si512( + _mm512_maskz_mov_epi16( + _mm512_cmpneq_epi16_mask(_mm512_and_si512(x, _mm512_set1_epi16(127)), _mm512_setzero_si512()), + _mm512_mask_blend_epi16( + _mm512_test_epi16_mask(x, _mm512_set1_epi16(120)), + _mm512_or_si512( + _mm512_and_si512( + _mm512_sllv_epi16( + _mm512_and_si512(x, _mm512_set1_epi16(3)), _mm512_sub_epi16(_mm512_set1_epi16(7), lg2mant)), + _mm512_set1_epi16(0x007f)), + _mm512_slli_epi16(_mm512_add_epi16(lg2mant, _mm512_set1_epi16(118)), 7)), + _mm512_or_si512( + _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(7)), 4), + _mm512_slli_epi16( + _mm512_add_epi16( + _mm512_srli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(120)), 3), _mm512_set1_epi16(120)), + 7)))), + _mm512_slli_epi16(_mm512_and_si512(x, _mm512_set1_epi16(128)), 8))); +} + +inline __m512bh CVT_FP8_TO_BF16(__m256i a) { +#ifdef SGLANG_CPU_FP8_CVT_FTZ + return cvt_e4m3_bf16_intrinsic_no_nan(a); +#else + return cvt_e4m3_bf16_intrinsic_with_denorm(a); +#endif +} + +#endif + +// vector to scalar reduction +#if defined(CPU_CAPABILITY_AVX512) && 0 +inline float vec_reduce_sum(const Vectorized& a) { + return _mm512_reduce_add_ps(__m512(a)); +} + +inline float vec_reduce_max(const Vectorized& a) { + return _mm512_reduce_max_ps(__m512(a)); +} +#else +inline float vec_reduce_sum(const Vectorized& a) { + return vec_reduce_all([](Vectorized& x, Vectorized& y) { return x + y; }, a); +} + +inline float vec_reduce_max(const Vectorized& a) { + return vec_reduce_all([](Vectorized& x, Vectorized& y) { return maximum(x, y); }, a); +} +#endif + +// https://github.com/InternLM/lmdeploy/blob/086481ed84b59bee3b8e4274e5fc69620040c048/lmdeploy/pytorch/kernels/cuda/w8a8_triton_kernels.py#L282 +template +inline void +quantize_row_int8(uint8_t* __restrict__ Aq, float& As, const scalar_t* __restrict__ A, int64_t K, float eps = 1e-7) { + float amax = 0.f; // absolute max + for (int64_t k = 0; k < K; ++k) { + const float val = static_cast(A[k]); + amax = std::max(amax, std::abs(val)); + } + + amax = std::max(amax, eps); + const float scale = amax / 127; + const float inv_scale = 127 / amax; + + for (int64_t k = 0; k < K; ++k) { + const float val = static_cast(A[k]) * inv_scale; + Aq[k] = (uint8_t)(std::round(val)) + 128; + } + As = scale; +} + +#if defined(CPU_CAPABILITY_AVX512) +template <> +inline void quantize_row_int8( + uint8_t* __restrict__ Aq, float& As, const at::BFloat16* __restrict__ A, int64_t K, float eps) { + const __m512 signBit = _mm512_set1_ps(-0.0f); + const __m512i off = _mm512_set1_epi32(128); + + // K is 32x, no remainder + float amax = 0.f; + __m512 vamax0 = _mm512_set1_ps(0.f); + __m512 vamax1 = _mm512_set1_ps(0.f); + for (int64_t k = 0; k < K; k += 32) { + __m512i va = _mm512_loadu_si512((void*)(A + k)); + __m512 va0 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 0)); + __m512 va1 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 1)); + vamax0 = _mm512_max_ps(vamax0, _mm512_andnot_ps(signBit, va0)); + vamax1 = _mm512_max_ps(vamax1, _mm512_andnot_ps(signBit, va1)); + } + amax = _mm512_reduce_max_ps(_mm512_max_ps(vamax0, vamax1)); + amax = std::max(amax, eps); + const float scale = amax / 127; + const float inv_scale = 127 / amax; + const __m512 vd = _mm512_set1_ps(inv_scale); + + for (int64_t k = 0; k < K; k += 32) { + __m512i va = _mm512_loadu_si512((void*)(A + k)); + __m512 va0 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 0)); + __m512 va1 = CVT_BF16_TO_FP32(_mm512_extracti32x8_epi32(va, 1)); + va0 = _mm512_mul_ps(va0, vd); + va1 = _mm512_mul_ps(va1, vd); + va0 = _mm512_roundscale_ps(va0, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); + va1 = _mm512_roundscale_ps(va1, (_MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)); + __m128i i0 = _mm512_cvtepi32_epi8(_mm512_add_epi32(_mm512_cvtps_epi32(va0), off)); + __m128i i1 = _mm512_cvtepi32_epi8(_mm512_add_epi32(_mm512_cvtps_epi32(va1), off)); + _mm256_storeu_si256(reinterpret_cast<__m256i*>(Aq + k), _mm256_set_m128i(i1, i0)); + } + As = scale; +} +#endif + +// transpose utils +// taken from my PR in ggml: https://github.com/ggml-org/llama.cpp/pull/8998 +#if defined(CPU_CAPABILITY_AVX512) +inline void transpose_16x16_32bit(__m512i* v) { + __m512i v1[16]; + v1[0] = _mm512_unpacklo_epi32(v[0], v[1]); + v1[1] = _mm512_unpackhi_epi32(v[0], v[1]); + v1[2] = _mm512_unpacklo_epi32(v[2], v[3]); + v1[3] = _mm512_unpackhi_epi32(v[2], v[3]); + v1[4] = _mm512_unpacklo_epi32(v[4], v[5]); + v1[5] = _mm512_unpackhi_epi32(v[4], v[5]); + v1[6] = _mm512_unpacklo_epi32(v[6], v[7]); + v1[7] = _mm512_unpackhi_epi32(v[6], v[7]); + v1[8] = _mm512_unpacklo_epi32(v[8], v[9]); + v1[9] = _mm512_unpackhi_epi32(v[8], v[9]); + v1[10] = _mm512_unpacklo_epi32(v[10], v[11]); + v1[11] = _mm512_unpackhi_epi32(v[10], v[11]); + v1[12] = _mm512_unpacklo_epi32(v[12], v[13]); + v1[13] = _mm512_unpackhi_epi32(v[12], v[13]); + v1[14] = _mm512_unpacklo_epi32(v[14], v[15]); + v1[15] = _mm512_unpackhi_epi32(v[14], v[15]); + + v[0] = _mm512_unpacklo_epi64(v1[0], v1[2]); + v[1] = _mm512_unpackhi_epi64(v1[0], v1[2]); + v[2] = _mm512_unpacklo_epi64(v1[1], v1[3]); + v[3] = _mm512_unpackhi_epi64(v1[1], v1[3]); + v[4] = _mm512_unpacklo_epi64(v1[4], v1[6]); + v[5] = _mm512_unpackhi_epi64(v1[4], v1[6]); + v[6] = _mm512_unpacklo_epi64(v1[5], v1[7]); + v[7] = _mm512_unpackhi_epi64(v1[5], v1[7]); + v[8] = _mm512_unpacklo_epi64(v1[8], v1[10]); + v[9] = _mm512_unpackhi_epi64(v1[8], v1[10]); + v[10] = _mm512_unpacklo_epi64(v1[9], v1[11]); + v[11] = _mm512_unpackhi_epi64(v1[9], v1[11]); + v[12] = _mm512_unpacklo_epi64(v1[12], v1[14]); + v[13] = _mm512_unpackhi_epi64(v1[12], v1[14]); + v[14] = _mm512_unpacklo_epi64(v1[13], v1[15]); + v[15] = _mm512_unpackhi_epi64(v1[13], v1[15]); + + v1[0] = _mm512_shuffle_i32x4(v[0], v[4], 0x88); + v1[1] = _mm512_shuffle_i32x4(v[1], v[5], 0x88); + v1[2] = _mm512_shuffle_i32x4(v[2], v[6], 0x88); + v1[3] = _mm512_shuffle_i32x4(v[3], v[7], 0x88); + v1[4] = _mm512_shuffle_i32x4(v[0], v[4], 0xdd); + v1[5] = _mm512_shuffle_i32x4(v[1], v[5], 0xdd); + v1[6] = _mm512_shuffle_i32x4(v[2], v[6], 0xdd); + v1[7] = _mm512_shuffle_i32x4(v[3], v[7], 0xdd); + v1[8] = _mm512_shuffle_i32x4(v[8], v[12], 0x88); + v1[9] = _mm512_shuffle_i32x4(v[9], v[13], 0x88); + v1[10] = _mm512_shuffle_i32x4(v[10], v[14], 0x88); + v1[11] = _mm512_shuffle_i32x4(v[11], v[15], 0x88); + v1[12] = _mm512_shuffle_i32x4(v[8], v[12], 0xdd); + v1[13] = _mm512_shuffle_i32x4(v[9], v[13], 0xdd); + v1[14] = _mm512_shuffle_i32x4(v[10], v[14], 0xdd); + v1[15] = _mm512_shuffle_i32x4(v[11], v[15], 0xdd); + + v[0] = _mm512_shuffle_i32x4(v1[0], v1[8], 0x88); + v[1] = _mm512_shuffle_i32x4(v1[1], v1[9], 0x88); + v[2] = _mm512_shuffle_i32x4(v1[2], v1[10], 0x88); + v[3] = _mm512_shuffle_i32x4(v1[3], v1[11], 0x88); + v[4] = _mm512_shuffle_i32x4(v1[4], v1[12], 0x88); + v[5] = _mm512_shuffle_i32x4(v1[5], v1[13], 0x88); + v[6] = _mm512_shuffle_i32x4(v1[6], v1[14], 0x88); + v[7] = _mm512_shuffle_i32x4(v1[7], v1[15], 0x88); + v[8] = _mm512_shuffle_i32x4(v1[0], v1[8], 0xdd); + v[9] = _mm512_shuffle_i32x4(v1[1], v1[9], 0xdd); + v[10] = _mm512_shuffle_i32x4(v1[2], v1[10], 0xdd); + v[11] = _mm512_shuffle_i32x4(v1[3], v1[11], 0xdd); + v[12] = _mm512_shuffle_i32x4(v1[4], v1[12], 0xdd); + v[13] = _mm512_shuffle_i32x4(v1[5], v1[13], 0xdd); + v[14] = _mm512_shuffle_i32x4(v1[6], v1[14], 0xdd); + v[15] = _mm512_shuffle_i32x4(v1[7], v1[15], 0xdd); +} + +// remove warning : ignoring attributes on template argument ‘__m512i’ [-Wignored-attributes] +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" + +// transpose from [2, 32] to [32, 2] +inline std::tuple<__m512i, __m512i> transpose_2x32_16bit(__m512i r0, __m512i r1) { + // r0: {a0, a1, ..., a31} + // r1: {b0, b1, ..., b31} + // + // d0: {a0, b0, ..., a15, b15} + // d1: {a16, b16, ..., a31, b31} + // + __m512i d0 = _mm512_unpacklo_epi16(r0, r1); + __m512i d1 = _mm512_unpackhi_epi16(r0, r1); + r0 = _mm512_shuffle_i32x4(d0, d1, 0x88); + r1 = _mm512_shuffle_i32x4(d0, d1, 0xdd); + d0 = _mm512_shuffle_i32x4(r0, r1, 0x88); + d1 = _mm512_shuffle_i32x4(r0, r1, 0xdd); + return std::make_tuple(d0, d1); +} +#pragma GCC diagnostic pop + +#endif + +} // anonymous namespace diff --git a/sgl-kernel/csrc/cutlass_extensions/common.hpp b/sgl-kernel/csrc/cutlass_extensions/common.hpp new file mode 100644 index 000000000..912ef8b16 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/common.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include "cuda_runtime.h" +#include "cutlass/cutlass.h" + +/** + * A wrapper for a kernel that is used to guard against compilation on + * architectures that will never use the kernel. The purpose of this is to + * reduce the size of the compiled binary. + * __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef + * into code that will be executed on the device where it is defined. + */ +template +struct enable_sm90_or_later : Kernel { + template + CUTLASS_DEVICE void operator()(Args&&... args) { +#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900 + Kernel::operator()(std::forward(args)...); +#endif + } +}; diff --git a/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp b/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp new file mode 100644 index 000000000..fcba4d40c --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/detail/collective/mixed_input_utils.hpp @@ -0,0 +1,482 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cute/arch/copy_sm90.hpp" +#include "cute/numeric/arithmetic_tuple.hpp" +#include "cute/util/type_traits.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/numeric_conversion.h" + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective::detail { + +template +struct MixedGroupedGemmInputUtils { + private: + using KernelSchedule = typename Collective::KernelSchedule; + using ConversionMode = typename Collective::ConversionMode; + using SmemLayoutA = typename Collective::SmemLayoutA; + using SmemLayoutB = typename Collective::SmemLayoutB; + using SmemLayoutScale = typename Collective::SmemLayoutScale; + using SwappedElementA = typename Collective::SwappedElementA; + using SwappedElementB = typename Collective::SwappedElementB; + using RealSwappedElementA = typename Collective::RealSwappedElementA; + using RealSwappedElementB = typename Collective::RealSwappedElementB; + using ElementScale = typename Collective::ElementScale; + using ElementZero = typename Collective::ElementZero; + using SmemCopyAtomScale = typename Collective::SmemCopyAtomScale; + static constexpr auto KernelConversionMode = Collective::KernelConversionMode; + static constexpr auto ModeHasScales = Collective::ModeHasScales; + static constexpr auto UseScaleLookupTable = Collective::UseScaleLookupTable; + + public: + static constexpr auto elements_per_smem_scale() { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return 0; + } else if constexpr (ModeHasScales) { + return cute::cosize_v; + } else { + static_assert(cutlass::detail::dependent_false, "Type not handled in scale smem allocation."); + } + } + + static constexpr auto elements_per_smem_zero() { + if constexpr ( + KernelConversionMode == ConversionMode::DirectConvert || + KernelConversionMode == ConversionMode::ConvertAndScale) { + return 0; + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + return cute::cosize_v; + } else { + static_assert(cutlass::detail::dependent_false, "Type not handled in scale smem allocation."); + } + } + + // These methods use some the public members of the class. For that reason, we define them after the public section. + static constexpr uint32_t compute_tma_transaction_bytes_mk() { + return cutlass::bits_to_bytes( + size<0>(SmemLayoutA{}) * size<1>(SmemLayoutA{}) * static_cast(cute::sizeof_bits_v)); + } + + static constexpr uint32_t compute_tma_transaction_bytes_nk() { + return cutlass::bits_to_bytes( + size<0>(SmemLayoutB{}) * size<1>(SmemLayoutB{}) * static_cast(cute::sizeof_bits_v)); + } + + static constexpr uint32_t compute_tma_transaction_bytes_extra() { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return 0; + } else if constexpr (ModeHasScales) { + constexpr uint32_t scale_tx_bytes = cutlass::bits_to_bytes( + size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * + static_cast(cute::sizeof_bits_v)); + static_assert(scale_tx_bytes % 128 == 0, "Each scale stage must be 128B aligned."); // required by TMA + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return scale_tx_bytes; + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // Scale and zero share smem layout + constexpr uint32_t zero_tx_bytes = cutlass::bits_to_bytes( + size<0>(SmemLayoutScale{}) * size<1>(SmemLayoutScale{}) * + static_cast(cute::sizeof_bits_v)); + static_assert(zero_tx_bytes % 128 == 0, "Each zero stage must be 128B aligned."); // required by TMA + return scale_tx_bytes + zero_tx_bytes; + } else { + static_assert( + cutlass::detail::dependent_false, "Type not handled in tma transaction bytes computation."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Type not handled in tma transaction bytes computation."); + } + } + + /// Utilities to copy A and extra inputs from smem to RF + template + CUTLASS_DEVICE static void copy_tensors_MK( + SmemTiledCopyA const& smem_tiled_copy_A, + TensorASmemView const& tCsA, + TensorACopyView& tCrA_copy_view, + cute::tuple const& partitioned_mma_extra_info, + cute::tuple const& tiled_copy_and_views, + int k_block, + int read_stage) { + copy(smem_tiled_copy_A, tCsA(_, _, k_block, read_stage), tCrA_copy_view(_, _, k_block)); + + if (k_block == 0) { + // We are starting a new k-tile so copy the scale + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + } else if constexpr (ModeHasScales) { + auto smem_tiled_copy_S = cute::get<0>(tiled_copy_and_views); + auto tCrS_copy_view = cute::get<1>(tiled_copy_and_views); + auto tCsS = cute::get<0>(partitioned_mma_extra_info); + copy(smem_tiled_copy_S, tCsS(_, _, k_block, read_stage), tCrS_copy_view(_, _, k_block)); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Nothing extra to do + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + auto tCsZ = cute::get<2>(partitioned_mma_extra_info); + auto tCrZ_copy_view = cute::get<2>(tiled_copy_and_views); + copy(smem_tiled_copy_S, tCsZ(_, _, k_block, read_stage), tCrZ_copy_view(_, _, k_block)); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } + } + + // The core converter uses a lookup table to converts i4 -> 8 bit value. + template < + class EngineIn, + class LayoutIn, + class EngineOut, + class LayoutOut, + class EngineScale, + class LayoutScale> + CUTLASS_DEVICE static void lookup_table_convert( // Accept mutable temporaries + Tensor const& src, + Tensor&& dst, + Tensor const& scales_neg, + Tensor const& scales_pos) { + lookup_table_convert(src, dst, scales_neg, scales_pos); + } + template + CUTLASS_DEVICE static void lookup_table_convert( + Tensor const& src, + Tensor& dst, + Tensor const& scales_neg, + Tensor const& scales_pos) { + constexpr int N = cute::cosize(LayoutIn{}); + static_assert(N == 4 || N == 8); + static_assert(cosize(LayoutScale{}) <= N / 4, "at least 4 consecutive weights must share the same scale."); + using SrcArray = cutlass::Array; + using DstArray = cutlass::Array; + using RegArray = cutlass::AlignedArray; + + // View the input as reg + auto&& src_reg = cute::recast(src)(0); + auto&& r = cute::recast(dst)(0); + + // Determines if to get from the signed or unsigned candidates + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + uint32_t sign; // ((reg & 0x88888888) | 0x64206420) >> 1 + asm volatile( + "{\n" + " lop3.b32 %0, %1, %2, %3, %4;\n" + "}\n" + : "=r"(sign) + : "r"(src_reg), "n"(0x88888888), "n"(0x64206420), "n"(immLut)); + sign = sign >> 1; + + // Ignore sign bit when indexing into LUT + uint32_t lut_idx = src_reg & 0x77777777; + Tensor scales_neg_ = cute::filter(scales_neg); + Tensor scales_pos_ = cute::filter(scales_pos); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < N / 4; ++i, lut_idx >>= 16, sign >>= 16) { + auto&& scale_neg_ = reinterpret_cast const&>(scales_neg_(i)); + auto&& scale_pos_ = reinterpret_cast const&>(scales_pos_(i)); + asm volatile( + "{\n" + " .reg .b32 pos, neg ;\n" + " prmt .b32 neg, %3, %4, %1 ;\n" + " prmt .b32 pos, %5, %6, %1 ;\n" + " prmt .b32 %0, pos, neg, %2 ;\n" + "}\n" + : "=r"(r[i]) + : "r"(lut_idx), "r"(sign), "r"(scale_neg_[0]), "r"(scale_neg_[1]), "r"(scale_pos_[0]), "r"(scale_pos_[1])); + } + } + + /// Utilities to dequantize A. + template + CUTLASS_DEVICE static void static_check_scale(Layout const& tensor) { + static_assert( + shape<0>(Layout{}) >= 4 && stride<0>(Layout{}) == 0, + "At least 4 adjacent weights in a thread must share the same scale."); + } + template + CUTLASS_DEVICE static void static_check_scale(Tensor const& tensor) { + static_check_scale(flatten(Layout{})); + } + + template + CUTLASS_DEVICE static void dequantize_A_kblock( + Tensor const& tCrA_load, + Tensor& tCrA_mma, + cute::tuple& partitioned_extra_info, + int const k_block) { + static_assert(is_rmem::value, "Input tensor for A conversion must come from registers"); + static_assert(is_rmem::value, "Output tensor for A conversion must come from registers"); + static_assert(cosize_v == cosize_v); + static_assert(size_v == cosize_v); + static_assert(size_v == cosize_v); + using SrcType = typename EngineIn::value_type; + using DstType = typename EngineOut::value_type; + + Tensor src = tCrA_load(_, _, k_block); + Tensor dst = tCrA_mma(_, _, k_block); + + CUTE_STATIC_ASSERT_V( + size(src(_, 0)) == cosize(src(_, 0).layout()), "The first mode of tensor src must be contiguous in memory"); + // try to make the size of the first mode equal to 32bit + int constexpr NumValPerSrcReg = cute::min(decltype(size(src(_, 0)))::value, ceil_div(32, sizeof_bits_v)); + Tensor src_vm = cute::group_modes<1, -1>(cute::zipped_divide(src, Int{})); + Tensor dst_vm = cute::group_modes<1, -1>(cute::zipped_divide(dst, Int{})); + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + } + } else if constexpr (UseScaleLookupTable) { + constexpr int num_elements = decltype(size(src))::value; + static_assert( + is_same_v, + "Lookup table only supports int4 being the quant type now."); + static_assert(sizeof_bits_v == 64, "Lookup table only supports 8 8bit scale values now."); + static_assert( + num_elements % 4 == 0 && num_elements >= 4, "Lookup table requires a vector size of 4x when converting."); + + Tensor tCrS_neg = cute::get<1>(partitioned_extra_info); + auto&& tCrS_pos = cute::get<2>(partitioned_extra_info); // modification to its value is needed + Tensor scales_neg = tCrS_neg(_, _, k_block); + Tensor scales_pos = tCrS_pos(_, _, k_block); + CUTE_STATIC_ASSERT_V(cute::size(src) == cute::size(scales_neg)); + + static_check_scale(scales_neg); + static_check_scale(scales_pos); + Tensor scales_neg_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales_neg, Int{})); + Tensor scales_pos_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales_pos, Int{})); + + if (k_block == 0) { + Tensor scales_neg_vm_ = filter(scales_neg_vm); + Tensor scales_pos_vm_ = filter(scales_pos_vm); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size(scales_neg_vm_.layout()); ++i) { + auto&& scale_neg_ = reinterpret_cast const&>(scales_neg_vm_(i)); + auto&& scale_pos_ = reinterpret_cast&>(scales_pos_vm_(i)); + constexpr uint32_t immLut = (0xf0 & 0xcc) ^ 0xaa; + asm volatile( + "{\n" + " lop3 .b32 %0, %2, %4, %5, %6;\n" + " xor .b32 %1, %3, %5; \n" + "}\n" + : "=r"(scale_pos_[0]), "=r"(scale_pos_[1]) + : "r"(scale_neg_[0]), "r"(scale_neg_[1]), "n"(0xFFFFFF00), "n"(0x80808080), "n"(immLut)); + } + } + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + lookup_table_convert(src_vm(_, i), dst_vm(_, i), scales_neg_vm(_, i), scales_pos_vm(_, i)); + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + Tensor scales = cute::get<1>(partitioned_extra_info)(_, _, k_block); + CUTE_STATIC_ASSERT_V(size(src) == size(scales)); + Tensor scales_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales, Int{})); + + if constexpr (is_same_v) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + dst_vm(j, i) *= scales_vm(j, i); + } + } + } else { + auto stage = make_tensor_like(src_vm(_, 0)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), stage); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + stage(j) *= scales_vm(j, i); + } + LayoutAwareConvert(stage, dst_vm(_, i)); + } + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + static_assert(is_same_v, "ElementScale and ElementZero must be the same."); + Tensor scales = cute::get<1>(partitioned_extra_info)(_, _, k_block); + Tensor zeros = cute::get<3>(partitioned_extra_info)(_, _, k_block); + CUTE_STATIC_ASSERT_V(size(src) == size(scales)); + CUTE_STATIC_ASSERT_V(size(src) == size(zeros)); + Tensor scales_vm = cute::group_modes<1, -1>(cute::zipped_divide(scales, Int{})); + Tensor zeros_vm = cute::group_modes<1, -1>(cute::zipped_divide(zeros, Int{})); + + if constexpr (is_same_v) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + dst_vm(j, i) = dst_vm(j, i) * scales_vm(j, i) + zeros_vm(j, i); + } + } + } else { + auto stage = make_tensor_like(src_vm(_, 0)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), stage); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < size<0>(dst_vm); ++j) { + stage(j) = stage(j) * scales_vm(j, i) + zeros_vm(j, i); + } + LayoutAwareConvert(stage, dst_vm(_, i)); + } + } + } else { + static_assert(cutlass::detail::dependent_false, "No A data is loaded."); + } + } + + template + CUTLASS_DEVICE static void convert_A_kblock( + Tensor const& tCrA_load, Tensor& tCrA_mma, int const k_block) { + static_assert(is_rmem::value, "Input tensor for A conversion must come from registers"); + static_assert(is_rmem::value, "Output tensor for A conversion must come from registers"); + static_assert(cosize_v == cosize_v); + static_assert(size_v == cosize_v); + static_assert(size_v == cosize_v); + using SrcType = typename EngineIn::value_type; + + Tensor src = tCrA_load(_, _, k_block); + Tensor dst = tCrA_mma(_, _, k_block); + + CUTE_STATIC_ASSERT_V( + size(src(_, 0)) == cosize(src(_, 0).layout()), "The first mode of tensor src must be contiguous in memory"); + // try to make the size of the first mode equal to 32bit + int constexpr NumValPerSrcReg = cute::min(decltype(size(src(_, 0)))::value, ceil_div(32, sizeof_bits_v)); + Tensor src_vm = cute::group_modes<1, -1>(cute::zipped_divide(src, Int{})); + Tensor dst_vm = cute::group_modes<1, -1>(cute::zipped_divide(dst, Int{})); + + // KernelConversionMode == ConversionMode::DirectConvert + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < size<1>(dst_vm); ++i) { + LayoutAwareConvert(src_vm(_, i), dst_vm(_, i)); + } + } + + /// Utilities for any additional inputs inside of the TMA load + template + CUTLASS_DEVICE static auto partition_extra_tma_inputs( + Params const& mainloop_params, + cute::tuple const& load_inputs, + TensorStorage& shared_tensors, + uint2 const& cluster_local_block_id, + int const m_coord, + int const l_coord) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(); + } else if constexpr (ModeHasScales) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE) + Tensor gS_mkl = get<2>(load_inputs); + auto block_tma_s = mainloop_params.tma_load_scale.get_slice(cluster_local_block_id.y); + Tensor gS = gS_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + + Tensor tSgS = block_tma_s.partition_S(gS); // (TMA,TMA_M,TMA_K,k) + Tensor tSsS = block_tma_s.partition_D(sS); // (TMA,TMA_M,TMA_K,PIPE) + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tSgS, tSsS); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor sZ = + make_tensor(make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{}); // (BLK_M,BLK_K,PIPE) + Tensor gZ_mkl = get<3>(load_inputs); + auto block_tma_z = mainloop_params.tma_load_zero.get_slice(cluster_local_block_id.y); + Tensor gZ = gZ_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + + Tensor tZgZ = block_tma_z.partition_S(gZ); // (TMA,TMA_M,TMA_K,k) + Tensor tZsZ = block_tma_z.partition_D(sZ); // (TMA,TMA_M,TMA_K,PIPE) + return cute::make_tuple(tSgS, tSsS, tZgZ, tZsZ); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for input partitioning."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for input partitioning."); + } + } + + /// Utilities for partitioning extra inputs for loading from smem in the mainloop. + template + CUTLASS_DEVICE static auto + partition_extra_mma_info(ThreadMma const& mma_thread_slice, TensorStorage& shared_tensors) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + return cute::make_tuple(); + } else if constexpr (UseScaleLookupTable) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsS = mma_thread_slice.partition_A(sS); + Tensor tCrS = make_tensor(mma_thread_slice.partition_fragment_A(sS(_, _, Int<0>{})).layout()); + + return cute::make_tuple(tCsS, tCrS); + } else if constexpr (ModeHasScales) { + Tensor sS = + make_tensor(make_smem_ptr(shared_tensors.smem_scale.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsS = mma_thread_slice.partition_A(sS); + Tensor tCrS = make_tensor(mma_thread_slice.partition_fragment_A(sS(_, _, Int<0>{})).layout()); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tCsS, tCrS); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor sZ = make_tensor( + make_smem_ptr(shared_tensors.smem_zero.begin()), SmemLayoutScale{}); // (BLK_M,BLK_SCALE_K,PIPE) + Tensor tCsZ = mma_thread_slice.partition_A(sZ); + Tensor tCrZ = make_tensor(mma_thread_slice.partition_fragment_A(sZ(_, _, Int<0>{})).layout()); + return cute::make_tuple(tCsS, tCrS, tCsZ, tCrZ); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } + + /// Returns the tiled copy and copy views for the extra inputs. + template + CUTLASS_DEVICE static auto retile_extra_mma_info( + TiledMma const& tiled_mma, cute::tuple& partitioned_extra_info, int const warp_group_thread_idx) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // nothing to do + return cute::make_tuple(); + } else if constexpr (ModeHasScales) { + auto smem_tiled_copy_S = make_tiled_copy_A(SmemCopyAtomScale{}, tiled_mma); + auto smem_thr_copy_S = smem_tiled_copy_S.get_thread_slice(warp_group_thread_idx); + Tensor tCrS_copy_view = smem_thr_copy_S.retile_D(cute::get<1>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K) + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor tCrZ_copy_view = smem_thr_copy_S.retile_D(cute::get<3>(partitioned_extra_info)); // (CPY,CPY_M,CPY_K) + return cute::make_tuple(smem_tiled_copy_S, tCrS_copy_view, tCrZ_copy_view); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in A -> RF path."); + } + } +}; + +} // namespace cutlass::gemm::collective::detail diff --git a/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h b/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h new file mode 100644 index 000000000..9f85bee28 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h @@ -0,0 +1,309 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/epilogue/threadblock/epilogue_per_row_per_col_scale.h + +#pragma once + +#include +#include + +namespace cutlass { +namespace epilogue { +namespace threadblock { + +template < + typename ThreadblockShape_, + int ThreadCount, + typename ScaleTileIterator_, + typename OutputTileIterator_, + typename ElementAccumulator_, + typename ElementCompute_, + typename ElementwiseFunctor_, + bool UseMasking_ = false> +class EpilogueVisitorPerRowPerCol { + public: + using ThreadblockShape = ThreadblockShape_; + static int const kThreadCount = ThreadCount; + + using ScaleTileIterator = ScaleTileIterator_; + using OutputTileIterator = OutputTileIterator_; + using ElementwiseFunctor = ElementwiseFunctor_; + + static int const kIterations = OutputTileIterator::kIterations; + static int const kElementsPerAccess = OutputTileIterator::kElementsPerAccess; + + using ElementOutput = typename OutputTileIterator::Element; + using LayoutOutput = cutlass::layout::RowMajor; + using ElementAccumulator = ElementAccumulator_; + + using AlphaScaleElementType = typename ScaleTileIterator::Element; + + using ElementCompute = ElementCompute_; + using AccumulatorFragment = Array; + using ComputeFragment = Array; + using OutputVector = Array; + + static int const kThreadsPerRow = OutputTileIterator::ThreadMap::Detail::kAccessWidth; + static bool const kHasMultiStepsInRow = (OutputTileIterator::ThreadMap::Iterations::kColumn > 1); + + /// Argument structure + struct Arguments { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + Arguments() : batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments(typename ElementwiseFunctor::Params elementwise_) + : elementwise(elementwise_), batch_stride_alpha(0), batch_stride_C(0), batch_stride_D(0) {} + + Arguments( + typename ElementwiseFunctor::Params elementwise_, + int64_t batch_stride_alpha_, + int64_t batch_stride_C_, + int64_t batch_stride_D_) + : elementwise(elementwise_), + batch_stride_alpha(batch_stride_alpha_), + batch_stride_C(batch_stride_C_), + batch_stride_D(batch_stride_D_) {} + }; + + struct Params { + typename ElementwiseFunctor::Params elementwise; + int64_t batch_stride_alpha; + int64_t batch_stride_C; + int64_t batch_stride_D; + + // + // Methods + // + CUTLASS_HOST_DEVICE + Params() {} + + CUTLASS_HOST_DEVICE + Params(Arguments const& args) + : elementwise(args.elementwise), + batch_stride_alpha(args.batch_stride_alpha), + batch_stride_C(args.batch_stride_C), + batch_stride_D(args.batch_stride_D) {} + }; + + /// Shared storage + struct SharedStorage {}; + + private: + Params const& params_; + SharedStorage& shared_storage_; + MatrixCoord extent_; + MatrixCoord extent_real_; + ElementwiseFunctor elementwise_; + + bool const with_bias_; + bool const per_token_quant_; + bool const per_channel_quant_; + + AlphaScaleElementType* ptr_alpha_row_; + AlphaScaleElementType* ptr_alpha_col_; + ScaleTileIterator iterator_alpha_col_; + OutputTileIterator iterator_C_; + OutputTileIterator iterator_D_; + + AlphaScaleElementType element_alpha_row_ = 1.0f; + AlphaScaleElementType element_alpha_col_ = 1.0f; + typename ScaleTileIterator::Fragment fragment_alpha_col_; + typename OutputTileIterator::Fragment fragment_C_; + typename OutputTileIterator::Fragment fragment_D_; + + ElementAccumulator beta_; + + int column_offset_; + + MatrixCoord thread_offset_; + + public: + CUTLASS_DEVICE + EpilogueVisitorPerRowPerCol( + Params const& params, + SharedStorage& shared_storage, + cutlass::MatrixCoord const& problem_size, + int thread_idx, + int warp_idx, + int lane_idx, + typename ScaleTileIterator::Params params_alpha_col, + typename OutputTileIterator::Params params_C, + typename OutputTileIterator::Params params_D, + bool with_bias, + bool per_token_quant, + bool per_channel_quant, + AlphaScaleElementType* ptr_alpha_row, + AlphaScaleElementType* ptr_alpha_col, + typename OutputTileIterator::Element* ptr_C, + typename OutputTileIterator::Element* ptr_D, + cutlass::MatrixCoord const& threadblock_offset = cutlass::MatrixCoord(0, 0), + int column_offset = 0, + cutlass::MatrixCoord const& problem_size_real = cutlass::MatrixCoord(0, 0)) + : params_(params), + shared_storage_(shared_storage), + extent_(problem_size), + elementwise_(params.elementwise), + with_bias_(with_bias), + per_token_quant_(per_token_quant), + per_channel_quant_(per_channel_quant), + ptr_alpha_row_(ptr_alpha_row), + ptr_alpha_col_(ptr_alpha_col), + iterator_alpha_col_(params_alpha_col, ptr_alpha_col, problem_size, thread_idx, threadblock_offset), + iterator_C_(params_C, ptr_C, problem_size, thread_idx, threadblock_offset), + iterator_D_(params_D, ptr_D, problem_size, thread_idx, threadblock_offset), + extent_real_(problem_size_real) { + if (!per_channel_quant_ && (ptr_alpha_col_ != nullptr)) { + element_alpha_col_ = *ptr_alpha_col_; + } + + if (!per_token_quant_ && (ptr_alpha_row_ != nullptr)) { + element_alpha_row_ = *ptr_alpha_row_; + } + } + + /// Helper to indicate split-K behavior + CUTLASS_DEVICE + void set_k_partition( + int split_k_index, ///< Index of this threadblock within split-K partitioned scheme + int split_k_slices) { ///< Total number of split-K slices + } + + /// Called to set the batch index + CUTLASS_DEVICE + void set_batch_index(int batch_idx) { + iterator_alpha_col_.add_pointer_offset(batch_idx * params_.batch_stride_alpha); + iterator_C_.add_pointer_offset(batch_idx * params_.batch_stride_C); + iterator_D_.add_pointer_offset(batch_idx * params_.batch_stride_D); + } + + /// Called at the start of the epilogue just before iterating over accumulator slices + CUTLASS_DEVICE + void begin_epilogue() { + if (per_channel_quant_) { + iterator_alpha_col_.load(fragment_alpha_col_); + } + + if (with_bias_) { + iterator_C_.load(fragment_C_); + } + } + + /// Called at the start of one step before starting accumulator exchange + CUTLASS_DEVICE + void begin_step(int step_idx) { + fragment_D_.clear(); + } + + /// Called at the start of a row + CUTLASS_DEVICE + void begin_row(int row_idx) { + // load alpha_row in begin_step only when per token(row) scaling is used + if (per_token_quant_) { + int thread_offset_row = + iterator_D_.thread_start_row() + OutputTileIterator::ThreadMap::iteration_offset(row_idx).row(); + + arch::global_load( + element_alpha_row_, ptr_alpha_row_ + thread_offset_row, thread_offset_row < extent_.row()); + } + } + + /// Called after accumulators have been exchanged for each accumulator vector + CUTLASS_DEVICE + void visit(int iter_idx, int row_idx, int column_idx, int frag_idx, AccumulatorFragment const& accum) { + NumericArrayConverter source_converter; + + ComputeFragment result = source_converter(accum); + if (per_channel_quant_) { + ComputeFragment alpha_col = reinterpret_cast(&fragment_alpha_col_)[column_idx]; + result = per_token_channel_scale_accumulator_(result, alpha_col, element_alpha_row_); + } else { + result = per_token_scale_accumulator_(result, element_alpha_col_, element_alpha_row_); + } + + if (with_bias_) { + NumericArrayConverter bias_converter; + OutputVector bias = reinterpret_cast(&fragment_C_)[column_idx]; + result = bias_accumulator_(result, bias_converter(bias)); + } + + // Convert to the output + NumericArrayConverter output_converter; + OutputVector& output = reinterpret_cast(&fragment_D_)[frag_idx]; + output = output_converter(result); + } + + /// Called at the end of a row + CUTLASS_DEVICE + void end_row(int row_idx) {} + + /// Called after all accumulator elements have been visited + CUTLASS_DEVICE + void end_step(int step_idx) { + iterator_D_.store(fragment_D_); + ++iterator_D_; + } + + /// Called after all steps have been completed + CUTLASS_DEVICE + void end_epilogue() {} + + private: + CUTLASS_DEVICE + ComputeFragment per_token_channel_scale_accumulator_( + ComputeFragment const& accum, ComputeFragment const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col[i] * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment per_token_scale_accumulator_( + ComputeFragment const& accum, AlphaScaleElementType const& scale_col, AlphaScaleElementType const& scale_row) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < ComputeFragment::kElements; ++i) { + result[i] = accum[i] * (scale_col * scale_row); + } + + return result; + } + + CUTLASS_DEVICE + ComputeFragment bias_accumulator_(ComputeFragment const& accum, ComputeFragment const& bias) { + ComputeFragment result; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < OutputVector::kElements; ++i) { + result[i] = accum[i] + bias[i]; + } + return result; + } +}; + +} // namespace threadblock +} // namespace epilogue +} // namespace cutlass diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl new file mode 100644 index 000000000..db1fdf1e7 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cute/arch/cluster_sm90.hpp" +#include "cute/tensor.hpp" +#include "cutlass/gemm/collective/builders/sm90_common.inl" +#include "cutlass/gemm/collective/collective_builder_decl.hpp" +#include "cutlass/gemm/collective/collective_mma_decl.hpp" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/pipeline/sm90_pipeline.hpp" + +// SM90 Collective Builders should be used only starting CUDA 12.0 +#if (__CUDACC_VER_MAJOR__ >= 12) +#define CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED +#endif + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// GMMA_TMA_WS_RS +template < + class ElementA_, + class GmemLayoutATag_, + int AlignmentA, + class ElementB_, + class GmemLayoutBTag_, + int AlignmentB, + class ElementAccumulator, + class TileShape_MNK, + class ClusterShape_MNK, + class StageCountType, + class KernelScheduleType> +struct CollectiveBuilderMixedInput< + arch::Sm90, + arch::OpClassTensorOp, + ElementA_, + GmemLayoutATag_, + AlignmentA, + ElementB_, + GmemLayoutBTag_, + AlignmentB, + ElementAccumulator, + TileShape_MNK, + ClusterShape_MNK, + StageCountType, + KernelScheduleType, + cute::enable_if_t< + (cute::is_same_v || + cute::is_same_v || + cute::is_same_v || + cute::is_same_v || + cute::is_same_v) && + (detail::is_use_rmem_A() || + // ConvertAndScale and ConvertAndScaleWithZero + cute::is_tuple::value || cute::is_tuple::value || + // DirectConvert + sizeof_bits::value != sizeof_bits::value)>> { + private: + using ScaleA = detail::deduce_mixed_width_dtype_t<1, ElementA_>; + using ScaleB = detail::deduce_mixed_width_dtype_t<1, ElementB_>; + using ZeroA = detail::deduce_mixed_width_dtype_t<2, ElementA_>; + using ZeroB = detail::deduce_mixed_width_dtype_t<2, ElementB_>; + static constexpr bool NeitherIsTuple = !cute::is_tuple::value && !cute::is_tuple::value; + // Determine if mixed input types. + static constexpr bool IsMixedInput = cute::sizeof_bits_v> != + cute::sizeof_bits_v>; + static constexpr bool IsArrayOfPointersGemm = cute::is_any_of_v< + KernelScheduleType, + KernelPtrArrayTmaWarpSpecializedCooperative, + KernelPtrArrayTmaWarpSpecializedPingpong>; + static_assert(IsMixedInput || !IsArrayOfPointersGemm, "Only mixed input grouped RS GEMM is supported."); + + public: + using ElementA = detail::deduce_mixed_width_dtype_t<0, ElementA_>; + using ElementB = detail::deduce_mixed_width_dtype_t<0, ElementB_>; + + static_assert( + !IsMixedInput || (cute::is_tuple::value ^ cute::is_tuple::value || + (NeitherIsTuple && (sizeof_bits::value != sizeof_bits::value))), + "Either A OR B must be a tuple or the widths of A and B must be different."); + + static constexpr bool IsANarrow = sizeof_bits::value < sizeof_bits::value; + + template + static auto get_stride(T const& t) { + if constexpr (not cute::is_layout>::value) { + return t; + } else { + if constexpr (cute::is_pointer_v) { + return &cute::stride(*t); + } else { + return cute::stride(t); + } + } + } + + using GmemLayoutATag = decltype(get_stride(GmemLayoutATag_{})); + using GmemLayoutBTag = decltype(get_stride(GmemLayoutBTag_{})); + + using ElementPairA = + cute::conditional_t, ElementA_>; + using ElementPairB = + cute::conditional_t, ElementB_>; + + static constexpr bool IsATransformed = cute::is_tuple::value; + using ElementScale = cute::conditional_t; + using ElementZero = cute::conditional_t; + + static_assert(is_static::value); + static_assert(is_static::value); + static_assert( + detail::is_aligned(), + "Should meet TMA alignment requirement\n"); +#ifndef CUTLASS_SM90_COLLECTIVE_BUILDER_SUPPORTED + static_assert(cutlass::detail::dependent_false, "Unsupported Toolkit for SM90 Collective Builder\n"); +#endif + static constexpr cute::GMMA::Major GmmaMajorA = detail::gmma_rs_tag_to_major_A(); + static constexpr cute::GMMA::Major GmmaMajorB = detail::gmma_rs_tag_to_major_B(); + // If A is scaled, then we don't need to swap. Otherwise, we must ensure B goes to rmem and we must swap the + // operands. + static constexpr bool SwapAB = + IsMixedInput ? !IsATransformed : detail::is_swapAB(); + static constexpr bool IsWarpSpecializedTransposeB = + detail::is_warpspecialized_transpose_B(); + static_assert(!IsMixedInput || !IsWarpSpecializedTransposeB, "Mixed input GEMM does not support WS transpose B."); + + // When we relax the above assertion, we must handle setting the tile mma GmmaMajorB correctly. + static constexpr cute::GMMA::Major TiledMmaGmmaMajorB = SwapAB ? GmmaMajorA : GmmaMajorB; + + // For fp32 types, map to tf32 MMA value type. + using ElementAMma = cute::conditional_t, tfloat32_t, ElementA>; + using ElementBMma = cute::conditional_t, tfloat32_t, ElementB>; + + // Handle mixed dtypes and MMA. + using RealElementA = cute::conditional_t; + using RealElementB = cute::conditional_t; + using RealElementAMma = cute::conditional_t; + // Always the same for element B. + using RealElementBMma = RealElementB; + + static_assert( + !IsMixedInput || TiledMmaGmmaMajorB == GMMA::Major::K || sizeof_bits::value == 16, + "Mixed input GEMM does not support MN major layout except for 16bit"); + + using AtomLayoutMNK = cute::conditional_t< + cute::is_any_of_v< + KernelScheduleType, + KernelTmaWarpSpecializedCooperative, + KernelPtrArrayTmaWarpSpecializedCooperative>, + Layout>, + Layout>>; + + using TiledMma = decltype(cute::make_tiled_mma( + cute::GMMA::rs_op_selector< + RealElementAMma, + RealElementBMma, + ElementAccumulator, + TileShape_MNK, + GMMA::Major::K, + GMMA::Major::K>(), + AtomLayoutMNK{})); + + using GmemTiledCopyA = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<1>(ClusterShape_MNK{}))); + using GmemTiledCopyB = decltype(detail::sm90_cluster_shape_to_tma_atom(shape<0>(ClusterShape_MNK{}))); + + using SmemLayoutAtomA = decltype(detail::rs_smem_selector< + GmmaMajorA, + ElementAMma, + decltype(cute::get<0>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{})), + IsWarpSpecializedTransposeB>()); + using SmemLayoutAtomB = decltype(detail::rs_smem_selector< + GmmaMajorB, + ElementBMma, + decltype(cute::get<1>(TileShape_MNK{})), + decltype(cute::get<2>(TileShape_MNK{})), + IsWarpSpecializedTransposeB>()); + + static constexpr size_t SmemAlignmentA = cutlass::detail::alignment_for_swizzle(SmemLayoutAtomA{}); + static constexpr size_t SmemAlignmentB = cutlass::detail::alignment_for_swizzle(SmemLayoutAtomB{}); + static constexpr int SmemAlignment = static_cast(cute::max(SmemAlignmentA, SmemAlignmentB)); + + // Handle mixed dtype array GEMM's size of tensor map storage. + static constexpr size_t TensorMapStorage = sizeof(cute::TmaDescriptor) * size_t(IsMixedInput) * 4; + static constexpr int KernelSmemCarveout = static_cast(TensorMapStorage); + static constexpr int Sm90ReducedSmemCapacityBytes = detail::sm90_smem_capacity_bytes - KernelSmemCarveout; + + static constexpr int PipelineStages = + IsMixedInput ? (IsArrayOfPointersGemm ? detail::compute_stage_count_or_override_single_affine_transformed_input< + Sm90ReducedSmemCapacityBytes, + RealElementA, + RealElementB, + ElementScale, + ElementZero, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{}) + : detail::compute_stage_count_or_override_single_affine_transformed_input< + detail::sm90_smem_capacity_bytes, + RealElementA, + RealElementB, + ElementScale, + ElementZero, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{})) + : detail::compute_stage_count_or_override< + detail::sm90_smem_capacity_bytes, + ElementAMma, + ElementBMma, + TileShape_MNK, + StageCountType::bytes, + SmemAlignment>(StageCountType{}); + + using DispatchPolicy = cute::conditional_t< + IsMixedInput, + cute::conditional_t< + IsArrayOfPointersGemm, + MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput, + MainloopSm90TmaGmmaRmemAWarpSpecializedMixedInput>, + MainloopSm90TmaGmmaRmemAWarpSpecialized>; + + using SmemCopyAtomA = cute::conditional_t>; + using SmemCopyAtomB = cute::conditional_t, void>; + + // We pack the scale data with the operand that will be optionally scaled and converted before MMA. + using StrideA = cute::conditional_t< + cute::is_layout>::value, + GmemLayoutATag_, + TagToStrideA_t>; + using StrideB = cute::conditional_t< + cute::is_layout>::value, + GmemLayoutBTag_, + TagToStrideB_t>; + + using CollectiveOp = CollectiveMmaArrayMixedInput< + DispatchPolicy, + TileShape_MNK, + ElementPairA, + StrideA, + ElementPairB, + StrideB, + TiledMma, + GmemTiledCopyA, + SmemLayoutAtomA, + SmemCopyAtomA, + cute::identity, + GmemTiledCopyB, + SmemLayoutAtomB, + SmemCopyAtomB, + cute::identity>; + + static_assert( + SmemAlignment == static_cast(cute::max(CollectiveOp::SmemAlignmentA, CollectiveOp::SmemAlignmentB))); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp new file mode 100644 index 000000000..8288ace95 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +///////////////////////////////////////////////////////////////////////////////////////////////// +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp" + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + class ArchTag, + class OpClass, + class ElementA, + class GmemLayoutA, + int AlignmentA, + class ElementB, + class GmemLayoutB, + int AlignmentB, + class ElementAccumulator, + class TileShape_MNK, + class ClusterShape_MNK, + class StageCountType, + class KernelScheduleType, + class Enable = void> +struct CollectiveBuilderMixedInput { + static_assert(sizeof(ElementA) == 0, "Could not build a collective for given parameters."); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// + +#include "cutlass_extensions/gemm/collective/builders/sm90_gmma_builder_mixed_input.inl" +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp new file mode 100644 index 000000000..08afdffd4 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/collective_mma_array_mixed_input.hpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cutlass/detail/dependent_false.hpp" + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + class DispatchPolicy, + class TileShape, + class ElementA, + class StrideA, + class ElementB, + class StrideB, + class TiledMma, + class GmemTiledCopyA, + class SmemLayoutAtomA, + class SmemCopyAtomA, + class TransformA, + class GmemTiledCopyB, + class SmemLayoutAtomB, + class SmemCopyAtomB, + class TransformB> +struct CollectiveMmaArrayMixedInput { + static_assert(cutlass::detail::dependent_false, "Could not find a mainloop specialization."); +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// + +#include "cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp" +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp new file mode 100644 index 000000000..b37d5696c --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/collective/sm90_mma_array_tma_gmma_rs_warpspecialized_mixed_input_.hpp @@ -0,0 +1,1538 @@ +/*************************************************************************************************** + * Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + **************************************************************************************************/ +#pragma once + +#include "cute/algorithm/functional.hpp" +#include "cute/algorithm/gemm.hpp" +#include "cute/arch/cluster_sm90.hpp" +#include "cute/arch/copy_sm90.hpp" +#include "cute/atom/mma_atom.hpp" +#include "cute/numeric/arithmetic_tuple.hpp" +#include "cutlass/cuda_host_adapter.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/numeric_types.h" +#include "cutlass/pipeline/pipeline.hpp" +#include "cutlass/trace.h" +#include "cutlass_extensions/detail/collective/mixed_input_utils.hpp" + +#define GROUP_SIZE 128 + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass::gemm::collective { +using namespace cute; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +// WarpSpecialized Mainloop +template < + int Stages, + class ClusterShape, + class KernelSchedule_, + class TileShape_, + class ElementAOptionalTuple, + class StrideA_, + class ElementBOptionalTuple, + class StrideB_, + class TiledMma_, + class GmemTiledCopyA_, + class SmemLayoutAtomA_, + class SmemCopyAtomA_, + class TransformA_, + class GmemTiledCopyB_, + class SmemLayoutAtomB_, + class SmemCopyAtomB_, + class TransformB_> +struct CollectiveMmaArrayMixedInput< + MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput, + TileShape_, + ElementAOptionalTuple, + StrideA_, + ElementBOptionalTuple, + StrideB_, + TiledMma_, + GmemTiledCopyA_, + SmemLayoutAtomA_, + SmemCopyAtomA_, + TransformA_, + GmemTiledCopyB_, + SmemLayoutAtomB_, + SmemCopyAtomB_, + TransformB_> { + public: + enum class ConversionMode { DirectConvert, ConvertAndScale, ConvertAndScaleWithZero }; + + // + // Type Aliases + // + using DispatchPolicy = MainloopSm90ArrayTmaGmmaWarpSpecializedMixedInput; + using TileShape = TileShape_; + using KernelSchedule = KernelSchedule_; + + private: + template + friend struct detail::MixedGroupedGemmInputUtils; + using CollectiveType = CollectiveMma< + DispatchPolicy, + TileShape_, + ElementAOptionalTuple, + StrideA_, + ElementBOptionalTuple, + StrideB_, + TiledMma_, + GmemTiledCopyA_, + SmemLayoutAtomA_, + SmemCopyAtomA_, + TransformA_, + GmemTiledCopyB_, + SmemLayoutAtomB_, + SmemCopyAtomB_, + TransformB_>; + using Utils = detail::MixedGroupedGemmInputUtils; + + // + // Type Aliases + // + using ScaleA = detail::deduce_mixed_width_dtype_t<1, ElementAOptionalTuple>; + using ScaleB = detail::deduce_mixed_width_dtype_t<1, ElementBOptionalTuple>; + using ZeroA = detail::deduce_mixed_width_dtype_t<2, ElementAOptionalTuple>; + using ZeroB = detail::deduce_mixed_width_dtype_t<2, ElementBOptionalTuple>; + + public: + static_assert( + cute::is_tuple::value ^ cute::is_tuple::value, + "Either A OR B must be a tuple. It must take the from {ElementOperand, [ElementScale], [ElementZero]}. Inputs in " + "[] are optional."); + + using ElementA = detail::deduce_mixed_width_dtype_t<0, ElementAOptionalTuple>; + using ElementB = detail::deduce_mixed_width_dtype_t<0, ElementBOptionalTuple>; + static constexpr bool IsATransformed = cute::is_tuple::value; + using ElementScale = cute::conditional_t; + using ElementZero = cute::conditional_t; + // For cases where we can't have a void type, we can use this to allow the code to compile when the scale / zero is + // void. + using NonVoidElementScale = cute::conditional_t, float, ElementScale>; + using NonVoidElementZero = cute::conditional_t, float, ElementZero>; + + using StrideA = StrideA_; + using InternalStrideA = cute::remove_pointer_t; + using StrideB = StrideB_; + using InternalStrideB = cute::remove_pointer_t; + + using StrideScale = cute::Stride, int64_t, int64_t>; + using NonVoidStrideScale = + cute::conditional_t, cute::Stride<_1, int64_t, int64_t>, StrideScale>; + + static_assert( + (IsATransformed && (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)) || + (!IsATransformed && (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)), + "The transformed type must be K-major."); + + static_assert( + (IsATransformed && (sizeof(ElementB) == 2)) || (!IsATransformed && (sizeof(ElementA) == 2)) || + ((cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value) && + (cutlass::gemm::detail::is_k_major() || is_layout::value || + is_layout::value)), + "The unscaled element must be 2 bytes OR both inputs must be K-major"); + + static_assert( + cutlass::gemm::detail::is_mn_major(), + "Scale must be MN major [Col Major if A is scaled, Row Major if B is scaled]."); + + using CtaShape_MNK = decltype(shape_div(TileShape{}, ClusterShape{})); + using TiledMma = TiledMma_; + using ElementAccumulator = typename TiledMma::ValTypeC; + using GmemTiledCopyA = GmemTiledCopyA_; + using GmemTiledCopyB = GmemTiledCopyB_; + using GmemTiledCopyScale = cute::SM90_TMA_LOAD; + using SmemLayoutAtomA = SmemLayoutAtomA_; + using SmemLayoutAtomB = SmemLayoutAtomB_; + using SmemCopyAtomA = SmemCopyAtomA_; + using SmemCopyAtomB = SmemCopyAtomB_; + using SmemCopyAtomScale = Copy_Atom; + + // We must ensure the type to be scaled goes to RF + static constexpr bool SwapAB = !IsATransformed; + using SwappedStrideA = cute::conditional_t; + using SwappedStrideB = cute::conditional_t; + using InternalSwappedStrideA = cute::conditional_t; + using InternalSwappedStrideB = cute::conditional_t; + using SwappedSmemLayoutAtomA = cute::conditional_t; + using SwappedSmemLayoutAtomB = cute::conditional_t; + using SwappedSmemCopyAtomA = cute::conditional_t; + using SwappedSmemCopyAtomB = cute::conditional_t; + // TMA converts f32 input to tf32 when copying from GMEM to SMEM + // For all other types, cast to size equivalent uint type to avoid any rounding by TMA. + static constexpr bool ConvertF32toTF32A = cute::is_same_v; + static constexpr bool ConvertF32toTF32B = cute::is_same_v; + using ConvertedElementA = cute::conditional_t>>; + using ConvertedElementB = cute::conditional_t>>; + using RealSwappedElementA = cute::conditional_t; + using RealSwappedElementB = cute::conditional_t; + using SwappedElementA = cute::conditional_t; + using SwappedElementB = cute::conditional_t; + + using TransformA = TransformA_; + using TransformB = TransformB_; + using SwappedTransformA = cute::conditional_t; + using SwappedTransformB = cute::conditional_t; + using ArchTag = typename DispatchPolicy::ArchTag; + + static constexpr int IsSubbyteA = cute::sizeof_bits_v < 8; + using TmaElementA = cute::conditional_t; + using TmaElementScale = uint_bit_t>; // in case we have array. translating to uint + // to satisfy tma descriptor's specialization + + using MainloopPipeline = cutlass::PipelineTmaAsync; + using PipelineState = cutlass::PipelineState; + using PipelineParams = typename MainloopPipeline::Params; + + static constexpr int NumProducerThreadEvents = 1; + + using SmemLayoutAtomScale = Layout(SwappedSmemLayoutAtomA{})), cute::Int<1>>>; + using ScaleTileShape = decltype(make_shape(shape<0>(TileShape{}), shape<1>(SmemLayoutAtomScale{}))); + + static_assert(cute::rank(SwappedSmemLayoutAtomA{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert( + (size<0>(TileShape{}) % size<0>(SwappedSmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SwappedSmemLayoutAtomA{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert(cute::rank(SwappedSmemLayoutAtomB{}) == 2, "SmemLayoutAtom must be rank 2 (M/N, K)"); + static_assert( + (size<1>(TileShape{}) % size<0>(SwappedSmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SwappedSmemLayoutAtomB{})) == 0, "SmemLayoutAtom must evenly divide tile shape."); + + static_assert(rank(SmemLayoutAtomScale{}) == 2, "SmemLayoutAtomScale must be rank 2"); + static_assert( + (size<0>(TileShape{}) % size<0>(SmemLayoutAtomScale{})) == 0, "SmemLayoutAtomScale must equal the tile shape."); + static_assert( + (size<2>(TileShape{}) % size<1>(SmemLayoutAtomScale{})) == 0, + "SmemLayoutAtomScale must evenly divide tile k shape."); + + /// Tile along modes in a way that maximizes the TMA box size. + using SmemLayoutA = decltype(detail::get_smem_layout( + SwappedSmemLayoutAtomA{}, select<0, 2>(TileShape{}), InternalSwappedStrideA{})); + using SmemLayoutB = decltype(detail::get_smem_layout( + SwappedSmemLayoutAtomB{}, select<1, 2>(TileShape{}), InternalSwappedStrideB{})); + + // It is assumed that the scales and zero-points share the same smem layout + using SmemLayoutScale = decltype(tile_to_shape( + SmemLayoutAtomScale{}, + make_shape(shape<0>(ScaleTileShape{}), shape<1>(ScaleTileShape{}), Int{}), + cute::conditional_t< + ::cutlass::gemm::detail::is_major<0, NonVoidStrideScale>(), + Step<_2, _1, _3>, + Step<_1, _2, _3>>{})); + + static_assert(DispatchPolicy::Stages >= 2, "Specialization requires Stages set to value 2 or more."); + static_assert( + not cute::is_base_of::value && + cute::is_base_of::value, + "MMA atom must source A from rmem and B operand from smem_desc for this mainloop."); + static_assert( + cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + static_assert( + cute::is_same_v || cute::is_same_v, + "GmemTiledCopy - invalid SM90 TMA copy atom specified."); + + // To relax them, we need to handle loading more than 1 row of scales for every main loop iteration. + // We must also handle updating the pipeline transaction bytes on the fly. + static_assert(size<1>(SmemLayoutAtomScale{}) == 1, "size<1>(SmemLayoutAtomScale) must be 1."); + + private: + static constexpr ConversionMode get_conversion_mode() { + if constexpr (cute::is_void_v) { + return ConversionMode::DirectConvert; + } else if constexpr (cute::is_void_v) { + return ConversionMode::ConvertAndScale; + } else { + return ConversionMode::ConvertAndScaleWithZero; + } + } + + public: + static constexpr ConversionMode KernelConversionMode = get_conversion_mode(); + static constexpr bool ModeHasScales = KernelConversionMode == ConversionMode::ConvertAndScale || + KernelConversionMode == ConversionMode::ConvertAndScaleWithZero; + static constexpr bool UseScaleLookupTable = + KernelConversionMode == ConversionMode::ConvertAndScale && cutlass::detail::is_Array_v; + static constexpr size_t SmemAlignmentA = cutlass::detail::alignment_for_swizzle(SmemLayoutA{}); + static constexpr size_t SmemAlignmentB = cutlass::detail::alignment_for_swizzle(SmemLayoutB{}); + static constexpr size_t SmemAlignmentScale = cute::max(SmemAlignmentA, SmemAlignmentB); + + static_assert(SmemAlignmentA >= 128 and SmemAlignmentB >= 128, "Require at least 128B alignment"); + + struct SharedStorage { + static constexpr int scale_elements = Utils::elements_per_smem_scale(); + static constexpr int zero_elements = Utils::elements_per_smem_zero(); + struct TensorStorage { + CUTE_ALIGNAS(SmemAlignmentA) cute::ArrayEngine> smem_A; + CUTE_ALIGNAS(SmemAlignmentB) cute::ArrayEngine> smem_B; + cute::ArrayEngine smem_scale; + cute::ArrayEngine smem_zero; + } tensors; + + struct TensorMapStorage { + cute::TmaDescriptor smem_tensormap_A; + cute::TmaDescriptor smem_tensormap_B; + cute::TmaDescriptor smem_tensormap_scale; + cute::TmaDescriptor smem_tensormap_zero; + }; + + using PipelineStorage = typename MainloopPipeline::SharedStorage; + PipelineStorage pipeline; + }; + using TensorStorage = typename SharedStorage::TensorStorage; + using TensorMapStorage = typename SharedStorage::TensorMapStorage; + using PipelineStorage = typename SharedStorage::PipelineStorage; + + static constexpr bool IsGroupedGemmKernel = !cute::is_same_v; + + // kernel Arguments + // Host side kernel arguments + struct Arguments { + ElementA const** ptr_A; + StrideA dA; + ElementB const** ptr_B; + StrideB dB; + ElementScale const** ptr_S = nullptr; + NonVoidStrideScale const* dS{}; + int chunk_size = 0; + ElementZero const** ptr_Z = nullptr; + }; + + // Device side kernel params + struct Params { + // Assumption: StrideA is congruent with Problem_MK + using LayoutA = + decltype(detail::get_gmem_layout(repeat_like(InternalSwappedStrideA{}, int32_t(0)), InternalSwappedStrideA{})); + using LayoutB = + decltype(detail::get_gmem_layout(repeat_like(InternalSwappedStrideB{}, int32_t(0)), InternalSwappedStrideB{})); + + using TMA_A = decltype(make_tma_copy( + GmemTiledCopyA{}, + make_tensor(detail::get_logical_ptr(static_cast(nullptr)), LayoutA{}), + SmemLayoutA{}(_, _, cute::Int<0>{}), + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{})), + size<1>(ClusterShape{}))); // mcast along N mode for this M load, if any + // Assumption: StrideB is congruent with Problem_NK + using TMA_B = decltype(make_tma_copy( + GmemTiledCopyB{}, + make_tensor(detail::get_logical_ptr(static_cast(nullptr)), LayoutB{}), + SmemLayoutB{}(_, _, cute::Int<0>{}), + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{})), + size<0>(ClusterShape{}))); // mcast along M mode for this N load, if any + + using TMA_Scale = decltype(make_tma_copy( + GmemTiledCopyScale{}, + make_tensor( + detail::get_logical_ptr(static_cast(nullptr)), + repeat_like(NonVoidStrideScale{}, int32_t(0)), + NonVoidStrideScale{}), + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{})); // mcast along N mode for this M load, if any. Scale is ALWAYS loaded with A for RF kernel + + using TMA_Zero = decltype(make_tma_copy( + GmemTiledCopyScale{}, + make_tensor( + detail::get_logical_ptr(static_cast(nullptr)), + repeat_like(NonVoidStrideScale{}, int32_t(0)), + NonVoidStrideScale{}), + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{})); // mcast along N mode for this M load, if any. Scale is ALWAYS loaded with A for RF kernel + + TMA_A tma_load_a; + TMA_B tma_load_b; + uint32_t tma_transaction_bytes = TmaTransactionBytes; + TMA_Scale tma_load_scale; + TMA_Zero tma_load_zero; + void* tensormaps; + SwappedElementA const** ptr_A; + SwappedStrideA ptr_dA; + SwappedElementB const** ptr_B; + SwappedStrideB ptr_dB; + NonVoidElementScale const** ptr_S; + NonVoidStrideScale const* dS; + NonVoidElementZero const** ptr_Z; + int64_t scale_k; + int chunk_size; + int reload_factor = (chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{}); + InternalSwappedStrideA dA; + InternalSwappedStrideB dB; + }; + + // + // Methods + // + + template + static constexpr Params to_underlying_arguments(ProblemShape problem_shapes, Arguments const& args, void* workspace) { + // These tensor shapes (only applicable for grouped gemm) and pointers are only used to create tensormap/tma desc. + // These will be replaced with correct values before the initial tma load. + auto init_shape = repeat_like(typename ProblemShape::UnderlyingProblemShape{}, int32_t(1)); + auto init_M = get<0>(init_shape); + auto init_N = get<1>(init_shape); + auto init_K = get<2>(init_shape); + + if constexpr (SwapAB) { + init_M = get<1>(init_shape); + init_N = get<0>(init_shape); + } + // Batches/Groups are managed by using appropriate pointers to input matrices + const uint32_t mock_L = 1; + SwappedElementA const* ptr_A_first_batch; + SwappedElementB const* ptr_B_first_batch; + SwappedStrideA ptr_dA; + SwappedStrideB ptr_dB; + InternalSwappedStrideA dA; + InternalSwappedStrideB dB; + + if constexpr (not SwapAB) { + ptr_A_first_batch = reinterpret_cast(args.ptr_A); + ptr_B_first_batch = reinterpret_cast(args.ptr_B); + } else { + ptr_A_first_batch = reinterpret_cast(args.ptr_B); + ptr_B_first_batch = reinterpret_cast(args.ptr_A); + } + + if constexpr (IsGroupedGemmKernel) { + // Strides for Grouped Gemm will be replaced prior to the first access regardless. + if constexpr (not SwapAB) { + ptr_dA = args.dA; + ptr_dB = args.dB; + } else { + ptr_dA = args.dB; + ptr_dB = args.dA; + } + dA = InternalSwappedStrideA{}; + if constexpr (is_layout::value) { + dA = make_layout( + transform_leaf( + dA.shape(), + [](auto x) { + if constexpr (not is_static_v) { + return static_cast(1); + } else { + return x; + } + }), + dA.stride()); + } + dB = InternalSwappedStrideB{}; + } else { + // Tensor shapes for Ptr-Array are initialized correctly only here. + auto problem_shape_MNK = problem_shapes.get_host_problem_shape(0); + init_M = get<0>(problem_shape_MNK); + init_N = get<1>(problem_shape_MNK); + init_K = get<2>(problem_shape_MNK); + + if constexpr (not SwapAB) { + dA = args.dA; + dB = args.dB; + } else { + dA = args.dB; + dB = args.dA; + } + ptr_dA = SwappedStrideA{}; + ptr_dB = SwappedStrideB{}; + } + Tensor tensor_a = make_tensor(ptr_A_first_batch, detail::get_gmem_layout(make_shape(init_M, init_K, mock_L), dA)); + Tensor tensor_b = make_tensor(ptr_B_first_batch, detail::get_gmem_layout(make_shape(init_N, init_K, mock_L), dB)); + + typename Params::TMA_A tma_load_a = make_tma_copy( + GmemTiledCopyA{}, + tensor_a, + SmemLayoutA{}(_, _, cute::Int<0>{}), + make_shape(shape<0>(TileShape{}), shape<2>(TileShape{})), + size<1>(ClusterShape{})); // mcast along N mode for this M load, if any + typename Params::TMA_B tma_load_b = make_tma_copy( + GmemTiledCopyB{}, + tensor_b, + SmemLayoutB{}(_, _, cute::Int<0>{}), + make_shape(shape<1>(TileShape{}), shape<2>(TileShape{})), + size<0>(ClusterShape{})); // mcast along M mode for this N load, if any + typename Params::TMA_Scale tma_load_scale{}; + typename Params::TMA_Zero tma_load_zero{}; + + void* tensormaps = workspace; + auto args_setup = + [&](auto ptr_A, auto ptr_B, int64_t scale_k = 0, int chunk_size = 0, int reload_factor = 1) -> Params { + return { + tma_load_a, + tma_load_b, + TmaTransactionBytes, + tma_load_scale, + tma_load_zero, + tensormaps, + reinterpret_cast(ptr_A), + ptr_dA, + reinterpret_cast(ptr_B), + ptr_dB, + reinterpret_cast(args.ptr_S), + args.dS, + reinterpret_cast(args.ptr_Z), + scale_k, + chunk_size, + reload_factor, + dA, + dB}; + }; + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return SwapAB ? args_setup(args.ptr_B, args.ptr_A) : args_setup(args.ptr_A, args.ptr_B); + } else if constexpr (ModeHasScales) { + auto fake_scale_k = 1; + ElementScale const* ptr_S = reinterpret_cast(args.ptr_S); + StrideScale dS{}; + Tensor tensor_scale = + make_tensor(detail::get_logical_ptr(ptr_S), make_layout(make_shape(init_M, fake_scale_k, mock_L), dS)); + tma_load_scale = make_tma_copy( + GmemTiledCopyScale{}, + tensor_scale, + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{}); // mcast along N mode for this M load, if any + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return SwapAB ? args_setup( + args.ptr_B, + args.ptr_A, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})) + : args_setup( + args.ptr_A, + args.ptr_B, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + ElementZero const* ptr_Z = reinterpret_cast(args.ptr_Z); + Tensor tensor_zero = + make_tensor(detail::get_logical_ptr(ptr_Z), make_layout(make_shape(init_M, fake_scale_k, mock_L), dS)); + tma_load_zero = make_tma_copy( + GmemTiledCopyScale{}, + tensor_zero, + SmemLayoutScale{}(_, _, cute::Int<0>{}), + ScaleTileShape{}, + _1{}); // mcast along N mode for this M load, if any + return SwapAB ? args_setup( + args.ptr_B, + args.ptr_A, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})) + : args_setup( + args.ptr_A, + args.ptr_B, + fake_scale_k, + args.chunk_size, + (args.chunk_size + size<2>(TileShape{}) - 1) / size<2>(TileShape{})); + + } else { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in to_underlying_arguments."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in to_underlying_arguments."); + } + } + + template + static size_t get_workspace_size(ProblemShape const& problem_shape, Arguments const& args, int sm_count) { + constexpr size_t SizeOfCuTensorMap = sizeof(cute::TmaDescriptor); + + // Calculating workspace size + auto calculate_workspace_size = [SizeOfCuTensorMap, sm_count](uint32_t num_input_tensors) { + return num_input_tensors * SizeOfCuTensorMap * sm_count; + }; + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies + return calculate_workspace_size(2); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies, + // followed by scale tensormap copies + return calculate_workspace_size(3); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // Allocate gmem space for input tensormaps per each SM, A tensormap copies followed by B tensormap copies, + // followed by scale and zeros tensormap copies + return calculate_workspace_size(4); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in get_workspace_size."); + } + } + + template + static cutlass::Status initialize_workspace( + ProblemShape const& problem_shape, + Arguments const& args, + void* workspace, + cudaStream_t stream, + CudaHostAdapter* cuda_adapter = nullptr) { + return cutlass::Status::kSuccess; + } + + template + CUTLASS_HOST_DEVICE static bool can_implement(ProblemShape problem_shapes, Arguments const& args) { + constexpr int tma_alignment_bits = 128; + constexpr int min_tma_aligned_elements_A = tma_alignment_bits / cutlass::sizeof_bits::value; + constexpr int min_tma_aligned_elements_B = tma_alignment_bits / cutlass::sizeof_bits::value; + + bool implementable = true; + if (problem_shapes.is_host_problem_shape_available()) { + // Check alignment for all problem sizes + for (int i = 0; i < problem_shapes.groups(); i++) { + auto problem_shape_MNKL = append<4>(problem_shapes.get_host_problem_shape(i), 1); + auto [M, N, K, L] = problem_shape_MNKL; + auto get_stride = [](auto stride) { + if constexpr (cute::is_pointer_v>) { + return *stride; + } else { + return stride; + } + }; + auto dA = get_stride(args.dA); + auto dB = get_stride(args.dB); + implementable = implementable && cutlass::detail::check_alignment( + detail::get_gmem_layout(cute::make_shape(M, K, L), dA)); + implementable = implementable && cutlass::detail::check_alignment( + detail::get_gmem_layout(cute::make_shape(N, K, L), dB)); + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + implementable = implementable && (args.ptr_S == nullptr); + implementable = implementable && (args.ptr_Z == nullptr); + } else if constexpr (ModeHasScales) { + const int scale_mn = SwapAB ? N : M; + const int scale_k = (K + args.chunk_size - 1) / args.chunk_size; + constexpr int min_tma_aligned_elements_scale = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment( + cute::make_shape(scale_mn, scale_k, L), StrideScale{}); + implementable = implementable && (args.chunk_size == K || ((args.chunk_size % size<2>(TileShape{})) == 0)); + implementable = implementable && args.chunk_size != 0; + implementable = implementable && (args.ptr_S != nullptr); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + implementable = implementable && (args.ptr_Z == nullptr); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + constexpr int min_tma_aligned_elements_zero = tma_alignment_bits / cutlass::sizeof_bits::value; + implementable = implementable && cutlass::detail::check_alignment( + cute::make_shape(scale_mn, scale_k, L), StrideScale{}); + implementable = implementable && (args.ptr_Z != nullptr); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in can_implement."); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in can_implement."); + } + } + } + + if (!implementable) { + CUTLASS_TRACE_HOST(" CAN IMPLEMENT: Problem Size doesn't meet the minimum alignment requirements for TMA.\n"); + } + return implementable; + } + + static constexpr int K_PIPE_MAX = DispatchPolicy::Stages; + static constexpr int K_PIPE_MMAS = 1; + static constexpr uint32_t TmaTransactionBytesMK = Utils::compute_tma_transaction_bytes_mk(); + static constexpr uint32_t TmaTransactionBytesNK = Utils::compute_tma_transaction_bytes_nk(); + static constexpr uint32_t TmaTransactionBytesExtra = Utils::compute_tma_transaction_bytes_extra(); + static constexpr uint32_t TmaTransactionBytes = + TmaTransactionBytesMK + TmaTransactionBytesNK + TmaTransactionBytesExtra; + + // Set up the data needed by this collective for load and mma. + // Returns a tuple of tensors. The collective and the kernel layer have the contract that the + // returned tuple must contain at least two elements, with the first two elements being: + // gA_mkl - The tma tensor, A after a local tile so it has shape (BLK_M,BLK_K,m,k,l) + // gB_nkl - The tma tensor, B after a local tile so it has shape (BLK_N,BLK_K,n,k,l) + // The rest of the tensors can be specified as needed by this collective. + template + CUTLASS_DEVICE auto load_init(ProblemShape_MNKL const& problem_shape_MNKL, Params const& mainloop_params) const { + using X = Underscore; + // Separate out problem shape for convenience + auto [M, N, K, L] = problem_shape_MNKL; + const int32_t mock_L = 1; + + // TMA requires special handling of strides to deal with coord codomain mapping + // Represent the full tensors -- get these from TMA + Tensor mA_mkl = mainloop_params.tma_load_a.get_tma_tensor( + shape(detail::get_gmem_layout(make_shape(M, K, mock_L), mainloop_params.dA))); // (m,k,l) + Tensor mB_nkl = mainloop_params.tma_load_b.get_tma_tensor( + shape(detail::get_gmem_layout(make_shape(N, K, mock_L), mainloop_params.dB))); // (n,k,l) + + // Make tiled views, defer the slice + Tensor gA_mkl = local_tile(mA_mkl, TileShape{}, make_coord(_, _, _), Step<_1, X, _1>{}); // (BLK_M,BLK_K,m,k,l) + Tensor gB_nkl = local_tile(mB_nkl, TileShape{}, make_coord(_, _, _), Step{}); // (BLK_N,BLK_K,n,k,l) + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(gA_mkl, gB_nkl); + } else if constexpr (ModeHasScales) { + // The real scale_k that actually works + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + + Tensor mS_mkl = mainloop_params.tma_load_scale.get_tma_tensor(make_shape(M, scale_k, L)); // (m,scale_k,l) + Tensor gS_mkl = local_tile(mS_mkl, ScaleTileShape{}, make_coord(_, _)); // (BLK_M,BLK_Scale_K,m,scale_k,l) + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(gA_mkl, gB_nkl, gS_mkl); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor mZ_mkl = mainloop_params.tma_load_zero.get_tma_tensor(make_shape(M, scale_k, L)); // (m,scale_k,l) + Tensor gZ_mkl = local_tile(mZ_mkl, ScaleTileShape{}, make_coord(_, _)); // (BLK_M,BLK_Scale_K,m,scale_k,l) + return cute::make_tuple(gA_mkl, gB_nkl, gS_mkl, gZ_mkl); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in load_init."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in load_init."); + } + } + + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Perform a collective-scoped matrix multiply-accumulate + // Producer Perspective + template + CUTLASS_DEVICE void load( + Params const& mainloop_params, + MainloopPipeline pipeline, + PipelineState smem_pipe_write, + cute::tuple const& load_inputs, + cute::tuple const& input_tensormaps, + BlockCoord const& blk_coord, + KTileIterator k_tile_iter, + int k_tile_count, + int thread_idx, + uint32_t block_rank_in_cluster, + TensorStorage& shared_tensors) { + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + static_assert(sizeof...(Ts) == 2, "Direct convert needs two inputs"); + static_assert(sizeof...(TMs) == 2, "Direct convert needs two tensormaps"); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + static_assert(sizeof...(Ts) == 3, "Scaled convert needs three inputs"); + static_assert(sizeof...(TMs) == 3, "Scaled convert needs three tensormaps"); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + static_assert(sizeof...(Ts) == 4, "Scaled and zero convert needs four inputs"); + static_assert(sizeof...(TMs) == 4, "Scaled and zero convert needs four tensormaps"); + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled in TMA load."); + } + + Tensor sA_ = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sB_ = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + Tensor sA = as_position_independent_swizzle_tensor(sA_); // (BLK_M,BLK_K,PIPE) + Tensor sB = as_position_independent_swizzle_tensor(sB_); // (BLK_N,BLK_K,PIPE) + + // + // Prepare the TMA loads for A and B + // + + constexpr uint32_t cluster_shape_x = get<0>(typename DispatchPolicy::ClusterShape()); + uint2 cluster_local_block_id = {block_rank_in_cluster % cluster_shape_x, block_rank_in_cluster / cluster_shape_x}; + + Tensor gA_mkl = get<0>(load_inputs); + Tensor gB_nkl = get<1>(load_inputs); + + auto block_tma_a = mainloop_params.tma_load_a.get_slice(cluster_local_block_id.y); + auto block_tma_b = mainloop_params.tma_load_b.get_slice(cluster_local_block_id.x); + + // Partition the inputs based on the current block coordinates. + auto [m_coord, n_coord, k_coord, l_coord] = blk_coord; + Tensor gA = gA_mkl(_, _, m_coord, _, l_coord); // (BLK_M,BLK_K,k) + Tensor gB = gB_nkl(_, _, n_coord, _, l_coord); // (BLK_N,BLK_K,k) + + // Applies the mapping from block_tma_a + Tensor tAgA = block_tma_a.partition_S(gA); // (TMA,TMA_M,TMA_K,k) + Tensor tAsA = block_tma_a.partition_D(sA); // (TMA,TMA_M,TMA_K,PIPE) + + Tensor tBgB = block_tma_b.partition_S(gB); // (TMA,TMA_N,TMA_K,k) + Tensor tBsB = block_tma_b.partition_D(sB); // (TMA,TMA_N,TMA_K,PIPE) + + uint16_t mcast_mask_a = 0; + uint16_t mcast_mask_b = 0; + uint16_t mcast_mask_s = 0; + + // Issue TmaLoads + // Maps the tile -> block, value + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int n = 0; n < size<1>(block_layout); ++n) { + mcast_mask_a |= (uint16_t(1) << block_layout(cluster_local_block_id.x, n, Int<0>{})); + } + } + + if constexpr (cute::is_same_v) { + auto block_layout = Layout{}; // (m,n) -> block_id + for (int m = 0; m < size<0>(block_layout); ++m) { + mcast_mask_b |= (uint16_t(1) << block_layout(m, cluster_local_block_id.y, Int<0>{})); + } + } + + auto extra_input_partitions = Utils::partition_extra_tma_inputs( + mainloop_params, load_inputs, shared_tensors, cluster_local_block_id, m_coord, l_coord); + + // Mainloop + CUTLASS_PRAGMA_NO_UNROLL + for (; k_tile_count > 0; --k_tile_count) { + // LOCK smem_pipe_write for _writing_ + pipeline.producer_acquire(smem_pipe_write); + + // + // Copy gmem to smem for *k_tile_iter + // + + using BarrierType = typename MainloopPipeline::ProducerBarrierType; + BarrierType* tma_barrier = pipeline.producer_get_barrier(smem_pipe_write); + + int write_stage = smem_pipe_write.index(); + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_a.with(get<0>(input_tensormaps), *tma_barrier, mcast_mask_a), + tAgA(_, _, _, *k_tile_iter), + tAsA(_, _, _, write_stage)); + copy( + mainloop_params.tma_load_b.with(get<1>(input_tensormaps), *tma_barrier, mcast_mask_b), + tBgB(_, _, _, *k_tile_iter), + tBsB(_, _, _, write_stage)); + } + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + // Nothing extra to do. + } else if constexpr (ModeHasScales) { + // scale copy + auto tSgS = get<0>(extra_input_partitions); + auto tSsS = get<1>(extra_input_partitions); + + // Temporary factor which will determine which k tile to reload from gmem. Needed so we don't modify tma + // transaction bytes on the fly. We must do a ceiling divide here to correctly handle with chunk_size == K. In + // that case, we don't require that K is a multiple of the threadblock tile K + const int scale_load_k = *k_tile_iter / 1; + // const int scale_load_k = *k_tile_iter / mainloop_params.reload_factor; // This will always be 0 when + // chunk_size == K. + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_scale.with(get<2>(input_tensormaps), *tma_barrier, mcast_mask_s), + tSgS(_, _, _, scale_load_k), + tSsS(_, _, _, write_stage)); + } + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + // Nothing extra to do + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + // zero copy + auto tZgZ = get<2>(extra_input_partitions); + auto tZsZ = get<3>(extra_input_partitions); + if (cute::elect_one_sync()) { + copy( + mainloop_params.tma_load_zero.with(get<3>(input_tensormaps), *tma_barrier, mcast_mask_s), + tZgZ(_, _, _, scale_load_k), + tZsZ(_, _, _, write_stage)); + } + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled for TMA copy op."); + } + } else { + static_assert(cutlass::detail::dependent_false, "Conversion mode not handled for TMA copy op."); + } + ++k_tile_iter; + + // Advance smem_pipe_write + ++smem_pipe_write; + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Perform a Producer Epilogue to prevent early exit of blocks in a Cluster + CUTLASS_DEVICE void load_tail(MainloopPipeline pipeline, PipelineState smem_pipe_write) { + int lane_predicate = cute::elect_one_sync(); + + // Issue the epilogue waits + if (lane_predicate) { + // This helps avoid early exit of blocks in Cluster. + // Waits for all stages to either be released (all + // Consumer UNLOCKs), or if the stage was never used + // then it would just be acquired since the phase was + // still inverted from make_producer_start_state. + pipeline.producer_tail(smem_pipe_write); + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// Perform a collective-scoped matrix multiply-accumulate + /// Consumer Perspective + template + CUTLASS_DEVICE void + mma(MainloopPipeline pipeline, + PipelineState smem_pipe_read, + FrgTensorC& accum, + int k_tile_count, + int thread_idx, + TensorStorage& shared_tensors, + Params const& mainloop_params) { + static_assert(is_rmem::value, "C tensor must be rmem resident."); + static_assert(cute::rank(SmemLayoutA{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::rank(SmemLayoutB{}) == 3, "Smem layout must be rank 3."); + static_assert(cute::rank(SwappedSmemLayoutAtomA{}) == 2, "SwappedSmemLayoutAtomA must be rank 2."); + static_assert(cute::rank(SwappedSmemLayoutAtomB{}) == 2, "SwappedSmemLayoutAtomB must be rank 2."); + static_assert( + !cute::is_void_v, + "SM90 GMMA mainloops must specify a non-void copy atom for smem sourced instructions."); + static_assert( + cute::is_void_v, + "SM90 GMMA mainloops cannot have a non-void copy atom for smem sourced instructions."); + + // Obtain warp index + int warp_idx = canonical_warp_idx_sync(); + [[maybe_unused]] int warp_group_thread_idx = thread_idx % 128; + + Tensor sA_ = make_tensor(make_smem_ptr(shared_tensors.smem_A.begin()), SmemLayoutA{}); // (BLK_M,BLK_K,PIPE) + Tensor sA = as_position_independent_swizzle_tensor(sA_); // (BLK_M,BLK_K,PIPE) + + Tensor sB = make_tensor(make_smem_ptr(shared_tensors.smem_B.begin()), SmemLayoutB{}); // (BLK_N,BLK_K,PIPE) + + // + // Define C accumulators and A/B partitioning + // + + // Layout of warp group to thread mapping + + static_assert( + stride<0>(typename TiledMma::BLayout{}) == 0 and + size<0>(typename TiledMma::BLayout{}) == NumThreadsPerWarpGroup, + "Stride of the first mode must be 0 and the size of the mode must be NumThreadsPerWarpGroup"); + + constexpr int MmaWarpGroups = size(TiledMma{}) / NumThreadsPerWarpGroup; + Layout warp_group_thread_layout = make_layout(Int{}, Int{}); + + int warp_group_idx = __shfl_sync(0xFFFFFFFF, thread_idx / NumThreadsPerWarpGroup, 0); + + TiledMma tiled_mma; + auto mma_thread_slice = tiled_mma.get_thread_slice(thread_idx); + Tensor tCsA = mma_thread_slice.partition_A(sA); + auto mma_warpgroup_slice = tiled_mma.get_slice(warp_group_thread_layout(warp_group_idx)); + + // Allocate fragments and descriptors + Tensor tCrA_mma = mma_thread_slice.partition_fragment_A(sA(_, _, Int<0>{})); // (MMA,MMA_M,MMA_K,PIPE) + Tensor tCrA_load = [&] { + if constexpr (not is_layout::value) { + // Make register tensor with MMA layout + return make_fragment_like(tCrA_mma); + } else { + // Make register tensor matching smem layout, converter will take care of de-swizzling + return make_tensor_like(tCsA(_, _, _, Int<0>{})); + } + }(); + Tensor tCsB = mma_warpgroup_slice.partition_B(sB); // (MMA,MMA_N,MMA_K,PIPE) + Tensor tCrB = mma_warpgroup_slice.make_fragment_B(tCsB); // (MMA,MMA_N,MMA_K,PIPE) + + // + // Copy Atom A retiling + // + auto smem_tiled_copy_A = make_tiled_copy_A(SwappedSmemCopyAtomA{}, tiled_mma); + auto smem_thr_copy_A = smem_tiled_copy_A.get_thread_slice(warp_group_thread_idx); + + Tensor tCrA_copy_view = smem_thr_copy_A.retile_D(tCrA_load); // (CPY,CPY_M,CPY_K) + + // Partition of thread -> shared and thread -> RF + auto partitioned_extra_info = Utils::partition_extra_mma_info(mma_thread_slice, shared_tensors); + auto copy_partitions_extra_info = + Utils::retile_extra_mma_info(tiled_mma, partitioned_extra_info, warp_group_thread_idx); + + CUTE_STATIC_ASSERT_V(size<1>(tCsA) == size<1>(tCrA_copy_view)); // CPY_M + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCrA_copy_view)); // CPY_K + CUTE_STATIC_ASSERT_V(size<1>(tCrA_mma) == size<1>(accum)); // MMA_M + CUTE_STATIC_ASSERT_V(size<1>(tCsB) == size<2>(accum)); // N + CUTE_STATIC_ASSERT_V(size<2>(tCsA) == size<2>(tCsB)); // K + CUTE_STATIC_ASSERT_V(size<3>(tCsA) == size<3>(tCsB)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sA)); // PIPE + CUTE_STATIC_ASSERT_V(Int{} == size<2>(sB)); // PIPE + + // + // PIPELINED MAIN LOOP + // + + // We release buffers to producer warps(dma load) with some mmas in flight + PipelineState smem_pipe_release = smem_pipe_read; + + multiply_add fma; + + constexpr int NumMMAsPerChunk = GROUP_SIZE / cute::get<0, 1>(tCsB.shape())(); + constexpr int NumChunksPerTileK = cute::size<1>(sA.shape())() / GROUP_SIZE; + cute::array intermediate_array; + + constexpr int K_BLOCK_MAX = size<2>(tCrA_load); + constexpr int K_WAIT_MAX = cute::min(K_BLOCK_MAX - 1, 7); + static_assert(K_BLOCK_MAX >= 4, "Consider increasing TileShapeK"); + + ConsumerToken barrier_token = {BarrierStatus::WaitAgain}; + // First k tile + { + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + int read_stage = smem_pipe_read.index(); + + ++smem_pipe_read; + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + + // copy smem->rmem for A operand + + Utils::copy_tensors_MK( + smem_tiled_copy_A, tCsA, tCrA_copy_view, partitioned_extra_info, copy_partitions_extra_info, 0, read_stage); + if (K_BLOCK_MAX > 1) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, tCsA, tCrA_copy_view, partitioned_extra_info, copy_partitions_extra_info, 1, read_stage); + } + + // src: tCrA_load, dst: tCrA_mma + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int chunk_id = 0; chunk_id < NumChunksPerTileK; ++chunk_id) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + CUTLASS_PRAGMA_UNROLL + for (int mma_id = 0; mma_id < NumMMAsPerChunk; ++mma_id) { + int k_block = chunk_id * NumMMAsPerChunk + mma_id; + + warpgroup_arrive(); + + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate_array[chunk_id]); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + + warpgroup_commit_batch(); + + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + if (k_block < K_BLOCK_MAX - 1) { + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + } + } + + CUTLASS_PRAGMA_UNROLL + for (int chunk_id_ = 0; chunk_id_ < NumChunksPerTileK; ++chunk_id_) { + warpgroup_fence_operand(intermediate_array[chunk_id_]); + + // Apply the group-wise scaling + // tCrS ((4, _2, _2), MMA_M, _1) + // accum ((2, _2, _2), MMA_M, _1) + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + + if (chunk_id_ == 0) { + accum(accum_coord) = + intermediate_array[chunk_id_](accum_coord) * static_cast(tCrS(scale_coord)[0]); + } else { + accum(accum_coord) = + fma(intermediate_array[chunk_id_](accum_coord), + static_cast(tCrS(scale_coord)[chunk_id_]), + accum(accum_coord)); + } + } + } + } + } + } + + --k_tile_count; + if (k_tile_count > 0) { + // Wait for K_BLOCK_MAX - 1 to be in flight to ensure that it is safe to overwrite the A registers for the first + // mma. + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 0, + smem_pipe_read.index()); + + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 1, + smem_pipe_read.index()); + + warpgroup_wait(); + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + } + } + + if (k_tile_count == 0) { + return; + } + + // Mainloop GMMAs + CUTLASS_PRAGMA_NO_UNROLL + for (; k_tile_count > 1; --k_tile_count) { + // + // Compute on k_tile + // + + int read_stage = smem_pipe_read.index(); + ++smem_pipe_read; + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int chunk_id = 0; chunk_id < NumChunksPerTileK; ++chunk_id) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + CUTLASS_PRAGMA_UNROLL + for (int mma_id = 0; mma_id < NumMMAsPerChunk; ++mma_id) { + int k_block = chunk_id * NumMMAsPerChunk + mma_id; + + warpgroup_arrive(); + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate_array[chunk_id]); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + warpgroup_commit_batch(); + + warpgroup_wait(); // We have K_BLOCK_MAX - 1 GMMA instructions pending for this stage, so we can + // release prior barrier + if (k_block == K_BLOCK_MAX - 1) { + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + + if (k_block == 0) { + barrier_token = pipeline.consumer_try_wait(smem_pipe_read); + } + + if (k_block == K_BLOCK_MAX - 1) { + // The last k_block + + CUTLASS_PRAGMA_UNROLL + for (int chunk_id_ = 0; chunk_id_ < NumChunksPerTileK; ++chunk_id_) { + warpgroup_fence_operand(intermediate_array[chunk_id_]); + + // Apply the group-wise scaling + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + + accum(accum_coord) = + fma(intermediate_array[chunk_id_](accum_coord), + static_cast(tCrS(scale_coord)[chunk_id_]), + accum(accum_coord)); + } + } + } + } + } + + pipeline.consumer_wait(smem_pipe_read, barrier_token); + + // copy scales when passing k_block=0 + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 0, + smem_pipe_read.index()); + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + 1, + smem_pipe_read.index()); + Utils::convert_A_kblock(tCrA_load, tCrA_mma, 0); + } else { + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + } + } + } + + { + // + // Last k tile + // + Tensor intermediate = make_fragment_like(accum); + + int read_stage = smem_pipe_read.index(); + + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + + // Unroll the K mode manually to set scale D to 1 + CUTLASS_PRAGMA_UNROLL + for (int k_block = 0; k_block < K_BLOCK_MAX; ++k_block) { + warpgroup_arrive(); + // (V,M) x (V,N) => (V,M,N) + cute::gemm(tiled_mma, tCrA_mma(_, _, k_block), tCrB(_, _, k_block, read_stage), intermediate); + tiled_mma.accumulate_ = GMMA::ScaleOut::One; + warpgroup_commit_batch(); + + warpgroup_wait(); + if (k_block == K_BLOCK_MAX - 1) { + // release prior barrier + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + + if (k_block < K_BLOCK_MAX - 2) { + Utils::copy_tensors_MK( + smem_tiled_copy_A, + tCsA, + tCrA_copy_view, + partitioned_extra_info, + copy_partitions_extra_info, + k_block + 2, + read_stage); + } + if (k_block < K_BLOCK_MAX - 1) { + Utils::convert_A_kblock(tCrA_load, tCrA_mma, k_block + 1); + } + + if ((k_block + 1) % NumMMAsPerChunk == 0) { + tiled_mma.accumulate_ = GMMA::ScaleOut::Zero; + warpgroup_fence_operand(intermediate); + + // Apply the group-wise scaling + auto tCrS = cute::get<1>(partitioned_extra_info); + for (int mma_m = 0; mma_m < size<1>(accum); mma_m++) { + for (int m = 0; m < size<0, 1>(accum); m++) { + for (int n = 0; n < size<0, 2>(accum); n++) { + for (int e = 0; e < size<0, 0>(accum); e++) { + auto accum_coord = make_coord(make_tuple(e, m, n), mma_m, 0); + auto scale_coord = make_coord(make_tuple(0, m, 0), mma_m, 0); + int scale_idx = k_block / NumMMAsPerChunk; + + accum(accum_coord) = fma( + intermediate(accum_coord), static_cast(tCrS(scale_coord)[scale_idx]), accum(accum_coord)); + } + } + } + } + } + } + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// Perform a Consumer Epilogue to release all buffers + CUTLASS_DEVICE void mma_tail(MainloopPipeline pipeline, PipelineState smem_pipe_release, int k_tile_count) { + // Prologue GMMAs + int prologue_mma_count = 1; + k_tile_count -= prologue_mma_count; + + smem_pipe_release.advance(k_tile_count); + + // Wait on all GMMAs to complete + warpgroup_wait<0>(); + + for (int count = 0; count < prologue_mma_count; ++count) { + pipeline.consumer_release(smem_pipe_release); // UNLOCK smem_pipe_release, done _computing_ on it + ++smem_pipe_release; + } + } + ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // + // Methods to perform different parts of TMA/Tensormap modifications + // + CUTLASS_DEVICE auto tensormaps_init( + Params const& mainloop_params, TensorMapStorage& shared_tensormaps, int32_t sm_count, int32_t sm_idx) { + cute::TmaDescriptor* gmem_tensormap = reinterpret_cast(mainloop_params.tensormaps); + + cute::TmaDescriptor* tma_desc_a = &gmem_tensormap[sm_idx]; + cute::TmaDescriptor* tma_desc_b = &gmem_tensormap[sm_idx + sm_count]; + cute::TmaDescriptor* tma_desc_scale = &gmem_tensormap[sm_idx + 2 * sm_count]; + cute::TmaDescriptor* tma_desc_zero = &gmem_tensormap[sm_idx + 3 * sm_count]; + + // Bringing tensormaps from params to smem for modification later + Tensor pA_tensormap = make_tensor(mainloop_params.tma_load_a.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sA_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_A), Int<1>{}, Int<1>{}); + Tensor pB_tensormap = make_tensor(mainloop_params.tma_load_b.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sB_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_B), Int<1>{}, Int<1>{}); + + if (cute::elect_one_sync()) { + copy(recast(pA_tensormap), recast(sA_tensormap)); + copy(recast(pB_tensormap), recast(sB_tensormap)); + } + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + Tensor pS_tensormap = make_tensor(mainloop_params.tma_load_scale.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sS_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_scale), Int<1>{}, Int<1>{}); + if (cute::elect_one_sync()) { + copy(recast(pS_tensormap), recast(sS_tensormap)); + } + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + Tensor pZ_tensormap = make_tensor(mainloop_params.tma_load_zero.get_tma_descriptor(), Int<1>{}, Int<1>{}); + Tensor sZ_tensormap = make_tensor(make_smem_ptr(&shared_tensormaps.smem_tensormap_zero), Int<1>{}, Int<1>{}); + if (cute::elect_one_sync()) { + copy(recast(pZ_tensormap), recast(sZ_tensormap)); + } + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_init."); + } + + __syncwarp(); + + if constexpr (KernelConversionMode == ConversionMode::DirectConvert) { + return cute::make_tuple(tma_desc_a, tma_desc_b); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + return cute::make_tuple(tma_desc_a, tma_desc_b, tma_desc_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + return cute::make_tuple(tma_desc_a, tma_desc_b, tma_desc_scale, tma_desc_zero); + } else { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_init."); + } + } + + // Replace address for the global tensor (to be done by single thread) + CUTLASS_DEVICE + void tensormaps_replace_global_address( + TensorMapStorage& shared_tensormaps, Params const& mainloop_params, int32_t next_batch) { + // Replacing global_address for the next batch + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_A, mainloop_params.ptr_A[next_batch]); + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_B, mainloop_params.ptr_B[next_batch]); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_scale, mainloop_params.ptr_S[next_batch]); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_replace_addr_in_shared_mem( + shared_tensormaps.smem_tensormap_zero, mainloop_params.ptr_Z[next_batch]); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_address."); + } + } + + // Replace dim and strides for the global tensor - used only for Grouped GEMM (to be done by single thread) + template + CUTLASS_DEVICE void tensormaps_replace_global_tensor_properties( + TensorMapStorage& shared_tensormaps, + Params const& mainloop_params, + int32_t next_group, + ProblemShape_MNKL problem_shape_mnkl) { + const uint32_t M = get<0>(problem_shape_mnkl); + const uint32_t N = get<1>(problem_shape_mnkl); + const uint32_t K = get<2>(problem_shape_mnkl); + + // Replace all dims for consistency + constexpr int MaxTensorRank = 5; + cute::array prob_shape_A = {1, 1, 1, 1, 1}; + cute::array prob_stride_A = {0, 0, 0, 0, 0}; + cute::array prob_shape_B = {1, 1, 1, 1, 1}; + cute::array prob_stride_B = {0, 0, 0, 0, 0}; + cute::array prob_shape_scale = {1, 1, 1, 1, 1}; + cute::array prob_stride_scale = {0, 0, 0, 0, 0}; + cute::array prob_shape_zero = {1, 1, 1, 1, 1}; + cute::array prob_stride_zero = {0, 0, 0, 0, 0}; + + SwappedElementA const* ptr_A = nullptr; + Tensor tensor_a = + make_tensor(ptr_A, detail::get_gmem_layout(make_shape(M, K, Int<1>{}), mainloop_params.ptr_dA[next_group])); + + SwappedElementB const* ptr_B = nullptr; + Tensor tensor_b = + make_tensor(ptr_B, detail::get_gmem_layout(make_shape(N, K, Int<1>{}), mainloop_params.ptr_dB[next_group])); + + cute::detail::fill_tma_gmem_shape_stride(mainloop_params.tma_load_a, tensor_a, prob_shape_A, prob_stride_A); + cute::detail::fill_tma_gmem_shape_stride(mainloop_params.tma_load_b, tensor_b, prob_shape_B, prob_stride_B); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + NonVoidElementScale const* ptr_S = nullptr; + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + Tensor tensor_scale = + make_tensor(detail::get_logical_ptr(ptr_S), make_shape(M, scale_k, Int<1>{}), mainloop_params.dS[next_group]); + cute::detail::fill_tma_gmem_shape_stride( + mainloop_params.tma_load_scale, tensor_scale, prob_shape_scale, prob_stride_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + ElementZero const* ptr_Z = nullptr; + // auto scale_k = K / mainloop_params.chunk_size; + auto scale_k = K / GROUP_SIZE; + Tensor tensor_zero = + make_tensor(detail::get_logical_ptr(ptr_Z), make_shape(M, scale_k, Int<1>{}), mainloop_params.dS[next_group]); + cute::detail::fill_tma_gmem_shape_stride( + mainloop_params.tma_load_zero, tensor_zero, prob_shape_zero, prob_stride_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_tensor_properties."); + } + + // Convert strides to byte strides + for (uint64_t& stride : prob_stride_A) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_B) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_scale) { + stride = (stride * sizeof_bits_v) / 8; + } + for (uint64_t& stride : prob_stride_zero) { + stride = (stride * sizeof_bits_v) / 8; + } + + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_A, prob_shape_A, prob_stride_A); + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_B, prob_shape_B, prob_stride_B); + + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_scale, prob_shape_scale, prob_stride_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_replace_dims_strides_in_shared_mem( + shared_tensormaps.smem_tensormap_zero, prob_shape_zero, prob_stride_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_replace_global_tensor_properties."); + } + } + + template + CUTLASS_DEVICE void tensormaps_perform_update( + TensorMapStorage& shared_tensormaps, + Params const& mainloop_params, + cute::tuple const& input_tensormaps, + ProblemShape_MNKL problem_shape_mnkl, + int32_t next_batch) { + if (cute::elect_one_sync()) { + // Replacing global_address for the next batch + tensormaps_replace_global_address(shared_tensormaps, mainloop_params, next_batch); + + if constexpr (IsGroupedGemmKernel) { + // Replacing global dims and strides for the next batch + tensormaps_replace_global_tensor_properties(shared_tensormaps, mainloop_params, next_batch, problem_shape_mnkl); + } + } + } + + template + CUTLASS_DEVICE void + tensormaps_cp_fence_release(TensorMapStorage& shared_tensormaps, cute::tuple const& input_tensormaps) { + if (cute::elect_one_sync()) { + cute::tma_desc_commit_group(); + cute::tma_desc_wait_group(); + } + // Entire warp must do this (i.e. it's aligned) + tma_descriptor_cp_fence_release(get<0>(input_tensormaps), shared_tensormaps.smem_tensormap_A); + tma_descriptor_cp_fence_release(get<1>(input_tensormaps), shared_tensormaps.smem_tensormap_B); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + tma_descriptor_cp_fence_release(get<2>(input_tensormaps), shared_tensormaps.smem_tensormap_scale); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + tma_descriptor_cp_fence_release(get<3>(input_tensormaps), shared_tensormaps.smem_tensormap_zero); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, + "Conversion mode not handled in tensormaps_cp_fence_release."); + } + } + + // The entire warp must call this function collectively (that is, the instructions are aligned) + template + CUTLASS_DEVICE void tensormaps_fence_acquire(cute::tuple const& input_tensormaps) { + cute::tma_descriptor_fence_acquire(get<0>(input_tensormaps)); + cute::tma_descriptor_fence_acquire(get<1>(input_tensormaps)); + if constexpr (KernelConversionMode == ConversionMode::ConvertAndScale) { + cute::tma_descriptor_fence_acquire(get<2>(input_tensormaps)); + } else if constexpr (KernelConversionMode == ConversionMode::ConvertAndScaleWithZero) { + cute::tma_descriptor_fence_acquire(get<3>(input_tensormaps)); + } else if constexpr (KernelConversionMode != ConversionMode::DirectConvert) { + static_assert( + cutlass::detail::dependent_false, "Conversion mode not handled in tensormaps_fence_acquire."); + } + } + + template + CUTLASS_DEVICE InputTensors tensors_perform_update( + InputTensors const& input_tensors, + [[maybe_unused]] Params const& mainloop_params, + [[maybe_unused]] ProblemShape_MNKL problem_shape_mnkl, + [[maybe_unused]] int32_t next_batch) { + return input_tensors; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm::collective + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh b/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh new file mode 100644 index 000000000..737916d89 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/cutlass_gemm_caller.cuh @@ -0,0 +1,62 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/c3x/cutlass_gemm_caller.cuh + +#pragma once + +// clang-format will break include orders +// clang-format off +#include + +#include +#include + +#include "cutlass/cutlass.h" + +#include "cute/tensor.hpp" +#include "cute/atom/mma_atom.hpp" +#include "cutlass/numeric_types.h" + +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/util/packed_stride.hpp" + +// clang-format on + +/** + * Helper function for checking CUTLASS errors + */ +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +template +void cutlass_gemm_caller( + torch::Device device, + cute::Shape prob_shape, + typename GemmKernel::MainloopArguments mainloop_args, + typename GemmKernel::EpilogueArguments epilogue_args, + typename GemmKernel::TileSchedulerArguments scheduler = {}) { + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = c10::cuda::current_device(); + hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, prob_shape, mainloop_args, epilogue_args, hw_info, scheduler}; + + // Launch the CUTLASS GEMM kernel. + using GemmOp = cutlass::gemm::device::GemmUniversalAdapter; + GemmOp gemm_op; + CUTLASS_CHECK(gemm_op.can_implement(args)); + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(device); + auto workspace = torch::empty(workspace_size, workspace_options); + + auto stream = at::cuda::getCurrentCUDAStream(device.index()); + + cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream); + CUTLASS_CHECK(status); +} diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp b/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp new file mode 100644 index 000000000..6019e4a53 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/dispatch_policy.hpp @@ -0,0 +1,38 @@ +// Adapted from https://github.com/vllm-project/vllm/blob/main/csrc/cutlass_extensions/gemm/dispatch_policy.hpp + +#pragma once + +#include "cutlass/gemm/dispatch_policy.hpp" + +namespace cutlass::gemm { + +////////////////////////////////////////////////////////////////////////////// + +// FP8 related policies (including Blocked Scaled Accumulation) +// `ScaleGranularityM` specifies scaling granularity along M, while zero-value +// `ScaleGranularityM` indicates that scaling granularity is +// `size<0>(TileShape_MNK{})` along M. +template +struct KernelTmaWarpSpecializedCooperativeFP8BlockScaledSubGroupMAccum : KernelTmaWarpSpecializedCooperative {}; + +// n-buffer in smem (Hopper TMA), pipelined with Hopper GMMA and TMA, Warp +// specialized dynamic schedule For FP8 kernels with Block Scaling +template < + int Stages_, + class ClusterShape_ = Shape<_1, _1, _1>, + class KernelSchedule = KernelTmaWarpSpecialized, + int ScaleGranularityM = 0 // `ScaleGranularityM` specifies scaling granularity along M, + // while zero-value `ScaleGranularityM` indicates that scaling + // granularity is `size<0>(TileShape_MNK{})` along M. + > +struct MainloopSm90TmaGmmaWarpSpecializedBlockScalingSubGroupMFP8 + : MainloopSm90TmaGmmaWarpSpecialized { + static_assert( + cute:: + is_same_v>, + "KernelSchedule must be one of the warp specialized policies"); +}; + +////////////////////////////////////////////////////////////////////////////// + +} // namespace cutlass::gemm diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh b/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh new file mode 100644 index 000000000..8deda43e5 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh @@ -0,0 +1,197 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/c3x/scaled_mm_blockwise_sm90_fp8_dispatch.cuh +#pragma once + +#include "cute/tensor.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/dispatch_policy.hpp" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/gemm/kernel/tile_scheduler_params.h" +#include "cutlass/numeric_types.h" +#include "cutlass/tensor_ref.h" +#include "cutlass_extensions/common.hpp" +#include "cutlass_extensions/gemm/cutlass_gemm_caller.cuh" +#include "cutlass_extensions/gemm/dispatch_policy.hpp" + +using namespace cute; + +template < + typename SchedulerType, + typename OutType, + int GroupSizeM_, + int GroupSizeN_, + int GroupSizeK_, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _2, _1>> +struct cutlass_3x_gemm_fp8_blockwise { + using GroupSizeM = Int; + using GroupSizeN = Int; + using GroupSizeK = Int; + using TileSizeM = Int; + + static_assert(TileSizeM_ % GroupSizeM_ == 0, "TileSizeM must be a multiple of GroupSizeM"); + + using ElementAB = cutlass::float_e4m3_t; + + // A matrix configuration + using ElementA = ElementAB; + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + // B matrix configuration + using ElementB = ElementAB; + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + // C/D matrix configuration + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementD = OutType; + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using ScaleTileShape = Shape<_1, _128, _128>; + using ScaleConfig = decltype(cutlass::detail::sm90_trivial_blockwise_scale_config(ScaleTileShape{})); + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + // Multiply-accumulate blocking/pipelining details + using ElementAccumulator = float; // Element type for internal accumulation + using ElementCompute = float; // Element type for compute + using TileShape = Shape; // Threadblock-level tile size + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecializedCooperative; + using EpilogueTileType = cutlass::epilogue::collective::EpilogueTileAuto; + using StoreEpilogueCompute = typename cutlass::epilogue::fusion::Sm90EVT; + + using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8BlockScaledAccum; + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + EpilogueTileType, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + EpilogueSchedule, + StoreEpilogueCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + SchedulerType>; +}; + +template +void cutlass_gemm_caller_blockwise( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + using GemmKernel = typename Gemm::GemmKernel; + using ElementAB = typename Gemm::ElementAB; + using ElementA = ElementAB; + using ElementB = ElementAB; + using ElementD = typename Gemm::ElementD; + using ElementBlockScale = float; + + using ScaleTileShape = Shape<_1, _128, _128>; + using ScaleConfig = decltype(cutlass::detail::sm90_trivial_blockwise_scale_config(ScaleTileShape{})); + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + + auto a_s_ptr = static_cast(a_scales.data_ptr()); + auto b_s_ptr = static_cast(b_scales.data_ptr()); + + using StrideA = typename GemmKernel::StrideA; + using StrideB = typename GemmKernel::StrideB; + using StrideD = typename GemmKernel::StrideD; + using StrideC = typename GemmKernel::StrideC; + + StrideA a_stride = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC c_stride = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_sfa = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_sfb = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, a_stride, b_ptr, b_stride, a_s_ptr, layout_sfa, b_s_ptr, layout_sfb}; + auto c_ptr = static_cast(out.data_ptr()); + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, c_stride, c_ptr, c_stride}; + + typename GemmKernel::TileSchedulerArguments scheduler; + + static constexpr bool UsesStreamKScheduler = + cute::is_same_v; + + if constexpr (UsesStreamKScheduler) { + using DecompositionMode = + typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode; + using ReductionMode = + typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::ReductionMode; + + scheduler.decomposition_mode = DecompositionMode::StreamK; + scheduler.reduction_mode = ReductionMode::Nondeterministic; + } + + cutlass_gemm_caller(a.device(), {m, n, k, 1}, mainloop_args, epilogue_args, scheduler); +} + +template +void cutlass_gemm_blockwise_sm90_fp8_dispatch( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + auto k = a.size(1); + auto n = b.size(1); + + if (k > 3 * n) { + cutlass_gemm_caller_blockwise>( + out, a, b, a_scales, b_scales); + } else { + cutlass_gemm_caller_blockwise< + cutlass_3x_gemm_fp8_blockwise>( + out, a, b, a_scales, b_scales); + } +} diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h new file mode 100644 index 000000000..b58d84318 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_universal_base_compat.h @@ -0,0 +1,356 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/device/gemm_universal_base_compat.h +#pragma once + +#include +#include +#include + +//////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace device { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +/* + This is the device layer from CUTLASS 2.10 (SHA - cc85b64cf676c45f98a17e3a47c0aafcf817f088) + It is replicated here since we needed to duplicate kernel level APIs for mixed dtype GEMMs + and SmoothQuant. The newer device layer is not compatible with these older kernel level APIs. + + Note: While CUTLASS 3.x supports stream-k, none of the kernels in the extensions folder support + that feature at the moment. + */ + +template +class GemmUniversalBaseCompat { + public: + using GemmKernel = GemmKernel_; + using ThreadblockShape = typename GemmKernel::Mma::Shape; + + using ElementA = typename GemmKernel::ElementA; + using LayoutA = typename GemmKernel::LayoutA; + using TensorRefA = TensorRef; + static ComplexTransform const kTransformA = GemmKernel::kTransformA; + + using ElementB = typename GemmKernel::ElementB; + using LayoutB = typename GemmKernel::LayoutB; + using TensorRefB = TensorRef; + static ComplexTransform const kTransformB = GemmKernel::kTransformB; + + using ElementC = typename GemmKernel::ElementC; + using LayoutC = typename GemmKernel::LayoutC; + using TensorRefC = TensorRef; + using TensorRefD = TensorRef; + + using ElementAccumulator = typename GemmKernel::Mma::Policy::Operator::ElementC; + + using EpilogueOutputOp = typename GemmKernel::EpilogueOutputOp; + using ThreadblockSwizzle = typename GemmKernel::ThreadblockSwizzle; + using Operator = typename GemmKernel::Operator; + + /// Argument structure + using Arguments = typename GemmKernel::Arguments; + + protected: + /// Kernel parameters object + typename GemmKernel::Params params_; + + protected: + /// Private helper to obtain the grid dimensions with fix-up for split-K + static void get_grid_shape_(gemm::GemmCoord& grid_tiled_shape, int& gemm_k_size, Arguments const& args) { + // Determine grid shape + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + gemm_k_size = args.problem_size.k(); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + } + + public: + /// Constructs the GEMM. + GemmUniversalBaseCompat() {} + + /// Determines whether the GEMM can execute the given problem. + static Status can_implement(Arguments const& args) { + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + ThreadblockSwizzle threadblock_swizzle; + dim3 grid = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + uint32_t const kGridYZMax = ((1 << (sizeof(uint16_t) * 8)) - 1); + + if (!(grid.y <= kGridYZMax && grid.z <= kGridYZMax)) { + return Status::kErrorInvalidProblem; + } + + return GemmKernel::can_implement(args); + } + + /// Gets the workspace size + static size_t get_workspace_size(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_workspace_size()"); + + size_t workspace_bytes = 0; + + // Determine grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + if (args.mode == GemmUniversalMode::kGemmSplitKParallel) { + // Split-K parallel always requires a temporary workspace + workspace_bytes = sizeof(ElementC) * size_t(args.batch_stride_D) * size_t(grid_tiled_shape.k()); + } else if (args.mode == GemmUniversalMode::kGemm && grid_tiled_shape.k() > 1) { + // Serial split-K only requires a temporary workspace if the number of partitions along the + // GEMM K dimension is greater than one. + workspace_bytes = sizeof(int) * size_t(grid_tiled_shape.m()) * size_t(grid_tiled_shape.n()); + } + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + workspace_bytes += GemmKernel::get_extra_workspace_size(args, grid_tiled_shape); + + return workspace_bytes; + } + + /// Computes the grid shape + static dim3 get_grid_shape(Arguments const& args) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::get_grid_shape()"); + + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + dim3 result = threadblock_swizzle.get_grid_shape(grid_tiled_shape); + + CUTLASS_TRACE_HOST( + " grid_tiled_shape: " << grid_tiled_shape << "\n" + << " result = {" << result << "}"); + + return result; + } + + /// Computes the maximum number of active blocks per multiprocessor + static int maximum_active_blocks(int smem_capacity = -1) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::maximum_active_blocks()"); + + int max_active_blocks = -1; + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + CUTLASS_TRACE_HOST(" smem_size: " << smem_size << " bytes"); + + if (smem_size <= (48 << 10)) { + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, smem_size); + + if (result == cudaSuccess) { + CUTLASS_TRACE_HOST(" max_active_blocks: " << max_active_blocks); + return max_active_blocks; + } + } else { + // Query assuming zero shared memory then compute occupancy limit based on SMEM + cudaError_t result = cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &max_active_blocks, Kernel, GemmKernel::kThreadCount, 0); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST( + " cudaOccupancyMaxActiveBlocksPerMultiprocessor() returned error " << cudaGetErrorString(result)); + + return -1; + } + + if (smem_capacity < 0) { + int device_idx = 0; + result = cudaGetDevice(&device_idx); + + if (result != cudaSuccess) { + return -1; + } + + cudaDeviceProp properties; + result = cudaGetDeviceProperties(&properties, device_idx); + + if (result != cudaSuccess) { + return -1; + } + + smem_capacity = static_cast(properties.sharedMemPerMultiprocessor); + } + + int occupancy = std::min(max_active_blocks, smem_capacity / smem_size); + + CUTLASS_TRACE_HOST(" occupancy: " << occupancy); + + return occupancy; + } + + CUTLASS_TRACE_HOST(" returning internal error"); + + return -1; + } + + /// Initializes GEMM state from arguments. + Status initialize(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST( + "GemmUniversalBaseCompat::initialize() - workspace " << workspace + << ", stream: " << (stream ? "non-null" : "null")); + + size_t workspace_bytes = get_workspace_size(args); + + CUTLASS_TRACE_HOST(" workspace_bytes: " << workspace_bytes); + + if (workspace_bytes) { + if (!workspace) { + CUTLASS_TRACE_HOST(" error: device workspace must not be null"); + + return Status::kErrorWorkspaceNull; + } + + if (args.mode == GemmUniversalMode::kGemm) { + CUTLASS_TRACE_HOST(" clearing device workspace"); + cudaError_t result = cudaMemsetAsync(workspace, 0, workspace_bytes, stream); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" cudaMemsetAsync() returned error " << cudaGetErrorString(result)); + + return Status::kErrorInternal; + } + } + } + + // Get CUDA grid shape + cutlass::gemm::GemmCoord grid_tiled_shape; + int gemm_k_size = 0; + + get_grid_shape_(grid_tiled_shape, gemm_k_size, args); + + // Initialize the Params structure + params_ = typename GemmKernel::Params(args, grid_tiled_shape, gemm_k_size, static_cast(workspace)); + + // Specify shared memory capacity for kernel. + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + if (smem_size >= (48 << 10)) { + cudaError_t result = + cudaFuncSetAttribute(Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size); + + if (result != cudaSuccess) { + return Status::kErrorInternal; + } + } + + return Status::kSuccess; + } + + /// Lightweight update given a subset of arguments + Status update(Arguments const& args, void* workspace = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat()::update() - workspace: " << workspace); + + size_t workspace_bytes = get_workspace_size(args); + + if (workspace_bytes && !workspace) { + return Status::kErrorWorkspaceNull; + } + + params_.update(args, workspace); + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status run(cudaStream_t stream = nullptr) { + CUTLASS_TRACE_HOST("GemmUniversalBaseCompat::run()"); + + // + // Configure grid and block dimensions + // + + ThreadblockSwizzle threadblock_swizzle; + + dim3 grid = threadblock_swizzle.get_grid_shape(params_.grid_tiled_shape); + dim3 block(GemmKernel::kThreadCount, 1, 1); + + int smem_size = int(sizeof(typename GemmKernel::SharedStorage)); + + // + // Launch kernel + // + + CUTLASS_TRACE_HOST(" grid: (" << grid << "), block: (" << block << "), SMEM: " << smem_size << " bytes"); + + // Launch + cutlass::Kernel<<>>(params_); + + // + // Query for errors + // + cudaError_t result = cudaGetLastError(); + + if (result != cudaSuccess) { + CUTLASS_TRACE_HOST(" grid launch failed with error " << cudaGetErrorString(result)); + return Status::kErrorInternal; + } + + return Status::kSuccess; + } + + /// Runs the kernel using initialized state. + Status operator()(cudaStream_t stream = nullptr) { + return run(stream); + } + + /// Runs the kernel using initialized state. + Status operator()(Arguments const& args, void* workspace = nullptr, cudaStream_t stream = nullptr) { + Status status = initialize(args, workspace, stream); + + if (status == Status::kSuccess) { + status = run(stream); + } + + return status; + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace device +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h new file mode 100644 index 000000000..905d11ba2 --- /dev/null +++ b/sgl-kernel/csrc/cutlass_extensions/gemm/gemm_with_epilogue_visitor.h @@ -0,0 +1,492 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/be1788106245496872d18e702978e59b6bfd50e0/cpp/tensorrt_llm/cutlass_extensions/include/cutlass_extensions/gemm/kernel/gemm_with_epilogue_visitor.h + +#pragma once + +#include +#include +#include +#include +#include + +///////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cutlass { +namespace gemm { +namespace kernel { + +///////////////////////////////////////////////////////////////////////////////////////////////// + +template < + typename Mma_, ///! Threadblock-scoped matrix multiply-accumulate + typename Epilogue_, ///! Epilogue + typename ThreadblockSwizzle_ ///! Threadblock swizzling function + > +struct GemmWithEpilogueVisitor { + public: + using Mma = Mma_; + using Epilogue = Epilogue_; + using EpilogueVisitor = typename Epilogue::Visitor; + using ThreadblockSwizzle = ThreadblockSwizzle_; + + using ElementA = typename Mma::IteratorA::Element; + using LayoutA = typename Mma::IteratorA::Layout; + using TensorRefA = TensorRef; + + using ElementB = typename Mma::IteratorB::Element; + using LayoutB = typename Mma::IteratorB::Layout; + using TensorRefB = TensorRef; + + using ElementCompute = typename EpilogueVisitor::ElementCompute; + using LayoutAlphaCol = cutlass::layout::RowMajor; + using LayoutAlphaRow = cutlass::layout::ColumnMajor; + using TensorRefAlphaCol = TensorRef; + using TensorRefAlphaRow = TensorRef; + + using ElementC = typename EpilogueVisitor::ElementOutput; + using LayoutC = typename Epilogue::Layout; + using TensorRefC = TensorRef; + + static ComplexTransform const kTransformA = Mma::kTransformA; + static ComplexTransform const kTransformB = Mma::kTransformB; + using Operator = typename Mma::Operator; + + using OperatorClass = typename Mma::Operator::OperatorClass; + using ThreadblockShape = typename Mma::Shape; + using WarpShape = typename Mma::Operator::Shape; + using InstructionShape = typename Mma::Policy::Operator::InstructionShape; + using ArchTag = typename Mma::ArchTag; + using EpilogueOutputOp = + typename Epilogue::Visitor::ElementwiseFunctor; // Define type so GemmUniversalBase doesn't complain + + static int const kStages = Mma::kStages; + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::kElementsPerAccess; + + /// Warp count (concept: GemmShape) + using WarpCount = typename Mma::WarpCount; + static int const kThreadCount = 32 * WarpCount::kCount; + + /// Split-K preserves splits that are 128b aligned + static int const kSplitKAlignment = const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value); + + // + // Structures + // + + /// Argument structure + struct Arguments { + // + // Data members + // + + GemmUniversalMode mode; + GemmCoord problem_size; + int batch_count; + + TensorRefA ref_A; + TensorRefB ref_B; + TensorRefAlphaCol ref_alpha_col; + TensorRefAlphaRow ref_alpha_row; + TensorRefC ref_C; + TensorRefC ref_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + int64_t batch_stride_D; + + typename EpilogueVisitor::Arguments epilogue_visitor; + + // + // Methods + // + + Arguments() : mode(GemmUniversalMode::kGemm), batch_count(1) {} + + /// constructs an arguments structure + Arguments( + GemmCoord problem_size_, + TensorRefA ref_A_, + TensorRefB ref_B_, + TensorRefAlphaCol ref_alpha_col_, + TensorRefAlphaRow ref_alpha_row_, + TensorRefC ref_C_, + TensorRefC ref_D_, + typename EpilogueVisitor::Arguments epilogue_visitor_) + : mode(GemmUniversalMode::kGemm), + problem_size(problem_size_), + batch_count(1), + ref_A(ref_A_), + ref_B(ref_B_), + ref_alpha_col(ref_alpha_col_), + ref_alpha_row(ref_alpha_row_), + ref_C(ref_C_), + ref_D(ref_D_), + batch_stride_A(0), + batch_stride_B(0), + batch_stride_D(0), + epilogue_visitor(epilogue_visitor_) {} + }; + + // + // Structure for precomputing values in host memory and passing to kernels + // + + /// Parameters structure + struct Params { + cutlass::gemm::GemmCoord problem_size; + cutlass::gemm::GemmCoord grid_tiled_shape; + int swizzle_log_tile; + + typename Mma::IteratorA::Params params_A; + typename Mma::IteratorB::Params params_B; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Params params_alpha_row; + typename EpilogueVisitor::OutputTileIterator::Params params_C; + typename EpilogueVisitor::OutputTileIterator::Params params_D; + + GemmUniversalMode mode; + int batch_count; + int gemm_k_size; + + void* ptr_A; + void* ptr_B; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_col; + typename EpilogueVisitor::ScaleTileIterator::Element* ptr_alpha_row; + ElementC* ptr_C; + ElementC* ptr_D; + + int64_t batch_stride_A; + int64_t batch_stride_B; + + typename EpilogueVisitor::Params epilogue_visitor; + + // + // Methods + // + + CUTLASS_HOST_DEVICE + Params() + : swizzle_log_tile(0), + params_A(0), + params_B(0), + params_alpha_col(0), + params_C(0), + params_D(0), + batch_count(0), + gemm_k_size(0), + mode(cutlass::gemm::GemmUniversalMode::kGemm), + ptr_A(nullptr), + ptr_B(nullptr), + ptr_alpha_col(nullptr), + ptr_alpha_row(nullptr), + ptr_C(nullptr), + ptr_D(nullptr), + batch_stride_A(0), + batch_stride_B(0) {} + + Params(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape_, int gemm_k_size_, int* workspace_) + : problem_size(args.problem_size), + swizzle_log_tile(0), + params_A(args.ref_A.layout()), + params_B(args.ref_B.layout()), + params_alpha_col(args.ref_alpha_col.layout()), + params_alpha_row(args.ref_alpha_col.layout()), + params_C(args.ref_C.layout()), + params_D(args.ref_D.layout()), + mode(args.mode), + batch_count(args.batch_count), + gemm_k_size(args.problem_size.k()), + ptr_A(args.ref_A.data()), + ptr_B(args.ref_B.data()), + ptr_alpha_col(args.ref_alpha_col.data()), + ptr_alpha_row(args.ref_alpha_row.data()), + ptr_C(args.ref_C.data()), + ptr_D(args.ref_D.data()), + batch_stride_A(args.batch_stride_A), + batch_stride_B(args.batch_stride_B), + epilogue_visitor(args.epilogue_visitor) { + ThreadblockSwizzle threadblock_swizzle; + + grid_tiled_shape = threadblock_swizzle.get_tiled_shape( + args.problem_size, {ThreadblockShape::kM, ThreadblockShape::kN, ThreadblockShape::kK}, args.batch_count); + + if (args.mode == GemmUniversalMode::kGemm || args.mode == GemmUniversalMode::kGemmSplitKParallel) { + int const kAlignK = + const_max(const_max(128 / sizeof_bits::value, 128 / sizeof_bits::value), 1); + + gemm_k_size = round_up(ceil_div(args.problem_size.k(), args.batch_count), kAlignK); + + if (gemm_k_size) { + grid_tiled_shape.k() = ceil_div(args.problem_size.k(), gemm_k_size); + } + } + + swizzle_log_tile = threadblock_swizzle.get_log_tile(grid_tiled_shape); + } + }; + + /// Shared memory storage structure + union SharedStorage { + typename Mma::SharedStorage main_loop; + + struct { + typename Epilogue::SharedStorage epilogue; + typename EpilogueVisitor::SharedStorage visitor; + } epilogue; + }; + + public: + // + // Methods + // + + CUTLASS_DEVICE + GemmWithEpilogueVisitor() {} + + /// Determines whether kernel satisfies alignment + static Status can_implement(cutlass::gemm::GemmCoord const& problem_size) { + CUTLASS_TRACE_HOST("GemmWithEpilogueVisitor::can_implement()"); + + static int const kAlignmentA = Mma::IteratorA::AccessType::kElements; + static int const kAlignmentB = Mma::IteratorB::AccessType::kElements; + static int const kAlignmentC = EpilogueVisitor::OutputTileIterator::kElementsPerAccess; + + bool isAMisaligned = false; + bool isBMisaligned = false; + bool isCMisaligned = false; + + if (platform::is_same::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } else if (platform::is_same::value) { + isAMisaligned = problem_size.m() % kAlignmentA; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isAMisaligned = problem_size.k() % kAlignmentA; + } + + if (platform::is_same::value) { + isBMisaligned = problem_size.n() % kAlignmentB; + } else if (platform::is_same::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isBMisaligned = problem_size.k() % kAlignmentB; + } + + if (platform::is_same::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } else if (platform::is_same::value) { + isCMisaligned = problem_size.m() % kAlignmentC; + } else if ( + platform::is_same>::value || + platform::is_same>::value) { + isCMisaligned = problem_size.n() % kAlignmentC; + } + + if (isAMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for A operand"); + return Status::kErrorMisalignedOperand; + } + + if (isBMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for B operand"); + return Status::kErrorMisalignedOperand; + } + + if (isCMisaligned) { + CUTLASS_TRACE_HOST(" returning kErrorMisalignedOperand for C operand"); + return Status::kErrorMisalignedOperand; + } + + CUTLASS_TRACE_HOST(" returning kSuccess"); + + return Status::kSuccess; + } + + static Status can_implement(Arguments const& args) { + return can_implement(args.problem_size); + } + + static size_t get_extra_workspace_size(Arguments const& args, cutlass::gemm::GemmCoord const& grid_tiled_shape) { + return 0; + } + +#define SPLIT_K_ENABLED 1 + + /// Executes one GEMM + CUTLASS_DEVICE + void run_kernel_(Params const& params, SharedStorage& shared_storage) { + // Compute threadblock location + ThreadblockSwizzle threadblock_swizzle; + + cutlass::gemm::GemmCoord threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // Early exit if CTA is out of range + if (params.grid_tiled_shape.m() <= threadblock_tile_offset.m() || + params.grid_tiled_shape.n() <= threadblock_tile_offset.n()) { + return; + } + + int offset_k = 0; + int problem_size_k = params.problem_size.k(); + + ElementA* ptr_A = static_cast(params.ptr_A); + ElementB* ptr_B = static_cast(params.ptr_B); + +#if SPLIT_K_ENABLED + // + // Fetch pointers based on mode. + // + if (params.mode == GemmUniversalMode::kGemm || params.mode == GemmUniversalMode::kGemmSplitKParallel) { + if (threadblock_tile_offset.k() + 1 < params.grid_tiled_shape.k()) { + problem_size_k = (threadblock_tile_offset.k() + 1) * params.gemm_k_size; + } + + offset_k = threadblock_tile_offset.k() * params.gemm_k_size; + } else if (params.mode == GemmUniversalMode::kBatched) { + ptr_A += threadblock_tile_offset.k() * params.batch_stride_A; + ptr_B += threadblock_tile_offset.k() * params.batch_stride_B; + } else if (params.mode == GemmUniversalMode::kArray) { + ptr_A = static_cast(params.ptr_A)[threadblock_tile_offset.k()]; + ptr_B = static_cast(params.ptr_B)[threadblock_tile_offset.k()]; + } +#endif + + // Compute initial location in logical coordinates + cutlass::MatrixCoord tb_offset_A{ + threadblock_tile_offset.m() * Mma::Shape::kM, + offset_k, + }; + + cutlass::MatrixCoord tb_offset_B{offset_k, threadblock_tile_offset.n() * Mma::Shape::kN}; + + // Compute position within threadblock + int thread_idx = threadIdx.x; + + // Construct iterators to A and B operands + typename Mma::IteratorA iterator_A( + params.params_A, ptr_A, {params.problem_size.m(), problem_size_k}, thread_idx, tb_offset_A); + + typename Mma::IteratorB iterator_B( + params.params_B, ptr_B, {problem_size_k, params.problem_size.n()}, thread_idx, tb_offset_B); + + // Broadcast the warp_id computed by lane 0 to ensure dependent code + // is compiled as warp-uniform. + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + int lane_idx = threadIdx.x % 32; + + // + // Main loop + // + + // Construct thread-scoped matrix multiply + Mma mma(shared_storage.main_loop, thread_idx, warp_idx, lane_idx); + + typename Mma::FragmentC accumulators; + + accumulators.clear(); + + // Compute threadblock-scoped matrix multiply-add + int gemm_k_iterations = (problem_size_k - offset_k + Mma::Shape::kK - 1) / Mma::Shape::kK; + + // Compute threadblock-scoped matrix multiply-add + mma(gemm_k_iterations, accumulators, iterator_A, iterator_B, accumulators); + + // + // Masked tile iterators constructed from members + // + + threadblock_tile_offset = threadblock_swizzle.get_tile_offset(params.swizzle_log_tile); + + // assume identity swizzle + MatrixCoord threadblock_offset( + threadblock_tile_offset.m() * Mma::Shape::kM, threadblock_tile_offset.n() * Mma::Shape::kN); + + int block_idx = threadblock_tile_offset.m() + threadblock_tile_offset.n() * params.grid_tiled_shape.m(); + + // + // Construct the epilogue visitor + // + + bool with_bias = true; + if (params.ptr_C == nullptr) { + with_bias = false; + } + + EpilogueVisitor epilogue_visitor( + params.epilogue_visitor, + shared_storage.epilogue.visitor, + params.problem_size.mn(), + thread_idx, + warp_idx, + lane_idx, + params.params_alpha_col, + params.params_C, + params.params_D, + with_bias, + true, + true, + params.ptr_alpha_row, + params.ptr_alpha_col, + params.ptr_C, + params.ptr_D, + threadblock_offset, + blockIdx.y * params.problem_size.m()); + + if (params.mode == GemmUniversalMode::kGemm) { + // Indicate which position in a serial reduction the output operator is currently updating + epilogue_visitor.set_k_partition(threadblock_tile_offset.k(), params.grid_tiled_shape.k()); + } else if (params.mode == GemmUniversalMode::kBatched || params.mode == GemmUniversalMode::kArray) { + epilogue_visitor.set_batch_index(threadblock_tile_offset.k()); + } + + // Construct the epilogue + Epilogue epilogue(shared_storage.epilogue.epilogue, thread_idx, warp_idx, lane_idx); + + // Execute the epilogue operator to update the destination tensor. + epilogue(epilogue_visitor, accumulators); + } + + template + CUTLASS_DEVICE void run_kernel(Params const& params, SharedStorage& shared_storage) { + if constexpr (platform::is_same::value) { + run_kernel_(params, shared_storage); + } else { + CUTLASS_NOT_IMPLEMENTED(); + } + } + + /// Executes one GEMM + CUTLASS_DEVICE + void operator()(Params const& params, SharedStorage& shared_storage) { + run_kernel(params, shared_storage); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////////////// + +} // namespace kernel +} // namespace gemm +} // namespace cutlass + +///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/sgl-kernel/csrc/elementwise/activation.cu b/sgl-kernel/csrc/elementwise/activation.cu new file mode 100644 index 000000000..43617f87f --- /dev/null +++ b/sgl-kernel/csrc/elementwise/activation.cu @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#ifndef USE_ROCM + +#include + +#include "utils.h" + +#else +#include "hip/hip_act_and_mul.cuh" +#endif + +// Adapted from flashinfer activation +// https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/csrc/activation.cu#L44 + +namespace detail { + +template +__device__ __forceinline__ float to_f32(const T& x) { +#if USE_ROCM + return castToFloat(x); +#else + return static_cast(x); +#endif +} + +template +__device__ __forceinline__ T from_f32(float f32) { +#if USE_ROCM + return castFromFloat(f32); +#else + return static_cast(f32); +#endif +} + +} // namespace detail + +template +__device__ __forceinline__ T silu(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val))); +} + +template +__device__ __forceinline__ T gelu(const T& x) { + constexpr float kAlpha = M_SQRT1_2; + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val * (0.5f * (1.0f + erf(f32_val * kAlpha)))); +} + +// gelu_quick(x) = x * torch.sigmoid(1.702 * x) +template +__device__ __forceinline__ T gelu_quick_act(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val * 1.702f))); +} + +template +__device__ __forceinline__ T gelu_tanh(const T& x) { + constexpr float kAlpha = 0.044715f; + constexpr float kBeta = 0.7978845608028654f; + float f32_val = detail::to_f32(x); + const float cdf = 0.5f * (1.0f + tanhf((kBeta * (f32_val + kAlpha * f32_val * f32_val * f32_val)))); + return detail::from_f32(f32_val * cdf); +} + +void silu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); +#if USE_ROCM + sgl_hip::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + flashinfer::activation::act_and_mul_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + + return true; + }); +} + +#if USE_ROCM +void gelu_quick(at::Tensor& out, const at::Tensor& input) { + int d = input.size(-1); + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(std::min(d / vec_size, 1024U)); + sgl_hip::activation::act_only_kernel + <<>>(static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} +#endif diff --git a/sgl-kernel/csrc/elementwise/activation.hip b/sgl-kernel/csrc/elementwise/activation.hip new file mode 100644 index 000000000..6ea8ca965 --- /dev/null +++ b/sgl-kernel/csrc/elementwise/activation.hip @@ -0,0 +1,173 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#ifndef USE_ROCM + +#include + +#include "utils_hip.h" + +#else +#include "hip/hip_act_and_mul_hip.cuh" +#endif + +// Adapted from flashinfer activation +// https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/csrc/activation.cu#L44 + +namespace detail { + +template +__device__ __forceinline__ float to_f32(const T& x) { +#if USE_ROCM + return castToFloat(x); +#else + return static_cast(x); +#endif +} + +template +__device__ __forceinline__ T from_f32(float f32) { +#if USE_ROCM + return castFromFloat(f32); +#else + return static_cast(f32); +#endif +} + +} // namespace detail + +template +__device__ __forceinline__ T silu(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val))); +} + +template +__device__ __forceinline__ T gelu(const T& x) { + constexpr float kAlpha = M_SQRT1_2; + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val * (0.5f * (1.0f + erf(f32_val * kAlpha)))); +} + +// gelu_quick(x) = x * torch.sigmoid(1.702 * x) +template +__device__ __forceinline__ T gelu_quick_act(const T& x) { + float f32_val = detail::to_f32(x); + return detail::from_f32(f32_val / (1.0f + expf(-f32_val * 1.702f))); +} + +template +__device__ __forceinline__ T gelu_tanh(const T& x) { + constexpr float kAlpha = 0.044715f; + constexpr float kBeta = 0.7978845608028654f; + float f32_val = detail::to_f32(x); + const float cdf = 0.5f * (1.0f + tanhf((kBeta * (f32_val + kAlpha * f32_val * f32_val * f32_val)))); + return detail::from_f32(f32_val * cdf); +} + +void silu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(::min(d / vec_size, 1024U)); +#if USE_ROCM + hipLaunchKernelGGL(( sgl_hip::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + hipLaunchKernelGGL(( flashinfer::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(::min(d / vec_size, 1024U)); +#if USE_ROCM + hipLaunchKernelGGL(( sgl_hip::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + hipLaunchKernelGGL(( flashinfer::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + return true; + }); +} + +void gelu_and_mul(at::Tensor& out, at::Tensor& input) { + int d = input.size(-1) / 2; + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(::min(d / vec_size, 1024U)); +#if USE_ROCM + hipLaunchKernelGGL(( sgl_hip::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#else + hipLaunchKernelGGL(( flashinfer::activation::act_and_mul_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); +#endif + + return true; + }); +} + +#if USE_ROCM +void gelu_quick(at::Tensor& out, const at::Tensor& input) { + int d = input.size(-1); + int64_t num_tokens = input.numel() / input.size(-1); + dim3 grid(num_tokens); + + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(input)); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + uint32_t vec_size = 16 / sizeof(c_type); + dim3 block(::min(d / vec_size, 1024U)); + hipLaunchKernelGGL(( sgl_hip::activation::act_only_kernel) + , dim3(grid), dim3(block), 0, stream, static_cast(out.data_ptr()), static_cast(input.data_ptr()), d); + + return true; + }); +} +#endif diff --git a/sgl-kernel/csrc/elementwise/cast.cu b/sgl-kernel/csrc/elementwise/cast.cu new file mode 100644 index 000000000..a1ff8703f --- /dev/null +++ b/sgl-kernel/csrc/elementwise/cast.cu @@ -0,0 +1,171 @@ +#include "pytorch_extension_utils.h" + +template +struct ConvertToFP8 { + static __device__ __nv_fp8_storage_t convert_to_fp8(T value) { + return 0; + } +}; + +template <> +struct ConvertToFP8<__nv_bfloat16> { + static __device__ __nv_fp8_storage_t convert_to_fp8(__nv_bfloat16 value) { + return __nv_cvt_bfloat16raw_to_fp8(value, __NV_SATFINITE, __NV_E4M3); + } +}; + +template <> +struct ConvertToFP8 { + static __device__ __nv_fp8_storage_t convert_to_fp8(half value) { + return __nv_cvt_halfraw_to_fp8(value, __NV_SATFINITE, __NV_E4M3); + } +}; + +template +struct ConvertFromFloat { + static __device__ T convert_from_float(float value) { + return 0; + } +}; + +template <> +struct ConvertFromFloat<__nv_bfloat16> { + static __device__ __nv_bfloat16 convert_from_float(float value) { + return __float2bfloat16(value); + } +}; + +template <> +struct ConvertFromFloat { + static __device__ half convert_from_float(float value) { + return __float2half(value); + } +}; + +template +__global__ void fused_downcast_kernel( + const T* cache_k, + const T* cache_v, + const float* k_scale, + const float* v_scale, + __nv_fp8_storage_t* output_k, + __nv_fp8_storage_t* output_v, + const int input_sl, + const int head, + const int dim, + const T max_fp8, + const T min_fp8, + const int64_t mult, + const int64_t offset, + const int64_t* loc) { + // TODO: change name + int token_idx = blockIdx.x; + int thread_idx = threadIdx.x; + int total_threads = blockDim.x; + + T k_scale_val = ConvertFromFloat::convert_from_float(k_scale[0]); + T v_scale_val = ConvertFromFloat::convert_from_float(v_scale[0]); + + T k_scale_inv = static_cast(1.f) / k_scale_val; + T v_scale_inv = static_cast(1.f) / v_scale_val; + + auto clamp = [&](T val) { return val > max_fp8 ? max_fp8 : (min_fp8 > val ? min_fp8 : val); }; + + if (token_idx < input_sl) { + int out_seq_idx = loc[token_idx]; + +#pragma unroll + for (int i = thread_idx; i < head * dim; i += total_threads) { + int in_idx = token_idx * head * dim + i; + int out_idx = (out_seq_idx * mult + offset) * head * dim + i; + + T k_val = cache_k[in_idx] * k_scale_inv; + k_val = clamp(k_val); + output_k[out_idx] = ConvertToFP8::convert_to_fp8(k_val); + + T v_val = cache_v[in_idx] * v_scale_inv; + v_val = clamp(v_val); + output_v[out_idx] = ConvertToFP8::convert_to_fp8(v_val); + } + } +} + +template +void downcast_fp8_impl( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset, + cudaStream_t stream) { + CHECK_INPUT(k); + CHECK_INPUT(v); + CHECK_INPUT(k_out); + CHECK_INPUT(v_out); + CHECK_INPUT(k_scale); + CHECK_INPUT(v_scale); + CHECK_INPUT(loc); + + int64_t input_sl = k.size(0); + int64_t head = k.size(1); + int64_t dim = k.size(2); + + dim3 grid(input_sl * head); + int vec_size = 8; + dim3 block(std::min(int(dim) / vec_size, 1024)); + + const T max_fp8 = static_cast(448.0f); + const T min_fp8 = static_cast(-448.0f); + + fused_downcast_kernel<<>>( + static_cast(k.data_ptr()), + static_cast(v.data_ptr()), + static_cast(k_scale.data_ptr()), + static_cast(v_scale.data_ptr()), + static_cast<__nv_fp8_storage_t*>(k_out.data_ptr()), + static_cast<__nv_fp8_storage_t*>(v_out.data_ptr()), + input_sl, + head, + dim, + max_fp8, + min_fp8, + mult, + offset, + static_cast(loc.data_ptr())); + + cudaError_t status = cudaGetLastError(); + TORCH_CHECK(status == cudaSuccess, "Kernel launch failed: " + std::string(cudaGetErrorString(status))); +} + +void downcast_fp8( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset, + int64_t cuda_stream) { + CHECK_INPUT(k); + CHECK_INPUT(v); + CHECK_INPUT(k_out); + CHECK_INPUT(v_out); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + switch (k.scalar_type()) { + case at::ScalarType::BFloat16: + downcast_fp8_impl<__nv_bfloat16>(k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset, stream); + break; + case at::ScalarType::Half: + downcast_fp8_impl<__half>(k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset, stream); + break; + default: + TORCH_CHECK(false, "Unsupported input type for downcast_fp8. Expected bfloat16 or float16."); + } +} diff --git a/sgl-kernel/csrc/elementwise/concat_mla.cu b/sgl-kernel/csrc/elementwise/concat_mla.cu new file mode 100644 index 000000000..b6c236333 --- /dev/null +++ b/sgl-kernel/csrc/elementwise/concat_mla.cu @@ -0,0 +1,117 @@ +#include +#include +#include + +#include "pytorch_extension_utils.h" + +constexpr int NUM_LOCAL_HEADS = 128; +constexpr int QK_NOPE_HEAD_DIM = 128; +constexpr int QK_ROPE_HEAD_DIM = 64; +constexpr int K_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM; + +constexpr int HEAD_CHUNK_SIZE = 16; +constexpr int NUM_HEAD_CHUNKS = NUM_LOCAL_HEADS / HEAD_CHUNK_SIZE; + +__forceinline__ __device__ int get_lane_id() { + int lane_id; + asm("mov.s32 %0, %laneid;" : "=r"(lane_id)); + return lane_id; +} + +int ceil_div(int a, int b) { + return (a + b - 1) / b; +} + +__global__ void concat_mla_k_kernel( + nv_bfloat16* k, + nv_bfloat16* k_nope, + nv_bfloat16* k_rope, + const int num_tokens, + const int k_stride_0, + const int k_stride_1, + const int k_nope_stride_0, + const int k_nope_stride_1, + const int k_rope_stride_0) { + const int flat_warp_id = (blockIdx.x * blockDim.x + threadIdx.x) / 32; + const int token_id = flat_warp_id / NUM_HEAD_CHUNKS; + const int head_chunk_id = flat_warp_id % NUM_HEAD_CHUNKS; + const int lane_id = get_lane_id(); + + if (token_id >= num_tokens) { + return; + } + + using KNopeBufType = int2; + static_assert(sizeof(KNopeBufType) == QK_NOPE_HEAD_DIM * sizeof(k[0]) / 32); + KNopeBufType k_nope_buf[HEAD_CHUNK_SIZE]; + + using KRopeBufType = int; + static_assert(sizeof(KRopeBufType) == QK_ROPE_HEAD_DIM * sizeof(k[0]) / 32); + KRopeBufType k_rope_buf; + + { + const int* base_addr = reinterpret_cast(k_rope + token_id * k_rope_stride_0); + k_rope_buf = *(base_addr + lane_id); + } + +#pragma unroll + for (int i = 0; i < HEAD_CHUNK_SIZE; ++i) { + const int head_id = head_chunk_id * HEAD_CHUNK_SIZE + i; + const int2* base_addr = reinterpret_cast(k_nope + token_id * k_nope_stride_0 + head_id * k_nope_stride_1); + k_nope_buf[i] = *(base_addr + lane_id); + } + +#pragma unroll + for (int i = 0; i < HEAD_CHUNK_SIZE; ++i) { + const int head_id = head_chunk_id * HEAD_CHUNK_SIZE + i; + + { + int2* base_addr = reinterpret_cast(k + token_id * k_stride_0 + head_id * k_stride_1); + *(base_addr + lane_id) = k_nope_buf[i]; + } + { + int* base_addr = reinterpret_cast(k + token_id * k_stride_0 + head_id * k_stride_1 + QK_NOPE_HEAD_DIM); + *(base_addr + lane_id) = k_rope_buf; + } + } +} + +inline void check_tensor(const at::Tensor& t, int64_t shape0, int64_t shape1, int64_t shape2, c10::ScalarType dtype) { + TORCH_CHECK_EQ(t.dim(), 3); + TORCH_CHECK_EQ(t.size(0), shape0); + TORCH_CHECK_EQ(t.size(1), shape1); + TORCH_CHECK_EQ(t.size(2), shape2); + TORCH_CHECK_EQ(t.dtype(), dtype); + TORCH_CHECK(t.device().is_cuda()); + TORCH_CHECK_EQ(((int64_t)t.data_ptr()) % 16, 0); // alignment +} + +void concat_mla_k(at::Tensor k, at::Tensor k_nope, at::Tensor k_rope) { + const int num_tokens = k.size(0); + + check_tensor(k, num_tokens, NUM_LOCAL_HEADS, K_HEAD_DIM, at::kBFloat16); + check_tensor(k_nope, num_tokens, NUM_LOCAL_HEADS, QK_NOPE_HEAD_DIM, at::kBFloat16); + check_tensor(k_rope, num_tokens, 1, QK_ROPE_HEAD_DIM, at::kBFloat16); + TORCH_CHECK_EQ(k.stride(2), 1); + TORCH_CHECK_EQ(k_nope.stride(2), 1); + TORCH_CHECK_EQ(k_rope.stride(2), 1); + + const auto stream = at::cuda::getCurrentCUDAStream().stream(); + + constexpr int num_warps_per_block = 32; + const int grid_size = ceil_div(num_tokens * NUM_HEAD_CHUNKS, num_warps_per_block); + const int block_size = num_warps_per_block * 32; + + concat_mla_k_kernel<<>>( + reinterpret_cast(k.data_ptr()), + reinterpret_cast(k_nope.data_ptr()), + reinterpret_cast(k_rope.data_ptr()), + num_tokens, + k.stride(0), + k.stride(1), + k_nope.stride(0), + k_nope.stride(1), + k_rope.stride(0)); + cudaError_t err = cudaGetLastError(); + TORCH_CHECK(err == cudaSuccess, "CUDA kernel launch failed: ", cudaGetErrorString(err)); +} diff --git a/sgl-kernel/csrc/elementwise/copy.cu b/sgl-kernel/csrc/elementwise/copy.cu new file mode 100644 index 000000000..09719f510 --- /dev/null +++ b/sgl-kernel/csrc/elementwise/copy.cu @@ -0,0 +1,58 @@ +#include +#include +#include + +#include + +template +struct InputArray { + int values[N]; +}; + +template +__global__ void copy_to_gpu_no_ce_kernel(const InputArray input_array, int* output) { + int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < N) { + output[idx] = input_array.values[idx]; + } +} + +template +void copy_to_gpu_no_ce_impl(const at::Tensor& input, at::Tensor& output) { + TORCH_CHECK(input.dim() == 1, "input must be 1-D"); + TORCH_CHECK(static_cast(input.numel()) == N, "input numel must equal template N"); + TORCH_CHECK(input.is_contiguous(), "input must be contiguous"); + TORCH_CHECK(input.dtype() == torch::kInt32, "input dtype must be int32"); + + TORCH_CHECK(output.dim() == 1, "output dim"); + TORCH_CHECK(static_cast(output.numel()) == N, "output size"); + TORCH_CHECK(output.is_contiguous(), "output contiguous"); + TORCH_CHECK(output.dtype() == torch::kInt32, "output dtype"); + + TORCH_CHECK(input.device().is_cpu(), "input must be a CPU tensor"); + TORCH_CHECK(output.device().is_cuda(), "output must be a CUDA tensor"); + + InputArray input_array; + const int* input_ptr = input.data_ptr(); + for (int i = 0; i < N; ++i) + input_array.values[i] = input_ptr[i]; + + // may use multi thread blocks if performance bottleneck + dim3 grid(1); + dim3 block(static_cast(input.numel())); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + copy_to_gpu_no_ce_kernel<<>>(input_array, output.data_ptr()); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void copy_to_gpu_no_ce(const at::Tensor& input, at::Tensor& output) { + int N = static_cast(input.numel()); + // Can use macro if there are more N needed + if (N == 72) { + copy_to_gpu_no_ce_impl<72>(input, output); + } else if (N == 64) { + copy_to_gpu_no_ce_impl<64>(input, output); + } else { + TORCH_CHECK(false, "unexpected N"); + } +} diff --git a/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu b/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu new file mode 100644 index 000000000..02d5b20c9 --- /dev/null +++ b/sgl-kernel/csrc/elementwise/fused_add_rms_norm_kernel.cu @@ -0,0 +1,59 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include + +#include "utils.h" + +using namespace flashinfer; + +void sgl_fused_add_rmsnorm( + torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps, bool enable_pdl) { + CHECK_INPUT(input); + CHECK_INPUT(residual); + CHECK_INPUT(weight); + auto device = input.device(); + CHECK_EQ(residual.device(), device); + CHECK_EQ(weight.device(), device); + CHECK_DIM(2, input); // input: (batch_size, hidden_size) + CHECK_DIM(2, residual); // residual: (batch_size, hidden_size) + CHECK_DIM(1, weight); // weight: (hidden_size) + CHECK_EQ(input.size(0), residual.size(0)); + CHECK_EQ(input.size(1), residual.size(1)); + CHECK_EQ(input.size(1), weight.size(0)); + unsigned int batch_size = input.size(0); + unsigned int hidden_size = input.size(1); + + cudaStream_t torch_current_stream = at::cuda::getCurrentCUDAStream(); + // support float16, bfloat16 and float32 + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), c_type, [&] { + cudaError_t status = norm::FusedAddRMSNorm( + static_cast(input.data_ptr()), + static_cast(residual.data_ptr()), + static_cast(weight.data_ptr()), + batch_size, + hidden_size, + input.stride(0), + residual.stride(0), + eps, + enable_pdl, + torch_current_stream); + TORCH_CHECK( + status == cudaSuccess, "FusedAddRMSNorm failed with error code " + std::string(cudaGetErrorString(status))); + return true; + }); +} diff --git a/sgl-kernel/csrc/elementwise/pos_enc.cuh b/sgl-kernel/csrc/elementwise/pos_enc.cuh new file mode 100644 index 000000000..a2e4e2ebb --- /dev/null +++ b/sgl-kernel/csrc/elementwise/pos_enc.cuh @@ -0,0 +1,467 @@ +/* + * Copyright (c) 2023 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SGL_POS_ENC_CUH_ +#define SGL_POS_ENC_CUH_ + +#include // upstream + +namespace flashinfer { + +namespace kv_buffer_saver { + +template +__device__ __forceinline__ void prepare( + vec_t& v_vec, + IdType& kv_cache_offset, + DType* v, + IdType* kv_cache_loc, + uint32_t idx, + uint32_t tx, + uint32_t kv_head_idx, + size_t v_stride_n, + size_t v_stride_h) { + kv_cache_offset = kv_cache_loc[idx]; + + DType* v_ptr = v + get_elem_offset_impl(idx, kv_head_idx, 0, v_stride_n, v_stride_h); + v_vec.cast_load(v_ptr + tx * vec_size); +} + +template +__device__ __forceinline__ void save( + IdType& kv_cache_offset, + vec_t& k_vec, + vec_t& v_vec, + DType* k_buffer, + DType* v_buffer, + uint32_t idx, + uint32_t tx, + uint32_t kv_head_idx, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h) { + DType* k_buffer_ptr = + k_buffer + get_elem_offset_impl(kv_cache_offset, kv_head_idx, 0, k_buffer_stride_n, k_buffer_stride_h); + DType* v_buffer_ptr = + v_buffer + get_elem_offset_impl(kv_cache_offset, kv_head_idx, 0, v_buffer_stride_n, v_buffer_stride_h); + k_vec.cast_store(k_buffer_ptr + tx * vec_size); + v_vec.cast_store(v_buffer_ptr + tx * vec_size); +} + +} // namespace kv_buffer_saver + +template < + bool save_kv_cache, + bool interleave, + uint32_t head_dim, + uint32_t vec_size, + uint32_t bdx, + typename DType, + typename IdType> +__global__ void BatchQKApplyRotaryPosIdsCosSinCacheEnhancedHeadParallelismKernel( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* __restrict__ cos_sin_cache, + IdType* __restrict__ pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* __restrict__ kv_cache_loc) { + uint32_t bx = blockIdx.x, tx = threadIdx.x, ty = threadIdx.y; + uint32_t by = blockIdx.y; + const uint32_t bdy = blockDim.y; + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + vec_t cos, sin; + if (bx * bdy + ty < nnz) { + const uint32_t idx = bx * bdy + ty; + const IdType pos = pos_ids[idx]; + + const int half_rotary_dim = rotary_dim / 2; + + // 1. if interleave: + // - cos = cos_sin_cache[pos_id][tx * vec_size // 2] + // - sin = cos_sin_cache[pos_id][(rot_dim // 2) + tx * vec_size // 2] + // 2. if not interleave + // - cos = cos_cache[pos_id][(tx * vec_size) % (rot_dim // 2)] + // - sin = sin_cache[pos_id][(rot_dim // 2) + (tx * vec_size) % (rot_dim // 2)] + if (tx * vec_size < rotary_dim) { + int sin_offset = rotary_dim / 2; + int vec_idx; + if constexpr (interleave) { + vec_idx = (tx * vec_size) / 2; // Force integer division + } else { + vec_idx = (tx * vec_size) % half_rotary_dim; // Use half_rotary_dim + } + cos.load(cos_sin_cache + (pos * rotary_dim) + vec_idx); + sin.load(cos_sin_cache + (pos * rotary_dim) + (sin_offset + vec_idx)); + } + + if (by < num_qo_heads) { + uint32_t qo_head_idx = by; + DType* q_ptr = q + get_elem_offset_impl(idx, qo_head_idx, 0, q_stride_n, q_stride_h); + DType* q_rope_ptr = q_rope + get_elem_offset_impl(idx, qo_head_idx, 0, q_rope_stride_n, q_rope_stride_h); + vec_t q_vec; + if constexpr (interleave) { + q_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(q_ptr, cos, sin, rotary_dim); + } else { + q_vec = vec_apply_llama_rope_cos_sin(q_ptr, cos, sin, rotary_dim); + } + q_vec.cast_store(q_rope_ptr + tx * vec_size); + } else { + uint32_t kv_head_idx = by - num_qo_heads; + DType* k_ptr = k + get_elem_offset_impl(idx, kv_head_idx, 0, k_stride_n, k_stride_h); + + DType* k_rope_ptr = k_rope + get_elem_offset_impl(idx, kv_head_idx, 0, k_rope_stride_n, k_rope_stride_h); + + vec_t v_vec; + IdType kv_cache_offset; + if constexpr (save_kv_cache) { + kv_buffer_saver::prepare( + v_vec, kv_cache_offset, v, kv_cache_loc, idx, tx, kv_head_idx, v_stride_n, v_stride_h); + } + + vec_t k_vec; + if constexpr (interleave) { + k_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(k_ptr, cos, sin, rotary_dim); + } else { + k_vec = vec_apply_llama_rope_cos_sin(k_ptr, cos, sin, rotary_dim); + } + k_vec.cast_store(k_rope_ptr + tx * vec_size); + + if constexpr (save_kv_cache) { + kv_buffer_saver::save( + kv_cache_offset, + k_vec, + v_vec, + k_buffer, + v_buffer, + idx, + tx, + kv_head_idx, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h); + } + } + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template < + bool save_kv_cache, + bool interleave, + uint32_t head_dim, + uint32_t vec_size, + uint32_t bdx, + typename DType, + typename IdType> +__global__ void BatchQKApplyRotaryPosIdsCosSinCacheEnhancedKernel( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* __restrict__ cos_sin_cache, + IdType* __restrict__ pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* __restrict__ kv_cache_loc) { + uint32_t bx = blockIdx.x, tx = threadIdx.x, ty = threadIdx.y; + const uint32_t bdy = blockDim.y; + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + vec_t cos, sin; + if (bx * bdy + ty < nnz) { + const uint32_t idx = bx * bdy + ty; + const IdType pos = pos_ids[idx]; + const int half_rotary_dim = rotary_dim / 2; + + // 1. if interleave: + // - cos = cos_sin_cache[pos_id][tx * vec_size // 2] + // - sin = cos_sin_cache[pos_id][(rot_dim // 2) + tx * vec_size // 2] + // 2. if not interleave + // - cos = cos_cache[pos_id][(tx * vec_size) % (rot_dim // 2)] + // - sin = sin_cache[pos_id][(rot_dim // 2) + (tx * vec_size) % (rot_dim // 2)] + if (tx * vec_size < rotary_dim) { + int sin_offset = rotary_dim / 2; + int vec_idx; + if constexpr (interleave) { + vec_idx = (tx * vec_size) / 2; // Force integer division + } else { + vec_idx = (tx * vec_size) % half_rotary_dim; // Use half_rotary_dim + } + cos.load(cos_sin_cache + (pos * rotary_dim) + vec_idx); + sin.load(cos_sin_cache + (pos * rotary_dim) + (sin_offset + vec_idx)); + } + + // not to unroll the loop, because num head might be large and might lead to worse performance +#pragma unroll 1 + for (uint32_t qo_head_idx = 0; qo_head_idx < num_qo_heads; ++qo_head_idx) { + DType* q_ptr = q + get_elem_offset_impl(idx, qo_head_idx, 0, q_stride_n, q_stride_h); + DType* q_rope_ptr = q_rope + get_elem_offset_impl(idx, qo_head_idx, 0, q_rope_stride_n, q_rope_stride_h); + vec_t q_vec; + if constexpr (interleave) { + q_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(q_ptr, cos, sin, rotary_dim); + } else { + q_vec = vec_apply_llama_rope_cos_sin(q_ptr, cos, sin, rotary_dim); + } + q_vec.cast_store(q_rope_ptr + tx * vec_size); + } + +#pragma unroll 1 + for (uint32_t kv_head_idx = 0; kv_head_idx < num_kv_heads; ++kv_head_idx) { + DType* k_ptr = k + get_elem_offset_impl(idx, kv_head_idx, 0, k_stride_n, k_stride_h); + + DType* k_rope_ptr = k_rope + get_elem_offset_impl(idx, kv_head_idx, 0, k_rope_stride_n, k_rope_stride_h); + + vec_t v_vec; + IdType kv_cache_offset; + if constexpr (save_kv_cache) { + kv_buffer_saver::prepare( + v_vec, kv_cache_offset, v, kv_cache_loc, idx, tx, kv_head_idx, v_stride_n, v_stride_h); + } + + vec_t k_vec; + if constexpr (interleave) { + k_vec = vec_apply_llama_rope_cos_sin_interleave_reuse_half(k_ptr, cos, sin, rotary_dim); + } else { + k_vec = vec_apply_llama_rope_cos_sin(k_ptr, cos, sin, rotary_dim); + } + k_vec.cast_store(k_rope_ptr + tx * vec_size); + + if constexpr (save_kv_cache) { + kv_buffer_saver::save( + kv_cache_offset, + k_vec, + v_vec, + k_buffer, + v_buffer, + idx, + tx, + kv_head_idx, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h); + } + } + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +#define DISPATCH_SAVE_KV_CACHE(save_kv_cache, SAVE_KV_CACHE, ...) \ + if (save_kv_cache) { \ + const bool SAVE_KV_CACHE = true; \ + __VA_ARGS__ \ + } else { \ + const bool SAVE_KV_CACHE = false; \ + __VA_ARGS__ \ + } + +template +cudaError_t BatchQKApplyRotaryPosIdsCosSinCacheEnhanced( + DType* q, + DType* k, + DType* v, + DType* q_rope, + DType* k_rope, + DType* k_buffer, + DType* v_buffer, + float* cos_sin_cache, + IdType* pos_ids, + uint32_t nnz, + uint32_t num_qo_heads, + uint32_t num_kv_heads, + uint32_t rotary_dim, + uint32_t head_dim, + size_t q_stride_n, + size_t q_stride_h, + size_t k_stride_n, + size_t k_stride_h, + size_t v_stride_n, + size_t v_stride_h, + size_t q_rope_stride_n, + size_t q_rope_stride_h, + size_t k_rope_stride_n, + size_t k_rope_stride_h, + size_t k_buffer_stride_n, + size_t k_buffer_stride_h, + size_t v_buffer_stride_n, + size_t v_buffer_stride_h, + IdType* kv_cache_loc, + bool interleave, + bool save_kv_cache, + bool enable_pdl, + cudaStream_t stream = nullptr) { + int dev_id = 0; + int num_sms = 0; + FLASHINFER_CUDA_CALL(cudaGetDevice(&dev_id)); + FLASHINFER_CUDA_CALL(cudaDeviceGetAttribute(&num_sms, cudaDevAttrMultiProcessorCount, dev_id)); + +#define LAUNCH_KERNEL_RAW(kernel_name) \ + do { \ + cudaLaunchConfig_t config = {}; \ + config.gridDim = nblks; \ + config.blockDim = nthrs; \ + config.dynamicSmemBytes = 0; \ + config.stream = stream; \ + cudaLaunchAttribute attrs[1] = {}; \ + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; \ + attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl; \ + config.numAttrs = 1; \ + config.attrs = attrs; \ + \ + FLASHINFER_CUDA_CALL(cudaLaunchKernelEx( \ + &config, \ + kernel_name, \ + q, \ + k, \ + v, \ + q_rope, \ + k_rope, \ + k_buffer, \ + v_buffer, \ + cos_sin_cache, \ + pos_ids, \ + nnz, \ + num_qo_heads, \ + num_kv_heads, \ + rotary_dim, \ + q_stride_n, \ + q_stride_h, \ + k_stride_n, \ + k_stride_h, \ + v_stride_n, \ + v_stride_h, \ + q_rope_stride_n, \ + q_rope_stride_h, \ + k_rope_stride_n, \ + k_rope_stride_h, \ + k_buffer_stride_n, \ + k_buffer_stride_h, \ + v_buffer_stride_n, \ + v_buffer_stride_h, \ + kv_cache_loc)); \ + } while (0) + + DISPATCH_SAVE_KV_CACHE(save_kv_cache, SAVE_KV_CACHE, { + DISPATCH_INTERLEAVE(interleave, INTERLEAVE, { + DISPATCH_HEAD_DIM(head_dim, HEAD_DIM, { + // operate on 16 Bytes at a time + constexpr uint32_t vec_size = std::max(16 / sizeof(DType), HEAD_DIM / 32); + // how many threads needed per head_dim + constexpr uint32_t bdx = HEAD_DIM / vec_size; + // how many threads needed per block + uint32_t num_threads = std::max(128U, bdx); + // how many tokens can we process in a block + uint32_t bdy = num_threads / bdx; + // how many blocks needed to process all tokens + uint32_t nblks_x = (nnz + bdy - 1) / bdy; + + auto kernel_0 = BatchQKApplyRotaryPosIdsCosSinCacheEnhancedKernel< + SAVE_KV_CACHE, + INTERLEAVE, + HEAD_DIM, + vec_size, + bdx, + DType, + IdType>; + + int num_blocks_per_sm_0 = 0; + FLASHINFER_CUDA_CALL(cudaOccupancyMaxActiveBlocksPerMultiprocessor( + &num_blocks_per_sm_0, kernel_0, num_threads, /*smem_size=*/0)); + uint32_t num_ctas_0 = num_blocks_per_sm_0 * num_sms; + + if ((nnz + bdy - 1) / bdy >= num_ctas_0) { + dim3 nblks(nblks_x); + dim3 nthrs(bdx, bdy); + LAUNCH_KERNEL_RAW(kernel_0); + } else { + dim3 nblks(nblks_x, num_qo_heads + num_kv_heads); + dim3 nthrs(bdx, bdy); + auto kernel_1 = BatchQKApplyRotaryPosIdsCosSinCacheEnhancedHeadParallelismKernel< + SAVE_KV_CACHE, + INTERLEAVE, + HEAD_DIM, + vec_size, + bdx, + DType, + IdType>; + LAUNCH_KERNEL_RAW(kernel_1); + } + }); + }); + }); +#undef LAUNCH_KERNEL_RAW + + return cudaSuccess; +} + +} // namespace flashinfer + +#endif // SGL_POS_ENC_CUH_ diff --git a/sgl-kernel/csrc/elementwise/rope.cu b/sgl-kernel/csrc/elementwise/rope.cu new file mode 100644 index 000000000..041558f61 --- /dev/null +++ b/sgl-kernel/csrc/elementwise/rope.cu @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "pos_enc.cuh" +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + bool enable_pdl, + int64_t cuda_stream, + const std::optional& v, + const std::optional& k_buffer, + const std::optional& v_buffer, + const std::optional& kv_cache_loc) { + CHECK_LAST_DIM_CONTIGUOUS(q); + CHECK_LAST_DIM_CONTIGUOUS(k); + + const bool save_kv_cache = v.has_value(); + if (save_kv_cache) { + TORCH_CHECK(v.has_value()); + TORCH_CHECK(k_buffer.has_value()); + TORCH_CHECK(v_buffer.has_value()); + TORCH_CHECK(kv_cache_loc.has_value()); + CHECK_LAST_DIM_CONTIGUOUS(v.value()); + CHECK_LAST_DIM_CONTIGUOUS(k_buffer.value()); + CHECK_LAST_DIM_CONTIGUOUS(v_buffer.value()); + CHECK_DIM(3, k_buffer.value()); // k_buffer: (nnz, H_K, D) + CHECK_DIM(3, v_buffer.value()); // v_buffer: (nnz, H_V, D) + CHECK_DIM(3, v.value()); // v: (nnz, H_V, D) + CHECK_DIM(1, kv_cache_loc.value()); // v: (n) + CHECK_INPUT(kv_cache_loc.value()); + } + size_t k_buffer_stride_n = save_kv_cache ? k_buffer->stride(0) : 0; + size_t k_buffer_stride_h = save_kv_cache ? k_buffer->stride(1) : 0; + size_t v_buffer_stride_n = save_kv_cache ? v_buffer->stride(0) : 0; + size_t v_buffer_stride_h = save_kv_cache ? v_buffer->stride(1) : 0; + size_t v_stride_n = save_kv_cache ? v->stride(0) : 0; + size_t v_stride_h = save_kv_cache ? v->stride(1) : 0; + auto kv_cache_loc_ptr = save_kv_cache ? static_cast(kv_cache_loc->data_ptr()) : nullptr; + + CHECK_INPUT(cos_sin_cache); + CHECK_INPUT(pos_ids); + auto device = q.device(); + CHECK_EQ(k.device(), device); + CHECK_EQ(cos_sin_cache.device(), device); + CHECK_EQ(pos_ids.device(), device); + CHECK_DIM(3, q); // q: (nnz, H_Q, D) + CHECK_DIM(3, k); // k: (nnz, H_K, D) + + // cos_sin_cache: (max_seq_len, R) + // First half of R is cos, second half is sin + CHECK_DIM(2, cos_sin_cache); + CHECK_EQ(q.size(0), k.size(0)); + CHECK_EQ(q.size(2), k.size(2)); + unsigned int rotary_dim = cos_sin_cache.size(1); + unsigned int num_qo_heads = q.size(1); + unsigned int num_kv_heads = k.size(1); + unsigned int head_dim = q.size(2); + unsigned int nnz = q.size(0); + size_t q_stride_n = q.stride(0); + size_t q_stride_h = q.stride(1); + size_t k_stride_n = k.stride(0); + size_t k_stride_h = k.stride(1); + + size_t q_rope_stride_n = q_rope.stride(0); + size_t q_rope_stride_h = q_rope.stride(1); + size_t k_rope_stride_n = k_rope.stride(0); + size_t k_rope_stride_h = k_rope.stride(1); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(q.scalar_type(), c_type, [&] { + // TODO temporarily only use `BatchQKApplyRotaryPosIdsCosSinCacheEnhanced` when save_kv_cache + // to avoid changing original code path; but this branch is feature-complete and should switch to this later + if (save_kv_cache) { + cudaError_t status = BatchQKApplyRotaryPosIdsCosSinCacheEnhanced( + static_cast(q.data_ptr()), + static_cast(k.data_ptr()), + save_kv_cache ? static_cast(v->data_ptr()) : nullptr, + static_cast(q_rope.data_ptr()), + static_cast(k_rope.data_ptr()), + save_kv_cache ? static_cast(k_buffer->data_ptr()) : nullptr, + save_kv_cache ? static_cast(v_buffer->data_ptr()) : nullptr, + static_cast(cos_sin_cache.data_ptr()), + static_cast(pos_ids.data_ptr()), + nnz, + num_qo_heads, + num_kv_heads, + rotary_dim, + head_dim, + q_stride_n, + q_stride_h, + k_stride_n, + k_stride_h, + v_stride_n, + v_stride_h, + q_rope_stride_n, + q_rope_stride_h, + k_rope_stride_n, + k_rope_stride_h, + k_buffer_stride_n, + k_buffer_stride_h, + v_buffer_stride_n, + v_buffer_stride_h, + kv_cache_loc_ptr, + interleave, + save_kv_cache, + enable_pdl, + stream); + TORCH_CHECK( + status == cudaSuccess, + "BatchQKApplyRotaryPosIdsCosSinCacheEnhanced failed with error code " + + std::string(cudaGetErrorString(status))); + } else { + TORCH_CHECK(!enable_pdl); + cudaError_t status = BatchQKApplyRotaryPosIdsCosSinCache( + static_cast(q.data_ptr()), + static_cast(k.data_ptr()), + static_cast(q_rope.data_ptr()), + static_cast(k_rope.data_ptr()), + static_cast(cos_sin_cache.data_ptr()), + static_cast(pos_ids.data_ptr()), + nnz, + num_qo_heads, + num_kv_heads, + rotary_dim, + head_dim, + q_stride_n, + q_stride_h, + k_stride_n, + k_stride_h, + q_rope_stride_n, + q_rope_stride_h, + k_rope_stride_n, + k_rope_stride_h, + interleave, + stream); + TORCH_CHECK( + status == cudaSuccess, + "BatchQKApplyRotaryPosIdsCosSinCache failed with error code " + std::string(cudaGetErrorString(status))); + } + return true; + }); +} diff --git a/sgl-kernel/csrc/flash_extension.cc b/sgl-kernel/csrc/flash_extension.cc new file mode 100644 index 000000000..f80db673f --- /dev/null +++ b/sgl-kernel/csrc/flash_extension.cc @@ -0,0 +1,63 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "sgl_flash_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From flash-attention + */ + m.def( + "fwd(Tensor! q," + " Tensor k," + " Tensor v," + " Tensor? k_new," + " Tensor? v_new," + " Tensor? q_v," + " Tensor!? out," + " Tensor? cu_seqlens_q," + " Tensor? cu_seqlens_k," + " Tensor? cu_seqlens_k_new," + " Tensor? seqused_q," + " Tensor? seqused_k," + " int? max_seqlen_q," + " int? max_seqlen_k," + " Tensor? page_table," + " Tensor? kv_batch_idx," + " Tensor? leftpad_k," + " Tensor? rotary_cos," + " Tensor? rotary_sin," + " Tensor? seqlens_rotary," + " Tensor? q_descale," + " Tensor? k_descale," + " Tensor? v_descale," + " float softmax_scale," + " bool is_causal," + " int window_size_left," + " int window_size_right," + " float softcap," + " bool is_rotary_interleaved," + " Tensor? scheduler_metadata," + " int num_splits," + " bool? pack_gqa," + " int sm_margin," + " Tensor? sinks) -> Tensor[]"); + m.impl("fwd", torch::kCUDA, make_pytorch_shim(&mha_fwd)); +} + +REGISTER_EXTENSION(flash_ops) diff --git a/sgl-kernel/csrc/gemm/awq_kernel.cu b/sgl-kernel/csrc/gemm/awq_kernel.cu new file mode 100644 index 000000000..eec933689 --- /dev/null +++ b/sgl-kernel/csrc/gemm/awq_kernel.cu @@ -0,0 +1,221 @@ +// Adapted from +// https://github.com/vllm-project/vllm/blob/eb59b5a6cba6727d3727c0372258db9002f687c1/csrc/quantization/awq/gemm_kernels.cu#L350 +#include +#include +#include +#include +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +#include +#endif + +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +__device__ uint4 dequantize_s4_to_fp16x2(uint32_t const& source) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750 + uint4 result; + + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = reinterpret_cast(source); + + // First, we extract the i4s and construct an intermediate fp16 number. + static constexpr uint32_t immLut = (0xf0 & 0xcc) | 0xaa; + static constexpr uint32_t BOTTOM_MASK = 0x000f000f; + static constexpr uint32_t TOP_MASK = 0x00f000f0; + static constexpr uint32_t I4s_TO_F16s_MAGIC_NUM = 0x64006400; + + // Note that the entire sequence only requires 1 shift instruction. This is + // thanks to the register packing format and the fact that we force our + // integers to be unsigned, and account for this in the fp16 subtractions. In + // addition, I exploit the fact that sub and fma have the same throughput in + // order to convert elt_23 and elt_67 to fp16 without having to shift them to + // the bottom bits before hand. + + // Shift right by 8 to now consider elt_45 and elt_67. Issue first to hide RAW + // dependency if we issue immediately before required. + const uint32_t top_i4s = i4s >> 8; + // Extract elt_01 - (i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[0]) + : "r"(i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_23 (i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[1]) + : "r"(i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_45 (top_i4s & 0x000f000f) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[2]) + : "r"(top_i4s), "n"(BOTTOM_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + // Extract elt_67 (top_i4s & 0x00f000f0) | 0x64006400 + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" + : "=r"(h[3]) + : "r"(top_i4s), "n"(TOP_MASK), "n"(I4s_TO_F16s_MAGIC_NUM), "n"(immLut)); + + // This is the half2 {1024, 1024} represented as an integer. + static constexpr uint32_t FP16_TOP_MAGIC_NUM = 0x64006400; + // This is the half2 {1 / 16, 1 / 16} represented as an integer. + static constexpr uint32_t ONE_SIXTEENTH = 0x2c002c00; + // This is the half2 {-64, -64} represented as an integer. + static constexpr uint32_t NEG_64 = 0xd400d400; + + // Finally, we construct the output numbers. + // Convert elt_01 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[0]) : "r"(h[0]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_23 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[1]) : "r"(h[1]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + // Convert elt_45 + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(h[2]) : "r"(h[2]), "r"(FP16_TOP_MAGIC_NUM)); + // Convert elt_67 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(h[3]) : "r"(h[3]), "r"(ONE_SIXTEENTH), "r"(NEG_64)); + + return result; +#else + assert(false); + return {}; +#endif +} + +__device__ uint4 dequantize_s4_to_bf16x2(uint32_t const& source) { +#if CUDA_VERSION >= 12000 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 + uint4 result; + uint32_t* h = reinterpret_cast(&result); + uint32_t const i4s = source; + + // Define masks and constants + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + static constexpr uint32_t MUL = 0x3F803F80; + static constexpr uint32_t ADD = 0xC300C300; + + int lo0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s, MASK, EX); + int hi0 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 4, MASK, EX); + int lo1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 8, MASK, EX); + int hi1 = lop3<(0xf0 & 0xcc) | 0xaa>(i4s >> 12, MASK, EX); + + nv_bfloat162* res = reinterpret_cast(h); + res[0] = __hfma2( + *reinterpret_cast(&lo0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[1] = __hfma2( + *reinterpret_cast(&hi0), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[2] = __hfma2( + *reinterpret_cast(&lo1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + res[3] = __hfma2( + *reinterpret_cast(&hi1), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + + return result; +#else + assert(false); + return {}; +#endif +#endif +} + +template +__global__ void __launch_bounds__(256) dequantize_weights( + int* __restrict__ qweight, + OutputT* __restrict__ scales, + int* __restrict__ qzeros, + OutputT* __restrict__ output, + int group_size, + int qweight_cols, + int qweight_rows) { +#if CUDA_VERSION >= 12000 + int col = blockIdx.x * blockDim.x + threadIdx.x; + int row = blockIdx.y * blockDim.y + threadIdx.y; + if (col >= qweight_cols || row >= qweight_rows) return; + + int group_idx = row / group_size; + int scale_offset = 8 * col + group_idx * qweight_cols * 8; + uint4 loaded_scale = *(uint4*)(scales + scale_offset); + + // Handle different data types + if constexpr (std::is_same::value) { + // FP16 path + uint4 zeros = dequantize_s4_to_fp16x2(qzeros[col + group_idx * qweight_cols]); + uint4 weight_fp16 = dequantize_s4_to_fp16x2(qweight[col + row * qweight_cols]); + + // Use PTX assembly for FP16 operations + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(zeros.x)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.x) : "r"(weight_fp16.x), "r"(loaded_scale.x)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(zeros.y)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.y) : "r"(weight_fp16.y), "r"(loaded_scale.y)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(zeros.z)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.z) : "r"(weight_fp16.z), "r"(loaded_scale.z)); + asm volatile("sub.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(zeros.w)); + asm volatile("mul.rn.f16x2 %0, %1, %2;\n" : "=r"(weight_fp16.w) : "r"(weight_fp16.w), "r"(loaded_scale.w)); + + OutputT* output_ptr = output + 8 * col + 8 * row * qweight_cols; + *(uint4*)output_ptr = weight_fp16; + } else if constexpr (std::is_same::value) { + uint4 weight_raw = dequantize_s4_to_bf16x2(qweight[col + row * qweight_cols]); + uint4 zero_raw = dequantize_s4_to_bf16x2(qzeros[col + group_idx * qweight_cols]); + uint4 scale_raw = *reinterpret_cast(scales + scale_offset); + + // Vectorized processing (each uint4 contains 4 nv_bfloat162) + nv_bfloat162* weight_vec = reinterpret_cast(&weight_raw); + nv_bfloat162* zero_vec = reinterpret_cast(&zero_raw); + nv_bfloat162* scale_vec = reinterpret_cast(&scale_raw); + +// Single instruction dual-channel operation +#pragma unroll + for (int i = 0; i < 4; ++i) { // uint4 = 4 * nv_bfloat162 + weight_vec[i] = __hmul2(__hsub2(weight_vec[i], zero_vec[i]), scale_vec[i]); + } + + // Directly store to OutputT array (guaranteed contiguous memory) + OutputT* output_ptr = output + 8 * col + row * qweight_cols * 8; + static_assert(sizeof(uint4) == 8 * sizeof(OutputT), "Memory layout mismatch"); + *reinterpret_cast(output_ptr) = weight_raw; + } +#endif +} + +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros) { + int qweight_rows = qweight.size(0); + int qweight_cols = qweight.size(1); + int group_size = qweight_rows / scales.size(0); + + int x_num_threads = 16; + int y_num_threads = 16; + int x_blocks = (qweight_cols + x_num_threads - 1) / x_num_threads; + int y_blocks = (qweight_rows + y_num_threads - 1) / y_num_threads; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(qweight)); + + auto output_tensor_options = torch::TensorOptions().dtype(scales.dtype()).device(scales.device()); + at::Tensor output = torch::empty({qweight_rows, qweight_cols * 8}, output_tensor_options); + + auto _qweight = reinterpret_cast(qweight.data_ptr()); + auto _zeros = reinterpret_cast(qzeros.data_ptr()); + + dim3 num_blocks(x_blocks, y_blocks); + dim3 threads_per_block(x_num_threads, y_num_threads); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (scales.scalar_type() == at::ScalarType::Half) { + auto _scales = reinterpret_cast(scales.data_ptr()); + auto _output = reinterpret_cast(output.data_ptr()); + dequantize_weights<<>>( + _qweight, _scales, _zeros, _output, group_size, qweight_cols, qweight_rows); + } else { + auto _scales = reinterpret_cast<__nv_bfloat16*>(scales.data_ptr()); + auto _output = reinterpret_cast<__nv_bfloat16*>(output.data_ptr()); + dequantize_weights<__nv_bfloat16><<>>( + _qweight, _scales, _zeros, _output, group_size, qweight_cols, qweight_rows); + } + + return output; +} diff --git a/sgl-kernel/csrc/gemm/bmm_fp8.cu b/sgl-kernel/csrc/gemm/bmm_fp8.cu new file mode 100644 index 000000000..4a82b4b27 --- /dev/null +++ b/sgl-kernel/csrc/gemm/bmm_fp8.cu @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include + +#include "pytorch_extension_utils.h" + +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle, + int64_t cuda_stream) { + TORCH_CHECK(A.is_cuda(), "A must be a CUDA tensor"); + TORCH_CHECK(B.is_cuda(), "B must be a CUDA tensor"); + TORCH_CHECK(D.is_cuda(), "D must be a CUDA tensor"); + TORCH_CHECK(A.dim() == 3, "Expected 3D tensor for A"); + TORCH_CHECK(B.dim() == 3, "Expected 3D tensor for B"); + TORCH_CHECK(D.dim() == 3, "Expected 3D tensor for D"); + TORCH_CHECK(A.size(0) == B.size(0) && A.size(0) == D.size(0), "Batch sizes must match"); + TORCH_CHECK(A.size(2) == B.size(1), "Incompatible matrix sizes"); + TORCH_CHECK(A.size(1) == D.size(1) && B.size(2) == D.size(2), "Result tensor has incorrect shape"); + + // PyTorch is row major by default. cuBLASLt is column major by default. + // We need row major D as expected. + // A ^ T * B = D, so D ^ T = B ^ T * A + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(B.scalar_type(), b_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(A.scalar_type(), a_type, [&] { + return DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(D.scalar_type(), d_type, [&] { + auto batch_size = A.size(0); + auto m = A.size(1); + auto k = A.size(2); + auto n = B.size(2); + + auto lt_handle = reinterpret_cast(cublas_handle); + auto stream = reinterpret_cast(cuda_stream); + + auto status = flashinfer::bmm_fp8::bmm_fp8_internal_cublaslt( + workspace_buffer.data_ptr(), + workspace_buffer.numel(), + static_cast(B.data_ptr()), + static_cast(A.data_ptr()), + static_cast(D.data_ptr()), + batch_size, + n, + m, + k, + static_cast(B_scale.data_ptr()), + static_cast(A_scale.data_ptr()), + lt_handle, + stream); + TORCH_CHECK( + status == CUBLAS_STATUS_SUCCESS, "bmm_fp8_internal_cublaslt failed: ", cublasGetStatusString(status)); + return true; + }); + }); + }); +} diff --git a/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu b/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu new file mode 100644 index 000000000..37aff1b9a --- /dev/null +++ b/sgl-kernel/csrc/gemm/dsv3_fused_a_gemm.cu @@ -0,0 +1,673 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/619709fc33bd5dc268f19d6a741fe7ed51c0f8f5/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3FusedAGemm.cu + * + * Copyright (c) 2019-2024, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2021, NAVER Corp. Authored by CLOVA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "utils.h" + +using bf16_t = __nv_bfloat16; + +__device__ void hmma_16_8_16_f32acc_bf16ab( + float (&d_reg)[4], const bf16_t (&a_reg)[8], const bf16_t (&b_reg)[4], float const (&c_reg)[4]) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t a0 = *reinterpret_cast(a_reg + 0); + uint32_t a1 = *reinterpret_cast(a_reg + 2); + uint32_t a2 = *reinterpret_cast(a_reg + 4); + uint32_t a3 = *reinterpret_cast(a_reg + 6); + uint32_t b0 = *reinterpret_cast(b_reg + 0); + uint32_t b1 = *reinterpret_cast(b_reg + 2); + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0, %1, %2, %3}," + "{%4, %5, %6, %7}," + "{%8, %9}," + "{%10, %11, %12, %13};\n" + : "=f"(d_reg[0]), "=f"(d_reg[1]), "=f"(d_reg[2]), "=f"(d_reg[3]) + : "r"(a0), + "r"(a1), + "r"(a2), + "r"(a3), + "r"(b0), + "r"(b1), + "f"(d_reg[0]), + "f"(d_reg[1]), + "f"(d_reg[2]), + "f"(d_reg[3])); +#endif +} + +extern "C" { +__device__ uint32_t __nvvm_get_smem_pointer(void*); +} + +__device__ void ldgsts_128(void const* gPtr, void* sPtr, uint32_t pred) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + if (pred) { + uint32_t smemPtrAsUint32 = __nvvm_get_smem_pointer(sPtr); + asm volatile("cp.async.cg.shared.global.L2::128B [%0], [%1], %2;\n" ::"r"(smemPtrAsUint32), "l"(gPtr), "n"(16)); + } +#endif +} + +__device__ void ldsm_x4(void* smem_ptr, uint32_t* reg_ptr) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("ldmatrix.sync.aligned.x4.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n" + : "=r"(reg_ptr[0]), "=r"(reg_ptr[1]), "=r"(reg_ptr[2]), "=r"(reg_ptr[3]) + : "r"(__nvvm_get_smem_pointer(smem_ptr))); +#endif +} + +template +__device__ int apply_swizzle_343_on_elem_row_col(int row_idx_, int col_idx_) { + uint32_t row_idx = *reinterpret_cast(&row_idx_); + uint32_t col_idx = *reinterpret_cast(&col_idx_); + row_idx = row_idx % 8; + row_idx = row_idx * (16 / sizeof(Type)); + col_idx = col_idx ^ row_idx; + return *reinterpret_cast(&col_idx); +} + +__device__ void initialize_barrier( + uint64_t* smem_barrier, // 64 bits user-manged barrier in smem + int thread_count = 1) // Thread count expected to arrive/wait on this barrier +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(smem_int_ptr), "r"(thread_count)); +#endif +} + +// Barrier wait +__device__ void wait_barrier( + uint64_t* smem_barrier, // 64 bits user-manged barrier in smem + int phase_bit) // Current phase bit the barrier waiting to flip +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile( + "{\n" + ".reg .pred P1;\n" + "LAB_WAIT:\n" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" + "@P1 bra DONE;\n" + "bra LAB_WAIT;\n" + "DONE:\n" + "}\n" ::"r"(smem_int_ptr), + "r"(phase_bit)); +#endif +} + +__device__ bool try_wait_barrier(uint64_t* smem_ptr, int phase_bit) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t wait_complete; + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_ptr); + asm volatile( + "{\n\t" + ".reg .pred P1; \n\t" + "mbarrier.try_wait.parity.shared::cta.b64 P1, [%1], %2; \n\t" + "selp.b32 %0, 1, 0, P1; \n\t" + "}" + : "=r"(wait_complete) + : "r"(smem_int_ptr), "r"(phase_bit)); + return static_cast(wait_complete); +#endif + return false; +} + +// Barrier arrive +__device__ void arrive_barrier(uint64_t* smem_barrier) // 64 bits user-manged barrier in smem +{ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile( + "{\n" + ".reg .b64 state; \n" + "mbarrier.arrive.shared::cta.b64 state, [%0];\n" + "}\n" ::"r"(smem_int_ptr)); +#endif +} + +__device__ void ldgsts_arrive(uint64_t* smem_barrier) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + uint32_t smem_int_ptr = __nvvm_get_smem_pointer(smem_barrier); + asm volatile("cp.async.mbarrier.arrive.noinc.shared.b64 [%0];" : : "r"(smem_int_ptr)); +#endif +} + +template +struct GmemLoaderA { + static constexpr int elem_bytes = 2; + static constexpr int vec_bytes = 16; + static constexpr int vec_elems = vec_bytes / elem_bytes; + static constexpr int thread_cnt = 64; + static_assert((tile_m * tile_k) % (vec_elems * thread_cnt) == 0); + static constexpr int a_inst_cnt_per_iter = (tile_m * tile_k) / (vec_elems * thread_cnt); + static_assert(gemm_k % tile_k == 0); + static constexpr int k_iter_cnt = gemm_k / tile_k; + + // Extra params to keep the order of k reduction... + static constexpr int mma_warp_cnt = 4; + static constexpr int per_mma_warp_k = tile_k / mma_warp_cnt; + static constexpr int k_each_chunk = gemm_k / mma_warp_cnt; + + private: + __device__ int k_project(int tile_k_idx) { + return (tile_k_idx / per_mma_warp_k * k_each_chunk) + (tile_k_idx % per_mma_warp_k); + } + + public: + __device__ GmemLoaderA(bf16_t const* gmem_a_local_, bf16_t* smem_a_, uint64_t* smem_barrier_) + : gmem_a(gmem_a_local_), smem_a(smem_a_), smem_barrier(smem_barrier_), local_tid(threadIdx.x % thread_cnt) {} + + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +// swizzle, that's what we want. +#pragma unroll + for (int i = 0; i < a_inst_cnt_per_iter; i++) { + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int m_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[i] = m_idx * tile_k + k_idx; + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + if (need_wait) { + wait_barrier(smem_barrier + 1 + stage_idx * 2, phase_bit); + } + int next_stage_idx = stage_idx + 1; + int next_phase_bit = next_stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + next_stage_idx = next_stage_idx == stage_cnt ? 0 : next_stage_idx; + if (loop_idx != k_iter_cnt - 1) { + need_wait = !try_wait_barrier(smem_barrier + 1 + next_stage_idx * 2, next_phase_bit); + } + +#pragma unroll + for (int i = 0; i < a_inst_cnt_per_iter; i++) { + int smem_offset = a_smem_offsets[i]; + bf16_t* smem_ptr_this_iter = smem_a + stage_idx * tile_m * tile_k + smem_offset; + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int m_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + int gmem_offset = m_idx * gemm_k + k_project(k_idx); + bf16_t const* gmem_ptr_this_iter = gmem_a + gmem_offset; + ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, true); + } + ldgsts_arrive(smem_barrier + stage_idx * 2); + + stage_idx = next_stage_idx; + phase_bit = next_phase_bit; + gmem_a += per_mma_warp_k; + } +#endif + } + + bf16_t const* gmem_a; + bf16_t* smem_a; + uint64_t* smem_barrier; + int local_tid; + int stage_idx = 0; + int phase_bit = 1; + bool need_wait = true; + + // per smem_stage, store with swizzle information + int a_smem_offsets[a_inst_cnt_per_iter]; +}; + +template +struct GmemLoaderB { + static constexpr int elem_bytes = 2; + static constexpr int vec_bytes = 16; + static constexpr int vec_elems = vec_bytes / elem_bytes; + static constexpr int thread_cnt = 64; + static_assert((tile_n * tile_k) % (vec_elems * thread_cnt) == 0); + static constexpr int b_inst_cnt_per_iter = (tile_n * tile_k) / (vec_elems * thread_cnt); + static_assert(gemm_k % tile_k == 0); + static constexpr int k_iter_cnt = gemm_k / tile_k; + + // Extra params to keep the order of k reduction... + static constexpr int mma_warp_cnt = 4; + static constexpr int per_mma_warp_k = tile_k / mma_warp_cnt; + static constexpr int k_each_chunk = gemm_k / mma_warp_cnt; + + private: + __device__ int k_project(int tile_k_idx) { + return (tile_k_idx / per_mma_warp_k * k_each_chunk) + (tile_k_idx % per_mma_warp_k); + } + + public: + __device__ GmemLoaderB(bf16_t const* gmem_b_local_, bf16_t* smem_b_, uint64_t* smem_barrier_, int gemm_n_) + : gmem_b(gmem_b_local_), + smem_b(smem_b_), + smem_barrier(smem_barrier_), + gemm_n(gemm_n_), + local_tid(threadIdx.x % thread_cnt) {} + + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +// swizzle, that's what we want. +#pragma unroll + for (int i = 0; i < b_inst_cnt_per_iter; i++) { + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int n_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(n_idx, k_idx); + b_smem_offsets[i] = n_idx * tile_k + k_idx; + preds[i] = n_idx < gemm_n; + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("griddepcontrol.wait;"); +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + if (need_wait) { + wait_barrier(smem_barrier + 1 + stage_idx * 2, phase_bit); + } + int next_stage_idx = stage_idx + 1; + int next_phase_bit = next_stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + next_stage_idx = next_stage_idx == stage_cnt ? 0 : next_stage_idx; + if (loop_idx != k_iter_cnt - 1) { + need_wait = !try_wait_barrier(smem_barrier + 1 + next_stage_idx * 2, next_phase_bit); + } +#pragma unroll + for (int i = 0; i < b_inst_cnt_per_iter; i++) { + int smem_offset = b_smem_offsets[i]; + bf16_t* smem_ptr_this_iter = smem_b + stage_idx * tile_n * tile_k + smem_offset; + int linear_idx = local_tid * vec_elems + i * thread_cnt * vec_elems; + int n_idx = linear_idx / tile_k; + int k_idx = linear_idx % tile_k; + int gmem_offset = n_idx * gemm_k + k_project(k_idx); + bf16_t const* gmem_ptr_this_iter = gmem_b + gmem_offset; + ldgsts_128(gmem_ptr_this_iter, smem_ptr_this_iter, preds[i]); + } + ldgsts_arrive(smem_barrier + stage_idx * 2); + + stage_idx = next_stage_idx; + phase_bit = next_phase_bit; + gmem_b += per_mma_warp_k; + } +#endif + } + + bf16_t const* gmem_b; + bf16_t* smem_b; + uint64_t* smem_barrier; + int gemm_n; + int local_tid; + int stage_idx = 0; + int phase_bit = 1; + bool need_wait = true; + + // per smem_stage, store with swizzle information + int b_smem_offsets[b_inst_cnt_per_iter]; + uint32_t preds[b_inst_cnt_per_iter]; +}; + +template +struct MmaComputer { + static constexpr int elem_bytes = 2; + static constexpr int thread_cnt = 128; + static_assert(gemm_k % tile_k == 0); + static_assert(tile_k % (thread_cnt / 32) == 0); + static constexpr int per_warp_tile_k = tile_k / (thread_cnt / 32); + static constexpr int k_iter_cnt = gemm_k / tile_k; + static constexpr int k_phase_cnt = per_warp_tile_k / 16; + static constexpr int m_iter_cnt = (tile_m + 15) / 16; + static constexpr int n_iter_cnt = (tile_n + 7) / 8; // Possible to have non-1 n_iter_cnt for ab_swap m16 case. + static_assert(m_iter_cnt == 1); + static_assert(n_iter_cnt == 1 || n_iter_cnt == 2); + + __device__ MmaComputer( + bf16_t* gmem_c_local_, bf16_t* smem_a_, bf16_t* smem_b_, uint64_t* smem_barrier_, int warp_idx_, int gemm_n_) + : gmem_c(gmem_c_local_), + smem_a(smem_a_), + smem_b(smem_b_), + smem_barrier(smem_barrier_), + warp_idx(warp_idx_ - (thread_cnt / 32)), + gemm_n(gemm_n_) {} + + private: + __device__ constexpr int internal_b_atom_func(int tid) { + if constexpr (tile_n < 8) { + return (tid % tile_n) + ((tid % 8) / tile_n * 0) + tid / 8 * 8 * tile_n; + } else { + return (tid % 8) + ((tid % 32) / 8 * (tile_n * 8)); + } + } + + public: + __device__ void prepare() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int linear_idx = (lane_idx % 16) + (lane_idx / 16) * 128 + i * 256; + int m_idx = linear_idx % tile_m; + int k_idx = linear_idx / tile_m + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(m_idx, k_idx); + a_smem_offsets[0][i] = m_idx * tile_k + k_idx; + } +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { +#pragma unroll + for (int i = 0; i < k_phase_cnt; i += 2) { // Special i+=2 for B. + int linear_idx = internal_b_atom_func(lane_idx) + i * tile_n * 16 + n_iter_idx * 8; + int n_idx = linear_idx % tile_n; + int k_idx = linear_idx / tile_n + warp_k_offset_in_tile_k; + k_idx = apply_swizzle_343_on_elem_row_col(n_idx, k_idx); + b_smem_offsets[n_iter_idx][i] = n_idx * tile_k + k_idx; + } + } +#endif + } + + __device__ void issue_mainloop() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 +#pragma unroll 1 + for (int loop_idx = 0; loop_idx < k_iter_cnt; loop_idx++) { + wait_barrier(smem_barrier + 0 + stage_idx * 2, phase_bit); + +#pragma unroll + for (int i = 0; i < k_phase_cnt; i++) { + int smem_offset = a_smem_offsets[0][i]; + bf16_t* smem_ptr_this_iter = smem_a + stage_idx * tile_m * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(a_reg[0][i])); + } + +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { +#pragma unroll + for (int i = 0; i < k_phase_cnt; i += 2) { + int smem_offset = b_smem_offsets[n_iter_idx][i]; + bf16_t* smem_ptr_this_iter = smem_b + stage_idx * tile_n * tile_k + smem_offset; + ldsm_x4(smem_ptr_this_iter, reinterpret_cast(b_reg[n_iter_idx][i])); + } + } + +#pragma unroll + for (int k_iter_idx = 0; k_iter_idx < k_phase_cnt; k_iter_idx++) { +#pragma unroll + for (int n_iter_idx = 0; n_iter_idx < n_iter_cnt; n_iter_idx++) { + hmma_16_8_16_f32acc_bf16ab( + acc_reg[0][n_iter_idx], a_reg[0][k_iter_idx], b_reg[n_iter_idx][k_iter_idx], acc_reg[0][n_iter_idx]); + } + } + ::arrive_barrier(smem_barrier + 1 + stage_idx * 2); + stage_idx += 1; + phase_bit = stage_idx == stage_cnt ? phase_bit ^ 1 : phase_bit; + stage_idx = stage_idx == stage_cnt ? 0 : stage_idx; + } +#endif + } + + __device__ void epi() { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); + // reorganize the acc_reg + constexpr int thread_m = 2; + constexpr int thread_n = 2 * n_iter_cnt; + constexpr int cta_mma_n = n_iter_cnt * 8; + float acc_reg_reorg[thread_m][thread_n]; + + for (int i = 0; i < thread_m; i++) { + for (int j = 0; j < thread_n; j++) { + acc_reg_reorg[i][j] = acc_reg[0][j / 2][(j % 2) + (i * 2)]; + } + } + + // 4 x cosize(smem_c_layout) + float* smem_c = reinterpret_cast(smem_a); + // coord -> index + auto smem_c_index_func = [&](int m_idx, int n_idx) { + int group_rows = 32 / cta_mma_n; + int group_cnt = 2; + return (m_idx % group_rows * cta_mma_n) + (m_idx / group_rows * (32 + group_cnt)) + n_idx; + }; + constexpr int cosize_smem_c = ((tile_m * cta_mma_n) / 32) * (32 + 2); + +// This should be optimized to STS.64 but can not be STS.128 due to the bank index. +#pragma unroll + for (int m_idx_thread = 0; m_idx_thread < thread_m; m_idx_thread++) { +#pragma unroll + for (int n_idx_thread = 0; n_idx_thread < thread_n; n_idx_thread++) { + int m_idx = (lane_idx / 4) + m_idx_thread * 8; + int n_idx = ((lane_idx % 4) * 2) + (n_idx_thread % 2) + (n_idx_thread / 2) * 8; + smem_c[cosize_smem_c * warp_idx + smem_c_index_func(m_idx, n_idx)] = acc_reg_reorg[m_idx_thread][n_idx_thread]; + } + } + asm volatile("bar.sync %0, %1;" : : "r"(1), "r"(thread_cnt)); + + if (warp_idx == 0) { + constexpr int final_acc_reg_cnt = (tile_m * tile_n + 31) / 32; + float acc_final[final_acc_reg_cnt]{}; + +#pragma unroll + for (int reg_idx = 0; reg_idx < final_acc_reg_cnt; reg_idx++) { + int linear_idx = reg_idx * 32 + lane_idx; + int m_idx = linear_idx % tile_m; + int n_idx = linear_idx / tile_m; + acc_final[reg_idx] += smem_c[smem_c_index_func(m_idx, n_idx) + 0 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 1 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 2 * cosize_smem_c] + + smem_c[smem_c_index_func(m_idx, n_idx) + 3 * cosize_smem_c]; + } + +#pragma unroll + for (int reg_idx = 0; reg_idx < final_acc_reg_cnt; reg_idx++) { + int linear_idx = reg_idx * 32 + lane_idx; + int m_idx = linear_idx % tile_m; + int n_idx = linear_idx / tile_m; + if (m_idx < tile_m && n_idx < gemm_n) { + gmem_c[n_idx * gemm_m + m_idx] = acc_final[reg_idx]; + } + } + } +#endif + } + + bf16_t* gmem_c; + bf16_t* smem_a; + bf16_t* smem_b; + uint64_t* smem_barrier; + int warp_idx; + int gemm_n; + int stage_idx = 0; + int phase_bit = 0; + int lane_idx = threadIdx.x % 32; + int warp_k_offset_in_tile_k = warp_idx * per_warp_tile_k; + + int a_smem_offsets[m_iter_cnt][k_phase_cnt]; + int b_smem_offsets[n_iter_cnt][k_phase_cnt]; + + bf16_t a_reg[m_iter_cnt][k_phase_cnt][8]; + bf16_t b_reg[n_iter_cnt][k_phase_cnt][4]; + float acc_reg[m_iter_cnt][n_iter_cnt][4]{}; +}; + +// AB swapped, kernel is k-major, k-major, m-major +template +__global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel( + bf16_t* output, bf16_t const* mat_a, bf16_t const* mat_b, int gemm_n) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900 + constexpr int load_thread_cnt = 128; + constexpr int compute_thread_cnt = 128; + constexpr int thread_cnt = load_thread_cnt + compute_thread_cnt; + (void)thread_cnt; + static_assert(gemm_m % 16 == 0); + static_assert(gemm_k % tile_k == 0); + static_assert(gemm_m % tile_m == 0); + static_assert( + tile_k == 128 || tile_k == 256 || tile_k == 512 || + tile_k == 1024); // tile_k must be larger than 64 since 4 warp splitK. + static_assert(tile_m == 16); + constexpr int g2s_vec_bytes = 16; + constexpr int a_elem_bytes = 2; + constexpr int b_elem_bytes = 2; + // constexpr int c_elem_bytes = 2; + static_assert((tile_m * a_elem_bytes + tile_n * b_elem_bytes) * tile_k * stage_cnt <= 225 * 1024); + static_assert((tile_m * tile_k * a_elem_bytes) % (load_thread_cnt * g2s_vec_bytes) == 0); + static_assert((tile_n * tile_k * b_elem_bytes) % (load_thread_cnt * g2s_vec_bytes) == 0); + + extern __shared__ char smem[]; + uint64_t* smem_barrier = reinterpret_cast(smem); // producer,consumer; producer,consumer; ... + bf16_t* smem_a = reinterpret_cast(smem + (stage_cnt * 8 * 2 + 1024) / 1024 * 1024); + bf16_t* smem_b = smem_a + tile_m * tile_k * stage_cnt; + + int cta_m_idx = tile_m * blockIdx.x; + int cta_n_idx = tile_n * blockIdx.y; + bf16_t const* gmem_a_local = mat_a + cta_m_idx * gemm_k; + bf16_t const* gmem_b_local = mat_b + cta_n_idx * gemm_k; + bf16_t* gmem_c_local = output + cta_n_idx * gemm_m + cta_m_idx; + + int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0); + + if (warp_idx == 4) { + for (int i = 0; i < stage_cnt; i++) { + initialize_barrier(smem_barrier + i * 2 + 0, load_thread_cnt); // producer + initialize_barrier(smem_barrier + i * 2 + 1, compute_thread_cnt); // consumer + } + } + __syncthreads(); + + if (warp_idx < 2) { + GmemLoaderA a_loader(gmem_a_local, smem_a, smem_barrier); + a_loader.prepare(); + a_loader.issue_mainloop(); + } else if (warp_idx < 4) { + GmemLoaderB b_loader(gmem_b_local, smem_b, smem_barrier, gemm_n); + b_loader.prepare(); + b_loader.issue_mainloop(); + } else { + MmaComputer mma_computer( + gmem_c_local, smem_a, smem_b, smem_barrier, warp_idx, gemm_n); + mma_computer.prepare(); + mma_computer.issue_mainloop(); + mma_computer.epi(); + } + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens, cudaStream_t const stream) { + constexpr int gemm_m = kHdOut; // 2112 + int const gemm_n = num_tokens; // 16 + constexpr int gemm_k = kHdIn; // 7168 + constexpr int batch_size = 1; + std::swap(mat_a, mat_b); + constexpr int tile_m = 16; + constexpr int tile_n = kTileN; // 8 or 16 + constexpr int tile_k = std::max(256, 1024 / tile_n); // 256 + constexpr int max_stage_cnt = 1024 * 192 / ((tile_m + tile_n) * tile_k * sizeof(bf16_t)); + constexpr int k_iter_cnt = gemm_k / tile_k; + constexpr int stage_cnt = + k_iter_cnt > max_stage_cnt ? max_stage_cnt : k_iter_cnt; // possible tunable for smallK > 1 wave n. // 22 + int cta_m_cnt = gemm_m / tile_m; + int cta_n_cnt = (gemm_n + tile_n - 1) / tile_n; + constexpr int barrier_bytes = (stage_cnt * 16 + 1023) / 1024 * 1024; // 4096 + constexpr int smem_bytes = ((tile_m * 2 + tile_n * 2) * tile_k * stage_cnt + barrier_bytes + 1023) / 1024 * 1024; + + dim3 grid(cta_m_cnt, cta_n_cnt, 1); + dim3 block_size(256); + cudaLaunchConfig_t config; + config.gridDim = grid; + config.blockDim = block_size; + config.dynamicSmemBytes = smem_bytes; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + if (smem_bytes >= (48 * 1024)) { + cudaFuncSetAttribute( + fused_a_gemm_kernel, + cudaFuncAttributeMaxDynamicSharedMemorySize, + smem_bytes); + } + cudaLaunchKernelEx( + &config, + fused_a_gemm_kernel, + output, + mat_a, + mat_b, + gemm_n); +} + +template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); + +template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens, cudaStream_t); + +void dsv3_fused_a_gemm(torch::Tensor& output, torch::Tensor const& mat_a, torch::Tensor const& mat_b) { + TORCH_CHECK(mat_a.dim() == 2 && mat_b.dim() == 2 && output.dim() == 2); + int const num_tokens = mat_a.size(0); + int const hd_in = mat_a.size(1); + int const hd_out = mat_b.size(1); + + constexpr int kHdIn = 7168; + constexpr int kHdOut = 2112; + TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16, "required 1 <= mat_a.shape[0] <= 16") + TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168") + TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112") + TORCH_CHECK(output.size(0) == num_tokens, "required output.shape[0] == mat_a.shape[0]") + TORCH_CHECK(output.size(1) == hd_out, "required output.shape[1] == mat_b.shape[1]") + + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); // Row-major + TORCH_CHECK(output.stride(1) == 1, "output must be a row major tensor"); // Row-major + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); // Column-major + + auto const data_type = mat_a.scalar_type(); + TORCH_CHECK( + mat_a.scalar_type() == torch::kBFloat16 && mat_b.scalar_type() == torch::kBFloat16, + "Only BFloat16 input dtype is supported") + TORCH_CHECK(output.scalar_type() == torch::kBFloat16, "Only BFloat16 output dtype is supported") + + auto const sm = getSMVersion(); + TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90"); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + if (num_tokens <= 8) { + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + num_tokens, + stream); + } else { + invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>( + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + num_tokens, + stream); + } +} diff --git a/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu b/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu new file mode 100644 index 000000000..e613bd75c --- /dev/null +++ b/sgl-kernel/csrc/gemm/dsv3_router_gemm_bf16_out.cu @@ -0,0 +1,284 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +// Custom FMA implementation using PTX assembly instructions +__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(d)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); +} + +// Convert 8 bfloat16 values from a uint4 to float array - optimized conversion +template +__device__ __forceinline__ void bf16_uint4_to_float8(uint4 const& vec, float* dst) { + __nv_bfloat16* bf16_ptr = reinterpret_cast<__nv_bfloat16*>(const_cast(&vec)); + +#pragma unroll + for (int i = 0; i < VPT; i++) { + dst[i] = __bfloat162float(bf16_ptr[i]); + } +} + +template +__global__ +__launch_bounds__(128, 1) void router_gemm_kernel_bf16_output(__nv_bfloat16* out, T const* mat_a, T const* mat_b) { + // Each block handles one expert column + int const n_idx = blockIdx.x; + int const tid = threadIdx.x; + constexpr int kWarpSize = 32; + constexpr int kNumWarps = kBlockSize / kWarpSize; + // Constants for this kernel + constexpr int k_elems_per_k_iteration = VPT * kBlockSize; + constexpr int k_iterations = kHiddenDim / k_elems_per_k_iteration; // Total K iterations + + // Initialize accumulators for all M rows + float acc[kNumTokens] = {}; + + // Shared memory for warp-level reduction + __shared__ float sm_reduction[kNumTokens][kNumWarps]; // kNumWarps + + // B matrix is in column-major order, so we can directly load a column for the n_idx expert + T const* b_col = mat_b + n_idx * kHiddenDim; + + // Pre-compute k_base values for each iteration to help compiler optimize + // int k_bases[k_iterations]; + int k_bases[k_iterations]; +#pragma unroll + for (int ki = 0; ki < k_iterations; ki++) { + k_bases[ki] = ki * k_elems_per_k_iteration + tid * VPT; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + // Process the GEMM in chunks + for (int ki = 0; ki < k_iterations; ki++) { + int const k_base = k_bases[ki]; + + // Load B matrix values using vector load (8 bf16 values) + uint4 b_vec = *reinterpret_cast(b_col + k_base); + + // Convert B values to float + float b_float[VPT]; + bf16_uint4_to_float8(b_vec, b_float); + +// Process each token +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + // Load both rows of A matrix using vector loads + uint4 a_vec = *reinterpret_cast(mat_a + (m_idx * kHiddenDim) + k_base); + + // Convert A values to float + float a_float[VPT]; + bf16_uint4_to_float8(a_vec, a_float); + +// Process elements in this chunk +#pragma unroll + for (int k = 0; k < VPT; k++) { + float a = a_float[k]; + float b = b_float[k]; + acc[m_idx] += a * b; + } + } + } + + // Perform warp-level reduction + int const warpSize = 32; + int const warpId = tid / warpSize; + int const laneId = tid % warpSize; + + // Register for warp-level reduction results + float warp_result[kNumTokens]; + +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + warp_result[m_idx] = acc[m_idx]; + } + +// Perform warp-level reduction using optimized butterfly pattern +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float sum = warp_result[m]; + + // Butterfly reduction pattern + sum += __shfl_xor_sync(0xffffffff, sum, 16); + sum += __shfl_xor_sync(0xffffffff, sum, 8); + sum += __shfl_xor_sync(0xffffffff, sum, 4); + sum += __shfl_xor_sync(0xffffffff, sum, 2); + sum += __shfl_xor_sync(0xffffffff, sum, 1); + + // Only the first thread in each warp stores to shared memory + if (laneId == 0) { + sm_reduction[m][warpId] = sum; + } + } + + __syncthreads(); + + // Final reduction across warps (only first thread) + if (tid == 0) { +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float final_sum = 0.0f; + +// Sum across the kNumWarps +#pragma unroll + for (int w = 0; w < kNumWarps; w++) { + final_sum += sm_reduction[m][w]; + } + + // Write final result + out[m * kNumExperts + n_idx] = __float2bfloat16(final_sum); + } + } +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeRouterGemmBf16Output(__nv_bfloat16* output, T const* mat_a, T const* mat_b, cudaStream_t stream) { + constexpr int VPT = 16 / sizeof(T); + constexpr int kBlockSize = 128; + cudaLaunchConfig_t config; + config.gridDim = kNumExperts; + config.blockDim = kBlockSize; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, + router_gemm_kernel_bf16_output, + output, + mat_a, + mat_b); +} + +// Template instantiations for DEFAULT_NUM_EXPERTS experts +template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 256, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for KIMI_K2_NUM_EXPERTS experts +template void invokeRouterGemmBf16Output<__nv_bfloat16, 1, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 2, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 3, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 4, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 5, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 6, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 7, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 8, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 9, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 10, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 11, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 12, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 13, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 14, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 15, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmBf16Output<__nv_bfloat16, 16, 384, 7168>( + __nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu b/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu new file mode 100644 index 000000000..4f09e6cf4 --- /dev/null +++ b/sgl-kernel/csrc/gemm/dsv3_router_gemm_entry.cu @@ -0,0 +1,161 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +static constexpr int DEFAULT_NUM_EXPERTS = 256; +static constexpr int KIMI_K2_NUM_EXPERTS = 384; +static constexpr int DEFAULT_HIDDEN_DIM = 7168; + +template +void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b, cudaStream_t stream); + +template +void invokeRouterGemmBf16Output(__nv_bfloat16* output, T const* mat_a, T const* mat_b, cudaStream_t stream); + +template +struct LoopUnroller { + static void unroll_float_output( + int num_tokens, float* output, __nv_bfloat16 const* input, __nv_bfloat16 const* weights, cudaStream_t stream) { + if (num_tokens == kBegin) { + invokeRouterGemmFloatOutput<__nv_bfloat16, kBegin, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + LoopUnroller::unroll_float_output( + num_tokens, output, input, weights, stream); + } + } + + static void unroll_bf16_output( + int num_tokens, + __nv_bfloat16* output, + __nv_bfloat16 const* input, + __nv_bfloat16 const* weights, + cudaStream_t stream) { + if (num_tokens == kBegin) { + invokeRouterGemmBf16Output<__nv_bfloat16, kBegin, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + LoopUnroller::unroll_bf16_output( + num_tokens, output, input, weights, stream); + } + } +}; + +template +struct LoopUnroller { + static void unroll_float_output( + int num_tokens, float* output, __nv_bfloat16 const* input, __nv_bfloat16 const* weights, cudaStream_t stream) { + if (num_tokens == kEnd) { + invokeRouterGemmFloatOutput<__nv_bfloat16, kEnd, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + throw std::invalid_argument("Invalid num_tokens, only supports 1 to 16"); + } + } + + static void unroll_bf16_output( + int num_tokens, + __nv_bfloat16* output, + __nv_bfloat16 const* input, + __nv_bfloat16 const* weights, + cudaStream_t stream) { + if (num_tokens == kEnd) { + invokeRouterGemmBf16Output<__nv_bfloat16, kEnd, kNumExperts, kHiddenDim>(output, input, weights, stream); + } else { + throw std::invalid_argument("Invalid num_tokens, only supports 1 to 16"); + } + } +}; + +void dsv3_router_gemm( + torch::Tensor& output, // [num_tokens, num_experts] + const torch::Tensor& mat_a, // [num_tokens, hidden_dim] + const torch::Tensor& mat_b // [num_experts, hidden_dim] +) { + TORCH_CHECK(output.dim() == 2 && mat_a.dim() == 2 && mat_b.dim() == 2); + + const int num_tokens = mat_a.size(0); + const int num_experts = mat_b.size(0); + const int hidden_dim = mat_a.size(1); + + TORCH_CHECK(mat_a.size(1) == mat_b.size(1), "mat_a and mat_b must have the same hidden_dim"); + TORCH_CHECK( + hidden_dim == DEFAULT_HIDDEN_DIM, + "Expected hidden_dim=", + DEFAULT_HIDDEN_DIM, + ", but got hidden_dim=", + hidden_dim); + TORCH_CHECK( + num_experts == DEFAULT_NUM_EXPERTS || num_experts == KIMI_K2_NUM_EXPERTS, + "Expected num_experts=", + DEFAULT_NUM_EXPERTS, + " or num_experts=", + KIMI_K2_NUM_EXPERTS, + ", but got num_experts=", + num_experts); + TORCH_CHECK( + num_tokens >= 1 && num_tokens <= 16, "currently num_tokens must be less than or equal to 16 for router_gemm"); + TORCH_CHECK(mat_a.dtype() == torch::kBFloat16, "mat_a must be bf16"); + TORCH_CHECK(mat_b.dtype() == torch::kBFloat16, "mat_b must be bf16"); + TORCH_CHECK( + output.dtype() == torch::kFloat32 || output.dtype() == torch::kBFloat16, "output must be float32 or bf16"); + + auto const sm = getSMVersion(); + TORCH_CHECK(sm >= 90, "required CUDA ARCH >= SM_90"); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (output.dtype() == torch::kFloat32) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_float_output( + num_tokens, + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } else if (num_experts == KIMI_K2_NUM_EXPERTS) { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_float_output( + num_tokens, + reinterpret_cast(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } + } else if (output.dtype() == torch::kBFloat16) { + if (num_experts == DEFAULT_NUM_EXPERTS) { + LoopUnroller<1, 16, DEFAULT_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_bf16_output( + num_tokens, + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } else if (num_experts == KIMI_K2_NUM_EXPERTS) { + LoopUnroller<1, 16, KIMI_K2_NUM_EXPERTS, DEFAULT_HIDDEN_DIM>::unroll_bf16_output( + num_tokens, + reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()), + reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), + stream); + } + } +} diff --git a/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu b/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu new file mode 100644 index 000000000..88a364e2c --- /dev/null +++ b/sgl-kernel/csrc/gemm/dsv3_router_gemm_float_out.cu @@ -0,0 +1,283 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/kernels/dsv3MinLatencyKernels/dsv3RouterGemm.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/main/cpp/tensorrt_llm/thop/dsv3RouterGemmOp.cpp + * + * Copyright (c) 2019-2023, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "cuda_bf16.h" +#include "cuda_runtime.h" +#include "utils.h" + +// Custom FMA implementation using PTX assembly instructions +__device__ __forceinline__ void fma(float2& d, float2 const& a, float2 const& b, float2 const& c) { + asm volatile("fma.rn.f32x2 %0, %1, %2, %3;\n" + : "=l"(reinterpret_cast(d)) + : "l"(reinterpret_cast(a)), + "l"(reinterpret_cast(b)), + "l"(reinterpret_cast(c))); +} + +// Convert 8 bfloat16 values from a uint4 to float array - optimized conversion +template +__device__ __forceinline__ void bf16_uint4_to_float8(uint4 const& vec, float* dst) { + __nv_bfloat16* bf16_ptr = reinterpret_cast<__nv_bfloat16*>(const_cast(&vec)); + +#pragma unroll + for (int i = 0; i < VPT; i++) { + dst[i] = __bfloat162float(bf16_ptr[i]); + } +} + +template +__global__ __launch_bounds__(128, 1) void router_gemm_kernel_float_output(float* out, T const* mat_a, T const* mat_b) { + // Each block handles one expert column + int const n_idx = blockIdx.x; + int const tid = threadIdx.x; + constexpr int kWarpSize = 32; + constexpr int kNumWarps = kBlockSize / kWarpSize; + // Constants for this kernel + constexpr int k_elems_per_k_iteration = VPT * kBlockSize; + constexpr int k_iterations = kHiddenDim / k_elems_per_k_iteration; // Total K iterations + + // Initialize accumulators for all M rows + float acc[kNumTokens] = {}; + + // Shared memory for warp-level reduction + __shared__ float sm_reduction[kNumTokens][kNumWarps]; // kNumWarps + + // B matrix is in column-major order, so we can directly load a column for the n_idx expert + T const* b_col = mat_b + n_idx * kHiddenDim; + + // Pre-compute k_base values for each iteration to help compiler optimize + // int k_bases[k_iterations]; + int k_bases[k_iterations]; +#pragma unroll + for (int ki = 0; ki < k_iterations; ki++) { + k_bases[ki] = ki * k_elems_per_k_iteration + tid * VPT; + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.wait;"); +#endif + + // Process the GEMM in chunks + for (int ki = 0; ki < k_iterations; ki++) { + int const k_base = k_bases[ki]; + + // Load B matrix values using vector load (8 bf16 values) + uint4 b_vec = *reinterpret_cast(b_col + k_base); + + // Convert B values to float + float b_float[VPT]; + bf16_uint4_to_float8(b_vec, b_float); + +// Process each token +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + // Load both rows of A matrix using vector loads + uint4 a_vec = *reinterpret_cast(mat_a + (m_idx * kHiddenDim) + k_base); + + // Convert A values to float + float a_float[VPT]; + bf16_uint4_to_float8(a_vec, a_float); + +// Process elements in this chunk +#pragma unroll + for (int k = 0; k < VPT; k++) { + float a = a_float[k]; + float b = b_float[k]; + acc[m_idx] += a * b; + } + } + } + + // Perform warp-level reduction + int const warpSize = 32; + int const warpId = tid / warpSize; + int const laneId = tid % warpSize; + + // Register for warp-level reduction results + float warp_result[kNumTokens]; + +#pragma unroll + for (int m_idx = 0; m_idx < kNumTokens; m_idx++) { + warp_result[m_idx] = acc[m_idx]; + } + +// Perform warp-level reduction using optimized butterfly pattern +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float sum = warp_result[m]; + + // Butterfly reduction pattern + sum += __shfl_xor_sync(0xffffffff, sum, 16); + sum += __shfl_xor_sync(0xffffffff, sum, 8); + sum += __shfl_xor_sync(0xffffffff, sum, 4); + sum += __shfl_xor_sync(0xffffffff, sum, 2); + sum += __shfl_xor_sync(0xffffffff, sum, 1); + + // Only the first thread in each warp stores to shared memory + if (laneId == 0) { + sm_reduction[m][warpId] = sum; + } + } + + __syncthreads(); + + // Final reduction across warps (only first thread) + if (tid == 0) { +#pragma unroll + for (int m = 0; m < kNumTokens; m++) { + float final_sum = 0.0f; + +// Sum across the kNumWarps +#pragma unroll + for (int w = 0; w < kNumWarps; w++) { + final_sum += sm_reduction[m][w]; + } + + // Write final result + out[m * kNumExperts + n_idx] = final_sum; + } + } +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + asm volatile("griddepcontrol.launch_dependents;"); +#endif +} + +template +void invokeRouterGemmFloatOutput(float* output, T const* mat_a, T const* mat_b, cudaStream_t stream) { + constexpr int VPT = 16 / sizeof(T); + constexpr int kBlockSize = 128; + cudaLaunchConfig_t config; + config.gridDim = kNumExperts; + config.blockDim = kBlockSize; + config.dynamicSmemBytes = 0; + config.stream = stream; + cudaLaunchAttribute attrs[1]; + attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; + attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL(); + config.numAttrs = 1; + config.attrs = attrs; + cudaLaunchKernelEx( + &config, + router_gemm_kernel_float_output, + output, + mat_a, + mat_b); +} + +// Template instantiations for DEFAULT_NUM_EXPERTS experts +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 256, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +// Template instantiations for KIMI_K2_NUM_EXPERTS experts +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 1, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 2, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 3, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 4, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 5, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 6, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 7, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 8, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 9, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 10, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 11, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 12, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 13, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 14, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 15, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); + +template void invokeRouterGemmFloatOutput<__nv_bfloat16, 16, 384, 7168>( + float*, __nv_bfloat16 const*, __nv_bfloat16 const*, cudaStream_t); diff --git a/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu b/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu new file mode 100644 index 000000000..b8b23c427 --- /dev/null +++ b/sgl-kernel/csrc/gemm/fp8_blockwise_gemm_kernel.cu @@ -0,0 +1,465 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/gemm/cutlass_gemm_caller.cuh" +#include "cutlass_extensions/gemm/fp8_blockwise_gemm_sm90_dispatch.cuh" +#include "utils.h" + +using namespace cute; + +template < + typename OutType, + typename MmaTileShape, + typename PerSmTileShape, + typename EpilogueTileShape, + typename ScalesPerTile, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _1, _1>> +void launch_sm100_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + static constexpr int ScaleMsPerTile = size<0>(ScalesPerTile{}); + static constexpr int ScaleGranularityM = size<0>(MmaTileShape{}) / ScaleMsPerTile; + static constexpr int ScaleGranularityN = size<1>(MmaTileShape{}) / size<1>(ScalesPerTile{}); + static constexpr int ScaleGranularityK = size<2>(MmaTileShape{}) / size<2>(ScalesPerTile{}); + + using ElementAB = cutlass::float_e4m3_t; + using ElementA = ElementAB; + using ElementB = ElementAB; + using ElementC = void; + using ElementD = OutType; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutD = cutlass::layout::RowMajor; + using LayoutC = LayoutD; + // This means both SFA and SFB are column-major. + using ScaleConfig = cutlass::detail::Sm100BlockwiseScaleConfig< + ScaleGranularityM, + ScaleGranularityN, + ScaleGranularityK, + cute::UMMA::Major::MN, + cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = AlignmentD; + + using ElementAccumulator = float; + using ElementBlockScale = float; + using ElementCompute = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + cutlass::arch::OpClassTensorOp, + PerSmTileShape, + ClusterShape, + EpilogueTileShape, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + ElementD, + LayoutD, + AlignmentD, + cutlass::epilogue::TmaWarpSpecialized1Sm>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::KernelTmaWarpSpecializedBlockwise1SmSm100>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, + CollectiveMainloop, + CollectiveEpilogue, + cutlass::gemm::PersistentScheduler>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto scales_a_ptr = static_cast(scales_a.data_ptr()); + auto scales_b_ptr = static_cast(scales_b.data_ptr()); + auto c_ptr = static_cast(out.data_ptr()); + + using StrideA = typename GemmKernel::StrideA; + using StrideB = typename GemmKernel::StrideB; + using StrideD = typename GemmKernel::StrideD; + using StrideC = typename GemmKernel::StrideD; + + StrideA a_stride = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB b_stride = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC c_stride = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, a_stride, b_ptr, b_stride, scales_a_ptr, layout_SFA, scales_b_ptr, layout_SFB}; + + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, c_stride, c_ptr, c_stride}; + epilogue_args.thread.alpha = 1.0f; + + typename GemmKernel::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, {m, n, k, 1}, mainloop_args, epilogue_args}; + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + size_t workspace_size = gemm_op.get_workspace_size(args); + cutlass::device_memory::allocation workspace(workspace_size); + + auto init_status = gemm_op.initialize(args, workspace.get()); + TORCH_CHECK(init_status == cutlass::Status::kSuccess, cutlassGetStatusString(init_status)); + + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template +void sm100_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + if (a.size(0) <= 128) { + using MmaTileShape = Shape<_64, _128, _128>; + using PerSmTileShape = Shape<_64, _128, _128>; + using EpilogueTileShape = Shape<_64, _64>; + using ScalesPerTile = Shape<_64, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } else { + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape = Shape<_128, _128, _128>; + using EpilogueTileShape = Shape<_128, _64>; + using ScalesPerTile = Shape<_128, _1, _1>; + launch_sm100_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); + } +} + +template < + typename OutType, + typename MmaTileShape, + typename PerSmTileShape, + typename EpilogueTileShape, + typename ScalesPerTile, + int TileSizeM_ = 128, + class ClusterShape = Shape<_1, _1, _1>> +void launch_sm120_fp8_blockwise_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using ElementBlockScale = float; + + // A matrix configuration + using ElementA = cutlass::float_e4m3_t; // Element type for A matrix operand + using LayoutATag = cutlass::layout::RowMajor; // Layout type for A matrix operand + constexpr int AlignmentA = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A matrix in units of + // elements (up to 16 bytes) + + // B matrix configuration + using ElementB = cutlass::float_e4m3_t; // Element type for B matrix operand + using LayoutBTag = cutlass::layout::ColumnMajor; // Layout type for B matrix operand + constexpr int AlignmentB = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of B matrix in units of + // elements (up to 16 bytes) + + // C/D matrix configuration + using ElementD = OutType; // Element type for D matrix operand + using ElementC = void; // Element type for C matrix operand + using LayoutCTag = cutlass::layout::RowMajor; // Layout type for C matrix operand + using LayoutDTag = cutlass::layout::RowMajor; // Layout type for D matrix operand + constexpr int AlignmentD = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrix in units of + // elements (up to 16 bytes) + constexpr int AlignmentC = + AlignmentD; // Memory access granularity/alignment of C matrix in units of elements (up to 16 bytes) + + // Kernel functional config + using ElementAccumulator = float; // Element type for internal accumulation + using ArchTag = cutlass::arch::Sm120; // Tag indicating the minimum SM that supports the intended feature + using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag - changed from OpClassBlockScaledTensorOp + + static constexpr int ScaleMsPerTile = size<0>(ScalesPerTile{}); + static constexpr int ScaleGranularityM = size<0>(MmaTileShape{}) / ScaleMsPerTile; + static constexpr int ScaleGranularityN = size<1>(MmaTileShape{}) / size<1>(ScalesPerTile{}); + static constexpr int ScaleGranularityK = size<2>(MmaTileShape{}) / size<2>(ScalesPerTile{}); + + using ScaleConfig = cutlass::detail::Sm120BlockwiseScaleConfig< + ScaleGranularityM, + ScaleGranularityN, + ScaleGranularityK, + cute::UMMA::Major::MN, + cute::UMMA::Major::K>; + // FP8 Block-wise scaling configuration + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); // Layout type for SFA matrix operand + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); // Layout type for SFB matrix operand + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + PerSmTileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + cutlass::epilogue::collective::EpilogueScheduleAuto // Epilogue schedule policy + >::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + cutlass::gemm::collective::KernelScheduleAuto // Kernel schedule policy. Auto defaults to cooperative kernel + // schedule + >::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + void>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = a.size(0); + int k = a.size(1); + int n = b.size(1); + + auto a_ptr = static_cast(a.data_ptr()); + auto b_ptr = static_cast(b.data_ptr()); + auto c_ptr = static_cast(out.data_ptr()); + + auto scales_a_ptr = static_cast(scales_a.data_ptr()); + auto scales_b_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideD = typename Gemm::GemmKernel::StrideD; + using StrideC = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + LayoutSFA layout_SFA = ScaleConfig::tile_atom_to_shape_SFA(make_shape(m, n, k, 1)); + LayoutSFB layout_SFB = ScaleConfig::tile_atom_to_shape_SFB(make_shape(m, n, k, 1)); + + typename GemmKernel::MainloopArguments mainloop_args{ + a_ptr, stride_a, b_ptr, stride_b, scales_a_ptr, layout_SFA, scales_b_ptr, layout_SFB}; + + typename GemmKernel::EpilogueArguments epilogue_args{{}, c_ptr, stride_c, c_ptr, stride_c}; + epilogue_args.thread.alpha = 1.0f; + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + mainloop_args, + epilogue_args, + }; + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess, cutlassGetStatusString(can_implement)) + + size_t workspace_size = gemm_op.get_workspace_size(args); + cutlass::device_memory::allocation workspace(workspace_size); + + auto init_status = gemm_op.initialize(args, workspace.get()); + TORCH_CHECK(init_status == cutlass::Status::kSuccess, cutlassGetStatusString(init_status)); + + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, cutlassGetStatusString(status)) +} + +template +void sm120_fp8_blockwise_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b) { + using MmaTileShape = Shape<_128, _128, _128>; + using PerSmTileShape = Shape<_128, _128, _128>; + using EpilogueTileShape = Shape<_128, _64>; + using ScalesPerTile = Shape<_128, _1, _1>; + launch_sm120_fp8_blockwise_scaled_mm( + out, a, b, scales_a, scales_b); +} + +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + auto is_contiguous_vector = [](const torch::Tensor& t) { + auto t_sizes = t.sizes(); + return t.is_contiguous() && + (t.dim() == 1 || (t.dim() == 2 && *std::min_element(t_sizes.begin(), t_sizes.end()) == 1)); + }; + + TORCH_CHECK(mat_a.size(0) == scales_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(mat_a.size(1) / 128 == scales_a.size(1), "size of scales_a is not matched"); + TORCH_CHECK(scales_a.stride(0) == 1 || is_contiguous_vector(scales_a), "scales_a must be M major"); + TORCH_CHECK(mat_b.size(0) / 128 == scales_b.size(0), "size of scales_b is not matched"); + TORCH_CHECK(mat_b.size(1) / 128 == scales_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_b.stride(0) == 1 || is_contiguous_vector(scales_b), "scales_b must be K major"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + + int64_t original_rows = mat_a.size(0); + torch::Tensor mat_a_padded = pad_tensor(mat_a, /*alignment=*/4); + torch::Tensor scales_a_padded = pad_tensor(scales_a, /*alignment=*/4, /*col_major=*/true); + torch::Tensor out_padded = torch::empty({mat_a_padded.size(0), mat_b.size(1)}, out.options()); + +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version == 90) { + torch::Tensor scales_b_contiguous = scales_b.contiguous(); + if (out_dtype == torch::kBFloat16) { + cutlass_gemm_blockwise_sm90_fp8_dispatch( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b_contiguous); + } else { + cutlass_gemm_blockwise_sm90_fp8_dispatch( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b_contiguous); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100 +#if CUDA_VERSION >= 12090 + || sm_version == 103 +#endif + ) { + if (out_dtype == torch::kBFloat16) { + sm100_fp8_blockwise_dispatch_shape( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } else { + sm100_fp8_blockwise_dispatch_shape(out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM120A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM120_SUPPORTED) +#if defined(CUDA_VERSION) && CUDA_VERSION >= 12080 + if (sm_version == 120) { + if (out_dtype == torch::kBFloat16) { + sm120_fp8_blockwise_dispatch_shape( + out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } else { + sm120_fp8_blockwise_dispatch_shape(out_padded, mat_a_padded, mat_b, scales_a_padded, scales_b); + } + return out_padded.slice(0, 0, original_rows); + } +#endif +#endif + + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented fp8_blockwise_scaled_mm for current compute capability: ", sm_version); +} diff --git a/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu b/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu new file mode 100644 index 000000000..0a9e6b7a5 --- /dev/null +++ b/sgl-kernel/csrc/gemm/fp8_gemm_kernel.cu @@ -0,0 +1,1252 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_template.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm89.h +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.16.0/cpp/tensorrt_llm/kernels/cutlass_kernels/fp8_rowwise_gemm/fp8_rowwise_gemm_kernel_template_sm90.h + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "math.hpp" +#include "utils.h" + +using namespace cute; + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CtaShape, + typename WarpShape, + int Stages, + bool WithBias, + typename FP8MathOperator = cutlass::arch::OpMultiplyAdd, + template typename EpilogueVisitor = cutlass::epilogue::threadblock::Sm80EVT, + typename ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<>> +struct DeviceGemmFp8RowwiseSm89 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + using ElementA = ElementType; + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using ElementB = ElementType; + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = OutElementType; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ElementOutput = OutElementType; + using LayoutOutput = cutlass::layout::RowMajor; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using ElementAccumulator = AccumElementType; + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm89; + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using InstructionShape = cutlass::gemm::GemmShape<16, 8, 32>; + // Number of epilogue stages in EVT + static constexpr int EVTEpilogueStages = 1; + + using OutputTileThreadMap = cutlass::epilogue::threadblock:: + OutputTileThreadLayout; + + // Definition of EVT + using accSrc = cutlass::epilogue::threadblock::VisitorAccFetch; + + using ComputeBScale = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiplies, + ElementComputeEpilogue, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using bScaleSrc = cutlass::epilogue::threadblock:: + VisitorRowBroadcast>; + using EpilogueBScale = cutlass::epilogue::threadblock::Sm80EVT; + + using ComputeAScale = cutlass::epilogue::threadblock:: + VisitorCompute; + using aScaleSrc = cutlass::epilogue::threadblock:: + VisitorColBroadcast>; + using EpilogueAScale = cutlass::epilogue::threadblock::Sm80EVT; + + // With bias + using biasSrc = + cutlass::epilogue::threadblock::VisitorRowBroadcast>; + using ComputeAScaleWithBias = cutlass::epilogue::threadblock::VisitorCompute< + cutlass::multiply_add, + ElementC, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EpilogueAScaleWithBias = + cutlass::epilogue::threadblock::Sm80EVT; + + using dTar = cutlass::epilogue::threadblock::VisitorAuxStore< + OutputTileThreadMap, + ElementC, + cutlass::FloatRoundStyle::round_to_nearest, + Stride>; + using EpilogueStore = typename cutlass::platform::conditional< + WithBias, + cutlass::epilogue::threadblock::Sm80EVT, + cutlass::epilogue::threadblock::Sm80EVT>::type; + + using EpilogueOp = EpilogueStore; + + using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmWithVisitor< + ElementA, + LayoutA, + cutlass::ComplexTransform::kNone, + AlignmentA, + ElementB, + LayoutB, + cutlass::ComplexTransform::kNone, + AlignmentB, + ElementC, + LayoutC, + AlignmentC, + ElementAccumulator, + ElementComputeEpilogue, + OperatorClass, + ArchTag, + CtaShape, + WarpShape, + InstructionShape, + EpilogueOp, + ThreadblockSwizzle, + Stages, + FP8MathOperator, + EVTEpilogueStages>::GemmKernel; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm89_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + int64_t lda = a.stride(0); + int64_t ldb = b.stride(1); + int64_t ldc = out.stride(0); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + typename Gemm::Arguments args( + cutlass::gemm::GemmUniversalMode::kGemm, // Mode + {m, n, k}, // Problem size + 1, // Split-k factor + {}, // Epilogue args + ptr_a, // a pointer + ptr_b, // b pointer + nullptr, // c pointer (unused) + nullptr, // d pointer (unused) + m * k, // batch stride a (unused) + n * k, // batch stride b (unused) + m * n, // batch stride c (unused) + m * n, // batch stride d (unused) + lda, // stride a + ldb, // stride b + ldc, // stride c (unused) + ldc); // stride d (unused) + if constexpr (WithBias) { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {ptr_bias, ElementOutput(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } else { + args.epilogue = { + { + { + {}, // Accumulator + {ptr_scales_b, ElementComputeEpilogue(0), {_0{}, _1{}, _0{}}}, + {} // Multiplies + }, + {ptr_scales_a, ElementComputeEpilogue(0), {_1{}, _0{}, _0{}}}, + {} // Multiplies + }, + {ptr_d, {n, _1{}, _0{}}}}; + } + + return args; +} + +template +void launch_sm89_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm89_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm89_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + true>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm89< + ElementInput, + ElementOutput, + AccumElementType, + CtaShape, + WarpShape, + Stages, + false>::Gemm; + return launch_sm89_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm89_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + uint32_t const n = out.size(1); + + if (m == 1) { + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 16) { + // M in (1, 16] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + // M in (16, 64] + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + // M in (64, 128] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + // M in (128, 256] + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 5>(out, a, b, scales_a, scales_b, bias); + } else if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<64, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 7>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 64, 128>, + cutlass::gemm::GemmShape<64, 32, 128>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else if (m <= 512) { + // M in (256, 512) + if (n <= 16384) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 4>(out, a, b, scales_a, scales_b, bias); + } + } else { + // M in (512, inf) + if (n <= 8192) { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 3>(out, a, b, scales_a, scales_b, bias); + } else { + return sm89_fp8_dispatch_bias< + OutType, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 32, 64>, + 2>(out, a, b, scales_a, scales_b, bias); + } + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm90 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + + // A matrix configuration + using ElementA = ElementType; // Element type for A matrix operand + using LayoutA = cutlass::layout::RowMajor; // Layout type for A matrix operand + static constexpr int AlignmentA = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of A + // matrix in units of elements (up to 16 bytes) + + // B matrix configuration + using ElementB = ElementType; // Element type for B matrix operand + using LayoutB = cutlass::layout::ColumnMajor; // Layout type for B matrix operand + static constexpr int AlignmentB = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of B + // matrix in units of elements (up to 16 bytes) + + // C/D matrix configuration + using ElementC = void; // Element type for C matrix operands + using LayoutC = cutlass::layout::RowMajor; // Layout type for C matrix operands + static constexpr int AlignmentC = + 128 / cutlass::sizeof_bits::value; // Memory access granularity/alignment of C matrices in + // units of elements (up to 16 bytes) + + // Output matrix configuration + using ElementOutput = OutElementType; // Element type for output matrix operands + using LayoutOutput = cutlass::layout::RowMajor; // Layout type for output matrix operands + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + // // Auxiliary matrix configuration and other fusion types + // using ElementBias = float; + + // Multiply-accumulate blocking/pipelining details + using ElementAccumulator = AccumElementType; // Element type for internal accumulation + using ElementCompute = float; // Element type for compute + using ElementComputeEpilogue = float; + using ArchTag = cutlass::arch::Sm90; // Tag indicating the minimum SM that supports the intended feature + using OperatorClass = cutlass::arch::OpClassTensorOp; // Operator class tag + using TileShape = CTAShape; // Threadblock-level tile size + + static constexpr bool PONG = false; + static constexpr bool FAST_ACCUM = true; + static constexpr bool USE_BIAS = false; + + using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized + // based on the tile size + using KernelSchedule = cutlass::gemm::collective::KernelScheduleAuto; // Kernel to launch based on the default + // setting in the Collective Builder + // Implement rowwise scaling epilogue. + using XScale = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using WScale = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementOutput, + ElementOutput, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using Compute0 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementComputeEpilogue, // First stage output type. + ElementComputeEpilogue, // First stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiplies, + ElementOutput, + ElementComputeEpilogue, // Second stage input types. + cutlass::FloatRoundStyle::round_to_nearest>; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion::Sm90Compute< + cutlass::multiply_add, + ElementOutput, + ElementComputeEpilogue, + cutlass::FloatRoundStyle::round_to_nearest>; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm90, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementComputeEpilogue, + ElementC, + LayoutC, + AlignmentC, + ElementOutput, + LayoutOutput, + AlignmentOutput, + cutlass::epilogue::TmaWarpSpecialized, + EpilogueEVT>::CollectiveOp; + + using DefaultSchedule = cutlass::gemm::KernelTmaWarpSpecialized; + using PongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpong; + using FastDefaultSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using FastPongSchedule = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + + using SlowAccum = DefaultSchedule; + using FastAccum = FastPongSchedule; // Default apply Pingpong + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutA, + AlignmentA, + ElementB, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; +}; + +template +typename Gemm::Arguments prepare_sm90_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementT = typename Gemm::ElementA; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + ElementOutput const* ptr_bias = nullptr; + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value()) + ptr_bias = reinterpret_cast(bias.value().data_ptr()); + } + ElementOutput* ptr_d = reinterpret_cast(out.data_ptr()); + ElementComputeEpilogue const* ptr_scales_a = reinterpret_cast(scales_a.data_ptr()); + ElementComputeEpilogue const* ptr_scales_b = reinterpret_cast(scales_b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {ptr_a, stride_a, ptr_b, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + ptr_d, + stride_d}}; + if constexpr (WithBias) { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {ptr_bias}, + {}, // Multiplies + }; + } else { + args.epilogue.thread = { + {ptr_scales_a}, + { + {ptr_scales_b}, + {}, // Accumulator + {} // Multiplies + }, + {}, // Multiplies + }; + } + + return args; +} + +template +void launch_sm90_fp8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm90_fp8_args(out, a, b, scales_a, scales_b, bias); + Gemm gemm_op; + + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template < + typename OutType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename TileSchedulerType> +void sm90_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias, + bool fast_accum = true, + bool use_persistent = false) { + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + + if (bias) { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + using Gemm = typename DeviceGemmFp8RowwiseSm90< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape, + ClusterShape, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>::Gemm; + return launch_sm90_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } +} + +template +void sm90_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + uint32_t const m = a.size(0); + using FastPingpongScheduler = cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum; + using FastBasicScheduler = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum; + using PersistentTileScheduler = cutlass::gemm::PersistentScheduler; + using BasicTileScheduler = void; + if (m <= 1) { + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + FastBasicScheduler, + BasicTileScheduler>(out, a, b, scales_a, scales_b, bias); + } + if (m <= 64) { + // m in [1, 64] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 256) { + // m in (64, 256] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_64, _64, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else if (m <= 1024) { + // m in (256, 1024] + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_1, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } else { + // m in (1024, inf) + return sm90_fp8_dispatch_bias< + OutType, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + FastPingpongScheduler, + PersistentTileScheduler>(out, a, b, scales_a, scales_b, bias); + } +} +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 +template < + typename ElementType, + typename OutElementType, + typename AccumElementType, + typename CTAShape, + typename ClusterShape, + typename MainloopScheduleType, + typename EpilogueScheduleType, + typename TileSchedulerType = void, + bool WithBias = false> +struct DeviceGemmFp8RowwiseSm100 { + static_assert(std::is_same_v, "ElementType must be FP8(e4m3)"); + using TileShape = CTAShape; + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + using ElementComputeEpilogue = float; + using ScaleA = cutlass::epilogue::fusion::Sm90ColBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<0>, cute::Int<0>>>; + + using ScaleB = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + ElementComputeEpilogue, + ElementComputeEpilogue, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Bias = cutlass::epilogue::fusion::Sm90RowBroadcast< + 0, + TileShape, + OutElementType, + OutElementType, + cute::Stride, cute::Int<1>, cute::Int<0>>>; + + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using LayoutA = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + + using LayoutB = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + + using ElementC = void; + using LayoutC = cutlass::layout::RowMajor; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using LayoutD = cutlass::layout::RowMajor; + static constexpr int AlignmentD = AlignmentC; + + using Compute1MulAdd = cutlass::epilogue::fusion:: + Sm90Compute; + using Compute1Mul = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute = typename std::conditional_t< + WithBias, + cutlass::epilogue::fusion::Sm90EVT, + cutlass::epilogue::fusion::Sm90EVT>; + using ArgumentType = typename EVTCompute::Arguments; + // MMA type + using ElementAccumulator = AccumElementType; + + // Epilogue types + using ElementCompute = float; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementC, + LayoutC, + AlignmentC, + OutElementType, + LayoutD, + AlignmentD, + EpilogueScheduleType, + EVTCompute>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + cutlass::arch::Sm100, + cutlass::arch::OpClassTensorOp, + ElementType, + LayoutA, + AlignmentA, + ElementType, + LayoutB, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopScheduleType>::CollectiveOp; + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + template + static auto args_from_tensor(torch::Tensor const& tensor) { + using Arguments = typename Descriptor::Arguments; + auto* data_ptr = static_cast(tensor.data_ptr()); + static_assert( + std::is_same_v || std::is_same_v || std::is_same_v); + return Arguments{data_ptr}; + } + + public: + static ArgumentType prepare_args( + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + std::optional const& bias = std::nullopt) { + auto a_args = args_from_tensor(a_scales); + auto b_args = args_from_tensor(b_scales); + + typename EVTCompute0::Arguments evt0_args{b_args, {}, {}}; + + if constexpr (WithBias) { + auto bias_args = args_from_tensor(bias.value()); + return ArgumentType{a_args, evt0_args, bias_args, {}}; + } else { + return ArgumentType{a_args, evt0_args, {}}; + } + } +}; + +template +typename GemmType::Gemm::Arguments prepare_sm100_fp8_args( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using Gemm = typename GemmType::Gemm; + using ElementT = typename Gemm::ElementA; + using ElementC = typename Gemm::ElementC; + using ElementOutput = typename Gemm::ElementD; + using ElementComputeEpilogue = float; + using GemmKernel = typename Gemm::GemmKernel; + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = StrideC; + using StrideAux = StrideC; + + int32_t m = a.size(0); + int32_t n = b.size(1); + int32_t k = a.size(1); + + ElementT const* ptr_a = reinterpret_cast(a.data_ptr()); + ElementT const* ptr_b = reinterpret_cast(b.data_ptr()); + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, cute::make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, cute::make_shape(n, k, 1)); + StrideC stride_c = cutlass::make_cute_packed_stride(StrideC{}, cute::make_shape(m, n, 1)); + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, cute::make_shape(m, n, 1)); + StrideAux aux_stride = stride_d; + + typename GemmKernel::MainloopArguments mainloop_args{ptr_a, stride_a, ptr_b, stride_b}; + + typename GemmKernel::ProblemShape prob_shape = {m, n, k, 1}; + cutlass::KernelHardwareInfo hw_info; + typename GemmKernel::TileSchedulerArguments scheduler = {}; + + auto ptr_c = static_cast(out.data_ptr()); + + auto prepare_epilogue_args = [&](const c10::optional& bias = c10::nullopt) { + if constexpr (WithBias) { + TORCH_CHECK(bias.has_value(), "Bias tensor is required but not provided."); + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b, bias.value()), ptr_c, stride_c, ptr_c, stride_d}; + } else { + return typename GemmKernel::EpilogueArguments{ + GemmType::prepare_args(scales_a, scales_b), ptr_c, stride_c, ptr_c, stride_d}; + } + }; + + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGemm, + prob_shape, + mainloop_args, + prepare_epilogue_args(bias), + hw_info, + scheduler}; + return args; +} + +template +void launch_sm100_fp8_scaled_mm( + torch::Tensor& out, + torch::Tensor const& a, + torch::Tensor const& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + auto args = prepare_sm100_fp8_args(out, a, b, scales_a, scales_b, bias); + + typename Gemm::Gemm gemm_op; + size_t workspace_size = gemm_op.get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + auto stream = at::cuda::getCurrentCUDAStream(a.get_device()); + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK(can_implement == cutlass::Status::kSuccess) + auto status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess) +} + +template +void sm100_fp8_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using CTAShapeDefault = Shape<_256, _128, _64>; + using ClusterShapeDefault = Shape<_2, _2, _1>; + + using CTAShape256 = Shape<_128, _128, _128>; + using ClusterShape256 = Shape<_2, _1, _1>; + + using CTAShape64 = Shape<_64, _64, _128>; + using ClusterShape64 = Shape<_1, _1, _1>; + + using CTAShape16 = Shape<_64, _64, _128>; + using ClusterShape16 = Shape<_1, _4, _1>; + + using MainloopScheduleType = cutlass::gemm::collective::KernelScheduleAuto; + using EpilogueScheduleType = cutlass::epilogue::collective::EpilogueScheduleAuto; + using TileSchedulerType = void; + + using ElementInput = cutlass::float_e4m3_t; + using ElementOutput = OutType; + using AccumElementType = float; + + // Gemm type with bias + using BiasGemmDefault = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm256 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape256, + ClusterShape256, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm64 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape64, + ClusterShape64, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + using BiasGemm16 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape16, + ClusterShape16, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + true>; + + // Gemm type without bias + using GemmDefault = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShapeDefault, + ClusterShapeDefault, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm256 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape256, + ClusterShape256, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm64 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape64, + ClusterShape64, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + using Gemm16 = DeviceGemmFp8RowwiseSm100< + ElementInput, + ElementOutput, + AccumElementType, + CTAShape16, + ClusterShape16, + MainloopScheduleType, + EpilogueScheduleType, + TileSchedulerType, + false>; + + // next power of 2 (minimum 16) + uint32_t const m = a.size(0); + uint32_t const mp2 = std::max(static_cast(16), next_pow_2(m)); + + if (bias) { + if (mp2 <= 16) { + // m in [1, 16] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 64) { + // m in (16, 64] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 256) { + // m in (64, 256] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + // m in (256, inf] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } + } else { + if (mp2 <= 16) { + // m in [1, 16] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 64) { + // m in (16, 64] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else if (mp2 <= 256) { + // m in (64, 256] + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } else { + return launch_sm100_fp8_scaled_mm(out, a, b, scales_a, scales_b, bias); + } + } +} + +template +void sm100_fp8_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + return sm100_fp8_dispatch_bias(out, a, b, scales_a, scales_b, bias); +} +#endif + +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + + TORCH_CHECK( + (mat_a.size(1) * mat_a.element_size()) % 16 == 0, "mat_a must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK( + (mat_b.size(0) * mat_b.element_size()) % 16 == 0, "mat_b must be multiple of 16 bytes for memory alignment"); + TORCH_CHECK(mat_a.scalar_type() == torch::kFloat8_e4m3fn, "mat_a must be Float8_e4m3fn"); + TORCH_CHECK(mat_b.scalar_type() == torch::kFloat8_e4m3fn, "mat_b must be Float8_e4m3fn"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + TORCH_CHECK((out.size(1) * out.element_size()) % 16 == 0, "out must be multiple of 16 bytes for memory alignment"); + + auto sm_version = getSMVersion(); + +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100 +#if CUDA_VERSION >= 12090 + || sm_version == 103 +#endif + ) { + if (out_dtype == torch::kBFloat16) { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm100_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + if (sm_version >= 90) { + if (out_dtype == torch::kBFloat16) { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + +#if defined CUDA_VERSION && CUDA_VERSION >= 12040 + if (sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_fp8_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } + return out; + } +#endif + + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented fp8_scaled_mm for current compute capability: ", sm_version); +} diff --git a/sgl-kernel/csrc/gemm/gptq/compat.cuh b/sgl-kernel/csrc/gemm/gptq/compat.cuh new file mode 100644 index 000000000..506eeb769 --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/compat.cuh @@ -0,0 +1,62 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _compat_cuh +#define _compat_cuh + +namespace sglang { +namespace gptq { +// atomicAdd for half types, to support CC < 7.x + +__device__ __forceinline__ void atomicAdd_half(half* address, half val) { + unsigned int* address_as_ui = (unsigned int*)((char*)address - ((size_t)address & 2)); + unsigned int old = *address_as_ui; + unsigned int assumed; + + do { + assumed = old; + __half_raw hsum; + hsum.x = (size_t)address & 2 ? (old >> 16) : (old & 0xffff); + half tmpres = __hadd(hsum, val); + hsum = __half_raw(tmpres); + old = (size_t)address & 2 ? (old & 0xffff) | (hsum.x << 16) : (old & 0xffff0000) | hsum.x; + old = atomicCAS(address_as_ui, assumed, old); + } while (assumed != old); +} + +// atomicAdd for half2 types + +__device__ __forceinline__ void atomicAdd_half2(half2* address, half2 val) { + unsigned int* address_as_ui = (unsigned int*)address; + unsigned int old = *address_as_ui; + unsigned int assumed; + do { + assumed = old; + half2 old_val = *((half2*)&old); + half2 new_val = __hadd2(old_val, val); + old = atomicCAS(address_as_ui, assumed, *((unsigned int*)&new_val)); + } while (assumed != old); +} + +// + +#if defined(__CUDA_ARCH__) || defined(USE_ROCM) +#if __CUDA_ARCH__ < 700 || defined(USE_ROCM) + +__device__ __forceinline__ void atomicAdd(half* address, half val) { + atomicAdd_half(address, val); +} + +#if __CUDA_ARCH__ < 600 || defined(USE_ROCM) +__device__ __forceinline__ void atomicAdd(half2* address, half2 val) { + atomicAdd_half2(address, val); +} +#endif + +#endif +#endif + +} // namespace gptq +} // namespace sglang +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu b/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu new file mode 100644 index 000000000..4dd5d8a24 --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/gptq_kernel.cu @@ -0,0 +1,1950 @@ +/* +Adapted from https://github.com/turboderp/exllamav2 and +https://github.com/qwopqwop200/GPTQ-for-LLaMa +*/ + +#include +#include +#include +#include +#include + +#include +#include + +#include "compat.cuh" +#include "matrix_view.cuh" +#include "qdq_2.cuh" +#include "qdq_3.cuh" +#include "qdq_4.cuh" +#include "qdq_8.cuh" + +namespace sglang { +namespace gptq { + +#define BLOCK_KN_SIZE 128 +#define BLOCK_M_SIZE_MAX 8 +#define MAX_GROUPS_IN_BLOCK (BLOCK_KN_SIZE / 32) +#define MAX_Q_GEMM_ROWS 50 +#define MAX_Q_GEMM_ROWS_8BIT 24 +#define MAX_ALT_GEMM_ROWS 8 +#define THREADS_X 32 +#define THREADS_Y 32 +#define DIVIDE(x, size) (((x) + (size) - 1) / (size)) + +#if defined(USE_ROCM) +#include +__host__ __forceinline__ hipblasStatus_t __compat_hipblasHgemm( + hipblasHandle_t handle, + hipblasOperation_t transA, + hipblasOperation_t transB, + int m, + int n, + int k, + const half* alpha, + const half* AP, + int lda, + const half* BP, + int ldb, + const half* beta, + half* CP, + int ldc) { + return hipblasHgemm( + handle, + transA, + transB, + m, + n, + k, + reinterpret_cast(alpha), + reinterpret_cast(AP), + lda, + reinterpret_cast(BP), + ldb, + reinterpret_cast(beta), + reinterpret_cast(CP), + ldc); +} +#define hipblasHgemm __compat_hipblasHgemm + +// Previous version of PyTorch were converting to rocBLAS instead of hipBLAS. +#define rocblas_operation_none HIPBLAS_OP_N +#define rocblas_hgemm __compat_hipblasHgemm +#endif + +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, const half2 g_result) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hadd2(result, g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __half2float(__low2half(result)) + __half2float(__high2half(result)); +} + +__forceinline__ __device__ half2 dot22_8(half2 (&dq)[4], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_16(half2 (&dq)[8], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ half2 dot22_32(half2 (&dq)[16], const half* a_ptr, const half2 g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + return __hfma2(result, __halves2half2(qs_h, qs_h), g_result); +} + +__forceinline__ __device__ float dot22_8_f(half2 (&dq)[4], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 4; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float dot22_16_f(half2 (&dq)[8], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ float +dot22_32_f(half2 (&dq)[16], const half* a_ptr, const float g_result, const float qs_f) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + float result_f = __half2float(__low2half(result)) + __half2float(__high2half(result)); + return fma(result_f, qs_f, g_result); +} + +__forceinline__ __device__ half dot22_8_h(half2 (&dq)[4], const half* a_ptr, const half g_result, const half qs_h) { + // Use FP32 accumulator to avoid potential overflow since unscaled weights are + // in the range -128..127 + + float result = {}; +#pragma unroll + for (int i = 0; i < 4; i++) { + half2 w01 = dq[i]; + float w0 = __low2float(w01); + float w1 = __high2float(w01); + float x0 = __half2float(*a_ptr++); + float x1 = __half2float(*a_ptr++); + result = fma(w0, x0, result); + result = fma(w1, x1, result); + } + float qs = __half2float(qs_h); + result *= qs; + half result_h = __float2half_rn(result); + return __hadd(result_h, g_result); +} + +__forceinline__ __device__ half dot22_16_h(half2 (&dq)[8], const half* a_ptr, const half g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 8; i++) + result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} + +__forceinline__ __device__ half dot22_32_h(half2 (&dq)[16], const half* a_ptr, const half g_result, const half qs_h) { + half2 result = {}; + const half2* a2_ptr = (const half2*)a_ptr; +#pragma unroll + for (int i = 0; i < 16; i += 1) + result = __hfma2(dq[i], *a2_ptr++, result); + half result_h = __hadd(__low2half(result), __high2half(result)); + return __hfma(result_h, qs_h, g_result); +} + +typedef void (*fp_gemm_half_q_half_gptq_kernel)( + const half*, + const uint32_t*, + const uint32_t*, + const half*, + half*, + const int, + const int, + const int, + const int, + const int*); + +template +__global__ void gemm_half_q_half_gptq_4bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + float scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + + // Column result + float block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_f(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + +#pragma unroll + for (int j = 0; j < 4; j++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][4]; + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = fma(dot22_8_f(dq[0], a_ptr + m * a_stride), scales[0], block_c[m][0]); + block_c[m][1] = fma(dot22_8_f(dq[1], a_ptr + m * a_stride), scales[1], block_c[m][1]); + block_c[m][2] = fma(dot22_8_f(dq[2], a_ptr + m * a_stride), scales[2], block_c[m][2]); + block_c[m][3] = fma(dot22_8_f(dq[3], a_ptr + m * a_stride), scales[3], block_c[m][3]); + } + + b_ptr += size_n; + a_ptr += 8; + } + + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(__float2half_rn(block_c[m][0]), __float2half_rn(block_c[m][1])); + half2 result23 = __halves2half2(__float2half_rn(block_c[m][2]), __float2half_rn(block_c[m][3])); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_2bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 2); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 1; j++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][8]; + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1); + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1); + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1); + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_16_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_16_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_16_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_16_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + + b_ptr += size_n; + a_ptr += 16; + } + + k += 16; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_3bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / 32 * 3; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 1; j++) { + int4 load_int4[3]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[2] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][16]; + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1); + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1); + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1); + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1); + +#pragma unroll + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_32_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_32_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_32_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_32_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + a_ptr += 32; + } + + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +template +__global__ void gemm_half_q_half_gptq_8bit_kernel( + const half* __restrict__ a, + const uint32_t* __restrict__ b_q_weight, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + half* __restrict__ c, + const int size_m, + const int size_n, + const int size_k, + const int groups, + const int* __restrict__ b_q_perm) { + MatrixView_half a_(a, size_m, size_k); + MatrixView_half_rw c_(c, size_m, size_n); + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto t = threadIdx.x; + + // Block + auto offset_n = blockIdx.x * BLOCK_KN_SIZE * 4; + auto offset_m = blockIdx.y * m_count; + auto offset_k = blockIdx.z * BLOCK_KN_SIZE; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + int n = offset_n + t * 4; + + // Preload block_a + __shared__ half block_a[m_count][BLOCK_KN_SIZE]; + + if (offset_k + t < end_k) { + for (int m = 0; m < m_count; ++m) { + const half* a_ptr = a_.item_ptr(offset_m + m, 0); + half* block_a_ptr = block_a[m]; + + half a0; + if (b_q_perm) + a0 = a_ptr[b_q_perm[offset_k + t]]; + else + a0 = a_ptr[offset_k + t]; + block_a_ptr[t] = a0; + } + } + + // Zero output + if (n >= size_n) return; + + if (blockIdx.z == 0) { + for (int m = 0; m < m_count; m++) + *((uint64_t*)c_.item_ptr(offset_m + m, n)) = 0; + } + + __syncthreads(); + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // a, b offset + int qk = offset_k / (32 / 8); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + const half* a_ptr = &block_a[0][0]; + int a_stride = BLOCK_KN_SIZE; + + // Initial group + int zeros[4]; + half scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + // Column result + half block_c[m_count][4] = {}; + + // Dequantize and multiply + int k = offset_k; + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4(scales, group, n); + } + +#pragma unroll + for (int j = 0; j < 4; j++) { + int4 load_int4[2]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][4]; + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1); + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1); + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1); + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1); + + for (int m = 0; m < m_count; m++) { + block_c[m][0] = dot22_8_h(dq[0], a_ptr + m * a_stride, block_c[m][0], scales[0]); + block_c[m][1] = dot22_8_h(dq[1], a_ptr + m * a_stride, block_c[m][1], scales[1]); + block_c[m][2] = dot22_8_h(dq[2], a_ptr + m * a_stride, block_c[m][2], scales[2]); + block_c[m][3] = dot22_8_h(dq[3], a_ptr + m * a_stride, block_c[m][3], scales[3]); + } + a_ptr += 8; + } + k += 32; + } + + for (int m = 0; m < m_count; m++) { + half2* out = (half2*)c_.item_ptr(offset_m + m, n); + half2 result01 = __halves2half2(block_c[m][0], block_c[m][1]); + half2 result23 = __halves2half2(block_c[m][2], block_c[m][3]); + atomicAdd(out, result01); + atomicAdd(out + 1, result23); + } +} + +fp_gemm_half_q_half_gptq_kernel pick_gemm_half_q_half_gptq_kernel(bool first_block, const int m_count, const int bit) { +#define SELECT_KERNEL(M_COUNT) \ + if (m_count == M_COUNT) { \ + if (bit == 2) return gemm_half_q_half_gptq_2bit_kernel; \ + if (bit == 3) return gemm_half_q_half_gptq_3bit_kernel; \ + if (bit == 4) return gemm_half_q_half_gptq_4bit_kernel; \ + if (bit == 8) return gemm_half_q_half_gptq_8bit_kernel; \ + } +#if BLOCK_M_SIZE_MAX >= 1 + SELECT_KERNEL(1); +#endif +#if BLOCK_M_SIZE_MAX >= 2 + SELECT_KERNEL(2); +#endif +#if BLOCK_M_SIZE_MAX >= 3 + SELECT_KERNEL(3); +#endif +#if BLOCK_M_SIZE_MAX >= 4 + SELECT_KERNEL(4); +#endif +#if BLOCK_M_SIZE_MAX >= 5 + SELECT_KERNEL(5); +#endif +#if BLOCK_M_SIZE_MAX >= 6 + SELECT_KERNEL(6); +#endif +#if BLOCK_M_SIZE_MAX >= 7 + SELECT_KERNEL(7); +#endif +#if BLOCK_M_SIZE_MAX >= 8 + SELECT_KERNEL(8); +#endif + return NULL; +} + +void gemm_half_q_half_cuda_part( + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_q_perm, + half* c, + int size_m, + int size_n, + int size_k, + int m_count, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE * 4); + gridDim.y = DIVIDE(size_m, m_count); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + fp_gemm_half_q_half_gptq_kernel kernel = pick_gemm_half_q_half_gptq_kernel(true, m_count, bit); + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, c, size_m, size_n, size_k, groups, b_q_perm); +} + +__global__ void reconstruct_exllama_8bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q8_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 8); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 4; p++) { + int4 load_int4[2]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][4]; + dequant_8bit_8(load_int4[0].x, load_int4[1].x, dq[0], size_n, zeros[0] + 1); + dequant_8bit_8(load_int4[0].y, load_int4[1].y, dq[1], size_n, zeros[1] + 1); + dequant_8bit_8(load_int4[0].z, load_int4[1].z, dq[2], size_n, zeros[2] + 1); + dequant_8bit_8(load_int4[0].w, load_int4[1].w, dq[3], size_n, zeros[3] + 1); + + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_4bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q4_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 4); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + half2 z1z16[4][2]; + half2 y1y16[4][2]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + dequant_4bit_8_prep_zero(zeros[0] + 1, z1z16[0], y1y16[0]); + dequant_4bit_8_prep_zero(zeros[1] + 1, z1z16[1], y1y16[1]); + dequant_4bit_8_prep_zero(zeros[2] + 1, z1z16[2], y1y16[2]); + dequant_4bit_8_prep_zero(zeros[3] + 1, z1z16[3], y1y16[3]); + } + + for (int p = 0; p < 4; p++) { + half2 dq[4][4]; + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + dequant_4bit_8_gptq(load_int4.x, dq[0], z1z16[0], y1y16[0], size_n, false); + dequant_4bit_8_gptq(load_int4.y, dq[1], z1z16[1], y1y16[1], size_n, false); + dequant_4bit_8_gptq(load_int4.z, dq[2], z1z16[2], y1y16[2], size_n, false); + dequant_4bit_8_gptq(load_int4.w, dq[3], z1z16[3], y1y16[3], size_n, false); + + b_ptr += size_n; + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 4; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_3bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q3_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / 32 * 3; + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 1; p++) { + int4 load_int4[3]; + load_int4[0] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[1] = *((int4*)b_ptr); + b_ptr += size_n; + load_int4[2] = *((int4*)b_ptr); + b_ptr += size_n; + + half2 dq[4][16]; + dequant_3bit_32(load_int4[0].x, load_int4[1].x, load_int4[2].x, dq[0], size_n, zeros[0] + 1); + dequant_3bit_32(load_int4[0].y, load_int4[1].y, load_int4[2].y, dq[1], size_n, zeros[1] + 1); + dequant_3bit_32(load_int4[0].z, load_int4[1].z, load_int4[2].z, dq[2], size_n, zeros[2] + 1); + dequant_3bit_32(load_int4[0].w, load_int4[1].w, load_int4[2].w, dq[3], size_n, zeros[3] + 1); + + if (b_q_perm) { + for (int j = 0; j < 16; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 16; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +__global__ void reconstruct_exllama_2bit_kernel( + const uint32_t* __restrict__ b_q_weight, + const int* __restrict__ b_q_perm, + const uint32_t* __restrict__ b_gptq_qzeros, + const half* __restrict__ b_gptq_scales, + const int size_k, + const int size_n, + const int groups, + half* __restrict__ b) { + MatrixView_half_rw b_(b, size_k, size_n); + MatrixView_q2_row b_gptq_qzeros_(b_gptq_qzeros, groups, size_n); + MatrixView_half b_gptq_scales_(b_gptq_scales, groups, size_n); + + auto offset_k = BLOCK_KN_SIZE * blockIdx.y; + auto offset_n = BLOCK_KN_SIZE * blockIdx.x * 4; + + int end_k = min(offset_k + BLOCK_KN_SIZE, size_k); + + // Preload remapping table + __shared__ int perm[BLOCK_KN_SIZE]; + auto t = threadIdx.x; + + if (b_q_perm) { + if (offset_k + t < size_k) perm[t] = b_q_perm[offset_k + t]; + } + + // Column + int n = offset_n + t * 4; + if (n >= size_n) return; + + // Find initial group + int groupsize = size_k / groups; + int group = offset_k / groupsize; + int nextgroup = offset_k + groupsize; + + // b offset + int qk = offset_k / (32 / 2); + + const uint32_t* b_ptr = b_q_weight + qk * size_n + n; + + // Initial zeros/scale + int zeros[4]; + half2 scales[4]; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + + __syncthreads(); + + int k = offset_k; + int lk = 0; + + while (k < end_k) { + if (k == nextgroup) { + group++; + nextgroup += groupsize; + b_gptq_qzeros_.item4(zeros, group, n); + b_gptq_scales_.item4_h2(scales, group, n); + } + + for (int p = 0; p < 2; p++) { + const int4* b_ptr4 = (int4*)b_ptr; + int4 load_int4 = *b_ptr4; + + half2 dq[4][8]; + dequant_2bit_16(load_int4.x, dq[0], size_n, zeros[0] + 1); + dequant_2bit_16(load_int4.y, dq[1], size_n, zeros[1] + 1); + dequant_2bit_16(load_int4.z, dq[2], size_n, zeros[2] + 1); + dequant_2bit_16(load_int4.w, dq[3], size_n, zeros[3] + 1); + + b_ptr += size_n; + // half* dqh = (half*)dq; + if (b_q_perm) { + for (int j = 0; j < 8; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + perm[lk++], n, __low2half(dq[0][j]), __low2half(dq[1][j]), __low2half(dq[2][j]), __low2half(dq[3][j])); + b_.set4( + perm[lk++], + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } else { + for (int j = 0; j < 8; j++) { + for (int v = 0; v < 4; v++) + dq[v][j] = __hmul2(scales[v], dq[v][j]); + b_.set4( + offset_k + lk++, + n, + __low2half(dq[0][j]), + __low2half(dq[1][j]), + __low2half(dq[2][j]), + __low2half(dq[3][j])); + b_.set4( + offset_k + lk++, + n, + __high2half(dq[0][j]), + __high2half(dq[1][j]), + __high2half(dq[2][j]), + __high2half(dq[3][j])); + } + } + } + k += 32; + } +} + +void reconstruct_exllama( + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_q_perm, + half* out, + int height, + int width, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + gridDim.y = DIVIDE(height, BLOCK_KN_SIZE); + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); + + auto reconstruct_exllama_kernel = reconstruct_exllama_4bit_kernel; + if (bit == 2) { + reconstruct_exllama_kernel = reconstruct_exllama_2bit_kernel; + } else if (bit == 3) { + reconstruct_exllama_kernel = reconstruct_exllama_3bit_kernel; + } else if (bit == 8) { + reconstruct_exllama_kernel = reconstruct_exllama_8bit_kernel; + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + reconstruct_exllama_kernel<<>>( + b_q_weight, b_q_perm, b_gptq_qzeros, b_gptq_scales, height, width, groups, out); +} + +__global__ void gemm_half_q_half_alt_4bit_kernel( + const half2* __restrict__ vec, + const uint32_t* __restrict__ mat, + half* __restrict__ mul, + const half* __restrict__ scales, + const uint32_t* __restrict__ zeros, + const int* __restrict__ g_idx, + int batch, + int height, + int width) { + int zero_width = width / 8; + int vec_height = height * 4; + const int blockwidth2 = BLOCK_KN_SIZE / 2; + auto b = blockIdx.y * BLOCK_M_SIZE_MAX; + int b_end = min(BLOCK_M_SIZE_MAX, batch - b); + auto h = BLOCK_KN_SIZE * blockIdx.z / 8; + int h_end = min(BLOCK_KN_SIZE / 8, height - h) * 4; + auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2]; + if (threadIdx.x < h_end) { + for (int m = 0; m < b_end; ++m) { + blockvec[m][threadIdx.x] = vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 + threadIdx.x]; + } + } + + __shared__ half2 deq2[256][8]; + auto val = threadIdx.x / 8; + auto off = threadIdx.x % 8; + for (; val < 256; val += BLOCK_KN_SIZE / 8) { + deq2[val][off] = __halves2half2(__int2half_rn(val & 0xF), __int2half_rn(val >> 4)); + } + + if (blockIdx.z == 0) { + for (int m = 0; m < b_end; m++) + mul[(b + m) * width + w] = __int2half_rn(0); + } + __syncthreads(); + + int i = width * h + w; + int g_h = h * 8; + int k = 0; + int z_w = w / 8; + int z_mod = (w % 8) * 4; + half2 res2; + half res[BLOCK_M_SIZE_MAX] = {}; + + unsigned int tmp; + while (k < h_end) { + tmp = mat[i]; + half2 scales_tmp[4]; + half2 zeros_tmp[4]; + for (int tmp_k = 0; tmp_k < 4; tmp_k++) { + int g = g_idx[g_h + (k + tmp_k) * 2]; + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1]; + half scale_f = scales[g * width + w]; + half scale_f2 = scales[g2 * width + w]; + half2 scale = __halves2half2(scale_f, scale_f2); + half2 zero = __halves2half2( + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xF) - 1)), + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xF) - 1))); + scales_tmp[tmp_k] = scale; + zeros_tmp[tmp_k] = zero; + } + for (int m = 0; m < b_end; m++) { +#ifndef USE_ROCM + res2 = {}; +#else + res2.x = __half_as_ushort(__float2half(0)); + res2.y = __half_as_ushort(__float2half(0)); +#endif + res2 = __hfma2(__hfma2(deq2[(tmp >> 0) & 0xff][off], scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 8) & 0xff][off], scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 16) & 0xff][off], scales_tmp[2], zeros_tmp[2]), blockvec[m][k + 2], res2); + res2 = __hfma2(__hfma2(deq2[(tmp >> 24) & 0xff][off], scales_tmp[3], zeros_tmp[3]), blockvec[m][k + 3], res2); +#ifndef USE_ROCM + res[m] = __hadd(res[m], __hadd(res2.x, res2.y)); +#else + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y))); +#endif + } + i += width; + k += 4; + } + for (int m = 0; m < b_end; m++) { + atomicAdd(&mul[(b + m) * width + w], res[m]); + } +} + +__global__ void gemm_half_q_half_alt_8bit_kernel( + const half2* __restrict__ vec, + const uint32_t* __restrict__ mat, + half* __restrict__ mul, + const half* __restrict__ scales, + const uint32_t* __restrict__ zeros, + const int* __restrict__ g_idx, + int batch, + int height, + int width) { + int zero_width = width / 4; + int vec_height = height * 2; + const int blockwidth2 = BLOCK_KN_SIZE / 2; + auto b = blockIdx.y * BLOCK_M_SIZE_MAX; + int b_end = min(BLOCK_M_SIZE_MAX, batch - b); + auto h = BLOCK_KN_SIZE * blockIdx.z / 4; + int h_end = min(BLOCK_KN_SIZE / 4, height - h) * 2; + auto w = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + + __shared__ half2 blockvec[BLOCK_M_SIZE_MAX][blockwidth2]; + if (threadIdx.x < h_end) { + for (int m = 0; m < b_end; ++m) { + blockvec[m][threadIdx.x] = vec[(m + b) * vec_height + blockIdx.z * BLOCK_KN_SIZE / 2 + threadIdx.x]; + } + } + + if (blockIdx.z == 0) { + for (int m = 0; m < b_end; m++) + mul[(b + m) * width + w] = __int2half_rn(0); + } + __syncthreads(); + + int i = width * h + w; + int g_h = h * 4; + int k = 0; + int z_w = w / 4; + int z_mod = (w % 4) * 8; + half2 res2; + half res[BLOCK_M_SIZE_MAX] = {}; + + unsigned int tmp; + while (k < h_end) { + tmp = mat[i]; + half2 scales_tmp[2]; + half2 zeros_tmp[2]; + for (int tmp_k = 0; tmp_k < 2; tmp_k++) { + int g = g_idx[g_h + (k + tmp_k) * 2]; + int g2 = g_idx[g_h + (k + tmp_k) * 2 + 1]; + half scale_f = scales[g * width + w]; + half scale_f2 = scales[g2 * width + w]; + half2 scale = __halves2half2(scale_f, scale_f2); + half2 zero = __halves2half2( + __hmul(scale_f, __int2half_rn(-((zeros[g * zero_width + z_w] >> z_mod) & 0xff) - 1)), + __hmul(scale_f2, __int2half_rn(-((zeros[g2 * zero_width + z_w] >> z_mod) & 0xff) - 1))); + scales_tmp[tmp_k] = scale; + zeros_tmp[tmp_k] = zero; + } + for (int m = 0; m < b_end; m++) { +#ifndef USE_ROCM + res2 = {}; +#else + res2.x = __half_as_ushort(__float2half(0)); + res2.y = __half_as_ushort(__float2half(0)); +#endif + half2 v12 = __halves2half2(__int2half_rn(tmp & 0xFF), __int2half_rn((tmp >> 8) & 0xFF)); + res2 = __hfma2(__hfma2(v12, scales_tmp[0], zeros_tmp[0]), blockvec[m][k + 0], res2); + half2 v34 = __halves2half2(__int2half_rn((tmp >> 16) & 0xFF), __int2half_rn((tmp >> 24) & 0xFF)); + res2 = __hfma2(__hfma2(v34, scales_tmp[1], zeros_tmp[1]), blockvec[m][k + 1], res2); +#ifndef USE_ROCM + res[m] = __hadd(res[m], __hadd(res2.x, res2.y)); +#else + res[m] = __hadd(res[m], __hadd(__ushort_as_half(res2.x), __ushort_as_half(res2.y))); +#endif + } + i += width; + k += 2; + } + for (int m = 0; m < b_end; m++) { + atomicAdd(&mul[(b + m) * width + w], res[m]); + } +} + +void gemm_half_q_half_alt( + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* c, + int size_m, + int size_n, + int size_k, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + blockDim.z = 1; + gridDim.x = DIVIDE(size_n, BLOCK_KN_SIZE); + gridDim.y = DIVIDE(size_m, BLOCK_M_SIZE_MAX); + gridDim.z = DIVIDE(size_k, BLOCK_KN_SIZE); + + auto kernel = gemm_half_q_half_alt_4bit_kernel; + if (bit == 8) { + kernel = gemm_half_q_half_alt_8bit_kernel; + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + (const half2*)a, b_q_weight, c, b_gptq_scales, b_gptq_qzeros, b_g_idx, size_m, size_k / 32 * bit, size_n); +} + +template +__global__ void reconstruct_gptq_kernel( + const uint32_t* __restrict__ w, + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int* __restrict__ g_idx, + const int height, + const int width, + const int group, + half* __restrict__ out) { + // Start of block + + auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + auto row = blockIdx.y * 32 / bit; + if (column >= width) return; + + // Views + + MatrixView_half_rw out_(out, height, width); + MatrixView_half w_scales_(w_scales, group, width); + T w_zeros_(w_zeros, group, width); + + uint32_t w_read = w[blockIdx.y * width + column]; + half* out_ptr = out_.item_ptr(row, column); + +#pragma unroll + for (int s = 0; s < 32; s += bit) { + int group = g_idx[row + s / bit]; + half w_scale = w_scales_.item(group, column); + uint32_t w_zero = w_zeros_.item(group, column) + 1; + half w_item = __hmul(__int2half_rn((int)((w_read >> s) & ((1 << bit) - 1)) - w_zero), w_scale); + *out_ptr = w_item; + out_ptr += out_.width; + } +} + +__global__ void reconstruct_gptq_3bit_kernel( + const uint32_t* __restrict__ w, + const half* __restrict__ w_scales, + const uint32_t* __restrict__ w_zeros, + const int* __restrict__ g_idx, + const int height, + const int width, + const int group, + half* __restrict__ out) { + // Start of block + auto column = BLOCK_KN_SIZE * blockIdx.x + threadIdx.x; + auto row = blockIdx.y * 32; + if (column >= width) return; + + // Views + + MatrixView_half_rw out_(out, height, width); + MatrixView_half w_scales_(w_scales, group, width); + MatrixView_q3_row w_zeros_(w_zeros, group, width); + + uint32_t w1 = w[(blockIdx.y * 3) * width + column]; + uint32_t w2 = w[(blockIdx.y * 3 + 1) * width + column]; + uint32_t w3 = w[(blockIdx.y * 3 + 2) * width + column]; + half* out_ptr = out_.item_ptr(row, column); + +#pragma unroll + for (int i = 0; i < 32; i += 1) { + int group = g_idx[row + i]; + half w_scale = w_scales_.item(group, column); + uint32_t w_zero = w_zeros_.item(group, column) + 1; + int w_item; + if (i == 10) { + w_item = (w1 >> 30) | ((w2 << 2) & 0x4); + } else if (i == 21) { + w_item = (w2 >> 31) | ((w3 << 1) & 0x6); + } else if (i < 10) { + w_item = ((w1 >> (i * 3)) & 0x7); + } else if (i < 21) { + w_item = ((w2 >> (i * 3 - 32)) & 0x7); + } else { + w_item = ((w3 >> (i * 3 - 64)) & 0x7); + } + *out_ptr = __hmul(__int2half_rn(w_item - w_zero), w_scale); + out_ptr += out_.width; + } +} + +void reconstruct_gptq( + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* out, + int height, + int width, + int groups, + int bit) { + dim3 blockDim, gridDim; + blockDim.x = BLOCK_KN_SIZE; + blockDim.y = 1; + gridDim.y = DIVIDE(height, 32 / bit); + gridDim.x = DIVIDE(width, BLOCK_KN_SIZE); + + auto kernel = reconstruct_gptq_kernel; + if (bit == 2) { + kernel = reconstruct_gptq_kernel; + } else if (bit == 8) { + kernel = reconstruct_gptq_kernel; + } else if (bit == 3) { + kernel = reconstruct_gptq_3bit_kernel; + gridDim.y = DIVIDE(height, 32); + } + + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>( + b_q_weight, b_gptq_scales, b_gptq_qzeros, b_g_idx, height, width, groups, out); +} + +void gemm_half_q_half_cuda( + cublasHandle_t cublas_handle, + const half* a, + const uint32_t* b_q_weight, + const uint32_t* b_gptq_qzeros, + const half* b_gptq_scales, + const int* b_g_idx, + half* c, + half* temp_dq, + int size_m, + int size_n, + int size_k, + int groups, + bool use_shuffle, + int bit) { + bool use_reconstruct; + if (use_shuffle) { + use_reconstruct = ((bit == 8 && size_m > MAX_Q_GEMM_ROWS_8BIT) || (bit != 8 && size_m > MAX_Q_GEMM_ROWS)); + } else { + // The 2/3-bit kernels are somehow slower than dequant + gemm baseline, so + // we disabled them for now. + use_reconstruct = (bit < 4 || size_m > MAX_ALT_GEMM_ROWS); + } + if (use_reconstruct) { + // Reconstruct FP16 matrix, then cuBLAS + if (use_shuffle) { + reconstruct_exllama(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq, size_k, size_n, groups, bit); + } else { + reconstruct_gptq(b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, temp_dq, size_k, size_n, groups, bit); + } + + const half alpha = __float2half(1.0f); + const half beta = __float2half(0.0f); + cublasHgemm( + cublas_handle, + CUBLAS_OP_N, + CUBLAS_OP_N, + size_n, + size_m, + size_k, + &alpha, + temp_dq, + size_n, + a, + size_k, + &beta, + c, + size_n); + } else if (use_shuffle) { + // Quantized matmul + int max_chunks = size_m / BLOCK_M_SIZE_MAX; + int last_chunk = max_chunks * BLOCK_M_SIZE_MAX; + int last_chunk_size = size_m - last_chunk; + + if (max_chunks) { + gemm_half_q_half_cuda_part( + a, + b_q_weight, + b_gptq_qzeros, + b_gptq_scales, + b_g_idx, + c, + last_chunk, + size_n, + size_k, + BLOCK_M_SIZE_MAX, + groups, + bit); + } + + if (last_chunk_size) { + gemm_half_q_half_cuda_part( + a + last_chunk * size_k, + b_q_weight, + b_gptq_qzeros, + b_gptq_scales, + b_g_idx, + c + last_chunk * size_n, + last_chunk_size, + size_n, + size_k, + last_chunk_size, + groups, + bit); + } + } else { + gemm_half_q_half_alt(a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, c, size_m, size_n, size_k, bit); + } +} + +__global__ void shuffle_4bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_4bit_8(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 8; + } +} + +__global__ void shuffle_8bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_8bit_4(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 4; + } +} + +__global__ void shuffle_2bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_2bit_16(b_ptr, size_n); + b_ptr += 1 * size_n; + k += 16; + } +} + +__global__ void shuffle_3bit_kernel(uint32_t* __restrict__ b_q_weight, const int size_k, const int size_n) { + auto n = blockIdx.x * THREADS_X + threadIdx.x; + if (n >= size_n) return; + int k = 0; + uint32_t* b_ptr = b_q_weight + n; + while (k < size_k) { + shuffle_3bit_32(b_ptr, size_n); + b_ptr += 3 * size_n; + k += 32; + } +} + +__global__ void make_sequential_4bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 3; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 8; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 3; + int w2_subrow = source_row & 0x07; + int w2_row_shift = w2_subrow << 2; + int wnew2_row_shift = i << 2; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000f0000000f; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +__global__ void make_sequential_2bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 4; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 16; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 4; + int w2_subrow = source_row & 0x0f; + int w2_row_shift = w2_subrow << 1; + int wnew2_row_shift = i << 1; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x0000000300000003; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +__global__ void make_sequential_3bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + auto w_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w_column >= w_width) return; + auto w_new_row = blockIdx.y * 3; + auto q_perm_idx = blockIdx.y << 5; + uint32_t dst[3] = {0, 0, 0}; + +#pragma unroll + for (int i = 0; i < 32; i++) { + int source_row = q_perm[q_perm_idx++]; + int z_w = (source_row / 32) * 3; + int z_mod = source_row % 32; + int z_bit; + + if (z_mod != 10) { + if (z_mod != 21) { + z_bit = z_mod; + if (z_bit > 21) { + z_bit *= 3; + z_bit -= 64; + z_w += 2; + } else if (z_bit > 10) { + z_bit *= 3; + z_bit -= 32; + z_w += 1; + } else { + z_bit *= 3; + } + } else { + z_w += 1; + } + } + + uint64_t src; + if (z_mod == 10) { + src = (w[z_w * w_width + w_column] >> 30) | ((w[(z_w + 1) * w_width + w_column] << 2) & 0x4); + } else if (z_mod == 21) { + src = (w[z_w * w_width + w_column] >> 31) | ((w[(z_w + 1) * w_width + w_column] << 1) & 0x6); + } else { + src = w[z_w * w_width + w_column]; + src >>= z_bit; + src &= 0x07; + } + + z_w = 0; + if (i != 10) { + if (i != 21) { + z_bit = i; + if (z_bit > 21) { + z_bit *= 3; + z_bit -= 64; + z_w += 2; + } else if (z_bit > 10) { + z_bit *= 3; + z_bit -= 32; + z_w += 1; + } else { + z_bit *= 3; + } + } else { + z_w += 1; + } + } + if (i == 10) { + dst[z_w] |= (src & 0x03) << 30; + dst[z_w + 1] |= ((src & 0x4) >> 2); + } else if (i == 21) { + dst[z_w] |= (src & 0x01) << 31; + dst[z_w + 1] |= ((src & 0x6) >> 1); + } else { + dst[z_w] |= (src << z_bit); + } + } + w_new[w_new_row * w_width + w_column] = dst[0]; + w_new[(w_new_row + 1) * w_width + w_column] = dst[1]; + w_new[(w_new_row + 2) * w_width + w_column] = dst[2]; +} + +__global__ void make_sequential_8bit_kernel( + const uint32_t* __restrict__ w, uint32_t* __restrict__ w_new, const int* __restrict__ q_perm, const int w_width) { + const uint64_t* w2 = (uint64_t*)w; + uint64_t* w_new2 = (uint64_t*)w_new; + int w2_stride = w_width >> 1; + auto w2_column = THREADS_X * blockIdx.x + threadIdx.x; + if (w2_column >= w2_stride) return; + auto w_new2_row = blockIdx.y; + int q_perm_idx = w_new2_row << 2; + uint64_t dst = 0; + +#pragma unroll + for (int i = 0; i < 4; i++) { + int source_row = q_perm[q_perm_idx++]; + + int w2_row = source_row >> 2; + int w2_subrow = source_row & 0x03; + int w2_row_shift = w2_subrow << 3; + int wnew2_row_shift = i << 3; + + uint64_t src = w2[w2_row * w2_stride + w2_column]; + src >>= w2_row_shift; + src &= 0x000000ff000000ff; + src <<= wnew2_row_shift; + dst |= src; + } + w_new2[w_new2_row * w2_stride + w2_column] = dst; +} + +void shuffle_exllama_weight(uint32_t* q_weight, int* q_perm, int height, int width, int bit) { + if (q_perm) { + uint32_t* new_qweight = NULL; + cudaMalloc(&new_qweight, height / 32 * bit * width * sizeof(uint32_t)); + + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = height / 32 * bit; + + auto kernel = make_sequential_4bit_kernel; + if (bit == 2) { + kernel = make_sequential_2bit_kernel; + } else if (bit == 3) { + kernel = make_sequential_3bit_kernel; + gridDim.y = height / 32; + } else if (bit == 8) { + kernel = make_sequential_8bit_kernel; + } + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + kernel<<>>(q_weight, new_qweight, q_perm, width); + // Replace qweights + cudaMemcpyAsync(q_weight, new_qweight, height / 32 * bit * width * sizeof(uint32_t), cudaMemcpyDeviceToDevice); + // Cleanup + cudaDeviceSynchronize(); + cudaFree(new_qweight); + } + dim3 blockDim, gridDim; + blockDim.x = THREADS_X; + blockDim.y = 1; + gridDim.x = DIVIDE(width, THREADS_X); + gridDim.y = 1; + auto shuffle_kernel = shuffle_4bit_kernel; + if (bit == 2) { + shuffle_kernel = shuffle_2bit_kernel; + } else if (bit == 3) { + shuffle_kernel = shuffle_3bit_kernel; + } else if (bit == 8) { + shuffle_kernel = shuffle_8bit_kernel; + } + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + shuffle_kernel<<>>(q_weight, height, width); +} + +} // namespace gptq +} // namespace sglang + +torch::Tensor gptq_gemm( + torch::Tensor a, + torch::Tensor b_q_weight, + torch::Tensor b_gptq_qzeros, + torch::Tensor b_gptq_scales, + torch::Tensor b_g_idx, + bool use_shuffle, + int64_t bit) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + at::Tensor c = torch::empty({a.size(0), b_q_weight.size(1)}, options); + at::Tensor temp_dq = torch::empty({b_q_weight.size(0) * 32 / bit, b_q_weight.size(1)}, options); + + sglang::gptq::gemm_half_q_half_cuda( + at::cuda::getCurrentCUDABlasHandle(), + (const half*)a.data_ptr(), + (const uint32_t*)b_q_weight.data_ptr(), + (const uint32_t*)b_gptq_qzeros.data_ptr(), + (const half*)b_gptq_scales.data_ptr(), + b_g_idx.device().is_meta() ? NULL : (const int*)b_g_idx.data_ptr(), + (half*)c.data_ptr(), + (half*)temp_dq.data_ptr(), + c.size(0), // m + c.size(1), // n + a.size(1), // k + b_gptq_qzeros.size(0), // group number + use_shuffle, + bit); + return c; +} + +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit) { + const at::cuda::OptionalCUDAGuard device_guard(device_of(q_weight)); + sglang::gptq::shuffle_exllama_weight( + (uint32_t*)q_weight.data_ptr(), + q_perm.device().is_meta() || q_perm.numel() == 0 ? NULL : (int*)q_perm.data_ptr(), + q_weight.size(0) * 32 / bit, + q_weight.size(1), + bit); +} diff --git a/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh b/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh new file mode 100644 index 000000000..3dfc8794c --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/matrix_view.cuh @@ -0,0 +1,269 @@ +/* +Adapted from https://github.com/turboderp/exllamav2 and +https://github.com/turboderp/exllama +*/ + +#ifndef _matrix_view_cuh +#define _matrix_view_cuh + +#include +#include + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +class MatrixView_half { + public: + const half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half(const half* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ half item(int row, int column) const { + return data[row * width + column]; + } + __device__ __forceinline__ half2 item_half2(int row, int column) const { + return ((half2*)data)[(row * width + column) / 2]; + } + __device__ __forceinline__ half2 item_half2half2(int row, int column) const { + return __half2half2(data[row * width + column]); + } + __device__ __forceinline__ const half* item_ptr(int row, int column) const { + return &data[row * width + column]; + } + + __device__ __forceinline__ void item4(half (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __low2half(i01); + items[1] = __high2half(i01); + items[2] = __low2half(i23); + items[3] = __high2half(i23); + } + __device__ __forceinline__ void item4_f(float (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2float(__low2half(i01)); + items[1] = __half2float(__high2half(i01)); + items[2] = __half2float(__low2half(i23)); + items[3] = __half2float(__high2half(i23)); + } + + __device__ __forceinline__ void item4_h2(half2 (&items)[4], int row, int column) const { + half2* ptr = (half2*)item_ptr(row, column); + half2 i01 = ptr[0]; + half2 i23 = ptr[1]; + items[0] = __half2half2(__low2half(i01)); + items[1] = __half2half2(__high2half(i01)); + items[2] = __half2half2(__low2half(i23)); + items[3] = __half2half2(__high2half(i23)); + } +}; + +class MatrixView_half_rw { + public: + half* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_half_rw(half* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ half item(int row, int column) const { + return data[row * width + column]; + } + __device__ __forceinline__ half2 item_half2(int row, int column) const { + return ((half2*)data)[(row * width + column) / 2]; + } + __device__ __forceinline__ half* item_ptr(int row, int column) { + return &data[row * width + column]; + } + __device__ __forceinline__ void set(int row, int column, half value) { + data[row * width + column] = value; + } + __device__ __forceinline__ void set_half2(int row, int column, half2 value) { + ((half2*)data)[(row * width + column) / 2] = value; + } + + __device__ __forceinline__ void set4(int row, int column, half v0, half v1, half v2, half v3) { + half2 v01 = __halves2half2(v0, v1); + half2 v23 = __halves2half2(v2, v3); + half2* ptr = (half2*)item_ptr(row, column); + ptr[0] = v01; + ptr[1] = v23; + } +}; + +class MatrixView_q4_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x07) * 4; + return (data[row * width / 8 + column / 8] >> shift) & 0x0f; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x07) * 4; + uint32_t d = data[row * width / 8 + column / 8] >> shift; + items[0] = d & 0x0f; + items[1] = (d >> 4) & 0x0f; + items[2] = (d >> 8) & 0x0f; + items[3] = (d >> 12) & 0x0f; + } +}; + +class MatrixView_q4_column { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q4_column(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (row & 0x07) * 4; + return (data[row / 8 * width + column] >> shift) & 0x0f; + } + + __device__ __forceinline__ uint32_t item_uint32_t(int row, int column) { + return data[row / 8 * width + column]; + } + __device__ __forceinline__ const uint32_t* item_uint32_ptr(int row, int column) { + return &data[row / 8 * width + column]; + } +}; + +class MatrixView_q2_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q2_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x0f) * 2; + return (data[row * width / 16 + column / 16] >> shift) & 0x03; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x0f) * 2; + uint32_t d = data[row * width / 16 + column / 16] >> shift; + items[0] = d & 0x03; + items[1] = (d >> 2) & 0x03; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x0f) * 2; + uint32_t d = data[row * width / 16 + column / 16] >> shift; + items[0] = d & 0x03; + items[1] = (d >> 2) & 0x03; + items[2] = (d >> 4) & 0x03; + items[3] = (d >> 6) & 0x03; + } +}; + +class MatrixView_q3_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q3_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int z_w = column * 3 / 32; + int z_mod = column & 0x1f; + + if (z_mod == 10) { + return (data[row * width * 3 / 32 + z_w] >> 30) | ((data[row * width * 3 / 32 + (z_w + 1)] << 2) & 0x4); + } else if (z_mod == 21) { + return (data[row * width * 3 / 32 + z_w] >> 31) | ((data[row * width * 3 / 32 + (z_w + 1)] << 1) & 0x6); + } else if (z_mod < 10) { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3)) & 0x07; + } else if (z_mod < 21) { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 32)) & 0x07; + } else { + return (data[row * width * 3 / 32 + z_w] >> (z_mod * 3 - 64)) & 0x07; + } + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x1f); + uint32_t d; + if (shift <= 4) { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3); + } else if (shift == 8) { + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 24) | + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0x0f) << 8); + } else if (shift <= 16) { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 32); + } else if (shift == 20) { + d = (data[row * width / 32 * 3 + column * 3 / 32] >> 28) | + ((data[row * width / 32 * 3 + column * 3 / 32 + 1] & 0xff) << 4); + } else { + d = data[row * width / 32 * 3 + column * 3 / 32] >> (shift * 3 - 64); + } + items[0] = d & 0x07; + items[1] = (d >> 3) & 0x07; + items[2] = (d >> 6) & 0x07; + items[3] = (d >> 9) & 0x07; + } +}; + +class MatrixView_q8_row { + public: + const uint32_t* data; + const int height; + const int width; + + __device__ __forceinline__ MatrixView_q8_row(const uint32_t* data, const int height, const int width) + : data(data), height(height), width(width) {} + + __device__ __forceinline__ int item(int row, int column) const { + int shift = (column & 0x03) * 8; + return (data[row * width / 4 + column / 4] >> shift) & 0xff; + } + + __device__ __forceinline__ void item2(int (&items)[2], int row, int column) const { + int shift = (column & 0x03) * 8; + uint32_t d = data[row * width / 4 + column / 4] >> shift; + items[0] = d & 0xff; + items[1] = (d >> 8) & 0xff; + } + + __device__ __forceinline__ void item4(int (&items)[4], int row, int column) const { + int shift = (column & 0x03) * 2; + uint32_t d = data[row * width / 4 + column / 4] >> shift; + items[0] = d & 0xff; + items[1] = (d >> 8) & 0xff; + items[2] = (d >> 16) & 0xff; + items[3] = (d >> 24) & 0xff; + } +}; + +} // namespace gptq +} // namespace sglang +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh b/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh new file mode 100644 index 000000000..4a75d7b56 --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/qdq_2.cuh @@ -0,0 +1,74 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_2_cuh +#define _qdq_2_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +// Permutation: +// +// ffddbb99 77553311 eeccaa88 66442200 + +__forceinline__ __device__ void shuffle_2bit_16(uint32_t* q, int stride) { + uint32_t qa = q[0]; + uint32_t qb = 0; + +#pragma unroll + for (int i = 0; i < 8; i++) { + uint32_t qa0 = qa & 0x03; + uint32_t qa1 = (qa & 0x0c) >> 2; + qa >>= 4; + qb |= (qa1 << (i * 2 + 16)); + qb |= (qa0 << (i * 2)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_2bit_16(const uint32_t q_0, half2 (&dq)[8], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y4_ = __float2half_rn(1.0f / 4.0f); + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y4 = __halves2half2(y4_, y4_); + const half2 y16 = __halves2half2(y16_, y16_); + const half2 y64 = __halves2half2(y64_, y64_); + + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z4_ = __hsub(__int2half_rn(-256), __int2half_rn(zero)); + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero)); + const half2 z1 = __half2half2(z1_.as_half); + const half2 z4 = __half2half2(z4_); + const half2 z16 = __half2half2(z16_); + const half2 z64 = __half2half2(z64_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x00030003) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x000c000c) | c0); // half2(q[ 2], q[ 3]) * 4 + 1024 + half2_uint32 q2((qa & 0x00300030) | c0); // half2(q[ 4], q[ 5]) * 16 + 1024 + half2_uint32 q3((qa & 0x00c000c0) | c0); // half2(q[ 6], q[ 7]) * 64 + 1024 + qa >>= 8; + half2_uint32 q4((qa & 0x00030003) | c0); // half2(q[ 8], q[ 8]) + 1024 + half2_uint32 q5((qa & 0x000c000c) | c0); // half2(q[10], q[11]) * 4 + 1024 + half2_uint32 q6((qa & 0x00300030) | c0); // half2(q[12], q[13]) * 16 + 1024 + half2_uint32 q7((qa & 0x00c000c0) | c0); // half2(q[14], q[15]) * 64 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y4, z4); + dq[2] = __hfma2(q2.as_half2, y16, z16); + dq[3] = __hfma2(q3.as_half2, y64, z64); + dq[4] = __hadd2(q4.as_half2, z1); + dq[5] = __hfma2(q5.as_half2, y4, z4); + dq[6] = __hfma2(q6.as_half2, y16, z16); + dq[7] = __hfma2(q7.as_half2, y64, z64); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh b/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh new file mode 100644 index 000000000..5996f342d --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/qdq_3.cuh @@ -0,0 +1,146 @@ +#ifndef _qdq_3_cuh +#define _qdq_3_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { +// Permutation: +// +// v9997775 55333111 u8886664 44222000 (u, v lsb) +// vjjjhhhf ffdddbbb uiiiggge eecccaaa +// vtttrrrp ppnnnlll usssqqqo oommmkkk + +__forceinline__ __device__ void shuffle_3bit_32(uint32_t* q, int stride) { + uint32_t qa = q[0 * stride]; + uint32_t qb = q[1 * stride]; + uint32_t qc = q[2 * stride]; + + // qa: aa999888 77766655 54443332 22111000 + // qb: lkkkjjji iihhhggg fffeeedd dcccbbba + // qc: vvvuuutt tsssrrrq qqpppooo nnnmmmll + + uint32_t qd = qc >> 26; + qc <<= 4; + qc |= qb >> 28; + qb <<= 2; + qb |= qa >> 30; + + // qa: ..999888 77766655 54443332 22111000 + // qb: ..jjjiii hhhgggff feeedddc ccbbbaaa + // qc: ..tttsss rrrqqqpp pooonnnm mmlllkkk + // qd: vvvuuu + + uint32_t za = 0; + uint32_t zb = 0; + uint32_t zc = 0; + + for (int i = 0; i < 5; i++) { + uint32_t t0 = qa & 0x07; + uint32_t t1 = (qa & 0x38) >> 3; + qa >>= 6; + za |= (t0 << (i * 3)); + za |= (t1 << (i * 3 + 16)); + } + for (int i = 0; i < 5; i++) { + uint32_t t0 = qb & 0x07; + uint32_t t1 = (qb & 0x38) >> 3; + qb >>= 6; + zb |= (t0 << (i * 3)); + zb |= (t1 << (i * 3 + 16)); + } + for (int i = 0; i < 5; i++) { + uint32_t t0 = qc & 0x07; + uint32_t t1 = (qc & 0x38) >> 3; + qc >>= 6; + zc |= (t0 << (i * 3)); + zc |= (t1 << (i * 3 + 16)); + } + + // za: 9997775 55333111 8886664 44222000 + // zb: jjjhhhf ffdddbbb iiiggge eecccaaa + // zc: tttrrrp ppnnnlll sssqqqo oommmkkk + // qd: vvvuuu + + za |= ((qd & 0x01) >> 0) << 15; + zb |= ((qd & 0x02) >> 1) << 15; + zc |= ((qd & 0x04) >> 2) << 15; + za |= ((qd & 0x08) >> 3) << 31; + zb |= ((qd & 0x10) >> 4) << 31; + zc |= ((qd & 0x20) >> 5) << 31; + + // za: v9997775 55333111 u8886664 44222000 (u, v lsb) + // zb: vjjjhhhf ffdddbbb uiiiggge eecccaaa + // zc: vtttrrrp ppnnnlll usssqqqo oommmkkk + + q[0 * stride] = za; + q[1 * stride] = zb; + q[2 * stride] = zc; +} + +__forceinline__ __device__ void dequant_3bit_32( + const uint32_t q_0, const uint32_t q_1, const uint32_t q_2, half2 (&dq)[16], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y8_ = __float2half_rn(1.0f / 8.0f); + const half y64_ = __float2half_rn(1.0f / 64.0f); + const half2 y8 = __halves2half2(y8_, y8_); + const half2 y64 = __halves2half2(y64_, y64_); + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z8_ = __hsub(__int2half_rn(-128), __int2half_rn(zero)); + const half z64_ = __hsub(__int2half_rn(-16), __int2half_rn(zero)); + const half2 z1 = __halves2half2(z1_.as_half, z1_.as_half); + const half2 z8 = __halves2half2(z8_, z8_); + const half2 z64 = __halves2half2(z64_, z64_); + + uint32_t qa = q_0; + uint32_t qb = q_1; + uint32_t qc = q_2; + + half2_uint32 q0((qa & 0x00070007) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00380038) | c0); // half2(q[ 2], q[ 3]) * 8 + 1024 + qa >>= 6; + half2_uint32 q2((qa & 0x00070007) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00380038) | c0); // half2(q[ 6], q[ 7]) * 8 + 1024 + half2_uint32 q4((qa & 0x01c001c0) | c0); // half2(q[ 8], q[ 9]) * 64 + 1024 + qa >>= 9; + qa &= 0x00010001; + half2_uint32 q5((qb & 0x00070007) | c0); // half2(q[10], q[11]) + 1024 + half2_uint32 q6((qb & 0x00380038) | c0); // half2(q[12], q[13]) * 8 + 1024 + qb >>= 6; + half2_uint32 q7((qb & 0x00070007) | c0); // half2(q[14], q[15]) + 1024 + half2_uint32 q8((qb & 0x00380038) | c0); // half2(q[16], q[17]) * 8 + 1024 + half2_uint32 q9((qb & 0x01c001c0) | c0); // half2(q[18], q[19]) * 64 + 1024 + qb >>= 8; + qb &= 0x00020002; + half2_uint32 q10((qc & 0x00070007) | c0); // half2(q[20], q[21]) + 1024 + half2_uint32 q11((qc & 0x00380038) | c0); // half2(q[22], q[23]) * 8 + 1024 + qc >>= 6; + half2_uint32 q12((qc & 0x00070007) | c0); // half2(q[24], q[25]) + 1024 + half2_uint32 q13((qc & 0x00380038) | c0); // half2(q[26], q[27]) * 8 + 1024 + half2_uint32 q14((qc & 0x01c001c0) | c0); // half2(q[28], q[29]) * 64 + 1024 + qc >>= 7; + qc &= 0x00040004; + half2_uint32 q15((qa | qb | qc) | c0); + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y8, z8); + dq[2] = __hadd2(q2.as_half2, z1); + dq[3] = __hfma2(q3.as_half2, y8, z8); + dq[4] = __hfma2(q4.as_half2, y64, z64); + dq[5] = __hadd2(q5.as_half2, z1); + dq[6] = __hfma2(q6.as_half2, y8, z8); + dq[7] = __hadd2(q7.as_half2, z1); + dq[8] = __hfma2(q8.as_half2, y8, z8); + dq[9] = __hfma2(q9.as_half2, y64, z64); + dq[10] = __hadd2(q10.as_half2, z1); + dq[11] = __hfma2(q11.as_half2, y8, z8); + dq[12] = __hadd2(q12.as_half2, z1); + dq[13] = __hfma2(q13.as_half2, y8, z8); + dq[14] = __hfma2(q14.as_half2, y64, z64); + dq[15] = __hadd2(q15.as_half2, z1); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh b/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh new file mode 100644 index 000000000..c96af4718 --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/qdq_4.cuh @@ -0,0 +1,114 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_4_cuh +#define _qdq_4_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { +// Permutation: +// +// 77775555 33331111 66664444 22220000 + +__forceinline__ __device__ void shuffle_4bit_8(uint32_t* q, int stride) { + uint32_t qa = q[0]; + uint32_t qb = 0; + +#pragma unroll + for (int i = 0; i < 4; i++) { + uint32_t qa0 = qa & 0x0f; + uint32_t qa1 = (qa & 0xf0) >> 4; + qa >>= 8; + qb |= (qa1 << (i * 4 + 16)); + qb |= (qa0 << (i * 4)); + } + q[0] = qb; +} + +__forceinline__ __device__ void dequant_4bit_8(const uint32_t q_0, half2 (&dq)[4], int stride, const uint32_t zero) { + const uint32_t c0 = 0x64006400; + const half y16_ = __float2half_rn(1.0f / 16.0f); + const half2 y16 = __halves2half2(y16_, y16_); + const half_uint16 z1_(0xe400 | zero); // half(-1024.0f - zero); + const half z16_ = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + const half2 z1 = __half2half2(z1_.as_half); + const half2 z16 = __half2half2(z16_); + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2(q[ 0], q[ 1]) + 1024 + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2(q[ 2], q[ 3]) * 16 + 1024 + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2(q[ 4], q[ 5]) + 1024 + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2(q[ 6], q[ 7]) * 16 + 1024 + + dq[0] = __hadd2(q0.as_half2, z1); + dq[1] = __hfma2(q1.as_half2, y16, z16); + dq[2] = __hadd2(q2.as_half2, z1); + dq[3] = __hfma2(q3.as_half2, y16, z16); +} + +__forceinline__ __device__ void +dequant_4bit_8_prep_zero_scale(const uint32_t zero, const half scale, half2 (&z1z16)[2], half2 (&y1y16)[2]) { + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + half2 scale2 = __half2half2(scale); + + z1z16[0] = __hmul2(scale2, __half2half2(z1.as_half)); + z1z16[1] = __hmul2(scale2, __half2half2(z16)); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __hmul2(scale2, __half2half2(y1)); + y1y16[1] = __hmul2(scale2, __half2half2(y16)); +} + +__forceinline__ __device__ void dequant_4bit_8_prep_zero(const uint32_t zero, half2 (&z1z16)[2], half2 (&y1y16)[2]) { + half_uint16 z1(0xe400 | zero); // half(-1024.0f - zero); + half z16 = __hsub(__int2half_rn(-64), __int2half_rn(zero)); + + z1z16[0] = __half2half2(z1.as_half); + z1z16[1] = __half2half2(z16); + + const half y1 = __float2half_rn(1.0f); + const half y16 = __float2half_rn(1.0f / 16.0f); + + y1y16[0] = __half2half2(y1); + y1y16[1] = __half2half2(y16); +} + +__forceinline__ __device__ void +dequant_4bit_8_gptq(const uint32_t q_0, half2 (&dq)[4], half2 (&z1z16)[2], half2 (&y1y16)[2], int stride, bool scaled) { + const uint32_t c0 = 0x64006400; + + uint32_t qa = q_0; + half2_uint32 q0((qa & 0x000f000f) | c0); // half2( q[0] + 1024, q[1] + 1024 ) + half2_uint32 q1((qa & 0x00f000f0) | c0); // half2( q[2] * 16 + 1024, q[3] * 16 + 1024 ) + qa >>= 8; + half2_uint32 q2((qa & 0x000f000f) | c0); // half2( q[4] + 1024, q[5] + 1024 ) + half2_uint32 q3((qa & 0x00f000f0) | c0); // half2( q[6] * 16 + 1024, q[7] * 16 + 1024 ) + + if (scaled) { + dq[0] = __hfma2(q0.as_half2, y1y16[0], + z1z16[0]); // half2( q[0] * s - z * s, q[1] * s - z * s) + dq[1] = __hfma2(q1.as_half2, y1y16[1], + z1z16[1]); // half2( q[2] * s - z * s, q[3] * s - z * s) + dq[2] = __hfma2(q2.as_half2, y1y16[0], z1z16[0]); + dq[3] = __hfma2(q3.as_half2, y1y16[1], z1z16[1]); + } else { + dq[0] = __hadd2(q0.as_half2, z1z16[0]); // half2( q[0] - z, q[1] - z ) + dq[1] = __hfma2(q1.as_half2, y1y16[1], + z1z16[1]); // half2( q[2] - z, q[3] - z ) + dq[2] = __hadd2(q2.as_half2, z1z16[0]); // half2( q[4] - z, q[5] - z ) + dq[3] = __hfma2(q3.as_half2, y1y16[1], + z1z16[1]); // half2( q[6] - z, q[7] - z ) + } +} +} // namespace gptq +} // namespace sglang + +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh b/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh new file mode 100644 index 000000000..c6a49d6dc --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/qdq_8.cuh @@ -0,0 +1,30 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_8_cuh +#define _qdq_8_cuh + +#include "qdq_util.cuh" + +namespace sglang { +namespace gptq { + +__forceinline__ __device__ void shuffle_8bit_4(uint32_t* q, int stride) {} + +__forceinline__ __device__ void +dequant_8bit_8(const uint32_t q_0, const uint32_t q_1, half2 (&dq)[4], int stride, const uint32_t zero) { + half dqh[8]; + for (int i = 0; i < 4; i++) + dqh[i] = dq_ns(exb(q_0, i * 8, 0xff), zero); + for (int i = 0; i < 4; i++) + dqh[i + 4] = dq_ns(exb(q_1, i * 8, 0xff), zero); + + for (int i = 0; i < 4; i++) + dq[i] = __halves2half2(dqh[i * 2], dqh[i * 2 + 1]); +} + +} // namespace gptq +} // namespace sglang + +#endif diff --git a/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh b/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh new file mode 100644 index 000000000..0977269c3 --- /dev/null +++ b/sgl-kernel/csrc/gemm/gptq/qdq_util.cuh @@ -0,0 +1,53 @@ +/* +Copied from https://github.com/turboderp/exllamav2 +*/ + +#ifndef _qdq_util_cuh +#define _qdq_util_cuh + +namespace sglang { +namespace gptq { + +union half2_uint32 { + uint32_t as_uint32; + half2 as_half2; + __device__ half2_uint32(uint32_t val) : as_uint32(val) {} + __device__ half2_uint32(half2 val) : as_half2(val) {} +}; + +union half_uint16 { + uint16_t as_uint16; + half as_half; + __device__ half_uint16(uint16_t val) : as_uint16(val) {} + __device__ half_uint16(half val) : as_half(val) {} +}; + +// Max_scale premultiplied by 1/256 + +__forceinline__ __device__ half dq_scale(const int qs, const half max_scale) { + int qs_i = qs + 1; + half qs_h = __int2half_rn(qs_i * qs_i); + qs_h = __hmul(qs_h, max_scale); + return qs_h; +} + +__forceinline__ __device__ half dq(const int q, const int qzero, const half scale) { + return __hmul(__int2half_rn(q - qzero), scale); +} + +__forceinline__ __device__ half dq_ns(const int q, const int qzero) { + // return __hsub(__int2half_rn(q), __int2half_rn(qzero)); + return __int2half_rn(q - qzero); +} + +__forceinline__ __device__ int exb(const uint32_t q, const int shift, const int mask) { + return (int)((q >> shift) & mask); +} + +__forceinline__ __device__ int exb(const uint32_t q1, const uint32_t q0, const int shift, const int mask) { + return (int)(__funnelshift_rc(q0, q1, shift) & mask); +} + +} // namespace gptq +} // namespace sglang +#endif diff --git a/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu b/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu new file mode 100644 index 000000000..f18c81865 --- /dev/null +++ b/sgl-kernel/csrc/gemm/int8_gemm_kernel.cu @@ -0,0 +1,747 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "cutlass_extensions/epilogue/epilogue_per_row_per_col_scale.h" +#include "cutlass_extensions/gemm/gemm_universal_base_compat.h" +#include "cutlass_extensions/gemm/gemm_with_epilogue_visitor.h" +#include "utils.h" + +using namespace cute; + +template < + typename ElementOutput, + typename ArchTag, + typename ThreadblockShape, + typename WarpShape, + typename InstructionShape, + int NumStages> +void cutlass_int8_scaled_mm( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + using ThreadblockSwizzle = cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<8>; + + using DefaultGemmConf = cutlass::gemm::device:: + DefaultGemmConfiguration; + using EpilogueOutputOp = typename DefaultGemmConf::EpilogueOutputOp; + + using GemmKernel_ = typename cutlass::gemm::kernel::DefaultGemm< + ElementInputA, + cutlass::layout::RowMajor, + DefaultGemmConf::kAlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + DefaultGemmConf::kAlignmentB, + ElementOutput, + cutlass::layout::RowMajor, + ElementAccumulator, + OperatorClass, + ArchTag, + ThreadblockShape, + WarpShape, + InstructionShape, + EpilogueOutputOp, + ThreadblockSwizzle, + NumStages, + true, + typename DefaultGemmConf::Operator>::GemmKernel; + + using AlphaColTileIterator = cutlass::epilogue::threadblock::PredicatedTileIterator< + cutlass::epilogue::threadblock::OutputTileOptimalThreadMap< + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Shape, + typename GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::Count, + GemmKernel_::Epilogue::OutputTileIterator::ThreadMap::kThreads, + GemmKernel_::Epilogue::OutputTileIterator::kElementsPerAccess, + cutlass::sizeof_bits::value>, + ElementCompute>; + + using EpilogueVisitor = typename cutlass::epilogue::threadblock::EpilogueVisitorPerRowPerCol< + ThreadblockShape, + GemmKernel_::kThreadCount, + AlphaColTileIterator, + typename GemmKernel_::Epilogue::OutputTileIterator, + ElementAccumulator, + ElementCompute, + EpilogueOutputOp>; + + using Epilogue = typename cutlass::epilogue::threadblock:: + EpilogueWithVisitorFromExistingEpilogue::Epilogue; + + using GemmKernel = + cutlass::gemm::kernel::GemmWithEpilogueVisitor; + + using Gemm = cutlass::gemm::device::GemmUniversalBaseCompat; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + int64_t lda = mat_a.stride(0); + int64_t ldb = mat_b.stride(1); + int64_t ldd = out.stride(0); + + ElementOutput* bias_ptr = nullptr; + int64_t ldc = 0; + if (bias) { + bias_ptr = static_cast(bias->data_ptr()); + } + + typename EpilogueOutputOp::Params linearScalingParams; + typename EpilogueVisitor::Arguments visitor_args{linearScalingParams}; + + typename Gemm::Arguments args{ + {m, n, k}, {a_ptr, lda}, {b_ptr, ldb}, {b_s_ptr, 0}, {a_s_ptr, 0}, {bias_ptr, ldc}, {o_ptr, ldd}, visitor_args}; + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm75_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + if (m <= 32) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 64>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 64) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (m <= 256) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 2>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm80_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 6>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128 && n < 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +// Dispatch shape for sm89 (L40S, L20, RTX 4090), according to: +// https://github.com/vllm-project/vllm/blob/main/csrc/quantization/cutlass_w8a8/scaled_mm_c2x_sm89_int8_dispatch.cuh +template +void sm89_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 16) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<16, 128, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 32) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 4>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 64, 128>, + cutlass::gemm::GemmShape<32, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 256) { + if (n <= 4096) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<64, 128, 128>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 8192) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (n <= 16384) { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<256, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 3>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<128, 128, 64>, + cutlass::gemm::GemmShape<64, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + cutlass_int8_scaled_mm< + ElementOutput, + ArchTag, + cutlass::gemm::GemmShape<32, 64, 128>, + cutlass::gemm::GemmShape<16, 64, 64>, + InstructionShape, + 5>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template < + typename ElementOutput, + typename TileShape, + typename ClusterShape, + typename MainloopScheduleType, + bool WithBias> +void cutlass_int8_scaled_mm_sm90( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + using ArchTag = cutlass::arch::Sm90; + + using ElementAccumulator = int32_t; + using ElementCompute = float; + using ElementInputA = int8_t; + using ElementInputB = int8_t; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentOutput = 128 / cutlass::sizeof_bits::value; + + using OperatorClass = cutlass::arch::OpClassTensorOp; + + using EpilogueScheduleType = cutlass::epilogue::TmaWarpSpecialized; + using TileSchedulerType = cutlass::gemm::PersistentScheduler; + + using XScale = cutlass::epilogue::fusion:: + Sm90ColBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<0>, Int<0>>>; + + using WScale = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementCompute, ElementCompute, Stride, Int<1>, Int<0>>>; + + using Bias = cutlass::epilogue::fusion:: + Sm90RowBroadcast<0, TileShape, ElementOutput, ElementOutput, Stride, Int<1>, Int<0>>>; + + using Accum = cutlass::epilogue::fusion::Sm90AccFetch; + + // Scale + using Compute0 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute0 = cutlass::epilogue::fusion::Sm90EVT; + + using Compute1 = cutlass::epilogue::fusion:: + Sm90Compute; + + using EVTCompute1 = cutlass::epilogue::fusion::Sm90EVT; + + // With bias + using ComputeWithBias = cutlass::epilogue::fusion:: + Sm90Compute; + using EVTComputeWithBias = cutlass::epilogue::fusion::Sm90EVT; + + using EpilogueEVT = typename cutlass::platform::conditional::type; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementCompute, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentC, + ElementOutput, + cutlass::layout::RowMajor, + AlignmentOutput, + EpilogueScheduleType, + EpilogueEVT>::CollectiveOp; + + using Stages = cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementInputA, + cutlass::layout::RowMajor, + AlignmentA, + ElementInputB, + cutlass::layout::ColumnMajor, + AlignmentB, + ElementAccumulator, + TileShape, + ClusterShape, + Stages, + MainloopScheduleType>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal< + Shape, // Indicates ProblemShape + CollectiveMainloop, + CollectiveEpilogue, + TileSchedulerType>; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + + Gemm gemm_op; + + int m = mat_a.size(0); + int k = mat_a.size(1); + int n = mat_b.size(1); + + auto a_ptr = static_cast(mat_a.data_ptr()); + auto b_ptr = static_cast(mat_b.data_ptr()); + auto o_ptr = static_cast(out.data_ptr()); + + auto a_s_ptr = static_cast(scales_a.data_ptr()); + auto b_s_ptr = static_cast(scales_b.data_ptr()); + + using StrideA = typename Gemm::GemmKernel::StrideA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using StrideD = typename Gemm::GemmKernel::StrideD; + + StrideA stride_a = cutlass::make_cute_packed_stride(StrideA{}, make_shape(m, k, 1)); + StrideB stride_b = cutlass::make_cute_packed_stride(StrideB{}, make_shape(n, k, 1)); + StrideC stride_c; + StrideD stride_d = cutlass::make_cute_packed_stride(StrideD{}, make_shape(m, n, 1)); + + typename Gemm::Arguments args = { + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {a_ptr, stride_a, b_ptr, stride_b}, + {{}, // epilogue.thread + nullptr, + stride_c, + o_ptr, + stride_d}}; + + if constexpr (WithBias) { + ElementOutput* bias_ptr = static_cast(bias->data_ptr()); + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {bias_ptr}, + {}, + }; + } else { + args.epilogue.thread = { + {a_s_ptr}, + {{b_s_ptr}, {}, {}}, + {}, + }; + } + + auto workspace = torch::empty( + gemm_op.get_workspace_size(args), torch::TensorOptions().dtype(torch::kUInt8).device(mat_a.device())); + + auto stream = at::cuda::getCurrentCUDAStream(mat_a.get_device()); + + auto can_implement = gemm_op.can_implement(args); + TORCH_CHECK( + can_implement == cutlass::Status::kSuccess, + "gemm cannot implement, error: ", + cutlassGetStatusString(can_implement)); + + auto status = gemm_op(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "gemm executioin failed, error: ", cutlassGetStatusString(status)); +} + +template +void sm90_dispatch_bias( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + if (bias) { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + cutlass_int8_scaled_mm_sm90( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +template +void sm90_dispatch_shape( + torch::Tensor& out, + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const c10::optional& bias) { + int m = mat_a.size(0); + int n = mat_b.size(1); + if (m <= 32) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_1, _8, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 64) { + if (n < 8192) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_1, _4, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _256>, + Shape<_1, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else if (m <= 128) { + if (n <= 4096) { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _64, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_64, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecialized>(out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + return sm90_dispatch_bias< + ElementOutput, + Shape<_128, _128, _128>, + Shape<_2, _1, _1>, + cutlass::gemm::KernelTmaWarpSpecializedPingpong>(out, mat_a, mat_b, scales_a, scales_b, bias); + } +} + +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias) { + TORCH_CHECK(mat_a.is_cuda(), "mat_a must be a CUDA tensor"); + TORCH_CHECK(mat_b.is_cuda(), "mat_b must be a CUDA tensor"); + TORCH_CHECK(mat_a.dim() == 2, "mat_a must be a 2D tensor"); + TORCH_CHECK(mat_b.dim() == 2, "mat_b must be a 2D tensor"); + TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor"); + TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor"); + TORCH_CHECK(mat_a.size(1) == mat_b.size(0), "mat_a and mat_b shapes cannot be multiplied"); + TORCH_CHECK(mat_a.size(1) % 16 == 0, "mat_a.size(1) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(0) % 16 == 0, "mat_b.size(0) must be multiple of 16 for memory alignment"); + TORCH_CHECK(mat_b.size(1) % 8 == 0, "mat_b.size(1) must be multiple of 8 for memory alignment"); // out.stride(0) + TORCH_CHECK(mat_a.scalar_type() == torch::kInt8, "mat_a must be Int8"); + TORCH_CHECK(mat_b.scalar_type() == torch::kInt8, "mat_b must be Int8"); + TORCH_CHECK(out_dtype == torch::kHalf || out_dtype == torch::kBFloat16, "out_dtype must be Half or BFloat16"); + + TORCH_CHECK(scales_a.numel() == mat_a.size(0), "size of scales_a is not matched"); + TORCH_CHECK(scales_b.numel() == mat_b.size(1), "size of scales_b is not matched"); + TORCH_CHECK(scales_a.is_contiguous(), "scales_a must be contiguous"); + TORCH_CHECK(scales_b.is_contiguous(), "scales_b msut be contiguous"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be Float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be Float32"); + + if (bias) { + TORCH_CHECK(bias->numel() == mat_b.size(1), "size of bias is not matched"); + TORCH_CHECK(bias->is_contiguous(), "bias must be contiguous"); + TORCH_CHECK(bias->dtype() == out_dtype, "bias dtype must match output dtype"); + } + + torch::Tensor out = torch::empty({mat_a.size(0), mat_b.size(1)}, mat_a.options().dtype(out_dtype)); + + auto sm_version = getSMVersion(); + + if (sm_version >= 75 && sm_version < 80) { + TORCH_CHECK(out_dtype == torch::kHalf, "out_dtype must be Half for SM75"); + sm75_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else if (sm_version >= 80 && sm_version < 90) { + // sm86/sm89 has a much smaller shared memory size (100K) than sm80 (160K) + if (sm_version == 86 || sm_version == 89) { + if (out_dtype == torch::kBFloat16) { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm89_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } else { + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } + } + } else if (sm_version == 90) { +#if defined CUDA_VERSION && CUDA_VERSION >= 12000 + // cutlass 3.x + if (out_dtype == torch::kBFloat16) { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm90_dispatch_shape(out, mat_a, mat_b, scales_a, scales_b, bias); + } +#else + // fallback to cutlass 2.x + if (out_dtype == torch::kBFloat16) { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } else { + sm80_dispatch_shape>( + out, mat_a, mat_b, scales_a, scales_b, bias); + } +#endif + } else { + TORCH_CHECK_NOT_IMPLEMENTED(false, "No implemented int8_scaled_mm for current compute capability."); + } + + return out; +} diff --git a/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu b/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu new file mode 100644 index 000000000..1c8de396d --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/awq_marlin_repack.cu @@ -0,0 +1,253 @@ +#include "marlin.cuh" + +namespace marlin { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +template +__global__ void awq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr, int size_k, int size_n) { + return; +} +#else + +template +__global__ void awq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, uint32_t* __restrict__ out_ptr, int size_k, int size_n) { + constexpr int pack_factor = 32 / num_bits; + + int k_tiles = size_k / tile_k_size; + int n_tiles = size_n / tile_n_size; + int block_k_tiles = div_ceil(k_tiles, gridDim.x); + + auto start_k_tile = blockIdx.x * block_k_tiles; + if (start_k_tile >= k_tiles) { + return; + } + + int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles); + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + extern __shared__ int4 sh[]; + + constexpr int tile_n_ints = tile_n_size / pack_factor; + + constexpr int stage_n_threads = tile_n_ints / 4; + constexpr int stage_k_threads = tile_k_size; + constexpr int stage_size = stage_k_threads * stage_n_threads; + + auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + cp_async_fence(); + return; + } + + int first_n = n_tile_id * tile_n_size; + int first_n_packed = first_n / pack_factor; + + int4* sh_ptr = sh + stage_size * pipe; + + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + int first_k = k_tile_id * tile_k_size; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast( + &(b_q_weight_ptr[(first_k + k_id) * (size_n / pack_factor) + first_n_packed + (n_id * 4)]))); + } + + cp_async_fence(); + }; + + auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + return; + } + + auto warp_id = threadIdx.x / 32; + auto th_id = threadIdx.x % 32; + + if (warp_id >= 4) { + return; + } + + int tc_col = th_id / 4; + int tc_row = (th_id % 4) * 2; + + constexpr int tc_offsets[4] = {0, 1, 8, 9}; + + int cur_n = warp_id * 16 + tc_col; + int cur_n_packed = cur_n / pack_factor; + int cur_n_pos = cur_n % pack_factor; + + constexpr int sh_stride = tile_n_ints; + constexpr uint32_t mask = (1 << num_bits) - 1; + + int4* sh_stage_ptr = sh + stage_size * pipe; + uint32_t* sh_stage_int_ptr = reinterpret_cast(sh_stage_ptr); + + // Undo interleaving + int cur_n_pos_unpacked; + if constexpr (num_bits == 4) { + constexpr int undo_pack[8] = {0, 4, 1, 5, 2, 6, 3, 7}; + cur_n_pos_unpacked = undo_pack[cur_n_pos]; + } else { + constexpr int undo_pack[4] = {0, 2, 1, 3}; + cur_n_pos_unpacked = undo_pack[cur_n_pos]; + } + + uint32_t vals[8]; +#pragma unroll + for (int i = 0; i < 4; i++) { + int cur_elem = tc_row + tc_offsets[i]; + + int packed_src_0 = sh_stage_int_ptr[cur_n_packed + sh_stride * cur_elem]; + int packed_src_1 = sh_stage_int_ptr[cur_n_packed + (8 / pack_factor) + sh_stride * cur_elem]; + + vals[i] = (packed_src_0 >> (cur_n_pos_unpacked * num_bits)) & mask; + vals[4 + i] = (packed_src_1 >> (cur_n_pos_unpacked * num_bits)) & mask; + } + + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor; + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size; + + // Result of: + // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h + if constexpr (num_bits == 4) { + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7}; + + uint32_t res = 0; +#pragma unroll + for (int i = 0; i < 8; i++) { + res |= vals[pack_idx[i]] << (i * 4); + } + + out_ptr[out_offset + th_id * 4 + warp_id] = res; + + } else { + constexpr int pack_idx[4] = {0, 2, 1, 3}; + + uint32_t res1 = 0; + uint32_t res2 = 0; +#pragma unroll + for (int i = 0; i < 4; i++) { + res1 |= vals[pack_idx[i]] << (i * 8); + res2 |= vals[4 + pack_idx[i]] << (i * 8); + } + + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1; + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2; + } + }; + + auto start_pipes = [&](int k_tile_id, int n_tile_id) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages - 1; pipe++) { + fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe); + } + + wait_for_stage(); + }; +#pragma unroll + for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) { + int n_tile_id = 0; + + start_pipes(k_tile_id, n_tile_id); + + while (n_tile_id < n_tiles) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages; pipe++) { + fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id, n_tile_id + pipe + repack_stages - 1); + repack_tile(pipe, k_tile_id, n_tile_id + pipe); + wait_for_stage(); + } + n_tile_id += repack_stages; + } + } +} +#endif +} // namespace marlin + +#define CALL_IF(NUM_BITS) \ + else if (num_bits == NUM_BITS) { \ + cudaFuncSetAttribute( \ + marlin::awq_marlin_repack_kernel, \ + cudaFuncAttributeMaxDynamicSharedMemorySize, \ + max_shared_mem); \ + marlin::awq_marlin_repack_kernel \ + <<>>(b_q_weight_ptr, out_ptr, size_k, size_n); \ + } + +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, int64_t size_n, int64_t num_bits) { + // Verify compatibility with marlin tile of 16x64 + TORCH_CHECK( + size_k % marlin::tile_k_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_k_size = ", + marlin::tile_k_size); + TORCH_CHECK( + size_n % marlin::tile_n_size == 0, + "size_n = ", + size_n, + " is not divisible by tile_n_size = ", + marlin::tile_n_size); + + TORCH_CHECK(num_bits == 4 || num_bits == 8, "num_bits must be 4 or 8. Got = ", num_bits); + int const pack_factor = 32 / num_bits; + + // Verify B + TORCH_CHECK(b_q_weight.size(0) == size_k, "b_q_weight.size(0) = ", b_q_weight.size(0), " is not size_k = ", size_k); + TORCH_CHECK( + (size_n / pack_factor) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", + b_q_weight.size(1), + ", size_n = ", + size_n, + ", pack_factor = ", + pack_factor); + + // Verify device and strides + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); + auto options = torch::TensorOptions().dtype(b_q_weight.dtype()).device(b_q_weight.device()); + torch::Tensor out = torch::empty({size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, options); + + // Get ptrs + uint32_t const* b_q_weight_ptr = reinterpret_cast(b_q_weight.data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + + // Get dev info + int dev = b_q_weight.get_device(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); + int blocks; + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + if (false) { + } + CALL_IF(4) + CALL_IF(8) + else { + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits); + } + + return out; +} diff --git a/sgl-kernel/csrc/gemm/marlin/dequant.h b/sgl-kernel/csrc/gemm/marlin/dequant.h new file mode 100644 index 000000000..e91b8b9f4 --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/dequant.h @@ -0,0 +1,459 @@ +/* +Fast Dequantization (Converting INT4/INT8/FP4/FP8 to FP16/BF16) + +The process of fast dequantization can be summarized as a combination +of bitwise operations and floating-point computations: + +weight =>(bit_op / bitwise operations)=> +f16_value =>(flop / floating-point computation)=> +dequantized_weight + +Since the dequantized weights typically require subtracting the zero point and +applying a scale factor, the floating-point computation step can be fused with +the zero-point subtraction and scaling operations. + +The following are the parts that need to be modified for the fused operation +of zero-point subtraction and scaling. + +## INT4 => FP16/BF16 or INT8 => FP16 + +The floating-point computation is `__hsub2` + +If has zero points: + + flop(bit_op(weight)) - flop(bit_op(zp)) + = sub(bit_op(weight), bias) - sub(bit_op(zp), bias) + = bit_op(weight) - bit_op(zp) + +so we don't need additional modification. + +If has float zero points: + + flop(bit_op(weight)) - fzp + = sub(bit_op(weight), bias) - fzp + = bit_op(weight) - (fzp + bias) + +where the `fzp + bias` can be computed at weight loading. But this +may have accuracy issue, so we should not use this in most cases. + +If has not zero points: + + scale(flop(bit_op(weight))) + = scale(sub(bit_op(weight), bias)) + = scale(bit_op(weight)) - scale(bias) + = fma(bit_op(weight), scale_factor, scale(bias)) + +where the `scale(bias)` can be cached. But this may have accuracy issue, +so we should not use this in most cases. + + +## INT8 => BF16 + +INT8 => BF16 is a special case, it use byte_perm instead of flop. +We cannot fused byte_perm with scaling. + + +## FP4/FP8 => FP16/BF16 + + scale(flop(bit_op(weight))) + = scale(mul(bit_op(weight), multiplier)) + = mul(bit_op(weight), scale_factor * multiplier) + +where `scale_factor * multiplier` can be computed at weight loading. + +*/ + +#include "marlin_dtypes.cuh" + +namespace MARLIN_NAMESPACE_NAME { + +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 +// Lookup-table based 3-input logical operation; explicitly used for +// dequantization as the compiler does not seem to automatically recognize it in +// all cases. +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +// Constructs destination register by taking bytes from 2 sources (based on +// mask) +template +__device__ inline uint32_t prmt(uint32_t a) { + uint32_t res; + asm volatile("prmt.b32 %0, %1, %2, %3;\n" : "=r"(res) : "r"(a), "n"(start_byte), "n"(mask)); + return res; +} + +template +__device__ inline void dequant(int q, scalar_t2* frag_b); + +// +// Efficiently dequantize 4bit values packed in an int32 value into a full +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below, +// with some small changes: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385 +// +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int MASK = 0x000f000f; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + q >>= 4; + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int LO = 0x000f000f; + const int HI = 0x00f000f0; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX); + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX); + // clang-format on + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point + // directly into `SUB` and `ADD`. + const int SUB = 0x64086408; + const int MUL = 0x2c002c00; + const int ADD = 0xd480d480; + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&SUB)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), *reinterpret_cast(&MUL), *reinterpret_cast(&ADD)); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + const int LO = 0x000f000f; + const int HI = 0x00f000f0; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX); + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX); + // clang-format on + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point + // directly into `SUB` and `ADD`. + const int SUB = 0x64006400; + const int MUL = 0x2c002c00; + const int ADD = 0xd400d400; + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&SUB)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), *reinterpret_cast(&MUL), *reinterpret_cast(&ADD)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + + // Guarantee that the `(a & b) | c` operations are LOP3s. + // clang-format off + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + q >>= 4; + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + // clang-format on + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t SUB = 0x43084308; + + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&SUB)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&SUB)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t SUB = 0x43004300; + + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&SUB)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&SUB)); +} + +// +// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or +// bf16 Reference: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175 +// +template <> +__device__ inline void dequant(int q, half2* frag_b) { + static constexpr uint32_t mask_for_elt_01 = 0x5250; + static constexpr uint32_t mask_for_elt_23 = 0x5351; + static constexpr uint32_t start_byte_for_fp16 = 0x64646464; + + uint32_t lo = prmt(q); + uint32_t hi = prmt(q); + + frag_b[0] = *reinterpret_cast(&lo); + frag_b[1] = *reinterpret_cast(&hi); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480; + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64006400; + frag_b[0] = __hsub2(frag_b[0], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + frag_b[1] = __hsub2(frag_b[1], *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + float fp32_intermediates[4]; + uint32_t* fp32_intermediates_casted = reinterpret_cast(fp32_intermediates); + + static constexpr uint32_t fp32_base = 0x4B000000; + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652); + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651); + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653); + + fp32_intermediates[0] -= 8388736.f; + fp32_intermediates[1] -= 8388736.f; + fp32_intermediates[2] -= 8388736.f; + fp32_intermediates[3] -= 8388736.f; + + uint32_t* bf16_result_ptr = reinterpret_cast(frag_b); + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632); + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + float fp32_intermediates[4]; + uint32_t* fp32_intermediates_casted = reinterpret_cast(fp32_intermediates); + + static constexpr uint32_t fp32_base = 0x4B000000; + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652); + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651); + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653); + + fp32_intermediates[0] -= 8388608.f; + fp32_intermediates[1] -= 8388608.f; + fp32_intermediates[2] -= 8388608.f; + fp32_intermediates[3] -= 8388608.f; + + uint32_t* bf16_result_ptr = reinterpret_cast(frag_b); + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632); + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + // Constants for FP8 (E4M3) and FP16 formats + constexpr int FP8_EXPONENT = 4, FP16_EXPONENT = 5; + constexpr int RIGHT_SHIFT = FP16_EXPONENT - FP8_EXPONENT; + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + // Constants for FP8 (E4M3) and FP16 formats + constexpr int FP8_EXPONENT = 4, FP16_EXPONENT = 5; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (FP16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1)); + const half2 bias_reg = __float2half2_rn(float(1 << BIAS_OFFSET)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + // Constants for FP8 (E4M3) and BF16 formats + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT; + + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to BF16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + // Constants for FP8 (E4M3) and BF16 formats + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (BF16_EXPONENT - 1)) - (1 << (FP8_EXPONENT - 1)); + // Add 127 (float exponent bias) to BIAS_OFFSET and shift to float exponent + // position + constexpr uint32_t BIAS = (BIAS_OFFSET + 127) << 23; + const nv_bfloat162 bias_reg = __float2bfloat162_rn(*reinterpret_cast(&BIAS)); + + // Convert to bfloat162 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, FP16_EXPONENT = 5; + constexpr int RIGHT_SHIFT = FP16_EXPONENT - FP4_EXPONENT; + constexpr int MASK = 0x70007000; + + // Extract and shift FP4 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 4; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, half2* frag_b) { + dequant(q, frag_b); + + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, FP16_EXPONENT = 5; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (FP16_EXPONENT - 1)) - (1 << (FP4_EXPONENT - 1)); + const half2 bias_reg = __float2half2_rn(float(1 << BIAS_OFFSET)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + // Constants for FP4 (E2M1) and FP16 formats + constexpr int FP4_EXPONENT = 2, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP4_EXPONENT; + constexpr int MASK = 0x70007000; + + // Extract and shift FP4 values to FP16 format + int Out1 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 4; + int Out2 = (q & 0x80008000) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +template <> +__device__ inline void dequant(int q, nv_bfloat162* frag_b) { + dequant(q, frag_b); + + // Constants for FP4 (E2M1) and BF16 formats + constexpr int FP4_EXPONENT = 2, BF16_EXPONENT = 8; + + // Construct and apply exponent bias + constexpr int BIAS_OFFSET = (1 << (BF16_EXPONENT - 1)) - (1 << (FP4_EXPONENT - 1)); + // Add 127 (float exponent bias) to BIAS_OFFSET and shift to float exponent + // position + constexpr uint32_t BIAS = (BIAS_OFFSET + 127) << 23; + const nv_bfloat162 bias_reg = __float2bfloat162_rn(*reinterpret_cast(&BIAS)); + + // Convert to half2 and apply bias + frag_b[1] = __hmul2(frag_b[1], bias_reg); + frag_b[0] = __hmul2(frag_b[0], bias_reg); +} + +template +__device__ inline void dequant_fp8_scales(int q, scalar_t2* frag_b); + +template <> +__device__ inline void dequant_fp8_scales(int q, half2* frag_b) { + int Out1 = (q & 0xFF00FF00) >> 1; + ; + q <<= 8; + int Out2 = (q & 0xFF00FF00) >> 1; + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +}; + +template <> +__device__ inline void dequant_fp8_scales(int q, nv_bfloat162* frag_b) { + constexpr int FP8_EXPONENT = 4, BF16_EXPONENT = 8; + constexpr int RIGHT_SHIFT = BF16_EXPONENT - FP8_EXPONENT; + constexpr int MASK = 0x7F007F00; + + // Extract and shift FP8 values to BF16 format + int Out1 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + q <<= 8; + int Out2 = ((q & 0x80008000) >> 1) | ((q & MASK) >> RIGHT_SHIFT); + + // Note: reverse indexing is intentional because weights are permuted + frag_b[1] = *reinterpret_cast(&Out1); + frag_b[0] = *reinterpret_cast(&Out2); +} + +#endif + +} // namespace MARLIN_NAMESPACE_NAME diff --git a/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu b/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu new file mode 100644 index 000000000..b3437fc4b --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/gptq_marlin.cu @@ -0,0 +1,1120 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "kernel.h" +#include "marlin_template.h" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace marlin { + +__global__ void MarlinDefault(MARLIN_KERNEL_PARAMS){}; + +using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS); + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + int size_m, + int size_k, + int lda, + int block_rows) {} + +} // namespace marlin + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + TORCH_CHECK_NOT_IMPLEMENTED(false, "marlin_gemm(..) requires CUDA_ARCH >= 8.0"); + return torch::empty({1, 1}); +} + +#else + +// For a given "a" of size [M,K] performs a permutation of the K columns based +// on the given "perm" indices. +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + int size_m, + int size_k, + int lda, + int block_rows) { + auto start_row = block_rows * blockIdx.x; + int finish_row = start_row + block_rows; + if (finish_row > size_m) { + finish_row = size_m; + } + int cur_block_rows = finish_row - start_row; + + int input_row_stride = lda * sizeof(half) / 16; + int output_row_stride = size_k * sizeof(half) / 16; + + auto permute_row = [&](int row) { + int iters = size_k / default_threads; + int rest = size_k % default_threads; + + int input_offset = row * input_row_stride; + int output_offset = row * output_row_stride; + + half const* a_row_half = reinterpret_cast(a_int4_ptr + input_offset); + half* out_half = reinterpret_cast(out_int4_ptr + output_offset); + + int base_k = 0; + + for (int i = 0; i < iters; i++) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + + base_k += default_threads; + } + + if (rest) { + if (threadIdx.x < rest) { + auto cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + } + } + }; + + for (int i = 0; i < cur_block_rows; i++) { + int cur_row = start_row + i; + if (cur_row < size_m) { + permute_row(cur_row); + } + } +} + +typedef struct { + int thread_k; + int thread_n; + int num_threads; +} thread_config_t; + +thread_config_t small_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {128, 128, 256}, + {64, 128, 128}, + {128, 64, 128}}; + +thread_config_t large_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {64, 256, 256}, + {64, 128, 128}, + {128, 64, 128}}; + +typedef struct { + int blocks_per_sm; + thread_config_t tb_cfg; +} exec_config_t; + +int get_scales_cache_size( + thread_config_t const& th_config, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full) { + bool cache_scales_chunk = has_act_order && !is_k_full; + + int tb_n = th_config.thread_n; + int tb_k = th_config.thread_k; + + // Get max scale groups per thread-block + int tb_groups; + if (group_size == -1) { + tb_groups = 1; + } else if (group_size == 0) { + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size + } else { + tb_groups = div_ceil(tb_k, group_size); + } + + if (cache_scales_chunk) { + int load_groups = tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K + load_groups = max(load_groups, 32); // We load at least 32 scale groups + return load_groups * tb_n * 2; + } else { + int tb_scales = tb_groups * tb_n * 2; + + return tb_scales * pipe_stages; + } +} + +int get_kernel_cache_size( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float) { + int pack_factor = 32 / num_bits; + + // Get B size + int tb_k = th_config.thread_k; + int tb_n = th_config.thread_n; + int tb_m = thread_m_blocks * 16; + int sh_a_size = pipe_stages * (tb_m * tb_k) * 2; + int sh_b_size = pipe_stages * (tb_k * tb_n / pack_factor) * 4; + int sh_red_size = tb_m * (tb_n + 8); + int sh_s_size = + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits, group_size, has_act_order, is_k_full); + int sh_g_idx_size = has_act_order && !is_k_full ? pipe_stages * tb_k / 4 : 0; + int sh_zp_size = 0; + if (has_zp) { + if (is_zp_float) + sh_zp_size = sh_s_size; + else if (num_bits == 4) + sh_zp_size = sh_s_size / 4; + else if (num_bits == 8) + sh_zp_size = sh_s_size / 2; + } + + int total_size = max(sh_b_size, sh_red_size) + sh_a_size + sh_s_size + sh_zp_size + sh_g_idx_size; + + return total_size; +} + +bool is_valid_config( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float, + int max_shared_mem) { + // Sanity + if (th_config.thread_k == -1 || th_config.thread_n == -1 || th_config.num_threads == -1) { + return false; + } + + // Verify K/N are divisible by thread K/N + if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) { + return false; + } + + // Verify min for thread K/N + if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) { + return false; + } + + // num_threads must be at least 128 (= 4 warps) + if (th_config.num_threads < 128) { + return false; + } + + // Check that pipeline fits into cache + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + return cache_size <= max_shared_mem; +} + +#define _GET_IF( \ + W_TYPE, THREAD_M_BLOCKS, THREAD_N_BLOCKS, THREAD_K_BLOCKS, M_BLOCK_SIZE_8, GROUP_BLOCKS, NUM_THREADS, IS_ZP_FLOAT) \ + else if ( \ + q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && thread_n_blocks == THREAD_N_BLOCKS && \ + thread_k_blocks == THREAD_K_BLOCKS && m_block_size_8 == M_BLOCK_SIZE_8 && group_blocks == GROUP_BLOCKS && \ + num_threads == NUM_THREADS && is_zp_float == IS_ZP_FLOAT) { \ + kernel = Marlin< \ + scalar_t, \ + W_TYPE.id(), \ + NUM_THREADS, \ + THREAD_M_BLOCKS, \ + THREAD_N_BLOCKS, \ + THREAD_K_BLOCKS, \ + M_BLOCK_SIZE_8, \ + pipe_stages, \ + GROUP_BLOCKS, \ + IS_ZP_FLOAT>; \ + } + +// COMMON: cases for (group_blocks in [-1, 2, 4, 8] and is_zp_float == false) +// this is the most common cases +// BIGGROUP: cases for big group size (group_blocks in [-1, 8]) +// FZP: cases for float-zero-point (is_zp_float = true) +// ACT: cases for act order case (group_blocks == 0) +// FP4: cases for nvfp4(e2m1) (group_blocks == 1) +#define COMMON_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 2, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define COMMON_GET_IF(W_TYPE) \ + COMMON_GET_IF_M1(W_TYPE, 8, 8, 256) \ + COMMON_GET_IF_M1(W_TYPE, 8, 4, 128) \ + COMMON_GET_IF_M1(W_TYPE, 4, 8, 128) \ + COMMON_GET_IF_M234(W_TYPE, 16, 4, 256) \ + COMMON_GET_IF_M234(W_TYPE, 8, 4, 128) \ + COMMON_GET_IF_M234(W_TYPE, 4, 8, 128) + +#define BIGGROUP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, -1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 8, NUM_THREADS, false) + +#define BIGGROUP_GET_IF(W_TYPE) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + BIGGROUP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + BIGGROUP_GET_IF_M1(W_TYPE, 4, 8, 128) \ + BIGGROUP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + BIGGROUP_GET_IF_M234(W_TYPE, 8, 4, 128) \ + BIGGROUP_GET_IF_M234(W_TYPE, 4, 8, 128) + +#define FP4_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define FP4_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 1, NUM_THREADS, false) + +#define FP4_GET_IF(W_TYPE) \ + FP4_GET_IF_M1(W_TYPE, 8, 8, 256) \ + FP4_GET_IF_M1(W_TYPE, 8, 4, 128) \ + FP4_GET_IF_M1(W_TYPE, 4, 8, 128) \ + FP4_GET_IF_M234(W_TYPE, 16, 4, 256) \ + FP4_GET_IF_M234(W_TYPE, 8, 4, 128) \ + FP4_GET_IF_M234(W_TYPE, 4, 8, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define FZP_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 4, NUM_THREADS, true) + +#define FZP_GET_IF(W_TYPE) \ + FZP_GET_IF_M1(W_TYPE, 8, 8, 256) \ + FZP_GET_IF_M1(W_TYPE, 8, 4, 128) \ + FZP_GET_IF_M1(W_TYPE, 4, 8, 128) \ + FZP_GET_IF_M234(W_TYPE, 16, 4, 256) \ + FZP_GET_IF_M234(W_TYPE, 8, 4, 128) \ + FZP_GET_IF_M234(W_TYPE, 4, 8, 128) + +// We currently have 4-bit models only with group_blocks == 4 +#define ACT_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + _GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) \ + _GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, 0, NUM_THREADS, false) + +#define ACT_GET_IF(W_TYPE) \ + ACT_GET_IF_M1(W_TYPE, 8, 8, 256) \ + ACT_GET_IF_M1(W_TYPE, 8, 4, 128) \ + ACT_GET_IF_M1(W_TYPE, 4, 8, 128) \ + ACT_GET_IF_M234(W_TYPE, 16, 4, 256) \ + ACT_GET_IF_M234(W_TYPE, 8, 4, 128) \ + ACT_GET_IF_M234(W_TYPE, 4, 8, 128) + +template +MarlinFuncPtr get_marlin_kernel( + const sglang::ScalarType q_type, + int thread_m_blocks, + int thread_n_blocks, + int thread_k_blocks, + bool m_block_size_8, + bool has_act_order, + bool has_zp, + int group_blocks, + int num_threads, + bool is_zp_float) { + int num_bits = q_type.size_bits(); + auto kernel = MarlinDefault; + if (false) { + } + + COMMON_GET_IF(sglang::kU4) + COMMON_GET_IF(sglang::kU4B8) + COMMON_GET_IF(sglang::kU8B128) + + FP4_GET_IF(sglang::kFE2M1f) + + BIGGROUP_GET_IF(sglang::kFE4M3fn) + + ACT_GET_IF(sglang::kU4B8) + ACT_GET_IF(sglang::kU8B128) + + if (std::is_same::value) { + if (false) { + } + FZP_GET_IF(sglang::kU4) + } + + return kernel; +} + +template +exec_config_t determine_exec_config( + const sglang::ScalarType& q_type, + int prob_m, + int prob_n, + int prob_k, + int thread_m_blocks, + bool m_block_size_8, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + bool has_zp, + bool is_zp_float, + int max_shared_mem, + int sms) { + exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}}; + thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs; + int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t) + : sizeof(small_batch_thread_configs) / sizeof(thread_config_t); + + for (int i = 0; i < thread_configs_size; i++) { + thread_config_t th_config = thread_configs[i]; + + if (!is_valid_config( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem)) { + continue; + } + + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + + int group_blocks = 0; + if (!has_act_order) { + group_blocks = group_size == -1 ? -1 : group_size / 16; + } + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + th_config.thread_n / 16, + th_config.thread_k / 16, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + th_config.num_threads, + is_zp_float); + + if (kernel == MarlinDefault) continue; + + // int m_tiles = div_ceil(prob_m, thread_m_blocks * 16); + // int n_tiles = prob_n / th_config.thread_n; + // int k_tiles = prob_k / th_config.thread_k; + + return {1, th_config}; + } + + return exec_cfg; +} + +template +void marlin_mm( + const void* A, + const void* B, + void* C, + void* C_tmp, + void* s, + void* s2, + void* zp, + void* g_idx, + void* perm, + void* a_tmp, + int prob_m, + int prob_n, + int prob_k, + int lda, + void* workspace, + sglang::ScalarType const& q_type, + bool has_act_order, + bool is_k_full, + bool has_zp, + int num_groups, + int group_size, + int dev, + cudaStream_t stream, + int thread_k_init, + int thread_n_init, + int sms, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + if (has_zp) { + TORCH_CHECK( + q_type == sglang::kU4 || q_type == sglang::kU8, + "q_type must be u4 or u8 when has_zp = True. Got = ", + q_type.str()); + } else { + TORCH_CHECK( + q_type == sglang::kU4B8 || q_type == sglang::kU8B128 || q_type == sglang::kFE4M3fn || q_type == sglang::kFE2M1f, + "q_type must be uint4b8, uint8b128, float8_e4m3fn or float4_e2m1f when " + "has_zp = False. Got = ", + q_type.str()); + } + + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, ", ", prob_n, ", ", prob_k, "]"); + + int group_blocks = 0; + if (has_act_order) { + if (is_k_full) { + TORCH_CHECK(group_size != -1); + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } else { + TORCH_CHECK(group_size == 0); + group_blocks = 0; + } + } else { + if (group_size == -1) { + group_blocks = -1; + } else { + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } + } + + int num_bits = q_type.size_bits(); + const int4* A_ptr = (const int4*)A; + const int4* B_ptr = (const int4*)B; + int4* C_ptr = (int4*)C; + int4* C_tmp_ptr = (int4*)C_tmp; + const int4* s_ptr = (const int4*)s; + const uint16_t* s2_ptr = (const uint16_t*)s2; + const int4* zp_ptr = (const int4*)zp; + const int* g_idx_ptr = (const int*)g_idx; + const int* perm_ptr = (const int*)perm; + int4* a_tmp_ptr = (int4*)a_tmp; + + int* locks = (int*)workspace; + + if (has_act_order) { + // Permute A columns + int block_rows = div_ceil(prob_m, sms); + // avoid ">>>" being formatted to "> > >" + // clang-format off + permute_cols_kernel<<>>( + A_ptr, perm_ptr, a_tmp_ptr, prob_m, prob_k, lda, block_rows); + // clang-format on + A_ptr = a_tmp_ptr; + lda = prob_k; + + // If we have a full K, then we can run the non-act-order version of Marlin + // (since the weight rows are reordered by increasing group ids, and by + // having a full K, we have full original groups) + if (is_k_full) has_act_order = false; + } + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + int max_par = 16; + if (prob_n <= 4096) max_par = 16 * 8; + int max_shared_mem_new = max_shared_mem; + int rest_m = prob_m; + int max_thread_m_blocks = 4; + while (rest_m) { + int par_count = rest_m / (max_thread_m_blocks * 16); + if (par_count > max_par) par_count = max_par; + int prob_m_split = par_count > 0 ? (par_count * (max_thread_m_blocks * 16)) : rest_m; + + int thread_k = thread_k_init; + int thread_n = thread_n_init; + + int thread_m_blocks = min(div_ceil(prob_m_split, 16), max_thread_m_blocks); + int m_block_size_8 = prob_m_split <= 8; + + // Set thread config + exec_config_t exec_cfg; + thread_config_t thread_tfg; + if (thread_k != -1 && thread_n != -1) { + thread_tfg = thread_config_t{thread_k, thread_n, default_threads}; + exec_cfg = exec_config_t{1, thread_tfg}; + TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, " is not divisible by thread_n = ", thread_n); + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k); + } else { + // Auto config + exec_cfg = determine_exec_config( + q_type, + prob_m_split, + prob_n, + prob_k, + thread_m_blocks, + m_block_size_8, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem, + sms); + thread_tfg = exec_cfg.tb_cfg; + if (thread_tfg.thread_k == -1 && max_thread_m_blocks > 1) { + max_thread_m_blocks--; + continue; + } + } + + int num_threads = thread_tfg.num_threads; + thread_k = thread_tfg.thread_k; + thread_n = thread_tfg.thread_n; + int blocks = sms * exec_cfg.blocks_per_sm; + if (exec_cfg.blocks_per_sm > 1) max_shared_mem_new = max_shared_mem / exec_cfg.blocks_per_sm - 1024; + + int thread_k_blocks = thread_k / 16; + int thread_n_blocks = thread_n / 16; + + TORCH_CHECK( + is_valid_config( + thread_tfg, + thread_m_blocks, + prob_m_split, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem_new), + "Invalid thread config: thread_m_blocks = ", + thread_m_blocks, + ", thread_k = ", + thread_tfg.thread_k, + ", thread_n = ", + thread_tfg.thread_n, + ", num_threads = ", + thread_tfg.num_threads, + " for MKN = [", + prob_m, + ", ", + prob_k, + ", ", + prob_n, + "] and num_bits = ", + num_bits, + ", prob_m_split = ", + prob_m_split, + ", group_size = ", + group_size, + ", has_act_order = ", + has_act_order, + ", is_k_full = ", + is_k_full, + ", has_zp = ", + has_zp, + ", is_zp_float = ", + is_zp_float, + ", max_shared_mem_new = ", + max_shared_mem_new); + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + thread_n_blocks, + thread_k_blocks, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + num_threads, + is_zp_float); + + if (kernel == MarlinDefault) { + TORCH_CHECK( + false, + "Unsupported shapes: MNK = [", + prob_m, + ", ", + prob_n, + ", ", + prob_k, + "]", + ", has_act_order = ", + has_act_order, + ", num_groups = ", + num_groups, + ", group_size = ", + group_size, + ", prob_m_split = ", + prob_m_split, + ", thread_m_blocks = ", + thread_m_blocks, + ", thread_n_blocks = ", + thread_n_blocks, + ", thread_k_blocks = ", + thread_k_blocks, + ", num_threads = ", + num_threads, + ", num_bits = ", + num_bits); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem_new); + + bool part_use_atomic_add = use_atomic_add && div_ceil(prob_m_split, 64) * prob_n <= 2048; + + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, s2_ptr, zp_ptr, g_idx_ptr, num_groups, + prob_m_split, prob_n, prob_k, lda, locks, part_use_atomic_add, + use_fp32_reduce, max_shared_mem_new); + // clang-format on + + A_ptr += prob_m_split * (lda / 8); + C_ptr += prob_m_split * (prob_n / 8); + rest_m -= prob_m_split; + } +} + +} // namespace marlin + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + sglang::ScalarType const b_q_type = sglang::ScalarType::from_id(b_q_type_id); + int pack_factor = 32 / b_q_type.size_bits(); + + // Verify A + TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), ", size_m = ", size_m); + TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), ", size_k = ", size_k); + + // Verify B + TORCH_CHECK( + size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", + b_q_weight.size(0), + ", size_k = ", + size_k, + ", tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + b_q_weight.size(1) % MARLIN_NAMESPACE_NAME::tile_size == 0, + "b_q_weight.size(1) = ", + b_q_weight.size(1), + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + int actual_size_n = (b_q_weight.size(1) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; + TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, ", actual_size_n = ", actual_size_n); + + // Verify device and strides + TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + TORCH_CHECK(a.stride(1) == 1, "A.stride(1) is not 1"); + // We use int4 (16 bytes) to load A, so A must aligned to 16 bytes + TORCH_CHECK(a.stride(0) % 8 == 0, "A.stride(0) must divisible by 8"); + TORCH_CHECK(((uint64_t)a.data_ptr()) % 16 == 0, "A must aligned to 16 bytes"); + + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + + TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + + // thread_k: `k` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_k = -1; + // thread_n: `n` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_n = -1; + // sms: number of SMs to use for the kernel + int sms = -1; + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + torch::Tensor c; + if (c_or_none.has_value()) { + c = c_or_none.value(); + TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + TORCH_CHECK(c.size(0) == size_m, "Shape mismatch: c.size(0) = ", c.size(0), ", size_m = ", size_m); + TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), ", size_n = ", size_n); + } else { + c = torch::empty({size_m, size_n}, options); + } + if (size_m == 0) return c; + + // Alloc C tmp buffer that is going to be used for the global reduce + torch::Tensor c_tmp; + auto options_fp32 = torch::TensorOptions().dtype(at::kFloat).device(a.device()); + if (use_fp32_reduce) { + int max_m_block_size = (size_m + 16 - 1) / 16 * 16; + max_m_block_size = min(max_m_block_size, 64); + int max_c_tmp_size = sms * max_m_block_size * MARLIN_NAMESPACE_NAME::max_thread_n; + c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + } else { + c_tmp = torch::empty({0}, options_fp32); + } + + // Detect groupsize and act_order + int num_groups = -1; + int group_size = -1; + + int rank = b_scales.sizes().size(); + TORCH_CHECK(rank == 2, "b_scales rank = ", rank, " is not 2"); + TORCH_CHECK(b_scales.size(1) == size_n, "b_scales dim 1 = ", b_scales.size(1), " is not size_n = ", size_n); + num_groups = b_scales.size(0); + + torch::Tensor g_idx, perm, a_tmp; + if (g_idx_or_none.has_value() && perm_or_none.has_value()) { + g_idx = g_idx_or_none.value(); + perm = perm_or_none.value(); + + TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + + // Verify g_idx and perm + TORCH_CHECK( + (g_idx.size(-1) == 0 && perm.size(-1) == 0) || (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", + g_idx.size(-1), + " and perm.size(-1) = ", + perm.size(-1), + ", where size_k = ", + size_k); + } else { + g_idx = torch::empty({0}, options); + perm = torch::empty({0}, options); + a_tmp = torch::empty({0}, options); + } + bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; + + if (has_act_order) { + a_tmp = torch::empty({size_m, size_k}, options); + if (is_k_full) { + TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by num_groups = ", num_groups); + group_size = size_k / num_groups; + } else { + group_size = 0; + } + + } else { + a_tmp = torch::empty({0}, options); + if (num_groups > 1) { + TORCH_CHECK( + size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(0) = ", b_scales.size(0)); + group_size = size_k / num_groups; + } else { + group_size = -1; + } + } + + torch::Tensor global_scale; + if (global_scale_or_none.has_value()) { + global_scale = global_scale_or_none.value(); + TORCH_CHECK(b_q_type == sglang::kFE2M1f, "global_scale can only be used for float4_e2m1f."); + } else { + global_scale = torch::empty({0}, options); + TORCH_CHECK(!(b_q_type == sglang::kFE2M1f), "the global_scale parameter must be passed for float4_e2m1f."); + } + + torch::Tensor b_zeros; + if (b_zeros_or_none.has_value()) { + b_zeros = b_zeros_or_none.value(); + TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + } else { + b_zeros = torch::empty({0}, options); + } + bool has_zp = b_zeros.size(-1) > 0; + if (has_zp) { + TORCH_CHECK( + b_q_type == sglang::kU4 || b_q_type == sglang::kU8, + "b_q_type must be u4 or u8 when has_zp = True. Got = ", + b_q_type.str()); + } else { + TORCH_CHECK( + b_q_type == sglang::kU4B8 || b_q_type == sglang::kU8B128 || b_q_type == sglang::kFE4M3fn || + b_q_type == sglang::kFE2M1f, + "b_q_type must be uint4b8, uint8b128, float8_e4m3fn or " + "float4_e2m1f when " + "has_zp = False. Got = ", + b_q_type.str()); + } + + if (has_zp && is_zp_float) { + TORCH_CHECK( + a.scalar_type() == at::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); + } + + // Verify b_zeros + if (has_zp) { + int rank = b_zeros.sizes().size(); + TORCH_CHECK(rank == 2, "b_zeros rank = ", rank, " is not 2"); + if (is_zp_float) { + TORCH_CHECK(b_zeros.size(1) == size_n, "b_zeros dim 1 = ", b_zeros.size(1), " is not size_n = ", size_n); + TORCH_CHECK( + num_groups == b_zeros.size(0), "b_zeros dim 0 = ", b_zeros.size(0), " is not num_groups = ", num_groups); + TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + } else { + TORCH_CHECK( + b_zeros.size(0) == num_groups, "b_zeros dim 0 = ", b_zeros.size(0), " is not num_groups = ", num_groups); + TORCH_CHECK( + b_zeros.size(1) == size_n / pack_factor, + "b_zeros dim 1 = ", + b_zeros.size(1), + " is not size_n / pack_factor = ", + size_n / pack_factor); + } + } + + // Verify workspace size + TORCH_CHECK( + size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", + size_n, + ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); + + int min_workspace_size = sms; + TORCH_CHECK( + workspace.numel() >= min_workspace_size, + "workspace.numel = ", + workspace.numel(), + " is below min_workspace_size = ", + min_workspace_size); + + int dev = a.get_device(); + if (a.scalar_type() == at::ScalarType::Half) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + scales_ptr = b_scales.data_ptr(); + } else { + scales_ptr = b_scales.data_ptr(); + } + + marlin::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + size_m, + size_n, + size_k, + a.stride(0), + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else if (a.scalar_type() == at::ScalarType::BFloat16) { + void* scales_ptr; + if (b_q_type == sglang::kFE2M1f) { + scales_ptr = b_scales.data_ptr(); + } else { + scales_ptr = b_scales.data_ptr(); + } + + marlin::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + scales_ptr, + global_scale.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + size_m, + size_n, + size_k, + a.stride(0), + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else { + TORCH_CHECK(false, "gpt_marlin_gemm only supports bfloat16 and float16"); + } + + return c; +} + +#endif diff --git a/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu b/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu new file mode 100644 index 000000000..3bdf84161 --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/gptq_marlin_repack.cu @@ -0,0 +1,329 @@ +#include "marlin.cuh" + +namespace marlin { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +template +__global__ void gptq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, + uint32_t const* __restrict__ perm_ptr, + uint32_t* __restrict__ out_ptr, + int size_k, + int size_n) { + return; +} +#else +template +__global__ void gptq_marlin_repack_kernel( + uint32_t const* __restrict__ b_q_weight_ptr, + uint32_t const* __restrict__ perm_ptr, + uint32_t* __restrict__ out_ptr, + int size_k, + int size_n) { + constexpr int pack_factor = 32 / num_bits; + + int k_tiles = size_k / tile_k_size; + int n_tiles = size_n / tile_n_size; + int block_k_tiles = div_ceil(k_tiles, gridDim.x); + + auto start_k_tile = blockIdx.x * block_k_tiles; + if (start_k_tile >= k_tiles) { + return; + } + + int finish_k_tile = min(start_k_tile + block_k_tiles, k_tiles); + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + extern __shared__ int4 sh[]; + + constexpr int perm_size = tile_k_size / 4; + + int4* sh_perm_ptr = sh; + int4* sh_pipe_ptr = sh_perm_ptr; + if constexpr (has_perm) { + sh_pipe_ptr += perm_size; + } + + constexpr int tile_ints = tile_k_size / pack_factor; + + constexpr int stage_n_threads = tile_n_size / 4; + constexpr int stage_k_threads = has_perm ? tile_k_size : tile_ints; + constexpr int stage_size = stage_k_threads * stage_n_threads; + + auto load_perm_to_shared = [&](int k_tile_id) { + int first_k_int4 = (k_tile_id * tile_k_size) / 4; + + int4 const* perm_int4_ptr = reinterpret_cast(perm_ptr); + + if (threadIdx.x < perm_size) { + sh_perm_ptr[threadIdx.x] = perm_int4_ptr[first_k_int4 + threadIdx.x]; + } + __syncthreads(); + }; + + auto fetch_to_shared = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + cp_async_fence(); + return; + } + + int first_n = n_tile_id * tile_n_size; + + int4* sh_ptr = sh_pipe_ptr + stage_size * pipe; + + if constexpr (has_perm) { + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + uint32_t const* sh_perm_int_ptr = reinterpret_cast(sh_perm_ptr); + + int src_k = sh_perm_int_ptr[k_id]; + int src_k_packed = src_k / pack_factor; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast(&(b_q_weight_ptr[src_k_packed * size_n + first_n + (n_id * 4)]))); + } + + } else { + if (threadIdx.x < stage_size) { + auto k_id = threadIdx.x / stage_n_threads; + auto n_id = threadIdx.x % stage_n_threads; + + int first_k = k_tile_id * tile_k_size; + int first_k_packed = first_k / pack_factor; + + cp_async4( + &sh_ptr[k_id * stage_n_threads + n_id], + reinterpret_cast(&(b_q_weight_ptr[(first_k_packed + k_id) * size_n + first_n + (n_id * 4)]))); + } + } + + cp_async_fence(); + }; + + auto repack_tile = [&](int pipe, int k_tile_id, int n_tile_id) { + if (n_tile_id >= n_tiles) { + return; + } + + auto warp_id = threadIdx.x / 32; + auto th_id = threadIdx.x % 32; + + if (warp_id >= 4) { + return; + } + + int tc_col = th_id / 4; + int tc_row = (th_id % 4) * 2; + + constexpr int tc_offsets[4] = {0, 1, 8, 9}; + + int cur_n = warp_id * 16 + tc_col; + + constexpr int sh_stride = 64; + constexpr uint32_t mask = (1 << num_bits) - 1; + + int4* sh_stage_ptr = sh_pipe_ptr + stage_size * pipe; + uint32_t* sh_stage_int_ptr = reinterpret_cast(sh_stage_ptr); + + uint32_t* sh_perm_int_ptr = reinterpret_cast(sh_perm_ptr); + + uint32_t vals[8]; + + if constexpr (has_perm) { + for (int i = 0; i < 4; i++) { + int k_idx = tc_row + tc_offsets[i]; + + uint32_t src_k = sh_perm_int_ptr[k_idx]; + uint32_t src_k_pos = src_k % pack_factor; + + uint32_t b1_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n]; + uint32_t b1_cur_val = (b1_val >> (src_k_pos * num_bits)) & mask; + + uint32_t b2_val = sh_stage_int_ptr[k_idx * sh_stride + cur_n + 8]; + uint32_t b2_cur_val = (b2_val >> (src_k_pos * num_bits)) & mask; + + vals[i] = b1_cur_val; + vals[4 + i] = b2_cur_val; + } + + } else { + uint32_t b1_vals[tile_ints]; + uint32_t b2_vals[tile_ints]; + +#pragma unroll + for (int i = 0; i < tile_ints; i++) { + b1_vals[i] = sh_stage_int_ptr[cur_n + sh_stride * i]; + b2_vals[i] = sh_stage_int_ptr[cur_n + 8 + sh_stride * i]; + } + +#pragma unroll + for (int i = 0; i < 4; i++) { + int cur_elem = tc_row + tc_offsets[i]; + int cur_int = cur_elem / pack_factor; + int cur_pos = cur_elem % pack_factor; + + vals[i] = (b1_vals[cur_int] >> (cur_pos * num_bits)) & mask; + vals[4 + i] = (b2_vals[cur_int] >> (cur_pos * num_bits)) & mask; + } + } + + constexpr int tile_size = tile_k_size * tile_n_size / pack_factor; + int out_offset = (k_tile_id * n_tiles + n_tile_id) * tile_size; + + // Result of: + // https://github.com/NVIDIA/FasterTransformer/blob/main/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h + if constexpr (num_bits == 4) { + constexpr int pack_idx[8] = {0, 2, 4, 6, 1, 3, 5, 7}; + + uint32_t res = 0; +#pragma unroll + for (int i = 0; i < 8; i++) { + res |= vals[pack_idx[i]] << (i * 4); + } + + out_ptr[out_offset + th_id * 4 + warp_id] = res; + + } else { + constexpr int pack_idx[4] = {0, 2, 1, 3}; + + uint32_t res1 = 0; + uint32_t res2 = 0; +#pragma unroll + for (int i = 0; i < 4; i++) { + res1 |= vals[pack_idx[i]] << (i * 8); + res2 |= vals[4 + pack_idx[i]] << (i * 8); + } + + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 0] = res1; + out_ptr[out_offset + th_id * 8 + (warp_id * 2) + 1] = res2; + } + }; + + auto start_pipes = [&](int k_tile_id, int n_tile_id) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages - 1; pipe++) { + fetch_to_shared(pipe, k_tile_id, n_tile_id + pipe); + } + + wait_for_stage(); + }; +#pragma unroll + for (int k_tile_id = start_k_tile; k_tile_id < finish_k_tile; k_tile_id++) { + int n_tile_id = 0; + + if constexpr (has_perm) { + load_perm_to_shared(k_tile_id); + } + + start_pipes(k_tile_id, n_tile_id); + + while (n_tile_id < n_tiles) { +#pragma unroll + for (int pipe = 0; pipe < repack_stages; pipe++) { + fetch_to_shared((pipe + repack_stages - 1) % repack_stages, k_tile_id, n_tile_id + pipe + repack_stages - 1); + repack_tile(pipe, k_tile_id, n_tile_id + pipe); + wait_for_stage(); + } + n_tile_id += repack_stages; + } + } +} +#endif +} // namespace marlin + +#define CALL_IF(NUM_BITS, HAS_PERM) \ + else if (num_bits == NUM_BITS && has_perm == HAS_PERM) { \ + cudaFuncSetAttribute( \ + marlin::gptq_marlin_repack_kernel, \ + cudaFuncAttributeMaxDynamicSharedMemorySize, \ + max_shared_mem); \ + marlin::gptq_marlin_repack_kernel \ + <<>>( \ + b_q_weight_ptr, perm_ptr, out_ptr, size_k, size_n); \ + } + +torch::Tensor +gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, int64_t size_k, int64_t size_n, int64_t num_bits) { + // Verify compatibility with marlin tile of 16x64 + TORCH_CHECK( + size_k % marlin::tile_k_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_k_size = ", + marlin::tile_k_size); + TORCH_CHECK( + size_n % marlin::tile_n_size == 0, + "size_n = ", + size_n, + " is not divisible by tile_n_size = ", + marlin::tile_n_size); + + TORCH_CHECK(num_bits == 4 || num_bits == 8, "num_bits must be 4 or 8. Got = ", num_bits); + int const pack_factor = 32 / num_bits; + + // Verify B + TORCH_CHECK( + (size_k / pack_factor) == b_q_weight.size(0), + "Shape mismatch: b_q_weight.size(0) = ", + b_q_weight.size(0), + ", size_k = ", + size_k, + ", pack_factor = ", + pack_factor); + TORCH_CHECK(b_q_weight.size(1) == size_n, "b_q_weight.size(1) = ", b_q_weight.size(1), " is not size_n = ", size_n); + + // Verify device and strides + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + TORCH_CHECK(b_q_weight.dtype() == at::kInt, "b_q_weight type is not kInt"); + + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + TORCH_CHECK(perm.dtype() == at::kInt, "perm type is not at::kInt"); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(b_q_weight)); + auto options = torch::TensorOptions().dtype(b_q_weight.dtype()).device(b_q_weight.device()); + torch::Tensor out = torch::empty({size_k / marlin::tile_size, size_n * marlin::tile_size / pack_factor}, options); + + // Detect if there is act_order + bool has_perm = perm.size(0) != 0; + + // Get ptrs + uint32_t const* b_q_weight_ptr = reinterpret_cast(b_q_weight.data_ptr()); + uint32_t const* perm_ptr = reinterpret_cast(perm.data_ptr()); + uint32_t* out_ptr = reinterpret_cast(out.data_ptr()); + + // Get dev info + int dev = b_q_weight.get_device(); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); + int blocks; + cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev); + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + if (false) { + } + CALL_IF(4, false) + CALL_IF(4, true) + CALL_IF(8, false) + CALL_IF(8, true) + else { + TORCH_CHECK(false, "Unsupported repack config: num_bits = ", num_bits, ", has_perm = ", has_perm); + } + + return out; +} diff --git a/sgl-kernel/csrc/gemm/marlin/kernel.h b/sgl-kernel/csrc/gemm/marlin/kernel.h new file mode 100644 index 000000000..72137b244 --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/kernel.h @@ -0,0 +1,36 @@ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "marlin.cuh" +#include "marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define MARLIN_KERNEL_PARAMS \ + const int4 *__restrict__ A, const int4 *__restrict__ B, int4 *__restrict__ C, int4 *__restrict__ C_tmp, \ + const int4 *__restrict__ scales_ptr, const uint16_t *__restrict__ scale2_ptr, const int4 *__restrict__ zp_ptr, \ + const int *__restrict__ g_idx, int num_groups, int prob_m, int prob_n, int prob_k, int lda, int *locks, \ + bool use_atomic_add, bool use_fp32_reduce, int max_shared_mem + +namespace MARLIN_NAMESPACE_NAME { +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin(MARLIN_KERNEL_PARAMS); + +} diff --git a/sgl-kernel/csrc/gemm/marlin/marlin.cuh b/sgl-kernel/csrc/gemm/marlin/marlin.cuh new file mode 100644 index 000000000..99903968d --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/marlin.cuh @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +namespace MARLIN_NAMESPACE_NAME { +// Marlin params + +// 8 warps are a good choice since every SM has 4 schedulers and having more +// than 1 warp per schedule allows some more latency hiding. At the same time, +// we want relatively few warps to have many registers per warp and small tiles. +static constexpr int default_threads = 256; + +static constexpr int pipe_stages = 4; // 4 pipeline stages fit into shared memory + +static constexpr int min_thread_n = 64; +static constexpr int min_thread_k = 64; +static constexpr int max_thread_n = 256; + +static constexpr int tile_size = 16; +static constexpr int max_par = 16; + +// Repack params +static constexpr int repack_stages = 8; + +static constexpr int repack_threads = 256; + +static constexpr int tile_k_size = tile_size; +static constexpr int tile_n_size = tile_k_size * 4; + +// Helpers +template +struct Vec { + T elems[n]; + __device__ T& operator[](int i) { + return elems[i]; + } +}; + +using I4 = Vec; + +constexpr int div_ceil(int a, int b) { + return (a + b - 1) / b; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 +// No support for async +#else + +__device__ inline void cp_async4_pred(void* smem_ptr, const void* glob_ptr, bool pred = true) { + const int BYTES = 16; + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + asm volatile( + "{\n" + " .reg .pred p;\n" + " setp.ne.b32 p, %0, 0;\n" + " @p cp.async.cg.shared.global [%1], [%2], %3;\n" + "}\n" ::"r"((int)pred), + "r"(smem), + "l"(glob_ptr), + "n"(BYTES)); +} + +__device__ inline void cp_async4(void* smem_ptr, const void* glob_ptr) { + const int BYTES = 16; + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + asm volatile( + "{\n" + " cp.async.cg.shared.global [%0], [%1], %2;\n" + "}\n" ::"r"(smem), + "l"(glob_ptr), + "n"(BYTES)); +} + +__device__ inline void cp_async_fence() { + asm volatile("cp.async.commit_group;\n" ::); +} + +template +__device__ inline void cp_async_wait() { + asm volatile("cp.async.wait_group %0;\n" ::"n"(n)); +} + +#endif + +} // namespace MARLIN_NAMESPACE_NAME diff --git a/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh b/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh new file mode 100644 index 000000000..f1b821f10 --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/marlin_dtypes.cuh @@ -0,0 +1,82 @@ +#ifndef _data_types_cuh +#define _data_types_cuh +#include +#include + +#include "marlin.cuh" + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +namespace MARLIN_NAMESPACE_NAME { + +template +class ScalarType {}; + +template <> +class ScalarType { + public: + using scalar_t = half; + using scalar_t2 = half2; + + // Matrix fragments for tensor core instructions; their precise layout is + // documented here: + // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#matrix-fragments-for-mma-m16n8k16-with-floating-point-type + using FragA = Vec; + using FragB = Vec; + using FragC = Vec; + using FragS = Vec; + using FragZP = Vec; + + static __device__ float inline num2float(const half x) { + return __half2float(x); + } + + static __device__ half2 inline num2num2(const half x) { + return __half2half2(x); + } + + static __device__ half2 inline nums2num2(const half x1, const half x2) { + return __halves2half2(x1, x2); + } + + static __host__ __device__ half inline float2num(const float x) { + return __float2half(x); + } +}; + +template <> +class ScalarType { + public: + using scalar_t = nv_bfloat16; + using scalar_t2 = nv_bfloat162; + + using FragA = Vec; + using FragB = Vec; + using FragC = Vec; + using FragS = Vec; + using FragZP = Vec; + +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 800 + static __device__ float inline num2float(const nv_bfloat16 x) { + return __bfloat162float(x); + } + + static __device__ nv_bfloat162 inline num2num2(const nv_bfloat16 x) { + return __bfloat162bfloat162(x); + } + + static __device__ nv_bfloat162 inline nums2num2(const nv_bfloat16 x1, const nv_bfloat16 x2) { + return __halves2bfloat162(x1, x2); + } + + static __host__ __device__ nv_bfloat16 inline float2num(const float x) { + return __float2bfloat16(x); + } +#endif +}; + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/sgl-kernel/csrc/gemm/marlin/marlin_template.h b/sgl-kernel/csrc/gemm/marlin/marlin_template.h new file mode 100644 index 000000000..01eb33878 --- /dev/null +++ b/sgl-kernel/csrc/gemm/marlin/marlin_template.h @@ -0,0 +1,1629 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin +#endif + +#include "dequant.h" +#include "marlin.cuh" +#include "marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const bool has_act_order, // whether act_order is enabled + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const int* __restrict__ g_idx, // int32 group indices of shape k + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool use_fp32_reduce // whether to use fp32 global reduce +) {} + +} // namespace marlin + +#else + +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32 +// output/accumulation. +template +__device__ inline void +mma(const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +template +__device__ inline void mma_trans( + const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + const typename ScalarType::FragB& frag_b2, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + const uint32_t* b2 = reinterpret_cast(&frag_b2); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +// Instruction for loading a full 16x16 matrix fragment of operand A from shared +// memory, directly in tensor core layout. +template +__device__ inline void ldsm(typename ScalarType::FragA& frag_a, const void* smem_ptr) { + uint32_t* a = reinterpret_cast(&frag_a); + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + if constexpr (count == 4) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3]) + : "r"(smem)); + } else if constexpr (count == 2) { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n" : "=r"(a[0]), "=r"(a[1]) : "r"(smem)); + } else if constexpr (count == 1) { + asm volatile("ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];\n" : "=r"(a[0]) : "r"(smem)); + } else { + static_assert(count == 1 || count == 2 || count == 4, "invalid count"); + } +} + +// Multiply dequantized values by the corresponding quantization scale; used +// only for grouped quantization. +template +__device__ inline void +scale(typename ScalarType::FragB& frag_b, typename ScalarType::FragS& frag_s, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s = ScalarType::num2num2(reinterpret_cast(&frag_s)[i]); + frag_b[0] = __hmul2(frag_b[0], s); + frag_b[1] = __hmul2(frag_b[1], s); +} + +template +__device__ inline void scale_and_sub(typename ScalarType::FragB& frag_b, scalar_t s, scalar_t zp) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s2 = ScalarType::num2num2(s); + scalar_t2 zp2 = ScalarType::num2num2(zp); + frag_b[0] = __hfma2(frag_b[0], s2, __hneg2(zp2)); + frag_b[1] = __hfma2(frag_b[1], s2, __hneg2(zp2)); +} + +template +__device__ inline void +sub_zp(typename ScalarType::FragB& frag_b, typename ScalarType::scalar_t2& frag_zp, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 zp = ScalarType::num2num2(reinterpret_cast(&frag_zp)[i]); + frag_b[0] = __hsub2(frag_b[0], zp); + frag_b[1] = __hsub2(frag_b[1], zp); +} + +// Same as above, but for act_order (each K is multiplied individually) +template +__device__ inline void scale4( + typename ScalarType::FragB& frag_b, + typename ScalarType::FragS& frag_s_1, + typename ScalarType::FragS& frag_s_2, + typename ScalarType::FragS& frag_s_3, + typename ScalarType::FragS& frag_s_4, + int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s_val_1_2; + s_val_1_2.x = reinterpret_cast(&frag_s_1)[i]; + s_val_1_2.y = reinterpret_cast(&frag_s_2)[i]; + + scalar_t2 s_val_3_4; + s_val_3_4.x = reinterpret_cast(&frag_s_3)[i]; + s_val_3_4.y = reinterpret_cast(&frag_s_4)[i]; + + frag_b[0] = __hmul2(frag_b[0], s_val_1_2); + frag_b[1] = __hmul2(frag_b[1], s_val_3_4); +} + +// Given 2 floats multiply by 2 scales (halves) +template +__device__ inline void scale_float(float* c, typename ScalarType::FragS& s) { + scalar_t* s_ptr = reinterpret_cast(&s); + c[0] = __fmul_rn(c[0], ScalarType::num2float(s_ptr[0])); + c[1] = __fmul_rn(c[1], ScalarType::num2float(s_ptr[1])); +} + +// Wait until barrier reaches `count`, then lock for current threadblock. +__device__ inline void barrier_acquire(int* lock, int count) { + if (threadIdx.x == 0) { + int state = -1; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state != count); + } + __syncthreads(); +} + +// Release barrier and increment visitation count. +__device__ inline void barrier_release(int* lock, bool reset = false) { + __syncthreads(); + if (threadIdx.x == 0) { + if (reset) { + lock[0] = 0; + return; + } + int val = 1; + // Make sure that all writes since acquiring this barrier are visible + // globally, while releasing the barrier. + asm volatile("fence.acq_rel.gpu;\n"); + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n" : : "l"(lock), "r"(val)); + } +} + +// Wait until value of lock to be negative, and then add 1 +__device__ inline void wait_negative_and_add(int* lock) { + if (threadIdx.x == 0) { + int state = 0; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state >= 0); + atomicAdd(lock, 1); + } + __syncthreads(); +} + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const uint16_t* __restrict__ scale2_ptr, // fp16 global scale (for nvfp4 + // only) + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int lda, // A.stride(0), equal to prob_k is A is contiguous + int* locks, // extra global storage for barrier synchronization + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce, // whether to use fp32 global reduce + int max_shared_mem) { + // Each threadblock processes one "stripe" of the B matrix with (roughly) the + // same size, which might involve multiple column "slices" (of width 16 * + // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM + // example: + // 0 1 3 + // 0 2 3 + // 1 2 4 + // While this kind of partitioning makes things somewhat more complicated, it + // ensures good utilization of all SMs for many kinds of shape and GPU + // configurations, while requiring as few slow global cross-threadblock + // reductions as possible. + using Dtype = ScalarType; + using scalar_t2 = typename ScalarType::scalar_t2; + using FragA = typename ScalarType::FragA; + using FragB = typename ScalarType::FragB; + using FragC = typename ScalarType::FragC; + using FragS = typename ScalarType::FragS; + using FragZP = typename ScalarType::FragZP; + + static constexpr auto w_type = sglang::ScalarType::from_id(w_type_id); + constexpr bool has_zp = w_type == sglang::kU4 || w_type == sglang::kU8; + constexpr bool is_int_type = + w_type == sglang::kU4 || w_type == sglang::kU8 || w_type == sglang::kU4B8 || w_type == sglang::kU8B128; + // see comments of dequant.h for more details + constexpr bool dequant_skip_flop = !is_int_type || + has_zp && !is_zp_float && !std::is_same::value || + has_zp && !is_zp_float && !(w_type == sglang::kU8); + + scalar_t2 global_scale; + + if constexpr (w_type == sglang::kFE2M1f) { + uint16_t val = scale2_ptr[0]; + global_scale = Dtype::num2num2(*reinterpret_cast(&val)); + } + + constexpr bool has_act_order = group_blocks == 0; + constexpr int m_block_size = m_block_size_8 ? 8 : (16 * thread_m_blocks); + + constexpr int pack_factor = 32 / w_type.size_bits(); + static_assert(thread_m_blocks == 1 || !m_block_size_8); + + // For larger GEMMs we run multiple batchsize 64 versions in parallel for a + // better partitioning with less reductions + int parallel = 1; + if (prob_m > m_block_size) { + parallel = prob_m / m_block_size; + prob_m = m_block_size; + } + + int k_tiles = prob_k / 16 / thread_k_blocks; + int n_tiles = prob_n / 16 / thread_n_blocks; + int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x); + + if constexpr (!has_act_order && group_blocks != -1) { + if (group_blocks >= thread_k_blocks) { + // Ensure that the number of tiles in each stripe is a multiple of the + // groupsize; this avoids an annoying special case where a stripe starts + // in the middle of group. + iters = (group_blocks / thread_k_blocks) * div_ceil(iters, (group_blocks / thread_k_blocks)); + } + } + + int slice_row = (iters * blockIdx.x) % k_tiles; + int slice_col_par = (iters * blockIdx.x) / k_tiles; + int slice_col = slice_col_par; + int slice_iters; // number of threadblock tiles in the current slice + int slice_count = 0; // total number of active threadblocks in the current slice + int slice_idx; // index of threadblock in current slice; numbered bottom to + // top + + int par_id = 0; + int locks_off = 0; + + // We can easily implement parallel problem execution by just remapping + // indices and advancing global pointers + if (slice_col_par >= n_tiles) { + A += (slice_col_par / n_tiles) * 16 * thread_m_blocks * lda / 8; + C += (slice_col_par / n_tiles) * 16 * thread_m_blocks * prob_n / 8; + slice_col = slice_col_par % n_tiles; + par_id = slice_col_par / n_tiles; + } + if (parallel * n_tiles >= gridDim.x) { + // when parallel * n_tiles >= sms + // then there are at most $sms$ conflict tile blocks + locks_off = blockIdx.x; + } else { + locks_off = (iters * blockIdx.x) / k_tiles - 1; + } + + // Compute all information about the current slice which is required for + // synchronization. + auto init_slice = [&](bool first_init = false) { + slice_iters = iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row); + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0; + if (slice_iters == 0) return; + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row; + slice_count = 1; + slice_idx = 0; + int col_first = iters * div_ceil(k_tiles * slice_col_par, iters); + if (col_first <= k_tiles * (slice_col_par + 1)) { + int col_off = col_first - k_tiles * slice_col_par; + slice_count = div_ceil(k_tiles - col_off, iters); + if (col_off > 0) slice_count++; + int delta_first = iters * blockIdx.x - col_first; + if (delta_first < 0 || (col_off == 0 && delta_first == 0)) + slice_idx = slice_count - 1; + else { + slice_idx = slice_count - 1 - delta_first / iters; + if (col_off > 0) slice_idx--; + } + } + if (parallel * n_tiles >= gridDim.x) { + if (slice_count > 1 && slice_idx == slice_count - 1) { + locks_off++; + } + } else { + locks_off++; + } + + if (first_init && use_atomic_add && slice_count > 1 && slice_idx == 0) { + constexpr int threads_per_m = 16 * thread_n_blocks / 8; + int m_per_thread = div_ceil(thread_m_blocks * 16, threads / threads_per_m); + if (m_block_size_8) m_per_thread = div_ceil(8, threads / threads_per_m); + for (int i = 0; i < m_per_thread; i++) { + int row = threads / threads_per_m * i + threadIdx.x / threads_per_m; + if (row < prob_m) { + int col = slice_col * 16 * thread_n_blocks / 8 + threadIdx.x % threads_per_m; + C[row * prob_n / 8 + col] = {0, 0, 0, 0}; + } + } + // After write zero to output, write a negative value to lock. + // Every SM that processes the same slice would wait for + // the negative value, and then atomicAdd 1 to it. + // After all SMs are processed, the lock value would back to 0 again. + __syncthreads(); + if (threadIdx.x == 0) locks[locks_off] = 1 - slice_count; + } + + if (slice_col == n_tiles) { + A += 16 * thread_m_blocks * lda / 8; + C += 16 * thread_m_blocks * prob_n / 8; + slice_col = 0; + par_id++; + } + }; + init_slice(true); + + // A sizes/strides + + // stride of the A matrix in global memory + int a_gl_stride = lda / 8; + // stride of an A matrix tile in shared memory + constexpr int a_sh_stride = 16 * thread_k_blocks / 8; + // delta between subsequent A tiles in global memory + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8; + // between subsequent accesses within a tile + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o); + // between shared memory writes + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o); + // between shared memory tile reads + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4)); + // within a shared memory tile + constexpr int a_sh_rd_delta_i = a_sh_stride * 16; + // overall size of a tile + constexpr int a_sh_stage = a_sh_stride * m_block_size; + // number of shared write iterations for a tile + constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta); + + // B sizes/strides + int b_gl_stride = 16 * prob_n / (pack_factor * 4); + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4; + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2; + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs; + + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks; + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads); + constexpr int b_sh_wr_delta = threads * b_thread_vecs; + constexpr int b_sh_rd_delta = threads * b_thread_vecs; + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks; + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta; + + // Scale sizes/strides without act_order + int s_gl_stride = prob_n / 8; + constexpr int s_sh_stride = 16 * thread_n_blocks / 8; + constexpr int s_tb_groups = !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks + ? thread_k_blocks / group_blocks / (w_type == sglang::kFE2M1f ? 2 : 1) + : 1; + constexpr int s_sh_stage = s_tb_groups * s_sh_stride; + int s_gl_rd_delta = s_gl_stride; + + // Scale size/strides with act_order + constexpr int tb_k = 16 * thread_k_blocks; + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0; + // constexpr int act_s_row_stride = 1; + // int act_s_col_stride = act_s_row_stride * num_groups; + constexpr int act_s_max_num_groups = 32; + int act_s_col_stride = 1; + int act_s_col_warp_stride = act_s_col_stride * 8; + + int tb_n_warps = thread_n_blocks / 4; + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps; + + // Zero-points sizes/strides + int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4; + constexpr int zp_sh_stride = is_zp_float ? 16 * thread_n_blocks / 8 : ((16 * thread_n_blocks) / pack_factor) / 4; + constexpr int zp_tb_groups = s_tb_groups; + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0; + int zp_gl_rd_delta = zp_gl_stride; + + // Global A read index of current thread. + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + a_gl_rd += a_gl_rd_delta_o * slice_row; + // Shared write index of current thread. + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + // Shared read index. + int a_sh_rd = a_sh_stride * ((threadIdx.x % 32) % (16 / (m_block_size_8 ? 2 : 1))) + + (threadIdx.x % 32) / (16 / (m_block_size_8 ? 2 : 1)); + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4)); + + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs; + b_gl_rd += b_sh_stride * slice_col; + b_gl_rd += b_gl_rd_delta_o * slice_row; + auto b_sh_wr = threadIdx.x * b_thread_vecs; + auto b_sh_rd = threadIdx.x * b_thread_vecs; + + // For act_order + constexpr int k_iter_size = tb_k / b_sh_wr_iters; + int slice_k_start = tb_k * slice_row; + int slice_k_finish = slice_k_start + tb_k * slice_iters; + int slice_k_start_shared_fetch = slice_k_start; + int slice_n_offset = act_s_col_tb_stride * slice_col; + + // No act_order + int s_gl_rd; + if constexpr (!has_act_order) { + if constexpr (group_blocks == -1) { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + } else { + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) / (w_type == sglang::kFE2M1f ? 2 : 1) + + s_sh_stride * slice_col + threadIdx.x; + } + } + auto s_sh_wr = threadIdx.x; + bool s_sh_wr_pred = threadIdx.x < s_sh_stride; + + // Zero-points + int zp_gl_rd; + if constexpr (has_zp) { + if constexpr (group_blocks == -1) { + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } else { + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) + zp_sh_stride * slice_col + threadIdx.x; + } + } + auto zp_sh_wr = threadIdx.x; + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride; + + // We use a different scale layout for grouped and column-wise quantization as + // we scale a `half2` tile in column-major layout in the former and in + // row-major in the latter case. + int s_sh_rd; + if constexpr (group_blocks != -1 && w_type == sglang::kFE2M1f) { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + int warp_row = warp_id / n_warps; + + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + s_sh_rd = s_sh_rd * 2 + warp_row % 2; + + } else if constexpr (group_blocks != -1) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + else if constexpr (group_blocks == -1 && (m_block_size_8 || (has_zp && !dequant_skip_flop))) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 8; + else + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4; + + // Zero-points have the same read layout as the scales + // (without column-wise case) + constexpr int num_col_threads = 8; + constexpr int num_row_threads = 4; + constexpr int num_ints_per_thread = 8 / pack_factor; + int zp_sh_rd; + if constexpr (has_zp) { + if constexpr (is_zp_float) { + if constexpr (group_blocks != -1) { + zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + } + } else { + zp_sh_rd = num_ints_per_thread * num_col_threads * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads); + } + } + + // Precompute which thread should not read memory in which iterations; this is + // needed if there are more threads than required for a certain tilesize or + // when the batchsize is not a multiple of 16. + bool a_sh_wr_pred[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_pred[i] = a_sh_wr_delta * i + a_sh_wr < a_sh_stride * prob_m; + + // To ensure that writing and reading A tiles to/from shared memory, the + // latter in fragment format, is fully bank conflict free, we need to use a + // rather fancy XOR-based layout. The key here is that neither reads nor + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the + // same shared memory banks. Further, it seems (based on NSight-Compute) that + // each warp must also write a consecutive memory segment? + auto transform_a = [&](int i) { + int row = i / a_gl_rd_delta_o; + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ (row % 8); + }; + // Since the computation of this remapping is non-trivial and, due to our main + // loop unrolls, all shared memory accesses are static, we simply precompute + // both transformed reads and writes. + int a_sh_wr_trans[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr); + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < thread_m_blocks; j++) + a_sh_rd_trans[i][j] = transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd); + } + + // Since B-accesses have non-constant stride they have to be computed at + // runtime; we break dependencies between subsequent accesses with a tile by + // maintining multiple pointers (we have enough registers), a tiny + // optimization. + const int4* B_ptr[b_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd; + + extern __shared__ int4 sh[]; + // Shared memory storage for global fetch pipelines. + constexpr int sh_red_size = (2 * thread_n_blocks + 1) * 16 * thread_m_blocks; + constexpr int sh_b_size = stages * b_sh_stage; + int4* sh_b = sh; + int4* sh_red = sh; + int4* sh_g_idx = sh_b + (sh_red_size > sh_b_size ? sh_red_size : sh_b_size); + int4* sh_zp = sh_g_idx + (stages * g_idx_stage); + constexpr int sh_s_size = has_act_order ? (act_s_max_num_groups * s_sh_stride) : (stages * s_sh_stage); + int4* sh_s = sh_zp + (stages * zp_sh_stage); + // shared memory reused by reduction should be smaller than + // shared memory used by weight. + static_assert(thread_m_blocks * 16 * thread_n_blocks * 16 / 8 <= stages * b_sh_stage); + int4* sh_a = sh_s + sh_s_size; + // constexpr int shm_size_used = + // stages * (g_idx_stage + zp_sh_stage) + sh_s_size + + // (sh_red_size > sh_b_size ? sh_red_size : sh_b_size); + + // Register storage for double buffer of shared memory reads. + FragA frag_a[2][thread_m_blocks]; + I4 frag_b_quant[2][b_thread_vecs]; + FragC frag_c[thread_m_blocks][4][2]; + FragS frag_s[2][4]; // No act-order + FragS act_frag_s[2][4][4]; // For act-order + int frag_qzp[2][num_ints_per_thread]; // Zero-points + FragZP frag_zp; // Zero-points in fp16 + FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ + + // Zero accumulators. + auto zero_accums = [&]() { +#pragma unroll + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++) + reinterpret_cast(frag_c)[i] = 0; + }; + + int sh_first_group_id = -1; + int sh_num_groups = -1; + + auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id, int last_group_id) { + sh_first_group_id = first_group_id; + sh_num_groups = last_group_id - first_group_id + 1; + + if (sh_num_groups > act_s_max_num_groups) { + sh_num_groups = act_s_max_num_groups; + } + + if (sh_first_group_id + sh_num_groups > num_groups) { + sh_num_groups = num_groups - sh_first_group_id; + } + + int row_offset = first_group_id * s_gl_stride; + + if (is_async) { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + cp_async4_pred( + &sh_s[(i * s_sh_stride) + threadIdx.x], + &scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]); + } + } + } else { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + sh_s[(i * s_sh_stride) + threadIdx.x] = + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]; + } + } + } + }; + // Asynchronously fetch the next A, B and s tile from global to the next + // shared memory pipeline location. + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) { + if (pred) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) { + cp_async4_pred( + &sh_a_stage[a_sh_wr_trans[i]], + &A[a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off], + a_sh_wr_pred[i]); + } + int4* sh_b_stage = sh_b + b_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < b_thread_vecs; j++) { + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j); + } + + B_ptr[i] += b_gl_rd_delta_o; + } + + if constexpr (has_act_order) { + // Fetch g_idx thread-block portion + int full_pipe = a_off; + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe; + if (cur_k < prob_k && cur_k < slice_k_finish) { + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + + int4 const* cur_g_idx_stage_ptr = reinterpret_cast(&g_idx[cur_k]); + + if (threadIdx.x < g_idx_stage) { + cp_async4_pred(&sh_g_idx_stage[threadIdx.x], &cur_g_idx_stage_ptr[threadIdx.x]); + } + } + } else { + if constexpr (group_blocks != -1) { + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch scales if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } else { + for (int i = 0; i < s_tb_groups; i++) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } + } + + if constexpr (has_zp && group_blocks != -1) { + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch zero-points if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } else { + for (int i = 0; i < zp_tb_groups; i++) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } + } + } + } + // Insert a fence even when we are winding down the pipeline to ensure that + // waiting is also correct at this point. + cp_async_fence(); + }; + + auto fetch_col_zp_to_shared = [&]() { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + }; + + auto fetch_col_scale_to_shared = [&]() { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + }; + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + // Load the next sub-tile from the current location in the shared memory pipe + // into the current register buffer. + auto fetch_to_registers = [&](int k, int pipe) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) + ldsm(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]); + int4* sh_b_stage = sh_b + b_sh_stage * pipe; + +#pragma unroll + for (int i = 0; i < b_thread_vecs; i++) { + frag_b_quant[k % 2][i] = *reinterpret_cast(&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]); + } + }; + + bool is_same_group[stages]; + int same_group_id[stages]; + + auto init_same_group = [&](int pipe) { + if constexpr (!has_act_order) { + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + int group_id_1 = sh_g_idx_int_ptr[0]; + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1]; + + is_same_group[pipe] = group_id_1 == group_id_2; + same_group_id[pipe] = group_id_1; + }; + + auto fetch_scales_to_registers = [&](int k, int full_pipe) { + int pipe = full_pipe % stages; + + if constexpr (!has_act_order) { + // No act-order case + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + } + } else if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_s_stage = + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd]; + } else { + reinterpret_cast(&frag_s[1])[0] = reinterpret_cast(&frag_s[0])[0]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = k_blocks / (group_blocks * (w_type == sglang::kFE2M1f ? 2 : 1)); + + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (w_type_id != sglang::kFE2M1f.id()) { + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride]; + } else { + reinterpret_cast(&frag_s[k % 2])[0] = + reinterpret_cast(sh_s_stage)[s_sh_rd + cur_group_id * (2 * s_sh_stride)]; + } + } + } + + return; + } + + // Act-order case + + // Determine K of the "current" thread-block + int cur_k = slice_k_start + tb_k * full_pipe; + if (cur_k >= prob_k || cur_k >= slice_k_finish) { + return; + } + + // Reset (to current thread-block) since we read g_idx portion from the + // shared memory + cur_k = 0; + + // Progress to current iteration + cur_k += k_iter_size * (k % b_sh_wr_iters); + + // Determine "position" inside the thread-block (based on warp and + // thread-id) + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N + + int warp_row = warp_id / n_warps; + int warp_col = warp_id % n_warps; + + cur_k += warp_row * 16; + + auto th_id = threadIdx.x % 32; + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix + + int s_col_shift = + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) + (th_id / 4) * act_s_col_stride; + + if (is_same_group[pipe]) { + if (k % 2 == 0) { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride + s_col_shift]; + } else { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + *(reinterpret_cast(&(act_frag_s[(k - 1) % 2][0][0]))); + } + + for (int i = 1; i < 4; i++) { + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))); + } + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + constexpr int k_frag_offsets[4] = {0, 1, 8, 9}; // Tensor core offsets per thread + +#pragma unroll + for (int i = 0; i < 4; i++) { + int actual_k = cur_k + k_frag_offsets[i]; + + int group_id = sh_g_idx_int_ptr[actual_k]; + int rel_group_id = group_id - sh_first_group_id; + + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = sh_s[rel_group_id * s_sh_stride + s_col_shift]; + } + }; + + auto fetch_zp_to_registers = [&](int k, int full_pipe) { + // This code does not handle group_blocks == 0, + // which signifies act_order. + // has_zp implies AWQ, which doesn't have act_order, + static_assert(!has_zp || group_blocks != 0); + + if constexpr (has_zp && !is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp))[zp_sh_rd + i]; + } + } + + } else if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = 0; + + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + sh_zp_stage += cur_group_id * zp_sh_stride; + +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } + + else if constexpr (has_zp && is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + if (k % b_sh_wr_iters == 0) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd]; + } + } else { + auto warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + int cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride]; + } + } + } + }; + + auto dequant_data = [&](int q, scalar_t2* frag_b_ptr) { + dequant(q, frag_b_ptr); + }; + + // Execute the actual tensor core matmul of a sub-tile. + bool is_first_matmul_in_slice = true; + auto matmul = [&](int k) { + int k2 = k % 2; + const bool is_new_zp = ((group_blocks != -1) && (group_blocks < thread_k_blocks || k == 0)) || + (group_blocks == -1 && is_first_matmul_in_slice); + if constexpr (has_zp && !is_zp_float) { + if (is_new_zp) { + if constexpr (group_blocks == -1) is_first_matmul_in_slice = false; + FragB frag_zp_0; + FragB frag_zp_1; + int zp_quant_0, zp_quant_1; + + if constexpr (w_type.size_bits() == 4) { + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = zp_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = frag_qzp[k2][1]; + } + + dequant_data(zp_quant_0, reinterpret_cast(&frag_zp)); + dequant_data(zp_quant_1, reinterpret_cast(&frag_zp) + 2); + } + } + if constexpr (!dequant_skip_flop && has_zp && is_zp_float) { + if (is_new_zp) { + reinterpret_cast(&frag_zp)[0] = reinterpret_cast(&frag_zpf[k2])[0]; + } + } + + if constexpr (w_type == sglang::kFE2M1f) { + int s_quant_0 = reinterpret_cast(frag_s[k2])[0]; + int s_quant_1 = reinterpret_cast(frag_s[k2])[1]; + + dequant_fp8_scales(s_quant_0, reinterpret_cast(&frag_s[k2])); + dequant_fp8_scales(s_quant_1, reinterpret_cast(&frag_s[k2]) + 2); + } + +// We have the m dimension as the inner loop in order to encourage overlapping +// dequantization and matmul operations. +#pragma unroll + for (int j = 0; j < 4; j++) { + FragB frag_b0; + FragB frag_b1; + int b_quant_0, b_quant_1; + + if constexpr (w_type_id == sglang::kFE2M1f.id()) { + b_quant_1 = frag_b_quant[k2][0][j]; + b_quant_0 = b_quant_1 << 8; + } else if constexpr (w_type.size_bits() == 4) { + b_quant_0 = frag_b_quant[k2][0][j]; + b_quant_1 = b_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + int* frag_b_quant_ptr = reinterpret_cast(frag_b_quant[k2]); + b_quant_0 = frag_b_quant_ptr[j * 2 + 0]; + b_quant_1 = frag_b_quant_ptr[j * 2 + 1]; + } + + dequant_data(b_quant_0, reinterpret_cast(&frag_b0)); + dequant_data(b_quant_1, reinterpret_cast(&frag_b1)); + + if constexpr (dequant_skip_flop && has_zp && !is_zp_float) { + sub_zp(frag_b0, frag_zp[j], 0); + sub_zp(frag_b1, frag_zp[j], 1); + } + + // Apply scale to frag_b0 + if constexpr (has_act_order) { + static_assert(group_blocks != -1); + scale4( + frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0); + scale4( + frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 1); + } else if constexpr (!dequant_skip_flop && has_zp && !is_zp_float && group_blocks == -1) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2 s2 = Dtype::nums2num2( + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 0])[idx], + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 1])[idx]); + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], s2); + scale_and_sub(frag_b0, s2.x, frag_zp[j].x); + scale_and_sub(frag_b1, s2.y, frag_zp[j].y); + } else if constexpr (!dequant_skip_flop && has_zp && group_blocks != -1) { + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], *reinterpret_cast(&frag_s[k2][j])); + scale_and_sub(frag_b0, frag_s[k2][j][0].x, frag_zp[j].x); + scale_and_sub(frag_b1, frag_s[k2][j][0].y, frag_zp[j].y); + } else if constexpr (group_blocks != -1) { + scale(frag_b0, frag_s[k2][j], 0); + scale(frag_b1, frag_s[k2][j], 1); + } + +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { + if constexpr (m_block_size_8) { + mma_trans(frag_a[k2][i], frag_b0, frag_b1, frag_c[i][j][0]); + } else { + mma(frag_a[k2][i], frag_b0, frag_c[i][j][0]); + mma(frag_a[k2][i], frag_b1, frag_c[i][j][1]); + } + } + } + }; + + // Since we slice across the k dimension of a tile in order to increase the + // number of warps while keeping the n dimension of a tile reasonable, we have + // multiple warps that accumulate their partial sums of the same output + // location; which we have to reduce over in the end. We do in shared memory. + auto thread_block_reduce = [&]() { + constexpr int red_off = threads / b_sh_stride_threads / 2; + if (red_off >= 1) { + auto red_idx = threadIdx.x / b_sh_stride_threads; + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2; + constexpr int red_sh_delta = b_sh_stride_threads; + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads); + + // Parallel logarithmic shared memory reduction. We make sure to avoid any + // unnecessary read or write iterations, e.g., for two warps we write only + // once by warp 1 and read only once by warp 0. + +#pragma unroll + for (int m_block = 0; m_block < thread_m_blocks; m_block++) { +#pragma unroll + for (int i = red_off; i > 0; i /= 2) { + if (i <= red_idx && red_idx < 2 * i) { +#pragma unroll + for (int j = 0; j < 4 * 2; j += (m_block_size_8 ? 2 : 1)) { + int red_sh_wr = red_sh_delta * j + (red_sh_rd - red_sh_stride * i); + if (i < red_off) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * j + red_sh_rd]); + float* c_wr = reinterpret_cast(&sh_red[red_sh_wr]); +#pragma unroll + for (int k = 0; k < 4; k++) + reinterpret_cast(frag_c)[4 * 2 * m_block + j][k] += c_rd[k] + c_wr[k]; + } + sh_red[red_sh_wr] = reinterpret_cast(&frag_c)[4 * 2 * m_block + j]; + } + } + __syncthreads(); + } + if (red_idx == 0) { +#pragma unroll + for (int i = 0; i < 4 * 2; i += (m_block_size_8 ? 2 : 1)) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * i + red_sh_rd]); +#pragma unroll + for (int j = 0; j < 4; j++) + reinterpret_cast(frag_c)[4 * 2 * m_block + i][j] += c_rd[j]; + } + } + __syncthreads(); + } + } + }; + + // Since multiple threadblocks may process parts of the same column slice, we + // finally have to globally reduce over the results. As the striped + // partitioning minimizes the number of such reductions and our outputs are + // usually rather small, we perform this reduction serially in L2 cache. + auto global_reduce_fp16 = [&](bool first = false, bool last = false) { + // We are very careful here to reduce directly in the output buffer to + // maximize L2 cache utilization in this step. To do this, we write out + // results in FP16 (but still reduce with FP32 compute). + constexpr int active_threads = 32 * thread_n_blocks / 4; + if (threadIdx.x < active_threads) { + int c_gl_stride = prob_n / 8; + int c_gl_wr_delta_o = 8 * c_gl_stride; + int c_gl_wr_delta_i = 4 * (active_threads / 32); + int c_gl_wr; + if constexpr (m_block_size_8) { + c_gl_wr = c_gl_stride * ((threadIdx.x % 4) * 2) + 4 * (threadIdx.x / 32) + (threadIdx.x % 32) / 8; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } else { + c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) + 4 * (threadIdx.x / 32) + threadIdx.x % 4; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } + constexpr int c_sh_wr_delta = active_threads; + auto c_sh_wr = threadIdx.x; + + int row = (threadIdx.x % 32) / 4; + + if (!first) { +// Interestingly, doing direct global accesses here really seems to mess up +// the compiler and lead to slowdowns, hence we also use async-copies even +// though these fetches are not actually asynchronous. +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + if constexpr (m_block_size_8) { + cp_async4_pred( + &sh_red[c_sh_wr + c_sh_wr_delta * i], + &C[c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i], + (threadIdx.x % 4) * 2 + i < prob_m); + } else { + cp_async4_pred( + &sh_red[c_sh_wr + c_sh_wr_delta * i], + &C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)], + i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m); + } + } + cp_async_fence(); + cp_async_wait<0>(); + } + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + bool mask = (!m_block_size_8) && (i < (thread_m_blocks - 1) * 4 || 8 * (i / 2) + row < prob_m) || + (m_block_size_8) && ((threadIdx.x % 4) * 2 + i < prob_m); + if (mask) { + if (!first) { + int4 c_red = sh_red[c_sh_wr + i * c_sh_wr_delta]; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta] += + Dtype::num2float(reinterpret_cast(&c_red)[j]); + } + } + if (!last) { + int4 c; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&c)[j] = + Dtype::float2num(reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta]); + } + if constexpr (m_block_size_8) + C[c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i] = c; + else + C[c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2)] = c; + } + } + } + } + }; + + // Globally reduce over threadblocks that compute the same column block. + // We use a tmp C buffer to reduce in full fp32 precision. + auto global_reduce_fp32 = [&](bool first = false, bool last = false) { + constexpr int tb_m = thread_m_blocks * 16; + constexpr int tb_n = thread_n_blocks * 16; + + constexpr int c_size = tb_m * tb_n * sizeof(float) / 16; + + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + + constexpr int num_floats = thread_m_blocks * 4 * 2 * 4; + constexpr int th_size = num_floats * sizeof(float) / 16; + + int c_cur_offset = locks_off * c_size; + + if (!is_th_active) { + return; + } + + if (!first) { + float* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k += (m_block_size_8 ? 2 : 1)) { + sh_red[threadIdx.x] = C_tmp[c_cur_offset + active_threads * k + threadIdx.x]; + + float* sh_c_ptr = reinterpret_cast(&sh_red[threadIdx.x]); +#pragma unroll + for (int f = 0; f < 4; f++) { + frag_c_ptr[k * 4 + f] += sh_c_ptr[f]; + } + } + } + + if (!last) { + int4* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k += (m_block_size_8 ? 2 : 1)) { + C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k]; + } + } + }; + + // Write out the reduce final result in the correct layout. We only actually + // reshuffle matrix fragments in this step, the reduction above is performed + // in fragment layout. + auto write_result = [&]() { + int c_gl_stride = prob_n / 8; + constexpr int c_sh_stride = 2 * thread_n_blocks + 1; + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks)); + constexpr int c_sh_rd_delta = c_sh_stride * (threads / (2 * thread_n_blocks)); + + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + c_gl_wr += (2 * thread_n_blocks) * slice_col; + int c_sh_wr; + if constexpr (m_block_size_8) { + c_sh_wr = (8 * c_sh_stride) * ((threadIdx.x % 32) % 4 * 2) + (threadIdx.x % 32) / 4; + c_sh_wr += 64 * (threadIdx.x / 32); + } else { + c_sh_wr = (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4; + c_sh_wr += 32 * (threadIdx.x / 32); + } + + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + + int c_gl_wr_end = c_gl_stride * prob_m; + // We first reorder in shared memory to guarantee the most efficient final + // global write patterns + auto write = [&](int idx, float c0, float c1, FragS& s) { + scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1)); + + // For per-column quantization we finally apply the scale here (only for + // 4-bit) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 4 && (has_zp && dequant_skip_flop || !has_zp)) { + res = __hmul2(res, s[0]); + } + + if constexpr (w_type == sglang::kFE2M1f) { + res = __hmul2(res, global_scale); + } + + if constexpr (m_block_size_8) { + ((scalar_t*)sh_red)[idx] = res.x; + ((scalar_t*)sh_red)[idx + 8 * c_sh_stride] = res.y; + } else { + ((scalar_t2*)sh_red)[idx] = res; + } + }; + + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + if constexpr (m_block_size_8) { + int wr = c_sh_wr + 16 * j; + write(wr, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write(wr + 8, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 1]); + } else { + int wr = c_sh_wr + 8 * j; + write( + wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0], frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]); + write( + wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2], frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]); + } + } + c_sh_wr += 16 * (4 * c_sh_stride); + } + } + __syncthreads(); + +#pragma unroll + for (int i = 0; i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks)); i++) { + if (c_gl_wr < c_gl_wr_end) { + if (use_atomic_add && slice_count > 1) { + scalar_t2* C_half2 = reinterpret_cast(&C[c_gl_wr]); + scalar_t2* sh_red_half2 = reinterpret_cast(&sh_red[c_sh_rd]); +#pragma unroll + for (int a = 0; a < 4; a++) { + atomicAdd(&C_half2[a], sh_red_half2[a]); + } + } else { + C[c_gl_wr] = sh_red[c_sh_rd]; + } + c_gl_wr += c_gl_wr_delta; + c_sh_rd += c_sh_rd_delta; + } + } + __syncthreads(); + }; + + // Start global fetch and register load pipelines. + auto start_pipes = [&]() { + +#pragma unroll + for (int i = 0; i < stages - 1; i++) { + if (has_act_order && i == 0) { + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + fetch_act_order_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]); + } + + if constexpr (has_zp && !is_zp_float && group_blocks == -1) { + if (i == 0) { + fetch_col_zp_to_shared(); + if constexpr (!dequant_skip_flop) { + fetch_col_scale_to_shared(); + } + } + } + fetch_to_shared(i, i, i < slice_iters); + } + + zero_accums(); + wait_for_stage(); + init_same_group(0); + fetch_to_registers(0, 0); + fetch_scales_to_registers(0, 0); + fetch_zp_to_registers(0, 0); + a_gl_rd += a_gl_rd_delta_o * (stages - 1); + if constexpr (has_act_order) { + slice_k_start_shared_fetch += tb_k * (stages - 1); + } + }; + if (slice_iters) { + start_pipes(); + } + + // Main loop. + while (slice_iters) { + // We unroll over both the global fetch and the register load pipeline to + // ensure all shared memory accesses are static. Note that both pipelines + // have even length meaning that the next iteration will always start at + // index 0. + +#pragma unroll + for (int pipe = 0; pipe < stages;) { +#pragma unroll + for (int k = 0; k < b_sh_wr_iters; k++) { + fetch_to_registers(k + 1, pipe % stages); + fetch_scales_to_registers(k + 1, pipe); + fetch_zp_to_registers(k + 1, pipe); + if (k == b_sh_wr_iters - 2) { + fetch_to_shared((pipe + stages - 1) % stages, pipe, slice_iters >= stages); + pipe++; + wait_for_stage(); + init_same_group(pipe % stages); + } + matmul(k); + } + slice_iters--; + if (slice_iters == 0) { + break; + } + } + + a_gl_rd += a_gl_rd_delta_o * stages; + + if constexpr (has_act_order) { + slice_k_start += tb_k * stages; + + if (slice_k_start < prob_k) { + slice_k_start_shared_fetch += tb_k * stages; + int first_group_id = g_idx[slice_k_start]; + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + int last_group_id = g_idx[last_g_idx]; + if (last_group_id >= sh_first_group_id + sh_num_groups) { + fetch_act_order_scales_to_shared(false, first_group_id, last_group_id); + __syncthreads(); + } + } + } + + // Process results and, if necessary, proceed to the next column slice. + // While this pattern may not be the most readable, other ways of writing + // the loop seemed to noticeably worse performance after compilation. + if (slice_iters == 0) { + cp_async_wait<0>(); + bool last = slice_idx == slice_count - 1; + // For per-column scales, we only fetch them here in the final step before + // write-out + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + cp_async_fence(); + } + } + + thread_block_reduce(); + if constexpr (!has_act_order && group_blocks == -1 && (has_zp && dequant_skip_flop || !has_zp)) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + cp_async_wait<0>(); + __syncthreads(); + if (threadIdx.x / 32 < thread_n_blocks / 4) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd + 0]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + if constexpr (m_block_size_8) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2* frag_s_half2 = reinterpret_cast(frag_s); +#pragma unroll + for (int i = 0; i < 8; i++) { + frag_s_half2[i] = Dtype::num2num2(reinterpret_cast(&frag_s_half2[i])[idx]); + } + } + } + } + } + + // For 8-bit channelwise, we apply the scale before the global reduction + // that converts the fp32 results to fp16 (so that we avoid possible + // overflow in fp16) + if constexpr ( + !has_act_order && group_blocks == -1 && w_type.size_bits() == 8 && (has_zp && dequant_skip_flop || !has_zp)) { + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + scale_float(reinterpret_cast(&frag_c[i][j][0][0]), frag_s[j / 2][2 * (j % 2) + 0]); + scale_float( + reinterpret_cast(&frag_c[i][j][0][2]), frag_s[j / 2][2 * (j % 2) + (m_block_size_8 ? 1 : 0)]); + + if constexpr (!m_block_size_8) { + scale_float(reinterpret_cast(&frag_c[i][j][1][0]), frag_s[j / 2][2 * (j % 2) + 1]); + scale_float(reinterpret_cast(&frag_c[i][j][1][2]), frag_s[j / 2][2 * (j % 2) + 1]); + } + } + } + } + } + + if (slice_count > 1 && !use_atomic_add) { + // only globally reduce if there is more than one block in a slice + barrier_acquire(&locks[locks_off], slice_idx); + if (use_fp32_reduce) { + global_reduce_fp32(slice_idx == 0, last); + } else { + global_reduce_fp16(slice_idx == 0, last); + } + barrier_release(&locks[locks_off], last); + } + if (use_atomic_add && slice_count > 1 && slice_idx != 0) wait_negative_and_add(&locks[locks_off]); + if (last || use_atomic_add) + // only the last block in a slice actually writes the result + write_result(); + slice_row = 0; + slice_col_par++; + slice_col++; + is_first_matmul_in_slice = true; + init_slice(); + + if (slice_iters) { + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles; + if (slice_col == 0) { +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] -= b_gl_stride; + } + + // Update slice k/n for scales loading + if constexpr (has_act_order) { + slice_k_start = tb_k * slice_row; + slice_k_finish = slice_k_start + tb_k * slice_iters; + slice_k_start_shared_fetch = slice_k_start; + slice_n_offset = act_s_col_tb_stride * slice_col; + + } else { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } + + start_pipes(); + } + } + } +} + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/sgl-kernel/csrc/gemm/math.hpp b/sgl-kernel/csrc/gemm/math.hpp new file mode 100644 index 000000000..6764e1fd6 --- /dev/null +++ b/sgl-kernel/csrc/gemm/math.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +inline constexpr uint32_t next_pow_2(uint32_t const num) { + if (num <= 1) return num; + return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1)); +} + +template +static inline constexpr auto div_ceil(A a, B b) { + return (a + b - 1) / b; +} + +// Round a down to the next multiple of b. The caller is responsible for making +// sure that b is non-zero +template +inline constexpr T round_to_previous_multiple_of(T a, T b) { + return a % b == 0 ? a : (a / b) * b; +} + +// Round a up to the next multiple of b. The caller is responsible for making +// sure that b is non-zero +template +inline constexpr T round_to_next_multiple_of(T a, T b) { + return a % b == 0 ? a : ((a / b) + 1) * b; +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu b/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu new file mode 100644 index 000000000..e18f2057b --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_expert_quant.cu @@ -0,0 +1,728 @@ +#include +#include +#include +#include +#include + +#include "nvfp4_quant.cuh" +#include "utils.h" + +// Quantizes the provided PackedVec into the uint32_t output +template +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec& vec, float SFScaleVal, uint8_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // Get absolute maximum values among the local 8 values. + auto localMax = __habs2(vec.elts[0]); + +// Local maximum value. +#pragma unroll + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + localMax = __hmax2(localMax, __habs2(vec.elts[i])); + } + + // Get the absolute maximum among all 16 values (two threads). + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax); + // Get the final absolute maximum values. + float vecMax = float(__hmax(localMax.x, localMax.y)); + + // Get the SF (max value of the vector / max value of e2m1). + // maximum value of e2m1 = 6.0. + // TODO: use half as compute data type. + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f)); + // 8 bits representation of the SF. + uint8_t fp8SFVal; + // Write the SF to global memory (STG.8). + if constexpr (UE8M0_SF) { + // Extract the 8 exponent bits from float32. + // float 32bits = 1 sign bit + 8 exponent bits + 23 mantissa bits. + uint32_t tmp = reinterpret_cast(SFValue) >> 23; + fp8SFVal = tmp & 0xff; + // Convert back to fp32. + reinterpret_cast(SFValue) = tmp << 23; + } else { + // Here SFValue is always positive, so E4M3 is the same as UE4M3. + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue); + reinterpret_cast<__nv_fp8_e4m3&>(fp8SFVal) = tmp; + // Convert back to fp32. + SFValue = float(tmp); + } + // Get the output scale. + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) * + // reciprocal(SFScaleVal)) + float outputScale = + SFValue != 0 ? reciprocal_approximate_ftz(SFValue * reciprocal_approximate_ftz(SFScaleVal)) : 0.0f; + + if (SFout) { + // Write the SF to global memory (STG.8). + *SFout = fp8SFVal; + } + + // Convert the input to float. + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + fp2Vals[i] = __half22float2(vec.elts[i]); + } else { + fp2Vals[i] = __bfloat1622float2(vec.elts[i]); + } + fp2Vals[i].x *= outputScale; + fp2Vals[i].y *= outputScale; + } + + // Convert to e2m1 values. + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals); + + // Write the e2m1 values to global memory. + return e2m1Vec; +#else + return 0; +#endif +} + +__device__ __forceinline__ float silu(const float& val) { + return val / (1.0f + __expf(-val)); +} + +template +inline __device__ void silu_and_mul(PackedVec& x_vec, const PackedVec& y_vec) { + float2 x[CVT_FP4_ELTS_PER_THREAD / 2]; + float2 y[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + x[i] = __half22float2(x_vec.elts[i]); + y[i] = __half22float2(y_vec.elts[i]); + x[i].x = silu(x[i].x) * y[i].x; + x[i].y = silu(x[i].y) * y[i].y; + x_vec.elts[i] = __float22half2_rn(x[i]); + } else { + x[i] = __bfloat1622float2(x_vec.elts[i]); + y[i] = __bfloat1622float2(y_vec.elts[i]); + x[i].x = silu(x[i].x) * y[i].x; + x[i].y = silu(x[i].y) * y[i].y; + x_vec.elts[i] = __float22bfloat162_rn(x[i]); + } + } +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + uint32_t* input_offset_by_experts, + uint32_t* output_scale_offset_by_experts, + int32_t* mask, + int n_experts, + bool low_latency) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Input tensor row/col loops. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + // TODO(kaixih@nvidia): For now, we assume mask is used together with + // silu_and_mal. Maybe we want a more general behavior of mask later. In the + // silu case, the input last dim doubles. + bool use_mask = mask != nullptr; + int actualColsPerRow = use_mask ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid; globalIdx < numRows * colsPerRow; globalIdx += gridDim.x * blockDim.x) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find index within the experts using different strategies based on expert + // count + int rowIdx_in_expert = 0; + int expert_idx = 0; + + if constexpr (SMALL_NUM_EXPERTS) { + for (int i = 0; i < n_experts; i++) { + uint32_t current_offset = __ldca(&input_offset_by_experts[i]); + uint32_t next_offset = __ldca(&input_offset_by_experts[i + 1]); + if (rowIdx >= current_offset && rowIdx < next_offset) { + rowIdx_in_expert = rowIdx - current_offset; + expert_idx = i; + break; + } + } + } else { + // Load input offsets into registers first, then do the computation. + // Local array size set to 17 because of register limit. + uint32_t local_offsets[17]; + for (int chunk_start = 0; chunk_start < n_experts; chunk_start += 16) { + *reinterpret_cast(local_offsets) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start])); + *reinterpret_cast(local_offsets + 4) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 4])); + *reinterpret_cast(local_offsets + 8) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 8])); + *reinterpret_cast(local_offsets + 12) = + __ldca(reinterpret_cast(&input_offset_by_experts[chunk_start + 12])); + local_offsets[16] = __ldca(&input_offset_by_experts[chunk_start + 16]); + +// Check against the 16 loaded offsets +#pragma unroll + for (int i = 0; i < 16; i++) { + if (rowIdx >= local_offsets[i] && rowIdx < local_offsets[i + 1]) { + rowIdx_in_expert = rowIdx - local_offsets[i]; + expert_idx = chunk_start + i; + break; + } + } + } + } + + // Early exit when using masks. + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + continue; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_mask) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + // The actual output_scales dim is computed from the padded numCols. + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + output_scale_offset_by_experts[expert_idx] * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4_expert( +#else +cvt_fp16_to_fp4_expert( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + int32_t* mask, + bool use_silu_and_mul, + int n_experts) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Input tensor row/col loops. + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int stride = (gridDim.x * blockDim.x) / n_experts; + int remainder = (gridDim.x * blockDim.x) % n_experts; + int expert_idx; + int tid_in_expert; + int actual_stride; + if (remainder > 0) { + int bound = remainder * (stride + 1); + if (tid < bound) { + expert_idx = tid / (stride + 1); + tid_in_expert = tid % (stride + 1); + actual_stride = stride + 1; + } else { + expert_idx = remainder + (tid - bound) / stride; + tid_in_expert = (tid - bound) % stride; + actual_stride = stride; + } + } else { + expert_idx = tid / stride; + tid_in_expert = tid % stride; + actual_stride = stride; + } + int m = numRows / n_experts; + int padded_m = (m + (128 - 1)) / 128 * 128; + + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + // TODO(kaixih@nvidia): For now, we assume mask is used together with + // silu_and_mal. Maybe we want a more general behavior of mask later. In the + // silu case, the input last dim doubles. + bool use_mask = mask != nullptr; + int actualColsPerRow = use_silu_and_mul ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid_in_expert + expert_idx * m * colsPerRow; globalIdx < (expert_idx + 1) * m * colsPerRow; + globalIdx += actual_stride) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find index within the experts + int rowIdx_in_expert = rowIdx - expert_idx * m; + + // Early exit when using masks. + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + break; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_silu_and_mul) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + // The actual output_scales dim is computed from the padded numCols. + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + expert_idx * padded_m * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +// Kernel for LARGE_M_TOPK = true (large m_topk optimized version) +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(1024, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, + int32_t numCols, + Type const* in, + float const* SFScale, + uint32_t* out, + uint32_t* SFout, + uint32_t* input_offset_by_experts, + uint32_t* output_scale_offset_by_experts, + int32_t* mask, + int n_experts) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + extern __shared__ uint32_t shared_input_offsets[]; + + // Load input offsets into shared memory. + // If n_experts is larger than 4, use vectorized int4 to save instructions. + // If n_experts is smaller than 4, read directly. + if constexpr (SMALL_NUM_EXPERTS) { + for (int i = threadIdx.x; i < n_experts + 1; i += blockDim.x) { + shared_input_offsets[i] = input_offset_by_experts[i]; + } + } else { + for (int i = threadIdx.x * 4; i < n_experts; i += blockDim.x * 4) { + *reinterpret_cast(&shared_input_offsets[i]) = *reinterpret_cast(&input_offset_by_experts[i]); + } + if (threadIdx.x == 0) { + shared_input_offsets[n_experts] = input_offset_by_experts[n_experts]; + } + } + + __syncthreads(); + + int tid = blockIdx.x * blockDim.x + threadIdx.x; + int colsPerRow = numCols / CVT_FP4_ELTS_PER_THREAD; + bool use_mask = mask != nullptr; + int actualColsPerRow = use_mask ? colsPerRow * 2 : colsPerRow; + + // Each global thread processes one element + for (int globalIdx = tid; globalIdx < numRows * colsPerRow; globalIdx += gridDim.x * blockDim.x) { + // Calculate which row and column this global thread should process + int rowIdx = globalIdx / colsPerRow; + int colIdx = globalIdx % colsPerRow; + + // Find expert using binary search for better performance with large m_topk + int rowIdx_in_expert = 0; + int expert_idx = 0; + + // Binary search through experts using shared memory + int left = 0, right = n_experts - 1; + while (left <= right) { + int mid = (left + right) / 2; + // Get offsets: shared_input_offsets[i] corresponds to + // input_offset_by_experts[i] + uint32_t mid_offset = shared_input_offsets[mid]; + uint32_t next_offset = shared_input_offsets[mid + 1]; + + if (rowIdx >= mid_offset && rowIdx < next_offset) { + rowIdx_in_expert = rowIdx - mid_offset; + expert_idx = mid; + break; + } else if (rowIdx < mid_offset) { + right = mid - 1; + } else { + left = mid + 1; + } + } + + if (use_mask && rowIdx_in_expert >= mask[expert_idx]) { + continue; + } + + int64_t inOffset = rowIdx * actualColsPerRow + colIdx; + + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + if (use_mask) { + PackedVec in_vec_mul = reinterpret_cast(in)[inOffset + colsPerRow]; + silu_and_mul(in_vec, in_vec_mul); + } + + int64_t outOffset = rowIdx * colsPerRow + colIdx; + auto& out_pos = out[outOffset]; + + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[expert_idx]; + + int factor = CVT_FP4_SF_VEC_SIZE * 4; + int32_t numCols_padded = (numCols + factor - 1) / factor * factor; + int numCols_SFout = numCols_padded / CVT_FP4_SF_VEC_SIZE / 4; + uint32_t* SFout_in_expert = SFout + output_scale_offset_by_experts[expert_idx] * numCols_SFout; + + auto sf_out = cvt_quant_to_fp4_get_sf_out_offset( + rowIdx_in_expert, colIdx, numCols, SFout_in_expert); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } +#endif +} + +template +void quant_impl( + void* output, + void* output_scale, + void* input, + void* input_global_scale, + void* input_offset_by_experts, + void* output_scale_offset_by_experts, + void* mask, + bool use_silu_and_mul, + int m_topk, + int k, + int n_experts, + cudaStream_t stream) { + // TODO: this multiProcessorCount should be cached. + int device; + cudaGetDevice(&device); + int multiProcessorCount; + cudaDeviceGetAttribute(&multiProcessorCount, cudaDevAttrMultiProcessorCount, device); + + // Grid, Block size. + // Each thread converts 8 values. + int const workSizePerRow = k / ELTS_PER_THREAD; + int const totalWorkSize = m_topk * workSizePerRow; + dim3 block(std::min(workSizePerRow, 512)); + // Get number of blocks per SM (assume we can fully utilize the SM). + int const numBlocksPerSM = 2048 / block.x; + dim3 grid(std::min(static_cast((totalWorkSize + block.x - 1) / block.x), multiProcessorCount * numBlocksPerSM)); + while (grid.x <= multiProcessorCount && block.x > 64) { + grid.x *= 2; + block.x = (block.x + 1) / 2; + } + + // TODO(kaixih@nvidia): Should relax this to allow any grid size. + if (mask != nullptr) { + grid.x = (grid.x + n_experts - 1) / n_experts * n_experts; + cvt_fp16_to_fp4_expert<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(mask), + use_silu_and_mul, + n_experts); + return; + } + + int const blockRepeat = (totalWorkSize + block.x * grid.x - 1) / (block.x * grid.x); + if (blockRepeat > 1) { + size_t shared_mem_size = (n_experts + 1) * sizeof(uint32_t); + if (n_experts >= 4) { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts); + } else { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts); + } + } else { + if (n_experts >= 16) { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts, + /* bool low_latency */ true); + } else { + cvt_fp16_to_fp4<<>>( + m_topk, + k, + reinterpret_cast(input), + reinterpret_cast(input_global_scale), + reinterpret_cast(output), + reinterpret_cast(output_scale), + reinterpret_cast(input_offset_by_experts), + reinterpret_cast(output_scale_offset_by_experts), + reinterpret_cast(mask), + n_experts, + /* bool low_latency */ true); + } + } +} + +// Avoid redefinition warnings +#undef CHECK_CONTIGUOUS +#undef CHECK_TH_CUDA +#undef CHECK_INPUT + +/*Quantization entry for fp4 experts quantization*/ +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, "must be contiguous") +#define CHECK_INPUT(x, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); + +// constexpr auto FP8 = at::ScalarType::Float8_e4m3fn; +constexpr auto HALF = at::ScalarType::Half; +constexpr auto BF16 = at::ScalarType::BFloat16; +constexpr auto FLOAT = at::ScalarType::Float; +constexpr auto INT = at::ScalarType::Int; +constexpr auto UINT8 = at::ScalarType::Byte; + +void scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version == 100 || sm_version == 103, "fp4_quant is only supported on sm100a/sm103a"); + + CHECK_INPUT(output, "output must be a CUDA tensor"); + CHECK_INPUT(output_scale, "output_scale must be a CUDA tensor"); + CHECK_INPUT(input, "input must be a CUDA tensor"); + CHECK_INPUT(input_global_scale, "input_global_scale must be a CUDA tensor"); + CHECK_INPUT(input_offset_by_experts, "input_offset_by_experts must be a CUDA tensor"); + CHECK_INPUT(output_scale_offset_by_experts, "output_scale_offset_by_experts must be a CUDA tensor"); + + TORCH_CHECK(output.dim() == 2); + TORCH_CHECK(output_scale.dim() == 2); + TORCH_CHECK(input.dim() == 2); + TORCH_CHECK(input_global_scale.dim() == 1); + TORCH_CHECK(input_offset_by_experts.dim() == 1); + TORCH_CHECK(output_scale_offset_by_experts.dim() == 1); + + TORCH_CHECK(input.scalar_type() == HALF || input.scalar_type() == BF16); + TORCH_CHECK(input_global_scale.scalar_type() == FLOAT); + TORCH_CHECK(input_offset_by_experts.scalar_type() == INT); + TORCH_CHECK(output_scale_offset_by_experts.scalar_type() == INT); + // output is uint8 (two nvfp4 values are packed into one uint8) + // output_scale is int32 (four fp8 values are packed into one int32) + TORCH_CHECK(output.scalar_type() == UINT8); + TORCH_CHECK(output_scale.scalar_type() == INT); + + const int BLOCK_SIZE = 16; + auto m_topk = input.size(0); + auto k = input.size(1); + TORCH_CHECK(k % BLOCK_SIZE == 0, "k must be a multiple of 16"); + auto n_experts = input_global_scale.size(0); + TORCH_CHECK(input_offset_by_experts.size(0) == n_experts + 1); + TORCH_CHECK(output_scale_offset_by_experts.size(0) == n_experts + 1); + TORCH_CHECK(output.size(0) == m_topk); + TORCH_CHECK(output.size(1) == k / 2); + int scales_k = k / BLOCK_SIZE; + // 4 means the swizzle requirement by nvidia nvfp4. + int padded_k = (scales_k + (4 - 1)) / 4 * 4; + // 4 means 4 fp8 values are packed into one int32 + TORCH_CHECK(output_scale.size(1) * 4 == padded_k); + + auto in_dtype = input.dtype(); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + if (in_dtype == at::ScalarType::Half) { + quant_impl( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + input_offset_by_experts.data_ptr(), + output_scale_offset_by_experts.data_ptr(), + nullptr, // mask + false, // use_silu_and_mul + m_topk, + k, + n_experts, + stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + quant_impl<__nv_bfloat16>( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + input_offset_by_experts.data_ptr(), + output_scale_offset_by_experts.data_ptr(), + nullptr, // mask + false, // use_silu_and_mul + m_topk, + k, + n_experts, + stream); + } else { + TORCH_CHECK(false, "Expected input data type to be half or bfloat16"); + } +} + +void silu_and_mul_scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version == 100 || sm_version == 103, "fp4_quant is only supported on sm100a/sm103a"); + + CHECK_INPUT(output, "output must be a CUDA tensor"); + CHECK_INPUT(output_scale, "output_scale must be a CUDA tensor"); + CHECK_INPUT(input, "input must be a CUDA tensor"); + CHECK_INPUT(input_global_scale, "input_global_scale must be a CUDA tensor"); + CHECK_INPUT(mask, "mask must be a CUDA tensor"); + + TORCH_CHECK(output.dim() == 2); + TORCH_CHECK(output_scale.dim() == 2); + TORCH_CHECK(input.dim() == 2); + TORCH_CHECK(input_global_scale.dim() == 1); + + TORCH_CHECK(input.scalar_type() == HALF || input.scalar_type() == BF16); + TORCH_CHECK(input_global_scale.scalar_type() == FLOAT); + TORCH_CHECK(mask.scalar_type() == INT); + // output is uint8 (two nvfp4 values are packed into one uint8) + // output_scale is int32 (four fp8 values are packed into one int32) + TORCH_CHECK(output.scalar_type() == UINT8); + TORCH_CHECK(output_scale.scalar_type() == INT); + + const int BLOCK_SIZE = 16; + auto m_topk = input.size(0); + auto k_by_2 = input.size(1); + auto k = k_by_2; + if (use_silu_and_mul) { + TORCH_CHECK(k_by_2 % 2 == 0, "k must be a multiple of 2"); + k = k_by_2 / 2; + } + auto n_experts = input_global_scale.size(0); + TORCH_CHECK(mask.size(0) == n_experts); + TORCH_CHECK(output.size(0) == m_topk); + TORCH_CHECK(output.size(1) == k / 2); + int scales_k = k / BLOCK_SIZE; + // 4 means the swizzle requirement by nvidia nvfp4. + int padded_k = (scales_k + (4 - 1)) / 4 * 4; + // 4 means 4 fp8 values are packed into one int32 + TORCH_CHECK(output_scale.size(1) * 4 == padded_k); + + auto in_dtype = input.dtype(); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + if (in_dtype == at::ScalarType::Half) { + quant_impl( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + nullptr, // input_offset_by_experts + nullptr, // output_scale_offset_by_experts + mask.data_ptr(), + use_silu_and_mul, + m_topk, + k, + n_experts, + stream); + } else if (in_dtype == at::ScalarType::BFloat16) { + quant_impl<__nv_bfloat16>( + output.data_ptr(), + output_scale.data_ptr(), + input.data_ptr(), + input_global_scale.data_ptr(), + nullptr, // input_offset_by_experts + nullptr, // output_scale_offset_by_experts + mask.data_ptr(), + use_silu_and_mul, + m_topk, + k, + n_experts, + stream); + } else { + TORCH_CHECK(false, "Expected input data type to be half or bfloat16"); + } +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_quant.cuh b/sgl-kernel/csrc/gemm/nvfp4_quant.cuh new file mode 100644 index 000000000..b2aa5f006 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_quant.cuh @@ -0,0 +1,176 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +// Get type2 from type or vice versa (applied to half and bfloat16) +template +struct TypeConverter { + using Type = half2; +}; // keep for generality + +template <> +struct TypeConverter { + using Type = half; +}; + +template <> +struct TypeConverter { + using Type = half2; +}; + +template <> +struct TypeConverter<__nv_bfloat162> { + using Type = __nv_bfloat16; +}; + +template <> +struct TypeConverter<__nv_bfloat16> { + using Type = __nv_bfloat162; +}; + +#define ELTS_PER_THREAD 8 + +constexpr int CVT_FP4_ELTS_PER_THREAD = 8; +constexpr int CVT_FP4_SF_VEC_SIZE = 16; + +// Convert 8 float32 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float (&array)[8]) { + // PTX instructions used here requires sm100a/sm103a. +#if CUTLASS_ARCH_MMA_SM100A_ENABLED || CUTLASS_ARCH_MMA_SM103A_ENABLED + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0]), + "f"(array[1]), + "f"(array[2]), + "f"(array[3]), + "f"(array[4]), + "f"(array[5]), + "f"(array[6]), + "f"(array[7])); + return val; +#else + return 0; +#endif +} + +// Convert 4 float2 values into 8 e2m1 values (represented as one uint32_t). +inline __device__ uint32_t fp32_vec_to_e2m1(float2 (&array)[4]) { + // PTX instructions used here requires sm100a/sm103a. +#if CUTLASS_ARCH_MMA_SM100A_ENABLED || CUTLASS_ARCH_MMA_SM103A_ENABLED + uint32_t val; + asm volatile( + "{\n" + ".reg .b8 byte0;\n" + ".reg .b8 byte1;\n" + ".reg .b8 byte2;\n" + ".reg .b8 byte3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte0, %2, %1;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte1, %4, %3;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte2, %6, %5;\n" + "cvt.rn.satfinite.e2m1x2.f32 byte3, %8, %7;\n" + "mov.b32 %0, {byte0, byte1, byte2, byte3};\n" + "}" + : "=r"(val) + : "f"(array[0].x), + "f"(array[0].y), + "f"(array[1].x), + "f"(array[1].y), + "f"(array[2].x), + "f"(array[2].y), + "f"(array[3].x), + "f"(array[3].y)); + return val; +#else + return 0; +#endif +} + +// Fast reciprocal. +inline __device__ float reciprocal_approximate_ftz(float a) { + float b; + asm volatile("rcp.approx.ftz.f32 %0, %1;\n" : "=f"(b) : "f"(a)); + return b; +} + +template +__device__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(int rowIdx, int colIdx, int numCols, SFType* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 || CVT_FP4_NUM_THREADS_PER_SF == 2); + + // One pair of threads write one SF to global memory. + // TODO: stage through smem for packed STG.32 + // is it better than STG.8 from 4 threads ? + if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF == 0) { + // SF vector index (16 elements share one SF in the K dimension). + int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF; + int32_t mIdx = rowIdx; + + // SF layout [numMTiles, numKTiles, 32 (mTile), 4 (mTile), 4(kTile)] + // --> index [mTileIdx, kTileIdx, outerMIdx, innerMIdx, innerKIdx] + + int32_t mTileIdx = mIdx / (32 * 4); + // SF vector size 16. + int factor = CVT_FP4_SF_VEC_SIZE * 4; + int32_t numKTiles = (numCols + factor - 1) / factor; + int64_t mTileStride = numKTiles * 32 * 4 * 4; + + int32_t kTileIdx = (kIdx / 4); + int64_t kTileStride = 32 * 4 * 4; + + // M tile layout [32, 4] is column-major. + int32_t outerMIdx = (mIdx % 32); + int64_t outerMStride = 4 * 4; + + int32_t innerMIdx = (mIdx % (32 * 4)) / 32; + int64_t innerMStride = 4; + + int32_t innerKIdx = (kIdx % 4); + int64_t innerKStride = 1; + + // Compute the global offset. + int64_t SFOffset = mTileIdx * mTileStride + kTileIdx * kTileStride + outerMIdx * outerMStride + + innerMIdx * innerMStride + innerKIdx * innerKStride; + + return reinterpret_cast(SFout) + SFOffset; + } +#endif + return nullptr; +} + +// Define a 16 bytes packed data type. +template +struct PackedVec { + typename TypeConverter::Type elts[4]; +}; + +template <> +struct PackedVec<__nv_fp8_e4m3> { + __nv_fp8x2_e4m3 elts[8]; +}; diff --git a/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu b/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu new file mode 100644 index 000000000..d960aa730 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_quant_entry.cu @@ -0,0 +1,74 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void scaled_fp4_quant_sm100a( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf); + +void scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts); + +void silu_and_mul_scaled_fp4_experts_quant_sm100a( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul); + +#endif + +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return scaled_fp4_quant_sm100a(output, input, output_sf, input_sf); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 quantization"); +} + +void scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return scaled_fp4_experts_quant_sm100a( + output, output_scale, input, input_global_scale, input_offset_by_experts, output_scale_offset_by_experts); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 experts quantization kernel"); +} + +void silu_and_mul_scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return silu_and_mul_scaled_fp4_experts_quant_sm100a( + output, output_scale, input, input_global_scale, mask, use_silu_and_mul); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 experts quantization kernel"); +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu b/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu new file mode 100644 index 000000000..d307f5fb7 --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_quant_kernels.cu @@ -0,0 +1,239 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "nvfp4_quant.cuh" +#include "utils.h" + +// Quantizes the provided PackedVec into the uint32_t output +template +__device__ uint32_t cvt_warp_fp16_to_fp4(PackedVec& vec, float SFScaleVal, uint8_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + // Get absolute maximum values among the local 8 values. + auto localMax = __habs2(vec.elts[0]); + +// Local maximum value. +#pragma unroll + for (int i = 1; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + localMax = __hmax2(localMax, __habs2(vec.elts[i])); + } + + // Get the absolute maximum among all 16 values (two threads). + localMax = __hmax2(__shfl_xor_sync(uint32_t(-1), localMax, 1), localMax); + // Get the final absolute maximum values. + float vecMax = float(__hmax(localMax.x, localMax.y)); + + // Get the SF (max value of the vector / max value of e2m1). + // maximum value of e2m1 = 6.0. + // TODO: use half as compute data type. + float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f)); + // 8 bits representation of the SF. + uint8_t fp8SFVal; + // Write the SF to global memory (STG.8). + if constexpr (UE8M0_SF) { + __nv_fp8_e8m0 tmp; + tmp.__x = __nv_cvt_float_to_e8m0(SFValue, __NV_SATFINITE, cudaRoundPosInf); + SFValue = static_cast(tmp); + fp8SFVal = tmp.__x; + } else { + // Here SFValue is always positive, so E4M3 is the same as UE4M3. + __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue); + fp8SFVal = tmp.__x; + SFValue = static_cast(tmp); + } + // Get the output scale. + // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) * + // reciprocal(SFScaleVal)) + float outputScale = + SFValue != 0 ? reciprocal_approximate_ftz(SFValue * reciprocal_approximate_ftz(SFScaleVal)) : 0.0f; + + if (SFout) { + // Write the SF to global memory (STG.8). + *SFout = fp8SFVal; + } + + // Convert the input to float. + float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2]; + +#pragma unroll + for (int i = 0; i < CVT_FP4_ELTS_PER_THREAD / 2; i++) { + if constexpr (std::is_same_v) { + fp2Vals[i] = __half22float2(vec.elts[i]); + } else { + fp2Vals[i] = __bfloat1622float2(vec.elts[i]); + } + fp2Vals[i].x *= outputScale; + fp2Vals[i].y *= outputScale; + } + + // Convert to e2m1 values. + uint32_t e2m1Vec = fp32_vec_to_e2m1(fp2Vals); + + // Write the e2m1 values to global memory. + return e2m1Vec; +#else + return 0; +#endif +} + +// Use UE4M3 by default. +template +__global__ void +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) +__launch_bounds__(512, 4) cvt_fp16_to_fp4( +#else +cvt_fp16_to_fp4( +#endif + int32_t numRows, int32_t numCols, Type const* in, float const* SFScale, uint32_t* out, uint32_t* SFout) { +#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 1000) + using PackedVec = PackedVec; + static constexpr int CVT_FP4_NUM_THREADS_PER_SF = (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD); + static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD, "Vec size is not matched."); + + // Get the global scaling factor, which will be applied to the SF. + // Note SFScale is the same as next GEMM's alpha, which is + // (448.f / (Alpha_A / 6.f)). + float const SFScaleVal = SFScale == nullptr ? 1.0f : SFScale[0]; + + // Input tensor row/col loops. + for (int rowIdx = blockIdx.x; rowIdx < numRows; rowIdx += gridDim.x) { + for (int colIdx = threadIdx.x; colIdx < numCols / CVT_FP4_ELTS_PER_THREAD; colIdx += blockDim.x) { + int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx; + PackedVec in_vec = reinterpret_cast(in)[inOffset]; + // Get the output tensor offset. + // Same as inOffset because 8 elements are packed into one uint32_t. + int64_t outOffset = inOffset; + auto& out_pos = out[outOffset]; + + auto sf_out = + cvt_quant_to_fp4_get_sf_out_offset(rowIdx, colIdx, numCols, SFout); + + out_pos = cvt_warp_fp16_to_fp4(in_vec, SFScaleVal, sf_out); + } + } +#endif +} + +template +void invokeFP4Quantization( + int m, + int n, + T const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream) { + // Grid, Block size. + // Each thread converts 8 values. + dim3 block(std::min(int(n / ELTS_PER_THREAD), 512)); + // Get number of blocks per SM (assume we can fully utilize the SM). + int const numBlocksPerSM = 2048 / block.x; + dim3 grid(std::min(int(m), multiProcessorCount * numBlocksPerSM)); + + // Launch the cvt kernel. + if (useUE8M0) { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } else { + cvt_fp16_to_fp4<<>>( + m, n, input, SFScale, reinterpret_cast(output), reinterpret_cast(SFOuput)); + } +} + +// Instantiate the function. +template void invokeFP4Quantization( + int m, + int n, + half const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +template void invokeFP4Quantization( + int m, + int n, + __nv_bfloat16 const* input, + float const* SFScale, + int64_t* output, + int32_t* SFOuput, + bool useUE8M0, + int multiProcessorCount, + cudaStream_t stream); + +inline int getMultiProcessorCount() { + static int multi_processor_count = []() { + int device_id = 0; + int count = 0; + + // Get the current CUDA device ID + CHECK_CUDA_SUCCESS(cudaGetDevice(&device_id)); + + // Get the number of multiprocessors for the current device + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&count, cudaDevAttrMultiProcessorCount, device_id)); + + return count; // Initialize the static variable + }(); + + return multi_processor_count; // Return the cached value on subsequent calls +} + +void scaled_fp4_quant_sm100a( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_sf, torch::Tensor const& input_sf) { + auto sm_version = getSMVersion(); + TORCH_CHECK(sm_version == 100 || sm_version == 103, "fp4_quant is only supported on sm100a/sm103a"); + + int32_t m = input.size(0); + int32_t n = input.size(1); + + TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16."); + + int multiProcessorCount = getMultiProcessorCount(); + + auto input_sf_ptr = static_cast(input_sf.data_ptr()); + auto sf_out = static_cast(output_sf.data_ptr()); + auto output_ptr = static_cast(output.data_ptr()); + at::cuda::CUDAGuard device_guard{(char)input.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(input.get_device()); + + // We don't support e8m0 scales at this moment. + bool useUE8M0 = false; + + switch (input.scalar_type()) { + case torch::kHalf: { + auto input_ptr = reinterpret_cast(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + case torch::kBFloat16: { + auto input_ptr = reinterpret_cast<__nv_bfloat16 const*>(input.data_ptr()); + invokeFP4Quantization(m, n, input_ptr, input_sf_ptr, output_ptr, sf_out, useUE8M0, multiProcessorCount, stream); + break; + } + default: { + std::cerr << "Observing: " << input.scalar_type() << " for the input datatype which is invalid"; + throw std::runtime_error("Unsupported input data type for quantize_to_fp4."); + } + } +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu new file mode 100644 index 000000000..8bcd8c52b --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_entry.cu @@ -0,0 +1,39 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 +void cutlass_scaled_fp4_mm_sm100a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); +#endif + +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + return cutlass_scaled_fp4_mm_sm100a(D, A, B, A_sf, B_sf, alpha); +#endif + TORCH_CHECK_NOT_IMPLEMENTED(false, "No compiled nvfp4 mm kernel."); +} diff --git a/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu new file mode 100644 index 000000000..a103545dd --- /dev/null +++ b/sgl-kernel/csrc/gemm/nvfp4_scaled_mm_kernels.cu @@ -0,0 +1,453 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +// clang-format off +#include "cutlass/cutlass.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/util/packed_stride.hpp" +// clang-format on + +/** + * Helper function for checking CUTLASS errors + */ +#define CUTLASS_CHECK(status) \ + { \ + cutlass::Status error = status; \ + TORCH_CHECK(error == cutlass::Status::kSuccess, cutlassGetStatusString(error)); \ + } + +using namespace cute; + +#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +// Config(half_t/bfloat16_t) for M <= 128 +template +struct KernelConfigM128 { + using OutputType = T; + using MmaTileShape = Shape<_128, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized1Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized1SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigM128::preferred_cluster(1, 4, 1); +template +const dim3 KernelConfigM128::fallback_cluster(1, 2, 1); + +// Config(half_t/bfloat16_t) for M <= 256 +template +struct KernelConfigM256 { + using OutputType = T; + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized2Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized2SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigM256::preferred_cluster(2, 4, 1); +template +const dim3 KernelConfigM256::fallback_cluster(2, 1, 1); + +// Default config(half_t/bfloat16_t) for M > 256 +template +struct KernelConfigDefault { + using OutputType = T; + using MmaTileShape = Shape<_256, _256, _256>; + using ClusterShape = Shape; + using EpilogueTile = Shape<_128, _64>; // Avoid register spilling + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized2Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized2SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +template +const dim3 KernelConfigDefault::preferred_cluster(4, 4, 1); +template +const dim3 KernelConfigDefault::fallback_cluster(2, 1, 1); + +struct KernelConfigFp32 { + using OutputType = float; + using MmaTileShape = Shape<_128, _128, _256>; + using ClusterShape = Shape; + using EpilogueTile = cutlass::epilogue::collective::EpilogueTileAuto; + using EpilogueSchedule = cutlass::epilogue::TmaWarpSpecialized1Sm; + using MainloopSchedule = cutlass::gemm::KernelTmaWarpSpecialized1SmNvf4Sm100; + const static dim3 preferred_cluster; + const static dim3 fallback_cluster; +}; +const dim3 KernelConfigFp32::preferred_cluster = dim3(1, 4, 1); +const dim3 KernelConfigFp32::fallback_cluster = dim3(1, 2, 1); + +template +struct Fp4GemmSm100 { + using Config = KernelConfig; // For generating args + using OutputType = typename KernelConfig::OutputType; + // A matrix configuration + using ElementA = cutlass::nv_float4_t; + using LayoutATag = cutlass::layout::RowMajor; + static constexpr int AlignmentA = 32; + + // B matrix configuration + using ElementB = cutlass::nv_float4_t; + using LayoutBTag = cutlass::layout::ColumnMajor; + static constexpr int AlignmentB = 32; + + // C/D matrix configuration + using ElementD = OutputType; + using ElementC = OutputType; + using LayoutCTag = cutlass::layout::RowMajor; + using LayoutDTag = cutlass::layout::RowMajor; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + // Kernel functional config + using ElementAccumulator = float; + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; + + // Kernel Perf config + using MmaTileShape = typename KernelConfig::MmaTileShape; + using ClusterShape = typename KernelConfig::ClusterShape; + using EpilogueTile = typename KernelConfig::EpilogueTile; + using EpilogueSchedule = typename KernelConfig::EpilogueSchedule; + using MainloopSchedule = typename KernelConfig::MainloopSchedule; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + MmaTileShape, + ClusterShape, + EpilogueTile, + ElementAccumulator, + ElementAccumulator, + void, + LayoutCTag, + AlignmentC, + ElementD, + LayoutDTag, + AlignmentD, + EpilogueSchedule, + cutlass::epilogue::fusion::LinearCombination>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + LayoutATag, + AlignmentA, + ElementB, + LayoutBTag, + AlignmentB, + ElementAccumulator, + MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + MainloopSchedule>::CollectiveOp; + + using GemmKernel = + cutlass::gemm::kernel::GemmUniversal, CollectiveMainloop, CollectiveEpilogue, void>; + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using StrideA = typename Gemm::GemmKernel::StrideA; + using LayoutA = decltype(cute::make_layout(make_shape(0, 0, 0), StrideA{})); + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFA; + using StrideB = typename Gemm::GemmKernel::StrideB; + using LayoutB = decltype(cute::make_layout(make_shape(0, 0, 0), StrideB{})); + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFB; + using StrideC = typename Gemm::GemmKernel::StrideC; + using LayoutC = decltype(cute::make_layout(make_shape(0, 0, 0), StrideC{})); + using StrideD = typename Gemm::GemmKernel::StrideD; + using LayoutD = decltype(cute::make_layout(make_shape(0, 0, 0), StrideD{})); +}; + +template +typename T::Gemm::Arguments args_from_options( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t M, + int64_t N, + int64_t K) { + using ElementA = typename T::Gemm::ElementA; + using ElementB = typename T::Gemm::ElementB; + using ElementSFA = cutlass::float_ue4m3_t; + using ElementSFB = cutlass::float_ue4m3_t; + using ElementD = typename T::Gemm::ElementD; + using ElementCompute = float; + using StrideA = typename T::StrideA; + using StrideB = typename T::StrideB; + using StrideD = typename T::StrideD; + using Sm1xxBlkScaledConfig = typename T::Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + int m = static_cast(M); + int n = static_cast(N); + int k = static_cast(K); + auto stride_A = cutlass::make_cute_packed_stride(StrideA{}, {m, k, 1}); + auto stride_B = cutlass::make_cute_packed_stride(StrideB{}, {n, k, 1}); + auto stride_D = cutlass::make_cute_packed_stride(StrideD{}, {m, n, 1}); + + auto layout_SFA = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + auto layout_SFB = Sm1xxBlkScaledConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + + typename T::Gemm::Arguments arguments{ + cutlass::gemm::GemmUniversalMode::kGemm, + {m, n, k, 1}, + {// Mainloop arguments + static_cast(A.data_ptr()), + stride_A, + static_cast(B.data_ptr()), + stride_B, + static_cast(A_sf.data_ptr()), + layout_SFA, + static_cast(B_sf.data_ptr()), + layout_SFB}, + { // Epilogue arguments + {}, // epilogue.thread + nullptr, + stride_D, + static_cast(D.data_ptr()), + stride_D}}; + auto& fusion_args = arguments.epilogue.thread; + fusion_args.alpha_ptr = static_cast(alpha.data_ptr()); + using KernelConfig = typename T::Config; + arguments.hw_info.cluster_shape = KernelConfig::preferred_cluster; + arguments.hw_info.cluster_shape_fallback = KernelConfig::fallback_cluster; + return arguments; +} + +template +void runGemm( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + typename T::Gemm gemm; + auto arguments = args_from_options(D, A, B, A_sf, B_sf, alpha, m, n, k); + + size_t workspace_size = T::Gemm::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(A.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + CUTLASS_CHECK(gemm.can_implement(arguments)); + + CUTLASS_CHECK(gemm.initialize(arguments, workspace.data_ptr(), stream)); + + CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream)); +} + +// Dispatch function to select appropriate config based on M +template +void cutlassFp4GemmDispatch( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + if (m <= 128) { + // m in [1, 128] + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (m <= 256) { + // m in (128, 256] + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + // m in (256, inf) + runGemm>>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } +} + +// Dispatch function to select appropriate config based on M +template <> +void cutlassFp4GemmDispatch( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + runGemm>(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); +} + +#else +template +void cutlassFp4GemmDispatch( + at::Tensor& D, + at::Tensor const& A, + at::Tensor const& B, + at::Tensor const& A_sf, + at::Tensor const& B_sf, + at::Tensor const& alpha, + int64_t m, + int64_t n, + int64_t k, + cudaStream_t stream) { + TORCH_CHECK( + false, + "Unsupported CUTLASS version. Set VLLM_CUTLASS_SRC_DIR to " + "a CUTLASS 3.8 source directory to enable support."); +} +#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) + +#define CHECK_TYPE(x, st, m) TORCH_CHECK(x.scalar_type() == st, "Inconsistency of Tensor type:", m) +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, "must be a CUDA tensor") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, "must be contiguous") +#define CHECK_INPUT(x, st, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); \ + CHECK_TYPE(x, st, m) + +constexpr auto FLOAT4_E2M1X2 = at::ScalarType::Byte; +constexpr auto SF_DTYPE = at::ScalarType::Float8_e4m3fn; + +void cutlass_scaled_fp4_mm_sm100a( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha) { + CHECK_INPUT(A, FLOAT4_E2M1X2, "a"); + CHECK_INPUT(B, FLOAT4_E2M1X2, "b"); + + CHECK_INPUT(A_sf, SF_DTYPE, "scale_a"); + CHECK_INPUT(B_sf, SF_DTYPE, "scale_b"); + + CHECK_INPUT(alpha, at::ScalarType::Float, "alpha"); + + TORCH_CHECK(A.dim() == 2, "a must be a matrix"); + TORCH_CHECK(B.dim() == 2, "b must be a matrix"); + TORCH_CHECK( + A.size(1) == B.size(1), + "a and b shapes cannot be multiplied (", + A.size(0), + "x", + A.size(1), + " and ", + B.size(0), + "x", + B.size(1), + ")"); + + auto const m = A.size(0); + auto const n = B.size(0); + auto const k = A.size(1) * 2; + + constexpr int alignment = 32; + TORCH_CHECK( + k % alignment == 0, + "Expected k to be divisible by ", + alignment, + ", but got a shape: (", + A.size(0), + "x", + A.size(1), + "), k: ", + k, + "."); + TORCH_CHECK( + n % alignment == 0, + "Expected n to be divisible by ", + alignment, + ", but got b shape: (", + B.size(0), + "x", + B.size(1), + ")."); + + auto round_up = [](int x, int y) { return (x + y - 1) / y * y; }; + int rounded_m = round_up(m, 128); + int rounded_n = round_up(n, 128); + // Since k is divisible by 32 (alignment), k / 16 is guaranteed to be an + // integer. + int rounded_k = round_up(k / 16, 4); + + TORCH_CHECK(A_sf.dim() == 2, "scale_a must be a matrix"); + TORCH_CHECK(B_sf.dim() == 2, "scale_b must be a matrix"); + TORCH_CHECK( + A_sf.size(1) == B_sf.size(1), + "scale_a and scale_b shapes cannot be multiplied (", + A_sf.size(0), + "x", + A_sf.size(1), + " and ", + B_sf.size(0), + "x", + B_sf.size(1), + ")"); + TORCH_CHECK( + A_sf.size(0) == rounded_m && A_sf.size(1) == rounded_k, + "scale_a must be padded and swizzled to a shape (", + rounded_m, + "x", + rounded_k, + "), but got a shape (", + A_sf.size(0), + "x", + A_sf.size(1), + ")"); + TORCH_CHECK( + B_sf.size(0) == rounded_n && B_sf.size(1) == rounded_k, + "scale_b must be padded and swizzled to a shape (", + rounded_n, + "x", + rounded_k, + "), but got a shape (", + B_sf.size(0), + "x", + B_sf.size(1), + ")"); + + auto out_dtype = D.dtype(); + at::cuda::CUDAGuard device_guard{(char)A.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(A.get_device()); + + if (out_dtype == at::ScalarType::Half) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::BFloat16) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else if (out_dtype == at::ScalarType::Float) { + cutlassFp4GemmDispatch(D, A, B, A_sf, B_sf, alpha, m, n, k, stream); + } else { + TORCH_CHECK(false, "Unsupported output data type of nvfp4 mm"); + } +} diff --git a/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu b/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu new file mode 100644 index 000000000..7afff7794 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_tensor_quant_fp8.cu @@ -0,0 +1,123 @@ +#include +#include + +#include +#include +#include + +#include "utils.h" + +template +__global__ void +per_tensor_absmax_kernel(const T* __restrict__ input, float* __restrict__ output_s, const int64_t num_elements) { + float max_value = 0.0f; + unsigned int tid = threadIdx.x; + unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / vec_size; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + const int32_t remaining_start = num_vec_elems * vec_size; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = static_cast(input[idx]); + max_value = fmaxf(max_value, fabsf(val)); + } + + max_value = blockReduceMax(max_value); + + if (tid == 0) { + atomicMaxFloat(output_s, max_value / FP8_E4M3_MAX); + } +} + +template +__global__ void per_tensor_quant_fp8_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output, + const float* __restrict__ scale, + const int64_t num_elements) { + const int gid = blockIdx.x * blockDim.x + threadIdx.x; + const int grid_size = blockDim.x * gridDim.x; + const float scale_val = 1.0f / (*scale); + + // We want to store 128 bits of data at a time. 16 = 128 / 8 bits + // Load is already vectorized, so 16 elements work for T. + const uint32_t VEC_SIZE = 16; + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = num_elements / VEC_SIZE; + + for (int32_t i = gid; i < num_vec_elems; i += grid_size) { + vec_t input_vec; + input_vec.cast_load(input + i * VEC_SIZE); + + DST_DTYPE output_arr[VEC_SIZE]; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + float val = fmax(fmin(static_cast(input_vec[j]) * scale_val, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + *(uint4*)(output + i * VEC_SIZE) = *(uint4*)output_arr; + } + + const int32_t remaining_start = num_vec_elems * VEC_SIZE; + for (int32_t idx = remaining_start + gid; idx < num_elements; idx += grid_size) { + float val = fmax(-FP8_E4M3_MAX, fmin(static_cast(input[idx]) * scale_val, FP8_E4M3_MAX)); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output[idx] = static_cast(val); +#else + output[idx] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } +} + +void sgl_per_tensor_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s, bool is_static) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + + const int block_size = 256; + const int num_elements = input.numel(); + const int num_blocks = min((num_elements + block_size - 1) / block_size, 1024); + + dim3 grid(num_blocks); + dim3 block(block_size); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (is_static == false) { + per_tensor_absmax_kernel<<>>( + static_cast(input.data_ptr()), static_cast(output_s.data_ptr()), num_elements); + } + + per_tensor_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + num_elements); + return true; + }); +} diff --git a/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu new file mode 100644 index 000000000..82daaef19 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_token_group_quant_8bit.cu @@ -0,0 +1,240 @@ +#include +#include + +#include +#include + +#include "utils.h" + +__device__ __forceinline__ float GroupReduceMax(float val, const int tid) { + unsigned mask = threadIdx.x % 32 >= 16 ? 0xffff0000 : 0x0000ffff; + + val = fmaxf(val, __shfl_xor_sync(mask, val, 8)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 4)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 2)); + val = fmaxf(val, __shfl_xor_sync(mask, val, 1)); + return val; +} + +template < + typename T, + typename DST_DTYPE, + bool IS_COLUMN_MAJOR = false, + bool SCALE_UE8M0 = false, + typename scale_packed_t = std::conditional_t> +__global__ void per_token_group_quant_8bit_kernel( + const T* __restrict__ input, + void* __restrict__ output_q, + scale_packed_t* __restrict__ output_s, + const int group_size, + const int num_groups, + const int groups_per_block, + const float eps, + const float min_8bit, + const float max_8bit, + const int num_groups_per_row = 0, + const int scale_stride = 0) { + const int threads_per_group = 16; + const int64_t local_group_id = threadIdx.x / threads_per_group; + const int lane_id = threadIdx.x % threads_per_group; + + const int64_t block_group_id = blockIdx.x * groups_per_block; + const int64_t global_group_id = block_group_id + local_group_id; + const int64_t block_group_offset = global_group_id * group_size; + + float local_absmax = eps; + + using scale_element_t = std::conditional_t; + static_assert(sizeof(scale_packed_t) % sizeof(scale_element_t) == 0); + + const T* group_input = input + block_group_offset; + DST_DTYPE* group_output = static_cast(output_q) + block_group_offset; + scale_element_t* scale_output; + + if constexpr (IS_COLUMN_MAJOR) { + const int num_elems_per_pack = static_cast(sizeof(scale_packed_t) / sizeof(scale_element_t)); + const int row_idx = global_group_id / num_groups_per_row; + const int col_idx_unpacked = global_group_id % num_groups_per_row; + const int col_idx = col_idx_unpacked / num_elems_per_pack; + const int pack_idx = col_idx_unpacked % num_elems_per_pack; + scale_output = reinterpret_cast(output_s) + + (col_idx * scale_stride * num_elems_per_pack + row_idx * num_elems_per_pack + pack_idx); + } else { + static_assert(!SCALE_UE8M0); + scale_output = output_s + global_group_id; + } + + constexpr uint32_t vec_size = 16 / sizeof(T); + using vec_t = flashinfer::vec_t; + + const int32_t num_vec_elems = group_size / vec_size; + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float abs_val = fabsf(val); + local_absmax = fmaxf(local_absmax, abs_val); + } + } + + local_absmax = GroupReduceMax(local_absmax, lane_id); + + float y_s = local_absmax / max_8bit; + if constexpr (SCALE_UE8M0) { + y_s = exp2f(ceilf(log2f(fmaxf(y_s, 1e-10f)))); + } + + // TODO can optimize + scale_element_t y_s_quant; + if constexpr (SCALE_UE8M0) { + y_s_quant = (uint8_t)(((int)log2f(y_s)) + 127); + } else { + y_s_quant = y_s; + } + + if (lane_id == 0) { + *scale_output = y_s_quant; + } + + for (int32_t i = lane_id; i < num_vec_elems; i += 16) { + vec_t input_vec; + input_vec.cast_load(group_input + i * vec_size); + +#pragma unroll + for (uint32_t j = 0; j < vec_size; ++j) { + float val = static_cast(input_vec[j]); + float q_val = fminf(fmaxf(val / y_s, min_8bit), max_8bit); + group_output[i * vec_size + j] = DST_DTYPE(q_val); + } + } +} + +void sgl_per_token_group_quant_8bit( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double min_8bit, + double max_8bit, + bool scale_ue8m0 = false) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + + const int num_groups = input.numel() / group_size; + + CHECK_EQ(input.numel() % group_size, 0); + CHECK_EQ(output_s.dim(), 2); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + constexpr int THREADS_PER_GROUP = 16; + + int groups_per_block = 1; + + if (num_groups % 16 == 0) { + groups_per_block = 16; + } else if (num_groups % 8 == 0) { + groups_per_block = 8; + } else if (num_groups % 4 == 0) { + groups_per_block = 4; + } else if (num_groups % 2 == 0) { + groups_per_block = 2; + } + + auto dst_type = output_q.scalar_type(); + const int num_blocks = num_groups / groups_per_block; + const int num_threads = groups_per_block * THREADS_PER_GROUP; + + const bool is_column_major = output_s.stride(0) < output_s.stride(1); + const int hidden_dim = input.size(input.dim() - 1); + const int num_groups_per_row = hidden_dim / group_size; + const int scale_stride = output_s.stride(1); + +#define LAUNCH_KERNEL(T, DST_DTYPE) \ + do { \ + dim3 grid(num_blocks); \ + dim3 block(num_threads); \ + if (is_column_major) { \ + if (scale_ue8m0) { \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit, \ + num_groups_per_row, \ + scale_stride); \ + } else { \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit, \ + num_groups_per_row, \ + scale_stride); \ + } \ + } else { \ + assert(!scale_ue8m0); \ + per_token_group_quant_8bit_kernel<<>>( \ + static_cast(input.data_ptr()), \ + output_q.data_ptr(), \ + static_cast(output_s.data_ptr()), \ + group_size, \ + num_groups, \ + groups_per_block, \ + (float)eps, \ + (float)min_8bit, \ + (float)max_8bit); \ + } \ + } while (0) + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (dst_type == at::ScalarType::Char) { + LAUNCH_KERNEL(scalar_t, int8_t); + return true; + } else if (dst_type == at::ScalarType::Float8_e4m3fn) { + LAUNCH_KERNEL(scalar_t, __nv_fp8_e4m3); + return true; + } + return false; + }); + +#undef LAUNCH_KERNEL +} + +void sgl_per_token_group_quant_int8( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double int8_min, + double int8_max) { + sgl_per_token_group_quant_8bit(input, output_q, output_s, group_size, eps, int8_min, int8_max); +} + +void sgl_per_token_group_quant_fp8( + torch::Tensor input, + torch::Tensor output_q, + torch::Tensor output_s, + int64_t group_size, + double eps, + double fp8_min, + double fp8_max, + bool scale_ue8m0) { + sgl_per_token_group_quant_8bit(input, output_q, output_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0); +} diff --git a/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu b/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu new file mode 100644 index 000000000..e73716c86 --- /dev/null +++ b/sgl-kernel/csrc/gemm/per_token_quant_fp8.cu @@ -0,0 +1,227 @@ +#include + +#include +#include + +#include "utils.h" + +static constexpr int kWarpSize = 32; + +// --------------------------------------------------------------------------- +// 1. Warp‑local, no shared memory +// • One warp handles one token. +// • Eight tokens per 256‑thread CTA. +// --------------------------------------------------------------------------- +template +__global__ void per_token_quant_fp8_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output_q, + float* __restrict__ output_s, + const int64_t hidden_dim, + const int64_t num_tokens) { + const int warp_id = threadIdx.x / kWarpSize; // 0‑7 (8 warps) + const int lane_id = threadIdx.x & (kWarpSize - 1); // 0‑31 + const int token_id = blockIdx.x * kTokensPerCTA + warp_id; + if (token_id >= num_tokens) return; + + // Global tensors for this token + const T* token_input = input + token_id * hidden_dim; + DST_DTYPE* token_output = output_q + token_id * hidden_dim; + float* token_scale = output_s + token_id; + + // + // Pass-1: Perform a warp reduce to find the max_value of a token's hidden_dim + // + float max_value = 0.f; + using vec_t = flashinfer::vec_t; + const int32_t num_vec_elems = hidden_dim / kVecSize; + + for (int32_t i = lane_id; i < num_vec_elems; i += kWarpSize) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + max_value = fmaxf(max_value, fabsf(static_cast(input_vec[j]))); + } + } + + float warp_max = warpReduceMax(max_value); + + __shared__ float scale; + scale = warp_max / FP8_E4M3_MAX; + // Broadcast scale + if (lane_id == 0) { + token_scale[0] = scale; + } + float scale_inv = (scale == 0.f) ? 0.f : 1.0f / scale; + + // + // Pass-2: quantize and write back + // + for (int i = lane_id; i < num_vec_elems; i += kWarpSize) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + DST_DTYPE output_arr[kVecSize]; +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = static_cast(input_vec[j]) * scale_inv; + val = fmaxf(fminf(val, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + if constexpr (kVecSize == 16) { + *(uint4*)(token_output + i * kVecSize) = *(uint4*)output_arr; + } else { + // Use element-wise copy for vector size 8 to ensure correctness + for (int k = 0; k < kVecSize; ++k) { + token_output[i * kVecSize + k] = output_arr[k]; + } + } + } +} + +// --------------------------------------------------------------------------- +// 2. Baseline kernel (1 token / CTA, CUB block reduce) +// --------------------------------------------------------------------------- +template +__global__ void per_token_quant_fp8_small_batch_kernel( + const T* __restrict__ input, + DST_DTYPE* __restrict__ output_q, + float* __restrict__ output_s, + const int64_t hidden_dim, + const int64_t num_tokens) { + const int token_idx = blockIdx.x; + if (token_idx >= num_tokens) return; + + const int tid = threadIdx.x; + const int block_dim = blockDim.x; + + const T* token_input = input + token_idx * hidden_dim; + DST_DTYPE* token_output = output_q + token_idx * hidden_dim; + + float max_value = 0.0f; + + // Use template parameter for vector size + using vec_t = flashinfer::vec_t; + const int32_t num_vec_elems = hidden_dim / kVecSize; + + // Find max using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = static_cast(input_vec[j]); + max_value = fmaxf(max_value, fabsf(val)); + } + } + + max_value = blockReduceMax(max_value); + + __shared__ float scale; + if (tid == 0) { + scale = max_value / FP8_E4M3_MAX; + output_s[token_idx] = scale; + } + __syncthreads(); + + const float scale_inv = 1.0f / scale; + + // Quantize using vectorized loads + for (int32_t i = tid; i < num_vec_elems; i += block_dim) { + vec_t input_vec; + input_vec.cast_load(token_input + i * kVecSize); + + DST_DTYPE output_arr[kVecSize]; +#pragma unroll + for (uint32_t j = 0; j < kVecSize; ++j) { + float val = fmaxf(fminf(static_cast(input_vec[j]) * scale_inv, FP8_E4M3_MAX), -FP8_E4M3_MAX); +#if !defined(USE_ROCM) || defined(HIP_FP8_TYPE_E4M3) + output_arr[j] = static_cast(val); +#else + output_arr[j] = c10::Float8_e4m3fnuz( + __hip_cvt_float_to_fp8(val, fp8::fp8_type::__default_saturation, fp8::fp8_type::__default_interpret), + c10::Float8_e4m3fnuz::from_bits()); +#endif + } + + if constexpr (kVecSize == 16) { + *(uint4*)(token_output + i * kVecSize) = *(uint4*)output_arr; + } else { + // Use element-wise copy for vector size 8 to ensure correctness + for (int k = 0; k < kVecSize; ++k) { + token_output[i * kVecSize + k] = output_arr[k]; + } + } + } +} + +void sgl_per_token_quant_fp8(torch::Tensor input, torch::Tensor output_q, torch::Tensor output_s) { + CHECK_INPUT(input); + CHECK_INPUT(output_q); + CHECK_INPUT(output_s); + const auto input_sizes = input.sizes(); + const int64_t num_tokens = input_sizes[0]; + const int64_t hidden_dim = input_sizes[1]; + TORCH_CHECK(hidden_dim % 8 == 0, "Hidden dimension must be divisible by 8, but got ", hidden_dim); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + const int sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + const int TOKENS_PER_CTA = 8; + const bool use_warp_kernel = (num_tokens >= sm_count * 2 * TOKENS_PER_CTA); + const bool use_vec16 = (hidden_dim % 16 == 0); + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(input.scalar_type(), scalar_t, [&] { + if (use_warp_kernel) { + // -------- warp‑local --------------------------------------------------- + constexpr int THREADS = TOKENS_PER_CTA * kWarpSize; // 256 + dim3 grid((num_tokens + TOKENS_PER_CTA - 1) / TOKENS_PER_CTA); + dim3 block(THREADS); + + if (use_vec16) { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else { + per_token_quant_fp8_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } + } else { + // -------- baseline ----------------------------------------------------- + constexpr int THREADS = 256; + dim3 grid(num_tokens); + dim3 block(THREADS); + + if (use_vec16) { + per_token_quant_fp8_small_batch_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } else { + per_token_quant_fp8_small_batch_kernel<<>>( + static_cast(input.data_ptr()), + static_cast<__nv_fp8_e4m3*>(output_q.data_ptr()), + static_cast(output_s.data_ptr()), + hidden_dim, + num_tokens); + } + } + return true; + }); +} diff --git a/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu b/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu new file mode 100644 index 000000000..79180737f --- /dev/null +++ b/sgl-kernel/csrc/gemm/qserve_w4a8_per_chn_gemm.cu @@ -0,0 +1,710 @@ +// Implemented by Haotian Tang and Shang Yang. +// @article{lin2024qserve, +// title={QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving}, +// author={Lin*, Yujun and Tang*, Haotian and Yang*, Shang and Zhang, Zhekai and Xiao, Guangxuan and Gan, Chuang and +// Han, Song}, journal={arXiv preprint arXiv:2405.04532}, year={2024} +// } +// @article{yang2025lserve, +// title={LServe: Efficient Long-sequence LLM Serving with Unified Sparse Attention}, +// author={Yang*, Shang and Guo*, Junxian and Tang, Haotian and Hu, Qinghao and Xiao, Guangxuan and Tang, Jiaming and +// Lin, Yujun and Liu, Zhijian and Lu, Yao and Han, Song}, year={2025} +// } + +// Adapted from https://github.com/mit-han-lab/omniserve/blob/main/kernels/csrc/qgemm/w4a8_per_chn/gemm_cuda.cu + +#include +#include +#include +#include + +#include "utils.h" + +#define OP_M 16 +#define OP_N 8 +#define OP_K 32 +#define INTRIN_M 16 +#define INTRIN_N 16 +#define INTRIN_K 32 +#define WARP_SIZE 32 +#define SMEM_PAD_A 0 +#define SMEM_PAD_B 0 +#define PACK_SIZE 16 +#if (__CUDACC_VER_MAJOR__ >= 11) && (__CUDACC_VER_MINOR__ >= 4) +#define L2_CACHEHINT(size) ".L2::" #size "B" +#else +#define L2_CACHEHINT(size) +#endif + +#define KERNEL_LAUNCH_CODE \ + constexpr int NUM_WARPS = (CTA_M / WARP_M) * (CTA_N / WARP_N) * (CTA_K / WARP_K); \ + constexpr int SCALES_SMEM_SIZE = (G >= CTA_K) ? (CTA_N * STAGES * 2) : (CTA_N * (CTA_K / G) * STAGES * 2); \ + constexpr int kSmemByteSize = \ + ((CTA_M * (CTA_K + SMEM_PAD_A) + CTA_N * (CTA_K + SMEM_PAD_B) / 2) * STAGES + SCALES_SMEM_SIZE) * \ + sizeof(int8_t); \ + if (kSmemByteSize >= 99 * 1024) { \ + printf( \ + "This kernel requires %d Bytes of shared memory, which exceeds " \ + "device limit.\n", \ + kSmemByteSize); \ + return; \ + } \ + int num_blocks_m = (num_out_feats + CTA_M - 1) / CTA_M; \ + int num_blocks_n = num_out_channels / CTA_N / 1; \ + const int log_tile = get_log_tile<8>((num_out_feats + CTA_M - 1) / CTA_M); \ + const int tile_shift = 1 << log_tile; \ + dim3 num_blocks(num_blocks_n* tile_shift, (num_blocks_m + tile_shift - 1) / tile_shift); \ + dim3 threads_per_block(WARP_SIZE, NUM_WARPS); \ + auto kernel_func = dense_kernel0; \ + cudaFuncSetAttribute(kernel_func, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemByteSize); \ + kernel_func<<>>( \ + in_feats, kernel, wscales, ascales, w_szs, a_ssums, out_feats, num_in_feats, num_out_channels, num_in_channels); + +template +__inline__ __host__ __device__ int get_log_tile(int n) { + if (N >= 8 && n >= 6) + return 3; + else if (N >= 4 && n >= 3) + return 2; + else if (N >= 2 && n >= 2) + return 1; + else + return 0; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +__inline__ __device__ uint2 get_block_idx_mapping(int blockIdx_x, int blockIdx_y, int log_tile) { + return make_uint2((blockIdx_x >> log_tile), (blockIdx_y << log_tile) + ((blockIdx_x) & ((1 << (log_tile)) - 1))); +} + +__inline__ __device__ uint32_t cast_smem_ptr_to_uint(void const* const ptr) { + uint32_t smem_int_ptr; + + asm("{.reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, " + "smem_ptr; }\n" + : "=r"(smem_int_ptr) + : "l"(ptr)); + + return smem_int_ptr; +} + +__inline__ __device__ void ldmatrix_m8n8_x4_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +__inline__ __device__ void ldmatrix_m8n8_x4_trans_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +// function from lmdeploy +__inline__ __device__ void cp_async_cg_A(uint32_t smem_int_ptr, const uint4* __restrict__ src, bool mask) { + const int cp_size = 16; + asm volatile("{" + " .reg .pred p;" + " setp.ne.b32 p, %0, 0;" + " @p cp.async.cg.shared.global" L2_CACHEHINT(128) " [%1], [%2], %3;" + "}" ::"r"((int)mask), + "r"(smem_int_ptr), + "l"(src), + "n"(cp_size)); +} + +__device__ __inline__ void mma_m16n8k32(void* C_warp, void* A_shared_warp, void* B_shared_warp) { + __asm__ __volatile__( + "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32" + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};" + : "=r"(((int*)C_warp)[0]), "=r"(((int*)C_warp)[1]), "=r"(((int*)C_warp)[2]), "=r"(((int*)C_warp)[3]) + : "r"(((unsigned*)A_shared_warp)[0]), + "r"(((unsigned*)A_shared_warp)[1]), + "r"(((unsigned*)A_shared_warp)[2]), + "r"(((unsigned*)A_shared_warp)[3]), + "r"(((unsigned*)B_shared_warp)[0]), + "r"(((unsigned*)B_shared_warp)[1]), + "r"(((int*)C_warp)[0]), + "r"(((int*)C_warp)[1]), + "r"(((int*)C_warp)[2]), + "r"(((int*)C_warp)[3])); +} + +template +__device__ __inline__ void global_to_share_one_stage_A( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask, + bool* preds) { + constexpr int total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int partial_global_iters = total_global_iters / SHARED_K_ITERS; + constexpr int cta_step_m_or_n = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int warp_step_m_or_n = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int threads_per_row = CTA_K / PACK_SIZE; + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K; + + if (mask) { +#pragma unroll + for (int _global_iter = 0; _global_iter < partial_global_iters; ++_global_iter) { + int global_iter = shared_iter_k * partial_global_iters + _global_iter; + + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols); + // *dst_ptr = *src_ptr; + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, preds[global_iter]); + } else { + if (preds[global_iter]) *(uint4*)dst_ptr = *src_ptr; + } + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_B( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int total_global_iters = (CTA_N * CTA_K) / 32 / CTA_SIZE; + constexpr int NUM_WARPS = CTA_SIZE / WARP_SIZE; + constexpr int warps_per_row = CTA_K / 32; + constexpr int cta_step_m_or_n = NUM_WARPS / warps_per_row; + constexpr int kSmemCol = CTA_K; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K * PACK_SIZE; + +#pragma unroll + for (int global_iter = 0; global_iter < total_global_iters; ++global_iter) { + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol * PACK_SIZE); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols * PACK_SIZE); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, mask); + } else { + if (mask) *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_zeros( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int threads_needed = CTA_N / PACK_SIZE / 1; + constexpr int threads_used = threads_needed < CTA_SIZE ? threads_needed : CTA_SIZE; + constexpr int total_global_iters = CTA_N / PACK_SIZE / threads_used; + constexpr int threads_per_row = CTA_N / PACK_SIZE; + constexpr int kSmemCol = CTA_N; + bool local_mask = mask & (threadIdx.y * WARP_SIZE + threadIdx.x < threads_used); + int g_idx = global_iter_k * CTA_K / G; + + void* dst_ptr = (void*)(dst + (threadIdx.x % threads_per_row) * PACK_SIZE); + uint4* src_ptr = (uint4*)(src + g_idx * global_ncols + cta_offset_n + (threadIdx.x % threads_per_row) * PACK_SIZE); + if (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, local_mask); + } else { + if (local_mask) { + *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void +share_to_reg_one_stage_A(int8_t* src, int8_t* dst, int warp_offset_m, int warp_offset_n, int k_0_1, int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int ld_col = (k_0_1 * INTRIN_K + (threadIdx.x / 16) * 16) / PACK_SIZE; + + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + int ld_row = warp_offset_m + shared_iter * INTRIN_M + (threadIdx.x % 16); + int ld_col_swizzled = ld_col ^ (ld_row / 2) & 3; + void* addr_ptr = (void*)(src + ld_row * kSmemCol + ld_col_swizzled * PACK_SIZE); + uint32_t addr = cast_smem_ptr_to_uint(addr_ptr); + ldmatrix_m8n8_x4_b16(dst, shared_iter, addr); + } +} + +template +__device__ __inline__ void share_to_reg_one_stage_B( + int8_t* src, + int8_t* dst, + int8_t* zeros, + int8_t* scales_i8, + int warp_offset_m, + int warp_offset_n, + int k_0_0, + int k_0_1, + int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_B; +#pragma unroll + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + uint4 loaded = + *((uint4*)(src) + warp_offset_n / 32 * kSmemCol + shared_iter * 32 / 32 * kSmemCol + k_0_1 * INTRIN_K + + threadIdx.x); + + auto ptr = (uint32_t*)dst + shared_iter * 8; + ptr[0] = loaded.x & 0x0F0F0F0F; + ptr[4] = (loaded.x & 0xF0F0F0F0) >> 4; + ptr[2] = loaded.y & 0x0F0F0F0F; + ptr[6] = (loaded.y & 0xF0F0F0F0) >> 4; + ptr[1] = loaded.z & 0x0F0F0F0F; + ptr[5] = (loaded.z & 0xF0F0F0F0) >> 4; + ptr[3] = loaded.w & 0x0F0F0F0F; + ptr[7] = (loaded.w & 0xF0F0F0F0) >> 4; + } +} + +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half2* __restrict__ w_szs, + half* __restrict__ a_ssums, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + constexpr int SPLITK = 1; + constexpr int NUM_WARPS_MN = CTA_M / WARP_M * CTA_N / WARP_N; + constexpr int NUM_WARPS = NUM_WARPS_MN * CTA_K / WARP_K; + constexpr int CTA_SIZE = NUM_WARPS * WARP_SIZE; + constexpr int CTA_SIZE_MN = NUM_WARPS_MN * WARP_SIZE; + constexpr int SLICES = CTA_K / WARP_K; + int num_blocks_n = (N + CTA_N - 1) / CTA_N; + int num_blocks_m = (M + CTA_M - 1) / CTA_M; + + int blockIdx_n = blockIdx.x; + int blockIdx_m = blockIdx.y; + const int log_tile = get_log_tile<8>((M + CTA_M - 1) / CTA_M); + const uint2 block_idx_mapping = get_block_idx_mapping(blockIdx_n, blockIdx_m, log_tile); + blockIdx_n = block_idx_mapping.x; + blockIdx_m = block_idx_mapping.y; + + int C_warp[CTA_M * CTA_N / CTA_SIZE_MN]; + constexpr int kSmemPadKA = CTA_K + SMEM_PAD_A; + constexpr int kSmemPadKB = CTA_K + SMEM_PAD_B; + constexpr int kSmemSizeAPerStage = CTA_M * kSmemPadKA; + constexpr int kSmemSizeBPerStage = CTA_N * kSmemPadKB / 2; + constexpr int kSmemSizeA = kSmemSizeAPerStage * STAGES; + constexpr int kSmemSizeB = kSmemSizeBPerStage * STAGES; + + constexpr int scales_load_interval = G >= CTA_K ? G / CTA_K : 1; + constexpr int scales_per_load = G < CTA_K ? CTA_K / G : 1; + constexpr int kSmemSizeScales = CTA_N * STAGES; + + extern __shared__ int8_t mem_shared[]; + int8_t* A_shared = mem_shared; + + int8_t* B_shared = mem_shared + kSmemSizeA; + int8_t* zeros_shared = mem_shared + kSmemSizeA + kSmemSizeB; + int8_t* scales_i8_shared = mem_shared + kSmemSizeA + kSmemSizeB + kSmemSizeScales; + + int8_t A_shared_warp_[2][WARP_M * WARP_K / WARP_SIZE]; + int8_t B_shared_warp_[2][WARP_N * WARP_K / WARP_SIZE]; + constexpr int A_total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int B_total_global_iters = (CTA_N * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int A_src_step_m = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_warp_step_m = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_threads_per_row = CTA_K / PACK_SIZE; + + constexpr int B_warps_per_row = CTA_K / 32; + constexpr int B_src_step_n = NUM_WARPS / B_warps_per_row; + + int cta_offset_m = blockIdx_m * CTA_M; + int cta_offset_n = blockIdx_n * CTA_N; + int warp_mn = threadIdx.y % NUM_WARPS_MN; + int slice_id = threadIdx.y / NUM_WARPS_MN; + int warp_offset_m = (warp_mn % (CTA_M / WARP_M)) * WARP_M; + int warp_offset_n = (warp_mn / (CTA_M / WARP_M)) * WARP_N; + int warp_offset_k = slice_id * WARP_K; + + for (int i = 0; i < CTA_M * CTA_N / CTA_SIZE_MN; i++) + C_warp[i] = 0; + + int gemm_iters = (K + CTA_K - 1) / CTA_K; + int k_0_0_ld = 0; + int k_0_0 = 0; + constexpr int prologue_stages = STAGES == 1 ? 1 : STAGES - 1; + int A_hoisted_row = threadIdx.y * A_warp_step_m + (threadIdx.x / A_threads_per_row); + int A_hoisted_col = (threadIdx.x % A_threads_per_row); + int A_hoisted_col_swizzled = A_hoisted_col ^ (A_hoisted_row / 2) & 3; + + int8_t* A_shared_hoisted = A_shared + A_hoisted_row * kSmemPadKA + A_hoisted_col_swizzled * PACK_SIZE; + int8_t* B_shared_hoisted = B_shared + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * kSmemPadKB * PACK_SIZE + threadIdx.x * PACK_SIZE; + int8_t* A_hoisted = A + cta_offset_m * K + A_hoisted_row * K + A_hoisted_col * PACK_SIZE; + int8_t* B_hoisted = B + cta_offset_n / 32 * K * PACK_SIZE + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * K * PACK_SIZE + threadIdx.x * PACK_SIZE; + + bool A_g2s_preds[A_total_global_iters]; +#pragma unroll + for (int i = 0; i < A_total_global_iters; i++) { + A_g2s_preds[i] = (cta_offset_m + A_hoisted_row + i * A_src_step_m) < M; + } + + int* C_shared = reinterpret_cast(mem_shared); + +#pragma unroll + for (k_0_0_ld = 0; k_0_0_ld < prologue_stages; ++k_0_0_ld) { + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + k_0_0_ld * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + true, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, B_shared_hoisted + k_0_0_ld * kSmemSizeBPerStage, K, cta_offset_m, cta_offset_n, k_0_0_ld, 0, true); + + if constexpr (STAGES > 1) __pipeline_commit(); + } + if constexpr (STAGES > 1) __pipeline_wait_prior(STAGES - 2); + __syncthreads(); + + share_to_reg_one_stage_A( + A_shared + warp_offset_k, A_shared_warp_[0], warp_offset_m, warp_offset_n, 0, WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared + warp_offset_k * PACK_SIZE, + B_shared_warp_[0], + zeros_shared, + scales_i8_shared, + warp_offset_m, + warp_offset_n, + 0, + 0, + WARP_N / 32); + constexpr int SHARED_K_ITERS = WARP_K / INTRIN_K; + + for (; k_0_0 < gemm_iters; ++k_0_0, ++k_0_0_ld) { + int ld_stage = k_0_0_ld % STAGES; + int compute_stage = k_0_0 % STAGES; + int8_t* A_shared_this_compute_stage; + int8_t* B_shared_this_compute_stage; + int8_t* zeros_shared_this_compute_stage; + int8_t* scales_i8_shared_this_compute_stage; + + for (int iter_k = 0; iter_k < SHARED_K_ITERS; ++iter_k) { + A_shared_this_compute_stage = A_shared + compute_stage * kSmemSizeAPerStage + warp_offset_k; + B_shared_this_compute_stage = B_shared + compute_stage * kSmemSizeBPerStage + warp_offset_k * PACK_SIZE; + zeros_shared_this_compute_stage = zeros_shared + (compute_stage)*CTA_N; + scales_i8_shared_this_compute_stage = scales_i8_shared + (compute_stage)*CTA_N; + + share_to_reg_one_stage_A( + A_shared_this_compute_stage, + A_shared_warp_[(iter_k + 1) % 2], + warp_offset_m, + warp_offset_n, + (iter_k + 1) % SHARED_K_ITERS, + WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared_this_compute_stage, + B_shared_warp_[(iter_k + 1) % 2], + zeros_shared_this_compute_stage, + scales_i8_shared_this_compute_stage, + warp_offset_m, + warp_offset_n, + k_0_0 + (iter_k == SHARED_K_ITERS - 1), + (iter_k + 1) % SHARED_K_ITERS, + WARP_N / 32); + int8_t* A_shared_warp = A_shared_warp_[iter_k % 2]; + int8_t* B_shared_warp = B_shared_warp_[iter_k % 2]; + + for (int j_0_4 = 0; j_0_4 < WARP_N / INTRIN_N; ++j_0_4) { + for (int i_0_3 = 0; i_0_3 < WARP_M / INTRIN_M; ++i_0_3) { + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16)); + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8 + 4), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16 + 8)); + } + } + + if (iter_k < SHARED_K_ITERS - 1) { + if constexpr (STAGES == 1) __syncthreads(); + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + } + + if (iter_k == SHARED_K_ITERS - 2) { + if constexpr (STAGES == 1 && SHARED_K_ITERS > 2) { + __syncthreads(); + } + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters); + if constexpr (STAGES > 1) { + __pipeline_commit(); + __pipeline_wait_prior(STAGES - 2); + } + compute_stage = (k_0_0 + 1) % STAGES; + __syncthreads(); + } + } + } + + __pipeline_commit(); + __pipeline_wait_prior(0); + __syncthreads(); + + if constexpr (SLICES > 1) { +#pragma unroll + for (int z = 0; z < SLICES; ++z) { + if (slice_id == z) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + if (z > 0) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] += C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + } + C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2] = C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id]; + }; + } + } + } + __syncthreads(); + } + if (slice_id == 0) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] = C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + }; + } + } + } + } + + int row_wb_thd = cta_offset_m + warp_offset_m + (threadIdx.x / 4); + int col_wb_thd = cta_offset_n + warp_offset_n + (threadIdx.x % 4) * 2; + if (slice_id == 0) { + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { + int row_wb_1 = row_wb_thd + ax0_0_1 * OP_M; + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { + int col_wb_1 = col_wb_thd + ax1_0_1 * 16; + int* C_warp_local = C_warp + ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8; + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; local_id += 2) { + int row_wb = row_wb_1 + (local_id % 4) / 2 * 8; + if (row_wb < M) { + int col_wb = col_wb_1 + (local_id / 4) * 8 + (local_id % 2); + float2 wscale = __half22float2(*(wscales + col_wb / 2)); + float2 w_sz = __half22float2(*(w_szs + col_wb / 2)); + float ascale = __half2float(ascales[row_wb]); + float a_ssum = __half2float(a_ssums[row_wb]); + float2 psums = + make_float2(__int2float_rn(C_warp_local[local_id]), __int2float_rn(C_warp_local[local_id + 1])); + psums.x = psums.x * wscale.x * ascale - w_sz.x * a_ssum; + psums.y = psums.y * wscale.y * ascale - w_sz.y * a_ssum; + *reinterpret_cast(C + row_wb * N + col_wb) = __float22half2_rn(psums); + } + }; + } + } + } +} +#else +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half2* __restrict__ w_szs, + half* __restrict__ a_ssums, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + // Not implemented for SM < 800 + assert(false); + return; +} +#endif + +void qserve_w4a8_per_chn_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + const torch::Tensor& _w_szs, + const torch::Tensor& _a_ssums, + torch::Tensor& _out_feats) { + // Check input tensor + TORCH_CHECK(_in_feats.is_cuda(), "_in_feats must be a CUDA tensor"); + TORCH_CHECK(_in_feats.dim() == 2, "_in_feats must be a 2D tensor"); + TORCH_CHECK(_in_feats.is_contiguous(), "_in_feats must be contiguous"); + TORCH_CHECK(_in_feats.scalar_type() == torch::kInt8, "_in_feats must be int8"); + // Check kernel tensor + TORCH_CHECK(_kernel.is_cuda(), "_kernel must be a CUDA tensor"); + TORCH_CHECK(_kernel.dim() == 2, "_kernel must be a 2D tensor"); + TORCH_CHECK(_kernel.is_contiguous(), "_kernel must be contiguous"); + TORCH_CHECK(_kernel.scalar_type() == torch::kInt8, "_kernel must be int8"); + // Check output tensor + TORCH_CHECK(_out_feats.is_cuda(), "_out_feats must be a CUDA tensor"); + TORCH_CHECK(_out_feats.is_contiguous(), "_out_feats must be contiguous"); + TORCH_CHECK(_out_feats.scalar_type() == torch::kHalf, "_out_feats must be half"); + + int num_in_feats = _in_feats.size(0); + int num_in_channels = _in_feats.size(1); + int num_out_feats = _out_feats.size(-2); + int num_out_channels = _out_feats.size(-1); + + // Check matmul shape + TORCH_CHECK(num_out_channels == _kernel.size(0), "num_out_channels must be equal to _kernel.size(0)"); + TORCH_CHECK(num_in_feats == num_out_feats, "num_in_feats must be equal to num_out_feats"); + + // Check _ascales + TORCH_CHECK(_ascales.is_cuda(), "_ascales must be a CUDA tensor"); + TORCH_CHECK(_ascales.is_contiguous(), "_ascales must be contiguous"); + TORCH_CHECK(_ascales.scalar_type() == torch::kHalf, "_ascales must be half"); + TORCH_CHECK(_ascales.numel() == num_in_feats, "_ascales must have num_in_feats elements"); + + // Check _wscales + TORCH_CHECK(_wscales.is_cuda(), "_wscales must be a CUDA tensor"); + TORCH_CHECK(_wscales.is_contiguous(), "_wscales must be contiguous"); + TORCH_CHECK(_wscales.scalar_type() == torch::kHalf, "_wscales must be half"); + TORCH_CHECK(_wscales.numel() == num_out_channels, "_wscales must have num_out_channels elements"); + + // Check _w_szs + TORCH_CHECK(_w_szs.is_cuda(), "_w_szs must be a CUDA tensor"); + TORCH_CHECK(_w_szs.is_contiguous(), "_w_szs must be contiguous"); + TORCH_CHECK(_w_szs.scalar_type() == torch::kHalf, "_w_szs must be half"); + TORCH_CHECK(_w_szs.numel() == num_out_channels, "_w_szs must have num_out_channels elements"); + + // Check _a_ssums + TORCH_CHECK(_a_ssums.is_cuda(), "_a_ssums must be a CUDA tensor"); + TORCH_CHECK(_a_ssums.is_contiguous(), "_a_ssums must be contiguous"); + TORCH_CHECK(_a_ssums.scalar_type() == torch::kHalf, "_a_ssums must be half"); + TORCH_CHECK(_a_ssums.numel() == num_in_feats, "_a_ssums must have num_in_feats elements"); + + auto in_feats = reinterpret_cast(_in_feats.data_ptr()); + auto kernel = reinterpret_cast(_kernel.data_ptr()); + auto w_szs = reinterpret_cast(_w_szs.data_ptr()); + auto a_ssums = reinterpret_cast(_a_ssums.data_ptr()); + auto wscales = reinterpret_cast(_wscales.data_ptr()); + auto ascales = reinterpret_cast(_ascales.data_ptr()); + auto out_feats = reinterpret_cast(_out_feats.data_ptr()); + auto stream = at::cuda::getCurrentCUDAStream(_in_feats.get_device()); + + auto sm_version = getSMVersion(); + if (sm_version >= 80) { + constexpr int G = 128; + + if (num_out_feats > 256) { + constexpr int CTA_M = 128; + constexpr int CTA_N = 128; + constexpr int CTA_K = 64; + constexpr int WARP_M = 64; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } else if (num_out_feats >= 128) { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else { + constexpr int CTA_M = 32; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented qserve_w4a8_per_chn_gemm for current compute capability: ", sm_version); + } + return; +} diff --git a/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu b/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu new file mode 100644 index 000000000..a99a203e8 --- /dev/null +++ b/sgl-kernel/csrc/gemm/qserve_w4a8_per_group_gemm.cu @@ -0,0 +1,795 @@ +// Implemented by Haotian Tang and Shang Yang. +// @article{lin2024qserve, +// title={QServe: W4A8KV4 Quantization and System Co-design for Efficient LLM Serving}, +// author={Lin*, Yujun and Tang*, Haotian and Yang*, Shang and Zhang, Zhekai and Xiao, Guangxuan and Gan, Chuang and +// Han, Song}, journal={arXiv preprint arXiv:2405.04532}, year={2024} +// } +// @article{yang2025lserve, +// title={LServe: Efficient Long-sequence LLM Serving with Unified Sparse Attention}, +// author={Yang*, Shang and Guo*, Junxian and Tang, Haotian and Hu, Qinghao and Xiao, Guangxuan and Tang, Jiaming and +// Lin, Yujun and Liu, Zhijian and Lu, Yao and Han, Song}, year={2025} +// } + +// Adapted from https://github.com/mit-han-lab/omniserve/blob/main/kernels/csrc/qgemm/w4a8_per_group/gemm_cuda.cu + +#include +#include +#include +#include + +#include "utils.h" + +#define OP_M 16 +#define OP_N 8 +#define OP_K 32 +#define INTRIN_M 16 +#define INTRIN_N 16 +#define INTRIN_K 32 +#define WARP_SIZE 32 +#define SMEM_PAD_A 0 +#define SMEM_PAD_B 0 +#define PACK_SIZE 16 +#if (__CUDACC_VER_MAJOR__ >= 11) && (__CUDACC_VER_MINOR__ >= 4) +#define L2_CACHEHINT(size) ".L2::" #size "B" +#else +#define L2_CACHEHINT(size) +#endif + +#define KERNEL_LAUNCH_CODE \ + constexpr int NUM_WARPS = (CTA_M / WARP_M) * (CTA_N / WARP_N) * (CTA_K / WARP_K); \ + constexpr int SCALES_SMEM_SIZE = (G >= CTA_K) ? (CTA_N * STAGES * 2) : (CTA_N * (CTA_K / G) * STAGES * 2); \ + constexpr int kSmemByteSize = \ + ((CTA_M * (CTA_K + SMEM_PAD_A) + CTA_N * (CTA_K + SMEM_PAD_B) / 2) * STAGES + SCALES_SMEM_SIZE) * \ + sizeof(int8_t); \ + if (kSmemByteSize >= 99 * 1024) { \ + printf( \ + "This kernel requires %d Bytes of shared memory, which exceeds " \ + "device limit.\n", \ + kSmemByteSize); \ + return; \ + } \ + int num_blocks_m = (num_out_feats + CTA_M - 1) / CTA_M; \ + int num_blocks_n = num_out_channels / CTA_N / 1; \ + const int log_tile = get_log_tile<8>((num_out_feats + CTA_M - 1) / CTA_M); \ + const int tile_shift = 1 << log_tile; \ + dim3 num_blocks(num_blocks_n* tile_shift, (num_blocks_m + tile_shift - 1) / tile_shift); \ + dim3 threads_per_block(WARP_SIZE, NUM_WARPS); \ + auto kernel_func = dense_kernel0; \ + cudaFuncSetAttribute(kernel_func, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemByteSize); \ + kernel_func<<>>( \ + in_feats, \ + kernel, \ + zeros, \ + scales_i8, \ + wscales, \ + ascales, \ + out_feats, \ + num_in_feats, \ + num_out_channels, \ + num_in_channels); + +template +__inline__ __host__ __device__ int get_log_tile(int n) { + if (N >= 8 && n >= 6) + return 3; + else if (N >= 4 && n >= 3) + return 2; + else if (N >= 2 && n >= 2) + return 1; + else + return 0; +} + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 +__inline__ __device__ uint2 get_block_idx_mapping(int blockIdx_x, int blockIdx_y, int log_tile) { + return make_uint2((blockIdx_x >> log_tile), (blockIdx_y << log_tile) + ((blockIdx_x) & ((1 << (log_tile)) - 1))); +} + +__inline__ __device__ uint32_t cast_smem_ptr_to_uint(void const* const ptr) { + uint32_t smem_int_ptr; + + asm("{.reg .u64 smem_ptr; cvta.to.shared.u64 smem_ptr, %1; cvt.u32.u64 %0, " + "smem_ptr; }\n" + : "=r"(smem_int_ptr) + : "l"(ptr)); + + return smem_int_ptr; +} + +__inline__ __device__ void ldmatrix_m8n8_x4_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +__inline__ __device__ void ldmatrix_m8n8_x4_trans_b16(int8_t* shared_warp, int ax0_0, uint32_t addr) { + __asm__ __volatile__( + "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16" + "{%0, %1, %2, %3}, [%4];" + : "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[0]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[1]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[2]), + "=r"(((unsigned*)(shared_warp + (ax0_0 * 16)))[3]) + : "r"(addr)); +} + +// function from lmdeploy +__inline__ __device__ void cp_async_cg_A(uint32_t smem_int_ptr, const uint4* __restrict__ src, bool mask) { + const int cp_size = 16; + asm volatile("{" + " .reg .pred p;" + " setp.ne.b32 p, %0, 0;" + " @p cp.async.cg.shared.global" L2_CACHEHINT(128) " [%1], [%2], %3;" + "}" ::"r"((int)mask), + "r"(smem_int_ptr), + "l"(src), + "n"(cp_size)); +} + +__device__ __inline__ void mma_m16n8k32(void* C_warp, void* A_shared_warp, void* B_shared_warp) { + __asm__ __volatile__( + "mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32" + "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};" + : "=r"(((int*)C_warp)[0]), "=r"(((int*)C_warp)[1]), "=r"(((int*)C_warp)[2]), "=r"(((int*)C_warp)[3]) + : "r"(((unsigned*)A_shared_warp)[0]), + "r"(((unsigned*)A_shared_warp)[1]), + "r"(((unsigned*)A_shared_warp)[2]), + "r"(((unsigned*)A_shared_warp)[3]), + "r"(((unsigned*)B_shared_warp)[0]), + "r"(((unsigned*)B_shared_warp)[1]), + "r"(((int*)C_warp)[0]), + "r"(((int*)C_warp)[1]), + "r"(((int*)C_warp)[2]), + "r"(((int*)C_warp)[3])); +} + +template +__device__ __inline__ void global_to_share_one_stage_A( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask, + bool* preds) { + constexpr int total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int partial_global_iters = total_global_iters / SHARED_K_ITERS; + constexpr int cta_step_m_or_n = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int warp_step_m_or_n = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int threads_per_row = CTA_K / PACK_SIZE; + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K; + + if (mask) { +#pragma unroll + for (int _global_iter = 0; _global_iter < partial_global_iters; ++_global_iter) { + int global_iter = shared_iter_k * partial_global_iters + _global_iter; + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, preds[global_iter]); + } else { + if (preds[global_iter]) *(uint4*)dst_ptr = *src_ptr; + } + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_B( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int total_global_iters = (CTA_N * CTA_K) / 32 / CTA_SIZE; + constexpr int NUM_WARPS = CTA_SIZE / WARP_SIZE; + constexpr int warps_per_row = CTA_K / 32; + constexpr int cta_step_m_or_n = NUM_WARPS / warps_per_row; + constexpr int kSmemCol = CTA_K; + int8_t* dst_hoisted = dst; + int8_t* src_hoisted = src + global_iter_k * CTA_K * PACK_SIZE; + +#pragma unroll + for (int global_iter = 0; global_iter < total_global_iters; ++global_iter) { + void* dst_ptr = (void*)(dst_hoisted + global_iter * cta_step_m_or_n * kSmemCol * PACK_SIZE); + uint4* src_ptr = (uint4*)(src_hoisted + global_iter * cta_step_m_or_n * global_ncols * PACK_SIZE); + if constexpr (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, mask); + } else { + if (mask) *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void global_to_share_one_stage_zeros( + int8_t* src, + int8_t* dst, + int global_ncols, + int cta_offset_m, + int cta_offset_n, + int global_iter_k, + int shared_iter_k, + bool mask) { + constexpr int threads_needed = CTA_N / PACK_SIZE / 1; + constexpr int threads_used = threads_needed < CTA_SIZE ? threads_needed : CTA_SIZE; + constexpr int total_global_iters = CTA_N / PACK_SIZE / threads_used; + constexpr int threads_per_row = CTA_N / PACK_SIZE; + constexpr int kSmemCol = CTA_N; + bool local_mask = mask & (threadIdx.y * WARP_SIZE + threadIdx.x < threads_used); + int g_idx = global_iter_k * CTA_K / G; + + void* dst_ptr = (void*)(dst + (threadIdx.x % threads_per_row) * PACK_SIZE); + uint4* src_ptr = (uint4*)(src + g_idx * global_ncols + cta_offset_n + (threadIdx.x % threads_per_row) * PACK_SIZE); + if (STAGES > 1) { + uint32_t addr = cast_smem_ptr_to_uint(dst_ptr); + cp_async_cg_A(addr, src_ptr, local_mask); + } else { + if (local_mask) { + *(uint4*)dst_ptr = *src_ptr; + } + } +} + +template +__device__ __inline__ void +share_to_reg_one_stage_A(int8_t* src, int8_t* dst, int warp_offset_m, int warp_offset_n, int k_0_1, int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_A; + int ld_col = (k_0_1 * INTRIN_K + (threadIdx.x / 16) * 16) / PACK_SIZE; + + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + int ld_row = warp_offset_m + shared_iter * INTRIN_M + (threadIdx.x % 16); + int ld_col_swizzled = ld_col ^ (ld_row / 2) & 3; + void* addr_ptr = (void*)(src + ld_row * kSmemCol + ld_col_swizzled * PACK_SIZE); + uint32_t addr = cast_smem_ptr_to_uint(addr_ptr); + ldmatrix_m8n8_x4_b16(dst, shared_iter, addr); + } +} + +template +__device__ __inline__ void share_to_reg_one_stage_B( + int8_t* src, + int8_t* dst, + int8_t* zeros, + int8_t* scales_i8, + int warp_offset_m, + int warp_offset_n, + int k_0_0, + int k_0_1, + int shared_iters) { + constexpr int kSmemCol = CTA_K + SMEM_PAD_B; +#pragma unroll + for (int shared_iter = 0; shared_iter < shared_iters; ++shared_iter) { + uint4 loaded = + *((uint4*)(src) + warp_offset_n / 32 * kSmemCol + shared_iter * 32 / 32 * kSmemCol + k_0_1 * INTRIN_K + + threadIdx.x); + uint32_t loaded_0 = loaded.x & 0x0F0F0F0F; + uint32_t loaded_4 = (loaded.x & 0xF0F0F0F0) >> 4; + uint32_t loaded_2 = loaded.y & 0x0F0F0F0F; + uint32_t loaded_6 = (loaded.y & 0xF0F0F0F0) >> 4; + uint32_t loaded_1 = loaded.z & 0x0F0F0F0F; + uint32_t loaded_5 = (loaded.z & 0xF0F0F0F0) >> 4; + uint32_t loaded_3 = loaded.w & 0x0F0F0F0F; + uint32_t loaded_7 = (loaded.w & 0xF0F0F0F0) >> 4; + + auto ptr = (uint32_t*)dst + shared_iter * 8; + int scales_zeros_offset = warp_offset_n + (threadIdx.x / 4) * 4 + shared_iter * 32; + uint32_t packed_scales = *reinterpret_cast(scales_i8 + scales_zeros_offset); + uint32_t packed_zeros = *reinterpret_cast(zeros + scales_zeros_offset); + + uint32_t scale_0 = packed_scales & 0xFF; + uint32_t zero_point_0 = __byte_perm(packed_zeros, 0, 0x00000000); + uint32_t ptr_0 = loaded_0 * scale_0; + uint32_t ptr_1 = loaded_1 * scale_0; + ptr[0] = __vadd4(ptr_0, zero_point_0); + ptr[1] = __vadd4(ptr_1, zero_point_0); + + uint32_t scale_1 = (packed_scales & 0xFF00) >> 8; + uint32_t zero_point_1 = __byte_perm(packed_zeros, 0, 0x00001111); + uint32_t ptr_2 = loaded_2 * scale_1; + uint32_t ptr_3 = loaded_3 * scale_1; + ptr[2] = __vadd4(ptr_2, zero_point_1); + ptr[3] = __vadd4(ptr_3, zero_point_1); + + uint32_t scale_2 = (packed_scales & 0xFF0000) >> 16; + uint32_t zero_point_2 = __byte_perm(packed_zeros, 0, 0x00002222); + uint32_t ptr_4 = loaded_4 * scale_2; + uint32_t ptr_5 = loaded_5 * scale_2; + ptr[4] = __vadd4(ptr_4, zero_point_2); + ptr[5] = __vadd4(ptr_5, zero_point_2); + + uint32_t scale_3 = (packed_scales & 0xFF000000) >> 24; + uint32_t zero_point_3 = __byte_perm(packed_zeros, 0, 0x00003333); + uint32_t ptr_6 = loaded_6 * scale_3; + uint32_t ptr_7 = loaded_7 * scale_3; + ptr[6] = __vadd4(ptr_6, zero_point_3); + ptr[7] = __vadd4(ptr_7, zero_point_3); + } +} + +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + int8_t* __restrict__ zeros, + int8_t* __restrict__ scales_i8, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + constexpr int SPLITK = 1; + constexpr int NUM_WARPS_MN = CTA_M / WARP_M * CTA_N / WARP_N; + constexpr int NUM_WARPS = NUM_WARPS_MN * CTA_K / WARP_K; + constexpr int CTA_SIZE = NUM_WARPS * WARP_SIZE; + constexpr int CTA_SIZE_MN = NUM_WARPS_MN * WARP_SIZE; + constexpr int SLICES = CTA_K / WARP_K; + int num_blocks_n = (N + CTA_N - 1) / CTA_N; + int num_blocks_m = (M + CTA_M - 1) / CTA_M; + + int blockIdx_n = blockIdx.x; + int blockIdx_m = blockIdx.y; + const int log_tile = get_log_tile<8>((M + CTA_M - 1) / CTA_M); + const uint2 block_idx_mapping = get_block_idx_mapping(blockIdx_n, blockIdx_m, log_tile); + blockIdx_n = block_idx_mapping.x; + blockIdx_m = block_idx_mapping.y; + + int C_warp[CTA_M * CTA_N / CTA_SIZE_MN]; + constexpr int kSmemPadKA = CTA_K + SMEM_PAD_A; + constexpr int kSmemPadKB = CTA_K + SMEM_PAD_B; + constexpr int kSmemSizeAPerStage = CTA_M * kSmemPadKA; + constexpr int kSmemSizeBPerStage = CTA_N * kSmemPadKB / 2; + constexpr int kSmemSizeA = kSmemSizeAPerStage * STAGES; + constexpr int kSmemSizeB = kSmemSizeBPerStage * STAGES; + + constexpr int scales_load_interval = G >= CTA_K ? G / CTA_K : 1; + constexpr int scales_per_load = G < CTA_K ? CTA_K / G : 1; + constexpr int kSmemSizeScales = CTA_N * STAGES; + + extern __shared__ int8_t mem_shared[]; + int8_t* A_shared = mem_shared; + + int8_t* B_shared = mem_shared + kSmemSizeA; + int8_t* zeros_shared = mem_shared + kSmemSizeA + kSmemSizeB; + int8_t* scales_i8_shared = mem_shared + kSmemSizeA + kSmemSizeB + kSmemSizeScales; + + int8_t A_shared_warp_[2][WARP_M * WARP_K / WARP_SIZE]; + int8_t B_shared_warp_[2][WARP_N * WARP_K / WARP_SIZE]; + constexpr int A_total_global_iters = (CTA_M * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int B_total_global_iters = (CTA_N * CTA_K) / PACK_SIZE / CTA_SIZE; + constexpr int A_src_step_m = (CTA_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_warp_step_m = (WARP_SIZE * PACK_SIZE) / CTA_K; + constexpr int A_threads_per_row = CTA_K / PACK_SIZE; + + constexpr int B_warps_per_row = CTA_K / 32; + constexpr int B_src_step_n = NUM_WARPS / B_warps_per_row; + + int cta_offset_m = blockIdx_m * CTA_M; + int cta_offset_n = blockIdx_n * CTA_N; + int warp_mn = threadIdx.y % NUM_WARPS_MN; + int slice_id = threadIdx.y / NUM_WARPS_MN; + int warp_offset_m = (warp_mn % (CTA_M / WARP_M)) * WARP_M; + int warp_offset_n = (warp_mn / (CTA_M / WARP_M)) * WARP_N; + int warp_offset_k = slice_id * WARP_K; + + for (int i = 0; i < CTA_M * CTA_N / CTA_SIZE_MN; i++) + C_warp[i] = 0; + + int gemm_iters = (K + CTA_K - 1) / CTA_K; + + int k_0_0_ld = 0; + int k_0_0 = 0; + constexpr int prologue_stages = STAGES == 1 ? 1 : STAGES - 1; + int A_hoisted_row = threadIdx.y * A_warp_step_m + (threadIdx.x / A_threads_per_row); + int A_hoisted_col = (threadIdx.x % A_threads_per_row); + int A_hoisted_col_swizzled = A_hoisted_col ^ (A_hoisted_row / 2) & 3; + + int8_t* A_shared_hoisted = A_shared + A_hoisted_row * kSmemPadKA + A_hoisted_col_swizzled * PACK_SIZE; + int8_t* B_shared_hoisted = B_shared + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * kSmemPadKB * PACK_SIZE + threadIdx.x * PACK_SIZE; + int8_t* A_hoisted = A + cta_offset_m * K + A_hoisted_row * K + A_hoisted_col * PACK_SIZE; + int8_t* B_hoisted = B + cta_offset_n / 32 * K * PACK_SIZE + (threadIdx.y % B_warps_per_row) * 32 * PACK_SIZE + + (threadIdx.y / B_warps_per_row) * K * PACK_SIZE + threadIdx.x * PACK_SIZE; + + bool A_g2s_preds[A_total_global_iters]; +#pragma unroll + for (int i = 0; i < A_total_global_iters; i++) { + A_g2s_preds[i] = (cta_offset_m + A_hoisted_row + i * A_src_step_m) < M; + } + + int* C_shared = reinterpret_cast(mem_shared); + +#pragma unroll + for (k_0_0_ld = 0; k_0_0_ld < prologue_stages; ++k_0_0_ld) { + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + k_0_0_ld * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + true, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, B_shared_hoisted + k_0_0_ld * kSmemSizeBPerStage, K, cta_offset_m, cta_offset_n, k_0_0_ld, 0, true); + global_to_share_one_stage_zeros( + zeros, zeros_shared + (k_0_0_ld)*CTA_N, N, cta_offset_m, cta_offset_n, k_0_0_ld, 0, k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + scales_i8, + scales_i8_shared + (k_0_0_ld)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + 0, + k_0_0_ld < gemm_iters); + + if constexpr (STAGES > 1) __pipeline_commit(); + } + if constexpr (STAGES > 1) __pipeline_wait_prior(STAGES - 2); + __syncthreads(); + + share_to_reg_one_stage_A( + A_shared + warp_offset_k, A_shared_warp_[0], warp_offset_m, warp_offset_n, 0, WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared + warp_offset_k * PACK_SIZE, + B_shared_warp_[0], + zeros_shared, + scales_i8_shared, + warp_offset_m, + warp_offset_n, + 0, + 0, + WARP_N / 32); + constexpr int SHARED_K_ITERS = WARP_K / INTRIN_K; + + for (; k_0_0 < gemm_iters; ++k_0_0, ++k_0_0_ld) { + int ld_stage = k_0_0_ld % STAGES; + int compute_stage = k_0_0 % STAGES; + int8_t* A_shared_this_compute_stage; + int8_t* B_shared_this_compute_stage; + int8_t* zeros_shared_this_compute_stage; + int8_t* scales_i8_shared_this_compute_stage; + + for (int iter_k = 0; iter_k < SHARED_K_ITERS; ++iter_k) { + A_shared_this_compute_stage = A_shared + compute_stage * kSmemSizeAPerStage + warp_offset_k; + B_shared_this_compute_stage = B_shared + compute_stage * kSmemSizeBPerStage + warp_offset_k * PACK_SIZE; + zeros_shared_this_compute_stage = zeros_shared + (compute_stage)*CTA_N; + scales_i8_shared_this_compute_stage = scales_i8_shared + (compute_stage)*CTA_N; + + share_to_reg_one_stage_A( + A_shared_this_compute_stage, + A_shared_warp_[(iter_k + 1) % 2], + warp_offset_m, + warp_offset_n, + (iter_k + 1) % SHARED_K_ITERS, + WARP_M / INTRIN_M); + share_to_reg_one_stage_B( + B_shared_this_compute_stage, + B_shared_warp_[(iter_k + 1) % 2], + zeros_shared_this_compute_stage, + scales_i8_shared_this_compute_stage, + warp_offset_m, + warp_offset_n, + k_0_0 + (iter_k == SHARED_K_ITERS - 1), + (iter_k + 1) % SHARED_K_ITERS, + WARP_N / 32); + int8_t* A_shared_warp = A_shared_warp_[iter_k % 2]; + int8_t* B_shared_warp = B_shared_warp_[iter_k % 2]; + + for (int j_0_4 = 0; j_0_4 < WARP_N / INTRIN_N; ++j_0_4) { + for (int i_0_3 = 0; i_0_3 < WARP_M / INTRIN_M; ++i_0_3) { + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16)); + mma_m16n8k32( + (void*)(C_warp + i_0_3 * WARP_N / INTRIN_N * 8 + j_0_4 * 8 + 4), + (void*)(A_shared_warp + i_0_3 * 16), + (void*)(B_shared_warp + j_0_4 * 16 + 8)); + } + } + + if (iter_k < SHARED_K_ITERS - 1) { + if constexpr (STAGES == 1) __syncthreads(); + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + } + + if (iter_k == SHARED_K_ITERS - 2) { + if constexpr (STAGES == 1 && SHARED_K_ITERS > 2) { + __syncthreads(); + } + global_to_share_one_stage_A( + A_hoisted, + A_shared_hoisted + ld_stage * kSmemSizeAPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters, + A_g2s_preds); + global_to_share_one_stage_B( + B_hoisted, + B_shared_hoisted + ld_stage * kSmemSizeBPerStage, + K, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k + 1, + k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + zeros, + zeros_shared + (ld_stage)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + global_to_share_one_stage_zeros( + scales_i8, + scales_i8_shared + (ld_stage)*CTA_N, + N, + cta_offset_m, + cta_offset_n, + k_0_0_ld, + iter_k, + k_0_0_ld < gemm_iters); + if constexpr (STAGES > 1) { + __pipeline_commit(); + __pipeline_wait_prior(STAGES - 2); + } + compute_stage = (k_0_0 + 1) % STAGES; + __syncthreads(); + } + } + } + __pipeline_commit(); + __pipeline_wait_prior(0); + __syncthreads(); + + if constexpr (SLICES > 1) { +#pragma unroll + for (int z = 0; z < SLICES; ++z) { + if (slice_id == z) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + if (z > 0) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] += C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + } + C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2] = C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id]; + }; + } + } + } + __syncthreads(); + } + if (slice_id == 0) { +#pragma unroll + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { +#pragma unroll + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { +#pragma unroll + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; ++local_id) { + C_warp[ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8 + local_id] = C_shared + [warp_offset_m * CTA_N + ax0_0_1 * OP_M * CTA_N + warp_offset_n + ax1_0_1 * 16 + + ((local_id % 4) / 2 * 8 + (threadIdx.x / 4)) * CTA_N + (local_id / 4) * 8 + (local_id % 2) + + (threadIdx.x % 4) * 2]; + }; + } + } + } + } + + int row_wb_thd = cta_offset_m + warp_offset_m + (threadIdx.x / 4); + int col_wb_thd = cta_offset_n + warp_offset_n + (threadIdx.x % 4) * 2; + if (slice_id == 0) { + for (int ax0_0_1 = 0; ax0_0_1 < WARP_M / INTRIN_M; ++ax0_0_1) { + int row_wb_1 = row_wb_thd + ax0_0_1 * OP_M; + for (int ax1_0_1 = 0; ax1_0_1 < WARP_N / INTRIN_N; ++ax1_0_1) { + int col_wb_1 = col_wb_thd + ax1_0_1 * 16; + int* C_warp_local = C_warp + ax0_0_1 * WARP_N / INTRIN_N * 8 + ax1_0_1 * 8; + for (int local_id = 0; local_id < OP_M * 16 / WARP_SIZE; local_id += 2) { + int row_wb = row_wb_1 + (local_id % 4) / 2 * 8; + if (row_wb < M) { + int col_wb = col_wb_1 + (local_id / 4) * 8 + (local_id % 2); + float2 wscale = __half22float2(*(wscales + col_wb / 2)); + float ascale = __half2float(ascales[row_wb]); + float2 psums = + make_float2(__int2float_rn(C_warp_local[local_id]), __int2float_rn(C_warp_local[local_id + 1])); + psums.x *= wscale.x * ascale; + psums.y *= wscale.y * ascale; + *reinterpret_cast(C + row_wb * N + col_wb) = __float22half2_rn(psums); + } + }; + } + } + } +} +#else +template +__global__ void dense_kernel0( + int8_t* __restrict__ A, + int8_t* __restrict__ B, + int8_t* __restrict__ zeros, + int8_t* __restrict__ scales_i8, + half2* __restrict__ wscales, + half* __restrict__ ascales, + half* __restrict__ C, + int M, + int64_t N, + int64_t K) { + // Not implemented for SM < 800 + assert(false); + return; +} +#endif + +void qserve_w4a8_per_group_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _zeros, + const torch::Tensor& _scales_i8, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + torch::Tensor& _out_feats) { + // Check input tensor + TORCH_CHECK(_in_feats.is_cuda(), "_in_feats must be a CUDA tensor"); + TORCH_CHECK(_in_feats.dim() == 2, "_in_feats must be a 2D tensor"); + TORCH_CHECK(_in_feats.is_contiguous(), "_in_feats must be contiguous"); + TORCH_CHECK(_in_feats.scalar_type() == torch::kInt8, "_in_feats must be int8"); + // Check kernel tensor + TORCH_CHECK(_kernel.is_cuda(), "_kernel must be a CUDA tensor"); + TORCH_CHECK(_kernel.dim() == 2, "_kernel must be a 2D tensor"); + TORCH_CHECK(_kernel.is_contiguous(), "_kernel must be contiguous"); + TORCH_CHECK(_kernel.scalar_type() == torch::kInt8, "_kernel must be int8"); + // Check output tensor + TORCH_CHECK(_out_feats.is_cuda(), "_out_feats must be a CUDA tensor"); + TORCH_CHECK(_out_feats.is_contiguous(), "_out_feats must be contiguous"); + TORCH_CHECK(_out_feats.scalar_type() == torch::kHalf, "_out_feats must be half"); + + int num_in_feats = _in_feats.size(0); + int num_in_channels = _in_feats.size(1); + int num_out_feats = _out_feats.size(-2); + int num_out_channels = _out_feats.size(-1); + + // Check matmul shape + TORCH_CHECK(num_out_channels == _kernel.size(0), "num_out_channels must be equal to _kernel.size(0)"); + TORCH_CHECK(num_in_feats == num_out_feats, "num_in_feats must be equal to num_out_feats"); + + // Check _ascales + TORCH_CHECK(_ascales.is_cuda(), "_ascales must be a CUDA tensor"); + TORCH_CHECK(_ascales.is_contiguous(), "_ascales must be contiguous"); + TORCH_CHECK(_ascales.scalar_type() == torch::kHalf, "_ascales must be half"); + TORCH_CHECK(_ascales.numel() == num_in_feats, "_ascales must have num_in_feats elements"); + + // Check _wscales + TORCH_CHECK(_wscales.is_cuda(), "_wscales must be a CUDA tensor"); + TORCH_CHECK(_wscales.is_contiguous(), "_wscales must be contiguous"); + TORCH_CHECK(_wscales.scalar_type() == torch::kHalf, "_wscales must be half"); + TORCH_CHECK(_wscales.numel() == num_out_channels, "_wscales must have num_out_channels elements"); + + // Check _scales_i8 + TORCH_CHECK(_scales_i8.is_cuda(), "_scales_i8 must be a CUDA tensor"); + TORCH_CHECK(_scales_i8.dim() == 2, "_scales_i8 must be a 2D tensor"); + TORCH_CHECK(_scales_i8.is_contiguous(), "_scales_i8 must be contiguous"); + TORCH_CHECK(_scales_i8.scalar_type() == torch::kInt8, "_scales_i8 must be int8"); + TORCH_CHECK(num_in_channels % _scales_i8.size(0) == 0, "num_in_channels must be divisible by _scales_i8.size(0)"); + TORCH_CHECK(num_out_channels == _scales_i8.size(1), "num_out_channels must be equal to _scales_i8.size(1)"); + + // Check _zeros + TORCH_CHECK(_zeros.is_cuda(), "_zeros must be a CUDA tensor"); + TORCH_CHECK(_zeros.dim() == 2, "_zeros must be a 2D tensor"); + TORCH_CHECK(_zeros.is_contiguous(), "_zeros must be contiguous"); + TORCH_CHECK(_zeros.scalar_type() == torch::kInt8, "_zeros must be int8"); + TORCH_CHECK(num_in_channels % _zeros.size(0) == 0, "num_in_channels must be divisible by _zeros.size(0)"); + TORCH_CHECK(num_out_channels == _zeros.size(1), "num_out_channels must be equal to _zeros.size(1)"); + + // Check group size + auto group_size = num_in_channels / _scales_i8.size(0); + TORCH_CHECK(group_size == 128, "group_size must be 128"); + + auto in_feats = reinterpret_cast(_in_feats.data_ptr()); + auto kernel = reinterpret_cast(_kernel.data_ptr()); + auto zeros = reinterpret_cast(_zeros.data_ptr()); + auto scales_i8 = reinterpret_cast(_scales_i8.data_ptr()); + auto wscales = reinterpret_cast(_wscales.data_ptr()); + auto ascales = reinterpret_cast(_ascales.data_ptr()); + // auto options = + // torch::TensorOptions().dtype(torch::kHalf).device(_in_feats.device()); + auto out_feats = reinterpret_cast(_out_feats.data_ptr()); + auto stream = at::cuda::getCurrentCUDAStream(_in_feats.get_device()); + auto sm_version = getSMVersion(); + if (sm_version >= 80) { + constexpr int G = 128; + + if (num_out_feats > 128) { + constexpr int CTA_M = 128; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 64; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else if (num_out_feats >= 128) { + if (num_in_channels <= 4096) { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 64; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 4; + KERNEL_LAUNCH_CODE + } else { + constexpr int CTA_M = 64; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + constexpr int CTA_M = 32; + constexpr int CTA_N = 64; + constexpr int CTA_K = 128; + constexpr int WARP_M = 32; + constexpr int WARP_N = 32; + constexpr int WARP_K = 64; + constexpr int STAGES = 3; + KERNEL_LAUNCH_CODE + } + } else { + TORCH_CHECK_NOT_IMPLEMENTED( + false, "No implemented qserve_w4a8_per_group_gemm for current compute capability: ", sm_version); + } + return; +} diff --git a/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu b/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu new file mode 100644 index 000000000..84b678551 --- /dev/null +++ b/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.cu @@ -0,0 +1,271 @@ +// Adapted from +// https://github.com/mlc-ai/xgrammar/blob/v0.1.18/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// clang-format off +#include +#include +#include +#include +#include + + +#if !defined(USE_ROCM) && (!defined(CUDA_VERSION) || CUDA_VERSION < 12040) +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for ApplyTokenBitmaskInplace"); +} +#else + +#ifndef CUDART_INF_FP16 +#ifndef USE_ROCM +#define CUDART_INF_FP16 __ushort_as_half((unsigned short)0x7C00U) +#endif +#endif + +#ifndef CUDART_INF_BF16 +#ifndef USE_ROCM +#define CUDART_INF_BF16 __ushort_as_bfloat16((unsigned short)0x7F80U) +#endif +#endif + +constexpr int32_t BITS_PER_BLOCK = 32; +constexpr int32_t THREADS_PER_THREAD_BLOCK = 256; + +template +__device__ T NegativeInfinity() { + return -INFINITY; +} + +template <> +__device__ __half NegativeInfinity<__half>() { +#ifdef USE_ROCM + return __float2half(-INFINITY); +#else + return -CUDART_INF_FP16; +#endif +} + +template <> +__device__ __nv_bfloat16 NegativeInfinity<__nv_bfloat16>() { +#ifdef USE_ROCM + return __nv_bfloat16(-INFINITY); +#else + return -CUDART_INF_BF16; +#endif +} + +template +__device__ PackedT PackedNegativeInfinity() { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + T packed[kAlignment]; +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + packed[i] = NegativeInfinity(); + } + return *reinterpret_cast(packed); +} + +template +__global__ void __launch_bounds__(THREADS_PER_THREAD_BLOCK) LogitsBitmaskKernel( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + constexpr uint32_t kPackedMask = (1 << kAlignment) - 1; + + const int batch_idx = (indices == nullptr) ? blockIdx.y : indices[blockIdx.y]; + + const int block_offset = blockIdx.x * THREADS_PER_THREAD_BLOCK * kBitsPerThread; + T* logits_gmem_ptr = logits + batch_idx * logits_stride + block_offset; + const int32_t* bitmask_gmem_ptr = bitmask + batch_idx * bitmask_stride + block_offset / BITS_PER_BLOCK; + const int bitmask_inner_idx = threadIdx.x % (BITS_PER_BLOCK / kAlignment); + T logits_reg[kAlignment]; + +#pragma unroll + for (int offset = threadIdx.x * kAlignment; offset < THREADS_PER_THREAD_BLOCK * kBitsPerThread; + offset += THREADS_PER_THREAD_BLOCK * kAlignment) { + if (block_offset + offset >= vocab_size) { + break; + } + + const uint32_t bitmask_val = + (~bitmask_gmem_ptr[offset / BITS_PER_BLOCK] >> (bitmask_inner_idx * kAlignment)) & kPackedMask; + + if (bitmask_val == 0) { + continue; + } + + if (bitmask_val == kPackedMask) { + *reinterpret_cast(logits_gmem_ptr + offset) = PackedNegativeInfinity(); + continue; + } + + *reinterpret_cast(logits_reg) = *reinterpret_cast(logits_gmem_ptr + offset); +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + if (((bitmask_val >> i) & 1)) { + logits_reg[i] = NegativeInfinity(); + } + } + *reinterpret_cast(logits_gmem_ptr + offset) = *reinterpret_cast(logits_reg); + } +} + +template ::value>> +constexpr auto CeilDiv(T numerator, T denominator) { + return (numerator + denominator - 1) / denominator; +} + +template +void ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + const int32_t num_blocks_per_row = CeilDiv(2048 / THREADS_PER_THREAD_BLOCK * 128, num_rows); + const int32_t num_bits_per_thread = CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * num_blocks_per_row); + + const dim3 block(THREADS_PER_THREAD_BLOCK); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + + if (num_bits_per_thread <= 4 && kAlignment <= 4) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 4), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 8 && kAlignment <= 8) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 8), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 16 && kAlignment <= 16) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 16), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 32), num_rows); + LogitsBitmaskKernel + <<>>(logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } +} + +template +void ApplyTokenBitmaskInplaceDispatchToPackedT( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + if (logits_stride % (sizeof(float4) / sizeof(T)) == 0) { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } else { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } +} + +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor."); + TORCH_CHECK(logits.is_contiguous(), "logits must be contiguous."); + TORCH_CHECK(logits.dim() == 1 || logits.dim() == 2, "logits must be a 1D or 2D tensor."); + std::pair logits_shape = + logits.dim() == 2 ? std::make_pair(static_cast(logits.size(0)), static_cast(logits.size(1))) + : std::make_pair(1, static_cast(logits.size(0))); + + TORCH_CHECK(bitmask.is_cuda(), "bitmask must be a CUDA tensor."); + TORCH_CHECK(bitmask.is_contiguous(), "bitmask must be contiguous."); + TORCH_CHECK(bitmask.dim() == 1 || bitmask.dim() == 2, "bitmask must be a 1D or 2D tensor."); + std::pair bitmask_shape = + bitmask.dim() == 2 ? std::make_pair(static_cast(bitmask.size(0)), static_cast(bitmask.size(1))) + : std::make_pair(1, static_cast(bitmask.size(0))); + + TORCH_CHECK(bitmask.dtype() == torch::kInt32, "bitmask must be of type int32."); + + TORCH_CHECK( + (logits_shape.second + BITS_PER_BLOCK - 1) / BITS_PER_BLOCK >= bitmask_shape.second, + "The provided logits's vocab size should be no less than the bitmask's vocab size " + "(converted from bitmask size). But got vocab size ", + logits_shape.second, + " vs bitmask size ", + bitmask_shape.second); + + int vocab_size = std::min(logits_shape.second, bitmask_shape.second * BITS_PER_BLOCK); + + int32_t num_rows = logits_shape.first; + int32_t* indices_ptr = nullptr; + if (indices) { + TORCH_CHECK(indices->is_cuda(), "indices must be a CUDA tensor."); + TORCH_CHECK(indices->is_contiguous(), "indices must be contiguous."); + TORCH_CHECK(indices->dim() == 1, "indices must be a 1D tensor."); + TORCH_CHECK(indices->dtype() == torch::kInt32, "indices must be of type int32."); + num_rows = indices->size(0); + indices_ptr = indices->data_ptr(); + } else { + TORCH_CHECK(logits_shape.first == bitmask_shape.first, "logits and bitmask must have the same batch size."); + } + + switch (logits.scalar_type()) { + case torch::kFloat32: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + logits.data_ptr(), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__half*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kBFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__nv_bfloat16*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + default: + TORCH_CHECK(false, "logits dtype must be float, half or bfloat16."); + break; + } +} +#endif +// clang-format on diff --git a/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.hip b/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.hip new file mode 100644 index 000000000..3b923bf39 --- /dev/null +++ b/sgl-kernel/csrc/grammar/apply_token_bitmask_inplace_cuda.hip @@ -0,0 +1,273 @@ +// !!! This is a file automatically generated by hipify!!! +#include +// Adapted from +// https://github.com/mlc-ai/xgrammar/blob/v0.1.18/python/xgrammar/kernels/apply_token_bitmask_inplace_cuda.cu + +/* + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// clang-format off +#include +#include +#include +#include +#include + + +#if !defined(USE_ROCM) && (!defined(DTK_VERSION) || DTK_VERSION < 12040) +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(false, "CUDA version must be >= 12.4 for ApplyTokenBitmaskInplace"); +} +#else + +#ifndef CUDART_INF_FP16 +#ifndef USE_ROCM +#define CUDART_INF_FP16 __ushort_as_half((unsigned short)0x7C00U) +#endif +#endif + +#ifndef CUDART_INF_BF16 +#ifndef USE_ROCM +#define CUDART_INF_BF16 __ushort_as_bfloat16((unsigned short)0x7F80U) +#endif +#endif + +constexpr int32_t BITS_PER_BLOCK = 32; +constexpr int32_t THREADS_PER_THREAD_BLOCK = 256; + +template +__device__ T NegativeInfinity() { + return -INFINITY; +} + +template <> +__device__ __half NegativeInfinity<__half>() { +#ifdef USE_ROCM + return __float2half(-INFINITY); +#else + return -CUDART_INF_FP16; +#endif +} + +template <> +__device__ __hip_bfloat16 NegativeInfinity<__hip_bfloat16>() { +#ifdef USE_ROCM + return __hip_bfloat16(-INFINITY); +#else + return -CUDART_INF_BF16; +#endif +} + +template +__device__ PackedT PackedNegativeInfinity() { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + T packed[kAlignment]; +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + packed[i] = NegativeInfinity(); + } + return *reinterpret_cast(packed); +} + +template +__global__ void __launch_bounds__(THREADS_PER_THREAD_BLOCK) LogitsBitmaskKernel( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + constexpr uint32_t kPackedMask = (1 << kAlignment) - 1; + + const int batch_idx = (indices == nullptr) ? blockIdx.y : indices[blockIdx.y]; + + const int block_offset = blockIdx.x * THREADS_PER_THREAD_BLOCK * kBitsPerThread; + T* logits_gmem_ptr = logits + batch_idx * logits_stride + block_offset; + const int32_t* bitmask_gmem_ptr = bitmask + batch_idx * bitmask_stride + block_offset / BITS_PER_BLOCK; + const int bitmask_inner_idx = threadIdx.x % (BITS_PER_BLOCK / kAlignment); + T logits_reg[kAlignment]; + +#pragma unroll + for (int offset = threadIdx.x * kAlignment; offset < THREADS_PER_THREAD_BLOCK * kBitsPerThread; + offset += THREADS_PER_THREAD_BLOCK * kAlignment) { + if (block_offset + offset >= vocab_size) { + break; + } + + const uint32_t bitmask_val = + (~bitmask_gmem_ptr[offset / BITS_PER_BLOCK] >> (bitmask_inner_idx * kAlignment)) & kPackedMask; + + if (bitmask_val == 0) { + continue; + } + + if (bitmask_val == kPackedMask) { + *reinterpret_cast(logits_gmem_ptr + offset) = PackedNegativeInfinity(); + continue; + } + + *reinterpret_cast(logits_reg) = *reinterpret_cast(logits_gmem_ptr + offset); +#pragma unroll + for (int i = 0; i < kAlignment; i++) { + if (((bitmask_val >> i) & 1)) { + logits_reg[i] = NegativeInfinity(); + } + } + *reinterpret_cast(logits_gmem_ptr + offset) = *reinterpret_cast(logits_reg); + } +} + +template ::value>> +constexpr auto CeilDiv(T numerator, T denominator) { + return (numerator + denominator - 1) / denominator; +} + +template +void ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + constexpr int kAlignment = sizeof(PackedT) / sizeof(T); + const int32_t num_blocks_per_row = CeilDiv(2048 / THREADS_PER_THREAD_BLOCK * 128, num_rows); + const int32_t num_bits_per_thread = CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * num_blocks_per_row); + + const dim3 block(THREADS_PER_THREAD_BLOCK); + hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA().stream(); + + if (num_bits_per_thread <= 4 && kAlignment <= 4) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 4), num_rows); + hipLaunchKernelGGL(( LogitsBitmaskKernel) + , dim3(grid), dim3(block), 0, stream, logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 8 && kAlignment <= 8) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 8), num_rows); + hipLaunchKernelGGL(( LogitsBitmaskKernel) + , dim3(grid), dim3(block), 0, stream, logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else if (num_bits_per_thread <= 16 && kAlignment <= 16) { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 16), num_rows); + hipLaunchKernelGGL(( LogitsBitmaskKernel) + , dim3(grid), dim3(block), 0, stream, logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } else { + const dim3 grid(CeilDiv(vocab_size, THREADS_PER_THREAD_BLOCK * 32), num_rows); + hipLaunchKernelGGL(( LogitsBitmaskKernel) + , dim3(grid), dim3(block), 0, stream, logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride); + } +} + +template +void ApplyTokenBitmaskInplaceDispatchToPackedT( + T* __restrict__ logits, + const int32_t* __restrict__ bitmask, + const int32_t* __restrict__ indices, + int32_t vocab_size, + int32_t logits_stride, + int32_t bitmask_stride, + int32_t num_rows) { + if (logits_stride % (sizeof(float4) / sizeof(T)) == 0) { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } else { + ApplyTokenBitmaskInplaceDispatchToBitsPerThread( + logits, bitmask, indices, vocab_size, logits_stride, bitmask_stride, num_rows); + } +} + +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt) { + TORCH_CHECK(logits.is_cuda(), "logits must be a CUDA tensor."); + TORCH_CHECK(logits.is_contiguous(), "logits must be contiguous."); + TORCH_CHECK(logits.dim() == 1 || logits.dim() == 2, "logits must be a 1D or 2D tensor."); + std::pair logits_shape = + logits.dim() == 2 ? std::make_pair(static_cast(logits.size(0)), static_cast(logits.size(1))) + : std::make_pair(1, static_cast(logits.size(0))); + + TORCH_CHECK(bitmask.is_cuda(), "bitmask must be a CUDA tensor."); + TORCH_CHECK(bitmask.is_contiguous(), "bitmask must be contiguous."); + TORCH_CHECK(bitmask.dim() == 1 || bitmask.dim() == 2, "bitmask must be a 1D or 2D tensor."); + std::pair bitmask_shape = + bitmask.dim() == 2 ? std::make_pair(static_cast(bitmask.size(0)), static_cast(bitmask.size(1))) + : std::make_pair(1, static_cast(bitmask.size(0))); + + TORCH_CHECK(bitmask.dtype() == torch::kInt32, "bitmask must be of type int32."); + + TORCH_CHECK( + (logits_shape.second + BITS_PER_BLOCK - 1) / BITS_PER_BLOCK >= bitmask_shape.second, + "The provided logits's vocab size should be no less than the bitmask's vocab size " + "(converted from bitmask size). But got vocab size ", + logits_shape.second, + " vs bitmask size ", + bitmask_shape.second); + + int vocab_size = ::min(logits_shape.second, bitmask_shape.second * BITS_PER_BLOCK); + + int32_t num_rows = logits_shape.first; + int32_t* indices_ptr = nullptr; + if (indices) { + TORCH_CHECK(indices->is_cuda(), "indices must be a CUDA tensor."); + TORCH_CHECK(indices->is_contiguous(), "indices must be contiguous."); + TORCH_CHECK(indices->dim() == 1, "indices must be a 1D tensor."); + TORCH_CHECK(indices->dtype() == torch::kInt32, "indices must be of type int32."); + num_rows = indices->size(0); + indices_ptr = indices->data_ptr(); + } else { + TORCH_CHECK(logits_shape.first == bitmask_shape.first, "logits and bitmask must have the same batch size."); + } + + switch (logits.scalar_type()) { + case torch::kFloat32: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + logits.data_ptr(), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__half*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + case torch::kBFloat16: { + ApplyTokenBitmaskInplaceDispatchToPackedT( + reinterpret_cast<__hip_bfloat16*>(logits.data_ptr()), + bitmask.data_ptr(), + indices_ptr, + vocab_size, + logits_shape.second, + bitmask_shape.second, + num_rows); + break; + } + default: + TORCH_CHECK(false, "logits dtype must be float, half or bfloat16."); + break; + } +} +#endif +// clang-format on diff --git a/sgl-kernel/csrc/kvcacheio/transfer.cu b/sgl-kernel/csrc/kvcacheio/transfer.cu new file mode 100644 index 000000000..bca9f326c --- /dev/null +++ b/sgl-kernel/csrc/kvcacheio/transfer.cu @@ -0,0 +1,573 @@ +#include +#include +#include + +#include + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#include "utils.h" // WARP_SIZE +#endif + +__device__ __forceinline__ void +transfer_item_warp(int32_t lane_id, const void* src_addr, void* dst_addr, int64_t item_size_bytes) { + const uint64_t* __restrict__ src = static_cast(src_addr); + uint64_t* __restrict__ dst = static_cast(dst_addr); + const int total_chunks = item_size_bytes / sizeof(uint64_t); + +#pragma unroll + for (int j = lane_id; j < total_chunks; j += WARP_SIZE) { +#ifndef USE_ROCM + uint64_t tmp; + asm volatile("ld.global.nc.b64 %0,[%1];" : "=l"(tmp) : "l"(src + j) : "memory"); + asm volatile("st.global.cg.b64 [%0],%1;" ::"l"(dst + j), "l"(tmp) : "memory"); + +#else + uint64_t tmp = __builtin_nontemporal_load(src + j); + __builtin_nontemporal_store(tmp, dst + j); +#endif + } +} + +template +__device__ __forceinline__ T* get_global_offset_lf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t layer_dim, + int64_t page_id, + int64_t item_size_bytes) { + // layer first + return base + layer_id * layer_dim + page_id * item_size_bytes; +} + +template +__device__ __forceinline__ T* get_global_offset_pf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t page_dim, + int64_t page_id, + int64_t item_size_bytes) { + // page first + return base + page_id * page_dim + layer_id * item_size_bytes; +} + +// get offset from layer base table when layers are not contiguous +template +__device__ __forceinline__ T* get_global_offset_lf_tbl( + T* /*unused*/, + const uintptr_t* __restrict__ layer_base_tbl, + int64_t layer_id, + int64_t /*unused*/, + int64_t page_id, + int64_t item_size_bytes) { + return reinterpret_cast(layer_base_tbl[layer_id]) + page_id * item_size_bytes; +} + +template +__global__ void transfer_kernel_impl( + const void* __restrict__ src_k, + void* __restrict__ dst_k, + const void* __restrict__ src_v, + void* __restrict__ dst_v, + const int64_t* __restrict__ src_indices, + const int64_t* __restrict__ dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t num_items, + int64_t items_per_warp, + int64_t item_size_bytes, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const uintptr_t* __restrict__ src_k_layer_tbl, + const uintptr_t* __restrict__ dst_k_layer_tbl, + const uintptr_t* __restrict__ src_v_layer_tbl, + const uintptr_t* __restrict__ dst_v_layer_tbl) { + int32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + int32_t lane_id = tid % WARP_SIZE; + int32_t warp_id = tid / WARP_SIZE; + + for (int i = 0; i < items_per_warp; ++i) { + int64_t item_id = warp_id * items_per_warp + i; + if (item_id >= num_items) { + break; + } + const int64_t src_page_id = src_indices[item_id]; + const int64_t dst_page_id = dst_indices[item_id]; + + // Loop over layers if necessary + for (int64_t layer_id = start_layer_id; layer_id < start_layer_id + num_layers_to_process; ++layer_id) { + const char* src_ptr = SrcOffsetFn( + static_cast(src_k), src_k_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_ptr = DstOffsetFn( + static_cast(dst_k), dst_k_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_ptr, dst_ptr, item_size_bytes); + + if constexpr (!IsMLA) { + const char* src_v_ptr = SrcOffsetFn( + static_cast(src_v), src_v_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_v_ptr = DstOffsetFn( + static_cast(dst_v), dst_v_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_v_ptr, dst_v_ptr, item_size_bytes); + } + } + } +} + +template +void transfer_kv_launcher( + const at::Tensor& src_k, + at::Tensor& dst_k, + const at::Tensor& src_v, + at::Tensor& dst_v, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t item_size, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const at::Tensor& src_k_layers, + const at::Tensor& dst_k_layers, + const at::Tensor& src_v_layers, + const at::Tensor& dst_v_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(src_indices.is_cuda(), "Source indices must be a CUDA tensor"); + TORCH_CHECK(dst_indices.is_cuda(), "Destination indices must be a CUDA tensor"); + TORCH_CHECK(src_indices.scalar_type() == at::kLong, "Source indices must be of type long"); + TORCH_CHECK(dst_indices.scalar_type() == at::kLong, "Destination indices must be of type long"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(item_size % 8 == 0, "Item byte size must be divisible by 8"); + + auto div_up = [](int64_t x, int64_t y) { return (x + y - 1) / y; }; + const int64_t num_items = src_indices.numel(); + const int64_t items_per_warp = div_up(num_items, block_quota * num_warps_per_block); + const int32_t num_blocks = div_up(num_items, items_per_warp * num_warps_per_block); + dim3 grid_dim(num_blocks, 1, 1); + const int32_t threads_per_block = num_warps_per_block * WARP_SIZE; + + const void* src_k_ptr = src_k.defined() ? src_k.data_ptr() : nullptr; + void* dst_k_ptr = dst_k.defined() ? dst_k.data_ptr() : nullptr; + const void* src_v_ptr = IsMLA || !src_v.defined() ? nullptr : src_v.data_ptr(); + void* dst_v_ptr = IsMLA || !dst_v.defined() ? nullptr : dst_v.data_ptr(); + const uintptr_t* src_k_tbl_ptr = src_k_layers.defined() ? src_k_layers.data_ptr() : nullptr; + const uintptr_t* dst_k_tbl_ptr = dst_k_layers.defined() ? dst_k_layers.data_ptr() : nullptr; + const uintptr_t* src_v_tbl_ptr = IsMLA || !src_v_layers.defined() ? nullptr : src_v_layers.data_ptr(); + const uintptr_t* dst_v_tbl_ptr = IsMLA || !dst_v_layers.defined() ? nullptr : dst_v_layers.data_ptr(); + + cudaStream_t torch_current_stream = at::cuda::getCurrentCUDAStream(); + transfer_kernel_impl<<>>( + src_k_ptr, + dst_k_ptr, + src_v_ptr, + dst_v_ptr, + src_indices.data_ptr(), + dst_indices.data_ptr(), + start_layer_id, + num_layers_to_process, + num_items, + items_per_warp, + item_size, + src_layout_dim, + dst_layout_dim, + src_k_tbl_ptr, + dst_k_tbl_ptr, + src_v_tbl_ptr, + dst_v_tbl_ptr); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void transfer_kv_per_layer( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_pf_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer( + const at::Tensor src_k_layers, + const at::Tensor dst_k_layers, + const at::Tensor src_v_layers, + const at::Tensor dst_v_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, false>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_k_layers, + dst_k_layers, + src_v_layers, + dst_v_layers, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_lf_pf( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, false>( + empty, + dst_k, + empty, + dst_v, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_k_layers, + empty, + src_v_layers, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_mla( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_mla_pf_lf( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla( + const at::Tensor src_layers, + const at::Tensor dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, true>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_layers, + dst_layers, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla_lf_pf( + const at::Tensor src_layers, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, true>( + empty, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_layers, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +inline void transfer_page_direct( + const at::Tensor src_buffer, + at::Tensor dst_buffer, + int64_t src_page_index, + int64_t dst_page_index, + int64_t page_size) { + dst_buffer.slice(0, dst_page_index, dst_page_index + page_size) + .copy_( + src_buffer.slice(0, src_page_index, src_page_index + page_size), + /* non_blocking= */ true); +} + +void transfer_kv_direct( + const std::vector& src_layers, + std::vector dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t page_size) { + TORCH_CHECK( + src_layers.size() == dst_layers.size(), "Source and destination layers must have the same number of layers"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + + const auto num_indices = src_indices_cpu.numel(); + const int64_t num_layers = src_layers.size(); + int64_t* src_indices_ptr = src_indices_cpu.data_ptr(); + int64_t* dst_indices_ptr = dst_indices_cpu.data_ptr(); + + int64_t start_index = 0; + int64_t end_index = 0; + + for (int64_t i = 0; i < num_indices; ++i) { + if (i < num_indices - 1) { + auto src_diff = src_indices_ptr[i + 1] - src_indices_ptr[i]; + auto dst_diff = dst_indices_ptr[i + 1] - dst_indices_ptr[i]; + + if (src_diff == 1 && dst_diff == 1) { + continue; + } + end_index = i + 1; + } else { // last batch + end_index = num_indices; + } + auto src_index = src_indices_ptr[start_index]; + auto dst_index = dst_indices_ptr[start_index]; + auto num_tokens = end_index - start_index; + + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct(src_layers[j], dst_layers[j], src_index, dst_index, num_tokens); + } + start_index = end_index; + } +} + +template +inline void transfer_kv_page_first_direct_impl( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t page_size) { + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + const int64_t num_pages = src_indices_cpu.size(0) / page_size; + + if constexpr (IsLf2Pf) { + const bool is_mla = dst_ptrs.size() == 1; + const int64_t num_layers = is_mla ? src_ptrs.size() : src_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item(); + auto d_index = dst_indices_cpu[i * page_size].item() / page_size; + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[j], dst_ptrs[0].select(0, d_index).select(0, start_layer_id + j), s_index, 0, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[j + num_layers], + dst_ptrs[1].select(0, d_index).select(0, start_layer_id + j), + s_index, + 0, + page_size); + } + } + } + } else { + const bool is_mla = src_ptrs.size() == 1; + const int64_t num_layers = is_mla ? dst_ptrs.size() : dst_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item() / page_size; + auto d_index = dst_indices_cpu[i * page_size].item(); + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[0].select(0, s_index).select(0, start_layer_id + j), dst_ptrs[j], 0, d_index, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[1].select(0, s_index).select(0, start_layer_id + j), + dst_ptrs[j + num_layers], + 0, + d_index, + page_size); + } + } + } + } +} + +void transfer_kv_per_layer_direct_pf_lf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t layer_id, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, layer_id, page_size); +} + +void transfer_kv_all_layer_direct_lf_pf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, 0, page_size); +} diff --git a/sgl-kernel/csrc/kvcacheio/transfer.hip b/sgl-kernel/csrc/kvcacheio/transfer.hip new file mode 100644 index 000000000..065fbec99 --- /dev/null +++ b/sgl-kernel/csrc/kvcacheio/transfer.hip @@ -0,0 +1,576 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +#include +#include +#include + +#include + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#include "utils_hip.h" // WARP_SIZE +#endif + +__device__ __forceinline__ void +transfer_item_warp(int32_t lane_id, const void* src_addr, void* dst_addr, int64_t item_size_bytes) { + const uint64_t* __restrict__ src = static_cast(src_addr); + uint64_t* __restrict__ dst = static_cast(dst_addr); + const int total_chunks = item_size_bytes / sizeof(uint64_t); + +#pragma unroll + for (int j = lane_id; j < total_chunks; j += WARP_SIZE) { +#ifndef USE_ROCM + uint64_t tmp; + asm volatile("ld.global.nc.b64 %0,[%1];" : "=l"(tmp) : "l"(src + j) : "memory"); + asm volatile("st.global.cg.b64 [%0],%1;" ::"l"(dst + j), "l"(tmp) : "memory"); + +#else + uint64_t tmp = __builtin_nontemporal_load(src + j); + __builtin_nontemporal_store(tmp, dst + j); +#endif + } +} + +template +__device__ __forceinline__ T* get_global_offset_lf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t layer_dim, + int64_t page_id, + int64_t item_size_bytes) { + // layer first + return base + layer_id * layer_dim + page_id * item_size_bytes; +} + +template +__device__ __forceinline__ T* get_global_offset_pf( + T* base, + const uintptr_t* __restrict__ /*unused*/, + int64_t layer_id, + int64_t page_dim, + int64_t page_id, + int64_t item_size_bytes) { + // page first + return base + page_id * page_dim + layer_id * item_size_bytes; +} + +// get offset from layer base table when layers are not contiguous +template +__device__ __forceinline__ T* get_global_offset_lf_tbl( + T* /*unused*/, + const uintptr_t* __restrict__ layer_base_tbl, + int64_t layer_id, + int64_t /*unused*/, + int64_t page_id, + int64_t item_size_bytes) { + return reinterpret_cast(layer_base_tbl[layer_id]) + page_id * item_size_bytes; +} + +template +__global__ void transfer_kernel_impl( + const void* __restrict__ src_k, + void* __restrict__ dst_k, + const void* __restrict__ src_v, + void* __restrict__ dst_v, + const int64_t* __restrict__ src_indices, + const int64_t* __restrict__ dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t num_items, + int64_t items_per_warp, + int64_t item_size_bytes, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const uintptr_t* __restrict__ src_k_layer_tbl, + const uintptr_t* __restrict__ dst_k_layer_tbl, + const uintptr_t* __restrict__ src_v_layer_tbl, + const uintptr_t* __restrict__ dst_v_layer_tbl) { + int32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + int32_t lane_id = tid % WARP_SIZE; + int32_t warp_id = tid / WARP_SIZE; + + for (int i = 0; i < items_per_warp; ++i) { + int64_t item_id = warp_id * items_per_warp + i; + if (item_id >= num_items) { + break; + } + const int64_t src_page_id = src_indices[item_id]; + const int64_t dst_page_id = dst_indices[item_id]; + + // Loop over layers if necessary + for (int64_t layer_id = start_layer_id; layer_id < start_layer_id + num_layers_to_process; ++layer_id) { + const char* src_ptr = SrcOffsetFn( + static_cast(src_k), src_k_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_ptr = DstOffsetFn( + static_cast(dst_k), dst_k_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_ptr, dst_ptr, item_size_bytes); + + if constexpr (!IsMLA) { + const char* src_v_ptr = SrcOffsetFn( + static_cast(src_v), src_v_layer_tbl, layer_id, src_layout_dim, src_page_id, item_size_bytes); + char* dst_v_ptr = DstOffsetFn( + static_cast(dst_v), dst_v_layer_tbl, layer_id, dst_layout_dim, dst_page_id, item_size_bytes); + transfer_item_warp(lane_id, src_v_ptr, dst_v_ptr, item_size_bytes); + } + } + } +} + +template +void transfer_kv_launcher( + const at::Tensor& src_k, + at::Tensor& dst_k, + const at::Tensor& src_v, + at::Tensor& dst_v, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t num_layers_to_process, + int64_t item_size, + int64_t src_layout_dim, + int64_t dst_layout_dim, + const at::Tensor& src_k_layers, + const at::Tensor& dst_k_layers, + const at::Tensor& src_v_layers, + const at::Tensor& dst_v_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(src_indices.is_cuda(), "Source indices must be a CUDA tensor"); + TORCH_CHECK(dst_indices.is_cuda(), "Destination indices must be a CUDA tensor"); + TORCH_CHECK(src_indices.scalar_type() == at::kLong, "Source indices must be of type long"); + TORCH_CHECK(dst_indices.scalar_type() == at::kLong, "Destination indices must be of type long"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(item_size % 8 == 0, "Item byte size must be divisible by 8"); + + auto div_up = [](int64_t x, int64_t y) { return (x + y - 1) / y; }; + const int64_t num_items = src_indices.numel(); + const int64_t items_per_warp = div_up(num_items, block_quota * num_warps_per_block); + const int32_t num_blocks = div_up(num_items, items_per_warp * num_warps_per_block); + dim3 grid_dim(num_blocks, 1, 1); + const int32_t threads_per_block = num_warps_per_block * WARP_SIZE; + + const void* src_k_ptr = src_k.defined() ? src_k.data_ptr() : nullptr; + void* dst_k_ptr = dst_k.defined() ? dst_k.data_ptr() : nullptr; + const void* src_v_ptr = IsMLA || !src_v.defined() ? nullptr : src_v.data_ptr(); + void* dst_v_ptr = IsMLA || !dst_v.defined() ? nullptr : dst_v.data_ptr(); + const uintptr_t* src_k_tbl_ptr = src_k_layers.defined() ? src_k_layers.data_ptr() : nullptr; + const uintptr_t* dst_k_tbl_ptr = dst_k_layers.defined() ? dst_k_layers.data_ptr() : nullptr; + const uintptr_t* src_v_tbl_ptr = IsMLA || !src_v_layers.defined() ? nullptr : src_v_layers.data_ptr(); + const uintptr_t* dst_v_tbl_ptr = IsMLA || !dst_v_layers.defined() ? nullptr : dst_v_layers.data_ptr(); + + hipStream_t torch_current_stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + hipLaunchKernelGGL(( transfer_kernel_impl), dim3(grid_dim), dim3(threads_per_block), 0, torch_current_stream, + src_k_ptr, + dst_k_ptr, + src_v_ptr, + dst_v_ptr, + src_indices.data_ptr(), + dst_indices.data_ptr(), + start_layer_id, + num_layers_to_process, + num_items, + items_per_warp, + item_size, + src_layout_dim, + dst_layout_dim, + src_k_tbl_ptr, + dst_k_tbl_ptr, + src_v_tbl_ptr, + dst_v_tbl_ptr); + C10_HIP_KERNEL_LAUNCH_CHECK(); +} + +void transfer_kv_per_layer( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_pf_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, false>( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer( + const at::Tensor src_k_layers, + const at::Tensor dst_k_layers, + const at::Tensor src_v_layers, + const at::Tensor dst_v_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, false>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_k_layers, + dst_k_layers, + src_v_layers, + dst_v_layers, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_lf_pf( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_k_layers.size(0), "Number of layers in source k tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, false>( + empty, + dst_k, + empty, + dst_v, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_k_layers, + empty, + src_v_layers, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_mla( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + 1, + item_size, + 0, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_per_layer_mla_pf_lf( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block) { + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf, true>( + src, + dst, + empty, + empty, + src_indices, + dst_indices, + layer_id, + 1, + item_size, + src_layout_dim, + 0, + empty, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla( + const at::Tensor src_layers, + const at::Tensor dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_lf_tbl, true>( + empty, + empty, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + 0, + src_layers, + dst_layers, + empty, + empty, + block_quota, + num_warps_per_block); +} + +void transfer_kv_all_layer_mla_lf_pf( + const at::Tensor src_layers, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block) { + TORCH_CHECK(num_layers == src_layers.size(0), "Number of layers in source tensor does not match num_layers"); + at::Tensor empty; + transfer_kv_launcher, get_global_offset_pf, true>( + empty, + dst, + empty, + empty, + src_indices, + dst_indices, + 0, + num_layers, + item_size, + 0, + dst_layout_dim, + src_layers, + empty, + empty, + empty, + block_quota, + num_warps_per_block); +} + +inline void transfer_page_direct( + const at::Tensor src_buffer, + at::Tensor dst_buffer, + int64_t src_page_index, + int64_t dst_page_index, + int64_t page_size) { + dst_buffer.slice(0, dst_page_index, dst_page_index + page_size) + .copy_( + src_buffer.slice(0, src_page_index, src_page_index + page_size), + /* non_blocking= */ true); +} + +void transfer_kv_direct( + const std::vector& src_layers, + std::vector dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t page_size) { + TORCH_CHECK( + src_layers.size() == dst_layers.size(), "Source and destination layers must have the same number of layers"); + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + + const auto num_indices = src_indices_cpu.numel(); + const int64_t num_layers = src_layers.size(); + int64_t* src_indices_ptr = src_indices_cpu.data_ptr(); + int64_t* dst_indices_ptr = dst_indices_cpu.data_ptr(); + + int64_t start_index = 0; + int64_t end_index = 0; + + for (int64_t i = 0; i < num_indices; ++i) { + if (i < num_indices - 1) { + auto src_diff = src_indices_ptr[i + 1] - src_indices_ptr[i]; + auto dst_diff = dst_indices_ptr[i + 1] - dst_indices_ptr[i]; + + if (src_diff == 1 && dst_diff == 1) { + continue; + } + end_index = i + 1; + } else { // last batch + end_index = num_indices; + } + auto src_index = src_indices_ptr[start_index]; + auto dst_index = dst_indices_ptr[start_index]; + auto num_tokens = end_index - start_index; + + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct(src_layers[j], dst_layers[j], src_index, dst_index, num_tokens); + } + start_index = end_index; + } +} + +template +inline void transfer_kv_page_first_direct_impl( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t start_layer_id, + int64_t page_size) { + TORCH_CHECK(src_indices.numel() == dst_indices.numel(), "Source and destination indices must have the same length"); + TORCH_CHECK(page_size > 0, "Page size must be positive"); + TORCH_CHECK(src_indices.numel() % page_size == 0, "Source indices size must be divisible by page size"); + + auto src_indices_cpu = src_indices.cpu(); + auto dst_indices_cpu = dst_indices.cpu(); + const int64_t num_pages = src_indices_cpu.size(0) / page_size; + + if constexpr (IsLf2Pf) { + const bool is_mla = dst_ptrs.size() == 1; + const int64_t num_layers = is_mla ? src_ptrs.size() : src_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item(); + auto d_index = dst_indices_cpu[i * page_size].item() / page_size; + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[j], dst_ptrs[0].select(0, d_index).select(0, start_layer_id + j), s_index, 0, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[j + num_layers], + dst_ptrs[1].select(0, d_index).select(0, start_layer_id + j), + s_index, + 0, + page_size); + } + } + } + } else { + const bool is_mla = src_ptrs.size() == 1; + const int64_t num_layers = is_mla ? dst_ptrs.size() : dst_ptrs.size() / 2; + + for (const auto i : c10::irange(num_pages)) { + auto s_index = src_indices_cpu[i * page_size].item() / page_size; + auto d_index = dst_indices_cpu[i * page_size].item(); + for (int64_t j = 0; j < num_layers; ++j) { + transfer_page_direct( + src_ptrs[0].select(0, s_index).select(0, start_layer_id + j), dst_ptrs[j], 0, d_index, page_size); + if (!is_mla) { + transfer_page_direct( + src_ptrs[1].select(0, s_index).select(0, start_layer_id + j), + dst_ptrs[j + num_layers], + 0, + d_index, + page_size); + } + } + } + } +} + +void transfer_kv_per_layer_direct_pf_lf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t layer_id, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, layer_id, page_size); +} + +void transfer_kv_all_layer_direct_lf_pf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t page_size) { + transfer_kv_page_first_direct_impl(src_ptrs, dst_ptrs, src_indices, dst_indices, 0, page_size); +} diff --git a/sgl-kernel/csrc/mamba/causal_conv1d.cu b/sgl-kernel/csrc/mamba/causal_conv1d.cu new file mode 100644 index 000000000..5eb3d9b1b --- /dev/null +++ b/sgl-kernel/csrc/mamba/causal_conv1d.cu @@ -0,0 +1,669 @@ +// clang-format off +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_fwd.cu +// and https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d_update.cu +#include +#include +#include + +#include "causal_conv1d.h" +#include +#include +#include // For C10_CUDA_CHECK and C10_CUDA_KERNEL_LAUNCH_CHECK + +#include +#include + +#define BOOL_SWITCH(COND, CONST_NAME, ...) \ + [&] { \ + if (COND) { \ + static constexpr bool CONST_NAME = true; \ + return __VA_ARGS__(); \ + } else { \ + static constexpr bool CONST_NAME = false; \ + return __VA_ARGS__(); \ + } \ + }() + +#define CHECK_SHAPE(x, ...) TORCH_CHECK(x.sizes() == torch::IntArrayRef({__VA_ARGS__}), #x " must have shape (" #__VA_ARGS__ ")") + +#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, NAME, ...) \ + if (ITYPE == at::ScalarType::Half) { \ + using input_t = at::Half; \ + using weight_t = at::Half; \ + __VA_ARGS__(); \ + } else if (ITYPE == at::ScalarType::BFloat16) { \ + using input_t = at::BFloat16; \ + using weight_t = at::BFloat16; \ + __VA_ARGS__(); \ + } else if (ITYPE == at::ScalarType::Float) { \ + using input_t = float; \ + using weight_t = float; \ + __VA_ARGS__(); \ + } else { \ + AT_ERROR(#NAME, " not implemented for input type '", toString(ITYPE), "'"); \ + } + + +template +void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + +template +void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + +void set_conv_params_fwd(ConvParamsBase ¶ms, + // sizes + const size_t batch, + const size_t dim, + const size_t seqlen, + const size_t width, + // device pointers + const at::Tensor x, + const at::Tensor weight, + const at::Tensor out, + const std::optional& bias, + bool silu_activation, + int64_t pad_slot_id, + const std::optional& query_start_loc = std::nullopt, + const std::optional& cache_indices = std::nullopt, + const std::optional& has_initial_state = std::nullopt) { + + // Reset the parameters + memset(¶ms, 0, sizeof(params)); + + params.batch = batch; + params.dim = dim; + params.seqlen = seqlen; + params.width = width; + params.pad_slot_id = pad_slot_id; + + params.silu_activation = silu_activation; + + // Set the pointers and strides. + params.x_ptr = x.data_ptr(); + params.weight_ptr = weight.data_ptr(); + params.bias_ptr = bias.has_value() ? bias.value().data_ptr() : nullptr; + params.out_ptr = out.data_ptr(); + // All stride are in elements, not bytes. + params.query_start_loc_ptr = query_start_loc.has_value() ? query_start_loc.value().data_ptr() : nullptr; + params.cache_indices_ptr = cache_indices.has_value() ? cache_indices.value().data_ptr() : nullptr; + params.has_initial_state_ptr = has_initial_state.has_value() ? has_initial_state.value().data_ptr() : nullptr; + const bool varlen = params.query_start_loc_ptr != nullptr; + params.x_batch_stride = x.stride(varlen ? 1 : 0); + params.x_c_stride = x.stride(varlen ? 0 : 1); + params.x_l_stride = x.stride(varlen ? 1 : -1); + params.weight_c_stride = weight.stride(0); + params.weight_width_stride = weight.stride(1); + params.out_batch_stride = out.stride(varlen ? 1 : 0); + params.out_c_stride = out.stride(varlen ? 0 : 1); + params.out_l_stride = out.stride(varlen ? 1 : -1); +} + + +void causal_conv1d_fwd(const at::Tensor &x, const at::Tensor &weight, + const std::optional &bias_, + const std::optional &conv_states, + const std::optional &query_start_loc, + const std::optional &cache_indices, + const std::optional &has_initial_state, + bool silu_activation, + // used to identify padding entries if cache_indices provided + // in case of padding, the kernel will return early + int64_t pad_slot_id) { + auto input_type = x.scalar_type(); + auto weight_type = weight.scalar_type(); + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16); + + TORCH_CHECK(x.is_cuda()); + TORCH_CHECK(weight.is_cuda()); + + const bool varlen = query_start_loc.has_value() ? true : false; + const auto sizes = x.sizes(); + const int batch_size = varlen ? query_start_loc.value().sizes()[0] - 1 : sizes[0]; + const int dim = varlen ? sizes[0] : sizes[1]; + const int seqlen = varlen ? sizes[1] : sizes[2]; + const int width = weight.size(-1); + if (varlen){ + CHECK_SHAPE(x, dim, seqlen); + } + else { + CHECK_SHAPE(x, batch_size, dim, seqlen); + } + CHECK_SHAPE(weight, dim, width); + + + + if (bias_.has_value()) { + auto bias = bias_.value(); + TORCH_CHECK(bias.scalar_type() == weight_type); + TORCH_CHECK(bias.is_cuda()); + TORCH_CHECK(bias.stride(-1) == 1); + CHECK_SHAPE(bias, dim); + } + + + if (has_initial_state.has_value()) { + auto has_initial_state_ = has_initial_state.value(); + TORCH_CHECK(has_initial_state_.scalar_type() == at::ScalarType::Bool); + TORCH_CHECK(has_initial_state_.is_cuda()); + CHECK_SHAPE(has_initial_state_, batch_size); + } + + + if (query_start_loc.has_value()) { + auto query_start_loc_ = query_start_loc.value(); + TORCH_CHECK(query_start_loc_.scalar_type() == at::ScalarType::Int); + TORCH_CHECK(query_start_loc_.is_cuda()); + } + + + if (cache_indices.has_value()) { + auto cache_indices_ = cache_indices.value(); + TORCH_CHECK(cache_indices_.scalar_type() == at::ScalarType::Int); + TORCH_CHECK(cache_indices_.is_cuda()); + CHECK_SHAPE(cache_indices_, batch_size); + } + + at::Tensor out = x; + + ConvParamsBase params; + set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out, + bias_, + silu_activation, + pad_slot_id, + query_start_loc, + cache_indices, + has_initial_state + ); + + if (conv_states.has_value()) { + auto conv_states_ = conv_states.value(); + TORCH_CHECK(conv_states_.scalar_type() == input_type); + TORCH_CHECK(conv_states_.is_cuda()); + params.conv_states_ptr = conv_states_.data_ptr(); + params.conv_states_batch_stride = conv_states_.stride(0); + params.conv_states_c_stride = conv_states_.stride(1); + params.conv_states_l_stride = conv_states_.stride(2); + } else { + params.conv_states_ptr = nullptr; + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)x.get_device()}; + auto stream = at::cuda::getCurrentCUDAStream().stream(); + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_fwd", [&] { + causal_conv1d_fwd_cuda(params, stream); + }); +} + + +void causal_conv1d_update(const at::Tensor &x, + const at::Tensor &conv_state, + const at::Tensor &weight, + const std::optional &bias_, + bool silu_activation, + const std::optional &cache_seqlens_, + const std::optional &conv_state_indices_, + // used to identify padding entries if cache_indices provided + // in case of padding, the kernel will return early + int64_t pad_slot_id) { + auto input_type = x.scalar_type(); + auto weight_type = weight.scalar_type(); + TORCH_CHECK(input_type == at::ScalarType::Float || input_type == at::ScalarType::Half || input_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == at::ScalarType::Float || weight_type == at::ScalarType::Half || weight_type == at::ScalarType::BFloat16); + TORCH_CHECK(weight_type == input_type, "weight type must equal to input type, other variations are disabled due to binary size limitations"); + TORCH_CHECK(conv_state.scalar_type() == input_type); + + TORCH_CHECK(x.is_cuda()); + TORCH_CHECK(conv_state.is_cuda()); + TORCH_CHECK(weight.is_cuda()); + + const auto sizes = x.sizes(); + const int batch_size = sizes[0]; + const int dim = sizes[1]; + const int seqlen = sizes[2]; + const int width = weight.size(-1); + const int conv_state_len = conv_state.size(2); + TORCH_CHECK(conv_state_len >= width - 1); + + CHECK_SHAPE(x, batch_size, dim, seqlen); + CHECK_SHAPE(weight, dim, width); + + TORCH_CHECK(width >= 2 && width <= 4, "causal_conv1d only supports width between 2 and 4"); + + if (bias_.has_value()) { + auto bias = bias_.value(); + TORCH_CHECK(bias.scalar_type() == weight_type); + TORCH_CHECK(bias.is_cuda()); + TORCH_CHECK(bias.stride(-1) == 1); + CHECK_SHAPE(bias, dim); + } + + at::Tensor out = x; + + ConvParamsBase params; + set_conv_params_fwd(params, batch_size, dim, seqlen, width, x, weight, out, + bias_, + silu_activation, + pad_slot_id); + params.conv_state_ptr = conv_state.data_ptr(); + params.conv_state_len = conv_state_len; + // All stride are in elements, not bytes. + params.conv_state_batch_stride = conv_state.stride(0); + params.conv_state_c_stride = conv_state.stride(1); + params.conv_state_l_stride = conv_state.stride(2); + + if (cache_seqlens_.has_value()) { + auto cache_seqlens = cache_seqlens_.value(); + TORCH_CHECK(cache_seqlens.scalar_type() == torch::kInt32); + TORCH_CHECK(cache_seqlens.is_cuda()); + TORCH_CHECK(cache_seqlens.stride(-1) == 1); + CHECK_SHAPE(cache_seqlens, batch_size); + params.cache_seqlens = cache_seqlens.data_ptr(); + } else { + params.cache_seqlens = nullptr; + } + + if (conv_state_indices_.has_value()) { + auto conv_state_indices = conv_state_indices_.value(); + TORCH_CHECK(conv_state_indices.scalar_type() == torch::kInt32) + TORCH_CHECK(conv_state_indices.is_cuda()); + TORCH_CHECK(conv_state_indices.stride(0) == 1) + CHECK_SHAPE(conv_state_indices, batch_size); + + int conv_state_entries = conv_state.size(0); + CHECK_SHAPE(conv_state, conv_state_entries, dim, conv_state_len); + + params.conv_state_indices_ptr = conv_state_indices.data_ptr(); + } else { + CHECK_SHAPE(conv_state, batch_size, dim, conv_state_len); + params.conv_state_indices_ptr = nullptr; + } + + // Otherwise the kernel will be launched from cuda:0 device + // Cast to char to avoid compiler warning about narrowing + at::cuda::CUDAGuard device_guard{(char)x.get_device()}; + auto stream = at::cuda::getCurrentCUDAStream().stream(); + DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(x.scalar_type(), "causal_conv1d_update", [&] { + causal_conv1d_update_cuda(params, stream); + }); +} + +template +struct Causal_conv1d_fwd_kernel_traits { + using input_t = input_t_; + using weight_t = weight_t_; + static constexpr int kNThreads = kNThreads_; + static constexpr int kWidth = kWidth_; + static constexpr int kNBytes = sizeof(input_t); + static_assert(kNBytes == 2 || kNBytes == 4); + static constexpr int kNElts = kNBytes == 4 ? 4 : 8; + static_assert(kWidth <= kNElts); + static constexpr bool kIsVecLoad = kIsVecLoad_; + using vec_t = typename BytesToType::Type; + using BlockLoadT = cub::BlockLoad; + using BlockLoadVecT = cub::BlockLoad; + using BlockStoreT = cub::BlockStore; + using BlockStoreVecT = cub::BlockStore; + static constexpr int kSmemIOSize = kIsVecLoad + ? 0 + : custom_max({sizeof(typename BlockLoadT::TempStorage), sizeof(typename BlockStoreT::TempStorage)}); + static constexpr int kSmemExchangeSize = kNThreads * kNBytes * kNElts; + static constexpr int kSmemSize = kSmemIOSize + kSmemExchangeSize; +}; + +template +__global__ __launch_bounds__(Ktraits::kNThreads) +void causal_conv1d_fwd_kernel(ConvParamsBase params) { + constexpr int kWidth = Ktraits::kWidth; + constexpr int kNThreads = Ktraits::kNThreads; + constexpr int kNElts = Ktraits::kNElts; + constexpr bool kIsVecLoad = Ktraits::kIsVecLoad; + using input_t = typename Ktraits::input_t; + using vec_t = typename Ktraits::vec_t; + using weight_t = typename Ktraits::weight_t; + + // Shared memory. + extern __shared__ char smem_[]; + auto& smem_load = reinterpret_cast(smem_); + auto& smem_load_vec = reinterpret_cast(smem_); + auto& smem_store = reinterpret_cast(smem_); + auto& smem_store_vec = reinterpret_cast(smem_); + vec_t *smem_exchange = reinterpret_cast(smem_ + Ktraits::kSmemIOSize); + + const bool kVarlen = params.query_start_loc_ptr != nullptr; + const int tidx = threadIdx.x; + const int batch_id = blockIdx.x; + const int channel_id = blockIdx.y; + const int *query_start_loc = kVarlen ? reinterpret_cast(params.query_start_loc_ptr) : nullptr; + const int sequence_start_index = kVarlen ? query_start_loc[batch_id] : batch_id; + const int seqlen = kVarlen ? query_start_loc[batch_id + 1] - sequence_start_index : params.seqlen; + + input_t *x = reinterpret_cast(params.x_ptr) + sequence_start_index * params.x_batch_stride + + channel_id * params.x_c_stride; + weight_t *weight = reinterpret_cast(params.weight_ptr) + channel_id * params.weight_c_stride; + input_t *out = reinterpret_cast(params.out_ptr) + sequence_start_index * params.out_batch_stride + + channel_id * params.out_c_stride; + float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast(params.bias_ptr)[channel_id]); + + bool has_initial_state = params.has_initial_state_ptr == nullptr ? false + : reinterpret_cast(params.has_initial_state_ptr)[batch_id]; + + int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr + : reinterpret_cast(params.cache_indices_ptr); + int cache_index = cache_indices == nullptr ? batch_id : cache_indices[batch_id]; + // cache_index == params.pad_slot_id is defined as padding, so we exit early + if (cache_index == params.pad_slot_id){ + return; + } + input_t *conv_states = params.conv_states_ptr == nullptr ? nullptr + : reinterpret_cast(params.conv_states_ptr) + cache_index * params.conv_states_batch_stride + channel_id * params.conv_states_c_stride; + + // Thread 0 will load the last elements of the previous chunk, so we initialize those to 0. + if (tidx == 0) { + input_t initial_state[kNElts] = {0}; + if (has_initial_state) { + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ initial_state[kNElts - 1 - (kWidth - 2) + w ] = conv_states[w]; } + } + smem_exchange[kNThreads - 1] = reinterpret_cast(initial_state)[0]; + } + + float weight_vals[kWidth]; + #pragma unroll + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); } + + constexpr int kChunkSize = kNThreads * kNElts; + const int n_chunks = (seqlen + kChunkSize - 1) / kChunkSize; + for (int chunk = 0; chunk < n_chunks; ++chunk) { + input_t x_vals_load[2 * kNElts] = {0}; + if constexpr(kIsVecLoad) { + typename Ktraits::BlockLoadVecT(smem_load_vec).Load(reinterpret_cast(x), *reinterpret_cast(&x_vals_load[kNElts]), (seqlen - chunk * kChunkSize) / kNElts); + } else { + __syncthreads(); + typename Ktraits::BlockLoadT(smem_load).Load(x, *reinterpret_cast(&x_vals_load[kNElts]), seqlen - chunk * kChunkSize); + } + x += kChunkSize; + __syncthreads(); + // Thread kNThreads - 1 don't write yet, so that thread 0 can read + // the last elements of the previous chunk. + if (tidx < kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast(x_vals_load)[1]; } + __syncthreads(); + reinterpret_cast(x_vals_load)[0] = smem_exchange[tidx > 0 ? tidx - 1 : kNThreads - 1]; + __syncthreads(); + // Now thread kNThreads - 1 can write the last elements of the current chunk. + if (tidx == kNThreads - 1) { smem_exchange[tidx] = reinterpret_cast(x_vals_load)[1]; } + + float x_vals[2 * kNElts]; + #pragma unroll + for (int i = 0; i < 2 * kNElts; ++i) { x_vals[i] = float(x_vals_load[i]); } + + float out_vals[kNElts]; + #pragma unroll + for (int i = 0; i < kNElts; ++i) { + out_vals[i] = bias_val; + #pragma unroll + for (int w = 0; w < kWidth; ++w) { + out_vals[i] += weight_vals[w] * x_vals[kNElts + i - (kWidth - w - 1)]; + } + } + + if (params.silu_activation) { + #pragma unroll + for (int i = 0; i < kNElts; ++i) { + out_vals[i] = out_vals[i] / (1 + expf(-out_vals[i])); + } + } + + input_t out_vals_store[kNElts]; + #pragma unroll + for (int i = 0; i < kNElts; ++i) { out_vals_store[i] = out_vals[i]; } + if constexpr(kIsVecLoad) { + typename Ktraits::BlockStoreVecT(smem_store_vec).Store(reinterpret_cast(out), reinterpret_cast(out_vals_store), (seqlen - chunk * kChunkSize) / kNElts); + } else { + typename Ktraits::BlockStoreT(smem_store).Store(out, out_vals_store, seqlen - chunk * kChunkSize); + } + out += kChunkSize; + + int final_state_position = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize); + // in case the final state is separated between the last "smem_exchange" and + // and the one before it (chunk = n_chunks - 1 and chunk = n_chunks - 2), + // (which occurs when `final_state_position` is a non-positivie index) + // we load the correct data from smem_exchange from both chunks, the last chunk iteration and the one before it + if (conv_states != nullptr && final_state_position < 0 && seqlen > kWidth){ + input_t vals_load[kNElts] = {0}; + if ((chunk == n_chunks - 2) && (tidx == kNThreads - 1)){ + // chunk = n_chunks - 2, a segment of the final state sits in the last index + reinterpret_cast(vals_load)[0] = smem_exchange[kNThreads - 1]; + #pragma unroll + for (int w = 0; w < -final_state_position; ++w){ + conv_states[w] = vals_load[kNElts + final_state_position + w]; + } + } + if ((chunk == n_chunks - 1) && tidx == 0){ + // chunk = n_chunks - 1, the second segment of the final state first positions + reinterpret_cast(vals_load)[0] = smem_exchange[0]; + for (int w = -final_state_position; w < kWidth - 1; ++w){ + conv_states[w] = vals_load[w + final_state_position]; + } + return; + } + } + } + // Final state is stored in the smem_exchange last token slot, + // in case seqlen < kWidth, we would need to take the final state from the + // initial state which is stored in conv_states + // in case seqlen > kWidth, we would need to load the last kWidth - 1 data + // and load it into conv_state accordingly + int last_thread = ((seqlen - (kWidth - 1)) - (n_chunks - 1) * kChunkSize) / kNElts; + if (conv_states != nullptr && tidx == last_thread) { + input_t x_vals_load[kNElts * 2] = {0}; + // in case we are on the first kWidth tokens + if (last_thread == 0 && seqlen < kWidth){ + // Need to take the initial state + reinterpret_cast(x_vals_load)[0] = smem_exchange[0]; + const int offset = seqlen - (kWidth - 1); + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + // pad the existing state + if ((w - seqlen) >= 0 && has_initial_state) { conv_states[w - seqlen] = conv_states[w]; } + else if ((w - seqlen) >= 0 && !has_initial_state) { conv_states[w - seqlen] = input_t(0.0f); } + } + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + if (offset + w >= 0) + conv_states[w] = x_vals_load[offset + w ]; + } + } + else { + // in case the final state is in between the threads data + const int offset = ((seqlen - (kWidth - 1)) % (kNElts)); + if ((offset + kWidth - 2) >= kNElts && (last_thread + 1 < kNThreads)){ + // In case last_thread == kNThreads - 1, accessing last_thread + 1 will result in a + // illegal access error on H100. + // Therefore, we access last_thread + 1, only if the final state data sits there + reinterpret_cast(x_vals_load)[1] = smem_exchange[last_thread + 1]; + } + reinterpret_cast(x_vals_load)[0] = smem_exchange[last_thread]; + #pragma unroll + for (int w = 0; w < kWidth - 1; ++w){ + conv_states[w] = x_vals_load[offset + w ]; + } + } + + } +} + + +template +void causal_conv1d_fwd_launch(ConvParamsBase ¶ms, cudaStream_t stream) { + static constexpr int kNElts = sizeof(input_t) == 4 ? 4 : 8; + const bool kVarlen = params.query_start_loc_ptr != nullptr; + BOOL_SWITCH(params.seqlen % kNElts == 0 && !kVarlen, kIsVecLoad, [&] { + using Ktraits = Causal_conv1d_fwd_kernel_traits; + constexpr int kSmemSize = Ktraits::kSmemSize; + dim3 grid(params.batch, params.dim); + + auto kernel = &causal_conv1d_fwd_kernel; + + if (kSmemSize >= 48 * 1024) { + #ifndef USE_ROCM + C10_CUDA_CHECK(cudaFuncSetAttribute( + kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize)); + #else + // There is a slight signature discrepancy in HIP and CUDA "FuncSetAttribute" function. + C10_CUDA_CHECK(cudaFuncSetAttribute( + (void *) kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize)); + std::cerr << "Warning (causal_conv1d fwd launch): attempting to set maxDynamicSharedMemorySize on an AMD GPU which is currently a non-op (in ROCm versions <= 6.1). This might lead to undefined behavior. \n" << std::endl; + #endif + } + kernel<<>>(params); + + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }); +} + +template +void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream) { + if (params.width == 2) { + causal_conv1d_fwd_launch<128, 2, input_t, weight_t>(params, stream); + } else if (params.width == 3) { + causal_conv1d_fwd_launch<128, 3, input_t, weight_t>(params, stream); + } else if (params.width == 4) { + causal_conv1d_fwd_launch<128, 4, input_t, weight_t>(params, stream); + } +} + + +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_fwd_cuda(ConvParamsBase ¶ms, cudaStream_t stream); + + + + +template +struct Causal_conv1d_update_kernel_traits { + using input_t = input_t_; + using weight_t = weight_t_; + static constexpr int kNThreads = kNThreads_; + static constexpr int kWidth = kWidth_; + static constexpr int kNBytes = sizeof(input_t); + static_assert(kNBytes == 2 || kNBytes == 4); +}; + +template +__global__ __launch_bounds__(Ktraits::kNThreads) +void causal_conv1d_update_kernel(ConvParamsBase params) { + constexpr int kWidth = Ktraits::kWidth; + constexpr int kNThreads = Ktraits::kNThreads; + using input_t = typename Ktraits::input_t; + using weight_t = typename Ktraits::weight_t; + + const int tidx = threadIdx.x; + const int batch_id = blockIdx.x; + const int channel_id = blockIdx.y * kNThreads + tidx; + if (channel_id >= params.dim) return; + + input_t *x = reinterpret_cast(params.x_ptr) + batch_id * params.x_batch_stride + + channel_id * params.x_c_stride; + + // If params.conv_state_batch_indices is set, then the conv state is gathered from the conv state tensor + // along the batch axis. Otherwise, the conv state coordinate is the same as the batch id. + const int conv_state_batch_coord = params.conv_state_indices_ptr == nullptr + ? batch_id + : params.conv_state_indices_ptr[batch_id]; + // conv_state_batch_coord == params.pad_slot_id is defined as padding so we exit early + if (conv_state_batch_coord == params.pad_slot_id){ + return; + } + input_t *conv_state = reinterpret_cast(params.conv_state_ptr) + + conv_state_batch_coord * params.conv_state_batch_stride + + channel_id * params.conv_state_c_stride; + + weight_t *weight = reinterpret_cast(params.weight_ptr) + channel_id * params.weight_c_stride; + input_t *out = reinterpret_cast(params.out_ptr) + batch_id * params.out_batch_stride + + channel_id * params.out_c_stride; + float bias_val = params.bias_ptr == nullptr ? 0.f : float(reinterpret_cast(params.bias_ptr)[channel_id]); + + int state_len = params.conv_state_len; + int advance_len = params.seqlen; + int cache_seqlen = kIsCircularBuffer ? params.cache_seqlens[batch_id] % state_len : 0; + int update_idx = cache_seqlen - (kWidth - 1); + update_idx = update_idx < 0 ? update_idx + state_len : update_idx; + + float weight_vals[kWidth] = {0}; + #pragma unroll + for (int i = 0; i < kWidth; ++i) { weight_vals[i] = float(weight[i * params.weight_width_stride]); } + + float x_vals[kWidth] = {0}; + if constexpr (!kIsCircularBuffer) { + #pragma unroll 2 + for (int i = 0; i < state_len - advance_len - (kWidth - 1); ++i) { + conv_state[i * params.conv_state_l_stride] = conv_state[(i + advance_len) * params.conv_state_l_stride]; + } + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i) { + input_t state_val = conv_state[(state_len - (kWidth - 1) + i) * params.conv_state_l_stride]; + if (i < advance_len + (kWidth - 1) && state_len - advance_len - (kWidth - 1) + i >= 0) { + conv_state[(state_len - advance_len - (kWidth - 1) + i) * params.conv_state_l_stride] = state_val; + } + x_vals[i] = float(state_val); + } + } else { + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i, update_idx = update_idx + 1 >= state_len ? update_idx + 1 - state_len : update_idx + 1) { + input_t state_val = conv_state[update_idx * params.conv_state_l_stride]; + x_vals[i] = float(state_val); + } + } + #pragma unroll 2 + for (int i = 0; i < params.seqlen; ++i) { + input_t x_val = x[i * params.x_l_stride]; + if constexpr (!kIsCircularBuffer) { + if (i < advance_len && state_len - advance_len + i >= 0) { + conv_state[(state_len - advance_len + i) * params.conv_state_l_stride] = x_val; + } + } else { + conv_state[update_idx * params.conv_state_l_stride] = x_val; + ++update_idx; + update_idx = update_idx >= state_len ? update_idx - state_len : update_idx; + } + x_vals[kWidth - 1] = float(x_val); + float out_val = bias_val; + #pragma unroll + for (int j = 0; j < kWidth; ++j) { out_val += weight_vals[j] * x_vals[j]; } + if (params.silu_activation) { out_val = out_val / (1 + expf(-out_val)); } + out[i * params.out_l_stride] = input_t(out_val); + // Shift the input buffer by 1 + #pragma unroll + for (int i = 0; i < kWidth - 1; ++i) { x_vals[i] = x_vals[i + 1]; } + } +} + +template +void causal_conv1d_update_launch(ConvParamsBase ¶ms, cudaStream_t stream) { + using Ktraits = Causal_conv1d_update_kernel_traits; + dim3 grid(params.batch, (params.dim + kNThreads - 1) / kNThreads); + auto kernel = params.cache_seqlens == nullptr + ? &causal_conv1d_update_kernel + : &causal_conv1d_update_kernel; + kernel<<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +template +void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream) { + if (params.width == 2) { + causal_conv1d_update_launch<64, 2, input_t, weight_t>(params, stream); + } else if (params.width == 3) { + causal_conv1d_update_launch<64, 3, input_t, weight_t>(params, stream); + } else if (params.width == 4) { + causal_conv1d_update_launch<64, 4, input_t, weight_t>(params, stream); + } +} + +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); +template void causal_conv1d_update_cuda(ConvParamsBase ¶ms, cudaStream_t stream); diff --git a/sgl-kernel/csrc/mamba/causal_conv1d.h b/sgl-kernel/csrc/mamba/causal_conv1d.h new file mode 100644 index 000000000..33f8e7432 --- /dev/null +++ b/sgl-kernel/csrc/mamba/causal_conv1d.h @@ -0,0 +1,159 @@ +/****************************************************************************** + * Copyright (c) 2024, Tri Dao. + ******************************************************************************/ +// clang-format off +// adapted from https://github.com/Dao-AILab/causal-conv1d/blob/main/csrc/causal_conv1d.h +#pragma once + +#include +#include +//////////////////////////////////////////////////////////////////////////////////////////////////// + +struct ConvParamsBase { + using index_t = uint32_t; + + int batch, dim, seqlen, width; + int64_t pad_slot_id; + bool silu_activation; + + index_t x_batch_stride; + index_t x_c_stride; + index_t x_l_stride; + index_t weight_c_stride; + index_t weight_width_stride; + index_t out_batch_stride; + index_t out_c_stride; + index_t out_l_stride; + + int conv_state_len; + index_t conv_state_batch_stride; + index_t conv_state_c_stride; + index_t conv_state_l_stride; + + // Common data pointers. + void *__restrict__ x_ptr; + void *__restrict__ weight_ptr; + void *__restrict__ bias_ptr; + void *__restrict__ out_ptr; + + void *__restrict__ conv_state_ptr; + void *__restrict__ query_start_loc_ptr; + void *__restrict__ has_initial_state_ptr; + void *__restrict__ cache_indices_ptr; + int32_t *__restrict__ cache_seqlens; + + // For the continuous batching case. Makes it so that the mamba state for + // the current batch doesn't need to be a contiguous tensor. + int32_t *__restrict__ conv_state_indices_ptr; + + void *__restrict__ seq_idx_ptr; + + // No __restrict__ since initial_states could be the same as final_states. + void * initial_states_ptr; + index_t initial_states_batch_stride; + index_t initial_states_l_stride; + index_t initial_states_c_stride; + + void * final_states_ptr; + index_t final_states_batch_stride; + index_t final_states_l_stride; + index_t final_states_c_stride; + + void * conv_states_ptr; + index_t conv_states_batch_stride; + index_t conv_states_l_stride; + index_t conv_states_c_stride; +}; + + +#ifndef USE_ROCM + #include + + template + __device__ inline T shuffle_xor(T val, int offset) { + return __shfl_xor_sync(uint32_t(-1), val, offset); + } + + constexpr size_t custom_max(std::initializer_list ilist) + { + return std::max(ilist); + } + + template + constexpr T constexpr_min(T a, T b) { + return std::min(a, b); + } + +#else + #include + + template + __device__ inline T shuffle_xor(T val, int offset) { + return __shfl_xor(val, offset); + } + constexpr size_t custom_max(std::initializer_list ilist) + { + return *std::max_element(ilist.begin(), ilist.end()); + } + + template + constexpr T constexpr_min(T a, T b) { + return a < b ? a : b; + } +#endif + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template struct BytesToType {}; + +template<> struct BytesToType<16> { + using Type = uint4; + static_assert(sizeof(Type) == 16); +}; + +template<> struct BytesToType<8> { + using Type = uint64_t; + static_assert(sizeof(Type) == 8); +}; + +template<> struct BytesToType<4> { + using Type = uint32_t; + static_assert(sizeof(Type) == 4); +}; + +template<> struct BytesToType<2> { + using Type = uint16_t; + static_assert(sizeof(Type) == 2); +}; + +template<> struct BytesToType<1> { + using Type = uint8_t; + static_assert(sizeof(Type) == 1); +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct SumOp { +__device__ inline T operator()(T const & x, T const & y) { return x + y; } +}; + +template +struct Allreduce { + static_assert(THREADS == 32 || THREADS == 16 || THREADS == 8 || THREADS == 4); + template + static __device__ inline T run(T x, Operator &op) { + constexpr int OFFSET = THREADS / 2; + x = op(x, __shfl_xor_sync(uint32_t(-1), x, OFFSET)); + return Allreduce::run(x, op); + } +}; + +template<> +struct Allreduce<2> { +template +static __device__ inline T run(T x, Operator &op) { + x = op(x, __shfl_xor_sync(uint32_t(-1), x, 1)); + return x; +} +}; diff --git a/sgl-kernel/csrc/memory/store.cu b/sgl-kernel/csrc/memory/store.cu new file mode 100644 index 000000000..c6dd97ebd --- /dev/null +++ b/sgl-kernel/csrc/memory/store.cu @@ -0,0 +1,147 @@ +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +using std::size_t; +using std::uint64_t; + +// Each warp will process 256 bytes per loop iteration +template +__global__ void store_kv_cache_256x1( + uint64_t* __restrict__ k_cache, + uint64_t* __restrict__ v_cache, + const T* __restrict__ out_loc, + const size_t length, + const uint64_t* __restrict__ k, + const uint64_t* __restrict__ v, + const size_t kv_cache_stride, + const size_t kv_input_stride, + const size_t num_items) { + const auto idx = blockIdx.x * blockDim.x + threadIdx.x; + const auto warp_id = idx / 32; + const auto lane_id = idx % 32; + if (warp_id >= length) return; + const auto offset = out_loc[warp_id]; + const auto k_dst = k_cache + offset * kv_cache_stride; + const auto v_dst = v_cache + offset * kv_cache_stride; + const auto k_src = k + warp_id * kv_input_stride; + const auto v_src = v + warp_id * kv_input_stride; + for (size_t i = 0; i < num_items; ++i) { + k_dst[lane_id + i * 32] = k_src[lane_id + i * 32]; + v_dst[lane_id + i * 32] = v_src[lane_id + i * 32]; + } +} + +// Each warp will process 128 bytes per loop iteration +template +__global__ void store_kv_cache_128x2( + uint64_t* __restrict__ k_cache, + uint64_t* __restrict__ v_cache, + const T* __restrict__ out_loc, + const size_t length, + const uint64_t* __restrict__ k, + const uint64_t* __restrict__ v, + const size_t kv_cache_stride, + const size_t kv_input_stride, + const size_t num_items) { + const auto idx = blockIdx.x * blockDim.x + threadIdx.x; + const auto warp_id = idx / 32; + const auto lane_id = idx % 32; + if (warp_id >= length) return; + const auto offset = out_loc[warp_id]; + const auto copy_k = lane_id < 16; + const auto copy_id = lane_id % 16; + const auto cache = copy_k ? k_cache : v_cache; + const auto input = copy_k ? k : v; + const auto dst = cache + offset * kv_cache_stride; + const auto src = input + warp_id * kv_input_stride; + for (size_t i = 0; i < num_items; ++i) { + dst[copy_id + i * 16] = src[copy_id + i * 16]; + } +} + +} // namespace + +auto store_kv_cache(at::Tensor k_cache, at::Tensor v_cache, at::Tensor out_loc, at::Tensor k, at::Tensor v) -> void { + const auto max_tokens = k_cache.size(0); + const auto num_tokens = out_loc.size(0); + k_cache = k_cache.view({max_tokens, -1}); + v_cache = v_cache.view({max_tokens, -1}); + k = k.view({num_tokens, -1}); + v = v.view({num_tokens, -1}); + + TORCH_CHECK( + k_cache.is_cuda() && v_cache.is_cuda() && out_loc.is_cuda() && k.is_cuda() && v.is_cuda(), + "All tensors must be CUDA tensors"); + TORCH_CHECK(k_cache.sizes() == v_cache.sizes(), "k_cache and v_cache must have the same size"); + TORCH_CHECK(k_cache.strides() == v_cache.strides(), "k_cache and v_cache must have the same strides"); + TORCH_CHECK(k.sizes() == v.sizes(), "k and v must have the same size"); + TORCH_CHECK(k.strides() == v.strides(), "k and v must have the same strides"); + TORCH_CHECK(k.stride(-1) == 1 && k_cache.stride(-1) == 1, "k and k_cache must be contiguous in head."); + TORCH_CHECK(k.size(-1) == k_cache.size(-1), "k and k_cache must have the same head size"); + TORCH_CHECK(out_loc.dim() == 1 && out_loc.is_contiguous(), "out_loc must be a 1D contiguous tensor"); + static_assert(sizeof(uint64_t) == 8, "uint64_t must be 8 bytes, our code assumes that"); + + const auto length = out_loc.size(0); + const auto elem_size = k.element_size(); + const auto size_bytes = elem_size * k.size(-1); + const auto kv_cache_stride_bytes = elem_size * k_cache.stride(-2); + const auto kv_input_stride_bytes = elem_size * k.stride(-2); + const auto kv_cache_stride = kv_cache_stride_bytes / 8; + const auto kv_input_stride = kv_input_stride_bytes / 8; + + const auto k_cache_ptr = static_cast(k_cache.data_ptr()); + const auto v_cache_ptr = static_cast(v_cache.data_ptr()); + const auto k_ptr = static_cast(k.data_ptr()); + const auto v_ptr = static_cast(v.data_ptr()); + const auto num_threads = 256; + const auto num_warps = num_threads / 32; + const auto num_blocks = (length + num_warps - 1) / num_warps; + const auto stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_INTEGRAL_TYPES(out_loc.scalar_type(), "store_kv_cache", [&] { + if constexpr (!std::is_same_v && !std::is_same_v) { + // do not instantiate the kernel if out_loc is not int32 or int64 + TORCH_CHECK(false, "out_loc must be of type int32 or int64, got: ", out_loc.scalar_type()); + } else { + if (size_bytes % 256 == 0) { + const auto items_per_warp = size_bytes / 256; + store_kv_cache_256x1<<>>( + k_cache_ptr, + v_cache_ptr, + out_loc.data_ptr(), + length, + k_ptr, + v_ptr, + kv_cache_stride, + kv_input_stride, + items_per_warp); + } else if (size_bytes % 128 == 0) { + const auto items_per_warp = size_bytes / 128; + store_kv_cache_128x2<<>>( + k_cache_ptr, + v_cache_ptr, + out_loc.data_ptr(), + length, + k_ptr, + v_ptr, + kv_cache_stride, + kv_input_stride, + items_per_warp); + } else { + TORCH_CHECK( + false, + "The last dimension size bytes of k and v must be" + " divisible by 128 at least, got: ", + size_bytes); + } + } + }); +} diff --git a/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu new file mode 100644 index 000000000..28ed28c1e --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/scaled_mm_entry.cu @@ -0,0 +1,91 @@ +#include +#include +#include + +int32_t get_sm_version_num() { + int32_t major_capability, minor_capability; + cudaDeviceGetAttribute(&major_capability, cudaDevAttrComputeCapabilityMajor, 0); + cudaDeviceGetAttribute(&minor_capability, cudaDevAttrComputeCapabilityMinor, 0); + int32_t version_num = major_capability * 10 + minor_capability; + return version_num; +} + +void cutlass_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk); + +void get_cutlass_w4a8_moe_mm_data_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void cutlass_w4a8_moe_mm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + cutlass_w4a8_moe_mm_sm90( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk); + return; +} + +void get_cutlass_w4a8_moe_mm_data( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + get_cutlass_w4a8_moe_mm_data_caller( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k); + return; +} diff --git a/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh new file mode 100644 index 000000000..8cd50c60c --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_get_group_starts.cuh @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +#include "cutlass/bfloat16.h" +#include "cutlass/float8.h" + +template +__global__ void int4_fp8_get_group_gemm_starts( + int32_t* expert_offsets, + ElementA** a_offsets, + ElementB** b_offsets, + ElementC** out_offsets, + ElementAccumulator** a_scales_offsets, + cutlass::bfloat16_t** b_scales_offsets, + ElementA* a_base_as_int, + ElementB* b_base_as_int, + ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + cutlass::bfloat16_t* b_scales_base_as_int, + int64_t n, + int64_t k, + bool per_act_token, + bool per_out_ch) { + int expert_id = threadIdx.x; + int32_t expert_offset = expert_offsets[expert_id]; + + a_offsets[expert_id] = a_base_as_int + expert_offset * k; + b_offsets[expert_id] = b_base_as_int + expert_id * k * n / 2; + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + a_scales_offsets[expert_id] = a_scales_base_as_int + (per_act_token ? expert_offset : 0); + b_scales_offsets[expert_id] = b_scales_base_as_int + (per_out_ch ? expert_id * n * k / 128 : expert_id); +} + +#define __CALL_W4A8_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + int4_fp8_get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + out_tensors.size(1), \ + a_tensors.size(1), \ + per_act_token, \ + per_out_ch); \ + } + +namespace { + +void run_int4_fp8_get_group_gemm_starts( + torch::Tensor const& expert_offsets, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kInt8); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kBFloat16); + + int num_experts = static_cast(expert_offsets.size(0)); + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + auto stream = at::cuda::getCurrentCUDAStream(expert_offsets.device().index()); + + if (false) { + } + __CALL_W4A8_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t) + __CALL_W4A8_GET_STARTS_KERNEL(torch::kFloat16, half) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} + +} // namespace diff --git a/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu new file mode 100644 index 000000000..bd63d2ee1 --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cu @@ -0,0 +1,386 @@ +#include +#include +#include + +#include + +#include "cutlass/cutlass.h" +#include "w4a8_grouped_mm_c3x.cuh" + +using namespace cute; + +namespace { + +enum class Sched { PP, CO }; + +template +struct SM90W4A8Config { + using KernelSchedule = std::conditional_t< + S == Sched::PP, + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpong, + cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperative>; + + using EpilogueSchedule = std::conditional_t< + S == Sched::PP, + cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong, + cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative>; + + using TileShape = cute::Shape, cute::Int, cute::Int>; + using ClusterShape = cute::Shape, cute::Int, cute::Int>; + using Cutlass3xW4A8Gemm = cutlass_3x_w4a8_group_gemm; +}; + +template +using SM90_PP = SM90W4A8Config; + +template +using SM90_CO = SM90W4A8Config; + +template +inline void invoke_gemm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size) { + using GemmT = typename Config::Cutlass3xW4A8Gemm; + cutlass_w4a8_group_gemm_caller( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); +} + +void dispatch_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + uint32_t const m = a_tensors.size(0) / topk; + uint32_t const n = d_tensors.size(1); + uint32_t const k = a_tensors.size(1); + + if (n == 4096 && k == 7168) { + // group gemm 1 + if (m <= 4) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 16) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 256) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 1024) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 7168 && k == 2048) { + // group gemm 2 + if (m <= 8) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 512) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 512 && k == 7168) { + // group gemm 1 for tp + if (m <= 4) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 16) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 256) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 1024) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else if (n == 7168 && k == 256) { + // group gemm 2 for tp + if (m <= 8) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else if (m <= 512) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } else { + if (k % 512 == 0) { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } else { + invoke_gemm>( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size); + } + } +} + +} // namespace + +void cutlass_w4a8_moe_mm_sm90( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk) { + dispatch_w4a8_moe_mm_sm90( + d_tensors, + a_tensors, + b_tensors, + a_scales, + b_scales, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk); +} diff --git a/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh new file mode 100644 index 000000000..d8b794997 --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_grouped_mm_c3x.cuh @@ -0,0 +1,277 @@ +#pragma once + +/** + * @file w4a8_grouped_mm_c3x.cuh + * @brief Implementation of grouped GEMM operation with int4 and fp8 mixed + * precision + * + * This file implements a grouped GEMM operation that multiplies FP8 matrices + * (A) with quantized INT4 matrices (B), applying per-block scaling factors. + * The implementation is optimized for NVIDIA Hopper GPUs, leveraging Tensor + * Cores for mixed precision arithmetic. + * + * Key features: + * - Supports grouped GEMM operations with multiple experts + * - Uses FP8 (e4m3) for matrix A + * - Uses INT4 quantization for matrix B with per-block scaling + * - Implements preprocessing for INT4 encoding and scale packing + * - Optimized for Hopper architecture with Tensor Core operations + */ + +#include +#include +#include +#include + +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass_extensions/gemm/collective/collective_builder_mixed_input.hpp" +#include "w4a8_get_group_starts.cuh" + +using namespace cute; + +namespace { + +// Type definitions +using MmaType = cutlass::float_e4m3_t; // FP8 e4m3 type +using QuantType = cutlass::int4b_t; // 4-bit integer type +using ElementAccumulator = float; // Accumulator type +using ElementScale = cutlass::bfloat16_t; // Scale type +using ElementC = cutlass::bfloat16_t; // Output type +using ElementD = ElementC; // Output type +using ProblemShape = cutlass::gemm::GroupProblemShape>; + +// Architecture-specific configurations +using ArchTag = cutlass::arch::Sm90; +using OperatorClass = cutlass::arch::OpClassTensorOp; +// constexpr int TileShapeK = 512; +// using TileShape = Shape<_128, _32, cute::Int>; +// using ClusterShape = Shape<_1, _1, _1>; + +// Layout configurations +using LayoutA = cutlass::layout::RowMajor; +using LayoutB = cutlass::layout::ColumnMajor; +using LayoutC = cutlass::layout::RowMajor; +using LayoutD = LayoutC; + +// Transposed layouts +using LayoutA_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutB_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutC_Transpose = typename cutlass::layout::LayoutTranspose::type; +using LayoutD_Transpose = typename cutlass::layout::LayoutTranspose::type; + +// Alignments +static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; +static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + +template +struct cutlass_3x_w4a8_group_gemm { + static constexpr int GroupSize = 128; + static constexpr int PackedScalesNum = get<2>(TileShape{}) / GroupSize; + using ElementScalePacked = cutlass::Array; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + TileShape, + ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutC_Transpose*, + AlignmentC, + ElementD, + LayoutD_Transpose*, + AlignmentD, + EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloopScaleOnly = typename cutlass::gemm::collective::CollectiveBuilderMixedInput< + ArchTag, + OperatorClass, + cute::tuple, + LayoutB_Transpose*, + AlignmentB, + MmaType, + LayoutA_Transpose*, + AlignmentA, + ElementAccumulator, + TileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + KernelSchedule>::CollectiveOp; + + // Define the final kernel and GEMM operation types + using GemmKernelScaleOnly = + cutlass::gemm::kernel::GemmUniversal; + + using GemmScaleOnly = cutlass::gemm::device::GemmUniversalAdapter; + + using StrideA = cute::remove_pointer_t>; + using StrideB = cute::remove_pointer_t>; + using StrideC = typename GemmKernelScaleOnly::InternalStrideC; + using StrideD = typename GemmKernelScaleOnly::InternalStrideD; + using StrideS = typename CollectiveMainloopScaleOnly::StrideScale; +}; + +/** + * @brief Main function to run int4 * fp8 grouped GEMM from PyTorch + * + * This function performs multiple GEMM operations in parallel where each + * operation multiplies an FP8 matrix (A) with a quantized INT4 matrix (B), + * applying per-channel scaling factors. It's designed for efficient execution + * on NVIDIA Hopper GPUs, leveraging Tensor Cores for optimal performance with + * mixed precision arithmetic. + * + * The function includes preprocessing steps for both INT4 tensors and scale + * factors to ensure optimal performance and correct operation. + * + * @param d_tensors Output tensor D with shape [total_m, total_n] + * @param a_tensors Tensor containing all A matrices (fp8_e4m3) with shape + * [total_m, K] + * @param b_tensors Tensor containing all B matrices (int4 packed as int8) with + * shape [E, N, K/2] + * @param a_scales Tensor containing A matrix scale factors + * @param b_scales Tensor containing B matrix scale factors with shape [E, + * K//512, N*4] + * @param expert_offsets Tensor containing expert offsets for determining group + * boundaries (int32) + * @param problem_sizes Tensor containing problem sizes with shape [num_experts, + * 3] (M, N, K for each group) (int32) + * @param a_strides Stride information for A tensors + * @param b_strides Stride information for B tensors + * @param d_strides Stride information for D tensors + * @param s_strides Stride information for scale tensors + * @param chunk_size Size of each chunk for scales (K / number of scale chunks) + */ +// template +template +void cutlass_w4a8_group_gemm_caller( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size) { + // using Gemm = cutlass_3x_w4a8_group_gemm; + using Args = typename Gemm::GemmScaleOnly::Arguments; + + int num_experts = static_cast(expert_offsets.size(0)); + bool per_act_token = a_scales.numel() != 1; + bool per_out_ch = b_scales.numel() != num_experts; + + // Check inputs + TORCH_CHECK(a_tensors.dim() == 2, "A tensor must be 2D"); + TORCH_CHECK(b_tensors.dim() == 3, "B tensor must be 3D [E, N, K/2]"); + TORCH_CHECK(b_scales.dim() == 3, "Scale tensor must be 3D [E, K//512, N*4]"); + TORCH_CHECK(a_scales.dim() == 1, "A Scale tensor must be 1D [1]"); + TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be a 1D tensor"); + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor"); + + // Check tensor shapes + TORCH_CHECK(problem_sizes.size(0) == num_experts, "problem_sizes must have num_experts rows"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have 3 columns (N, M, K)"); + TORCH_CHECK(b_tensors.size(0) == num_experts, "B tensor first dimension must match number of groups"); + TORCH_CHECK(b_scales.size(0) == num_experts, "Scale tensor first dimension must match number of groups"); + TORCH_CHECK(b_tensors.size(2) * 2 == a_tensors.size(1), "B tensor K/2 dimension must match A tensor K dimension"); + + // Check tensor types + TORCH_CHECK(a_tensors.scalar_type() == torch::kFloat8_e4m3fn, "A tensor must be fp8 (float_e4m3_t) type"); + TORCH_CHECK(b_tensors.scalar_type() == torch::kInt8, "B tensor must contain packed int4 values (stored as int8)"); + TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "Expert offsets must be int32 type"); + TORCH_CHECK(problem_sizes.scalar_type() == torch::kInt32, "Problem sizes must be int32 type"); + + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a_tensors.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = a_tensors.device().index(); + hw_info.sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + + Args arguments; + decltype(arguments.epilogue.thread) fusion_args; + fusion_args.alpha = 0; + fusion_args.beta = 0; + fusion_args.alpha_ptr = a_scales.data_ptr(); + ; + fusion_args.beta_ptr = nullptr; + fusion_args.alpha_ptr_array = nullptr; + fusion_args.beta_ptr_array = nullptr; + fusion_args.dAlpha = {cute::_0{}, cute::_0{}, 0}; + fusion_args.dBeta = {cute::_0{}, cute::_0{}, 0}; + + ProblemShape::UnderlyingProblemShape* problem_sizes_as_shapes = + static_cast(problem_sizes.data_ptr()); + + run_int4_fp8_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_tensors, + b_tensors, + d_tensors, + a_scales, + b_scales); + + arguments = Args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + {static_cast(b_ptrs.data_ptr()), + static_cast(b_strides.data_ptr()), + static_cast(a_ptrs.data_ptr()), + static_cast(a_strides.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + static_cast(s_strides.data_ptr()), + static_cast(chunk_size)}, + {fusion_args, + nullptr, + nullptr, + static_cast(out_ptrs.data_ptr()), + static_cast(d_strides.data_ptr())}, + hw_info}; + + // Instantiate and run GEMM + typename Gemm::GemmScaleOnly gemm; + size_t workspace_size = Gemm::GemmScaleOnly::get_workspace_size(arguments); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a_tensors.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + + cutlass::Status status = gemm.can_implement(arguments); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM implementation not supported"); + } + + status = gemm.initialize(arguments, workspace.data_ptr(), stream); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM initialization failed"); + } + + status = gemm.run(stream); + if (status != cutlass::Status::kSuccess) { + TORCH_CHECK(false, "GEMM execution failed"); + } +} + +} // namespace diff --git a/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu new file mode 100644 index 000000000..f30f7b025 --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe/w4a8/w4a8_moe_data.cu @@ -0,0 +1,79 @@ +#include +#include +#include + +#include + +constexpr uint64_t THREADS_PER_EXPERT = 512; + +__global__ void compute_problem_sizes_w4a8( + const int32_t* __restrict__ topk_ids, + int32_t* problem_sizes1, + int32_t* problem_sizes2, + int32_t* atomic_buffer, + const int topk_length, + const int n, + const int k) { + int expert_id = blockIdx.x; + + int occurrences = 0; + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + occurrences += (topk_ids[i] == expert_id); + } + atomicAdd(&atomic_buffer[expert_id], occurrences); + __syncthreads(); + + if (threadIdx.x == 0) { + int final_occurrences = atomic_buffer[expert_id]; + problem_sizes1[expert_id * 3] = 2 * n; + problem_sizes1[expert_id * 3 + 1] = final_occurrences; + problem_sizes1[expert_id * 3 + 2] = k; + problem_sizes2[expert_id * 3] = k; + problem_sizes2[expert_id * 3 + 1] = final_occurrences; + problem_sizes2[expert_id * 3 + 2] = n; + } +} + +__global__ void compute_expert_offsets_w4a8( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* atomic_buffer, + const int num_experts) { + int32_t tot_offset = 0; + expert_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + tot_offset += problem_sizes1[i * 3 + 1]; + expert_offsets[i + 1] = tot_offset; + } +} + +void get_cutlass_w4a8_moe_mm_data_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index()); + auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device()); + torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32); + + int num_threads = min(THREADS_PER_EXPERT, topk_ids.numel()); + compute_problem_sizes_w4a8<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(problem_sizes1.data_ptr()), + static_cast(problem_sizes2.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + n, + k); + compute_expert_offsets_w4a8<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); +} diff --git a/sgl-kernel/csrc/moe/cutlass_moe_helper.cu b/sgl-kernel/csrc/moe/cutlass_moe_helper.cu new file mode 100644 index 000000000..576ad233b --- /dev/null +++ b/sgl-kernel/csrc/moe/cutlass_moe_helper.cu @@ -0,0 +1,142 @@ +#pragma once + +#include +#include +#include + +#include "cutlass/bfloat16.h" +#include "cutlass/float8.h" + +template < + typename ElementAB, + typename ElementC, + typename ElementAccumulator, + typename LayoutSFA, + typename LayoutSFB, + typename ScaleConfig> +__global__ void get_group_gemm_starts( + int32_t* expert_offsets, + ElementAB** a_offsets, + ElementAB** b_offsets, + ElementC** out_offsets, + ElementAccumulator** a_scales_offsets, + ElementAccumulator** b_scales_offsets, + ElementAB* a_base_as_int, + ElementAB* b_base_as_int, + ElementC* out_base_as_int, + ElementAccumulator* a_scales_base_as_int, + ElementAccumulator* b_scales_base_as_int, + LayoutSFA* layout_sfa_base_as_int, + LayoutSFB* layout_sfb_base_as_int, + int* problem_sizes, + int* problem_sizes_transpose, + bool transpose = false) { + int64_t expert_id = static_cast(threadIdx.x); + + if (expert_id >= gridDim.x * blockDim.x) { + return; + } + + int m = problem_sizes[expert_id * 3]; + int n = problem_sizes[expert_id * 3 + 1]; + int k = problem_sizes[expert_id * 3 + 2]; + if (transpose) { + problem_sizes_transpose[expert_id * 3] = n; + problem_sizes_transpose[expert_id * 3 + 1] = m; + problem_sizes_transpose[expert_id * 3 + 2] = k; + } + + int64_t expert_offset = static_cast(expert_offsets[expert_id]); + int64_t a_stride = 0; + int64_t b_stride = 0; + int64_t a_scale_stride = 0; + int64_t b_scale_stride = 0; + if (!transpose) { + a_stride = expert_offset * k; + b_stride = expert_id * k * n; + a_scale_stride = expert_offset * k / 128; + b_scale_stride = expert_id * k * n / 128 / 128; + } else { + a_stride = expert_id * k * n; + b_stride = expert_offset * k; + a_scale_stride = expert_id * k * n / 128 / 128; + b_scale_stride = expert_offset * k / 128; + } + a_offsets[expert_id] = a_base_as_int + a_stride; + b_offsets[expert_id] = b_base_as_int + b_stride; + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + a_scales_offsets[expert_id] = a_scales_base_as_int + a_scale_stride; + b_scales_offsets[expert_id] = b_scales_base_as_int + b_scale_stride; + + LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id; + + if (!transpose) { + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(m, n, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(m, n, k, 1)); + } else { + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA(cute::make_shape(n, m, k, 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB(cute::make_shape(n, m, k, 1)); + } +} + +#define __CALL_GET_STARTS_KERNEL(TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, ScaleConfig) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(a_ptrs.data_ptr()), \ + static_cast(b_ptrs.data_ptr()), \ + static_cast(out_ptrs.data_ptr()), \ + static_cast(a_scales_ptrs.data_ptr()), \ + static_cast(b_scales_ptrs.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + reinterpret_cast(layout_sfa.data_ptr()), \ + reinterpret_cast(layout_sfb.data_ptr()), \ + static_cast(problem_sizes.data_ptr()), \ + static_cast(problem_sizes_transpose.data_ptr()), \ + transpose); \ + } + +namespace { +template +void run_get_group_gemm_starts( + torch::Tensor const& expert_offsets, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& layout_sfa, + torch::Tensor const& layout_sfb, + torch::Tensor const& problem_sizes, + torch::Tensor& problem_sizes_transpose, + bool transpose = false) { + TORCH_CHECK(a_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(b_tensors.dtype() == torch::kFloat8_e4m3fn); + TORCH_CHECK(a_scales.dtype() == torch::kFloat32); + TORCH_CHECK(b_scales.dtype() == torch::kFloat32); + TORCH_CHECK(out_tensors.size(1) % 128 == 0 or out_tensors.size(0) % 128 == 0); + TORCH_CHECK(a_tensors.size(1) % 128 == 0 or a_tensors.size(0) % 128 == 0); + + int num_experts = (int)expert_offsets.size(0); + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + if (false) { + } + __CALL_GET_STARTS_KERNEL(torch::kBFloat16, cutlass::bfloat16_t, LayoutSFA, LayoutSFB, ScaleConfig) + __CALL_GET_STARTS_KERNEL(torch::kFloat16, half, LayoutSFA, LayoutSFB, ScaleConfig) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} +} // namespace diff --git a/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu b/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu new file mode 100644 index 000000000..b2e1fc83c --- /dev/null +++ b/sgl-kernel/csrc/moe/fp8_blockwise_moe_kernel.cu @@ -0,0 +1,810 @@ +#include +#include +#include +#include + +#include "cute/tensor.hpp" +#include "cutlass/cutlass.h" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/default_epilogue.hpp" +#include "cutlass/epilogue/dispatch_policy.hpp" +#include "cutlass/epilogue/thread/activation.h" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/gemm/kernel/tile_scheduler_params.h" +#include "cutlass/tensor_ref.h" +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/tensor_view_io.h" +#include "cutlass_moe_helper.cu" +#include "utils.h" + +using namespace cute; + +using ProblemShape = cutlass::gemm::GroupProblemShape>; + +template +void launch_sm90_fp8_blockwise_scaled_group_mm( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + using ElementA = cutlass::float_e4m3_t; + using ElementB = cutlass::float_e4m3_t; + using ElementC = void; + using ElementD = OutType; + using ElementAccumulator = float; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = LayoutD; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ArchTag = cutlass::arch::Sm90; + using OperatorClass = cutlass::arch::OpClassTensorOp; + static constexpr auto RoundStyle = cutlass::FloatRoundStyle::round_to_nearest; + using CustomEVTIdentity = // acc + cutlass::epilogue::fusion::Sm90EVT< + cutlass::epilogue::fusion:: + Sm90Compute, + cutlass::epilogue::fusion::Sm90AccFetch>; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + ElementC, // Use void to avoid load Matrix C + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentC, + typename ScheduleConfig::EpilogueSchedule, + CustomEVTIdentity>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename ScheduleConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + int num_experts = (int)expert_offsets.size(0); + Gemm gemm_op; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(stride_a.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(stride_b.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + cutlass::KernelHardwareInfo hw_info; + hw_info.device_id = c10::cuda::current_device(); + hw_info.sm_count = at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, + nullptr, + static_cast(stride_c.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(stride_c.data_ptr())}; + + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info}; + + at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + auto status = gemm_op.initialize(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void launch_sm100_fp8_blockwise_scaled_group_mm( + torch::Tensor& out_ptrs, + const torch::Tensor& a_ptrs, + const torch::Tensor& b_ptrs, + const torch::Tensor& a_scales_ptrs, + const torch::Tensor& b_scales_ptrs, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + using ProblemShape = cutlass::gemm::GroupProblemShape>; + using ElementA = cutlass::float_e4m3_t; + using ElementB = cutlass::float_e4m3_t; + using ElementC = OutType; + using ElementD = ElementC; + using ElementAccumulator = float; + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = LayoutD; + + static constexpr int AlignmentA = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentB = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + + using ArchTag = cutlass::arch::Sm100; + using OperatorClass = cutlass::arch::OpClassTensorOp; + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::epilogue::collective::EpilogueTileAuto, + ElementAccumulator, + ElementAccumulator, + void, + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentC, + typename ScheduleConfig::EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + OperatorClass, + ElementA, + cute::tuple, + AlignmentA, + ElementB, + cute::tuple, + AlignmentB, + ElementAccumulator, + typename ScheduleConfig::MmaTileShape, + typename ScheduleConfig::ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename ScheduleConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm = cutlass::gemm::device::GemmUniversalAdapter; + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + int num_experts = (int)expert_offsets.size(0); + // Create an instance of the GEMM + Gemm gemm_op; + + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(stride_a.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(stride_b.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + cutlass::KernelHardwareInfo hw_info; + + hw_info.device_id = 0; + // sm_count is the number of SMs on the current device, since we only support SM100 blackwell, so we set it to 148 + hw_info.sm_count = 148; + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, + nullptr, + static_cast(stride_c.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(stride_c.data_ptr())}; + + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info}; + + at::cuda::CUDAGuard device_guard{(char)a_ptrs.get_device()}; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a_ptrs.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + auto status = gemm_op.initialize(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +template +void sm100_fp8_blockwise_group_mm_dispatch_shape( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + // Check the first matrix size to decide on the configuration + // Assuming all matrices in the group have similar size characteristics + // bool use_small_config = a[0].size(0) <= 128; + struct MmaConfig1 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_256, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise2SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized2Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<128, 1, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + struct MmaConfig2 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + struct MmaConfig3 { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_1, _1, _1>; // Layout type for SFB matrix operand + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedBlockwise1SmSm100; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; + using ScaleConfig = + cutlass::detail::Sm100BlockwiseScaleConfig<1, 128, 128, cute::UMMA::Major::K, cute::UMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + int num_experts = (int)expert_offsets.size(0); + torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int); + torch::Tensor output_t = output.t(); + torch::Tensor a_t = a.t(); + torch::Tensor b_t = b.transpose(1, 2); + torch::Tensor scales_a_t = scales_a.t(); + torch::Tensor scales_b_t = scales_b.transpose(1, 2); + + if (a.size(0) <= 2048 && a.size(1) >= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + b_t, + a_t, + output_t, + scales_b_t, + scales_a_t, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose, + true); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes_transpose, + expert_offsets, + workspace); + output = output_t.t(); + } else if (a.size(0) > 2048 && a.size(1) >= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm100_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } +} + +template +void sm90_fp8_blockwise_group_mm_dispatch_shape( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + struct MmaConfigSmallM { + // Swap A/B + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _32, _128>; + using ClusterShape = Shape<_2, _1, _1>; + // TODO: Check Pingpong or Cooperative + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8BlockScaledAccum; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<128, 1, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + struct MmaConfigH20LargeK { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_64, _128, _128>; + using ClusterShape = Shape<_2, _1, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedPingpongFP8BlockScaledAccum; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedPingpong; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + struct MmaConfigHx00AndH20SmallK { + using ElementA = cutlass::float_e4m3_t; + using MmaTileShape = Shape<_128, _128, _128>; + using ClusterShape = Shape<_1, _2, _1>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeFP8BlockScaledAccum; + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecializedCooperative; + using ScaleConfig = + cutlass::detail::Sm90BlockwiseScaleConfig<1, 128, 128, cute::GMMA::Major::K, cute::GMMA::Major::K>; + using LayoutSFA = decltype(ScaleConfig::deduce_layoutSFA()); + using LayoutSFB = decltype(ScaleConfig::deduce_layoutSFB()); + }; + + int num_experts = (int)expert_offsets.size(0); + torch::TensorOptions options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + torch::Tensor problem_sizes_transpose = torch::empty(num_experts * 3, options_int); + torch::Tensor output_t = output.t(); + torch::Tensor a_t = a.t(); + torch::Tensor b_t = b.transpose(1, 2); + torch::Tensor scales_a_t = scales_a.t(); + torch::Tensor scales_b_t = scales_b.transpose(1, 2); + + const std::string H20_device_type_str("NVIDIA H20"); + bool is_h20_device = std::string(at::cuda::getCurrentDeviceProperties()->name) == H20_device_type_str; + + if (a.size(0) <= 2048) { + run_get_group_gemm_starts( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + b_t, + a_t, + output_t, + scales_b_t, + scales_a_t, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose, + true); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes_transpose, + expert_offsets, + workspace); + output = output_t.t(); + } else { + if (is_h20_device && a.size(1) > 128) { + // For H20 with K > 128, use Pingpong Schedule + run_get_group_gemm_starts< + MmaConfigH20LargeK::LayoutSFA, + MmaConfigH20LargeK::LayoutSFB, + MmaConfigH20LargeK::ScaleConfig>( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + // For H20 with K <= 128, and H100 & H200 & H800, use Cooperative Schedule + run_get_group_gemm_starts< + MmaConfigHx00AndH20SmallK::LayoutSFA, + MmaConfigHx00AndH20SmallK::LayoutSFB, + MmaConfigHx00AndH20SmallK::ScaleConfig>( + expert_offsets, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + output, + scales_a, + scales_b, + layout_sfa, + layout_sfb, + problem_sizes, + problem_sizes_transpose); + launch_sm90_fp8_blockwise_scaled_group_mm( + out_ptrs, + a_ptrs, + b_ptrs, + a_scales_ptrs, + b_scales_ptrs, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + } +} + +/** + * @brief Performs blockwise grouped matrix multiplication on FP8 quantized inputs, + * with per-block scaling. + * + * This function dispatches to hardware-specific implementations (e.g., SM100 FP8) + * to compute: + * C_i = scale_a[i] * A_i * scale_b[i] * B_i + * for each expert group `i`, using input `problem_sizes` and `expert_offsets` + * to describe the individual matrix dimensions and their offsets. + * + * Input tensors A and B must be quantized to 8-bit formats and dequantized before multiplication. + * The output tensor is written with bfloat16 or half precision. + * + * @param output Output tensor (must be of type bfloat16 or half). + * @param a Input tensor A (must be kFloat8_e4m3fn). + * @param b Input tensor B (must be kFloat8_e4m3fn). + * @param scales_a Scaling factors for tensor A, float32 per expert group. + * @param scales_b Scaling factors for tensor B, float32 per expert group. + * @param stride_a Stride information for tensor A (int32). + * @param stride_b Stride information for tensor B (int32). + * @param stride_c Stride information for output tensor C (int32). + * @param layout_sfa Layout descriptor for A (int32), e.g., row-major/column-major. + * @param layout_sfb Layout descriptor for B (int32). + * @param problem_sizes 2D int32 tensor of shape (num_experts, 3), specifying (M, N, K) + * for each grouped matrix multiplication problem. + * @param expert_offsets 1D int32 tensor of size (num_experts), used to index into + * the grouped input tensors for dispatch. + * @note Performance Optimization: + * If the batch size (a.size(0)) is smaller than 512, the implementation + * will internally transpose input matrices to align with the optimal memory access + * pattern for better GPU efficiency. This transformation is done within the kernel. + */ +void fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace) { + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be 2D tensor"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have shape (num_experts, 3)"); + TORCH_CHECK( + problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets"); + TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32"); + TORCH_CHECK(a.scalar_type() == torch::kFloat8_e4m3fn, "a must be kFloat8_e4m3fn"); + TORCH_CHECK(b.scalar_type() == torch::kFloat8_e4m3fn, "b must be kFloat8_e4m3fn"); + TORCH_CHECK( + output.scalar_type() == torch::kBFloat16 || output.scalar_type() == torch::kHalf, + "output must be bfloat16 or half"); + TORCH_CHECK(scales_a.scalar_type() == torch::kFloat32, "scales_a must be float32"); + TORCH_CHECK(scales_b.scalar_type() == torch::kFloat32, "scales_b must be float32"); + TORCH_CHECK(stride_a.scalar_type() == torch::kInt64, "stride_a must be int64"); + TORCH_CHECK(stride_b.scalar_type() == torch::kInt64, "stride_b must be int64"); + TORCH_CHECK(stride_c.scalar_type() == torch::kInt64, "stride_c must be int64"); + TORCH_CHECK(layout_sfa.scalar_type() == torch::kInt32, "layout_sfa must be int32"); + TORCH_CHECK(layout_sfb.scalar_type() == torch::kInt32, "layout_sfb must be int32"); + TORCH_CHECK(expert_offsets.scalar_type() == torch::kInt32, "expert_offsets must be int32"); + + TORCH_CHECK(output.dim() == 2, "output must be 2D tensor"); + TORCH_CHECK(a.dim() == 2, "a must be 2D tensor"); + TORCH_CHECK(b.dim() == 3, "b must be 3D tensor"); + TORCH_CHECK(scales_a.dim() == 2, "scales_a must be 2D tensor"); + TORCH_CHECK(scales_b.dim() == 3, "scales_b must be 3D tensor"); + TORCH_CHECK(stride_a.dim() == 1, "stride_a must be 1D tensor"); + TORCH_CHECK(stride_b.dim() == 1, "stride_b must be 1D tensor"); + TORCH_CHECK(stride_c.dim() == 1, "stride_c must be 1D tensor"); + TORCH_CHECK(layout_sfa.dim() == 2, "layout_sfa must be 1D tensor"); + TORCH_CHECK(layout_sfb.dim() == 2, "layout_sfb must be 1D tensor"); + TORCH_CHECK(a_ptrs.dim() == 1, "a_ptrs must be 1D tensor"); + TORCH_CHECK(b_ptrs.dim() == 1, "b_ptrs must be 1D tensor"); + TORCH_CHECK(out_ptrs.dim() == 1, "out_ptrs must be 1D tensor"); + TORCH_CHECK(a_scales_ptrs.dim() == 1, "a_scales_ptrs must be 1D tensor"); + TORCH_CHECK(b_scales_ptrs.dim() == 1, "b_scales_ptrs must be 1D tensor"); + TORCH_CHECK(expert_offsets.dim() == 1, "expert_offsets must be 1D tensor"); + TORCH_CHECK(workspace.dim() == 1, "workspace must be 1D tensor"); + + bool can_implement = false; + auto sm_version = getSMVersion(); + +#if defined(CUTLASS_ARCH_MMA_SM100A_SUPPORTED) || defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED) +#if defined CUDA_VERSION && CUDA_VERSION >= 12080 + if (sm_version == 100 +#if CUDA_VERSION >= 12090 + || sm_version == 103 +#endif + ) { + if (output.scalar_type() == torch::kBFloat16) { + sm100_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + sm100_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + can_implement = true; + } +#endif +#endif + +#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED) && defined(CUTLASS_ARCH_MMA_MODIFIABLE_TMA_SM90_SUPPORTED) + if (sm_version == 90) { + if (output.scalar_type() == torch::kBFloat16) { + sm90_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } else { + sm90_fp8_blockwise_group_mm_dispatch_shape( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace); + } + can_implement = true; + } +#endif + TORCH_CHECK_NOT_IMPLEMENTED( + can_implement, "No implemented fp8_blockwise_scaled_grouped_mm for current compute capability: ", sm_version); +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py b/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py new file mode 100644 index 000000000..b3ed863a3 --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/generate_kernels.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +import glob +import itertools +import os +import subprocess + +import jinja2 + +FILE_HEAD = """ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { +""".strip() + +TEMPLATE = ( + "template __global__ void Marlin<" + "{{scalar_t}}, " + "{{w_type_id}}, " + "{{threads}}, " + "{{thread_m_blocks}}, " + "{{thread_n_blocks}}, " + "{{thread_k_blocks}}, " + "{{'true' if m_block_size_8 else 'false'}}, " + "{{stages}}, " + "{{'true' if has_act_order else 'false'}}, " + "{{'true' if has_zp else 'false'}}, " + "{{group_blocks}}, " + "{{'true' if is_zp_float else 'false'}}>" + "( MARLIN_KERNEL_PARAMS );" +) + +KERNEL_FILE_TEMPLATE = ( + "// auto generated by generate.py\n" + "// clang-format off\n" + "#pragma once\n\n" + "{% for kernel_file in kernel_files %}" + '#include "{{ kernel_file }}"\n' + "{% endfor %}" +) + +KERNEL_FILE_NAME = "kernel_marlin.cuh" + +# int8 with zero point case (sglang::kU8) is also supported, +# we don't add it to reduce wheel size. +SCALAR_TYPES = ["sglang::kU4", "sglang::kU4B8", "sglang::kU8B128"] +THREAD_CONFIGS = [(128, 128, 256), (64, 256, 256), (64, 128, 128)] + +THREAD_M_BLOCKS = [0.5, 1, 2, 3, 4] +# group_blocks: +# = 0 : act order case +# = -1 : channelwise quantization +# > 0 : group_size=16*group_blocks +GROUP_BLOCKS = [0, -1, 2, 4, 8] +DTYPES = ["fp16", "bf16"] + + +def remove_old_kernels(): + for filename in glob.glob(os.path.dirname(__file__) + "/kernel_*.cuh"): + subprocess.call(["rm", "-f", filename]) + + +def generate_new_kernels(): + kernel_files = set() + for scalar_type, dtype in itertools.product(SCALAR_TYPES, DTYPES): + has_zp = "B" not in scalar_type + all_template_str_list = [] + + for group_blocks, m_blocks, thread_configs in itertools.product( + GROUP_BLOCKS, THREAD_M_BLOCKS, THREAD_CONFIGS + ): + + has_act_order = group_blocks == 0 + if has_zp and has_act_order: + continue + if thread_configs[2] == 256: + if m_blocks <= 1 and thread_configs[0] != 128: + continue + if m_blocks > 1 and thread_configs[0] != 64: + continue + + k_blocks = thread_configs[0] // 16 + n_blocks = thread_configs[1] // 16 + threads = thread_configs[2] + + c_dtype = "half" if dtype == "fp16" else "nv_bfloat16" + + template_str = jinja2.Template(TEMPLATE).render( + scalar_t=c_dtype, + w_type_id=scalar_type + ".id()", + threads=threads, + thread_m_blocks=max(m_blocks, 1), + thread_n_blocks=n_blocks, + thread_k_blocks=k_blocks, + m_block_size_8=m_blocks == 0.5, + stages="pipe_stages", + has_act_order=has_act_order, + has_zp=has_zp, + group_blocks=group_blocks, + is_zp_float=False, + ) + + all_template_str_list.append(template_str) + + file_content = FILE_HEAD + "\n\n" + file_content += "\n\n".join(all_template_str_list) + "\n\n}\n" + filename = f"kernel_{dtype}_{scalar_type[8:].lower()}.cuh" + + with open(os.path.join(os.path.dirname(__file__), filename), "w") as f: + f.write(file_content) + kernel_files.add(filename) + + kernel_files = list(kernel_files) + kernel_files.sort() + + file_content = jinja2.Template(KERNEL_FILE_TEMPLATE).render( + kernel_files=kernel_files + ) + with open(os.path.join(os.path.dirname(__file__), KERNEL_FILE_NAME), "w") as f: + f.write(file_content) + + +if __name__ == "__main__": + remove_old_kernels() + generate_new_kernels() diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h new file mode 100644 index 000000000..afa7c377b --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel.h @@ -0,0 +1,41 @@ +#pragma once + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16 +#endif + +#include "gemm/marlin/marlin.cuh" +#include "gemm/marlin/marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define MARLIN_KERNEL_PARAMS \ + const int4 *__restrict__ A, const int4 *__restrict__ B, int4 *__restrict__ C, int4 *__restrict__ C_tmp, \ + const int4 *__restrict__ scales_ptr, const int4 *__restrict__ zp_ptr, const int *__restrict__ g_idx, \ + const int32_t *__restrict__ sorted_token_ids_ptr, const int32_t *__restrict__ expert_ids_ptr, \ + const int32_t *__restrict__ num_tokens_past_padded_ptr, const float *__restrict__ topk_weights_ptr, int top_k, \ + bool mul_topk_weights, bool is_ep, int num_groups, int prob_m, int prob_n, int prob_k, int *locks, \ + bool use_atomic_add, bool use_fp32_reduce + +namespace MARLIN_NAMESPACE_NAME { +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const bool has_act_order, // whether act_order is enabled + const bool has_zp, // whether zero-points are enabled + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin(MARLIN_KERNEL_PARAMS); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh new file mode 100644 index 000000000..7e83bed8f --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4.cuh @@ -0,0 +1,90 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh new file mode 100644 index 000000000..60e2dea31 --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku4b8.cuh @@ -0,0 +1,110 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh new file mode 100644 index 000000000..7eb6b18de --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_bf16_ku8b128.cuh @@ -0,0 +1,110 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh new file mode 100644 index 000000000..ec41e018b --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4.cuh @@ -0,0 +1,90 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh new file mode 100644 index 000000000..7df28701b --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku4b8.cuh @@ -0,0 +1,110 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh new file mode 100644 index 000000000..1150844e2 --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_fp16_ku8b128.cuh @@ -0,0 +1,110 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel.h" +#include "marlin_template.h" + +namespace MARLIN_NAMESPACE_NAME { + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +template __global__ void Marlin( MARLIN_KERNEL_PARAMS ); + +} diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh new file mode 100644 index 000000000..bb828dc5b --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/kernel_marlin.cuh @@ -0,0 +1,10 @@ +// auto generated by generate.py +// clang-format off +#pragma once + +#include "kernel_bf16_ku4.cuh" +#include "kernel_bf16_ku4b8.cuh" +#include "kernel_bf16_ku8b128.cuh" +#include "kernel_fp16_ku4.cuh" +#include "kernel_fp16_ku4b8.cuh" +#include "kernel_fp16_ku8b128.cuh" diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h b/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h new file mode 100644 index 000000000..ade562af6 --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/marlin_template.h @@ -0,0 +1,1805 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ +#pragma once + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16 +#endif + +#include "gemm/marlin/marlin.cuh" +#include "gemm/marlin/marlin_dtypes.cuh" +#include "scalar_type.hpp" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const bool has_act_order, // whether act_order is enabled + const bool has_zp, // whether zero-points are enabled + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids + const int32_t* __restrict__ expert_ids_ptr, // moe expert ids + const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens + const float* __restrict__ topk_weights_ptr, // moe top weights + int top_k, // num of experts per token + bool mul_topk_weights, // mul topk weights or not + bool is_ep, // expert parallelism + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce // whether to use fp32 global reduce +) {} + +} // namespace MARLIN_NAMESPACE_NAME + +#else + +// m16n8k16 tensor core mma instruction with fp16 inputs and fp32 +// output/accumulation. +template +__device__ inline void +mma(const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]), "f"(c[0]), "f"(c[1]), "f"(c[2]), "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +template +__device__ inline void mma_trans( + const typename ScalarType::FragA& a_frag, + const typename ScalarType::FragB& frag_b, + const typename ScalarType::FragB& frag_b2, + typename ScalarType::FragC& frag_c) { + const uint32_t* a = reinterpret_cast(&a_frag); + const uint32_t* b = reinterpret_cast(&frag_b); + const uint32_t* b2 = reinterpret_cast(&frag_b2); + float* c = reinterpret_cast(&frag_c); + if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.f16.f16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else if constexpr (std::is_same::value) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" + : "=f"(c[0]), "=f"(c[1]), "=f"(c[2]), "=f"(c[3]) + : "r"(b[0]), + "r"(b2[0]), + "r"(b[1]), + "r"(b2[1]), + "r"(a[0]), + "r"(a[1]), + "f"(c[0]), + "f"(c[1]), + "f"(c[2]), + "f"(c[3])); + } else { + STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t); + } +} + +// Instruction for loading a full 16x16 matrix fragment of operand A from shared +// memory, directly in tensor core layout. +template +__device__ inline void ldsm(typename ScalarType::FragA& frag_a, const void* smem_ptr) { + uint32_t* a = reinterpret_cast(&frag_a); + uint32_t smem = static_cast(__cvta_generic_to_shared(smem_ptr)); + if constexpr (count == 4) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(a[0]), "=r"(a[1]), "=r"(a[2]), "=r"(a[3]) + : "r"(smem)); + } else if constexpr (count == 2) { + asm volatile("ldmatrix.sync.aligned.m8n8.x2.shared.b16 {%0,%1}, [%2];\n" : "=r"(a[0]), "=r"(a[1]) : "r"(smem)); + } else if constexpr (count == 1) { + asm volatile("ldmatrix.sync.aligned.m8n8.x1.shared.b16 {%0}, [%1];\n" : "=r"(a[0]) : "r"(smem)); + } else { + static_assert(count == 1 || count == 2 || count == 4, "invalid count"); + } +} + +// Lookup-table based 3-input logical operation; explicitly used for +// dequantization as the compiler does not seem to automatically recognize it in +// all cases. +template +__device__ inline int lop3(int a, int b, int c) { + int res; + asm volatile("lop3.b32 %0, %1, %2, %3, %4;\n" : "=r"(res) : "r"(a), "r"(b), "r"(c), "n"(lut)); + return res; +} + +// Constructs destination register by taking bytes from 2 sources (based on +// mask) +template +__device__ inline uint32_t prmt(uint32_t a) { + uint32_t res; + asm volatile("prmt.b32 %0, %1, %2, %3;\n" : "=r"(res) : "r"(a), "n"(start_byte), "n"(mask)); + return res; +} + +template +__device__ inline typename ScalarType::FragB dequant(int q, typename ScalarType::FragB& frag_b); + +// +// Efficiently dequantize 4bit values packed in an int32 value into a full +// B-fragment of 4 fp16 values. We mostly follow the strategy in the link below, +// with some small changes: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L215-L287 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L327-L385 +// +template <> +__device__ inline typename ScalarType::FragB dequant(int q, typename ScalarType::FragB& frag_b) { + const int LO = 0x000f000f; + const int HI = 0x00f000f0; + const int EX = 0x64006400; + // Guarantee that the `(a & b) | c` operations are LOP3s. + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, LO, EX); + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, HI, EX); + // We want signed int4 outputs, hence we fuse the `-8` symmetric zero point + // directly into `SUB` and `ADD`. + const int SUB = 0x64086408; + const int MUL = 0x2c002c00; + const int ADD = 0xd480d480; + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&SUB)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), *reinterpret_cast(&MUL), *reinterpret_cast(&ADD)); + return frag_b; +} + +template <> +__device__ inline typename ScalarType::FragB +dequant(int q, typename ScalarType::FragB& frag_b) { + static constexpr uint32_t MASK = 0x000f000f; + static constexpr uint32_t EX = 0x43004300; + + // Guarantee that the `(a & b) | c` operations are LOP3s. + + int lo = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + q >>= 4; + int hi = lop3<(0xf0 & 0xcc) | 0xaa>(q, MASK, EX); + + static constexpr uint32_t MUL = 0x3F803F80; + static constexpr uint32_t ADD = 0xC308C308; + + frag_b[0] = __hfma2( + *reinterpret_cast(&lo), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + frag_b[1] = __hfma2( + *reinterpret_cast(&hi), + *reinterpret_cast(&MUL), + *reinterpret_cast(&ADD)); + return frag_b; +} + +// +// Fast Int8ToFp16/Int8ToBf16: Efficiently dequantize 8bit int values to fp16 or +// bf16 Reference: +// - FP16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L53-L85 +// - BF16: +// https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/cutlass_extensions/include/cutlass_extensions/interleaved_numeric_conversion.h#L125-L175 +// +template <> +__device__ inline typename ScalarType::FragB dequant(int q, typename ScalarType::FragB& frag_b) { + static constexpr uint32_t mask_for_elt_01 = 0x5250; + static constexpr uint32_t mask_for_elt_23 = 0x5351; + static constexpr uint32_t start_byte_for_fp16 = 0x64646464; + + uint32_t lo = prmt(q); + uint32_t hi = prmt(q); + + static constexpr uint32_t I8s_TO_F16s_MAGIC_NUM = 0x64806480; + + frag_b[0] = __hsub2(*reinterpret_cast(&lo), *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + frag_b[1] = __hsub2(*reinterpret_cast(&hi), *reinterpret_cast(&I8s_TO_F16s_MAGIC_NUM)); + return frag_b; +} + +template <> +__device__ inline typename ScalarType::FragB +dequant(int q, typename ScalarType::FragB& frag_b) { + float fp32_intermediates[4]; + uint32_t* fp32_intermediates_casted = reinterpret_cast(fp32_intermediates); + + static constexpr uint32_t fp32_base = 0x4B000000; + fp32_intermediates_casted[0] = __byte_perm(q, fp32_base, 0x7650); + fp32_intermediates_casted[1] = __byte_perm(q, fp32_base, 0x7652); + fp32_intermediates_casted[2] = __byte_perm(q, fp32_base, 0x7651); + fp32_intermediates_casted[3] = __byte_perm(q, fp32_base, 0x7653); + + fp32_intermediates[0] -= 8388736.f; + fp32_intermediates[1] -= 8388736.f; + fp32_intermediates[2] -= 8388736.f; + fp32_intermediates[3] -= 8388736.f; + + uint32_t* bf16_result_ptr = reinterpret_cast(&frag_b); + bf16_result_ptr[0] = __byte_perm(fp32_intermediates_casted[0], fp32_intermediates_casted[1], 0x7632); + bf16_result_ptr[1] = __byte_perm(fp32_intermediates_casted[2], fp32_intermediates_casted[3], 0x7632); + + return frag_b; +} + +// Multiply dequantized values by the corresponding quantization scale; used +// only for grouped quantization. +template +__device__ inline void +scale(typename ScalarType::FragB& frag_b, typename ScalarType::FragS& frag_s, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s = ScalarType::num2num2(reinterpret_cast(&frag_s)[i]); + frag_b[0] = __hmul2(frag_b[0], s); + frag_b[1] = __hmul2(frag_b[1], s); +} + +template +__device__ inline void scale_and_sub(typename ScalarType::FragB& frag_b, scalar_t s, scalar_t zp) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s2 = ScalarType::num2num2(s); + scalar_t2 zp2 = ScalarType::num2num2(zp); + frag_b[0] = __hfma2(frag_b[0], s2, __hneg2(zp2)); + frag_b[1] = __hfma2(frag_b[1], s2, __hneg2(zp2)); +} + +template +__device__ inline void +sub_zp(typename ScalarType::FragB& frag_b, typename ScalarType::scalar_t2& frag_zp, int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 zp = ScalarType::num2num2(reinterpret_cast(&frag_zp)[i]); + frag_b[0] = __hsub2(frag_b[0], zp); + frag_b[1] = __hsub2(frag_b[1], zp); +} + +// Same as above, but for act_order (each K is multiplied individually) +template +__device__ inline void scale4( + typename ScalarType::FragB& frag_b, + typename ScalarType::FragS& frag_s_1, + typename ScalarType::FragS& frag_s_2, + typename ScalarType::FragS& frag_s_3, + typename ScalarType::FragS& frag_s_4, + int i) { + using scalar_t2 = typename ScalarType::scalar_t2; + scalar_t2 s_val_1_2; + s_val_1_2.x = reinterpret_cast(&frag_s_1)[i]; + s_val_1_2.y = reinterpret_cast(&frag_s_2)[i]; + + scalar_t2 s_val_3_4; + s_val_3_4.x = reinterpret_cast(&frag_s_3)[i]; + s_val_3_4.y = reinterpret_cast(&frag_s_4)[i]; + + frag_b[0] = __hmul2(frag_b[0], s_val_1_2); + frag_b[1] = __hmul2(frag_b[1], s_val_3_4); +} + +// Given 2 floats multiply by 2 scales (halves) +template +__device__ inline void scale_float(float* c, typename ScalarType::FragS& s) { + scalar_t* s_ptr = reinterpret_cast(&s); + c[0] = __fmul_rn(c[0], ScalarType::num2float(s_ptr[0])); + c[1] = __fmul_rn(c[1], ScalarType::num2float(s_ptr[1])); +} + +// Wait until barrier reaches `count`, then lock for current threadblock. +__device__ inline void barrier_acquire(int* lock, int count) { + if (threadIdx.x == 0) { + int state = -1; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state != count); + } + __syncthreads(); +} + +// Release barrier and increment visitation count. +__device__ inline void barrier_release(int* lock, bool reset = false) { + __syncthreads(); + if (threadIdx.x == 0) { + if (reset) { + lock[0] = 0; + return; + } + int val = 1; + // Make sure that all writes since acquiring this barrier are visible + // globally, while releasing the barrier. + asm volatile("fence.acq_rel.gpu;\n"); + asm volatile("red.relaxed.gpu.global.add.s32 [%0], %1;\n" : : "l"(lock), "r"(val)); + } +} + +// Wait until value of lock to be negative, and then add 1 +__device__ inline void wait_negative_and_add(int* lock) { + if (threadIdx.x == 0) { + int state = 0; + do + // Guarantee that subsequent writes by this threadblock will be visible + // globally. + asm volatile("ld.global.acquire.gpu.b32 %0, [%1];\n" : "=r"(state) : "l"(lock)); + while (state >= 0); + atomicAdd(lock, 1); + } + __syncthreads(); +} + +template < + typename scalar_t, // compute dtype, half or nv_float16 + const sglang::ScalarTypeId w_type_id, // weight ScalarType id + const int threads, // number of threads in a threadblock + const int thread_m_blocks, // number of 16x16 blocks in the m + // dimension (batchsize) of the + // threadblock + const int thread_n_blocks, // same for n dimension (output) + const int thread_k_blocks, // same for k dimension (reduction) + const bool m_block_size_8, // whether m_block_size == 8 + // only works when thread_m_blocks == 1 + const int stages, // number of stages for the async global->shared + // fetch pipeline + const bool has_act_order, // whether act_order is enabled + const bool has_zp, // whether zero-points are enabled + const int group_blocks, // number of consecutive 16x16 blocks + // with a separate quantization scale + const bool is_zp_float // is zero point of float16 type? + > +__global__ void Marlin( + const int4* __restrict__ A, // fp16 input matrix of shape mxk + const int4* __restrict__ B, // 4bit quantized weight matrix of shape kxn + int4* __restrict__ C, // fp16 output buffer of shape mxn + int4* __restrict__ C_tmp, // fp32 tmp output buffer (for reduce) + const int4* __restrict__ scales_ptr, // fp16 quantization scales of shape + // (k/groupsize)xn + const int4* __restrict__ zp_ptr, // 4bit packed zero-points of shape + // (k/groupsize)x(n/pack_factor) + const int* __restrict__ g_idx, // int32 group indices of shape k + const int32_t* __restrict__ sorted_token_ids_ptr, // moe sorted_ids + const int32_t* __restrict__ expert_ids_ptr, // moe expert ids + const int32_t* __restrict__ num_tokens_past_padded_ptr, // moe num tokens + const float* __restrict__ topk_weights_ptr, // moe top weights + int top_k, // num of experts per token + bool mul_topk_weights, // mul topk weights or not + bool is_ep, // expert parallelism + int num_groups, // number of scale groups per output channel + int prob_m, // batch dimension m + int prob_n, // output dimension n + int prob_k, // reduction dimension k + int* locks, // extra global storage for barrier synchronization + bool use_atomic_add, // whether to use atomic add to reduce + bool use_fp32_reduce // whether to use fp32 global reduce +) { + // Each threadblock processes one "stripe" of the B matrix with (roughly) the + // same size, which might involve multiple column "slices" (of width 16 * + // `thread_n_blocks`). Stripes are defined as shown in the 3x3 matrix 5 SM + // example: + // 0 1 3 + // 0 2 3 + // 1 2 4 + // While this kind of partitioning makes things somewhat more complicated, it + // ensures good utilization of all SMs for many kinds of shape and GPU + // configurations, while requiring as few slow global cross-threadblock + // reductions as possible. + using Dtype = ScalarType; + using scalar_t2 = typename ScalarType::scalar_t2; + using FragA = typename ScalarType::FragA; + using FragB = typename ScalarType::FragB; + using FragC = typename ScalarType::FragC; + using FragS = typename ScalarType::FragS; + using FragZP = typename ScalarType::FragZP; + + extern __shared__ int4 sh[]; + static constexpr auto w_type = sglang::ScalarType::from_id(w_type_id); + + constexpr int pack_factor = 32 / w_type.size_bits(); + static_assert(thread_m_blocks == 1 || !m_block_size_8); + constexpr int moe_block_size = m_block_size_8 ? 8 : (16 * thread_m_blocks); + const int group_size = (!has_act_order && group_blocks == -1) ? prob_k : prob_k / num_groups; + const int scales_expert_stride = prob_n * prob_k / group_size / 8; + const int zp_expert_stride = + is_zp_float ? prob_n * prob_k / group_size / 8 : prob_n * prob_k / group_size / (pack_factor * 4); + + // parallel: num valid moe blocks + int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + int parallel = num_tokens_past_padded / moe_block_size; + int num_valid_blocks = parallel; + if (is_ep) { + for (int i = 0; i < parallel; i++) { + if (expert_ids_ptr[i] == -1) num_valid_blocks--; + } + } + int num_invalid_blocks = parallel - num_valid_blocks; + parallel = num_valid_blocks; + + int k_tiles = prob_k / 16 / thread_k_blocks; + int n_tiles = prob_n / 16 / thread_n_blocks; + int iters = div_ceil(k_tiles * n_tiles * parallel, gridDim.x); + + if constexpr (!has_act_order && group_blocks != -1) { + if (group_blocks >= thread_k_blocks) { + // Ensure that the number of tiles in each stripe is a multiple of the + // groupsize; this avoids an annoying special case where a stripe starts + // in the middle of group. + iters = (group_blocks / thread_k_blocks) * div_ceil(iters, (group_blocks / thread_k_blocks)); + } + } + + int slice_row = (iters * blockIdx.x) % k_tiles; + int slice_col_par = (iters * blockIdx.x) / k_tiles; + int slice_col = slice_col_par; + int slice_iters; // number of threadblock tiles in the current slice + int slice_count = 0; // total number of active threadblocks in the current slice + int slice_idx; // index of threadblock in current slice; numbered bottom to + // top + + int par_id = 0; + int block_id = -1; + int64_t expert_id = 0; // use int64 to avoid computation result overflow + int old_expert_id = 0; + int64_t B_expert_off = 0; + + int4* sh_block_sorted_ids_int4 = sh; + int32_t* sh_block_sorted_ids = reinterpret_cast(sh_block_sorted_ids_int4); + int4* sh_block_topk_weights_int4 = sh_block_sorted_ids_int4 + moe_block_size / 4; + scalar_t2* sh_block_topk_weights = reinterpret_cast(sh_block_topk_weights_int4); + int4* sh_new = sh_block_topk_weights_int4 + moe_block_size / 4; + + int32_t block_num_valid_tokens = 0; + int32_t locks_off = 0; + + // We can easily implement parallel problem execution by just remapping + // indices and advancing global pointers + if (slice_col_par >= n_tiles) { + slice_col = slice_col_par % n_tiles; + par_id = slice_col_par / n_tiles; + } + if (parallel * n_tiles >= gridDim.x) { + // when parallel * n_tiles >= sms + // then there are at most $sms$ conflict tile blocks + locks_off = blockIdx.x; + } else { + locks_off = (iters * blockIdx.x) / k_tiles - 1; + } + + // read moe block data given block_id + // block_sorted_ids / block_num_valid_tokens / block_topk_weights + auto read_moe_block_data = [&](int block_id) { + block_num_valid_tokens = moe_block_size; +#pragma unroll + for (int i = 0; i < moe_block_size / 4; i++) { + int4 sorted_token_ids_int4 = + reinterpret_cast(sorted_token_ids_ptr)[block_id * moe_block_size / 4 + i]; + int* sorted_token_ids = reinterpret_cast(&sorted_token_ids_int4); +#pragma unroll + for (int j = 0; j < 4; j++) { + if (sorted_token_ids[j] >= prob_m * top_k) { + block_num_valid_tokens = i * 4 + j; + break; + } + } + if (block_num_valid_tokens != moe_block_size) break; + } + + __syncthreads(); + int tid4 = threadIdx.x / 4; + if (threadIdx.x % 4 == 0 && threadIdx.x < block_num_valid_tokens) { + sh_block_sorted_ids_int4[tid4] = + reinterpret_cast(sorted_token_ids_ptr)[block_id * moe_block_size / 4 + tid4]; + + if (mul_topk_weights) { +#pragma unroll + for (int i = 0; i < 4; i++) { + sh_block_topk_weights[tid4 * 4 + i] = + Dtype::num2num2(Dtype::float2num(topk_weights_ptr[sh_block_sorted_ids[tid4 * 4 + i]])); + } + } + } + __syncthreads(); + }; + + // when move to next moe block, find the next block_id and expert_id + // and then read moe block data + auto update_next_moe_block_data = [&]() { + if (par_id >= parallel) return; + + old_expert_id = expert_id; + if (num_invalid_blocks > 0) { + int skip_count = block_id == -1 ? par_id : 0; + block_id++; + for (int i = block_id; i < num_tokens_past_padded / moe_block_size; i++) { + expert_id = expert_ids_ptr[i]; + if (expert_id != -1) { + if (skip_count == 0) { + block_id = i; + break; + }; + skip_count--; + }; + } + } else { + block_id = par_id; + expert_id = expert_ids_ptr[block_id]; + } + + B_expert_off = expert_id * prob_n * prob_k / (pack_factor * 4); + scales_ptr += (expert_id - old_expert_id) * scales_expert_stride; + if constexpr (has_zp) { + zp_ptr += (expert_id - old_expert_id) * zp_expert_stride; + } + if constexpr (has_act_order) { + g_idx += (expert_id - old_expert_id) * prob_k; + } + + read_moe_block_data(block_id); + }; + + // Compute all information about the current slice which is required for + // synchronization. + auto init_slice = [&](bool first_init = false) { + slice_iters = iters * (blockIdx.x + 1) - (k_tiles * slice_col_par + slice_row); + if (slice_iters < 0 || slice_col_par >= n_tiles * parallel) slice_iters = 0; + if (slice_iters == 0) return; + if (slice_row + slice_iters > k_tiles) slice_iters = k_tiles - slice_row; + slice_count = 1; + slice_idx = 0; + int col_first = iters * div_ceil(k_tiles * slice_col_par, iters); + if (col_first <= k_tiles * (slice_col_par + 1)) { + int col_off = col_first - k_tiles * slice_col_par; + slice_count = div_ceil(k_tiles - col_off, iters); + if (col_off > 0) slice_count++; + int delta_first = iters * blockIdx.x - col_first; + if (delta_first < 0 || (col_off == 0 && delta_first == 0)) + slice_idx = slice_count - 1; + else { + slice_idx = slice_count - 1 - delta_first / iters; + if (col_off > 0) slice_idx--; + } + } + if (parallel * n_tiles >= gridDim.x) { + if (slice_count > 1 && slice_idx == slice_count - 1) { + locks_off++; + } + } else { + locks_off++; + } + + if (first_init && use_atomic_add && slice_count > 1 && slice_idx == 0) { + constexpr int threads_per_m = 16 * thread_n_blocks / 8; + int m_per_thread = div_ceil(block_num_valid_tokens, threads / threads_per_m); + for (int i = 0; i < m_per_thread; i++) { + int row = threads / threads_per_m * i + threadIdx.x / threads_per_m; + if (row < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[row]; + int col = slice_col * 16 * thread_n_blocks / 8 + threadIdx.x % threads_per_m; + C[sorted_row * prob_n / 8 + col] = {0, 0, 0, 0}; + } + } + // After write zero to output, write a negative value to lock. + // Every SM that processes the same slice would wait for + // the negative value, and then atomicAdd 1 to it. + // After all SMs are processed, the lock value would back to 0 again. + __syncthreads(); + if (threadIdx.x == 0) locks[locks_off] = 1 - slice_count; + } + + if (slice_col == n_tiles) { + slice_col = 0; + par_id++; + update_next_moe_block_data(); + } + }; + + update_next_moe_block_data(); + init_slice(true); + + // A sizes/strides + + // stride of the A matrix in global memory + int a_gl_stride = prob_k / 8; + // stride of an A matrix tile in shared memory + constexpr int a_sh_stride = 16 * thread_k_blocks / 8; + // delta between subsequent A tiles in global memory + constexpr int a_gl_rd_delta_o = 16 * thread_k_blocks / 8; + // between subsequent accesses within a tile + int a_gl_rd_delta_i = a_gl_stride * (threads / a_gl_rd_delta_o); + // between shared memory writes + constexpr int a_sh_wr_delta = a_sh_stride * (threads / a_gl_rd_delta_o); + // between shared memory tile reads + constexpr int a_sh_rd_delta_o = 2 * ((threads / 32) / (thread_n_blocks / 4)); + // within a shared memory tile + constexpr int a_sh_rd_delta_i = a_sh_stride * 16; + // overall size of a tile + constexpr int a_sh_stage = a_sh_stride * (16 * thread_m_blocks); + // number of shared write iterations for a tile + constexpr int a_sh_wr_iters = div_ceil(a_sh_stage, a_sh_wr_delta); + + // B sizes/strides + int b_gl_stride = 16 * prob_n / (pack_factor * 4); + constexpr int b_sh_stride = ((thread_n_blocks * 16) * 16 / pack_factor) / 4; + constexpr int b_thread_vecs = w_type.size_bits() == 4 ? 1 : 2; + constexpr int b_sh_stride_threads = b_sh_stride / b_thread_vecs; + + int b_gl_rd_delta_o = b_gl_stride * thread_k_blocks; + int b_gl_rd_delta_i = b_gl_stride * (threads / b_sh_stride_threads); + constexpr int b_sh_wr_delta = threads * b_thread_vecs; + constexpr int b_sh_rd_delta = threads * b_thread_vecs; + constexpr int b_sh_stage = b_sh_stride * thread_k_blocks; + constexpr int b_sh_wr_iters = b_sh_stage / b_sh_wr_delta; + + // Scale sizes/strides without act_order + int s_gl_stride = prob_n / 8; + constexpr int s_sh_stride = 16 * thread_n_blocks / 8; + constexpr int s_tb_groups = + !has_act_order && group_blocks != -1 && group_blocks < thread_k_blocks ? thread_k_blocks / group_blocks : 1; + constexpr int s_sh_stage = s_tb_groups * s_sh_stride; + int s_gl_rd_delta = s_gl_stride; + + // Scale size/strides with act_order + constexpr int tb_k = 16 * thread_k_blocks; + constexpr int g_idx_stage = has_act_order ? (tb_k * sizeof(int)) / 16 : 0; + // constexpr int act_s_row_stride = 1; + // int act_s_col_stride = act_s_row_stride * num_groups; + int act_s_col_stride = 1; + int act_s_col_warp_stride = act_s_col_stride * 8; + int tb_n_warps = thread_n_blocks / 4; + int act_s_col_tb_stride = act_s_col_warp_stride * tb_n_warps; + + // Zero-points sizes/strides + int zp_gl_stride = is_zp_float ? prob_n / 8 : (prob_n / pack_factor) / 4; + constexpr int zp_sh_stride = is_zp_float ? 16 * thread_n_blocks / 8 : ((16 * thread_n_blocks) / pack_factor) / 4; + constexpr int zp_tb_groups = s_tb_groups; + constexpr int zp_sh_stage = has_zp ? zp_tb_groups * zp_sh_stride : 0; + int zp_gl_rd_delta = zp_gl_stride; + + // Global A read index of current thread. + int a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + a_gl_rd += a_gl_rd_delta_o * slice_row; + // Shared write index of current thread. + int a_sh_wr = a_sh_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); + // Shared read index. + int a_sh_rd = a_sh_stride * ((threadIdx.x % 32) % (16 / (m_block_size_8 ? 2 : 1))) + + (threadIdx.x % 32) / (16 / (m_block_size_8 ? 2 : 1)); + a_sh_rd += 2 * ((threadIdx.x / 32) / (thread_n_blocks / 4)); + + int b_gl_rd = b_gl_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads) * b_thread_vecs; + b_gl_rd += b_sh_stride * slice_col; + b_gl_rd += b_gl_rd_delta_o * slice_row; + int b_sh_wr = threadIdx.x * b_thread_vecs; + int b_sh_rd = threadIdx.x * b_thread_vecs; + + // For act_order + constexpr int k_iter_size = tb_k / b_sh_wr_iters; + int slice_k_start = tb_k * slice_row; + int slice_k_finish = slice_k_start + tb_k * slice_iters; + int slice_k_start_shared_fetch = slice_k_start; + int slice_n_offset = act_s_col_tb_stride * slice_col; + + // No act_order + int s_gl_rd; + if constexpr (!has_act_order) { + if constexpr (group_blocks == -1) { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + } else { + s_gl_rd = s_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) + s_sh_stride * slice_col + threadIdx.x; + } + } + int s_sh_wr = threadIdx.x; + bool s_sh_wr_pred = threadIdx.x < s_sh_stride; + + // Zero-points + int zp_gl_rd; + if constexpr (has_zp) { + if constexpr (group_blocks == -1) { + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } else { + zp_gl_rd = zp_gl_stride * ((thread_k_blocks * slice_row) / group_blocks) + zp_sh_stride * slice_col + threadIdx.x; + } + } + int zp_sh_wr = threadIdx.x; + bool zp_sh_wr_pred = threadIdx.x < zp_sh_stride; + + // We use a different scale layout for grouped and column-wise quantization as + // we scale a `half2` tile in column-major layout in the former and in + // row-major in the latter case. + int s_sh_rd; + if constexpr (group_blocks != -1) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + else if constexpr (group_blocks == -1 && (m_block_size_8 || has_zp)) + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 8; + else + s_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) % 4; + + // Zero-points have the same read layout as the scales + // (without column-wise case) + constexpr int num_col_threads = 8; + constexpr int num_row_threads = 4; + constexpr int num_ints_per_thread = 8 / pack_factor; + int zp_sh_rd; + if constexpr (has_zp) { + if constexpr (is_zp_float) { + if constexpr (group_blocks != -1) { + zp_sh_rd = 8 * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + (threadIdx.x % 32) / 4; + } + } else { + zp_sh_rd = num_ints_per_thread * num_col_threads * ((threadIdx.x / 32) % (thread_n_blocks / 4)) + + num_ints_per_thread * ((threadIdx.x % 32) / num_row_threads); + } + } + + // To ensure that writing and reading A tiles to/from shared memory, the + // latter in fragment format, is fully bank conflict free, we need to use a + // rather fancy XOR-based layout. The key here is that neither reads nor + // writes of the 16-byte `int4` blocks of 8 consecutive threads involve the + // same shared memory banks. Further, it seems (based on NSight-Compute) that + // each warp must also write a consecutive memory segment? + auto transform_a = [&](int i) { + int row = i / a_gl_rd_delta_o; + return a_gl_rd_delta_o * row + (i % a_gl_rd_delta_o) ^ row; + }; + // Since the computation of this remapping is non-trivial and, due to our main + // loop unrolls, all shared memory accesses are static, we simply precompute + // both transformed reads and writes. + int a_sh_wr_trans[a_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) + a_sh_wr_trans[i] = transform_a(a_sh_wr_delta * i + a_sh_wr); + int a_sh_rd_trans[b_sh_wr_iters][thread_m_blocks]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < thread_m_blocks; j++) + a_sh_rd_trans[i][j] = transform_a(a_sh_rd_delta_o * i + a_sh_rd_delta_i * j + a_sh_rd); + } + + // Since B-accesses have non-constant stride they have to be computed at + // runtime; we break dependencies between subsequent accesses with a tile by + // maintining multiple pointers (we have enough registers), a tiny + // optimization. + const int4* B_ptr[b_sh_wr_iters]; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] = B + b_gl_rd_delta_i * i + b_gl_rd; + + // Shared memory storage for global fetch pipelines. + int4* sh_a = sh_new; + int4* sh_b = sh_a + (stages * a_sh_stage); + int4* sh_g_idx = sh_b + (stages * b_sh_stage); + int4* sh_zp = sh_g_idx + (stages * g_idx_stage); + int4* sh_s = sh_zp + (stages * zp_sh_stage); + int4* sh_red = sh_b; + + // Register storage for double buffer of shared memory reads. + FragA frag_a[2][thread_m_blocks]; + I4 frag_b_quant[2][b_thread_vecs]; + FragC frag_c[thread_m_blocks][4][2]; + FragS frag_s[2][4]; // No act-order + FragS act_frag_s[2][4][4]; // For act-order + int frag_qzp[2][num_ints_per_thread]; // Zero-points + FragZP frag_zp; // Zero-points in fp16 + FragZP frag_zpf[2]; // Zero-points in fp16 in HQQ + + // Zero accumulators. + auto zero_accums = [&]() { +#pragma unroll + for (int i = 0; i < thread_m_blocks * 4 * 2 * 4; i++) + reinterpret_cast(frag_c)[i] = 0; + }; + + int sh_first_group_id = -1; + int sh_num_groups = -1; + constexpr int sh_max_num_groups = 32; + + auto fetch_act_order_scales_to_shared = [&](bool is_async, int first_group_id, int last_group_id) { + sh_first_group_id = first_group_id; + sh_num_groups = last_group_id - first_group_id + 1; + + if (sh_num_groups < sh_max_num_groups) { + sh_num_groups = sh_max_num_groups; + } + + if (sh_first_group_id + sh_num_groups > num_groups) { + sh_num_groups = num_groups - sh_first_group_id; + } + + int row_offset = first_group_id * s_gl_stride; + + if (is_async) { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + cp_async4_pred( + &sh_s[(i * s_sh_stride) + threadIdx.x], + &scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]); + } + } + } else { + for (int i = 0; i < sh_num_groups; i++) { + if (threadIdx.x < s_sh_stride) { + sh_s[(i * s_sh_stride) + threadIdx.x] = + scales_ptr[row_offset + (i * s_gl_stride) + slice_n_offset + threadIdx.x]; + } + } + } + }; + // Asynchronously fetch the next A, B and s tile from global to the next + // shared memory pipeline location. + int a_remaining_load_count_in_slice = stages; + auto fetch_to_shared = [&](int pipe, int a_off, bool pred = true) { + if (pred) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; + if (prob_k > thread_k_blocks * 16 * stages || slice_col == 0 || a_remaining_load_count_in_slice > 0) { + a_remaining_load_count_in_slice--; +#pragma unroll + for (int i = 0; i < a_sh_wr_iters; i++) { + int a_idx = a_gl_rd_delta_i * i + a_gl_rd + a_gl_rd_delta_o * a_off; + int row = a_idx / a_gl_stride; + int64_t sorted_row = 0; + if (!m_block_size_8 || row < 8) sorted_row = sh_block_sorted_ids[row] / top_k; + int64_t true_idx = sorted_row * a_gl_stride + a_idx % a_gl_stride; + cp_async4_pred(&sh_a_stage[a_sh_wr_trans[i]], &A[true_idx], row < block_num_valid_tokens); + } + } + int4* sh_b_stage = sh_b + b_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) { +#pragma unroll + for (int j = 0; j < b_thread_vecs; j++) { + cp_async4(&sh_b_stage[b_sh_wr_delta * i + b_sh_wr + j], B_ptr[i] + j + B_expert_off); + } + + B_ptr[i] += b_gl_rd_delta_o; + } + + if constexpr (has_act_order) { + // Fetch g_idx thread-block portion + int full_pipe = a_off; + int cur_k = slice_k_start_shared_fetch + tb_k * full_pipe; + if (cur_k < prob_k && cur_k < slice_k_finish) { + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + + int4 const* cur_g_idx_stage_ptr = reinterpret_cast(&g_idx[cur_k]); + + if (threadIdx.x < g_idx_stage) { + cp_async4_pred(&sh_g_idx_stage[threadIdx.x], &cur_g_idx_stage_ptr[threadIdx.x]); + } + } + } else { + if constexpr (group_blocks != -1) { + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch scales if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } else { + for (int i = 0; i < s_tb_groups; i++) { + if (s_sh_wr_pred) { + cp_async4(&sh_s_stage[i * s_sh_stride + s_sh_wr], &scales_ptr[s_gl_rd]); + } + s_gl_rd += s_gl_rd_delta; + } + } + } + + if constexpr (has_zp && group_blocks != -1) { + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + if constexpr (group_blocks >= thread_k_blocks) { + // Only fetch zero-points if this tile starts a new group + if (pipe % (group_blocks / thread_k_blocks) == 0) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } else { + for (int i = 0; i < zp_tb_groups; i++) { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp_stage[i * zp_sh_stride + zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + zp_gl_rd += zp_gl_rd_delta; + } + } + } + } + } + // Insert a fence even when we are winding down the pipeline to ensure that + // waiting is also correct at this point. + cp_async_fence(); + }; + + auto fetch_col_zp_to_shared = [&]() { + if (zp_sh_wr_pred) { + cp_async4(&sh_zp[zp_sh_wr], &zp_ptr[zp_gl_rd]); + } + }; + + auto fetch_col_scale_to_shared = [&]() { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + }; + + // Wait until the next thread tile has been loaded to shared memory. + auto wait_for_stage = [&]() { + // We only have `stages - 2` active fetches since we are double buffering + // and can only issue the next fetch when it is guaranteed that the previous + // shared memory load is fully complete (as it may otherwise be + // overwritten). + cp_async_wait(); + __syncthreads(); + }; + + // Load the next sub-tile from the current location in the shared memory pipe + // into the current register buffer. + auto fetch_to_registers = [&](int k, int pipe) { + int4* sh_a_stage = sh_a + a_sh_stage * pipe; +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) + ldsm(frag_a[k % 2][i], &sh_a_stage[a_sh_rd_trans[k % b_sh_wr_iters][i]]); + int4* sh_b_stage = sh_b + b_sh_stage * pipe; + +#pragma unroll + for (int i = 0; i < b_thread_vecs; i++) { + frag_b_quant[k % 2][i] = *reinterpret_cast(&sh_b_stage[b_sh_rd_delta * (k % b_sh_wr_iters) + b_sh_rd + i]); + } + }; + + bool is_same_group[stages]; + int same_group_id[stages]; + + auto init_same_group = [&](int pipe) { + if constexpr (!has_act_order) { + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + int group_id_1 = sh_g_idx_int_ptr[0]; + int group_id_2 = sh_g_idx_int_ptr[tb_k - 1]; + + is_same_group[pipe] = group_id_1 == group_id_2; + same_group_id[pipe] = group_id_1; + }; + + auto fetch_scales_to_registers = [&](int k, int full_pipe) { + int pipe = full_pipe % stages; + + if constexpr (!has_act_order) { + // No act-order case + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + } + } else if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + int4* sh_s_stage = + sh_s + s_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd]; + } else { + int warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = k_blocks / group_blocks; + + int4* sh_s_stage = sh_s + s_sh_stage * pipe; + + reinterpret_cast(&frag_s[k % 2])[0] = sh_s_stage[s_sh_rd + cur_group_id * s_sh_stride]; + } + } + + return; + } + + // Act-order case + + // Determine K of the "current" thread-block + int cur_k = slice_k_start + tb_k * full_pipe; + if (cur_k >= prob_k || cur_k >= slice_k_finish) { + return; + } + + // Reset (to current thread-block) since we read g_idx portion from the + // shared memory + cur_k = 0; + + // Progress to current iteration + cur_k += k_iter_size * (k % b_sh_wr_iters); + + // Determine "position" inside the thread-block (based on warp and + // thread-id) + int warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; // Each warp processes 4 16-size tiles over N + + int warp_row = warp_id / n_warps; + int warp_col = warp_id % n_warps; + + cur_k += warp_row * 16; + + int th_id = threadIdx.x % 32; + cur_k += (th_id % 4) * 2; // Due to tensor-core layout for fp16 B matrix + + int s_col_shift = + /*slice_n_offset +*/ (act_s_col_warp_stride * warp_col) + (th_id / 4) * act_s_col_stride; + + if (is_same_group[pipe]) { + if (k % 2 == 0) { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + sh_s[(same_group_id[pipe] - sh_first_group_id) * s_sh_stride + s_col_shift]; + } else { + *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))) = + *(reinterpret_cast(&(act_frag_s[(k - 1) % 2][0][0]))); + } + + for (int i = 1; i < 4; i++) { + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = *(reinterpret_cast(&(act_frag_s[k % 2][0][0]))); + } + return; + } + + int4* sh_g_idx_stage = sh_g_idx + g_idx_stage * pipe; + int* sh_g_idx_int_ptr = reinterpret_cast(sh_g_idx_stage); + + constexpr int k_frag_offsets[4] = {0, 1, 8, 9}; // Tensor core offsets per thread + +#pragma unroll + for (int i = 0; i < 4; i++) { + int actual_k = cur_k + k_frag_offsets[i]; + + int group_id = sh_g_idx_int_ptr[actual_k]; + int rel_group_id = group_id - sh_first_group_id; + + *(reinterpret_cast(&(act_frag_s[k % 2][i][0]))) = sh_s[rel_group_id * s_sh_stride + s_col_shift]; + } + }; + + auto fetch_zp_to_registers = [&](int k, int full_pipe) { + // This code does not handle group_blocks == 0, + // which signifies act_order. + // has_zp implies AWQ, which doesn't have act_order, + static_assert(!has_zp || group_blocks != 0); + + if constexpr (has_zp && !is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks == -1) { + // load only when starting a new slice + if (k == 0 && full_pipe == 0) { +#pragma unroll + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp))[zp_sh_rd + i]; + } + } + + } else if constexpr (group_blocks >= thread_k_blocks) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } else { + int warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + int cur_group_id = 0; + + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + sh_zp_stage += cur_group_id * zp_sh_stride; + + for (int i = 0; i < num_ints_per_thread; i++) { + frag_qzp[k % 2][i] = (reinterpret_cast(sh_zp_stage))[zp_sh_rd + i]; + } + } + } + + else if constexpr (has_zp && is_zp_float) { + int pipe = full_pipe % stages; + + if constexpr (group_blocks != -1) { + if constexpr (group_blocks >= thread_k_blocks) { + int4* sh_zp_stage = + sh_zp + zp_sh_stage * ((group_blocks / thread_k_blocks) * (pipe / (group_blocks / thread_k_blocks))); + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd]; + } else { + int warp_id = threadIdx.x / 32; + int n_warps = thread_n_blocks / 4; + + int warp_row = warp_id / n_warps; + + int cur_k = warp_row * 16; + cur_k += k_iter_size * (k % b_sh_wr_iters); + + int k_blocks = cur_k / 16; + // Suppress bogus and persistent divide-by-zero warning +#pragma nv_diagnostic push +#pragma nv_diag_suppress divide_by_zero + int cur_group_id = k_blocks / group_blocks; +#pragma nv_diagnostic pop + + int4* sh_zp_stage = sh_zp + zp_sh_stage * pipe; + + reinterpret_cast(&frag_zpf[k % 2])[0] = sh_zp_stage[zp_sh_rd + cur_group_id * zp_sh_stride]; + } + } + } + }; + + // Execute the actual tensor core matmul of a sub-tile. + bool is_first_matmul_in_slice = true; + auto matmul = [&](int k) { + int k2 = k % 2; + const bool is_new_zp = ((group_blocks != -1) && (group_blocks < thread_k_blocks || k == 0)) || + (group_blocks == -1 && is_first_matmul_in_slice); + if constexpr (has_zp && !is_zp_float) { + if (is_new_zp) { + if constexpr (group_blocks == -1) is_first_matmul_in_slice = false; + FragB frag_zp_0; + FragB frag_zp_1; + int zp_quant_0, zp_quant_1; + + if constexpr (w_type.size_bits() == 4) { + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = zp_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + zp_quant_0 = frag_qzp[k2][0]; + zp_quant_1 = frag_qzp[k2][1]; + } + + dequant(zp_quant_0, frag_zp_0); + dequant(zp_quant_1, frag_zp_1); + + frag_zp[0] = frag_zp_0[0]; + frag_zp[1] = frag_zp_0[1]; + frag_zp[2] = frag_zp_1[0]; + frag_zp[3] = frag_zp_1[1]; + } + } +// We have the m dimension as the inner loop in order to encourage overlapping +// dequantization and matmul operations. +#pragma unroll + for (int j = 0; j < 4; j++) { + FragB frag_b0; + FragB frag_b1; + int b_quant_0, b_quant_1; + + if constexpr (w_type.size_bits() == 4) { + b_quant_0 = frag_b_quant[k2][0][j]; + b_quant_1 = b_quant_0 >> 8; + } else { + static_assert(w_type.size_bits() == 8); + int* frag_b_quant_ptr = reinterpret_cast(frag_b_quant[k2]); + b_quant_0 = frag_b_quant_ptr[j * 2 + 0]; + b_quant_1 = frag_b_quant_ptr[j * 2 + 1]; + } + + dequant(b_quant_0, frag_b0); + dequant(b_quant_1, frag_b1); + + // Apply scale to frag_b0 + if constexpr (has_act_order) { + static_assert(group_blocks != -1); + scale4( + frag_b0, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k2][2][j], act_frag_s[k2][3][j], 0); + scale4( + frag_b1, act_frag_s[k2][0][j], act_frag_s[k2][1][j], act_frag_s[k][2][j], act_frag_s[k2][3][j], 1); + + } else if constexpr (has_zp && !is_zp_float && group_blocks == -1) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2 s2 = Dtype::nums2num2( + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 0])[idx], + reinterpret_cast(&frag_s[j / 2][j % 2 * 2 + 1])[idx]); + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], s2); + scale_and_sub(frag_b0, s2.x, frag_zp[j].x); + scale_and_sub(frag_b1, s2.y, frag_zp[j].y); + } else if constexpr (has_zp && !is_zp_float && group_blocks != -1) { + if (is_new_zp) frag_zp[j] = __hmul2(frag_zp[j], *reinterpret_cast(&frag_s[k2][j])); + scale_and_sub(frag_b0, frag_s[k % 2][j][0].x, frag_zp[j].x); + scale_and_sub(frag_b1, frag_s[k % 2][j][0].y, frag_zp[j].y); + } else if constexpr (has_zp && is_zp_float && group_blocks != -1) { + if (is_new_zp) frag_zpf[k2][j] = __hmul2(frag_zpf[k2][j], *reinterpret_cast(&frag_s[k2][j])); + scale_and_sub(frag_b0, frag_s[k2][j].x, frag_zpf[k2][j].x); + scale_and_sub(frag_b1, frag_s[k2][j].y, frag_zpf[k2][j].y); + } else if constexpr (group_blocks != -1) { + scale(frag_b0, frag_s[k2][j], 0); + scale(frag_b1, frag_s[k2][j], 1); + } + +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { + if constexpr (m_block_size_8) { + mma_trans(frag_a[k2][i], frag_b0, frag_b1, frag_c[i][j][0]); + } else { + mma(frag_a[k2][i], frag_b0, frag_c[i][j][0]); + mma(frag_a[k2][i], frag_b1, frag_c[i][j][1]); + } + } + } + }; + + // Since we slice across the k dimension of a tile in order to increase the + // number of warps while keeping the n dimension of a tile reasonable, we have + // multiple warps that accumulate their partial sums of the same output + // location; which we have to reduce over in the end. We do in shared memory. + auto thread_block_reduce = [&]() { + constexpr int red_off = threads / b_sh_stride_threads / 2; + if (red_off >= 1) { + int red_idx = threadIdx.x / b_sh_stride_threads; + constexpr int red_sh_stride = b_sh_stride_threads * 4 * 2; + constexpr int red_sh_delta = b_sh_stride_threads; + int red_sh_rd = red_sh_stride * (threadIdx.x / b_sh_stride_threads) + (threadIdx.x % b_sh_stride_threads); + + // Parallel logarithmic shared memory reduction. We make sure to avoid any + // unnecessary read or write iterations, e.g., for two warps we write only + // once by warp 1 and read only once by warp 0. + +#pragma unroll + for (int m_block = 0; m_block < thread_m_blocks; m_block++) { +#pragma unroll + for (int i = red_off; i > 0; i /= 2) { + if (i <= red_idx && red_idx < 2 * i) { +#pragma unroll + for (int j = 0; j < 4 * 2; j += (m_block_size_8 ? 2 : 1)) { + int red_sh_wr = red_sh_delta * j + (red_sh_rd - red_sh_stride * i); + if (i < red_off) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * j + red_sh_rd]); + float* c_wr = reinterpret_cast(&sh_red[red_sh_wr]); +#pragma unroll + for (int k = 0; k < 4; k++) + reinterpret_cast(frag_c)[4 * 2 * m_block + j][k] += c_rd[k] + c_wr[k]; + } + sh_red[red_sh_wr] = reinterpret_cast(&frag_c)[4 * 2 * m_block + j]; + } + } + __syncthreads(); + } + if (red_idx == 0) { +#pragma unroll + for (int i = 0; i < 4 * 2; i += (m_block_size_8 ? 2 : 1)) { + float* c_rd = reinterpret_cast(&sh_red[red_sh_delta * i + red_sh_rd]); +#pragma unroll + for (int j = 0; j < 4; j++) + reinterpret_cast(frag_c)[4 * 2 * m_block + i][j] += c_rd[j]; + } + } + __syncthreads(); + } + } + }; + + // Since multiple threadblocks may process parts of the same column slice, we + // finally have to globally reduce over the results. As the striped + // partitioning minimizes the number of such reductions and our outputs are + // usually rather small, we perform this reduction serially in L2 cache. + auto global_reduce_fp16 = [&](bool first = false, bool last = false) { + // We are very careful here to reduce directly in the output buffer to + // maximize L2 cache utilization in this step. To do this, we write out + // results in FP16 (but still reduce with FP32 compute). + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + if (!is_th_active) { + return; + } + + int c_gl_stride = prob_n / 8; + int c_gl_wr_delta_o = 8 * c_gl_stride; + int c_gl_wr_delta_i = 4 * (active_threads / 32); + int c_gl_wr; + if constexpr (m_block_size_8) { + c_gl_wr = c_gl_stride * ((threadIdx.x % 4) * 2) + 4 * (threadIdx.x / 32) + (threadIdx.x % 32) / 8; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } else { + c_gl_wr = c_gl_stride * ((threadIdx.x % 32) / 4) + 4 * (threadIdx.x / 32) + threadIdx.x % 4; + c_gl_wr += (2 * thread_n_blocks) * slice_col; + } + constexpr int c_sh_wr_delta = active_threads; + int c_sh_wr = threadIdx.x; + + if (!first) { + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + int c_idx; + if constexpr (m_block_size_8) + c_idx = c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i; + else + c_idx = c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2); + if (c_idx / c_gl_stride < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[c_idx / c_gl_stride]; + int64_t true_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride; + sh_red[c_sh_wr + c_sh_wr_delta * i] = C[true_idx]; + } + } + } + +#pragma unroll + for (int i = 0; i < (m_block_size_8 ? 2 : thread_m_blocks * 4); i++) { + if (!first) { + int4 c_red = sh_red[c_sh_wr + i * c_sh_wr_delta]; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta] += + Dtype::num2float(reinterpret_cast(&c_red)[j]); + } + } + if (!last) { + int4 c; +#pragma unroll + for (int j = 0; j < 2 * 4; j++) { + int delta = 0; + if constexpr (m_block_size_8) { + delta = j % 2 == 1 ? -2 : 0; + } + reinterpret_cast(&c)[j] = + Dtype::float2num(reinterpret_cast(&frag_c)[4 * 2 * 4 * (i / 4) + 4 * j + (i % 4) + delta]); + } + + int c_idx; + if constexpr (m_block_size_8) + c_idx = c_gl_wr + i * c_gl_stride + (threadIdx.x % 8) / 4 * c_gl_wr_delta_i; + else + c_idx = c_gl_wr + c_gl_wr_delta_o * (i / 2) + c_gl_wr_delta_i * (i % 2); + if (c_idx / c_gl_stride < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[c_idx / c_gl_stride]; + int64_t true_idx = sorted_row * c_gl_stride + c_idx % c_gl_stride; + C[true_idx] = c; + } + } + } + }; + + // Globally reduce over threadblocks that compute the same column block. + // We use a tmp C buffer to reduce in full fp32 precision. + auto global_reduce_fp32 = [&](bool first = false, bool last = false) { + constexpr int tb_m = thread_m_blocks * 16; + constexpr int tb_n = thread_n_blocks * 16; + + constexpr int c_size = tb_m * tb_n * sizeof(float) / 16; + + constexpr int active_threads = 32 * thread_n_blocks / 4; + bool is_th_active = threadIdx.x < active_threads; + + constexpr int num_floats = thread_m_blocks * 4 * 2 * 4; + constexpr int th_size = num_floats * sizeof(float) / 16; + + int c_cur_offset = locks_off * c_size; + + if (!is_th_active) { + return; + } + + if (!first) { + float* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k++) { + if constexpr (m_block_size_8) { + if (k % 2) continue; + } else { + if (k / 8 * 16 + (threadIdx.x % 32) / 4 >= block_num_valid_tokens) continue; + } + + sh_red[threadIdx.x] = C_tmp[c_cur_offset + active_threads * k + threadIdx.x]; + + float* sh_c_ptr = reinterpret_cast(&sh_red[threadIdx.x]); +#pragma unroll + for (int f = 0; f < 4; f++) { + frag_c_ptr[k * 4 + f] += sh_c_ptr[f]; + } + } + } + + if (!last) { + int4* frag_c_ptr = reinterpret_cast(&frag_c); +#pragma unroll + for (int k = 0; k < th_size; k++) { + if constexpr (m_block_size_8) { + if (k % 2) continue; + } else { + if (k / 8 * 16 + (threadIdx.x % 32) / 4 >= block_num_valid_tokens) continue; + } + + C_tmp[c_cur_offset + active_threads * k + threadIdx.x] = frag_c_ptr[k]; + } + } + }; + + // Write out the reduce final result in the correct layout. We only actually + // reshuffle matrix fragments in this step, the reduction above is performed + // in fragment layout. + auto write_result = [&]() { + int c_gl_stride = prob_n / 8; + constexpr int c_sh_stride = 2 * thread_n_blocks + 1; + int c_gl_wr_delta = c_gl_stride * (threads / (2 * thread_n_blocks)); + constexpr int c_sh_rd_delta = c_sh_stride * (threads / (2 * thread_n_blocks)); + + int c_gl_wr = c_gl_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + c_gl_wr += (2 * thread_n_blocks) * slice_col; + int c_sh_wr; + if constexpr (m_block_size_8) { + c_sh_wr = (8 * c_sh_stride) * ((threadIdx.x % 32) % 4 * 2) + (threadIdx.x % 32) / 4; + c_sh_wr += 64 * (threadIdx.x / 32); + } else { + c_sh_wr = (4 * c_sh_stride) * ((threadIdx.x % 32) / 4) + (threadIdx.x % 32) % 4; + c_sh_wr += 32 * (threadIdx.x / 32); + } + + int c_sh_rd = c_sh_stride * (threadIdx.x / (2 * thread_n_blocks)) + (threadIdx.x % (2 * thread_n_blocks)); + + // We first reorder in shared memory to guarantee the most efficient final + // global write patterns + auto write = [&](int idx, float c0, float c1, FragS& s) { + scalar_t2 res = Dtype::nums2num2(Dtype::float2num(c0), Dtype::float2num(c1)); + + // For per-column quantization we finally apply the scale here (only for + // 4-bit) + if constexpr (!has_act_order && group_blocks == -1 && w_type.size_bits() == 4 && !has_zp) { + res = __hmul2(res, s[0]); + } + + if constexpr (m_block_size_8) { + ((scalar_t*)sh_red)[idx] = res.x; + ((scalar_t*)sh_red)[idx + 8 * c_sh_stride] = res.y; + } else { + ((scalar_t2*)sh_red)[idx] = res; + } + }; + + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + if constexpr (m_block_size_8) { + int wr = c_sh_wr + 16 * j; + write(wr, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write(wr + 8, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 1]); + } else { + int wr = c_sh_wr + 8 * j; + write( + wr + (4 * c_sh_stride) * 0 + 0, frag_c[i][j][0][0], frag_c[i][j][0][1], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 8 + 0, frag_c[i][j][0][2], frag_c[i][j][0][3], frag_s[j / 2][2 * (j % 2) + 0]); + write( + wr + (4 * c_sh_stride) * 0 + 4, frag_c[i][j][1][0], frag_c[i][j][1][1], frag_s[j / 2][2 * (j % 2) + 1]); + write( + wr + (4 * c_sh_stride) * 8 + 4, frag_c[i][j][1][2], frag_c[i][j][1][3], frag_s[j / 2][2 * (j % 2) + 1]); + } + } + c_sh_wr += 16 * (4 * c_sh_stride); + } + } + __syncthreads(); + +#pragma unroll + for (int i = 0; i < div_ceil(16 * thread_m_blocks, threads / (2 * thread_n_blocks)); i++) { + int row = c_gl_wr / c_gl_stride; + if (row < block_num_valid_tokens) { + int64_t sorted_row = sh_block_sorted_ids[row]; + int64_t true_idx = sorted_row * c_gl_stride + c_gl_wr % c_gl_stride; + scalar_t2 topk_weight_score; + if (mul_topk_weights) topk_weight_score = sh_block_topk_weights[row]; + if (use_atomic_add && slice_count > 1 || mul_topk_weights) { + scalar_t2* C_half2 = reinterpret_cast(&C[true_idx]); + scalar_t2* sh_red_half2 = reinterpret_cast(&sh_red[c_sh_rd]); +#pragma unroll + for (int a = 0; a < 4; a++) { + scalar_t2 res = sh_red_half2[a]; + if (mul_topk_weights) { + res = __hmul2(res, topk_weight_score); + } + + if (use_atomic_add && slice_count > 1) { + atomicAdd(&C_half2[a], res); + } else { + C_half2[a] = res; + }; + } + } else { + C[true_idx] = sh_red[c_sh_rd]; + } + c_gl_wr += c_gl_wr_delta; + c_sh_rd += c_sh_rd_delta; + } + } + __syncthreads(); + }; + + // Start global fetch and register load pipelines. + auto start_pipes = [&]() { + +#pragma unroll + for (int i = 0; i < stages - 1; i++) { + if (has_act_order && i == 0) { + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + fetch_act_order_scales_to_shared(true, g_idx[slice_k_start], g_idx[last_g_idx]); + } + + if constexpr (has_zp && !is_zp_float && group_blocks == -1) { + if (i == 0) { + fetch_col_zp_to_shared(); + fetch_col_scale_to_shared(); + } + } + fetch_to_shared(i, i, i < slice_iters); + } + + zero_accums(); + wait_for_stage(); + init_same_group(0); + fetch_to_registers(0, 0); + fetch_scales_to_registers(0, 0); + fetch_zp_to_registers(0, 0); + a_gl_rd += a_gl_rd_delta_o * (stages - 1); + slice_k_start_shared_fetch += tb_k * (stages - 1); + }; + if (slice_iters) { + start_pipes(); + } + + // Main loop. + while (slice_iters) { + // We unroll over both the global fetch and the register load pipeline to + // ensure all shared memory accesses are static. Note that both pipelines + // have even length meaning that the next iteration will always start at + // index 0. + +#pragma unroll + for (int pipe = 0; pipe < stages;) { +#pragma unroll + for (int k = 0; k < b_sh_wr_iters; k++) { + fetch_to_registers(k + 1, pipe % stages); + fetch_scales_to_registers(k + 1, pipe); + fetch_zp_to_registers(k + 1, pipe); + if (k == b_sh_wr_iters - 2) { + fetch_to_shared((pipe + stages - 1) % stages, pipe, slice_iters >= stages); + pipe++; + wait_for_stage(); + init_same_group(pipe % stages); + } + matmul(k); + } + slice_iters--; + if (slice_iters == 0) { + break; + } + } + a_remaining_load_count_in_slice = 0; + + a_gl_rd += a_gl_rd_delta_o * stages; + slice_k_start += tb_k * stages; + slice_k_start_shared_fetch += tb_k * stages; + + if constexpr (has_act_order) { + int first_group_id = g_idx[slice_k_start]; + int last_g_idx = slice_k_start + stages * tb_k * 2; + if (last_g_idx >= prob_k) { + last_g_idx = prob_k - 1; + } + int last_group_id = g_idx[last_g_idx]; + if (last_group_id >= sh_first_group_id + sh_num_groups) { + fetch_act_order_scales_to_shared(false, first_group_id, last_group_id); + __syncthreads(); + } + } + + // Process results and, if necessary, proceed to the next column slice. + // While this pattern may not be the most readable, other ways of writing + // the loop seemed to noticeably worse performance after compilation. + if (slice_iters == 0) { + cp_async_wait<0>(); + bool last = slice_idx == slice_count - 1; + // For per-column scales, we only fetch them here in the final step before + // write-out + if constexpr (!has_act_order && group_blocks == -1 && !has_zp) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + if (s_sh_wr_pred) { + cp_async4(&sh_s[s_sh_wr], &scales_ptr[s_gl_rd]); + } + cp_async_fence(); + } + } + + thread_block_reduce(); + if constexpr (!has_act_order && group_blocks == -1 && !has_zp) { + if (w_type.size_bits() == 8 || (last || use_atomic_add)) { + cp_async_wait<0>(); + __syncthreads(); + if (threadIdx.x / 32 < thread_n_blocks / 4) { + reinterpret_cast(&frag_s)[0] = sh_s[s_sh_rd + 0]; + reinterpret_cast(&frag_s)[1] = sh_s[s_sh_rd + 4]; + if constexpr (m_block_size_8) { + int idx = (threadIdx.x / 4) % 2; + scalar_t2* frag_s_half2 = reinterpret_cast(frag_s); +#pragma unroll + for (int i = 0; i < 8; i++) { + frag_s_half2[i] = Dtype::num2num2(reinterpret_cast(&frag_s_half2[i])[idx]); + } + } + } + } + } + + // For 8-bit channelwise, we apply the scale before the global reduction + // that converts the fp32 results to fp16 (so that we avoid possible + // overflow in fp16) + if constexpr (!has_act_order && group_blocks == -1 && w_type.size_bits() == 8 && !has_zp) { + if (threadIdx.x / 32 < thread_n_blocks / 4) { +#pragma unroll + for (int i = 0; i < thread_m_blocks; i++) { +#pragma unroll + for (int j = 0; j < 4; j++) { + scale_float(reinterpret_cast(&frag_c[i][j][0][0]), frag_s[j / 2][2 * (j % 2) + 0]); + scale_float( + reinterpret_cast(&frag_c[i][j][0][2]), frag_s[j / 2][2 * (j % 2) + (m_block_size_8 ? 1 : 0)]); + + if constexpr (!m_block_size_8) { + scale_float(reinterpret_cast(&frag_c[i][j][1][0]), frag_s[j / 2][2 * (j % 2) + 1]); + scale_float(reinterpret_cast(&frag_c[i][j][1][2]), frag_s[j / 2][2 * (j % 2) + 1]); + } + } + } + } + } + + if (slice_count > 1 && !use_atomic_add) { + // only globally reduce if there is more than one block in a slice + barrier_acquire(&locks[locks_off], slice_idx); + if (use_fp32_reduce) { + global_reduce_fp32(slice_idx == 0, last); + } else { + global_reduce_fp16(slice_idx == 0, last); + } + barrier_release(&locks[locks_off], last); + } + if (use_atomic_add && slice_count > 1 && slice_idx != 0) wait_negative_and_add(&locks[locks_off]); + if (last || use_atomic_add) + // only the last block in a slice actually writes the result + write_result(); + if (slice_row) a_remaining_load_count_in_slice = stages; + slice_row = 0; + slice_col_par++; + slice_col++; + is_first_matmul_in_slice = true; + init_slice(); + if (slice_iters) { + a_gl_rd = a_gl_stride * (threadIdx.x / a_gl_rd_delta_o) + (threadIdx.x % a_gl_rd_delta_o); +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] += b_sh_stride - b_gl_rd_delta_o * k_tiles; + if (slice_col == 0) { +#pragma unroll + for (int i = 0; i < b_sh_wr_iters; i++) + B_ptr[i] -= b_gl_stride; + } + + // Update slice k/n for scales loading + if constexpr (has_act_order) { + slice_k_start = tb_k * slice_row; + slice_k_finish = slice_k_start + tb_k * slice_iters; + slice_k_start_shared_fetch = slice_k_start; + slice_n_offset = act_s_col_tb_stride * slice_col; + + } else { + s_gl_rd = s_sh_stride * slice_col + threadIdx.x; + zp_gl_rd = zp_sh_stride * slice_col + threadIdx.x; + } + + start_pipes(); + } + } + } +} + +} // namespace MARLIN_NAMESPACE_NAME + +#endif diff --git a/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu b/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu new file mode 100644 index 000000000..b249f6415 --- /dev/null +++ b/sgl-kernel/csrc/moe/marlin_moe_wna16/ops.cu @@ -0,0 +1,1111 @@ +/* + * Modified by Neural Magic + * Copyright (C) Marlin.2024 Elias Frantar + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Adapted from https://github.com/IST-DASLab/marlin + */ + +#ifndef MARLIN_NAMESPACE_NAME +#define MARLIN_NAMESPACE_NAME marlin_moe_wna16 +#endif + +#include "kernel.h" +#include "kernel_marlin.cuh" + +#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ + static_assert( \ + std::is_same::value || std::is_same::value, \ + "only float16 and bfloat16 is supported"); + +namespace MARLIN_NAMESPACE_NAME { + +__global__ void MarlinDefault(MARLIN_KERNEL_PARAMS){}; + +using MarlinFuncPtr = void (*)(MARLIN_KERNEL_PARAMS); + +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 800 + +template +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + const int32_t* __restrict__ sorted_token_ids_ptr, + const int32_t* __restrict__ expert_ids_ptr, + const int32_t* __restrict__ num_tokens_past_padded_ptr, + int size_m, + int size_k, + int top_k) {}; +} + +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional const& c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + TORCH_CHECK_NOT_IMPLEMENTED(false, "marlin_gemm(..) requires CUDA_ARCH >= 8.0"); + return torch::empty({1, 1}); +} + +#else + +// For a given "a" of size [M,K] performs a permutation of the K columns based +// on the given "perm" indices. +template +__global__ void permute_cols_kernel( + int4 const* __restrict__ a_int4_ptr, + int const* __restrict__ perm_int_ptr, + int4* __restrict__ out_int4_ptr, + const int32_t* __restrict__ sorted_token_ids_ptr, + const int32_t* __restrict__ expert_ids_ptr, + const int32_t* __restrict__ num_tokens_past_padded_ptr, + int size_m, + int size_k, + int top_k) { + int num_tokens_past_padded = num_tokens_past_padded_ptr[0]; + int num_moe_blocks = div_ceil(num_tokens_past_padded, moe_block_size); + int32_t block_sorted_ids[moe_block_size]; + int block_num_valid_tokens = 0; + int64_t old_expert_id = 0; + int64_t expert_id = 0; + int row_stride = size_k * sizeof(half) / 16; + + auto read_moe_block_data = [&](int block_id) { + block_num_valid_tokens = moe_block_size; + int4* tmp_block_sorted_ids = reinterpret_cast(block_sorted_ids); + for (int i = 0; i < moe_block_size / 4; i++) { + tmp_block_sorted_ids[i] = ((int4*)sorted_token_ids_ptr)[block_id * moe_block_size / 4 + i]; + } + for (int i = 0; i < moe_block_size; i++) { + if (block_sorted_ids[i] >= size_m * top_k) { + block_num_valid_tokens = i; + break; + }; + } + }; + + auto permute_row = [&](int row) { + int iters = size_k / default_threads; + int rest = size_k % default_threads; + + int in_offset = (row / top_k) * row_stride; + int out_offset = row * row_stride; + + half const* a_row_half = reinterpret_cast(a_int4_ptr + in_offset); + half* out_half = reinterpret_cast(out_int4_ptr + out_offset); + + int base_k = 0; + + for (int i = 0; i < iters; i++) { + int cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + + base_k += default_threads; + } + + if (rest) { + if (threadIdx.x < rest) { + int cur_k = base_k + threadIdx.x; + int src_pos = perm_int_ptr[cur_k]; + + out_half[cur_k] = a_row_half[src_pos]; + } + } + }; + + for (int index = blockIdx.x; index < num_moe_blocks; index += gridDim.x) { + old_expert_id = expert_id; + int tmp_expert_id = expert_ids_ptr[index]; + if (tmp_expert_id == -1) continue; + expert_id = tmp_expert_id; + perm_int_ptr += (expert_id - old_expert_id) * size_k; + read_moe_block_data(index); + + for (int i = 0; i < block_num_valid_tokens; i++) + permute_row(block_sorted_ids[i]); + } +} + +typedef struct { + int thread_k; + int thread_n; + int num_threads; +} thread_config_t; + +thread_config_t small_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {128, 128, 256}, + {64, 128, 128}}; + +thread_config_t large_batch_thread_configs[] = { + // Ordered by priority + + // thread_k, thread_n, num_threads + {64, 256, 256}, + {64, 128, 128}}; + +typedef struct { + int blocks_per_sm; + thread_config_t tb_cfg; +} exec_config_t; + +int get_scales_cache_size( + thread_config_t const& th_config, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full) { + bool cache_scales_chunk = has_act_order && !is_k_full; + + int tb_n = th_config.thread_n; + int tb_k = th_config.thread_k; + + // Get max scale groups per thread-block + int tb_groups; + if (group_size == -1) { + tb_groups = 1; + } else if (group_size == 0) { + tb_groups = div_ceil(tb_k, 32); // Worst case is 32 group size + } else { + tb_groups = div_ceil(tb_k, group_size); + } + + if (cache_scales_chunk) { + int load_groups = tb_groups * pipe_stages * 2; // Chunk size is 2x pipeline over dim K + load_groups = max(load_groups, 32); // We load at least 32 scale groups + return load_groups * tb_n * 2; + + } else { + int tb_scales = tb_groups * tb_n * 2; + + return tb_scales * pipe_stages; + } +} + +int get_kernel_cache_size( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float) { + int pack_factor = 32 / num_bits; + + // Get B size + int tb_k = th_config.thread_k; + int tb_n = th_config.thread_n; + int tb_m = thread_m_blocks * 16; + + // shm size for block_sorted_ids/block_topk_weights + // both of them requires tb_m * 4 bytes (tb_m * int32 or tb_m * float32) + int sh_block_meta_size = tb_m * 4 * 2; + int sh_a_size = pipe_stages * (tb_m * tb_k) * 2; + int sh_b_size = pipe_stages * (tb_k * tb_n / pack_factor) * 4; + int sh_s_size = + get_scales_cache_size(th_config, prob_m, prob_n, prob_k, num_bits, group_size, has_act_order, is_k_full); + int sh_g_idx_size = has_act_order && !is_k_full ? pipe_stages * tb_k / 4 : 0; + int sh_zp_size = 0; + if (has_zp) { + if (is_zp_float) + sh_zp_size = sh_s_size; + else if (num_bits == 4) + sh_zp_size = sh_s_size / 4; + else if (num_bits == 8) + sh_zp_size = sh_s_size / 2; + } + + int total_size = sh_a_size + sh_b_size + sh_s_size + sh_zp_size + sh_g_idx_size + sh_block_meta_size; + + return total_size; +} + +bool is_valid_config( + thread_config_t const& th_config, + int thread_m_blocks, + int prob_m, + int prob_n, + int prob_k, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + int has_zp, + int is_zp_float, + int max_shared_mem) { + // Sanity + if (th_config.thread_k == -1 || th_config.thread_n == -1 || th_config.num_threads == -1) { + return false; + } + + // Verify K/N are divisible by thread K/N + if (prob_k % th_config.thread_k != 0 || prob_n % th_config.thread_n != 0) { + return false; + } + + // Verify min for thread K/N + if (th_config.thread_n < min_thread_n || th_config.thread_k < min_thread_k) { + return false; + } + + // num_threads must be at least 128 (= 4 warps) + if (th_config.num_threads < 128) { + return false; + } + + // Check that pipeline fits into cache + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + return cache_size <= max_shared_mem; +} + +#define __GET_IF( \ + W_TYPE, \ + THREAD_M_BLOCKS, \ + THREAD_N_BLOCKS, \ + THREAD_K_BLOCKS, \ + M_BLOCK_SIZE_8, \ + HAS_ACT_ORDER, \ + HAS_ZP, \ + GROUP_BLOCKS, \ + NUM_THREADS, \ + IS_ZP_FLOAT) \ + else if ( \ + q_type == W_TYPE && thread_m_blocks == THREAD_M_BLOCKS && thread_n_blocks == THREAD_N_BLOCKS && \ + thread_k_blocks == THREAD_K_BLOCKS && m_block_size_8 == M_BLOCK_SIZE_8 && has_act_order == HAS_ACT_ORDER && \ + has_zp == HAS_ZP && group_blocks == GROUP_BLOCKS && num_threads == NUM_THREADS && is_zp_float == IS_ZP_FLOAT) { \ + kernel = Marlin< \ + scalar_t, \ + W_TYPE.id(), \ + NUM_THREADS, \ + THREAD_M_BLOCKS, \ + THREAD_N_BLOCKS, \ + THREAD_K_BLOCKS, \ + M_BLOCK_SIZE_8, \ + pipe_stages, \ + HAS_ACT_ORDER, \ + HAS_ZP, \ + GROUP_BLOCKS, \ + IS_ZP_FLOAT>; \ + } + +#define GPTQ_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, true, false, 0, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, true, false, 0, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, false, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, false, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, false, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, false, 8, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, false, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, false, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, false, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, false, 8, NUM_THREADS, false) + +#define GPTQ_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, true, false, 0, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, true, false, 0, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, true, false, 0, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, false, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, false, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, false, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, false, 8, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, false, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, false, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, false, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, false, 8, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, false, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, false, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, false, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, false, 8, NUM_THREADS, false) + +#define AWQ_GET_IF_M1(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, true, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, true, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, true, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, true, 8, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, true, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, true, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, true, 8, NUM_THREADS, false) + +#define AWQ_GET_IF_M234(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, true, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, true, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, true, 8, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, true, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, true, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, true, 8, NUM_THREADS, false) \ + \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, true, -1, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, true, 2, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, false) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, true, 8, NUM_THREADS, false) + +// We currently have 4-bit models only with group_blocks == 4 +#define HQQ_GET_IF(W_TYPE, N_BLOCKS, K_BLOCKS, NUM_THREADS) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, true, false, true, 4, NUM_THREADS, true) \ + __GET_IF(W_TYPE, 1, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, true) \ + __GET_IF(W_TYPE, 2, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, true) \ + __GET_IF(W_TYPE, 3, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, true) \ + __GET_IF(W_TYPE, 4, N_BLOCKS, K_BLOCKS, false, false, true, 4, NUM_THREADS, true) + +template +MarlinFuncPtr get_marlin_kernel( + const sglang::ScalarType q_type, + int thread_m_blocks, + int thread_n_blocks, + int thread_k_blocks, + bool m_block_size_8, + bool has_act_order, + bool has_zp, + int group_blocks, + int num_threads, + bool is_zp_float) { + int num_bits = q_type.size_bits(); + auto kernel = MarlinDefault; + if (false) { + } + GPTQ_GET_IF_M1(sglang::kU4B8, 8, 8, 256) + GPTQ_GET_IF_M1(sglang::kU4B8, 8, 4, 128) + + GPTQ_GET_IF_M234(sglang::kU4B8, 16, 4, 256) + GPTQ_GET_IF_M234(sglang::kU4B8, 8, 4, 128) + + GPTQ_GET_IF_M1(sglang::kU8B128, 8, 8, 256) + GPTQ_GET_IF_M1(sglang::kU8B128, 8, 4, 128) + + GPTQ_GET_IF_M234(sglang::kU8B128, 16, 4, 256) + GPTQ_GET_IF_M234(sglang::kU8B128, 8, 4, 128) + + AWQ_GET_IF_M1(sglang::kU4, 8, 8, 256) + AWQ_GET_IF_M1(sglang::kU4, 8, 4, 128) + + AWQ_GET_IF_M234(sglang::kU4, 16, 4, 256) + AWQ_GET_IF_M234(sglang::kU4, 8, 4, 128) + + return kernel; +} + +template +exec_config_t determine_exec_config( + const sglang::ScalarType& q_type, + int prob_m, + int prob_n, + int prob_k, + int thread_m_blocks, + bool m_block_size_8, + int num_bits, + int group_size, + bool has_act_order, + bool is_k_full, + bool has_zp, + bool is_zp_float, + int max_shared_mem) { + exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}}; + thread_config_t* thread_configs = thread_m_blocks > 1 ? large_batch_thread_configs : small_batch_thread_configs; + int thread_configs_size = thread_m_blocks > 1 ? sizeof(large_batch_thread_configs) / sizeof(thread_config_t) + : sizeof(small_batch_thread_configs) / sizeof(thread_config_t); + + int count = 0; + constexpr int device_max_reg_size = 255 * 1024; + for (int i = 0; i < thread_configs_size; i++) { + thread_config_t th_config = thread_configs[i]; + + if (!is_valid_config( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem)) { + continue; + } + + int cache_size = get_kernel_cache_size( + th_config, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float); + + int group_blocks = 0; + if (!has_act_order) { + group_blocks = group_size == -1 ? -1 : group_size / 16; + } + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + th_config.thread_n / 16, + th_config.thread_k / 16, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + th_config.num_threads, + is_zp_float); + + if (kernel == MarlinDefault) continue; + + if (thread_m_blocks > 1) { + exec_cfg = {1, th_config}; + break; + } else { + cudaFuncAttributes attr; + cudaFuncGetAttributes(&attr, kernel); + int reg_size = max(attr.numRegs, 1) * th_config.num_threads * 4; + int allow_count = min(device_max_reg_size / reg_size, max_shared_mem / (cache_size + 1024)); + allow_count = max(min(allow_count, 4), 1); + if (allow_count > count) { + count = allow_count; + exec_cfg = {count, th_config}; + }; + } + } + + return exec_cfg; +} + +template +void marlin_mm( + const void* A, + const void* B, + void* C, + void* C_tmp, + void* s, + void* zp, + void* g_idx, + void* perm, + void* a_tmp, + void* sorted_token_ids, + void* expert_ids, + void* num_tokens_past_padded, + void* topk_weights, + int moe_block_size, + int top_k, + bool mul_topk_weights, + bool is_ep, + int prob_m, + int prob_n, + int prob_k, + void* workspace, + sglang::ScalarType const& q_type, + bool has_act_order, + bool is_k_full, + bool has_zp, + int num_groups, + int group_size, + int dev, + cudaStream_t stream, + int thread_k, + int thread_n, + int sms, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + int thread_m_blocks = div_ceil(moe_block_size, 16); + bool m_block_size_8 = moe_block_size == 8; + + if (has_zp) { + TORCH_CHECK( + q_type == sglang::kU4 || q_type == sglang::kU8, + "q_type must be u4 or u8 when has_zp = True. Got = ", + q_type.str()); + } else { + TORCH_CHECK( + q_type == sglang::kU4B8 || q_type == sglang::kU8B128, + "q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ", + q_type.str()); + } + + TORCH_CHECK(prob_m > 0 && prob_n > 0 && prob_k > 0, "Invalid MNK = [", prob_m, ", ", prob_n, ", ", prob_k, "]"); + + int group_blocks = 0; + if (has_act_order) { + if (is_k_full) { + TORCH_CHECK(group_size != -1); + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } else { + TORCH_CHECK(group_size == 0); + group_blocks = 0; + } + } else { + if (group_size == -1) { + group_blocks = -1; + } else { + group_blocks = group_size / 16; + TORCH_CHECK( + prob_k % group_blocks == 0, "prob_k = ", prob_k, " is not divisible by group_blocks = ", group_blocks); + } + } + + int num_bits = q_type.size_bits(); + const int4* A_ptr = (const int4*)A; + const int4* B_ptr = (const int4*)B; + int4* C_ptr = (int4*)C; + int4* C_tmp_ptr = (int4*)C_tmp; + const int4* s_ptr = (const int4*)s; + const int4* zp_ptr = (const int4*)zp; + const int* g_idx_ptr = (const int*)g_idx; + const int* perm_ptr = (const int*)perm; + int4* a_tmp_ptr = (int4*)a_tmp; + const int32_t* sorted_token_ids_ptr = (const int32_t*)sorted_token_ids; + const int32_t* expert_ids_ptr = (const int32_t*)expert_ids; + const int32_t* num_tokens_past_padded_ptr = (const int32_t*)num_tokens_past_padded; + const float* topk_weights_ptr = (const float*)topk_weights; + int* locks = (int*)workspace; + + if (has_act_order) { + // Permute A columns + auto kernel = permute_cols_kernel<8>; + if (moe_block_size == 8) { + } else if (moe_block_size == 16) + kernel = permute_cols_kernel<16>; + else if (moe_block_size == 32) + kernel = permute_cols_kernel<32>; + else if (moe_block_size == 48) + kernel = permute_cols_kernel<48>; + else if (moe_block_size == 64) + kernel = permute_cols_kernel<64>; + else + TORCH_CHECK(false, "unsupported moe_block_size ", moe_block_size); + + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, perm_ptr, a_tmp_ptr, sorted_token_ids_ptr, expert_ids_ptr, + num_tokens_past_padded_ptr, prob_m, prob_k, top_k); + // clang-format on + A_ptr = a_tmp_ptr; + prob_m = prob_m * top_k; + top_k = 1; + + // If we have a full K, then we can run the non-act-order version of Marlin + // (since the weight rows are reordered by increasing group ids, and by + // having a full K, we have full original groups) + if (is_k_full) has_act_order = false; + } + + int max_shared_mem = 0; + cudaDeviceGetAttribute(&max_shared_mem, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev); + TORCH_CHECK(max_shared_mem > 0); + + // Set thread config + exec_config_t exec_cfg; + thread_config_t thread_tfg; + if (thread_k != -1 && thread_n != -1) { + thread_tfg = thread_config_t{thread_k, thread_n, default_threads}; + exec_cfg = exec_config_t{1, thread_tfg}; + TORCH_CHECK(prob_n % thread_n == 0, "prob_n = ", prob_n, " is not divisible by thread_n = ", thread_n); + TORCH_CHECK(prob_k % thread_k == 0, "prob_k = ", prob_k, " is not divisible by thread_k = ", thread_k); + } else { + // Auto config + exec_cfg = determine_exec_config( + q_type, + prob_m, + prob_n, + prob_k, + thread_m_blocks, + m_block_size_8, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem); + thread_tfg = exec_cfg.tb_cfg; + } + + int num_threads = thread_tfg.num_threads; + thread_k = thread_tfg.thread_k; + thread_n = thread_tfg.thread_n; + int blocks = sms * exec_cfg.blocks_per_sm; + if (exec_cfg.blocks_per_sm > 1) max_shared_mem = max_shared_mem / exec_cfg.blocks_per_sm - 1024; + + int thread_k_blocks = thread_k / 16; + int thread_n_blocks = thread_n / 16; + + TORCH_CHECK( + is_valid_config( + thread_tfg, + thread_m_blocks, + prob_m, + prob_n, + prob_k, + num_bits, + group_size, + has_act_order, + is_k_full, + has_zp, + is_zp_float, + max_shared_mem), + "Invalid thread config: thread_m_blocks = ", + thread_m_blocks, + ", thread_k = ", + thread_tfg.thread_k, + ", thread_n = ", + thread_tfg.thread_n, + ", num_threads = ", + thread_tfg.num_threads, + " for MKN = [", + prob_m, + ", ", + prob_k, + ", ", + prob_n, + "] and num_bits = ", + num_bits, + ", group_size = ", + group_size, + ", has_act_order = ", + has_act_order, + ", is_k_full = ", + is_k_full, + ", has_zp = ", + has_zp, + ", is_zp_float = ", + is_zp_float, + ", max_shared_mem = ", + max_shared_mem); + + auto kernel = get_marlin_kernel( + q_type, + thread_m_blocks, + thread_n_blocks, + thread_k_blocks, + m_block_size_8, + has_act_order, + has_zp, + group_blocks, + num_threads, + is_zp_float); + + if (kernel == MarlinDefault) { + TORCH_CHECK( + false, + "Unsupported shapes: MNK = [", + prob_m, + ", ", + prob_n, + ", ", + prob_k, + "]", + ", has_act_order = ", + has_act_order, + ", num_groups = ", + num_groups, + ", group_size = ", + group_size, + ", thread_m_blocks = ", + thread_m_blocks, + ", thread_n_blocks = ", + thread_n_blocks, + ", thread_k_blocks = ", + thread_k_blocks, + ", num_bits = ", + num_bits); + } + + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, max_shared_mem); + // avoid ">>>" being formatted to "> > >" + // clang-format off + kernel<<>>( + A_ptr, B_ptr, C_ptr, C_tmp_ptr, s_ptr, zp_ptr, g_idx_ptr, + sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr, + topk_weights_ptr, top_k, mul_topk_weights, is_ep, num_groups, prob_m, + prob_n, prob_k, locks, use_atomic_add, use_fp32_reduce); + // clang-format on +} + +} // namespace MARLIN_NAMESPACE_NAME + +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional const& c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float) { + sglang::ScalarType const b_q_type = sglang::ScalarType::from_id(b_q_type_id); + int pack_factor = 32 / b_q_type.size_bits(); + + if (moe_block_size != 8) { + TORCH_CHECK(moe_block_size % 16 == 0, "unsupported moe_block_size=", moe_block_size); + TORCH_CHECK(moe_block_size >= 16 && moe_block_size <= 64, "unsupported moe_block_size=", moe_block_size); + } + + // Verify A + TORCH_CHECK(a.size(0) == size_m, "Shape mismatch: a.size(0) = ", a.size(0), ", size_m = ", size_m); + TORCH_CHECK(a.size(1) == size_k, "Shape mismatch: a.size(1) = ", a.size(1), ", size_k = ", size_k); + + // Verify B + TORCH_CHECK( + size_k % MARLIN_NAMESPACE_NAME::tile_size == 0, + "size_k = ", + size_k, + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + (size_k / MARLIN_NAMESPACE_NAME::tile_size) == b_q_weight.size(1), + "Shape mismatch: b_q_weight.size(1) = ", + b_q_weight.size(1), + ", size_k = ", + size_k, + ", tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + TORCH_CHECK( + b_q_weight.size(2) % MARLIN_NAMESPACE_NAME::tile_size == 0, + "b_q_weight.size(2) = ", + b_q_weight.size(2), + " is not divisible by tile_size = ", + MARLIN_NAMESPACE_NAME::tile_size); + int actual_size_n = (b_q_weight.size(2) / MARLIN_NAMESPACE_NAME::tile_size) * pack_factor; + TORCH_CHECK(size_n == actual_size_n, "size_n = ", size_n, ", actual_size_n = ", actual_size_n); + + // Verify device and strides + TORCH_CHECK(a.device().is_cuda(), "A is not on GPU"); + TORCH_CHECK(a.is_contiguous(), "A is not contiguous"); + + TORCH_CHECK(b_q_weight.device().is_cuda(), "b_q_weight is not on GPU"); + TORCH_CHECK(b_q_weight.is_contiguous(), "b_q_weight is not contiguous"); + + TORCH_CHECK(b_scales.device().is_cuda(), "b_scales is not on GPU"); + TORCH_CHECK(b_scales.is_contiguous(), "b_scales is not contiguous"); + + // thread_k: `k` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_k = -1; + // thread_n: `n` size of a thread_tile in `weights` (can usually be left as + // auto -1) + int thread_n = -1; + // sms: number of SMs to use for the kernel + int sms = -1; + cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, a.get_device()); + + // Alloc buffers + const at::cuda::OptionalCUDAGuard device_guard(device_of(a)); + auto options = torch::TensorOptions().dtype(a.dtype()).device(a.device()); + torch::Tensor c; + if (c_or_none.has_value()) { + c = c_or_none.value(); + TORCH_CHECK(c.device().is_cuda(), "c is not on GPU"); + TORCH_CHECK(c.is_contiguous(), "c is not contiguous"); + TORCH_CHECK( + c.size(0) == size_m * top_k, "Shape mismatch: c.size(0) = ", c.size(0), ", size_m * topk = ", size_m * top_k); + TORCH_CHECK(c.size(1) == size_n, "Shape mismatch: c.size(1) = ", c.size(1), ", size_n = ", size_n); + } else { + c = torch::empty({size_m * top_k, size_n}, options); + } + + // Alloc C tmp buffer that is going to be used for the global reduce + torch::Tensor c_tmp; + auto options_fp32 = torch::TensorOptions().dtype(at::kFloat).device(a.device()); + if (use_fp32_reduce && !use_atomic_add) { + // max num of threadblocks is sms * 4 + long max_c_tmp_size = min( + (long)size_n * sorted_token_ids.size(0), (long)sms * 4 * moe_block_size * MARLIN_NAMESPACE_NAME::max_thread_n); + if (moe_block_size == 8) max_c_tmp_size *= 2; + c_tmp = torch::empty({max_c_tmp_size}, options_fp32); + } else { + c_tmp = torch::empty({0}, options_fp32); + } + + // Detect groupsize and act_order + int num_groups = -1; + int group_size = -1; + + int rank = b_scales.sizes().size(); + TORCH_CHECK(rank == 3, "b_scales rank = ", rank, " is not 3"); + TORCH_CHECK(b_scales.size(2) == size_n, "b_scales dim 2 = ", b_scales.size(2), " is not size_n = ", size_n); + num_groups = b_scales.size(1); + + torch::Tensor g_idx, perm, a_tmp; + ; + if (g_idx_or_none.has_value() && perm_or_none.has_value()) { + g_idx = g_idx_or_none.value(); + perm = perm_or_none.value(); + + TORCH_CHECK(g_idx.device().is_cuda(), "g_idx is not on GPU"); + TORCH_CHECK(g_idx.is_contiguous(), "g_idx is not contiguous"); + TORCH_CHECK(perm.device().is_cuda(), "perm is not on GPU"); + TORCH_CHECK(perm.is_contiguous(), "perm is not contiguous"); + + // Verify g_idx and perm + TORCH_CHECK( + (g_idx.size(-1) == 0 && perm.size(-1) == 0) || (g_idx.size(-1) == size_k && perm.size(-1) == size_k), + "Unexpected g_idx.size(-1) = ", + g_idx.size(-1), + " and perm.size(-1) = ", + perm.size(-1), + ", where size_k = ", + size_k); + } else { + g_idx = torch::empty({0}, options); + perm = torch::empty({0}, options); + a_tmp = torch::empty({0}, options); + } + bool has_act_order = g_idx.size(-1) > 0 && perm.size(-1) > 0; + + if (has_act_order) { + a_tmp = torch::empty({size_m * top_k, size_k}, options); + if (is_k_full) { + TORCH_CHECK(num_groups > 1, "For act_order, num_groups must be > 1"); + TORCH_CHECK(size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by num_groups = ", num_groups); + group_size = size_k / num_groups; + } else { + group_size = 0; + } + + } else { + a_tmp = torch::empty({0}, options); + if (num_groups > 1) { + TORCH_CHECK( + size_k % num_groups == 0, "size_k = ", size_k, ", is not divisible by b_scales.size(1) = ", b_scales.size(1)); + group_size = size_k / num_groups; + } else { + group_size = -1; + } + } + + torch::Tensor b_zeros; + if (b_zeros_or_none.has_value()) { + b_zeros = b_zeros_or_none.value(); + TORCH_CHECK(b_zeros.device().is_cuda(), "b_zeros is not on GPU"); + TORCH_CHECK(b_zeros.is_contiguous(), "b_zeros is not contiguous"); + } else { + b_zeros = torch::empty({0}, options); + } + bool has_zp = b_zeros.size(-1) > 0; + + if (has_zp) { + TORCH_CHECK(b_q_type == sglang::kU4, "b_q_type must be u4 when has_zp = True. Got = ", b_q_type.str()); + } else { + TORCH_CHECK( + b_q_type == sglang::kU4B8 || b_q_type == sglang::kU8B128, + "b_q_type must be uint4b8 or uint8b128 when has_zp = False. Got = ", + b_q_type.str()); + } + + if (has_zp && is_zp_float) { + TORCH_CHECK( + a.scalar_type() == at::ScalarType::Half, + "Computation type must be float16 (half) when using float zero " + "points."); + } + + // Verify b_zeros + if (has_zp) { + int rank = b_zeros.sizes().size(); + TORCH_CHECK(rank == 3, "b_zeros rank = ", rank, " is not 3"); + if (is_zp_float) { + TORCH_CHECK(b_zeros.size(2) == size_n, "b_zeros dim 2 = ", b_zeros.size(2), " is not size_n = ", size_n); + TORCH_CHECK( + num_groups == b_zeros.size(1), "b_zeros dim 1 = ", b_zeros.size(1), " is not num_groups = ", num_groups); + TORCH_CHECK(num_groups != -1, "num_groups must be != -1"); + } else { + TORCH_CHECK( + b_zeros.size(1) == num_groups, "b_zeros dim 1 = ", b_zeros.size(1), " is not num_groups = ", num_groups); + TORCH_CHECK( + b_zeros.size(2) == size_n / pack_factor, + "b_zeros dim 2 = ", + b_zeros.size(2), + " is not size_n / pack_factor = ", + size_n / pack_factor); + } + } + + // Verify workspace size + TORCH_CHECK( + size_n % MARLIN_NAMESPACE_NAME::min_thread_n == 0, + "size_n = ", + size_n, + ", is not divisible by min_thread_n = ", + MARLIN_NAMESPACE_NAME::min_thread_n); + + int max_n_tiles = size_n / MARLIN_NAMESPACE_NAME::min_thread_n; + int min_workspace_size = min(max_n_tiles * (int)(sorted_token_ids.size(0) / moe_block_size), sms * 4); + TORCH_CHECK( + workspace.numel() >= min_workspace_size, + "workspace.numel = ", + workspace.numel(), + " is below min_workspace_size = ", + min_workspace_size); + + int dev = a.get_device(); + if (a.scalar_type() == at::ScalarType::Half) { + MARLIN_NAMESPACE_NAME::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + b_scales.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + sorted_token_ids.data_ptr(), + expert_ids.data_ptr(), + num_tokens_past_padded.data_ptr(), + topk_weights.data_ptr(), + moe_block_size, + top_k, + mul_topk_weights, + is_ep, + size_m, + size_n, + size_k, + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else if (a.scalar_type() == at::ScalarType::BFloat16) { + MARLIN_NAMESPACE_NAME::marlin_mm( + a.data_ptr(), + b_q_weight.data_ptr(), + c.data_ptr(), + c_tmp.data_ptr(), + b_scales.data_ptr(), + b_zeros.data_ptr(), + g_idx.data_ptr(), + perm.data_ptr(), + a_tmp.data_ptr(), + sorted_token_ids.data_ptr(), + expert_ids.data_ptr(), + num_tokens_past_padded.data_ptr(), + topk_weights.data_ptr(), + moe_block_size, + top_k, + mul_topk_weights, + is_ep, + size_m, + size_n, + size_k, + workspace.data_ptr(), + b_q_type, + has_act_order, + is_k_full, + has_zp, + num_groups, + group_size, + dev, + at::cuda::getCurrentCUDAStream(dev), + thread_k, + thread_n, + sms, + use_atomic_add, + use_fp32_reduce, + is_zp_float); + } else { + TORCH_CHECK(false, "moe_wna16_marlin_gemm only supports bfloat16 and float16"); + } + + return c; +} + +#endif diff --git a/sgl-kernel/csrc/moe/moe_align_kernel.cu b/sgl-kernel/csrc/moe/moe_align_kernel.cu new file mode 100644 index 000000000..64429e1ac --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_align_kernel.cu @@ -0,0 +1,364 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include + +#include "utils.h" +#define WARP_SIZE 64 +#define VEC_SIZE 4 +using Vec = int4; + +template +__global__ void count_and_sort_expert_tokens_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ cumsum_buffer, + size_t numel) { + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = blockDim.x * gridDim.x; + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1); + sorted_token_ids[rank_post_pad] = i; + } +} + +#ifdef __CUDA_ARCH__ +__device__ __forceinline__ int warp_exclusive_scan(int v, unsigned mask = 0xffffffffu) { + int original = v; +#pragma unroll + for (int offset = 1; offset < WARP_SIZE; offset <<= 1) { + int n = __shfl_up(v, offset); + if ((threadIdx.x & (WARP_SIZE - 1)) >= offset) v += n; + } + return v - original; +} +#endif + +template +__global__ void moe_align_block_size_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + int32_t* __restrict__ cumsum, + bool pad_sorted_token_ids, + const int32_t scan_size) { + extern __shared__ int32_t smem[]; + int32_t* shared_counts = smem; // [num_experts] + int32_t* prefix = shared_counts + num_experts; // [num_experts + 1] + int32_t* scan_buf = prefix + num_experts + 1; // [scan_size] + __shared__ int32_t s_total_tokens_post_pad; + + const size_t tid = threadIdx.x; + const size_t stride = blockDim.x; + + if (tid < num_experts) { + shared_counts[tid] = 0; + } + + __syncthreads(); + + for (size_t i = tid; i < numel; i += stride) { + int expert_id = topk_ids[i] + 1; + atomicAdd(&shared_counts[expert_id], 1); + } + + __syncthreads(); + + int32_t padded_count = 0; + if (tid < num_experts) { + int32_t count = shared_counts[tid]; + padded_count = (count + block_size - 1) / block_size * block_size; + scan_buf[tid] = padded_count; + } + +#ifndef __CUDA_ARCH__ // HIP + + if (tid >= num_experts && tid < scan_size) { + scan_buf[tid] = 0; + } + + __syncthreads(); + + // Blelloch scan + int offset = 1; +#pragma unroll + for (int d = scan_size >> 1; d > 0; d >>= 1) { + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + scan_buf[bi] += scan_buf[ai]; + } + offset <<= 1; + __syncthreads(); + } + + // down-sweep + if (tid == 0) { + prefix[num_experts] = scan_buf[scan_size - 1]; + scan_buf[scan_size - 1] = 0; + } + __syncthreads(); + +#pragma unroll + for (int d = 1; d < scan_size; d <<= 1) { + offset >>= 1; + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + if (bi < scan_size) { + int temp = scan_buf[ai]; + scan_buf[ai] = scan_buf[bi]; + scan_buf[bi] += temp; + } + } + __syncthreads(); + } + + if (tid < num_experts) { + prefix[tid] = scan_buf[tid]; + } + + if (tid == 0) { + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + +#else // CUDA + + // Intra warp prefix sum + int32_t* warp_sums = scan_buf + scan_size; // [<= 32] + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid & (WARP_SIZE - 1); + const int num_warps_for_scan = (scan_size + WARP_SIZE - 1) / WARP_SIZE; + const int warp_sum = warp_exclusive_scan(padded_count) + padded_count; + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = warp_sum; + __syncthreads(); + + // warp0 accumulate all the block's prefix sum + if (tid < WARP_SIZE) { + int val = (tid < num_warps_for_scan) ? warp_sums[tid] : 0; + int incl = warp_exclusive_scan(val) + val; + warp_sums[tid] = incl; + } + __syncthreads(); + + // Every thread obtains the whole block's sum + if (tid == 0) { + prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + + // Fill 0 to scan_buf extended area (tid >= num_expert) + if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0; + __syncthreads(); + + // Perform 2 level exclusive-prefix-sum to scan_buf + int v = (tid < scan_size) ? scan_buf[tid] : 0; + int pre = warp_exclusive_scan(v); + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = pre + v; + __syncthreads(); + + if (warp_id == 0) { + int val = (lane_id < num_warps_for_scan) ? warp_sums[lane_id] : 0; + warp_sums[lane_id] = warp_exclusive_scan(val); + } + __syncthreads(); + + int offset = warp_sums[warp_id]; + if (tid < scan_size) scan_buf[tid] = pre + offset; + __syncthreads(); + + // Write prefix[0..num_experts - 1] and cumsum + if (tid < num_experts) prefix[tid] = scan_buf[tid]; +#endif + + if (tid <= num_experts) { + cumsum[tid] = prefix[tid]; + } + // fill expert_ids + const int32_t num_blocks = s_total_tokens_post_pad / block_size; + for (int32_t i = tid; i < num_blocks; i += stride) { + int32_t block_start = i * block_size; + int left = 0, right = num_experts; + while (left < right) { + int mid = (left + right) >> 1; + if (prefix[mid] <= block_start) { + left = mid + 1; + } else { + right = mid; + } + } + expert_ids[i] = left - 2; + } + + if (pad_sorted_token_ids) { + Vec fill_vec; + fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; + int32_t total_vecs = (s_total_tokens_post_pad + VEC_SIZE - 1) / VEC_SIZE; + Vec* out_ptr = reinterpret_cast(sorted_token_ids); + for (int32_t i = tid; i < total_vecs; i += stride) { + out_ptr[i] = fill_vec; + } + } +} + +template +__global__ void moe_align_block_size_small_batch_expert_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + bool pad_sorted_token_ids) { + const size_t tid = threadIdx.x; + const size_t stride = blockDim.x; + + extern __shared__ int32_t shared_mem[]; + int32_t* cumsum = shared_mem; + int32_t* tokens_cnts = (int32_t*)(shared_mem + num_experts + 1); + + for (int i = 0; i < num_experts; ++i) { + tokens_cnts[(threadIdx.x + 1) * num_experts + i] = 0; + } + + for (size_t i = tid; i < numel; i += stride) { + ++tokens_cnts[(threadIdx.x + 1) * num_experts + topk_ids[i] + 1]; + } + + __syncthreads(); + + if (threadIdx.x < num_experts) { + tokens_cnts[threadIdx.x] = 0; + for (int i = 1; i <= blockDim.x; ++i) { + tokens_cnts[i * num_experts + threadIdx.x] += tokens_cnts[(i - 1) * num_experts + threadIdx.x]; + } + } + + __syncthreads(); + + if (threadIdx.x == 0) { + cumsum[0] = 0; + for (int i = 1; i <= num_experts; ++i) { + cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[blockDim.x * num_experts + i - 1], block_size) * block_size; + } + *total_tokens_post_pad = static_cast(cumsum[num_experts]); + } + + __syncthreads(); + + if (threadIdx.x < num_experts) { + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1]; i += block_size) { + expert_ids[i / block_size] = threadIdx.x - 1; + } + } + + if (pad_sorted_token_ids) { + Vec fill_vec; + fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; + int32_t total_vecs = (*total_tokens_post_pad + VEC_SIZE - 1) / VEC_SIZE; + Vec* out_ptr = reinterpret_cast(sorted_token_ids); + for (int32_t i = tid; i < total_vecs; i += stride) { + out_ptr[i] = fill_vec; + } + } + + __syncthreads(); + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = tokens_cnts[threadIdx.x * num_experts + expert_id] + cumsum[expert_id]; + sorted_token_ids[rank_post_pad] = i; + ++tokens_cnts[threadIdx.x * num_experts + expert_id]; + } +} + +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor cumsum_buffer, + bool pad_sorted_token_ids) { + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + int threads = 1024; + + threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; + + DISPATCH_INTEGRAL_TYPES(topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] { + bool small_batch_expert_mode = (topk_ids.numel() < 1024) && (num_experts <= 64); + + if (small_batch_expert_mode) { + const int32_t threads = max((int32_t)num_experts, WARP_SIZE); + const int32_t shared_mem_size = ((threads + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t); + + auto small_batch_expert_kernel = moe_align_block_size_small_batch_expert_kernel; + small_batch_expert_kernel<<<1, threads, shared_mem_size, stream>>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + pad_sorted_token_ids); + } else { + auto align_kernel = moe_align_block_size_kernel; + + const size_t scan_size = next_pow2(num_experts); + const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); + align_kernel<<<1, threads, shared_mem_size, stream>>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + cumsum_buffer.data_ptr(), + pad_sorted_token_ids, + scan_size); + + const int block_threads = std::min(256, (int)threads); + const int num_blocks = (topk_ids.numel() + block_threads - 1) / block_threads; + const int max_blocks = 65535; + const int actual_blocks = std::min(num_blocks, max_blocks); + + auto sort_kernel = count_and_sort_expert_tokens_kernel; + sort_kernel<<>>( + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + cumsum_buffer.data_ptr(), + topk_ids.numel()); + } + }); +} diff --git a/sgl-kernel/csrc/moe/moe_align_kernel.hip b/sgl-kernel/csrc/moe/moe_align_kernel.hip new file mode 100644 index 000000000..c462ac6ce --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_align_kernel.hip @@ -0,0 +1,367 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include + +#include "utils_hip.h" +#define WARP_SIZE 64 +#define VEC_SIZE 4 +using Vec = int4; + +template +__global__ void count_and_sort_expert_tokens_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ cumsum_buffer, + size_t numel) { + const size_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const size_t stride = blockDim.x * gridDim.x; + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = atomicAdd(&cumsum_buffer[expert_id], 1); + sorted_token_ids[rank_post_pad] = i; + } +} + +#ifdef __DTK_ARCH__ +__device__ __forceinline__ int warp_exclusive_scan(int v, unsigned mask = 0xffffffffu) { + int original = v; +#pragma unroll + for (int offset = 1; offset < WARP_SIZE; offset <<= 1) { + int n = __shfl_up(v, offset); + if ((threadIdx.x & (WARP_SIZE - 1)) >= offset) v += n; + } + return v - original; +} +#endif + +template +__global__ void moe_align_block_size_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + int32_t* __restrict__ cumsum, + bool pad_sorted_token_ids, + const int32_t scan_size) { + extern __shared__ int32_t smem[]; + int32_t* shared_counts = smem; // [num_experts] + int32_t* prefix = shared_counts + num_experts; // [num_experts + 1] + int32_t* scan_buf = prefix + num_experts + 1; // [scan_size] + __shared__ int32_t s_total_tokens_post_pad; + + const size_t tid = threadIdx.x; + const size_t stride = blockDim.x; + + if (tid < num_experts) { + shared_counts[tid] = 0; + } + + __syncthreads(); + + for (size_t i = tid; i < numel; i += stride) { + int expert_id = topk_ids[i] + 1; + atomicAdd(&shared_counts[expert_id], 1); + } + + __syncthreads(); + + int32_t padded_count = 0; + if (tid < num_experts) { + int32_t count = shared_counts[tid]; + padded_count = (count + block_size - 1) / block_size * block_size; + scan_buf[tid] = padded_count; + } + +#ifndef __DTK_ARCH__ // HIP + + if (tid >= num_experts && tid < scan_size) { + scan_buf[tid] = 0; + } + + __syncthreads(); + + // Blelloch scan + int offset = 1; +#pragma unroll + for (int d = scan_size >> 1; d > 0; d >>= 1) { + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + scan_buf[bi] += scan_buf[ai]; + } + offset <<= 1; + __syncthreads(); + } + + // down-sweep + if (tid == 0) { + prefix[num_experts] = scan_buf[scan_size - 1]; + scan_buf[scan_size - 1] = 0; + } + __syncthreads(); + +#pragma unroll + for (int d = 1; d < scan_size; d <<= 1) { + offset >>= 1; + if (tid < d) { + int ai = offset * (2 * tid + 1) - 1; + int bi = offset * (2 * tid + 2) - 1; + if (bi < scan_size) { + int temp = scan_buf[ai]; + scan_buf[ai] = scan_buf[bi]; + scan_buf[bi] += temp; + } + } + __syncthreads(); + } + + if (tid < num_experts) { + prefix[tid] = scan_buf[tid]; + } + + if (tid == 0) { + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + +#else // CUDA + + // Intra warp prefix sum + int32_t* warp_sums = scan_buf + scan_size; // [<= 32] + const int warp_id = tid / WARP_SIZE; + const int lane_id = tid & (WARP_SIZE - 1); + const int num_warps_for_scan = (scan_size + WARP_SIZE - 1) / WARP_SIZE; + const int warp_sum = warp_exclusive_scan(padded_count) + padded_count; + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = warp_sum; + __syncthreads(); + + // warp0 accumulate all the block's prefix sum + if (tid < WARP_SIZE) { + int val = (tid < num_warps_for_scan) ? warp_sums[tid] : 0; + int incl = warp_exclusive_scan(val) + val; + warp_sums[tid] = incl; + } + __syncthreads(); + + // Every thread obtains the whole block's sum + if (tid == 0) { + prefix[num_experts] = warp_sums[num_warps_for_scan - 1]; + s_total_tokens_post_pad = prefix[num_experts]; + *total_tokens_post_pad = s_total_tokens_post_pad; + } + __syncthreads(); + + // Fill 0 to scan_buf extended area (tid >= num_expert) + if (tid >= num_experts && tid < scan_size) scan_buf[tid] = 0; + __syncthreads(); + + // Perform 2 level exclusive-prefix-sum to scan_buf + int v = (tid < scan_size) ? scan_buf[tid] : 0; + int pre = warp_exclusive_scan(v); + if (lane_id == WARP_SIZE - 1) warp_sums[warp_id] = pre + v; + __syncthreads(); + + if (warp_id == 0) { + int val = (lane_id < num_warps_for_scan) ? warp_sums[lane_id] : 0; + warp_sums[lane_id] = warp_exclusive_scan(val); + } + __syncthreads(); + + int offset = warp_sums[warp_id]; + if (tid < scan_size) scan_buf[tid] = pre + offset; + __syncthreads(); + + // Write prefix[0..num_experts - 1] and cumsum + if (tid < num_experts) prefix[tid] = scan_buf[tid]; +#endif + + if (tid <= num_experts) { + cumsum[tid] = prefix[tid]; + } + // fill expert_ids + const int32_t num_blocks = s_total_tokens_post_pad / block_size; + for (int32_t i = tid; i < num_blocks; i += stride) { + int32_t block_start = i * block_size; + int left = 0, right = num_experts; + while (left < right) { + int mid = (left + right) >> 1; + if (prefix[mid] <= block_start) { + left = mid + 1; + } else { + right = mid; + } + } + expert_ids[i] = left - 2; + } + + if (pad_sorted_token_ids) { + Vec fill_vec; + fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; + int32_t total_vecs = (s_total_tokens_post_pad + VEC_SIZE - 1) / VEC_SIZE; + Vec* out_ptr = reinterpret_cast(sorted_token_ids); + for (int32_t i = tid; i < total_vecs; i += stride) { + out_ptr[i] = fill_vec; + } + } +} + +template +__global__ void moe_align_block_size_small_batch_expert_kernel( + const scalar_t* __restrict__ topk_ids, + int32_t* __restrict__ sorted_token_ids, + int32_t* __restrict__ expert_ids, + int32_t* __restrict__ total_tokens_post_pad, + int32_t num_experts, + int32_t block_size, + size_t numel, + bool pad_sorted_token_ids) { + const size_t tid = threadIdx.x; + const size_t stride = blockDim.x; + + extern __shared__ int32_t shared_mem[]; + int32_t* cumsum = shared_mem; + int32_t* tokens_cnts = (int32_t*)(shared_mem + num_experts + 1); + + for (int i = 0; i < num_experts; ++i) { + tokens_cnts[(threadIdx.x + 1) * num_experts + i] = 0; + } + + for (size_t i = tid; i < numel; i += stride) { + ++tokens_cnts[(threadIdx.x + 1) * num_experts + topk_ids[i] + 1]; + } + + __syncthreads(); + + if (threadIdx.x < num_experts) { + tokens_cnts[threadIdx.x] = 0; + for (int i = 1; i <= blockDim.x; ++i) { + tokens_cnts[i * num_experts + threadIdx.x] += tokens_cnts[(i - 1) * num_experts + threadIdx.x]; + } + } + + __syncthreads(); + + if (threadIdx.x == 0) { + cumsum[0] = 0; + for (int i = 1; i <= num_experts; ++i) { + cumsum[i] = cumsum[i - 1] + CEILDIV(tokens_cnts[blockDim.x * num_experts + i - 1], block_size) * block_size; + } + *total_tokens_post_pad = static_cast(cumsum[num_experts]); + } + + __syncthreads(); + + if (threadIdx.x < num_experts) { + for (int i = cumsum[threadIdx.x]; i < cumsum[threadIdx.x + 1]; i += block_size) { + expert_ids[i / block_size] = threadIdx.x - 1; + } + } + + if (pad_sorted_token_ids) { + Vec fill_vec; + fill_vec.x = fill_vec.y = fill_vec.z = fill_vec.w = numel; + int32_t total_vecs = (*total_tokens_post_pad + VEC_SIZE - 1) / VEC_SIZE; + Vec* out_ptr = reinterpret_cast(sorted_token_ids); + for (int32_t i = tid; i < total_vecs; i += stride) { + out_ptr[i] = fill_vec; + } + } + + __syncthreads(); + + for (size_t i = tid; i < numel; i += stride) { + int32_t expert_id = topk_ids[i] + 1; + int32_t rank_post_pad = tokens_cnts[threadIdx.x * num_experts + expert_id] + cumsum[expert_id]; + sorted_token_ids[rank_post_pad] = i; + ++tokens_cnts[threadIdx.x * num_experts + expert_id]; + } +} + +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor cumsum_buffer, + bool pad_sorted_token_ids) { + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + + int threads = 1024; + + threads = ((threads + WARP_SIZE - 1) / WARP_SIZE) * WARP_SIZE; + + DISPATCH_INTEGRAL_TYPES(topk_ids.scalar_type(), "moe_align_block_size_kernel", [&] { + bool small_batch_expert_mode = (topk_ids.numel() < 1024) && (num_experts <= 64); + + if (small_batch_expert_mode) { + const int32_t threads = max((int32_t)num_experts, WARP_SIZE); + const int32_t shared_mem_size = ((threads + 1) * num_experts + (num_experts + 1)) * sizeof(int32_t); + + auto small_batch_expert_kernel = moe_align_block_size_small_batch_expert_kernel; + hipLaunchKernelGGL(( small_batch_expert_kernel), dim3(1), dim3(threads), shared_mem_size, stream, + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + pad_sorted_token_ids); + } else { + auto align_kernel = moe_align_block_size_kernel; + + const size_t scan_size = next_pow2(num_experts); + const size_t shared_mem_size = (num_experts + (num_experts + 1) + scan_size + WARP_SIZE) * sizeof(int32_t); + hipLaunchKernelGGL(( align_kernel), dim3(1), dim3(threads), shared_mem_size, stream, + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + experts_ids.data_ptr(), + num_tokens_post_pad.data_ptr(), + num_experts, + block_size, + topk_ids.numel(), + cumsum_buffer.data_ptr(), + pad_sorted_token_ids, + scan_size); + + const int block_threads = ::min(256, (int)threads); + const int num_blocks = (topk_ids.numel() + block_threads - 1) / block_threads; + const int max_blocks = 65535; + const int actual_blocks = ::min(num_blocks, max_blocks); + + auto sort_kernel = count_and_sort_expert_tokens_kernel; + hipLaunchKernelGGL(( sort_kernel), dim3(actual_blocks), dim3(block_threads), 0, stream, + topk_ids.data_ptr(), + sorted_token_ids.data_ptr(), + cumsum_buffer.data_ptr(), + topk_ids.numel()); + } + }); +} diff --git a/sgl-kernel/csrc/moe/moe_fused_gate.cu b/sgl-kernel/csrc/moe/moe_fused_gate.cu new file mode 100644 index 000000000..782a884fb --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_fused_gate.cu @@ -0,0 +1,521 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +template +using AlignedArray = cutlass::AlignedArray; +using bfloat16_t = cutlass::bfloat16_t; +using float16_t = cutlass::half_t; +using float32_t = float; + +// QQ NOTE: to handle the case for at::Half, error: more than one operator ">" matches these operands: built-in operator +// "arithmetic > arithmetic" function "operator>(const __half &, const __half &)" +template +__device__ inline bool cmp_gt(const T& a, const T& b) { + if constexpr (std::is_same::value) { + // at::Half (or float16_t in our native case) causes ambiguity, so we cast to float. + return static_cast(a) > static_cast(b); + } else { + // For types like float, at::BFloat16, or cutlass::half_t / cutlass::bfloat16_t, assume operator> works as expected. + return a > b; + } +} + +template +__device__ inline bool cmp_eq(const T& a, const T& b) { + if constexpr (std::is_same::value) { + return static_cast(a) == static_cast(b); + } else { + return a == b; + } +} + +// Fixed constants common to both dynamic and static template versions: +static constexpr int WARP_SIZE = 32; +static constexpr int WARPS_PER_CTA = 6; +static constexpr int MAX_VPT = 32; // maximum VPT we support, > params.VPT = num_expert / num_expert_group + +// Create an alias for Array using AlignedArray +template +using Array = AlignedArray; +// QQ: NOTE expression must have a constant value, this has to be > params.VPT +template +using AccessType = AlignedArray; + +template +__device__ void moe_fused_gate_impl( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output, + Params params) { + int tidx = threadIdx.x; + int64_t thread_row = + blockIdx.x * params.ROWS_PER_CTA + threadIdx.y * params.ROWS_PER_WARP + tidx / params.THREADS_PER_ROW; + if (thread_row >= num_rows) { + return; + } + + // Calculate topk_excluding_share_expert_fusion from topk + int64_t topk_excluding_share_expert_fusion = topk - num_fused_shared_experts; + + // Cast pointers to type T: + auto* input_ptr = reinterpret_cast(input); + auto* bias_ptr = reinterpret_cast(bias); + auto* thread_row_ptr = input_ptr + thread_row * params.NUM_EXPERTS; + + int thread_group_idx = tidx % params.THREADS_PER_ROW; + int first_elt_read_by_thread = thread_group_idx * params.VPT; + + // Create local arrays for the row chunk and bias chunk and then reinterpret the address of row_chunk as a pointer to + // AccessType. + T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + Array row_chunk; + AccessType const* vec_thread_read_ptr = reinterpret_cast const*>(thread_read_ptr); + + T* bias_thread_read_ptr = bias_ptr + first_elt_read_by_thread; + Array bias_chunk; + AccessType const* vec_bias_thread_read_ptr = reinterpret_cast const*>(bias_thread_read_ptr); + +// QQ NOTE: doing the follow will be slower than loop assign and more importantly +// have misaligned address issue when params.VPT < 8 and mismatch with MAX_VPT +// AccessType* row_chunk_vec_ptr = reinterpret_cast*>(&row_chunk); +// row_chunk_vec_ptr[0] = vec_thread_read_ptr[0]; +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + row_chunk[ii] = vec_thread_read_ptr[0][ii]; + bias_chunk[ii] = vec_bias_thread_read_ptr[0][ii]; + } + + __syncthreads(); + +////////////////////// Sigmoid ////////////////////// +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + row_chunk[ii] = static_cast(1.0f / (1.0f + expf(-float(row_chunk[ii])))); + } + __syncthreads(); + +////////////////////// Add Bias ////////////////////// +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + bias_chunk[ii] = row_chunk[ii] + bias_chunk[ii]; + } + +////////////////////// Exclude Groups ////////////////////// +#pragma unroll + for (int k_idx = 0; k_idx < params.THREADS_PER_ROW - topk_group; + ++k_idx) { // QQ NOTE Here params.THREADS_PER_ROW = num_expert_group + int expert = first_elt_read_by_thread; + // local argmax + T max_val = static_cast(-FLT_MAX); + T max_val_second = static_cast(-FLT_MAX); +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + T val = bias_chunk[ii]; + + if (cmp_gt(val, max_val)) { + max_val_second = max_val; + max_val = val; + } else if (cmp_gt(val, max_val_second)) { + max_val_second = val; + } + } + + // QQ NOTE: currently fixed to pick top2 sigmoid weight value in each expert group and sum them as the group weight + // to select expert groups + T max_sum = max_val + max_val_second; + +// argmin reduce +#pragma unroll + for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + T other_max_sum = + static_cast(__shfl_xor_sync(0xFFFFFFFF, static_cast(max_sum), mask, params.THREADS_PER_ROW)); + int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW); + + // higher indices win + if (cmp_gt(max_sum, other_max_sum) || (cmp_eq(other_max_sum, max_sum) && other_expert > expert)) { + max_sum = other_max_sum; + expert = other_expert; + } + } + + // clear the max value in the thread + if (k_idx < params.THREADS_PER_ROW - topk_group) { + int const thread_to_clear_in_group = expert / params.VPT; + + if (thread_group_idx == thread_to_clear_in_group) { +#pragma unroll + for (int ii = 0; ii < params.VPT; ++ii) { + bias_chunk[ii] = static_cast(FLT_MAX); + } + } + } + } + + __syncthreads(); + + ////////////////////// Topk ////////////////////// + float output_sum = 0.0f; + for (int k_idx = 0; k_idx < topk_excluding_share_expert_fusion; ++k_idx) { + // local argmax + T max_val = bias_chunk[0]; + int expert = first_elt_read_by_thread; + + if (!cmp_eq(max_val, static_cast(FLT_MAX))) { +#pragma unroll + for (int ii = 1; ii < params.VPT; ++ii) { + T val = bias_chunk[ii]; + if (cmp_gt(val, max_val)) { + max_val = val; + expert = first_elt_read_by_thread + ii; + } + } + } else { + max_val = static_cast(-FLT_MAX); + } + + // argmax reduce +#pragma unroll + for (int mask = params.THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + T other_max = + static_cast(__shfl_xor_sync(0xFFFFFFFF, static_cast(max_val), mask, params.THREADS_PER_ROW)); + int other_expert = __shfl_xor_sync(0xFFFFFFFF, expert, mask, params.THREADS_PER_ROW); + + // lower indices to win + if (cmp_gt(other_max, max_val) || (cmp_eq(other_max, max_val) && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + int thread_to_clear_in_group = expert / params.VPT; + int64_t idx = topk * thread_row + k_idx; + + if (thread_group_idx == thread_to_clear_in_group) { + int expert_to_clear_in_thread = expert % params.VPT; + + // clear the max value in the thread + bias_chunk[expert_to_clear_in_thread] = static_cast(-FLT_MAX); + + // store output + output_ptr[idx] = static_cast(row_chunk[expert_to_clear_in_thread]); + indices_ptr[idx] = static_cast(expert); + } + + // accumulate sum for all elements + if (thread_group_idx == 0) { + output_sum += output_ptr[idx]; + } + + __syncthreads(); + } + + if (thread_group_idx == 0 && num_fused_shared_experts > 0) { + int64_t last_idx = topk * thread_row + topk_excluding_share_expert_fusion; + int64_t expert_offset = 0; + indices_ptr[last_idx] = static_cast(params.NUM_EXPERTS + expert_offset); + + // Set the weight to the sum of all weights divided by routed_scaling_factor + output_ptr[last_idx] = output_sum / routed_scaling_factor; + + if (num_fused_shared_experts > 1) { + for (int i = 1; i < num_fused_shared_experts; ++i) { + ++last_idx; + ++expert_offset; + indices_ptr[last_idx] = static_cast(params.NUM_EXPERTS + expert_offset); + // Set the weight to the sum of all weights divided by routed_scaling_factor + output_ptr[last_idx] = output_sum / routed_scaling_factor; + } + } + } + __syncthreads(); + + ////////////////////// Rescale Output ////////////////////// + if (thread_group_idx == 0) { +#pragma unroll + for (int ii = 0; ii < topk; ++ii) { + int64_t const idx = topk * thread_row + ii; + output_ptr[idx] = output_ptr[idx] / output_sum; + if (apply_routed_scaling_factor_on_output) { + output_ptr[idx] *= routed_scaling_factor; + } + } + } +} + +//------------------------------------------------------------------------------ +// Templated Kernel Version (using compile-time constants) +//------------------------------------------------------------------------------ +template +struct KernelParams { + static constexpr int VPT = VPT_; + static constexpr int NUM_EXPERTS = NUM_EXPERTS_; + static constexpr int THREADS_PER_ROW = THREADS_PER_ROW_; + static constexpr int ROWS_PER_WARP = ROWS_PER_WARP_; + static constexpr int ROWS_PER_CTA = ROWS_PER_CTA_; + static constexpr int WARPS_PER_CTA = WARPS_PER_CTA_; +}; + +template < + typename T, + int VPT, + int NUM_EXPERTS, + int THREADS_PER_ROW, + int ROWS_PER_WARP, + int ROWS_PER_CTA, + int WARPS_PER_CTA> +__global__ void moe_fused_gate_kernel( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + KernelParams params; + moe_fused_gate_impl( + input, + bias, + output_ptr, + indices_ptr, + num_rows, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + params); +} + +// Macro to compute compile-time constants and launch the kernel. +#define LAUNCH_MOE_GATE_CONFIG(T, EXPERTS, EXPERT_GROUP) \ + do { \ + constexpr int VPT = (EXPERTS) / (EXPERT_GROUP); \ + /* If EXPERT_GROUP > WARP_SIZE, fall back to 1 row per warp */ \ + constexpr int ROWS_PER_WARP = ((EXPERT_GROUP) <= WARP_SIZE) ? (WARP_SIZE / (EXPERT_GROUP)) : 1; \ + constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; \ + moe_fused_gate_kernel \ + <<>>( \ + input.data_ptr(), \ + bias.data_ptr(), \ + output.data_ptr(), \ + indices.data_ptr(), \ + num_rows, \ + topk_group, \ + topk, \ + num_fused_shared_experts, \ + routed_scaling_factor, \ + apply_routed_scaling_factor_on_output); \ + dispatched = true; \ + } while (0) + +//------------------------------------------------------------------------------ +// Dynamic Kernel Version (parameters computed at runtime) +//------------------------------------------------------------------------------ +struct KernelParamsDynamic { + int VPT; + int NUM_EXPERTS; + int THREADS_PER_ROW; + int ROWS_PER_WARP; + int ROWS_PER_CTA; + int WARPS_PER_CTA; +}; + +template +__global__ void moe_fused_gate_kernel_dynamic( + void* input, + void* bias, + float* output_ptr, + int32_t* indices_ptr, + int64_t num_rows, + int64_t num_experts, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + KernelParamsDynamic params; + params.NUM_EXPERTS = num_experts; // e.g, for deepseek v3, this is 256 + params.VPT = num_experts / num_expert_group; // e.g., for deepseek v3, this is 256 / 8 = 32 + params.THREADS_PER_ROW = num_expert_group; // fixed as num_expert_group, e.g., for deepseek v3, this is 8 + params.WARPS_PER_CTA = WARPS_PER_CTA; // fixed as 6 + params.ROWS_PER_WARP = std::max(1, WARP_SIZE / num_expert_group); // WARP_SIZE is fixed as 32 + params.ROWS_PER_CTA = params.WARPS_PER_CTA * params.ROWS_PER_WARP; + + moe_fused_gate_impl( + input, + bias, + output_ptr, + indices_ptr, + num_rows, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + params); +} + +//------------------------------------------------------------------------------ +// Host Launcher Function +//------------------------------------------------------------------------------ +std::vector moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output) { + int64_t num_rows = input.size(0); + int32_t num_experts = input.size(1); + auto options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCUDA); + auto output = torch::empty({num_rows, topk}, options); + auto indices = torch::empty({num_rows, topk}, options.dtype(torch::kInt32)); + + // Compute grid dimensions based on runtime value for num_expert_group. + int64_t rows_per_warp = std::max(1, WARP_SIZE / num_expert_group); + int64_t num_warps = (num_rows + rows_per_warp - 1) / rows_per_warp; + int64_t num_blocks = (num_warps + WARPS_PER_CTA - 1) / WARPS_PER_CTA; + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + dim3 block_dim(WARP_SIZE, WARPS_PER_CTA); + + // Check 1: Ensure that num_experts is a power of 2. + TORCH_CHECK((num_experts & (num_experts - 1)) == 0, "num_experts must be a power of 2, but got ", num_experts); + + // Check 2: Ensure that num_experts is divisible by num_expert_group. (this also means num_expert_group is power of 2) + TORCH_CHECK( + num_experts % num_expert_group == 0, + "num_experts must be divisible by num_expert_group, but got ", + num_experts, + " / ", + num_expert_group); + + int computed_vpt = num_experts / num_expert_group; + // Check 3: Ensure that num_experts/num_expert_group does not exceed MAX_VPT=32. Maximum VPT indicate max value per + // threads we can process. + TORCH_CHECK( + computed_vpt <= MAX_VPT, + "Per group experts: num_experts / num_expert_group = (", + computed_vpt, + ") exceeds the maximum supported (", + MAX_VPT, + ")"); + + // Dispatch to templated kernel for known compile-time configurations. + // We currently only support for: + // Case 1: 256 experts, with 8 or 16 groups. + // Case 2: 128 experts, with 4 or 8 groups. + // Case 3: other cases, require 8 <= num_experts / num_expert_group <= 32 + bool dispatched = false; + switch (num_experts) { + case 256: + if (num_expert_group == 8) + // This is deepseek v3 case. Here VPT = 256/8 = 32, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 8); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 8); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 8); + } else if (num_expert_group == 16) + // Here VPT = 256/16 = 16, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 256, 16); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 256, 16); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 256, 16); + } + break; + case 128: + if (num_expert_group == 4) + // VPT = 128/4 = 32, ROWS_PER_WARP = 32/16 = 2, ROWS_PER_CTA = 6 * 2 = 12. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 4); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 4); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 4); + } else if (num_expert_group == 8) + // VPT = 128/8 = 16, ROWS_PER_WARP = 32/8 = 4, ROWS_PER_CTA = 6 * 4 = 24. + if (input.scalar_type() == at::kBFloat16) { + LAUNCH_MOE_GATE_CONFIG(bfloat16_t, 128, 8); + } else if (input.scalar_type() == at::kHalf) { + LAUNCH_MOE_GATE_CONFIG(float16_t, 128, 8); + } else if (input.scalar_type() == at::kFloat) { + LAUNCH_MOE_GATE_CONFIG(float32_t, 128, 8); + } + break; + default: + break; + } + if (!dispatched) { + // Fallback to the dynamic kernel if none of the supported combinations match. + // currently only support num_experts / num_expert_group <= 32 for dynamic kernels + if (input.scalar_type() == at::kBFloat16) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else if (input.scalar_type() == at::kHalf) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else if (input.scalar_type() == at::kFloat) { + moe_fused_gate_kernel_dynamic<<>>( + input.data_ptr(), + bias.data_ptr(), + output.data_ptr(), + indices.data_ptr(), + num_rows, + num_experts, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output); + } else { + TORCH_CHECK(false, "Unsupported data type for moe_fused_gate"); + } + } + return {output, indices}; +} diff --git a/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu new file mode 100644 index 000000000..6ca4a7846 --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.cu @@ -0,0 +1,591 @@ +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu +// which is originally adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#ifndef USE_ROCM +#include +#include +#include +#else +#include +#include +#endif + +#include "utils.h" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +// Define reduction operators based on CUDA version +// CUDA 13 (12.9+) deprecated cub::Max/Min in favor of cuda::maximum/minimum +#if CUDA_VERSION >= 12090 +using MaxReduceOp = cuda::maximum<>; +using MinReduceOp = cuda::minimum<>; +#else +using MaxReduceOp = cub::Max; +using MinReduceOp = cub::Min; +#endif + +/// Aligned array type +template < + typename T, + /// Number of elements in the array + int N, + /// Alignment requirement in bytes + int Alignment = sizeof(T) * N> +class alignas(Alignment) AlignedArray { + T data[N]; +}; + +// ========================== Util functions to convert types ========================== +template +__device__ float convert_to_float(T x) { + if constexpr (std::is_same_v) { + return __half2float(x); + } else if constexpr (std::is_same_v) { + return __bfloat162float(x); + } else if constexpr (std::is_same_v) { + return x; + } else { + return static_cast(x); + } +} + +// ====================== Softmax things =============================== +// We have our own implementation of softmax here so we can support transposing the output +// in the softmax kernel when we extend this module to support expert-choice routing. +template +__launch_bounds__(TPB) __global__ + void moeSoftmax(const T* input, const bool* finished, float* output, const int num_cols) { + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + __shared__ float normalizing_factor; + __shared__ float float_max; + + const int thread_row_offset = blockIdx.x * num_cols; + + float threadData(-FLT_MAX); + + // Don't touch finished rows. + if ((finished != nullptr) && finished[blockIdx.x]) { + return; + } + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData = max(convert_to_float(input[idx]), threadData); + } + + const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, MaxReduceOp()); + + if (threadIdx.x == 0) { + float_max = maxElem; + } + __syncthreads(); + + threadData = 0; + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData += exp((convert_to_float(input[idx]) - float_max)); + } + + const auto Z = BlockReduce(tmpStorage).Sum(threadData); + + if (threadIdx.x == 0) { + normalizing_factor = 1.f / Z; + } + __syncthreads(); + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + const float val = exp((convert_to_float(input[idx]) - float_max)) * normalizing_factor; + output[idx] = val; + } +} + +template +__launch_bounds__(TPB) __global__ void moeTopK( + const float* inputs_after_softmax, + const bool* finished, + float* output, + int* indices, + const int num_experts, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize) { + using cub_kvp = cub::KeyValuePair; + using BlockReduce = cub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + cub_kvp thread_kvp; + cub::ArgMax arg_max; + + const int block_row = blockIdx.x; + + const bool row_is_active = finished ? !finished[block_row] : true; + const int thread_read_offset = blockIdx.x * num_experts; + float row_sum_for_renormalize = 0; + for (int k_idx = 0; k_idx < k; ++k_idx) { + thread_kvp.key = 0; + thread_kvp.value = -1.f; // This is OK because inputs are probabilities + + cub_kvp inp_kvp; + for (int expert = threadIdx.x; expert < num_experts; expert += TPB) { + const int idx = thread_read_offset + expert; + inp_kvp.key = expert; + inp_kvp.value = inputs_after_softmax[idx]; + + for (int prior_k = 0; prior_k < k_idx; ++prior_k) { + const int prior_winning_expert = indices[k * block_row + prior_k]; + + if (prior_winning_expert == expert) { + inp_kvp = thread_kvp; + } + } + + thread_kvp = arg_max(inp_kvp, thread_kvp); + } + + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max); + if (threadIdx.x == 0) { + // Ignore experts the node isn't responsible for with expert parallelism + const int expert = result_kvp.key; + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + const int idx = k * block_row + k_idx; + output[idx] = result_kvp.value; + indices[idx] = should_process_row ? (expert - start_expert) : num_experts; + assert(indices[idx] >= 0); + row_sum_for_renormalize += result_kvp.value; + } + __syncthreads(); + } + + if (renormalize && threadIdx.x == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * block_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +// ====================== TopK softmax things =============================== + +/* + A Top-K gating softmax written to exploit when the number of experts in the MoE layers + are a small power of 2. This allows us to cleanly share the rows among the threads in + a single warp and eliminate communication between warps (so no need to use shared mem). + + It fuses the softmax, max and argmax into a single kernel. + + Limitations: + 1) This implementation is intended for when the number of experts is a small power of 2. + 2) This implementation assumes k is small, but will work for any k. +*/ + +template +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSoftmax( + const T* input, + const bool* finished, + float* output, + const int num_rows, + int* indices, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize) { + // We begin by enforcing compile time assertions and setting up compile time constants. + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2"); + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2"); + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2"); + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16"); + + // Number of bytes each thread pulls in per load + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static constexpr int ELTS_PER_ROW = NUM_EXPERTS; + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT; + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG; + + // Restrictions based on previous section. + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg"); + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp"); + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2"); + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size"); + + // We have NUM_EXPERTS elements per row. We specialize for small #experts + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT; + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW; + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; + + // Restrictions for previous section. + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp"); + + // ===================== From this point, we finally start computing run-time variables. ======================== + + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps. + // This, each block processes a chunk of rows. We start by computing the start row for each block. + const int cta_base_row = blockIdx.x * ROWS_PER_CTA; + + // Now, using the base row per thread block, we compute the base row per warp. + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP; + + // The threads in a warp are split into sub-groups that will work on a row. + // We compute row offset for each thread sub-group + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW; + const int thread_row = warp_base_row + thread_row_in_warp; + + // Threads with indices out of bounds should early exit here. + if (thread_row >= num_rows) { + return; + } + const bool row_is_active = finished ? !finished[thread_row] : true; + + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the + // row it will read. + const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW; + + // Now, we compute the group each thread belong to in order to determine the first column to start loads. + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW; + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG; + const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory, + // this can support all powers of 2 up to 16. + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here. + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS. + using AccessType = AlignedArray; + + // Finally, we pull in the data from global mem + T row_chunk_temp[VPT]; + AccessType* row_chunk_vec_ptr = reinterpret_cast(&row_chunk_temp); + const AccessType* vec_thread_read_ptr = reinterpret_cast(thread_read_ptr); +#pragma unroll + for (int ii = 0; ii < LDG_PER_THREAD; ++ii) { + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW]; + } + + float row_chunk[VPT]; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = convert_to_float(row_chunk_temp[ii]); + } + + // First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just + // convert to float afterwards for the exp + sum reduction. + float thread_max = row_chunk[0]; +#pragma unroll + for (int ii = 1; ii < VPT; ++ii) { + thread_max = max(thread_max, row_chunk[ii]); + } + +// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + thread_max = max(thread_max, SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, thread_max, mask, THREADS_PER_ROW)); + } + + // From this point, thread max in all the threads have the max within the row. + // Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum. + float row_sum = 0; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = expf(row_chunk[ii] - thread_max); + row_sum += row_chunk[ii]; + } + +// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + row_sum += SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, row_sum, mask, THREADS_PER_ROW); + } + + // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables + // respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to + // compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row. + // However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the + // argmax after computing the softmax. + const float reciprocal_row_sum = 1.f / row_sum; + +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum; + } + + // Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along + // with the max index. + int start_col = first_elt_read_by_thread; + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW; + + float row_sum_for_renormalize = 0; + + for (int k_idx = 0; k_idx < k; ++k_idx) { + // First, each thread does the local argmax + float max_val = row_chunk[0]; + int expert = start_col; +#pragma unroll + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) { +#pragma unroll + for (int ii = 0; ii < ELTS_PER_LDG; ++ii) { + float val = row_chunk[ldg * ELTS_PER_LDG + ii]; + + // No check on the experts here since columns with the smallest index are processed first and only + // updated if > (not >=) + if (val > max_val) { + max_val = val; + expert = col + ii; + } + } + } + +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max. +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can +// then blank out their max with -inf and the warp can run more iterations... +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW); + int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW); + + // We want lower indices to "win" in every thread so we break ties this way + if (other_max > max_val || (other_max == max_val && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + // Write the max for this k iteration to global memory. + if (thread_group_idx == 0) { + // Add a guard to ignore experts not included by this node + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + // The lead thread from each sub-group will write out the final results to global memory. (This will be a + // single) thread per row of the input/output matrices. + const int idx = k * thread_row + k_idx; + output[idx] = max_val; + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS; + row_sum_for_renormalize += max_val; + } + + // Finally, we clear the value in the thread with the current max if there is another iteration to run. + if (k_idx + 1 < k) { + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG; + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW; + + // Only the thread in the group which produced the max will reset the "winning" value to -inf. + if (thread_group_idx == thread_to_clear_in_group) { + const int offset_for_expert = expert % ELTS_PER_LDG; + // Safe to set to any negative value since row_chunk values must be between 0 and 1. + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f; + } + } + } + + // Fuse renormalization of topk_weights into this kernel + if (renormalize && thread_group_idx == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; +#pragma unroll + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * thread_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +namespace detail { +// Constructs some constants needed to partition the work across threads at compile time. +template +struct TopkConstants { + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, ""); + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE)); + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG; + static constexpr int THREADS_PER_ROW = EXPERTS / VPT; + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW; +}; +} // namespace detail + +template +void topkGatingSoftmaxLauncherHelper( + const T* input, + const bool* finished, + float* output, + int* indices, + const int num_rows, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + cudaStream_t stream) { + static constexpr std::size_t MAX_BYTES_PER_LDG = 16; + + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS); + using Constants = detail::TopkConstants; + static constexpr int VPT = Constants::VPT; + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP; + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP; + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB; + + dim3 block_dim(WARP_SIZE, WARPS_PER_TB); + topkGatingSoftmax<<>>( + input, finished, output, num_rows, indices, k, start_expert, end_expert, renormalize); +} + +#define LAUNCH_SOFTMAX(TYPE, NUM_EXPERTS, WARPS_PER_TB) \ + topkGatingSoftmaxLauncherHelper( \ + gating_output, nullptr, topk_weights, topk_indices, num_tokens, topk, 0, num_experts, renormalize, stream); + +template +void topkGatingSoftmaxKernelLauncher( + const T* gating_output, + float* topk_weights, + int* topk_indices, + float* softmax_workspace, + const int num_tokens, + const int num_experts, + const int topk, + const bool renormalize, + cudaStream_t stream) { + static constexpr int WARPS_PER_TB = 4; + switch (num_experts) { + case 1: + LAUNCH_SOFTMAX(T, 1, WARPS_PER_TB); + break; + case 2: + LAUNCH_SOFTMAX(T, 2, WARPS_PER_TB); + break; + case 4: + LAUNCH_SOFTMAX(T, 4, WARPS_PER_TB); + break; + case 8: + LAUNCH_SOFTMAX(T, 8, WARPS_PER_TB); + break; + case 16: + LAUNCH_SOFTMAX(T, 16, WARPS_PER_TB); + break; + case 32: + LAUNCH_SOFTMAX(T, 32, WARPS_PER_TB); + break; + case 64: + LAUNCH_SOFTMAX(T, 64, WARPS_PER_TB); + break; + case 128: + LAUNCH_SOFTMAX(T, 128, WARPS_PER_TB); + break; + case 256: + LAUNCH_SOFTMAX(T, 256, WARPS_PER_TB); + break; + default: { + TORCH_CHECK( + softmax_workspace != nullptr, + "softmax_workspace must be provided for num_experts that are not a power of 2."); + static constexpr int TPB = 256; + moeSoftmax<<>>(gating_output, nullptr, softmax_workspace, num_experts); + moeTopK<<>>( + softmax_workspace, nullptr, topk_weights, topk_indices, num_experts, topk, 0, num_experts, renormalize); + } + } +} + +void topk_softmax( + torch::Tensor& topk_weights, // [num_tokens, topk] + torch::Tensor& topk_indices, // [num_tokens, topk] + torch::Tensor& gating_output, + const bool renormalize) // [num_tokens, num_experts] +{ + // Check data type + TORCH_CHECK( + gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half || + gating_output.scalar_type() == at::ScalarType::BFloat16, + "gating_output must be float32, float16, or bfloat16"); + + // Check dimensions + TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]"); + TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]"); + TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]"); + + // Check shapes + TORCH_CHECK( + gating_output.size(0) == topk_weights.size(0), + "First dimension of topk_weights must match num_tokens in gating_output"); + TORCH_CHECK( + gating_output.size(0) == topk_indices.size(0), + "First dimension of topk_indices must match num_tokens in gating_output"); + TORCH_CHECK( + topk_weights.size(-1) == topk_indices.size(-1), + "Second dimension of topk_indices must match topk in topk_weights"); + TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts"); + + const int num_experts = static_cast(gating_output.size(-1)); + const int num_tokens = static_cast(gating_output.size(0)); + const int topk = static_cast(topk_weights.size(-1)); + + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); + const bool needs_workspace = !is_pow_2 || num_experts > 256; + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; + + const at::cuda::OptionalCUDAGuard device_guard(device_of(gating_output)); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + torch::Tensor softmax_workspace = + torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float)); + + const at::ScalarType dtype = gating_output.scalar_type(); + if (dtype == at::ScalarType::Float) { + topkGatingSoftmaxKernelLauncher( + gating_output.data_ptr(), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else if (dtype == at::ScalarType::Half) { + topkGatingSoftmaxKernelLauncher<__half>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else if (dtype == at::ScalarType::BFloat16) { + topkGatingSoftmaxKernelLauncher<__nv_bfloat16>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else { + TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype); + } +} diff --git a/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.hip b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.hip new file mode 100644 index 000000000..96329f22c --- /dev/null +++ b/sgl-kernel/csrc/moe/moe_topk_softmax_kernels.hip @@ -0,0 +1,594 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +// Adapt from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/moe/topk_softmax_kernels.cu +// which is originally adapted from +// https://github.com/NVIDIA/TensorRT-LLM/blob/v0.7.1/cpp/tensorrt_llm/kernels/mixtureOfExperts/moe_kernels.cu +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#ifndef USE_ROCM +#include +#include +#include +#else +#include +#include +#endif + +#include "utils_hip.h" + +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +// Define reduction operators based on CUDA version +// CUDA 13 (12.9+) deprecated hipcub::Max/Min in favor of cuda::maximum/minimum +#if DTK_VERSION >= 12090 +using MaxReduceOp = cuda::maximum<>; +using MinReduceOp = cuda::minimum<>; +#else +using MaxReduceOp = hipcub::Max; +using MinReduceOp = hipcub::Min; +#endif + +/// Aligned array type +template < + typename T, + /// Number of elements in the array + int N, + /// Alignment requirement in bytes + int Alignment = sizeof(T) * N> +class alignas(Alignment) AlignedArray { + T data[N]; +}; + +// ========================== Util functions to convert types ========================== +template +__device__ float convert_to_float(T x) { + if constexpr (std::is_same_v) { + return __half2float(x); + } else if constexpr (std::is_same_v) { + return __bfloat162float(x); + } else if constexpr (std::is_same_v) { + return x; + } else { + return static_cast(x); + } +} + +// ====================== Softmax things =============================== +// We have our own implementation of softmax here so we can support transposing the output +// in the softmax kernel when we extend this module to support expert-choice routing. +template +__launch_bounds__(TPB) __global__ + void moeSoftmax(const T* input, const bool* finished, float* output, const int num_cols) { + using BlockReduce = hipcub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + __shared__ float normalizing_factor; + __shared__ float float_max; + + const int thread_row_offset = blockIdx.x * num_cols; + + float threadData(-FLT_MAX); + + // Don't touch finished rows. + if ((finished != nullptr) && finished[blockIdx.x]) { + return; + } + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData = max(convert_to_float(input[idx]), threadData); + } + + const float maxElem = BlockReduce(tmpStorage).Reduce(threadData, MaxReduceOp()); + + if (threadIdx.x == 0) { + float_max = maxElem; + } + __syncthreads(); + + threadData = 0; + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + threadData += exp((convert_to_float(input[idx]) - float_max)); + } + + const auto Z = BlockReduce(tmpStorage).Sum(threadData); + + if (threadIdx.x == 0) { + normalizing_factor = 1.f / Z; + } + __syncthreads(); + + for (int ii = threadIdx.x; ii < num_cols; ii += TPB) { + const int idx = thread_row_offset + ii; + const float val = exp((convert_to_float(input[idx]) - float_max)) * normalizing_factor; + output[idx] = val; + } +} + +template +__launch_bounds__(TPB) __global__ void moeTopK( + const float* inputs_after_softmax, + const bool* finished, + float* output, + int* indices, + const int num_experts, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize) { + using cub_kvp = hipcub::KeyValuePair; + using BlockReduce = hipcub::BlockReduce; + __shared__ typename BlockReduce::TempStorage tmpStorage; + + cub_kvp thread_kvp; + hipcub::ArgMax arg_max; + + const int block_row = blockIdx.x; + + const bool row_is_active = finished ? !finished[block_row] : true; + const int thread_read_offset = blockIdx.x * num_experts; + float row_sum_for_renormalize = 0; + for (int k_idx = 0; k_idx < k; ++k_idx) { + thread_kvp.key = 0; + thread_kvp.value = -1.f; // This is OK because inputs are probabilities + + cub_kvp inp_kvp; + for (int expert = threadIdx.x; expert < num_experts; expert += TPB) { + const int idx = thread_read_offset + expert; + inp_kvp.key = expert; + inp_kvp.value = inputs_after_softmax[idx]; + + for (int prior_k = 0; prior_k < k_idx; ++prior_k) { + const int prior_winning_expert = indices[k * block_row + prior_k]; + + if (prior_winning_expert == expert) { + inp_kvp = thread_kvp; + } + } + + thread_kvp = arg_max(inp_kvp, thread_kvp); + } + + const cub_kvp result_kvp = BlockReduce(tmpStorage).Reduce(thread_kvp, arg_max); + if (threadIdx.x == 0) { + // Ignore experts the node isn't responsible for with expert parallelism + const int expert = result_kvp.key; + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + const int idx = k * block_row + k_idx; + output[idx] = result_kvp.value; + indices[idx] = should_process_row ? (expert - start_expert) : num_experts; + assert(indices[idx] >= 0); + row_sum_for_renormalize += result_kvp.value; + } + __syncthreads(); + } + + if (renormalize && threadIdx.x == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * block_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +// ====================== TopK softmax things =============================== + +/* + A Top-K gating softmax written to exploit when the number of experts in the MoE layers + are a small power of 2. This allows us to cleanly share the rows among the threads in + a single warp and eliminate communication between warps (so no need to use shared mem). + + It fuses the softmax, max and argmax into a single kernel. + + Limitations: + 1) This implementation is intended for when the number of experts is a small power of 2. + 2) This implementation assumes k is small, but will work for any k. +*/ + +template +__launch_bounds__(WARPS_PER_CTA* WARP_SIZE) __global__ void topkGatingSoftmax( + const T* input, + const bool* finished, + float* output, + const int num_rows, + int* indices, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize) { + // We begin by enforcing compile time assertions and setting up compile time constants. + static_assert(VPT == (VPT & -VPT), "VPT must be power of 2"); + static_assert(NUM_EXPERTS == (NUM_EXPERTS & -NUM_EXPERTS), "NUM_EXPERTS must be power of 2"); + static_assert(BYTES_PER_LDG == (BYTES_PER_LDG & -BYTES_PER_LDG), "BYTES_PER_LDG must be power of 2"); + static_assert(BYTES_PER_LDG <= 16, "BYTES_PER_LDG must be leq 16"); + + // Number of bytes each thread pulls in per load + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static constexpr int ELTS_PER_ROW = NUM_EXPERTS; + static constexpr int THREADS_PER_ROW = ELTS_PER_ROW / VPT; + static constexpr int LDG_PER_THREAD = VPT / ELTS_PER_LDG; + + // Restrictions based on previous section. + static_assert(VPT % ELTS_PER_LDG == 0, "The elements per thread must be a multiple of the elements per ldg"); + static_assert(WARP_SIZE % THREADS_PER_ROW == 0, "The threads per row must cleanly divide the threads per warp"); + static_assert(THREADS_PER_ROW == (THREADS_PER_ROW & -THREADS_PER_ROW), "THREADS_PER_ROW must be power of 2"); + static_assert(THREADS_PER_ROW <= WARP_SIZE, "THREADS_PER_ROW can be at most warp size"); + + // We have NUM_EXPERTS elements per row. We specialize for small #experts + static constexpr int ELTS_PER_WARP = WARP_SIZE * VPT; + static constexpr int ROWS_PER_WARP = ELTS_PER_WARP / ELTS_PER_ROW; + static constexpr int ROWS_PER_CTA = WARPS_PER_CTA * ROWS_PER_WARP; + + // Restrictions for previous section. + static_assert(ELTS_PER_WARP % ELTS_PER_ROW == 0, "The elts per row must cleanly divide the total elt per warp"); + + // ===================== From this point, we finally start computing run-time variables. ======================== + + // Compute CTA and warp rows. We pack multiple rows into a single warp, and a block contains WARPS_PER_CTA warps. + // This, each block processes a chunk of rows. We start by computing the start row for each block. + const int cta_base_row = blockIdx.x * ROWS_PER_CTA; + + // Now, using the base row per thread block, we compute the base row per warp. + const int warp_base_row = cta_base_row + threadIdx.y * ROWS_PER_WARP; + + // The threads in a warp are split into sub-groups that will work on a row. + // We compute row offset for each thread sub-group + const int thread_row_in_warp = threadIdx.x / THREADS_PER_ROW; + const int thread_row = warp_base_row + thread_row_in_warp; + + // Threads with indices out of bounds should early exit here. + if (thread_row >= num_rows) { + return; + } + const bool row_is_active = finished ? !finished[thread_row] : true; + + // We finally start setting up the read pointers for each thread. First, each thread jumps to the start of the + // row it will read. + const T* thread_row_ptr = input + thread_row * ELTS_PER_ROW; + + // Now, we compute the group each thread belong to in order to determine the first column to start loads. + const int thread_group_idx = threadIdx.x % THREADS_PER_ROW; + const int first_elt_read_by_thread = thread_group_idx * ELTS_PER_LDG; + const T* thread_read_ptr = thread_row_ptr + first_elt_read_by_thread; + + // Determine the pointer type to use to read in the data depending on the BYTES_PER_LDG template param. In theory, + // this can support all powers of 2 up to 16. + // NOTE(woosuk): The original implementation uses CUTLASS aligned array here. + // We defined our own aligned array and use it here to avoid the dependency on CUTLASS. + using AccessType = AlignedArray; + + // Finally, we pull in the data from global mem + T row_chunk_temp[VPT]; + AccessType* row_chunk_vec_ptr = reinterpret_cast(&row_chunk_temp); + const AccessType* vec_thread_read_ptr = reinterpret_cast(thread_read_ptr); +#pragma unroll + for (int ii = 0; ii < LDG_PER_THREAD; ++ii) { + row_chunk_vec_ptr[ii] = vec_thread_read_ptr[ii * THREADS_PER_ROW]; + } + + float row_chunk[VPT]; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = convert_to_float(row_chunk_temp[ii]); + } + + // First, we perform a max reduce within the thread. We can do the max in fp16 safely (I think) and just + // convert to float afterwards for the exp + sum reduction. + float thread_max = row_chunk[0]; +#pragma unroll + for (int ii = 1; ii < VPT; ++ii) { + thread_max = max(thread_max, row_chunk[ii]); + } + +// Now, we find the max within the thread group and distribute among the threads. We use a butterfly reduce. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + thread_max = max(thread_max, SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, thread_max, mask, THREADS_PER_ROW)); + } + + // From this point, thread max in all the threads have the max within the row. + // Now, we subtract the max from each element in the thread and take the exp. We also compute the thread local sum. + float row_sum = 0; +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = expf(row_chunk[ii] - thread_max); + row_sum += row_chunk[ii]; + } + +// Now, we perform the sum reduce within each thread group. Similar to the max reduce, we use a bufferfly pattern. +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + row_sum += SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, row_sum, mask, THREADS_PER_ROW); + } + + // From this point, all threads have the max and the sum for their rows in the thread_max and thread_sum variables + // respectively. Finally, we can scale the rows for the softmax. Technically, for top-k gating we don't need to + // compute the entire softmax row. We can likely look at the maxes and only compute for the top-k values in the row. + // However, this kernel will likely not be a bottle neck and it seems better to closer match torch and find the + // argmax after computing the softmax. + const float reciprocal_row_sum = 1.f / row_sum; + +#pragma unroll + for (int ii = 0; ii < VPT; ++ii) { + row_chunk[ii] = row_chunk[ii] * reciprocal_row_sum; + } + + // Now, softmax_res contains the softmax of the row chunk. Now, I want to find the topk elements in each row, along + // with the max index. + int start_col = first_elt_read_by_thread; + static constexpr int COLS_PER_GROUP_LDG = ELTS_PER_LDG * THREADS_PER_ROW; + + float row_sum_for_renormalize = 0; + + for (int k_idx = 0; k_idx < k; ++k_idx) { + // First, each thread does the local argmax + float max_val = row_chunk[0]; + int expert = start_col; +#pragma unroll + for (int ldg = 0, col = start_col; ldg < LDG_PER_THREAD; ++ldg, col += COLS_PER_GROUP_LDG) { +#pragma unroll + for (int ii = 0; ii < ELTS_PER_LDG; ++ii) { + float val = row_chunk[ldg * ELTS_PER_LDG + ii]; + + // No check on the experts here since columns with the smallest index are processed first and only + // updated if > (not >=) + if (val > max_val) { + max_val = val; + expert = col + ii; + } + } + } + +// Now, we perform the argmax reduce. We use the butterfly pattern so threads reach consensus about the max. +// This will be useful for K > 1 so that the threads can agree on "who" had the max value. That thread can +// then blank out their max with -inf and the warp can run more iterations... +#pragma unroll + for (int mask = THREADS_PER_ROW / 2; mask > 0; mask /= 2) { + float other_max = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, max_val, mask, THREADS_PER_ROW); + int other_expert = SGLANG_SHFL_XOR_SYNC_WIDTH(0xffffffff, expert, mask, THREADS_PER_ROW); + + // We want lower indices to "win" in every thread so we break ties this way + if (other_max > max_val || (other_max == max_val && other_expert < expert)) { + max_val = other_max; + expert = other_expert; + } + } + + // Write the max for this k iteration to global memory. + if (thread_group_idx == 0) { + // Add a guard to ignore experts not included by this node + const bool node_uses_expert = expert >= start_expert && expert < end_expert; + const bool should_process_row = row_is_active && node_uses_expert; + + // The lead thread from each sub-group will write out the final results to global memory. (This will be a + // single) thread per row of the input/output matrices. + const int idx = k * thread_row + k_idx; + output[idx] = max_val; + indices[idx] = should_process_row ? (expert - start_expert) : NUM_EXPERTS; + row_sum_for_renormalize += max_val; + } + + // Finally, we clear the value in the thread with the current max if there is another iteration to run. + if (k_idx + 1 < k) { + const int ldg_group_for_expert = expert / COLS_PER_GROUP_LDG; + const int thread_to_clear_in_group = (expert / ELTS_PER_LDG) % THREADS_PER_ROW; + + // Only the thread in the group which produced the max will reset the "winning" value to -inf. + if (thread_group_idx == thread_to_clear_in_group) { + const int offset_for_expert = expert % ELTS_PER_LDG; + // Safe to set to any negative value since row_chunk values must be between 0 and 1. + row_chunk[ldg_group_for_expert * ELTS_PER_LDG + offset_for_expert] = -10000.f; + } + } + } + + // Fuse renormalization of topk_weights into this kernel + if (renormalize && thread_group_idx == 0) { + float row_sum_for_renormalize_inv = 1.f / row_sum_for_renormalize; +#pragma unroll + for (int k_idx = 0; k_idx < k; ++k_idx) { + const int idx = k * thread_row + k_idx; + output[idx] = output[idx] * row_sum_for_renormalize_inv; + } + } +} + +namespace detail { +// Constructs some constants needed to partition the work across threads at compile time. +template +struct TopkConstants { + static constexpr int ELTS_PER_LDG = BYTES_PER_LDG / sizeof(T); + static_assert(EXPERTS / (ELTS_PER_LDG * WARP_SIZE) == 0 || EXPERTS % (ELTS_PER_LDG * WARP_SIZE) == 0, ""); + static constexpr int VECs_PER_THREAD = MAX(1, EXPERTS / (ELTS_PER_LDG * WARP_SIZE)); + static constexpr int VPT = VECs_PER_THREAD * ELTS_PER_LDG; + static constexpr int THREADS_PER_ROW = EXPERTS / VPT; + static constexpr int ROWS_PER_WARP = WARP_SIZE / THREADS_PER_ROW; +}; +} // namespace detail + +template +void topkGatingSoftmaxLauncherHelper( + const T* input, + const bool* finished, + float* output, + int* indices, + const int num_rows, + const int k, + const int start_expert, + const int end_expert, + const bool renormalize, + hipStream_t stream) { + static constexpr std::size_t MAX_BYTES_PER_LDG = 16; + + static constexpr int BYTES_PER_LDG = MIN(MAX_BYTES_PER_LDG, sizeof(T) * EXPERTS); + using Constants = detail::TopkConstants; + static constexpr int VPT = Constants::VPT; + static constexpr int ROWS_PER_WARP = Constants::ROWS_PER_WARP; + const int num_warps = (num_rows + ROWS_PER_WARP - 1) / ROWS_PER_WARP; + const int num_blocks = (num_warps + WARPS_PER_TB - 1) / WARPS_PER_TB; + + dim3 block_dim(WARP_SIZE, WARPS_PER_TB); + hipLaunchKernelGGL(( topkGatingSoftmax), dim3(num_blocks), dim3(block_dim), 0, stream, + input, finished, output, num_rows, indices, k, start_expert, end_expert, renormalize); +} + +#define LAUNCH_SOFTMAX(TYPE, NUM_EXPERTS, WARPS_PER_TB) \ + topkGatingSoftmaxLauncherHelper( \ + gating_output, nullptr, topk_weights, topk_indices, num_tokens, topk, 0, num_experts, renormalize, stream); + +template +void topkGatingSoftmaxKernelLauncher( + const T* gating_output, + float* topk_weights, + int* topk_indices, + float* softmax_workspace, + const int num_tokens, + const int num_experts, + const int topk, + const bool renormalize, + hipStream_t stream) { + static constexpr int WARPS_PER_TB = 4; + switch (num_experts) { + case 1: + LAUNCH_SOFTMAX(T, 1, WARPS_PER_TB); + break; + case 2: + LAUNCH_SOFTMAX(T, 2, WARPS_PER_TB); + break; + case 4: + LAUNCH_SOFTMAX(T, 4, WARPS_PER_TB); + break; + case 8: + LAUNCH_SOFTMAX(T, 8, WARPS_PER_TB); + break; + case 16: + LAUNCH_SOFTMAX(T, 16, WARPS_PER_TB); + break; + case 32: + LAUNCH_SOFTMAX(T, 32, WARPS_PER_TB); + break; + case 64: + LAUNCH_SOFTMAX(T, 64, WARPS_PER_TB); + break; + case 128: + LAUNCH_SOFTMAX(T, 128, WARPS_PER_TB); + break; + case 256: + LAUNCH_SOFTMAX(T, 256, WARPS_PER_TB); + break; + default: { + TORCH_CHECK( + softmax_workspace != nullptr, + "softmax_workspace must be provided for num_experts that are not a power of 2."); + static constexpr int TPB = 256; + hipLaunchKernelGGL(( moeSoftmax), dim3(num_tokens), dim3(TPB), 0, stream, gating_output, nullptr, softmax_workspace, num_experts); + hipLaunchKernelGGL(( moeTopK), dim3(num_tokens), dim3(TPB), 0, stream, + softmax_workspace, nullptr, topk_weights, topk_indices, num_experts, topk, 0, num_experts, renormalize); + } + } +} + +void topk_softmax( + torch::Tensor& topk_weights, // [num_tokens, topk] + torch::Tensor& topk_indices, // [num_tokens, topk] + torch::Tensor& gating_output, + const bool renormalize) // [num_tokens, num_experts] +{ + // Check data type + TORCH_CHECK( + gating_output.scalar_type() == at::ScalarType::Float || gating_output.scalar_type() == at::ScalarType::Half || + gating_output.scalar_type() == at::ScalarType::BFloat16, + "gating_output must be float32, float16, or bfloat16"); + + // Check dimensions + TORCH_CHECK(gating_output.dim() == 2, "gating_output must be 2D tensor [num_tokens, num_experts]"); + TORCH_CHECK(topk_weights.dim() == 2, "topk_weights must be 2D tensor [num_tokens, topk]"); + TORCH_CHECK(topk_indices.dim() == 2, "topk_indices must be 2D tensor [num_tokens, topk]"); + + // Check shapes + TORCH_CHECK( + gating_output.size(0) == topk_weights.size(0), + "First dimension of topk_weights must match num_tokens in gating_output"); + TORCH_CHECK( + gating_output.size(0) == topk_indices.size(0), + "First dimension of topk_indices must match num_tokens in gating_output"); + TORCH_CHECK( + topk_weights.size(-1) == topk_indices.size(-1), + "Second dimension of topk_indices must match topk in topk_weights"); + TORCH_CHECK(topk_weights.size(-1) <= gating_output.size(-1), "topk must be less than or equal to num_experts"); + + const int num_experts = static_cast(gating_output.size(-1)); + const int num_tokens = static_cast(gating_output.size(0)); + const int topk = static_cast(topk_weights.size(-1)); + + const bool is_pow_2 = (num_experts != 0) && ((num_experts & (num_experts - 1)) == 0); + const bool needs_workspace = !is_pow_2 || num_experts > 256; + const int64_t workspace_size = needs_workspace ? num_tokens * num_experts : 0; + + const at::hip::OptionalHIPGuardMasqueradingAsCUDA device_guard(device_of(gating_output)); + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + torch::Tensor softmax_workspace = + torch::empty({workspace_size}, gating_output.options().dtype(at::ScalarType::Float)); + + const at::ScalarType dtype = gating_output.scalar_type(); + if (dtype == at::ScalarType::Float) { + topkGatingSoftmaxKernelLauncher( + gating_output.data_ptr(), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else if (dtype == at::ScalarType::Half) { + topkGatingSoftmaxKernelLauncher<__half>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else if (dtype == at::ScalarType::BFloat16) { + topkGatingSoftmaxKernelLauncher<__hip_bfloat16>( + reinterpret_cast(gating_output.data_ptr()), + topk_weights.data_ptr(), + topk_indices.data_ptr(), + softmax_workspace.data_ptr(), + num_tokens, + num_experts, + topk, + renormalize, + stream); + } else { + TORCH_CHECK(false, "Unsupported gating_output dtype: ", dtype); + } +} diff --git a/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu b/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu new file mode 100644 index 000000000..c68ca552d --- /dev/null +++ b/sgl-kernel/csrc/moe/nvfp4_blockwise_moe.cu @@ -0,0 +1,471 @@ +#include +#include +#include +#include +#include + +#include + +#include "cute/tensor.hpp" +#include "cutlass/epilogue/collective/collective_builder.hpp" +#include "cutlass/epilogue/collective/default_epilogue.hpp" +#include "cutlass/epilogue/thread/linear_combination.h" +#include "cutlass/gemm/collective/collective_builder.hpp" +#include "cutlass/gemm/device/gemm_universal_adapter.h" +#include "cutlass/gemm/dispatch_policy.hpp" +#include "cutlass/gemm/group_array_problem_shape.hpp" +#include "cutlass/gemm/kernel/gemm_universal.hpp" +#include "cutlass/tensor_ref.h" +#include "cutlass/util/command_line.h" +#include "cutlass/util/distribution.h" +#include "cutlass/util/host_tensor.h" +#include "cutlass/util/packed_stride.hpp" +#include "cutlass/util/reference/device/gemm.h" +#include "cutlass/util/reference/device/tensor_compare.h" +#include "cutlass/util/reference/host/gett.hpp" +#include "cutlass/util/reference/host/tensor_compare.h" +#include "cutlass/util/reference/host/tensor_fill.h" +#include "cutlass/util/reference/host/tensor_norm.h" +#include "cutlass/util/tensor_view_io.h" + +using namespace cute; + +template < + typename ElementAB, + typename ElementC, + typename ElementSF, + typename ElementAccumulator, + typename LayoutSFA, + typename LayoutSFB, + typename ScaleConfig> +__global__ void __get_group_gemm_starts( + ElementAB** a_offsets, + ElementAB** b_offsets, + ElementC** out_offsets, + ElementSF** a_scales_offsets, + ElementSF** b_scales_offsets, + ElementAccumulator** alpha_offsets, + LayoutSFA* layout_sfa_base_as_int, + LayoutSFB* layout_sfb_base_as_int, + ElementAB* a_base_as_int, + ElementAB* b_base_as_int, + ElementC* out_base_as_int, + ElementSF* a_scales_base_as_int, + ElementSF* b_scales_base_as_int, + ElementAccumulator* alphas_base_as_int, + const int32_t* expert_offsets, + const int32_t* sf_offsets, + const int32_t* problem_sizes_as_shapes, + const int K, + const int N) { + int64_t expert_id = threadIdx.x; + if (expert_id >= gridDim.x * blockDim.x) { + return; + } + // Originally int32_t but upcasting to int64_t to avoid overflow + // during offset calculations + int64_t expert_offset = static_cast(expert_offsets[expert_id]); + int64_t sf_offset = static_cast(sf_offsets[expert_id]); + // size for block in block scale. + int64_t group_size = 16; + int64_t m = static_cast(problem_sizes_as_shapes[expert_id * 3]); + int64_t n = static_cast(problem_sizes_as_shapes[expert_id * 3 + 1]); + int64_t k = static_cast(problem_sizes_as_shapes[expert_id * 3 + 2]); + assert((m >= 0 && n == N && k == K && k % 2 == 0) && "unexpected problem sizes"); + + int64_t half_k = static_cast(k / 2); + int64_t group_k = static_cast(k / group_size); + // Shape of A as uint8/byte = [M, K // 2] + // Shape of B as uint8/byte = [E, N, K // 2] + a_offsets[expert_id] = a_base_as_int + expert_offset * half_k; + + b_offsets[expert_id] = b_base_as_int + expert_id * n * half_k; + // Shape of C = [M, N] + out_offsets[expert_id] = out_base_as_int + expert_offset * n; + // Shape of a_scale = [sum(sf_sizes), K // group_size] + a_scales_offsets[expert_id] = a_scales_base_as_int + sf_offset * group_k; + + assert((reinterpret_cast(a_scales_offsets[expert_id]) % 128) == 0 && "TMA requires 128-byte alignment"); + + // Shape of B scale = [E, N, K // group_size] + b_scales_offsets[expert_id] = b_scales_base_as_int + expert_id * n * group_k; + assert((reinterpret_cast(b_scales_offsets[expert_id]) % 128) == 0 && "TMA requires 128-byte alignment"); + // Shape of alpha = [E] + alpha_offsets[expert_id] = alphas_base_as_int + expert_id; + + LayoutSFA* layout_sfa_ptr = layout_sfa_base_as_int + expert_id; + LayoutSFB* layout_sfb_ptr = layout_sfb_base_as_int + expert_id; + + *layout_sfa_ptr = ScaleConfig::tile_atom_to_shape_SFA( + cute::make_shape(static_cast(m), static_cast(n), static_cast(k), 1)); + *layout_sfb_ptr = ScaleConfig::tile_atom_to_shape_SFB( + cute::make_shape(static_cast(m), static_cast(n), static_cast(k), 1)); +} + +#define __CALL_GET_STARTS_KERNEL_BLOCKSCALE( \ + ELEMENT_AB_TYPE, SF_TYPE, TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, ScaleConfig) \ + else if (out_tensors.dtype() == TENSOR_C_TYPE) { \ + __get_group_gemm_starts \ + <<<1, num_experts, 0, stream>>>( \ + static_cast(a_starts.data_ptr()), \ + static_cast(b_starts.data_ptr()), \ + static_cast(out_starts.data_ptr()), \ + static_cast(a_scales_starts.data_ptr()), \ + static_cast(b_scales_starts.data_ptr()), \ + static_cast(alpha_starts.data_ptr()), \ + reinterpret_cast(layout_sfa.data_ptr()), \ + reinterpret_cast(layout_sfb.data_ptr()), \ + static_cast(a_tensors.data_ptr()), \ + static_cast(b_tensors.data_ptr()), \ + static_cast(out_tensors.data_ptr()), \ + static_cast(a_scales.data_ptr()), \ + static_cast(b_scales.data_ptr()), \ + static_cast(alphas.data_ptr()), \ + static_cast(expert_offsets.data_ptr()), \ + static_cast(sf_offsets.data_ptr()), \ + static_cast(problem_sizes.data_ptr()), \ + K, \ + N); \ + } + +template +void run_get_group_gemm_starts( + const torch::Tensor& a_starts, + const torch::Tensor& b_starts, + const torch::Tensor& out_starts, + const torch::Tensor& a_scales_starts, + const torch::Tensor& b_scales_starts, + const torch::Tensor& alpha_starts, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + /*these are used for their base addresses*/ + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& out_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& alphas, + torch::Tensor const& expert_offsets, + torch::Tensor const& sf_offsets, + torch::Tensor const& problem_sizes, + int M, + int N, + int K) { + int num_experts = (int)expert_offsets.size(0); + auto stream = at::cuda::getCurrentCUDAStream(a_tensors.device().index()); + + TORCH_CHECK(out_tensors.size(1) == N, "Output tensor shape doesn't match expected shape"); + TORCH_CHECK( + K / 2 == b_tensors.size(2), + "b_tensors(dim = 2) and a_tensors(dim = 1) trailing" + " dimension must match"); + if (false) { + } + //(ELEMENT_AB_TYPE, BS_TYPE, TENSOR_C_TYPE, C_TYPE, LayoutSFA, LayoutSFB, + // ScaleConfig) + __CALL_GET_STARTS_KERNEL_BLOCKSCALE( + cutlass::float_e2m1_t, + cutlass::float_ue4m3_t, + torch::kBFloat16, + cutlass::bfloat16_t, + LayoutSFA, + LayoutSFB, + ScaleConfig) + __CALL_GET_STARTS_KERNEL_BLOCKSCALE( + cutlass::float_e2m1_t, cutlass::float_ue4m3_t, torch::kFloat16, half, LayoutSFA, LayoutSFB, ScaleConfig) + else { + TORCH_CHECK(false, "Invalid output type (must be float16 or bfloat16)"); + } +} + +template +void run_fp4_blockwise_scaled_group_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets, + int M, + int N, + int K) { + using ProblemShape = cutlass::gemm::GroupProblemShape>; + using ElementType = cutlass::float_e2m1_t; + using ElementSFType = cutlass::float_ue4m3_t; + using ElementA = cutlass::nv_float4_t; + using ElementB = cutlass::nv_float4_t; + + using ElementC = OutType; + using ElementD = ElementC; + using ElementAccumulator = float; + // Layout definitions + using LayoutA = cutlass::layout::RowMajor; + using LayoutB = cutlass::layout::ColumnMajor; + using LayoutC = cutlass::layout::RowMajor; + using LayoutD = LayoutC; + + // Alignment constraints + static constexpr int AlignmentA = 32; + static constexpr int AlignmentB = 32; + static constexpr int AlignmentC = 128 / cutlass::sizeof_bits::value; + static constexpr int AlignmentD = 128 / cutlass::sizeof_bits::value; + + // Architecture definitions + using ArchTag = cutlass::arch::Sm100; + using EpilogueOperatorClass = cutlass::arch::OpClassTensorOp; // Epilogue Operator class tag + using MainloopOperatorClass = cutlass::arch::OpClassBlockScaledTensorOp; // Mainloop Operator class tag + using StageCountType = cutlass::gemm::collective::StageCountAuto; // Stage count maximized based + // on the tile size + + using ClusterShape = Shape<_1, _1, _1>; + struct MMA1SMConfig { + using MmaTileShape = Shape<_128, _128, _128>; + using KernelSchedule = cutlass::gemm::KernelPtrArrayTmaWarpSpecialized1SmNvf4Sm100; // Kernel to launch + using EpilogueSchedule = cutlass::epilogue::PtrArrayTmaWarpSpecialized1Sm; // Epilogue to launch + }; + + using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder< + ArchTag, + EpilogueOperatorClass, + typename MMA1SMConfig::MmaTileShape, + ClusterShape, + Shape<_128, _64>, + ElementAccumulator, + ElementAccumulator, + ElementC, + LayoutC*, + AlignmentC, + ElementD, + LayoutC*, + AlignmentD, + typename MMA1SMConfig::EpilogueSchedule>::CollectiveOp; + + using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder< + ArchTag, + MainloopOperatorClass, + ElementA, + LayoutA*, + AlignmentA, + ElementB, + LayoutB*, + AlignmentB, + ElementAccumulator, + typename MMA1SMConfig::MmaTileShape, + ClusterShape, + cutlass::gemm::collective::StageCountAutoCarveout( + sizeof(typename CollectiveEpilogue::SharedStorage))>, + typename MMA1SMConfig::KernelSchedule>::CollectiveOp; + + using GemmKernel = cutlass::gemm::kernel::GemmUniversal; + + using Gemm1SM = cutlass::gemm::device::GemmUniversalAdapter; + using Gemm = Gemm1SM; + using StrideA = typename Gemm::GemmKernel::InternalStrideA; + using StrideB = typename Gemm::GemmKernel::InternalStrideB; + using StrideC = typename Gemm::GemmKernel::InternalStrideC; + using StrideD = typename Gemm::GemmKernel::InternalStrideD; + + using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFA; + using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::InternalLayoutSFB; + using ScaleConfig = typename Gemm::GemmKernel::CollectiveMainloop::Sm1xxBlkScaledConfig; + + using UnderlyingProblemShape = ProblemShape::UnderlyingProblemShape; + int num_experts = static_cast(expert_offsets.size(0)); + auto options_int = torch::TensorOptions().dtype(torch::kInt64).device(a.device()); + + torch::Tensor a_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_ptrs = torch::empty(num_experts, options_int); + torch::Tensor out_ptrs = torch::empty(num_experts, options_int); + torch::Tensor a_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor b_scales_ptrs = torch::empty(num_experts, options_int); + torch::Tensor alpha_ptrs = torch::empty(num_experts, options_int); + torch::Tensor layout_sfa = torch::empty({num_experts, 5}, options_int); + torch::Tensor layout_sfb = torch::empty({num_experts, 5}, options_int); + + run_get_group_gemm_starts( + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + alpha_ptrs, + layout_sfa, + layout_sfb, + a, + b, + output, + a_blockscale, + b_blockscales, + alphas, + expert_offsets, + sf_offsets, + problem_sizes, + M, + N, + K); + + // Create an instance of the GEMM + Gemm gemm_op; + + // Initialize problem_sizes_as_shapes correctly + UnderlyingProblemShape* problem_sizes_as_shapes = static_cast(problem_sizes.data_ptr()); + + // Set the Scheduler info + cutlass::KernelHardwareInfo hw_info; + using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm100GroupParams< + typename ProblemShape::UnderlyingProblemShape>::RasterOrderOptions; + typename Gemm::GemmKernel::TileSchedulerArguments scheduler; + scheduler.raster_order = RasterOrderOptions::AlongM; + hw_info.device_id = a.get_device(); + static std::unordered_map cached_sm_counts; + if (cached_sm_counts.find(hw_info.device_id) == cached_sm_counts.end()) { + cached_sm_counts[hw_info.device_id] = + cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id); + } + hw_info.sm_count = min(cached_sm_counts[hw_info.device_id], INT_MAX); + + // Mainloop Arguments + typename GemmKernel::MainloopArguments mainloop_args{ + static_cast(a_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(b_ptrs.data_ptr()), + static_cast(ab_strides.data_ptr()), + static_cast(a_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfa.data_ptr()), + static_cast(b_scales_ptrs.data_ptr()), + reinterpret_cast(layout_sfb.data_ptr())}; + + // Epilogue Arguments + typename GemmKernel::EpilogueArguments epilogue_args{ + {}, // epilogue.thread + nullptr, + static_cast(c_strides.data_ptr()), + static_cast(out_ptrs.data_ptr()), + static_cast(c_strides.data_ptr())}; + auto& fusion_args = epilogue_args.thread; + fusion_args.alpha_ptr_array = reinterpret_cast(alpha_ptrs.data_ptr()); + fusion_args.dAlpha = {_0{}, _0{}, 1}; + + // Gemm Arguments + typename GemmKernel::Arguments args{ + cutlass::gemm::GemmUniversalMode::kGrouped, + {num_experts, problem_sizes_as_shapes, nullptr}, + mainloop_args, + epilogue_args, + hw_info, + scheduler}; + + size_t workspace_size = Gemm::get_workspace_size(args); + auto const workspace_options = torch::TensorOptions().dtype(torch::kUInt8).device(a.device()); + auto workspace = torch::empty(workspace_size, workspace_options); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(a.get_device()); + + auto can_implement_status = gemm_op.can_implement(args); + TORCH_CHECK(can_implement_status == cutlass::Status::kSuccess, "Failed to implement GEMM"); + + // Run the GEMM + auto status = gemm_op.initialize(args, workspace.data_ptr()); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to initialize GEMM"); + + status = gemm_op.run(args, workspace.data_ptr(), stream); + TORCH_CHECK(status == cutlass::Status::kSuccess, "Failed to run GEMM"); +} + +#define CHECK_TYPE(x, st, m) TORCH_CHECK(x.scalar_type() == st, ": Inconsistency of Tensor type:", m) +#define CHECK_TH_CUDA(x, m) TORCH_CHECK(x.is_cuda(), m, ": must be a CUDA tensor.") +#define CHECK_CONTIGUOUS(x, m) TORCH_CHECK(x.is_contiguous(), m, ": must be contiguous.") +#define CHECK_INPUT(x, st, m) \ + CHECK_TH_CUDA(x, m); \ + CHECK_CONTIGUOUS(x, m); \ + CHECK_TYPE(x, st, m) + +void cutlass_fp4_group_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets) { +#if defined ENABLE_NVFP4 && ENABLE_NVFP4 + + constexpr auto FLOAT4_E2M1X2 = at::ScalarType::Byte; + constexpr auto SF_DTYPE = at::ScalarType::Float8_e4m3fn; + // Input validation + CHECK_INPUT(a, FLOAT4_E2M1X2, "a"); + CHECK_INPUT(b, FLOAT4_E2M1X2, "b"); + CHECK_INPUT(a_blockscale, SF_DTYPE, "a_blockscale"); + CHECK_INPUT(b_blockscales, SF_DTYPE, "b_blockscales"); + CHECK_INPUT(alphas, at::ScalarType::Float, "alphas"); + + TORCH_CHECK( + a_blockscale.dim() == 2, + "expected a_blockscale to be of shape [num_experts, rounded_m," + " k // group_size], observed rank: ", + a_blockscale.dim()) + TORCH_CHECK( + b_blockscales.dim() == 3, + "expected b_blockscale to be of shape: " + " [num_experts, n, k // group_size], observed rank: ", + b_blockscales.dim()) + TORCH_CHECK(problem_sizes.dim() == 2, "problem_sizes must be a 2D tensor"); + TORCH_CHECK(problem_sizes.size(1) == 3, "problem_sizes must have the shape (num_experts, 3)"); + TORCH_CHECK( + problem_sizes.size(0) == expert_offsets.size(0), "Number of experts in problem_sizes must match expert_offsets"); + TORCH_CHECK(problem_sizes.dtype() == torch::kInt32, "problem_sizes must be int32."); + + int M = static_cast(a.size(0)); + int N = static_cast(b.size(1)); + int E = static_cast(b.size(0)); + int K = static_cast(2 * b.size(2)); + + if (output.scalar_type() == torch::kBFloat16) { + run_fp4_blockwise_scaled_group_mm( + output, + a, + b, + a_blockscale, + b_blockscales, + alphas, + ab_strides, + c_strides, + problem_sizes, + expert_offsets, + sf_offsets, + M, + N, + K); + } else { + run_fp4_blockwise_scaled_group_mm( + output, + a, + b, + a_blockscale, + b_blockscales, + alphas, + ab_strides, + c_strides, + problem_sizes, + expert_offsets, + sf_offsets, + M, + N, + K); + } +#else + TORCH_CHECK_NOT_IMPLEMENTED( + false, + "No compiled cutlass_fp4_group_mm kernel, sgl-kernel must " + "be compiled with ENABLE_NVFP4 for SM100+ and CUDA " + "12.8 or above."); +#endif +} diff --git a/sgl-kernel/csrc/moe/prepare_moe_input.cu b/sgl-kernel/csrc/moe/prepare_moe_input.cu new file mode 100644 index 000000000..46fff0649 --- /dev/null +++ b/sgl-kernel/csrc/moe/prepare_moe_input.cu @@ -0,0 +1,392 @@ +#include +#include +#include + +#include +#include + +#include "cutlass/array.h" +#include "utils.h" + +constexpr uint64_t THREADS_PER_EXPERT = 512; + +__global__ void compute_problem_sizes( + const int* __restrict__ topk_ids, + int32_t* problem_sizes1, + int32_t* problem_sizes2, + int32_t* atomic_buffer, + const int64_t topk_length, + const int64_t n, + const int64_t k) { + int expert_id = blockIdx.x; + + int occurrences = 0; + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + occurrences += (topk_ids[i] == expert_id); + } + atomicAdd(&atomic_buffer[expert_id], occurrences); + __syncthreads(); + + if (threadIdx.x == 0) { + int final_occurrences = atomic_buffer[expert_id]; + problem_sizes1[expert_id * 3] = final_occurrences; + problem_sizes1[expert_id * 3 + 1] = static_cast(2 * n); + problem_sizes1[expert_id * 3 + 2] = static_cast(k); + problem_sizes2[expert_id * 3] = final_occurrences; + problem_sizes2[expert_id * 3 + 1] = static_cast(k); + problem_sizes2[expert_id * 3 + 2] = static_cast(n); + } +} + +__global__ void compute_expert_offsets( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* atomic_buffer, + const int64_t num_experts) { + int32_t tot_offset = 0; + expert_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + tot_offset += problem_sizes1[i * 3]; + expert_offsets[i + 1] = tot_offset; + } +} + +__global__ void compute_expert_blockscale_offsets( + const int32_t* __restrict__ problem_sizes1, + int32_t* expert_offsets, + int32_t* blockscale_offsets, + int32_t* atomic_buffer, + const int64_t num_experts) { + int32_t tot_offset = 0; + int32_t tot_rounded_offset = 0; + expert_offsets[0] = 0; + blockscale_offsets[0] = 0; + for (int i = 0; i < num_experts; ++i) { + atomic_buffer[i] = tot_offset; + int num_tokens = problem_sizes1[i * 3]; + int rounded_num_tokens = (num_tokens + (128 - 1)) / 128 * 128; + tot_offset += num_tokens; + tot_rounded_offset += rounded_num_tokens; + expert_offsets[i + 1] = tot_offset; + blockscale_offsets[i + 1] = tot_rounded_offset; + } +} + +__global__ void compute_arg_sorts( + const int32_t* __restrict__ topk_ids, + int32_t* input_permutation, + int32_t* output_permutation, + int32_t* atomic_buffer, + const int64_t topk_length, + const int64_t topk) { + int expert_id = blockIdx.x; + + for (int i = threadIdx.x; i < topk_length; i += THREADS_PER_EXPERT) { + if (topk_ids[i] == expert_id) { + int start = atomicAdd(&atomic_buffer[expert_id], 1); + input_permutation[start] = i / topk; + output_permutation[i] = start; + } + } +} + +void get_moe_prepare_input_caller( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + auto stream = at::cuda::getCurrentCUDAStream(topk_ids.device().index()); + auto options_int32 = torch::TensorOptions().dtype(torch::kInt32).device(topk_ids.device()); + torch::Tensor atomic_buffer = torch::zeros(num_experts, options_int32); + + uint32_t num_threads = static_cast(min(THREADS_PER_EXPERT, topk_ids.numel())); + uint32_t num_blocks = static_cast(num_experts); + + compute_problem_sizes<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(problem_sizes1.data_ptr()), + static_cast(problem_sizes2.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + n, + k); + if (blockscale_offsets.has_value()) { + compute_expert_blockscale_offsets<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(blockscale_offsets.value().data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); + } else { + compute_expert_offsets<<<1, 1, 0, stream>>>( + static_cast(problem_sizes1.data_ptr()), + static_cast(expert_offsets.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + num_experts); + } + compute_arg_sorts<<>>( + static_cast(topk_ids.data_ptr()), + static_cast(input_permutation.data_ptr()), + static_cast(output_permutation.data_ptr()), + static_cast(atomic_buffer.data_ptr()), + topk_ids.numel(), + topk_ids.size(1)); +} + +void prepare_moe_input( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k) { + TORCH_CHECK(topk_ids.dtype() == torch::kInt32); + get_moe_prepare_input_caller( + topk_ids, + expert_offsets, + blockscale_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k); + return; +} + +template +__global__ void shuffleRowsKernel( + const T* input, + const int32_t* dst2src_map, + T* output, + int64_t num_src_rows, + int64_t num_dst_rows, + int64_t num_cols) { + int64_t dest_row_idx = blockIdx.x; + int64_t const source_row_idx = dst2src_map[dest_row_idx]; + + if (blockIdx.x < num_dst_rows) { + // Load 128-bits per thread + constexpr uint64_t ELEM_PER_THREAD = 128 / sizeof(T) / 8; + using DataElem = cutlass::Array; + + // Duplicate and permute rows + auto const* source_row_ptr = reinterpret_cast(input + source_row_idx * num_cols); + auto* dest_row_ptr = reinterpret_cast(output + dest_row_idx * num_cols); + + auto const start_offset = threadIdx.x; + auto const stride = blockDim.x; + auto const num_elems_in_col = num_cols / ELEM_PER_THREAD; + + for (auto elem_index = start_offset; elem_index < num_elems_in_col; elem_index += stride) { + dest_row_ptr[elem_index] = source_row_ptr[elem_index]; + } + } +} + +#define DECLARE_SHUFFLE_ROWS(T) \ + __global__ void shuffleRowsKernel( \ + const T* input, \ + const int32_t* dst2src_map, \ + T* output, \ + int64_t num_src_rows, \ + int64_t num_dest_rows, \ + int64_t num_cols); + +DECLARE_SHUFFLE_ROWS(float); +DECLARE_SHUFFLE_ROWS(half); +DECLARE_SHUFFLE_ROWS(__nv_bfloat16); +DECLARE_SHUFFLE_ROWS(__nv_fp8_e4m3); +DECLARE_SHUFFLE_ROWS(uint8_t); + +#define SHUFFLE_ROWS(T) \ + shuffleRowsKernel<<>>( \ + reinterpret_cast(input), \ + static_cast(dst2src_map.data_ptr()), \ + reinterpret_cast(output), \ + num_src_rows, \ + num_dst_rows, \ + num_cols) + +#define DTYPE_DISPATCH_CASE(T, CUDA_T) \ + case T: \ + SHUFFLE_ROWS(CUDA_T); \ + break; + +void shuffle_rows_caller( + const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) { + TORCH_CHECK( + input_tensor.scalar_type() == output_tensor.scalar_type(), + "Input and output tensors must have the same data type"); + auto stream = at::cuda::getCurrentCUDAStream().stream(); + uint32_t blocks = static_cast(output_tensor.size(0)); + uint32_t threads = 256; + int64_t num_dst_rows = output_tensor.size(0); + int64_t num_src_rows = input_tensor.size(0); + int64_t num_cols = input_tensor.size(1); + const void* input = input_tensor.data_ptr(); + void* output = output_tensor.data_ptr(); + switch (input_tensor.scalar_type()) { + DTYPE_DISPATCH_CASE(torch::kFloat16, half); + DTYPE_DISPATCH_CASE(torch::kBFloat16, __nv_bfloat16); + DTYPE_DISPATCH_CASE(torch::kFloat32, float); + DTYPE_DISPATCH_CASE(torch::kFloat8_e4m3fn, __nv_fp8_e4m3); + DTYPE_DISPATCH_CASE(torch::kUInt8, uint8_t); + default: + TORCH_CHECK(false, "[moe replicate input] data type dispatch fail!"); + } + return; +} + +void shuffle_rows(const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor) { + shuffle_rows_caller(input_tensor, dst2src_map, output_tensor); + return; +} + +template +__global__ void apply_shuffle_mul_sum_kernel( + const scalar_t* __restrict__ input_tensor, // [m * topk, k] (expert-major layout) + scalar_t* __restrict__ output_tensor, // [m, k] (token-major layout) + const int32_t* __restrict__ permutation, // [m * topk] (c_map: token-major-idx -> expert-major-idx) + int m, + int topk, + int row_stride, + const scalar_t* __restrict__ factors) // [m * topk] (topk_weights, token-major layout) +{ + int i = blockIdx.x; + if (i >= m) { + return; + } + + constexpr uint32_t vec_size = 16 / sizeof(scalar_t); + using t = float; + using vec_t = flashinfer::vec_t; + int thread_idx = threadIdx.x; + int stride = blockDim.x; + + for (int d_vec_idx = thread_idx; d_vec_idx < row_stride / vec_size; d_vec_idx += stride) { + int d = d_vec_idx * vec_size; + vec_t sum_vec; + sum_vec.fill(0.0f); + + for (int j = 0; j < topk; ++j) { + int token_major_idx = i * topk + j; + int src_row = permutation[token_major_idx]; + + vec_t val_vec; + val_vec.cast_load(input_tensor + src_row * row_stride + d); + + t factor = 1.0; + if (factors != nullptr) { + factor = factors[token_major_idx]; + } + +#pragma unroll + for (int k = 0; k < vec_size; ++k) { + sum_vec[k] += factor * val_vec[k]; + } + } + sum_vec.cast_store(output_tensor + i * row_stride + d); + } + + // remainder part + int remainder_start = (row_stride / vec_size) * vec_size; + for (int d = remainder_start + thread_idx; d < row_stride; d += stride) { + t sum_val = 0.0; + for (int j = 0; j < topk; ++j) { + int token_major_idx = i * topk + j; + int src_row = permutation[token_major_idx]; + t val = input_tensor[src_row * row_stride + d]; + + t factor = 1.0; + if (factors != nullptr) { + factor = factors[token_major_idx]; + } + sum_val += factor * val; + } + output_tensor[i * row_stride + d] = sum_val; + } +} + +void get_apply_shuffle_mul_sum_caller( + const torch::Tensor& input_tensor, // [m * topk, row_stride], bf16/f16 + torch::Tensor& output_tensor, // [m, row_stride], bf16/f16 + const torch::Tensor& permutation, // [m * topk], int32 + const std::optional& factors_opt) // optional [m * topk], bf16/f16 +{ + TORCH_CHECK(input_tensor.dim() == 2, "input_tensor must be 2D [m * topk, row_stride]"); + TORCH_CHECK(output_tensor.dim() == 2, "output_tensor must be 2D [m, row_stride]"); + TORCH_CHECK(permutation.dim() == 1, "permutation must be 1D [m * topk]"); + + int m = output_tensor.size(0); + int topk = int(permutation.size(0) / m); + int row_stride = output_tensor.size(1); + + TORCH_CHECK(permutation.size(0) == m * topk, "permutation size must match m * topk"); + + auto scalar_type = output_tensor.scalar_type(); + uint32_t vec_size = 16 / sizeof(scalar_type); + auto blockDim = std::min(row_stride / vec_size, 1024U); + dim3 block(blockDim); + + dim3 grid(m); // blockIdx.x = j, blockIdx.y = i + auto stream = at::cuda::getCurrentCUDAStream(input_tensor.device().index()); + + const int32_t* perm_ptr = permutation.data_ptr(); + + void* factors_ptr = nullptr; + if (factors_opt.has_value()) { + TORCH_CHECK(factors_opt->dtype() == output_tensor.dtype(), "Factors must match output dtype"); + TORCH_CHECK(factors_opt->numel() == m * topk, "Factors must have shape [m * topk]"); + factors_ptr = factors_opt->data_ptr(); + } + + DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(output_tensor.scalar_type(), scalar_t, [&] { + apply_shuffle_mul_sum_kernel<<>>( + static_cast(input_tensor.data_ptr()), + static_cast(output_tensor.data_ptr()), + perm_ptr, + m, + topk, + row_stride, + static_cast(factors_ptr)); + return true; + }); +} + +/** + * @brief Applies a permutation-based shuffle, element-wise multiplication, and reduction over the second dimension. + * + * This function performs the equivalent of the following PyTorch expression: + * + * (c2[c_map].view(m, topk, k) * topk_weights.view(m, topk, 1).to(out_dtype)).sum(dim=1) + * + * Specifically: + * - `input` is shuffled using the `permutation` tensor. + * - The shuffled tensor is reshaped and multiplied element-wise with `factors` (e.g., top-k weights). + * - The result is summed along dimension 1 (the top-k dimension), and stored in `output`. + * + * @param input Input tensor of shape (m * topk, k), representing c2. + * @param output Output tensor of shape (m, k), where the final reduced results are stored. + * @param permutation Index tensor (e.g., c_map) that maps positions in `input` to shuffled layout. + * @param factors Optional scaling factors (e.g., top-k weights), shape (m * topk) or (m, topk). + */ +void apply_shuffle_mul_sum( + const torch::Tensor& input, + torch::Tensor& output, + const torch::Tensor& permutation, + const std::optional& factors) { + get_apply_shuffle_mul_sum_caller(input, output, permutation, factors); +} diff --git a/sgl-kernel/csrc/spatial/cuda_utils.h b/sgl-kernel/csrc/spatial/cuda_utils.h new file mode 100644 index 000000000..65132975f --- /dev/null +++ b/sgl-kernel/csrc/spatial/cuda_utils.h @@ -0,0 +1,46 @@ +#include +#include + +#include + +#define CUDA_RT(call) \ + do { \ + cudaError_t _status = (call); \ + if (_status != cudaSuccess) { \ + std::cerr << "ERROR: CUDA RT call \"" << #call << "\" in line " << __LINE__ << " of file " << __FILE__ \ + << " failed with " << cudaGetErrorString(_status) << std::endl; \ + TORCH_CHECK( \ + false, \ + c10::str( \ + "ERROR: CUDA RT call \"", \ + #call, \ + "\" in line ", \ + __LINE__, \ + " of file ", \ + __FILE__, \ + " failed with ", \ + cudaGetErrorString(_status))); \ + } \ + } while (0) + +#define CUDA_DRV(call) \ + do { \ + CUresult _status = (call); \ + if (_status != CUDA_SUCCESS) { \ + const char* err_str; \ + cuGetErrorString(_status, &err_str); \ + std::cerr << "ERROR: CUDA DRV call \"" << #call << "\" in line " << __LINE__ << " of file " << __FILE__ \ + << " failed with " << err_str << std::endl; \ + TORCH_CHECK( \ + false, \ + c10::str( \ + "ERROR: CUDA DRV call \"", \ + #call, \ + "\" in line ", \ + __LINE__, \ + " of file ", \ + __FILE__, \ + " failed with ", \ + err_str)); \ + } \ + } while (0) diff --git a/sgl-kernel/csrc/spatial/greenctx_stream.cu b/sgl-kernel/csrc/spatial/greenctx_stream.cu new file mode 100644 index 000000000..0366565ef --- /dev/null +++ b/sgl-kernel/csrc/spatial/greenctx_stream.cu @@ -0,0 +1,98 @@ +// Documentation: https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__GREEN__CONTEXTS.html +#include + +#include + +#include "cuda_utils.h" +#include "greenctx_stream.h" + +static int CUDA_DRIVER_VERSION; + +using PFN_cuGreenCtxStreamCreate = CUresult(CUDAAPI*)(CUstream*, CUgreenCtx, unsigned int, int); + +auto probe_cuGreenCtxStreamCreate() -> PFN_cuGreenCtxStreamCreate { + static PFN_cuGreenCtxStreamCreate pfn = nullptr; + CUDA_DRV(cuGetProcAddress("cuGreenCtxStreamCreate", reinterpret_cast(&pfn), CUDA_DRIVER_VERSION, 0, nullptr)); + return pfn; +} + +static std::vector create_greenctx_stream_fallback(CUgreenCtx gctx[2]) { + CUstream streamA, streamB; + CUcontext ctx; + + CUDA_DRV(cuCtxFromGreenCtx(&ctx, gctx[0])); + CUDA_DRV(cuCtxPushCurrent(ctx)); + CUDA_DRV(cuStreamCreate(&streamA, CU_STREAM_NON_BLOCKING)); + CUDA_DRV(cuCtxPopCurrent(nullptr)); + + CUDA_DRV(cuCtxFromGreenCtx(&ctx, gctx[1])); + CUDA_DRV(cuCtxPushCurrent(ctx)); + CUDA_DRV(cuStreamCreate(&streamB, CU_STREAM_NON_BLOCKING)); + CUDA_DRV(cuCtxPopCurrent(nullptr)); + + return {(int64_t)streamA, (int64_t)streamB}; +} + +inline void destroy_green_context(CUgreenCtx gctx) { + if (!gctx) return; + CUDA_DRV(cuGreenCtxDestroy(gctx)); +} + +static std::vector create_greenctx_stream_direct_dynamic(CUgreenCtx gctx[2]) { + // This symbol is introduced in CUDA 12.5 + const static auto pfn = probe_cuGreenCtxStreamCreate(); + if (!pfn) { + TORCH_WARN("cuGreenCtxStreamCreate(cuda>=12.5) is not available, using fallback"); + return create_greenctx_stream_fallback(gctx); + } + + CUstream streamA, streamB; + CUDA_DRV(pfn(&streamA, gctx[0], CU_STREAM_NON_BLOCKING, 0)); + CUDA_DRV(pfn(&streamB, gctx[1], CU_STREAM_NON_BLOCKING, 0)); + + return {(int64_t)streamA, (int64_t)streamB}; +} + +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device) { + CUDA_DRV(cuDriverGetVersion(&CUDA_DRIVER_VERSION)); + + CUgreenCtx gctx[3]; + CUdevResourceDesc desc[3]; + CUdevResource input; + CUdevResource resources[4]; + + TORCH_CHECK(smA > 0 && smB > 0, "SM counts must be positive"); + + CUDA_DRV(cuDeviceGetDevResource((CUdevice)device, &input, CU_DEV_RESOURCE_TYPE_SM)); + + const unsigned minCount = static_cast(smA + smB); + const unsigned minCountA = static_cast(smA); + TORCH_CHECK(minCount <= input.sm.smCount, "Not enough SMs available for the requested configuration"); + + unsigned nbGroups = 1; + CUDA_DRV(cuDevSmResourceSplitByCount(&resources[2], &nbGroups, &input, &resources[3], 0, minCount)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[2], &resources[2], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[2], desc[2], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + CUDA_DRV(cuGreenCtxGetDevResource(gctx[2], &input, CU_DEV_RESOURCE_TYPE_SM)); + nbGroups = 1; + CUDA_DRV(cuDevSmResourceSplitByCount(&resources[0], &nbGroups, &input, &resources[1], 0, minCountA)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[0], &resources[0], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[0], desc[0], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + CUDA_DRV(cuDevResourceGenerateDesc(&desc[1], &resources[1], 1)); + CUDA_DRV(cuGreenCtxCreate(&gctx[1], desc[1], (CUdevice)device, CU_GREEN_CTX_DEFAULT_STREAM)); + + const int smCountA = resources[0].sm.smCount; + const int smCountB = resources[1].sm.smCount; + + std::vector streams = create_greenctx_stream_direct_dynamic(gctx); + + destroy_green_context(gctx[2]); + + std::vector vec = { + streams[0], // streamA + streams[1], // streamB + (int64_t)smCountA, + (int64_t)smCountB}; + + return vec; +} diff --git a/sgl-kernel/csrc/spatial/greenctx_stream.h b/sgl-kernel/csrc/spatial/greenctx_stream.h new file mode 100644 index 000000000..2577e9f29 --- /dev/null +++ b/sgl-kernel/csrc/spatial/greenctx_stream.h @@ -0,0 +1,2 @@ +#include +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device); diff --git a/sgl-kernel/csrc/spatial_extension.cc b/sgl-kernel/csrc/spatial_extension.cc new file mode 100644 index 000000000..27833cd70 --- /dev/null +++ b/sgl-kernel/csrc/spatial_extension.cc @@ -0,0 +1,29 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "sgl_kernel_ops.h" + +TORCH_LIBRARY_FRAGMENT(sgl_kernel, m) { + /* + * From csrc/spatial + */ + m.def("create_greenctx_stream_by_value(int smA, int smB, int device) -> int[]"); + m.impl("create_greenctx_stream_by_value", &create_greenctx_stream_by_value); +} + +REGISTER_EXTENSION(spatial_ops) diff --git a/sgl-kernel/csrc/speculative/eagle_utils.cu b/sgl-kernel/csrc/speculative/eagle_utils.cu new file mode 100644 index 000000000..7bf5db274 --- /dev/null +++ b/sgl-kernel/csrc/speculative/eagle_utils.cu @@ -0,0 +1,408 @@ +/* + * Copyright (c) 2025 by SGLang team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef USE_ROCM +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#endif + +typedef enum { FULL_MASK = 0, QLEN_ONLY = 1, QLEN_ONLY_BITPACKING = 2 } TreeMaskMode; + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask [draft_token*(seq_len[0]+draft_token) | draft_token*(seq_len[1]+draft_token) | ..] = +// [sum(verified_seq_len)*draft_token+bs*draft_token*draft_token] positions [bs * draft_token] retrive_index [b, +// draft_token] retrive_next_token [b, draft_token] retrive_next_sibling [b, draft_token] +__global__ void build_tree_efficient( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + bool* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + int tree_mask_mode) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_tree_idx = draft_token_num * draft_token_num * bid; + for (int i = 0; i < bid; i++) { + seq_tree_idx += verified_seq_len[i] * draft_token_num; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx; + if (tree_mask_mode == FULL_MASK) { + token_tree_idx = seq_tree_idx + (seq_len + draft_token_num) * tid + seq_len + 1; + } else { + token_tree_idx = draft_token_num * draft_token_num * bid + draft_token_num * tid + 1; + } + tree_mask[token_tree_idx - 1] = true; + for (int i = 0; i < draft_token_num - 1; i++) { + tree_mask[token_tree_idx + i] = false; + } + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + tree_mask[token_tree_idx + cur_position] = true; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask: [draft_token*num_bytes_per_item | .. ] = [bs*draft_token*num_bytes_per_item] +// positions [bs * draft_token] +// retrive_index [bs, draft_token] +// retrive_next_token [bs, draft_token] +// retrive_next_sibling [bs, draft_token] +__global__ void build_tree_efficient_partial_packed( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + uint8_t* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + size_t num_bytes_per_item) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx = (bid * draft_token_num + tid) * num_bytes_per_item; + tree_mask[token_tree_idx] = 1; // little endian + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + int byte_idx = (cur_position + 1) / 8; + int bit_idx = (cur_position + 1) % 8; + tree_mask[token_tree_idx + byte_idx] |= (1 << bit_idx); + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num, + int64_t tree_mask_mode) { + // TODO (ying) check shape + // TODO (ying) check type + int bs = parent_list.size(0); + dim3 grid(bs); + dim3 block(draft_token_num); + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + if (tree_mask_mode == QLEN_ONLY_BITPACKING) { + size_t num_bytes_per_item = 1; + if (draft_token_num > 16) { + num_bytes_per_item = 4; + } else if (draft_token_num > 8) { + num_bytes_per_item = 2; + } + build_tree_efficient_partial_packed<<>>( + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + num_bytes_per_item); + } else { + build_tree_efficient<<>>( + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + int32_t(tree_mask_mode)); + } +} + +template +__global__ void VerifyTreeGreedy( + IdType* predicts, + IdType* accept_index, + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + IdType2* target_predict, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens) { + uint32_t bx = blockIdx.x; + + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + IdType2 target_token_id = target_predict[last_accepted_retrive_idx]; + + if (draft_token_id == target_token_id) { + // accept token + predicts[last_accepted_retrive_idx] = target_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + predicts[last_accepted_retrive_idx] = target_predict[last_accepted_retrive_idx]; +} + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// target_predict: [bs, num_draft_tokens] +void verify_tree_greedy( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict, + int64_t cuda_stream = 0) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(target_predict); + auto device = target_predict.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(target_predict.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, target_predict); + unsigned int batch_size = candidates.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_predict.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, target_predict.size(1)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'candidates' to be of type long (torch.int64)."); + } + if (retrive_index.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_index' to be of type long (torch.int64)."); + } + if (retrive_next_token.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type long (torch.int64)."); + } + if (retrive_next_sibling.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type long (torch.int64)."); + } + if (target_predict.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'target_predict' to be of type long (torch.int64)."); + } + + cudaStream_t stream = reinterpret_cast(cuda_stream); + dim3 grid(batch_size); + dim3 block(1); + + VerifyTreeGreedy<<>>( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(target_predict.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens); +} diff --git a/sgl-kernel/csrc/speculative/eagle_utils.hip b/sgl-kernel/csrc/speculative/eagle_utils.hip new file mode 100644 index 000000000..4903f78c4 --- /dev/null +++ b/sgl-kernel/csrc/speculative/eagle_utils.hip @@ -0,0 +1,411 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +/* + * Copyright (c) 2025 by SGLang team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#ifndef USE_ROCM +#include "pytorch_extension_utils.h" +#else +#include "pytorch_extension_utils_rocm.h" +#endif + +typedef enum { FULL_MASK = 0, QLEN_ONLY = 1, QLEN_ONLY_BITPACKING = 2 } TreeMaskMode; + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask [draft_token*(seq_len[0]+draft_token) | draft_token*(seq_len[1]+draft_token) | ..] = +// [sum(verified_seq_len)*draft_token+bs*draft_token*draft_token] positions [bs * draft_token] retrive_index [b, +// draft_token] retrive_next_token [b, draft_token] retrive_next_sibling [b, draft_token] +__global__ void build_tree_efficient( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + bool* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + int tree_mask_mode) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_tree_idx = draft_token_num * draft_token_num * bid; + for (int i = 0; i < bid; i++) { + seq_tree_idx += verified_seq_len[i] * draft_token_num; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx; + if (tree_mask_mode == FULL_MASK) { + token_tree_idx = seq_tree_idx + (seq_len + draft_token_num) * tid + seq_len + 1; + } else { + token_tree_idx = draft_token_num * draft_token_num * bid + draft_token_num * tid + 1; + } + tree_mask[token_tree_idx - 1] = true; + for (int i = 0; i < draft_token_num - 1; i++) { + tree_mask[token_tree_idx + i] = false; + } + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + tree_mask[token_tree_idx + cur_position] = true; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +// parent_list [bs, topk * (depth - 1) + 1)] +// selected_index [bs, draft_token_num - 1] +// verified_seq_len [bs] +// tree_mask: [draft_token*num_bytes_per_item | .. ] = [bs*draft_token*num_bytes_per_item] +// positions [bs * draft_token] +// retrive_index [bs, draft_token] +// retrive_next_token [bs, draft_token] +// retrive_next_sibling [bs, draft_token] +__global__ void build_tree_efficient_partial_packed( + int64_t* parent_list, + int64_t* selected_index, + int64_t* verified_seq_len, + uint8_t* tree_mask, + int64_t* positions, + int64_t* retrive_index, + int64_t* retrive_next_token, + int64_t* retrive_next_sibling, + int topk, + int depth, + int draft_token_num, + size_t num_bytes_per_item) { + int bid = blockIdx.x; + int tid = threadIdx.x; + + if (tid >= draft_token_num) { + return; + } + int seq_len = verified_seq_len[bid]; + int token_tree_idx = (bid * draft_token_num + tid) * num_bytes_per_item; + tree_mask[token_tree_idx] = 1; // little endian + + int position = 0; + if (tid == 0) { + positions[bid * draft_token_num] = seq_len; + + int retrive_index_offset = bid * draft_token_num; + for (int i = draft_token_num - 1; i > 0; --i) { + int current_token_idx = retrive_index_offset + i; + retrive_index[bid * draft_token_num + i] = current_token_idx; + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + i - 1] / topk; + int parent_position = 0; + if (parent_tb_idx > 0) { + int parent_token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (; parent_position < draft_token_num; ++parent_position) { + if (selected_index[bid * (draft_token_num - 1) + parent_position] == parent_token_idx) { + ++parent_position; + break; + } + } + } + if (parent_position == draft_token_num) { + printf( + "WARNING: invalid eagle tree!!! Detected a token with no parent token selected. " + "Please check if the logprob has nan. The token will be ignored to keep proceeding.\n"); + continue; + } + + if (retrive_next_token[bid * draft_token_num + parent_position] == -1) { + retrive_next_token[bid * draft_token_num + parent_position] = i; + } else { + int origin_next_token = retrive_next_token[bid * draft_token_num + parent_position]; + retrive_next_token[bid * draft_token_num + parent_position] = i; + retrive_next_sibling[bid * draft_token_num + i] = origin_next_token; + } + } + retrive_index[bid * draft_token_num] = bid * draft_token_num; + } else { + int cur_position = tid - 1; + while (true) { + position += 1; + int byte_idx = (cur_position + 1) / 8; + int bit_idx = (cur_position + 1) % 8; + tree_mask[token_tree_idx + byte_idx] |= (1 << bit_idx); + int parent_tb_idx = selected_index[bid * (draft_token_num - 1) + cur_position] / topk; + if (parent_tb_idx == 0) { + break; + } + + int token_idx = parent_list[bid * (topk * (depth - 1) + 1) + parent_tb_idx]; + for (cur_position = 0; cur_position < draft_token_num; ++cur_position) { + if (selected_index[bid * (draft_token_num - 1) + cur_position] == token_idx) { + break; + } + } + } + positions[bid * draft_token_num + tid] = position + seq_len; + } +} + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num, + int64_t tree_mask_mode) { + // TODO (ying) check shape + // TODO (ying) check type + int bs = parent_list.size(0); + dim3 grid(bs); + dim3 block(draft_token_num); + const hipStream_t stream = at::hip::getCurrentHIPStreamMasqueradingAsCUDA(); + + if (tree_mask_mode == QLEN_ONLY_BITPACKING) { + size_t num_bytes_per_item = 1; + if (draft_token_num > 16) { + num_bytes_per_item = 4; + } else if (draft_token_num > 8) { + num_bytes_per_item = 2; + } + hipLaunchKernelGGL(( build_tree_efficient_partial_packed), dim3(grid), dim3(block), 0, stream, + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + num_bytes_per_item); + } else { + hipLaunchKernelGGL(( build_tree_efficient), dim3(grid), dim3(block), 0, stream, + static_cast(parent_list.data_ptr()), + static_cast(selected_index.data_ptr()), + static_cast(verified_seq_len.data_ptr()), + static_cast(tree_mask.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + int32_t(topk), + int32_t(depth), + int32_t(draft_token_num), + int32_t(tree_mask_mode)); + } +} + +template +__global__ void VerifyTreeGreedy( + IdType* predicts, + IdType* accept_index, + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + IdType2* target_predict, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens) { + uint32_t bx = blockIdx.x; + + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + IdType2 target_token_id = target_predict[last_accepted_retrive_idx]; + + if (draft_token_id == target_token_id) { + // accept token + predicts[last_accepted_retrive_idx] = target_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + predicts[last_accepted_retrive_idx] = target_predict[last_accepted_retrive_idx]; +} + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// target_predict: [bs, num_draft_tokens] +void verify_tree_greedy( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict, + int64_t cuda_stream = 0) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(target_predict); + auto device = target_predict.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(target_predict.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, target_predict); + unsigned int batch_size = candidates.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_predict.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, target_predict.size(1)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'candidates' to be of type long (torch.int64)."); + } + if (retrive_index.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_index' to be of type long (torch.int64)."); + } + if (retrive_next_token.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type long (torch.int64)."); + } + if (retrive_next_sibling.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type long (torch.int64)."); + } + if (target_predict.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'target_predict' to be of type long (torch.int64)."); + } + + hipStream_t stream = reinterpret_cast(cuda_stream); + dim3 grid(batch_size); + dim3 block(1); + + hipLaunchKernelGGL(( VerifyTreeGreedy), dim3(grid), dim3(block), 0, stream, + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(target_predict.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens); +} diff --git a/sgl-kernel/csrc/speculative/packbit.cu b/sgl-kernel/csrc/speculative/packbit.cu new file mode 100644 index 000000000..1decc3ded --- /dev/null +++ b/sgl-kernel/csrc/speculative/packbit.cu @@ -0,0 +1,51 @@ +// This is only a plugin used for flashinfer 0.1.6. The new version does not need it. +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "pytorch_extension_utils.h" + +using namespace flashinfer; + +// bitorder = "little" +void segment_packbits( + at::Tensor x, + at::Tensor input_indptr, + at::Tensor output_indptr, + at::Tensor y, + int64_t batch_size, + int64_t cuda_stream) { + CHECK_INPUT(x); + CHECK_INPUT(input_indptr); + CHECK_INPUT(output_indptr); + auto device = x.device(); + CHECK_EQ(input_indptr.device(), device); + CHECK_EQ(output_indptr.device(), device); + CHECK_EQ(y.device(), device); + CHECK_GE(output_indptr.size(0), batch_size + 1); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + cudaError_t status = quantization::SegmentPackBits( + static_cast(x.data_ptr()), + static_cast(y.data_ptr()), + static_cast(input_indptr.data_ptr()), + static_cast(output_indptr.data_ptr()), + batch_size, + quantization::BitOrder::kLittle, + stream); +} diff --git a/sgl-kernel/csrc/speculative/speculative_sampling.cu b/sgl-kernel/csrc/speculative/speculative_sampling.cu new file mode 100644 index 000000000..ca545e99e --- /dev/null +++ b/sgl-kernel/csrc/speculative/speculative_sampling.cu @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "pytorch_extension_utils.h" +#include "speculative_sampling.cuh" + +using namespace flashinfer; + +// predicts: [tot_num_draft_tokens] +// accept_index: [bs, num_spec_step] +// accept_token_num: [bs] +// candidates: [bs, num_draft_tokens] +// retrive_index: [bs, num_draft_tokens] +// retrive_next_token: [bs, num_draft_tokens] +// retrive_next_sibling: [bs, num_draft_tokens] +// uniform_samples: [bs, num_draft_tokens] +// target_probs: [bs, num_draft_tokens, vocab_size] +void tree_speculative_sampling_target_only( + at::Tensor predicts, + at::Tensor accept_index, + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor uniform_samples_for_final_sampling, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single, + double threshold_acc, + bool deterministic = true, + int64_t cuda_stream = 0) { + CHECK_INPUT(candidates); + CHECK_INPUT(retrive_index); + CHECK_INPUT(retrive_next_token); + CHECK_INPUT(retrive_next_sibling); + CHECK_INPUT(uniform_samples); + CHECK_INPUT(uniform_samples_for_final_sampling); + CHECK_INPUT(target_probs); + auto device = target_probs.device(); + CHECK_EQ(candidates.device(), device); + CHECK_EQ(retrive_index.device(), device); + CHECK_EQ(retrive_next_token.device(), device); + CHECK_EQ(retrive_next_sibling.device(), device); + CHECK_EQ(uniform_samples.device(), device); + CHECK_EQ(uniform_samples_for_final_sampling.device(), device); + CHECK_EQ(target_probs.device(), device); + CHECK_DIM(1, predicts); + CHECK_DIM(2, accept_index); + CHECK_DIM(1, accept_token_num); + CHECK_DIM(2, candidates); + CHECK_DIM(2, retrive_index); + CHECK_DIM(2, retrive_next_token); + CHECK_DIM(2, retrive_next_sibling); + CHECK_DIM(2, uniform_samples); + CHECK_DIM(3, target_probs); + CHECK_DIM(3, draft_probs); + unsigned int batch_size = uniform_samples.size(0); + unsigned int num_spec_step = accept_index.size(1); + unsigned int num_draft_tokens = candidates.size(1); + unsigned int vocab_size = target_probs.size(2); + CHECK_EQ(batch_size, candidates.size(0)); + CHECK_EQ(batch_size, retrive_index.size(0)); + CHECK_EQ(batch_size, retrive_next_token.size(0)); + CHECK_EQ(batch_size, retrive_next_sibling.size(0)); + CHECK_EQ(batch_size, target_probs.size(0)); + CHECK_EQ(num_draft_tokens, retrive_index.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_token.size(1)); + CHECK_EQ(num_draft_tokens, retrive_next_sibling.size(1)); + CHECK_EQ(num_draft_tokens, uniform_samples.size(1)); + CHECK_EQ(num_draft_tokens, target_probs.size(1)); + CHECK_EQ(vocab_size, target_probs.size(2)); + CHECK_EQ(batch_size, accept_index.size(0)); + CHECK_EQ(batch_size, accept_token_num.size(0)); + if (predicts.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'predicts' to be of type int (torch.int32)."); + } + if (accept_index.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_index' to be of type int (torch.int32)."); + } + if (accept_token_num.scalar_type() != at::kInt) { + throw std::runtime_error("Expected 'accept_token_num' to be of type int (torch.int32)."); + } + if (candidates.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'candidates' to be of type long (torch.int64)."); + } + if (retrive_index.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_index' to be of type long (torch.int64)."); + } + if (retrive_next_token.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_token' to be of type long (torch.int64)."); + } + if (retrive_next_sibling.scalar_type() != at::kLong) { + throw std::runtime_error("Expected 'retrive_next_sibling' to be of type long (torch.int64)."); + } + if (uniform_samples.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'uniform_samples' to be of type float (torch.float32)."); + } + if (uniform_samples_for_final_sampling.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'uniform_samples_for_final_sampling' to be of type float (torch.float32)."); + } + if (target_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + if (draft_probs.scalar_type() != at::kFloat) { + throw std::runtime_error("Expected 'target_probs' to be of type float (torch.float32)."); + } + CHECK_GE(threshold_single, 0); + CHECK_GE(1, threshold_single); + CHECK_GE(threshold_acc, 0); + CHECK_GE(1, threshold_acc); + + cudaStream_t stream = reinterpret_cast(cuda_stream); + cudaError_t status = sampling::TreeSpeculativeSamplingTargetOnly( + static_cast(predicts.data_ptr()), + static_cast(accept_index.data_ptr()), + static_cast(accept_token_num.data_ptr()), + static_cast(candidates.data_ptr()), + static_cast(retrive_index.data_ptr()), + static_cast(retrive_next_token.data_ptr()), + static_cast(retrive_next_sibling.data_ptr()), + static_cast(uniform_samples.data_ptr()), + static_cast(uniform_samples_for_final_sampling.data_ptr()), + static_cast(target_probs.data_ptr()), + static_cast(draft_probs.data_ptr()), + batch_size, + num_spec_step, + num_draft_tokens, + vocab_size, + static_cast(threshold_single), + static_cast(threshold_acc), + deterministic, + stream); + + TORCH_CHECK( + status == cudaSuccess, + "TreeSpeculativeSamplingTargetOnly failed with error code " + std::string(cudaGetErrorString(status))); +} diff --git a/sgl-kernel/csrc/speculative/speculative_sampling.cuh b/sgl-kernel/csrc/speculative/speculative_sampling.cuh new file mode 100644 index 000000000..59f18bc2f --- /dev/null +++ b/sgl-kernel/csrc/speculative/speculative_sampling.cuh @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2025 by SGLang team. + * Copyright (c) 2024-2025 by FlashInfer team. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef SPECULATIVE_SAMPLING_CUH_ +#define SPECULATIVE_SAMPLING_CUH_ + +#include + +#include + +namespace flashinfer { + +namespace sampling { + +using namespace cub; + +template < + uint32_t BLOCK_THREADS, + BlockScanAlgorithm SCAN_ALGORITHM, + BlockReduceAlgorithm REDUCE_ALGORITHM, + uint32_t VEC_SIZE, + bool DETERMINISTIC, + typename DType, + typename IdType, + typename IdType2> +__global__ void TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* accept_index, // mutable + IdType* accept_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single, + DType threshold_acc) { + const uint32_t bx = blockIdx.x, tx = threadIdx.x; + + extern __shared__ __align__(alignof(SamplingTempStorage)) + uint8_t smem_sampling[]; + auto& temp_storage = + reinterpret_cast&>(smem_sampling); + + DType prob_acc = 0.0; + uint32_t cur_prob_offset = bx * num_draft_tokens * d; + DType coin = uniform_samples[bx * num_draft_tokens]; + IdType2 last_accepted_retrive_idx = retrive_index[bx * num_draft_tokens]; + accept_index[bx * num_speculative_tokens] = last_accepted_retrive_idx; + uint32_t num_accepted_tokens = 0; + IdType2 cur_index = 0; + + for (uint32_t j = 1; j < num_speculative_tokens; ++j) { + cur_index = retrive_next_token[bx * num_draft_tokens + cur_index]; + while (cur_index != -1) { + IdType2 draft_index = retrive_index[bx * num_draft_tokens + cur_index]; + IdType2 draft_token_id = candidates[bx * num_draft_tokens + cur_index]; + DType target_prob_single = target_probs[cur_prob_offset + draft_token_id]; + prob_acc += target_prob_single; + + if (coin <= prob_acc / threshold_acc || target_prob_single >= threshold_single) { + // accept token + prob_acc = 0.; + cur_prob_offset = (bx * num_draft_tokens + cur_index) * d; + coin = uniform_samples[bx * num_draft_tokens + cur_index]; + predicts[last_accepted_retrive_idx] = draft_token_id; + ++num_accepted_tokens; + accept_index[bx * num_speculative_tokens + num_accepted_tokens] = draft_index; + last_accepted_retrive_idx = draft_index; + break; + } else { + // FIXME: leverage draft probs + draft_probs[cur_prob_offset + draft_token_id] = target_probs[cur_prob_offset + draft_token_id]; + cur_index = retrive_next_sibling[bx * num_draft_tokens + cur_index]; + } + } + if (cur_index == -1) break; + } + accept_token_num[bx] = num_accepted_tokens; + + // we need a different coin for the final sampling + coin = uniform_samples_for_final_sampling[bx]; + + // sample from relu(target_probs - draft_probs) + DType sum_relu_q_minus_p(0); + vec_t q_vec, p_vec; + DType relu_q_minus_p[VEC_SIZE]; + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + sum_relu_q_minus_p += BlockReduce(temp_storage.block_prim.reduce) + .Sum(relu_q_minus_p); + __syncthreads(); + } + if (tx == 0) { + temp_storage.block_aggregate.value = sum_relu_q_minus_p; + } + // init the first rejected token to (d - 1) + temp_storage.sampled_id = d - 1; + __syncthreads(); + sum_relu_q_minus_p = temp_storage.block_aggregate.value; + DType u = coin * sum_relu_q_minus_p; + + DType aggregate_relu_q_minus_p(0); + for (uint32_t i = 0; i < ceil_div(d, BLOCK_THREADS * VEC_SIZE); ++i) { + q_vec.fill(DType(0)); + p_vec.fill(DType(0)); + if ((i * BLOCK_THREADS + tx) * VEC_SIZE < d) { + q_vec.load(target_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + if (num_accepted_tokens != num_speculative_tokens - 1) { + // there is no draft_probs for the bonus token + p_vec.load(draft_probs + cur_prob_offset + i * BLOCK_THREADS * VEC_SIZE + tx * VEC_SIZE); + } + } + + vec_t relu_q_minus_p_vec; +#pragma unroll + for (uint32_t j = 0; j < VEC_SIZE; ++j) { + relu_q_minus_p_vec[j] = max(q_vec[j] - p_vec[j], DType(0)); + } + + DeviceSamplingFromProb( + i, d, [&](DType x) { return x > 0; }, u, relu_q_minus_p_vec, aggregate_relu_q_minus_p, &temp_storage); + if (aggregate_relu_q_minus_p > u) { + break; + } + } + __syncthreads(); + // set the first rejected token + predicts[last_accepted_retrive_idx] = temp_storage.sampled_id; + // value at not used indices are undefined +} + +template +cudaError_t TreeSpeculativeSamplingTargetOnly( + IdType* predicts, // mutable + IdType* output_token_ids, // mutable + IdType* output_accepted_token_num, // mutable + IdType2* candidates, + IdType2* retrive_index, + IdType2* retrive_next_token, + IdType2* retrive_next_sibling, + DType* uniform_samples, + DType* uniform_samples_for_final_sampling, + DType* target_probs, + DType* draft_probs, + uint32_t batch_size, + uint32_t num_speculative_tokens, + uint32_t num_draft_tokens, + uint32_t d, + DType threshold_single = 1, + DType threshold_acc = 1, + bool deterministic = true, + cudaStream_t stream = 0) { + constexpr uint32_t BLOCK_THREADS = 1024; + const uint32_t vec_size = std::gcd(16 / sizeof(DType), d); + + const uint32_t smem_size = sizeof(SamplingTempStorage); + dim3 nblks(batch_size); + dim3 nthrs(BLOCK_THREADS); + float capped_threshold_acc = fmaxf(threshold_acc, 1e-9f); + void* args[] = { + &predicts, + &output_token_ids, + &output_accepted_token_num, + &candidates, + &retrive_index, + &retrive_next_token, + &retrive_next_sibling, + &uniform_samples, + &uniform_samples_for_final_sampling, + &target_probs, + &draft_probs, + &batch_size, + &num_speculative_tokens, + &num_draft_tokens, + &d, + &threshold_single, + &capped_threshold_acc}; + DISPATCH_ALIGNED_VEC_SIZE( + vec_size, VEC_SIZE, {DISPATCH_DETERMINISTIC(deterministic, DETERMINISTIC, { + auto kernel = TreeSpeculativeSamplingTargetOnly< + BLOCK_THREADS, + SCAN_ALGO, + REDUCE_ALGO, + VEC_SIZE, + DETERMINISTIC, + DType, + IdType, + IdType2>; + FLASHINFER_CUDA_CALL(cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size)); + FLASHINFER_CUDA_CALL(cudaLaunchKernel((void*)kernel, nblks, nthrs, args, smem_size, stream)); + })}); + return cudaSuccess; +} + +} // namespace sampling + +} // namespace flashinfer + +#endif // SPECULATIVE_SAMPLING_CUH_ diff --git a/sgl-kernel/include/hip/hip_act_and_mul.cuh b/sgl-kernel/include/hip/hip_act_and_mul.cuh new file mode 100644 index 000000000..ddb1b702d --- /dev/null +++ b/sgl-kernel/include/hip/hip_act_and_mul.cuh @@ -0,0 +1,87 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include "utils.h" + +#define kBitsToLoad 128 +#define kBytesToLoad (kBitsToLoad / 8) + +// Adapted from +// [flashinfer::activation::act_and_mul_kernel](https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/include/flashinfer/activation.cuh#L29) + +namespace sgl_hip { +namespace activation { + +template +__global__ void act_and_mul_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * 2 * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); + y_vec.cast_load(input + offset + d + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]) * y_vec[i]; + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx], y = input[offset + remaining_offset + d + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x) * y; + } +} + +template +__global__ void act_only_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]); + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x); + } +} + +} // namespace activation +} // namespace sgl_hip diff --git a/sgl-kernel/include/hip/hip_act_and_mul_hip.cuh b/sgl-kernel/include/hip/hip_act_and_mul_hip.cuh new file mode 100644 index 000000000..d45409fa5 --- /dev/null +++ b/sgl-kernel/include/hip/hip_act_and_mul_hip.cuh @@ -0,0 +1,90 @@ +// !!! This is a file automatically generated by hipify!!! +#include +#include "hip/hip_runtime.h" +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include "utils_hip.h" + +#define kBitsToLoad 128 +#define kBytesToLoad (kBitsToLoad / 8) + +// Adapted from +// [flashinfer::activation::act_and_mul_kernel](https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/include/flashinfer/activation.cuh#L29) + +namespace sgl_hip { +namespace activation { + +template +__global__ void act_and_mul_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * 2 * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); + y_vec.cast_load(input + offset + d + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]) * y_vec[i]; + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx], y = input[offset + remaining_offset + d + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x) * y; + } +} + +template +__global__ void act_only_kernel(T* __restrict__ out, const T* __restrict__ input, const int d) { + constexpr uint32_t vec_size = kBytesToLoad / sizeof(T); + const int64_t token_idx = blockIdx.x; + const int64_t thread_idx = threadIdx.x; + const int64_t stride = blockDim.x; + const int64_t offset = token_idx * d; + +#pragma unroll 1 + for (uint32_t idx = thread_idx; idx < d / vec_size; idx += stride) { + sgl_hip::vec_t x_vec, y_vec, out_vec; + x_vec.cast_load(input + offset + idx * vec_size); +#pragma unroll + for (uint32_t i = 0; i < vec_size; ++i) { + out_vec[i] = Activation(x_vec[i]); + } + out_vec.cast_store(out + token_idx * d + idx * vec_size); + } + + const int64_t remaining_offset = d - d % (stride * vec_size); + // process the remaining elements +#pragma unroll 1 + for (int64_t idx = thread_idx; idx < d % (stride * vec_size); idx += stride) { + T x = input[offset + remaining_offset + idx]; + out[token_idx * d + remaining_offset + idx] = Activation(x); + } +} + +} // namespace activation +} // namespace sgl_hip diff --git a/sgl-kernel/include/hip/hip_math_def.h b/sgl-kernel/include/hip/hip_math_def.h new file mode 100644 index 000000000..356ed953f --- /dev/null +++ b/sgl-kernel/include/hip/hip_math_def.h @@ -0,0 +1,94 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#ifdef USE_ROCM + +#include +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +namespace amdgpu { + +template +__forceinline__ __device__ T shfl_xor_sync(unsigned mask, T var, int laneMask, int width = warpSize); + +template +__forceinline__ __device__ destDtype cast(srcDtype val); + +// specialization +template <> +__forceinline__ __device__ float shfl_xor_sync(unsigned mask, float var, int laneMask, int width) { + return __shfl_xor(var, laneMask, width); +} + +template <> +__forceinline__ __device__ int shfl_xor_sync(unsigned mask, int var, int laneMask, int width) { + return __shfl_xor(var, laneMask, width); +} + +template <> +__forceinline__ __device__ float cast(float val) { + return val; +} + +template <> +__forceinline__ __device__ float cast(__half val) { + return __half2float(val); +} + +template <> +__forceinline__ __device__ float cast(__hip_bfloat16 val) { + return __bfloat162float(val); +} + +template <> +__forceinline__ __device__ __half cast(float fval) { + return __float2half(fval); +} + +template <> +__forceinline__ __device__ __hip_bfloat16 cast(float fval) { + return __float2bfloat16(fval); +} + +} // namespace amdgpu + +template +__forceinline__ __device__ T __shfl_xor_sync(unsigned mask, T var, int laneMask, int width = warpSize) { + return amdgpu::shfl_xor_sync(mask, var, laneMask, width); +} + +template +__device__ __forceinline__ float castToFloat(srcDtype val) { + return amdgpu::cast(val); +} + +template +__device__ __forceinline__ dstDtype castFromFloat(float val) { + return amdgpu::cast(val); +} + +// operator overload to support flashinfer +__host__ __device__ __forceinline__ __half operator*(const __half& x, const __half& y) { + __half h_x = x; + __half h_y = y; + return __hmul(h_x, h_y); +} + +#endif diff --git a/sgl-kernel/include/hip/hip_vec_dtypes.h b/sgl-kernel/include/hip/hip_vec_dtypes.h new file mode 100644 index 000000000..a68a6986e --- /dev/null +++ b/sgl-kernel/include/hip/hip_vec_dtypes.h @@ -0,0 +1,101 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#if USE_ROCM + +#include +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491)d + +#define SGL_HIP_INLINE inline __attribute__((always_inline)) __device__ + +namespace sgl_hip { + +template +struct vec_t; + +template +SGL_HIP_INLINE void cast_load_impl(vec_t& dst, const srcDtype* src); + +template +SGL_HIP_INLINE void cast_store_impl(dstDtype* dst_ptr, const vec_t& src); + +template +struct vec_t { + SGL_HIP_INLINE float_t& operator[](size_t i); + SGL_HIP_INLINE const float_t& operator[](size_t i) const; + SGL_HIP_INLINE float_t* ptr(); + + SGL_HIP_INLINE void load(const float_t* ptr); + SGL_HIP_INLINE void store(float_t* ptr) const; + + template + SGL_HIP_INLINE void cast_from(const vec_t& src); + template + SGL_HIP_INLINE void cast_load(const T* ptr); + template + SGL_HIP_INLINE void cast_store(T* ptr) const; +}; + +} // namespace sgl_hip + +// **** impl ***** + +namespace sgl_hip { + +template +SGL_HIP_INLINE void cast_load_impl(vec_t& dst, const srcDtype* src_ptr) { + if constexpr (std::is_same::value) { + dst.load(src_ptr); + } else { + vec_t tmp; + tmp.load(src_ptr); + dst.cast_from(tmp); + } +} + +template +SGL_HIP_INLINE void cast_store_impl(dstDtype* dst_ptr, const vec_t& src) { + if constexpr (std::is_same::value) { + src.store(dst_ptr); + } else { + vec_t tmp; + tmp.cast_from(src); + tmp.store(dst_ptr); + } +} + +template +template +SGL_HIP_INLINE void vec_t::cast_load(const T* ptr) { + cast_load_impl(*this, ptr); +} + +template +template +SGL_HIP_INLINE void vec_t::cast_store(T* ptr) const { + cast_store_impl(ptr, *this); +} + +} // namespace sgl_hip + +#include "impl/hip_vec_bf16_impl.h" +#include "impl/hip_vec_fp32_impl.h" +#include "impl/hip_vec_half_impl.h" +#endif diff --git a/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h b/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h new file mode 100644 index 000000000..b783f3f43 --- /dev/null +++ b/sgl-kernel/include/hip/impl/hip_vec_bf16_impl.h @@ -0,0 +1,177 @@ +#pragma once + +#if USE_ROCM + +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +using nv_bfloat16 = __hip_bfloat16; +using nv_bfloat162 = __hip_bfloat162; + +__BF16_HOST_DEVICE_STATIC__ __hip_bfloat162 make_bfloat162(const __hip_bfloat16 x, const __hip_bfloat16 y) { + __hip_bfloat162 t; + t.x = x; + t.y = y; + return t; +} + +namespace sgl_hip { + +// nv_bfloat16 x 1 +template <> +struct vec_t { + nv_bfloat16 data; + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *ptr = data; +} + +// nv_bfloat16 x 2 +template <> +struct vec_t { + nv_bfloat162 data; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *((nv_bfloat162*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *((nv_bfloat162*)ptr) = data; +} + +template <> +struct vec_t { + uint2 data; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)(&data))[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr); + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const nv_bfloat16* ptr) { + data = *((uint2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(nv_bfloat16* ptr) const { + *((uint2*)ptr) = data; +} + +// nv_bfloat16 x 8 or more + +template +struct vec_t { + uint4 data[vec_size / 8]; + + SGL_HIP_INLINE nv_bfloat16& operator[](size_t i) { + return ((nv_bfloat16*)data)[i]; + } + SGL_HIP_INLINE const nv_bfloat16& operator[](size_t i) const { + return ((const nv_bfloat16*)data)[i]; + } + SGL_HIP_INLINE nv_bfloat16* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const nv_bfloat16* ptr) { +#pragma unoll + for (size_t i = 0; i < vec_size / 8; ++i) { + data[i] = ((uint4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(nv_bfloat16* ptr) const { +#pragma unoll + for (size_t i = 0; i < vec_size / 8; ++i) { + ((uint4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip + +#endif diff --git a/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h b/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h new file mode 100644 index 000000000..97cba6320 --- /dev/null +++ b/sgl-kernel/include/hip/impl/hip_vec_fp32_impl.h @@ -0,0 +1,129 @@ +#pragma once + +#if USE_ROCM + +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +namespace sgl_hip { + +template <> +struct vec_t { + float data; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(&data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(&data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr); + SGL_HIP_INLINE void store(float* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const float* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(float* ptr) const { + *ptr = data; +} + +// float x 2 + +template <> +struct vec_t { + float2 data; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(&data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(&data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr); + SGL_HIP_INLINE void store(float* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const float* ptr) { + data = *((float2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(float* ptr) const { + *((float2*)ptr) = data; +} + +// float x 4 or more +template +struct vec_t { + float4 data[vec_size / 4]; + + SGL_HIP_INLINE float& operator[](size_t i) { + return ((float*)(data))[i]; + } + SGL_HIP_INLINE const float& operator[](size_t i) const { + return ((const float*)(data))[i]; + } + SGL_HIP_INLINE float* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const float* ptr) { +#pragma unroll + for (size_t i = 0; i < vec_size / 4; ++i) { + data[i] = ((float4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(float* ptr) const { +#pragma unroll + for (size_t i = 0; i < vec_size / 4; ++i) { + ((float4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip + +#endif diff --git a/sgl-kernel/include/hip/impl/hip_vec_half_impl.h b/sgl-kernel/include/hip/impl/hip_vec_half_impl.h new file mode 100644 index 000000000..767b9c62f --- /dev/null +++ b/sgl-kernel/include/hip/impl/hip_vec_half_impl.h @@ -0,0 +1,172 @@ +#pragma once + +#if USE_ROCM + +#include +#include + +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) + +using half = __half; +using half2 = __half2; + +namespace sgl_hip { + +// half x 1 +template <> +struct vec_t { + half data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *ptr; +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *ptr = data; +} + +// half x 2 +template <> +struct vec_t { + half2 data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *((half2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *((half2*)ptr) = data; +} + +// half x 4 + +template <> +struct vec_t { + uint2 data; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)(&data))[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)(&data))[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr); + SGL_HIP_INLINE void store(half* ptr) const; + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +SGL_HIP_INLINE void vec_t::load(const half* ptr) { + data = *((uint2*)ptr); +} + +SGL_HIP_INLINE void vec_t::store(half* ptr) const { + *((uint2*)ptr) = data; +} + +// half x 8 or more + +template +struct vec_t { + uint4 data[vec_size / 8]; + + SGL_HIP_INLINE half& operator[](size_t i) { + return ((half*)data)[i]; + } + SGL_HIP_INLINE const half& operator[](size_t i) const { + return ((const half*)data)[i]; + } + SGL_HIP_INLINE half* ptr() { + return reinterpret_cast(&data); + } + SGL_HIP_INLINE void load(const half* ptr) { +#pragma unroll + for (size_t i = 0; i < vec_size / 8; ++i) { + data[i] = ((uint4*)ptr)[i]; + } + } + SGL_HIP_INLINE void store(half* ptr) const { +#pragma unroll + for (size_t i = 0; i < vec_size / 8; ++i) { + ((uint4*)ptr)[i] = data[i]; + } + } + template + SGL_HIP_INLINE void cast_from(const vec_t& src) { + cast_from_impl(*this, src); + } + template + SGL_HIP_INLINE void cast_load(const T* ptr) { + cast_load_impl(*this, ptr); + } + template + SGL_HIP_INLINE void cast_store(T* ptr) const { + cast_store_impl(ptr, *this); + } +}; + +} // namespace sgl_hip +#endif diff --git a/sgl-kernel/include/pytorch_extension_utils_rocm.h b/sgl-kernel/include/pytorch_extension_utils_rocm.h new file mode 100644 index 000000000..fa5fd129f --- /dev/null +++ b/sgl-kernel/include/pytorch_extension_utils_rocm.h @@ -0,0 +1,20 @@ +#include + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) diff --git a/sgl-kernel/include/scalar_type.hpp b/sgl-kernel/include/scalar_type.hpp new file mode 100644 index 000000000..9161a38b7 --- /dev/null +++ b/sgl-kernel/include/scalar_type.hpp @@ -0,0 +1,330 @@ +#pragma once + +// For TORCH_CHECK +#include + +namespace sglang { + +// +// ScalarType can represent a wide range of floating point and integer types, +// in particular it can be used to represent sub-byte data types (something +// that torch.dtype currently does not support). +// +// The type definitions on the Python side can be found in: vllm/scalar_type.py +// these type definitions should be kept up to date with any Python API changes +// here. +// +class ScalarType { + public: + enum NanRepr : uint8_t { + NAN_NONE = 0, // nans are not supported + NAN_IEEE_754 = 1, // nans are: exp all 1s, mantissa not all 0s + NAN_EXTD_RANGE_MAX_MIN = 2, // nans are: exp all 1s, mantissa all 1s + + NAN_REPR_ID_MAX + }; + + constexpr ScalarType( + uint8_t exponent, + uint8_t mantissa, + bool signed_, + int32_t bias, + bool finite_values_only = false, + NanRepr nan_repr = NAN_IEEE_754) + : exponent(exponent), + mantissa(mantissa), + signed_(signed_), + bias(bias), + finite_values_only(finite_values_only), + nan_repr(nan_repr) {}; + + static constexpr ScalarType int_(uint8_t size_bits, int32_t bias = 0) { + return ScalarType(0, size_bits - 1, true, bias); + } + + static constexpr ScalarType uint(uint8_t size_bits, int32_t bias = 0) { + return ScalarType(0, size_bits, false, bias); + } + + // IEEE 754 compliant floating point type + static constexpr ScalarType float_IEEE754(uint8_t exponent, uint8_t mantissa) { + TORCH_CHECK(mantissa > 0 && exponent > 0); + return ScalarType(exponent, mantissa, true, 0, false, NAN_IEEE_754); + } + + // IEEE 754 non-compliant floating point type + static constexpr ScalarType float_(uint8_t exponent, uint8_t mantissa, bool finite_values_only, NanRepr nan_repr) { + TORCH_CHECK(nan_repr < NAN_REPR_ID_MAX, "Invalid NanRepr"); + TORCH_CHECK(mantissa > 0 && exponent > 0); + TORCH_CHECK( + nan_repr != NAN_IEEE_754, + "use `float_IEEE754` constructor for floating point types that " + "follow IEEE 754 conventions"); + return ScalarType(exponent, mantissa, true, 0, finite_values_only, nan_repr); + } + + uint8_t const exponent; // size of the exponent field (0 for integer types) + uint8_t const mantissa; // size of the mantissa field (size of the integer + // excluding the sign bit for integer types) + bool const signed_; // flag if the type supports negative numbers (i.e. has a + // sign bit) + int32_t const bias; // stored values equal value + bias, + // used for quantized type + + // Extra Floating point info + bool const finite_values_only; // i.e. no +/-inf if true + NanRepr const nan_repr; // how NaNs are represented + // (not applicable for integer types) + + using Id = int64_t; + + private: + // Field size in id + template + static constexpr size_t member_id_field_width() { + using T = std::decay_t; + return std::is_same_v ? 1 : sizeof(T) * 8; + } + + template + static constexpr auto reduce_members_helper(Fn f, Init val, Member member, Rest... rest) { + auto new_val = f(val, member); + if constexpr (sizeof...(rest) > 0) { + return reduce_members_helper(f, new_val, rest...); + } else { + return new_val; + }; + } + + template + constexpr auto reduce_members(Fn f, Init init) const { + // Should be in constructor order for `from_id` + return reduce_members_helper(f, init, exponent, mantissa, signed_, bias, finite_values_only, nan_repr); + }; + + template + static constexpr auto reduce_member_types(Fn f, Init init) { + constexpr auto dummy_type = ScalarType(0, 0, false, 0, false, NAN_NONE); + return dummy_type.reduce_members(f, init); + }; + + static constexpr auto id_size_bits() { + return reduce_member_types( + [](int acc, auto member) -> int { return acc + member_id_field_width(); }, 0); + } + + public: + // unique id for this scalar type that can be computed at compile time for + // c++17 template specialization this is not needed once we migrate to + // c++20 and can pass literal classes as template parameters + constexpr Id id() const { + static_assert(id_size_bits() <= sizeof(Id) * 8, "ScalarType id is too large to be stored"); + + auto or_and_advance = [](std::pair result, auto member) -> std::pair { + auto [id, bit_offset] = result; + auto constexpr bits = member_id_field_width(); + return {id | (int64_t(member) & ((uint64_t(1) << bits) - 1)) << bit_offset, bit_offset + bits}; + }; + return reduce_members(or_and_advance, std::pair{}).first; + } + + // create a ScalarType from an id, for c++17 template specialization, + // this is not needed once we migrate to c++20 and can pass literal + // classes as template parameters + static constexpr ScalarType from_id(Id id) { + auto extract_and_advance = [id](auto result, auto member) { + using T = decltype(member); + auto [tuple, bit_offset] = result; + auto constexpr bits = member_id_field_width(); + auto extracted_val = static_cast((int64_t(id) >> bit_offset) & ((uint64_t(1) << bits) - 1)); + auto new_tuple = std::tuple_cat(tuple, std::make_tuple(extracted_val)); + return std::pair{new_tuple, bit_offset + bits}; + }; + + auto [tuple_args, _] = reduce_member_types(extract_and_advance, std::pair, int>{}); + return std::apply([](auto... args) { return ScalarType(args...); }, tuple_args); + } + + constexpr int64_t size_bits() const { + return mantissa + exponent + is_signed(); + } + constexpr bool is_signed() const { + return signed_; + } + constexpr bool is_integer() const { + return exponent == 0; + } + constexpr bool is_floating_point() const { + return exponent > 0; + } + constexpr bool is_ieee_754() const { + return is_floating_point() && finite_values_only == false && nan_repr == NAN_IEEE_754; + } + constexpr bool has_nans() const { + return is_floating_point() && nan_repr != NAN_NONE; + } + constexpr bool has_infs() const { + return is_floating_point() && finite_values_only == false; + } + constexpr bool has_bias() const { + return bias != 0; + } + + private: + double _floating_point_max() const { + TORCH_CHECK(mantissa <= 52 && exponent <= 11, "Cannot represent max/min as a double for type ", str()); + + uint64_t max_mantissa = (uint64_t(1) << mantissa) - 1; + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN) { + max_mantissa -= 1; + } + + uint64_t max_exponent = (uint64_t(1) << exponent) - 2; + if (nan_repr == NAN_EXTD_RANGE_MAX_MIN || nan_repr == NAN_NONE) { + TORCH_CHECK(exponent < 11, "Cannot represent max/min as a double for type ", str()); + max_exponent += 1; + } + + // adjust the exponent to match that of a double + // for now we assume the exponent bias is the standard 2^(e-1) -1, (where e + // is the exponent bits), there is some precedent for non-standard biases, + // example `float8_e4m3b11fnuz` here: https://github.com/jax-ml/ml_dtypes + // but to avoid premature over complication we are just assuming the + // standard exponent bias until there is a need to support non-standard + // biases + uint64_t exponent_bias = (uint64_t(1) << (exponent - 1)) - 1; + uint64_t exponent_bias_double = (uint64_t(1) << 10) - 1; // double e = 11 + + uint64_t max_exponent_double = max_exponent - exponent_bias + exponent_bias_double; + + // shift the mantissa into the position for a double and + // the exponent + uint64_t double_raw = (max_mantissa << (52 - mantissa)) | (max_exponent_double << 52); + + return *reinterpret_cast(&double_raw); + } + + constexpr std::variant _raw_max() const { + if (is_floating_point()) { + return {_floating_point_max()}; + } else { + TORCH_CHECK(size_bits() < 64 || size_bits() == 64 && is_signed(), "Cannot represent max as a int64_t"); + return {(int64_t(1) << mantissa) - 1}; + } + } + + constexpr std::variant _raw_min() const { + if (is_floating_point()) { + TORCH_CHECK(is_signed(), "We currently assume all floating point types are signed"); + constexpr uint64_t sign_bit_double = (uint64_t(1) << 63); + + double max = _floating_point_max(); + uint64_t max_raw = *reinterpret_cast(&max); + uint64_t min_raw = max_raw | sign_bit_double; + return {*reinterpret_cast(&min_raw)}; + } else { + TORCH_CHECK(!is_signed() || size_bits() <= 64, "Cannot represent min as a int64_t"); + if (is_signed()) { + // set the top bit to 1 (i.e. INT64_MIN) and the rest to 0 + // then perform an arithmetic shift right to set all the bits above + // (size_bits() - 1) to 1 + return {INT64_MIN >> (64 - size_bits())}; + } else { + return {int64_t(0)}; + } + } + } + + public: + // Max representable value for this scalar type. + // (accounting for bias if there is one) + constexpr std::variant max() const { + return std::visit([this](auto x) -> std::variant { return {x - bias}; }, _raw_max()); + } + + // Min representable value for this scalar type. + // (accounting for bias if there is one) + constexpr std::variant min() const { + return std::visit([this](auto x) -> std::variant { return {x - bias}; }, _raw_min()); + } + + std::string str() const { + /* naming generally follows: https://github.com/jax-ml/ml_dtypes + * for floating point types (leading f) the scheme is: + * `float_em[flags]` + * flags: + * - no-flags: means it follows IEEE 754 conventions + * - f: means finite values only (no infinities) + * - n: means nans are supported (non-standard encoding) + * for integer types the scheme is: + * `[u]int[b]` + * - if bias is not present it means its zero + */ + if (is_floating_point()) { + auto ret = + "float" + std::to_string(size_bits()) + "_e" + std::to_string(exponent) + "m" + std::to_string(mantissa); + if (!is_ieee_754()) { + if (finite_values_only) { + ret += "f"; + } + if (nan_repr != NAN_NONE) { + ret += "n"; + } + } + return ret; + } else { + auto ret = ((is_signed()) ? "int" : "uint") + std::to_string(size_bits()); + if (has_bias()) { + ret += "b" + std::to_string(bias); + } + return ret; + } + } + + constexpr bool operator==(ScalarType const& other) const { + return mantissa == other.mantissa && exponent == other.exponent && bias == other.bias && signed_ == other.signed_ && + finite_values_only == other.finite_values_only && nan_repr == other.nan_repr; + } +}; + +using ScalarTypeId = ScalarType::Id; + +// "rust style" names generally following: +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L60-L70 +static inline constexpr auto kS4 = ScalarType::int_(4); +static inline constexpr auto kU4 = ScalarType::uint(4); +static inline constexpr auto kU4B8 = ScalarType::uint(4, 8); +static inline constexpr auto kS8 = ScalarType::int_(8); +static inline constexpr auto kU8 = ScalarType::uint(8); +static inline constexpr auto kU8B128 = ScalarType::uint(8, 128); + +static inline constexpr auto kFE2M1f = ScalarType::float_(2, 1, true, ScalarType::NAN_NONE); +static inline constexpr auto kFE3M2f = ScalarType::float_(3, 2, true, ScalarType::NAN_NONE); +static inline constexpr auto kFE4M3fn = ScalarType::float_(4, 3, true, ScalarType::NAN_EXTD_RANGE_MAX_MIN); +static inline constexpr auto kFE5M2 = ScalarType::float_IEEE754(5, 2); +static inline constexpr auto kFE8M7 = ScalarType::float_IEEE754(8, 7); +static inline constexpr auto kFE5M10 = ScalarType::float_IEEE754(5, 10); + +// Fixed width style names, generally following: +// https://github.com/pytorch/pytorch/blob/6d9f74f0af54751311f0dd71f7e5c01a93260ab3/torch/csrc/api/include/torch/types.h#L47-L57 +static inline constexpr auto kInt4 = kS4; +static inline constexpr auto kUint4 = kU4; +static inline constexpr auto kUint4b8 = kU4B8; +static inline constexpr auto kInt8 = kS8; +static inline constexpr auto kUint8 = kU8; +static inline constexpr auto kUint8b128 = kU8B128; + +static inline constexpr auto kFloat4_e2m1f = kFE2M1f; +static inline constexpr auto kFloat6_e3m2f = kFE3M2f; +static inline constexpr auto kFloat8_e4m3fn = kFE4M3fn; +static inline constexpr auto kFloat8_e5m2 = kFE5M2; +static inline constexpr auto kFloat16_e8m7 = kFE8M7; +static inline constexpr auto kFloat16_e5m10 = kFE5M10; + +// colloquial names +static inline constexpr auto kHalf = kFE5M10; +static inline constexpr auto kFloat16 = kHalf; +static inline constexpr auto kBFloat16 = kFE8M7; + +static inline constexpr auto kFloat16Id = kFloat16.id(); +}; // namespace sglang diff --git a/sgl-kernel/include/sgl_flash_kernel_ops.h b/sgl-kernel/include/sgl_flash_kernel_ops.h new file mode 100644 index 000000000..383e207c3 --- /dev/null +++ b/sgl-kernel/include/sgl_flash_kernel_ops.h @@ -0,0 +1,86 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "sgl_kernel_torch_shim.h" + +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE) + +#define _CONCAT(A, B) A##B +#define CONCAT(A, B) _CONCAT(A, B) + +#define _STRINGIFY(A) #A +#define STRINGIFY(A) _STRINGIFY(A) + +#define REGISTER_EXTENSION(NAME) \ + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \ + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, STRINGIFY(NAME), nullptr, 0, nullptr}; \ + return PyModule_Create(&module); \ + } + +/* + * From flash-attention + */ +std::vector mha_fwd( + at::Tensor& q, // (b, s_q, h, d) or (total_q, h, d) if there is cu_seqlens_q + const at::Tensor& k, // (b_k, s_k, h_k, d) or (total_k, h_k, d) if there is cu_seqlens_k or (num_pages, page_size, + // h_k, d) if there is page_table. + const at::Tensor& v, // (b_k, s_k, h_k, dv) or (total_k, h_k, dv) if there is cu_seqlens_k or (num_pages, + // page_size, h_k, dv) if there is page_table. + std::optional& + k_new_, // (b, s_k_new, h_k, d) or (total_k_new, h_k, d) if there is cu_seqlens_k_new + std::optional& + v_new_, // (b, s_k_new, h_k, dv) or (total_k_new, h_k, dv) if there is cu_seqlens_k_new + std::optional& q_v_, // (b, s_q, h, dv) or (total_q_new, h, dv) if there is cu_seqlens_q + std::optional& out_, // (b, s_q, h, dv) or (total_q, h, dv) if there is cu_seqlens_q + std::optional& cu_seqlens_q_, // b+1 + std::optional& cu_seqlens_k_, // b+1 + std::optional& cu_seqlens_k_new_, // b+1 + std::optional& + seqused_q_, // b. If given, only this many elements of each batch element's queries and outputs are used. + std::optional& + seqused_k_, // b. If given, only this many elements of each batch element's keys are used. + std::optional max_seqlen_q_, + // TODO: check if we need max_seqlen_k + std::optional max_seqlen_k_, + std::optional& page_table_, // (b_k, max_num_pages_per_seq) + std::optional& kv_batch_idx_, // b. indices to index into the KV cache + std::optional& leftpad_k_, // b + std::optional& rotary_cos_, // seqlen_ro x (rotary_dim / 2) + std::optional& rotary_sin_, // seqlen_ro x (rotary_dim / 2) + std::optional& seqlens_rotary_, // b + std::optional& q_descale_, // (b, h_k), not (b, h) + std::optional& k_descale_, // (b, h_k) + std::optional& v_descale_, // (b, h_k) + float const softmax_scale, + bool is_causal, + int window_size_left, + int window_size_right, + float const softcap, + bool const is_rotary_interleaved, // if true, rotary combines indices 0 & 1, else indices 0 & rotary_dim / 2 + std::optional& scheduler_metadata_, // (b + 1) + int num_splits, + std::optional pack_gqa_, + int const sm_margin, + std::optional& sinks_); diff --git a/sgl-kernel/include/sgl_kernel_ops.h b/sgl-kernel/include/sgl_kernel_ops.h new file mode 100644 index 000000000..a13af546a --- /dev/null +++ b/sgl-kernel/include/sgl_kernel_ops.h @@ -0,0 +1,771 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "scalar_type.hpp" + +#define _CONCAT(A, B) A##B +#define CONCAT(A, B) _CONCAT(A, B) + +#define _STRINGIFY(A) #A +#define STRINGIFY(A) _STRINGIFY(A) + +#define TORCH_LIBRARY_EXPAND(NAME, MODULE) TORCH_LIBRARY(NAME, MODULE) + +#define REGISTER_EXTENSION(NAME) \ + PyMODINIT_FUNC CONCAT(PyInit_, NAME)() { \ + static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, STRINGIFY(NAME), nullptr, 0, nullptr}; \ + return PyModule_Create(&module); \ + } + +using fptr_t = int64_t; + +/* + * From csrc/allreduce + */ +#ifdef USE_ROCM +// ROCM custom allreduce +fptr_t init_custom_ar( + torch::Tensor& meta, + torch::Tensor& rank_data, + const std::vector& handles, + const std::vector& offsets, + int64_t rank, + bool full_nvlink); +void all_reduce_reg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out); +void all_reduce_unreg(fptr_t _fa, torch::Tensor& inp, torch::Tensor& reg_buffer, torch::Tensor& out); +void dispose(fptr_t _fa); +int64_t meta_size(); +void register_buffer( + fptr_t _fa, torch::Tensor& t, const std::vector& handles, const std::vector& offsets); +std::tuple> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_graph_buffers( + fptr_t _fa, const std::vector& handles, const std::vector>& offsets); +torch::Tensor allocate_meta_buffer(int64_t size); +torch::Tensor get_meta_buffer_ipc_handle(torch::Tensor& inp); +// quick allreduce +fptr_t init_custom_qr(int64_t rank, int64_t world_size, std::optional qr_max_size = std::nullopt); +void qr_destroy(fptr_t _fa); +torch::Tensor qr_get_handle(fptr_t _fa); +void qr_open_handles(fptr_t _fa, const std::vector& handles); +void qr_all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, int64_t quant_level, bool cast_bf2half = false); +int64_t qr_max_size(); +#else +// custom allreduce +fptr_t +init_custom_ar(const std::vector& fake_ipc_ptrs, torch::Tensor& rank_data, int64_t rank, bool full_nvlink); +void dispose(fptr_t _fa); +int64_t meta_size(); +void all_reduce(fptr_t _fa, torch::Tensor& inp, torch::Tensor& out, fptr_t _reg_buffer, int64_t reg_buffer_sz_bytes); +std::tuple, std::vector> get_graph_buffer_ipc_meta(fptr_t _fa); +void register_buffer(fptr_t _fa, const std::vector& fake_ipc_ptrs); +void register_graph_buffers( + fptr_t _fa, const std::vector>& handles, const std::vector>& offsets); + +// mscclpp +torch::Tensor mscclpp_generate_unique_id(); +fptr_t mscclpp_init_context( + const torch::Tensor& unique_id, + const int64_t rank, + const int64_t world_size, + torch::Tensor& scratch, + torch::Tensor& put_buffer, + const int64_t nranks_per_node, + const std::vector& rank_to_node, + const std::vector& rank_to_ib, + const int64_t context_selection); +void mscclpp_allreduce(fptr_t _context, torch::Tensor& inp, torch::Tensor& out, int64_t nthreads, int64_t nblocks); +#endif + +/* + * From csrc/attention + */ +void lightning_attention_decode( + const torch::Tensor& q, + const torch::Tensor& k, + const torch::Tensor& v, + const torch::Tensor& past_kv, + const torch::Tensor& slope, + torch::Tensor output, + torch::Tensor new_kv); +void merge_state( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged); +void merge_state_v2( + at::Tensor v_a, at::Tensor s_a, at::Tensor v_b, at::Tensor s_b, at::Tensor v_merged, at::Tensor s_merged); +void cutlass_mla_decode( + torch::Tensor const& out, + torch::Tensor const& q_nope, + torch::Tensor const& q_pe, + torch::Tensor const& kv_c_and_k_pe_cache, + torch::Tensor const& seq_lens, + torch::Tensor const& page_table, + torch::Tensor const& workspace, + double sm_scale, + int64_t num_kv_splits = 1 /* Set to 1 to avoid cuda_graph issue by default. */); +int64_t cutlass_mla_get_workspace_size( + int64_t max_seq_len, + int64_t num_batches, + int64_t sm_count = 0, + int64_t num_kv_splits = 1 /* Set to 1 to avoid cuda_graph issue by default. */); + +/* + * From csrc/elementwise + */ +void rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, bool enable_pdl); +void sgl_fused_add_rmsnorm( + torch::Tensor input, torch::Tensor residual, torch::Tensor weight, double eps, bool enable_pdl); +void gemma_rmsnorm(at::Tensor& output, at::Tensor& input, at::Tensor& weight, double eps, bool enable_pdl); +void gemma_fused_add_rmsnorm(at::Tensor& input, at::Tensor& residual, at::Tensor& weight, double eps, bool enable_pdl); +void silu_and_mul(at::Tensor& out, at::Tensor& input); +void gelu_tanh_and_mul(at::Tensor& out, at::Tensor& input); +void gelu_and_mul(at::Tensor& out, at::Tensor& input); + +void apply_rope_pos_ids_cos_sin_cache( + at::Tensor q, + at::Tensor k, + at::Tensor q_rope, + at::Tensor k_rope, + at::Tensor cos_sin_cache, + at::Tensor pos_ids, + bool interleave, + bool enable_pdl, + int64_t cuda_stream, + const std::optional& v, + const std::optional& k_buffer, + const std::optional& v_buffer, + const std::optional& kv_cache_loc); + +void downcast_fp8( + at::Tensor& k, + at::Tensor& v, + at::Tensor& k_out, + at::Tensor& v_out, + at::Tensor& k_scale, + at::Tensor& v_scale, + at::Tensor& loc, + int64_t mult, + int64_t offset, + int64_t cuda_stream); + +#ifdef USE_ROCM +void gelu_quick(at::Tensor& out, const at::Tensor& input); +#endif + +/* + * From csrc/gemm + */ +torch::Tensor awq_dequantize(torch::Tensor qweight, torch::Tensor scales, torch::Tensor qzeros); +void cutlass_scaled_fp4_mm( + torch::Tensor& D, + torch::Tensor const& A, + torch::Tensor const& B, + torch::Tensor const& A_sf, + torch::Tensor const& B_sf, + torch::Tensor const& alpha); +torch::Tensor int8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype, + const c10::optional& bias); +torch::Tensor fp8_blockwise_scaled_mm( + const torch::Tensor& mat_a, + const torch::Tensor& mat_b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Dtype& out_dtype); +void scaled_fp4_quant( + torch::Tensor& output, torch::Tensor const& input, torch::Tensor& output_scale, torch::Tensor const& input_scale); +void sgl_per_token_group_quant_fp8( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double fp8_min, + double fp8_max, + bool scale_ue8m0); +void sgl_per_token_group_quant_int8( + at::Tensor input, + at::Tensor output_q, + at::Tensor output_s, + int64_t group_size, + double eps, + double int8_min, + double int8_max); +void sgl_per_tensor_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s, bool is_static); +void sgl_per_token_quant_fp8(at::Tensor input, at::Tensor output_q, at::Tensor output_s); +void bmm_fp8( + at::Tensor A, + at::Tensor B, + at::Tensor D, + at::Tensor A_scale, + at::Tensor B_scale, + at::Tensor workspace_buffer, + int64_t cublas_handle, + int64_t cuda_stream); +void dsv3_router_gemm(torch::Tensor& output, const torch::Tensor& mat_a, const torch::Tensor& mat_b); +void dsv3_fused_a_gemm(torch::Tensor& output, torch::Tensor const& mat_a, torch::Tensor const& mat_b); + +torch::Tensor gptq_marlin_gemm( + torch::Tensor& a, + std::optional c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& global_scale_or_none, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float); + +torch::Tensor gptq_gemm( + torch::Tensor a, + torch::Tensor b_q_weight, + torch::Tensor b_gptq_qzeros, + torch::Tensor b_gptq_scales, + torch::Tensor b_g_idx, + bool use_shuffle, + int64_t bit); + +void gptq_shuffle(torch::Tensor q_weight, torch::Tensor q_perm, int64_t bit); + +torch::Tensor +gptq_marlin_repack(torch::Tensor& b_q_weight, torch::Tensor& perm, int64_t size_k, int64_t size_n, int64_t num_bits); + +torch::Tensor awq_marlin_repack(torch::Tensor& b_q_weight, int64_t size_k, int64_t size_n, int64_t num_bits); + +/* + * From csrc/moe + */ +void moe_align_block_size( + torch::Tensor topk_ids, + int64_t num_experts, + int64_t block_size, + torch::Tensor sorted_token_ids, + torch::Tensor experts_ids, + torch::Tensor num_tokens_post_pad, + torch::Tensor cumsum_buffer, + bool pad_sorted_token_ids); + +void topk_softmax( + torch::Tensor& topk_weights, torch::Tensor& topk_indices, torch::Tensor& gating_output, bool renormalize); + +std::vector moe_fused_gate( + at::Tensor& input, + at::Tensor& bias, + int64_t num_expert_group, + int64_t topk_group, + int64_t topk, + int64_t num_fused_shared_experts, + double routed_scaling_factor, + bool apply_routed_scaling_factor_on_output); + +void fp8_blockwise_scaled_grouped_mm( + torch::Tensor& output, + torch::Tensor& a_ptrs, + torch::Tensor& b_ptrs, + torch::Tensor& out_ptrs, + torch::Tensor& a_scales_ptrs, + torch::Tensor& b_scales_ptrs, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& scales_a, + const torch::Tensor& scales_b, + const torch::Tensor& stride_a, + const torch::Tensor& stride_b, + const torch::Tensor& stride_c, + const torch::Tensor& layout_sfa, + const torch::Tensor& layout_sfb, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& workspace); + +void prepare_moe_input( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + const std::optional& blockscale_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void shuffle_rows(const torch::Tensor& input_tensor, const torch::Tensor& dst2src_map, torch::Tensor& output_tensor); + +void apply_shuffle_mul_sum( + const torch::Tensor& input, + torch::Tensor& output, + const torch::Tensor& permutation, + const std::optional& factors); + +void cutlass_fp4_group_mm( + torch::Tensor& output, + const torch::Tensor& a, + const torch::Tensor& b, + const torch::Tensor& a_blockscale, + const torch::Tensor& b_blockscales, + const torch::Tensor& alphas, + const torch::Tensor& ab_strides, + const torch::Tensor& c_strides, + const torch::Tensor& problem_sizes, + const torch::Tensor& expert_offsets, + const torch::Tensor& sf_offsets); + +void scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& input_offset_by_experts, + torch::Tensor const& output_scale_offset_by_experts); + +void silu_and_mul_scaled_fp4_experts_quant( + torch::Tensor& output, + torch::Tensor& output_scale, + torch::Tensor const& input, + torch::Tensor const& input_global_scale, + torch::Tensor const& mask, + bool use_silu_and_mul); +/* + * From csrc/moe/cutlass_moe/w4a8 + */ +void get_cutlass_w4a8_moe_mm_data( + const torch::Tensor& topk_ids, + torch::Tensor& expert_offsets, + torch::Tensor& problem_sizes1, + torch::Tensor& problem_sizes2, + torch::Tensor& input_permutation, + torch::Tensor& output_permutation, + const int64_t num_experts, + const int64_t n, + const int64_t k); + +void cutlass_w4a8_moe_mm( + torch::Tensor& d_tensors, + torch::Tensor const& a_tensors, + torch::Tensor const& b_tensors, + torch::Tensor const& a_scales, + torch::Tensor const& b_scales, + torch::Tensor const& expert_offsets, + torch::Tensor const& problem_sizes, + torch::Tensor const& a_strides, + torch::Tensor const& b_strides, + torch::Tensor const& d_strides, + torch::Tensor const& s_strides, + int64_t chunk_size, + int64_t topk); + +torch::Tensor moe_wna16_marlin_gemm( + torch::Tensor& a, + std::optional const& c_or_none, + torch::Tensor& b_q_weight, + torch::Tensor& b_scales, + std::optional const& b_zeros_or_none, + std::optional const& g_idx_or_none, + std::optional const& perm_or_none, + torch::Tensor& workspace, + torch::Tensor& sorted_token_ids, + torch::Tensor& expert_ids, + torch::Tensor& num_tokens_past_padded, + torch::Tensor& topk_weights, + int64_t moe_block_size, + int64_t top_k, + bool mul_topk_weights, + bool is_ep, + sglang::ScalarTypeId const& b_q_type_id, + int64_t size_m, + int64_t size_n, + int64_t size_k, + bool is_k_full, + bool use_atomic_add, + bool use_fp32_reduce, + bool is_zp_float); + +/* + * From csrc/speculative + */ +void tree_speculative_sampling_target_only( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor uniform_samples, + at::Tensor uniform_samples_for_final_sampling, + at::Tensor target_probs, + at::Tensor draft_probs, + double threshold_single = 1, + double threshold_acc = 1, + bool deterministic = true, + int64_t cuda_stream = 0); + +void verify_tree_greedy( + at::Tensor predicts, // mutable + at::Tensor accept_index, // mutable + at::Tensor accept_token_num, // mutable + at::Tensor candidates, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + at::Tensor target_predict, + int64_t cuda_stream = 0); + +void build_tree_kernel_efficient( + at::Tensor parent_list, + at::Tensor selected_index, + at::Tensor verified_seq_len, + at::Tensor tree_mask, + at::Tensor positions, + at::Tensor retrive_index, + at::Tensor retrive_next_token, + at::Tensor retrive_next_sibling, + int64_t topk, + int64_t depth, + int64_t draft_token_num, + int64_t tree_mask_mode); + +void segment_packbits( + at::Tensor x, + at::Tensor input_indptr, + at::Tensor output_indptr, + at::Tensor y, + int64_t batch_size, + int64_t cuda_stream = 0); + +/* + * From csrc/kvcacheio + */ +void transfer_kv_per_layer( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_pf_lf( + const at::Tensor src_k, + at::Tensor dst_k, + const at::Tensor src_v, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer( + const at::Tensor src_k_layers, + const at::Tensor dst_k_layers, + const at::Tensor src_v_layers, + const at::Tensor dst_v_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_lf_pf( + const at::Tensor src_k_layers, + at::Tensor dst_k, + const at::Tensor src_v_layers, + at::Tensor dst_v, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_mla( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_per_layer_mla_pf_lf( + const at::Tensor src, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t layer_id, + int64_t item_size, + int64_t src_layout_dim, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_mla( + const at::Tensor src_layers, + const at::Tensor dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_all_layer_mla_lf_pf( + const at::Tensor src_layers, + at::Tensor dst, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t item_size, + int64_t dst_layout_dim, + int64_t num_layers, + int64_t block_quota, + int64_t num_warps_per_block); + +void transfer_kv_direct( + const std::vector& src_layers, + std::vector dst_layers, + const at::Tensor src_indices, + const at::Tensor dst_indices, + int64_t page_size); + +void transfer_kv_per_layer_direct_pf_lf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t layer_id, + int64_t page_size); + +void transfer_kv_all_layer_direct_lf_pf( + const std::vector& src_ptrs, + std::vector dst_ptrs, + const at::Tensor& src_indices, + const at::Tensor& dst_indices, + int64_t page_size); + +/* + * From FlashInfer + */ +void min_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_min_p_arr, + double min_p_val, + bool deterministic, + std::optional gen); + +void top_k_renorm_probs( + at::Tensor probs, at::Tensor renorm_probs, std::optional maybe_top_k_arr, int64_t top_k_val); + +void top_p_renorm_probs( + at::Tensor probs, at::Tensor renorm_probs, std::optional maybe_top_p_arr, double top_p_val); + +void top_k_top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_top_k_arr, + double top_k_val, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + std::optional gen); + +void top_p_sampling_from_probs( + at::Tensor probs, + at::Tensor output, + std::optional maybe_indices, + std::optional maybe_top_p_arr, + double top_p_val, + bool deterministic, + std::optional gen); + +void top_k_mask_logits( + at::Tensor logits, at::Tensor mask_logits, std::optional maybe_top_k_arr, int64_t top_k_val); + +namespace flash { +/* + * From fa2 sparse + */ +std::vector mha_fwd_sparse( + at::Tensor& q, // batch_size x seqlen_q x num_heads x head_size + const at::Tensor& k, // batch_size x seqlen_k x num_heads_k x head_size + const at::Tensor& v, // batch_size x seqlen_k x num_heads_k x head_size + const at::Tensor& block_count, + const at::Tensor& block_offset, + const at::Tensor& column_count, + const at::Tensor& column_index, + const std::optional& out_, // batch_size x seqlen_q x num_heads x head_size + const std::optional& alibi_slopes_, // num_heads or batch_size x num_heads + const double p_dropout, + const double softmax_scale, + bool is_causal, + const double softcap, + const bool return_softmax, + std::optional gen_); + +std::vector mha_varlen_fwd_sparse( + at::Tensor& q, // total_q x num_heads x head_size, total_q := \sum_{i=0}^{b} s_i + const at::Tensor& k, // total_k x num_heads_k x head_size, total_k := \sum_{i=0}^{b} s_i. + const at::Tensor& v, // total_k x num_heads_k x head_size, total_k := \sum_{i=0}^{b} s_i. + const at::Tensor& block_count, + const at::Tensor& block_offset, + const at::Tensor& column_count, + const at::Tensor& column_index, + const c10::optional& out_, // total_q x num_heads x head_size, total_k := \sum_{i=0}^{b} s_i + const at::Tensor& cu_seqlens_q, // b+1 + const at::Tensor& cu_seqlens_k, // b+1 + const c10::optional& + seqused_k, // b. If given, only this many elements of each batch element's keys are used. + const c10::optional& alibi_slopes_, // num_heads or b x num_heads + int64_t max_seqlen_q, + const int64_t max_seqlen_k, + const double p_dropout, + const double softmax_scale, + const bool zero_tensors, + bool is_causal, + const double softcap, + const bool return_softmax, + c10::optional gen_); +} // namespace flash + +void convert_vertical_slash_indexes( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal); + +void convert_vertical_slash_indexes_mergehead( + torch::Tensor& block_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& block_offset, // [BATCH, N_HEADS, NUM_ROWS, NNZ_S] + torch::Tensor& column_count, // [BATCH, N_HEADS, NUM_ROWS] + torch::Tensor& column_index, // [BATCH, N_HEADS, NUM_ROWS, NNZ_V] + torch::Tensor q_seqlens, // [BATCH, ] + torch::Tensor kv_seqlens, // [BATCH, ] + torch::Tensor vertical_indexes, // [BATCH, N_HEADS, NNZ_V] + torch::Tensor slash_indexes, // [BATCH, N_HEADS, NNZ_S] + torch::Tensor vertical_indices_count, // [N_HEADS, ] + torch::Tensor slash_indices_count, + int64_t context_size, + int64_t block_size_M, + int64_t block_size_N, + bool causal); + +/* + * From XGrammar + */ +void ApplyTokenBitmaskInplace(at::Tensor logits, at::Tensor bitmask, at::optional indices = at::nullopt); + +/* + * From QServe + */ +void qserve_w4a8_per_chn_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + const torch::Tensor& _w_szs, + const torch::Tensor& _a_ssums, + torch::Tensor& _out_feats); + +void qserve_w4a8_per_group_gemm( + const torch::Tensor& _in_feats, + const torch::Tensor& _kernel, + const torch::Tensor& _zeros, + const torch::Tensor& _scales_i8, + const torch::Tensor& _wscales, + const torch::Tensor& _ascales, + torch::Tensor& _out_feats); + +/* + * From csrc/spatial + */ +std::vector create_greenctx_stream_by_value(int64_t smA, int64_t smB, int64_t device); + +/* + * From csrc/memory + */ +void store_kv_cache(at::Tensor k_cache, at::Tensor v_cache, at::Tensor out_loc, at::Tensor k, at::Tensor v); + +void copy_to_gpu_no_ce(const at::Tensor& input, at::Tensor& output); +void concat_mla_k(torch::Tensor k, torch::Tensor k_nope, torch::Tensor k_rope); + +/* + * From csrc/mamba + */ +void causal_conv1d_update( + const at::Tensor& x, + const at::Tensor& conv_state, + const at::Tensor& weight, + const std::optional& bias_, + bool silu_activation, + const std::optional& cache_seqlens_, + const std::optional& conv_state_indices_, + int64_t pad_slot_id); + +void causal_conv1d_fwd( + const at::Tensor& x, + const at::Tensor& weight, + const std::optional& bias_, + const std::optional& conv_states, + const std::optional& query_start_loc, + const std::optional& cache_indices, + const std::optional& has_initial_state, + bool silu_activation, + int64_t pad_slot_id); diff --git a/sgl-kernel/include/sgl_kernel_torch_shim.h b/sgl-kernel/include/sgl_kernel_torch_shim.h new file mode 100644 index 000000000..5a26f9076 --- /dev/null +++ b/sgl-kernel/include/sgl_kernel_torch_shim.h @@ -0,0 +1,122 @@ +/*Adapt from: +https://github.com/neuralmagic/vllm-flash-attention/blob/90eacc1af2a7c3de62ea249e929ed5faccf38954/csrc/common/pytorch_shim.h + Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include + +/** + * Unfortunately, the type signatures of the flash_attn ops are not compatible + * with the PyTorch library bindings. To get around that we use + * `make_pytorch_shim` which creates a lambda that exposes the API using + * PyTorch compatible types to the types, then converts them to the types + * expected by the flash_attn ops. This shims allows us to make minimal changes + * to `flash_api.cpp` making it easier to synchronize with upstream changes. + * + * The `pytorch_library_compatible_type` struct is used to map from the + * flash_attn ops types to a PyTorch library compatible one. The main issues is + * that the following types are not support by PyTorch library bindings: + * - `int` + * - `float` + * - `std::optional &` + * - `std::optional &` + * So we convert them to (respectively): + * - `int64_t` + * - `double` + * - `const std::optional&` + * - `const std::optional&` + */ + +template +struct pytorch_library_compatible_type { + using type = T; + static T convert_from_type(T arg) { + return arg; + } +}; + +template +using pytorch_library_compatible_type_t = typename pytorch_library_compatible_type::type; + +template +T convert_from_pytorch_compatible_type(pytorch_library_compatible_type_t arg) { + return pytorch_library_compatible_type::convert_from_type(arg); +} + +// Map `c10::optional &` -> `const c10::optional&` +// (NOTE: this is bit unsafe but non of the ops in flash_attn mutate +// the optional container) +template +struct pytorch_library_compatible_type&> { + using type = const c10::optional&; + static c10::optional& convert_from_type(const c10::optional& arg) { + return const_cast&>(arg); + } +}; + +// Map `c10::optional` -> +// `c10::optional>` +// (NOTE: tested for `c10::optional` -> `c10::optional`) +template +struct pytorch_library_compatible_type> { + using type = c10::optional>; + static c10::optional> convert_from_type(c10::optional arg) { + return arg; + } +}; + +// Map `c10::optional&` -> `const c10::optional&` +template <> +struct pytorch_library_compatible_type&> { + using type = const c10::optional&; + static c10::optional& convert_from_type(const c10::optional& arg) { + return const_cast&>(reinterpret_cast&>(arg)); + } +}; + +// Map `int` -> `int64_t` +template <> +struct pytorch_library_compatible_type { + using type = int64_t; + static int convert_from_type(int64_t arg) { + TORCH_CHECK(arg <= std::numeric_limits::max(), "int64_t value is too large to be converted to int"); + TORCH_CHECK(arg >= std::numeric_limits::min(), "int64_t value is too small to be converted to int"); + return arg; + } +}; + +// Map `float` -> `double` +template <> +struct pytorch_library_compatible_type { + using type = double; + static float convert_from_type(double arg) { + TORCH_CHECK( + std::abs(arg) <= std::numeric_limits::max(), "double value is too large to be converted to float"); + return arg; + } +}; + +// +// Shim Utils +// + +template +auto make_pytorch_shim(Ret (*fun)(Args... args)) { + return [fun](pytorch_library_compatible_type_t... args) { + return fun(convert_from_pytorch_compatible_type(args)...); + }; +} diff --git a/sgl-kernel/include/utils.h b/sgl-kernel/include/utils.h new file mode 100644 index 000000000..7cfef8136 --- /dev/null +++ b/sgl-kernel/include/utils.h @@ -0,0 +1,449 @@ +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include + +#ifdef USE_ROCM +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = __half; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = __hip_bfloat16; \ + return __VA_ARGS__(); \ + } +#endif // USE_ROCM + +#ifndef USE_ROCM +// Adapt from FlashInfer +#ifdef FLASHINFER_ENABLE_F16 +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = nv_half; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_F16(c_type, ...) +#endif // FLASHINFER_ENABLE_F16 + +#ifdef FLASHINFER_ENABLE_BF16 +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = nv_bfloat16; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_BF16(c_type, ...) +#endif // FLASHINFER_ENABLE_BF16 + +#ifdef FLASHINFER_ENABLE_FP8_E4M3 +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) \ + case at::ScalarType::Float8_e4m3fn: { \ + using c_type = __nv_fp8_e4m3; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E4M3 + +#ifdef FLASHINFER_ENABLE_FP8_E5M2 +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) \ + case at::ScalarType::Float8_e5m2: { \ + using c_type = __nv_fp8_e5m2; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E5M2 + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch fp8 data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH(var_name, cond, ...) \ + [&]() -> bool { \ + switch (cond) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch " var_name " " << int(cond); \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH_U16x2(var1_name, var2_name, cond1, cond2, ...) \ + [&]() -> bool { \ + switch (pack_u16(cond1, cond2)) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch (" var1_name ", " var2_name "): (" << int(cond1) << ", " \ + << int(cond2) << ")"; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_CASE(case_expr, case_var, ...) \ + case case_expr: { \ + constexpr auto case_var = case_expr; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_U16x2(case_expr1, case_expr2, case_var1, case_var2, ...) \ + case pack_u16(case_expr1, case_expr2): { \ + constexpr auto case_var1 = case_expr1; \ + constexpr auto case_var2 = case_expr2; \ + return __VA_ARGS__(); \ + } + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +inline void check_shape(const at::Tensor& a, const at::Tensor& b, const char* a_name, const char* b_name) { + TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ", a.dim(), " vs ", b.dim()); + for (int i = 0; i < a.dim(); ++i) { + TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name, ".size(", i, ")"); + } +} + +inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) { + return (uint32_t(a) << 16) | uint32_t(b); +} + +#define CHECK_GQA_HEAD_DIVISIBLE(num_qo_heads, num_kv_heads) \ + TORCH_CHECK( \ + num_qo_heads % num_kv_heads == 0, \ + "num_qo_heads(", \ + num_qo_heads, \ + ") must be divisible by num_kv_heads(", \ + num_kv_heads, \ + ")") + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b) + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) + +inline bool is_float8_tensor(const at::Tensor& tensor) { + return tensor.scalar_type() == at::ScalarType::Float8_e4m3fn || tensor.scalar_type() == at::ScalarType::Float8_e5m2; +} +#endif // USE_ROCM + +struct cuda_error : public std::runtime_error { + /** + * @brief Constructs a `cuda_error` object with the given `message`. + * + * @param message The error char array used to construct `cuda_error` + */ + cuda_error(const char* message) : std::runtime_error(message) {} + /** + * @brief Constructs a `cuda_error` object with the given `message` string. + * + * @param message The `std::string` used to construct `cuda_error` + */ + cuda_error(std::string const& message) : cuda_error{message.c_str()} {} +}; + +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + cudaError_t e = cmd; \ + if (e != cudaSuccess) { \ + std::stringstream _message; \ + auto s = cudaGetErrorString(e); \ + _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__; \ + throw cuda_error(_message.str()); \ + } \ + } while (0) + +#define CHECK_IS_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_IS_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_CUDA_INPUT(x) \ + CHECK_IS_CUDA(x); \ + CHECK_IS_CONTIGUOUS(x) + +inline int getSMVersion() { + int device{-1}; + CHECK_CUDA_SUCCESS(cudaGetDevice(&device)); + int sm_major = 0; + int sm_minor = 0; + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_major, cudaDevAttrComputeCapabilityMajor, device)); + CHECK_CUDA_SUCCESS(cudaDeviceGetAttribute(&sm_minor, cudaDevAttrComputeCapabilityMinor, device)); + return sm_major * 10 + sm_minor; +} + +inline bool isDeviceType(const std::string& device_type) { + int deviceCount; + CHECK_CUDA_SUCCESS(cudaGetDeviceCount(&deviceCount)); + + int device_id = -1; + if (deviceCount >= 1) { + CHECK_CUDA_SUCCESS(cudaGetDevice(&device_id)); + } else { + return false; + } + + cudaDeviceProp prop; + CHECK_CUDA_SUCCESS(cudaGetDeviceProperties(&prop, device_id)); + if (device_type == std::string(prop.name)) { + return true; + } + return false; +} + +inline bool getBoolEnv(char const* name) { + char const* env = std::getenv(name); + return env && env[0] == '1' && env[1] == '\0'; +} + +inline bool getEnvEnablePDL() { + static std::once_flag flag; + static bool enablePDL = false; + std::call_once(flag, [&]() { + if (getSMVersion() >= 90) { + // PDL will be enabled by setting the env variables `TRTLLM_ENABLE_PDL` to `1` + enablePDL = getBoolEnv("TRTLLM_ENABLE_PDL"); + } + }); + return enablePDL; +} + +// SGLANG_SHFL_XOR_* adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/cuda_compat.h#L19-L28 +#ifndef USE_ROCM +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width)) +#else +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor((var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor((var), (lane_mask), (width)) +#endif + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + case at::ScalarType::Float: { \ + using c_type = float; \ + return __VA_ARGS__(); \ + } \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_CASE_INTEGRAL_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) + +#define DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) + +#define CEILDIV(x, y) (((x) + (y) - 1) / (y)) + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#else +#if defined(__GFX9__) || !defined(__HIP_DEVICE_COMPILE__) +#define WARP_SIZE 64 +#else +#define WARP_SIZE 32 +#endif +#endif + +#ifdef USE_ROCM + +#include "hip/hip_math_def.h" +#include "hip/hip_vec_dtypes.h" + +#else + +template +__device__ __forceinline__ float castToFloat(srcDtype val) { + return static_cast(val); +} + +template +__device__ __forceinline__ dstDtype castFromFloat(float val) { + return static_cast(val); +} + +#endif + +// add FP8 support +// #ifndef USE_ROCM +// #include +// using FP8_TYPE = c10::Float8_e4m3fn; +// C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +// #else // USE_ROCM +// #if HIP_FP8_TYPE_FNUZ +// #include +// using FP8_TYPE = c10::Float8_e4m3fnuz; +// constexpr auto FP8_E4M3_MAX = 224.0f; +// #else +// #if HIP_FP8_TYPE_E4M3 +// #include +// using FP8_TYPE = c10::Float8_e4m3fn; +// C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +// #else +// #error "fp8 is not supported in this processor (arch < gfx942)." +// #endif // HIP_FP8_TYPE_E4M3 +// #endif // HIP_FP8_TYPE_FNUZ +// #endif // USE_ROCM + +#define FULL_MASK 0xffffffff + +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) { +#ifndef USE_ROCM + float old; + old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) + : __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value))); + return old; +#else + int* addr_as_i = (int*)addr; + int old = *addr_as_i, assumed; + do { + assumed = old; + old = atomicCAS(addr_as_i, assumed, __float_as_int(fmaxf(value, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +#endif +} + +__device__ __forceinline__ float warpReduceMax(float value) { + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 16)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 8)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 4)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 2)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 1)); + return value; +} + +__device__ __forceinline__ float blockReduceMax(float value) { + static __shared__ float warpLevelMaxs[WARP_SIZE]; + const int laneId = threadIdx.x % WARP_SIZE; + const int warpId = threadIdx.x / WARP_SIZE; + + value = warpReduceMax(value); + + if (laneId == 0) warpLevelMaxs[warpId] = value; + __syncthreads(); + + value = (threadIdx.x < blockDim.x / WARP_SIZE) ? warpLevelMaxs[laneId] : 0; + if (warpId == 0) value = warpReduceMax(value); + + return value; +} + +// Pads to a multiple of `alignment` rows. +inline torch::Tensor pad_tensor(const torch::Tensor& tensor, int64_t alignment = 4, bool is_column_major = false) { + int64_t rows = tensor.size(0); + int64_t cols = tensor.size(1); + int64_t pad_rows = (alignment - (rows % alignment)) % alignment; // Compute padding size + + if (pad_rows == 0) { + return tensor; // Already aligned + } + + torch::Tensor padding = torch::zeros({pad_rows, cols}, tensor.options()); + torch::Tensor tensor_padded = torch::cat({tensor, padding}, 0); // Pad along rows + + // Ensure column-major layout + if (is_column_major) { + return tensor_padded.t().contiguous().t(); + } + return tensor_padded; +} + +// Get the next power of 2 of a number +inline uint32_t next_pow2(uint32_t x) noexcept { + if (x <= 1) return 1; + return 1u << (32 - __builtin_clz(x - 1)); +} diff --git a/sgl-kernel/include/utils_hip.h b/sgl-kernel/include/utils_hip.h new file mode 100644 index 000000000..db1056b0f --- /dev/null +++ b/sgl-kernel/include/utils_hip.h @@ -0,0 +1,451 @@ +// !!! This is a file automatically generated by hipify!!! +#include +/* Copyright 2025 SGLang Team. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include + +#ifdef USE_ROCM +// Adapted from flashinfer-rocm [PR#491](https://github.com/flashinfer-ai/flashinfer/pull/491) +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = __half; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = __hip_bfloat16; \ + return __VA_ARGS__(); \ + } +#endif // USE_ROCM + +#ifndef USE_ROCM +// Adapt from FlashInfer +#ifdef FLASHINFER_ENABLE_F16 +#define _DISPATCH_CASE_F16(c_type, ...) \ + case at::ScalarType::Half: { \ + using c_type = nv_half; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_F16(c_type, ...) +#endif // FLASHINFER_ENABLE_F16 + +#ifdef FLASHINFER_ENABLE_BF16 +#define _DISPATCH_CASE_BF16(c_type, ...) \ + case at::ScalarType::BFloat16: { \ + using c_type = nv_bfloat16; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_BF16(c_type, ...) +#endif // FLASHINFER_ENABLE_BF16 + +#ifdef FLASHINFER_ENABLE_FP8_E4M3 +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) \ + case at::ScalarType::Float8_e4m3fn: { \ + using c_type = __nv_fp8_e4m3; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E4M3(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E4M3 + +#ifdef FLASHINFER_ENABLE_FP8_E5M2 +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) \ + case at::ScalarType::Float8_e5m2: { \ + using c_type = __nv_fp8_e5m2; \ + return __VA_ARGS__(); \ + } +#else +#define _DISPATCH_CASE_FP8_E5M2(c_type, ...) +#endif // FLASHINFER_ENABLE_FP8_E5M2 + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FP8(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch fp8 data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E4M3(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_FP8_E5M2(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH(var_name, cond, ...) \ + [&]() -> bool { \ + switch (cond) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch " var_name " " << int(cond); \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_SWITCH_U16x2(var1_name, var2_name, cond1, cond2, ...) \ + [&]() -> bool { \ + switch (pack_u16(cond1, cond2)) { \ + __VA_ARGS__ \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch (" var1_name ", " var2_name "): (" << int(cond1) << ", " \ + << int(cond2) << ")"; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define _DISPATCH_CASE(case_expr, case_var, ...) \ + case case_expr: { \ + constexpr auto case_var = case_expr; \ + return __VA_ARGS__(); \ + } + +#define _DISPATCH_CASE_U16x2(case_expr1, case_expr2, case_var1, case_var2, ...) \ + case pack_u16(case_expr1, case_expr2): { \ + constexpr auto case_var1 = case_expr1; \ + constexpr auto case_var2 = case_expr2; \ + return __VA_ARGS__(); \ + } + +#define DISPATCH_BOOL(expr, const_expr, ...) \ + [&]() -> bool { \ + if (expr) { \ + constexpr bool const_expr = true; \ + return __VA_ARGS__(); \ + } else { \ + constexpr bool const_expr = false; \ + return __VA_ARGS__(); \ + } \ + }() + +inline void check_shape(const at::Tensor& a, const at::Tensor& b, const char* a_name, const char* b_name) { + TORCH_CHECK(a.dim() == b.dim(), a_name, ".dim() != ", b_name, ".dim(). ", a.dim(), " vs ", b.dim()); + for (int i = 0; i < a.dim(); ++i) { + TORCH_CHECK(a.size(i) == b.size(i), a_name, ".size(", i, ") != ", b_name, ".size(", i, ")"); + } +} + +inline constexpr uint32_t pack_u16(uint16_t a, uint16_t b) { + return (uint32_t(a) << 16) | uint32_t(b); +} + +#define CHECK_GQA_HEAD_DIVISIBLE(num_qo_heads, num_kv_heads) \ + TORCH_CHECK( \ + num_qo_heads % num_kv_heads == 0, \ + "num_qo_heads(", \ + num_qo_heads, \ + ") must be divisible by num_kv_heads(", \ + num_kv_heads, \ + ")") + +#define CHECK_CUDA(x) TORCH_CHECK(x.is_cuda(), #x " must be a CUDA tensor") + +#define CHECK_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_LAST_DIM_CONTIGUOUS(x) \ + TORCH_CHECK(x.strides()[x.strides().size() - 1] == 1, #x "must be contiguous at last dimension") + +#define CHECK_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_CONTIGUOUS(x) +#define CHECK_LAST_DIM_CONTIGUOUS_INPUT(x) \ + CHECK_CUDA(x); \ + CHECK_LAST_DIM_CONTIGUOUS(x) + +#define CHECK_DIM(d, x) TORCH_CHECK(x.dim() == d, #x " must be a " #d "D tensor") + +#define CHECK_SHAPE(a, b) check_shape(a, b, #a, #b) + +#define CHECK_EQ(a, b) TORCH_CHECK((a) == (b), "CHECK_EQ(" #a ", " #b ") failed. ", a, " vs ", b) + +#define CHECK_GE(a, b) TORCH_CHECK((a) >= (b), "CHECK_GE(" #a ", " #b ") failed. ", a, " vs ", b) + +inline bool is_float8_tensor(const at::Tensor& tensor) { + return tensor.scalar_type() == at::ScalarType::Float8_e4m3fn || tensor.scalar_type() == at::ScalarType::Float8_e5m2; +} +#endif // USE_ROCM + +struct cuda_error : public std::runtime_error { + /** + * @brief Constructs a `cuda_error` object with the given `message`. + * + * @param message The error char array used to construct `cuda_error` + */ + cuda_error(const char* message) : std::runtime_error(message) {} + /** + * @brief Constructs a `cuda_error` object with the given `message` string. + * + * @param message The `std::string` used to construct `cuda_error` + */ + cuda_error(std::string const& message) : cuda_error{message.c_str()} {} +}; + +#define CHECK_CUDA_SUCCESS(cmd) \ + do { \ + hipError_t e = cmd; \ + if (e != hipSuccess) { \ + std::stringstream _message; \ + auto s = hipGetErrorString(e); \ + _message << std::string(s) + "\n" << __FILE__ << ':' << __LINE__; \ + throw cuda_error(_message.str()); \ + } \ + } while (0) + +#define CHECK_IS_CUDA(x) TORCH_CHECK(x.device().is_cuda(), #x " must be a CUDA tensor") +#define CHECK_IS_CONTIGUOUS(x) TORCH_CHECK(x.is_contiguous(), #x " must be contiguous") +#define CHECK_CUDA_INPUT(x) \ + CHECK_IS_CUDA(x); \ + CHECK_IS_CONTIGUOUS(x) + +inline int getSMVersion() { + int device{-1}; + CHECK_CUDA_SUCCESS(hipGetDevice(&device)); + int sm_major = 0; + int sm_minor = 0; + CHECK_CUDA_SUCCESS(hipDeviceGetAttribute(&sm_major, hipDeviceAttributeComputeCapabilityMajor, device)); + CHECK_CUDA_SUCCESS(hipDeviceGetAttribute(&sm_minor, hipDeviceAttributeComputeCapabilityMinor, device)); + return sm_major * 10 + sm_minor; +} + +inline bool isDeviceType(const std::string& device_type) { + int deviceCount; + CHECK_CUDA_SUCCESS(hipGetDeviceCount(&deviceCount)); + + int device_id = -1; + if (deviceCount >= 1) { + CHECK_CUDA_SUCCESS(hipGetDevice(&device_id)); + } else { + return false; + } + + hipDeviceProp_t prop; + CHECK_CUDA_SUCCESS(hipGetDeviceProperties(&prop, device_id)); + if (device_type == std::string(prop.name)) { + return true; + } + return false; +} + +inline bool getBoolEnv(char const* name) { + char const* env = std::getenv(name); + return env && env[0] == '1' && env[1] == '\0'; +} + +inline bool getEnvEnablePDL() { + static std::once_flag flag; + static bool enablePDL = false; + std::call_once(flag, [&]() { + if (getSMVersion() >= 90) { + // PDL will be enabled by setting the env variables `TRTLLM_ENABLE_PDL` to `1` + enablePDL = getBoolEnv("TRTLLM_ENABLE_PDL"); + } + }); + return enablePDL; +} + +// SGLANG_SHFL_XOR_* adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/csrc/cuda_compat.h#L19-L28 +#ifndef USE_ROCM +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor_sync((mask), (var), (lane_mask), (width)) +#else +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor((var), (lane_mask)) +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) __shfl_xor((var), (lane_mask), (width)) +#endif + +#define DISPATCH_PYTORCH_DTYPE_TO_CTYPE_FLOAT_FP16(pytorch_dtype, c_type, ...) \ + [&]() -> bool { \ + switch (pytorch_dtype) { \ + case at::ScalarType::Float: { \ + using c_type = float; \ + return __VA_ARGS__(); \ + } \ + _DISPATCH_CASE_F16(c_type, __VA_ARGS__) \ + _DISPATCH_CASE_BF16(c_type, __VA_ARGS__) \ + default: \ + std::ostringstream oss; \ + oss << __PRETTY_FUNCTION__ << " failed to dispatch data type " << pytorch_dtype; \ + TORCH_CHECK(false, oss.str()); \ + return false; \ + } \ + }() + +#define DISPATCH_CASE_INTEGRAL_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Byte, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Char, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Short, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Int, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Long, __VA_ARGS__) + +#define DISPATCH_INTEGRAL_TYPES(TYPE, NAME, ...) \ + AT_DISPATCH_SWITCH(TYPE, NAME, DISPATCH_CASE_INTEGRAL_TYPES(__VA_ARGS__)) + +#define CEILDIV(x, y) (((x) + (y) - 1) / (y)) + +#ifndef USE_ROCM +#define WARP_SIZE 32 +#else +#if defined(__GFX9__) || !defined(__HIP_DEVICE_COMPILE__) +#define WARP_SIZE 64 +#else +#define WARP_SIZE 32 +#endif +#endif + +#ifdef USE_ROCM + +#include "hip/hip_math_def.h" +#include "hip/hip_vec_dtypes.h" + +#else + +template +__device__ __forceinline__ float castToFloat(srcDtype val) { + return static_cast(val); +} + +template +__device__ __forceinline__ dstDtype castFromFloat(float val) { + return static_cast(val); +} + +#endif + +// add FP8 support +// #ifndef USE_ROCM +// #include +// using FP8_TYPE = c10::Float8_e4m3fn; +// C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +// #else // USE_ROCM +// #if HIP_FP8_TYPE_FNUZ +// #include +// using FP8_TYPE = c10::Float8_e4m3fnuz; +// constexpr auto FP8_E4M3_MAX = 224.0f; +// #else +// #if HIP_FP8_TYPE_E4M3 +// #include +// using FP8_TYPE = c10::Float8_e4m3fn; +// C10_HOST_DEVICE constexpr auto FP8_E4M3_MAX = std::numeric_limits::max(); +// #else +// #error "fp8 is not supported in this processor (arch < gfx942)." +// #endif // HIP_FP8_TYPE_E4M3 +// #endif // HIP_FP8_TYPE_FNUZ +// #endif // USE_ROCM + +#define FULL_MASK 0xffffffff + +__device__ __forceinline__ float atomicMaxFloat(float* addr, float value) { +#ifndef USE_ROCM + float old; + old = (value >= 0) ? __int_as_float(atomicMax((int*)addr, __float_as_int(value))) + : __uint_as_float(atomicMin((unsigned int*)addr, __float_as_uint(value))); + return old; +#else + int* addr_as_i = (int*)addr; + int old = *addr_as_i, assumed; + do { + assumed = old; + old = atomicCAS(addr_as_i, assumed, __float_as_int(fmaxf(value, __int_as_float(assumed)))); + } while (assumed != old); + return __int_as_float(old); +#endif +} + +__device__ __forceinline__ float warpReduceMax(float value) { + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 16)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 8)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 4)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 2)); + value = fmaxf(value, __shfl_xor_sync(FULL_MASK, value, 1)); + return value; +} + +__device__ __forceinline__ float blockReduceMax(float value) { + static __shared__ float warpLevelMaxs[WARP_SIZE]; + const int laneId = threadIdx.x % WARP_SIZE; + const int warpId = threadIdx.x / WARP_SIZE; + + value = warpReduceMax(value); + + if (laneId == 0) warpLevelMaxs[warpId] = value; + __syncthreads(); + + value = (threadIdx.x < blockDim.x / WARP_SIZE) ? warpLevelMaxs[laneId] : 0; + if (warpId == 0) value = warpReduceMax(value); + + return value; +} + +// Pads to a multiple of `alignment` rows. +inline torch::Tensor pad_tensor(const torch::Tensor& tensor, int64_t alignment = 4, bool is_column_major = false) { + int64_t rows = tensor.size(0); + int64_t cols = tensor.size(1); + int64_t pad_rows = (alignment - (rows % alignment)) % alignment; // Compute padding size + + if (pad_rows == 0) { + return tensor; // Already aligned + } + + torch::Tensor padding = torch::zeros({pad_rows, cols}, tensor.options()); + torch::Tensor tensor_padded = torch::cat({tensor, padding}, 0); // Pad along rows + + // Ensure column-major layout + if (is_column_major) { + return tensor_padded.t().contiguous().t(); + } + return tensor_padded; +} + +// Get the next power of 2 of a number +inline uint32_t next_pow2(uint32_t x) noexcept { + if (x <= 1) return 1; + return 1u << (32 - __builtin_clz(x - 1)); +} diff --git a/sgl-kernel/pyproject.toml b/sgl-kernel/pyproject.toml new file mode 100644 index 000000000..778001ecb --- /dev/null +++ b/sgl-kernel/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = [ + "scikit-build-core>=0.10", + "torch>=2.8.0", + "wheel", +] +build-backend = "scikit_build_core.build" + +[project] +name = "sgl-kernel" +version = "0.3.9.post2" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] + +[tool.scikit-build] +cmake.build-type = "Release" +minimum-version = "build-system.requires" + +wheel.py-api = "cp310" +wheel.license-files = [] +wheel.packages = ["python/sgl_kernel"] diff --git a/sgl-kernel/pyproject_cpu.toml b/sgl-kernel/pyproject_cpu.toml new file mode 100644 index 000000000..79ae096e8 --- /dev/null +++ b/sgl-kernel/pyproject_cpu.toml @@ -0,0 +1,36 @@ +[build-system] +requires = [ + "scikit-build-core>=0.10", + "torch>=2.7.1", + "wheel", +] +build-backend = "scikit_build_core.build" + +[project] +name = "sgl-kernel" +version = "0.3.9.post2" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: CPU" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] + +[tool.scikit-build] +cmake.source-dir = "csrc/cpu" +cmake.build-type = "Release" +minimum-version = "build-system.requires" diff --git a/sgl-kernel/pyproject_rocm.toml b/sgl-kernel/pyproject_rocm.toml new file mode 100644 index 000000000..4c787f885 --- /dev/null +++ b/sgl-kernel/pyproject_rocm.toml @@ -0,0 +1,32 @@ +[build-system] +requires = [ + "setuptools>=75.0", + "scikit-build-core>=0.10", + "torch>=2.8.0", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "sgl-kernel" +version = "0.3.9.post2" +description = "Kernel Library for SGLang" +readme = "README.md" +requires-python = ">=3.10" +license = { file = "LICENSE" } +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Environment :: GPU :: NVIDIA CUDA" +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/sgl-project/sglang/tree/main/sgl-kernel" +"Bug Tracker" = "https://github.com/sgl-project/sglang/issues" + +[tool.wheel] +exclude = [ + "dist*", + "tests*", +] diff --git a/sgl-kernel/python/sgl_kernel/__init__.py b/sgl-kernel/python/sgl_kernel/__init__.py new file mode 100755 index 000000000..76c87d30b --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/__init__.py @@ -0,0 +1,117 @@ +import ctypes +import os +import platform + +import torch + +SYSTEM_ARCH = platform.machine() + +cuda_path = f"/usr/local/cuda/targets/{SYSTEM_ARCH}-linux/lib/libcudart.so.12" +if os.path.exists(cuda_path): + ctypes.CDLL(cuda_path, mode=ctypes.RTLD_GLOBAL) + +from sgl_kernel import common_ops +from sgl_kernel.allreduce import * +from sgl_kernel.attention import ( + cutlass_mla_decode, + cutlass_mla_get_workspace_size, + lightning_attention_decode, + merge_state, + merge_state_v2, +) +from sgl_kernel.cutlass_moe import cutlass_w4a8_moe_mm, get_cutlass_w4a8_moe_mm_data +from sgl_kernel.elementwise import ( + FusedSetKVBufferArg, + apply_rope_with_cos_sin_cache_inplace, + concat_mla_k, + copy_to_gpu_no_ce, + downcast_fp8, + fused_add_rmsnorm, + gelu_and_mul, + gelu_tanh_and_mul, + gemma_fused_add_rmsnorm, + gemma_rmsnorm, + rmsnorm, + silu_and_mul, +) +from sgl_kernel.mamba import causal_conv1d_fwd, causal_conv1d_update + +if torch.version.hip is not None: + from sgl_kernel.elementwise import gelu_quick + +from sgl_kernel.fused_moe import fused_marlin_moe +from sgl_kernel.gemm import ( + awq_dequantize, + bmm_fp8, + cutlass_scaled_fp4_mm, + dsv3_fused_a_gemm, + dsv3_router_gemm, + fp8_blockwise_scaled_mm, + fp8_scaled_mm, + gptq_gemm, + gptq_marlin_gemm, + gptq_shuffle, + int8_scaled_mm, + qserve_w4a8_per_chn_gemm, + qserve_w4a8_per_group_gemm, + scaled_fp4_experts_quant, + scaled_fp4_grouped_quant, + scaled_fp4_quant, + sgl_per_tensor_quant_fp8, + sgl_per_token_group_quant_fp8, + sgl_per_token_group_quant_int8, + sgl_per_token_quant_fp8, + shuffle_rows, + silu_and_mul_scaled_fp4_grouped_quant, +) +from sgl_kernel.grammar import apply_token_bitmask_inplace_cuda +from sgl_kernel.kvcacheio import ( + transfer_kv_all_layer, + transfer_kv_all_layer_mla, + transfer_kv_per_layer, + transfer_kv_per_layer_mla, +) +from sgl_kernel.marlin import ( + awq_marlin_moe_repack, + awq_marlin_repack, + gptq_marlin_repack, +) +from sgl_kernel.memory import set_kv_buffer_kernel +from sgl_kernel.moe import ( + apply_shuffle_mul_sum, + cutlass_fp4_group_mm, + fp8_blockwise_scaled_grouped_mm, + moe_align_block_size, + moe_fused_gate, + prepare_moe_input, + topk_softmax, +) +from sgl_kernel.sampling import ( + min_p_sampling_from_probs, + top_k_mask_logits, + top_k_renorm_prob, + top_k_top_p_sampling_from_logits, + top_k_top_p_sampling_from_probs, + top_p_renorm_prob, + top_p_sampling_from_probs, +) +from sgl_kernel.speculative import ( + build_tree_kernel_efficient, + segment_packbits, + tree_speculative_sampling_target_only, + verify_tree_greedy, +) +from sgl_kernel.top_k import fast_topk +from sgl_kernel.version import __version__ + + +def create_greenctx_stream_by_value(*args, **kwargs): + from sgl_kernel.spatial import create_greenctx_stream_by_value as _impl + + return _impl(*args, **kwargs) + + +def get_sm_available(*args, **kwargs): + from sgl_kernel.spatial import get_sm_available as _impl + + return _impl(*args, **kwargs) diff --git a/sgl-kernel/python/sgl_kernel/_fa4_interface.py b/sgl-kernel/python/sgl_kernel/_fa4_interface.py new file mode 100644 index 000000000..512b0aaef --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/_fa4_interface.py @@ -0,0 +1,376 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/203b9b3dba39d5d08dffb49c09aa622984dff07d/flash_attn/cute/interface.py + +# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. +# [2025-07-04] Version in Cute-DSL, for Hopper and Blackwell. You'd need to install nvidia-cutlass-dsl==4.1.0. + + +import math +from typing import Optional, Tuple + +import cuda.bindings.driver as cuda +import cutlass +import cutlass.cute as cute +import torch +from cutlass.cute.runtime import from_dlpack +from flash_attn.cute.flash_fwd import FlashAttentionForwardSm90 +from flash_attn.cute.flash_fwd_sm100 import FlashAttentionForwardSm100 + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +torch2cute_dtype_map = { + torch.float16: cutlass.Float16, + torch.bfloat16: cutlass.BFloat16, + torch.float32: cutlass.Float32, +} + + +def _flash_attn_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + seqused_q: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + softcap: Optional[float] = None, + window_size_left: Optional[int] = None, + window_size_right: Optional[int] = None, + learnable_sink: Optional[torch.Tensor] = None, + # m_block_size: int = 128, + # n_block_size: int = 64, + # num_threads: int = 128, + m_block_size: int = 128, + n_block_size: int = 128, + num_threads: int = 384, + pack_gqa: Optional[bool] = None, + _compute_capability: Optional[int] = None, + return_softmax_lse: Optional[bool] = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + q, k, v = [maybe_contiguous(t) for t in (q, k, v)] + num_head, head_dim = q.shape[-2:] + if cu_seqlens_q is None: + batch_size, seqlen_q = q.shape[:2] + total_q = batch_size * seqlen_q + else: + batch_size = cu_seqlens_q.shape[0] - 1 + seqlen_q = None + total_q = q.shape[0] + if page_table is not None: + assert cu_seqlens_k is None, "page_table is not supported with cu_seqlens_k" + assert page_table.dtype == torch.int32, "page_table must be int32" + assert ( + page_table.stride(-1) == 1 + ), "page_table must be contiguous in the last dimension" + max_num_pages_per_seq = page_table.shape[1] + assert page_table.shape == (batch_size, max_num_pages_per_seq) + num_pages, page_size = k.shape[:2] + seqlen_k = num_pages * page_size + else: + num_pages, page_size = None, None + seqlen_k = k.shape[-3] + num_head_kv = k.shape[-2] + head_dim_v = v.shape[-1] + if cu_seqlens_k is None: + if page_table is None: + assert k.shape == (batch_size, seqlen_k, num_head_kv, head_dim) + assert v.shape == (batch_size, seqlen_k, num_head_kv, head_dim_v) + else: + assert k.shape == (num_pages, page_size, num_head_kv, head_dim) + assert v.shape == (num_pages, page_size, num_head_kv, head_dim_v) + else: + assert k.shape == (seqlen_k, num_head_kv, head_dim) + assert v.shape == (seqlen_k, num_head_kv, head_dim_v) + assert cu_seqlens_k.shape == ( + batch_size + 1, + ), "cu_seqlens_k must have shape (batch_size + 1,)" + if cu_seqlens_q is not None: + assert cu_seqlens_q.shape == ( + batch_size + 1, + ), "cu_seqlens_q must have shape (batch_size + 1,)" + assert seqused_q is None or seqused_q.shape == ( + batch_size, + ), "seqused_q must have shape (batch_size,)" + assert seqused_k is None or seqused_k.shape == ( + batch_size, + ), "seqused_k must have shape (batch_size,)" + assert q.dtype in [ + torch.float16, + torch.bfloat16, + ], "inputs must be float16 or bfloat16" + assert q.dtype == k.dtype == v.dtype, "inputs must have the same dtype" + for t in [cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k]: + if t is not None: + assert ( + t.dtype == torch.int32 + ), "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be int32" + assert ( + t.stride(0) == 1 + ), "cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k must be contiguous" + if learnable_sink is not None: + assert learnable_sink.shape == (num_head,) + assert learnable_sink.dtype == torch.bfloat16, "learnable_sink must be bfloat16" + assert all( + t is None or t.is_cuda + for t in ( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + page_table, + learnable_sink, + ) + ), "inputs must be on CUDA device" + assert num_head % num_head_kv == 0, "num_head must be divisible by num_head_kv" + assert head_dim <= 256, "head_dim must be less than or equal to 256" + alignment = 16 // q.element_size() + assert head_dim % alignment == 0, f"head_dim must be divisible by {alignment}" + assert head_dim_v % alignment == 0, f"head_dim_v must be divisible by {alignment}" + if softmax_scale is None: + softmax_scale = 1.0 / math.sqrt(head_dim) + if softcap == 0.0: + softcap = None + qhead_per_kvhead = num_head // num_head_kv + if pack_gqa is None: + pack_gqa = qhead_per_kvhead > 1 + + out_torch_dtype = q.dtype + device = q.device + q_batch_seqlen_shape = ( + (batch_size, seqlen_q) if cu_seqlens_q is None else (total_q,) + ) + out = torch.empty( + *q_batch_seqlen_shape, + num_head, + head_dim_v, + dtype=out_torch_dtype, + device=device, + ) + lse_shape = ( + (batch_size, num_head, seqlen_q) + if cu_seqlens_q is None + else (num_head, total_q) + ) + lse = ( + torch.empty(lse_shape, dtype=torch.float32, device=device) + if return_softmax_lse + else None + ) + + dtype = torch2cute_dtype_map[q.dtype] + q_tensor, k_tensor, v_tensor, o_tensor = [ + from_dlpack(t.detach(), assumed_align=16).mark_layout_dynamic( + leading_dim=t.ndim - 1 + ) + for t in (q, k, v, out) + ] + lse_tensor = ( + from_dlpack(lse.detach(), assumed_align=4).mark_layout_dynamic( + leading_dim=lse.ndim - 1 + ) + if lse is not None + else None + ) + ( + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + learnable_sink_tensor, + ) = [ + ( + from_dlpack(t.detach(), assumed_align=4).mark_layout_dynamic(leading_dim=0) + if t is not None + else None + ) + for t in (cu_seqlens_q, cu_seqlens_k, seqused_q, seqused_k, learnable_sink) + ] + page_table_tensor = ( + from_dlpack(page_table.detach(), assumed_align=4).mark_layout_dynamic( + leading_dim=1 + ) + if page_table is not None + else None + ) + if causal: + window_size_right = 0 + local = window_size_left is not None or window_size_right is not None + if window_size_left is not None or window_size_right is not None: + if window_size_left is None and window_size_right == 0: + causal, local = True, False + else: + causal, local = False, True + compute_capability = ( + torch.cuda.get_device_capability()[0] + if _compute_capability is None + else _compute_capability + ) + assert compute_capability in [ + 9, + 10, + ], "Unsupported compute capability. Supported: 9.x, 10.x" + current_stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream) + + if compute_capability == 9: # TODO: tune block size according to hdim + if head_dim == head_dim_v == 128 and not causal and not local: + n_block_size = 192 + if compute_capability == 10: + # TODO: fix the varlen case + if ( + pack_gqa + and (128 % qhead_per_kvhead != 0) + or (cu_seqlens_q is not None or seqused_q is not None) + ): + pack_gqa = False + + compile_key = ( + dtype, + head_dim, + head_dim_v, + qhead_per_kvhead, + causal, + softcap is not None, + lse is None, + cu_seqlens_q is None, + cu_seqlens_k is None, + seqused_q is None, + seqused_k is None, + page_table is not None, + window_size_left is not None, + window_size_right is not None, + learnable_sink is not None, + m_block_size, + n_block_size, + num_threads, + pack_gqa, + compute_capability, + ) + if compile_key not in _flash_attn_fwd.compile_cache: + if compute_capability == 9: + assert page_table is None, "paged KV not supported on SM 9.0" + # fa_fwd = FlashAttentionForwardSm80( + fa_fwd = FlashAttentionForwardSm90( + dtype, + head_dim, + head_dim_v, + qhead_per_kvhead, + is_causal=causal, + is_local=local, + pack_gqa=pack_gqa, + m_block_size=m_block_size, + n_block_size=n_block_size, + # num_stages=1, + num_stages=2, + num_threads=num_threads, + Q_in_regs=False, + ) + elif compute_capability == 10: + assert page_size in [ + None, + 128, + ], "Only page_size=128 is supported for paged KV on SM 10.0" + fa_fwd = FlashAttentionForwardSm100( + head_dim, + head_dim_v, + qhead_per_kvhead=qhead_per_kvhead, + is_causal=causal, + is_local=local, + pack_gqa=pack_gqa, + is_persistent=not causal + and not local + and cu_seqlens_q is None + and seqused_q is None, + ) + else: + raise ValueError( + f"Unsupported compute capability: {compute_capability}. Supported: 9.x, 10.x" + ) + # TODO: check @can_implement + _flash_attn_fwd.compile_cache[compile_key] = cute.compile( + fa_fwd, + q_tensor, + k_tensor, + v_tensor, + o_tensor, + lse_tensor, + softmax_scale, + current_stream, + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + page_table_tensor, + softcap, + window_size_left, + window_size_right, + learnable_sink_tensor, + ) + _flash_attn_fwd.compile_cache[compile_key]( + q_tensor, + k_tensor, + v_tensor, + o_tensor, + lse_tensor, + softmax_scale, + current_stream, + cu_seqlens_q_tensor, + cu_seqlens_k_tensor, + seqused_q_tensor, + seqused_k_tensor, + page_table_tensor, + softcap, + window_size_left, + window_size_right, + learnable_sink_tensor, + ) + return out, lse + + +_flash_attn_fwd.compile_cache = {} + + +def flash_attn_varlen_func( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k: Optional[torch.Tensor] = None, + seqused_q: Optional[torch.Tensor] = None, + seqused_k: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + softmax_scale: Optional[float] = None, + causal: bool = False, + window_size: Tuple[Optional[int], Optional[int]] = (None, None), + learnable_sink: Optional[torch.Tensor] = None, + softcap: float = 0.0, + pack_gqa: Optional[bool] = None, + return_softmax_lse: Optional[bool] = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + out, lse = _flash_attn_fwd( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + page_table=page_table, + softmax_scale=softmax_scale, + causal=causal, + window_size_left=window_size[0], + window_size_right=window_size[1], + learnable_sink=learnable_sink, + softcap=softcap, + pack_gqa=pack_gqa, + return_softmax_lse=return_softmax_lse, + ) + + return (out, lse) if return_softmax_lse else out diff --git a/sgl-kernel/python/sgl_kernel/allreduce.py b/sgl-kernel/python/sgl_kernel/allreduce.py new file mode 100644 index 000000000..544fc1d77 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/allreduce.py @@ -0,0 +1,173 @@ +from typing import List, Optional, Tuple + +import torch + +if torch.version.hip is not None: + # ROCM custom allreduce + def init_custom_ar( + meta: torch.Tensor, + rank_data: torch.Tensor, + handles: List[str], + offsets: List[int], + rank: int, + full_nvlink: bool, + ) -> int: + return torch.ops.sgl_kernel.init_custom_ar.default( + meta, rank_data, handles, offsets, rank, full_nvlink + ) + + def all_reduce_reg(fa: int, inp: torch.Tensor, out: torch.Tensor) -> None: + torch.ops.sgl_kernel.all_reduce_reg.default(fa, inp, out) + + def all_reduce_unreg( + fa: int, inp: torch.Tensor, reg_buffer: torch.Tensor, out: torch.Tensor + ) -> None: + torch.ops.sgl_kernel.all_reduce_unreg.default(fa, inp, reg_buffer, out) + + def dispose(fa: int) -> None: + torch.ops.sgl_kernel.dispose.default(fa) + + def meta_size() -> int: + return torch.ops.sgl_kernel.meta_size.default() + + def register_buffer( + fa: int, t: torch.Tensor, handles: List[str], offsets: List[int] + ) -> None: + return torch.ops.sgl_kernel.register_buffer.default(fa, t, handles, offsets) + + def get_graph_buffer_ipc_meta(fa: int) -> Tuple[torch.Tensor, List[int]]: + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_graph_buffers( + fa: int, handles: List[str], offsets: List[List[int]] + ) -> None: + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) + + def allocate_meta_buffer(size: int) -> torch.Tensor: + return torch.ops.sgl_kernel.allocate_meta_buffer.default(size) + + def get_meta_buffer_ipc_handle(inp: torch.Tensor) -> torch.Tensor: + return torch.ops.sgl_kernel.get_meta_buffer_ipc_handle.default(inp) + + # ROCM quick allreduce + def init_custom_qr( + rank: int, world_size: int, qr_max_size: Optional[int] = None + ) -> int: + return torch.ops.sgl_kernel.init_custom_qr.default( + world_size, rank, qr_max_size + ) + + def qr_get_handle(fa: int) -> torch.Tensor: + return torch.ops.sgl_kernel.qr_get_handle.default(fa) + + def qr_open_handles(fa: int, handles: list[torch.Tensor]) -> None: + torch.ops.sgl_kernel.qr_open_handles.default(fa, handles) + + def qr_all_reduce( + fa: int, + profile: int, + inp: torch.Tensor, + out: torch.Tensor, + cast_bf162half: bool, + ) -> None: + torch.ops.sgl_kernel.qr_all_reduce.default( + fa, profile, inp, out, cast_bf162half + ) + + def qr_destroy(fa: int) -> None: + torch.ops.sgl_kernel.qr_destroy.default(fa) + + def qr_max_size() -> int: + return torch.ops.sgl_kernel.qr_max_size.default() + + # mscclpp + def mscclpp_generate_unique_id() -> bytes: + raise NotImplementedError() + + def mscclpp_init_context( + unique_id: bytes, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, + ) -> int: + raise NotImplementedError() + + def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int + ) -> None: + raise NotImplementedError() + +else: + + def init_custom_ar( + ipc_tensors: List[int], rank_data: torch.Tensor, rank: int, full_nvlink: bool + ) -> int: + return torch.ops.sgl_kernel.init_custom_ar.default( + ipc_tensors, rank_data, rank, full_nvlink + ) + + def dispose(fa: int) -> None: + torch.ops.sgl_kernel.dispose.default(fa) + + def all_reduce( + fa: int, + inp: torch.Tensor, + out: torch.Tensor, + reg_buffer: int, + reg_buffer_sz_bytes: int, + ) -> None: + torch.ops.sgl_kernel.all_reduce.default( + fa, inp, out, reg_buffer, reg_buffer_sz_bytes + ) + + def get_graph_buffer_ipc_meta(fa) -> Tuple[List[int], List[int]]: + return torch.ops.sgl_kernel.get_graph_buffer_ipc_meta.default(fa) + + def register_buffer(fa: int, fake_ipc_ptrs: List[int]) -> None: + return torch.ops.sgl_kernel.register_buffer.default(fa, fake_ipc_ptrs) + + def register_graph_buffers( + fa: int, handles: List[List[int]], offsets: List[List[int]] + ) -> None: + torch.ops.sgl_kernel.register_graph_buffers.default(fa, handles, offsets) + + def meta_size() -> int: + return torch.ops.sgl_kernel.meta_size.default() + + def mscclpp_generate_unique_id() -> torch.Tensor: + return torch.ops.sgl_kernel.mscclpp_generate_unique_id.default() + + def mscclpp_init_context( + unique_id: torch.Tensor, + rank: int, + world_size: int, + scratch: torch.Tensor, + put_buffer: torch.Tensor, + nranks_per_node: int, + rank_to_node: List[int], + rank_to_ib: List[int], + context_selection: int, + ) -> int: + return torch.ops.sgl_kernel.mscclpp_init_context.default( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + context_selection, + ) + + def mscclpp_allreduce( + context: int, inp: torch.Tensor, out: torch.Tensor, nthreads: int, nblocks: int + ) -> None: + torch.ops.sgl_kernel.mscclpp_allreduce.default( + context, inp, out, nthreads, nblocks + ) diff --git a/sgl-kernel/python/sgl_kernel/attention.py b/sgl-kernel/python/sgl_kernel/attention.py new file mode 100644 index 000000000..f15b4fa24 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/attention.py @@ -0,0 +1,138 @@ +from typing import Optional, Tuple + +import torch + + +def lightning_attention_decode(q, k, v, past_kv, slope, output, new_kv): + torch.ops.sgl_kernel.lightning_attention_decode.default( + q, k, v, past_kv, slope, output, new_kv + ) + + +def merge_state( + v_a: torch.Tensor, + s_a: torch.Tensor, + v_b: torch.Tensor, + s_b: torch.Tensor, + v_merged: Optional[torch.Tensor] = None, + s_merged: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + # Avoid creating new tensors if they are already provided + if v_merged is None: + v_merged = torch.empty_like(v_a) + if s_merged is None: + s_merged = torch.empty_like(s_a) + torch.ops.sgl_kernel.merge_state.default(v_a, s_a, v_b, s_b, v_merged, s_merged) + return v_merged, s_merged + + +def merge_state_v2( + v_a: torch.Tensor, + s_a: torch.Tensor, + v_b: torch.Tensor, + s_b: torch.Tensor, + v_merged: Optional[torch.Tensor] = None, + s_merged: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + # TODO(DefTruth): Currently, the custom merge_attn_states kernel + # does not support the FP8 data type and non - CUDA devices. + # It may be necessary to fall back to using the Triton kernel. + + # Avoid creating new tensors if they are already provided + if v_merged is None: + v_merged = torch.empty_like(v_a) + if s_merged is None: + s_merged = torch.empty_like(s_a) + torch.ops.sgl_kernel.merge_state_v2.default(v_a, s_a, v_b, s_b, v_merged, s_merged) + return v_merged, s_merged + + +def cutlass_mla_decode( + q_nope: torch.Tensor, + q_pe: torch.Tensor, + kv_c_and_k_pe_cache: torch.Tensor, + seq_lens: torch.Tensor, + page_table: torch.Tensor, + workspace: torch.Tensor, + sm_scale: float, + num_kv_splits: int = 1, # Set to 1 to avoid cuda_graph issue by default. +) -> torch.Tensor: + assert q_nope.ndim == 3, f"q_nope must be a 3D tensor, but got {q_nope.ndim}" + assert q_pe.ndim == 3, f"q_pe must be a 3D tensor, but got {q_pe.ndim}" + assert ( + kv_c_and_k_pe_cache.ndim == 3 + ), f"kv_c_and_k_pe_cache must be a 3D tensor, but got {kv_c_and_k_pe_cache.ndim}" + + B_q, H, D_q_nope = q_nope.shape + B_q_2, H_2, D_q_pe = q_pe.shape + assert (B_q == B_q_2) and (H == H_2) + + _, PAGE_SIZE, D_ckv = kv_c_and_k_pe_cache.shape + + D_latent = 512 + D_rope = 64 + assert D_q_nope == D_latent + assert D_q_pe == D_rope + assert D_ckv == D_latent + D_rope + + MAX_HEADS = 128 + assert H <= MAX_HEADS, f"H must be <= {MAX_HEADS}, but got {H}" + if H < MAX_HEADS: + q_nope_padded = q_nope.new_empty((B_q, MAX_HEADS, D_q_nope)) + q_nope_padded[:, :H] = q_nope + q_nope = q_nope_padded + + q_pe_padded = q_pe.new_empty((B_q, MAX_HEADS, D_q_pe)) + q_pe_padded[:, :H] = q_pe + q_pe = q_pe_padded + + assert len(page_table.shape) == 2 + B_block_table, block_num = page_table.shape + assert B_block_table == B_q + assert block_num > 0, f"block num must be greater than 0, got {block_num}" + assert block_num % (128 / PAGE_SIZE) == 0 + + # TODO(kaixih@nvidia): support fp8 + assert q_nope.dtype in ( + torch.float16, + torch.bfloat16, + ), f"q_nope.dtype needs to be fp16 or bf16 but got {q_nope.dtype}." + assert q_nope.dtype == q_pe.dtype == kv_c_and_k_pe_cache.dtype + assert ( + seq_lens.dtype == torch.int32 + ), f"seq_lens.dtype needs to be int32 but got {seq_lens.dtype}." + assert ( + page_table.dtype == torch.int32 + ), f"page_table.dtype needs to be int32 but got {page_table.dtype}." + + out = q_nope.new_empty((B_q, MAX_HEADS, D_latent)) + + torch.ops.sgl_kernel.cutlass_mla_decode.default( + out, + q_nope, + q_pe, + kv_c_and_k_pe_cache, + seq_lens, + page_table, + workspace, + sm_scale, + num_kv_splits, + ) + return out[:, :H].contiguous() + + +def cutlass_mla_get_workspace_size( + max_seq_len: int, + num_batches: int, + sm_count: int = 0, + num_kv_splits: int = 1, # Set to 1 to avoid cuda_graph issue by default. +) -> int: + assert max_seq_len > 0, f"max_seq_len must be greater than 0, got {max_seq_len}" + assert num_batches > 0, f"num_batches must be greater than 0, got {num_batches}" + return torch.ops.sgl_kernel.cutlass_mla_get_workspace_size.default( + max_seq_len, num_batches, sm_count, num_kv_splits + ) diff --git a/sgl-kernel/python/sgl_kernel/cutlass_moe.py b/sgl-kernel/python/sgl_kernel/cutlass_moe.py new file mode 100644 index 000000000..52256bda1 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/cutlass_moe.py @@ -0,0 +1,112 @@ +import torch + + +def get_cutlass_w4a8_moe_mm_data( + topk_ids: torch.Tensor, + expert_offsets: torch.Tensor, + problem_sizes1: torch.Tensor, + problem_sizes2: torch.Tensor, + input_permutation: torch.Tensor, + output_permutation: torch.Tensor, + num_experts: int, + n: int, + k: int, +): + """ + Prepare data necessary to perform CUTLASS grouped matrix multiplications + used in CUTLASS-based fused MoE. + + The function takes in topk_ids (token-expert mapping) and uses it to + compute: + - expert_offsets: Indices that mark at which token index each expert begins + its computation after the input is sorted with + input_permutation. The number of tokens computed with + expert E is expert_offsets[E + 1] - expert_offsets[E] + - problem_sizes1, problem_sizes2: MxNxK sizes of each expert's + multiplication in two grouped MMs used in + the fused MoE operation. + - input_permutation: Permutation that must be used to shuffle the input + before executing the MMs. + - output_permutation: Permutation that must be used to shuffle the output + after executing the MMs. + """ + torch.ops.sgl_kernel.get_cutlass_w4a8_moe_mm_data.default( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + ) + + +def cutlass_w4a8_moe_mm( + d: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + a_scales: torch.Tensor, + b_scales: torch.Tensor, + experts_offsets: torch.tensor, + problem_sizes: torch.tensor, + a_strides: torch.tensor, + b_strides: torch.tensor, + d_strides: torch.tensor, + s_strides: torch.tensor, + chunk_size: int = 128, + topk: int = 8, +): + """ + Perform grouped matrix multiplication between int4 weights and fp8 activations. + + This function executes multiple GEMM operations in parallel, which is useful for + scenarios like Mixture of Experts (MoE) where different inputs go through different + experts. The implementation leverages NVIDIA Hopper architecture features for + optimal performance with quantized weights. + + Args: + d: Output matrices of shape [total_m, total_n] + a: Activation matrices in FP8 (float_e4m3_t) format + Each tensor should be of shape [total_m, K] in row-major layout + b: Weight matrices in packed int4 format + Each tensor should be of shape [E, N, K//2] in column-major layout + where each byte contains two 4-bit integers + a_scales: Scale factors for the inputs + b_scales: Scale factors for the quantized weights + Each tensor should be of shape [E, K//512, N*8] + experts_offsets: Tensor containing expert offsets for determining group boundaries + problem_sizes: with shape [num_experts, 3] (M, N, K for each group) (int32) + a_strides: Strides information for A matrices + b_strides: Strides information for B matrices + d_strides: Strides information for D matrices + s_strides: Strides information for b_scales matrices + chunk_size: Number of elements each scale value applies to (K//512), default to 128 + + Requirements: + - All tensors must be on a CUDA device + - Requires an NVIDIA Hopper GPU (H100) + - A tensors must be in float8_e4m3fn format + - B tensors must contain packed int4 values (stored as int8) + + Note: + The function computes: D = (A * (B * scales)) + for each group of tensors in parallel + """ + + torch.ops.sgl_kernel.cutlass_w4a8_moe_mm.default( + d, + a, + b, + a_scales, + b_scales, + experts_offsets, + problem_sizes, + a_strides, + b_strides, + d_strides, + s_strides, + chunk_size, + topk, + ) diff --git a/sgl-kernel/python/sgl_kernel/elementwise.py b/sgl-kernel/python/sgl_kernel/elementwise.py new file mode 100644 index 000000000..af3adfd4a --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/elementwise.py @@ -0,0 +1,381 @@ +from dataclasses import dataclass +from typing import List, Optional + +import torch +from sgl_kernel.utils import get_cuda_stream, is_arch_support_pdl + + +# These implementations extensively draw from and build upon the FlashInfer project https://github.com/flashinfer-ai/flashinfer +# Kudos to @yzh119 +def rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, + enable_pdl: Optional[bool] = None, +) -> torch.Tensor: + r"""Root mean square normalization. + + ``out[i] = (input[i] / RMS(input)) * weight[i]`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + out: Optional[torch.Tensor] + The output tensor, if specified, the kernel will update this tensor inplace. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + + Returns + ------- + output: torch.Tensor + Normalized tensor, shape (batch_size, hidden_size). + """ + if out is None: + out = torch.empty_like(input) + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.rmsnorm.default(out, input, weight, eps, enable_pdl) + return out + + +def fused_add_rmsnorm( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + enable_pdl: Optional[bool] = None, +) -> None: + r"""Fused add root mean square normalization. + + Step 1: + ``residual[i] += input[i]`` + + Step 2: + ``input[i] = (residual[i] / RMS(residual)) * weight[i]`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + residual: torch.Tensor + Residual tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + """ + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.fused_add_rmsnorm.default( + input, residual, weight, eps, enable_pdl + ) + + +def gemma_rmsnorm( + input: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + out: Optional[torch.Tensor] = None, + enable_pdl: Optional[bool] = None, +) -> torch.Tensor: + r"""Gemma-style root mean square normalization. + + ``out[i] = (input[i] / RMS(input)) * (weight[i] + 1)`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + out: Optional[torch.Tensor] + The output tensor, if specified, the kernel will update this tensor inplace. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + + Returns + ------- + output: torch.Tensor + Gemma Normalized tensor, shape (batch_size, hidden_size). + """ + if out is None: + out = torch.empty_like(input) + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.gemma_rmsnorm.default(out, input, weight, eps, enable_pdl) + return out + + +def gemma_fused_add_rmsnorm( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-6, + enable_pdl: Optional[bool] = None, +) -> None: + r"""Gemma-style fused add root mean square normalization. + + Step 1: + ``residual[i] += input[i]`` + + Step 2: + ``input[i] = (residual[i] / RMS(residual)) * (weight + 1)`` + + Parameters + ---------- + input: torch.Tensor + Input tensor, shape (batch_size, hidden_size). + residual: torch.Tensor + Residual tensor, shape (batch_size, hidden_size). + weight: torch.Tensor + Weight tensor, shape (hidden_size,). + eps: float + Epsilon for numerical stability. + enable_pdl: Optional[bool] + Whether to enable `programmatic dependent launch + `_ + If None, will be automatically enabled on Hopper architecture. + """ + if enable_pdl is None: + enable_pdl = is_arch_support_pdl() + torch.ops.sgl_kernel.gemma_fused_add_rmsnorm.default( + input, residual, weight, eps, enable_pdl + ) + + +def _check_shape(input: torch.Tensor, output: torch.Tensor) -> None: + assert input.ndim == output.ndim, f"{input.ndim} != {output.ndim}" + assert ( + input.shape[:-1] == output.shape[:-1] + ), f"{input.shape[:-1]} != {output.shape[:-1]}" + assert ( + input.shape[-1] == 2 * output.shape[-1] + ), f"{input.shape[-1]} != {2 * output.shape[-1]}" + + +def silu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.silu_and_mul.default(out, input) + return out + + +def gelu_tanh_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_tanh_and_mul.default(out, input) + return out + + +def gelu_and_mul(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError("The pointers must be multiple of 16 bytes.") + if out is not None: + _check_shape(input, out) + else: + out = torch.empty( + input.shape[:-1] + (input.shape[-1] // 2,), + device=input.device, + dtype=input.dtype, + ) + torch.ops.sgl_kernel.gelu_and_mul.default(out, input) + return out + + +if torch.version.hip is not None: + + def gelu_quick(input: torch.Tensor, out: torch.Tensor = None) -> torch.Tensor: + """ + Quick-GELU: y = x * sigmoid(1.702 * x) + + The CUDA/HIP kernel uses 128-bit (16-byte) vector loads & stores, + so the last-dimension byte length must be a multiple of 16 bytes. + """ + if input.shape[-1] * input.dtype.itemsize % 16 != 0: + raise ValueError( + f"The last dimension ({input.shape[-1]}) x itemsize " + f"({input.dtype.itemsize}) must be a multiple of 16 bytes." + ) + + if out is not None: + assert input.shape == out.shape, f"{input.shape} != {out.shape}" + else: + out = torch.empty_like(input) + + torch.ops.sgl_kernel.gelu_quick(out, input) + return out + + +@dataclass +class FusedSetKVBufferArg: + """ + value : Optional[torch.Tensor] + Value tensor, shape: ``(nnz, num_v_heads * head_size)``. + k_buffer : Optional[torch.Tensor] + Buffer for keys, shape: ``(nnz, num_k_heads * head_size)``. + v_buffer : Optional[torch.Tensor] + Buffer for values, shape: ``(nnz, num_v_heads * head_size)``. + k_scale : Optional[float] + Scale factor for keys. + v_scale : Optional[float] + Scale factor for values. + cache_loc : Optional[torch.Tensor] + Cache location tensor, used for indexing kv cache. + """ + + value: torch.Tensor + k_buffer: torch.Tensor + v_buffer: torch.Tensor + k_scale: Optional[float] + v_scale: Optional[float] + cache_loc: torch.Tensor + + +def apply_rope_with_cos_sin_cache_inplace( + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + head_size: int, + cos_sin_cache: torch.Tensor, + is_neox: bool = True, + fused_set_kv_buffer_arg: Optional[FusedSetKVBufferArg] = None, + enable_pdl: Optional[bool] = None, +) -> None: + r""" + Apply rotary embedding to keys and queries with precomputed cos/sin values. + This is designed to be compatible with the SGL/vLLM implementation. + The result is inplace applied to the input tensors. + + Parameters + ---------- + positions : torch.Tensor + Position indices, shape: ``(nnz)``. + query : torch.Tensor + Query tensor, shape: ``(nnz, num_q_heads * head_size)``. + key : torch.Tensor + Key tensor, shape: ``(nnz, num_k_heads * head_size)``. + cos_sin_cache : torch.Tensor + Cosine and Sine cache tensor, shape: ``(max_seq_len, rotary_dim)``. + Cosine is the first half and Sine is the second half on rotary_dim. + is_neox : bool + Whether to use Neox style RoPE, default: ``True``. + + * If ``True``, the last dimension of the query/key tensor is not interleaved, i.e., + we rotate the first half dimensions ``([..., :head_dim//2])`` and the second half + dimensions ``([..., head_dim//2:])``. + + * If ``False``, the last dimension of the query/key tensor is interleaved, i.e., + we rotate the even dimensions ``([..., ::2])`` and odd dimensions ``([..., 1::2])``. + fused_set_kv_buffer_arg : FusedSetKVBufferArg + Fuse the set-kv-buffer operation into this kernel + + Note + ---- + The rotary dimension is determined by the cosine cache and sine cache. + """ + if cos_sin_cache.dtype != torch.float32: + raise ValueError("cos_sin_cache should be float32") + + if enable_pdl is None: + # the non-fused branch does not yet support PDL, but after we switch to our impl for that branch it will + enable_pdl = is_arch_support_pdl() and (fused_set_kv_buffer_arg is not None) + + if (a := fused_set_kv_buffer_arg) is not None: + assert a.k_scale is None, "k_scale is not yet supported" + assert a.v_scale is None, "v_scale is not yet supported" + assert a.cache_loc.dtype == torch.int64, f"{a.cache_loc.dtype=}" + + def _view_3d(x): + return x.view(x.shape[0], -1, head_size) + + torch.ops.sgl_kernel.apply_rope_pos_ids_cos_sin_cache.default( + _view_3d(query), + _view_3d(key), + _view_3d(query), + _view_3d(key), + cos_sin_cache, + positions.long(), + (not is_neox), + enable_pdl, + get_cuda_stream(), + ( + _view_3d(fused_set_kv_buffer_arg.value) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + _view_3d(fused_set_kv_buffer_arg.k_buffer) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + _view_3d(fused_set_kv_buffer_arg.v_buffer) + if fused_set_kv_buffer_arg is not None + else None + ), + ( + fused_set_kv_buffer_arg.cache_loc + if fused_set_kv_buffer_arg is not None + else None + ), + ) + + +def downcast_fp8( + k: torch.Tensor, + v: torch.Tensor, + k_out: torch.Tensor, + v_out: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + loc: torch.Tensor, + mult: int = 1, + offset: int = 0, +) -> None: + torch.ops.sgl_kernel.downcast_fp8( + k, v, k_out, v_out, k_scale, v_scale, loc, mult, offset, get_cuda_stream() + ) + + +def copy_to_gpu_no_ce(input: List[int], output: torch.Tensor): + torch.ops.sgl_kernel.copy_to_gpu_no_ce(input, output) + + +def concat_mla_k( + k: torch.Tensor, + k_nope: torch.Tensor, + k_rope: torch.Tensor, +): + torch.ops.sgl_kernel.concat_mla_k(k, k_nope, k_rope) diff --git a/sgl-kernel/python/sgl_kernel/flash_attn.py b/sgl-kernel/python/sgl_kernel/flash_attn.py new file mode 100644 index 000000000..ada7b6d07 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/flash_attn.py @@ -0,0 +1,459 @@ +from functools import lru_cache +from typing import Optional, Union + +import torch +import torch.nn as nn + +# try: +# from sgl_kernel import flash_ops +# except: +# raise ImportError("Can not import sgl_kernel. Please check your installation.") + +try: + from ._fa4_interface import flash_attn_varlen_func as flash_attn_varlen_func_v4 +except ImportError: + flash_attn_varlen_func_v4 = None + + +@lru_cache(maxsize=1) +def is_fa3_supported(device=None) -> bool: + # There some fa3 FYI + # FA3 can fail without a enough shared memory for a some shapes, such as higher + # hidden_dim or some special cases. + # Right now, fa3 is supported for sm80/sm87 and sm86/sm89. The main different + # Between sm80/sm87 and sm86/sm89 is the shared memory size. you can follow the link below for more information + # https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared-memory-8-x + # And for sgl-kernel right now, we can build fa3 on sm80/sm86/sm89/sm90a. + # That means if you use A100/A*0/L20/L40/L40s/4090 you can use fa3. + return (torch.version.cuda >= "12.3") and ( + torch.cuda.get_device_capability(device)[0] == 9 + or torch.cuda.get_device_capability(device)[0] == 8 + ) + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +# def flash_attn_with_kvcache( +# q, +# k_cache, +# v_cache, +# k=None, +# v=None, +# qv=None, +# rotary_cos=None, +# rotary_sin=None, +# cache_seqlens: Optional[Union[(int, torch.Tensor)]] = None, +# cache_batch_idx: Optional[torch.Tensor] = None, +# cache_leftpad: Optional[torch.Tensor] = None, +# page_table: Optional[torch.Tensor] = None, +# cu_seqlens_q: Optional[torch.Tensor] = None, +# cu_seqlens_k_new: Optional[torch.Tensor] = None, +# max_seqlen_q: Optional[int] = None, +# rotary_seqlens: Optional[torch.Tensor] = None, +# q_descale: Optional[torch.Tensor] = None, +# k_descale: Optional[torch.Tensor] = None, +# v_descale: Optional[torch.Tensor] = None, +# softmax_scale=None, +# causal=False, +# window_size=(-1, -1), # -1 means infinite context window +# softcap=0.0, # 0.0 means deactivated +# rotary_interleaved=True, +# scheduler_metadata=None, +# num_splits=0, # Can be tuned for speed +# pack_gqa=None, # Can be tuned for speed +# sm_margin=0, # Can be tuned if some SMs are used for communication +# return_softmax_lse=False, +# sinks=None, +# ver=3, +# ): +# """ +# If k and v are not None, k_cache and v_cache will be updated *inplace* with the new values from +# k and v. This is useful for incremental decoding: you can pass in the cached keys/values from +# the previous step, and update them with the new keys/values from the current step, and do +# attention with the updated cache, all in 1 kernel. + +# If you pass in k / v, you must make sure that the cache is large enough to hold the new values. +# For example, the KV cache could be pre-allocated with the max sequence length, and you can use +# cache_seqlens to keep track of the current sequence lengths of each sequence in the batch. + +# Also apply rotary embedding if rotary_cos and rotary_sin are passed in. The key @k will be +# rotated by rotary_cos and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. +# If causal or local (i.e., window_size != (-1, -1)), the query @q will be rotated by rotary_cos +# and rotary_sin at indices cache_seqlens, cache_seqlens + 1, etc. +# If not causal and not local, the query @q will be rotated by rotary_cos and rotary_sin at +# indices cache_seqlens only (i.e. we consider all tokens in @q to be at position cache_seqlens). + +# See tests/test_flash_attn.py::test_flash_attn_kvcache for examples of how to use this function. + +# Supports multi-query and grouped-query attention (MQA/GQA) by passing in KV with fewer heads +# than Q. Note that the number of heads in Q must be divisible by the number of heads in KV. +# For example, if Q has 6 heads and K, V have 2 heads, head 0, 1, 2 of Q will attention to head +# 0 of K, V, and head 3, 4, 5 of Q will attention to head 1 of K, V. + +# If causal=True, the causal mask is aligned to the bottom right corner of the attention matrix. +# For example, if seqlen_q = 2 and seqlen_k = 5, the causal mask (1 = keep, 0 = masked out) is: +# 1 1 1 1 0 +# 1 1 1 1 1 +# If seqlen_q = 5 and seqlen_k = 2, the causal mask is: +# 0 0 +# 0 0 +# 0 0 +# 1 0 +# 1 1 +# If the row of the mask is all zero, the output will be zero. + +# If window_size != (-1, -1), implements sliding window local attention. Query at position i +# will only attend to keys between +# [i + seqlen_k - seqlen_q - window_size[0], i + seqlen_k - seqlen_q + window_size[1]] inclusive. + +# Note: Does not support backward pass. + +# Arguments: +# q: (batch_size, seqlen, nheads, headdim) +# k_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim) if there's no page_table, +# or (num_blocks, page_block_size, nheads_k, headdim) if there's a page_table (i.e. paged KV cache) +# page_block_size must be a multiple of 256. +# v_cache: (batch_size_cache, seqlen_cache, nheads_k, headdim_v) if there's no page_table, +# or (num_blocks, page_block_size, nheads_k, headdim_v) if there's a page_table (i.e. paged KV cache) +# k [optional]: (batch_size, seqlen_new, nheads_k, headdim). If not None, we concatenate +# k with k_cache, starting at the indices specified by cache_seqlens. +# v [optional]: (batch_size, seqlen_new, nheads_k, headdim_v). Similar to k. +# qv [optional]: (batch_size, seqlen, nheads, headdim_v) +# rotary_cos [optional]: (seqlen_ro, rotary_dim / 2). If not None, we apply rotary embedding +# to k and q. Only applicable if k and v are passed in. rotary_dim must be divisible by 16. +# rotary_sin [optional]: (seqlen_ro, rotary_dim / 2). Similar to rotary_cos. +# cache_seqlens: int, or (batch_size,), dtype torch.int32. The sequence lengths of the +# KV cache. +# cache_batch_idx: (batch_size,), dtype torch.int32. The indices used to index into the KV cache. +# If None, we assume that the batch indices are [0, 1, 2, ..., batch_size - 1]. +# If the indices are not distinct, and k and v are provided, the values updated in the cache +# might come from any of the duplicate indices. +# cache_leftpad: (batch_size,), dtype torch.int32. The index that the KV cache starts. If None, assume 0. +# page_table [optional]: (batch_size, max_num_blocks_per_seq), dtype torch.int32. +# softmax_scale: float. The scaling of QK^T before applying softmax. +# Default to 1 / sqrt(headdim). +# causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). +# window_size: (left, right). If not (-1, -1), implements sliding window local attention. +# softcap: float. Anything > 0 activates softcapping attention. +# rotary_interleaved: bool. Only applicable if rotary_cos and rotary_sin are passed in. +# If True, rotary embedding will combine dimensions 0 & 1, 2 & 3, etc. If False, +# rotary embedding will combine dimensions 0 & rotary_dim / 2, 1 & rotary_dim / 2 + 1 +# (i.e. GPT-NeoX style). +# num_splits: int. If > 1, split the key/value into this many chunks along the sequence. +# If num_splits == 1, we don't split the key/value. If num_splits == 0, we use a heuristic +# to automatically determine the number of splits. +# Don't change this unless you know what you are doing. +# return_softmax_lse: bool. Whether to return the logsumexp of the attention scores. + +# Return: +# out: (batch_size, seqlen, nheads, headdim). +# softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The +# logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax +# normalization factor). +# """ +# if ver == 4: +# raise NotImplementedError("haven't implemented flash_attn_with_kvcache for fa4") + +# assert k_cache.stride(-1) == 1, "k_cache must have contiguous last dimension" +# assert v_cache.stride(-1) == 1, "v_cache must have contiguous last dimension" +# if softmax_scale is None: +# softmax_scale = (q.shape[-1] + (qv.shape[-1] if qv is not None else 0)) ** ( +# -0.5 +# ) +# if cache_seqlens is not None and isinstance(cache_seqlens, int): +# cache_seqlens = torch.full( +# (k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device +# ) +# cache_seqlens = maybe_contiguous(cache_seqlens) + +# q, k_cache, k, v = [maybe_contiguous(x) for x in (q, k_cache, k, v)] +# v_cache = ( +# v_cache.contiguous() +# if v_cache.stride(-1) != 1 and v_cache.stride(-3) != 1 +# else v_cache +# ) +# cu_seqlens_q, cu_seqlens_k_new = [ +# maybe_contiguous(x) for x in (cu_seqlens_q, cu_seqlens_k_new) +# ] +# page_table, cache_batch_idx, cache_leftpad = [ +# maybe_contiguous(x) for x in (page_table, cache_batch_idx, cache_leftpad) +# ] +# rotary_cos, rotary_sin = [maybe_contiguous(x) for x in (rotary_cos, rotary_sin)] +# rotary_seqlens = maybe_contiguous(rotary_seqlens) + +# if hasattr(torch.version, 'hip') and torch.version.hip is not None: +# # HIP环境回退 +# from flash_attn import flash_attn_with_kvcache as fa_with_kv +# out, softmax_lse, *rest = fa_with_kv( +# q, k, v, k_cache, v_cache, cache_seqlens, cache_batch_idx, +# block_tables, softmax_scale, causal, alibi_slopes, out +# ) +# else: +# out, softmax_lse, *rest = torch.ops.sgl_kernel.fwd.default( +# q, +# k_cache, +# v_cache, +# k, +# v, +# qv, +# None, # out +# cu_seqlens_q, +# None, # cu_seqlens_k +# cu_seqlens_k_new, +# None, # seqused_q +# cache_seqlens, +# max_seqlen_q, +# None, # max_seqlen_k +# page_table, +# cache_batch_idx, +# cache_leftpad, +# rotary_cos, +# rotary_sin, +# rotary_seqlens, +# q_descale, +# k_descale, +# v_descale, +# softmax_scale, +# causal, +# window_size[0], +# window_size[1], +# softcap, +# rotary_interleaved, +# scheduler_metadata, +# num_splits, +# pack_gqa, +# sm_margin, +# sinks, +# ) +# return (out, softmax_lse) if return_softmax_lse else out +def flash_attn_with_kvcache( + q, + k_cache, + v_cache, + k=None, + v=None, + qv=None, + rotary_cos=None, + rotary_sin=None, + cache_seqlens: Optional[Union[(int, torch.Tensor)]] = None, + cache_batch_idx: Optional[torch.Tensor] = None, + cache_leftpad: Optional[torch.Tensor] = None, + page_table: Optional[torch.Tensor] = None, + cu_seqlens_q: Optional[torch.Tensor] = None, + cu_seqlens_k_new: Optional[torch.Tensor] = None, + max_seqlen_q: Optional[int] = None, + rotary_seqlens: Optional[torch.Tensor] = None, + q_descale: Optional[torch.Tensor] = None, + k_descale: Optional[torch.Tensor] = None, + v_descale: Optional[torch.Tensor] = None, + softmax_scale=None, + causal=False, + window_size=(-1, -1), # -1 means infinite context window + softcap=0.0, # 0.0 means deactivated + rotary_interleaved=True, + scheduler_metadata=None, + num_splits=0, # Can be tuned for speed + pack_gqa=None, # Can be tuned for speed + sm_margin=0, # Can be tuned if some SMs are used for communication + return_softmax_lse=False, + sinks=None, + ver=3, +): + if ver == 4: + raise NotImplementedError("haven't implemented flash_attn_with_kvcache for fa4") + + # HIP环境检测和回退 + # if hasattr(torch.version, 'hip') and torch.version.hip is not None: + # # 简单PyTorch回退，处理实际的张量形状 + # # q: [1, 4, 256], k_cache: [411528, 1, 1, 256], v_cache: [411528, 1, 1, 256] + + # if softmax_scale is None: + # softmax_scale = (q.shape[-1]) ** (-0.5) + + # # 重塑以匹配attention计算 + # q_reshaped = q.unsqueeze(1) # [1, 1, 4, 256] + # k_reshaped = k_cache.squeeze(1).squeeze(1) # [411528, 256] + # v_reshaped = v_cache.squeeze(1).squeeze(1) # [411528, 256] + + # # 简单的点积attention + # scores = torch.matmul(q, k_reshaped.T) * softmax_scale # [1, 4, 411528] + # attn_weights = torch.softmax(scores, dim=-1) + # out = torch.matmul(attn_weights, v_reshaped) # [1, 4, 256] + + # if return_softmax_lse: + # softmax_lse = torch.zeros(1, 4, 1, device=q.device) + # return out, softmax_lse + # return out + + # 原始sgl_kernel实现 + assert k_cache.stride(-1) == 1, "k_cache must have contiguous last dimension" + assert v_cache.stride(-1) == 1, "v_cache must have contiguous last dimension" + if softmax_scale is None: + softmax_scale = (q.shape[-1] + (qv.shape[-1] if qv is not None else 0)) ** (-0.5) + if cache_seqlens is not None and isinstance(cache_seqlens, int): + cache_seqlens = torch.full( + (k_cache.shape[0],), cache_seqlens, dtype=torch.int32, device=k_cache.device + ) + cache_seqlens = maybe_contiguous(cache_seqlens) + + q, k_cache, k, v = [maybe_contiguous(x) for x in (q, k_cache, k, v)] + v_cache = ( + v_cache.contiguous() + if v_cache.stride(-1) != 1 and v_cache.stride(-3) != 1 + else v_cache + ) + cu_seqlens_q, cu_seqlens_k_new = [ + maybe_contiguous(x) for x in (cu_seqlens_q, cu_seqlens_k_new) + ] + page_table, cache_batch_idx, cache_leftpad = [ + maybe_contiguous(x) for x in (page_table, cache_batch_idx, cache_leftpad) + ] + rotary_cos, rotary_sin = [maybe_contiguous(x) for x in (rotary_cos, rotary_sin)] + rotary_seqlens = maybe_contiguous(rotary_seqlens) + + out, softmax_lse, *rest = torch.ops.sgl_kernel.fwd.default( + q, + k_cache, + v_cache, + k, + v, + qv, + None, # out + cu_seqlens_q, + None, # cu_seqlens_k + cu_seqlens_k_new, + None, # seqused_q + cache_seqlens, + max_seqlen_q, + None, # max_seqlen_k + page_table, + cache_batch_idx, + cache_leftpad, + rotary_cos, + rotary_sin, + rotary_seqlens, + q_descale, + k_descale, + v_descale, + softmax_scale, + causal, + window_size[0], + window_size[1], + softcap, + rotary_interleaved, + scheduler_metadata, + num_splits, + pack_gqa, + sm_margin, + sinks, + ) + return (out, softmax_lse) if return_softmax_lse else out + + +def flash_attn_varlen_func( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + seqused_q=None, + seqused_k=None, + softmax_scale=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(-1, -1), + softcap=0.0, + num_splits=1, + pack_gqa=None, + sm_margin=0, + return_softmax_lse=False, + sinks=None, + ver=3, +): + if ver == 4: + assert ( + flash_attn_varlen_func_v4 is not None + ), "FA4 is not available, please check your installation." + # Using `(-1, -1)` as no sliding window causes correctness issues for FA4. + if window_size == (-1, -1): + window_size = (None, None) + return flash_attn_varlen_func_v4( + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + # max_seqlen_q, + # max_seqlen_k, + seqused_q=seqused_q, + seqused_k=seqused_k, + softmax_scale=softmax_scale, + causal=causal, + # qv=qv, + # q_descale=q_descale, + # k_descale=k_descale, + # v_descale=v_descale, + window_size=window_size, + softcap=softcap, + # num_splits=num_splits, + pack_gqa=pack_gqa, + # sm_margin=sm_margin, + return_softmax_lse=return_softmax_lse, + learnable_sink=sinks, + ) + + if not is_fa3_supported(): + raise NotImplementedError( + "flash_attn at sgl-kernel is only supported on sm90 and above" + ) + + if softmax_scale is None: + softmax_scale = (q.shape[-1] + (qv.shape[-1] if qv is not None else 0)) ** ( + -0.5 + ) + + out, softmax_lse, *rest = torch.ops.sgl_kernel.fwd.default( + q, + k, + v, + None, # k_new + None, # v_new + qv, # qv + None, # out + cu_seqlens_q, + cu_seqlens_k, + None, # cu_seqlens_k_new + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + None, # page_table, + None, # kv_batch_idx + None, # leftpad_k + None, # rotary cos + None, # rotary sin + None, # seqlens_rotary + q_descale, + k_descale, + v_descale, + softmax_scale, + causal, + window_size[0], + window_size[1], + softcap, + is_rotary_interleaved=False, + scheduler_metadata=None, + num_splits=num_splits, + pack_gqa=pack_gqa, + sm_margin=sm_margin, + sinks=sinks, + ) + + return (out, softmax_lse, *rest) if return_softmax_lse else out diff --git a/sgl-kernel/python/sgl_kernel/fused_moe.py b/sgl-kernel/python/sgl_kernel/fused_moe.py new file mode 100644 index 000000000..c9a11bfc0 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/fused_moe.py @@ -0,0 +1,225 @@ +import functools +from typing import Optional + +import torch +from sgl_kernel import silu_and_mul + + +def get_scalar_type(num_bits: int, has_zp: bool): + from sgl_kernel.scalar_type import scalar_types + + if has_zp: + assert num_bits == 4 + return scalar_types.uint4 + else: + return scalar_types.uint4b8 if num_bits == 4 else scalar_types.uint8b128 + + +def fused_marlin_moe( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + gating_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + global_num_experts: int = -1, + expert_map: Optional[torch.Tensor] = None, + g_idx1: Optional[torch.Tensor] = None, + g_idx2: Optional[torch.Tensor] = None, + sort_indices1: Optional[torch.Tensor] = None, + sort_indices2: Optional[torch.Tensor] = None, + w1_zeros: Optional[torch.Tensor] = None, + w2_zeros: Optional[torch.Tensor] = None, + workspace: Optional[torch.Tensor] = None, + num_bits: int = 8, + is_k_full: bool = True, + inplace: bool = False, +) -> torch.Tensor: + """ + This function computes a Mixture of Experts (MoE) layer using two sets of + weights, w1 and w2, and top-k gating mechanism. + + Parameters: + - hidden_states (torch.Tensor): The input tensor to the MoE layer. + - w1 (torch.Tensor): The first set of expert weights. + - w2 (torch.Tensor): The second set of expert weights. + - w1_scale (torch.Tensor): Scale to be used for w1. + - w2_scale (torch.Tensor): Scale to be used for w2. + - gating_output (torch.Tensor): The output of the gating operation + (before softmax). + - g_idx1 (Optional[torch.Tensor]): The first set of act_order indices. + - g_idx2 (Optional[torch.Tensor]): The second set of act_order indices. + - sort_indices1 (Optional[torch.Tensor]): The first act_order input + permutation. + - sort_indices2 (Optional[torch.Tensor]): The second act_order input + permutation. + - topk_weights (torch.Tensor): Top-k weights. + - topk_ids (torch.Tensor): Indices of topk-k elements. + - w1_zeros (Optional[torch.Tensor]): Optional zero points to be used for w1. + - w2_zeros (Optional[torch.Tensor]): Optional zero points to be used for w2. + - num_bits (bool): The number of bits in expert weights quantization. + + Returns: + - torch.Tensor: The output tensor after applying the MoE layer. + """ + # Delay the import to avoid circular dependency + from sglang.srt.layers.moe.fused_moe_triton import ( + moe_align_block_size, + try_get_optimal_moe_config, + ) + + # Check constraints. + assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" + assert hidden_states.shape[1] == w1.shape[1] * 16, "Hidden size mismatch w1" + assert hidden_states.shape[1] == w2.shape[2] // ( + num_bits // 2 + ), "Hidden size mismatch w2" + assert hidden_states.is_contiguous(), "Hidden_states must be contiguous" + assert w1.is_contiguous(), "Expert weights1 must be contiguous" + assert w2.is_contiguous(), "Expert weights2 must be contiguous" + assert hidden_states.dtype in [torch.float16, torch.bfloat16] + assert num_bits in [4, 8] + + M, K = hidden_states.shape + E = w1.shape[0] + N = w2.shape[1] * 16 + topk = topk_ids.shape[1] + + get_config_func = functools.partial( + try_get_optimal_moe_config, + w1.shape, + w2.shape, + topk_ids.shape[1], + None, + is_marlin=True, + ) + config = get_config_func(M) + + block_size_m = config["BLOCK_SIZE_M"] + + if global_num_experts == -1: + global_num_experts = E + sorted_token_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_ids, block_size_m, global_num_experts + ) + + if workspace is None: + max_workspace_size = (max(2 * N, K) // 64) * ( + sorted_token_ids.size(0) // block_size_m + ) + device = hidden_states.device + sms = torch.cuda.get_device_properties(device).multi_processor_count + max_workspace_size = min(max_workspace_size, sms * 4) + workspace = torch.zeros( + max_workspace_size, dtype=torch.int, device=device, requires_grad=False + ) + + scalar_type1 = get_scalar_type(num_bits, w1_zeros is not None) + scalar_type2 = get_scalar_type(num_bits, w2_zeros is not None) + + intermediate_cache2 = torch.empty( + (M * topk_ids.shape[1], N), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache13 = torch.empty( + (M * topk_ids.shape[1] * max(2 * N, K),), + device=hidden_states.device, + dtype=hidden_states.dtype, + ) + intermediate_cache1 = intermediate_cache13[: M * topk_ids.shape[1] * 2 * N] + intermediate_cache1 = intermediate_cache1.view(-1, 2 * N) + intermediate_cache3 = intermediate_cache13[: M * topk_ids.shape[1] * K] + intermediate_cache3 = intermediate_cache3.view(-1, K) + + use_atomic_add = ( + hidden_states.dtype == torch.half + or torch.cuda.get_device_capability(hidden_states.device)[0] >= 9 + ) + + intermediate_cache1 = torch.ops.sgl_kernel.moe_wna16_marlin_gemm.default( + hidden_states, + intermediate_cache1, + w1, + w1_scale, + w1_zeros, + g_idx1, + sort_indices1, + workspace, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + topk_weights, + moe_block_size=block_size_m, + top_k=topk, + mul_topk_weights=False, + is_ep=expert_map is not None, + b_q_type_id=scalar_type1.id, + size_m=M, + size_n=2 * N, + size_k=K, + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=True, + is_zp_float=False, + ) + + silu_and_mul(intermediate_cache1.view(-1, 2 * N), intermediate_cache2) + + if expert_map is not None: + intermediate_cache3.zero_() + + intermediate_cache3 = torch.ops.sgl_kernel.moe_wna16_marlin_gemm.default( + intermediate_cache2, + intermediate_cache3, + w2, + w2_scale, + w2_zeros, + g_idx2, + sort_indices2, + workspace, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + topk_weights, + moe_block_size=block_size_m, + top_k=1, + mul_topk_weights=True, + is_ep=expert_map is not None, + b_q_type_id=scalar_type2.id, + size_m=M * topk, + size_n=K, + size_k=N, + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=True, + is_zp_float=False, + ).view(-1, topk, K) + + output = hidden_states if inplace else torch.empty_like(hidden_states) + return torch.sum( + intermediate_cache3.view(*intermediate_cache3.shape), dim=1, out=output + ) + + +def fused_marlin_moe_fake( + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, + gating_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + g_idx1: Optional[torch.Tensor] = None, + g_idx2: Optional[torch.Tensor] = None, + sort_indices1: Optional[torch.Tensor] = None, + sort_indices2: Optional[torch.Tensor] = None, + w1_zeros: Optional[torch.Tensor] = None, + w2_zeros: Optional[torch.Tensor] = None, + num_bits: int = 8, + is_k_full: bool = True, +) -> torch.Tensor: + return torch.empty_like(hidden_states) diff --git a/sgl-kernel/python/sgl_kernel/gemm.py b/sgl-kernel/python/sgl_kernel/gemm.py new file mode 100644 index 000000000..36672877d --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/gemm.py @@ -0,0 +1,550 @@ +from typing import Optional, Tuple + +import torch +from sgl_kernel.scalar_type import ScalarType +from sgl_kernel.utils import _get_cache_buf, get_cuda_stream + + +def awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.ByteTensor: + return torch.ops.sgl_kernel.awq_dequantize.default(qweight, scales, qzeros) + + +def int8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.int8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def fp8_blockwise_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype): + return torch.ops.sgl_kernel.fp8_blockwise_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + ) + + +def fp8_scaled_mm(mat_a, mat_b, scales_a, scales_b, out_dtype, bias=None): + return torch.ops.sgl_kernel.fp8_scaled_mm.default( + mat_a, + mat_b, + scales_a, + scales_b, + out_dtype, + bias, + ) + + +def _bmm_fp8_internal( + workspace_buffer: torch.Tensor, + A: torch.Tensor, + B: torch.Tensor, + D: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, +) -> None: + cublas_handle = torch.cuda.current_blas_handle() + torch.ops.sgl_kernel.bmm_fp8.default( + A, + B, + D, + A_scale, + B_scale, + workspace_buffer, + cublas_handle, + get_cuda_stream(), + ) + + +def bmm_fp8( + A: torch.Tensor, + B: torch.Tensor, + A_scale: torch.Tensor, + B_scale: torch.Tensor, + dtype: torch.dtype, + out: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out is None: + out = torch.empty( + (A.shape[0], A.shape[1], B.shape[2]), + device=A.device, + dtype=dtype, + ) + workspace_buffer = _get_cache_buf("bmm_fp8_workspace", 32 * 1024 * 1024, A.device) + _bmm_fp8_internal(workspace_buffer, A, B, out, A_scale, B_scale) + return out + + +def dsv3_fused_a_gemm( + mat_a: torch.Tensor, + mat_b: torch.Tensor, + output: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if output is None: + output = torch.empty( + (mat_a.shape[0], mat_b.shape[1]), + device=mat_a.device, + dtype=mat_a.dtype, + ) + torch.ops.sgl_kernel.dsv3_fused_a_gemm.default(output, mat_a, mat_b) + return output + + +def sgl_per_token_group_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + group_size: int, + eps: float, + fp8_min: float, + fp8_max: float, + scale_ue8m0: bool, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_group_quant_fp8.default( + input, output_q, output_s, group_size, eps, fp8_min, fp8_max, scale_ue8m0 + ) + + +def sgl_per_token_group_quant_int8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + group_size: int, + eps: float, + int8_min: float, + int8_max: float, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_group_quant_int8.default( + input, output_q, output_s, group_size, eps, int8_min, int8_max + ) + + +def sgl_per_tensor_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, + is_static: bool, +) -> None: + torch.ops.sgl_kernel.sgl_per_tensor_quant_fp8.default( + input, output_q, output_s, is_static + ) + + +def sgl_per_token_quant_fp8( + input: torch.Tensor, + output_q: torch.Tensor, + output_s: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.sgl_per_token_quant_fp8.default(input, output_q, output_s) + + +def cutlass_scaled_fp4_mm( + a: torch.Tensor, + b: torch.Tensor, + block_scale_a: torch.Tensor, + block_scale_b: torch.Tensor, + alpha: torch.Tensor, + out_dtype: torch.dtype, +) -> torch.Tensor: + assert a.ndim == 2 and b.ndim == 2 + m, n = a.shape[0], b.shape[0] + out = torch.empty((m, n), dtype=out_dtype, device=a.device) + torch.ops.sgl_kernel.cutlass_scaled_fp4_mm.default( + out, a, b, block_scale_a, block_scale_b, alpha + ) + return out + + +def scaled_fp4_quant( + input: torch.Tensor, input_global_scale: torch.Tensor +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP4 and return quantized tensor and scale. + + This function quantizes the last dimension of the given tensor `input`. For + every 16 consecutive elements, a single dynamically computed scaling factor + is shared. This scaling factor is quantized using the `input_global_scale` + and is stored in a swizzled layout (see + https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-scale-factor-b-layout-4x). + + Args: + input: The input tensor to be quantized to FP4 + input_global_scale: A scalar scaling factor for the entire tensor. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP4 but every + two values are packed into a uint8 and float8_e4m3 scaling factors + in a sizzled layout. + """ + assert input.ndim >= 1, f"input.ndim needs to be >= 1, but got {input.ndim}." + other_dims = 1 if input.ndim == 1 else -1 + input = input.reshape(other_dims, input.shape[-1]) + m, n = input.shape + block_size = 16 + device = input.device + + assert n % block_size == 0, f"last dim has to be multiple of 16, but got {n}." + assert input.dtype in ( + torch.float16, + torch.bfloat16, + ), f"input.dtype needs to be fp16 or bf16 but got {input.dtype}." + + # Two fp4 values will be packed into an uint8. + output = torch.empty((m, n // 2), device=device, dtype=torch.uint8) + + # We use the rounded values to store the swizzled values. Then, the scaling + # factors in float8_e4m3fn are packed into an int32 for every 4 values. + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // block_size + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + # padded part should be zeroed out + if rounded_n > scale_n: + output_scale = torch.zeros( + (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 + ) + else: + output_scale = torch.empty( + (rounded_m, rounded_n // 4), device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.scaled_fp4_quant.default( + output, input, output_scale, input_global_scale + ) + output_scale = output_scale.view(torch.float8_e4m3fn) + return output, output_scale + + +def qserve_w4a8_per_chn_gemm( + in_feats: torch.Tensor, + kernel: torch.Tensor, + wscales: torch.Tensor, + ascales: torch.Tensor, + w_szs: torch.Tensor, + a_ssums: torch.Tensor, + out_feats: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out_feats is None: + # NOTE(HandH1998): qserve_w4a8_per_chn_gemm only supports out dtype=torch.float16 now + out_feats = torch.empty( + (in_feats.shape[0], kernel.shape[0]), + device=in_feats.device, + dtype=torch.float16, + ) + torch.ops.sgl_kernel.qserve_w4a8_per_chn_gemm.default( + in_feats, kernel, wscales, ascales, w_szs, a_ssums, out_feats + ) + return out_feats + + +def qserve_w4a8_per_group_gemm( + in_feats: torch.Tensor, + kernel: torch.Tensor, + zeros: torch.Tensor, + scales_i8: torch.Tensor, + wscales: torch.Tensor, + ascales: torch.Tensor, + out_feats: Optional[torch.Tensor] = None, +) -> torch.Tensor: + if out_feats is None: + # NOTE(HandH1998): qserve_w4a8_per_group_gemm only supports out dtype=torch.float16 now + out_feats = torch.empty( + (in_feats.shape[0], kernel.shape[0]), + device=in_feats.device, + dtype=torch.float16, + ) + torch.ops.sgl_kernel.qserve_w4a8_per_group_gemm.default( + in_feats, kernel, zeros, scales_i8, wscales, ascales, out_feats + ) + return out_feats + + +def dsv3_router_gemm( + hidden_states: torch.Tensor, + router_weights: torch.Tensor, + out_dtype: torch.dtype = torch.bfloat16, +) -> torch.Tensor: + output = torch.empty( + hidden_states.shape[0], + router_weights.shape[0], + device=hidden_states.device, + dtype=out_dtype, + ) + torch.ops.sgl_kernel.dsv3_router_gemm( + output, + hidden_states, + router_weights, + ) + return output + + +def shuffle_rows(input_tensor, dst2src_map, output_tensor_shape): + output_tensor = torch.empty( + output_tensor_shape, + device=input_tensor.device, + dtype=input_tensor.dtype, + ) + torch.ops.sgl_kernel.shuffle_rows.default(input_tensor, dst2src_map, output_tensor) + return output_tensor + + +def scaled_fp4_grouped_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + mask: torch.Tensor, +): + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + grouped gemm inputs (e.g., grouped_gemm_nt_masked for flashinfer). + Args: + input: The input tensor to be quantized to FP4, with shape (l, m, k) + l is number of groups, m is number of tokens per group, k is number of features. + input_global_scale: A scalar scaling factor for the entire tensor, with + shape (l,). + Outputs: + output: The quantized tensor in FP4, with shape (m, k // 2, l) but the physical + layout is (l, m, k // 2). `// 2` is because two fp4 values are packed into + an uint8. + output_scales: The blockscale tensor in FP8-E4M3, with shape (32, 4, rm, 4, rk, l) + but the physical layout is (l, rm, rk, 32, 4, 4). + Note: + For the shape of output_scales, `32 * 4 * rm` is a padded m to nearest multiple of 128. + `4 * rk` is a padded `k // 16` to nearest multiple of 4. These layout constants are + required by the NVIDIA Blackwell MMA operations. + """ + device = input_tensor.device + l, m, k = input_tensor.shape + sf_vec_size = 16 + assert k % sf_vec_size == 0, f"k must be multiple of 16, but got {k}." + + scale_k = k // sf_vec_size + padded_k = (scale_k + (4 - 1)) // 4 * 4 + padded_k_int32 = padded_k // 4 + padded_m = (m + (128 - 1)) // 128 * 128 + output = torch.empty(l, m, k // 2, device=device, dtype=torch.uint8) + output_scales = torch.empty( + l, padded_m, padded_k_int32, device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.silu_and_mul_scaled_fp4_experts_quant.default( + output.view(l * m, k // 2), + output_scales.view(l * padded_m, padded_k_int32), + input_tensor.view(l * m, k), + input_global_scale, + mask, + use_silu_and_mul=False, + ) + # The physical layout of the output is (l, m, k // 2), but we want to return a + # logical layout (m, k // 2, l) required by the flashinfer masked group gemm. + output = output.permute(1, 2, 0) + # The physical layout of the output scales is already swizzled as (l, rm, rk, 32, 4, 4), a + # requirement for the flashinfer masked group gemm, where rm=m/128 and rk=k/4. The logic + # layout is (32, 4, rm, 4, rk, l). + output_scales = output_scales.view(torch.float8_e4m3fn).view( + l, padded_m // 128, padded_k // 4, 32, 4, 4 + ) + output_scales = output_scales.permute(3, 4, 1, 5, 2, 0) + return output, output_scales + + +def silu_and_mul_scaled_fp4_grouped_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + mask: torch.Tensor, +): + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + grouped gemm inputs (e.g., grouped_gemm_nt_masked for flashinfer). + Args: + input: The input tensor to be quantized to FP4, with shape (l, m, k * 2) + l is number of groups, m is number of tokens per group, k is number of features. + input_global_scale: A scalar scaling factor for the entire tensor, with + shape (l,). + mask: The mask tensor, with shape (l,) + Outputs: + output: The quantized tensor in FP4, with shape (m, k // 2, l) but the physical + layout is (l, m, k // 2). `// 2` is because two fp4 values are packed into + an uint8. + output_scales: The blockscale tensor in FP8-E4M3, with shape (32, 4, rm, 4, rk, l) + but the physical layout is (l, rm, rk, 32, 4, 4). + Note: + For the shape of output_scales, `32 * 4 * rm` is a padded m to nearest multiple of 128. + `4 * rk` is a padded `k // 16` to nearest multiple of 4. These layout constants are + required by the NVIDIA Blackwell MMA operations. + """ + device = input_tensor.device + l, m, k_by_2 = input_tensor.shape + k = k_by_2 // 2 + sf_vec_size = 16 + assert k % sf_vec_size == 0, f"k must be multiple of 16, but got {k}." + + scale_k = k // sf_vec_size + padded_k = (scale_k + (4 - 1)) // 4 * 4 + padded_k_int32 = padded_k // 4 + padded_m = (m + (128 - 1)) // 128 * 128 + output = torch.empty(l, m, k // 2, device=device, dtype=torch.uint8) + output_scales = torch.empty( + l, padded_m, padded_k_int32, device=device, dtype=torch.int32 + ) + + torch.ops.sgl_kernel.silu_and_mul_scaled_fp4_experts_quant.default( + output.view(l * m, k // 2), + output_scales.view(l * padded_m, padded_k_int32), + input_tensor.view(l * m, k_by_2), + input_global_scale, + mask, + use_silu_and_mul=True, + ) + # The physical layout of the output is (l, m, k // 2), but we want to return a + # logical layout (m, k // 2, l) required by the flashinfer masked group gemm. + output = output.permute(1, 2, 0) + # The physical layout of the output scales is already swizzled as (l, rm, rk, 32, 4, 4), a + # requirement for the flashinfer masked group gemm, where rm=m/128 and rk=k/4. The logic + # layout is (32, 4, rm, 4, rk, l). + output_scales = output_scales.view(torch.float8_e4m3fn).view( + l, padded_m // 128, padded_k // 4, 32, 4, 4 + ) + output_scales = output_scales.permute(3, 4, 1, 5, 2, 0) + return output, output_scales + + +def scaled_fp4_experts_quant( + input_tensor: torch.Tensor, + input_global_scale: torch.Tensor, + expert_offsets: torch.Tensor, + blockscale_offsets: torch.Tensor, + topk: int, + expert_map: Optional[torch.Tensor] = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """ + Quantize input tensor to FP4 and return quantized tensor and scale, for + packed MoE Inputs. + Args: + input: The input tensor to be quantized to FP4 + expert_map: The expert map tensor + input_global_scale: A scalar scaling factor for the entire tensor. + expert_offsets: The expert offsets tensor + blockscale_offsets: The blockscale offsets tensor + Outputs: + output: The quantized tensor in FP4 + output_scales: The blockscale tensor in FP8-E4M3 + """ + assert ( + input_tensor.ndim == 2 + ), f"input.ndim needs to be == 2, but got {input_tensor.ndim}." + if expert_map is not None: + (m, k) = input_tensor.shape + output_tensor_shape = (m * topk, k) + input_tensor = shuffle_rows(input_tensor, expert_map, output_tensor_shape) + m_numtopk, k = input_tensor.shape + # Control the maximum number of tokens per expert supported by the + # NVFP4 MoE Expert Quantization. This is used to prevent the kernel + # from running out of memory. This value can also be increased to support + # larger models. + import os + + MAX_TOKENS_PER_EXPERT = os.environ.get("MODELOPT_MAX_TOKENS_PER_EXPERT", 65536) + assert m_numtopk <= MAX_TOKENS_PER_EXPERT * topk, ( + f"m_numtopk must be less than MAX_TOKENS_PER_EXPERT(" + f"{MAX_TOKENS_PER_EXPERT})" + f" for cutlass_moe_fp4, observed m_numtopk = {m_numtopk}. Use" + f" MODELOPT_MAX_TOKENS_PER_EXPERT to set this value." + ) + scales_k = k // 16 + padded_k = (scales_k + (4 - 1)) // 4 + + # output is uint8 and packed fp4 values + output = torch.empty( + m_numtopk, k // 2, device=input_tensor.device, dtype=torch.uint8 + ) + # padded part should be zeroed out + if padded_k > scales_k: + output_scales = torch.zeros( + MAX_TOKENS_PER_EXPERT * topk, + padded_k, + dtype=torch.int32, + device=input_tensor.device, + ) + else: + output_scales = torch.empty( + MAX_TOKENS_PER_EXPERT * topk, + padded_k, + dtype=torch.int32, + device=input_tensor.device, + ) + torch.ops.sgl_kernel.scaled_fp4_experts_quant.default( + output, + output_scales, + input_tensor, + input_global_scale, + expert_offsets, + blockscale_offsets, + ) + output_scales = output_scales.view(torch.float8_e4m3fn) + return output, output_scales + + +# GPTQ kernels +def gptq_marlin_gemm( + a: torch.Tensor, + c: Optional[torch.Tensor], + b_q_weight: torch.Tensor, + b_scales: torch.Tensor, + global_scale: Optional[torch.Tensor], + b_zeros: Optional[torch.Tensor], + g_idx: Optional[torch.Tensor], + perm: Optional[torch.Tensor], + workspace: torch.Tensor, + b_q_type: ScalarType, + size_m: int, + size_n: int, + size_k: int, + is_k_full: bool = True, + use_atomic_add: bool = False, + use_fp32_reduce: bool = False, + is_zp_float: bool = False, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_marlin_gemm( + a, + c, + b_q_weight, + b_scales, + global_scale, + b_zeros, + g_idx, + perm, + workspace, + b_q_type.id, + size_m, + size_n, + size_k, + is_k_full, + use_atomic_add, + use_fp32_reduce, + is_zp_float, + ) + + +def gptq_gemm( + a: torch.Tensor, + b_q_weight: torch.Tensor, + b_gptq_qzeros: torch.Tensor, + b_gptq_scales: torch.Tensor, + b_g_idx: torch.Tensor, + use_shuffle: bool, + bit: int, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit + ) + + +def gptq_shuffle(q_weight: torch.Tensor, q_perm: torch.Tensor, bit: int) -> None: + torch.torch.ops.sgl_kernel.gptq_shuffle(q_weight, q_perm, bit) diff --git a/sgl-kernel/python/sgl_kernel/grammar.py b/sgl-kernel/python/sgl_kernel/grammar.py new file mode 100644 index 000000000..971f94bc2 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/grammar.py @@ -0,0 +1,15 @@ +from typing import List, Optional, Union + +import torch + + +def apply_token_bitmask_inplace_cuda( + logits: torch.Tensor, + bitmask: torch.Tensor, + indices: Optional[Union[List[int], torch.Tensor]] = None, +) -> None: + if isinstance(indices, list): + indices = torch.tensor(indices, dtype=torch.int32, device=logits.device) + if indices is not None: + indices = indices.to(logits.device) + torch.ops.sgl_kernel.apply_token_bitmask_inplace_cuda(logits, bitmask, indices) diff --git a/sgl-kernel/python/sgl_kernel/kvcacheio.py b/sgl-kernel/python/sgl_kernel/kvcacheio.py new file mode 100644 index 000000000..5714b6a0d --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/kvcacheio.py @@ -0,0 +1,243 @@ +from typing import List + +import torch + + +def is_hip() -> bool: + return torch.version.hip is not None + + +_is_hip = is_hip() + + +def transfer_kv_per_layer( + src_k: torch.Tensor, + dst_k: torch.Tensor, + src_v: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + item_size, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_per_layer_pf_lf( + src_k: torch.Tensor, + dst_k: torch.Tensor, + src_v: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + item_size: int, + src_layout_dim: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_pf_lf( + src_k, + dst_k, + src_v, + dst_v, + src_indices, + dst_indices, + layer_id, + item_size, + src_layout_dim, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer( + src_k_layers: torch.Tensor, + dst_k_layers: torch.Tensor, + src_v_layers: torch.Tensor, + dst_v_layers: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer( + src_k_layers, + dst_k_layers, + src_v_layers, + dst_v_layers, + src_indices, + dst_indices, + item_size, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_lf_pf( + src_k_layers: torch.Tensor, + dst_k: torch.Tensor, + src_v_layers: torch.Tensor, + dst_v: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + dst_layout_dim: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_lf_pf( + src_k_layers, + dst_k, + src_v_layers, + dst_v, + src_indices, + dst_indices, + item_size, + dst_layout_dim, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_direct( + src_layers: List[torch.Tensor], + dst_layers: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_direct( + src_layers, dst_layers, src_indices, dst_indices, page_size + ) + + +def transfer_kv_per_layer_direct_pf_lf( + src_ptrs: List[torch.Tensor], + dst_ptrs: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_direct_pf_lf( + src_ptrs, dst_ptrs, src_indices, dst_indices, layer_id, page_size + ) + + +def transfer_kv_all_layer_direct_lf_pf( + src_ptrs: List[torch.Tensor], + dst_ptrs: List[torch.Tensor], + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + page_size: int, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_direct_lf_pf( + src_ptrs, dst_ptrs, src_indices, dst_indices, page_size + ) + + +def transfer_kv_per_layer_mla( + src: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_mla( + src, + dst, + src_indices, + dst_indices, + item_size, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_per_layer_mla_pf_lf( + src: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + layer_id: int, + item_size: int, + src_layout_dim: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_per_layer_mla_pf_lf( + src, + dst, + src_indices, + dst_indices, + layer_id, + item_size, + src_layout_dim, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_mla( + src_layers: torch.Tensor, + dst_layers: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_mla( + src_layers, + dst_layers, + src_indices, + dst_indices, + item_size, + num_layers, + block_quota, + num_warps_per_block, + ) + + +def transfer_kv_all_layer_mla_lf_pf( + src_layers: torch.Tensor, + dst: torch.Tensor, + src_indices: torch.Tensor, + dst_indices: torch.Tensor, + item_size: int, + dst_layout_dim: int, + num_layers: int, + block_quota: int = 2, + num_warps_per_block: int = 16 if _is_hip else 32, +): + torch.ops.sgl_kernel.transfer_kv_all_layer_mla_lf_pf( + src_layers, + dst, + src_indices, + dst_indices, + item_size, + dst_layout_dim, + num_layers, + block_quota, + num_warps_per_block, + ) diff --git a/sgl-kernel/python/sgl_kernel/mamba.py b/sgl-kernel/python/sgl_kernel/mamba.py new file mode 100644 index 000000000..85aa5b947 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/mamba.py @@ -0,0 +1,50 @@ +from typing import Optional + +import torch + + +# mamba +def causal_conv1d_fwd( + x: torch.Tensor, + weight: torch.Tensor, + bias_: Optional[torch.Tensor], + conv_states: Optional[torch.Tensor], + query_start_loc: Optional[torch.Tensor], + cache_indices: Optional[torch.Tensor], + has_initial_state: Optional[torch.Tensor], + silu_activation: bool, + pad_slot_id: int, +): + torch.ops.sgl_kernel.causal_conv1d_fwd( + x, + weight, + bias_, + conv_states, + query_start_loc, + cache_indices, + has_initial_state, + silu_activation, + pad_slot_id, + ) + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias_: Optional[torch.Tensor], + silu_activation: bool, + cache_seqlens: Optional[torch.Tensor], + conv_state_indices: Optional[torch.Tensor], + pad_slot_id: int, +): + torch.ops.sgl_kernel.causal_conv1d_update( + x, + conv_state, + weight, + bias_, + silu_activation, + cache_seqlens, + conv_state_indices, + pad_slot_id, + ) diff --git a/sgl-kernel/python/sgl_kernel/marlin.py b/sgl-kernel/python/sgl_kernel/marlin.py new file mode 100644 index 000000000..823d3fd4b --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/marlin.py @@ -0,0 +1,44 @@ +import torch + + +def gptq_marlin_repack( + b_q_weight, + perm, + size_k, + size_n, + num_bits, +) -> torch.Tensor: + return torch.ops.sgl_kernel.gptq_marlin_repack( + b_q_weight, + perm, + size_k, + size_n, + num_bits, + ) + + +def awq_marlin_repack( + b_q_weight: torch.Tensor, size_k: int, size_n: int, num_bits: int +) -> torch.Tensor: + return torch.ops.sgl_kernel.awq_marlin_repack(b_q_weight, size_k, size_n, num_bits) + + +def awq_marlin_moe_repack( + b_q_weight: torch.Tensor, + perm: torch.Tensor, + size_k: int, + size_n: int, + num_bits: int, +) -> torch.Tensor: + num_experts = b_q_weight.shape[0] + assert size_k % 16 == 0 + output = torch.empty( + (num_experts, size_k // 16, size_n * (num_bits // 2)), + device=b_q_weight.device, + dtype=b_q_weight.dtype, + ) + for e in range(num_experts): + output[e] = torch.ops.sgl_kernel.awq_marlin_repack( + b_q_weight[e], size_k, size_n, num_bits + ) + return output diff --git a/sgl-kernel/python/sgl_kernel/memory.py b/sgl-kernel/python/sgl_kernel/memory.py new file mode 100644 index 000000000..eb997db0c --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/memory.py @@ -0,0 +1,18 @@ +import torch + + +def set_kv_buffer_kernel( + k_cache: torch.Tensor, + v_cache: torch.Tensor, + loc: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + fallback: bool = False, +): + try: + if fallback: + raise RuntimeError("Fallback to torch implementation") + torch.ops.sgl_kernel.store_kv_cache(k_cache, v_cache, loc, k, v) + except RuntimeError: # ok, fallback to torch implementation + k_cache[loc] = k + v_cache[loc] = v diff --git a/sgl-kernel/python/sgl_kernel/moe.py b/sgl-kernel/python/sgl_kernel/moe.py new file mode 100755 index 000000000..66fec9f2b --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/moe.py @@ -0,0 +1,197 @@ +from typing import Any, Dict, Optional + +import torch + + +def moe_align_block_size( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + cumsum_buffer, + pad_sorted_token_ids=False, +): + torch.ops.sgl_kernel.moe_align_block_size.default( + topk_ids, + num_experts, + block_size, + sorted_token_ids, + experts_ids, + num_tokens_post_pad, + cumsum_buffer, + pad_sorted_token_ids, + ) + + +def topk_softmax( + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + gating_output: float, + renormalize: bool = False, +) -> None: + torch.ops.sgl_kernel.topk_softmax.default( + topk_weights, topk_ids, gating_output, renormalize + ) + + +def moe_fused_gate( + input_tensor, + bias, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts=0, + routed_scaling_factor=0, + apply_routed_scaling_factor_on_output=False, +): + # This fused kernel function is used to select topk expert in a hierarchical 2-layer fashion + # it split group of expert into num_expert_group, and use top2 expert weight sum in each group + # as the group weight to select expert groups and then select topk experts within the selected groups + # the #experts is decided by the input tensor shape and we currently only support power of 2 #experts + # and #experts should be divisible by num_expert_group. #expert/num_expert_group <= 32 is limited for now. + # for non-supported case, we suggest to use the biased_grouped_topk func in sglang.srt.layers.moe.topk + # num_fused_shared_experts: if > 0, the last several experts will be + # replaced with shared experts. the shared experts will be divided by the + # routed_scaling_factor - this is intended to cancel out later when routed+shared + # output is scaled so that shared experts are not scaled. + # routed_scaling_factor: if > 0, the experts will be scaled by this factor + # apply_routed_scaling_factor_on_output: if true, output will be + # scaled by the routed_scaling_factor + return torch.ops.sgl_kernel.moe_fused_gate.default( + input_tensor, + bias, + num_expert_group, + topk_group, + topk, + num_fused_shared_experts, + routed_scaling_factor, + apply_routed_scaling_factor_on_output, + ) + + +def fp8_blockwise_scaled_grouped_mm( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace, +): + torch.ops.sgl_kernel.fp8_blockwise_scaled_grouped_mm.default( + output, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a, + b, + scales_a, + scales_b, + stride_a, + stride_b, + stride_c, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets, + workspace, + ) + + +def prepare_moe_input( + topk_ids, + expert_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + blockscale_offsets: Optional[torch.Tensor] = None, +): + torch.ops.sgl_kernel.prepare_moe_input.default( + topk_ids, + expert_offsets, + blockscale_offsets, + problem_sizes1, + problem_sizes2, + input_permutation, + output_permutation, + num_experts, + n, + k, + ) + + +def apply_shuffle_mul_sum( + input, + output, + permutation, + factors, +): + torch.ops.sgl_kernel.apply_shuffle_mul_sum.default( + input, output, permutation, factors + ) + + +def cutlass_fp4_group_mm( + a_fp4, + b_fp4, + a_blockscale, + b_blockscale, + alphas, + out_dtype, + device, + params: Dict[str, Any], +): + """ + An FP4 Blockscaled Group Gemm that takes in a_tensors, b_tensors and runs + the gemms for each combination based on the specified problem sizes. + + This is used as the MoE gemm during NVFP4 Quantized FusedMoE forward. + - a/b_tensors: the NVFP4 a_ptrs and b_ptrs tensors which are quantized + input and expert weights. + - a_/b_scales: The blockscales in FP8-E4M3 precision + - ab_strides/c_strides: Strides for the a/b tensors between rows. + - expert_offsets/sf_offsets: Indices that mark at which token index + each expert begins its computation. The number of tokens + computed with expert E is expert_offsets[E + 1] - + expert_offsets[E] And the sf_size per expert is + sf_offset[E+1] - sf_offset[E] + - problem_sizes: MxNxK sizes of each expert's multiplication in two grouped + MMs used in the fused MoE operation. + """ + m_topk = a_fp4.shape[0] + n = b_fp4.shape[1] + c_shape = (m_topk, n) + c = torch.empty(c_shape, device=device, dtype=out_dtype) + torch.ops.sgl_kernel.cutlass_fp4_group_mm.default( + c, + a_fp4, + b_fp4, + a_blockscale, + b_blockscale, + alphas, + params["ab_strides"], + params["c_strides"], + params["problem_sizes"], + params["expert_offsets"], + params["blockscale_offsets"], + ) + return c.to(dtype=out_dtype) diff --git a/sgl-kernel/python/sgl_kernel/sampling.py b/sgl-kernel/python/sgl_kernel/sampling.py new file mode 100644 index 000000000..4ee6f24d3 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/sampling.py @@ -0,0 +1,543 @@ +from typing import Optional, Union + +import torch +from sgl_kernel.utils import _to_tensor_scalar_tuple + + +def _top_k_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, +) -> torch.Tensor: + probs = probs.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_k_renorm_probs.default( + probs, renorm_probs, maybe_top_k_arr, top_k_val + ) + return renorm_probs + + +def top_k_renorm_probs( + probs: torch.Tensor, + top_k: Union[torch.Tensor, int], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for renormalizing probabilities by top-k thresholding. + + Parameters + ---------- + probs: torch.Tensor + Probabilities, shape ``(batch_size, num_classes)``. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for + for re-normalizing probabilities, should be in ``(0, num_classes)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We keep the top-k probabilities, set the rest to zero, and renormalize the probabilities. + + Returns + ------- + renorm_probs: torch.Tensor + Renormalized probabilities, shape ``(batch_size, num_classes)``. + + Note + ---- + This combination of ``top_k_renorm_probs`` and ``sampling_from_probs`` should be equivalent to + ``top_k_sampling_from_probs``. + """ + return _top_k_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_k)) + + +top_k_renorm_prob = top_k_renorm_probs + + +def _top_p_renorm_probs_internal( + probs: torch.Tensor, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, +) -> torch.Tensor: + probs = probs.float() + maybe_top_p_arr = maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + renorm_probs = torch.empty_like(probs) + torch.ops.sgl_kernel.top_p_renorm_probs.default( + probs, renorm_probs, maybe_top_p_arr, top_p_val + ) + return renorm_probs + + +def top_p_renorm_probs( + probs: torch.Tensor, + top_p: Union[torch.Tensor, float], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for renormalizing probabilities by top-p thresholding. + + Parameters + ---------- + probs: torch.Tensor + Probabilities, shape ``(batch_size, num_classes)``. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-p threshold for for + re-normalizing probabilities, should be in ``(0, 1)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We mask out the probabilities less than `threshold` where the cumulative sum + of ``probs[probs >= threshold]`` is `top_p`, and renormalize the probabilities. + + Returns + ------- + renorm_probs: torch.Tensor + Renormalized probabilities, shape ``(batch_size, num_classes)``. + + Note + ---- + This combination of ``top_p_renorm_probs`` and ``sampling_from_probs`` should be equivalent to + ``top_p_sampling_from_probs``. + + """ + return _top_p_renorm_probs_internal(probs, *_to_tensor_scalar_tuple(top_p)) + + +top_p_renorm_prob = top_p_renorm_probs + + +def _top_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.top_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_top_p_arr, + top_p_val, + deterministic, + generator, + ) + return samples + + +def top_p_sampling_from_probs( + probs: torch.Tensor, + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-p sampling (nucleus sampling) from probabilities, + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_p_sampling_from_probs_internal( + probs, indices, *_to_tensor_scalar_tuple(top_p), deterministic, generator + ) + + +def _top_k_top_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, + maybe_top_p_arr: Optional[torch.Tensor], + top_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + maybe_top_p_arr = ( + maybe_top_p_arr.float() if maybe_top_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.top_k_top_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_top_k_arr, + top_k_val, + maybe_top_p_arr, + top_p_val, + deterministic, + generator, + ) + return samples + + +def top_k_top_p_sampling_from_probs( + probs: torch.Tensor, + top_k: Union[torch.Tensor, int], + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + filter_apply_order: str = "top_k_first", + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-k and top-p sampling from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + filter_apply_order: str + The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. + If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. + If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if filter_apply_order == "top_k_first": + renorm_probs = top_k_renorm_probs(probs, top_k) + return top_p_sampling_from_probs( + renorm_probs, + top_p, + indices, + deterministic, + check_nan=check_nan, + generator=generator, + ) + elif filter_apply_order == "joint": + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_k_top_p_sampling_from_probs_internal( + probs, + indices, + *_to_tensor_scalar_tuple(top_k), + *_to_tensor_scalar_tuple(top_p), + deterministic, + generator, + ) + else: + raise ValueError(f"Invalid filter_apply_order: {filter_apply_order}") + + +def _min_p_sampling_from_probs_internal( + probs: torch.Tensor, + indices: Optional[torch.Tensor], + maybe_min_p_arr: Optional[torch.Tensor], + min_p_val: float, + deterministic: bool, + generator: Optional[torch.Generator], +) -> torch.Tensor: + with probs.device as device: + probs = probs.float() + maybe_min_p_arr = ( + maybe_min_p_arr.float() if maybe_min_p_arr is not None else None + ) + samples = torch.empty(probs.size(0), dtype=torch.int32, device=device) + torch.ops.sgl_kernel.min_p_sampling_from_probs.default( + probs, + samples, + indices, + maybe_min_p_arr, + min_p_val, + deterministic, + generator, + ) + return samples + + +def min_p_sampling_from_probs( + probs: torch.Tensor, + min_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for `min_p sampling `_ from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + probs: torch.Tensor + Probabilities for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of probabilities. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + min_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for min-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + """ + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _min_p_sampling_from_probs_internal( + probs, indices, *_to_tensor_scalar_tuple(min_p), deterministic, generator + ) + + +def _top_k_mask_logits_internal( + logits: torch.Tensor, + maybe_top_k_arr: Optional[torch.Tensor], + top_k_val: int, +) -> torch.Tensor: + logits = logits.float() + maybe_top_k_arr = maybe_top_k_arr.int() if maybe_top_k_arr is not None else None + mask_logits = torch.empty_like(logits) + torch.ops.sgl_kernel.top_k_mask_logits.default( + logits, mask_logits, maybe_top_k_arr, top_k_val + ) + return mask_logits + + +def top_k_mask_logits( + logits: torch.Tensor, + top_k: Union[torch.Tensor, int], +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for masking logits by top-k thresholding. + + Parameters + ---------- + logits: torch.Tensor + Logits before softmax, shape ``(batch_size, num_classes)``. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the top-k threshold for for + for masking logits, should be in ``(0, num_classes)``. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + We keep the top-k logits, set the rest to negative infinity. + + Returns + ------- + masked_logits: torch.Tensor + Masked logits, shape ``(batch_size, num_classes)``. + + Examples + -------- + + >>> import torch + >>> import flashinfer + >>> torch.manual_seed(42) + >>> batch_size = 4 + >>> vocab_size = 5 + >>> top_k = 3 + >>> logits = torch.randn(batch_size, vocab_size).to(0) + >>> logits + tensor([[ 1.9269, 1.4873, 0.9007, -2.1055, -0.7581], + [ 1.0783, 0.8008, 1.6806, 0.3559, -0.6866], + [-0.4934, 0.2415, -0.2316, 0.0418, -0.2516], + [ 0.8599, -0.3097, -0.3957, 0.8034, -0.6216]], device='cuda:0') + >>> masked_logits = flashinfer.sampling.top_k_mask_logits(logits, top_k) + >>> masked_logits + tensor([[ 1.9269, 1.4873, 0.9007, -inf, -inf], + [ 1.0783, 0.8008, 1.6806, -inf, -inf], + [ -inf, 0.2415, -0.2316, 0.0418, -inf], + [ 0.8599, -0.3097, -inf, 0.8034, -inf]], device='cuda:0') + + Note + ---- + The combination of ``top_k_mask_logits`` and ``softmax`` should be equivalent to ``top_k_renorm_probs``. + + See Also + -------- + top_k_renorm_probs + """ + return _top_k_mask_logits_internal(logits, *_to_tensor_scalar_tuple(top_k)) + + +def top_k_top_p_sampling_from_logits( + logits: torch.Tensor, + top_k: Union[torch.Tensor, int], + top_p: Union[torch.Tensor, float], + indices: Optional[torch.Tensor] = None, + filter_apply_order: str = "top_k_first", + deterministic: bool = True, + generator: Optional[torch.Generator] = None, + check_nan: bool = False, +) -> torch.Tensor: + r"""Adapt from https://github.com/flashinfer-ai/flashinfer/flashinfer/sampling.py + Fused GPU kernel for top-k and top-p sampling from probabilities, + + this operator implements GPU-based rejection sampling without explicit sorting. + Check the `blog post `_ for more details. + + The multiple rounds of rejection sampling are implemented in a single CUDA kernel, + which is more efficient than the naive implementation that launches a series of kernels. + + Parameters + ---------- + logits: torch.Tensor + Pre-softmax logits for sampling. When indices is not provided, shape should be ``(batch_size, num_classes)`` + and the i-th output will be sampled from the i-th row of logits. When indices is provided, + shape should be ``(unique_batch_size, num_classes)`` where unique_batch_size is the number of unique + probability distributions. + top_k: Union[torch.Tensor, int] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-k sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + top_p: Union[torch.Tensor, float] + Either a scalar or a tensor of shape ``(batch_size,)``, representing the threshold for top-p sampling. + If a scalar, the same threshold is used for all requests. + If a tensor, each request has its own threshold. + indices: Optional[torch.Tensor] + Optional indices tensor of shape ``(batch_size,)`` that maps each output to a row in probs. + For example, if indices[i] = j, then the i-th output will be sampled from probs[j]. + This allows reusing the same probability distribution for multiple outputs. + If indices is not provided, the i-th output will be sampled from the i-th row of probs. + filter_apply_order: str + The order of applying top-k and top-p sampling, should be either ``"top_k_first"`` or ``"joint"``. + If ``"top_k_first"``, we first apply top-k filter, then apply top-p sampling on the top-k results. + If ``"joint"``, we apply top-k and top-p filter simultaneously in each round. Default is ``"top_k_first"``. + deterministic: bool + Whether to use deterministic kernel implementation, default is ``True``. + generator: Optional[torch.Generator] + A random number generator for the operation. + check_nan: bool + Whether to check nan in :attr:`probs`, default is ``False``. + + Returns + ------- + samples: torch.Tensor + Sampled categories, shape ``(batch_size,)``. + + Note + ---- + This function expects float32 inputs, and the output is int32. + + """ + if filter_apply_order == "top_k_first": + masked_logits = top_k_mask_logits(logits, top_k) + probs = torch.softmax(masked_logits, dim=-1) + return top_p_sampling_from_probs( + probs, + top_p, + indices, + deterministic, + check_nan=check_nan, + generator=generator, + ) + elif filter_apply_order == "joint": + probs = torch.softmax(logits, dim=-1) + if check_nan: + if torch.any(torch.isnan(probs)): + raise ValueError("Input probs contains NaN.") + return _top_k_top_p_sampling_from_probs_internal( + probs, + indices, + *_to_tensor_scalar_tuple(top_k), + *_to_tensor_scalar_tuple(top_p), + deterministic, + generator, + ) + else: + raise ValueError(f"Invalid filter_apply_order: {filter_apply_order}") diff --git a/sgl-kernel/python/sgl_kernel/scalar_type.py b/sgl-kernel/python/sgl_kernel/scalar_type.py new file mode 100644 index 000000000..571c1dca7 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/scalar_type.py @@ -0,0 +1,352 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import functools +import struct +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Union + +_SCALAR_TYPES_ID_MAP = {} + + +# Mirrors enum in `core/scalar_type.hpp` +class NanRepr(Enum): + NONE = 0 # nans are not supported + IEEE_754 = 1 # nans are: Exp all 1s, mantissa not all 0s + EXTD_RANGE_MAX_MIN = 2 # nans are: Exp all 1s, mantissa all 1s + + +# This ScalarType class is a parallel implementation of the C++ ScalarType +# class found in csrc/core/scalar_type.hpp. These two classes should be kept +# in sync until the inductor fully supports custom C++ classes. +@dataclass(frozen=True) +class ScalarType: + """ + ScalarType can represent a wide range of floating point and integer + types, in particular it can be used to represent sub-byte data types + (something that torch.dtype currently does not support). It is also + capable of representing types with a bias, i.e.: + `stored_value = value + bias`, + this is useful for quantized types (e.g. standard GPTQ 4bit uses a bias + of 8). The implementation for this class can be found in + csrc/core/scalar_type.hpp, these type signatures should be kept in sync + with that file. + """ + + exponent: int + """ + Number of bits in the exponent if this is a floating point type + (zero if this an integer type) + """ + + mantissa: int + """ + Number of bits in the mantissa if this is a floating point type, + or the number bits representing an integer excluding the sign bit if + this an integer type. + """ + + signed: bool + "If the type is signed (i.e. has a sign bit)" + + bias: int + """ + bias used to encode the values in this scalar type + (value = stored_value - bias, default 0) for example if we store the + type as an unsigned integer with a bias of 128 then the value 0 will be + stored as 128 and -1 will be stored as 127 and 1 will be stored as 129. + """ + + _finite_values_only: bool = False + """ + Private: if infs are supported, used `has_infs()` instead. + """ + + nan_repr: NanRepr = NanRepr.IEEE_754 + """ + How NaNs are represent in this scalar type, returns NanRepr value. + (not applicable for integer types) + """ + + def _floating_point_max_int(self) -> int: + assert ( + self.mantissa <= 52 and self.exponent <= 11 + ), f"Cannot represent max/min as a double for type {self.__str__()}" + + max_mantissa = (1 << self.mantissa) - 1 + if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN: + max_mantissa = max_mantissa - 1 + + max_exponent = (1 << self.exponent) - 2 + if self.nan_repr == NanRepr.EXTD_RANGE_MAX_MIN or self.nan_repr == NanRepr.NONE: + assert ( + self.exponent < 11 + ), f"Cannot represent max/min as a double for type {self.__str__()}" + max_exponent = max_exponent + 1 + + # adjust the exponent to match that of a double + # for now we assume the exponent bias is the standard 2^(e-1) -1, (where + # e is the exponent bits), there is some precedent for non-standard + # biases, example `float8_e4m3b11fnuz` here: + # https://github.com/jax-ml/ml_dtypes but to avoid premature over + # complication we are just assuming the standard exponent bias until + # there is a need to support non-standard biases + exponent_bias = (1 << (self.exponent - 1)) - 1 + exponent_bias_double = (1 << 10) - 1 # double e = 11 + + max_exponent_double = max_exponent - exponent_bias + exponent_bias_double + + # shift the mantissa and exponent into the proper positions for an + # IEEE double and bitwise-or them together. + return (max_mantissa << (52 - self.mantissa)) | (max_exponent_double << 52) + + def _floating_point_max(self) -> float: + double_raw = self._floating_point_max_int() + return struct.unpack("!d", struct.pack("!Q", double_raw))[0] + + def _raw_max(self) -> Union[int, float]: + if self.is_floating_point(): + return self._floating_point_max() + else: + assert ( + self.size_bits < 64 or self.size_bits == 64 and self.is_signed() + ), "Cannot represent max as an int" + return (1 << self.mantissa) - 1 + + def _raw_min(self) -> Union[int, float]: + if self.is_floating_point(): + assert ( + self.is_signed() + ), "We currently assume all floating point types are signed" + sign_bit_double = 1 << 63 + + max_raw = self._floating_point_max_int() + min_raw = max_raw | sign_bit_double + return struct.unpack("!d", struct.pack("!Q", min_raw))[0] + else: + assert ( + not self.is_signed() or self.size_bits <= 64 + ), "Cannot represent min as a int64_t" + + if self.is_signed(): + return -(1 << (self.size_bits - 1)) + else: + return 0 + + @functools.cached_property + def id(self) -> int: + """ + Convert the ScalarType to an int which can be passed to pytorch custom + ops. This layout of the int must be kept in sync with the C++ + ScalarType's from_id method. + """ + val = 0 + offset = 0 + + def or_and_advance(member, bit_width): + nonlocal val + nonlocal offset + bit_mask = (1 << bit_width) - 1 + val = val | (int(member) & bit_mask) << offset + offset = offset + bit_width + + or_and_advance(self.exponent, 8) + or_and_advance(self.mantissa, 8) + or_and_advance(self.signed, 1) + or_and_advance(self.bias, 32) + or_and_advance(self._finite_values_only, 1) + or_and_advance(self.nan_repr.value, 8) + + assert offset <= 64, f"ScalarType fields too big {offset} to fit into an int64" + + _SCALAR_TYPES_ID_MAP[val] = self + + return val + + @property + def size_bits(self) -> int: + return self.exponent + self.mantissa + int(self.signed) + + def min(self) -> Union[int, float]: + """ + Min representable value for this scalar type. + (accounting for bias if there is one) + """ + return self._raw_min() - self.bias + + def max(self) -> Union[int, float]: + """ + Max representable value for this scalar type. + (accounting for bias if there is one) + """ + return self._raw_max() - self.bias + + def is_signed(self) -> bool: + """ + If the type is signed (i.e. has a sign bit), same as `signed` + added for consistency with: + https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html + """ + return self.signed + + def is_floating_point(self) -> bool: + "If the type is a floating point type" + return self.exponent != 0 + + def is_integer(self) -> bool: + "If the type is an integer type" + return self.exponent == 0 + + def has_bias(self) -> bool: + "If the type has a non-zero bias" + return self.bias != 0 + + def has_infs(self) -> bool: + "If the type is floating point and supports infinity" + return not self._finite_values_only + + def has_nans(self) -> bool: + return self.nan_repr != NanRepr.NONE + + def is_ieee_754(self) -> bool: + """ + If the type is a floating point type that follows IEEE 754 + conventions + """ + return self.nan_repr == NanRepr.IEEE_754 and not self._finite_values_only + + def __str__(self) -> str: + """ + naming generally follows: https://github.com/jax-ml/ml_dtypes + for floating point types (leading f) the scheme is: + `float_em[flags]` + flags: + - no-flags: means it follows IEEE 754 conventions + - f: means finite values only (no infinities) + - n: means nans are supported (non-standard encoding) + for integer types the scheme is: + `[u]int[b]` + - if bias is not present it means its zero + """ + if self.is_floating_point(): + ret = ( + "float" + + str(self.size_bits) + + "_e" + + str(self.exponent) + + "m" + + str(self.mantissa) + ) + + if not self.is_ieee_754(): + if self._finite_values_only: + ret = ret + "f" + if self.nan_repr != NanRepr.NONE: + ret = ret + "n" + + return ret + else: + ret = ("int" if self.is_signed() else "uint") + str(self.size_bits) + if self.has_bias(): + ret = ret + "b" + str(self.bias) + return ret + + def __repr__(self) -> str: + return "ScalarType." + self.__str__() + + # __len__ needs to be defined (and has to throw TypeError) for pytorch's + # opcheck to work. + def __len__(self) -> int: + raise TypeError + + # + # Convenience Constructors + # + + @classmethod + def int_(cls, size_bits: int, bias: Optional[int]) -> "ScalarType": + "Create a signed integer scalar type (size_bits includes sign-bit)." + ret = cls(0, size_bits - 1, True, bias if bias else 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def uint(cls, size_bits: int, bias: Optional[int]) -> "ScalarType": + """Create a unsigned integer scalar type.""" + ret = cls(0, size_bits, False, bias if bias else 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def float_IEEE754(cls, exponent: int, mantissa: int) -> "ScalarType": + """ + Create a standard floating point type + (i.e. follows IEEE 754 conventions). + """ + assert mantissa > 0 and exponent > 0 + ret = cls(exponent, mantissa, True, 0) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def float_( + cls, exponent: int, mantissa: int, finite_values_only: bool, nan_repr: NanRepr + ) -> "ScalarType": + """ + Create a non-standard floating point type + (i.e. does not follow IEEE 754 conventions). + """ + assert mantissa > 0 and exponent > 0 + assert nan_repr != NanRepr.IEEE_754, ( + "use `float_IEEE754` constructor for floating point types that " + "follow IEEE 754 conventions" + ) + ret = cls(exponent, mantissa, True, 0, finite_values_only, nan_repr) + ret.id # noqa B018: make sure the id is cached + return ret + + @classmethod + def from_id(cls, scalar_type_id: int): + if scalar_type_id not in _SCALAR_TYPES_ID_MAP: + raise ValueError(f"scalar_type_id {scalar_type_id} doesn't exists.") + return _SCALAR_TYPES_ID_MAP[scalar_type_id] + + +# naming generally follows: https://github.com/jax-ml/ml_dtypes +# for floating point types (leading f) the scheme is: +# `float_em[flags]` +# flags: +# - no-flags: means it follows IEEE 754 conventions +# - f: means finite values only (no infinities) +# - n: means nans are supported (non-standard encoding) +# for integer types the scheme is: +# `[u]int[b]` +# - if bias is not present it means its zero + + +class scalar_types: + int4 = ScalarType.int_(4, None) + uint4 = ScalarType.uint(4, None) + int8 = ScalarType.int_(8, None) + uint8 = ScalarType.uint(8, None) + float8_e4m3fn = ScalarType.float_(4, 3, True, NanRepr.EXTD_RANGE_MAX_MIN) + float8_e5m2 = ScalarType.float_IEEE754(5, 2) + float16_e8m7 = ScalarType.float_IEEE754(8, 7) + float16_e5m10 = ScalarType.float_IEEE754(5, 10) + + # fp6, https://github.com/usyd-fsalab/fp6_llm/tree/main + float6_e3m2f = ScalarType.float_(3, 2, True, NanRepr.NONE) + + # fp4, https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + float4_e2m1f = ScalarType.float_(2, 1, True, NanRepr.NONE) + + # "gptq" types + uint2b2 = ScalarType.uint(2, 2) + uint3b4 = ScalarType.uint(3, 4) + uint4b8 = ScalarType.uint(4, 8) + uint8b128 = ScalarType.uint(8, 128) + + # colloquial names + bfloat16 = float16_e8m7 + float16 = float16_e5m10 diff --git a/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py b/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py new file mode 100644 index 000000000..29b2f0405 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/sparse_flash_attn.py @@ -0,0 +1,293 @@ +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + + +def maybe_contiguous(x): + return x.contiguous() if x is not None and x.stride(-1) != 1 else x + + +# Sparse attention utils +def convert_vertical_slash_indexes( + q_seqlens: torch.Tensor, # [BATCH, ] + kv_seqlens: torch.Tensor, # [BATCH, ] + vertical_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_V] + slash_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_S] + context_size: int, + block_size_M: int, + block_size_N: int, + causal: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size = slash_indexes.size(0) + num_heads = slash_indexes.size(1) + nnz_slash = slash_indexes.size(2) + nnz_vertical = vertical_indexes.size(2) + num_rows = (context_size + block_size_M - 1) // block_size_M + + block_count = torch.zeros( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + block_offset = torch.zeros( + batch_size, + num_heads, + num_rows, + nnz_slash, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + column_count = torch.zeros( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + column_index = torch.zeros( + batch_size, + num_heads, + num_rows, + nnz_vertical, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + + torch.ops.sgl_kernel.convert_vertical_slash_indexes.default( + block_count, + block_offset, + column_count, + column_index, + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + context_size, + block_size_M, + block_size_N, + causal, + ) + return block_count, block_offset, column_count, column_index + + +def convert_vertical_slash_indexes_mergehead( + q_seqlens: torch.Tensor, # [BATCH, ] + kv_seqlens: torch.Tensor, # [BATCH, ] + vertical_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_V] + slash_indexes: torch.Tensor, # [BATCH, N_HEADS, NNZ_S] + # [N_HEADS] : different head use different number of indices + vertical_indices_count: torch.Tensor, + slash_indices_count: torch.Tensor, + context_size: int, + block_size_M: int, + block_size_N: int, + causal: bool = True, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + batch_size = slash_indexes.size(0) + num_heads = slash_indexes.size(1) + nnz_slash = slash_indexes.size(2) + nnz_vertical = vertical_indexes.size(2) + num_rows = (context_size + block_size_M - 1) // block_size_M + + block_count = torch.empty( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + block_offset = torch.empty( + batch_size, + num_heads, + num_rows, + nnz_slash, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + column_count = torch.empty( + batch_size, num_heads, num_rows, dtype=q_seqlens.dtype, device=q_seqlens.device + ) + column_index = torch.empty( + batch_size, + num_heads, + num_rows, + nnz_vertical, + dtype=q_seqlens.dtype, + device=q_seqlens.device, + ) + + torch.ops.sgl_kernel.convert_vertical_slash_indexes_mergehead.default( + block_count, + block_offset, + column_count, + column_index, + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + vertical_indices_count, + slash_indices_count, + context_size, + block_size_M, + block_size_N, + causal, + ) + return block_count, block_offset, column_count, column_index + + +def sparse_attn_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + dropout_p=0.0, + softmax_scale=None, + causal=False, + softcap=0.0, # 0.0 means deactivated + alibi_slopes=None, + deterministic=False, + return_attn_probs=False, + *, + return_softmax_lse=False, + out=None, +): + """Compute attention with vertical and slash sparsity patterns. + Most Arguments are the same with the flash_attn_func interface, except for 4 extra args: + block_count and block_offset for slash sparsity patterns, and + column_count and column_index for vertical sparsity patterns. + For more details please refer to Appendix C.4.2 of paper https://arxiv.org/abs/2407.02490. + + Arguments: + q: (batch_size, seqlen, nheads, headdim) + k: (batch_size, seqlen, nheads_k, headdim) + v: (batch_size, seqlen, nheads_k, headdim) + block_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + block_offset: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_S) + column_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + column_index: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_V) + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + Default to 1 / sqrt(headdim). + causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). + alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of + (-alibi_slope * |i + seqlen_k - seqlen_q - j|) + is added to the attention score of query i and key j. + deterministic: bool. Whether to use the deterministic implementation of the backward pass, + which is slightly slower and uses more memory. The forward pass is always deterministic. + return_attn_probs: bool. Whether to return the attention probabilities. This option is for + testing only. The returned probabilities are not guaranteed to be correct + (they might not have the right scaling). + Return: + out: (batch_size, seqlen, nheads, headdim). + softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The + logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax + normalization factor). + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + + q, k, v = [maybe_contiguous(x) for x in (q, k, v)] + out, softmax_lse = torch.ops.sgl_kernel.fwd_sparse.default( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + out, + alibi_slopes, + dropout_p, + softmax_scale, + causal, + softcap, + return_attn_probs and dropout_p > 0, + None, + ) + return (out, softmax_lse) if return_softmax_lse else out + + +def sparse_attn_varlen_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + dropout_p=0.0, + softmax_scale=None, + causal=False, + softcap=0.0, # 0.0 means deactivated + alibi_slopes=None, + deterministic=False, + return_attn_probs=False, + *, + return_softmax_lse=False, + out=None, +): + """Compute attention with vertical and slash sparsity patterns. + Most Arguments are the same with the flash_attn_varlen_func interface, except for 4 extra args: + block_count and block_offset for slash sparsity patterns, and + column_count and column_index for vertical sparsity patterns. + For more details please refer to Appendix C.4.2 of paper https://arxiv.org/abs/2407.02490. + + Arguments: + q: (total_q, nheads, headdim), where total_q = total number of query tokens in the batch. + k: (total_k, nheads_k, headdim), where total_k = total number of key tokens in the batch. + v: (total_k, nheads_k, headdim), where total_k = total number of key tokens in the batch. + block_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + block_offset: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_S) + column_count: (batch_size, nheads, cdiv(seqlen, BLOCK_M)) + column_index: (batch_size, nheads, cdiv(seqlen, BLOCK_M), NNZ_V) + cu_seqlens_q: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths + of the sequences in the batch, used to index into q. + cu_seqlens_k: (batch_size + 1,), dtype torch.int32. The cumulative sequence lengths + of the sequences in the batch, used to index into kv. + max_seqlen_q: int. Maximum query sequence length in the batch. + max_seqlen_k: int. Maximum key sequence length in the batch. + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + Default to 1 / sqrt(headdim). + causal: bool. Whether to apply causal attention mask (e.g., for auto-regressive modeling). + softcap: float. Anything > 0 activates softcapping attention. + alibi_slopes: (nheads,) or (batch_size, nheads), fp32. A bias of + (-alibi_slope * |i + seqlen_k - seqlen_q - j|) + is added to the attention score of query i and key j. + deterministic: bool. Whether to use the deterministic implementation of the backward pass, + which is slightly slower and uses more memory. The forward pass is always deterministic. + return_attn_probs: bool. Whether to return the attention probabilities. This option is for + testing only. The returned probabilities are not guaranteed to be correct + (they might not have the right scaling). + Return: + out: (total, nheads, headdim). + softmax_lse [optional, if return_softmax_lse=True]: (nheads, total_q_seqlen). The + logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax + normalization factor). + """ + if softmax_scale is None: + softmax_scale = q.shape[-1] ** (-0.5) + + q, k, v = [maybe_contiguous(x) for x in (q, k, v)] + out, softmax_lse = torch.ops.sgl_kernel.varlen_fwd_sparse.default( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + out, + cu_seqlens_q, + cu_seqlens_k, + None, + alibi_slopes, + max_seqlen_q, + max_seqlen_k, + dropout_p, + softmax_scale, + False, + causal, + softcap, + return_attn_probs and dropout_p > 0, + None, + ) + return (out, softmax_lse) if return_softmax_lse else out diff --git a/sgl-kernel/python/sgl_kernel/spatial.py b/sgl-kernel/python/sgl_kernel/spatial.py new file mode 100644 index 000000000..201ec7b75 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/spatial.py @@ -0,0 +1,63 @@ +import torch +from torch.cuda.streams import ExternalStream + +try: + from . import spatial_ops # triggers TORCH extension registration +except Exception as _e: + _spatial_import_error = _e +else: + _spatial_import_error = None + +_IMPORT_ERROR = ImportError( + "Failed to load sgl_kernel.spatial_ops extension. Ensure CUDA Driver >= 12.4" +) + + +def create_greenctx_stream_by_value( + SM_a: int, SM_b: int, device_id: int = None +) -> tuple[ExternalStream, ExternalStream]: + """ + Create two streams for greenctx. + Args: + sm_A (int): The SM of stream A. + sm_B (int): The weight of stream B. + device_id (int): The device id. + Returns: + tuple[ExternalStream, ExternalStream]: The two streams. + """ + if _spatial_import_error is not None: + raise _IMPORT_ERROR from _spatial_import_error + if device_id is None: + device_id = torch.cuda.current_device() + + res = torch.ops.sgl_kernel.create_greenctx_stream_by_value(SM_a, SM_b, device_id) + + stream_a = ExternalStream( + stream_ptr=res[0], device=torch.device(f"cuda:{device_id}") + ) + stream_b = ExternalStream( + stream_ptr=res[1], device=torch.device(f"cuda:{device_id}") + ) + + return stream_a, stream_b + + +def get_sm_available(device_id: int = None) -> int: + """ + Get the SMs available on the device. + Args: + device_id (int): The device id. + Returns: + int: The SMs available. + """ + if _spatial_import_error is not None: + raise _IMPORT_ERROR from _spatial_import_error + if device_id is None: + device_id = torch.cuda.current_device() + + device_props = torch.cuda.get_device_properties(device_id) + + # Get the number of Streaming Multiprocessors (SMs) + sm_count = device_props.multi_processor_count + + return sm_count diff --git a/sgl-kernel/python/sgl_kernel/speculative.py b/sgl-kernel/python/sgl_kernel/speculative.py new file mode 100644 index 000000000..ea2e3ac8a --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/speculative.py @@ -0,0 +1,107 @@ +import torch +from sgl_kernel.utils import get_cuda_stream + + +def tree_speculative_sampling_target_only( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + uniform_samples: torch.Tensor, + uniform_samples_for_final_sampling: torch.Tensor, + target_probs: torch.Tensor, + draft_probs: torch.Tensor, + threshold_single: float = 1.0, + threshold_acc: float = 1.0, + deterministic: bool = True, +) -> None: + torch.ops.sgl_kernel.tree_speculative_sampling_target_only.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + uniform_samples, + uniform_samples_for_final_sampling, + target_probs, + draft_probs, + threshold_single, + threshold_acc, + deterministic, + get_cuda_stream(), + ) + + +def verify_tree_greedy( + predicts: torch.Tensor, # mutable + accept_index: torch.Tensor, # mutable + accept_token_num: torch.Tensor, # mutable + candidates: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + target_predict: torch.Tensor, +) -> None: + torch.ops.sgl_kernel.verify_tree_greedy.default( + predicts, + accept_index, + accept_token_num, + candidates, + retrive_index, + retrive_next_token, + retrive_next_sibling, + target_predict, + get_cuda_stream(), + ) + + +def build_tree_kernel_efficient( + parent_list: torch.Tensor, + selected_index: torch.Tensor, + verified_seq_len: torch.Tensor, + tree_mask: torch.Tensor, + positions: torch.Tensor, + retrive_index: torch.Tensor, + retrive_next_token: torch.Tensor, + retrive_next_sibling: torch.Tensor, + topk: int, + depth: int, + draft_token_num: int, + tree_mask_mode: int, +) -> None: + torch.ops.sgl_kernel.build_tree_kernel_efficient.default( + parent_list, + selected_index, + verified_seq_len, + tree_mask, + positions, + retrive_index, + retrive_next_token, + retrive_next_sibling, + topk, + depth, + draft_token_num, + tree_mask_mode, + ) + + +def segment_packbits( + x: torch.Tensor, + input_indptr: torch.Tensor, + output_indptr: torch.Tensor, + y: torch.Tensor, + batch_size: int, +) -> None: + torch.ops.sgl_kernel.segment_packbits.default( + x, + input_indptr, + output_indptr, + y, + batch_size, + torch.cuda.current_stream().cuda_stream, + ) diff --git a/sgl-kernel/python/sgl_kernel/testing/__init__.py b/sgl-kernel/python/sgl_kernel/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py b/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py new file mode 100644 index 000000000..e26208048 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/testing/rotary_embedding.py @@ -0,0 +1,217 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import pytest +import torch +from sgl_kernel import FusedSetKVBufferArg, apply_rope_with_cos_sin_cache_inplace + + +# vLLM torch native +def _apply_rotary_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + is_neox_style: bool, +) -> torch.Tensor: + """ + Args: + x: [num_tokens, num_heads, head_size] + cos: [num_tokens, head_size // 2] + sin: [num_tokens, head_size // 2] + is_neox_style: Whether to use the Neox-style or GPT-J-style rotary + positional embeddings. + """ + cos = cos.unsqueeze(-2).to(x.dtype) + sin = sin.unsqueeze(-2).to(x.dtype) + if is_neox_style: + x1, x2 = torch.chunk(x, 2, dim=-1) + else: + x1 = x[..., ::2] + x2 = x[..., 1::2] + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_neox_style: + return torch.cat((o1, o2), dim=-1) + else: + return torch.stack((o1, o2), dim=-1).flatten(-2) + + +class RotaryEmbedding(torch.nn.Module): + # Reference: https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/rotary_embedding.py + def __init__( + self, + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + ) -> None: + super().__init__() + self.head_size = head_size + self.rotary_dim = rotary_dim + self.max_position_embeddings = max_position_embeddings + self.base = base + self.is_neox_style = is_neox_style + self.dtype = dtype + + cache = self._compute_cos_sin_cache() + self.cos_sin_cache: torch.Tensor + self.register_buffer("cos_sin_cache", cache, persistent=False) + + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor: + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim + ) + ) + return inv_freq + + def _compute_cos_sin_cache(self) -> torch.Tensor: + """Compute the cos and sin cache.""" + inv_freq = self._compute_inv_freq(self.base) + t = torch.arange(self.max_position_embeddings, dtype=torch.float) + + freqs = torch.einsum("i,j -> ij", t, inv_freq) + cos = freqs.cos() + sin = freqs.sin() + cache = torch.cat((cos, sin), dim=-1) + return cache + + def forward_native( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """A PyTorch-native implementation of forward().""" + if offsets is not None: + positions = positions + offsets + + positions = positions.flatten() + num_tokens = positions.shape[0] + cos_sin = self.cos_sin_cache.index_select(0, positions) + + # Modification: float32 is required for the rotary embedding to work correctly + query = query.to(torch.float32) + key = key.to(torch.float32) + + cos, sin = cos_sin.chunk(2, dim=-1) + + query_shape = query.shape + query = query.view(num_tokens, -1, self.head_size) + query_rot = query[..., : self.rotary_dim] + query_pass = query[..., self.rotary_dim :] + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style) + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape) + + key_shape = key.shape + key = key.view(num_tokens, -1, self.head_size) + key_rot = key[..., : self.rotary_dim] + key_pass = key[..., self.rotary_dim :] + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style) + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape) + + # Modification: convert to the correct dtype + query = query.to(self.dtype) + key = key.to(self.dtype) + return query, key + + +class FlashInferRotaryEmbedding(RotaryEmbedding): + def forward_cuda( + self, + positions: torch.Tensor, + query: torch.Tensor, + key: torch.Tensor, + fused_set_kv_buffer_arg: Optional[FusedSetKVBufferArg] = None, + offsets: Optional[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + + apply_rope_with_cos_sin_cache_inplace( + positions=positions, + query=query, + key=key, + fused_set_kv_buffer_arg=fused_set_kv_buffer_arg, + head_size=self.head_size, + cos_sin_cache=self.cos_sin_cache, + is_neox=self.is_neox_style, + ) + + return query, key + + +class MHATokenToKVPool: + KV_POOL_SIZE = 16384 + + def __init__( + self, + head_num: int, + head_dim: int, + ): + self.head_num = head_num + self.head_dim = head_dim + self.size = MHATokenToKVPool.KV_POOL_SIZE + self.page_size = 1 + self.store_dtype = torch.bfloat16 + self.device = "cuda" + self.layer_num = 1 + self.start_layer = 0 + self._create_buffers() + + def _create_buffers(self): + self.k_buffer = [ + torch.zeros( + (self.size + self.page_size, self.head_num, self.head_dim), + dtype=self.store_dtype, + device=self.device, + ) + for _ in range(self.layer_num) + ] + self.v_buffer = [ + torch.zeros( + (self.size + self.page_size, self.head_num, self.head_dim), + dtype=self.store_dtype, + device=self.device, + ) + for _ in range(self.layer_num) + ] + + def set_kv_buffer( + self, + loc: torch.Tensor, + cache_k: torch.Tensor, + cache_v: torch.Tensor, + ): + layer_id = 0 + self.k_buffer[layer_id - self.start_layer][loc] = cache_k + self.v_buffer[layer_id - self.start_layer][loc] = cache_v + + +def create_inputs( + head_size: int, + batch_size: int, + seq_len: int, + device, + dtype: torch.dtype, + num_q_heads: int, + num_kv_heads: int, +): + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q_heads * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv_heads * head_size, dtype=dtype, device=device + ) + value = torch.randn( + batch_size * seq_len, num_kv_heads * head_size, dtype=dtype, device=device + ) + out_cache_loc = torch.randperm( + MHATokenToKVPool.KV_POOL_SIZE, dtype=torch.int64, device=device + )[: batch_size * seq_len].clone() + + return dict( + pos_ids=pos_ids, query=query, key=key, value=value, out_cache_loc=out_cache_loc + ) diff --git a/sgl-kernel/python/sgl_kernel/top_k.py b/sgl-kernel/python/sgl_kernel/top_k.py new file mode 100644 index 000000000..fc29a6db8 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/top_k.py @@ -0,0 +1,11 @@ +import torch + + +def fast_topk(values, topk, dim): + if topk == 1: + # Use max along the specified dimension to get both value and index + return torch.max(values, dim=dim, keepdim=True) + else: + # Use topk for efficiency with larger k values + # TODO: implement faster cuda kernels for large vocab sizes + return torch.topk(values, topk, dim=dim) diff --git a/sgl-kernel/python/sgl_kernel/utils.py b/sgl-kernel/python/sgl_kernel/utils.py new file mode 100644 index 000000000..f2fa0b617 --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/utils.py @@ -0,0 +1,50 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import functools +from typing import Dict, Tuple + +import torch + + +def get_cuda_stream() -> int: + return torch.cuda.current_stream().cuda_stream + + +_cache_buf: Dict[Tuple[str, torch.device], torch.Tensor] = {} + + +def _get_cache_buf(name: str, bytes: int, device: torch.device) -> torch.Tensor: + key = (name, device) + buf = _cache_buf.get(key) + if buf is None: + buf = torch.empty(bytes, dtype=torch.uint8, device=device) + _cache_buf[key] = buf + return buf + + +def _to_tensor_scalar_tuple(x): + if isinstance(x, torch.Tensor): + return (x, 0) + else: + return (None, x) + + +@functools.lru_cache(maxsize=1) +def is_arch_support_pdl() -> bool: + # Hopper arch's compute capability == 9.0 + device = torch.cuda.current_device() + major, minor = torch.cuda.get_device_capability(device) + return major >= 9 diff --git a/sgl-kernel/python/sgl_kernel/version.py b/sgl-kernel/python/sgl_kernel/version.py new file mode 100644 index 000000000..c2c2a2efd --- /dev/null +++ b/sgl-kernel/python/sgl_kernel/version.py @@ -0,0 +1 @@ +__version__ = "0.3.9.post2" diff --git a/sgl-kernel/rename_wheels.sh b/sgl-kernel/rename_wheels.sh new file mode 100755 index 000000000..018eeb27b --- /dev/null +++ b/sgl-kernel/rename_wheels.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -ex + +WHEEL_DIR="dist" + +wheel_files=($WHEEL_DIR/*.whl) +for wheel in "${wheel_files[@]}"; do + intermediate_wheel="${wheel/linux/manylinux2014}" + + # Extract the current python version from the wheel name + if [[ $intermediate_wheel =~ -cp([0-9]+)- ]]; then + cp_version="${BASH_REMATCH[1]}" + else + echo "Could not extract Python version from wheel name: $intermediate_wheel" + continue + fi + + # Detect CUDA version and add appropriate suffix + if ls /usr/local/ | grep -q "12.4"; then + new_wheel="${intermediate_wheel/-cp${cp_version}/+cu124-cp${cp_version}}" + elif ls /usr/local/ | grep -q "12.8"; then + new_wheel="${intermediate_wheel/-cp${cp_version}/+cu128-cp${cp_version}}" + else + new_wheel="$intermediate_wheel" + fi + + if [[ "$wheel" != "$new_wheel" ]]; then + echo "Renaming $wheel to $new_wheel" + mv -- "$wheel" "$new_wheel" + fi +done +echo "Wheel renaming completed." diff --git a/sgl-kernel/setup_hip.py b/sgl-kernel/setup_hip.py new file mode 100644 index 000000000..d7691b53b --- /dev/null +++ b/sgl-kernel/setup_hip.py @@ -0,0 +1,100 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import platform +import sys +from pathlib import Path + +from setuptools import find_packages, setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +root = Path(__file__).parent.resolve() +arch = platform.machine().lower() + + +def _get_version(): + with open(root / "pyproject.toml") as f: + for line in f: + if line.startswith("version"): + return line.split("=")[1].strip().strip('"') + + +operator_namespace = "sgl_kernel" +include_dirs = [ + root / "include", + root / "csrc", +] + +sources = [ + "csrc/allreduce/custom_all_reduce.hip", + "csrc/allreduce/quick_all_reduce.cu", + "csrc/common_extension_rocm.cc", + "csrc/elementwise/activation.cu", + "csrc/grammar/apply_token_bitmask_inplace_cuda.cu", + "csrc/moe/moe_align_kernel.cu", + "csrc/moe/moe_topk_softmax_kernels.cu", + "csrc/speculative/eagle_utils.cu", + "csrc/kvcacheio/transfer.cu", +] + +cxx_flags = [ + "-O3", + "-Wno-switch-bool", + "-Wno-macro-redefined", + "-Wno-deprecated-declarations", + "-w", +] +libraries = ["c10", "torch", "torch_python"] +extra_link_args = ["-Wl,-rpath,$ORIGIN/../../torch/lib", f"-L/usr/lib/{arch}-linux-gnu"] + +hipcc_flags = [ + "-fPIC", + "-O3", + "-std=c++17", + "-D__HIP_PLATFORM_HCC__=1", + "--offload-arch=gfx928", + "--offload-arch=gfx936", + "--gpu-max-threads-per-block=1024", + "-Wno-macro-redefined", + "-Wno-deprecated-declarations", + "-funroll-loops", + "-Rpass-analysis=unroll-loops", + "-w", +] + +ext_modules = [ + CUDAExtension( + name="sgl_kernel.common_ops", + sources=sources, + include_dirs=include_dirs, + extra_compile_args={ + "nvcc": hipcc_flags, + "cxx": cxx_flags, + }, + libraries=libraries, + extra_link_args=extra_link_args, + py_limited_api=False, + ), +] + +setup( + name="sgl-kernel", + version=_get_version(), + packages=find_packages(where="python"), + package_dir={"": "python"}, + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)}, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) \ No newline at end of file diff --git a/sgl-kernel/setup_rocm.py b/sgl-kernel/setup_rocm.py new file mode 100644 index 000000000..6e3466ec3 --- /dev/null +++ b/sgl-kernel/setup_rocm.py @@ -0,0 +1,116 @@ +# Copyright 2025 SGLang Team. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import os +import platform +import sys +from pathlib import Path + +import torch +from setuptools import find_packages, setup +from torch.utils.cpp_extension import BuildExtension, CUDAExtension + +root = Path(__file__).parent.resolve() +arch = platform.machine().lower() + + +def _get_version(): + with open(root / "pyproject.toml") as f: + for line in f: + if line.startswith("version"): + return line.split("=")[1].strip().strip('"') + + +operator_namespace = "sgl_kernel" +include_dirs = [ + root / "include", + root / "include" / "impl", + root / "csrc", +] + +sources = [ + "csrc/allreduce/custom_all_reduce.hip", + "csrc/allreduce/quick_all_reduce.cu", + "csrc/common_extension_rocm.cc", + "csrc/elementwise/activation.cu", + "csrc/grammar/apply_token_bitmask_inplace_cuda.cu", + "csrc/moe/moe_align_kernel.cu", + "csrc/moe/moe_topk_softmax_kernels.cu", + "csrc/speculative/eagle_utils.cu", + "csrc/kvcacheio/transfer.cu", +] + +cxx_flags = ["-O3"] +libraries = ["hiprtc", "amdhip64", "c10", "torch", "torch_python"] +extra_link_args = ["-Wl,-rpath,$ORIGIN/../../torch/lib", f"-L/usr/lib/{arch}-linux-gnu"] + +default_target = "gfx942" +amdgpu_target = os.environ.get("AMDGPU_TARGET", default_target) + +if torch.cuda.is_available(): + try: + amdgpu_target = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + except Exception as e: + print(f"Warning: Failed to detect GPU properties: {e}") +else: + print(f"Warning: torch.cuda not available. Using default target: {amdgpu_target}") + +if amdgpu_target not in ["gfx942", "gfx950"]: + print( + f"Warning: Unsupported GPU architecture detected '{amdgpu_target}'. Expected 'gfx942' or 'gfx950'." + ) + sys.exit(1) + +fp8_macro = ( + "-DHIP_FP8_TYPE_FNUZ" if amdgpu_target == "gfx942" else "-DHIP_FP8_TYPE_E4M3" +) + +hipcc_flags = [ + "-DNDEBUG", + f"-DOPERATOR_NAMESPACE={operator_namespace}", + "-O3", + "-Xcompiler", + "-fPIC", + "-std=c++17", + f"--amdgpu-target={amdgpu_target}", + "-DENABLE_BF16", + "-DENABLE_FP8", + fp8_macro, +] + +ext_modules = [ + CUDAExtension( + name="sgl_kernel.common_ops", + sources=sources, + include_dirs=include_dirs, + extra_compile_args={ + "nvcc": hipcc_flags, + "cxx": cxx_flags, + }, + libraries=libraries, + extra_link_args=extra_link_args, + py_limited_api=False, + ), +] + +setup( + name="sgl-kernel", + version=_get_version(), + packages=find_packages(where="python"), + package_dir={"": "python"}, + ext_modules=ext_modules, + cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)}, + options={"bdist_wheel": {"py_limited_api": "cp39"}}, +) diff --git a/sgl-kernel/tests/spatial/test_greenctx_stream.py b/sgl-kernel/tests/spatial/test_greenctx_stream.py new file mode 100644 index 000000000..c57bc3360 --- /dev/null +++ b/sgl-kernel/tests/spatial/test_greenctx_stream.py @@ -0,0 +1,25 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import create_greenctx_stream_by_value, get_sm_available + + +def test_green_ctx(): + A = torch.randn(5120, 5120).cuda() + B = torch.randn(5120, 5120).cuda() + C = torch.matmul(A, B) + sm_counts = get_sm_available(0) + stream_group = create_greenctx_stream_by_value(sm_counts // 2, sm_counts // 2, 0) + with torch.cuda.stream(stream_group[0]): + for _ in range(100): + result_0 = torch.matmul(A, B) + with torch.cuda.stream(stream_group[1]): + for _ in range(100): + result_1 = torch.matmul(A, B) + torch.cuda.synchronize() + assert torch.allclose(result_0, C) + assert torch.allclose(result_1, C) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/speculative/test_eagle_utils.py b/sgl-kernel/tests/speculative/test_eagle_utils.py new file mode 100644 index 000000000..503355387 --- /dev/null +++ b/sgl-kernel/tests/speculative/test_eagle_utils.py @@ -0,0 +1,87 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import verify_tree_greedy + + +def test_verify_tree_greedy(): + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int64, + device="cuda", + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int64, + device="cuda", + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device="cuda") + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i][j]) < 10: + target_logits[i][j][18] = 10 + + target_predict = torch.argmax(target_logits, dim=-1) + predict_shape = (12,) + + bs = candidates.shape[0] + num_spec_step = 4 + + predicts = torch.full( + predict_shape, -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_index = torch.full( + (bs, num_spec_step), -1, dtype=torch.int32, device="cuda" + ) # mutable + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device="cuda") # mutable + + verify_tree_greedy( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + target_predict=target_predict, + ) + + # Check the expected output. + assert predicts.tolist() == [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18] + assert accept_index.tolist() == [ + [0, 3, 4, 5], + [6, 10, 11, -1], + ] + assert accept_token_num.tolist() == [3, 2] + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/speculative/test_speculative_sampling.py b/sgl-kernel/tests/speculative/test_speculative_sampling.py new file mode 100644 index 000000000..a9b59bb2e --- /dev/null +++ b/sgl-kernel/tests/speculative/test_speculative_sampling.py @@ -0,0 +1,129 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import tree_speculative_sampling_target_only + +test_cases = [ + ( + 1, + 1, + [3, -1, -1, 4, 5, 18, 11, -1, -1, -1, 12, 18], + [[0, 3, 4, 5], [6, 10, 11, -1]], + [3, 2], + ), + ( + 0, # threshold_single + 0, # threshold_acc + [1, 2, 18, -1, -1, -1, 11, -1, -1, -1, 12, 18], + [[0, 1, 2, -1], [6, 10, 11, -1]], + [2, 2], + ), +] + + +@pytest.mark.parametrize( + "threshold_single, threshold_acc, expected_predicts, expected_accept_index, expected_accept_token_num", + test_cases, +) +def test_tree_speculative_sampling_target_only( + threshold_single, + threshold_acc, + expected_predicts, + expected_accept_index, + expected_accept_token_num, +): + """ + Tests the tree_speculative_sampling_target_only function using Pytest parameterization. + """ + device = "cuda" + + candidates = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [7, 8, 9, 10, 11, 12], + ], + dtype=torch.int64, + device=device, + ) + retrive_index = torch.tensor( + [ + [0, 1, 2, 3, 4, 5], + [6, 7, 8, 9, 10, 11], + ], + dtype=torch.int64, + device=device, + ) + retrive_next_token = torch.tensor( + [ + [1, 2, -1, 4, 5, -1], + [4, 2, 3, -1, 5, -1], + ], + dtype=torch.int64, + device=device, + ) + retrive_next_sibling = torch.tensor( + [ + [-1, 3, -1, -1, -1, -1], + [-1, -1, -1, -1, 1, -1], + ], + dtype=torch.int64, + device=device, + ) + + target_logits = torch.full((2, 6, 20), 1, dtype=torch.float32, device=device) + target_logits[0, 0, 3] = 10 + target_logits[0, 3, 4] = 10 + target_logits[0, 4, 5] = 10 + target_logits[1, 0, 11] = 10 + target_logits[1, 4, 12] = 10 + + for i in range(target_logits.shape[0]): + for j in range(target_logits.shape[1]): + if torch.max(target_logits[i, j]) < 10: + target_logits[i, j, 18] = 10 + + temperatures = torch.tensor([0.01, 0.01], dtype=torch.float32, device=device) + bs, num_draft_tokens = candidates.shape + num_spec_step = len(expected_accept_index[0]) + predict_shape = (len(expected_predicts),) + + predicts = torch.full(predict_shape, -1, dtype=torch.int32, device=device) + accept_index = torch.full((bs, num_spec_step), -1, dtype=torch.int32, device=device) + accept_token_num = torch.full((bs,), 0, dtype=torch.int32, device=device) + + expanded_temperature = temperatures.unsqueeze(1).unsqueeze(1) + target_probs = F.softmax(target_logits / expanded_temperature, dim=-1) + draft_probs = torch.full_like(target_probs, 0, dtype=torch.float32, device=device) + coins = torch.rand(bs, num_draft_tokens, device=device, dtype=torch.float32) + coins_for_final_sampling = torch.rand(bs, device=device).to(torch.float32) + + tree_speculative_sampling_target_only( + predicts=predicts, + accept_index=accept_index, + accept_token_num=accept_token_num, + candidates=candidates, + retrive_index=retrive_index, + retrive_next_token=retrive_next_token, + retrive_next_sibling=retrive_next_sibling, + uniform_samples=coins, + uniform_samples_for_final_sampling=coins_for_final_sampling, + target_probs=target_probs, + draft_probs=draft_probs, + threshold_single=threshold_single, + threshold_acc=threshold_acc, + deterministic=True, + ) + + assert ( + predicts.tolist() == expected_predicts + ), f"Predicts mismatch for thresholds ({threshold_single}, {threshold_acc})" + assert ( + accept_index.tolist() == expected_accept_index + ), f"Accept index mismatch for thresholds ({threshold_single}, {threshold_acc})" + assert ( + accept_token_num.tolist() == expected_accept_token_num + ), f"Accept token num mismatch for thresholds ({threshold_single}, {threshold_acc})" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_activation.py b/sgl-kernel/tests/test_activation.py new file mode 100644 index 000000000..43593441e --- /dev/null +++ b/sgl-kernel/tests/test_activation.py @@ -0,0 +1,39 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_activation.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_silu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.silu(x[..., :dim]) + y = sgl_kernel.silu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_tanh_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="tanh") + y = sgl_kernel.gelu_tanh_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("dim", [128, 256, 512, 2048, 4096, 11008, 16384]) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +@pytest.mark.parametrize("seq_len", [1, 2, 4, 8, 16, 32, 64, 128, 512]) +def test_fused_gelu_mul(dim, batch_size, seq_len): + x = torch.randn(batch_size, seq_len, 2 * dim).to(0).to(torch.float16) + y_ref = x[..., dim:] * torch.nn.functional.gelu(x[..., :dim], approximate="none") + y = sgl_kernel.gelu_and_mul(x) + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_apply_token_bitmask_inplace.py b/sgl-kernel/tests/test_apply_token_bitmask_inplace.py new file mode 100644 index 000000000..480479134 --- /dev/null +++ b/sgl-kernel/tests/test_apply_token_bitmask_inplace.py @@ -0,0 +1,23 @@ +import pytest +import torch +from sgl_kernel import apply_token_bitmask_inplace_cuda + + +def test_apply_token_bitmask_inplace_kernel(): + neginf = float("-inf") + bool_mask = torch.tensor([0, 1, 0, 1, 0, 1, 0, 1, 0, 1], dtype=torch.bool) + logits = torch.tensor( + [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0], dtype=torch.float32 + ) + expected = torch.where(bool_mask, logits, neginf) + + logits_gpu = logits.to("cuda") + bitmask = torch.tensor([0b1010101010], dtype=torch.int32).to("cuda") + apply_token_bitmask_inplace_cuda(logits_gpu, bitmask) + torch.cuda.synchronize() + torch.testing.assert_close(logits_gpu, expected.to("cuda")) + + +if __name__ == "__main__": + test_apply_token_bitmask_inplace_kernel() + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_awq_dequant.py b/sgl-kernel/tests/test_awq_dequant.py new file mode 100644 index 000000000..da68e88d0 --- /dev/null +++ b/sgl-kernel/tests/test_awq_dequant.py @@ -0,0 +1,115 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import awq_dequantize + + +def reverse_awq_order(t: torch.Tensor): + bits = 4 + AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + reverse_order_tensor = torch.arange( + t.shape[-1], + dtype=torch.int32, + device=t.device, + ) + reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits) + reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER] + reverse_order_tensor = reverse_order_tensor.view(-1) + + t = t[:, reverse_order_tensor] & 0xF + return t + + +# qweights - [R , C // 8], int32 +# scales - [R // G, C ], float16 +# zeros - [R // G, C // 8], int32 +def awq_dequantize_torch( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, group_size: int +) -> torch.Tensor: + + if group_size == -1: + group_size = qweight.shape[0] + + bits = 4 + shifts = torch.arange(0, 32, bits, device=qzeros.device) + + iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + + iweights = iweights.view(iweights.shape[0], -1) + + zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + zeros = zeros.view(qzeros.shape[0], -1) + zeros = reverse_awq_order(zeros) + + iweights = reverse_awq_order(iweights) + + iweights = torch.bitwise_and(iweights, (2**bits) - 1) + zeros = torch.bitwise_and(zeros, (2**bits) - 1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +def sglang_awq_dequantize( + qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor +) -> torch.Tensor: + return awq_dequantize(qweight, scales, qzeros) + + +@pytest.mark.parametrize( + "qweight_row,qweight_col,is_bf16_act", + list( + itertools.product( + [3584, 18944, 128, 256, 512, 1024, 1536], + [448, 576, 4736, 16, 32, 64, 128, 72], + [True, False], + ) + ), +) +def test_awq_dequant_compare_implementations( + qweight_row: int, qweight_col: int, is_bf16_act: bool +): + device = torch.device("cuda") + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_row, qweight_col), + dtype=torch.int32, + device=device, + ) + group_size = qweight_row + scales_row = qweight_row // group_size + scales_col = qweight_col * 8 + + if is_bf16_act: + scales = torch.rand(scales_row, scales_col, dtype=torch.bfloat16, device=device) + else: + scales = torch.rand(scales_row, scales_col, dtype=torch.float16, device=device) + + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (scales_row, qweight_col), + dtype=torch.int32, + device=device, + ) + + # Run both implementations + torch_out = awq_dequantize_torch(qweight, scales, qzeros, group_size) + sglang_out = sglang_awq_dequantize(qweight, scales, qzeros) + + # Compare results + torch.testing.assert_close( + torch_out.to(torch.float32), sglang_out.to(torch.float32), rtol=1e-3, atol=1e-5 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_bmm_fp8.py b/sgl-kernel/tests/test_bmm_fp8.py new file mode 100644 index 000000000..e0be92896 --- /dev/null +++ b/sgl-kernel/tests/test_bmm_fp8.py @@ -0,0 +1,43 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_bmm_fp8.py + +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import bmm_fp8 + + +def to_float8(x, dtype=torch.float8_e4m3fn): + finfo = torch.finfo(dtype) + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) + scale = finfo.max / amax + x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max) + return x_scl_sat.to(dtype), scale.float().reciprocal() + + +@pytest.mark.parametrize("input_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("mat2_dtype", [torch.float8_e4m3fn, torch.float8_e5m2]) +@pytest.mark.parametrize("res_dtype", [torch.bfloat16, torch.float16]) +def test_bmm_fp8(input_dtype, mat2_dtype, res_dtype): + if input_dtype == torch.float8_e5m2 and mat2_dtype == torch.float8_e5m2: + pytest.skip("Invalid combination: both input and mat2 are e5m2") + + input = torch.randn([16, 48, 64], device="cuda", dtype=torch.bfloat16) + input_fp8, input_inv_s = to_float8(input, dtype=input_dtype) + + # mat2 row major -> column major + mat2 = torch.randn([16, 80, 64], device="cuda", dtype=torch.bfloat16).transpose( + -2, -1 + ) + mat2_fp8, mat2_inv_s = to_float8(mat2, dtype=mat2_dtype) + + res = torch.empty([16, 48, 80], device="cuda", dtype=res_dtype) + bmm_fp8(input_fp8, mat2_fp8, input_inv_s, mat2_inv_s, res_dtype, res) + + reference = torch.bmm(input, mat2) + cos_sim = F.cosine_similarity(reference.reshape(-1), res.reshape(-1), dim=0) + assert cos_sim > 0.99 + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_causal_conv1d.py b/sgl-kernel/tests/test_causal_conv1d.py new file mode 100644 index 000000000..a10e1f45e --- /dev/null +++ b/sgl-kernel/tests/test_causal_conv1d.py @@ -0,0 +1,489 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/mamba/test_causal_conv1d.py +from typing import Optional + +import torch +from sgl_kernel import causal_conv1d_fwd +from sgl_kernel import causal_conv1d_update as causal_conv1d_update_kernel + +PAD_SLOT_ID = -1 + + +def causal_conv1d_fn( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + query_start_loc: Optional[torch.Tensor] = None, + cache_indices: Optional[torch.Tensor] = None, + has_initial_state: Optional[torch.Tensor] = None, + conv_states: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim, seqlen) or (dim,cu_seq_len) for varlen + sequences are concatenated from left to right for varlen + weight: (dim, width) + bias: (dim,) + query_start_loc: (batch + 1) int32 + The cumulative sequence lengths of the sequences in + the batch, used to index into sequence. prepended by 0. + for example: query_start_loc = torch.Tensor([0,10,16,17]), + x.shape=(dim,17) + cache_indices: (batch) int32 + indicates the corresponding state index, + like so: conv_state = conv_states[cache_indices[batch_id]] + has_initial_state: (batch) bool + indicates whether should the kernel take the current state as initial + state for the calculations + conv_states: (...,dim,width - 1) itype + updated inplace if provided + activation: either None or "silu" or "swish" + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1, 20, pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + if x.stride(-1) != 1: + x = x.contiguous() + bias = bias.contiguous() if bias is not None else None + + causal_conv1d_fwd( + x, + weight, + bias, + conv_states, + query_start_loc, + cache_indices, + has_initial_state, + activation in ["silu", "swish"], + pad_slot_id, + ) + return x + + +def causal_conv1d_update( + x: torch.Tensor, + conv_state: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + activation: Optional[str] = None, + cache_seqlens: Optional[torch.Tensor] = None, + conv_state_indices: Optional[torch.Tensor] = None, + pad_slot_id: int = PAD_SLOT_ID, +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the conv_state + starting at the index + @cache_seqlens % state_len. + conv_state_indices: (batch,), dtype int32 + If not None, the conv_state is a larger tensor along the batch dim, + and we are selecting the batch coords specified by conv_state_indices. + Useful for a continuous batching scenario. + pad_slot_id: int + if cache_indices is passed, lets the kernel identify padded + entries that will not be processed, + for example: cache_indices = [pad_slot_id, 1 ,20 ,pad_slot_id] + in this case, the kernel will not process entries at + indices 0 and 3 + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError( + f"activation must be None, silu, or swish, actual: {activation}" + ) + activation_val = activation in ["silu", "swish"] + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + causal_conv1d_update_kernel( + x, + conv_state, + weight, + bias, + activation_val, + cache_seqlens, + conv_state_indices, + pad_slot_id, + ) + if unsqueeze: + x = x.squeeze(-1) + return x + + +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F + + +def causal_conv1d_ref( + x: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor] = None, + initial_states: Optional[torch.Tensor] = None, + return_final_states: bool = False, + final_states_out: Optional[torch.Tensor] = None, + activation: Optional[str] = "silu", +): + """ + x: (batch, dim, seqlen) + weight: (dim, width) + bias: (dim,) + initial_states: (batch, dim, width - 1) + final_states_out: (batch, dim, width - 1) + + out: (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + x = x.to(weight.dtype) + seqlen = x.shape[-1] + dim, width = weight.shape + if initial_states is None: + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=width - 1, groups=dim) + else: + x = torch.cat([initial_states, x], dim=-1) + out = F.conv1d(x, weight.unsqueeze(1), bias, padding=0, groups=dim) + out = out[..., :seqlen] + if return_final_states: + final_states = F.pad(x, (width - 1 - x.shape[-1], 0)).to( + dtype_in + ) # (batch, dim, width - 1) + if final_states_out is not None: + final_states_out.copy_(final_states) + else: + final_states_out = final_states + out = (out if activation is None else F.silu(out)).to(dtype=dtype_in) + return (out, None) if not return_final_states else (out, final_states_out) + + +def causal_conv1d_update_ref( + x, conv_state, weight, bias=None, activation=None, cache_seqlens=None +): + """ + x: (batch, dim) or (batch, dim, seqlen) + conv_state: (batch, dim, state_len), where state_len >= width - 1 + weight: (dim, width) + bias: (dim,) + cache_seqlens: (batch,), dtype int32. + If not None, the conv_state is treated as a circular buffer. + The conv_state will be updated by copying x to the + conv_state starting at the index + @cache_seqlens % state_len before performing the convolution. + + out: (batch, dim) or (batch, dim, seqlen) + """ + if activation not in [None, "silu", "swish"]: + raise NotImplementedError("activation must be None, silu, or swish") + dtype_in = x.dtype + unsqueeze = x.dim() == 2 + if unsqueeze: + x = x.unsqueeze(-1) + batch, dim, seqlen = x.shape + width = weight.shape[1] + state_len = conv_state.shape[-1] + assert conv_state.shape == (batch, dim, state_len) + assert weight.shape == (dim, width) + if cache_seqlens is None: + x_new = torch.cat([conv_state, x], dim=-1).to( + weight.dtype + ) # (batch, dim, state_len + seqlen) + conv_state.copy_(x_new[:, :, -state_len:]) + else: + width_idx = torch.arange( + -(width - 1), 0, dtype=torch.long, device=x.device + ).unsqueeze(0) + cache_seqlens.unsqueeze(1) + width_idx = ( + torch.remainder(width_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + ) + x_new = torch.cat([conv_state.gather(2, width_idx), x], dim=-1).to(weight.dtype) + copy_idx = torch.arange(seqlen, dtype=torch.long, device=x.device).unsqueeze( + 0 + ) + cache_seqlens.unsqueeze(1) + copy_idx = torch.remainder(copy_idx, state_len).unsqueeze(1).expand(-1, dim, -1) + conv_state.scatter_(2, copy_idx, x) + out = F.conv1d(x_new, weight.unsqueeze(1), bias, padding=0, groups=dim)[ + :, :, -seqlen: + ] + if unsqueeze: + out = out.squeeze(-1) + return (out if activation is None else F.silu(out)).to(dtype=dtype_in) + + +@pytest.mark.parametrize("itype", [torch.bfloat16, torch.float]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +@pytest.mark.parametrize("has_initial_state", [True, False]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize( + "seqlen", [1, 8, 16, 32, 64, 128, 256, 512, 784, 1024, 1025, 2048, 4096] +) +@pytest.mark.parametrize("dim", [64]) +@pytest.mark.parametrize("batch", [1]) +def test_causal_conv1d( + batch, dim, seqlen, width, has_bias, silu_activation, has_initial_state, itype +): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype).contiguous() + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + if has_initial_state: + initial_states = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + has_initial_state_tensor = torch.ones(batch, dtype=torch.bool, device=x.device) + else: + initial_states = None + has_initial_state_tensor = None + x_ref = x.clone() + weight_ref = weight.clone() + bias_ref = bias.clone() if bias is not None else None + initial_states_ref = initial_states.clone() if initial_states is not None else None + activation = None if not silu_activation else "silu" + out = causal_conv1d_fn( + x, + weight, + bias, + activation=activation, + conv_states=initial_states, + has_initial_state=has_initial_state_tensor, + ) + out_ref, final_states_ref = causal_conv1d_ref( + x_ref, + weight_ref, + bias_ref, + initial_states=initial_states_ref, + return_final_states=True, + activation=activation, + ) + if has_initial_state: + assert initial_states is not None and final_states_ref is not None + assert torch.allclose(initial_states, final_states_ref, rtol=rtol, atol=atol) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +def test_causal_conv1d_update(dim, width, seqlen, has_bias, silu_activation, itype): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + batch = 2 + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype) + x_ref = x.clone() + conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state.detach().clone() + activation = None if not silu_activation else "silu" + out = causal_conv1d_update(x, conv_state, weight, bias, activation=activation) + out_ref = causal_conv1d_update_ref( + x_ref, conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state, conv_state_ref) + assert torch.allclose(out, out_ref, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("itype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [False, True]) +@pytest.mark.parametrize("has_bias", [False, True]) +@pytest.mark.parametrize("seqlen", [1, 4, 5]) +@pytest.mark.parametrize("width", [2, 3, 4]) +@pytest.mark.parametrize("dim", [2048, 2048 + 16, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +def test_causal_conv1d_update_with_batch_gather( + with_padding, dim, width, seqlen, has_bias, silu_activation, itype +): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + batch_size = 3 + padding = 5 if with_padding else 0 + padded_batch_size = batch_size + padding + total_entries = 10 * batch_size + + x = torch.randn(padded_batch_size, dim, 1, device=device, dtype=itype) + x_ref = x.clone() + + conv_state_indices = torch.randperm(total_entries)[:batch_size].to( + dtype=torch.int32, device=device + ) + unused_states_bool = torch.ones(total_entries, dtype=torch.bool, device=device) + unused_states_bool[conv_state_indices] = False + padded_state_indices = torch.concat( + [ + conv_state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=0, + ) + conv_state = torch.randn(total_entries, dim, width - 1, device=device, dtype=itype) + conv_state_for_padding_test = conv_state.clone() + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + conv_state_ref = conv_state[conv_state_indices, :].detach().clone() + activation = None if not silu_activation else "silu" + out = causal_conv1d_update( + x, + conv_state, + weight, + bias, + activation=activation, + conv_state_indices=padded_state_indices, + pad_slot_id=PAD_SLOT_ID, + ) + out_ref = causal_conv1d_update_ref( + x_ref[:batch_size], conv_state_ref, weight, bias, activation=activation + ) + + assert torch.equal(conv_state[conv_state_indices, :], conv_state_ref) + assert torch.allclose(out[:batch_size], out_ref, rtol=rtol, atol=atol) + assert torch.equal( + conv_state[unused_states_bool], conv_state_for_padding_test[unused_states_bool] + ) + + +@pytest.mark.parametrize("itype", [torch.bfloat16]) +@pytest.mark.parametrize("silu_activation", [True]) +@pytest.mark.parametrize("has_bias", [True]) +@pytest.mark.parametrize("width", [4]) +@pytest.mark.parametrize( + "seqlen", [8, 16, 32, 64, 128, 256, 512, 784, 1024, 2048, 2049, 4096] +) +@pytest.mark.parametrize("dim", [64, 4096]) +# tests correctness in case subset of the sequences are padded +@pytest.mark.parametrize("with_padding", [True, False]) +def test_causal_conv1d_varlen( + with_padding, dim, seqlen, width, has_bias, silu_activation, itype +): + device = "cuda" + torch.cuda.empty_cache() + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + + seqlens = [] + batch_size = 4 + if seqlen < 10: + batch_size = 1 + padding = 3 if with_padding else 0 + padded_batch_size = batch_size + padding + nsplits = padded_batch_size - 1 + + eos_pos = torch.randperm(seqlen - 1)[:nsplits].sort().values + seqlens.append( + torch.diff( + torch.cat([torch.tensor([-1]), eos_pos, torch.tensor([seqlen - 1])]) + ).tolist() + ) + assert sum(seqlens[-1]) == seqlen + assert all(s > 0 for s in seqlens[-1]) + + total_entries = batch_size * 10 + cumsum = torch.cumsum(torch.tensor(seqlens[0]), dim=0).to(torch.int32) + cumsum = torch.concat([torch.tensor([0], dtype=torch.int32), cumsum], dim=0) + x = torch.randn(1, 4096 + dim + 64, seqlen, device=device, dtype=itype)[ + :, 4096 : 4096 + dim, : + ] + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + x_ref = x.clone() + weight_ref = weight.clone() + bias_ref = bias.clone() if bias is not None else None + activation = None if not silu_activation else "silu" + final_states = torch.randn( + total_entries, dim, width - 1, device=x.device, dtype=x.dtype + ) + final_states_ref = final_states.clone() + has_initial_states = torch.randint( + 0, 2, (cumsum.shape[0] - 1,), dtype=torch.bool, device=x.device + ) + state_indices = torch.randperm(total_entries, dtype=torch.int32, device=x.device)[ + :batch_size + ] + padded_state_indices = torch.concat( + [ + state_indices, + torch.as_tensor([PAD_SLOT_ID] * padding, dtype=torch.int32, device=device), + ], + dim=-1, + ) + + out = causal_conv1d_fn( + x.squeeze(0), + weight, + bias, + cumsum.cuda(), + padded_state_indices, + has_initial_states, + final_states, + activation, + PAD_SLOT_ID, + ) + out_ref = [] + out_ref_b = [] + + splits = [torch.split(var, seqlens[0], dim=-1) for var in (x_ref)] + for i in range(len(seqlens[0])): + x_s = [v[i].unsqueeze(0) for v in splits][0] + if padded_state_indices[i] == PAD_SLOT_ID: + continue + out_ref_b.append( + causal_conv1d_ref( + x_s, + weight_ref, + bias_ref, + activation=activation, + return_final_states=True, + final_states_out=final_states_ref[padded_state_indices[i]].unsqueeze(0), + initial_states=( + final_states_ref[padded_state_indices[i]].unsqueeze(0) + if has_initial_states[i] + else None + ), + ) + ) + out_ref.append(torch.cat([t[0] for t in out_ref_b], dim=2)) + out_ref_tensor = torch.cat(out_ref, dim=0) + + unpadded_out = out[:, : out_ref_tensor.shape[-1]] + assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol) + assert torch.allclose( + final_states[state_indices], + final_states_ref[state_indices], + rtol=rtol, + atol=atol, + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_custom_allreduce.py b/sgl-kernel/tests/test_custom_allreduce.py new file mode 100644 index 000000000..d729392c3 --- /dev/null +++ b/sgl-kernel/tests/test_custom_allreduce.py @@ -0,0 +1,185 @@ +import ctypes +import multiprocessing as mp +import random +import socket +import unittest +from typing import Any, List, Optional + +import sgl_kernel.allreduce as custom_ops +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup + +from sglang.srt.distributed.device_communicators.cuda_wrapper import CudaRTLibrary + + +def _run_correctness_worker(world_size, rank, distributed_init_port, test_sizes): + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + dist.init_process_group( + backend="nccl", + init_method=distributed_init_method, + rank=rank, + world_size=world_size, + ) + group = dist.group.WORLD + + try: + device = torch.device(f"cuda:{rank}") + max_size = 8192 * 1024 + meta_ptrs = TestCustomAllReduce.create_shared_buffer( + custom_ops.meta_size() + max_size, group=group + ) + + rank_data = torch.empty(8 * 1024 * 1024, dtype=torch.uint8, device=device) + buffer_ptrs = TestCustomAllReduce.create_shared_buffer(max_size, group=group) + + custom_ptr = custom_ops.init_custom_ar(meta_ptrs, rank_data, rank, True) + custom_ops.register_buffer(custom_ptr, buffer_ptrs) + + test_loop = 10 + for sz in test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(test_loop): + inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp1_ref = inp1.clone() + out1 = torch.empty_like(inp1) + + custom_ops.all_reduce( + custom_ptr, inp1, out1, buffer_ptrs[rank], max_size + ) + + dist.all_reduce(inp1_ref, group=group) + + torch.testing.assert_close(out1, inp1_ref) + + finally: + dist.barrier(group=group) + if custom_ptr is not None: + custom_ops.dispose(custom_ptr) + if buffer_ptrs: + TestCustomAllReduce.free_shared_buffer(buffer_ptrs, group) + if meta_ptrs: + TestCustomAllReduce.free_shared_buffer(meta_ptrs, group) + + dist.destroy_process_group(group=group) + + +def get_open_port() -> int: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + except OSError: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("::1", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, test_target: Any, target_args: tuple = () +) -> None: + mp.set_start_method("spawn", force=True) + + procs = [] + distributed_init_port = get_open_port() + for i in range(world_size): + proc_args = (world_size, i, distributed_init_port) + target_args + proc = mp.Process(target=test_target, args=proc_args, name=f"Worker-{i}") + proc.start() + procs.append(proc) + + for i in range(world_size): + procs[i].join() + assert ( + procs[i].exitcode == 0 + ), f"Process {i} failed with exit code {procs[i].exitcode}" + + +class TestCustomAllReduce(unittest.TestCase): + test_sizes = [ + 512, + 2560, + 4096, + 5120, + 7680, + 32768, + 262144, + 524288, + 1048576, + 2097152, + ] + world_sizes = [2, 4, 8] + + @staticmethod + def create_shared_buffer( + size_in_bytes: int, group: Optional[ProcessGroup] = None + ) -> List[int]: + lib = CudaRTLibrary() + pointer = lib.cudaMalloc(size_in_bytes) + handle = lib.cudaIpcGetMemHandle(pointer) + if group is None: + group = dist.group.WORLD + world_size = dist.get_world_size(group=group) + rank = dist.get_rank(group=group) + + handle_bytes = ctypes.string_at(ctypes.addressof(handle), ctypes.sizeof(handle)) + input_tensor = torch.ByteTensor(list(handle_bytes)).to(f"cuda:{rank}") + gathered_tensors = [torch.empty_like(input_tensor) for _ in range(world_size)] + dist.all_gather(gathered_tensors, input_tensor, group=group) + + handles = [] + handle_type = type(handle) + for tensor in gathered_tensors: + bytes_list = tensor.cpu().tolist() + bytes_data = bytes(bytes_list) + handle_obj = handle_type() + ctypes.memmove(ctypes.addressof(handle_obj), bytes_data, len(bytes_data)) + handles.append(handle_obj) + + pointers: List[int] = [] + for i, h in enumerate(handles): + if i == rank: + pointers.append(pointer.value) + else: + try: + opened_ptr = lib.cudaIpcOpenMemHandle(h) + pointers.append(opened_ptr.value) + except Exception as e: + print(f"Rank {rank}: Failed to open IPC handle from rank {i}: {e}") + raise + + dist.barrier(group=group) + return pointers + + @staticmethod + def free_shared_buffer( + pointers: List[int], group: Optional[ProcessGroup] = None + ) -> None: + if group is None: + group = dist.group.WORLD + rank = dist.get_rank(group=group) + lib = CudaRTLibrary() + if pointers and len(pointers) > rank and pointers[rank] is not None: + lib.cudaFree(ctypes.c_void_p(pointers[rank])) + dist.barrier(group=group) + + def test_correctness(self): + for world_size in self.world_sizes: + available_gpus = torch.cuda.device_count() + if world_size > available_gpus: + print( + f"Skipping world_size={world_size}, requires {world_size} GPUs, found {available_gpus}" + ) + continue + + print(f"Running test for world_size={world_size}") + multi_process_parallel( + world_size, _run_correctness_worker, target_args=(self.test_sizes,) + ) + print(f"custom allreduce tp = {world_size}: OK") + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_cutlass_mla.py b/sgl-kernel/tests/test_cutlass_mla.py new file mode 100644 index 000000000..71de8327a --- /dev/null +++ b/sgl-kernel/tests/test_cutlass_mla.py @@ -0,0 +1,104 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import cutlass_mla_decode, cutlass_mla_get_workspace_size +from torch import Tensor + +# Disable tests on SM103 until the accuracy issues are fixed. +if torch.cuda.get_device_capability() != (10, 0): + pytest.skip( + reason="Cutlass MLA Requires compute capability of 10.", + allow_module_level=True, + ) + + +def ref_mla( + out: Tensor, # (bs, num_heads, v_head_dim) + query: Tensor, # (bs, num_heads, head_dim) + kv_cache: Tensor, # (num_blocks, block_size, head_dim) + scale: float, + block_tables: Tensor, # (bs, max_num_blocks) + seq_lens: Tensor, # (bs,) +): + bs, num_heads, v_head_dim = out.shape + head_dim = query.shape[2] + + for i in range(bs): + # gather and flatten KV-cache + kv = kv_cache[block_tables[i]] # (max_num_blocks, block_size, head_dim) + kv = kv.view(1, -1, head_dim)[:, : seq_lens[i]] # (1, seq_len, head_dim) + v = kv[:, :, :v_head_dim] + + q = query[i].view(num_heads, 1, head_dim) + o = F.scaled_dot_product_attention(q, kv, v, scale=scale, enable_gqa=True) + out[i] = o.view(num_heads, v_head_dim) + + return out + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("mean_seq_len", [128, 1024, 4096]) +@pytest.mark.parametrize("bs", [1, 2, 4]) +@pytest.mark.parametrize("varlen", [False, True]) +@pytest.mark.parametrize("block_size", [1, 16, 64, 128]) +@pytest.mark.parametrize("num_heads", [16, 32, 64, 128]) +@pytest.mark.parametrize("num_kv_splits", [-1, 1]) +def test_cutlass_mla_decode( + dtype: torch.dtype, + mean_seq_len: int, + bs: int, + varlen: bool, + block_size: int, + num_heads: int, + num_kv_splits: int, +): + torch.set_default_dtype(dtype) + torch.set_default_device("cuda") + torch.manual_seed(42) + + d = 576 + h_q = num_heads + dv = 512 + + q_nope_dim = 128 + q_pe_dim = 64 + scale = (q_nope_dim + q_pe_dim) ** (-0.5) + if varlen: + seq_lens = torch.empty(bs).normal_(mean_seq_len, mean_seq_len / 2) + seq_lens = seq_lens.clip(2).to(torch.int32) + else: + seq_lens = torch.full((bs,), mean_seq_len, dtype=torch.int32) + max_seq_len = seq_lens.max().item() + block_num = (max_seq_len + block_size - 1) // block_size + + # Pad block_num so that small blocks can be packed into full 128-sized CUTLASS tiles. + # One 128-wide tile can hold (128 // block_size) small blocks. + pack_factor = 128 // block_size + block_num = ((block_num + pack_factor - 1) // pack_factor) * pack_factor + + # Lager q values to detect split kv error + q = torch.randn(bs, h_q, d) * 100.0 + block_table = torch.randint(0, bs * block_num, (bs, block_num), dtype=torch.int32) + + kv_cache = torch.randn(block_table.numel(), block_size, d) + + workspace_size = cutlass_mla_get_workspace_size( + block_num * block_size, bs, num_kv_splits=num_kv_splits + ) + workspace = torch.empty(workspace_size, device="cuda", dtype=torch.uint8) + + q_nope = torch.empty((h_q, bs, dv)).transpose(0, 1) + q_nope.copy_(q[:, :, :dv]) + q_pe = q[:, :, dv:].clone() + + out_ref = q.new_zeros(bs, h_q, dv) + ref_mla(out_ref, q, kv_cache, scale, block_table, seq_lens) + out = cutlass_mla_decode( + q_nope, q_pe, kv_cache, seq_lens, block_table, workspace, scale, num_kv_splits + ) + + torch.testing.assert_close(out, out_ref, atol=1e-2, rtol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py b/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py new file mode 100644 index 000000000..3f9e60077 --- /dev/null +++ b/sgl-kernel/tests/test_cutlass_w4a8_moe_mm.py @@ -0,0 +1,283 @@ +import pytest +import torch +from sgl_kernel import cutlass_w4a8_moe_mm, sgl_per_tensor_quant_fp8 +from utils import is_hopper + + +def pack_int4_values_to_int8(int4_values_interleaved: torch.Tensor) -> torch.Tensor: + if int4_values_interleaved.shape[-1] % 2 != 0: + raise ValueError( + "the last dim size of int4_values_interleaved tensor must be even." + ) + + input_tensor_int8 = int4_values_interleaved.to(torch.int8) + + low_nibbles = input_tensor_int8[..., 0::2] + high_nibbles = input_tensor_int8[..., 1::2] + + packed_tensor = (high_nibbles << 4) | (low_nibbles & 0x0F) + + return packed_tensor.to(torch.int8) + + +def pack_interleave(num_experts, ref_weight, ref_scale): + n, k = ref_weight.shape[1], ref_weight.shape[2] + + weight = pack_int4_values_to_int8(ref_weight.cpu()).cuda() + w_q = weight.view((num_experts, n, k // 2)).view(torch.int8) + w_q = w_q.contiguous() + + alignment = 4 if k % 512 == 0 else 1 + scale_interleaved = ref_scale.reshape( + ref_scale.shape[0], + ref_scale.shape[1], + (ref_scale.shape[2] // alignment), + alignment, + ) # [E, N, K/4, 4] + scale_interleaved = scale_interleaved.permute(0, 2, 1, 3) # [E, K/4, N, 4] + scale_interleaved = scale_interleaved.reshape( + ref_scale.shape[0], + ref_scale.shape[2] // alignment, + ref_scale.shape[1] * alignment, + ) # [E, K/4, N*4] + w_scale = scale_interleaved.contiguous() + + return w_q, w_scale + + +@pytest.mark.skipif( + not is_hopper(), + reason="cutlass_w4a8_moe_mm is only supported on sm90", +) +@pytest.mark.parametrize("batch_size", [1, 2, 4, 8, 16]) +def test_int4_fp8_grouped_gemm_single_expert(batch_size): + # Test parameters + num_experts = 1 + m = batch_size # batch size + k = 512 # input dimension + n = 1024 # output dimension + torch.manual_seed(0) + dtype = torch.bfloat16 + device = "cuda" + debug = False + + print(f"\nTesting with batch_size={batch_size}") + + # Create input tensors with ones + if debug: + a = torch.ones(m, k, dtype=torch.bfloat16, device=device) + ref_w = torch.ones(num_experts, n, k, dtype=torch.int8, device=device) + ref_w_scale = torch.ones(num_experts, n, k // 128, dtype=dtype, device=device) + else: + a = torch.randn(m, k, dtype=dtype, device=device) + ref_w = torch.randint( + -8, 8, (num_experts, n, k), dtype=torch.int8, device=device + ) + affine_coeff = 0.005 + ref_w_scale = ( + torch.randn(num_experts, n, k // 128, dtype=dtype, device=device) + * affine_coeff + ) + + w, w_scale = pack_interleave(num_experts, ref_w, ref_w_scale) + + # Create expert offsets and problem sizes + expert_offsets = torch.tensor([0, m], dtype=torch.int32, device=device) + problem_sizes = torch.tensor([[n, m, k]], dtype=torch.int32, device=device) + + a_strides = torch.full((num_experts, 3), k, device=device, dtype=torch.int64) + c_strides = torch.full((num_experts, 3), n, device=device, dtype=torch.int64) + b_strides = a_strides + s_strides = c_strides + + # Quantize input + a_q, a_scale = _per_tensor_quant_fp8(a) + + # Create output tensor + c = torch.empty((m, n), dtype=torch.bfloat16, device=device) + cutlass_w4a8_moe_mm( + c, + a_q, + w, + a_scale, + w_scale, + expert_offsets[:-1], + problem_sizes, + a_strides, + b_strides, + c_strides, + s_strides, + 128, + 8, + ) + c = c.to(dtype) + + # Reference implementation + experts_selection_result = torch.full((m,), 0) + c_ref = ref_grouped_gemm( + c, a_q, a_scale, ref_w, ref_w_scale, num_experts, experts_selection_result + ) + + # Compare results + try: + torch.testing.assert_close(c, c_ref, rtol=1e-2, atol=0.1) + except AssertionError as e: + # torch.set_printoptions(threshold=10_000) + print(f" FAILURE: tensors are NOT close.") + print(f" Ref tensor: {c_ref.flatten()}") + print(f" Cutlass tensor: {c.flatten()}") + print( + f" Max absolute difference: {torch.max(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print( + f" Mean absolute difference: {torch.mean(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print(f" AssertionError: {e}") + raise + + +def _per_tensor_quant_fp8( + x: torch.Tensor, + dtype: torch.dtype = torch.float8_e4m3fn, +): + assert x.is_contiguous(), "`x` is not contiguous" + + x_q = torch.empty_like(x, device=x.device, dtype=dtype) + x_s = torch.empty( + 1, + device=x.device, + dtype=torch.float32, + ) + sgl_per_tensor_quant_fp8(x, x_q, x_s, is_static=False) + return x_q, x_s + + +@pytest.mark.skipif( + not is_hopper(), + reason="cutlass_w4a8_moe_mm is only supported on sm90", +) +@pytest.mark.parametrize("batch_size", [2, 4, 8, 16, 32]) +@pytest.mark.parametrize("k", [512, 1024, 2048, 4096, 7168]) +@pytest.mark.parametrize("n", [256, 512, 1024, 2048]) +@pytest.mark.parametrize("num_experts", [2, 4, 6, 8]) +def test_int4_fp8_grouped_gemm_multi_experts(batch_size, k, n, num_experts): + torch.manual_seed(0) + dtype = torch.bfloat16 + device = "cuda" + debug = False + + print( + f"\nTesting with batch_size={batch_size}, k={k}, n={n}, num_experts={num_experts}" + ) + + if debug: + a = torch.ones(batch_size, k, dtype=torch.bfloat16, device=device) + ref_w = torch.ones(num_experts, n, k, dtype=torch.int8, device=device) + ref_w_scale = torch.ones(num_experts, n, k // 128, dtype=dtype, device=device) + else: + a = torch.randn(batch_size, k, dtype=dtype, device=device) + ref_w = torch.randint( + -8, 8, (num_experts, n, k), dtype=torch.int8, device=device + ) + affine_coeff = 0.005 + ref_w_scale = ( + torch.randn(num_experts, n, k // 128, dtype=dtype, device=device) + * affine_coeff + ) + + w, w_scale = pack_interleave(num_experts, ref_w, ref_w_scale) + + # random select experts + experts_selection_result = torch.randint( + 0, num_experts, (batch_size,), device=device + ) + permutation = torch.argsort(experts_selection_result) + expert_token_counts = torch.bincount( + experts_selection_result, minlength=num_experts + ) + + # Create problem sizes and offsets for active experts + problem_sizes = [] + for i in range(num_experts): + problem_sizes.append([n, expert_token_counts[i].item(), k]) + problem_sizes = torch.tensor(problem_sizes, dtype=torch.int32, device=device) + + expert_offsets = [] + offset = 0 + for i in range(num_experts): + expert_offsets.append(offset) + offset += problem_sizes[i][1].item() + expert_offsets = torch.tensor(expert_offsets, dtype=torch.int32, device=device) + + # Permute input and quantize + a_q, a_scale = _per_tensor_quant_fp8(a) + a_q_perm = a_q[permutation] + + # Create stride tensors + a_strides = torch.full((num_experts, 3), k, device=device, dtype=torch.int64) + c_strides = torch.full((num_experts, 3), n, device=device, dtype=torch.int64) + b_strides = a_strides + s_strides = c_strides + + c_perm = torch.empty((batch_size, n), dtype=torch.bfloat16, device=device) + cutlass_w4a8_moe_mm( + c_perm, + a_q_perm, + w, + a_scale, + w_scale, + expert_offsets, + problem_sizes, + a_strides, + b_strides, + c_strides, + s_strides, + 128, + 8, + ) + + # Un-permute the result + c = torch.empty_like(c_perm) + c[permutation] = c_perm + c = c.to(dtype) + + c_ref = ref_grouped_gemm( + c, a_q, a_scale, ref_w, ref_w_scale, num_experts, experts_selection_result + ) + + # Compare results + try: + torch.testing.assert_close(c, c_ref, rtol=1e-2, atol=0.1) + except AssertionError as e: + print(f" FAILURE: tensors are NOT close.") + print( + f" Max absolute difference: {torch.max(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print( + f" Mean absolute difference: {torch.mean(torch.abs(c.to(c_ref.dtype) - c_ref))}" + ) + print(f" AssertionError: {e}") + raise + + +def ref_grouped_gemm( + c, a_q, a_scale, w, w_scale, num_experts, experts_selection_result +): + dtype = torch.bfloat16 + c_ref = torch.zeros_like(c) + for i in range(num_experts): + token_idx = torch.where(experts_selection_result == i)[0] + if len(token_idx) == 0: + continue + a = a_q[token_idx] + + ref_w_scale_repeat = w_scale[i].repeat_interleave(128, dim=1).to(torch.float32) + ref_w = w[i].to(torch.float32) * ref_w_scale_repeat + c = torch.matmul(a.to(torch.float32), ref_w.t()) * a_scale + c_ref[token_idx] = c.to(dtype) + + return c_ref + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_dsv3_fused_a_gemm.py b/sgl-kernel/tests/test_dsv3_fused_a_gemm.py new file mode 100644 index 000000000..914af95e2 --- /dev/null +++ b/sgl-kernel/tests/test_dsv3_fused_a_gemm.py @@ -0,0 +1,32 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import dsv3_fused_a_gemm + + +@pytest.mark.parametrize("num_tokens", [i + 1 for i in range(16)]) +def test_dsv3_fused_a_gemm(num_tokens): + kHdIn = 7168 + kHdOut = 2112 + + mat_a = torch.randn( + (num_tokens, kHdIn), dtype=torch.bfloat16, device="cuda" + ).contiguous() + mat_b = torch.randn((kHdOut, kHdIn), dtype=torch.bfloat16, device="cuda").transpose( + 0, 1 + ) + output = torch.empty( + (num_tokens, kHdOut), dtype=torch.bfloat16, device="cuda" + ).contiguous() + + ref = F.linear(mat_a, mat_b.T) + + output = dsv3_fused_a_gemm(mat_a, mat_b) + + assert torch.allclose( + output, ref, rtol=1e-2, atol=1e-3 + ), "Fused GEMM output mismatch with torch.nn.functional.linear reference" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_dsv3_router_gemm.py b/sgl-kernel/tests/test_dsv3_router_gemm.py new file mode 100644 index 000000000..575769d6d --- /dev/null +++ b/sgl-kernel/tests/test_dsv3_router_gemm.py @@ -0,0 +1,35 @@ +import pytest +import torch +import torch.nn.functional as F +from sgl_kernel import dsv3_router_gemm + + +@pytest.mark.parametrize("num_tokens", [i + 1 for i in range(16)]) +@pytest.mark.parametrize("num_experts", [256, 384]) +def test_dsv3_router_gemm(num_tokens, num_experts): + hidden_dim = 7168 + + mat_a = torch.randn( + (num_tokens, hidden_dim), dtype=torch.bfloat16, device="cuda" + ).contiguous() + mat_b = torch.randn( + (num_experts, hidden_dim), dtype=torch.bfloat16, device="cuda" + ).contiguous() + + bf16_ref = F.linear(mat_a, mat_b) + float_ref = bf16_ref.to(torch.float32) + + bf16_output = dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.bfloat16) + float_output = dsv3_router_gemm(mat_a, mat_b, out_dtype=torch.float32) + + assert torch.allclose( + bf16_output, bf16_ref, rtol=1e-2, atol=1e-3 + ), "Router GEMM output in bf16 dtype mismatch with torch.nn.functional.linear reference" + + assert torch.allclose( + float_output, float_ref, rtol=1e-2, atol=1e-3 + ), "Router GEMM output in float32 dtype mismatch with torch.nn.functional.linear reference" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_flash_attention.py b/sgl-kernel/tests/test_flash_attention.py new file mode 100644 index 000000000..159390e54 --- /dev/null +++ b/sgl-kernel/tests/test_flash_attention.py @@ -0,0 +1,1368 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/test_flash_attn.py +import itertools +import math +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + +apply_rotary_emb = None + + +def is_hopper(): + # Only Hopper supports different V headdim + return torch.cuda.get_device_properties(0).major == 9 + + +def is_fa3_supported(device=None) -> bool: + # There some fa3 FYI + # FA3 can fail without a enough shared memory for a some shapes, such as higher + # hidden_dim or some special cases. + # Right now, fa3 is supported for sm80/sm87 and sm86/sm89. The main different + # Between sm80/sm87 and sm86/sm89 is the shared memory size. you can follow the link below for more information + # https://docs.nvidia.com/cuda/cuda-c-programming-guide/#shared-memory-8-x + # And for sgl-kernel right now, we can build fa3 on sm80/sm86/sm89/sm90a. + # That means if you use A100/A*0/L20/L40/L40s/4090 you can use fa3. + return (torch.version.cuda >= "12.3") and ( + torch.cuda.get_device_capability(device)[0] == 9 + or torch.cuda.get_device_capability(device)[0] == 8 + ) + + +DISABLE_BACKWARD = True +# For CI test, we close them to True. +# DISABLE_SPLIT = os.getenv("FLASH_ATTENTION_DISABLE_SPLIT", "FALSE") == "TRUE" +# DISABLE_PAGEDKV = os.getenv("FLASH_ATTENTION_DISABLE_PAGEDKV", "FALSE") == "TRUE" +# DISABLE_APPENDKV = os.getenv("FLASH_ATTENTION_DISABLE_APPENDKV", "FALSE") == "TRUE" +# DISABLE_LOCAL = os.getenv("FLASH_ATTENTION_DISABLE_LOCAL", "FALSE") == "TRUE" +# DISABLE_SOFTCAP = os.getenv("FLASH_ATTENTION_DISABLE_SOFTCAP", "FALSE") == "TRUE" +# DISABLE_PACKGQA = os.getenv("FLASH_ATTENTION_DISABLE_PACKGQA", "FALSE") == "TRUE" +# DISABLE_FP16 = os.getenv("FLASH_ATTENTION_DISABLE_FP16", "FALSE") == "TRUE" +# DISABLE_FP8 = ( +# os.getenv("FLASH_ATTENTION_DISABLE_FP8", "FALSE") == "TRUE" +# or torch.cuda.get_device_capability("cuda")[0] < 9 +# ) + +DISABLE_SPLIT = False +DISABLE_PAGEDKV = True +DISABLE_APPENDKV = False +DISABLE_LOCAL = False +DISABLE_SOFTCAP = True +DISABLE_PACKGQA = False +DISABLE_FP16 = True +DISABLE_FP8 = True + + +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/main/hopper/padding.py +def unpad_input(hidden_states, attention_mask, unused_mask=None): + """ + Arguments: + hidden_states: (batch, seqlen, ...) + attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. + unused_mask: (batch, seqlen), bool / int, 1 means the element is allocated but unused. + Return: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask + unused_mask. + indices: (total_nnz), the indices of masked tokens from the flattened input sequence. + cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states. + max_seqlen_in_batch: int + seqused: (batch), returns the number of tokens selected in attention_mask + unused_mask. + """ + all_masks = ( + (attention_mask + unused_mask) if unused_mask is not None else attention_mask + ) + seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) + used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the + # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim + # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to + # index with integer indices. + return ( + rearrange(hidden_states, "b s ... -> (b s) ...")[indices], + indices, + cu_seqlens, + max_seqlen_in_batch, + used_seqlens_in_batch, + ) + + +def generate_random_padding_mask( + max_seqlen, batch_size, device, mode="random", zero_lengths=False +): + assert mode in ["full", "random", "third"] + if mode == "full": + lengths = torch.full( + (batch_size, 1), max_seqlen, device=device, dtype=torch.int32 + ) + elif mode == "random": + lengths = torch.randint( + max(0 if zero_lengths else 1, max_seqlen - 20), + max_seqlen + 1, + (batch_size, 1), + device=device, + ) + elif mode == "third": + lengths = torch.randint( + max_seqlen // 3, max_seqlen + 1, (batch_size, 1), device=device + ) + + if zero_lengths: + # Generate zero-lengths every 5 batches and the last batch. + for i in range(batch_size): + if i % 5 == 0: + lengths[i] = 0 + lengths[-1] = 0 + padding_mask = ( + repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) + < lengths + ) + return padding_mask + + +def pad_input(hidden_states, indices, batch, seqlen): + """ + Arguments: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. + indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. + batch: int, batch size for the padded sequence. + seqlen: int, maximum sequence length for the padded sequence. + Return: + hidden_states: (batch, seqlen, ...) + """ + dim = hidden_states.shape[1:] + output = torch.zeros( + (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype + ) + output[indices] = hidden_states + return rearrange(output, "(b s) ... -> b s ...", b=batch) + + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] < 0: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + torch.logical_and( + col_idx < row_idx + sk - sq - window_size[0], + col_idx >= sink_token_length, + ), + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(-1, -1), # -1 means infinite window size + sink_token_length=0, + sinks: Optional[torch.Tensor] = None, + softcap=0.0, + upcast=True, + reorder_ops=False, + intermediate_dtype=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads, head_dim) + v: (batch_size, seqlen_k, nheads, head_dim_v) + qv: (batch_size, seqlen_q, nheads, head_dim_v) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim_v) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + qv = qv.float() if qv is not None else None + if q_descale is not None: + q_descale = repeat(q_descale, "b h -> b 1 (h g) 1", g=q.shape[2] // k.shape[2]) + q = (q.float() * q_descale).to(q.dtype) + qv = (qv.float() * q_descale).to(qv.dtype) if qv is not None else None + if k_descale is not None: + k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) + if v_descale is not None: + v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + dv = v.shape[-1] + softmax_scale = 1.0 / math.sqrt(d if qv is None else d + dv) + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q * softmax_scale, k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale) + if qv is not None: + scores = scores + torch.einsum("bthd,bshd->bhts", qv * softmax_scale, v) + if softcap > 0: + scores = torch.tanh(scores / softcap) * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + sink_token_length, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + if sinks is None: + attention = torch.softmax(scores, dim=-1).to(v.dtype) + else: + scores_fp32 = scores.to(torch.float32) + logits_max = torch.amax(scores_fp32, dim=-1, keepdim=True) + sinks = rearrange(sinks, "h -> h 1 1") + logits_or_sinks_max = torch.maximum(sinks, logits_max) + unnormalized_scores = torch.exp(scores_fp32 - logits_or_sinks_max) + normalizer = unnormalized_scores.sum(dim=-1, keepdim=True) + torch.exp( + sinks - logits_or_sinks_max + ) + attention = (unnormalized_scores / normalizer).to(v.dtype) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + # Without this we might get NaN in dv + if key_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0 + ) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + if intermediate_dtype is not None: + attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) + + +def generate_qkv( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + kvpacked=False, + qkvpacked=False, + add_unused_qkv=False, + query_unused_mask=None, + key_unused_mask=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, d) + k: (batch_size, seqlen_k, nheads_k, d) + v: (batch_size, seqlen_k, nheads_k, d) + query_padding_mask: (batch_size, seqlen), bool + key_padding_mask: (batch_size, seqlen), bool + """ + assert not (kvpacked and qkvpacked) + batch_size, seqlen_q, nheads, d = q.shape + _, seqlen_k, nheads_k, _ = k.shape + assert k.shape == (batch_size, seqlen_k, nheads_k, d) + assert v.shape == (batch_size, seqlen_k, nheads_k, d) + if query_unused_mask is not None or key_unused_mask is not None: + assert not kvpacked + assert not qkvpacked + + if query_padding_mask is not None: + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( + q, + query_padding_mask, + query_unused_mask, + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + else: + q_unpad = rearrange(q, "b s h d -> (b s) h d") + cu_seqlens_q = torch.arange( + 0, + (batch_size + 1) * seqlen_q, + step=seqlen_q, + dtype=torch.int32, + device=q_unpad.device, + ) + seqused_q = None + max_seqlen_q = seqlen_q + output_pad_fn = lambda output_unpad: rearrange( + output_unpad, "(b s) h d -> b s h d", b=batch_size + ) + + if key_padding_mask is not None: + k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( + k, key_padding_mask, key_unused_mask + ) + v_unpad, _, _, _, _ = unpad_input(v, key_padding_mask, key_unused_mask) + else: + k_unpad = rearrange(k, "b s h d -> (b s) h d") + v_unpad = rearrange(v, "b s h d -> (b s) h d") + cu_seqlens_k = torch.arange( + 0, + (batch_size + 1) * seqlen_k, + step=seqlen_k, + dtype=torch.int32, + device=k_unpad.device, + ) + seqused_k = None + max_seqlen_k = seqlen_k + + if qkvpacked: + assert (query_padding_mask == key_padding_mask).all() + assert nheads == nheads_k + qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) + qkv = torch.stack([q, k, v], dim=2) + if query_padding_mask is not None: + dqkv_pad_fn = lambda dqkv_unpad: pad_input( + dqkv_unpad, indices_q, batch_size, seqlen_q + ) + else: + dqkv_pad_fn = lambda dqkv_unpad: rearrange( + dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + qkv_unpad.detach().requires_grad_(), + cu_seqlens_q, + max_seqlen_q, + qkv.detach().requires_grad_(), + output_pad_fn, + dqkv_pad_fn, + ) + elif kvpacked: + kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) + kv = torch.stack([k, v], dim=2) + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dkv_pad_fn = lambda dkv_unpad: pad_input( + dkv_unpad, indices_k, batch_size, seqlen_k + ) + else: + dkv_pad_fn = lambda dkv_unpad: rearrange( + dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + kv_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + kv.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dkv_pad_fn, + ) + else: + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dk_pad_fn = lambda dk_unpad: pad_input( + dk_unpad, indices_k, batch_size, seqlen_k + ) + else: + dk_pad_fn = lambda dk_unpad: rearrange( + dk_unpad, "(b s) h d -> b s h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + k_unpad.detach().requires_grad_(), + v_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + k.detach().requires_grad_(), + v.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize( + "dtype", [torch.bfloat16] + ([torch.float8_e4m3fn] if not DISABLE_FP8 else []) +) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("has_sink", [False, True]) +# @pytest.mark.parametrize("has_sink", [False]) +@pytest.mark.parametrize("new_kv", [False] + ([True] if not DISABLE_APPENDKV else [])) +# @pytest.mark.parametrize("new_kv", [True]) +# @pytest.mark.parametrize( +# "causal,local", +# [(False, False), (True, False)] + ([(False, True)] if not DISABLE_LOCAL else []), +# ) +# @pytest.mark.parametrize("causal,local", [(False, False), (True, False)]) +@pytest.mark.parametrize("causal,local", [(False, False)]) +@pytest.mark.parametrize( + "seqlen_new_eq_seqlen_q", [True, False] if not DISABLE_APPENDKV else [True] +) +# @pytest.mark.parametrize("seqlen_new_eq_seqlen_q", [True]) +# @pytest.mark.parametrize("has_rotary_seqlens", [False, True]) +@pytest.mark.parametrize("has_rotary_seqlens", [False]) +@pytest.mark.parametrize( + "rotary_interleaved", [False, True] if not DISABLE_APPENDKV else [False] +) +# @pytest.mark.parametrize("rotary_interleaved", [True]) +@pytest.mark.parametrize( + "rotary_fraction", + ( + [0.0, 0.5, 1.0] + if (not DISABLE_APPENDKV) and (apply_rotary_emb is not None) + else [0.0] + ), +) +# @pytest.mark.parametrize("rotary_fraction", [0.0]) +@pytest.mark.parametrize( + "page_size", [None] + ([1, 4, 128] if not DISABLE_PAGEDKV else []) +) +# @pytest.mark.parametrize("page_size", [None]) +# @pytest.mark.parametrize("has_leftpad", [False, True]) +@pytest.mark.parametrize("has_leftpad", [False]) +# @pytest.mark.parametrize("has_batch_idx", [False, True]) +@pytest.mark.parametrize("has_batch_idx", [False]) +# @pytest.mark.parametrize("varlen_q", [False, True]) +@pytest.mark.parametrize("varlen_q", [False]) +# @pytest.mark.parametrize("d", [32, 59, 64, 80, 128, 256]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +@pytest.mark.parametrize("d", [64]) +# @pytest.mark.parametrize("d", [192]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 128), + (1, 339), + (3, 1024), + (64, 800), + (64, 256), + (3, 799), + (64, 2048), + (16, 20000), + # (1, 128 * 1024), + # (16, 128 * 1024), + (128, 128), + (256, 512), # To test appending KV with more than 1 block + (2048, 3577), # Enough tile to test persistent scheduler + ], +) +# @pytest.mark.parametrize('seqlen_q,seqlen_k', [(256, 128)]) +def test_flash_attn_kvcache( + seqlen_q, + seqlen_k, + d, + varlen_q, + has_batch_idx, + has_leftpad, + page_size, + rotary_fraction, + rotary_interleaved, + has_rotary_seqlens, + seqlen_new_eq_seqlen_q, + causal, + local, + new_kv, + mha_type, + dtype, + has_sink, +): + from sgl_kernel.flash_attn import flash_attn_with_kvcache + + if page_size is not None and seqlen_k % page_size != 0: + pytest.skip() + if seqlen_q > seqlen_k and new_kv: + pytest.skip() + if not new_kv and rotary_fraction > 0.0: + pytest.skip() + if rotary_fraction == 0.0 and has_rotary_seqlens: + pytest.skip() + device = "cuda" + # set seed + torch.random.manual_seed(0) + batch_size = 5 + # batch_size = 1 + batch_size_cache = batch_size if not has_batch_idx else batch_size * 2 + nheads = 6 + # nheads = 1 + # rotary_dim must be a multiple of 16, and must be <= d + rotary_dim = math.floor(int(rotary_fraction * d) / 16) * 16 + nheads_k = nheads if mha_type == "mha" else (1 if mha_type == "mqa" else 3) + assert nheads % nheads_k == 0 + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + + if has_sink: + sinks = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + sinks = None + + if dtype == torch.float8_e4m3fn or not is_hopper(): + # for fp8 and ampere arch, we not support v head dim != qk head dim + dv_vals = [d] + for dv in dv_vals: + has_qv = d == 64 and dv >= 256 + q = ( + torch.randn(batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + if has_qv: + qv = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv = None + if varlen_q: + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random" + ) + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, *rest = unpad_input( + q, query_padding_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + qv_unpad = ( + rearrange(qv, "b s ... -> (b s) ...")[indices_q] if has_qv else None + ) + else: + query_padding_mask = None + q_unpad = q + qv_unpad = qv + cu_seqlens_q, max_seqlen_q = None, None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + + seqlen_new = ( + seqlen_q + if seqlen_new_eq_seqlen_q + else torch.randint(1, seqlen_q + 1, (1,)).item() + ) + cu_seqlens_k_new = None + key_new_padding_mask = None + if new_kv: + k = ( + torch.randn( + batch_size, seqlen_new, nheads_k, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + v = ( + torch.randn( + batch_size, seqlen_new, nheads_k, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + if varlen_q: # k & v are also varlen + key_new_padding_mask = generate_random_padding_mask( + seqlen_new, batch_size, device, mode="random" + ) + k_unpad, indices_k, cu_seqlens_k_new, *rest = unpad_input( + k, key_new_padding_mask + ) + v_unpad, *rest = unpad_input(v, key_new_padding_mask) + else: + k_unpad, v_unpad = k, v + else: + k, v, k_unpad, v_unpad = None, None, None, None + if page_size is None: + k_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + d, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + v_cache = ( + torch.randn( + batch_size_cache, + seqlen_k, + nheads_k, + dv, + device=device, + dtype=dtype_ref, + ) + .to(dtype) + .to(dtype_ref) + ) + page_table = None + else: + ( + k_cache, + v_cache, + page_table, + k_cache_paged, + v_cache_paged, + num_blocks, + ) = _generate_block_kvcache( + seqlen_k, + page_size, + batch_size_cache, + nheads_k, + d, + dv, + device, + dtype, + dtype_ref, + ) + cache_seqlens = torch.randint( + 0 if new_kv else 1, + # If we don't use seqlen_q in the case of causal and rotary, cos/sin won't be long enough + ( + ( + seqlen_k + - (seqlen_q if (causal or local) and rotary_dim > 1 else seqlen_new) + + 1 + ) + if new_kv + else (seqlen_k + 1) + ), + (batch_size,), + dtype=torch.int32, + device=device, + ) + if has_leftpad: + cache_leftpad = torch.cat( + [ + ( + torch.randint( + 0, + cache_seqlens[i].item(), + (1,), + dtype=torch.int32, + device=device, + ) + if cache_seqlens[i].item() > 0 + else torch.zeros(1, dtype=torch.int32, device=device) + ) + for i in range(batch_size) + ] + ) + else: + cache_leftpad = None + if has_batch_idx: + cache_batch_idx = torch.randperm( + batch_size_cache, dtype=torch.int32, device=device + )[:batch_size] + else: + cache_batch_idx = None + arange = rearrange(torch.arange(seqlen_k, device=device), "s -> 1 s") + cache_seqlens_expanded = rearrange(cache_seqlens, "b -> b 1") + if not new_kv: + key_padding_mask = arange < cache_seqlens_expanded + else: + k_new_seqlens = ( + key_new_padding_mask.sum(-1, keepdims=True) if varlen_q else seqlen_new + ) + key_padding_mask = arange < cache_seqlens_expanded + k_new_seqlens + if has_leftpad: + key_padding_mask = torch.logical_and( + key_padding_mask, + arange >= cache_leftpad.unsqueeze(-1).expand(-1, seqlen_k), + ) + # cache_seqlens = torch.tensor([64], dtype=torch.int32, device=device) + rotary_seqlens = cache_seqlens if not has_rotary_seqlens else cache_seqlens // 2 + if rotary_dim > 0: + angle = ( + torch.rand( + seqlen_k if page_size is None else num_blocks * page_size, + rotary_dim // 2, + device=device, + ) + * 2 + * math.pi + ) + cos = torch.cos(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + sin = torch.sin(angle).to(dtype=dtype_ref).to(dtype).to(dtype_ref) + if causal or local: + q_ro = apply_rotary_emb( + q, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + q_ro = rearrange( + apply_rotary_emb( + rearrange(q, "b s h d -> b 1 (s h) d"), + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ), + "b 1 (s h) d -> b s h d", + s=seqlen_q, + ) + # q_ro = q + k_ro = apply_rotary_emb( + k, + cos, + sin, + seqlen_offsets=rotary_seqlens, + interleaved=rotary_interleaved, + ) + else: + cos, sin = None, None + q_ro, k_ro = q, k + # k_cache[:, 64:] = -1 + k_cache_ref = ( + k_cache if not has_batch_idx else k_cache[cache_batch_idx] + ).clone() + v_cache_ref = ( + v_cache if not has_batch_idx else v_cache[cache_batch_idx] + ).clone() + if new_kv: + update_mask = torch.logical_and( + cache_seqlens_expanded <= arange, + arange < cache_seqlens_expanded + k_new_seqlens, + ) + k_to_update = rearrange(k_ro, "b s ... -> (b s) ...") + v_to_update = rearrange(v, "b s ... -> (b s) ...") + if varlen_q: + k_to_update = k_to_update[indices_k] + v_to_update = v_to_update[indices_k] + k_cache_ref[update_mask] = k_to_update + v_cache_ref[update_mask] = v_to_update + k_cache_rep = repeat( + k_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + v_cache_rep = repeat( + v_cache_ref, "b s h d -> b s (h g) d", g=nheads // nheads_k + ) + out_ref, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + key_leftpad=cache_leftpad, + sinks=sinks, + ) + out_pt, _ = attention_ref( + q_ro, + k_cache_rep, + v_cache_rep, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv, + window_size=window_size, + upcast=False, + reorder_ops=True, + key_leftpad=cache_leftpad, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + sinks=sinks, + ) + q = q.to(dtype) + q_unpad = q_unpad.to(dtype) if varlen_q else None + k_cache = k_cache.to(dtype) + v_cache = v_cache.to(dtype) + k_cache_paged = k_cache_paged.to(dtype) if page_size is not None else None + v_cache_paged = v_cache_paged.to(dtype) if page_size is not None else None + k = k.to(dtype) if k is not None else None + v = v.to(dtype) if v is not None else None + k_unpad = k_unpad.to(dtype) if k_unpad is not None else None + v_unpad = v_unpad.to(dtype) if v_unpad is not None else None + qv = qv.to(dtype) if qv is not None else None + qv_unpad = qv_unpad.to(dtype) if (varlen_q and qv is not None) else None + cos = cos.to(dtype) if cos is not None else None + sin = sin.to(dtype) if sin is not None else None + k_cache_saved = k_cache.clone() if page_size is None else k_cache_paged.clone() + v_cache_saved = v_cache.clone() if page_size is None else v_cache_paged.clone() + num_splits_vals = [1, 0] if not DISABLE_SPLIT else [1] + precompute_metadata_vals = [False] + for num_splits, precompute_metadata in itertools.product( + num_splits_vals, precompute_metadata_vals + ): + scheduler_metadata = None + # Repeat to test metadata reuse + for _ in range(1 if not precompute_metadata else 2): + if page_size is None: + k_cache.copy_(k_cache_saved) + v_cache.copy_(v_cache_saved) + else: + k_cache_paged.copy_(k_cache_saved) + v_cache_paged.copy_(v_cache_saved) + out, lse, *rest = flash_attn_with_kvcache( + q if not varlen_q else q_unpad, + k_cache if page_size is None else k_cache_paged, + v_cache if page_size is None else v_cache_paged, + k if not new_kv or not varlen_q else k_unpad, + v if not new_kv or not varlen_q else v_unpad, + qv=qv if not varlen_q else qv_unpad, + rotary_cos=cos, + rotary_sin=sin, + cache_seqlens=cache_seqlens, + cache_batch_idx=cache_batch_idx, + cache_leftpad=cache_leftpad, + page_table=page_table, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k_new=cu_seqlens_k_new, + max_seqlen_q=max_seqlen_q, + rotary_seqlens=rotary_seqlens, + causal=causal, + window_size=window_size, + rotary_interleaved=rotary_interleaved, + scheduler_metadata=scheduler_metadata, + num_splits=num_splits, + return_softmax_lse=True, + sinks=sinks, + ) + if varlen_q: + out = output_pad_fn(out) + # out = flash_attn_with_kvcache( + # q, k_cache, v_cache, cache_seqlens=cache_seqlens, causal=causal, window_size=window_size + # ) + # out = flash_attn_with_kvcache(q, k_cache, v_cache, causal=causal, window_size=window_size) + # qk = torch.einsum("bqhd,bkhd->bhqk", q, k_cache_ref) + # m = qk.amax(-1, keepdim=True) + # s_tmp = torch.exp((qk - m) / math.sqrt(d)) + # o1 = torch.einsum('bhst,bthd->bshd', s_tmp, v_cache_ref) + # lse_ref = torch.logsumexp(qk / math.sqrt(d), -1) + # probs = torch.softmax(qk, dim=-1) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most twice the numerical error + # of a Pytorch implementation. + if new_kv: + if page_size is None: + k_cache_select = ( + k_cache.to(dtype_ref) + if not has_batch_idx + else k_cache.to(dtype_ref)[cache_batch_idx] + ) + v_cache_select = ( + v_cache.to(dtype_ref) + if not has_batch_idx + else v_cache.to(dtype_ref)[cache_batch_idx] + ) + else: + k_cache_select = rearrange( + k_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + v_cache_select = rearrange( + v_cache_paged.to(dtype_ref)[ + ( + page_table + if not has_batch_idx + else page_table[cache_batch_idx] + ).flatten() + ], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k].to(dtype_ref) + k_cache_ref = k_cache_ref.to(dtype).to(dtype_ref) + v_cache_ref = v_cache_ref.to(dtype).to(dtype_ref) + if dtype is not torch.float8_e4m3fn: + assert torch.equal(v_cache_select, v_cache_ref) + else: + assert torch.allclose( + v_cache_select, v_cache_ref, rtol=1e-3, atol=1e-3 + ) + # breakpoint() + # if rotary_dim == 0 and dtype is not torch.float8_e4m3fn: + if rotary_dim == 0: + assert torch.equal(k_cache_select, k_cache_ref) + else: + # if not torch.allclose(k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3): + # breakpoint() + if dtype is not torch.float8_e4m3fn: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-3, atol=1e-3 + ) + else: + assert torch.allclose( + k_cache_select, k_cache_ref, rtol=1e-1, atol=1e-1 + ) + mult = 4 if dtype == torch.float8_e4m3fn else 2 + assert (out - out_ref).abs().max().item() <= mult * ( + out_pt - out_ref + ).abs().max().item() + 1e-5 + mult_mean = 3 if dtype == torch.float8_e4m3fn else 1.5 + assert (out - out_ref).abs().mean().item() <= mult_mean * ( + out_pt - out_ref + ).abs().mean().item() + + +def _generate_block_kvcache( + seqlen_k, page_size, batch_size, nheads_k, d, dv, device, dtype, dtype_ref +): + num_blocks = math.ceil(seqlen_k / page_size) * batch_size * 3 + k_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, d, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + v_cache_paged = ( + torch.randn(num_blocks, page_size, nheads_k, dv, device=device, dtype=dtype_ref) + .to(dtype) + .to(dtype_ref) + ) + page_table = rearrange( + torch.randperm(num_blocks, dtype=torch.int32, device=device), + "(b nblocks) -> b nblocks", + b=batch_size, + ) + k_cache = rearrange( + k_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + v_cache = rearrange( + v_cache_paged[page_table.flatten()], + "(b nblocks) block_size ... -> b (nblocks block_size) ...", + b=batch_size, + )[:, :seqlen_k] + return k_cache, v_cache, page_table, k_cache_paged, v_cache_paged, num_blocks + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize( + "dtype", [torch.bfloat16] + ([torch.float8_e4m3fn] if not DISABLE_FP8 else []) +) +# @pytest.mark.parametrize("dtype", [torch.bfloat16]) +# @pytest.mark.parametrize("dtype", [torch.float8_e4m3fn]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mha"]) +@pytest.mark.parametrize("has_sink", [False, True]) +# @pytest.mark.parametrize("has_sink", [False]) +# @pytest.mark.parametrize("has_qv", [False, True]) +@pytest.mark.parametrize("has_qv", [False]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +@pytest.mark.parametrize("softcap", [0.0] + ([15.0] if not DISABLE_SOFTCAP else [])) +# @pytest.mark.parametrize("softcap", [0.0]) +@pytest.mark.parametrize("local", [False]) +# @pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +@pytest.mark.parametrize("add_unused_qkv", [False, True]) +# @pytest.mark.parametrize("add_unused_qkv", [True]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128]) +# @pytest.mark.parametrize("d", COMPILED_HDIMS) +@pytest.mark.parametrize("d", [128]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + (1, 1), + (1, 3), + (2, 1), + (511, 1), + (3, 513), + (64, 128), + (128, 128), + (256, 256), + (113, 203), + (128, 217), + (113, 211), + (108, 256), + (256, 512), + (307, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (2048, 2048), + ], +) +def test_flash_attn_varlen_output( + seqlen_q, + seqlen_k, + d, + add_unused_qkv, + causal, + local, + softcap, + deterministic, + has_qv, + mha_type, + dtype, + has_sink, +): + from sgl_kernel.flash_attn import flash_attn_varlen_func + + device = "cuda" + # set seed + torch.random.manual_seed(seqlen_q + seqlen_k + d + int(causal) * 2 + int(local)) + # batch_size = 40 + # nheads = 16 + batch_size = 9 if seqlen_q <= 2048 else 2 + nheads = 6 + # batch_size = 2 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (2 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + if dtype == torch.float8_e4m3fn: + dv_vals = [d] + for dv in dv_vals: + q_ref = torch.randn( + batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref + ) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4).detach().requires_grad_() + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + v_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + if has_qv: + qv_ref = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv_ref = None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = (-1, -1) if not local else torch.randint(0, seqlen_k, (2,)) + + if has_sink: + sinks = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + sinks = None + + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [ + torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) + * 2 + for _ in range(3) + ] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().requires_grad_() for x in (q_ref, k_ref, v_ref)] + qv = qv_ref.detach() if has_qv else None + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random", zero_lengths=False + ) + key_padding_mask = generate_random_padding_mask( + seqlen_k, batch_size, device, mode="random", zero_lengths=True + ) + + def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device): + if add_unused: + another_mask = generate_random_padding_mask(max_seq_len, bs, device) + attn_mask = torch.logical_and(padding_mask, another_mask) + unused_mask = torch.logical_xor( + torch.logical_or(padding_mask, another_mask), attn_mask + ) + else: + attn_mask = padding_mask + unused_mask = None + return attn_mask, unused_mask + + query_padding_mask, query_unused_mask = _gen_unused_masks( + query_padding_mask, add_unused_qkv, seqlen_q, batch_size, q.device + ) + key_padding_mask, key_unused_mask = _gen_unused_masks( + key_padding_mask, add_unused_qkv, seqlen_k, batch_size, k.device + ) + + ( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q, + k, + v, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) = generate_qkv( + q, + k, + v, + query_padding_mask, + key_padding_mask, + kvpacked=False, + query_unused_mask=query_unused_mask, + key_unused_mask=key_unused_mask, + ) + q_unpad, k_unpad, v_unpad = [ + x.detach().to(dtype).requires_grad_() for x in (q_unpad, k_unpad, v_unpad) + ] + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + sinks=sinks, + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + sinks=sinks, + ) + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + + if query_unused_mask is not None: + q_zero_masking = rearrange(query_unused_mask, "b s -> b s 1 1") + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + pack_gqa_vals = [False, True] if not DISABLE_PACKGQA else [False] + num_splits_vals = [1, 3] if not DISABLE_SPLIT else [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out_unpad, lse, *rest = flash_attn_varlen_func( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + seqused_q=seqused_q, + seqused_k=seqused_k, + causal=causal, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + softcap=softcap, + return_softmax_lse=True, + sinks=sinks, + ) + out = output_pad_fn(out_unpad) + if query_unused_mask is not None: + out.masked_fill_(q_zero_masking, 0.0) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + + # Check that FlashAttention's numerical error is at most 3x the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * ( + out_pt - out_ref + ).abs().max().item() + fwd_atol + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not has_qv: + g_unpad = torch.randn_like(out_unpad) + do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2) + dq_unpad, dk_unpad, dv_unpad = torch.autograd.grad( + out_unpad, (q_unpad, k_unpad, v_unpad), g_unpad + ) + dq = dq_pad_fn(dq_unpad) + dk = dk_pad_fn(dk_unpad) + dv = dk_pad_fn(dv_unpad) + if key_unused_mask is not None: + k_zero_masking = rearrange(key_unused_mask, "b s -> b s 1 1") + dk.masked_fill_(k_zero_masking, 0.0) + dv.masked_fill_(k_zero_masking, 0.0) + if query_unused_mask is not None: + dq.masked_fill_(q_zero_masking, 0.0) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + g = output_pad_fn(g_unpad) + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad(out_ref, (q_ref, k_ref, v_ref), g) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + + if not DISABLE_BACKWARD and dtype != torch.float8_e4m3fn and not has_qv: + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dq - dq_ref).abs().max().item() <= rtol * ( + dq_pt - dq_ref + ).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dk - dk_ref).abs().max().item() <= rtol * ( + dk_pt - dk_ref + ).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dv - dv_ref).abs().max().item() <= rtol * ( + dv_pt - dv_ref + ).abs().max().item() + dv_atol + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_flash_attention_4.py b/sgl-kernel/tests/test_flash_attention_4.py new file mode 100644 index 000000000..f50492923 --- /dev/null +++ b/sgl-kernel/tests/test_flash_attention_4.py @@ -0,0 +1,877 @@ +# Adapted from https://github.com/Dao-AILab/flash-attention/blob/b31ae1e4cd22cf5f820a2995b74b7cd3bd54355a/tests/cute/test_flash_attn.py + +# Copyright (c) 2025, Jay Shah, Ganesh Bikshandi, Ying Zhang, Vijay Thakkar, Pradeep Ramani, Tri Dao. + +import itertools +import math +from functools import partial + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange, repeat +from sgl_kernel.flash_attn import flash_attn_varlen_func +from utils import is_hopper + +flash_attn_varlen_func = partial(flash_attn_varlen_func, ver=4) + + +def unpad_input(hidden_states, attention_mask, unused_mask=None): + """ + Arguments: + hidden_states: (batch, seqlen, ...) + attention_mask: (batch, seqlen), bool / int, 1 means valid and 0 means not valid. + unused_mask: (batch, seqlen), bool / int, 1 means the element is allocated but unused. + Return: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens selected in attention_mask + unused_mask. + indices: (total_nnz), the indices of masked tokens from the flattened input sequence. + cu_seqlens: (batch + 1), the cumulative sequence lengths, used to index into hidden_states. + max_seqlen_in_batch: int + seqused: (batch), returns the number of tokens selected in attention_mask + unused_mask. + """ + all_masks = ( + (attention_mask + unused_mask) if unused_mask is not None else attention_mask + ) + seqlens_in_batch = all_masks.sum(dim=-1, dtype=torch.int32) + used_seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) + indices = torch.nonzero(all_masks.flatten(), as_tuple=False).flatten() + max_seqlen_in_batch = seqlens_in_batch.max().item() + cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0)) + # TD [2022-03-04] We don't want to index with a bool mask, because Pytorch will expand the + # bool mask, then call nonzero to get the indices, then index with those. The indices is @dim + # times larger than it needs to be, wasting memory. It's faster and more memory-efficient to + # index with integer indices. + return ( + rearrange(hidden_states, "b s ... -> (b s) ...")[indices], + indices, + cu_seqlens, + max_seqlen_in_batch, + used_seqlens_in_batch, + ) + + +def pad_input(hidden_states, indices, batch, seqlen): + """ + Arguments: + hidden_states: (total_nnz, ...), where total_nnz = number of tokens in selected in attention_mask. + indices: (total_nnz), the indices that represent the non-masked tokens of the original padded input sequence. + batch: int, batch size for the padded sequence. + seqlen: int, maximum sequence length for the padded sequence. + Return: + hidden_states: (batch, seqlen, ...) + """ + dim = hidden_states.shape[1:] + output = torch.zeros( + (batch * seqlen), *dim, device=hidden_states.device, dtype=hidden_states.dtype + ) + output[indices] = hidden_states + return rearrange(output, "(b s) ... -> b s ...", b=batch) + + +def generate_random_padding_mask( + max_seqlen, batch_size, device, mode="random", zero_lengths=False +): + assert mode in ["full", "random", "third"] + if mode == "full": + lengths = torch.full( + (batch_size, 1), max_seqlen, device=device, dtype=torch.int32 + ) + elif mode == "random": + lengths = torch.randint( + max(0 if zero_lengths else 1, max_seqlen - 20), + max_seqlen + 1, + (batch_size, 1), + device=device, + ) + elif mode == "third": + lengths = torch.randint( + max_seqlen // 3, max_seqlen + 1, (batch_size, 1), device=device + ) + + if zero_lengths: + # Generate zero-lengths every 5 batches and the last batch. + for i in range(batch_size): + if i % 5 == 0: + lengths[i] = 0 + lengths[-1] = 0 + padding_mask = ( + repeat(torch.arange(max_seqlen, device=device), "s -> b s", b=batch_size) + < lengths + ) + return padding_mask + + +def generate_qkv( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + qv=None, + kvpacked=False, + qkvpacked=False, + query_unused_mask=None, + key_unused_mask=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, d) + k: (batch_size, seqlen_k, nheads_k, d) + v: (batch_size, seqlen_k, nheads_k, d_v) + query_padding_mask: (batch_size, seqlen), bool + key_padding_mask: (batch_size, seqlen), bool + """ + assert not (kvpacked and qkvpacked) + batch_size, seqlen_q, nheads, d = q.shape + d_v = v.shape[-1] + _, seqlen_k, nheads_k, _ = k.shape + assert k.shape == (batch_size, seqlen_k, nheads_k, d) + assert v.shape == (batch_size, seqlen_k, nheads_k, d_v) + if query_unused_mask is not None or key_unused_mask is not None: + assert not kvpacked + assert not qkvpacked + + if query_padding_mask is not None: + q_unpad, indices_q, cu_seqlens_q, max_seqlen_q, seqused_q = unpad_input( + q, query_padding_mask, query_unused_mask + ) + output_pad_fn = lambda output_unpad: pad_input( + output_unpad, indices_q, batch_size, seqlen_q + ) + qv_unpad = ( + rearrange(qv, "b s ... -> (b s) ...")[indices_q] if qv is not None else None + ) + else: + q_unpad = rearrange(q, "b s h d -> (b s) h d") + cu_seqlens_q = torch.arange( + 0, + (batch_size + 1) * seqlen_q, + step=seqlen_q, + dtype=torch.int32, + device=q_unpad.device, + ) + seqused_q = None + max_seqlen_q = seqlen_q + output_pad_fn = lambda output_unpad: rearrange( + output_unpad, "(b s) h d -> b s h d", b=batch_size + ) + qv_unpad = rearrange(qv, "b s ... -> (b s) ...") if qv is not None else None + + if key_padding_mask is not None: + k_unpad, indices_k, cu_seqlens_k, max_seqlen_k, seqused_k = unpad_input( + k, key_padding_mask, key_unused_mask + ) + v_unpad, *rest = unpad_input(v, key_padding_mask, key_unused_mask) + else: + k_unpad = rearrange(k, "b s h d -> (b s) h d") + v_unpad = rearrange(v, "b s h d -> (b s) h d") + cu_seqlens_k = torch.arange( + 0, + (batch_size + 1) * seqlen_k, + step=seqlen_k, + dtype=torch.int32, + device=k_unpad.device, + ) + seqused_k = None + max_seqlen_k = seqlen_k + + if qkvpacked: + assert (query_padding_mask == key_padding_mask).all() + assert nheads == nheads_k + qkv_unpad = torch.stack([q_unpad, k_unpad, v_unpad], dim=1) + qkv = torch.stack([q, k, v], dim=2) + if query_padding_mask is not None: + dqkv_pad_fn = lambda dqkv_unpad: pad_input( + dqkv_unpad, indices_q, batch_size, seqlen_q + ) + else: + dqkv_pad_fn = lambda dqkv_unpad: rearrange( + dqkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + qkv_unpad.detach().requires_grad_(), + cu_seqlens_q, + max_seqlen_q, + qkv.detach().requires_grad_(), + output_pad_fn, + dqkv_pad_fn, + ) + elif kvpacked: + kv_unpad = torch.stack([k_unpad, v_unpad], dim=1) + kv = torch.stack([k, v], dim=2) + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dkv_pad_fn = lambda dkv_unpad: pad_input( + dkv_unpad, indices_k, batch_size, seqlen_k + ) + else: + dkv_pad_fn = lambda dkv_unpad: rearrange( + dkv_unpad, "(b s) t h d -> b s t h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + kv_unpad.detach().requires_grad_(), + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + kv.detach().requires_grad_(), + output_pad_fn, + dq_pad_fn, + dkv_pad_fn, + ) + else: + dq_pad_fn = output_pad_fn + if key_padding_mask is not None: + dk_pad_fn = lambda dk_unpad: pad_input( + dk_unpad, indices_k, batch_size, seqlen_k + ) + else: + dk_pad_fn = lambda dk_unpad: rearrange( + dk_unpad, "(b s) h d -> b s h d", b=batch_size + ) + return ( + q_unpad.detach().requires_grad_(), + k_unpad.detach().requires_grad_(), + v_unpad.detach().requires_grad_(), + qv_unpad.detach() if qv is not None else None, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q.detach().requires_grad_(), + k.detach().requires_grad_(), + v.detach().requires_grad_(), + qv.detach() if qv is not None else None, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) + + +def construct_local_mask( + seqlen_q, + seqlen_k, + window_size=(None, None), + sink_token_length=0, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + if window_size[0] is None: + return col_idx > row_idx + sk - sq + window_size[1] + else: + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + return torch.logical_or( + col_idx > torch.minimum(row_idx + sk - sq + window_size[1], sk), + torch.logical_and( + col_idx < row_idx + sk - sq - window_size[0], + col_idx >= sink_token_length, + ), + ) + + +def construct_chunk_mask( + seqlen_q, + seqlen_k, + attention_chunk, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + device=None, +): + row_idx = rearrange( + torch.arange(seqlen_q, device=device, dtype=torch.long), "s -> s 1" + ) + col_idx = torch.arange(seqlen_k, device=device, dtype=torch.long) + if key_leftpad is not None: + key_leftpad = rearrange(key_leftpad, "b -> b 1 1 1") + col_idx = repeat(col_idx, "s -> b 1 1 s", b=key_leftpad.shape[0]) + col_idx = torch.where(col_idx >= key_leftpad, col_idx - key_leftpad, 2**32) + sk = ( + seqlen_k + if key_padding_mask is None + else rearrange(key_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sq = ( + seqlen_q + if query_padding_mask is None + else rearrange(query_padding_mask.sum(-1), "b -> b 1 1 1") + ) + sk = torch.full_like(col_idx, seqlen_k) if key_padding_mask is None else sk + # Subtract remainder instead of divide and then multiply to take care of negative values + col_limit_left_chunk = row_idx + sk - sq - (row_idx + sk - sq) % attention_chunk + return torch.logical_or( + col_idx < col_limit_left_chunk, + col_idx >= col_limit_left_chunk + attention_chunk, + ) + + +def attention_ref( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + key_leftpad=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + qv=None, + q_descale=None, + k_descale=None, + v_descale=None, + window_size=(None, None), + attention_chunk=0, + sink_token_length=0, + learnable_sink=None, + softcap=0.0, + upcast=True, + reorder_ops=False, + intermediate_dtype=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads, head_dim) + v: (batch_size, seqlen_k, nheads, head_dim_v) + qv: (batch_size, seqlen_q, nheads, head_dim_v) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling k, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim_v) + attention: (batch_size, nheads, seqlen_q, seqlen_k), softmax after dropout + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + qv = qv.float() if qv is not None else None + if q_descale is not None: + q_descale = repeat(q_descale, "b h -> b 1 (h g) 1", g=q.shape[2] // k.shape[2]) + q = (q.float() * q_descale).to(q.dtype) + qv = (qv.float() * q_descale).to(qv.dtype) if qv is not None else None + if k_descale is not None: + k = (k.float() * rearrange(k_descale, "b h -> b 1 h 1")).to(dtype=k.dtype) + if v_descale is not None: + v = (v.float() * rearrange(v_descale, "b h -> b 1 h 1")).to(dtype=v.dtype) + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + dv = v.shape[-1] + softmax_scale = 1.0 / math.sqrt(d if qv is None else d + dv) + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q * softmax_scale, k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k * softmax_scale) + if qv is not None: + scores = scores + torch.einsum("bthd,bshd->bhts", qv * softmax_scale, v) + if softcap > 0: + scores = torch.tanh(scores / softcap) * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + local_mask = None + if window_size[0] is not None or window_size[1] is not None: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + sink_token_length, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + if attention_chunk > 0: + chunk_mask = construct_chunk_mask( + seqlen_q, + seqlen_k, + attention_chunk, + query_padding_mask, + key_padding_mask, + key_leftpad=key_leftpad, + device=q.device, + ) + local_mask = ( + torch.logical_or(local_mask, chunk_mask) + if local_mask is not None + else chunk_mask + ) + if local_mask is not None: + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + if learnable_sink is None: + attention = torch.softmax(scores, dim=-1).to(v.dtype) + else: + scores_fp32 = scores.to(torch.float32) + logits_max = torch.amax(scores_fp32, dim=-1, keepdim=True) + learnable_sink = rearrange(learnable_sink, "h -> h 1 1") + logits_or_sinks_max = torch.maximum(learnable_sink, logits_max) + unnormalized_scores = torch.exp(scores_fp32 - logits_or_sinks_max) + normalizer = unnormalized_scores.sum(dim=-1, keepdim=True) + torch.exp( + learnable_sink - logits_or_sinks_max + ) + attention = (unnormalized_scores / normalizer).to(v.dtype) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + # Without this we might get NaN in dv + if key_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), 0.0 + ) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if local_mask is not None: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + if intermediate_dtype is not None: + attention_drop = attention_drop.to(intermediate_dtype).to(attention_drop.dtype) + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + return output.to(dtype=dtype_og), attention.to(dtype=dtype_og) + + +@pytest.mark.skipif( + is_hopper(), + reason="skip on hopper", +) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float8_e4m3fn]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("mha_type", ["mha", "mqa", "gqa"]) +# @pytest.mark.parametrize("mha_type", ["mqa"]) +@pytest.mark.parametrize("has_learnable_sink", [False, True]) +# @pytest.mark.parametrize("has_learnable_sink", [False]) +# @pytest.mark.parametrize("has_qv", [False, True]) +@pytest.mark.parametrize("has_qv", [False]) +# @pytest.mark.parametrize("deterministic", [False, True]) +@pytest.mark.parametrize("deterministic", [False]) +# @pytest.mark.parametrize("softcap", [0.0, 15.0]) +@pytest.mark.parametrize("softcap", [0.0]) +@pytest.mark.parametrize("local", [False, True]) +# @pytest.mark.parametrize("local", [False]) +@pytest.mark.parametrize("causal", [False, True]) +# @pytest.mark.parametrize("causal", [False]) +# @pytest.mark.parametrize("add_unused_qkv", [False, True]) +@pytest.mark.parametrize("add_unused_qkv", [False]) +# @pytest.mark.parametrize("d", [32, 64, 96, 128, 160, 192, 224, 256]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128, 160, 192, 256]) +# @pytest.mark.parametrize('d', [32, 64, 96, 128, 160, 192]) +# @pytest.mark.parametrize('d', [56, 80]) +# @pytest.mark.parametrize('d', [32, 40, 64, 80, 96, 128]) +# @pytest.mark.parametrize("d", [64, 96, 128]) +@pytest.mark.parametrize("d", [128, 192]) +# @pytest.mark.parametrize("d", [192]) +@pytest.mark.parametrize( + "seqlen_q,seqlen_k", + [ + # (1, 1), + # (1, 3), + # (2, 1), + (511, 1), + (3, 513), + (64, 128), + (128, 128), + (256, 256), + (113, 203), + (128, 217), + (113, 211), + (108, 256), + (256, 512), + (307, 256), + (640, 128), + (512, 256), + (1024, 1024), + (1023, 1024), + (1024, 1023), + (2048, 2048), + ], +) +def test_flash_attn_varlen_output( + seqlen_q, + seqlen_k, + d, + add_unused_qkv, + causal, + local, + softcap, + deterministic, + has_qv, + has_learnable_sink, + mha_type, + dtype, +): + if ( + causal or local + ): # Right now we only support causal attention with seqlen_k == seqlen_q + seqlen_k = seqlen_q + device = "cuda" + # set seed + torch.random.manual_seed(seqlen_q + seqlen_k + d + int(causal) * 2 + int(local)) + batch_size = 49 if seqlen_q <= 1024 else 7 + nheads = 6 + # batch_size = 1 + # nheads = 1 + nheads_kv = nheads if mha_type == "mha" else (3 if mha_type == "gqa" else 1) + dtype_ref = torch.bfloat16 if dtype == torch.float8_e4m3fn else dtype + # dv_vals = [128, d] if d > 128 and d <= 192 else ([256, 512, d] if d <= 64 else [d]) + dv_vals = [128] if d == 192 else ([d] if d != 128 else [64, d]) + if dtype == torch.float8_e4m3fn: + dv_vals = [d] + # attention_chunk_vals = [torch.randint(1, seqlen_k * 2, (1,)).item(), 0] if seqlen_q <= seqlen_k else [0] + attention_chunk_vals = [0] + for dv, attention_chunk in itertools.product(dv_vals, attention_chunk_vals): + q_ref = torch.randn( + batch_size, seqlen_q, nheads, d, device=device, dtype=dtype_ref + ) + if softcap > 0.0: + # Ensure the values of qk are at least within softcap range. + q_ref = (q_ref * softcap / 4).detach().requires_grad_() + q_ref = q_ref.to(dtype).to(dtype_ref).requires_grad_() + k_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, d, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + v_ref = ( + torch.randn( + batch_size, seqlen_k, nheads_kv, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + .requires_grad_() + ) + if has_qv: + qv_ref = ( + torch.randn( + batch_size, seqlen_q, nheads, dv, device=device, dtype=dtype_ref + ) + .to(dtype) + .to(dtype_ref) + ) + else: + qv_ref = None + # Put window_size after QKV randn so that window_size changes from test to test + window_size = ( + (None, None) if not local else torch.randint(0, seqlen_k, (2,)).tolist() + ) + if has_learnable_sink: + learnable_sink = torch.randn(nheads, dtype=torch.bfloat16, device=device) + else: + learnable_sink = None + if dtype == torch.float8_e4m3fn: + q_descale, k_descale, v_descale = [ + torch.rand(batch_size, nheads_kv, device=device, dtype=torch.float32) + * 2 + for _ in range(3) + ] + else: + q_descale, k_descale, v_descale = None, None, None + q, k, v = [x.detach().requires_grad_() for x in (q_ref, k_ref, v_ref)] + qv = qv_ref.detach() if has_qv else None + query_padding_mask = generate_random_padding_mask( + seqlen_q, batch_size, device, mode="random", zero_lengths=False + ) + # TODO: test zero_lengths + key_padding_mask = generate_random_padding_mask( + # seqlen_k, batch_size, device, mode="random", zero_lengths=True + seqlen_k, + batch_size, + device, + mode="random", + zero_lengths=False, + ) + + def _gen_unused_masks(padding_mask, add_unused, max_seq_len, bs, device): + if add_unused: + another_mask = generate_random_padding_mask(max_seq_len, bs, device) + attn_mask = torch.logical_and(padding_mask, another_mask) + unused_mask = torch.logical_xor( + torch.logical_or(padding_mask, another_mask), attn_mask + ) + else: + attn_mask = padding_mask + unused_mask = None + return attn_mask, unused_mask + + query_padding_mask, query_unused_mask = _gen_unused_masks( + query_padding_mask, add_unused_qkv, seqlen_q, batch_size, q.device + ) + # query_padding_mask[:] = True + # query_unused_mask = None + key_padding_mask, key_unused_mask = _gen_unused_masks( + key_padding_mask, add_unused_qkv, seqlen_k, batch_size, k.device + ) + + if causal or local: + key_padding_mask = query_padding_mask + + ( + q_unpad, + k_unpad, + v_unpad, + qv_unpad, + cu_seqlens_q, + cu_seqlens_k, + seqused_q, + seqused_k, + max_seqlen_q, + max_seqlen_k, + q, + k, + v, + qv, + output_pad_fn, + dq_pad_fn, + dk_pad_fn, + ) = generate_qkv( + q, + k, + v, + query_padding_mask, + key_padding_mask, + qv=qv, + kvpacked=False, + query_unused_mask=query_unused_mask, + key_unused_mask=key_unused_mask, + ) + q_unpad, k_unpad, v_unpad = [ + x.detach().to(dtype).requires_grad_() for x in (q_unpad, k_unpad, v_unpad) + ] + out_ref, attn_ref = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + attention_chunk=attention_chunk, + learnable_sink=learnable_sink, + softcap=softcap, + ) + out_pt, attn_pt = attention_ref( + q_ref, + k_ref, + v_ref, + query_padding_mask, + key_padding_mask, + causal=causal, + qv=qv_ref, + q_descale=q_descale, + k_descale=k_descale, + v_descale=v_descale, + window_size=window_size, + attention_chunk=attention_chunk, + learnable_sink=learnable_sink, + softcap=softcap, + upcast=False, + reorder_ops=True, + intermediate_dtype=dtype if dtype == torch.float8_e4m3fn else None, + ) + + print(f"Pytorch max diff: {(out_pt - out_ref).abs().max().item()}") + print(f"Pytorch mean diff: {(out_pt - out_ref).abs().mean().item()}") + + if query_unused_mask is not None: + q_zero_masking = rearrange(query_unused_mask, "b s -> b s 1 1") + + # Numerical error if we just do any arithmetic on out_ref + fwd_atol = 2 * (out_ref + 0.3 - 0.3 - out_ref).abs().max().item() + rtol = 2 if softcap == 0.0 else 3 + + pack_gqa_vals = [False, True, None] + # num_splits_vals = [1, 3] + num_splits_vals = [1] + for pack_gqa, num_splits in itertools.product(pack_gqa_vals, num_splits_vals): + out_unpad, lse = flash_attn_varlen_func( + q_unpad, + k_unpad, + v_unpad, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=None, + max_seqlen_k=None, + # seqused_q=seqused_q, + # seqused_k=seqused_k, + causal=causal, + # qv=qv_unpad, + # q_descale=q_descale, + # k_descale=k_descale, v_descale=v_descale, + window_size=window_size, + # attention_chunk=attention_chunk, + sinks=learnable_sink, + softcap=softcap, + pack_gqa=pack_gqa, + return_softmax_lse=True, + ) + out = output_pad_fn(out_unpad) + if query_unused_mask is not None: + out.masked_fill_(q_zero_masking, 0.0) + print(f"Output max diff: {(out - out_ref).abs().max().item()}") + print(f"Output mean diff: {(out - out_ref).abs().mean().item()}") + # if not causal: + # print(f"LSE max diff: {(lse - lse_ref).abs().max().item()}") + # breakpoint() + + # Check that FlashAttention's numerical error is at most 3x the numerical error + # of a Pytorch implementation. + assert (out - out_ref).abs().max().item() <= rtol * ( + out_pt - out_ref + ).abs().max().item() + fwd_atol + + if ( + dtype != torch.float8_e4m3fn + and not has_qv + and not dv > 256 + and not attention_chunk != 0 + and dv == d + and not has_learnable_sink + and False + ): + g_unpad = torch.randn_like(out_unpad) + do_o = ((g_unpad.float() * out_unpad.float()).sum(-1)).transpose(-1, -2) + # import flash_attn_3_cuda + # dq_unpad, dk_unpad, dv_unpad, softmax_d, dq_accum, lse_log2 = flash_attn_3_cuda.bwd_varlen( + # g_unpad, + # q_unpad, + # k_unpad, + # v_unpad, + # out_unpad, + # lse, + # None, + # None, + # None, + # cu_seqlens_q, + # cu_seqlens_k, + # None, None, + # max_seqlen_q, + # max_seqlen_k, + # d ** (-0.5), + # causal, + # window_size[0], window_size[1], + # softcap, + # deterministic, + # 0, # sm_margin + # ) + dq_unpad, dk_unpad, dv_unpad = torch.autograd.grad( + out_unpad, (q_unpad, k_unpad, v_unpad), g_unpad + ) + dq = dq_pad_fn(dq_unpad) + dk = dk_pad_fn(dk_unpad) + dv = dk_pad_fn(dv_unpad) + if key_unused_mask is not None: + k_zero_masking = rearrange(key_unused_mask, "b s -> b s 1 1") + dk.masked_fill_(k_zero_masking, 0.0) + dv.masked_fill_(k_zero_masking, 0.0) + if query_unused_mask is not None: + dq.masked_fill_(q_zero_masking, 0.0) + # print(f"dO_O max diff: {(softmax_d - do_o).abs().max().item()}") + # assert (softmax_d - do_o).abs().max().item() <= 1e-5 + # assert dq_accum.abs().max().item() == 0.0 + g = output_pad_fn(g_unpad) + + # qk = torch.einsum('bthd,bshd->bhts', q / (d ** 0.5), k).float() + # qk = torch.masked_fill(qk, rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf")) + # dS = torch.einsum('bthd,bshd->bhts', g.float(), v.float()) + # P = torch.softmax(qk, -1) + # dP = P * (dS - (g.float() * out.float()).sum(-1).transpose(1, 2).unsqueeze(-1)) + # dQ = torch.einsum('bhts,bshd->bthd', dP, k.float()) + # dV = torch.einsum('bhts,bthd->bshd', P, g.float()) + # dK = torch.einsum('bhts,bthd->bshd', dP, q.float()) + + # dq, dk, dv = torch.autograd.grad(out, (q, k, v), g) + dq_ref, dk_ref, dv_ref = torch.autograd.grad( + out_ref, (q_ref, k_ref, v_ref), g + ) + dq_pt, dk_pt, dv_pt = torch.autograd.grad(out_pt, (q_ref, k_ref, v_ref), g) + print(f"dQ max diff: {(dq - dq_ref).abs().max().item()}") + print(f"dK max diff: {(dk - dk_ref).abs().max().item()}") + print(f"dV max diff: {(dv - dv_ref).abs().max().item()}") + print(f"dQ mean diff: {(dq - dq_ref).abs().mean().item()}") + print(f"dK mean diff: {(dk - dk_ref).abs().mean().item()}") + print(f"dV mean diff: {(dv - dv_ref).abs().mean().item()}") + print(f"dQ Pytorch max diff: {(dq_pt - dq_ref).abs().max().item()}") + print(f"dK Pytorch max diff: {(dk_pt - dk_ref).abs().max().item()}") + print(f"dV Pytorch max diff: {(dv_pt - dv_ref).abs().max().item()}") + print(f"dQ Pytorch mean diff: {(dq_pt - dq_ref).abs().mean().item()}") + print(f"dK Pytorch mean diff: {(dk_pt - dk_ref).abs().mean().item()}") + print(f"dV Pytorch mean diff: {(dv_pt - dv_ref).abs().mean().item()}") + # breakpoint() + dq_atol = 2 * (dq_ref + 0.3 - 0.3 - dq_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dq - dq_ref).abs().max().item() <= rtol * ( + dq_pt - dq_ref + ).abs().max().item() + dq_atol + dk_atol = 2 * (dk_ref + 0.3 - 0.3 - dk_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dk - dk_ref).abs().max().item() <= rtol * ( + dk_pt - dk_ref + ).abs().max().item() + dk_atol + dv_atol = 2 * (dv_ref + 0.3 - 0.3 - dv_ref).abs().max().item() + ( + 0 if softcap == 0 else 3e-4 + ) + assert (dv - dv_ref).abs().max().item() <= rtol * ( + dv_pt - dv_ref + ).abs().max().item() + dv_atol + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_fp4_gemm.py b/sgl-kernel/tests/test_fp4_gemm.py new file mode 100644 index 000000000..47401618b --- /dev/null +++ b/sgl-kernel/tests/test_fp4_gemm.py @@ -0,0 +1,154 @@ +import pytest +import torch +from sgl_kernel import cutlass_scaled_fp4_mm, scaled_fp4_quant + +skip_condition = torch.cuda.get_device_capability() < (10, 0) + +DTYPES = [torch.float16, torch.bfloat16] +# m, n, k +SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)] +PAD_SHAPES = [(150, 128, 64), (128, 128, 96)] +SHAPES.extend(PAD_SHAPES) + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +kE2M1ToFloatArray = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, +] + + +def e2m1_to_fp32(int4_value): + signBit = int4_value & 0x8 + int4_absValue = int4_value & 0x7 + float_result = kE2M1ToFloatArray[int4_absValue] + if signBit: + float_result = -float_result + return float_result + + +def break_fp4_bytes(a, dtype): + assert a.dtype == torch.uint8 + m, n = a.shape + a = a.flatten() + # Get upper 4 bits + highHalfByte = (a & 0xF0) >> 4 + # Get lower 4 bits + lowHalfByte = a & 0x0F + fH = torch.tensor([e2m1_to_fp32(x) for x in highHalfByte]).to(a.device) + fL = torch.tensor([e2m1_to_fp32(x) for x in lowHalfByte]).to(a.device) + # [0xAB, 0xCD] -> [0xB, 0xA, 0xD, 0xC] + out = torch.stack((fL, fH), dim=-1).reshape(m, n * 2) + return out + + +def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): + sf_m, sf_k = a_sf_swizzled.shape + m_tiles = (m + 128 - 1) // 128 + f = block_size * 4 + k_tiles = (k + f - 1) // f + tmp = torch.reshape(a_sf_swizzled, (1, m_tiles, k_tiles, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + out = tmp.reshape(m_tiles * 128, k_tiles * f // block_size) + return out[0:m, 0:k] + + +def dequantize_to_dtype( + tensor_fp4, tensor_sf, global_scale, dtype, device, block_size=16 +): + """Dequantize the fp4 tensor back to high precision.""" + # Two fp4 values are packed into one uint8. + assert tensor_fp4.dtype == torch.uint8 + m, packed_k = tensor_fp4.shape + k = packed_k * 2 + tensor_f32 = break_fp4_bytes(tensor_fp4, dtype) + tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_sf = tensor_sf.view(torch.float8_e4m3fn) + tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf_dtype = tensor_sf.to(torch.float32) / global_scale + + # scale the tensor + out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + return out + + +def get_ref_results( + a_fp4, + b_fp4, + a_sf, + b_sf, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + device, +): + _, m_k = a_fp4.shape + _, n_k = b_fp4.shape + assert m_k == n_k + a_in_dtype = dequantize_to_dtype( + a_fp4, a_sf, a_global_scale, dtype=dtype, device=device, block_size=block_size + ) + b_in_dtype = dequantize_to_dtype( + b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size + ) + return torch.matmul(a_in_dtype, b_in_dtype.t()) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_nvfp4_gemm( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + m, n, packed_k = shape + k = packed_k * 2 + block_size = 16 + a_dtype = torch.randn((m, k), dtype=dtype, device="cuda") + b_dtype = torch.randn((n, k), dtype=dtype, device="cuda") + + a_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1) + ).to(torch.float32) + b_global_scale = ( + (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1) + ).to(torch.float32) + alpha = 1.0 / (a_global_scale * b_global_scale) + a_fp4, a_scale_interleaved = scaled_fp4_quant(a_dtype, a_global_scale) + b_fp4, b_scale_interleaved = scaled_fp4_quant(b_dtype, b_global_scale) + + expected_out = get_ref_results( + a_fp4, + b_fp4, + a_scale_interleaved, + b_scale_interleaved, + a_global_scale, + b_global_scale, + m, + n, + dtype, + block_size, + "cuda", + ) + out = cutlass_scaled_fp4_mm( + a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype + ) + + torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_fp4_quantize.py b/sgl-kernel/tests/test_fp4_quantize.py new file mode 100644 index 000000000..3e83e47ac --- /dev/null +++ b/sgl-kernel/tests/test_fp4_quantize.py @@ -0,0 +1,261 @@ +import pytest +import torch +from sgl_kernel import ( + scaled_fp4_grouped_quant, + scaled_fp4_quant, + silu_and_mul, + silu_and_mul_scaled_fp4_grouped_quant, +) + +skip_condition = torch.cuda.get_device_capability() < (10, 0) + +DTYPES = [torch.float16, torch.bfloat16] +SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)] +PAD_SHAPES = [ + (90, 64), + (150, 64), + (128, 48), + (128, 80), + (150, 80), + (90, 48), + (90, 128), + (150, 128), + (150, 48), + (90, 80), +] + +FLOAT4_E2M1_MAX = 6.0 +FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max + +# E2M1 to float +# 0111 -> 6 +# 0110 -> 4 +# 0101 -> 3 +# 0100 -> 2 +# 0011 -> 1.5 +# 0010 -> 1 +# 0001 -> 0.5 +# 0000 -> 0 +E2M1_TO_FLOAT32 = [ + 0.0, + 0.5, + 1.0, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + 0.0, + -0.5, + -1.0, + -1.5, + -2.0, + -3.0, + -4.0, + -6.0, +] +BLOCK_SIZE = 16 + + +def cast_from_fp4(x, m, n): + # The fp4 values are packed in uint8 as [v_1st | v_2nd] + v_2nd = x & 0xF + v_1st = (x >> 4) & 0xF + c = torch.stack((v_2nd, v_1st), dim=-1) + out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()]) + out = out.reshape(m, n).to(torch.float32) + return out + + +def cast_to_fp4(x): + sign = torch.sign(x) + x = torch.abs(x) + x[(x >= 0.0) & (x <= 0.25)] = 0.0 + x[(x > 0.25) & (x < 0.75)] = 0.5 + x[(x >= 0.75) & (x <= 1.25)] = 1.0 + x[(x > 1.25) & (x < 1.75)] = 1.5 + x[(x >= 1.75) & (x <= 2.5)] = 2.0 + x[(x > 2.5) & (x < 3.5)] = 3.0 + x[(x >= 3.5) & (x <= 5.0)] = 4.0 + x[x > 5.0] = 6.0 + return x * sign + + +def get_reciprocal(x): + if isinstance(x, torch.Tensor): + return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x) + elif isinstance(x, (float, int)): + return 0.0 if x == 0 else 1.0 / x + else: + raise TypeError("Input must be a float, int, or a torch.Tensor.") + + +def ref_nvfp4_quant(x, global_scale): + assert global_scale.dtype == torch.float32 + assert x.ndim == 2 + m, n = x.shape + x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE)) + vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) + scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX)) + scale = scale.to(torch.float8_e4m3fn).to(torch.float32) + output_scale = get_reciprocal(scale * get_reciprocal(global_scale)) + + scaled_x = x.to(torch.float32) * output_scale + clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n) + return cast_to_fp4(clipped_x), scale.squeeze(-1) + + +def recover_swizzled_scales(scale, m, n): + rounded_m = ((m + 128 - 1) // 128) * 128 + scale_n = n // BLOCK_SIZE + rounded_n = ((scale_n + 4 - 1) // 4) * 4 + # Recover the swizzled scaling factor to linear layout + tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4)) + tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5)) + result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32) + return result[:m, :scale_n] + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("dtype", DTYPES) +@pytest.mark.parametrize("shape", SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4( + dtype: torch.dtype, + shape: tuple[int, int], +) -> None: + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + m, n = shape + + x = torch.randn((m, n), dtype=dtype) + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("pad_shape", PAD_SHAPES) +@torch.inference_mode() +def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None: + torch.manual_seed(42) + dtype = torch.float16 + torch.set_default_device("cuda:0") + + m, n = pad_shape + + x = torch.randn((m, n), dtype=dtype) + + tensor_amax = torch.abs(x).max().to(torch.float32) + global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + out_ref, scale_ref = ref_nvfp4_quant(x, global_scale) + + out, out_scale = scaled_fp4_quant(x, global_scale) + + scale_ans = recover_swizzled_scales(out_scale, m, n) + out_ans = cast_from_fp4(out, m, n) + + torch.testing.assert_close(out_ans, out_ref) + torch.testing.assert_close(scale_ans, scale_ref) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("shape", [(2, 512, 2048), (2, 100, 128), (2, 128, 96)]) +def test_quantize_to_fp4_grouped(shape): + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + l, m, k = shape + x = torch.randn((l, m, k), dtype=torch.bfloat16) + max_m = m // 2 + assert max_m <= m + mask = torch.randint(1, max_m, (l,), dtype=torch.int32) + tensor_amax = x.abs().amax(dim=(1, 2)).to(torch.float32) + x_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + output, output_scales = scaled_fp4_grouped_quant( + x, + x_sf_global, + mask, + ) + # output in logical (m, k, l), but its physical layout is (l, m, k). + # So permute first to (l, m, k). + output = output.permute(2, 0, 1) + # output_scale in logical (32, 4, rm, 4, rk, l), but its physical layout is (l, rm, rk, 32, 4, 4). + # So permute first to (l, rm, rk, 32, 4, 4). + padded_m = ((m + 128 - 1) // 128) * 128 + output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1) + for i in range(l): + a_fp4, a_scale_interleaved = scaled_fp4_quant(x[i], x_sf_global[i]) + torch.testing.assert_close(a_fp4[: mask[i]], output[i][: mask[i]]) + # Recover swizzled scales to linear layout and drop padded values, so + # no extra checks on padding are needed. + scale_ref = recover_swizzled_scales(a_scale_interleaved, m, k) + scale_ans = recover_swizzled_scales(output_scales[i], m, k) + torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]]) + + +@pytest.mark.skipif( + skip_condition, reason="Nvfp4 Requires compute capability of 10 or above." +) +@pytest.mark.parametrize("shape", [(32, 100, 2048), (32, 512, 2048), (6, 6144, 2048)]) +def test_silu_and_mul_quantize_to_fp4_grouped(shape): + torch.manual_seed(42) + torch.set_default_device("cuda:0") + + l, m, k = shape + x = torch.randn((l, m, k * 2), dtype=torch.bfloat16) + max_m = m // 2 + assert max_m <= m + mask = torch.randint(1, max_m, (l,), dtype=torch.int32) + + ref_y = silu_and_mul(x) + tensor_amax = ref_y.abs().amax(dim=(1, 2)).to(torch.float32) + y_sf_global = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax + ref_output, ref_output_scales = scaled_fp4_grouped_quant( + ref_y, + y_sf_global, + mask, + ) + output, output_scales = silu_and_mul_scaled_fp4_grouped_quant( + x, + y_sf_global, + mask, + ) + + # output in logical (m, k, l), but its physical layout is (l, m, k). + # So permute first to (l, m, k). + output = output.permute(2, 0, 1) + ref_output = ref_output.permute(2, 0, 1) + + # output_scale in logical (32, 4, rm, 4, rk, l), but its physical layout is (l, rm, rk, 32, 4, 4). + # So permute first to (l, rm, rk, 32, 4, 4). + padded_m = ((m + 128 - 1) // 128) * 128 + output_scales = output_scales.permute(5, 2, 4, 0, 1, 3).view(l, padded_m, -1) + ref_output_scales = ref_output_scales.permute(5, 2, 4, 0, 1, 3).view( + l, padded_m, -1 + ) + + for i in range(l): + torch.testing.assert_close(ref_output[i, : mask[i]], output[i, : mask[i]]) + # We need to recover the swizzled scales to linear layout before applying mask slice. + scale_ref = recover_swizzled_scales(ref_output_scales[i], m, k) + scale_ans = recover_swizzled_scales(output_scales[i], m, k) + torch.testing.assert_close(scale_ref[: mask[i]], scale_ans[: mask[i]]) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_fp8_blockwise_gemm.py b/sgl-kernel/tests/test_fp8_blockwise_gemm.py new file mode 100644 index 000000000..4c1dde336 --- /dev/null +++ b/sgl-kernel/tests/test_fp8_blockwise_gemm.py @@ -0,0 +1,93 @@ +import os +import random +from typing import Optional, Type + +import pytest +import torch +from sgl_kernel import fp8_blockwise_scaled_mm + + +def cdiv(a: int, b: int) -> int: + return -(a // -b) + + +def scale_shape(shape, group_shape): + assert len(shape) == len(group_shape) + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: Type[torch.dtype], + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + # We treat N-dimensional group scaling as extended numpy-style broadcasting + # in numpy simply stretches dimensions with an extent of 1 to match the + # the target shape by repeating the data along that dimension (broadcasting) + # , we extend these semantics to say if the extent of a dimension in the + # source shape is not 1 and does not match the target shape we repeat each + # element along that dimension src_shape[dim] // target_shape[dim] times + # example if we have: + # a = [[1, 2], and target_shape = (2, 4) + # [3, 4]] + # then we would expand a to: + # a = [[1, 1, 2, 2], + # [3, 3, 4, 4]] + # NOTE this function this function does not explicitly broadcast dimensions + # with an extent of 1, since this can be done implicitly by pytorch + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + output = torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + if bias is not None: + output = output + bias + return output + + +def _test_accuracy_once(M, N, K, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn).t() + scale_a_group_shape = (1, 128) + scale_b_group_shape = (128, 128) + scale_a_shape = scale_shape(a_fp8.shape, scale_a_group_shape) + scale_b_shape = scale_shape(b_fp8.shape, scale_b_group_shape) + scale_a = torch.randn(scale_a_shape, device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn(scale_b_shape, device=device, dtype=torch.float32) * 0.001 + scale_a = scale_a.t().contiguous().t() + scale_b = scale_b.t().contiguous().t() + o = baseline_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + o1 = fp8_blockwise_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype) + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("M", [1, 3, 5, 127, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 14080]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 14080, 16384]) +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) +def test_accuracy(M, N, K, out_dtype): + _test_accuracy_once(M, N, K, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_fp8_blockwise_moe.py b/sgl-kernel/tests/test_fp8_blockwise_moe.py new file mode 100755 index 000000000..6f2279393 --- /dev/null +++ b/sgl-kernel/tests/test_fp8_blockwise_moe.py @@ -0,0 +1,221 @@ +import random +from typing import Tuple + +import pytest +import torch +from sgl_kernel import fp8_blockwise_scaled_grouped_mm + + +def cdiv(a: int, b: int) -> int: + return -(a // -b) + + +def scale_shape(shape, group_shape): + return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape))) + + +def to_fp8(tensor: torch.Tensor) -> torch.Tensor: + finfo = torch.finfo(torch.float8_e4m3fn) + return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to( + dtype=torch.float8_e4m3fn + ) + + +# Copy from: https://github.com/deepseek-ai/DeepGEMM/blob/main/deep_gemm/utils.py +def calc_diff(x, y): + x, y = x.double(), y.double() + denominator = (x * x + y * y).sum() + sim = 2 * (x * y).sum() / denominator + return 1 - sim + + +def ceil_div(x: int, y: int) -> int: + return (x + y - 1) // y + + +def per_token_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad(x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), dtype=x.dtype, device=x.device + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), x_view.size(2) + ) + + +def baseline_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: type[torch.dtype], +) -> torch.Tensor: + + def group_broadcast(t, shape): + for i, s in enumerate(shape): + if t.shape[i] != s and t.shape[i] != 1: + assert s % t.shape[i] == 0 + t = ( + t.unsqueeze(i + 1) + .expand(*t.shape[: i + 1], s // t.shape[i], *t.shape[i + 1 :]) + .flatten(i, i + 1) + ) + return t + + scale_a = group_broadcast(scale_a, a.shape) + scale_b = group_broadcast(scale_b, b.shape) + + return torch.mm( + (scale_a * a.to(dtype=torch.float32)), (scale_b * b.to(dtype=torch.float32)) + ).to(out_dtype) + + +def is_sm100_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 10) and ( + torch.version.cuda >= "12.8" + ) + + +def is_sm90_supported(device=None) -> bool: + return (torch.cuda.get_device_capability(device)[0] == 9) and ( + torch.version.cuda >= "12.3" + ) + + +@pytest.mark.skipif( + not (is_sm100_supported() or is_sm90_supported()), + reason="fp8_blockwise_scaled_grouped_mm at sgl-kernel is only supported on sm100 or sm90", +) +@pytest.mark.parametrize("num_experts", [8, 16, 32, 64, 128]) +@pytest.mark.parametrize("out_dtype", [torch.half, torch.bfloat16]) +def test_fp8_blockwise_scaled_grouped_mm(num_experts, out_dtype): + device = "cuda" + alignment = 128 + n_g = random.randint(1, 64) * 128 + k_g = random.randint(1, 64) * 128 + + expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int32) + problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32) + layout_sfa = torch.zeros((num_experts, 5), device=device, dtype=torch.int32) + layout_sfb = torch.zeros((num_experts, 5), device=device, dtype=torch.int32) + + a_tensors = [] + b_tensors = [] + a_scales_tensors = [] + b_scales_tensors = [] + baseline_tensors = [] + + for g in range(num_experts): + m_g = random.randint(1, 256) + expert_offsets[g + 1] = expert_offsets[g] + m_g + problem_sizes[g][:] = torch.tensor([m_g, n_g, k_g], device=device) + + a = torch.randn((m_g, k_g), device=device, dtype=out_dtype) # (M, K):(K, 1) + b = torch.randn((n_g, k_g), device=device, dtype=out_dtype).t() # (K, N):(1, K) + + a_g, a_scale = per_token_cast_to_fp8( + a + ) # ag -- (M, K):(K, 1), a_scale() -- (M, k):(k, 1) + b_g, b_scale = per_block_cast_to_fp8( + b + ) # bg -- (K, N):(N, 1), b_scale() -- (k, n):(n, 1) + a_tensors.append(a_g) + b_tensors.append(b_g) + a_scales_tensors.append(a_scale) + b_scales_tensors.append(b_scale) + + baseline = torch.mm(a, b) + baseline_tensors.append(baseline) + a_stack = torch.empty( + (expert_offsets[-1], k_g), device=device, dtype=torch.float8_e4m3fn + ) + b_stack = torch.empty( + (num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn + ) + a_scale_stack = torch.empty( + (expert_offsets[-1], (k_g // 128)), device=device, dtype=torch.float32 + ) + b_scale_stack = torch.empty( + (num_experts, n_g // 128, k_g // 128), device=device, dtype=torch.float32 + ) + + for g in range(num_experts): + # Matrix A is Row-Major. + a_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_tensors[ + g + ] # a_stack[expert_offsets[g] : expert_offsets[g + 1], :] -- (M, K):(K, 1) + b_stack[g] = b_tensors[g].t() # b_stack[g] -- (N, K):(K, 1) + + # We need K-Major scale factor + a_scale_stack[expert_offsets[g] : expert_offsets[g + 1], :] = a_scales_tensors[ + g + ] + b_scale_stack[g] = b_scales_tensors[ + g + ].t() # b_scale_stack[g] -- (k, n):(n, 1), we need transpose & contiguous later + b_stack = b_stack.transpose(1, 2) # Transpose Matrix B to Column-Major. + b_scale_stack = b_scale_stack.transpose(1, 2) + + c_out = torch.empty((expert_offsets[-1], n_g), device=device, dtype=out_dtype) + a_strides = torch.full( + (num_experts,), a_stack.stride(0), device=device, dtype=torch.int64 + ) + c_strides = torch.full( + (num_experts,), c_out.stride(0), device=device, dtype=torch.int64 + ) + workspace = torch.empty((1024 * 1024 * 1024), device=device, dtype=torch.uint8) + a_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + b_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + out_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + a_scales_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + b_scales_ptrs = torch.empty((num_experts,), device=device, dtype=torch.int64) + + fp8_blockwise_scaled_grouped_mm( + c_out, + a_ptrs, + b_ptrs, + out_ptrs, + a_scales_ptrs, + b_scales_ptrs, + a_stack, + b_stack, + a_scale_stack, + b_scale_stack, + a_strides, + a_strides, + c_strides, + layout_sfa, + layout_sfb, + problem_sizes, + expert_offsets[:-1], + workspace, + ) + + for g in range(num_experts): + baseline = baseline_tensors[g] + actual = c_out[expert_offsets[g] : expert_offsets[g + 1]] + diff = calc_diff(actual, baseline) + assert diff < 0.001 + print( + f"m_g={baseline.shape[0]} n_g={n_g} k_g={k_g} num_experts={num_experts}, out_dtype={out_dtype}, diff={diff:.5f}: OK" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_fp8_gemm.py b/sgl-kernel/tests/test_fp8_gemm.py new file mode 100644 index 000000000..e70e62af2 --- /dev/null +++ b/sgl-kernel/tests/test_fp8_gemm.py @@ -0,0 +1,49 @@ +import pytest +import torch +from sgl_kernel import fp8_scaled_mm + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + o = o.to(torch.float32) + temp1 = o * scale_a.view(-1, 1) + temp2 = temp1 * scale_b.view(1, -1) + final = temp2.to(out_dtype) + if bias is not None: + final = final + bias.view(1, -1) + return final + + +def _test_accuracy_once(M, N, K, with_bias, out_dtype, device): + fp8_info = torch.finfo(torch.float8_e4m3fn) + fp8_max, fp8_min = fp8_info.max, fp8_info.min + a_fp32 = (torch.rand(M, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + a_fp8 = a_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + b_fp32 = (torch.rand(N, K, dtype=torch.float32, device=device) - 0.5) * 2 * fp8_max + b_fp8 = b_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + scale_a = torch.randn((M,), device=device, dtype=torch.float32) * 0.001 + scale_b = torch.randn((N,), device=device, dtype=torch.float32) * 0.001 + if with_bias: + bias = torch.randn((N,), device=device, dtype=out_dtype) + else: + bias = None + b_fp8 = b_fp8.t() + o = torch_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + o1 = fp8_scaled_mm(a_fp8, b_fp8, scale_a, scale_b, out_dtype, bias) + rtol = 0.02 + atol = 1 + torch.testing.assert_close(o, o1, rtol=rtol, atol=atol) + print(f"M={M}, N={N}, K={K}, with_bias={with_bias}, out_dtype={out_dtype}: OK") + + +@pytest.mark.parametrize("M", [1, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("N", [16, 128, 512, 1024, 4096]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("with_bias", [True, False]) +@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16]) +def test_accuracy(M, N, K, with_bias, out_dtype): + _test_accuracy_once(M, N, K, with_bias, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_gptq_kernel.py b/sgl-kernel/tests/test_gptq_kernel.py new file mode 100644 index 000000000..0ef4fad2c --- /dev/null +++ b/sgl-kernel/tests/test_gptq_kernel.py @@ -0,0 +1,131 @@ +import pytest +import torch +from sgl_kernel import gptq_gemm + +from sglang.srt.layers.quantization.utils import pack_cols, pack_rows + + +def torch_dequantize(q_weight, q_zeros, scales, g_idx, use_shuffle, bit, K, N): + assert bit == 4, "Reference dequantization only supports 4-bit" + group_size = K // scales.shape[0] + pack_factor = 32 // bit + + # unpack q_weight: (K//pack_factor, N) -> (K, N) + unpacked_q_weight = torch.empty( + q_weight.shape[0] * pack_factor, + q_weight.shape[1], + dtype=torch.uint8, + device=q_weight.device, + ) + for i in range(pack_factor): + unpacked_q_weight[i::pack_factor, :] = (q_weight >> (i * 4)) & 0x0F + + # unpack q_zeros: (num_groups, N//pack_factor) -> (num_groups, N) + unpacked_q_zeros = torch.empty( + q_zeros.shape[0], + q_zeros.shape[1] * pack_factor, + dtype=torch.uint8, + device=q_zeros.device, + ) + for i in range(pack_factor): + unpacked_q_zeros[:, i::pack_factor] = (q_zeros >> (i * 4)) & 0x0F + + unpacked_q_zeros += 1 + unpacked_q_zeros = unpacked_q_zeros.to(scales.dtype) + + scale_zeros = unpacked_q_zeros * scales # (num_groups, N) + + current_g_idx = torch.tensor( + [i // group_size for i in range(K)], dtype=torch.int32, device=q_weight.device + ) + + scale_mat = scales[current_g_idx] # (K, N) + scale_zeros_mat = scale_zeros[current_g_idx] # (K, N) + + # dequant: weight * scale - scale_zeros + dequantized_b = unpacked_q_weight.to(scales.dtype) * scale_mat - scale_zeros_mat + + return dequantized_b.reshape(K, N) + + +def torch_gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit +): + K, N = a.shape[1], b_q_weight.shape[1] + + b_dequant = torch_dequantize( + b_q_weight, b_gptq_qzeros, b_gptq_scales, b_g_idx, use_shuffle, bit, K, N + ) + c = torch.matmul(a, b_dequant) + return c + + +def _test_gptq_gemm_once(M, N, K, bit, group_size, use_shuffle, dtype, device="cuda"): + + b_fp = torch.randn(K, N, dtype=dtype, device=device) + + assert K % group_size == 0, "K must be divisible by group_size" + num_groups = K // group_size + + if use_shuffle: + return + else: + g_idx = torch.tensor( + [i // group_size for i in range(K)], dtype=torch.int32, device=device + ) + b_shuffled = b_fp[g_idx] + + b_grouped = b_shuffled.reshape(num_groups, group_size, N) + + b_max = torch.max(b_grouped, dim=1, keepdim=True)[0] + b_min = torch.min(b_grouped, dim=1, keepdim=True)[0] + + scales = (b_max - b_min) / (2**bit - 1) + scales = scales.clamp(min=1e-6) + + zeros_float = (-b_min / scales).round() + + q_b = ( + (b_grouped / scales + zeros_float).round().clamp(0, 2**bit - 1).to(torch.uint8) + ) + + q_zeros_unpacked = zeros_float.to(torch.uint8) - 1 + + b_q_weight = pack_rows(q_b.reshape(K, N), bit, K, N) + + q_zeros_unpacked = q_zeros_unpacked.reshape(num_groups, N) + b_gptq_qzeros = pack_cols(q_zeros_unpacked, bit, num_groups, N) + b_gptq_scales = scales.squeeze(1) + + a = torch.randn(M, K, dtype=dtype, device=device) + + c_ref = torch_gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, g_idx, use_shuffle, bit + ) + c_out = gptq_gemm( + a, b_q_weight, b_gptq_qzeros, b_gptq_scales, g_idx, use_shuffle, bit + ) + + rtol = 4e-2 + atol = 4e-2 + torch.testing.assert_close(c_ref, c_out, rtol=rtol, atol=atol) + print( + f"✅ Test passed: M={M}, N={N}, K={K}, bit={bit}, group_size={group_size}, use_shuffle={use_shuffle}, dtype={dtype}" + ) + + +@pytest.mark.parametrize("M", [1, 8, 128]) +@pytest.mark.parametrize("N", [2048, 4096]) +@pytest.mark.parametrize("K", [2048, 4096]) +@pytest.mark.parametrize("bit", [4]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("use_shuffle", [False]) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_gptq_gemm(M, N, K, bit, group_size, use_shuffle, dtype): + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + _test_gptq_gemm_once(M, N, K, bit, group_size, use_shuffle, dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/sgl-kernel/tests/test_int8_gemm.py b/sgl-kernel/tests/test_int8_gemm.py new file mode 100644 index 000000000..80f32cd02 --- /dev/null +++ b/sgl-kernel/tests/test_int8_gemm.py @@ -0,0 +1,48 @@ +import pytest +import torch +from sgl_kernel import int8_scaled_mm +from utils import is_sm10x + + +def to_int8(tensor: torch.Tensor) -> torch.Tensor: + return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) + + +def torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias): + o = torch.matmul(a.to(torch.float32), b.to(torch.float32)) + if bias is not None: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + bias + else: + o = o.to(torch.float32) * scale_a.view(-1, 1) * scale_b.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, with_bias, out_dtype, device): + a = to_int8(torch.randn((M, K), device=device) * 5) + b = to_int8(torch.randn((N, K), device=device).t() * 5) + scale_a = torch.randn((M,), device="cuda", dtype=torch.float32) + scale_b = torch.randn((N,), device="cuda", dtype=torch.float32) + if with_bias: + bias = torch.randn((N,), device="cuda", dtype=out_dtype) * 10 + else: + bias = None + o = int8_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + o1 = torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias) + torch.testing.assert_close(o, o1) + + +@pytest.mark.skipif( + is_sm10x(), + reason="int8_scaled_mm is only supported on sm90 and lower", +) +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [16, 128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("with_bias", [True, False]) +@pytest.mark.parametrize("out_dtype", [torch.float16, torch.bfloat16]) +def test_accuracy(M, N, K, with_bias, out_dtype): + _test_accuracy_once(M, N, K, with_bias, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_kvcacheio.py b/sgl-kernel/tests/test_kvcacheio.py new file mode 100644 index 000000000..07fcc2413 --- /dev/null +++ b/sgl-kernel/tests/test_kvcacheio.py @@ -0,0 +1,485 @@ +import pytest +import torch +from sgl_kernel.kvcacheio import ( + transfer_kv_all_layer, + transfer_kv_all_layer_direct_lf_pf, + transfer_kv_all_layer_mla, + transfer_kv_direct, + transfer_kv_per_layer, + transfer_kv_per_layer_direct_pf_lf, + transfer_kv_per_layer_mla, +) + + +def ref_copy_with_indices(src_pool, dst_pool, src_indices, dst_indices): + dst_pool[dst_indices] = src_pool[src_indices].to(dst_pool.device) + + +def ref_copy_with_indices_pf_direct( + src_pool, dst_pool, src_indices, dst_indices, page_size, layer_id, lf_to_pf=False +): + if lf_to_pf: + for i in range(0, len(src_indices), page_size): + dst_pool[dst_indices[i] // page_size][layer_id] = src_pool[layer_id][ + src_indices[i : i + page_size] + ].to(dst_pool.device) + else: + for i in range(0, len(src_indices), page_size): + dst_pool[layer_id][dst_indices[i : i + page_size]] = src_pool[ + src_indices[i] // page_size + ][layer_id].to(dst_pool.device) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("num_items_to_transfer", [1, 128, 1024]) +@pytest.mark.parametrize("page_size", [1, 16, 64]) +@pytest.mark.parametrize("item_size", [256]) +@pytest.mark.parametrize("total_items_in_pool", [10240]) +@pytest.mark.parametrize("is_mla", [False, True]) +@pytest.mark.parametrize("all_layers", [False, True]) +def test_transfer_kv( + dtype: torch.dtype, + num_items_to_transfer: int, + item_size: int, + page_size: int, + total_items_in_pool: int, + is_mla: bool, + all_layers: bool, +): + """ + Tests the per-layer transfer functions, treating tensors as memory pools. + """ + + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + device = "cuda" + torch.cuda.manual_seed(42) + + num_layers = 4 # A small number of layers for pool creation + + total_pages_in_pool = total_items_in_pool // page_size + num_pages_to_transfer = num_items_to_transfer // page_size + if num_pages_to_transfer == 0: + torch.set_default_dtype(original_dtype) + return + page_indices = torch.randperm(total_pages_in_pool, dtype=torch.int64) + src_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[:num_pages_to_transfer] + ] + ) + src_indices_device = src_indices_host.to(device) + dst_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[num_pages_to_transfer : 2 * num_pages_to_transfer] + ] + ) + dst_indices_device = dst_indices_host.to(device) + + # Prepare memory pools based on whether it's an MLA case. + if is_mla: + src_pool_host = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + dst_pool_ref = torch.zeros_like(src_pool_host).to(device) + dst_pool_kernel = torch.zeros_like(dst_pool_ref) + dst_pool_direct = torch.zeros_like(dst_pool_ref) + else: + src_k_pool = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + src_v_pool = torch.randn( + num_layers, total_items_in_pool, item_size + ).pin_memory() + dst_k_pool_ref = torch.zeros_like(src_k_pool).to(device) + dst_v_pool_ref = torch.zeros_like(src_v_pool).to(device) + dst_k_pool_kernel = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_kernel = torch.zeros_like(dst_v_pool_ref) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + + torch.cuda.synchronize() + + # We will test the per-layer function on the first layer (index 0) of the pool. + layer_idx_to_test = 0 + + if is_mla: + if not all_layers: + ref_copy_with_indices( + src_pool_host[layer_idx_to_test], + dst_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + transfer_kv_per_layer_mla( + src_pool_host[layer_idx_to_test], + dst_pool_kernel[layer_idx_to_test], + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + ) + transfer_kv_direct( + [src_pool_host[layer_idx_to_test]], + [dst_pool_direct[layer_idx_to_test]], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + else: + for layer_id in range(num_layers): + ref_copy_with_indices( + src_pool_host[layer_id], + dst_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + src_layers_device = torch.tensor( + [src_pool_host[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + dst_layers_device = torch.tensor( + [ + dst_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + transfer_kv_all_layer_mla( + src_layers_device, + dst_layers_device, + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + num_layers=num_layers, + ) + transfer_kv_direct( + [src_pool_host[layer_id] for layer_id in range(num_layers)], + [dst_pool_direct[layer_id] for layer_id in range(num_layers)], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_kernel, dst_pool_ref) + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + else: + if not all_layers: + ref_copy_with_indices( + src_k_pool[layer_idx_to_test], + dst_k_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + ref_copy_with_indices( + src_v_pool[layer_idx_to_test], + dst_v_pool_ref[layer_idx_to_test], + src_indices_host, + dst_indices_device, + ) + transfer_kv_per_layer( + src_k_pool[layer_idx_to_test], + dst_k_pool_kernel[layer_idx_to_test], + src_v_pool[layer_idx_to_test], + dst_v_pool_kernel[layer_idx_to_test], + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + ) + transfer_kv_direct( + [src_k_pool[layer_idx_to_test], src_v_pool[layer_idx_to_test]], + [ + dst_k_pool_direct[layer_idx_to_test], + dst_v_pool_direct[layer_idx_to_test], + ], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + else: + for layer_id in range(num_layers): + ref_copy_with_indices( + src_k_pool[layer_id], + dst_k_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + ref_copy_with_indices( + src_v_pool[layer_id], + dst_v_pool_ref[layer_id], + src_indices_host, + dst_indices_device, + ) + + src_k_layers_device = torch.tensor( + [src_k_pool[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + src_v_layers_device = torch.tensor( + [src_v_pool[layer_id].data_ptr() for layer_id in range(num_layers)], + dtype=torch.uint64, + device=device, + ) + dst_k_layers_device = torch.tensor( + [ + dst_k_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + dst_v_layers_device = torch.tensor( + [ + dst_v_pool_kernel[layer_id].data_ptr() + for layer_id in range(num_layers) + ], + dtype=torch.uint64, + device=device, + ) + transfer_kv_all_layer( + src_k_layers_device, + dst_k_layers_device, + src_v_layers_device, + dst_v_layers_device, + src_indices_device, + dst_indices_device, + item_size=item_size * dtype.itemsize, + num_layers=num_layers, + ) + transfer_kv_direct( + [src_k_pool[layer_id] for layer_id in range(num_layers)] + + [src_v_pool[layer_id] for layer_id in range(num_layers)], + [dst_k_pool_direct[layer_id] for layer_id in range(num_layers)] + + [dst_v_pool_direct[layer_id] for layer_id in range(num_layers)], + src_indices_host, + dst_indices_device, + page_size=page_size, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_kernel, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_kernel, dst_v_pool_ref) + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + + torch.set_default_dtype(original_dtype) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +@pytest.mark.parametrize("num_items_to_transfer", [128, 1024, 8192]) +@pytest.mark.parametrize("page_size", [16, 64, 128]) +@pytest.mark.parametrize("item_size", [256]) +@pytest.mark.parametrize("total_items_in_pool", [20480]) +@pytest.mark.parametrize("is_mla", [False, True]) +@pytest.mark.parametrize("lf_to_pf", [False, True]) +def test_transfer_kv_pf_direct( + dtype: torch.dtype, + num_items_to_transfer: int, + item_size: int, + page_size: int, + total_items_in_pool: int, + is_mla: bool, + lf_to_pf: bool, +): + original_dtype = torch.get_default_dtype() + torch.set_default_dtype(dtype) + device = "cuda" + torch.cuda.manual_seed(42) + + num_layers = 4 + + total_pages_in_pool = total_items_in_pool // page_size + num_pages_to_transfer = num_items_to_transfer // page_size + if num_pages_to_transfer == 0: + torch.set_default_dtype(original_dtype) + return + page_indices = torch.randperm(total_pages_in_pool, dtype=torch.int64) + src_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[:num_pages_to_transfer] + ] + ) + src_indices_device = src_indices_host.to(device) + dst_indices_host = torch.cat( + [ + torch.arange(p * page_size, (p + 1) * page_size) + for p in page_indices[num_pages_to_transfer : 2 * num_pages_to_transfer] + ] + ) + dst_indices_device = dst_indices_host.to(device) + + # We will test the per-layer function on the first layer (index 0) of the pool. + layer_idx_to_test = 0 + + if lf_to_pf: + if is_mla: + src_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_pool_ptrs = [src_pool[i] for i in range(num_layers)] + dst_pool_ref = torch.zeros( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + dst_pool_direct = torch.zeros_like(dst_pool_ref) + torch.cuda.synchronize() + + transfer_kv_all_layer_direct_lf_pf( + src_pool_ptrs, + [dst_pool_direct], + src_indices_host, + dst_indices_host, + page_size, + ) + for i in range(num_layers): + ref_copy_with_indices_pf_direct( + src_pool, + dst_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + + else: + src_k_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_k_pool_ptrs = [src_k_pool[i] for i in range(num_layers)] + src_v_pool = torch.randn(num_layers, total_items_in_pool, item_size).to( + device + ) + src_v_pool_ptrs = [src_v_pool[i] for i in range(num_layers)] + dst_k_pool_ref = torch.zeros( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + torch.cuda.synchronize() + + transfer_kv_all_layer_direct_lf_pf( + src_k_pool_ptrs + src_v_pool_ptrs, + [dst_k_pool_direct, dst_v_pool_direct], + src_indices_host, + dst_indices_host, + page_size, + ) + for i in range(num_layers): + ref_copy_with_indices_pf_direct( + src_k_pool, + dst_k_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + ref_copy_with_indices_pf_direct( + src_v_pool, + dst_v_pool_ref, + src_indices_device, + dst_indices_host, + page_size, + i, + lf_to_pf=True, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + else: + if is_mla: + src_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + + dst_pool_ref = torch.zeros(num_layers, total_items_in_pool, item_size).to( + device + ) + dst_pool_direct = torch.zeros_like(dst_pool_ref) + dst_pool_direct_ptrs = [dst_pool_direct[i] for i in range(num_layers)] + torch.cuda.synchronize() + + transfer_kv_per_layer_direct_pf_lf( + [src_pool], + [dst_pool_direct_ptrs[layer_idx_to_test]], + src_indices_host, + dst_indices_host, + layer_idx_to_test, + page_size, + ) + ref_copy_with_indices_pf_direct( + src_pool, + dst_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + torch.cuda.synchronize() + torch.testing.assert_close(dst_pool_direct, dst_pool_ref) + else: + src_k_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + src_v_pool = torch.randn( + total_pages_in_pool, num_layers, page_size, item_size + ).pin_memory() + + dst_k_pool_ref = torch.zeros(num_layers, total_items_in_pool, item_size).to( + device + ) + dst_k_pool_direct = torch.zeros_like(dst_k_pool_ref) + dst_k_pool_direct_ptrs = [dst_k_pool_direct[i] for i in range(num_layers)] + + dst_v_pool_ref = torch.zeros_like(dst_k_pool_ref) + dst_v_pool_direct = torch.zeros_like(dst_v_pool_ref) + dst_v_pool_direct_ptrs = [dst_v_pool_direct[i] for i in range(num_layers)] + torch.cuda.synchronize() + + transfer_kv_per_layer_direct_pf_lf( + [src_k_pool, src_v_pool], + [ + dst_k_pool_direct_ptrs[layer_idx_to_test], + dst_v_pool_direct_ptrs[layer_idx_to_test], + ], + src_indices_host, + dst_indices_host, + layer_idx_to_test, + page_size, + ) + + ref_copy_with_indices_pf_direct( + src_k_pool, + dst_k_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + ref_copy_with_indices_pf_direct( + src_v_pool, + dst_v_pool_ref, + src_indices_host, + dst_indices_device, + page_size, + layer_idx_to_test, + lf_to_pf=False, + ) + + torch.cuda.synchronize() + torch.testing.assert_close(dst_k_pool_direct, dst_k_pool_ref) + torch.testing.assert_close(dst_v_pool_direct, dst_v_pool_ref) + torch.set_default_dtype(original_dtype) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_lightning_attention_decode.py b/sgl-kernel/tests/test_lightning_attention_decode.py new file mode 100644 index 000000000..f2f0ba258 --- /dev/null +++ b/sgl-kernel/tests/test_lightning_attention_decode.py @@ -0,0 +1,84 @@ +import pytest +import torch +from sgl_kernel import lightning_attention_decode + + +def naive_lightning_attention_decode(q, k, v, past_kv, slope): + """Naive implementation of lightning attention decode""" + original_dtype = q.dtype + ratio = torch.exp(-slope) # [h, 1, 1] + + kv = past_kv + b, h, n, d = q.shape + + output = [] + for i in range(n): + kv = ratio * kv.to(torch.float32) + torch.einsum( + "... n d, ... n e -> ... d e", + k[:, :, i : i + 1], + v[:, :, i : i + 1], + ) + qkv = torch.einsum( + "... n e, ... e d -> ... n d", + q[:, :, i : i + 1].to(torch.float32), + kv.to(torch.float32), + ) + output.append(qkv) + output = torch.cat(output, dim=-2) + + return output.to(original_dtype), kv + + +configs = [ + # (batch_size, num_heads, dim, embed_dim) + (1, 8, 64, 64), + (2, 8, 64, 64), + (1, 32, 32, 64), + (2, 32, 32, 64), + (4, 32, 64, 64), + (4, 32, 64, 64), + (16, 64, 96, 96), + (64, 64, 96, 96), +] + +dtypes = [torch.float32, torch.float16, torch.bfloat16] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") +@pytest.mark.parametrize("dtype", dtypes) +@pytest.mark.parametrize("batch_size,num_heads,dim,embed_dim", configs) +def test_lightning_attention_decode(dtype, batch_size, num_heads, dim, embed_dim): + device = torch.device("cuda") + + q = torch.randn(batch_size, num_heads, 1, dim, device=device, dtype=dtype) + k = torch.randn(batch_size, num_heads, 1, dim, device=device, dtype=dtype) + v = torch.randn(batch_size, num_heads, 1, embed_dim, device=device, dtype=dtype) + past_kv = torch.randn(batch_size, num_heads, dim, embed_dim, device=device) + slope = torch.randn(num_heads, 1, 1, device=device) + + ref_output, ref_new_kv = naive_lightning_attention_decode(q, k, v, past_kv, slope) + + output = torch.empty_like(ref_output) + new_kv = torch.empty_like(ref_new_kv) + lightning_attention_decode(q, k, v, past_kv, slope, output, new_kv) + + rtol = 1e-2 + atol = 1e-2 + + torch.testing.assert_close( + output, + ref_output, + rtol=rtol, + atol=atol, + ) + + torch.testing.assert_close( + new_kv, + ref_new_kv, + rtol=rtol, + atol=atol, + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_marlin_gemm.py b/sgl-kernel/tests/test_marlin_gemm.py new file mode 100644 index 000000000..9c4d48a39 --- /dev/null +++ b/sgl-kernel/tests/test_marlin_gemm.py @@ -0,0 +1,121 @@ +import pytest +import torch +from sgl_kernel import gptq_marlin_gemm +from sgl_kernel.scalar_type import scalar_types + +from sglang.srt.layers.quantization.marlin_utils import marlin_make_workspace +from sglang.test.test_marlin_utils import awq_marlin_quantize, marlin_quantize + +MNK_FACTORS = [ + (1, 1, 1), + (1, 4, 8), + (1, 7, 5), + (13, 17, 67), + (26, 37, 13), + (67, 13, 11), + (257, 13, 11), + (658, 13, 11), +] + + +# uint4 for awq +# uint4b8 for gptq +@pytest.mark.parametrize("k_chunk", [128]) +@pytest.mark.parametrize("n_chunk", [64, 256]) +@pytest.mark.parametrize("quant_type", [scalar_types.uint4, scalar_types.uint4b8]) +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128]) +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS) +@pytest.mark.parametrize("act_order", [False, True]) +@pytest.mark.parametrize("is_k_full", [False, True]) +@pytest.mark.parametrize("use_atomic_add", [False, True]) +@pytest.mark.parametrize("use_fp32_reduce", [False, True]) +def test_gptq_marlin_gemm( + k_chunk, + n_chunk, + quant_type, + group_size, + mnk_factors, + act_order, + is_k_full, + use_atomic_add, + use_fp32_reduce, +): + m_factor, n_factor, k_factor = mnk_factors + has_zp = quant_type in [scalar_types.uint4, scalar_types.uint8] + + size_m = m_factor + size_k = k_chunk * k_factor + size_n = n_chunk * n_factor + + if act_order: + if group_size == -1: + return + if group_size == size_k: + return + if has_zp: + return + + if size_k % group_size != 0: + return + + a_input = torch.randn((size_m, size_k), dtype=torch.float16, device="cuda") + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + if has_zp: + # AWQ style, unsigned + runtime zero-point + if group_size == 16: + return + w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize( + b_weight, quant_type, group_size + ) + g_idx = None + sort_indices = None + marlin_s2 = None + else: + # GPTQ style, unsigned + symmetric bias + if group_size == 16: + return + w_ref, marlin_q_w, marlin_s, g_idx, sort_indices, _ = marlin_quantize( + b_weight, quant_type, group_size, act_order + ) + marlin_zp = None + marlin_s2 = None + + workspace = marlin_make_workspace(w_ref.device) + + # marlin gemm + output = gptq_marlin_gemm( + a_input, + None, + marlin_q_w, + marlin_s, + marlin_s2, + marlin_zp, + g_idx, + sort_indices, + workspace, + quant_type, + a_input.shape[0], + b_weight.shape[1], + a_input.shape[1], + is_k_full=is_k_full, + use_atomic_add=use_atomic_add, + use_fp32_reduce=use_fp32_reduce, + is_zp_float=False, + ) + # ref gemm + output_ref = torch.matmul(a_input, w_ref) + + torch.cuda.synchronize() + + max_diff = torch.mean(torch.abs(output - output_ref)) / torch.mean( + torch.abs(output_ref) + ) + + assert max_diff < 0.04 + + +if __name__ == "__main__": + import subprocess + + subprocess.call(["pytest", "--tb=short", str(__file__)]) diff --git a/sgl-kernel/tests/test_marlin_repack.py b/sgl-kernel/tests/test_marlin_repack.py new file mode 100644 index 000000000..508099106 --- /dev/null +++ b/sgl-kernel/tests/test_marlin_repack.py @@ -0,0 +1,148 @@ +import numpy as np +import pytest +import torch +from sgl_kernel import awq_marlin_repack, gptq_marlin_repack +from sgl_kernel.scalar_type import scalar_types + +from sglang.srt.layers.quantization.utils import ( + gptq_quantize_weights, + pack_cols, + pack_rows, + quantize_weights, + sort_weights, +) +from sglang.test.test_marlin_utils import get_weight_perm, marlin_weights + +GPTQ_MARLIN_TILE = 16 +MARLIN_K_CHUNKS = [128] +MARLIN_N_CHUNKS = [64, 256] + +MNK_FACTORS = [ + (1, 1, 1), + (1, 4, 8), + (1, 7, 5), + (13, 17, 67), + (26, 37, 13), + (67, 13, 11), + (257, 13, 11), + (658, 13, 11), +] + + +def awq_pack( + q_w: torch.Tensor, + num_bits: int, + size_k: int, + size_n: int, +): + assert q_w.shape == (size_k, size_n) + + # Interleave column dim (for the dequantize code) and pack it to int32 + if num_bits == 4: + interleave = np.array([0, 2, 4, 6, 1, 3, 5, 7]) + elif num_bits == 8: + interleave = np.array([0, 2, 1, 3]) + else: + raise Exception("num_bits must be 4 or 8, got {}".format(num_bits)) + + q_w = q_w.reshape((-1, len(interleave)))[:, interleave].ravel() + q_w = q_w.reshape((-1, size_n)).contiguous() + + return pack_cols(q_w, num_bits, size_k, size_n) + + +@pytest.mark.parametrize("num_bits", [4, 8]) +@pytest.mark.parametrize("k_tiles,n_tiles", [(1, 1), (2, 2)]) +@pytest.mark.parametrize("group_size", [16, 32]) +def test_awq_marlin_repack_correct(num_bits, k_tiles, n_tiles, group_size): + tile_k, tile_n = 16, 64 + size_k = k_tiles * tile_k + size_n = n_tiles * tile_n + pack_factor = 32 // num_bits + + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + w_ref, q_w, s, zp = quantize_weights( + b_weight, scalar_types.uint4, group_size, zero_points=True + ) + + q_w_awq = awq_pack(q_w, num_bits, size_k, size_n) + + weight_perm = get_weight_perm(num_bits) + q_w_marlin = marlin_weights(q_w, size_k, size_n, num_bits, weight_perm) + + out_gpu = awq_marlin_repack(q_w_awq, size_k, size_n, num_bits) + assert out_gpu.is_cuda and out_gpu.dtype == torch.int32 + + expected_cols = size_n * tile_k // pack_factor + assert list(out_gpu.shape) == [size_k // tile_k, expected_cols] + + torch.cuda.synchronize() + + torch.testing.assert_close(out_gpu, q_w_marlin) + + +@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS) +@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS) +@pytest.mark.parametrize("quant_type", [scalar_types.uint4b8]) +@pytest.mark.parametrize("group_size", [-1, 32, 64, 128]) +@pytest.mark.parametrize("act_order", [False, True]) +@pytest.mark.parametrize("mnk_factors", MNK_FACTORS) +def test_gptq_marlin_repack( + k_chunk, n_chunk, quant_type, group_size, act_order, mnk_factors +): + m_factor, n_factor, k_factor = mnk_factors + + size_k = k_chunk * k_factor + size_n = n_chunk * n_factor + + # Filter act_order + if act_order: + if group_size == -1: + return + if group_size == size_k: + return + + # Normalize group_size + if group_size == -1: + group_size = size_k + assert group_size <= size_k + + if size_k % group_size != 0: + pytest.skip("size_k must be divisible by group_size") + + # Create input + b_weight = torch.randn((size_k, size_n), dtype=torch.float16, device="cuda") + + # Quantize (and apply act_order if provided) + w_ref, q_w, s, g_idx, rand_perm = gptq_quantize_weights( + b_weight, quant_type, group_size, act_order + ) + + q_w_gptq = pack_rows(q_w, quant_type.size_bits, size_k, size_n) + + # For act_order, sort the "weights" and "g_idx" so that group ids are + # increasing + sort_indices = torch.empty(0, dtype=torch.int, device=b_weight.device) + if act_order: + q_w, g_idx, sort_indices = sort_weights(q_w, g_idx) + + marlin_layout_perm = get_weight_perm(quant_type.size_bits) + q_w_marlin_ref = marlin_weights( + q_w, size_k, size_n, quant_type.size_bits, marlin_layout_perm + ) + + # Run Marlin repack GPU kernel + q_w_marlin = gptq_marlin_repack( + q_w_gptq, sort_indices, size_k, size_n, quant_type.size_bits + ) + + torch.cuda.synchronize() + + torch.testing.assert_close(q_w_marlin, q_w_marlin_ref) + + +if __name__ == "__main__": + import subprocess + + subprocess.call(["pytest", "--tb=short", str(__file__)]) diff --git a/sgl-kernel/tests/test_merge_state.py b/sgl-kernel/tests/test_merge_state.py new file mode 100644 index 000000000..70b9628d9 --- /dev/null +++ b/sgl-kernel/tests/test_merge_state.py @@ -0,0 +1,142 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/55576c626421b5ee7e7ebe74afd26465c8ae863f/flashinfer/triton/kernels/cascade.py + +from typing import List + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import merge_state + + +def check_input(x: torch.Tensor): + assert x.is_cuda, f"{str(x)} must be a CUDA Tensor" + assert x.is_contiguous(), f"{str(x)} must be contiguous" + + +def check_dim(d, x: torch.Tensor): + assert x.dim() == d, f"{str(x)} must be a {d}D tensor" + + +def check_shape(a: torch.Tensor, b: torch.Tensor): + assert a.dim() == b.dim(), "tensors should have same dim" + for i in range(a.dim()): + assert a.size(i) == b.size( + i + ), f"tensors shape mismatch, {a.size()} and {b.size()}" + + +def check_device(tensors: List[torch.Tensor]): + device = tensors[0].device + for t in tensors: + assert ( + t.device == device + ), f"All tensors should be on the same device, but got {device} and {t.device}" + + +@triton.jit +def state_merge(o, m, d, other_o, other_m, other_d): + m_max = tl.maximum(m, other_m) + d = d * tl.exp2(m - m_max) + other_d * tl.exp2(other_m - m_max) + o = o * tl.exp2(m - m_max) + other_o * tl.exp2(other_m - m_max) + return o, m_max, d + + +@triton.jit +def state_normalize(o, m, d): + o = o / d + return o, m, d + + +@triton.jit +def state_get_lse(o, m, d): + return m + tl.log2(d) + + +@triton.jit +def merge_state_kernel( + v_a_ptr, + s_a_ptr, + v_b_ptr, + s_b_ptr, + v_merged_ptr, + s_merged_ptr, + num_heads, + head_dim, + bdx: tl.constexpr, + bdy: tl.constexpr, +): + pos = tl.program_id(axis=0) + for tx in tl.range(bdx): + for head_idx in tl.range(bdy): + s_a_val = tl.load(s_a_ptr + pos * num_heads + head_idx) + s_b_val = tl.load(s_b_ptr + pos * num_heads + head_idx) + + offsets = (pos * num_heads + head_idx) * head_dim + tx + v_a = tl.load(v_a_ptr + offsets) + v_b = tl.load(v_b_ptr + offsets) + + v_merged, s_max, d = state_merge( + o=v_a, m=s_a_val, d=1, other_o=v_b, other_m=s_b_val, other_d=1 + ) + v_merged, s_max, d = state_normalize(v_merged, s_max, d) + v_merged_offset = (pos * num_heads + head_idx) * head_dim + tx + tl.store(v_merged_ptr + v_merged_offset, v_merged) + + if s_merged_ptr: + tl.store( + s_merged_ptr + pos * num_heads + head_idx, + tl.log2(d) + s_max, + ) + + +def merge_state_triton( + v_a: torch.Tensor, s_a: torch.Tensor, v_b: torch.Tensor, s_b: torch.Tensor +): + check_input(v_a) + check_input(s_a) + check_input(v_b) + check_input(s_b) + check_device([v_a, s_a, v_b, s_b]) + check_dim(3, v_a) + check_dim(2, s_a) + check_dim(3, v_b) + check_dim(2, s_b) + check_shape(v_a, v_b) + check_shape(s_a, s_b) + assert v_a.size(0) == s_a.size(0) + assert v_a.size(1) == s_b.size(1) + s_a = s_a.to(torch.float32) + s_b = s_b.to(torch.float32) + seq_len = v_a.size(0) + num_heads = v_a.size(1) + head_dim = v_a.size(2) + v_merged = torch.empty_like(v_a).to(s_a.device) + s_merged = torch.empty((seq_len, num_heads)).to(s_a.device) + bdx = head_dim + bdy = num_heads + + merge_state_kernel[lambda meta: (seq_len,)]( + v_a, s_a, v_b, s_b, v_merged, s_merged, num_heads, head_dim, bdx=bdx, bdy=bdy + ) + + return v_merged, s_merged + + +@pytest.mark.parametrize("seq_len", [2048]) +@pytest.mark.parametrize("num_heads", [32]) +@pytest.mark.parametrize("head_dim", [128]) +def test_merge_state(seq_len, num_heads, head_dim): + va = torch.randn(seq_len, num_heads, head_dim).half().to("cuda:0") + sa = torch.randn(seq_len, num_heads, dtype=torch.float32).to("cuda:0") + vb = torch.randn(seq_len, num_heads, head_dim).half().to("cuda:0") + sb = torch.randn(seq_len, num_heads, dtype=torch.float32).to("cuda:0") + v_merged, s_merged = merge_state_triton(va, sa, vb, sb) + v_merged_std, s_merged_std = merge_state(va, sa, vb, sb) + + assert torch.allclose(v_merged, v_merged_std, atol=1e-2) + assert torch.allclose(s_merged, s_merged_std, atol=1e-2) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_merge_state_v2.py b/sgl-kernel/tests/test_merge_state_v2.py new file mode 100644 index 000000000..62326f75c --- /dev/null +++ b/sgl-kernel/tests/test_merge_state_v2.py @@ -0,0 +1,400 @@ +from typing import Optional + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import merge_state, merge_state_v2 + + +@triton.jit +def merge_state_kernel( + output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_merged + output_lse, # [NUM_TOKENS, NUM_HEADS] s_merged + prefix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_a + prefix_lse, # [NUM_TOKENS, NUM_HEADS] s_a + suffix_output, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] v_b + suffix_lse, # [NUM_TOKENS, NUM_HEADS] s_b + HEAD_SIZE: tl.constexpr, + PADDED_HEAD_SIZE: tl.constexpr, + OUTPUT_LSE: tl.constexpr, +): + token_idx = tl.program_id(0) + num_tokens = tl.num_programs(0) + head_idx = tl.program_id(1) + num_heads = tl.num_programs(1) + + p_lse = tl.load(prefix_lse + token_idx * num_heads + head_idx) + s_lse = tl.load(suffix_lse + token_idx * num_heads + head_idx) + p_lse = float("-inf") if p_lse == float("inf") else p_lse + s_lse = float("-inf") if s_lse == float("inf") else s_lse + + max_lse = tl.maximum(p_lse, s_lse) + p_lse = p_lse - max_lse + s_lse = s_lse - max_lse + out_se = tl.exp(p_lse) + tl.exp(s_lse) + + if OUTPUT_LSE: + out_lse = tl.log(out_se) + max_lse + tl.store(output_lse + token_idx * num_heads + head_idx, out_lse) + + head_arange = tl.arange(0, PADDED_HEAD_SIZE) + head_mask = head_arange < HEAD_SIZE + p_out = tl.load( + prefix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + s_out = tl.load( + suffix_output + + token_idx * num_heads * HEAD_SIZE + + head_idx * HEAD_SIZE + + head_arange, + mask=head_mask, + ) + + p_scale = tl.exp(p_lse) / out_se + s_scale = tl.exp(s_lse) / out_se + out = p_out * p_scale + s_out * s_scale + tl.store( + output + token_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE + head_arange, + out, + mask=head_mask, + ) + + +def merge_state_triton( + prefix_output: torch.Tensor, + prefix_lse: torch.Tensor, + suffix_output: torch.Tensor, + suffix_lse: torch.Tensor, + output: Optional[torch.Tensor] = None, + output_lse: Optional[torch.Tensor] = None, +) -> None: + num_tokens = output.shape[0] + num_query_heads = output.shape[1] + head_size = output.shape[2] + padded_head_size = triton.next_power_of_2(head_size) + # Avoid creating new tensors if they are already provided + if output is None: + output = torch.empty_like(prefix_output) + if output_lse is None: + output_lse = torch.empty_like(prefix_lse) + + merge_state_kernel[(num_tokens, num_query_heads)]( + output, + output_lse, + prefix_output, + prefix_lse, + suffix_output, + suffix_lse, + head_size, + padded_head_size, + output_lse is not None, + ) + return output, output_lse + + +# Naive PyTorch Implements of Merge Attn States +def merge_state_torch( + prefix_output: torch.Tensor, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + prefix_lse: torch.Tensor, # [NUM_TOKENS, NUM_HEADS] + suffix_output: torch.Tensor, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + suffix_lse: torch.Tensor, # [NUM_TOKENS, NUM_HEADS] + output: Optional[torch.Tensor] = None, # [NUM_TOKENS, NUM_HEADS, HEAD_SIZE] + output_lse: Optional[torch.Tensor] = None, # [NUM_TOKENS, NUM_HEADS] +): + # Avoid creating new tensors if they are already provided + if output is None: + output = torch.empty_like(prefix_output) + if output_lse is None: + output_lse = torch.empty_like(prefix_lse) + p_lse = prefix_lse + s_lse = suffix_lse + # inf -> -inf + p_lse[p_lse == torch.inf] = -torch.inf + s_lse[s_lse == torch.inf] = -torch.inf + # max_lse [NUM_HEADS, NUM_TOKENS] + max_lse = torch.maximum(p_lse, s_lse) + p_lse = p_lse - max_lse + s_lse = s_lse - max_lse + p_lse_exp = torch.exp(p_lse) + s_lse_exp = torch.exp(s_lse) + out_se = p_lse_exp + s_lse_exp + if output_lse is not None: + output_lse = torch.log(out_se) + max_lse + p_scale = p_lse_exp / out_se + s_scale = s_lse_exp / out_se + p_scale = p_scale.unsqueeze(2) # [NUM_TOKENS, NUM_HEADS, 1] + s_scale = s_scale.unsqueeze(2) # [NUM_TOKENS, NUM_HEADS, 1] + output = prefix_output * p_scale + suffix_output * s_scale + return output, output_lse + + +NUM_BATCH_TOKENS = [256, 512, 613, 1024, 1536] +NUM_QUERY_HEADS = [8, 16, 32] +HEAD_SIZES = [32, 48, 64, 128, 256] +DTYPES = [torch.half, torch.bfloat16] + +all_case_info: list[tuple] = [] + + +def generate_markdown_table(): + global all_case_info + table_header = ( + "| tokens | heads | headsize | dtype " + "| device | torch | triton | v1 | v2 | speedup(vs triton) | speedup(vs v1)|" + ) + table_separator = ( + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |" + ) + + def shortly_dtype(dtype: torch.dtype) -> str: + return str(dtype).removeprefix("torch.") + + def shortly_device(device: str) -> str: + return device.removeprefix("NVIDIA").strip() + + print(table_header) + print(table_separator) + for info in all_case_info: + ( + num_tokens, + num_heads, + head_size, + dtype, + device, + time_torch, + time_triton, + time_v1, + time_v2, + ) = info + dtype = shortly_dtype(dtype) + device = shortly_device(device) + improved_triton = time_triton / time_v2 + improved_v1 = time_v1 / time_v2 + print( + f"| {num_tokens} | {num_heads} | {head_size} " + f"| {dtype} | {device} | {time_torch:.4f}ms " + f"| {time_triton:.4f}ms " + f"| {time_v1:.4f}ms " + f"| {time_v2:.4f}ms " + f"| {improved_triton:.4f}x " + f"| {improved_v1:.4f}x |" + ) + + +@pytest.mark.parametrize("num_tokens", NUM_BATCH_TOKENS) +@pytest.mark.parametrize("num_query_heads", NUM_QUERY_HEADS) +@pytest.mark.parametrize("head_size", HEAD_SIZES) +@pytest.mark.parametrize("output_dtype", DTYPES) +@torch.inference_mode() +def test_merge_attn_states( + num_tokens: int, num_query_heads: int, head_size: int, output_dtype: torch.dtype +): + if not torch.cuda.is_available(): + pytest.skip( + "Currently only support compare triton merge_attn_states " + "with custom cuda merge_attn_states kernel" + ) + + NUM_TOKENS = num_tokens + NUM_HEADS = num_query_heads + HEAD_SIZE = head_size + + print( + f"\nNUM_TOKENS:{NUM_TOKENS}, NUM_HEADS:{NUM_HEADS}, " + f"HEAD_SIZE:{HEAD_SIZE}, DTYPE: {output_dtype}, " + f"Device: {torch.cuda.get_device_name()}" + ) + + # prefix_lse and suffix_lse contain inf and normal values + prefix_lse = torch.randn(NUM_TOKENS, NUM_HEADS, dtype=torch.float32, device="cuda") + suffix_lse = torch.randn(NUM_TOKENS, NUM_HEADS, dtype=torch.float32, device="cuda") + + # Generate boolean masks + mask_prefix = torch.rand(NUM_TOKENS, NUM_HEADS) < 0.1 + mask_suffix = torch.rand(NUM_TOKENS, NUM_HEADS) < 0.1 + # Ensure that the same position is not True at the same time + combined_mask = torch.logical_and(mask_prefix, mask_suffix) + mask_prefix = torch.logical_and(mask_prefix, ~combined_mask) + mask_suffix = torch.logical_and(mask_suffix, ~combined_mask) + + prefix_lse[mask_prefix] = float("inf") + suffix_lse[mask_suffix] = float("inf") + + # Other input tensors (need to be initialized but + # no actual calculation needed) + output = torch.zeros( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + output_lse = torch.zeros( + (NUM_TOKENS, NUM_HEADS), dtype=torch.float32, device="cuda" + ) + prefix_output = torch.randn( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + suffix_output = torch.randn( + (NUM_TOKENS, NUM_HEADS, HEAD_SIZE), dtype=output_dtype, device="cuda" + ) + + warmup_times = 2 + repeat_times = 20 + + def perf_kernel_fn( + output_fn: torch.Tensor, + output_lse_fn: torch.Tensor, + kernel_fn: callable, + fn_type: str = "torch", + ): + # Avoid inplace inf -> -inf, we have to use prefix_lse + # and suffix_lse for other kernel. + if fn_type == "torch": + prefix_lse_ = prefix_lse.clone() + suffix_lse_ = suffix_lse.clone() + else: + prefix_lse_ = prefix_lse + suffix_lse_ = suffix_lse + + if fn_type == "cuda_v1": + # merge_state v1 kernel not support float32 + if output_dtype not in (torch.half, torch.bfloat16): + return 0, output_fn, output_lse_fn + + total_time = 0 + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + + try: + for _ in range(warmup_times): + output_fn, output_lse_fn = kernel_fn( + prefix_output, + prefix_lse_, + suffix_output, + suffix_lse_, + output_fn, + output_lse_fn, + ) + torch.cuda.synchronize() + + for _ in range(repeat_times): + start.record() + output_fn, output_lse_fn = kernel_fn( + prefix_output, + prefix_lse_, + suffix_output, + suffix_lse_, + output_fn, + output_lse_fn, + ) + end.record() + torch.cuda.synchronize() + total_time += start.elapsed_time(end) + + avg_time = total_time / repeat_times + return avg_time, output_fn, output_lse_fn + except Exception as e: + return 0, output_fn, output_lse_fn + + # 0. Run the Torch kernel + output_torch = output.clone() + output_lse_torch = output_lse.clone() + time_torch, output_torch, output_lse_torch = perf_kernel_fn( + output_torch, output_lse_torch, merge_state_torch, fn_type="torch" + ) + + # 1. Run the Triton kernel + output_ref_triton = output.clone() + output_lse_ref_triton = output_lse.clone() + time_triton, output_ref_triton, output_lse_ref_triton = perf_kernel_fn( + output_ref_triton, + output_lse_ref_triton, + merge_state_triton, + fn_type="triton", + ) + + # 2. Run the merge_state V1 kernel + output_v1 = output.clone() + output_lse_v1 = output_lse.clone() + time_v1, output_v1, output_lse_v1 = perf_kernel_fn( + output_v1, output_lse_v1, merge_state, fn_type="cuda_v1" + ) + + # 3. Run the merge_state V2 kernel + output_v2 = output.clone() + output_lse_v2 = output_lse.clone() + time_v2, output_v2, output_lse_v2 = perf_kernel_fn( + output_v2, output_lse_v2, merge_state_v2, fn_type="cuda_v2" + ) + + # 4. Performance compare + improved = time_triton / time_v2 + print(f" Torch time: {time_torch:.6f}ms") + print(f" Triton time: {time_triton:.6f}ms") + print(f"CUDA v1 time: {time_v1:.6f}ms") + print(f"CUDA v2 time: {time_v2:.6f}ms, Performance: {improved:.5f}x") + print("-" * 100) + + # 5. Correctness compare + # Liger Kernel: Efficient Triton Kernels for LLM Training + # https://arxiv.org/pdf/2410.10989, 3.3 Correctness + # use rtol = 1e-2 for bfloat16. + rtol = 1e-2 if output_dtype == torch.bfloat16 else 1e-3 + + def diff(a: torch.Tensor, b: torch.Tensor): + max_diff = torch.max(torch.abs(a.float() - b.float())) + return max_diff + + # Use Triton output as reference because we want to replace + # the Triton kernel with custom CUDA kernel for merge attn + # states operation. + output_ref = output_ref_triton + output_lse_ref = output_lse_ref_triton + torch.testing.assert_close( + output_v2.float(), output_ref.float(), atol=1e-3, rtol=rtol + ) + print("Output all match, max abs diff:") + print(f"(Triton vs Torch) : {diff(output_torch, output_ref)}") + print(f"(CUDA v2 vs Torch) : {diff(output_torch, output_v2)}") + print(f"(CUDA v2 vs Triton): {diff(output_ref, output_v2)}") + print("-" * 100) + + torch.testing.assert_close( + output_lse_v2.float(), output_lse_ref.float(), atol=1e-3, rtol=rtol + ) + print("Output LSE all match, max abs diff:") + print(f"(Triton vs Torch) : {diff(output_lse_torch, output_lse_ref)}") + print(f"(CUDA v2 vs Torch) : {diff(output_lse_torch, output_lse_v2)}") + print(f"(CUDA v2 vs Triton): {diff(output_lse_ref, output_lse_v2)}") + print("-" * 100) + + print( + "All output values test passed! All inf values " + "are correctly replaced with -inf." + ) + print("-" * 100) + + device = torch.cuda.get_device_name() + all_case_info.append( + ( + NUM_TOKENS, + NUM_HEADS, + HEAD_SIZE, + output_dtype, + device, + time_torch, + time_triton, + time_v1, + time_v2, + ) + ) + if len(all_case_info) == ( + len(NUM_BATCH_TOKENS) * len(HEAD_SIZES) * len(NUM_QUERY_HEADS) * len(DTYPES) + ): + generate_markdown_table() + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_moe_align.py b/sgl-kernel/tests/test_moe_align.py new file mode 100644 index 000000000..90f04ec95 --- /dev/null +++ b/sgl-kernel/tests/test_moe_align.py @@ -0,0 +1,250 @@ +import itertools + +import pytest +import torch +import triton +import triton.language as tl +from sgl_kernel import moe_align_block_size + + +def ceil_div(a, b): + return (a + b - 1) // b + + +@triton.jit +def moe_align_block_size_stage1( + topk_ids_ptr, + tokens_cnts_ptr, + num_experts: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = pid * tokens_per_thread + off_c = (pid + 1) * num_experts + + for i in range(tokens_per_thread): + if start_idx + i < numel: + idx = tl.load(topk_ids_ptr + start_idx + i) + token_cnt = tl.load(tokens_cnts_ptr + off_c + idx) + tl.store(tokens_cnts_ptr + off_c + idx, token_cnt + 1) + + +@triton.jit +def moe_align_block_size_stage2( + tokens_cnts_ptr, + num_experts: tl.constexpr, +): + pid = tl.program_id(0) + last_cnt = 0 + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + i * num_experts + pid) + last_cnt = last_cnt + token_cnt + tl.store(tokens_cnts_ptr + i * num_experts + pid, last_cnt) + + +@triton.jit +def moe_align_block_size_stage3( + total_tokens_post_pad_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, +): + last_cumsum = 0 + off_cnt = num_experts * num_experts + for i in range(1, num_experts + 1): + token_cnt = tl.load(tokens_cnts_ptr + off_cnt + i - 1) + last_cumsum = last_cumsum + tl.cdiv(token_cnt, block_size) * block_size + tl.store(cumsum_ptr + i, last_cumsum) + tl.store(total_tokens_post_pad_ptr, last_cumsum) + + +@triton.jit +def moe_align_block_size_stage4( + topk_ids_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + tokens_cnts_ptr, + cumsum_ptr, + num_experts: tl.constexpr, + block_size: tl.constexpr, + numel: tl.constexpr, + tokens_per_thread: tl.constexpr, +): + pid = tl.program_id(0) + start_idx = tl.load(cumsum_ptr + pid) + end_idx = tl.load(cumsum_ptr + pid + 1) + + for i in range(start_idx, end_idx, block_size): + tl.store(expert_ids_ptr + i // block_size, pid) + + start_idx = pid * tokens_per_thread + off_t = pid * num_experts + + for i in range(start_idx, tl.minimum(start_idx + tokens_per_thread, numel)): + expert_id = tl.load(topk_ids_ptr + i) + token_cnt = tl.load(tokens_cnts_ptr + off_t + expert_id) + rank_post_pad = token_cnt + tl.load(cumsum_ptr + expert_id) + tl.store(sorted_token_ids_ptr + rank_post_pad, i) + tl.store(tokens_cnts_ptr + off_t + expert_id, token_cnt + 1) + + +def moe_align_block_size_triton( + topk_ids: torch.Tensor, + num_experts: int, + block_size: int, + sorted_token_ids: torch.Tensor, + expert_ids: torch.Tensor, + num_tokens_post_pad: torch.Tensor, +) -> None: + numel = topk_ids.numel() + grid = (num_experts,) + tokens_cnts = torch.zeros( + (num_experts + 1, num_experts), dtype=torch.int32, device=topk_ids.device + ) + cumsum = torch.zeros((num_experts + 1,), dtype=torch.int32, device=topk_ids.device) + tokens_per_thread = ceil_div(numel, num_experts) + + moe_align_block_size_stage1[grid]( + topk_ids, + tokens_cnts, + num_experts, + numel, + tokens_per_thread, + ) + moe_align_block_size_stage2[grid]( + tokens_cnts, + num_experts, + ) + moe_align_block_size_stage3[(1,)]( + num_tokens_post_pad, + tokens_cnts, + cumsum, + num_experts, + block_size, + ) + moe_align_block_size_stage4[grid]( + topk_ids, + sorted_token_ids, + expert_ids, + tokens_cnts, + cumsum, + num_experts, + block_size, + numel, + tokens_per_thread, + ) + + +@pytest.mark.parametrize( + "block_size,num_tokens,topk,num_experts,pad_sorted_token_ids", + list( + itertools.product( + [32, 64, 128, 256], # block_size + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096], # num_tokens + [1, 2, 4, 8, 16, 32, 64], # topk + [64, 160, 256, 257, 260, 264], # num_experts + [True, False], # pad_sorted_token_ids + ) + ), +) +def test_moe_align_block_size_compare_implementations( + block_size, num_tokens, topk, num_experts, pad_sorted_token_ids +): + + topk_ids = torch.argsort(torch.rand(num_tokens, num_experts, device="cuda"), dim=1)[ + :, :topk + ] + + max_num_tokens_padded = topk_ids.numel() + (num_experts + 1) * (block_size - 1) + + sorted_ids_cuda = torch.empty( + (max_num_tokens_padded,), dtype=torch.int32, device=topk_ids.device + ) + if not pad_sorted_token_ids: + sorted_ids_cuda.fill_(topk_ids.numel()) + max_num_m_blocks = max_num_tokens_padded // block_size + expert_ids_cuda = torch.zeros( + (max_num_m_blocks,), dtype=torch.int32, device=topk_ids.device + ) + num_tokens_post_pad_cuda = torch.empty( + (1), dtype=torch.int32, device=topk_ids.device + ) + cumsum_buffer = torch.empty( + num_experts + 2, dtype=torch.int32, device=topk_ids.device + ) + + sorted_ids_triton = torch.empty_like(sorted_ids_cuda) + sorted_ids_triton.fill_(topk_ids.numel()) + expert_ids_triton = torch.zeros_like(expert_ids_cuda) + num_tokens_post_pad_triton = torch.empty_like(num_tokens_post_pad_cuda) + + moe_align_block_size( + topk_ids, + num_experts + 1, + block_size, + sorted_ids_cuda, + expert_ids_cuda, + num_tokens_post_pad_cuda, + cumsum_buffer, + pad_sorted_token_ids, + ) + + moe_align_block_size_triton( + topk_ids, + num_experts + 1, + block_size, + sorted_ids_triton, + expert_ids_triton, + num_tokens_post_pad_triton, + ) + + assert torch.allclose(expert_ids_cuda, expert_ids_triton, atol=0, rtol=0), ( + f"Expert IDs mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA expert_ids: {expert_ids_cuda}\n" + f"Triton expert_ids: {expert_ids_triton}" + ) + + assert torch.allclose( + num_tokens_post_pad_cuda, num_tokens_post_pad_triton, atol=0, rtol=0 + ), ( + f"Num tokens post pad mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA num_tokens_post_pad: {num_tokens_post_pad_cuda}\n" + f"Triton num_tokens_post_pad: {num_tokens_post_pad_triton}" + ) + + # Select an expert to check + expert_idx = expert_ids_cuda.max().item() + + # Get the first and last block id where expert_ids_cuda == expert_idx + matching_indices = torch.where(expert_ids_cuda == expert_idx)[0] + block_sorted_start = matching_indices[0].item() * block_size + block_sorted_end = min( + (matching_indices[-1].item() + 1) * block_size, num_tokens_post_pad_cuda.item() + ) + + selected_sorted_ids_cuda = sorted_ids_cuda[ + block_sorted_start:block_sorted_end + ].sort()[0] + selected_sorted_ids_triton = sorted_ids_triton[ + block_sorted_start:block_sorted_end + ].sort()[0] + + assert torch.allclose( + selected_sorted_ids_cuda, + selected_sorted_ids_triton, + atol=0, + rtol=0, + ), ( + f"Sorted IDs mismatch for block_size={block_size}, " + f"num_tokens={num_tokens}, topk={topk}\n" + f"CUDA sorted_ids: {selected_sorted_ids_cuda}\n" + f"Triton sorted_ids: {selected_sorted_ids_triton}" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_moe_fused_gate.py b/sgl-kernel/tests/test_moe_fused_gate.py new file mode 100644 index 000000000..983895752 --- /dev/null +++ b/sgl-kernel/tests/test_moe_fused_gate.py @@ -0,0 +1,103 @@ +import pytest +import torch +from sgl_kernel import moe_fused_gate + +from sglang.srt.layers.moe.topk import biased_grouped_topk + + +@pytest.mark.parametrize( + "seq_length", + list(range(1, 10)) + + [16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536], +) +@pytest.mark.parametrize( + "params", + [ + (128, 4, 2, 4), + (256, 8, 4, 8), # deepseek v3 + (512, 16, 8, 16), + ], +) +@pytest.mark.parametrize("num_fused_shared_experts", [0, 1, 2]) +@pytest.mark.parametrize("apply_routed_scaling_factor_on_output", [False, True]) +def test_moe_fused_gate_combined( + seq_length, params, num_fused_shared_experts, apply_routed_scaling_factor_on_output +): + num_experts, num_expert_group, topk_group, topk = params + dtype = torch.float32 + + torch.manual_seed(seq_length) + tensor = torch.rand((seq_length, num_experts), dtype=dtype, device="cuda") + scores = tensor.clone() + bias = torch.rand(num_experts, dtype=dtype, device="cuda") + topk = topk + num_fused_shared_experts + + output, indices = moe_fused_gate( + tensor, + bias, + num_expert_group=num_expert_group, + topk_group=topk_group, + topk=topk, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + ref_output, ref_indices = biased_grouped_topk( + scores, + scores, + bias, + topk=topk, + renormalize=True, + num_expert_group=num_expert_group, + topk_group=topk_group, + num_fused_shared_experts=num_fused_shared_experts, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + ) + + # When num_fused_shared_experts > 0, ignore the comparison of the last topk dimension + if num_fused_shared_experts > 0: + original_indices = indices.clone() + original_ref_indices = ref_indices.clone() + + indices = indices[:, :-1] + ref_indices = ref_indices[:, :-1] + + valid_min = num_experts + valid_max = num_experts + num_fused_shared_experts + shared_indices = original_indices[:, -1] + shared_ref_indices = original_ref_indices[:, -1] + if shared_indices is not None: + assert torch.all( + (shared_indices >= valid_min) & (shared_indices < valid_max) + ), f"Shared expert indices out of range: found values outside [{valid_min}, {valid_max})" + if shared_ref_indices is not None: + assert torch.all( + (shared_ref_indices >= valid_min) & (shared_ref_indices < valid_max) + ), f"Shared expert reference indices out of range: found values outside [{valid_min}, {valid_max})" + + idx_check = torch.allclose( + ref_indices.sort()[0].to(torch.int32), + indices.sort()[0].to(torch.int32), + rtol=1e-04, + atol=1e-05, + ) + output_check = torch.allclose( + ref_output.sort()[0].to(torch.float32), + output.sort()[0].to(torch.float32), + rtol=1e-02, + atol=1e-03, + ) + + assert idx_check, ( + f"Indices mismatch at seq_length {seq_length}, dtype {dtype}, " + f"params {params}, num_fused_shared_experts {num_fused_shared_experts}" + ) + assert output_check, ( + f"Output mismatch at seq_length {seq_length}, dtype {dtype}, " + f"params {params}, num_fused_shared_experts {num_fused_shared_experts}" + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_moe_topk_softmax.py b/sgl-kernel/tests/test_moe_topk_softmax.py new file mode 100644 index 000000000..b9a802c51 --- /dev/null +++ b/sgl-kernel/tests/test_moe_topk_softmax.py @@ -0,0 +1,139 @@ +import itertools + +import pytest +import torch +from sgl_kernel import topk_softmax + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_softmax(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.float32, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + ) + + # Native torch implementation + softmax_output = torch.softmax(gating_output, dim=-1) + topk_weights_ref, topk_indices_ref = torch.topk(softmax_output, topk, dim=-1) + + # Verify the top-k weights and indices match the torch native ones + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: torch={topk_indices_ref} vs SGLang={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: torch={topk_indices_ref}, SGLang={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk, dtype", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + [torch.float16, torch.bfloat16, torch.float32], # dtype + ) + ), +) +def test_topk_softmax_dtype_regression(num_tokens, num_experts, topk, dtype): + gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda") + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights_ref, + topk_indices_ref, + gating_output.float(), + ) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang old interface={topk_indices_ref} vs SGLang new interface={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang old interface={topk_indices_ref}, SGLang new interface={topk_indices}" + + +@pytest.mark.parametrize( + "num_tokens, num_experts, topk", + list( + itertools.product( + [1, 16, 128, 512, 1024, 2048], # num_tokens + [4, 8, 16, 32, 64, 128, 256], # num_experts + [1, 2, 4], # topk + ) + ), +) +def test_topk_softmax_renormalize(num_tokens, num_experts, topk): + gating_output = torch.randn( + (num_tokens, num_experts), dtype=torch.bfloat16, device="cuda" + ) + + topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda") + topk_indices = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + + topk_softmax( + topk_weights, + topk_indices, + gating_output, + renormalize=True, + ) + + topk_weights_ref = torch.empty( + (num_tokens, topk), dtype=torch.float32, device="cuda" + ) + topk_indices_ref = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda") + token_expert_indices_ref = torch.empty( + (num_tokens, topk), dtype=torch.int32, device="cuda" + ) + + topk_softmax( + topk_weights_ref, + topk_indices_ref, + gating_output, + ) + topk_weights_ref = topk_weights_ref / topk_weights_ref.sum(dim=-1, keepdim=True) + + assert torch.allclose( + topk_weights_ref, topk_weights, atol=1e-3, rtol=1e-3 + ), f"Weights mismatch: SGLang w/o fused renormalize={topk_indices_ref} vs SGLang w/ fused renormalize={topk_weights}" + + assert torch.allclose( + topk_indices_ref.int(), topk_indices, atol=0, rtol=0 + ), f"Indices mismatch: SGLang w/o fused renormalize={topk_indices_ref}, SGLang w/ fused renormalize={topk_indices}" + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_mscclpp.py b/sgl-kernel/tests/test_mscclpp.py new file mode 100644 index 000000000..0a4332bd3 --- /dev/null +++ b/sgl-kernel/tests/test_mscclpp.py @@ -0,0 +1,146 @@ +import multiprocessing as mp +import os +import socket +import unittest +from enum import IntEnum +from typing import Any + +import sgl_kernel.allreduce as custom_ops +import torch +import torch.distributed as dist + + +class MscclContextSelection(IntEnum): + MSCCL1SHOT1NODELL = 1 + MSCCL1SHOT2NODELL = 2 + + +def _run_correctness_worker(world_size, rank, distributed_init_port, test_sizes): + device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + dist.init_process_group( + backend="nccl", + init_method=distributed_init_method, + rank=rank, + world_size=world_size, + ) + group = dist.group.WORLD + cpu_group = torch.distributed.new_group(list(range(world_size)), backend="gloo") + if rank == 0: + unique_id = [custom_ops.mscclpp_generate_unique_id()] + else: + unique_id = [None] + dist.broadcast_object_list( + unique_id, src=0, device=torch.device("cpu"), group=cpu_group + ) + unique_id = unique_id[0] + rank_to_node, rank_to_ib = list(range(world_size)), list(range(world_size)) + for r in range(world_size): + rank_to_node[r] = r // 8 + rank_to_ib[r] = rank % 8 + MAX_BYTES = 2**20 + scratch = torch.empty( + MAX_BYTES * 8, dtype=torch.bfloat16, device=torch.cuda.current_device() + ) + put_buffer = torch.empty( + MAX_BYTES, dtype=torch.bfloat16, device=torch.cuda.current_device() + ) + print(f"[{rank}] start mscclpp_context init") + nranks_per_node = torch.cuda.device_count() + selection = int(MscclContextSelection.MSCCL1SHOT1NODELL) + mscclpp_context = custom_ops.mscclpp_init_context( + unique_id, + rank, + world_size, + scratch, + put_buffer, + nranks_per_node, + rank_to_node, + rank_to_ib, + selection, + ) + try: + test_loop = 10 + for sz in test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + if sz * dtype.itemsize > MAX_BYTES: + continue + if rank == 0: + print(f"mscclpp allreduce test sz {sz}, dtype {dtype}") + for _ in range(test_loop): + inp1 = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp1_ref = inp1.clone() + out1 = torch.empty_like(inp1) + custom_ops.mscclpp_allreduce( + mscclpp_context, inp1, out1, nthreads=512, nblocks=21 + ) + dist.all_reduce(inp1_ref, group=group) + torch.testing.assert_close(out1, inp1_ref) + finally: + dist.barrier(group=group) + dist.destroy_process_group(group=group) + + +def get_open_port() -> int: + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + except OSError: + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("::1", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, test_target: Any, target_args: tuple = () +) -> None: + mp.set_start_method("spawn", force=True) + + procs = [] + distributed_init_port = get_open_port() + for i in range(world_size): + proc_args = (world_size, i, distributed_init_port) + target_args + proc = mp.Process(target=test_target, args=proc_args, name=f"Worker-{i}") + proc.start() + procs.append(proc) + + for i in range(world_size): + procs[i].join() + assert ( + procs[i].exitcode == 0 + ), f"Process {i} failed with exit code {procs[i].exitcode}" + + +class TestMSCCLAllReduce(unittest.TestCase): + test_sizes = [ + 512, + 2560, + 4096, + 5120, + 7680, + 32768, + 262144, + 524288, + ] + world_sizes = [8] + + def test_correctness(self): + for world_size in self.world_sizes: + available_gpus = torch.cuda.device_count() + if world_size > available_gpus: + print( + f"Skipping world_size={world_size}, found {available_gpus} and now ray is not supported here" + ) + continue + + print(f"Running test for world_size={world_size}") + multi_process_parallel( + world_size, _run_correctness_worker, target_args=(self.test_sizes,) + ) + print(f"custom allreduce tp = {world_size}: OK") + + +if __name__ == "__main__": + unittest.main() diff --git a/sgl-kernel/tests/test_norm.py b/sgl-kernel/tests/test_norm.py new file mode 100644 index 000000000..d22da931f --- /dev/null +++ b/sgl-kernel/tests/test_norm.py @@ -0,0 +1,133 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/4e8eb1879f9c3ba6d75511e5893183bf8f289a62/tests/test_norm.py + +import pytest +import sgl_kernel +import torch + + +def llama_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * w.float() + x = x.to(orig_dtype) + return x + + +def gemma_rms_norm(x, w, eps=1e-6): + orig_dtype = x.dtype + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x + + +def gemma_fused_add_rms_norm(x, residual, w, eps=1e-6): + orig_dtype = x.dtype + x = x + residual + residual = x + x = x.float() + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = x * (1.0 + w.float()) + x = x.to(orig_dtype) + return x, residual + + +def fused_add_rms_norm(x, residual, weight, eps): + orig_dtype = x.dtype + x = x.to(torch.float32) + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + eps) + x = (x * weight.float()).to(orig_dtype) + return x, residual + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = llama_rms_norm(x, w) + if specify_out: + y = torch.empty_like(x) + sgl_kernel.rmsnorm(x, w, out=y) + else: + y = sgl_kernel.rmsnorm(x, w) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + sgl_kernel.fused_add_rmsnorm(x_fused, residual_fused, weight, eps) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +@pytest.mark.parametrize("specify_out", [True, False]) +def test_gemma_norm(batch_size, hidden_size, dtype, specify_out): + x = torch.randn(batch_size, hidden_size).to(0).to(dtype) + w = torch.randn(hidden_size).to(0).to(dtype) + + y_ref = gemma_rms_norm(x, w) + if specify_out: + y = torch.empty_like(x) + sgl_kernel.gemma_rmsnorm(x, w, out=y) + else: + y = sgl_kernel.gemma_rmsnorm(x, w) + + torch.testing.assert_close(y_ref, y, rtol=1e-3, atol=1e-3) + + +@pytest.mark.parametrize("batch_size", [1, 19, 99, 989]) +@pytest.mark.parametrize("hidden_size", [111, 500, 1024, 3072, 3584, 4096, 8192, 16384]) +@pytest.mark.parametrize("dtype", [torch.float16]) +def test_gemma_fused_add_rmsnorm(batch_size, hidden_size, dtype): + eps = 1e-6 + + x = torch.randn(batch_size, hidden_size, dtype=dtype, device="cuda") + residual = torch.randn_like(x) + weight = torch.randn(hidden_size, dtype=dtype, device="cuda") + + x_native, residual_native = gemma_fused_add_rms_norm( + x.clone(), residual.clone(), weight, eps + ) + + x_fused = x.clone() + residual_fused = residual.clone() + sgl_kernel.gemma_fused_add_rmsnorm(x_fused, residual_fused, weight, eps) + + torch.testing.assert_close(x_fused, x_native, rtol=1e-3, atol=1e-3) + torch.testing.assert_close(residual_fused, residual_native, rtol=1e-3, atol=1e-3) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_tensor_quant_fp8.py b/sgl-kernel/tests/test_per_tensor_quant_fp8.py new file mode 100644 index 000000000..0840f298f --- /dev/null +++ b/sgl-kernel/tests/test_per_tensor_quant_fp8.py @@ -0,0 +1,67 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_tensor_quant_fp8 + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +def sglang_scaled_fp8_quant( + input: torch.Tensor, + scale: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + fp8_type_: torch.dtype = torch.float8_e4m3fn + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + is_static = True + if scale is None: + scale = torch.zeros(1, device=input.device, dtype=torch.float32) + is_static = False + sgl_per_tensor_quant_fp8(input, output, scale, is_static) + + return output, scale + + +def torch_scaled_fp8_quant(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 2048, 4096])), +) +def test_per_tensor_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x) + torch_out = torch_scaled_fp8_quant(x, sglang_scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + scale = torch.rand(1, dtype=torch.float32, device=device) + sglang_out, sglang_scale = sglang_scaled_fp8_quant(x, scale) + torch_out = torch_scaled_fp8_quant(x, scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_token_group_quant_8bit.py b/sgl-kernel/tests/test_per_token_group_quant_8bit.py new file mode 100644 index 000000000..778d14d31 --- /dev/null +++ b/sgl-kernel/tests/test_per_token_group_quant_8bit.py @@ -0,0 +1,97 @@ +import itertools + +import pytest +import torch + +from sglang.srt.layers.quantization import deep_gemm_wrapper +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_8bit as triton_per_token_group_quant_8bit, +) +from sglang.srt.layers.quantization.fp8_kernel import sglang_per_token_group_quant_8bit +from sglang.srt.layers.quantization.utils import assert_fp8_all_close +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +@pytest.mark.parametrize( + "num_tokens, hidden_dim, group_size, dst_dtype, flags", + list( + itertools.product( + [127, 128, 512, 1024, 4096, 8192], # num_tokens + [256, 512, 1024, 2048, 4096], # hidden_dim + [8, 16, 32, 64, 128], # group_size + # TODO test int8 + [fp8_type_], # dtype + [ + dict( + column_major_scales=False, + scale_tma_aligned=False, + scale_ue8m0=False, + ), + dict( + column_major_scales=True, + scale_tma_aligned=False, + scale_ue8m0=False, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=False, + ), + dict( + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + ), + ], + ) + ), +) +def test_per_token_group_quant_with_column_major( + num_tokens, + hidden_dim, + group_size, + dst_dtype, + flags, +): + if flags["scale_ue8m0"] and ((group_size != 128) or (hidden_dim % 512 != 0)): + pytest.skip() + return + if flags["scale_ue8m0"] and not deep_gemm_wrapper.DEEPGEMM_BLACKWELL: + pytest.skip("scale_ue8m0 only supported on Blackwell") + return + + x = torch.randn(num_tokens, hidden_dim, device="cuda", dtype=torch.bfloat16) + + execute_kwargs = dict( + x=x, + group_size=group_size, + eps=1e-10, + dst_dtype=dst_dtype, + **flags, + ) + + x_q_triton, x_s_triton = triton_per_token_group_quant_8bit(**execute_kwargs) + x_q_sglang, x_s_sglang = sglang_per_token_group_quant_8bit(**execute_kwargs) + + # torch.set_printoptions(profile="full") + # print(f"{x_q_triton=}") + # print(f"{x_s_triton=}") + # print(f"{x_q_sglang=}") + # print(f"{x_s_sglang=}") + # torch.set_printoptions(profile="default") + + assert_fp8_all_close(x_q_triton, x_q_sglang) + torch.testing.assert_close( + x_s_triton.contiguous(), + x_s_sglang.contiguous(), + rtol=1e-3, + atol=1e-5, + msg=lambda message: message + f" {x_s_triton=} {x_s_sglang=}", + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_per_token_quant_fp8.py b/sgl-kernel/tests/test_per_token_quant_fp8.py new file mode 100644 index 000000000..40ec9d897 --- /dev/null +++ b/sgl-kernel/tests/test_per_token_quant_fp8.py @@ -0,0 +1,57 @@ +import itertools +from typing import Optional, Tuple + +import pytest +import torch +from sgl_kernel import sgl_per_token_quant_fp8 + +from sglang.srt.utils import is_hip + +_is_hip = is_hip() +fp8_type_ = torch.float8_e4m3fnuz if _is_hip else torch.float8_e4m3fn + + +def torch_per_token_quant_fp8(tensor, inv_scale): + # The reference implementation that fully aligns to + # the kernel being tested. + finfo = torch.finfo(torch.float8_e4m3fn) + inv_scale = inv_scale.view(-1, 1) + scale = inv_scale.reciprocal() + qweight = (tensor.to(torch.float32) * scale).clamp(min=finfo.min, max=finfo.max) + qweight = qweight.to(torch.float8_e4m3fn) + return qweight + + +def sglang_per_token_quant_fp8( + input: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + scale = torch.zeros(input.size(0), device=input.device, dtype=torch.float32) + output = torch.empty_like(input, device=input.device, dtype=fp8_type_) + + sgl_per_token_quant_fp8(input, output, scale) + scale = scale.reshape(-1, 1) + + return output, scale + + +@pytest.mark.parametrize( + "num_tokens,hidden_dim", + list(itertools.product([128, 256, 512], [512, 1368, 2048, 4096])), +) +def test_per_token_quant_compare_implementations( + num_tokens: int, + hidden_dim: int, +): + device = torch.device("cuda") + x = torch.rand((num_tokens, hidden_dim), dtype=torch.float16, device=device) + + sglang_out, sglang_scale = sglang_per_token_quant_fp8(x) + torch_out = torch_per_token_quant_fp8(x, sglang_scale) + + torch.testing.assert_close( + sglang_out.float(), torch_out.float(), rtol=1e-3, atol=1e-3 + ) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py b/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py new file mode 100644 index 000000000..9410710d7 --- /dev/null +++ b/sgl-kernel/tests/test_qserve_w4a8_per_chn_gemm.py @@ -0,0 +1,118 @@ +import pytest +import torch +from sgl_kernel import qserve_w4a8_per_chn_gemm + + +# Adapted from https://github.com/mit-han-lab/omniserve/blob/main/omniserve/modeling/layers/quantized_linear/w4a8_linear.py +def convert_to_qserve_format(qweight, scale, zero): + assert qweight.min() >= 0 and qweight.max() <= 15, "Quantized weight out of range" + in_features = qweight.shape[1] + out_features = qweight.shape[0] + assert in_features % 32 == 0, "Input features must be divisible by 32" + assert out_features % 32 == 0, "Output features must be divisible by 32" + + # ---- Repack the weight ---- # + # pack to M // 32, K // 32, (8, 4), ([2], 2, 2, 4) + qweight_unpack_reorder = ( + qweight.reshape( + out_features // 32, + 2, + 2, + 8, + in_features // 32, + 2, + 4, + 4, + ) + .permute(0, 4, 3, 6, 1, 5, 2, 7) + .contiguous() + ) + qweight_unpack_reorder = ( + qweight_unpack_reorder.permute(0, 1, 2, 3, 5, 6, 7, 4) + .contiguous() + .to(torch.int8) + ) + # B_fp16_reorder = B_fp16_reorder[:, :, :, :, :, :, [3, 2, 1, 0]].contiguous() + # [16, 0, 17, 1, ...] + qweight_unpack_repacked = ( + qweight_unpack_reorder[..., 1] << 4 + ) + qweight_unpack_reorder[..., 0] + qweight_unpack_repacked = qweight_unpack_repacked.reshape( + out_features // 32, in_features // 32, 32, 16 + ) + qweight_unpack_repacked = qweight_unpack_repacked.reshape( + out_features, in_features // 2 + ).contiguous() + + # ---- Pack the scales ---- # + scale = scale.reshape(out_features).to(torch.float16).contiguous() + szero = zero.reshape(out_features).to(torch.float16).contiguous() * scale + + return qweight_unpack_repacked, scale, szero + + +# INT4 Quantization +def asym_quantize_tensor(tensor): + tensor_min = tensor.min(dim=-1, keepdim=True)[0] + tensor_max = tensor.max(dim=-1, keepdim=True)[0] + q_min = 0 + q_max = 15 + tensor_scale = (tensor_max - tensor_min) / (q_max - q_min) + tensor_zero = q_min - torch.round(tensor_min / tensor_scale) + tensor_q = torch.clamp( + torch.round(tensor / tensor_scale) + tensor_zero, q_min, q_max + ).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16), tensor_zero.to(torch.int8) + + +# INT8 Quantization +def sym_quantize_tensor(tensor): + tensor_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 127 + tensor_q = torch.clamp(torch.round(tensor / tensor_scale), -128, 127).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16) + + +def torch_w4a8_per_chn_gemm(a, b, a_scale, b_scale, b_zero, out_dtype): + print(a.shape) + print(b.shape) + print(b_zero.shape) + o = torch.matmul( + a.to(torch.float16), (b.to(torch.float16) - b_zero.to(torch.float16)).t() + ) + o = o * a_scale.view(-1, 1) * b_scale.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, out_dtype, device): + # to avoid overflow, multiply 0.01 + a = torch.randn((M, K), device=device, dtype=torch.float32) * 0.01 + b = torch.randn((N, K), device=device, dtype=torch.float32) * 0.01 + + # symmetric quantize a + a_q, a_scale = sym_quantize_tensor(a) + # asymmetric quantize b + b_q, b_scale, b_zero = asym_quantize_tensor(b) + # convert to qserve format + b_q_format, b_scale_format, b_szero_format = convert_to_qserve_format( + b_q, b_scale, b_zero + ) + + # cal sum of every row of a + a_sum = a.sum(dim=-1, keepdim=True).to(torch.float16) + out = qserve_w4a8_per_chn_gemm( + a_q, b_q_format, b_scale_format, a_scale, b_szero_format, a_sum + ) + ref_out = torch_w4a8_per_chn_gemm(a_q, b_q, a_scale, b_scale, b_zero, out_dtype) + torch.testing.assert_close(out, ref_out, rtol=1e-3, atol=1e-2) + + +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("out_dtype", [torch.float16]) +def test_accuracy(M, N, K, out_dtype): + _test_accuracy_once(M, N, K, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py b/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py new file mode 100644 index 000000000..fc26a2e60 --- /dev/null +++ b/sgl-kernel/tests/test_qserve_w4a8_per_group_gemm.py @@ -0,0 +1,183 @@ +import pytest +import torch +from sgl_kernel import qserve_w4a8_per_group_gemm + + +# Adapted from https://github.com/mit-han-lab/omniserve/blob/main/omniserve/modeling/layers/quantized_linear/w4a8_linear.py +def convert_to_qserve_format(qweight, chn_scale, scale_i8, zero_i8, group_size): + assert qweight.min() >= 0 and qweight.max() <= 15, "Quantized weight out of range" + in_features = qweight.shape[1] + out_features = qweight.shape[0] + assert in_features % 32 == 0, "Input features must be divisible by 32" + assert out_features % 32 == 0, "Output features must be divisible by 32" + assert group_size == 128, "Group size must be 128" + assert ( + in_features % group_size == 0 + ), "Input features must be divisible by group_size" + + # ---- Repack the weight ---- # + # pack to M // 32, K // 32, (8, 4), ([2], 2, 2, 4) + qweight_unpack_reorder = ( + qweight.reshape( + out_features // 32, + 2, + 2, + 8, + in_features // 32, + 2, + 4, + 4, + ) + .permute(0, 4, 3, 6, 1, 5, 2, 7) + .contiguous() + ) + qweight_unpack_reorder = ( + qweight_unpack_reorder.permute(0, 1, 2, 3, 5, 6, 7, 4) + .contiguous() + .to(torch.int8) + ) + # B_fp16_reorder = B_fp16_reorder[:, :, :, :, :, :, [3, 2, 1, 0]].contiguous() + # [16, 0, 17, 1, ...] + qweigth_unpack_repacked = ( + qweight_unpack_reorder[..., 1] << 4 + ) + qweight_unpack_reorder[..., 0] + qweigth_unpack_repacked = qweigth_unpack_repacked.reshape( + out_features // 32, in_features // 32, 32, 16 + ) + qweigth_unpack_repacked = qweigth_unpack_repacked.reshape( + out_features, in_features // 2 + ) + + # ---- Pack the scales ---- # + chn_scale = chn_scale.reshape(out_features) + + scale_i8 = ( + scale_i8.reshape(out_features, in_features // group_size) + .transpose(0, 1) + .contiguous() + ) + scale_i8 = scale_i8.reshape(in_features // group_size, out_features // 32, 32) + scale_i8 = ( + scale_i8.reshape(in_features // group_size, out_features // 32, 4, 8) + .transpose(-2, -1) + .contiguous() + ) + scale_i8 = scale_i8.reshape(in_features // group_size, out_features).contiguous() + + # ---- Pack the zeros ---- # + zero_i8 = -zero_i8 + # zero_i8 = zero_i8.int() # convert to 2-complement + + zero_i8 = ( + zero_i8.reshape(out_features, in_features // group_size) + .transpose(0, 1) + .contiguous() + ) + zero_i8 = zero_i8.reshape(in_features // group_size, out_features // 32, 32) + # for the last dimension, organize as 0, 8, 16, 24, 1, 9, 17, 25, ... following the requirement of tensor core gemm + zero_i8 = ( + zero_i8.reshape(in_features // group_size, out_features // 32, 4, 8) + .transpose(-2, -1) + .contiguous() + ) + zero_i8 = ( + zero_i8.reshape(in_features // group_size, out_features).contiguous() * scale_i8 + ) + + return qweigth_unpack_repacked, chn_scale, scale_i8, zero_i8 + + +# Progressive Group INT4 Quantization +def progressive_group_quantize_tensor(tensor, group_size): + assert group_size == 128, "Group size must be 128" + assert ( + tensor.shape[-1] % group_size == 0 + ), "Input features must be divisible by group_size" + # Channel scale + # NOTE(HandH1998): use protective quantization range + chn_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 119 + tensor_i8 = torch.clamp(torch.round(tensor / chn_scale), -119, 119) + + # Group scale + tensor_i8 = tensor_i8.reshape(-1, group_size) + tensor_i8_min = tensor_i8.min(dim=-1, keepdim=True)[0] + tensor_i8_max = tensor_i8.max(dim=-1, keepdim=True)[0] + q_min = 0 + q_max = 15 + scale_i8 = torch.round((tensor_i8_max - tensor_i8_min) / (q_max - q_min)) + zero_i8 = q_min - torch.round(tensor_i8_min / scale_i8) + tensor_q = ( + torch.clamp(torch.round(tensor_i8 / scale_i8) + zero_i8, q_min, q_max) + .reshape(tensor.shape[0], -1) + .to(torch.int8) + ) + return ( + tensor_q, + chn_scale.to(torch.float16), + scale_i8.reshape(tensor.shape[0], -1).to(torch.int8), + zero_i8.reshape(tensor.shape[0], -1).to(torch.int8), + ) + + +# INT8 Quantization +def sym_quantize_tensor(tensor): + tensor_scale = tensor.abs().max(dim=-1, keepdim=True)[0] / 127 + tensor_q = torch.clamp(torch.round(tensor / tensor_scale), -128, 127).to(torch.int8) + return tensor_q, tensor_scale.to(torch.float16) + + +def torch_w4a8_per_group_gemm( + a, b, a_scale, b_chn_scale, b_scale_i8, b_zero_i8, group_size, out_dtype +): + assert group_size == 128, "Group size must be 128" + b_dq = ( + b.reshape(-1, group_size).to(torch.float32) + - b_zero_i8.reshape(-1, 1).to(torch.float32) + ) * b_scale_i8.reshape(-1, 1).to(torch.float32) + b_dq = b_dq.reshape(b.shape[0], b.shape[1]) + o = torch.matmul(a.to(torch.float32), b_dq.t()) + o = o * a_scale.view(-1, 1) * b_chn_scale.view(1, -1) + return o.to(out_dtype) + + +def _test_accuracy_once(M, N, K, group_size, out_dtype, device): + # to avoid overflow, multiply 0.01 + a = torch.randn((M, K), device=device, dtype=torch.float32) * 0.01 + b = torch.randn((N, K), device=device, dtype=torch.float32) * 0.01 + + # symmetric quantize a + a_q, a_scale = sym_quantize_tensor(a) + # asymmetric quantize b + b_q, b_chn_scale, b_scale_i8, b_zero_i8 = progressive_group_quantize_tensor( + b, group_size + ) + # convert to qserve format + b_q_format, b_chn_scale_format, b_scale_i8_format, b_zero_i8_format = ( + convert_to_qserve_format(b_q, b_chn_scale, b_scale_i8, b_zero_i8, group_size) + ) + + out = qserve_w4a8_per_group_gemm( + a_q, + b_q_format, + b_zero_i8_format, + b_scale_i8_format, + b_chn_scale_format, + a_scale, + ) + ref_out = torch_w4a8_per_group_gemm( + a_q, b_q, a_scale, b_chn_scale, b_scale_i8, b_zero_i8, group_size, out_dtype + ) + torch.testing.assert_close(out, ref_out, rtol=1e-3, atol=1e-5) + + +@pytest.mark.parametrize("M", [1, 16, 32, 64, 128, 512, 1024, 4096, 8192]) +@pytest.mark.parametrize("N", [128, 512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("K", [512, 1024, 4096, 8192, 16384]) +@pytest.mark.parametrize("group_size", [128]) +@pytest.mark.parametrize("out_dtype", [torch.float16]) +def test_accuracy(M, N, K, group_size, out_dtype): + _test_accuracy_once(M, N, K, group_size, out_dtype, "cuda") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_rotary_embedding.py b/sgl-kernel/tests/test_rotary_embedding.py new file mode 100644 index 000000000..d9f9364b0 --- /dev/null +++ b/sgl-kernel/tests/test_rotary_embedding.py @@ -0,0 +1,138 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import pytest +import torch +from sgl_kernel import FusedSetKVBufferArg, apply_rope_with_cos_sin_cache_inplace +from sgl_kernel.testing.rotary_embedding import ( + FlashInferRotaryEmbedding, + MHATokenToKVPool, + RotaryEmbedding, + create_inputs, +) + + +@pytest.mark.parametrize( + "head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype, device, batch_size, seq_len, num_q_heads, num_kv_heads, save_kv_cache", + [ + # GPT-OSS cases + *[ + ( + 64, + 64, + 4096, + 8000, + True, + torch.bfloat16, + "cuda", + batch_size, + seq_len, + 64, + 8, + save_kv_cache, + ) + for batch_size, seq_len in ( + (1, 1), + (32, 1), + (128, 1), + (512, 1), + (2, 512), + (4, 4096), + ) + for save_kv_cache in (False, True) + ], + # Other cases + (64, 64, 32, 8000, True, torch.bfloat16, "cuda", 32, 32, 1, 1, False), + (256, 128, 4096, 10000, True, torch.bfloat16, "cuda", 2, 512, 4, 2, False), + (512, 128, 311, 10000, True, torch.bfloat16, "cuda", 3, 39, 4, 2, False), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 32, 8, False), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 16, 4, False), + (512, 128, 311, 10000, False, torch.bfloat16, "cuda", 3, 39, 4, 2, False), + ], +) +def test_correctness( + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q_heads: int, + num_kv_heads: int, + save_kv_cache: bool, +): + config = dict( + head_size=head_size, + rotary_dim=rotary_dim, + max_position_embeddings=max_position_embeddings, + base=base, + is_neox_style=is_neox_style, + dtype=dtype, + ) + + rope_ref = RotaryEmbedding(**config).to(device) + rope_flashinfer = FlashInferRotaryEmbedding(**config).to(device) + + inputs = create_inputs( + head_size=head_size, + batch_size=batch_size, + seq_len=seq_len, + device=device, + dtype=dtype, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + ) + + if save_kv_cache: + pool_ref = MHATokenToKVPool(head_num=num_kv_heads, head_dim=head_size) + pool_flashinfer = MHATokenToKVPool(head_num=num_kv_heads, head_dim=head_size) + + query_ref, key_ref = inputs["query"].clone(), inputs["key"].clone() + query_flashinfer, key_flashinfer = inputs["query"].clone(), inputs["key"].clone() + + query_ref_out, key_ref_out = rope_ref.forward_native( + inputs["pos_ids"], query_ref, key_ref + ) + if save_kv_cache: + pool_ref.set_kv_buffer( + loc=inputs["out_cache_loc"], + cache_k=key_ref_out.view(-1, num_kv_heads, head_size), + cache_v=inputs["value"].view(-1, num_kv_heads, head_size), + ) + + query_flashinfer_out, key_flashinfer_out = rope_flashinfer.forward_cuda( + inputs["pos_ids"], + query_flashinfer, + key_flashinfer, + fused_set_kv_buffer_arg=( + FusedSetKVBufferArg( + value=inputs["value"], + k_buffer=pool_flashinfer.k_buffer[0].view(-1, num_kv_heads * head_size), + v_buffer=pool_flashinfer.v_buffer[0].view(-1, num_kv_heads * head_size), + k_scale=None, + v_scale=None, + cache_loc=inputs["out_cache_loc"], + ) + if save_kv_cache + else None + ), + ) + + torch.testing.assert_close( + query_ref_out, query_flashinfer_out, atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(key_ref_out, key_flashinfer_out, atol=1e-2, rtol=1e-2) + if save_kv_cache: + for field in ["k_buffer", "v_buffer"]: + x_ref = getattr(pool_ref, field)[0] + x_flashinfer = getattr(pool_flashinfer, field)[0] + torch.testing.assert_close(x_ref, x_flashinfer, atol=1e-2, rtol=1e-2) + nonzero_ref = x_ref != 0 + nonzero_flashinfer = x_ref != 0 + assert torch.all(nonzero_ref == nonzero_flashinfer) + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_sampling.py b/sgl-kernel/tests/test_sampling.py new file mode 100644 index 000000000..dc5734cb7 --- /dev/null +++ b/sgl-kernel/tests/test_sampling.py @@ -0,0 +1,185 @@ +# Adapted from https://github.com/flashinfer-ai/flashinfer/blob/93e1a2634e22355b0856246b032b285ad1d1da6b/tests/test_sampling.py + +import pytest +import sgl_kernel +import torch + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [100]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_sampling_from_probs_logits_top_k_first_alignment( + batch_size, vocab_size, k, p +): + torch.manual_seed(42) + logits = torch.randn(batch_size, vocab_size, device="cuda:0") * 5 + generator_logits = torch.Generator("cuda:0") + generator_probs = generator_logits.clone_state() + samples = sgl_kernel.sampling.top_k_top_p_sampling_from_logits( + logits, k, p, filter_apply_order="top_k_first", generator=generator_logits + ) + samples_ref = sgl_kernel.sampling.top_k_top_p_sampling_from_probs( + torch.softmax(logits, dim=-1), + k, + p, + filter_apply_order="top_k_first", + generator=generator_probs, + ) + assert torch.all(samples == samples_ref) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [100]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_sampling_from_probs_logits_joint_alignment( + batch_size, vocab_size, k, p +): + torch.manual_seed(42) + logits = torch.randn(batch_size, vocab_size, device="cuda:0") * 5 + generator_logits = torch.Generator("cuda:0") + generator_probs = generator_logits.clone_state() + samples = sgl_kernel.sampling.top_k_top_p_sampling_from_logits( + logits, k, p, filter_apply_order="joint", generator=generator_logits + ) + samples_ref = sgl_kernel.sampling.top_k_top_p_sampling_from_probs( + torch.softmax(logits, dim=-1), + k, + p, + filter_apply_order="joint", + generator=generator_probs, + ) + assert torch.all(samples == samples_ref) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5]) +def test_top_k_top_p_joint_sampling_from_probs(batch_size, vocab_size, p): + torch.manual_seed(42) + if p == 0.1: + k = int(vocab_size * 0.5) + elif p == 0.5: + k = int(vocab_size * 0.1) + else: + raise ValueError("p not recognized") + eps = 1e-4 + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + # top-p mask + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask_top_p = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask_top_p.scatter_add_(1, indices, (cdf > (1 - p) - eps).int()) + # top-k mask + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask_top_k = (normalized_prob >= pivot.unsqueeze(-1)).int() + # overall mask + mask = torch.minimum(mask_top_p, mask_top_k) + top_p_tensor = torch.full((batch_size,), p, device="cuda:0") + top_k_tensor = torch.full((batch_size,), k, device="cuda:0") + + num_trails = 1000 + for _ in range(num_trails): + samples = sgl_kernel.top_k_top_p_sampling_from_probs( + normalized_prob, + top_k_tensor, + top_p_tensor, + filter_apply_order="joint", + ) + assert torch.all(samples < vocab_size) and torch.all(samples >= 0) + assert torch.all(mask[torch.arange(batch_size), samples] == 1), normalized_prob[ + torch.arange(batch_size), samples + ] + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.1, 0.5, 0.9]) +def test_top_p_renorm_probs(batch_size, vocab_size, p): + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + cdf = torch.cumsum(sorted_prob, dim=-1) + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask.scatter_add_(1, indices, (cdf >= (1 - p)).int()) + renorm_prob_ground_truth = normalized_prob.clone() + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_p_renorm_prob(normalized_prob, p) + torch.testing.assert_close( + renorm_prob_ground_truth, + renorm_prob, + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("k", [10, 100, 500]) +def test_top_k_renorm_probs(batch_size, vocab_size, k): + if k > vocab_size: + pytest.skip("k should be less than vocab_size") + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, _ = torch.sort(normalized_prob, descending=True) + pivot = sorted_prob[:, k - 1] + mask = (normalized_prob >= pivot.unsqueeze(-1)).int() + renorm_prob_ground_truth = normalized_prob.clone() + renorm_prob_ground_truth[mask == 0] = 0 + renorm_prob_ground_truth = renorm_prob_ground_truth / renorm_prob_ground_truth.sum( + dim=-1, keepdim=True + ) + + renorm_prob = sgl_kernel.top_k_renorm_prob(normalized_prob, k) + for i in range(batch_size): + torch.testing.assert_close( + renorm_prob_ground_truth[i], + renorm_prob[i], + rtol=1e-3, + atol=1e-3, + ) + + +@pytest.mark.parametrize("batch_size", [1, 99, 989]) +@pytest.mark.parametrize("vocab_size", [111, 32000, 128256]) +@pytest.mark.parametrize("p", [0.05, 0.1, 0.2, 0.7, 1]) +def test_min_p_sampling(batch_size, vocab_size, p): + torch.manual_seed(42) + pre_norm_prob = torch.rand(batch_size, vocab_size, device="cuda:0") + normalized_prob = pre_norm_prob / pre_norm_prob.sum(dim=-1, keepdim=True) + sorted_prob, indices = torch.sort(normalized_prob, descending=False) + # scale min-p + top_probs = sorted_prob[:, -1].unsqueeze(-1) + scaled_p = p * top_probs + # min-p mask + mask = torch.zeros(batch_size, vocab_size, dtype=torch.int32, device="cuda:0") + mask.scatter_add_(1, indices, (sorted_prob >= scaled_p).int()) + min_p_tensor = torch.full((batch_size,), p, device="cuda:0") + + num_trails = 1000 + for _ in range(num_trails): + samples = sgl_kernel.min_p_sampling_from_probs( + normalized_prob, + min_p_tensor, + ) + + assert torch.all(mask[torch.arange(batch_size), samples] == 1), samples[ + torch.nonzero(mask[torch.arange(batch_size), samples] == 0) + ] + + assert torch.all(mask[torch.arange(batch_size), samples] == 1), samples[ + torch.nonzero(mask[torch.arange(batch_size), samples] == 0) + ] + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/test_sparse_flash_attn.py b/sgl-kernel/tests/test_sparse_flash_attn.py new file mode 100644 index 000000000..28c64cb61 --- /dev/null +++ b/sgl-kernel/tests/test_sparse_flash_attn.py @@ -0,0 +1,492 @@ +import math +from typing import List, Optional, Tuple + +import pytest +import torch +from einops import rearrange, repeat +from sgl_kernel.sparse_flash_attn import ( + convert_vertical_slash_indexes, + convert_vertical_slash_indexes_mergehead, + sparse_attn_func, +) +from test_flash_attention import construct_local_mask, is_fa3_supported + + +def ref_attn( + q, + k, + v, + query_padding_mask=None, + key_padding_mask=None, + attn_bias=None, + dropout_p=0.0, + dropout_mask=None, + causal=False, + window_size=(-1, -1), # -1 means infinite window size + softcap=0.0, + upcast=True, + reorder_ops=False, + key_leftpad=None, +): + """ + Arguments: + q: (batch_size, seqlen_q, nheads, head_dim) + k: (batch_size, seqlen_k, nheads_k, head_dim) + v: (batch_size, seqlen_k, nheads_k, head_dim) + query_padding_mask: (batch_size, seqlen_q) + key_padding_mask: (batch_size, seqlen_k) + attn_bias: broadcastable to (batch_size, nheads, seqlen_q, seqlen_k) + dropout_p: float + dropout_mask: (batch_size, nheads, seqlen_q, seqlen_k) + causal: whether to apply causal masking + window_size: (int, int), left and right window size + upcast: whether to cast all inputs to fp32, do all computation in fp32, then cast + output back to fp16/bf16. + reorder_ops: whether to change the order of operations (scaling k instead of scaling q, etc.) + without changing the math. This is to estimate the numerical error from operation + reordering. + Output: + output: (batch_size, seqlen_q, nheads, head_dim) + lse: (batch_size, nheads, seqlen_q) + """ + if causal: + window_size = (window_size[0], 0) + dtype_og = q.dtype + if upcast: + q, k, v = q.float(), k.float(), v.float() + seqlen_q, seqlen_k = q.shape[1], k.shape[1] + k = repeat(k, "b s h d -> b s (h g) d", g=q.shape[2] // k.shape[2]) + v = repeat(v, "b s h d -> b s (h g) d", g=q.shape[2] // v.shape[2]) + d = q.shape[-1] + if not reorder_ops: + scores = torch.einsum("bthd,bshd->bhts", q / math.sqrt(d), k) + else: + scores = torch.einsum("bthd,bshd->bhts", q, k / math.sqrt(d)) + + lse_ref = scores.logsumexp(dim=-1) + + if softcap > 0: + scores = scores / softcap + scores = scores.tanh() + scores = scores * softcap + if key_padding_mask is not None: + scores.masked_fill_( + rearrange(~key_padding_mask, "b s -> b 1 1 s"), float("-inf") + ) + if window_size[0] >= 0 or window_size[1] >= 0: + local_mask = construct_local_mask( + seqlen_q, + seqlen_k, + window_size, + query_padding_mask, + key_padding_mask, + q.device, + key_leftpad=key_leftpad, + ) + scores.masked_fill_(local_mask, float("-inf")) + if attn_bias is not None: + scores = scores + attn_bias + attention = torch.softmax(scores, dim=-1).to(v.dtype) + # Some rows might be completely masked out so we fill them with zero instead of NaN + if window_size[0] >= 0 or window_size[1] >= 0: + attention = attention.masked_fill( + torch.all(local_mask, dim=-1, keepdim=True), 0.0 + ) + # We want to mask here so that the attention matrix doesn't have any NaNs + # Otherwise we'll get NaN in dV + if query_padding_mask is not None: + attention = attention.masked_fill( + rearrange(~query_padding_mask, "b s -> b 1 s 1"), 0.0 + ) + dropout_scaling = 1.0 / (1 - dropout_p) + # attention_drop = attention.masked_fill(~dropout_mask, 0.0) * dropout_scaling + # output = torch.einsum('bhts,bshd->bthd', attention_drop , v) + if dropout_mask is not None: + attention_drop = attention.masked_fill(~dropout_mask, 0.0) + else: + attention_drop = attention + output = torch.einsum("bhts,bshd->bthd", attention_drop, v * dropout_scaling) + if query_padding_mask is not None: + output.masked_fill_(rearrange(~query_padding_mask, "b s -> b s 1 1"), 0.0) + + return output.to(dtype=dtype_og), lse_ref + + +def ref_paged_attn( + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + query_lens: List[int], + kv_lens: List[int], + block_tables: torch.Tensor, + scale: float, + sliding_window: Optional[int] = None, + soft_cap: Optional[float] = None, +) -> torch.Tensor: + num_seqs = len(query_lens) + block_tables = block_tables.cpu().numpy() + _, block_size, num_kv_heads, head_size = key_cache.shape + + outputs: List[torch.Tensor] = [] + start_idx = 0 + for i in range(num_seqs): + query_len = query_lens[i] + kv_len = kv_lens[i] + # clone to avoid clobbering the query tensor + q = query[start_idx : start_idx + query_len].clone() + q *= scale + + num_kv_blocks = (kv_len + block_size - 1) // block_size + block_indices = block_tables[i, :num_kv_blocks] + + k = key_cache[block_indices].view(-1, num_kv_heads, head_size) + k = k[:kv_len] + v = value_cache[block_indices].view(-1, num_kv_heads, head_size) + v = v[:kv_len] + + if q.shape[1] != k.shape[1]: + k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1) + v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1) + attn = torch.einsum("qhd,khd->hqk", q, k).float() + empty_mask = torch.ones(query_len, kv_len) + mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool() + if sliding_window is not None: + sliding_window_mask = ( + torch.triu( + empty_mask, diagonal=kv_len - (query_len + sliding_window) + 1 + ) + .bool() + .logical_not() + ) + mask |= sliding_window_mask + if soft_cap is not None: + attn = soft_cap * torch.tanh(attn / soft_cap) + attn.masked_fill_(mask, float("-inf")) + attn = torch.softmax(attn, dim=-1).to(v.dtype) + out = torch.einsum("hqk,khd->qhd", attn, v) + + outputs.append(out) + start_idx += query_len + + return torch.cat(outputs, dim=0) + + +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("batch_size", [1, 2]) +@pytest.mark.parametrize( + "seq_lens", + [ + (1, 1), + (1, 1024), + (1, 2048), + (1023, 2049), + (1023, 1023), + (32, 32), + (65, 65), + (129, 129), + ], +) +@pytest.mark.parametrize("num_heads", [1, 2, 4]) +@pytest.mark.parametrize("head_size", [128]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("NNZ_S", [0, 1, 2, 3, 7, 15, 32]) +@torch.inference_mode() +def test_sparse_attention( + batch_size, + seq_lens, + num_heads, + head_size, + dtype, + NNZ_S, +) -> None: + torch.set_default_device("cuda") + torch.cuda.manual_seed_all(0) + block_size_M = 64 + block_size_N = 64 + seqlen_q, seqlen_k = seq_lens + q = torch.randn( + batch_size, seqlen_q, num_heads, head_size, dtype=dtype, requires_grad=False + ) + k = torch.randn( + batch_size, seqlen_k, num_heads, head_size, dtype=dtype, requires_grad=False + ) + v = torch.randn( + batch_size, seqlen_k, num_heads, head_size, dtype=dtype, requires_grad=False + ) + NUM_ROWS = (seqlen_q + block_size_M - 1) // block_size_M + if NNZ_S * block_size_N > seqlen_k: + return + NNZ_V = seqlen_k - NNZ_S * block_size_N + block_count = torch.tensor( + [NNZ_S] * batch_size * NUM_ROWS * num_heads, dtype=torch.int32 + ).reshape(batch_size, num_heads, NUM_ROWS) + column_count = torch.tensor( + [NNZ_V] * batch_size * NUM_ROWS * num_heads, dtype=torch.int32 + ).reshape(batch_size, num_heads, NUM_ROWS) + block_offset = torch.tensor( + [[i * block_size_N for i in range(NNZ_S)]] * batch_size * NUM_ROWS * num_heads, + dtype=torch.int32, + ).reshape(batch_size, num_heads, NUM_ROWS, NNZ_S) + column_index = torch.tensor( + [[NNZ_S * block_size_N + i for i in range(NNZ_V)]] + * batch_size + * NUM_ROWS + * num_heads, + dtype=torch.int32, + ).reshape(batch_size, num_heads, NUM_ROWS, NNZ_V) + out, lse = sparse_attn_func( + q, + k, + v, + block_count, + block_offset, + column_count, + column_index, + return_softmax_lse=True, + ) + + ref_out, ref_lse = ref_attn(q, k, v) + + torch.testing.assert_close( + out, ref_out, atol=2e-2, rtol=1e-2 + ), f"{torch.max(torch.abs(out - ref_out))}" + torch.testing.assert_close( + lse, ref_lse, atol=2e-2, rtol=1e-2 + ), f"{torch.max(torch.abs(lse - ref_lse))}" + + +# sparse attention utils +# origin +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("causal", [True, False]) +def test_convert_vertical_slash_indexes(causal): + # Prepare small, hand-checkable inputs + q_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") # [BATCH] + kv_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + vertical_indexes = torch.tensor( + [[[1, 3]]], dtype=torch.int32, device="cuda" + ) # [BATCH, N_HEADS, NNZ_V] + slash_indexes = torch.tensor( + [[[2]]], dtype=torch.int32, device="cuda" + ) # [BATCH, N_HEADS, NNZ_S] + context_size = 4 + block_size_M = 2 + block_size_N = 2 + + # Call your CUDA kernel wrapper + block_count, block_offset, column_count, column_index = ( + convert_vertical_slash_indexes( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + context_size, + block_size_M, + block_size_N, + causal=causal, + ) + ) + + # Manually create expected outputs for this input + # There are 2 rows (blocks): row0 (tokens 0-1), row1 (tokens 2-3) + # Fill these expected tensors based on your CUDA kernel's logic + # For demonstration, we assume: + # - block_count: how many slash indices fall into each block + # - block_offset: the value of those indices + # - column_count: number of valid vertical indices per block + # - column_index: the actual vertical indices + + expected_column_index = torch.tensor( + [[[[0, 0], [0, 0]]]], dtype=torch.int32, device="cuda" + ) + + # If causal=False, update these tensors according to expected behavior + if not causal: + # Update these values if your kernel produces different output in non-causal mode + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]]]], dtype=torch.int32, device="cuda" + ) + + # Assert that outputs match expectations + assert torch.equal(column_index, expected_column_index) + + +# mergehead +@pytest.mark.skipif( + not is_fa3_supported(), + reason="flash_attn at sgl-kernel is only supported on sm90 or sm80", +) +@pytest.mark.parametrize("causal", [True, False]) +def test_convert_vertical_slash_indexes_mergehead(causal): + # Prepare small, hand-checkable inputs for mergehead version + q_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + kv_seqlens = torch.tensor([4], dtype=torch.int32, device="cuda") + vertical_indexes = torch.tensor( + [ + [ + [1, 3], # head 0 + [2, 0], # head 1 + ] + ], + dtype=torch.int32, + device="cuda", + ) # [BATCH, N_HEADS, NNZ_V] + slash_indexes = torch.tensor( + [ + [ + [2, 0], # head 0 + [1, 3], # head 1 + ] + ], + dtype=torch.int32, + device="cuda", + ) # [BATCH, N_HEADS, NNZ_S] + vertical_indices_count = torch.tensor([2, 1], dtype=torch.int32, device="cuda") + slash_indices_count = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + context_size = 4 + block_size_M = 2 + block_size_N = 2 + + # Call your CUDA kernel wrapper + block_count, block_offset, column_count, column_index = ( + convert_vertical_slash_indexes_mergehead( + q_seqlens, + kv_seqlens, + vertical_indexes, + slash_indexes, + vertical_indices_count, + slash_indices_count, + context_size, + block_size_M, + block_size_N, + causal=causal, + ) + ) + + # Manually create expected outputs for this input + # For demonstration, assume: + # - batch=1, head=2, num_rows=2, nnz_v=2, nnz_s=2 + # Fill these expected tensors according to your kernel's behavior + + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]], [[-1079459945, -1077788999], [-1080050043, -1104625879]]]], + dtype=torch.int32, + device="cuda", + ) + + if not causal: + # If non-causal mode output is different, update these values + expected_column_index = torch.tensor( + [[[[1, 0], [1, 3]], [[2, -1077788999], [2, -1104625879]]]], + dtype=torch.int32, + device="cuda", + ) + + # Assert that outputs match expectations + assert torch.equal(column_index, expected_column_index) + + +# skip cause use fa2 for test +# @pytest.mark.parametrize("seq_lens", [[(1024, 1328)], +# [(1024, 1328), (1, 2048)], +# [(1025, 1328), (2, 2048)], +# [(1025, 2049), (2, 1281)], +# ]) +# @pytest.mark.parametrize("head_size", [128]) +# @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +# @torch.inference_mode() +# def test_sparse_attention_varlen( +# seq_lens, +# head_size, +# dtype, +# ) -> None: +# torch.set_default_device("cuda") +# torch.cuda.manual_seed_all(0) +# block_size_M = 64 +# block_size_N = 64 +# num_seqs = len(seq_lens) +# query_lens = [x[0] for x in seq_lens] +# kv_lens = [x[1] for x in seq_lens] +# num_heads = 1 +# query = torch.randn(sum(query_lens), +# num_heads, +# head_size, +# dtype=dtype) +# key = torch.randn(sum(kv_lens), +# num_heads, +# head_size, +# dtype=dtype) +# value = torch.randn_like(key) +# cu_query_lens = torch.tensor([0] + query_lens, +# dtype=torch.int32).cumsum(dim=0, +# dtype=torch.int32) +# cu_kv_lens = torch.tensor([0] + kv_lens, +# dtype=torch.int32).cumsum(dim=0, +# dtype=torch.int32) +# max_query_len = max(query_lens) +# max_kv_len = max(kv_lens) + +# NUM_ROWS = (max_query_len + block_size_M - 1) // block_size_M +# NNZ_S = 20 +# NNZ_V = 2048 +# batch_size = len(query_lens) + +# block_counts = [] +# column_counts = [] +# block_offsets = [] +# column_indices = [] +# for b in range(batch_size): +# block_counts.append(torch.tensor([NNZ_S] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS)) +# columns = kv_lens[b] - NNZ_S * block_size_N +# column_counts.append(torch.tensor([columns] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS)) +# block_offsets.append(torch.tensor([[i * block_size_N for i in range(NNZ_S)]] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS, NNZ_S)) +# column_indices.append(torch.tensor([[NNZ_S * block_size_N + i for i in range(NNZ_V)]] * NUM_ROWS * num_heads, dtype=torch.int32).reshape(num_heads, NUM_ROWS, NNZ_V)) +# block_count = torch.concat(block_counts).reshape(batch_size, num_heads, NUM_ROWS) +# column_count = torch.concat(column_counts).reshape(batch_size, num_heads, NUM_ROWS) +# block_offset = torch.concat(block_offsets).reshape(batch_size, num_heads, NUM_ROWS, NNZ_S) +# column_index = torch.concat(column_indices).reshape(batch_size, num_heads, NUM_ROWS, NNZ_V) +# out, lse = sparse_attn_varlen_func( +# query, +# key, +# value, +# block_count, +# block_offset, +# column_count, +# column_index, +# cu_seqlens_q=cu_query_lens, +# cu_seqlens_k=cu_kv_lens, +# max_seqlen_q=max_query_len, +# max_seqlen_k=max_kv_len, +# return_softmax_lse=True, +# ) + +# max_num_blocks_per_seq = (max_kv_len + 2048 - 1) // 2048 +# block_tables = torch.randint(0, +# 2048, +# (len(query_lens), max_num_blocks_per_seq), +# dtype=torch.int32) +# scale = head_size**-0.5 + +# ref_out, ref_lse, _ = ref_paged_attn( +# query, +# key, +# value, +# query_lens=query_lens, +# kv_lens=kv_lens, +# block_tables=block_tables, +# scale=scale +# ) + +# torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=1e-2), \ +# f"{torch.max(torch.abs(out - ref_out))}" +# torch.testing.assert_close(lse, ref_lse, atol=2e-2, rtol=1e-2), \ +# f"{torch.max(torch.abs(lse - ref_lse))}" + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/sgl-kernel/tests/utils.py b/sgl-kernel/tests/utils.py new file mode 100644 index 000000000..8fa9a2234 --- /dev/null +++ b/sgl-kernel/tests/utils.py @@ -0,0 +1,9 @@ +import torch + + +def is_sm10x(): + return torch.cuda.get_device_capability() >= (10, 0) + + +def is_hopper(): + return torch.cuda.get_device_capability() == (9, 0) diff --git a/sgl-router/.cargo/config.toml b/sgl-router/.cargo/config.toml new file mode 100644 index 000000000..305d57fa3 --- /dev/null +++ b/sgl-router/.cargo/config.toml @@ -0,0 +1,15 @@ +[build] +rustflags = [] +incremental = true + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] diff --git a/sgl-router/.coveragerc b/sgl-router/.coveragerc new file mode 100644 index 000000000..5bab1e8d2 --- /dev/null +++ b/sgl-router/.coveragerc @@ -0,0 +1,9 @@ +[run] +source = py_src/sglang_router +omit = + py_src/sglang_router/mini_lb.py + +[report] +fail_under = 80 +omit = + py_src/sglang_router/mini_lb.py diff --git a/sgl-router/Cargo.toml b/sgl-router/Cargo.toml new file mode 100644 index 000000000..4ecfae55d --- /dev/null +++ b/sgl-router/Cargo.toml @@ -0,0 +1,121 @@ +[package] +name = "sglang_router_rs" +version = "0.0.0" +edition = "2021" + +[features] +default = ["grpc-client"] +grpc-client = [] +grpc-server = [] + +[lib] +name = "sglang_router_rs" +# Pure Rust library: Just omit crate-type (defaults to rlib) +# Python/C binding + Rust library: Use ["cdylib", "rlib"] +crate-type = ["cdylib", "rlib"] + +[[bin]] +name = "sglang-router" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +axum = { version = "0.8.4", features = ["macros", "ws", "tracing"] } +tower = { version = "0.5", features = ["full"] } +tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "limit", "request-id", "util"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +bytes = "1.8.0" +rand = "0.9.2" +reqwest = { version = "0.12.8", features = ["stream", "blocking", "json"] } +futures-util = "0.3" +futures = "0.3" +pyo3 = { version = "0.25.1", features = ["extension-module"] } +dashmap = "6.1.0" +http = "1.1.0" +tokio = { version = "1.42.0", features = ["full"] } +async-trait = "0.1" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "chrono"] } +tracing-log = "0.2" +tracing-appender = "0.2.3" +chrono = "0.4" +kube = { version = "1.1.0", features = ["runtime", "derive"] } +k8s-openapi = { version = "0.25.0", features = ["v1_33"] } +metrics = "0.24.2" +metrics-exporter-prometheus = "0.17.0" +uuid = { version = "1.10", features = ["v4", "serde"] } +thiserror = "2.0.12" +regex = "1.10" +url = "2.5.4" +tokio-stream = { version = "0.1", features = ["sync"] } +anyhow = "1.0" +tokenizers = { version = "0.22.0" } +tiktoken-rs = { version = "0.7.0" } +minijinja = { version = "2.0" } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +hf-hub = { version = "0.4.3", features = ["tokio"] } +rmcp = { version = "0.6.3", features = ["client", "server", + "transport-child-process", + "transport-sse-client-reqwest", + "transport-streamable-http-client-reqwest", + "transport-streamable-http-server", + "transport-streamable-http-server-session", + "reqwest", + "auth"] } +serde_yaml = "0.9" + +# gRPC and Protobuf dependencies +tonic = { version = "0.12", features = ["tls", "gzip", "transport"] } +prost = "0.13" +prost-types = "0.13" +deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } +backoff = { version = "0.4", features = ["tokio"] } +strum = { version = "0.26", features = ["derive"] } +once_cell = "1.21.3" + +[build-dependencies] +tonic-build = "0.12" +prost-build = "0.13" + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +tower = { version = "0.5", features = ["util"] } +http-body-util = "0.1" +portpicker = "0.1" +tempfile = "3.8" +lazy_static = "1.4" + +[[bench]] +name = "request_processing" +harness = false +path = "benches/request_processing.rs" + +[[bench]] +name = "tokenizer_benchmark" +harness = false +path = "benches/tokenizer_benchmark.rs" + +[[bench]] +name = "tool_parser_benchmark" +harness = false +path = "benches/tool_parser_benchmark.rs" + +[profile.release] +lto = "thin" +codegen-units = 1 + +[profile.dev] +opt-level = 0 +debug = true +split-debuginfo = "unpacked" +incremental = true + + +[profile.dev.build-override] +opt-level = 3 +codegen-units = 1 + +[profile.dev-opt] +inherits = "dev" +opt-level = 1 diff --git a/sgl-router/MANIFEST.in b/sgl-router/MANIFEST.in new file mode 100644 index 000000000..4baa6c84f --- /dev/null +++ b/sgl-router/MANIFEST.in @@ -0,0 +1,5 @@ +# Must include: +include Cargo.toml # Rust project configuration +include build.rs # Build script for protobuf generation +recursive-include src *.rs # Rust source files +recursive-include src/proto *.proto # Protobuf definitions diff --git a/sgl-router/Makefile b/sgl-router/Makefile new file mode 100644 index 000000000..feda1ee63 --- /dev/null +++ b/sgl-router/Makefile @@ -0,0 +1,92 @@ +# SGLang Router Makefile +# Provides convenient shortcuts for common development tasks + +.PHONY: help bench bench-quick bench-baseline bench-compare test build clean + +help: ## Show this help message + @echo "SGLang Router Development Commands" + @echo "==================================" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' + @echo "" + +build: ## Build the project in release mode + @echo "Building SGLang Router..." + @cargo build --release + +test: ## Run all tests + @echo "Running tests..." + @cargo test + +bench: ## Run full benchmark suite + @echo "Running full benchmarks..." + @python3 scripts/run_benchmarks.py + +bench-quick: ## Run quick benchmarks only + @echo "Running quick benchmarks..." + @python3 scripts/run_benchmarks.py --quick + +bench-baseline: ## Save current performance as baseline + @echo "Saving performance baseline..." + @python3 scripts/run_benchmarks.py --save-baseline main + +bench-compare: ## Compare with saved baseline + @echo "Comparing with baseline..." + @python3 scripts/run_benchmarks.py --compare-baseline main + +bench-ci: ## Run benchmarks suitable for CI (quick mode) + @echo "Running CI benchmarks..." + @python3 scripts/run_benchmarks.py --quick + +clean: ## Clean build artifacts + @echo "Cleaning build artifacts..." + @cargo clean + +docs: ## Generate and open documentation + @echo "Generating documentation..." + @cargo doc --open + +check: ## Run cargo check and clippy + @echo "Running cargo check..." + @cargo check + @echo "Running clippy..." + @cargo clippy + +fmt: ## Format code with rustfmt + @echo "Formatting code..." + @cargo fmt + +# Development workflow shortcuts +dev-setup: build test ## Set up development environment + @echo "Development environment ready!" + +pre-commit: fmt check test bench-quick ## Run pre-commit checks + @echo "Pre-commit checks passed!" + +# Benchmark analysis shortcuts +bench-report: ## Open benchmark HTML report + @if [ -f "target/criterion/request_processing/report/index.html" ]; then \ + echo "Opening benchmark report..."; \ + if command -v xdg-open >/dev/null 2>&1; then \ + xdg-open target/criterion/request_processing/report/index.html; \ + elif command -v open >/dev/null 2>&1; then \ + open target/criterion/request_processing/report/index.html; \ + else \ + echo "Please open target/criterion/request_processing/report/index.html in your browser"; \ + fi \ + else \ + echo "No benchmark report found. Run 'make bench' first."; \ + fi + +bench-clean: ## Clean benchmark results + @echo "Cleaning benchmark results..." + @rm -rf target/criterion + +# Performance monitoring +perf-monitor: ## Run continuous performance monitoring + @echo "Starting performance monitoring..." + @if command -v watch >/dev/null 2>&1; then \ + watch -n 300 'make bench-quick'; \ + else \ + echo "Warning: 'watch' command not found. Install it or run 'make bench-quick' manually."; \ + fi diff --git a/sgl-router/README.md b/sgl-router/README.md new file mode 100644 index 000000000..af73536b3 --- /dev/null +++ b/sgl-router/README.md @@ -0,0 +1,402 @@ +# SGLang Router + +SGLang router is a standalone Rust module that enables data parallelism across SGLang instances, providing high-performance request routing and advanced load balancing. The router supports multiple load balancing algorithms including cache-aware, power of two, random, and round robin, and acts as a specialized load balancer for prefill-decode disaggregated serving architectures. + +## Documentation + +- **User Guide**: [docs.sglang.ai/advanced_features/router.html](https://docs.sglang.ai/advanced_features/router.html) + +## Quick Start + +### Prerequisites + +**Rust and Cargo:** +```bash +# Install rustup (Rust installer and version manager) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +# Follow the installation prompts, then reload your shell +source $HOME/.cargo/env + +# Verify installation +rustc --version +cargo --version +``` + +**Python with pip installed** + +### Installation + +#### Option A: Build and Install Wheel (Recommended) +```bash +# Install build dependencies +pip install setuptools-rust wheel build + +# Build the wheel package +python -m build + +# Install the generated wheel +pip install dist/*.whl + +# One-liner for development (rebuild + install) +python -m build && pip install --force-reinstall dist/*.whl +``` + +#### Option B: Development Mode + +```bash +# Currently broken +pip install -e . +``` + +⚠️ **Warning**: Editable installs may suffer performance degradation. Use wheel builds for performance testing. + +### Basic Usage + +```bash +# Build Rust components +cargo build +``` + +#### Using the Rust Binary Directly (Alternative to Python) +```bash +# Build the Rust binary +cargo build --release + +# Launch router with worker URLs in regular mode +./target/release/sglang-router \ + --worker-urls http://worker1:8000 http://worker2:8000 + +# Or use cargo run +cargo run --release -- \ + --worker-urls http://worker1:8000 http://worker2:8000 +``` + +#### Launch Router with Python (Original Method) +```bash +# Launch router with worker URLs +python -m sglang_router.launch_router \ + --worker-urls http://worker1:8000 http://worker2:8000 +``` + +#### Launch Router with Worker URLs in prefill-decode mode +```bash +# Note that the prefill and decode URLs must be provided in the following format: +# http://: for decode nodes +# http://: bootstrap-port for prefill nodes, where bootstrap-port is optional + +# Using Rust binary directly +./target/release/sglang-router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --prefill http://127.0.0.3:30003 9003 \ + --prefill http://127.0.0.4:30004 9004 \ + --decode http://127.0.0.5:30005 \ + --decode http://127.0.0.6:30006 \ + --decode http://127.0.0.7:30007 \ + --host 0.0.0.0 \ + --port 8080 + +# Or using Python launcher +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --prefill http://127.0.0.3:30003 9003 \ + --prefill http://127.0.0.4:30004 9004 \ + --decode http://127.0.0.5:30005 \ + --decode http://127.0.0.6:30006 \ + --decode http://127.0.0.7:30007 \ + --host 0.0.0.0 \ + --port 8080 +```` + +## Configuration + +### Logging + +Enable structured logging with optional file output: + +```python +from sglang_router import Router + +# Console logging (default) +router = Router(worker_urls=["http://worker1:8000", "http://worker2:8000"]) + +# File logging enabled +router = Router( + worker_urls=["http://worker1:8000", "http://worker2:8000"], + log_dir="./logs" # Daily log files created here +) +``` + +Set log level with `--log-level` flag ([documentation](https://docs.sglang.ai/backend/server_arguments.html#logging)). + +### Metrics + +Prometheus metrics endpoint available at `127.0.0.1:29000` by default. + +```bash +# Custom metrics configuration +python -m sglang_router.launch_router \ + --worker-urls http://localhost:8080 http://localhost:8081 \ + --prometheus-host 0.0.0.0 \ + --prometheus-port 9000 +``` + +### Retries and Circuit Breakers + +- Retries (regular router) are enabled by default with exponential backoff and jitter. You can tune them via CLI: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://localhost:8080 http://localhost:8081 \ + --retry-max-retries 3 \ + --retry-initial-backoff-ms 100 \ + --retry-max-backoff-ms 10000 \ + --retry-backoff-multiplier 2.0 \ + --retry-jitter-factor 0.1 +``` + +- Circuit Breaker defaults protect workers and auto-recover. Tune thresholds/timeouts: + +```bash +python -m sglang_router.launch_router \ + --worker-urls http://localhost:8080 http://localhost:8081 \ + --cb-failure-threshold 5 \ + --cb-success-threshold 2 \ + --cb-timeout-duration-secs 30 \ + --cb-window-duration-secs 60 +``` + +Behavior summary: +- Closed → Open after N consecutive failures (failure-threshold) +- Open → HalfOpen after timeout (timeout-duration-secs) +- HalfOpen → Closed after M consecutive successes (success-threshold) +- Any failure in HalfOpen reopens immediately + +Retry predicate (regular router): retry on 408/429/500/502/503/504, otherwise return immediately. Backoff/jitter observed between attempts. + +### Request ID Tracking + +Track requests across distributed systems with configurable headers: + +```bash +# Use custom request ID headers +python -m sglang_router.launch_router \ + --worker-urls http://localhost:8080 \ + --request-id-headers x-trace-id x-request-id +``` + +Default headers: `x-request-id`, `x-correlation-id`, `x-trace-id`, `request-id` + +## Advanced Features + +### Kubernetes Service Discovery + +Automatic worker discovery and management in Kubernetes environments. + +#### Basic Service Discovery + +```bash +python -m sglang_router.launch_router \ + --service-discovery \ + --selector app=sglang-worker role=inference \ + --service-discovery-namespace default +``` + +#### PD (Prefill-Decode) Mode + +For disaggregated prefill/decode routing: + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --service-discovery \ + --prefill-selector app=sglang component=prefill \ + --decode-selector app=sglang component=decode \ + --service-discovery-namespace sglang-system + +# With separate routing policies: +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --prefill-policy cache_aware \ + --decode-policy power_of_two \ + --service-discovery \ + --prefill-selector app=sglang component=prefill \ + --decode-selector app=sglang component=decode \ + --service-discovery-namespace sglang-system + +# in lws case, such as tp16(1 leader pod, 1 worker pod) +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --service-discovery \ + --prefill-selector app=sglang component=prefill role=leader\ + --decode-selector app=sglang component=decode role=leader\ + --service-discovery-namespace sglang-system +``` + +#### Kubernetes Pod Configuration + +**Prefill Server Pod:** +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: sglang-prefill-1 + labels: + app: sglang + component: prefill + annotations: + sglang.ai/bootstrap-port: "9001" # Optional: Bootstrap port +spec: + containers: + - name: sglang + image: lmsys/sglang:latest + ports: + - containerPort: 8000 # Main API port + - containerPort: 9001 # Optional: Bootstrap port +``` + +**Decode Server Pod:** +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: sglang-decode-1 + labels: + app: sglang + component: decode +spec: + containers: + - name: sglang + image: lmsys/sglang:latest + ports: + - containerPort: 8000 +``` + +#### RBAC Configuration + +**Namespace-scoped (recommended):** +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sglang-router + namespace: sglang-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: sglang-system + name: sglang-router +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: sglang-router + namespace: sglang-system +subjects: +- kind: ServiceAccount + name: sglang-router + namespace: sglang-system +roleRef: + kind: Role + name: sglang-router + apiGroup: rbac.authorization.k8s.io +``` + +#### Complete PD Example + +```bash +python -m sglang_router.launch_router \ + --pd-disaggregation \ + --policy cache_aware \ + --service-discovery \ + --prefill-selector app=sglang component=prefill environment=production \ + --decode-selector app=sglang component=decode environment=production \ + --service-discovery-namespace production \ + --host 0.0.0.0 \ + --port 8080 \ + --prometheus-host 0.0.0.0 \ + --prometheus-port 9090 +``` + +### Command Line Arguments Reference + +#### Service Discovery +- `--service-discovery`: Enable Kubernetes service discovery +- `--service-discovery-port`: Port for worker URLs (default: 8000) +- `--service-discovery-namespace`: Kubernetes namespace to watch +- `--selector`: Label selectors for regular mode (format: `key1=value1 key2=value2`) + +#### PD Mode +- `--pd-disaggregation`: Enable Prefill-Decode disaggregated mode +- `--prefill`: Initial prefill server (format: `URL BOOTSTRAP_PORT`) +- `--decode`: Initial decode server URL +- `--prefill-selector`: Label selector for prefill pods +- `--decode-selector`: Label selector for decode pods +- `--policy`: Routing policy (`cache_aware`, `random`, `power_of_two`, `round_robin`) +- `--prefill-policy`: Separate routing policy for prefill nodes (optional, overrides `--policy` for prefill) +- `--decode-policy`: Separate routing policy for decode nodes (optional, overrides `--policy` for decode) + +## Development + +### Build Process + +```bash +# Build Rust project +cargo build + +# Build Python binding (see Installation section above) +``` + +**Note**: When modifying Rust code, you must rebuild the wheel for changes to take effect. + +### Troubleshooting + +**VSCode Rust Analyzer Issues:** +Set `rust-analyzer.linkedProjects` to the absolute path of `Cargo.toml`: + +```json +{ + "rust-analyzer.linkedProjects": ["/workspaces/sglang/sgl-router/Cargo.toml"] +} +``` + +### CI/CD Pipeline + +The continuous integration pipeline includes comprehensive testing, benchmarking, and publishing: + +#### Build & Test + +1. **Build Wheels**: Uses `cibuildwheel` for manylinux x86_64 packages +2. **Build Source Distribution**: Creates source distribution for pip fallback +3. **Rust HTTP Server Benchmarking**: Performance testing of router overhead +4. **Basic Inference Testing**: End-to-end validation through the router +5. **PD Disaggregation Testing**: Benchmark and sanity checks for prefill-decode load balancing + +#### Publishing +- **PyPI Publishing**: Wheels and source distributions are published only when the version changes in `pyproject.toml` +- **Container Images**: Docker images published using `/docker/Dockerfile.router` + +## Features +- **High Performance**: Rust-based routing with connection pooling and optimized request handling +- **Advanced Load Balancing**: Multiple algorithms including: + - **Cache-Aware**: Intelligent routing based on cache locality for optimal performance + - **Power of Two**: Chooses the less loaded of two randomly selected workers + - **Random**: Distributes requests randomly across available workers + - **Round Robin**: Sequential distribution across workers in rotation +- **Prefill-Decode Disaggregation**: Specialized load balancing for separated prefill and decode servers +- **Service Discovery**: Automatic Kubernetes worker discovery and health management +- **Monitoring**: Comprehensive Prometheus metrics and structured logging +- **Scalability**: Handles thousands of concurrent connections with efficient resource utilization diff --git a/sgl-router/benches/request_processing.rs b/sgl-router/benches/request_processing.rs new file mode 100644 index 000000000..eba5680aa --- /dev/null +++ b/sgl-router/benches/request_processing.rs @@ -0,0 +1,692 @@ +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use serde_json::{from_str, to_string, to_value, to_vec}; +use std::time::Instant; + +use sglang_router_rs::core::{BasicWorker, Worker, WorkerType}; +use sglang_router_rs::protocols::spec::{ + ChatCompletionRequest, ChatMessage, CompletionRequest, GenerateParameters, GenerateRequest, + SamplingParams, StringOrArray, UserMessageContent, +}; +use sglang_router_rs::routers::http::pd_types::{ + generate_room_id, get_hostname, RequestWithBootstrap, +}; + +fn create_test_worker() -> BasicWorker { + BasicWorker::new( + "http://test-server:8000".to_string(), + WorkerType::Prefill { + bootstrap_port: Some(5678), + }, + ) +} + +// Helper function to get bootstrap info from worker +fn get_bootstrap_info(worker: &BasicWorker) -> (String, Option) { + let hostname = get_hostname(worker.url()); + let bootstrap_port = match worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + (hostname, bootstrap_port) +} + +/// Create a default GenerateRequest for benchmarks with minimal fields set +fn default_generate_request() -> GenerateRequest { + GenerateRequest { + text: None, + prompt: None, + input_ids: None, + stream: false, + parameters: None, + sampling_params: None, + return_logprob: false, + // SGLang Extensions + lora_path: None, + session_params: None, + return_hidden_states: false, + rid: None, + } +} + +/// Create a default ChatCompletionRequest for benchmarks with minimal fields set +fn default_chat_completion_request() -> ChatCompletionRequest { + ChatCompletionRequest { + model: String::new(), + messages: vec![], + max_tokens: None, + max_completion_tokens: None, + temperature: None, + top_p: None, + n: None, + stream: false, + stream_options: None, + stop: None, + presence_penalty: None, + frequency_penalty: None, + logit_bias: None, + logprobs: false, + top_logprobs: None, + user: None, + response_format: None, + seed: None, + tools: None, + tool_choice: None, + parallel_tool_calls: None, + function_call: None, + functions: None, + // SGLang Extensions + top_k: None, + min_p: None, + min_tokens: None, + repetition_penalty: None, + regex: None, + ebnf: None, + stop_token_ids: None, + no_stop_trim: false, + ignore_eos: false, + continue_final_message: false, + skip_special_tokens: true, + // SGLang Extensions + lora_path: None, + session_params: None, + separate_reasoning: true, + stream_reasoning: true, + chat_template_kwargs: None, + return_hidden_states: false, + } +} + +/// Create a default CompletionRequest for benchmarks with minimal fields set +fn default_completion_request() -> CompletionRequest { + CompletionRequest { + model: String::new(), + prompt: StringOrArray::String(String::new()), + suffix: None, + max_tokens: None, + temperature: None, + top_p: None, + n: None, + stream: false, + stream_options: None, + logprobs: None, + echo: false, + stop: None, + presence_penalty: None, + frequency_penalty: None, + best_of: None, + logit_bias: None, + user: None, + seed: None, + // SGLang Extensions + top_k: None, + min_p: None, + min_tokens: None, + repetition_penalty: None, + regex: None, + ebnf: None, + json_schema: None, + stop_token_ids: None, + no_stop_trim: false, + ignore_eos: false, + skip_special_tokens: true, + // SGLang Extensions + lora_path: None, + session_params: None, + return_hidden_states: false, + other: serde_json::Map::new(), + } +} + +// Sample request data for benchmarks +fn create_sample_generate_request() -> GenerateRequest { + GenerateRequest { + text: Some("Write a story about artificial intelligence".to_string()), + parameters: Some(GenerateParameters { + max_new_tokens: Some(100), + temperature: Some(0.8), + top_p: Some(0.9), + top_k: Some(50), + repetition_penalty: Some(1.0), + ..Default::default() + }), + sampling_params: Some(SamplingParams { + temperature: Some(0.8), + top_p: Some(0.9), + top_k: Some(50), + frequency_penalty: Some(0.0), + presence_penalty: Some(0.0), + repetition_penalty: Some(1.0), + ..Default::default() + }), + ..default_generate_request() + } +} + +fn create_sample_chat_completion_request() -> ChatCompletionRequest { + ChatCompletionRequest { + model: "gpt-3.5-turbo".to_string(), + messages: vec![ + ChatMessage::System { + role: "system".to_string(), + content: "You are a helpful assistant".to_string(), + name: None, + }, + ChatMessage::User { + role: "user".to_string(), + content: UserMessageContent::Text( + "Explain quantum computing in simple terms".to_string(), + ), + name: None, + }, + ], + max_tokens: Some(150), + max_completion_tokens: Some(150), + temperature: Some(0.7), + top_p: Some(1.0), + n: Some(1), + presence_penalty: Some(0.0), + frequency_penalty: Some(0.0), + parallel_tool_calls: Some(true), + ..default_chat_completion_request() + } +} + +fn create_sample_completion_request() -> CompletionRequest { + CompletionRequest { + model: "text-davinci-003".to_string(), + prompt: StringOrArray::String("Complete this sentence: The future of AI is".to_string()), + max_tokens: Some(50), + temperature: Some(0.8), + top_p: Some(1.0), + n: Some(1), + presence_penalty: Some(0.0), + frequency_penalty: Some(0.0), + best_of: Some(1), + ..default_completion_request() + } +} + +fn create_large_chat_completion_request() -> ChatCompletionRequest { + let mut messages = vec![ChatMessage::System { + role: "system".to_string(), + content: "You are a helpful assistant with extensive knowledge.".to_string(), + name: None, + }]; + + // Add many user/assistant pairs to simulate a long conversation + for i in 0..50 { + messages.push(ChatMessage::User { + role: "user".to_string(), + content: UserMessageContent::Text(format!("Question {}: What do you think about topic number {} which involves complex reasoning about multiple interconnected systems and their relationships?", i, i)), + name: None, + }); + messages.push(ChatMessage::Assistant { + role: "assistant".to_string(), + content: Some(format!("Answer {}: This is a detailed response about topic {} that covers multiple aspects and provides comprehensive analysis of the interconnected systems you mentioned.", i, i)), + name: None, + tool_calls: None, + function_call: None, + reasoning_content: None, + }); + } + + ChatCompletionRequest { + model: "gpt-4".to_string(), + messages, + max_tokens: Some(1000), + max_completion_tokens: Some(1000), + temperature: Some(0.7), + top_p: Some(0.95), + n: Some(1), + presence_penalty: Some(0.1), + frequency_penalty: Some(0.1), + top_logprobs: Some(5), + user: Some("benchmark_user".to_string()), + seed: Some(42), + parallel_tool_calls: Some(true), + ..default_chat_completion_request() + } +} + +// Benchmark JSON serialization +fn bench_json_serialization(c: &mut Criterion) { + let mut group = c.benchmark_group("json_serialization"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + let large_chat_req = create_large_chat_completion_request(); + + group.bench_function("generate_request", |b| { + b.iter(|| { + let json = to_string(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&completion_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("large_chat_completion_request", |b| { + b.iter(|| { + let json = to_string(black_box(&large_chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_request_to_bytes", |b| { + b.iter(|| { + let bytes = to_vec(black_box(&generate_req)).unwrap(); + black_box(bytes); + }); + }); + + group.finish(); +} + +// Benchmark JSON deserialization +fn bench_json_deserialization(c: &mut Criterion) { + let mut group = c.benchmark_group("json_deserialization"); + + let generate_json = to_string(&create_sample_generate_request()).unwrap(); + let chat_json = to_string(&create_sample_chat_completion_request()).unwrap(); + let completion_json = to_string(&create_sample_completion_request()).unwrap(); + let large_chat_json = to_string(&create_large_chat_completion_request()).unwrap(); + + group.bench_function("generate_request", |b| { + b.iter(|| { + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("chat_completion_request", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("completion_request", |b| { + b.iter(|| { + let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap(); + black_box(req); + }); + }); + + group.bench_function("large_chat_completion_request", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&large_chat_json)).unwrap(); + black_box(req); + }); + }); + + group.finish(); +} + +// Benchmark bootstrap injection (replaces request adaptation) +fn bench_bootstrap_injection(c: &mut Criterion) { + let mut group = c.benchmark_group("bootstrap_injection"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + let large_chat_req = create_large_chat_completion_request(); + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + group.bench_function("generate_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &generate_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &chat_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &completion_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("large_chat_completion_bootstrap_injection", |b| { + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: &large_chat_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(black_box(&request_with_bootstrap)).unwrap(); + black_box(json); + }); + }); + + group.finish(); +} + +// Benchmark direct JSON routing (replaces regular routing) +fn bench_direct_json_routing(c: &mut Criterion) { + let mut group = c.benchmark_group("direct_json_routing"); + + let generate_req = create_sample_generate_request(); + let chat_req = create_sample_chat_completion_request(); + let completion_req = create_sample_completion_request(); + + group.bench_function("generate_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_to_json_string", |b| { + b.iter(|| { + let json = to_string(black_box(&generate_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("generate_to_bytes", |b| { + b.iter(|| { + let bytes = to_vec(black_box(&generate_req)).unwrap(); + black_box(bytes); + }); + }); + + group.bench_function("chat_completion_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("chat_completion_to_json_string", |b| { + b.iter(|| { + let json = to_string(black_box(&chat_req)).unwrap(); + black_box(json); + }); + }); + + group.bench_function("completion_to_json", |b| { + b.iter(|| { + let json = to_value(black_box(&completion_req)).unwrap(); + black_box(json); + }); + }); + + group.finish(); +} + +// Benchmark throughput with different request sizes +fn bench_throughput_by_size(c: &mut Criterion) { + let mut group = c.benchmark_group("throughput_by_size"); + + // Create requests of different sizes + let small_generate = GenerateRequest { + text: Some("Hi".to_string()), + ..default_generate_request() + }; + + let medium_generate = GenerateRequest { + text: Some("Write a medium length story about AI".repeat(10)), + ..default_generate_request() + }; + + let large_generate = GenerateRequest { + text: Some("Write a very long and detailed story about artificial intelligence and its impact on society".repeat(100)), + ..default_generate_request() + }; + + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + for (name, req) in [ + ("small", &small_generate), + ("medium", &medium_generate), + ("large", &large_generate), + ] { + let json = to_string(req).unwrap(); + let size_bytes = json.len(); + let hostname_clone = hostname.clone(); + + group.throughput(Throughput::Bytes(size_bytes as u64)); + group.bench_with_input(BenchmarkId::new("serialize", name), &req, |b, req| { + b.iter(|| { + let json = to_string(black_box(req)).unwrap(); + black_box(json); + }); + }); + + group.bench_with_input( + BenchmarkId::new("deserialize", name), + &json, + |b, json_str| { + b.iter(|| { + let req: GenerateRequest = black_box(from_str(json_str)).unwrap(); + black_box(req); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("bootstrap_inject", name), + &req, + move |b, req| { + let hostname = hostname_clone.clone(); + b.iter(|| { + let request_with_bootstrap = RequestWithBootstrap { + original: req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let json = to_value(&request_with_bootstrap).unwrap(); + black_box(json); + }); + }, + ); + } + + group.finish(); +} + +// Benchmark full round-trip: deserialize -> inject bootstrap -> serialize +fn bench_full_round_trip(c: &mut Criterion) { + let mut group = c.benchmark_group("full_round_trip"); + + let generate_json = to_string(&create_sample_generate_request()).unwrap(); + let chat_json = to_string(&create_sample_chat_completion_request()).unwrap(); + let completion_json = to_string(&create_sample_completion_request()).unwrap(); + let worker = create_test_worker(); + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + + group.bench_function("generate_openai_to_pd_pipeline", |b| { + b.iter(|| { + // Deserialize OpenAI request + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + // Create wrapper with bootstrap fields + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + // Serialize final request + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("chat_completion_openai_to_pd_pipeline", |b| { + b.iter(|| { + let req: ChatCompletionRequest = from_str(black_box(&chat_json)).unwrap(); + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("completion_openai_to_pd_pipeline", |b| { + b.iter(|| { + let req: CompletionRequest = from_str(black_box(&completion_json)).unwrap(); + let request_with_bootstrap = RequestWithBootstrap { + original: &req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let pd_json = to_string(&request_with_bootstrap).unwrap(); + black_box(pd_json); + }); + }); + + group.bench_function("generate_direct_json_pipeline", |b| { + b.iter(|| { + // Deserialize OpenAI request + let req: GenerateRequest = from_str(black_box(&generate_json)).unwrap(); + // Convert to JSON for direct routing (no bootstrap injection) + let routing_json = to_value(&req).unwrap(); + let json_string = to_string(&routing_json).unwrap(); + black_box(json_string); + }); + }); + + group.finish(); +} + +fn benchmark_summary(c: &mut Criterion) { + let group = c.benchmark_group("benchmark_summary"); + + println!("\nSGLang Router Performance Benchmark Suite"); + println!("============================================="); + + // Quick performance overview + let generate_req = create_sample_generate_request(); + let worker = create_test_worker(); + + println!("\nQuick Performance Overview:"); + + // Measure serialization + let start = Instant::now(); + for _ in 0..1000 { + let _ = black_box(to_string(&generate_req).unwrap()); + } + let serialize_time = start.elapsed().as_nanos() / 1000; + println!(" * Serialization (avg): {:>8} ns/req", serialize_time); + + // Measure deserialization + let json = to_string(&generate_req).unwrap(); + let start = Instant::now(); + for _ in 0..1000 { + let _: GenerateRequest = black_box(from_str(&json).unwrap()); + } + let deserialize_time = start.elapsed().as_nanos() / 1000; + println!( + " * Deserialization (avg): {:>8} ns/req", + deserialize_time + ); + + // Measure bootstrap injection (replaces adaptation) + let (hostname, bootstrap_port) = get_bootstrap_info(&worker); + let start = Instant::now(); + for _ in 0..1000 { + let request_with_bootstrap = RequestWithBootstrap { + original: &generate_req, + bootstrap_host: hostname.clone(), + bootstrap_port, + bootstrap_room: generate_room_id(), + }; + let _ = black_box(to_value(&request_with_bootstrap).unwrap()); + } + let inject_time = start.elapsed().as_nanos() / 1000; + println!(" * Bootstrap Injection (avg): {:>6} ns/req", inject_time); + + // Calculate ratios + let total_pipeline = serialize_time + deserialize_time + inject_time; + println!(" * Total Pipeline (avg): {:>8} ns/req", total_pipeline); + + println!("\nPerformance Insights:"); + if deserialize_time > serialize_time * 2 { + println!(" • Deserialization is significantly faster than serialization"); + } + if inject_time < serialize_time / 10 { + println!( + " • Bootstrap injection overhead is negligible ({:.1}% of serialization)", + (inject_time as f64 / serialize_time as f64) * 100.0 + ); + } + if total_pipeline < 100_000 { + println!(" • Total pipeline latency is excellent (< 100μs)"); + } + + println!("\nSimplification Benefits:"); + println!(" • Eliminated complex type conversion layer"); + println!(" • Reduced memory allocations"); + println!(" • Automatic field preservation (no manual mapping)"); + println!(" • Direct JSON manipulation improves performance"); + + println!("\nRecommendations:"); + if serialize_time > deserialize_time { + println!(" • Focus optimization efforts on serialization rather than deserialization"); + } + println!(" • PD mode overhead is minimal - safe to use for latency-sensitive workloads"); + println!(" • Consider batching small requests to improve overall throughput"); + + println!("\n{}", "=".repeat(50)); + + group.finish(); +} + +criterion_group!( + benches, + benchmark_summary, + bench_json_serialization, + bench_json_deserialization, + bench_bootstrap_injection, + bench_direct_json_routing, + bench_throughput_by_size, + bench_full_round_trip +); +criterion_main!(benches); diff --git a/sgl-router/benches/tokenizer_benchmark.rs b/sgl-router/benches/tokenizer_benchmark.rs new file mode 100644 index 000000000..a40abcc4e --- /dev/null +++ b/sgl-router/benches/tokenizer_benchmark.rs @@ -0,0 +1,1400 @@ +//! Comprehensive tokenizer benchmark with clean summary output +//! Each test adds a row to the final summary table + +use criterion::{black_box, criterion_group, BenchmarkId, Criterion, Throughput}; +use sglang_router_rs::tokenizer::{ + huggingface::HuggingFaceTokenizer, sequence::Sequence, stop::*, stream::DecodeStream, traits::*, +}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; + +// Include the common test utilities +#[path = "../tests/common/mod.rs"] +mod common; +use common::ensure_tokenizer_cached; + +// Cache the tokenizer path for the entire benchmark run +static TOKENIZER_PATH: OnceLock = OnceLock::new(); + +fn get_tokenizer_path() -> &'static PathBuf { + TOKENIZER_PATH.get_or_init(ensure_tokenizer_cached) +} + +// Production target: 100k tokens per second +const TARGET_TOKENS_PER_SECOND: u64 = 100_000; + +// Typical prompt sizes +const SHORT_PROMPT: &str = "What is the capital of France?"; +const MEDIUM_PROMPT: &str = "Write a detailed explanation of quantum computing, including its principles, current applications, and future potential. Be sure to cover both the theoretical foundations and practical implementations."; +const LONG_PROMPT: &str = "You are an expert software engineer. Review the following code and provide detailed feedback on performance optimizations, potential bugs, and architectural improvements. Consider scalability, maintainability, and best practices. The code implements a distributed caching system with the following requirements: 1) High availability across multiple regions, 2) Sub-millisecond latency for cache hits, 3) Automatic failover and recovery, 4) Support for both LRU and LFU eviction policies, 5) Real-time monitoring and alerting. Please analyze each component thoroughly and suggest concrete improvements with code examples where appropriate."; + +// System prompts can be quite large +fn generate_system_prompt(size: usize) -> String { + let base = "You are a helpful AI assistant with expertise in "; + let domains = vec![ + "mathematics", + "physics", + "chemistry", + "biology", + "computer science", + "engineering", + "medicine", + "law", + "economics", + "philosophy", + ]; + + let mut prompt = base.to_string(); + while prompt.len() < size { + for domain in &domains { + prompt.push_str(domain); + prompt.push_str(", "); + if prompt.len() >= size { + break; + } + } + } + prompt +} + +// Global results storage +lazy_static::lazy_static! { + static ref BENCHMARK_RESULTS: Mutex> = Mutex::new(BTreeMap::new()); +} + +fn add_result(category: &str, result: String) { + let mut results = BENCHMARK_RESULTS.lock().unwrap(); + let index = results.len(); + results.insert(format!("{:03}_{}", index, category), result); +} + +fn bench_encode_throughput(c: &mut Criterion) { + let tokenizer_path = get_tokenizer_path(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + // Pre-generate system prompts + let system_1k = generate_system_prompt(1000); + let system_4k = generate_system_prompt(4000); + let system_16k = generate_system_prompt(16000); + + let test_cases = vec![ + ("short_30B", SHORT_PROMPT), + ("medium_230B", MEDIUM_PROMPT), + ("long_670B", LONG_PROMPT), + ("system_1KB", system_1k.as_str()), + ("system_4KB", system_4k.as_str()), + ("system_16KB", system_16k.as_str()), + ]; + + let mut group = c.benchmark_group("encode_throughput"); + + for (name, prompt) in test_cases { + let prompt_len = prompt.len(); + let tokenizer_clone = tokenizer.clone(); + + // Get token count once + let encoding = tokenizer.encode(prompt).unwrap(); + let token_count = encoding.token_ids().len(); + + // Track if metrics have been printed for this test case + let printed = Arc::new(AtomicBool::new(false)); + + group.throughput(Throughput::Bytes(prompt_len as u64)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode(prompt).unwrap()); + } + let duration = start.elapsed(); + + // Store result only once per test case + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let chars_per_sec = (iters as f64 * prompt_len as f64) / duration.as_secs_f64(); + let tokens_per_sec = + (iters as f64 * token_count as f64) / duration.as_secs_f64(); + + let result = format!( + "{:<15} | {:>8} | {:>8} | {:>12.0} | {:>12.0} | {:>10.0} | {:>10}", + name, + prompt_len, + token_count, + chars_per_sec, + tokens_per_sec, + ops_per_sec, + 1 + ); + add_result("encode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_batch_encode(c: &mut Criterion) { + let tokenizer_path = get_tokenizer_path(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let batch_sizes = vec![1, 8, 16, 32, 64, 128]; + let prompt = MEDIUM_PROMPT; + let prompt_len = prompt.len(); + let encoding = tokenizer.encode(prompt).unwrap(); + let token_count = encoding.token_ids().len(); + + let mut group = c.benchmark_group("batch_encode"); + + for batch_size in batch_sizes { + let prompts: Vec<&str> = vec![prompt; batch_size]; + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.throughput(Throughput::Elements(batch_size as u64)); + group.bench_with_input( + BenchmarkId::from_parameter(batch_size), + &batch_size, + |b, &size| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode_batch(&prompts).unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let prompts_per_sec = (iters as f64 * size as f64) / duration.as_secs_f64(); + let tokens_per_sec = prompts_per_sec * token_count as f64; + let chars_per_sec = prompts_per_sec * prompt_len as f64; + + let result = format!( + "{:<15} | {:>8} | {:>8} | {:>12.0} | {:>12.0} | {:>10.0} | {:>10}", + format!("batch_{}", size), + prompt_len * size, + token_count * size, + prompts_per_sec, + tokens_per_sec, + chars_per_sec, + 1 + ); + add_result("batch", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_concurrent_encode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let client_counts = vec![1, 4, 8, 16, 32]; + + let mut group = c.benchmark_group("concurrent_encode"); + group.measurement_time(Duration::from_secs(2)); + + for num_clients in client_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_clients), + &num_clients, + |b, &clients| { + let printed_clone = printed.clone(); + + b.iter_custom(|_iters| { + let tokenizer = tokenizer_clone.clone(); + let total_operations = Arc::new(AtomicU64::new(0)); + let total_chars = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..clients) + .map(|client_id| { + let tokenizer = tokenizer.clone(); + let total_ops = total_operations.clone(); + let total_ch = total_chars.clone(); + + thread::spawn(move || { + let prompts = [SHORT_PROMPT, MEDIUM_PROMPT, LONG_PROMPT]; + let prompt = prompts[client_id % prompts.len()]; + let mut local_ops = 0u64; + let mut local_chars = 0u64; + + while start.elapsed() < Duration::from_millis(500) { + let _ = tokenizer.encode(prompt).unwrap(); + local_ops += 1; + local_chars += prompt.len() as u64; + } + + total_ops.fetch_add(local_ops, Ordering::Relaxed); + total_ch.fetch_add(local_chars, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_ch = total_chars.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let chars_per_sec = total_ch as f64 / duration.as_secs_f64(); + let per_client = ops_per_sec / clients as f64; + + let result = format!( + "{:<15} | {:>10} | {:>12.0} | {:>12.0} | {:>15.0}", + format!("{}_clients", clients), + total_ops, + ops_per_sec, + chars_per_sec, + per_client + ); + add_result("concurrent", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_decode_performance(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let test_text = "The quick brown fox jumps over the lazy dog. ".repeat(10); + let encoding = tokenizer.encode(&test_text).unwrap(); + let tokens = encoding.token_ids(); + let num_tokens = tokens.len(); + + let mut group = c.benchmark_group("decode_performance"); + + // Test direct decode + let printed_direct = Arc::new(AtomicBool::new(false)); + group.bench_function("direct_decode", |b| { + let printed = printed_direct.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.decode(tokens, false).unwrap()); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "Direct", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test DecodeStream + let printed_stream = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_stream", |b| { + let printed = printed_stream.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + let mut output = String::new(); + for token in tokens { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + } + black_box(output); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "DecodeStream", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test Sequence + let printed_seq = Arc::new(AtomicBool::new(false)); + group.bench_function("sequence_decode", |b| { + let printed = printed_seq.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let mut sequence = Sequence::new(tokenizer.clone()); + let mut output = String::new(); + for token in tokens { + let text = sequence.append_token(*token).unwrap(); + output.push_str(&text); + } + black_box(output); + } + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let tokens_per_sec = ops_per_sec * num_tokens as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + "Sequence", num_tokens, tokens_per_sec, ops_per_sec, 1 + ); + add_result("decode", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_streaming_decode_100k(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let sample_text = "The quick brown fox jumps over the lazy dog. ".repeat(1000); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let all_tokens = encoding.token_ids(); + + let mut group = c.benchmark_group("streaming_100k"); + group.measurement_time(Duration::from_secs(1)); + + // Test DecodeStream + let printed_stream = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_stream_100k", |b| { + let printed = printed_stream.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let start = Instant::now(); + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + let mut output = String::new(); + let mut tokens_processed = 0u64; + + for token in all_tokens.iter().cycle() { + if start.elapsed() >= Duration::from_millis(500) { + break; + } + + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + tokens_processed += 1; + } + + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let tokens_per_sec = tokens_processed as f64 / duration.as_secs_f64(); + let status = if tokens_per_sec >= TARGET_TOKENS_PER_SECOND as f64 { + "PASS" + } else { + "BELOW" + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>12} | {:>10} | {:>12}", + "DecodeStream", + tokens_processed, + tokens_per_sec, + TARGET_TOKENS_PER_SECOND, + 1, + status + ); + add_result("streaming_100k", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Test Sequence + let printed_seq = Arc::new(AtomicBool::new(false)); + group.bench_function("sequence_100k", |b| { + let printed = printed_seq.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let start = Instant::now(); + let mut sequence = Sequence::new(tokenizer.clone()); + let mut output = String::new(); + let mut tokens_processed = 0u64; + + for token in all_tokens.iter().cycle() { + if start.elapsed() >= Duration::from_millis(500) { + break; + } + + let text = sequence.append_token(*token).unwrap(); + output.push_str(&text); + tokens_processed += 1; + } + + let duration = start.elapsed(); + + if !printed.load(Ordering::Relaxed) { + let tokens_per_sec = tokens_processed as f64 / duration.as_secs_f64(); + let status = if tokens_per_sec >= TARGET_TOKENS_PER_SECOND as f64 { + "PASS" + } else { + "BELOW" + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>12} | {:>10} | {:>12}", + "Sequence", + tokens_processed, + tokens_per_sec, + TARGET_TOKENS_PER_SECOND, + 1, + status + ); + add_result("streaming_100k", result); + + printed.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_latency_distribution(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + // Test latency for individual token processing + let sample_tokens = vec![1, 450, 6635, 3290, 491, 278, 3474, 29892]; + + let mut group = c.benchmark_group("latency"); + + // Encode latency + let system_4k = generate_system_prompt(4000); + let test_cases = vec![ + ("encode_short", SHORT_PROMPT), + ("encode_medium", MEDIUM_PROMPT), + ("encode_long", LONG_PROMPT), + ("encode_4KB", system_4k.as_str()), + ]; + + for (name, prompt) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + // Only collect detailed latency on first iteration + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + + // Warm up + for _ in 0..100 { + let _ = tokenizer.encode(prompt).unwrap(); + } + + // Measure for statistics + for _ in 0..1000 { + let start = Instant::now(); + let _ = tokenizer.encode(prompt).unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<20} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + name, + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + printed_clone.store(true, Ordering::Relaxed); + + // Return median for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + black_box(tokenizer.encode(prompt).unwrap()); + } + start.elapsed() + }; + + total_duration + }); + }); + } + + // Decode token latency + let printed_decode = Arc::new(AtomicBool::new(false)); + group.bench_function("decode_token", |b| { + let printed_clone = printed_decode.clone(); + let tokenizer = tokenizer.clone(); + let tokens = sample_tokens.clone(); + + b.iter_custom(|iters| { + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + + for token in tokens.iter().cycle().take(1000) { + let start = Instant::now(); + let _ = decoder.step(*token).unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<20} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + "decode_token", + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + // Check target latency + let target_latency = Duration::from_micros(10); + if p50 > target_latency { + let warning = format!( + "WARNING: P50 latency exceeds target of {:?} for 100k tokens/sec", + target_latency + ); + add_result("latency_warning", warning); + } + + printed_clone.store(true, Ordering::Relaxed); + + // Return approximate time for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + for token in tokens.iter().take(10) { + black_box(decoder.step(*token).unwrap()); + } + } + start.elapsed() + }; + + total_duration + }); + }); + + group.finish(); +} + +fn bench_concurrent_streaming(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let num_sequences = 16; + let tokens_per_sequence = 10_000; + + let sample_text = "The quick brown fox jumps over the lazy dog. ".repeat(100); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let token_batch: Vec = encoding.token_ids().to_vec(); + + let mut group = c.benchmark_group("concurrent_streaming"); + group.measurement_time(Duration::from_secs(2)); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("concurrent_16_sequences", |b| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + let tokens = token_batch.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..num_sequences) + .map(|_seq_id| { + let tokenizer = tokenizer.clone(); + let tokens = tokens.clone(); + let total_tokens = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let mut output = String::new(); + let mut local_count = 0u64; + + for token in tokens.iter().cycle().take(tokens_per_sequence) { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + local_count += 1; + } + + total_tokens.fetch_add(local_count, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let throughput = total as f64 / duration.as_secs_f64(); + let per_seq = throughput / num_sequences as f64; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>15.0} | {:>15}", + format!("{}_sequences", num_sequences), + total, + throughput, + per_seq, + num_sequences + ); + add_result("concurrent_streaming", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_stop_sequences(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let config = StopSequenceConfig::default() + .with_stop_sequence("") + .with_stop_sequence("\n\n") + .with_stop_sequence("###") + .with_stop_token(2); + + let sample_text = "Hello world! This is a test. ### Stop here. Continue after.".repeat(100); + let encoding = tokenizer.encode(&sample_text).unwrap(); + let tokens = encoding.token_ids(); + + let mut group = c.benchmark_group("stop_sequences"); + + // No stops + let printed_no_stop = Arc::new(AtomicBool::new(false)); + group.bench_function("no_stops", |b| { + let printed_clone = printed_no_stop.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + let mut total_tokens = 0u64; + + for _ in 0..iters { + let mut decoder = StopSequenceDecoder::new( + tokenizer.clone(), + StopSequenceConfig::default(), + false, + ); + for token in tokens { + let _ = decoder.process_token(*token).unwrap(); + total_tokens += 1; + } + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let tokens_per_sec = total_tokens as f64 / duration.as_secs_f64(); + let seq_per_sec = iters as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<20} | {:>10} | {:>12} | {:>12.0} | {:>10.0}", + "No stops", iters, total_tokens, tokens_per_sec, seq_per_sec + ); + add_result("stop_sequences", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // With stops + let printed_with_stops = Arc::new(AtomicBool::new(false)); + group.bench_function("with_stops", |b| { + let printed_clone = printed_with_stops.clone(); + let tokenizer = tokenizer.clone(); + let config = config.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + let mut total_tokens = 0u64; + let mut total_sequences = 0u64; + + for _ in 0..iters { + let mut decoder = + StopSequenceDecoder::new(tokenizer.clone(), config.clone(), false); + let mut sequence_tokens = 0u64; + + for token in tokens { + let result = decoder.process_token(*token).unwrap(); + sequence_tokens += 1; + + if matches!( + result, + SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_) + ) { + break; + } + } + + total_tokens += sequence_tokens; + total_sequences += 1; + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let tokens_per_sec = total_tokens as f64 / duration.as_secs_f64(); + let seq_per_sec = total_sequences as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<20} | {:>10} | {:>12} | {:>12.0} | {:>10.0}", + "With stops", total_sequences, total_tokens, tokens_per_sec, seq_per_sec + ); + add_result("stop_sequences", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_multithreaded_encode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let operations_per_thread = 1000; + + // Test with medium-sized prompt for balanced workload + let test_prompt = MEDIUM_PROMPT; + + let mut group = c.benchmark_group("multithreaded_encode"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + + b.iter_custom(|_iters| { + let total_operations = Arc::new(AtomicU64::new(0)); + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let total_ops = total_operations.clone(); + let total_tok = total_tokens.clone(); + + thread::spawn(move || { + for _ in 0..operations_per_thread { + let encoding = tokenizer.encode(test_prompt).unwrap(); + total_tok.fetch_add( + encoding.token_ids().len() as u64, + Ordering::Relaxed, + ); + } + total_ops + .fetch_add(operations_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_tok = total_tokens.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let tokens_per_sec = total_tok as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = tokens_per_sec; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (tokens_per_sec / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<20} | {:>10} | {:>12.0} | {:>12.0} | {:>10} | {:>11.1}%", + format!("encode_{}_threads", threads), + total_ops, + ops_per_sec, + tokens_per_sec, + threads, + efficiency + ); + add_result("mt_encode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_multithreaded_decode(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let tokens_per_thread = 5000; + + // Generate tokens for decoding + let test_text = "The quick brown fox jumps over the lazy dog. ".repeat(100); + let encoding = tokenizer.encode(&test_text).unwrap(); + let test_tokens: Vec = encoding.token_ids().to_vec(); + + let mut group = c.benchmark_group("multithreaded_decode"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let tokenizer_clone = tokenizer.clone(); + let tokens = test_tokens.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer_clone.clone(); + let tokens = tokens.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let tokens = tokens.clone(); + let total_tok = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let mut output = String::new(); + let mut local_tokens = 0u64; + + for token in tokens.iter().cycle().take(tokens_per_thread) { + if let Some(text) = decoder.step(*token).unwrap() { + output.push_str(&text); + } + local_tokens += 1; + } + + total_tok.fetch_add(local_tokens, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let tokens_per_sec = total as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = tokens_per_sec; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (tokens_per_sec / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<20} | {:>12} | {:>12.0} | {:>10} | {:>11.1}%", + format!("decode_{}_threads", threads), + total, + tokens_per_sec, + threads, + efficiency + ); + add_result("mt_decode", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_memory_efficiency(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let large_text = "The quick brown fox jumps over the lazy dog. ".repeat(1000); + let encoding = tokenizer.encode(&large_text).unwrap(); + + let mut group = c.benchmark_group("memory"); + + // Track owned baseline time + let mut owned_time_ns = 0.0; + + // Owned + let printed_owned = Arc::new(AtomicBool::new(false)); + group.bench_function("token_ids_owned", |b| { + let printed_clone = printed_owned.clone(); + let encoding = encoding.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let _ = black_box(encoding.token_ids()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_call = duration.as_nanos() as f64 / iters as f64; + owned_time_ns = time_per_call; + + let result = format!( + "{:<20} | {:>12.0} | {:>11.0}ns | {:>12}", + "token_ids(owned)", ops_per_sec, time_per_call, "baseline" + ); + add_result("memory", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Reference + let printed_ref = Arc::new(AtomicBool::new(false)); + + group.bench_function("token_ids_ref", |b| { + let printed_clone = printed_ref.clone(); + let encoding = encoding.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let _ = black_box(encoding.token_ids()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_call = duration.as_nanos() as f64 / iters as f64; + + // Calculate improvement + let improvement = if owned_time_ns > 0.0 { + format!("{:.1}x faster", owned_time_ns / time_per_call) + } else { + "N/A".to_string() + }; + + let result = format!( + "{:<20} | {:>12.0} | {:>11.0}ns | {:>12}", + "token_ids_ref(ref)", ops_per_sec, time_per_call, improvement + ); + add_result("memory", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_scaling_characteristics(c: &mut Criterion) { + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(get_tokenizer_path().to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let thread_counts = vec![1, 2, 4, 8, 16]; + let tokens_per_thread = 10000; + + let mut group = c.benchmark_group("scaling"); + group.measurement_time(Duration::from_secs(2)); + + let mut baseline_throughput = 0.0; + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let tokenizer = tokenizer.clone(); + + b.iter_custom(|_iters| { + let total_tokens = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_| { + let tokenizer = tokenizer.clone(); + let total_tokens = total_tokens.clone(); + + thread::spawn(move || { + let mut decoder = DecodeStream::new(tokenizer, &[], false); + let sample_tokens = [1, 450, 6635, 3290, 491]; + + for token in sample_tokens.iter().cycle().take(tokens_per_thread) { + let _ = decoder.step(*token).unwrap(); + } + + total_tokens.fetch_add(tokens_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total = total_tokens.load(Ordering::Relaxed); + let throughput = total as f64 / duration.as_secs_f64(); + + if threads == 1 { + baseline_throughput = throughput; + } + + let efficiency = if threads == 1 { + 100.0 + } else { + (throughput / (baseline_throughput * threads as f64)) * 100.0 + }; + + let result = format!( + "{:<15} | {:>12} | {:>12.0} | {:>11.1}%", + format!("{}_threads", threads), + total, + throughput, + efficiency + ); + add_result("scaling", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +// Print final summary table +fn print_summary() { + println!("\n{}", "=".repeat(120)); + println!("TOKENIZER BENCHMARK SUMMARY"); + println!("{}", "=".repeat(120)); + + let results = BENCHMARK_RESULTS.lock().unwrap(); + + let mut current_category = String::new(); + for (key, value) in results.iter() { + let category = key.split('_').skip(1).collect::>().join("_"); + + if category != current_category { + current_category = category.clone(); + + // Print section header based on category + println!("\n{}", "-".repeat(120)); + match category.as_str() { + "encode" => { + println!("ENCODING THROUGHPUT"); + println!( + "{:<15} | {:>8} | {:>8} | {:>12} | {:>12} | {:>10} | {:>10}", + "Test Case", + "Size(B)", + "Tokens", + "Chars/sec", + "Tokens/sec", + "Ops/sec", + "Thread" + ); + } + "batch" => { + println!("BATCH ENCODING"); + println!( + "{:<15} | {:>8} | {:>8} | {:>12} | {:>12} | {:>10} | {:>10}", + "Batch Size", + "Size(B)", + "Tokens", + "Prompts/sec", + "Tokens/sec", + "Chars/sec", + "Thread" + ); + } + "concurrent" => { + println!("CONCURRENT ENCODING"); + println!( + "{:<15} | {:>10} | {:>12} | {:>12} | {:>15}", + "Clients", "Total Ops", "Ops/sec", "Chars/sec", "Per-Client/sec" + ); + } + "mt_encode" => { + println!("MULTI-THREADED ENCODING"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10} | {:>12}", + "Configuration", + "Total Ops", + "Ops/sec", + "Tokens/sec", + "Threads", + "Efficiency" + ); + } + "decode" => { + println!("DECODE PERFORMANCE"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10}", + "Method", "Tokens", "Tokens/sec", "Ops/sec", "Thread" + ); + } + "mt_decode" => { + println!("MULTI-THREADED DECODING"); + println!( + "{:<20} | {:>12} | {:>12} | {:>10} | {:>12}", + "Configuration", "Total Tokens", "Tokens/sec", "Threads", "Efficiency" + ); + } + "streaming_100k" => { + println!("STREAMING DECODE (100K Target)"); + println!( + "{:<20} | {:>12} | {:>12} | {:>12} | {:>10} | {:>12}", + "Method", "Tokens", "Tokens/sec", "Target", "Thread", "Status" + ); + } + "concurrent_streaming" => { + println!("CONCURRENT STREAMING"); + println!( + "{:<20} | {:>10} | {:>12} | {:>15} | {:>15}", + "Sequences", "Total", "Aggregate/sec", "Per-Seq/sec", "Threads" + ); + } + "stop_sequences" => { + println!("STOP SEQUENCE PERFORMANCE"); + println!( + "{:<20} | {:>10} | {:>12} | {:>12} | {:>10}", + "Config", "Sequences", "Tokens", "Tokens/sec", "Seq/sec" + ); + } + "latency" => { + println!("LATENCY DISTRIBUTION"); + println!( + "{:<20} | {:>10} | {:>10} | {:>10} | {:>10} | {:>10}", + "Operation", "P50(µs)", "P95(µs)", "P99(µs)", "Max(µs)", "Samples" + ); + } + "scaling" => { + println!("SCALING CHARACTERISTICS"); + println!( + "{:<15} | {:>12} | {:>12} | {:>12}", + "Threads", "Total Tokens", "Tokens/sec", "Efficiency" + ); + } + "memory" => { + println!("MEMORY EFFICIENCY"); + println!( + "{:<20} | {:>12} | {:>12} | {:>12}", + "Operation", "Calls/sec", "Time/call", "Improvement" + ); + } + _ => {} + } + println!("{}", "-".repeat(120)); + } + + println!("{}", value); + } + + println!("\n{}", "=".repeat(120)); +} + +fn run_benchmarks(c: &mut Criterion) { + bench_encode_throughput(c); + bench_batch_encode(c); + bench_concurrent_encode(c); + bench_multithreaded_encode(c); + bench_decode_performance(c); + bench_multithreaded_decode(c); + bench_streaming_decode_100k(c); + bench_concurrent_streaming(c); + bench_stop_sequences(c); + bench_latency_distribution(c); + bench_scaling_characteristics(c); + bench_memory_efficiency(c); + + // Print summary at the end + print_summary(); +} + +criterion_group!(benches, run_benchmarks); +criterion::criterion_main!(benches); diff --git a/sgl-router/benches/tool_parser_benchmark.rs b/sgl-router/benches/tool_parser_benchmark.rs new file mode 100644 index 000000000..35bd7f820 --- /dev/null +++ b/sgl-router/benches/tool_parser_benchmark.rs @@ -0,0 +1,848 @@ +//! Comprehensive tool parser benchmark for measuring performance under various scenarios +//! +//! This benchmark tests: +//! - Single parser parsing performance +//! - Registry creation overhead +//! - Concurrent parsing with shared parsers +//! - Streaming vs complete parsing +//! - Different model formats (JSON, Mistral, Qwen, Pythonic, etc.) + +use criterion::{black_box, criterion_group, BenchmarkId, Criterion, Throughput}; +use sglang_router_rs::tool_parser::{ + registry::ParserRegistry, state::ParseState, types::StreamResult, +}; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; + +// Test data for different parser formats - realistic complex examples +const JSON_SIMPLE: &str = r#"{"name": "code_interpreter", "arguments": "{\"language\": \"python\", \"code\": \"import numpy as np\\nimport matplotlib.pyplot as plt\\n\\n# Generate sample data\\nx = np.linspace(0, 10, 100)\\ny = np.sin(x) * np.exp(-x/10)\\n\\n# Create the plot\\nplt.figure(figsize=(10, 6))\\nplt.plot(x, y, 'b-', linewidth=2)\\nplt.grid(True)\\nplt.xlabel('Time (s)')\\nplt.ylabel('Amplitude')\\nplt.title('Damped Oscillation')\\nplt.show()\"}"}"#; + +const JSON_ARRAY: &str = r#"[{"name": "web_search", "arguments": "{\"query\": \"latest developments in quantum computing 2024\", \"num_results\": 10, \"search_type\": \"news\", \"date_range\": \"2024-01-01:2024-12-31\", \"exclude_domains\": [\"reddit.com\", \"facebook.com\"], \"language\": \"en\"}"}, {"name": "analyze_sentiment", "arguments": "{\"text\": \"The breakthrough in quantum error correction represents a significant milestone. Researchers are optimistic about practical applications within the next decade.\", \"granularity\": \"sentence\", \"aspects\": [\"technology\", \"timeline\", \"impact\"], \"confidence_threshold\": 0.85}"}, {"name": "create_summary", "arguments": "{\"content_ids\": [\"doc_1234\", \"doc_5678\", \"doc_9012\"], \"max_length\": 500, \"style\": \"technical\", \"include_citations\": true}"}]"#; + +const JSON_WITH_PARAMS: &str = r#"{"name": "database_query", "parameters": {"connection_string": "postgresql://user:pass@localhost:5432/analytics", "query": "SELECT customer_id, COUNT(*) as order_count, SUM(total_amount) as lifetime_value, AVG(order_amount) as avg_order_value FROM orders WHERE created_at >= '2024-01-01' GROUP BY customer_id HAVING COUNT(*) > 5 ORDER BY lifetime_value DESC LIMIT 100", "timeout_ms": 30000, "read_consistency": "strong", "partition_key": "customer_id"}}"#; + +const MISTRAL_FORMAT: &str = r#"I'll help you analyze the sales data and create visualizations. Let me start by querying the database and then create some charts. + +[TOOL_CALLS] [{"name": "sql_query", "arguments": {"database": "sales_analytics", "query": "WITH monthly_sales AS (SELECT DATE_TRUNC('month', order_date) as month, SUM(total_amount) as revenue, COUNT(DISTINCT customer_id) as unique_customers, COUNT(*) as total_orders FROM orders WHERE order_date >= CURRENT_DATE - INTERVAL '12 months' GROUP BY DATE_TRUNC('month', order_date)) SELECT month, revenue, unique_customers, total_orders, LAG(revenue) OVER (ORDER BY month) as prev_month_revenue, (revenue - LAG(revenue) OVER (ORDER BY month)) / LAG(revenue) OVER (ORDER BY month) * 100 as growth_rate FROM monthly_sales ORDER BY month DESC", "format": "json", "timeout": 60000}}] + +Based on the query results, I can see interesting trends in your sales data."#; + +const MISTRAL_MULTI: &str = r#"Let me help you with a comprehensive analysis of your application's performance. + +[TOOL_CALLS] [{"name": "get_metrics", "arguments": {"service": "api-gateway", "metrics": ["latency_p50", "latency_p95", "latency_p99", "error_rate", "requests_per_second"], "start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z", "aggregation": "5m", "filters": {"environment": "production", "region": "us-east-1"}}}, {"name": "analyze_logs", "arguments": {"log_group": "/aws/lambda/process-orders", "query": "fields @timestamp, @message, @requestId, duration | filter @message like /ERROR/ | stats count() by bin(@timestamp, 5m) as time_window", "start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z", "limit": 1000}}, {"name": "get_traces", "arguments": {"service": "order-processing", "operation": "ProcessOrder", "min_duration_ms": 1000, "max_results": 100, "include_downstream": true}}] + +Now let me create a comprehensive report based on this data."#; + +const QWEN_FORMAT: &str = r#"Let me search for information about machine learning frameworks and their performance benchmarks. + + +{"name": "academic_search", "arguments": {"query": "transformer architecture optimization techniques GPU inference latency reduction", "databases": ["arxiv", "ieee", "acm"], "year_range": [2020, 2024], "citation_count_min": 10, "include_code": true, "page_size": 25, "sort_by": "relevance"}} + + +I found several interesting papers on optimization techniques."#; + +const QWEN_MULTI: &str = r#"I'll help you set up a complete data pipeline for your analytics system. + + +{"name": "create_data_pipeline", "arguments": {"name": "customer_analytics_etl", "source": {"type": "kafka", "config": {"bootstrap_servers": "kafka1:9092,kafka2:9092", "topic": "customer_events", "consumer_group": "analytics_consumer", "auto_offset_reset": "earliest"}}, "transformations": [{"type": "filter", "condition": "event_type IN ('purchase', 'signup', 'churn')"}, {"type": "aggregate", "window": "1h", "group_by": ["customer_id", "event_type"], "metrics": ["count", "sum(amount)"]}], "destination": {"type": "bigquery", "dataset": "analytics", "table": "customer_metrics", "write_mode": "append"}}} + + +{"name": "schedule_job", "arguments": {"job_id": "customer_analytics_etl", "schedule": "0 */4 * * *", "timezone": "UTC", "retry_policy": {"max_attempts": 3, "backoff_multiplier": 2, "max_backoff": 3600}, "notifications": {"on_failure": ["ops-team@company.com"], "on_success": null}, "monitoring": {"sla_minutes": 30, "alert_threshold": 0.95}}} + + +{"name": "create_dashboard", "arguments": {"title": "Customer Analytics Dashboard", "widgets": [{"type": "time_series", "title": "Customer Acquisition", "query": "SELECT DATE(timestamp) as date, COUNT(DISTINCT customer_id) as new_customers FROM analytics.customer_metrics WHERE event_type = 'signup' GROUP BY date ORDER BY date", "visualization": "line"}, {"type": "metric", "title": "Total Revenue", "query": "SELECT SUM(amount) as total FROM analytics.customer_metrics WHERE event_type = 'purchase' AND DATE(timestamp) = CURRENT_DATE()", "format": "currency"}, {"type": "table", "title": "Top Customers", "query": "SELECT customer_id, COUNT(*) as purchases, SUM(amount) as total_spent FROM analytics.customer_metrics WHERE event_type = 'purchase' GROUP BY customer_id ORDER BY total_spent DESC LIMIT 10"}], "refresh_interval": 300}} + + +The data pipeline has been configured and the dashboard is ready."#; + +const LLAMA_FORMAT: &str = r#"<|python_tag|>{"name": "execute_code", "arguments": "{\"code\": \"import pandas as pd\\nimport numpy as np\\nfrom sklearn.model_selection import train_test_split\\nfrom sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.metrics import classification_report, confusion_matrix\\nimport joblib\\n\\n# Load and preprocess data\\ndf = pd.read_csv('/data/customer_churn.csv')\\nprint(f'Dataset shape: {df.shape}')\\nprint(f'Missing values: {df.isnull().sum().sum()}')\\n\\n# Feature engineering\\ndf['tenure_months'] = pd.to_datetime('today') - pd.to_datetime(df['signup_date'])\\ndf['tenure_months'] = df['tenure_months'].dt.days // 30\\ndf['avg_monthly_spend'] = df['total_spend'] / df['tenure_months'].clip(lower=1)\\n\\n# Prepare features and target\\nfeature_cols = ['tenure_months', 'avg_monthly_spend', 'support_tickets', 'product_usage_hours', 'feature_adoption_score']\\nX = df[feature_cols]\\ny = df['churned']\\n\\n# Split and train\\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)\\nrf_model = RandomForestClassifier(n_estimators=100, max_depth=10, min_samples_split=5, random_state=42)\\nrf_model.fit(X_train, y_train)\\n\\n# Evaluate\\ny_pred = rf_model.predict(X_test)\\nprint('Classification Report:')\\nprint(classification_report(y_test, y_pred))\\n\\n# Save model\\njoblib.dump(rf_model, '/models/churn_predictor_v1.pkl')\\nprint('Model saved successfully!')\"}"}"#; + +const PYTHONIC_FORMAT: &str = r#"[retrieve_context(query="How do transformer models handle long-range dependencies in natural language processing tasks?", index="ml_knowledge_base", top_k=5, similarity_threshold=0.75, rerank=True, include_metadata=True, filters={"category": "deep_learning", "year": {"$gte": 2020}})]"#; + +const PYTHONIC_MULTI: &str = r#"[fetch_api_data(endpoint="https://api.weather.com/v1/forecast", params={"lat": 37.7749, "lon": -122.4194, "units": "metric", "days": 7, "hourly": True}, headers={"API-Key": "${WEATHER_API_KEY}"}, timeout=30, retry_count=3), process_weather_data(data="${response}", extract_fields=["temperature", "humidity", "precipitation", "wind_speed", "uv_index"], aggregation="daily", calculate_trends=True), generate_report(data="${processed_data}", template="weather_forecast", format="html", include_charts=True, language="en")]"#; + +const DEEPSEEK_FORMAT: &str = r#"I'll analyze your codebase and identify potential security vulnerabilities. + +🤔[{"name": "scan_repository", "arguments": {"repo_path": "/src/application", "scan_types": ["security", "dependencies", "secrets", "code_quality"], "file_patterns": ["*.py", "*.js", "*.java", "*.go"], "exclude_dirs": ["node_modules", ".git", "vendor", "build"], "vulnerability_databases": ["cve", "nvd", "ghsa"], "min_severity": "medium", "check_dependencies": true, "deep_scan": true, "parallel_workers": 8}}] + +Let me examine the scan results and provide recommendations."#; + +const KIMIK2_FORMAT: &str = r#"⍼validate_and_deploy⍁{"deployment_config": {"application": "payment-service", "version": "2.3.1", "environment": "staging", "region": "us-west-2", "deployment_strategy": "blue_green", "health_check": {"endpoint": "/health", "interval": 30, "timeout": 5, "healthy_threshold": 2, "unhealthy_threshold": 3}, "rollback_on_failure": true, "canary_config": {"percentage": 10, "duration_minutes": 30, "metrics": ["error_rate", "latency_p99", "success_rate"], "thresholds": {"error_rate": 0.01, "latency_p99": 500, "success_rate": 0.99}}, "pre_deployment_hooks": ["run_tests", "security_scan", "backup_database"], "post_deployment_hooks": ["smoke_tests", "notify_team", "update_documentation"]}}"#; + +const GLM4_FORMAT: &str = r#" +analyze_customer_behavior +dataset_id=customer_interactions_2024 +analysis_type=cohort_retention +cohort_definition=signup_month +retention_periods=[1, 7, 14, 30, 60, 90, 180, 365] +segment_by=["acquisition_channel", "pricing_tier", "industry", "company_size"] +metrics=["active_users", "revenue", "feature_usage", "engagement_score"] +statistical_tests=["chi_square", "anova", "trend_analysis"] +visualization_types=["heatmap", "line_chart", "funnel", "sankey"] +export_format=dashboard +confidence_level=0.95 +"#; + +const STEP3_FORMAT: &str = r#" + +orchestrate_ml_pipeline + +fraud_detection_model_v3 +s3://ml-datasets/transactions/2024/ + + {"remove_duplicates": true, "handle_missing": "interpolate", "outlier_method": "isolation_forest"} + {"create_ratios": true, "time_features": ["hour", "day_of_week", "month"], "aggregations": ["mean", "std", "max"]} + {"method": "robust_scaler", "clip_outliers": true} + +{"algorithm": "xgboost", "hyperparameters": {"n_estimators": 500, "max_depth": 8, "learning_rate": 0.01, "subsample": 0.8}, "cross_validation": {"method": "stratified_kfold", "n_splits": 5}} +["auc_roc", "precision_recall", "f1", "confusion_matrix"] +sagemaker_endpoint +{"drift_detection": true, "performance_threshold": 0.92, "alert_emails": ["ml-team@company.com"]} + + +"#; + +const GPT_OSS_FORMAT: &str = r#"{"collection": "technical_documentation", "query_embedding": [0.0234, -0.1456, 0.0891, 0.2341, -0.0567, 0.1234, 0.0456, -0.0789, 0.1567, 0.0234, -0.1123, 0.0678, 0.2345, -0.0456, 0.0891, 0.1234, -0.0567, 0.0789, 0.1456, -0.0234, 0.0891, 0.1567, -0.0678, 0.0345, 0.1234, -0.0456, 0.0789, 0.1891, -0.0234, 0.0567, 0.1345, -0.0891], "top_k": 10, "similarity_metric": "cosine", "filters": {"language": "en", "last_updated": {"$gte": "2023-01-01"}, "categories": {"$in": ["api", "sdk", "integration"]}}, "include_metadata": true, "rerank_with_cross_encoder": true}"#; + +// Large test data for stress testing +fn generate_large_json(num_tools: usize) -> String { + let mut tools = Vec::new(); + for i in 0..num_tools { + tools.push(format!( + r#"{{"name": "tool_{}", "arguments": {{"param1": "value{}", "param2": {}, "param3": true}}}}"#, + i, i, i + )); + } + format!("[{}]", tools.join(", ")) +} + +// Global results storage +lazy_static::lazy_static! { + static ref BENCHMARK_RESULTS: Mutex> = Mutex::new(BTreeMap::new()); +} + +fn add_result(category: &str, result: String) { + let mut results = BENCHMARK_RESULTS.lock().unwrap(); + let index = results.len(); + results.insert(format!("{:03}_{}", index, category), result); +} + +fn bench_registry_creation(c: &mut Criterion) { + let mut group = c.benchmark_group("registry_creation"); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("new_registry", |b| { + let printed_clone = printed.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let registry = black_box(ParserRegistry::new()); + // Force evaluation to prevent optimization + black_box(registry.list_parsers()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Registry Creation", ops_per_sec, time_per_op, "N/A" + ); + add_result("registry", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_parser_lookup(c: &mut Criterion) { + let registry = Arc::new(ParserRegistry::new()); + let models = vec![ + "gpt-4", + "mistral-large", + "qwen-72b", + "llama-3.2", + "deepseek-v3", + "unknown-model", + ]; + + let mut group = c.benchmark_group("parser_lookup"); + + for model in models { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + + group.bench_function(model, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = black_box(registry.get_parser(model)); + // Force evaluation + black_box(parser.is_some()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_nanos() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}ns | {:>15}", + format!("Lookup {}", model), + ops_per_sec, + time_per_op, + if registry.get_parser(model).is_some() { + "Found" + } else { + "Fallback" + } + ); + add_result("lookup", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_complete_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ParserRegistry::new()); + + let test_cases = vec![ + ("json_simple", "json", JSON_SIMPLE), + ("json_array", "json", JSON_ARRAY), + ("json_params", "json", JSON_WITH_PARAMS), + ("mistral_single", "mistral", MISTRAL_FORMAT), + ("mistral_multi", "mistral", MISTRAL_MULTI), + ("qwen_single", "qwen", QWEN_FORMAT), + ("qwen_multi", "qwen", QWEN_MULTI), + ("llama", "llama", LLAMA_FORMAT), + ("pythonic_single", "pythonic", PYTHONIC_FORMAT), + ("pythonic_multi", "pythonic", PYTHONIC_MULTI), + ("deepseek", "deepseek", DEEPSEEK_FORMAT), + ("kimik2", "kimik2", KIMIK2_FORMAT), + ("glm4", "glm4_moe", GLM4_FORMAT), + ("step3", "step3", STEP3_FORMAT), + ("gpt_oss", "gpt_oss", GPT_OSS_FORMAT), + ]; + + let mut group = c.benchmark_group("complete_parsing"); + + for (name, parser_name, input) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + let input_len = input.len(); + + group.throughput(Throughput::Bytes(input_len as u64)); + group.bench_function(name, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser(parser_name).expect("Parser not found"); + + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(input).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let bytes_per_sec = (iters as f64 * input_len as f64) / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}µs", + name, input_len, ops_per_sec, bytes_per_sec, time_per_op + ); + add_result("complete", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_streaming_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ParserRegistry::new()); + + // Streaming test with chunked input + let chunks = vec![ + r#"{"na"#, + r#"me": "sear"#, + r#"ch", "argu"#, + r#"ments": {"qu"#, + r#"ery": "rust prog"#, + r#"ramming", "li"#, + r#"mit": 10, "off"#, + r#"set": 0}"#, + r#"}"#, + ]; + + let mut group = c.benchmark_group("streaming_parsing"); + + let printed = Arc::new(AtomicBool::new(false)); + group.bench_function("json_streaming", |b| { + let printed_clone = printed.clone(); + let registry = registry.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser("json").expect("Parser not found"); + + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let mut state = ParseState::new(); + let mut complete_tools = Vec::new(); + + rt.block_on(async { + for chunk in &chunks { + if let StreamResult::ToolComplete(tool) = + parser.parse_incremental(chunk, &mut state).await.unwrap() + { + complete_tools.push(tool); + } + } + }); + + black_box(complete_tools); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + let chunks_per_sec = (iters as f64 * chunks.len() as f64) / duration.as_secs_f64(); + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}µs", + "JSON Streaming", + chunks.len(), + ops_per_sec, + chunks_per_sec, + time_per_op + ); + add_result("streaming", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_concurrent_parsing(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ParserRegistry::new()); + let parser = registry.get_parser("json").expect("Parser not found"); + + let thread_counts = vec![1, 2, 4, 8, 16, 32]; + let operations_per_thread = 100; + + let mut group = c.benchmark_group("concurrent_parsing"); + group.measurement_time(Duration::from_secs(3)); + + for num_threads in thread_counts { + let printed = Arc::new(AtomicBool::new(false)); + let parser_clone = parser.clone(); + + group.bench_with_input( + BenchmarkId::from_parameter(num_threads), + &num_threads, + |b, &threads| { + let printed_clone = printed.clone(); + let parser = parser_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|_iters| { + let total_operations = Arc::new(AtomicU64::new(0)); + let total_parsed = Arc::new(AtomicU64::new(0)); + let start = Instant::now(); + + let handles: Vec<_> = (0..threads) + .map(|_thread_id| { + let parser = parser.clone(); + let total_ops = total_operations.clone(); + let total_p = total_parsed.clone(); + let rt = rt.clone(); + + thread::spawn(move || { + let test_inputs = [JSON_SIMPLE, JSON_ARRAY, JSON_WITH_PARAMS]; + + for i in 0..operations_per_thread { + let input = test_inputs[i % test_inputs.len()]; + let result = + rt.block_on(async { parser.parse_complete(input).await }); + + if let Ok(tools) = result { + total_p.fetch_add(tools.len() as u64, Ordering::Relaxed); + } + } + + total_ops + .fetch_add(operations_per_thread as u64, Ordering::Relaxed); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let total_ops = total_operations.load(Ordering::Relaxed); + let total_p = total_parsed.load(Ordering::Relaxed); + let ops_per_sec = total_ops as f64 / duration.as_secs_f64(); + let tools_per_sec = total_p as f64 / duration.as_secs_f64(); + + let result = format!( + "{:<25} | {:>10} | {:>12.0} | {:>12.0} | {:>10}", + format!("{}_threads", threads), + total_ops, + ops_per_sec, + tools_per_sec, + threads + ); + add_result("concurrent", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }, + ); + } + + group.finish(); +} + +fn bench_large_payloads(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ParserRegistry::new()); + let parser = registry.get_parser("json").expect("Parser not found"); + + let sizes = vec![1, 10, 50, 100, 500]; + + let mut group = c.benchmark_group("large_payloads"); + + for size in sizes { + let large_json = generate_large_json(size); + let input_len = large_json.len(); + let printed = Arc::new(AtomicBool::new(false)); + let parser_clone = parser.clone(); + + group.throughput(Throughput::Bytes(input_len as u64)); + group.bench_with_input(BenchmarkId::from_parameter(size), &size, |b, &num_tools| { + let printed_clone = printed.clone(); + let parser = parser_clone.clone(); + let rt = rt.handle().clone(); + let input = &large_json; + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(input).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let bytes_per_sec = (iters as f64 * input_len as f64) / duration.as_secs_f64(); + let time_per_op = duration.as_millis() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>10} | {:>10} | {:>12.0} | {:>12.0} | {:>10.1}ms", + format!("{}_tools", num_tools), + num_tools, + input_len, + ops_per_sec, + bytes_per_sec, + time_per_op + ); + add_result("large", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + } + + group.finish(); +} + +fn bench_parser_reuse(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("parser_reuse"); + + // Benchmark creating new registry each time + let printed_new = Arc::new(AtomicBool::new(false)); + group.bench_function("new_registry_each_time", |b| { + let printed_clone = printed_new.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let registry = ParserRegistry::new(); + let parser = registry.get_parser("json").unwrap(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "New Registry Each Time", ops_per_sec, time_per_op, "Baseline" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark reusing registry + let printed_reuse = Arc::new(AtomicBool::new(false)); + let shared_registry = Arc::new(ParserRegistry::new()); + + group.bench_function("reuse_registry", |b| { + let printed_clone = printed_reuse.clone(); + let registry = shared_registry.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser("json").unwrap(); + + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Reuse Registry", ops_per_sec, time_per_op, "Optimized" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + // Benchmark reusing parser + let printed_parser = Arc::new(AtomicBool::new(false)); + let shared_parser = shared_registry.get_parser("json").unwrap(); + + group.bench_function("reuse_parser", |b| { + let printed_clone = printed_parser.clone(); + let parser = shared_parser.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + let result = rt.block_on(async { parser.parse_complete(JSON_SIMPLE).await }); + black_box(result.unwrap()); + } + let duration = start.elapsed(); + + if !printed_clone.load(Ordering::Relaxed) { + let ops_per_sec = iters as f64 / duration.as_secs_f64(); + let time_per_op = duration.as_micros() as f64 / iters as f64; + + let result = format!( + "{:<25} | {:>12.0} | {:>12.1}µs | {:>15}", + "Reuse Parser", ops_per_sec, time_per_op, "Best" + ); + add_result("reuse", result); + + printed_clone.store(true, Ordering::Relaxed); + } + + duration + }); + }); + + group.finish(); +} + +fn bench_latency_distribution(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let registry = Arc::new(ParserRegistry::new()); + + let test_cases = vec![ + ("json", JSON_SIMPLE), + ("mistral", MISTRAL_FORMAT), + ("qwen", QWEN_FORMAT), + ("pythonic", PYTHONIC_FORMAT), + ]; + + let mut group = c.benchmark_group("latency"); + + for (parser_name, input) in test_cases { + let printed = Arc::new(AtomicBool::new(false)); + let registry_clone = registry.clone(); + + group.bench_function(parser_name, |b| { + let printed_clone = printed.clone(); + let registry = registry_clone.clone(); + let rt = rt.handle().clone(); + + b.iter_custom(|iters| { + let parser = registry.get_parser(parser_name).expect("Parser not found"); + + let total_duration = if !printed_clone.load(Ordering::Relaxed) { + let mut latencies = Vec::new(); + + // Warm up + for _ in 0..100 { + let parser = parser.clone(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + } + + // Measure for statistics + for _ in 0..1000 { + let parser = parser.clone(); + let start = Instant::now(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + let latency = start.elapsed(); + latencies.push(latency); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + let max = latencies.last().unwrap(); + + let result = format!( + "{:<25} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10.1} | {:>10}", + parser_name, + p50.as_micros() as f64, + p95.as_micros() as f64, + p99.as_micros() as f64, + max.as_micros() as f64, + 1000 + ); + add_result("latency", result); + + printed_clone.store(true, Ordering::Relaxed); + + // Return median for consistency + p50 * iters as u32 + } else { + // Regular benchmark iterations + let start = Instant::now(); + for _ in 0..iters { + let parser = parser.clone(); + rt.block_on(async { parser.parse_complete(input).await }) + .unwrap(); + } + start.elapsed() + }; + + total_duration + }); + }); + } + + group.finish(); +} + +// Print final summary table +fn print_summary() { + println!("\n{}", "=".repeat(120)); + println!("TOOL PARSER BENCHMARK SUMMARY"); + println!("{}", "=".repeat(120)); + + let results = BENCHMARK_RESULTS.lock().unwrap(); + + let mut current_category = String::new(); + for (key, value) in results.iter() { + let category = key.split('_').skip(1).collect::>().join("_"); + + if category != current_category { + current_category = category.clone(); + + // Print section header based on category + println!("\n{}", "-".repeat(120)); + match category.as_str() { + "registry" => { + println!("REGISTRY OPERATIONS"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Operation", "Ops/sec", "Time/op", "Notes" + ); + } + "lookup" => { + println!("PARSER LOOKUP PERFORMANCE"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Model", "Lookups/sec", "Time/lookup", "Result" + ); + } + "complete" => { + println!("COMPLETE PARSING PERFORMANCE"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>12}", + "Parser Format", "Size(B)", "Ops/sec", "Bytes/sec", "Time/op" + ); + } + "streaming" => { + println!("STREAMING PARSING PERFORMANCE"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>12}", + "Parser", "Chunks", "Ops/sec", "Chunks/sec", "Time/op" + ); + } + "concurrent" => { + println!("CONCURRENT PARSING"); + println!( + "{:<25} | {:>10} | {:>12} | {:>12} | {:>10}", + "Configuration", "Total Ops", "Ops/sec", "Tools/sec", "Threads" + ); + } + "large" => { + println!("LARGE PAYLOAD PARSING"); + println!( + "{:<25} | {:>10} | {:>10} | {:>12} | {:>12} | {:>12}", + "Payload", "Tools", "Size(B)", "Ops/sec", "Bytes/sec", "Time/op" + ); + } + "reuse" => { + println!("PARSER REUSE COMPARISON"); + println!( + "{:<25} | {:>12} | {:>12} | {:>15}", + "Strategy", "Ops/sec", "Time/op", "Performance" + ); + } + "latency" => { + println!("LATENCY DISTRIBUTION"); + println!( + "{:<25} | {:>10} | {:>10} | {:>10} | {:>10} | {:>10}", + "Parser", "P50(µs)", "P95(µs)", "P99(µs)", "Max(µs)", "Samples" + ); + } + _ => {} + } + println!("{}", "-".repeat(120)); + } + + println!("{}", value); + } + + println!("\n{}", "=".repeat(120)); + + // Print performance analysis + println!("\nPERFORMANCE ANALYSIS:"); + println!("{}", "-".repeat(120)); + + // Calculate and display key metrics + if let Some(new_registry) = results.get("007_reuse") { + if let Some(reuse_parser) = results.get("009_reuse") { + // Extract ops/sec values + let new_ops: f64 = new_registry + .split('|') + .nth(1) + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(0.0); + let reuse_ops: f64 = reuse_parser + .split('|') + .nth(1) + .and_then(|s| s.trim().parse().ok()) + .unwrap_or(0.0); + + if new_ops > 0.0 && reuse_ops > 0.0 { + let improvement = (reuse_ops / new_ops - 1.0) * 100.0; + println!("Parser Reuse Improvement: {:.1}% faster", improvement); + + if improvement < 100.0 { + println!("⚠️ WARNING: Parser reuse improvement is lower than expected!"); + println!(" Expected: >100% improvement with singleton pattern"); + println!(" Actual: {:.1}% improvement", improvement); + println!(" Recommendation: Implement global singleton registry"); + } + } + } + } + + println!("{}", "=".repeat(120)); +} + +fn run_benchmarks(c: &mut Criterion) { + bench_registry_creation(c); + bench_parser_lookup(c); + bench_complete_parsing(c); + bench_streaming_parsing(c); + bench_concurrent_parsing(c); + bench_large_payloads(c); + bench_parser_reuse(c); + bench_latency_distribution(c); + + // Print summary at the end + print_summary(); +} + +criterion_group!(benches, run_benchmarks); +criterion::criterion_main!(benches); diff --git a/sgl-router/build.rs b/sgl-router/build.rs new file mode 100644 index 000000000..90b3c6101 --- /dev/null +++ b/sgl-router/build.rs @@ -0,0 +1,35 @@ +fn main() -> Result<(), Box> { + // Only regenerate if the proto file changes + println!("cargo:rerun-if-changed=src/proto/sglang_scheduler.proto"); + + // Configure protobuf compilation with custom settings + let config = prost_build::Config::new(); + + // Skip serde for types that use prost_types::Struct + // These cause conflicts and we don't need serde for all generated types + + // Configure tonic-build for gRPC code generation + tonic_build::configure() + // Generate both client and server code + .build_server(true) + .build_client(true) + // Add a module-level attribute for documentation and clippy warnings + .server_mod_attribute( + "sglang.grpc.scheduler", + "#[allow(unused, clippy::mixed_attributes_style)]", + ) + .client_mod_attribute( + "sglang.grpc.scheduler", + "#[allow(unused, clippy::mixed_attributes_style)]", + ) + // Compile the proto file with the custom config + .compile_protos_with_config( + config, + &["src/proto/sglang_scheduler.proto"], + &["src/proto"], + )?; + + println!("cargo:warning=Protobuf compilation completed successfully"); + + Ok(()) +} diff --git a/sgl-router/py_src/sglang_router/__init__.py b/sgl-router/py_src/sglang_router/__init__.py new file mode 100644 index 000000000..9c7fa208e --- /dev/null +++ b/sgl-router/py_src/sglang_router/__init__.py @@ -0,0 +1,3 @@ +from sglang_router.version import __version__ + +__all__ = ["__version__"] diff --git a/sgl-router/py_src/sglang_router/launch_router.py b/sgl-router/py_src/sglang_router/launch_router.py new file mode 100644 index 000000000..506842f84 --- /dev/null +++ b/sgl-router/py_src/sglang_router/launch_router.py @@ -0,0 +1,109 @@ +import argparse +import logging +import sys +from typing import List, Optional + +import setproctitle +from sglang_router.mini_lb import MiniLoadBalancer +from sglang_router.router_args import RouterArgs + +logger = logging.getLogger("router") + +try: + from sglang_router.router import Router +except ImportError: + Router = None + logger.warning( + "Rust Router is not installed, only python MiniLB (debugging only) is available" + ) + + +def launch_router(args: argparse.Namespace) -> Optional[Router]: + """ + Launch the SGLang router with the configuration from parsed arguments. + + Args: + args: Namespace object containing router configuration + Can be either raw argparse.Namespace or converted RouterArgs + + Returns: + Router instance if successful, None if failed + """ + setproctitle.setproctitle("sglang::router") + try: + # Convert to RouterArgs if needed + if not isinstance(args, RouterArgs): + router_args = RouterArgs.from_cli_args(args) + else: + router_args = args + + if router_args.mini_lb: + mini_lb = MiniLoadBalancer(router_args) + mini_lb.start() + else: + if Router is None: + raise RuntimeError("Rust Router is not installed") + router_args._validate_router_args() + router = Router.from_args(router_args) + router.start() + + except Exception as e: + logger.error(f"Error starting router: {e}") + raise e + + +class CustomHelpFormatter( + argparse.RawDescriptionHelpFormatter, argparse.ArgumentDefaultsHelpFormatter +): + """Custom formatter that preserves both description formatting and shows defaults""" + + pass + + +def parse_router_args(args: List[str]) -> RouterArgs: + """Parse command line arguments and return RouterArgs instance.""" + parser = argparse.ArgumentParser( + description="""SGLang Router - High-performance request distribution across worker nodes + +Usage: +This launcher enables starting a router with individual worker instances. It is useful for +multi-node setups or when you want to start workers and router separately. + +Examples: + # Regular mode + python -m sglang_router.launch_router --worker-urls http://worker1:8000 http://worker2:8000 + + # PD disaggregated mode with same policy for both + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 9000 --prefill http://prefill2:8000 \\ + --decode http://decode1:8001 --decode http://decode2:8001 \\ + --policy cache_aware + + # PD mode with optional bootstrap ports + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 9000 \\ # With bootstrap port + --prefill http://prefill2:8000 none \\ # Explicitly no bootstrap port + --prefill http://prefill3:8000 \\ # Defaults to no bootstrap port + --decode http://decode1:8001 --decode http://decode2:8001 + + # PD mode with different policies for prefill and decode + python -m sglang_router.launch_router --pd-disaggregation \\ + --prefill http://prefill1:8000 --prefill http://prefill2:8000 \\ + --decode http://decode1:8001 --decode http://decode2:8001 \\ + --prefill-policy cache_aware --decode-policy power_of_two + + """, + formatter_class=CustomHelpFormatter, + ) + + RouterArgs.add_cli_args(parser, use_router_prefix=False) + return RouterArgs.from_cli_args(parser.parse_args(args), use_router_prefix=False) + + +def main() -> None: + router_args = parse_router_args(sys.argv[1:]) + launch_router(router_args) + + +if __name__ == "__main__": + main() diff --git a/sgl-router/py_src/sglang_router/launch_server.py b/sgl-router/py_src/sglang_router/launch_server.py new file mode 100644 index 000000000..cce0faf7a --- /dev/null +++ b/sgl-router/py_src/sglang_router/launch_server.py @@ -0,0 +1,203 @@ +import argparse +import copy +import logging +import multiprocessing as mp +import os +import random +import signal +import sys +import time +from typing import List + +import requests +from setproctitle import setproctitle +from sglang_router.launch_router import RouterArgs, launch_router + +from sglang.srt.entrypoints.http_server import launch_server +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import is_port_available + + +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s - %(filename)s:%(lineno)d", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + + +logger = setup_logger() + + +# Create new process group +def run_server(server_args, dp_rank): + """ + Note: + + 1. Without os.setpgrp(), all processes share the same PGID. When you press Ctrl+C, the terminal sends SIGINT to all processes in the group simultaneously. + This can cause leaf processes to terminate first, which messes up the cleaning order and produces orphaned processes. + + Terminal (PGID=100) + └── Main Python Process (PGID=100) + └── Server Process 1 (PGID=100) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=100) + └── Scheduler 2 + └── Detokenizer 2 + + 2. With os.setpgrp(), the main Python process and its children are in a separate group. Now: + + Terminal (PGID=100) + └── Main Python Process (PGID=200) + └── Server Process 1 (PGID=300) + └── Scheduler 1 + └── Detokenizer 1 + └── Server Process 2 (PGID=400) + └── Scheduler 2 + └── Detokenizer 2 + """ + # create new process group + os.setpgrp() + + setproctitle("sglang::server") + # Set SGLANG_DP_RANK environment variable + os.environ["SGLANG_DP_RANK"] = str(dp_rank) + + launch_server(server_args) + + +def launch_server_process( + server_args: ServerArgs, worker_port: int, dp_id: int +) -> mp.Process: + """Launch a single server process with the given args and port.""" + server_args = copy.deepcopy(server_args) + server_args.port = worker_port + server_args.base_gpu_id = dp_id * server_args.tp_size + server_args.dp_size = 1 + + proc = mp.Process(target=run_server, args=(server_args, dp_id)) + proc.start() + return proc + + +def wait_for_server_health(host: str, port: int, timeout: int = 300) -> bool: + """Wait for server to be healthy by checking /health endpoint.""" + start_time = time.perf_counter() + url = f"http://{host}:{port}/health" + + while time.perf_counter() - start_time < timeout: + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + return True + except requests.exceptions.RequestException: + pass + time.sleep(1) + return False + + +def find_available_ports(base_port: int, count: int) -> List[int]: + """Find consecutive available ports starting from base_port.""" + available_ports = [] + current_port = base_port + + while len(available_ports) < count: + if is_port_available(current_port): + available_ports.append(current_port) + current_port += random.randint(100, 1000) + + return available_ports + + +def cleanup_processes(processes: List[mp.Process]): + for process in processes: + logger.info(f"Terminating process group {process.pid}") + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + # Process group may already be terminated + pass + + # Wait for processes to terminate + for process in processes: + process.join(timeout=5) + if process.is_alive(): + logger.warning( + f"Process {process.pid} did not terminate gracefully, forcing kill" + ) + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + logger.info("All process groups terminated") + + +def main(): + # CUDA runtime isn't fork-safe, which can lead to subtle bugs or crashes + mp.set_start_method("spawn") + + parser = argparse.ArgumentParser( + description="Launch SGLang router and server processes" + ) + + ServerArgs.add_cli_args(parser) + RouterArgs.add_cli_args(parser, use_router_prefix=True, exclude_host_port=True) + parser.add_argument( + "--router-dp-worker-base-port", + type=int, + default=31000, + help="Base port number for data parallel workers", + ) + # No extra retry/CB flags here; RouterArgs.add_cli_args already defines them with router- prefix + + args = parser.parse_args() + server_args = ServerArgs.from_cli_args(args) + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Find available ports for workers + worker_ports = find_available_ports( + args.router_dp_worker_base_port, server_args.dp_size + ) + + # Start server processes + server_processes = [] + + for i, worker_port in enumerate(worker_ports): + logger.info(f"Launching DP server process {i} on port {worker_port}") + proc = launch_server_process(server_args, worker_port, i) + server_processes.append(proc) + + signal.signal(signal.SIGINT, lambda sig, frame: cleanup_processes(server_processes)) + signal.signal( + signal.SIGTERM, lambda sig, frame: cleanup_processes(server_processes) + ) + signal.signal( + signal.SIGQUIT, lambda sig, frame: cleanup_processes(server_processes) + ) + + # Update router args with worker URLs + router_args.worker_urls = [ + f"http://{server_args.host}:{port}" for port in worker_ports + ] + + # Start the router + try: + launch_router(router_args) + except Exception as e: + logger.error(f"Failed to start router: {e}") + cleanup_processes(server_processes) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/sgl-router/py_src/sglang_router/mini_lb.py b/sgl-router/py_src/sglang_router/mini_lb.py new file mode 100644 index 000000000..920d5c38f --- /dev/null +++ b/sgl-router/py_src/sglang_router/mini_lb.py @@ -0,0 +1,395 @@ +""" +Minimal HTTP load balancer for prefill and decode servers for testing. +""" + +import asyncio +import ipaddress +import logging +import random +import urllib +from http import HTTPStatus +from itertools import chain +from typing import Optional + +import aiohttp +import orjson +import uvicorn +from fastapi import FastAPI, HTTPException +from fastapi.responses import ORJSONResponse, Response, StreamingResponse +from sglang_router.router_args import RouterArgs + +logger = logging.getLogger(__name__) + +AIOHTTP_STREAM_READ_CHUNK_SIZE = ( + 1024 * 64 +) # 64KB, to prevent aiohttp's "Chunk too big" error + + +def maybe_wrap_ipv6_address(address: str) -> str: + try: + ipaddress.IPv6Address(address) + return f"[{address}]" + except ValueError: + return address + + +class MiniLoadBalancer: + def __init__( + self, + router_args: RouterArgs, + ): + self._validate_router_args(router_args) + + self.host = router_args.host + self.port = router_args.port + self.timeout = router_args.request_timeout_secs + self.prefill_urls = [url[0] for url in router_args.prefill_urls] + self.prefill_bootstrap_ports = [url[1] for url in router_args.prefill_urls] + self.decode_urls = router_args.decode_urls + + def _validate_router_args(self, router_args: RouterArgs): + logger.warning( + "\x1b[33mMiniLB is only for debugging purposes, it only supports random policy!\033[0m" + ) + + # NOTE: too many arguments unsupported, just validate some important ones + if router_args.policy != "random": + logger.warning("[MiniLB] Overriding policy to random") + router_args.policy = "random" + + if not router_args.pd_disaggregation: + raise ValueError("MiniLB only supports PD disaggregation mode") + + if len(router_args.prefill_urls) == 0 or len(router_args.decode_urls) == 0: + raise ValueError( + "MiniLB requires at least one prefill and one decode server" + ) + + def start(self): + global lb + lb = self + uvicorn.run(app, host=self.host, port=self.port) + + def select_pair(self): + assert len(self.prefill_urls) > 0, "No prefill servers available" + assert len(self.decode_urls) > 0, "No decode servers available" + pidx = random.randint(0, len(self.prefill_urls) - 1) + didx = random.randint(0, len(self.decode_urls) - 1) + return ( + self.prefill_urls[pidx], + self.prefill_bootstrap_ports[pidx], + self.decode_urls[didx], + ) + + async def generate( + self, modified_request, prefill_server, decode_server, endpoint + ) -> ORJSONResponse: + assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}" + + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout( + total=self.timeout + ) # Add timeout for request reliability + ) as session: + tasks = [ + session.post(f"{prefill_server}/{endpoint}", json=modified_request), + session.post(f"{decode_server}/{endpoint}", json=modified_request), + ] + + # Wait for both responses to complete. Prefill should end first. + prefill_response, decode_response = await asyncio.gather(*tasks) + + if "return_logprob" in modified_request: + + prefill_json = await prefill_response.json() + ret_json = await decode_response.json() + + # merge `meta_info.input_token_logprobs` from prefill to decode + if "meta_info" in ret_json: + if "input_token_logprobs" in ret_json["meta_info"]: + ret_json["meta_info"]["input_token_logprobs"] = ( + prefill_json["meta_info"]["input_token_logprobs"] + + ret_json["meta_info"]["input_token_logprobs"] + ) + else: + ret_json = await decode_response.json() + + return ORJSONResponse( + content=ret_json, + status_code=decode_response.status, + ) + + async def generate_stream( + self, modified_request, prefill_server, decode_server, endpoint="generate" + ): + assert endpoint[0] != "/", f"Endpoint should not start with '/': {endpoint}" + + async def stream_results(): + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout( + total=self.timeout + ) # Add timeout for request reliability + ) as session: + # Create the tasks for both prefill and decode requests + tasks = [ + session.post(f"{prefill_server}/{endpoint}", json=modified_request), + session.post(f"{decode_server}/{endpoint}", json=modified_request), + ] + # Wait for both responses to complete. Since this is streaming, they return immediately. + prefill_response, decode_response = await asyncio.gather(*tasks) + + if modified_request.get("return_logprob", False): + prefill_chunks = [] + async for chunk in prefill_response.content: + prefill_chunks.append(chunk) + + first_prefill_chunk = ( + prefill_chunks[0].decode("utf-8")[5:].strip("\n") + ) + first_prefill_chunk_json = orjson.loads(first_prefill_chunk) + + async for chunk in decode_response.content: + # Note: This is inefficient + # merge prefill input_token_logprobs, output_token_logprobs to decode + decoded_chunk = chunk.decode("utf-8") + if ( + decoded_chunk + and decoded_chunk.startswith("data:") + and "[DONE]" not in decoded_chunk + ): + ret_json = orjson.loads(decoded_chunk[5:].strip("\n")) + ret_json["meta_info"]["input_token_logprobs"] = ( + first_prefill_chunk_json["meta_info"][ + "input_token_logprobs" + ] + + ret_json["meta_info"]["input_token_logprobs"] + ) + + yield b"data: " + orjson.dumps(ret_json) + b"\n\n" + else: + yield chunk + else: + async for chunk in decode_response.content.iter_chunked( + AIOHTTP_STREAM_READ_CHUNK_SIZE + ): + yield chunk + + return StreamingResponse( + stream_results(), + media_type="text/event-stream", + ) + + +app = FastAPI() +lb: Optional[MiniLoadBalancer] = None + + +@app.get("/health") +async def health_check(): + return Response(status_code=200) + + +@app.get("/health_generate") +async def health_generate(): + async with aiohttp.ClientSession() as session: + # Create the tasks + tasks = [] + for server in chain(lb.prefill_urls, lb.decode_urls): + tasks.append(session.get(f"{server}/health_generate")) + for i, response in enumerate(asyncio.as_completed(tasks)): + await response + return Response(status_code=200) + + +@app.post("/flush_cache") +async def flush_cache(): + async with aiohttp.ClientSession() as session: + # Create the tasks + tasks = [] + for server in chain(lb.prefill_urls, lb.decode_urls): + tasks.append(session.post(f"{server}/flush_cache")) + for i, response in enumerate(asyncio.as_completed(tasks)): + await response + return Response(status_code=200) + + +@app.get("/get_server_info") +async def get_server_info(): + prefill_infos = [] + decode_infos = [] + all_internal_states = [] + + async with aiohttp.ClientSession() as session: + for server in lb.prefill_urls: + server_info = await session.get(f"{server}/get_server_info") + prefill_infos.append(await server_info.json()) + for server in lb.decode_urls: + server_info = await session.get(f"{server}/get_server_info") + info_json = await server_info.json() + decode_infos.append(info_json) + # Extract internal_states from decode servers + if "internal_states" in info_json: + all_internal_states.extend(info_json["internal_states"]) + + # Return format expected by bench_one_batch_server.py + if all_internal_states: + return { + "internal_states": all_internal_states, + "prefill": prefill_infos, + "decode": decode_infos, + } + else: + # Fallback with dummy data if no internal states found + return { + "internal_states": [ + { + "last_gen_throughput": 0.0, + "avg_spec_accept_length": None, + } + ], + "prefill": prefill_infos, + "decode": decode_infos, + } + + +@app.get("/get_model_info") +async def get_model_info(): + if not lb or not lb.prefill_urls: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail="There is no server registered", + ) + + target_server_url = lb.prefill_urls[0] + endpoint_url = f"{target_server_url}/get_model_info" + + async with aiohttp.ClientSession() as session: + try: + async with session.get(endpoint_url) as response: + if response.status != 200: + error_text = await response.text() + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail=( + f"Failed to get model info from {target_server_url}" + f"Status: {response.status}, Response: {error_text}" + ), + ) + + model_info_json = await response.json() + return ORJSONResponse(content=model_info_json) + + except aiohttp.ClientError as e: + raise HTTPException( + status_code=HTTPStatus.SERVICE_UNAVAILABLE, + detail=f"Failed to get model info from backend", + ) + + +@app.post("/generate") +async def handle_generate_request(request_data: dict): + prefill_server, bootstrap_port, decode_server = lb.select_pair() + + # Parse and transform prefill_server for bootstrap data + parsed_url = urllib.parse.urlparse(prefill_server) + hostname = maybe_wrap_ipv6_address(parsed_url.hostname) + modified_request = request_data.copy() + + batch_size = _get_request_batch_size(modified_request) + if batch_size is not None: + modified_request.update( + { + "bootstrap_host": [hostname] * batch_size, + "bootstrap_port": [bootstrap_port] * batch_size, + "bootstrap_room": [ + _generate_bootstrap_room() for _ in range(batch_size) + ], + } + ) + else: + modified_request.update( + { + "bootstrap_host": hostname, + "bootstrap_port": bootstrap_port, + "bootstrap_room": _generate_bootstrap_room(), + } + ) + + if request_data.get("stream", False): + return await lb.generate_stream( + modified_request, prefill_server, decode_server, "generate" + ) + else: + return await lb.generate( + modified_request, prefill_server, decode_server, "generate" + ) + + +async def _forward_to_backend(request_data: dict, endpoint_name: str): + prefill_server, bootstrap_port, decode_server = lb.select_pair() + + # Parse and transform prefill_server for bootstrap data + parsed_url = urllib.parse.urlparse(prefill_server) + hostname = maybe_wrap_ipv6_address(parsed_url.hostname) + modified_request = request_data.copy() + modified_request.update( + { + "bootstrap_host": hostname, + "bootstrap_port": bootstrap_port, + "bootstrap_room": _generate_bootstrap_room(), + } + ) + + if request_data.get("stream", False): + return await lb.generate_stream( + modified_request, + prefill_server, + decode_server, + endpoint=endpoint_name, + ) + else: + return await lb.generate( + modified_request, + prefill_server, + decode_server, + endpoint=endpoint_name, + ) + + +@app.post("/v1/chat/completions") +async def handle_chat_completion_request(request_data: dict): + return await _forward_to_backend(request_data, "v1/chat/completions") + + +@app.post("/v1/completions") +async def handle_completion_request(request_data: dict): + return await _forward_to_backend(request_data, "v1/completions") + + +def _generate_bootstrap_room(): + return random.randint(0, 2**63 - 1) + + +# We may utilize `GenerateReqInput`'s logic later +def _get_request_batch_size(request): + if (text := request.get("text")) is not None: + return None if isinstance(text, str) else len(text) + if (input_ids := request.get("input_ids")) is not None: + return None if isinstance(input_ids[0], int) else len(input_ids) + return None + + +@app.get("/v1/models") +async def get_models(): + prefill_server = lb.prefill_urls[0] # Get the first prefill server + async with aiohttp.ClientSession() as session: + try: + response = await session.get(f"{prefill_server}/v1/models") + if response.status != 200: + raise HTTPException( + status_code=response.status, + detail=f"Prefill server error: Status {response.status}", + ) + return ORJSONResponse(content=await response.json()) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) diff --git a/sgl-router/py_src/sglang_router/router.py b/sgl-router/py_src/sglang_router/router.py new file mode 100644 index 000000000..72a99ffbb --- /dev/null +++ b/sgl-router/py_src/sglang_router/router.py @@ -0,0 +1,129 @@ +from typing import Optional + +from sglang_router.router_args import RouterArgs +from sglang_router_rs import PolicyType +from sglang_router_rs import Router as _Router + + +def policy_from_str(policy_str: Optional[str]) -> PolicyType: + """Convert policy string to PolicyType enum.""" + if policy_str is None: + return None + policy_map = { + "random": PolicyType.Random, + "round_robin": PolicyType.RoundRobin, + "cache_aware": PolicyType.CacheAware, + "power_of_two": PolicyType.PowerOfTwo, + } + return policy_map[policy_str] + + +class Router: + """ + A high-performance router for distributing requests across worker nodes. + + Args: + worker_urls: List of URLs for worker nodes that will handle requests. Each URL should include + the protocol, host, and port (e.g., ['http://worker1:8000', 'http://worker2:8000']) + policy: Load balancing policy to use. Options: + - PolicyType.Random: Randomly select workers + - PolicyType.RoundRobin: Distribute requests in round-robin fashion + - PolicyType.CacheAware: Distribute requests based on cache state and load balance + - PolicyType.PowerOfTwo: Select best of two random workers based on load (PD mode only) + host: Host address to bind the router server. Default: '127.0.0.1' + port: Port number to bind the router server. Default: 3001 + worker_startup_timeout_secs: Timeout in seconds for worker startup. Default: 300 + worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10 + cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker + if the match rate exceeds threshold, otherwise routes to the worker with the smallest + tree. Default: 0.5 + balance_abs_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 32 + balance_rel_threshold: Load balancing is triggered when (max_load - min_load) > abs_threshold + AND max_load > min_load * rel_threshold. Otherwise, use cache aware. Default: 1.0001 + eviction_interval_secs: Interval in seconds between cache eviction operations in cache-aware + routing. Default: 60 + max_payload_size: Maximum payload size in bytes. Default: 256MB + max_tree_size: Maximum size of the approximation tree for cache-aware routing. Default: 2^24 + dp_aware: Enable data parallelism aware schedule. Default: False + api_key: The api key used for the authorization with the worker. + Useful when the dp aware scheduling strategy is enabled. + Default: None + log_dir: Directory to store log files. If None, logs are only output to console. Default: None + log_level: Logging level. Options: 'debug', 'info', 'warning', 'error', 'critical'. + service_discovery: Enable Kubernetes service discovery. When enabled, the router will + automatically discover worker pods based on the selector. Default: False + selector: Dictionary mapping of label keys to values for Kubernetes pod selection. + Example: {"app": "sglang-worker"}. Default: {} + service_discovery_port: Port to use for service discovery. The router will generate + worker URLs using this port. Default: 80 + service_discovery_namespace: Kubernetes namespace to watch for pods. If not provided, + watches pods across all namespaces (requires cluster-wide permissions). Default: None + prefill_selector: Dictionary mapping of label keys to values for Kubernetes pod selection + for prefill servers (PD mode only). Default: {} + decode_selector: Dictionary mapping of label keys to values for Kubernetes pod selection + for decode servers (PD mode only). Default: {} + prometheus_port: Port to expose Prometheus metrics. Default: None + prometheus_host: Host address to bind the Prometheus metrics server. Default: None + pd_disaggregation: Enable PD (Prefill-Decode) disaggregated mode. Default: False + prefill_urls: List of (url, bootstrap_port) tuples for prefill servers (PD mode only) + decode_urls: List of URLs for decode servers (PD mode only) + prefill_policy: Specific load balancing policy for prefill nodes (PD mode only). + If not specified, uses the main policy. Default: None + decode_policy: Specific load balancing policy for decode nodes (PD mode only). + If not specified, uses the main policy. Default: None + request_id_headers: List of HTTP headers to check for request IDs. If not specified, + uses common defaults: ['x-request-id', 'x-correlation-id', 'x-trace-id', 'request-id']. + Example: ['x-my-request-id', 'x-custom-trace-id']. Default: None + bootstrap_port_annotation: Kubernetes annotation name for bootstrap port (PD mode). + Default: 'sglang.ai/bootstrap-port' + request_timeout_secs: Request timeout in seconds. Default: 600 + max_concurrent_requests: Maximum number of concurrent requests allowed for rate limiting. Default: 256 + queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100 + queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60 + rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None + cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: [] + health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3 + health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2 + health_check_timeout_secs: Timeout in seconds for health check requests. Default: 5 + health_check_interval_secs: Interval in seconds between runtime health checks. Default: 60 + health_check_endpoint: Health check endpoint path. Default: '/health' + model_path: Model path for loading tokenizer (HuggingFace model ID or local path). Default: None + tokenizer_path: Explicit tokenizer path (overrides model_path tokenizer if provided). Default: None + """ + + def __init__(self, router: _Router): + self._router = router + + @staticmethod + def from_args(args: RouterArgs) -> "Router": + """Create a router from a RouterArgs instance.""" + + args_dict = vars(args) + # Convert RouterArgs to _Router parameters + args_dict["worker_urls"] = ( + [] + if args_dict["service_discovery"] or args_dict["pd_disaggregation"] + else args_dict["worker_urls"] + ) + args_dict["policy"] = policy_from_str(args_dict["policy"]) + args_dict["prefill_urls"] = ( + args_dict["prefill_urls"] if args_dict["pd_disaggregation"] else None + ) + args_dict["decode_urls"] = ( + args_dict["decode_urls"] if args_dict["pd_disaggregation"] else None + ) + args_dict["prefill_policy"] = policy_from_str(args_dict["prefill_policy"]) + args_dict["decode_policy"] = policy_from_str(args_dict["decode_policy"]) + + # remoge mini_lb parameter + args_dict.pop("mini_lb") + + return Router(_Router(**args_dict)) + + def start(self) -> None: + """Start the router server. + + This method blocks until the server is shut down. + """ + self._router.start() diff --git a/sgl-router/py_src/sglang_router/router_args.py b/sgl-router/py_src/sglang_router/router_args.py new file mode 100644 index 000000000..ad0a2ac9f --- /dev/null +++ b/sgl-router/py_src/sglang_router/router_args.py @@ -0,0 +1,577 @@ +import argparse +import dataclasses +import logging +from typing import Dict, List, Optional + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class RouterArgs: + # Worker configuration + worker_urls: List[str] = dataclasses.field(default_factory=list) + host: str = "127.0.0.1" + port: int = 30000 + + # PD-specific configuration + mini_lb: bool = False + pd_disaggregation: bool = False # Enable PD disaggregated mode + prefill_urls: List[tuple] = dataclasses.field( + default_factory=list + ) # List of (url, bootstrap_port) + decode_urls: List[str] = dataclasses.field(default_factory=list) + + # Routing policy + policy: str = "cache_aware" + prefill_policy: Optional[str] = None # Specific policy for prefill nodes in PD mode + decode_policy: Optional[str] = None # Specific policy for decode nodes in PD mode + worker_startup_timeout_secs: int = 600 + worker_startup_check_interval: int = 30 + cache_threshold: float = 0.3 + balance_abs_threshold: int = 64 + balance_rel_threshold: float = 1.5 + eviction_interval_secs: int = 120 + max_tree_size: int = 2**26 + max_payload_size: int = 512 * 1024 * 1024 # 512MB default for large batches + dp_aware: bool = False + api_key: Optional[str] = None + log_dir: Optional[str] = None + log_level: Optional[str] = None + # Service discovery configuration + service_discovery: bool = False + selector: Dict[str, str] = dataclasses.field(default_factory=dict) + service_discovery_port: int = 80 + service_discovery_namespace: Optional[str] = None + # PD service discovery configuration + prefill_selector: Dict[str, str] = dataclasses.field(default_factory=dict) + decode_selector: Dict[str, str] = dataclasses.field(default_factory=dict) + bootstrap_port_annotation: str = "sglang.ai/bootstrap-port" + # Prometheus configuration + prometheus_port: Optional[int] = None + prometheus_host: Optional[str] = None + # Request ID headers configuration + request_id_headers: Optional[List[str]] = None + # Request timeout in seconds + request_timeout_secs: int = 1800 + # Max concurrent requests for rate limiting + max_concurrent_requests: int = 256 + # Queue size for pending requests when max concurrent limit reached + queue_size: int = 100 + # Maximum time (in seconds) a request can wait in queue before timing out + queue_timeout_secs: int = 60 + # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests + rate_limit_tokens_per_second: Optional[int] = None + # CORS allowed origins + cors_allowed_origins: List[str] = dataclasses.field(default_factory=list) + # Retry configuration + retry_max_retries: int = 5 + retry_initial_backoff_ms: int = 50 + retry_max_backoff_ms: int = 30_000 + retry_backoff_multiplier: float = 1.5 + retry_jitter_factor: float = 0.2 + disable_retries: bool = False + # Health check configuration + health_failure_threshold: int = 3 + health_success_threshold: int = 2 + health_check_timeout_secs: int = 5 + health_check_interval_secs: int = 60 + health_check_endpoint: str = "/health" + # Circuit breaker configuration + cb_failure_threshold: int = 10 + cb_success_threshold: int = 3 + cb_timeout_duration_secs: int = 60 + cb_window_duration_secs: int = 120 + disable_circuit_breaker: bool = False + # Tokenizer configuration + model_path: Optional[str] = None + tokenizer_path: Optional[str] = None + + @staticmethod + def add_cli_args( + parser: argparse.ArgumentParser, + use_router_prefix: bool = False, + exclude_host_port: bool = False, + ): + """ + Add router-specific arguments to an argument parser. + + Args: + parser: The argument parser to add arguments to + use_router_prefix: If True, prefix all arguments with 'router-' to avoid conflicts + exclude_host_port: If True, don't add host and port arguments (used when inheriting from server) + """ + prefix = "router-" if use_router_prefix else "" + + # Worker configuration + if not exclude_host_port: + parser.add_argument( + "--host", + type=str, + default=RouterArgs.host, + help="Host address to bind the router server", + ) + parser.add_argument( + "--port", + type=int, + default=RouterArgs.port, + help="Port number to bind the router server", + ) + + parser.add_argument( + "--worker-urls", + type=str, + nargs="*", + default=[], + help="List of worker URLs (e.g., http://worker1:8000 http://worker2:8000)", + ) + + # Routing policy configuration + parser.add_argument( + f"--{prefix}policy", + type=str, + default=RouterArgs.policy, + choices=["random", "round_robin", "cache_aware", "power_of_two"], + help="Load balancing policy to use. In PD mode, this is used for both prefill and decode unless overridden", + ) + parser.add_argument( + f"--{prefix}prefill-policy", + type=str, + default=None, + choices=["random", "round_robin", "cache_aware", "power_of_two"], + help="Specific policy for prefill nodes in PD mode. If not specified, uses the main policy", + ) + parser.add_argument( + f"--{prefix}decode-policy", + type=str, + default=None, + choices=["random", "round_robin", "cache_aware", "power_of_two"], + help="Specific policy for decode nodes in PD mode. If not specified, uses the main policy", + ) + + # PD-specific arguments + parser.add_argument( + f"--{prefix}mini-lb", + action="store_true", + help="Enable MiniLB", + ) + parser.add_argument( + f"--{prefix}pd-disaggregation", + action="store_true", + help="Enable PD (Prefill-Decode) disaggregated mode", + ) + parser.add_argument( + f"--{prefix}prefill", + nargs="+", + action="append", + help="Prefill server URL and optional bootstrap port. Can be specified multiple times. " + "Format: --prefill URL [BOOTSTRAP_PORT]. " + "BOOTSTRAP_PORT can be a port number, 'none', or omitted (defaults to none).", + ) + parser.add_argument( + f"--{prefix}decode", + nargs=1, + action="append", + metavar=("URL",), + help="Decode server URL. Can be specified multiple times.", + ) + parser.add_argument( + f"--{prefix}worker-startup-timeout-secs", + type=int, + default=RouterArgs.worker_startup_timeout_secs, + help="Timeout in seconds for worker startup", + ) + parser.add_argument( + f"--{prefix}worker-startup-check-interval", + type=int, + default=RouterArgs.worker_startup_check_interval, + help="Interval in seconds between checks for worker startup", + ) + parser.add_argument( + f"--{prefix}cache-threshold", + type=float, + default=RouterArgs.cache_threshold, + help="Cache threshold (0.0-1.0) for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}balance-abs-threshold", + type=int, + default=RouterArgs.balance_abs_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}balance-rel-threshold", + type=float, + default=RouterArgs.balance_rel_threshold, + help="Load balancing is triggered when (max_load - min_load) > abs_threshold AND max_load > min_load * rel_threshold. Otherwise, use cache aware", + ) + parser.add_argument( + f"--{prefix}eviction-interval-secs", + type=int, + default=RouterArgs.eviction_interval_secs, + help="Interval in seconds between cache eviction operations", + ) + parser.add_argument( + f"--{prefix}max-tree-size", + type=int, + default=RouterArgs.max_tree_size, + help="Maximum size of the approximation tree for cache-aware routing", + ) + parser.add_argument( + f"--{prefix}max-payload-size", + type=int, + default=RouterArgs.max_payload_size, + help="Maximum payload size in bytes", + ) + parser.add_argument( + f"--{prefix}dp-aware", + action="store_true", + help="Enable data parallelism aware schedule", + ) + parser.add_argument( + f"--{prefix}api-key", + type=str, + default=None, + help="The api key used for the authorization with the worker. Useful when the dp aware scheduling strategy is enaled.", + ) + parser.add_argument( + f"--{prefix}log-dir", + type=str, + default=None, + help="Directory to store log files. If not specified, logs are only output to console.", + ) + parser.add_argument( + f"--{prefix}log-level", + type=str, + default="info", + choices=["debug", "info", "warning", "error", "critical"], + help="Set the logging level. If not specified, defaults to INFO.", + ) + parser.add_argument( + f"--{prefix}service-discovery", + action="store_true", + help="Enable Kubernetes service discovery", + ) + parser.add_argument( + f"--{prefix}selector", + type=str, + nargs="+", + default={}, + help="Label selector for Kubernetes service discovery (format: key1=value1 key2=value2)", + ) + parser.add_argument( + f"--{prefix}service-discovery-port", + type=int, + default=RouterArgs.service_discovery_port, + help="Port to use for discovered worker pods", + ) + parser.add_argument( + f"--{prefix}service-discovery-namespace", + type=str, + help="Kubernetes namespace to watch for pods. If not provided, watches all namespaces (requires cluster-wide permissions)", + ) + parser.add_argument( + f"--{prefix}prefill-selector", + type=str, + nargs="+", + default={}, + help="Label selector for prefill server pods in PD mode (format: key1=value1 key2=value2)", + ) + parser.add_argument( + f"--{prefix}decode-selector", + type=str, + nargs="+", + default={}, + help="Label selector for decode server pods in PD mode (format: key1=value1 key2=value2)", + ) + # Prometheus configuration + parser.add_argument( + f"--{prefix}prometheus-port", + type=int, + default=29000, + help="Port to expose Prometheus metrics. If not specified, Prometheus metrics are disabled", + ) + parser.add_argument( + f"--{prefix}prometheus-host", + type=str, + default="127.0.0.1", + help="Host address to bind the Prometheus metrics server", + ) + parser.add_argument( + f"--{prefix}request-id-headers", + type=str, + nargs="*", + help="Custom HTTP headers to check for request IDs (e.g., x-request-id x-trace-id). If not specified, uses common defaults.", + ) + parser.add_argument( + f"--{prefix}request-timeout-secs", + type=int, + default=RouterArgs.request_timeout_secs, + help="Request timeout in seconds", + ) + # Retry configuration + parser.add_argument( + f"--{prefix}retry-max-retries", + type=int, + default=RouterArgs.retry_max_retries, + ) + parser.add_argument( + f"--{prefix}retry-initial-backoff-ms", + type=int, + default=RouterArgs.retry_initial_backoff_ms, + ) + parser.add_argument( + f"--{prefix}retry-max-backoff-ms", + type=int, + default=RouterArgs.retry_max_backoff_ms, + ) + parser.add_argument( + f"--{prefix}retry-backoff-multiplier", + type=float, + default=RouterArgs.retry_backoff_multiplier, + ) + parser.add_argument( + f"--{prefix}retry-jitter-factor", + type=float, + default=RouterArgs.retry_jitter_factor, + ) + parser.add_argument( + f"--{prefix}disable-retries", + action="store_true", + help="Disable retries (equivalent to setting retry_max_retries=1)", + ) + # Circuit breaker configuration + parser.add_argument( + f"--{prefix}cb-failure-threshold", + type=int, + default=RouterArgs.cb_failure_threshold, + ) + parser.add_argument( + f"--{prefix}cb-success-threshold", + type=int, + default=RouterArgs.cb_success_threshold, + ) + parser.add_argument( + f"--{prefix}cb-timeout-duration-secs", + type=int, + default=RouterArgs.cb_timeout_duration_secs, + ) + parser.add_argument( + f"--{prefix}cb-window-duration-secs", + type=int, + default=RouterArgs.cb_window_duration_secs, + ) + parser.add_argument( + f"--{prefix}disable-circuit-breaker", + action="store_true", + help="Disable circuit breaker (equivalent to setting cb_failure_threshold to u32::MAX)", + ) + # Health check configuration + parser.add_argument( + f"--{prefix}health-failure-threshold", + type=int, + default=RouterArgs.health_failure_threshold, + help="Number of consecutive health check failures before marking worker unhealthy", + ) + parser.add_argument( + f"--{prefix}health-success-threshold", + type=int, + default=RouterArgs.health_success_threshold, + help="Number of consecutive health check successes before marking worker healthy", + ) + parser.add_argument( + f"--{prefix}health-check-timeout-secs", + type=int, + default=RouterArgs.health_check_timeout_secs, + help="Timeout in seconds for health check requests", + ) + parser.add_argument( + f"--{prefix}health-check-interval-secs", + type=int, + default=RouterArgs.health_check_interval_secs, + help="Interval in seconds between runtime health checks", + ) + parser.add_argument( + f"--{prefix}health-check-endpoint", + type=str, + default=RouterArgs.health_check_endpoint, + help="Health check endpoint path", + ) + parser.add_argument( + f"--{prefix}max-concurrent-requests", + type=int, + default=RouterArgs.max_concurrent_requests, + help="Maximum number of concurrent requests allowed (for rate limiting)", + ) + parser.add_argument( + f"--{prefix}queue-size", + type=int, + default=RouterArgs.queue_size, + help="Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately)", + ) + parser.add_argument( + f"--{prefix}queue-timeout-secs", + type=int, + default=RouterArgs.queue_timeout_secs, + help="Maximum time (in seconds) a request can wait in queue before timing out", + ) + parser.add_argument( + f"--{prefix}rate-limit-tokens-per-second", + type=int, + default=RouterArgs.rate_limit_tokens_per_second, + help="Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests", + ) + parser.add_argument( + f"--{prefix}cors-allowed-origins", + type=str, + nargs="*", + default=[], + help="CORS allowed origins (e.g., http://localhost:3000 https://example.com)", + ) + # Tokenizer configuration + parser.add_argument( + f"--{prefix}model-path", + type=str, + default=None, + help="Model path for loading tokenizer (HuggingFace model ID or local path)", + ) + parser.add_argument( + f"--{prefix}tokenizer-path", + type=str, + default=None, + help="Explicit tokenizer path (overrides model_path tokenizer if provided)", + ) + + @classmethod + def from_cli_args( + cls, args: argparse.Namespace, use_router_prefix: bool = False + ) -> "RouterArgs": + """ + Create RouterArgs instance from parsed command line arguments. + + Args: + args: Parsed command line arguments + use_router_prefix: If True, look for arguments with 'router-' prefix + """ + prefix = "router_" if use_router_prefix else "" + cli_args_dict = vars(args) + args_dict = {} + + for attr in dataclasses.fields(cls): + # Auto strip prefix from args + if f"{prefix}{attr.name}" in cli_args_dict: + args_dict[attr.name] = cli_args_dict[f"{prefix}{attr.name}"] + elif attr.name in cli_args_dict: + args_dict[attr.name] = cli_args_dict[attr.name] + + # parse special arguments and remove "--prefill" and "--decode" from cli_args_dict + args_dict["prefill_urls"] = cls._parse_prefill_urls( + cli_args_dict.get(f"{prefix}prefill", None) + ) + args_dict["decode_urls"] = cls._parse_decode_urls( + cli_args_dict.get(f"{prefix}decode", None) + ) + args_dict["selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}selector", None) + ) + args_dict["prefill_selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}prefill_selector", None) + ) + args_dict["decode_selector"] = cls._parse_selector( + cli_args_dict.get(f"{prefix}decode_selector", None) + ) + + # Mooncake-specific annotation + args_dict["bootstrap_port_annotation"] = "sglang.ai/bootstrap-port" + + return cls(**args_dict) + + def _validate_router_args(self): + # Validate configuration based on mode + if self.pd_disaggregation: + # Validate PD configuration - skip URL requirements if using service discovery + if not self.service_discovery: + if not self.prefill_urls: + raise ValueError("PD disaggregation mode requires --prefill") + if not self.decode_urls: + raise ValueError("PD disaggregation mode requires --decode") + + # Warn about policy usage in PD mode + if self.prefill_policy and self.decode_policy and self.policy: + logger.warning( + "Both --prefill-policy and --decode-policy are specified. " + "The main --policy flag will be ignored for PD mode." + ) + elif self.prefill_policy and not self.decode_policy and self.policy: + logger.info( + f"Using --prefill-policy '{self.prefill_policy}' for prefill nodes " + f"and --policy '{self.policy}' for decode nodes." + ) + elif self.decode_policy and not self.prefill_policy and self.policy: + logger.info( + f"Using --policy '{self.policy}' for prefill nodes " + f"and --decode-policy '{self.decode_policy}' for decode nodes." + ) + + @staticmethod + def _parse_selector(selector_list): + if not selector_list: + return {} + + selector = {} + for item in selector_list: + if "=" in item: + key, value = item.split("=", 1) + selector[key] = value + return selector + + @staticmethod + def _parse_prefill_urls(prefill_list): + """Parse prefill URLs from --prefill arguments. + + Format: --prefill URL [BOOTSTRAP_PORT] + Example: + --prefill http://prefill1:8080 9000 # With bootstrap port + --prefill http://prefill2:8080 none # Explicitly no bootstrap port + --prefill http://prefill3:8080 # Defaults to no bootstrap port + """ + if not prefill_list: + return [] + + prefill_urls = [] + for prefill_args in prefill_list: + + url = prefill_args[0] + + # Handle optional bootstrap port + if len(prefill_args) >= 2: + bootstrap_port_str = prefill_args[1] + # Handle 'none' as None + if bootstrap_port_str.lower() == "none": + bootstrap_port = None + else: + try: + bootstrap_port = int(bootstrap_port_str) + except ValueError: + raise ValueError( + f"Invalid bootstrap port: {bootstrap_port_str}. Must be a number or 'none'" + ) + else: + # No bootstrap port specified, default to None + bootstrap_port = None + + prefill_urls.append((url, bootstrap_port)) + + return prefill_urls + + @staticmethod + def _parse_decode_urls(decode_list): + """Parse decode URLs from --decode arguments. + + Format: --decode URL + Example: --decode http://decode1:8081 --decode http://decode2:8081 + """ + if not decode_list: + return [] + + # decode_list is a list of single-element lists due to nargs=1 + return [url[0] for url in decode_list] diff --git a/sgl-router/py_src/sglang_router/version.py b/sgl-router/py_src/sglang_router/version.py new file mode 100644 index 000000000..c11f861af --- /dev/null +++ b/sgl-router/py_src/sglang_router/version.py @@ -0,0 +1 @@ +__version__ = "0.1.9" diff --git a/sgl-router/py_test/__init__.py b/sgl-router/py_test/__init__.py new file mode 100644 index 000000000..893097780 --- /dev/null +++ b/sgl-router/py_test/__init__.py @@ -0,0 +1 @@ +"""Test package root for router Python tests.""" diff --git a/sgl-router/py_test/conftest.py b/sgl-router/py_test/conftest.py new file mode 100644 index 000000000..894e12bf5 --- /dev/null +++ b/sgl-router/py_test/conftest.py @@ -0,0 +1,8 @@ +import sys +from pathlib import Path + +# Ensure local sources in py_src are importable ahead of any installed package +_ROOT = Path(__file__).resolve().parents[1] +_SRC = _ROOT / "py_src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) diff --git a/sgl-router/py_test/e2e/conftest.py b/sgl-router/py_test/e2e/conftest.py new file mode 100644 index 000000000..7987f328c --- /dev/null +++ b/sgl-router/py_test/e2e/conftest.py @@ -0,0 +1,512 @@ +import json +import logging +import os +import shutil +import signal +import socket +import subprocess +import time +from pathlib import Path +from types import SimpleNamespace +from typing import Callable, Optional +from urllib.parse import urlparse + +import pytest +import requests + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, +) + +logger = logging.getLogger(__name__) + + +def _find_available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _parse_url(base_url: str) -> tuple[str, str]: + """Parse a base URL and return (host, port) as strings. + + This is more robust than simple string splitting and supports different schemes + and URL shapes like trailing paths. + """ + parsed = urlparse(base_url) + return parsed.hostname or "127.0.0.1", ( + str(parsed.port) if parsed.port is not None else "" + ) + + +def _wait_router_health(base_url: str, timeout: float) -> None: + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{base_url}/health", timeout=5) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(2) + raise TimeoutError("Router failed to become healthy in time") + + +def _popen_launch_router( + model: str, + base_url: str, + dp_size: int, + timeout: float, + policy: str = "cache_aware", +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + prom_port = _find_available_port() + + cmd = [ + "python3", + "-m", + "sglang_router.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--dp", + str(dp_size), + "--router-policy", + policy, + "--allow-auto-truncate", + "--router-prometheus-port", + str(prom_port), + "--router-prometheus-host", + "127.0.0.1", + ] + + proc = subprocess.Popen(cmd) + _wait_router_health(base_url, timeout) + return proc + + +def _popen_launch_worker( + model: str, + base_url: str, + *, + dp_size: int | None = None, + api_key: str | None = None, + base_gpu_id: int | None = 0, +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--host", + host, + "--port", + port, + "--base-gpu-id", + str(base_gpu_id or 0), + ] + if dp_size is not None: + cmd += ["--dp-size", str(dp_size)] + if api_key is not None: + cmd += ["--api-key", api_key] + return subprocess.Popen(cmd) + + +def _popen_launch_router_only( + base_url: str, + policy: str = "round_robin", + timeout: float = 120.0, + *, + dp_aware: bool = False, + api_key: str | None = None, +) -> subprocess.Popen: + host, port = _parse_url(base_url) + + prom_port = _find_available_port() + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + host, + "--port", + port, + "--policy", + policy, + ] + if dp_aware: + cmd += ["--dp-aware"] + if api_key is not None: + cmd += ["--api-key", api_key] + cmd += [ + "--prometheus-port", + str(prom_port), + "--prometheus-host", + "127.0.0.1", + ] + proc = subprocess.Popen(cmd) + _wait_router_health(base_url, timeout) + return proc + + +def _terminate(proc: subprocess.Popen, timeout: float = 120) -> None: + if proc is None: + return + proc.terminate() + start = time.perf_counter() + while proc.poll() is None: + if time.perf_counter() - start > timeout: + proc.kill() + break + time.sleep(1) + + +def _which(cmd: str) -> Optional[str]: + try: + return shutil.which(cmd) + except Exception as e: + logger.warning("shutil.which(%r) failed: %s", cmd, e) + return None + + +def _graceful_stop_popen(p: subprocess.Popen) -> None: + if p is None: + return + try: + if p.poll() is None: + p.terminate() + for _ in range(5): + if p.poll() is not None: + break + time.sleep(1) + if p.poll() is None: + p.kill() + except Exception as e: + logger.warning("Exception during graceful stop of popen: %s", e) + + +def _pid_alive(pid: int) -> bool: + try: + os.kill(pid, 0) + return True + except Exception: + return False + + +def _graceful_stop_pid(pid: int) -> None: + try: + if _pid_alive(pid): + try: + os.kill(pid, signal.SIGTERM) + except Exception: + pass + for _ in range(5): + if not _pid_alive(pid): + break + time.sleep(1) + if _pid_alive(pid): + try: + os.kill(pid, signal.SIGKILL) + except Exception: + pass + except Exception: + pass + + +def _graceful_stop_any(obj) -> None: + try: + if isinstance(obj, subprocess.Popen): + _graceful_stop_popen(obj) + return + if isinstance(obj, int): + _graceful_stop_pid(obj) + return + proc_obj = getattr(obj, "proc", None) + if isinstance(proc_obj, subprocess.Popen): + _graceful_stop_popen(proc_obj) + except Exception: + pass + + +@pytest.fixture(scope="session") +def genai_bench_runner() -> Callable[..., None]: + """Provide a callable to run genai-bench and validate metrics. + + Usage in tests: + def test(..., genai_bench_runner): + genai_bench_runner(router_url=..., model_path=..., experiment_folder=...) + """ + + def _run( + *, + router_url: str, + model_path: str, + experiment_folder: str, + timeout_sec: int | None = None, + thresholds: dict | None = None, + extra_env: dict | None = None, + num_concurrency: int = 32, + traffic_scenario: str = "D(4000,100)", + max_requests_per_run: int | None = None, + clean_experiment: bool = True, + kill_procs: list | None = None, + drain_delay_sec: int = 6, + ) -> None: + cli = _which("genai-bench") + if not cli: + pytest.fail( + "genai-bench CLI not found; please install it to run benchmarks" + ) + + # Clean previous experiment folder under current working directory + if clean_experiment: + exp_dir = Path.cwd() / experiment_folder + if exp_dir.exists(): + shutil.rmtree(exp_dir, ignore_errors=True) + + # Default requests per run if not provided + mrr = ( + max_requests_per_run + if max_requests_per_run is not None + else num_concurrency * 3 + ) + + cmd = [ + cli, + "benchmark", + "--api-backend", + "openai", + "--api-base", + router_url, + "--api-key", + "dummy-token", + "--api-model-name", + model_path, + "--model-tokenizer", + model_path, + "--task", + "text-to-text", + "--num-concurrency", + str(num_concurrency), + "--traffic-scenario", + traffic_scenario, + "--max-requests-per-run", + str(mrr), + "--max-time-per-run", + "2", + "--experiment-folder-name", + experiment_folder, + "--experiment-base-dir", + str(Path.cwd()), + ] + + env = os.environ.copy() + if extra_env: + env.update(extra_env) + + to = timeout_sec or int(os.environ.get("GENAI_BENCH_TEST_TIMEOUT", "120")) + proc = subprocess.Popen( + cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True + ) + stdout = stderr = "" + rc = None + try: + try: + stdout, stderr = proc.communicate(timeout=to) + except subprocess.TimeoutExpired: + # Simple: kill the CLI process if it doesn't exit in time + try: + proc.kill() + except Exception: + pass + stdout, stderr = proc.communicate() + rc = proc.returncode + + # Prefer exact path under cwd; fallback to rglob search + base = Path.cwd() + direct = base / experiment_folder + candidates = [direct] if direct.is_dir() else [] + if not candidates: + for p in base.rglob(experiment_folder): + if p.is_dir() and p.name == experiment_folder: + candidates = [p] + break + if not candidates: + raise AssertionError( + "Benchmark failed: experiment folder not found: " + f"{experiment_folder}\nExit code: {rc}\nSTDOUT (tail):\n{stdout[-1000:]}\nSTDERR (tail):\n{stderr[-1000:]}" + ) + actual_folder = candidates[0] + + json_files = [ + p + for p in actual_folder.rglob("*.json") + if "experiment_metadata" not in p.name + ] + if not json_files: + raise AssertionError( + "Benchmark failed: no JSON results found\n" + f"Exit code: {rc}\nSTDOUT (tail):\n{stdout[-1000:]}\nSTDERR (tail):\n{stderr[-1000:]}" + ) + + th = thresholds # None means "log only", no validation + + for jf in json_files: + with jf.open("r") as f: + data = json.load(f) + stats = data.get("aggregated_metrics", {}).get("stats", {}) + ttft_mean = float(stats.get("ttft", {}).get("mean", float("inf"))) + e2e_latency_mean = float( + stats.get("e2e_latency", {}).get("mean", float("inf")) + ) + input_tp_mean = float(stats.get("input_throughput", {}).get("mean", 0.0)) + output_tp_mean = float(stats.get("output_throughput", {}).get("mean", 0.0)) + + logger.info( + "genai-bench[%s] %s ttft_mean=%.3fs e2e_latency_mean=%.3fs input_tp_mean=%.1f tok/s output_tp_mean=%.1f tok/s", + experiment_folder, + jf.name, + ttft_mean, + e2e_latency_mean, + input_tp_mean, + output_tp_mean, + ) + + if th is not None: + assert ( + ttft_mean <= th["ttft_mean_max"] + ), f"TTFT validation failed: {ttft_mean} > {th['ttft_mean_max']} (file={jf.name})" + assert ( + e2e_latency_mean <= th["e2e_latency_mean_max"] + ), f"E2E latency validation failed: {e2e_latency_mean} > {th['e2e_latency_mean_max']} (file={jf.name})" + assert ( + input_tp_mean >= th["input_throughput_mean_min"] + ), f"Input throughput validation failed: {input_tp_mean} < {th['input_throughput_mean_min']} (file={jf.name})" + assert ( + output_tp_mean >= th["output_throughput_mean_min"] + ), f"Output throughput validation failed: {output_tp_mean} < {th['output_throughput_mean_min']} (file={jf.name})" + + finally: + # Always attempt to stop workers to avoid resource leakage + if kill_procs: + # Give router/workers a small grace period to finish any last drains + if drain_delay_sec > 0: + try: + time.sleep(drain_delay_sec) + except Exception: + pass + for p in kill_procs: + _graceful_stop_any(p) + try: + time.sleep(2) + except Exception: + pass + + return _run + + +def pytest_configure(config): + config.addinivalue_line("markers", "e2e: mark as end-to-end test") + + +@pytest.fixture(scope="session") +def e2e_model() -> str: + # Always use the default test model + return DEFAULT_MODEL_NAME_FOR_TEST + + +@pytest.fixture +def e2e_router(e2e_model: str): + # Keep this available but tests below use router-only to avoid GPU contention + base_url = DEFAULT_URL_FOR_TEST + proc = _popen_launch_router( + e2e_model, base_url, dp_size=2, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_router_only_rr(): + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + proc = _popen_launch_router_only(base_url, policy="round_robin") + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture(scope="session") +def e2e_primary_worker(e2e_model: str): + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + proc = _popen_launch_worker(e2e_model, base_url) + # Router health gate will handle worker readiness + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_router_only_rr_dp_aware_api(): + """Router-only with dp-aware enabled and an API key.""" + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + api_key = "secret" + proc = _popen_launch_router_only( + base_url, policy="round_robin", timeout=180.0, dp_aware=True, api_key=api_key + ) + try: + yield SimpleNamespace(proc=proc, url=base_url, api_key=api_key) + finally: + _terminate(proc) + + +@pytest.fixture +def e2e_worker_dp2_api(e2e_model: str, e2e_router_only_rr_dp_aware_api): + """Worker with dp-size=2 and the same API key as the dp-aware router.""" + port = _find_available_port() + base_url = f"http://127.0.0.1:{port}" + api_key = e2e_router_only_rr_dp_aware_api.api_key + proc = _popen_launch_worker(e2e_model, base_url, dp_size=2, api_key=api_key) + try: + yield SimpleNamespace(proc=proc, url=base_url) + finally: + _terminate(proc) + + +@pytest.fixture(scope="session") +def e2e_two_workers_dp2(e2e_model: str): + """Launch two workers, each with dp_size=2, mapped to GPUs [0,1] and [2,3].""" + workers = [] + try: + # Worker A on GPUs 0-1 + port_a = _find_available_port() + url_a = f"http://127.0.0.1:{port_a}" + proc_a = _popen_launch_worker(e2e_model, url_a, dp_size=2, base_gpu_id=0) + workers.append(SimpleNamespace(proc=proc_a, url=url_a)) + + # Worker B on GPUs 2-3 + port_b = _find_available_port() + url_b = f"http://127.0.0.1:{port_b}" + proc_b = _popen_launch_worker(e2e_model, url_b, dp_size=2, base_gpu_id=2) + workers.append(SimpleNamespace(proc=proc_b, url=url_b)) + + yield workers + finally: + for w in workers: + _terminate(w.proc) diff --git a/sgl-router/py_test/e2e/test_pd_router.py b/sgl-router/py_test/e2e/test_pd_router.py new file mode 100644 index 000000000..f6a73cd01 --- /dev/null +++ b/sgl-router/py_test/e2e/test_pd_router.py @@ -0,0 +1,262 @@ +import logging +import os +import socket +import subprocess +import time +from types import SimpleNamespace +from typing import Optional + +import pytest +import requests + +from sglang.test.run_eval import run_eval + +logger = logging.getLogger(__name__) + + +def _find_available_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_health(url: str, timeout: float = 180.0) -> None: + start = time.perf_counter() + with requests.Session() as session: + while time.perf_counter() - start < timeout: + try: + r = session.get(f"{url}/health", timeout=5) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(1) + raise TimeoutError(f"Service at {url} failed to become healthy in time") + + +def _detect_ib_device() -> Optional[str]: + """Return first active IB device name (e.g., mlx5_0) or None if unavailable.""" + # Fast check that ibv_devinfo exists + try: + subprocess.run( + ["ibv_devinfo", "-l"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=1, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return None + + for i in range(12): + dev = f"mlx5_{i}" + try: + res = subprocess.run( + ["ibv_devinfo", dev], + capture_output=True, + text=True, + timeout=2, + ) + if res.returncode == 0 and ("state:" in res.stdout): + for line in res.stdout.splitlines(): + if "state:" in line and "PORT_ACTIVE" in line: + return dev + except Exception: + pass + return None + + +def _popen_launch_prefill_worker( + model: str, + bootstrap_port: int, + ib_device: Optional[str] = None, + base_gpu_id: int = 0, +) -> SimpleNamespace: + port = _find_available_port() + url = f"http://127.0.0.1:{port}" + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--disaggregation-mode", + "prefill", + "--host", + "127.0.0.1", + "--port", + str(port), + "--disaggregation-bootstrap-port", + str(bootstrap_port), + "--base-gpu-id", + str(base_gpu_id), + ] + if ib_device: + cmd += ["--disaggregation-ib-device", ib_device] + proc = subprocess.Popen(cmd) + _wait_health(url, timeout=300.0) + return SimpleNamespace(proc=proc, url=url, bootstrap_port=bootstrap_port) + + +def _popen_launch_decode_worker( + model: str, ib_device: Optional[str] = None, base_gpu_id: int = 0 +) -> SimpleNamespace: + port = _find_available_port() + url = f"http://127.0.0.1:{port}" + cmd = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + model, + "--disaggregation-mode", + "decode", + "--host", + "127.0.0.1", + "--port", + str(port), + "--base-gpu-id", + str(base_gpu_id), + ] + if ib_device: + cmd += ["--disaggregation-ib-device", ib_device] + proc = subprocess.Popen(cmd) + _wait_health(url, timeout=300.0) + return SimpleNamespace(proc=proc, url=url) + + +def _terminate(proc: subprocess.Popen, timeout: float = 120) -> None: + if proc is None: + return + proc.terminate() + start = time.perf_counter() + while proc.poll() is None: + if time.perf_counter() - start > timeout: + proc.kill() + break + time.sleep(1) + + +@pytest.fixture(scope="module") +def pd_cluster(e2e_model: str): + """Start 2 prefill + 2 decode workers and one PD router, once per module.""" + # Environment capability checks: require sgl_kernel and GPU backend + try: + import sgl_kernel # noqa: F401 + except Exception as e: # pragma: no cover - environment dependent + pytest.fail(f"PD e2e requires sgl_kernel but it is not available: {e}") + + try: + import torch # noqa: F401 + except Exception as e: # pragma: no cover - environment dependent + pytest.fail( + f"PD e2e requires torch but it is not available or misconfigured: {e}" + ) + + if not torch.cuda.is_available(): # pragma: no cover - environment dependent + pytest.fail("PD e2e requires CUDA backend, but CUDA is not available") + + workers: list[SimpleNamespace] = [] + router_proc = None + try: + ib_device = _detect_ib_device() + + # Launch 4 workers across 4 GPUs: prefill on 0,1 and decode on 2,3 + pf1 = _popen_launch_prefill_worker( + e2e_model, + bootstrap_port=_find_available_port(), + ib_device=ib_device, + base_gpu_id=0, + ) + pf2 = _popen_launch_prefill_worker( + e2e_model, + bootstrap_port=_find_available_port(), + ib_device=ib_device, + base_gpu_id=1, + ) + dc1 = _popen_launch_decode_worker(e2e_model, ib_device=ib_device, base_gpu_id=2) + dc2 = _popen_launch_decode_worker(e2e_model, ib_device=ib_device, base_gpu_id=3) + prefills = [pf1, pf2] + decodes = [dc1, dc2] + workers.extend(prefills + decodes) + + # PD router with two prefill and two decode endpoints + rport = _find_available_port() + router_url = f"http://127.0.0.1:{rport}" + pport = _find_available_port() + + prefill = [(pf.url, pf.bootstrap_port) for pf in prefills] + decode = [dc.url for dc in decodes] + + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(rport), + "--policy", + "round_robin", + "--pd-disaggregation", + ] + for url, bport in prefill: + cmd += ["--prefill", url, str(bport)] + for url in decode: + cmd += ["--decode", url] + cmd += [ + "--prometheus-port", + str(pport), + "--prometheus-host", + "127.0.0.1", + ] + + router_proc = subprocess.Popen(cmd) + _wait_health(router_url, timeout=180.0) + + yield SimpleNamespace( + router_url=router_url, workers=workers, router_proc=router_proc + ) + finally: + if router_proc is not None: + _terminate(router_proc) + for w in workers: + _terminate(w.proc) + + +@pytest.mark.e2e +def test_pd_mmlu(e2e_model: str, pd_cluster): + """ + Launch 4 workers, start a PD router (2 prefill + 2 decode), then run MMLU. + """ + args = SimpleNamespace( + base_url=pd_cluster.router_url, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 + + +@pytest.mark.e2e +def test_pd_genai_bench(e2e_model: str, pd_cluster, genai_bench_runner): + """ + Launch 4 workers, start a PD router (2 prefill + 2 decode), then run a + short genai-bench benchmark and validate aggregate metrics. + """ + # Run genai-bench against the shared router + policy_label = "benchmark_round_robin_pd" + genai_bench_runner( + router_url=pd_cluster.router_url, + model_path=e2e_model, + experiment_folder=policy_label, + thresholds={ + "ttft_mean_max": 12, + "e2e_latency_mean_max": 15, + "input_throughput_mean_min": 400, + "output_throughput_mean_min": 20, + }, + kill_procs=pd_cluster.workers, + ) diff --git a/sgl-router/py_test/e2e/test_regular_router.py b/sgl-router/py_test/e2e/test_regular_router.py new file mode 100644 index 000000000..856ecda72 --- /dev/null +++ b/sgl-router/py_test/e2e/test_regular_router.py @@ -0,0 +1,169 @@ +import threading +import time +from types import SimpleNamespace + +import pytest +import requests + +from sglang.test.run_eval import run_eval + + +@pytest.mark.e2e +def test_mmlu(e2e_router_only_rr, e2e_two_workers_dp2, e2e_model): + # Attach two dp=2 workers (total 4 GPUs) to a fresh router-only instance + base = e2e_router_only_rr.url + for w in e2e_two_workers_dp2: + r = requests.post(f"{base}/add_worker", params={"url": w.url}, timeout=180) + r.raise_for_status() + + args = SimpleNamespace( + base_url=base, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 + + +@pytest.mark.e2e +def test_genai_bench( + e2e_router_only_rr, e2e_two_workers_dp2, e2e_model, genai_bench_runner +): + """Attach a worker to the regular router and run a short genai-bench.""" + base = e2e_router_only_rr.url + for w in e2e_two_workers_dp2: + r = requests.post(f"{base}/add_worker", params={"url": w.url}, timeout=180) + r.raise_for_status() + + genai_bench_runner( + router_url=base, + model_path=e2e_model, + experiment_folder="benchmark_round_robin_regular", + thresholds={ + "ttft_mean_max": 6, + "e2e_latency_mean_max": 14, + "input_throughput_mean_min": 1000, + "output_throughput_mean_min": 12, + }, + kill_procs=e2e_two_workers_dp2, + ) + + +@pytest.mark.e2e +def test_add_and_remove_worker_live(e2e_router_only_rr, e2e_primary_worker, e2e_model): + base = e2e_router_only_rr.url + worker_url = e2e_primary_worker.url + + r = requests.post(f"{base}/add_worker", params={"url": worker_url}, timeout=180) + r.raise_for_status() + + with requests.Session() as s: + for i in range(8): + r = s.post( + f"{base}/v1/completions", + json={ + "model": e2e_model, + "prompt": f"x{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=120, + ) + r.raise_for_status() + + # Remove the worker + r = requests.post(f"{base}/remove_worker", params={"url": worker_url}, timeout=60) + r.raise_for_status() + + +@pytest.mark.e2e +def test_lazy_fault_tolerance_live(e2e_router_only_rr, e2e_primary_worker, e2e_model): + base = e2e_router_only_rr.url + worker = e2e_primary_worker + + r = requests.post(f"{base}/add_worker", params={"url": worker.url}, timeout=180) + r.raise_for_status() + + def killer(): + time.sleep(10) + try: + worker.proc.terminate() + except Exception: + pass + + t = threading.Thread(target=killer, daemon=True) + t.start() + + args = SimpleNamespace( + base_url=base, + model=e2e_model, + eval_name="mmlu", + num_examples=32, + num_threads=16, + temperature=0.0, + ) + metrics = run_eval(args) + assert 0.0 <= metrics["score"] <= 1.0 + + +@pytest.mark.e2e +def test_dp_aware_worker_expansion_and_api_key( + e2e_model, + e2e_router_only_rr_dp_aware_api, + e2e_worker_dp2_api, +): + """ + Launch a router-only instance in dp_aware mode and a single worker with dp_size=2 + and API key protection. Verify expansion, auth enforcement, and basic eval. + """ + import os + + router_url = e2e_router_only_rr_dp_aware_api.url + worker_url = e2e_worker_dp2_api.url + api_key = e2e_router_only_rr_dp_aware_api.api_key + + # Attach worker; router should expand to dp_size logical workers + r = requests.post( + f"{router_url}/add_worker", params={"url": worker_url}, timeout=180 + ) + r.raise_for_status() + + r = requests.get(f"{router_url}/list_workers", timeout=30) + r.raise_for_status() + urls = r.json().get("urls", []) + assert len(urls) == 2 + assert set(urls) == {f"{worker_url}@0", f"{worker_url}@1"} + + # Verify API key enforcement path-through + # 1) Without Authorization -> 401 from backend + r = requests.post( + f"{router_url}/v1/completions", + json={"model": e2e_model, "prompt": "hi", "max_tokens": 1}, + timeout=60, + ) + assert r.status_code == 401 + + # 2) With correct Authorization -> 200 + r = requests.post( + f"{router_url}/v1/completions", + json={"model": e2e_model, "prompt": "hi", "max_tokens": 1}, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=60, + ) + assert r.status_code == 200 + + # Finally, run MMLU eval through the router with auth + os.environ["OPENAI_API_KEY"] = api_key + args = SimpleNamespace( + base_url=router_url, + model=e2e_model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + metrics = run_eval(args) + assert metrics["score"] >= 0.65 diff --git a/sgl-router/py_test/fixtures/__init__.py b/sgl-router/py_test/fixtures/__init__.py new file mode 100644 index 000000000..4ac754df8 --- /dev/null +++ b/sgl-router/py_test/fixtures/__init__.py @@ -0,0 +1 @@ +"""Shared fixtures for router integration tests.""" diff --git a/sgl-router/py_test/fixtures/mock_worker.py b/sgl-router/py_test/fixtures/mock_worker.py new file mode 100644 index 000000000..13b3c71a6 --- /dev/null +++ b/sgl-router/py_test/fixtures/mock_worker.py @@ -0,0 +1,252 @@ +""" +Lightweight mock worker HTTP server for router integration tests. + +Implements minimal endpoints used by the router: +- GET /health, /health_generate +- POST /generate, /v1/completions, /v1/chat/completions +- POST /flush_cache +- GET /get_server_info, /get_model_info, /v1/models + +Behavior knobs are controlled via CLI flags to simulate failures, latency, and load. +""" + +import argparse +import asyncio +import json +import os +import random +import signal +import sys +import time +from contextlib import asynccontextmanager +from typing import Optional + +import uvicorn +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse + +# Global state (per-process) +_inflight = 0 +_failures_seen = 0 + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser() + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, required=True) + p.add_argument("--worker-id", default=None) + p.add_argument("--latency-ms", type=int, default=0) + p.add_argument("--timeout", action="store_true") + p.add_argument("--status-code", type=int, default=200) + p.add_argument("--fail-first-n", type=int, default=0) + p.add_argument("--random-fail-rate", type=float, default=0.0) + p.add_argument("--require-api-key", action="store_true") + p.add_argument("--api-key", default=None) + p.add_argument("--max-payload-bytes", type=int, default=10 * 1024 * 1024) + p.add_argument("--stream", action="store_true") + p.add_argument("--dp-size", type=int, default=1) + p.add_argument("--crash-on-request", action="store_true") + p.add_argument("--health-fail-after-ms", type=int, default=0) + return p.parse_args() + + +def _extract_worker_id(args: argparse.Namespace) -> str: + if args.worker_id: + return str(args.worker_id) + # default to port (unique enough for tests) + return f"worker-{args.port}" + + +def create_app(args: argparse.Namespace) -> FastAPI: + app = FastAPI() + worker_id = _extract_worker_id(args) + start_ts = time.time() + crashed = {"done": False} + + async def maybe_delay(): + if args.latency_ms > 0: + await asyncio.sleep(args.latency_ms / 1000.0) + + def should_fail() -> Optional[int]: + global _failures_seen + # Fail first N requests (500) + if args.fail_first_n > 0 and _failures_seen < args.fail_first_n: + _failures_seen += 1 + return 500 + # Random failure probability (500) + if args.random_fail_rate > 0.0 and random.random() < args.random_fail_rate: + return 500 + # Forced status code override (non-200) for all responses + if args.status_code != 200: + return int(args.status_code) + return None + + def check_api_key(request: Request): + if not args.require_api_key: + return + auth = request.headers.get("Authorization") + if not auth or not auth.startswith("Bearer "): + raise HTTPException(status_code=401, detail="Unauthorized") + key = auth.split(" ", 1)[1] + if args.api_key and key != args.api_key: + raise HTTPException(status_code=401, detail="Unauthorized") + + @asynccontextmanager + async def track_inflight(): + global _inflight + _inflight += 1 + try: + yield + finally: + _inflight -= 1 + + @app.get("/health") + async def health(): + if ( + args.health_fail_after_ms + and (time.time() - start_ts) * 1000.0 >= args.health_fail_after_ms + ): + return PlainTextResponse("bad", status_code=500) + return PlainTextResponse("ok", status_code=200) + + @app.get("/health_generate") + async def health_generate(): + return PlainTextResponse("ok", status_code=200) + + @app.post("/flush_cache") + async def flush_cache(): + return PlainTextResponse("ok", status_code=200) + + @app.get("/get_model_info") + async def get_model_info(): + return JSONResponse({"model": "mock", "vocab_size": 32000}) + + @app.get("/v1/models") + async def list_models(): + return JSONResponse({"data": [{"id": "mock", "object": "model"}]}) + + @app.get("/get_server_info") + async def get_server_info(request: Request): + # Enforce API key on server info when required (used by dp_aware probing) + check_api_key(request) + return JSONResponse( + { + "worker_id": worker_id, + "load_in_flight": _inflight, + "cache": {"size": 0, "hit_rate": 0.0}, + "dp_size": int(args.dp_size), + } + ) + + @app.get("/get_load") + async def get_load(): + return JSONResponse({"load": _inflight}) + + def make_json_response(obj: dict, status_code: int = 200) -> JSONResponse: + resp = JSONResponse(obj, status_code=status_code) + resp.headers["X-Worker-Id"] = worker_id + return resp + + async def handle_text_request(request: Request): + # Authorization + check_api_key(request) + + # Payload limit + body = await request.body() + if len(body) > args.max_payload_bytes: + return make_json_response({"error": "payload too large"}, status_code=413) + + # Simulate crash on first request + if args.crash_on_request and not crashed["done"]: + crashed["done"] = True + os._exit(1) + + # Optional timeout (simulate hang) + if args.timeout: + await asyncio.sleep(3600) + + # Optional latency + await maybe_delay() + + # Optional failures + fail_code = should_fail() + if fail_code is not None and fail_code != 200: + return make_json_response( + {"error": f"mock failure {fail_code}"}, status_code=fail_code + ) + + # Build response echoing minimal shape + try: + data = await request.json() + except (json.JSONDecodeError, ValueError): + data = {} + + now = time.time() + ret = { + "id": f"cmpl-{int(now*1000)}", + "object": "text_completion", + "created": int(now), + "model": "mock", + "choices": [ + { + "text": "ok", + "index": 0, + "finish_reason": "stop", + } + ], + "worker_id": worker_id, + "echo": data, + } + return make_json_response(ret, status_code=200) + + async def handle_stream_request(request: Request): + check_api_key(request) + + async def gen(): + # minimal 2-chunk stream then [DONE] + for i in range(2): + await asyncio.sleep(0.01) + chunk = { + "choices": [{"delta": {"content": "x"}}], + "worker_id": worker_id, + } + yield f"data: {json.dumps(chunk)}\n\n" + yield "data: [DONE]\n\n" + + headers = {"X-Worker-Id": worker_id} + return StreamingResponse(gen(), media_type="text/event-stream", headers=headers) + + @app.post("/generate") + async def generate(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + @app.post("/v1/completions") + async def completions(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + @app.post("/v1/chat/completions") + async def chat_completions(request: Request): + async with track_inflight(): + if args.stream: + return await handle_stream_request(request) + return await handle_text_request(request) + + return app + + +def main() -> None: + args = _parse_args() + app = create_app(args) + # Handle SIGTERM gracefully for fast test teardown + signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) + uvicorn.run(app, host=args.host, port=args.port, log_level="warning") + + +if __name__ == "__main__": + main() diff --git a/sgl-router/py_test/fixtures/ports.py b/sgl-router/py_test/fixtures/ports.py new file mode 100644 index 000000000..d616cffa1 --- /dev/null +++ b/sgl-router/py_test/fixtures/ports.py @@ -0,0 +1,8 @@ +import socket + + +def find_free_port() -> int: + """Return an available TCP port on localhost.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] diff --git a/sgl-router/py_test/fixtures/router_manager.py b/sgl-router/py_test/fixtures/router_manager.py new file mode 100644 index 000000000..c536a0015 --- /dev/null +++ b/sgl-router/py_test/fixtures/router_manager.py @@ -0,0 +1,158 @@ +import subprocess +import time +from dataclasses import dataclass +from typing import Dict, List, Optional + +import requests + +from .ports import find_free_port + + +@dataclass +class ProcHandle: + process: subprocess.Popen + url: str + + +class RouterManager: + """Helper to spawn a router process and interact with admin endpoints.""" + + def __init__(self): + self._children: List[subprocess.Popen] = [] + + def start_router( + self, + worker_urls: Optional[List[str]] = None, + policy: str = "round_robin", + port: Optional[int] = None, + extra: Optional[Dict] = None, + # PD options + pd_disaggregation: bool = False, + prefill_urls: Optional[List[tuple]] = None, + decode_urls: Optional[List[str]] = None, + prefill_policy: Optional[str] = None, + decode_policy: Optional[str] = None, + ) -> ProcHandle: + worker_urls = worker_urls or [] + port = port or find_free_port() + cmd = [ + "python3", + "-m", + "sglang_router.launch_router", + "--host", + "127.0.0.1", + "--port", + str(port), + "--policy", + policy, + ] + # Avoid Prometheus port collisions by assigning a free port per router + prom_port = find_free_port() + cmd.extend( + ["--prometheus-port", str(prom_port), "--prometheus-host", "127.0.0.1"] + ) + if worker_urls: + cmd.extend(["--worker-urls", *worker_urls]) + + # PD routing configuration + if pd_disaggregation: + cmd.append("--pd-disaggregation") + if prefill_urls: + for url, bport in prefill_urls: + if bport is None: + cmd.extend(["--prefill", url, "none"]) + else: + cmd.extend(["--prefill", url, str(bport)]) + if decode_urls: + for url in decode_urls: + cmd.extend(["--decode", url]) + if prefill_policy: + cmd.extend(["--prefill-policy", prefill_policy]) + if decode_policy: + cmd.extend(["--decode-policy", decode_policy]) + + # Map supported extras to CLI flags (subset for integration) + if extra: + flag_map = { + "max_payload_size": "--max-payload-size", + "dp_aware": "--dp-aware", + "api_key": "--api-key", + # Health/monitoring + "worker_startup_check_interval": "--worker-startup-check-interval", + # Cache-aware tuning + "cache_threshold": "--cache-threshold", + "balance_abs_threshold": "--balance-abs-threshold", + "balance_rel_threshold": "--balance-rel-threshold", + # Retry + "retry_max_retries": "--retry-max-retries", + "retry_initial_backoff_ms": "--retry-initial-backoff-ms", + "retry_max_backoff_ms": "--retry-max-backoff-ms", + "retry_backoff_multiplier": "--retry-backoff-multiplier", + "retry_jitter_factor": "--retry-jitter-factor", + "disable_retries": "--disable-retries", + # Circuit breaker + "cb_failure_threshold": "--cb-failure-threshold", + "cb_success_threshold": "--cb-success-threshold", + "cb_timeout_duration_secs": "--cb-timeout-duration-secs", + "cb_window_duration_secs": "--cb-window-duration-secs", + "disable_circuit_breaker": "--disable-circuit-breaker", + # Rate limiting + "max_concurrent_requests": "--max-concurrent-requests", + "queue_size": "--queue-size", + "queue_timeout_secs": "--queue-timeout-secs", + "rate_limit_tokens_per_second": "--rate-limit-tokens-per-second", + } + for k, v in extra.items(): + if v is None: + continue + flag = flag_map.get(k) + if not flag: + continue + if isinstance(v, bool): + if v: + cmd.append(flag) + else: + cmd.extend([flag, str(v)]) + + proc = subprocess.Popen(cmd) + self._children.append(proc) + url = f"http://127.0.0.1:{port}" + self._wait_health(url) + return ProcHandle(process=proc, url=url) + + def _wait_health(self, base_url: str, timeout: float = 30.0): + start = time.time() + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{base_url}/health", timeout=2) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(0.2) + raise TimeoutError(f"Router at {base_url} did not become healthy") + + def add_worker(self, base_url: str, worker_url: str) -> None: + r = requests.post(f"{base_url}/add_worker", params={"url": worker_url}) + assert r.status_code == 200, f"add_worker failed: {r.status_code} {r.text}" + + def remove_worker(self, base_url: str, worker_url: str) -> None: + r = requests.post(f"{base_url}/remove_worker", params={"url": worker_url}) + assert r.status_code == 200, f"remove_worker failed: {r.status_code} {r.text}" + + def list_workers(self, base_url: str) -> list[str]: + r = requests.get(f"{base_url}/list_workers") + assert r.status_code == 200, f"list_workers failed: {r.status_code} {r.text}" + data = r.json() + return data.get("urls", []) + + def stop_all(self): + for p in self._children: + if p.poll() is None: + p.terminate() + try: + p.wait(timeout=5) + except subprocess.TimeoutExpired: + p.kill() + self._children.clear() diff --git a/sgl-router/py_test/integration/__init__.py b/sgl-router/py_test/integration/__init__.py new file mode 100644 index 000000000..1e342eca0 --- /dev/null +++ b/sgl-router/py_test/integration/__init__.py @@ -0,0 +1 @@ +"""Integration test package for the router.""" diff --git a/sgl-router/py_test/integration/conftest.py b/sgl-router/py_test/integration/conftest.py new file mode 100644 index 000000000..21b9369d7 --- /dev/null +++ b/sgl-router/py_test/integration/conftest.py @@ -0,0 +1,109 @@ +import os +import subprocess +import time +from pathlib import Path +from typing import Dict, Iterable, List, Tuple + +import pytest +import requests + +from ..fixtures.ports import find_free_port +from ..fixtures.router_manager import RouterManager + + +def pytest_configure(config): + config.addinivalue_line("markers", "integration: mark as router integration test") + + +@pytest.fixture +def router_manager() -> Iterable[RouterManager]: + mgr = RouterManager() + try: + yield mgr + finally: + mgr.stop_all() + + +def _spawn_mock_worker(args: List[str]) -> Tuple[subprocess.Popen, str, str]: + repo_root = Path(__file__).resolve().parents[2] + script = repo_root / "py_test" / "fixtures" / "mock_worker.py" + port = find_free_port() + worker_id = f"worker-{port}" + base_cmd = [ + "python3", + str(script), + "--port", + str(port), + "--worker-id", + worker_id, + ] + cmd = base_cmd + args + proc = subprocess.Popen(cmd) + url = f"http://127.0.0.1:{port}" + _wait_health(url) + return proc, url, worker_id + + +def _wait_health(url: str, timeout: float = 10.0): + start = time.time() + with requests.Session() as s: + while time.time() - start < timeout: + try: + r = s.get(f"{url}/health", timeout=1) + if r.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(0.1) + raise TimeoutError(f"Mock worker at {url} did not become healthy") + + +@pytest.fixture +def mock_worker(): + """Start a single healthy mock worker; yields (process, url, worker_id).""" + proc, url, worker_id = _spawn_mock_worker([]) + try: + yield proc, url, worker_id + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture +def mock_workers(): + """Factory to start N workers with custom args. + + Usage: + procs, urls, ids = mock_workers(n=3, args=["--latency-ms", "5"]) # same args for all + ... + """ + + procs: List[subprocess.Popen] = [] + + def _start(n: int, args: List[str] | None = None): + args = args or [] + new_procs: List[subprocess.Popen] = [] + urls: List[str] = [] + ids: List[str] = [] + for _ in range(n): + p, url, wid = _spawn_mock_worker(args) + procs.append(p) + new_procs.append(p) + urls.append(url) + ids.append(wid) + return new_procs, urls, ids + + try: + yield _start + finally: + for p in procs: + if p.poll() is None: + p.terminate() + try: + p.wait(timeout=3) + except subprocess.TimeoutExpired: + p.kill() diff --git a/sgl-router/py_test/integration/load_balancing/__init__.py b/sgl-router/py_test/integration/load_balancing/__init__.py new file mode 100644 index 000000000..77b8c2460 --- /dev/null +++ b/sgl-router/py_test/integration/load_balancing/__init__.py @@ -0,0 +1 @@ +"""Load balancing integration tests.""" diff --git a/sgl-router/py_test/integration/load_balancing/test_cache_aware.py b/sgl-router/py_test/integration/load_balancing/test_cache_aware.py new file mode 100644 index 000000000..acbbd3682 --- /dev/null +++ b/sgl-router/py_test/integration/load_balancing/test_cache_aware.py @@ -0,0 +1,73 @@ +import collections +import concurrent.futures +import uuid + +import pytest +import requests + + +@pytest.mark.integration +def test_cache_aware_affinity(mock_workers, router_manager): + # Two workers; same prompt should stick to one due to cache tree + _, urls, ids = mock_workers(n=2) + rh = router_manager.start_router(worker_urls=urls, policy="cache_aware") + + counts = collections.Counter() + with requests.Session() as s: + for i in range(12): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "repeated prompt for cache", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + counts[wid] += 1 + + # Expect strong skew toward one worker (tree match); majority > 80% + top = max(counts.values()) + assert top >= 10, counts + + +@pytest.mark.integration +def test_cache_aware_diverse_prompts_balances(mock_workers, router_manager): + # Add latency so concurrent requests overlap and influence load-based selection + _, urls, ids = mock_workers(n=3, args=["--latency-ms", "30"]) + rh = router_manager.start_router( + worker_urls=urls, + policy="cache_aware", + extra={ + "cache_threshold": 0.99, + "balance_abs_threshold": 0, + "balance_rel_threshold": 1.0, + }, + ) + + counts = collections.Counter() + + def call(i): + # Use diverse, unrelated prompts to avoid prefix matches entirely + prompt = str(uuid.uuid4()) + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": prompt, + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex: + for wid in ex.map(call, range(40)): + counts[wid] += 1 + + # Expect participation of at least two workers + assert sum(1 for v in counts.values() if v > 0) >= 2, counts diff --git a/sgl-router/py_test/integration/load_balancing/test_power_of_two.py b/sgl-router/py_test/integration/load_balancing/test_power_of_two.py new file mode 100644 index 000000000..c56f4d38a --- /dev/null +++ b/sgl-router/py_test/integration/load_balancing/test_power_of_two.py @@ -0,0 +1,89 @@ +import collections +import concurrent.futures +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_power_of_two_prefers_less_loaded(mock_workers, router_manager): + # Start two workers: one slow (higher inflight), one fast + # Router monitors /get_load and Power-of-Two uses cached loads to choose + # Start one slow and one fast worker using the fixture factory + procs_slow, urls_slow, ids_slow = mock_workers(n=1, args=["--latency-ms", "200"]) + procs_fast, urls_fast, ids_fast = mock_workers(n=1, args=["--latency-ms", "0"]) + procs = procs_slow + procs_fast + urls = urls_slow + urls_fast + ids = ids_slow + ids_fast + slow_id = ids_slow[0] + + rh = router_manager.start_router( + worker_urls=urls, + policy="power_of_two", + extra={"worker_startup_check_interval": 1}, + ) + + # Prime: fire a burst to create measurable load on slow worker, then wait for monitor tick + + def _prime_call(i): + try: + requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"warm-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_prime_call, range(128))) + time.sleep(2) + + # Apply direct background load on the slow worker to amplify load diff + def _direct_load(i): + try: + requests.post( + f"{slow_url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"bg-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_direct_load, range(128))) + time.sleep(1) + + def call(i): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + counts = collections.Counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + for wid in ex.map(call, range(200)): + counts[wid] += 1 + + # Expect the slow worker (higher latency/inflight) to receive fewer requests + fast_worker_id = [i for i in ids if i != slow_id][0] + assert counts[slow_id] < counts[fast_worker_id], counts diff --git a/sgl-router/py_test/integration/load_balancing/test_random.py b/sgl-router/py_test/integration/load_balancing/test_random.py new file mode 100644 index 000000000..41a613e12 --- /dev/null +++ b/sgl-router/py_test/integration/load_balancing/test_random.py @@ -0,0 +1,33 @@ +import collections +import math + +import pytest +import requests + + +@pytest.mark.integration +def test_random_distribution(mock_workers, router_manager): + procs, urls, ids = mock_workers(n=4) + rh = router_manager.start_router(worker_urls=urls, policy="random") + + counts = collections.Counter() + N = 200 + with requests.Session() as s: + for i in range(N): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + counts[wid] += 1 + + # simple statistical tolerance: each worker should be within ±50% of mean + mean = N / len(ids) + for wid in ids: + assert 0.5 * mean <= counts[wid] <= 1.5 * mean, counts diff --git a/sgl-router/py_test/integration/load_balancing/test_round_robin.py b/sgl-router/py_test/integration/load_balancing/test_round_robin.py new file mode 100644 index 000000000..966f3747a --- /dev/null +++ b/sgl-router/py_test/integration/load_balancing/test_round_robin.py @@ -0,0 +1,34 @@ +import collections +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_round_robin_distribution(mock_workers, router_manager): + procs, urls, ids = mock_workers(n=3) + + rh = router_manager.start_router(worker_urls=urls, policy="round_robin") + + counts = collections.Counter() + with requests.Session() as s: + for i in range(30): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"hello {i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid in ids + counts[wid] += 1 + + # Expect near-even distribution across 3 workers + # 30 requests -> ideally 10 each; allow small tolerance ±3 + for wid in ids: + assert 7 <= counts[wid] <= 13, counts diff --git a/sgl-router/py_test/integration/test_api_auth.py b/sgl-router/py_test/integration/test_api_auth.py new file mode 100644 index 000000000..b8ba5c670 --- /dev/null +++ b/sgl-router/py_test/integration/test_api_auth.py @@ -0,0 +1,38 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_router_api_key_enforcement(router_manager, mock_workers): + # Start backend requiring API key; router should forward Authorization header transparently + _, urls, _ = mock_workers( + n=1, args=["--require-api-key", "--api-key", "correct_api_key"] + ) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={}, + ) + + # No auth -> 401 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + ) + assert r.status_code == 401 + + # Invalid auth -> 401 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + headers={"Authorization": "Bearer wrong"}, + ) + assert r.status_code == 401 + + # Correct auth -> 200 + r = requests.post( + f"{rh.url}/v1/completions", + json={"model": "test-model", "prompt": "x", "max_tokens": 1, "stream": False}, + headers={"Authorization": "Bearer correct_api_key"}, + ) + assert r.status_code == 200 diff --git a/sgl-router/py_test/integration/test_circuit_breaker.py b/sgl-router/py_test/integration/test_circuit_breaker.py new file mode 100644 index 000000000..7e7ba409b --- /dev/null +++ b/sgl-router/py_test/integration/test_circuit_breaker.py @@ -0,0 +1,191 @@ +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_circuit_breaker_opens_and_recovers(router_manager, mock_workers): + # A single worker that fails first 3 requests, then succeeds + _, [wurl], _ = mock_workers(n=1, args=["--fail-first-n", "3"]) # fails first 3 + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "cb_failure_threshold": 3, + "cb_success_threshold": 2, + "cb_timeout_duration_secs": 3, + "cb_window_duration_secs": 10, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "trigger", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + saw_503 = False + for _ in range(8): + r = post_once() + if r.status_code == 503: + saw_503 = True + break + assert saw_503, "circuit breaker did not open to return 503" + + time.sleep(4) + r1 = post_once() + r2 = post_once() + assert r1.status_code == 200 and r2.status_code == 200 + + +@pytest.mark.integration +def test_circuit_breaker_half_open_failure_reopens(router_manager, mock_workers): + _, [wurl], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "cb_failure_threshold": 2, + "cb_success_threshold": 2, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 5, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + opened = False + for _ in range(8): + r = post_once() + if r.status_code == 503: + opened = True + break + assert opened, "circuit breaker did not open" + + time.sleep(3) + r = post_once() + assert r.status_code == 500 + r2 = post_once() + assert r2.status_code == 503 + + +@pytest.mark.integration +def test_circuit_breaker_disable_flag(router_manager, mock_workers): + _, [wurl], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[wurl], + policy="round_robin", + extra={ + "disable_circuit_breaker": True, + "disable_retries": True, + }, + ) + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + assert r.status_code == 500 + + +@pytest.mark.integration +def test_circuit_breaker_per_worker_isolation(router_manager, mock_workers): + _, [fail_url], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + _, [ok_url], _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[fail_url, ok_url], + policy="round_robin", + extra={ + "cb_failure_threshold": 2, + "cb_success_threshold": 1, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 10, + "disable_retries": True, + }, + ) + + def post_once(): + return requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "y", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + + failures = 0 + successes_after_open = 0 + opened = False + for _ in range(30): + r = post_once() + if not opened: + if r.status_code == 500: + failures += 1 + if failures >= 2: + _ = post_once() + _ = post_once() + opened = True + else: + if r.status_code == 200: + successes_after_open += 1 + else: + assert False, f"Unexpected non-200 after CB open: {r.status_code}" + assert opened and successes_after_open >= 5 + + +@pytest.mark.integration +def test_circuit_breaker_with_retries(router_manager, mock_workers): + _, [fail_url], _ = mock_workers(n=1, args=["--status-code", "500"]) # always fail + _, [ok_url], _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[fail_url, ok_url], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + "cb_failure_threshold": 2, + "cb_success_threshold": 1, + "cb_timeout_duration_secs": 2, + "cb_window_duration_secs": 10, + }, + ) + + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "z", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 diff --git a/sgl-router/py_test/integration/test_fault_tolerance.py b/sgl-router/py_test/integration/test_fault_tolerance.py new file mode 100644 index 000000000..78e5968ce --- /dev/null +++ b/sgl-router/py_test/integration/test_fault_tolerance.py @@ -0,0 +1,36 @@ +import concurrent.futures +import subprocess +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_worker_crash_reroute_with_retries(router_manager, mock_workers): + # Start one healthy and one that will crash on first request + _, [ok_url], _ = mock_workers(n=1) + _, [crash_url], _ = mock_workers(n=1, args=["--crash-on-request"]) + rh = router_manager.start_router( + worker_urls=[crash_url, ok_url], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + }, + ) + + # A single request should succeed via retry to the healthy worker + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "crash", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + # mock_workers fixture handles cleanup diff --git a/sgl-router/py_test/integration/test_payload_size.py b/sgl-router/py_test/integration/test_payload_size.py new file mode 100644 index 000000000..b3583ab28 --- /dev/null +++ b/sgl-router/py_test/integration/test_payload_size.py @@ -0,0 +1,33 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_payload_size_limit(router_manager, mock_workers): + # Start one backend and a router with a 1MB payload limit + _, urls, _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={"max_payload_size": 1 * 1024 * 1024}, # 1MB + ) + + # Payload just under 1MB should succeed + payload_small = { + "model": "test-model", + "prompt": "x" * int(0.5 * 1024 * 1024), # ~0.5MB + "max_tokens": 1, + "stream": False, + } + r = requests.post(f"{rh.url}/v1/completions", json=payload_small) + assert r.status_code == 200 + + # Payload over 1MB should fail with 413 + payload_large = { + "model": "test-model", + "prompt": "x" * int(1.2 * 1024 * 1024), # ~1.2MB + "max_tokens": 1, + "stream": False, + } + r = requests.post(f"{rh.url}/v1/completions", json=payload_large) + assert r.status_code == 413 diff --git a/sgl-router/py_test/integration/test_pd_routing.py b/sgl-router/py_test/integration/test_pd_routing.py new file mode 100644 index 000000000..d0ae7d552 --- /dev/null +++ b/sgl-router/py_test/integration/test_pd_routing.py @@ -0,0 +1,127 @@ +import collections +import concurrent.futures +import subprocess +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_pd_power_of_two_decode_attribution(router_manager, mock_workers): + # Start two prefill and three decode mock workers via fixture + _, prefill_urls_raw, prefill_ids = mock_workers(n=2) + _, decode_urls_raw, decode_ids_list = mock_workers(n=3) + prefill_urls = [(u, None) for u in prefill_urls_raw] + decode_urls = list(decode_urls_raw) + decode_ids = set(decode_ids_list) + + rh = router_manager.start_router( + policy="power_of_two", + pd_disaggregation=True, + prefill_urls=prefill_urls, + decode_urls=decode_urls, + extra={"worker_startup_check_interval": 1}, + ) + + counts = collections.Counter() + with requests.Session() as s: + for i in range(30): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid in decode_ids + counts[wid] += 1 + + assert sum(1 for v in counts.values() if v > 0) >= 2 + + +@pytest.mark.integration +def test_pd_power_of_two_skews_to_faster_decode(router_manager, mock_workers): + # Start two prefill workers (fast) + _, prefill_urls_raw, _ = mock_workers(n=2) + + # Start two decode workers: one slow, one fast + _, [decode_slow_url], [slow_id] = mock_workers( + n=1, args=["--latency-ms", "300"] + ) # slower decode + _, [decode_fast_url], [fast_id] = mock_workers(n=1) + decode_urls_raw = [decode_slow_url, decode_fast_url] + + prefill_urls = [(u, None) for u in prefill_urls_raw] + decode_urls = list(decode_urls_raw) + + rh = router_manager.start_router( + policy="power_of_two", + pd_disaggregation=True, + prefill_urls=prefill_urls, + decode_urls=decode_urls, + extra={"worker_startup_check_interval": 1}, + ) + + def _prime_call(i): + try: + requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"warm-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_prime_call, range(128))) + time.sleep(2) + + def _direct_decode_load(i): + try: + requests.post( + f"{decode_slow_url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"bg-{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + except Exception: + pass + + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + list(ex.map(_direct_decode_load, range(128))) + time.sleep(1) + + def call(i): + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=8, + ) + assert r.status_code == 200 + return r.headers.get("X-Worker-Id") or r.json().get("worker_id") + + counts = collections.Counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=32) as ex: + for wid in ex.map(call, range(200)): + counts[wid] += 1 + + assert counts[slow_id] < counts[fast_id], counts diff --git a/sgl-router/py_test/integration/test_rate_limiting.py b/sgl-router/py_test/integration/test_rate_limiting.py new file mode 100644 index 000000000..4297d77c9 --- /dev/null +++ b/sgl-router/py_test/integration/test_rate_limiting.py @@ -0,0 +1,91 @@ +import concurrent.futures +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_rate_limit_and_queue(router_manager, mock_workers): + # One fast backend + _, urls, _ = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={ + "max_concurrent_requests": 2, + "queue_size": 0, # no queue -> immediate 429 when limit exceeded + }, + ) + + def call_once(i): + try: + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"p{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=3, + ) + return r.status_code + except Exception: + return 599 + + # Fire a burst of concurrent requests + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex: + results = list(ex.map(call_once, range(16))) + + # Expect some to succeed and some to be rate limited (429) + assert any(code == 200 for code in results) + assert any(code == 429 for code in results) + + +@pytest.mark.integration +def test_rate_limit_queue_and_timeout(router_manager, mock_workers): + # Slow backend: ~2s per request ensures queue wait > timeout + _, urls, _ = mock_workers(n=1, args=["--latency-ms", "2000"]) # 2.0s per request + + # Allow 1 concurrent, queue up to 1, with 1s queue timeout + rh = router_manager.start_router( + worker_urls=urls, + policy="round_robin", + extra={ + "max_concurrent_requests": 1, + "queue_size": 1, + "queue_timeout_secs": 1, + }, + ) + + def call_once(i): + try: + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"q{i}", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + return r.status_code + except Exception: + return 599 + + # Fire 4 concurrent requests: 1 runs (~2s), 1 queued (times out at 1s -> 408), 2 overflow -> 429 + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex: + results = list(ex.map(call_once, range(4))) + + # We expect: + # - Some 200s (processed) + # - At least one 408 (queued too long and timed out) + # - Remaining non-200s are either 429 (queue overflow) or additional 408s depending on scheduling + assert any(code == 200 for code in results) + assert any(code == 408 for code in results), results + non200 = [c for c in results if c != 200] + assert len(non200) >= 2 and all(c in (408, 429) for c in non200), results diff --git a/sgl-router/py_test/integration/test_retries.py b/sgl-router/py_test/integration/test_retries.py new file mode 100644 index 000000000..5f3d4ffee --- /dev/null +++ b/sgl-router/py_test/integration/test_retries.py @@ -0,0 +1,65 @@ +import concurrent.futures +import subprocess +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_retry_reroutes_to_healthy_worker(router_manager, mock_workers): + # Worker A always 500; Worker B healthy + # Worker A always 500; Worker B/C healthy + _, [url_a], [id_a] = mock_workers(n=1, args=["--status-code", "500"]) # fail + _, [url_b], [id_b] = mock_workers(n=1) + _, [url_c], [id_c] = mock_workers(n=1) + rh = router_manager.start_router( + worker_urls=[url_a, url_b, url_c], + policy="round_robin", + extra={ + "retry_max_retries": 3, + "retry_initial_backoff_ms": 10, + "retry_max_backoff_ms": 50, + }, + ) + + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid == id_b # should have retried onto healthy worker + # mock_workers fixture handles cleanup + + +@pytest.mark.integration +def test_disable_retries_surfaces_failure(router_manager, mock_workers): + # Single failing worker, retries disabled -> should return 500 + _, [url], [wid] = mock_workers(n=1, args=["--status-code", "500"]) # always fail + rh = router_manager.start_router( + worker_urls=[url], + policy="round_robin", + extra={ + "disable_retries": True, + }, + ) + + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "x", + "max_tokens": 1, + "stream": False, + }, + timeout=5, + ) + assert r.status_code == 500 + # mock_workers fixture handles cleanup diff --git a/sgl-router/py_test/integration/test_service_discovery_shim.py b/sgl-router/py_test/integration/test_service_discovery_shim.py new file mode 100644 index 000000000..5cc1d6734 --- /dev/null +++ b/sgl-router/py_test/integration/test_service_discovery_shim.py @@ -0,0 +1,36 @@ +import pytest +import requests + + +@pytest.mark.integration +def test_discovery_shim_add_remove(router_manager, mock_workers): + # Start router without workers + rh = router_manager.start_router(worker_urls=[], policy="round_robin") + + # Initially empty + urls = router_manager.list_workers(rh.url) + assert urls == [] + + # Add a worker (simulate discovery event) + _, [wurl], [wid] = mock_workers(n=1) + router_manager.add_worker(rh.url, wurl) + urls = router_manager.list_workers(rh.url) + assert wurl in urls + + # Can serve a request + r = requests.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": "hi", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + + # Remove worker (simulate pod deletion) + router_manager.remove_worker(rh.url, wurl) + urls = router_manager.list_workers(rh.url) + assert wurl not in urls + # mock_workers fixture handles cleanup diff --git a/sgl-router/py_test/integration/test_worker_management.py b/sgl-router/py_test/integration/test_worker_management.py new file mode 100644 index 000000000..8acb94114 --- /dev/null +++ b/sgl-router/py_test/integration/test_worker_management.py @@ -0,0 +1,61 @@ +import collections +import subprocess +import time + +import pytest +import requests + + +@pytest.mark.integration +def test_add_and_remove_worker(mock_worker, router_manager, mock_workers): + # Start with a single worker + proc1, url1, id1 = mock_worker + rh = router_manager.start_router(worker_urls=[url1], policy="round_robin") + + # Add a second worker + + procs2, urls2, ids2 = mock_workers(n=1) + url2 = urls2[0] + id2 = ids2[0] + router_manager.add_worker(rh.url, url2) + + # Send some requests and ensure both workers are seen + seen = set() + with requests.Session() as s: + for i in range(20): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"x{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + seen.add(wid) + if len(seen) == 2: + break + + assert id1 in seen and id2 in seen + + # Now remove the second worker + router_manager.remove_worker(rh.url, url2) + + # After removal, subsequent requests should only come from first worker + with requests.Session() as s: + for i in range(10): + r = s.post( + f"{rh.url}/v1/completions", + json={ + "model": "test-model", + "prompt": f"y{i}", + "max_tokens": 1, + "stream": False, + }, + ) + assert r.status_code == 200 + wid = r.headers.get("X-Worker-Id") or r.json().get("worker_id") + assert wid == id1 + # mock_workers fixture handles cleanup diff --git a/sgl-router/py_test/unit/__init__.py b/sgl-router/py_test/unit/__init__.py new file mode 100644 index 000000000..42cbd8bee --- /dev/null +++ b/sgl-router/py_test/unit/__init__.py @@ -0,0 +1,7 @@ +""" +Unit tests for sglang_router. + +This package contains fast, isolated unit tests for Python components +of the SGLang router. These tests focus on testing individual functions +and classes in isolation without starting actual router instances. +""" diff --git a/sgl-router/py_test/unit/test_arg_parser.py b/sgl-router/py_test/unit/test_arg_parser.py new file mode 100644 index 000000000..04d8a112d --- /dev/null +++ b/sgl-router/py_test/unit/test_arg_parser.py @@ -0,0 +1,628 @@ +""" +Unit tests for argument parsing functionality in sglang_router. + +These tests focus on testing the argument parsing logic in isolation, +without starting actual router instances. +""" + +import argparse +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, parse_router_args +from sglang_router.router import policy_from_str + + +class TestRouterArgs: + """Test RouterArgs dataclass and its methods.""" + + def test_default_values(self): + """Test that RouterArgs has correct default values.""" + args = RouterArgs() + + # Test basic defaults + assert args.host == "127.0.0.1" + assert args.port == 30000 + assert args.policy == "cache_aware" + assert args.worker_urls == [] + assert args.pd_disaggregation is False + assert args.prefill_urls == [] + assert args.decode_urls == [] + + # Test PD-specific defaults + assert args.prefill_policy is None + assert args.decode_policy is None + + # Test service discovery defaults + assert args.service_discovery is False + assert args.selector == {} + assert args.service_discovery_port == 80 + assert args.service_discovery_namespace is None + + # Test retry and circuit breaker defaults + assert args.retry_max_retries == 5 + assert args.cb_failure_threshold == 10 + assert args.disable_retries is False + assert args.disable_circuit_breaker is False + + def test_parse_selector_valid(self): + """Test parsing valid selector arguments.""" + # Test single key-value pair + result = RouterArgs._parse_selector(["app=worker"]) + assert result == {"app": "worker"} + + # Test multiple key-value pairs + result = RouterArgs._parse_selector(["app=worker", "env=prod", "version=v1"]) + assert result == {"app": "worker", "env": "prod", "version": "v1"} + + # Test empty list + result = RouterArgs._parse_selector([]) + assert result == {} + + # Test None + result = RouterArgs._parse_selector(None) + assert result == {} + + def test_parse_selector_invalid(self): + """Test parsing invalid selector arguments.""" + # Test malformed selector (no equals sign) + result = RouterArgs._parse_selector(["app"]) + assert result == {} + + # Test multiple equals signs (should use first one) + result = RouterArgs._parse_selector(["app=worker=extra"]) + assert result == {"app": "worker=extra"} + + def test_parse_prefill_urls_valid(self): + """Test parsing valid prefill URL arguments.""" + # Test with bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000", "9000"]]) + assert result == [("http://prefill1:8000", 9000)] + + # Test with 'none' bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000", "none"]]) + assert result == [("http://prefill1:8000", None)] + + # Test without bootstrap port + result = RouterArgs._parse_prefill_urls([["http://prefill1:8000"]]) + assert result == [("http://prefill1:8000", None)] + + # Test multiple prefill URLs + result = RouterArgs._parse_prefill_urls( + [ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ["http://prefill3:8000"], + ] + ) + expected = [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ("http://prefill3:8000", None), + ] + assert result == expected + + # Test empty list + result = RouterArgs._parse_prefill_urls([]) + assert result == [] + + # Test None + result = RouterArgs._parse_prefill_urls(None) + assert result == [] + + def test_parse_prefill_urls_invalid(self): + """Test parsing invalid prefill URL arguments.""" + # Test invalid bootstrap port + with pytest.raises(ValueError, match="Invalid bootstrap port"): + RouterArgs._parse_prefill_urls([["http://prefill1:8000", "invalid"]]) + + def test_parse_decode_urls_valid(self): + """Test parsing valid decode URL arguments.""" + # Test single decode URL + result = RouterArgs._parse_decode_urls([["http://decode1:8001"]]) + assert result == ["http://decode1:8001"] + + # Test multiple decode URLs + result = RouterArgs._parse_decode_urls( + [["http://decode1:8001"], ["http://decode2:8001"]] + ) + assert result == ["http://decode1:8001", "http://decode2:8001"] + + # Test empty list + result = RouterArgs._parse_decode_urls([]) + assert result == [] + + # Test None + result = RouterArgs._parse_decode_urls(None) + assert result == [] + + def test_from_cli_args_basic(self): + """Test creating RouterArgs from basic CLI arguments.""" + args = SimpleNamespace( + host="0.0.0.0", + port=30001, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="round_robin", + prefill=None, + decode=None, + router_policy="round_robin", + router_pd_disaggregation=False, + router_prefill_policy=None, + router_decode_policy=None, + router_worker_startup_timeout_secs=300, + router_worker_startup_check_interval=15, + router_cache_threshold=0.7, + router_balance_abs_threshold=128, + router_balance_rel_threshold=2.0, + router_eviction_interval=180, + router_max_tree_size=2**28, + router_max_payload_size=1024 * 1024 * 1024, # 1GB + router_dp_aware=True, + router_api_key="test-key", + router_log_dir="/tmp/logs", + router_log_level="debug", + router_service_discovery=True, + router_selector=["app=worker", "env=test"], + router_service_discovery_port=8080, + router_service_discovery_namespace="default", + router_prefill_selector=["app=prefill"], + router_decode_selector=["app=decode"], + router_prometheus_port=29000, + router_prometheus_host="0.0.0.0", + router_request_id_headers=["x-request-id", "x-trace-id"], + router_request_timeout_secs=1200, + router_max_concurrent_requests=512, + router_queue_size=200, + router_queue_timeout_secs=120, + router_rate_limit_tokens_per_second=100, + router_cors_allowed_origins=["http://localhost:3000"], + router_retry_max_retries=3, + router_retry_initial_backoff_ms=100, + router_retry_max_backoff_ms=10000, + router_retry_backoff_multiplier=2.0, + router_retry_jitter_factor=0.1, + router_cb_failure_threshold=5, + router_cb_success_threshold=2, + router_cb_timeout_duration_secs=30, + router_cb_window_duration_secs=60, + router_disable_retries=False, + router_disable_circuit_breaker=False, + router_health_failure_threshold=2, + router_health_success_threshold=1, + router_health_check_timeout_secs=3, + router_health_check_interval_secs=30, + router_health_check_endpoint="/healthz", + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Test basic configuration + assert router_args.host == "0.0.0.0" + assert router_args.port == 30001 + assert router_args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert router_args.policy == "round_robin" + + # Test PD configuration + assert router_args.pd_disaggregation is False + assert router_args.prefill_urls == [] + assert router_args.decode_urls == [] + + # Test service discovery + assert router_args.service_discovery is True + assert router_args.selector == {"app": "worker", "env": "test"} + assert router_args.service_discovery_port == 8080 + assert router_args.service_discovery_namespace == "default" + assert router_args.prefill_selector == {"app": "prefill"} + assert router_args.decode_selector == {"app": "decode"} + + # Test other configurations + assert router_args.dp_aware is True + assert router_args.api_key == "test-key" + assert router_args.log_dir == "/tmp/logs" + assert router_args.log_level == "debug" + assert router_args.prometheus_port == 29000 + assert router_args.prometheus_host == "0.0.0.0" + assert router_args.request_id_headers == ["x-request-id", "x-trace-id"] + assert router_args.request_timeout_secs == 1200 + assert router_args.max_concurrent_requests == 512 + assert router_args.queue_size == 200 + assert router_args.queue_timeout_secs == 120 + assert router_args.rate_limit_tokens_per_second == 100 + assert router_args.cors_allowed_origins == ["http://localhost:3000"] + + # Test retry configuration + assert router_args.retry_max_retries == 3 + assert router_args.retry_initial_backoff_ms == 100 + assert router_args.retry_max_backoff_ms == 10000 + assert router_args.retry_backoff_multiplier == 2.0 + assert router_args.retry_jitter_factor == 0.1 + + # Test circuit breaker configuration + assert router_args.cb_failure_threshold == 5 + assert router_args.cb_success_threshold == 2 + assert router_args.cb_timeout_duration_secs == 30 + assert router_args.cb_window_duration_secs == 60 + assert router_args.disable_retries is False + assert router_args.disable_circuit_breaker is False + + # Test health check configuration + assert router_args.health_failure_threshold == 2 + assert router_args.health_success_threshold == 1 + assert router_args.health_check_timeout_secs == 3 + assert router_args.health_check_interval_secs == 30 + assert router_args.health_check_endpoint == "/healthz" + + # Note: model_path and tokenizer_path are not available in current RouterArgs + + def test_from_cli_args_pd_mode(self): + """Test creating RouterArgs from CLI arguments in PD mode.""" + args = SimpleNamespace( + host="127.0.0.1", + port=30000, + worker_urls=[], + policy="cache_aware", + prefill=[ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ], + decode=[["http://decode1:8001"], ["http://decode2:8001"]], + router_prefill=[ + ["http://prefill1:8000", "9000"], + ["http://prefill2:8000", "none"], + ], + router_decode=[["http://decode1:8001"], ["http://decode2:8001"]], + router_policy="cache_aware", + router_pd_disaggregation=True, + router_prefill_policy="power_of_two", + router_decode_policy="round_robin", + # Include all required fields with defaults + router_worker_startup_timeout_secs=600, + router_worker_startup_check_interval=30, + router_cache_threshold=0.3, + router_balance_abs_threshold=64, + router_balance_rel_threshold=1.5, + router_eviction_interval=120, + router_max_tree_size=2**26, + router_max_payload_size=512 * 1024 * 1024, + router_dp_aware=False, + router_api_key=None, + router_log_dir=None, + router_log_level=None, + router_service_discovery=False, + router_selector=None, + router_service_discovery_port=80, + router_service_discovery_namespace=None, + router_prefill_selector=None, + router_decode_selector=None, + router_prometheus_port=None, + router_prometheus_host=None, + router_request_id_headers=None, + router_request_timeout_secs=1800, + router_max_concurrent_requests=256, + router_queue_size=100, + router_queue_timeout_secs=60, + router_rate_limit_tokens_per_second=None, + router_cors_allowed_origins=[], + router_retry_max_retries=5, + router_retry_initial_backoff_ms=50, + router_retry_max_backoff_ms=30000, + router_retry_backoff_multiplier=1.5, + router_retry_jitter_factor=0.2, + router_cb_failure_threshold=10, + router_cb_success_threshold=3, + router_cb_timeout_duration_secs=60, + router_cb_window_duration_secs=120, + router_disable_retries=False, + router_disable_circuit_breaker=False, + router_health_failure_threshold=3, + router_health_success_threshold=2, + router_health_check_timeout_secs=5, + router_health_check_interval_secs=60, + router_health_check_endpoint="/health", + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=True) + + # Test PD configuration + assert router_args.pd_disaggregation is True + assert router_args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert router_args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert router_args.prefill_policy == "power_of_two" + assert router_args.decode_policy == "round_robin" + assert router_args.policy == "cache_aware" # Main policy still set + + def test_from_cli_args_without_prefix(self): + """Test creating RouterArgs from CLI arguments without router prefix.""" + args = SimpleNamespace( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="random", + prefill=None, + decode=None, + pd_disaggregation=False, + prefill_policy=None, + decode_policy=None, + worker_startup_timeout_secs=600, + worker_startup_check_interval=30, + cache_threshold=0.3, + balance_abs_threshold=64, + balance_rel_threshold=1.5, + eviction_interval=120, + max_tree_size=2**26, + max_payload_size=512 * 1024 * 1024, + dp_aware=False, + api_key=None, + log_dir=None, + log_level=None, + service_discovery=False, + selector=None, + service_discovery_port=80, + service_discovery_namespace=None, + prefill_selector=None, + decode_selector=None, + prometheus_port=None, + prometheus_host=None, + request_id_headers=None, + request_timeout_secs=1800, + max_concurrent_requests=256, + queue_size=100, + queue_timeout_secs=60, + rate_limit_tokens_per_second=None, + cors_allowed_origins=[], + retry_max_retries=5, + retry_initial_backoff_ms=50, + retry_max_backoff_ms=30000, + retry_backoff_multiplier=1.5, + retry_jitter_factor=0.2, + cb_failure_threshold=10, + cb_success_threshold=3, + cb_timeout_duration_secs=60, + cb_window_duration_secs=120, + disable_retries=False, + disable_circuit_breaker=False, + health_failure_threshold=3, + health_success_threshold=2, + health_check_timeout_secs=5, + health_check_interval_secs=60, + health_check_endpoint="/health", + model_path=None, + tokenizer_path=None, + ) + + router_args = RouterArgs.from_cli_args(args, use_router_prefix=False) + + assert router_args.host == "127.0.0.1" + assert router_args.port == 30000 + assert router_args.worker_urls == ["http://worker1:8000"] + assert router_args.policy == "random" + assert router_args.pd_disaggregation is False + + +class TestPolicyFromStr: + """Test policy string to enum conversion.""" + + def test_valid_policies(self): + """Test conversion of valid policy strings.""" + from sglang_router_rs import PolicyType + + assert policy_from_str("random") == PolicyType.Random + assert policy_from_str("round_robin") == PolicyType.RoundRobin + assert policy_from_str("cache_aware") == PolicyType.CacheAware + assert policy_from_str("power_of_two") == PolicyType.PowerOfTwo + + def test_invalid_policy(self): + """Test conversion of invalid policy string.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + +class TestParseRouterArgs: + """Test the parse_router_args function.""" + + def test_parse_basic_args(self): + """Test parsing basic router arguments.""" + args = [ + "--host", + "0.0.0.0", + "--port", + "30001", + "--worker-urls", + "http://worker1:8000", + "http://worker2:8000", + "--policy", + "round_robin", + ] + + router_args = parse_router_args(args) + + assert router_args.host == "0.0.0.0" + assert router_args.port == 30001 + assert router_args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert router_args.policy == "round_robin" + + def test_parse_pd_args(self): + """Test parsing PD disaggregated mode arguments.""" + args = [ + "--pd-disaggregation", + "--prefill", + "http://prefill1:8000", + "9000", + "--prefill", + "http://prefill2:8000", + "none", + "--decode", + "http://decode1:8001", + "--decode", + "http://decode2:8001", + "--prefill-policy", + "power_of_two", + "--decode-policy", + "round_robin", + ] + + router_args = parse_router_args(args) + + assert router_args.pd_disaggregation is True + assert router_args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert router_args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert router_args.prefill_policy == "power_of_two" + assert router_args.decode_policy == "round_robin" + + def test_parse_service_discovery_args(self): + """Test parsing service discovery arguments.""" + args = [ + "--service-discovery", + "--selector", + "app=worker", + "env=prod", + "--service-discovery-port", + "8080", + "--service-discovery-namespace", + "default", + ] + + router_args = parse_router_args(args) + + assert router_args.service_discovery is True + assert router_args.selector == {"app": "worker", "env": "prod"} + assert router_args.service_discovery_port == 8080 + assert router_args.service_discovery_namespace == "default" + + def test_parse_retry_and_circuit_breaker_args(self): + """Test parsing retry and circuit breaker arguments.""" + args = [ + "--retry-max-retries", + "3", + "--retry-initial-backoff-ms", + "100", + "--retry-max-backoff-ms", + "10000", + "--retry-backoff-multiplier", + "2.0", + "--retry-jitter-factor", + "0.1", + "--disable-retries", + "--cb-failure-threshold", + "5", + "--cb-success-threshold", + "2", + "--cb-timeout-duration-secs", + "30", + "--cb-window-duration-secs", + "60", + "--disable-circuit-breaker", + ] + + router_args = parse_router_args(args) + + # Test retry configuration + assert router_args.retry_max_retries == 3 + assert router_args.retry_initial_backoff_ms == 100 + assert router_args.retry_max_backoff_ms == 10000 + assert router_args.retry_backoff_multiplier == 2.0 + assert router_args.retry_jitter_factor == 0.1 + assert router_args.disable_retries is True + + # Test circuit breaker configuration + assert router_args.cb_failure_threshold == 5 + assert router_args.cb_success_threshold == 2 + assert router_args.cb_timeout_duration_secs == 30 + assert router_args.cb_window_duration_secs == 60 + assert router_args.disable_circuit_breaker is True + + def test_parse_rate_limiting_args(self): + """Test parsing rate limiting arguments.""" + args = [ + "--max-concurrent-requests", + "512", + "--queue-size", + "200", + "--queue-timeout-secs", + "120", + "--rate-limit-tokens-per-second", + "100", + ] + + router_args = parse_router_args(args) + + assert router_args.max_concurrent_requests == 512 + assert router_args.queue_size == 200 + assert router_args.queue_timeout_secs == 120 + assert router_args.rate_limit_tokens_per_second == 100 + + def test_parse_health_check_args(self): + """Test parsing health check arguments.""" + args = [ + "--health-failure-threshold", + "2", + "--health-success-threshold", + "1", + "--health-check-timeout-secs", + "3", + "--health-check-interval-secs", + "30", + "--health-check-endpoint", + "/healthz", + ] + + router_args = parse_router_args(args) + + assert router_args.health_failure_threshold == 2 + assert router_args.health_success_threshold == 1 + assert router_args.health_check_timeout_secs == 3 + assert router_args.health_check_interval_secs == 30 + assert router_args.health_check_endpoint == "/healthz" + + def test_parse_cors_args(self): + """Test parsing CORS arguments.""" + args = [ + "--cors-allowed-origins", + "http://localhost:3000", + "https://example.com", + ] + + router_args = parse_router_args(args) + + assert router_args.cors_allowed_origins == [ + "http://localhost:3000", + "https://example.com", + ] + + def test_parse_tokenizer_args(self): + """Test parsing tokenizer arguments.""" + # Note: model-path and tokenizer-path arguments are not available in current implementation + # This test is skipped until those arguments are added + pytest.skip("Tokenizer arguments not available in current implementation") + + def test_parse_invalid_args(self): + """Test parsing invalid arguments.""" + # Test invalid policy + with pytest.raises(SystemExit): + parse_router_args(["--policy", "invalid_policy"]) + + # Test invalid bootstrap port + with pytest.raises(ValueError, match="Invalid bootstrap port"): + parse_router_args( + [ + "--pd-disaggregation", + "--prefill", + "http://prefill1:8000", + "invalid_port", + ] + ) + + def test_help_output(self): + """Test that help output is generated correctly.""" + with pytest.raises(SystemExit) as exc_info: + parse_router_args(["--help"]) + + # SystemExit with code 0 indicates help was displayed + assert exc_info.value.code == 0 diff --git a/sgl-router/py_test/unit/test_router_config.py b/sgl-router/py_test/unit/test_router_config.py new file mode 100644 index 000000000..ed0d9db4b --- /dev/null +++ b/sgl-router/py_test/unit/test_router_config.py @@ -0,0 +1,421 @@ +""" +Unit tests for router configuration validation and setup. + +These tests focus on testing the router configuration logic in isolation, +including validation of configuration parameters and their interactions. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router +from sglang_router.router import policy_from_str +from sglang_router_rs import PolicyType + + +class TestRouterConfigValidation: + """Test router configuration validation logic.""" + + def test_valid_basic_config(self): + """Test that a valid basic configuration passes validation.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="cache_aware", + ) + + # Should not raise any exceptions + assert args.host == "127.0.0.1" + assert args.port == 30000 + assert args.worker_urls == ["http://worker1:8000", "http://worker2:8000"] + assert args.policy == "cache_aware" + + def test_valid_pd_config(self): + """Test that a valid PD configuration passes validation.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + pd_disaggregation=True, + prefill_urls=[ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ], + decode_urls=["http://decode1:8001", "http://decode2:8001"], + policy="cache_aware", + ) + + assert args.pd_disaggregation is True + assert args.prefill_urls == [ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ] + assert args.decode_urls == ["http://decode1:8001", "http://decode2:8001"] + assert args.policy == "cache_aware" + + def test_pd_config_without_urls_raises_error(self): + """Test that PD mode without URLs raises validation error.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + # This should raise an error when trying to launch + with pytest.raises( + ValueError, match="PD disaggregation mode requires --prefill" + ): + launch_router(args) + + def test_pd_config_with_service_discovery_allows_empty_urls(self): + """Test that PD mode with service discovery allows empty URLs.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error when service discovery is enabled + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_regular_mode_without_workers_allows_empty_urls(self): + """Test that regular mode allows empty worker URLs.""" + args = RouterArgs(worker_urls=[], service_discovery=False) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_cache_threshold_validation(self): + """Test cache threshold validation.""" + # Valid cache threshold + args = RouterArgs(cache_threshold=0.5) + assert args.cache_threshold == 0.5 + + # Edge cases + args = RouterArgs(cache_threshold=0.0) + assert args.cache_threshold == 0.0 + + args = RouterArgs(cache_threshold=1.0) + assert args.cache_threshold == 1.0 + + def test_balance_threshold_validation(self): + """Test load balancing threshold validation.""" + # Valid thresholds + args = RouterArgs(balance_abs_threshold=64, balance_rel_threshold=1.5) + assert args.balance_abs_threshold == 64 + assert args.balance_rel_threshold == 1.5 + + # Edge cases + args = RouterArgs(balance_abs_threshold=0, balance_rel_threshold=1.0) + assert args.balance_abs_threshold == 0 + assert args.balance_rel_threshold == 1.0 + + def test_timeout_validation(self): + """Test timeout parameter validation.""" + # Valid timeouts + args = RouterArgs( + worker_startup_timeout_secs=600, + worker_startup_check_interval=30, + request_timeout_secs=1800, + queue_timeout_secs=60, + ) + assert args.worker_startup_timeout_secs == 600 + assert args.worker_startup_check_interval == 30 + assert args.request_timeout_secs == 1800 + assert args.queue_timeout_secs == 60 + + def test_retry_config_validation(self): + """Test retry configuration validation.""" + # Valid retry config + args = RouterArgs( + retry_max_retries=5, + retry_initial_backoff_ms=50, + retry_max_backoff_ms=30000, + retry_backoff_multiplier=1.5, + retry_jitter_factor=0.2, + disable_retries=False, + ) + assert args.retry_max_retries == 5 + assert args.retry_initial_backoff_ms == 50 + assert args.retry_max_backoff_ms == 30000 + assert args.retry_backoff_multiplier == 1.5 + assert args.retry_jitter_factor == 0.2 + assert args.disable_retries is False + + def test_circuit_breaker_config_validation(self): + """Test circuit breaker configuration validation.""" + # Valid circuit breaker config + args = RouterArgs( + cb_failure_threshold=10, + cb_success_threshold=3, + cb_timeout_duration_secs=60, + cb_window_duration_secs=120, + disable_circuit_breaker=False, + ) + assert args.cb_failure_threshold == 10 + assert args.cb_success_threshold == 3 + assert args.cb_timeout_duration_secs == 60 + assert args.cb_window_duration_secs == 120 + assert args.disable_circuit_breaker is False + + def test_health_check_config_validation(self): + """Test health check configuration validation.""" + # Valid health check config + args = RouterArgs( + health_failure_threshold=3, + health_success_threshold=2, + health_check_timeout_secs=5, + health_check_interval_secs=60, + health_check_endpoint="/health", + ) + assert args.health_failure_threshold == 3 + assert args.health_success_threshold == 2 + assert args.health_check_timeout_secs == 5 + assert args.health_check_interval_secs == 60 + assert args.health_check_endpoint == "/health" + + def test_rate_limiting_config_validation(self): + """Test rate limiting configuration validation.""" + # Valid rate limiting config + args = RouterArgs( + max_concurrent_requests=256, + queue_size=100, + queue_timeout_secs=60, + rate_limit_tokens_per_second=100, + ) + assert args.max_concurrent_requests == 256 + assert args.queue_size == 100 + assert args.queue_timeout_secs == 60 + assert args.rate_limit_tokens_per_second == 100 + + def test_service_discovery_config_validation(self): + """Test service discovery configuration validation.""" + # Valid service discovery config + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + assert args.service_discovery is True + assert args.selector == {"app": "worker", "env": "prod"} + assert args.service_discovery_port == 8080 + assert args.service_discovery_namespace == "default" + + def test_pd_service_discovery_config_validation(self): + """Test PD service discovery configuration validation.""" + # Valid PD service discovery config + args = RouterArgs( + pd_disaggregation=True, + service_discovery=True, + prefill_selector={"app": "prefill"}, + decode_selector={"app": "decode"}, + bootstrap_port_annotation="sglang.ai/bootstrap-port", + ) + assert args.pd_disaggregation is True + assert args.service_discovery is True + assert args.prefill_selector == {"app": "prefill"} + assert args.decode_selector == {"app": "decode"} + assert args.bootstrap_port_annotation == "sglang.ai/bootstrap-port" + + def test_prometheus_config_validation(self): + """Test Prometheus configuration validation.""" + # Valid Prometheus config + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + assert args.prometheus_port == 29000 + assert args.prometheus_host == "127.0.0.1" + + def test_cors_config_validation(self): + """Test CORS configuration validation.""" + # Valid CORS config + args = RouterArgs( + cors_allowed_origins=["http://localhost:3000", "https://example.com"] + ) + assert args.cors_allowed_origins == [ + "http://localhost:3000", + "https://example.com", + ] + + def test_tokenizer_config_validation(self): + """Test tokenizer configuration validation.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + def test_dp_aware_config_validation(self): + """Test data parallelism aware configuration validation.""" + # Valid DP aware config + args = RouterArgs(dp_aware=True, api_key="test-api-key") + assert args.dp_aware is True + assert args.api_key == "test-api-key" + + def test_request_id_headers_validation(self): + """Test request ID headers configuration validation.""" + # Valid request ID headers config + args = RouterArgs( + request_id_headers=["x-request-id", "x-trace-id", "x-correlation-id"] + ) + assert args.request_id_headers == [ + "x-request-id", + "x-trace-id", + "x-correlation-id", + ] + + def test_policy_consistency_validation(self): + """Test policy consistency validation in PD mode.""" + # Test with both prefill and decode policies specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy="round_robin", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_policy_fallback_validation(self): + """Test policy fallback validation in PD mode.""" + # Test with only prefill policy specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy=None, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_policy_enum_conversion(self): + """Test policy string to enum conversion.""" + # Test all valid policy conversions + assert policy_from_str("random") == PolicyType.Random + assert policy_from_str("round_robin") == PolicyType.RoundRobin + assert policy_from_str("cache_aware") == PolicyType.CacheAware + assert policy_from_str("power_of_two") == PolicyType.PowerOfTwo + + def test_invalid_policy_enum_conversion(self): + """Test invalid policy string to enum conversion.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + def test_config_immutability(self): + """Test that configuration objects are properly immutable.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + # Test that we can't modify the configuration after creation + # (This is more of a design test - dataclasses are mutable by default) + original_host = args.host + args.host = "0.0.0.0" + assert args.host == "0.0.0.0" # Dataclasses are mutable + assert args.host != original_host + + def test_config_defaults_consistency(self): + """Test that configuration defaults are consistent.""" + args1 = RouterArgs() + args2 = RouterArgs() + + # Both instances should have the same defaults + assert args1.host == args2.host + assert args1.port == args2.port + assert args1.policy == args2.policy + assert args1.worker_urls == args2.worker_urls + assert args1.pd_disaggregation == args2.pd_disaggregation + + def test_config_serialization(self): + """Test that configuration can be serialized/deserialized.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + cache_threshold=0.5, + ) + + # Test that we can access all attributes + assert hasattr(args, "host") + assert hasattr(args, "port") + assert hasattr(args, "worker_urls") + assert hasattr(args, "policy") + assert hasattr(args, "cache_threshold") + + def test_config_with_none_values(self): + """Test configuration with None values.""" + args = RouterArgs( + api_key=None, + log_dir=None, + log_level=None, + prometheus_port=None, + prometheus_host=None, + request_id_headers=None, + rate_limit_tokens_per_second=None, + service_discovery_namespace=None, + ) + + # All None values should be preserved + assert args.api_key is None + assert args.log_dir is None + assert args.log_level is None + assert args.prometheus_port is None + assert args.prometheus_host is None + assert args.request_id_headers is None + assert args.rate_limit_tokens_per_second is None + assert args.service_discovery_namespace is None + + def test_config_with_empty_lists(self): + """Test configuration with empty lists.""" + args = RouterArgs( + worker_urls=[], prefill_urls=[], decode_urls=[], cors_allowed_origins=[] + ) + + # All empty lists should be preserved + assert args.worker_urls == [] + assert args.prefill_urls == [] + assert args.decode_urls == [] + assert args.cors_allowed_origins == [] + + def test_config_with_empty_dicts(self): + """Test configuration with empty dictionaries.""" + args = RouterArgs(selector={}, prefill_selector={}, decode_selector={}) + + # All empty dictionaries should be preserved + assert args.selector == {} + assert args.prefill_selector == {} + assert args.decode_selector == {} diff --git a/sgl-router/py_test/unit/test_startup_sequence.py b/sgl-router/py_test/unit/test_startup_sequence.py new file mode 100644 index 000000000..133c7eb16 --- /dev/null +++ b/sgl-router/py_test/unit/test_startup_sequence.py @@ -0,0 +1,1053 @@ +""" +Unit tests for startup sequence logic in sglang_router. + +These tests focus on testing the startup sequence logic in isolation, +including router initialization, configuration validation, and startup flow. +""" + +import logging +from types import SimpleNamespace +from unittest.mock import MagicMock, call, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router +from sglang_router.router import policy_from_str + + +# Local helper mirroring the router logger setup used in production +def setup_logger(): + logger = logging.getLogger("router") + logger.setLevel(logging.INFO) + if not logger.handlers: + formatter = logging.Formatter( + "[Router (Python)] %(asctime)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logger.addHandler(handler) + return logger + + +from sglang_router_rs import PolicyType + + +class TestSetupLogger: + """Test logger setup functionality.""" + + def test_setup_logger_returns_logger(self): + """Test that setup_logger returns a logger instance.""" + logger = setup_logger() + + assert isinstance(logger, logging.Logger) + assert logger.name == "router" + assert logger.level == logging.INFO + + def test_setup_logger_has_handler(self): + """Test that setup_logger configures a handler.""" + logger = setup_logger() + + assert len(logger.handlers) > 0 + handler = logger.handlers[0] + assert isinstance(handler, logging.StreamHandler) + + def test_setup_logger_has_formatter(self): + """Test that setup_logger configures a formatter.""" + logger = setup_logger() + + handler = logger.handlers[0] + formatter = handler.formatter + + assert formatter is not None + assert "[Router (Python)]" in formatter._fmt + + def test_setup_logger_multiple_calls(self): + """Test that multiple calls to setup_logger work correctly.""" + logger1 = setup_logger() + logger2 = setup_logger() + + # Should return the same logger instance + assert logger1 is logger2 + + +class TestPolicyFromStr: + """Test policy string to enum conversion in startup context.""" + + def test_policy_conversion_in_startup(self): + """Test policy conversion during startup sequence.""" + # Test all valid policies + policies = ["random", "round_robin", "cache_aware", "power_of_two"] + expected_enums = [ + PolicyType.Random, + PolicyType.RoundRobin, + PolicyType.CacheAware, + PolicyType.PowerOfTwo, + ] + + for policy_str, expected_enum in zip(policies, expected_enums): + result = policy_from_str(policy_str) + assert result == expected_enum + + def test_invalid_policy_in_startup(self): + """Test handling of invalid policy during startup.""" + with pytest.raises(KeyError): + policy_from_str("invalid_policy") + + +class TestRouterInitialization: + """Test router initialization logic.""" + + def test_router_initialization_basic(self): + """Test basic router initialization.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + # capture needed fields from RouterArgs + captured_args.update( + dict( + host=router_args.host, + port=router_args.port, + worker_urls=router_args.worker_urls, + policy=policy_from_str(router_args.policy), + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called and captured fields match + router_mod.from_args.assert_called_once() + assert captured_args["host"] == "127.0.0.1" + assert captured_args["port"] == 30000 + assert captured_args["worker_urls"] == ["http://worker1:8000"] + assert captured_args["policy"] == PolicyType.CacheAware + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_pd_mode(self): + """Test router initialization in PD mode.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + policy="power_of_two", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + pd_disaggregation=router_args.pd_disaggregation, + prefill_urls=router_args.prefill_urls, + decode_urls=router_args.decode_urls, + policy=policy_from_str(router_args.policy), + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called with PD parameters + router_mod.from_args.assert_called_once() + assert captured_args["pd_disaggregation"] is True + assert captured_args["prefill_urls"] == [("http://prefill1:8000", 9000)] + assert captured_args["decode_urls"] == ["http://decode1:8001"] + assert captured_args["policy"] == PolicyType.PowerOfTwo + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_service_discovery(self): + """Test router initialization with service discovery.""" + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + service_discovery=router_args.service_discovery, + selector=router_args.selector, + service_discovery_port=router_args.service_discovery_port, + service_discovery_namespace=router_args.service_discovery_namespace, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify Router.from_args was called with service discovery parameters + router_mod.from_args.assert_called_once() + assert captured_args["service_discovery"] is True + assert captured_args["selector"] == {"app": "worker", "env": "prod"} + assert captured_args["service_discovery_port"] == 8080 + assert captured_args["service_discovery_namespace"] == "default" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_retry_config(self): + """Test router initialization with retry configuration.""" + args = RouterArgs( + retry_max_retries=3, + retry_initial_backoff_ms=100, + retry_max_backoff_ms=10000, + retry_backoff_multiplier=2.0, + retry_jitter_factor=0.1, + disable_retries=False, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + retry_max_retries=router_args.retry_max_retries, + retry_initial_backoff_ms=router_args.retry_initial_backoff_ms, + retry_max_backoff_ms=router_args.retry_max_backoff_ms, + retry_backoff_multiplier=router_args.retry_backoff_multiplier, + retry_jitter_factor=router_args.retry_jitter_factor, + disable_retries=router_args.disable_retries, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with retry parameters + router_mod.from_args.assert_called_once() + assert captured_args["retry_max_retries"] == 3 + assert captured_args["retry_initial_backoff_ms"] == 100 + assert captured_args["retry_max_backoff_ms"] == 10000 + assert captured_args["retry_backoff_multiplier"] == 2.0 + assert captured_args["retry_jitter_factor"] == 0.1 + assert captured_args["disable_retries"] is False + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_circuit_breaker_config(self): + """Test router initialization with circuit breaker configuration.""" + args = RouterArgs( + cb_failure_threshold=5, + cb_success_threshold=2, + cb_timeout_duration_secs=30, + cb_window_duration_secs=60, + disable_circuit_breaker=False, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + cb_failure_threshold=router_args.cb_failure_threshold, + cb_success_threshold=router_args.cb_success_threshold, + cb_timeout_duration_secs=router_args.cb_timeout_duration_secs, + cb_window_duration_secs=router_args.cb_window_duration_secs, + disable_circuit_breaker=router_args.disable_circuit_breaker, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with circuit breaker parameters + router_mod.from_args.assert_called_once() + assert captured_args["cb_failure_threshold"] == 5 + assert captured_args["cb_success_threshold"] == 2 + assert captured_args["cb_timeout_duration_secs"] == 30 + assert captured_args["cb_window_duration_secs"] == 60 + assert captured_args["disable_circuit_breaker"] is False + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_rate_limiting_config(self): + """Test router initialization with rate limiting configuration.""" + args = RouterArgs( + max_concurrent_requests=512, + queue_size=200, + queue_timeout_secs=120, + rate_limit_tokens_per_second=100, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + max_concurrent_requests=router_args.max_concurrent_requests, + queue_size=router_args.queue_size, + queue_timeout_secs=router_args.queue_timeout_secs, + rate_limit_tokens_per_second=router_args.rate_limit_tokens_per_second, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with rate limiting parameters + router_mod.from_args.assert_called_once() + assert captured_args["max_concurrent_requests"] == 512 + assert captured_args["queue_size"] == 200 + assert captured_args["queue_timeout_secs"] == 120 + assert captured_args["rate_limit_tokens_per_second"] == 100 + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_health_check_config(self): + """Test router initialization with health check configuration.""" + args = RouterArgs( + health_failure_threshold=2, + health_success_threshold=1, + health_check_timeout_secs=3, + health_check_interval_secs=30, + health_check_endpoint="/healthz", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + health_failure_threshold=router_args.health_failure_threshold, + health_success_threshold=router_args.health_success_threshold, + health_check_timeout_secs=router_args.health_check_timeout_secs, + health_check_interval_secs=router_args.health_check_interval_secs, + health_check_endpoint=router_args.health_check_endpoint, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with health check parameters + router_mod.from_args.assert_called_once() + assert captured_args["health_failure_threshold"] == 2 + assert captured_args["health_success_threshold"] == 1 + assert captured_args["health_check_timeout_secs"] == 3 + assert captured_args["health_check_interval_secs"] == 30 + assert captured_args["health_check_endpoint"] == "/healthz" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_prometheus_config(self): + """Test router initialization with Prometheus configuration.""" + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + prometheus_port=router_args.prometheus_port, + prometheus_host=router_args.prometheus_host, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with Prometheus parameters + router_mod.from_args.assert_called_once() + assert captured_args["prometheus_port"] == 29000 + assert captured_args["prometheus_host"] == "127.0.0.1" + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_cors_config(self): + """Test router initialization with CORS configuration.""" + args = RouterArgs( + cors_allowed_origins=["http://localhost:3000", "https://example.com"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict(cors_allowed_origins=router_args.cors_allowed_origins) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify router was created with CORS parameters + router_mod.from_args.assert_called_once() + assert captured_args["cors_allowed_origins"] == [ + "http://localhost:3000", + "https://example.com", + ] + + # Verify router.start() was called + mock_router_instance.start.assert_called_once() + + # Function returns None; ensure start was invoked + + def test_router_initialization_with_tokenizer_config(self): + """Test router initialization with tokenizer configuration.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + +class TestStartupValidation: + """Test startup validation logic.""" + + def test_pd_mode_validation_during_startup(self): + """Test PD mode validation during startup.""" + # PD mode without URLs should fail + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + with pytest.raises( + ValueError, match="PD disaggregation mode requires --prefill" + ): + launch_router(args) + + def test_pd_mode_with_service_discovery_validation(self): + """Test PD mode with service discovery validation during startup.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + result = launch_router(args) + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_warning_during_startup(self): + """Test policy warning during startup in PD mode.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log warning about policy usage + mock_logger.warning.assert_called_once() + warning_call = mock_logger.warning.call_args[0][0] + assert ( + "Both --prefill-policy and --decode-policy are specified" + in warning_call + ) + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_info_during_startup(self): + """Test policy info logging during startup in PD mode.""" + # Test with only prefill policy specified + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy="power_of_two", + decode_policy=None, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log info about policy usage + mock_logger.info.assert_called_once() + info_call = mock_logger.info.call_args[0][0] + assert "Using --prefill-policy 'power_of_two'" in info_call + assert "and --policy 'cache_aware'" in info_call + + # Should create router instance + router_mod.from_args.assert_called_once() + + def test_policy_info_decode_only_during_startup(self): + """Test policy info logging during startup with only decode policy specified.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + prefill_policy=None, + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # The policy messages are emitted by router_args logger + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Should log info about policy usage + mock_logger.info.assert_called_once() + info_call = mock_logger.info.call_args[0][0] + assert "Using --policy 'cache_aware'" in info_call + assert "and --decode-policy 'round_robin'" in info_call + + # Should create router instance + router_mod.from_args.assert_called_once() + + +class TestStartupErrorHandling: + """Test startup error handling logic.""" + + def test_router_creation_error_handling(self): + """Test error handling when router creation fails.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + # Simulate router creation failure in from_args + router_mod.from_args = MagicMock( + side_effect=Exception("Router creation failed") + ) + + with patch("sglang_router.launch_router.logger") as mock_logger: + with pytest.raises(Exception, match="Router creation failed"): + launch_router(args) + + # Should log error + mock_logger.error.assert_called_once() + error_call = mock_logger.error.call_args[0][0] + assert "Error starting router: Router creation failed" in error_call + + def test_router_start_error_handling(self): + """Test error handling when router start fails.""" + args = RouterArgs( + host="127.0.0.1", port=30000, worker_urls=["http://worker1:8000"] + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + # Simulate router start failure + mock_router_instance.start.side_effect = Exception("Router start failed") + + with patch("sglang_router.launch_router.logger") as mock_logger: + with pytest.raises(Exception, match="Router start failed"): + launch_router(args) + + # Should log error + mock_logger.error.assert_called_once() + error_call = mock_logger.error.call_args[0][0] + assert "Error starting router: Router start failed" in error_call + + +# --- Added unit tests for Router wrapper and launch_server helpers --- + + +def _install_sglang_stubs(monkeypatch): + """Install lightweight stubs for sglang.srt to avoid heavy deps during unit tests.""" + import sys + import types + + sglang_mod = types.ModuleType("sglang") + srt_mod = types.ModuleType("sglang.srt") + entry_mod = types.ModuleType("sglang.srt.entrypoints") + http_server_mod = types.ModuleType("sglang.srt.entrypoints.http_server") + server_args_mod = types.ModuleType("sglang.srt.server_args") + utils_mod = types.ModuleType("sglang.srt.utils") + + def launch_server(_args): + return None + + class ServerArgs: + # Minimal fields used by launch_server_process + def __init__(self): + self.port = 0 + self.base_gpu_id = 0 + self.dp_size = 1 + self.tp_size = 1 + + @staticmethod + def add_cli_args(_parser): + return None + + @staticmethod + def from_cli_args(_args): + sa = ServerArgs() + if hasattr(_args, "dp_size"): + sa.dp_size = _args.dp_size + if hasattr(_args, "tp_size"): + sa.tp_size = _args.tp_size + if hasattr(_args, "host"): + sa.host = _args.host + else: + sa.host = "127.0.0.1" + return sa + + def is_port_available(_port: int) -> bool: + return True + + http_server_mod.launch_server = launch_server + server_args_mod.ServerArgs = ServerArgs + utils_mod.is_port_available = is_port_available + + # Also stub external deps imported at module top-level + def _dummy_get(*_a, **_k): + raise NotImplementedError + + requests_stub = types.SimpleNamespace( + exceptions=types.SimpleNamespace(RequestException=Exception), get=_dummy_get + ) + setproctitle_stub = types.SimpleNamespace(setproctitle=lambda *_a, **_k: None) + + monkeypatch.setitem(sys.modules, "requests", requests_stub) + monkeypatch.setitem(sys.modules, "setproctitle", setproctitle_stub) + + monkeypatch.setitem(sys.modules, "sglang", sglang_mod) + monkeypatch.setitem(sys.modules, "sglang.srt", srt_mod) + monkeypatch.setitem(sys.modules, "sglang.srt.entrypoints", entry_mod) + monkeypatch.setitem( + sys.modules, "sglang.srt.entrypoints.http_server", http_server_mod + ) + monkeypatch.setitem(sys.modules, "sglang.srt.server_args", server_args_mod) + monkeypatch.setitem(sys.modules, "sglang.srt.utils", utils_mod) + + +def test_router_defaults_and_start(monkeypatch): + """Router wrapper: defaults normalization and start() call. + + Mocks the Rust-backed _Router to avoid native deps. + """ + from sglang_router import router as router_mod + + captured = {} + + class FakeRouter: + def __init__(self, **kwargs): + captured.update(kwargs) + + def start(self): + captured["started"] = True + + monkeypatch.setattr(router_mod, "_Router", FakeRouter, raising=True) + + from sglang_router.router_args import RouterArgs as _RouterArgs + + Router = router_mod.Router + args = _RouterArgs( + worker_urls=["http://w1:8000"], + policy="round_robin", + selector=None, + prefill_selector=None, + decode_selector=None, + cors_allowed_origins=None, + ) + + r = Router.from_args(args) + + # Defaults preserved/normalized by Router.from_args + assert captured["selector"] is None + assert captured["prefill_selector"] is None + assert captured["decode_selector"] is None + assert captured["cors_allowed_origins"] is None + assert captured["worker_urls"] == ["http://w1:8000"] + from sglang_router_rs import PolicyType + + assert captured["policy"] == PolicyType.RoundRobin + + r.start() + assert captured.get("started") is True + + +def test_find_available_ports_and_wait_health(monkeypatch): + """launch_server helpers: port finding and health waiting with transient error.""" + _install_sglang_stubs(monkeypatch) + import importlib + + ls = importlib.import_module("sglang_router.launch_server") + + # Deterministic increments + monkeypatch.setattr(ls.random, "randint", lambda a, b: 100) + ports = ls.find_available_ports(30000, 3) + assert ports == [30000, 30100, 30200] + + calls = {"n": 0} + + class Ok: + status_code = 200 + + def fake_get(_url, timeout=5): + calls["n"] += 1 + if calls["n"] == 1: + raise ls.requests.exceptions.RequestException("boom") + return Ok() + + monkeypatch.setattr(ls.requests, "get", fake_get) + monkeypatch.setattr(ls.time, "sleep", lambda _s: None) + base = {"t": 0.0} + monkeypatch.setattr( + ls.time, + "perf_counter", + lambda: (base.__setitem__("t", base["t"] + 0.1) or base["t"]), + ) + + assert ls.wait_for_server_health("127.0.0.1", 12345, timeout=1) + + +def test_launch_server_process_and_cleanup(monkeypatch): + """launch_server: process creation args and cleanup SIGTERM/SIGKILL logic.""" + _install_sglang_stubs(monkeypatch) + import importlib + + ls = importlib.import_module("sglang_router.launch_server") + + created = {} + + class FakeProcess: + def __init__(self, target, args): + created["target"] = target + created["args"] = args + self.pid = 4242 + self._alive = True + + def start(self): + created["started"] = True + + def join(self, timeout=None): + return None + + def is_alive(self): + return self._alive + + monkeypatch.setattr(ls.mp, "Process", FakeProcess) + + import sys as _sys + + SA = _sys.modules["sglang.srt.server_args"].ServerArgs + sa = SA() + sa.tp_size = 2 + + proc = ls.launch_server_process(sa, worker_port=31001, dp_id=3) + assert created.get("started") is True + targ, targ_args = created["target"], created["args"] + assert targ is ls.run_server + passed_sa = targ_args[0] + assert passed_sa.port == 31001 + assert passed_sa.base_gpu_id == 3 * 2 + assert passed_sa.dp_size == 1 + + # cleanup_processes + p1 = FakeProcess(target=None, args=()) + p1._alive = False + p2 = FakeProcess(target=None, args=()) + p2._alive = True + + calls = [] + + def fake_killpg(pid, sig): + calls.append((pid, sig)) + + monkeypatch.setattr(ls.os, "killpg", fake_killpg) + + ls.cleanup_processes([p1, p2]) + + import signal as _sig + + assert (p1.pid, _sig.SIGTERM) in calls and (p2.pid, _sig.SIGTERM) in calls + assert (p2.pid, _sig.SIGKILL) in calls + + def test_validation_error_handling(self): + """Test error handling when validation fails.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + with patch("sglang_router.launch_router.logger") as mock_logger: + + with pytest.raises( + ValueError, match="PD disaggregation mode requires --prefill" + ): + launch_router(args) + + # Should log error for validation failures + mock_logger.error.assert_called_once() + + +class TestStartupFlow: + """Test complete startup flow.""" + + def test_complete_startup_flow_basic(self): + """Test complete startup flow for basic configuration.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000", "http://worker2:8000"], + policy="cache_aware", + cache_threshold=0.5, + balance_abs_threshold=32, + balance_rel_threshold=1.5, + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + def test_complete_startup_flow_pd_mode(self): + """Test complete startup flow for PD mode configuration.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[ + ("http://prefill1:8000", 9000), + ("http://prefill2:8000", None), + ], + decode_urls=["http://decode1:8001", "http://decode2:8001"], + policy="power_of_two", + prefill_policy="cache_aware", + decode_policy="round_robin", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + with patch("sglang_router.router_args.logger") as mock_logger: + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + # Verify policy warning was logged + mock_logger.warning.assert_called_once() + + def test_complete_startup_flow_with_all_features(self): + """Test complete startup flow with all features enabled.""" + args = RouterArgs( + host="0.0.0.0", + port=30001, + worker_urls=["http://worker1:8000"], + policy="round_robin", + service_discovery=True, + selector={"app": "worker"}, + service_discovery_port=8080, + service_discovery_namespace="default", + dp_aware=True, + api_key="test-key", + log_dir="/tmp/logs", + log_level="debug", + prometheus_port=29000, + prometheus_host="0.0.0.0", + request_id_headers=["x-request-id", "x-trace-id"], + request_timeout_secs=1200, + max_concurrent_requests=512, + queue_size=200, + queue_timeout_secs=120, + rate_limit_tokens_per_second=100, + cors_allowed_origins=["http://localhost:3000"], + retry_max_retries=3, + retry_initial_backoff_ms=100, + retry_max_backoff_ms=10000, + retry_backoff_multiplier=2.0, + retry_jitter_factor=0.1, + cb_failure_threshold=5, + cb_success_threshold=2, + cb_timeout_duration_secs=30, + cb_window_duration_secs=60, + health_failure_threshold=2, + health_success_threshold=1, + health_check_timeout_secs=3, + health_check_interval_secs=30, + health_check_endpoint="/healthz", + ) + + with patch("sglang_router.launch_router.Router") as router_mod: + captured_args = {} + mock_router_instance = MagicMock() + + def fake_from_args(router_args): + captured_args.update( + dict( + host=router_args.host, + port=router_args.port, + worker_urls=router_args.worker_urls, + policy=policy_from_str(router_args.policy), + service_discovery=router_args.service_discovery, + selector=router_args.selector, + service_discovery_port=router_args.service_discovery_port, + service_discovery_namespace=router_args.service_discovery_namespace, + dp_aware=router_args.dp_aware, + api_key=router_args.api_key, + log_dir=router_args.log_dir, + log_level=router_args.log_level, + prometheus_port=router_args.prometheus_port, + prometheus_host=router_args.prometheus_host, + request_id_headers=router_args.request_id_headers, + request_timeout_secs=router_args.request_timeout_secs, + max_concurrent_requests=router_args.max_concurrent_requests, + queue_size=router_args.queue_size, + queue_timeout_secs=router_args.queue_timeout_secs, + rate_limit_tokens_per_second=router_args.rate_limit_tokens_per_second, + cors_allowed_origins=router_args.cors_allowed_origins, + retry_max_retries=router_args.retry_max_retries, + retry_initial_backoff_ms=router_args.retry_initial_backoff_ms, + retry_max_backoff_ms=router_args.retry_max_backoff_ms, + retry_backoff_multiplier=router_args.retry_backoff_multiplier, + retry_jitter_factor=router_args.retry_jitter_factor, + cb_failure_threshold=router_args.cb_failure_threshold, + cb_success_threshold=router_args.cb_success_threshold, + cb_timeout_duration_secs=router_args.cb_timeout_duration_secs, + cb_window_duration_secs=router_args.cb_window_duration_secs, + health_failure_threshold=router_args.health_failure_threshold, + health_success_threshold=router_args.health_success_threshold, + health_check_timeout_secs=router_args.health_check_timeout_secs, + health_check_interval_secs=router_args.health_check_interval_secs, + health_check_endpoint=router_args.health_check_endpoint, + ) + ) + return mock_router_instance + + router_mod.from_args = MagicMock(side_effect=fake_from_args) + + result = launch_router(args) + + # Verify complete flow + router_mod.from_args.assert_called_once() + mock_router_instance.start.assert_called_once() + + # Verify key parameters were propagated into RouterArgs + assert captured_args["host"] == "0.0.0.0" + assert captured_args["port"] == 30001 + assert captured_args["worker_urls"] == ["http://worker1:8000"] + assert captured_args["policy"] == PolicyType.RoundRobin + assert captured_args["service_discovery"] is True + assert captured_args["selector"] == {"app": "worker"} + assert captured_args["service_discovery_port"] == 8080 + assert captured_args["service_discovery_namespace"] == "default" + assert captured_args["dp_aware"] is True + assert captured_args["api_key"] == "test-key" + assert captured_args["log_dir"] == "/tmp/logs" + assert captured_args["log_level"] == "debug" + assert captured_args["prometheus_port"] == 29000 + assert captured_args["prometheus_host"] == "0.0.0.0" + assert captured_args["request_id_headers"] == ["x-request-id", "x-trace-id"] + assert captured_args["request_timeout_secs"] == 1200 + assert captured_args["max_concurrent_requests"] == 512 + assert captured_args["queue_size"] == 200 + assert captured_args["queue_timeout_secs"] == 120 + assert captured_args["rate_limit_tokens_per_second"] == 100 + assert captured_args["cors_allowed_origins"] == ["http://localhost:3000"] + assert captured_args["retry_max_retries"] == 3 + assert captured_args["retry_initial_backoff_ms"] == 100 + assert captured_args["retry_max_backoff_ms"] == 10000 + assert captured_args["retry_backoff_multiplier"] == 2.0 + assert captured_args["retry_jitter_factor"] == 0.1 + assert captured_args["cb_failure_threshold"] == 5 + assert captured_args["cb_success_threshold"] == 2 + assert captured_args["cb_timeout_duration_secs"] == 30 + assert captured_args["cb_window_duration_secs"] == 60 + assert captured_args["health_failure_threshold"] == 2 + assert captured_args["health_success_threshold"] == 1 + assert captured_args["health_check_timeout_secs"] == 3 + assert captured_args["health_check_interval_secs"] == 30 + assert captured_args["health_check_endpoint"] == "/healthz" diff --git a/sgl-router/py_test/unit/test_validation.py b/sgl-router/py_test/unit/test_validation.py new file mode 100644 index 000000000..1a3e54612 --- /dev/null +++ b/sgl-router/py_test/unit/test_validation.py @@ -0,0 +1,506 @@ +""" +Unit tests for validation logic in sglang_router. + +These tests focus on testing the validation logic in isolation, +including parameter validation, URL validation, and configuration validation. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from sglang_router.launch_router import RouterArgs, launch_router + + +class TestURLValidation: + """Test URL validation logic.""" + + def test_valid_worker_urls(self): + """Test validation of valid worker URLs.""" + valid_urls = [ + "http://worker1:8000", + "https://worker2:8000", + "http://localhost:8000", + "http://127.0.0.1:8000", + "http://192.168.1.100:8000", + "http://worker.example.com:8000", + ] + + for url in valid_urls: + args = RouterArgs(worker_urls=[url]) + # Should not raise any validation errors + assert url in args.worker_urls + + def test_valid_prefill_urls(self): + """Test validation of valid prefill URLs.""" + valid_prefill_urls = [ + ("http://prefill1:8000", 9000), + ("https://prefill2:8000", None), + ("http://localhost:8000", 9000), + ("http://127.0.0.1:8000", None), + ] + + for url, bootstrap_port in valid_prefill_urls: + args = RouterArgs(prefill_urls=[(url, bootstrap_port)]) + # Should not raise any validation errors + assert (url, bootstrap_port) in args.prefill_urls + + def test_valid_decode_urls(self): + """Test validation of valid decode URLs.""" + valid_decode_urls = [ + "http://decode1:8001", + "https://decode2:8001", + "http://localhost:8001", + "http://127.0.0.1:8001", + ] + + for url in valid_decode_urls: + args = RouterArgs(decode_urls=[url]) + # Should not raise any validation errors + assert url in args.decode_urls + + def test_malformed_urls(self): + """Test handling of malformed URLs.""" + # Note: The current implementation doesn't validate URL format + # This test documents the current behavior + malformed_urls = [ + "not-a-url", + "ftp://worker1:8000", # Wrong protocol + "http://", # Missing host + ":8000", # Missing protocol and host + "http://worker1", # Missing port + ] + + for url in malformed_urls: + args = RouterArgs(worker_urls=[url]) + # Currently, malformed URLs are accepted + # This might be something to improve in the future + assert url in args.worker_urls + + +class TestPortValidation: + """Test port validation logic.""" + + def test_valid_ports(self): + """Test validation of valid port numbers.""" + valid_ports = [1, 80, 8000, 30000, 65535] + + for port in valid_ports: + args = RouterArgs(port=port) + assert args.port == port + + def test_invalid_ports(self): + """Test handling of invalid port numbers.""" + # Note: The current implementation doesn't validate port ranges + # This test documents the current behavior + invalid_ports = [0, -1, 65536, 70000] + + for port in invalid_ports: + args = RouterArgs(port=port) + # Currently, invalid ports are accepted + # This might be something to improve in the future + assert args.port == port + + def test_bootstrap_port_validation(self): + """Test validation of bootstrap ports in PD mode.""" + valid_bootstrap_ports = [1, 80, 9000, 30000, 65535, None] + + for bootstrap_port in valid_bootstrap_ports: + args = RouterArgs(prefill_urls=[("http://prefill1:8000", bootstrap_port)]) + assert args.prefill_urls[0][1] == bootstrap_port + + +class TestParameterValidation: + """Test parameter validation logic.""" + + def test_cache_threshold_validation(self): + """Test cache threshold parameter validation.""" + # Valid cache thresholds + valid_thresholds = [0.0, 0.1, 0.5, 0.9, 1.0] + + for threshold in valid_thresholds: + args = RouterArgs(cache_threshold=threshold) + assert args.cache_threshold == threshold + + def test_balance_threshold_validation(self): + """Test load balancing threshold parameter validation.""" + # Valid absolute thresholds + valid_abs_thresholds = [0, 1, 32, 64, 128, 1000] + for threshold in valid_abs_thresholds: + args = RouterArgs(balance_abs_threshold=threshold) + assert args.balance_abs_threshold == threshold + + # Valid relative thresholds + valid_rel_thresholds = [1.0, 1.1, 1.5, 2.0, 10.0] + for threshold in valid_rel_thresholds: + args = RouterArgs(balance_rel_threshold=threshold) + assert args.balance_rel_threshold == threshold + + def test_timeout_validation(self): + """Test timeout parameter validation.""" + # Valid timeouts + valid_timeouts = [1, 30, 60, 300, 600, 1800, 3600] + + for timeout in valid_timeouts: + args = RouterArgs( + worker_startup_timeout_secs=timeout, + worker_startup_check_interval=timeout, + request_timeout_secs=timeout, + queue_timeout_secs=timeout, + ) + assert args.worker_startup_timeout_secs == timeout + assert args.worker_startup_check_interval == timeout + assert args.request_timeout_secs == timeout + assert args.queue_timeout_secs == timeout + + def test_retry_parameter_validation(self): + """Test retry parameter validation.""" + # Valid retry parameters + valid_retry_counts = [0, 1, 3, 5, 10] + for count in valid_retry_counts: + args = RouterArgs(retry_max_retries=count) + assert args.retry_max_retries == count + + # Valid backoff parameters + valid_backoff_ms = [1, 50, 100, 1000, 30000] + for backoff in valid_backoff_ms: + args = RouterArgs( + retry_initial_backoff_ms=backoff, retry_max_backoff_ms=backoff + ) + assert args.retry_initial_backoff_ms == backoff + assert args.retry_max_backoff_ms == backoff + + # Valid multiplier parameters + valid_multipliers = [1.0, 1.5, 2.0, 3.0] + for multiplier in valid_multipliers: + args = RouterArgs(retry_backoff_multiplier=multiplier) + assert args.retry_backoff_multiplier == multiplier + + # Valid jitter parameters + valid_jitter = [0.0, 0.1, 0.2, 0.5] + for jitter in valid_jitter: + args = RouterArgs(retry_jitter_factor=jitter) + assert args.retry_jitter_factor == jitter + + def test_circuit_breaker_parameter_validation(self): + """Test circuit breaker parameter validation.""" + # Valid failure thresholds + valid_failure_thresholds = [1, 3, 5, 10, 20] + for threshold in valid_failure_thresholds: + args = RouterArgs(cb_failure_threshold=threshold) + assert args.cb_failure_threshold == threshold + + # Valid success thresholds + valid_success_thresholds = [1, 2, 3, 5] + for threshold in valid_success_thresholds: + args = RouterArgs(cb_success_threshold=threshold) + assert args.cb_success_threshold == threshold + + # Valid timeout durations + valid_timeouts = [10, 30, 60, 120, 300] + for timeout in valid_timeouts: + args = RouterArgs( + cb_timeout_duration_secs=timeout, cb_window_duration_secs=timeout + ) + assert args.cb_timeout_duration_secs == timeout + assert args.cb_window_duration_secs == timeout + + def test_health_check_parameter_validation(self): + """Test health check parameter validation.""" + # Valid failure thresholds + valid_failure_thresholds = [1, 2, 3, 5, 10] + for threshold in valid_failure_thresholds: + args = RouterArgs(health_failure_threshold=threshold) + assert args.health_failure_threshold == threshold + + # Valid success thresholds + valid_success_thresholds = [1, 2, 3, 5] + for threshold in valid_success_thresholds: + args = RouterArgs(health_success_threshold=threshold) + assert args.health_success_threshold == threshold + + # Valid timeouts and intervals + valid_times = [1, 5, 10, 30, 60, 120] + for time_val in valid_times: + args = RouterArgs( + health_check_timeout_secs=time_val, health_check_interval_secs=time_val + ) + assert args.health_check_timeout_secs == time_val + assert args.health_check_interval_secs == time_val + + def test_rate_limiting_parameter_validation(self): + """Test rate limiting parameter validation.""" + # Valid concurrent request limits + valid_limits = [1, 10, 64, 256, 512, 1000] + for limit in valid_limits: + args = RouterArgs(max_concurrent_requests=limit) + assert args.max_concurrent_requests == limit + + # Valid queue sizes + valid_queue_sizes = [0, 10, 50, 100, 500, 1000] + for size in valid_queue_sizes: + args = RouterArgs(queue_size=size) + assert args.queue_size == size + + # Valid token rates + valid_rates = [1, 10, 50, 100, 500, 1000] + for rate in valid_rates: + args = RouterArgs(rate_limit_tokens_per_second=rate) + assert args.rate_limit_tokens_per_second == rate + + def test_tree_size_validation(self): + """Test tree size parameter validation.""" + # Valid tree sizes (powers of 2) + valid_sizes = [2**10, 2**20, 2**24, 2**26, 2**28, 2**30] + + for size in valid_sizes: + args = RouterArgs(max_tree_size=size) + assert args.max_tree_size == size + + def test_payload_size_validation(self): + """Test payload size parameter validation.""" + # Valid payload sizes + valid_sizes = [ + 1024, # 1KB + 1024 * 1024, # 1MB + 10 * 1024 * 1024, # 10MB + 100 * 1024 * 1024, # 100MB + 512 * 1024 * 1024, # 512MB + 1024 * 1024 * 1024, # 1GB + ] + + for size in valid_sizes: + args = RouterArgs(max_payload_size=size) + assert args.max_payload_size == size + + +class TestConfigurationValidation: + """Test configuration validation logic.""" + + def test_pd_mode_validation(self): + """Test PD mode configuration validation.""" + # Valid PD configuration + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + ) + + assert args.pd_disaggregation is True + assert len(args.prefill_urls) > 0 + assert len(args.decode_urls) > 0 + + def test_service_discovery_validation(self): + """Test service discovery configuration validation.""" + # Valid service discovery configuration + args = RouterArgs( + service_discovery=True, + selector={"app": "worker", "env": "prod"}, + service_discovery_port=8080, + service_discovery_namespace="default", + ) + + assert args.service_discovery is True + assert args.selector == {"app": "worker", "env": "prod"} + assert args.service_discovery_port == 8080 + assert args.service_discovery_namespace == "default" + + def test_pd_service_discovery_validation(self): + """Test PD service discovery configuration validation.""" + # Valid PD service discovery configuration + args = RouterArgs( + pd_disaggregation=True, + service_discovery=True, + prefill_selector={"app": "prefill"}, + decode_selector={"app": "decode"}, + ) + + assert args.pd_disaggregation is True + assert args.service_discovery is True + assert args.prefill_selector == {"app": "prefill"} + assert args.decode_selector == {"app": "decode"} + + def test_policy_validation(self): + """Test policy configuration validation.""" + # Valid policies + valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"] + + for policy in valid_policies: + args = RouterArgs(policy=policy) + assert args.policy == policy + + def test_pd_policy_validation(self): + """Test PD policy configuration validation.""" + # Valid PD policies + valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"] + + for prefill_policy in valid_policies: + for decode_policy in valid_policies: + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", None)], + decode_urls=["http://decode1:8001"], + prefill_policy=prefill_policy, + decode_policy=decode_policy, + ) + assert args.prefill_policy == prefill_policy + assert args.decode_policy == decode_policy + + def test_cors_validation(self): + """Test CORS configuration validation.""" + # Valid CORS origins + valid_origins = [ + [], + ["http://localhost:3000"], + ["https://example.com"], + ["http://localhost:3000", "https://example.com"], + ["*"], # Wildcard (if supported) + ] + + for origins in valid_origins: + args = RouterArgs(cors_allowed_origins=origins) + assert args.cors_allowed_origins == origins + + def test_logging_validation(self): + """Test logging configuration validation.""" + # Valid log levels + valid_log_levels = ["debug", "info", "warning", "error", "critical"] + + for level in valid_log_levels: + args = RouterArgs(log_level=level) + assert args.log_level == level + + def test_prometheus_validation(self): + """Test Prometheus configuration validation.""" + # Valid Prometheus configuration + args = RouterArgs(prometheus_port=29000, prometheus_host="127.0.0.1") + + assert args.prometheus_port == 29000 + assert args.prometheus_host == "127.0.0.1" + + def test_tokenizer_validation(self): + """Test tokenizer configuration validation.""" + # Note: model_path and tokenizer_path are not available in current RouterArgs + pytest.skip("Tokenizer configuration not available in current implementation") + + def test_request_id_headers_validation(self): + """Test request ID headers configuration validation.""" + # Valid request ID headers + valid_headers = [ + ["x-request-id"], + ["x-request-id", "x-trace-id"], + ["x-request-id", "x-trace-id", "x-correlation-id"], + ["custom-header"], + ] + + for headers in valid_headers: + args = RouterArgs(request_id_headers=headers) + assert args.request_id_headers == headers + + +class TestLaunchValidation: + """Test launch-time validation logic.""" + + def test_pd_mode_requires_urls(self): + """Test that PD mode requires prefill and decode URLs.""" + # PD mode without URLs should fail + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=False, + ) + + with pytest.raises( + ValueError, match="PD disaggregation mode requires --prefill" + ): + launch_router(args) + + def test_pd_mode_with_service_discovery_allows_empty_urls(self): + """Test that PD mode with service discovery allows empty URLs.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[], + decode_urls=[], + service_discovery=True, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_regular_mode_allows_empty_worker_urls(self): + """Test that regular mode allows empty worker URLs.""" + args = RouterArgs(worker_urls=[], service_discovery=False) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_valid_config(self): + """Test launching with valid configuration.""" + args = RouterArgs( + host="127.0.0.1", + port=30000, + worker_urls=["http://worker1:8000"], + policy="cache_aware", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_pd_config(self): + """Test launching with valid PD configuration.""" + args = RouterArgs( + pd_disaggregation=True, + prefill_urls=[("http://prefill1:8000", 9000)], + decode_urls=["http://decode1:8001"], + policy="cache_aware", + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() + + def test_launch_with_service_discovery_config(self): + """Test launching with valid service discovery configuration.""" + args = RouterArgs( + service_discovery=True, + selector={"app": "worker"}, + service_discovery_port=8080, + ) + + # Should not raise validation error + with patch("sglang_router.launch_router.Router") as router_mod: + mock_router_instance = MagicMock() + router_mod.from_args = MagicMock(return_value=mock_router_instance) + + launch_router(args) + + # Should create router instance via from_args + router_mod.from_args.assert_called_once() diff --git a/sgl-router/pyproject.toml b/sgl-router/pyproject.toml new file mode 100644 index 000000000..9a7606f6a --- /dev/null +++ b/sgl-router/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools-rust>=1.5.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "sglang-router" +version = "0.1.9" +description = "High-performance Rust-based load balancer for SGLang with multiple routing algorithms and prefill-decode disaggregation support" +authors = [{name = "Byron Hsu", email = "byronhsu1230@gmail.com"}] +requires-python = ">=3.8" +readme = "README.md" +license = { text = "Apache-2.0" } +classifiers = [ + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Rust", + "Programming Language :: Python :: 3", +] + +[project.optional-dependencies] +dev = [ + "requests>=2.25.0", +] + + +# https://github.com/PyO3/setuptools-rust?tab=readme-ov-file +[tool.setuptools.packages] +find = { where = ["py_src"] } + +# workaround for https://github.com/pypa/twine/issues/1216 +[tool.setuptools] +license-files = [] diff --git a/sgl-router/pytest.ini b/sgl-router/pytest.ini new file mode 100644 index 000000000..c9f400753 --- /dev/null +++ b/sgl-router/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = py_test +python_files = test_*.py +python_classes = Test* +python_functions = test_* diff --git a/sgl-router/scripts/run_benchmarks.py b/sgl-router/scripts/run_benchmarks.py new file mode 100755 index 000000000..a7ece9d9a --- /dev/null +++ b/sgl-router/scripts/run_benchmarks.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +SGLang Router Benchmark Runner + +A Python script to run Rust benchmarks with various options and modes. +Replaces the shell script for better maintainability and integration. +""" + +import argparse +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + + +class BenchmarkRunner: + """Handles running Rust benchmarks for the SGLang router.""" + + def __init__(self, project_root: str): + self.project_root = Path(project_root) + self.timestamp = time.strftime("%a %b %d %H:%M:%S UTC %Y", time.gmtime()) + + def run_command( + self, cmd: List[str], capture_output: bool = False + ) -> subprocess.CompletedProcess: + """Run a command and handle errors.""" + try: + if capture_output: + result = subprocess.run( + cmd, capture_output=True, text=True, cwd=self.project_root + ) + else: + result = subprocess.run(cmd, cwd=self.project_root) + return result + except subprocess.CalledProcessError as e: + print(f"Error running command: {' '.join(cmd)}") + print(f"Exit code: {e.returncode}") + sys.exit(1) + + def print_header(self): + """Print the benchmark runner header.""" + print("SGLang Router Benchmark Runner") + print("=" * 30) + print(f"Project: {self.project_root.absolute()}") + print(f"Timestamp: {self.timestamp}") + print() + + def build_release(self): + """Build the project in release mode.""" + print("Building in release mode...") + result = self.run_command(["cargo", "build", "--release", "--quiet"]) + if result.returncode != 0: + print("Failed to build in release mode") + sys.exit(1) + + def run_benchmarks( + self, + quick_mode: bool = False, + save_baseline: Optional[str] = None, + compare_baseline: Optional[str] = None, + ) -> str: + """Run benchmarks with specified options.""" + bench_args = ["cargo", "bench", "--bench", "request_processing"] + + if quick_mode: + bench_args.append("benchmark_summary") + print("Running quick benchmarks...") + else: + print("Running full benchmark suite...") + + # Note: Criterion baselines are handled via target directory structure + # For now, we'll implement baseline functionality via file copying + if save_baseline: + print(f"Will save results as baseline: {save_baseline}") + + if compare_baseline: + print(f"Will compare with baseline: {compare_baseline}") + + print(f"Executing: {' '.join(bench_args)}") + result = self.run_command(bench_args, capture_output=True) + + if result.returncode != 0: + print("Benchmark execution failed!") + print("STDOUT:", result.stdout) + print("STDERR:", result.stderr) + sys.exit(1) + + # Handle baseline saving after successful run + if save_baseline: + self._save_baseline(save_baseline, result.stdout) + + return result.stdout + + def _save_baseline(self, filename: str, output: str): + """Save benchmark results to a file as baseline.""" + filepath = self.project_root / filename + with open(filepath, "w") as f: + f.write(output) + print(f"Baseline saved to: {filepath}") + + def parse_benchmark_results(self, output: str) -> Dict[str, str]: + """Parse benchmark output to extract performance metrics.""" + results = {} + + # Look for performance overview section + lines = output.split("\n") + parsing_overview = False + + for line in lines: + line = line.strip() + + if "Quick Performance Overview:" in line: + parsing_overview = True + continue + + if parsing_overview and line.startswith("* "): + # Parse lines like "* Serialization (avg): 481 ns/req" + if "Serialization (avg):" in line: + results["serialization_time"] = self._extract_time(line) + elif "Deserialization (avg):" in line: + results["deserialization_time"] = self._extract_time(line) + elif "Bootstrap Injection (avg):" in line: + results["bootstrap_injection_time"] = self._extract_time(line) + elif "Total Pipeline (avg):" in line: + results["total_time"] = self._extract_time(line) + + # Stop parsing after the overview section + if parsing_overview and line.startswith("Performance Insights:"): + break + + return results + + def _extract_time(self, line: str) -> str: + """Extract time value from a benchmark line.""" + # Extract number followed by ns/req + import re + + match = re.search(r"(\d+)\s*ns/req", line) + return match.group(1) if match else "N/A" + + def validate_thresholds(self, results: Dict[str, str]) -> bool: + """Validate benchmark results against performance thresholds.""" + thresholds = { + "serialization_time": 2000, # 2μs max + "deserialization_time": 2000, # 2μs max + "bootstrap_injection_time": 5000, # 5μs max + "total_time": 10000, # 10μs max + } + + all_passed = True + print("\nPerformance Threshold Validation:") + print("=" * 35) + + for metric, threshold in thresholds.items(): + if metric in results and results[metric] != "N/A": + try: + value = int(results[metric]) + passed = value <= threshold + status = "✓ PASS" if passed else "✗ FAIL" + print(f"{metric:20}: {value:>6}ns <= {threshold:>6}ns {status}") + if not passed: + all_passed = False + except ValueError: + print(f"{metric:20}: Invalid value: {results[metric]}") + all_passed = False + else: + print(f"{metric:20}: No data available") + all_passed = False + + print() + if all_passed: + print("All performance thresholds passed!") + else: + print("Some performance thresholds failed!") + + return all_passed + + def save_results_to_file( + self, results: Dict[str, str], filename: str = "benchmark_results.env" + ): + """Save benchmark results to a file for CI consumption.""" + filepath = self.project_root / filename + with open(filepath, "w") as f: + for key, value in results.items(): + f.write(f"{key}={value}\n") + print(f"Results saved to: {filepath}") + + +def main(): + parser = argparse.ArgumentParser(description="Run SGLang router benchmarks") + parser.add_argument( + "--quick", action="store_true", help="Run quick benchmarks (summary only)" + ) + parser.add_argument( + "--save-baseline", type=str, help="Save benchmark results as baseline" + ) + parser.add_argument( + "--compare-baseline", type=str, help="Compare with saved baseline" + ) + parser.add_argument( + "--validate-thresholds", + action="store_true", + help="Validate results against performance thresholds", + ) + parser.add_argument( + "--save-results", action="store_true", help="Save results to file for CI" + ) + + args = parser.parse_args() + + # Determine project root (script is in scripts/ subdirectory) + script_dir = Path(__file__).parent + project_root = script_dir.parent + + runner = BenchmarkRunner(str(project_root)) + runner.print_header() + + # Build in release mode + runner.build_release() + + # Run benchmarks + output = runner.run_benchmarks( + quick_mode=args.quick, + save_baseline=args.save_baseline, + compare_baseline=args.compare_baseline, + ) + + # Print the raw output + print(output) + + # Parse and validate results if requested + if args.validate_thresholds or args.save_results: + results = runner.parse_benchmark_results(output) + + if args.save_results: + runner.save_results_to_file(results) + + if args.validate_thresholds: + passed = runner.validate_thresholds(results) + if not passed: + print("Validation failed - performance regression detected!") + sys.exit(1) + + print("\nBenchmark run completed successfully!") + + +if __name__ == "__main__": + main() diff --git a/sgl-router/setup.py b/sgl-router/setup.py new file mode 100644 index 000000000..730a91ceb --- /dev/null +++ b/sgl-router/setup.py @@ -0,0 +1,21 @@ +import os + +from setuptools import setup +from setuptools_rust import Binding, RustExtension + +no_rust = os.environ.get("SGLANG_ROUTER_BUILD_NO_RUST") == "1" + +rust_extensions = [] +if not no_rust: + rust_extensions.append( + RustExtension( + target="sglang_router_rs", + path="Cargo.toml", + binding=Binding.PyO3, + ) + ) + +setup( + rust_extensions=rust_extensions, + zip_safe=False, +) diff --git a/sgl-router/src/config/mod.rs b/sgl-router/src/config/mod.rs new file mode 100644 index 000000000..4622ff781 --- /dev/null +++ b/sgl-router/src/config/mod.rs @@ -0,0 +1,28 @@ +pub mod types; +pub mod validation; + +pub use types::*; +pub use validation::*; + +/// Configuration errors +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("Validation failed: {reason}")] + ValidationFailed { reason: String }, + + #[error("Invalid value for field '{field}': {value} - {reason}")] + InvalidValue { + field: String, + value: String, + reason: String, + }, + + #[error("Incompatible configuration: {reason}")] + IncompatibleConfig { reason: String }, + + #[error("Missing required field: {field}")] + MissingRequired { field: String }, +} + +/// Result type for configuration operations +pub type ConfigResult = Result; diff --git a/sgl-router/src/config/types.rs b/sgl-router/src/config/types.rs new file mode 100644 index 000000000..5f5b227af --- /dev/null +++ b/sgl-router/src/config/types.rs @@ -0,0 +1,1241 @@ +use super::ConfigResult; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Main router configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RouterConfig { + /// Routing mode configuration + pub mode: RoutingMode, + /// Worker connection mode + #[serde(default)] + pub connection_mode: ConnectionMode, + /// Policy configuration + pub policy: PolicyConfig, + /// Server host address + pub host: String, + /// Server port + pub port: u16, + /// Maximum payload size in bytes + pub max_payload_size: usize, + /// Request timeout in seconds + pub request_timeout_secs: u64, + /// Worker startup timeout in seconds + pub worker_startup_timeout_secs: u64, + /// Worker health check interval in seconds + pub worker_startup_check_interval_secs: u64, + /// Enable data parallelism aware schedule + pub dp_aware: bool, + /// The api key used for the authorization with the worker + pub api_key: Option, + /// Service discovery configuration (optional) + pub discovery: Option, + /// Metrics configuration (optional) + pub metrics: Option, + /// Log directory (None = stdout only) + pub log_dir: Option, + /// Log level (None = info) + pub log_level: Option, + /// Custom request ID headers to check (defaults to common headers) + pub request_id_headers: Option>, + /// Maximum concurrent requests allowed (for rate limiting) + pub max_concurrent_requests: usize, + /// Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately) + pub queue_size: usize, + /// Maximum time (in seconds) a request can wait in queue before timing out + pub queue_timeout_secs: u64, + /// Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests + pub rate_limit_tokens_per_second: Option, + /// CORS allowed origins + pub cors_allowed_origins: Vec, + /// Retry configuration + pub retry: RetryConfig, + /// Circuit breaker configuration + pub circuit_breaker: CircuitBreakerConfig, + /// Disable retries (overrides retry.max_retries to 1 when true) + #[serde(default)] + pub disable_retries: bool, + /// Disable circuit breaker (overrides circuit_breaker.failure_threshold to u32::MAX when true) + #[serde(default)] + pub disable_circuit_breaker: bool, + /// Health check configuration + pub health_check: HealthCheckConfig, + /// Enable Inference Gateway mode (false = proxy mode, true = IGW mode) + #[serde(default)] + pub enable_igw: bool, + /// Model path for loading tokenizer (can be a HuggingFace model ID or local path) + pub model_path: Option, + /// Explicit tokenizer path (overrides model_path tokenizer if provided) + pub tokenizer_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +#[serde(tag = "type")] +pub enum ConnectionMode { + #[default] + #[serde(rename = "http")] + Http, + #[serde(rename = "grpc")] + Grpc, +} + +/// Routing mode configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum RoutingMode { + #[serde(rename = "regular")] + Regular { + /// List of worker URLs + worker_urls: Vec, + }, + #[serde(rename = "prefill_decode")] + PrefillDecode { + /// Prefill worker URLs with optional bootstrap ports + prefill_urls: Vec<(String, Option)>, + /// Decode worker URLs + decode_urls: Vec, + /// Optional separate policy for prefill workers + #[serde(skip_serializing_if = "Option::is_none")] + prefill_policy: Option, + /// Optional separate policy for decode workers + #[serde(skip_serializing_if = "Option::is_none")] + decode_policy: Option, + }, + #[serde(rename = "openai")] + OpenAI { + /// OpenAI-compatible API base(s), provided via worker URLs + worker_urls: Vec, + }, +} + +impl RoutingMode { + pub fn is_pd_mode(&self) -> bool { + matches!(self, RoutingMode::PrefillDecode { .. }) + } + + pub fn worker_count(&self) -> usize { + match self { + RoutingMode::Regular { worker_urls } => worker_urls.len(), + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + .. + } => prefill_urls.len() + decode_urls.len(), + // OpenAI mode represents a single upstream + RoutingMode::OpenAI { .. } => 1, + } + } + + /// Get the effective prefill policy for PD mode + /// Falls back to the main policy if no specific prefill policy is set + pub fn get_prefill_policy<'a>(&'a self, main_policy: &'a PolicyConfig) -> &'a PolicyConfig { + match self { + RoutingMode::PrefillDecode { prefill_policy, .. } => { + prefill_policy.as_ref().unwrap_or(main_policy) + } + _ => main_policy, + } + } + + /// Get the effective decode policy for PD mode + /// Falls back to the main policy if no specific decode policy is set + pub fn get_decode_policy<'a>(&'a self, main_policy: &'a PolicyConfig) -> &'a PolicyConfig { + match self { + RoutingMode::PrefillDecode { decode_policy, .. } => { + decode_policy.as_ref().unwrap_or(main_policy) + } + _ => main_policy, + } + } +} + +/// Policy configuration for routing +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum PolicyConfig { + #[serde(rename = "random")] + Random, + + #[serde(rename = "round_robin")] + RoundRobin, + + #[serde(rename = "cache_aware")] + CacheAware { + /// Minimum prefix match ratio to use cache-based routing + cache_threshold: f32, + /// Absolute load difference threshold for load balancing + balance_abs_threshold: usize, + /// Relative load ratio threshold for load balancing + balance_rel_threshold: f32, + /// Interval between cache eviction cycles (seconds) + eviction_interval_secs: u64, + /// Maximum cache tree size per tenant + max_tree_size: usize, + }, + + #[serde(rename = "power_of_two")] + PowerOfTwo { + /// Interval for load monitoring (seconds) + load_check_interval_secs: u64, + }, +} + +impl PolicyConfig { + pub fn name(&self) -> &'static str { + match self { + PolicyConfig::Random => "random", + PolicyConfig::RoundRobin => "round_robin", + PolicyConfig::CacheAware { .. } => "cache_aware", + PolicyConfig::PowerOfTwo { .. } => "power_of_two", + } + } +} + +/// Service discovery configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiscoveryConfig { + /// Enable service discovery + pub enabled: bool, + /// Kubernetes namespace (None = all namespaces) + pub namespace: Option, + /// Service discovery port + pub port: u16, + /// Check interval for service discovery + pub check_interval_secs: u64, + /// Regular mode selector + pub selector: HashMap, + /// PD mode prefill selector + pub prefill_selector: HashMap, + /// PD mode decode selector + pub decode_selector: HashMap, + /// Bootstrap port annotation key + pub bootstrap_port_annotation: String, +} + +impl Default for DiscoveryConfig { + fn default() -> Self { + Self { + enabled: false, + namespace: None, + port: 8000, + check_interval_secs: 120, + selector: HashMap::new(), + prefill_selector: HashMap::new(), + decode_selector: HashMap::new(), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + } + } +} + +/// Retry configuration for request handling +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetryConfig { + /// Maximum number of retry attempts + pub max_retries: u32, + /// Initial backoff delay in milliseconds + pub initial_backoff_ms: u64, + /// Maximum backoff delay in milliseconds + pub max_backoff_ms: u64, + /// Backoff multiplier for exponential backoff + pub backoff_multiplier: f32, + /// Jitter factor applied to backoff (0.0 - 1.0) + /// Effective delay D' = D * (1 + U[-j, +j]) + #[serde(default = "default_retry_jitter_factor")] + pub jitter_factor: f32, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + max_retries: 5, + initial_backoff_ms: 50, + max_backoff_ms: 30000, + backoff_multiplier: 1.5, + jitter_factor: 0.2, + } + } +} + +fn default_retry_jitter_factor() -> f32 { + 0.2 +} + +/// Health check configuration for worker monitoring +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HealthCheckConfig { + /// Number of consecutive failures before marking unhealthy + pub failure_threshold: u32, + /// Number of consecutive successes before marking healthy + pub success_threshold: u32, + /// Timeout for health check requests in seconds + pub timeout_secs: u64, + /// Interval between health checks in seconds + pub check_interval_secs: u64, + /// Health check endpoint path + pub endpoint: String, +} + +impl Default for HealthCheckConfig { + fn default() -> Self { + Self { + failure_threshold: 3, + success_threshold: 2, + timeout_secs: 5, + check_interval_secs: 60, + endpoint: "/health".to_string(), + } + } +} + +/// Circuit breaker configuration for worker reliability +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircuitBreakerConfig { + /// Number of consecutive failures before opening circuit + pub failure_threshold: u32, + /// Number of consecutive successes before closing circuit + pub success_threshold: u32, + /// Time before attempting to recover from open state (in seconds) + pub timeout_duration_secs: u64, + /// Window duration for failure tracking (in seconds) + pub window_duration_secs: u64, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + failure_threshold: 10, + success_threshold: 3, + timeout_duration_secs: 60, + window_duration_secs: 120, + } + } +} + +/// Metrics configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetricsConfig { + /// Prometheus metrics port + pub port: u16, + /// Prometheus metrics host + pub host: String, +} + +impl Default for MetricsConfig { + fn default() -> Self { + Self { + port: 29000, + host: "127.0.0.1".to_string(), + } + } +} + +impl Default for RouterConfig { + fn default() -> Self { + Self { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3001, + max_payload_size: 536_870_912, // 512MB + request_timeout_secs: 1800, // 30 minutes + worker_startup_timeout_secs: 600, + worker_startup_check_interval_secs: 30, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 256, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + } + } +} + +impl RouterConfig { + /// Create a new configuration with mode and policy + pub fn new(mode: RoutingMode, policy: PolicyConfig) -> Self { + Self { + mode, + policy, + ..Default::default() + } + } + + /// Validate the configuration + pub fn validate(&self) -> ConfigResult<()> { + crate::config::validation::ConfigValidator::validate(self) + } + + /// Get the routing mode type as a string + pub fn mode_type(&self) -> &'static str { + match self.mode { + RoutingMode::Regular { .. } => "regular", + RoutingMode::PrefillDecode { .. } => "prefill_decode", + RoutingMode::OpenAI { .. } => "openai", + } + } + + /// Check if service discovery is enabled + pub fn has_service_discovery(&self) -> bool { + self.discovery.as_ref().is_some_and(|d| d.enabled) + } + + /// Check if metrics are enabled + pub fn has_metrics(&self) -> bool { + self.metrics.is_some() + } + + /// Compute the effective retry config considering disable flag + pub fn effective_retry_config(&self) -> RetryConfig { + let mut cfg = self.retry.clone(); + if self.disable_retries { + cfg.max_retries = 1; + } + cfg + } + + /// Compute the effective circuit breaker config considering disable flag + pub fn effective_circuit_breaker_config(&self) -> CircuitBreakerConfig { + let mut cfg = self.circuit_breaker.clone(); + if self.disable_circuit_breaker { + cfg.failure_threshold = u32::MAX; + } + cfg + } + + /// Check if running in IGW (Inference Gateway) mode + pub fn is_igw_mode(&self) -> bool { + self.enable_igw + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ============= RouterConfig Tests ============= + + #[test] + fn test_router_config_default() { + let config = RouterConfig::default(); + + assert!( + matches!(config.mode, RoutingMode::Regular { worker_urls } if worker_urls.is_empty()) + ); + assert!(matches!(config.policy, PolicyConfig::Random)); + assert_eq!(config.host, "127.0.0.1"); + assert_eq!(config.port, 3001); + assert_eq!(config.max_payload_size, 536_870_912); + assert_eq!(config.request_timeout_secs, 1800); + assert_eq!(config.worker_startup_timeout_secs, 600); + assert_eq!(config.worker_startup_check_interval_secs, 30); + assert!(config.discovery.is_none()); + assert!(config.metrics.is_none()); + assert!(config.log_dir.is_none()); + assert!(config.log_level.is_none()); + } + + #[test] + fn test_router_config_new() { + let mode = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string(), "http://worker2".to_string()], + }; + let policy = PolicyConfig::RoundRobin; + + let config = RouterConfig::new(mode, policy); + + match config.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls.len(), 2); + assert_eq!(worker_urls[0], "http://worker1"); + assert_eq!(worker_urls[1], "http://worker2"); + } + _ => panic!("Expected Regular mode"), + } + + assert!(matches!(config.policy, PolicyConfig::RoundRobin)); + // Other fields should be default + assert_eq!(config.host, "127.0.0.1"); + assert_eq!(config.port, 3001); + } + + #[test] + fn test_router_config_serialization() { + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }, + policy: PolicyConfig::Random, + host: "0.0.0.0".to_string(), + port: 8080, + max_payload_size: 1024, + request_timeout_secs: 30, + worker_startup_timeout_secs: 60, + worker_startup_check_interval_secs: 5, + dp_aware: false, + api_key: None, + discovery: Some(DiscoveryConfig::default()), + metrics: Some(MetricsConfig::default()), + log_dir: Some("/var/log".to_string()), + log_level: Some("debug".to_string()), + request_id_headers: None, + max_concurrent_requests: 64, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(config.host, deserialized.host); + assert_eq!(config.port, deserialized.port); + assert_eq!(config.max_payload_size, deserialized.max_payload_size); + assert!(deserialized.discovery.is_some()); + assert!(deserialized.metrics.is_some()); + } + + // ============= RoutingMode Tests ============= + + #[test] + fn test_routing_mode_is_pd_mode() { + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + assert!(!regular.is_pd_mode()); + + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), Some(8001))], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + assert!(pd.is_pd_mode()); + } + + #[test] + fn test_routing_mode_worker_count() { + let regular = RoutingMode::Regular { + worker_urls: vec![ + "http://worker1".to_string(), + "http://worker2".to_string(), + "http://worker3".to_string(), + ], + }; + assert_eq!(regular.worker_count(), 3); + + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1".to_string(), Some(8001)), + ("http://prefill2".to_string(), None), + ], + decode_urls: vec![ + "http://decode1".to_string(), + "http://decode2".to_string(), + "http://decode3".to_string(), + ], + prefill_policy: None, + decode_policy: None, + }; + assert_eq!(pd.worker_count(), 5); + + let empty_regular = RoutingMode::Regular { + worker_urls: vec![], + }; + assert_eq!(empty_regular.worker_count(), 0); + } + + #[test] + fn test_routing_mode_serialization() { + // Test Regular mode + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + let json = serde_json::to_string(®ular).unwrap(); + assert!(json.contains("\"type\":\"regular\"")); + assert!(json.contains("\"worker_urls\"")); + + // Test PrefillDecode mode + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), Some(8001))], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + let json = serde_json::to_string(&pd).unwrap(); + assert!(json.contains("\"type\":\"prefill_decode\"")); + assert!(json.contains("\"prefill_urls\"")); + assert!(json.contains("\"decode_urls\"")); + } + + // ============= PolicyConfig Tests ============= + + #[test] + fn test_policy_config_name() { + assert_eq!(PolicyConfig::Random.name(), "random"); + assert_eq!(PolicyConfig::RoundRobin.name(), "round_robin"); + + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.8, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 1000, + }; + assert_eq!(cache_aware.name(), "cache_aware"); + + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }; + assert_eq!(power_of_two.name(), "power_of_two"); + } + + #[test] + fn test_policy_config_serialization() { + // Test Random + let random = PolicyConfig::Random; + let json = serde_json::to_string(&random).unwrap(); + assert_eq!(json, r#"{"type":"random"}"#); + + // Test CacheAware with all parameters + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.8, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 1000, + }; + let json = serde_json::to_string(&cache_aware).unwrap(); + assert!(json.contains("\"type\":\"cache_aware\"")); + assert!(json.contains("\"cache_threshold\":0.8")); + assert!(json.contains("\"balance_abs_threshold\":10")); + + // Test PowerOfTwo + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }; + let json = serde_json::to_string(&power_of_two).unwrap(); + assert!(json.contains("\"type\":\"power_of_two\"")); + assert!(json.contains("\"load_check_interval_secs\":60")); + } + + #[test] + fn test_cache_aware_parameters() { + let cache_aware = PolicyConfig::CacheAware { + cache_threshold: 0.75, + balance_abs_threshold: 20, + balance_rel_threshold: 2.0, + eviction_interval_secs: 600, + max_tree_size: 5000, + }; + + match cache_aware { + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + assert!((cache_threshold - 0.75).abs() < 0.0001); + assert_eq!(balance_abs_threshold, 20); + assert!((balance_rel_threshold - 2.0).abs() < 0.0001); + assert_eq!(eviction_interval_secs, 600); + assert_eq!(max_tree_size, 5000); + } + _ => panic!("Expected CacheAware"), + } + } + + #[test] + fn test_power_of_two_parameters() { + let power_of_two = PolicyConfig::PowerOfTwo { + load_check_interval_secs: 120, + }; + + match power_of_two { + PolicyConfig::PowerOfTwo { + load_check_interval_secs, + } => { + assert_eq!(load_check_interval_secs, 120); + } + _ => panic!("Expected PowerOfTwo"), + } + } + + // ============= DiscoveryConfig Tests ============= + + #[test] + fn test_discovery_config_default() { + let config = DiscoveryConfig::default(); + + assert!(!config.enabled); + assert!(config.namespace.is_none()); + assert_eq!(config.port, 8000); + assert_eq!(config.check_interval_secs, 120); + assert!(config.selector.is_empty()); + assert!(config.prefill_selector.is_empty()); + assert!(config.decode_selector.is_empty()); + assert_eq!(config.bootstrap_port_annotation, "sglang.ai/bootstrap-port"); + } + + #[test] + fn test_discovery_config_with_selectors() { + let mut selector = HashMap::new(); + selector.insert("app".to_string(), "sglang".to_string()); + selector.insert("role".to_string(), "worker".to_string()); + + let config = DiscoveryConfig { + enabled: true, + namespace: Some("default".to_string()), + port: 9000, + check_interval_secs: 30, + selector: selector.clone(), + prefill_selector: selector.clone(), + decode_selector: selector.clone(), + bootstrap_port_annotation: "custom.io/port".to_string(), + }; + + assert!(config.enabled); + assert_eq!(config.namespace, Some("default".to_string())); + assert_eq!(config.port, 9000); + assert_eq!(config.selector.len(), 2); + assert_eq!(config.selector.get("app"), Some(&"sglang".to_string())); + } + + #[test] + fn test_discovery_config_namespace() { + // Test None namespace (all namespaces) + let config = DiscoveryConfig { + namespace: None, + ..Default::default() + }; + assert!(config.namespace.is_none()); + + // Test specific namespace + let config = DiscoveryConfig { + namespace: Some("production".to_string()), + ..Default::default() + }; + assert_eq!(config.namespace, Some("production".to_string())); + } + + // ============= MetricsConfig Tests ============= + + #[test] + fn test_metrics_config_default() { + let config = MetricsConfig::default(); + + assert_eq!(config.port, 29000); + assert_eq!(config.host, "127.0.0.1"); + } + + #[test] + fn test_metrics_config_custom() { + let config = MetricsConfig { + port: 9090, + host: "0.0.0.0".to_string(), + }; + + assert_eq!(config.port, 9090); + assert_eq!(config.host, "0.0.0.0"); + } + + // ============= RouterConfig Utility Methods Tests ============= + + #[test] + fn test_mode_type() { + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + ..Default::default() + }; + assert_eq!(config.mode_type(), "regular"); + + let config = RouterConfig { + mode: RoutingMode::PrefillDecode { + prefill_urls: vec![], + decode_urls: vec![], + prefill_policy: None, + decode_policy: None, + }, + ..Default::default() + }; + assert_eq!(config.mode_type(), "prefill_decode"); + } + + #[test] + fn test_has_service_discovery() { + let config = RouterConfig::default(); + assert!(!config.has_service_discovery()); + + let config = RouterConfig { + discovery: Some(DiscoveryConfig { + enabled: false, + ..Default::default() + }), + ..Default::default() + }; + assert!(!config.has_service_discovery()); + + let config = RouterConfig { + discovery: Some(DiscoveryConfig { + enabled: true, + ..Default::default() + }), + ..Default::default() + }; + assert!(config.has_service_discovery()); + } + + #[test] + fn test_has_metrics() { + let config = RouterConfig::default(); + assert!(!config.has_metrics()); + + let config = RouterConfig { + metrics: Some(MetricsConfig::default()), + ..Default::default() + }; + assert!(config.has_metrics()); + } + + // ============= Edge Cases ============= + + #[test] + fn test_large_worker_lists() { + let large_urls: Vec = (0..1000).map(|i| format!("http://worker{}", i)).collect(); + + let mode = RoutingMode::Regular { + worker_urls: large_urls.clone(), + }; + + assert_eq!(mode.worker_count(), 1000); + + // Test serialization with large list + let config = RouterConfig { + mode, + ..Default::default() + }; + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + match deserialized.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls.len(), 1000); + } + _ => panic!("Expected Regular mode"), + } + } + + #[test] + fn test_unicode_in_config() { + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec!["http://работник1".to_string(), "http://工作者2".to_string()], + }, + log_dir: Some("/日志/目录".to_string()), + ..Default::default() + }; + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + match deserialized.mode { + RoutingMode::Regular { worker_urls } => { + assert_eq!(worker_urls[0], "http://работник1"); + assert_eq!(worker_urls[1], "http://工作者2"); + } + _ => panic!("Expected Regular mode"), + } + + assert_eq!(deserialized.log_dir, Some("/日志/目录".to_string())); + } + + #[test] + fn test_empty_string_fields() { + let config = RouterConfig { + host: "".to_string(), + log_dir: Some("".to_string()), + log_level: Some("".to_string()), + ..Default::default() + }; + + assert_eq!(config.host, ""); + assert_eq!(config.log_dir, Some("".to_string())); + assert_eq!(config.log_level, Some("".to_string())); + } + + // ============= Complex Configuration Tests ============= + + #[test] + fn test_full_pd_mode_config() { + let config = RouterConfig { + mode: RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8000".to_string(), Some(8001)), + ("http://prefill2:8000".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: None, + decode_policy: None, + }, + policy: PolicyConfig::PowerOfTwo { + load_check_interval_secs: 30, + }, + host: "0.0.0.0".to_string(), + port: 3000, + max_payload_size: 1048576, + request_timeout_secs: 120, + worker_startup_timeout_secs: 60, + worker_startup_check_interval_secs: 5, + dp_aware: false, + api_key: None, + discovery: Some(DiscoveryConfig { + enabled: true, + namespace: Some("sglang".to_string()), + ..Default::default() + }), + metrics: Some(MetricsConfig { + port: 9090, + host: "0.0.0.0".to_string(), + }), + log_dir: Some("/var/log/sglang".to_string()), + log_level: Some("info".to_string()), + request_id_headers: None, + max_concurrent_requests: 64, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + assert!(config.mode.is_pd_mode()); + assert_eq!(config.mode.worker_count(), 4); + assert_eq!(config.policy.name(), "power_of_two"); + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + } + + #[test] + fn test_full_regular_mode_config() { + let mut selector = HashMap::new(); + selector.insert("app".to_string(), "sglang".to_string()); + + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + "http://worker3:8000".to_string(), + ], + }, + policy: PolicyConfig::CacheAware { + cache_threshold: 0.9, + balance_abs_threshold: 5, + balance_rel_threshold: 1.2, + eviction_interval_secs: 600, + max_tree_size: 10000, + }, + host: "0.0.0.0".to_string(), + port: 3001, + max_payload_size: 536870912, + request_timeout_secs: 300, + worker_startup_timeout_secs: 180, + worker_startup_check_interval_secs: 15, + dp_aware: false, + api_key: None, + discovery: Some(DiscoveryConfig { + enabled: true, + namespace: None, + port: 8080, + check_interval_secs: 45, + selector, + ..Default::default() + }), + metrics: Some(MetricsConfig::default()), + log_dir: None, + log_level: Some("debug".to_string()), + request_id_headers: None, + max_concurrent_requests: 64, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + assert!(!config.mode.is_pd_mode()); + assert_eq!(config.mode.worker_count(), 3); + assert_eq!(config.policy.name(), "cache_aware"); + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + } + + #[test] + fn test_config_with_all_options() { + let mut selectors = HashMap::new(); + selectors.insert("env".to_string(), "prod".to_string()); + selectors.insert("version".to_string(), "v1".to_string()); + + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }, + policy: PolicyConfig::RoundRobin, + host: "::1".to_string(), // IPv6 + port: 8888, + max_payload_size: 1024 * 1024 * 512, // 512MB + request_timeout_secs: 900, + worker_startup_timeout_secs: 600, + worker_startup_check_interval_secs: 20, + dp_aware: false, + api_key: None, + discovery: Some(DiscoveryConfig { + enabled: true, + namespace: Some("production".to_string()), + port: 8443, + check_interval_secs: 120, + selector: selectors.clone(), + prefill_selector: selectors.clone(), + decode_selector: selectors, + bootstrap_port_annotation: "mycompany.io/bootstrap".to_string(), + }), + metrics: Some(MetricsConfig { + port: 9999, + host: "::".to_string(), // IPv6 any + }), + log_dir: Some("/opt/logs/sglang".to_string()), + log_level: Some("trace".to_string()), + request_id_headers: None, + max_concurrent_requests: 64, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: HealthCheckConfig::default(), + enable_igw: false, + queue_size: 100, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + assert!(config.has_service_discovery()); + assert!(config.has_metrics()); + assert_eq!(config.mode_type(), "regular"); + + // Test round-trip serialization + let json = serde_json::to_string_pretty(&config).unwrap(); + let deserialized: RouterConfig = serde_json::from_str(&json).unwrap(); + + assert_eq!(deserialized.host, "::1"); + assert_eq!(deserialized.port, 8888); + assert_eq!( + deserialized.discovery.unwrap().namespace, + Some("production".to_string()) + ); + } + + // ============= Policy Fallback Tests ============= + + #[test] + fn test_pd_policy_fallback_both_specified() { + // When both prefill and decode policies are specified, they should be used + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }; + + let main_policy = PolicyConfig::Random; + + // Both specific policies should be used + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { .. } => {} // Success + _ => panic!("Expected CacheAware for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::PowerOfTwo { .. } => {} // Success + _ => panic!("Expected PowerOfTwo for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_only_prefill() { + // When only prefill policy is specified, decode should use main policy + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: None, + }; + + let main_policy = PolicyConfig::RoundRobin; + + // Prefill should use specific policy + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { .. } => {} // Success + _ => panic!("Expected CacheAware for prefill"), + } + + // Decode should fall back to main policy + match pd.get_decode_policy(&main_policy) { + PolicyConfig::RoundRobin => {} // Success + _ => panic!("Expected RoundRobin for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_only_decode() { + // When only decode policy is specified, prefill should use main policy + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }; + + let main_policy = PolicyConfig::Random; + + // Prefill should fall back to main policy + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::Random => {} // Success + _ => panic!("Expected Random for prefill"), + } + + // Decode should use specific policy + match pd.get_decode_policy(&main_policy) { + PolicyConfig::PowerOfTwo { .. } => {} // Success + _ => panic!("Expected PowerOfTwo for decode"), + } + } + + #[test] + fn test_pd_policy_fallback_none_specified() { + // When no specific policies are specified, both should use main policy + let pd = RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1".to_string(), None)], + decode_urls: vec!["http://decode1".to_string()], + prefill_policy: None, + decode_policy: None, + }; + + let main_policy = PolicyConfig::CacheAware { + cache_threshold: 0.7, + balance_abs_threshold: 20, + balance_rel_threshold: 1.5, + eviction_interval_secs: 300, + max_tree_size: 2000, + }; + + // Both should fall back to main policy + match pd.get_prefill_policy(&main_policy) { + PolicyConfig::CacheAware { + cache_threshold, .. + } => { + assert!((cache_threshold - 0.7).abs() < 0.0001); + } + _ => panic!("Expected CacheAware for prefill"), + } + + match pd.get_decode_policy(&main_policy) { + PolicyConfig::CacheAware { + cache_threshold, .. + } => { + assert!((cache_threshold - 0.7).abs() < 0.0001); + } + _ => panic!("Expected CacheAware for decode"), + } + } + + #[test] + fn test_regular_mode_policy_fallback() { + // For regular mode, the helper methods should just return the main policy + let regular = RoutingMode::Regular { + worker_urls: vec!["http://worker1".to_string()], + }; + + let main_policy = PolicyConfig::RoundRobin; + + // Both methods should return main policy for regular mode + match regular.get_prefill_policy(&main_policy) { + PolicyConfig::RoundRobin => {} // Success + _ => panic!("Expected RoundRobin for regular mode"), + } + + match regular.get_decode_policy(&main_policy) { + PolicyConfig::RoundRobin => {} // Success + _ => panic!("Expected RoundRobin for regular mode"), + } + } +} diff --git a/sgl-router/src/config/validation.rs b/sgl-router/src/config/validation.rs new file mode 100644 index 000000000..710ad3fc8 --- /dev/null +++ b/sgl-router/src/config/validation.rs @@ -0,0 +1,776 @@ +use super::*; + +/// Configuration validator +pub struct ConfigValidator; + +impl ConfigValidator { + /// Validate a complete router configuration + pub fn validate(config: &RouterConfig) -> ConfigResult<()> { + // Check if service discovery is enabled + let has_service_discovery = config.discovery.as_ref().is_some_and(|d| d.enabled); + + Self::validate_mode(&config.mode, has_service_discovery)?; + Self::validate_policy(&config.policy)?; + Self::validate_server_settings(config)?; + + if let Some(discovery) = &config.discovery { + Self::validate_discovery(discovery, &config.mode)?; + } + + if let Some(metrics) = &config.metrics { + Self::validate_metrics(metrics)?; + } + + Self::validate_compatibility(config)?; + + // Validate effective retry/CB configs (respect disable flags) + let retry_cfg = config.effective_retry_config(); + let cb_cfg = config.effective_circuit_breaker_config(); + Self::validate_retry(&retry_cfg)?; + Self::validate_circuit_breaker(&cb_cfg)?; + + Ok(()) + } + + /// Validate routing mode configuration + fn validate_mode(mode: &RoutingMode, has_service_discovery: bool) -> ConfigResult<()> { + match mode { + RoutingMode::Regular { worker_urls } => { + // Validate URLs if any are provided + if !worker_urls.is_empty() { + Self::validate_urls(worker_urls)?; + } + // Note: We allow empty worker URLs even without service discovery + // to let the router start and fail at runtime when routing requests. + // This matches legacy behavior and test expectations. + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } => { + // Only require URLs if service discovery is disabled + if !has_service_discovery { + if prefill_urls.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "PD mode requires at least one prefill worker URL".to_string(), + }); + } + if decode_urls.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "PD mode requires at least one decode worker URL".to_string(), + }); + } + } + + // Validate URLs if any are provided + if !prefill_urls.is_empty() { + let prefill_url_strings: Vec = + prefill_urls.iter().map(|(url, _)| url.clone()).collect(); + Self::validate_urls(&prefill_url_strings)?; + } + if !decode_urls.is_empty() { + Self::validate_urls(decode_urls)?; + } + + // Validate bootstrap ports + for (_url, port) in prefill_urls { + if let Some(port) = port { + if *port == 0 { + return Err(ConfigError::InvalidValue { + field: "bootstrap_port".to_string(), + value: port.to_string(), + reason: "Port must be between 1 and 65535".to_string(), + }); + } + } + } + + // Validate optional prefill and decode policies + if let Some(p_policy) = prefill_policy { + Self::validate_policy(p_policy)?; + } + if let Some(d_policy) = decode_policy { + Self::validate_policy(d_policy)?; + } + } + RoutingMode::OpenAI { worker_urls } => { + // Require exactly one worker URL for OpenAI router + if worker_urls.len() != 1 { + return Err(ConfigError::ValidationFailed { + reason: "OpenAI mode requires exactly one --worker-urls entry".to_string(), + }); + } + // Validate URL format + if let Err(e) = url::Url::parse(&worker_urls[0]) { + return Err(ConfigError::ValidationFailed { + reason: format!("Invalid OpenAI worker URL '{}': {}", &worker_urls[0], e), + }); + } + } + } + Ok(()) + } + + /// Validate policy configuration + fn validate_policy(policy: &PolicyConfig) -> ConfigResult<()> { + match policy { + PolicyConfig::Random | PolicyConfig::RoundRobin => { + // No specific validation needed + } + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold: _, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + if !(0.0..=1.0).contains(cache_threshold) { + return Err(ConfigError::InvalidValue { + field: "cache_threshold".to_string(), + value: cache_threshold.to_string(), + reason: "Must be between 0.0 and 1.0".to_string(), + }); + } + + if *balance_rel_threshold < 1.0 { + return Err(ConfigError::InvalidValue { + field: "balance_rel_threshold".to_string(), + value: balance_rel_threshold.to_string(), + reason: "Must be >= 1.0".to_string(), + }); + } + + if *eviction_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "eviction_interval_secs".to_string(), + value: eviction_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if *max_tree_size == 0 { + return Err(ConfigError::InvalidValue { + field: "max_tree_size".to_string(), + value: max_tree_size.to_string(), + reason: "Must be > 0".to_string(), + }); + } + } + PolicyConfig::PowerOfTwo { + load_check_interval_secs, + } => { + if *load_check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "load_check_interval_secs".to_string(), + value: load_check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + } + } + Ok(()) + } + + /// Validate server configuration + fn validate_server_settings(config: &RouterConfig) -> ConfigResult<()> { + if config.port == 0 { + return Err(ConfigError::InvalidValue { + field: "port".to_string(), + value: config.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if config.max_payload_size == 0 { + return Err(ConfigError::InvalidValue { + field: "max_payload_size".to_string(), + value: config.max_payload_size.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.request_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "request_timeout_secs".to_string(), + value: config.request_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.worker_startup_timeout_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "worker_startup_timeout_secs".to_string(), + value: config.worker_startup_timeout_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + if config.worker_startup_check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "worker_startup_check_interval_secs".to_string(), + value: config.worker_startup_check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + Ok(()) + } + + /// Validate service discovery configuration + fn validate_discovery(discovery: &DiscoveryConfig, mode: &RoutingMode) -> ConfigResult<()> { + if !discovery.enabled { + return Ok(()); // No validation needed if disabled + } + + if discovery.port == 0 { + return Err(ConfigError::InvalidValue { + field: "discovery.port".to_string(), + value: discovery.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if discovery.check_interval_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "discovery.check_interval_secs".to_string(), + value: discovery.check_interval_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + + // Validate selectors based on mode + match mode { + RoutingMode::Regular { .. } => { + if discovery.selector.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "Regular mode with service discovery requires a non-empty selector" + .to_string(), + }); + } + } + RoutingMode::PrefillDecode { .. } => { + if discovery.prefill_selector.is_empty() && discovery.decode_selector.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "PD mode with service discovery requires at least one non-empty selector (prefill or decode)".to_string(), + }); + } + } + RoutingMode::OpenAI { .. } => { + // OpenAI mode doesn't use service discovery + return Err(ConfigError::ValidationFailed { + reason: "OpenAI mode does not support service discovery".to_string(), + }); + } + } + + Ok(()) + } + + /// Validate metrics configuration + fn validate_metrics(metrics: &MetricsConfig) -> ConfigResult<()> { + if metrics.port == 0 { + return Err(ConfigError::InvalidValue { + field: "metrics.port".to_string(), + value: metrics.port.to_string(), + reason: "Port must be > 0".to_string(), + }); + } + + if metrics.host.is_empty() { + return Err(ConfigError::InvalidValue { + field: "metrics.host".to_string(), + value: metrics.host.clone(), + reason: "Host cannot be empty".to_string(), + }); + } + + Ok(()) + } + + /// Validate retry configuration + fn validate_retry(retry: &RetryConfig) -> ConfigResult<()> { + if retry.max_retries < 1 { + return Err(ConfigError::InvalidValue { + field: "retry.max_retries".to_string(), + value: retry.max_retries.to_string(), + reason: "Must be >= 1 (set to 1 to effectively disable retries)".to_string(), + }); + } + if retry.initial_backoff_ms == 0 { + return Err(ConfigError::InvalidValue { + field: "retry.initial_backoff_ms".to_string(), + value: retry.initial_backoff_ms.to_string(), + reason: "Must be > 0".to_string(), + }); + } + if retry.max_backoff_ms < retry.initial_backoff_ms { + return Err(ConfigError::InvalidValue { + field: "retry.max_backoff_ms".to_string(), + value: retry.max_backoff_ms.to_string(), + reason: "Must be >= initial_backoff_ms".to_string(), + }); + } + if retry.backoff_multiplier < 1.0 { + return Err(ConfigError::InvalidValue { + field: "retry.backoff_multiplier".to_string(), + value: retry.backoff_multiplier.to_string(), + reason: "Must be >= 1.0".to_string(), + }); + } + if !(0.0..=1.0).contains(&retry.jitter_factor) { + return Err(ConfigError::InvalidValue { + field: "retry.jitter_factor".to_string(), + value: retry.jitter_factor.to_string(), + reason: "Must be between 0.0 and 1.0".to_string(), + }); + } + Ok(()) + } + + /// Validate circuit breaker configuration + fn validate_circuit_breaker(cb: &CircuitBreakerConfig) -> ConfigResult<()> { + if cb.failure_threshold < 1 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.failure_threshold".to_string(), + value: cb.failure_threshold.to_string(), + reason: "Must be >= 1 (set to u32::MAX to effectively disable CB)".to_string(), + }); + } + if cb.success_threshold < 1 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.success_threshold".to_string(), + value: cb.success_threshold.to_string(), + reason: "Must be >= 1".to_string(), + }); + } + if cb.timeout_duration_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.timeout_duration_secs".to_string(), + value: cb.timeout_duration_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + if cb.window_duration_secs == 0 { + return Err(ConfigError::InvalidValue { + field: "circuit_breaker.window_duration_secs".to_string(), + value: cb.window_duration_secs.to_string(), + reason: "Must be > 0".to_string(), + }); + } + Ok(()) + } + + /// Validate compatibility between different configuration sections + fn validate_compatibility(config: &RouterConfig) -> ConfigResult<()> { + // IGW mode is independent - skip other compatibility checks when enabled + if config.enable_igw { + return Ok(()); + } + + // Validate gRPC connection mode requires tokenizer configuration + if config.connection_mode == ConnectionMode::Grpc + && config.tokenizer_path.is_none() + && config.model_path.is_none() + { + return Err(ConfigError::ValidationFailed { + reason: "gRPC connection mode requires either --tokenizer-path or --model-path to be specified".to_string(), + }); + } + + // All policies are now supported for both router types thanks to the unified trait design + // No mode/policy restrictions needed anymore + + // Check if service discovery is enabled for worker count validation + let has_service_discovery = config.discovery.as_ref().is_some_and(|d| d.enabled); + + // Only validate worker counts if service discovery is disabled + if !has_service_discovery { + // Check if power-of-two policy makes sense with insufficient workers + if let PolicyConfig::PowerOfTwo { .. } = &config.policy { + let worker_count = config.mode.worker_count(); + if worker_count < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: "Power-of-two policy requires at least 2 workers".to_string(), + }); + } + } + + // For PD mode, validate that policies have sufficient workers + if let RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } = &config.mode + { + // Check power-of-two for prefill + if let Some(PolicyConfig::PowerOfTwo { .. }) = prefill_policy { + if prefill_urls.len() < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: "Power-of-two policy for prefill requires at least 2 prefill workers".to_string(), + }); + } + } + + // Check power-of-two for decode + if let Some(PolicyConfig::PowerOfTwo { .. }) = decode_policy { + if decode_urls.len() < 2 { + return Err(ConfigError::IncompatibleConfig { + reason: + "Power-of-two policy for decode requires at least 2 decode workers" + .to_string(), + }); + } + } + } + } + + // Service discovery is conflict with dp_aware routing for now + // since it's not fully supported yet + if has_service_discovery && config.dp_aware { + return Err(ConfigError::IncompatibleConfig { + reason: "DP-aware routing is not compatible with service discovery".to_string(), + }); + } + + Ok(()) + } + + /// Validate URL format + fn validate_urls(urls: &[String]) -> ConfigResult<()> { + for url in urls { + if url.is_empty() { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL cannot be empty".to_string(), + }); + } + + if !url.starts_with("http://") + && !url.starts_with("https://") + && !url.starts_with("grpc://") + { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL must start with http://, https://, or grpc://".to_string(), + }); + } + + // Basic URL validation + match ::url::Url::parse(url) { + Ok(parsed) => { + // Additional validation + if parsed.host_str().is_none() { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: "URL must have a valid host".to_string(), + }); + } + } + Err(e) => { + return Err(ConfigError::InvalidValue { + field: "worker_url".to_string(), + value: url.clone(), + reason: format!("Invalid URL format: {}", e), + }); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_validate_regular_mode() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["http://worker:8000".to_string()], + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_empty_worker_urls() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Empty worker URLs are now allowed to match legacy behavior + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_empty_worker_urls_with_service_discovery() { + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![], + }, + PolicyConfig::Random, + ); + + // Enable service discovery + config.discovery = Some(DiscoveryConfig { + enabled: true, + selector: vec![("app".to_string(), "test".to_string())] + .into_iter() + .collect(), + ..Default::default() + }); + + // Should pass validation since service discovery is enabled + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_invalid_urls() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["invalid-url".to_string()], + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_err()); + } + + #[test] + fn test_validate_cache_aware_thresholds() { + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + ], + }, + PolicyConfig::CacheAware { + cache_threshold: 1.5, // Invalid: > 1.0 + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + assert!(ConfigValidator::validate(&config).is_err()); + } + + #[test] + fn test_validate_cache_aware_single_worker() { + // Cache-aware with single worker should be allowed (even if not optimal) + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["http://worker1:8000".to_string()], + }, + PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_pd_mode() { + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), Some(8081))], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::Random, + ); + + assert!(ConfigValidator::validate(&config).is_ok()); + } + + #[test] + fn test_validate_roundrobin_with_pd_mode() { + // RoundRobin with PD mode is now supported + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), None)], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::RoundRobin, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_cache_aware_with_pd_mode() { + // CacheAware with PD mode is now supported + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8000".to_string(), None)], + decode_urls: vec!["http://decode:8000".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_power_of_two_with_regular_mode() { + // PowerOfTwo with Regular mode is now supported + let config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec![ + "http://worker1:8000".to_string(), + "http://worker2:8000".to_string(), + ], + }, + PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_pd_mode_with_separate_policies() { + // Test PD mode with different policies for prefill and decode + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8000".to_string(), None), + ("http://prefill2:8000".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 60, + max_tree_size: 1000, + }), + decode_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), + }, + PolicyConfig::Random, // Main policy as fallback + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_pd_mode_power_of_two_insufficient_workers() { + // Test that power-of-two policy requires at least 2 workers + let config = RouterConfig::new( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill1:8000".to_string(), None)], // Only 1 prefill + decode_urls: vec![ + "http://decode1:8000".to_string(), + "http://decode2:8000".to_string(), + ], + prefill_policy: Some(PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }), // Requires 2+ workers + decode_policy: None, + }, + PolicyConfig::Random, + ); + + let result = ConfigValidator::validate(&config); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("prefill requires at least 2")); + } + } + + #[test] + fn test_validate_grpc_requires_tokenizer() { + // Test that gRPC connection mode requires tokenizer configuration + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + // Set connection mode to gRPC without tokenizer config + config.connection_mode = ConnectionMode::Grpc; + config.tokenizer_path = None; + config.model_path = None; + + let result = ConfigValidator::validate(&config); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("gRPC connection mode requires")); + } + } + + #[test] + fn test_validate_grpc_with_model_path() { + // Test that gRPC works with model_path + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + config.connection_mode = ConnectionMode::Grpc; + config.model_path = Some("meta-llama/Llama-3-8B".to_string()); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } + + #[test] + fn test_validate_grpc_with_tokenizer_path() { + // Test that gRPC works with tokenizer_path + let mut config = RouterConfig::new( + RoutingMode::Regular { + worker_urls: vec!["grpc://worker:50051".to_string()], + }, + PolicyConfig::Random, + ); + + config.connection_mode = ConnectionMode::Grpc; + config.tokenizer_path = Some("/path/to/tokenizer.json".to_string()); + + let result = ConfigValidator::validate(&config); + assert!(result.is_ok()); + } +} diff --git a/sgl-router/src/core/circuit_breaker.rs b/sgl-router/src/core/circuit_breaker.rs new file mode 100644 index 000000000..5c374233e --- /dev/null +++ b/sgl-router/src/core/circuit_breaker.rs @@ -0,0 +1,555 @@ +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tracing::info; + +/// Circuit breaker configuration +#[derive(Debug, Clone)] +pub struct CircuitBreakerConfig { + /// Number of consecutive failures to open the circuit + pub failure_threshold: u32, + /// Success threshold to close circuit from half-open + pub success_threshold: u32, + /// Duration to wait before attempting half-open + pub timeout_duration: Duration, + /// Time window for failure counting + pub window_duration: Duration, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + failure_threshold: 5, + success_threshold: 2, + timeout_duration: Duration::from_secs(30), + window_duration: Duration::from_secs(60), + } + } +} + +/// Circuit breaker state +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CircuitState { + /// Normal operation - requests are allowed + Closed, + /// Circuit is open - requests are rejected + Open, + /// Testing if service has recovered - limited requests allowed + HalfOpen, +} + +impl std::fmt::Display for CircuitState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CircuitState::Closed => write!(f, "Closed"), + CircuitState::Open => write!(f, "Open"), + CircuitState::HalfOpen => write!(f, "HalfOpen"), + } + } +} + +/// Circuit breaker implementation +#[derive(Debug)] +pub struct CircuitBreaker { + state: Arc>, + consecutive_failures: Arc, + consecutive_successes: Arc, + total_failures: Arc, + total_successes: Arc, + last_failure_time: Arc>>, + last_state_change: Arc>, + config: CircuitBreakerConfig, +} + +impl CircuitBreaker { + /// Create a new circuit breaker with default configuration + pub fn new() -> Self { + Self::with_config(CircuitBreakerConfig::default()) + } + + /// Create a new circuit breaker with custom configuration + pub fn with_config(config: CircuitBreakerConfig) -> Self { + Self { + state: Arc::new(RwLock::new(CircuitState::Closed)), + consecutive_failures: Arc::new(AtomicU32::new(0)), + consecutive_successes: Arc::new(AtomicU32::new(0)), + total_failures: Arc::new(AtomicU64::new(0)), + total_successes: Arc::new(AtomicU64::new(0)), + last_failure_time: Arc::new(RwLock::new(None)), + last_state_change: Arc::new(RwLock::new(Instant::now())), + config, + } + } + + /// Check if a request can be executed + pub fn can_execute(&self) -> bool { + // First check if we need to transition from Open to HalfOpen + self.check_and_update_state(); + + let state = *self.state.read().unwrap(); + match state { + CircuitState::Closed => true, + CircuitState::Open => false, + CircuitState::HalfOpen => true, // Allow limited requests in half-open state + } + } + + /// Get the current state + pub fn state(&self) -> CircuitState { + self.check_and_update_state(); + *self.state.read().unwrap() + } + + /// Record the outcome of a request + pub fn record_outcome(&self, success: bool) { + if success { + self.record_success(); + } else { + self.record_failure(); + } + } + + /// Record a successful request + pub fn record_success(&self) { + self.total_successes.fetch_add(1, Ordering::Relaxed); + self.consecutive_failures.store(0, Ordering::Release); + let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1; + // Outcome-level metrics are recorded at the worker level where the worker label is known + + let current_state = *self.state.read().unwrap(); + + match current_state { + CircuitState::HalfOpen => { + // Check if we've reached the success threshold to close the circuit + if successes >= self.config.success_threshold { + self.transition_to(CircuitState::Closed); + } + } + CircuitState::Closed => { + // Already closed, nothing to do + } + CircuitState::Open => { + // Shouldn't happen, but if it does, stay open + tracing::warn!("Success recorded while circuit is open"); + } + } + } + + /// Record a failed request + pub fn record_failure(&self) { + self.total_failures.fetch_add(1, Ordering::Relaxed); + self.consecutive_successes.store(0, Ordering::Release); + let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1; + // Outcome-level metrics are recorded at the worker level where the worker label is known + + // Update last failure time + { + let mut last_failure = self.last_failure_time.write().unwrap(); + *last_failure = Some(Instant::now()); + } + + let current_state = *self.state.read().unwrap(); + + match current_state { + CircuitState::Closed => { + // Check if we've reached the failure threshold to open the circuit + if failures >= self.config.failure_threshold { + self.transition_to(CircuitState::Open); + } + } + CircuitState::HalfOpen => { + // Single failure in half-open state reopens the circuit + self.transition_to(CircuitState::Open); + } + CircuitState::Open => { + // Already open, nothing to do + } + } + } + + /// Check and update state based on timeout + fn check_and_update_state(&self) { + let current_state = *self.state.read().unwrap(); + + if current_state == CircuitState::Open { + // Check if timeout has expired + let last_change = *self.last_state_change.read().unwrap(); + if last_change.elapsed() >= self.config.timeout_duration { + self.transition_to(CircuitState::HalfOpen); + } + } + } + + /// Transition to a new state + fn transition_to(&self, new_state: CircuitState) { + let mut state = self.state.write().unwrap(); + let old_state = *state; + + if old_state != new_state { + *state = new_state; + + // Update last state change time + let mut last_change = self.last_state_change.write().unwrap(); + *last_change = Instant::now(); + + // Reset counters based on transition + match new_state { + CircuitState::Closed => { + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + CircuitState::Open => { + self.consecutive_successes.store(0, Ordering::Release); + } + CircuitState::HalfOpen => { + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + } + + let from = match old_state { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + let to = match new_state { + CircuitState::Closed => "closed", + CircuitState::Open => "open", + CircuitState::HalfOpen => "half_open", + }; + info!("Circuit breaker state transition: {} -> {}", from, to); + // Transition metrics are recorded at the worker level where the worker label is known + } + } + + /// Get the number of consecutive failures + pub fn failure_count(&self) -> u32 { + self.consecutive_failures.load(Ordering::Acquire) + } + + /// Get the number of consecutive successes + pub fn success_count(&self) -> u32 { + self.consecutive_successes.load(Ordering::Acquire) + } + + /// Get total failures + pub fn total_failures(&self) -> u64 { + self.total_failures.load(Ordering::Relaxed) + } + + /// Get total successes + pub fn total_successes(&self) -> u64 { + self.total_successes.load(Ordering::Relaxed) + } + + /// Get time since last failure + pub fn time_since_last_failure(&self) -> Option { + self.last_failure_time.read().unwrap().map(|t| t.elapsed()) + } + + /// Get time since last state change + pub fn time_since_last_state_change(&self) -> Duration { + self.last_state_change.read().unwrap().elapsed() + } + + /// Check if the circuit is in a half-open state + pub fn is_half_open(&self) -> bool { + self.state() == CircuitState::HalfOpen + } + + /// Record a test success (for health check probing) + pub fn record_test_success(&self) { + if self.is_half_open() { + self.record_success(); + } + } + + /// Record a test failure (for health check probing) + pub fn record_test_failure(&self) { + if self.is_half_open() { + self.record_failure(); + } + } + + /// Reset the circuit breaker to closed state + pub fn reset(&self) { + self.transition_to(CircuitState::Closed); + self.consecutive_failures.store(0, Ordering::Release); + self.consecutive_successes.store(0, Ordering::Release); + } + + /// Force the circuit to open (for manual intervention) + pub fn force_open(&self) { + self.transition_to(CircuitState::Open); + } + + /// Get circuit breaker statistics + pub fn stats(&self) -> CircuitBreakerStats { + CircuitBreakerStats { + state: self.state(), + consecutive_failures: self.failure_count(), + consecutive_successes: self.success_count(), + total_failures: self.total_failures(), + total_successes: self.total_successes(), + time_since_last_failure: self.time_since_last_failure(), + time_since_last_state_change: self.time_since_last_state_change(), + } + } +} + +impl Clone for CircuitBreaker { + fn clone(&self) -> Self { + Self { + state: Arc::clone(&self.state), + consecutive_failures: Arc::clone(&self.consecutive_failures), + consecutive_successes: Arc::clone(&self.consecutive_successes), + total_failures: Arc::clone(&self.total_failures), + total_successes: Arc::clone(&self.total_successes), + last_failure_time: Arc::clone(&self.last_failure_time), + last_state_change: Arc::clone(&self.last_state_change), + config: self.config.clone(), + } + } +} + +impl Default for CircuitBreaker { + fn default() -> Self { + Self::new() + } +} + +/// Circuit breaker statistics +#[derive(Debug, Clone)] +pub struct CircuitBreakerStats { + pub state: CircuitState, + pub consecutive_failures: u32, + pub consecutive_successes: u32, + pub total_failures: u64, + pub total_successes: u64, + pub time_since_last_failure: Option, + pub time_since_last_state_change: Duration, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread; + + #[test] + fn test_circuit_breaker_initial_state() { + let cb = CircuitBreaker::new(); + assert_eq!(cb.state(), CircuitState::Closed); + assert!(cb.can_execute()); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 0); + } + + #[test] + fn test_circuit_opens_on_threshold() { + let config = CircuitBreakerConfig { + failure_threshold: 3, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Record failures up to threshold + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + cb.record_failure(); + + // Circuit should now be open + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + assert_eq!(cb.failure_count(), 3); + } + + #[test] + fn test_circuit_half_open_after_timeout() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + timeout_duration: Duration::from_millis(100), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Open the circuit + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + // Wait for timeout + thread::sleep(Duration::from_millis(150)); + + // Circuit should be half-open + assert_eq!(cb.state(), CircuitState::HalfOpen); + assert!(cb.can_execute()); + } + + #[test] + fn test_circuit_closes_on_success_threshold() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + success_threshold: 2, + timeout_duration: Duration::from_millis(50), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Open the circuit + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + // Wait for timeout + thread::sleep(Duration::from_millis(100)); + assert_eq!(cb.state(), CircuitState::HalfOpen); + + // Record successes + cb.record_success(); + assert_eq!(cb.state(), CircuitState::HalfOpen); + cb.record_success(); + + // Circuit should now be closed + assert_eq!(cb.state(), CircuitState::Closed); + assert!(cb.can_execute()); + } + + #[test] + fn test_circuit_reopens_on_half_open_failure() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + timeout_duration: Duration::from_millis(50), + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Open the circuit + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + // Wait for timeout + thread::sleep(Duration::from_millis(100)); + assert_eq!(cb.state(), CircuitState::HalfOpen); + + // Record a failure in half-open state + cb.record_failure(); + + // Circuit should reopen immediately + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + } + + #[test] + fn test_success_resets_failure_count() { + let config = CircuitBreakerConfig { + failure_threshold: 3, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Record some failures + cb.record_failure(); + cb.record_failure(); + assert_eq!(cb.failure_count(), 2); + + // Success should reset failure count + cb.record_success(); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 1); + + // Can now record more failures without opening + cb.record_failure(); + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Closed); + } + + #[test] + fn test_manual_reset() { + let config = CircuitBreakerConfig { + failure_threshold: 1, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + // Open the circuit + cb.record_failure(); + assert_eq!(cb.state(), CircuitState::Open); + + // Manual reset + cb.reset(); + assert_eq!(cb.state(), CircuitState::Closed); + assert_eq!(cb.failure_count(), 0); + assert_eq!(cb.success_count(), 0); + } + + #[test] + fn test_force_open() { + let cb = CircuitBreaker::new(); + assert_eq!(cb.state(), CircuitState::Closed); + + cb.force_open(); + assert_eq!(cb.state(), CircuitState::Open); + assert!(!cb.can_execute()); + } + + #[test] + fn test_stats() { + let config = CircuitBreakerConfig { + failure_threshold: 2, + ..Default::default() + }; + let cb = CircuitBreaker::with_config(config); + + cb.record_success(); + cb.record_failure(); + cb.record_failure(); + + let stats = cb.stats(); + assert_eq!(stats.state, CircuitState::Open); + assert_eq!(stats.consecutive_failures, 2); + assert_eq!(stats.consecutive_successes, 0); + assert_eq!(stats.total_failures, 2); + assert_eq!(stats.total_successes, 1); + } + + #[test] + fn test_clone() { + let cb1 = CircuitBreaker::new(); + cb1.record_failure(); + + let cb2 = cb1.clone(); + assert_eq!(cb2.failure_count(), 1); + + // Changes to cb1 affect cb2 (shared state) + cb1.record_failure(); + assert_eq!(cb2.failure_count(), 2); + } + + #[test] + fn test_thread_safety() { + use std::sync::Arc; + + let cb = Arc::new(CircuitBreaker::new()); + let mut handles = vec![]; + + // Spawn threads that record failures + for _ in 0..10 { + let cb_clone = Arc::clone(&cb); + let handle = thread::spawn(move || { + for _ in 0..100 { + cb_clone.record_failure(); + } + }); + handles.push(handle); + } + + // Wait for all threads + for handle in handles { + handle.join().unwrap(); + } + + // Should have recorded 1000 failures + assert_eq!(cb.total_failures(), 1000); + } +} diff --git a/sgl-router/src/core/error.rs b/sgl-router/src/core/error.rs new file mode 100644 index 000000000..74e0a0d25 --- /dev/null +++ b/sgl-router/src/core/error.rs @@ -0,0 +1,240 @@ +//! Error types for the SGLang router core +//! +//! This module defines error types used throughout the router for worker operations. + +use std::fmt; + +/// Worker-related errors +#[derive(Debug)] +pub enum WorkerError { + /// Health check failed + HealthCheckFailed { url: String, reason: String }, + /// Worker not found + WorkerNotFound { url: String }, + /// Invalid worker configuration + InvalidConfiguration { message: String }, + /// Network error + NetworkError { url: String, error: String }, + /// Worker is at capacity + WorkerAtCapacity { url: String }, + /// Invalid URL format + InvalidUrl { url: String }, +} + +impl fmt::Display for WorkerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WorkerError::HealthCheckFailed { url, reason } => { + write!(f, "Health check failed for worker {}: {}", url, reason) + } + WorkerError::WorkerNotFound { url } => { + write!(f, "Worker not found: {}", url) + } + WorkerError::InvalidConfiguration { message } => { + write!(f, "Invalid worker configuration: {}", message) + } + WorkerError::NetworkError { url, error } => { + write!(f, "Network error for worker {}: {}", url, error) + } + WorkerError::WorkerAtCapacity { url } => { + write!(f, "Worker at capacity: {}", url) + } + WorkerError::InvalidUrl { url } => { + write!(f, "Invalid URL format: {}", url) + } + } + } +} + +impl std::error::Error for WorkerError {} + +/// Result type for worker operations +pub type WorkerResult = Result; + +/// Convert from reqwest errors to worker errors +impl From for WorkerError { + fn from(err: reqwest::Error) -> Self { + WorkerError::NetworkError { + url: err.url().map(|u| u.to_string()).unwrap_or_default(), + error: err.to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + + #[test] + fn test_health_check_failed_display() { + let error = WorkerError::HealthCheckFailed { + url: "http://worker1:8080".to_string(), + reason: "Connection refused".to_string(), + }; + assert_eq!( + error.to_string(), + "Health check failed for worker http://worker1:8080: Connection refused" + ); + } + + #[test] + fn test_worker_not_found_display() { + let error = WorkerError::WorkerNotFound { + url: "http://worker2:8080".to_string(), + }; + assert_eq!(error.to_string(), "Worker not found: http://worker2:8080"); + } + + #[test] + fn test_invalid_configuration_display() { + let error = WorkerError::InvalidConfiguration { + message: "Missing port number".to_string(), + }; + assert_eq!( + error.to_string(), + "Invalid worker configuration: Missing port number" + ); + } + + #[test] + fn test_network_error_display() { + let error = WorkerError::NetworkError { + url: "http://worker3:8080".to_string(), + error: "Timeout after 30s".to_string(), + }; + assert_eq!( + error.to_string(), + "Network error for worker http://worker3:8080: Timeout after 30s" + ); + } + + #[test] + fn test_worker_at_capacity_display() { + let error = WorkerError::WorkerAtCapacity { + url: "http://worker4:8080".to_string(), + }; + assert_eq!(error.to_string(), "Worker at capacity: http://worker4:8080"); + } + + #[test] + fn test_worker_error_implements_std_error() { + let error = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + // Verify it implements Error trait + let _: &dyn Error = &error; + assert!(error.source().is_none()); + } + + #[test] + fn test_error_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + } + + #[test] + fn test_worker_result_type_alias() { + // Test Ok variant + let result: WorkerResult = Ok(42); + assert!(matches!(result, Ok(42))); + + // Test Err variant + let error = WorkerError::WorkerNotFound { + url: "test".to_string(), + }; + let result: WorkerResult = Err(error); + assert!(result.is_err()); + } + + #[test] + fn test_empty_url_handling() { + // Test empty URLs in error variants + let error1 = WorkerError::HealthCheckFailed { + url: "".to_string(), + reason: "No connection".to_string(), + }; + assert_eq!( + error1.to_string(), + "Health check failed for worker : No connection" + ); + + let error2 = WorkerError::NetworkError { + url: "".to_string(), + error: "DNS failure".to_string(), + }; + assert_eq!(error2.to_string(), "Network error for worker : DNS failure"); + + let error3 = WorkerError::WorkerNotFound { + url: "".to_string(), + }; + assert_eq!(error3.to_string(), "Worker not found: "); + } + + #[test] + fn test_special_characters_in_messages() { + // Test with special characters + let error = WorkerError::InvalidConfiguration { + message: "Invalid JSON: {\"error\": \"test\"}".to_string(), + }; + assert_eq!( + error.to_string(), + "Invalid worker configuration: Invalid JSON: {\"error\": \"test\"}" + ); + + // Test with unicode + let error2 = WorkerError::HealthCheckFailed { + url: "http://测试:8080".to_string(), + reason: "连接被拒绝".to_string(), + }; + assert_eq!( + error2.to_string(), + "Health check failed for worker http://测试:8080: 连接被拒绝" + ); + } + + #[test] + fn test_very_long_error_messages() { + let long_message = "A".repeat(10000); + let error = WorkerError::InvalidConfiguration { + message: long_message.clone(), + }; + let display = error.to_string(); + assert!(display.contains(&long_message)); + assert_eq!( + display.len(), + "Invalid worker configuration: ".len() + long_message.len() + ); + } + + // Mock reqwest error for testing conversion + #[test] + fn test_reqwest_error_conversion() { + // Test that NetworkError is the correct variant + let network_error = WorkerError::NetworkError { + url: "http://example.com".to_string(), + error: "connection timeout".to_string(), + }; + + match network_error { + WorkerError::NetworkError { url, error } => { + assert_eq!(url, "http://example.com"); + assert_eq!(error, "connection timeout"); + } + _ => panic!("Expected NetworkError variant"), + } + } + + #[test] + fn test_error_equality() { + // WorkerError doesn't implement PartialEq, but we can test that + // the same error construction produces the same display output + let error1 = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + let error2 = WorkerError::WorkerNotFound { + url: "http://test".to_string(), + }; + assert_eq!(error1.to_string(), error2.to_string()); + } +} diff --git a/sgl-router/src/core/mod.rs b/sgl-router/src/core/mod.rs new file mode 100644 index 000000000..b46810b4c --- /dev/null +++ b/sgl-router/src/core/mod.rs @@ -0,0 +1,24 @@ +//! Core abstractions for the SGLang router +//! +//! This module contains the fundamental types and traits used throughout the router: +//! - Worker trait and implementations +//! - Error types +//! - Circuit breaker for reliability +//! - Common utilities + +pub mod circuit_breaker; +pub mod error; +pub mod retry; +pub mod token_bucket; +pub mod worker; + +// Re-export commonly used types at the module level +pub use circuit_breaker::{ + CircuitBreaker, CircuitBreakerConfig, CircuitBreakerStats, CircuitState, +}; +pub use error::{WorkerError, WorkerResult}; +pub use retry::{is_retryable_status, BackoffCalculator, RetryError, RetryExecutor}; +pub use worker::{ + start_health_checker, BasicWorker, ConnectionMode, DPAwareWorker, HealthChecker, HealthConfig, + Worker, WorkerCollection, WorkerFactory, WorkerLoadGuard, WorkerType, +}; diff --git a/sgl-router/src/core/retry.rs b/sgl-router/src/core/retry.rs new file mode 100644 index 000000000..8a00424a5 --- /dev/null +++ b/sgl-router/src/core/retry.rs @@ -0,0 +1,409 @@ +use crate::config::types::RetryConfig; +use axum::http::StatusCode; +use axum::response::Response; +use rand::Rng; +use std::time::Duration; +use tracing::debug; + +/// Check if an HTTP status code indicates a retryable error +pub fn is_retryable_status(status: StatusCode) -> bool { + matches!( + status, + StatusCode::REQUEST_TIMEOUT + | StatusCode::TOO_MANY_REQUESTS + | StatusCode::INTERNAL_SERVER_ERROR + | StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT + ) +} + +/// Computes exponential backoff with optional jitter. +#[derive(Debug, Clone)] +pub struct BackoffCalculator; + +impl BackoffCalculator { + /// Calculate backoff delay for a given attempt index (0-based). + pub fn calculate_delay(config: &RetryConfig, attempt: u32) -> Duration { + // Base exponential backoff + let pow = config.backoff_multiplier.powi(attempt as i32); + let mut delay_ms = (config.initial_backoff_ms as f32 * pow) as u64; + if delay_ms > config.max_backoff_ms { + delay_ms = config.max_backoff_ms; + } + + // Apply jitter in range [-j, +j] + let jitter = config.jitter_factor.clamp(0.0, 1.0); + if jitter > 0.0 { + let mut rng = rand::rng(); + let jitter_scale: f32 = rng.random_range(-jitter..=jitter); + let jitter_ms = (delay_ms as f32 * jitter_scale) + .round() + .max(-(delay_ms as f32)); + let adjusted = (delay_ms as i64 + jitter_ms as i64).max(0) as u64; + return Duration::from_millis(adjusted); + } + + Duration::from_millis(delay_ms) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RetryError { + #[error("no available workers")] + NoAvailableWorkers, + #[error("maximum retry attempts exceeded")] + MaxRetriesExceeded, +} + +/// A thin async retry executor for generic operations. +#[derive(Debug, Clone, Default)] +pub struct RetryExecutor; + +impl RetryExecutor { + /// Execute an async operation with retries and backoff. + /// The `operation` closure is invoked each attempt with the attempt index. + pub async fn execute_with_retry( + config: &RetryConfig, + mut operation: F, + ) -> Result + where + F: FnMut(u32) -> Fut, + Fut: std::future::Future>, + { + let max = config.max_retries.max(1); + let mut attempt: u32 = 0; + loop { + match operation(attempt).await { + Ok(val) => return Ok(val), + Err(_) => { + // Use the number of failures so far (0-indexed) to compute delay, + // so the first retry uses `initial_backoff_ms`. + let is_last = attempt + 1 >= max; + if is_last { + return Err(RetryError::MaxRetriesExceeded); + } + let delay = BackoffCalculator::calculate_delay(config, attempt); + attempt += 1; // advance to the next attempt after computing delay + tokio::time::sleep(delay).await; + } + } + } + } + + /// Execute an operation that returns an HTTP Response with retries and backoff. + /// + /// Usage pattern: + /// - `operation(attempt)`: perform one attempt (0-based). Construct and send the request, + /// then return the `Response`. Do any per-attempt bookkeeping (e.g., load tracking, + /// circuit-breaker outcome recording) inside this closure. + /// - `should_retry(&response, attempt)`: decide if the given response should be retried + /// (e.g., based on HTTP status). Returning false short-circuits and returns the response. + /// - `on_backoff(delay, next_attempt)`: called before sleeping between attempts. + /// Use this to record metrics. + /// - `on_exhausted()`: called when the executor has exhausted all retry attempts. + /// + /// Example: + /// ```ignore + /// let resp = RetryExecutor::execute_response_with_retry( + /// &retry_cfg, + /// |attempt| async move { + /// let worker = select_cb_aware_worker()?; + /// let resp = send_request(worker).await; + /// worker.record_outcome(resp.status().is_success()); + /// resp + /// }, + /// |res, _| matches!(res.status(), StatusCode::REQUEST_TIMEOUT | StatusCode::TOO_MANY_REQUESTS | StatusCode::INTERNAL_SERVER_ERROR | StatusCode::BAD_GATEWAY | StatusCode::SERVICE_UNAVAILABLE | StatusCode::GATEWAY_TIMEOUT), + /// |delay, attempt| RouterMetrics::record_retry_backoff_duration(delay, attempt), + /// || RouterMetrics::record_retries_exhausted("/route"), + /// ).await; + /// ``` + pub async fn execute_response_with_retry( + config: &RetryConfig, + mut operation: Op, + should_retry: ShouldRetry, + on_backoff: OnBackoff, + mut on_exhausted: OnExhausted, + ) -> Response + where + Op: FnMut(u32) -> Fut, + Fut: std::future::Future, + ShouldRetry: Fn(&Response, u32) -> bool, + OnBackoff: Fn(Duration, u32), + OnExhausted: FnMut(), + { + let max = config.max_retries.max(1); + + let mut attempt: u32 = 0; + loop { + let response = operation(attempt).await; + let is_last = attempt + 1 >= max; + + if !should_retry(&response, attempt) { + return response; + } + + if is_last { + // Exhausted retries + on_exhausted(); + return response; + } + + // Backoff before next attempt + let next_attempt = attempt + 1; + // Compute delay based on the number of failures so far (0-indexed) + let delay = BackoffCalculator::calculate_delay(config, attempt); + debug!( + attempt = attempt, + next_attempt = next_attempt, + delay_ms = delay.as_millis() as u64, + "Retry backoff" + ); + on_backoff(delay, next_attempt); + tokio::time::sleep(delay).await; + + attempt = next_attempt; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::StatusCode; + use axum::response::IntoResponse; + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::Arc; + + fn base_retry_config() -> RetryConfig { + RetryConfig { + max_retries: 3, + initial_backoff_ms: 1, + max_backoff_ms: 4, + backoff_multiplier: 2.0, + jitter_factor: 0.0, + } + } + + #[test] + fn test_backoff_no_jitter_progression_and_cap() { + let cfg = RetryConfig { + max_retries: 10, + initial_backoff_ms: 100, + max_backoff_ms: 250, + backoff_multiplier: 2.0, + jitter_factor: 0.0, + }; + // attempt=0 => 100ms + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 0), + Duration::from_millis(100) + ); + // attempt=1 => 200ms + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 1), + Duration::from_millis(200) + ); + // attempt=2 => 400ms -> capped to 250ms + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 2), + Duration::from_millis(250) + ); + // large attempt still capped + assert_eq!( + BackoffCalculator::calculate_delay(&cfg, 10), + Duration::from_millis(250) + ); + } + + #[test] + fn test_backoff_with_jitter_within_bounds() { + let cfg = RetryConfig { + max_retries: 5, + initial_backoff_ms: 100, + max_backoff_ms: 10_000, + backoff_multiplier: 2.0, + jitter_factor: 0.5, + }; + // attempt=2 => base 400ms, jitter in [0.5x, 1.5x] + let base = 400.0; + for _ in 0..50 { + let d = BackoffCalculator::calculate_delay(&cfg, 2).as_millis() as f32; + assert!(d >= base * 0.5 - 1.0 && d <= base * 1.5 + 1.0); + } + } + + #[tokio::test] + async fn test_execute_with_retry_success_after_failures() { + let cfg = base_retry_config(); + let remaining = Arc::new(AtomicU32::new(2)); + let calls = Arc::new(AtomicU32::new(0)); + + let res: Result = RetryExecutor::execute_with_retry(&cfg, { + let remaining = remaining.clone(); + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + let remaining = remaining.clone(); + async move { + if remaining + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_sub(1)) + .is_ok() + { + Err(()) + } else { + Ok(42u32) + } + } + } + }) + .await; + + assert!(res.is_ok()); + assert_eq!(res.unwrap(), 42); + assert_eq!(calls.load(Ordering::Relaxed), 3); // 2 fails + 1 success + } + + #[tokio::test] + async fn test_execute_with_retry_exhausted() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let res: Result = RetryExecutor::execute_with_retry(&cfg, { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { Err(()) } + } + }) + .await; + + assert!(matches!(res, Err(RetryError::MaxRetriesExceeded))); + assert_eq!(calls.load(Ordering::Relaxed), cfg.max_retries); + } + + #[tokio::test] + async fn test_execute_response_with_retry_success_path_and_hooks() { + let cfg = base_retry_config(); + let remaining = Arc::new(AtomicU32::new(2)); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let remaining = remaining.clone(); + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + let remaining = remaining.clone(); + async move { + if remaining + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| v.checked_sub(1)) + .is_ok() + { + (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response() + } else { + (StatusCode::OK, "ok").into_response() + } + } + } + }, + |res, _attempt| !res.status().is_success(), // retry until success + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(calls.load(Ordering::Relaxed), 3); // 2 fails + 1 success + assert_eq!(backoffs.load(Ordering::Relaxed), 2); + assert_eq!(exhausted.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn test_execute_response_with_retry_non_retryable_short_circuit() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { (StatusCode::BAD_REQUEST, "bad").into_response() } + } + }, + |_res, _attempt| false, // never retry + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(calls.load(Ordering::Relaxed), 1); + assert_eq!(backoffs.load(Ordering::Relaxed), 0); + assert_eq!(exhausted.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn test_execute_response_with_retry_exhausted_hooks() { + let cfg = base_retry_config(); + let calls = Arc::new(AtomicU32::new(0)); + let backoffs = Arc::new(AtomicU32::new(0)); + let exhausted = Arc::new(AtomicU32::new(0)); + + let response = RetryExecutor::execute_response_with_retry( + &cfg, + { + let calls = calls.clone(); + move |_attempt| { + calls.fetch_add(1, Ordering::Relaxed); + async move { (StatusCode::SERVICE_UNAVAILABLE, "fail").into_response() } + } + }, + |_res, _attempt| true, // keep retrying + { + let backoffs = backoffs.clone(); + move |_delay, _next_attempt| { + backoffs.fetch_add(1, Ordering::Relaxed); + } + }, + { + let exhausted = exhausted.clone(); + move || { + exhausted.fetch_add(1, Ordering::Relaxed); + } + }, + ) + .await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(calls.load(Ordering::Relaxed), cfg.max_retries); + assert_eq!(backoffs.load(Ordering::Relaxed), cfg.max_retries - 1); + assert_eq!(exhausted.load(Ordering::Relaxed), 1); + } +} diff --git a/sgl-router/src/core/token_bucket.rs b/sgl-router/src/core/token_bucket.rs new file mode 100644 index 000000000..65117331a --- /dev/null +++ b/sgl-router/src/core/token_bucket.rs @@ -0,0 +1,195 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{Mutex, Notify}; +use tracing::{debug, trace}; + +/// Token bucket for rate limiting +/// +/// This implementation provides: +/// - Smooth rate limiting with configurable refill rate +/// - Burst capacity handling +/// - Fair queuing for waiting requests +#[derive(Clone)] +pub struct TokenBucket { + inner: Arc>, + notify: Arc, + capacity: f64, + refill_rate: f64, // tokens per second +} + +struct TokenBucketInner { + tokens: f64, + last_refill: Instant, +} + +impl TokenBucket { + /// Create a new token bucket + /// + /// # Arguments + /// * `capacity` - Maximum number of tokens (burst capacity) + /// * `refill_rate` - Tokens added per second + pub fn new(capacity: usize, refill_rate: usize) -> Self { + let capacity = capacity as f64; + let refill_rate = refill_rate as f64; + + // Ensure refill_rate is not zero to prevent division by zero + let refill_rate = if refill_rate > 0.0 { + refill_rate + } else { + 1.0 // Default to 1 token per second if zero + }; + + Self { + inner: Arc::new(Mutex::new(TokenBucketInner { + tokens: capacity, // Start full + last_refill: Instant::now(), + })), + notify: Arc::new(Notify::new()), + capacity, + refill_rate, + } + } + + /// Try to acquire tokens immediately + pub async fn try_acquire(&self, tokens: f64) -> Result<(), ()> { + let mut inner = self.inner.lock().await; + + // Refill tokens based on elapsed time + let now = Instant::now(); + let elapsed = now.duration_since(inner.last_refill).as_secs_f64(); + let refill_amount = elapsed * self.refill_rate; + + inner.tokens = (inner.tokens + refill_amount).min(self.capacity); + inner.last_refill = now; + + trace!( + "Token bucket: {} tokens available, requesting {}", + inner.tokens, + tokens + ); + + if inner.tokens >= tokens { + inner.tokens -= tokens; + debug!( + "Token bucket: acquired {} tokens, {} remaining", + tokens, inner.tokens + ); + Ok(()) + } else { + Err(()) + } + } + + /// Acquire tokens, waiting if necessary + pub async fn acquire(&self, tokens: f64) -> Result<(), tokio::time::error::Elapsed> { + // First try to acquire immediately + if self.try_acquire(tokens).await.is_ok() { + return Ok(()); + } + + // Calculate wait time + let wait_time = { + let inner = self.inner.lock().await; + let tokens_needed = tokens - inner.tokens; + let wait_secs = tokens_needed / self.refill_rate; + Duration::from_secs_f64(wait_secs) + }; + + debug!( + "Token bucket: waiting {:?} for {} tokens", + wait_time, tokens + ); + + // Wait for tokens to be available + tokio::time::timeout(wait_time, async { + loop { + // Check if we can acquire now + if self.try_acquire(tokens).await.is_ok() { + return; + } + + // Wait for notification or small interval + tokio::select! { + _ = self.notify.notified() => {}, + _ = tokio::time::sleep(Duration::from_millis(10)) => {}, + } + } + }) + .await?; + + Ok(()) + } + + /// Acquire tokens with custom timeout + pub async fn acquire_timeout( + &self, + tokens: f64, + timeout: Duration, + ) -> Result<(), tokio::time::error::Elapsed> { + tokio::time::timeout(timeout, self.acquire(tokens)).await? + } + + /// Return tokens to the bucket (for cancelled requests) + pub async fn return_tokens(&self, tokens: f64) { + let mut inner = self.inner.lock().await; + inner.tokens = (inner.tokens + tokens).min(self.capacity); + self.notify.notify_waiters(); + debug!( + "Token bucket: returned {} tokens, {} available", + tokens, inner.tokens + ); + } + + /// Get current available tokens (for monitoring) + pub async fn available_tokens(&self) -> f64 { + let mut inner = self.inner.lock().await; + + // Refill before checking + let now = Instant::now(); + let elapsed = now.duration_since(inner.last_refill).as_secs_f64(); + let refill_amount = elapsed * self.refill_rate; + + inner.tokens = (inner.tokens + refill_amount).min(self.capacity); + inner.last_refill = now; + + inner.tokens + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_token_bucket_basic() { + let bucket = TokenBucket::new(10, 5); // 10 capacity, 5 per second + + // Should succeed - bucket starts full + assert!(bucket.try_acquire(5.0).await.is_ok()); + assert!(bucket.try_acquire(5.0).await.is_ok()); + + // Should fail - no tokens left + assert!(bucket.try_acquire(1.0).await.is_err()); + + // Wait for refill + tokio::time::sleep(Duration::from_millis(300)).await; + + // Should have ~1.5 tokens now + assert!(bucket.try_acquire(1.0).await.is_ok()); + } + + #[tokio::test] + async fn test_token_bucket_refill() { + let bucket = TokenBucket::new(10, 10); // 10 capacity, 10 per second + + // Use all tokens + assert!(bucket.try_acquire(10.0).await.is_ok()); + + // Wait for partial refill + tokio::time::sleep(Duration::from_millis(500)).await; + + // Should have ~5 tokens + let available = bucket.available_tokens().await; + assert!((4.0..=6.0).contains(&available)); + } +} diff --git a/sgl-router/src/core/worker.rs b/sgl-router/src/core/worker.rs new file mode 100644 index 000000000..f25fc6eea --- /dev/null +++ b/sgl-router/src/core/worker.rs @@ -0,0 +1,1980 @@ +use super::{CircuitBreaker, CircuitBreakerConfig, WorkerError, WorkerResult}; +use crate::grpc::SglangSchedulerClient; +use crate::metrics::RouterMetrics; +use async_trait::async_trait; +use futures; +use serde_json; +use std::fmt; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, LazyLock}; +use tokio::sync::Mutex; + +// Shared HTTP client for worker operations (health checks, server info, etc.) +static WORKER_CLIENT: LazyLock = LazyLock::new(|| { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) // Default timeout, overridden per request + .build() + .expect("Failed to create worker HTTP client") +}); + +/// Core worker abstraction that represents a backend service +#[async_trait] +pub trait Worker: Send + Sync + fmt::Debug { + /// Get the worker's URL + fn url(&self) -> &str; + + /// Get the worker's type (Regular, Prefill, or Decode) + fn worker_type(&self) -> WorkerType; + + /// Get the worker's connection mode (HTTP or gRPC) + fn connection_mode(&self) -> ConnectionMode; + + /// Check if the worker is currently healthy + fn is_healthy(&self) -> bool; + + /// Set the worker's health status + fn set_healthy(&self, healthy: bool); + + /// Perform an async health check on the worker + async fn check_health_async(&self) -> WorkerResult<()>; + + /// Synchronous health check wrapper (for compatibility) + fn check_health(&self) -> WorkerResult<()> { + // Use a small runtime for synchronous contexts + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| WorkerError::HealthCheckFailed { + url: self.url().to_string(), + reason: format!("Failed to create runtime: {}", e), + })? + .block_on(self.check_health_async()) + } + + /// Get the current load (number of active requests) + fn load(&self) -> usize; + + /// Increment the load counter + fn increment_load(&self); + + /// Decrement the load counter + fn decrement_load(&self); + + /// Reset the load counter to 0 (for sync/recovery) + fn reset_load(&self) { + // Default implementation - does nothing + // Workers that track load should override this + } + + /// Get the number of processed requests + fn processed_requests(&self) -> usize; + + /// Increment the processed requests counter + fn increment_processed(&self); + + /// Get worker-specific metadata + fn metadata(&self) -> &WorkerMetadata; + + /// Clone the worker (for trait objects) + fn clone_worker(&self) -> Box; + + /// Get the circuit breaker for this worker + fn circuit_breaker(&self) -> &CircuitBreaker; + + /// Check if the worker is available (healthy + circuit closed/half-open) + fn is_available(&self) -> bool { + self.is_healthy() && self.circuit_breaker().can_execute() + } + + /// Record the outcome of a request to this worker + fn record_outcome(&self, success: bool) { + // Record outcome-level metric with worker label + let outcome_str = if success { "success" } else { "failure" }; + RouterMetrics::record_cb_outcome(self.url(), outcome_str); + + // Record into circuit breaker and infer state change for metrics + let before = self.circuit_breaker().state(); + self.circuit_breaker().record_outcome(success); + let after = self.circuit_breaker().state(); + + if before != after { + let from = match before { + crate::core::CircuitState::Closed => "closed", + crate::core::CircuitState::Open => "open", + crate::core::CircuitState::HalfOpen => "half_open", + }; + let to = match after { + crate::core::CircuitState::Closed => "closed", + crate::core::CircuitState::Open => "open", + crate::core::CircuitState::HalfOpen => "half_open", + }; + RouterMetrics::record_cb_state_transition(self.url(), from, to); + } + + let state_code = match self.circuit_breaker().state() { + crate::core::CircuitState::Closed => 0u8, + crate::core::CircuitState::Open => 1u8, + crate::core::CircuitState::HalfOpen => 2u8, + }; + RouterMetrics::set_cb_state(self.url(), state_code); + } + + // === DP-aware methods === + + /// Check if this worker is DP-aware + fn is_dp_aware(&self) -> bool { + false + } + + /// Get the base URL without any DP rank suffix + fn base_url(&self) -> &str { + self.url() + } + + /// Get DP rank if this is a DP-aware worker + fn dp_rank(&self) -> Option { + None + } + + /// Get DP size if this worker is part of a DP group + fn dp_size(&self) -> Option { + None + } + + /// Transform a request for DP-aware routing + async fn prepare_request(&self, req: serde_json::Value) -> WorkerResult { + Ok(req) + } + + /// Get the actual endpoint URL for requests + fn endpoint_url(&self, route: &str) -> String { + format!("{}{}", self.base_url(), route) + } + + /// Check if this worker can handle a specific request + fn can_handle(&self, _req: &serde_json::Value) -> bool { + true + } +} + +/// Connection mode for worker communication +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ConnectionMode { + /// HTTP/REST connection + Http, + /// gRPC connection + Grpc { + /// Optional port for gRPC endpoint (if different from URL) + port: Option, + }, +} + +impl fmt::Display for ConnectionMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ConnectionMode::Http => write!(f, "HTTP"), + ConnectionMode::Grpc { port } => match port { + Some(p) => write!(f, "gRPC(port:{})", p), + None => write!(f, "gRPC"), + }, + } + } +} + +/// Worker type classification +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum WorkerType { + /// Regular worker for standard routing + Regular, + /// Prefill worker for PD disaggregated mode + Prefill { + /// Bootstrap port for communication with decode workers + bootstrap_port: Option, + }, + /// Decode worker for PD disaggregated mode + Decode, +} + +impl fmt::Display for WorkerType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + WorkerType::Regular => write!(f, "Regular"), + WorkerType::Prefill { bootstrap_port } => match bootstrap_port { + Some(port) => write!(f, "Prefill(bootstrap:{})", port), + None => write!(f, "Prefill"), + }, + WorkerType::Decode => write!(f, "Decode"), + } + } +} + +/// Health check configuration +#[derive(Debug, Clone)] +pub struct HealthConfig { + /// Timeout for health checks in seconds + pub timeout_secs: u64, + /// Interval between health checks in seconds + pub check_interval_secs: u64, + /// Health check endpoint path + pub endpoint: String, + /// Number of consecutive failures before marking unhealthy + pub failure_threshold: u32, + /// Number of consecutive successes before marking healthy + pub success_threshold: u32, +} + +impl Default for HealthConfig { + fn default() -> Self { + Self { + timeout_secs: 5, + check_interval_secs: 30, + endpoint: "/health".to_string(), + failure_threshold: 3, + success_threshold: 2, + } + } +} + +/// Metadata associated with a worker +#[derive(Debug, Clone)] +pub struct WorkerMetadata { + /// Worker URL + pub url: String, + /// Worker type + pub worker_type: WorkerType, + /// Connection mode + pub connection_mode: ConnectionMode, + /// Additional labels/tags + pub labels: std::collections::HashMap, + /// Health check configuration + pub health_config: HealthConfig, +} + +/// Basic worker implementation +#[derive(Clone)] +pub struct BasicWorker { + metadata: WorkerMetadata, + load_counter: Arc, + processed_counter: Arc, + healthy: Arc, + consecutive_failures: Arc, + consecutive_successes: Arc, + circuit_breaker: CircuitBreaker, + /// Optional gRPC client for gRPC workers + grpc_client: Option>>, +} + +impl fmt::Debug for BasicWorker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BasicWorker") + .field("metadata", &self.metadata) + .field("healthy", &self.healthy.load(Ordering::Relaxed)) + .field("circuit_breaker", &self.circuit_breaker) + .field("has_grpc_client", &self.grpc_client.is_some()) + .finish() + } +} + +impl BasicWorker { + pub fn new(url: String, worker_type: WorkerType) -> Self { + Self::with_connection_mode(url, worker_type, ConnectionMode::Http) + } + + pub fn with_connection_mode( + url: String, + worker_type: WorkerType, + connection_mode: ConnectionMode, + ) -> Self { + let metadata = WorkerMetadata { + url: url.clone(), + worker_type, + connection_mode, + labels: std::collections::HashMap::new(), + health_config: HealthConfig::default(), + }; + + Self { + metadata, + load_counter: Arc::new(AtomicUsize::new(0)), + processed_counter: Arc::new(AtomicUsize::new(0)), + healthy: Arc::new(AtomicBool::new(true)), + consecutive_failures: Arc::new(AtomicUsize::new(0)), + consecutive_successes: Arc::new(AtomicUsize::new(0)), + circuit_breaker: CircuitBreaker::new(), + grpc_client: None, + } + } + + pub fn with_labels(mut self, labels: std::collections::HashMap) -> Self { + self.metadata.labels = labels; + self + } + + pub fn with_health_config(mut self, config: HealthConfig) -> Self { + self.metadata.health_config = config; + self + } + + pub fn with_circuit_breaker_config(mut self, config: CircuitBreakerConfig) -> Self { + self.circuit_breaker = CircuitBreaker::with_config(config); + self + } + + /// Set the gRPC client for gRPC workers + pub fn with_grpc_client(mut self, client: SglangSchedulerClient) -> Self { + self.grpc_client = Some(Arc::new(Mutex::new(client))); + self + } + + pub fn normalised_url(&self) -> WorkerResult<&str> { + if self.url().contains("@") { + // Need to extract the URL from "http://host:port@dp_rank" + let parts: Vec<&str> = self.url().split('@').collect(); + if parts.len() != 2 { + return Err(WorkerError::InvalidUrl { + url: self.url().to_string(), + }); + } + // Ensure the second part (the dp_rank) can be parsed as an integer + match parts[1].parse::() { + Ok(_) => Ok(parts[0]), + Err(_) => Err(WorkerError::InvalidUrl { + url: self.url().to_string(), + }), + } + } else { + Ok(self.url()) + } + } +} + +#[async_trait] +impl Worker for BasicWorker { + fn url(&self) -> &str { + &self.metadata.url + } + + fn worker_type(&self) -> WorkerType { + self.metadata.worker_type.clone() + } + + fn connection_mode(&self) -> ConnectionMode { + self.metadata.connection_mode.clone() + } + + fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::Acquire) + } + + fn set_healthy(&self, healthy: bool) { + self.healthy.store(healthy, Ordering::Release); + RouterMetrics::set_worker_health(self.url(), healthy); + } + + async fn check_health_async(&self) -> WorkerResult<()> { + use std::time::Duration; + + let health_result = match &self.metadata.connection_mode { + ConnectionMode::Http => { + // Perform HTTP health check + let url = self.normalised_url()?; + let health_url = format!("{}{}", url, self.metadata.health_config.endpoint); + let timeout = Duration::from_secs(self.metadata.health_config.timeout_secs); + + // Use the shared client with a custom timeout for this request + match WORKER_CLIENT.get(&health_url).timeout(timeout).send().await { + Ok(response) => response.status().is_success(), + Err(_) => false, + } + } + ConnectionMode::Grpc { .. } => { + // Perform gRPC health check + if let Some(grpc_client) = &self.grpc_client { + let mut client = grpc_client.lock().await; + match client.health_check().await { + Ok(response) => { + tracing::debug!( + "gRPC health check succeeded for {}: healthy={}", + self.metadata.url, + response.healthy + ); + response.healthy + } + Err(e) => { + tracing::warn!( + "gRPC health check RPC failed for {}: {:?}", + self.metadata.url, + e + ); + false + } + } + } else { + tracing::error!("No gRPC client available for worker {}", self.metadata.url); + false + } + } + }; + + if health_result { + // Health check succeeded + self.consecutive_failures.store(0, Ordering::Release); + let successes = self.consecutive_successes.fetch_add(1, Ordering::AcqRel) + 1; + + // Mark healthy if we've reached the success threshold + if !self.is_healthy() + && successes >= self.metadata.health_config.success_threshold as usize + { + self.set_healthy(true); + self.consecutive_successes.store(0, Ordering::Release); + } + Ok(()) + } else { + // Health check failed + self.consecutive_successes.store(0, Ordering::Release); + let failures = self.consecutive_failures.fetch_add(1, Ordering::AcqRel) + 1; + + // Mark unhealthy if we've reached the failure threshold + if self.is_healthy() + && failures >= self.metadata.health_config.failure_threshold as usize + { + self.set_healthy(false); + self.consecutive_failures.store(0, Ordering::Release); + } + + Err(WorkerError::HealthCheckFailed { + url: self.metadata.url.clone(), + reason: format!("Health check failed (consecutive failures: {})", failures), + }) + } + } + + fn load(&self) -> usize { + self.load_counter.load(Ordering::Relaxed) + } + + fn increment_load(&self) { + self.load_counter.fetch_add(1, Ordering::Relaxed); + } + + fn decrement_load(&self) { + self.load_counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| { + current.checked_sub(1) + }) + .ok(); + } + + fn reset_load(&self) { + self.load_counter.store(0, Ordering::Relaxed); + } + + fn processed_requests(&self) -> usize { + self.processed_counter.load(Ordering::Relaxed) + } + + fn increment_processed(&self) { + self.processed_counter.fetch_add(1, Ordering::Relaxed); + } + + fn metadata(&self) -> &WorkerMetadata { + &self.metadata + } + + fn clone_worker(&self) -> Box { + Box::new(self.clone()) + } + + fn circuit_breaker(&self) -> &CircuitBreaker { + &self.circuit_breaker + } +} + +/// A DP-aware worker that handles data-parallel routing +#[derive(Debug, Clone)] +pub struct DPAwareWorker { + /// The underlying basic worker + base_worker: BasicWorker, + /// DP rank for this worker + dp_rank: usize, + /// Total DP size + dp_size: usize, + /// Base URL without DP suffix + base_url: String, +} + +impl DPAwareWorker { + /// Create a new DP-aware worker of any type + pub fn new(base_url: String, dp_rank: usize, dp_size: usize, worker_type: WorkerType) -> Self { + // Create URL with DP rank suffix for identification + let worker_url = format!("{}@{}", base_url, dp_rank); + let base_worker = BasicWorker::new(worker_url, worker_type); + + Self { + base_worker, + dp_rank, + dp_size, + base_url, + } + } +} + +#[async_trait] +impl Worker for DPAwareWorker { + fn url(&self) -> &str { + self.base_worker.url() + } + + fn worker_type(&self) -> WorkerType { + self.base_worker.worker_type() + } + + fn connection_mode(&self) -> ConnectionMode { + self.base_worker.connection_mode() + } + + fn is_healthy(&self) -> bool { + self.base_worker.is_healthy() + } + + fn set_healthy(&self, healthy: bool) { + self.base_worker.set_healthy(healthy); + } + + async fn check_health_async(&self) -> WorkerResult<()> { + // Delegate to the base worker's health check logic + self.base_worker.check_health_async().await + } + + fn load(&self) -> usize { + self.base_worker.load() + } + + fn increment_load(&self) { + self.base_worker.increment_load(); + } + + fn decrement_load(&self) { + self.base_worker.decrement_load(); + } + + fn reset_load(&self) { + self.base_worker.reset_load(); + } + + fn processed_requests(&self) -> usize { + self.base_worker.processed_requests() + } + + fn increment_processed(&self) { + self.base_worker.increment_processed(); + } + + fn metadata(&self) -> &WorkerMetadata { + self.base_worker.metadata() + } + + fn clone_worker(&self) -> Box { + Box::new(self.clone()) + } + + fn circuit_breaker(&self) -> &CircuitBreaker { + self.base_worker.circuit_breaker() + } + + // DP-aware specific implementations + + fn is_dp_aware(&self) -> bool { + true + } + + fn base_url(&self) -> &str { + &self.base_url + } + + fn dp_rank(&self) -> Option { + Some(self.dp_rank) + } + + fn dp_size(&self) -> Option { + Some(self.dp_size) + } + + async fn prepare_request(&self, mut req: serde_json::Value) -> WorkerResult { + // Inject data_parallel_rank into the request + if let Some(map) = req.as_object_mut() { + map.insert( + "data_parallel_rank".to_string(), + serde_json::json!(self.dp_rank), + ); + Ok(req) + } else { + Err(WorkerError::InvalidConfiguration { + message: "Request must be a JSON object for DP-aware routing".to_string(), + }) + } + } + + fn endpoint_url(&self, route: &str) -> String { + // Use base URL for actual requests + format!("{}{}", self.base_url, route) + } +} + +/// Worker factory for creating workers of different types +pub struct WorkerFactory; + +impl WorkerFactory { + /// Create a regular worker + pub fn create_regular(url: String) -> Box { + Box::new(BasicWorker::new(url, WorkerType::Regular)) + } + + /// Create a regular worker with custom circuit breaker configuration + pub fn create_regular_with_config( + url: String, + circuit_breaker_config: CircuitBreakerConfig, + ) -> Box { + Box::new( + BasicWorker::new(url, WorkerType::Regular) + .with_circuit_breaker_config(circuit_breaker_config), + ) + } + + /// Create a prefill worker with optional bootstrap port + pub fn create_prefill(url: String, bootstrap_port: Option) -> Box { + Box::new(BasicWorker::new( + url, + WorkerType::Prefill { bootstrap_port }, + )) + } + + /// Create a prefill worker with custom circuit breaker configuration + pub fn create_prefill_with_config( + url: String, + bootstrap_port: Option, + circuit_breaker_config: CircuitBreakerConfig, + ) -> Box { + Box::new( + BasicWorker::new(url, WorkerType::Prefill { bootstrap_port }) + .with_circuit_breaker_config(circuit_breaker_config), + ) + } + + /// Create a decode worker + pub fn create_decode(url: String) -> Box { + Box::new(BasicWorker::new(url, WorkerType::Decode)) + } + + /// Create a decode worker with custom circuit breaker configuration + pub fn create_decode_with_config( + url: String, + circuit_breaker_config: CircuitBreakerConfig, + ) -> Box { + Box::new( + BasicWorker::new(url, WorkerType::Decode) + .with_circuit_breaker_config(circuit_breaker_config), + ) + } + + /// Create workers from URLs with automatic type detection + #[allow(clippy::type_complexity)] + pub fn create_from_urls( + regular_urls: Vec, + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + ) -> ( + Vec>, + Vec>, + Vec>, + ) { + let regular_workers: Vec> = + regular_urls.into_iter().map(Self::create_regular).collect(); + + let prefill_workers: Vec> = prefill_urls + .into_iter() + .map(|(url, port)| Self::create_prefill(url, port)) + .collect(); + + let decode_workers: Vec> = + decode_urls.into_iter().map(Self::create_decode).collect(); + + (regular_workers, prefill_workers, decode_workers) + } + + /// Create a gRPC worker + pub fn create_grpc(url: String, worker_type: WorkerType, port: Option) -> Box { + Box::new(BasicWorker::with_connection_mode( + url, + worker_type, + ConnectionMode::Grpc { port }, + )) + } + + /// Create a gRPC worker with custom circuit breaker configuration + pub fn create_grpc_with_config( + url: String, + worker_type: WorkerType, + port: Option, + circuit_breaker_config: CircuitBreakerConfig, + ) -> Box { + Box::new( + BasicWorker::with_connection_mode(url, worker_type, ConnectionMode::Grpc { port }) + .with_circuit_breaker_config(circuit_breaker_config), + ) + } + + /// Create a DP-aware worker of specified type + pub fn create_dp_aware( + base_url: String, + dp_rank: usize, + dp_size: usize, + worker_type: WorkerType, + ) -> Box { + Box::new(DPAwareWorker::new(base_url, dp_rank, dp_size, worker_type)) + } + + /// Get DP size from a worker + async fn get_worker_dp_size(url: &str, api_key: &Option) -> WorkerResult { + let mut req_builder = WORKER_CLIENT.get(format!("{}/get_server_info", url)); + + if let Some(key) = api_key { + req_builder = req_builder.bearer_auth(key); + } + + let response = req_builder + .send() + .await + .map_err(|e| WorkerError::NetworkError { + url: url.to_string(), + error: e.to_string(), + })?; + + if !response.status().is_success() { + return Err(WorkerError::NetworkError { + url: url.to_string(), + error: format!("Server returned: {}", response.status()), + }); + } + + let info: serde_json::Value = + response + .json() + .await + .map_err(|e| WorkerError::NetworkError { + url: url.to_string(), + error: format!("Failed to parse JSON: {}", e), + })?; + + let dp_size = info + .get("dp_size") + .and_then(|v| v.as_u64()) + .ok_or_else(|| WorkerError::InvalidConfiguration { + message: "dp_size not found in server info".to_string(), + })?; + + if dp_size > usize::MAX as u64 { + return Err(WorkerError::InvalidConfiguration { + message: format!("dp_size is too large: {}", dp_size), + }); + } + + Ok(dp_size as usize) + } + + /// Private helper to create DP-aware workers of any type + async fn create_dp_aware_workers_of_type( + url: &str, + api_key: &Option, + worker_type: WorkerType, + ) -> WorkerResult>> { + let dp_size = Self::get_worker_dp_size(url, api_key).await?; + + let workers = (0..dp_size) + .map(|rank| Self::create_dp_aware(url.to_string(), rank, dp_size, worker_type.clone())) + .collect(); + + Ok(workers) + } + + /// Create DP-aware regular workers from a single URL + pub async fn create_dp_aware_regular_workers( + url: &str, + api_key: &Option, + ) -> WorkerResult>> { + Self::create_dp_aware_workers_of_type(url, api_key, WorkerType::Regular).await + } + + /// Create DP-aware prefill workers from a single URL + pub async fn create_dp_aware_prefill_workers( + url: &str, + bootstrap_port: Option, + api_key: &Option, + ) -> WorkerResult>> { + Self::create_dp_aware_workers_of_type(url, api_key, WorkerType::Prefill { bootstrap_port }) + .await + } + + /// Create DP-aware decode workers from a single URL + pub async fn create_dp_aware_decode_workers( + url: &str, + api_key: &Option, + ) -> WorkerResult>> { + Self::create_dp_aware_workers_of_type(url, api_key, WorkerType::Decode).await + } + + /// Create workers based on configuration (for regular router) + pub async fn create_workers( + urls: Vec, + dp_aware: bool, + api_key: &Option, + ) -> WorkerResult>> { + if dp_aware { + // Create futures for all worker creations + let worker_futs = urls + .iter() + .map(|url| Self::create_dp_aware_regular_workers(url, api_key)); + + // Execute all futures concurrently and flatten results + let all_workers = futures::future::try_join_all(worker_futs) + .await? + .into_iter() + .flatten() + .collect(); + + Ok(all_workers) + } else { + Ok(urls + .into_iter() + .map(|url| Self::create_regular(url)) + .collect()) + } + } +} + +/// Helper trait for collections of workers +pub trait WorkerCollection { + fn healthy_workers(&self) -> Vec<&dyn Worker>; + fn total_load(&self) -> usize; + fn find_worker(&self, url: &str) -> Option<&dyn Worker>; + fn find_worker_mut(&mut self, url: &str) -> Option<&mut Box>; +} + +impl WorkerCollection for Vec> { + fn healthy_workers(&self) -> Vec<&dyn Worker> { + self.iter() + .filter(|w| w.is_healthy()) + .map(|w| w.as_ref()) + .collect() + } + + fn total_load(&self) -> usize { + self.iter().map(|w| w.load()).sum() + } + + fn find_worker(&self, url: &str) -> Option<&dyn Worker> { + self.iter().find(|w| w.url() == url).map(|w| w.as_ref()) + } + + fn find_worker_mut(&mut self, url: &str) -> Option<&mut Box> { + self.iter_mut().find(|w| w.url() == url) + } +} + +/// Convert a list of worker URLs to worker trait objects +pub fn urls_to_workers(urls: Vec) -> Vec> { + urls.into_iter() + .map(WorkerFactory::create_regular) + .collect() +} + +/// Convert worker trait objects back to URLs +pub fn workers_to_urls(workers: &[Box]) -> Vec { + workers.iter().map(|w| w.url().to_string()).collect() +} + +/// RAII guard for worker load management +pub struct WorkerLoadGuard<'a> { + workers: Vec<&'a dyn Worker>, +} + +impl<'a> WorkerLoadGuard<'a> { + /// Create a new load guard for a single worker + pub fn new(worker: &'a dyn Worker) -> Self { + worker.increment_load(); + Self { + workers: vec![worker], + } + } + + /// Create a new load guard for multiple workers + pub fn new_multi(workers: Vec<&'a dyn Worker>) -> Self { + // Increment load counters for all workers + for worker in &workers { + worker.increment_load(); + } + Self { workers } + } +} + +impl<'a> Drop for WorkerLoadGuard<'a> { + fn drop(&mut self) { + // Decrement load counters for all workers + for worker in &self.workers { + worker.decrement_load(); + } + } +} + +/// Health checker handle with graceful shutdown +pub struct HealthChecker { + handle: tokio::task::JoinHandle<()>, + shutdown: Arc, +} + +impl fmt::Debug for HealthChecker { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HealthChecker") + .field("shutdown", &self.shutdown.load(Ordering::Relaxed)) + .finish() + } +} + +impl HealthChecker { + /// Shutdown the health checker gracefully + pub async fn shutdown(self) { + self.shutdown.store(true, Ordering::Release); + let _ = self.handle.await; + } +} + +/// Start an async background health checker for a collection of workers +pub fn start_health_checker( + workers: std::sync::Arc>>>, + check_interval_secs: u64, +) -> HealthChecker { + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown_clone = shutdown.clone(); + + let handle = tokio::spawn(async move { + let mut interval = + tokio::time::interval(tokio::time::Duration::from_secs(check_interval_secs)); + + // Counter for periodic load reset (every 10 health check cycles) + let mut check_count = 0u64; + const LOAD_RESET_INTERVAL: u64 = 10; + + loop { + interval.tick().await; + + // Check for shutdown signal + if shutdown_clone.load(Ordering::Acquire) { + tracing::debug!("Health checker shutting down"); + break; + } + + check_count += 1; + + // Check health of all workers + let workers_to_check = match workers.read() { + Ok(guard) => guard.iter().map(|w| w.clone_worker()).collect::>(), + Err(poisoned) => { + tracing::error!("Worker lock poisoned: {}", poisoned); + continue; + } + }; + + // Periodically reset load counters to prevent drift + // Only do this when we believe all workers should be idle + if check_count.is_multiple_of(LOAD_RESET_INTERVAL) { + let max_load = workers_to_check.iter().map(|w| w.load()).max().unwrap_or(0); + // Only reset if load appears to be very low (likely drift) + if max_load <= 2 { + tracing::debug!( + "Resetting load counters to prevent drift (max_load: {})", + max_load + ); + for worker in &workers_to_check { + worker.reset_load(); + } + } + } + + // Perform health checks concurrently + let health_checks = workers_to_check.iter().map(|worker| { + let worker_url = worker.url().to_string(); + let was_healthy = worker.is_healthy(); + + async move { + match worker.check_health_async().await { + Ok(_) => { + if !was_healthy { + tracing::info!("Worker {} is now healthy", worker_url); + } + } + Err(e) => { + if was_healthy { + tracing::warn!("Worker {} health check failed: {}", worker_url, e); + } else { + // Worker was already unhealthy, log at debug level + tracing::debug!("Worker {} remains unhealthy: {}", worker_url, e); + } + } + } + } + }); + + // Execute all health checks concurrently + futures::future::join_all(health_checks).await; + } + }); + + HealthChecker { handle, shutdown } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::RwLock; + use std::thread; + use std::time::Duration; + use tokio::time::timeout; + + // Test WorkerType + #[test] + fn test_worker_type_display() { + assert_eq!(WorkerType::Regular.to_string(), "Regular"); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + } + .to_string(), + "Prefill(bootstrap:8080)" + ); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: None + } + .to_string(), + "Prefill" + ); + assert_eq!(WorkerType::Decode.to_string(), "Decode"); + } + + #[test] + fn test_worker_type_equality() { + assert_eq!(WorkerType::Regular, WorkerType::Regular); + assert_ne!(WorkerType::Regular, WorkerType::Decode); + assert_eq!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + }, + WorkerType::Prefill { + bootstrap_port: Some(8080) + } + ); + assert_ne!( + WorkerType::Prefill { + bootstrap_port: Some(8080) + }, + WorkerType::Prefill { + bootstrap_port: Some(8081) + } + ); + } + + #[test] + fn test_worker_type_clone() { + let original = WorkerType::Prefill { + bootstrap_port: Some(8080), + }; + let cloned = original.clone(); + assert_eq!(original, cloned); + } + + // Test HealthConfig + #[test] + fn test_health_config_default() { + let config = HealthConfig::default(); + assert_eq!(config.timeout_secs, 5); + assert_eq!(config.check_interval_secs, 30); + assert_eq!(config.endpoint, "/health"); + assert_eq!(config.failure_threshold, 3); + assert_eq!(config.success_threshold, 2); + } + + #[test] + fn test_health_config_custom() { + let config = HealthConfig { + timeout_secs: 10, + check_interval_secs: 60, + endpoint: "/healthz".to_string(), + failure_threshold: 5, + success_threshold: 3, + }; + assert_eq!(config.timeout_secs, 10); + assert_eq!(config.check_interval_secs, 60); + assert_eq!(config.endpoint, "/healthz"); + assert_eq!(config.failure_threshold, 5); + assert_eq!(config.success_threshold, 3); + } + + // Test BasicWorker + #[test] + fn test_basic_worker_creation() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + assert_eq!(worker.url(), "http://test:8080"); + assert_eq!(worker.worker_type(), WorkerType::Regular); + assert!(worker.is_healthy()); + assert_eq!(worker.load(), 0); + assert_eq!(worker.processed_requests(), 0); + } + + #[test] + fn test_worker_with_labels() { + let mut labels = std::collections::HashMap::new(); + labels.insert("env".to_string(), "prod".to_string()); + labels.insert("zone".to_string(), "us-west".to_string()); + + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular) + .with_labels(labels.clone()); + + assert_eq!(worker.metadata().labels, labels); + } + + #[test] + fn test_worker_with_health_config() { + let custom_config = HealthConfig { + timeout_secs: 15, + check_interval_secs: 45, + endpoint: "/custom-health".to_string(), + failure_threshold: 4, + success_threshold: 2, + }; + + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular) + .with_health_config(custom_config.clone()); + + assert_eq!(worker.metadata().health_config.timeout_secs, 15); + assert_eq!(worker.metadata().health_config.check_interval_secs, 45); + assert_eq!(worker.metadata().health_config.endpoint, "/custom-health"); + } + + // Test Worker trait implementation + #[test] + fn test_worker_url() { + let worker = BasicWorker::new("http://worker1:8080".to_string(), WorkerType::Regular); + assert_eq!(worker.url(), "http://worker1:8080"); + } + + #[test] + fn test_worker_type_getter() { + let regular = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + assert_eq!(regular.worker_type(), WorkerType::Regular); + + let prefill = BasicWorker::new( + "http://test:8080".to_string(), + WorkerType::Prefill { + bootstrap_port: Some(9090), + }, + ); + assert_eq!( + prefill.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + + let decode = BasicWorker::new("http://test:8080".to_string(), WorkerType::Decode); + assert_eq!(decode.worker_type(), WorkerType::Decode); + } + + #[test] + fn test_health_status() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + + // Initial state is healthy + assert!(worker.is_healthy()); + + // Set unhealthy + worker.set_healthy(false); + assert!(!worker.is_healthy()); + + // Set healthy again + worker.set_healthy(true); + assert!(worker.is_healthy()); + } + + #[test] + fn test_load_counter_operations() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + + // Initial load is 0 + assert_eq!(worker.load(), 0); + + // Increment once + worker.increment_load(); + assert_eq!(worker.load(), 1); + + // Increment twice more + worker.increment_load(); + worker.increment_load(); + assert_eq!(worker.load(), 3); + + // Decrement once + worker.decrement_load(); + assert_eq!(worker.load(), 2); + + // Decrement to 0 + worker.decrement_load(); + worker.decrement_load(); + assert_eq!(worker.load(), 0); + + // Decrement below 0 should stay at 0 + worker.decrement_load(); + assert_eq!(worker.load(), 0); + } + + #[test] + fn test_processed_counter() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + + // Initial count is 0 + assert_eq!(worker.processed_requests(), 0); + + // Increment multiple times + for i in 1..=100 { + worker.increment_processed(); + assert_eq!(worker.processed_requests(), i); + } + } + + #[test] + fn test_clone_worker() { + let original = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + original.increment_load(); + original.increment_processed(); + original.set_healthy(false); + + let cloned = original.clone_worker(); + + // Verify cloned worker has same URL and type + assert_eq!(cloned.url(), original.url()); + assert_eq!(cloned.worker_type(), original.worker_type()); + + // Load counters should be independent (cloned shares the Arc) + assert_eq!(cloned.load(), original.load()); + + // Modify original and verify clone is affected (shared state) + original.increment_load(); + assert_eq!(cloned.load(), original.load()); + } + + // Test concurrent operations + #[tokio::test] + async fn test_concurrent_load_increments() { + let worker = Arc::new(BasicWorker::new( + "http://test:8080".to_string(), + WorkerType::Regular, + )); + + let mut handles = vec![]; + + // Spawn 100 tasks incrementing load + for _ in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.increment_load(); + }); + handles.push(handle); + } + + // Wait for all tasks + for handle in handles { + handle.await.unwrap(); + } + + // Final count should be 100 + assert_eq!(worker.load(), 100); + } + + #[tokio::test] + async fn test_concurrent_load_decrements() { + let worker = Arc::new(BasicWorker::new( + "http://test:8080".to_string(), + WorkerType::Regular, + )); + + // Set initial load to 100 + for _ in 0..100 { + worker.increment_load(); + } + assert_eq!(worker.load(), 100); + + let mut handles = vec![]; + + // Spawn 100 tasks decrementing load + for _ in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.decrement_load(); + }); + handles.push(handle); + } + + // Wait for all tasks + for handle in handles { + handle.await.unwrap(); + } + + // Final count should be 0 + assert_eq!(worker.load(), 0); + } + + #[tokio::test] + async fn test_concurrent_health_updates() { + let worker = Arc::new(BasicWorker::new( + "http://test:8080".to_string(), + WorkerType::Regular, + )); + + let mut handles = vec![]; + + // Spawn threads randomly setting health status + for i in 0..100 { + let worker_clone = Arc::clone(&worker); + let handle = tokio::spawn(async move { + worker_clone.set_healthy(i % 2 == 0); + tokio::time::sleep(Duration::from_micros(10)).await; + }); + handles.push(handle); + } + + // Wait for all tasks + for handle in handles { + handle.await.unwrap(); + } + } + + // Test WorkerFactory + #[test] + fn test_create_regular_worker() { + let worker = WorkerFactory::create_regular("http://regular:8080".to_string()); + assert_eq!(worker.url(), "http://regular:8080"); + assert_eq!(worker.worker_type(), WorkerType::Regular); + } + + #[test] + fn test_create_prefill_worker() { + // With bootstrap port + let worker1 = WorkerFactory::create_prefill("http://prefill:8080".to_string(), Some(9090)); + assert_eq!(worker1.url(), "http://prefill:8080"); + assert_eq!( + worker1.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + + // Without bootstrap port + let worker2 = WorkerFactory::create_prefill("http://prefill:8080".to_string(), None); + assert_eq!( + worker2.worker_type(), + WorkerType::Prefill { + bootstrap_port: None + } + ); + } + + #[test] + fn test_create_decode_worker() { + let worker = WorkerFactory::create_decode("http://decode:8080".to_string()); + assert_eq!(worker.url(), "http://decode:8080"); + assert_eq!(worker.worker_type(), WorkerType::Decode); + } + + #[test] + fn test_create_from_urls() { + let regular_urls = vec![ + "http://regular1:8080".to_string(), + "http://regular2:8080".to_string(), + ]; + let prefill_urls = vec![ + ("http://prefill1:8080".to_string(), Some(9090)), + ("http://prefill2:8080".to_string(), None), + ]; + let decode_urls = vec![ + "http://decode1:8080".to_string(), + "http://decode2:8080".to_string(), + ]; + + let (regular, prefill, decode) = + WorkerFactory::create_from_urls(regular_urls, prefill_urls, decode_urls); + + assert_eq!(regular.len(), 2); + assert_eq!(prefill.len(), 2); + assert_eq!(decode.len(), 2); + + assert_eq!(regular[0].url(), "http://regular1:8080"); + assert_eq!(prefill[0].url(), "http://prefill1:8080"); + assert_eq!(decode[0].url(), "http://decode1:8080"); + } + + // Test WorkerCollection trait + #[test] + fn test_healthy_workers_filter() { + let workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + WorkerFactory::create_regular("http://w3:8080".to_string()), + ]; + + // Set some workers unhealthy + workers[0].set_healthy(false); + workers[2].set_healthy(false); + + let healthy = workers.healthy_workers(); + assert_eq!(healthy.len(), 1); + assert_eq!(healthy[0].url(), "http://w2:8080"); + } + + #[test] + fn test_total_load_calculation() { + let workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + WorkerFactory::create_regular("http://w3:8080".to_string()), + ]; + + // Set different loads + workers[0].increment_load(); + workers[0].increment_load(); // load = 2 + + workers[1].increment_load(); + workers[1].increment_load(); + workers[1].increment_load(); // load = 3 + + workers[2].increment_load(); // load = 1 + + assert_eq!(workers.total_load(), 6); + } + + #[test] + fn test_find_worker() { + let workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + WorkerFactory::create_regular("http://w3:8080".to_string()), + ]; + + // Found case + let found = workers.find_worker("http://w2:8080"); + assert!(found.is_some()); + assert_eq!(found.unwrap().url(), "http://w2:8080"); + + // Not found case + let not_found = workers.find_worker("http://w4:8080"); + assert!(not_found.is_none()); + } + + #[test] + fn test_find_worker_mut() { + let mut workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + ]; + + // Find and modify + if let Some(worker) = workers.find_worker_mut("http://w1:8080") { + worker.set_healthy(false); + } + + // Verify modification + assert!(!workers[0].is_healthy()); + assert!(workers[1].is_healthy()); + } + + // Test WorkerLoadGuard + #[test] + fn test_load_guard_single_worker() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + assert_eq!(worker.load(), 0); + + { + let _guard = WorkerLoadGuard::new(&worker); + assert_eq!(worker.load(), 1); + } + + // Guard dropped, load decremented + assert_eq!(worker.load(), 0); + } + + #[test] + fn test_load_guard_multiple_workers() { + let workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + WorkerFactory::create_regular("http://w3:8080".to_string()), + ]; + + let worker_refs: Vec<&dyn Worker> = workers.iter().map(|w| w.as_ref()).collect(); + + { + let _guard = WorkerLoadGuard::new_multi(worker_refs); + // All loads incremented + assert_eq!(workers[0].load(), 1); + assert_eq!(workers[1].load(), 1); + assert_eq!(workers[2].load(), 1); + } + + // All loads decremented + assert_eq!(workers[0].load(), 0); + assert_eq!(workers[1].load(), 0); + assert_eq!(workers[2].load(), 0); + } + + #[test] + fn test_load_guard_panic_safety() { + let worker = Arc::new(BasicWorker::new( + "http://test:8080".to_string(), + WorkerType::Regular, + )); + assert_eq!(worker.load(), 0); + + // Clone for use inside catch_unwind + let worker_clone = Arc::clone(&worker); + + // Use AssertUnwindSafe wrapper for the test + // This is safe because we're only testing the load counter behavior, + // not the grpc_client which is None for HTTP workers + use std::panic::AssertUnwindSafe; + + // This will panic, but the guard should still clean up + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + let _guard = WorkerLoadGuard::new(worker_clone.as_ref()); + assert_eq!(worker_clone.load(), 1); + panic!("Test panic"); + })); + + // Verify panic occurred + assert!(result.is_err()); + + // Load should be decremented even after panic + assert_eq!(worker.load(), 0); + } + + // Test helper functions + #[test] + fn test_urls_to_workers() { + let urls = vec!["http://w1:8080".to_string(), "http://w2:8080".to_string()]; + + let workers = urls_to_workers(urls); + assert_eq!(workers.len(), 2); + assert_eq!(workers[0].url(), "http://w1:8080"); + assert_eq!(workers[1].url(), "http://w2:8080"); + assert_eq!(workers[0].worker_type(), WorkerType::Regular); + } + + #[test] + fn test_workers_to_urls() { + let workers: Vec> = vec![ + WorkerFactory::create_regular("http://w1:8080".to_string()), + WorkerFactory::create_regular("http://w2:8080".to_string()), + ]; + + let urls = workers_to_urls(&workers); + assert_eq!(urls, vec!["http://w1:8080", "http://w2:8080"]); + } + + // Test synchronous health check wrapper + #[test] + fn test_check_health_sync_wrapper() { + // We can't easily test the actual HTTP call without mocking, + // but we can verify the sync wrapper works + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + + // This will fail because there's no server at this URL, + // but it tests that the sync wrapper doesn't panic + let result = worker.check_health(); + assert!(result.is_err()); + } + + // Test HealthChecker background task + #[tokio::test] + async fn test_health_checker_startup() { + let workers = Arc::new(RwLock::new(vec![WorkerFactory::create_regular( + "http://w1:8080".to_string(), + )])); + + let checker = start_health_checker(workers.clone(), 60); + + // Verify it starts without panic + tokio::time::sleep(Duration::from_millis(100)).await; + + // Shutdown + checker.shutdown().await; + } + + #[tokio::test] + async fn test_health_checker_shutdown() { + let workers = Arc::new(RwLock::new(vec![WorkerFactory::create_regular( + "http://w1:8080".to_string(), + )])); + + let checker = start_health_checker(workers.clone(), 60); + + // Shutdown should complete quickly + let shutdown_result = timeout(Duration::from_secs(1), checker.shutdown()).await; + assert!(shutdown_result.is_ok()); + } + + // Performance test for load counter + #[test] + fn test_load_counter_performance() { + use std::time::Instant; + + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + let iterations = 1_000_000; + + let start = Instant::now(); + for _ in 0..iterations { + worker.increment_load(); + } + let duration = start.elapsed(); + + let ops_per_sec = iterations as f64 / duration.as_secs_f64(); + println!("Load counter operations per second: {:.0}", ops_per_sec); + + // Should be well over 1M ops/sec + assert!(ops_per_sec > 1_000_000.0); + } + + // ===== Tests for DPAwareWorker ===== + + #[test] + fn test_dp_aware_worker_creation() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 2, 4, WorkerType::Regular); + + assert_eq!(dp_worker.url(), "http://worker1:8080@2"); + assert_eq!(dp_worker.base_url(), "http://worker1:8080"); + assert!(dp_worker.is_dp_aware()); + assert_eq!(dp_worker.dp_rank(), Some(2)); + assert_eq!(dp_worker.dp_size(), Some(4)); + assert_eq!(dp_worker.worker_type(), WorkerType::Regular); + } + + #[test] + fn test_dp_aware_worker_creation_prefill() { + let dp_worker = DPAwareWorker::new( + "http://worker1:8080".to_string(), + 1, + 2, + WorkerType::Prefill { + bootstrap_port: Some(9090), + }, + ); + + assert_eq!(dp_worker.url(), "http://worker1:8080@1"); + assert!(dp_worker.is_dp_aware()); + assert_eq!( + dp_worker.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + } + + #[test] + fn test_dp_aware_worker_creation_decode() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 0, 4, WorkerType::Decode); + + assert_eq!(dp_worker.url(), "http://worker1:8080@0"); + assert!(dp_worker.is_dp_aware()); + assert_eq!(dp_worker.worker_type(), WorkerType::Decode); + } + + #[tokio::test] + async fn test_dp_aware_prepare_request() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 3, 8, WorkerType::Regular); + + let original_req = serde_json::json!({ + "prompt": "Hello", + "max_tokens": 100 + }); + + let prepared_req = dp_worker.prepare_request(original_req).await.unwrap(); + + assert_eq!(prepared_req["prompt"], "Hello"); + assert_eq!(prepared_req["max_tokens"], 100); + assert_eq!(prepared_req["data_parallel_rank"], 3); + } + + #[tokio::test] + async fn test_dp_aware_prepare_request_invalid() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 0, 4, WorkerType::Regular); + + // Non-object JSON should fail + let invalid_req = serde_json::json!("not an object"); + let result = dp_worker.prepare_request(invalid_req).await; + + assert!(result.is_err()); + match result.unwrap_err() { + WorkerError::InvalidConfiguration { message } => { + assert!(message.contains("JSON object")); + } + _ => panic!("Expected InvalidConfiguration error"), + } + } + + #[test] + fn test_dp_aware_endpoint_url() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 1, 4, WorkerType::Regular); + + assert_eq!( + dp_worker.endpoint_url("/generate"), + "http://worker1:8080/generate" + ); + assert_eq!( + dp_worker.endpoint_url("/health"), + "http://worker1:8080/health" + ); + } + + #[test] + fn test_dp_aware_worker_delegated_methods() { + let dp_worker = + DPAwareWorker::new("http://worker1:8080".to_string(), 0, 2, WorkerType::Regular); + + // Test health status + assert!(dp_worker.is_healthy()); + dp_worker.set_healthy(false); + assert!(!dp_worker.is_healthy()); + + // Test load tracking + assert_eq!(dp_worker.load(), 0); + dp_worker.increment_load(); + assert_eq!(dp_worker.load(), 1); + dp_worker.decrement_load(); + assert_eq!(dp_worker.load(), 0); + + // Test processed tracking + assert_eq!(dp_worker.processed_requests(), 0); + dp_worker.increment_processed(); + assert_eq!(dp_worker.processed_requests(), 1); + } + + // ===== Tests for WorkerFactory async methods ===== + + #[tokio::test] + async fn test_factory_create_dp_aware() { + let worker = WorkerFactory::create_dp_aware( + "http://worker1:8080".to_string(), + 1, + 4, + WorkerType::Regular, + ); + + assert_eq!(worker.url(), "http://worker1:8080@1"); + assert!(worker.is_dp_aware()); + assert_eq!(worker.dp_rank(), Some(1)); + assert_eq!(worker.dp_size(), Some(4)); + assert_eq!(worker.worker_type(), WorkerType::Regular); + } + + #[tokio::test] + async fn test_factory_create_dp_aware_prefill() { + let worker = WorkerFactory::create_dp_aware( + "http://worker1:8080".to_string(), + 0, + 2, + WorkerType::Prefill { + bootstrap_port: Some(8090), + }, + ); + + assert_eq!(worker.url(), "http://worker1:8080@0"); + assert!(worker.is_dp_aware()); + assert_eq!( + worker.worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(8090) + } + ); + } + + #[tokio::test] + async fn test_factory_create_workers_regular() { + let urls = vec!["http://w1:8080".to_string(), "http://w2:8080".to_string()]; + + let workers = WorkerFactory::create_workers(urls, false, &None) + .await + .unwrap(); + + assert_eq!(workers.len(), 2); + assert!(!workers[0].is_dp_aware()); + assert!(!workers[1].is_dp_aware()); + assert_eq!(workers[0].url(), "http://w1:8080"); + assert_eq!(workers[1].url(), "http://w2:8080"); + } + + // ===== Circuit Breaker Integration Tests ===== + + #[test] + fn test_worker_circuit_breaker() { + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular); + + // Initial state should be available + assert!(worker.is_available()); + assert_eq!( + worker.circuit_breaker().state(), + crate::core::CircuitState::Closed + ); + + // Record some failures + worker.record_outcome(false); + worker.record_outcome(false); + + // Still available (default threshold is 5) + assert!(worker.is_available()); + + // Record more failures to open circuit + worker.record_outcome(false); + worker.record_outcome(false); + worker.record_outcome(false); + + // Circuit should be open, worker not available + assert!(!worker.is_available()); + assert!(worker.is_healthy()); // Still healthy + assert!(!worker.circuit_breaker().can_execute()); // But circuit is open + } + + #[test] + fn test_worker_with_circuit_breaker_config() { + let config = crate::core::CircuitBreakerConfig { + failure_threshold: 2, + success_threshold: 1, + timeout_duration: Duration::from_millis(100), + window_duration: Duration::from_secs(60), + }; + + let worker = BasicWorker::new("http://test:8080".to_string(), WorkerType::Regular) + .with_circuit_breaker_config(config); + + // Should open after 2 failures + worker.record_outcome(false); + assert!(worker.is_available()); + worker.record_outcome(false); + assert!(!worker.is_available()); + + // Wait for timeout + thread::sleep(Duration::from_millis(150)); + + // Should be half-open + assert!(worker.is_available()); + assert_eq!( + worker.circuit_breaker().state(), + crate::core::CircuitState::HalfOpen + ); + + // Success should close it + worker.record_outcome(true); + assert_eq!( + worker.circuit_breaker().state(), + crate::core::CircuitState::Closed + ); + } + + #[test] + fn test_dp_aware_worker_circuit_breaker() { + let dp_worker = + DPAwareWorker::new("http://worker:8080".to_string(), 0, 2, WorkerType::Regular); + + // Should have circuit breaker + assert!(dp_worker.is_available()); + + // Record failures + for _ in 0..5 { + dp_worker.record_outcome(false); + } + + // Should not be available + assert!(!dp_worker.is_available()); + assert_eq!( + dp_worker.circuit_breaker().state(), + crate::core::CircuitState::Open + ); + } + + // ===== Integration tests ===== + + #[tokio::test] + async fn test_mixed_worker_types() { + // Create a mix of worker types + let regular = WorkerFactory::create_regular("http://regular:8080".to_string()); + let prefill = WorkerFactory::create_prefill("http://prefill:8080".to_string(), Some(9090)); + let decode = WorkerFactory::create_decode("http://decode:8080".to_string()); + let dp_aware_regular = + WorkerFactory::create_dp_aware("http://dp:8080".to_string(), 0, 2, WorkerType::Regular); + let dp_aware_prefill = WorkerFactory::create_dp_aware( + "http://dp-prefill:8080".to_string(), + 1, + 2, + WorkerType::Prefill { + bootstrap_port: None, + }, + ); + let dp_aware_decode = WorkerFactory::create_dp_aware( + "http://dp-decode:8080".to_string(), + 0, + 4, + WorkerType::Decode, + ); + + let workers: Vec> = vec![ + regular, + prefill, + decode, + dp_aware_regular, + dp_aware_prefill, + dp_aware_decode, + ]; + + // Test that they all implement Worker trait properly + for worker in &workers { + assert!(worker.is_healthy()); + assert_eq!(worker.load(), 0); + assert_eq!(worker.processed_requests(), 0); + } + + // Test specific behaviors + assert!(!workers[0].is_dp_aware()); // regular + assert!(!workers[1].is_dp_aware()); // prefill + assert!(!workers[2].is_dp_aware()); // decode + assert!(workers[3].is_dp_aware()); // dp_aware_regular + assert!(workers[4].is_dp_aware()); // dp_aware_prefill + assert!(workers[5].is_dp_aware()); // dp_aware_decode + + // Test worker types + assert_eq!(workers[0].worker_type(), WorkerType::Regular); + assert_eq!( + workers[1].worker_type(), + WorkerType::Prefill { + bootstrap_port: Some(9090) + } + ); + assert_eq!(workers[2].worker_type(), WorkerType::Decode); + assert_eq!(workers[3].worker_type(), WorkerType::Regular); + assert_eq!( + workers[4].worker_type(), + WorkerType::Prefill { + bootstrap_port: None + } + ); + assert_eq!(workers[5].worker_type(), WorkerType::Decode); + } +} diff --git a/sgl-router/src/grpc/client.rs b/sgl-router/src/grpc/client.rs new file mode 100644 index 000000000..8561b79db --- /dev/null +++ b/sgl-router/src/grpc/client.rs @@ -0,0 +1,327 @@ +use std::time::Duration; +use tonic::{transport::Channel, Request}; +use tracing::debug; + +// Include the generated protobuf code +pub mod proto { + tonic::include_proto!("sglang.grpc.scheduler"); +} + +// The generated module structure depends on the package name in the .proto file +// package sglang.grpc.scheduler; generates a nested module structure + +/// gRPC client for SGLang scheduler +pub struct SglangSchedulerClient { + client: proto::sglang_scheduler_client::SglangSchedulerClient, +} + +impl SglangSchedulerClient { + /// Create a new client and connect to the scheduler + pub async fn connect(endpoint: &str) -> Result> { + debug!("Connecting to SGLang scheduler at {}", endpoint); + + // Convert grpc:// to http:// for tonic + let http_endpoint = if endpoint.starts_with("grpc://") { + endpoint.replace("grpc://", "http://") + } else { + endpoint.to_string() + }; + + let channel = Channel::from_shared(http_endpoint)? + .timeout(Duration::from_secs(30)) + .connect() + .await?; + + let client = proto::sglang_scheduler_client::SglangSchedulerClient::new(channel); + + Ok(Self { client }) + } + + /// Initialize the connection + pub async fn initialize( + &mut self, + client_id: String, + ) -> Result> { + let request = Request::new(proto::InitializeRequest { + client_id, + client_version: "0.1.0".to_string(), + mode: proto::initialize_request::Mode::Regular as i32, + }); + + let response = self.client.initialize(request).await?; + Ok(response.into_inner()) + } + + /// Submit a generation request (returns streaming response) + pub async fn generate_stream( + &mut self, + req: proto::GenerateRequest, + ) -> Result, Box> { + let request = Request::new(req); + let response = self.client.generate(request).await?; + Ok(response.into_inner()) + } + + /// Perform health check + pub async fn health_check( + &mut self, + ) -> Result> { + debug!("Sending health check request"); + let request = Request::new(proto::HealthCheckRequest { + include_detailed_metrics: false, + }); + + let response = self.client.health_check(request).await?; + debug!("Health check response received"); + Ok(response.into_inner()) + } + + /// Abort a request + pub async fn abort_request( + &mut self, + request_id: String, + reason: String, + ) -> Result<(), Box> { + let request = Request::new(proto::AbortRequest { request_id, reason }); + + self.client.abort(request).await?; + Ok(()) + } + + /// Flush cache + pub async fn flush_cache( + &mut self, + flush_all: bool, + session_ids: &[String], + ) -> Result> { + let request = Request::new(proto::FlushCacheRequest { + flush_all, + session_ids: session_ids.to_vec(), + }); + + let response = self.client.flush_cache(request).await?; + Ok(response.into_inner()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_proto_types_compilation() { + // Test that protobuf types can be constructed + let init_req = proto::InitializeRequest { + client_id: "test-client".to_string(), + client_version: "0.1.0".to_string(), + mode: 0, + }; + assert_eq!(init_req.client_id, "test-client"); + assert_eq!(init_req.client_version, "0.1.0"); + assert_eq!(init_req.mode, 0); + } + + #[test] + fn test_generate_request_construction() { + let sampling_params = proto::SamplingParams { + temperature: 0.7, + max_new_tokens: 128, + top_p: 0.9, + top_k: 50, + stop: vec!["".to_string()], + ..Default::default() + }; + + let gen_req = proto::GenerateRequest { + request_id: "test-req-123".to_string(), + input: Some(proto::generate_request::Input::Text( + "Hello world".to_string(), + )), + sampling_params: Some(sampling_params), + return_logprob: true, + logprob_start_len: 0, + top_logprobs_num: 5, + ..Default::default() + }; + + assert_eq!(gen_req.request_id, "test-req-123"); + if let Some(proto::generate_request::Input::Text(text)) = &gen_req.input { + assert_eq!(text, "Hello world"); + } + assert!(gen_req.return_logprob); + assert_eq!(gen_req.top_logprobs_num, 5); + + let params = gen_req.sampling_params.unwrap(); + assert_eq!(params.temperature, 0.7); + assert_eq!(params.max_new_tokens, 128); + assert_eq!(params.stop, vec![""]); + } + + #[test] + fn test_health_check_request() { + let health_req = proto::HealthCheckRequest { + include_detailed_metrics: true, + }; + assert!(health_req.include_detailed_metrics); + } + + #[test] + fn test_abort_request_construction() { + let abort_req = proto::AbortRequest { + request_id: "req-456".to_string(), + reason: "User canceled".to_string(), + }; + assert_eq!(abort_req.request_id, "req-456"); + assert_eq!(abort_req.reason, "User canceled"); + } + + #[test] + fn test_flush_cache_request() { + let flush_req = proto::FlushCacheRequest { + flush_all: true, + session_ids: vec!["session1".to_string(), "session2".to_string()], + }; + assert!(flush_req.flush_all); + assert_eq!(flush_req.session_ids.len(), 2); + assert_eq!(flush_req.session_ids[0], "session1"); + } + + #[test] + fn test_sampling_params_defaults() { + let params = proto::SamplingParams::default(); + assert_eq!(params.temperature, 0.0); + assert_eq!(params.max_new_tokens, 0); + assert_eq!(params.top_p, 0.0); + assert_eq!(params.top_k, 0); + assert!(params.stop.is_empty()); + } + + #[test] + fn test_multimodal_inputs() { + let mm_inputs = proto::MultimodalInputs { + image_urls: vec!["http://example.com/image.jpg".to_string()], + video_urls: vec![], + audio_urls: vec![], + image_data: vec![], + video_data: vec![], + audio_data: vec![], + modalities: vec!["image".to_string()], + ..Default::default() + }; + + assert_eq!(mm_inputs.image_urls.len(), 1); + assert_eq!(mm_inputs.image_urls[0], "http://example.com/image.jpg"); + assert_eq!(mm_inputs.modalities[0], "image"); + } + + #[test] + fn test_session_params() { + let session_params = proto::SessionParams { + session_id: "sess-789".to_string(), + request_id: "req-101".to_string(), + offset: 100, + replace: true, + drop_previous_output: false, + }; + + assert_eq!(session_params.session_id, "sess-789"); + assert_eq!(session_params.request_id, "req-101"); + assert_eq!(session_params.offset, 100); + assert!(session_params.replace); + assert!(!session_params.drop_previous_output); + } + + #[test] + fn test_embed_request() { + let embed_req = proto::EmbedRequest { + request_id: "embed-req-202".to_string(), + input: Some(proto::embed_request::Input::Text( + "This is a test sentence for embedding".to_string(), + )), + log_metrics: true, + data_parallel_rank: 0, + ..Default::default() + }; + + assert_eq!(embed_req.request_id, "embed-req-202"); + if let Some(proto::embed_request::Input::Text(text)) = &embed_req.input { + assert_eq!(text, "This is a test sentence for embedding"); + } + assert!(embed_req.log_metrics); + assert_eq!(embed_req.data_parallel_rank, 0); + } + + #[tokio::test] + async fn test_client_connect_invalid_endpoint() { + // Test connecting to an invalid endpoint should return error + let result = SglangSchedulerClient::connect("invalid://endpoint").await; + assert!(result.is_err()); + } + + #[test] + fn test_tokenized_input() { + let tokenized = proto::TokenizedInput { + original_text: "Hello world".to_string(), + input_ids: vec![1, 15043, 1917, 2], + }; + + assert_eq!(tokenized.original_text, "Hello world"); + assert_eq!(tokenized.input_ids, vec![1, 15043, 1917, 2]); + } + + // Test response type construction + #[test] + fn test_generate_stream_chunk() { + let chunk = proto::GenerateStreamChunk { + token_id: 1234, + text: " world".to_string(), + prompt_tokens: 5, + completion_tokens: 2, + cached_tokens: 3, + generation_time: 0.025, + queue_time: 10, + ..Default::default() + }; + + assert_eq!(chunk.token_id, 1234); + assert_eq!(chunk.text, " world"); + assert_eq!(chunk.prompt_tokens, 5); + assert_eq!(chunk.completion_tokens, 2); + assert_eq!(chunk.cached_tokens, 3); + assert_eq!(chunk.generation_time, 0.025); + assert_eq!(chunk.queue_time, 10); + } + + #[test] + fn test_model_info() { + let model_info = proto::ModelInfo { + model_name: "Meta-Llama-3-8B-Instruct".to_string(), + max_context_length: 8192, + vocab_size: 128256, + supports_tool_calling: true, + supports_vision: false, + special_tokens: vec![ + "<|begin_of_text|>".to_string(), + "<|end_of_text|>".to_string(), + ], + model_type: "llama".to_string(), + num_layers: 32, + hidden_size: 4096, + num_attention_heads: 32, + num_key_value_heads: 8, + tokenizer_type: "llama".to_string(), + eos_token_ids: vec![128001, 128009], + pad_token_id: 128001, + bos_token_id: 128000, + }; + + assert_eq!(model_info.model_name, "Meta-Llama-3-8B-Instruct"); + assert_eq!(model_info.max_context_length, 8192); + assert_eq!(model_info.vocab_size, 128256); + assert!(model_info.supports_tool_calling); + assert!(!model_info.supports_vision); + assert_eq!(model_info.special_tokens.len(), 2); + assert_eq!(model_info.num_layers, 32); + assert_eq!(model_info.eos_token_ids, vec![128001, 128009]); + } +} diff --git a/sgl-router/src/grpc/mod.rs b/sgl-router/src/grpc/mod.rs new file mode 100644 index 000000000..331a6a538 --- /dev/null +++ b/sgl-router/src/grpc/mod.rs @@ -0,0 +1,8 @@ +//! gRPC client module for communicating with SGLang scheduler +//! +//! This module provides a gRPC client implementation for the SGLang router. + +pub mod client; + +// Re-export the client +pub use client::{proto, SglangSchedulerClient}; diff --git a/sgl-router/src/lib.rs b/sgl-router/src/lib.rs new file mode 100644 index 000000000..938a1ba0c --- /dev/null +++ b/sgl-router/src/lib.rs @@ -0,0 +1,508 @@ +use pyo3::prelude::*; +pub mod config; +pub mod logging; +use std::collections::HashMap; + +pub mod core; +#[cfg(feature = "grpc-client")] +pub mod grpc; +pub mod mcp; +pub mod metrics; +pub mod middleware; +pub mod policies; +pub mod protocols; +pub mod reasoning_parser; +pub mod routers; +pub mod server; +pub mod service_discovery; +pub mod tokenizer; +pub mod tool_parser; +pub mod tree; +use crate::metrics::PrometheusConfig; + +#[pyclass(eq)] +#[derive(Clone, PartialEq, Debug)] +pub enum PolicyType { + Random, + RoundRobin, + CacheAware, + PowerOfTwo, +} + +#[pyclass] +#[derive(Debug, Clone, PartialEq)] +struct Router { + host: String, + port: u16, + worker_urls: Vec, + policy: PolicyType, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + dp_aware: bool, + api_key: Option, + log_dir: Option, + log_level: Option, + service_discovery: bool, + selector: HashMap, + service_discovery_port: u16, + service_discovery_namespace: Option, + prefill_selector: HashMap, + decode_selector: HashMap, + bootstrap_port_annotation: String, + prometheus_port: Option, + prometheus_host: Option, + request_timeout_secs: u64, + request_id_headers: Option>, + pd_disaggregation: bool, + prefill_urls: Option)>>, + decode_urls: Option>, + prefill_policy: Option, + decode_policy: Option, + max_concurrent_requests: usize, + cors_allowed_origins: Vec, + // Retry configuration + retry_max_retries: u32, + retry_initial_backoff_ms: u64, + retry_max_backoff_ms: u64, + retry_backoff_multiplier: f32, + retry_jitter_factor: f32, + disable_retries: bool, + // Circuit breaker configuration + cb_failure_threshold: u32, + cb_success_threshold: u32, + cb_timeout_duration_secs: u64, + cb_window_duration_secs: u64, + disable_circuit_breaker: bool, + // Health check configuration + health_failure_threshold: u32, + health_success_threshold: u32, + health_check_timeout_secs: u64, + health_check_interval_secs: u64, + health_check_endpoint: String, + // IGW (Inference Gateway) configuration + enable_igw: bool, + queue_size: usize, + queue_timeout_secs: u64, + rate_limit_tokens_per_second: Option, + // Connection mode (determined from worker URLs) + connection_mode: config::ConnectionMode, + // Model path for tokenizer + model_path: Option, + // Explicit tokenizer path + tokenizer_path: Option, +} + +impl Router { + /// Determine connection mode from worker URLs + fn determine_connection_mode(worker_urls: &[String]) -> config::ConnectionMode { + // Only consider it gRPC if explicitly specified with grpc:// or grpcs:// scheme + for url in worker_urls { + if url.starts_with("grpc://") || url.starts_with("grpcs://") { + return config::ConnectionMode::Grpc; + } + } + // Default to HTTP for all other cases (including http://, https://, or no scheme) + config::ConnectionMode::Http + } + + /// Convert PyO3 Router to RouterConfig + pub fn to_router_config(&self) -> config::ConfigResult { + use config::{ + DiscoveryConfig, MetricsConfig, PolicyConfig as ConfigPolicyConfig, RoutingMode, + }; + + // Convert policy helper function + let convert_policy = |policy: &PolicyType| -> ConfigPolicyConfig { + match policy { + PolicyType::Random => ConfigPolicyConfig::Random, + PolicyType::RoundRobin => ConfigPolicyConfig::RoundRobin, + PolicyType::CacheAware => ConfigPolicyConfig::CacheAware { + cache_threshold: self.cache_threshold, + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + eviction_interval_secs: self.eviction_interval_secs, + max_tree_size: self.max_tree_size, + }, + PolicyType::PowerOfTwo => ConfigPolicyConfig::PowerOfTwo { + load_check_interval_secs: 5, // Default value + }, + } + }; + + // Determine routing mode + let mode = if self.enable_igw { + // IGW mode - routing mode is not used in IGW, but we need to provide a placeholder + RoutingMode::Regular { + worker_urls: vec![], + } + } else if self.pd_disaggregation { + RoutingMode::PrefillDecode { + prefill_urls: self.prefill_urls.clone().unwrap_or_default(), + decode_urls: self.decode_urls.clone().unwrap_or_default(), + prefill_policy: self.prefill_policy.as_ref().map(convert_policy), + decode_policy: self.decode_policy.as_ref().map(convert_policy), + } + } else { + RoutingMode::Regular { + worker_urls: self.worker_urls.clone(), + } + }; + + // Convert main policy + let policy = convert_policy(&self.policy); + + // Service discovery configuration + let discovery = if self.service_discovery { + Some(DiscoveryConfig { + enabled: true, + namespace: self.service_discovery_namespace.clone(), + port: self.service_discovery_port, + check_interval_secs: 60, + selector: self.selector.clone(), + prefill_selector: self.prefill_selector.clone(), + decode_selector: self.decode_selector.clone(), + bootstrap_port_annotation: self.bootstrap_port_annotation.clone(), + }) + } else { + None + }; + + // Metrics configuration + let metrics = match (self.prometheus_port, self.prometheus_host.as_ref()) { + (Some(port), Some(host)) => Some(MetricsConfig { + port, + host: host.clone(), + }), + _ => None, + }; + + Ok(config::RouterConfig { + mode, + policy, + host: self.host.clone(), + port: self.port, + connection_mode: self.connection_mode.clone(), + max_payload_size: self.max_payload_size, + request_timeout_secs: self.request_timeout_secs, + worker_startup_timeout_secs: self.worker_startup_timeout_secs, + worker_startup_check_interval_secs: self.worker_startup_check_interval, + dp_aware: self.dp_aware, + api_key: self.api_key.clone(), + discovery, + metrics, + log_dir: self.log_dir.clone(), + log_level: self.log_level.clone(), + request_id_headers: self.request_id_headers.clone(), + max_concurrent_requests: self.max_concurrent_requests, + queue_size: self.queue_size, + queue_timeout_secs: self.queue_timeout_secs, + rate_limit_tokens_per_second: self.rate_limit_tokens_per_second, + cors_allowed_origins: self.cors_allowed_origins.clone(), + retry: config::RetryConfig { + max_retries: self.retry_max_retries, + initial_backoff_ms: self.retry_initial_backoff_ms, + max_backoff_ms: self.retry_max_backoff_ms, + backoff_multiplier: self.retry_backoff_multiplier, + jitter_factor: self.retry_jitter_factor, + }, + circuit_breaker: config::CircuitBreakerConfig { + failure_threshold: self.cb_failure_threshold, + success_threshold: self.cb_success_threshold, + timeout_duration_secs: self.cb_timeout_duration_secs, + window_duration_secs: self.cb_window_duration_secs, + }, + disable_retries: self.disable_retries, + disable_circuit_breaker: self.disable_circuit_breaker, + health_check: config::HealthCheckConfig { + failure_threshold: self.health_failure_threshold, + success_threshold: self.health_success_threshold, + timeout_secs: self.health_check_timeout_secs, + check_interval_secs: self.health_check_interval_secs, + endpoint: self.health_check_endpoint.clone(), + }, + enable_igw: self.enable_igw, + model_path: self.model_path.clone(), + tokenizer_path: self.tokenizer_path.clone(), + }) + } +} + +#[pymethods] +impl Router { + #[new] + #[pyo3(signature = ( + worker_urls, + policy = PolicyType::RoundRobin, + host = String::from("127.0.0.1"), + port = 3001, + worker_startup_timeout_secs = 600, + worker_startup_check_interval = 30, + cache_threshold = 0.3, + balance_abs_threshold = 64, + balance_rel_threshold = 1.5, + eviction_interval_secs = 120, + max_tree_size = 2usize.pow(26), + max_payload_size = 512 * 1024 * 1024, // 512MB default for large batches + dp_aware = false, + api_key = None, + log_dir = None, + log_level = None, + service_discovery = false, + selector = HashMap::new(), + service_discovery_port = 80, + service_discovery_namespace = None, + prefill_selector = HashMap::new(), + decode_selector = HashMap::new(), + bootstrap_port_annotation = String::from("sglang.ai/bootstrap-port"), + prometheus_port = None, + prometheus_host = None, + request_timeout_secs = 1800, // Add configurable request timeout + request_id_headers = None, // Custom request ID headers + pd_disaggregation = false, // New flag for PD mode + prefill_urls = None, + decode_urls = None, + prefill_policy = None, + decode_policy = None, + max_concurrent_requests = 256, + cors_allowed_origins = vec![], + // Retry defaults + retry_max_retries = 5, + retry_initial_backoff_ms = 50, + retry_max_backoff_ms = 30_000, + retry_backoff_multiplier = 1.5, + retry_jitter_factor = 0.2, + disable_retries = false, + // Circuit breaker defaults + cb_failure_threshold = 10, + cb_success_threshold = 3, + cb_timeout_duration_secs = 60, + cb_window_duration_secs = 120, + disable_circuit_breaker = false, + // Health check defaults + health_failure_threshold = 3, + health_success_threshold = 2, + health_check_timeout_secs = 5, + health_check_interval_secs = 60, + health_check_endpoint = String::from("/health"), + // IGW defaults + enable_igw = false, + queue_size = 100, + queue_timeout_secs = 60, + rate_limit_tokens_per_second = None, + // Tokenizer defaults + model_path = None, + tokenizer_path = None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + worker_urls: Vec, + policy: PolicyType, + host: String, + port: u16, + worker_startup_timeout_secs: u64, + worker_startup_check_interval: u64, + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + eviction_interval_secs: u64, + max_tree_size: usize, + max_payload_size: usize, + dp_aware: bool, + api_key: Option, + log_dir: Option, + log_level: Option, + service_discovery: bool, + selector: HashMap, + service_discovery_port: u16, + service_discovery_namespace: Option, + prefill_selector: HashMap, + decode_selector: HashMap, + bootstrap_port_annotation: String, + prometheus_port: Option, + prometheus_host: Option, + request_timeout_secs: u64, + request_id_headers: Option>, + pd_disaggregation: bool, + prefill_urls: Option)>>, + decode_urls: Option>, + prefill_policy: Option, + decode_policy: Option, + max_concurrent_requests: usize, + cors_allowed_origins: Vec, + retry_max_retries: u32, + retry_initial_backoff_ms: u64, + retry_max_backoff_ms: u64, + retry_backoff_multiplier: f32, + retry_jitter_factor: f32, + disable_retries: bool, + cb_failure_threshold: u32, + cb_success_threshold: u32, + cb_timeout_duration_secs: u64, + cb_window_duration_secs: u64, + disable_circuit_breaker: bool, + health_failure_threshold: u32, + health_success_threshold: u32, + health_check_timeout_secs: u64, + health_check_interval_secs: u64, + health_check_endpoint: String, + enable_igw: bool, + queue_size: usize, + queue_timeout_secs: u64, + rate_limit_tokens_per_second: Option, + model_path: Option, + tokenizer_path: Option, + ) -> PyResult { + // Determine connection mode from worker URLs + let mut all_urls = worker_urls.clone(); + + // Add prefill URLs if in PD mode + if let Some(ref prefill_urls) = prefill_urls { + for (url, _) in prefill_urls { + all_urls.push(url.clone()); + } + } + + // Add decode URLs if in PD mode + if let Some(ref decode_urls) = decode_urls { + all_urls.extend(decode_urls.clone()); + } + + let connection_mode = Self::determine_connection_mode(&all_urls); + + Ok(Router { + host, + port, + worker_urls, + policy, + worker_startup_timeout_secs, + worker_startup_check_interval, + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + max_payload_size, + dp_aware, + api_key, + log_dir, + log_level, + service_discovery, + selector, + service_discovery_port, + service_discovery_namespace, + prefill_selector, + decode_selector, + bootstrap_port_annotation, + prometheus_port, + prometheus_host, + request_timeout_secs, + request_id_headers, + pd_disaggregation, + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + max_concurrent_requests, + cors_allowed_origins, + retry_max_retries, + retry_initial_backoff_ms, + retry_max_backoff_ms, + retry_backoff_multiplier, + retry_jitter_factor, + disable_retries, + cb_failure_threshold, + cb_success_threshold, + cb_timeout_duration_secs, + cb_window_duration_secs, + disable_circuit_breaker, + health_failure_threshold, + health_success_threshold, + health_check_timeout_secs, + health_check_interval_secs, + health_check_endpoint, + enable_igw, + queue_size, + queue_timeout_secs, + rate_limit_tokens_per_second, + connection_mode, + model_path, + tokenizer_path, + }) + } + + fn start(&self) -> PyResult<()> { + // Convert to RouterConfig and validate + let router_config = self.to_router_config().map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!("Configuration error: {}", e)) + })?; + + // Validate the configuration + router_config.validate().map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Configuration validation failed: {}", + e + )) + })?; + + // Create service discovery config if enabled + let service_discovery_config = if self.service_discovery { + Some(service_discovery::ServiceDiscoveryConfig { + enabled: true, + selector: self.selector.clone(), + check_interval: std::time::Duration::from_secs(60), + port: self.service_discovery_port, + namespace: self.service_discovery_namespace.clone(), + pd_mode: self.pd_disaggregation, + prefill_selector: self.prefill_selector.clone(), + decode_selector: self.decode_selector.clone(), + bootstrap_port_annotation: self.bootstrap_port_annotation.clone(), + }) + } else { + None + }; + + // Create Prometheus config if enabled + let prometheus_config = Some(PrometheusConfig { + port: self.prometheus_port.unwrap_or(29000), + host: self + .prometheus_host + .clone() + .unwrap_or_else(|| "127.0.0.1".to_string()), + }); + + // Use tokio runtime instead of actix-web System for better compatibility + let runtime = tokio::runtime::Runtime::new() + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + + // Block on the async startup function + runtime.block_on(async move { + server::startup(server::ServerConfig { + host: self.host.clone(), + port: self.port, + router_config, + max_payload_size: self.max_payload_size, + log_dir: self.log_dir.clone(), + log_level: self.log_level.clone(), + service_discovery_config, + prometheus_config, + request_timeout_secs: self.request_timeout_secs, + request_id_headers: self.request_id_headers.clone(), + }) + .await + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + }) + } +} + +#[pymodule] +fn sglang_router_rs(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + Ok(()) +} diff --git a/sgl-router/src/logging.rs b/sgl-router/src/logging.rs new file mode 100644 index 000000000..5c5b63e0e --- /dev/null +++ b/sgl-router/src/logging.rs @@ -0,0 +1,163 @@ +use std::path::PathBuf; +use tracing::Level; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_appender::rolling::{RollingFileAppender, Rotation}; +use tracing_log::LogTracer; +use tracing_subscriber::fmt::time::ChronoUtc; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{EnvFilter, Layer}; + +/// Configuration for the logging system +#[derive(Debug, Clone)] +pub struct LoggingConfig { + /// Log level for the application (default: INFO) + pub level: Level, + /// Whether to use json format for logs (default: false) + pub json_format: bool, + /// Path to store log files. If None, logs will only go to stdout/stderr + pub log_dir: Option, + /// Whether to colorize logs when output is a terminal (default: true) + pub colorize: bool, + /// Log file name to use if log_dir is specified (default: "sgl-router") + pub log_file_name: String, + /// Custom log targets to filter (default: "sglang_router_rs") + pub log_targets: Option>, +} + +impl Default for LoggingConfig { + fn default() -> Self { + Self { + level: Level::INFO, + json_format: false, + log_dir: None, + colorize: true, + log_file_name: "sgl-router".to_string(), + log_targets: Some(vec!["sglang_router_rs".to_string()]), + } + } +} + +/// Guard that keeps the file appender worker thread alive +/// +/// This must be kept in scope for the duration of the program +/// to ensure logs are properly written to files +#[allow(dead_code)] +pub struct LogGuard { + _file_guard: Option, +} + +/// Initialize the logging system with the given configuration +/// +/// # Arguments +/// * `config` - Configuration for the logging system +/// +/// # Returns +/// A LogGuard that must be kept alive for the duration of the program +/// +/// # Panics +/// Will not panic, as initialization errors are handled gracefully +pub fn init_logging(config: LoggingConfig) -> LogGuard { + // Forward logs to tracing - ignore errors to allow for multiple initialization + let _ = LogTracer::init(); + + // Convert log level to filter string + let level_filter = match config.level { + Level::TRACE => "trace", + Level::DEBUG => "debug", + Level::INFO => "info", + Level::WARN => "warn", + Level::ERROR => "error", + }; + + // Create env filter + let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| { + // Format: =,=,... + let filter_string = if let Some(targets) = &config.log_targets { + targets + .iter() + .enumerate() + .map(|(i, target)| { + if i > 0 { + format!(",{}={}", target, level_filter) + } else { + format!("{}={}", target, level_filter) + } + }) + .collect::() + } else { + format!("sglang_router_rs={}", level_filter) + }; + + EnvFilter::new(filter_string) + }); + + // Setup stdout/stderr layer + let mut layers = Vec::new(); + + // Standard timestamp format: YYYY-MM-DD HH:MM:SS + let time_format = "%Y-%m-%d %H:%M:%S".to_string(); + + // Configure the console stdout layer + let stdout_layer = tracing_subscriber::fmt::layer() + .with_ansi(config.colorize) + .with_file(true) + .with_line_number(true) + .with_timer(ChronoUtc::new(time_format.clone())); + + let stdout_layer = if config.json_format { + stdout_layer.json().flatten_event(true).boxed() + } else { + stdout_layer.boxed() + }; + + layers.push(stdout_layer); + + // Create a file appender if log_dir is specified + let mut file_guard = None; + + if let Some(log_dir) = &config.log_dir { + let file_name = config.log_file_name.clone(); + let log_dir = PathBuf::from(log_dir); + + // Create log directory if it doesn't exist + if !log_dir.exists() { + if let Err(e) = std::fs::create_dir_all(&log_dir) { + eprintln!("Failed to create log directory: {}", e); + return LogGuard { _file_guard: None }; + } + } + + let file_appender = RollingFileAppender::new(Rotation::DAILY, log_dir, file_name); + + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + file_guard = Some(guard); + + let file_layer = tracing_subscriber::fmt::layer() + .with_ansi(false) // Never use ANSI colors in log files + .with_file(true) + .with_line_number(true) + .with_timer(ChronoUtc::new(time_format)) + .with_writer(non_blocking); + + let file_layer = if config.json_format { + file_layer.json().flatten_event(true).boxed() + } else { + file_layer.boxed() + }; + + layers.push(file_layer); + } + + // Initialize the subscriber with all layers + // Use try_init to handle errors gracefully in case another subscriber is already set + let _ = tracing_subscriber::registry() + .with(env_filter) + .with(layers) + .try_init(); + + // Return the guard to keep the file appender worker thread alive + LogGuard { + _file_guard: file_guard, + } +} diff --git a/sgl-router/src/main.rs b/sgl-router/src/main.rs new file mode 100644 index 000000000..8ec24a722 --- /dev/null +++ b/sgl-router/src/main.rs @@ -0,0 +1,636 @@ +use clap::{ArgAction, Parser, ValueEnum}; +use sglang_router_rs::config::{ + CircuitBreakerConfig, ConfigError, ConfigResult, ConnectionMode, DiscoveryConfig, + HealthCheckConfig, MetricsConfig, PolicyConfig, RetryConfig, RouterConfig, RoutingMode, +}; +use sglang_router_rs::metrics::PrometheusConfig; +use sglang_router_rs::server::{self, ServerConfig}; +use sglang_router_rs::service_discovery::ServiceDiscoveryConfig; +use std::collections::HashMap; + +// Helper function to parse prefill arguments from command line +fn parse_prefill_args() -> Vec<(String, Option)> { + let args: Vec = std::env::args().collect(); + let mut prefill_entries = Vec::new(); + let mut i = 0; + + while i < args.len() { + if args[i] == "--prefill" && i + 1 < args.len() { + let url = args[i + 1].clone(); + let bootstrap_port = if i + 2 < args.len() && !args[i + 2].starts_with("--") { + // Check if next arg is a port number + if let Ok(port) = args[i + 2].parse::() { + i += 1; // Skip the port argument + Some(port) + } else if args[i + 2].to_lowercase() == "none" { + i += 1; // Skip the "none" argument + None + } else { + None + } + } else { + None + }; + prefill_entries.push((url, bootstrap_port)); + i += 2; // Skip --prefill and URL + } else { + i += 1; + } + } + + prefill_entries +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq, ValueEnum)] +pub enum Backend { + #[value(name = "sglang")] + Sglang, + #[value(name = "vllm")] + Vllm, + #[value(name = "trtllm")] + Trtllm, + #[value(name = "openai")] + Openai, + #[value(name = "anthropic")] + Anthropic, +} + +impl std::fmt::Display for Backend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + Backend::Sglang => "sglang", + Backend::Vllm => "vllm", + Backend::Trtllm => "trtllm", + Backend::Openai => "openai", + Backend::Anthropic => "anthropic", + }; + write!(f, "{}", s) + } +} + +#[derive(Parser, Debug)] +#[command(name = "sglang-router")] +#[command(about = "SGLang Router - High-performance request distribution across worker nodes")] +#[command(long_about = r#" +SGLang Router - High-performance request distribution across worker nodes + +Usage: +This launcher enables starting a router with individual worker instances. It is useful for +multi-node setups or when you want to start workers and router separately. + +Examples: + # Regular mode + sglang-router --worker-urls http://worker1:8000 http://worker2:8000 + + # PD disaggregated mode with same policy for both + sglang-router --pd-disaggregation \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 9002 \ + --decode http://127.0.0.3:30003 \ + --decode http://127.0.0.4:30004 \ + --policy cache_aware + + # PD mode with different policies for prefill and decode + sglang-router --pd-disaggregation \ + --prefill http://127.0.0.1:30001 9001 \ + --prefill http://127.0.0.2:30002 \ + --decode http://127.0.0.3:30003 \ + --decode http://127.0.0.4:30004 \ + --prefill-policy cache_aware --decode-policy power_of_two + +"#)] +struct CliArgs { + /// Host address to bind the router server + #[arg(long, default_value = "127.0.0.1")] + host: String, + + /// Port number to bind the router server + #[arg(long, default_value_t = 30000)] + port: u16, + + /// List of worker URLs (e.g., http://worker1:8000 http://worker2:8000) + #[arg(long, num_args = 0..)] + worker_urls: Vec, + + /// Load balancing policy to use + #[arg(long, default_value = "cache_aware", value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + policy: String, + + /// Enable PD (Prefill-Decode) disaggregated mode + #[arg(long, default_value_t = false)] + pd_disaggregation: bool, + + /// Decode server URL (can be specified multiple times) + #[arg(long, action = ArgAction::Append)] + decode: Vec, + + /// Specific policy for prefill nodes in PD mode + #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + prefill_policy: Option, + + /// Specific policy for decode nodes in PD mode + #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two"])] + decode_policy: Option, + + /// Timeout in seconds for worker startup + #[arg(long, default_value_t = 600)] + worker_startup_timeout_secs: u64, + + /// Interval in seconds between checks for worker startup + #[arg(long, default_value_t = 30)] + worker_startup_check_interval: u64, + + /// Cache threshold (0.0-1.0) for cache-aware routing + #[arg(long, default_value_t = 0.3)] + cache_threshold: f32, + + /// Absolute threshold for load balancing + #[arg(long, default_value_t = 64)] + balance_abs_threshold: usize, + + /// Relative threshold for load balancing + #[arg(long, default_value_t = 1.5)] + balance_rel_threshold: f32, + + /// Interval in seconds between cache eviction operations + #[arg(long, default_value_t = 120)] + eviction_interval: u64, + + /// Maximum size of the approximation tree for cache-aware routing + #[arg(long, default_value_t = 67108864)] // 2^26 + max_tree_size: usize, + + /// Maximum payload size in bytes + #[arg(long, default_value_t = 536870912)] // 512MB + max_payload_size: usize, + + /// Enable data parallelism aware schedule + #[arg(long, default_value_t = false)] + dp_aware: bool, + + /// API key for worker authorization + #[arg(long)] + api_key: Option, + + /// Backend to route requests to (sglang, vllm, trtllm, openai, anthropic) + #[arg(long, value_enum, default_value_t = Backend::Sglang, alias = "runtime")] + backend: Backend, + + /// Directory to store log files + #[arg(long)] + log_dir: Option, + + /// Set the logging level + #[arg(long, default_value = "info", value_parser = ["debug", "info", "warn", "error"])] + log_level: String, + + /// Enable Kubernetes service discovery + #[arg(long, default_value_t = false)] + service_discovery: bool, + + /// Label selector for Kubernetes service discovery (format: key1=value1 key2=value2) + #[arg(long, num_args = 0..)] + selector: Vec, + + /// Port to use for discovered worker pods + #[arg(long, default_value_t = 80)] + service_discovery_port: u16, + + /// Kubernetes namespace to watch for pods + #[arg(long)] + service_discovery_namespace: Option, + + /// Label selector for prefill server pods in PD mode + #[arg(long, num_args = 0..)] + prefill_selector: Vec, + + /// Label selector for decode server pods in PD mode + #[arg(long, num_args = 0..)] + decode_selector: Vec, + + /// Port to expose Prometheus metrics + #[arg(long, default_value_t = 29000)] + prometheus_port: u16, + + /// Host address to bind the Prometheus metrics server + #[arg(long, default_value = "127.0.0.1")] + prometheus_host: String, + + /// Custom HTTP headers to check for request IDs + #[arg(long, num_args = 0..)] + request_id_headers: Vec, + + /// Request timeout in seconds + #[arg(long, default_value_t = 1800)] + request_timeout_secs: u64, + + /// Maximum number of concurrent requests allowed + #[arg(long, default_value_t = 256)] + max_concurrent_requests: usize, + + /// CORS allowed origins + #[arg(long, num_args = 0..)] + cors_allowed_origins: Vec, + + // Retry configuration + /// Maximum number of retries + #[arg(long, default_value_t = 5)] + retry_max_retries: u32, + + /// Initial backoff in milliseconds for retries + #[arg(long, default_value_t = 50)] + retry_initial_backoff_ms: u64, + + /// Maximum backoff in milliseconds for retries + #[arg(long, default_value_t = 30000)] + retry_max_backoff_ms: u64, + + /// Backoff multiplier for exponential backoff + #[arg(long, default_value_t = 1.5)] + retry_backoff_multiplier: f32, + + /// Jitter factor for retry backoff + #[arg(long, default_value_t = 0.2)] + retry_jitter_factor: f32, + + /// Disable retries + #[arg(long, default_value_t = false)] + disable_retries: bool, + + // Circuit breaker configuration + /// Number of failures before circuit breaker opens + #[arg(long, default_value_t = 10)] + cb_failure_threshold: u32, + + /// Number of successes before circuit breaker closes + #[arg(long, default_value_t = 3)] + cb_success_threshold: u32, + + /// Timeout duration in seconds for circuit breaker + #[arg(long, default_value_t = 60)] + cb_timeout_duration_secs: u64, + + /// Window duration in seconds for circuit breaker + #[arg(long, default_value_t = 120)] + cb_window_duration_secs: u64, + + /// Disable circuit breaker + #[arg(long, default_value_t = false)] + disable_circuit_breaker: bool, + + // Health check configuration + /// Number of consecutive health check failures before marking worker unhealthy + #[arg(long, default_value_t = 3)] + health_failure_threshold: u32, + + /// Number of consecutive health check successes before marking worker healthy + #[arg(long, default_value_t = 2)] + health_success_threshold: u32, + + /// Timeout in seconds for health check requests + #[arg(long, default_value_t = 5)] + health_check_timeout_secs: u64, + + /// Interval in seconds between runtime health checks + #[arg(long, default_value_t = 60)] + health_check_interval_secs: u64, + + /// Health check endpoint path + #[arg(long, default_value = "/health")] + health_check_endpoint: String, + + // IGW (Inference Gateway) configuration + /// Enable Inference Gateway mode + #[arg(long, default_value_t = false)] + enable_igw: bool, + + // Tokenizer configuration + /// Model path for loading tokenizer (HuggingFace model ID or local path) + #[arg(long)] + model_path: Option, + + /// Explicit tokenizer path (overrides model_path tokenizer if provided) + #[arg(long)] + tokenizer_path: Option, +} + +impl CliArgs { + /// Determine connection mode from worker URLs + fn determine_connection_mode(worker_urls: &[String]) -> ConnectionMode { + // Only consider it gRPC if explicitly specified with grpc:// or grpcs:// scheme + for url in worker_urls { + if url.starts_with("grpc://") || url.starts_with("grpcs://") { + return ConnectionMode::Grpc; + } + } + // Default to HTTP for all other cases (including http://, https://, or no scheme) + ConnectionMode::Http + } + + /// Parse selector strings into HashMap + fn parse_selector(selector_list: &[String]) -> HashMap { + let mut map = HashMap::new(); + for item in selector_list { + if let Some(eq_pos) = item.find('=') { + let key = item[..eq_pos].to_string(); + let value = item[eq_pos + 1..].to_string(); + map.insert(key, value); + } + } + map + } + + /// Convert policy string to PolicyConfig + fn parse_policy(&self, policy_str: &str) -> PolicyConfig { + match policy_str { + "random" => PolicyConfig::Random, + "round_robin" => PolicyConfig::RoundRobin, + "cache_aware" => PolicyConfig::CacheAware { + cache_threshold: self.cache_threshold, + balance_abs_threshold: self.balance_abs_threshold, + balance_rel_threshold: self.balance_rel_threshold, + eviction_interval_secs: self.eviction_interval, + max_tree_size: self.max_tree_size, + }, + "power_of_two" => PolicyConfig::PowerOfTwo { + load_check_interval_secs: 5, // Default value + }, + _ => PolicyConfig::RoundRobin, // Fallback + } + } + + /// Convert CLI arguments to RouterConfig + fn to_router_config( + &self, + prefill_urls: Vec<(String, Option)>, + ) -> ConfigResult { + // Determine routing mode + let mode = if self.enable_igw { + // IGW mode - routing mode is not used in IGW, but we need to provide a placeholder + RoutingMode::Regular { + worker_urls: vec![], + } + } else if matches!(self.backend, Backend::Openai) { + // OpenAI backend mode - use worker_urls as base(s) + RoutingMode::OpenAI { + worker_urls: self.worker_urls.clone(), + } + } else if self.pd_disaggregation { + let decode_urls = self.decode.clone(); + + // Validate PD configuration if not using service discovery + if !self.service_discovery && (prefill_urls.is_empty() || decode_urls.is_empty()) { + return Err(ConfigError::ValidationFailed { + reason: "PD disaggregation mode requires --prefill and --decode URLs when not using service discovery".to_string(), + }); + } + + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy: self.prefill_policy.as_ref().map(|p| self.parse_policy(p)), + decode_policy: self.decode_policy.as_ref().map(|p| self.parse_policy(p)), + } + } else { + // Regular mode + if !self.service_discovery && self.worker_urls.is_empty() { + return Err(ConfigError::ValidationFailed { + reason: "Regular mode requires --worker-urls when not using service discovery" + .to_string(), + }); + } + RoutingMode::Regular { + worker_urls: self.worker_urls.clone(), + } + }; + + // Main policy + let policy = self.parse_policy(&self.policy); + + // Service discovery configuration + let discovery = if self.service_discovery { + Some(DiscoveryConfig { + enabled: true, + namespace: self.service_discovery_namespace.clone(), + port: self.service_discovery_port, + check_interval_secs: 60, + selector: Self::parse_selector(&self.selector), + prefill_selector: Self::parse_selector(&self.prefill_selector), + decode_selector: Self::parse_selector(&self.decode_selector), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + }) + } else { + None + }; + + // Metrics configuration + let metrics = Some(MetricsConfig { + port: self.prometheus_port, + host: self.prometheus_host.clone(), + }); + + // Determine connection mode from all worker URLs + let mut all_urls = Vec::new(); + match &mode { + RoutingMode::Regular { worker_urls } => { + all_urls.extend(worker_urls.clone()); + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + .. + } => { + for (url, _) in prefill_urls { + all_urls.push(url.clone()); + } + all_urls.extend(decode_urls.clone()); + } + RoutingMode::OpenAI { .. } => { + // For connection-mode detection, skip URLs; OpenAI forces HTTP below. + } + } + let connection_mode = match &mode { + RoutingMode::OpenAI { .. } => ConnectionMode::Http, + _ => Self::determine_connection_mode(&all_urls), + }; + + // Build RouterConfig + Ok(RouterConfig { + mode, + policy, + connection_mode, + host: self.host.clone(), + port: self.port, + max_payload_size: self.max_payload_size, + request_timeout_secs: self.request_timeout_secs, + worker_startup_timeout_secs: self.worker_startup_timeout_secs, + worker_startup_check_interval_secs: self.worker_startup_check_interval, + dp_aware: self.dp_aware, + api_key: self.api_key.clone(), + discovery, + metrics, + log_dir: self.log_dir.clone(), + log_level: Some(self.log_level.clone()), + request_id_headers: if self.request_id_headers.is_empty() { + None + } else { + Some(self.request_id_headers.clone()) + }, + max_concurrent_requests: self.max_concurrent_requests, + queue_size: 100, // Default queue size + queue_timeout_secs: 60, // Default timeout + cors_allowed_origins: self.cors_allowed_origins.clone(), + retry: RetryConfig { + max_retries: self.retry_max_retries, + initial_backoff_ms: self.retry_initial_backoff_ms, + max_backoff_ms: self.retry_max_backoff_ms, + backoff_multiplier: self.retry_backoff_multiplier, + jitter_factor: self.retry_jitter_factor, + }, + circuit_breaker: CircuitBreakerConfig { + failure_threshold: self.cb_failure_threshold, + success_threshold: self.cb_success_threshold, + timeout_duration_secs: self.cb_timeout_duration_secs, + window_duration_secs: self.cb_window_duration_secs, + }, + disable_retries: self.disable_retries, + disable_circuit_breaker: self.disable_circuit_breaker, + health_check: HealthCheckConfig { + failure_threshold: self.health_failure_threshold, + success_threshold: self.health_success_threshold, + timeout_secs: self.health_check_timeout_secs, + check_interval_secs: self.health_check_interval_secs, + endpoint: self.health_check_endpoint.clone(), + }, + enable_igw: self.enable_igw, + rate_limit_tokens_per_second: None, + model_path: self.model_path.clone(), + tokenizer_path: self.tokenizer_path.clone(), + }) + } + + /// Create ServerConfig from CLI args and RouterConfig + fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig { + // Create service discovery config if enabled + let service_discovery_config = if self.service_discovery { + Some(ServiceDiscoveryConfig { + enabled: true, + selector: Self::parse_selector(&self.selector), + check_interval: std::time::Duration::from_secs(60), + port: self.service_discovery_port, + namespace: self.service_discovery_namespace.clone(), + pd_mode: self.pd_disaggregation, + prefill_selector: Self::parse_selector(&self.prefill_selector), + decode_selector: Self::parse_selector(&self.decode_selector), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + }) + } else { + None + }; + + // Create Prometheus config + let prometheus_config = Some(PrometheusConfig { + port: self.prometheus_port, + host: self.prometheus_host.clone(), + }); + + ServerConfig { + host: self.host.clone(), + port: self.port, + router_config, + max_payload_size: self.max_payload_size, + log_dir: self.log_dir.clone(), + log_level: Some(self.log_level.clone()), + service_discovery_config, + prometheus_config, + request_timeout_secs: self.request_timeout_secs, + request_id_headers: if self.request_id_headers.is_empty() { + None + } else { + Some(self.request_id_headers.clone()) + }, + } + } +} + +fn main() -> Result<(), Box> { + // Parse prefill arguments manually before clap parsing + let prefill_urls = parse_prefill_args(); + + // Filter out prefill arguments and their values before passing to clap + let mut filtered_args: Vec = Vec::new(); + let raw_args: Vec = std::env::args().collect(); + let mut i = 0; + + while i < raw_args.len() { + if raw_args[i] == "--prefill" && i + 1 < raw_args.len() { + // Skip --prefill and its URL + i += 2; + // Also skip bootstrap port if present + if i < raw_args.len() + && !raw_args[i].starts_with("--") + && (raw_args[i].parse::().is_ok() || raw_args[i].to_lowercase() == "none") + { + i += 1; + } + } else { + filtered_args.push(raw_args[i].clone()); + i += 1; + } + } + + // Parse CLI arguments with clap using filtered args + let cli_args = CliArgs::parse_from(filtered_args); + + // Print startup info + println!("SGLang Router starting..."); + println!("Host: {}:{}", cli_args.host, cli_args.port); + let mode_str = if cli_args.enable_igw { + "IGW (Inference Gateway)".to_string() + } else if matches!(cli_args.backend, Backend::Openai) { + "OpenAI Backend".to_string() + } else if cli_args.pd_disaggregation { + "PD Disaggregated".to_string() + } else { + format!("Regular ({})", cli_args.backend) + }; + println!("Mode: {}", mode_str); + + // Warn for runtimes that are parsed but not yet implemented + match cli_args.backend { + Backend::Vllm | Backend::Trtllm | Backend::Anthropic => { + println!( + "WARNING: runtime '{}' not implemented yet; falling back to regular routing. \ +Provide --worker-urls or PD flags as usual.", + cli_args.backend + ); + } + Backend::Sglang | Backend::Openai => {} + } + + if !cli_args.enable_igw { + println!("Policy: {}", cli_args.policy); + + if cli_args.pd_disaggregation && !prefill_urls.is_empty() { + println!("Prefill nodes: {:?}", prefill_urls); + println!("Decode nodes: {:?}", cli_args.decode); + } + } + + // Convert to RouterConfig + let router_config = cli_args.to_router_config(prefill_urls)?; + + // Validate configuration + router_config.validate()?; + + // Create ServerConfig + let server_config = cli_args.to_server_config(router_config); + + // Create a new runtime for the server (like Python binding does) + let runtime = tokio::runtime::Runtime::new()?; + + // Block on the async startup function + runtime.block_on(async move { server::startup(server_config).await })?; + + Ok(()) +} diff --git a/sgl-router/src/mcp/client_manager.rs b/sgl-router/src/mcp/client_manager.rs new file mode 100644 index 000000000..a2a6d7a7e --- /dev/null +++ b/sgl-router/src/mcp/client_manager.rs @@ -0,0 +1,535 @@ +use backoff::ExponentialBackoffBuilder; +use dashmap::DashMap; +use rmcp::{ + model::{ + CallToolRequestParam, GetPromptRequestParam, GetPromptResult, Prompt, + ReadResourceRequestParam, ReadResourceResult, Resource, Tool as McpTool, + }, + service::RunningService, + transport::{ + sse_client::SseClientConfig, streamable_http_client::StreamableHttpClientTransportConfig, + ConfigureCommandExt, SseClientTransport, StreamableHttpClientTransport, TokioChildProcess, + }, + RoleClient, ServiceExt, +}; +use serde::{Deserialize, Serialize}; +use std::{borrow::Cow, collections::HashMap, time::Duration}; + +use crate::mcp::{ + config::{McpConfig, McpServerConfig, McpTransport}, + error::{McpError, McpResult}, +}; + +/// Information about an available tool +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolInfo { + pub name: String, + pub description: String, + pub server: String, + pub parameters: Option, +} + +/// Information about an available prompt +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PromptInfo { + pub name: String, + pub description: Option, + pub server: String, + pub arguments: Option>, +} + +/// Information about an available resource +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResourceInfo { + pub uri: String, + pub name: String, + pub description: Option, + pub mime_type: Option, + pub server: String, +} + +/// Manages MCP client connections and tool execution +pub struct McpClientManager { + /// Map of server_name -> MCP client + clients: HashMap>, + /// Map of tool_name -> (server_name, tool_definition) + tools: DashMap, + /// Map of prompt_name -> (server_name, prompt_definition) + prompts: DashMap, + /// Map of resource_uri -> (server_name, resource_definition) + resources: DashMap, +} + +impl McpClientManager { + /// Create a new manager and connect to all configured servers + pub async fn new(config: McpConfig) -> McpResult { + let mut mgr = Self { + clients: HashMap::new(), + tools: DashMap::new(), + prompts: DashMap::new(), + resources: DashMap::new(), + }; + + for server_config in config.servers { + match Self::connect_server(&server_config).await { + Ok(client) => { + mgr.load_server_inventory(&server_config.name, &client) + .await; + mgr.clients.insert(server_config.name.clone(), client); + } + Err(e) => { + tracing::error!( + "Failed to connect to server '{}': {}", + server_config.name, + e + ); + } + } + } + + if mgr.clients.is_empty() { + return Err(McpError::ConnectionFailed( + "Failed to connect to any MCP servers".to_string(), + )); + } + Ok(mgr) + } + + /// Discover and cache tools/prompts/resources for a connected server + async fn load_server_inventory( + &self, + server_name: &str, + client: &RunningService, + ) { + // Tools + match client.peer().list_all_tools().await { + Ok(ts) => { + tracing::info!("Discovered {} tools from '{}'", ts.len(), server_name); + for t in ts { + if self.tools.contains_key(t.name.as_ref()) { + tracing::warn!( + "Tool '{}' from server '{}' is overwriting an existing tool.", + &t.name, + server_name + ); + } + self.tools + .insert(t.name.to_string(), (server_name.to_string(), t)); + } + } + Err(e) => tracing::warn!("Failed to list tools from '{}': {}", server_name, e), + } + + // Prompts + match client.peer().list_all_prompts().await { + Ok(ps) => { + tracing::info!("Discovered {} prompts from '{}'", ps.len(), server_name); + for p in ps { + if self.prompts.contains_key(&p.name) { + tracing::warn!( + "Prompt '{}' from server '{}' is overwriting an existing prompt.", + &p.name, + server_name + ); + } + self.prompts + .insert(p.name.clone(), (server_name.to_string(), p)); + } + } + Err(e) => tracing::debug!("No prompts or failed to list on '{}': {}", server_name, e), + } + + // Resources + match client.peer().list_all_resources().await { + Ok(rs) => { + tracing::info!("Discovered {} resources from '{}'", rs.len(), server_name); + for r in rs { + if self.resources.contains_key(&r.uri) { + tracing::warn!( + "Resource '{}' from server '{}' is overwriting an existing resource.", + &r.uri, + server_name + ); + } + self.resources + .insert(r.uri.clone(), (server_name.to_string(), r)); + } + } + Err(e) => tracing::debug!("No resources or failed to list on '{}': {}", server_name, e), + } + } + + /// Connect to a single MCP server with retry logic for remote transports + async fn connect_server(config: &McpServerConfig) -> McpResult> { + let needs_retry = matches!( + &config.transport, + McpTransport::Sse { .. } | McpTransport::Streamable { .. } + ); + if needs_retry { + Self::connect_server_with_retry(config).await + } else { + Self::connect_server_impl(config).await + } + } + + /// Connect with exponential backoff retry for remote servers + async fn connect_server_with_retry( + config: &McpServerConfig, + ) -> McpResult> { + let backoff = ExponentialBackoffBuilder::new() + .with_initial_interval(Duration::from_secs(1)) + .with_max_interval(Duration::from_secs(30)) + .with_max_elapsed_time(Some(Duration::from_secs(120))) + .build(); + + backoff::future::retry(backoff, || async { + match Self::connect_server_impl(config).await { + Ok(client) => Ok(client), + Err(e) => { + tracing::warn!("Failed to connect to '{}', retrying: {}", config.name, e); + Err(backoff::Error::transient(e)) + } + } + }) + .await + } + + /// Internal implementation of server connection + async fn connect_server_impl( + config: &McpServerConfig, + ) -> McpResult> { + tracing::info!( + "Connecting to MCP server '{}' via {:?}", + config.name, + config.transport + ); + + match &config.transport { + McpTransport::Stdio { + command, + args, + envs, + } => { + let transport = TokioChildProcess::new( + tokio::process::Command::new(command).configure(|cmd| { + cmd.args(args) + .envs(envs.iter()) + .stderr(std::process::Stdio::inherit()); + }), + ) + .map_err(|e| McpError::Transport(format!("create stdio transport: {}", e)))?; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize stdio client: {}", e)) + })?; + + tracing::info!("Connected to stdio server '{}'", config.name); + Ok(client) + } + + McpTransport::Sse { url, token } => { + let transport = if let Some(tok) = token { + let client = reqwest::Client::builder() + .default_headers({ + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + format!("Bearer {}", tok).parse().map_err(|e| { + McpError::Transport(format!("auth token: {}", e)) + })?, + ); + headers + }) + .build() + .map_err(|e| McpError::Transport(format!("build HTTP client: {}", e)))?; + + let cfg = SseClientConfig { + sse_endpoint: url.clone().into(), + ..Default::default() + }; + + SseClientTransport::start_with_client(client, cfg) + .await + .map_err(|e| McpError::Transport(format!("create SSE transport: {}", e)))? + } else { + SseClientTransport::start(url.as_str()) + .await + .map_err(|e| McpError::Transport(format!("create SSE transport: {}", e)))? + }; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize SSE client: {}", e)) + })?; + + tracing::info!("Connected to SSE server '{}' at {}", config.name, url); + Ok(client) + } + + McpTransport::Streamable { url, token } => { + let transport = if let Some(tok) = token { + let mut cfg = StreamableHttpClientTransportConfig::with_uri(url.as_str()); + cfg.auth_header = Some(format!("Bearer {}", tok)); + StreamableHttpClientTransport::from_config(cfg) + } else { + StreamableHttpClientTransport::from_uri(url.as_str()) + }; + + let client = ().serve(transport).await.map_err(|e| { + McpError::ConnectionFailed(format!("initialize streamable client: {}", e)) + })?; + + tracing::info!( + "Connected to streamable HTTP server '{}' at {}", + config.name, + url + ); + Ok(client) + } + } + } + + // ===== Helpers ===== + + fn client_for(&self, server_name: &str) -> McpResult<&RunningService> { + self.clients + .get(server_name) + .ok_or_else(|| McpError::ServerNotFound(server_name.to_string())) + } + + fn tool_entry(&self, name: &str) -> McpResult<(String, McpTool)> { + self.tools + .get(name) + .map(|e| e.value().clone()) + .ok_or_else(|| McpError::ToolNotFound(name.to_string())) + } + + fn prompt_entry(&self, name: &str) -> McpResult<(String, Prompt)> { + self.prompts + .get(name) + .map(|e| e.value().clone()) + .ok_or_else(|| McpError::PromptNotFound(name.to_string())) + } + + fn resource_entry(&self, uri: &str) -> McpResult<(String, Resource)> { + self.resources + .get(uri) + .map(|e| e.value().clone()) + .ok_or_else(|| McpError::ResourceNotFound(uri.to_string())) + } + + // ===== Tool Methods ===== + + /// Call a tool by name + pub async fn call_tool( + &self, + tool_name: &str, + arguments: Option>, + ) -> McpResult { + let (server_name, _tool) = self.tool_entry(tool_name)?; + let client = self.client_for(&server_name)?; + + tracing::debug!("Calling tool '{}' on '{}'", tool_name, server_name); + + client + .peer() + .call_tool(CallToolRequestParam { + name: Cow::Owned(tool_name.to_string()), + arguments, + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Tool call failed: {}", e))) + } + + /// Get all available tools + pub fn list_tools(&self) -> Vec { + self.tools + .iter() + .map(|entry| { + let tool_name = entry.key().clone(); + let (server_name, tool) = entry.value(); + ToolInfo { + name: tool_name, + description: tool.description.as_deref().unwrap_or_default().to_string(), + server: server_name.clone(), + parameters: Some(serde_json::Value::Object((*tool.input_schema).clone())), + } + }) + .collect() + } + + /// Get a specific tool by name + pub fn get_tool(&self, name: &str) -> Option { + self.tools.get(name).map(|entry| { + let (server_name, tool) = entry.value(); + ToolInfo { + name: name.to_string(), + description: tool.description.as_deref().unwrap_or_default().to_string(), + server: server_name.clone(), + parameters: Some(serde_json::Value::Object((*tool.input_schema).clone())), + } + }) + } + + /// Check if a tool exists + pub fn has_tool(&self, name: &str) -> bool { + self.tools.contains_key(name) + } + + /// Get list of connected servers + pub fn list_servers(&self) -> Vec { + self.clients.keys().cloned().collect() + } + + // ===== Prompt Methods ===== + + /// Get a prompt by name with arguments + pub async fn get_prompt( + &self, + prompt_name: &str, + arguments: Option>, + ) -> McpResult { + let (server_name, _prompt) = self.prompt_entry(prompt_name)?; + let client = self.client_for(&server_name)?; + + tracing::debug!("Getting prompt '{}' from '{}'", prompt_name, server_name); + + client + .peer() + .get_prompt(GetPromptRequestParam { + name: prompt_name.to_string(), + arguments, + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to get prompt: {}", e))) + } + + /// List all available prompts + pub fn list_prompts(&self) -> Vec { + self.prompts + .iter() + .map(|entry| { + let name = entry.key().clone(); + let (server_name, prompt) = entry.value(); + PromptInfo { + name, + description: prompt.description.clone(), + server: server_name.clone(), + arguments: prompt + .arguments + .clone() + .map(|args| args.into_iter().map(|arg| serde_json::json!(arg)).collect()), + } + }) + .collect() + } + + /// Get a specific prompt info by name + pub fn get_prompt_info(&self, name: &str) -> Option { + self.prompts.get(name).map(|entry| { + let (server_name, prompt) = entry.value(); + PromptInfo { + name: name.to_string(), + description: prompt.description.clone(), + server: server_name.clone(), + arguments: prompt + .arguments + .clone() + .map(|args| args.into_iter().map(|arg| serde_json::json!(arg)).collect()), + } + }) + } + + // ===== Resource Methods ===== + + /// Read a resource by URI + pub async fn read_resource(&self, uri: &str) -> McpResult { + let (server_name, _resource) = self.resource_entry(uri)?; + let client = self.client_for(&server_name)?; + + tracing::debug!("Reading resource '{}' from '{}'", uri, server_name); + + client + .peer() + .read_resource(ReadResourceRequestParam { + uri: uri.to_string(), + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to read resource: {}", e))) + } + + /// List all available resources + pub fn list_resources(&self) -> Vec { + self.resources + .iter() + .map(|entry| { + let uri = entry.key().clone(); + let (server_name, resource) = entry.value(); + ResourceInfo { + uri, + name: resource.name.clone(), + description: resource.description.clone(), + mime_type: resource.mime_type.clone(), + server: server_name.clone(), + } + }) + .collect() + } + + /// Get a specific resource info by URI + pub fn get_resource_info(&self, uri: &str) -> Option { + self.resources.get(uri).map(|entry| { + let (server_name, resource) = entry.value(); + ResourceInfo { + uri: uri.to_string(), + name: resource.name.clone(), + description: resource.description.clone(), + mime_type: resource.mime_type.clone(), + server: server_name.clone(), + } + }) + } + + /// Subscribe to resource changes + pub async fn subscribe_resource(&self, uri: &str) -> McpResult<()> { + let (server_name, _resource) = self.resource_entry(uri)?; + let client = self.client_for(&server_name)?; + + tracing::debug!("Subscribing to '{}' on '{}'", uri, server_name); + + client + .peer() + .subscribe(rmcp::model::SubscribeRequestParam { + uri: uri.to_string(), + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to subscribe: {}", e))) + } + + /// Unsubscribe from resource changes + pub async fn unsubscribe_resource(&self, uri: &str) -> McpResult<()> { + let (server_name, _resource) = self.resource_entry(uri)?; + let client = self.client_for(&server_name)?; + + tracing::debug!("Unsubscribing from '{}' on '{}'", uri, server_name); + + client + .peer() + .unsubscribe(rmcp::model::UnsubscribeRequestParam { + uri: uri.to_string(), + }) + .await + .map_err(|e| McpError::ToolExecution(format!("Failed to unsubscribe: {}", e))) + } + + /// Disconnect from all servers (for cleanup) + pub async fn shutdown(&mut self) { + for (name, client) in self.clients.drain() { + if let Err(e) = client.cancel().await { + tracing::warn!("Error disconnecting from '{}': {}", name, e); + } + } + self.tools.clear(); + self.prompts.clear(); + self.resources.clear(); + } +} diff --git a/sgl-router/src/mcp/config.rs b/sgl-router/src/mcp/config.rs new file mode 100644 index 000000000..1adf6a7d7 --- /dev/null +++ b/sgl-router/src/mcp/config.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpConfig { + pub servers: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpServerConfig { + pub name: String, + #[serde(flatten)] + pub transport: McpTransport, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "protocol", rename_all = "lowercase")] +pub enum McpTransport { + Stdio { + command: String, + #[serde(default)] + args: Vec, + #[serde(default)] + envs: HashMap, + }, + Sse { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + token: Option, + }, + Streamable { + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + token: Option, + }, +} + +impl McpConfig { + /// Load configuration from a YAML file + pub async fn from_file(path: &str) -> Result> { + let content = tokio::fs::read_to_string(path).await?; + let config: Self = serde_yaml::from_str(&content)?; + Ok(config) + } + + /// Load configuration from environment variables (optional) + pub fn from_env() -> Option { + // This could be expanded to read from env vars + // For now, return None to indicate env config not implemented + None + } +} diff --git a/sgl-router/src/mcp/error.rs b/sgl-router/src/mcp/error.rs new file mode 100644 index 000000000..03b8b4cd1 --- /dev/null +++ b/sgl-router/src/mcp/error.rs @@ -0,0 +1,42 @@ +use thiserror::Error; + +pub type McpResult = Result; + +#[derive(Debug, Error)] +pub enum McpError { + #[error("Server not found: {0}")] + ServerNotFound(String), + + #[error("Tool not found: {0}")] + ToolNotFound(String), + + #[error("Transport error: {0}")] + Transport(String), + + #[error("Tool execution failed: {0}")] + ToolExecution(String), + + #[error("Connection failed: {0}")] + ConnectionFailed(String), + + #[error("Configuration error: {0}")] + Config(String), + + #[error("Authentication error: {0}")] + Auth(String), + + #[error("Resource not found: {0}")] + ResourceNotFound(String), + + #[error("Prompt not found: {0}")] + PromptNotFound(String), + + #[error(transparent)] + Sdk(#[from] Box), + + #[error(transparent)] + Io(#[from] std::io::Error), + + #[error(transparent)] + Http(#[from] reqwest::Error), +} diff --git a/sgl-router/src/mcp/mod.rs b/sgl-router/src/mcp/mod.rs new file mode 100644 index 000000000..6cebc4c7d --- /dev/null +++ b/sgl-router/src/mcp/mod.rs @@ -0,0 +1,18 @@ +// MCP Client for SGLang Router +// +// This module provides a complete MCP (Model Context Protocol) client implementation +// supporting multiple transport types (stdio, SSE, HTTP) and all MCP features: +// - Tools: Discovery and execution +// - Prompts: Reusable templates for LLM interactions +// - Resources: File/data access with subscription support +// - OAuth: Secure authentication for remote servers + +pub mod client_manager; +pub mod config; +pub mod error; +pub mod oauth; + +// Re-export the main types for convenience +pub use client_manager::{McpClientManager, PromptInfo, ResourceInfo, ToolInfo}; +pub use config::{McpConfig, McpServerConfig, McpTransport}; +pub use error::{McpError, McpResult}; diff --git a/sgl-router/src/mcp/oauth.rs b/sgl-router/src/mcp/oauth.rs new file mode 100644 index 000000000..3d13ea2be --- /dev/null +++ b/sgl-router/src/mcp/oauth.rs @@ -0,0 +1,191 @@ +// OAuth authentication support for MCP servers + +use axum::{ + extract::{Query, State}, + response::Html, + routing::get, + Router, +}; +use rmcp::transport::auth::OAuthState; +use serde::Deserialize; +use std::{net::SocketAddr, sync::Arc}; +use tokio::sync::{oneshot, Mutex}; + +use crate::mcp::error::{McpError, McpResult}; + +/// OAuth callback parameters +#[derive(Debug, Deserialize)] +struct CallbackParams { + code: String, + #[allow(dead_code)] + state: Option, +} + +/// State for the callback server +#[derive(Clone)] +struct CallbackState { + code_receiver: Arc>>>, +} + +/// HTML page returned after successful OAuth callback +const CALLBACK_HTML: &str = r#" + + + + OAuth Success + + + +

+
✓
+

Authentication Successful!

+

You can now close this window and return to your application.

+
+ + +"#; + +/// OAuth authentication helper for MCP servers +pub struct OAuthHelper { + server_url: String, + redirect_uri: String, + callback_port: u16, +} + +impl OAuthHelper { + /// Create a new OAuth helper + pub fn new(server_url: String, redirect_uri: String, callback_port: u16) -> Self { + Self { + server_url, + redirect_uri, + callback_port, + } + } + + /// Perform OAuth authentication flow + pub async fn authenticate( + &self, + scopes: &[&str], + ) -> McpResult { + // Initialize OAuth state machine + let mut oauth_state = OAuthState::new(&self.server_url, None) + .await + .map_err(|e| McpError::Auth(format!("Failed to initialize OAuth: {}", e)))?; + + oauth_state + .start_authorization(scopes, &self.redirect_uri) + .await + .map_err(|e| McpError::Auth(format!("Failed to start authorization: {}", e)))?; + + // Get authorization URL + let auth_url = oauth_state + .get_authorization_url() + .await + .map_err(|e| McpError::Auth(format!("Failed to get authorization URL: {}", e)))?; + + tracing::info!("OAuth authorization URL: {}", auth_url); + + // Start callback server and wait for code + let auth_code = self.start_callback_server().await?; + + // Exchange code for token + oauth_state + .handle_callback(&auth_code) + .await + .map_err(|e| McpError::Auth(format!("Failed to handle OAuth callback: {}", e)))?; + + // Get authorization manager + oauth_state + .into_authorization_manager() + .ok_or_else(|| McpError::Auth("Failed to get authorization manager".to_string())) + } + + /// Start a local HTTP server to receive the OAuth callback + async fn start_callback_server(&self) -> McpResult { + let (code_sender, code_receiver) = oneshot::channel::(); + + let state = CallbackState { + code_receiver: Arc::new(Mutex::new(Some(code_sender))), + }; + + // Create router for callback + let app = Router::new() + .route("/callback", get(Self::callback_handler)) + .with_state(state); + + let addr = SocketAddr::from(([127, 0, 0, 1], self.callback_port)); + + // Start server in background + let listener = tokio::net::TcpListener::bind(addr).await.map_err(|e| { + McpError::Auth(format!( + "Failed to bind to callback port {}: {}", + self.callback_port, e + )) + })?; + + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + + tracing::info!( + "OAuth callback server started on port {}", + self.callback_port + ); + + // Wait for authorization code + code_receiver + .await + .map_err(|_| McpError::Auth("Failed to receive authorization code".to_string())) + } + + /// Handle OAuth callback + async fn callback_handler( + Query(params): Query, + State(state): State, + ) -> Html { + tracing::debug!("Received OAuth callback with code"); + + // Send code to waiting task + if let Some(sender) = state.code_receiver.lock().await.take() { + let _ = sender.send(params.code); + } + + Html(CALLBACK_HTML.to_string()) + } +} + +/// Create an OAuth-authenticated client +pub async fn create_oauth_client( + server_url: String, + _sse_url: String, + redirect_uri: String, + callback_port: u16, + scopes: &[&str], +) -> McpResult> { + let helper = OAuthHelper::new(server_url, redirect_uri, callback_port); + let auth_manager = helper.authenticate(scopes).await?; + + let client = rmcp::transport::auth::AuthClient::new(reqwest::Client::default(), auth_manager); + + Ok(client) +} diff --git a/sgl-router/src/metrics.rs b/sgl-router/src/metrics.rs new file mode 100644 index 000000000..afcccb549 --- /dev/null +++ b/sgl-router/src/metrics.rs @@ -0,0 +1,1011 @@ +use metrics::{counter, describe_counter, describe_gauge, describe_histogram, gauge, histogram}; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::time::Duration; + +#[derive(Debug, Clone)] +pub struct PrometheusConfig { + pub port: u16, + pub host: String, +} + +impl Default for PrometheusConfig { + fn default() -> Self { + Self { + port: 29000, + host: "0.0.0.0".to_string(), + } + } +} + +pub fn init_metrics() { + // Request metrics + describe_counter!( + "sgl_router_requests_total", + "Total number of requests by route and method" + ); + describe_histogram!( + "sgl_router_request_duration_seconds", + "Request duration in seconds by route" + ); + describe_counter!( + "sgl_router_request_errors_total", + "Total number of request errors by route and error type" + ); + describe_counter!( + "sgl_router_retries_total", + "Total number of request retries by route" + ); + describe_histogram!( + "sgl_router_retry_backoff_duration_seconds", + "Backoff duration in seconds by attempt index" + ); + describe_counter!( + "sgl_router_retries_exhausted_total", + "Total number of requests that exhausted retries by route" + ); + + // Circuit breaker metrics + describe_gauge!( + "sgl_router_cb_state", + "Circuit breaker state per worker (0=closed, 1=open, 2=half_open)" + ); + describe_counter!( + "sgl_router_cb_state_transitions_total", + "Total number of circuit breaker state transitions by worker" + ); + describe_counter!( + "sgl_router_cb_outcomes_total", + "Total number of circuit breaker outcomes by worker and outcome type (success/failure)" + ); + + // Worker metrics + describe_gauge!( + "sgl_router_active_workers", + "Number of currently active workers" + ); + describe_gauge!( + "sgl_router_worker_health", + "Worker health status (1=healthy, 0=unhealthy)" + ); + describe_gauge!("sgl_router_worker_load", "Current load on each worker"); + describe_counter!( + "sgl_router_processed_requests_total", + "Total requests processed by each worker" + ); + + // Policy metrics + describe_counter!( + "sgl_router_policy_decisions_total", + "Total routing policy decisions by policy and worker" + ); + describe_counter!("sgl_router_cache_hits_total", "Total cache hits"); + describe_counter!("sgl_router_cache_misses_total", "Total cache misses"); + describe_gauge!( + "sgl_router_tree_size", + "Current tree size for cache-aware routing" + ); + describe_counter!( + "sgl_router_load_balancing_events_total", + "Total load balancing trigger events" + ); + describe_gauge!("sgl_router_max_load", "Maximum worker load"); + describe_gauge!("sgl_router_min_load", "Minimum worker load"); + + // PD-specific metrics + describe_counter!("sgl_router_pd_requests_total", "Total PD requests by route"); + describe_counter!( + "sgl_router_pd_prefill_requests_total", + "Total prefill requests per worker" + ); + describe_counter!( + "sgl_router_pd_decode_requests_total", + "Total decode requests per worker" + ); + describe_counter!( + "sgl_router_pd_errors_total", + "Total PD errors by error type" + ); + describe_counter!( + "sgl_router_pd_prefill_errors_total", + "Total prefill server errors" + ); + describe_counter!( + "sgl_router_pd_decode_errors_total", + "Total decode server errors" + ); + describe_counter!( + "sgl_router_pd_stream_errors_total", + "Total streaming errors per worker" + ); + describe_histogram!( + "sgl_router_pd_request_duration_seconds", + "PD request duration by route" + ); + + // Service discovery metrics + describe_counter!( + "sgl_router_discovery_updates_total", + "Total service discovery update events" + ); + describe_gauge!( + "sgl_router_discovery_workers_added", + "Number of workers added in last discovery update" + ); + describe_gauge!( + "sgl_router_discovery_workers_removed", + "Number of workers removed in last discovery update" + ); + + // Generate request specific metrics + describe_histogram!( + "sgl_router_generate_duration_seconds", + "Generate request duration" + ); + + // Running requests gauge for cache-aware policy + describe_gauge!( + "sgl_router_running_requests", + "Number of running requests per worker" + ); + + // Tokenizer metrics + describe_histogram!( + "sgl_tokenizer_encode_duration_seconds", + "Time to encode text to tokens" + ); + describe_histogram!( + "sgl_tokenizer_decode_duration_seconds", + "Time to decode tokens to text" + ); + describe_histogram!( + "sgl_tokenizer_encode_batch_duration_seconds", + "Time to encode a batch of texts" + ); + describe_counter!( + "sgl_tokenizer_encode_requests_total", + "Total number of encode requests by tokenizer type" + ); + describe_counter!( + "sgl_tokenizer_decode_requests_total", + "Total number of decode requests by tokenizer type" + ); + describe_counter!( + "sgl_tokenizer_encode_errors_total", + "Total number of encode errors by error type" + ); + describe_counter!( + "sgl_tokenizer_decode_errors_total", + "Total number of decode errors by error type" + ); + describe_histogram!( + "sgl_tokenizer_tokens_per_encode", + "Number of tokens produced per encode operation" + ); + describe_histogram!( + "sgl_tokenizer_chars_per_encode", + "Number of characters in input text per encode" + ); + describe_histogram!( + "sgl_tokenizer_tokens_per_decode", + "Number of tokens decoded per operation" + ); + describe_gauge!( + "sgl_tokenizer_vocab_size", + "Vocabulary size of the loaded tokenizer" + ); + + // Stop sequence detection metrics + describe_counter!( + "sgl_tokenizer_stop_sequences_detected_total", + "Total stop sequences detected by type" + ); + describe_counter!( + "sgl_tokenizer_partial_matches_total", + "Total partial stop sequence matches (jailed text)" + ); + describe_histogram!( + "sgl_tokenizer_stop_detection_duration_seconds", + "Time to check for stop sequences per token" + ); + + // Streaming decode metrics + describe_counter!( + "sgl_tokenizer_stream_tokens_total", + "Total tokens processed in streaming decode" + ); + describe_counter!( + "sgl_tokenizer_stream_incomplete_utf8_total", + "Total incomplete UTF-8 sequences detected" + ); + describe_histogram!( + "sgl_tokenizer_stream_step_duration_seconds", + "Time per streaming decode step" + ); + + // Factory metrics + describe_counter!( + "sgl_tokenizer_factory_loads_total", + "Total tokenizer loads by file type" + ); + describe_counter!( + "sgl_tokenizer_factory_errors_total", + "Total tokenizer loading errors by type" + ); + describe_histogram!( + "sgl_tokenizer_factory_load_duration_seconds", + "Time to load and initialize tokenizer" + ); +} + +pub fn start_prometheus(config: PrometheusConfig) { + // Initialize metric descriptions + init_metrics(); + + let duration_matcher = Matcher::Suffix(String::from("duration_seconds")); + let duration_bucket = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + let ip_addr: IpAddr = config + .host + .parse() + .unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + PrometheusBuilder::new() + .with_http_listener(socket_addr) + .upkeep_timeout(Duration::from_secs(5 * 60)) + .set_buckets_for_metric(duration_matcher, &duration_bucket) + .expect("failed to set duration bucket") + .install() + .expect("failed to install Prometheus metrics exporter"); +} + +pub struct RouterMetrics; + +pub struct TokenizerMetrics; + +impl RouterMetrics { + // Request metrics + pub fn record_request(route: &str) { + counter!("sgl_router_requests_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_request_duration(route: &str, duration: Duration) { + histogram!("sgl_router_request_duration_seconds", + "route" => route.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_request_error(route: &str, error_type: &str) { + counter!("sgl_router_request_errors_total", + "route" => route.to_string(), + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_retry(route: &str) { + counter!("sgl_router_retries_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_retry_backoff_duration(duration: Duration, attempt: u32) { + histogram!("sgl_router_retry_backoff_duration_seconds", + "attempt" => attempt.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_retries_exhausted(route: &str) { + counter!("sgl_router_retries_exhausted_total", + "route" => route.to_string() + ) + .increment(1); + } + + // Worker metrics + pub fn set_active_workers(count: usize) { + gauge!("sgl_router_active_workers").set(count as f64); + } + + pub fn set_worker_health(worker_url: &str, healthy: bool) { + gauge!("sgl_router_worker_health", + "worker" => worker_url.to_string() + ) + .set(if healthy { 1.0 } else { 0.0 }); + } + + pub fn set_worker_load(worker_url: &str, load: usize) { + gauge!("sgl_router_worker_load", + "worker" => worker_url.to_string() + ) + .set(load as f64); + } + + pub fn record_processed_request(worker_url: &str) { + counter!("sgl_router_processed_requests_total", + "worker" => worker_url.to_string() + ) + .increment(1); + } + + // Policy metrics + pub fn record_policy_decision(policy: &str, worker: &str) { + counter!("sgl_router_policy_decisions_total", + "policy" => policy.to_string(), + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_cache_hit() { + counter!("sgl_router_cache_hits_total").increment(1); + } + + pub fn record_cache_miss() { + counter!("sgl_router_cache_misses_total").increment(1); + } + + pub fn set_tree_size(worker: &str, size: usize) { + gauge!("sgl_router_tree_size", + "worker" => worker.to_string() + ) + .set(size as f64); + } + + pub fn record_load_balancing_event() { + counter!("sgl_router_load_balancing_events_total").increment(1); + } + + pub fn set_load_range(max_load: usize, min_load: usize) { + gauge!("sgl_router_max_load").set(max_load as f64); + gauge!("sgl_router_min_load").set(min_load as f64); + } + + // PD-specific metrics + pub fn record_pd_request(route: &str) { + counter!("sgl_router_pd_requests_total", + "route" => route.to_string() + ) + .increment(1); + } + + pub fn record_pd_request_duration(route: &str, duration: Duration) { + histogram!("sgl_router_pd_request_duration_seconds", + "route" => route.to_string() + ) + .record(duration.as_secs_f64()); + } + + pub fn record_pd_prefill_request(worker: &str) { + counter!("sgl_router_pd_prefill_requests_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_decode_request(worker: &str) { + counter!("sgl_router_pd_decode_requests_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_error(error_type: &str) { + counter!("sgl_router_pd_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_pd_prefill_error(worker: &str) { + counter!("sgl_router_pd_prefill_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_decode_error(worker: &str) { + counter!("sgl_router_pd_decode_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + pub fn record_pd_stream_error(worker: &str) { + counter!("sgl_router_pd_stream_errors_total", + "worker" => worker.to_string() + ) + .increment(1); + } + + // Service discovery metrics + pub fn record_discovery_update(added: usize, removed: usize) { + counter!("sgl_router_discovery_updates_total").increment(1); + gauge!("sgl_router_discovery_workers_added").set(added as f64); + gauge!("sgl_router_discovery_workers_removed").set(removed as f64); + } + + // Generate request metrics + pub fn record_generate_duration(duration: Duration) { + histogram!("sgl_router_generate_duration_seconds").record(duration.as_secs_f64()); + } + + // Running requests for cache-aware policy + pub fn set_running_requests(worker: &str, count: usize) { + gauge!("sgl_router_running_requests", + "worker" => worker.to_string() + ) + .set(count as f64); + } + + // Circuit breaker metrics + pub fn set_cb_state(worker: &str, state_code: u8) { + gauge!("sgl_router_cb_state", + "worker" => worker.to_string() + ) + .set(state_code as f64); + } + + pub fn record_cb_state_transition(worker: &str, from: &str, to: &str) { + counter!("sgl_router_cb_state_transitions_total", + "worker" => worker.to_string(), + "from" => from.to_string(), + "to" => to.to_string() + ) + .increment(1); + } + + pub fn record_cb_outcome(worker: &str, outcome: &str) { + counter!("sgl_router_cb_outcomes_total", + "worker" => worker.to_string(), + "outcome" => outcome.to_string() + ) + .increment(1); + } +} + +impl TokenizerMetrics { + // Encoding metrics + pub fn record_encode_request(tokenizer_type: &str) { + counter!("sgl_tokenizer_encode_requests_total", + "tokenizer_type" => tokenizer_type.to_string() + ) + .increment(1); + } + + pub fn record_encode_duration(duration: Duration) { + histogram!("sgl_tokenizer_encode_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_encode_error(error_type: &str) { + counter!("sgl_tokenizer_encode_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_tokens_per_encode(token_count: usize) { + histogram!("sgl_tokenizer_tokens_per_encode").record(token_count as f64); + } + + pub fn record_chars_per_encode(char_count: usize) { + histogram!("sgl_tokenizer_chars_per_encode").record(char_count as f64); + } + + // Decoding metrics + pub fn record_decode_request(tokenizer_type: &str) { + counter!("sgl_tokenizer_decode_requests_total", + "tokenizer_type" => tokenizer_type.to_string() + ) + .increment(1); + } + + pub fn record_decode_duration(duration: Duration) { + histogram!("sgl_tokenizer_decode_duration_seconds").record(duration.as_secs_f64()); + } + + pub fn record_decode_error(error_type: &str) { + counter!("sgl_tokenizer_decode_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_tokens_per_decode(token_count: usize) { + histogram!("sgl_tokenizer_tokens_per_decode").record(token_count as f64); + } + + // Batch encoding metrics + pub fn record_encode_batch_duration(duration: Duration, batch_size: usize) { + histogram!("sgl_tokenizer_encode_batch_duration_seconds", + "batch_size" => batch_size.to_string() + ) + .record(duration.as_secs_f64()); + } + + // Stop sequence detection metrics + pub fn record_stop_sequence_detected(stop_type: &str) { + counter!("sgl_tokenizer_stop_sequences_detected_total", + "type" => stop_type.to_string() + ) + .increment(1); + } + + pub fn record_partial_match() { + counter!("sgl_tokenizer_partial_matches_total").increment(1); + } + + pub fn record_stop_detection_duration(duration: Duration) { + histogram!("sgl_tokenizer_stop_detection_duration_seconds").record(duration.as_secs_f64()); + } + + // Streaming decode metrics + pub fn record_stream_token() { + counter!("sgl_tokenizer_stream_tokens_total").increment(1); + } + + pub fn record_incomplete_utf8() { + counter!("sgl_tokenizer_stream_incomplete_utf8_total").increment(1); + } + + pub fn record_stream_step_duration(duration: Duration) { + histogram!("sgl_tokenizer_stream_step_duration_seconds").record(duration.as_secs_f64()); + } + + // Factory metrics + pub fn record_factory_load(file_type: &str) { + counter!("sgl_tokenizer_factory_loads_total", + "file_type" => file_type.to_string() + ) + .increment(1); + } + + pub fn record_factory_error(error_type: &str) { + counter!("sgl_tokenizer_factory_errors_total", + "error_type" => error_type.to_string() + ) + .increment(1); + } + + pub fn record_factory_load_duration(duration: Duration) { + histogram!("sgl_tokenizer_factory_load_duration_seconds").record(duration.as_secs_f64()); + } + + // Vocabulary metrics + pub fn set_vocab_size(tokenizer_type: &str, size: usize) { + gauge!("sgl_tokenizer_vocab_size", + "tokenizer_type" => tokenizer_type.to_string() + ) + .set(size as f64); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + // ============= PrometheusConfig Tests ============= + + #[test] + fn test_prometheus_config_default() { + let config = PrometheusConfig::default(); + assert_eq!(config.port, 29000); + assert_eq!(config.host, "0.0.0.0"); + } + + #[test] + fn test_prometheus_config_custom() { + let config = PrometheusConfig { + port: 8080, + host: "127.0.0.1".to_string(), + }; + assert_eq!(config.port, 8080); + assert_eq!(config.host, "127.0.0.1"); + } + + #[test] + fn test_prometheus_config_clone() { + let config = PrometheusConfig { + port: 9090, + host: "192.168.1.1".to_string(), + }; + let cloned = config.clone(); + assert_eq!(cloned.port, config.port); + assert_eq!(cloned.host, config.host); + } + + // ============= IP Address Parsing Tests ============= + + #[test] + fn test_valid_ipv4_parsing() { + let test_cases = vec!["127.0.0.1", "192.168.1.1", "0.0.0.0"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + assert!(matches!(ip_addr, IpAddr::V4(_))); + } + } + + #[test] + fn test_valid_ipv6_parsing() { + let test_cases = vec!["::1", "2001:db8::1", "::"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + assert!(matches!(ip_addr, IpAddr::V6(_))); + } + } + + #[test] + fn test_invalid_ip_parsing() { + let test_cases = vec!["invalid", "256.256.256.256", "hostname"]; + + for ip_str in test_cases { + let config = PrometheusConfig { + port: 29000, + host: ip_str.to_string(), + }; + + let ip_addr: IpAddr = config + .host + .parse() + .unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + + // Should fall back to 0.0.0.0 + assert_eq!(ip_addr, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))); + } + } + + // ============= Socket Address Creation Tests ============= + + #[test] + fn test_socket_addr_creation() { + let test_cases = vec![("127.0.0.1", 8080), ("0.0.0.0", 29000), ("::1", 9090)]; + + for (host, port) in test_cases { + let config = PrometheusConfig { + port, + host: host.to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.port(), port); + assert_eq!(socket_addr.ip().to_string(), host); + } + } + + #[test] + fn test_socket_addr_with_different_ports() { + let ports = vec![0, 80, 8080, 65535]; + + for port in ports { + let config = PrometheusConfig { + port, + host: "127.0.0.1".to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.port(), port); + } + } + + // ============= Duration Bucket Tests ============= + + #[test] + fn test_duration_bucket_coverage() { + let test_cases: [(f64, &str); 7] = [ + (0.0005, "sub-millisecond"), + (0.005, "5ms"), + (0.05, "50ms"), + (1.0, "1s"), + (10.0, "10s"), + (60.0, "1m"), + (240.0, "4m"), + ]; + + let buckets: [f64; 20] = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + for (duration, label) in test_cases { + let bucket_found = buckets + .iter() + .any(|&b| (b - duration).abs() < 0.0001 || b > duration); + assert!(bucket_found, "No bucket found for {} ({})", duration, label); + } + } + + // ============= Matcher Configuration Tests ============= + + #[test] + fn test_duration_suffix_matcher() { + let matcher = Matcher::Suffix(String::from("duration_seconds")); + + // Test matching behavior + let _matching_metrics = [ + "request_duration_seconds", + "response_duration_seconds", + "sgl_router_request_duration_seconds", + ]; + + let _non_matching_metrics = ["duration_total", "duration_seconds_total", "other_metric"]; + + // Note: We can't directly test Matcher matching without the internals, + // but we can verify the matcher is created correctly + match matcher { + Matcher::Suffix(suffix) => assert_eq!(suffix, "duration_seconds"), + _ => panic!("Expected Suffix matcher"), + } + } + + // ============= Builder Configuration Tests ============= + + #[test] + fn test_prometheus_builder_configuration() { + // This test verifies the builder configuration without actually starting Prometheus + let _config = PrometheusConfig::default(); + + let duration_matcher = Matcher::Suffix(String::from("duration_seconds")); + let duration_bucket = [ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 30.0, 45.0, + 60.0, 90.0, 120.0, 180.0, 240.0, + ]; + + // Verify bucket configuration + assert_eq!(duration_bucket.len(), 20); + + // Verify matcher is suffix type + match duration_matcher { + Matcher::Suffix(s) => assert_eq!(s, "duration_seconds"), + _ => panic!("Expected Suffix matcher"), + } + } + + // ============= Upkeep Timeout Tests ============= + + #[test] + fn test_upkeep_timeout_duration() { + let timeout = Duration::from_secs(5 * 60); + assert_eq!(timeout.as_secs(), 300); + } + + // ============= Custom Bucket Tests ============= + + #[test] + fn test_custom_buckets_for_different_metrics() { + // Test that we can create different bucket configurations + let request_buckets = [0.001, 0.01, 0.1, 1.0, 10.0]; + let generate_buckets = [0.1, 0.5, 1.0, 5.0, 30.0, 60.0]; + + assert_eq!(request_buckets.len(), 5); + assert_eq!(generate_buckets.len(), 6); + + // Verify each set is sorted + for i in 1..request_buckets.len() { + assert!(request_buckets[i] > request_buckets[i - 1]); + } + + for i in 1..generate_buckets.len() { + assert!(generate_buckets[i] > generate_buckets[i - 1]); + } + } + + // ============= RouterMetrics Tests ============= + + #[test] + fn test_metrics_static_methods() { + // Test that all static methods can be called without panic + RouterMetrics::record_request("/generate"); + RouterMetrics::record_request_duration("/generate", Duration::from_millis(100)); + RouterMetrics::record_request_error("/generate", "timeout"); + RouterMetrics::record_retry("/generate"); + + RouterMetrics::set_active_workers(5); + RouterMetrics::set_worker_health("http://worker1", true); + RouterMetrics::set_worker_load("http://worker1", 10); + RouterMetrics::record_processed_request("http://worker1"); + + RouterMetrics::record_policy_decision("random", "http://worker1"); + RouterMetrics::record_cache_hit(); + RouterMetrics::record_cache_miss(); + RouterMetrics::set_tree_size("http://worker1", 1000); + RouterMetrics::record_load_balancing_event(); + RouterMetrics::set_load_range(20, 5); + + RouterMetrics::record_pd_request("/v1/chat/completions"); + RouterMetrics::record_pd_request_duration("/v1/chat/completions", Duration::from_secs(1)); + RouterMetrics::record_pd_prefill_request("http://prefill1"); + RouterMetrics::record_pd_decode_request("http://decode1"); + RouterMetrics::record_pd_error("invalid_request"); + RouterMetrics::record_pd_prefill_error("http://prefill1"); + RouterMetrics::record_pd_decode_error("http://decode1"); + RouterMetrics::record_pd_stream_error("http://decode1"); + + RouterMetrics::record_discovery_update(3, 1); + RouterMetrics::record_generate_duration(Duration::from_secs(2)); + RouterMetrics::set_running_requests("http://worker1", 15); + } + + #[test] + fn test_tokenizer_metrics_static_methods() { + // Test that all tokenizer metric methods can be called without panic + + // Encoding metrics + TokenizerMetrics::record_encode_request("huggingface"); + TokenizerMetrics::record_encode_duration(Duration::from_millis(10)); + TokenizerMetrics::record_encode_error("invalid_input"); + TokenizerMetrics::record_tokens_per_encode(100); + TokenizerMetrics::record_chars_per_encode(500); + + // Decoding metrics + TokenizerMetrics::record_decode_request("huggingface"); + TokenizerMetrics::record_decode_duration(Duration::from_millis(5)); + TokenizerMetrics::record_decode_error("invalid_tokens"); + TokenizerMetrics::record_tokens_per_decode(50); + + // Batch encoding + TokenizerMetrics::record_encode_batch_duration(Duration::from_millis(100), 10); + + // Stop sequence detection + TokenizerMetrics::record_stop_sequence_detected("token"); + TokenizerMetrics::record_stop_sequence_detected("string"); + TokenizerMetrics::record_partial_match(); + TokenizerMetrics::record_stop_detection_duration(Duration::from_micros(100)); + + // Streaming decode + TokenizerMetrics::record_stream_token(); + TokenizerMetrics::record_incomplete_utf8(); + TokenizerMetrics::record_stream_step_duration(Duration::from_micros(50)); + + // Factory metrics + TokenizerMetrics::record_factory_load("json"); + TokenizerMetrics::record_factory_error("unsupported_format"); + TokenizerMetrics::record_factory_load_duration(Duration::from_millis(200)); + + // Vocabulary metrics + TokenizerMetrics::set_vocab_size("huggingface", 50000); + } + + // ============= Port Availability Tests ============= + + #[test] + fn test_port_already_in_use() { + // Skip this test if we can't bind to the port + let port = 29123; // Use a different port to avoid conflicts + + if let Ok(_listener) = TcpListener::bind(("127.0.0.1", port)) { + // Port is available, we can test + let config = PrometheusConfig { + port, + host: "127.0.0.1".to_string(), + }; + + // Just verify config is created correctly + assert_eq!(config.port, port); + } + } + + // ============= Integration Test Helpers ============= + + #[test] + fn test_metrics_endpoint_accessibility() { + // This would be an integration test in practice + // Here we just verify the configuration + let config = PrometheusConfig { + port: 29000, + host: "127.0.0.1".to_string(), + }; + + let ip_addr: IpAddr = config.host.parse().unwrap(); + let socket_addr = SocketAddr::new(ip_addr, config.port); + + assert_eq!(socket_addr.to_string(), "127.0.0.1:29000"); + } + + #[test] + fn test_concurrent_metric_updates() { + // Test that metric updates can be called concurrently + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use std::thread; + + let done = Arc::new(AtomicBool::new(false)); + let mut handles = vec![]; + + for i in 0..3 { + let done_clone = done.clone(); + let handle = thread::spawn(move || { + let worker = format!("http://worker{}", i); + while !done_clone.load(Ordering::Relaxed) { + RouterMetrics::set_worker_load(&worker, i * 10); + RouterMetrics::record_processed_request(&worker); + thread::sleep(Duration::from_millis(1)); + } + }); + handles.push(handle); + } + + // Let threads run briefly + thread::sleep(Duration::from_millis(10)); + done.store(true, Ordering::Relaxed); + + // Wait for all threads + for handle in handles { + handle.join().unwrap(); + } + } + + // ============= Edge Cases Tests ============= + + #[test] + fn test_empty_string_metrics() { + // Test that empty strings don't cause issues + RouterMetrics::record_request(""); + RouterMetrics::set_worker_health("", true); + RouterMetrics::record_policy_decision("", ""); + } + + #[test] + fn test_very_long_metric_labels() { + let long_label = "a".repeat(1000); + + RouterMetrics::record_request(&long_label); + RouterMetrics::set_worker_health(&long_label, false); + } + + #[test] + fn test_special_characters_in_labels() { + let special_labels = [ + "test/with/slashes", + "test-with-dashes", + "test_with_underscores", + "test.with.dots", + "test:with:colons", + ]; + + for label in special_labels { + RouterMetrics::record_request(label); + RouterMetrics::set_worker_health(label, true); + } + } + + #[test] + fn test_extreme_metric_values() { + // Test extreme values + RouterMetrics::set_active_workers(0); + RouterMetrics::set_active_workers(usize::MAX); + + RouterMetrics::set_worker_load("worker", 0); + RouterMetrics::set_worker_load("worker", usize::MAX); + + RouterMetrics::record_request_duration("route", Duration::from_nanos(1)); + // 24 hours + RouterMetrics::record_request_duration("route", Duration::from_secs(86400)); + } +} diff --git a/sgl-router/src/middleware.rs b/sgl-router/src/middleware.rs new file mode 100644 index 000000000..abe137572 --- /dev/null +++ b/sgl-router/src/middleware.rs @@ -0,0 +1,502 @@ +use axum::{ + extract::Request, extract::State, http::HeaderValue, http::StatusCode, middleware::Next, + response::IntoResponse, response::Response, +}; +use rand::Rng; +use std::sync::Arc; +use std::time::Duration; +use std::time::Instant; +use tokio::sync::{mpsc, oneshot}; +use tower::{Layer, Service}; +use tower_http::trace::{MakeSpan, OnRequest, OnResponse, TraceLayer}; +use tracing::{debug, error, field::Empty, info, info_span, warn, Span}; + +pub use crate::core::token_bucket::TokenBucket; + +use crate::server::AppState; + +/// Generate OpenAI-compatible request ID based on endpoint +fn generate_request_id(path: &str) -> String { + let prefix = if path.contains("/chat/completions") { + "chatcmpl-" + } else if path.contains("/completions") { + "cmpl-" + } else if path.contains("/generate") { + "gnt-" + } else { + "req-" + }; + + // Generate a random string similar to OpenAI's format + let chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + let mut rng = rand::rng(); + let random_part: String = (0..24) + .map(|_| { + let idx = rng.random_range(0..chars.len()); + chars.chars().nth(idx).unwrap() + }) + .collect(); + + format!("{}{}", prefix, random_part) +} + +/// Extension type for storing request ID +#[derive(Clone, Debug)] +pub struct RequestId(pub String); + +/// Tower Layer for request ID middleware +#[derive(Clone)] +pub struct RequestIdLayer { + headers: Arc>, +} + +impl RequestIdLayer { + pub fn new(headers: Vec) -> Self { + Self { + headers: Arc::new(headers), + } + } +} + +impl Layer for RequestIdLayer { + type Service = RequestIdMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + RequestIdMiddleware { + inner, + headers: self.headers.clone(), + } + } +} + +/// Tower Service for request ID middleware +#[derive(Clone)] +pub struct RequestIdMiddleware { + inner: S, + headers: Arc>, +} + +impl Service for RequestIdMiddleware +where + S: Service + Send + 'static, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = std::pin::Pin< + Box> + Send>, + >; + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, mut req: Request) -> Self::Future { + let headers = self.headers.clone(); + + // Extract request ID from headers or generate new one + let mut request_id = None; + + for header_name in headers.iter() { + if let Some(header_value) = req.headers().get(header_name) { + if let Ok(value) = header_value.to_str() { + request_id = Some(value.to_string()); + break; + } + } + } + + let request_id = request_id.unwrap_or_else(|| generate_request_id(req.uri().path())); + + // Insert request ID into request extensions + req.extensions_mut().insert(RequestId(request_id.clone())); + + // Create a span with the request ID for this request + let span = tracing::info_span!( + "http_request", + method = %req.method(), + uri = %req.uri(), + version = ?req.version(), + request_id = %request_id + ); + + // Log within the span + let _enter = span.enter(); + tracing::info!( + target: "sglang_router_rs::request", + "started processing request" + ); + drop(_enter); + + // Capture values we need in the async block + let method = req.method().clone(); + let uri = req.uri().clone(); + let version = req.version(); + + // Call the inner service + let future = self.inner.call(req); + + Box::pin(async move { + let start_time = Instant::now(); + let mut response = future.await?; + let latency = start_time.elapsed(); + + // Add request ID to response headers + response.headers_mut().insert( + "x-request-id", + HeaderValue::from_str(&request_id) + .unwrap_or_else(|_| HeaderValue::from_static("invalid-request-id")), + ); + + // Log the response with proper request ID in span + let status = response.status(); + let span = tracing::info_span!( + "http_request", + method = %method, + uri = %uri, + version = ?version, + request_id = %request_id, + status = %status, + latency = ?latency + ); + + let _enter = span.enter(); + if status.is_server_error() { + tracing::error!( + target: "sglang_router_rs::response", + "request failed with server error" + ); + } else if status.is_client_error() { + tracing::warn!( + target: "sglang_router_rs::response", + "request failed with client error" + ); + } else { + tracing::info!( + target: "sglang_router_rs::response", + "finished processing request" + ); + } + + Ok(response) + }) + } +} + +// ============= Logging Middleware ============= + +/// Custom span maker that includes request ID +#[derive(Clone, Debug)] +pub struct RequestSpan; + +impl MakeSpan for RequestSpan { + fn make_span(&mut self, request: &Request) -> Span { + // Don't try to extract request ID here - it won't be available yet + // The RequestIdLayer runs after TraceLayer creates the span + info_span!( + "http_request", + method = %request.method(), + uri = %request.uri(), + version = ?request.version(), + request_id = Empty, // Will be set later + status_code = Empty, + latency = Empty, + error = Empty, + ) + } +} + +/// Custom on_request handler +#[derive(Clone, Debug)] +pub struct RequestLogger; + +impl OnRequest for RequestLogger { + fn on_request(&mut self, request: &Request, span: &Span) { + let _enter = span.enter(); + + // Try to get the request ID from extensions + // This will work if RequestIdLayer has already run + if let Some(request_id) = request.extensions().get::() { + span.record("request_id", request_id.0.as_str()); + } + + // Don't log here - we already log in RequestIdService with the proper request_id + } +} + +/// Custom on_response handler +#[derive(Clone, Debug)] +pub struct ResponseLogger { + _start_time: Instant, +} + +impl Default for ResponseLogger { + fn default() -> Self { + Self { + _start_time: Instant::now(), + } + } +} + +impl OnResponse for ResponseLogger { + fn on_response(self, response: &Response, latency: std::time::Duration, span: &Span) { + let status = response.status(); + + // Record these in the span for structured logging/observability tools + span.record("status_code", status.as_u16()); + span.record("latency", format!("{:?}", latency)); + + // Don't log here - RequestIdService handles all logging with proper request IDs + } +} + +/// Create a configured TraceLayer for HTTP logging +/// Note: Actual request/response logging with request IDs is done in RequestIdService +pub fn create_logging_layer() -> TraceLayer< + tower_http::classify::SharedClassifier, + RequestSpan, + RequestLogger, + ResponseLogger, +> { + TraceLayer::new_for_http() + .make_span_with(RequestSpan) + .on_request(RequestLogger) + .on_response(ResponseLogger::default()) +} + +/// Structured logging data for requests +#[derive(Debug, serde::Serialize)] +pub struct RequestLogEntry { + pub timestamp: String, + pub request_id: String, + pub method: String, + pub uri: String, + pub status: u16, + pub latency_ms: u64, + pub user_agent: Option, + pub remote_addr: Option, + pub error: Option, +} + +/// Log a request with structured data +pub fn log_request(entry: RequestLogEntry) { + if entry.status >= 500 { + tracing::error!( + target: "sglang_router_rs::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + error = ?entry.error, + "HTTP request failed" + ); + } else if entry.status >= 400 { + tracing::warn!( + target: "sglang_router_rs::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + "HTTP request client error" + ); + } else { + tracing::info!( + target: "sglang_router_rs::http", + request_id = %entry.request_id, + method = %entry.method, + uri = %entry.uri, + status = entry.status, + latency_ms = entry.latency_ms, + user_agent = ?entry.user_agent, + remote_addr = ?entry.remote_addr, + "HTTP request completed" + ); + } +} + +// ============ Concurrency Limiting with Queue Support ============ + +/// Request queue entry +pub struct QueuedRequest { + /// Time when the request was queued + queued_at: Instant, + /// Channel to send the permit back when acquired + permit_tx: oneshot::Sender>, +} + +/// Queue metrics for monitoring +#[derive(Debug, Default)] +pub struct QueueMetrics { + pub total_queued: std::sync::atomic::AtomicU64, + pub current_queued: std::sync::atomic::AtomicU64, + pub total_timeout: std::sync::atomic::AtomicU64, + pub total_rejected: std::sync::atomic::AtomicU64, +} + +/// Queue processor that handles queued requests +pub struct QueueProcessor { + token_bucket: Arc, + queue_rx: mpsc::Receiver, + queue_timeout: Duration, +} + +impl QueueProcessor { + pub fn new( + token_bucket: Arc, + queue_rx: mpsc::Receiver, + queue_timeout: Duration, + ) -> Self { + Self { + token_bucket, + queue_rx, + queue_timeout, + } + } + + pub async fn run(mut self) { + info!("Starting concurrency queue processor"); + + // Process requests in a single task to reduce overhead + while let Some(queued) = self.queue_rx.recv().await { + // Check timeout immediately + let elapsed = queued.queued_at.elapsed(); + if elapsed >= self.queue_timeout { + warn!("Request already timed out in queue"); + let _ = queued.permit_tx.send(Err(StatusCode::REQUEST_TIMEOUT)); + continue; + } + + let remaining_timeout = self.queue_timeout - elapsed; + + // Try to acquire token for this request + if self.token_bucket.try_acquire(1.0).await.is_ok() { + // Got token immediately + debug!("Queue: acquired token immediately for queued request"); + let _ = queued.permit_tx.send(Ok(())); + } else { + // Need to wait for token + let token_bucket = self.token_bucket.clone(); + + // Spawn task only when we actually need to wait + tokio::spawn(async move { + if token_bucket + .acquire_timeout(1.0, remaining_timeout) + .await + .is_ok() + { + debug!("Queue: acquired token after waiting"); + let _ = queued.permit_tx.send(Ok(())); + } else { + warn!("Queue: request timed out waiting for token"); + let _ = queued.permit_tx.send(Err(StatusCode::REQUEST_TIMEOUT)); + } + }); + } + } + + warn!("Concurrency queue processor shutting down"); + } +} + +/// State for the concurrency limiter +pub struct ConcurrencyLimiter { + pub queue_tx: Option>, +} + +impl ConcurrencyLimiter { + /// Create new concurrency limiter with optional queue + pub fn new( + token_bucket: Arc, + queue_size: usize, + queue_timeout: Duration, + ) -> (Self, Option) { + if queue_size > 0 { + let (queue_tx, queue_rx) = mpsc::channel(queue_size); + let processor = QueueProcessor::new(token_bucket, queue_rx, queue_timeout); + + ( + Self { + queue_tx: Some(queue_tx), + }, + Some(processor), + ) + } else { + (Self { queue_tx: None }, None) + } + } +} + +/// Middleware function for concurrency limiting with optional queuing +pub async fn concurrency_limit_middleware( + State(app_state): State>, + request: Request, + next: Next, +) -> Response { + let token_bucket = app_state.context.rate_limiter.clone(); + + // Try to acquire token immediately + if token_bucket.try_acquire(1.0).await.is_ok() { + debug!("Acquired token immediately"); + let response = next.run(request).await; + + // Return the token to the bucket + token_bucket.return_tokens(1.0).await; + + response + } else { + // No tokens available, try to queue if enabled + if let Some(queue_tx) = &app_state.concurrency_queue_tx { + debug!("No tokens available, attempting to queue request"); + + // Create a channel for the token response + let (permit_tx, permit_rx) = oneshot::channel(); + + let queued = QueuedRequest { + queued_at: Instant::now(), + permit_tx, + }; + + // Try to send to queue + match queue_tx.try_send(queued) { + Ok(_) => { + // Wait for token from queue processor + match permit_rx.await { + Ok(Ok(())) => { + debug!("Acquired token from queue"); + let response = next.run(request).await; + + // Return the token to the bucket + token_bucket.return_tokens(1.0).await; + + response + } + Ok(Err(status)) => { + warn!("Queue returned error status: {}", status); + status.into_response() + } + Err(_) => { + error!("Queue response channel closed"); + StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + } + } + Err(_) => { + warn!("Request queue is full, returning 429"); + StatusCode::TOO_MANY_REQUESTS.into_response() + } + } + } else { + warn!("No tokens available and queuing is disabled, returning 429"); + StatusCode::TOO_MANY_REQUESTS.into_response() + } + } +} diff --git a/sgl-router/src/policies/cache_aware.rs b/sgl-router/src/policies/cache_aware.rs new file mode 100644 index 000000000..47d95c835 --- /dev/null +++ b/sgl-router/src/policies/cache_aware.rs @@ -0,0 +1,423 @@ +/* + Cache-Aware Load Balancing Router + + This router combines two strategies to optimize both cache utilization and request distribution: + + 1. Cache-Aware Routing (Approximate Tree) + 2. Load Balancing (Shortest Queue with Balance Thresholds) + + The router dynamically switches between these strategies based on load conditions: + - Uses load balancing when the system is imbalanced + - Uses cache-aware routing when the system is balanced + + A system is considered imbalanced if both conditions are met: + 1. (max - min) > abs_threshold + 2. max > rel_threshold * min + + Strategy Details: + + 1. Cache-Aware Routing (Approximate Tree) + ------------------------------------------- + This strategy maintains an approximate radix tree for each worker based on request history, + eliminating the need for direct cache state queries. The tree stores raw text characters + instead of token IDs to avoid tokenization overhead. + + Process: + a. For each request, find the worker with the highest prefix match + b. If match rate > cache_threshold: + Route to the worker with highest match (likely has relevant data cached) + c. If match rate ≤ cache_threshold: + Route to the worker with smallest tree size (most available cache capacity) + d. Background maintenance: + Periodically evict least recently used leaf nodes to prevent memory overflow + + 2. Load Balancing (Shortest Queue) + ------------------------------------------- + This strategy tracks pending request counts per worker and routes new requests + to the least busy worker when the system is detected to be imbalanced. + + Configuration Parameters: + ------------------------ + 1. cache_threshold: (float, 0.0 to 1.0) + Minimum prefix match ratio to use highest-match routing. + Below this threshold, routes to worker with most available cache space. + + 2. balance_abs_threshold: (integer) + Absolute difference threshold for load imbalance detection. + System is potentially imbalanced if (max_load - min_load) > abs_threshold + + 3. balance_rel_threshold: (float) + Relative ratio threshold for load imbalance detection. + System is potentially imbalanced if max_load > min_load * rel_threshold + Used in conjunction with abs_threshold to determine final imbalance state. + + 4. eviction_interval_secs: (integer) + Interval between LRU eviction cycles for the approximate trees. + + 5. max_tree_size: (integer) + Maximum nodes per tree. When exceeded, LRU leaf nodes are evicted + during the next eviction cycle. +*/ + +use super::{get_healthy_worker_indices, CacheAwareConfig, LoadBalancingPolicy}; +use crate::core::Worker; +use crate::metrics::RouterMetrics; +use crate::tree::Tree; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; +use tracing::debug; + +/// Cache-aware routing policy +/// +/// Routes requests based on cache affinity when load is balanced, +/// switches to shortest-queue routing when load is imbalanced. +#[derive(Debug)] +pub struct CacheAwarePolicy { + config: CacheAwareConfig, + tree: Arc>, + eviction_handle: Option>, +} + +impl CacheAwarePolicy { + pub fn new() -> Self { + Self::with_config(CacheAwareConfig::default()) + } + + pub fn with_config(config: CacheAwareConfig) -> Self { + let tree = Arc::new(Mutex::new(Tree::new())); + + // Start background eviction thread if configured + let eviction_handle = if config.eviction_interval_secs > 0 { + let tree_clone = Arc::clone(&tree); + let max_tree_size = config.max_tree_size; + let interval = config.eviction_interval_secs; + + Some(thread::spawn(move || loop { + thread::sleep(Duration::from_secs(interval)); + + if let Ok(tree_guard) = tree_clone.lock() { + tree_guard.evict_tenant_by_size(max_tree_size); + debug!("Cache eviction completed, max_size: {}", max_tree_size); + } + })) + } else { + None + }; + + Self { + config, + tree, + eviction_handle, + } + } + + /// Initialize the tree with worker URLs (used only during initial setup) + pub fn init_workers(&self, workers: &[Box]) { + if let Ok(tree) = self.tree.lock() { + for worker in workers { + tree.insert("", worker.url()); + } + } + } + + /// Add a single worker to the tree (incremental update) + pub fn add_worker(&self, url: &str) { + if let Ok(tree) = self.tree.lock() { + tree.insert("", url); + } + } + + /// Remove a worker from the tree + pub fn remove_worker(&self, url: &str) { + if let Ok(tree) = self.tree.lock() { + tree.remove_tenant(url); + } + } + + /// Run cache eviction to prevent unbounded growth + pub fn evict_cache(&self, max_size: usize) { + if let Ok(tree) = self.tree.lock() { + tree.evict_tenant_by_size(max_size); + } + } +} + +impl LoadBalancingPolicy for CacheAwarePolicy { + fn select_worker( + &self, + workers: &[Box], + request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + // Get current load statistics + let loads: Vec = workers.iter().map(|w| w.load()).collect(); + let max_load = *loads.iter().max().unwrap_or(&0); + let min_load = *loads.iter().min().unwrap_or(&0); + + // Check if load is imbalanced + let is_imbalanced = max_load.saturating_sub(min_load) > self.config.balance_abs_threshold + && (max_load as f32) > (min_load as f32 * self.config.balance_rel_threshold); + + if is_imbalanced { + // Log load balancing trigger + let worker_loads: Vec<(String, usize)> = workers + .iter() + .map(|w| (w.url().to_string(), w.load())) + .collect(); + + debug!( + "Load balancing triggered | max: {} | min: {} | workers: {:?}", + max_load, min_load, worker_loads + ); + + RouterMetrics::record_load_balancing_event(); + RouterMetrics::set_load_range(max_load, min_load); + + // Use shortest queue when imbalanced + let min_load_idx = healthy_indices + .iter() + .min_by_key(|&&idx| workers[idx].load()) + .copied()?; + + // Even in imbalanced mode, update the tree to maintain cache state + if let Some(text) = request_text { + if let Ok(tree) = self.tree.lock() { + tree.insert(text, workers[min_load_idx].url()); + } + } + + // Increment processed counter + workers[min_load_idx].increment_processed(); + RouterMetrics::record_processed_request(workers[min_load_idx].url()); + RouterMetrics::record_policy_decision(self.name(), workers[min_load_idx].url()); + + return Some(min_load_idx); + } + + // Use cache-aware routing when balanced + let text = request_text.unwrap_or(""); + + if let Ok(tree) = self.tree.lock() { + let (matched_text, matched_worker) = tree.prefix_match(text); + let match_rate = if text.is_empty() { + 0.0 + } else { + matched_text.chars().count() as f32 / text.chars().count() as f32 + }; + + let selected_url = if match_rate > self.config.cache_threshold { + RouterMetrics::record_cache_hit(); + matched_worker.to_string() + } else { + RouterMetrics::record_cache_miss(); + tree.get_smallest_tenant() + }; + + // Find the index of the selected worker + if let Some(selected_idx) = workers.iter().position(|w| w.url() == selected_url) { + // Only proceed if the worker is healthy + if workers[selected_idx].is_healthy() { + // Update the tree with this request + tree.insert(text, &selected_url); + + // Increment processed counter + workers[selected_idx].increment_processed(); + RouterMetrics::record_processed_request(&selected_url); + + return Some(selected_idx); + } + } else { + // Selected worker no longer exists, remove it from tree + tree.remove_tenant(&selected_url); + debug!("Removed stale worker {} from cache tree", selected_url); + } + + // Fallback to first healthy worker + return healthy_indices.first().copied(); + } + + // Fallback to first healthy worker if tree operations fail + healthy_indices.first().copied() + } + + fn name(&self) -> &'static str { + "cache_aware" + } + + fn needs_request_text(&self) -> bool { + true // Cache-aware policy needs request text for cache affinity + } + + fn on_request_complete(&self, worker_url: &str, success: bool) { + // Could track success rates per worker for more intelligent routing + if !success { + // Optionally reduce affinity for failed requests + tracing::debug!( + "Request to {} completed with success={}", + worker_url, + success + ); + } + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn select_worker_pair( + &self, + prefill_workers: &[Box], + decode_workers: &[Box], + request_text: Option<&str>, + ) -> Option<(usize, usize)> { + // DEPRECATED: This method is no longer used when separate policies are configured. + // The PD router now uses separate policies for prefill and decode selection. + // This implementation remains for backward compatibility when a single policy is used. + + // In PD mode with single policy: + // - Prefill: Use cache-aware routing for better cache utilization + // - Decode: Use least-load routing for better load distribution + + // Select prefill worker using cache-aware logic + let prefill_idx = self.select_worker(prefill_workers, request_text)?; + + // Select decode worker using least-load logic + let healthy_decode = get_healthy_worker_indices(decode_workers); + if healthy_decode.is_empty() { + return None; + } + + let decode_idx = healthy_decode + .iter() + .min_by_key(|&&idx| decode_workers[idx].load()) + .copied()?; + + Some((prefill_idx, decode_idx)) + } +} + +impl Default for CacheAwarePolicy { + fn default() -> Self { + Self::new() + } +} + +impl Drop for CacheAwarePolicy { + fn drop(&mut self) { + // Note: We can't properly stop the eviction thread since it's in an infinite loop + // In a production system, we'd use a channel or atomic flag to signal shutdown + if let Some(handle) = self.eviction_handle.take() { + // The thread will continue running until the program exits + // This is acceptable for now since the router typically runs for the lifetime of the program + drop(handle); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + + #[test] + fn test_cache_aware_with_balanced_load() { + // Create policy without eviction thread for testing + let config = CacheAwareConfig { + eviction_interval_secs: 0, // Disable eviction thread + ..Default::default() + }; + let policy = CacheAwarePolicy::with_config(config); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Initialize the policy with workers + policy.init_workers(&workers); + + // First request should be distributed + let idx1 = policy.select_worker(&workers, Some("hello world")).unwrap(); + + // Same request should go to same worker (cache hit) + let idx2 = policy.select_worker(&workers, Some("hello world")).unwrap(); + assert_eq!(idx1, idx2); + + // Similar request should also go to same worker + let idx3 = policy.select_worker(&workers, Some("hello")).unwrap(); + assert_eq!(idx1, idx3); + } + + #[test] + fn test_cache_aware_with_imbalanced_load() { + let policy = CacheAwarePolicy::with_config(CacheAwareConfig { + cache_threshold: 0.5, + balance_abs_threshold: 5, + balance_rel_threshold: 2.0, + eviction_interval_secs: 0, // Disable eviction thread + max_tree_size: 10000, + }); + + let worker1 = BasicWorker::new("http://w1:8000".to_string(), WorkerType::Regular); + let worker2 = BasicWorker::new("http://w2:8000".to_string(), WorkerType::Regular); + + // Create significant load imbalance + for _ in 0..20 { + worker1.increment_load(); + } + // worker2 has load 0 + + let workers: Vec> = vec![Box::new(worker1), Box::new(worker2)]; + policy.init_workers(&workers); + + // Should select worker2 (lower load) despite cache affinity + for _ in 0..5 { + let idx = policy.select_worker(&workers, Some("test")).unwrap(); + assert_eq!(idx, 1); // Should always pick worker2 + } + } + + #[test] + fn test_cache_aware_worker_removal() { + let config = CacheAwareConfig { + eviction_interval_secs: 0, // Disable eviction thread + ..Default::default() + }; + let policy = CacheAwarePolicy::with_config(config); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + ]; + + policy.init_workers(&workers); + + // Route some requests + policy.select_worker(&workers, Some("test1")); + policy.select_worker(&workers, Some("test2")); + + // Remove a worker + policy.remove_worker("http://w1:8000"); + workers[0].set_healthy(false); + + // All requests should now go to worker2 + let idx = policy.select_worker(&workers, Some("test1")).unwrap(); + assert_eq!(idx, 1); + } +} diff --git a/sgl-router/src/policies/factory.rs b/sgl-router/src/policies/factory.rs new file mode 100644 index 000000000..c65785d63 --- /dev/null +++ b/sgl-router/src/policies/factory.rs @@ -0,0 +1,94 @@ +//! Factory for creating load balancing policies + +use super::{ + CacheAwareConfig, CacheAwarePolicy, LoadBalancingPolicy, PowerOfTwoPolicy, RandomPolicy, + RoundRobinPolicy, +}; +use crate::config::PolicyConfig; +use std::sync::Arc; + +/// Factory for creating policy instances +pub struct PolicyFactory; + +impl PolicyFactory { + /// Create a policy from configuration + pub fn create_from_config(config: &PolicyConfig) -> Arc { + match config { + PolicyConfig::Random => Arc::new(RandomPolicy::new()), + PolicyConfig::RoundRobin => Arc::new(RoundRobinPolicy::new()), + PolicyConfig::PowerOfTwo { .. } => Arc::new(PowerOfTwoPolicy::new()), + PolicyConfig::CacheAware { + cache_threshold, + balance_abs_threshold, + balance_rel_threshold, + eviction_interval_secs, + max_tree_size, + } => { + let config = CacheAwareConfig { + cache_threshold: *cache_threshold, + balance_abs_threshold: *balance_abs_threshold, + balance_rel_threshold: *balance_rel_threshold, + eviction_interval_secs: *eviction_interval_secs, + max_tree_size: *max_tree_size, + }; + Arc::new(CacheAwarePolicy::with_config(config)) + } + } + } + + /// Create a policy by name (for dynamic loading) + pub fn create_by_name(name: &str) -> Option> { + match name.to_lowercase().as_str() { + "random" => Some(Arc::new(RandomPolicy::new())), + "round_robin" | "roundrobin" => Some(Arc::new(RoundRobinPolicy::new())), + "power_of_two" | "poweroftwo" => Some(Arc::new(PowerOfTwoPolicy::new())), + "cache_aware" | "cacheaware" => Some(Arc::new(CacheAwarePolicy::new())), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_create_from_config() { + // Test Random + let policy = PolicyFactory::create_from_config(&PolicyConfig::Random); + assert_eq!(policy.name(), "random"); + + // Test RoundRobin + let policy = PolicyFactory::create_from_config(&PolicyConfig::RoundRobin); + assert_eq!(policy.name(), "round_robin"); + + // Test PowerOfTwo + let policy = PolicyFactory::create_from_config(&PolicyConfig::PowerOfTwo { + load_check_interval_secs: 60, + }); + assert_eq!(policy.name(), "power_of_two"); + + // Test CacheAware + let policy = PolicyFactory::create_from_config(&PolicyConfig::CacheAware { + cache_threshold: 0.7, + balance_abs_threshold: 10, + balance_rel_threshold: 1.5, + eviction_interval_secs: 30, + max_tree_size: 1000, + }); + assert_eq!(policy.name(), "cache_aware"); + } + + #[test] + fn test_create_by_name() { + assert!(PolicyFactory::create_by_name("random").is_some()); + assert!(PolicyFactory::create_by_name("RANDOM").is_some()); + assert!(PolicyFactory::create_by_name("round_robin").is_some()); + assert!(PolicyFactory::create_by_name("RoundRobin").is_some()); + assert!(PolicyFactory::create_by_name("power_of_two").is_some()); + assert!(PolicyFactory::create_by_name("PowerOfTwo").is_some()); + assert!(PolicyFactory::create_by_name("cache_aware").is_some()); + assert!(PolicyFactory::create_by_name("CacheAware").is_some()); + assert!(PolicyFactory::create_by_name("unknown").is_none()); + } +} diff --git a/sgl-router/src/policies/mod.rs b/sgl-router/src/policies/mod.rs new file mode 100644 index 000000000..97ce9ca6f --- /dev/null +++ b/sgl-router/src/policies/mod.rs @@ -0,0 +1,148 @@ +//! Load balancing policies for SGLang router +//! +//! This module provides a unified abstraction for routing policies that work +//! across both regular and prefill-decode (PD) routing modes. + +use crate::core::Worker; +use std::fmt::Debug; + +mod cache_aware; +mod factory; +mod power_of_two; +mod random; +mod round_robin; + +pub use cache_aware::CacheAwarePolicy; +pub use factory::PolicyFactory; +pub use power_of_two::PowerOfTwoPolicy; +pub use random::RandomPolicy; +pub use round_robin::RoundRobinPolicy; + +/// Core trait for load balancing policies +/// +/// This trait provides a unified interface for implementing routing algorithms +/// that can work with both regular single-worker selection and PD dual-worker selection. +pub trait LoadBalancingPolicy: Send + Sync + Debug { + /// Select a single worker from the available workers + /// + /// This is used for regular routing mode where requests go to a single worker. + fn select_worker( + &self, + workers: &[Box], + request_text: Option<&str>, + ) -> Option; + + /// Select a pair of workers (prefill and decode) for PD routing + /// + /// Returns indices of (prefill_worker, decode_worker) from their respective arrays. + /// Default implementation uses select_worker for each array independently. + fn select_worker_pair( + &self, + prefill_workers: &[Box], + decode_workers: &[Box], + request_text: Option<&str>, + ) -> Option<(usize, usize)> { + // Default implementation: independently select from each pool + let prefill_idx = self.select_worker(prefill_workers, request_text)?; + let decode_idx = self.select_worker(decode_workers, request_text)?; + Some((prefill_idx, decode_idx)) + } + + /// Update policy state after request completion + /// + /// This is called when a request completes (successfully or not) to allow + /// policies to update their internal state. + fn on_request_complete(&self, _worker_url: &str, _success: bool) { + // Default: no-op for stateless policies + } + + /// Get policy name for metrics and debugging + fn name(&self) -> &'static str; + + /// Check if this policy needs request text for routing decisions + fn needs_request_text(&self) -> bool { + false // Default: most policies don't need request text + } + + /// Update worker load information + /// + /// This is called periodically with current load information for load-aware policies. + fn update_loads(&self, _loads: &std::collections::HashMap) { + // Default: no-op for policies that don't use load information + } + + /// Reset any internal state + /// + /// This is useful for policies that maintain state (e.g., round-robin counters). + fn reset(&self) { + // Default: no-op for stateless policies + } + + /// Get as Any for downcasting + fn as_any(&self) -> &dyn std::any::Any; +} + +/// Configuration for cache-aware policy +#[derive(Debug, Clone)] +pub struct CacheAwareConfig { + pub cache_threshold: f32, + pub balance_abs_threshold: usize, + pub balance_rel_threshold: f32, + pub eviction_interval_secs: u64, + pub max_tree_size: usize, +} + +impl Default for CacheAwareConfig { + fn default() -> Self { + Self { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + eviction_interval_secs: 30, + max_tree_size: 10000, + } + } +} + +/// Helper function to filter healthy workers and return their indices +pub(crate) fn get_healthy_worker_indices(workers: &[Box]) -> Vec { + workers + .iter() + .enumerate() + .filter(|(_, w)| w.is_healthy() && w.circuit_breaker().can_execute()) + .map(|(idx, _)| idx) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + + #[test] + fn test_get_healthy_worker_indices() { + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w3:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // All healthy initially + let indices = get_healthy_worker_indices(&workers); + assert_eq!(indices, vec![0, 1, 2]); + + // Mark one unhealthy + workers[1].set_healthy(false); + let indices = get_healthy_worker_indices(&workers); + assert_eq!(indices, vec![0, 2]); + } +} diff --git a/sgl-router/src/policies/power_of_two.rs b/sgl-router/src/policies/power_of_two.rs new file mode 100644 index 000000000..c10fc2949 --- /dev/null +++ b/sgl-router/src/policies/power_of_two.rs @@ -0,0 +1,201 @@ +//! Power-of-two choices load balancing policy + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::core::Worker; +use crate::metrics::RouterMetrics; +use rand::Rng; +use std::collections::HashMap; +use std::sync::RwLock; +use tracing::info; + +/// Power-of-two choices policy +/// +/// Randomly selects two workers and routes to the one with lower load. +/// This provides good load distribution with minimal coordination overhead. +#[derive(Debug)] +pub struct PowerOfTwoPolicy { + /// Cached load information from external monitoring + cached_loads: RwLock>, +} + +impl PowerOfTwoPolicy { + pub fn new() -> Self { + Self { + cached_loads: RwLock::new(HashMap::new()), + } + } + + fn get_worker_load(&self, worker: &dyn Worker) -> isize { + // First check cached loads (from external monitoring) + if let Ok(loads) = self.cached_loads.read() { + if let Some(&load) = loads.get(worker.url()) { + return load; + } + } + + // Fall back to local load counter + worker.load() as isize + } +} + +impl LoadBalancingPolicy for PowerOfTwoPolicy { + fn select_worker( + &self, + workers: &[Box], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + if healthy_indices.len() == 1 { + return Some(healthy_indices[0]); + } + + // Select two random workers + let mut rng = rand::rng(); + let idx1 = rng.random_range(0..healthy_indices.len()); + let mut idx2 = rng.random_range(0..healthy_indices.len()); + + // Ensure we pick two different workers + while idx2 == idx1 { + idx2 = rng.random_range(0..healthy_indices.len()); + } + + let worker_idx1 = healthy_indices[idx1]; + let worker_idx2 = healthy_indices[idx2]; + + // Compare loads and select the less loaded one + let load1 = self.get_worker_load(workers[worker_idx1].as_ref()); + let load2 = self.get_worker_load(workers[worker_idx2].as_ref()); + + // Log selection for debugging + let selected_idx = if load1 <= load2 { + worker_idx1 + } else { + worker_idx2 + }; + + info!( + "Power-of-two selection: {}={} vs {}={} -> selected {}", + workers[worker_idx1].url(), + load1, + workers[worker_idx2].url(), + load2, + workers[selected_idx].url() + ); + + // Increment processed counter + workers[selected_idx].increment_processed(); + RouterMetrics::record_processed_request(workers[selected_idx].url()); + RouterMetrics::record_policy_decision(self.name(), workers[selected_idx].url()); + + Some(selected_idx) + } + + fn name(&self) -> &'static str { + "power_of_two" + } + + fn update_loads(&self, loads: &HashMap) { + if let Ok(mut cached) = self.cached_loads.write() { + *cached = loads.clone(); + } + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +impl Default for PowerOfTwoPolicy { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + + #[test] + fn test_power_of_two_selection() { + let policy = PowerOfTwoPolicy::new(); + let worker1 = BasicWorker::new("http://w1:8000".to_string(), WorkerType::Regular); + let worker2 = BasicWorker::new("http://w2:8000".to_string(), WorkerType::Regular); + let worker3 = BasicWorker::new("http://w3:8000".to_string(), WorkerType::Regular); + + // Set different loads + for _ in 0..10 { + worker1.increment_load(); + } + for _ in 0..5 { + worker2.increment_load(); + } + // worker3 has load 0 + + let workers: Vec> = + vec![Box::new(worker1), Box::new(worker2), Box::new(worker3)]; + + // Run multiple selections + let mut selected_counts = [0; 3]; + for _ in 0..100 { + if let Some(idx) = policy.select_worker(&workers, None) { + selected_counts[idx] += 1; + } + } + + // Worker with lowest load (worker3) should be selected most often + assert!(selected_counts[2] > selected_counts[1]); + assert!(selected_counts[1] > selected_counts[0]); + } + + #[test] + fn test_power_of_two_with_cached_loads() { + let policy = PowerOfTwoPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Update cached loads + let mut loads = HashMap::new(); + loads.insert("http://w1:8000".to_string(), 100); + loads.insert("http://w2:8000".to_string(), 10); + policy.update_loads(&loads); + + // Should prefer worker2 with lower cached load + let mut w2_selected = 0; + for _ in 0..50 { + if let Some(idx) = policy.select_worker(&workers, None) { + if idx == 1 { + w2_selected += 1; + } + } + } + + // Worker2 should be selected significantly more often + assert!(w2_selected > 35); // Should win most of the time + } + + #[test] + fn test_power_of_two_single_worker() { + let policy = PowerOfTwoPolicy::new(); + let workers: Vec> = vec![Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + ))]; + + // With single worker, should always select it + assert_eq!(policy.select_worker(&workers, None), Some(0)); + } +} diff --git a/sgl-router/src/policies/random.rs b/sgl-router/src/policies/random.rs new file mode 100644 index 000000000..4912d0dd2 --- /dev/null +++ b/sgl-router/src/policies/random.rs @@ -0,0 +1,121 @@ +//! Random load balancing policy + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::core::Worker; +use crate::metrics::RouterMetrics; +use rand::Rng; + +/// Random selection policy +/// +/// Selects workers randomly with uniform distribution among healthy workers. +#[derive(Debug, Default)] +pub struct RandomPolicy; + +impl RandomPolicy { + pub fn new() -> Self { + Self + } +} + +impl LoadBalancingPolicy for RandomPolicy { + fn select_worker( + &self, + workers: &[Box], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + let mut rng = rand::rng(); + let random_idx = rng.random_range(0..healthy_indices.len()); + let worker = workers[healthy_indices[random_idx]].url(); + + RouterMetrics::record_processed_request(worker); + RouterMetrics::record_policy_decision(self.name(), worker); + Some(healthy_indices[random_idx]) + } + + fn name(&self) -> &'static str { + "random" + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + use std::collections::HashMap; + + #[test] + fn test_random_selection() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w3:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Test multiple selections to ensure randomness + let mut counts = HashMap::new(); + for _ in 0..100 { + if let Some(idx) = policy.select_worker(&workers, None) { + *counts.entry(idx).or_insert(0) += 1; + } + } + + // All workers should be selected at least once + assert_eq!(counts.len(), 3); + assert!(counts.values().all(|&count| count > 0)); + } + + #[test] + fn test_random_with_unhealthy_workers() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Mark first worker as unhealthy + workers[0].set_healthy(false); + + // Should always select the healthy worker (index 1) + for _ in 0..10 { + assert_eq!(policy.select_worker(&workers, None), Some(1)); + } + } + + #[test] + fn test_random_no_healthy_workers() { + let policy = RandomPolicy::new(); + let workers: Vec> = vec![Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + ))]; + + workers[0].set_healthy(false); + assert_eq!(policy.select_worker(&workers, None), None); + } +} diff --git a/sgl-router/src/policies/round_robin.rs b/sgl-router/src/policies/round_robin.rs new file mode 100644 index 000000000..fcb60233f --- /dev/null +++ b/sgl-router/src/policies/round_robin.rs @@ -0,0 +1,140 @@ +//! Round-robin load balancing policy + +use super::{get_healthy_worker_indices, LoadBalancingPolicy}; +use crate::core::Worker; +use crate::metrics::RouterMetrics; +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Round-robin selection policy +/// +/// Selects workers in sequential order, cycling through all healthy workers. +#[derive(Debug, Default)] +pub struct RoundRobinPolicy { + counter: AtomicUsize, +} + +impl RoundRobinPolicy { + pub fn new() -> Self { + Self { + counter: AtomicUsize::new(0), + } + } +} + +impl LoadBalancingPolicy for RoundRobinPolicy { + fn select_worker( + &self, + workers: &[Box], + _request_text: Option<&str>, + ) -> Option { + let healthy_indices = get_healthy_worker_indices(workers); + + if healthy_indices.is_empty() { + return None; + } + + // Get and increment counter atomically + let count = self.counter.fetch_add(1, Ordering::Relaxed); + let selected_idx = count % healthy_indices.len(); + let worker = workers[healthy_indices[selected_idx]].url(); + + RouterMetrics::record_processed_request(worker); + RouterMetrics::record_policy_decision(self.name(), worker); + Some(healthy_indices[selected_idx]) + } + + fn name(&self) -> &'static str { + "round_robin" + } + + fn reset(&self) { + self.counter.store(0, Ordering::Relaxed); + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + + #[test] + fn test_round_robin_selection() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w3:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Should select workers in order: 0, 1, 2, 0, 1, 2, ... + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + } + + #[test] + fn test_round_robin_with_unhealthy_workers() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w3:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Mark middle worker as unhealthy + workers[1].set_healthy(false); + + // Should skip unhealthy worker: 0, 2, 0, 2, ... + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(2)); + } + + #[test] + fn test_round_robin_reset() { + let policy = RoundRobinPolicy::new(); + let workers: Vec> = vec![ + Box::new(BasicWorker::new( + "http://w1:8000".to_string(), + WorkerType::Regular, + )), + Box::new(BasicWorker::new( + "http://w2:8000".to_string(), + WorkerType::Regular, + )), + ]; + + // Advance the counter + assert_eq!(policy.select_worker(&workers, None), Some(0)); + assert_eq!(policy.select_worker(&workers, None), Some(1)); + + // Reset should start from beginning + policy.reset(); + assert_eq!(policy.select_worker(&workers, None), Some(0)); + } +} diff --git a/sgl-router/src/proto/sglang_scheduler.proto b/sgl-router/src/proto/sglang_scheduler.proto new file mode 100644 index 000000000..1ea2855a4 --- /dev/null +++ b/sgl-router/src/proto/sglang_scheduler.proto @@ -0,0 +1,541 @@ +syntax = "proto3"; + +package sglang.grpc.scheduler; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/struct.proto"; + +// Service definition for SGLang scheduler communication +// This protocol bridges the Rust router and Python scheduler +service SglangScheduler { + // Initialize connection and get model info + rpc Initialize(InitializeRequest) returns (InitializeResponse); + + // Submit a generation request (supports streaming) + rpc Generate(GenerateRequest) returns (stream GenerateResponse); + + // Submit an embedding request + rpc Embed(EmbedRequest) returns (EmbedResponse); + + // Health check and metrics + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // Abort a running request + rpc Abort(AbortRequest) returns (AbortResponse); + + // Flush KV cache + rpc FlushCache(FlushCacheRequest) returns (FlushCacheResponse); +} + +// ===================== +// Common Types +// ===================== + +// Sampling parameters matching SGLang's SamplingParams +message SamplingParams { + float temperature = 1; + float top_p = 2; + int32 top_k = 3; + float min_p = 4; + float frequency_penalty = 5; + float presence_penalty = 6; + float repetition_penalty = 7; + + int32 max_new_tokens = 8; + repeated string stop = 9; + repeated int32 stop_token_ids = 10; + bool skip_special_tokens = 11; + bool spaces_between_special_tokens = 12; + + // Structured generation + oneof constraint { + string regex = 13; + string json_schema = 14; + string ebnf_grammar = 15; + } + + // LoRA adapter + string lora_path = 16; + + // Speculative decoding + int32 n = 17; // Number of samples + + // Token healing + bool token_healing = 18; + + // Additional parameters + int32 min_new_tokens = 19; + bool ignore_eos = 20; + bool no_stop_trim = 21; + int32 stream_interval = 22; + map logit_bias = 23; + string structural_tag = 24; + + // Custom parameters for extensibility + google.protobuf.Struct custom_params = 25; +} + +// Session parameters for continual prompting +message SessionParams { + string session_id = 1; + string request_id = 2; + int32 offset = 3; + bool replace = 4; + bool drop_previous_output = 5; +} + +// Disaggregated serving parameters +message DisaggregatedParams { + string bootstrap_host = 1; + int32 bootstrap_port = 2; + int32 bootstrap_room = 3; +} + +// ===================== +// Initialize +// ===================== + +message InitializeRequest { + string client_id = 1; + string client_version = 2; + + // Operating mode + enum Mode { + REGULAR = 0; // Normal mode with local scheduler + PREFILL = 1; // Prefill-only mode for disaggregated serving + DECODE = 2; // Decode-only mode for disaggregated serving + } + Mode mode = 3; +} + +message InitializeResponse { + bool success = 1; + string scheduler_version = 2; + + // Model information + ModelInfo model_info = 3; + + // Server capabilities + ServerCapabilities capabilities = 4; + + // Error message if success is false + string error_message = 5; +} + +message ModelInfo { + string model_name = 1; + int32 max_context_length = 2; + int32 vocab_size = 3; + bool supports_tool_calling = 4; + bool supports_vision = 5; + repeated string special_tokens = 6; + + // Additional model metadata + string model_type = 7; + int32 num_layers = 8; + int32 hidden_size = 9; + int32 num_attention_heads = 10; + int32 num_key_value_heads = 11; + + // Tokenizer info + string tokenizer_type = 12; + repeated int32 eos_token_ids = 13; + int32 pad_token_id = 14; + int32 bos_token_id = 15; +} + +message ServerCapabilities { + bool continuous_batching = 1; + bool disaggregated_serving = 2; + bool speculative_decoding = 3; + int32 max_batch_size = 4; + int32 max_num_batched_tokens = 5; + int32 max_prefill_tokens = 6; + string attention_backend = 7; // "flashinfer", "triton", "torch" + + // Additional capabilities + bool supports_lora = 8; + bool supports_grammar = 9; + bool supports_multimodal = 10; + repeated string supported_modalities = 11; // ["image", "video", "audio"] + bool supports_custom_logit_processor = 12; + bool supports_session = 13; + + // Hardware info + int32 num_gpus = 14; + string gpu_type = 15; + int64 total_gpu_memory = 16; + + // Parallelism info + int32 tensor_parallel_size = 17; + int32 pipeline_parallel_size = 18; + int32 data_parallel_size = 19; +} + +// ===================== +// Generate Request +// ===================== + +message GenerateRequest { + string request_id = 1; + + // Input can be either text or tokenized + oneof input { + string text = 2; + TokenizedInput tokenized = 3; + } + + // Multimodal inputs + MultimodalInputs mm_inputs = 4; + + // Generation parameters + SamplingParams sampling_params = 5; + + // Return options + bool return_logprob = 6; + int32 logprob_start_len = 7; + int32 top_logprobs_num = 8; + repeated int32 token_ids_logprob = 9; + bool return_hidden_states = 10; + + // Session management + SessionParams session_params = 11; + + // For disaggregated serving + DisaggregatedParams disaggregated_params = 12; + + // Custom logit processor (serialized) + string custom_logit_processor = 13; + + // Request metadata + google.protobuf.Timestamp timestamp = 14; + bool log_metrics = 15; + + // Input embeddings (alternative to text/tokens) + repeated float input_embeds = 16; + + // LoRA adapter ID (if pre-loaded) + string lora_id = 17; + + // Data parallel routing + int32 data_parallel_rank = 18; + + // For load balancing + int32 dp_balance_id = 19; +} + +message TokenizedInput { + string original_text = 1; // For reference + repeated int32 input_ids = 2; +} + +message MultimodalInputs { + // Simplified multimodal handling - actual data processed by tokenizer + repeated string image_urls = 1; + repeated string video_urls = 2; + repeated string audio_urls = 3; + + // Pre-processed multimodal features (if available) + google.protobuf.Struct processed_features = 4; + + // Raw data for direct processing + repeated bytes image_data = 5; + repeated bytes video_data = 6; + repeated bytes audio_data = 7; + + // Modality metadata + repeated string modalities = 8; +} + +// ===================== +// Generate Response +// ===================== + +message GenerateResponse { + string request_id = 1; + + // Response type + oneof response { + GenerateStreamChunk chunk = 2; + GenerateComplete complete = 3; + GenerateError error = 4; + } +} + +message GenerateStreamChunk { + // Generated token + int32 token_id = 1; + string text = 2; + + // Cumulative counts + int32 prompt_tokens = 3; + int32 completion_tokens = 4; + int32 cached_tokens = 5; + + // Logprobs (if requested) + LogProbs logprobs = 6; + + // Hidden states (if requested) + repeated float hidden_states = 7; + + // Metadata + float generation_time = 8; // Time to generate this token + int32 queue_time = 9; // Time spent in queue +} + +message GenerateComplete { + // Final output + repeated int32 output_ids = 1; + string output_text = 2; + + // Finish reason + enum FinishReason { + // The model generated a stop sequence. + STOP = 0; + // The model reached the maximum generation length. + LENGTH = 1; + // The model generated an end-of-sequence (EOS) token. + EOS_TOKEN = 2; + // The model generated a user-provided stop string. + STOP_STR = 3; + // The request was aborted by the user or system. + ABORT = 4; + } + FinishReason finish_reason = 3; + + // Final counts + int32 prompt_tokens = 4; + int32 completion_tokens = 5; + int32 cached_tokens = 6; + + // Performance metrics + float total_generation_time = 7; + float time_to_first_token = 8; + float tokens_per_second = 9; + + // Spec decode metrics + int32 spec_verify_count = 10; + + // All logprobs if requested + repeated LogProbs all_logprobs = 11; + + // All hidden states if requested + repeated HiddenStates all_hidden_states = 12; +} + +message GenerateError { + string message = 1; + string http_status_code = 2; + string details = 3; +} + +message LogProbs { + repeated float token_logprobs = 1; + repeated int32 token_ids = 2; + + // Top logprobs at each position + repeated TopLogProbs top_logprobs = 3; + + // Decoded text for tokens + repeated string token_texts = 4; +} + +message TopLogProbs { + repeated float values = 1; + repeated int32 token_ids = 2; + repeated string token_texts = 3; +} + +message HiddenStates { + repeated float values = 1; + int32 layer = 2; + int32 position = 3; +} + +// ===================== +// Embedding Request +// ===================== + +message EmbedRequest { + string request_id = 1; + + oneof input { + string text = 2; + TokenizedInput tokenized = 3; + } + + // Multimodal inputs + MultimodalInputs mm_inputs = 4; + + // Dummy sampling params for compatibility + // EmbedRequest doesn't use sampling_params + SamplingParams sampling_params = 5; + + bool log_metrics = 6; + + // Token type IDs for models that require them + repeated int32 token_type_ids = 7; + + // Data parallel routing + int32 data_parallel_rank = 8; + + // For cross-encoder requests + bool is_cross_encoder = 9; + repeated string texts = 10; // For cross-encoder batch +} + +message EmbedResponse { + string request_id = 1; + + oneof response { + EmbedComplete complete = 2; + EmbedError error = 3; + } +} + +message EmbedComplete { + repeated float embedding = 1; + int32 prompt_tokens = 2; + int32 cached_tokens = 3; + + // Additional metadata + int32 embedding_dim = 4; + float generation_time = 5; + + // For batch embeddings + repeated Embedding batch_embeddings = 6; +} + +message Embedding { + repeated float values = 1; + int32 index = 2; +} + +message EmbedError { + string message = 1; + string code = 2; + string details = 3; +} + +// ===================== +// Management Operations +// ===================== + +message HealthCheckRequest { + bool include_detailed_metrics = 1; +} + +message HealthCheckResponse { + bool healthy = 1; + + // Current load metrics + int32 num_requests_running = 2; + int32 num_requests_waiting = 3; + float gpu_cache_usage = 4; + float gpu_memory_usage = 5; + + // KV cache metrics + int32 kv_cache_total_blocks = 6; + int32 kv_cache_used_blocks = 7; + float kv_cache_hit_rate = 8; + + // Additional metrics + int32 num_grammar_queue_requests = 9; + float generation_throughput = 10; // tokens/sec + float average_queue_time = 11; // seconds + float average_generation_time = 12; // seconds + + // System metrics + float cpu_usage = 13; + int64 memory_usage = 14; + + // Disaggregation metrics + int32 num_prefill_requests = 15; + int32 num_decode_requests = 16; + + // Detailed metrics (optional) + google.protobuf.Struct detailed_metrics = 17; +} + +message AbortRequest { + string request_id = 1; + string reason = 2; +} + +message AbortResponse { + bool success = 1; + string message = 2; +} + +message FlushCacheRequest { + bool flush_all = 1; + repeated string session_ids = 2; // Flush specific sessions +} + +message FlushCacheResponse { + bool success = 1; + int32 num_entries_flushed = 2; + int64 memory_freed = 3; // bytes + string message = 4; +} + +// ===================== +// Additional Operations (Future) +// ===================== + +// Load LoRA adapter +message LoadLoRARequest { + string adapter_id = 1; + string adapter_path = 2; + int32 rank = 3; +} + +message LoadLoRAResponse { + bool success = 1; + string adapter_id = 2; + string message = 3; +} + +// Unload LoRA adapter +message UnloadLoRARequest { + string adapter_id = 1; +} + +message UnloadLoRAResponse { + bool success = 1; + string message = 2; +} + +// Update weights +message UpdateWeightsRequest { + oneof source { + string disk_path = 1; + bytes tensor_data = 2; + string remote_url = 3; + } + string weight_name = 4; +} + +message UpdateWeightsResponse { + bool success = 1; + string message = 2; +} + +// Get internal state for debugging +message GetInternalStateRequest { + repeated string state_keys = 1; +} + +message GetInternalStateResponse { + google.protobuf.Struct state = 1; +} + +// Set internal state for testing +message SetInternalStateRequest { + google.protobuf.Struct state = 1; +} + +message SetInternalStateResponse { + bool success = 1; + string message = 2; +} diff --git a/sgl-router/src/protocols/mod.rs b/sgl-router/src/protocols/mod.rs new file mode 100644 index 000000000..5243c645f --- /dev/null +++ b/sgl-router/src/protocols/mod.rs @@ -0,0 +1,5 @@ +// Protocol definitions and validation for various LLM APIs +// This module provides a structured approach to handling different API protocols + +pub mod spec; +pub mod validation; diff --git a/sgl-router/src/protocols/spec.rs b/sgl-router/src/protocols/spec.rs new file mode 100644 index 000000000..43e60244c --- /dev/null +++ b/sgl-router/src/protocols/spec.rs @@ -0,0 +1,2681 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::HashMap; + +// # Protocol Specifications +// +// This module contains all protocol definitions for OpenAI and SGLang APIs. +// +// ## Table of Contents +// +// 1. **OPENAI SPEC - Chat Completions API** +// - Message Types +// - Response Format Types +// - Tool/Function Types +// - Streaming Delta Types +// - Request/Response structures +// +// 2. **OPENAI SPEC - Completions API** +// - Request/Response structures +// - Streaming support +// +// 3. **OPENAI SPEC - Responses API** +// - Tool Definitions +// - Reasoning Configuration +// - Input/Output Items +// - Service Tier & Tool Choice +// - Request/Response structures +// +// 4. **OPENAI SPEC - Common** +// - Shared Request Components +// - Tool Choice Types +// - Usage Tracking +// - Logprobs Types +// - Error Response Types +// +// 5. **SGLANG SPEC - GENERATE API** +// - Generate Parameters +// - Sampling Parameters +// - Request/Response structures +// +// 6. **SGLANG SPEC - RERANK API** +// - Request/Response structures +// +// 7. **COMMON** +// - GenerationRequest trait +// - StringOrArray & LoRAPath types +// - Helper functions + +// ================================================================== +// = OPENAI SPEC - Chat Completions API = +// ================================================================== + +// ============= Message Types ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ChatMessage { + System { + role: String, + content: String, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + User { + role: String, // "user" + content: UserMessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + Assistant { + role: String, // "assistant" + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tool_calls: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + function_call: Option, + /// Reasoning content for O1-style models (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + }, + Tool { + role: String, // "tool" + content: String, + tool_call_id: String, + }, + Function { + role: String, // "function" + content: String, + name: String, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum UserMessageContent { + Text(String), + Parts(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum ContentPart { + #[serde(rename = "text")] + Text { text: String }, + #[serde(rename = "image_url")] + ImageUrl { image_url: ImageUrl }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ImageUrl { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, // "auto", "low", or "high" +} + +// ============= Response Format Types ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum ResponseFormat { + #[serde(rename = "text")] + Text, + #[serde(rename = "json_object")] + JsonObject, + #[serde(rename = "json_schema")] + JsonSchema { json_schema: JsonSchemaFormat }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +// ============= Streaming Delta Types ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatMessageDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub function_call: Option, + /// Reasoning content delta for O1-style models (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCallDelta { + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "type")] + pub tool_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +// ============= Request ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionRequest { + /// ID of the model to use + pub model: String, + + /// A list of messages comprising the conversation so far + pub messages: Vec, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// An alternative to sampling with temperature + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// How many chat completion choices to generate for each input message + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + + /// If set, partial message deltas will be sent + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + + /// The maximum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + + /// An upper bound for the number of tokens that can be generated for a completion + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub presence_penalty: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_penalty: Option, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// A unique identifier representing your end-user + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// If specified, our system will make a best effort to sample deterministically + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + /// Whether to return log probabilities of the output tokens + #[serde(default)] + pub logprobs: bool, + + /// An integer between 0 and 20 specifying the number of most likely tokens to return + #[serde(skip_serializing_if = "Option::is_none")] + pub top_logprobs: Option, + + /// An object specifying the format that the model must output + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + + /// A list of tools the model may call + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + + /// Controls which (if any) tool is called by the model + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// Whether to enable parallel function calling during tool use + #[serde(skip_serializing_if = "Option::is_none")] + pub parallel_tool_calls: Option, + + /// Deprecated: use tools instead + #[serde(skip_serializing_if = "Option::is_none")] + pub functions: Option>, + + /// Deprecated: use tool_choice instead + #[serde(skip_serializing_if = "Option::is_none")] + pub function_call: Option, + + // ============= SGLang Extensions ============= + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Continue generating from final assistant message + #[serde(default)] + pub continue_final_message: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + // ============= SGLang Extensions ============= + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Separate reasoning content from final answer (O1-style models) + #[serde(default = "default_true")] + pub separate_reasoning: bool, + + /// Stream reasoning tokens during generation + #[serde(default = "default_true")] + pub stream_reasoning: bool, + + /// Chat template kwargs + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template_kwargs: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, +} + +impl GenerationRequest for ChatCompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Extract text from messages for routing decisions + self.messages + .iter() + .filter_map(|msg| match msg { + ChatMessage::System { content, .. } => Some(content.clone()), + ChatMessage::User { content, .. } => match content { + UserMessageContent::Text(text) => Some(text.clone()), + UserMessageContent::Parts(parts) => { + let texts: Vec = parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text.clone()), + _ => None, + }) + .collect(); + Some(texts.join(" ")) + } + }, + ChatMessage::Assistant { + content, + reasoning_content, + .. + } => { + // Combine content and reasoning content for routing decisions + let main_content = content.clone().unwrap_or_default(); + let reasoning = reasoning_content.clone().unwrap_or_default(); + if main_content.is_empty() && reasoning.is_empty() { + None + } else { + Some(format!("{} {}", main_content, reasoning).trim().to_string()) + } + } + ChatMessage::Tool { content, .. } => Some(content.clone()), + ChatMessage::Function { content, .. } => Some(content.clone()), + }) + .collect::>() + .join(" ") + } +} + +// ============= Regular Response ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionResponse { + pub id: String, + pub object: String, // "chat.completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatChoice { + pub index: u32, + pub message: ChatMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "tool_calls", "content_filter", "function_call" + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer + /// Hidden states from the model (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + pub hidden_states: Option>, +} + +// ============= Streaming Response ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionStreamResponse { + pub id: String, + pub object: String, // "chat.completion.chunk" + pub created: u64, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatStreamChoice { + pub index: u32, + pub delta: ChatMessageDelta, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, +} + +// ================================================================== +// = OPENAI SPEC - Completions API = +// ================================================================== +// Completions API request types (v1/completions) - DEPRECATED but still supported + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionRequest { + /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang) + pub model: String, + + /// The prompt(s) to generate completions for + pub prompt: StringOrArray, + + /// The suffix that comes after a completion of inserted text + #[serde(skip_serializing_if = "Option::is_none")] + pub suffix: Option, + + /// The maximum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// An alternative to sampling with temperature (nucleus sampling) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// How many completions to generate for each prompt + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + + /// Whether to stream back partial progress + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// Include the log probabilities on the logprobs most likely tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + + /// Echo back the prompt in addition to the completion + #[serde(default)] + pub echo: bool, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub presence_penalty: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_penalty: Option, + + /// Generates best_of completions server-side and returns the "best" + #[serde(skip_serializing_if = "Option::is_none")] + pub best_of: Option, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// A unique identifier representing your end-user + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// If specified, our system will make a best effort to sample deterministically + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + // ============= SGLang Extensions ============= + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// JSON schema constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + // ============= SGLang Extensions ============= + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Additional fields including bootstrap info for PD routing + #[serde(flatten)] + pub other: serde_json::Map, +} + +impl GenerationRequest for CompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + match &self.prompt { + StringOrArray::String(s) => s.clone(), + StringOrArray::Array(v) => v.join(" "), + } + } +} + +// ============= Regular Response ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "content_filter", etc. + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer + /// Hidden states from the model (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + pub hidden_states: Option>, +} + +// ============= Streaming Response ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub choices: Vec, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, +} + +// ================================================================== +// = OPENAI SPEC - Responses API = +// ================================================================== + +// ============= Tool Definitions ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseTool { + #[serde(rename = "type")] + pub r#type: ResponseToolType, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponseToolType { + WebSearchPreview, + CodeInterpreter, +} + +// ============= Reasoning Configuration ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseReasoningParam { + #[serde(default = "default_reasoning_effort")] + pub effort: Option, +} + +fn default_reasoning_effort() -> Option { + Some(ReasoningEffort::Medium) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + Low, + Medium, + High, +} + +// ============= Input/Output Items ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseInputOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + summary: Vec, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_tool_call")] + FunctionToolCall { + id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseContentPart { + #[serde(rename = "output_text")] + OutputText { + text: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + annotations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + logprobs: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseReasoningContent { + #[serde(rename = "reasoning_text")] + ReasoningText { text: String }, +} + +// ============= Output Items for Response ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + status: String, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + summary: Vec, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_tool_call")] + FunctionToolCall { + id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + status: String, + }, +} + +// ============= Service Tier ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ServiceTier { + Auto, + Default, + Flex, + Scale, + Priority, +} + +impl Default for ServiceTier { + fn default() -> Self { + Self::Auto + } +} + +// ============= Truncation ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Truncation { + Auto, + Disabled, +} + +impl Default for Truncation { + fn default() -> Self { + Self::Disabled + } +} + +// ============= Response Status ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponseStatus { + Queued, + InProgress, + Completed, + Failed, + Cancelled, +} + +// ============= Include Fields ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum IncludeField { + #[serde(rename = "code_interpreter_call.outputs")] + CodeInterpreterCallOutputs, + #[serde(rename = "computer_call_output.output.image_url")] + ComputerCallOutputImageUrl, + #[serde(rename = "file_search_call.results")] + FileSearchCallResults, + #[serde(rename = "message.input_image.image_url")] + MessageInputImageUrl, + #[serde(rename = "message.output_text.logprobs")] + MessageOutputTextLogprobs, + #[serde(rename = "reasoning.encrypted_content")] + ReasoningEncryptedContent, +} + +// ============= Usage Info ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct UsageInfo { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PromptTokenUsageInfo { + pub cached_tokens: u32, +} + +// ============= Response Usage Format ============= + +/// OpenAI Responses API usage format (different from standard UsageInfo) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseUsage { + pub input_tokens: u32, + pub output_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens_details: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct InputTokensDetails { + pub cached_tokens: u32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OutputTokensDetails { + pub reasoning_tokens: u32, +} + +impl UsageInfo { + /// Convert to OpenAI Responses API format + pub fn to_response_usage(&self) -> ResponseUsage { + ResponseUsage { + input_tokens: self.prompt_tokens, + output_tokens: self.completion_tokens, + total_tokens: self.total_tokens, + input_tokens_details: self.prompt_tokens_details.as_ref().map(|details| { + InputTokensDetails { + cached_tokens: details.cached_tokens, + } + }), + output_tokens_details: self.reasoning_tokens.map(|tokens| OutputTokensDetails { + reasoning_tokens: tokens, + }), + } + } +} + +impl From for ResponseUsage { + fn from(usage: UsageInfo) -> Self { + usage.to_response_usage() + } +} + +impl ResponseUsage { + /// Convert back to standard UsageInfo format + pub fn to_usage_info(&self) -> UsageInfo { + UsageInfo { + prompt_tokens: self.input_tokens, + completion_tokens: self.output_tokens, + total_tokens: self.total_tokens, + reasoning_tokens: self + .output_tokens_details + .as_ref() + .map(|details| details.reasoning_tokens), + prompt_tokens_details: self.input_tokens_details.as_ref().map(|details| { + PromptTokenUsageInfo { + cached_tokens: details.cached_tokens, + } + }), + } + } +} + +fn generate_request_id() -> String { + format!("resp_{}", uuid::Uuid::new_v4().simple()) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponsesRequest { + // ============= Core OpenAI API fields ============= + /// Run the request in the background + #[serde(default)] + pub background: bool, + + /// Fields to include in the response + #[serde(skip_serializing_if = "Option::is_none")] + pub include: Option>, + + /// Input content - can be string or structured items + pub input: ResponseInput, + + /// System instructions for the model + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option, + + /// Maximum number of output tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + + /// Maximum number of tool calls + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tool_calls: Option, + + /// Additional metadata + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + + /// Model to use (optional to match vLLM) + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + + /// Whether to enable parallel tool calls + #[serde(default = "default_true")] + pub parallel_tool_calls: bool, + + /// ID of previous response to continue from + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + + /// Reasoning configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + + /// Service tier + #[serde(default)] + pub service_tier: ServiceTier, + + /// Whether to store the response + #[serde(default = "default_true")] + pub store: bool, + + /// Whether to stream the response + #[serde(default)] + pub stream: bool, + + /// Temperature for sampling + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// Tool choice behavior + #[serde(default)] + pub tool_choice: ToolChoice, + + /// Available tools + #[serde(default)] + pub tools: Vec, + + /// Number of top logprobs to return + #[serde(default)] + pub top_logprobs: u32, + + /// Top-p sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// Truncation behavior + #[serde(default)] + pub truncation: Truncation, + + /// User identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + // ============= SGLang Extensions ============= + /// Request ID + #[serde(default = "generate_request_id")] + pub request_id: String, + + /// Request priority + #[serde(default)] + pub priority: i32, + + /// Frequency penalty + #[serde(default)] + pub frequency_penalty: f32, + + /// Presence penalty + #[serde(default)] + pub presence_penalty: f32, + + /// Stop sequences + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + + /// Top-k sampling parameter + #[serde(default = "default_top_k")] + pub top_k: i32, + + /// Min-p sampling parameter + #[serde(default)] + pub min_p: f32, + + /// Repetition penalty + #[serde(default = "default_repetition_penalty")] + pub repetition_penalty: f32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ResponseInput { + Text(String), + Items(Vec), +} + +fn default_top_k() -> i32 { + -1 +} + +fn default_repetition_penalty() -> f32 { + 1.0 +} + +impl ResponsesRequest { + /// Default sampling parameters + const DEFAULT_TEMPERATURE: f32 = 0.7; + const DEFAULT_TOP_P: f32 = 1.0; + + /// Convert to sampling parameters for generation + pub fn to_sampling_params( + &self, + default_max_tokens: u32, + default_params: Option>, + ) -> HashMap { + let mut params = HashMap::new(); + + // Use max_output_tokens if available + let max_tokens = if let Some(max_output) = self.max_output_tokens { + std::cmp::min(max_output, default_max_tokens) + } else { + default_max_tokens + }; + + // Avoid exceeding context length by minus 1 token + let max_tokens = max_tokens.saturating_sub(1); + + // Temperature + let temperature = self.temperature.unwrap_or_else(|| { + default_params + .as_ref() + .and_then(|p| p.get("temperature")) + .and_then(|v| v.as_f64()) + .map(|v| v as f32) + .unwrap_or(Self::DEFAULT_TEMPERATURE) + }); + + // Top-p + let top_p = self.top_p.unwrap_or_else(|| { + default_params + .as_ref() + .and_then(|p| p.get("top_p")) + .and_then(|v| v.as_f64()) + .map(|v| v as f32) + .unwrap_or(Self::DEFAULT_TOP_P) + }); + + params.insert( + "max_new_tokens".to_string(), + serde_json::Value::Number(serde_json::Number::from(max_tokens)), + ); + params.insert( + "temperature".to_string(), + serde_json::Value::Number(serde_json::Number::from_f64(temperature as f64).unwrap()), + ); + params.insert( + "top_p".to_string(), + serde_json::Value::Number(serde_json::Number::from_f64(top_p as f64).unwrap()), + ); + params.insert( + "frequency_penalty".to_string(), + serde_json::Value::Number( + serde_json::Number::from_f64(self.frequency_penalty as f64).unwrap(), + ), + ); + params.insert( + "presence_penalty".to_string(), + serde_json::Value::Number( + serde_json::Number::from_f64(self.presence_penalty as f64).unwrap(), + ), + ); + params.insert( + "top_k".to_string(), + serde_json::Value::Number(serde_json::Number::from(self.top_k)), + ); + params.insert( + "min_p".to_string(), + serde_json::Value::Number(serde_json::Number::from_f64(self.min_p as f64).unwrap()), + ); + params.insert( + "repetition_penalty".to_string(), + serde_json::Value::Number( + serde_json::Number::from_f64(self.repetition_penalty as f64).unwrap(), + ), + ); + + if let Some(ref stop) = self.stop { + match serde_json::to_value(stop) { + Ok(value) => params.insert("stop".to_string(), value), + Err(_) => params.insert("stop".to_string(), serde_json::Value::Null), + }; + } + + // Apply any additional default parameters + if let Some(default_params) = default_params { + for (key, value) in default_params { + params.entry(key).or_insert(value); + } + } + + params + } +} + +impl GenerationRequest for ResponsesRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + self.model.as_deref() + } + + fn extract_text_for_routing(&self) -> String { + match &self.input { + ResponseInput::Text(text) => text.clone(), + ResponseInput::Items(items) => items + .iter() + .filter_map(|item| match item { + ResponseInputOutputItem::Message { content, .. } => { + let texts: Vec = content + .iter() + .map(|part| match part { + ResponseContentPart::OutputText { text, .. } => text.clone(), + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::Reasoning { content, .. } => { + let texts: Vec = content + .iter() + .map(|part| match part { + ResponseReasoningContent::ReasoningText { text } => text.clone(), + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::FunctionToolCall { arguments, .. } => { + Some(arguments.clone()) + } + }) + .collect::>() + .join(" "), + } + } +} + +fn generate_response_id() -> String { + format!("resp_{}", uuid::Uuid::new_v4().simple()) +} + +fn current_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_secs() as i64 +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponsesResponse { + /// Response ID + #[serde(default = "generate_response_id")] + pub id: String, + + /// Object type + #[serde(default = "default_object_type")] + pub object: String, + + /// Creation timestamp + #[serde(default = "current_timestamp")] + pub created_at: i64, + + /// Model name + pub model: String, + + /// Output items + #[serde(default)] + pub output: Vec, + + /// Response status + pub status: ResponseStatus, + + /// Usage information + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + + /// Whether parallel tool calls are enabled + #[serde(default = "default_true")] + pub parallel_tool_calls: bool, + + /// Tool choice setting + #[serde(default = "default_tool_choice")] + pub tool_choice: String, + + /// Available tools + #[serde(default)] + pub tools: Vec, +} + +fn default_object_type() -> String { + "response".to_string() +} + +fn default_tool_choice() -> String { + "auto".to_string() +} + +impl ResponsesResponse { + /// Create a response from a request + #[allow(clippy::too_many_arguments)] + pub fn from_request( + request: &ResponsesRequest, + _sampling_params: &HashMap, + model_name: String, + created_time: i64, + output: Vec, + status: ResponseStatus, + usage: Option, + ) -> Self { + Self { + id: request.request_id.clone(), + object: "response".to_string(), + created_at: created_time, + model: model_name, + output, + status, + usage, + parallel_tool_calls: request.parallel_tool_calls, + tool_choice: match &request.tool_choice { + ToolChoice::Value(ToolChoiceValue::Auto) => "auto".to_string(), + ToolChoice::Value(ToolChoiceValue::Required) => "required".to_string(), + ToolChoice::Value(ToolChoiceValue::None) => "none".to_string(), + ToolChoice::Function { .. } => "function".to_string(), + }, + tools: request.tools.clone(), + } + } + + /// Create a new response with default values + pub fn new(request_id: String, model: String, status: ResponseStatus) -> Self { + Self { + id: request_id, + object: "response".to_string(), + created_at: current_timestamp(), + model, + output: Vec::new(), + status, + usage: None, + parallel_tool_calls: true, + tool_choice: "auto".to_string(), + tools: Vec::new(), + } + } + + /// Add an output item to the response + pub fn add_output(&mut self, item: ResponseOutputItem) { + self.output.push(item); + } + + /// Set the usage information + pub fn set_usage(&mut self, usage: UsageInfo) { + self.usage = Some(usage); + } + + /// Update the status + pub fn set_status(&mut self, status: ResponseStatus) { + self.status = status; + } + + /// Check if the response is complete + pub fn is_complete(&self) -> bool { + matches!(self.status, ResponseStatus::Completed) + } + + /// Check if the response is in progress + pub fn is_in_progress(&self) -> bool { + matches!(self.status, ResponseStatus::InProgress) + } + + /// Check if the response failed + pub fn is_failed(&self) -> bool { + matches!(self.status, ResponseStatus::Failed) + } + + /// Check if the response was cancelled + pub fn is_cancelled(&self) -> bool { + matches!(self.status, ResponseStatus::Cancelled) + } + + /// Check if the response is queued + pub fn is_queued(&self) -> bool { + matches!(self.status, ResponseStatus::Queued) + } + + /// Convert usage to OpenAI Responses API format + pub fn usage_in_response_format(&self) -> Option { + self.usage.as_ref().map(|usage| usage.to_response_usage()) + } + + /// Get the response as a JSON value with usage in response format + pub fn to_response_format(&self) -> serde_json::Value { + let mut response = serde_json::to_value(self).unwrap_or(serde_json::Value::Null); + + // Convert usage to response format if present + if let Some(usage) = &self.usage { + if let Ok(usage_value) = serde_json::to_value(usage.to_response_usage()) { + response["usage"] = usage_value; + } + } + + response + } +} + +// ============= Helper Functions ============= + +impl ResponseOutputItem { + /// Create a new message output item + pub fn new_message( + id: String, + role: String, + content: Vec, + status: String, + ) -> Self { + Self::Message { + id, + role, + content, + status, + } + } + + /// Create a new reasoning output item + pub fn new_reasoning( + id: String, + summary: Vec, + content: Vec, + status: Option, + ) -> Self { + Self::Reasoning { + id, + summary, + content, + status, + } + } + + /// Create a new function tool call output item + pub fn new_function_tool_call( + id: String, + name: String, + arguments: String, + output: Option, + status: String, + ) -> Self { + Self::FunctionToolCall { + id, + name, + arguments, + output, + status, + } + } +} + +impl ResponseContentPart { + /// Create a new text content part + pub fn new_text( + text: String, + annotations: Vec, + logprobs: Option, + ) -> Self { + Self::OutputText { + text, + annotations, + logprobs, + } + } +} + +impl ResponseReasoningContent { + /// Create a new reasoning text content + pub fn new_reasoning_text(text: String) -> Self { + Self::ReasoningText { text } + } +} + +impl UsageInfo { + /// Create a new usage info with token counts + pub fn new(prompt_tokens: u32, completion_tokens: u32, reasoning_tokens: Option) -> Self { + Self { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + reasoning_tokens, + prompt_tokens_details: None, + } + } + + /// Create usage info with cached token details + pub fn new_with_cached( + prompt_tokens: u32, + completion_tokens: u32, + reasoning_tokens: Option, + cached_tokens: u32, + ) -> Self { + Self { + prompt_tokens, + completion_tokens, + total_tokens: prompt_tokens + completion_tokens, + reasoning_tokens, + prompt_tokens_details: Some(PromptTokenUsageInfo { cached_tokens }), + } + } +} + +// ================================================================== +// = OPENAI SPEC - Common = +// ================================================================== + +// ============= Shared Request Components ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct StreamOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub include_usage: Option, +} + +// ============= Tool Choice Types ============= + +/// Tool choice value for simple string options +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolChoiceValue { + Auto, + Required, + None, +} + +/// Tool choice for both Chat Completion and Responses APIs +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ToolChoice { + Value(ToolChoiceValue), + Function { + #[serde(rename = "type")] + tool_type: String, // "function" + function: FunctionChoice, + }, +} + +impl Default for ToolChoice { + fn default() -> Self { + Self::Value(ToolChoiceValue::Auto) + } +} + +/// Function choice specification for ToolChoice::Function +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionChoice { + pub name: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Tool { + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: Function, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Function { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: Value, // JSON Schema +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCall { + pub id: String, + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: FunctionCallResponse, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum FunctionCall { + None, + Auto, + Function { name: String }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallResponse { + pub name: String, + #[serde(default)] + pub arguments: Option, // JSON string +} + +// ============= Usage Tracking ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub completion_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionTokensDetails { + pub reasoning_tokens: Option, +} + +// ============= Logprobs Types ============= + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct LogProbs { + pub tokens: Vec, + pub token_logprobs: Vec>, + pub top_logprobs: Vec>>, + pub text_offset: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatLogProbs { + pub content: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatLogProbsContent { + pub token: String, + pub logprob: f32, + pub bytes: Option>, + pub top_logprobs: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TopLogProb { + pub token: String, + pub logprob: f32, + pub bytes: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorResponse { + pub error: ErrorDetail, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorDetail { + pub message: String, + #[serde(rename = "type")] + pub error_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub param: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, +} + +// ================================================================== +// = SGLANG SPEC - GENERATE API = +// ================================================================== + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum InputIds { + Single(Vec), + Batch(Vec>), +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct GenerateParameters { + #[serde(skip_serializing_if = "Option::is_none")] + pub best_of: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub decoder_input_details: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub details: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub do_sample: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_new_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub return_full_text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub truncate: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub typical_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub watermark: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct SamplingParams { + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_new_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub presence_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ignore_eos: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_special_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub min_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub min_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_stop_trim: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GenerateRequest { + /// The prompt to generate from (OpenAI style) + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt: Option, + + /// Text input - SGLang native format + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + + /// Input IDs for tokenized input + #[serde(skip_serializing_if = "Option::is_none")] + pub input_ids: Option, + + /// Generation parameters + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parameters: Option, + + /// Sampling parameters (sglang style) + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_params: Option, + + /// Whether to stream the response + #[serde(default)] + pub stream: bool, + + /// Whether to return logprobs + #[serde(default)] + pub return_logprob: bool, + + // ============= SGLang Extensions ============= + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Request ID for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, +} + +impl GenerationRequest for GenerateRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + // Generate requests typically don't have a model field + None + } + + fn extract_text_for_routing(&self) -> String { + // Check fields in priority order: text, prompt, inputs + if let Some(ref text) = self.text { + return text.clone(); + } + + if let Some(ref prompt) = self.prompt { + return match prompt { + StringOrArray::String(s) => s.clone(), + StringOrArray::Array(v) => v.join(" "), + }; + } + + if let Some(ref input_ids) = self.input_ids { + return match input_ids { + InputIds::Single(ids) => ids + .iter() + .map(|&id| id.to_string()) + .collect::>() + .join(" "), + InputIds::Batch(batches) => batches + .iter() + .flat_map(|batch| batch.iter().map(|&id| id.to_string())) + .collect::>() + .join(" "), + }; + } + + // No text input found + String::new() + } +} + +// ================================================================== +// = SGLANG SPEC - RERANK API = +// ================================================================== + +// Constants for rerank API +pub const DEFAULT_MODEL_NAME: &str = "default"; + +/// Rerank request for scoring documents against a query +/// Used for RAG systems and document relevance scoring +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankRequest { + /// The query text to rank documents against + pub query: String, + + /// List of documents to be ranked + pub documents: Vec, + + /// Model to use for reranking + #[serde(default = "default_model_name")] + pub model: String, + + /// Maximum number of documents to return (optional) + pub top_k: Option, + + /// Whether to return documents in addition to scores + #[serde(default = "default_return_documents")] + pub return_documents: bool, + + // SGLang specific extensions + /// Request ID for tracking + pub rid: Option, + + /// User identifier + pub user: Option, +} + +fn default_model_name() -> String { + DEFAULT_MODEL_NAME.to_string() +} + +fn default_return_documents() -> bool { + true +} + +/// Individual rerank result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResult { + /// Relevance score for the document + pub score: f32, + + /// The document text (if return_documents was true) + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, + + /// Original index of the document in the request + pub index: usize, + + /// Additional metadata about the ranking + #[serde(skip_serializing_if = "Option::is_none")] + pub meta_info: Option>, +} + +/// Rerank response containing sorted results +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResponse { + /// Ranked results sorted by score (highest first) + pub results: Vec, + + /// Model used for reranking + pub model: String, + + /// Usage information + pub usage: Option, + + /// Response object type + #[serde(default = "default_rerank_object")] + pub object: String, + + /// Response ID + pub id: String, + + /// Creation timestamp + pub created: i64, +} + +fn default_rerank_object() -> String { + "rerank".to_string() +} + +/// V1 API compatibility format for rerank requests +/// Matches Python's V1RerankReqInput +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct V1RerankReqInput { + pub query: String, + pub documents: Vec, +} + +/// Convert V1RerankReqInput to RerankRequest +impl From for RerankRequest { + fn from(v1: V1RerankReqInput) -> Self { + RerankRequest { + query: v1.query, + documents: v1.documents, + model: default_model_name(), + top_k: None, + return_documents: true, + rid: None, + user: None, + } + } +} + +/// Implementation of GenerationRequest trait for RerankRequest +impl GenerationRequest for RerankRequest { + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn is_stream(&self) -> bool { + false // Reranking doesn't support streaming + } + + fn extract_text_for_routing(&self) -> String { + self.query.clone() + } +} + +impl RerankRequest { + pub fn validate(&self) -> Result<(), String> { + // Validate query is not empty + if self.query.trim().is_empty() { + return Err("Query cannot be empty".to_string()); + } + + // Validate documents list + if self.documents.is_empty() { + return Err("Documents list cannot be empty".to_string()); + } + + // Validate top_k if specified + if let Some(k) = self.top_k { + if k == 0 { + return Err("top_k must be greater than 0".to_string()); + } + if k > self.documents.len() { + // This is allowed but we log a warning + tracing::warn!( + "top_k ({}) is greater than number of documents ({})", + k, + self.documents.len() + ); + } + } + + Ok(()) + } + + /// Get the effective top_k value + pub fn effective_top_k(&self) -> usize { + self.top_k.unwrap_or(self.documents.len()) + } +} + +impl RerankResponse { + pub fn new(results: Vec, model: String, request_id: String) -> Self { + RerankResponse { + results, + model, + usage: None, + object: default_rerank_object(), + id: request_id, + created: current_timestamp(), + } + } + + /// Sort results by score in descending order + pub fn sort_by_score(&mut self) { + self.results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + } + + /// Apply top_k limit to results + pub fn apply_top_k(&mut self, k: usize) { + self.results.truncate(k); + } +} + +// ================================================================== +// = COMMON = +// ================================================================== + +/// Helper function for serde default value +pub fn default_true() -> bool { + true +} + +/// Common trait for all generation requests across different APIs +pub trait GenerationRequest: Send + Sync { + /// Check if the request is for streaming + fn is_stream(&self) -> bool; + + /// Get the model name if specified + fn get_model(&self) -> Option<&str>; + + /// Extract text content for routing decisions + fn extract_text_for_routing(&self) -> String; +} + +/// Helper type for string or array of strings +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum StringOrArray { + String(String), + Array(Vec), +} +impl StringOrArray { + /// Get the number of items in the StringOrArray + pub fn len(&self) -> usize { + match self { + StringOrArray::String(_) => 1, + StringOrArray::Array(arr) => arr.len(), + } + } + + /// Check if the StringOrArray is empty + pub fn is_empty(&self) -> bool { + match self { + StringOrArray::String(s) => s.is_empty(), + StringOrArray::Array(arr) => arr.is_empty(), + } + } + + /// Convert to a vector of strings + pub fn to_vec(&self) -> Vec { + match self { + StringOrArray::String(s) => vec![s.clone()], + StringOrArray::Array(arr) => arr.clone(), + } + } +} + +/// LoRA adapter path - can be single path or batch of paths (SGLang extension) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum LoRAPath { + Single(Option), + Batch(Vec>), +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json; + + // ================================================================== + // = RERANK REQUEST TESTS = + // ================================================================== + + #[test] + fn test_rerank_request_serialization() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + model: "test-model".to_string(), + top_k: Some(5), + return_documents: true, + rid: Some(StringOrArray::String("req-123".to_string())), + user: Some("user-456".to_string()), + }; + + let serialized = serde_json::to_string(&request).unwrap(); + let deserialized: RerankRequest = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(deserialized.query, request.query); + assert_eq!(deserialized.documents, request.documents); + assert_eq!(deserialized.model, request.model); + assert_eq!(deserialized.top_k, request.top_k); + assert_eq!(deserialized.return_documents, request.return_documents); + assert_eq!(deserialized.rid, request.rid); + assert_eq!(deserialized.user, request.user); + } + + #[test] + fn test_rerank_request_deserialization_with_defaults() { + let json = r#"{ + "query": "test query", + "documents": ["doc1", "doc2"] + }"#; + + let request: RerankRequest = serde_json::from_str(json).unwrap(); + + assert_eq!(request.query, "test query"); + assert_eq!(request.documents, vec!["doc1", "doc2"]); + assert_eq!(request.model, default_model_name()); + assert_eq!(request.top_k, None); + assert!(request.return_documents); + assert_eq!(request.rid, None); + assert_eq!(request.user, None); + } + + #[test] + fn test_rerank_request_validation_success() { + let request = RerankRequest { + query: "valid query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + model: "test-model".to_string(), + top_k: Some(2), + return_documents: true, + rid: None, + user: None, + }; + + assert!(request.validate().is_ok()); + } + + #[test] + fn test_rerank_request_validation_empty_query() { + let request = RerankRequest { + query: "".to_string(), + documents: vec!["doc1".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + let result = request.validate(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "Query cannot be empty"); + } + + #[test] + fn test_rerank_request_validation_whitespace_query() { + let request = RerankRequest { + query: " ".to_string(), + documents: vec!["doc1".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + let result = request.validate(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "Query cannot be empty"); + } + + #[test] + fn test_rerank_request_validation_empty_documents() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec![], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + let result = request.validate(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "Documents list cannot be empty"); + } + + #[test] + fn test_rerank_request_validation_top_k_zero() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + model: "test-model".to_string(), + top_k: Some(0), + return_documents: true, + rid: None, + user: None, + }; + + let result = request.validate(); + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "top_k must be greater than 0"); + } + + #[test] + fn test_rerank_request_validation_top_k_greater_than_docs() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + model: "test-model".to_string(), + top_k: Some(5), + return_documents: true, + rid: None, + user: None, + }; + + // This should pass but log a warning + assert!(request.validate().is_ok()); + } + + #[test] + fn test_rerank_request_effective_top_k() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string(), "doc3".to_string()], + model: "test-model".to_string(), + top_k: Some(2), + return_documents: true, + rid: None, + user: None, + }; + + assert_eq!(request.effective_top_k(), 2); + } + + #[test] + fn test_rerank_request_effective_top_k_none() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string(), "doc3".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + assert_eq!(request.effective_top_k(), 3); + } + + // ================================================================== + // = RERANK RESPONSE TESTS = + // ================================================================== + + #[test] + fn test_rerank_response_creation() { + let results = vec![ + RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }, + RerankResult { + score: 0.6, + document: Some("doc2".to_string()), + index: 1, + meta_info: None, + }, + ]; + + let response = RerankResponse::new( + results.clone(), + "test-model".to_string(), + "req-123".to_string(), + ); + + assert_eq!(response.results.len(), 2); + assert_eq!(response.model, "test-model"); + assert_eq!(response.id, "req-123"); + assert_eq!(response.object, "rerank"); + assert!(response.created > 0); + } + + #[test] + fn test_rerank_response_serialization() { + let results = vec![RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }]; + + let response = + RerankResponse::new(results, "test-model".to_string(), "req-123".to_string()); + + let serialized = serde_json::to_string(&response).unwrap(); + let deserialized: RerankResponse = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(deserialized.results.len(), response.results.len()); + assert_eq!(deserialized.model, response.model); + assert_eq!(deserialized.id, response.id); + assert_eq!(deserialized.object, response.object); + } + + #[test] + fn test_rerank_response_sort_by_score() { + let results = vec![ + RerankResult { + score: 0.6, + document: Some("doc2".to_string()), + index: 1, + meta_info: None, + }, + RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }, + RerankResult { + score: 0.4, + document: Some("doc3".to_string()), + index: 2, + meta_info: None, + }, + ]; + + let mut response = + RerankResponse::new(results, "test-model".to_string(), "req-123".to_string()); + + response.sort_by_score(); + + assert_eq!(response.results[0].score, 0.8); + assert_eq!(response.results[0].index, 0); + assert_eq!(response.results[1].score, 0.6); + assert_eq!(response.results[1].index, 1); + assert_eq!(response.results[2].score, 0.4); + assert_eq!(response.results[2].index, 2); + } + + #[test] + fn test_rerank_response_apply_top_k() { + let results = vec![ + RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }, + RerankResult { + score: 0.6, + document: Some("doc2".to_string()), + index: 1, + meta_info: None, + }, + RerankResult { + score: 0.4, + document: Some("doc3".to_string()), + index: 2, + meta_info: None, + }, + ]; + + let mut response = + RerankResponse::new(results, "test-model".to_string(), "req-123".to_string()); + + response.apply_top_k(2); + + assert_eq!(response.results.len(), 2); + assert_eq!(response.results[0].score, 0.8); + assert_eq!(response.results[1].score, 0.6); + } + + #[test] + fn test_rerank_response_apply_top_k_larger_than_results() { + let results = vec![RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }]; + + let mut response = + RerankResponse::new(results, "test-model".to_string(), "req-123".to_string()); + + response.apply_top_k(5); + + assert_eq!(response.results.len(), 1); + } + + // ================================================================== + // = RERANK RESULT TESTS = + // ================================================================== + + #[test] + fn test_rerank_result_serialization() { + let result = RerankResult { + score: 0.85, + document: Some("test document".to_string()), + index: 42, + meta_info: Some(HashMap::from([ + ("confidence".to_string(), Value::String("high".to_string())), + ( + "processing_time".to_string(), + Value::Number(serde_json::Number::from(150)), + ), + ])), + }; + + let serialized = serde_json::to_string(&result).unwrap(); + let deserialized: RerankResult = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(deserialized.score, result.score); + assert_eq!(deserialized.document, result.document); + assert_eq!(deserialized.index, result.index); + assert_eq!(deserialized.meta_info, result.meta_info); + } + + #[test] + fn test_rerank_result_serialization_without_document() { + let result = RerankResult { + score: 0.85, + document: None, + index: 42, + meta_info: None, + }; + + let serialized = serde_json::to_string(&result).unwrap(); + let deserialized: RerankResult = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(deserialized.score, result.score); + assert_eq!(deserialized.document, result.document); + assert_eq!(deserialized.index, result.index); + assert_eq!(deserialized.meta_info, result.meta_info); + } + + // ================================================================== + // = V1 COMPATIBILITY TESTS = + // ================================================================== + + #[test] + fn test_v1_rerank_req_input_serialization() { + let v1_input = V1RerankReqInput { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + }; + + let serialized = serde_json::to_string(&v1_input).unwrap(); + let deserialized: V1RerankReqInput = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(deserialized.query, v1_input.query); + assert_eq!(deserialized.documents, v1_input.documents); + } + + #[test] + fn test_v1_to_rerank_request_conversion() { + let v1_input = V1RerankReqInput { + query: "test query".to_string(), + documents: vec!["doc1".to_string(), "doc2".to_string()], + }; + + let request: RerankRequest = v1_input.into(); + + assert_eq!(request.query, "test query"); + assert_eq!(request.documents, vec!["doc1", "doc2"]); + assert_eq!(request.model, default_model_name()); + assert_eq!(request.top_k, None); + assert!(request.return_documents); + assert_eq!(request.rid, None); + assert_eq!(request.user, None); + } + + // ================================================================== + // = GENERATION REQUEST TRAIT TESTS = + // ================================================================== + + #[test] + fn test_rerank_request_generation_request_trait() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + assert_eq!(request.get_model(), Some("test-model")); + assert!(!request.is_stream()); + assert_eq!(request.extract_text_for_routing(), "test query"); + } + + // ================================================================== + // = EDGE CASES AND STRESS TESTS = + // ================================================================== + + #[test] + fn test_rerank_request_very_long_query() { + let long_query = "a".repeat(100000); + let request = RerankRequest { + query: long_query, + documents: vec!["doc1".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: None, + user: None, + }; + + assert!(request.validate().is_ok()); + } + + #[test] + fn test_rerank_request_many_documents() { + let documents: Vec = (0..1000).map(|i| format!("doc{}", i)).collect(); + let request = RerankRequest { + query: "test query".to_string(), + documents, + model: "test-model".to_string(), + top_k: Some(100), + return_documents: true, + rid: None, + user: None, + }; + + assert!(request.validate().is_ok()); + assert_eq!(request.effective_top_k(), 100); + } + + #[test] + fn test_rerank_request_special_characters() { + let request = RerankRequest { + query: "query with émojis 🚀 and unicode: 测试".to_string(), + documents: vec![ + "doc with émojis 🎉".to_string(), + "doc with unicode: 测试".to_string(), + ], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: Some(StringOrArray::String("req-🚀-123".to_string())), + user: Some("user-🎉-456".to_string()), + }; + + assert!(request.validate().is_ok()); + } + + #[test] + fn test_rerank_request_rid_array() { + let request = RerankRequest { + query: "test query".to_string(), + documents: vec!["doc1".to_string()], + model: "test-model".to_string(), + top_k: None, + return_documents: true, + rid: Some(StringOrArray::Array(vec![ + "req1".to_string(), + "req2".to_string(), + ])), + user: None, + }; + + assert!(request.validate().is_ok()); + } + + #[test] + fn test_rerank_response_with_usage_info() { + let results = vec![RerankResult { + score: 0.8, + document: Some("doc1".to_string()), + index: 0, + meta_info: None, + }]; + + let mut response = + RerankResponse::new(results, "test-model".to_string(), "req-123".to_string()); + + response.usage = Some(UsageInfo { + prompt_tokens: 100, + completion_tokens: 50, + total_tokens: 150, + reasoning_tokens: None, + prompt_tokens_details: None, + }); + + let serialized = serde_json::to_string(&response).unwrap(); + let deserialized: RerankResponse = serde_json::from_str(&serialized).unwrap(); + + assert!(deserialized.usage.is_some()); + let usage = deserialized.usage.unwrap(); + assert_eq!(usage.prompt_tokens, 100); + assert_eq!(usage.completion_tokens, 50); + assert_eq!(usage.total_tokens, 150); + } + + // ================================================================== + // = INTEGRATION TESTS = + // ================================================================== + + #[test] + fn test_full_rerank_workflow() { + // Create request + let request = RerankRequest { + query: "machine learning".to_string(), + documents: vec![ + "Introduction to machine learning algorithms".to_string(), + "Deep learning for computer vision".to_string(), + "Natural language processing basics".to_string(), + "Statistics and probability theory".to_string(), + ], + model: "rerank-model".to_string(), + top_k: Some(2), + return_documents: true, + rid: Some(StringOrArray::String("req-123".to_string())), + user: Some("user-456".to_string()), + }; + + // Validate request + assert!(request.validate().is_ok()); + + // Simulate reranking results (in real scenario, this would come from the model) + let results = vec![ + RerankResult { + score: 0.95, + document: Some("Introduction to machine learning algorithms".to_string()), + index: 0, + meta_info: None, + }, + RerankResult { + score: 0.87, + document: Some("Deep learning for computer vision".to_string()), + index: 1, + meta_info: None, + }, + RerankResult { + score: 0.72, + document: Some("Natural language processing basics".to_string()), + index: 2, + meta_info: None, + }, + RerankResult { + score: 0.45, + document: Some("Statistics and probability theory".to_string()), + index: 3, + meta_info: None, + }, + ]; + + // Create response + let mut response = RerankResponse::new( + results, + request.model.clone(), + request + .rid + .as_ref() + .and_then(|r| match r { + StringOrArray::String(s) => Some(s.clone()), + StringOrArray::Array(arr) => arr.first().cloned(), + }) + .unwrap_or_else(|| "unknown".to_string()), + ); + + // Sort by score + response.sort_by_score(); + + // Apply top_k + response.apply_top_k(request.effective_top_k()); + + // Verify results + assert_eq!(response.results.len(), 2); + assert_eq!(response.results[0].score, 0.95); + assert_eq!(response.results[0].index, 0); + assert_eq!(response.results[1].score, 0.87); + assert_eq!(response.results[1].index, 1); + assert_eq!(response.model, "rerank-model"); + + // Serialize and deserialize + let serialized = serde_json::to_string(&response).unwrap(); + let deserialized: RerankResponse = serde_json::from_str(&serialized).unwrap(); + assert_eq!(deserialized.results.len(), 2); + assert_eq!(deserialized.model, response.model); + } +} diff --git a/sgl-router/src/protocols/validation.rs b/sgl-router/src/protocols/validation.rs new file mode 100644 index 000000000..460ce2148 --- /dev/null +++ b/sgl-router/src/protocols/validation.rs @@ -0,0 +1,1164 @@ +// Core validation infrastructure for API parameter validation + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +// Import types from spec module +use crate::protocols::spec::{ + ChatCompletionRequest, ChatMessage, ResponseFormat, StringOrArray, UserMessageContent, +}; + +/// Validation constants for OpenAI API parameters +pub mod constants { + /// Temperature range: 0.0 to 2.0 (OpenAI spec) + pub const TEMPERATURE_RANGE: (f32, f32) = (0.0, 2.0); + + /// Top-p range: 0.0 to 1.0 (exclusive of 0.0) + pub const TOP_P_RANGE: (f32, f32) = (0.0, 1.0); + + /// Presence penalty range: -2.0 to 2.0 (OpenAI spec) + pub const PRESENCE_PENALTY_RANGE: (f32, f32) = (-2.0, 2.0); + + /// Frequency penalty range: -2.0 to 2.0 (OpenAI spec) + pub const FREQUENCY_PENALTY_RANGE: (f32, f32) = (-2.0, 2.0); + + /// Logprobs range for completions API: 0 to 5 + pub const LOGPROBS_RANGE: (u32, u32) = (0, 5); + + /// Top logprobs range for chat completions: 0 to 20 + pub const TOP_LOGPROBS_RANGE: (u32, u32) = (0, 20); + + /// Maximum number of stop sequences allowed + pub const MAX_STOP_SEQUENCES: usize = 4; + + /// SGLang-specific validation constants + pub mod sglang { + /// Min-p range: 0.0 to 1.0 (SGLang extension) + pub const MIN_P_RANGE: (f32, f32) = (0.0, 1.0); + + /// Top-k minimum value: -1 to disable, otherwise positive + pub const TOP_K_MIN: i32 = -1; + + /// Repetition penalty range: 0.0 to 2.0 (SGLang extension) + /// 1.0 = no penalty, >1.0 = discourage repetition, <1.0 = encourage repetition + pub const REPETITION_PENALTY_RANGE: (f32, f32) = (0.0, 2.0); + } +} + +/// Core validation error types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ValidationError { + /// Parameter value out of valid range + OutOfRange { + parameter: String, + value: String, + min: String, + max: String, + }, + /// Invalid parameter value format or type + InvalidValue { + parameter: String, + value: String, + reason: String, + }, + /// Cross-parameter validation failure + ConflictingParameters { + parameter1: String, + parameter2: String, + reason: String, + }, + /// Required parameter missing + MissingRequired { parameter: String }, + /// Too many items in array parameter + TooManyItems { + parameter: String, + count: usize, + max: usize, + }, + /// Custom validation error + Custom(String), +} + +impl Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ValidationError::OutOfRange { + parameter, + value, + min, + max, + } => { + write!( + f, + "Parameter '{}' must be between {} and {}, got {}", + parameter, min, max, value + ) + } + ValidationError::InvalidValue { + parameter, + value, + reason, + } => { + write!( + f, + "Invalid value for parameter '{}': {} ({})", + parameter, value, reason + ) + } + ValidationError::ConflictingParameters { + parameter1, + parameter2, + reason, + } => { + write!( + f, + "Conflicting parameters '{}' and '{}': {}", + parameter1, parameter2, reason + ) + } + ValidationError::MissingRequired { parameter } => { + write!(f, "Required parameter '{}' is missing", parameter) + } + ValidationError::TooManyItems { + parameter, + count, + max, + } => { + write!( + f, + "Parameter '{}' has too many items: {} (maximum: {})", + parameter, count, max + ) + } + ValidationError::Custom(msg) => write!(f, "{}", msg), + } + } +} + +impl std::error::Error for ValidationError {} + +/// Core validation utility functions +pub mod utils { + use super::*; + + /// Validate that a numeric value is within the specified range (inclusive) + pub fn validate_range( + value: T, + range: &(T, T), + param_name: &str, + ) -> Result + where + T: PartialOrd + Display + Copy, + { + if value >= range.0 && value <= range.1 { + Ok(value) + } else { + Err(ValidationError::OutOfRange { + parameter: param_name.to_string(), + value: value.to_string(), + min: range.0.to_string(), + max: range.1.to_string(), + }) + } + } + + /// Validate that a positive number is actually positive + pub fn validate_positive(value: T, param_name: &str) -> Result + where + T: PartialOrd + Display + Copy + Default, + { + if value > T::default() { + Ok(value) + } else { + Err(ValidationError::InvalidValue { + parameter: param_name.to_string(), + value: value.to_string(), + reason: "must be positive".to_string(), + }) + } + } + + /// Validate that an array doesn't exceed maximum length + pub fn validate_max_items( + items: &[T], + max_count: usize, + param_name: &str, + ) -> Result<(), ValidationError> { + if items.len() <= max_count { + Ok(()) + } else { + Err(ValidationError::TooManyItems { + parameter: param_name.to_string(), + count: items.len(), + max: max_count, + }) + } + } + + /// Validate that a required parameter is present + pub fn validate_required<'a, T>( + value: &'a Option, + param_name: &str, + ) -> Result<&'a T, ValidationError> { + value + .as_ref() + .ok_or_else(|| ValidationError::MissingRequired { + parameter: param_name.to_string(), + }) + } + + /// Validate top_k parameter (SGLang extension) + pub fn validate_top_k(top_k: i32) -> Result { + if top_k == constants::sglang::TOP_K_MIN || top_k > 0 { + Ok(top_k) + } else { + Err(ValidationError::InvalidValue { + parameter: "top_k".to_string(), + value: top_k.to_string(), + reason: "must be -1 (disabled) or positive".to_string(), + }) + } + } + + /// Generic validation function for sampling options + pub fn validate_sampling_options( + request: &T, + ) -> Result<(), ValidationError> { + // Validate temperature (0.0 to 2.0) + if let Some(temp) = request.get_temperature() { + validate_range(temp, &constants::TEMPERATURE_RANGE, "temperature")?; + } + + // Validate top_p (0.0 to 1.0) + if let Some(top_p) = request.get_top_p() { + validate_range(top_p, &constants::TOP_P_RANGE, "top_p")?; + } + + // Validate frequency_penalty (-2.0 to 2.0) + if let Some(freq_penalty) = request.get_frequency_penalty() { + validate_range( + freq_penalty, + &constants::FREQUENCY_PENALTY_RANGE, + "frequency_penalty", + )?; + } + + // Validate presence_penalty (-2.0 to 2.0) + if let Some(pres_penalty) = request.get_presence_penalty() { + validate_range( + pres_penalty, + &constants::PRESENCE_PENALTY_RANGE, + "presence_penalty", + )?; + } + + Ok(()) + } + + /// Generic validation function for stop conditions + pub fn validate_stop_conditions( + request: &T, + ) -> Result<(), ValidationError> { + if let Some(stop) = request.get_stop_sequences() { + match stop { + StringOrArray::String(s) => { + if s.is_empty() { + return Err(ValidationError::InvalidValue { + parameter: "stop".to_string(), + value: "empty string".to_string(), + reason: "stop sequences cannot be empty".to_string(), + }); + } + } + StringOrArray::Array(arr) => { + validate_max_items(arr, constants::MAX_STOP_SEQUENCES, "stop")?; + for (i, s) in arr.iter().enumerate() { + if s.is_empty() { + return Err(ValidationError::InvalidValue { + parameter: format!("stop[{}]", i), + value: "empty string".to_string(), + reason: "stop sequences cannot be empty".to_string(), + }); + } + } + } + } + } + + Ok(()) + } + + /// Generic validation function for token limits + pub fn validate_token_limits( + request: &T, + ) -> Result<(), ValidationError> { + // Validate max_tokens if provided + if let Some(max_tokens) = request.get_max_tokens() { + validate_positive(max_tokens, "max_tokens")?; + } + + // Validate min_tokens if provided (SGLang extension) + if let Some(min_tokens) = request.get_min_tokens() { + validate_positive(min_tokens, "min_tokens")?; + } + + Ok(()) + } + + /// Generic validation function for logprobs + pub fn validate_logprobs( + request: &T, + ) -> Result<(), ValidationError> { + // Validate logprobs (completions API - 0 to 5) + if let Some(logprobs) = request.get_logprobs() { + validate_range(logprobs, &constants::LOGPROBS_RANGE, "logprobs")?; + } + + // Validate top_logprobs (chat API - 0 to 20) + if let Some(top_logprobs) = request.get_top_logprobs() { + validate_range(top_logprobs, &constants::TOP_LOGPROBS_RANGE, "top_logprobs")?; + } + + Ok(()) + } + + /// Generic cross-parameter validation + pub fn validate_cross_parameters( + request: &T, + ) -> Result<(), ValidationError> { + // Check min_tokens <= max_tokens if both are specified + if let (Some(min_tokens), Some(max_tokens)) = + (request.get_min_tokens(), request.get_max_tokens()) + { + if min_tokens > max_tokens { + return Err(ValidationError::ConflictingParameters { + parameter1: "min_tokens".to_string(), + parameter2: "max_tokens".to_string(), + reason: format!( + "min_tokens ({}) cannot be greater than max_tokens ({})", + min_tokens, max_tokens + ), + }); + } + } + + Ok(()) + } + + /// Validate conflicting structured output constraints + pub fn validate_conflicting_parameters( + param1_name: &str, + param1_value: bool, + param2_name: &str, + param2_value: bool, + reason: &str, + ) -> Result<(), ValidationError> { + if param1_value && param2_value { + return Err(ValidationError::ConflictingParameters { + parameter1: param1_name.to_string(), + parameter2: param2_name.to_string(), + reason: reason.to_string(), + }); + } + Ok(()) + } + + /// Validate that only one option from a set is active + pub fn validate_mutually_exclusive_options( + options: &[(&str, bool)], + error_msg: &str, + ) -> Result<(), ValidationError> { + let active_count = options.iter().filter(|(_, is_active)| *is_active).count(); + if active_count > 1 { + return Err(ValidationError::Custom(error_msg.to_string())); + } + Ok(()) + } + + /// Generic validation for SGLang extensions + pub fn validate_sglang_extensions( + request: &T, + ) -> Result<(), ValidationError> { + // Validate top_k (-1 to disable, or positive) + if let Some(top_k) = request.get_top_k() { + validate_top_k(top_k)?; + } + + // Validate min_p (0.0 to 1.0) + if let Some(min_p) = request.get_min_p() { + validate_range(min_p, &constants::sglang::MIN_P_RANGE, "min_p")?; + } + + // Validate repetition_penalty (0.0 to 2.0) + if let Some(rep_penalty) = request.get_repetition_penalty() { + validate_range( + rep_penalty, + &constants::sglang::REPETITION_PENALTY_RANGE, + "repetition_penalty", + )?; + } + + Ok(()) + } + + /// Generic validation for n parameter (number of completions) + pub fn validate_completion_count( + request: &T, + ) -> Result<(), ValidationError> { + const N_RANGE: (u32, u32) = (1, 10); + + if let Some(n) = request.get_n() { + validate_range(n, &N_RANGE, "n")?; + } + + Ok(()) + } + + /// Validate that an array is not empty + pub fn validate_non_empty_array( + items: &[T], + param_name: &str, + ) -> Result<(), ValidationError> { + if items.is_empty() { + return Err(ValidationError::MissingRequired { + parameter: param_name.to_string(), + }); + } + Ok(()) + } + + /// Validate common request parameters that are shared across all API types + pub fn validate_common_request_params(request: &T) -> Result<(), ValidationError> + where + T: SamplingOptionsProvider + + StopConditionsProvider + + TokenLimitsProvider + + LogProbsProvider + + SGLangExtensionsProvider + + CompletionCountProvider + + ?Sized, + { + // Validate all standard parameters + validate_sampling_options(request)?; + validate_stop_conditions(request)?; + validate_token_limits(request)?; + validate_logprobs(request)?; + + // Validate SGLang extensions and completion count + validate_sglang_extensions(request)?; + validate_completion_count(request)?; + + // Perform cross-parameter validation + validate_cross_parameters(request)?; + + Ok(()) + } +} + +/// Core validation traits for different parameter categories +pub trait SamplingOptionsProvider { + /// Get temperature parameter + fn get_temperature(&self) -> Option; + + /// Get top_p parameter + fn get_top_p(&self) -> Option; + + /// Get frequency penalty parameter + fn get_frequency_penalty(&self) -> Option; + + /// Get presence penalty parameter + fn get_presence_penalty(&self) -> Option; +} + +/// Trait for validating stop conditions +pub trait StopConditionsProvider { + /// Get stop sequences + fn get_stop_sequences(&self) -> Option<&StringOrArray>; +} + +/// Trait for validating token limits +pub trait TokenLimitsProvider { + /// Get maximum tokens parameter + fn get_max_tokens(&self) -> Option; + + /// Get minimum tokens parameter (SGLang extension) + fn get_min_tokens(&self) -> Option; +} + +/// Trait for validating logprobs parameters +pub trait LogProbsProvider { + /// Get logprobs parameter (completions API) + fn get_logprobs(&self) -> Option; + + /// Get top_logprobs parameter (chat API) + fn get_top_logprobs(&self) -> Option; +} + +/// Trait for SGLang-specific extensions +pub trait SGLangExtensionsProvider { + /// Get top_k parameter + fn get_top_k(&self) -> Option { + None + } + + /// Get min_p parameter + fn get_min_p(&self) -> Option { + None + } + + /// Get repetition_penalty parameter + fn get_repetition_penalty(&self) -> Option { + None + } +} + +/// Trait for n parameter (number of completions) +pub trait CompletionCountProvider { + /// Get n parameter + fn get_n(&self) -> Option { + None + } +} + +/// Comprehensive validation trait that combines all validation aspects +pub trait ValidatableRequest: + SamplingOptionsProvider + + StopConditionsProvider + + TokenLimitsProvider + + LogProbsProvider + + SGLangExtensionsProvider + + CompletionCountProvider +{ + /// Perform comprehensive validation of the entire request + fn validate(&self) -> Result<(), ValidationError> { + // Use the common validation function + utils::validate_common_request_params(self) + } +} + +// ================================================================== +// = OPENAI CHAT COMPLETION VALIDATION = +// ================================================================== + +impl SamplingOptionsProvider for ChatCompletionRequest { + fn get_temperature(&self) -> Option { + self.temperature + } + fn get_top_p(&self) -> Option { + self.top_p + } + fn get_frequency_penalty(&self) -> Option { + self.frequency_penalty + } + fn get_presence_penalty(&self) -> Option { + self.presence_penalty + } +} + +impl StopConditionsProvider for ChatCompletionRequest { + fn get_stop_sequences(&self) -> Option<&StringOrArray> { + self.stop.as_ref() + } +} + +impl TokenLimitsProvider for ChatCompletionRequest { + fn get_max_tokens(&self) -> Option { + // Prefer max_completion_tokens over max_tokens if both are set + self.max_completion_tokens.or(self.max_tokens) + } + + fn get_min_tokens(&self) -> Option { + self.min_tokens + } +} + +impl LogProbsProvider for ChatCompletionRequest { + fn get_logprobs(&self) -> Option { + // For chat API, logprobs is a boolean, return 1 if true for validation purposes + if self.logprobs { + Some(1) + } else { + None + } + } + + fn get_top_logprobs(&self) -> Option { + self.top_logprobs + } +} + +impl SGLangExtensionsProvider for ChatCompletionRequest { + fn get_top_k(&self) -> Option { + self.top_k + } + + fn get_min_p(&self) -> Option { + self.min_p + } + + fn get_repetition_penalty(&self) -> Option { + self.repetition_penalty + } +} + +impl CompletionCountProvider for ChatCompletionRequest { + fn get_n(&self) -> Option { + self.n + } +} + +impl ChatCompletionRequest { + /// Validate message-specific requirements + pub fn validate_messages(&self) -> Result<(), ValidationError> { + // Ensure messages array is not empty + utils::validate_non_empty_array(&self.messages, "messages")?; + + // Validate message content is not empty + for (i, msg) in self.messages.iter().enumerate() { + if let ChatMessage::User { content, .. } = msg { + match content { + UserMessageContent::Text(text) if text.is_empty() => { + return Err(ValidationError::InvalidValue { + parameter: format!("messages[{}].content", i), + value: "empty".to_string(), + reason: "message content cannot be empty".to_string(), + }); + } + UserMessageContent::Parts(parts) if parts.is_empty() => { + return Err(ValidationError::InvalidValue { + parameter: format!("messages[{}].content", i), + value: "empty array".to_string(), + reason: "message content parts cannot be empty".to_string(), + }); + } + _ => {} + } + } + } + + Ok(()) + } + + /// Validate response format if specified + pub fn validate_response_format(&self) -> Result<(), ValidationError> { + if let Some(ResponseFormat::JsonSchema { json_schema }) = &self.response_format { + if json_schema.name.is_empty() { + return Err(ValidationError::InvalidValue { + parameter: "response_format.json_schema.name".to_string(), + value: "empty".to_string(), + reason: "JSON schema name cannot be empty".to_string(), + }); + } + } + Ok(()) + } + + /// Validate chat API specific logprobs requirements + pub fn validate_chat_logprobs(&self) -> Result<(), ValidationError> { + // In chat API, if logprobs=true, top_logprobs must be specified + if self.logprobs && self.top_logprobs.is_none() { + return Err(ValidationError::MissingRequired { + parameter: "top_logprobs".to_string(), + }); + } + + // If top_logprobs is specified, logprobs should be true + if self.top_logprobs.is_some() && !self.logprobs { + return Err(ValidationError::InvalidValue { + parameter: "logprobs".to_string(), + value: "false".to_string(), + reason: "must be true when top_logprobs is specified".to_string(), + }); + } + + Ok(()) + } + + /// Validate cross-parameter relationships specific to chat completions + pub fn validate_chat_cross_parameters(&self) -> Result<(), ValidationError> { + // Validate that both max_tokens and max_completion_tokens aren't set + utils::validate_conflicting_parameters( + "max_tokens", + self.max_tokens.is_some(), + "max_completion_tokens", + self.max_completion_tokens.is_some(), + "cannot specify both max_tokens and max_completion_tokens", + )?; + + // Validate that tools and functions aren't both specified (deprecated) + utils::validate_conflicting_parameters( + "tools", + self.tools.is_some(), + "functions", + self.functions.is_some(), + "functions is deprecated, use tools instead", + )?; + + // Validate structured output constraints don't conflict with JSON response format + let has_json_format = matches!( + self.response_format, + Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) + ); + + utils::validate_conflicting_parameters( + "response_format", + has_json_format, + "regex", + self.regex.is_some(), + "cannot use regex constraint with JSON response format", + )?; + + utils::validate_conflicting_parameters( + "response_format", + has_json_format, + "ebnf", + self.ebnf.is_some(), + "cannot use EBNF constraint with JSON response format", + )?; + + // Only one structured output constraint should be active + let structured_constraints = [ + ("regex", self.regex.is_some()), + ("ebnf", self.ebnf.is_some()), + ( + "json_schema", + matches!( + self.response_format, + Some(ResponseFormat::JsonSchema { .. }) + ), + ), + ]; + + utils::validate_mutually_exclusive_options( + &structured_constraints, + "Only one structured output constraint (regex, ebnf, or json_schema) can be active at a time", + )?; + + Ok(()) + } +} + +impl ValidatableRequest for ChatCompletionRequest { + fn validate(&self) -> Result<(), ValidationError> { + // Call the common validation function from the validation module + utils::validate_common_request_params(self)?; + + // Then validate chat-specific parameters + self.validate_messages()?; + self.validate_response_format()?; + self.validate_chat_logprobs()?; + self.validate_chat_cross_parameters()?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::constants::*; + use super::utils::*; + use super::*; + use crate::protocols::spec::StringOrArray; + + // Mock request type for testing validation traits + #[derive(Debug, Default)] + struct MockRequest { + temperature: Option, + stop: Option, + max_tokens: Option, + min_tokens: Option, + } + + impl SamplingOptionsProvider for MockRequest { + fn get_temperature(&self) -> Option { + self.temperature + } + fn get_top_p(&self) -> Option { + None + } + fn get_frequency_penalty(&self) -> Option { + None + } + fn get_presence_penalty(&self) -> Option { + None + } + } + + impl StopConditionsProvider for MockRequest { + fn get_stop_sequences(&self) -> Option<&StringOrArray> { + self.stop.as_ref() + } + } + + impl TokenLimitsProvider for MockRequest { + fn get_max_tokens(&self) -> Option { + self.max_tokens + } + fn get_min_tokens(&self) -> Option { + self.min_tokens + } + } + + impl LogProbsProvider for MockRequest { + fn get_logprobs(&self) -> Option { + None + } + fn get_top_logprobs(&self) -> Option { + None + } + } + + impl SGLangExtensionsProvider for MockRequest {} + impl CompletionCountProvider for MockRequest {} + impl ValidatableRequest for MockRequest {} + + #[test] + fn test_range_validation() { + // Valid range + assert!(validate_range(1.5f32, &TEMPERATURE_RANGE, "temperature").is_ok()); + // Invalid range + assert!(validate_range(-0.1f32, &TEMPERATURE_RANGE, "temperature").is_err()); + assert!(validate_range(3.0f32, &TEMPERATURE_RANGE, "temperature").is_err()); + } + + #[test] + fn test_sglang_top_k_validation() { + assert!(validate_top_k(-1).is_ok()); // Disabled + assert!(validate_top_k(50).is_ok()); // Valid positive + assert!(validate_top_k(0).is_err()); // Invalid + assert!(validate_top_k(-5).is_err()); // Invalid + } + + #[test] + fn test_stop_sequences_limits() { + let request = MockRequest { + stop: Some(StringOrArray::Array(vec![ + "stop1".to_string(), + "stop2".to_string(), + "stop3".to_string(), + "stop4".to_string(), + "stop5".to_string(), // Too many + ])), + ..Default::default() + }; + assert!(request.validate().is_err()); + } + + #[test] + fn test_token_limits_conflict() { + let request = MockRequest { + min_tokens: Some(100), + max_tokens: Some(50), // min > max + ..Default::default() + }; + assert!(request.validate().is_err()); + } + + #[test] + fn test_valid_request() { + let request = MockRequest { + temperature: Some(1.0), + stop: Some(StringOrArray::Array(vec!["stop".to_string()])), + max_tokens: Some(100), + min_tokens: Some(10), + }; + assert!(request.validate().is_ok()); + } + + // Chat completion specific tests + #[cfg(test)] + mod chat_tests { + use super::*; + + fn create_valid_chat_request() -> ChatCompletionRequest { + ChatCompletionRequest { + model: "gpt-4".to_string(), + messages: vec![ChatMessage::User { + role: "user".to_string(), + content: UserMessageContent::Text("Hello".to_string()), + name: None, + }], + temperature: Some(1.0), + top_p: Some(0.9), + n: Some(1), + stream: false, + stream_options: None, + stop: None, + max_tokens: Some(100), + max_completion_tokens: None, + presence_penalty: Some(0.0), + frequency_penalty: Some(0.0), + logit_bias: None, + user: None, + seed: None, + logprobs: false, + top_logprobs: None, + response_format: None, + tools: None, + tool_choice: None, + parallel_tool_calls: None, + functions: None, + function_call: None, + // SGLang extensions + top_k: None, + min_p: None, + min_tokens: None, + repetition_penalty: None, + regex: None, + ebnf: None, + stop_token_ids: None, + no_stop_trim: false, + ignore_eos: false, + continue_final_message: false, + skip_special_tokens: true, + lora_path: None, + session_params: None, + separate_reasoning: true, + stream_reasoning: true, + chat_template_kwargs: None, + return_hidden_states: false, + } + } + + #[test] + fn test_chat_validation_basics() { + // Valid request + assert!(create_valid_chat_request().validate().is_ok()); + + // Empty messages + let mut request = create_valid_chat_request(); + request.messages = vec![]; + assert!(request.validate().is_err()); + + // Invalid temperature + let mut request = create_valid_chat_request(); + request.temperature = Some(3.0); + assert!(request.validate().is_err()); + } + + #[test] + fn test_chat_conflicts() { + let mut request = create_valid_chat_request(); + + // Conflicting max_tokens + request.max_tokens = Some(100); + request.max_completion_tokens = Some(200); + assert!(request.validate().is_err()); + + // Logprobs without top_logprobs + request.max_tokens = None; + request.logprobs = true; + request.top_logprobs = None; + assert!(request.validate().is_err()); + } + + #[test] + fn test_sglang_extensions() { + let mut request = create_valid_chat_request(); + + // Valid SGLang parameters + request.top_k = Some(-1); + request.min_p = Some(0.1); + request.repetition_penalty = Some(1.2); + assert!(request.validate().is_ok()); + + // Invalid parameters + request.top_k = Some(0); // Invalid + assert!(request.validate().is_err()); + } + + #[test] + fn test_parameter_ranges() { + let mut request = create_valid_chat_request(); + + // Test temperature range (0.0 to 2.0) + request.temperature = Some(1.5); + assert!(request.validate().is_ok()); + request.temperature = Some(-0.1); + assert!(request.validate().is_err()); + request.temperature = Some(3.0); + assert!(request.validate().is_err()); + + // Test top_p range (0.0 to 1.0) + request.temperature = Some(1.0); // Reset + request.top_p = Some(0.9); + assert!(request.validate().is_ok()); + request.top_p = Some(-0.1); + assert!(request.validate().is_err()); + request.top_p = Some(1.5); + assert!(request.validate().is_err()); + + // Test frequency_penalty range (-2.0 to 2.0) + request.top_p = Some(0.9); // Reset + request.frequency_penalty = Some(1.5); + assert!(request.validate().is_ok()); + request.frequency_penalty = Some(-2.5); + assert!(request.validate().is_err()); + request.frequency_penalty = Some(3.0); + assert!(request.validate().is_err()); + + // Test presence_penalty range (-2.0 to 2.0) + request.frequency_penalty = Some(0.0); // Reset + request.presence_penalty = Some(-1.5); + assert!(request.validate().is_ok()); + request.presence_penalty = Some(-3.0); + assert!(request.validate().is_err()); + request.presence_penalty = Some(2.5); + assert!(request.validate().is_err()); + + // Test repetition_penalty range (0.0 to 2.0) + request.presence_penalty = Some(0.0); // Reset + request.repetition_penalty = Some(1.2); + assert!(request.validate().is_ok()); + request.repetition_penalty = Some(-0.1); + assert!(request.validate().is_err()); + request.repetition_penalty = Some(2.1); + assert!(request.validate().is_err()); + + // Test min_p range (0.0 to 1.0) + request.repetition_penalty = Some(1.0); // Reset + request.min_p = Some(0.5); + assert!(request.validate().is_ok()); + request.min_p = Some(-0.1); + assert!(request.validate().is_err()); + request.min_p = Some(1.5); + assert!(request.validate().is_err()); + } + + #[test] + fn test_structured_output_conflicts() { + let mut request = create_valid_chat_request(); + + // JSON response format with regex should conflict + request.response_format = Some(ResponseFormat::JsonObject); + request.regex = Some(".*".to_string()); + assert!(request.validate().is_err()); + + // JSON response format with EBNF should conflict + request.regex = None; + request.ebnf = Some("grammar".to_string()); + assert!(request.validate().is_err()); + + // Multiple structured constraints should conflict + request.response_format = None; + request.regex = Some(".*".to_string()); + request.ebnf = Some("grammar".to_string()); + assert!(request.validate().is_err()); + + // Only one constraint should work + request.ebnf = None; + request.regex = Some(".*".to_string()); + assert!(request.validate().is_ok()); + + request.regex = None; + request.ebnf = Some("grammar".to_string()); + assert!(request.validate().is_ok()); + + request.ebnf = None; + request.response_format = Some(ResponseFormat::JsonObject); + assert!(request.validate().is_ok()); + } + + #[test] + fn test_stop_sequences_validation() { + let mut request = create_valid_chat_request(); + + // Valid stop sequences + request.stop = Some(StringOrArray::Array(vec![ + "stop1".to_string(), + "stop2".to_string(), + ])); + assert!(request.validate().is_ok()); + + // Too many stop sequences (max 4) + request.stop = Some(StringOrArray::Array(vec![ + "stop1".to_string(), + "stop2".to_string(), + "stop3".to_string(), + "stop4".to_string(), + "stop5".to_string(), + ])); + assert!(request.validate().is_err()); + + // Empty stop sequence should fail + request.stop = Some(StringOrArray::String("".to_string())); + assert!(request.validate().is_err()); + + // Empty string in array should fail + request.stop = Some(StringOrArray::Array(vec![ + "stop1".to_string(), + "".to_string(), + ])); + assert!(request.validate().is_err()); + } + + #[test] + fn test_logprobs_validation() { + let mut request = create_valid_chat_request(); + + // Valid logprobs configuration + request.logprobs = true; + request.top_logprobs = Some(10); + assert!(request.validate().is_ok()); + + // logprobs=true without top_logprobs should fail + request.top_logprobs = None; + assert!(request.validate().is_err()); + + // top_logprobs without logprobs=true should fail + request.logprobs = false; + request.top_logprobs = Some(10); + assert!(request.validate().is_err()); + + // top_logprobs out of range (0-20) + request.logprobs = true; + request.top_logprobs = Some(25); + assert!(request.validate().is_err()); + } + + #[test] + fn test_n_parameter_validation() { + let mut request = create_valid_chat_request(); + + // Valid n values (1-10) + request.n = Some(1); + assert!(request.validate().is_ok()); + request.n = Some(5); + assert!(request.validate().is_ok()); + request.n = Some(10); + assert!(request.validate().is_ok()); + + // Invalid n values + request.n = Some(0); + assert!(request.validate().is_err()); + request.n = Some(15); + assert!(request.validate().is_err()); + } + + #[test] + fn test_min_max_tokens_validation() { + let mut request = create_valid_chat_request(); + + // Valid token limits + request.min_tokens = Some(10); + request.max_tokens = Some(100); + assert!(request.validate().is_ok()); + + // min_tokens > max_tokens should fail + request.min_tokens = Some(150); + request.max_tokens = Some(100); + assert!(request.validate().is_err()); + + // Should work with max_completion_tokens instead + request.max_tokens = None; + request.max_completion_tokens = Some(200); + request.min_tokens = Some(50); + assert!(request.validate().is_ok()); + + // min_tokens > max_completion_tokens should fail + request.min_tokens = Some(250); + assert!(request.validate().is_err()); + } + } +} diff --git a/sgl-router/src/reasoning_parser/README.md b/sgl-router/src/reasoning_parser/README.md new file mode 100644 index 000000000..92a6ffce7 --- /dev/null +++ b/sgl-router/src/reasoning_parser/README.md @@ -0,0 +1,474 @@ +# Reasoning Parser Architecture + +## 1. Executive Summary + +### High-Level Overview + +The reasoning parser layer provides a unified interface for detecting and extracting reasoning content from Large Language Model (LLM) outputs, particularly from models that support Chain-of-Thought (CoT) reasoning with explicit thinking blocks. The architecture follows a trait-based design pattern enabling pluggable parser implementations while maintaining consistent APIs across different model families that use various reasoning token formats. + +**Key Components:** +- **Factory Pattern**: Registry-based creation and pooling of model-specific parsers +- **Trait System**: `ReasoningParser` trait for implementation flexibility +- **Parser Pooling**: Efficient reuse of parser instances across concurrent requests +- **Streaming Support**: Incremental parsing with partial token buffering +- **Model Detection**: Pattern-based matching for automatic parser selection +- **State Management**: Stateful parsing for streaming scenarios with buffer management +- **Thread Safety**: Arc based sharing for high-concurrency environments +- **Extensibility**: Easy addition of new model-specific parsers + +**Data Flow:** +1. Request → Factory (model detection) → Pooled Parser Retrieval +2. One-Shot: Text → Parser → ParserResult (normal + reasoning text) +3. Streaming: Chunks → Parser (stateful) → Incremental ParserResult +4. Buffer Management: Partial Tokens → Buffer → Complete Token Detection +5. Reset: Parser State → Clear Buffers → Ready for Reuse + +### Architecture Highlights + +- **Model-Specific Parsers**: DeepSeek-R1, Qwen3, Kimi, GLM45, Step3 variants +- **Parser Pooling**: Singleton instances per model type for memory efficiency +- **High Concurrency**: Mutex-protected parsers handle 1000+ req/sec +- **Buffer Overflow Protection**: Configurable max buffer size (default 64KB) +- **Partial Token Detection**: Intelligent buffering for incomplete delimiters +- **Passthrough Mode**: Graceful fallback for unknown models +- **Zero-Copy Where Possible**: Efficient string handling in hot paths + +## 2. Mermaid Diagrams + +### Component Flow Diagram + +```mermaid +graph TB + subgraph Input + R[Request] --> MID[Model ID] + end + + subgraph Factory Layer + MID --> PF[ParserFactory] + PF --> REG[ParserRegistry] + REG --> PM[Pattern Matching] + PM --> PP[Parser Pool] + end + + subgraph Parser Pool + PP --> DS[DeepSeek-R1] + PP --> QW[Qwen3] + PP --> QWT[Qwen3-Thinking] + PP --> KM[Kimi] + PP --> GL[GLM45] + PP --> S3[Step3] + PP --> PT[Passthrough] + end + + subgraph Parser Instance + DS --> BP[BaseReasoningParser] + QW --> BP + KM --> BP + GL --> BP + S3 --> BP + end + + subgraph Processing + BP --> DAP[detect_and_parse] + BP --> PSI[parse_streaming] + BP --> RST[reset] + end + + subgraph State Management + BP --> BUF[Buffer] + BP --> IR[in_reasoning flag] + BP --> STS[stripped_think_start] + end + + subgraph Output + DAP --> PR[ParserResult] + PSI --> PR + PR --> NT[normal_text] + PR --> RT[reasoning_text] + end +``` + +### Sequence Flow Diagram + +```mermaid +sequenceDiagram + participant C as Client + participant F as ParserFactory + participant R as Registry + participant P as Parser Pool + participant BP as BaseParser + participant PR as ParserResult + + C->>F: get_pooled("deepseek-r1-model") + F->>R: find_pooled_parser_for_model() + R->>R: pattern_match("deepseek-r1") + R->>P: get_pooled_parser("deepseek_r1") + + alt Parser exists in pool + P-->>F: Arc> + else Create new parser + P->>BP: new DeepSeekR1Parser() + P->>P: insert into pool + P-->>F: Arc> + end + + F-->>C: PooledParser + + C->>BP: lock().parse_reasoning_streaming_incremental() + loop streaming chunks + C->>BP: parse_reasoning_streaming_incremental(chunk) + BP->>BP: buffer.push_str(chunk) + BP->>BP: check partial tokens + + alt Complete token found + BP->>PR: create result + BP->>BP: clear buffer + BP-->>C: ParserResult + else Partial token + BP->>BP: keep buffering + BP-->>C: ParserResult::default() + end + end + + C->>BP: reset() + BP->>BP: clear buffers & flags + C->>BP: unlock() +``` + +### Class/Type Diagram + +```mermaid +classDiagram + class ReasoningParser { + <> + +detect_and_parse_reasoning(&mut self, text: &str) Result~ParserResult~ + +parse_reasoning_streaming_incremental(&mut self, text: &str) Result~ParserResult~ + +reset(&mut self) + +model_type(&self) &str + } + + class ParserResult { + +normal_text: String + +reasoning_text: String + +new(normal: String, reasoning: String) Self + +normal(text: String) Self + +reasoning(text: String) Self + +is_empty() bool + } + + class ParserConfig { + +think_start_token: String + +think_end_token: String + +stream_reasoning: bool + +max_buffer_size: usize + +initial_in_reasoning: bool + +default() Self + } + + class BaseReasoningParser { + -config: ParserConfig + -in_reasoning: bool + -buffer: String + -stripped_think_start: bool + -model_type: String + +new(config: ParserConfig) Self + +with_model_type(model: String) Self + -is_partial_token(&self, text: &str) bool + } + + class DeepSeekR1Parser { + -base: BaseReasoningParser + +new() Self + } + + class Qwen3Parser { + -base: BaseReasoningParser + +new() Self + } + + class QwenThinkingParser { + -base: BaseReasoningParser + +new() Self + } + + class KimiParser { + -base: BaseReasoningParser + +new() Self + } + + class Glm45Parser { + -base: BaseReasoningParser + +new() Self + } + + class Step3Parser { + -base: BaseReasoningParser + +new() Self + } + + class ParserFactory { + -registry: ParserRegistry + +new() Self + +get_pooled(model_id: &str) PooledParser + +create(model_id: &str) Result~Box~dyn ReasoningParser~~ + +clear_pool() + } + + class ParserRegistry { + -creators: Arc~RwLock~HashMap~~ + -pool: Arc~RwLock~HashMap~~ + -patterns: Arc~RwLock~Vec~~ + +register_parser(name: &str, creator: F) + +register_pattern(pattern: &str, parser_name: &str) + +get_pooled_parser(name: &str) Option~PooledParser~ + +find_pooled_parser_for_model(model: &str) Option~PooledParser~ + } + + ReasoningParser <|.. BaseReasoningParser + ReasoningParser <|.. DeepSeekR1Parser + ReasoningParser <|.. Qwen3Parser + ReasoningParser <|.. QwenThinkingParser + ReasoningParser <|.. KimiParser + ReasoningParser <|.. Glm45Parser + ReasoningParser <|.. Step3Parser + + DeepSeekR1Parser o-- BaseReasoningParser + Qwen3Parser o-- BaseReasoningParser + QwenThinkingParser o-- BaseReasoningParser + KimiParser o-- BaseReasoningParser + Glm45Parser o-- BaseReasoningParser + Step3Parser o-- BaseReasoningParser + + BaseReasoningParser o-- ParserConfig + ParserFactory o-- ParserRegistry + ParserRegistry o-- ReasoningParser +``` + +## 3. Module-by-Module Deep Dive + +### 3.1 mod.rs (Main Module) + +**Key Responsibilities:** +- Module organization and public API surface +- Re-exports for convenient access to core types +- Separation of concerns across submodules + +**Module Structure:** +- `factory`: Parser creation and pooling logic +- `parsers`: Concrete parser implementations +- `traits`: Core trait definitions and types + +### 3.2 traits.rs (Trait Definitions) + +**ParserResult Methods**: +- `new()`: Create with both normal and reasoning text +- `normal()`: Create with only normal text (convenience) +- `reasoning()`: Create with only reasoning text (convenience) +- `is_empty()`: Check if result contains any text + +**ReasoningParser Trait**: +- **`detect_and_parse_reasoning`**: One-shot parsing for complete text +- **`parse_reasoning_streaming_incremental`**: Stateful streaming parser +- **`reset`**: Clear state for parser reuse +- **`model_type`**: Identify parser variant for debugging + +**ParserConfig Defaults**: +- Default tokens: `` and `` +- Stream reasoning: true (immediate output) +- Max buffer: 65536 bytes (64KB) +- Initial state: false (explicit reasoning blocks) + +### 3.3 factory.rs (Parser Creation & Pooling) + +**ParserRegistry Methods**: + +1. **`register_parser`**: + - Register creator function for parser type + - Lazy instantiation when requested + - Thread-safe registration + +2. **`register_pattern`**: + - Map model ID patterns to parser names + - First-match-wins ordering + - Case-insensitive matching + +3. **`get_pooled_parser`**: + - Check pool for existing instance + - Create and pool if not present + - Return Arc for sharing + +4. **`find_pooled_parser_for_model`**: + - Pattern match against model ID + - Delegate to get_pooled_parser + - Case-insensitive comparison + +**ParserFactory Methods**: + +1. **`new()`**: + - Register all built-in parsers + - Setup model pattern mappings + - Initialize empty pool + +2. **`get_pooled`**: + - Primary API for getting parsers + - Automatic passthrough fallback + - Guaranteed non-null return + +3. **`create`**: + - Create fresh parser instance + - No pooling (for testing/isolation) + - Returns Result for error handling + +**Registered Parsers**: +- `base`: Generic configurable parser +- `deepseek_r1`: DeepSeek-R1 (initial_in_reasoning=true) +- `qwen3`: Qwen3 base model (initial_in_reasoning=false) +- `qwen3_thinking`: Qwen3 thinking variant (initial_in_reasoning=true) +- `kimi`: Kimi with Unicode tokens +- `glm45`: GLM-4.5 parser +- `step3`: Step3 parser +- `passthrough`: No-op fallback parser + +**Model Pattern Mappings**: +``` +"deepseek-r1" → "deepseek_r1" +"qwen3-thinking" → "qwen3_thinking" +"qwen-thinking" → "qwen3_thinking" +"qwen3" → "qwen3" +"qwen" → "qwen3" +"glm45" → "glm45" +"kimi" → "kimi" +"step3" → "step3" +``` + +### 3.4 parsers/base.rs (Base Implementation) + +**Key Methods:** + +**`detect_and_parse_reasoning`**: +``` +Algorithm: +1. Check buffer overflow protection +2. Detect reasoning presence (in_reasoning OR contains start_token) +3. If no reasoning → return as normal text +4. Remove start token and trim +5. If no end token → assume truncated reasoning +6. Split on end token +7. Extract reasoning and normal portions +``` + +**`parse_reasoning_streaming_incremental`**: +``` +Algorithm: +1. Check buffer capacity +2. Append text to buffer +3. Check if buffer is partial token prefix +4. If partial → buffer and return empty +5. Strip start token if present +6. Find end token position +7. Handle based on state: + - In reasoning + end found → split and return both + - In reasoning + streaming → return accumulated reasoning + - Not in reasoning → return as normal text + - In reasoning + no end → continue buffering +``` + +**Critical Features:** + +1. **Partial Token Detection**: + - Prevents premature token matching + - Buffers incomplete delimiters + - Essential for streaming correctness + +2. **Buffer Management**: + - Overflow protection + - Accumulation for partial content + - Clear on complete token detection + +3. **State Tracking**: + - `in_reasoning`: Current parsing state + - `stripped_think_start`: Prevent double processing + - `buffer`: Accumulated partial content + + +## 4. Extensibility Guide + +### Adding a New Parser + +**Step 1: Create Parser Implementation** + +```rust +// src/reasoning_parser/parsers/mymodel.rs +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParserConfig, ReasoningParser}; + +pub struct MyModelParser { + base: BaseReasoningParser, +} + +impl MyModelParser { + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // or true for implicit + }; + + Self { + base: BaseReasoningParser::new(config) + .with_model_type("mymodel".to_string()), + } + } +} + +impl ReasoningParser for MyModelParser { + // Delegate to base or implement custom logic + fn detect_and_parse_reasoning(&mut self, text: &str) + -> Result { + self.base.detect_and_parse_reasoning(text) + } + + // ... other trait methods +} +``` + +**Step 2: Register in Factory** + +```rust +// In factory.rs ParserFactory::new() +registry.register_parser("mymodel", || { + Box::new(MyModelParser::new()) +}); + +// Register patterns +registry.register_pattern("my-model", "mymodel"); +registry.register_pattern("mymodel", "mymodel"); +``` + +**Step 3: Export from Module** + +```rust +// In parsers/mod.rs +pub use self::mymodel::MyModelParser; + +// In reasoning_parser/mod.rs +pub use parsers::MyModelParser; +``` + +### Custom Parsing Logic + +For parsers requiring custom logic beyond configuration: + +```rust +impl ReasoningParser for CustomParser { + fn parse_reasoning_streaming_incremental(&mut self, text: &str) + -> Result { + // Custom state machine + // Custom token detection + // Custom buffering strategy + // Return appropriate ParserResult + } +} +``` diff --git a/sgl-router/src/reasoning_parser/factory.rs b/sgl-router/src/reasoning_parser/factory.rs new file mode 100644 index 000000000..771f0e856 --- /dev/null +++ b/sgl-router/src/reasoning_parser/factory.rs @@ -0,0 +1,566 @@ +// Factory and registry for creating model-specific reasoning parsers. +// Now with parser pooling support for efficient reuse across requests. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock}; + +use crate::reasoning_parser::parsers::{ + BaseReasoningParser, DeepSeekR1Parser, Glm45Parser, KimiParser, Qwen3Parser, + QwenThinkingParser, Step3Parser, +}; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ReasoningParser}; + +/// Type alias for pooled parser instances. +pub type PooledParser = Arc>>; + +/// Type alias for parser creator functions. +type ParserCreator = Arc Box + Send + Sync>; + +/// Registry for model-specific parsers with pooling support. +#[derive(Clone)] +pub struct ParserRegistry { + /// Creator functions for parsers (used when pool is empty) + creators: Arc>>, + /// Pooled parser instances for reuse + pool: Arc>>, + /// Model pattern to parser name mappings + patterns: Arc>>, // (pattern, parser_name) +} + +impl ParserRegistry { + /// Create a new empty registry. + pub fn new() -> Self { + Self { + creators: Arc::new(RwLock::new(HashMap::new())), + pool: Arc::new(RwLock::new(HashMap::new())), + patterns: Arc::new(RwLock::new(Vec::new())), + } + } + + /// Register a parser creator for a given parser type. + pub fn register_parser(&self, name: &str, creator: F) + where + F: Fn() -> Box + Send + Sync + 'static, + { + let mut creators = self.creators.write().unwrap(); + creators.insert(name.to_string(), Arc::new(creator)); + } + + /// Register a model pattern to parser mapping. + /// Patterns are checked in order, first match wins. + pub fn register_pattern(&self, pattern: &str, parser_name: &str) { + let mut patterns = self.patterns.write().unwrap(); + patterns.push((pattern.to_string(), parser_name.to_string())); + } + + /// Get a pooled parser by exact name. + /// Returns a shared parser instance from the pool, creating one if needed. + pub fn get_pooled_parser(&self, name: &str) -> Option { + // First check if we have a pooled instance + { + let pool = self.pool.read().unwrap(); + if let Some(parser) = pool.get(name) { + return Some(Arc::clone(parser)); + } + } + + // If not in pool, create one and add to pool + let creators = self.creators.read().unwrap(); + if let Some(creator) = creators.get(name) { + let parser = Arc::new(Mutex::new(creator())); + + // Add to pool for future use + let mut pool = self.pool.write().unwrap(); + pool.insert(name.to_string(), Arc::clone(&parser)); + + Some(parser) + } else { + None + } + } + + /// Get a parser by exact name (creates new instance, not pooled). + /// Use this for compatibility or when you need a fresh instance. + pub fn get_parser(&self, name: &str) -> Option> { + let creators = self.creators.read().unwrap(); + creators.get(name).map(|creator| creator()) + } + + /// Find a pooled parser for a given model ID by pattern matching. + pub fn find_pooled_parser_for_model(&self, model_id: &str) -> Option { + let patterns = self.patterns.read().unwrap(); + let model_lower = model_id.to_lowercase(); + + for (pattern, parser_name) in patterns.iter() { + if model_lower.contains(&pattern.to_lowercase()) { + return self.get_pooled_parser(parser_name); + } + } + None + } + + /// Find a parser for a given model ID by pattern matching (creates new instance). + pub fn find_parser_for_model(&self, model_id: &str) -> Option> { + let patterns = self.patterns.read().unwrap(); + let model_lower = model_id.to_lowercase(); + + for (pattern, parser_name) in patterns.iter() { + if model_lower.contains(&pattern.to_lowercase()) { + return self.get_parser(parser_name); + } + } + None + } + + /// Clear the parser pool, forcing new instances to be created. + /// Useful for testing or when parsers need to be reset globally. + pub fn clear_pool(&self) { + let mut pool = self.pool.write().unwrap(); + pool.clear(); + } +} + +impl Default for ParserRegistry { + fn default() -> Self { + Self::new() + } +} + +/// Factory for creating reasoning parsers based on model type. +#[derive(Clone)] +pub struct ParserFactory { + registry: ParserRegistry, +} + +impl ParserFactory { + /// Create a new factory with default parsers registered. + pub fn new() -> Self { + let registry = ParserRegistry::new(); + + // Register base parser + registry.register_parser("base", || { + Box::new(BaseReasoningParser::new(ParserConfig::default())) + }); + + // Register DeepSeek-R1 parser (starts with in_reasoning=true) + registry.register_parser("deepseek_r1", || Box::new(DeepSeekR1Parser::new())); + + // Register Qwen3 parser (starts with in_reasoning=false) + registry.register_parser("qwen3", || Box::new(Qwen3Parser::new())); + + // Register Qwen3-thinking parser (starts with in_reasoning=true) + registry.register_parser("qwen3_thinking", || Box::new(QwenThinkingParser::new())); + + // Register Kimi parser with Unicode tokens (starts with in_reasoning=false) + registry.register_parser("kimi", || Box::new(KimiParser::new())); + + // Register GLM45 parser (same format as Qwen3 but separate for debugging) + registry.register_parser("glm45", || Box::new(Glm45Parser::new())); + + // Register Step3 parser (same format as DeepSeek-R1 but separate for debugging) + registry.register_parser("step3", || Box::new(Step3Parser::new())); + + // Register model patterns + registry.register_pattern("deepseek-r1", "deepseek_r1"); + registry.register_pattern("qwen3-thinking", "qwen3_thinking"); + registry.register_pattern("qwen-thinking", "qwen3_thinking"); + registry.register_pattern("qwen3", "qwen3"); + registry.register_pattern("qwen", "qwen3"); + registry.register_pattern("glm45", "glm45"); + registry.register_pattern("kimi", "kimi"); + registry.register_pattern("step3", "step3"); + + Self { registry } + } + + /// Get a pooled parser for the given model ID. + /// Returns a shared instance that can be used concurrently. + /// Falls back to a passthrough parser if model is not recognized. + pub fn get_pooled(&self, model_id: &str) -> PooledParser { + // First try to find by pattern + if let Some(parser) = self.registry.find_pooled_parser_for_model(model_id) { + return parser; + } + + // Fall back to no-op parser (get or create passthrough in pool) + self.registry + .get_pooled_parser("passthrough") + .unwrap_or_else(|| { + // Register passthrough if not already registered + self.registry.register_parser("passthrough", || { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, + }; + Box::new( + BaseReasoningParser::new(config).with_model_type("passthrough".to_string()), + ) + }); + self.registry.get_pooled_parser("passthrough").unwrap() + }) + } + + /// Create a new parser instance for the given model ID. + /// Returns a fresh instance (not pooled). + /// Use this when you need an isolated parser instance. + pub fn create(&self, model_id: &str) -> Result, ParseError> { + // First try to find by pattern + if let Some(parser) = self.registry.find_parser_for_model(model_id) { + return Ok(parser); + } + + // Fall back to no-op parser (base parser without reasoning detection) + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, + }; + Ok(Box::new( + BaseReasoningParser::new(config).with_model_type("passthrough".to_string()), + )) + } + + /// Get the internal registry for custom registration. + pub fn registry(&self) -> &ParserRegistry { + &self.registry + } + + /// Clear the parser pool. + /// Useful for testing or when parsers need to be reset globally. + pub fn clear_pool(&self) { + self.registry.clear_pool(); + } +} + +impl Default for ParserFactory { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_factory_creates_deepseek_r1() { + let factory = ParserFactory::new(); + let parser = factory.create("deepseek-r1-distill").unwrap(); + assert_eq!(parser.model_type(), "deepseek_r1"); + } + + #[test] + fn test_factory_creates_qwen3() { + let factory = ParserFactory::new(); + let parser = factory.create("qwen3-7b").unwrap(); + assert_eq!(parser.model_type(), "qwen3"); + } + + #[test] + fn test_factory_creates_kimi() { + let factory = ParserFactory::new(); + let parser = factory.create("kimi-chat").unwrap(); + assert_eq!(parser.model_type(), "kimi"); + } + + #[test] + fn test_factory_fallback_to_passthrough() { + let factory = ParserFactory::new(); + let parser = factory.create("unknown-model").unwrap(); + assert_eq!(parser.model_type(), "passthrough"); + } + + #[test] + fn test_case_insensitive_matching() { + let factory = ParserFactory::new(); + let parser1 = factory.create("DeepSeek-R1").unwrap(); + let parser2 = factory.create("QWEN3").unwrap(); + let parser3 = factory.create("Kimi").unwrap(); + + assert_eq!(parser1.model_type(), "deepseek_r1"); + assert_eq!(parser2.model_type(), "qwen3"); + assert_eq!(parser3.model_type(), "kimi"); + } + + #[test] + fn test_step3_model() { + let factory = ParserFactory::new(); + let step3 = factory.create("step3-model").unwrap(); + assert_eq!(step3.model_type(), "step3"); + } + + #[test] + fn test_glm45_model() { + let factory = ParserFactory::new(); + let glm45 = factory.create("glm45-v2").unwrap(); + assert_eq!(glm45.model_type(), "glm45"); + } + + #[test] + fn test_pooled_parser_reuse() { + let factory = ParserFactory::new(); + + // Get the same parser twice - should be the same instance + let parser1 = factory.get_pooled("deepseek-r1"); + let parser2 = factory.get_pooled("deepseek-r1"); + + // Both should point to the same Arc + assert!(Arc::ptr_eq(&parser1, &parser2)); + + // Different models should get different parsers + let parser3 = factory.get_pooled("qwen3"); + assert!(!Arc::ptr_eq(&parser1, &parser3)); + } + + #[test] + fn test_pooled_parser_concurrent_access() { + use std::thread; + + let factory = ParserFactory::new(); + let parser = factory.get_pooled("deepseek-r1"); + + // Spawn multiple threads that use the same parser + let mut handles = vec![]; + + for i in 0..3 { + let parser_clone = Arc::clone(&parser); + let handle = thread::spawn(move || { + let mut parser = parser_clone.lock().unwrap(); + let input = format!("thread {} reasoninganswer", i); + let result = parser.detect_and_parse_reasoning(&input).unwrap(); + assert_eq!(result.normal_text, "answer"); + assert!(result.reasoning_text.contains("reasoning")); + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_pool_clearing() { + let factory = ParserFactory::new(); + + // Get a pooled parser + let parser1 = factory.get_pooled("deepseek-r1"); + + // Clear the pool + factory.clear_pool(); + + // Get another parser - should be a new instance + let parser2 = factory.get_pooled("deepseek-r1"); + + // They should be different instances (different Arc pointers) + assert!(!Arc::ptr_eq(&parser1, &parser2)); + } + + #[test] + fn test_passthrough_parser_pooling() { + let factory = ParserFactory::new(); + + // Unknown models should get passthrough parser + let parser1 = factory.get_pooled("unknown-model-1"); + let parser2 = factory.get_pooled("unknown-model-2"); + + // Both should use the same passthrough parser instance + assert!(Arc::ptr_eq(&parser1, &parser2)); + + // Verify it's actually a passthrough parser + let parser = parser1.lock().unwrap(); + assert_eq!(parser.model_type(), "passthrough"); + } + + #[test] + fn test_high_concurrency_parser_access() { + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::thread; + use std::time::Instant; + + let factory = ParserFactory::new(); + let num_threads = 100; + let requests_per_thread = 50; + let models = vec!["deepseek-r1", "qwen3", "kimi", "qwen3-thinking"]; + + // Track successful operations + let success_count = Arc::new(AtomicUsize::new(0)); + let error_count = Arc::new(AtomicUsize::new(0)); + + let start = Instant::now(); + let mut handles = vec![]; + + for thread_id in 0..num_threads { + let factory = factory.clone(); + let models = models.clone(); + let success_count = Arc::clone(&success_count); + let error_count = Arc::clone(&error_count); + + let handle = thread::spawn(move || { + for request_id in 0..requests_per_thread { + // Rotate through different models + let model = &models[(thread_id + request_id) % models.len()]; + let parser = factory.get_pooled(model); + + // Use blocking lock - this is the realistic scenario + // In production, requests would wait for the parser to be available + // Handle poisoned locks gracefully + let mut p = match parser.lock() { + Ok(guard) => guard, + Err(_poisoned) => { + // Lock was poisoned by a panicking thread + // In production, we might want to recreate the parser + // For testing, we'll just skip this iteration + error_count.fetch_add(1, Ordering::Relaxed); + continue; + } + }; + + // Simulate realistic parsing work with substantial text + // Typical reasoning can be 500-5000 tokens + let reasoning_text = format!( + "Thread {} is processing request {}. Let me think through this step by step. \ + First, I need to understand the problem. The problem involves analyzing data \ + and making calculations. Let me break this down: \n\ + 1. Initial analysis shows that we have multiple variables to consider. \ + 2. The data suggests a pattern that needs further investigation. \ + 3. Computing the values: {} * {} = {}. \ + 4. Cross-referencing with previous results indicates consistency. \ + 5. The mathematical proof follows from the axioms... \ + 6. Considering edge cases and boundary conditions... \ + 7. Validating against known constraints... \ + 8. The conclusion follows logically from premises A, B, and C. \ + This reasoning chain demonstrates the validity of our approach.", + thread_id, request_id, thread_id, request_id, thread_id * request_id + ); + + let answer_text = format!( + "Based on my analysis, the answer for thread {} request {} is: \ + The solution involves multiple steps as outlined in the reasoning. \ + The final result is {} with confidence level high. \ + This conclusion is supported by rigorous mathematical analysis \ + and has been validated against multiple test cases. \ + The implementation should handle edge cases appropriately.", + thread_id, + request_id, + thread_id * request_id + ); + + let input = format!("{}{}", reasoning_text, answer_text); + + match p.detect_and_parse_reasoning(&input) { + Ok(result) => { + // Verify parsing worked correctly with substantial content + // Note: Some parsers with stream_reasoning=true won't accumulate reasoning text + assert!(result + .normal_text + .contains(&format!("thread {}", thread_id))); + + // For parsers that accumulate reasoning (stream_reasoning=false) + // the reasoning_text should be populated + if !result.reasoning_text.is_empty() { + assert!(result + .reasoning_text + .contains(&format!("Thread {}", thread_id))); + assert!(result.reasoning_text.len() > 500); // Ensure substantial reasoning + } + + // Normal text should always be present + assert!(result.normal_text.len() > 100); // Ensure substantial answer + success_count.fetch_add(1, Ordering::Relaxed); + } + Err(e) => { + eprintln!("Parse error: {:?}", e); + error_count.fetch_add(1, Ordering::Relaxed); + } + } + + // Explicitly drop the lock to release it quickly + drop(p); + } + }); + handles.push(handle); + } + + // Wait for all threads + for handle in handles { + handle.join().unwrap(); + } + + let duration = start.elapsed(); + let total_requests = num_threads * requests_per_thread; + let successes = success_count.load(Ordering::Relaxed); + let errors = error_count.load(Ordering::Relaxed); + + // Print stats for debugging + println!( + "High concurrency test: {} threads, {} requests each", + num_threads, requests_per_thread + ); + println!( + "Completed in {:?}, {} successes, {} errors", + duration, successes, errors + ); + println!( + "Throughput: {:.0} requests/sec", + (total_requests as f64) / duration.as_secs_f64() + ); + + // All requests should succeed + assert_eq!(successes, total_requests); + assert_eq!(errors, 0); + + // Performance check: should handle at least 1000 req/sec + let throughput = (total_requests as f64) / duration.as_secs_f64(); + assert!( + throughput > 1000.0, + "Throughput too low: {:.0} req/sec", + throughput + ); + } + + #[test] + fn test_concurrent_pool_modifications() { + use std::thread; + + let factory = ParserFactory::new(); + let mut handles = vec![]; + + // Thread 1: Continuously get parsers + let factory1 = factory.clone(); + handles.push(thread::spawn(move || { + for _ in 0..100 { + let _parser = factory1.get_pooled("deepseek-r1"); + } + })); + + // Thread 2: Continuously clear pool + let factory2 = factory.clone(); + handles.push(thread::spawn(move || { + for _ in 0..10 { + factory2.clear_pool(); + thread::sleep(std::time::Duration::from_micros(100)); + } + })); + + // Thread 3: Get different parsers + let factory3 = factory.clone(); + handles.push(thread::spawn(move || { + for i in 0..100 { + let models = ["qwen3", "kimi", "unknown"]; + let _parser = factory3.get_pooled(models[i % 3]); + } + })); + + // Wait for all threads - should not deadlock or panic + for handle in handles { + handle.join().unwrap(); + } + } +} diff --git a/sgl-router/src/reasoning_parser/mod.rs b/sgl-router/src/reasoning_parser/mod.rs new file mode 100644 index 000000000..95ffcbc4f --- /dev/null +++ b/sgl-router/src/reasoning_parser/mod.rs @@ -0,0 +1,10 @@ +pub mod factory; +pub mod parsers; +pub mod traits; + +pub use factory::{ParserFactory, ParserRegistry, PooledParser}; +pub use parsers::{ + BaseReasoningParser, DeepSeekR1Parser, Glm45Parser, KimiParser, Qwen3Parser, + QwenThinkingParser, Step3Parser, +}; +pub use traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; diff --git a/sgl-router/src/reasoning_parser/parsers/base.rs b/sgl-router/src/reasoning_parser/parsers/base.rs new file mode 100644 index 000000000..0fd2818b9 --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/base.rs @@ -0,0 +1,386 @@ +// Base implementation of reasoning parser that handles common logic +// for detecting and extracting reasoning blocks from text. + +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; +use tracing as log; + +/// Base reasoning parser implementation. +/// +/// This parser handles the common logic for detecting reasoning blocks +/// delimited by start and end tokens (e.g., and ). +#[derive(Debug, Clone)] +pub struct BaseReasoningParser { + config: ParserConfig, + in_reasoning: bool, + buffer: String, + stripped_think_start: bool, + model_type: String, +} + +impl BaseReasoningParser { + /// Create a new BaseReasoningParser with the given configuration. + pub fn new(config: ParserConfig) -> Self { + let in_reasoning = config.initial_in_reasoning; + Self { + config, + in_reasoning, + buffer: String::new(), + stripped_think_start: false, + model_type: "base".to_string(), + } + } + + /// Create with custom model type identifier. + pub fn with_model_type(mut self, model_type: String) -> Self { + self.model_type = model_type; + self + } + + /// Check if the current buffer is a prefix of one of the tokens. + fn is_partial_token(&self, text: &str) -> bool { + (self.config.think_start_token.starts_with(text) && self.config.think_start_token != text) + || (self.config.think_end_token.starts_with(text) + && self.config.think_end_token != text) + } +} + +impl ReasoningParser for BaseReasoningParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + log::debug!("detect_and_parse_reasoning called with text: {:?}", text); + + // Check input size against buffer limit + if text.len() > self.config.max_buffer_size { + return Err(ParseError::BufferOverflow(text.len())); + } + + let in_reasoning = self.in_reasoning || text.contains(&self.config.think_start_token); + log::debug!("in_reasoning: {}", in_reasoning); + + if !in_reasoning { + log::debug!("No reasoning detected, returning normal text."); + return Ok(ParserResult::normal(text.to_string())); + } + + // The text is considered to be in a reasoning block. + let processed_text = text + .replace(&self.config.think_start_token, "") + .trim() + .to_string(); + log::debug!( + "Processed text after removing think_start_token: {:?}", + processed_text + ); + + if !processed_text.contains(&self.config.think_end_token) { + log::debug!( + "Reasoning truncated, think_end_token not found. Returning reasoning text." + ); + // Assume reasoning was truncated before end token + return Ok(ParserResult::reasoning(processed_text)); + } + + // Extract reasoning content + let splits: Vec<&str> = processed_text + .splitn(2, &self.config.think_end_token) + .collect(); + let reasoning_text = splits.first().unwrap_or(&"").to_string(); + let normal_text = splits + .get(1) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + log::debug!("Extracted reasoning_text: {:?}", reasoning_text); + log::debug!("Extracted normal_text: {:?}", normal_text); + + Ok(ParserResult::new(normal_text, reasoning_text)) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + // Check if adding this text would exceed buffer limit + if self.buffer.len() + text.len() > self.config.max_buffer_size { + return Err(ParseError::BufferOverflow(self.buffer.len() + text.len())); + } + + // Incrementally parse the streaming text + self.buffer.push_str(text); + let mut current_text = self.buffer.clone(); + + log::debug!( + "parse_reasoning_streaming_incremental called with text: {:?}", + text + ); + log::debug!("current buffer: {:?}", self.buffer); + log::debug!("current_text: {:?}", current_text); + log::debug!( + "in_reasoning: {}, stripped_think_start: {}, stream_reasoning: {}", + self.in_reasoning, + self.stripped_think_start, + self.config.stream_reasoning + ); + + // If the current text is a prefix of a token, keep buffering + if self.is_partial_token(¤t_text) { + return Ok(ParserResult::default()); + } + + // Strip start token if present + if !self.stripped_think_start && current_text.contains(&self.config.think_start_token) { + current_text = current_text.replace(&self.config.think_start_token, ""); + self.buffer = current_text.clone(); + self.stripped_think_start = true; + self.in_reasoning = true; + } + + // Handle end of reasoning block + let think_end_idx = if self.in_reasoning { + current_text + .find(&self.config.think_end_token) + .unwrap_or(current_text.len()) + } else { + current_text.len() + }; + + if self.in_reasoning && think_end_idx < current_text.len() { + let reasoning_text = ¤t_text[..think_end_idx]; + self.buffer.clear(); + self.in_reasoning = false; + let start_idx = think_end_idx + self.config.think_end_token.len(); + let normal_text = if start_idx < current_text.len() { + ¤t_text[start_idx..] + } else { + "" + }; + return Ok(ParserResult::new( + normal_text.to_string(), + reasoning_text.trim().to_string(), + )); + } + + // Continue with reasoning content + if self.in_reasoning && self.config.stream_reasoning { + // Stream the content immediately + let reasoning_text = current_text; + self.buffer.clear(); + Ok(ParserResult::reasoning(reasoning_text)) + } else if !self.in_reasoning { + // If we're not in a reasoning block, return as normal text + // CRITICAL FIX: Return current_text (with buffer) not just text + // This prevents buffer loss when partial tokens are followed by normal text + let normal_text = current_text; + self.buffer.clear(); + Ok(ParserResult::normal(normal_text)) + } else { + // If we are in a reasoning block but no end token is found, buffer it + Ok(ParserResult::default()) + } + } + + fn reset(&mut self) { + self.in_reasoning = self.config.initial_in_reasoning; + self.buffer.clear(); + self.stripped_think_start = false; + } + + fn model_type(&self) -> &str { + &self.model_type + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_parser( + initial_in_reasoning: bool, + stream_reasoning: bool, + ) -> BaseReasoningParser { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning, + max_buffer_size: 65536, + initial_in_reasoning, + }; + BaseReasoningParser::new(config) + } + + #[test] + fn test_detect_and_parse_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("with reasoning and more text.") + .unwrap(); + assert_eq!(result.normal_text, "and more text."); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_detect_and_parse_no_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("This is a test without reasoning.") + .unwrap(); + assert_eq!(result.normal_text, "This is a test without reasoning."); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_detect_and_parse_truncated_reasoning() { + let mut parser = create_test_parser(false, true); + let result = parser + .detect_and_parse_reasoning("with truncated reasoning") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "with truncated reasoning"); + } + + #[test] + fn test_parse_streaming_partial_token() { + let mut parser = create_test_parser(false, true); + let result = parser + .parse_reasoning_streaming_incremental("with reasoning and more text.") + .unwrap(); + assert_eq!(result.normal_text, " and more text."); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_parse_streaming_no_end_token() { + let mut parser = create_test_parser(true, true); + let result = parser + .parse_reasoning_streaming_incremental("with reasoning") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "with reasoning"); + } + + #[test] + fn test_initial_in_reasoning_true() { + // Parser starts with in_reasoning=true (like DeepSeek-R1) + let mut parser = create_test_parser(true, true); + let result = parser + .detect_and_parse_reasoning("no think tags here") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "no think tags here"); + } + + #[test] + fn test_buffer_loss_bug_fix() { + // Critical test for buffer preservation + let mut parser = create_test_parser(false, true); + + // Step 1: Send partial end tag when not in reasoning mode + let result1 = parser.parse_reasoning_streaming_incremental("reasoning ") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, "reasoning "); + + // Continue streaming reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("content ") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "content "); + + // End reasoning block + let result3 = parser + .parse_reasoning_streaming_incremental("more normal") + .unwrap(); + assert_eq!(result3.normal_text, " normal"); + assert_eq!(result3.reasoning_text, "more"); + } + + #[test] + fn test_reset_state() { + let mut parser = create_test_parser(false, true); + + // Process some text + parser + .parse_reasoning_streaming_incremental("reasoning normal") + .unwrap(); + + // Reset and verify state + parser.reset(); + assert!(!parser.in_reasoning); + assert!(parser.buffer.is_empty()); + assert!(!parser.stripped_think_start); + } + + #[test] + fn test_buffer_overflow_detect_and_parse() { + let config = ParserConfig { + max_buffer_size: 10, // Set a very small buffer + ..Default::default() + }; + let mut parser = BaseReasoningParser::new(config); + + let large_text = "a".repeat(20); + let result = parser.detect_and_parse_reasoning(&large_text); + + assert!(result.is_err()); + match result { + Err(ParseError::BufferOverflow(size)) => { + assert_eq!(size, 20); + } + _ => panic!("Expected BufferOverflow error"), + } + } + + #[test] + fn test_buffer_overflow_streaming() { + let config = ParserConfig { + max_buffer_size: 10, // Set a very small buffer + ..Default::default() + }; + let mut parser = BaseReasoningParser::new(config); + + // Send a partial token that will be buffered + let result1 = parser.parse_reasoning_streaming_incremental(" { + assert_eq!(size, 21); // 4 + 17 + } + _ => panic!("Expected BufferOverflow error"), + } + } +} diff --git a/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs b/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs new file mode 100644 index 000000000..62a7aadec --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/deepseek_r1.rs @@ -0,0 +1,112 @@ +// DeepSeek-R1 specific reasoning parser. +// This parser starts with in_reasoning=true, assuming all text is reasoning +// until an end token is encountered. + +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// DeepSeek-R1 reasoning parser. +/// +/// This parser assumes reasoning from the start of text (in_reasoning=true) +/// and uses and tokens. +pub struct DeepSeekR1Parser { + base: BaseReasoningParser, +} + +impl DeepSeekR1Parser { + /// Create a new DeepSeek-R1 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Always starts with reasoning + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("deepseek_r1".to_string()), + } + } +} + +impl Default for DeepSeekR1Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for DeepSeekR1Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deepseek_r1_initial_state() { + let mut parser = DeepSeekR1Parser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_deepseek_r1_with_end_token() { + let mut parser = DeepSeekR1Parser::new(); + + // Should extract reasoning until end token + let result = parser + .detect_and_parse_reasoning("reasoning contentnormal content") + .unwrap(); + assert_eq!(result.normal_text, "normal content"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_deepseek_r1_streaming() { + let mut parser = DeepSeekR1Parser::new(); + + // First chunk - all reasoning + let result1 = parser + .parse_reasoning_streaming_incremental("thinking about") + .unwrap(); + assert_eq!(result1.reasoning_text, "thinking about"); + assert_eq!(result1.normal_text, ""); + + // Second chunk - ends reasoning + let result2 = parser + .parse_reasoning_streaming_incremental(" the problemanswer") + .unwrap(); + assert_eq!(result2.reasoning_text, "the problem"); // Text is trimmed + assert_eq!(result2.normal_text, "answer"); + } + + #[test] + fn test_model_type() { + let parser = DeepSeekR1Parser::new(); + assert_eq!(parser.model_type(), "deepseek_r1"); + } +} diff --git a/sgl-router/src/reasoning_parser/parsers/glm45.rs b/sgl-router/src/reasoning_parser/parsers/glm45.rs new file mode 100644 index 000000000..e4e56723c --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/glm45.rs @@ -0,0 +1,118 @@ +// GLM45 specific reasoning parser. +// Uses the same format as Qwen3 but has its own implementation for debugging. + +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// GLM45 reasoning parser. +/// +/// This parser uses the same format as Qwen3 (...) but has +/// its own implementation for better debugging and potential future customization. +pub struct Glm45Parser { + base: BaseReasoningParser, +} + +impl Glm45Parser { + /// Create a new GLM45 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token like Qwen3 + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("glm45".to_string()), + } + } +} + +impl Default for Glm45Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Glm45Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_glm45_initial_state() { + let mut parser = Glm45Parser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_glm45_with_tokens() { + let mut parser = Glm45Parser::new(); + + // Should extract reasoning with proper tokens + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_glm45_streaming() { + let mut parser = Glm45Parser::new(); + + // First chunk - normal text + let result1 = parser + .parse_reasoning_streaming_incremental("normal text ") + .unwrap(); + assert_eq!(result1.normal_text, "normal text "); + assert_eq!(result1.reasoning_text, ""); + + // Second chunk - enters reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + + // Third chunk - exits reasoning + let result3 = parser + .parse_reasoning_streaming_incremental("answer") + .unwrap(); + assert_eq!(result3.normal_text, "answer"); + assert_eq!(result3.reasoning_text, ""); + } + + #[test] + fn test_model_type() { + let parser = Glm45Parser::new(); + assert_eq!(parser.model_type(), "glm45"); + } +} diff --git a/sgl-router/src/reasoning_parser/parsers/kimi.rs b/sgl-router/src/reasoning_parser/parsers/kimi.rs new file mode 100644 index 000000000..3e11a5711 --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/kimi.rs @@ -0,0 +1,137 @@ +// Kimi specific reasoning parser. +// This parser uses Unicode tokens and starts with in_reasoning=false. + +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// Kimi reasoning parser. +/// +/// This parser uses Unicode tokens (◁think▷ and ◁/think▷) and requires +/// explicit start tokens to enter reasoning mode. +pub struct KimiParser { + base: BaseReasoningParser, +} + +impl KimiParser { + /// Create a new Kimi parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "◁think▷".to_string(), + think_end_token: "◁/think▷".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("kimi".to_string()), + } + } +} + +impl Default for KimiParser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for KimiParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_kimi_initial_state() { + let mut parser = KimiParser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_kimi_with_unicode_tokens() { + let mut parser = KimiParser::new(); + + // Should extract reasoning with Unicode tokens + let result = parser + .detect_and_parse_reasoning("◁think▷reasoning content◁/think▷answer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_kimi_partial_unicode() { + let mut parser = KimiParser::new(); + + // Test partial Unicode token buffering + let result1 = parser + .parse_reasoning_streaming_incremental("◁thi") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, ""); + + // Complete the token + let result2 = parser + .parse_reasoning_streaming_incremental("nk▷reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + } + + #[test] + fn test_kimi_streaming() { + let mut parser = KimiParser::new(); + + // Normal text first + let result1 = parser + .parse_reasoning_streaming_incremental("normal ") + .unwrap(); + assert_eq!(result1.normal_text, "normal "); + assert_eq!(result1.reasoning_text, ""); + + // Enter reasoning with Unicode token + let result2 = parser + .parse_reasoning_streaming_incremental("◁think▷thinking") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "thinking"); + + // Exit reasoning + let result3 = parser + .parse_reasoning_streaming_incremental("◁/think▷answer") + .unwrap(); + assert_eq!(result3.normal_text, "answer"); + assert_eq!(result3.reasoning_text, ""); // Already returned in stream mode + } + + #[test] + fn test_model_type() { + let parser = KimiParser::new(); + assert_eq!(parser.model_type(), "kimi"); + } +} diff --git a/sgl-router/src/reasoning_parser/parsers/mod.rs b/sgl-router/src/reasoning_parser/parsers/mod.rs new file mode 100644 index 000000000..a940a055c --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/mod.rs @@ -0,0 +1,13 @@ +pub mod base; +pub mod deepseek_r1; +pub mod glm45; +pub mod kimi; +pub mod qwen3; +pub mod step3; + +pub use base::BaseReasoningParser; +pub use deepseek_r1::DeepSeekR1Parser; +pub use glm45::Glm45Parser; +pub use kimi::KimiParser; +pub use qwen3::{Qwen3Parser, QwenThinkingParser}; +pub use step3::Step3Parser; diff --git a/sgl-router/src/reasoning_parser/parsers/qwen3.rs b/sgl-router/src/reasoning_parser/parsers/qwen3.rs new file mode 100644 index 000000000..8c5ce9e8c --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/qwen3.rs @@ -0,0 +1,178 @@ +// Qwen3 specific reasoning parser. +// This parser starts with in_reasoning=false, requiring an explicit +// start token to enter reasoning mode. + +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// Qwen3 reasoning parser. +/// +/// This parser requires explicit tokens to enter reasoning mode +/// (in_reasoning=false initially). +pub struct Qwen3Parser { + base: BaseReasoningParser, +} + +impl Qwen3Parser { + /// Create a new Qwen3 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: false, // Requires explicit start token + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("qwen3".to_string()), + } + } +} + +impl Default for Qwen3Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Qwen3Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +/// QwenThinking parser - variant that assumes reasoning from start. +/// +/// This is for qwen*thinking models that behave like DeepSeek-R1. +pub struct QwenThinkingParser { + base: BaseReasoningParser, +} + +impl QwenThinkingParser { + /// Create a new QwenThinking parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Assumes reasoning from start + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("qwen_thinking".to_string()), + } + } +} + +impl Default for QwenThinkingParser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for QwenThinkingParser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_qwen3_initial_state() { + let mut parser = Qwen3Parser::new(); + + // Should NOT treat text as reasoning without start token + let result = parser + .detect_and_parse_reasoning("This is normal content") + .unwrap(); + assert_eq!(result.normal_text, "This is normal content"); + assert_eq!(result.reasoning_text, ""); + } + + #[test] + fn test_qwen3_with_tokens() { + let mut parser = Qwen3Parser::new(); + + // Should extract reasoning with proper tokens + let result = parser + .detect_and_parse_reasoning("reasoninganswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning"); + } + + #[test] + fn test_qwen_thinking_initial_state() { + let mut parser = QwenThinkingParser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_qwen3_streaming() { + let mut parser = Qwen3Parser::new(); + + // First chunk - normal text (no start token yet) + let result1 = parser + .parse_reasoning_streaming_incremental("normal text ") + .unwrap(); + assert_eq!(result1.normal_text, "normal text "); + assert_eq!(result1.reasoning_text, ""); + + // Second chunk - enters reasoning + let result2 = parser + .parse_reasoning_streaming_incremental("reasoning") + .unwrap(); + assert_eq!(result2.normal_text, ""); + assert_eq!(result2.reasoning_text, "reasoning"); + } + + #[test] + fn test_model_types() { + let qwen3 = Qwen3Parser::new(); + assert_eq!(qwen3.model_type(), "qwen3"); + + let qwen_thinking = QwenThinkingParser::new(); + assert_eq!(qwen_thinking.model_type(), "qwen_thinking"); + } +} diff --git a/sgl-router/src/reasoning_parser/parsers/step3.rs b/sgl-router/src/reasoning_parser/parsers/step3.rs new file mode 100644 index 000000000..cec0bcd15 --- /dev/null +++ b/sgl-router/src/reasoning_parser/parsers/step3.rs @@ -0,0 +1,123 @@ +// Step3 specific reasoning parser. +// Uses the same format as DeepSeek-R1 but has its own implementation for debugging. + +use crate::reasoning_parser::parsers::BaseReasoningParser; +use crate::reasoning_parser::traits::{ParseError, ParserConfig, ParserResult, ReasoningParser}; + +/// Step3 reasoning parser. +/// +/// This parser uses the same format as DeepSeek-R1 (...) but has +/// its own implementation for better debugging and potential future customization. +pub struct Step3Parser { + base: BaseReasoningParser, +} + +impl Step3Parser { + /// Create a new Step3 parser. + pub fn new() -> Self { + let config = ParserConfig { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, + initial_in_reasoning: true, // Assumes reasoning from start like DeepSeek-R1 + }; + + Self { + base: BaseReasoningParser::new(config).with_model_type("step3".to_string()), + } + } +} + +impl Default for Step3Parser { + fn default() -> Self { + Self::new() + } +} + +impl ReasoningParser for Step3Parser { + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result { + self.base.detect_and_parse_reasoning(text) + } + + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result { + self.base.parse_reasoning_streaming_incremental(text) + } + + fn reset(&mut self) { + self.base.reset() + } + + fn model_type(&self) -> &str { + self.base.model_type() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_step3_initial_state() { + let mut parser = Step3Parser::new(); + + // Should treat text as reasoning even without start token + let result = parser + .detect_and_parse_reasoning("This is reasoning content") + .unwrap(); + assert_eq!(result.normal_text, ""); + assert_eq!(result.reasoning_text, "This is reasoning content"); + } + + #[test] + fn test_step3_with_end_token() { + let mut parser = Step3Parser::new(); + + // Should handle text with end token + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_step3_with_both_tokens() { + let mut parser = Step3Parser::new(); + + // Should handle both start and end tokens + let result = parser + .detect_and_parse_reasoning("reasoning contentanswer") + .unwrap(); + assert_eq!(result.normal_text, "answer"); + assert_eq!(result.reasoning_text, "reasoning content"); + } + + #[test] + fn test_step3_streaming() { + let mut parser = Step3Parser::new(); + + // First chunk - treated as reasoning (initial_in_reasoning=true) + let result1 = parser + .parse_reasoning_streaming_incremental("reasoning text ") + .unwrap(); + assert_eq!(result1.normal_text, ""); + assert_eq!(result1.reasoning_text, "reasoning text "); + + // Second chunk - continues reasoning until end token + let result2 = parser + .parse_reasoning_streaming_incremental("more reasoninganswer") + .unwrap(); + assert_eq!(result2.normal_text, "answer"); + assert_eq!(result2.reasoning_text, "more reasoning"); + } + + #[test] + fn test_model_type() { + let parser = Step3Parser::new(); + assert_eq!(parser.model_type(), "step3"); + } +} diff --git a/sgl-router/src/reasoning_parser/traits.rs b/sgl-router/src/reasoning_parser/traits.rs new file mode 100644 index 000000000..160fa51d9 --- /dev/null +++ b/sgl-router/src/reasoning_parser/traits.rs @@ -0,0 +1,130 @@ +use std::fmt; + +/// Result of parsing text for reasoning content. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ParserResult { + /// The normal text outside of reasoning blocks. + pub normal_text: String, + + /// The extracted reasoning text from within reasoning blocks. + pub reasoning_text: String, +} + +impl ParserResult { + /// Create a new ParserResult with the given normal and reasoning text. + pub fn new(normal_text: String, reasoning_text: String) -> Self { + Self { + normal_text, + reasoning_text, + } + } + + /// Create a result with only normal text. + pub fn normal(text: String) -> Self { + Self { + normal_text: text, + reasoning_text: String::new(), + } + } + + /// Create a result with only reasoning text. + pub fn reasoning(text: String) -> Self { + Self { + normal_text: String::new(), + reasoning_text: text, + } + } + + /// Check if this result contains any text. + pub fn is_empty(&self) -> bool { + self.normal_text.is_empty() && self.reasoning_text.is_empty() + } +} + +/// Trait for parsing reasoning content from LLM outputs. +pub trait ReasoningParser: Send + Sync { + /// Detects and parses reasoning from the input text (one-time parsing). + /// + /// This method is used for non-streaming scenarios where the complete + /// text is available at once. + /// + /// Returns an error if the text exceeds buffer limits or contains invalid UTF-8. + fn detect_and_parse_reasoning(&mut self, text: &str) -> Result; + + /// Parses reasoning incrementally from streaming input. + /// + /// This method maintains internal state across calls to handle partial + /// tokens and chunk boundaries correctly. + /// + /// Returns an error if the buffer exceeds max_buffer_size. + fn parse_reasoning_streaming_incremental( + &mut self, + text: &str, + ) -> Result; + + /// Reset the parser state for reuse. + /// + /// This should clear any buffers and reset flags to initial state. + fn reset(&mut self); + + /// Get the model type this parser is designed for. + fn model_type(&self) -> &str; +} + +/// Error types for reasoning parsing operations. +#[derive(Debug, thiserror::Error)] +pub enum ParseError { + #[error("Invalid UTF-8 in stream: {0}")] + Utf8Error(#[from] std::str::Utf8Error), + + #[error("Buffer overflow: {0} bytes exceeds maximum")] + BufferOverflow(usize), + + #[error("Unknown model type: {0}")] + UnknownModel(String), + + #[error("Parser configuration error: {0}")] + ConfigError(String), +} + +/// Configuration for parser behavior. +#[derive(Debug, Clone)] +pub struct ParserConfig { + /// The token that marks the start of reasoning content. + pub think_start_token: String, + + /// The token that marks the end of reasoning content. + pub think_end_token: String, + + /// Whether to stream reasoning content as it arrives. + pub stream_reasoning: bool, + + /// Maximum buffer size in bytes. + pub max_buffer_size: usize, + + /// Initial state for in_reasoning flag (fixed per parser type). + pub initial_in_reasoning: bool, +} + +impl Default for ParserConfig { + fn default() -> Self { + Self { + think_start_token: "".to_string(), + think_end_token: "".to_string(), + stream_reasoning: true, + max_buffer_size: 65536, // 64KB default + initial_in_reasoning: false, // Default to false (explicit reasoning) + } + } +} + +impl fmt::Display for ParserResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "ParserResult {{ normal: {} chars, reasoning: {} chars }}", + self.normal_text.len(), + self.reasoning_text.len() + ) + } +} diff --git a/sgl-router/src/routers/factory.rs b/sgl-router/src/routers/factory.rs new file mode 100644 index 000000000..d1bdc0fce --- /dev/null +++ b/sgl-router/src/routers/factory.rs @@ -0,0 +1,195 @@ +//! Factory for creating router instances + +use super::{ + http::{openai_router::OpenAIRouter, pd_router::PDRouter, router::Router}, + RouterTrait, +}; +use crate::config::{ConnectionMode, PolicyConfig, RoutingMode}; +use crate::policies::PolicyFactory; +use crate::server::AppContext; +use std::sync::Arc; + +/// Factory for creating router instances based on configuration +pub struct RouterFactory; + +impl RouterFactory { + /// Create a router instance from application context + pub async fn create_router(ctx: &Arc) -> Result, String> { + // Check if IGW mode is enabled + if ctx.router_config.enable_igw { + return Self::create_igw_router(ctx).await; + } + + // Check connection mode and route to appropriate implementation + match ctx.router_config.connection_mode { + ConnectionMode::Grpc => { + // Route to gRPC implementation based on routing mode + match &ctx.router_config.mode { + RoutingMode::Regular { worker_urls } => { + Self::create_grpc_router(worker_urls, &ctx.router_config.policy, ctx).await + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } => { + Self::create_grpc_pd_router( + prefill_urls, + decode_urls, + prefill_policy.as_ref(), + decode_policy.as_ref(), + &ctx.router_config.policy, + ctx, + ) + .await + } + RoutingMode::OpenAI { .. } => { + Err("OpenAI mode requires HTTP connection_mode".to_string()) + } + } + } + ConnectionMode::Http => { + // Route to HTTP implementation based on routing mode + match &ctx.router_config.mode { + RoutingMode::Regular { worker_urls } => { + Self::create_regular_router(worker_urls, &ctx.router_config.policy, ctx) + .await + } + RoutingMode::PrefillDecode { + prefill_urls, + decode_urls, + prefill_policy, + decode_policy, + } => { + Self::create_pd_router( + prefill_urls, + decode_urls, + prefill_policy.as_ref(), + decode_policy.as_ref(), + &ctx.router_config.policy, + ctx, + ) + .await + } + RoutingMode::OpenAI { worker_urls, .. } => { + Self::create_openai_router(worker_urls.clone(), ctx).await + } + } + } + } + } + + /// Create a regular router with injected policy + async fn create_regular_router( + worker_urls: &[String], + policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + // Create policy + let policy = PolicyFactory::create_from_config(policy_config); + + // Create regular router with injected policy and context + let router = Router::new(worker_urls.to_vec(), policy, ctx).await?; + + Ok(Box::new(router)) + } + + /// Create a PD router with injected policy + async fn create_pd_router( + prefill_urls: &[(String, Option)], + decode_urls: &[String], + prefill_policy_config: Option<&PolicyConfig>, + decode_policy_config: Option<&PolicyConfig>, + main_policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + // Create policies - use specific policies if provided, otherwise fall back to main policy + let prefill_policy = + PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config)); + let decode_policy = + PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config)); + + // Create PD router with separate policies and context + let router = PDRouter::new( + prefill_urls.to_vec(), + decode_urls.to_vec(), + prefill_policy, + decode_policy, + ctx, + ) + .await?; + + Ok(Box::new(router)) + } + + /// Create a gRPC router with injected policy + pub async fn create_grpc_router( + worker_urls: &[String], + policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + use super::grpc::router::GrpcRouter; + + // Create policy + let policy = PolicyFactory::create_from_config(policy_config); + + // Create gRPC router with context + let router = GrpcRouter::new(worker_urls.to_vec(), policy, ctx).await?; + + Ok(Box::new(router)) + } + + /// Create a gRPC PD router with tokenizer and worker configuration + pub async fn create_grpc_pd_router( + prefill_urls: &[(String, Option)], + decode_urls: &[String], + prefill_policy_config: Option<&PolicyConfig>, + decode_policy_config: Option<&PolicyConfig>, + main_policy_config: &PolicyConfig, + ctx: &Arc, + ) -> Result, String> { + use super::grpc::pd_router::GrpcPDRouter; + + // Create policies - use specific policies if provided, otherwise fall back to main policy + let prefill_policy = + PolicyFactory::create_from_config(prefill_policy_config.unwrap_or(main_policy_config)); + let decode_policy = + PolicyFactory::create_from_config(decode_policy_config.unwrap_or(main_policy_config)); + + // Create gRPC PD router with context + let router = GrpcPDRouter::new( + prefill_urls.to_vec(), + decode_urls.to_vec(), + prefill_policy, + decode_policy, + ctx, + ) + .await?; + + Ok(Box::new(router)) + } + + /// Create an OpenAI router + async fn create_openai_router( + worker_urls: Vec, + ctx: &Arc, + ) -> Result, String> { + // Use the first worker URL as the OpenAI-compatible base + let base_url = worker_urls + .first() + .cloned() + .ok_or_else(|| "OpenAI mode requires at least one worker URL".to_string())?; + + let router = + OpenAIRouter::new(base_url, Some(ctx.router_config.circuit_breaker.clone())).await?; + + Ok(Box::new(router)) + } + + /// Create an IGW router (placeholder for future implementation) + async fn create_igw_router(_ctx: &Arc) -> Result, String> { + // For now, return an error indicating IGW is not yet implemented + Err("IGW mode is not yet implemented".to_string()) + } +} diff --git a/sgl-router/src/routers/grpc/mod.rs b/sgl-router/src/routers/grpc/mod.rs new file mode 100644 index 000000000..a6a5d8eec --- /dev/null +++ b/sgl-router/src/routers/grpc/mod.rs @@ -0,0 +1,4 @@ +//! gRPC router implementations + +pub mod pd_router; +pub mod router; diff --git a/sgl-router/src/routers/grpc/pd_router.rs b/sgl-router/src/routers/grpc/pd_router.rs new file mode 100644 index 000000000..d227b460d --- /dev/null +++ b/sgl-router/src/routers/grpc/pd_router.rs @@ -0,0 +1,328 @@ +// PD (Prefill-Decode) gRPC Router Implementation + +use crate::config::types::RetryConfig; +use crate::core::{ + BasicWorker, CircuitBreakerConfig, HealthChecker, HealthConfig, Worker, WorkerType, +}; +use crate::grpc::SglangSchedulerClient; +use crate::metrics::RouterMetrics; +use crate::policies::LoadBalancingPolicy; +use crate::reasoning_parser::ParserFactory; +use crate::routers::{RouterTrait, WorkerManagement}; +use crate::tokenizer::traits::Tokenizer; +use crate::tool_parser::ParserRegistry; +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use tracing::{info, warn}; + +/// gRPC PD (Prefill-Decode) router implementation for SGLang +#[allow(dead_code)] // Fields will be used once implementation is complete +pub struct GrpcPDRouter { + /// Prefill worker connections + prefill_workers: Arc>>>, + /// Decode worker connections + decode_workers: Arc>>>, + /// gRPC clients for prefill workers + prefill_grpc_clients: Arc>>, + /// gRPC clients for decode workers + decode_grpc_clients: Arc>>, + /// Load balancing policy for prefill + prefill_policy: Arc, + /// Load balancing policy for decode + decode_policy: Arc, + /// Tokenizer for handling text encoding/decoding + tokenizer: Arc, + /// Reasoning parser factory for structured reasoning outputs + reasoning_parser_factory: ParserFactory, + /// Tool parser registry for function/tool calls + tool_parser_registry: &'static ParserRegistry, + /// Worker health checkers + _prefill_health_checker: Option, + _decode_health_checker: Option, + /// Configuration + timeout_secs: u64, + interval_secs: u64, + dp_aware: bool, + api_key: Option, + retry_config: RetryConfig, + circuit_breaker_config: CircuitBreakerConfig, +} + +impl GrpcPDRouter { + /// Create a new gRPC PD router + pub async fn new( + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + prefill_policy: Arc, + decode_policy: Arc, + ctx: &Arc, + ) -> Result { + // Update metrics + RouterMetrics::set_active_workers(prefill_urls.len() + decode_urls.len()); + + // Extract necessary components from context + let tokenizer = ctx + .tokenizer + .as_ref() + .ok_or_else(|| "gRPC PD router requires tokenizer".to_string())? + .clone(); + let reasoning_parser_factory = ctx + .reasoning_parser_factory + .as_ref() + .ok_or_else(|| "gRPC PD router requires reasoning parser factory".to_string())? + .clone(); + let tool_parser_registry = ctx + .tool_parser_registry + .ok_or_else(|| "gRPC PD router requires tool parser registry".to_string())?; + + // Convert config CircuitBreakerConfig to core CircuitBreakerConfig + let circuit_breaker_config = ctx.router_config.effective_circuit_breaker_config(); + let core_cb_config = CircuitBreakerConfig { + failure_threshold: circuit_breaker_config.failure_threshold, + success_threshold: circuit_breaker_config.success_threshold, + timeout_duration: Duration::from_secs(circuit_breaker_config.timeout_duration_secs), + window_duration: Duration::from_secs(circuit_breaker_config.window_duration_secs), + }; + + // Create gRPC clients for prefill workers + let mut prefill_grpc_clients = HashMap::new(); + for (url, _bootstrap_port) in &prefill_urls { + match SglangSchedulerClient::connect(url).await { + Ok(client) => { + prefill_grpc_clients.insert(url.clone(), client); + info!("Connected to gRPC prefill worker at {}", url); + } + Err(e) => { + warn!("Failed to connect to gRPC prefill worker at {}: {}", url, e); + // Continue with other workers + } + } + } + + // Create gRPC clients for decode workers + let mut decode_grpc_clients = HashMap::new(); + for url in &decode_urls { + match SglangSchedulerClient::connect(url).await { + Ok(client) => { + decode_grpc_clients.insert(url.clone(), client); + info!("Connected to gRPC decode worker at {}", url); + } + Err(e) => { + warn!("Failed to connect to gRPC decode worker at {}: {}", url, e); + // Continue with other workers + } + } + } + + if prefill_grpc_clients.is_empty() && decode_grpc_clients.is_empty() { + return Err("Failed to connect to any gRPC workers".to_string()); + } + + // Create Prefill Worker trait objects with gRPC connection mode + let prefill_workers: Vec> = prefill_urls + .iter() + .map(|(url, bootstrap_port)| { + let worker = BasicWorker::with_connection_mode( + url.clone(), + WorkerType::Prefill { + bootstrap_port: *bootstrap_port, + }, + crate::core::ConnectionMode::Grpc { + port: *bootstrap_port, + }, + ) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }); + Box::new(worker) as Box + }) + .collect(); + + // Create Decode Worker trait objects with gRPC connection mode + let decode_workers: Vec> = decode_urls + .iter() + .map(|url| { + let worker = BasicWorker::with_connection_mode( + url.clone(), + WorkerType::Decode, + crate::core::ConnectionMode::Grpc { port: None }, + ) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }); + Box::new(worker) as Box + }) + .collect(); + + // Initialize policies with workers if needed + if let Some(cache_aware) = prefill_policy + .as_any() + .downcast_ref::() + { + cache_aware.init_workers(&prefill_workers); + } + + if let Some(cache_aware) = decode_policy + .as_any() + .downcast_ref::() + { + cache_aware.init_workers(&decode_workers); + } + + let prefill_workers = Arc::new(RwLock::new(prefill_workers)); + let decode_workers = Arc::new(RwLock::new(decode_workers)); + + let prefill_health_checker = crate::core::start_health_checker( + Arc::clone(&prefill_workers), + ctx.router_config.worker_startup_check_interval_secs, + ); + let decode_health_checker = crate::core::start_health_checker( + Arc::clone(&decode_workers), + ctx.router_config.worker_startup_check_interval_secs, + ); + + Ok(GrpcPDRouter { + prefill_workers, + decode_workers, + prefill_grpc_clients: Arc::new(RwLock::new(prefill_grpc_clients)), + decode_grpc_clients: Arc::new(RwLock::new(decode_grpc_clients)), + prefill_policy, + decode_policy, + tokenizer, + reasoning_parser_factory, + tool_parser_registry, + _prefill_health_checker: Some(prefill_health_checker), + _decode_health_checker: Some(decode_health_checker), + timeout_secs: ctx.router_config.worker_startup_timeout_secs, + interval_secs: ctx.router_config.worker_startup_check_interval_secs, + dp_aware: ctx.router_config.dp_aware, + api_key: ctx.router_config.api_key.clone(), + retry_config: ctx.router_config.effective_retry_config(), + circuit_breaker_config: core_cb_config, + }) + } +} + +impl std::fmt::Debug for GrpcPDRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GrpcPDRouter") + .field( + "prefill_workers_count", + &self.prefill_workers.read().unwrap().len(), + ) + .field( + "decode_workers_count", + &self.decode_workers.read().unwrap().len(), + ) + .field("timeout_secs", &self.timeout_secs) + .field("interval_secs", &self.interval_secs) + .field("dp_aware", &self.dp_aware) + .finish() + } +} + +#[async_trait] +impl RouterTrait for GrpcPDRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn health_generate(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_server_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_models(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_model_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_generate( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::GenerateRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_chat( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::ChatCompletionRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::CompletionRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_embeddings(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_rerank(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn flush_cache(&self) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_worker_loads(&self) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + fn router_type(&self) -> &'static str { + "grpc_pd" + } + + fn readiness(&self) -> Response { + (StatusCode::SERVICE_UNAVAILABLE).into_response() + } +} + +#[async_trait] +impl WorkerManagement for GrpcPDRouter { + async fn add_worker(&self, _worker_url: &str) -> Result { + Err("Not implemented".to_string()) + } + + fn remove_worker(&self, _worker_url: &str) {} + + fn get_worker_urls(&self) -> Vec { + vec![] + } +} diff --git a/sgl-router/src/routers/grpc/router.rs b/sgl-router/src/routers/grpc/router.rs new file mode 100644 index 000000000..5c499125f --- /dev/null +++ b/sgl-router/src/routers/grpc/router.rs @@ -0,0 +1,266 @@ +// gRPC Router Implementation + +use crate::config::types::RetryConfig; +use crate::core::{ + BasicWorker, CircuitBreakerConfig, HealthChecker, HealthConfig, Worker, WorkerType, +}; +use crate::grpc::SglangSchedulerClient; +use crate::metrics::RouterMetrics; +use crate::policies::LoadBalancingPolicy; +use crate::reasoning_parser::ParserFactory; +use crate::routers::{RouterTrait, WorkerManagement}; +use crate::tokenizer::traits::Tokenizer; +use crate::tool_parser::ParserRegistry; +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use tracing::{info, warn}; + +/// gRPC router implementation for SGLang +#[allow(dead_code)] // Fields will be used once implementation is complete +pub struct GrpcRouter { + /// Worker connections + workers: Arc>>>, + /// gRPC clients for each worker + grpc_clients: Arc>>, + /// Load balancing policy + policy: Arc, + /// Tokenizer for handling text encoding/decoding + tokenizer: Arc, + /// Reasoning parser factory for structured reasoning outputs + reasoning_parser_factory: ParserFactory, + /// Tool parser registry for function/tool calls + tool_parser_registry: &'static ParserRegistry, + /// Worker health checker + _health_checker: Option, + /// Configuration + timeout_secs: u64, + interval_secs: u64, + dp_aware: bool, + api_key: Option, + retry_config: RetryConfig, + circuit_breaker_config: CircuitBreakerConfig, +} + +impl GrpcRouter { + /// Create a new gRPC router + pub async fn new( + worker_urls: Vec, + policy: Arc, + ctx: &Arc, + ) -> Result { + // Update metrics + RouterMetrics::set_active_workers(worker_urls.len()); + + // Extract necessary components from context + let tokenizer = ctx + .tokenizer + .as_ref() + .ok_or_else(|| "gRPC router requires tokenizer".to_string())? + .clone(); + let reasoning_parser_factory = ctx + .reasoning_parser_factory + .as_ref() + .ok_or_else(|| "gRPC router requires reasoning parser factory".to_string())? + .clone(); + let tool_parser_registry = ctx + .tool_parser_registry + .ok_or_else(|| "gRPC router requires tool parser registry".to_string())?; + + // Convert config CircuitBreakerConfig to core CircuitBreakerConfig + let circuit_breaker_config = ctx.router_config.effective_circuit_breaker_config(); + let core_cb_config = CircuitBreakerConfig { + failure_threshold: circuit_breaker_config.failure_threshold, + success_threshold: circuit_breaker_config.success_threshold, + timeout_duration: Duration::from_secs(circuit_breaker_config.timeout_duration_secs), + window_duration: Duration::from_secs(circuit_breaker_config.window_duration_secs), + }; + + // Create gRPC clients for each worker + let mut grpc_clients = HashMap::new(); + for url in &worker_urls { + match SglangSchedulerClient::connect(url).await { + Ok(client) => { + grpc_clients.insert(url.clone(), client); + info!("Connected to gRPC worker at {}", url); + } + Err(e) => { + warn!("Failed to connect to gRPC worker at {}: {}", url, e); + // Continue with other workers + } + } + } + + if grpc_clients.is_empty() { + return Err("Failed to connect to any gRPC workers".to_string()); + } + + // Create Worker trait objects with gRPC connection mode + let mut workers: Vec> = Vec::new(); + + // Move clients from the HashMap to the workers + for url in &worker_urls { + if let Some(client) = grpc_clients.remove(url) { + let worker = BasicWorker::with_connection_mode( + url.clone(), + WorkerType::Regular, + crate::core::ConnectionMode::Grpc { port: None }, + ) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }) + .with_grpc_client(client); + + workers.push(Box::new(worker) as Box); + } else { + warn!("No gRPC client for worker {}, skipping", url); + } + } + + // Initialize policy with workers if needed + if let Some(cache_aware) = policy + .as_any() + .downcast_ref::() + { + cache_aware.init_workers(&workers); + } + + let workers = Arc::new(RwLock::new(workers)); + let health_checker = crate::core::start_health_checker( + Arc::clone(&workers), + ctx.router_config.worker_startup_check_interval_secs, + ); + + Ok(GrpcRouter { + workers, + grpc_clients: Arc::new(RwLock::new(grpc_clients)), + policy, + tokenizer, + reasoning_parser_factory, + tool_parser_registry, + _health_checker: Some(health_checker), + timeout_secs: ctx.router_config.worker_startup_timeout_secs, + interval_secs: ctx.router_config.worker_startup_check_interval_secs, + dp_aware: ctx.router_config.dp_aware, + api_key: ctx.router_config.api_key.clone(), + retry_config: ctx.router_config.effective_retry_config(), + circuit_breaker_config: core_cb_config, + }) + } +} + +impl std::fmt::Debug for GrpcRouter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GrpcRouter") + .field("workers_count", &self.workers.read().unwrap().len()) + .field("timeout_secs", &self.timeout_secs) + .field("interval_secs", &self.interval_secs) + .field("dp_aware", &self.dp_aware) + .finish() + } +} + +#[async_trait] +impl RouterTrait for GrpcRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn health_generate(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_server_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_models(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_model_info(&self, _req: Request) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_generate( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::GenerateRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_chat( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::ChatCompletionRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &crate::protocols::spec::CompletionRequest, + ) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_embeddings(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn route_rerank(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn flush_cache(&self) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + async fn get_worker_loads(&self) -> Response { + (StatusCode::NOT_IMPLEMENTED).into_response() + } + + fn router_type(&self) -> &'static str { + "grpc" + } + + fn readiness(&self) -> Response { + (StatusCode::SERVICE_UNAVAILABLE).into_response() + } +} + +#[async_trait] +impl WorkerManagement for GrpcRouter { + async fn add_worker(&self, _worker_url: &str) -> Result { + Err("Not implemented".to_string()) + } + + fn remove_worker(&self, _worker_url: &str) {} + + fn get_worker_urls(&self) -> Vec { + self.workers + .read() + .unwrap() + .iter() + .map(|w| w.url().to_string()) + .collect() + } +} diff --git a/sgl-router/src/routers/header_utils.rs b/sgl-router/src/routers/header_utils.rs new file mode 100644 index 000000000..0adab5bf0 --- /dev/null +++ b/sgl-router/src/routers/header_utils.rs @@ -0,0 +1,53 @@ +use axum::body::Body; +use axum::extract::Request; +use axum::http::HeaderMap; + +/// Copy request headers to a Vec of name-value string pairs +/// Used for forwarding headers to backend workers +pub fn copy_request_headers(req: &Request) -> Vec<(String, String)> { + req.headers() + .iter() + .filter_map(|(name, value)| { + // Convert header value to string, skipping non-UTF8 headers + value + .to_str() + .ok() + .map(|v| (name.to_string(), v.to_string())) + }) + .collect() +} + +/// Convert headers from reqwest Response to axum HeaderMap +/// Filters out hop-by-hop headers that shouldn't be forwarded +pub fn preserve_response_headers(reqwest_headers: &HeaderMap) -> HeaderMap { + let mut headers = HeaderMap::new(); + + for (name, value) in reqwest_headers.iter() { + // Skip hop-by-hop headers that shouldn't be forwarded + let name_str = name.as_str().to_lowercase(); + if should_forward_header(&name_str) { + // The original name and value are already valid, so we can just clone them + headers.insert(name.clone(), value.clone()); + } + } + + headers +} + +/// Determine if a header should be forwarded from backend to client +fn should_forward_header(name: &str) -> bool { + // List of headers that should NOT be forwarded (hop-by-hop headers) + !matches!( + name, + "connection" | + "keep-alive" | + "proxy-authenticate" | + "proxy-authorization" | + "te" | + "trailers" | + "transfer-encoding" | + "upgrade" | + "content-encoding" | // Let axum/hyper handle encoding + "host" // Should not forward the backend's host header + ) +} diff --git a/sgl-router/src/routers/http/mod.rs b/sgl-router/src/routers/http/mod.rs new file mode 100644 index 000000000..9f955b651 --- /dev/null +++ b/sgl-router/src/routers/http/mod.rs @@ -0,0 +1,6 @@ +//! HTTP router implementations + +pub mod openai_router; +pub mod pd_router; +pub mod pd_types; +pub mod router; diff --git a/sgl-router/src/routers/http/openai_router.rs b/sgl-router/src/routers/http/openai_router.rs new file mode 100644 index 000000000..551dd1aa3 --- /dev/null +++ b/sgl-router/src/routers/http/openai_router.rs @@ -0,0 +1,379 @@ +//! OpenAI router implementation (reqwest-based) + +use crate::config::CircuitBreakerConfig; +use crate::core::{CircuitBreaker, CircuitBreakerConfig as CoreCircuitBreakerConfig}; +use crate::protocols::spec::{ChatCompletionRequest, CompletionRequest, GenerateRequest}; +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, +}; +use futures_util::StreamExt; +use std::{ + any::Any, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// Router for OpenAI backend +#[derive(Debug)] +pub struct OpenAIRouter { + /// HTTP client for upstream OpenAI-compatible API + client: reqwest::Client, + /// Base URL for identification (no trailing slash) + base_url: String, + /// Circuit breaker + circuit_breaker: CircuitBreaker, + /// Health status + healthy: AtomicBool, +} + +impl OpenAIRouter { + /// Create a new OpenAI router + pub async fn new( + base_url: String, + circuit_breaker_config: Option, + ) -> Result { + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(300)) + .build() + .map_err(|e| format!("Failed to create HTTP client: {}", e))?; + + let base_url = base_url.trim_end_matches('/').to_string(); + + // Convert circuit breaker config + let core_cb_config = circuit_breaker_config + .map(|cb| CoreCircuitBreakerConfig { + failure_threshold: cb.failure_threshold, + success_threshold: cb.success_threshold, + timeout_duration: std::time::Duration::from_secs(cb.timeout_duration_secs), + window_duration: std::time::Duration::from_secs(cb.window_duration_secs), + }) + .unwrap_or_default(); + + let circuit_breaker = CircuitBreaker::with_config(core_cb_config); + + Ok(Self { + client, + base_url, + circuit_breaker, + healthy: AtomicBool::new(true), + }) + } +} + +#[async_trait] +impl super::super::WorkerManagement for OpenAIRouter { + async fn add_worker(&self, _worker_url: &str) -> Result { + Err("Cannot add workers to OpenAI router".to_string()) + } + + fn remove_worker(&self, _worker_url: &str) { + // No-op for OpenAI router + } + + fn get_worker_urls(&self) -> Vec { + vec![self.base_url.clone()] + } +} + +#[async_trait] +impl super::super::RouterTrait for OpenAIRouter { + fn as_any(&self) -> &dyn Any { + self + } + + async fn health(&self, _req: Request) -> Response { + // Simple upstream probe: GET {base}/v1/models without auth + let url = format!("{}/v1/models", self.base_url); + match self + .client + .get(&url) + .timeout(std::time::Duration::from_secs(2)) + .send() + .await + { + Ok(resp) => { + let code = resp.status(); + // Treat success and auth-required as healthy (endpoint reachable) + if code.is_success() || code.as_u16() == 401 || code.as_u16() == 403 { + (StatusCode::OK, "OK").into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Upstream status: {}", code), + ) + .into_response() + } + } + Err(e) => ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Upstream error: {}", e), + ) + .into_response(), + } + } + + async fn health_generate(&self, _req: Request) -> Response { + // For OpenAI, health_generate is the same as health + self.health(_req).await + } + + async fn get_server_info(&self, _req: Request) -> Response { + let info = serde_json::json!({ + "router_type": "openai", + "workers": 1, + "base_url": &self.base_url + }); + (StatusCode::OK, info.to_string()).into_response() + } + + async fn get_models(&self, req: Request) -> Response { + // Proxy to upstream /v1/models; forward Authorization header if provided + let headers = req.headers(); + + let mut upstream = self.client.get(format!("{}/v1/models", self.base_url)); + + if let Some(auth) = headers + .get("authorization") + .or_else(|| headers.get("Authorization")) + { + upstream = upstream.header("Authorization", auth); + } + + match upstream.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + let content_type = res.headers().get(CONTENT_TYPE).cloned(); + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(axum::body::Body::from(body)); + *response.status_mut() = status; + if let Some(ct) = content_type { + response.headers_mut().insert(CONTENT_TYPE, ct); + } + response + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read upstream response: {}", e), + ) + .into_response(), + } + } + Err(e) => ( + StatusCode::BAD_GATEWAY, + format!("Failed to contact upstream: {}", e), + ) + .into_response(), + } + } + + async fn get_model_info(&self, _req: Request) -> Response { + // Not directly supported without model param; return 501 + ( + StatusCode::NOT_IMPLEMENTED, + "get_model_info not implemented for OpenAI router", + ) + .into_response() + } + + async fn route_generate( + &self, + _headers: Option<&HeaderMap>, + _body: &GenerateRequest, + ) -> Response { + // Generate endpoint is SGLang-specific, not supported for OpenAI backend + ( + StatusCode::NOT_IMPLEMENTED, + "Generate endpoint not supported for OpenAI backend", + ) + .into_response() + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + ) -> Response { + if !self.circuit_breaker.can_execute() { + return (StatusCode::SERVICE_UNAVAILABLE, "Circuit breaker open").into_response(); + } + + // Serialize request body, removing SGLang-only fields + let mut payload = match serde_json::to_value(body) { + Ok(v) => v, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Failed to serialize request: {}", e), + ) + .into_response(); + } + }; + if let Some(obj) = payload.as_object_mut() { + for key in [ + "top_k", + "min_p", + "min_tokens", + "regex", + "ebnf", + "stop_token_ids", + "no_stop_trim", + "ignore_eos", + "continue_final_message", + "skip_special_tokens", + "lora_path", + "session_params", + "separate_reasoning", + "stream_reasoning", + "chat_template_kwargs", + "return_hidden_states", + "repetition_penalty", + ] { + obj.remove(key); + } + } + + let url = format!("{}/v1/chat/completions", self.base_url); + let mut req = self.client.post(&url).json(&payload); + + // Forward Authorization header if provided + if let Some(h) = headers { + if let Some(auth) = h.get("authorization").or_else(|| h.get("Authorization")) { + req = req.header("Authorization", auth); + } + } + + // Accept SSE when stream=true + if body.stream { + req = req.header("Accept", "text/event-stream"); + } + + let resp = match req.send().await { + Ok(r) => r, + Err(e) => { + self.circuit_breaker.record_failure(); + return ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Failed to contact upstream: {}", e), + ) + .into_response(); + } + }; + + let status = StatusCode::from_u16(resp.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + if !body.stream { + // Capture Content-Type before consuming response body + let content_type = resp.headers().get(CONTENT_TYPE).cloned(); + match resp.bytes().await { + Ok(body) => { + self.circuit_breaker.record_success(); + let mut response = Response::new(axum::body::Body::from(body)); + *response.status_mut() = status; + if let Some(ct) = content_type { + response.headers_mut().insert(CONTENT_TYPE, ct); + } + response + } + Err(e) => { + self.circuit_breaker.record_failure(); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response: {}", e), + ) + .into_response() + } + } + } else { + // Stream SSE bytes to client + let stream = resp.bytes_stream(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + let mut s = stream; + while let Some(chunk) = s.next().await { + match chunk { + Ok(bytes) => { + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + }); + let mut response = Response::new(Body::from_stream( + tokio_stream::wrappers::UnboundedReceiverStream::new(rx), + )); + *response.status_mut() = status; + response + .headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + response + } + } + + async fn route_completion( + &self, + _headers: Option<&HeaderMap>, + _body: &CompletionRequest, + ) -> Response { + // Completion endpoint not implemented for OpenAI backend + ( + StatusCode::NOT_IMPLEMENTED, + "Completion endpoint not implemented for OpenAI backend", + ) + .into_response() + } + + async fn flush_cache(&self) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "flush_cache not supported for OpenAI router", + ) + .into_response() + } + + async fn get_worker_loads(&self) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "get_worker_loads not supported for OpenAI router", + ) + .into_response() + } + + fn router_type(&self) -> &'static str { + "openai" + } + + fn readiness(&self) -> Response { + if self.healthy.load(Ordering::Acquire) && self.circuit_breaker.can_execute() { + (StatusCode::OK, "Ready").into_response() + } else { + (StatusCode::SERVICE_UNAVAILABLE, "Not ready").into_response() + } + } + + async fn route_embeddings(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Embeddings endpoint not implemented for OpenAI backend", + ) + .into_response() + } + + async fn route_rerank(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + ( + StatusCode::NOT_IMPLEMENTED, + "Rerank endpoint not implemented for OpenAI backend", + ) + .into_response() + } +} diff --git a/sgl-router/src/routers/http/pd_router.rs b/sgl-router/src/routers/http/pd_router.rs new file mode 100644 index 000000000..528ead5f5 --- /dev/null +++ b/sgl-router/src/routers/http/pd_router.rs @@ -0,0 +1,2500 @@ +// PD (Prefill-Decode) Router Implementation +// This module handles routing for disaggregated prefill-decode systems +use super::pd_types::{api_path, PDRouterError}; +use crate::config::types::RetryConfig; +use crate::core::{ + is_retryable_status, BasicWorker, CircuitBreakerConfig, HealthChecker, HealthConfig, + RetryExecutor, Worker, WorkerFactory, WorkerLoadGuard, WorkerType, +}; +use crate::metrics::RouterMetrics; +use crate::policies::LoadBalancingPolicy; +use crate::protocols::spec::{ + ChatCompletionRequest, ChatMessage, CompletionRequest, GenerateRequest, StringOrArray, + UserMessageContent, +}; +use crate::routers::header_utils; +use crate::routers::{RouterTrait, WorkerManagement}; +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use futures_util::StreamExt; +use reqwest::Client; +use serde::Serialize; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error, info, warn}; + +#[derive(Debug)] +pub struct PDRouter { + pub prefill_workers: Arc>>>, + pub decode_workers: Arc>>>, + pub prefill_policy: Arc, + pub decode_policy: Arc, + pub worker_startup_timeout_secs: u64, + pub worker_startup_check_interval_secs: u64, + pub worker_loads: Arc>>, + pub load_monitor_handle: Option>>, + pub client: Client, + // Dedicated client for prefill fire-and-forget (non-logprob) requests + pub prefill_client: Client, + pub retry_config: RetryConfig, + pub circuit_breaker_config: CircuitBreakerConfig, + _prefill_health_checker: Option, + _decode_health_checker: Option, + // Channel for sending prefill responses to background workers for draining + prefill_drain_tx: mpsc::Sender, +} + +// Request context for PD router operations +#[derive(Clone)] +struct PDRequestContext { + route: &'static str, + batch_size: Option, + is_stream: bool, + return_logprob: bool, + request_text: Option, +} + +impl PDRouter { + // Dynamic worker management methods for service discovery + + // Private helper method to perform health check on a new server + async fn wait_for_server_health(&self, url: &str) -> Result<(), PDRouterError> { + crate::routers::http::router::Router::wait_for_healthy_workers( + &[url.to_string()], + self.worker_startup_timeout_secs, + self.worker_startup_check_interval_secs, + ) + .await + .map_err(|_| PDRouterError::HealthCheckFailed { + url: url.to_string(), + }) + } + + // Generic helper for processing all workers with an endpoint + async fn process_workers( + &self, + workers: &RwLock>>, + worker_type: &str, + endpoint: &str, + ) -> (Vec, Vec) { + let mut results = Vec::new(); + let mut errors = Vec::new(); + + // Get worker URLs first to avoid holding lock across await + let urls = match workers.read() { + Ok(workers) => workers + .iter() + .map(|w| w.url().to_string()) + .collect::>(), + Err(_) => { + errors.push(format!("Failed to access {} workers", worker_type)); + Vec::new() + } + }; + + // Process each worker + for worker_url in urls { + let url = format!("{}/{}", worker_url, endpoint); + match self.client.post(&url).send().await { + Ok(res) if res.status().is_success() => { + results.push(format!("{} {}: OK", worker_type, worker_url)); + } + Ok(res) => { + errors.push(format!( + "{} {} returned status: {}", + worker_type, + worker_url, + res.status() + )); + } + Err(e) => { + errors.push(format!("{} {} error: {}", worker_type, worker_url, e)); + } + } + } + + (results, errors) + } + + // Helper to get worker URLs from a worker collection + fn get_worker_urls( + workers: &RwLock>>, + worker_type: &str, + ) -> Result, String> { + workers + .read() + .map(|workers| { + workers + .iter() + .map(|w| w.url().to_string()) + .collect::>() + }) + .map_err(|_| format!("Failed to access {} workers", worker_type)) + } + + // Generic helper for proxying requests to the first worker + async fn proxy_to_first_worker( + &self, + workers: &RwLock>>, + endpoint: &str, + worker_type: &str, + headers: Option>, + ) -> Response { + // Get first worker URL to avoid holding lock across await + let first_worker_url = match workers.read() { + Ok(workers) => workers.first().map(|w| w.url().to_string()), + Err(_) => { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to access {} workers", worker_type), + ) + .into_response(); + } + }; + + if let Some(worker_url) = first_worker_url { + let url = format!("{}/{}", worker_url, endpoint); + let mut request_builder = self.client.get(&url); + + // Add headers if provided + if let Some(headers) = headers { + for (name, value) in headers { + request_builder = request_builder.header(name, value); + } + } + + match request_builder.send().await { + Ok(res) if res.status().is_success() => { + let response_headers = header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(axum::body::Body::from(body)); + *response.status_mut() = StatusCode::OK; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + error!("Failed to read response body: {}", e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response body: {}", e), + ) + .into_response() + } + } + } + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + ( + status, + format!("{} server returned status: {}", worker_type, res.status()), + ) + .into_response() + } + Err(e) => { + error!("Failed to proxy request to {} server: {}", worker_type, e); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to proxy request: {}", e), + ) + .into_response() + } + } + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("No {} servers available", worker_type), + ) + .into_response() + } + } + + pub async fn add_prefill_server( + &self, + url: String, + bootstrap_port: Option, + ) -> Result { + // Wait for the new server to be healthy + self.wait_for_server_health(&url).await?; + + // Create Worker for the new prefill server with circuit breaker configuration + let worker = WorkerFactory::create_prefill_with_config( + url.clone(), + bootstrap_port, + self.circuit_breaker_config.clone(), + ); + + // Add to prefill workers list + let mut workers = self + .prefill_workers + .write() + .map_err(|_| PDRouterError::LockError { + operation: "prefill_workers write".to_string(), + })?; + + // Check if already exists + if workers.iter().any(|w| w.url() == url) { + return Err(PDRouterError::WorkerAlreadyExists { url: url.clone() }); + } + + workers.push(worker); + + // Update cache-aware policy if applicable + drop(workers); // Release write lock + if let Some(cache_policy) = self + .prefill_policy + .as_any() + .downcast_ref::() + { + cache_policy.add_worker(&url); + } + + info!("Added prefill server: {}", url); + Ok(format!("Successfully added prefill server: {}", url)) + } + + pub async fn add_decode_server(&self, url: String) -> Result { + // Wait for the new server to be healthy + self.wait_for_server_health(&url).await?; + + // Create Worker for the new decode server with circuit breaker configuration + let worker = WorkerFactory::create_decode_with_config( + url.clone(), + self.circuit_breaker_config.clone(), + ); + + // Add to decode workers list + let mut workers = self + .decode_workers + .write() + .map_err(|_| PDRouterError::LockError { + operation: "decode_workers write".to_string(), + })?; + + // Check if already exists + if workers.iter().any(|w| w.url() == url) { + return Err(PDRouterError::WorkerAlreadyExists { url: url.clone() }); + } + + workers.push(worker); + + // Update cache-aware policy if applicable + drop(workers); // Release write lock + if let Some(cache_policy) = self + .decode_policy + .as_any() + .downcast_ref::() + { + cache_policy.add_worker(&url); + } + + info!("Added decode server: {}", url); + Ok(format!("Successfully added decode server: {}", url)) + } + + pub async fn remove_prefill_server(&self, url: &str) -> Result { + let mut workers = self + .prefill_workers + .write() + .map_err(|_| PDRouterError::LockError { + operation: "prefill_workers write".to_string(), + })?; + + // Find and remove the server + let initial_len = workers.len(); + workers.retain(|w| w.url() != url); + + if workers.len() == initial_len { + return Err(PDRouterError::WorkerNotFound { + url: url.to_string(), + }); + } + + // Remove from cache-aware policy if applicable + if let Some(cache_policy) = self + .prefill_policy + .as_any() + .downcast_ref::() + { + cache_policy.remove_worker(url); + } + + info!("Removed prefill server: {}", url); + Ok(format!("Successfully removed prefill server: {}", url)) + } + + pub async fn remove_decode_server(&self, url: &str) -> Result { + let mut workers = self + .decode_workers + .write() + .map_err(|_| PDRouterError::LockError { + operation: "decode_workers write".to_string(), + })?; + + // Find and remove the server + let initial_len = workers.len(); + workers.retain(|w| w.url() != url); + + if workers.len() == initial_len { + return Err(PDRouterError::WorkerNotFound { + url: url.to_string(), + }); + } + + // Remove from cache-aware policy if applicable + if let Some(cache_policy) = self + .decode_policy + .as_any() + .downcast_ref::() + { + cache_policy.remove_worker(url); + } + + info!("Removed decode server: {}", url); + Ok(format!("Successfully removed decode server: {}", url)) + } + + #[allow(clippy::too_many_arguments)] + pub async fn new( + prefill_urls: Vec<(String, Option)>, + decode_urls: Vec, + prefill_policy: Arc, + decode_policy: Arc, + ctx: &Arc, + ) -> Result { + // Convert config CircuitBreakerConfig to core CircuitBreakerConfig + let circuit_breaker_config = ctx.router_config.effective_circuit_breaker_config(); + let core_cb_config = CircuitBreakerConfig { + failure_threshold: circuit_breaker_config.failure_threshold, + success_threshold: circuit_breaker_config.success_threshold, + timeout_duration: Duration::from_secs(circuit_breaker_config.timeout_duration_secs), + window_duration: Duration::from_secs(circuit_breaker_config.window_duration_secs), + }; + + // Convert URLs to Worker trait objects with health check config + let prefill_workers: Vec> = prefill_urls + .into_iter() + .map(|(url, port)| { + let worker = BasicWorker::new( + url, + WorkerType::Prefill { + bootstrap_port: port, + }, + ) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }); + Box::new(worker) as Box + }) + .collect(); + + let decode_workers: Vec> = decode_urls + .into_iter() + .map(|url| { + let worker = BasicWorker::new(url, WorkerType::Decode) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }); + Box::new(worker) as Box + }) + .collect(); + + // Wait for PD workers to be healthy (skip if empty - for service discovery mode) + let all_urls: Vec = prefill_workers + .iter() + .chain(decode_workers.iter()) + .map(|worker| worker.url().to_string()) + .collect(); + if !all_urls.is_empty() { + crate::routers::http::router::Router::wait_for_healthy_workers( + &all_urls, + ctx.router_config.worker_startup_timeout_secs, + ctx.router_config.worker_startup_check_interval_secs, + ) + .await?; + } + + // Initialize cache-aware policies with workers + if let Some(cache_policy) = prefill_policy + .as_any() + .downcast_ref::() + { + cache_policy.init_workers(&prefill_workers); + } + + if let Some(cache_policy) = decode_policy + .as_any() + .downcast_ref::() + { + cache_policy.init_workers(&decode_workers); + } + + // Set up background load monitoring for power-of-two selection + let (tx, rx) = tokio::sync::watch::channel(HashMap::new()); + let worker_loads = Arc::new(rx); + + let load_monitor_handle = + if prefill_policy.name() == "power_of_two" || decode_policy.name() == "power_of_two" { + let monitor_urls = all_urls.clone(); + let monitor_interval = ctx.router_config.worker_startup_check_interval_secs; + let monitor_client = ctx.client.clone(); + let prefill_policy_clone = Arc::clone(&prefill_policy); + let decode_policy_clone = Arc::clone(&decode_policy); + + Some(Arc::new(tokio::spawn(async move { + Self::monitor_worker_loads_with_client( + monitor_urls, + tx, + monitor_interval, + monitor_client, + prefill_policy_clone, + decode_policy_clone, + ) + .await; + }))) + } else { + None + }; + + let prefill_workers = Arc::new(RwLock::new(prefill_workers)); + let decode_workers = Arc::new(RwLock::new(decode_workers)); + + // Start health checkers for both worker pools + let prefill_health_checker = crate::core::start_health_checker( + Arc::clone(&prefill_workers), + ctx.router_config.health_check.check_interval_secs, + ); + let decode_health_checker = crate::core::start_health_checker( + Arc::clone(&decode_workers), + ctx.router_config.health_check.check_interval_secs, + ); + + // Build a dedicated prefill client for fire-and-forget semantics + let prefill_client = reqwest::Client::builder() + .pool_max_idle_per_host(0) + .http1_only() + .connect_timeout(Duration::from_millis(300)) + .timeout(Duration::from_secs(ctx.router_config.request_timeout_secs)) + .build() + .map_err(|e| format!("Failed to build prefill client: {}", e))?; + + // Create bounded channel for prefill response draining + // Larger buffer for high concurrency scenarios + let (prefill_drain_tx, mut prefill_drain_rx) = mpsc::channel::(2000); + + // Spawn a coordinator with limited concurrent drain tasks + // This prevents unbounded task spawning under extreme load + tokio::spawn(async move { + info!("Prefill drain coordinator started"); + + // Use a semaphore to limit concurrent drain operations + let max_concurrent_drains = 100; + let semaphore = Arc::new(tokio::sync::Semaphore::new(max_concurrent_drains)); + + while let Some(response) = prefill_drain_rx.recv().await { + let permit = semaphore.clone().acquire_owned().await; + + match permit { + Ok(permit) => { + // Spawn a task to drain this response + tokio::spawn(async move { + let url = response.url().to_string(); + let status = response.status(); + + if !status.is_success() { + error!("Prefill drain: error status={} url={}", status, url); + RouterMetrics::record_pd_prefill_error(&url); + } + + // Drain the response body efficiently + // Use streaming to avoid loading entire body into memory + let start = std::time::Instant::now(); + let mut stream = response.bytes_stream(); + let mut bytes_drained = 0; + + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(chunk) => bytes_drained += chunk.len(), + Err(e) => { + debug!( + "Prefill drain: error streaming url={} error={}", + url, e + ); + break; + } + } + } + + let elapsed = start.elapsed(); + if elapsed > Duration::from_millis(100) { + // Only log slow drains + debug!( + "Prefill drain: slow drain {} bytes from {} in {:?}", + bytes_drained, url, elapsed + ); + } + + // Permit is automatically released when dropped + drop(permit); + }); + } + Err(_) => { + // Semaphore closed, shutting down + break; + } + } + } + info!("Prefill drain coordinator shutting down"); + }); + + Ok(PDRouter { + prefill_workers, + decode_workers, + prefill_policy, + decode_policy, + worker_startup_timeout_secs: ctx.router_config.worker_startup_timeout_secs, + worker_startup_check_interval_secs: ctx + .router_config + .worker_startup_check_interval_secs, + worker_loads, + load_monitor_handle, + client: ctx.client.clone(), + prefill_client, + prefill_drain_tx, + retry_config: ctx.router_config.effective_retry_config(), + circuit_breaker_config: core_cb_config, + _prefill_health_checker: Some(prefill_health_checker), + _decode_health_checker: Some(decode_health_checker), + }) + } + + // Helper to handle server selection errors + fn handle_server_selection_error(error: String) -> Response { + error!("Failed to select PD pair error={}", error); + RouterMetrics::record_pd_error("server_selection"); + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("No available servers: {}", error), + ) + .into_response() + } + + // Helper to handle serialization errors + fn handle_serialization_error(error: impl std::fmt::Display) -> Response { + error!("Failed to serialize request error={}", error); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to serialize request", + ) + .into_response() + } + + // Helper to determine batch size from a GenerateRequest + fn get_generate_batch_size(req: &GenerateRequest) -> Option { + // Check prompt array + if let Some(StringOrArray::Array(arr)) = &req.prompt { + if !arr.is_empty() { + return Some(arr.len()); + } + } + // Check text array + if let Some(text) = &req.text { + if text.contains("[") && text.contains("]") { + // This is a simplified check - in reality we'd need to parse JSON + return None; // For now, fall back to non-batch + } + } + None + } + + // Helper to determine batch size from a ChatCompletionRequest + fn get_chat_batch_size(req: &ChatCompletionRequest) -> Option { + // Check 'n' parameter for multiple responses + if let Some(n) = req.n { + if n > 1 { + return Some(n as usize); + } + } + None + } + + // Helper to determine batch size from a CompletionRequest + fn get_completion_batch_size(req: &CompletionRequest) -> Option { + // Check prompt array + if let StringOrArray::Array(arr) = &req.prompt { + if !arr.is_empty() { + return Some(arr.len()); + } + } + None + } + + // Helper to inject bootstrap fields into an existing JSON request value + fn inject_bootstrap_into_value( + mut original: Value, + prefill_worker: &dyn Worker, + batch_size: Option, + ) -> Result { + let bootstrap_port = match prefill_worker.worker_type() { + crate::core::WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + let hostname = super::pd_types::get_hostname(prefill_worker.url()); + + let obj = original + .as_object_mut() + .ok_or_else(|| "Request must be a JSON object".to_string())?; + + if let Some(n) = batch_size { + let mut hosts = Vec::with_capacity(n); + let mut ports = Vec::with_capacity(n); + let mut rooms = Vec::with_capacity(n); + for _ in 0..n { + hosts.push(hostname.clone()); + ports.push(bootstrap_port); + rooms.push(super::pd_types::generate_room_id()); + } + obj.insert( + "bootstrap_host".to_string(), + Value::Array(hosts.into_iter().map(serde_json::Value::from).collect()), + ); + obj.insert( + "bootstrap_port".to_string(), + Value::Array( + ports + .into_iter() + .map(|p| match p { + Some(v) => serde_json::Value::from(v), + None => Value::Null, + }) + .collect(), + ), + ); + obj.insert( + "bootstrap_room".to_string(), + Value::Array(rooms.into_iter().map(serde_json::Value::from).collect()), + ); + } else { + obj.insert( + "bootstrap_host".to_string(), + serde_json::Value::from(hostname), + ); + obj.insert( + "bootstrap_port".to_string(), + match bootstrap_port { + Some(v) => serde_json::Value::from(v), + None => Value::Null, + }, + ); + obj.insert( + "bootstrap_room".to_string(), + serde_json::Value::from(super::pd_types::generate_room_id()), + ); + } + Ok(original) + } + + // Execute the dual dispatch to prefill and decode servers with retries and bootstrap injection + async fn execute_dual_dispatch( + &self, + headers: Option<&HeaderMap>, + original_request: &T, + context: PDRequestContext, + ) -> Response { + let start_time = Instant::now(); + + let route = context.route; + RetryExecutor::execute_response_with_retry( + &self.retry_config, + // Operation per attempt + { + let original_request = original_request.clone(); + move |attempt: u32| { + let original_request = original_request.clone(); + let context = context.clone(); + async move { + // Select workers fresh for each attempt + let (prefill, decode) = + match self.select_pd_pair(context.request_text.as_deref()).await { + Ok(pair) => pair, + Err(e) => { + RouterMetrics::record_pd_error("server_selection"); + return Self::handle_server_selection_error(e); + } + }; + + debug!( + "PD retry attempt {} using prefill={} decode={}", + attempt, + prefill.url(), + decode.url() + ); + + // Serialize the original request + let mut json_request = match serde_json::to_value(&original_request) { + Ok(v) => v, + Err(e) => return Self::handle_serialization_error(e), + }; + + // Inject bootstrap based on current prefill worker + json_request = match Self::inject_bootstrap_into_value( + json_request, + prefill.as_ref(), + context.batch_size, + ) { + Ok(v) => v, + Err(e) => return Self::handle_serialization_error(e), + }; + + // Execute the actual dual dispatch + let response = self + .execute_dual_dispatch_internal( + headers, + json_request, + context, + prefill.as_ref(), + decode.as_ref(), + start_time, + ) + .await; + + // Record outcomes for circuit breakers + let _status = response.status(); + let not_error = _status.is_success() || _status.is_client_error(); + prefill.record_outcome(not_error); + decode.record_outcome(not_error); + + response + } + } + }, + // Should retry predicate + |res, _attempt| is_retryable_status(res.status()), + // On backoff hook + |delay, attempt| { + RouterMetrics::record_retry(route); + RouterMetrics::record_retry_backoff_duration(delay, attempt); + }, + // On exhausted hook + || RouterMetrics::record_retries_exhausted(route), + ) + .await + } + + async fn handle_decode_error_response( + &self, + res: reqwest::Response, + context: &PDRequestContext, + prefill: &dyn Worker, + decode: &dyn Worker, + ) -> Response { + let status = res.status(); + + if context.is_stream { + // Handle streaming error response + let response_headers = header_utils::preserve_response_headers(res.headers()); + let error_payload = match res.bytes().await { + Ok(error_body) => { + if let Ok(error_json) = serde_json::from_slice::(&error_body) { + json!({ "message": error_json, "status": status.as_u16() }) + } else { + json!({ "message": String::from_utf8_lossy(&error_body).to_string(), "status": status.as_u16() }) + } + } + Err(e) => { + json!({ "message": format!("Decode server error: {}", e), "status": status.as_u16() }) + } + }; + + let sse_data = format!( + "data: {{'error': {}}}", + serde_json::to_string(&error_payload).unwrap_or_default() + ); + let error_stream = tokio_stream::once(Ok(axum::body::Bytes::from(sse_data))); + + let decode_url = decode.url().to_string(); + self.create_streaming_response( + error_stream, + status, + None, + context.return_logprob, + Some(decode_url), + Some(response_headers), + prefill, + decode, + ) + } else { + // Handle non-streaming error response + match res.bytes().await { + Ok(error_body) => (status, error_body).into_response(), + Err(e) => (status, format!("Decode server error: {}", e)).into_response(), + } + } + } + + // Internal method that performs the actual dual dispatch (without retry logic) + async fn execute_dual_dispatch_internal( + &self, + headers: Option<&HeaderMap>, + json_request: Value, + context: PDRequestContext, + prefill: &dyn Worker, + decode: &dyn Worker, + start_time: Instant, + ) -> Response { + // For non-streaming: use guard for automatic load management + // For streaming: load will be managed in create_streaming_response + let _guard = if !context.is_stream { + Some(WorkerLoadGuard::new_multi(vec![prefill, decode])) + } else { + None + }; + + // Build decode request with shared client + let decode_request = self.build_post_with_headers( + &self.client, + decode.url(), + context.route, + &json_request, + headers, + false, + ); + + // Send both requests concurrently + debug!( + "Sending concurrent requests to prefill={} decode={}", + prefill.url(), + decode.url() + ); + + if context.return_logprob { + // Build prefill request with shared client when we need response body + let prefill_request = self.build_post_with_headers( + &self.client, + prefill.url(), + context.route, + &json_request, + headers, + false, + ); + // When we need logprobs, wait for both responses + let (prefill_result, decode_result) = + tokio::join!(prefill_request.send(), decode_request.send()); + debug!("Received responses from both servers"); + + // Update metrics + let duration = start_time.elapsed(); + RouterMetrics::record_pd_request_duration(context.route, duration); + RouterMetrics::record_pd_request(context.route); + RouterMetrics::record_pd_prefill_request(prefill.url()); + RouterMetrics::record_pd_decode_request(decode.url()); + + // Process decode response with prefill for logprobs + debug!("Processing decode response with logprobs"); + match decode_result { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + debug!("Decode response status: {}", status); + + if !status.is_success() { + RouterMetrics::record_pd_decode_error(decode.url()); + error!( + "Decode server returned error status decode_url={} status={}", + decode.url(), + status + ); + + return self + .handle_decode_error_response(res, &context, prefill, decode) + .await; + } + + // Process prefill response for logprobs + let prefill_body = match self + .process_prefill_response( + prefill_result, + prefill.url(), + context.return_logprob, + ) + .await + { + Ok((_, body)) => body, + Err(error_response) => return error_response, + }; + + if context.is_stream { + // Streaming response with logprobs + let prefill_logprobs = prefill_body + .as_ref() + .and_then(|body| serde_json::from_slice::(body).ok()) + .and_then(|json| { + json.pointer("/meta_info/input_token_logprobs").cloned() + }); + + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + self.create_streaming_response( + res.bytes_stream(), + status, + prefill_logprobs, + context.return_logprob, + None, + Some(response_headers), + prefill, + decode, + ) + } else { + // Non-streaming response with logprobs + self.process_non_streaming_response( + res, + status, + context.return_logprob, + prefill_body, + ) + .await + } + } + Err(e) => { + error!( + decode_url = %decode.url(), + error = %e, + "Decode request failed" + ); + RouterMetrics::record_pd_decode_error(decode.url()); + ( + StatusCode::BAD_GATEWAY, + format!("Decode server error: {}", e), + ) + .into_response() + } + } + } else { + // When we don't need logprobs, only wait for decode response + // Send both requests concurrently but don't wait for prefill + // Use dedicated prefill client with Connection: close + let prefill_future = self + .build_post_with_headers( + &self.prefill_client, + prefill.url(), + context.route, + &json_request, + headers, + true, + ) + .send(); + let decode_future = decode_request.send(); + + // Send prefill response to background worker for draining + // This ensures HTTP compliance without blocking + let drain_tx = self.prefill_drain_tx.clone(); + let prefill_url = prefill.url().to_string(); + tokio::spawn(async move { + if let Ok(response) = prefill_future.await { + // Try to send to drain worker + // If channel is full (under extreme load), drain inline as fallback + match drain_tx.try_send(response) { + Ok(_) => { + // Successfully queued for draining + debug!("Prefill response queued for draining"); + } + Err(mpsc::error::TrySendError::Full(response)) => { + // Channel full - drain inline as fallback + warn!("Prefill drain channel full (capacity exceeded), draining inline for {}", prefill_url); + RouterMetrics::record_pd_prefill_error(&prefill_url); + + // Drain inline with timeout to prevent blocking too long + let drain_future = async { + let mut stream = response.bytes_stream(); + while stream.next().await.is_some() { + // Just drain + } + }; + + match tokio::time::timeout(Duration::from_secs(1), drain_future).await { + Ok(_) => debug!("Inline drain completed for {}", prefill_url), + Err(_) => error!("Inline drain timeout for {}", prefill_url), + } + } + Err(mpsc::error::TrySendError::Closed(_)) => { + error!("Prefill drain channel closed!"); + } + } + } + }); + + // Wait only for decode response + let decode_result = decode_future.await; + debug!("Received decode response"); + + // Update metrics + let duration = start_time.elapsed(); + RouterMetrics::record_pd_request_duration(context.route, duration); + RouterMetrics::record_pd_request(context.route); + RouterMetrics::record_pd_prefill_request(prefill.url()); + RouterMetrics::record_pd_decode_request(decode.url()); + + // Process decode response immediately + debug!("Processing decode response (no logprobs)"); + match decode_result { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + debug!("Decode response status: {}", status); + + if !status.is_success() { + RouterMetrics::record_pd_decode_error(decode.url()); + error!( + "Decode server returned error status decode_url={} status={}", + decode.url(), + status + ); + + self.handle_decode_error_response(res, &context, prefill, decode) + .await + } else if context.is_stream { + // Streaming response without logprobs - direct passthrough + let decode_url = decode.url().to_string(); + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + self.create_streaming_response( + res.bytes_stream(), + status, + None, + false, + Some(decode_url), + Some(response_headers), + prefill, + decode, + ) + } else { + // Non-streaming response without logprobs - direct passthrough like fast version + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(decode_body) => { + let mut response = + Response::new(axum::body::Body::from(decode_body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + error!("Failed to read decode response: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read response") + .into_response() + } + } + } + } + Err(e) => { + error!( + decode_url = %decode.url(), + error = %e, + "Decode request failed" + ); + RouterMetrics::record_pd_decode_error(decode.url()); + ( + StatusCode::BAD_GATEWAY, + format!("Decode server error: {}", e), + ) + .into_response() + } + } + } + } + + // Check if either prefill or decode policy needs request text + fn policies_need_request_text(&self) -> bool { + self.prefill_policy.needs_request_text() || self.decode_policy.needs_request_text() + } + + // Select a pair of prefill and decode servers considering circuit breaker state + async fn select_pd_pair( + &self, + request_text: Option<&str>, + ) -> Result<(Box, Box), String> { + // Get read locks for both worker lists + let prefill_workers = self + .prefill_workers + .read() + .map_err(|e| format!("Failed to acquire prefill workers lock: {}", e))?; + let decode_workers = self + .decode_workers + .read() + .map_err(|e| format!("Failed to acquire decode workers lock: {}", e))?; + + // Select workers using helper function + let prefill = Self::pick_worker_by_policy( + &prefill_workers, + &*self.prefill_policy, + request_text, + "prefill", + )?; + + let decode = Self::pick_worker_by_policy( + &decode_workers, + &*self.decode_policy, + request_text, + "decode", + )?; + + Ok((prefill, decode)) + } + + // Helper function to select a worker using the policy + fn pick_worker_by_policy( + workers: &[Box], + policy: &dyn LoadBalancingPolicy, + request_text: Option<&str>, + worker_type: &str, + ) -> Result, String> { + // Check if we have any workers + if workers.is_empty() { + return Err(format!( + "No {} workers available. Please check if {} servers are configured and healthy.", + worker_type, worker_type + )); + } + + // Filter available workers (healthy + circuit breaker not open) + let available_workers: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .map(|w| w.clone_worker()) + .collect(); + + if available_workers.is_empty() { + return Err(format!( + "No available {} workers (all circuits open or unhealthy)", + worker_type + )); + } + + // Let policy select from available workers only + match policy.select_worker(&available_workers, request_text) { + Some(idx) => Ok(available_workers[idx].clone_worker()), + None => Err(format!("Policy could not select a {} worker", worker_type)), + } + } + + // Background task to monitor worker loads with shared client + async fn monitor_worker_loads_with_client( + worker_urls: Vec, + tx: tokio::sync::watch::Sender>, + interval_secs: u64, + client: Client, + prefill_policy: Arc, + decode_policy: Arc, + ) { + loop { + let mut loads = HashMap::new(); + + let futures: Vec<_> = worker_urls + .iter() + .map(|url| { + let client = client.clone(); + let url = url.clone(); + async move { + let load = get_worker_load(&client, &url).await.unwrap_or(0); + (url, load) + } + }) + .collect(); + + let results = futures_util::future::join_all(futures).await; + + for (url, load) in results { + loads.insert(url, load); + } + + debug!("Worker loads updated: {:?}", loads); + + // Update both policies with current loads + prefill_policy.update_loads(&loads); + decode_policy.update_loads(&loads); + + // Check if receiver is still active + if tx.send(loads).is_err() { + info!("Load monitor receiver dropped, shutting down monitor task"); + break; + } + + tokio::time::sleep(Duration::from_secs(interval_secs)).await; + } + } + + // Helper to create a streaming response + #[allow(clippy::too_many_arguments)] + fn create_streaming_response( + &self, + stream: impl futures_util::Stream> + Send + 'static, + status: StatusCode, + prefill_logprobs: Option, + return_logprob: bool, + decode_url: Option, + headers: Option, + prefill: &dyn Worker, + decode: &dyn Worker, + ) -> Response { + // For streaming, increment load now - will be decremented when streaming completes + prefill.increment_load(); + decode.increment_load(); + + // Store URLs to find workers later for decrementing + let prefill_url = prefill.url().to_string(); + let decode_url_str = decode.url().to_string(); + + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + // Clone the worker collections for the spawned task + let prefill_workers = self.prefill_workers.clone(); + let decode_workers = self.decode_workers.clone(); + + tokio::spawn(async move { + // Use a flag to track whether stream completed successfully + let mut stream_completed = false; + + futures_util::pin_mut!(stream); + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(chunk) => { + // Check for stream end marker to decrement load early + let is_done = chunk + .as_ref() + .windows(12) + .any(|window| window == b"data: [DONE]"); + + let result = if return_logprob && prefill_logprobs.is_some() { + // Try to merge logprobs + Self::merge_streaming_logprobs(prefill_logprobs.clone(), &chunk) + .unwrap_or(chunk) + } else { + chunk + }; + + if tx.send(Ok(result)).is_err() { + break; + } + + // If we see the done marker, decrement load immediately + if is_done { + stream_completed = true; + break; + } + } + Err(e) => { + if let Some(ref url) = decode_url { + error!("Stream error from decode server {}: {}", url, e); + RouterMetrics::record_pd_stream_error(url); + } + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + + // Always decrement load after streaming (either completes or errors) + // Find and decrement prefill worker + if let Ok(prefill_workers_guard) = prefill_workers.read() { + for worker in prefill_workers_guard.iter() { + if worker.url() == prefill_url.as_str() { + worker.decrement_load(); + debug!( + "Decremented load for prefill worker: {} (stream_completed: {})", + prefill_url, stream_completed + ); + break; + } + } + } + + // Find and decrement decode worker + if let Ok(decode_workers_guard) = decode_workers.read() { + for worker in decode_workers_guard.iter() { + if worker.url() == decode_url_str.as_str() { + worker.decrement_load(); + debug!( + "Decremented load for decode worker: {} (stream_completed: {})", + decode_url_str, stream_completed + ); + break; + } + } + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + + // Use provided headers or create new ones, then ensure content-type is set for streaming + let mut headers = headers.unwrap_or_default(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + *response.headers_mut() = headers; + + response + } + + // Helper to process non-streaming decode response with logprob merging + async fn process_non_streaming_response( + &self, + res: reqwest::Response, + status: StatusCode, + return_logprob: bool, + prefill_body: Option, + ) -> Response { + let response = res.bytes().await; + let decode_body = match response { + Ok(decode_body) => decode_body, + Err(e) => { + error!("Failed to read decode response: {}", e); + return (StatusCode::INTERNAL_SERVER_ERROR, "Failed to read response") + .into_response(); + } + }; + + if !return_logprob { + return (status, decode_body).into_response(); + } + + let Some(prefill_body) = prefill_body else { + return (status, decode_body).into_response(); + }; + + // Merge logprobs from prefill and decode + let (Ok(prefill_json), Ok(mut decode_json)) = ( + serde_json::from_slice::(&prefill_body), + serde_json::from_slice::(&decode_body), + ) else { + warn!("Failed to parse responses for logprob merging"); + return (status, decode_body).into_response(); + }; + + Self::merge_logprobs_in_json(&prefill_json, &mut decode_json); + + // Return merged response + match serde_json::to_vec(&decode_json) { + Ok(body) => (status, body).into_response(), + Err(e) => { + error!("Failed to serialize merged response: {}", e); + (status, decode_body).into_response() + } + } + } + + // Helper to process prefill response and extract body if needed for logprobs + async fn process_prefill_response( + &self, + prefill_result: Result, + prefill_url: &str, + return_logprob: bool, + ) -> Result<(StatusCode, Option), Response> { + // Check prefill result first - it's critical for disaggregated mode + let prefill_response = match prefill_result { + Ok(response) => response, + Err(e) => { + RouterMetrics::record_pd_prefill_error(prefill_url); + error!( + "Prefill server failed (CRITICAL) prefill_url={} error={}. Decode will timeout without prefill KV cache.", + prefill_url, + e + ); + + // Return error immediately - don't wait for decode to timeout + return Err(( + StatusCode::BAD_GATEWAY, + format!( + "Prefill server error: {}. This will cause decode timeout.", + e + ), + ) + .into_response()); + } + }; + + let prefill_status = StatusCode::from_u16(prefill_response.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + // Check if prefill succeeded + if !prefill_status.is_success() { + RouterMetrics::record_pd_prefill_error(prefill_url); + + // Get error body from prefill + let error_msg = prefill_response + .text() + .await + .unwrap_or_else(|_| "Unknown prefill error".to_string()); + + error!( + "Prefill server returned error status prefill_url={} status={} body={}", + prefill_url, prefill_status, error_msg + ); + + return Err(( + prefill_status, + format!("Prefill server error ({}): {}", prefill_status, error_msg), + ) + .into_response()); + } + + // Read prefill body if needed for logprob merging + let prefill_body = if return_logprob { + match prefill_response.bytes().await { + Ok(body) => Some(body), + Err(e) => { + warn!("Failed to read prefill response body for logprobs: {}", e); + None + } + } + } else { + // For non-logprob requests, just consume the response without storing + debug!("Consuming prefill response body (non-logprob request)"); + match prefill_response.bytes().await { + Ok(_) => debug!("Prefill response consumed successfully"), + Err(e) => warn!("Error consuming prefill response: {}", e), + } + None + }; + + Ok((prefill_status, prefill_body)) + } + + fn build_post_with_headers( + &self, + client: &Client, + url: &str, + route: &str, + json_request: &Value, + headers: Option<&HeaderMap>, + connection_close: bool, + ) -> reqwest::RequestBuilder { + let mut request = client.post(api_path(url, route)).json(json_request); + if connection_close { + request = request.header("Connection", "close"); + } + if let Some(headers) = headers { + for (name, value) in headers.iter() { + let name_lc = name.as_str().to_ascii_lowercase(); + // Whitelist important end-to-end headers, skip hop-by-hop + let forward = matches!( + name_lc.as_str(), + "authorization" | "x-request-id" | "x-correlation-id" + ) || name_lc.starts_with("x-request-id-"); + if forward { + if let Ok(val) = value.to_str() { + request = request.header(name, val); + } + } + } + } + request + } + + // Helper to merge logprobs from prefill and decode responses + fn merge_logprobs_in_json(prefill_json: &Value, decode_json: &mut Value) -> bool { + if let (Some(prefill_meta), Some(decode_meta)) = ( + prefill_json.get("meta_info"), + decode_json.get_mut("meta_info"), + ) { + if let (Some(prefill_logprobs), Some(decode_logprobs)) = ( + prefill_meta.get("input_token_logprobs"), + decode_meta.get_mut("input_token_logprobs"), + ) { + if let (Some(prefill_arr), Some(decode_arr)) = + (prefill_logprobs.as_array(), decode_logprobs.as_array_mut()) + { + let mut merged = prefill_arr.clone(); + merged.extend(decode_arr.clone()); + decode_meta["input_token_logprobs"] = Value::Array(merged); + return true; + } + } + } + false + } + + // Simple helper to merge logprobs in streaming responses + fn merge_streaming_logprobs( + prefill_logprobs: Option, + decode_chunk: &[u8], + ) -> Result { + // Skip non-data chunks + let chunk_str = std::str::from_utf8(decode_chunk).map_err(|_| ())?; + if !chunk_str.starts_with("data: ") || chunk_str.contains("[DONE]") { + return Err(()); + } + + // Parse JSON from chunk + let json_str = chunk_str.trim_start_matches("data: ").trim(); + let mut decode_json: Value = serde_json::from_str(json_str).map_err(|_| ())?; + + // Merge prefill logprobs if available + if let Some(ref p_logprobs) = prefill_logprobs { + if let Some(meta) = decode_json.get_mut("meta_info") { + if let Some(d_logprobs) = meta.get_mut("input_token_logprobs") { + if let (Some(p_arr), Some(d_arr)) = + (p_logprobs.as_array(), d_logprobs.as_array()) + { + let mut merged = p_arr.clone(); + merged.extend(d_arr.clone()); + *d_logprobs = Value::Array(merged); + } + } + } + } + + // Re-serialize + let merged_str = format!( + "data: {}\n\n", + serde_json::to_string(&decode_json).unwrap_or_default() + ); + Ok(bytes::Bytes::from(merged_str)) + } +} + +// Helper functions + +async fn get_worker_load(client: &Client, worker_url: &str) -> Option { + match client.get(format!("{}/get_load", worker_url)).send().await { + Ok(res) if res.status().is_success() => match res.bytes().await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(data) => data + .get("load") + .and_then(|v| v.as_i64()) + .map(|v| v as isize), + Err(e) => { + debug!("Failed to parse load response from {}: {}", worker_url, e); + None + } + }, + Err(e) => { + debug!("Failed to read load response from {}: {}", worker_url, e); + None + } + }, + Ok(res) => { + debug!( + "Worker {} returned non-success status: {}", + worker_url, + res.status() + ); + None + } + Err(e) => { + debug!("Failed to get load from {}: {}", worker_url, e); + None + } + } +} + +#[async_trait] +impl WorkerManagement for PDRouter { + async fn add_worker(&self, _worker_url: &str) -> Result { + // For PD router, we don't support adding workers via this generic method + Err( + "PD router requires specific add_prefill_server or add_decode_server methods" + .to_string(), + ) + } + + fn remove_worker(&self, worker_url: &str) { + // For PD router, we would need to know if it's a prefill or decode server + // For now, try both + if let Ok(mut workers) = self.prefill_workers.write() { + if let Some(index) = workers.iter().position(|w| w.url() == worker_url) { + workers.remove(index); + info!("Removed prefill worker: {}", worker_url); + return; + } + } + + if let Ok(mut workers) = self.decode_workers.write() { + if let Some(index) = workers.iter().position(|w| w.url() == worker_url) { + workers.remove(index); + info!("Removed decode worker: {}", worker_url); + } + } + } + + fn get_worker_urls(&self) -> Vec { + let mut urls = Vec::new(); + + // Add prefill worker URLs + if let Ok(workers) = self.prefill_workers.read() { + for worker in workers.iter() { + urls.push(worker.url().to_string()); + } + } + + // Add decode worker URLs + if let Ok(workers) = self.decode_workers.read() { + for worker in workers.iter() { + urls.push(worker.url().to_string()); + } + } + + urls + } +} + +#[async_trait] +impl RouterTrait for PDRouter { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health(&self, _req: Request) -> Response { + // This is a server readiness check - checking if we have healthy workers + // Workers handle their own health checks in the background + let mut all_healthy = true; + let mut unhealthy_servers = Vec::new(); + + // Check prefill servers + for worker in self.prefill_workers.read().unwrap().iter() { + if !worker.is_healthy() { + all_healthy = false; + unhealthy_servers.push(format!("Prefill: {}", worker.url())); + } + } + + // Check decode servers + for worker in self.decode_workers.read().unwrap().iter() { + if !worker.is_healthy() { + all_healthy = false; + unhealthy_servers.push(format!("Decode: {}", worker.url())); + } + } + + if all_healthy { + (StatusCode::OK, "All servers healthy").into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Unhealthy servers: {:?}", unhealthy_servers), + ) + .into_response() + } + } + + async fn health_generate(&self, _req: Request) -> Response { + // Test model generation capability by selecting a random pair and testing them + // Note: This endpoint actually causes the model to generate tokens, so we only test one pair + + // Select a random worker pair using the policy + let (prefill, decode) = match self.select_pd_pair(None).await { + Ok(pair) => pair, + Err(e) => { + return ( + StatusCode::SERVICE_UNAVAILABLE, + format!("No healthy worker pair available: {}", e), + ) + .into_response(); + } + }; + + // Test prefill server's health_generate + let prefill_url = format!("{}/health_generate", prefill.url()); + let (prefill_result, decode_result) = tokio::join!( + self.client.get(&prefill_url).send(), + self.client + .get(format!("{}/health_generate", decode.url())) + .send() + ); + + // Check results + let mut errors = Vec::new(); + + match prefill_result { + Ok(res) if res.status().is_success() => { + debug!( + "Health generate passed for prefill server: {}", + prefill.url() + ); + } + Ok(res) => { + errors.push(format!( + "Prefill {} returned status {}", + prefill.url(), + res.status() + )); + } + Err(e) => { + errors.push(format!("Prefill {} error: {}", prefill.url(), e)); + } + } + + match decode_result { + Ok(res) if res.status().is_success() => { + debug!("Health generate passed for decode server: {}", decode.url()); + } + Ok(res) => { + errors.push(format!( + "Decode {} returned status {}", + decode.url(), + res.status() + )); + } + Err(e) => { + errors.push(format!("Decode {} error: {}", decode.url(), e)); + } + } + + if errors.is_empty() { + ( + StatusCode::OK, + format!( + "Health generate passed on selected pair: prefill={}, decode={}", + prefill.url(), + decode.url() + ), + ) + .into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Health generate failed: {:?}", errors), + ) + .into_response() + } + } + + async fn get_server_info(&self, _req: Request) -> Response { + // Get info from the first decode server to match sglang's server info format + // Note: We use decode workers for server info to match expected format + self.proxy_to_first_worker(&self.decode_workers, "get_server_info", "decode", None) + .await + } + + async fn get_models(&self, req: Request) -> Response { + // Extract headers first to avoid Send issues + let headers = header_utils::copy_request_headers(&req); + + // Proxy to first prefill worker + self.proxy_to_first_worker(&self.prefill_workers, "v1/models", "prefill", Some(headers)) + .await + } + + async fn get_model_info(&self, req: Request) -> Response { + // Extract headers first to avoid Send issues + let headers = header_utils::copy_request_headers(&req); + + // Proxy to first prefill worker + self.proxy_to_first_worker( + &self.prefill_workers, + "get_model_info", + "prefill", + Some(headers), + ) + .await + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + ) -> Response { + // Extract parameters + let is_stream = body.stream; + let return_logprob = body.return_logprob; + + // Extract text for cache-aware routing + let request_text = if self.policies_need_request_text() { + body.text + .as_deref() + .or_else(|| { + body.prompt.as_ref().and_then(|p| match p { + StringOrArray::String(s) => Some(s.as_str()), + StringOrArray::Array(v) => v.first().map(|s| s.as_str()), + }) + }) + .map(|s| s.to_string()) + } else { + None + }; + + // Calculate batch size + let batch_size = Self::get_generate_batch_size(body); + + // Create context + let context = PDRequestContext { + route: "/generate", + batch_size, + is_stream, + return_logprob, + request_text, + }; + + // Execute with retry and bootstrap injection + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + ) -> Response { + // Extract parameters + let is_stream = body.stream; + let return_logprob = body.logprobs; + + // Extract text for cache-aware routing + let request_text = if self.policies_need_request_text() { + body.messages.first().and_then(|msg| match msg { + ChatMessage::User { content, .. } => match content { + UserMessageContent::Text(text) => Some(text.clone()), + UserMessageContent::Parts(_) => None, + }, + ChatMessage::System { content, .. } => Some(content.clone()), + _ => None, + }) + } else { + None + }; + + // Calculate batch size + let batch_size = Self::get_chat_batch_size(body); + + // Create context + let context = PDRequestContext { + route: "/v1/chat/completions", + batch_size, + is_stream, + return_logprob, + request_text, + }; + + // Execute with retry and bootstrap injection + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + ) -> Response { + // Extract parameters + let is_stream = body.stream; + let return_logprob = body.logprobs.is_some(); + + // Extract text for cache-aware routing + let request_text = if self.policies_need_request_text() { + match &body.prompt { + StringOrArray::String(s) => Some(s.clone()), + StringOrArray::Array(v) => v.first().map(|s| s.to_string()), + } + } else { + None + }; + + // Calculate batch size + let batch_size = Self::get_completion_batch_size(body); + + // Create context + let context = PDRequestContext { + route: "/v1/completions", + batch_size, + is_stream, + return_logprob, + request_text, + }; + + // Execute with retry and bootstrap injection + self.execute_dual_dispatch(headers, body, context).await + } + + async fn route_embeddings(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + todo!() + } + + async fn route_rerank(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + todo!() + } + + async fn flush_cache(&self) -> Response { + // Process both prefill and decode workers + let (prefill_results, prefill_errors) = self + .process_workers(&self.prefill_workers, "Prefill", "flush_cache") + .await; + let (decode_results, decode_errors) = self + .process_workers(&self.decode_workers, "Decode", "flush_cache") + .await; + + // Combine results and errors + let mut results = prefill_results; + results.extend(decode_results); + let mut errors = prefill_errors; + errors.extend(decode_errors); + + if errors.is_empty() { + ( + StatusCode::OK, + format!("Cache flushed successfully: {:?}", results), + ) + .into_response() + } else { + ( + StatusCode::PARTIAL_CONTENT, + format!( + "Partial success. Results: {:?}, Errors: {:?}", + results, errors + ), + ) + .into_response() + } + } + + async fn get_worker_loads(&self) -> Response { + let mut loads = HashMap::new(); + let mut errors = Vec::new(); + + // Process prefill workers + match Self::get_worker_urls(&self.prefill_workers, "prefill") { + Ok(urls) => { + for worker_url in urls { + match get_worker_load(&self.client, &worker_url).await { + Some(load) => { + loads.insert(format!("prefill_{}", worker_url), load); + } + None => { + errors.push(format!("Failed to get load from prefill {}", worker_url)); + } + } + } + } + Err(e) => errors.push(e), + } + + // Process decode workers + match Self::get_worker_urls(&self.decode_workers, "decode") { + Ok(urls) => { + for worker_url in urls { + match get_worker_load(&self.client, &worker_url).await { + Some(load) => { + loads.insert(format!("decode_{}", worker_url), load); + } + None => { + errors.push(format!("Failed to get load from decode {}", worker_url)); + } + } + } + } + Err(e) => errors.push(e), + } + + let response_data = serde_json::json!({ + "loads": loads, + "errors": errors + }); + + (StatusCode::OK, Json(response_data)).into_response() + } + + fn router_type(&self) -> &'static str { + "pd" + } + + fn readiness(&self) -> Response { + // PD router is ready if it has at least one healthy prefill AND one healthy decode worker + let healthy_prefill_count = self + .prefill_workers + .read() + .unwrap() + .iter() + .filter(|w| w.is_healthy()) + .count(); + + let healthy_decode_count = self + .decode_workers + .read() + .unwrap() + .iter() + .filter(|w| w.is_healthy()) + .count(); + + let total_prefill = self.prefill_workers.read().unwrap().len(); + let total_decode = self.decode_workers.read().unwrap().len(); + + if healthy_prefill_count > 0 && healthy_decode_count > 0 { + Json(json!({ + "status": "ready", + "prefill": { + "healthy": healthy_prefill_count, + "total": total_prefill + }, + "decode": { + "healthy": healthy_decode_count, + "total": total_decode + } + })) + .into_response() + } else { + let mut reasons = Vec::new(); + if healthy_prefill_count == 0 { + reasons.push("no healthy prefill workers"); + } + if healthy_decode_count == 0 { + reasons.push("no healthy decode workers"); + } + + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "status": "not_ready", + "reason": reasons.join(", "), + "prefill": { + "healthy": healthy_prefill_count, + "total": total_prefill + }, + "decode": { + "healthy": healthy_decode_count, + "total": total_decode + } + })), + ) + .into_response() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::{BasicWorker, WorkerType}; + use crate::policies::RandomPolicy; + + fn create_test_pd_router() -> PDRouter { + let prefill_policy = Arc::new(RandomPolicy::new()); + let decode_policy = Arc::new(RandomPolicy::new()); + + PDRouter { + prefill_workers: Arc::new(RwLock::new(vec![])), + decode_workers: Arc::new(RwLock::new(vec![])), + prefill_policy, + decode_policy, + worker_startup_timeout_secs: 5, + worker_startup_check_interval_secs: 1, + worker_loads: Arc::new(tokio::sync::watch::channel(HashMap::new()).1), + load_monitor_handle: None, + client: Client::new(), + prefill_client: Client::new(), + prefill_drain_tx: mpsc::channel(100).0, + retry_config: RetryConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + _prefill_health_checker: None, + _decode_health_checker: None, + } + } + + fn create_test_worker(url: String, worker_type: WorkerType, healthy: bool) -> Box { + let worker = BasicWorker::new(url, worker_type); + worker.set_healthy(healthy); + Box::new(worker) + } + + // ============= Worker Management Tests ============= + + #[tokio::test] + async fn test_add_prefill_server_already_exists() { + let router = create_test_pd_router(); + + // Add a worker first + let worker = create_test_worker( + "http://localhost:8000".to_string(), + WorkerType::Prefill { + bootstrap_port: Some(8080), + }, + true, + ); + router.prefill_workers.write().unwrap().push(worker); + + // Try to add the same URL again - this would fail during health check in real scenario + // For unit test, we test the duplicate check logic + let workers = router.prefill_workers.read().unwrap(); + let exists = workers.iter().any(|w| w.url() == "http://localhost:8000"); + assert!(exists); + } + + #[tokio::test] + async fn test_remove_prefill_server_success() { + let router = create_test_pd_router(); + + // Add servers first + let worker1 = create_test_worker( + "http://worker1".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let worker2 = create_test_worker( + "http://worker2".to_string(), + WorkerType::Prefill { + bootstrap_port: Some(8080), + }, + true, + ); + + router.prefill_workers.write().unwrap().push(worker1); + router.prefill_workers.write().unwrap().push(worker2); + + // Remove one + let result = router.remove_prefill_server("http://worker1").await; + + assert!(result.is_ok()); + assert!(result.unwrap().contains("Successfully removed")); + + let workers = router.prefill_workers.read().unwrap(); + assert_eq!(workers.len(), 1); + assert_eq!(workers[0].url(), "http://worker2"); + } + + #[tokio::test] + async fn test_remove_prefill_server_not_found() { + let router = create_test_pd_router(); + + let result = router.remove_prefill_server("http://nonexistent").await; + + assert!(result.is_err()); + match result.unwrap_err() { + PDRouterError::WorkerNotFound { url } => { + assert_eq!(url, "http://nonexistent"); + } + _ => panic!("Expected WorkerNotFound error"), + } + } + + #[tokio::test] + async fn test_remove_decode_server_success() { + let router = create_test_pd_router(); + + // Add server first + let worker = create_test_worker("http://decode1".to_string(), WorkerType::Decode, true); + router.decode_workers.write().unwrap().push(worker); + + let result = router.remove_decode_server("http://decode1").await; + + assert!(result.is_ok()); + assert!(result.unwrap().contains("Successfully removed")); + + let workers = router.decode_workers.read().unwrap(); + assert_eq!(workers.len(), 0); + } + + // ============= Lock Error Handling Tests ============= + + #[test] + fn test_lock_operations() { + let router = create_test_pd_router(); + + // Test read/write locks work correctly + { + let read_guard = router.prefill_workers.read().unwrap(); + assert_eq!(read_guard.len(), 0); + } + + { + let mut write_guard = router.prefill_workers.write().unwrap(); + write_guard.push(create_test_worker( + "http://test".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + )); + } + + { + let read_guard = router.prefill_workers.read().unwrap(); + assert_eq!(read_guard.len(), 1); + } + } + + // ============= Bootstrap Injection Tests ============= + // Note: These tests are commented out as we've moved to the optimized bootstrap injection + // approach that doesn't use the Bootstrap trait on GenerateReqInput anymore. + + // TODO: Add new tests for the optimized bootstrap injection approach using + // RequestWithBootstrap and BatchRequestWithBootstrap wrappers + + // ============= Worker Selection Tests ============= + + #[tokio::test] + async fn test_select_healthy_prefill_worker() { + let router = create_test_pd_router(); + + // Add mix of healthy and unhealthy workers + let healthy_worker = create_test_worker( + "http://healthy".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let unhealthy_worker = create_test_worker( + "http://unhealthy".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + false, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + router + .prefill_workers + .write() + .unwrap() + .push(unhealthy_worker); + router.prefill_workers.write().unwrap().push(healthy_worker); + router.decode_workers.write().unwrap().push(decode_worker); + + let result = router.select_pd_pair(None).await; + + assert!(result.is_ok()); + let (prefill, _decode) = result.unwrap(); + + // Should select the healthy worker + assert_eq!(prefill.url(), "http://healthy"); + assert!(prefill.is_healthy()); + } + + #[tokio::test] + async fn test_empty_worker_lists() { + let router = create_test_pd_router(); + + let result = router.select_pd_pair(None).await; + + assert!(result.is_err()); + assert!(result.unwrap_err().contains("No prefill workers available")); + } + + // ============= Health Endpoints Tests ============= + + #[tokio::test] + async fn test_health_endpoints() { + let router = create_test_pd_router(); + + // Add healthy workers + let prefill_worker = create_test_worker( + "http://localhost:8000".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let decode_worker = create_test_worker( + "http://localhost:8001".to_string(), + WorkerType::Decode, + true, + ); + + router.prefill_workers.write().unwrap().push(prefill_worker); + router.decode_workers.write().unwrap().push(decode_worker); + + // Test health endpoint + let http_req = axum::http::Request::builder() + .body(axum::body::Body::empty()) + .unwrap(); + let response = router.health(http_req).await; + + assert_eq!(response.status(), 200); + + // Test readiness endpoint + let response = router.readiness(); + assert_eq!(response.status(), 200); + } + + // ============= Load Monitoring Tests ============= + + #[tokio::test] + async fn test_load_monitor_updates() { + let power_of_two_policy = Arc::new(crate::policies::PowerOfTwoPolicy::new()); + let mut router = create_test_pd_router(); + router.prefill_policy = power_of_two_policy.clone(); + router.decode_policy = power_of_two_policy; + + // Create load channel + let (tx, rx) = tokio::sync::watch::channel(HashMap::new()); + router.worker_loads = Arc::new(rx); + + // Simulate load updates + let mut loads = HashMap::new(); + loads.insert("http://worker1".to_string(), 10); + loads.insert("http://worker2".to_string(), 5); + + let _ = tx.send(loads.clone()); + + // Router should receive updates + let received = router.worker_loads.borrow().clone(); + assert_eq!(received.get("http://worker1"), Some(&10)); + assert_eq!(received.get("http://worker2"), Some(&5)); + } + + // ============= Worker Load Tests ============= + + #[test] + fn test_worker_load_metrics() { + let prefill_worker = create_test_worker( + "http://prefill".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + // Create load guard for both workers + let _guard = + WorkerLoadGuard::new_multi(vec![prefill_worker.as_ref(), decode_worker.as_ref()]); + + // Load should be incremented + assert_eq!(prefill_worker.load(), 1); + assert_eq!(decode_worker.load(), 1); + + // Drop guard - load should decrement + drop(_guard); + + assert_eq!(prefill_worker.load(), 0); + assert_eq!(decode_worker.load(), 0); + } + + #[tokio::test] + async fn test_streaming_load_tracking() { + use futures_util::StreamExt; + use tokio::time::{sleep, Duration}; + + let router = create_test_pd_router(); + + // Add workers + let prefill_worker = create_test_worker( + "http://prefill".to_string(), + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + let decode_worker = + create_test_worker("http://decode".to_string(), WorkerType::Decode, true); + + router.prefill_workers.write().unwrap().push(prefill_worker); + router.decode_workers.write().unwrap().push(decode_worker); + + // Get references to the workers - clone to avoid holding lock across await + let (prefill_ref, decode_ref) = { + let workers = router.prefill_workers.read().unwrap(); + let prefill = workers[0].clone_worker(); + drop(workers); + let workers = router.decode_workers.read().unwrap(); + let decode = workers[0].clone_worker(); + (prefill, decode) + }; + + // Initially load should be 0 + assert_eq!(prefill_ref.load(), 0); + assert_eq!(decode_ref.load(), 0); + + // Create a mock streaming response + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx); + + // Call create_streaming_response which should increment load + let _response = router.create_streaming_response( + stream.map(Ok), + StatusCode::OK, + None, + false, + None, + None, + prefill_ref.as_ref(), + decode_ref.as_ref(), + ); + + // Load should be incremented immediately + assert_eq!(prefill_ref.load(), 1); + assert_eq!(decode_ref.load(), 1); + + // Send some data through the stream + tx.send(bytes::Bytes::from("test data")).unwrap(); + + // Give time for the spawned task to process + sleep(Duration::from_millis(10)).await; + + // Load should still be 1 (streaming in progress) + assert_eq!(prefill_ref.load(), 1); + assert_eq!(decode_ref.load(), 1); + + // Close the stream + drop(tx); + + // Give time for cleanup + sleep(Duration::from_millis(100)).await; + + // Load should be decremented after streaming completes + assert_eq!(prefill_ref.load(), 0); + assert_eq!(decode_ref.load(), 0); + } + + // ============= Concurrent Operations Tests ============= + + #[tokio::test] + async fn test_concurrent_worker_operations() { + let router = Arc::new(create_test_pd_router()); + + let mut handles = vec![]; + + // Spawn tasks to add workers + for i in 0..5 { + let router_clone = Arc::clone(&router); + let url = format!("http://worker{}", i); + let handle = tokio::spawn(async move { + let worker = create_test_worker( + url, + WorkerType::Prefill { + bootstrap_port: None, + }, + true, + ); + router_clone.prefill_workers.write().unwrap().push(worker); + }); + handles.push(handle); + } + + // Wait for all tasks + for handle in handles { + let _ = handle.await; + } + + // Check final state + let workers = router.prefill_workers.read().unwrap(); + assert_eq!(workers.len(), 5); + } +} diff --git a/sgl-router/src/routers/http/pd_types.rs b/sgl-router/src/routers/http/pd_types.rs new file mode 100644 index 000000000..a2b28a57d --- /dev/null +++ b/sgl-router/src/routers/http/pd_types.rs @@ -0,0 +1,81 @@ +// Custom error type for PD router operations +#[derive(Debug, thiserror::Error)] +pub enum PDRouterError { + #[error("Worker already exists: {url}")] + WorkerAlreadyExists { url: String }, + + #[error("Worker not found: {url}")] + WorkerNotFound { url: String }, + + #[error("Lock acquisition failed: {operation}")] + LockError { operation: String }, + + #[error("Health check failed for worker: {url}")] + HealthCheckFailed { url: String }, + + #[error("Invalid worker configuration: {reason}")] + InvalidConfiguration { reason: String }, + + #[error("Network error: {message}")] + NetworkError { message: String }, + + #[error("Timeout waiting for worker: {url}")] + Timeout { url: String }, +} + +// Helper functions for workers +pub fn api_path(url: &str, api_path: &str) -> String { + if api_path.starts_with("/") { + format!("{}{}", url, api_path) + } else { + format!("{}/{}", url, api_path) + } +} + +pub fn get_hostname(url: &str) -> String { + // Simple hostname extraction without external dependencies + let url = url + .trim_start_matches("http://") + .trim_start_matches("https://"); + url.split(':').next().unwrap_or("localhost").to_string() +} + +use serde::Serialize; + +// Optimized bootstrap wrapper for single requests +#[derive(Serialize)] +pub struct RequestWithBootstrap<'a, T: Serialize> { + #[serde(flatten)] + pub original: &'a T, + pub bootstrap_host: String, + pub bootstrap_port: Option, + pub bootstrap_room: u64, +} + +// Optimized bootstrap wrapper for batch requests +#[derive(Serialize)] +pub struct BatchRequestWithBootstrap<'a, T: Serialize> { + #[serde(flatten)] + pub original: &'a T, + pub bootstrap_host: Vec, + pub bootstrap_port: Vec>, + pub bootstrap_room: Vec, +} + +// Helper to generate bootstrap room ID +pub fn generate_room_id() -> u64 { + // Generate a value in the range [0, 2^63 - 1] to match Python's random.randint(0, 2**63 - 1) + rand::random::() & (i64::MAX as u64) +} + +// PD-specific routing policies +#[derive(Debug, Clone, PartialEq)] +pub enum PDSelectionPolicy { + Random, + PowerOfTwo, + CacheAware { + cache_threshold: f32, + balance_abs_threshold: usize, + balance_rel_threshold: f32, + }, +} diff --git a/sgl-router/src/routers/http/router.rs b/sgl-router/src/routers/http/router.rs new file mode 100644 index 000000000..176c38602 --- /dev/null +++ b/sgl-router/src/routers/http/router.rs @@ -0,0 +1,1387 @@ +use crate::config::types::RetryConfig; +use crate::core::{ + is_retryable_status, BasicWorker, CircuitBreakerConfig, HealthChecker, HealthConfig, + RetryExecutor, Worker, WorkerFactory, WorkerType, +}; +use crate::metrics::RouterMetrics; +use crate::policies::LoadBalancingPolicy; +use crate::protocols::spec::{ + ChatCompletionRequest, CompletionRequest, GenerateRequest, GenerationRequest, +}; +use crate::routers::header_utils; +use crate::routers::{RouterTrait, WorkerManagement}; +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_LENGTH, header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode}, + response::{IntoResponse, Response}, + Json, +}; +use futures_util::StreamExt; +use reqwest::Client; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tracing::{debug, error, info, warn}; + +/// Regular router that uses injected load balancing policies +#[derive(Debug)] +pub struct Router { + workers: Arc>>>, + policy: Arc, + client: Client, + worker_startup_timeout_secs: u64, + worker_startup_check_interval_secs: u64, + dp_aware: bool, + api_key: Option, + retry_config: RetryConfig, + circuit_breaker_config: CircuitBreakerConfig, + _worker_loads: Arc>>, + _load_monitor_handle: Option>>, + _health_checker: Option, +} + +impl Router { + /// Create a new router with injected policy and client + #[allow(clippy::too_many_arguments)] + pub async fn new( + worker_urls: Vec, + policy: Arc, + ctx: &Arc, + ) -> Result { + // Update active workers gauge + RouterMetrics::set_active_workers(worker_urls.len()); + + // Wait for workers to be healthy (skip if empty - for service discovery mode) + if !worker_urls.is_empty() { + Self::wait_for_healthy_workers( + &worker_urls, + ctx.router_config.worker_startup_timeout_secs, + ctx.router_config.worker_startup_check_interval_secs, + ) + .await?; + } + + let worker_urls = if ctx.router_config.dp_aware { + // worker address now in the format of "http://host:port@dp_rank" + Self::get_dp_aware_workers(&worker_urls, &ctx.router_config.api_key) + .map_err(|e| format!("Failed to get dp-aware workers: {}", e))? + } else { + worker_urls + }; + + // Convert config CircuitBreakerConfig to core CircuitBreakerConfig + let circuit_breaker_config = ctx.router_config.effective_circuit_breaker_config(); + let core_cb_config = CircuitBreakerConfig { + failure_threshold: circuit_breaker_config.failure_threshold, + success_threshold: circuit_breaker_config.success_threshold, + timeout_duration: Duration::from_secs(circuit_breaker_config.timeout_duration_secs), + window_duration: Duration::from_secs(circuit_breaker_config.window_duration_secs), + }; + + // Create Worker trait objects from URLs with health check config + let workers: Vec> = worker_urls + .iter() + .map(|url| { + let worker = BasicWorker::new(url.clone(), WorkerType::Regular) + .with_circuit_breaker_config(core_cb_config.clone()) + .with_health_config(HealthConfig { + timeout_secs: ctx.router_config.health_check.timeout_secs, + check_interval_secs: ctx.router_config.health_check.check_interval_secs, + endpoint: ctx.router_config.health_check.endpoint.clone(), + failure_threshold: ctx.router_config.health_check.failure_threshold, + success_threshold: ctx.router_config.health_check.success_threshold, + }); + Box::new(worker) as Box + }) + .collect(); + + // Initialize policy with workers if needed (e.g., for cache-aware) + if let Some(cache_aware) = policy + .as_any() + .downcast_ref::() + { + cache_aware.init_workers(&workers); + } + + let workers = Arc::new(RwLock::new(workers)); + let health_checker = crate::core::start_health_checker( + Arc::clone(&workers), + ctx.router_config.worker_startup_check_interval_secs, + ); + + // Setup load monitoring for PowerOfTwo policy + let (tx, rx) = tokio::sync::watch::channel(HashMap::new()); + let worker_loads = Arc::new(rx); + + let load_monitor_handle = if policy.name() == "power_of_two" { + let monitor_urls = worker_urls.clone(); + let monitor_interval = ctx.router_config.worker_startup_check_interval_secs; + let policy_clone = Arc::clone(&policy); + let client_clone = ctx.client.clone(); + + Some(Arc::new(tokio::spawn(async move { + Self::monitor_worker_loads( + monitor_urls, + tx, + monitor_interval, + policy_clone, + client_clone, + ) + .await; + }))) + } else { + None + }; + + Ok(Router { + workers, + policy, + client: ctx.client.clone(), + worker_startup_timeout_secs: ctx.router_config.worker_startup_timeout_secs, + worker_startup_check_interval_secs: ctx + .router_config + .worker_startup_check_interval_secs, + dp_aware: ctx.router_config.dp_aware, + api_key: ctx.router_config.api_key.clone(), + retry_config: ctx.router_config.effective_retry_config(), + circuit_breaker_config: core_cb_config, + _worker_loads: worker_loads, + _load_monitor_handle: load_monitor_handle, + _health_checker: Some(health_checker), + }) + } + + /// Get the current list of worker URLs + pub fn get_worker_urls(&self) -> Vec { + self.workers + .read() + .unwrap() + .iter() + .map(|w| w.url().to_string()) + .collect() + } + + pub async fn wait_for_healthy_workers( + worker_urls: &[String], + worker_startup_timeout_secs: u64, + worker_startup_check_interval_secs: u64, + ) -> Result<(), String> { + if worker_urls.is_empty() { + return Err( + "Timeout waiting for workers to become healthy: no workers provided".to_string(), + ); + } + + // Perform health check asynchronously + Self::wait_for_healthy_workers_async( + worker_urls, + worker_startup_timeout_secs, + worker_startup_check_interval_secs, + ) + .await + } + + async fn wait_for_healthy_workers_async( + worker_urls: &[String], + worker_startup_timeout_secs: u64, + worker_startup_check_interval_secs: u64, + ) -> Result<(), String> { + info!( + "Waiting for {} workers to become healthy (timeout: {}s)", + worker_urls.len(), + worker_startup_timeout_secs + ); + + let start_time = std::time::Instant::now(); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .map_err(|e| format!("Failed to create HTTP client: {}", e))?; + + loop { + if start_time.elapsed() > Duration::from_secs(worker_startup_timeout_secs) { + error!( + "Timeout {}s waiting for workers {:?} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + worker_startup_timeout_secs, worker_urls + ); + return Err(format!( + "Timeout {}s waiting for workers {:?} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + worker_startup_timeout_secs, worker_urls + )); + } + + // Perform all health checks concurrently + let mut health_checks = Vec::new(); + for url in worker_urls { + let client_clone = client.clone(); + let url_clone = url.clone(); + + let check_health = tokio::spawn(async move { + let health_url = format!("{}/health", url_clone); + match client_clone.get(&health_url).send().await { + Ok(res) => { + if res.status().is_success() { + None + } else { + Some((url_clone, format!("status: {}", res.status()))) + } + } + Err(_) => Some((url_clone, "not ready".to_string())), + } + }); + + health_checks.push(check_health); + } + + // Wait for all health checks to complete + let results = futures::future::join_all(health_checks).await; + + let mut all_healthy = true; + let mut unhealthy_workers = Vec::new(); + + for result in results { + match result { + Ok(None) => { + // Worker is healthy + } + Ok(Some((url, reason))) => { + all_healthy = false; + unhealthy_workers.push((url, reason)); + } + Err(e) => { + all_healthy = false; + unhealthy_workers + .push(("unknown".to_string(), format!("task error: {}", e))); + } + } + } + + if all_healthy { + info!("All {} workers are healthy", worker_urls.len()); + return Ok(()); + } else { + debug!( + "Waiting for {} workers to become healthy ({} unhealthy: {:?})", + worker_urls.len(), + unhealthy_workers.len(), + unhealthy_workers + ); + tokio::time::sleep(Duration::from_secs(worker_startup_check_interval_secs)).await; + } + } + } + + fn get_worker_dp_size(worker_url: &str, api_key: &Option) -> Result { + let sync_client = reqwest::blocking::Client::new(); + let mut req_builder = sync_client.get(format!("{}/get_server_info", worker_url)); + if let Some(key) = api_key { + req_builder = req_builder.bearer_auth(key); + } + + match req_builder.send() { + Ok(res) => { + if res.status().is_success() { + let server_info = res + .text() + .map_err(|e| format!("failed to read text from response: {}", e))?; + + let server_info: serde_json::Value = serde_json::from_str(&server_info) + .map_err(|e| format!("failed to decode JSON: {}", e))?; + + let dp_size = server_info + .get("dp_size") + .and_then(|v| v.as_u64()) + .ok_or_else(|| String::from("dp_size not found or not an u64"))?; + + Ok(if dp_size > usize::MAX as u64 { + return Err(format!("dp_size is too large: {}", dp_size)); + } else { + dp_size as usize + }) + } else { + Err(format!("unexpected status code: {}", res.status())) + } + } + Err(e) => Err(format!("error response: {}", e)), + } + } + + // Given a list of workers, return a list of workers with dp_rank as suffix + fn get_dp_aware_workers( + worker_urls: &[String], + api_key: &Option, + ) -> Result, String> { + let mut dp_aware_workers: Vec = Vec::new(); + + for url in worker_urls { + match Self::get_worker_dp_size(url, api_key) { + Ok(dp_size) => { + for i in 0..dp_size { + dp_aware_workers.push(format!("{}@{}", url, i)); + } + } + Err(e) => return Err(format!("Failed to get DP size for {}: {}", url, e)), + } + } + + Ok(dp_aware_workers) + } + + fn select_first_worker(&self) -> Result { + let workers_guard = self.workers.read().unwrap(); + if workers_guard.is_empty() { + Err("No workers are available".to_string()) + } else { + Ok(workers_guard[0].url().to_string()) + } + } + + pub async fn send_health_check(&self, worker_url: &str) -> Response { + let health_url = if self.dp_aware { + // Need to extract the URL from "http://host:port@dp_rank" + match Self::extract_dp_rank(worker_url) { + Ok((worker_url_prefix, _dp_rank)) => worker_url_prefix, + Err(e) => { + error!("Failed to extract dp_rank for health check: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to extract dp_rank: {}", e), + ) + .into_response(); + } + } + } else { + worker_url + }; + + let request_builder = self.client.get(format!("{}/health", health_url)); + + let response = match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + match res.bytes().await { + Ok(body) => (status, body).into_response(), + Err(e) => { + error!( + worker_url = %health_url, + error = %e, + "Failed to read health response body" + ); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response body: {}", e), + ) + .into_response() + } + } + } + Err(e) => { + error!( + worker_url = %health_url, + error = %e, + "Failed to send health request to worker" + ); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to send request to worker {}: {}", health_url, e), + ) + .into_response() + } + }; + + // Don't record metrics for health checks + response + } + + // Helper method to proxy GET requests to the first available worker + async fn proxy_get_request(&self, req: Request, endpoint: &str) -> Response { + let headers = header_utils::copy_request_headers(&req); + + match self.select_first_worker() { + Ok(worker_url) => { + let mut request_builder = self.client.get(format!("{}/{}", worker_url, endpoint)); + for (name, value) in headers { + let name_lc = name.to_lowercase(); + if name_lc != "content-type" && name_lc != "content-length" { + request_builder = request_builder.header(name, value); + } + } + + match request_builder.send().await { + Ok(res) => { + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + // Preserve headers from backend + let response_headers = + header_utils::preserve_response_headers(res.headers()); + + match res.bytes().await { + Ok(body) => { + let mut response = Response::new(axum::body::Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read response: {}", e), + ) + .into_response(), + } + } + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Request failed: {}", e), + ) + .into_response(), + } + } + Err(e) => (StatusCode::SERVICE_UNAVAILABLE, e).into_response(), + } + } + + // New method to route typed requests directly + /// Select worker considering circuit breaker state + fn select_worker_with_circuit_breaker(&self, text: Option<&str>) -> Option> { + let workers = self.workers.read().ok()?; + let available: Vec> = workers + .iter() + .filter(|w| w.is_available()) + .map(|w| w.clone_worker()) + .collect(); + if available.is_empty() { + return None; + } + let idx = self.policy.select_worker(&available, text)?; + Some(available[idx].clone_worker()) + } + + pub async fn route_typed_request( + &self, + headers: Option<&HeaderMap>, + typed_req: &T, + route: &str, + ) -> Response { + let start = Instant::now(); + let is_stream = typed_req.is_stream(); + let text = typed_req.extract_text_for_routing(); + + let response = RetryExecutor::execute_response_with_retry( + &self.retry_config, + // operation per attempt + |_: u32| async { + let worker = match self.select_worker_with_circuit_breaker(Some(&text)) { + Some(w) => w, + None => { + RouterMetrics::record_request_error(route, "no_available_workers"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "No available workers (all circuits open or unhealthy)", + ) + .into_response(); + } + }; + + // Optional load tracking for cache-aware policy + let load_incremented = if self.policy.name() == "cache_aware" { + worker.increment_load(); + RouterMetrics::set_running_requests(worker.url(), worker.load()); + true + } else { + false + }; + + // Keep a clone for potential cleanup on retry + let worker_for_cleanup = if load_incremented { + Some(worker.clone_worker()) + } else { + None + }; + + let response = self + .send_typed_request( + headers, + typed_req, + route, + worker.url(), + is_stream, + load_incremented, + ) + .await; + + worker.record_outcome(response.status().is_success()); + + // For retryable failures, we need to decrement load since send_typed_request + // won't have done it (it only decrements on success or non-retryable failures) + if is_retryable_status(response.status()) && load_incremented { + if let Some(cleanup_worker) = worker_for_cleanup { + cleanup_worker.decrement_load(); + RouterMetrics::set_running_requests( + cleanup_worker.url(), + cleanup_worker.load(), + ); + } + } + + response + }, + // should_retry predicate + |res, _attempt| is_retryable_status(res.status()), + // on_backoff hook + |delay, attempt| { + RouterMetrics::record_retry(route); + RouterMetrics::record_retry_backoff_duration(delay, attempt); + }, + // on_exhausted hook + || RouterMetrics::record_retries_exhausted(route), + ) + .await; + + if response.status().is_success() { + let duration = start.elapsed(); + RouterMetrics::record_request(route); + RouterMetrics::record_generate_duration(duration); + } else if !is_retryable_status(response.status()) { + RouterMetrics::record_request_error(route, "non_retryable_error"); + } + + response + } + + // TODO (rui): Better accommodate to the Worker abstraction + fn extract_dp_rank(worker_url: &str) -> Result<(&str, usize), String> { + let parts: Vec<&str> = worker_url.split('@').collect(); + if parts.len() != 2 { + return Err(format!("invalid worker_url format: {}", worker_url)); + } + + // Parse the second part (dp_rank) into an integer + match parts[1].parse::() { + Ok(dp_rank) => Ok((parts[0], dp_rank)), + Err(_) => Err(format!( + "failed to parse dp_rank from worker_url: {}", + worker_url + )), + } + } + + // Send typed request directly without conversion + async fn send_typed_request( + &self, + headers: Option<&HeaderMap>, + typed_req: &T, + route: &str, + worker_url: &str, + is_stream: bool, + load_incremented: bool, // Whether load was incremented for this request + ) -> Response { + let mut request_builder = if self.dp_aware { + let (worker_url_prefix, dp_rank) = match Self::extract_dp_rank(worker_url) { + Ok(tup) => tup, + Err(e) => { + error!("Failed to extract dp_rank: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to extract dp_rank: {}", e), + ) + .into_response(); + } + }; + + // Parse the request body + let mut json_val = match serde_json::to_value(typed_req) { + Ok(j) => j, + Err(e) => { + return ( + StatusCode::BAD_REQUEST, + format!("Convert into serde_json::Value failed: {}", e), + ) + .into_response(); + } + }; + + // Insert the data_parallel_rank field + if let Some(map) = json_val.as_object_mut() { + map.insert( + String::from("data_parallel_rank"), + serde_json::json!(dp_rank), + ); + debug!( + "Modified request body: {}", + serde_json::to_string(&json_val).unwrap_or(String::from("ERR")) + ); + } else { + return ( + StatusCode::BAD_REQUEST, + "Failed to insert the data_parallel_rank field into the request body", + ) + .into_response(); + } + + self.client + .post(format!("{}{}", worker_url_prefix, route)) + .json(&json_val) + } else { + self.client + .post(format!("{}{}", worker_url, route)) + .json(typed_req) // Use json() directly with typed request + }; + + // Copy all headers from original request if provided + if let Some(headers) = headers { + for (name, value) in headers { + // Skip Content-Type and Content-Length as .json() sets them + if *name != CONTENT_TYPE && *name != CONTENT_LENGTH { + request_builder = request_builder.header(name, value); + } + } + } + + let res = match request_builder.send().await { + Ok(res) => res, + Err(e) => { + error!( + "Failed to send typed request worker_url={} route={} error={}", + worker_url, route, e + ); + + // Decrement load on error if it was incremented + if load_incremented { + if let Ok(workers_guard) = self.workers.read() { + if let Some(worker) = workers_guard.iter().find(|w| w.url() == worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + } + + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Request failed: {}", e), + ) + .into_response(); + } + }; + + let status = StatusCode::from_u16(res.status().as_u16()) + .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + + if !is_stream { + // For non-streaming requests, preserve headers + let response_headers = header_utils::preserve_response_headers(res.headers()); + + let response = match res.bytes().await { + Ok(body) => { + let mut response = Response::new(axum::body::Body::from(body)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + Err(e) => { + // IMPORTANT: Decrement load on error before returning + if load_incremented { + if let Ok(workers_guard) = self.workers.read() { + if let Some(worker) = + workers_guard.iter().find(|w| w.url() == worker_url) + { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + } + + let error_msg = format!("Failed to get response body: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, error_msg).into_response() + } + }; + + // Decrement load counter for non-streaming requests if it was incremented + if load_incremented { + if let Ok(workers_guard) = self.workers.read() { + if let Some(worker) = workers_guard.iter().find(|w| w.url() == worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(worker_url, worker.load()); + } + } + } + + response + } else if load_incremented { + // For streaming with load tracking, we need to manually decrement when done + let workers = Arc::clone(&self.workers); + let worker_url = worker_url.to_string(); + + // Preserve headers for streaming response + let mut response_headers = header_utils::preserve_response_headers(res.headers()); + // Ensure we set the correct content-type for SSE + response_headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + + let stream = res.bytes_stream(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + // Spawn task to forward stream and detect completion + tokio::spawn(async move { + let mut stream = stream; + let mut decremented = false; + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + // Check for stream end marker + if bytes + .as_ref() + .windows(12) + .any(|window| window == b"data: [DONE]") + { + if let Ok(workers_guard) = workers.read() { + if let Some(worker) = + workers_guard.iter().find(|w| w.url() == worker_url) + { + worker.decrement_load(); + RouterMetrics::set_running_requests( + &worker_url, + worker.load(), + ); + decremented = true; + } + } + } + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + if !decremented { + if let Ok(workers_guard) = workers.read() { + if let Some(worker) = workers_guard.iter().find(|w| w.url() == worker_url) { + worker.decrement_load(); + RouterMetrics::set_running_requests(&worker_url, worker.load()); + } + } + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } else { + // For requests without load tracking, just stream + // Preserve headers for streaming response + let mut response_headers = header_utils::preserve_response_headers(res.headers()); + // Ensure we set the correct content-type for SSE + response_headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream")); + + let stream = res.bytes_stream(); + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + // Spawn task to forward stream + tokio::spawn(async move { + let mut stream = stream; + while let Some(chunk) = stream.next().await { + match chunk { + Ok(bytes) => { + if tx.send(Ok(bytes)).is_err() { + break; + } + } + Err(e) => { + let _ = tx.send(Err(format!("Stream error: {}", e))); + break; + } + } + } + }); + + let stream = UnboundedReceiverStream::new(rx); + let body = Body::from_stream(stream); + + let mut response = Response::new(body); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response + } + } + + pub async fn add_worker(&self, worker_url: &str) -> Result { + let start_time = std::time::Instant::now(); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(self.worker_startup_timeout_secs)) + .build() + .map_err(|e| format!("Failed to create HTTP client: {}", e))?; + + loop { + if start_time.elapsed() > Duration::from_secs(self.worker_startup_timeout_secs) { + error!( + "Timeout {}s waiting for worker {} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + self.worker_startup_timeout_secs, worker_url + ); + return Err(format!( + "Timeout {}s waiting for worker {} to become healthy. Please set --router-worker-startup-timeout-secs (sglang_router.launch_server) or --worker-startup-timeout-secs (sglang_worker.router) to a larger value", + self.worker_startup_timeout_secs, worker_url + )); + } + + match client.get(format!("{}/health", worker_url)).send().await { + Ok(res) => { + if res.status().is_success() { + let mut workers_guard = self.workers.write().unwrap(); + if self.dp_aware { + // Need to contact the worker to extract the dp_size, + // and add them as multiple workers + let url_vec = vec![String::from(worker_url)]; + let dp_url_vec = Self::get_dp_aware_workers(&url_vec, &self.api_key) + .map_err(|e| format!("Failed to get dp-aware workers: {}", e))?; + let mut worker_added: bool = false; + for dp_url in &dp_url_vec { + if workers_guard.iter().any(|w| w.url() == dp_url) { + warn!("Worker {} already exists", dp_url); + continue; + } + info!("Added worker: {}", dp_url); + let new_worker = WorkerFactory::create_regular_with_config( + dp_url.to_string(), + self.circuit_breaker_config.clone(), + ); + workers_guard.push(new_worker); + worker_added = true; + } + if !worker_added { + return Err(format!("No worker added for {}", worker_url)); + } + } else { + if workers_guard.iter().any(|w| w.url() == worker_url) { + return Err(format!("Worker {} already exists", worker_url)); + } + info!("Added worker: {}", worker_url); + let new_worker = WorkerFactory::create_regular_with_config( + worker_url.to_string(), + self.circuit_breaker_config.clone(), + ); + workers_guard.push(new_worker); + } + + RouterMetrics::set_active_workers(workers_guard.len()); + + // If cache aware policy, initialize the worker in the tree + if let Some(cache_aware) = + self.policy + .as_any() + .downcast_ref::() + { + // Get updated workers after adding + drop(workers_guard); + let workers_guard = self.workers.read().unwrap(); + cache_aware.init_workers(&workers_guard); + } + + return Ok(format!("Successfully added worker: {}", worker_url)); + } else { + debug!( + "Worker {} health check pending - status: {}", + worker_url, + res.status() + ); + // if the url does not have http or https prefix, warn users + if !worker_url.starts_with("http://") && !worker_url.starts_with("https://") + { + warn!("The worker url {} does not have http or https prefix. Please add the prefix to the url.", worker_url); + } + + tokio::time::sleep(Duration::from_secs( + self.worker_startup_check_interval_secs, + )) + .await; + continue; + } + } + Err(e) => { + debug!("Worker {} health check pending - error: {}", worker_url, e); + + // if the url does not have http or https prefix, warn users + if !worker_url.starts_with("http://") && !worker_url.starts_with("https://") { + warn!("The worker url {} does not have http or https prefix. Please add the prefix to the url.", worker_url); + } + + tokio::time::sleep(Duration::from_secs( + self.worker_startup_check_interval_secs, + )) + .await; + continue; + } + } + } + } + + pub fn remove_worker(&self, worker_url: &str) { + if self.dp_aware { + // remove dp-aware workers in a prefix-matching fashion + // without contacting the remote worker + let mut candidate_workers: Vec = Vec::new(); + let mut removed_workers: Vec = Vec::new(); + let worker_url_prefix = format!("{}@", worker_url); + + { + // find the candidate workers to be removed + let workers_guard = self.workers.read().unwrap(); + for w in workers_guard.iter() { + if w.url().starts_with(&worker_url_prefix) { + candidate_workers.push(w.url().to_string()); + } + } + } + + { + // do the removing on the worker_urls + let mut workers_guard = self.workers.write().unwrap(); + for dp_url in candidate_workers.iter() { + if let Some(index) = workers_guard.iter().position(|w| w.url() == dp_url) { + workers_guard.remove(index); + info!("Removed worker: {}", dp_url); + removed_workers.push(dp_url.to_string()); + } else { + warn!("Worker {} not found, skipping removal", dp_url); + continue; + } + } + RouterMetrics::set_active_workers(workers_guard.len()); + } + + // If cache aware policy, remove the workers from the tree + if let Some(cache_aware) = self + .policy + .as_any() + .downcast_ref::() + { + for dp_url in removed_workers.iter() { + cache_aware.remove_worker(dp_url); + info!("Removed worker from tree: {}", dp_url); + } + } + } else { + let mut workers_guard = self.workers.write().unwrap(); + if let Some(index) = workers_guard.iter().position(|w| w.url() == worker_url) { + workers_guard.remove(index); + info!("Removed worker: {}", worker_url); + RouterMetrics::set_active_workers(workers_guard.len()); + } else { + warn!("Worker {} not found, skipping removal", worker_url); + return; + } + + // If cache aware policy, remove the workers from the tree + if let Some(cache_aware) = self + .policy + .as_any() + .downcast_ref::() + { + cache_aware.remove_worker(worker_url); + info!("Removed worker from tree: {}", worker_url); + } + } + } + + async fn get_worker_load(&self, worker_url: &str) -> Option { + let worker_url = if self.dp_aware { + // Need to extract the URL from "http://host:port@dp_rank" + let (worker_url_prefix, _dp_rank) = match Self::extract_dp_rank(worker_url) { + Ok(tup) => tup, + Err(e) => { + error!("Failed to extract dp_rank: {}", e); + return None; + } + }; + worker_url_prefix + } else { + worker_url + }; + + match self + .client + .get(format!("{}/get_load", worker_url)) + .send() + .await + { + Ok(res) if res.status().is_success() => match res.bytes().await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(data) => data + .get("load") + .and_then(|v| v.as_i64()) + .map(|v| v as isize), + Err(e) => { + debug!("Failed to parse load response from {}: {}", worker_url, e); + None + } + }, + Err(e) => { + debug!("Failed to read load response from {}: {}", worker_url, e); + None + } + }, + Ok(res) => { + debug!( + "Worker {} returned non-success status: {}", + worker_url, + res.status() + ); + None + } + Err(e) => { + debug!("Failed to get load from {}: {}", worker_url, e); + None + } + } + } + + // Background task to monitor worker loads + async fn monitor_worker_loads( + worker_urls: Vec, + tx: tokio::sync::watch::Sender>, + interval_secs: u64, + policy: Arc, + client: Client, + ) { + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + + loop { + interval.tick().await; + + let mut loads = HashMap::new(); + for url in &worker_urls { + if let Some(load) = Self::get_worker_load_static(&client, url).await { + loads.insert(url.clone(), load); + } + } + + if !loads.is_empty() { + // Update policy with new loads + policy.update_loads(&loads); + + // Send to watchers + if let Err(e) = tx.send(loads) { + error!("Failed to send load update: {}", e); + } + } + } + } + + // Static version of get_worker_load for use in monitoring task + async fn get_worker_load_static(client: &reqwest::Client, worker_url: &str) -> Option { + let worker_url = if worker_url.contains("@") { + // Need to extract the URL from "http://host:port@dp_rank" + let (worker_url_prefix, _dp_rank) = match Self::extract_dp_rank(worker_url) { + Ok(tup) => tup, + Err(e) => { + debug!("Failed to extract dp_rank: {}", e); + return None; + } + }; + worker_url_prefix + } else { + worker_url + }; + + match client.get(format!("{}/get_load", worker_url)).send().await { + Ok(res) if res.status().is_success() => match res.bytes().await { + Ok(bytes) => match serde_json::from_slice::(&bytes) { + Ok(data) => data + .get("load") + .and_then(|v| v.as_i64()) + .map(|v| v as isize), + Err(e) => { + debug!("Failed to parse load response from {}: {}", worker_url, e); + None + } + }, + Err(e) => { + debug!("Failed to read load response from {}: {}", worker_url, e); + None + } + }, + Ok(res) => { + debug!( + "Worker {} returned non-success status: {}", + worker_url, + res.status() + ); + None + } + Err(e) => { + debug!("Failed to get load from {}: {}", worker_url, e); + None + } + } + } +} + +use async_trait::async_trait; + +#[async_trait] +impl WorkerManagement for Router { + async fn add_worker(&self, worker_url: &str) -> Result { + Router::add_worker(self, worker_url).await + } + + fn remove_worker(&self, worker_url: &str) { + Router::remove_worker(self, worker_url) + } + + fn get_worker_urls(&self) -> Vec { + Router::get_worker_urls(self) + } +} + +#[async_trait] +impl RouterTrait for Router { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + async fn health(&self, _req: Request) -> Response { + let workers = self.workers.read().unwrap(); + let unhealthy_servers: Vec<_> = workers + .iter() + .filter(|w| !w.is_healthy()) + .map(|w| w.url().to_string()) + .collect(); + + if unhealthy_servers.is_empty() { + (StatusCode::OK, "All servers healthy").into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Unhealthy servers: {:?}", unhealthy_servers), + ) + .into_response() + } + } + + async fn health_generate(&self, req: Request) -> Response { + self.proxy_get_request(req, "health_generate").await + } + + async fn get_server_info(&self, req: Request) -> Response { + self.proxy_get_request(req, "get_server_info").await + } + + async fn get_models(&self, req: Request) -> Response { + self.proxy_get_request(req, "v1/models").await + } + + async fn get_model_info(&self, req: Request) -> Response { + self.proxy_get_request(req, "get_model_info").await + } + + async fn route_generate( + &self, + headers: Option<&HeaderMap>, + body: &GenerateRequest, + ) -> Response { + self.route_typed_request(headers, body, "/generate").await + } + + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + ) -> Response { + self.route_typed_request(headers, body, "/v1/chat/completions") + .await + } + + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + ) -> Response { + self.route_typed_request(headers, body, "/v1/completions") + .await + } + + async fn route_embeddings(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + todo!() + } + + async fn route_rerank(&self, _headers: Option<&HeaderMap>, _body: Body) -> Response { + todo!() + } + + async fn flush_cache(&self) -> Response { + // Get all worker URLs + let worker_urls = self.get_worker_urls(); + + // Send requests to all workers concurrently without headers + let mut tasks = Vec::new(); + for worker_url in &worker_urls { + let worker_url = if self.dp_aware { + // Need to extract the URL from "http://host:port@dp_rank" + let (worker_url_prefix, _dp_rank) = match Self::extract_dp_rank(worker_url) { + Ok(tup) => tup, + Err(e) => { + error!("Failed to extract dp_rank: {}", e); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to extract dp_rank: {}", e), + ) + .into_response(); + } + }; + worker_url_prefix + } else { + worker_url + }; + let request_builder = self.client.post(format!("{}/flush_cache", worker_url)); + tasks.push(request_builder.send()); + } + + // Wait for all responses + let results = futures_util::future::join_all(tasks).await; + + // Check if all succeeded + let all_success = results.iter().all(|r| { + r.as_ref() + .map(|res| res.status().is_success()) + .unwrap_or(false) + }); + + if all_success { + (StatusCode::OK, "Cache flushed on all servers").into_response() + } else { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Cache flush failed on one or more servers", + ) + .into_response() + } + } + + async fn get_worker_loads(&self) -> Response { + let urls = self.get_worker_urls(); + let mut loads = Vec::new(); + + // Get loads from all workers + for url in &urls { + let load = self.get_worker_load(url).await.unwrap_or(-1); + loads.push(serde_json::json!({ + "worker": url, + "load": load + })); + } + + Json(serde_json::json!({ + "workers": loads + })) + .into_response() + } + + fn router_type(&self) -> &'static str { + "regular" + } + + fn readiness(&self) -> Response { + // Regular router is ready if it has at least one healthy worker + let healthy_count = self + .workers + .read() + .unwrap() + .iter() + .filter(|w| w.is_healthy()) + .count(); + + if healthy_count > 0 { + Json(serde_json::json!({ + "status": "ready", + "healthy_workers": healthy_count, + "total_workers": self.workers.read().unwrap().len() + })) + .into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "status": "not_ready", + "reason": "no healthy workers available", + "total_workers": self.workers.read().unwrap().len() + })), + ) + .into_response() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::policies::RandomPolicy; + use std::collections::HashMap; + + fn create_test_regular_router() -> Router { + let workers = vec![ + WorkerFactory::create_regular("http://worker1:8080".to_string()), + WorkerFactory::create_regular("http://worker2:8080".to_string()), + ]; + let (_, rx) = tokio::sync::watch::channel(HashMap::new()); + Router { + workers: Arc::new(RwLock::new(workers)), + policy: Arc::new(RandomPolicy::new()), + worker_startup_timeout_secs: 5, + worker_startup_check_interval_secs: 1, + dp_aware: false, + api_key: None, + client: Client::new(), + retry_config: RetryConfig::default(), + circuit_breaker_config: CircuitBreakerConfig::default(), + _worker_loads: Arc::new(rx), + _load_monitor_handle: None, + _health_checker: None, + } + } + + #[test] + fn test_router_get_worker_urls_regular() { + let router = create_test_regular_router(); + let urls = router.get_worker_urls(); + + assert_eq!(urls.len(), 2); + assert!(urls.contains(&"http://worker1:8080".to_string())); + assert!(urls.contains(&"http://worker2:8080".to_string())); + } + + #[test] + fn test_select_first_worker_regular() { + let router = create_test_regular_router(); + let result = router.select_first_worker(); + + assert!(result.is_ok()); + assert_eq!(result.unwrap(), "http://worker1:8080"); + } + + #[tokio::test] + async fn test_wait_for_healthy_workers_empty_list() { + // Empty list will return error immediately + let result = Router::wait_for_healthy_workers(&[], 1, 1).await; + assert!(result.is_err()); + assert!(result.unwrap_err().contains("no workers provided")); + } + + #[tokio::test] + async fn test_wait_for_healthy_workers_invalid_urls() { + // This test will timeout quickly since the URLs are invalid + let result = + Router::wait_for_healthy_workers(&["http://nonexistent:8080".to_string()], 1, 1).await; + assert!(result.is_err()); + assert!(result.unwrap_err().contains("Timeout")); + } +} diff --git a/sgl-router/src/routers/mod.rs b/sgl-router/src/routers/mod.rs new file mode 100644 index 000000000..e610fedb3 --- /dev/null +++ b/sgl-router/src/routers/mod.rs @@ -0,0 +1,107 @@ +//! Router implementations + +use async_trait::async_trait; +use axum::{ + body::Body, + extract::Request, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, +}; +use std::fmt::Debug; + +use crate::protocols::spec::{ChatCompletionRequest, CompletionRequest, GenerateRequest}; + +pub mod factory; +pub mod grpc; +pub mod header_utils; +pub mod http; + +pub use factory::RouterFactory; +// Re-export HTTP routers for convenience (keeps routers::openai_router path working) +pub use http::{openai_router, pd_router, pd_types, router}; + +/// Worker management trait for administrative operations +/// +/// This trait is separate from RouterTrait to allow Send futures +/// for use in service discovery and other background tasks +#[async_trait] +pub trait WorkerManagement: Send + Sync { + /// Add a worker to the router + async fn add_worker(&self, worker_url: &str) -> Result; + + /// Remove a worker from the router + fn remove_worker(&self, worker_url: &str); + + /// Get all worker URLs + fn get_worker_urls(&self) -> Vec; +} + +/// Core trait for all router implementations +/// +/// This trait provides a unified interface for routing requests, +/// regardless of whether it's a regular router or PD router. +#[async_trait] +pub trait RouterTrait: Send + Sync + Debug + WorkerManagement { + /// Get a reference to self as Any for downcasting + fn as_any(&self) -> &dyn std::any::Any; + + /// Route a health check request + async fn health(&self, req: Request) -> Response; + + /// Route a health generate request + async fn health_generate(&self, req: Request) -> Response; + + /// Get server information + async fn get_server_info(&self, req: Request) -> Response; + + /// Get available models + async fn get_models(&self, req: Request) -> Response; + + /// Get model information + async fn get_model_info(&self, req: Request) -> Response; + + /// Route a generate request + async fn route_generate(&self, headers: Option<&HeaderMap>, body: &GenerateRequest) + -> Response; + + /// Route a chat completion request + async fn route_chat( + &self, + headers: Option<&HeaderMap>, + body: &ChatCompletionRequest, + ) -> Response; + + /// Route a completion request + async fn route_completion( + &self, + headers: Option<&HeaderMap>, + body: &CompletionRequest, + ) -> Response; + + async fn route_embeddings(&self, headers: Option<&HeaderMap>, body: Body) -> Response; + + async fn route_rerank(&self, headers: Option<&HeaderMap>, body: Body) -> Response; + + /// Flush cache on all workers + async fn flush_cache(&self) -> Response; + + /// Get worker loads (for monitoring) + async fn get_worker_loads(&self) -> Response; + + /// Get router type name + fn router_type(&self) -> &'static str; + + /// Check if this is a PD router + fn is_pd_mode(&self) -> bool { + self.router_type() == "pd" + } + + /// Server liveness check - is the server process running + fn liveness(&self) -> Response { + // Simple liveness check - if we can respond, we're alive + (StatusCode::OK, "OK").into_response() + } + + /// Server readiness check - is the server ready to handle requests + fn readiness(&self) -> Response; +} diff --git a/sgl-router/src/server.rs b/sgl-router/src/server.rs new file mode 100644 index 000000000..2762f9765 --- /dev/null +++ b/sgl-router/src/server.rs @@ -0,0 +1,474 @@ +use crate::config::RouterConfig; +use crate::logging::{self, LoggingConfig}; +use crate::metrics::{self, PrometheusConfig}; +use crate::middleware::TokenBucket; +use crate::protocols::spec::{ChatCompletionRequest, CompletionRequest, GenerateRequest}; +use crate::reasoning_parser::ParserFactory; +use crate::routers::{RouterFactory, RouterTrait}; +use crate::service_discovery::{start_service_discovery, ServiceDiscoveryConfig}; +use crate::tokenizer::{factory as tokenizer_factory, traits::Tokenizer}; +use crate::tool_parser::ParserRegistry; +use axum::{ + extract::{Query, Request, State}, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, + Json, Router, +}; +use reqwest::Client; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpListener; +use tokio::signal; +use tokio::spawn; +use tracing::{error, info, warn, Level}; + +#[derive(Clone)] +pub struct AppContext { + pub client: Client, + pub router_config: RouterConfig, + pub rate_limiter: Arc, + pub tokenizer: Option>, + pub reasoning_parser_factory: Option, + pub tool_parser_registry: Option<&'static ParserRegistry>, +} + +impl AppContext { + pub fn new( + router_config: RouterConfig, + client: Client, + max_concurrent_requests: usize, + rate_limit_tokens_per_second: Option, + ) -> Result { + let rate_limit_tokens = rate_limit_tokens_per_second.unwrap_or(max_concurrent_requests); + let rate_limiter = Arc::new(TokenBucket::new(max_concurrent_requests, rate_limit_tokens)); + + // Initialize gRPC-specific components only when in gRPC mode + let (tokenizer, reasoning_parser_factory, tool_parser_registry) = + if router_config.connection_mode == crate::config::ConnectionMode::Grpc { + // Get tokenizer path (required for gRPC mode) + let tokenizer_path = router_config + .tokenizer_path + .clone() + .or_else(|| router_config.model_path.clone()) + .ok_or_else(|| { + "gRPC mode requires either --tokenizer-path or --model-path to be specified" + .to_string() + })?; + + // Initialize all gRPC components + let tokenizer = Some( + tokenizer_factory::create_tokenizer(&tokenizer_path) + .map_err(|e| format!("Failed to create tokenizer: {}", e))?, + ); + let reasoning_parser_factory = Some(ParserFactory::new()); + let tool_parser_registry = Some(ParserRegistry::new()); + + (tokenizer, reasoning_parser_factory, tool_parser_registry) + } else { + // HTTP mode doesn't need these components + (None, None, None) + }; + + Ok(Self { + client, + router_config, + rate_limiter, + tokenizer, + reasoning_parser_factory, + tool_parser_registry, + }) + } +} + +#[derive(Clone)] +pub struct AppState { + pub router: Arc, + pub context: Arc, + pub concurrency_queue_tx: Option>, +} + +// Fallback handler for unmatched routes +async fn sink_handler() -> Response { + StatusCode::NOT_FOUND.into_response() +} + +// Health check endpoints +async fn liveness(State(state): State>) -> Response { + state.router.liveness() +} + +async fn readiness(State(state): State>) -> Response { + state.router.readiness() +} + +async fn health(State(state): State>, req: Request) -> Response { + state.router.health(req).await +} + +async fn health_generate(State(state): State>, req: Request) -> Response { + state.router.health_generate(req).await +} + +async fn get_server_info(State(state): State>, req: Request) -> Response { + state.router.get_server_info(req).await +} + +async fn v1_models(State(state): State>, req: Request) -> Response { + state.router.get_models(req).await +} + +async fn get_model_info(State(state): State>, req: Request) -> Response { + state.router.get_model_info(req).await +} + +// Generation endpoints +// The RouterTrait now accepts optional headers and typed body directly +async fn generate( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state.router.route_generate(Some(&headers), &body).await +} + +async fn v1_chat_completions( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state.router.route_chat(Some(&headers), &body).await +} + +async fn v1_completions( + State(state): State>, + headers: http::HeaderMap, + Json(body): Json, +) -> Response { + state.router.route_completion(Some(&headers), &body).await +} + +// Worker management endpoints +async fn add_worker( + State(state): State>, + Query(params): Query>, +) -> Response { + let worker_url = match params.get("url") { + Some(url) => url.to_string(), + None => { + return ( + StatusCode::BAD_REQUEST, + "Worker URL required. Provide 'url' query parameter", + ) + .into_response(); + } + }; + + match state.router.add_worker(&worker_url).await { + Ok(message) => (StatusCode::OK, message).into_response(), + Err(error) => (StatusCode::BAD_REQUEST, error).into_response(), + } +} + +async fn list_workers(State(state): State>) -> Response { + let worker_list = state.router.get_worker_urls(); + Json(serde_json::json!({ "urls": worker_list })).into_response() +} + +async fn remove_worker( + State(state): State>, + Query(params): Query>, +) -> Response { + let worker_url = match params.get("url") { + Some(url) => url.to_string(), + None => return StatusCode::BAD_REQUEST.into_response(), + }; + + state.router.remove_worker(&worker_url); + ( + StatusCode::OK, + format!("Successfully removed worker: {}", worker_url), + ) + .into_response() +} + +async fn flush_cache(State(state): State>, _req: Request) -> Response { + state.router.flush_cache().await +} + +async fn get_loads(State(state): State>, _req: Request) -> Response { + state.router.get_worker_loads().await +} + +pub struct ServerConfig { + pub host: String, + pub port: u16, + pub router_config: RouterConfig, + pub max_payload_size: usize, + pub log_dir: Option, + pub log_level: Option, + pub service_discovery_config: Option, + pub prometheus_config: Option, + pub request_timeout_secs: u64, + pub request_id_headers: Option>, +} + +/// Build the Axum application with all routes and middleware +pub fn build_app( + app_state: Arc, + max_payload_size: usize, + request_id_headers: Vec, + cors_allowed_origins: Vec, +) -> Router { + // Create routes + let protected_routes = Router::new() + .route("/generate", post(generate)) + .route("/v1/chat/completions", post(v1_chat_completions)) + .route("/v1/completions", post(v1_completions)) + .route_layer(axum::middleware::from_fn_with_state( + app_state.clone(), + crate::middleware::concurrency_limit_middleware, + )); + + let public_routes = Router::new() + .route("/liveness", get(liveness)) + .route("/readiness", get(readiness)) + .route("/health", get(health)) + .route("/health_generate", get(health_generate)) + .route("/v1/models", get(v1_models)) + .route("/get_model_info", get(get_model_info)) + .route("/get_server_info", get(get_server_info)); + + let admin_routes = Router::new() + .route("/add_worker", post(add_worker)) + .route("/remove_worker", post(remove_worker)) + .route("/list_workers", get(list_workers)) + .route("/flush_cache", post(flush_cache)) + .route("/get_loads", get(get_loads)); + + // Build app with all routes and middleware + Router::new() + .merge(protected_routes) + .merge(public_routes) + .merge(admin_routes) + // Request body size limiting + .layer(tower_http::limit::RequestBodyLimitLayer::new( + max_payload_size, + )) + // Request ID layer - must be added AFTER logging layer in the code + // so it executes BEFORE logging layer at runtime (layers execute bottom-up) + .layer(crate::middleware::RequestIdLayer::new(request_id_headers)) + // Custom logging layer that can now see request IDs from extensions + .layer(crate::middleware::create_logging_layer()) + // CORS (should be outermost) + .layer(create_cors_layer(cors_allowed_origins)) + // Fallback + .fallback(sink_handler) + // State - apply last to get Router> + .with_state(app_state) +} + +pub async fn startup(config: ServerConfig) -> Result<(), Box> { + // Only initialize logging if not already done (for Python bindings support) + static LOGGING_INITIALIZED: AtomicBool = AtomicBool::new(false); + + let _log_guard = if !LOGGING_INITIALIZED.swap(true, Ordering::SeqCst) { + Some(logging::init_logging(LoggingConfig { + level: config + .log_level + .as_deref() + .and_then(|s| match s.to_uppercase().parse::() { + Ok(l) => Some(l), + Err(_) => { + warn!("Invalid log level string: '{}'. Defaulting to INFO.", s); + None + } + }) + .unwrap_or(Level::INFO), + json_format: false, + log_dir: config.log_dir.clone(), + colorize: true, + log_file_name: "sgl-router".to_string(), + log_targets: None, + })) + } else { + None + }; + + // Initialize prometheus metrics exporter + if let Some(prometheus_config) = config.prometheus_config { + metrics::start_prometheus(prometheus_config); + } + + info!( + "Starting router on {}:{} | mode: {:?} | policy: {:?} | max_payload: {}MB", + config.host, + config.port, + config.router_config.mode, + config.router_config.policy, + config.max_payload_size / (1024 * 1024) + ); + + let client = Client::builder() + .pool_idle_timeout(Some(Duration::from_secs(50))) + .pool_max_idle_per_host(500) // Increase to 500 connections per host + .timeout(Duration::from_secs(config.request_timeout_secs)) + .connect_timeout(Duration::from_secs(10)) // Separate connection timeout + .tcp_nodelay(true) + .tcp_keepalive(Some(Duration::from_secs(30))) // Keep connections alive + .build() + .expect("Failed to create HTTP client"); + + // Create the application context with all dependencies + let app_context = Arc::new(AppContext::new( + config.router_config.clone(), + client.clone(), + config.router_config.max_concurrent_requests, + config.router_config.rate_limit_tokens_per_second, + )?); + + // Create router with the context + let router = RouterFactory::create_router(&app_context).await?; + + // Set up concurrency limiter with queue if configured + let (limiter, processor) = crate::middleware::ConcurrencyLimiter::new( + app_context.rate_limiter.clone(), + config.router_config.queue_size, + Duration::from_secs(config.router_config.queue_timeout_secs), + ); + + // Start queue processor if enabled + if let Some(processor) = processor { + tokio::spawn(processor.run()); + info!( + "Started request queue with size: {}, timeout: {}s", + config.router_config.queue_size, config.router_config.queue_timeout_secs + ); + } + + // Create app state with router and context + let app_state = Arc::new(AppState { + router: Arc::from(router), + context: app_context.clone(), + concurrency_queue_tx: limiter.queue_tx.clone(), + }); + let router_arc = Arc::clone(&app_state.router); + + // Start the service discovery if enabled + if let Some(service_discovery_config) = config.service_discovery_config { + if service_discovery_config.enabled { + match start_service_discovery(service_discovery_config, router_arc).await { + Ok(handle) => { + info!("Service discovery started"); + // Spawn a task to handle the service discovery thread + spawn(async move { + if let Err(e) = handle.await { + error!("Service discovery task failed: {:?}", e); + } + }); + } + Err(e) => { + error!("Failed to start service discovery: {}", e); + warn!("Continuing without service discovery"); + } + } + } + } + + info!( + "Router ready | workers: {:?}", + app_state.router.get_worker_urls() + ); + + // Configure request ID headers + let request_id_headers = config.request_id_headers.clone().unwrap_or_else(|| { + vec![ + "x-request-id".to_string(), + "x-correlation-id".to_string(), + "x-trace-id".to_string(), + "request-id".to_string(), + ] + }); + + // Build the application + let app = build_app( + app_state, + config.max_payload_size, + request_id_headers, + config.router_config.cors_allowed_origins.clone(), + ); + + // Create TCP listener - use the configured host + let addr = format!("{}:{}", config.host, config.port); + let listener = TcpListener::bind(&addr).await?; + + // Start server with graceful shutdown + info!("Starting server on {}", addr); + + // Serve the application with graceful shutdown + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .map_err(|e| Box::new(e) as Box)?; + + Ok(()) +} + +// Graceful shutdown handler +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => { + info!("Received Ctrl+C, starting graceful shutdown"); + }, + _ = terminate => { + info!("Received terminate signal, starting graceful shutdown"); + }, + } +} + +// CORS Layer Creation +fn create_cors_layer(allowed_origins: Vec) -> tower_http::cors::CorsLayer { + use tower_http::cors::Any; + + let cors = if allowed_origins.is_empty() { + // Allow all origins if none specified + tower_http::cors::CorsLayer::new() + .allow_origin(Any) + .allow_methods(Any) + .allow_headers(Any) + .expose_headers(Any) + } else { + // Restrict to specific origins + let origins: Vec = allowed_origins + .into_iter() + .filter_map(|origin| origin.parse().ok()) + .collect(); + + tower_http::cors::CorsLayer::new() + .allow_origin(origins) + .allow_methods([http::Method::GET, http::Method::POST, http::Method::OPTIONS]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .expose_headers([http::header::HeaderName::from_static("x-request-id")]) + }; + + cors.max_age(Duration::from_secs(3600)) +} diff --git a/sgl-router/src/service_discovery.rs b/sgl-router/src/service_discovery.rs new file mode 100644 index 000000000..c27317f86 --- /dev/null +++ b/sgl-router/src/service_discovery.rs @@ -0,0 +1,1168 @@ +use crate::routers::RouterTrait; + +use futures::{StreamExt, TryStreamExt}; +use k8s_openapi::api::core::v1::Pod; +use kube::{ + api::Api, + runtime::watcher::{watcher, Config}, + runtime::WatchStreamExt, + Client, +}; +use std::collections::{HashMap, HashSet}; + +use rustls; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::task; +use tokio::time; +use tracing::{debug, error, info, warn}; + +/// Represents the service discovery configuration +#[derive(Debug, Clone)] +pub struct ServiceDiscoveryConfig { + pub enabled: bool, + pub selector: HashMap, + pub check_interval: Duration, + pub port: u16, + pub namespace: Option, + // PD mode specific configuration + pub pd_mode: bool, + pub prefill_selector: HashMap, + pub decode_selector: HashMap, + // Bootstrap port annotation specific to mooncake implementation + pub bootstrap_port_annotation: String, +} + +impl Default for ServiceDiscoveryConfig { + fn default() -> Self { + ServiceDiscoveryConfig { + enabled: false, + selector: HashMap::new(), + check_interval: Duration::from_secs(60), + port: 8000, // Standard port for modern services + namespace: None, // None means watch all namespaces + pd_mode: false, + prefill_selector: HashMap::new(), + decode_selector: HashMap::new(), + bootstrap_port_annotation: "sglang.ai/bootstrap-port".to_string(), + } + } +} + +/// Pod type for PD mode service discovery +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum PodType { + Prefill, + Decode, + Regular, +} + +/// Represents a Kubernetes pod's information used for worker management +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PodInfo { + pub name: String, + pub ip: String, + pub status: String, + pub is_ready: bool, + pub pod_type: Option, + pub bootstrap_port: Option, +} + +impl PodInfo { + /// Check if a pod matches any of the given selectors + fn matches_selector(pod: &Pod, selector: &HashMap) -> bool { + if selector.is_empty() { + return false; + } + + pod.metadata + .labels + .as_ref() + .is_some_and(|labels| selector.iter().all(|(k, v)| labels.get(k) == Some(v))) + } + + /// Check if a pod should be included in service discovery + pub fn should_include(pod: &Pod, config: &ServiceDiscoveryConfig) -> bool { + if config.pd_mode { + // In PD mode, at least one selector must be non-empty + if config.prefill_selector.is_empty() && config.decode_selector.is_empty() { + warn!("PD mode enabled but both prefill_selector and decode_selector are empty"); + return false; + } + // In PD mode, pod must match either prefill or decode selector + Self::matches_selector(pod, &config.prefill_selector) + || Self::matches_selector(pod, &config.decode_selector) + } else { + // In regular mode, pod must match the general selector + if config.selector.is_empty() { + warn!("Regular mode enabled but selector is empty"); + return false; + } + Self::matches_selector(pod, &config.selector) + } + } + + /// Unified PodInfo creation with optional PD configuration + pub fn from_pod(pod: &Pod, config: Option<&ServiceDiscoveryConfig>) -> Option { + let name = pod.metadata.name.clone()?; + let status = pod.status.clone()?; + let pod_ip = status.pod_ip?; + + let is_ready = if let Some(conditions) = &status.conditions { + conditions + .iter() + .any(|condition| condition.type_ == "Ready" && condition.status == "True") + } else { + false + }; + + let pod_status = status.phase.unwrap_or_else(|| "Unknown".to_string()); + + // Determine pod type based on labels if config is provided and in PD mode + let pod_type = if let Some(config) = config { + if config.pd_mode { + // Use simplified helper methods for cleaner logic + if Self::matches_selector(pod, &config.prefill_selector) { + Some(PodType::Prefill) + } else if Self::matches_selector(pod, &config.decode_selector) { + Some(PodType::Decode) + } else { + Some(PodType::Regular) + } + } else { + Some(PodType::Regular) + } + } else { + // No config provided, default to None (for backwards compatibility) + None + }; + + // Extract bootstrap port from annotations for prefill pods + let bootstrap_port = if matches!(pod_type, Some(PodType::Prefill)) { + if let Some(config) = config { + pod.metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(&config.bootstrap_port_annotation)) + .and_then(|port_str| port_str.parse::().ok()) + } else { + None + } + } else { + None + }; + + Some(PodInfo { + name, + ip: pod_ip, + status: pod_status, + is_ready, + pod_type, + bootstrap_port, + }) + } + + /// Returns true if the pod is in a state where it can accept traffic + pub fn is_healthy(&self) -> bool { + self.is_ready && self.status == "Running" + } + + /// Generates a worker URL for this pod + pub fn worker_url(&self, port: u16) -> String { + format!("http://{}:{}", self.ip, port) + } +} + +pub async fn start_service_discovery( + config: ServiceDiscoveryConfig, + router: Arc, +) -> Result, kube::Error> { + // Don't initialize anything if service discovery is disabled + if !config.enabled { + // Return a generic error when service discovery is disabled + return Err(kube::Error::Api(kube::error::ErrorResponse { + status: "Disabled".to_string(), + message: "Service discovery is disabled".to_string(), + reason: "ConfigurationError".to_string(), + code: 400, + })); + } + + let _ = rustls::crypto::ring::default_provider().install_default(); + + // Initialize Kubernetes client + let client = Client::try_default().await?; + + // Log the appropriate selectors based on mode + if config.pd_mode { + let prefill_selector = config + .prefill_selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + let decode_selector = config + .decode_selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + info!( + "Starting K8s service discovery | PD mode | prefill: '{}' | decode: '{}'", + prefill_selector, decode_selector + ); + } else { + let label_selector = config + .selector + .iter() + .map(|(k, v)| format!("{}={}", k, v)) + .collect::>() + .join(","); + + info!( + "Starting K8s service discovery | selector: '{}'", + label_selector + ); + } + + // Create the task that will run in the background + let handle = task::spawn(async move { + // We'll track pods we've already added to avoid duplicates + let tracked_pods = Arc::new(Mutex::new(HashSet::new())); + + // Create a watcher for pods + let pods: Api = if let Some(namespace) = &config.namespace { + Api::namespaced(client, namespace) + } else { + Api::all(client) + }; + + debug!("K8s service discovery initialized"); + + // Create Arcs for configuration data + let config_arc = Arc::new(config.clone()); + let port = config.port; + + let mut retry_delay = Duration::from_secs(1); + const MAX_RETRY_DELAY: Duration = Duration::from_secs(300); // 5 minutes max + + loop { + // Create a watcher with the proper parameters according to the kube-rs API + let watcher_config = Config::default(); + let watcher_stream = watcher(pods.clone(), watcher_config).applied_objects(); + + // Clone Arcs for the closures + let config_clone = Arc::clone(&config_arc); + let tracked_pods_clone = Arc::clone(&tracked_pods); + + // Simplified label selector filter using helper method + let filtered_stream = watcher_stream.filter_map(move |obj_res| { + let config_inner = Arc::clone(&config_clone); + + async move { + match obj_res { + Ok(pod) => { + if PodInfo::should_include(&pod, &config_inner) { + Some(Ok(pod)) + } else { + None + } + } + Err(e) => Some(Err(e)), + } + } + }); + + // Clone again for the next closure + let tracked_pods_clone2 = Arc::clone(&tracked_pods_clone); + let router_clone = Arc::clone(&router); + let config_clone2 = Arc::clone(&config_arc); + + match filtered_stream + .try_for_each(move |pod| { + let tracked_pods_inner = Arc::clone(&tracked_pods_clone2); + let router_inner = Arc::clone(&router_clone); + let config_inner = Arc::clone(&config_clone2); + + async move { + let pod_info = PodInfo::from_pod(&pod, Some(&config_inner)); + + if let Some(pod_info) = pod_info { + if pod.metadata.deletion_timestamp.is_some() { + handle_pod_deletion( + &pod_info, + tracked_pods_inner, + router_inner, + port, + config_inner.pd_mode, + ) + .await; + } else { + handle_pod_event( + &pod_info, + tracked_pods_inner, + router_inner, + port, + config_inner.pd_mode, + ) + .await; + } + } + Ok(()) + } + }) + .await + { + Ok(_) => { + // Reset retry delay on success + retry_delay = Duration::from_secs(1); + } + Err(err) => { + error!("Error in Kubernetes watcher: {}", err); + warn!( + "Retrying in {} seconds with exponential backoff", + retry_delay.as_secs() + ); + time::sleep(retry_delay).await; + + // Exponential backoff with jitter + retry_delay = std::cmp::min(retry_delay * 2, MAX_RETRY_DELAY); + } + } + + // If the watcher exits for some reason, wait a bit before restarting + warn!( + "Kubernetes watcher exited, restarting in {} seconds", + config_arc.check_interval.as_secs() + ); + time::sleep(config_arc.check_interval).await; + } + }); + + Ok(handle) +} + +async fn handle_pod_event( + pod_info: &PodInfo, + tracked_pods: Arc>>, + router: Arc, + port: u16, + pd_mode: bool, +) { + let worker_url = pod_info.worker_url(port); + + // If pod is healthy, try to add it (with atomic check-and-insert) + if pod_info.is_healthy() { + // Atomic check-and-insert to prevent race conditions + let should_add = { + let mut tracker = match tracked_pods.lock() { + Ok(tracker) => tracker, + Err(e) => { + error!("Failed to acquire tracked_pods lock: {}", e); + return; + } + }; + + if tracker.contains(pod_info) { + false // Already tracked + } else { + // Reserve the spot to prevent other threads from adding the same pod + tracker.insert(pod_info.clone()); + true + } + }; + + if should_add { + info!( + "Adding pod: {} | type: {:?} | url: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + + // Handle PD mode with specific pod types + let result = if pd_mode && pod_info.pod_type.is_some() { + // Need to import PDRouter type + use crate::routers::http::pd_router::PDRouter; + + // Try to downcast to PDRouter + if let Some(pd_router) = router.as_any().downcast_ref::() { + match &pod_info.pod_type { + Some(PodType::Prefill) => pd_router + .add_prefill_server(worker_url.clone(), pod_info.bootstrap_port) + .await + .map_err(|e| e.to_string()), + Some(PodType::Decode) => pd_router + .add_decode_server(worker_url.clone()) + .await + .map_err(|e| e.to_string()), + Some(PodType::Regular) | None => { + // Fall back to regular add_worker for regular pods + router.add_worker(&worker_url).await + } + } + } else { + Err("PD mode enabled but router is not a PDRouter".to_string()) + } + } else { + // Regular mode or no pod type specified + router.add_worker(&worker_url).await + }; + + match result { + Ok(_) => { + debug!("Worker added: {}", worker_url); + } + Err(e) => { + error!("Failed to add worker {} to router: {}", worker_url, e); + // Remove from tracking since addition failed + if let Ok(mut tracker) = tracked_pods.lock() { + tracker.remove(pod_info); + } + } + } + } + } +} + +async fn handle_pod_deletion( + pod_info: &PodInfo, + tracked_pods: Arc>>, + router: Arc, + port: u16, + pd_mode: bool, +) { + let worker_url = pod_info.worker_url(port); + + let was_tracked = { + let mut tracked = match tracked_pods.lock() { + Ok(tracked) => tracked, + Err(e) => { + error!("Failed to acquire tracked_pods lock during deletion: {}", e); + return; + } + }; + tracked.remove(pod_info) + }; + + if was_tracked { + info!( + "Removing pod: {} | type: {:?} | url: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + + // Handle PD mode removal + if pd_mode && pod_info.pod_type.is_some() { + use crate::routers::http::pd_router::PDRouter; + + // Try to downcast to PDRouter for PD-specific removal + if let Some(pd_router) = router.as_any().downcast_ref::() { + match &pod_info.pod_type { + Some(PodType::Prefill) => { + if let Err(e) = pd_router.remove_prefill_server(&worker_url).await { + error!("Failed to remove prefill server {}: {}", worker_url, e); + } + } + Some(PodType::Decode) => { + if let Err(e) = pd_router.remove_decode_server(&worker_url).await { + error!("Failed to remove decode server {}: {}", worker_url, e); + } + } + Some(PodType::Regular) | None => { + // Fall back to regular remove_worker + router.remove_worker(&worker_url); + } + } + } else { + // PD mode but not a PDRouter, use generic removal + router.remove_worker(&worker_url); + } + } else { + // Regular mode removal + router.remove_worker(&worker_url); + } + } else { + // This case might occur if a pod is deleted before it was ever marked healthy and added. + // Or if the event is duplicated. No action needed on the router if it wasn't tracked (and thus not added). + debug!( + "Pod deletion event for untracked/already removed pod: {} (type: {:?}). Worker URL: {}", + pod_info.name, pod_info.pod_type, worker_url + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::api::core::v1::{Pod, PodCondition, PodSpec, PodStatus}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time; + + // Helper function to create a Pod for testing PodInfo::from_pod + fn create_k8s_pod( + name: Option<&str>, + ip: Option<&str>, + phase: Option<&str>, + ready_status: Option<&str>, + deletion_timestamp: Option")) +} + +/// Factory function to create tokenizer from a model name or path (async version) +pub async fn create_tokenizer_async( + model_name_or_path: &str, +) -> Result> { + // Check if it's a file path + let path = Path::new(model_name_or_path); + if path.exists() { + return create_tokenizer_from_file(model_name_or_path); + } + + // Check if it's a GPT model name that should use Tiktoken + if model_name_or_path.contains("gpt-") + || model_name_or_path.contains("davinci") + || model_name_or_path.contains("curie") + || model_name_or_path.contains("babbage") + || model_name_or_path.contains("ada") + { + let tokenizer = TiktokenTokenizer::from_model_name(model_name_or_path)?; + return Ok(Arc::new(tokenizer)); + } + + // Try to download tokenizer files from HuggingFace + match download_tokenizer_from_hf(model_name_or_path).await { + Ok(cache_dir) => { + // Look for tokenizer.json in the cache directory + let tokenizer_path = cache_dir.join("tokenizer.json"); + if tokenizer_path.exists() { + create_tokenizer_from_file(tokenizer_path.to_str().unwrap()) + } else { + // Try other common tokenizer file names + let possible_files = ["tokenizer_config.json", "vocab.json"]; + for file_name in &possible_files { + let file_path = cache_dir.join(file_name); + if file_path.exists() { + return create_tokenizer_from_file(file_path.to_str().unwrap()); + } + } + Err(Error::msg(format!( + "Downloaded model '{}' but couldn't find a suitable tokenizer file", + model_name_or_path + ))) + } + } + Err(e) => Err(Error::msg(format!( + "Failed to download tokenizer from HuggingFace: {}", + e + ))), + } +} + +/// Factory function to create tokenizer from a model name or path (blocking version) +pub fn create_tokenizer(model_name_or_path: &str) -> Result> { + // Check if it's a file path + let path = Path::new(model_name_or_path); + if path.exists() { + return create_tokenizer_from_file(model_name_or_path); + } + + // Check if it's a GPT model name that should use Tiktoken + if model_name_or_path.contains("gpt-") + || model_name_or_path.contains("davinci") + || model_name_or_path.contains("curie") + || model_name_or_path.contains("babbage") + || model_name_or_path.contains("ada") + { + let tokenizer = TiktokenTokenizer::from_model_name(model_name_or_path)?; + return Ok(Arc::new(tokenizer)); + } + + // Only use tokio for HuggingFace downloads + // Check if we're already in a tokio runtime + if let Ok(handle) = tokio::runtime::Handle::try_current() { + // We're in a runtime, use block_in_place + tokio::task::block_in_place(|| handle.block_on(create_tokenizer_async(model_name_or_path))) + } else { + // No runtime, create a temporary one + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(create_tokenizer_async(model_name_or_path)) + } +} + +/// Get information about a tokenizer file +pub fn get_tokenizer_info(file_path: &str) -> Result { + let path = Path::new(file_path); + + if !path.exists() { + return Err(Error::msg(format!("File not found: {}", file_path))); + } + + let extension = path + .extension() + .and_then(std::ffi::OsStr::to_str) + .map(|s| s.to_lowercase()); + + match extension.as_deref() { + Some("json") => Ok(TokenizerType::HuggingFace(file_path.to_string())), + _ => { + // Try auto-detection + use std::fs::File; + use std::io::Read; + + let mut file = File::open(file_path)?; + let mut buffer = vec![0u8; 512]; + let bytes_read = file.read(&mut buffer)?; + buffer.truncate(bytes_read); + + if is_likely_json(&buffer) { + Ok(TokenizerType::HuggingFace(file_path.to_string())) + } else { + Err(Error::msg("Unknown tokenizer type")) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_json_detection() { + assert!(is_likely_json(b"{\"test\": \"value\"}")); + assert!(is_likely_json(b" \n\t{\"test\": \"value\"}")); + assert!(is_likely_json(b"[1, 2, 3]")); + assert!(!is_likely_json(b"not json")); + assert!(!is_likely_json(b"")); + } + + #[test] + fn test_mock_tokenizer_creation() { + let tokenizer = create_tokenizer_from_file("mock").unwrap(); + assert_eq!(tokenizer.vocab_size(), 8); // Mock tokenizer has 8 tokens + } + + #[test] + fn test_file_not_found() { + let result = create_tokenizer_from_file("/nonexistent/file.json"); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("File not found")); + } + } + + #[test] + fn test_create_tiktoken_tokenizer() { + // Test creating tokenizer for GPT models + let tokenizer = create_tokenizer("gpt-4").unwrap(); + assert!(tokenizer.vocab_size() > 0); + + // Test encoding and decoding + let text = "Hello, world!"; + let encoding = tokenizer.encode(text).unwrap(); + let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap(); + assert_eq!(decoded, text); + } + + #[tokio::test] + async fn test_download_tokenizer_from_hf() { + // Test with a small model that should have tokenizer files + // Skip this test if HF_TOKEN is not set and we're in CI + if std::env::var("CI").is_ok() && std::env::var("HF_TOKEN").is_err() { + println!("Skipping HF download test in CI without HF_TOKEN"); + return; + } + + // Try to create tokenizer for a known small model + let result = create_tokenizer_async("bert-base-uncased").await; + + // The test might fail due to network issues or rate limiting + // so we just check that the function executes without panic + match result { + Ok(tokenizer) => { + assert!(tokenizer.vocab_size() > 0); + println!("Successfully downloaded and created tokenizer"); + } + Err(e) => { + println!("Download failed (this might be expected): {}", e); + // Don't fail the test - network issues shouldn't break CI + } + } + } +} diff --git a/sgl-router/src/tokenizer/hub.rs b/sgl-router/src/tokenizer/hub.rs new file mode 100644 index 000000000..c9d2cd1a4 --- /dev/null +++ b/sgl-router/src/tokenizer/hub.rs @@ -0,0 +1,238 @@ +use hf_hub::api::tokio::ApiBuilder; +use std::env; +use std::path::{Path, PathBuf}; + +const IGNORED: [&str; 5] = [ + ".gitattributes", + "LICENSE", + "LICENSE.txt", + "README.md", + "USE_POLICY.md", +]; + +const HF_TOKEN_ENV_VAR: &str = "HF_TOKEN"; + +/// Checks if a file is a model weight file +fn is_weight_file(filename: &str) -> bool { + filename.ends_with(".bin") + || filename.ends_with(".safetensors") + || filename.ends_with(".h5") + || filename.ends_with(".msgpack") + || filename.ends_with(".ckpt.index") +} + +/// Checks if a file is an image file +fn is_image(filename: &str) -> bool { + filename.ends_with(".png") + || filename.ends_with("PNG") + || filename.ends_with(".jpg") + || filename.ends_with("JPG") + || filename.ends_with(".jpeg") + || filename.ends_with("JPEG") +} + +/// Checks if a file is a tokenizer file +fn is_tokenizer_file(filename: &str) -> bool { + filename.ends_with("tokenizer.json") + || filename.ends_with("tokenizer_config.json") + || filename.ends_with("special_tokens_map.json") + || filename.ends_with("vocab.json") + || filename.ends_with("merges.txt") + || filename.ends_with(".model") // SentencePiece models + || filename.ends_with(".tiktoken") +} + +/// Attempt to download tokenizer files from Hugging Face +/// Returns the directory containing the downloaded tokenizer files +pub async fn download_tokenizer_from_hf(model_id: impl AsRef) -> anyhow::Result { + let model_id = model_id.as_ref(); + let token = env::var(HF_TOKEN_ENV_VAR).ok(); + let api = ApiBuilder::new() + .with_progress(true) + .with_token(token) + .build()?; + let model_name = model_id.display().to_string(); + + let repo = api.model(model_name.clone()); + + let info = match repo.info().await { + Ok(info) => info, + Err(e) => { + return Err(anyhow::anyhow!( + "Failed to fetch model '{}' from HuggingFace: {}. Is this a valid HuggingFace ID?", + model_name, + e + )); + } + }; + + if info.siblings.is_empty() { + return Err(anyhow::anyhow!( + "Model '{}' exists but contains no downloadable files.", + model_name + )); + } + + let mut cache_dir = None; + let mut tokenizer_files_found = false; + + // First, identify all tokenizer files to download + let tokenizer_files: Vec<_> = info + .siblings + .iter() + .filter(|sib| { + !IGNORED.contains(&sib.rfilename.as_str()) + && !is_image(&sib.rfilename) + && !is_weight_file(&sib.rfilename) + && is_tokenizer_file(&sib.rfilename) + }) + .collect(); + + if tokenizer_files.is_empty() { + return Err(anyhow::anyhow!( + "No tokenizer files found for model '{}'.", + model_name + )); + } + + // Download all tokenizer files + for sib in tokenizer_files { + match repo.get(&sib.rfilename).await { + Ok(path) => { + if cache_dir.is_none() { + cache_dir = path.parent().map(|p| p.to_path_buf()); + } + tokenizer_files_found = true; + } + Err(e) => { + return Err(anyhow::anyhow!( + "Failed to download tokenizer file '{}' from model '{}': {}", + sib.rfilename, + model_name, + e + )); + } + } + } + + if !tokenizer_files_found { + return Err(anyhow::anyhow!( + "No tokenizer files could be downloaded for model '{}'.", + model_name + )); + } + + match cache_dir { + Some(dir) => Ok(dir), + None => Err(anyhow::anyhow!( + "Invalid HF cache path for model '{}'", + model_name + )), + } +} + +/// Attempt to download a model from Hugging Face (including weights) +/// Returns the directory it is in +/// If ignore_weights is true, model weight files will be skipped +pub async fn from_hf(name: impl AsRef, ignore_weights: bool) -> anyhow::Result { + let name = name.as_ref(); + let token = env::var(HF_TOKEN_ENV_VAR).ok(); + let api = ApiBuilder::new() + .with_progress(true) + .with_token(token) + .build()?; + let model_name = name.display().to_string(); + + let repo = api.model(model_name.clone()); + + let info = match repo.info().await { + Ok(info) => info, + Err(e) => { + return Err(anyhow::anyhow!( + "Failed to fetch model '{}' from HuggingFace: {}. Is this a valid HuggingFace ID?", + model_name, + e + )); + } + }; + + if info.siblings.is_empty() { + return Err(anyhow::anyhow!( + "Model '{}' exists but contains no downloadable files.", + model_name + )); + } + + let mut p = PathBuf::new(); + let mut files_downloaded = false; + + for sib in info.siblings { + if IGNORED.contains(&sib.rfilename.as_str()) || is_image(&sib.rfilename) { + continue; + } + + // If ignore_weights is true, skip weight files + if ignore_weights && is_weight_file(&sib.rfilename) { + continue; + } + + match repo.get(&sib.rfilename).await { + Ok(path) => { + p = path; + files_downloaded = true; + } + Err(e) => { + return Err(anyhow::anyhow!( + "Failed to download file '{}' from model '{}': {}", + sib.rfilename, + model_name, + e + )); + } + } + } + + if !files_downloaded { + let file_type = if ignore_weights { + "non-weight" + } else { + "valid" + }; + return Err(anyhow::anyhow!( + "No {} files found for model '{}'.", + file_type, + model_name + )); + } + + match p.parent() { + Some(p) => Ok(p.to_path_buf()), + None => Err(anyhow::anyhow!("Invalid HF cache path: {}", p.display())), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_tokenizer_file() { + assert!(is_tokenizer_file("tokenizer.json")); + assert!(is_tokenizer_file("tokenizer_config.json")); + assert!(is_tokenizer_file("special_tokens_map.json")); + assert!(is_tokenizer_file("vocab.json")); + assert!(is_tokenizer_file("merges.txt")); + assert!(is_tokenizer_file("spiece.model")); + assert!(!is_tokenizer_file("model.bin")); + assert!(!is_tokenizer_file("README.md")); + } + + #[test] + fn test_is_weight_file() { + assert!(is_weight_file("model.bin")); + assert!(is_weight_file("model.safetensors")); + assert!(is_weight_file("pytorch_model.bin")); + assert!(!is_weight_file("tokenizer.json")); + assert!(!is_weight_file("config.json")); + } +} diff --git a/sgl-router/src/tokenizer/huggingface.rs b/sgl-router/src/tokenizer/huggingface.rs new file mode 100644 index 000000000..02dce5a0a --- /dev/null +++ b/sgl-router/src/tokenizer/huggingface.rs @@ -0,0 +1,234 @@ +use super::traits::{ + Decoder, Encoder, Encoding, SpecialTokens, TokenIdType, Tokenizer as TokenizerTrait, +}; +use anyhow::{Error, Result}; +use std::collections::HashMap; +use tokenizers::tokenizer::Tokenizer as HfTokenizer; + +use super::chat_template::{ChatMessage, ChatTemplateProcessor}; + +/// HuggingFace tokenizer wrapper +pub struct HuggingFaceTokenizer { + tokenizer: HfTokenizer, + special_tokens: SpecialTokens, + vocab: HashMap, + reverse_vocab: HashMap, + chat_template: Option, +} + +impl HuggingFaceTokenizer { + /// Create a tokenizer from a HuggingFace tokenizer JSON file + pub fn from_file(file_path: &str) -> Result { + Self::from_file_with_chat_template(file_path, None) + } + + /// Create a tokenizer from a HuggingFace tokenizer JSON file with an optional chat template + pub fn from_file_with_chat_template( + file_path: &str, + chat_template_path: Option<&str>, + ) -> Result { + let tokenizer = HfTokenizer::from_file(file_path) + .map_err(|e| Error::msg(format!("Failed to load tokenizer: {}", e)))?; + + // Extract special tokens + let special_tokens = Self::extract_special_tokens(&tokenizer); + + // Build vocab mappings + let vocab = tokenizer.get_vocab(false); + let reverse_vocab: HashMap = vocab + .iter() + .map(|(token, &id)| (id, token.clone())) + .collect(); + + // Load chat template + let chat_template = if let Some(template_path) = chat_template_path { + // Load from specified .jinja file + Self::load_chat_template_from_file(template_path)? + } else { + // Try to load from tokenizer_config.json + Self::load_chat_template(file_path) + }; + + Ok(HuggingFaceTokenizer { + tokenizer, + special_tokens, + vocab, + reverse_vocab, + chat_template, + }) + } + + /// Create from an existing HuggingFace tokenizer + pub fn from_tokenizer(tokenizer: HfTokenizer) -> Self { + let special_tokens = Self::extract_special_tokens(&tokenizer); + let vocab = tokenizer.get_vocab(false); + let reverse_vocab: HashMap = vocab + .iter() + .map(|(token, &id)| (id, token.clone())) + .collect(); + + HuggingFaceTokenizer { + tokenizer, + special_tokens, + vocab, + reverse_vocab, + chat_template: None, + } + } + + /// Extract special tokens from the tokenizer + fn extract_special_tokens(tokenizer: &HfTokenizer) -> SpecialTokens { + // Try to get special tokens from the tokenizer + // This is a simplified version - actual implementation would need to handle various formats + let vocab = tokenizer.get_vocab(true); + + let find_token = |patterns: &[&str]| -> Option { + for pattern in patterns { + if vocab.contains_key(*pattern) { + return Some(pattern.to_string()); + } + } + None + }; + + SpecialTokens { + bos_token: find_token(&["", "<|startoftext|>", "", "[CLS]"]), + eos_token: find_token(&["", "<|endoftext|>", "", "[SEP]"]), + unk_token: find_token(&["", "", "[UNK]"]), + sep_token: find_token(&["[SEP]", "", ""]), + pad_token: find_token(&["", "", "[PAD]"]), + cls_token: find_token(&["[CLS]", "", ""]), + mask_token: find_token(&["[MASK]", "", ""]), + additional_special_tokens: vec![], + } + } + + /// Try to load chat template from tokenizer_config.json + fn load_chat_template(tokenizer_path: &str) -> Option { + // Try to find tokenizer_config.json in the same directory + let path = std::path::Path::new(tokenizer_path); + let dir = path.parent()?; + let config_path = dir.join("tokenizer_config.json"); + + if config_path.exists() { + if let Ok(template) = + super::chat_template::load_chat_template_from_config(config_path.to_str()?) + { + return template; + } + } + None + } + + /// Load chat template from a .jinja file + fn load_chat_template_from_file(template_path: &str) -> Result> { + use std::fs; + + let content = fs::read_to_string(template_path) + .map_err(|e| Error::msg(format!("Failed to read chat template file: {}", e)))?; + + // Clean up the template (similar to Python implementation) + let template = content.trim().replace("\\n", "\n"); + + Ok(Some(template)) + } + + /// Set or override the chat template + pub fn set_chat_template(&mut self, template: String) { + self.chat_template = Some(template); + } + + /// Apply chat template if available + pub fn apply_chat_template( + &self, + messages: &[ChatMessage], + add_generation_prompt: bool, + ) -> Result { + if let Some(ref template) = self.chat_template { + let processor = ChatTemplateProcessor::new( + template.clone(), + self.special_tokens.bos_token.clone(), + self.special_tokens.eos_token.clone(), + ); + processor.apply_chat_template(messages, add_generation_prompt) + } else { + // Fallback to simple formatting if no template is available + let mut result = String::new(); + for msg in messages { + result.push_str(&format!("{}: {}\n", msg.role, msg.content)); + } + if add_generation_prompt { + result.push_str("assistant: "); + } + Ok(result) + } + } +} + +impl Encoder for HuggingFaceTokenizer { + fn encode(&self, input: &str) -> Result { + self.tokenizer + .encode(input, false) + .map_err(|e| Error::msg(format!("Encoding failed: {}", e))) + .map(|encoding| Encoding::Hf(Box::new(encoding))) + } + + fn encode_batch(&self, inputs: &[&str]) -> Result> { + let encodings = self + .tokenizer + .encode_batch(inputs.to_vec(), false) + .map_err(|e| Error::msg(format!("Batch encoding failed: {}", e)))?; + + Ok(encodings + .into_iter() + .map(|e| Encoding::Hf(Box::new(e))) + .collect()) + } +} + +impl Decoder for HuggingFaceTokenizer { + fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result { + self.tokenizer + .decode(token_ids, skip_special_tokens) + .map_err(|e| Error::msg(format!("Decoding failed: {}", e))) + } +} + +impl TokenizerTrait for HuggingFaceTokenizer { + fn vocab_size(&self) -> usize { + self.tokenizer.get_vocab_size(false) + } + + fn get_special_tokens(&self) -> &SpecialTokens { + &self.special_tokens + } + + fn token_to_id(&self, token: &str) -> Option { + self.vocab.get(token).copied() + } + + fn id_to_token(&self, id: TokenIdType) -> Option { + self.reverse_vocab.get(&id).cloned() + } +} + +#[cfg(test)] +mod tests { + use super::ChatMessage; + + #[test] + fn test_chat_message_creation() { + let msg = ChatMessage::system("You are a helpful assistant"); + assert_eq!(msg.role, "system"); + assert_eq!(msg.content, "You are a helpful assistant"); + + let user_msg = ChatMessage::user("Hello!"); + assert_eq!(user_msg.role, "user"); + + let assistant_msg = ChatMessage::assistant("Hi there!"); + assert_eq!(assistant_msg.role, "assistant"); + } + + // Note: Actual tokenizer tests would require a real tokenizer file + // These would be integration tests rather than unit tests +} diff --git a/sgl-router/src/tokenizer/mock.rs b/sgl-router/src/tokenizer/mock.rs new file mode 100644 index 000000000..afb91543c --- /dev/null +++ b/sgl-router/src/tokenizer/mock.rs @@ -0,0 +1,112 @@ +//! Mock tokenizer implementation for testing + +use super::traits::{Decoder, Encoder, Encoding, SpecialTokens, Tokenizer as TokenizerTrait}; +use anyhow::Result; +use std::collections::HashMap; + +/// Mock tokenizer for testing purposes +pub struct MockTokenizer { + vocab: HashMap, + reverse_vocab: HashMap, + special_tokens: SpecialTokens, +} + +impl Default for MockTokenizer { + fn default() -> Self { + Self::new() + } +} + +impl MockTokenizer { + pub fn new() -> Self { + let mut vocab = HashMap::new(); + let mut reverse_vocab = HashMap::new(); + + // Add some basic tokens + let tokens = vec![ + ("Hello", 1), + ("world", 2), + ("test", 3), + ("token", 4), + (" ", 5), + (".", 6), + ("", 999), + ("", 1000), + ]; + + for (token, id) in tokens { + vocab.insert(token.to_string(), id); + reverse_vocab.insert(id, token.to_string()); + } + + let special_tokens = SpecialTokens { + bos_token: Some("".to_string()), + eos_token: Some("".to_string()), + unk_token: Some("".to_string()), + sep_token: None, + pad_token: None, + cls_token: None, + mask_token: None, + additional_special_tokens: vec![], + }; + + Self { + vocab, + reverse_vocab, + special_tokens, + } + } +} + +impl Encoder for MockTokenizer { + fn encode(&self, input: &str) -> Result { + // Simple word-based tokenization for testing + let tokens: Vec = input + .split_whitespace() + .filter_map(|word| self.vocab.get(word).copied()) + .collect(); + + Ok(Encoding::Sp(tokens)) + } + + fn encode_batch(&self, inputs: &[&str]) -> Result> { + inputs.iter().map(|input| self.encode(input)).collect() + } +} + +impl Decoder for MockTokenizer { + fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { + let tokens: Vec = token_ids + .iter() + .filter_map(|id| { + self.reverse_vocab.get(id).and_then(|token| { + if skip_special_tokens && (token == "" || token == "") { + None + } else { + Some(token.clone()) + } + }) + }) + .collect(); + + Ok(tokens.join(" ")) + } +} + +impl TokenizerTrait for MockTokenizer { + fn vocab_size(&self) -> usize { + self.vocab.len() + } + + fn get_special_tokens(&self) -> &SpecialTokens { + &self.special_tokens + } + + fn token_to_id(&self, token: &str) -> Option { + self.vocab.get(token).copied() + } + + fn id_to_token(&self, id: u32) -> Option { + self.reverse_vocab.get(&id).cloned() + } +} diff --git a/sgl-router/src/tokenizer/mod.rs b/sgl-router/src/tokenizer/mod.rs new file mode 100644 index 000000000..98a23f761 --- /dev/null +++ b/sgl-router/src/tokenizer/mod.rs @@ -0,0 +1,123 @@ +use anyhow::Result; +use std::ops::Deref; +use std::sync::Arc; + +pub mod factory; +pub mod hub; +pub mod mock; +pub mod sequence; +pub mod stop; +pub mod stream; +pub mod traits; + +// Feature-gated modules + +pub mod chat_template; + +pub mod huggingface; + +pub mod tiktoken; + +#[cfg(test)] +mod tests; + +// Re-exports +pub use factory::{ + create_tokenizer, create_tokenizer_async, create_tokenizer_from_file, + create_tokenizer_with_chat_template, TokenizerType, +}; +pub use sequence::Sequence; +pub use stop::{SequenceDecoderOutput, StopSequenceConfig, StopSequenceDecoder}; +pub use stream::DecodeStream; +pub use traits::{Decoder, Encoder, Encoding, SpecialTokens, Tokenizer as TokenizerTrait}; + +pub use huggingface::HuggingFaceTokenizer; + +pub use chat_template::ChatMessage; + +pub use tiktoken::{TiktokenModel, TiktokenTokenizer}; + +/// Main tokenizer wrapper that provides a unified interface for different tokenizer implementations +#[derive(Clone)] +pub struct Tokenizer(Arc); + +impl Tokenizer { + /// Create a tokenizer from a file path + pub fn from_file(file_path: &str) -> Result { + Ok(Tokenizer(factory::create_tokenizer_from_file(file_path)?)) + } + + /// Create a tokenizer from a file path with an optional chat template + pub fn from_file_with_chat_template( + file_path: &str, + chat_template_path: Option<&str>, + ) -> Result { + Ok(Tokenizer(factory::create_tokenizer_with_chat_template( + file_path, + chat_template_path, + )?)) + } + + /// Create a tokenizer from an Arc + pub fn from_arc(tokenizer: Arc) -> Self { + Tokenizer(tokenizer) + } + + /// Create a stateful sequence object for decoding token_ids into text + pub fn decode_stream( + &self, + prompt_token_ids: &[u32], + skip_special_tokens: bool, + ) -> DecodeStream { + DecodeStream::new(self.0.clone(), prompt_token_ids, skip_special_tokens) + } + + /// Direct encode method + pub fn encode(&self, input: &str) -> Result { + self.0.encode(input) + } + + /// Direct batch encode method + pub fn encode_batch(&self, inputs: &[&str]) -> Result> { + self.0.encode_batch(inputs) + } + + /// Direct decode method + pub fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result { + self.0.decode(token_ids, skip_special_tokens) + } + + /// Get vocabulary size + pub fn vocab_size(&self) -> usize { + self.0.vocab_size() + } + + /// Get special tokens + pub fn get_special_tokens(&self) -> &SpecialTokens { + self.0.get_special_tokens() + } + + /// Convert token string to ID + pub fn token_to_id(&self, token: &str) -> Option { + self.0.token_to_id(token) + } + + /// Convert ID to token string + pub fn id_to_token(&self, id: u32) -> Option { + self.0.id_to_token(id) + } +} + +impl Deref for Tokenizer { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From> for Tokenizer { + fn from(tokenizer: Arc) -> Self { + Tokenizer(tokenizer) + } +} diff --git a/sgl-router/src/tokenizer/sequence.rs b/sgl-router/src/tokenizer/sequence.rs new file mode 100644 index 000000000..99801438d --- /dev/null +++ b/sgl-router/src/tokenizer/sequence.rs @@ -0,0 +1,238 @@ +use super::traits::{TokenIdType, Tokenizer as TokenizerTrait}; +use anyhow::Result; +use std::sync::Arc; + +/// Maintains state for an ongoing sequence of tokens and their decoded text +/// This provides a cleaner abstraction for managing token sequences +pub struct Sequence { + /// The tokenizer used for encoding/decoding + tokenizer: Arc, + + /// The current sequence of token ids + token_ids: Vec, + + /// The position in the current sequence the last decoded token completed + prefix_offset: usize, + + /// Current position in the sequence + read_offset: usize, +} + +impl std::fmt::Debug for Sequence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Sequence") + .field("tokenizer", &"Arc") + .field( + "token_ids", + &format_args!("{}", { + let token_ids = self.token_ids(); + if token_ids.len() <= 20 { + format!("{:?}", token_ids) + } else { + let first_ten = &token_ids[..10]; + let last_ten = &token_ids[token_ids.len() - 10..]; + format!("{:?} ... {:?}", first_ten, last_ten) + } + }), + ) + .field("prefix_offset", &self.prefix_offset) + .field("read_offset", &self.read_offset) + .field("token count", &self.token_ids.len()) + .finish() + } +} + +impl Sequence { + /// Create a new empty sequence + pub fn new(tokenizer: Arc) -> Self { + Self { + tokenizer, + token_ids: Vec::new(), + prefix_offset: 0, + read_offset: 0, + } + } + + /// Create a sequence with initial tokens + pub fn with_tokens(tokenizer: Arc, token_ids: Vec) -> Self { + let len = token_ids.len(); + Self { + tokenizer, + token_ids, + prefix_offset: 0, + read_offset: len, + } + } + + /// Check if the sequence is empty + pub fn is_empty(&self) -> bool { + self.token_ids.is_empty() + } + + /// Get the length of the sequence + pub fn len(&self) -> usize { + self.token_ids.len() + } + + /// Clear the sequence + pub fn clear(&mut self) { + self.token_ids.clear(); + self.prefix_offset = 0; + self.read_offset = 0; + } + + /// Append text to the sequence by encoding it + pub fn append_text(&mut self, input: &str) -> Result<()> { + let encoding = self.tokenizer.encode(input)?; + self.token_ids.extend(encoding.token_ids()); + Ok(()) + } + + /// Append a single token to the sequence and return newly decoded text + /// Based on HuggingFace TGI incremental decoding + pub fn append_token(&mut self, token_id: TokenIdType) -> Result { + // Store the old read offset before adding the new token + let old_read_offset = self.read_offset; + + self.token_ids.push(token_id); + self.read_offset = self.token_ids.len(); + + // If this is the first token or we're at the beginning, decode everything + if self.prefix_offset == 0 && old_read_offset == 0 { + let text = self.tokenizer.decode(&self.token_ids, false)?; + if text.ends_with("�") { + // Incomplete UTF-8 sequence, wait for more tokens + return Ok(String::new()); + } + self.prefix_offset = 0; + return Ok(text); + } + + // Decode the text up to the previous position + let prefix_text = self + .tokenizer + .decode(&self.token_ids[self.prefix_offset..old_read_offset], false)?; + + // Decode the text including the new token + let new_text = self + .tokenizer + .decode(&self.token_ids[self.prefix_offset..], false)?; + + // Handle multi-byte character boundaries + let mut prefix_text_len = prefix_text.len(); + while !new_text.is_char_boundary(prefix_text_len) && prefix_text_len > 0 { + prefix_text_len -= 1; + } + + if new_text.len() > prefix_text.len() { + if new_text.ends_with("�") { + // Incomplete UTF-8 sequence, wait for more tokens + return Ok(String::new()); + } else { + // Return the new text portion + let incremental_text = new_text[prefix_text_len..].to_string().replace("�", ""); + self.prefix_offset = old_read_offset; + return Ok(incremental_text); + } + } + + Ok(String::new()) + } + + /// Get a reference to the tokenizer + pub fn tokenizer(&self) -> &Arc { + &self.tokenizer + } + + /// Get the current token ids + pub fn token_ids(&self) -> &[TokenIdType] { + &self.token_ids + } + + /// Decode the entire sequence to text + pub fn text(&self) -> Result { + self.tokenizer.decode(&self.token_ids, false) + } + + /// Get the prefix offset + pub fn prefix_offset(&self) -> usize { + self.prefix_offset + } + + /// Get the read offset + pub fn read_offset(&self) -> usize { + self.read_offset + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tokenizer::mock::MockTokenizer; + + #[test] + fn test_sequence_new() { + let tokenizer = Arc::new(MockTokenizer::new()); + let seq = Sequence::new(tokenizer); + assert!(seq.is_empty()); + assert_eq!(seq.len(), 0); + } + + #[test] + fn test_sequence_append_text() { + let tokenizer = Arc::new(MockTokenizer::new()); + let mut seq = Sequence::new(tokenizer); + + seq.append_text("Hello").unwrap(); + assert!(!seq.is_empty()); + assert!(!seq.is_empty()); + + let text = seq.text().unwrap(); + assert_eq!(text, "Hello"); + } + + #[test] + fn test_sequence_append_token() { + let tokenizer = Arc::new(MockTokenizer::new()); + let mut seq = Sequence::new(tokenizer.clone()); + + // Start with an empty sequence and append token 1 ("Hello") + let text1 = seq.append_token(1).unwrap(); + assert_eq!(text1, "Hello"); + + // Now append token 2 ("world") + // The mock tokenizer will decode [1, 2] as "Hello world" (with a space) + let text2 = seq.append_token(2).unwrap(); + // The incremental text should be " world" (with the space that the mock tokenizer adds) + assert_eq!(text2, " world"); + + // Verify the full text + assert_eq!(seq.text().unwrap(), "Hello world"); + } + + #[test] + fn test_sequence_clear() { + let tokenizer = Arc::new(MockTokenizer::new()); + let mut seq = Sequence::new(tokenizer); + + seq.append_text("Hello world").unwrap(); + assert!(!seq.is_empty()); + + seq.clear(); + assert!(seq.is_empty()); + assert_eq!(seq.len(), 0); + assert_eq!(seq.prefix_offset(), 0); + assert_eq!(seq.read_offset(), 0); + } + + #[test] + fn test_sequence_debug() { + let tokenizer = Arc::new(MockTokenizer::new()); + let mut seq = Sequence::new(tokenizer); + + seq.append_text("Test").unwrap(); + let debug_str = format!("{:?}", seq); + assert!(debug_str.contains("Sequence")); + assert!(debug_str.contains("token count")); + } +} diff --git a/sgl-router/src/tokenizer/stop.rs b/sgl-router/src/tokenizer/stop.rs new file mode 100644 index 000000000..1efda15b6 --- /dev/null +++ b/sgl-router/src/tokenizer/stop.rs @@ -0,0 +1,506 @@ +use super::traits::{self, TokenIdType}; +use anyhow::Result; +use std::collections::HashSet; +use std::sync::Arc; + +/// Output from the sequence decoder +#[derive(Debug, Clone, PartialEq)] +pub enum SequenceDecoderOutput { + /// Normal text output + Text(String), + /// Text is being held due to partial stop sequence match + Held, + /// Stop sequence matched (hidden - not included in output) + Stopped, + /// Stop sequence matched with text (visible - included in output) + StoppedWithText(String), +} + +/// Configuration for stop sequences +#[derive(Debug, Clone, Default)] +pub struct StopSequenceConfig { + /// Token IDs that trigger a stop + pub stop_tokens: HashSet, + /// String sequences that trigger a stop + pub stop_sequences: Vec, + /// Token IDs for visible stops (included in output) + pub visible_stop_tokens: HashSet, + /// String sequences for visible stops (included in output) + pub visible_stop_sequences: Vec, +} + +impl StopSequenceConfig { + /// Builder pattern - add a stop token + pub fn with_stop_token(mut self, token_id: TokenIdType) -> Self { + self.stop_tokens.insert(token_id); + self + } + + /// Builder pattern - add a stop sequence + pub fn with_stop_sequence(mut self, sequence: impl Into) -> Self { + self.stop_sequences.push(sequence.into()); + self + } + + /// Builder pattern - add a visible stop token + pub fn with_visible_stop_token(mut self, token_id: TokenIdType) -> Self { + self.visible_stop_tokens.insert(token_id); + self + } + + /// Builder pattern - add a visible stop sequence + pub fn with_visible_stop_sequence(mut self, sequence: impl Into) -> Self { + self.visible_stop_sequences.push(sequence.into()); + self + } +} + +/// Decoder that handles stop sequences +pub struct StopSequenceDecoder { + tokenizer: Arc, + config: StopSequenceConfig, + /// Buffer for partial matches (the "jail") + jail_buffer: String, + /// Accumulated tokens + token_buffer: Vec, + /// Offset where the prefix text starts (for context) + prefix_offset: usize, + /// Offset marking the end of previously decoded text + read_offset: usize, + /// Whether we've stopped + stopped: bool, + skip_special_tokens: bool, +} + +impl StopSequenceDecoder { + /// Create a new stop sequence decoder + pub fn new( + tokenizer: Arc, + config: StopSequenceConfig, + skip_special_tokens: bool, + ) -> Self { + StopSequenceDecoder { + tokenizer, + config, + jail_buffer: String::new(), + token_buffer: Vec::new(), + prefix_offset: 0, + read_offset: 0, + stopped: false, + skip_special_tokens, + } + } + + /// Process a single token + pub fn process_token(&mut self, token_id: TokenIdType) -> Result { + if self.stopped { + return Ok(SequenceDecoderOutput::Stopped); + } + + // Check for token-level stops first + if self.config.stop_tokens.contains(&token_id) { + self.stopped = true; + + // Flush any jailed text before stopping + if !self.jail_buffer.is_empty() { + let output = self.jail_buffer.clone(); + self.jail_buffer.clear(); + return Ok(SequenceDecoderOutput::StoppedWithText(output)); + } + return Ok(SequenceDecoderOutput::Stopped); + } + + if self.config.visible_stop_tokens.contains(&token_id) { + self.stopped = true; + + // Include jailed text plus the stop token + let stop_text = self + .tokenizer + .decode(&[token_id], self.skip_special_tokens)?; + let output = format!("{}{}", self.jail_buffer, stop_text); + self.jail_buffer.clear(); + return Ok(SequenceDecoderOutput::StoppedWithText(output)); + } + + // Add token to buffer + self.token_buffer.push(token_id); + + // Use incremental decoding like DecodeStream + // First decode the previous context (what we've already output) + let prefix_text = if self.read_offset > self.prefix_offset { + self.tokenizer.decode( + &self.token_buffer[self.prefix_offset..self.read_offset], + self.skip_special_tokens, + )? + } else { + String::new() + }; + + // Now decode from prefix to current position + let new_full_text = self.tokenizer.decode( + &self.token_buffer[self.prefix_offset..], + self.skip_special_tokens, + )?; + + // Check for incomplete UTF-8 sequence + if new_full_text.ends_with("�") { + // Wait for more tokens to complete the sequence + return Ok(SequenceDecoderOutput::Held); + } + + // Calculate only the NEW text since last successful decode + let new_text = if new_full_text.len() > prefix_text.len() { + &new_full_text[prefix_text.len()..] + } else { + // No new text produced (can happen with special tokens) + return Ok(SequenceDecoderOutput::Held); + }; + + // Combine jail buffer with new text for checking + let check_text = format!("{}{}", self.jail_buffer, new_text); + + // Check for complete stop sequences + for stop_seq in &self.config.stop_sequences { + if let Some(pos) = check_text.find(stop_seq) { + self.stopped = true; + + // Output text before the stop sequence + let output = check_text[..pos].to_string(); + self.jail_buffer.clear(); + return Ok(if output.is_empty() { + SequenceDecoderOutput::Stopped + } else { + SequenceDecoderOutput::StoppedWithText(output) + }); + } + } + + // Check for visible stop sequences + for stop_seq in &self.config.visible_stop_sequences { + if let Some(pos) = check_text.find(stop_seq) { + self.stopped = true; + + // Include the stop sequence in output + let end_pos = pos + stop_seq.len(); + let output = check_text[..end_pos].to_string(); + self.jail_buffer.clear(); + return Ok(SequenceDecoderOutput::StoppedWithText(output)); + } + } + + // Check for partial matches at the end of check_text + let mut partial_match_len = 0; + for stop_seq in self + .config + .stop_sequences + .iter() + .chain(&self.config.visible_stop_sequences) + { + // Check all possible suffixes that could be a prefix of stop_seq + for i in 1..=check_text.len().min(stop_seq.len() - 1) { + let suffix = &check_text[check_text.len() - i..]; + if stop_seq.starts_with(suffix) { + partial_match_len = partial_match_len.max(i); + } + } + } + + if partial_match_len > 0 { + // Split: output safe text, jail the potential match + let safe_end = check_text.len() - partial_match_len; + let safe_text = &check_text[..safe_end]; + self.jail_buffer = check_text[safe_end..].to_string(); + + // Update offsets for next iteration + self.prefix_offset = self.read_offset; + self.read_offset = self.token_buffer.len(); + + if safe_text.is_empty() { + Ok(SequenceDecoderOutput::Held) + } else { + Ok(SequenceDecoderOutput::Text(safe_text.to_string())) + } + } else { + // No partial matches - output everything + self.jail_buffer.clear(); + + // Update offsets for next iteration + self.prefix_offset = self.read_offset; + self.read_offset = self.token_buffer.len(); + + Ok(SequenceDecoderOutput::Text(check_text)) + } + } + + /// Process multiple tokens + pub fn process_tokens( + &mut self, + token_ids: &[TokenIdType], + ) -> Result> { + let mut outputs = Vec::new(); + for &token_id in token_ids { + outputs.push(self.process_token(token_id)?); + } + Ok(outputs) + } + + /// Flush any held text + pub fn flush(&mut self) -> SequenceDecoderOutput { + if !self.jail_buffer.is_empty() { + let output = self.jail_buffer.clone(); + self.jail_buffer.clear(); + SequenceDecoderOutput::Text(output) + } else { + SequenceDecoderOutput::Text(String::new()) + } + } + + /// Check if decoding has stopped + pub fn is_stopped(&self) -> bool { + self.stopped + } + + /// Reset the decoder state + pub fn reset(&mut self) { + self.jail_buffer.clear(); + self.token_buffer.clear(); + self.prefix_offset = 0; + self.read_offset = 0; + self.stopped = false; + } +} + +/// Builder for StopSequenceDecoder +pub struct StopSequenceDecoderBuilder { + tokenizer: Arc, + config: StopSequenceConfig, + skip_special_tokens: bool, +} + +impl StopSequenceDecoderBuilder { + pub fn new(tokenizer: Arc) -> Self { + StopSequenceDecoderBuilder { + tokenizer, + config: StopSequenceConfig::default(), + skip_special_tokens: true, + } + } + + pub fn stop_token(mut self, token_id: TokenIdType) -> Self { + self.config.stop_tokens.insert(token_id); + self + } + + pub fn stop_sequence(mut self, sequence: impl Into) -> Self { + self.config.stop_sequences.push(sequence.into()); + self + } + + pub fn visible_stop_token(mut self, token_id: TokenIdType) -> Self { + self.config.visible_stop_tokens.insert(token_id); + self + } + + pub fn visible_stop_sequence(mut self, sequence: impl Into) -> Self { + self.config.visible_stop_sequences.push(sequence.into()); + self + } + + pub fn skip_special_tokens(mut self, skip: bool) -> Self { + self.skip_special_tokens = skip; + self + } + + pub fn build(self) -> StopSequenceDecoder { + StopSequenceDecoder::new(self.tokenizer, self.config, self.skip_special_tokens) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tokenizer::mock::MockTokenizer; + + #[test] + fn test_stop_token_detection() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_stop_token(999); // token + + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process tokens before stop + let result = decoder.process_token(1).unwrap(); // "Hello" + assert!(matches!(result, SequenceDecoderOutput::Text(_))); + + // Process stop token + let result = decoder.process_token(999).unwrap(); // + assert_eq!(result, SequenceDecoderOutput::Stopped); + + // Further tokens should also return Stopped + let result = decoder.process_token(2).unwrap(); + assert_eq!(result, SequenceDecoderOutput::Stopped); + } + + #[test] + fn test_visible_stop_token() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_visible_stop_token(999); + + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + let result = decoder.process_token(999).unwrap(); + assert!(matches!(result, SequenceDecoderOutput::StoppedWithText(_))); + } + + #[test] + fn test_builder_pattern() { + let tokenizer = Arc::new(MockTokenizer::new()); + + let decoder = StopSequenceDecoderBuilder::new(tokenizer) + .stop_token(999) + .stop_sequence("STOP") + .visible_stop_token(1000) + .skip_special_tokens(true) + .build(); + + assert!(!decoder.is_stopped()); + } + + #[test] + fn test_incremental_decoding_no_repetition() { + // This test verifies the critical fix: no repeated output + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default(); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process tokens one by one and collect outputs + let mut outputs = Vec::new(); + + // Token 1: "Hello" + let result = decoder.process_token(1).unwrap(); + if let SequenceDecoderOutput::Text(text) = result { + outputs.push(text.clone()); + } + + // Token 2: "world" + let result = decoder.process_token(2).unwrap(); + if let SequenceDecoderOutput::Text(text) = result { + outputs.push(text.clone()); + } + + // Token 3: "test" + let result = decoder.process_token(3).unwrap(); + if let SequenceDecoderOutput::Text(text) = result { + outputs.push(text.clone()); + } + + // CRITICAL: Each output should be unique (no accumulation) + // The fix ensures we only output NEW text, not accumulated text + assert_eq!(outputs.len(), 3); + + // Verify no text is repeated + for i in 0..outputs.len() { + for j in i + 1..outputs.len() { + // No output should contain another (no accumulation) + assert!(!outputs[j].contains(&outputs[i])); + } + } + } + + #[test] + fn test_stop_sequence_detection() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_stop_sequence("test"); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process "Hello world" + decoder.process_token(1).unwrap(); // "Hello" + decoder.process_token(2).unwrap(); // "world" + + // Process "test" which should trigger stop + let result = decoder.process_token(3).unwrap(); // "test" + + // Should stop when we hit "test" + assert!(matches!( + result, + SequenceDecoderOutput::Stopped | SequenceDecoderOutput::StoppedWithText(_) + )); + } + + #[test] + fn test_flush_after_partial() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_stop_sequence("NEVER_MATCH"); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process a token + decoder.process_token(1).unwrap(); // "Hello" + + // Flush should return any remaining text in jail + let result = decoder.flush(); + + // After processing, flush should work + assert!(matches!(result, SequenceDecoderOutput::Text(_))); + } + + #[test] + fn test_reset_functionality() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_stop_token(999); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process and stop + decoder.process_token(1).unwrap(); + decoder.process_token(999).unwrap(); + assert!(decoder.is_stopped()); + + // Reset should clear everything + decoder.reset(); + assert!(!decoder.is_stopped()); + + // Should be able to process again + let result = decoder.process_token(2).unwrap(); + assert!(matches!(result, SequenceDecoderOutput::Text(_))); + } + + #[test] + fn test_visible_stop_sequence() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default().with_visible_stop_sequence("world"); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process "Hello" + decoder.process_token(1).unwrap(); + + // Process "world" - should include it in output + let result = decoder.process_token(2).unwrap(); + + if let SequenceDecoderOutput::StoppedWithText(text) = result { + // Should include "world" in the output + assert!(text.contains("world")); + } else { + panic!("Expected StoppedWithText with visible stop sequence"); + } + } + + #[test] + fn test_multiple_tokens_processing() { + let tokenizer = Arc::new(MockTokenizer::new()); + let config = StopSequenceConfig::default(); + let mut decoder = StopSequenceDecoder::new(tokenizer, config, false); + + // Process multiple tokens at once + let results = decoder.process_tokens(&[1, 2, 3]).unwrap(); + + // Should get results for each token + assert_eq!(results.len(), 3); + + // Each result should be Text (no stops configured) + for result in results { + assert!(matches!( + result, + SequenceDecoderOutput::Text(_) | SequenceDecoderOutput::Held + )); + } + } +} diff --git a/sgl-router/src/tokenizer/stream.rs b/sgl-router/src/tokenizer/stream.rs new file mode 100644 index 000000000..848be8a8c --- /dev/null +++ b/sgl-router/src/tokenizer/stream.rs @@ -0,0 +1,105 @@ +// src/tokenizer/stream.rs + +use super::traits::{self, TokenIdType}; +use anyhow::Result; +use std::sync::Arc; + +const INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET: usize = 5; + +/// DecodeStream will keep the state necessary to produce individual chunks of +/// strings given an input stream of token_ids +pub struct DecodeStream { + /// The tokenizer used to decode token_ids + tokenizer: Arc, + + skip_special_tokens: bool, + + /// A temporary buffer of the necessary token_ids needed + /// to produce valid string chunks + all_token_ids: Vec, + + prefix_offset: usize, + read_offset: usize, +} + +impl DecodeStream { + pub fn new( + tokenizer: Arc, + prompt_token_ids: &[TokenIdType], + skip_special_tokens: bool, + ) -> Self { + let num_input_tokens = prompt_token_ids.len(); + let prompt_token_ids = prompt_token_ids.to_vec(); + Self { + tokenizer, + skip_special_tokens, + all_token_ids: prompt_token_ids, + prefix_offset: num_input_tokens + .saturating_sub(INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET), + read_offset: num_input_tokens, + } + } + + /// Step appends a token_id to the internal state and tries to produce a text chunk. + /// Returning `None` means the given id is not enough to produce a chunk. + pub fn step(&mut self, id: TokenIdType) -> Result> { + self.all_token_ids.push(id); + + let prefix_text = self.tokenizer.decode( + &self.all_token_ids[self.prefix_offset..self.read_offset], + self.skip_special_tokens, + )?; + + let new_text = self.tokenizer.decode( + &self.all_token_ids[self.prefix_offset..], + self.skip_special_tokens, + )?; + + if new_text.len() > prefix_text.len() && !new_text.ends_with("�") { + let new_text = new_text[prefix_text.len()..].to_string(); + + self.prefix_offset = self.read_offset; + self.read_offset = self.all_token_ids.len(); + + Ok(Some(new_text)) + } else { + Ok(None) + } + } + + /// Process multiple tokens at once + pub fn step_batch(&mut self, token_ids: &[u32]) -> Result> { + let mut chunks = Vec::new(); + + for &token_id in token_ids { + if let Some(text) = self.step(token_id)? { + chunks.push(text); + } + } + + Ok(chunks) + } + + /// Force flush any remaining text + pub fn flush(&mut self) -> Result> { + if self.read_offset < self.all_token_ids.len() { + let remaining = self.tokenizer.decode( + &self.all_token_ids[self.read_offset..], + self.skip_special_tokens, + )?; + + self.read_offset = self.all_token_ids.len(); + + if !remaining.is_empty() { + return Ok(Some(remaining)); + } + } + + Ok(None) + } + + /// Get all tokens processed so far + pub fn tokens(&self) -> &[u32] { + &self.all_token_ids + } +} diff --git a/sgl-router/src/tokenizer/tests.rs b/sgl-router/src/tokenizer/tests.rs new file mode 100644 index 000000000..2c4d4b108 --- /dev/null +++ b/sgl-router/src/tokenizer/tests.rs @@ -0,0 +1,143 @@ +#[cfg(test)] +use super::*; +#[cfg(test)] +use std::sync::Arc; + +#[test] +fn test_mock_tokenizer_encode() { + let tokenizer = mock::MockTokenizer::new(); + let encoding = tokenizer.encode("Hello world").unwrap(); + let token_ids = encoding.token_ids(); + assert_eq!(token_ids, &[1, 2]); // "Hello" -> 1, "world" -> 2 +} + +#[test] +fn test_mock_tokenizer_decode() { + let tokenizer = mock::MockTokenizer::new(); + let text = tokenizer.decode(&[1, 2], false).unwrap(); + assert_eq!(text, "Hello world"); +} + +#[test] +fn test_mock_tokenizer_decode_skip_special() { + let tokenizer = mock::MockTokenizer::new(); + + // With special tokens + let text = tokenizer.decode(&[1000, 1, 2, 999], false).unwrap(); + assert_eq!(text, " Hello world "); + + // Without special tokens + let text = tokenizer.decode(&[1000, 1, 2, 999], true).unwrap(); + assert_eq!(text, "Hello world"); +} + +#[test] +fn test_tokenizer_wrapper() { + let mock_tokenizer = Arc::new(mock::MockTokenizer::new()); + let tokenizer = Tokenizer::from_arc(mock_tokenizer); + + // Test encoding + let encoding = tokenizer.encode("Hello world").unwrap(); + assert_eq!(encoding.token_ids(), &[1, 2]); + + // Test decoding + let text = tokenizer.decode(&[1, 2], false).unwrap(); + assert_eq!(text, "Hello world"); + + // Test vocab size + assert_eq!(tokenizer.vocab_size(), 8); + + // Test token to ID + assert_eq!(tokenizer.token_to_id("Hello"), Some(1)); + assert_eq!(tokenizer.token_to_id("unknown"), None); + + // Test ID to token + assert_eq!(tokenizer.id_to_token(1), Some("Hello".to_string())); + assert_eq!(tokenizer.id_to_token(9999), None); +} + +#[test] +fn test_decode_stream_basic() { + let mock_tokenizer = Arc::new(mock::MockTokenizer::new()); + let tokenizer = Tokenizer::from_arc(mock_tokenizer); + + // Create a decode stream with initial tokens + let initial_tokens = vec![1, 2]; // "Hello world" + let mut stream = tokenizer.decode_stream(&initial_tokens, false); + + // Add a new token + let result = stream.step(3).unwrap(); // "test" + // Since we're using a mock, the actual incremental behavior depends on implementation + // For now, we just verify it doesn't crash + assert!(result.is_some() || result.is_none()); +} + +#[test] +fn test_decode_stream_flush() { + let mock_tokenizer = Arc::new(mock::MockTokenizer::new()); + let tokenizer = Tokenizer::from_arc(mock_tokenizer); + + let initial_tokens = vec![1]; + let mut stream = tokenizer.decode_stream(&initial_tokens, false); + + // Add tokens + stream.step(2).unwrap(); + stream.step(3).unwrap(); + + // Flush remaining + let flushed = stream.flush().unwrap(); + // The flush behavior depends on the implementation + assert!(flushed.is_some() || flushed.is_none()); +} + +#[test] +fn test_special_tokens() { + let mock_tokenizer = Arc::new(mock::MockTokenizer::new()); + let tokenizer = Tokenizer::from_arc(mock_tokenizer); + + let special_tokens = tokenizer.get_special_tokens(); + assert_eq!(special_tokens.bos_token, Some("".to_string())); + assert_eq!(special_tokens.eos_token, Some("".to_string())); + assert_eq!(special_tokens.unk_token, Some("".to_string())); + assert!(special_tokens.sep_token.is_none()); + assert!(special_tokens.pad_token.is_none()); +} + +#[test] +fn test_batch_encode() { + let tokenizer = mock::MockTokenizer::new(); + let inputs = vec!["Hello", "world", "test"]; + let encodings = tokenizer.encode_batch(&inputs).unwrap(); + + assert_eq!(encodings.len(), 3); + assert_eq!(encodings[0].token_ids(), &[1]); // "Hello" -> 1 + assert_eq!(encodings[1].token_ids(), &[2]); // "world" -> 2 + assert_eq!(encodings[2].token_ids(), &[3]); // "test" -> 3 +} + +#[test] +fn test_thread_safety() { + use std::thread; + + let mock_tokenizer = Arc::new(mock::MockTokenizer::new()); + let tokenizer = Tokenizer::from_arc(mock_tokenizer); + + // Spawn multiple threads that use the same tokenizer + let handles: Vec<_> = (0..10) + .map(|i| { + let tokenizer_clone = tokenizer.clone(); + thread::spawn(move || { + let text = "Hello test".to_string(); + let encoding = tokenizer_clone.encode(&text).unwrap(); + let decoded = tokenizer_clone.decode(encoding.token_ids(), false).unwrap(); + assert!(decoded.contains("Hello") || decoded.contains("test")); + i + }) + }) + .collect(); + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } +} diff --git a/sgl-router/src/tokenizer/tiktoken.rs b/sgl-router/src/tokenizer/tiktoken.rs new file mode 100644 index 000000000..0af5a9791 --- /dev/null +++ b/sgl-router/src/tokenizer/tiktoken.rs @@ -0,0 +1,276 @@ +use super::traits::{ + Decoder, Encoder, Encoding, SpecialTokens, TokenIdType, Tokenizer as TokenizerTrait, +}; +use anyhow::{Error, Result}; +use tiktoken_rs::{cl100k_base, p50k_base, p50k_edit, r50k_base, CoreBPE}; + +/// Tiktoken tokenizer wrapper for OpenAI GPT models +pub struct TiktokenTokenizer { + tokenizer: CoreBPE, + #[allow(dead_code)] + model: TiktokenModel, + special_tokens: SpecialTokens, + vocab_size: usize, +} + +/// Supported Tiktoken models +#[derive(Debug, Clone, Copy)] +pub enum TiktokenModel { + /// GPT-4, GPT-3.5-turbo, text-embedding-ada-002 + Cl100kBase, + /// Codex models, text-davinci-002, text-davinci-003 + P50kBase, + /// Use for edit models like text-davinci-edit-001, code-davinci-edit-001 + P50kEdit, + /// GPT-3 models like davinci + R50kBase, +} + +impl TiktokenTokenizer { + /// Create a new Tiktoken tokenizer for the specified model + pub fn new(model: TiktokenModel) -> Result { + let tokenizer = + match model { + TiktokenModel::Cl100kBase => cl100k_base() + .map_err(|e| Error::msg(format!("Failed to load cl100k_base: {}", e)))?, + TiktokenModel::P50kBase => p50k_base() + .map_err(|e| Error::msg(format!("Failed to load p50k_base: {}", e)))?, + TiktokenModel::P50kEdit => p50k_edit() + .map_err(|e| Error::msg(format!("Failed to load p50k_edit: {}", e)))?, + TiktokenModel::R50kBase => r50k_base() + .map_err(|e| Error::msg(format!("Failed to load r50k_base: {}", e)))?, + }; + + // Extract special tokens (tiktoken-rs doesn't expose them directly) + // We'll use common ones for GPT models + let special_tokens = Self::get_special_tokens_for_model(model); + + // Get vocabulary size (this is an approximation) + let vocab_size = match model { + TiktokenModel::Cl100kBase => 100256, // cl100k has ~100k tokens + TiktokenModel::P50kBase | TiktokenModel::P50kEdit => 50281, // p50k has ~50k tokens + TiktokenModel::R50kBase => 50257, // r50k has ~50k tokens + }; + + Ok(TiktokenTokenizer { + tokenizer, + model, + special_tokens, + vocab_size, + }) + } + + /// Create a tokenizer from a model string (e.g., "gpt-4", "gpt-3.5-turbo") + pub fn from_model_name(model_name: &str) -> Result { + let model = Self::model_from_name(model_name)?; + Self::new(model) + } + + /// Determine the appropriate model from a model name + fn model_from_name(model_name: &str) -> Result { + // Based on OpenAI's model-to-encoding mapping + if model_name.contains("gpt-4") + || model_name.contains("gpt-3.5") + || model_name.contains("turbo") + { + Ok(TiktokenModel::Cl100kBase) + } else if model_name.contains("davinci-002") + || model_name.contains("davinci-003") + || model_name.contains("codex") + { + Ok(TiktokenModel::P50kBase) + } else if model_name.contains("edit") { + Ok(TiktokenModel::P50kEdit) + } else if model_name.contains("davinci") + || model_name.contains("curie") + || model_name.contains("babbage") + || model_name.contains("ada") + { + Ok(TiktokenModel::R50kBase) + } else { + // Return an error for unrecognized model names to prevent silent failures + Err(anyhow::anyhow!( + "Unrecognized OpenAI model name: '{}'. Expected GPT-3, GPT-3.5, GPT-4, or related model names", + model_name + )) + } + } + + /// Get special tokens for a specific model + fn get_special_tokens_for_model(model: TiktokenModel) -> SpecialTokens { + // These are common special tokens for GPT models + // The actual token IDs might vary by model + match model { + TiktokenModel::Cl100kBase => SpecialTokens { + bos_token: Some("<|endoftext|>".to_string()), + eos_token: Some("<|endoftext|>".to_string()), + unk_token: None, + sep_token: None, + pad_token: Some("<|endoftext|>".to_string()), + cls_token: None, + mask_token: None, + additional_special_tokens: vec![ + "<|fim_prefix|>".to_string(), + "<|fim_middle|>".to_string(), + "<|fim_suffix|>".to_string(), + "<|endofprompt|>".to_string(), + ], + }, + _ => SpecialTokens { + bos_token: Some("<|endoftext|>".to_string()), + eos_token: Some("<|endoftext|>".to_string()), + unk_token: None, + sep_token: None, + pad_token: Some("<|endoftext|>".to_string()), + cls_token: None, + mask_token: None, + additional_special_tokens: vec![], + }, + } + } +} + +impl Encoder for TiktokenTokenizer { + fn encode(&self, input: &str) -> Result { + let tokens = self.tokenizer.encode_ordinary(input); + Ok(Encoding::Tiktoken(tokens)) + } + + fn encode_batch(&self, inputs: &[&str]) -> Result> { + inputs.iter().map(|input| self.encode(input)).collect() + } +} + +impl Decoder for TiktokenTokenizer { + fn decode(&self, token_ids: &[TokenIdType], _skip_special_tokens: bool) -> Result { + // tiktoken-rs 0.7.0 now uses u32 (Rank type) + self.tokenizer + .decode(token_ids.to_vec()) + .map_err(|e| Error::msg(format!("Decoding failed: {}", e))) + } +} + +impl TokenizerTrait for TiktokenTokenizer { + fn vocab_size(&self) -> usize { + self.vocab_size + } + + fn get_special_tokens(&self) -> &SpecialTokens { + &self.special_tokens + } + + fn token_to_id(&self, _token: &str) -> Option { + // Tiktoken doesn't provide direct token-to-id mapping + // We'd need to encode the token and check if it produces a single ID + None + } + + fn id_to_token(&self, _id: TokenIdType) -> Option { + // Tiktoken doesn't provide direct id-to-token mapping + // We can only decode IDs to text + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tiktoken_creation() { + let tokenizer = TiktokenTokenizer::new(TiktokenModel::Cl100kBase).unwrap(); + assert_eq!(tokenizer.vocab_size(), 100256); + } + + #[test] + fn test_model_from_name() { + assert!(matches!( + TiktokenTokenizer::model_from_name("gpt-4").unwrap(), + TiktokenModel::Cl100kBase + )); + assert!(matches!( + TiktokenTokenizer::model_from_name("gpt-3.5-turbo").unwrap(), + TiktokenModel::Cl100kBase + )); + assert!(matches!( + TiktokenTokenizer::model_from_name("text-davinci-003").unwrap(), + TiktokenModel::P50kBase + )); + assert!(matches!( + TiktokenTokenizer::model_from_name("text-davinci-edit-001").unwrap(), + TiktokenModel::P50kEdit + )); + assert!(matches!( + TiktokenTokenizer::model_from_name("davinci").unwrap(), + TiktokenModel::R50kBase + )); + } + + #[test] + fn test_encode_decode() { + let tokenizer = TiktokenTokenizer::new(TiktokenModel::Cl100kBase).unwrap(); + + let text = "Hello, world!"; + let encoding = tokenizer.encode(text).unwrap(); + + let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap(); + assert_eq!(decoded, text); + } + + #[test] + fn test_batch_encode() { + let tokenizer = TiktokenTokenizer::new(TiktokenModel::Cl100kBase).unwrap(); + + let texts = vec!["Hello", "World", "Test"]; + let encodings = tokenizer.encode_batch(&texts).unwrap(); + + assert_eq!(encodings.len(), 3); + for (i, encoding) in encodings.iter().enumerate() { + let decoded = tokenizer.decode(encoding.token_ids(), false).unwrap(); + assert_eq!(decoded, texts[i]); + } + } + + #[test] + fn test_special_tokens() { + let tokenizer = TiktokenTokenizer::new(TiktokenModel::Cl100kBase).unwrap(); + let special_tokens = tokenizer.get_special_tokens(); + + assert!(special_tokens.eos_token.is_some()); + assert_eq!(special_tokens.eos_token.as_ref().unwrap(), "<|endoftext|>"); + } + + #[test] + fn test_unrecognized_model_name_returns_error() { + // Test that unrecognized model names return an error + let result = TiktokenTokenizer::from_model_name("distilgpt-2"); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("Unrecognized OpenAI model name")); + } + + let result = TiktokenTokenizer::from_model_name("bert-base-uncased"); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("Unrecognized OpenAI model name")); + } + + let result = TiktokenTokenizer::from_model_name("llama-7b"); + assert!(result.is_err()); + if let Err(e) = result { + assert!(e.to_string().contains("Unrecognized OpenAI model name")); + } + } + + #[test] + fn test_recognized_model_names() { + // Test that recognized model names work correctly + assert!(TiktokenTokenizer::from_model_name("gpt-4").is_ok()); + assert!(TiktokenTokenizer::from_model_name("gpt-3.5-turbo").is_ok()); + assert!(TiktokenTokenizer::from_model_name("text-davinci-003").is_ok()); + assert!(TiktokenTokenizer::from_model_name("code-davinci-002").is_ok()); + assert!(TiktokenTokenizer::from_model_name("text-curie-001").is_ok()); + assert!(TiktokenTokenizer::from_model_name("text-babbage-001").is_ok()); + assert!(TiktokenTokenizer::from_model_name("text-ada-001").is_ok()); + } +} diff --git a/sgl-router/src/tokenizer/traits.rs b/sgl-router/src/tokenizer/traits.rs new file mode 100644 index 000000000..275dd822f --- /dev/null +++ b/sgl-router/src/tokenizer/traits.rs @@ -0,0 +1,83 @@ +use anyhow::Result; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +/// Type alias for token IDs +pub type TokenIdType = u32; + +/// Core encoding trait - separate from decoding for modularity +pub trait Encoder: Send + Sync { + fn encode(&self, input: &str) -> Result; + fn encode_batch(&self, inputs: &[&str]) -> Result>; +} + +/// Core decoding trait - can be implemented independently +pub trait Decoder: Send + Sync { + fn decode(&self, token_ids: &[TokenIdType], skip_special_tokens: bool) -> Result; +} + +/// Combined tokenizer trait +pub trait Tokenizer: Encoder + Decoder { + fn vocab_size(&self) -> usize; + fn get_special_tokens(&self) -> &SpecialTokens; + fn token_to_id(&self, token: &str) -> Option; + fn id_to_token(&self, id: TokenIdType) -> Option; +} + +/// Contains the results of tokenizing text: token IDs, string tokens, and their spans +#[derive(Debug, Clone)] +pub enum Encoding { + /// Hugging Face + Hf(Box), + /// Sentence Piece + Sp(Vec), + /// Tiktoken (for GPT models) - now uses u32 in tiktoken-rs 0.7.0 + Tiktoken(Vec), +} + +impl Encoding { + /// Returns a reference to token IDs - zero-copy operation + pub fn token_ids(&self) -> &[TokenIdType] { + match self { + Encoding::Hf(inner) => inner.get_ids(), + Encoding::Sp(inner) => inner, + Encoding::Tiktoken(inner) => inner, + } + } + + /// Deprecated: Use token_ids() instead (kept for compatibility) + #[deprecated(since = "0.1.0", note = "Use token_ids() instead")] + pub fn token_ids_ref(&self) -> &[TokenIdType] { + self.token_ids() + } + + /// Get a hash of the token IDs for caching purposes + pub fn get_hash(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.hash(&mut hasher); + hasher.finish() + } +} + +/// Hash implementation for Encoding +impl Hash for Encoding { + fn hash(&self, state: &mut H) { + match self { + Encoding::Hf(inner) => inner.get_ids().hash(state), + Encoding::Sp(inner) => inner.hash(state), + Encoding::Tiktoken(inner) => inner.hash(state), + } + } +} + +#[derive(Debug, Clone)] +pub struct SpecialTokens { + pub bos_token: Option, + pub eos_token: Option, + pub unk_token: Option, + pub sep_token: Option, + pub pad_token: Option, + pub cls_token: Option, + pub mask_token: Option, + pub additional_special_tokens: Vec, +} diff --git a/sgl-router/src/tool_parser/errors.rs b/sgl-router/src/tool_parser/errors.rs new file mode 100644 index 000000000..30129a596 --- /dev/null +++ b/sgl-router/src/tool_parser/errors.rs @@ -0,0 +1,32 @@ +use thiserror::Error; + +/// Result type for tool parser operations +pub type ToolParserResult = Result; + +/// Errors that can occur during tool parsing +#[derive(Debug, Error)] +pub enum ToolParserError { + #[error("Parsing failed: {0}")] + ParsingFailed(String), + + #[error("Model not supported: {0}")] + ModelNotSupported(String), + + #[error("Parse depth exceeded: max {0}")] + DepthExceeded(usize), + + #[error("Invalid JSON: {0}")] + JsonError(#[from] serde_json::Error), + + #[error("Regex error: {0}")] + RegexError(#[from] regex::Error), + + #[error("Incomplete tool call")] + Incomplete, + + #[error("Invalid tool name: {0}")] + InvalidToolName(String), + + #[error("Token not found: {0}")] + TokenNotFound(String), +} diff --git a/sgl-router/src/tool_parser/mod.rs b/sgl-router/src/tool_parser/mod.rs new file mode 100644 index 000000000..41b8fae2f --- /dev/null +++ b/sgl-router/src/tool_parser/mod.rs @@ -0,0 +1,30 @@ +/// Tool parser module for handling function/tool calls in model outputs +/// +/// This module provides infrastructure for parsing tool calls from various model formats. +// Core modules +pub mod errors; +pub mod partial_json; +pub mod python_literal_parser; +pub mod registry; +pub mod state; +pub mod traits; +pub mod types; + +// Parser implementations +pub mod parsers; + +#[cfg(test)] +mod tests; + +// Re-export commonly used types +pub use errors::{ToolParserError, ToolParserResult}; +pub use registry::ParserRegistry; +pub use state::{ParsePhase, ParseState}; +pub use traits::{PartialJsonParser, ToolParser}; +pub use types::{FunctionCall, PartialToolCall, StreamResult, TokenConfig, ToolCall}; + +// Re-export parsers for convenience +pub use parsers::{ + DeepSeekParser, Glm4MoeParser, GptOssParser, JsonParser, KimiK2Parser, LlamaParser, + MistralParser, PythonicParser, QwenParser, Step3Parser, +}; diff --git a/sgl-router/src/tool_parser/parsers/deepseek_parser.rs b/sgl-router/src/tool_parser/parsers/deepseek_parser.rs new file mode 100644 index 000000000..5e467bf2b --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/deepseek_parser.rs @@ -0,0 +1,277 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// DeepSeek V3 format parser for tool calls +/// +/// Handles the DeepSeek V3 specific format that uses Unicode tokens: +/// `<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>{name}\n```json\n{args}\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>` +/// +/// Features: +/// - Unicode token delimiters +/// - JSON arguments in code blocks +/// - Support for multiple sequential tool calls +pub struct DeepSeekParser { + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, + /// Regex for extracting complete tool calls + tool_call_extractor: Regex, + /// Regex for extracting function details + func_detail_extractor: Regex, +} + +impl DeepSeekParser { + /// Create a new DeepSeek parser + pub fn new() -> Self { + // Use (?s) flag for DOTALL mode to handle newlines + let tool_call_pattern = r"(?s)<｜tool▁call▁begin｜>.*?<｜tool▁call▁end｜>"; + let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); + + let func_detail_pattern = r"(?s)<｜tool▁call▁begin｜>(.*?)<｜tool▁sep｜>(.*?)\n```json\n(.*?)\n```<｜tool▁call▁end｜>"; + let func_detail_extractor = Regex::new(func_detail_pattern).expect("Valid regex pattern"); + + Self { + partial_json: PartialJson::default(), + tool_call_extractor, + func_detail_extractor, + } + } + + /// Check if text contains DeepSeek tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("<｜tool▁calls▁begin｜>") + } + + /// Extract all tool call blocks from text + fn extract_tool_calls<'a>(&self, text: &'a str) -> Vec<&'a str> { + self.tool_call_extractor + .find_iter(text) + .map(|m| m.as_str()) + .collect() + } + + /// Parse a single tool call block + fn parse_tool_call(&self, block: &str) -> ToolParserResult> { + if let Some(captures) = self.func_detail_extractor.captures(block) { + // Get function type (should be "function") + let func_type = captures.get(1).map_or("", |m| m.as_str()); + if func_type != "function" { + return Ok(None); + } + + // Get function name + let func_name = captures.get(2).map_or("", |m| m.as_str()).trim(); + + // Get JSON arguments + let json_args = captures.get(3).map_or("{}", |m| m.as_str()).trim(); + + // Parse JSON arguments + match serde_json::from_str::(json_args) { + Ok(value) => { + // Create arguments object + let args = if value.is_object() { + value + } else { + // If not an object, wrap it + serde_json::json!({ "value": value }) + }; + + let arguments = serde_json::to_string(&args) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate ID + let id = format!("deepseek_call_{}", uuid::Uuid::new_v4()); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: func_name.to_string(), + arguments, + }, + })) + } + Err(_) => Ok(None), + } + } else { + Ok(None) + } + } +} + +impl Default for DeepSeekParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for DeepSeekParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains DeepSeek format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + // Extract all tool call blocks + let tool_blocks = self.extract_tool_calls(text); + let mut tools = Vec::new(); + + for block in tool_blocks { + if let Some(tool) = self.parse_tool_call(block)? { + tools.push(tool); + } + } + + Ok(tools) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for tool markers + if !self.has_tool_markers(&state.buffer) { + // No markers found, return as incomplete + return Ok(StreamResult::Incomplete); + } + + // Look for start of tool calls + if let Some(start_pos) = state.buffer.find("<｜tool▁calls▁begin｜>") { + // Look for individual tool call start + let search_from = start_pos + "<｜tool▁calls▁begin｜>".len(); + if let Some(call_start) = state.buffer[search_from..].find("<｜tool▁call▁begin｜>") + { + let call_start_abs = search_from + call_start; + + // Look for the end of this tool call + let search_end_from = call_start_abs + "<｜tool▁call▁begin｜>".len(); + if let Some(call_end) = state.buffer[search_end_from..].find("<｜tool▁call▁end｜>") + { + let call_end_abs = search_end_from + call_end + "<｜tool▁call▁end｜>".len(); + + // Extract and parse the complete tool call + let tool_call_text = &state.buffer[call_start_abs..call_end_abs]; + + if let Some(tool) = self.parse_tool_call(tool_call_text)? { + // Remove the processed part from buffer + state.buffer.drain(..call_end_abs); + + return Ok(StreamResult::ToolComplete(tool)); + } + } else { + // Tool call not complete yet, try to extract partial info + let partial = &state.buffer[search_end_from..]; + + // Try to extract function name + if let Some(sep_pos) = partial.find("<｜tool▁sep｜>") { + if let Some(_func_start) = partial[..sep_pos].rfind("function") { + // We have the function type marker + let after_sep = &partial[sep_pos + "<｜tool▁sep｜>".len()..]; + + // Look for function name (ends at newline before ```json) + if let Some(name_end) = after_sep.find("\n```json\n") { + let func_name = after_sep[..name_end].trim(); + + if !state.in_string { + state.in_string = true; // Mark name as sent + return Ok(StreamResult::ToolName { + index: 0, + name: func_name.to_string(), + }); + } + + // Try to extract partial arguments + let args_start = name_end + "\n```json\n".len(); + let partial_args = &after_sep[args_start..]; + + // Check if we can parse partial JSON + if !partial_args.is_empty() { + match self.partial_json.parse_value(partial_args) { + Ok((value, _consumed)) => { + let args_str = serde_json::to_string(&value) + .unwrap_or_else(|_| "{}".to_string()); + + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + Err(_) => { + // Can't parse yet, keep buffering + } + } + } + } + } + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + self.has_tool_markers(text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_deepseek_single_tool() { + let parser = DeepSeekParser::new(); + let input = r#"Some text +<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Tokyo", "units": "celsius"} +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>More text"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Tokyo")); + } + + #[tokio::test] + async fn test_parse_deepseek_multiple_tools() { + let parser = DeepSeekParser::new(); + let input = r#"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Tokyo"} +```<｜tool▁call▁end｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Paris"} +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Tokyo")); + assert!(result[1].function.arguments.contains("Paris")); + } + + #[test] + fn test_detect_format() { + let parser = DeepSeekParser::new(); + assert!(parser.detect_format("<｜tool▁calls▁begin｜>")); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format("[TOOL_CALLS]")); + } +} diff --git a/sgl-router/src/tool_parser/parsers/glm4_moe_parser.rs b/sgl-router/src/tool_parser/parsers/glm4_moe_parser.rs new file mode 100644 index 000000000..017de1256 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/glm4_moe_parser.rs @@ -0,0 +1,292 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// GLM-4 MoE format parser for tool calls +/// +/// Handles the GLM-4 MoE specific format: +/// `{name}\n{key}\n{value}\n` +/// +/// Features: +/// - XML-style tags for tool calls +/// - Key-value pairs for arguments +/// - Support for multiple sequential tool calls +pub struct Glm4MoeParser { + /// Regex for extracting complete tool calls + tool_call_extractor: Regex, + /// Regex for extracting function details + func_detail_extractor: Regex, + /// Regex for extracting argument key-value pairs + arg_extractor: Regex, +} + +impl Glm4MoeParser { + /// Create a new GLM-4 MoE parser + pub fn new() -> Self { + // Use (?s) flag for DOTALL mode to handle newlines + let tool_call_pattern = r"(?s).*?"; + let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); + + let func_detail_pattern = r"(?s)([^\n]*)\n(.*)"; + let func_detail_extractor = Regex::new(func_detail_pattern).expect("Valid regex pattern"); + + let arg_pattern = r"(?s)(.*?)\s*(.*?)"; + let arg_extractor = Regex::new(arg_pattern).expect("Valid regex pattern"); + + Self { + tool_call_extractor, + func_detail_extractor, + arg_extractor, + } + } + + /// Check if text contains GLM-4 MoE tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("") + } + + /// Parse arguments from key-value pairs + fn parse_arguments(&self, args_text: &str) -> ToolParserResult> { + let mut arguments = serde_json::Map::new(); + + for capture in self.arg_extractor.captures_iter(args_text) { + let key = capture.get(1).map_or("", |m| m.as_str()).trim(); + let value_str = capture.get(2).map_or("", |m| m.as_str()).trim(); + + // Try to parse the value as JSON first, fallback to string + let value = if let Ok(json_val) = serde_json::from_str::(value_str) { + json_val + } else { + // Try parsing as Python literal (similar to Python's ast.literal_eval) + if value_str == "true" || value_str == "True" { + Value::Bool(true) + } else if value_str == "false" || value_str == "False" { + Value::Bool(false) + } else if value_str == "null" || value_str == "None" { + Value::Null + } else if let Ok(num) = value_str.parse::() { + Value::Number(num.into()) + } else if let Ok(num) = value_str.parse::() { + if let Some(n) = serde_json::Number::from_f64(num) { + Value::Number(n) + } else { + Value::String(value_str.to_string()) + } + } else { + Value::String(value_str.to_string()) + } + }; + + arguments.insert(key.to_string(), value); + } + + Ok(arguments) + } + + /// Parse a single tool call block + fn parse_tool_call(&self, block: &str) -> ToolParserResult> { + if let Some(captures) = self.func_detail_extractor.captures(block) { + // Get function name + let func_name = captures.get(1).map_or("", |m| m.as_str()).trim(); + + // Get arguments text + let args_text = captures.get(2).map_or("", |m| m.as_str()); + + // Parse arguments + let arguments = self.parse_arguments(args_text)?; + + let arguments_str = serde_json::to_string(&arguments) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate ID + let id = format!("glm4_call_{}", uuid::Uuid::new_v4()); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: func_name.to_string(), + arguments: arguments_str, + }, + })) + } else { + Ok(None) + } + } +} + +impl Default for Glm4MoeParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for Glm4MoeParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains GLM-4 MoE format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + // Extract all tool call blocks + let mut tools = Vec::new(); + for mat in self.tool_call_extractor.find_iter(text) { + if let Some(tool) = self.parse_tool_call(mat.as_str())? { + tools.push(tool); + } + } + + Ok(tools) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for tool markers + if !self.has_tool_markers(&state.buffer) { + // No markers found, return as incomplete + return Ok(StreamResult::Incomplete); + } + + // Look for start of tool call + if let Some(start_pos) = state.buffer.find("") { + // Look for the end of this tool call + let search_from = start_pos + "".len(); + if let Some(end_pos) = state.buffer[search_from..].find("") { + let end_abs = search_from + end_pos + "".len(); + + // Extract and parse the complete tool call + let tool_call_text = &state.buffer[start_pos..end_abs]; + + if let Some(tool) = self.parse_tool_call(tool_call_text)? { + // Remove the processed part from buffer + state.buffer.drain(..end_abs); + + return Ok(StreamResult::ToolComplete(tool)); + } + } else { + // Tool call not complete yet, try to extract partial info + let partial = &state.buffer[search_from..]; + + // Try to extract function name (first line after ) + if let Some(name_end) = partial.find('\n') { + let func_name = partial[..name_end].trim(); + + if !func_name.is_empty() && !state.in_string { + state.in_string = true; // Mark name as sent + return Ok(StreamResult::ToolName { + index: 0, + name: func_name.to_string(), + }); + } + + // Try to extract partial arguments + let args_text = &partial[name_end + 1..]; + let partial_args = self.parse_arguments(args_text)?; + + if !partial_args.is_empty() { + let args_str = serde_json::to_string(&partial_args) + .unwrap_or_else(|_| "{}".to_string()); + + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + self.has_tool_markers(text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_glm4_single_tool() { + let parser = Glm4MoeParser::new(); + let input = r#"Some text +get_weather +city +Beijing +date +2024-06-27 +More text"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Beijing")); + assert!(result[0].function.arguments.contains("2024-06-27")); + } + + #[tokio::test] + async fn test_parse_glm4_multiple_tools() { + let parser = Glm4MoeParser::new(); + let input = r#"get_weather +city +Beijing + +get_weather +city +Shanghai +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Beijing")); + assert!(result[1].function.arguments.contains("Shanghai")); + } + + #[tokio::test] + async fn test_parse_glm4_mixed_types() { + let parser = Glm4MoeParser::new(); + let input = r#"process_data +count +42 +active +true +name +test +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process_data"); + + // Parse arguments to check types + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["count"], 42); + assert_eq!(args["active"], true); + assert_eq!(args["name"], "test"); + } + + #[test] + fn test_detect_format() { + let parser = Glm4MoeParser::new(); + assert!(parser.detect_format("")); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format("[TOOL_CALLS]")); + } +} diff --git a/sgl-router/src/tool_parser/parsers/gpt_oss_parser.rs b/sgl-router/src/tool_parser/parsers/gpt_oss_parser.rs new file mode 100644 index 000000000..646161a72 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/gpt_oss_parser.rs @@ -0,0 +1,292 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// GPT-OSS format parser for tool calls +/// +/// Handles the GPT-OSS specific channel format: +/// `<|channel|>commentary to={namespace.function}<|constrain|>json<|message|>{json_args}<|call|>` +/// +/// Features: +/// - Channel-based format with commentary +/// - Namespaced function calls +/// - JSON arguments +pub struct GptOssParser { + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, + /// Regex for extracting complete function calls + function_call_extractor: Regex, + /// Regex for extracting streaming function calls + streaming_extractor: Regex, +} + +impl GptOssParser { + /// Create a new GPT-OSS parser + pub fn new() -> Self { + // Pattern for complete function calls with to= parameter + // Handles optional <|start|>assistant prefix and whitespace after function name + let function_call_pattern = r"(?s)(?:<\|start\|>assistant)?<\|channel\|>commentary to=([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)\s*<\|constrain\|>json<\|message\|>(.*?)<\|call\|>(?:commentary)?"; + let function_call_extractor = + Regex::new(function_call_pattern).expect("Valid regex pattern"); + + // Pattern for streaming function calls (incomplete) + let streaming_pattern = r"(?s)(?:<\|start\|>assistant)?<\|channel\|>commentary to=([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)\s*<\|constrain\|>json<\|message\|>(.*)"; + let streaming_extractor = Regex::new(streaming_pattern).expect("Valid regex pattern"); + + Self { + partial_json: PartialJson::default(), + function_call_extractor, + streaming_extractor, + } + } + + /// Check if text contains GPT-OSS tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("<|channel|>commentary to=") + } + + /// Extract function name from full namespace (e.g., "functions.get_weather" -> "get_weather") + fn extract_function_name(&self, full_name: &str) -> String { + if let Some(dot_pos) = full_name.rfind('.') { + full_name[dot_pos + 1..].to_string() + } else { + full_name.to_string() + } + } +} + +impl Default for GptOssParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for GptOssParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains GPT-OSS format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + let mut tools = Vec::new(); + let mut _tool_index = 0; + + // Extract all function calls + for captures in self.function_call_extractor.captures_iter(text) { + if let (Some(name_match), Some(args_match)) = (captures.get(1), captures.get(2)) { + let full_function_name = name_match.as_str(); + let args_content = args_match.as_str().trim(); + + // Extract actual function name + let function_name = self.extract_function_name(full_function_name); + + // Parse JSON arguments + let arguments = if args_content.is_empty() { + "{}".to_string() + } else { + match serde_json::from_str::(args_content) { + Ok(value) => serde_json::to_string(&value) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?, + Err(_) => { + // Skip malformed JSON + continue; + } + } + }; + + // Generate unique ID + let id = format!("gpt_oss_call_{}", uuid::Uuid::new_v4()); + + tools.push(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: function_name, + arguments, + }, + }); + + _tool_index += 1; + } + } + + Ok(tools) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for tool markers + if !self.has_tool_markers(&state.buffer) { + // No markers found, clear buffer and return + state.buffer.clear(); + return Ok(StreamResult::Incomplete); + } + + // Try to match streaming pattern + if let Some(captures) = self.streaming_extractor.captures(&state.buffer) { + if let (Some(name_match), Some(args_match)) = (captures.get(1), captures.get(2)) { + let full_function_name = name_match.as_str(); + let partial_args = args_match.as_str(); + + // Extract actual function name + let function_name = self.extract_function_name(full_function_name); + + // Send function name if not sent yet + if !state.in_string { + state.in_string = true; // Mark name as sent + return Ok(StreamResult::ToolName { + index: 0, + name: function_name.clone(), + }); + } + + // Check if we have a complete function call + if let Some(complete_match) = self.function_call_extractor.captures(&state.buffer) { + if let Some(args_match) = complete_match.get(2) { + let args_content = args_match.as_str().trim(); + + // Parse JSON arguments + let arguments = if args_content.is_empty() { + "{}".to_string() + } else { + match serde_json::from_str::(args_content) { + Ok(value) => serde_json::to_string(&value) + .unwrap_or_else(|_| "{}".to_string()), + Err(_) => "{}".to_string(), + } + }; + + // Generate unique ID + let id = format!("gpt_oss_call_{}", uuid::Uuid::new_v4()); + + let tool = ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: function_name, + arguments, + }, + }; + + // Remove the processed part from buffer + let complete_end = complete_match.get(0).unwrap().end(); + state.buffer.drain(..complete_end); + + // Reset state for next tool + state.in_string = false; + + return Ok(StreamResult::ToolComplete(tool)); + } + } else { + // Try to parse partial JSON for streaming arguments + if !partial_args.is_empty() { + // Look for the end of JSON (before <|call|>) + let json_part = if let Some(call_pos) = partial_args.find("<|call|>") { + &partial_args[..call_pos] + } else { + partial_args + }; + + match self.partial_json.parse_value(json_part) { + Ok((value, _consumed)) => { + let args_str = serde_json::to_string(&value) + .unwrap_or_else(|_| "{}".to_string()); + + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + Err(_) => { + // Can't parse yet, keep buffering + } + } + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + self.has_tool_markers(text) || text.contains("<|channel|>commentary") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_gpt_oss_single_tool() { + let parser = GptOssParser::new(); + let input = r#"Some text +<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location": "San Francisco"}<|call|> +More text"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("San Francisco")); + } + + #[tokio::test] + async fn test_parse_gpt_oss_multiple_tools() { + let parser = GptOssParser::new(); + let input = r#"<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location": "Paris"}<|call|>commentary +<|channel|>commentary to=functions.search<|constrain|>json<|message|>{"query": "Paris tourism"}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "search"); + assert!(result[0].function.arguments.contains("Paris")); + assert!(result[1].function.arguments.contains("Paris tourism")); + } + + #[tokio::test] + async fn test_parse_gpt_oss_with_prefix() { + let parser = GptOssParser::new(); + let input = r#"<|start|>assistant<|channel|>commentary to=functions.test<|constrain|>json<|message|>{"key": "value"}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } + + #[tokio::test] + async fn test_parse_gpt_oss_empty_args() { + let parser = GptOssParser::new(); + let input = + r#"<|channel|>commentary to=functions.get_time<|constrain|>json<|message|>{}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); + assert_eq!(result[0].function.arguments, "{}"); + } + + #[test] + fn test_detect_format() { + let parser = GptOssParser::new(); + assert!(parser.detect_format("<|channel|>commentary to=")); + assert!(parser.detect_format("<|channel|>commentary")); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format("[TOOL_CALLS]")); + } +} diff --git a/sgl-router/src/tool_parser/parsers/json_parser.rs b/sgl-router/src/tool_parser/parsers/json_parser.rs new file mode 100644 index 000000000..b8430dc9e --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/json_parser.rs @@ -0,0 +1,619 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, TokenConfig, ToolCall}, +}; + +/// JSON format parser for tool calls +/// +/// Handles various JSON formats for function calling: +/// - Single tool call: {"name": "fn", "arguments": {...}} +/// - Multiple tool calls: [{"name": "fn1", "arguments": {...}}, ...] +/// - With parameters instead of arguments: {"name": "fn", "parameters": {...}} +/// +/// Supports configurable token markers for different models +pub struct JsonParser { + /// Token configuration for parsing + token_config: TokenConfig, + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, + /// Regex patterns for extracting content between tokens + extractors: Vec, +} + +impl JsonParser { + /// Create a new JSON parser with default configuration + pub fn new() -> Self { + Self::with_config(TokenConfig { + start_tokens: vec![], + end_tokens: vec![], + separator: ", ".to_string(), + }) + } + + /// Create a parser with custom token configuration + pub fn with_config(config: TokenConfig) -> Self { + // Build extraction patterns for each token pair + let extractors: Vec = config + .iter_pairs() + .filter_map(|(start, end)| { + if !start.is_empty() && !end.is_empty() { + // Use (?s) flag to enable DOTALL mode so . matches newlines + let pattern = + format!(r"(?s){}(.*?){}", regex::escape(start), regex::escape(end)); + Regex::new(&pattern).ok() + } else { + None + } + }) + .collect(); + + Self { + token_config: config, + partial_json: PartialJson::default(), + extractors, + } + } + + /// Extract JSON content from text, handling wrapper tokens if configured + fn extract_json_content<'a>(&self, text: &'a str) -> &'a str { + let mut content = text; + + // Try each extractor pattern (for tokens with both start and end) + for extractor in &self.extractors { + if let Some(captures) = extractor.captures(content) { + if let Some(matched) = captures.get(1) { + return matched.as_str().trim(); + } + } + } + + // Handle special case where there's a start token but no end token + for (start, end) in self.token_config.iter_pairs() { + if !start.is_empty() && end.is_empty() { + // Find the start token and extract everything after it + if let Some(pos) = content.find(start) { + content = &content[pos + start.len()..]; + return content.trim(); + } + } + } + + content.trim() + } + + /// Try to extract a JSON object or array from text that may contain other content + fn extract_json_from_text(&self, text: &str) -> Option { + // Look for JSON object starting with { + if let Some(start) = text.find('{') { + let mut depth = 0; + let mut in_string = false; + let mut escape_next = false; + + for (i, ch) in text[start..].char_indices() { + if escape_next { + escape_next = false; + continue; + } + + match ch { + '\\' if in_string => escape_next = true, + '"' if !in_string => in_string = true, + '"' if in_string => in_string = false, + '{' if !in_string => depth += 1, + '}' if !in_string => { + depth -= 1; + if depth == 0 { + return Some(text[start..start + i + 1].to_string()); + } + } + _ => {} + } + } + } + + // Look for JSON array starting with [ + if let Some(start) = text.find('[') { + let mut depth = 0; + let mut in_string = false; + let mut escape_next = false; + + for (i, ch) in text[start..].char_indices() { + if escape_next { + escape_next = false; + continue; + } + + match ch { + '\\' if in_string => escape_next = true, + '"' if !in_string => in_string = true, + '"' if in_string => in_string = false, + '[' if !in_string => depth += 1, + ']' if !in_string => { + depth -= 1; + if depth == 0 { + return Some(text[start..start + i + 1].to_string()); + } + } + _ => {} + } + } + } + + None + } + + /// Parse a single JSON object into a ToolCall + fn parse_single_object(&self, obj: &Value) -> ToolParserResult> { + // Check if this looks like a tool call + let name = obj + .get("name") + .or_else(|| obj.get("function")) + .and_then(|v| v.as_str()); + + if let Some(name) = name { + // Get arguments - support both "arguments" and "parameters" keys + let empty_obj = Value::Object(serde_json::Map::new()); + let args = obj + .get("arguments") + .or_else(|| obj.get("parameters")) + .unwrap_or(&empty_obj); + + // Convert arguments to JSON string + let arguments = serde_json::to_string(args) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate a unique ID if not provided + let id = obj + .get("id") + .and_then(|v| v.as_str()) + .map(String::from) + .unwrap_or_else(|| format!("call_{}", uuid::Uuid::new_v4())); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: name.to_string(), + arguments, + }, + })) + } else { + Ok(None) + } + } + + /// Parse JSON value(s) into tool calls + fn parse_json_value(&self, value: &Value) -> ToolParserResult> { + let mut tools = Vec::new(); + + match value { + Value::Array(arr) => { + // Parse each element in the array + for item in arr { + if let Some(tool) = self.parse_single_object(item)? { + tools.push(tool); + } + } + } + Value::Object(_) => { + // Single tool call + if let Some(tool) = self.parse_single_object(value)? { + tools.push(tool); + } + } + _ => { + // Not a valid tool call format + return Ok(vec![]); + } + } + + Ok(tools) + } + + /// Check if text contains potential tool call markers + fn has_tool_markers(&self, text: &str) -> bool { + // If no start tokens configured, check for JSON structure + if self.token_config.start_tokens.is_empty() { + // For JSON, we just need to see the start of an object or array + return text.contains('{') || text.contains('['); + } + + // Check for any start token + self.token_config + .start_tokens + .iter() + .any(|token| text.contains(token)) + } +} + +impl Default for JsonParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for JsonParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if we have multiple start tokens (e.g., multiple <|python_tag|> markers) + if !self.token_config.start_tokens.is_empty() { + let start_token = &self.token_config.start_tokens[0]; + if !start_token.is_empty() && text.matches(start_token).count() > 1 { + // We have multiple occurrences of the start token + let mut all_tools = Vec::new(); + let mut remaining = text; + + while let Some(start_pos) = remaining.find(start_token.as_str()) { + // Extract content after this start token + let after_token = &remaining[start_pos + start_token.len()..]; + + // Find where this JSON ends (look for the next start token or end of string) + let end_pos = if let Some(next_start) = after_token.find(start_token.as_str()) { + next_start + } else { + after_token.len() + }; + + let json_content = &after_token[..end_pos]; + + // Try to extract and parse JSON from this segment + if let Some(extracted) = self.extract_json_from_text(json_content) { + if let Ok(value) = serde_json::from_str::(&extracted) { + if let Ok(tools) = self.parse_json_value(&value) { + all_tools.extend(tools); + } + } + } + + // Move to the next segment + remaining = &remaining[start_pos + start_token.len() + end_pos..]; + if remaining.is_empty() { + break; + } + } + + if !all_tools.is_empty() { + return Ok(all_tools); + } + } + } + + // Extract JSON content from wrapper tokens if present + let json_content = self.extract_json_content(text); + + // Try to parse as JSON first + match serde_json::from_str::(json_content) { + Ok(value) => self.parse_json_value(&value), + Err(_) => { + // If parse failed, check if we have multiple JSON objects separated by the configured separator + // This handles cases like: {"name": "func1", ...};{"name": "func2", ...} + if !self.token_config.separator.is_empty() + && json_content.contains(&self.token_config.separator) + { + let mut all_tools = Vec::new(); + + // Split by separator and try to parse each part + let parts: Vec<&str> = + json_content.split(&self.token_config.separator).collect(); + for part in parts { + let trimmed = part.trim(); + if trimmed.is_empty() { + continue; + } + + // Try to parse this part as JSON + if let Ok(value) = serde_json::from_str::(trimmed) { + if let Ok(tools) = self.parse_json_value(&value) { + all_tools.extend(tools); + } + } else if let Some(extracted) = self.extract_json_from_text(trimmed) { + // Try extracting JSON from this part + if let Ok(value) = serde_json::from_str::(&extracted) { + if let Ok(tools) = self.parse_json_value(&value) { + all_tools.extend(tools); + } + } + } + } + + if !all_tools.is_empty() { + return Ok(all_tools); + } + } + + // If no wrapper tokens configured and parse failed, + // try to extract JSON from mixed text + if self.token_config.start_tokens.is_empty() { + if let Some(extracted) = self.extract_json_from_text(text) { + if let Ok(value) = serde_json::from_str::(&extracted) { + return self.parse_json_value(&value); + } + } + } + // Not valid JSON, return empty + Ok(vec![]) + } + } + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check if we have potential tool calls + if !self.has_tool_markers(&state.buffer) { + // No tool markers, return as incomplete + return Ok(StreamResult::Incomplete); + } + + // Extract JSON content first to check for separators + let extracted_json = self.extract_json_content(&state.buffer); + + // Handle multiple JSON objects with separators + // Check if we have a separator and potentially multiple JSON objects + let separator = &self.token_config.separator; + if !separator.is_empty() && extracted_json.contains(separator.as_str()) { + // Try to find a complete JSON object before the separator + if let Some(separator_pos) = extracted_json.find(separator.as_str()) { + // Get JSON before separator + let before_separator = &extracted_json[..separator_pos]; + + // Try to parse the JSON before the separator + match serde_json::from_str::(before_separator) { + Ok(value) => { + // Parse tool calls from this JSON + let tools = self.parse_json_value(&value)?; + if !tools.is_empty() { + // We need to figure out how much to remove from the original buffer + // Find where the separator is in the original buffer and remove up to and including it + if let Some(sep_in_original) = state.buffer.find(separator.as_str()) { + let remaining = + state.buffer[sep_in_original + separator.len()..].to_string(); + state.buffer = remaining; + } + + // Return the first tool as complete + if let Some(tool) = tools.into_iter().next() { + return Ok(StreamResult::ToolComplete(tool)); + } + } + } + Err(_) => { + // Failed to parse, continue to try other methods + } + } + } + } + + // Handle multiple start tokens (e.g., multiple <|python_tag|> markers) + if !self.token_config.start_tokens.is_empty() { + let start_token = &self.token_config.start_tokens[0]; + if !start_token.is_empty() { + // Find all occurrences of start token + let occurrences: Vec<_> = + state.buffer.match_indices(start_token.as_str()).collect(); + if occurrences.len() > 1 { + // We have multiple start tokens, try to process the first complete one + let first_pos = occurrences[0].0; + let second_pos = occurrences[1].0; + + // Extract content between first and second start token + let first_json_section = &state.buffer[first_pos..second_pos]; + let json_content = self.extract_json_content(first_json_section); + + // Try to parse this as complete JSON + if let Ok(value) = serde_json::from_str::(json_content) { + // Parse tool calls from this JSON + let tools = self.parse_json_value(&value)?; + if !tools.is_empty() { + // Remove the processed section from buffer + let remaining = state.buffer[second_pos..].to_string(); + state.buffer = remaining; + + // Return the first tool as complete + if let Some(tool) = tools.into_iter().next() { + return Ok(StreamResult::ToolComplete(tool)); + } + } + } + } + } + } + + // Regular single JSON parsing + // Extract JSON content + let json_content = self.extract_json_content(&state.buffer); + + // Try to parse with partial JSON parser + match self.partial_json.parse_value(json_content) { + Ok((value, consumed)) => { + // Check if we have a complete JSON structure + if consumed == json_content.len() { + // Check if this is truly complete or just has null from incomplete parsing + // We need to ensure the JSON actually ends properly (not cut off mid-key) + let trimmed = json_content.trim(); + let looks_complete = trimmed.ends_with('}') || trimmed.ends_with(']'); + + if looks_complete { + // Complete JSON, parse tool calls + let tools = self.parse_json_value(&value)?; + if !tools.is_empty() { + // Clear buffer since we consumed everything + state.buffer.clear(); + + // Return the first tool as complete + // TODO simplified version, address more complex version + if let Some(tool) = tools.into_iter().next() { + return Ok(StreamResult::ToolComplete(tool)); + } + } + } + } else { + // Partial JSON, try to extract tool name + if let Some(name) = value.get("name").and_then(|v| v.as_str()) { + // TODO simplified version, address more complex version + // Just return the tool name once we see it + if !state.in_string { + state.in_string = true; // Use as a flag for "name sent" + return Ok(StreamResult::ToolName { + index: 0, + name: name.to_string(), + }); + } + + // Check for complete arguments + if let Some(args) = + value.get("arguments").or_else(|| value.get("parameters")) + { + if let Ok(args_str) = serde_json::to_string(args) { + // Return arguments as a single update + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + } + } + } + } + Err(_) => { + // Failed to parse even as partial JSON + // Keep buffering + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + // Check if text contains JSON-like structure + if self.has_tool_markers(text) { + // Try to extract and parse + let json_content = self.extract_json_content(text); + + // Check if it looks like valid JSON for tool calls + if let Ok(value) = serde_json::from_str::(json_content) { + match value { + Value::Object(ref obj) => { + // Check for tool call structure + obj.contains_key("name") || obj.contains_key("function") + } + Value::Array(ref arr) => { + // Check if array contains tool-like objects + arr.iter().any(|v| { + if let Some(obj) = v.as_object() { + obj.contains_key("name") || obj.contains_key("function") + } else { + false + } + }) + } + _ => false, + } + } else { + false + } + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_single_tool_call() { + let parser = JsonParser::new(); + let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + } + + #[tokio::test] + async fn test_parse_multiple_tool_calls() { + let parser = JsonParser::new(); + let input = r#"[ + {"name": "get_weather", "arguments": {"location": "SF"}}, + {"name": "search", "arguments": {"query": "news"}} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "search"); + } + + #[tokio::test] + async fn test_parse_with_parameters_key() { + let parser = JsonParser::new(); + let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + assert!(result[0].function.arguments.contains("10")); + } + + #[tokio::test] + async fn test_parse_with_wrapper_tokens() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } + + #[test] + fn test_detect_format() { + let parser = JsonParser::new(); + + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(parser.detect_format(r#"[{"name": "test"}]"#)); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format(r#"{"key": "value"}"#)); + } + + #[tokio::test] + async fn test_streaming_parse() { + // Just verify that streaming eventually produces a complete tool call + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Send complete JSON in one go + // TODO simplified version, address more complex version + let full_json = r#"{"name": "get_weather", "arguments": {"location": "SF"}}"#; + + let result = parser + .parse_incremental(full_json, &mut state) + .await + .unwrap(); + + // Should get a complete tool immediately with complete JSON + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + assert!(tool.function.arguments.contains("SF")); + } + _ => panic!("Expected ToolComplete for complete JSON input"), + } + } +} diff --git a/sgl-router/src/tool_parser/parsers/kimik2_parser.rs b/sgl-router/src/tool_parser/parsers/kimik2_parser.rs new file mode 100644 index 000000000..52f92bd90 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/kimik2_parser.rs @@ -0,0 +1,270 @@ +use async_trait::async_trait; +use regex::Regex; + +use crate::tool_parser::{ + errors::ToolParserResult, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// Kimi K2 format parser for tool calls +/// +/// Handles the Kimi K2 specific format: +/// `<|tool_calls_section_begin|><|tool_call_begin|>functions.{name}:{index}<|tool_call_argument_begin|>{json_args}<|tool_call_end|><|tool_calls_section_end|>` +/// +/// Features: +/// - Token-based delimiters +/// - Function calls with explicit indexing +/// - JSON arguments +pub struct KimiK2Parser { + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, + /// Regex for extracting complete tool calls + tool_call_extractor: Regex, + /// Regex for extracting partial tool calls (streaming) + stream_tool_call_extractor: Regex, +} + +impl KimiK2Parser { + /// Create a new Kimi K2 parser + pub fn new() -> Self { + // Pattern for complete tool calls + let tool_call_pattern = r"<\|tool_call_begin\|>\s*(?P[\w\.]+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P\{.*?\})\s*<\|tool_call_end\|>"; + let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); + + // Pattern for streaming (partial) tool calls + let stream_pattern = r"<\|tool_call_begin\|>\s*(?P[\w\.]+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P\{.*)"; + let stream_tool_call_extractor = Regex::new(stream_pattern).expect("Valid regex pattern"); + + Self { + partial_json: PartialJson::default(), + tool_call_extractor, + stream_tool_call_extractor, + } + } + + /// Check if text contains Kimi K2 tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("<|tool_calls_section_begin|>") + } + + /// Parse function ID to extract name and index + fn parse_function_id(&self, id: &str) -> Option<(String, usize)> { + // Format: functions.{name}:{index} or namespace.functions.{name}:{index} + // Extract everything after the last dot before the colon as the function name + if let Some(colon_pos) = id.rfind(':') { + let before_colon = &id[..colon_pos]; + let index_str = &id[colon_pos + 1..]; + + // Find the last dot to extract the function name + if let Some(dot_pos) = before_colon.rfind('.') { + let func_name = &before_colon[dot_pos + 1..]; + + if let Ok(index) = index_str.parse::() { + return Some((func_name.to_string(), index)); + } + } + } + None + } +} + +impl Default for KimiK2Parser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for KimiK2Parser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains Kimi K2 format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + let mut tools = Vec::new(); + + // Extract all tool calls + for captures in self.tool_call_extractor.captures_iter(text) { + if let (Some(id_match), Some(args_match)) = ( + captures.name("tool_call_id"), + captures.name("function_arguments"), + ) { + let function_id = id_match.as_str(); + let function_args = args_match.as_str(); + + // Parse function ID + if let Some((func_name, _index)) = self.parse_function_id(function_id) { + // Validate JSON arguments + if serde_json::from_str::(function_args).is_ok() { + // Generate unique ID + let id = format!("kimi_call_{}", uuid::Uuid::new_v4()); + + tools.push(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: func_name, + arguments: function_args.to_string(), + }, + }); + } + } + } + } + + Ok(tools) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for tool markers + let has_tool_call = + self.has_tool_markers(&state.buffer) || state.buffer.contains("<|tool_call_begin|>"); + + if !has_tool_call { + // No markers found, clear buffer and return + state.buffer.clear(); + return Ok(StreamResult::Incomplete); + } + + // Try to match streaming pattern + if let Some(captures) = self.stream_tool_call_extractor.captures(&state.buffer) { + if let (Some(id_match), Some(args_match)) = ( + captures.name("tool_call_id"), + captures.name("function_arguments"), + ) { + let function_id = id_match.as_str(); + let partial_args = args_match.as_str(); + + // Parse function ID + if let Some((func_name, _index)) = self.parse_function_id(function_id) { + // Send function name if not sent yet + if !state.in_string { + state.in_string = true; // Mark name as sent + return Ok(StreamResult::ToolName { + index: 0, + name: func_name.clone(), + }); + } + + // Check if we have a complete tool call + if let Some(end_pos) = partial_args.find("<|tool_call_end|>") { + // Extract just the JSON part + let json_args = &partial_args[..end_pos]; + + // Validate and parse JSON + if serde_json::from_str::(json_args).is_ok() { + // Generate unique ID + let id = format!("kimi_call_{}", uuid::Uuid::new_v4()); + + let tool = ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: func_name, + arguments: json_args.to_string(), + }, + }; + + // Find where this tool call ends in the buffer + if let Some(tool_end) = state.buffer.find("<|tool_call_end|>") { + let end_pos = tool_end + "<|tool_call_end|>".len(); + state.buffer.drain(..end_pos); + } + + // Reset state for next tool + state.in_string = false; + + return Ok(StreamResult::ToolComplete(tool)); + } + } else { + // Try to parse partial JSON for streaming arguments + match self.partial_json.parse_value(partial_args) { + Ok((value, _consumed)) => { + let args_str = serde_json::to_string(&value) + .unwrap_or_else(|_| "{}".to_string()); + + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + Err(_) => { + // Can't parse yet, keep buffering + } + } + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + self.has_tool_markers(text) || text.contains("<|tool_call_begin|>") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_kimi_single_tool() { + let parser = KimiK2Parser::new(); + let input = r#"Some text +<|tool_calls_section_begin|> +<|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"location": "Tokyo", "units": "celsius"}<|tool_call_end|> +<|tool_calls_section_end|>More text"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Tokyo")); + } + + #[tokio::test] + async fn test_parse_kimi_multiple_tools() { + let parser = KimiK2Parser::new(); + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"query": "rust"}<|tool_call_end|> +<|tool_call_begin|>functions.calculate:1<|tool_call_argument_begin|>{"expression": "2+2"}<|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "calculate"); + } + + #[tokio::test] + async fn test_parse_kimi_with_whitespace() { + let parser = KimiK2Parser::new(); + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {"key": "value"} <|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } + + #[test] + fn test_detect_format() { + let parser = KimiK2Parser::new(); + assert!(parser.detect_format("<|tool_calls_section_begin|>")); + assert!(parser.detect_format("<|tool_call_begin|>")); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format("[TOOL_CALLS]")); + } +} diff --git a/sgl-router/src/tool_parser/parsers/llama_parser.rs b/sgl-router/src/tool_parser/parsers/llama_parser.rs new file mode 100644 index 000000000..678c964c5 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/llama_parser.rs @@ -0,0 +1,156 @@ +use async_trait::async_trait; + +use super::json_parser::JsonParser; +use crate::tool_parser::{ + errors::ToolParserResult, + state::ParseState, + traits::ToolParser, + types::{StreamResult, TokenConfig, ToolCall}, +}; + +/// Llama 3.2 format parser for tool calls +/// +/// Handles the Llama 3.2 specific format: +/// `<|python_tag|>{"name": "func", "arguments": {...}}` +/// +/// Also supports plain JSON without the python_tag prefix +pub struct LlamaParser { + /// Underlying JSON parser with Llama-specific configuration + json_parser: JsonParser, +} + +impl LlamaParser { + /// Create a new Llama parser + pub fn new() -> Self { + // Configure JSON parser with Llama's python_tag token + // Note: No end token for python_tag format + let json_parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["<|python_tag|>".to_string()], + end_tokens: vec!["".to_string()], // Empty end token + separator: ";".to_string(), // Llama uses semicolon for multiple calls (though not well supported) + }); + + Self { json_parser } + } +} + +impl Default for LlamaParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for LlamaParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // First try with the configured python_tag parser + let result = self.json_parser.parse_complete(text).await?; + + if !result.is_empty() { + return Ok(result); + } + + // If no results and text starts with '{', try plain JSON + if text.trim_start().starts_with('{') { + // Create a temporary plain JSON parser + let plain_parser = JsonParser::new(); + return plain_parser.parse_complete(text).await; + } + + Ok(vec![]) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + // Try with the python_tag parser first + let result = self.json_parser.parse_incremental(chunk, state).await?; + + // If we get Incomplete and buffer starts with '{', might be plain JSON + if matches!(result, StreamResult::Incomplete) && state.buffer.trim_start().starts_with('{') + { + // Check if we have python_tag in the buffer + if !state.buffer.contains("<|python_tag|>") { + // Likely plain JSON, create temporary parser + let plain_parser = JsonParser::new(); + return plain_parser.parse_incremental("", state).await; + } + } + + Ok(result) + } + + fn detect_format(&self, text: &str) -> bool { + // Llama format if contains python_tag or starts with JSON object + text.contains("<|python_tag|>") + || (text.trim_start().starts_with('{') + && (text.contains(r#""name""#) || text.contains(r#""function""#))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_with_python_tag() { + let parser = LlamaParser::new(); + let input = r#"<|python_tag|>{"name": "search", "arguments": {"query": "weather"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); + assert!(result[0].function.arguments.contains("weather")); + } + + #[tokio::test] + async fn test_parse_plain_json() { + let parser = LlamaParser::new(); + let input = r#"{"name": "calculate", "arguments": {"x": 5, "y": 10}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + } + + #[tokio::test] + async fn test_parse_with_text_before() { + let parser = LlamaParser::new(); + let input = r#"Let me help you with that. <|python_tag|>{"name": "get_time", "arguments": {"timezone": "UTC"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); + } + + #[test] + fn test_detect_format() { + let parser = LlamaParser::new(); + + assert!(parser.detect_format(r#"<|python_tag|>{"name": "test"}"#)); + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format(r#"{"key": "value"}"#)); // No name field + } + + #[tokio::test] + async fn test_single_call_with_semicolon() { + let parser = LlamaParser::new(); + // Note: Llama 3.2 doesn't handle multiple calls well + // Test that we can at least parse a single call followed by semicolon + let input = r#"<|python_tag|>{"name": "func1", "arguments": {"x": 1}};"#; + + let result = parser.parse_complete(input).await.unwrap(); + + // We expect this to either parse the first JSON object or fail gracefully + // Since the semicolon makes it invalid JSON, it will likely return empty + // This is acceptable as Llama 3.2 doesn't reliably support parallel calls + + // If it parses anything, it should be func1 + if !result.is_empty() { + assert_eq!(result[0].function.name, "func1"); + } + } +} diff --git a/sgl-router/src/tool_parser/parsers/mistral_parser.rs b/sgl-router/src/tool_parser/parsers/mistral_parser.rs new file mode 100644 index 000000000..68a3568aa --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/mistral_parser.rs @@ -0,0 +1,347 @@ +use async_trait::async_trait; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// Mistral format parser for tool calls +/// +/// Handles the Mistral-specific format: +/// `[TOOL_CALLS] [{"name": "func", "arguments": {...}}, ...]` +/// +/// Features: +/// - Bracket counting for proper JSON array extraction +/// - Support for multiple tool calls in a single array +/// - String-aware parsing to handle nested brackets in JSON +pub struct MistralParser { + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, +} + +impl MistralParser { + /// Create a new Mistral parser + pub fn new() -> Self { + Self { + partial_json: PartialJson::default(), + } + } + + /// Extract JSON array using bracket counting + /// + /// Handles nested brackets in JSON content by tracking: + /// - String boundaries (quotes) + /// - Escape sequences + /// - Bracket depth + fn extract_json_array<'a>(&self, text: &'a str) -> Option<&'a str> { + const BOT_TOKEN: &str = "[TOOL_CALLS] ["; + + // Find the start of the token + let start_idx = text.find(BOT_TOKEN)?; + + // Start from the opening bracket after [TOOL_CALLS] + // The -1 is to include the opening bracket that's part of the token + let json_start = start_idx + BOT_TOKEN.len() - 1; + + let mut bracket_count = 0; + let mut in_string = false; + let mut escape_next = false; + + let bytes = text.as_bytes(); + + for i in json_start..text.len() { + let char = bytes[i]; + + if escape_next { + escape_next = false; + continue; + } + + if char == b'\\' { + escape_next = true; + continue; + } + + if char == b'"' && !escape_next { + in_string = !in_string; + continue; + } + + if !in_string { + if char == b'[' { + bracket_count += 1; + } else if char == b']' { + bracket_count -= 1; + if bracket_count == 0 { + // Found the matching closing bracket + return Some(&text[json_start..=i]); + } + } + } + } + + // Incomplete array (no matching closing bracket found) + None + } + + /// Parse tool calls from a JSON array + fn parse_json_array(&self, json_str: &str) -> ToolParserResult> { + let value: Value = serde_json::from_str(json_str) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + let mut tools = Vec::new(); + + if let Value::Array(arr) = value { + for (index, item) in arr.iter().enumerate() { + if let Some(tool) = self.parse_single_object(item, index)? { + tools.push(tool); + } + } + } else { + // Single object case (shouldn't happen with Mistral format, but handle it) + if let Some(tool) = self.parse_single_object(&value, 0)? { + tools.push(tool); + } + } + + Ok(tools) + } + + /// Parse a single JSON object into a ToolCall + fn parse_single_object(&self, obj: &Value, index: usize) -> ToolParserResult> { + let name = obj.get("name").and_then(|v| v.as_str()); + + if let Some(name) = name { + // Get arguments - Mistral uses "arguments" key + let empty_obj = Value::Object(serde_json::Map::new()); + let args = obj.get("arguments").unwrap_or(&empty_obj); + + // Convert arguments to JSON string + let arguments = serde_json::to_string(args) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate ID with index for multiple tools + let id = format!("mistral_call_{}", index); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: name.to_string(), + arguments, + }, + })) + } else { + Ok(None) + } + } + + /// Check if text contains Mistral tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("[TOOL_CALLS]") + } +} + +impl Default for MistralParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for MistralParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains Mistral format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + // Extract JSON array from Mistral format + if let Some(json_array) = self.extract_json_array(text) { + self.parse_json_array(json_array) + } else { + // Markers present but no complete array found + Ok(vec![]) + } + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check if we have the start marker + if !self.has_tool_markers(&state.buffer) { + return Ok(StreamResult::Incomplete); + } + + // Try to extract complete JSON array + if let Some(json_array) = self.extract_json_array(&state.buffer) { + // Parse with partial JSON to handle incomplete content + match self.partial_json.parse_value(json_array) { + Ok((value, consumed)) => { + // Check if we have a complete JSON structure + if consumed == json_array.len() { + // Complete JSON, parse tool calls + let tools = if let Value::Array(arr) = value { + let mut result = Vec::new(); + for (index, item) in arr.iter().enumerate() { + if let Some(tool) = self.parse_single_object(item, index)? { + result.push(tool); + } + } + result + } else { + vec![] + }; + + if !tools.is_empty() { + // Clear buffer since we consumed everything + state.buffer.clear(); + + // Return the first tool (simplified for Phase 3) + // Full multi-tool streaming will be implemented later + if let Some(tool) = tools.into_iter().next() { + return Ok(StreamResult::ToolComplete(tool)); + } + } + } else { + // Partial JSON - try to extract tool name for streaming + if let Value::Array(arr) = value { + if let Some(first_tool) = arr.first() { + if let Some(name) = first_tool.get("name").and_then(|v| v.as_str()) + { + // Check if we've already sent the name + if !state.in_string { + state.in_string = true; // Use as flag for "name sent" + return Ok(StreamResult::ToolName { + index: 0, + name: name.to_string(), + }); + } + + // Check for arguments + if let Some(args) = first_tool.get("arguments") { + if let Ok(args_str) = serde_json::to_string(args) { + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + } + } + } + } + } + } + Err(_) => { + // Failed to parse even as partial JSON + // Keep buffering + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + // Check if text contains Mistral-specific markers + if self.has_tool_markers(text) { + // Try to extract and validate the array + if let Some(json_array) = self.extract_json_array(text) { + // Check if it's valid JSON + if let Ok(value) = serde_json::from_str::(json_array) { + // Check if it contains tool-like structures + match value { + Value::Array(ref arr) => arr.iter().any(|v| { + v.as_object().is_some_and(|o| { + o.contains_key("name") && o.contains_key("arguments") + }) + }), + Value::Object(ref obj) => { + obj.contains_key("name") && obj.contains_key("arguments") + } + _ => false, + } + } else { + false + } + } else { + // Has markers but no complete array - might be streaming + true + } + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_mistral_format() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "Paris", "units": "celsius"}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Paris")); + } + + #[tokio::test] + async fn test_parse_multiple_tools() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [ + {"name": "search", "arguments": {"query": "rust programming"}}, + {"name": "calculate", "arguments": {"expression": "2 + 2"}} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "calculate"); + } + + #[tokio::test] + async fn test_nested_brackets_in_json() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "process", "arguments": {"data": [1, 2, [3, 4]], "config": {"nested": [5, 6]}}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process"); + // JSON serialization removes spaces, so check for [3,4] without spaces + assert!(result[0].function.arguments.contains("[3,4]")); + } + + #[tokio::test] + async fn test_escaped_quotes_in_strings() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"message": "He said \"Hello [World]\""}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "echo"); + } + + #[test] + fn test_detect_format() { + let parser = MistralParser::new(); + + assert!(parser.detect_format(r#"[TOOL_CALLS] [{"name": "test", "arguments": {}}]"#)); + assert!( + parser.detect_format(r#"Some text [TOOL_CALLS] [{"name": "test", "arguments": {}}]"#) + ); + assert!(!parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(!parser.detect_format("plain text")); + } +} diff --git a/sgl-router/src/tool_parser/parsers/mod.rs b/sgl-router/src/tool_parser/parsers/mod.rs new file mode 100644 index 000000000..693aeedf4 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/mod.rs @@ -0,0 +1,27 @@ +/// Parser implementations for different model formats +/// +/// This module contains concrete parser implementations for various model-specific +/// tool/function call formats. +// Individual parser modules +pub mod deepseek_parser; +pub mod glm4_moe_parser; +pub mod gpt_oss_parser; +pub mod json_parser; +pub mod kimik2_parser; +pub mod llama_parser; +pub mod mistral_parser; +pub mod pythonic_parser; +pub mod qwen_parser; +pub mod step3_parser; + +// Re-export parser types for convenience +pub use deepseek_parser::DeepSeekParser; +pub use glm4_moe_parser::Glm4MoeParser; +pub use gpt_oss_parser::GptOssParser; +pub use json_parser::JsonParser; +pub use kimik2_parser::KimiK2Parser; +pub use llama_parser::LlamaParser; +pub use mistral_parser::MistralParser; +pub use pythonic_parser::PythonicParser; +pub use qwen_parser::QwenParser; +pub use step3_parser::Step3Parser; diff --git a/sgl-router/src/tool_parser/parsers/pythonic_parser.rs b/sgl-router/src/tool_parser/parsers/pythonic_parser.rs new file mode 100644 index 000000000..6e0a1ecff --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/pythonic_parser.rs @@ -0,0 +1,434 @@ +/// Pythonic format parser for tool calls +/// +/// Handles Python function call syntax within square brackets: +/// ```text +/// [tool1(arg1=val1, arg2=val2), tool2(arg1=val3)] +/// ``` +/// +/// This format is used by Llama-4 models and uses Python literals +/// rather than JSON for arguments. +use async_trait::async_trait; +use regex::Regex; +use serde_json::{json, Value}; + +use crate::tool_parser::{ + errors::ToolParserResult, + python_literal_parser::parse_python_literal, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// Parser for Pythonic tool call format +pub struct PythonicParser { + /// Regex to detect tool calls in Pythonic format + tool_call_regex: Regex, + /// Regex to parse function calls - cached for reuse + call_regex: Regex, +} + +impl PythonicParser { + /// Create a new Pythonic parser + pub fn new() -> Self { + // Simple regex to detect start of Pythonic tool calls + // We'll use manual parsing for the actual extraction + let pattern = r"\[[a-zA-Z_]\w*\("; + let tool_call_regex = Regex::new(pattern).expect("Valid regex pattern"); + + // Compile the function call regex once + let call_regex = Regex::new(r"(?s)^([a-zA-Z_]\w*)\((.*)\)$").expect("Valid regex pattern"); + + Self { + tool_call_regex, + call_regex, + } + } + + /// Extract tool calls using bracket counting (similar to MistralParser) + fn extract_tool_calls(&self, text: &str) -> Option { + // Find the start of a tool call list - look for [ followed by a function name + let chars: Vec = text.chars().collect(); + + for start_idx in 0..chars.len() { + if chars[start_idx] != '[' { + continue; + } + + // Check if this looks like a tool call + // Skip whitespace after [ + let mut check_idx = start_idx + 1; + while check_idx < chars.len() && chars[check_idx].is_whitespace() { + check_idx += 1; + } + + // Check if we have a function name (starts with letter or underscore) + if check_idx >= chars.len() + || (!chars[check_idx].is_alphabetic() && chars[check_idx] != '_') + { + continue; + } + + // Now count brackets to find the matching ] + let mut bracket_count = 0; + let mut _paren_count = 0; + let mut _brace_count = 0; + let mut in_string = false; + let mut string_char = ' '; + let mut escape_next = false; + + for i in start_idx..chars.len() { + let ch = chars[i]; + + if escape_next { + escape_next = false; + continue; + } + + if ch == '\\' && in_string { + escape_next = true; + continue; + } + + if !in_string && (ch == '"' || ch == '\'') { + in_string = true; + string_char = ch; + } else if in_string && ch == string_char && !escape_next { + in_string = false; + } else if !in_string { + match ch { + '[' => bracket_count += 1, + ']' => { + bracket_count -= 1; + if bracket_count == 0 { + // Found the matching bracket + let extracted: String = chars[start_idx..=i].iter().collect(); + // Verify this actually contains a function call + if extracted.contains('(') && extracted.contains(')') { + return Some(extracted); + } + } + } + '(' => _paren_count += 1, + ')' => _paren_count -= 1, + '{' => _brace_count += 1, + '}' => _brace_count -= 1, + _ => {} + } + } + } + } + None + } + + /// Strip special tokens that Llama 4 might output + fn strip_special_tokens(text: &str) -> String { + text.replace("<|python_start|>", "") + .replace("<|python_end|>", "") + } + + /// Parse a single function call from Python syntax + fn parse_function_call(&self, call_str: &str) -> ToolParserResult> { + // Use cached regex instead of creating new one + if let Some(captures) = self.call_regex.captures(call_str.trim()) { + let function_name = captures.get(1).unwrap().as_str(); + let args_str = captures.get(2).unwrap().as_str(); + + // Parse arguments + let arguments = self.parse_arguments(args_str)?; + + Ok(Some(ToolCall { + id: format!("call_{}", uuid::Uuid::new_v4()), + r#type: "function".to_string(), + function: FunctionCall { + name: function_name.to_string(), + arguments: serde_json::to_string(&arguments)?, + }, + })) + } else { + Ok(None) + } + } + + /// Parse Python-style arguments into JSON + fn parse_arguments(&self, args_str: &str) -> ToolParserResult { + if args_str.trim().is_empty() { + return Ok(json!({})); + } + + let mut result = serde_json::Map::new(); + let mut current_key = String::new(); + let mut current_value = String::new(); + let mut in_key = true; + let mut depth = 0; + let mut in_string = false; + let mut string_char = ' '; + let mut escape_next = false; + + let chars: Vec = args_str.chars().collect(); + let mut i = 0; + + while i < chars.len() { + let ch = chars[i]; + + if escape_next { + if in_key { + current_key.push(ch); + } else { + current_value.push(ch); + } + escape_next = false; + i += 1; + continue; + } + + if ch == '\\' && in_string { + escape_next = true; + current_value.push(ch); + i += 1; + continue; + } + + // Handle string literals + if !in_string && (ch == '"' || ch == '\'') { + in_string = true; + string_char = ch; + if !in_key { + current_value.push(ch); + } + } else if in_string && ch == string_char && !escape_next { + in_string = false; + if !in_key { + current_value.push(ch); + } + } else if in_string { + if in_key { + current_key.push(ch); + } else { + current_value.push(ch); + } + } else { + // Not in string + match ch { + '=' if in_key && depth == 0 => { + in_key = false; + } + ',' if depth == 0 => { + // End of current argument + if !current_key.is_empty() { + let value = parse_python_literal(current_value.trim())?; + result.insert(current_key.trim().to_string(), value); + } + current_key.clear(); + current_value.clear(); + in_key = true; + } + '[' | '{' | '(' => { + depth += 1; + if !in_key { + current_value.push(ch); + } + } + ']' | '}' | ')' => { + depth -= 1; + if !in_key { + current_value.push(ch); + } + } + _ => { + if in_key { + if !ch.is_whitespace() || !current_key.is_empty() { + current_key.push(ch); + } + } else { + current_value.push(ch); + } + } + } + } + + i += 1; + } + + // Handle the last argument + if !current_key.is_empty() { + let value = parse_python_literal(current_value.trim())?; + result.insert(current_key.trim().to_string(), value); + } + + Ok(Value::Object(result)) + } +} + +#[async_trait] +impl ToolParser for PythonicParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + let cleaned = Self::strip_special_tokens(text); + + // Extract tool calls using bracket counting + if let Some(tool_calls_text) = self.extract_tool_calls(&cleaned) { + // Remove the outer brackets + let tool_calls_str = &tool_calls_text[1..tool_calls_text.len() - 1]; + + // Split into individual function calls + let mut calls = Vec::new(); + let mut current_call = String::new(); + let mut paren_depth = 0; + let mut in_string = false; + let mut string_char = ' '; + + for ch in tool_calls_str.chars() { + if !in_string && (ch == '"' || ch == '\'') { + in_string = true; + string_char = ch; + current_call.push(ch); + } else if in_string && ch == string_char { + in_string = false; + current_call.push(ch); + } else if in_string { + current_call.push(ch); + } else { + match ch { + '(' => { + paren_depth += 1; + current_call.push(ch); + } + ')' => { + paren_depth -= 1; + current_call.push(ch); + } + ',' if paren_depth == 0 => { + // End of current function call + if let Some(call) = self.parse_function_call(current_call.trim())? { + calls.push(call); + } + current_call.clear(); + } + _ => { + if !ch.is_whitespace() || !current_call.is_empty() { + current_call.push(ch); + } + } + } + } + } + + // Handle the last call (important for single calls or the last call in a list) + if !current_call.trim().is_empty() { + if let Some(call) = self.parse_function_call(current_call.trim())? { + calls.push(call); + } + } + + Ok(calls) + } else { + Ok(vec![]) + } + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + // For Pythonic format, we accumulate until we have a complete tool call + // This is a simplified implementation + state.buffer.push_str(chunk); + + // Try to parse if we have a complete tool call + let cleaned = Self::strip_special_tokens(&state.buffer); + if self.extract_tool_calls(&cleaned).is_some() { + let result = self.parse_complete(&state.buffer).await?; + if !result.is_empty() { + state.buffer.clear(); + return Ok(StreamResult::ToolComplete( + result.into_iter().next().unwrap(), + )); + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + let cleaned = Self::strip_special_tokens(text); + self.tool_call_regex.is_match(&cleaned) + } +} + +impl Default for PythonicParser { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_single_function_call() { + let parser = PythonicParser::new(); + let input = r#"[search_web(query="Rust programming", max_results=5)]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search_web"); + + let args: Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "Rust programming"); + assert_eq!(args["max_results"], 5); + } + + #[tokio::test] + async fn test_multiple_function_calls() { + let parser = PythonicParser::new(); + let input = r#"[get_weather(city="Tokyo"), search(query="news")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "search"); + } + + #[tokio::test] + async fn test_python_literals() { + let parser = PythonicParser::new(); + let input = r#"[test(flag=True, disabled=False, optional=None)]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["flag"], true); + assert_eq!(args["disabled"], false); + assert_eq!(args["optional"], Value::Null); + } + + #[tokio::test] + async fn test_special_tokens() { + let parser = PythonicParser::new(); + let input = r#"<|python_start|>[calculate(x=10, y=20)]<|python_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + + let args: Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["x"], 10); + assert_eq!(args["y"], 20); + } + + #[tokio::test] + async fn test_llama4_format() { + let parser = PythonicParser::new(); + let input = r#"[get_weather(city="London", units="celsius")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["city"], "London"); + assert_eq!(args["units"], "celsius"); + } +} diff --git a/sgl-router/src/tool_parser/parsers/qwen_parser.rs b/sgl-router/src/tool_parser/parsers/qwen_parser.rs new file mode 100644 index 000000000..29ad2083c --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/qwen_parser.rs @@ -0,0 +1,396 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + partial_json::PartialJson, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// Qwen format parser for tool calls +/// +/// Handles the Qwen 2.5/3 specific format: +/// `\n{"name": "func", "arguments": {...}}\n` +/// +/// Features: +/// - XML-style tags with JSON content +/// - Support for multiple sequential tool calls +/// - Newline-aware parsing +pub struct QwenParser { + /// Parser for handling incomplete JSON during streaming + partial_json: PartialJson, + /// Regex for extracting tool calls + extractor: Regex, +} + +impl QwenParser { + /// Create a new Qwen parser + pub fn new() -> Self { + // Use (?s) flag for DOTALL mode to handle newlines + let pattern = r"(?s)\n(.*?)\n"; + let extractor = Regex::new(pattern).expect("Valid regex pattern"); + + Self { + partial_json: PartialJson::default(), + extractor, + } + } + + /// Extract all tool call blocks from text + fn extract_tool_calls<'a>(&self, text: &'a str) -> Vec<&'a str> { + self.extractor + .captures_iter(text) + .filter_map(|cap| cap.get(1).map(|m| m.as_str())) + .collect() + } + + /// Parse a single JSON object into a ToolCall + fn parse_single_object(&self, obj: &Value, index: usize) -> ToolParserResult> { + let name = obj.get("name").and_then(|v| v.as_str()); + + if let Some(name) = name { + // Get arguments - Qwen uses "arguments" key + let empty_obj = Value::Object(serde_json::Map::new()); + let args = obj.get("arguments").unwrap_or(&empty_obj); + + // Convert arguments to JSON string + let arguments = serde_json::to_string(args) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate ID with index for multiple tools + let id = format!("qwen_call_{}", index); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: name.to_string(), + arguments, + }, + })) + } else { + Ok(None) + } + } + + /// Check if text contains Qwen tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("") + } + + /// Find the start position of a tool call + fn find_tool_start(&self, text: &str) -> Option { + text.find("\n") + } + + /// Find the end position of a tool call + fn find_tool_end(&self, text: &str, start_pos: usize) -> Option { + let search_from = start_pos + "\n".len(); + text[search_from..] + .find("\n") + .map(|pos| search_from + pos + "\n".len()) + } + + /// Check if buffer ends with a partial token + fn ends_with_partial_token(&self, buffer: &str) -> Option { + // Check for partial start token + let start_token = "\n"; + // Use inclusive range to check if entire buffer could be a prefix + for i in 1..=start_token.len().min(buffer.len()) { + if start_token.starts_with(&buffer[buffer.len() - i..]) { + return Some(i); + } + } + + // Check for partial end token + let end_token = "\n"; + // Only check if buffer ends with a partial match (not the complete token without newline) + // If buffer ends with "", that's not a partial token - it's missing the newline + if buffer.ends_with("") { + // This is a complete end tag, just missing the leading newline + // Not a partial token situation + return None; + } + // Use inclusive range to check if entire buffer could be a prefix + (1..=end_token.len().min(buffer.len())) + .find(|&i| end_token.starts_with(&buffer[buffer.len() - i..])) + } +} + +impl Default for QwenParser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for QwenParser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains Qwen format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + // Extract all tool call blocks + let tool_blocks = self.extract_tool_calls(text); + let mut tools = Vec::new(); + + for (index, json_str) in tool_blocks.iter().enumerate() { + // Parse each JSON block + match serde_json::from_str::(json_str.trim()) { + Ok(value) => { + if let Some(tool) = self.parse_single_object(&value, index)? { + tools.push(tool); + } + } + Err(_) => { + // Skip malformed JSON blocks + continue; + } + } + } + + Ok(tools) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for partial token at end of buffer + if let Some(_partial_len) = self.ends_with_partial_token(&state.buffer) { + // Hold back the partial token + return Ok(StreamResult::Incomplete); + } + + // Check if we have the start marker + if !self.has_tool_markers(&state.buffer) { + return Ok(StreamResult::Incomplete); + } + + // Find start and end positions + if let Some(start_pos) = self.find_tool_start(&state.buffer) { + // Check if we have the complete tool call + if let Some(end_pos) = self.find_tool_end(&state.buffer, start_pos) { + // Extract the JSON content + let json_start = start_pos + "\n".len(); + let json_end = end_pos - "\n".len(); + let json_str = &state.buffer[json_start..json_end]; + + // Parse the complete JSON + match serde_json::from_str::(json_str.trim()) { + Ok(value) => { + if let Some(tool) = self.parse_single_object(&value, 0)? { + // Clear the consumed part from buffer using drain for efficiency + state.buffer.drain(..end_pos); + return Ok(StreamResult::ToolComplete(tool)); + } + } + Err(_) => { + // JSON parsing failed, might be incomplete + } + } + } else { + // We have start but no end yet - try partial parsing + let json_start = start_pos + "\n".len(); + let partial_json = &state.buffer[json_start..]; + + // Remove trailing newline if present (might be start of end token) + let partial_json = partial_json.trim_end(); + + // Try to parse with partial JSON parser + match self.partial_json.parse_value(partial_json) { + Ok((value, _consumed)) => { + // Extract tool name if available + if let Some(name) = value.get("name").and_then(|v| v.as_str()) { + // Check if we've already sent the name + if !state.in_string { + state.in_string = true; // Use as flag for "name sent" + return Ok(StreamResult::ToolName { + index: 0, + name: name.to_string(), + }); + } + + // Check for arguments + if let Some(args) = value.get("arguments") { + if let Ok(args_str) = serde_json::to_string(args) { + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + } + } + } + Err(_) => { + // Failed to parse even as partial JSON + // Keep buffering + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + // Check if text contains Qwen-specific markers. If not, it's not this format. + if !self.has_tool_markers(text) { + return false; + } + + // Try to extract tool calls to see if we have a complete, valid one. + let tool_blocks = self.extract_tool_calls(text); + for json_str in &tool_blocks { + if let Ok(value) = serde_json::from_str::(json_str.trim()) { + if let Some(obj) = value.as_object() { + if obj.contains_key("name") && obj.contains_key("arguments") { + // Found a valid, complete tool call. + return true; + } + } + } + } + + // If we have the marker but no valid complete tool call, + // it could be a partial stream. We should detect this as the format. + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_qwen_format() { + let parser = QwenParser::new(); + let input = r#" +{"name": "get_weather", "arguments": {"location": "Beijing", "units": "celsius"}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Beijing")); + } + + #[tokio::test] + async fn test_parse_multiple_tools() { + let parser = QwenParser::new(); + let input = r#" +{"name": "search", "arguments": {"query": "rust programming"}} + + +{"name": "calculate", "arguments": {"expression": "2 + 2"}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "calculate"); + } + + #[tokio::test] + async fn test_with_normal_text() { + let parser = QwenParser::new(); + let input = r#"Let me help you with that. + +{"name": "get_info", "arguments": {"topic": "Rust"}} + +Here are the results."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_info"); + } + + #[tokio::test] + async fn test_nested_json_structures() { + let parser = QwenParser::new(); + let input = r#" +{ + "name": "process_data", + "arguments": { + "data": { + "nested": { + "array": [1, 2, 3], + "object": {"key": "value"} + } + } + } +} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process_data"); + assert!(result[0].function.arguments.contains("nested")); + } + + #[test] + fn test_detect_format() { + let parser = QwenParser::new(); + + assert!(parser.detect_format( + r#" +{"name": "test", "arguments": {}} +"# + )); + + assert!(parser.detect_format( + r#"Text before +{"name": "test", "arguments": {}} + text after"# + )); + + assert!(!parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(!parser.detect_format("plain text")); + + // Partial format should still be detected + assert!(parser.detect_format("")); + } + + #[tokio::test] + async fn test_streaming_partial() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "\n", + r#"{"name": "search","#, + r#" "arguments": {"query":"#, + r#" "rust"}}"#, + "\n", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "search"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "search"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); // At least one should be found + } +} diff --git a/sgl-router/src/tool_parser/parsers/step3_parser.rs b/sgl-router/src/tool_parser/parsers/step3_parser.rs new file mode 100644 index 000000000..721d5c037 --- /dev/null +++ b/sgl-router/src/tool_parser/parsers/step3_parser.rs @@ -0,0 +1,348 @@ +use async_trait::async_trait; +use regex::Regex; +use serde_json::Value; + +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + state::ParseState, + traits::ToolParser, + types::{FunctionCall, StreamResult, ToolCall}, +}; + +/// Step3 format parser for tool calls +/// +/// Handles the Step3 specific format with steptml XML: +/// `<｜tool_calls_begin｜><｜tool_call_begin｜>function<｜tool_sep｜>{v}<｜tool_call_end｜><｜tool_calls_end｜>` +/// +/// Features: +/// - Unicode token delimiters +/// - StepTML XML format for invocations +/// - Support for multiple sequential tool calls +pub struct Step3Parser { + /// Regex for extracting tool call blocks + tool_call_extractor: Regex, + /// Regex for extracting steptml invocations + invoke_extractor: Regex, + /// Regex for extracting parameters + param_extractor: Regex, +} + +impl Step3Parser { + /// Create a new Step3 parser + pub fn new() -> Self { + // Pattern for individual tool calls + let tool_call_pattern = r"(?s)<｜tool_call_begin｜>.*?<｜tool_call_end｜>"; + let tool_call_extractor = Regex::new(tool_call_pattern).expect("Valid regex pattern"); + + // Pattern for steptml invocations + let invoke_pattern = r#"(?s)(.+?)"#; + let invoke_extractor = Regex::new(invoke_pattern).expect("Valid regex pattern"); + + // Pattern for steptml parameters - using non-greedy match for values to handle < characters + let param_pattern = r#"(?s)(.+?)"#; + let param_extractor = Regex::new(param_pattern).expect("Valid regex pattern"); + + Self { + tool_call_extractor, + invoke_extractor, + param_extractor, + } + } + + /// Check if text contains Step3 tool markers + fn has_tool_markers(&self, text: &str) -> bool { + text.contains("<｜tool_calls_begin｜>") + } + + /// Parse parameters from steptml format + fn parse_steptml_parameters( + &self, + params_text: &str, + ) -> ToolParserResult> { + let mut parameters = serde_json::Map::new(); + + for capture in self.param_extractor.captures_iter(params_text) { + let param_name = capture.get(1).map_or("", |m| m.as_str()).trim(); + let param_value_str = capture.get(2).map_or("", |m| m.as_str()).trim(); + + // Try to parse the value as JSON first, fallback to string + let param_value = if let Ok(json_val) = serde_json::from_str::(param_value_str) { + json_val + } else { + // Try parsing as Python literal + if param_value_str == "true" || param_value_str == "True" { + Value::Bool(true) + } else if param_value_str == "false" || param_value_str == "False" { + Value::Bool(false) + } else if param_value_str == "null" || param_value_str == "None" { + Value::Null + } else if let Ok(num) = param_value_str.parse::() { + Value::Number(num.into()) + } else if let Ok(num) = param_value_str.parse::() { + if let Some(n) = serde_json::Number::from_f64(num) { + Value::Number(n) + } else { + Value::String(param_value_str.to_string()) + } + } else { + Value::String(param_value_str.to_string()) + } + }; + + parameters.insert(param_name.to_string(), param_value); + } + + Ok(parameters) + } + + /// Parse a single tool call block + fn parse_tool_call(&self, block: &str) -> ToolParserResult> { + // Check if it contains function marker and tool separator + if !block.contains("function") || !block.contains("<｜tool_sep｜>") { + return Ok(None); + } + + // Split by tool separator + let parts: Vec<&str> = block.split("<｜tool_sep｜>").collect(); + if parts.len() != 2 { + return Ok(None); + } + + // Check if it's a function type + if !parts[0].contains("function") { + return Ok(None); + } + + let invoke_part = parts[1]; + + // Extract steptml invoke + if let Some(captures) = self.invoke_extractor.captures(invoke_part) { + let func_name = captures.get(1).map_or("", |m| m.as_str()).trim(); + + // Validate function name is not empty + if func_name.is_empty() { + return Ok(None); + } + + let params_text = captures.get(2).map_or("", |m| m.as_str()); + + // Parse parameters + let parameters = self.parse_steptml_parameters(params_text)?; + + let arguments_str = serde_json::to_string(¶meters) + .map_err(|e| ToolParserError::ParsingFailed(e.to_string()))?; + + // Generate ID + let id = format!("step3_call_{}", uuid::Uuid::new_v4()); + + Ok(Some(ToolCall { + id, + r#type: "function".to_string(), + function: FunctionCall { + name: func_name.to_string(), + arguments: arguments_str, + }, + })) + } else { + Ok(None) + } + } +} + +impl Default for Step3Parser { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl ToolParser for Step3Parser { + async fn parse_complete(&self, text: &str) -> ToolParserResult> { + // Check if text contains Step3 format + if !self.has_tool_markers(text) { + return Ok(vec![]); + } + + // Find the tool calls section + if let Some(start_pos) = text.find("<｜tool_calls_begin｜>") { + let search_from = start_pos + "<｜tool_calls_begin｜>".len(); + + // Find the end of tool calls section + if let Some(end_pos) = text[search_from..].find("<｜tool_calls_end｜>") { + let tool_section = &text[search_from..search_from + end_pos]; + + // Extract all tool call blocks + let mut tools = Vec::new(); + for mat in self.tool_call_extractor.find_iter(tool_section) { + if let Some(tool) = self.parse_tool_call(mat.as_str())? { + tools.push(tool); + } + } + + return Ok(tools); + } + } + + Ok(vec![]) + } + + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult { + state.buffer.push_str(chunk); + + // Check for tool markers + if !self.has_tool_markers(&state.buffer) { + // No markers found, return as incomplete + return Ok(StreamResult::Incomplete); + } + + // Look for start of tool calls + if let Some(start_pos) = state.buffer.find("<｜tool_calls_begin｜>") { + let search_from = start_pos + "<｜tool_calls_begin｜>".len(); + + // Look for individual tool call start + if let Some(call_start) = state.buffer[search_from..].find("<｜tool_call_begin｜>") { + let call_start_abs = search_from + call_start; + + // Look for the end of this tool call + let search_end_from = call_start_abs + "<｜tool_call_begin｜>".len(); + if let Some(call_end) = state.buffer[search_end_from..].find("<｜tool_call_end｜>") + { + let call_end_abs = search_end_from + call_end + "<｜tool_call_end｜>".len(); + + // Extract and parse the complete tool call + let tool_call_text = &state.buffer[call_start_abs..call_end_abs]; + + if let Some(tool) = self.parse_tool_call(tool_call_text)? { + // Remove the processed part from buffer + state.buffer.drain(..call_end_abs); + + return Ok(StreamResult::ToolComplete(tool)); + } + } else { + // Tool call not complete yet, try to extract partial info + let partial = &state.buffer[search_end_from..]; + + // Check for tool separator + if let Some(sep_pos) = partial.find("<｜tool_sep｜>") { + // Check if it's a function + if partial[..sep_pos].contains("function") { + let after_sep = &partial[sep_pos + "<｜tool_sep｜>".len()..]; + + // Try to extract function name from steptml:invoke + if let Some(name_match) = self.invoke_extractor.captures(after_sep) { + let func_name = name_match.get(1).map_or("", |m| m.as_str()).trim(); + + if !state.in_string && !func_name.is_empty() { + state.in_string = true; // Mark name as sent + return Ok(StreamResult::ToolName { + index: 0, + name: func_name.to_string(), + }); + } + + // Try to extract partial parameters + if let Some(params_text) = name_match.get(2) { + let parameters = + self.parse_steptml_parameters(params_text.as_str())?; + + if !parameters.is_empty() { + let args_str = serde_json::to_string(¶meters) + .unwrap_or_else(|_| "{}".to_string()); + + return Ok(StreamResult::ToolArguments { + index: 0, + arguments: args_str, + }); + } + } + } + } + } + } + } + } + + Ok(StreamResult::Incomplete) + } + + fn detect_format(&self, text: &str) -> bool { + self.has_tool_markers(text) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_parse_step3_single_tool() { + let parser = Step3Parser::new(); + let input = r#"Some text +<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +Tokyo +celsius +<｜tool_call_end｜> +<｜tool_calls_end｜>More text"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("Tokyo")); + assert!(result[0].function.arguments.contains("celsius")); + } + + #[tokio::test] + async fn test_parse_step3_multiple_tools() { + let parser = Step3Parser::new(); + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +rust programming +<｜tool_call_end｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +2 + 2 +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "calculate"); + } + + #[tokio::test] + async fn test_parse_step3_mixed_types() { + let parser = Step3Parser::new(); + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +42 +true +1.5 +test +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process_data"); + + // Parse arguments to check types + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["count"], 42); + assert_eq!(args["active"], true); + assert_eq!(args["rate"], 1.5); + assert_eq!(args["name"], "test"); + } + + #[test] + fn test_detect_format() { + let parser = Step3Parser::new(); + assert!(parser.detect_format("<｜tool_calls_begin｜>")); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format("[TOOL_CALLS]")); + } +} diff --git a/sgl-router/src/tool_parser/partial_json.rs b/sgl-router/src/tool_parser/partial_json.rs new file mode 100644 index 000000000..4a4504fe0 --- /dev/null +++ b/sgl-router/src/tool_parser/partial_json.rs @@ -0,0 +1,527 @@ +use crate::tool_parser::{ + errors::{ToolParserError, ToolParserResult}, + traits::PartialJsonParser, +}; +use serde_json::{Map, Value}; + +/// Parser for incomplete JSON +pub struct PartialJson { + /// Maximum depth for nested structures + max_depth: usize, + /// Whether to allow incomplete values + allow_incomplete: bool, +} + +impl PartialJson { + /// Create a new partial JSON parser + pub fn new(max_depth: usize, allow_incomplete: bool) -> Self { + Self { + max_depth, + allow_incomplete, + } + } + + /// Parse potentially incomplete JSON, returning parsed value and consumed bytes + pub fn parse_value(&self, input: &str) -> ToolParserResult<(Value, usize)> { + let mut parser = Parser::new(input, self.max_depth, self.allow_incomplete); + let value = parser.parse_value(0)?; + Ok((value, parser.position)) + } +} + +impl Default for PartialJson { + fn default() -> Self { + Self::new(32, true) + } +} + +impl PartialJsonParser for PartialJson { + fn parse(&self, input: &str) -> ToolParserResult<(Value, usize)> { + self.parse_value(input) + } + + fn is_complete(&self, input: &str) -> bool { + // Try to parse as complete JSON + serde_json::from_str::(input).is_ok() + } + + fn max_depth(&self) -> usize { + self.max_depth + } +} + +/// Internal parser state +struct Parser<'a> { + chars: std::iter::Peekable>, + position: usize, + max_depth: usize, + allow_incomplete: bool, +} + +impl<'a> Parser<'a> { + fn new(input: &'a str, max_depth: usize, allow_incomplete: bool) -> Self { + Self { + chars: input.chars().peekable(), + position: 0, + max_depth, + allow_incomplete, + } + } + + fn peek(&mut self) -> Option { + self.chars.peek().copied() + } + + fn advance(&mut self) { + if self.chars.next().is_some() { + self.position += 1; + } + } + + fn skip_whitespace(&mut self) { + while let Some(ch) = self.peek() { + if ch.is_whitespace() { + self.advance(); + } else { + break; + } + } + } + + fn parse_value(&mut self, depth: usize) -> ToolParserResult { + if depth > self.max_depth { + return Err(ToolParserError::DepthExceeded(self.max_depth)); + } + + self.skip_whitespace(); + + match self.peek() { + Some('{') => self.parse_object(depth + 1), + Some('[') => self.parse_array(depth + 1), + Some('"') => self.parse_string(), + Some('t') | Some('f') => self.parse_bool(), + Some('n') => self.parse_null(), + Some(c) if c == '-' || c.is_ascii_digit() => self.parse_number(), + _ => { + if self.allow_incomplete { + Ok(Value::Null) + } else { + Err(ToolParserError::ParsingFailed( + "Unexpected character".into(), + )) + } + } + } + } + + fn parse_object(&mut self, depth: usize) -> ToolParserResult { + if depth > self.max_depth { + return Err(ToolParserError::DepthExceeded(self.max_depth)); + } + + let mut object = Map::new(); + + // Consume '{' + self.advance(); + self.skip_whitespace(); + + // Check for empty object + if self.peek() == Some('}') { + self.advance(); + return Ok(Value::Object(object)); + } + + loop { + // Parse key + let key = match self.parse_string() { + Ok(Value::String(s)) => s, + Err(_) if self.allow_incomplete => { + // Incomplete object + return Ok(Value::Object(object)); + } + Err(e) => return Err(e), + _ => return Err(ToolParserError::ParsingFailed("Expected string key".into())), + }; + + self.skip_whitespace(); + + // Expect ':' + if self.peek() != Some(':') { + if self.allow_incomplete { + // Add null value for incomplete pair + object.insert(key, Value::Null); + return Ok(Value::Object(object)); + } + return Err(ToolParserError::ParsingFailed("Expected ':'".into())); + } + self.advance(); + self.skip_whitespace(); + + // Parse value (keep same depth - we already incremented in parse_object) + let value = match self.parse_value(depth) { + Ok(v) => v, + Err(_) if self.allow_incomplete => { + // Add null for incomplete value + object.insert(key, Value::Null); + return Ok(Value::Object(object)); + } + Err(e) => return Err(e), + }; + + object.insert(key, value); + self.skip_whitespace(); + + match self.peek() { + Some(',') => { + self.advance(); + self.skip_whitespace(); + // Check for trailing comma + if self.peek() == Some('}') { + self.advance(); + return Ok(Value::Object(object)); + } + } + Some('}') => { + self.advance(); + return Ok(Value::Object(object)); + } + None if self.allow_incomplete => { + return Ok(Value::Object(object)); + } + _ => { + if self.allow_incomplete { + return Ok(Value::Object(object)); + } + return Err(ToolParserError::ParsingFailed("Expected ',' or '}'".into())); + } + } + } + } + + fn parse_array(&mut self, depth: usize) -> ToolParserResult { + if depth > self.max_depth { + return Err(ToolParserError::DepthExceeded(self.max_depth)); + } + + let mut array = Vec::new(); + + // Consume '[' + self.advance(); + self.skip_whitespace(); + + // Check for empty array + if self.peek() == Some(']') { + self.advance(); + return Ok(Value::Array(array)); + } + + loop { + // Parse value (keep same depth - we already incremented in parse_object) + let value = match self.parse_value(depth) { + Ok(v) => v, + Err(_) if self.allow_incomplete => { + return Ok(Value::Array(array)); + } + Err(e) => return Err(e), + }; + + array.push(value); + self.skip_whitespace(); + + match self.peek() { + Some(',') => { + self.advance(); + self.skip_whitespace(); + // Check for trailing comma + if self.peek() == Some(']') { + self.advance(); + return Ok(Value::Array(array)); + } + } + Some(']') => { + self.advance(); + return Ok(Value::Array(array)); + } + None if self.allow_incomplete => { + return Ok(Value::Array(array)); + } + _ => { + if self.allow_incomplete { + return Ok(Value::Array(array)); + } + return Err(ToolParserError::ParsingFailed("Expected ',' or ']'".into())); + } + } + } + } + + fn parse_string(&mut self) -> ToolParserResult { + if self.peek() != Some('"') { + return Err(ToolParserError::ParsingFailed("Expected '\"'".into())); + } + + // Consume opening quote + self.advance(); + + let mut string = String::new(); + let mut escaped = false; + + while let Some(ch) = self.peek() { + if escaped { + // Handle escape sequences + let escaped_char = match ch { + '"' | '\\' | '/' => ch, + 'b' => '\u{0008}', + 'f' => '\u{000C}', + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'u' => { + // Unicode escape + self.advance(); + let hex = self.parse_unicode_escape()?; + string.push(hex); + escaped = false; + continue; + } + _ => ch, // Invalid escape, but be lenient + }; + string.push(escaped_char); + escaped = false; + } else if ch == '\\' { + escaped = true; + } else if ch == '"' { + // End of string + self.advance(); + return Ok(Value::String(string)); + } else { + string.push(ch); + } + self.advance(); + } + + // Incomplete string + if self.allow_incomplete { + Ok(Value::String(string)) + } else { + Err(ToolParserError::ParsingFailed("Unterminated string".into())) + } + } + + fn parse_unicode_escape(&mut self) -> ToolParserResult { + let mut hex = String::new(); + for _ in 0..4 { + if let Some(ch) = self.peek() { + if ch.is_ascii_hexdigit() { + hex.push(ch); + self.advance(); + } else { + break; + } + } else { + break; + } + } + + if hex.len() == 4 { + u32::from_str_radix(&hex, 16) + .ok() + .and_then(char::from_u32) + .ok_or_else(|| ToolParserError::ParsingFailed("Invalid unicode escape".into())) + } else if self.allow_incomplete { + Ok('\u{FFFD}') // Replacement character + } else { + Err(ToolParserError::ParsingFailed( + "Incomplete unicode escape".into(), + )) + } + } + + fn parse_number(&mut self) -> ToolParserResult { + let mut number = String::new(); + + // Handle negative sign + if self.peek() == Some('-') { + number.push('-'); + self.advance(); + } + + // Parse integer part + if self.peek() == Some('0') { + number.push('0'); + self.advance(); + } else { + while let Some(ch) = self.peek() { + if ch.is_ascii_digit() { + number.push(ch); + self.advance(); + } else { + break; + } + } + } + + // Parse decimal part + if self.peek() == Some('.') { + number.push('.'); + self.advance(); + + while let Some(ch) = self.peek() { + if ch.is_ascii_digit() { + number.push(ch); + self.advance(); + } else { + break; + } + } + } + + // Parse exponent + if let Some(ch) = self.peek() { + if ch == 'e' || ch == 'E' { + number.push(ch); + self.advance(); + + if let Some(sign) = self.peek() { + if sign == '+' || sign == '-' { + number.push(sign); + self.advance(); + } + } + + while let Some(ch) = self.peek() { + if ch.is_ascii_digit() { + number.push(ch); + self.advance(); + } else { + break; + } + } + } + } + + // Try to parse as integer first, then as float + if let Ok(n) = number.parse::() { + Ok(Value::Number(serde_json::Number::from(n))) + } else if let Ok(n) = number.parse::() { + Ok(Value::Number( + serde_json::Number::from_f64(n).unwrap_or_else(|| serde_json::Number::from(0)), + )) + } else if self.allow_incomplete { + Ok(Value::Number(serde_json::Number::from(0))) + } else { + Err(ToolParserError::ParsingFailed("Invalid number".into())) + } + } + + fn parse_bool(&mut self) -> ToolParserResult { + let mut word = String::new(); + + // Peek at upcoming characters to validate it looks like a boolean + let mut temp_chars = self.chars.clone(); + while let Some(&ch) = temp_chars.peek() { + if ch.is_alphabetic() && word.len() < 5 { + // "false" is 5 chars + word.push(ch); + temp_chars.next(); + } else { + break; + } + } + + // Check if it's a valid boolean prefix + let is_valid = word == "true" + || word == "false" + || (self.allow_incomplete && ("true".starts_with(&word) || "false".starts_with(&word))); + + if !is_valid { + return Err(ToolParserError::ParsingFailed("Invalid boolean".into())); + } + + // Now actually consume the characters + word.clear(); + while let Some(ch) = self.peek() { + if ch.is_alphabetic() { + word.push(ch); + self.advance(); + } else { + break; + } + } + + match word.as_str() { + "true" => Ok(Value::Bool(true)), + "false" => Ok(Value::Bool(false)), + partial if self.allow_incomplete => { + if "true".starts_with(partial) { + Ok(Value::Bool(true)) + } else if "false".starts_with(partial) { + Ok(Value::Bool(false)) + } else { + Err(ToolParserError::ParsingFailed("Invalid boolean".into())) + } + } + _ => Err(ToolParserError::ParsingFailed("Invalid boolean".into())), + } + } + + fn parse_null(&mut self) -> ToolParserResult { + let mut word = String::new(); + + // Peek at upcoming characters to validate it looks like "null" + let mut temp_chars = self.chars.clone(); + while let Some(&ch) = temp_chars.peek() { + if ch.is_alphabetic() && word.len() < 4 { + // "null" is 4 chars + word.push(ch); + temp_chars.next(); + } else { + break; + } + } + + // Check if it's a valid null prefix + let is_valid = word == "null" || (self.allow_incomplete && "null".starts_with(&word)); + + if !is_valid { + return Err(ToolParserError::ParsingFailed("Invalid null".into())); + } + + // Now actually consume the characters + word.clear(); + while let Some(ch) = self.peek() { + if ch.is_alphabetic() { + word.push(ch); + self.advance(); + } else { + break; + } + } + + if word == "null" || (self.allow_incomplete && "null".starts_with(&word)) { + Ok(Value::Null) + } else { + Err(ToolParserError::ParsingFailed("Invalid null".into())) + } + } +} + +/// Utility function to check if a string contains complete JSON +pub fn is_complete_json(input: &str) -> bool { + serde_json::from_str::(input).is_ok() +} + +/// Utility function to find common prefix between two strings +pub fn find_common_prefix(s1: &str, s2: &str) -> usize { + s1.chars() + .zip(s2.chars()) + .take_while(|(a, b)| a == b) + .count() +} + +/// Utility function to compute diff between old and new strings +pub fn compute_diff(old: &str, new: &str) -> String { + let common_len = find_common_prefix(old, new); + // Convert character count to byte offset + new.chars().skip(common_len).collect() +} diff --git a/sgl-router/src/tool_parser/python_literal_parser.rs b/sgl-router/src/tool_parser/python_literal_parser.rs new file mode 100644 index 000000000..4acc69d34 --- /dev/null +++ b/sgl-router/src/tool_parser/python_literal_parser.rs @@ -0,0 +1,442 @@ +/// Minimal Python literal parser for Pythonic tool call format +/// +/// This module provides a recursive descent parser for Python literals +/// (strings, numbers, booleans, None, lists, dicts) without requiring +/// a full Python AST parser. +use serde_json::{json, Value}; +use std::collections::HashMap; + +use crate::tool_parser::errors::{ToolParserError, ToolParserResult}; + +/// Token types for Python literals +#[derive(Debug, Clone, PartialEq)] +enum Token { + // Literals + String(String), + Number(String), + True, + False, + None, + + // Delimiters + LeftBracket, // [ + RightBracket, // ] + LeftBrace, // { + RightBrace, // } + LeftParen, // ( + RightParen, // ) + Comma, // , + Colon, // : + Equals, // = + + // Identifier for function names + Identifier(String), + + // End of input + Eof, +} + +/// Lexer for Python literals +struct Lexer { + input: Vec, + position: usize, +} + +impl Lexer { + fn new(input: &str) -> Self { + Self { + input: input.chars().collect(), + position: 0, + } + } + + fn current_char(&self) -> Option { + self.input.get(self.position).copied() + } + + fn advance(&mut self) { + if self.position < self.input.len() { + self.position += 1; + } + } + + fn skip_whitespace(&mut self) { + while let Some(ch) = self.current_char() { + if ch.is_whitespace() { + self.advance(); + } else { + break; + } + } + } + + fn read_string(&mut self, quote_char: char) -> Result { + let mut result = String::new(); + self.advance(); // Skip opening quote + + while let Some(ch) = self.current_char() { + if ch == '\\' { + self.advance(); + if let Some(escaped) = self.current_char() { + match escaped { + 'n' => result.push('\n'), + 't' => result.push('\t'), + 'r' => result.push('\r'), + '\\' => result.push('\\'), + '\'' => result.push('\''), + '"' => result.push('"'), + _ => { + result.push('\\'); + result.push(escaped); + } + } + self.advance(); + } + } else if ch == quote_char { + self.advance(); // Skip closing quote + return Ok(result); + } else { + result.push(ch); + self.advance(); + } + } + + Err(ToolParserError::ParsingFailed("Unterminated string".into())) + } + + fn read_number(&mut self) -> String { + let mut result = String::new(); + + // Handle negative numbers + if self.current_char() == Some('-') { + result.push('-'); + self.advance(); + } + + // Read digits and decimal point + while let Some(ch) = self.current_char() { + if ch.is_ascii_digit() || ch == '.' || ch == 'e' || ch == 'E' || ch == '+' || ch == '-' + { + result.push(ch); + self.advance(); + } else { + break; + } + } + + result + } + + fn read_identifier(&mut self) -> String { + let mut result = String::new(); + + while let Some(ch) = self.current_char() { + if ch.is_alphanumeric() || ch == '_' { + result.push(ch); + self.advance(); + } else { + break; + } + } + + result + } + + fn next_token(&mut self) -> Result { + self.skip_whitespace(); + + match self.current_char() { + None => Ok(Token::Eof), + Some('[') => { + self.advance(); + Ok(Token::LeftBracket) + } + Some(']') => { + self.advance(); + Ok(Token::RightBracket) + } + Some('{') => { + self.advance(); + Ok(Token::LeftBrace) + } + Some('}') => { + self.advance(); + Ok(Token::RightBrace) + } + Some('(') => { + self.advance(); + Ok(Token::LeftParen) + } + Some(')') => { + self.advance(); + Ok(Token::RightParen) + } + Some(',') => { + self.advance(); + Ok(Token::Comma) + } + Some(':') => { + self.advance(); + Ok(Token::Colon) + } + Some('=') => { + self.advance(); + Ok(Token::Equals) + } + Some('"') => Ok(Token::String(self.read_string('"')?)), + Some('\'') => Ok(Token::String(self.read_string('\'')?)), + Some(ch) if ch == '-' || ch.is_ascii_digit() => Ok(Token::Number(self.read_number())), + Some(ch) if ch.is_alphabetic() || ch == '_' => { + let ident = self.read_identifier(); + match ident.as_str() { + "True" => Ok(Token::True), + "False" => Ok(Token::False), + "None" => Ok(Token::None), + _ => Ok(Token::Identifier(ident)), + } + } + Some(ch) => Err(ToolParserError::ParsingFailed(format!( + "Unexpected character: {}", + ch + ))), + } + } +} + +/// Parser for Python literals +pub struct PythonLiteralParser { + lexer: Lexer, + current_token: Token, +} + +impl PythonLiteralParser { + pub fn new(input: &str) -> Result { + let mut lexer = Lexer::new(input); + let current_token = lexer.next_token()?; + Ok(Self { + lexer, + current_token, + }) + } + + fn advance(&mut self) -> Result<(), ToolParserError> { + self.current_token = self.lexer.next_token()?; + Ok(()) + } + + fn expect(&mut self, expected: Token) -> Result<(), ToolParserError> { + if self.current_token == expected { + self.advance()?; + Ok(()) + } else { + Err(ToolParserError::ParsingFailed(format!( + "Expected {:?}, got {:?}", + expected, self.current_token + ))) + } + } + + /// Parse a Python literal value + pub fn parse_value(&mut self) -> Result { + match &self.current_token.clone() { + Token::String(s) => { + let value = s.clone(); + self.advance()?; + Ok(json!(value)) + } + Token::Number(n) => { + let value = if let Ok(int_val) = n.parse::() { + json!(int_val) + } else if let Ok(float_val) = n.parse::() { + json!(float_val) + } else { + return Err(ToolParserError::ParsingFailed(format!( + "Invalid number: {}", + n + ))); + }; + self.advance()?; + Ok(value) + } + Token::True => { + self.advance()?; + Ok(json!(true)) + } + Token::False => { + self.advance()?; + Ok(json!(false)) + } + Token::None => { + self.advance()?; + Ok(Value::Null) + } + Token::LeftBracket => self.parse_list(), + Token::LeftBrace => self.parse_dict(), + _ => Err(ToolParserError::ParsingFailed(format!( + "Unexpected token: {:?}", + self.current_token + ))), + } + } + + /// Parse a Python list: [item1, item2, ...] + fn parse_list(&mut self) -> Result { + self.expect(Token::LeftBracket)?; + let mut items = Vec::new(); + + // Handle empty list + if self.current_token == Token::RightBracket { + self.advance()?; + return Ok(json!(items)); + } + + loop { + items.push(self.parse_value()?); + + if self.current_token == Token::Comma { + self.advance()?; + // Handle trailing comma + if self.current_token == Token::RightBracket { + break; + } + } else if self.current_token == Token::RightBracket { + break; + } else { + return Err(ToolParserError::ParsingFailed(format!( + "Expected ',' or ']', got {:?}", + self.current_token + ))); + } + } + + self.expect(Token::RightBracket)?; + Ok(json!(items)) + } + + /// Parse a Python dict: {key1: value1, key2: value2, ...} + fn parse_dict(&mut self) -> Result { + self.expect(Token::LeftBrace)?; + let mut map = HashMap::new(); + + // Handle empty dict + if self.current_token == Token::RightBrace { + self.advance()?; + return Ok(json!(map)); + } + + loop { + // Parse key (must be a string) + let key = match &self.current_token { + Token::String(s) => { + let k = s.clone(); + self.advance()?; + k + } + _ => { + return Err(ToolParserError::ParsingFailed(format!( + "Expected string key, got {:?}", + self.current_token + ))) + } + }; + + self.expect(Token::Colon)?; + + // Parse value + let value = self.parse_value()?; + map.insert(key, value); + + if self.current_token == Token::Comma { + self.advance()?; + // Handle trailing comma + if self.current_token == Token::RightBrace { + break; + } + } else if self.current_token == Token::RightBrace { + break; + } else { + return Err(ToolParserError::ParsingFailed(format!( + "Expected ',' or '}}', got {:?}", + self.current_token + ))); + } + } + + self.expect(Token::RightBrace)?; + Ok(json!(map)) + } +} + +/// Parse a Python literal string into a JSON value +pub fn parse_python_literal(input: &str) -> ToolParserResult { + let mut parser = PythonLiteralParser::new(input)?; + parser.parse_value() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_primitives() { + assert_eq!(parse_python_literal("True").unwrap(), json!(true)); + assert_eq!(parse_python_literal("False").unwrap(), json!(false)); + assert_eq!(parse_python_literal("None").unwrap(), Value::Null); + assert_eq!(parse_python_literal("42").unwrap(), json!(42)); + assert_eq!(parse_python_literal("12.345").unwrap(), json!(12.345)); + assert_eq!(parse_python_literal("-42").unwrap(), json!(-42)); + assert_eq!(parse_python_literal("\"hello\"").unwrap(), json!("hello")); + assert_eq!(parse_python_literal("'world'").unwrap(), json!("world")); + } + + #[test] + fn test_parse_list() { + assert_eq!(parse_python_literal("[]").unwrap(), json!([])); + assert_eq!(parse_python_literal("[1, 2, 3]").unwrap(), json!([1, 2, 3])); + assert_eq!( + parse_python_literal("[\"a\", \"b\", \"c\"]").unwrap(), + json!(["a", "b", "c"]) + ); + assert_eq!( + parse_python_literal("[True, False, None]").unwrap(), + json!([true, false, null]) + ); + // Nested list + assert_eq!( + parse_python_literal("[[1, 2], [3, 4]]").unwrap(), + json!([[1, 2], [3, 4]]) + ); + } + + #[test] + fn test_parse_dict() { + assert_eq!(parse_python_literal("{}").unwrap(), json!({})); + assert_eq!( + parse_python_literal("{\"a\": 1, \"b\": 2}").unwrap(), + json!({"a": 1, "b": 2}) + ); + assert_eq!( + parse_python_literal("{'x': True, 'y': False}").unwrap(), + json!({"x": true, "y": false}) + ); + // Nested dict + assert_eq!( + parse_python_literal("{\"nested\": {\"value\": [1, 2, 3]}}").unwrap(), + json!({"nested": {"value": [1, 2, 3]}}) + ); + } + + #[test] + fn test_complex_nested() { + let input = r#"{"config": {"nested": {"value": [1, 2, 3]}, "enabled": True}}"#; + let expected = json!({ + "config": { + "nested": { + "value": [1, 2, 3] + }, + "enabled": true + } + }); + assert_eq!(parse_python_literal(input).unwrap(), expected); + } +} diff --git a/sgl-router/src/tool_parser/registry.rs b/sgl-router/src/tool_parser/registry.rs new file mode 100644 index 000000000..f694d680c --- /dev/null +++ b/sgl-router/src/tool_parser/registry.rs @@ -0,0 +1,224 @@ +use crate::tool_parser::parsers::{ + DeepSeekParser, Glm4MoeParser, GptOssParser, JsonParser, KimiK2Parser, LlamaParser, + MistralParser, PythonicParser, QwenParser, Step3Parser, +}; +use crate::tool_parser::traits::ToolParser; +use once_cell::sync::Lazy; +use std::collections::HashMap; +use std::sync::Arc; + +/// Global singleton registry instance - created once and reused +pub static GLOBAL_REGISTRY: Lazy = Lazy::new(ParserRegistry::new_internal); + +/// Registry for tool parsers and model mappings +pub struct ParserRegistry { + /// Map of parser name to parser instance + parsers: HashMap>, + /// Map of model name/pattern to parser name + model_mapping: HashMap, + /// Default parser to use when no match found + default_parser: String, +} + +impl ParserRegistry { + /// Get the global singleton instance + pub fn new() -> &'static Self { + &GLOBAL_REGISTRY + } + + /// Create a new instance for testing (not the singleton) + #[cfg(test)] + pub fn new_for_testing() -> Self { + Self::new_internal() + } + + /// Internal constructor for creating the singleton instance + fn new_internal() -> Self { + let mut registry = Self { + parsers: HashMap::new(), + model_mapping: HashMap::new(), + default_parser: "json".to_string(), + }; + + // Register default parsers + registry.register_default_parsers(); + + // Register default model mappings + registry.register_default_mappings(); + + registry + } + + /// Register a parser + pub fn register_parser(&mut self, name: impl Into, parser: Arc) { + self.parsers.insert(name.into(), parser); + } + + /// Map a model name/pattern to a parser + pub fn map_model(&mut self, model: impl Into, parser: impl Into) { + self.model_mapping.insert(model.into(), parser.into()); + } + + /// Get parser for a specific model + pub fn get_parser(&self, model: &str) -> Option> { + // Try exact match first + if let Some(parser_name) = self.model_mapping.get(model) { + if let Some(parser) = self.parsers.get(parser_name) { + return Some(parser.clone()); + } + } + + // Try prefix matching with more specific patterns first + // Collect all matching patterns and sort by specificity (longer = more specific) + let mut matches: Vec<(&String, &String)> = self + .model_mapping + .iter() + .filter(|(pattern, _)| { + if pattern.ends_with('*') { + let prefix = &pattern[..pattern.len() - 1]; + model.starts_with(prefix) + } else { + false + } + }) + .collect(); + + // Sort by pattern length in descending order (longer patterns are more specific) + matches.sort_by_key(|(pattern, _)| std::cmp::Reverse(pattern.len())); + + // Return the first matching parser + for (_, parser_name) in matches { + if let Some(parser) = self.parsers.get(parser_name) { + return Some(parser.clone()); + } + } + + // Fall back to default parser if it exists + self.parsers.get(&self.default_parser).cloned() + } + + /// List all registered parsers + pub fn list_parsers(&self) -> Vec<&str> { + self.parsers.keys().map(|s| s.as_str()).collect() + } + + /// List all model mappings + pub fn list_mappings(&self) -> Vec<(&str, &str)> { + self.model_mapping + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())) + .collect() + } + + /// Register default parsers + fn register_default_parsers(&mut self) { + // JSON parser - most common format + self.register_parser("json", Arc::new(JsonParser::new())); + + // Mistral parser - [TOOL_CALLS] [...] format + self.register_parser("mistral", Arc::new(MistralParser::new())); + + // Qwen parser - ... format + self.register_parser("qwen", Arc::new(QwenParser::new())); + + // Pythonic parser - [func(arg=val)] format + self.register_parser("pythonic", Arc::new(PythonicParser::new())); + + // Llama parser - <|python_tag|>{...} or plain JSON format + self.register_parser("llama", Arc::new(LlamaParser::new())); + + // DeepSeek V3 parser - Unicode tokens with JSON blocks + self.register_parser("deepseek", Arc::new(DeepSeekParser::new())); + + // GLM-4 MoE parser - XML-style key-value format + self.register_parser("glm4_moe", Arc::new(Glm4MoeParser::new())); + + // Step3 parser - StepTML XML format + self.register_parser("step3", Arc::new(Step3Parser::new())); + + // Kimi K2 parser - Token-based with indexed functions + self.register_parser("kimik2", Arc::new(KimiK2Parser::new())); + + // GPT-OSS parser - Channel format + self.register_parser("gpt_oss", Arc::new(GptOssParser::new())); + } + + /// Register default model mappings + fn register_default_mappings(&mut self) { + // OpenAI models + self.map_model("gpt-4*", "json"); + self.map_model("gpt-3.5*", "json"); + self.map_model("gpt-4o*", "json"); + + // Anthropic models + self.map_model("claude-*", "json"); + + // Mistral models - use Mistral parser + self.map_model("mistral-*", "mistral"); + self.map_model("mixtral-*", "mistral"); + + // Qwen models - use Qwen parser + self.map_model("qwen*", "qwen"); + self.map_model("Qwen*", "qwen"); + + // Llama models + // Llama 4 uses pythonic format + self.map_model("llama-4*", "pythonic"); + self.map_model("meta-llama-4*", "pythonic"); + // Llama 3.2 uses python_tag format + self.map_model("llama-3.2*", "llama"); + self.map_model("meta-llama-3.2*", "llama"); + // Other Llama models use JSON + self.map_model("llama-*", "json"); + self.map_model("meta-llama-*", "json"); + + // DeepSeek models + // DeepSeek V3 uses custom Unicode token format + self.map_model("deepseek-v3*", "deepseek"); + self.map_model("deepseek-ai/DeepSeek-V3*", "deepseek"); + // DeepSeek V2 uses pythonic format + self.map_model("deepseek-*", "pythonic"); + + // GLM models + // GLM-4 MoE uses XML-style format + self.map_model("glm-4-moe*", "glm4_moe"); + self.map_model("THUDM/glm-4-moe*", "glm4_moe"); + self.map_model("glm-4.5*", "glm4_moe"); + // Other GLM models may use JSON + self.map_model("glm-*", "json"); + + // Step3 models + self.map_model("step3*", "step3"); + self.map_model("Step-3*", "step3"); + + // Kimi models + self.map_model("kimi-k2*", "kimik2"); + self.map_model("Kimi-K2*", "kimik2"); + self.map_model("moonshot*/Kimi-K2*", "kimik2"); + + // GPT-OSS models (T4-style) + self.map_model("gpt-oss*", "gpt_oss"); + self.map_model("t4-*", "gpt_oss"); + + // Other models default to JSON + self.map_model("gemini-*", "json"); + self.map_model("palm-*", "json"); + self.map_model("gemma-*", "json"); + } + + /// Set the default parser + pub fn set_default_parser(&mut self, name: impl Into) { + self.default_parser = name.into(); + } + + /// Check if a parser is registered + pub fn has_parser(&self, name: &str) -> bool { + self.parsers.contains_key(name) + } +} + +impl Default for &'static ParserRegistry { + fn default() -> Self { + ParserRegistry::new() + } +} diff --git a/sgl-router/src/tool_parser/state.rs b/sgl-router/src/tool_parser/state.rs new file mode 100644 index 000000000..096a9352f --- /dev/null +++ b/sgl-router/src/tool_parser/state.rs @@ -0,0 +1,181 @@ +use crate::tool_parser::types::{PartialToolCall, ToolCall}; + +/// Current phase of parsing +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParsePhase { + /// Looking for start of tool call + Searching, + /// Parsing function name + InName, + /// Parsing function arguments + InArguments, + /// Tool call complete + Complete, +} + +/// State for streaming parser +#[derive(Debug, Clone)] +pub struct ParseState { + /// Buffer for accumulating input + pub buffer: String, + /// Position of last consumed character + pub consumed: usize, + /// Current partial tool being parsed + pub partial_tool: Option, + /// Completed tool calls + pub completed_tools: Vec, + /// Current parsing phase + pub phase: ParsePhase, + /// Bracket/brace depth for JSON parsing + pub bracket_depth: i32, + /// Whether currently inside a string literal + pub in_string: bool, + /// Whether next character should be escaped + pub escape_next: bool, + /// Current tool index (for streaming) + pub tool_index: usize, +} + +impl ParseState { + /// Create a new parse state + pub fn new() -> Self { + Self { + buffer: String::new(), + consumed: 0, + partial_tool: None, + completed_tools: Vec::new(), + phase: ParsePhase::Searching, + bracket_depth: 0, + in_string: false, + escape_next: false, + tool_index: 0, + } + } + + /// Reset state for parsing next tool + pub fn reset(&mut self) { + self.partial_tool = None; + self.phase = ParsePhase::Searching; + self.bracket_depth = 0; + self.in_string = false; + self.escape_next = false; + } + + /// Process a single character for JSON parsing + pub fn process_char(&mut self, ch: char) { + // Handle escape sequences + if self.escape_next { + self.escape_next = false; + self.buffer.push(ch); + return; + } + + if ch == '\\' && self.in_string { + self.escape_next = true; + self.buffer.push(ch); + return; + } + + // Track string boundaries + if ch == '"' && !self.escape_next { + self.in_string = !self.in_string; + } + + // Track bracket depth for JSON + if !self.in_string { + match ch { + '{' | '[' => { + self.bracket_depth += 1; + } + '}' | ']' => { + self.bracket_depth -= 1; + if self.bracket_depth == 0 && self.partial_tool.is_some() { + // Complete tool call found + self.phase = ParsePhase::Complete; + } + } + _ => {} + } + } + + self.buffer.push(ch); + } + + /// Check if we have a complete JSON object/array + pub fn has_complete_json(&self) -> bool { + self.bracket_depth == 0 && !self.in_string && !self.buffer.is_empty() + } + + /// Extract content from buffer starting at position + pub fn extract_from(&self, start: usize) -> &str { + if start >= self.buffer.len() { + return ""; + } + + // Find the nearest character boundary at or after start + let mut safe_start = start; + while safe_start < self.buffer.len() && !self.buffer.is_char_boundary(safe_start) { + safe_start += 1; + } + + if safe_start < self.buffer.len() { + &self.buffer[safe_start..] + } else { + "" + } + } + + /// Mark content as consumed up to position + pub fn consume_to(&mut self, position: usize) { + if position > self.consumed { + self.consumed = position; + } + } + + /// Get unconsumed content + pub fn unconsumed(&self) -> &str { + if self.consumed >= self.buffer.len() { + return ""; + } + + // Find the nearest character boundary at or after consumed + let mut safe_consumed = self.consumed; + while safe_consumed < self.buffer.len() && !self.buffer.is_char_boundary(safe_consumed) { + safe_consumed += 1; + } + + if safe_consumed < self.buffer.len() { + &self.buffer[safe_consumed..] + } else { + "" + } + } + + /// Clear consumed content from buffer + pub fn clear_consumed(&mut self) { + if self.consumed > 0 { + // Find the nearest character boundary at or before consumed + let mut safe_consumed = self.consumed; + while safe_consumed > 0 && !self.buffer.is_char_boundary(safe_consumed) { + safe_consumed -= 1; + } + + if safe_consumed > 0 { + self.buffer.drain(..safe_consumed); + self.consumed = self.consumed.saturating_sub(safe_consumed); + } + } + } + + /// Add completed tool + pub fn add_completed_tool(&mut self, tool: ToolCall) { + self.completed_tools.push(tool); + self.tool_index += 1; + } +} + +impl Default for ParseState { + fn default() -> Self { + Self::new() + } +} diff --git a/sgl-router/src/tool_parser/tests.rs b/sgl-router/src/tool_parser/tests.rs new file mode 100644 index 000000000..27a258ad5 --- /dev/null +++ b/sgl-router/src/tool_parser/tests.rs @@ -0,0 +1,886 @@ +use super::*; +use crate::tool_parser::parsers::JsonParser; +use crate::tool_parser::partial_json::{ + compute_diff, find_common_prefix, is_complete_json, PartialJson, +}; +use crate::tool_parser::traits::ToolParser; +use crate::tool_parser::types::TokenConfig; + +#[test] +fn test_parse_state_new() { + let state = ParseState::new(); + assert_eq!(state.phase, ParsePhase::Searching); + assert_eq!(state.buffer, ""); + assert_eq!(state.consumed, 0); + assert_eq!(state.bracket_depth, 0); + assert!(!state.in_string); + assert!(!state.escape_next); +} + +#[test] +fn test_parse_state_process_char() { + let mut state = ParseState::new(); + + // Test bracket tracking + state.process_char('{'); + assert_eq!(state.bracket_depth, 1); + + state.process_char('}'); + assert_eq!(state.bracket_depth, 0); + + // Test string tracking + state.process_char('"'); + assert!(state.in_string); + + state.process_char('"'); + assert!(!state.in_string); + + // Test escape handling + state.process_char('"'); + state.process_char('\\'); + assert!(state.escape_next); + + state.process_char('"'); + assert!(!state.escape_next); + assert!(state.in_string); // Still in string because quote was escaped +} + +#[test] +fn test_token_config() { + let config = TokenConfig { + start_tokens: vec!["".to_string(), "[".to_string()], + end_tokens: vec!["".to_string(), "]".to_string()], + separator: ", ".to_string(), + }; + + let pairs: Vec<_> = config.iter_pairs().collect(); + assert_eq!(pairs.len(), 2); + assert_eq!(pairs[0], ("", "")); + assert_eq!(pairs[1], ("[", "]")); +} + +#[test] +fn test_parser_registry() { + let registry = ParserRegistry::new(); + + // Test has default mappings + assert!(!registry.list_mappings().is_empty()); + + // Test model pattern matching + let mappings = registry.list_mappings(); + let has_gpt = mappings.iter().any(|(m, _)| m.starts_with("gpt")); + assert!(has_gpt); +} + +#[test] +fn test_parser_registry_pattern_matching() { + let mut registry = ParserRegistry::new_for_testing(); + + // Test that model mappings work by checking the list + registry.map_model("test-model", "json"); + + // Verify through list_mappings + let mappings = registry.list_mappings(); + let has_test = mappings + .iter() + .any(|(m, p)| *m == "test-model" && *p == "json"); + assert!(has_test); +} + +#[test] +fn test_tool_call_serialization() { + let tool_call = ToolCall { + id: "call-123".to_string(), + r#type: "function".to_string(), + function: FunctionCall { + name: "search".to_string(), + arguments: r#"{"query": "rust programming"}"#.to_string(), + }, + }; + + let json = serde_json::to_string(&tool_call).unwrap(); + assert!(json.contains("call-123")); + assert!(json.contains("search")); + assert!(json.contains("rust programming")); + + let parsed: ToolCall = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.id, "call-123"); + assert_eq!(parsed.function.name, "search"); +} + +#[test] +fn test_partial_json_parser() { + let parser = PartialJson::default(); + + // Test complete JSON + let input = r#"{"name": "test", "value": 42}"#; + let (value, consumed) = parser.parse_value(input).unwrap(); + assert_eq!(value["name"], "test"); + assert_eq!(value["value"], 42); + assert_eq!(consumed, input.len()); + + // Test incomplete JSON object + let input = r#"{"name": "test", "value": "#; + let (value, _consumed) = parser.parse_value(input).unwrap(); + assert_eq!(value["name"], "test"); + assert!(value["value"].is_null()); + + // Test incomplete string + let input = r#"{"name": "tes"#; + let (value, _consumed) = parser.parse_value(input).unwrap(); + assert_eq!(value["name"], "tes"); + + // Test incomplete array + let input = r#"[1, 2, "#; + let (value, _consumed) = parser.parse_value(input).unwrap(); + assert!(value.is_array()); + assert_eq!(value[0], 1); + assert_eq!(value[1], 2); +} + +#[test] +fn test_partial_json_depth_limit() { + // max_depth of 3 allows nesting up to 3 levels + // Set allow_incomplete to false to get errors instead of partial results + let parser = PartialJson::new(3, false); + + // This should work (simple object) + let input = r#"{"a": 1}"#; + let result = parser.parse_value(input); + assert!(result.is_ok()); + + // This should work (nested to depth 3) + let input = r#"{"a": {"b": {"c": 1}}}"#; + let result = parser.parse_value(input); + assert!(result.is_ok()); + + // This should fail (nested to depth 4, exceeds limit) + let input = r#"{"a": {"b": {"c": {"d": 1}}}}"#; + let result = parser.parse_value(input); + assert!(result.is_err()); +} + +#[test] +fn test_is_complete_json() { + assert!(is_complete_json(r#"{"name": "test"}"#)); + assert!(is_complete_json(r#"[1, 2, 3]"#)); + assert!(is_complete_json(r#""string""#)); + assert!(is_complete_json("42")); + assert!(is_complete_json("true")); + assert!(is_complete_json("null")); + + assert!(!is_complete_json(r#"{"name": "#)); + assert!(!is_complete_json(r#"[1, 2, "#)); + assert!(!is_complete_json(r#""unclosed"#)); +} + +#[test] +fn test_find_common_prefix() { + assert_eq!(find_common_prefix("hello", "hello"), 5); + assert_eq!(find_common_prefix("hello", "help"), 3); + assert_eq!(find_common_prefix("hello", "world"), 0); + assert_eq!(find_common_prefix("", "hello"), 0); + assert_eq!(find_common_prefix("hello", ""), 0); +} + +#[test] +fn test_compute_diff() { + assert_eq!(compute_diff("hello", "hello world"), " world"); + assert_eq!(compute_diff("", "hello"), "hello"); + assert_eq!(compute_diff("hello", "hello"), ""); + assert_eq!(compute_diff("test", "hello"), "hello"); +} + +#[test] +fn test_stream_result_variants() { + // Test Incomplete + let result = StreamResult::Incomplete; + matches!(result, StreamResult::Incomplete); + + // Test ToolName + let result = StreamResult::ToolName { + index: 0, + name: "test".to_string(), + }; + if let StreamResult::ToolName { index, name } = result { + assert_eq!(index, 0); + assert_eq!(name, "test"); + } else { + panic!("Expected ToolName variant"); + } + + // Test ToolComplete + let tool = ToolCall { + id: "123".to_string(), + r#type: "function".to_string(), + function: FunctionCall { + name: "test".to_string(), + arguments: "{}".to_string(), + }, + }; + let result = StreamResult::ToolComplete(tool.clone()); + if let StreamResult::ToolComplete(t) = result { + assert_eq!(t.id, "123"); + } else { + panic!("Expected ToolComplete variant"); + } +} + +#[test] +fn test_partial_tool_call() { + let mut partial = PartialToolCall { + name: None, + arguments_buffer: String::new(), + start_position: 0, + name_sent: false, + streamed_args: String::new(), + }; + + // Set name + partial.name = Some("test_function".to_string()); + assert_eq!(partial.name.as_ref().unwrap(), "test_function"); + + // Append arguments + partial.arguments_buffer.push_str(r#"{"key": "value"}"#); + assert_eq!(partial.arguments_buffer, r#"{"key": "value"}"#); + + // Update streaming state + partial.name_sent = true; + partial.streamed_args = r#"{"key": "#.to_string(); + assert!(partial.name_sent); + assert_eq!(partial.streamed_args, r#"{"key": "#); +} + +#[tokio::test] +async fn test_json_parser_complete_single() { + let parser = JsonParser::new(); + + // Test single tool call with arguments + let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco", "units": "celsius"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("San Francisco")); + assert!(result[0].function.arguments.contains("celsius")); +} + +#[tokio::test] +async fn test_json_parser_complete_array() { + let parser = JsonParser::new(); + + // Test array of tool calls + let input = r#"[ + {"name": "get_weather", "arguments": {"location": "SF"}}, + {"name": "get_news", "arguments": {"query": "technology"}} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "get_news"); +} + +#[tokio::test] +async fn test_json_parser_with_parameters() { + let parser = JsonParser::new(); + + // Test with "parameters" instead of "arguments" + let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20, "operation": "add"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + assert!(result[0].function.arguments.contains("10")); + assert!(result[0].function.arguments.contains("20")); + assert!(result[0].function.arguments.contains("add")); +} + +#[tokio::test] +async fn test_json_parser_with_tokens() { + // Test with custom wrapper tokens + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["[TOOL_CALLS] [".to_string()], + end_tokens: vec!["]".to_string()], + separator: ", ".to_string(), + }); + + let input = r#"[TOOL_CALLS] [{"name": "search", "arguments": {"query": "rust programming"}}]"#; + let result = parser.parse_complete(input).await.unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); +} + +#[tokio::test] +async fn test_multiline_json_with_tokens() { + // Test that regex with (?s) flag properly handles multi-line JSON + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + // Pretty-printed multi-line JSON + let input = r#"{ + "name": "get_weather", + "arguments": { + "location": "San Francisco", + "units": "celsius", + "include_forecast": true + } +}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + assert!(result[0].function.arguments.contains("San Francisco")); + assert!(result[0].function.arguments.contains("celsius")); + assert!(result[0].function.arguments.contains("true")); +} + +#[tokio::test] +async fn test_multiline_json_array() { + // Test multi-line JSON array without wrapper tokens + let parser = JsonParser::new(); + + let input = r#"[ + { + "name": "function1", + "arguments": { + "param1": "value1", + "param2": 42 + } + }, + { + "name": "function2", + "parameters": { + "data": [1, 2, 3], + "flag": false + } + } +]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "function1"); + assert_eq!(result[1].function.name, "function2"); + assert!(result[0].function.arguments.contains("value1")); + assert!(result[1].function.arguments.contains("[1,2,3]")); +} + +#[test] +fn test_json_parser_format_detection() { + let parser = JsonParser::new(); + + // Should detect valid tool call formats + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(parser.detect_format(r#"{"name": "test", "parameters": {"x": 1}}"#)); + assert!(parser.detect_format(r#"[{"name": "test"}]"#)); + + // Should not detect non-tool formats + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format(r#"{"key": "value"}"#)); + assert!(!parser.detect_format(r#"{"data": {"nested": true}}"#)); +} + +#[tokio::test] +async fn test_json_parser_streaming() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Test with complete JSON + let full_json = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#; + + let result = parser + .parse_incremental(full_json, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + assert!(tool.function.arguments.contains("San Francisco")); + } + _ => panic!("Expected ToolComplete for complete JSON"), + } +} + +#[tokio::test] +async fn test_registry_with_json_parser() { + let registry = ParserRegistry::new(); + + // JSON parser should be registered by default + assert!(registry.has_parser("json")); + + // Should get JSON parser for OpenAI models + let parser = registry.get_parser("gpt-4-turbo").unwrap(); + + // Test that the parser works + let input = r#"{"name": "test", "arguments": {"x": 1}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} + +#[tokio::test] +async fn test_json_parser_invalid_input() { + let parser = JsonParser::new(); + + // Invalid JSON should return empty results + assert_eq!(parser.parse_complete("not json").await.unwrap().len(), 0); + assert_eq!(parser.parse_complete("{invalid}").await.unwrap().len(), 0); + assert_eq!(parser.parse_complete("").await.unwrap().len(), 0); +} + +#[tokio::test] +async fn test_json_parser_empty_arguments() { + let parser = JsonParser::new(); + + // Tool call with no arguments + let input = r#"{"name": "get_time"}"#; + let result = parser.parse_complete(input).await.unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); + assert_eq!(result[0].function.arguments, "{}"); +} + +#[cfg(test)] +mod failure_cases { + use super::*; + + #[tokio::test] + async fn test_malformed_tool_missing_name() { + let parser = JsonParser::new(); + + // Missing name field + let input = r#"{"arguments": {"x": 1}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should return empty for tool without name"); + + // Empty name + let input = r#"{"name": "", "arguments": {"x": 1}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1, "Should accept empty name string"); + assert_eq!(result[0].function.name, ""); + } + + #[tokio::test] + async fn test_invalid_arguments_json() { + let parser = JsonParser::new(); + + // Arguments is a string instead of object + let input = r#"{"name": "test", "arguments": "not an object"}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + // Should serialize the string as JSON + assert!(result[0].function.arguments.contains("not an object")); + + // Arguments is a number + let input = r#"{"name": "test", "arguments": 42}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.arguments, "42"); + + // Arguments is null + let input = r#"{"name": "test", "arguments": null}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.arguments, "null"); + } + + #[tokio::test] + async fn test_broken_wrapper_tokens() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + // Missing end token + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!( + result.len(), + 0, + "Should fail to parse without complete wrapper" + ); + + // Missing start token - parser looks for complete wrapper, so this won't parse + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!( + result.len(), + 0, + "Should not parse JSON with incomplete wrapper" + ); + + // Mismatched tokens + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should fail with mismatched tokens"); + } + + #[tokio::test] + async fn test_invalid_json_structures() { + let parser = JsonParser::new(); + + // Trailing comma + let input = r#"{"name": "test", "arguments": {"x": 1,}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should reject JSON with trailing comma"); + + // Missing quotes on keys + let input = r#"{name: "test", arguments: {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should reject invalid JSON syntax"); + + // Unclosed object + let input = r#"{"name": "test", "arguments": {"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should reject incomplete JSON"); + } +} + +#[cfg(test)] +mod edge_cases { + use super::*; + + #[tokio::test] + async fn test_unicode_in_names_and_arguments() { + let parser = JsonParser::new(); + + // Unicode in function name + let input = r#"{"name": "获取天气", "arguments": {"location": "北京"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "获取天气"); + assert!(result[0].function.arguments.contains("北京")); + + // Emoji in arguments + let input = r#"{"name": "send_message", "arguments": {"text": "Hello 👋 World 🌍"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("👋")); + assert!(result[0].function.arguments.contains("🌍")); + } + + #[tokio::test] + async fn test_escaped_characters() { + let parser = JsonParser::new(); + + // Escaped quotes in arguments + let input = r#"{"name": "echo", "arguments": {"text": "He said \"hello\""}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains(r#"\"hello\""#)); + + // Escaped backslashes + let input = r#"{"name": "path", "arguments": {"dir": "C:\\Users\\test"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("\\\\")); + + // Newlines and tabs + let input = r#"{"name": "format", "arguments": {"text": "line1\nline2\ttabbed"}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("\\n")); + assert!(result[0].function.arguments.contains("\\t")); + } + + #[tokio::test] + async fn test_very_large_payloads() { + let parser = JsonParser::new(); + + // Large arguments object + let mut large_args = r#"{"name": "process", "arguments": {"#.to_string(); + for i in 0..1000 { + large_args.push_str(&format!(r#""field_{}": "value_{}","#, i, i)); + } + large_args.push_str(r#""final": "value"}}"#); + + let result = parser.parse_complete(&large_args).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process"); + assert!(result[0].function.arguments.contains("field_999")); + + // Large array of tool calls + let mut large_array = "[".to_string(); + for i in 0..100 { + if i > 0 { + large_array.push(','); + } + large_array.push_str(&format!(r#"{{"name": "func_{}", "arguments": {{}}}}"#, i)); + } + large_array.push(']'); + + let result = parser.parse_complete(&large_array).await.unwrap(); + assert_eq!(result.len(), 100); + assert_eq!(result[99].function.name, "func_99"); + } + + #[tokio::test] + async fn test_mixed_array_tools_and_non_tools() { + let parser = JsonParser::new(); + + // Array with both tool calls and non-tool objects + let input = r#"[ + {"name": "tool1", "arguments": {}}, + {"not_a_tool": "just_data"}, + {"name": "tool2", "parameters": {"x": 1}}, + {"key": "value", "another": "field"} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2, "Should only parse valid tool calls"); + assert_eq!(result[0].function.name, "tool1"); + assert_eq!(result[1].function.name, "tool2"); + } + + #[tokio::test] + async fn test_duplicate_keys_in_json() { + let parser = JsonParser::new(); + + // JSON with duplicate keys (last one wins in most parsers) + let input = r#"{"name": "first", "name": "second", "arguments": {"x": 1, "x": 2}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!( + result[0].function.name, "second", + "Last duplicate key should win" + ); + assert!( + result[0].function.arguments.contains("2"), + "Last duplicate value should win" + ); + } + + #[tokio::test] + async fn test_null_values_in_arguments() { + let parser = JsonParser::new(); + + // Null values in arguments + let input = r#"{"name": "test", "arguments": {"required": "value", "optional": null}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("null")); + + // Array with null + let input = r#"{"name": "test", "arguments": {"items": [1, null, "three"]}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("null")); + } + + #[tokio::test] + async fn test_multiple_token_pairs_with_conflicts() { + // Test with overlapping token patterns + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["<<".to_string(), "".to_string()], + end_tokens: vec![">>".to_string(), "".to_string()], + separator: ", ".to_string(), + }); + + // First pattern + let input = r#"<<{"name": "test1", "arguments": {}}>>"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test1"); + + // Second pattern + let input = r#"{"name": "test2", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test2"); + + // Nested patterns (should use first match) + let input = r#"<{"name": "test3", "arguments": {}}>"#; + let result = parser.parse_complete(input).await.unwrap(); + // This is tricky - depends on regex behavior + // The parser should handle this gracefully + assert!(result.len() <= 1, "Should not parse multiple times"); + } + + #[tokio::test] + async fn test_streaming_with_partial_chunks() { + let parser = JsonParser::new(); + + // Test 1: Very incomplete JSON (just opening brace) should return Incomplete + let mut state1 = ParseState::new(); + let partial = r#"{"#; + let result = parser + .parse_incremental(partial, &mut state1) + .await + .unwrap(); + assert!( + matches!(result, StreamResult::Incomplete), + "Should return Incomplete for just opening brace" + ); + + // Test 2: Complete JSON should return ToolComplete + let mut state2 = ParseState::new(); + let complete = r#"{"name": "get_weather", "arguments": {"location": "SF"}}"#; + let result = parser + .parse_incremental(complete, &mut state2) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = + serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "SF"); + } + _ => panic!("Expected ToolComplete for complete JSON"), + } + + // Test 3: Partial JSON with name + // The PartialJson parser can complete partial JSON by filling in missing values + let mut state3 = ParseState::new(); + let partial_with_name = r#"{"name": "test", "argum"#; + let result = parser + .parse_incremental(partial_with_name, &mut state3) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test"); + // Arguments will be empty object since "argum" is incomplete + assert_eq!(tool.function.arguments, "{}"); + } + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "test"); + } + StreamResult::Incomplete => { + // Also acceptable if parser decides to wait + } + _ => panic!("Unexpected result for partial JSON with name"), + } + } + + #[tokio::test] + async fn test_special_json_values() { + let parser = JsonParser::new(); + + // Boolean values + let input = r#"{"name": "toggle", "arguments": {"enabled": true, "disabled": false}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("true")); + assert!(result[0].function.arguments.contains("false")); + + // Numbers (including float and negative) + let input = r#"{"name": "calc", "arguments": {"int": 42, "float": 3.14, "negative": -17}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("42")); + assert!(result[0].function.arguments.contains("3.14")); + assert!(result[0].function.arguments.contains("-17")); + + // Empty arrays and objects + let input = r#"{"name": "test", "arguments": {"empty_arr": [], "empty_obj": {}}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("[]")); + assert!(result[0].function.arguments.contains("{}")); + } + + #[tokio::test] + async fn test_function_field_alternative() { + let parser = JsonParser::new(); + + // Using "function" instead of "name" + let input = r#"{"function": "test_func", "arguments": {"x": 1}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test_func"); + + // Both "name" and "function" present (name should take precedence) + let input = r#"{"name": "primary", "function": "secondary", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "primary"); + } + + #[tokio::test] + async fn test_whitespace_handling() { + let parser = JsonParser::new(); + + // Extra whitespace everywhere + let input = r#" { + "name" : "test" , + "arguments" : { + "key" : "value" + } + } "#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + // Minified JSON (no whitespace) + let input = r#"{"name":"compact","arguments":{"a":1,"b":2}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "compact"); + } +} + +#[cfg(test)] +mod stress_tests { + use super::*; + + #[tokio::test] + async fn test_deeply_nested_arguments() { + let parser = JsonParser::new(); + + // Deeply nested structure + let input = r#"{ + "name": "nested", + "arguments": { + "level1": { + "level2": { + "level3": { + "level4": { + "level5": { + "value": "deep" + } + } + } + } + } + } + }"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert!(result[0].function.arguments.contains("deep")); + } + + #[tokio::test] + async fn test_concurrent_parser_usage() { + // Test that parser can be used concurrently + let parser = std::sync::Arc::new(JsonParser::new()); + + let mut handles = vec![]; + + for i in 0..10 { + let parser_clone = parser.clone(); + let handle = tokio::spawn(async move { + let input = format!(r#"{{"name": "func_{}", "arguments": {{}}}}"#, i); + let result = parser_clone.parse_complete(&input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, format!("func_{}", i)); + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + } +} diff --git a/sgl-router/src/tool_parser/traits.rs b/sgl-router/src/tool_parser/traits.rs new file mode 100644 index 000000000..19263688d --- /dev/null +++ b/sgl-router/src/tool_parser/traits.rs @@ -0,0 +1,35 @@ +use crate::tool_parser::{ + errors::ToolParserResult, + state::ParseState, + types::{StreamResult, ToolCall}, +}; +use async_trait::async_trait; + +/// Core trait for all tool parsers +#[async_trait] +pub trait ToolParser: Send + Sync { + /// Parse complete tool calls from final output + async fn parse_complete(&self, output: &str) -> ToolParserResult>; + + /// Parse tool calls from model output (streaming) + async fn parse_incremental( + &self, + chunk: &str, + state: &mut ParseState, + ) -> ToolParserResult; + + /// Check if text contains tool calls in this parser's format + fn detect_format(&self, text: &str) -> bool; +} + +/// Trait for partial JSON parsing +pub trait PartialJsonParser: Send + Sync { + /// Parse potentially incomplete JSON + fn parse(&self, input: &str) -> ToolParserResult<(serde_json::Value, usize)>; + + /// Check if JSON is complete + fn is_complete(&self, input: &str) -> bool; + + /// Get the maximum parsing depth + fn max_depth(&self) -> usize; +} diff --git a/sgl-router/src/tool_parser/types.rs b/sgl-router/src/tool_parser/types.rs new file mode 100644 index 000000000..0638d1c2a --- /dev/null +++ b/sgl-router/src/tool_parser/types.rs @@ -0,0 +1,73 @@ +use serde::{Deserialize, Serialize}; + +/// Parsed tool call from model output (OpenAI format) +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ToolCall { + /// Unique identifier for the tool call + pub id: String, + /// Type of tool call (currently always "function") + #[serde(rename = "type")] + pub r#type: String, + /// Function call details + pub function: FunctionCall, +} + +/// Function call within a tool call +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct FunctionCall { + /// Name of the function to call + pub name: String, + /// Arguments as JSON string + pub arguments: String, +} + +/// Streaming parse result +#[derive(Debug, Clone)] +pub enum StreamResult { + /// Need more data to continue parsing + Incomplete, + /// Found a tool name (for streaming) + ToolName { index: usize, name: String }, + /// Found incremental arguments (for streaming) + ToolArguments { index: usize, arguments: String }, + /// Completed parsing a tool + ToolComplete(ToolCall), + /// Normal text (not part of tool call) + NormalText(String), +} + +/// Token configuration for parsing +#[derive(Debug, Clone)] +pub struct TokenConfig { + /// Start tokens for tool calls + pub start_tokens: Vec, + /// End tokens for tool calls + pub end_tokens: Vec, + /// Separator between multiple tool calls + pub separator: String, +} + +impl TokenConfig { + /// Iterate over start/end token pairs + pub fn iter_pairs(&self) -> impl Iterator { + self.start_tokens + .iter() + .zip(self.end_tokens.iter()) + .map(|(s, e)| (s.as_str(), e.as_str())) + } +} + +/// Simple partial tool call for streaming +#[derive(Debug, Clone)] +pub struct PartialToolCall { + /// Tool name (if parsed) + pub name: Option, + /// Buffer for accumulating arguments + pub arguments_buffer: String, + /// Start position in the input buffer + pub start_position: usize, + /// Whether the name has been sent (for streaming) + pub name_sent: bool, + /// Arguments already streamed + pub streamed_args: String, +} diff --git a/sgl-router/src/tree.rs b/sgl-router/src/tree.rs new file mode 100644 index 000000000..ea95a6946 --- /dev/null +++ b/sgl-router/src/tree.rs @@ -0,0 +1,1478 @@ +use dashmap::mapref::entry::Entry; +use dashmap::DashMap; +use tracing::info; + +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::RwLock; + +use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; + +type NodeRef = Arc; + +#[derive(Debug)] +struct Node { + children: DashMap, + text: RwLock, + tenant_last_access_time: DashMap, + parent: RwLock>, +} + +#[derive(Debug)] +pub struct Tree { + root: NodeRef, + pub tenant_char_count: DashMap, +} + +// For the heap + +struct EvictionEntry { + timestamp: u128, + tenant: String, + node: NodeRef, +} + +impl Eq for EvictionEntry {} + +#[allow(clippy::non_canonical_partial_ord_impl)] +impl PartialOrd for EvictionEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.timestamp.cmp(&other.timestamp)) + } +} + +impl Ord for EvictionEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.timestamp.cmp(&other.timestamp) + } +} + +impl PartialEq for EvictionEntry { + fn eq(&self, other: &Self) -> bool { + self.timestamp == other.timestamp + } +} + +// For char operations +// Note that in rust, `.len()` or slice is operated on the "byte" level. It causes issues for UTF-8 characters because one character might use multiple bytes. +// https://en.wikipedia.org/wiki/UTF-8 + +fn shared_prefix_count(a: &str, b: &str) -> usize { + let mut i = 0; + let mut a_iter = a.chars(); + let mut b_iter = b.chars(); + + loop { + match (a_iter.next(), b_iter.next()) { + (Some(a_char), Some(b_char)) if a_char == b_char => { + i += 1; + } + _ => break, + } + } + + i +} + +fn slice_by_chars(s: &str, start: usize, end: usize) -> String { + s.chars().skip(start).take(end - start).collect() +} + +impl Default for Tree { + fn default() -> Self { + Self::new() + } +} + +impl Tree { + /* + Thread-safe multi tenant radix tree + + 1. Storing data for multiple tenants (the overlap of multiple radix tree) + 2. Node-level lock to enable concurrent acesss on nodes + 3. Leaf LRU eviction based on tenant access time + */ + + pub fn new() -> Self { + Tree { + root: Arc::new(Node { + children: DashMap::new(), + text: RwLock::new("".to_string()), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(None), + }), + tenant_char_count: DashMap::new(), + } + } + + pub fn insert(&self, text: &str, tenant: &str) { + // Insert text into tree with given tenant + + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + curr.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + self.tenant_char_count + .entry(tenant.to_string()) + .or_insert(0); + + let mut prev = Arc::clone(&self.root); + + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + + curr = prev; + + // dashmap.entry locks the entry until the op is done + // if using contains_key + insert, there will be an issue that + // 1. "apple" and "app" entered at the same time + // 2. and get inserted to the dashmap concurrently, so only one is inserted + + match curr.children.entry(first_char) { + Entry::Vacant(entry) => { + /* + no matched + [curr] + becomes + [curr] => [new node] + */ + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let curr_text_count = curr_text.chars().count(); + let new_node = Arc::new(Node { + children: DashMap::new(), + text: RwLock::new(curr_text), + tenant_last_access_time: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + }); + + // Attach tenant to the new node (map is empty here) and increment count once + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += curr_text_count) + .or_insert(curr_text_count); + new_node + .tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + + entry.insert(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + curr_idx = text_count; + } + + Entry::Occupied(mut entry) => { + // matched + let matched_node = entry.get().clone(); + + let matched_node_text = matched_node.text.read().unwrap().to_owned(); + let matched_node_text_count = matched_node_text.chars().count(); + + let curr_text = slice_by_chars(text, curr_idx, text_count); + let shared_count = shared_prefix_count(&matched_node_text, &curr_text); + + if shared_count < matched_node_text_count { + /* + split the matched node + [curr] -> [matched_node] => + becomes + [curr] -> [new_node] -> [contracted_matched_node] + */ + + let matched_text = slice_by_chars(&matched_node_text, 0, shared_count); + let contracted_text = slice_by_chars( + &matched_node_text, + shared_count, + matched_node_text_count, + ); + let matched_text_count = matched_text.chars().count(); + + let new_node = Arc::new(Node { + text: RwLock::new(matched_text), + children: DashMap::new(), + parent: RwLock::new(Some(Arc::clone(&curr))), + tenant_last_access_time: matched_node.tenant_last_access_time.clone(), + }); + + let first_new_char = contracted_text.chars().nth(0).unwrap(); + new_node + .children + .insert(first_new_char, Arc::clone(&matched_node)); + + entry.insert(Arc::clone(&new_node)); + + *matched_node.text.write().unwrap() = contracted_text; + *matched_node.parent.write().unwrap() = Some(Arc::clone(&new_node)); + + prev = Arc::clone(&new_node); + + // Atomically attach tenant to the new split node and increment count once + match prev.tenant_last_access_time.entry(tenant.to_string()) { + Entry::Vacant(v) => { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_text_count) + .or_insert(matched_text_count); + v.insert(timestamp_ms); + } + Entry::Occupied(mut o) => { + o.insert(timestamp_ms); + } + } + + curr_idx += shared_count; + } else { + // move to next node + prev = Arc::clone(&matched_node); + + // Atomically attach tenant to existing node and increment count once + match prev.tenant_last_access_time.entry(tenant.to_string()) { + Entry::Vacant(v) => { + self.tenant_char_count + .entry(tenant.to_string()) + .and_modify(|count| *count += matched_node_text_count) + .or_insert(matched_node_text_count); + v.insert(timestamp_ms); + } + Entry::Occupied(mut o) => { + o.insert(timestamp_ms); + } + } + curr_idx += shared_count; + } + } + } + } + } + + #[allow(unused_assignments)] + pub fn prefix_match(&self, text: &str) -> (String, String) { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + if let Some(entry) = curr.children.get(&first_char) { + let matched_node = entry.value().clone(); + let matched_text_guard = matched_node.text.read().unwrap(); + let shared_count = shared_prefix_count(&matched_text_guard, &curr_text); + let matched_node_text_count = matched_text_guard.chars().count(); + drop(matched_text_guard); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } else { + // No match found, stop here + break; + } + } + + curr = prev.clone(); + + // Select the first tenant (key in the map) + let tenant = curr + .tenant_last_access_time + .iter() + .next() + .map(|kv| kv.key().to_owned()) + .unwrap_or("empty".to_string()); + + // Traverse from the curr node to the root and update the timestamp + + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + if !tenant.eq("empty") { + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.clone(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + let ret_text = slice_by_chars(text, 0, curr_idx); + (ret_text, tenant) + } + + #[allow(unused_assignments)] + pub fn prefix_match_tenant(&self, text: &str, tenant: &str) -> String { + let mut curr = Arc::clone(&self.root); + let mut curr_idx = 0; + + let mut prev = Arc::clone(&self.root); + let text_count = text.chars().count(); + + while curr_idx < text_count { + let first_char = text.chars().nth(curr_idx).unwrap(); + let curr_text = slice_by_chars(text, curr_idx, text_count); + + curr = prev.clone(); + + if let Some(entry) = curr.children.get(&first_char) { + let matched_node = entry.value().clone(); + + // Only continue matching if this node belongs to the specified tenant + if !matched_node.tenant_last_access_time.contains_key(tenant) { + break; + } + + let matched_text_guard = matched_node.text.read().unwrap(); + let shared_count = shared_prefix_count(&matched_text_guard, &curr_text); + let matched_node_text_count = matched_text_guard.chars().count(); + drop(matched_text_guard); + + if shared_count == matched_node_text_count { + // Full match with current node's text, continue to next node + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + } else { + // Partial match, stop here + curr_idx += shared_count; + prev = Arc::clone(&matched_node); + break; + } + } else { + // No match found, stop here + break; + } + } + + curr = prev.clone(); + + // Only update timestamp if we found a match for the specified tenant + if curr.tenant_last_access_time.contains_key(tenant) { + let timestamp_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis(); + + let mut current_node = Some(curr); + while let Some(node) = current_node { + node.tenant_last_access_time + .insert(tenant.to_string(), timestamp_ms); + current_node = node.parent.read().unwrap().clone(); + } + } + + slice_by_chars(text, 0, curr_idx) + } + + fn leaf_of(node: &NodeRef) -> Vec { + /* + Return the list of tenants if it's a leaf for the tenant + */ + let mut candidates: HashMap = node + .tenant_last_access_time + .iter() + .map(|entry| (entry.key().clone(), true)) + .collect(); + + for child in node.children.iter() { + for tenant in child.value().tenant_last_access_time.iter() { + candidates.insert(tenant.key().clone(), false); + } + } + + candidates + .into_iter() + .filter(|(_, is_leaf)| *is_leaf) + .map(|(tenant, _)| tenant) + .collect() + } + + pub fn evict_tenant_by_size(&self, max_size: usize) { + // Calculate used size and collect leaves + let mut stack = vec![Arc::clone(&self.root)]; + let mut pq = BinaryHeap::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + // Add leaves to priority queue + for tenant in Tree::leaf_of(&curr) { + if let Some(timestamp) = curr.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(&curr), + })); + } + } + } + + info!("Before eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + + // Process eviction + while let Some(Reverse(entry)) = pq.pop() { + let EvictionEntry { tenant, node, .. } = entry; + + if let Some(used_size) = self.tenant_char_count.get(&tenant) { + if *used_size <= max_size { + continue; + } + } + + // Decrement when removing tenant from node + if node.tenant_last_access_time.contains_key(&tenant) { + let node_len = node.text.read().unwrap().chars().count(); + self.tenant_char_count + .entry(tenant.clone()) + .and_modify(|count| { + *count = count.saturating_sub(node_len); + }); + } + + // Remove tenant from node + node.tenant_last_access_time.remove(&tenant); + + // Remove empty nodes + if node.children.is_empty() && node.tenant_last_access_time.is_empty() { + if let Some(parent) = node.parent.read().unwrap().as_ref() { + let text_guard = node.text.read().unwrap(); + if let Some(first_char) = text_guard.chars().next() { + parent.children.remove(&first_char); + } + } + } + + // Add parent to queue if it becomes a leaf + if let Some(parent) = node.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant) { + if let Some(timestamp) = parent.tenant_last_access_time.get(&tenant) { + pq.push(Reverse(EvictionEntry { + timestamp: *timestamp, + tenant: tenant.clone(), + node: Arc::clone(parent), + })); + } + } + }; + } + + info!("After eviction - Used size per tenant:"); + for entry in self.tenant_char_count.iter() { + info!("Tenant: {}, Size: {}", entry.key(), entry.value()); + } + } + + pub fn remove_tenant(&self, tenant: &str) { + // 1. Find all the leaves for the tenant + let mut stack = vec![Arc::clone(&self.root)]; + let mut queue = VecDeque::new(); + + while let Some(curr) = stack.pop() { + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + + if Tree::leaf_of(&curr).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(&curr)); + } + } + + // 2. Start from the leaves and traverse up to the root, removing the tenant from each node + while let Some(curr) = queue.pop_front() { + // remove tenant from node + curr.tenant_last_access_time.remove(&tenant.to_string()); + + // remove empty nodes + if curr.children.is_empty() && curr.tenant_last_access_time.is_empty() { + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + let text_guard = curr.text.read().unwrap(); + if let Some(first_char) = text_guard.chars().next() { + parent.children.remove(&first_char); + } + } + } + + // add parent to queue if it becomes a leaf + if let Some(parent) = curr.parent.read().unwrap().as_ref() { + if Tree::leaf_of(parent).contains(&tenant.to_string()) { + queue.push_back(Arc::clone(parent)); + } + } + } + + // 3. Remove the tenant from the tenant_char_count map + self.tenant_char_count.remove(&tenant.to_string()); + } + + pub fn get_tenant_char_count(&self) -> HashMap { + self.tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect() + } + + pub fn get_smallest_tenant(&self) -> String { + // Return a placeholder if there are no tenants + if self.tenant_char_count.is_empty() { + return "empty".to_string(); + } + + // Find the tenant with minimum char count + let mut min_tenant = None; + let mut min_count = usize::MAX; + + for entry in self.tenant_char_count.iter() { + let tenant = entry.key(); + let count = *entry.value(); + + if count < min_count { + min_count = count; + min_tenant = Some(tenant.clone()); + } + } + + // Return the found tenant or "empty" if somehow none was found + min_tenant.unwrap_or_else(|| "empty".to_string()) + } + + pub fn get_used_size_per_tenant(&self) -> HashMap { + // perform a DFS to traverse all nodes and calculate the total size used by each tenant + + let mut used_size_per_tenant: HashMap = HashMap::new(); + let mut stack = vec![Arc::clone(&self.root)]; + + while let Some(curr) = stack.pop() { + let text_count = curr.text.read().unwrap().chars().count(); + + for tenant in curr.tenant_last_access_time.iter() { + let size = used_size_per_tenant + .entry(tenant.key().clone()) + .or_insert(0); + *size += text_count; + } + + for child in curr.children.iter() { + stack.push(Arc::clone(child.value())); + } + } + + used_size_per_tenant + } + + fn node_to_string(node: &NodeRef, prefix: &str, is_last: bool) -> String { + let mut result = String::new(); + + // Add prefix and branch character + result.push_str(prefix); + result.push_str(if is_last { "└── " } else { "├── " }); + + // Add node text + let node_text = node.text.read().unwrap(); + result.push_str(&format!("'{}' [", node_text)); + + // Add tenant information with timestamps + let mut tenant_info = Vec::new(); + for entry in node.tenant_last_access_time.iter() { + let tenant_id = entry.key(); + let timestamp_ms = entry.value(); + + // Convert milliseconds to seconds and remaining milliseconds + let seconds = (timestamp_ms / 1000) as u64; + let millis = (timestamp_ms % 1000) as u32; + + // Create SystemTime from Unix timestamp + let system_time = UNIX_EPOCH + Duration::from_secs(seconds); + + // Format time as HH:MM:SS.mmm + let datetime = system_time.duration_since(UNIX_EPOCH).unwrap(); + let hours = (datetime.as_secs() % 86400) / 3600; + let minutes = (datetime.as_secs() % 3600) / 60; + let seconds = datetime.as_secs() % 60; + + tenant_info.push(format!( + "{} | {:02}:{:02}:{:02}.{:03}", + tenant_id, hours, minutes, seconds, millis + )); + } + + result.push_str(&tenant_info.join(", ")); + result.push_str("]\n"); + + // Process children + let children: Vec<_> = node.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last_child = i == child_count - 1; + let new_prefix = format!("{}{}", prefix, if is_last { " " } else { "│ " }); + + result.push_str(&Tree::node_to_string( + entry.value(), + &new_prefix, + is_last_child, + )); + } + + result + } + + pub fn pretty_print(&self) { + if self.root.children.is_empty() { + return; + } + + let mut result = String::new(); + let children: Vec<_> = self.root.children.iter().collect(); + let child_count = children.len(); + + for (i, entry) in children.iter().enumerate() { + let is_last = i == child_count - 1; + result.push_str(&Tree::node_to_string(entry.value(), "", is_last)); + } + + println!("{result}"); + } +} + +// Unit tests +#[cfg(test)] +mod tests { + use rand::distr::Alphanumeric; + use rand::distr::SampleString; + use rand::rng as thread_rng; + use rand::Rng; + use std::thread; + use std::time::Instant; + + use super::*; + + #[test] + fn test_get_smallest_tenant() { + let tree = Tree::new(); + + // Test empty tree + assert_eq!(tree.get_smallest_tenant(), "empty"); + + // Insert data for tenant1 - "ap" + "icot" = 6 chars + tree.insert("ap", "tenant1"); + tree.insert("icot", "tenant1"); + + // Insert data for tenant2 - "cat" = 3 chars + tree.insert("cat", "tenant2"); + + // Test - tenant2 should be smallest with 3 chars vs 6 chars + assert_eq!( + tree.get_smallest_tenant(), + "tenant2", + "Expected tenant2 to be smallest with 3 characters." + ); + + // Insert overlapping data for tenant3 and tenant4 to test equal counts + // tenant3: "do" = 2 chars + // tenant4: "hi" = 2 chars + tree.insert("do", "tenant3"); + tree.insert("hi", "tenant4"); + + // Test - should return either tenant3 or tenant4 (both have 2 chars) + let smallest = tree.get_smallest_tenant(); + assert!( + smallest == "tenant3" || smallest == "tenant4", + "Expected either tenant3 or tenant4 (both have 2 characters), got {}", + smallest + ); + + // Add more text to tenant4 to make it larger + tree.insert("hello", "tenant4"); // Now tenant4 has "hi" + "hello" = 6 chars + + // Now tenant3 should be smallest (2 chars vs 6 chars for tenant4) + assert_eq!( + tree.get_smallest_tenant(), + "tenant3", + "Expected tenant3 to be smallest with 2 characters" + ); + + // Test eviction + tree.evict_tenant_by_size(3); // This should evict tenants with more than 3 chars + + let post_eviction_smallest = tree.get_smallest_tenant(); + println!("Smallest tenant after eviction: {}", post_eviction_smallest); + } + + #[test] + fn test_tenant_char_count() { + let tree = Tree::new(); + + // Phase 1: Initial insertions + tree.insert("apple", "tenant1"); + tree.insert("apricot", "tenant1"); + tree.insert("banana", "tenant1"); + tree.insert("amplify", "tenant2"); + tree.insert("application", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 1 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 1: Initial insertions" + ); + + // Phase 2: Additional insertions + tree.insert("apartment", "tenant1"); + tree.insert("appetite", "tenant2"); + tree.insert("ball", "tenant1"); + tree.insert("box", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 2 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 2: Additional insertions" + ); + + // Phase 3: Overlapping insertions + tree.insert("zebra", "tenant1"); + tree.insert("zebra", "tenant2"); + tree.insert("zero", "tenant1"); + tree.insert("zero", "tenant2"); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 3 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!( + maintained_counts, computed_sizes, + "Phase 3: Overlapping insertions" + ); + + // Phase 4: Eviction test + tree.evict_tenant_by_size(10); + + let computed_sizes = tree.get_used_size_per_tenant(); + let maintained_counts: HashMap = tree + .tenant_char_count + .iter() + .map(|entry| (entry.key().clone(), *entry.value())) + .collect(); + + println!("Phase 4 - Maintained vs Computed counts:"); + println!( + "Maintained: {:?}\nComputed: {:?}", + maintained_counts, computed_sizes + ); + assert_eq!(maintained_counts, computed_sizes, "Phase 4: After eviction"); + } + + fn random_string(len: usize) -> String { + Alphanumeric.sample_string(&mut thread_rng(), len) + } + + #[test] + fn test_cold_start() { + let tree = Tree::new(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + + assert_eq!(matched_text, ""); + assert_eq!(tenant, "empty"); + } + + #[test] + fn test_exact_match_seq() { + let tree = Tree::new(); + tree.insert("hello", "tenant1"); + tree.pretty_print(); + tree.insert("apple", "tenant2"); + tree.pretty_print(); + tree.insert("banana", "tenant3"); + tree.pretty_print(); + + let (matched_text, tenant) = tree.prefix_match("hello"); + assert_eq!(matched_text, "hello"); + assert_eq!(tenant, "tenant1"); + + let (matched_text, tenant) = tree.prefix_match("apple"); + assert_eq!(matched_text, "apple"); + assert_eq!(tenant, "tenant2"); + + let (matched_text, tenant) = tree.prefix_match("banana"); + assert_eq!(matched_text, "banana"); + assert_eq!(tenant, "tenant3"); + } + + #[test] + fn test_exact_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + let texts = ["hello", "apple", "banana"]; + let tenants = ["tenant1", "tenant2", "tenant3"]; + + let mut handles = vec![]; + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for i in 0..3 { + let tree_clone = Arc::clone(&tree_clone); + let text = texts[i]; + let tenant = tenants[i]; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_partial_match_concurrent() { + let tree = Arc::new(Tree::new()); + + // spawn 3 threads for insert + let tree_clone = Arc::clone(&tree); + + static TEXTS: [&str; 3] = ["apple", "apabc", "acbdeds"]; + + let mut handles = vec![]; + + for text in TEXTS.iter() { + let tree_clone = Arc::clone(&tree_clone); + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + // spawn 3 threads for match + let mut handles = vec![]; + + let tree_clone = Arc::clone(&tree); + + for text in TEXTS.iter() { + let tree_clone = Arc::clone(&tree_clone); + let tenant = "tenant0"; + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_group_prefix_insert_match_concurrent() { + static PREFIXES: [&str; 4] = [ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffixes = [ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + for suffix in suffixes.iter() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix, suffix); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // check matching using multi threads + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(prefix); + let tenant = format!("tenant{}", i); + assert_eq!(matched_text, *prefix); + assert_eq!(matched_tenant, tenant); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_mixed_concurrent_insert_match() { + // ensure it does not deadlock instead of doing correctness check + + static PREFIXES: [&str; 4] = [ + "Clock strikes midnight, I'm still wide awake", + "Got dreams bigger than these city lights", + "Time waits for no one, gotta make my move", + "Started from the bottom, that's no metaphor", + ]; + let suffixes = [ + "Got too much to prove, ain't got time to lose", + "History in the making, yeah, you can't erase this", + ]; + let tree = Arc::new(Tree::new()); + + let mut handles = vec![]; + + for (i, prefix) in PREFIXES.iter().enumerate() { + for suffix in suffixes.iter() { + let tree_clone = Arc::clone(&tree); + let text = format!("{} {}", prefix, suffix); + let tenant = format!("tenant{}", i); + + let handle = thread::spawn(move || { + tree_clone.insert(&text, &tenant); + }); + + handles.push(handle); + } + } + + // check matching using multi threads + for prefix in PREFIXES.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (_matched_text, _matched_tenant) = tree_clone.prefix_match(prefix); + }); + + handles.push(handle); + } + + // wait + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_utf8_split_seq() { + // The string should be indexed and split by a utf-8 value basis instead of byte basis + // use .chars() to get the iterator of the utf-8 value + let tree = Arc::new(Tree::new()); + + static TEST_PAIRS: [(&str, &str); 3] = [ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Insert sequentially + for (text, tenant) in TEST_PAIRS.iter() { + tree.insert(text, tenant); + } + + tree.pretty_print(); + + // Test sequentially + + for (text, tenant) in TEST_PAIRS.iter() { + let (matched_text, matched_tenant) = tree.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, *tenant); + } + } + + #[test] + fn test_utf8_split_concurrent() { + let tree = Arc::new(Tree::new()); + + static TEST_PAIRS: [(&str, &str); 3] = [ + ("你好嗎", "tenant1"), + ("你好喔", "tenant2"), + ("你心情好嗎", "tenant3"), + ]; + + // Create multiple threads for insertion + let mut handles = vec![]; + + for (text, tenant) in TEST_PAIRS.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + tree_clone.insert(text, tenant); + }); + + handles.push(handle); + } + + // Wait for all insertions to complete + for handle in handles { + handle.join().unwrap(); + } + + tree.pretty_print(); + + // Create multiple threads for matching + let mut handles = vec![]; + + for (text, tenant) in TEST_PAIRS.iter() { + let tree_clone = Arc::clone(&tree); + + let handle = thread::spawn(move || { + let (matched_text, matched_tenant) = tree_clone.prefix_match(text); + assert_eq!(matched_text, *text); + assert_eq!(matched_tenant, *tenant); + }); + + handles.push(handle); + } + + // Wait for all matches to complete + for handle in handles { + handle.join().unwrap(); + } + } + + #[test] + fn test_simple_eviction() { + let tree = Tree::new(); + let max_size = 5; + + // Insert strings for both tenants + tree.insert("hello", "tenant1"); // size 5 + + tree.insert("hello", "tenant2"); // size 5 + thread::sleep(Duration::from_millis(10)); + tree.insert("world", "tenant2"); // size 5, total for tenant2 = 10 + + tree.pretty_print(); + + // Verify initial sizes + let sizes_before = tree.get_used_size_per_tenant(); + assert_eq!(sizes_before.get("tenant1").unwrap(), &5); // "hello" = 5 + assert_eq!(sizes_before.get("tenant2").unwrap(), &10); // "hello" + "world" = 10 + + // Evict - should remove "hello" from tenant2 as it's the oldest + tree.evict_tenant_by_size(max_size); + + tree.pretty_print(); + + // Verify sizes after eviction + let sizes_after = tree.get_used_size_per_tenant(); + assert_eq!(sizes_after.get("tenant1").unwrap(), &5); // Should be unchanged + assert_eq!(sizes_after.get("tenant2").unwrap(), &5); // Only "world" remains + + // Verify "world" remains for tenant2 + let (matched, tenant) = tree.prefix_match("world"); + assert_eq!(matched, "world"); + assert_eq!(tenant, "tenant2"); + } + + #[test] + fn test_advanced_eviction() { + let tree = Tree::new(); + + // Set limits for each tenant + let max_size: usize = 100; + + // Define prefixes + let prefixes = ["aqwefcisdf", "iajsdfkmade", "kjnzxcvewqe", "iejksduqasd"]; + + // Insert strings with shared prefixes + for _i in 0..100 { + for (j, prefix) in prefixes.iter().enumerate() { + let random_suffix = random_string(10); + let text = format!("{}{}", prefix, random_suffix); + let tenant = format!("tenant{}", j + 1); + tree.insert(&text, &tenant); + } + } + + // Perform eviction + tree.evict_tenant_by_size(max_size); + + // Check sizes after eviction + let sizes_after = tree.get_used_size_per_tenant(); + // Verify all tenants are under their size limits + for (tenant, &size) in sizes_after.iter() { + assert!( + size <= max_size, + "Tenant {} exceeds size limit. Current size: {}, Limit: {}", + tenant, + size, + max_size + ); + } + } + + #[test] + fn test_concurrent_operations_with_eviction() { + // Ensure eviction works fine with concurrent insert and match operations for a given period + + let tree = Arc::new(Tree::new()); + let mut handles = vec![]; + let test_duration = Duration::from_secs(10); + let start_time = Instant::now(); + let max_size = 100; // Single max size for all tenants + + // Spawn eviction thread + { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + while start_time.elapsed() < test_duration { + // Run eviction + tree.evict_tenant_by_size(max_size); + + // Sleep for 5 seconds + thread::sleep(Duration::from_secs(5)); + } + }); + handles.push(handle); + } + + // Spawn 4 worker threads + for thread_id in 0..4 { + let tree = Arc::clone(&tree); + let handle = thread::spawn(move || { + let mut rng = rand::rng(); + let tenant = format!("tenant{}", thread_id + 1); + let prefix = format!("prefix{}", thread_id); + + while start_time.elapsed() < test_duration { + // Random decision: match or insert (70% match, 30% insert) + if rng.random_bool(0.7) { + // Perform match operation + let random_len = rng.random_range(3..10); + let search_str = format!("{}{}", prefix, random_string(random_len)); + let (_matched, _) = tree.prefix_match(&search_str); + } else { + // Perform insert operation + let random_len = rng.random_range(5..15); + let insert_str = format!("{}{}", prefix, random_string(random_len)); + tree.insert(&insert_str, &tenant); + // println!("Thread {} inserted: {}", thread_id, insert_str); + } + + // Small random sleep to vary timing + thread::sleep(Duration::from_millis(rng.random_range(10..100))); + } + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().unwrap(); + } + + // final eviction + tree.evict_tenant_by_size(max_size); + + // Final size check + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes after test completion: {:?}", final_sizes); + + // Verify all tenants are under limit + for (_, &size) in final_sizes.iter() { + assert!( + size <= max_size, + "Tenant exceeds size limit. Final size: {}, Limit: {}", + size, + max_size + ); + } + } + + #[test] + fn test_leaf_of() { + let tree = Tree::new(); + + // Single node + tree.insert("hello", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves, vec!["tenant1"]); + + // Node with multiple tenants + tree.insert("hello", "tenant2"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert_eq!(leaves.len(), 2); + assert!(leaves.contains(&"tenant1".to_string())); + assert!(leaves.contains(&"tenant2".to_string())); + + // Non-leaf node + tree.insert("hi", "tenant1"); + let leaves = Tree::leaf_of(&tree.root.children.get(&'h').unwrap()); + assert!(leaves.is_empty()); + } + + #[test] + fn test_get_used_size_per_tenant() { + let tree = Tree::new(); + + // Single tenant + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + + // Multiple tenants sharing nodes + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + let sizes = tree.get_used_size_per_tenant(); + + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant1").unwrap(), &10); + assert_eq!(sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // UTF-8 characters + tree.insert("你好", "tenant3"); + let sizes = tree.get_used_size_per_tenant(); + tree.pretty_print(); + println!("{:?}", sizes); + assert_eq!(sizes.get("tenant3").unwrap(), &2); // 2 Chinese characters + + tree.pretty_print(); + } + + #[test] + fn test_prefix_match_tenant() { + let tree = Tree::new(); + + // Insert overlapping prefixes for different tenants + tree.insert("hello", "tenant1"); // tenant1: hello + tree.insert("hello", "tenant2"); // tenant2: hello + tree.insert("hello world", "tenant2"); // tenant2: hello -> world + tree.insert("help", "tenant1"); // tenant1: hel -> p + tree.insert("helicopter", "tenant2"); // tenant2: hel -> icopter + + // Test tenant1's data + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), "hello"); // Full match for tenant1 + assert_eq!(tree.prefix_match_tenant("help", "tenant1"), "help"); // Exclusive to tenant1 + assert_eq!(tree.prefix_match_tenant("hel", "tenant1"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("hello world", "tenant1"), "hello"); // Should stop at tenant1's boundary + assert_eq!(tree.prefix_match_tenant("helicopter", "tenant1"), "hel"); // Should stop at tenant1's boundary + + // Test tenant2's data + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); // Full match for tenant2 + assert_eq!( + tree.prefix_match_tenant("hello world", "tenant2"), + "hello world" + ); // Exclusive to tenant2 + assert_eq!( + tree.prefix_match_tenant("helicopter", "tenant2"), + "helicopter" + ); // Exclusive to tenant2 + assert_eq!(tree.prefix_match_tenant("hel", "tenant2"), "hel"); // Shared prefix + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "hel"); // Should stop at tenant2's boundary + + // Test non-existent tenant + assert_eq!(tree.prefix_match_tenant("hello", "tenant3"), ""); // Non-existent tenant + assert_eq!(tree.prefix_match_tenant("help", "tenant3"), ""); // Non-existent tenant + } + + #[test] + fn test_simple_tenant_eviction() { + let tree = Tree::new(); + + // Insert data for multiple tenants + tree.insert("hello", "tenant1"); + tree.insert("world", "tenant1"); + tree.insert("hello", "tenant2"); + tree.insert("help", "tenant2"); + + // Verify initial state + let initial_sizes = tree.get_used_size_per_tenant(); + assert_eq!(initial_sizes.get("tenant1").unwrap(), &10); // "hello" + "world" + assert_eq!(initial_sizes.get("tenant2").unwrap(), &6); // "hello" + "p" + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + // Verify after eviction + let final_sizes = tree.get_used_size_per_tenant(); + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + assert_eq!( + final_sizes.get("tenant2").unwrap(), + &6, + "tenant2 should be unaffected" + ); + + // Verify tenant1's data is inaccessible + assert_eq!(tree.prefix_match_tenant("hello", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("world", "tenant1"), ""); + + // Verify tenant2's data is still accessible + assert_eq!(tree.prefix_match_tenant("hello", "tenant2"), "hello"); + assert_eq!(tree.prefix_match_tenant("help", "tenant2"), "help"); + } + + #[test] + fn test_complex_tenant_eviction() { + let tree = Tree::new(); + + // Create a more complex tree structure with shared prefixes + tree.insert("apple", "tenant1"); + tree.insert("application", "tenant1"); + tree.insert("apple", "tenant2"); + tree.insert("appetite", "tenant2"); + tree.insert("banana", "tenant1"); + tree.insert("banana", "tenant2"); + tree.insert("ball", "tenant2"); + + // Verify initial state + let initial_sizes = tree.get_used_size_per_tenant(); + println!("Initial sizes: {:?}", initial_sizes); + tree.pretty_print(); + + // Evict tenant1 + tree.remove_tenant("tenant1"); + + // Verify final state + let final_sizes = tree.get_used_size_per_tenant(); + println!("Final sizes: {:?}", final_sizes); + tree.pretty_print(); + + // Verify tenant1 is completely removed + assert!( + !final_sizes.contains_key("tenant1"), + "tenant1 should be completely removed" + ); + + // Verify all tenant1's data is inaccessible + assert_eq!(tree.prefix_match_tenant("apple", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("application", "tenant1"), ""); + assert_eq!(tree.prefix_match_tenant("banana", "tenant1"), ""); + + // Verify tenant2's data is intact + assert_eq!(tree.prefix_match_tenant("apple", "tenant2"), "apple"); + assert_eq!(tree.prefix_match_tenant("appetite", "tenant2"), "appetite"); + assert_eq!(tree.prefix_match_tenant("banana", "tenant2"), "banana"); + assert_eq!(tree.prefix_match_tenant("ball", "tenant2"), "ball"); + + // Verify the tree structure is still valid for tenant2 + let tenant2_size = final_sizes.get("tenant2").unwrap(); + assert_eq!(tenant2_size, &(5 + 5 + 6 + 2)); // "apple" + "etite" + "banana" + "ll" + } +} diff --git a/sgl-router/tests/api_endpoints_test.rs b/sgl-router/tests/api_endpoints_test.rs new file mode 100644 index 000000000..8b2e29714 --- /dev/null +++ b/sgl-router/tests/api_endpoints_test.rs @@ -0,0 +1,1669 @@ +mod common; + +use axum::{ + body::Body, + extract::Request, + http::{header::CONTENT_TYPE, StatusCode}, +}; +use common::mock_worker::{HealthStatus, MockWorker, MockWorkerConfig, WorkerType}; +use reqwest::Client; +use serde_json::json; +use sglang_router_rs::config::{ + CircuitBreakerConfig, ConnectionMode, PolicyConfig, RetryConfig, RouterConfig, RoutingMode, +}; +use sglang_router_rs::routers::{RouterFactory, RouterTrait}; +use std::sync::Arc; +use tower::ServiceExt; + +/// Test context that manages mock workers +struct TestContext { + workers: Vec, + router: Arc, + client: Client, + config: RouterConfig, +} + +impl TestContext { + async fn new(worker_configs: Vec) -> Self { + // Create default router config + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3002, + max_payload_size: 256 * 1024 * 1024, + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + discovery: None, + dp_aware: false, + api_key: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + Self::new_with_config(config, worker_configs).await + } + + async fn new_with_config( + mut config: RouterConfig, + worker_configs: Vec, + ) -> Self { + let mut workers = Vec::new(); + let mut worker_urls = Vec::new(); + + // Start mock workers if any + for worker_config in worker_configs { + let mut worker = MockWorker::new(worker_config); + let url = worker.start().await.unwrap(); + worker_urls.push(url); + workers.push(worker); + } + + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + } + + // Update config with worker URLs if not already set + if let RoutingMode::Regular { + worker_urls: ref mut urls, + } = config.mode + { + if urls.is_empty() { + *urls = worker_urls.clone(); + } + } + + let client = Client::builder() + .timeout(std::time::Duration::from_secs(config.request_timeout_secs)) + .build() + .unwrap(); + + // Create app context + let app_context = common::create_test_context(config.clone()); + + // Create router + let router = RouterFactory::create_router(&app_context).await.unwrap(); + let router = Arc::from(router); + + // Wait for router to discover workers + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + } + + Self { + workers, + router, + client, + config, + } + } + + async fn create_app(&self) -> axum::Router { + common::test_app::create_test_app( + Arc::clone(&self.router), + self.client.clone(), + &self.config, + ) + } + + async fn shutdown(mut self) { + for worker in &mut self.workers { + worker.stop().await; + } + } +} + +#[cfg(test)] +mod health_tests { + use super::*; + + #[tokio::test] + async fn test_liveness_endpoint() { + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/liveness") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_readiness_with_healthy_workers() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18001, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/readiness") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_readiness_with_unhealthy_workers() { + let ctx = TestContext::new(vec![]).await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/readiness") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // With no workers, readiness should return SERVICE_UNAVAILABLE + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_health_endpoint_details() { + let ctx = TestContext::new(vec![ + MockWorkerConfig { + port: 18003, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + MockWorkerConfig { + port: 18004, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + ]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/health") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // The health endpoint returns plain text, not JSON + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_str = String::from_utf8_lossy(&body); + assert!(body_str.contains("All servers healthy")); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_health_generate_endpoint() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18005, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/health_generate") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.is_object()); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod generation_tests { + use super::*; + + #[tokio::test] + async fn test_generate_success() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18101, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "text": "Hello, world!", + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.get("text").is_some()); + assert!(body_json.get("meta_info").is_some()); + let meta_info = &body_json["meta_info"]; + assert!(meta_info.get("finish_reason").is_some()); + assert_eq!(meta_info["finish_reason"]["type"], "stop"); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_generate_streaming() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18102, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "text": "Stream test", + "stream": true + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // For streaming responses, the router might use chunked encoding or other streaming mechanisms + // The exact content-type can vary based on the router implementation + // Just verify we got a successful response + // Note: In a real implementation, we'd check for text/event-stream or appropriate streaming headers + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_generate_with_worker_failure() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18103, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 1.0, // Always fail + }]) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "text": "This should fail", + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_chat_completions_success() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18104, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "model": "test-model", + "messages": [ + {"role": "user", "content": "Hello!"} + ], + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.get("choices").is_some()); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod model_info_tests { + use super::*; + + #[tokio::test] + async fn test_get_server_info() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18201, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/get_server_info") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.is_object()); + // Check for actual sglang server fields + assert!(body_json.get("version").is_some()); + assert!(body_json.get("model_path").is_some()); + assert!(body_json.get("tokenizer_path").is_some()); + assert!(body_json.get("port").is_some()); + assert!(body_json.get("max_num_batched_tokens").is_some()); + assert!(body_json.get("schedule_policy").is_some()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_get_model_info() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18202, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/get_model_info") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.is_object()); + // Check for actual sglang model info fields + assert_eq!( + body_json.get("model_path").and_then(|v| v.as_str()), + Some("mock-model-path") + ); + assert_eq!( + body_json.get("tokenizer_path").and_then(|v| v.as_str()), + Some("mock-tokenizer-path") + ); + assert_eq!( + body_json.get("is_generation").and_then(|v| v.as_bool()), + Some(true) + ); + assert!(body_json.get("preferred_sampling_params").is_some()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_models() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18203, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/v1/models") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!(body_json.get("object").is_some()); + assert_eq!( + body_json.get("object").and_then(|v| v.as_str()), + Some("list") + ); + + let data = body_json.get("data").and_then(|v| v.as_array()); + assert!(data.is_some()); + + let models = data.unwrap(); + assert!(!models.is_empty()); + + let first_model = &models[0]; + assert_eq!( + first_model.get("id").and_then(|v| v.as_str()), + Some("mock-model") + ); + assert_eq!( + first_model.get("object").and_then(|v| v.as_str()), + Some("model") + ); + assert!(first_model.get("created").is_some()); + assert_eq!( + first_model.get("owned_by").and_then(|v| v.as_str()), + Some("organization-owner") + ); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_model_info_with_no_workers() { + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + // Test server info with no workers + let req = Request::builder() + .method("GET") + .uri("/get_server_info") + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + // Router may return various error codes when no workers + assert!( + resp.status() == StatusCode::OK + || resp.status() == StatusCode::SERVICE_UNAVAILABLE + || resp.status() == StatusCode::NOT_FOUND + || resp.status() == StatusCode::INTERNAL_SERVER_ERROR, + "Unexpected status code: {:?}", + resp.status() + ); + + // Test model info with no workers + let req = Request::builder() + .method("GET") + .uri("/get_model_info") + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + // Router may return various error codes when no workers + assert!( + resp.status() == StatusCode::OK + || resp.status() == StatusCode::SERVICE_UNAVAILABLE + || resp.status() == StatusCode::NOT_FOUND + || resp.status() == StatusCode::INTERNAL_SERVER_ERROR, + "Unexpected status code: {:?}", + resp.status() + ); + + // Test v1/models with no workers + let req = Request::builder() + .method("GET") + .uri("/v1/models") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + // Router may return various error codes when no workers + assert!( + resp.status() == StatusCode::OK + || resp.status() == StatusCode::SERVICE_UNAVAILABLE + || resp.status() == StatusCode::NOT_FOUND + || resp.status() == StatusCode::INTERNAL_SERVER_ERROR, + "Unexpected status code: {:?}", + resp.status() + ); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_model_info_with_multiple_workers() { + let ctx = TestContext::new(vec![ + MockWorkerConfig { + port: 18204, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + MockWorkerConfig { + port: 18205, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + ]) + .await; + + let app = ctx.create_app().await; + + // Test that model info is consistent across workers + for _ in 0..5 { + let req = Request::builder() + .method("GET") + .uri("/get_model_info") + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + body_json.get("model_path").and_then(|v| v.as_str()), + Some("mock-model-path") + ); + } + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_model_info_with_unhealthy_worker() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18206, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 1.0, // Always fail + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/get_model_info") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Worker with fail_rate: 1.0 should always return an error status + assert!( + resp.status() == StatusCode::INTERNAL_SERVER_ERROR + || resp.status() == StatusCode::SERVICE_UNAVAILABLE, + "Expected error status for always-failing worker, got: {:?}", + resp.status() + ); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod worker_management_tests { + use super::*; + + #[tokio::test] + async fn test_add_new_worker() { + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + // Start a mock worker + let mut worker = MockWorker::new(MockWorkerConfig { + port: 18301, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }); + let url = worker.start().await.unwrap(); + + // Add the worker + let req = Request::builder() + .method("POST") + .uri(format!("/add_worker?url={}", url)) + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // List workers to verify + let req = Request::builder() + .method("GET") + .uri("/list_workers") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let workers = body_json["urls"].as_array().unwrap(); + assert!(workers.iter().any(|w| w.as_str().unwrap() == url)); + + worker.stop().await; + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_remove_existing_worker() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18302, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // Get the worker URL + let req = Request::builder() + .method("GET") + .uri("/list_workers") + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let workers = body_json["urls"].as_array().unwrap(); + let worker_url = workers[0].as_str().unwrap(); + + // Remove the worker + let req = Request::builder() + .method("POST") + .uri(format!("/remove_worker?url={}", worker_url)) + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // Verify it's removed + let req = Request::builder() + .method("GET") + .uri("/list_workers") + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let workers = body_json["urls"].as_array().unwrap(); + assert!(workers.is_empty()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_add_worker_invalid_url() { + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + // Invalid URL format + let req = Request::builder() + .method("POST") + .uri("/add_worker?url=not-a-valid-url") + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + // Missing URL parameter + let req = Request::builder() + .method("POST") + .uri("/add_worker") + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + // Empty URL + let req = Request::builder() + .method("POST") + .uri("/add_worker?url=") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_add_duplicate_worker() { + // Start a mock worker + let mut worker = MockWorker::new(MockWorkerConfig { + port: 18303, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }); + let url = worker.start().await.unwrap(); + + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + // Add worker first time + let req = Request::builder() + .method("POST") + .uri(format!("/add_worker?url={}", url)) + .body(Body::empty()) + .unwrap(); + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + + // Try to add same worker again + let req = Request::builder() + .method("POST") + .uri(format!("/add_worker?url={}", url)) + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + // Should return error for duplicate + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + worker.stop().await; + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_add_unhealthy_worker() { + // Start unhealthy worker + let mut worker = MockWorker::new(MockWorkerConfig { + port: 18304, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Unhealthy, + response_delay_ms: 0, + fail_rate: 0.0, + }); + let url = worker.start().await.unwrap(); + + let ctx = TestContext::new(vec![]).await; + let app = ctx.create_app().await; + + // Try to add unhealthy worker + let req = Request::builder() + .method("POST") + .uri(format!("/add_worker?url={}", url)) + .body(Body::empty()) + .unwrap(); + let resp = app.oneshot(req).await.unwrap(); + + // Router should reject unhealthy workers + assert!( + resp.status() == StatusCode::BAD_REQUEST + || resp.status() == StatusCode::SERVICE_UNAVAILABLE + ); + + worker.stop().await; + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod router_policy_tests { + use super::*; + + #[tokio::test] + async fn test_random_policy() { + let ctx = TestContext::new(vec![ + MockWorkerConfig { + port: 18801, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + MockWorkerConfig { + port: 18802, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + ]) + .await; + + // Send multiple requests and verify they succeed + let app = ctx.create_app().await; + + for i in 0..10 { + let payload = json!({ + "text": format!("Request {}", i), + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + } + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_worker_selection() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18203, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let _payload = json!({ + "text": "Test selection", + "stream": false + }); + + // Check that router has the worker + let worker_urls = ctx.router.get_worker_urls(); + assert_eq!(worker_urls.len(), 1); + assert!(worker_urls[0].contains("18203")); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod error_tests { + use super::*; + + #[tokio::test] + async fn test_404_not_found() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18401, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // Test unknown endpoint + let req = Request::builder() + .method("GET") + .uri("/unknown_endpoint") + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + // Test POST to unknown endpoint + let req = Request::builder() + .method("POST") + .uri("/api/v2/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + serde_json::to_string(&json!({"text": "test"})).unwrap(), + )) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_method_not_allowed() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18402, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // GET request to POST-only endpoint + let req = Request::builder() + .method("GET") + .uri("/generate") + .body(Body::empty()) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + // Note: Axum returns 405 for wrong methods on matched routes + assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + + // POST request to GET-only endpoint + let req = Request::builder() + .method("POST") + .uri("/health") + .header(CONTENT_TYPE, "application/json") + .body(Body::from("{}")) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_payload_too_large() { + // Create context with small payload limit + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3010, + max_payload_size: 1024, // 1KB limit + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + let ctx = TestContext::new_with_config( + config, + vec![MockWorkerConfig { + port: 18403, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }], + ) + .await; + + // Note: The server would have payload size middleware configured + // but we cannot test it directly through the test app + // This test is kept for documentation purposes + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_invalid_json_payload() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18404, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // Send invalid JSON + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from("{invalid json}")) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + // Send empty body + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_missing_required_fields() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18405, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // Missing messages in chat completion + let payload = json!({ + "model": "test-model" + // missing "messages" + }); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Axum validates JSON schema - returns 422 for validation errors + assert_eq!(resp.status(), StatusCode::UNPROCESSABLE_ENTITY); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_invalid_model() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18406, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "model": "invalid-model-name-that-does-not-exist", + "messages": [{"role": "user", "content": "Hello"}], + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Mock worker accepts any model, but real implementation might return 400 + assert!(resp.status().is_success() || resp.status() == StatusCode::BAD_REQUEST); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod cache_tests { + use super::*; + + #[tokio::test] + async fn test_flush_cache() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18501, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("POST") + .uri("/flush_cache") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // The response might be empty or contain a message + let body_bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + if !body_bytes.is_empty() { + if let Ok(body) = serde_json::from_slice::(&body_bytes) { + // Check that we got a successful response with expected fields + assert!(body.is_object()); + assert!(body.get("message").is_some() || body.get("status").is_some()); + } + } + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_get_loads() { + let ctx = TestContext::new(vec![ + MockWorkerConfig { + port: 18502, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + MockWorkerConfig { + port: 18503, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + ]) + .await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("GET") + .uri("/get_loads") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let body = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .unwrap(); + let body_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + + // Verify the response contains load information + assert!(body_json.is_object()); + // The exact structure depends on the implementation + // but should contain worker load information + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_flush_cache_no_workers() { + let ctx = TestContext::new(vec![]).await; + + let app = ctx.create_app().await; + + let req = Request::builder() + .method("POST") + .uri("/flush_cache") + .body(Body::empty()) + .unwrap(); + + let resp = app.oneshot(req).await.unwrap(); + // Should either succeed (no-op) or return service unavailable + assert!( + resp.status() == StatusCode::OK || resp.status() == StatusCode::SERVICE_UNAVAILABLE + ); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod load_balancing_tests { + use super::*; + + #[tokio::test] + async fn test_request_distribution() { + // Create multiple workers + let ctx = TestContext::new(vec![ + MockWorkerConfig { + port: 18601, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + MockWorkerConfig { + port: 18602, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }, + ]) + .await; + + let app = ctx.create_app().await; + + // Send multiple requests and track distribution + let mut request_count = 0; + for i in 0..10 { + let payload = json!({ + "text": format!("Request {}", i), + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + if resp.status() == StatusCode::OK { + request_count += 1; + } + } + + // With random policy, all requests should succeed + assert_eq!(request_count, 10); + + ctx.shutdown().await; + } +} + +#[cfg(test)] +mod pd_mode_tests { + use super::*; + + #[tokio::test] + async fn test_pd_mode_routing() { + // Create PD mode configuration with prefill and decode workers + let mut prefill_worker = MockWorker::new(MockWorkerConfig { + port: 18701, + worker_type: WorkerType::Prefill, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }); + + let mut decode_worker = MockWorker::new(MockWorkerConfig { + port: 18702, + worker_type: WorkerType::Decode, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }); + + let prefill_url = prefill_worker.start().await.unwrap(); + let decode_url = decode_worker.start().await.unwrap(); + + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + + // Extract port from prefill URL + let prefill_port = prefill_url + .split(':') + .next_back() + .and_then(|p| p.trim_end_matches('/').parse::().ok()) + .unwrap_or(9000); + + let config = RouterConfig { + mode: RoutingMode::PrefillDecode { + prefill_urls: vec![(prefill_url, Some(prefill_port))], + decode_urls: vec![decode_url], + prefill_policy: None, + decode_policy: None, + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3011, + max_payload_size: 256 * 1024 * 1024, + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + discovery: None, + metrics: None, + log_dir: None, + dp_aware: false, + api_key: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + // Create app context + let app_context = common::create_test_context(config); + + // Create router - this might fail due to health check issues + let router_result = RouterFactory::create_router(&app_context).await; + + // Clean up workers + prefill_worker.stop().await; + decode_worker.stop().await; + + // For now, just verify the configuration was attempted + assert!(router_result.is_err() || router_result.is_ok()); + } +} + +#[cfg(test)] +mod request_id_tests { + use super::*; + + #[tokio::test] + async fn test_request_id_generation() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 18901, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + let app = ctx.create_app().await; + + // Test 1: Request without any request ID header should generate one + let payload = json!({ + "text": "Test request", + "stream": false + }); + + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + // Check that response has x-request-id header + let request_id = resp.headers().get("x-request-id"); + assert!( + request_id.is_some(), + "Response should have x-request-id header" + ); + + let id_value = request_id.unwrap().to_str().unwrap(); + assert!( + id_value.starts_with("gnt-"), + "Generate endpoint should have gnt- prefix" + ); + assert!( + id_value.len() > 4, + "Request ID should have content after prefix" + ); + + // Test 2: Request with custom x-request-id should preserve it + let custom_id = "custom-request-id-123"; + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .header("x-request-id", custom_id) + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let response_id = resp.headers().get("x-request-id"); + assert!(response_id.is_some()); + assert_eq!(response_id.unwrap(), custom_id); + + // Test 3: Different endpoints should have different prefixes + let chat_payload = json!({ + "messages": [{"role": "user", "content": "Hello"}], + "model": "test-model" + }); + + let req = Request::builder() + .method("POST") + .uri("/v1/chat/completions") + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_string(&chat_payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let request_id = resp.headers().get("x-request-id"); + assert!(request_id.is_some()); + assert!(request_id + .unwrap() + .to_str() + .unwrap() + .starts_with("chatcmpl-")); + + // Test 4: Alternative request ID headers should be recognized + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .header("x-correlation-id", "correlation-123") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let response_id = resp.headers().get("x-request-id"); + assert!(response_id.is_some()); + assert_eq!(response_id.unwrap(), "correlation-123"); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_request_id_with_custom_headers() { + // Create config with custom request ID headers + let config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3002, + max_payload_size: 256 * 1024 * 1024, + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + discovery: None, + metrics: None, + dp_aware: false, + api_key: None, + log_dir: None, + log_level: None, + request_id_headers: Some(vec!["custom-id".to_string(), "trace-id".to_string()]), + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + let ctx = TestContext::new_with_config( + config, + vec![MockWorkerConfig { + port: 18902, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }], + ) + .await; + + let app = ctx.create_app().await; + + let payload = json!({ + "text": "Test request", + "stream": false + }); + + // Test custom header is recognized + let req = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .header("custom-id", "my-custom-id") + .body(Body::from(serde_json::to_string(&payload).unwrap())) + .unwrap(); + + let resp = app.clone().oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let response_id = resp.headers().get("x-request-id"); + assert!(response_id.is_some()); + assert_eq!(response_id.unwrap(), "my-custom-id"); + + ctx.shutdown().await; + } +} diff --git a/sgl-router/tests/common/mock_mcp_server.rs b/sgl-router/tests/common/mock_mcp_server.rs new file mode 100644 index 000000000..daeec8001 --- /dev/null +++ b/sgl-router/tests/common/mock_mcp_server.rs @@ -0,0 +1,177 @@ +// tests/common/mock_mcp_server.rs - Mock MCP server for testing +use rmcp::{ + handler::server::{router::tool::ToolRouter, wrapper::Parameters}, + model::*, + service::RequestContext, + tool, tool_handler, tool_router, + transport::streamable_http_server::{ + session::local::LocalSessionManager, StreamableHttpService, + }, + ErrorData as McpError, RoleServer, ServerHandler, +}; +use tokio::net::TcpListener; + +/// Mock MCP server that returns hardcoded responses for testing +pub struct MockMCPServer { + pub port: u16, + pub server_handle: Option>, +} + +/// Simple test server with mock search tools +#[derive(Clone)] +pub struct MockSearchServer { + tool_router: ToolRouter, +} + +#[tool_router] +impl MockSearchServer { + pub fn new() -> Self { + Self { + tool_router: Self::tool_router(), + } + } + + #[tool(description = "Mock web search tool")] + fn brave_web_search( + &self, + Parameters(params): Parameters>, + ) -> Result { + let query = params + .get("query") + .and_then(|v| v.as_str()) + .unwrap_or("test"); + Ok(CallToolResult::success(vec![Content::text(format!( + "Mock search results for: {}", + query + ))])) + } + + #[tool(description = "Mock local search tool")] + fn brave_local_search( + &self, + Parameters(_params): Parameters>, + ) -> Result { + Ok(CallToolResult::success(vec![Content::text( + "Mock local search results", + )])) + } +} + +#[tool_handler] +impl ServerHandler for MockSearchServer { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::V_2024_11_05, + capabilities: ServerCapabilities::builder().enable_tools().build(), + server_info: Implementation::from_build_env(), + instructions: Some("Mock server for testing".to_string()), + } + } + + async fn initialize( + &self, + _request: InitializeRequestParam, + _context: RequestContext, + ) -> Result { + Ok(self.get_info()) + } +} + +impl MockMCPServer { + /// Start a mock MCP server on an available port + pub async fn start() -> Result> { + // Find an available port + let listener = TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + + // Create the MCP service using rmcp's StreamableHttpService + let service = StreamableHttpService::new( + || Ok(MockSearchServer::new()), + LocalSessionManager::default().into(), + Default::default(), + ); + + let app = axum::Router::new().nest_service("/mcp", service); + + let server_handle = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("Mock MCP server failed to start"); + }); + + // Give the server a moment to start + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + Ok(MockMCPServer { + port, + server_handle: Some(server_handle), + }) + } + + /// Get the full URL for this mock server + pub fn url(&self) -> String { + format!("http://127.0.0.1:{}/mcp", self.port) + } + + /// Stop the mock server + pub async fn stop(&mut self) { + if let Some(handle) = self.server_handle.take() { + handle.abort(); + // Wait a moment for cleanup + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + } +} + +impl Drop for MockMCPServer { + fn drop(&mut self) { + if let Some(handle) = self.server_handle.take() { + handle.abort(); + } + } +} + +#[cfg(test)] +mod tests { + #[allow(unused_imports)] + use super::MockMCPServer; + + #[tokio::test] + async fn test_mock_server_startup() { + let mut server = MockMCPServer::start().await.unwrap(); + assert!(server.port > 0); + assert!(server.url().contains(&server.port.to_string())); + server.stop().await; + } + + #[tokio::test] + async fn test_mock_server_with_rmcp_client() { + let mut server = MockMCPServer::start().await.unwrap(); + + // Test that we can connect with rmcp client + use rmcp::transport::StreamableHttpClientTransport; + use rmcp::ServiceExt; + + let transport = StreamableHttpClientTransport::from_uri(server.url().as_str()); + let client = ().serve(transport).await; + + assert!(client.is_ok(), "Should be able to connect to mock server"); + + if let Ok(client) = client { + // Test listing tools + let tools = client.peer().list_all_tools().await; + assert!(tools.is_ok(), "Should be able to list tools"); + + if let Ok(tools) = tools { + assert_eq!(tools.len(), 2, "Should have 2 tools"); + assert!(tools.iter().any(|t| t.name == "brave_web_search")); + assert!(tools.iter().any(|t| t.name == "brave_local_search")); + } + + // Shutdown by dropping the client + drop(client); + } + + server.stop().await; + } +} diff --git a/sgl-router/tests/common/mock_openai_server.rs b/sgl-router/tests/common/mock_openai_server.rs new file mode 100644 index 000000000..643fd5e98 --- /dev/null +++ b/sgl-router/tests/common/mock_openai_server.rs @@ -0,0 +1,238 @@ +//! Mock servers for testing + +#![allow(dead_code)] + +use axum::{ + body::Body, + extract::{Request, State}, + http::{HeaderValue, StatusCode}, + response::sse::{Event, KeepAlive}, + response::{IntoResponse, Response, Sse}, + routing::post, + Json, Router, +}; +use futures_util::stream::{self, StreamExt}; +use serde_json::json; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::net::TcpListener; + +/// Mock OpenAI API server for testing +pub struct MockOpenAIServer { + addr: SocketAddr, + _handle: tokio::task::JoinHandle<()>, +} + +#[derive(Clone)] +struct MockServerState { + require_auth: bool, + expected_auth: Option, +} + +impl MockOpenAIServer { + /// Create and start a new mock OpenAI server + pub async fn new() -> Self { + Self::new_with_auth(None).await + } + + /// Create and start a new mock OpenAI server with optional auth requirement + pub async fn new_with_auth(expected_auth: Option) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let state = Arc::new(MockServerState { + require_auth: expected_auth.is_some(), + expected_auth, + }); + + let app = Router::new() + .route("/v1/chat/completions", post(mock_chat_completions)) + .route("/v1/completions", post(mock_completions)) + .route("/v1/models", post(mock_models).get(mock_models)) + .with_state(state); + + let handle = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + // Give the server a moment to start + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + + Self { + addr, + _handle: handle, + } + } + + /// Get the base URL for this mock server + pub fn base_url(&self) -> String { + format!("http://{}", self.addr) + } +} + +/// Mock chat completions endpoint +async fn mock_chat_completions(req: Request) -> Response { + let (_, body) = req.into_parts(); + let body_bytes = match axum::body::to_bytes(body, usize::MAX).await { + Ok(bytes) => bytes, + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + + let request: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(req) => req, + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + + // Extract model from request or use default (owned String to satisfy 'static in stream) + let model: String = request + .get("model") + .and_then(|v| v.as_str()) + .unwrap_or("gpt-3.5-turbo") + .to_string(); + + // If stream requested, return SSE + let is_stream = request + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + if is_stream { + let created = 1677652288u64; + // Single chunk then [DONE] + let model_chunk = model.clone(); + let event_stream = stream::once(async move { + let chunk = json!({ + "id": "chatcmpl-123456789", + "object": "chat.completion.chunk", + "created": created, + "model": model_chunk, + "choices": [{ + "index": 0, + "delta": { + "content": "Hello!" + }, + "finish_reason": null + }] + }); + Ok::<_, std::convert::Infallible>(Event::default().data(chunk.to_string())) + }) + .chain(stream::once(async { Ok(Event::default().data("[DONE]")) })); + + Sse::new(event_stream) + .keep_alive(KeepAlive::default()) + .into_response() + } else { + // Create a mock non-streaming response + let response = json!({ + "id": "chatcmpl-123456789", + "object": "chat.completion", + "created": 1677652288, + "model": model, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm a mock OpenAI assistant. How can I help you today?" + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21 + } + }); + + Json(response).into_response() + } +} + +/// Mock completions endpoint (legacy) +async fn mock_completions(req: Request) -> Response { + let (_, body) = req.into_parts(); + let body_bytes = match axum::body::to_bytes(body, usize::MAX).await { + Ok(bytes) => bytes, + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + + let request: serde_json::Value = match serde_json::from_slice(&body_bytes) { + Ok(req) => req, + Err(_) => return StatusCode::BAD_REQUEST.into_response(), + }; + + let model = request["model"].as_str().unwrap_or("text-davinci-003"); + + let response = json!({ + "id": "cmpl-123456789", + "object": "text_completion", + "created": 1677652288, + "model": model, + "choices": [{ + "text": " This is a mock completion response.", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 5, + "completion_tokens": 7, + "total_tokens": 12 + } + }); + + Json(response).into_response() +} + +/// Mock models endpoint +async fn mock_models(State(state): State>, req: Request) -> Response { + // Optionally enforce Authorization header + if state.require_auth { + let auth = req + .headers() + .get("authorization") + .or_else(|| req.headers().get("Authorization")) + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()); + let auth_ok = match (&state.expected_auth, auth) { + (Some(expected), Some(got)) => &got == expected, + (None, Some(_)) => true, + _ => false, + }; + if !auth_ok { + let mut response = Response::new(Body::from( + json!({ + "error": { + "message": "Unauthorized", + "type": "invalid_request_error" + } + }) + .to_string(), + )); + *response.status_mut() = StatusCode::UNAUTHORIZED; + response + .headers_mut() + .insert("WWW-Authenticate", HeaderValue::from_static("Bearer")); + return response; + } + } + + let response = json!({ + "object": "list", + "data": [ + { + "id": "gpt-4", + "object": "model", + "created": 1677610602, + "owned_by": "openai" + }, + { + "id": "gpt-3.5-turbo", + "object": "model", + "created": 1677610602, + "owned_by": "openai" + } + ] + }); + + Json(response).into_response() +} diff --git a/sgl-router/tests/common/mock_worker.rs b/sgl-router/tests/common/mock_worker.rs new file mode 100755 index 000000000..16d721607 --- /dev/null +++ b/sgl-router/tests/common/mock_worker.rs @@ -0,0 +1,614 @@ +// Mock worker for testing - these functions are used by integration tests +#![allow(dead_code)] + +use axum::{ + extract::{Json, State}, + http::StatusCode, + response::sse::{Event, KeepAlive}, + response::{IntoResponse, Response, Sse}, + routing::{get, post}, + Router, +}; +use futures_util::stream::{self, StreamExt}; +use serde_json::json; +use std::convert::Infallible; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::sync::RwLock; +use uuid::Uuid; + +/// Configuration for mock worker behavior +#[derive(Clone)] +pub struct MockWorkerConfig { + pub port: u16, + pub worker_type: WorkerType, + pub health_status: HealthStatus, + pub response_delay_ms: u64, + pub fail_rate: f32, +} + +#[derive(Clone, Debug)] +pub enum WorkerType { + Regular, + Prefill, + Decode, +} + +#[derive(Clone, Debug)] +pub enum HealthStatus { + Healthy, + Unhealthy, + Degraded, +} + +/// Mock worker server for testing +pub struct MockWorker { + config: Arc>, + shutdown_handle: Option>, + shutdown_tx: Option>, +} + +impl MockWorker { + pub fn new(config: MockWorkerConfig) -> Self { + Self { + config: Arc::new(RwLock::new(config)), + shutdown_handle: None, + shutdown_tx: None, + } + } + + /// Start the mock worker server + pub async fn start(&mut self) -> Result> { + let config = self.config.clone(); + let port = config.read().await.port; + + // If port is 0, find an available port + let port = if port == 0 { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + drop(listener); + config.write().await.port = port; + port + } else { + port + }; + + let app = Router::new() + .route("/health", get(health_handler)) + .route("/health_generate", get(health_generate_handler)) + .route("/get_server_info", get(server_info_handler)) + .route("/get_model_info", get(model_info_handler)) + .route("/generate", post(generate_handler)) + .route("/v1/chat/completions", post(chat_completions_handler)) + .route("/v1/completions", post(completions_handler)) + .route("/flush_cache", post(flush_cache_handler)) + .route("/v1/models", get(v1_models_handler)) + .with_state(config); + + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + self.shutdown_tx = Some(shutdown_tx); + + // Spawn the server in a separate task + let handle = tokio::spawn(async move { + let listener = match tokio::net::TcpListener::bind(("127.0.0.1", port)).await { + Ok(l) => l, + Err(e) => { + eprintln!("Failed to bind to port {}: {}", port, e); + return; + } + }; + + let server = axum::serve(listener, app).with_graceful_shutdown(async move { + let _ = shutdown_rx.await; + }); + + if let Err(e) = server.await { + eprintln!("Server error: {}", e); + } + }); + + self.shutdown_handle = Some(handle); + + // Wait for the server to start + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + let url = format!("http://127.0.0.1:{}", port); + Ok(url) + } + + /// Stop the mock worker server + pub async fn stop(&mut self) { + if let Some(shutdown_tx) = self.shutdown_tx.take() { + let _ = shutdown_tx.send(()); + } + + if let Some(handle) = self.shutdown_handle.take() { + // Wait for the server to shut down + let _ = tokio::time::timeout(tokio::time::Duration::from_secs(5), handle).await; + } + } +} + +impl Drop for MockWorker { + fn drop(&mut self) { + // Clean shutdown when dropped + if let Some(shutdown_tx) = self.shutdown_tx.take() { + let _ = shutdown_tx.send(()); + } + } +} + +// Handler implementations + +/// Check if request should fail based on configured fail_rate +async fn should_fail(config: &MockWorkerConfig) -> bool { + rand::random::() < config.fail_rate +} + +async fn health_handler(State(config): State>>) -> Response { + let config = config.read().await; + + match config.health_status { + HealthStatus::Healthy => Json(json!({ + "status": "healthy", + "timestamp": SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(), + "worker_type": format!("{:?}", config.worker_type), + })) + .into_response(), + HealthStatus::Unhealthy => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "status": "unhealthy", + "error": "Worker is not responding" + })), + ) + .into_response(), + HealthStatus::Degraded => Json(json!({ + "status": "degraded", + "warning": "High load detected" + })) + .into_response(), + } +} + +async fn health_generate_handler(State(config): State>>) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": "Random failure for testing" + })), + ) + .into_response(); + } + + if matches!(config.health_status, HealthStatus::Healthy) { + Json(json!({ + "status": "ok", + "queue_length": 0, + "processing_time_ms": config.response_delay_ms + })) + .into_response() + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({ + "error": "Generation service unavailable" + })), + ) + .into_response() + } +} + +async fn server_info_handler(State(config): State>>) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": "Random failure for testing" + })), + ) + .into_response(); + } + + Json(json!({ + "model_path": "mock-model-path", + "tokenizer_path": "mock-tokenizer-path", + "port": config.port, + "host": "127.0.0.1", + "max_num_batched_tokens": 32768, + "max_prefill_tokens": 16384, + "mem_fraction_static": 0.88, + "tp_size": 1, + "dp_size": 1, + "stream_interval": 8, + "dtype": "float16", + "device": "cuda", + "enable_flashinfer": true, + "enable_p2p_check": true, + "context_length": 32768, + "chat_template": null, + "disable_radix_cache": false, + "enable_torch_compile": false, + "trust_remote_code": false, + "show_time_cost": false, + "waiting_queue_size": 0, + "running_queue_size": 0, + "req_to_token_ratio": 1.2, + "min_running_requests": 0, + "max_running_requests": 2048, + "max_req_num": 8192, + "max_batch_tokens": 32768, + "schedule_policy": "lpm", + "schedule_conservativeness": 1.0, + "version": "0.3.0", + "internal_states": [{ + "waiting_queue_size": 0, + "running_queue_size": 0 + }] + })) + .into_response() +} + +async fn model_info_handler(State(config): State>>) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": "Random failure for testing" + })), + ) + .into_response(); + } + + Json(json!({ + "model_path": "mock-model-path", + "tokenizer_path": "mock-tokenizer-path", + "is_generation": true, + "preferred_sampling_params": { + "temperature": 0.7, + "top_p": 0.9, + "top_k": 40, + "max_tokens": 2048 + } + })) + .into_response() +} + +async fn generate_handler( + State(config): State>>, + Json(payload): Json, +) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": "Random failure for testing" + })), + ) + .into_response(); + } + + if config.response_delay_ms > 0 { + tokio::time::sleep(tokio::time::Duration::from_millis(config.response_delay_ms)).await; + } + + let is_stream = payload + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + if is_stream { + let stream_delay = config.response_delay_ms; + + // Check if it's a batch request + let is_batch = payload.get("text").and_then(|t| t.as_array()).is_some(); + + let batch_size = if is_batch { + payload + .get("text") + .and_then(|t| t.as_array()) + .map(|arr| arr.len()) + .unwrap_or(1) + } else { + 1 + }; + + let mut events = Vec::new(); + + // Generate events for each item in batch + for i in 0..batch_size { + let timestamp_start = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + + let data = json!({ + "text": format!("Mock response {}", i + 1), + "meta_info": { + "prompt_tokens": 10, + "completion_tokens": 5, + "completion_tokens_wo_jump_forward": 5, + "input_token_logprobs": null, + "output_token_logprobs": null, + "first_token_latency": stream_delay as f64 / 1000.0, + "time_to_first_token": stream_delay as f64 / 1000.0, + "time_per_output_token": 0.01, + "end_time": timestamp_start + (stream_delay as f64 / 1000.0), + "start_time": timestamp_start, + "finish_reason": { + "type": "stop", + "reason": "length" + } + }, + "stage": "mid" + }); + + events.push(Ok::<_, Infallible>(Event::default().data(data.to_string()))); + } + + // Add [DONE] event + events.push(Ok(Event::default().data("[DONE]"))); + + let stream = stream::iter(events); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() + } else { + Json(json!({ + "text": "This is a mock response.", + "meta_info": { + "prompt_tokens": 10, + "completion_tokens": 5, + "completion_tokens_wo_jump_forward": 5, + "input_token_logprobs": null, + "output_token_logprobs": null, + "first_token_latency": config.response_delay_ms as f64 / 1000.0, + "time_to_first_token": config.response_delay_ms as f64 / 1000.0, + "time_per_output_token": 0.01, + "finish_reason": { + "type": "stop", + "reason": "length" + } + } + })) + .into_response() + } +} + +async fn chat_completions_handler( + State(config): State>>, + Json(payload): Json, +) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": { + "message": "Random failure for testing", + "type": "internal_error", + "code": "internal_error" + } + })), + ) + .into_response(); + } + + if config.response_delay_ms > 0 { + tokio::time::sleep(tokio::time::Duration::from_millis(config.response_delay_ms)).await; + } + + let is_stream = payload + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + if is_stream { + let request_id = format!("chatcmpl-{}", Uuid::new_v4()); + + let stream = stream::once(async move { + let chunk = json!({ + "id": request_id, + "object": "chat.completion.chunk", + "created": timestamp, + "model": "mock-model", + "choices": [{ + "index": 0, + "delta": { + "content": "This is a mock chat response." + }, + "finish_reason": null + }] + }); + + Ok::<_, Infallible>(Event::default().data(chunk.to_string())) + }) + .chain(stream::once(async { Ok(Event::default().data("[DONE]")) })); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() + } else { + Json(json!({ + "id": format!("chatcmpl-{}", Uuid::new_v4()), + "object": "chat.completion", + "created": timestamp, + "model": "mock-model", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "This is a mock chat response." + }, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15 + } + })) + .into_response() + } +} + +async fn completions_handler( + State(config): State>>, + Json(payload): Json, +) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": { + "message": "Random failure for testing", + "type": "internal_error", + "code": "internal_error" + } + })), + ) + .into_response(); + } + + if config.response_delay_ms > 0 { + tokio::time::sleep(tokio::time::Duration::from_millis(config.response_delay_ms)).await; + } + + let is_stream = payload + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + if is_stream { + let request_id = format!("cmpl-{}", Uuid::new_v4()); + + let stream = stream::once(async move { + let chunk = json!({ + "id": request_id, + "object": "text_completion", + "created": timestamp, + "model": "mock-model", + "choices": [{ + "text": "This is a mock completion.", + "index": 0, + "logprobs": null, + "finish_reason": null + }] + }); + + Ok::<_, Infallible>(Event::default().data(chunk.to_string())) + }) + .chain(stream::once(async { Ok(Event::default().data("[DONE]")) })); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() + } else { + Json(json!({ + "id": format!("cmpl-{}", Uuid::new_v4()), + "object": "text_completion", + "created": timestamp, + "model": "mock-model", + "choices": [{ + "text": "This is a mock completion.", + "index": 0, + "logprobs": null, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15 + } + })) + .into_response() + } +} + +async fn flush_cache_handler(State(config): State>>) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": "Random failure for testing" + })), + ) + .into_response(); + } + + Json(json!({ + "message": "Cache flushed successfully" + })) + .into_response() +} + +async fn v1_models_handler(State(config): State>>) -> Response { + let config = config.read().await; + + if should_fail(&config).await { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({ + "error": { + "message": "Random failure for testing", + "type": "internal_error", + "code": "internal_error" + } + })), + ) + .into_response(); + } + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + Json(json!({ + "object": "list", + "data": [{ + "id": "mock-model", + "object": "model", + "created": timestamp, + "owned_by": "organization-owner" + }] + })) + .into_response() +} + +impl Default for MockWorkerConfig { + fn default() -> Self { + Self { + port: 0, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + } + } +} diff --git a/sgl-router/tests/common/mod.rs b/sgl-router/tests/common/mod.rs new file mode 100644 index 000000000..0170cd765 --- /dev/null +++ b/sgl-router/tests/common/mod.rs @@ -0,0 +1,103 @@ +// These modules are used by tests and benchmarks +#![allow(dead_code)] + +pub mod mock_mcp_server; +pub mod mock_openai_server; +pub mod mock_worker; +pub mod test_app; + +use sglang_router_rs::config::RouterConfig; +use sglang_router_rs::server::AppContext; +use std::fs; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, OnceLock}; + +/// Helper function to create AppContext for tests +pub fn create_test_context(config: RouterConfig) -> Arc { + Arc::new( + AppContext::new( + config.clone(), + reqwest::Client::new(), + config.max_concurrent_requests, + config.rate_limit_tokens_per_second, + ) + .expect("Failed to create AppContext in test"), + ) +} + +// Tokenizer download configuration +const TINYLLAMA_TOKENIZER_URL: &str = + "https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0/resolve/main/tokenizer.json"; +const CACHE_DIR: &str = ".tokenizer_cache"; +const TINYLLAMA_TOKENIZER_FILENAME: &str = "tinyllama_tokenizer.json"; + +// Global mutex to prevent concurrent downloads +static DOWNLOAD_MUTEX: OnceLock> = OnceLock::new(); + +/// Downloads the TinyLlama tokenizer from HuggingFace if not already cached. +/// Returns the path to the cached tokenizer file. +/// +/// This function is thread-safe and will only download the tokenizer once +/// even if called from multiple threads concurrently. +pub fn ensure_tokenizer_cached() -> PathBuf { + // Get or initialize the mutex + let mutex = DOWNLOAD_MUTEX.get_or_init(|| Mutex::new(())); + + // Lock to ensure only one thread downloads at a time + let _guard = mutex.lock().unwrap(); + + let cache_dir = PathBuf::from(CACHE_DIR); + let tokenizer_path = cache_dir.join(TINYLLAMA_TOKENIZER_FILENAME); + + // Create cache directory if it doesn't exist + if !cache_dir.exists() { + fs::create_dir_all(&cache_dir).expect("Failed to create cache directory"); + } + + // Download tokenizer if not already cached + if !tokenizer_path.exists() { + println!("Downloading TinyLlama tokenizer from HuggingFace..."); + + // Use blocking reqwest client since we're in tests/benchmarks + let client = reqwest::blocking::Client::new(); + let response = client + .get(TINYLLAMA_TOKENIZER_URL) + .send() + .expect("Failed to download tokenizer"); + + if !response.status().is_success() { + panic!("Failed to download tokenizer: HTTP {}", response.status()); + } + + let content = response.bytes().expect("Failed to read tokenizer content"); + + // Verify we got actual JSON content + if content.len() < 100 { + panic!("Downloaded content too small: {} bytes", content.len()); + } + + fs::write(&tokenizer_path, content).expect("Failed to write tokenizer to cache"); + println!( + "Tokenizer downloaded and cached successfully ({} bytes)", + tokenizer_path.metadata().unwrap().len() + ); + } + + tokenizer_path +} + +/// Common test prompts for consistency across tests +pub const TEST_PROMPTS: [&str; 4] = [ + "deep learning is", + "Deep learning is", + "has anyone seen nemo lately", + "another prompt", +]; + +/// Pre-computed hashes for verification +pub const EXPECTED_HASHES: [u64; 4] = [ + 1209591529327510910, + 4181375434596349981, + 6245658446118930933, + 5097285695902185237, +]; diff --git a/sgl-router/tests/common/test_app.rs b/sgl-router/tests/common/test_app.rs new file mode 100644 index 000000000..83d7d456a --- /dev/null +++ b/sgl-router/tests/common/test_app.rs @@ -0,0 +1,52 @@ +use axum::Router; +use reqwest::Client; +use sglang_router_rs::{ + config::RouterConfig, + routers::RouterTrait, + server::{build_app, AppContext, AppState}, +}; +use std::sync::Arc; + +/// Create a test Axum application using the actual server's build_app function +#[allow(dead_code)] +pub fn create_test_app( + router: Arc, + client: Client, + router_config: &RouterConfig, +) -> Router { + // Create AppContext + let app_context = Arc::new( + AppContext::new( + router_config.clone(), + client, + router_config.max_concurrent_requests, + router_config.rate_limit_tokens_per_second, + ) + .expect("Failed to create AppContext in test"), + ); + + // Create AppState with the test router and context + let app_state = Arc::new(AppState { + router, + context: app_context, + concurrency_queue_tx: None, // No queue for tests + }); + + // Configure request ID headers (use defaults if not specified) + let request_id_headers = router_config.request_id_headers.clone().unwrap_or_else(|| { + vec![ + "x-request-id".to_string(), + "x-correlation-id".to_string(), + "x-trace-id".to_string(), + "request-id".to_string(), + ] + }); + + // Use the actual server's build_app function + build_app( + app_state, + router_config.max_payload_size, + request_id_headers, + router_config.cors_allowed_origins.clone(), + ) +} diff --git a/sgl-router/tests/mcp_test.rs b/sgl-router/tests/mcp_test.rs new file mode 100644 index 000000000..9821bffa6 --- /dev/null +++ b/sgl-router/tests/mcp_test.rs @@ -0,0 +1,457 @@ +// This test suite validates the complete MCP implementation against the +// functionality required for SGLang responses API integration. +// +// Test Coverage: +// - Core MCP server functionality +// - Tool session management (individual and multi-tool) +// - Tool execution and error handling +// - Schema adaptation and validation +// - Mock server integration for reliable testing + +mod common; + +use common::mock_mcp_server::MockMCPServer; +use serde_json::json; +use sglang_router_rs::mcp::{McpClientManager, McpConfig, McpError, McpServerConfig, McpTransport}; +use std::collections::HashMap; + +/// Create a new mock server for testing (each test gets its own) +async fn create_mock_server() -> MockMCPServer { + MockMCPServer::start() + .await + .expect("Failed to start mock MCP server") +} + +// Core MCP Server Tests + +#[tokio::test] +async fn test_mcp_server_initialization() { + // Test that we can create an empty configuration + let config = McpConfig { servers: vec![] }; + + // Should fail with no servers + let result = McpClientManager::new(config).await; + assert!(result.is_err(), "Should fail with no servers configured"); +} + +#[tokio::test] +async fn test_server_connection_with_mock() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let result = McpClientManager::new(config).await; + assert!(result.is_ok(), "Should connect to mock server"); + + let mut manager = result.unwrap(); + + let servers = manager.list_servers(); + assert_eq!(servers.len(), 1); + assert!(servers.contains(&"mock_server".to_string())); + + let tools = manager.list_tools(); + assert_eq!(tools.len(), 2, "Should have 2 tools from mock server"); + + assert!(manager.has_tool("brave_web_search")); + assert!(manager.has_tool("brave_local_search")); + + manager.shutdown().await; +} + +#[tokio::test] +async fn test_tool_availability_checking() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let mut manager = McpClientManager::new(config).await.unwrap(); + + let test_tools = vec!["brave_web_search", "brave_local_search", "calculator"]; + for tool in test_tools { + let available = manager.has_tool(tool); + match tool { + "brave_web_search" | "brave_local_search" => { + assert!( + available, + "Tool {} should be available from mock server", + tool + ); + } + "calculator" => { + assert!( + !available, + "Tool {} should not be available from mock server", + tool + ); + } + _ => {} + } + } + + manager.shutdown().await; +} + +#[tokio::test] +async fn test_multi_server_connection() { + let mock_server1 = create_mock_server().await; + let mock_server2 = create_mock_server().await; + + let config = McpConfig { + servers: vec![ + McpServerConfig { + name: "mock_server_1".to_string(), + transport: McpTransport::Streamable { + url: mock_server1.url(), + token: None, + }, + }, + McpServerConfig { + name: "mock_server_2".to_string(), + transport: McpTransport::Streamable { + url: mock_server2.url(), + token: None, + }, + }, + ], + }; + + // Note: This will fail to connect to both servers in the current implementation + // since they return the same tools. The manager will connect to the first one. + let result = McpClientManager::new(config).await; + + if let Ok(mut manager) = result { + let servers = manager.list_servers(); + assert!(!servers.is_empty(), "Should have at least one server"); + + let tools = manager.list_tools(); + assert!(tools.len() >= 2, "Should have tools from servers"); + + manager.shutdown().await; + } +} + +#[tokio::test] +async fn test_tool_execution_with_mock() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let mut manager = McpClientManager::new(config).await.unwrap(); + + let result = manager + .call_tool( + "brave_web_search", + Some( + json!({ + "query": "rust programming", + "count": 1 + }) + .as_object() + .unwrap() + .clone(), + ), + ) + .await; + + assert!( + result.is_ok(), + "Tool execution should succeed with mock server" + ); + + let response = result.unwrap(); + assert!(!response.content.is_empty(), "Should have content"); + + // Check the content + if let rmcp::model::RawContent::Text(text) = &response.content[0].raw { + assert!(text + .text + .contains("Mock search results for: rust programming")); + } else { + panic!("Expected text content"); + } + + manager.shutdown().await; +} + +#[tokio::test] +async fn test_concurrent_tool_execution() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let mut manager = McpClientManager::new(config).await.unwrap(); + + // Execute tools sequentially (true concurrent execution would require Arc) + let tool_calls = vec![ + ("brave_web_search", json!({"query": "test1"})), + ("brave_local_search", json!({"query": "test2"})), + ]; + + for (tool_name, args) in tool_calls { + let result = manager + .call_tool(tool_name, Some(args.as_object().unwrap().clone())) + .await; + + assert!(result.is_ok(), "Tool {} should succeed", tool_name); + let response = result.unwrap(); + assert!(!response.content.is_empty(), "Should have content"); + } + + manager.shutdown().await; +} + +// Error Handling Tests + +#[tokio::test] +async fn test_tool_execution_errors() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let mut manager = McpClientManager::new(config).await.unwrap(); + + // Try to call unknown tool + let result = manager + .call_tool("unknown_tool", Some(serde_json::Map::new())) + .await; + assert!(result.is_err(), "Should fail for unknown tool"); + + match result.unwrap_err() { + McpError::ToolNotFound(name) => { + assert_eq!(name, "unknown_tool"); + } + _ => panic!("Expected ToolNotFound error"), + } + + manager.shutdown().await; +} + +#[tokio::test] +async fn test_connection_without_server() { + let config = McpConfig { + servers: vec![McpServerConfig { + name: "nonexistent".to_string(), + transport: McpTransport::Streamable { + url: "http://localhost:9999/mcp".to_string(), + token: None, + }, + }], + }; + + let result = McpClientManager::new(config).await; + assert!(result.is_err(), "Should fail when no server is running"); + + if let Err(e) = result { + let error_msg = e.to_string(); + assert!( + error_msg.contains("Failed to connect") || error_msg.contains("Connection"), + "Error should be connection-related: {}", + error_msg + ); + } +} + +// Schema Validation Tests + +#[tokio::test] +async fn test_tool_info_structure() { + let mock_server = create_mock_server().await; + + let config = McpConfig { + servers: vec![McpServerConfig { + name: "mock_server".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + let manager = McpClientManager::new(config).await.unwrap(); + + let tools = manager.list_tools(); + let brave_search = tools + .iter() + .find(|t| t.name == "brave_web_search") + .expect("Should have brave_web_search tool"); + + assert_eq!(brave_search.name, "brave_web_search"); + assert!(brave_search.description.contains("Mock web search")); + assert_eq!(brave_search.server, "mock_server"); + assert!(brave_search.parameters.is_some()); +} + +// SSE Parsing Tests (simplified since we don't expose parse_sse_event) + +#[tokio::test] +async fn test_sse_connection() { + let mock_server = create_mock_server().await; + + // Test SSE transport configuration + let config = McpConfig { + servers: vec![McpServerConfig { + name: "sse_server".to_string(), + transport: McpTransport::Sse { + // Mock server doesn't support SSE, but we can test the config + url: format!("http://127.0.0.1:{}/sse", mock_server.port), + token: Some("test_token".to_string()), + }, + }], + }; + + // This will fail to connect but tests the configuration + let result = McpClientManager::new(config).await; + assert!(result.is_err(), "Mock server doesn't support SSE"); +} + +// Connection Type Tests + +#[tokio::test] +async fn test_transport_types() { + // Test different transport configurations + + // HTTP/Streamable transport + let http_config = McpServerConfig { + name: "http_server".to_string(), + transport: McpTransport::Streamable { + url: "http://localhost:8080/mcp".to_string(), + token: Some("auth_token".to_string()), + }, + }; + assert_eq!(http_config.name, "http_server"); + + // SSE transport + let sse_config = McpServerConfig { + name: "sse_server".to_string(), + transport: McpTransport::Sse { + url: "http://localhost:8081/sse".to_string(), + token: None, + }, + }; + assert_eq!(sse_config.name, "sse_server"); + + // STDIO transport + let stdio_config = McpServerConfig { + name: "stdio_server".to_string(), + transport: McpTransport::Stdio { + command: "mcp-server".to_string(), + args: vec!["--port".to_string(), "8082".to_string()], + envs: HashMap::new(), + }, + }; + assert_eq!(stdio_config.name, "stdio_server"); +} + +// Integration Pattern Tests + +#[tokio::test] +async fn test_complete_workflow() { + let mock_server = create_mock_server().await; + + // 1. Initialize configuration + let config = McpConfig { + servers: vec![McpServerConfig { + name: "integration_test".to_string(), + transport: McpTransport::Streamable { + url: mock_server.url(), + token: None, + }, + }], + }; + + // 2. Connect to server + let mut manager = McpClientManager::new(config) + .await + .expect("Should connect to mock server"); + + // 3. Verify server connection + let servers = manager.list_servers(); + assert_eq!(servers.len(), 1); + assert_eq!(servers[0], "integration_test"); + + // 4. Check available tools + let tools = manager.list_tools(); + assert_eq!(tools.len(), 2); + + // 5. Verify specific tools exist + assert!(manager.has_tool("brave_web_search")); + assert!(manager.has_tool("brave_local_search")); + assert!(!manager.has_tool("nonexistent_tool")); + + // 6. Execute a tool + let result = manager + .call_tool( + "brave_web_search", + Some( + json!({ + "query": "SGLang router MCP integration", + "count": 1 + }) + .as_object() + .unwrap() + .clone(), + ), + ) + .await; + + assert!(result.is_ok(), "Tool execution should succeed"); + let response = result.unwrap(); + assert!(!response.content.is_empty(), "Should return content"); + + // 7. Clean shutdown + manager.shutdown().await; + + // Verify all required capabilities for responses API integration + let capabilities = [ + "MCP server initialization", + "Tool server connection and discovery", + "Tool availability checking", + "Tool execution", + "Error handling and robustness", + "Multi-server support", + "Schema adaptation", + "Mock server integration (no external dependencies)", + ]; + + assert_eq!(capabilities.len(), 8); +} diff --git a/sgl-router/tests/request_formats_test.rs b/sgl-router/tests/request_formats_test.rs new file mode 100644 index 000000000..606ca0a41 --- /dev/null +++ b/sgl-router/tests/request_formats_test.rs @@ -0,0 +1,392 @@ +mod common; + +use common::mock_worker::{HealthStatus, MockWorker, MockWorkerConfig, WorkerType}; +use reqwest::Client; +use serde_json::json; +use sglang_router_rs::config::{ + CircuitBreakerConfig, ConnectionMode, PolicyConfig, RetryConfig, RouterConfig, RoutingMode, +}; +use sglang_router_rs::routers::{RouterFactory, RouterTrait}; +use std::sync::Arc; + +/// Test context that manages mock workers +struct TestContext { + workers: Vec, + router: Arc, +} + +impl TestContext { + async fn new(worker_configs: Vec) -> Self { + let mut config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3003, + max_payload_size: 256 * 1024 * 1024, + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + let mut workers = Vec::new(); + let mut worker_urls = Vec::new(); + + for worker_config in worker_configs { + let mut worker = MockWorker::new(worker_config); + let url = worker.start().await.unwrap(); + worker_urls.push(url); + workers.push(worker); + } + + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + } + + config.mode = RoutingMode::Regular { worker_urls }; + + let app_context = common::create_test_context(config); + let router = RouterFactory::create_router(&app_context).await.unwrap(); + let router = Arc::from(router); + + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + } + + Self { workers, router } + } + + async fn shutdown(mut self) { + // Small delay to ensure any pending operations complete + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + for worker in &mut self.workers { + worker.stop().await; + } + + // Another small delay to ensure cleanup completes + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + + async fn make_request( + &self, + endpoint: &str, + body: serde_json::Value, + ) -> Result { + let client = Client::new(); + + // Get any worker URL for testing + let worker_urls = self.router.get_worker_urls(); + if worker_urls.is_empty() { + return Err("No available workers".to_string()); + } + + let worker_url = &worker_urls[0]; + + let response = client + .post(format!("{}{}", worker_url, endpoint)) + .json(&body) + .send() + .await + .map_err(|e| format!("Request failed: {}", e))?; + + if !response.status().is_success() { + return Err(format!("Request failed with status: {}", response.status())); + } + + response + .json::() + .await + .map_err(|e| format!("Failed to parse response: {}", e)) + } +} + +#[cfg(test)] +mod request_format_tests { + use super::*; + + #[tokio::test] + async fn test_generate_request_formats() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19001, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test 1: Basic text request + let payload = json!({ + "text": "Hello, world!", + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + // Test 2: Request with sampling parameters + let payload = json!({ + "text": "Tell me a story", + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 100, + "top_p": 0.9 + }, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + // Test 3: Request with input_ids + let payload = json!({ + "input_ids": [1, 2, 3, 4, 5], + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 50 + }, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_chat_completions_formats() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19002, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test 1: Basic chat completion + let payload = json!({ + "model": "test-model", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello!"} + ], + "stream": false + }); + + let result = ctx.make_request("/v1/chat/completions", payload).await; + assert!(result.is_ok()); + + let response = result.unwrap(); + assert!(response.get("choices").is_some()); + assert!(response.get("id").is_some()); + assert_eq!( + response.get("object").and_then(|v| v.as_str()), + Some("chat.completion") + ); + + // Test 2: Chat completion with parameters + let payload = json!({ + "model": "test-model", + "messages": [ + {"role": "user", "content": "Tell me a joke"} + ], + "temperature": 0.8, + "max_tokens": 150, + "top_p": 0.95, + "stream": false + }); + + let result = ctx.make_request("/v1/chat/completions", payload).await; + assert!(result.is_ok()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_completions_formats() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19003, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test 1: Basic completion + let payload = json!({ + "model": "test-model", + "prompt": "Once upon a time", + "max_tokens": 50, + "stream": false + }); + + let result = ctx.make_request("/v1/completions", payload).await; + assert!(result.is_ok()); + + let response = result.unwrap(); + assert!(response.get("choices").is_some()); + assert_eq!( + response.get("object").and_then(|v| v.as_str()), + Some("text_completion") + ); + + // Test 2: Completion with array prompt + let payload = json!({ + "model": "test-model", + "prompt": ["First prompt", "Second prompt"], + "temperature": 0.5, + "stream": false + }); + + let result = ctx.make_request("/v1/completions", payload).await; + assert!(result.is_ok()); + + // Test 3: Completion with logprobs + let payload = json!({ + "model": "test-model", + "prompt": "The capital of France is", + "max_tokens": 10, + "logprobs": 5, + "stream": false + }); + + let result = ctx.make_request("/v1/completions", payload).await; + assert!(result.is_ok()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_batch_requests() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19004, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test batch text generation + let payload = json!({ + "text": ["First text", "Second text", "Third text"], + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 50 + }, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + // Test batch with input_ids + let payload = json!({ + "input_ids": [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_special_parameters() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19005, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test with return_logprob + let payload = json!({ + "text": "Test", + "return_logprob": true, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + // Test with json_schema + let payload = json!({ + "text": "Generate JSON", + "sampling_params": { + "temperature": 0.0, + "json_schema": "$$ANY$$" + }, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + // Test with ignore_eos + let payload = json!({ + "text": "Continue forever", + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 100, + "ignore_eos": true + }, + "stream": false + }); + + let result = ctx.make_request("/generate", payload).await; + assert!(result.is_ok()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_error_handling() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 19006, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 0.0, + }]) + .await; + + // Test with empty body - should still work with mock worker + let payload = json!({}); + + let result = ctx.make_request("/generate", payload).await; + // Mock worker accepts empty body + assert!(result.is_ok()); + + ctx.shutdown().await; + } +} diff --git a/sgl-router/tests/responses_api_test.rs b/sgl-router/tests/responses_api_test.rs new file mode 100644 index 000000000..dc2253799 --- /dev/null +++ b/sgl-router/tests/responses_api_test.rs @@ -0,0 +1,210 @@ +// Integration test for Responses API + +use sglang_router_rs::protocols::spec::{ + GenerationRequest, ReasoningEffort, ResponseInput, ResponseReasoningParam, ResponseStatus, + ResponseTool, ResponseToolType, ResponsesRequest, ResponsesResponse, ServiceTier, ToolChoice, + ToolChoiceValue, Truncation, UsageInfo, +}; + +#[test] +fn test_responses_request_creation() { + let request = ResponsesRequest { + background: false, + include: None, + input: ResponseInput::Text("Hello, world!".to_string()), + instructions: Some("Be helpful".to_string()), + max_output_tokens: Some(100), + max_tool_calls: None, + metadata: None, + model: Some("test-model".to_string()), + parallel_tool_calls: true, + previous_response_id: None, + reasoning: Some(ResponseReasoningParam { + effort: Some(ReasoningEffort::Medium), + }), + service_tier: ServiceTier::Auto, + store: true, + stream: false, + temperature: Some(0.7), + tool_choice: ToolChoice::Value(ToolChoiceValue::Auto), + tools: vec![ResponseTool { + r#type: ResponseToolType::WebSearchPreview, + }], + top_logprobs: 5, + top_p: Some(0.9), + truncation: Truncation::Disabled, + user: Some("test-user".to_string()), + request_id: "resp_test123".to_string(), + priority: 0, + frequency_penalty: 0.0, + presence_penalty: 0.0, + stop: None, + top_k: -1, + min_p: 0.0, + repetition_penalty: 1.0, + }; + + // Test GenerationRequest trait implementation + assert!(!request.is_stream()); + assert_eq!(request.get_model(), Some("test-model")); + let routing_text = request.extract_text_for_routing(); + assert_eq!(routing_text, "Hello, world!"); +} + +#[test] +fn test_sampling_params_conversion() { + let request = ResponsesRequest { + background: false, + include: None, + input: ResponseInput::Text("Test".to_string()), + instructions: None, + max_output_tokens: Some(50), + max_tool_calls: None, + metadata: None, + model: Some("test-model".to_string()), + parallel_tool_calls: true, // Use default true + previous_response_id: None, + reasoning: None, + service_tier: ServiceTier::Auto, + store: true, // Use default true + stream: false, + temperature: Some(0.8), + tool_choice: ToolChoice::Value(ToolChoiceValue::Auto), + tools: vec![], + top_logprobs: 0, // Use default 0 + top_p: Some(0.95), + truncation: Truncation::Auto, + user: None, + request_id: "resp_test456".to_string(), + priority: 0, + frequency_penalty: 0.1, + presence_penalty: 0.2, + stop: None, + top_k: 10, + min_p: 0.05, + repetition_penalty: 1.1, + }; + + let params = request.to_sampling_params(1000, None); + + // Check that parameters are converted correctly + assert!(params.contains_key("temperature")); + assert!(params.contains_key("top_p")); + assert!(params.contains_key("frequency_penalty")); + assert!(params.contains_key("max_new_tokens")); +} + +#[test] +fn test_responses_response_creation() { + let response = ResponsesResponse::new( + "resp_test789".to_string(), + "test-model".to_string(), + ResponseStatus::Completed, + ); + + assert_eq!(response.id, "resp_test789"); + assert_eq!(response.model, "test-model"); + assert!(response.is_complete()); + assert!(!response.is_in_progress()); + assert!(!response.is_failed()); +} + +#[test] +fn test_usage_conversion() { + let usage_info = UsageInfo::new_with_cached(15, 25, Some(8), 3); + let response_usage = usage_info.to_response_usage(); + + assert_eq!(response_usage.input_tokens, 15); + assert_eq!(response_usage.output_tokens, 25); + assert_eq!(response_usage.total_tokens, 40); + + // Check details are converted correctly + assert!(response_usage.input_tokens_details.is_some()); + assert_eq!( + response_usage + .input_tokens_details + .as_ref() + .unwrap() + .cached_tokens, + 3 + ); + + assert!(response_usage.output_tokens_details.is_some()); + assert_eq!( + response_usage + .output_tokens_details + .as_ref() + .unwrap() + .reasoning_tokens, + 8 + ); + + // Test reverse conversion + let back_to_usage = response_usage.to_usage_info(); + assert_eq!(back_to_usage.prompt_tokens, 15); + assert_eq!(back_to_usage.completion_tokens, 25); + assert_eq!(back_to_usage.reasoning_tokens, Some(8)); +} + +#[test] +fn test_reasoning_param_default() { + let param = ResponseReasoningParam { + effort: Some(ReasoningEffort::Medium), + }; + + // Test JSON serialization/deserialization preserves default + let json = serde_json::to_string(¶m).unwrap(); + let parsed: ResponseReasoningParam = serde_json::from_str(&json).unwrap(); + + assert!(matches!(parsed.effort, Some(ReasoningEffort::Medium))); +} + +#[test] +fn test_json_serialization() { + let request = ResponsesRequest { + background: true, + include: None, + input: ResponseInput::Text("Test input".to_string()), + instructions: Some("Test instructions".to_string()), + max_output_tokens: Some(200), + max_tool_calls: Some(5), + metadata: None, + model: Some("gpt-4".to_string()), + parallel_tool_calls: false, + previous_response_id: None, + reasoning: Some(ResponseReasoningParam { + effort: Some(ReasoningEffort::High), + }), + service_tier: ServiceTier::Priority, + store: false, + stream: true, + temperature: Some(0.9), + tool_choice: ToolChoice::Value(ToolChoiceValue::Required), + tools: vec![ResponseTool { + r#type: ResponseToolType::CodeInterpreter, + }], + top_logprobs: 10, + top_p: Some(0.8), + truncation: Truncation::Auto, + user: Some("test_user".to_string()), + request_id: "resp_comprehensive_test".to_string(), + priority: 1, + frequency_penalty: 0.3, + presence_penalty: 0.4, + stop: None, + top_k: 50, + min_p: 0.1, + repetition_penalty: 1.2, + }; + + // Test that everything can be serialized to JSON and back + let json = serde_json::to_string(&request).expect("Serialization should work"); + let parsed: ResponsesRequest = + serde_json::from_str(&json).expect("Deserialization should work"); + + assert_eq!(parsed.request_id, "resp_comprehensive_test"); + assert_eq!(parsed.model, Some("gpt-4".to_string())); + assert!(parsed.background); + assert!(parsed.stream); + assert_eq!(parsed.tools.len(), 1); +} diff --git a/sgl-router/tests/streaming_tests.rs b/sgl-router/tests/streaming_tests.rs new file mode 100644 index 000000000..29190a312 --- /dev/null +++ b/sgl-router/tests/streaming_tests.rs @@ -0,0 +1,370 @@ +mod common; + +use common::mock_worker::{HealthStatus, MockWorker, MockWorkerConfig, WorkerType}; +use futures_util::StreamExt; +use reqwest::Client; +use serde_json::json; +use sglang_router_rs::config::{ + CircuitBreakerConfig, ConnectionMode, PolicyConfig, RetryConfig, RouterConfig, RoutingMode, +}; +use sglang_router_rs::routers::{RouterFactory, RouterTrait}; +use std::sync::Arc; + +/// Test context that manages mock workers +struct TestContext { + workers: Vec, + router: Arc, +} + +impl TestContext { + async fn new(worker_configs: Vec) -> Self { + let mut config = RouterConfig { + mode: RoutingMode::Regular { + worker_urls: vec![], + }, + policy: PolicyConfig::Random, + host: "127.0.0.1".to_string(), + port: 3004, + max_payload_size: 256 * 1024 * 1024, + request_timeout_secs: 600, + worker_startup_timeout_secs: 1, + worker_startup_check_interval_secs: 1, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + rate_limit_tokens_per_second: None, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + let mut workers = Vec::new(); + let mut worker_urls = Vec::new(); + + for worker_config in worker_configs { + let mut worker = MockWorker::new(worker_config); + let url = worker.start().await.unwrap(); + worker_urls.push(url); + workers.push(worker); + } + + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(200)).await; + } + + config.mode = RoutingMode::Regular { worker_urls }; + + let app_context = common::create_test_context(config); + let router = RouterFactory::create_router(&app_context).await.unwrap(); + let router = Arc::from(router); + + if !workers.is_empty() { + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + } + + Self { workers, router } + } + + async fn shutdown(mut self) { + // Small delay to ensure any pending operations complete + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + for worker in &mut self.workers { + worker.stop().await; + } + + // Another small delay to ensure cleanup completes + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + + async fn make_streaming_request( + &self, + endpoint: &str, + body: serde_json::Value, + ) -> Result, String> { + let client = Client::new(); + + // Get any worker URL for testing + let worker_urls = self.router.get_worker_urls(); + if worker_urls.is_empty() { + return Err("No available workers".to_string()); + } + + let worker_url = &worker_urls[0]; + + let response = client + .post(format!("{}{}", worker_url, endpoint)) + .json(&body) + .send() + .await + .map_err(|e| format!("Request failed: {}", e))?; + + if !response.status().is_success() { + return Err(format!("Request failed with status: {}", response.status())); + } + + // Check if it's a streaming response + let content_type = response + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + + if !content_type.contains("text/event-stream") { + return Err("Response is not a stream".to_string()); + } + + let mut stream = response.bytes_stream(); + let mut events = Vec::new(); + + while let Some(chunk) = stream.next().await { + if let Ok(bytes) = chunk { + let text = String::from_utf8_lossy(&bytes); + for line in text.lines() { + if let Some(stripped) = line.strip_prefix("data: ") { + events.push(stripped.to_string()); + } + } + } + } + + Ok(events) + } +} + +#[cfg(test)] +mod streaming_tests { + use super::*; + + #[tokio::test] + async fn test_generate_streaming() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20001, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 10, + fail_rate: 0.0, + }]) + .await; + + let payload = json!({ + "text": "Stream test", + "stream": true, + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 10 + } + }); + + let result = ctx.make_streaming_request("/generate", payload).await; + assert!(result.is_ok()); + + let events = result.unwrap(); + // Should have at least one data chunk and [DONE] + assert!(events.len() >= 2); + assert_eq!(events.last().unwrap(), "[DONE]"); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_chat_completions_streaming() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20002, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 10, + fail_rate: 0.0, + }]) + .await; + + let payload = json!({ + "model": "test-model", + "messages": [ + {"role": "user", "content": "Count to 3"} + ], + "stream": true, + "max_tokens": 20 + }); + + let result = ctx + .make_streaming_request("/v1/chat/completions", payload) + .await; + assert!(result.is_ok()); + + let events = result.unwrap(); + assert!(events.len() >= 2); // At least one chunk + [DONE] + + // Verify events are valid JSON (except [DONE]) + for event in &events { + if event != "[DONE]" { + let parsed: Result = serde_json::from_str(event); + assert!(parsed.is_ok(), "Invalid JSON in SSE event: {}", event); + + let json = parsed.unwrap(); + assert_eq!( + json.get("object").and_then(|v| v.as_str()), + Some("chat.completion.chunk") + ); + } + } + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_v1_completions_streaming() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20003, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 10, + fail_rate: 0.0, + }]) + .await; + + let payload = json!({ + "model": "test-model", + "prompt": "Once upon a time", + "stream": true, + "max_tokens": 15 + }); + + let result = ctx.make_streaming_request("/v1/completions", payload).await; + assert!(result.is_ok()); + + let events = result.unwrap(); + assert!(events.len() >= 2); // At least one chunk + [DONE] + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_streaming_with_error() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20004, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 0, + fail_rate: 1.0, // Always fail + }]) + .await; + + let payload = json!({ + "text": "This should fail", + "stream": true + }); + + let result = ctx.make_streaming_request("/generate", payload).await; + // With fail_rate: 1.0, the request should fail + assert!(result.is_err()); + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_streaming_timeouts() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20005, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 100, // Slow response + fail_rate: 0.0, + }]) + .await; + + let payload = json!({ + "text": "Slow stream", + "stream": true, + "sampling_params": { + "max_new_tokens": 5 + } + }); + + let start = std::time::Instant::now(); + let result = ctx.make_streaming_request("/generate", payload).await; + let elapsed = start.elapsed(); + + assert!(result.is_ok()); + let events = result.unwrap(); + + // Should have received multiple chunks over time + assert!(!events.is_empty()); + assert!(elapsed.as_millis() >= 100); // At least one delay + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_batch_streaming() { + let ctx = TestContext::new(vec![MockWorkerConfig { + port: 20006, + worker_type: WorkerType::Regular, + health_status: HealthStatus::Healthy, + response_delay_ms: 10, + fail_rate: 0.0, + }]) + .await; + + // Batch request with streaming + let payload = json!({ + "text": ["First", "Second", "Third"], + "stream": true, + "sampling_params": { + "max_new_tokens": 5 + } + }); + + let result = ctx.make_streaming_request("/generate", payload).await; + assert!(result.is_ok()); + + let events = result.unwrap(); + // Should have multiple events for batch + assert!(events.len() >= 4); // At least 3 responses + [DONE] + + ctx.shutdown().await; + } + + #[tokio::test] + async fn test_sse_format_parsing() { + // Test SSE format parsing + let parse_sse_chunk = |chunk: &[u8]| -> Vec { + let text = String::from_utf8_lossy(chunk); + text.lines() + .filter(|line| line.starts_with("data: ")) + .map(|line| line[6..].to_string()) + .collect() + }; + + let sse_data = + b"data: {\"text\":\"Hello\"}\n\ndata: {\"text\":\" world\"}\n\ndata: [DONE]\n\n"; + let events = parse_sse_chunk(sse_data); + + assert_eq!(events.len(), 3); + assert_eq!(events[0], "{\"text\":\"Hello\"}"); + assert_eq!(events[1], "{\"text\":\" world\"}"); + assert_eq!(events[2], "[DONE]"); + + // Test with mixed content + let mixed = b"event: message\ndata: {\"test\":true}\n\n: comment\ndata: [DONE]\n\n"; + let events = parse_sse_chunk(mixed); + + assert_eq!(events.len(), 2); + assert_eq!(events[0], "{\"test\":true}"); + assert_eq!(events[1], "[DONE]"); + } +} diff --git a/sgl-router/tests/test_chat_template.rs b/sgl-router/tests/test_chat_template.rs new file mode 100644 index 000000000..4a0e73bd0 --- /dev/null +++ b/sgl-router/tests/test_chat_template.rs @@ -0,0 +1,150 @@ +#[cfg(test)] +mod tests { + use sglang_router_rs::tokenizer::chat_template::{ChatMessage, ChatTemplateProcessor}; + + #[test] + fn test_chat_message_helpers() { + let system_msg = ChatMessage::system("You are a helpful assistant"); + assert_eq!(system_msg.role, "system"); + assert_eq!(system_msg.content, "You are a helpful assistant"); + + let user_msg = ChatMessage::user("Hello!"); + assert_eq!(user_msg.role, "user"); + assert_eq!(user_msg.content, "Hello!"); + + let assistant_msg = ChatMessage::assistant("Hi there!"); + assert_eq!(assistant_msg.role, "assistant"); + assert_eq!(assistant_msg.content, "Hi there!"); + } + + #[test] + fn test_llama_style_template() { + // Test a Llama-style chat template + let template = r#" +{%- if messages[0]['role'] == 'system' -%} + {%- set system_message = messages[0]['content'] -%} + {%- set messages = messages[1:] -%} +{%- else -%} + {%- set system_message = '' -%} +{%- endif -%} + +{{- bos_token }} +{%- if system_message %} +{{- '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }} +{%- endif %} + +{%- for message in messages %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }} +{%- endfor %} + +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} +"#; + + let processor = ChatTemplateProcessor::new( + template.to_string(), + Some("<|begin_of_text|>".to_string()), + Some("<|end_of_text|>".to_string()), + ); + + let messages = vec![ + ChatMessage::system("You are a helpful assistant"), + ChatMessage::user("What is 2+2?"), + ]; + + let result = processor.apply_chat_template(&messages, true).unwrap(); + + // Check that the result contains expected markers + assert!(result.contains("<|begin_of_text|>")); + assert!(result.contains("<|start_header_id|>system<|end_header_id|>")); + assert!(result.contains("You are a helpful assistant")); + assert!(result.contains("<|start_header_id|>user<|end_header_id|>")); + assert!(result.contains("What is 2+2?")); + assert!(result.contains("<|start_header_id|>assistant<|end_header_id|>")); + } + + #[test] + fn test_chatml_template() { + // Test a ChatML-style template + let template = r#" +{%- for message in messages %} + {{- '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} +"#; + + let processor = ChatTemplateProcessor::new(template.to_string(), None, None); + + let messages = vec![ + ChatMessage::user("Hello"), + ChatMessage::assistant("Hi there!"), + ChatMessage::user("How are you?"), + ]; + + let result = processor.apply_chat_template(&messages, true).unwrap(); + + // Check ChatML format + assert!(result.contains("<|im_start|>user\nHello<|im_end|>")); + assert!(result.contains("<|im_start|>assistant\nHi there!<|im_end|>")); + assert!(result.contains("<|im_start|>user\nHow are you?<|im_end|>")); + assert!(result.ends_with("<|im_start|>assistant\n")); + } + + #[test] + fn test_template_without_generation_prompt() { + let template = r#" +{%- for message in messages -%} +{{ message.role }}: {{ message.content }} +{% endfor -%} +{%- if add_generation_prompt -%} +assistant: +{%- endif -%} +"#; + + let processor = ChatTemplateProcessor::new(template.to_string(), None, None); + + let messages = vec![ChatMessage::user("Test")]; + + // Test without generation prompt + let result = processor.apply_chat_template(&messages, false).unwrap(); + assert_eq!(result.trim(), "user: Test"); + + // Test with generation prompt + let result_with_prompt = processor.apply_chat_template(&messages, true).unwrap(); + assert!(result_with_prompt.contains("assistant:")); + } + + #[test] + fn test_template_with_special_tokens() { + let template = r#"{{ bos_token }}{% for msg in messages %}{{ msg.content }}{{ eos_token }}{% endfor %}"#; + + let processor = ChatTemplateProcessor::new( + template.to_string(), + Some("".to_string()), + Some("".to_string()), + ); + + let messages = vec![ChatMessage::user("Hello")]; + + let result = processor.apply_chat_template(&messages, false).unwrap(); + assert_eq!(result, "Hello"); + } + + #[test] + fn test_empty_messages() { + let template = + r#"{% for msg in messages %}{{ msg.role }}: {{ msg.content }}\n{% endfor %}"#; + + let processor = ChatTemplateProcessor::new(template.to_string(), None, None); + + let messages = vec![]; + let result = processor.apply_chat_template(&messages, false).unwrap(); + assert_eq!(result, ""); + } + + // Integration test with actual tokenizer file loading would go here + // but requires a real tokenizer_config.json file +} diff --git a/sgl-router/tests/test_chat_template_loading.rs b/sgl-router/tests/test_chat_template_loading.rs new file mode 100644 index 000000000..ad1501233 --- /dev/null +++ b/sgl-router/tests/test_chat_template_loading.rs @@ -0,0 +1,183 @@ +#[cfg(test)] +mod tests { + use std::fs; + use tempfile::TempDir; + + #[test] + fn test_load_chat_template_from_file() { + use sglang_router_rs::tokenizer::chat_template::ChatMessage; + use sglang_router_rs::tokenizer::huggingface::HuggingFaceTokenizer; + + // Create temporary directory + let temp_dir = TempDir::new().unwrap(); + let template_path = temp_dir.path().join("template.jinja"); + + // Write a test template + let template_content = r#" +{%- for message in messages %} + {{- '<|' + message['role'] + '|>' + message['content'] }} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|assistant|>' }} +{%- endif %} +"#; + fs::write(&template_path, template_content).unwrap(); + + // Create a mock tokenizer config + let tokenizer_config = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": { + "type": "Whitespace" + }, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "vocab": { + "hello": 0, + "world": 1, + "": 2, + "": 3 + }, + "merges": [] + } + }"#; + + let tokenizer_path = temp_dir.path().join("tokenizer.json"); + fs::write(&tokenizer_path, tokenizer_config).unwrap(); + + // Load tokenizer with custom chat template + let tokenizer = HuggingFaceTokenizer::from_file_with_chat_template( + tokenizer_path.to_str().unwrap(), + Some(template_path.to_str().unwrap()), + ) + .unwrap(); + + // Test that the custom template is used + let messages = vec![ + ChatMessage::user("Hello"), + ChatMessage::assistant("Hi there"), + ]; + + let result = tokenizer.apply_chat_template(&messages, true).unwrap(); + + // Verify the custom template format + assert!(result.contains("<|user|>Hello")); + assert!(result.contains("<|assistant|>Hi there")); + assert!(result.ends_with("<|assistant|>")); + } + + #[test] + fn test_override_existing_template() { + use sglang_router_rs::tokenizer::chat_template::ChatMessage; + use sglang_router_rs::tokenizer::huggingface::HuggingFaceTokenizer; + + // Create temporary directory + let temp_dir = TempDir::new().unwrap(); + + // Create tokenizer config with a built-in template + let tokenizer_config_path = temp_dir.path().join("tokenizer_config.json"); + let config_with_template = r#"{ + "chat_template": "built-in: {% for msg in messages %}{{ msg.content }}{% endfor %}" + }"#; + fs::write(&tokenizer_config_path, config_with_template).unwrap(); + + // Create the actual tokenizer file + let tokenizer_json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": { + "type": "Whitespace" + }, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "vocab": { + "test": 0, + "": 1, + "": 2 + }, + "merges": [] + } + }"#; + let tokenizer_path = temp_dir.path().join("tokenizer.json"); + fs::write(&tokenizer_path, tokenizer_json).unwrap(); + + // Create custom template that should override + let custom_template_path = temp_dir.path().join("custom.jinja"); + let custom_template = + r#"CUSTOM: {% for msg in messages %}[{{ msg.role }}]: {{ msg.content }}{% endfor %}"#; + fs::write(&custom_template_path, custom_template).unwrap(); + + // Load with custom template - should override the built-in one + let tokenizer = HuggingFaceTokenizer::from_file_with_chat_template( + tokenizer_path.to_str().unwrap(), + Some(custom_template_path.to_str().unwrap()), + ) + .unwrap(); + + let messages = vec![ChatMessage::user("Test")]; + let result = tokenizer.apply_chat_template(&messages, false).unwrap(); + + // Should use CUSTOM template, not built-in + assert!(result.starts_with("CUSTOM:")); + assert!(result.contains("[user]: Test")); + assert!(!result.contains("built-in:")); + } + + #[test] + fn test_set_chat_template_after_creation() { + use sglang_router_rs::tokenizer::chat_template::ChatMessage; + use sglang_router_rs::tokenizer::huggingface::HuggingFaceTokenizer; + + // Create temporary directory and tokenizer file + let temp_dir = TempDir::new().unwrap(); + let tokenizer_json = r#"{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [], + "normalizer": null, + "pre_tokenizer": { + "type": "Whitespace" + }, + "post_processor": null, + "decoder": null, + "model": { + "type": "BPE", + "vocab": { + "test": 0, + "": 1, + "": 2 + }, + "merges": [] + } + }"#; + let tokenizer_path = temp_dir.path().join("tokenizer.json"); + fs::write(&tokenizer_path, tokenizer_json).unwrap(); + + // Load tokenizer without custom template + let mut tokenizer = + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()).unwrap(); + + // Set a template after creation (mimics Python's behavior) + let new_template = + "NEW: {% for msg in messages %}{{ msg.role }}: {{ msg.content }}; {% endfor %}"; + tokenizer.set_chat_template(new_template.to_string()); + + let messages = vec![ChatMessage::user("Hello"), ChatMessage::assistant("World")]; + let result = tokenizer.apply_chat_template(&messages, false).unwrap(); + + assert!(result.starts_with("NEW:")); + assert!(result.contains("user: Hello;")); + assert!(result.contains("assistant: World;")); + } +} diff --git a/sgl-router/tests/test_openai_routing.rs b/sgl-router/tests/test_openai_routing.rs new file mode 100644 index 000000000..ec38a6dd5 --- /dev/null +++ b/sgl-router/tests/test_openai_routing.rs @@ -0,0 +1,419 @@ +//! Comprehensive integration tests for OpenAI backend functionality + +use axum::{ + body::Body, + extract::Request, + http::{Method, StatusCode}, + routing::post, + Router, +}; +use serde_json::json; +use sglang_router_rs::{ + config::{RouterConfig, RoutingMode}, + protocols::spec::{ + ChatCompletionRequest, ChatMessage, CompletionRequest, GenerateRequest, UserMessageContent, + }, + routers::{openai_router::OpenAIRouter, RouterTrait}, +}; +use std::sync::Arc; +use tower::ServiceExt; + +mod common; +use common::mock_openai_server::MockOpenAIServer; + +/// Helper function to create a minimal chat completion request for testing +fn create_minimal_chat_request() -> ChatCompletionRequest { + let val = json!({ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hello"} + ], + "max_tokens": 100 + }); + serde_json::from_value(val).unwrap() +} + +/// Helper function to create a minimal completion request for testing +fn create_minimal_completion_request() -> CompletionRequest { + CompletionRequest { + model: "gpt-3.5-turbo".to_string(), + prompt: sglang_router_rs::protocols::spec::StringOrArray::String("Hello".to_string()), + suffix: None, + max_tokens: Some(100), + temperature: None, + top_p: None, + n: None, + stream: false, + stream_options: None, + logprobs: None, + echo: false, + stop: None, + presence_penalty: None, + frequency_penalty: None, + best_of: None, + logit_bias: None, + user: None, + seed: None, + top_k: None, + min_p: None, + min_tokens: None, + repetition_penalty: None, + regex: None, + ebnf: None, + json_schema: None, + stop_token_ids: None, + no_stop_trim: false, + ignore_eos: false, + skip_special_tokens: true, + lora_path: None, + session_params: None, + return_hidden_states: false, + other: serde_json::Map::new(), + } +} + +// ============= Basic Unit Tests ============= + +/// Test basic OpenAI router creation and configuration +#[tokio::test] +async fn test_openai_router_creation() { + let router = OpenAIRouter::new("https://api.openai.com".to_string(), None).await; + + assert!(router.is_ok(), "Router creation should succeed"); + + let router = router.unwrap(); + assert_eq!(router.router_type(), "openai"); + assert!(!router.is_pd_mode()); +} + +/// Test health endpoints +#[tokio::test] +async fn test_openai_router_health() { + let router = OpenAIRouter::new("https://api.openai.com".to_string(), None) + .await + .unwrap(); + + let req = Request::builder() + .method(Method::GET) + .uri("/health") + .body(Body::empty()) + .unwrap(); + + let response = router.health(req).await; + assert_eq!(response.status(), StatusCode::OK); +} + +/// Test server info endpoint +#[tokio::test] +async fn test_openai_router_server_info() { + let router = OpenAIRouter::new("https://api.openai.com".to_string(), None) + .await + .unwrap(); + + let req = Request::builder() + .method(Method::GET) + .uri("/info") + .body(Body::empty()) + .unwrap(); + + let response = router.get_server_info(req).await; + assert_eq!(response.status(), StatusCode::OK); + + let (_, body) = response.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + + assert!(body_str.contains("openai")); +} + +/// Test models endpoint +#[tokio::test] +async fn test_openai_router_models() { + // Use mock server for deterministic models response + let mock_server = MockOpenAIServer::new().await; + let router = OpenAIRouter::new(mock_server.base_url(), None) + .await + .unwrap(); + + let req = Request::builder() + .method(Method::GET) + .uri("/models") + .body(Body::empty()) + .unwrap(); + + let response = router.get_models(req).await; + assert_eq!(response.status(), StatusCode::OK); + + let (_, body) = response.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + let models: serde_json::Value = serde_json::from_str(&body_str).unwrap(); + + assert_eq!(models["object"], "list"); + assert!(models["data"].is_array()); +} + +/// Test router factory with OpenAI routing mode +#[tokio::test] +async fn test_router_factory_openai_mode() { + let routing_mode = RoutingMode::OpenAI { + worker_urls: vec!["https://api.openai.com".to_string()], + }; + + let router_config = + RouterConfig::new(routing_mode, sglang_router_rs::config::PolicyConfig::Random); + + let app_context = common::create_test_context(router_config); + + let router = sglang_router_rs::routers::RouterFactory::create_router(&app_context).await; + assert!( + router.is_ok(), + "Router factory should create OpenAI router successfully" + ); + + let router = router.unwrap(); + assert_eq!(router.router_type(), "openai"); +} + +/// Test that unsupported endpoints return proper error codes +#[tokio::test] +async fn test_unsupported_endpoints() { + let router = OpenAIRouter::new("https://api.openai.com".to_string(), None) + .await + .unwrap(); + + // Test generate endpoint (SGLang-specific, should not be supported) + let generate_request = GenerateRequest { + prompt: None, + text: Some("Hello world".to_string()), + input_ids: None, + parameters: None, + sampling_params: None, + stream: false, + return_logprob: false, + lora_path: None, + session_params: None, + return_hidden_states: false, + rid: None, + }; + + let response = router.route_generate(None, &generate_request).await; + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); + + // Test completion endpoint (should also not be supported) + let completion_request = create_minimal_completion_request(); + let response = router.route_completion(None, &completion_request).await; + assert_eq!(response.status(), StatusCode::NOT_IMPLEMENTED); +} + +// ============= Mock Server E2E Tests ============= + +/// Test chat completion with mock OpenAI server +#[tokio::test] +async fn test_openai_router_chat_completion_with_mock() { + // Start a mock OpenAI server + let mock_server = MockOpenAIServer::new().await; + let base_url = mock_server.base_url(); + + // Create router pointing to mock server + let router = OpenAIRouter::new(base_url, None).await.unwrap(); + + // Create a minimal chat completion request + let mut chat_request = create_minimal_chat_request(); + chat_request.messages = vec![ChatMessage::User { + role: "user".to_string(), + content: UserMessageContent::Text("Hello, how are you?".to_string()), + name: None, + }]; + chat_request.temperature = Some(0.7); + + // Route the request + let response = router.route_chat(None, &chat_request).await; + + // Should get a successful response from mock server + assert_eq!(response.status(), StatusCode::OK); + + let (_, body) = response.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + let chat_response: serde_json::Value = serde_json::from_str(&body_str).unwrap(); + + // Verify it's a valid chat completion response + assert_eq!(chat_response["object"], "chat.completion"); + assert_eq!(chat_response["model"], "gpt-3.5-turbo"); + assert!(!chat_response["choices"].as_array().unwrap().is_empty()); +} + +/// Test full E2E flow with Axum server +#[tokio::test] +async fn test_openai_e2e_with_server() { + // Start mock OpenAI server + let mock_server = MockOpenAIServer::new().await; + let base_url = mock_server.base_url(); + + // Create router + let router = OpenAIRouter::new(base_url, None).await.unwrap(); + + // Create Axum app with chat completions endpoint + let app = Router::new().route( + "/v1/chat/completions", + post({ + let router = Arc::new(router); + move |req: Request| { + let router = router.clone(); + async move { + let (parts, body) = req.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + + let chat_request: ChatCompletionRequest = + serde_json::from_str(&body_str).unwrap(); + + router.route_chat(Some(&parts.headers), &chat_request).await + } + } + }), + ); + + // Make a request to the server + let request = Request::builder() + .method(Method::POST) + .uri("/v1/chat/completions") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hello, world!" + } + ], + "max_tokens": 100 + }) + .to_string(), + )) + .unwrap(); + + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let response_json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + + // Verify the response structure + assert_eq!(response_json["object"], "chat.completion"); + assert_eq!(response_json["model"], "gpt-3.5-turbo"); + assert!(!response_json["choices"].as_array().unwrap().is_empty()); +} + +/// Test streaming chat completions pass-through with mock server +#[tokio::test] +async fn test_openai_router_chat_streaming_with_mock() { + let mock_server = MockOpenAIServer::new().await; + let base_url = mock_server.base_url(); + let router = OpenAIRouter::new(base_url, None).await.unwrap(); + + // Build a streaming chat request + let val = json!({ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hello"} + ], + "max_tokens": 10, + "stream": true + }); + let chat_request: ChatCompletionRequest = serde_json::from_value(val).unwrap(); + + let response = router.route_chat(None, &chat_request).await; + assert_eq!(response.status(), StatusCode::OK); + + // Should be SSE + let headers = response.headers(); + let ct = headers + .get("content-type") + .unwrap() + .to_str() + .unwrap() + .to_ascii_lowercase(); + assert!(ct.contains("text/event-stream")); + + // Read entire stream body and assert chunks + DONE + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let text = String::from_utf8(body.to_vec()).unwrap(); + assert!(text.contains("chat.completion.chunk")); + assert!(text.contains("[DONE]")); +} + +/// Test circuit breaker functionality +#[tokio::test] +async fn test_openai_router_circuit_breaker() { + // Create router with circuit breaker config + let cb_config = sglang_router_rs::config::CircuitBreakerConfig { + failure_threshold: 2, + success_threshold: 1, + timeout_duration_secs: 1, + window_duration_secs: 10, + }; + + let router = OpenAIRouter::new( + "http://invalid-url-that-will-fail".to_string(), + Some(cb_config), + ) + .await + .unwrap(); + + let chat_request = create_minimal_chat_request(); + + // First few requests should fail and record failures + for _ in 0..3 { + let response = router.route_chat(None, &chat_request).await; + // Should get either an error or circuit breaker response + assert!( + response.status() == StatusCode::INTERNAL_SERVER_ERROR + || response.status() == StatusCode::SERVICE_UNAVAILABLE + ); + } +} + +/// Test that Authorization header is forwarded in /v1/models +#[tokio::test] +async fn test_openai_router_models_auth_forwarding() { + // Start a mock server that requires Authorization + let expected_auth = "Bearer test-token".to_string(); + let mock_server = MockOpenAIServer::new_with_auth(Some(expected_auth.clone())).await; + let router = OpenAIRouter::new(mock_server.base_url(), None) + .await + .unwrap(); + + // 1) Without auth header -> expect 401 + let req = Request::builder() + .method(Method::GET) + .uri("/models") + .body(Body::empty()) + .unwrap(); + + let response = router.get_models(req).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + + // 2) With auth header -> expect 200 + let req = Request::builder() + .method(Method::GET) + .uri("/models") + .header("Authorization", expected_auth) + .body(Body::empty()) + .unwrap(); + + let response = router.get_models(req).await; + assert_eq!(response.status(), StatusCode::OK); + + let (_, body) = response.into_parts(); + let body_bytes = axum::body::to_bytes(body, usize::MAX).await.unwrap(); + let body_str = String::from_utf8(body_bytes.to_vec()).unwrap(); + let models: serde_json::Value = serde_json::from_str(&body_str).unwrap(); + assert_eq!(models["object"], "list"); +} diff --git a/sgl-router/tests/test_pd_routing.rs b/sgl-router/tests/test_pd_routing.rs new file mode 100644 index 000000000..7071106a4 --- /dev/null +++ b/sgl-router/tests/test_pd_routing.rs @@ -0,0 +1,918 @@ +#[cfg(test)] +mod test_pd_routing { + use serde_json::json; + use sglang_router_rs::config::{ + CircuitBreakerConfig, ConnectionMode, PolicyConfig, RetryConfig, RouterConfig, RoutingMode, + }; + use sglang_router_rs::core::{WorkerFactory, WorkerType}; + use sglang_router_rs::routers::http::pd_types::get_hostname; + use sglang_router_rs::routers::http::pd_types::PDSelectionPolicy; + use sglang_router_rs::routers::RouterFactory; + + // Test-only struct to help validate PD request parsing + #[derive(Debug)] + struct PDRequest { + pub is_stream: bool, + pub batch_size: Option, + } + + impl PDRequest { + // Extract PD-relevant info from JSON for testing + pub fn from_json(json: &serde_json::Value) -> Self { + let is_stream = json + .get("stream") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + // Detect batch size from text or input_ids + let batch_size = if let Some(text) = json.get("text") { + text.as_array().map(|arr| arr.len()) + } else if let Some(input_ids) = json.get("input_ids") { + input_ids.as_array().map(|arr| arr.len()) + } else { + None + }; + + PDRequest { + is_stream, + batch_size, + } + } + } + + // ======================================================================== + // Phase 1: Basic PD Components and Router Creation + // ======================================================================== + + #[test] + fn test_worker_types() { + use sglang_router_rs::core::{WorkerFactory, WorkerType}; + + // Test worker creation for prefill servers + let prefill_worker = + WorkerFactory::create_prefill("http://prefill:8080".to_string(), Some(9000)); + assert_eq!(prefill_worker.url(), "http://prefill:8080"); + match prefill_worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => { + assert_eq!(bootstrap_port, Some(9000)); + } + _ => panic!("Expected Prefill worker type"), + } + + // Test worker creation for decode servers + let decode_worker = WorkerFactory::create_decode("http://decode:8080".to_string()); + assert_eq!(decode_worker.url(), "http://decode:8080"); + match decode_worker.worker_type() { + WorkerType::Decode => (), + _ => panic!("Expected Decode worker type"), + } + + // Test regular worker creation + let regular_worker = WorkerFactory::create_regular("http://regular:8080".to_string()); + assert_eq!(regular_worker.url(), "http://regular:8080"); + match regular_worker.worker_type() { + WorkerType::Regular => (), + _ => panic!("Expected Regular worker type"), + } + } + + #[test] + fn test_pd_selection_policies() { + // Test all PD selection policy variants + // Note: These policies are only used when pd_disaggregation=true + let policies = vec![ + PDSelectionPolicy::Random, + PDSelectionPolicy::PowerOfTwo, + PDSelectionPolicy::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + ]; + + for policy in policies { + // Verify each policy can be created and matched + match &policy { + PDSelectionPolicy::Random => { + assert!(matches!(policy, PDSelectionPolicy::Random)); + } + PDSelectionPolicy::PowerOfTwo => { + assert!(matches!(policy, PDSelectionPolicy::PowerOfTwo)); + } + PDSelectionPolicy::CacheAware { + cache_threshold, .. + } => { + assert!(*cache_threshold >= 0.0 && *cache_threshold <= 1.0); + } + } + } + } + + #[tokio::test] + async fn test_pd_router_configuration() { + // Test PD router configuration with various policies + // In the new structure, RoutingMode and PolicyConfig are separate + let test_cases = vec![ + ( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://prefill1:8080".to_string(), Some(9000)), + ("http://prefill2:8080".to_string(), None), + ], + decode_urls: vec![ + "http://decode1:8080".to_string(), + "http://decode2:8080".to_string(), + ], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::Random, + ), + ( + RoutingMode::PrefillDecode { + prefill_urls: vec![("http://prefill:8080".to_string(), Some(9000))], + decode_urls: vec!["http://decode:8080".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::PowerOfTwo { + load_check_interval_secs: 5, + }, + ), + ( + RoutingMode::PrefillDecode { + prefill_urls: vec![ + ("http://p1:8080".to_string(), Some(9000)), + ("http://p2:8080".to_string(), Some(9001)), + ("http://p3:8080".to_string(), Some(9002)), + ], + decode_urls: vec!["http://d1:8080".to_string(), "http://d2:8080".to_string()], + prefill_policy: None, + decode_policy: None, + }, + PolicyConfig::CacheAware { + cache_threshold: 0.7, + balance_abs_threshold: 20, + balance_rel_threshold: 1.2, + eviction_interval_secs: 60, + max_tree_size: 1000000, + }, + ), + ]; + + for (mode, policy) in test_cases { + let config = RouterConfig { + mode, + policy, + host: "127.0.0.1".to_string(), + port: 3001, + max_payload_size: 1024 * 1024, + request_timeout_secs: 60, + worker_startup_timeout_secs: 10, + worker_startup_check_interval_secs: 1, + dp_aware: false, + api_key: None, + discovery: None, + metrics: None, + log_dir: None, + log_level: None, + request_id_headers: None, + max_concurrent_requests: 64, + queue_size: 0, + queue_timeout_secs: 60, + cors_allowed_origins: vec![], + retry: RetryConfig::default(), + circuit_breaker: CircuitBreakerConfig::default(), + disable_retries: false, + disable_circuit_breaker: false, + health_check: sglang_router_rs::config::HealthCheckConfig::default(), + enable_igw: false, + rate_limit_tokens_per_second: None, + connection_mode: ConnectionMode::Http, + model_path: None, + tokenizer_path: None, + }; + + // Router creation will fail due to health checks, but config should be valid + let app_context = + sglang_router_rs::server::AppContext::new(config, reqwest::Client::new(), 64, None) + .expect("Failed to create AppContext"); + let app_context = std::sync::Arc::new(app_context); + let result = RouterFactory::create_router(&app_context).await; + assert!(result.is_err()); + let error_msg = result.unwrap_err(); + // Error should be about health/timeout, not configuration + assert!( + error_msg.contains("healthy") || error_msg.contains("timeout"), + "Unexpected error: {}", + error_msg + ); + } + } + + // ======================================================================== + // Phase 2: Bootstrap Injection and Request Handling + // ======================================================================== + + #[test] + fn test_pd_request_from_json() { + // Test PDRequest parsing from single text request + let single_json = json!({ + "text": "Hello world", + "stream": false, + "temperature": 0.7, + "max_tokens": 100 + }); + + let pd_req = PDRequest::from_json(&single_json); + assert!(!pd_req.is_stream); + assert_eq!(pd_req.batch_size, None); + + // Test PDRequest parsing from batch text request + let batch_json = json!({ + "text": ["Hello", "World", "Test"], + "stream": true, + "temperature": 0.5 + }); + + let pd_req = PDRequest::from_json(&batch_json); + assert!(pd_req.is_stream); + assert_eq!(pd_req.batch_size, Some(3)); + + // Test PDRequest parsing from input_ids request + let ids_json = json!({ + "input_ids": [[1, 2, 3], [4, 5, 6]], + "stream": false + }); + + let pd_req = PDRequest::from_json(&ids_json); + assert!(!pd_req.is_stream); + assert_eq!(pd_req.batch_size, Some(2)); + + // Test PDRequest parsing from chat request + let chat_json = json!({ + "messages": [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello"} + ], + "stream": true + }); + + let pd_req = PDRequest::from_json(&chat_json); + assert!(pd_req.is_stream); + assert_eq!(pd_req.batch_size, None); + } + + #[test] + fn test_bootstrap_injection_simulation() { + // Since we can't test the actual inject_bootstrap_fields function here + // (it's private in the router module), we'll test the expected behavior + + // Simulate bootstrap injection for single request + let mut single_json = json!({ + "text": "Hello world", + "stream": false, + "temperature": 0.7 + }); + + // Create a prefill worker to simulate injection + let prefill_worker = + WorkerFactory::create_prefill("http://prefill1:8080".to_string(), Some(9000)); + + // Extract bootstrap port from worker type + let bootstrap_port = match prefill_worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + + // Simulate what inject_bootstrap_fields would do + single_json["bootstrap_host"] = json!(get_hostname(prefill_worker.url())); + single_json["bootstrap_port"] = json!(bootstrap_port); + single_json["bootstrap_room"] = json!(12345u64); // Random room ID + + // Verify bootstrap fields are added correctly + assert_eq!(single_json["bootstrap_host"], "prefill1"); + assert_eq!(single_json["bootstrap_port"], json!(Some(9000))); + assert!(single_json["bootstrap_room"].is_u64()); + assert_eq!(single_json["temperature"], 0.7); // Original field preserved + + // Simulate bootstrap injection for batch request + let mut batch_json = json!({ + "text": ["Hello", "World", "Test"], + "stream": true + }); + + let batch_size = 3; + let hostname = get_hostname(prefill_worker.url()); + batch_json["bootstrap_host"] = json!(vec![hostname; batch_size]); + batch_json["bootstrap_port"] = json!(vec![bootstrap_port; batch_size]); + batch_json["bootstrap_room"] = json!(vec![111u64, 222u64, 333u64]); + + // Verify batch bootstrap fields + assert!(batch_json["bootstrap_host"].is_array()); + assert_eq!( + batch_json["bootstrap_host"].as_array().unwrap().len(), + batch_size + ); + assert!(batch_json["bootstrap_port"].is_array()); + assert!(batch_json["bootstrap_room"].is_array()); + assert_eq!(batch_json["stream"], true); // Original field preserved + } + + #[test] + fn test_request_serialization() { + // Test that requests can be properly serialized and deserialized + let request = json!({ + "text": "Test prompt", + "stream": false, + "temperature": 0.7, + "max_tokens": 100, + "top_p": 0.9, + "frequency_penalty": 0.5, + "bootstrap_host": "prefill1", + "bootstrap_port": 9000, + "bootstrap_room": 12345u64 + }); + + // Convert to bytes (as would happen in the router) + let bytes = serde_json::to_vec(&request).unwrap(); + + // Parse back from bytes + let parsed: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + + // Verify all fields are preserved + assert_eq!(parsed["text"], "Test prompt"); + assert_eq!(parsed["stream"], false); + assert_eq!(parsed["temperature"], 0.7); + assert_eq!(parsed["max_tokens"], 100); + assert_eq!(parsed["bootstrap_host"], "prefill1"); + assert_eq!(parsed["bootstrap_port"], 9000); + assert_eq!(parsed["bootstrap_room"], 12345); + } + + #[test] + fn test_hostname_extraction() { + // Test various URL formats + let test_cases = vec![ + ("http://localhost:8080", "localhost"), + ("http://10.0.0.1:8080", "10.0.0.1"), + ("https://api.example.com:443", "api.example.com"), + ("http://prefill-server", "prefill-server"), + ("http://[::1]:8080", "["), // IPv6 edge case + ("prefill:8080", "prefill"), // No protocol + ]; + + for (url, expected_hostname) in test_cases { + assert_eq!(get_hostname(url), expected_hostname); + } + } + + #[test] + fn test_pd_request_edge_cases() { + // Test empty request + let empty_json = json!({}); + let pd_req = PDRequest::from_json(&empty_json); + assert!(!pd_req.is_stream); + assert_eq!(pd_req.batch_size, None); + + // Test request with only stream field + let stream_only = json!({ + "stream": true + }); + let pd_req = PDRequest::from_json(&stream_only); + assert!(pd_req.is_stream); + assert_eq!(pd_req.batch_size, None); + + // Test request with empty text array + let empty_batch = json!({ + "text": [] + }); + let pd_req = PDRequest::from_json(&empty_batch); + assert_eq!(pd_req.batch_size, Some(0)); + + // Test request with non-array text (should be None) + let non_array_text = json!({ + "text": "single string" + }); + let pd_req = PDRequest::from_json(&non_array_text); + assert_eq!(pd_req.batch_size, None); + } + + // ======================================================================== + // Phase 2: Background Load Monitoring Tests + // ======================================================================== + + #[tokio::test] + async fn test_background_load_monitoring() { + use std::collections::HashMap; + use tokio::sync::watch; + + // Create a watch channel for testing + let (tx, rx) = watch::channel(HashMap::new()); + + // Simulate load updates + let mut loads = HashMap::new(); + loads.insert("http://prefill1:8080".to_string(), 10); + loads.insert("http://prefill2:8080".to_string(), 20); + loads.insert("http://decode1:8080".to_string(), 5); + loads.insert("http://decode2:8080".to_string(), 15); + + // Send the loads + tx.send(loads.clone()).unwrap(); + + // Verify receiver gets the update + let received_loads = rx.borrow(); + assert_eq!(received_loads.get("http://prefill1:8080"), Some(&10)); + assert_eq!(received_loads.get("http://prefill2:8080"), Some(&20)); + assert_eq!(received_loads.get("http://decode1:8080"), Some(&5)); + assert_eq!(received_loads.get("http://decode2:8080"), Some(&15)); + } + + #[test] + fn test_load_monitoring_configuration() { + // Test that load monitoring is only enabled for PowerOfTwo policy + let policies = vec![ + (PDSelectionPolicy::Random, false), + (PDSelectionPolicy::PowerOfTwo, true), + ( + PDSelectionPolicy::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }, + false, + ), + ]; + + for (policy, should_monitor) in policies { + match policy { + PDSelectionPolicy::PowerOfTwo => assert!(should_monitor), + _ => assert!(!should_monitor), + } + } + } + + #[tokio::test] + async fn test_watch_channel_behavior() { + use std::collections::HashMap; + use tokio::sync::watch; + + // Test watch channel's broadcast behavior + let (tx, rx1) = watch::channel(HashMap::new()); + let rx2 = rx1.clone(); + + // Initial state - empty map + assert!(rx1.borrow().is_empty()); + assert!(rx2.borrow().is_empty()); + + // Update 1 + let mut loads = HashMap::new(); + loads.insert("worker1".to_string(), 10); + tx.send(loads.clone()).unwrap(); + + // Both receivers see the update + assert_eq!(rx1.borrow().get("worker1"), Some(&10)); + assert_eq!(rx2.borrow().get("worker1"), Some(&10)); + + // Update 2 - overwrites previous + loads.insert("worker1".to_string(), 20); + loads.insert("worker2".to_string(), 30); + tx.send(loads).unwrap(); + + // Both receivers see the latest state + assert_eq!(rx1.borrow().get("worker1"), Some(&20)); + assert_eq!(rx2.borrow().get("worker2"), Some(&30)); + } + + // ======================================================================== + // Tests based on bench_one_batch_server.py patterns + // ======================================================================== + + #[test] + fn test_generate_request_formats() { + // Based on bench_one_batch_server.py request patterns + + // Test 1: Batch request with input_ids (most common in benchmarks) + let batch_request = json!({ + "input_ids": [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 16, + "ignore_eos": true, + }, + "return_logprob": false, + "stream": true + }); + + let pd_req = PDRequest::from_json(&batch_request); + assert!(pd_req.is_stream); + assert_eq!(pd_req.batch_size, Some(3)); + + // Test 2: Request with return_logprob (critical for PD) + let logprob_request = json!({ + "input_ids": [[1, 2, 3]], + "sampling_params": { + "temperature": 0.7, + "max_new_tokens": 8, + }, + "return_logprob": true, + "stream": false + }); + + assert_eq!(logprob_request["return_logprob"], true); + assert_eq!(logprob_request["stream"], false); + + // Test 3: Large batch sizes from benchmark + let batch_sizes = vec![1, 16, 64]; // From bench_one_batch_server.py + for bs in batch_sizes { + let request = json!({ + "input_ids": vec![vec![1, 2, 3]; bs], + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 16, + }, + "stream": true + }); + + let pd_req = PDRequest::from_json(&request); + assert_eq!(pd_req.batch_size, Some(bs)); + } + } + + #[test] + fn test_sampling_params_handling() { + // Test various sampling parameters from bench_one_batch_server.py + let sampling_params_variations = vec![ + json!({ + "temperature": 0.0, + "max_new_tokens": 8, + "ignore_eos": true + }), + json!({ + "temperature": 0.7, + "max_new_tokens": 16, + "ignore_eos": false, + "top_p": 0.9, + "frequency_penalty": 0.5 + }), + json!({ + "temperature": 1.0, + "max_new_tokens": 64, + "json_schema": "$$ANY$$" // Structured output + }), + ]; + + for params in sampling_params_variations { + let request = json!({ + "input_ids": [[1, 2, 3]], + "sampling_params": params.clone(), + "stream": false + }); + + // Verify params are preserved + assert_eq!(request["sampling_params"], params); + } + } + + #[test] + fn test_streaming_response_parsing() { + // Test SSE format parsing from streaming responses + let sse_chunks = ["data: {\"text\":\"Hello\",\"meta_info\":{\"completion_tokens\":1,\"finish_reason\":null}}", + "data: {\"text\":\" world\",\"meta_info\":{\"completion_tokens\":2,\"finish_reason\":null}}", + "data: {\"text\":\"!\",\"meta_info\":{\"completion_tokens\":3,\"finish_reason\":{\"type\":\"length\"}}}", + "data: [DONE]"]; + + for chunk in &sse_chunks[..3] { + assert!(chunk.starts_with("data: ")); + let json_str = &chunk[6..]; // Skip "data: " + let parsed: serde_json::Value = serde_json::from_str(json_str).unwrap(); + assert!(parsed["meta_info"]["completion_tokens"].is_u64()); + } + + // Test [DONE] detection + assert_eq!(sse_chunks[3], "data: [DONE]"); + } + + #[test] + fn test_ttft_calculation() { + // Test Time To First Token calculation pattern + let first_token_response = json!({ + "text": "Hello", + "meta_info": { + "completion_tokens": 1, + "finish_reason": null + } + }); + + // TTFT is calculated when completion_tokens == 1 + assert_eq!(first_token_response["meta_info"]["completion_tokens"], 1); + assert!(first_token_response["meta_info"]["finish_reason"].is_null()); + } + + #[test] + fn test_throughput_metrics() { + // Test throughput calculation patterns from bench_one_batch_server.py + let batch_size = 16; + let input_len = 1024; + let output_len = 16; + let ttft = 0.5; // seconds + let total_latency = 2.0; // seconds + + // Input throughput = batch_size * input_len / ttft + let input_throughput = (batch_size as f64) * (input_len as f64) / ttft; + assert!((input_throughput - 32768.0).abs() < 0.01); + + // Output throughput = batch_size * output_len / (latency - ttft) + let output_throughput = (batch_size as f64) * (output_len as f64) / (total_latency - ttft); + assert!((output_throughput - 170.67).abs() < 0.01); + } + + #[test] + fn test_error_response_handling() { + // Test error response format from bench_one_batch_server.py + let error_response = json!({ + "error": "Request has failed. Invalid input format." + }); + + assert!(error_response.get("error").is_some()); + assert!(error_response["error"].as_str().unwrap().contains("failed")); + } + + #[test] + fn test_structured_output_request() { + // Test structured output format (json_schema) + let structured_request = json!({ + "text": "What is the capital of France? Answer in JSON.", + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 64, + "json_schema": "$$ANY$$" + }, + "stream": false + }); + + assert_eq!( + structured_request["sampling_params"]["json_schema"], + "$$ANY$$" + ); + } + + #[test] + fn test_bootstrap_injection_with_benchmark_requests() { + use sglang_router_rs::core::{WorkerFactory, WorkerType}; + + // Test bootstrap injection with actual benchmark request patterns + let mut benchmark_request = json!({ + "input_ids": vec![vec![1, 2, 3, 4]; 16], // Batch size 16 + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 8, + "ignore_eos": true + }, + "return_logprob": true, + "stream": true + }); + + // Create a prefill worker to simulate injection + let prefill_worker = + WorkerFactory::create_prefill("http://prefill:8080".to_string(), Some(9000)); + + // Extract bootstrap port from worker type + let bootstrap_port = match prefill_worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + let batch_size = 16; + let hostname = get_hostname(prefill_worker.url()); + + benchmark_request["bootstrap_host"] = json!(vec![hostname; batch_size]); + benchmark_request["bootstrap_port"] = json!(vec![bootstrap_port; batch_size]); + benchmark_request["bootstrap_room"] = + json!((0..batch_size).map(|_| 12345u64).collect::>()); + + // Verify bootstrap fields match batch size + assert_eq!( + benchmark_request["bootstrap_host"] + .as_array() + .unwrap() + .len(), + batch_size + ); + assert_eq!( + benchmark_request["bootstrap_port"] + .as_array() + .unwrap() + .len(), + batch_size + ); + assert_eq!( + benchmark_request["bootstrap_room"] + .as_array() + .unwrap() + .len(), + batch_size + ); + + // Verify original fields are preserved + assert_eq!(benchmark_request["return_logprob"], true); + assert_eq!(benchmark_request["stream"], true); + } + + #[test] + fn test_server_info_response_format() { + // Test server info format expected by bench_one_batch_server.py + let server_info = json!({ + "internal_states": [{ + "avg_spec_accept_length": 3.5, + "last_gen_throughput": 2048.5, + "load": 16 + }], + "prefill": [ + {"url": "http://prefill1:8080", "load": 10}, + {"url": "http://prefill2:8080", "load": 20} + ], + "decode": [ + {"url": "http://decode1:8080", "load": 5}, + {"url": "http://decode2:8080", "load": 15} + ] + }); + + // Verify structure matches what benchmark expects + assert!(server_info["internal_states"][0]["avg_spec_accept_length"].is_f64()); + assert!(server_info["internal_states"][0]["last_gen_throughput"].is_f64()); + assert!(server_info["prefill"].is_array()); + assert!(server_info["decode"].is_array()); + } + + // ======================================================================== + // Comprehensive Endpoint Coverage Test + // ======================================================================== + + #[test] + fn test_pd_endpoints_coverage() { + // Document all endpoints from Python mini_lb.py and verify implementation status + let implemented_endpoints = vec![ + ("/health", "GET", true), + ("/health_generate", "GET", true), // Note: Python uses POST, we use GET + ("/get_server_info", "GET", true), + ("/v1/models", "GET", true), + ("/get_model_info", "GET", true), + ("/generate", "POST", true), + ("/v1/chat/completions", "POST", true), + ("/v1/completions", "POST", true), + ("/flush_cache", "POST", true), + ("/get_loads", "GET", true), + ("/register", "POST", false), // NOT IMPLEMENTED - needs dynamic worker management + ]; + + let implemented_count = implemented_endpoints + .iter() + .filter(|(_, _, impl_status)| *impl_status) + .count(); + let total_count = implemented_endpoints.len(); + + // We've implemented 10 out of 11 endpoints (register is not needed for Phase 1/2) + assert_eq!(implemented_count, 10); + assert_eq!(total_count, 11); + + // Document the missing endpoint + let missing: Vec<_> = implemented_endpoints + .iter() + .filter(|(_, _, impl_status)| !impl_status) + .map(|(endpoint, method, _)| format!("{} {}", method, endpoint)) + .collect(); + + assert_eq!(missing, vec!["POST /register"]); + } + + #[test] + fn test_large_batch_bootstrap_injection() { + // Test bootstrap injection performance with very large batches + // This simulates the bench_one_batch_server.py scenario + let large_batch_sizes = vec![1024, 4096, 8192]; + + for batch_size in large_batch_sizes { + let start = std::time::Instant::now(); + + // Simulate a large batch request + let mut large_batch_request = json!({ + "input_ids": vec![vec![1, 2, 3, 4]; batch_size], + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 16, + }, + "stream": true + }); + + // Create a prefill worker to simulate injection + let prefill_worker = + WorkerFactory::create_prefill("http://prefill:8080".to_string(), Some(9000)); + + // Extract bootstrap port from worker type + let bootstrap_port = match prefill_worker.worker_type() { + WorkerType::Prefill { bootstrap_port } => bootstrap_port, + _ => None, + }; + let hostname = get_hostname(prefill_worker.url()); + + large_batch_request["bootstrap_host"] = json!(vec![hostname; batch_size]); + large_batch_request["bootstrap_port"] = json!(vec![bootstrap_port; batch_size]); + large_batch_request["bootstrap_room"] = json!((0..batch_size) + .map(|_| rand::random::()) + .collect::>()); + + let elapsed = start.elapsed(); + + // Verify bootstrap fields are correctly sized + assert_eq!( + large_batch_request["bootstrap_host"] + .as_array() + .unwrap() + .len(), + batch_size + ); + assert_eq!( + large_batch_request["bootstrap_port"] + .as_array() + .unwrap() + .len(), + batch_size + ); + assert_eq!( + large_batch_request["bootstrap_room"] + .as_array() + .unwrap() + .len(), + batch_size + ); + + // Bootstrap injection should be reasonably fast even for large batches + println!( + "Bootstrap injection for batch_size {} took {:?}", + batch_size, elapsed + ); + assert!( + elapsed.as_millis() < 1000, + "Bootstrap injection took too long for batch size {}", + batch_size + ); + } + } + + #[test] + fn test_payload_size_calculation() { + // Test payload size estimation for bench_one_batch_server.py scenarios + let test_cases = vec![ + (1, 1024, 16), // Small batch + (16, 1024, 16), // Medium batch + (64, 1024, 16), // Large batch + (8192, 4096, 5), // Benchmark scenario + ]; + + for (batch_size, input_len, _output_len) in test_cases { + // Estimate payload size (rough calculation) + // Each token is ~4 bytes (i32), plus JSON overhead + let tokens_size = batch_size * input_len * 4; // 4 bytes per token + let json_overhead = batch_size * 100; // ~100 bytes overhead per request + let total_size = tokens_size + json_overhead; + + println!( + "Batch size: {}, Input len: {}, Estimated payload: {} MB", + batch_size, + input_len, + total_size / (1024 * 1024) + ); + + // For the benchmark case (8192, 4096), this should be ~134 MB + if batch_size == 8192 && input_len == 4096 { + assert!( + total_size > 100 * 1024 * 1024, + "Benchmark payload should be > 100MB" + ); + assert!( + total_size < 200 * 1024 * 1024, + "Benchmark payload should be < 200MB" + ); + } + } + } + + #[test] + fn test_policy_type_to_pd_selection_policy_mapping() { + // Test that PDSelectionPolicy doesn't include RoundRobin + let pd_policy_count = 3; // Random, PowerOfTwo, CacheAware + assert_eq!( + pd_policy_count, 3, + "PDSelectionPolicy should have exactly 3 variants" + ); + + // Verify that each PDSelectionPolicy variant can be created + let _random = PDSelectionPolicy::Random; + let _po2 = PDSelectionPolicy::PowerOfTwo; + let _cache_aware = PDSelectionPolicy::CacheAware { + cache_threshold: 0.5, + balance_abs_threshold: 32, + balance_rel_threshold: 1.1, + }; + } +} diff --git a/sgl-router/tests/tokenizer_integration.rs b/sgl-router/tests/tokenizer_integration.rs new file mode 100644 index 000000000..9f0597297 --- /dev/null +++ b/sgl-router/tests/tokenizer_integration.rs @@ -0,0 +1,320 @@ +//! Integration tests for tokenizers using real tokenizer data +//! +//! These tests download the TinyLlama tokenizer from HuggingFace to verify our tokenizer +//! implementation works correctly with real-world tokenizer files. + +mod common; +use common::{ensure_tokenizer_cached, EXPECTED_HASHES, TEST_PROMPTS}; + +use sglang_router_rs::tokenizer::{ + factory, huggingface::HuggingFaceTokenizer, sequence::Sequence, stop::*, stream::DecodeStream, + traits::*, +}; +use std::sync::Arc; + +const LONG_TEST_PROMPTS: [(&str, &str); 6] = [ + ("Tell me about the following text.", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat."), + ("Tell me about the following text.", "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."), + ("Tell me about the following text.", "Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt."), + ("Tell me about the following text.", "Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem."), + // Tennis-themed prompt for variety + ("Tell me about the following text.", "In the ancient realm of Tennisia, the very magic of the land is drawn from the sport itself. Forehands light the skies, backhands carve the earth, and serves rumble like thunder across kingdoms. At the center of this balance lie four sacred Grand Slam relics: the Sapphire Trophy of Melbourne, the Emerald Chalice of Paris, the Ruby Crown of London, and the Diamond Orb of New York. Together, they keep the game's spirit alive. + But the relics are scattered, guarded by champions of legendary skill. The first is the Fire King of Clay, ruler of the crimson courts, whose topspin arcs blaze high and heavy, scorching all who dare stand across from him. The second is the Tempest Trickster, master of the baseline fortress, whose footwork and precision can turn back any storm, and whose returns arrive as if pulled by invisible strings. The third is the Shadow-Dancer of the Highlands, a tactician who thrives in the long rallies of twilight, changing pace and spin until opponents lose their rhythm. The fourth and final guardian is a towering Diamond Titan, a net-charging colossus whose volleys shatter the air itself. + Into this arena of gods steps the Silver-Wristed Knight — a player of impossible grace, whose game is an art form. His quest: to claim each relic not for glory, but to restore harmony to the rankings of the realm. + He travels across the Kingdom of Clay, where the points stretch like marathons and the air tastes of iron; through the Grasslands of London, where the ball skids low and the margins are razor-thin; over the Hard Courts of the East, where rallies turn into duels of endurance; and finally to the Cathedral of Lights in New York, where night matches burn with fevered energy. + Each battle is played under enchanted floodlights, the lines patrolled by spectral line judges whose calls are final. The crowd's roar swells with every break point, and the Silver-Wristed Knight's racket glows brightest when the match teeters at deuce. There are moments when doubt grips him — when his serve falters or his touch deserts him — but each challenge teaches a new stroke, culminating in the legendary Forehand of Dawn. + When the last relic is claimed, he stands not as a conqueror but as a custodian of the game, knowing that rivalries forge the very magic he protects. The balance is restored — until the next season begins."), + // Emoji stress test + ("Tell me about the following text.", "😀😃😄😁😆🥹😅😂🤣🥲☺️😊😇🙂🙃😉🤩😎 🤪🥳🤓🙄🤪😵👻") +]; + +fn compute_hashes_for_tokenizer(tokenizer: &E, prompts: &[&str]) -> Vec { + prompts + .iter() + .map(|&prompt| { + tokenizer + .encode(prompt) + .expect("Failed to encode prompt") + .get_hash() + }) + .collect() +} + +#[test] +fn test_huggingface_tokenizer_hashes() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load HuggingFace tokenizer"); + + let prompt_hashes = compute_hashes_for_tokenizer(&tokenizer, &TEST_PROMPTS); + + println!( + "HF Tokenizer: {:?}\nComputed Hashes: {:?}\nExpected Hashes: {:?}", + tokenizer_path, prompt_hashes, EXPECTED_HASHES + ); + + assert_eq!(prompt_hashes, EXPECTED_HASHES); +} + +#[test] +fn test_tokenizer_encode_decode_lifecycle() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load HuggingFace tokenizer"); + + for prompt in TEST_PROMPTS.iter() { + let encoding = tokenizer.encode(prompt).expect("Failed to encode prompt"); + + let decoded = tokenizer + .decode(encoding.token_ids(), false) + .expect("Failed to decode token_ids"); + + assert_eq!(decoded, *prompt, "Encode-decode mismatch for: {}", prompt); + } +} + +#[test] +fn test_sequence_operations() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + for prompt in TEST_PROMPTS.iter() { + let encoding = tokenizer.encode(prompt).expect("Failed to encode prompt"); + + // Test Sequence with append_text + let mut sequence = Sequence::new(tokenizer.clone()); + sequence.append_text(prompt).expect("Failed to append text"); + + assert_eq!( + sequence.len(), + encoding.token_ids().len(), + "Sequence length mismatch" + ); + assert_eq!(sequence.text().unwrap(), *prompt, "Sequence text mismatch"); + + // Test incremental decoding with append_token + let mut decoder = Sequence::new(tokenizer.clone()); + let mut output = String::new(); + + for token_id in encoding.token_ids() { + let text = decoder + .append_token(*token_id) + .expect("Failed to append token"); + output.push_str(&text); + } + + assert_eq!(decoder.len(), sequence.len(), "Decoder length mismatch"); + assert_eq!( + decoder.token_ids(), + sequence.token_ids(), + "Token IDs mismatch" + ); + assert_eq!(output, *prompt, "Incremental decode mismatch"); + } +} + +#[test] +fn test_decode_stream() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + for prompt in TEST_PROMPTS.iter() { + let encoding = tokenizer.encode(prompt).expect("Failed to encode prompt"); + + let mut decoder = DecodeStream::new(tokenizer.clone(), &[], false); + let mut output = String::new(); + + for token_id in encoding.token_ids() { + if let Some(text) = decoder.step(*token_id).expect("Failed to decode token") { + output.push_str(&text); + } + } + + assert_eq!(output, *prompt, "DecodeStream output mismatch"); + } +} + +#[test] +fn test_long_sequence_incremental_decode_with_prefill() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + for (input_text, output_text) in LONG_TEST_PROMPTS.iter() { + let input_encoding = tokenizer + .encode(input_text) + .expect("Failed to encode input"); + + let output_encoding = tokenizer + .encode(output_text) + .expect("Failed to encode output"); + + let mut decoder = DecodeStream::new(tokenizer.clone(), input_encoding.token_ids(), false); + + let mut output = String::new(); + for token_id in output_encoding.token_ids() { + if let Some(text) = decoder.step(*token_id).expect("Failed to decode token") { + output.push_str(&text); + } + } + + assert_eq!(output.trim(), *output_text, "Long sequence decode mismatch"); + } +} + +#[test] +fn test_stop_sequence_decoder() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + // Test with various stop sequences + let test_cases = vec![ + ( + "Hello world! Stop here. Continue after.", + "Stop", + "Hello world! ", + ), + ("Testing stop sequences.", ".", "Testing stop sequences"), + ("No stop sequence here", "xyz", "No stop sequence here"), + ]; + + for (input, stop_seq, expected) in test_cases { + let config = StopSequenceConfig::default().with_stop_sequence(stop_seq); + + let mut decoder = StopSequenceDecoder::new(tokenizer.clone(), config, false); + + let encoding = tokenizer.encode(input).expect("Failed to encode"); + let mut output = String::new(); + let mut stopped = false; + + for token_id in encoding.token_ids() { + match decoder.process_token(*token_id).unwrap() { + SequenceDecoderOutput::Text(text) => output.push_str(&text), + SequenceDecoderOutput::StoppedWithText(text) => { + output.push_str(&text); + stopped = true; + break; + } + SequenceDecoderOutput::Stopped => { + stopped = true; + break; + } + SequenceDecoderOutput::Held => {} + } + } + + if !stopped { + // Flush any remaining text + if let SequenceDecoderOutput::Text(text) = decoder.flush() { + output.push_str(&text); + } + } + + println!( + "Input: '{}', Stop: '{}', Output: '{}', Expected: '{}'", + input, stop_seq, output, expected + ); + + // The test should check if output starts with expected + // since stop sequences might not be perfectly aligned with token boundaries + assert!( + output.starts_with(expected) || output == input, + "Stop sequence test failed" + ); + } +} + +#[test] +fn test_factory_creation() { + // Test factory creation method + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = factory::create_tokenizer(tokenizer_path.to_str().unwrap()) + .expect("Failed to create tokenizer via factory"); + + let encoding = tokenizer.encode(TEST_PROMPTS[0]).expect("Failed to encode"); + + let decoded = tokenizer + .decode(encoding.token_ids(), false) + .expect("Failed to decode"); + + assert_eq!(decoded, TEST_PROMPTS[0]); +} + +#[test] +fn test_batch_encoding() { + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"); + + let encodings = tokenizer + .encode_batch(&TEST_PROMPTS) + .expect("Failed to batch encode"); + + assert_eq!(encodings.len(), TEST_PROMPTS.len()); + + for (i, encoding) in encodings.iter().enumerate() { + let decoded = tokenizer + .decode(encoding.token_ids(), false) + .expect("Failed to decode"); + assert_eq!(decoded, TEST_PROMPTS[i]); + } +} + +#[test] +fn test_special_tokens() { + use sglang_router_rs::tokenizer::traits::Tokenizer as TokenizerTrait; + + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"); + + let special_tokens = tokenizer.get_special_tokens(); + + // TinyLlama should have at least BOS and EOS tokens + assert!(special_tokens.bos_token.is_some()); + assert!(special_tokens.eos_token.is_some()); + + println!("Special tokens: {:?}", special_tokens); +} + +#[test] +fn test_thread_safety() { + use std::thread; + + let tokenizer_path = ensure_tokenizer_cached(); + let tokenizer = Arc::new( + HuggingFaceTokenizer::from_file(tokenizer_path.to_str().unwrap()) + .expect("Failed to load tokenizer"), + ); + + let handles: Vec<_> = TEST_PROMPTS + .iter() + .map(|&prompt| { + let tokenizer_clone = tokenizer.clone(); + thread::spawn(move || { + let encoding = tokenizer_clone + .encode(prompt) + .expect("Failed to encode in thread"); + let decoded = tokenizer_clone + .decode(encoding.token_ids(), false) + .expect("Failed to decode in thread"); + assert_eq!(decoded, prompt); + }) + }) + .collect(); + + for handle in handles { + handle.join().expect("Thread panicked"); + } +} diff --git a/sgl-router/tests/tool_parser_deepseek.rs b/sgl-router/tests/tool_parser_deepseek.rs new file mode 100644 index 000000000..45168c13e --- /dev/null +++ b/sgl-router/tests/tool_parser_deepseek.rs @@ -0,0 +1,183 @@ +//! DeepSeek V3 Parser Integration Tests + +use sglang_router_rs::tool_parser::{DeepSeekParser, ParseState, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_deepseek_complete_parsing() { + let parser = DeepSeekParser::new(); + + // Test single tool call + let input = r#"Let me help you with that. +<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Tokyo", "units": "celsius"} +```<｜tool▁call▁end｜><｜tool▁calls▁end｜> +The weather in Tokyo is..."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + // Verify arguments + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["location"], "Tokyo"); + assert_eq!(args["units"], "celsius"); +} + +#[tokio::test] +async fn test_deepseek_multiple_tools() { + let parser = DeepSeekParser::new(); + + let input = r#"<｜tool▁calls▁begin｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>search +```json +{"query": "rust programming"} +```<｜tool▁call▁end｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>translate +```json +{"text": "Hello World", "to": "ja"} +```<｜tool▁call▁end｜> +<｜tool▁calls▁end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_deepseek_streaming() { + let parser = DeepSeekParser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>", + "function<｜tool▁sep｜>get_weather\n", + "```json\n", + r#"{"location": "#, + r#""Beijing", "#, + r#""units": "metric"}"#, + "\n```<｜tool▁call▁end｜><｜tool▁calls▁end｜>", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "get_weather"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); +} + +#[tokio::test] +async fn test_deepseek_nested_json() { + let parser = DeepSeekParser::new(); + + let input = r#"<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>process +```json +{ + "data": { + "nested": { + "deep": [1, 2, 3] + } + } +} +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["data"]["nested"]["deep"].is_array()); +} + +#[test] +fn test_deepseek_format_detection() { + let parser = DeepSeekParser::new(); + + // Should detect DeepSeek format + assert!(parser.detect_format("<｜tool▁calls▁begin｜>")); + assert!(parser.detect_format("text with <｜tool▁calls▁begin｜> marker")); + + // Should not detect other formats + assert!(!parser.detect_format("[TOOL_CALLS]")); + assert!(!parser.detect_format("")); + assert!(!parser.detect_format("plain text")); +} + +#[tokio::test] +async fn test_deepseek_malformed_json_handling() { + let parser = DeepSeekParser::new(); + + // Malformed JSON should be skipped + let input = r#"<｜tool▁calls▁begin｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>broken +```json +{invalid json} +```<｜tool▁call▁end｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>valid +```json +{"key": "value"} +```<｜tool▁call▁end｜> +<｜tool▁calls▁end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + // Only the valid tool call should be parsed + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "valid"); +} + +#[tokio::test] +async fn test_normal_text_extraction() { + let parser = DeepSeekParser::new(); + + // Python extracts text before tool calls as normal_text + let input = r#"Let me help you with that. +<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Tokyo"} +```<｜tool▁call▁end｜><｜tool▁calls▁end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + // TODO: Verify normal text extraction when parser returns it + // In Python: normal_text = "Let me help you with that." +} + +#[tokio::test] +async fn test_multiple_tool_calls() { + let parser = DeepSeekParser::new(); + + let input = r#"<｜tool▁calls▁begin｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Tokyo"} +```<｜tool▁call▁end｜> +<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather +```json +{"location": "Paris"} +```<｜tool▁call▁end｜> +<｜tool▁calls▁end｜><｜end▁of▁sentence｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "get_weather"); +} diff --git a/sgl-router/tests/tool_parser_edge_cases.rs b/sgl-router/tests/tool_parser_edge_cases.rs new file mode 100644 index 000000000..5738f650b --- /dev/null +++ b/sgl-router/tests/tool_parser_edge_cases.rs @@ -0,0 +1,330 @@ +//! Edge Cases and Error Handling Tests +//! +//! Tests for malformed input, edge cases, and error recovery + +use sglang_router_rs::tool_parser::{ + JsonParser, MistralParser, ParseState, ParserRegistry, PythonicParser, QwenParser, + StreamResult, ToolParser, +}; + +#[tokio::test] +async fn test_empty_input() { + let registry = ParserRegistry::new(); + let parsers = vec!["json", "mistral", "qwen", "pythonic", "llama"]; + + for parser_name in parsers { + let parser = registry + .get_parser(&format!("test-{}", parser_name)) + .unwrap(); + let result = parser.parse_complete("").await.unwrap(); + assert_eq!( + result.len(), + 0, + "Parser {} should return empty for empty input", + parser_name + ); + } +} + +#[tokio::test] +async fn test_plain_text_no_tools() { + let plain_text = "This is just a regular response with no tool calls whatsoever."; + + let json_parser = JsonParser::new(); + assert_eq!( + json_parser.parse_complete(plain_text).await.unwrap().len(), + 0 + ); + + let mistral_parser = MistralParser::new(); + assert_eq!( + mistral_parser + .parse_complete(plain_text) + .await + .unwrap() + .len(), + 0 + ); + + let qwen_parser = QwenParser::new(); + assert_eq!( + qwen_parser.parse_complete(plain_text).await.unwrap().len(), + 0 + ); + + let pythonic_parser = PythonicParser::new(); + assert_eq!( + pythonic_parser + .parse_complete(plain_text) + .await + .unwrap() + .len(), + 0 + ); +} + +#[tokio::test] +async fn test_incomplete_json() { + let json_parser = JsonParser::new(); + + let incomplete_cases = vec![ + r#"{"name": "test""#, // Missing closing brace + r#"{"name": "test", "arguments":"#, // Incomplete arguments + r#"{"name": "test", "arguments": {"#, // Incomplete nested object + ]; + + for input in incomplete_cases { + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!( + result.len(), + 0, + "Should not parse incomplete JSON: {}", + input + ); + } + + // This case might actually parse because [{"name": "test"}] is complete + // The trailing comma suggests more items but the first item is valid + let _result = json_parser + .parse_complete(r#"[{"name": "test"},"#) + .await + .unwrap(); + // This could parse the first element or return empty - implementation dependent +} + +#[tokio::test] +async fn test_malformed_mistral() { + let parser = MistralParser::new(); + + let malformed_cases = vec![ + "[TOOL_CALLS]", // Missing array + "[TOOL_CALLS] {", // Not an array + "[TOOL_CALLS] [", // Incomplete array + "[TOOL_CALLS] [{]", // Invalid JSON in array + "[TOOL_CALLS] [{\"name\": }]", // Invalid value + ]; + + for input in malformed_cases { + // Parser might return error or empty vec for malformed input + if let Ok(result) = parser.parse_complete(input).await { + assert_eq!( + result.len(), + 0, + "Should not parse malformed Mistral: {}", + input + ); + } + // Error is also acceptable for malformed input + } +} + +#[tokio::test] +async fn test_missing_required_fields() { + let json_parser = JsonParser::new(); + + // Missing name field + let input = r#"{"arguments": {"x": 1}}"#; + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should not parse without name field"); + + // Name is not a string + let input = r#"{"name": 123, "arguments": {}}"#; + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should not parse with non-string name"); +} + +#[tokio::test] +async fn test_very_long_strings() { + let json_parser = JsonParser::new(); + + let long_string = "x".repeat(10000); + let input = format!( + r#"{{"name": "test", "arguments": {{"data": "{}"}}}}"#, + long_string + ); + + let result = json_parser.parse_complete(&input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["data"].as_str().unwrap().len(), 10000); +} + +#[tokio::test] +async fn test_unicode_edge_cases() { + let json_parser = JsonParser::new(); + + // Various Unicode characters including emojis, CJK, RTL text + let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍 مرحبا עולם"}}"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Hello 世界 🌍 مرحبا עולם"); +} + +#[tokio::test] +async fn test_nested_brackets_in_strings() { + // Test that parsers correctly handle brackets within string literals + + let mistral_parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"text": "Array: [1, 2, 3]"}}]"#; + let result = mistral_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Array: [1, 2, 3]"); + + let pythonic_parser = PythonicParser::new(); + let input = r#"[echo(text="List: [a, b, c]")]"#; + let result = pythonic_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "List: [a, b, c]"); +} + +#[tokio::test] +async fn test_multiple_formats_in_text() { + // Test that parsers don't get confused by other formats in the text + + let json_parser = JsonParser::new(); + let input = r#" + Here's some text with [TOOL_CALLS] that shouldn't trigger. + {"name": "actual_tool", "arguments": {}} + And some more text with tags. + "#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "actual_tool"); +} + +#[tokio::test] +async fn test_escaped_characters() { + let json_parser = JsonParser::new(); + + let input = r#"{"name": "write", "arguments": {"content": "Line 1\nLine 2\r\nLine 3\tTabbed\\Backslash\"Quote"}}"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + let content = args["content"].as_str().unwrap(); + assert!(content.contains('\n')); + assert!(content.contains('\t')); + assert!(content.contains('\\')); + assert!(content.contains('"')); +} + +#[tokio::test] +async fn test_numeric_edge_cases() { + let json_parser = JsonParser::new(); + + let input = r#"{ + "name": "calculate", + "arguments": { + "int": 42, + "float": 123.456, + "scientific": 1.23e-4, + "negative": -999, + "zero": 0, + "large": 9007199254740991 + } + }"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["int"], 42); + assert_eq!(args["float"], 123.456); + assert_eq!(args["scientific"], 0.000123); + assert_eq!(args["negative"], -999); + assert_eq!(args["zero"], 0); + assert_eq!(args["large"], 9007199254740991i64); +} + +#[tokio::test] +async fn test_null_and_boolean_values() { + let json_parser = JsonParser::new(); + + let input = r#"{ + "name": "configure", + "arguments": { + "enabled": true, + "disabled": false, + "optional": null + } + }"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["enabled"], true); + assert_eq!(args["disabled"], false); + assert_eq!(args["optional"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_partial_token_at_buffer_boundary() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // Test case that would fail with the bug: + // Send exactly "\n" + let result = parser.parse_incremental("\n{\"name\": \"test\", \"arguments\": {}}\n", + &mut state, + ) + .await + .unwrap(); + + // Should successfully parse after completing + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test"); + } + _ => { + // In Phase 2 simplified streaming, might get Incomplete + // The important thing is it didn't fail to recognize the partial token + } + } +} + +#[tokio::test] +async fn test_exact_prefix_lengths() { + let parser = QwenParser::new(); + + // Test various exact prefix lengths that would be missed by exclusive range + let test_cases = vec![ + ("<", 1), // 1-char prefix + ("", 11), // 11-char prefix (full start without \n) + ]; + + for (prefix, expected_len) in test_cases { + let mut state = ParseState::new(); + let result = parser.parse_incremental(prefix, &mut state).await.unwrap(); + assert!( + matches!(result, StreamResult::Incomplete), + "Prefix '{}' (len {}) should be incomplete", + prefix, + expected_len + ); + assert_eq!( + state.buffer, prefix, + "Buffer should contain the prefix '{}'", + prefix + ); + } +} diff --git a/sgl-router/tests/tool_parser_glm4_moe.rs b/sgl-router/tests/tool_parser_glm4_moe.rs new file mode 100644 index 000000000..bae8fe727 --- /dev/null +++ b/sgl-router/tests/tool_parser_glm4_moe.rs @@ -0,0 +1,194 @@ +//! GLM-4 MoE Parser Integration Tests + +use sglang_router_rs::tool_parser::{Glm4MoeParser, ParseState, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_glm4_complete_parsing() { + let parser = Glm4MoeParser::new(); + + // Test single tool call + let input = r#"Let me search for that. +get_weather +city +Beijing +date +2024-12-25 + +The weather will be..."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + // Verify arguments + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Beijing"); + assert_eq!(args["date"], "2024-12-25"); +} + +#[tokio::test] +async fn test_glm4_multiple_tools() { + let parser = Glm4MoeParser::new(); + + let input = r#"search +query +rust tutorials + +translate +text +Hello World +target_lang +zh +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_glm4_type_conversion() { + let parser = Glm4MoeParser::new(); + + // Test various value types + let input = r#"process +count +42 +rate +1.5 +enabled +true +data +null +text +string value +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["count"], 42); + assert_eq!(args["rate"], 1.5); + assert_eq!(args["enabled"], true); + assert_eq!(args["data"], serde_json::Value::Null); + assert_eq!(args["text"], "string value"); +} + +#[tokio::test] +async fn test_glm4_streaming() { + let parser = Glm4MoeParser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "", + "get_weather\n", + "city\n", + "Shanghai\n", + "units\n", + "celsius\n", + "", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "get_weather"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); +} + +#[test] +fn test_glm4_format_detection() { + let parser = Glm4MoeParser::new(); + + // Should detect GLM-4 format + assert!(parser.detect_format("")); + assert!(parser.detect_format("text with marker")); + + // Should not detect other formats + assert!(!parser.detect_format("[TOOL_CALLS]")); + assert!(!parser.detect_format("<｜tool▁calls▁begin｜>")); + assert!(!parser.detect_format("plain text")); +} + +#[tokio::test] +async fn test_glm4_python_literal_values() { + let parser = Glm4MoeParser::new(); + + // Test Python-style boolean values + let input = r#"config +debug +True +verbose +False +optional +None +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["debug"], true); + assert_eq!(args["verbose"], false); + assert_eq!(args["optional"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_python_literals() { + let parser = Glm4MoeParser::new(); + + let input = r#"test_func +bool_true +True +bool_false +False +none_val +None +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test_func"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["bool_true"], true); + assert_eq!(args["bool_false"], false); + assert_eq!(args["none_val"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_nested_values() { + let parser = Glm4MoeParser::new(); + + let input = r#"process +data +{"nested": {"key": "value"}} +list +[1, 2, 3] +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["data"].is_object()); + assert!(args["list"].is_array()); +} diff --git a/sgl-router/tests/tool_parser_gpt_oss.rs b/sgl-router/tests/tool_parser_gpt_oss.rs new file mode 100644 index 000000000..50dc0be15 --- /dev/null +++ b/sgl-router/tests/tool_parser_gpt_oss.rs @@ -0,0 +1,201 @@ +//! GPT-OSS Parser Integration Tests + +use sglang_router_rs::tool_parser::{GptOssParser, ParseState, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_gpt_oss_complete_parsing() { + let parser = GptOssParser::new(); + + // Test single tool call + let input = r#"Let me search for that information. +<|channel|>commentary to=functions.search<|constrain|>json<|message|>{"query": "rust programming", "limit": 10}<|call|> +Here are the results..."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); + + // Verify arguments + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "rust programming"); + assert_eq!(args["limit"], 10); +} + +#[tokio::test] +async fn test_gpt_oss_multiple_tools() { + let parser = GptOssParser::new(); + + let input = r#"<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{"location": "Paris"}<|call|>commentary +<|channel|>commentary to=functions.search<|constrain|>json<|message|>{"query": "Paris tourism"}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "search"); +} + +#[tokio::test] +async fn test_gpt_oss_with_namespace() { + let parser = GptOssParser::new(); + + // Test with different namespace patterns + let input = r#"<|channel|>commentary to=api.users.create<|constrain|>json<|message|>{"name": "John", "email": "john@example.com"}<|call|> +<|channel|>commentary to=tools.calculator.add<|constrain|>json<|message|>{"x": 10, "y": 20}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "create"); // Should extract last part + assert_eq!(result[1].function.name, "add"); +} + +#[tokio::test] +async fn test_gpt_oss_with_assistant_prefix() { + let parser = GptOssParser::new(); + + // Test with <|start|>assistant prefix + let input = r#"<|start|>assistant<|channel|>commentary to=functions.test<|constrain|>json<|message|>{"key": "value"}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} + +#[tokio::test] +async fn test_gpt_oss_empty_args() { + let parser = GptOssParser::new(); + + // Test with empty arguments + let input = + r#"<|channel|>commentary to=functions.get_time<|constrain|>json<|message|>{}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); + assert_eq!(result[0].function.arguments, "{}"); +} + +#[tokio::test] +async fn test_gpt_oss_streaming() { + let parser = GptOssParser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "<|channel|>commentary to=", + "functions.calculate", + "<|constrain|>json<|message|>", + r#"{"x": 10"#, + r#", "y": 20}"#, + "<|call|>", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "calculate"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "calculate"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); +} + +#[test] +fn test_gpt_oss_format_detection() { + let parser = GptOssParser::new(); + + // Should detect GPT-OSS format + assert!(parser.detect_format("<|channel|>commentary to=")); + assert!(parser.detect_format("<|channel|>commentary")); + assert!(parser.detect_format("text with <|channel|>commentary to= marker")); + + // Should not detect other formats + assert!(!parser.detect_format("[TOOL_CALLS]")); + assert!(!parser.detect_format("")); + assert!(!parser.detect_format("plain text")); +} + +#[tokio::test] +async fn test_gpt_oss_with_whitespace() { + let parser = GptOssParser::new(); + + // Test with whitespace after function name + let input = r#"<|channel|>commentary to=functions.test <|constrain|>json<|message|>{"key": "value"}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} + +#[tokio::test] +async fn test_gpt_oss_complex_json() { + let parser = GptOssParser::new(); + + // Test with complex nested JSON + let input = r#"<|channel|>commentary to=functions.process<|constrain|>json<|message|>{ + "nested": { + "data": [1, 2, 3], + "config": { + "enabled": true + } + } +}<|call|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["nested"]["data"].is_array()); + assert_eq!(args["nested"]["config"]["enabled"], true); +} + +#[tokio::test] +async fn test_commentary_without_function() { + let parser = GptOssParser::new(); + + // Python should extract commentary as normal text + let input = r#"<|channel|>commentary<|message|>**Action plan**: 1. Do X 2. Do Y<|end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); // No tool calls + // TODO: Verify normal text = "**Action plan**: 1. Do X 2. Do Y" +} + +#[tokio::test] +async fn test_final_channel() { + let parser = GptOssParser::new(); + + let input = r#"<|channel|>commentary to=functions.test<|constrain|>json<|message|>{"x": 1}<|call|> +<|channel|>final<|message|>The result is calculated.<|return|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + // TODO: Verify normal text = "The result is calculated." +} + +#[tokio::test] +async fn test_mixed_commentary_and_calls() { + let parser = GptOssParser::new(); + + let input = r#"<|channel|>commentary<|message|>Let me think<|end|> +<|channel|>commentary to=functions.calc<|constrain|>json<|message|>{"x": 5}<|call|> +<|channel|>commentary<|message|>Processing...<|end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calc"); + // TODO: Verify normal text = "Let me think Processing..." +} diff --git a/sgl-router/tests/tool_parser_json.rs b/sgl-router/tests/tool_parser_json.rs new file mode 100644 index 000000000..c8c42b70f --- /dev/null +++ b/sgl-router/tests/tool_parser_json.rs @@ -0,0 +1,147 @@ +//! JSON Parser Integration Tests +//! +//! Tests for the JSON parser which handles OpenAI, Claude, and generic JSON formats + +use serde_json::json; +use sglang_router_rs::tool_parser::{JsonParser, ToolParser}; + +#[tokio::test] +async fn test_simple_json_tool_call() { + let parser = JsonParser::new(); + let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["location"], "San Francisco"); +} + +#[tokio::test] +async fn test_json_array_of_tools() { + let parser = JsonParser::new(); + let input = r#"[ + {"name": "get_weather", "arguments": {"location": "SF"}}, + {"name": "search", "arguments": {"query": "news"}} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "search"); +} + +#[tokio::test] +async fn test_json_with_parameters_key() { + let parser = JsonParser::new(); + let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["x"], 10); + assert_eq!(args["y"], 20); +} + +#[tokio::test] +async fn test_json_extraction_from_text() { + let parser = JsonParser::new(); + let input = r#"I'll help you with that. {"name": "search", "arguments": {"query": "rust"}} Let me search for that."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); +} + +#[tokio::test] +async fn test_json_with_nested_objects() { + let parser = JsonParser::new(); + let input = r#"{ + "name": "update_config", + "arguments": { + "settings": { + "theme": "dark", + "language": "en", + "notifications": { + "email": true, + "push": false + } + } + } + }"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "update_config"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["settings"]["theme"], "dark"); + assert_eq!(args["settings"]["notifications"]["email"], true); +} + +#[tokio::test] +async fn test_json_with_special_characters() { + let parser = JsonParser::new(); + let input = r#"{"name": "echo", "arguments": {"text": "Line 1\nLine 2\tTabbed", "path": "C:\\Users\\test"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Line 1\nLine 2\tTabbed"); + assert_eq!(args["path"], "C:\\Users\\test"); +} + +#[tokio::test] +async fn test_json_with_unicode() { + let parser = JsonParser::new(); + let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍", "emoji": "😊"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Hello 世界 🌍"); + assert_eq!(args["emoji"], "😊"); +} + +#[tokio::test] +async fn test_json_empty_arguments() { + let parser = JsonParser::new(); + let input = r#"{"name": "ping", "arguments": {}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "ping"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args, json!({})); +} + +#[tokio::test] +async fn test_json_invalid_format() { + let parser = JsonParser::new(); + + // Missing closing brace + let input = r#"{"name": "test", "arguments": {"key": "value""#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); + + // Not JSON at all + let input = "This is just plain text"; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); +} + +#[tokio::test] +async fn test_json_format_detection() { + let parser = JsonParser::new(); + + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(parser.detect_format(r#"[{"name": "test"}]"#)); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format(r#"{"key": "value"}"#)); // No name field +} diff --git a/sgl-router/tests/tool_parser_kimik2.rs b/sgl-router/tests/tool_parser_kimik2.rs new file mode 100644 index 000000000..66be2e88f --- /dev/null +++ b/sgl-router/tests/tool_parser_kimik2.rs @@ -0,0 +1,160 @@ +//! Kimi K2 Parser Integration Tests + +use sglang_router_rs::tool_parser::{KimiK2Parser, ParseState, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_kimik2_complete_parsing() { + let parser = KimiK2Parser::new(); + + // Test single tool call + let input = r#"Let me help you with that. +<|tool_calls_section_begin|> +<|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"location": "Tokyo", "units": "celsius"}<|tool_call_end|> +<|tool_calls_section_end|> +The weather in Tokyo is..."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + // Verify arguments + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["location"], "Tokyo"); + assert_eq!(args["units"], "celsius"); +} + +#[tokio::test] +async fn test_kimik2_multiple_tools() { + let parser = KimiK2Parser::new(); + + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"query": "rust tutorials"}<|tool_call_end|> +<|tool_call_begin|>functions.translate:1<|tool_call_argument_begin|>{"text": "Hello", "to": "ja"}<|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_kimik2_with_whitespace() { + let parser = KimiK2Parser::new(); + + // Test with extra whitespace + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {"key": "value", "num": 42} <|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["key"], "value"); + assert_eq!(args["num"], 42); +} + +#[tokio::test] +async fn test_kimik2_streaming() { + let parser = KimiK2Parser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "<|tool_calls_section_begin|>\n", + "<|tool_call_begin|>functions.", + "calculate:0", + "<|tool_call_argument_begin|>", + r#"{"x": 10, "#, + r#""y": 20}"#, + "<|tool_call_end|>\n", + "<|tool_calls_section_end|>", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "calculate"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "calculate"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); +} + +#[test] +fn test_kimik2_format_detection() { + let parser = KimiK2Parser::new(); + + // Should detect Kimi K2 format + assert!(parser.detect_format("<|tool_calls_section_begin|>")); + assert!(parser.detect_format("<|tool_call_begin|>")); + assert!(parser.detect_format("text with <|tool_calls_section_begin|> marker")); + + // Should not detect other formats + assert!(!parser.detect_format("[TOOL_CALLS]")); + assert!(!parser.detect_format("")); + assert!(!parser.detect_format("plain text")); +} + +#[tokio::test] +async fn test_kimik2_sequential_indices() { + let parser = KimiK2Parser::new(); + + // Test with proper sequential indexing + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|>functions.first:0<|tool_call_argument_begin|>{"param": "a"}<|tool_call_end|> +<|tool_call_begin|>functions.second:1<|tool_call_argument_begin|>{"param": "b"}<|tool_call_end|> +<|tool_call_begin|>functions.third:2<|tool_call_argument_begin|>{"param": "c"}<|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[0].function.name, "first"); + assert_eq!(result[1].function.name, "second"); + assert_eq!(result[2].function.name, "third"); +} + +#[tokio::test] +async fn test_function_index_extraction() { + let parser = KimiK2Parser::new(); + + let input = r#"Text before tool calls. +<|tool_calls_section_begin|> +<|tool_call_begin|>functions.search:0<|tool_call_argument_begin|>{"query": "rust"}<|tool_call_end|> +<|tool_call_begin|>functions.calc:1<|tool_call_argument_begin|>{"x": 10}<|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "calc"); + // TODO: Verify indices are preserved: 0 and 1 + // TODO: Verify normal text = "Text before tool calls." +} + +#[tokio::test] +async fn test_namespace_extraction() { + let parser = KimiK2Parser::new(); + + let input = r#"<|tool_calls_section_begin|> +<|tool_call_begin|>api.tools.search:0<|tool_call_argument_begin|>{"q": "test"}<|tool_call_end|> +<|tool_calls_section_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); // Should extract after last dot +} diff --git a/sgl-router/tests/tool_parser_llama.rs b/sgl-router/tests/tool_parser_llama.rs new file mode 100644 index 000000000..4f86ef2b2 --- /dev/null +++ b/sgl-router/tests/tool_parser_llama.rs @@ -0,0 +1,424 @@ +//! Llama Parser Integration Tests +//! +//! Tests for the Llama parser which handles <|python_tag|> format and plain JSON + +use sglang_router_rs::tool_parser::{LlamaParser, ToolParser}; + +#[tokio::test] +async fn test_llama_python_tag_format() { + let parser = LlamaParser::new(); + let input = r#"<|python_tag|>{"name": "search", "arguments": {"query": "weather"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "weather"); +} + +#[tokio::test] +async fn test_llama_plain_json_fallback() { + let parser = LlamaParser::new(); + let input = r#"{"name": "calculate", "arguments": {"x": 5, "y": 10}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["x"], 5); + assert_eq!(args["y"], 10); +} + +#[tokio::test] +async fn test_llama_with_text_before() { + let parser = LlamaParser::new(); + let input = r#"Let me help you with that. <|python_tag|>{"name": "get_time", "arguments": {"timezone": "UTC"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["timezone"], "UTC"); +} + +#[tokio::test] +async fn test_llama_with_nested_json() { + let parser = LlamaParser::new(); + let input = r#"<|python_tag|>{ + "name": "update_settings", + "arguments": { + "preferences": { + "theme": "dark", + "language": "en" + }, + "notifications": true + } + }"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "update_settings"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["preferences"]["theme"], "dark"); + assert_eq!(args["notifications"], true); +} + +#[tokio::test] +async fn test_llama_empty_arguments() { + let parser = LlamaParser::new(); + + // With python_tag + let input = r#"<|python_tag|>{"name": "ping", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "ping"); + + // Plain JSON + let input = r#"{"name": "ping", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "ping"); +} + +#[tokio::test] +async fn test_llama_format_detection() { + let parser = LlamaParser::new(); + + assert!(parser.detect_format(r#"<|python_tag|>{"name": "test"}"#)); + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); + assert!(!parser.detect_format("plain text")); + assert!(!parser.detect_format(r#"{"key": "value"}"#)); // No name field +} + +#[tokio::test] +async fn test_llama_invalid_json_after_tag() { + let parser = LlamaParser::new(); + + let input = r#"<|python_tag|>{"name": invalid}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); +} + +#[tokio::test] +async fn test_llama_real_world_output() { + let parser = LlamaParser::new(); + + // Actual output from Llama 3.2 model - simplified for testing + let input = r#"I'll search for that information for you. + +<|python_tag|>{"name": "web_search", "arguments": {"query": "Llama 3.2 model capabilities", "num_results": 5, "search_type": "recent"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "web_search"); + + // Test with nicely formatted JSON + let formatted_input = r#"<|python_tag|>{ + "name": "get_current_time", + "arguments": { + "timezone": "America/New_York", + "format": "ISO8601" + } +}"#; + + let result2 = parser.parse_complete(formatted_input).await.unwrap(); + assert_eq!(result2.len(), 1); + assert_eq!(result2[0].function.name, "get_current_time"); +} + +#[tokio::test] +async fn test_llama_json_array_format() { + let parser = LlamaParser::new(); + + // Plain JSON array (should work as fallback) + let input = r#"[{"name": "func1", "arguments": {}}, {"name": "func2", "arguments": {}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + // Current implementation might handle this through JSON fallback + assert!(!result.is_empty()); +} + +#[tokio::test] +async fn test_single_json() { + // Test parsing plain JSON without python_tag + let parser = LlamaParser::new(); + let text = r#"{"name": "get_weather", "arguments": {"city": "Paris"}}"#; + + let result = parser.parse_complete(text).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Paris"); +} + +#[tokio::test] +async fn test_multiple_json_with_separator() { + // Test multiple JSON objects with semicolon separator + let parser = LlamaParser::new(); + let text = r#"<|python_tag|>{"name": "get_weather", "arguments": {"city": "Paris"}};{"name": "get_tourist_attractions", "arguments": {"city": "Paris"}}"#; + + let result = parser.parse_complete(text).await.unwrap(); + // Note: Current implementation may only parse the first one due to semicolon handling + assert!(!result.is_empty()); + assert_eq!(result[0].function.name, "get_weather"); +} + +#[tokio::test] +async fn test_multiple_json_with_separator_customized() { + // Test multiple JSON objects with python_tag repeated + let parser = LlamaParser::new(); + let text = r#"<|python_tag|>{"name": "get_weather", "arguments": {}}<|python_tag|>{"name": "get_tourist_attractions", "arguments": {}}"#; + + let result = parser.parse_complete(text).await.unwrap(); + // Current implementation may handle this differently + assert!(!result.is_empty()); + assert_eq!(result[0].function.name, "get_weather"); +} + +#[tokio::test] +async fn test_json_with_trailing_text() { + // Test JSON with trailing text after + let parser = LlamaParser::new(); + let text = r#"{"name": "get_weather", "arguments": {}} Some follow-up text"#; + + let result = parser.parse_complete(text).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); +} + +#[tokio::test] +async fn test_invalid_then_valid_json() { + // Test error recovery - invalid JSON followed by valid JSON + let parser = LlamaParser::new(); + let text = r#"{"name": "get_weather", "arguments": {{"name": "get_weather", "arguments": {}}"#; + + let result = parser.parse_complete(text).await.unwrap(); + // Should parse at least one valid JSON + if !result.is_empty() { + assert_eq!(result[0].function.name, "get_weather"); + } +} + +#[tokio::test] +async fn test_plain_text_only() { + // Test plain text with no tool calls + let parser = LlamaParser::new(); + let text = "This is just plain explanation text."; + + let result = parser.parse_complete(text).await.unwrap(); + assert_eq!(result.len(), 0); +} + +#[tokio::test] +async fn test_with_python_tag_prefix() { + // Test text before python_tag + let parser = LlamaParser::new(); + let text = r#"Some intro. <|python_tag|>{"name": "get_weather", "arguments": {}}"#; + + let result = parser.parse_complete(text).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); +} + +// ============================================================================ +// STREAMING TESTS +// ============================================================================ + +#[tokio::test] +async fn test_llama_streaming_simple() { + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // Send complete JSON at once + let full_json = r#"<|python_tag|>{"name": "search", "arguments": {"query": "weather"}}"#; + + let result = parser + .parse_incremental(full_json, &mut state) + .await + .unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "search"); + } + _ => panic!("Expected ToolComplete for complete JSON input"), + } +} + +#[tokio::test] +async fn test_llama_streaming_partial() { + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // Stream in chunks + let chunks = vec![ + r#"<|python"#, + r#"_tag|>{"name": "#, + r#""calculate", "#, + r#""arguments": {"x": 10}"#, + r#"}"#, + ]; + + let mut got_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "calculate"); + got_complete = true; + } + } + + assert!(got_complete, "Should have completed parsing"); +} + +#[tokio::test] +async fn test_llama_streaming_plain_json() { + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // Stream plain JSON without python_tag + let chunks = vec![ + r#"{"name": "#, + r#""search", "#, + r#""arguments": "#, + r#"{"query": "#, + r#""test"}}"#, + ]; + + let mut got_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "search"); + got_complete = true; + } + } + + assert!(got_complete, "Should have completed parsing"); +} + +#[tokio::test] +async fn test_llama_streaming_with_text_before() { + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let chunks = vec![ + r#"Let me help you. "#, + r#"<|python_tag|>"#, + r#"{"name": "get_time","#, + r#" "arguments": {"#, + r#""timezone": "UTC"}}"#, + ]; + + let mut got_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "get_time"); + got_complete = true; + } + } + + assert!(got_complete, "Should have completed parsing"); +} + +#[tokio::test] +async fn test_llama_streaming_multiple_tools() { + // Test streaming multiple tool calls with semicolon separator + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = + r#"<|python_tag|>{"name": "func1", "arguments": {}};{"name": "func2", "arguments": {}}"#; + + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + // Should get first tool complete + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "func1"); + } + _ => panic!("Expected first tool to be complete"), + } + + // Process remaining buffer to get second tool + let result2 = parser.parse_incremental("", &mut state).await.unwrap(); + match result2 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "func2"); + } + _ => panic!("Expected second tool to be complete"), + } +} + +#[tokio::test] +async fn test_llama_streaming_multiple_tools_chunked() { + // Test streaming multiple tool calls arriving in chunks + let parser = LlamaParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // First chunk - incomplete first JSON + let chunk1 = r#"<|python_tag|>{"name": "get_weather", "arguments""#; + let result1 = parser.parse_incremental(chunk1, &mut state).await.unwrap(); + + // Should be incomplete or have tool name + match result1 { + sglang_router_rs::tool_parser::StreamResult::Incomplete + | sglang_router_rs::tool_parser::StreamResult::ToolName { .. } + | sglang_router_rs::tool_parser::StreamResult::ToolArguments { .. } => { + // Expected - could get tool name or be incomplete or even partial args + } + _ => panic!( + "Expected incomplete or tool name for partial JSON, got: {:?}", + result1 + ), + } + + // Second chunk - complete first JSON and separator + let chunk2 = r#": {"city": "Paris"}};{"name": "#; + let result2 = parser.parse_incremental(chunk2, &mut state).await.unwrap(); + + // Should get first tool complete + match result2 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["city"], "Paris"); + } + _ => panic!("Expected first tool to be complete after separator"), + } + + // Third chunk - complete second JSON + let chunk3 = r#""get_time", "arguments": {"timezone": "UTC"}}"#; + let result3 = parser.parse_incremental(chunk3, &mut state).await.unwrap(); + + // Should get second tool complete + match result3 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_time"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["timezone"], "UTC"); + } + _ => { + // If not complete yet, try one more empty chunk + let result4 = parser.parse_incremental("", &mut state).await.unwrap(); + match result4 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_time"); + let args: serde_json::Value = + serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["timezone"], "UTC"); + } + _ => panic!("Expected second tool to be complete"), + } + } + } +} diff --git a/sgl-router/tests/tool_parser_mistral.rs b/sgl-router/tests/tool_parser_mistral.rs new file mode 100644 index 000000000..3801006f5 --- /dev/null +++ b/sgl-router/tests/tool_parser_mistral.rs @@ -0,0 +1,153 @@ +//! Mistral Parser Integration Tests +//! +//! Tests for the Mistral parser which handles [TOOL_CALLS] format + +use serde_json::json; +use sglang_router_rs::tool_parser::{MistralParser, ToolParser}; + +#[tokio::test] +async fn test_mistral_single_tool() { + let parser = MistralParser::new(); + let input = r#"Let me search for that. +[TOOL_CALLS] [{"name": "search_web", "arguments": {"query": "latest news", "max_results": 5}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search_web"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "latest news"); + assert_eq!(args["max_results"], 5); +} + +#[tokio::test] +async fn test_mistral_multiple_tools() { + let parser = MistralParser::new(); + let input = r#"I'll help you with both tasks. +[TOOL_CALLS] [ + {"name": "get_weather", "arguments": {"city": "Tokyo", "units": "celsius"}}, + {"name": "search_news", "arguments": {"query": "AI developments", "limit": 10}} +]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + + assert_eq!(result[0].function.name, "get_weather"); + let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args0["city"], "Tokyo"); + + assert_eq!(result[1].function.name, "search_news"); + let args1: serde_json::Value = serde_json::from_str(&result[1].function.arguments).unwrap(); + assert_eq!(args1["query"], "AI developments"); +} + +#[tokio::test] +async fn test_mistral_nested_json() { + let parser = MistralParser::new(); + let input = r#"Processing complex data. +[TOOL_CALLS] [{"name": "process_data", "arguments": {"config": {"nested": {"value": [1, 2, 3]}}, "enabled": true}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3])); + assert_eq!(args["enabled"], true); +} + +#[tokio::test] +async fn test_mistral_with_text_after() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}}] + +And here's some text after the tool call that should be ignored."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} + +#[tokio::test] +async fn test_mistral_empty_arguments() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "ping", "arguments": {}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "ping"); +} + +#[tokio::test] +async fn test_mistral_with_brackets_in_strings() { + let parser = MistralParser::new(); + let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"text": "Array notation: arr[0] = value[1]"}}]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Array notation: arr[0] = value[1]"); +} + +#[tokio::test] +async fn test_mistral_format_detection() { + let parser = MistralParser::new(); + + assert!(parser.detect_format("[TOOL_CALLS] [")); + assert!(parser.detect_format("Some text [TOOL_CALLS] [")); + assert!(!parser.detect_format("Just plain text")); + assert!(!parser.detect_format("[{\"name\": \"test\"}]")); // JSON array without TOOL_CALLS +} + +#[tokio::test] +async fn test_mistral_malformed_json() { + let parser = MistralParser::new(); + + // Missing closing bracket + let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}"#; + if let Ok(result) = parser.parse_complete(input).await { + assert_eq!(result.len(), 0); + } + // Error is also acceptable for malformed input + + // Invalid JSON inside + let input = r#"[TOOL_CALLS] [{"name": invalid}]"#; + if let Ok(result) = parser.parse_complete(input).await { + assert_eq!(result.len(), 0); + } + // Error is also acceptable for malformed input +} + +#[tokio::test] +async fn test_mistral_real_world_output() { + let parser = MistralParser::new(); + + // Actual output from Mistral model + let input = r#"I'll search for information about Rust programming and check the weather in San Francisco. + +[TOOL_CALLS] [ + { + "name": "web_search", + "arguments": { + "query": "Rust programming language features 2024", + "max_results": 3, + "include_snippets": true + } + }, + { + "name": "get_weather", + "arguments": { + "location": "San Francisco, CA", + "units": "fahrenheit", + "include_forecast": false + } + } +] + +Let me execute these searches for you."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "web_search"); + assert_eq!(result[1].function.name, "get_weather"); +} diff --git a/sgl-router/tests/tool_parser_mixed_edge_cases.rs b/sgl-router/tests/tool_parser_mixed_edge_cases.rs new file mode 100644 index 000000000..595fb74a0 --- /dev/null +++ b/sgl-router/tests/tool_parser_mixed_edge_cases.rs @@ -0,0 +1,301 @@ +//! Mixed Format and Additional Edge Case Tests +//! +//! Tests for edge cases across parsers and mixed format scenarios + +use serde_json::json; +use sglang_router_rs::tool_parser::{ + JsonParser, LlamaParser, MistralParser, ParseState, PythonicParser, QwenParser, StreamResult, + ToolParser, +}; + +#[tokio::test] +async fn test_mixed_formats_in_text() { + // Test that parsers correctly ignore other formats' markers + + let json_parser = JsonParser::new(); + let input = r#" + Some text with [TOOL_CALLS] marker that shouldn't trigger. + Also has tags and [function()] syntax. + But here's the actual JSON: {"name": "test", "arguments": {}} + "#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + // Mistral parser should ignore JSON and other formats + let mistral_parser = MistralParser::new(); + let input = r#" + {"name": "fake"} [function()] + [TOOL_CALLS] [{"name": "real", "arguments": {}}] + "#; + + let result = mistral_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "real"); +} + +#[tokio::test] +async fn test_format_markers_in_string_content() { + // Test that format markers inside string content don't interfere + + let pythonic_parser = PythonicParser::new(); + let input = r#"[echo(text="Use [TOOL_CALLS] and in text")]"#; + + let result = pythonic_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Use [TOOL_CALLS] and in text"); + + let qwen_parser = QwenParser::new(); + let input = r#" +{"name": "log", "arguments": {"msg": "Found [function()] pattern"}} +"#; + + let result = qwen_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["msg"], "Found [function()] pattern"); +} + +#[tokio::test] +async fn test_deeply_nested_json_structures() { + let json_parser = JsonParser::new(); + + let input = r#"{ + "name": "deep_process", + "arguments": { + "level1": { + "level2": { + "level3": { + "level4": { + "level5": { + "data": [1, 2, [3, [4, 5]]] + } + } + } + } + } + } + }"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "deep_process"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["level1"]["level2"]["level3"]["level4"]["level5"]["data"].is_array()); +} + +#[tokio::test] +async fn test_multiple_sequential_calls_different_formats() { + // Simulate a scenario where different parts of text have different formats + // (though each parser will only recognize its own format) + + let llama_parser = LlamaParser::new(); + + // Llama parser currently only returns the first tool found + let input = r#"First call: <|python_tag|>{"name": "call1", "arguments": {}}"#; + + let result = llama_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "call1"); + + // Test plain JSON separately + let input2 = r#"{"name": "call2", "arguments": {"x": 1}}"#; + let result2 = llama_parser.parse_complete(input2).await.unwrap(); + assert_eq!(result2.len(), 1); + assert_eq!(result2[0].function.name, "call2"); +} + +#[tokio::test] +async fn test_empty_and_whitespace_variations() { + let json_parser = JsonParser::new(); + + // Various whitespace scenarios + let cases = vec![ + r#" {"name":"compact","arguments":{}} "#, + r#" + + {"name": "spaced", "arguments": {}} + + "#, + r#" {"name": "tabbed", "arguments": {}} "#, // tabs + ]; + + for input in cases { + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1, "Should parse regardless of whitespace"); + } +} + +#[tokio::test] +async fn test_special_json_values() { + let json_parser = JsonParser::new(); + + // Test various special JSON values + let input = r#"{ + "name": "test_special", + "arguments": { + "float_e": 1.23e10, + "float_neg_e": 1.23e-10, + "hex_like": "0x1234", + "very_long_num": 99999999999999999999, + "special_strings": ["", " ", "\u0000", "\u001f"], + "escaped": "\\n\\r\\t\\\"\\\\", + "unicode": "\u4e2d\u6587" + } + }"#; + + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test_special"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["special_strings"].is_array()); + assert!(args["escaped"].is_string()); +} + +#[tokio::test] +async fn test_parser_recovery_after_invalid_input() { + let mut state = ParseState::new(); + let parser = JsonParser::new(); + + // Send invalid JSON first + let _ = parser.parse_incremental(r#"{"broken": "#, &mut state).await; + + // Clear state and try valid JSON + state.buffer.clear(); + let result = parser + .parse_incremental(r#"{"name": "valid", "arguments": {}}"#, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "valid"); + } + _ => { + // Might be incomplete depending on implementation + } + } +} + +#[tokio::test] +async fn test_boundary_cases_for_extraction() { + // Test edge cases in JSON extraction from text + + let json_parser = JsonParser::new(); + + // JSON at the very beginning + let input = r#"{"name": "start", "arguments": {}} and then text"#; + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "start"); + + // JSON at the very end + let input = r#"Some text first {"name": "end", "arguments": {}}"#; + let result = json_parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "end"); + + // Multiple JSON objects in text (should find first valid one) + let input = + r#"Text {"name": "first", "arguments": {}} more {"name": "second", "arguments": {}}"#; + let result = json_parser.parse_complete(input).await.unwrap(); + assert!(!result.is_empty()); + assert_eq!(result[0].function.name, "first"); +} + +#[tokio::test] +async fn test_pythonic_edge_cases() { + let parser = PythonicParser::new(); + + // Function name with underscores and numbers + let input = r#"[func_name_2(param_1="value")]"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "func_name_2"); + + // Empty string argument + let input = r#"[process(text="")]"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], ""); +} + +#[tokio::test] +async fn test_mistral_with_pretty_json() { + let parser = MistralParser::new(); + + // Pretty-printed JSON in Mistral format + let input = r#"[TOOL_CALLS] [ + { + "name": "formatted", + "arguments": { + "nested": { + "key": "value" + }, + "array": [ + 1, + 2, + 3 + ] + } + } + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "formatted"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["nested"]["key"], "value"); + assert_eq!(args["array"], json!([1, 2, 3])); +} + +#[tokio::test] +async fn test_qwen_with_cdata_like_content() { + let parser = QwenParser::new(); + + // Test with content that looks like CDATA but isn't + // Note: QwenParser expects exactly "\n" with the newline + let input = r#" +{"name": "process", "arguments": {"xml": ""}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["xml"], ""); +} + +#[tokio::test] +async fn test_extremely_long_function_names() { + let parser = PythonicParser::new(); + + let long_name = "very_long_function_name_that_might_appear_in_generated_code_somewhere"; + let input = format!(r#"[{}(param="value")]"#, long_name); + + let result = parser.parse_complete(&input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, long_name); +} + +#[tokio::test] +async fn test_json_with_duplicate_keys() { + let parser = JsonParser::new(); + + // JSON with duplicate keys (last one should win per JSON spec) + let input = r#"{"name": "test", "arguments": {"key": "first", "key": "second"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + // JSON parsers typically keep the last value for duplicate keys + assert_eq!(args["key"], "second"); +} diff --git a/sgl-router/tests/tool_parser_pythonic.rs b/sgl-router/tests/tool_parser_pythonic.rs new file mode 100644 index 000000000..4297fdbd3 --- /dev/null +++ b/sgl-router/tests/tool_parser_pythonic.rs @@ -0,0 +1,559 @@ +//! Pythonic Parser Integration Tests +//! +//! Tests for the Pythonic parser which handles Python function call syntax + +use serde_json::json; +use sglang_router_rs::tool_parser::{PythonicParser, ToolParser}; + +#[tokio::test] +async fn test_pythonic_single_function() { + let parser = PythonicParser::new(); + let input = r#"[get_weather(city="London", units="celsius")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["city"], "London"); + assert_eq!(args["units"], "celsius"); +} + +#[tokio::test] +async fn test_pythonic_multiple_functions() { + let parser = PythonicParser::new(); + let input = + r#"[search_web(query="Rust programming", max_results=5), get_time(timezone="UTC")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search_web"); + assert_eq!(result[1].function.name, "get_time"); + + let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args0["query"], "Rust programming"); + assert_eq!(args0["max_results"], 5); +} + +#[tokio::test] +async fn test_pythonic_with_python_literals() { + let parser = PythonicParser::new(); + let input = r#"[configure(enabled=True, disabled=False, optional=None)]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["enabled"], true); + assert_eq!(args["disabled"], false); + assert_eq!(args["optional"], json!(null)); +} + +#[tokio::test] +async fn test_pythonic_with_lists_and_dicts() { + let parser = PythonicParser::new(); + let input = + r#"[process_data(items=[1, 2, 3], config={"key": "value", "nested": {"deep": True}})]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["items"], json!([1, 2, 3])); + assert_eq!(args["config"]["key"], "value"); + assert_eq!(args["config"]["nested"]["deep"], true); +} + +#[tokio::test] +async fn test_pythonic_with_special_tokens() { + let parser = PythonicParser::new(); + + // Llama 4 sometimes outputs these tokens + let input = r#"<|python_start|>[calculate(x=10, y=20)]<|python_end|>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "calculate"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["x"], 10); + assert_eq!(args["y"], 20); +} + +#[tokio::test] +async fn test_pythonic_with_nested_parentheses() { + let parser = PythonicParser::new(); + let input = r#"[math_eval(expression="(2 + 3) * (4 - 1)", round_to=2)]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["expression"], "(2 + 3) * (4 - 1)"); + assert_eq!(args["round_to"], 2); +} + +#[tokio::test] +async fn test_pythonic_with_escaped_quotes() { + let parser = PythonicParser::new(); + let input = r#"[echo(text="She said \"Hello\" to him")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "She said \"Hello\" to him"); +} + +#[tokio::test] +async fn test_pythonic_empty_arguments() { + let parser = PythonicParser::new(); + let input = r#"[ping()]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "ping"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args, json!({})); +} + +#[tokio::test] +async fn test_pythonic_format_detection() { + let parser = PythonicParser::new(); + + assert!(parser.detect_format("[function_name(")); + assert!(parser.detect_format("[get_weather(city=\"NYC\")]")); + assert!(!parser.detect_format("Just plain text")); + assert!(!parser.detect_format("[1, 2, 3]")); // Plain list + assert!(!parser.detect_format("{\"name\": \"test\"}")); // JSON +} + +#[tokio::test] +async fn test_pythonic_invalid_syntax() { + let parser = PythonicParser::new(); + + // Missing closing bracket + let input = r#"[function(arg=value"#; + if let Ok(result) = parser.parse_complete(input).await { + assert_eq!(result.len(), 0); + } + // Error is also acceptable for invalid syntax + + // Invalid Python syntax - empty parameter name + // Note: The parser currently accepts this invalid syntax and returns a result + // This is a known limitation of the current implementation + let input = r#"[function(=value)]"#; + if let Ok(result) = parser.parse_complete(input).await { + // The parser incorrectly accepts this, returning 1 result + // We'll accept this behavior for now but note it's not ideal + assert!(result.len() <= 1, "Should parse at most one function"); + } + // Error would be the correct behavior +} + +#[tokio::test] +async fn test_pythonic_real_world_llama4() { + let parser = PythonicParser::new(); + + // Actual output from Llama 4 model + let input = r#"I'll help you with multiple tasks. Let me search for information and perform calculations. + +[web_search(query="latest Rust features", max_results=3, safe_search=True), + calculate(expression="42 * 3.14159", precision=2), + get_weather(city="San Francisco", units="fahrenheit", include_forecast=False)] + +These functions will provide the information you need."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[0].function.name, "web_search"); + assert_eq!(result[1].function.name, "calculate"); + assert_eq!(result[2].function.name, "get_weather"); + + let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args0["query"], "latest Rust features"); + assert_eq!(args0["safe_search"], true); +} + +#[tokio::test] +async fn test_pythonic_nested_brackets_in_lists() { + let parser = PythonicParser::new(); + + // Test nested brackets within list arguments + let input = r#"[process_matrix(data=[[1, 2], [3, 4]], labels=["row[0]", "row[1]"])]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "process_matrix"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["data"], json!([[1, 2], [3, 4]])); + assert_eq!(args["labels"], json!(["row[0]", "row[1]"])); +} + +#[tokio::test] +async fn test_pythonic_nested_brackets_in_dicts() { + let parser = PythonicParser::new(); + + // Test nested brackets within dictionary arguments + let input = + r#"[analyze(config={"patterns": ["[a-z]+", "[0-9]+"], "nested": {"list": [1, [2, 3]]}})]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "analyze"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["config"]["patterns"], json!(["[a-z]+", "[0-9]+"])); + assert_eq!(args["config"]["nested"]["list"], json!([1, [2, 3]])); +} + +#[tokio::test] +async fn test_pythonic_mixed_quotes() { + let parser = PythonicParser::new(); + + // Test mixed quote types in arguments + let input = r#"[format_text(single='Hello', double="World", mixed="It's \"quoted\"")]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "format_text"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["single"], "Hello"); + assert_eq!(args["double"], "World"); + assert_eq!(args["mixed"], "It's \"quoted\""); +} + +#[tokio::test] +async fn test_pythonic_complex_nesting() { + let parser = PythonicParser::new(); + + // Test complex nested structures + let input = r#"[transform( + matrix=[[1, [2, 3]], [4, [5, [6, 7]]]], + operations=[{"type": "scale", "factor": [2, 3]}, {"type": "rotate", "angle": 90}], + metadata={"tags": ["nested[0]", "nested[1]"], "config": {"depth": [1, 2, 3]}} + )]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "transform"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["matrix"].is_array()); + assert!(args["operations"].is_array()); + assert_eq!(args["operations"][0]["type"], "scale"); + assert_eq!(args["metadata"]["config"]["depth"], json!([1, 2, 3])); +} + +#[tokio::test] +async fn test_parse_streaming_no_brackets() { + // Test parsing text with no brackets (no tool calls) + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "This is just normal text without any tool calls."; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::Incomplete => { + // Expected - no tool calls found + assert_eq!(state.buffer, text); + } + _ => panic!("Should return Incomplete for text without tool calls"), + } +} + +#[tokio::test] +async fn test_parse_streaming_complete_tool_call() { + // Test parsing a complete tool call + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "Here's a tool call: [get_weather(location='New York', unit='celsius')]"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "New York"); + assert_eq!(args["unit"], "celsius"); + assert_eq!(state.buffer, ""); + } + _ => panic!("Should return ToolComplete for complete tool call"), + } +} + +#[tokio::test] +async fn test_parse_streaming_text_before_tool_call() { + // Test parsing text that appears before a tool call + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "This is some text before [get_weather(location='London')]"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "London"); + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_partial_tool_call() { + // Test parsing a partial tool call that spans multiple chunks + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // First chunk with opening bracket but no closing bracket + let text1 = "Let me check the weather: [get_weather(location="; + let result1 = parser.parse_incremental(text1, &mut state).await.unwrap(); + + match result1 { + sglang_router_rs::tool_parser::StreamResult::Incomplete => { + assert!(state.buffer.contains("[get_weather(location=")); + } + _ => panic!("First chunk should return Incomplete"), + } + + // Second chunk completing the tool call + let text2 = "'Paris')]"; + let result2 = parser.parse_incremental(text2, &mut state).await.unwrap(); + + match result2 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "Paris"); + assert_eq!(state.buffer, ""); + } + _ => panic!("Second chunk should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_bracket_without_text_before() { + // Test parsing a tool call that starts at the beginning of the text + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "[search(query='python programming')]"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "search"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["query"], "python programming"); + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_text_after_tool_call() { + // Test parsing text that appears after a tool call + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // First chunk with complete tool call and some text after + let text = "[get_weather(location='Tokyo')] Here's the forecast:"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + // Text after tool call should remain in buffer + // Note: Current implementation may clear buffer, this behavior needs verification + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_multiple_tool_calls() { + // Test parsing multiple tool calls in sequence + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "[get_weather(location='Berlin'), search(query='restaurants')]"; + + // Current implementation may handle this as a single parse + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + // The parser should handle multiple tools in one bracket pair + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(_) => { + // Expected behavior - parses first tool + } + _ => { + // Also acceptable if it returns Incomplete waiting for more + } + } +} + +#[tokio::test] +async fn test_parse_streaming_opening_bracket_only() { + // Test parsing text with only an opening bracket but no closing bracket + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "Let's try this: ["; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::Incomplete => { + assert!(state.buffer.ends_with("[")); + } + _ => panic!("Should return Incomplete for partial bracket"), + } +} + +#[tokio::test] +async fn test_parse_streaming_nested_brackets() { + // Test parsing tool calls with nested brackets in arguments + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = "[get_weather(location='New York', unit='celsius', data=[1, 2, 3])]"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "New York"); + assert_eq!(args["unit"], "celsius"); + assert_eq!(args["data"], json!([1, 2, 3])); + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_nested_brackets_dict() { + // Test parsing tool calls with nested dictionaries and lists + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = r#"[search(query='test', config={'options': [1, 2], 'nested': {'key': 'value'}})]"#; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "search"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["query"], "test"); + assert_eq!(args["config"]["options"], json!([1, 2])); + assert_eq!(args["config"]["nested"]["key"], "value"); + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_multiple_tools_with_nested_brackets() { + // Test parsing multiple tool calls with nested brackets + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let text = + "[get_weather(location='Paris', data=[10, 20]), search(query='test', filters=['a', 'b'])]"; + let result = parser.parse_incremental(text, &mut state).await.unwrap(); + + // Should parse both tools successfully + match result { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + // At least gets the first tool + assert_eq!(tool.function.name, "get_weather"); + } + _ => panic!("Should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_partial_nested_brackets() { + // Test parsing partial tool calls with nested brackets across chunks + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + // First chunk with nested brackets but incomplete + let text1 = "Here's a call: [get_weather(location='Tokyo', data=[1, 2"; + let result1 = parser.parse_incremental(text1, &mut state).await.unwrap(); + + match result1 { + sglang_router_rs::tool_parser::StreamResult::Incomplete => { + assert!(state + .buffer + .contains("[get_weather(location='Tokyo', data=[1, 2")); + } + _ => panic!("First chunk should return Incomplete"), + } + + // Second chunk completing the nested brackets + let text2 = ", 3])]"; + let result2 = parser.parse_incremental(text2, &mut state).await.unwrap(); + + match result2 { + sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "Tokyo"); + assert_eq!(args["data"], json!([1, 2, 3])); + } + _ => panic!("Second chunk should return ToolComplete"), + } +} + +#[tokio::test] +async fn test_parse_streaming_with_python_start_and_end_token() { + // Test parsing a message that starts with <|python_start|> and <|python_end|> across chunks + let parser = PythonicParser::new(); + let mut state = sglang_router_rs::tool_parser::ParseState::new(); + + let chunks = vec![ + "Here's a call: ", + "<|python_", + "start|>[get_weather(location=", + "'Tokyo', data=[1, 2", + ", 3])]<|python_end|>", + ]; + + let mut got_tool = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let sglang_router_rs::tool_parser::StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["location"], "Tokyo"); + assert_eq!(args["data"], json!([1, 2, 3])); + got_tool = true; + } + } + + assert!(got_tool, "Should have parsed the tool call"); +} + +#[tokio::test] +async fn test_detect_and_parse_with_python_start_and_end_token() { + // Test parsing a message that starts with <|python_start|> and contains a valid tool call + let parser = PythonicParser::new(); + + let text = "User wants to get the weather in Mars. <|python_start|>[get_weather(location='Mars', unit='celsius')]<|python_end|> In this way we will get the weather in Mars."; + let result = parser.parse_complete(text).await.unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["location"], "Mars"); + assert_eq!(args["unit"], "celsius"); +} diff --git a/sgl-router/tests/tool_parser_qwen.rs b/sgl-router/tests/tool_parser_qwen.rs new file mode 100644 index 000000000..979c105b0 --- /dev/null +++ b/sgl-router/tests/tool_parser_qwen.rs @@ -0,0 +1,259 @@ +//! Qwen Parser Integration Tests +//! +//! Tests for the Qwen parser which handles ... format + +use serde_json::json; +use sglang_router_rs::tool_parser::{ParseState, QwenParser, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_qwen_single_tool() { + let parser = QwenParser::new(); + let input = r#" +{"name": "get_weather", "arguments": {"city": "Beijing", "units": "celsius"}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["city"], "Beijing"); + assert_eq!(args["units"], "celsius"); +} + +#[tokio::test] +async fn test_qwen_multiple_sequential_tools() { + let parser = QwenParser::new(); + let input = r#"Let me help you with that. + +{"name": "search", "arguments": {"query": "Qwen model"}} + + +{"name": "translate", "arguments": {"text": "Hello", "to": "zh"}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_qwen_pretty_printed_json() { + let parser = QwenParser::new(); + let input = r#" +{ + "name": "create_document", + "arguments": { + "title": "Test Document", + "content": "This is a test", + "metadata": { + "author": "Qwen", + "tags": ["test", "example"] + } + } +} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "create_document"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["metadata"]["author"], "Qwen"); + assert_eq!(args["metadata"]["tags"], json!(["test", "example"])); +} + +#[tokio::test] +async fn test_qwen_with_text_between() { + let parser = QwenParser::new(); + let input = r#"First, let me search for information. + +{"name": "search", "arguments": {"query": "test"}} + + +Now I'll translate something. + + +{"name": "translate", "arguments": {"text": "world", "to": "es"}} + +Done!"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "search"); + assert_eq!(result[1].function.name, "translate"); +} + +#[tokio::test] +async fn test_qwen_empty_arguments() { + let parser = QwenParser::new(); + let input = r#" +{"name": "get_time", "arguments": {}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_time"); +} + +#[tokio::test] +async fn test_qwen_with_newlines_in_strings() { + let parser = QwenParser::new(); + let input = r#" +{"name": "write_file", "arguments": {"content": "Line 1\nLine 2\nLine 3", "path": "/tmp/test.txt"}} +"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["content"], "Line 1\nLine 2\nLine 3"); +} + +#[tokio::test] +async fn test_qwen_format_detection() { + let parser = QwenParser::new(); + + assert!(parser.detect_format("")); + assert!(parser.detect_format("Some text \n{")); + assert!(!parser.detect_format("Just plain text")); + assert!(!parser.detect_format("{\"name\": \"test\"}")); // Plain JSON +} + +#[tokio::test] +async fn test_qwen_incomplete_tags() { + let parser = QwenParser::new(); + + // Missing closing tag + let input = r#" +{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); + + // Missing opening tag + let input = r#"{"name": "test", "arguments": {}} +"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); +} + +#[tokio::test] +async fn test_qwen_real_world_output() { + let parser = QwenParser::new(); + + // Actual output from Qwen model + let input = r#"I'll help you search for information and perform calculations. + + +{ + "name": "web_search", + "arguments": { + "query": "quantum computing breakthroughs 2024", + "language": "en", + "region": "us", + "safe_search": true + } +} + + +Let me also calculate something for you: + + +{ + "name": "calculator", + "arguments": { + "expression": "sqrt(144) + 3^2", + "precision": 2 + } +} + + +These tools will provide the information you need."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "web_search"); + assert_eq!(result[1].function.name, "calculator"); + + let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args0["query"], "quantum computing breakthroughs 2024"); + assert_eq!(args0["safe_search"], true); +} + +#[tokio::test] +async fn test_buffer_drain_optimization() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // First chunk - incomplete tool call + let chunk1 = "\n{\"name\": \"test1\", "; + let _result = parser.parse_incremental(chunk1, &mut state).await.unwrap(); + // Phase 2 simplified streaming might not handle partial JSON correctly + // The important thing is buffer accumulation works + assert!(!state.buffer.is_empty()); + + // Complete first tool and start second + let chunk2 = "\"arguments\": {}}\n\n{\"name\": \"test2\", "; + let result = parser.parse_incremental(chunk2, &mut state).await.unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test1"); + // After consuming the first tool, buffer should contain only the second tool start + assert!(state.buffer.starts_with("")); + assert!(state.buffer.contains("test2")); + } + _ => { + // Phase 2 simplified streaming might return Incomplete + // The important thing is the buffer is managed correctly + } + } + + // Complete the second tool + let chunk3 = "\"arguments\": {\"x\": 1}}\n"; + let result = parser.parse_incremental(chunk3, &mut state).await.unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test2"); + // Buffer should be empty after consuming all tools + assert!(state.buffer.is_empty() || !state.buffer.contains("")); + } + _ => { + // Phase 2 simplified streaming might handle this differently + } + } +} + +#[tokio::test] +async fn test_buffer_efficiency_with_multiple_tools() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // Send multiple complete tools at once + let input = r#" +{"name": "tool1", "arguments": {"a": 1}} + +{"name": "tool2", "arguments": {"b": 2}} + +{"name": "tool3", "arguments": {"c": 3}} +"#; + + // This should efficiently process tools using drain() without creating new strings + let result = parser.parse_incremental(input, &mut state).await.unwrap(); + + // In Phase 2, this will likely parse only the first tool + // The important thing is that drain() doesn't cause any issues + match result { + StreamResult::ToolComplete(tool) => { + assert!(["tool1", "tool2", "tool3"].contains(&tool.function.name.as_str())); + } + _ => { + // Simplified streaming might return Incomplete + } + } + + // Verify no memory issues or panics occurred with drain() + // Test passes if we reach this point without panic +} diff --git a/sgl-router/tests/tool_parser_registry.rs b/sgl-router/tests/tool_parser_registry.rs new file mode 100644 index 000000000..c98405eaf --- /dev/null +++ b/sgl-router/tests/tool_parser_registry.rs @@ -0,0 +1,194 @@ +//! Parser Registry Integration Tests +//! +//! Tests for model-to-parser mappings and registry functionality + +use sglang_router_rs::tool_parser::ParserRegistry; + +#[tokio::test] +async fn test_registry_has_all_parsers() { + let registry = ParserRegistry::new(); + let parsers = registry.list_parsers(); + + assert!(parsers.contains(&"json")); + assert!(parsers.contains(&"mistral")); + assert!(parsers.contains(&"qwen")); + assert!(parsers.contains(&"pythonic")); + assert!(parsers.contains(&"llama")); +} + +#[tokio::test] +async fn test_openai_models_use_json() { + let registry = ParserRegistry::new(); + + let models = vec!["gpt-4", "gpt-4-turbo", "gpt-3.5-turbo", "gpt-4o"]; + for model in models { + let parser = registry.get_parser(model).unwrap(); + let test_input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } +} + +#[tokio::test] +async fn test_anthropic_models_use_json() { + let registry = ParserRegistry::new(); + + let models = vec!["claude-3-opus", "claude-3-sonnet", "claude-2.1"]; + for model in models { + let parser = registry.get_parser(model).unwrap(); + let test_input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + } +} + +#[tokio::test] +async fn test_mistral_models() { + let registry = ParserRegistry::new(); + + let models = vec!["mistral-large", "mistral-medium", "mixtral-8x7b"]; + for model in models { + let parser = registry.get_parser(model).unwrap(); + let test_input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}}]"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } +} + +#[tokio::test] +async fn test_qwen_models() { + let registry = ParserRegistry::new(); + + let models = vec!["qwen2.5-72b", "Qwen2-7B", "qwen-max"]; + for model in models { + let parser = registry.get_parser(model).unwrap(); + let test_input = r#" +{"name": "test", "arguments": {}} +"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + } +} + +#[tokio::test] +async fn test_llama_model_variants() { + let registry = ParserRegistry::new(); + + // Llama 4 uses pythonic + let parser = registry.get_parser("llama-4-70b").unwrap(); + let test_input = r#"[get_weather(city="NYC")]"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "get_weather"); + + // Llama 3.2 uses python_tag + let parser = registry.get_parser("llama-3.2-8b").unwrap(); + let test_input = r#"<|python_tag|>{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + // Other Llama models use JSON + let parser = registry.get_parser("llama-2-70b").unwrap(); + let test_input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); +} + +#[tokio::test] +async fn test_deepseek_models() { + let registry = ParserRegistry::new(); + + // DeepSeek uses pythonic format (simplified, v3 would need custom parser) + let parser = registry.get_parser("deepseek-coder").unwrap(); + let test_input = r#"[function(arg="value")]"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "function"); +} + +#[tokio::test] +async fn test_unknown_model_fallback() { + let registry = ParserRegistry::new(); + + // Unknown models should fall back to JSON parser + let parser = registry.get_parser("unknown-model-xyz").unwrap(); + let test_input = r#"{"name": "fallback", "arguments": {}}"#; + let result = parser.parse_complete(test_input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "fallback"); +} + +#[tokio::test] +async fn test_pattern_specificity() { + let registry = ParserRegistry::new(); + + // Test that more specific patterns take precedence + // llama-4* should match before llama-* + let parser = registry.get_parser("llama-4-70b").unwrap(); + assert!(parser.detect_format(r#"[test_function(x=1)]"#)); // Pythonic format + + let parser = registry.get_parser("llama-3-70b").unwrap(); + assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#)); // JSON format +} + +#[tokio::test] +async fn test_real_world_model_outputs() { + let registry = ParserRegistry::new(); + + // Test with realistic outputs from different models + let test_cases = vec![ + ( + "gpt-4", + r#"I'll help you with that. + +{"name": "search_web", "arguments": {"query": "latest AI news", "max_results": 5}} + +Let me search for that information."#, + "search_web", + ), + ( + "mistral-large", + r#"Let me search for information about Rust. + +[TOOL_CALLS] [ + {"name": "search", "arguments": {"query": "Rust programming"}}, + {"name": "get_weather", "arguments": {"city": "San Francisco"}} +] + +I've initiated the search."#, + "search", + ), + ( + "qwen2.5", + r#"I'll check the weather for you. + + +{ + "name": "get_weather", + "arguments": { + "location": "Tokyo", + "units": "celsius" + } +} + + +The weather information has been requested."#, + "get_weather", + ), + ]; + + for (model, output, expected_name) in test_cases { + let parser = registry.get_parser(model).unwrap(); + let result = parser.parse_complete(output).await.unwrap(); + assert!(!result.is_empty(), "No tools parsed for model {}", model); + assert_eq!( + result[0].function.name, expected_name, + "Wrong function name for model {}", + model + ); + } +} diff --git a/sgl-router/tests/tool_parser_step3.rs b/sgl-router/tests/tool_parser_step3.rs new file mode 100644 index 000000000..6c1808b31 --- /dev/null +++ b/sgl-router/tests/tool_parser_step3.rs @@ -0,0 +1,245 @@ +//! Step3 Parser Integration Tests + +use sglang_router_rs::tool_parser::{ParseState, Step3Parser, StreamResult, ToolParser}; + +#[tokio::test] +async fn test_step3_complete_parsing() { + let parser = Step3Parser::new(); + + // Test single tool call + let input = r#"Let me help you. +<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +rust programming +10 +<｜tool_call_end｜> +<｜tool_calls_end｜> +Here are the results..."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); + + // Verify arguments + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "rust programming"); + assert_eq!(args["limit"], 10); +} + +#[tokio::test] +async fn test_step3_multiple_tools() { + let parser = Step3Parser::new(); + + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +Tokyo +<｜tool_call_end｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +tech +5 +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "get_weather"); + assert_eq!(result[1].function.name, "get_news"); +} + +#[tokio::test] +async fn test_step3_type_conversion() { + let parser = Step3Parser::new(); + + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +100 +2.5 +true +null +hello world +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["count"], 100); + assert_eq!(args["rate"], 2.5); + assert_eq!(args["active"], true); + assert_eq!(args["optional"], serde_json::Value::Null); + assert_eq!(args["text"], "hello world"); +} + +#[tokio::test] +async fn test_step3_streaming() { + let parser = Step3Parser::new(); + let mut state = ParseState::new(); + + // Simulate streaming chunks + let chunks = vec![ + "<｜tool_calls_begin｜>\n", + "<｜tool_call_begin｜>function", + "<｜tool_sep｜>", + "\n10", + "\n20", + "\n<｜tool_call_end｜>", + "\n<｜tool_calls_end｜>", + ]; + + let mut found_name = false; + let mut found_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + + match result { + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "calc"); + found_name = true; + } + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "calc"); + found_complete = true; + } + _ => {} + } + } + + assert!(found_name || found_complete); +} + +#[test] +fn test_step3_format_detection() { + let parser = Step3Parser::new(); + + // Should detect Step3 format + assert!(parser.detect_format("<｜tool_calls_begin｜>")); + assert!(parser.detect_format("text with <｜tool_calls_begin｜> marker")); + + // Should not detect other formats + assert!(!parser.detect_format("[TOOL_CALLS]")); + assert!(!parser.detect_format("")); + assert!(!parser.detect_format("plain text")); +} + +#[tokio::test] +async fn test_step3_nested_steptml() { + let parser = Step3Parser::new(); + + // Test with complex parameter values + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +{"nested": {"key": "value"}} +[1, 2, 3] +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "config"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["settings"].is_object()); + assert!(args["array"].is_array()); +} + +#[tokio::test] +async fn test_step3_python_literals() { + let parser = Step3Parser::new(); + + // Test Python-style literals + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +True +False +None +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["bool_true"], true); + assert_eq!(args["bool_false"], false); + assert_eq!(args["none_value"], serde_json::Value::Null); +} + +#[tokio::test] +async fn test_steptml_format() { + let parser = Step3Parser::new(); + + let input = r#"Text before. +<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +rust lang +10 +<｜tool_call_end｜> +<｜tool_calls_end｜>Text after."#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["query"], "rust lang"); + assert_eq!(args["limit"], 10); + // TODO: Verify normal text extraction +} + +#[tokio::test] +async fn test_json_parameter_values() { + let parser = Step3Parser::new(); + + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +{"nested": {"value": true}} +[1, 2, 3] +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert!(args["settings"].is_object()); + assert!(args["items"].is_array()); +} + +#[tokio::test] +async fn test_step3_parameter_with_angle_brackets() { + let parser = Step3Parser::new(); + + // Test parameter value containing < character + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +a < b && b > c +comparison test +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "compare"); + + // Verify the parameter value was parsed correctly + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["expression"], "a < b && b > c"); + assert_eq!(args["context"], "comparison test"); +} + +#[tokio::test] +async fn test_step3_empty_function_name() { + let parser = Step3Parser::new(); + + // Test empty function name + let input = r#"<｜tool_calls_begin｜> +<｜tool_call_begin｜>function<｜tool_sep｜> +value +<｜tool_call_end｜> +<｜tool_calls_end｜>"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0); // Should reject empty function name +} diff --git a/sgl-router/tests/tool_parser_streaming.rs b/sgl-router/tests/tool_parser_streaming.rs new file mode 100644 index 000000000..f0e9ddedb --- /dev/null +++ b/sgl-router/tests/tool_parser_streaming.rs @@ -0,0 +1,341 @@ +//! Streaming Parser Tests +//! +//! Tests for incremental/streaming parsing capabilities across all parsers + +use sglang_router_rs::tool_parser::{ + JsonParser, LlamaParser, MistralParser, ParseState, PythonicParser, QwenParser, StreamResult, + ToolParser, +}; + +#[tokio::test] +async fn test_json_streaming_simple() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Phase 2 note: This test sends the full JSON at once in the last chunk + // In real streaming, chunks would be smaller + let full_json = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#; + + let result = parser + .parse_incremental(full_json, &mut state) + .await + .unwrap(); + + // With complete JSON sent at once, we should get ToolComplete + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + } + _ => { + panic!("Expected ToolComplete for complete JSON input"); + } + } +} + +#[tokio::test] +async fn test_json_streaming_array() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Stream a JSON array of tools + let chunks = vec![ + r#"["#, + r#"{"name": "tool1", "#, + r#""arguments": {}}, "#, + r#"{"name": "tool2", "#, + r#""arguments": {"x": 1"#, + r#"}}]"#, + ]; + + let mut tool_count = 0; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let StreamResult::ToolComplete(_) = result { + tool_count += 1; + } + } + + // Current implementation may handle this differently + // We're mainly testing that it doesn't crash + assert!(tool_count <= 2, "Should parse at most 2 tools"); +} + +#[tokio::test] +async fn test_mistral_streaming() { + let parser = MistralParser::new(); + let mut state = ParseState::new(); + + let chunks = vec![ + r#"Here is the result: "#, + r#"[TOOL_CALLS] ["#, + r#"{"name": "#, + r#""search", "#, + r#""arguments": "#, + r#"{"query": "#, + r#""rust lang""#, + r#"}}]"#, + ]; + + let mut got_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "search"); + got_complete = true; + } + } + + assert!(got_complete, "Should have completed parsing"); +} + +#[tokio::test] +async fn test_pythonic_streaming() { + let parser = PythonicParser::new(); + let mut state = ParseState::new(); + + // Send complete pythonic format at once + let full_input = r#"[get_weather(city="London", units="celsius")]"#; + + let result = parser + .parse_incremental(full_input, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "get_weather"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert_eq!(args["city"], "London"); + } + _ => { + panic!("Expected ToolComplete for complete pythonic input"); + } + } +} + +#[tokio::test] +async fn test_llama_streaming_with_python_tag() { + let parser = LlamaParser::new(); + let mut state = ParseState::new(); + + let chunks = vec![ + r#"Let me help. "#, + r#"<|python"#, + r#"_tag|>"#, + r#"{"name": "#, + r#""calculate", "#, + r#""arguments": "#, + r#"{"x": 10}"#, + r#"}"#, + ]; + + let mut got_complete = false; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + if let StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "calculate"); + got_complete = true; + } + } + + assert!(got_complete, "Should have completed parsing"); +} + +#[tokio::test] +async fn test_qwen_streaming() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // Send complete Qwen format at once (with exact format expected by parser) + // Note: Parser expects newline after both tags + let full_input = "\n{\"name\": \"translate\", \"arguments\": {\"text\": \"hello\", \"to\": \"zh\"}}\n"; + + let result = parser + .parse_incremental(full_input, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "translate"); + } + other => { + panic!( + "Expected ToolComplete for complete Qwen input, got: {:?}", + other + ); + } + } +} + +#[tokio::test] +async fn test_streaming_incomplete_stays_incomplete() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Send truly incomplete JSON that can't be auto-completed + let chunks = vec![r#"{"na"#, r#"me": "#]; + + for chunk in chunks { + let result = parser.parse_incremental(chunk, &mut state).await.unwrap(); + // Should return Incomplete for partial JSON that can't be auto-completed + assert!( + matches!(result, StreamResult::Incomplete), + "Should return Incomplete for partial JSON, got: {:?}", + result + ); + } + + // Buffer should contain the accumulated incomplete JSON + assert!(!state.buffer.is_empty()); +} + +#[tokio::test] +async fn test_streaming_with_text_before_tool() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // For streaming, the parser expects clean JSON + // Mixed text extraction only works in parse_complete, not parse_incremental + let full_input = r#"{"name": "test", "arguments": {}}"#; + + let result = parser + .parse_incremental(full_input, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test"); + } + other => { + panic!("Expected ToolComplete, got: {:?}", other); + } + } +} + +#[tokio::test] +async fn test_streaming_buffer_accumulation() { + let parser = JsonParser::new(); + + // Test: Complete JSON should clear buffer after parsing + let mut state = ParseState::new(); + + // Send partial JSON that can't be interpreted as complete + let result1 = parser + .parse_incremental(r#"{"na"#, &mut state) + .await + .unwrap(); + + assert!(matches!(result1, StreamResult::Incomplete)); + assert!( + !state.buffer.is_empty(), + "Buffer should accumulate incomplete JSON" + ); + + // Send rest of JSON + let result2 = parser + .parse_incremental(r#"me": "test", "arguments": {}}"#, &mut state) + .await + .unwrap(); + + match result2 { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "test"); + assert!( + state.buffer.is_empty(), + "Buffer should be cleared after complete parse" + ); + } + _ => panic!( + "Expected ToolComplete for complete JSON, got: {:?}", + result2 + ), + } +} + +#[tokio::test] +async fn test_streaming_multiple_tools_sequential() { + let parser = QwenParser::new(); + let mut state = ParseState::new(); + + // Send complete Qwen format with newlines + let full_input = r#" +{"name": "tool1", "arguments": {}} +"#; + + let result = parser + .parse_incremental(full_input, &mut state) + .await + .unwrap(); + + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "tool1"); + } + _ => { + panic!("Expected ToolComplete for first tool"); + } + } +} + +#[tokio::test] +async fn test_streaming_reset_after_error() { + let parser = JsonParser::new(); + + // First attempt with invalid JSON + let mut state1 = ParseState::new(); + let _ = parser + .parse_incremental(r#"{"name": invalid}"#, &mut state1) + .await; + + // Second attempt with valid JSON should work with fresh state + let mut state2 = ParseState::new(); + let result = parser + .parse_incremental(r#"{"name": "test", "arguments": {}}"#, &mut state2) + .await + .unwrap(); + + if let StreamResult::ToolComplete(tool) = result { + assert_eq!(tool.function.name, "test"); + } +} + +#[tokio::test] +async fn test_streaming_with_unicode_chunks() { + let parser = JsonParser::new(); + let mut state = ParseState::new(); + + // Send complete JSON with unicode + let full_input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍"}}"#; + + let result = parser + .parse_incremental(full_input, &mut state) + .await + .unwrap(); + + // Phase 2 may return partial results even with complete JSON + // The important thing is that unicode is handled without crashes + match result { + StreamResult::ToolComplete(tool) => { + assert_eq!(tool.function.name, "translate"); + let args: serde_json::Value = serde_json::from_str(&tool.function.arguments).unwrap(); + assert!(args["text"].as_str().unwrap().contains("世界")); + } + StreamResult::ToolName { name, .. } => { + assert_eq!(name, "translate"); + // Phase 2 partial streaming behavior - acceptable + } + StreamResult::ToolArguments { arguments, .. } => { + // Verify unicode was preserved + let args: serde_json::Value = serde_json::from_str(&arguments).unwrap(); + assert!(args["text"].as_str().unwrap().contains("世界")); + } + other => { + panic!("Unexpected result: {:?}", other); + } + } +} diff --git a/sgl-router/tests/tool_parser_wrapper_tokens.rs b/sgl-router/tests/tool_parser_wrapper_tokens.rs new file mode 100644 index 000000000..d2cc6b2f7 --- /dev/null +++ b/sgl-router/tests/tool_parser_wrapper_tokens.rs @@ -0,0 +1,247 @@ +//! Wrapper Token Tests +//! +//! Tests for JSON parser with custom wrapper tokens + +use sglang_router_rs::tool_parser::{JsonParser, TokenConfig, ToolParser}; + +#[tokio::test] +async fn test_json_with_xml_style_wrapper() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + let input = + r#"Some text before {"name": "test", "arguments": {"x": 1}} and after"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["x"], 1); +} + +#[tokio::test] +async fn test_json_with_multiple_wrapper_pairs() { + // Test with multiple start/end token pairs + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string(), "<>".to_string()], + end_tokens: vec!["".to_string(), "<>".to_string()], + separator: ", ".to_string(), + }); + + // Test first pair + let input1 = r#"{"name": "tool1", "arguments": {}}"#; + let result1 = parser.parse_complete(input1).await.unwrap(); + assert_eq!(result1.len(), 1); + assert_eq!(result1[0].function.name, "tool1"); + + // Test second pair + let input2 = r#"<>{"name": "tool2", "arguments": {}}<>"#; + let result2 = parser.parse_complete(input2).await.unwrap(); + assert_eq!(result2.len(), 1); + assert_eq!(result2[0].function.name, "tool2"); +} + +#[tokio::test] +async fn test_json_with_only_start_token() { + // Test when only start token is provided (no end token) + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec![">>>FUNCTION:".to_string()], + end_tokens: vec!["".to_string()], // Empty end token + separator: ", ".to_string(), + }); + + let input = r#"Some preamble >>>FUNCTION:{"name": "execute", "arguments": {"cmd": "ls"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "execute"); +} + +#[tokio::test] +async fn test_json_with_custom_separator() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["[FUNC]".to_string()], + end_tokens: vec!["[/FUNC]".to_string()], + separator: " | ".to_string(), // Custom separator + }); + + // Though we're not testing multiple tools here, the separator is configured + let input = r#"[FUNC]{"name": "test", "arguments": {}}[/FUNC]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} + +#[tokio::test] +async fn test_json_with_nested_wrapper_tokens_in_content() { + // Known limitation: When wrapper tokens appear inside JSON strings, + // the simple regex-based extraction may fail. This would require + // a more sophisticated parser that understands JSON string escaping. + + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + let input = + r#"{"name": "echo", "arguments": {"text": "Use and tags"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + + // This is a known limitation - the parser may fail when end tokens appear in content + // For now, we accept this behavior + if result.is_empty() { + // Parser failed due to nested tokens - this is expected + assert_eq!( + result.len(), + 0, + "Known limitation: nested wrapper tokens in content" + ); + } else { + // If it does parse, verify it's correct + assert_eq!(result[0].function.name, "echo"); + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["text"], "Use and tags"); + } +} + +#[tokio::test] +async fn test_json_extraction_without_wrapper_tokens() { + // Default parser without wrapper tokens should extract JSON from text + let parser = JsonParser::new(); + + let input = r#" + Here is some text before the JSON. + {"name": "search", "arguments": {"query": "test"}} + And here is some text after. + "#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "search"); +} + +#[tokio::test] +async fn test_json_with_multiline_wrapper_content() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["```json\n".to_string()], + end_tokens: vec!["\n```".to_string()], + separator: ", ".to_string(), + }); + + let input = r#"Here's the function call: +```json +{ + "name": "format_code", + "arguments": { + "language": "rust", + "code": "fn main() {}" + } +} +``` +Done!"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "format_code"); +} + +#[tokio::test] +async fn test_json_with_special_chars_in_tokens() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["{{FUNC[[".to_string()], + end_tokens: vec!["]]FUNC}}".to_string()], + separator: ", ".to_string(), + }); + + let input = r#"{{FUNC[[{"name": "test", "arguments": {"special": "[]{}"}}]]FUNC}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); + + let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap(); + assert_eq!(args["special"], "[]{}"); +} + +#[tokio::test] +async fn test_json_multiple_tools_with_wrapper() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + // Multiple wrapped JSON objects + let input = r#" + {"name": "tool1", "arguments": {}} + Some text between. + {"name": "tool2", "arguments": {"x": 1}} + "#; + + // Current implementation might handle this as separate calls + // Let's test that at least the first one is parsed + let result = parser.parse_complete(input).await.unwrap(); + assert!(!result.is_empty(), "Should parse at least one tool"); + assert_eq!(result[0].function.name, "tool1"); +} + +#[tokio::test] +async fn test_json_wrapper_with_array() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + let input = r#"[ + {"name": "func1", "arguments": {}}, + {"name": "func2", "arguments": {"param": "value"}} + ]"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].function.name, "func1"); + assert_eq!(result[1].function.name, "func2"); +} + +#[tokio::test] +async fn test_json_incomplete_wrapper_tokens() { + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec!["".to_string()], + end_tokens: vec!["".to_string()], + separator: ", ".to_string(), + }); + + // Missing end token + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should not parse without closing token"); + + // Missing start token + let input = r#"{"name": "test", "arguments": {}}"#; + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 0, "Should not parse without opening token"); +} + +#[tokio::test] +async fn test_json_empty_wrapper_tokens() { + // Test with empty wrapper tokens (should behave like default) + let parser = JsonParser::with_config(TokenConfig { + start_tokens: vec![], + end_tokens: vec![], + separator: ", ".to_string(), + }); + + let input = r#"{"name": "test", "arguments": {"key": "value"}}"#; + + let result = parser.parse_complete(input).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].function.name, "test"); +} diff --git a/test/README.md b/test/README.md new file mode 100644 index 000000000..1854ec955 --- /dev/null +++ b/test/README.md @@ -0,0 +1,41 @@ +# Run Unit Tests + +SGLang uses the built-in library [unittest](https://docs.python.org/3/library/unittest.html) as the testing framework. + +## Test Backend Runtime +```bash +cd sglang/test/srt + +# Run a single file +python3 test_srt_endpoint.py + +# Run a single test +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode + +# Run a suite with multiple files +python3 run_suite.py --suite per-commit +``` + +## Test Frontend Language +```bash +cd sglang/test/lang + +# Run a single file +python3 test_srt_backend.py +``` + +## Adding or Updating Tests in CI + +- Create new test files under `test/srt` or `test/lang` depending on the type of test. +- Ensure they are referenced in the respective `run_suite.py` (e.g., `test/srt/run_suite.py`) so they’re picked up in CI. For most small test cases, they can be added to the `per-commit` suite. Sort the test cases alphabetically. +- The CI will run the `per-commit` and `nightly` automatically. If you need special setup or custom test groups, you may modify the workflows in [`.github/workflows/`](https://github.com/sgl-project/sglang/tree/main/.github/workflows). + + +## Writing Elegant Test Cases + +- Examine existing tests in [sglang/test](https://github.com/sgl-project/sglang/tree/main/test) for practical examples. +- Keep each test function focused on a single scenario or piece of functionality. +- Give tests descriptive names reflecting their purpose. +- Use robust assertions (e.g., assert, unittest methods) to validate outcomes. +- Clean up resources to avoid side effects and preserve test independence. +- Reduce the test time by using smaller models and reusing the server for multiple test cases. diff --git a/test/lang/run_suite.py b/test/lang/run_suite.py new file mode 100644 index 000000000..04efba51f --- /dev/null +++ b/test/lang/run_suite.py @@ -0,0 +1,38 @@ +import argparse +import glob + +from sglang.test.test_utils import TestFile, run_unittest_files + +suites = { + "per-commit": [ + TestFile("test_srt_backend.py"), + # Skip this due to some OPENAI_API_KEY issues + # "test_openai_backend.py", + ], +} + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1000, + help="The time limit for running one file in seconds.", + ) + arg_parser.add_argument( + "--suite", + type=str, + default=list(suites.keys())[0], + choices=list(suites.keys()) + ["all"], + help="The suite to run", + ) + args = arg_parser.parse_args() + + if args.suite == "all": + files = glob.glob("**/test_*.py", recursive=True) + else: + files = suites[args.suite] + + exit_code = run_unittest_files(files, args.timeout_per_file) + exit(exit_code) diff --git a/test/lang/test_anthropic_backend.py b/test/lang/test_anthropic_backend.py new file mode 100644 index 000000000..dc8f0b17e --- /dev/null +++ b/test/lang/test_anthropic_backend.py @@ -0,0 +1,25 @@ +import json +import unittest + +from sglang import Anthropic, set_default_backend +from sglang.test.test_programs import test_mt_bench, test_stream +from sglang.test.test_utils import CustomTestCase + + +class TestAnthropicBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = Anthropic("claude-3-haiku-20240307") + set_default_backend(cls.backend) + + def test_mt_bench(self): + test_mt_bench() + + def test_stream(self): + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_bind_cache.py b/test/lang/test_bind_cache.py new file mode 100644 index 000000000..d5beefae6 --- /dev/null +++ b/test/lang/test_bind_cache.py @@ -0,0 +1,51 @@ +import unittest + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestBind(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = sgl.Runtime(model_path=DEFAULT_MODEL_NAME_FOR_TEST) + sgl.set_default_backend(cls.backend) + + @classmethod + def tearDownClass(cls): + cls.backend.shutdown() + + def test_bind(self): + @sgl.function + def few_shot_qa(s, prompt, question): + s += prompt + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + few_shot_qa_2 = few_shot_qa.bind( + prompt="The following are questions with answers.\n\n" + ) + + tracer = few_shot_qa_2.trace() + print(tracer.last_node.print_graph_dfs() + "\n") + + def test_cache(self): + @sgl.function + def few_shot_qa(s, prompt, question): + s += prompt + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + few_shot_qa_2 = few_shot_qa.bind( + prompt="Answer the following questions as if you were a 5-year-old kid.\n\n" + ) + few_shot_qa_2.cache(self.backend) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_choices.py b/test/lang/test_choices.py new file mode 100644 index 000000000..89e7ca1c7 --- /dev/null +++ b/test/lang/test_choices.py @@ -0,0 +1,91 @@ +import unittest + +import numpy as np + +from sglang.lang.choices import ( + greedy_token_selection, + token_length_normalized, + unconditional_likelihood_normalized, +) +from sglang.test.test_utils import CustomTestCase + +MOCK_CHOICES_INPUT_DATA = { + "choices": [ + "organ", # ["organ"] + "organism", # ["organ", "ism"] + "antidisestablishmentarianism", # ["ant", "id", "is", "est", "ablish", "ment", "arian", "ism"] + ], + "normalized_prompt_logprobs": [-0.1, -0.2, -0.05], + "input_token_logprobs": [ + [[-0.1, 1, None]], + [[-0.1, 1, None], [-0.3, 2, None]], + [ + [-0.4, 3, None], + [-0.25, 4, None], + [-0.1, 5, None], + [-0.01, 6, None], + [-0.01, 7, None], + [-0.01, 8, None], + [-0.01, 9, None], + [-0.01, 2, None], + ], + ], + "output_token_logprobs": [ + [[-0.1, 10, None]], + [[-0.1, 10, None]], + [[-0.1, 10, None]], + ], + "unconditional_token_logprobs": [ + [[None, 1, None]], + [[None, 1, None], [-1.4, 2, None]], + [ + [None, 3, None], + [-0.25, 4, None], + [-0.1, 5, None], + [-0.01, 6, None], + [-0.01, 7, None], + [-0.01, 8, None], + [-0.01, 9, None], + [-0.01, 2, None], + ], + ], +} + + +class TestChoices(CustomTestCase): + + def test_token_length_normalized(self): + """Confirm 'antidisestablishmentarianism' is selected due to high confidences for + its later tokens resulting in highest token length normalized prompt logprob.""" + decision = token_length_normalized(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "antidisestablishmentarianism" + + def test_greedy_token_selection(self): + """Confirm 'organ' is selected due it having the joint highest initial token + logprob, and a higher average logprob than organism's second token.""" + decision = greedy_token_selection(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "organ" + assert np.allclose( + decision.meta_info["greedy_logprob_matrix"], + [ + [-0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1, -0.1], + [-0.1, -0.3, -0.2, -0.2, -0.2, -0.2, -0.2, -0.2], + [-0.4, -0.25, -0.1, -0.01, -0.01, -0.01, -0.01, -0.01], + ], + atol=0.01, + ) + + def test_unconditional_likelihood_normalized(self): + """Confirm 'organism' is selected due to it having the highest average token logprob + once normalized by the unconditional token logprobs.""" + decision = unconditional_likelihood_normalized(**MOCK_CHOICES_INPUT_DATA) + assert decision.decision == "organism" + assert np.allclose( + decision.meta_info["normalized_unconditional_prompt_logprobs"], + [-0.1, 0.5, -0.05], + atol=0.01, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_litellm_backend.py b/test/lang/test_litellm_backend.py new file mode 100644 index 000000000..74c3a187a --- /dev/null +++ b/test/lang/test_litellm_backend.py @@ -0,0 +1,25 @@ +import json +import unittest + +from sglang import LiteLLM, set_default_backend +from sglang.test.test_programs import test_mt_bench, test_stream +from sglang.test.test_utils import CustomTestCase + + +class TestAnthropicBackend(CustomTestCase): + chat_backend = None + + @classmethod + def setUpClass(cls): + cls.chat_backend = LiteLLM("gpt-3.5-turbo") + set_default_backend(cls.chat_backend) + + def test_mt_bench(self): + test_mt_bench() + + def test_stream(self): + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_openai_backend.py b/test/lang/test_openai_backend.py new file mode 100644 index 000000000..dbef7d450 --- /dev/null +++ b/test/lang/test_openai_backend.py @@ -0,0 +1,92 @@ +import unittest + +from sglang import OpenAI, set_default_backend +from sglang.test.test_programs import ( + test_chat_completion_speculative, + test_completion_speculative, + test_decode_int, + test_decode_json, + test_expert_answer, + test_few_shot_qa, + test_image_qa, + test_mt_bench, + test_parallel_decoding, + test_parallel_encoding, + test_react, + test_select, + test_stream, + test_tool_use, +) +from sglang.test.test_utils import CustomTestCase + + +class TestOpenAIBackend(CustomTestCase): + instruct_backend = None + chat_backend = None + chat_vision_backend = None + + @classmethod + def setUpClass(cls): + cls.instruct_backend = OpenAI("gpt-3.5-turbo-instruct") + cls.chat_backend = OpenAI("gpt-3.5-turbo") + cls.chat_vision_backend = OpenAI("gpt-4-turbo") + + def test_few_shot_qa(self): + set_default_backend(self.instruct_backend) + test_few_shot_qa() + + def test_mt_bench(self): + set_default_backend(self.chat_backend) + test_mt_bench() + + def test_select(self): + set_default_backend(self.instruct_backend) + test_select(check_answer=True) + + def test_decode_int(self): + set_default_backend(self.instruct_backend) + test_decode_int() + + def test_decode_json(self): + set_default_backend(self.instruct_backend) + test_decode_json() + + def test_expert_answer(self): + set_default_backend(self.instruct_backend) + test_expert_answer() + + def test_tool_use(self): + set_default_backend(self.instruct_backend) + test_tool_use() + + def test_react(self): + set_default_backend(self.instruct_backend) + test_react() + + def test_parallel_decoding(self): + set_default_backend(self.instruct_backend) + test_parallel_decoding() + + def test_parallel_encoding(self): + set_default_backend(self.instruct_backend) + test_parallel_encoding() + + def test_image_qa(self): + set_default_backend(self.chat_vision_backend) + test_image_qa() + + def test_stream(self): + set_default_backend(self.instruct_backend) + test_stream() + + def test_completion_speculative(self): + set_default_backend(self.instruct_backend) + test_completion_speculative() + + def test_chat_completion_speculative(self): + set_default_backend(self.chat_backend) + test_chat_completion_speculative() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_separate_reasoning.py b/test/lang/test_separate_reasoning.py new file mode 100644 index 000000000..1709cecfd --- /dev/null +++ b/test/lang/test_separate_reasoning.py @@ -0,0 +1,68 @@ +""" +Tests for the separate_reasoning functionality in sglang. + +Usage: +python3 -m unittest test/lang/test_separate_reasoning.py +""" + +import unittest + +from sglang import assistant, gen, separate_reasoning, user +from sglang.lang.ir import SglExprList, SglSeparateReasoning +from sglang.test.test_utils import CustomTestCase + + +class TestSeparateReasoning(CustomTestCase): + def test_separate_reasoning_creation(self): + """Test that SglSeparateReasoning objects are created correctly.""" + # Test with valid model type and gen expression + test_gen = gen("test") + expr = separate_reasoning(test_gen, model_type="deepseek-r1") + self.assertIsInstance(expr, SglExprList) + self.assertEqual(len(expr.expr_list), 2) + self.assertEqual(expr.expr_list[0], test_gen) + reasoning_expr = expr.expr_list[1] + self.assertIsInstance(reasoning_expr, SglSeparateReasoning) + self.assertEqual(reasoning_expr.model_type, "deepseek-r1") + self.assertEqual(reasoning_expr.name, "test_reasoning_content") + + # Test with another valid model type + expr = separate_reasoning(test_gen, model_type="qwen3") + self.assertIsInstance(expr, SglExprList) + self.assertEqual(expr.expr_list[1].model_type, "qwen3") + + def test_separate_reasoning_name_processing(self): + """Test that separate_reasoning correctly processes names.""" + test_gen = gen("test_var") + expr = separate_reasoning(test_gen, model_type="deepseek-r1") + reasoning_expr = expr.expr_list[1] + self.assertEqual(reasoning_expr.name, "test_var_reasoning_content") + + # Test the process_name_for_reasoning method + self.assertEqual( + reasoning_expr.process_name_for_reasoning("another_var"), + "another_var_reasoning_content", + ) + + def test_separate_reasoning_repr(self): + """Test the string representation of SglSeparateReasoning.""" + test_gen = gen("test_var") + expr = separate_reasoning(test_gen, model_type="deepseek-r1") + reasoning_expr = expr.expr_list[1] + self.assertEqual( + repr(reasoning_expr), + "SeparateReasoning(model_type=deepseek-r1, name=test_var_reasoning_content)", + ) + + def test_separate_reasoning_with_invalid_model_type(self): + """Test that separate_reasoning accepts any model type during creation.""" + # Create with invalid model type + test_gen = gen("test") + expr = separate_reasoning(test_gen, model_type="invalid-model") + self.assertIsInstance(expr, SglExprList) + self.assertEqual(expr.expr_list[1].model_type, "invalid-model") + # The actual validation happens in the ReasoningParser constructor + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_separate_reasoning_execution.py b/test/lang/test_separate_reasoning_execution.py new file mode 100644 index 000000000..481488f6a --- /dev/null +++ b/test/lang/test_separate_reasoning_execution.py @@ -0,0 +1,195 @@ +""" +Tests for the execution of separate_reasoning functionality in sglang. + +Usage: +python3 -m unittest test/lang/test_separate_reasoning_execution.py +""" + +import threading +import time +import unittest +from unittest.mock import MagicMock, patch + +from sglang import assistant, gen, separate_reasoning, user +from sglang.lang.interpreter import StreamExecutor +from sglang.lang.ir import SglGen, SglSeparateReasoning +from sglang.test.test_utils import CustomTestCase + + +# Helper function to create events that won't block program exit +def create_daemon_event(): + event = threading.Event() + return event + + +class MockReasoningParser: + def __init__(self, model_type): + self.model_type = model_type + self.parse_non_stream_called = False + self.parse_stream_chunk_called = False + + def parse_non_stream(self, full_text): + self.parse_non_stream_called = True + # Simulate parsing by adding a prefix to indicate reasoning + reasoning = f"[REASONING from {self.model_type}]: {full_text}" + normal_text = f"[NORMAL from {self.model_type}]: {full_text}" + return reasoning, normal_text + + def parse_stream_chunk(self, chunk_text): + self.parse_stream_chunk_called = True + # Simulate parsing by adding a prefix to indicate reasoning + reasoning = f"[REASONING from {self.model_type}]: {chunk_text}" + normal_text = f"[NORMAL from {self.model_type}]: {chunk_text}" + return reasoning, normal_text + + +class TestSeparateReasoningExecution(CustomTestCase): + def setUp(self): + """Set up for the test.""" + super().setUp() + # Store any events created during the test + self.events = [] + + def tearDown(self): + """Clean up any threads that might have been created during the test.""" + super().tearDown() + + # Set all events to ensure any waiting threads are released + for event in self.events: + event.set() + + def tearDown(self): + super().tearDown() + # wake up all threads + for ev in self.events: + ev.set() + + @patch("sglang.srt.parser.reasoning_parser.ReasoningParser") + def test_execute_separate_reasoning(self, mock_parser_class): + """Test that _execute_separate_reasoning correctly calls the ReasoningParser.""" + # Setup mock parser + mock_parser = MockReasoningParser("deepseek-r1") + mock_parser_class.return_value = mock_parser + + # Create a mock backend to avoid AttributeError in __del__ + mock_backend = MagicMock() + + # Create a StreamExecutor with necessary setup + executor = StreamExecutor( + backend=mock_backend, + arguments={}, + default_sampling_para={}, + chat_template={ + "role_map": {"user": "user", "assistant": "assistant"} + }, # Simple chat template + stream=False, + use_thread=False, + ) + + # Set up the executor with a variable and its value + var_name = "test_var" + reasoning_name = f"{var_name}_reasoning_content" + var_value = "Test content" + executor.variables = {var_name: var_value} + + # Create events and track them for cleanup + var_event = create_daemon_event() + reasoning_event = create_daemon_event() + self.events.extend([var_event, reasoning_event]) + + executor.variable_event = {var_name: var_event, reasoning_name: reasoning_event} + executor.variable_event[var_name].set() # Mark as ready + + # Set up the current role + executor.cur_role = "assistant" + executor.cur_role_begin_pos = 0 + executor.text_ = var_value + + # Create a gen expression and a separate_reasoning expression + gen_expr = SglGen(var_name) + expr = SglSeparateReasoning("deepseek-r1", expr=gen_expr) + + # Execute separate_reasoning + executor._execute_separate_reasoning(expr) + + # Verify that the parser was created with the correct model type + mock_parser_class.assert_called_once_with("deepseek-r1") + + # Verify that parse_non_stream was called + self.assertTrue(mock_parser.parse_non_stream_called) + + # Verify that the variables were updated correctly + reasoning_name = f"{var_name}_reasoning_content" + self.assertIn(reasoning_name, executor.variables) + self.assertEqual( + executor.variables[reasoning_name], + f"[REASONING from deepseek-r1]: {var_value}", + ) + self.assertEqual( + executor.variables[var_name], f"[NORMAL from deepseek-r1]: {var_value}" + ) + + # Verify that the variable event was set + self.assertIn(reasoning_name, executor.variable_event) + self.assertTrue(executor.variable_event[reasoning_name].is_set()) + + # Verify that the text was updated + self.assertEqual(executor.text_, f"[NORMAL from deepseek-r1]: {var_value}") + + @patch("sglang.srt.parser.reasoning_parser.ReasoningParser") + def test_reasoning_parser_integration(self, mock_parser_class): + """Test the integration between separate_reasoning and ReasoningParser.""" + # Setup mock parsers for different model types + deepseek_parser = MockReasoningParser("deepseek-r1") + qwen_parser = MockReasoningParser("qwen3") + + # Configure the mock to return different parsers based on model type + def get_parser(model_type): + if model_type == "deepseek-r1": + return deepseek_parser + elif model_type == "qwen3": + return qwen_parser + else: + raise ValueError(f"Unsupported model type: {model_type}") + + mock_parser_class.side_effect = get_parser + + # Test with DeepSeek-R1 model + test_text = "This is a test" + reasoning, normal_text = deepseek_parser.parse_non_stream(test_text) + + self.assertEqual(reasoning, f"[REASONING from deepseek-r1]: {test_text}") + self.assertEqual(normal_text, f"[NORMAL from deepseek-r1]: {test_text}") + + # Test with Qwen3 model + reasoning, normal_text = qwen_parser.parse_non_stream(test_text) + + self.assertEqual(reasoning, f"[REASONING from qwen3]: {test_text}") + self.assertEqual(normal_text, f"[NORMAL from qwen3]: {test_text}") + + @patch("sglang.srt.parser.reasoning_parser.ReasoningParser") + def test_reasoning_parser_invalid_model(self, mock_parser_class): + """Test that ReasoningParser raises an error for invalid model types.""" + + # Configure the mock to raise an error for invalid model types + def get_parser(model_type): + if model_type in ["deepseek-r1", "qwen3"]: + return MockReasoningParser(model_type) + elif model_type is None: + raise ValueError("Model type must be specified") + else: + raise ValueError(f"Unsupported model type: {model_type}") + + mock_parser_class.side_effect = get_parser + + with self.assertRaises(ValueError) as context: + mock_parser_class("invalid-model") + self.assertIn("Unsupported model type", str(context.exception)) + + with self.assertRaises(ValueError) as context: + mock_parser_class(None) + self.assertIn("Model type must be specified", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_srt_backend.py b/test/lang/test_srt_backend.py new file mode 100644 index 000000000..0e05eb906 --- /dev/null +++ b/test/lang/test_srt_backend.py @@ -0,0 +1,86 @@ +""" +Usage: +python3 -m unittest test_srt_backend.TestSRTBackend.test_gen_min_new_tokens +python3 -m unittest test_srt_backend.TestSRTBackend.test_hellaswag_select +""" + +import unittest + +import sglang as sgl +from sglang.test.test_programs import ( + test_decode_int, + test_decode_json_regex, + test_dtype_gen, + test_expert_answer, + test_few_shot_qa, + test_gen_min_new_tokens, + test_hellaswag_select, + test_mt_bench, + test_parallel_decoding, + test_regex, + test_select, + test_stream, + test_tool_use, +) +from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestSRTBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = sgl.Runtime( + model_path=DEFAULT_MODEL_NAME_FOR_TEST, cuda_graph_max_bs=4 + ) + sgl.set_default_backend(cls.backend) + + @classmethod + def tearDownClass(cls): + cls.backend.shutdown() + + def test_few_shot_qa(self): + test_few_shot_qa() + + def test_mt_bench(self): + test_mt_bench() + + def test_select(self): + test_select(check_answer=False) + + def test_decode_int(self): + test_decode_int() + + def test_decode_json_regex(self): + test_decode_json_regex() + + def test_expert_answer(self): + test_expert_answer() + + def test_tool_use(self): + test_tool_use() + + def test_parallel_decoding(self): + test_parallel_decoding() + + def test_stream(self): + test_stream() + + def test_regex(self): + test_regex() + + def test_dtype_gen(self): + test_dtype_gen() + + def test_hellaswag_select(self): + # Run twice to capture more bugs + for _ in range(2): + accuracy, latency = test_hellaswag_select() + self.assertGreater(accuracy, 0.60) + + def test_gen_min_new_tokens(self): + test_gen_min_new_tokens() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_tracing.py b/test/lang/test_tracing.py new file mode 100644 index 000000000..3f02ac52b --- /dev/null +++ b/test/lang/test_tracing.py @@ -0,0 +1,129 @@ +import unittest + +import sglang as sgl +from sglang.lang.backend.base_backend import BaseBackend +from sglang.lang.chat_template import get_chat_template +from sglang.test.test_utils import CustomTestCase + + +class TestTracing(CustomTestCase): + def test_few_shot_qa(self): + @sgl.function + def few_shot_qa(s, question): + s += "The following are questions with answers.\n\n" + s += "Q: What is the capital of France?\n" + s += "A: Paris\n" + s += "Q: " + question + "\n" + s += "A:" + sgl.gen("answer", stop="\n") + + tracer = few_shot_qa.trace() + # print(tracer.last_node.print_graph_dfs() + "\n") + + def test_select(self): + @sgl.function + def capital(s): + s += "The capital of France is" + s += sgl.select("capital", ["Paris. ", "London. "]) + s += "It is a city" + sgl.gen("description", stop=".") + + tracer = capital.trace() + # print(tracer.last_node.print_graph_dfs() + "\n") + + def test_raise_warning(self): + @sgl.function + def wrong(s, question): + s += f"I want to ask {question}" + + try: + tracer = wrong.trace() + raised = False + except TypeError: + raised = True + + assert raised + + def test_multi_function(self): + @sgl.function + def expand(s, tip): + s += ( + "Please expand the following tip into a detailed paragraph:" + + tip + + "\n" + ) + s += sgl.gen("detailed_tip") + + @sgl.function + def tip_suggestion(s, topic): + s += "Here are 2 tips for " + topic + ".\n" + + s += "1." + sgl.gen("tip_1", stop=["\n", ":", "."]) + "\n" + s += "2." + sgl.gen("tip_2", stop=["\n", ":", "."]) + "\n" + + branch1 = expand(tip=s["tip_1"]) + branch2 = expand(tip=s["tip_2"]) + + s += "Tip 1: " + branch1["detailed_tip"] + "\n" + s += "Tip 2: " + branch2["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + compiled = tip_suggestion.compile() + # compiled.print_graph() + + sgl.set_default_backend(sgl.OpenAI("gpt-3.5-turbo-instruct")) + state = compiled.run(topic="staying healthy") + # print(state.text() + "\n") + + states = compiled.run_batch( + [ + {"topic": "staying healthy"}, + {"topic": "staying happy"}, + {"topic": "earning money"}, + ], + temperature=0, + ) + # for s in states: + # print(s.text() + "\n") + + def test_role(self): + @sgl.function + def multi_turn_chat(s): + s += sgl.user("Who are you?") + s += sgl.assistant(sgl.gen("answer_1")) + s += sgl.user("Who created you?") + s += sgl.assistant(sgl.gen("answer_2")) + + backend = BaseBackend() + backend.chat_template = get_chat_template("llama-2-chat") + + compiled = multi_turn_chat.compile(backend=backend) + # compiled.print_graph() + + def test_fork(self): + @sgl.function + def tip_suggestion(s): + s += ( + "Here are three tips for staying healthy: " + "1. Balanced Diet; " + "2. Regular Exercise; " + "3. Adequate Sleep\n" + ) + + forks = s.fork(3) + for i in range(3): + forks[i] += f"Now, expand tip {i+1} into a paragraph:\n" + forks[i] += sgl.gen(f"detailed_tip") + + s += "Tip 1:" + forks[0]["detailed_tip"] + "\n" + s += "Tip 2:" + forks[1]["detailed_tip"] + "\n" + s += "Tip 3:" + forks[2]["detailed_tip"] + "\n" + s += "In summary" + sgl.gen("summary") + + tracer = tip_suggestion.trace() + # print(tracer.last_node.print_graph_dfs()) + + a = tip_suggestion.run(backend=sgl.OpenAI("gpt-3.5-turbo-instruct")) + # print(a.text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/lang/test_vertexai_backend.py b/test/lang/test_vertexai_backend.py new file mode 100644 index 000000000..83ce7fc0b --- /dev/null +++ b/test/lang/test_vertexai_backend.py @@ -0,0 +1,53 @@ +import unittest + +from sglang import VertexAI, set_default_backend +from sglang.test.test_programs import ( + test_expert_answer, + test_few_shot_qa, + test_image_qa, + test_mt_bench, + test_parallel_decoding, + test_parallel_encoding, + test_stream, +) +from sglang.test.test_utils import CustomTestCase + + +class TestVertexAIBackend(CustomTestCase): + backend = None + + @classmethod + def setUpClass(cls): + cls.backend = VertexAI("gemini-1.5-pro-001") + + def test_few_shot_qa(self): + set_default_backend(self.backend) + test_few_shot_qa() + + def test_mt_bench(self): + set_default_backend(self.backend) + test_mt_bench() + + def test_expert_answer(self): + set_default_backend(self.backend) + test_expert_answer(check_answer=False) + + def test_parallel_decoding(self): + set_default_backend(self.backend) + test_parallel_decoding() + + def test_parallel_encoding(self): + set_default_backend(self.backend) + test_parallel_encoding() + + def test_image_qa(self): + set_default_backend(self.backend) + test_image_qa() + + def test_stream(self): + set_default_backend(self.backend) + test_stream() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 000000000..2f4c80e30 --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/test/srt/ascend/test_ascend_deepep.py b/test/srt/ascend/test_ascend_deepep.py new file mode 100644 index 000000000..6ccd34d27 --- /dev/null +++ b/test/srt/ascend/test_ascend_deepep.py @@ -0,0 +1,121 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-R1-0528-W8A8": { + "accuracy": 0.95, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendDeepEP(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + + cls.common_args = [ + "--trust-remote-code", + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + 0.9, + "--max-running-requests", + 32, + "--disable-radix-cache", + "--chunked-prefill-size", + 32768, + "--disable-cuda-graph", + "--tp-size", + 16, + "--dp-size", + 1, + "--ep-size", + 16, + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "auto", + ] + + cls.extra_envs = { + "HCCL_BUFFSIZE": "500", + "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "32", + } + os.environ.update(cls.extra_envs) + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=1500, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_graph_tp1_bf16.py b/test/srt/ascend/test_ascend_graph_tp1_bf16.py new file mode 100644 index 000000000..95c6b7bcf --- /dev/null +++ b/test/srt/ascend/test_ascend_graph_tp1_bf16.py @@ -0,0 +1,95 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 150, + "output_throughput": 30, + }, +} + + +class TestAscendGraphTp1Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_graph_tp2_bf16.py b/test/srt/ascend/test_ascend_graph_tp2_bf16.py new file mode 100644 index 000000000..f7c3c6537 --- /dev/null +++ b/test/srt/ascend/test_ascend_graph_tp2_bf16.py @@ -0,0 +1,97 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendGraphTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py b/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py new file mode 100644 index 000000000..6de97b04d --- /dev/null +++ b/test/srt/ascend/test_ascend_mla_fia_w8a8int8.py @@ -0,0 +1,103 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V2-Lite-W8A8": { + "accuracy": 0.34, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendMlaW8A8Int8(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--tp-size", + 2, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + os.environ["ASCEND_USE_FIA"] = "true" + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_mla_w8a8int8.py b/test/srt/ascend/test_ascend_mla_w8a8int8.py new file mode 100644 index 000000000..70f7edab4 --- /dev/null +++ b/test/srt/ascend/test_ascend_mla_w8a8int8.py @@ -0,0 +1,101 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "/root/.cache/modelscope/hub/models/vllm-ascend/DeepSeek-V2-Lite-W8A8": { + "accuracy": 0.34, + "latency": 1000, + "output_throughput": 6, + }, +} + + +class TestAscendMlaW8A8Int8(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + "--tp-size", + 4, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_tp1_bf16.py b/test/srt/ascend/test_ascend_tp1_bf16.py new file mode 100644 index 000000000..f854605ce --- /dev/null +++ b/test/srt/ascend/test_ascend_tp1_bf16.py @@ -0,0 +1,96 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.84, + "latency": 150, + "output_throughput": 30, + }, +} + + +class TestAscendTp1Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_tp2_bf16.py b/test/srt/ascend/test_ascend_tp2_bf16.py new file mode 100644 index 000000000..d5e141c9f --- /dev/null +++ b/test/srt/ascend/test_ascend_tp2_bf16.py @@ -0,0 +1,98 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_tp2_fia_bf16.py b/test/srt/ascend/test_ascend_tp2_fia_bf16.py new file mode 100644 index 000000000..bdd1c5733 --- /dev/null +++ b/test/srt/ascend/test_ascend_tp2_fia_bf16.py @@ -0,0 +1,101 @@ +import os +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen2.5-7B-Instruct": { + "accuracy": 0.85, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp2Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + 0.8, + "--attention-backend", + "ascend", + "--tp-size", + 2, + "--disable-radix-cache", + ] + + def test_a_gsm8k(self): + os.environ["ASCEND_USE_FIA"] = "true" + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_tp4_bf16.py b/test/srt/ascend/test_ascend_tp4_bf16.py new file mode 100644 index 000000000..bb7d90e4f --- /dev/null +++ b/test/srt/ascend/test_ascend_tp4_bf16.py @@ -0,0 +1,101 @@ +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + +TEST_MODEL_MATRIX = { + "Qwen/Qwen3-30B-A3B-Instruct-2507": { + "accuracy": 0.90, + "latency": 180, + "output_throughput": 20, + }, +} + + +class TestAscendTp4Bf16(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.models = TEST_MODEL_MATRIX.keys() + cls.base_url = DEFAULT_URL_FOR_TEST + cls.url = urlparse(DEFAULT_URL_FOR_TEST) + cls.common_args = [ + "--trust-remote-code", + "--mem-fraction-static", + 0.7, + "--max-running-requests", + 32, + "--attention-backend", + "ascend", + "--cuda-graph-max-bs", + 32, + "--tp-size", + 4, + ] + + def test_a_gsm8k(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing accuracy: {model} ===##") + + process = popen_launch_server( + model, + self.base_url, + timeout=1800, + other_args=[ + *self.common_args, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1319, + max_new_tokens=512, + parallel=128, + host=f"http://{self.url.hostname}", + port=int(self.url.port), + ) + + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreaterEqual( + metrics["accuracy"], + TEST_MODEL_MATRIX[model]["accuracy"], + ) + finally: + kill_process_tree(process.pid) + + def test_b_throughput(self): + for model in self.models: + with self.subTest(model=model): + print(f"##=== Testing throughput: {model} ===##") + + output_throughput = run_bench_offline_throughput( + model, + [ + *self.common_args, + ], + ) + + print(f"##=== {model} throughput: {output_throughput} ===##") + + if is_in_ci(): + self.assertGreater( + output_throughput, + TEST_MODEL_MATRIX[model]["output_throughput"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ascend/test_ascend_w8a8_quantization.py b/test/srt/ascend/test_ascend_w8a8_quantization.py new file mode 100644 index 000000000..bf139f46a --- /dev/null +++ b/test/srt/ascend/test_ascend_w8a8_quantization.py @@ -0,0 +1,104 @@ +""" +Usage: +python3 -m unittest test_ascend_w8a8_quantization.TestAscendW8A8.test_gsm8k +""" + +import os +import time +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +if "ASCEND_RT_VISIBLE_DEVICES" not in os.environ: + os.environ["ASCEND_RT_VISIBLE_DEVICES"] = "0,1" +DEFAULT_PORT_FOR_SRT_TEST_RUNNER = ( + 7000 + int(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "0")[0]) * 100 +) +DEFAULT_URL_FOR_TEST = f"http://127.0.0.1:{DEFAULT_PORT_FOR_SRT_TEST_RUNNER + 1000}" + + +class TestAscendW8A8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "vllm-ascend/Qwen2.5-0.5B-Instruct-w8a8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--disable-cuda-graph", + "--device", + "npu", + "--attention-backend", + "ascend", + "--quantization", + "w8a8_int8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + base_url = DEFAULT_URL_FOR_TEST + url = urlparse(base_url) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{url.hostname}", + port=int(url.port), + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreaterEqual(metrics["accuracy"], 0.25) + self.assertGreaterEqual(metrics["output_throughput"], 1000) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + + if is_in_ci(): + self.assertGreaterEqual(throughput, 25) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/configs/deepseek_v3.yaml b/test/srt/configs/deepseek_v3.yaml new file mode 100644 index 000000000..82d059cba --- /dev/null +++ b/test/srt/configs/deepseek_v3.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-8192-1024-concurrency1 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 1 --num-prompts 5 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency2 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 2 --num-prompts 10 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency4 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 4 --num-prompts 20 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency8 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 8 --num-prompts 32 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency16 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 16 --num-prompts 48 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency24 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 24 --num-prompts 72 --output-file deepseek_v3_results.jsonl + + - name: sglang-8192-1024-concurrency32 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 32 --num-prompts 96 --output-file deepseek_v3_results.jsonl diff --git a/test/srt/configs/deepseek_v3_long_context.yaml b/test/srt/configs/deepseek_v3_long_context.yaml new file mode 100644 index 000000000..df416a429 --- /dev/null +++ b/test/srt/configs/deepseek_v3_long_context.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-32000-100-concurrency1 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 1 --num-prompts 5 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency2 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 2 --num-prompts 10 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency4 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 4 --num-prompts 20 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency8 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 8 --num-prompts 32 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency16 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 16 --num-prompts 48 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency24 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 24 --num-prompts 72 --output-file deepseek_v3_long_context_results.jsonl + + - name: sglang-32000-100-concurrency32 + server_cmd: python3 -m sglang.launch_server --model deepseek-ai/DeepSeek-V3-0324 --tp 8 --trust-remote-code --disable-radix-cache --max-prefill-tokens 32768 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 32000 --random-output-len 100 --max-concurrency 32 --num-prompts 96 --output-file deepseek_v3_long_context_results.jsonl diff --git a/test/srt/configs/llama_405b.yaml b/test/srt/configs/llama_405b.yaml new file mode 100644 index 000000000..db0c816fb --- /dev/null +++ b/test/srt/configs/llama_405b.yaml @@ -0,0 +1,28 @@ +tasks: + - name: sglang-8192-1024-concurrency1 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 1 --num-prompts 5 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency2 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 2 --num-prompts 10 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency4 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 4 --num-prompts 20 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency8 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 8 --num-prompts 32 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency16 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 16 --num-prompts 48 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency24 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 24 --num-prompts 72 --output-file llama_405b_results.jsonl + + - name: sglang-8192-1024-concurrency32 + server_cmd: python3 -m sglang.launch_server --model nvidia/Llama-3.1-405B-Instruct-FP8 --tp 8 + client_cmd: python3 -m sglang.bench_serving --dataset-name random --random-range-ratio 1 --random-input-len 8192 --random-output-len 1024 --max-concurrency 32 --num-prompts 96 --output-file llama_405b_results.jsonl diff --git a/test/srt/configs/random_config.yaml b/test/srt/configs/random_config.yaml new file mode 100644 index 000000000..eae8c27f4 --- /dev/null +++ b/test/srt/configs/random_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: vllm-128-4 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: vllm-2000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: vllm-4000-200 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: vllm-32000-100 + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/test/srt/configs/random_flashinfer_vs_triton_config.yaml b/test/srt/configs/random_flashinfer_vs_triton_config.yaml new file mode 100644 index 000000000..7f4a386dd --- /dev/null +++ b/test/srt/configs/random_flashinfer_vs_triton_config.yaml @@ -0,0 +1,25 @@ +tasks: + - name: sglang-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-triton-128-4 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 128 --random-output 4 --request-rate 24 --num-prompt 1440 + - name: sglang-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-triton-2000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 2000 --random-output 100 --request-rate 2 --num-prompt 120 + - name: sglang-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-triton-4000-200 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 4000 --random-output 200 --request-rate 8 --num-prompt 480 + - name: sglang-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 + - name: sglang-triton-32000-100 + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache --attention-backend triton + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name random --random-input 32000 --random-output 100 --request-rate 1 --num-prompt 60 diff --git a/test/srt/configs/sharegpt_config.yaml b/test/srt/configs/sharegpt_config.yaml new file mode 100644 index 000000000..a80b96c8e --- /dev/null +++ b/test/srt/configs/sharegpt_config.yaml @@ -0,0 +1,7 @@ +tasks: + - name: sglang-benchmark + server_cmd: python3 -m sglang.launch_server --model-path meta-llama/Llama-3.1-8B-Instruct --disable-radix-cache + client_cmd: python3 -m sglang.bench_serving --backend sglang --dataset-name sharegpt --request-rate 16 + - name: vllm-benchmark + server_cmd: python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-3.1-8B-Instruct --disable-log-requests + client_cmd: python3 -m sglang.bench_serving --backend vllm --dataset-name sharegpt --request-rate 16 diff --git a/test/srt/cpu/test_activation.py b/test/srt/cpu/test_activation.py new file mode 100644 index 000000000..1234fc631 --- /dev/null +++ b/test/srt/cpu/test_activation.py @@ -0,0 +1,55 @@ +import itertools +import unittest + +import sgl_kernel +import torch +import torch.nn.functional as F +from utils import GeluAndMul, SiluAndMul, precision + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestActivation(CustomTestCase): + M = [128, 129, 257] + N = [22016, 22018] + dtype = [torch.float16, torch.bfloat16] + + def _silu_and_mul_test(self, m, n, dtype): + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.silu_and_mul_cpu(x) + ref_out = SiluAndMul(x) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def _gelu_and_mul_test(self, m, n, dtype): + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.gelu_and_mul_cpu(x) + ref_out = GeluAndMul(x, approximate="none") + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def _gelu_tanh_and_mul_test(self, m, n, dtype): + x = torch.randn([m, n], dtype=dtype) + + out = torch.ops.sgl_kernel.gelu_tanh_and_mul_cpu(x) + ref_out = GeluAndMul(x, approximate="tanh") + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_activation(self): + for params in itertools.product(self.M, self.N, self.dtype): + with self.subTest(m=params[0], n=params[1], dtype=params[2]): + self._silu_and_mul_test(*params) + self._gelu_and_mul_test(*params) + self._gelu_tanh_and_mul_test(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_binding.py b/test/srt/cpu/test_binding.py new file mode 100644 index 000000000..d3cc329af --- /dev/null +++ b/test/srt/cpu/test_binding.py @@ -0,0 +1,28 @@ +import re +import unittest + +import sgl_kernel +import torch + +kernel = torch.ops.sgl_kernel + +from sglang.test.test_utils import CustomTestCase + + +class TestGemm(CustomTestCase): + def test_binding(self): + start_id = 1 + n_cpu = 6 + + expected_cores = list(map(str, range(start_id, start_id + n_cpu))) + cpu_ids = ",".join(expected_cores) + output = kernel.init_cpu_threads_env(cpu_ids) + + bindings = re.findall(r"OMP tid: \d+, core (\d+)", output) + self.assertEqual(len(bindings), n_cpu) + + self.assertEqual(bindings, expected_cores) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_decode.py b/test/srt/cpu/test_decode.py new file mode 100644 index 000000000..c77378e1a --- /dev/null +++ b/test/srt/cpu/test_decode.py @@ -0,0 +1,170 @@ +import unittest + +import sgl_kernel +import torch +from torch.nn.functional import scaled_dot_product_attention + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestDecodeAttention(CustomTestCase): + def _run_sdpa_forward_decode( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + seq_len_q = 1 + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out = ( + scaled_dot_product_attention( + per_req_query.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out + start_q, start_kv = end_q, end_kv + + return output + + def _test_grouped_decode_attention_once(self, B, H_Q, H_KV, D, D_V, device): + dtype = torch.bfloat16 + # This represents the number of tokens already in the sequence + seq_len = 1024 + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + num_kv_splits = 8 + enable_gqa = H_Q != H_KV + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device=device) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device=device) + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device=device) + + key = torch.randn(B, H_KV, D, dtype=dtype) + value = torch.randn(B, H_KV, D_V, dtype=dtype) + loc = torch.randint(0, 10, (B,)).to(torch.int64) + + # set kv cache + k_buffer[loc] = key + v_buffer[loc] = value + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype, device=device) + + req_to_token = ( + torch.arange(total_tokens, device=device) + .reshape(B, seq_len) + .to(torch.int32) + ) + b_req_idx = torch.arange(B, device=device).to(torch.int64) + b_seq_len = torch.full((B,), seq_len, device=device).to(torch.int64) + + attn_logits = torch.empty( + (B, H_Q, num_kv_splits, D_V + 1), + dtype=torch.float32, + device=device, + ) + + # k_buffer, v_buffer, query, key and value supports non-contiguous tensors + k_buffer = k_buffer.transpose(0, 1).contiguous().transpose(0, 1) + v_buffer = v_buffer.transpose(0, 1).contiguous().transpose(0, 1) + q = q.transpose(0, 1).contiguous().transpose(0, 1) + key = key.transpose(0, 1).contiguous().transpose(0, 1) + value = value.transpose(0, 1).contiguous().transpose(0, 1) + torch.ops.sgl_kernel.decode_attention_cpu( + q, + k_buffer, + v_buffer, + o, + key, + value, + loc, + attn_logits, + req_to_token, + b_req_idx, + b_seq_len, + sm_scale, + logit_cap, + ) + + self._run_sdpa_forward_decode( + q, + o_grouped, + k_buffer, + v_buffer, + req_to_token, + b_req_idx, + b_seq_len, + scaling=sm_scale, + enable_gqa=enable_gqa, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + self.assertGreater(cos_sim.item(), 0.99) + torch.testing.assert_close(o, o_grouped, atol=3e-2, rtol=1e-6) + + def _test_grouped_decode_attention(self, device="cuda"): + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 16, 16), + (2, 32, 8, 33, 55), + (2, 16, 1, 64, 64), + (2, 64, 1, 13, 13), + (2, 128, 1, 80, 80), + (2, 128, 2, 512, 512), + (1, 16, 1, 576, 512), + (1, 16, 16, 576, 512), + (1, 22, 1, 576, 512), + (1, 40, 8, 128, 128), + ] + + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once( + B, H_Q, H_KV, D, D_V, device=device + ) + + def test_grouped_decode_attention(self): + self._test_grouped_decode_attention("cpu") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_extend.py b/test/srt/cpu/test_extend.py new file mode 100644 index 000000000..9c6f5b394 --- /dev/null +++ b/test/srt/cpu/test_extend.py @@ -0,0 +1,190 @@ +import unittest + +import sgl_kernel +import torch +from torch.nn.functional import scaled_dot_product_attention + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestExtendAttention(CustomTestCase): + + def _run_sdpa_forward_extend( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + extend_prefix_lens: torch.Tensor, + extend_seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + + assert seq_lens.shape[0] == extend_prefix_lens.shape[0] + assert seq_lens.shape[0] == extend_seq_lens.shape[0] + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + + extend_seq_len_q = extend_seq_lens[seq_idx] + prefill_seq_len_q = extend_prefix_lens[seq_idx] + + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + extend_seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + per_req_query_redudant = torch.empty( + (per_req_query.shape[0], seq_len_kv, per_req_query.shape[2]), + dtype=per_req_query.dtype, + device=per_req_query.device, + ) + + per_req_query_redudant[:, prefill_seq_len_q:, :] = per_req_query + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out_redudant = ( + scaled_dot_product_attention( + per_req_query_redudant.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out_redudant[prefill_seq_len_q:, :, :] + start_q, start_kv = end_q, end_kv + return output + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D, DV, mla=False): + dtype = torch.bfloat16 + + b_seq_len_prefix = torch.randint(1, N_CTX // 2, (B,), dtype=torch.int32) + if mla: + b_seq_len_prefix.zero_() + b_seq_len_extend = torch.randint(1, N_CTX // 2, (B,), dtype=torch.int32) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32) + req_to_tokens = torch.empty((B, max_len_in_batch), dtype=torch.int32) + b_start_loc = torch.zeros((B,), dtype=torch.int32) + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + for i in range(B): + req_to_tokens[i, : b_seq_len[i]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + + H_BUF = 1 if mla else H_KV + k_buffer = torch.randn((total_token_num, H_BUF, D), dtype=dtype) + v_buffer = torch.randn((total_token_num, H_BUF, DV), dtype=dtype) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype) + v_extend = torch.empty((extend_token_num, H_KV, DV), dtype=dtype) + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype) + + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.randn( + (b_seq_len_extend[i], H_Q, D), dtype=dtype + ) + + # q_extend, k_extend, v_extend, k_buffer and v_buffer supports non-contiguous tensors + q_extend = q_extend.transpose(0, 1).contiguous().transpose(0, 1) + k_extend = k_extend.transpose(0, 1).contiguous().transpose(0, 1) + v_extend = v_extend.transpose(0, 1).contiguous().transpose(0, 1) + k_buffer = k_buffer.transpose(0, 1).contiguous().transpose(0, 1) + v_buffer = v_buffer.transpose(0, 1).contiguous().transpose(0, 1) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + b_start_loc_extend = torch.zeros_like(b_seq_len) + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + + # handle index type + b_req_idx = b_req_idx.to(torch.int64) + b_seq_len = b_seq_len.to(torch.int64) + + enable_gqa = H_Q != H_KV + o_ref = torch.empty((extend_token_num, H_Q, DV), dtype=dtype) + self._run_sdpa_forward_extend( + q_extend, + o_ref, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_seq_len_prefix, + b_seq_len_extend, + scaling=sm_scale, + enable_gqa=enable_gqa, + causal=True, + ) + + o_extend = torch.empty((extend_token_num, H_Q, DV), dtype=dtype) + torch.ops.sgl_kernel.extend_attention_cpu( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + req_to_tokens, + b_req_idx, + b_seq_len, + b_seq_len_extend, + b_start_loc_extend, + max_len_extend, + sm_scale, + logit_cap, + ) + + torch.testing.assert_close(o_ref, o_extend, atol=1e-2, rtol=1e-2) + + def test_extend_attention(self): + for is_mla in [True, False]: + self._test_extend_attention_once(1, 123, 1, 1, 128, 96, is_mla) + self._test_extend_attention_once(1, 123, 16, 1, 128, 96, is_mla) + self._test_extend_attention_once(4, 1230, 16, 4, 128, 96, is_mla) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_gemm.py b/test/srt/cpu/test_gemm.py new file mode 100644 index 000000000..a9fe5066a --- /dev/null +++ b/test/srt/cpu/test_gemm.py @@ -0,0 +1,189 @@ +import itertools +import unittest + +# TODO: use interface in cpu.py +import sgl_kernel +import torch +import torch.nn as nn +from utils import ( + convert_weight, + native_w8a8_per_token_matmul, + per_token_quant_int8, + precision, +) + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class Mod(nn.Module): + def __init__(self, input_channel, output_channel, has_bias): + super(Mod, self).__init__() + self.linear = torch.nn.Linear(input_channel, output_channel, has_bias) + + def forward(self, x): + return self.linear(x) + + +class TestGemm(CustomTestCase): + M = [1, 101] + N = [16, 32 * 13] + K = [32 * 16] + has_bias = [False, True] + + M_int8 = [2, 128] + N_int8 = [32 * 12] + K_int8 = [32 * 17] + + M_fp8 = [1, 11] + N_fp8 = [128, 224] + K_fp8 = [512, 576] + + def _bf16_gemm(self, M, N, K, has_bias): + + mat1 = torch.randn(M, K, dtype=torch.bfloat16) + mat2 = torch.randn(N, K, dtype=torch.bfloat16) + + ref = torch.matmul(mat1.float(), mat2.float().t()) + if has_bias: + bias = torch.randn(N, dtype=torch.float32) + ref.add_(bias.bfloat16()) + + ref = ref.bfloat16() + + out = torch.ops.sgl_kernel.weight_packed_linear( + mat1, mat2, bias if has_bias else None, False + ) + + packed_mat2 = torch.ops.sgl_kernel.convert_weight_packed(mat2) + out2 = torch.ops.sgl_kernel.weight_packed_linear( + mat1, packed_mat2, bias if has_bias else None, True + ) + + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, out, atol=atol, rtol=rtol) + torch.testing.assert_close(ref, out2, atol=atol, rtol=rtol) + + def test_bf16_gemm(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._bf16_gemm(*params) + + def _int8_gemm(self, M, N, K, has_bias): + dtype = torch.bfloat16 + A = torch.randn((M, K), dtype=dtype) / 10 + Aq, As = per_token_quant_int8(A) + + factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + B = (torch.rand((N, K), dtype=torch.float32) - 0.5) * 2 + Bq = (B * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + Bs = torch.rand(N) * factor_for_scale + + bias = torch.randn(N) if has_bias else None + ref_out = native_w8a8_per_token_matmul(Aq, Bq, As, Bs, bias, dtype) + + atol = rtol = precision[ref_out.dtype] + + Aq2, As2 = torch.ops.sgl_kernel.per_token_quant_int8_cpu(A) + out = torch.ops.sgl_kernel.int8_scaled_mm_cpu( + Aq2, Bq, As2, Bs, bias if has_bias else None, torch.bfloat16, False + ) + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + # test the fused version + fused_out = torch.ops.sgl_kernel.int8_scaled_mm_with_quant( + A, Bq, Bs, bias if has_bias else None, torch.bfloat16, False + ) + torch.testing.assert_close(ref_out, fused_out, atol=atol, rtol=rtol) + + def test_int8_gemm(self): + for params in itertools.product( + self.M_int8, + self.N_int8, + self.K_int8, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._int8_gemm(*params) + + def _fp8_gemm(self, M, N, K, has_bias): + prepack = True + chunk = False + scale_block_size_N = 64 + scale_block_size_K = 128 + assert scale_block_size_N <= N + assert scale_block_size_K <= K + A_dtype = torch.bfloat16 + + model = Mod(K, N, has_bias).eval() + if chunk: + data = torch.randn(M, K + 6, dtype=A_dtype).narrow(1, 0, K) + else: + data = torch.randn(M, K, dtype=A_dtype) + + weight = model.linear.weight # (N, K) + + if has_bias: + bias = model.linear.bias + + fp8_weight, scales, dq_weight = convert_weight( + weight, [scale_block_size_N, scale_block_size_K], A_dtype + ) + + if has_bias: + ref = torch.matmul(data.to(A_dtype), dq_weight.T) + bias.to(A_dtype) + else: + ref = torch.matmul(data.to(A_dtype), dq_weight.T) + + if prepack: + fp8_weight = torch.ops.sgl_kernel.convert_weight_packed(fp8_weight) + + opt = torch.ops.sgl_kernel.fp8_scaled_mm_cpu( + data, + fp8_weight, + scales, + [scale_block_size_N, scale_block_size_K], + bias if has_bias else None, + data.dtype, + prepack, + ) + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, opt, atol=atol, rtol=rtol) + + def test_fp8_gemm(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.has_bias, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + has_bias=params[3], + ): + self._fp8_gemm(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_mla.py b/test/srt/cpu/test_mla.py new file mode 100644 index 000000000..1f0718d7a --- /dev/null +++ b/test/srt/cpu/test_mla.py @@ -0,0 +1,157 @@ +import itertools +import unittest + +import sgl_kernel +import torch +from torch.nn.functional import scaled_dot_product_attention +from utils import precision + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestMLA(CustomTestCase): + def _run_sdpa_forward_decode( + self, + query: torch.Tensor, + output: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + key: torch.Tensor, + loc: torch.Tensor, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + seq_lens: torch.Tensor, + scaling=None, + enable_gqa=False, + causal=False, + ): + # set kv cache + k_cache[loc] = key + + # [num_tokens, num_heads, head_size] -> [num_heads, num_tokens, head_size] + query = query.movedim(0, query.dim() - 2) + + start_q, start_kv = 0, 0 + for seq_idx in range(seq_lens.shape[0]): + seq_len_q = 1 + seq_len_kv = seq_lens[seq_idx] + end_q = start_q + seq_len_q + end_kv = start_kv + seq_len_kv + + per_req_query = query[:, start_q:end_q, :] + + # get key and value from cache. per_req_tokens contains the kv cache + # index for each token in the sequence. + req_pool_idx = req_pool_indices[seq_idx] + per_req_tokens = req_to_token[req_pool_idx, :seq_len_kv] + per_req_key = k_cache[per_req_tokens].movedim(0, query.dim() - 2) + per_req_value = v_cache[per_req_tokens].movedim(0, query.dim() - 2) + + per_req_out = ( + scaled_dot_product_attention( + per_req_query.unsqueeze(0), + per_req_key.unsqueeze(0), + per_req_value.unsqueeze(0), + enable_gqa=enable_gqa, + scale=scaling, + is_causal=causal, + ) + .squeeze(0) + .movedim(query.dim() - 2, 0) + ) + output[start_q:end_q, :, :] = per_req_out + start_q, start_kv = end_q, end_kv + + return output + + def _test_grouped_decode_attention_once(self, B, H_Q, H_KV, D, D_V, seq_len): + dtype = torch.bfloat16 + + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + logit_cap = 0.0 + num_kv_splits = 8 + enable_gqa = H_Q != H_KV + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype) + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype) + v_buffer = k_buffer.narrow(2, 0, D_V) + + key = torch.randn(B, H_KV, D, dtype=dtype) + value = key.narrow(2, 0, D_V) + # make sure no duplicates in loc + loc = torch.randperm(total_tokens)[:B].to(torch.int64) + + k_buffer2 = k_buffer.clone() + v_buffer2 = k_buffer2.narrow(2, 0, D_V) + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype) + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype) + + req_to_token = torch.arange(total_tokens).reshape(B, seq_len).to(torch.int32) + b_req_idx = torch.arange(B).to(torch.int64) + b_seq_len = torch.full((B,), seq_len).to(torch.int64) + + attn_logits = torch.empty( + (B, H_Q, num_kv_splits, D_V + 1), + dtype=torch.float32, + ) + + torch.ops.sgl_kernel.decode_attention_cpu( + q, + k_buffer2, + v_buffer2, + o, + key, + value, + loc, + attn_logits, + req_to_token, + b_req_idx, + b_seq_len, + sm_scale, + logit_cap, + ) + + self._run_sdpa_forward_decode( + q, + o_grouped, + k_buffer, + v_buffer, + key, + loc, + req_to_token, + b_req_idx, + b_seq_len, + scaling=sm_scale, + enable_gqa=enable_gqa, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + atol = rtol = precision[q.dtype] + self.assertGreater(cos_sim.item(), 0.99) + torch.testing.assert_close(o, o_grouped, atol=atol, rtol=rtol) + torch.testing.assert_close(k_buffer, k_buffer2, atol=atol, rtol=rtol) + torch.testing.assert_close(v_buffer, v_buffer2, atol=atol, rtol=rtol) + + def test_grouped_decode_attention(self): + configs = [ + (1, 22, 1, 576, 512, 8 * 111), + (4, 22, 1, 576, 512, 8 * 128), + (40, 22, 1, 576, 512, 8 * 133), + ] + + for B, H_Q, H_KV, D, D_V, seqlen in configs: + self._test_grouped_decode_attention_once(B, H_Q, H_KV, D, D_V, seqlen) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_moe.py b/test/srt/cpu/test_moe.py new file mode 100644 index 000000000..96eb28020 --- /dev/null +++ b/test/srt/cpu/test_moe.py @@ -0,0 +1,265 @@ +import itertools +import math +import unittest + +# TODO: use interface in cpu.py +import sgl_kernel +import torch + +kernel = torch.ops.sgl_kernel + +torch.manual_seed(1234) + +from utils import ( + BLOCK_K, + BLOCK_N, + factor_for_scale, + fp8_max, + fp8_min, + native_fp8_fused_moe, + precision, + scaled_weight, + torch_naive_fused_moe, + torch_w8a8_per_column_fused_moe, +) + +from sglang.test.test_utils import CustomTestCase + + +def fused_moe(a, w1, w2, score, topk, renormalize, prepack): + + G = 1 + topk_group = 1 + + B, D = a.shape + topk_weights = torch.empty(B, topk, dtype=torch.float32) + topk_ids = torch.empty(B, topk, dtype=torch.int32) + topk_weights, topk_ids = kernel.grouped_topk_cpu( + a, score, topk, renormalize, G, topk_group, 0, None, None + ) + + packed_w1 = kernel.convert_weight_packed(w1) if prepack else w1 + packed_w2 = kernel.convert_weight_packed(w2) if prepack else w2 + + inplace = True + return kernel.fused_experts_cpu( + a, + packed_w1, + packed_w2, + topk_weights, + topk_ids, + inplace, + False, + False, + None, + None, + None, + None, + None, + prepack, + ) + + +class TestFusedExperts(CustomTestCase): + M = [2, 114] + N = [32] + K = [32] + E = [4] + topk = [2] + renormalize = [False, True] + + M_int8 = [1, 39] + N_int8 = [128] + K_int8 = [256] + E_int8 = [8] + topk_int8 = [3] + + M_fp8 = [2, 121] + N_fp8 = [352, 512] + K_fp8 = [256, 320] + E_fp8 = [8] + topk_fp8 = [4] + + def _bf16_moe(self, m, n, k, e, topk, renormalize): + dtype = torch.bfloat16 + prepack = True + + a = torch.randn((m, k), device="cpu", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cpu", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cpu", dtype=dtype) / 10 + score = torch.randn((m, e), device="cpu", dtype=dtype) + + torch_output = torch_naive_fused_moe(a, w1, w2, score, topk, renormalize) + fused_output = fused_moe(a, w1, w2, score, topk, renormalize, prepack) + + atol = rtol = precision[torch_output.dtype] + torch.testing.assert_close(torch_output, fused_output, atol=atol, rtol=rtol) + + def test_bf16_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.topk, + self.renormalize, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + e=params[3], + topk=params[4], + renormalize=params[5], + ): + self._bf16_moe(*params) + + def _int8_moe(self, M, N, K, E, topk): + dtype = torch.bfloat16 + prepack = True + + # Initialize int8 quantization parameters + int8_factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / math.sqrt(K) + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * int8_factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * int8_factor_for_scale + + # Calculate routing + score = torch.randn((M, E), dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + ref_out = torch_w8a8_per_column_fused_moe( + a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, topk + ) + + inplace = True + packed_w1 = kernel.convert_weight_packed(w1) if prepack else w1 + packed_w2 = kernel.convert_weight_packed(w2) if prepack else w2 + out = kernel.fused_experts_cpu( + a, + packed_w1, + packed_w2, + topk_weight, + topk_ids.to(torch.int32), + inplace, + True, + False, + w1_s, + w2_s, + None, + None, + None, + prepack, + ) + + atol = rtol = precision[ref_out.dtype] + # Increase the tolerance for large input shapes + if M > 35: + atol = rtol = 0.02 + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_int8_moe(self): + for params in itertools.product( + self.M_int8, + self.N_int8, + self.K_int8, + self.E_int8, + self.topk_int8, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + ): + self._int8_moe(*params) + + def _fp8_moe(self, M, N, K, E, topk): + dtype = torch.bfloat16 + + a = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + + w1_fp32 = torch.randn(E, 2 * N, K) + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = torch.randn(E, K, N) + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w1s = ( + torch.randn(E, math.ceil(2 * N / BLOCK_N), math.ceil(K / BLOCK_K)) + * factor_for_scale + ) + w2s = ( + torch.randn(E, math.ceil(K / BLOCK_N), math.ceil(N / BLOCK_K)) + * factor_for_scale + ) + + w1_scaled = scaled_weight(w1, w1s) + w2_scaled = scaled_weight(w2, w2s) + + score = torch.randn((M, E), dtype=dtype) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + w1 = kernel.convert_weight_packed(w1) + w2 = kernel.convert_weight_packed(w2) + + ref_out = native_fp8_fused_moe( + a, w1_scaled, w2_scaled, topk_weight, topk_ids, topk + ) + out = kernel.fused_experts_cpu( + a, + w1, + w2, + topk_weight, + topk_ids.to(torch.int32), + False, + False, + True, + w1s, + w2s, + [BLOCK_N, BLOCK_K], + None, + None, + True, + ) + + atol = rtol = precision[dtype] + torch.testing.assert_close(ref_out.bfloat16(), out, atol=atol, rtol=rtol) + + def test_fp8_moe(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.E_fp8, + self.topk_fp8, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + ): + self._fp8_moe(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_norm.py b/test/srt/cpu/test_norm.py new file mode 100644 index 000000000..75bacb198 --- /dev/null +++ b/test/srt/cpu/test_norm.py @@ -0,0 +1,90 @@ +import itertools +import unittest +from typing import Optional, Tuple, Union + +import sgl_kernel +import torch +from utils import make_non_contiguous, precision + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestNorm(CustomTestCase): + M = [4096, 1024] + N = [4096, 4096 + 13] + dtype = [torch.float16, torch.bfloat16] + + def _forward_native( + self, + x: torch.Tensor, + weight: torch.Tensor, + variance_epsilon: float = 1e-6, + residual: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + orig_dtype = x.dtype + x = x.to(torch.float32) + if residual is not None: + x = x + residual.to(torch.float32) + residual = x.to(orig_dtype) + + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + x = x.to(orig_dtype) * weight + if residual is None: + return x + else: + return x, residual + + def _norm_test(self, m, n, dtype): + + x = torch.randn([m, n], dtype=dtype) + x = make_non_contiguous(x) + hidden_size = x.size(-1) + weight = torch.randn(hidden_size, dtype=dtype) + variance_epsilon = 1e-6 + + out = torch.ops.sgl_kernel.rmsnorm_cpu(x, weight, variance_epsilon) + ref_out = self._forward_native(x, weight, variance_epsilon) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + ref_x = x.clone() + residual = torch.randn([m, hidden_size], dtype=dtype) + ref_residual = residual.clone() + + torch.ops.sgl_kernel.fused_add_rmsnorm_cpu( + x, residual, weight, variance_epsilon + ) + + ref_x, ref_residual = self._forward_native( + ref_x, weight, variance_epsilon, ref_residual + ) + + torch.testing.assert_close(x, ref_x, atol=atol, rtol=rtol) + torch.testing.assert_close(residual, ref_residual, atol=atol, rtol=rtol) + + def _l2norm_test(self, m, n, dtype): + + x = torch.randn([m, n], dtype=dtype) + hidden_size = x.size(-1) + fake_ones_weight = torch.ones(hidden_size, dtype=dtype) + variance_epsilon = 1e-6 + + out = torch.ops.sgl_kernel.l2norm_cpu(x, variance_epsilon) + ref_out = self._forward_native(x, fake_ones_weight, variance_epsilon) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_norm(self): + for params in itertools.product(self.M, self.N, self.dtype): + with self.subTest(m=params[0], n=params[1], dtype=params[2]): + self._norm_test(*params) + self._l2norm_test(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_qkv_proj_with_rope.py b/test/srt/cpu/test_qkv_proj_with_rope.py new file mode 100644 index 000000000..dc90cc559 --- /dev/null +++ b/test/srt/cpu/test_qkv_proj_with_rope.py @@ -0,0 +1,432 @@ +import unittest + +import sgl_kernel +import torch +from utils import ( + convert_weight, + native_w8a8_per_token_matmul, + per_token_quant_int8, + precision, +) + +from sglang.srt.layers.rotary_embedding import _apply_rotary_emb +from sglang.test.test_utils import CustomTestCase + +convert_weight_packed = torch.ops.sgl_kernel.convert_weight_packed +qkv_proj_with_rope = torch.ops.sgl_kernel.qkv_proj_with_rope +qkv_proj_with_rope_fused_weight = torch.ops.sgl_kernel.qkv_proj_with_rope_fused_weight +torch.manual_seed(1234) +# constants +kv_lora_rank = 512 +qk_head_dim = 192 +qk_nope_head_dim = 128 +qk_rope_head_dim = 64 +rotary_dim = qk_rope_head_dim +num_heads = 22 +q_lora_rank = 1536 +hidden_size = 7168 +B = 1 +eps = 1e-6 + + +def layernorm(x, weight, variance_epsilon=1e-6, residual=None): + orig_dtype = x.dtype + x = x.to(torch.float32) + variance = x.pow(2).mean(dim=-1, keepdim=True) + x = x * torch.rsqrt(variance + variance_epsilon) + return (x * weight).to(orig_dtype) + + +def rotary_emb(q_pe, k_pe, pos, cos_sin_cache): + orig_dtype = q_pe.dtype + q_pe = q_pe.float() + k_pe = k_pe.float() + cos_sin_cache = cos_sin_cache.float() + + query_rot = q_pe[..., :rotary_dim] + key_rot = k_pe[..., :rotary_dim] + cos_sin = cos_sin_cache[pos] + cos, sin = cos_sin.chunk(2, dim=-1) + query_rot = _apply_rotary_emb(query_rot, cos, sin, False) + key_rot = _apply_rotary_emb(key_rot, cos, sin, False) + return query_rot.to(orig_dtype), key_rot.to(orig_dtype) + + +def native_torch( + q_input, + hidden_states, + q_a_proj_weight, + norm_weight1, + q_b_proj_weight, + w_kc, + kv_a_proj_weight, + norm_weight2, + pos, + cos_sin_cache, +): + + q = torch.matmul(hidden_states, q_a_proj_weight.t()) + q = layernorm(q, norm_weight1) + q = torch.matmul(q, q_b_proj_weight.t()).view(-1, num_heads, qk_head_dim) + + q_nope, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + q_nope_out = torch.bmm(q_nope.transpose(0, 1), w_kc) + + q_input[..., :kv_lora_rank] = q_nope_out.transpose(0, 1) + latent_cache = torch.matmul(hidden_states, kv_a_proj_weight.t()) + v_input = latent_cache[..., :kv_lora_rank] + v_input = layernorm(v_input.contiguous(), norm_weight2).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + k_pe = k_input[..., kv_lora_rank:] + + q_pe, k_pe = rotary_emb(q_pe, k_pe, pos, cos_sin_cache) + q_input[..., kv_lora_rank:] = q_pe + k_input[..., kv_lora_rank:] = k_pe + + return q_input, k_input, v_input + + +def native_torch_int8( + q_input, + hidden_states, + w1_q, + w1_s, + norm_weight1, + w2_q, + w2_s, + w_kc, + w3_q, + w3_s, + norm_weight2, + pos, + cos_sin_cache, +): + + a_q, a_s = per_token_quant_int8(hidden_states) + q = native_w8a8_per_token_matmul(a_q, w1_q, a_s, w1_s, None, torch.bfloat16) + q = layernorm(q, norm_weight1) + + a_q, a_s = per_token_quant_int8(q) + q = native_w8a8_per_token_matmul(a_q, w2_q, a_s, w2_s, None, torch.bfloat16).view( + -1, num_heads, qk_head_dim + ) + + q_nope, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + q_nope_out = torch.bmm(q_nope.transpose(0, 1), w_kc) + + q_input[..., :kv_lora_rank] = q_nope_out.transpose(0, 1) + a_q, a_s = per_token_quant_int8(hidden_states) + latent_cache = native_w8a8_per_token_matmul( + a_q, w3_q, a_s, w3_s, None, torch.bfloat16 + ) + v_input = latent_cache[..., :kv_lora_rank] + v_input = layernorm(v_input.contiguous(), norm_weight2).unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + k_pe = k_input[..., kv_lora_rank:] + + q_pe, k_pe = rotary_emb(q_pe, k_pe, pos, cos_sin_cache) + q_input[..., kv_lora_rank:] = q_pe + k_input[..., kv_lora_rank:] = k_pe + + return q_input, k_input, v_input + + +class TestQKVProjWithROPE(CustomTestCase): + def test_bf16_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + fused_weight = torch.cat([q_a_proj_weight, kv_a_proj_weight], dim=0) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + q_ref, k_ref, v_ref = native_torch( + q_input, + hidden_states, + q_a_proj_weight, + norm_weight1, + q_b_proj_weight, + w_kc.transpose(1, 2), + kv_a_proj_weight, + norm_weight2, + pos, + cos_sin_cache, + ) + qa_packed = convert_weight_packed(q_a_proj_weight) + qb_packed = convert_weight_packed(q_b_proj_weight) + kva_packed = convert_weight_packed(kv_a_proj_weight) + wkc_packed = convert_weight_packed(w_kc) + fused_weight_packed = convert_weight_packed(fused_weight) + + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + qa_packed, + qb_packed, + kva_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + False, + None, + None, + None, + True, + None, + ) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + fused_weight_packed, + qb_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + False, + None, + None, + True, + None, + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + torch.testing.assert_close(q_ref, q_out, atol=atol, rtol=rtol) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + def test_int8_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + + w1_q, w1_s = per_token_quant_int8(q_a_proj_weight) + w2_q, w2_s = per_token_quant_int8(q_b_proj_weight) + w3_q, w3_s = per_token_quant_int8(kv_a_proj_weight) + q_ref, k_ref, v_ref = native_torch_int8( + q_input, + hidden_states, + w1_q, + w1_s, + norm_weight1, + w2_q, + w2_s, + w_kc.transpose(1, 2), + w3_q, + w3_s, + norm_weight2, + pos, + cos_sin_cache, + ) + w1_q_packed = convert_weight_packed(w1_q) + w2_q_packed = convert_weight_packed(w2_q) + w3_q_packed = convert_weight_packed(w3_q) + wkc_packed = convert_weight_packed(w_kc) + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + w1_q_packed, + w2_q_packed, + w3_q_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + True, + False, + w1_s, + w2_s, + w3_s, + True, + None, + ) + fused_weight = torch.cat([w1_q, w3_q], dim=0) + fused_weight_s = torch.cat([w1_s, w3_s], dim=0) + w_fused_q_packed = convert_weight_packed(fused_weight) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + w_fused_q_packed, + w2_q_packed, + wkc_packed, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + True, + False, + fused_weight_s, + w2_s, + True, + None, + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + torch.testing.assert_close(q_ref, q_out, atol=atol, rtol=rtol) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + def test_fp8_qkv_proj_with_rope(self): + dtype = torch.bfloat16 + hidden_states = torch.randn(B, hidden_size, dtype=dtype) / hidden_size + q_input = torch.empty( + B, num_heads, kv_lora_rank + qk_rope_head_dim, dtype=dtype + ) + q_a_proj_weight = torch.randn(q_lora_rank, hidden_size, dtype=dtype) * 0.1 + norm_weight1 = torch.randn(q_lora_rank, dtype=dtype) + q_b_proj_weight = ( + torch.randn(num_heads * qk_head_dim, q_lora_rank, dtype=dtype) * 0.1 + ) + w_kc = torch.randn(num_heads, kv_lora_rank, qk_nope_head_dim, dtype=dtype) * 0.1 + kv_a_proj_weight = ( + torch.randn(kv_lora_rank + qk_rope_head_dim, hidden_size, dtype=dtype) * 0.1 + ) + norm_weight2 = torch.randn(kv_lora_rank, dtype=dtype) + pos = torch.randint(10, 100, (B,)) + cos_sin_cache = torch.randn(100, rotary_dim, dtype=dtype) + + scale_block_size_N = 128 + scale_block_size_K = 128 + fp8_q_a_proj_weight, q_a_proj_weight_scale_inv, q_a_proj_weight_dq = ( + convert_weight( + q_a_proj_weight, + [scale_block_size_N, scale_block_size_K], + torch.bfloat16, + ) + ) + fp8_q_b_proj_weight, q_b_proj_weight_scale_inv, q_b_proj_weight_dq = ( + convert_weight( + q_b_proj_weight, + [scale_block_size_N, scale_block_size_K], + torch.bfloat16, + ) + ) + ( + fp8_kv_a_proj_with_mqa_weight, + kv_a_proj_with_mqa_weight_scale_inv, + kv_a_proj_with_mqa_weight_dq, + ) = convert_weight( + kv_a_proj_weight, [scale_block_size_N, scale_block_size_K], torch.bfloat16 + ) + q_ref, k_ref, v_ref = native_torch( + q_input, + hidden_states, + q_a_proj_weight_dq, + norm_weight1, + q_b_proj_weight_dq, + w_kc.transpose(1, 2), + kv_a_proj_with_mqa_weight_dq, + norm_weight2, + pos, + cos_sin_cache, + ) + fp8_q_a_proj_weight_packed = convert_weight_packed(fp8_q_a_proj_weight) + fp8_q_b_proj_weight_packed = convert_weight_packed(fp8_q_b_proj_weight) + fp8_kv_a_proj_with_mqa_weight_packed = convert_weight_packed( + fp8_kv_a_proj_with_mqa_weight + ) + w_kc = convert_weight_packed(w_kc) + q_out, k_out, v_out = qkv_proj_with_rope( + hidden_states, + fp8_q_a_proj_weight_packed, + fp8_q_b_proj_weight_packed, + fp8_kv_a_proj_with_mqa_weight_packed, + w_kc, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + True, + q_a_proj_weight_scale_inv.float(), + q_b_proj_weight_scale_inv.float(), + kv_a_proj_with_mqa_weight_scale_inv.float(), + True, + [scale_block_size_N, scale_block_size_K], + ) + + fused_weight = torch.cat( + [fp8_q_a_proj_weight, fp8_kv_a_proj_with_mqa_weight], dim=0 + ) + fused_weight_s = torch.cat( + [q_a_proj_weight_scale_inv, kv_a_proj_with_mqa_weight_scale_inv], dim=0 + ) + fused_weight_packed = convert_weight_packed(fused_weight) + fused_q_out, fused_k_out, fused_v_out = qkv_proj_with_rope_fused_weight( + hidden_states, + fused_weight_packed, + fp8_q_b_proj_weight_packed, + w_kc, + norm_weight1, + norm_weight2, + pos, + cos_sin_cache, + eps, + False, + True, + fused_weight_s.float(), + q_b_proj_weight_scale_inv.float(), + True, + [scale_block_size_N, scale_block_size_K], + q_lora_rank, + kv_lora_rank, + qk_rope_head_dim, + ) + atol = rtol = precision[q_ref.dtype] + # Due to the change in multiplication order, the error is amplified. + # In the model, with fewer layers, this doesn't cause issues, but in + # tests with more layers, we need to enlarge the tolerance to pass the tests. + torch.testing.assert_close(q_ref, q_out, atol=1e-1, rtol=1e-1) + torch.testing.assert_close(k_ref, k_out, atol=atol, rtol=rtol) + torch.testing.assert_close(v_ref, v_out, atol=atol, rtol=rtol) + torch.testing.assert_close(fused_q_out, q_out) + torch.testing.assert_close(fused_k_out, k_out) + torch.testing.assert_close(fused_v_out, v_out) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_rope.py b/test/srt/cpu/test_rope.py new file mode 100644 index 000000000..38481e5e3 --- /dev/null +++ b/test/srt/cpu/test_rope.py @@ -0,0 +1,178 @@ +import unittest + +import sgl_kernel +import torch +from utils import precision + +from sglang.srt.layers.rotary_embedding import ( + DeepseekScalingRotaryEmbedding, + RotaryEmbedding, +) +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestROPE(CustomTestCase): + def test_deepseek_v2_rope(self): + num_head = 16 + seq_len = 1024 + q_head_dim = 192 + qk_nope_head_dim = 128 + qk_rope_head_dim = 64 + max_pos = 256 + k_dim = 576 + rotary_dim = 64 + is_neox_style = False + + # Create cos_sin_cache + freqs = torch.rand(max_pos, qk_rope_head_dim // 2) + cos = freqs.cos() * 0.7 + sin = freqs.sin() * 0.7 + cos_sin_cache = torch.cat((cos, sin), dim=-1).to(torch.bfloat16) + positions = torch.randint(0, max_pos, (seq_len,)) + + rope = DeepseekScalingRotaryEmbedding( + qk_rope_head_dim, + rotary_dim, + max_pos, + 16, # not used since cos_sin_cache is provided + is_neox_style, + 1.0, + torch.bfloat16, + device="cpu", + ) + rope.register_buffer("cos_sin_cache", cos_sin_cache) + + for dtype in [torch.bfloat16]: + enable_autocast = True + + with torch.no_grad(), torch.amp.autocast("cpu", enabled=enable_autocast): + q = torch.randn(seq_len, num_head, q_head_dim, dtype=dtype) + q_clone = q.clone() + k = torch.randn(seq_len, 1, k_dim, dtype=dtype) + k_clone = k.clone() + _, q_pe = q.split([qk_nope_head_dim, qk_rope_head_dim], dim=-1) + _, q_pe_clone = q_clone.split( + [qk_nope_head_dim, qk_rope_head_dim], dim=-1 + ) + k_pe = k[:, :, k_dim - qk_rope_head_dim :] + k_pe_clone = k_clone[:, :, k_dim - qk_rope_head_dim :] + + # ref kernel + q_pe, k_pe = rope.forward_native( + query=q_pe, + key=k_pe, + positions=positions, + ) + + # fused rope kernel + q_pe_clone, k_pe_clone = torch.ops.sgl_kernel.rotary_embedding_cpu( + positions, + q_pe_clone, + k_pe_clone, + rope.head_size, + cos_sin_cache, + False, + ) + + atol = rtol = precision[q_pe.dtype] + torch.testing.assert_close(q_pe, q_pe_clone, atol=atol, rtol=rtol) + torch.testing.assert_close(k_pe, k_pe_clone, atol=atol, rtol=rtol) + torch.testing.assert_close(k_pe, k_pe_clone) + + def test_origin_rope(self): + def single_test( + head_size: int, + rotary_dim: int, + max_position_embeddings: int, + base: int, + is_neox_style: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q_heads: int, + num_kv_heads: int, + ): + torch.manual_seed(100) + rope_ref = RotaryEmbedding( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + ).to(device) + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, + num_q_heads * head_size, + dtype=dtype, + device=device, + ) + key = torch.randn( + batch_size * seq_len, + num_kv_heads * head_size, + dtype=dtype, + device=device, + ) + + query_ref, key_ref = query.clone(), key.clone() + query_cpu, key_cpu = query.clone(), key.clone() + + query_ref_out, key_ref_out = rope_ref.forward_native( + pos_ids, query_ref, key_ref + ) + query_cpu_out, key_cpu_out = torch.ops.sgl_kernel.rotary_embedding_cpu( + pos_ids, + query_cpu, + key_cpu, + rope_ref.head_size, + rope_ref.cos_sin_cache.to(query.dtype), + rope_ref.is_neox_style, + ) + torch.testing.assert_close( + query_ref_out, query_cpu_out, atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(key_ref_out, key_cpu_out, atol=1e-2, rtol=1e-2) + + test_config = [ + (64, 64, 32, 8000, True, torch.bfloat16, "cpu", 32, 32, 1, 1), + (256, 128, 4096, 10000, True, torch.bfloat16, "cpu", 2, 512, 32, 8), + (512, 128, 311, 10000, True, torch.bfloat16, "cpu", 3, 39, 4, 2), + (128, 128, 2048, 10000, False, torch.bfloat16, "cpu", 2, 512, 32, 8), + (128, 128, 2048, 10000, False, torch.bfloat16, "cpu", 2, 512, 16, 4), + (512, 128, 311, 10000, False, torch.bfloat16, "cpu", 3, 39, 4, 2), + ] + + for ( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + device, + batch_size, + seq_len, + num_q_heads, + num_kv_heads, + ) in test_config: + single_test( + head_size, + rotary_dim, + max_position_embeddings, + base, + is_neox_style, + dtype, + device, + batch_size, + seq_len, + num_q_heads, + num_kv_heads, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_shared_expert.py b/test/srt/cpu/test_shared_expert.py new file mode 100644 index 000000000..17818aebb --- /dev/null +++ b/test/srt/cpu/test_shared_expert.py @@ -0,0 +1,223 @@ +import itertools +import math +import unittest + +# TODO: use interface in cpu.py +import sgl_kernel +import torch +import torch.nn as nn +from utils import ( + BLOCK_K, + BLOCK_N, + SiluAndMul, + factor_for_scale, + fp8_max, + fp8_min, + per_token_quant_int8, + precision, + scaled_weight, + torch_naive_moe, + torch_w8a8_per_column_moe, +) + +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +class TestSharedExpert(CustomTestCase): + M = [2, 121] + N = [32, 32 * 4] + K = [32, 32 * 2] + routed_scaling_factor = [16] + + M_fp8 = [2, 12] + N_fp8 = [512] + K_fp8 = [256] + + def _bf16_shared_expert(self, m, n, k, routed_scaling_factor): + dtype = torch.bfloat16 + prepack = True + + hidden_states = torch.randn(m, k, dtype=dtype) / k + w1 = torch.randn(2 * n, k, dtype=dtype) + w2 = torch.randn(k, n, dtype=dtype) + fused_output = torch.randn(m, k, dtype=dtype) / k + + # fused moe mutates content in hs + hidden_states2 = hidden_states.clone() + + # bfloat16 + ref = torch_naive_moe( + hidden_states.float(), + w1.float(), + w2.float(), + fused_output.float(), + routed_scaling_factor, + ).to(dtype=dtype) + res = torch.ops.sgl_kernel.shared_expert_cpu( + hidden_states, + w1, + w2, + fused_output, + routed_scaling_factor, + True, + False, + False, + None, + None, + None, + None, + None, + False, + ) + + atol = rtol = precision[ref.dtype] + torch.testing.assert_close(ref, res, atol=atol, rtol=rtol) + + def test_bf16_shared_expert(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.routed_scaling_factor, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + routed_scaling_factor=params[3], + ): + self._bf16_shared_expert(*params) + + def _int8_shared_expert(self, m, n, k, routed_scaling_factor): + dtype = torch.bfloat16 + prepack = True + + hidden_states = torch.randn(m, k, dtype=dtype) / k + w1 = torch.randn(2 * n, k, dtype=dtype) + w2 = torch.randn(k, n, dtype=dtype) + fused_output = torch.randn(m, k, dtype=dtype) / k + + # fused moe mutates content in hs + hidden_states2 = hidden_states.clone() + + w1_q, w1_s = per_token_quant_int8(w1) + w2_q, w2_s = per_token_quant_int8(w2) + ref2 = torch_w8a8_per_column_moe( + hidden_states2.float(), + w1_q, + w2_q, + w1_s, + w2_s, + fused_output.float(), + routed_scaling_factor, + ).to(dtype=dtype) + res2 = torch.ops.sgl_kernel.shared_expert_cpu( + hidden_states2, + w1_q, + w2_q, + fused_output, + routed_scaling_factor, + True, + True, + False, + w1_s, + w2_s, + None, + None, + None, + False, + ) + + atol = rtol = precision[ref2.dtype] + torch.testing.assert_close(ref2, res2, atol=atol, rtol=rtol) + + def test_int8_shared_expert(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.routed_scaling_factor, + ): + with self.subTest( + m=params[0], + n=params[1], + k=params[2], + routed_scaling_factor=params[3], + ): + self._int8_shared_expert(*params) + + def _fp8_shared_expert(self, M, N, K, routed_scaling_factor): + dtype = torch.bfloat16 + prepack = True + + a = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + + w1_fp32 = torch.randn(1, 2 * N, K) + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = torch.randn(1, K, N) + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w1s = torch.randn(1, 2 * N // BLOCK_N, K // BLOCK_K) * factor_for_scale + w2s = torch.randn(1, K // BLOCK_N, N // BLOCK_K) * factor_for_scale + + w1_scaled = scaled_weight(w1, w1s).view(2 * N, K) + w2_scaled = scaled_weight(w2, w2s).view(K, N) + + # change back to 2D + w1, w2 = w1.squeeze(0), w2.squeeze(0) + w1s, w2s = w1s.squeeze(0), w2s.squeeze(0) + w1_scaled, w2_scaled = w1_scaled.squeeze(0), w2_scaled.squeeze(0) + + fused_out = torch.randn(M, K, dtype=dtype) / math.sqrt(K) + a2 = a.clone() + + # ref + ic0 = torch.matmul(a.float(), w1_scaled.transpose(0, 1)) + ic1 = SiluAndMul(ic0) + shared_out = torch.matmul(ic1, w2_scaled.transpose(0, 1)) + ref_out = shared_out + fused_out.float() * routed_scaling_factor + ref_out = ref_out.to(dtype=dtype) + + w1 = torch.ops.sgl_kernel.convert_weight_packed(w1) # [2N, K] + w2 = torch.ops.sgl_kernel.convert_weight_packed(w2) # [K, N] + out = torch.ops.sgl_kernel.shared_expert_cpu( + a2, + w1, + w2, + fused_out, + routed_scaling_factor, + True, + False, + True, + w1s, + w2s, + [BLOCK_N, BLOCK_K], + None, + None, + True, + ) + + atol = rtol = precision[ref_out.dtype] + torch.testing.assert_close(ref_out, out, atol=atol, rtol=rtol) + + def test_fp8_shared_expert(self): + for params in itertools.product( + self.M_fp8, + self.N_fp8, + self.K_fp8, + self.routed_scaling_factor, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + routed_scaling_factor=params[3], + ): + self._fp8_shared_expert(*params) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/test_topk.py b/test/srt/cpu/test_topk.py new file mode 100644 index 000000000..4b4ce21ae --- /dev/null +++ b/test/srt/cpu/test_topk.py @@ -0,0 +1,199 @@ +import itertools +import unittest + +import sgl_kernel +import torch +from utils import precision + +from sglang.srt.layers.moe.topk import ( + biased_grouped_topk_impl as native_biased_grouped_topk, +) +from sglang.srt.layers.moe.topk import fused_topk_torch_native as native_fused_topk +from sglang.srt.layers.moe.topk import grouped_topk_gpu as native_grouped_topk +from sglang.srt.models.llama4 import Llama4MoE +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(1234) + + +# This is used by the Deepseek-V2 model +class TestGroupedTopK(CustomTestCase): + def _run_single_test(self, M, E, G, topk, topk_group, renormalize, dtype): + torch.manual_seed(1234) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_grouped_topk( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + G, + topk_group, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.grouped_topk_cpu( + hidden_states, + gating_output, + topk, + renormalize, + G, + topk_group, + 0, + None, + None, + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_grouped_topk(self): + for renormalize in [True, False]: + self._run_single_test(123, 8, 2, 2, 1, renormalize, torch.bfloat16) + self._run_single_test(123, 16, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(1123, 32, 4, 3, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 64, 1, 6, 1, renormalize, torch.bfloat16) + self._run_single_test(123, 256, 8, 4, 8, renormalize, torch.bfloat16) + self._run_single_test(123, 160, 8, 6, 2, renormalize, torch.bfloat16) + + +# DeepSeek V2/V3/R1 uses biased_grouped_top +class TestBiasedGroupedTopK(CustomTestCase): + def _run_single_test( + self, M, E, G, topk, topk_group, renormalize, dtype, bias_dtype + ): + torch.manual_seed(1234) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + correction_bias = torch.randn(E, dtype=bias_dtype) + + ref_topk_weights, ref_topk_ids = native_biased_grouped_topk( + hidden_states.float(), + gating_output.float(), + correction_bias.float(), + topk, + renormalize, + G, + topk_group, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.biased_grouped_topk_cpu( + hidden_states, + gating_output, + correction_bias, + topk, + renormalize, + G, + topk_group, + 0, + None, + None, + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_biased_grouped_topk(self): + for renormalize in [True, False]: + for bias_dtype in [torch.float32, torch.bfloat16]: + self._run_single_test( + 122, 256, 8, 8, 2, renormalize, torch.bfloat16, bias_dtype + ) + + +class TestTopK(CustomTestCase): + def _run_single_test(self, M, E, topk, renormalize, dtype): + torch.manual_seed(1998) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_fused_topk( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + ) + + # fused version + topk_weights, topk_ids = torch.ops.sgl_kernel.topk_softmax_cpu( + hidden_states, gating_output, topk, renormalize + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_topk(self): + for renormalize in [True, False]: + self._run_single_test(123, 8, 2, renormalize, torch.bfloat16) + self._run_single_test(123, 16, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 32, 3, renormalize, torch.bfloat16) + self._run_single_test(123, 64, 6, renormalize, torch.bfloat16) + self._run_single_test(123, 256, 4, renormalize, torch.bfloat16) + self._run_single_test(123, 160, 6, renormalize, torch.bfloat16) + + +class TestCustomTopK(CustomTestCase): + def _run_single_test( + self, M, E, topk, renormalize, dtype, native_custom_f, fused_custom_f + ): + torch.manual_seed(16) + + # expand gating_output by M, otherwise bfloat16 fall into same value aftering truncating + hidden_states = torch.randn(M, 100, dtype=dtype) + gating_output = torch.randn(M, E, dtype=dtype) * 2 * M + + ref_topk_weights, ref_topk_ids = native_custom_f( + hidden_states.float(), + gating_output.float(), + topk, + renormalize, + ) + + # fused version + topk_weights, topk_ids = fused_custom_f( + hidden_states, gating_output, topk, renormalize + ) + + res = torch.zeros(M, E, dtype=torch.float) + ref = torch.zeros(M, E, dtype=torch.float) + res.scatter_(1, topk_ids.long(), topk_weights) + ref.scatter_(1, ref_topk_ids.long(), ref_topk_weights) + torch.testing.assert_close(res, ref) + + def test_custom_topk(self): + test_custom_functions = [ + (Llama4MoE.custom_routing_function, torch.ops.sgl_kernel.topk_sigmoid_cpu) + ] + for native_custom_f, fused_custom_f in test_custom_functions: + self._run_single_test( + 123, 8, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + self._run_single_test( + 123, 16, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + self._run_single_test( + 123, 32, 1, False, torch.bfloat16, native_custom_f, fused_custom_f + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/cpu/utils.py b/test/srt/cpu/utils.py new file mode 100644 index 000000000..6435dad74 --- /dev/null +++ b/test/srt/cpu/utils.py @@ -0,0 +1,274 @@ +import math + +import torch +import torch.nn.functional as F + +precision = { + torch.bfloat16: 1e-2, + torch.float16: 1e-3, + torch.float32: 1e-5, +} + + +BLOCK_N, BLOCK_K = 64, 128 +factor_for_scale = 1e-3 +fp8_max, fp8_min = 400, -400 + + +def SiluAndMul(x: torch.Tensor) -> torch.Tensor: + d = x.shape[-1] // 2 + return F.silu(x[..., :d]) * x[..., d:] + + +def GeluAndMul(x: torch.Tensor, approximate="tanh") -> torch.Tensor: + d = x.shape[-1] // 2 + return F.gelu(x[..., :d], approximate=approximate) * x[..., d:] + + +def per_token_quant_int8(x): + x = x.float() + absmax = x.abs().max(dim=-1).values + absmax = absmax.clamp_min(1e-10).unsqueeze(-1) + scale_x = absmax / 127 + x_q = x.mul(127 / absmax) + x_q = torch.round(x_q).to(torch.int8) + + return x_q, scale_x + + +def convert_weight(weight, scale_block_size, A_dtype): + N, K = weight.size() + fp8_max = 448.0 + scale_block_size_N, scale_block_size_K = scale_block_size # (128, 128) + + pad_N = (scale_block_size_N - (N % scale_block_size_N)) % scale_block_size_N + pad_K = (scale_block_size_K - (K % scale_block_size_K)) % scale_block_size_K + + if pad_N > 0 or pad_K > 0: + weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N)) + + weight_blocks = weight.view( + math.ceil(N / scale_block_size_N), + scale_block_size_N, + math.ceil(K / scale_block_size_K), + scale_block_size_K, + ) # (8, 128, 8, 128) + weight_blocks = weight_blocks.permute(0, 2, 1, 3).contiguous() # (8, 8, 128, 128) + + # Step 2: compute per-block max abs values → scale + abs_max = weight_blocks.abs().amax(dim=(-2, -1), keepdim=True) # (8, 8, 1, 1) + scales = abs_max / fp8_max + scales = torch.where( + scales == 0, torch.ones_like(scales), scales + ) # avoid division by zero + + q_fp8 = (weight_blocks / scales).to(torch.float8_e4m3fn) + q_fp8_reshape = q_fp8.permute(0, 2, 1, 3).contiguous() + + if pad_N > 0 or pad_K > 0: + q_fp8_reshape = q_fp8_reshape.view(N + pad_N, K + pad_K) + q_fp8_reshape = q_fp8_reshape[:N, :K].contiguous() + else: + q_fp8_reshape = q_fp8_reshape.view(N, K) + + dq_weight = q_fp8.float() * scales + dq_weight = dq_weight.permute(0, 2, 1, 3).contiguous() # (8, 128, 8, 128) + + if pad_N > 0 or pad_K > 0: + w_dq = dq_weight.view(N + pad_N, K + pad_K).to(A_dtype) + w_dq = w_dq[:N, :K].contiguous() + else: + w_dq = dq_weight.view(N, K).to(A_dtype) + + scales = scales.view( + math.ceil(N / scale_block_size_N), math.ceil(K / scale_block_size_K) + ) + + return q_fp8_reshape, scales, w_dq + + +def native_w8a8_per_token_matmul(A, B, As, Bs, bias, output_dtype=torch.bfloat16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + if bias is not None: + C.add_(bias.view(1, -1)) + + return C.reshape(origin_C_shape).to(output_dtype) + + +def torch_naive_moe(a, w1, w2, b, routed_scaling_factor): + + ic1 = torch.matmul(a, w1.transpose(0, 1)) + ic2 = SiluAndMul(ic1) + ic3 = torch.matmul(ic2, w2.transpose(0, 1)) + + return ic3 + b * routed_scaling_factor + + +def torch_w8a8_per_column_moe(a, w1_q, w2_q, w1_s, w2_s, b, routed_scaling_factor): + + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + + ic1 = native_w8a8_per_token_matmul( + a_q, w1_q, a_s, w1_s, bias=None, output_dtype=torch.float32 + ) + ic2 = SiluAndMul(ic1) + + a1_q, a1_s = per_token_quant_int8(ic2) + ic3 = native_w8a8_per_token_matmul( + a1_q, w2_q, a1_s, w2_s, bias=None, output_dtype=torch.float32 + ) + + return ic3 + b * routed_scaling_factor + + +def scaled_weight(weight, scales): + E, N, K = weight.shape + pad_N = (BLOCK_N - (N % BLOCK_N)) % BLOCK_N + pad_K = (BLOCK_K - (K % BLOCK_K)) % BLOCK_K + + if pad_N > 0 or pad_K > 0: + weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N)) + + weight_block = ( + weight.view(E, math.ceil(N / BLOCK_N), BLOCK_N, math.ceil(K / BLOCK_K), BLOCK_K) + .permute(0, 1, 3, 2, 4) + .float() + .contiguous() + ) + + weight_scaled = ( + ( + weight_block + * scales.view(E, math.ceil(N / BLOCK_N), math.ceil(K / BLOCK_K), 1, 1) + ) + .permute(0, 1, 3, 2, 4) + .contiguous() + ) + if pad_N > 0 or pad_K > 0: + weight_scaled = weight_scaled.view(E, N + pad_N, K + pad_K) + weight_scaled = weight_scaled[..., :N, :K].contiguous() + else: + weight_scaled = weight_scaled.view(E, N, K) + return weight_scaled + + +def torch_naive_fused_moe(a, w1, w2, score, topk, renormalize): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + + if renormalize: + topk_weight = topk_weight / topk_weight.sum(dim=-1, keepdim=True) + + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose( + 0, 1 + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +def torch_w8a8_per_column_fused_moe(a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=torch.float32, device=a.device) + + # Calculate routing + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + a_q[mask], + w1[i], + a_s[mask], + w1_s[i], + bias=None, + output_dtype=torch.float32, + ) + # Activation function + act_out = SiluAndMul(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = per_token_quant_int8(act_out) + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, + w2[i], + act_out_s, + w2_s[i], + bias=None, + output_dtype=torch.float32, + ) + # Apply routing weights and sum + return ( + (out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)) + .sum(dim=1) + .to(a.dtype) + ) + + +def native_fp8_fused_moe(a, w1, w2, topk_weight, topk_ids, topk): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D).float() + out = torch.zeros(B * topk, w2.shape[1], dtype=torch.float32, device=a.device) + + # Calculate routing + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + ic0 = torch.matmul(a[mask], w1[i].transpose(0, 1)) + ic1 = SiluAndMul(ic0) + out[mask] = torch.matmul(ic1, w2[i].transpose(0, 1)) + + return ( + (out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)) + .sum(dim=1) + .to(a.dtype) + ) + + +def make_non_contiguous(x: torch.Tensor) -> torch.Tensor: + """ + Make a tensor non-contiguous by slicing it via last dimension. + """ + last_dim = x.shape[-1] + return x[..., : last_dim // 2] if x.is_contiguous() else x diff --git a/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json b/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json new file mode 100644 index 000000000..f1652ec96 --- /dev/null +++ b/test/srt/double-sparsity-config-Llama-3.1-8B-Instruct.json @@ -0,0 +1 @@ +{"model.layers.0.self_attn.q_proj": [[39, 106, 104, 102, 33, 95, 13, 44, 11, 29, 12, 10, 53, 126, 27, 114, 121, 8, 124, 113, 112, 15, 23, 89, 69, 111, 54, 80, 4, 1, 20, 24, 83, 63, 115, 122, 66, 42, 22, 110, 3, 73, 21, 61, 97, 19, 25, 88, 117, 119, 116, 85, 70, 5, 56, 118, 68, 123, 2, 86, 71, 127, 93, 49, 109, 50, 67, 52, 91, 40, 17, 108, 60, 55, 78, 62, 65, 47, 6, 87, 51, 84, 58, 82, 94, 79, 57, 103, 76, 48, 0, 18, 81, 96, 9, 31, 16, 92, 26, 43, 30, 37, 74, 100, 46, 38, 32, 14, 90, 36, 101, 75, 35, 125, 45, 7, 72, 41, 77, 105, 28, 59, 34, 99, 98, 107, 120, 64], [39, 104, 102, 106, 33, 44, 95, 53, 126, 111, 114, 110, 29, 14, 113, 54, 121, 112, 109, 80, 119, 12, 63, 25, 78, 123, 42, 10, 24, 127, 108, 115, 122, 60, 116, 97, 9, 93, 61, 17, 117, 56, 82, 91, 21, 124, 62, 0, 43, 19, 31, 26, 49, 52, 57, 59, 1, 50, 41, 15, 46, 86, 88, 81, 118, 125, 2, 3, 55, 105, 84, 75, 120, 70, 47, 85, 32, 51, 37, 16, 22, 23, 89, 87, 67, 30, 68, 69, 77, 48, 66, 27, 36, 107, 20, 76, 94, 79, 96, 83, 99, 38, 100, 18, 8, 35, 28, 101, 13, 72, 92, 7, 4, 98, 90, 103, 6, 34, 74, 11, 65, 40, 71, 45, 58, 64, 73, 5], [106, 104, 39, 102, 2, 44, 95, 53, 121, 3, 4, 33, 1, 0, 122, 111, 126, 115, 63, 71, 119, 112, 113, 70, 109, 29, 61, 52, 114, 54, 127, 21, 10, 72, 5, 110, 116, 49, 123, 11, 12, 55, 24, 45, 15, 19, 117, 73, 80, 50, 6, 23, 64, 124, 75, 69, 25, 83, 14, 77, 97, 42, 78, 13, 79, 85, 27, 56, 87, 86, 57, 17, 118, 91, 81, 74, 65, 76, 9, 41, 88, 18, 51, 22, 62, 93, 89, 67, 105, 84, 82, 38, 48, 120, 60, 47, 26, 20, 43, 68, 28, 66, 108, 59, 58, 8, 125, 98, 46, 31, 90, 36, 94, 92, 34, 107, 35, 7, 99, 100, 30, 32, 37, 101, 16, 96, 40, 103], [39, 104, 106, 102, 33, 44, 53, 111, 113, 95, 54, 109, 41, 116, 124, 63, 121, 122, 61, 56, 114, 110, 60, 108, 127, 112, 125, 43, 52, 126, 12, 80, 119, 59, 62, 82, 123, 97, 117, 25, 37, 0, 88, 29, 57, 120, 26, 91, 115, 78, 24, 21, 118, 49, 93, 1, 42, 14, 58, 107, 10, 84, 67, 7, 9, 86, 87, 3, 50, 51, 64, 98, 18, 20, 72, 66, 100, 17, 32, 81, 69, 79, 2, 101, 105, 85, 15, 77, 31, 22, 75, 70, 68, 90, 55, 19, 89, 11, 74, 76, 8, 38, 30, 4, 65, 5, 45, 92, 23, 46, 73, 13, 94, 47, 35, 36, 6, 48, 34, 28, 99, 27, 96, 103, 71, 83, 16, 40], [41, 103, 38, 33, 42, 93, 109, 95, 88, 44, 107, 17, 25, 20, 110, 21, 47, 40, 78, 116, 115, 126, 59, 61, 64, 57, 48, 83, 119, 12, 45, 125, 11, 18, 124, 118, 53, 56, 89, 70, 22, 77, 86, 50, 92, 4, 87, 24, 97, 65, 49, 73, 26, 62, 3, 16, 108, 66, 72, 104, 67, 80, 58, 71, 79, 23, 91, 122, 19, 43, 113, 39, 14, 102, 111, 27, 9, 121, 10, 117, 29, 60, 69, 90, 76, 120, 0, 84, 15, 114, 51, 74, 68, 98, 6, 81, 85, 2, 1, 82, 100, 112, 99, 13, 52, 123, 94, 8, 35, 28, 55, 96, 63, 54, 7, 105, 5, 36, 127, 37, 32, 75, 34, 106, 31, 30, 101, 46], [38, 103, 44, 41, 107, 33, 42, 88, 95, 40, 93, 87, 25, 78, 110, 47, 48, 26, 20, 18, 21, 17, 64, 59, 45, 22, 109, 89, 118, 62, 91, 115, 116, 53, 49, 61, 126, 90, 16, 11, 119, 92, 111, 65, 125, 124, 43, 85, 83, 60, 24, 4, 77, 73, 104, 117, 57, 120, 86, 3, 76, 97, 29, 23, 102, 39, 51, 67, 70, 12, 122, 71, 9, 15, 30, 113, 56, 66, 0, 106, 7, 50, 108, 100, 114, 28, 46, 121, 80, 94, 72, 10, 6, 79, 69, 55, 19, 58, 74, 68, 123, 81, 27, 13, 2, 82, 98, 35, 8, 34, 36, 1, 101, 5, 105, 54, 63, 96, 37, 84, 127, 52, 32, 31, 14, 75, 112, 99], [103, 41, 107, 33, 38, 93, 110, 116, 64, 42, 48, 20, 123, 61, 124, 17, 126, 115, 88, 49, 25, 53, 47, 78, 57, 122, 21, 59, 95, 18, 44, 89, 77, 97, 16, 11, 125, 66, 86, 120, 119, 109, 4, 65, 12, 118, 71, 62, 22, 24, 80, 14, 50, 74, 73, 51, 70, 3, 7, 121, 67, 83, 27, 111, 72, 5, 69, 76, 23, 45, 84, 29, 60, 9, 31, 10, 81, 26, 8, 87, 19, 13, 0, 40, 2, 56, 113, 85, 82, 127, 91, 43, 34, 108, 117, 114, 6, 15, 96, 106, 79, 75, 94, 102, 100, 101, 68, 98, 63, 58, 99, 105, 55, 30, 90, 36, 1, 35, 32, 46, 112, 28, 104, 39, 37, 92, 52, 54], [103, 41, 33, 38, 42, 93, 95, 107, 110, 25, 59, 17, 48, 116, 88, 109, 44, 57, 11, 83, 20, 125, 78, 21, 62, 86, 115, 70, 97, 77, 4, 118, 18, 47, 43, 64, 12, 24, 50, 126, 22, 124, 45, 3, 61, 72, 65, 39, 89, 84, 113, 53, 73, 29, 49, 120, 66, 80, 79, 27, 71, 122, 10, 117, 40, 9, 92, 16, 67, 81, 19, 23, 119, 123, 51, 105, 36, 8, 15, 55, 108, 111, 60, 90, 14, 26, 87, 104, 74, 56, 91, 75, 0, 28, 69, 68, 100, 2, 102, 99, 58, 52, 85, 6, 63, 34, 54, 94, 127, 96, 76, 32, 121, 30, 1, 31, 7, 114, 82, 46, 37, 13, 112, 35, 5, 101, 98, 106], [102, 41, 112, 101, 108, 103, 33, 104, 29, 27, 115, 30, 42, 38, 16, 45, 114, 20, 32, 100, 121, 89, 98, 111, 87, 62, 59, 126, 13, 63, 107, 11, 43, 88, 117, 75, 118, 25, 84, 34, 31, 50, 51, 55, 23, 95, 56, 58, 53, 78, 94, 97, 82, 92, 8, 18, 127, 113, 110, 19, 40, 21, 35, 39, 85, 119, 37, 125, 96, 54, 6, 72, 90, 91, 14, 26, 124, 105, 9, 44, 28, 17, 116, 120, 60, 48, 80, 57, 1, 123, 79, 77, 86, 24, 69, 73, 15, 122, 68, 99, 81, 74, 4, 52, 71, 67, 7, 65, 76, 66, 61, 109, 106, 46, 93, 2, 64, 12, 10, 49, 3, 47, 83, 70, 36, 22, 0, 5], [107, 104, 102, 33, 108, 45, 95, 92, 29, 42, 103, 21, 99, 43, 85, 101, 23, 50, 27, 28, 113, 98, 19, 15, 120, 83, 26, 46, 74, 97, 34, 12, 86, 123, 52, 127, 90, 79, 39, 7, 17, 41, 112, 49, 126, 77, 48, 38, 125, 89, 44, 61, 111, 18, 100, 115, 32, 25, 35, 119, 93, 62, 30, 60, 84, 96, 31, 80, 122, 87, 114, 53, 55, 47, 56, 110, 13, 57, 63, 37, 51, 54, 116, 117, 68, 124, 105, 78, 40, 81, 118, 88, 9, 91, 11, 109, 76, 24, 59, 71, 121, 82, 6, 20, 1, 72, 106, 16, 58, 73, 5, 94, 67, 10, 69, 14, 4, 22, 75, 2, 36, 70, 8, 66, 3, 65, 64, 0], [102, 103, 33, 29, 41, 42, 104, 45, 64, 1, 107, 108, 66, 27, 16, 112, 88, 115, 3, 50, 101, 20, 127, 67, 118, 75, 58, 116, 53, 72, 13, 84, 68, 19, 86, 25, 89, 70, 11, 30, 21, 77, 117, 4, 6, 14, 56, 59, 69, 82, 80, 100, 126, 95, 32, 92, 97, 39, 85, 124, 79, 90, 121, 52, 54, 26, 5, 113, 71, 7, 123, 73, 51, 8, 0, 63, 111, 38, 23, 57, 15, 119, 10, 43, 60, 78, 22, 83, 87, 9, 74, 37, 99, 98, 76, 65, 61, 55, 17, 81, 18, 31, 24, 94, 28, 2, 47, 44, 40, 110, 114, 96, 105, 46, 109, 120, 34, 48, 12, 35, 62, 93, 106, 49, 91, 122, 125, 36], [103, 45, 102, 33, 26, 42, 95, 87, 79, 123, 104, 19, 113, 112, 41, 29, 21, 76, 108, 120, 30, 71, 10, 97, 81, 27, 101, 74, 44, 32, 115, 122, 89, 17, 12, 23, 7, 92, 25, 53, 15, 100, 90, 117, 13, 39, 105, 99, 83, 51, 85, 84, 50, 4, 20, 109, 61, 88, 35, 16, 98, 54, 94, 110, 125, 55, 66, 127, 70, 47, 65, 124, 116, 46, 80, 67, 114, 49, 57, 56, 78, 38, 86, 18, 60, 82, 11, 5, 121, 119, 126, 62, 48, 58, 59, 28, 111, 93, 37, 63, 2, 73, 52, 6, 8, 24, 3, 96, 106, 43, 40, 107, 1, 34, 118, 22, 0, 68, 69, 77, 91, 64, 72, 36, 75, 31, 14, 9], [104, 103, 33, 43, 44, 111, 102, 90, 49, 89, 8, 41, 93, 37, 116, 113, 95, 122, 123, 23, 74, 42, 14, 17, 83, 68, 85, 106, 91, 92, 13, 78, 63, 127, 115, 81, 112, 87, 121, 3, 70, 48, 72, 120, 28, 124, 53, 71, 30, 19, 47, 86, 59, 51, 96, 56, 94, 114, 108, 118, 76, 79, 55, 38, 29, 110, 5, 82, 15, 21, 11, 20, 26, 66, 6, 84, 22, 50, 9, 27, 100, 65, 61, 64, 57, 25, 35, 7, 16, 101, 80, 45, 52, 36, 46, 109, 107, 24, 126, 2, 4, 12, 67, 125, 75, 10, 32, 117, 97, 1, 69, 60, 77, 88, 98, 99, 73, 105, 58, 18, 62, 54, 34, 31, 0, 119, 40, 39], [103, 104, 102, 33, 43, 44, 111, 90, 115, 106, 113, 89, 42, 74, 56, 93, 48, 95, 85, 127, 41, 72, 92, 37, 107, 71, 17, 87, 123, 112, 122, 50, 83, 22, 55, 57, 116, 14, 53, 124, 49, 114, 26, 47, 110, 23, 63, 70, 98, 100, 51, 108, 121, 59, 13, 35, 28, 81, 94, 79, 32, 120, 29, 76, 101, 105, 126, 38, 7, 82, 9, 6, 118, 24, 125, 91, 20, 88, 15, 99, 36, 109, 18, 62, 34, 46, 84, 19, 3, 61, 2, 30, 11, 27, 86, 52, 58, 75, 25, 68, 12, 31, 117, 69, 54, 96, 119, 65, 78, 80, 8, 45, 16, 77, 40, 1, 21, 73, 10, 4, 66, 64, 60, 67, 0, 39, 97, 5], [104, 44, 103, 43, 33, 49, 111, 102, 95, 41, 37, 123, 27, 91, 9, 84, 42, 89, 61, 122, 24, 76, 124, 116, 30, 85, 79, 96, 51, 48, 118, 28, 87, 92, 98, 127, 59, 23, 109, 121, 20, 15, 88, 29, 100, 106, 53, 34, 35, 101, 126, 16, 120, 54, 108, 115, 47, 93, 83, 17, 56, 18, 90, 113, 55, 38, 22, 63, 8, 81, 13, 94, 72, 117, 60, 50, 114, 36, 12, 71, 45, 57, 105, 32, 110, 119, 62, 74, 58, 97, 31, 112, 125, 52, 19, 46, 86, 5, 80, 68, 73, 3, 78, 99, 107, 6, 70, 75, 40, 14, 65, 21, 39, 7, 25, 77, 64, 2, 11, 82, 26, 10, 69, 4, 0, 66, 1, 67], [102, 33, 44, 104, 43, 103, 79, 28, 27, 87, 95, 9, 12, 124, 49, 21, 23, 99, 76, 90, 85, 35, 32, 31, 29, 80, 94, 61, 98, 114, 84, 91, 18, 15, 73, 89, 81, 106, 112, 62, 51, 41, 54, 56, 58, 59, 117, 45, 119, 88, 26, 110, 105, 52, 115, 42, 111, 24, 30, 92, 109, 38, 123, 37, 108, 113, 57, 118, 60, 125, 82, 25, 86, 75, 96, 19, 16, 14, 40, 46, 77, 20, 101, 93, 122, 126, 36, 53, 100, 55, 47, 71, 39, 11, 72, 34, 50, 63, 116, 48, 107, 70, 121, 13, 120, 17, 127, 78, 83, 3, 22, 7, 6, 5, 8, 66, 74, 97, 69, 10, 68, 67, 64, 4, 65, 0, 1, 2], [39, 41, 101, 97, 29, 110, 24, 43, 32, 49, 121, 84, 120, 63, 3, 56, 44, 8, 59, 21, 38, 98, 40, 6, 60, 127, 53, 71, 2, 25, 52, 116, 99, 13, 62, 4, 125, 72, 46, 18, 23, 7, 31, 68, 85, 15, 92, 42, 79, 76, 115, 123, 87, 75, 109, 57, 108, 95, 102, 12, 54, 0, 36, 66, 61, 19, 50, 124, 94, 65, 111, 82, 69, 126, 77, 80, 83, 106, 113, 51, 33, 100, 114, 91, 14, 20, 78, 10, 47, 27, 117, 86, 48, 118, 67, 89, 11, 112, 58, 16, 35, 88, 30, 45, 28, 5, 17, 70, 93, 73, 90, 34, 105, 22, 26, 55, 64, 119, 107, 9, 122, 104, 103, 96, 81, 1, 74, 37], [39, 41, 97, 32, 101, 43, 29, 26, 116, 31, 54, 80, 99, 55, 125, 18, 21, 59, 24, 78, 73, 111, 23, 76, 71, 16, 87, 82, 119, 3, 25, 28, 11, 121, 112, 12, 123, 63, 98, 120, 85, 83, 47, 91, 94, 56, 13, 103, 7, 33, 17, 88, 51, 127, 4, 95, 104, 19, 38, 108, 60, 86, 100, 79, 124, 77, 30, 27, 109, 81, 52, 50, 84, 61, 67, 37, 93, 49, 92, 5, 10, 118, 46, 58, 69, 74, 110, 14, 53, 45, 122, 126, 57, 113, 114, 15, 42, 107, 68, 22, 72, 9, 62, 90, 89, 70, 115, 48, 102, 8, 75, 36, 40, 6, 117, 105, 2, 0, 96, 20, 106, 44, 35, 65, 66, 1, 34, 64], [6, 39, 41, 70, 101, 72, 8, 43, 74, 60, 108, 53, 92, 110, 98, 113, 50, 32, 44, 47, 61, 87, 62, 95, 28, 111, 26, 38, 58, 57, 67, 124, 18, 52, 56, 51, 99, 97, 112, 123, 118, 125, 116, 59, 109, 33, 40, 89, 120, 106, 19, 121, 30, 46, 14, 31, 68, 126, 88, 11, 4, 104, 24, 45, 2, 63, 127, 10, 119, 105, 80, 90, 107, 83, 34, 114, 36, 78, 3, 27, 69, 42, 55, 9, 20, 73, 122, 5, 15, 86, 25, 49, 48, 29, 21, 94, 66, 100, 22, 0, 54, 65, 75, 84, 17, 96, 93, 91, 35, 77, 12, 71, 115, 102, 103, 79, 7, 23, 81, 117, 37, 85, 16, 13, 64, 76, 1, 82], [41, 39, 70, 101, 10, 32, 19, 43, 87, 97, 11, 83, 46, 14, 60, 4, 123, 15, 125, 38, 44, 108, 56, 86, 16, 9, 118, 53, 18, 5, 109, 2, 92, 76, 78, 62, 61, 23, 120, 127, 24, 68, 49, 59, 42, 66, 84, 75, 21, 110, 8, 27, 106, 72, 63, 111, 47, 52, 121, 50, 73, 126, 28, 40, 17, 99, 67, 107, 79, 122, 89, 82, 33, 124, 105, 77, 58, 26, 1, 85, 25, 114, 119, 51, 22, 57, 98, 48, 31, 71, 94, 103, 115, 116, 13, 7, 102, 74, 113, 3, 20, 96, 6, 0, 81, 90, 69, 45, 104, 29, 34, 100, 12, 36, 91, 88, 80, 64, 54, 95, 30, 55, 35, 65, 93, 37, 112, 117], [40, 102, 31, 107, 22, 39, 78, 98, 9, 6, 73, 88, 12, 81, 17, 67, 76, 106, 25, 70, 48, 85, 86, 110, 91, 92, 114, 74, 82, 2, 58, 15, 83, 59, 14, 27, 108, 117, 127, 66, 79, 111, 47, 30, 51, 50, 5, 21, 57, 52, 109, 36, 89, 23, 35, 49, 101, 63, 72, 90, 122, 54, 100, 26, 84, 56, 124, 33, 18, 55, 62, 126, 29, 123, 11, 95, 119, 20, 96, 28, 77, 19, 24, 4, 87, 32, 115, 37, 99, 41, 53, 97, 125, 112, 8, 104, 71, 43, 93, 3, 116, 65, 16, 10, 64, 75, 94, 113, 120, 105, 68, 60, 46, 103, 7, 61, 69, 34, 13, 44, 0, 118, 80, 45, 121, 1, 42, 38], [40, 73, 70, 14, 39, 17, 110, 102, 114, 85, 82, 91, 51, 106, 99, 49, 48, 41, 98, 83, 36, 75, 115, 46, 88, 119, 92, 30, 69, 57, 28, 32, 15, 109, 126, 63, 86, 31, 123, 84, 6, 37, 72, 107, 90, 76, 77, 53, 71, 60, 22, 23, 124, 55, 93, 100, 21, 33, 58, 111, 81, 8, 24, 18, 125, 19, 44, 112, 79, 113, 59, 2, 74, 65, 62, 97, 35, 0, 61, 45, 78, 89, 101, 5, 4, 66, 67, 50, 42, 122, 11, 117, 26, 16, 108, 56, 3, 10, 9, 121, 25, 116, 94, 1, 105, 118, 12, 47, 120, 20, 96, 43, 29, 87, 64, 52, 13, 104, 80, 127, 7, 38, 54, 95, 27, 34, 68, 103], [40, 99, 41, 107, 9, 66, 96, 37, 91, 36, 73, 24, 58, 111, 94, 31, 47, 86, 114, 50, 92, 89, 10, 52, 29, 59, 34, 6, 39, 18, 127, 44, 100, 22, 76, 21, 56, 122, 121, 57, 102, 79, 70, 12, 46, 84, 81, 85, 63, 116, 72, 88, 43, 110, 5, 83, 87, 45, 69, 27, 3, 2, 8, 78, 51, 115, 106, 67, 74, 7, 54, 118, 19, 42, 104, 124, 11, 97, 4, 75, 105, 35, 33, 62, 55, 28, 30, 0, 82, 77, 95, 93, 120, 98, 20, 25, 117, 32, 53, 17, 90, 26, 71, 108, 64, 14, 23, 101, 48, 49, 65, 15, 119, 125, 103, 80, 68, 112, 1, 126, 13, 16, 113, 123, 61, 109, 38, 60], [40, 41, 102, 37, 92, 32, 90, 88, 96, 107, 23, 31, 74, 39, 84, 97, 25, 75, 82, 35, 79, 10, 91, 15, 7, 4, 18, 116, 20, 98, 87, 99, 69, 33, 86, 111, 100, 122, 121, 127, 65, 58, 13, 8, 56, 47, 95, 9, 50, 114, 81, 110, 59, 118, 72, 48, 77, 17, 29, 24, 21, 52, 104, 22, 54, 28, 94, 83, 85, 51, 57, 14, 73, 36, 16, 19, 66, 76, 71, 11, 93, 78, 120, 27, 6, 108, 5, 117, 43, 49, 63, 3, 67, 2, 61, 26, 64, 70, 68, 0, 101, 105, 124, 62, 55, 30, 80, 89, 103, 34, 44, 113, 45, 60, 1, 38, 106, 42, 115, 109, 12, 125, 119, 126, 53, 112, 123, 46], [103, 105, 96, 47, 59, 45, 42, 30, 19, 93, 14, 112, 113, 32, 86, 0, 127, 104, 38, 21, 84, 57, 48, 89, 72, 124, 87, 81, 51, 80, 55, 44, 25, 63, 126, 94, 118, 123, 34, 52, 68, 26, 40, 7, 115, 27, 3, 33, 88, 11, 120, 8, 125, 1, 23, 66, 70, 110, 77, 5, 107, 39, 50, 73, 58, 46, 13, 119, 6, 91, 9, 69, 15, 17, 76, 74, 43, 54, 24, 79, 95, 75, 22, 82, 78, 90, 12, 121, 92, 16, 71, 31, 114, 4, 117, 116, 18, 20, 67, 61, 53, 97, 37, 28, 41, 49, 35, 60, 36, 122, 99, 10, 85, 2, 109, 100, 101, 108, 83, 102, 56, 111, 65, 62, 98, 29, 64, 106], [104, 42, 103, 96, 45, 34, 47, 93, 32, 89, 27, 30, 84, 94, 26, 21, 81, 112, 116, 87, 105, 19, 51, 102, 86, 25, 38, 88, 80, 15, 41, 23, 119, 33, 20, 118, 52, 111, 114, 113, 44, 110, 85, 57, 98, 125, 124, 13, 48, 54, 11, 107, 43, 73, 55, 37, 106, 12, 100, 59, 10, 76, 24, 127, 77, 82, 115, 60, 14, 72, 101, 68, 90, 121, 29, 91, 0, 63, 97, 58, 75, 9, 61, 46, 126, 108, 4, 39, 56, 18, 22, 28, 70, 117, 120, 53, 109, 66, 6, 123, 3, 1, 95, 92, 49, 31, 50, 122, 99, 40, 69, 5, 62, 67, 83, 79, 16, 36, 64, 78, 2, 8, 74, 7, 35, 65, 71, 17], [103, 104, 96, 42, 45, 105, 30, 19, 93, 59, 47, 89, 113, 14, 86, 112, 84, 21, 32, 124, 80, 87, 57, 81, 44, 0, 72, 127, 88, 115, 25, 26, 48, 63, 119, 123, 51, 68, 118, 34, 52, 15, 55, 98, 126, 61, 43, 73, 24, 110, 102, 46, 107, 125, 121, 23, 116, 114, 41, 56, 54, 66, 27, 8, 3, 70, 91, 76, 6, 11, 39, 13, 1, 33, 97, 74, 4, 94, 22, 77, 58, 7, 37, 95, 82, 50, 36, 17, 117, 120, 53, 122, 75, 16, 90, 79, 67, 111, 40, 5, 64, 9, 2, 78, 18, 38, 31, 71, 60, 65, 12, 100, 108, 92, 85, 69, 29, 49, 99, 101, 62, 20, 109, 28, 35, 10, 106, 83], [42, 104, 103, 96, 45, 30, 89, 112, 34, 27, 113, 32, 102, 47, 93, 26, 105, 84, 94, 21, 81, 86, 88, 87, 124, 25, 59, 80, 44, 15, 19, 38, 114, 20, 57, 85, 52, 51, 23, 118, 127, 125, 119, 43, 100, 29, 33, 41, 46, 116, 82, 13, 126, 48, 11, 107, 77, 73, 98, 72, 122, 106, 54, 37, 95, 55, 24, 76, 12, 115, 10, 14, 97, 39, 110, 68, 101, 123, 56, 63, 92, 4, 0, 6, 91, 53, 90, 121, 50, 40, 61, 111, 22, 75, 9, 18, 16, 70, 117, 31, 60, 3, 58, 99, 66, 49, 74, 67, 1, 108, 109, 79, 2, 120, 35, 62, 64, 36, 28, 7, 83, 8, 69, 65, 5, 78, 71, 17], [100, 39, 31, 50, 91, 25, 47, 97, 96, 42, 116, 40, 90, 60, 41, 46, 19, 70, 44, 89, 22, 109, 48, 33, 95, 77, 27, 107, 2, 43, 101, 0, 18, 17, 108, 80, 49, 13, 53, 7, 10, 103, 64, 45, 21, 94, 12, 74, 79, 76, 118, 115, 124, 5, 73, 120, 35, 56, 99, 28, 55, 81, 54, 110, 14, 114, 126, 6, 68, 125, 127, 123, 93, 112, 113, 38, 58, 57, 61, 122, 32, 121, 75, 62, 84, 88, 119, 111, 59, 106, 3, 63, 72, 52, 30, 36, 26, 51, 86, 34, 98, 102, 29, 117, 16, 9, 8, 67, 83, 105, 11, 71, 65, 15, 4, 69, 1, 24, 104, 87, 66, 20, 82, 37, 92, 78, 23, 85], [40, 25, 64, 101, 42, 41, 45, 110, 72, 113, 56, 112, 108, 68, 107, 111, 114, 39, 2, 61, 125, 55, 119, 51, 97, 63, 31, 121, 22, 57, 54, 126, 59, 74, 123, 80, 87, 96, 52, 62, 18, 120, 7, 48, 0, 84, 58, 28, 91, 70, 115, 93, 11, 5, 14, 49, 60, 17, 30, 76, 66, 44, 19, 24, 88, 77, 3, 100, 13, 50, 1, 79, 109, 69, 90, 95, 99, 102, 104, 33, 53, 71, 21, 73, 8, 26, 106, 65, 75, 116, 117, 46, 98, 105, 89, 92, 67, 47, 35, 34, 81, 12, 4, 6, 122, 36, 78, 85, 10, 94, 43, 82, 83, 9, 118, 86, 103, 15, 127, 37, 16, 38, 27, 124, 23, 20, 32, 29], [0, 39, 22, 42, 96, 41, 91, 18, 10, 107, 79, 40, 14, 21, 19, 50, 77, 71, 80, 88, 70, 84, 48, 5, 45, 85, 44, 17, 11, 81, 100, 46, 28, 60, 76, 83, 72, 110, 66, 63, 57, 36, 47, 109, 2, 108, 56, 125, 120, 35, 111, 93, 61, 73, 114, 24, 112, 49, 55, 113, 87, 90, 121, 30, 64, 4, 99, 126, 116, 15, 51, 103, 52, 54, 6, 82, 97, 43, 115, 86, 119, 123, 58, 7, 101, 117, 69, 53, 13, 33, 59, 9, 62, 32, 68, 75, 31, 8, 3, 1, 12, 92, 29, 20, 78, 65, 104, 37, 27, 106, 122, 74, 16, 23, 34, 95, 25, 118, 26, 89, 94, 67, 98, 38, 105, 102, 127, 124], [108, 54, 112, 119, 59, 62, 113, 110, 123, 111, 26, 51, 61, 107, 55, 125, 89, 115, 114, 45, 121, 98, 58, 57, 63, 48, 52, 49, 34, 109, 122, 41, 126, 43, 127, 38, 32, 95, 56, 94, 100, 33, 120, 53, 50, 24, 124, 29, 40, 101, 44, 90, 25, 85, 8, 102, 60, 97, 88, 86, 83, 27, 42, 118, 46, 99, 66, 105, 4, 12, 19, 10, 47, 116, 96, 13, 82, 22, 16, 92, 30, 64, 31, 37, 117, 72, 28, 91, 87, 106, 69, 2, 103, 17, 9, 36, 70, 5, 84, 65, 76, 104, 75, 11, 39, 0, 78, 73, 20, 77, 23, 3, 6, 81, 68, 15, 74, 79, 80, 21, 35, 7, 93, 18, 71, 14, 67, 1]], "model.layers.0.self_attn.k_proj": [[0, 97, 42, 47, 103, 86, 93, 1, 25, 38, 88, 49, 40, 108, 65, 91, 117, 48, 57, 66, 78, 46, 68, 9, 64, 50, 52, 19, 45, 31, 118, 2, 55, 51, 70, 67, 85, 61, 77, 127, 82, 17, 63, 120, 80, 126, 87, 6, 75, 124, 72, 123, 3, 54, 62, 122, 14, 121, 7, 69, 53, 5, 56, 12, 10, 84, 41, 119, 115, 81, 34, 4, 76, 101, 107, 105, 60, 26, 74, 23, 79, 89, 15, 73, 24, 83, 22, 21, 18, 58, 43, 28, 33, 116, 29, 35, 27, 11, 112, 32, 30, 92, 16, 37, 104, 98, 59, 99, 94, 39, 90, 114, 13, 96, 36, 113, 100, 109, 95, 125, 111, 20, 106, 71, 8, 110, 44, 102], [39, 97, 105, 102, 112, 46, 65, 43, 29, 113, 64, 51, 77, 68, 111, 25, 86, 125, 18, 67, 106, 70, 9, 80, 17, 4, 59, 11, 116, 31, 124, 24, 126, 12, 88, 3, 108, 78, 52, 71, 73, 16, 45, 47, 90, 66, 42, 22, 53, 57, 21, 61, 122, 118, 20, 1, 6, 121, 114, 93, 109, 54, 89, 62, 76, 120, 74, 48, 69, 123, 33, 87, 50, 13, 55, 7, 10, 127, 84, 95, 83, 115, 41, 23, 26, 119, 2, 117, 60, 103, 56, 5, 30, 37, 72, 81, 40, 100, 101, 91, 14, 104, 94, 15, 92, 19, 38, 85, 8, 0, 49, 79, 75, 35, 36, 28, 63, 98, 82, 32, 34, 99, 110, 44, 27, 58, 96, 107], [38, 39, 97, 109, 105, 89, 25, 49, 106, 78, 44, 40, 20, 48, 36, 126, 33, 64, 62, 31, 104, 108, 82, 27, 122, 29, 120, 94, 34, 96, 113, 117, 51, 42, 107, 103, 11, 8, 115, 16, 18, 84, 123, 14, 30, 37, 124, 91, 125, 50, 88, 56, 61, 41, 17, 87, 114, 66, 21, 47, 12, 53, 54, 19, 60, 26, 52, 73, 28, 24, 23, 118, 90, 32, 81, 83, 59, 102, 85, 15, 1, 74, 93, 57, 121, 43, 75, 58, 3, 112, 70, 92, 9, 69, 13, 127, 111, 79, 5, 6, 80, 119, 22, 63, 55, 76, 99, 7, 116, 35, 110, 77, 68, 4, 101, 72, 86, 46, 67, 10, 71, 45, 0, 2, 95, 98, 65, 100], [107, 39, 108, 64, 40, 47, 105, 97, 113, 29, 65, 30, 3, 66, 36, 83, 48, 89, 116, 122, 101, 90, 2, 95, 17, 13, 16, 33, 74, 38, 85, 23, 19, 63, 87, 27, 109, 5, 71, 114, 24, 81, 26, 84, 104, 53, 50, 92, 82, 93, 4, 34, 22, 121, 106, 76, 1, 14, 78, 68, 35, 79, 115, 103, 120, 123, 9, 42, 55, 41, 69, 7, 126, 118, 88, 18, 127, 28, 112, 51, 59, 80, 54, 124, 111, 117, 62, 91, 15, 43, 77, 96, 125, 110, 98, 6, 45, 49, 119, 20, 100, 94, 60, 25, 75, 58, 86, 56, 52, 44, 0, 57, 21, 61, 12, 10, 46, 31, 102, 11, 32, 70, 8, 72, 99, 67, 37, 73], [103, 105, 107, 33, 0, 37, 65, 83, 24, 28, 95, 87, 78, 19, 84, 23, 25, 77, 34, 96, 125, 54, 94, 14, 92, 35, 21, 89, 79, 5, 116, 123, 22, 59, 13, 18, 29, 55, 73, 81, 36, 12, 106, 52, 127, 111, 102, 56, 121, 15, 97, 42, 63, 46, 48, 120, 11, 45, 91, 51, 71, 76, 126, 93, 85, 66, 119, 122, 90, 8, 68, 60, 26, 98, 2, 118, 40, 39, 109, 16, 113, 57, 32, 58, 3, 27, 112, 62, 61, 44, 30, 7, 80, 115, 114, 124, 41, 74, 117, 50, 17, 110, 49, 31, 69, 53, 47, 108, 4, 64, 82, 9, 104, 67, 100, 75, 43, 1, 88, 72, 38, 101, 10, 20, 70, 86, 99, 6], [104, 38, 43, 103, 95, 42, 45, 46, 105, 112, 84, 113, 53, 50, 59, 125, 96, 90, 115, 88, 60, 23, 13, 62, 34, 61, 91, 82, 58, 124, 0, 57, 92, 63, 111, 21, 123, 120, 119, 20, 79, 19, 55, 54, 17, 15, 126, 22, 81, 87, 27, 127, 44, 72, 49, 85, 51, 86, 56, 52, 117, 80, 48, 75, 114, 31, 89, 18, 94, 116, 29, 121, 122, 1, 33, 12, 16, 118, 32, 7, 35, 101, 64, 40, 78, 47, 24, 109, 65, 69, 71, 14, 76, 73, 100, 11, 97, 110, 41, 8, 26, 4, 108, 67, 39, 93, 25, 36, 70, 28, 10, 66, 74, 6, 77, 37, 107, 99, 5, 68, 83, 9, 98, 3, 102, 106, 30, 2], [39, 111, 32, 1, 0, 109, 2, 6, 48, 4, 40, 41, 29, 108, 67, 84, 66, 15, 107, 11, 65, 81, 115, 70, 89, 73, 52, 21, 30, 57, 104, 119, 80, 42, 118, 87, 124, 123, 13, 106, 72, 3, 49, 86, 58, 98, 63, 27, 114, 110, 127, 113, 126, 76, 53, 26, 25, 14, 83, 59, 96, 55, 77, 88, 19, 94, 121, 93, 23, 125, 7, 78, 51, 120, 46, 82, 116, 71, 68, 105, 85, 24, 97, 8, 61, 74, 103, 10, 54, 47, 117, 122, 34, 91, 18, 92, 20, 62, 100, 12, 31, 75, 56, 5, 90, 9, 79, 28, 33, 101, 69, 99, 22, 36, 95, 35, 43, 50, 60, 16, 112, 37, 44, 102, 17, 45, 64, 38], [64, 66, 4, 106, 6, 67, 1, 71, 10, 69, 105, 46, 43, 104, 37, 44, 92, 47, 109, 103, 94, 15, 8, 83, 85, 7, 56, 53, 73, 86, 126, 116, 32, 78, 22, 82, 75, 24, 77, 49, 9, 120, 76, 79, 50, 91, 114, 52, 127, 48, 17, 16, 35, 19, 124, 18, 102, 12, 20, 68, 58, 57, 63, 60, 115, 13, 113, 121, 123, 14, 81, 117, 45, 62, 59, 55, 51, 111, 122, 125, 23, 61, 119, 118, 110, 21, 3, 100, 112, 33, 54, 38, 72, 34, 2, 11, 80, 65, 29, 70, 88, 27, 0, 36, 98, 84, 89, 97, 74, 93, 30, 5, 107, 90, 96, 26, 87, 95, 28, 39, 108, 41, 42, 99, 101, 31, 25, 40]], "model.layers.0.self_attn.qk_proj": [[104, 39, 33, 42, 103, 41, 97, 64, 0, 93, 108, 25, 107, 89, 105, 113, 126, 29, 102, 121, 111, 48, 57, 95, 59, 115, 47, 123, 70, 88, 53, 63, 96, 30, 24, 20, 43, 87, 116, 23, 38, 109, 78, 124, 49, 19, 61, 112, 52, 51, 50, 120, 27, 83, 119, 22, 32, 114, 45, 21, 106, 125, 122, 40, 44, 127, 86, 54, 17, 118, 56, 85, 55, 14, 84, 46, 65, 81, 68, 1, 91, 3, 62, 4, 66, 18, 82, 77, 9, 79, 117, 2, 90, 92, 110, 8, 80, 67, 13, 76, 15, 75, 60, 12, 26, 16, 31, 11, 73, 10, 58, 71, 74, 7, 5, 69, 28, 101, 6, 34, 98, 94, 72, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 107, 105, 25, 113, 126, 29, 89, 95, 102, 111, 59, 47, 38, 123, 121, 124, 88, 115, 50, 53, 106, 116, 48, 85, 119, 30, 70, 43, 63, 40, 87, 96, 24, 21, 57, 49, 23, 122, 27, 20, 22, 44, 114, 109, 52, 19, 120, 54, 61, 45, 67, 127, 17, 78, 32, 81, 83, 84, 62, 56, 125, 86, 51, 46, 14, 1, 118, 92, 112, 55, 79, 26, 18, 12, 2, 82, 15, 76, 65, 90, 3, 77, 91, 66, 117, 110, 73, 74, 4, 31, 9, 16, 13, 68, 75, 10, 80, 60, 8, 7, 58, 11, 71, 28, 98, 94, 6, 5, 72, 69, 34, 101, 100, 35, 36, 37, 99], [104, 39, 33, 42, 103, 41, 97, 108, 93, 0, 64, 25, 89, 29, 105, 107, 113, 95, 126, 102, 59, 123, 111, 115, 121, 47, 57, 88, 63, 24, 48, 124, 96, 38, 53, 116, 30, 87, 19, 52, 119, 43, 23, 20, 21, 40, 61, 127, 27, 109, 51, 84, 50, 22, 112, 106, 49, 32, 78, 86, 125, 14, 85, 54, 56, 17, 45, 120, 114, 70, 122, 44, 3, 118, 81, 46, 83, 65, 62, 82, 1, 91, 92, 15, 55, 76, 2, 117, 110, 12, 90, 79, 80, 26, 67, 16, 18, 77, 68, 11, 66, 13, 73, 9, 4, 31, 75, 6, 58, 74, 60, 72, 71, 10, 8, 28, 69, 7, 34, 98, 5, 94, 101, 100, 37, 35, 36, 99], [39, 104, 42, 33, 103, 41, 97, 108, 64, 0, 105, 93, 107, 113, 126, 89, 25, 29, 59, 124, 123, 111, 119, 63, 102, 115, 121, 95, 53, 47, 48, 50, 57, 87, 106, 96, 116, 43, 88, 38, 49, 30, 61, 23, 24, 22, 54, 20, 51, 6, 122, 44, 114, 27, 40, 109, 52, 78, 85, 125, 21, 83, 45, 17, 19, 118, 86, 84, 112, 62, 65, 81, 56, 55, 14, 120, 79, 127, 32, 68, 66, 67, 1, 46, 3, 9, 91, 110, 18, 4, 12, 73, 82, 26, 2, 72, 70, 92, 77, 15, 117, 76, 13, 16, 90, 58, 80, 75, 10, 31, 60, 74, 11, 7, 71, 69, 28, 5, 98, 94, 101, 34, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 108, 0, 93, 105, 25, 89, 107, 113, 126, 115, 102, 59, 29, 123, 47, 95, 119, 121, 63, 53, 111, 124, 57, 48, 116, 43, 6, 20, 30, 23, 88, 96, 24, 106, 109, 38, 87, 50, 52, 22, 21, 49, 27, 17, 40, 61, 19, 112, 122, 78, 3, 54, 14, 44, 45, 114, 51, 91, 84, 86, 46, 65, 85, 125, 1, 127, 118, 120, 32, 81, 56, 62, 82, 83, 26, 73, 72, 55, 15, 9, 2, 110, 4, 79, 66, 76, 92, 117, 12, 67, 18, 16, 77, 11, 13, 68, 31, 80, 90, 75, 60, 58, 10, 74, 71, 70, 7, 98, 34, 28, 69, 5, 94, 101, 8, 100, 37, 36, 35, 99], [39, 104, 42, 33, 103, 41, 97, 108, 0, 64, 25, 93, 89, 105, 107, 126, 95, 113, 29, 102, 115, 123, 59, 48, 121, 53, 124, 6, 30, 88, 111, 47, 87, 63, 96, 106, 116, 23, 119, 38, 27, 24, 43, 57, 20, 49, 40, 22, 61, 44, 50, 54, 52, 122, 21, 109, 1, 51, 86, 81, 85, 19, 78, 114, 3, 84, 45, 125, 112, 32, 127, 14, 83, 118, 120, 17, 2, 46, 56, 55, 76, 62, 82, 91, 92, 18, 72, 77, 65, 79, 9, 117, 15, 12, 66, 90, 110, 67, 68, 4, 26, 16, 31, 13, 80, 73, 75, 10, 74, 60, 7, 11, 71, 58, 98, 5, 94, 28, 34, 70, 69, 101, 8, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 97, 41, 0, 64, 93, 108, 113, 105, 107, 25, 89, 126, 47, 29, 95, 102, 30, 121, 59, 123, 115, 48, 111, 53, 6, 124, 88, 24, 50, 119, 49, 116, 27, 38, 106, 63, 43, 57, 96, 87, 23, 21, 32, 20, 125, 52, 61, 22, 109, 112, 44, 85, 45, 51, 19, 83, 122, 114, 40, 14, 54, 55, 78, 17, 120, 2, 84, 86, 62, 1, 118, 127, 66, 91, 3, 56, 72, 110, 4, 81, 18, 82, 77, 46, 68, 26, 79, 65, 16, 13, 9, 15, 76, 67, 12, 80, 92, 75, 117, 31, 90, 73, 60, 74, 58, 10, 11, 71, 7, 28, 70, 5, 69, 98, 34, 94, 101, 8, 100, 37, 35, 36, 99], [39, 33, 104, 42, 103, 97, 41, 93, 64, 108, 0, 25, 105, 113, 89, 107, 126, 29, 95, 47, 59, 102, 48, 88, 53, 115, 116, 123, 124, 111, 24, 57, 30, 38, 96, 63, 87, 52, 121, 19, 43, 23, 27, 20, 50, 119, 51, 85, 32, 6, 40, 21, 106, 49, 109, 61, 127, 78, 125, 86, 122, 84, 22, 54, 81, 120, 45, 112, 14, 114, 91, 17, 83, 56, 46, 44, 118, 2, 67, 16, 90, 110, 66, 62, 1, 117, 4, 79, 65, 76, 92, 72, 68, 82, 15, 18, 75, 13, 26, 55, 12, 9, 3, 80, 31, 77, 11, 73, 10, 58, 74, 60, 70, 28, 7, 98, 71, 34, 94, 101, 69, 5, 100, 37, 8, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 64, 0, 108, 93, 105, 107, 113, 25, 95, 89, 29, 126, 48, 59, 124, 102, 88, 47, 38, 115, 123, 111, 53, 50, 106, 21, 27, 44, 30, 96, 43, 87, 116, 121, 109, 52, 23, 24, 119, 40, 63, 49, 57, 85, 19, 22, 114, 86, 122, 20, 127, 78, 54, 83, 56, 51, 17, 32, 84, 61, 14, 81, 90, 125, 120, 112, 45, 46, 118, 66, 1, 70, 82, 15, 62, 12, 79, 91, 55, 2, 76, 26, 65, 67, 18, 92, 6, 110, 9, 117, 73, 3, 72, 77, 16, 31, 68, 10, 13, 80, 74, 75, 4, 60, 11, 28, 58, 7, 98, 94, 34, 71, 5, 69, 8, 101, 100, 37, 35, 36, 99], [33, 104, 39, 42, 103, 41, 97, 89, 25, 93, 108, 29, 64, 95, 0, 113, 102, 107, 105, 126, 38, 87, 59, 88, 96, 48, 27, 24, 53, 47, 32, 116, 30, 115, 124, 85, 123, 19, 21, 40, 106, 23, 57, 43, 109, 121, 111, 22, 44, 20, 83, 63, 122, 78, 52, 50, 84, 86, 81, 49, 127, 2, 56, 45, 17, 26, 119, 66, 14, 114, 54, 61, 51, 125, 70, 91, 82, 4, 120, 112, 76, 12, 46, 118, 92, 18, 1, 67, 15, 55, 117, 68, 79, 65, 62, 80, 16, 90, 13, 77, 73, 74, 72, 31, 75, 9, 11, 10, 3, 110, 71, 7, 58, 60, 6, 28, 8, 34, 94, 98, 101, 69, 5, 100, 37, 36, 35, 99], [39, 104, 33, 42, 103, 0, 41, 64, 97, 108, 25, 93, 113, 105, 107, 126, 89, 59, 29, 115, 111, 47, 57, 121, 102, 63, 123, 48, 95, 124, 70, 52, 88, 96, 116, 53, 119, 43, 87, 61, 49, 23, 24, 30, 38, 50, 19, 125, 106, 20, 51, 21, 32, 109, 56, 22, 54, 14, 78, 1, 112, 120, 83, 67, 45, 40, 114, 44, 122, 118, 84, 127, 27, 46, 86, 81, 68, 65, 3, 17, 62, 85, 55, 91, 110, 2, 79, 18, 66, 77, 82, 4, 76, 9, 13, 117, 92, 15, 90, 26, 12, 73, 60, 11, 16, 31, 74, 58, 75, 80, 10, 8, 71, 7, 72, 28, 5, 69, 6, 98, 101, 94, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 0, 41, 64, 108, 97, 107, 105, 113, 93, 126, 25, 89, 59, 121, 63, 111, 115, 70, 123, 47, 29, 95, 102, 119, 48, 124, 96, 53, 116, 49, 57, 51, 88, 52, 87, 50, 61, 27, 30, 65, 109, 43, 106, 22, 120, 24, 23, 21, 114, 38, 56, 19, 122, 40, 54, 14, 45, 125, 118, 83, 84, 112, 20, 46, 44, 127, 55, 32, 86, 81, 78, 62, 17, 67, 82, 85, 3, 91, 2, 90, 117, 79, 66, 77, 92, 68, 4, 110, 76, 15, 13, 60, 73, 26, 31, 1, 9, 12, 75, 18, 80, 8, 58, 74, 11, 16, 71, 10, 28, 7, 69, 5, 98, 72, 34, 94, 6, 101, 100, 37, 36, 35, 99], [39, 33, 104, 42, 103, 97, 41, 108, 89, 93, 95, 105, 0, 113, 25, 64, 29, 126, 107, 30, 102, 27, 124, 48, 38, 47, 115, 53, 106, 88, 24, 59, 23, 116, 123, 21, 96, 40, 32, 87, 86, 111, 50, 121, 44, 85, 43, 22, 122, 20, 109, 63, 49, 70, 57, 119, 52, 83, 81, 19, 51, 14, 91, 17, 114, 118, 78, 54, 127, 125, 45, 112, 61, 26, 92, 84, 2, 56, 76, 66, 18, 15, 82, 12, 65, 62, 79, 46, 1, 120, 117, 80, 55, 73, 31, 68, 90, 110, 16, 67, 77, 3, 8, 13, 75, 74, 4, 9, 6, 10, 11, 60, 34, 58, 71, 28, 98, 94, 7, 72, 69, 101, 5, 100, 36, 37, 35, 99], [33, 104, 39, 42, 103, 41, 97, 93, 108, 89, 25, 0, 64, 95, 105, 113, 29, 107, 126, 102, 121, 115, 53, 30, 88, 38, 48, 47, 96, 59, 124, 123, 57, 43, 87, 21, 111, 116, 22, 23, 27, 63, 32, 40, 51, 52, 50, 106, 20, 109, 86, 44, 83, 122, 24, 14, 49, 78, 119, 127, 19, 112, 84, 81, 61, 54, 17, 114, 85, 55, 120, 91, 45, 46, 70, 18, 82, 62, 79, 65, 12, 2, 56, 92, 3, 66, 118, 4, 125, 90, 73, 26, 117, 13, 1, 68, 8, 80, 15, 76, 6, 31, 67, 77, 110, 16, 11, 9, 75, 10, 60, 74, 28, 7, 71, 5, 58, 98, 34, 101, 94, 69, 100, 72, 37, 35, 36, 99], [39, 104, 33, 42, 103, 0, 97, 41, 108, 93, 64, 113, 25, 107, 126, 47, 105, 89, 59, 29, 115, 124, 48, 63, 121, 123, 53, 95, 57, 111, 116, 102, 52, 30, 96, 24, 49, 88, 119, 51, 87, 23, 38, 20, 106, 78, 32, 61, 83, 44, 43, 27, 21, 22, 50, 127, 109, 122, 86, 54, 85, 19, 114, 6, 17, 125, 14, 3, 40, 112, 4, 81, 118, 46, 45, 56, 65, 62, 84, 2, 55, 120, 82, 66, 8, 91, 18, 76, 68, 79, 26, 117, 1, 12, 9, 11, 67, 73, 90, 70, 13, 15, 16, 110, 92, 77, 31, 60, 75, 80, 71, 10, 7, 58, 74, 28, 5, 98, 34, 69, 94, 72, 101, 37, 100, 35, 36, 99], [39, 104, 33, 42, 103, 41, 64, 0, 97, 108, 93, 113, 25, 105, 107, 89, 126, 115, 29, 102, 59, 123, 95, 111, 47, 6, 48, 121, 53, 88, 63, 57, 119, 124, 43, 49, 116, 30, 52, 109, 24, 38, 106, 87, 50, 96, 61, 20, 23, 21, 19, 44, 27, 114, 14, 122, 125, 51, 54, 85, 22, 112, 56, 40, 120, 84, 83, 118, 127, 17, 65, 62, 78, 55, 45, 46, 32, 66, 81, 91, 3, 86, 1, 67, 8, 76, 18, 79, 117, 68, 2, 73, 110, 26, 90, 82, 77, 9, 4, 13, 92, 12, 16, 80, 74, 15, 60, 31, 11, 75, 10, 7, 58, 28, 71, 70, 94, 69, 5, 98, 34, 101, 72, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 64, 105, 107, 25, 89, 95, 113, 126, 29, 102, 59, 30, 111, 121, 47, 123, 88, 38, 124, 115, 87, 6, 24, 43, 106, 50, 53, 48, 116, 119, 40, 27, 21, 23, 96, 86, 109, 85, 63, 20, 44, 83, 32, 122, 22, 84, 49, 52, 57, 17, 14, 91, 114, 81, 51, 19, 61, 54, 127, 45, 56, 65, 120, 92, 125, 90, 79, 66, 2, 76, 78, 112, 82, 46, 118, 26, 15, 18, 62, 55, 80, 3, 13, 67, 8, 68, 9, 31, 12, 117, 1, 74, 73, 16, 110, 4, 77, 11, 75, 10, 60, 58, 71, 7, 28, 34, 94, 98, 5, 72, 69, 100, 70, 101, 35, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 93, 64, 0, 89, 29, 105, 113, 25, 126, 107, 95, 115, 47, 124, 111, 116, 88, 59, 48, 121, 123, 53, 57, 87, 102, 24, 38, 63, 30, 20, 23, 96, 43, 21, 6, 40, 127, 22, 106, 119, 52, 50, 27, 51, 109, 49, 61, 32, 54, 17, 78, 84, 44, 112, 114, 19, 83, 118, 85, 82, 56, 120, 45, 122, 14, 125, 81, 1, 46, 117, 86, 65, 91, 67, 62, 68, 26, 2, 92, 90, 55, 4, 79, 12, 77, 66, 15, 18, 13, 76, 31, 3, 9, 110, 16, 73, 75, 80, 11, 74, 10, 60, 58, 8, 7, 98, 71, 70, 5, 72, 69, 28, 34, 94, 101, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 64, 97, 41, 0, 108, 93, 107, 113, 89, 105, 126, 25, 29, 59, 121, 115, 102, 95, 47, 111, 123, 124, 48, 63, 53, 57, 88, 119, 43, 30, 38, 24, 50, 116, 96, 49, 61, 51, 106, 87, 20, 52, 54, 19, 23, 21, 44, 27, 22, 81, 6, 14, 125, 122, 85, 32, 112, 114, 109, 127, 120, 55, 78, 62, 40, 84, 83, 56, 2, 45, 17, 86, 118, 46, 1, 67, 82, 91, 65, 3, 110, 68, 79, 4, 66, 12, 26, 77, 18, 73, 15, 90, 117, 92, 76, 9, 70, 13, 80, 11, 16, 10, 75, 31, 74, 58, 60, 7, 71, 69, 72, 8, 28, 5, 98, 94, 34, 101, 37, 100, 35, 99, 36], [39, 33, 104, 42, 103, 41, 108, 97, 0, 64, 113, 25, 93, 89, 107, 105, 29, 95, 126, 102, 115, 59, 48, 124, 123, 121, 111, 38, 88, 53, 63, 21, 24, 47, 106, 116, 30, 87, 23, 49, 22, 27, 96, 119, 85, 43, 20, 57, 122, 40, 52, 109, 44, 65, 32, 14, 83, 17, 81, 51, 19, 61, 50, 54, 114, 125, 78, 91, 127, 3, 84, 112, 70, 79, 56, 86, 67, 120, 46, 45, 12, 62, 18, 2, 118, 26, 76, 66, 73, 55, 82, 92, 4, 16, 13, 6, 90, 1, 15, 9, 117, 110, 31, 77, 80, 74, 10, 68, 75, 72, 7, 71, 11, 28, 58, 60, 94, 98, 34, 5, 8, 69, 101, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 41, 64, 97, 108, 0, 93, 105, 113, 89, 107, 59, 25, 29, 126, 95, 123, 47, 121, 48, 111, 115, 102, 63, 124, 70, 57, 87, 119, 53, 116, 88, 30, 109, 49, 43, 50, 24, 56, 96, 114, 61, 38, 52, 65, 51, 106, 40, 127, 20, 44, 83, 85, 120, 118, 54, 22, 86, 23, 46, 21, 19, 122, 78, 112, 27, 125, 81, 45, 14, 32, 62, 17, 84, 67, 91, 82, 90, 117, 15, 1, 55, 18, 3, 110, 76, 12, 13, 26, 66, 68, 77, 60, 72, 31, 2, 4, 9, 16, 75, 79, 92, 58, 73, 11, 10, 80, 6, 74, 7, 98, 28, 71, 69, 5, 34, 94, 100, 8, 101, 37, 35, 36, 99], [39, 104, 33, 42, 103, 41, 97, 108, 107, 0, 93, 113, 64, 105, 126, 89, 25, 95, 29, 30, 47, 59, 123, 102, 88, 124, 121, 111, 70, 115, 38, 43, 116, 24, 48, 53, 119, 106, 50, 19, 96, 63, 87, 57, 44, 21, 49, 23, 109, 27, 85, 52, 22, 14, 40, 51, 122, 20, 32, 56, 54, 114, 61, 17, 84, 127, 125, 2, 120, 62, 4, 83, 81, 67, 86, 78, 55, 79, 66, 112, 82, 118, 46, 26, 91, 68, 65, 1, 45, 15, 76, 16, 72, 18, 92, 117, 73, 12, 3, 9, 90, 77, 11, 110, 74, 13, 31, 80, 10, 71, 60, 58, 75, 7, 28, 94, 98, 69, 8, 34, 5, 101, 100, 37, 6, 35, 36, 99], [39, 104, 33, 42, 103, 41, 0, 97, 108, 105, 93, 64, 25, 89, 113, 107, 95, 29, 126, 102, 115, 59, 111, 123, 70, 124, 48, 53, 87, 24, 88, 121, 106, 47, 38, 27, 63, 21, 116, 119, 43, 50, 96, 22, 30, 40, 20, 23, 19, 57, 52, 49, 122, 114, 109, 85, 44, 54, 17, 51, 84, 78, 81, 65, 61, 3, 45, 112, 86, 120, 125, 56, 127, 32, 14, 91, 62, 82, 118, 46, 15, 92, 12, 79, 1, 18, 110, 26, 9, 76, 72, 83, 73, 67, 55, 117, 2, 80, 16, 66, 31, 77, 13, 10, 75, 4, 68, 90, 11, 60, 58, 7, 74, 6, 28, 34, 98, 5, 69, 71, 94, 101, 100, 35, 8, 36, 37, 99], [39, 104, 33, 42, 103, 41, 97, 108, 0, 64, 25, 93, 107, 105, 89, 29, 113, 95, 126, 102, 115, 123, 48, 124, 88, 53, 59, 121, 30, 87, 47, 38, 96, 22, 63, 111, 106, 23, 43, 27, 49, 24, 65, 40, 116, 20, 119, 50, 44, 109, 122, 52, 114, 57, 51, 21, 85, 14, 84, 54, 70, 83, 81, 127, 61, 17, 86, 82, 19, 32, 79, 56, 112, 46, 78, 118, 45, 120, 62, 91, 55, 2, 117, 125, 3, 12, 18, 9, 72, 92, 13, 76, 26, 16, 66, 31, 1, 67, 90, 68, 110, 11, 77, 10, 73, 15, 80, 6, 4, 60, 75, 71, 74, 58, 98, 28, 34, 7, 94, 5, 69, 101, 8, 100, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 41, 93, 64, 0, 108, 25, 107, 113, 89, 105, 102, 126, 95, 29, 47, 30, 123, 59, 111, 88, 115, 24, 48, 121, 38, 124, 53, 119, 63, 49, 43, 57, 96, 50, 27, 52, 23, 116, 21, 87, 19, 106, 51, 20, 61, 32, 84, 44, 22, 109, 40, 78, 85, 114, 127, 14, 118, 83, 125, 17, 86, 112, 66, 122, 45, 120, 2, 54, 81, 62, 56, 1, 3, 68, 65, 4, 70, 46, 26, 79, 55, 12, 82, 6, 91, 15, 92, 72, 18, 80, 76, 117, 13, 9, 77, 90, 16, 67, 110, 31, 11, 10, 73, 60, 58, 75, 74, 71, 7, 28, 5, 98, 69, 34, 94, 8, 101, 100, 35, 37, 99, 36], [39, 104, 33, 42, 103, 41, 97, 108, 64, 93, 105, 95, 29, 107, 0, 89, 25, 126, 88, 113, 102, 123, 124, 47, 121, 48, 53, 30, 115, 111, 59, 38, 43, 87, 106, 116, 40, 96, 22, 50, 109, 6, 24, 63, 23, 49, 119, 27, 21, 54, 44, 32, 52, 61, 83, 114, 122, 57, 20, 86, 56, 85, 81, 112, 45, 127, 19, 84, 51, 14, 125, 120, 1, 78, 118, 46, 62, 82, 17, 91, 15, 65, 18, 3, 117, 90, 67, 79, 26, 31, 92, 12, 66, 2, 77, 68, 55, 76, 16, 110, 13, 73, 58, 4, 80, 10, 9, 75, 74, 11, 72, 98, 60, 70, 7, 71, 34, 28, 94, 69, 8, 5, 101, 100, 36, 35, 37, 99], [39, 104, 33, 42, 103, 41, 97, 0, 64, 108, 93, 25, 113, 29, 89, 105, 59, 126, 107, 95, 115, 111, 102, 48, 123, 57, 47, 87, 124, 53, 121, 6, 63, 116, 38, 52, 88, 24, 96, 109, 30, 43, 119, 61, 106, 21, 49, 127, 44, 50, 54, 19, 23, 40, 51, 84, 20, 125, 27, 114, 56, 112, 14, 86, 85, 22, 83, 45, 122, 17, 65, 32, 120, 46, 81, 66, 78, 55, 1, 91, 3, 18, 79, 82, 62, 118, 26, 67, 12, 110, 90, 2, 76, 13, 77, 92, 31, 117, 10, 4, 15, 16, 9, 80, 75, 60, 73, 58, 68, 11, 74, 8, 71, 7, 72, 28, 70, 69, 98, 5, 101, 34, 94, 100, 35, 37, 36, 99], [39, 33, 104, 42, 103, 41, 97, 108, 0, 93, 25, 89, 107, 64, 113, 105, 126, 29, 95, 102, 59, 111, 123, 48, 124, 47, 30, 38, 53, 24, 43, 121, 6, 57, 88, 27, 115, 63, 96, 23, 44, 106, 116, 87, 21, 83, 49, 119, 109, 85, 52, 20, 32, 22, 50, 40, 14, 122, 19, 51, 17, 86, 78, 54, 114, 127, 81, 61, 45, 46, 84, 65, 62, 91, 56, 125, 112, 66, 120, 18, 2, 12, 15, 68, 67, 26, 82, 90, 3, 55, 118, 4, 79, 92, 13, 31, 1, 117, 76, 9, 8, 73, 80, 11, 77, 10, 16, 110, 75, 60, 74, 7, 71, 28, 58, 72, 34, 94, 98, 69, 5, 70, 101, 100, 37, 35, 36, 99], [104, 39, 33, 42, 103, 64, 0, 41, 108, 97, 93, 113, 107, 126, 25, 89, 59, 29, 105, 121, 111, 63, 115, 47, 102, 123, 95, 57, 88, 48, 124, 6, 53, 119, 21, 24, 61, 87, 116, 43, 96, 30, 106, 49, 51, 20, 83, 50, 114, 52, 67, 38, 109, 27, 65, 54, 125, 56, 122, 1, 112, 14, 4, 85, 44, 22, 120, 127, 23, 78, 19, 32, 81, 45, 40, 55, 86, 46, 118, 84, 17, 62, 2, 3, 12, 15, 82, 66, 79, 68, 26, 76, 18, 91, 9, 117, 110, 73, 8, 77, 92, 13, 16, 90, 7, 75, 10, 11, 80, 74, 71, 60, 31, 58, 70, 69, 28, 5, 98, 101, 94, 72, 34, 100, 37, 35, 99, 36], [39, 104, 33, 42, 103, 41, 108, 97, 93, 0, 105, 64, 25, 95, 29, 89, 115, 113, 126, 107, 102, 59, 124, 48, 123, 111, 53, 43, 88, 106, 121, 30, 47, 38, 119, 24, 87, 50, 96, 27, 22, 63, 21, 20, 57, 17, 49, 23, 52, 86, 116, 44, 40, 54, 122, 114, 6, 109, 51, 84, 65, 78, 19, 112, 127, 61, 125, 14, 83, 32, 85, 3, 81, 45, 18, 56, 46, 70, 79, 120, 1, 15, 62, 91, 82, 55, 76, 118, 9, 67, 2, 12, 8, 92, 90, 73, 110, 4, 31, 13, 68, 117, 16, 26, 10, 66, 80, 77, 75, 11, 74, 58, 60, 71, 28, 69, 7, 98, 94, 34, 5, 101, 100, 72, 36, 35, 37, 99], [39, 33, 104, 42, 103, 97, 41, 93, 0, 108, 25, 95, 107, 29, 113, 89, 64, 105, 102, 126, 115, 59, 88, 124, 47, 27, 38, 30, 24, 53, 121, 87, 48, 23, 123, 116, 96, 21, 106, 111, 32, 50, 44, 43, 85, 19, 109, 40, 63, 52, 83, 22, 49, 119, 54, 122, 56, 84, 81, 14, 86, 125, 20, 57, 66, 127, 114, 112, 17, 91, 61, 78, 2, 1, 70, 51, 45, 18, 12, 118, 82, 65, 15, 120, 46, 55, 90, 76, 77, 16, 92, 4, 3, 26, 8, 62, 67, 79, 117, 73, 80, 75, 13, 31, 68, 74, 9, 110, 10, 6, 71, 11, 28, 60, 58, 7, 34, 98, 94, 5, 69, 72, 100, 101, 35, 37, 36, 99], [39, 104, 33, 42, 103, 97, 0, 41, 93, 108, 64, 25, 105, 89, 113, 29, 95, 107, 126, 102, 59, 115, 48, 47, 123, 124, 53, 70, 88, 111, 116, 30, 43, 52, 24, 57, 23, 38, 121, 96, 87, 63, 22, 119, 106, 78, 51, 50, 40, 32, 20, 19, 44, 109, 49, 61, 54, 122, 127, 91, 83, 21, 112, 27, 125, 81, 45, 85, 84, 118, 1, 86, 56, 17, 62, 46, 79, 114, 55, 3, 14, 66, 120, 65, 2, 18, 68, 82, 110, 12, 8, 26, 117, 77, 90, 67, 75, 16, 92, 4, 31, 13, 76, 15, 80, 9, 73, 10, 28, 11, 74, 58, 60, 7, 71, 34, 98, 5, 94, 69, 6, 101, 72, 100, 37, 35, 36, 99]], "model.layers.1.self_attn.q_proj": [[103, 12, 107, 104, 46, 33, 42, 105, 49, 123, 50, 117, 62, 48, 57, 122, 19, 6, 59, 31, 29, 52, 54, 56, 5, 44, 115, 23, 92, 63, 8, 76, 70, 80, 66, 79, 72, 20, 116, 85, 120, 111, 84, 65, 78, 83, 25, 3, 22, 18, 127, 55, 90, 9, 27, 7, 126, 11, 87, 74, 37, 58, 82, 16, 77, 124, 4, 67, 2, 35, 13, 109, 15, 125, 88, 102, 71, 75, 17, 34, 94, 21, 100, 86, 14, 61, 24, 81, 114, 69, 45, 1, 98, 112, 73, 0, 51, 10, 39, 113, 101, 38, 32, 30, 36, 26, 121, 89, 41, 43, 97, 108, 106, 47, 40, 28, 93, 96, 64, 60, 91, 99, 118, 53, 119, 68, 110, 95], [103, 107, 104, 42, 105, 117, 46, 123, 50, 62, 24, 48, 33, 3, 67, 57, 122, 111, 59, 70, 56, 7, 127, 25, 65, 29, 49, 63, 115, 31, 79, 58, 13, 124, 16, 74, 44, 6, 66, 0, 4, 11, 116, 22, 9, 92, 82, 54, 5, 77, 101, 23, 125, 45, 84, 52, 1, 90, 78, 12, 98, 20, 94, 27, 126, 71, 2, 83, 120, 17, 73, 109, 38, 18, 87, 37, 19, 86, 32, 36, 15, 75, 64, 80, 55, 35, 81, 10, 14, 102, 100, 8, 113, 99, 76, 61, 121, 69, 26, 85, 53, 72, 97, 21, 30, 112, 89, 68, 118, 39, 43, 88, 41, 106, 114, 34, 110, 91, 95, 93, 60, 47, 108, 40, 51, 28, 96, 119], [100, 44, 102, 113, 37, 38, 59, 115, 35, 57, 122, 116, 34, 104, 114, 103, 109, 123, 46, 107, 50, 48, 62, 120, 117, 54, 42, 105, 52, 49, 125, 63, 112, 111, 98, 61, 56, 60, 53, 126, 121, 101, 33, 25, 99, 110, 127, 32, 84, 86, 58, 47, 82, 124, 88, 108, 21, 94, 40, 119, 41, 45, 36, 43, 89, 29, 118, 7, 51, 78, 77, 96, 17, 13, 30, 72, 106, 81, 95, 70, 92, 97, 93, 27, 11, 4, 79, 14, 9, 16, 55, 71, 74, 26, 83, 20, 67, 65, 75, 5, 0, 87, 66, 19, 12, 76, 85, 80, 91, 39, 3, 90, 24, 31, 28, 18, 6, 2, 22, 1, 64, 8, 15, 23, 68, 69, 73, 10], [103, 107, 104, 42, 33, 105, 46, 123, 117, 50, 62, 57, 122, 48, 49, 9, 29, 125, 111, 63, 31, 92, 90, 115, 10, 56, 4, 18, 54, 83, 69, 35, 7, 84, 44, 20, 23, 71, 22, 59, 100, 79, 75, 14, 5, 58, 127, 25, 113, 77, 8, 74, 86, 87, 27, 82, 70, 66, 65, 99, 120, 24, 13, 2, 73, 6, 12, 52, 78, 85, 55, 124, 114, 38, 1, 47, 108, 116, 72, 121, 11, 45, 88, 89, 81, 126, 112, 32, 3, 80, 93, 21, 15, 37, 16, 30, 97, 26, 94, 19, 53, 28, 101, 60, 118, 102, 17, 109, 91, 36, 39, 119, 98, 61, 110, 0, 34, 76, 95, 41, 51, 43, 40, 106, 96, 67, 68, 64], [104, 38, 97, 103, 42, 43, 48, 56, 90, 45, 87, 110, 47, 125, 50, 120, 53, 121, 49, 124, 61, 126, 58, 123, 30, 6, 31, 122, 57, 62, 68, 12, 119, 84, 52, 67, 81, 29, 16, 4, 105, 70, 112, 75, 117, 77, 78, 28, 63, 127, 7, 51, 82, 59, 13, 10, 41, 76, 20, 66, 85, 1, 18, 9, 8, 83, 15, 55, 64, 100, 60, 89, 113, 108, 23, 115, 116, 69, 72, 88, 98, 24, 73, 3, 34, 2, 91, 118, 54, 17, 79, 65, 5, 14, 109, 35, 11, 21, 80, 86, 107, 101, 37, 32, 19, 71, 74, 106, 99, 44, 25, 0, 92, 93, 36, 22, 26, 96, 94, 27, 111, 114, 33, 46, 40, 102, 95, 39], [104, 38, 103, 97, 42, 43, 87, 47, 45, 110, 49, 53, 50, 123, 126, 61, 124, 6, 29, 57, 90, 120, 121, 48, 12, 66, 31, 13, 117, 64, 28, 59, 81, 56, 10, 68, 58, 122, 62, 78, 7, 44, 75, 105, 52, 67, 20, 125, 51, 16, 1, 72, 119, 54, 15, 2, 9, 55, 18, 69, 115, 113, 82, 127, 100, 80, 11, 109, 91, 99, 4, 35, 96, 8, 85, 71, 63, 60, 5, 116, 70, 17, 118, 22, 101, 3, 36, 84, 37, 89, 107, 14, 23, 46, 73, 79, 106, 112, 74, 27, 77, 30, 25, 76, 65, 21, 41, 98, 108, 24, 19, 88, 0, 114, 26, 32, 34, 111, 86, 83, 94, 33, 92, 93, 40, 95, 39, 102], [104, 103, 97, 38, 42, 43, 13, 49, 110, 47, 53, 50, 121, 56, 126, 59, 45, 87, 123, 124, 61, 6, 122, 48, 31, 117, 120, 29, 37, 78, 81, 90, 127, 28, 118, 119, 67, 51, 12, 68, 16, 55, 105, 70, 113, 58, 52, 82, 10, 35, 19, 72, 125, 8, 69, 24, 71, 66, 79, 75, 1, 23, 115, 112, 57, 54, 11, 62, 91, 73, 116, 18, 7, 85, 60, 25, 80, 44, 101, 20, 98, 30, 114, 41, 84, 17, 15, 76, 108, 88, 32, 63, 89, 34, 77, 46, 96, 27, 4, 64, 74, 14, 36, 100, 22, 21, 5, 107, 94, 9, 86, 92, 99, 2, 3, 65, 106, 83, 111, 26, 93, 109, 33, 0, 95, 102, 40, 39], [120, 49, 97, 124, 57, 103, 61, 104, 59, 87, 38, 18, 105, 121, 42, 122, 43, 54, 53, 50, 23, 31, 47, 35, 45, 48, 110, 101, 92, 126, 82, 84, 123, 117, 113, 56, 118, 115, 127, 51, 62, 41, 55, 109, 90, 12, 27, 89, 28, 114, 94, 91, 108, 81, 19, 26, 102, 44, 58, 16, 77, 78, 60, 88, 37, 15, 116, 95, 80, 119, 100, 107, 75, 6, 10, 29, 52, 20, 36, 111, 72, 46, 13, 73, 79, 74, 25, 22, 85, 33, 63, 106, 9, 86, 68, 30, 32, 11, 17, 98, 4, 93, 67, 71, 7, 125, 66, 1, 24, 76, 21, 96, 112, 3, 99, 69, 83, 34, 14, 8, 64, 70, 5, 39, 65, 2, 40, 0], [107, 42, 41, 40, 103, 33, 46, 118, 95, 24, 126, 109, 116, 125, 18, 14, 82, 115, 114, 63, 53, 56, 93, 113, 119, 111, 112, 58, 84, 54, 108, 117, 85, 59, 55, 49, 83, 60, 121, 20, 79, 80, 61, 87, 13, 27, 75, 76, 25, 50, 21, 99, 110, 48, 51, 124, 90, 92, 26, 7, 38, 89, 86, 77, 71, 94, 123, 8, 81, 74, 29, 23, 72, 120, 10, 15, 127, 22, 37, 32, 91, 96, 98, 73, 30, 52, 16, 34, 12, 122, 28, 11, 106, 47, 19, 31, 17, 35, 45, 102, 78, 97, 43, 9, 101, 88, 70, 36, 69, 67, 44, 66, 57, 3, 4, 65, 100, 62, 68, 104, 5, 105, 39, 0, 6, 1, 2, 64], [101, 98, 120, 108, 112, 127, 114, 47, 113, 34, 61, 49, 115, 62, 59, 122, 32, 111, 30, 118, 58, 36, 121, 119, 46, 48, 60, 123, 42, 50, 55, 53, 63, 94, 124, 41, 40, 125, 116, 37, 35, 126, 107, 56, 109, 44, 117, 54, 100, 88, 103, 38, 89, 22, 91, 45, 21, 68, 84, 102, 33, 57, 39, 85, 51, 25, 1, 93, 92, 6, 11, 64, 4, 52, 77, 69, 67, 8, 0, 71, 86, 99, 12, 24, 83, 95, 96, 97, 10, 81, 78, 80, 28, 9, 18, 110, 16, 7, 26, 2, 66, 104, 13, 79, 72, 31, 15, 82, 20, 3, 5, 19, 14, 65, 23, 73, 27, 70, 75, 76, 90, 43, 87, 29, 74, 17, 106, 105], [40, 42, 107, 41, 33, 112, 95, 46, 103, 116, 93, 20, 115, 114, 18, 121, 109, 27, 122, 15, 118, 78, 60, 16, 19, 72, 58, 54, 92, 25, 126, 125, 53, 17, 81, 26, 23, 119, 108, 12, 111, 13, 62, 83, 22, 56, 10, 49, 117, 48, 77, 55, 124, 11, 68, 120, 79, 14, 34, 87, 71, 113, 50, 29, 86, 59, 21, 74, 3, 57, 75, 127, 67, 5, 76, 80, 8, 104, 70, 89, 66, 84, 6, 61, 36, 82, 91, 9, 51, 98, 73, 38, 37, 90, 106, 85, 97, 69, 99, 28, 24, 52, 105, 1, 39, 31, 110, 63, 32, 2, 45, 4, 123, 94, 101, 88, 96, 43, 7, 47, 44, 30, 102, 65, 100, 35, 0, 64], [107, 41, 42, 103, 40, 109, 33, 85, 112, 108, 46, 95, 126, 54, 24, 66, 49, 68, 111, 117, 113, 38, 4, 114, 62, 122, 119, 125, 56, 22, 27, 1, 93, 61, 7, 118, 32, 36, 53, 18, 0, 17, 121, 10, 71, 6, 23, 8, 94, 78, 67, 2, 120, 98, 115, 116, 127, 59, 25, 81, 92, 60, 11, 21, 110, 37, 9, 15, 55, 80, 30, 57, 69, 77, 48, 64, 75, 12, 100, 3, 20, 19, 88, 63, 82, 123, 13, 16, 73, 34, 52, 5, 96, 124, 87, 70, 51, 101, 26, 35, 45, 99, 14, 79, 74, 43, 65, 58, 102, 29, 50, 83, 76, 86, 72, 84, 47, 28, 90, 91, 106, 44, 89, 97, 31, 105, 39, 104], [52, 55, 102, 57, 117, 111, 56, 95, 109, 54, 51, 101, 114, 63, 97, 53, 50, 127, 59, 104, 48, 122, 24, 86, 61, 98, 103, 90, 100, 112, 88, 26, 93, 33, 115, 41, 80, 110, 29, 31, 42, 116, 96, 125, 108, 16, 113, 85, 120, 27, 124, 126, 121, 60, 46, 49, 44, 22, 123, 74, 99, 37, 119, 30, 39, 17, 62, 81, 118, 107, 47, 58, 38, 43, 32, 40, 94, 10, 25, 20, 28, 21, 36, 45, 23, 91, 35, 34, 105, 84, 19, 92, 12, 76, 87, 106, 14, 89, 11, 18, 82, 15, 69, 78, 7, 13, 3, 75, 77, 72, 9, 8, 83, 5, 67, 79, 73, 6, 71, 4, 66, 2, 70, 68, 65, 0, 64, 1], [103, 104, 42, 41, 0, 33, 90, 44, 114, 109, 111, 63, 56, 59, 122, 102, 29, 61, 86, 127, 53, 101, 125, 121, 54, 51, 3, 95, 66, 16, 52, 10, 4, 112, 107, 48, 46, 83, 65, 50, 120, 84, 77, 49, 7, 76, 15, 69, 55, 71, 19, 115, 74, 12, 57, 25, 28, 78, 17, 72, 6, 85, 14, 91, 5, 9, 100, 123, 118, 80, 1, 126, 64, 124, 47, 21, 70, 13, 18, 2, 22, 75, 117, 23, 81, 68, 82, 11, 106, 87, 73, 113, 88, 67, 94, 96, 79, 105, 116, 108, 35, 110, 8, 20, 34, 39, 40, 98, 26, 24, 38, 30, 60, 36, 62, 99, 58, 89, 27, 37, 43, 45, 119, 32, 93, 92, 31, 97], [46, 44, 33, 52, 24, 111, 104, 103, 109, 124, 41, 102, 123, 57, 125, 107, 120, 108, 51, 114, 122, 61, 127, 59, 54, 90, 56, 85, 48, 29, 42, 53, 63, 95, 86, 55, 116, 76, 101, 50, 27, 80, 12, 22, 58, 60, 37, 110, 113, 26, 88, 119, 47, 21, 97, 81, 74, 93, 126, 96, 112, 34, 19, 117, 83, 7, 23, 3, 20, 15, 25, 10, 49, 77, 18, 43, 31, 89, 4, 84, 100, 0, 115, 17, 69, 98, 99, 78, 121, 87, 16, 9, 38, 28, 66, 14, 75, 92, 30, 11, 32, 91, 6, 62, 94, 105, 65, 118, 45, 106, 13, 8, 36, 35, 70, 79, 72, 67, 82, 71, 2, 68, 64, 73, 5, 1, 40, 39], [103, 57, 44, 101, 104, 41, 42, 33, 52, 109, 90, 55, 114, 46, 12, 111, 29, 54, 63, 56, 59, 127, 122, 61, 53, 51, 125, 95, 85, 72, 112, 81, 76, 48, 17, 3, 50, 11, 4, 100, 7, 121, 73, 115, 15, 120, 47, 69, 82, 19, 66, 80, 65, 23, 86, 78, 28, 67, 123, 102, 107, 0, 20, 22, 13, 16, 84, 88, 70, 10, 91, 30, 9, 126, 94, 5, 18, 14, 71, 83, 110, 25, 34, 58, 21, 32, 49, 6, 89, 2, 26, 8, 124, 77, 93, 64, 68, 75, 62, 45, 79, 117, 113, 87, 96, 43, 92, 24, 60, 98, 38, 106, 99, 27, 105, 1, 74, 36, 119, 108, 118, 31, 116, 35, 37, 97, 40, 39], [104, 101, 105, 45, 110, 97, 44, 106, 43, 64, 114, 95, 89, 113, 56, 116, 57, 93, 48, 119, 47, 58, 50, 68, 1, 15, 6, 70, 9, 84, 2, 76, 81, 16, 13, 74, 3, 14, 69, 11, 80, 20, 55, 0, 65, 60, 28, 72, 78, 7, 63, 17, 115, 71, 36, 46, 66, 8, 19, 121, 5, 109, 52, 73, 4, 118, 25, 40, 75, 34, 108, 41, 126, 82, 112, 59, 107, 111, 122, 61, 51, 42, 124, 123, 35, 125, 99, 62, 98, 67, 79, 88, 54, 85, 117, 49, 103, 27, 102, 38, 53, 32, 96, 91, 100, 90, 10, 21, 24, 12, 30, 22, 26, 127, 29, 86, 87, 83, 120, 77, 94, 23, 39, 18, 37, 92, 33, 31], [114, 104, 101, 97, 105, 45, 69, 44, 89, 110, 106, 43, 63, 95, 72, 75, 3, 93, 113, 56, 4, 57, 14, 55, 76, 119, 116, 58, 50, 48, 84, 47, 74, 20, 70, 67, 1, 9, 81, 80, 13, 60, 28, 118, 123, 16, 115, 18, 73, 64, 17, 11, 15, 22, 126, 78, 12, 7, 121, 46, 68, 77, 112, 62, 109, 65, 85, 61, 2, 83, 24, 23, 52, 127, 125, 108, 122, 6, 40, 51, 41, 79, 107, 10, 54, 26, 96, 42, 66, 21, 49, 19, 86, 25, 103, 38, 120, 37, 36, 124, 111, 90, 8, 102, 35, 87, 92, 27, 88, 91, 5, 82, 34, 100, 39, 117, 98, 94, 0, 71, 53, 32, 59, 29, 30, 99, 31, 33], [104, 64, 45, 110, 101, 105, 44, 106, 97, 43, 89, 113, 56, 57, 1, 119, 116, 0, 48, 58, 50, 47, 95, 93, 114, 68, 3, 84, 70, 123, 13, 2, 5, 63, 76, 28, 55, 60, 127, 59, 118, 14, 74, 122, 9, 81, 80, 15, 20, 46, 66, 8, 51, 16, 126, 36, 109, 112, 26, 40, 52, 4, 67, 41, 75, 108, 12, 69, 7, 11, 65, 107, 42, 77, 17, 53, 18, 78, 117, 61, 88, 6, 39, 35, 115, 72, 98, 111, 124, 32, 82, 71, 121, 91, 99, 86, 22, 85, 38, 62, 34, 10, 125, 73, 90, 49, 120, 19, 87, 30, 25, 21, 102, 83, 96, 27, 23, 100, 94, 24, 79, 92, 54, 103, 29, 31, 37, 33], [55, 63, 104, 97, 101, 72, 105, 45, 44, 110, 106, 43, 114, 52, 89, 93, 95, 77, 113, 58, 57, 56, 119, 116, 78, 15, 48, 60, 47, 83, 13, 62, 70, 50, 84, 115, 126, 80, 81, 118, 75, 8, 90, 17, 79, 38, 28, 91, 73, 20, 18, 12, 121, 112, 22, 125, 6, 69, 21, 16, 25, 86, 85, 46, 19, 37, 124, 29, 23, 9, 109, 76, 74, 102, 14, 61, 3, 10, 4, 59, 127, 111, 123, 108, 68, 71, 53, 92, 94, 36, 120, 24, 103, 35, 82, 54, 99, 27, 107, 51, 100, 11, 122, 117, 5, 88, 26, 41, 34, 96, 42, 87, 40, 32, 33, 67, 31, 30, 1, 49, 39, 98, 2, 7, 64, 65, 66, 0], [104, 0, 45, 106, 43, 103, 112, 1, 97, 102, 127, 2, 64, 116, 60, 93, 117, 67, 113, 4, 88, 115, 65, 50, 70, 71, 66, 57, 77, 78, 49, 63, 59, 69, 16, 76, 52, 62, 20, 48, 15, 73, 122, 74, 109, 22, 51, 58, 114, 123, 11, 81, 82, 8, 3, 124, 125, 107, 53, 87, 120, 17, 56, 83, 42, 19, 61, 84, 118, 18, 7, 24, 55, 40, 72, 39, 121, 86, 119, 6, 14, 54, 13, 12, 75, 10, 5, 38, 85, 9, 26, 21, 79, 92, 80, 89, 126, 111, 110, 44, 23, 68, 47, 32, 27, 25, 105, 98, 95, 108, 90, 28, 41, 91, 30, 100, 29, 37, 36, 96, 31, 34, 46, 94, 101, 33, 99, 35], [104, 103, 106, 45, 43, 97, 112, 102, 5, 0, 116, 3, 93, 68, 69, 127, 88, 20, 66, 4, 67, 60, 114, 117, 11, 71, 57, 70, 113, 49, 73, 115, 83, 2, 76, 122, 16, 82, 77, 78, 63, 51, 59, 48, 124, 8, 6, 22, 1, 15, 74, 81, 50, 65, 19, 58, 62, 52, 118, 123, 84, 85, 56, 120, 87, 75, 125, 61, 121, 109, 53, 55, 7, 64, 24, 17, 86, 54, 79, 107, 110, 80, 18, 72, 13, 42, 119, 10, 12, 14, 23, 47, 9, 111, 92, 108, 126, 29, 40, 27, 89, 44, 38, 21, 90, 46, 39, 28, 95, 25, 33, 26, 91, 31, 36, 41, 100, 30, 105, 94, 99, 37, 32, 96, 34, 35, 98, 101], [103, 114, 104, 116, 122, 59, 51, 97, 106, 48, 49, 43, 20, 124, 58, 18, 62, 88, 45, 57, 118, 63, 19, 55, 22, 102, 86, 108, 92, 123, 61, 79, 52, 112, 47, 125, 14, 121, 80, 126, 110, 120, 29, 33, 113, 17, 56, 75, 127, 84, 53, 82, 117, 24, 13, 115, 12, 50, 90, 46, 23, 93, 81, 26, 60, 119, 83, 91, 44, 10, 87, 27, 72, 54, 9, 32, 89, 15, 41, 111, 95, 94, 25, 28, 30, 36, 39, 38, 98, 77, 31, 85, 7, 37, 100, 6, 16, 99, 96, 105, 101, 5, 109, 35, 76, 78, 74, 21, 34, 73, 40, 11, 68, 8, 3, 71, 70, 66, 69, 4, 0, 1, 65, 67, 107, 2, 42, 64], [114, 118, 51, 103, 104, 49, 55, 53, 110, 125, 20, 123, 106, 122, 120, 121, 102, 56, 61, 43, 63, 124, 108, 126, 111, 88, 45, 19, 62, 119, 47, 54, 17, 58, 97, 18, 92, 22, 112, 79, 14, 80, 57, 41, 36, 99, 101, 100, 86, 116, 35, 90, 30, 98, 37, 34, 75, 59, 31, 44, 95, 32, 105, 96, 12, 13, 117, 127, 89, 91, 10, 50, 81, 33, 60, 24, 93, 72, 83, 94, 38, 9, 115, 46, 27, 52, 26, 7, 82, 113, 23, 84, 6, 87, 48, 29, 15, 5, 85, 109, 21, 28, 68, 25, 16, 11, 107, 3, 78, 39, 8, 77, 69, 66, 76, 42, 70, 65, 71, 74, 73, 4, 67, 40, 2, 0, 1, 64], [104, 102, 97, 44, 4, 103, 24, 107, 106, 29, 50, 126, 120, 55, 21, 62, 116, 59, 125, 95, 48, 3, 71, 121, 78, 67, 119, 45, 76, 0, 112, 82, 15, 65, 91, 123, 8, 92, 9, 49, 13, 1, 66, 10, 11, 68, 52, 122, 53, 80, 117, 51, 18, 110, 83, 6, 7, 58, 73, 77, 127, 75, 57, 113, 47, 101, 85, 108, 54, 81, 5, 20, 70, 16, 12, 111, 19, 43, 115, 74, 37, 32, 2, 69, 26, 42, 30, 56, 124, 64, 114, 46, 61, 72, 88, 96, 17, 118, 99, 22, 87, 89, 14, 98, 34, 63, 40, 41, 84, 27, 25, 94, 109, 79, 100, 60, 86, 23, 105, 36, 90, 35, 28, 93, 39, 33, 31, 38], [97, 102, 104, 44, 103, 105, 107, 106, 29, 24, 53, 50, 48, 126, 120, 116, 55, 119, 95, 57, 21, 121, 117, 125, 45, 82, 16, 123, 61, 78, 63, 15, 51, 62, 111, 11, 127, 92, 52, 13, 56, 9, 115, 5, 91, 72, 68, 12, 7, 59, 3, 2, 83, 6, 76, 118, 23, 108, 0, 49, 18, 54, 65, 30, 60, 75, 17, 80, 79, 89, 90, 77, 43, 98, 10, 32, 112, 74, 42, 19, 86, 81, 70, 14, 34, 84, 26, 25, 94, 35, 100, 27, 22, 85, 96, 36, 66, 124, 99, 37, 40, 28, 87, 20, 4, 8, 122, 113, 110, 109, 101, 47, 58, 71, 39, 31, 69, 67, 33, 73, 46, 64, 114, 41, 88, 38, 1, 93], [104, 102, 97, 44, 103, 29, 107, 106, 24, 50, 21, 126, 55, 120, 68, 82, 53, 52, 95, 11, 117, 78, 5, 13, 0, 125, 9, 59, 12, 62, 2, 6, 48, 92, 65, 15, 7, 45, 58, 51, 72, 3, 70, 116, 105, 18, 14, 123, 91, 46, 8, 121, 75, 57, 119, 109, 113, 122, 66, 76, 54, 26, 49, 110, 85, 88, 73, 17, 67, 71, 112, 30, 108, 114, 47, 60, 74, 80, 63, 23, 101, 1, 64, 79, 61, 43, 10, 22, 77, 41, 4, 42, 19, 111, 56, 37, 35, 124, 16, 90, 118, 84, 87, 34, 20, 69, 127, 100, 96, 115, 89, 36, 40, 32, 33, 94, 98, 83, 99, 25, 86, 81, 27, 28, 93, 38, 31, 39], [103, 102, 104, 105, 48, 97, 59, 44, 106, 107, 24, 116, 29, 117, 78, 21, 11, 120, 50, 126, 52, 16, 62, 55, 82, 53, 57, 74, 13, 51, 9, 18, 123, 95, 110, 119, 121, 75, 2, 15, 54, 85, 68, 125, 19, 122, 7, 88, 6, 70, 14, 112, 72, 92, 8, 80, 3, 45, 113, 30, 109, 66, 0, 17, 26, 76, 71, 65, 49, 58, 5, 1, 115, 114, 91, 47, 60, 22, 4, 77, 108, 67, 64, 10, 111, 32, 63, 124, 12, 87, 25, 43, 93, 56, 84, 86, 79, 83, 96, 42, 23, 41, 118, 94, 20, 61, 27, 69, 100, 127, 73, 46, 101, 98, 37, 40, 34, 81, 89, 90, 38, 36, 35, 99, 39, 28, 31, 33], [114, 103, 98, 44, 43, 49, 57, 48, 97, 105, 122, 95, 40, 14, 127, 63, 121, 107, 124, 83, 19, 24, 21, 120, 42, 78, 119, 94, 54, 32, 90, 58, 117, 110, 111, 74, 51, 92, 53, 84, 25, 26, 62, 29, 36, 18, 86, 109, 38, 115, 50, 46, 56, 112, 116, 15, 59, 123, 113, 85, 81, 33, 87, 99, 125, 88, 10, 17, 22, 31, 60, 100, 104, 118, 126, 82, 70, 93, 91, 52, 61, 28, 37, 55, 101, 108, 30, 35, 79, 47, 45, 6, 8, 102, 23, 16, 39, 13, 76, 11, 89, 77, 96, 12, 20, 34, 80, 27, 7, 9, 72, 4, 106, 75, 3, 68, 66, 73, 2, 41, 71, 65, 5, 67, 1, 69, 64, 0], [44, 63, 98, 62, 60, 111, 51, 49, 103, 124, 43, 59, 48, 127, 42, 97, 86, 17, 105, 22, 24, 120, 95, 55, 114, 92, 29, 50, 40, 99, 113, 110, 45, 90, 102, 25, 18, 57, 37, 36, 54, 77, 61, 47, 85, 28, 88, 94, 33, 74, 21, 87, 84, 101, 93, 100, 31, 26, 27, 108, 123, 125, 14, 91, 118, 35, 116, 115, 38, 117, 10, 52, 112, 121, 20, 56, 15, 109, 30, 81, 89, 13, 82, 78, 122, 104, 70, 83, 39, 126, 119, 96, 58, 32, 8, 19, 53, 71, 66, 7, 16, 4, 79, 34, 23, 107, 2, 46, 72, 67, 11, 68, 6, 41, 106, 80, 12, 0, 75, 1, 69, 64, 3, 73, 65, 9, 5, 76], [103, 105, 97, 42, 44, 111, 40, 121, 89, 95, 9, 119, 22, 58, 49, 57, 4, 48, 120, 124, 94, 62, 122, 65, 50, 60, 67, 11, 109, 51, 13, 59, 56, 92, 43, 99, 18, 79, 53, 127, 55, 16, 0, 70, 84, 63, 54, 5, 1, 73, 36, 81, 24, 26, 114, 46, 20, 71, 6, 98, 2, 82, 90, 112, 23, 66, 80, 86, 3, 77, 74, 85, 113, 100, 126, 12, 72, 76, 10, 7, 32, 38, 21, 115, 91, 78, 17, 118, 83, 125, 123, 27, 19, 29, 69, 68, 25, 47, 45, 102, 88, 64, 75, 8, 61, 14, 93, 116, 28, 52, 110, 34, 31, 37, 117, 87, 30, 15, 35, 101, 33, 96, 41, 107, 39, 108, 104, 106], [40, 109, 43, 44, 49, 97, 42, 103, 48, 53, 124, 105, 37, 118, 98, 120, 127, 62, 57, 56, 36, 95, 29, 51, 60, 59, 74, 92, 114, 22, 90, 84, 121, 55, 78, 81, 63, 10, 89, 54, 122, 16, 77, 58, 108, 24, 15, 115, 70, 38, 126, 119, 85, 86, 111, 125, 13, 47, 61, 82, 21, 25, 123, 8, 101, 93, 94, 83, 87, 45, 11, 102, 76, 19, 79, 18, 99, 20, 28, 88, 46, 116, 68, 80, 117, 32, 52, 100, 106, 9, 14, 17, 23, 67, 1, 4, 75, 96, 110, 26, 41, 2, 112, 33, 91, 113, 27, 30, 35, 50, 71, 107, 34, 72, 6, 12, 0, 3, 7, 64, 73, 69, 66, 39, 5, 31, 104, 65]], "model.layers.1.self_attn.k_proj": [[0, 39, 40, 43, 110, 4, 97, 11, 106, 41, 65, 95, 78, 84, 79, 62, 117, 74, 123, 9, 66, 50, 72, 122, 93, 5, 57, 87, 91, 73, 26, 47, 13, 69, 28, 68, 86, 16, 70, 7, 2, 59, 77, 81, 53, 112, 56, 67, 127, 17, 21, 46, 48, 10, 82, 121, 3, 64, 116, 76, 55, 63, 118, 124, 115, 58, 6, 114, 30, 89, 71, 113, 18, 51, 111, 14, 8, 108, 96, 54, 12, 120, 85, 60, 61, 109, 126, 75, 119, 15, 80, 125, 23, 1, 88, 52, 29, 49, 103, 38, 83, 94, 19, 34, 32, 45, 99, 33, 90, 36, 104, 27, 101, 22, 20, 24, 92, 98, 44, 105, 35, 100, 31, 107, 102, 37, 25, 42], [64, 40, 106, 107, 39, 46, 69, 111, 1, 33, 10, 66, 93, 7, 65, 2, 5, 113, 109, 95, 67, 75, 72, 117, 126, 12, 3, 123, 16, 112, 114, 81, 9, 61, 50, 4, 78, 53, 6, 102, 60, 124, 85, 15, 23, 47, 116, 21, 26, 125, 0, 92, 63, 27, 74, 71, 110, 108, 86, 22, 58, 68, 62, 11, 56, 70, 121, 83, 49, 80, 24, 19, 82, 94, 118, 55, 77, 17, 8, 98, 127, 57, 59, 52, 34, 51, 73, 25, 122, 119, 96, 14, 32, 99, 101, 87, 18, 37, 36, 79, 88, 76, 115, 84, 41, 20, 48, 91, 44, 35, 54, 89, 100, 28, 30, 97, 43, 105, 90, 120, 29, 45, 13, 42, 38, 31, 104, 103], [39, 97, 43, 105, 104, 106, 110, 31, 90, 29, 87, 45, 48, 54, 117, 83, 91, 81, 12, 15, 44, 78, 51, 28, 77, 10, 20, 0, 11, 89, 6, 122, 69, 7, 50, 2, 16, 126, 17, 79, 56, 125, 86, 8, 18, 52, 116, 65, 84, 60, 62, 1, 121, 13, 3, 55, 9, 73, 57, 19, 124, 53, 4, 119, 35, 58, 115, 64, 47, 75, 118, 59, 67, 49, 74, 5, 102, 114, 21, 61, 127, 24, 112, 103, 120, 80, 111, 34, 113, 46, 98, 23, 72, 96, 40, 101, 41, 68, 71, 93, 27, 32, 70, 107, 100, 30, 99, 42, 85, 36, 63, 82, 66, 37, 123, 88, 38, 94, 76, 109, 22, 108, 14, 26, 92, 33, 25, 95], [45, 0, 106, 105, 40, 39, 1, 65, 97, 108, 50, 47, 92, 31, 9, 61, 15, 127, 77, 63, 7, 122, 3, 6, 19, 110, 53, 59, 48, 4, 51, 118, 2, 120, 54, 68, 56, 23, 64, 49, 66, 43, 27, 114, 35, 87, 93, 126, 69, 70, 5, 36, 89, 79, 11, 71, 82, 8, 20, 119, 94, 75, 18, 121, 98, 62, 125, 76, 81, 37, 12, 78, 38, 32, 83, 22, 74, 58, 14, 111, 34, 13, 67, 115, 57, 84, 60, 109, 112, 99, 26, 21, 30, 123, 17, 116, 46, 73, 96, 113, 85, 107, 103, 80, 117, 124, 25, 33, 10, 24, 86, 91, 29, 104, 42, 88, 41, 72, 55, 100, 90, 44, 102, 101, 28, 52, 16, 95], [64, 40, 109, 65, 46, 108, 41, 107, 42, 66, 50, 0, 37, 3, 119, 57, 56, 33, 113, 116, 67, 48, 1, 74, 5, 58, 31, 68, 6, 76, 4, 14, 9, 13, 25, 47, 29, 71, 49, 20, 111, 11, 81, 112, 8, 7, 92, 15, 16, 2, 80, 61, 51, 94, 110, 85, 10, 60, 24, 84, 21, 123, 70, 59, 118, 126, 63, 100, 96, 54, 82, 83, 23, 102, 30, 79, 45, 39, 121, 99, 86, 19, 104, 73, 98, 27, 77, 12, 103, 32, 75, 91, 88, 87, 117, 90, 69, 38, 105, 120, 22, 44, 35, 18, 34, 106, 125, 17, 62, 78, 53, 43, 115, 122, 127, 52, 114, 124, 26, 55, 72, 89, 36, 28, 93, 101, 97, 95], [40, 0, 109, 42, 107, 39, 33, 48, 65, 64, 66, 113, 50, 52, 3, 115, 29, 117, 127, 60, 71, 70, 4, 16, 24, 73, 69, 67, 77, 68, 116, 57, 74, 76, 8, 11, 58, 82, 28, 83, 78, 63, 59, 62, 15, 86, 2, 124, 112, 84, 23, 54, 121, 122, 46, 81, 119, 125, 123, 120, 61, 26, 44, 5, 6, 1, 7, 41, 101, 37, 100, 111, 94, 99, 56, 27, 36, 35, 38, 96, 31, 98, 32, 105, 34, 55, 126, 118, 47, 87, 25, 9, 95, 45, 108, 110, 51, 114, 49, 91, 53, 30, 20, 88, 85, 90, 21, 72, 10, 19, 89, 17, 102, 14, 43, 12, 92, 106, 18, 79, 104, 22, 13, 75, 103, 93, 80, 97], [0, 108, 40, 43, 42, 39, 65, 55, 120, 126, 114, 3, 33, 50, 72, 117, 6, 2, 52, 7, 9, 41, 31, 109, 68, 5, 48, 70, 15, 125, 49, 13, 121, 62, 18, 12, 112, 78, 74, 115, 46, 85, 11, 110, 76, 69, 27, 93, 59, 116, 123, 28, 67, 38, 53, 88, 1, 4, 16, 82, 17, 20, 118, 84, 86, 90, 80, 63, 61, 60, 19, 47, 37, 83, 111, 96, 81, 57, 35, 87, 54, 21, 99, 75, 89, 36, 22, 51, 10, 73, 77, 98, 100, 25, 24, 66, 94, 23, 79, 30, 113, 34, 32, 14, 127, 124, 8, 119, 26, 71, 58, 101, 122, 44, 102, 45, 29, 64, 56, 97, 92, 105, 95, 91, 107, 106, 104, 103], [39, 41, 106, 33, 108, 104, 31, 0, 69, 34, 57, 113, 120, 25, 16, 112, 30, 47, 45, 1, 67, 84, 119, 46, 87, 2, 122, 124, 107, 5, 8, 51, 11, 121, 58, 71, 62, 68, 53, 9, 56, 35, 59, 15, 18, 60, 90, 92, 13, 125, 96, 86, 70, 76, 12, 7, 61, 24, 36, 81, 49, 4, 27, 126, 55, 123, 50, 66, 63, 127, 73, 101, 52, 75, 117, 77, 89, 54, 38, 85, 21, 110, 48, 102, 100, 28, 29, 72, 93, 22, 116, 118, 80, 97, 115, 3, 82, 43, 64, 95, 74, 23, 65, 19, 111, 20, 88, 17, 91, 98, 26, 37, 83, 99, 109, 32, 114, 79, 103, 6, 14, 44, 10, 94, 78, 40, 42, 105]], "model.layers.1.self_attn.qk_proj": [[50, 48, 0, 57, 64, 117, 56, 116, 59, 126, 113, 120, 122, 62, 123, 53, 55, 49, 65, 127, 46, 110, 1, 63, 97, 61, 124, 47, 58, 109, 67, 119, 121, 29, 3, 112, 108, 54, 51, 40, 60, 107, 4, 45, 125, 114, 43, 104, 70, 68, 2, 76, 33, 5, 66, 16, 42, 106, 95, 7, 77, 111, 93, 14, 6, 52, 41, 9, 13, 20, 11, 115, 84, 12, 71, 82, 75, 74, 81, 44, 73, 79, 69, 103, 78, 39, 10, 17, 15, 18, 72, 80, 87, 105, 24, 118, 31, 23, 8, 22, 83, 19, 90, 86, 85, 92, 89, 21, 25, 88, 26, 91, 102, 28, 36, 38, 27, 37, 98, 94, 101, 35, 100, 96, 34, 99, 32, 30], [50, 48, 0, 64, 57, 117, 56, 120, 59, 126, 116, 62, 113, 122, 46, 53, 55, 123, 127, 63, 65, 1, 110, 58, 49, 97, 61, 109, 47, 108, 124, 29, 40, 112, 121, 51, 68, 119, 114, 60, 43, 107, 54, 4, 45, 125, 66, 104, 2, 42, 67, 6, 70, 52, 106, 93, 33, 7, 95, 69, 3, 115, 77, 14, 20, 12, 41, 5, 76, 13, 111, 9, 71, 16, 44, 103, 11, 39, 84, 82, 17, 73, 105, 10, 18, 118, 78, 74, 79, 72, 75, 31, 80, 15, 81, 24, 87, 23, 8, 89, 88, 22, 83, 85, 102, 26, 19, 25, 90, 92, 21, 28, 86, 27, 98, 91, 38, 101, 37, 35, 30, 94, 36, 32, 100, 34, 99, 96], [50, 48, 57, 64, 0, 117, 56, 116, 120, 59, 126, 113, 62, 122, 53, 46, 123, 63, 127, 65, 110, 55, 58, 108, 97, 1, 47, 109, 51, 49, 29, 112, 121, 61, 124, 119, 67, 40, 3, 54, 114, 107, 60, 43, 6, 45, 4, 104, 125, 70, 2, 42, 66, 52, 106, 95, 68, 13, 71, 93, 115, 33, 5, 14, 76, 84, 111, 69, 11, 41, 16, 77, 12, 20, 9, 7, 73, 75, 15, 10, 39, 79, 78, 82, 18, 17, 103, 44, 81, 74, 72, 80, 118, 105, 31, 24, 87, 8, 23, 83, 22, 19, 89, 86, 88, 21, 85, 28, 25, 92, 102, 90, 26, 91, 98, 27, 37, 101, 38, 30, 94, 32, 35, 100, 36, 99, 96, 34], [50, 48, 0, 64, 57, 117, 56, 116, 59, 120, 126, 113, 122, 62, 123, 46, 53, 65, 127, 55, 58, 110, 63, 124, 49, 1, 97, 3, 61, 47, 108, 121, 109, 51, 112, 29, 114, 67, 54, 40, 119, 60, 6, 45, 2, 107, 104, 43, 125, 66, 52, 42, 5, 95, 106, 68, 115, 70, 111, 71, 93, 69, 20, 4, 76, 33, 41, 11, 12, 13, 77, 9, 14, 7, 103, 16, 78, 10, 84, 44, 79, 81, 80, 39, 82, 18, 73, 17, 31, 74, 75, 105, 72, 24, 118, 15, 23, 22, 8, 87, 88, 83, 86, 21, 89, 19, 92, 27, 90, 25, 85, 102, 28, 26, 91, 37, 38, 101, 98, 100, 94, 34, 35, 36, 30, 96, 32, 99], [50, 48, 64, 0, 57, 117, 56, 120, 116, 126, 62, 59, 113, 122, 46, 123, 63, 53, 58, 110, 127, 55, 108, 49, 1, 109, 61, 124, 47, 40, 65, 29, 97, 51, 54, 107, 112, 43, 121, 45, 68, 2, 66, 119, 4, 6, 104, 60, 114, 106, 42, 52, 125, 67, 3, 5, 70, 93, 115, 41, 111, 33, 71, 95, 77, 13, 103, 16, 69, 7, 44, 39, 20, 80, 12, 75, 84, 9, 14, 74, 17, 11, 76, 78, 73, 10, 105, 81, 79, 15, 72, 82, 31, 18, 8, 118, 87, 23, 24, 22, 83, 19, 86, 102, 89, 25, 88, 27, 21, 92, 28, 90, 91, 85, 26, 38, 101, 98, 37, 94, 36, 34, 100, 99, 32, 30, 35, 96], [50, 48, 57, 0, 64, 117, 116, 56, 126, 59, 120, 122, 113, 62, 46, 123, 55, 53, 110, 58, 109, 63, 49, 127, 97, 124, 29, 1, 61, 40, 108, 47, 65, 3, 43, 112, 67, 114, 51, 107, 54, 119, 121, 4, 2, 125, 104, 68, 45, 60, 6, 42, 52, 106, 66, 33, 93, 20, 41, 95, 70, 77, 13, 80, 111, 12, 115, 11, 69, 71, 10, 5, 84, 7, 16, 39, 74, 103, 9, 82, 14, 17, 76, 73, 79, 44, 78, 15, 75, 72, 81, 8, 105, 31, 118, 23, 18, 87, 24, 88, 22, 102, 83, 85, 21, 19, 89, 92, 26, 25, 86, 90, 91, 27, 28, 38, 101, 37, 36, 98, 100, 35, 34, 94, 99, 96, 30, 32], [50, 48, 57, 117, 0, 64, 120, 56, 59, 116, 122, 126, 62, 113, 123, 55, 46, 53, 110, 127, 29, 58, 108, 63, 49, 97, 1, 124, 47, 61, 109, 65, 51, 112, 114, 40, 121, 67, 119, 54, 3, 43, 107, 125, 52, 60, 45, 104, 70, 2, 93, 6, 42, 66, 115, 106, 33, 13, 20, 16, 77, 80, 95, 69, 71, 12, 68, 11, 84, 111, 4, 75, 78, 41, 103, 79, 9, 15, 7, 76, 82, 18, 14, 81, 8, 5, 73, 39, 10, 74, 44, 17, 72, 105, 118, 23, 87, 31, 24, 85, 22, 19, 26, 92, 83, 25, 89, 86, 21, 90, 88, 102, 27, 28, 91, 37, 98, 36, 101, 94, 38, 35, 34, 30, 99, 96, 100, 32], [50, 48, 57, 0, 117, 64, 56, 116, 122, 59, 120, 113, 62, 126, 123, 53, 55, 46, 127, 110, 63, 97, 1, 49, 108, 65, 58, 112, 29, 109, 54, 61, 119, 47, 124, 114, 40, 121, 51, 125, 60, 107, 45, 43, 70, 104, 4, 52, 77, 106, 42, 95, 115, 33, 93, 6, 68, 66, 71, 13, 11, 2, 3, 15, 20, 14, 12, 75, 80, 67, 84, 41, 7, 16, 9, 78, 82, 44, 69, 111, 5, 39, 76, 103, 73, 17, 81, 79, 74, 18, 10, 8, 105, 31, 24, 118, 87, 23, 21, 19, 72, 25, 86, 22, 83, 85, 89, 90, 92, 26, 88, 27, 102, 28, 98, 91, 37, 101, 36, 94, 38, 30, 34, 32, 100, 35, 96, 99], [50, 48, 57, 64, 0, 117, 56, 120, 59, 116, 122, 126, 62, 113, 123, 53, 63, 46, 127, 1, 97, 110, 109, 61, 55, 108, 47, 124, 65, 40, 49, 112, 58, 29, 119, 54, 121, 3, 67, 51, 107, 43, 60, 125, 114, 68, 45, 4, 42, 104, 70, 106, 66, 95, 52, 2, 20, 115, 33, 11, 77, 13, 93, 6, 41, 71, 76, 15, 7, 14, 39, 16, 5, 84, 82, 78, 111, 80, 12, 103, 73, 9, 74, 81, 75, 69, 44, 18, 17, 10, 31, 79, 24, 105, 87, 8, 118, 23, 72, 85, 19, 83, 25, 90, 88, 89, 22, 86, 21, 92, 102, 28, 26, 27, 91, 98, 38, 101, 36, 37, 34, 35, 94, 96, 32, 100, 30, 99], [50, 48, 57, 64, 0, 117, 56, 59, 116, 126, 62, 122, 120, 113, 123, 46, 53, 97, 108, 127, 55, 65, 110, 63, 47, 1, 109, 49, 112, 61, 58, 124, 29, 119, 40, 54, 114, 51, 121, 107, 43, 67, 60, 45, 3, 104, 125, 70, 52, 42, 68, 66, 115, 106, 95, 33, 4, 15, 20, 77, 6, 2, 13, 93, 12, 14, 75, 79, 41, 69, 78, 5, 84, 7, 76, 80, 11, 82, 71, 39, 73, 103, 44, 18, 16, 81, 9, 10, 111, 87, 17, 8, 74, 118, 31, 24, 105, 23, 21, 22, 19, 25, 85, 88, 92, 90, 89, 83, 26, 72, 86, 28, 102, 27, 98, 37, 38, 91, 101, 34, 35, 96, 100, 30, 94, 32, 36, 99], [50, 48, 64, 57, 0, 117, 56, 116, 59, 126, 120, 113, 122, 123, 62, 65, 53, 46, 55, 1, 63, 58, 110, 127, 49, 97, 61, 124, 108, 109, 47, 54, 121, 51, 29, 40, 45, 112, 119, 114, 125, 66, 60, 43, 107, 70, 2, 67, 104, 3, 42, 52, 106, 33, 95, 115, 41, 71, 68, 20, 111, 13, 4, 93, 7, 5, 12, 77, 6, 10, 76, 84, 78, 14, 39, 69, 17, 81, 103, 16, 75, 73, 74, 82, 44, 9, 11, 80, 15, 31, 105, 79, 118, 18, 24, 8, 72, 87, 23, 89, 22, 19, 83, 25, 88, 92, 86, 28, 85, 21, 90, 102, 26, 101, 38, 91, 37, 27, 98, 35, 36, 94, 100, 34, 32, 96, 30, 99], [50, 48, 64, 0, 57, 117, 56, 126, 116, 59, 120, 62, 122, 113, 46, 123, 55, 53, 110, 58, 63, 127, 49, 65, 1, 67, 108, 3, 109, 61, 124, 40, 97, 51, 47, 29, 119, 121, 43, 112, 4, 60, 107, 54, 114, 45, 125, 68, 104, 2, 66, 42, 106, 70, 52, 71, 6, 111, 41, 33, 77, 95, 93, 115, 20, 69, 16, 13, 39, 5, 103, 10, 80, 7, 75, 11, 84, 73, 15, 44, 12, 74, 78, 76, 17, 9, 14, 82, 105, 81, 8, 18, 31, 79, 23, 118, 87, 24, 72, 19, 21, 88, 22, 83, 25, 90, 85, 89, 102, 26, 86, 92, 28, 91, 27, 101, 38, 98, 36, 37, 94, 35, 32, 30, 34, 99, 100, 96], [50, 48, 57, 64, 117, 0, 116, 120, 56, 122, 62, 126, 113, 59, 123, 46, 110, 53, 127, 55, 97, 109, 63, 108, 40, 49, 112, 61, 47, 1, 58, 124, 65, 29, 119, 51, 114, 54, 107, 121, 43, 3, 4, 60, 6, 42, 52, 104, 67, 125, 106, 45, 68, 2, 66, 115, 33, 95, 70, 15, 79, 16, 20, 93, 41, 13, 84, 11, 12, 77, 39, 71, 5, 44, 76, 14, 75, 73, 103, 78, 82, 7, 18, 9, 69, 10, 105, 80, 17, 74, 31, 111, 81, 87, 118, 24, 72, 8, 23, 88, 22, 26, 19, 25, 21, 92, 85, 86, 83, 90, 89, 28, 27, 102, 91, 98, 101, 38, 37, 34, 30, 36, 35, 94, 32, 99, 100, 96], [50, 48, 57, 64, 0, 117, 56, 116, 59, 113, 126, 122, 62, 120, 123, 53, 46, 97, 63, 127, 110, 55, 49, 1, 61, 109, 58, 124, 65, 108, 112, 47, 29, 121, 54, 40, 51, 114, 119, 60, 43, 107, 125, 6, 104, 52, 2, 67, 45, 42, 66, 33, 95, 106, 70, 3, 68, 4, 41, 115, 77, 76, 93, 5, 7, 16, 13, 78, 12, 20, 71, 84, 9, 39, 69, 79, 11, 18, 44, 10, 15, 80, 75, 103, 81, 74, 111, 14, 82, 73, 17, 72, 31, 24, 8, 105, 118, 87, 23, 19, 85, 88, 21, 86, 92, 25, 89, 22, 26, 83, 102, 90, 28, 27, 91, 38, 36, 98, 37, 101, 35, 94, 34, 100, 32, 99, 30, 96], [50, 48, 0, 57, 117, 64, 56, 120, 116, 59, 126, 122, 62, 113, 123, 53, 46, 108, 63, 55, 110, 127, 1, 49, 47, 61, 97, 65, 29, 58, 124, 3, 109, 67, 112, 119, 121, 40, 114, 51, 54, 60, 107, 6, 52, 43, 125, 2, 4, 45, 104, 66, 68, 106, 42, 33, 5, 95, 70, 93, 111, 71, 13, 77, 115, 41, 75, 7, 12, 79, 9, 39, 69, 20, 78, 84, 76, 73, 103, 82, 18, 14, 44, 80, 16, 10, 11, 74, 17, 81, 87, 72, 15, 105, 118, 24, 31, 8, 23, 90, 83, 22, 85, 88, 25, 19, 26, 86, 102, 92, 89, 21, 28, 27, 91, 101, 38, 37, 35, 98, 94, 96, 30, 36, 32, 100, 34, 99], [50, 48, 57, 0, 64, 117, 56, 116, 120, 126, 59, 122, 113, 62, 53, 63, 55, 46, 123, 58, 127, 110, 108, 65, 47, 61, 109, 49, 1, 40, 124, 97, 29, 121, 51, 119, 112, 114, 43, 54, 4, 60, 67, 107, 3, 68, 6, 45, 104, 125, 42, 2, 106, 66, 33, 52, 93, 115, 7, 41, 95, 20, 13, 111, 77, 70, 39, 9, 78, 71, 103, 75, 44, 84, 74, 12, 5, 73, 69, 80, 72, 76, 11, 79, 18, 81, 82, 10, 17, 16, 31, 14, 15, 105, 118, 8, 23, 87, 24, 83, 90, 102, 85, 88, 89, 25, 26, 22, 91, 19, 28, 98, 86, 21, 27, 92, 37, 38, 101, 94, 36, 35, 96, 30, 99, 100, 32, 34], [50, 48, 57, 117, 64, 0, 56, 116, 59, 62, 120, 126, 122, 113, 53, 46, 55, 123, 110, 127, 49, 58, 63, 109, 47, 97, 29, 108, 124, 54, 61, 65, 112, 114, 121, 40, 1, 51, 119, 60, 43, 107, 104, 66, 125, 52, 68, 2, 42, 45, 4, 67, 6, 33, 3, 70, 106, 115, 93, 95, 79, 13, 7, 5, 20, 16, 12, 11, 75, 78, 41, 77, 80, 9, 84, 82, 15, 14, 10, 103, 72, 44, 76, 69, 81, 73, 39, 71, 18, 111, 17, 118, 87, 105, 74, 31, 24, 23, 83, 8, 85, 21, 88, 26, 89, 86, 22, 92, 90, 25, 19, 102, 28, 27, 98, 91, 38, 101, 37, 94, 35, 30, 36, 34, 32, 100, 99, 96], [50, 48, 57, 117, 0, 64, 56, 59, 116, 126, 120, 122, 62, 113, 123, 53, 46, 127, 63, 110, 97, 55, 49, 58, 1, 61, 29, 124, 109, 47, 112, 108, 121, 67, 65, 51, 54, 119, 3, 114, 40, 107, 60, 66, 45, 125, 43, 2, 104, 70, 52, 6, 95, 13, 93, 106, 42, 4, 33, 5, 12, 115, 16, 68, 9, 77, 20, 14, 76, 84, 79, 7, 111, 75, 78, 71, 69, 82, 18, 15, 103, 41, 17, 39, 44, 81, 73, 11, 10, 80, 72, 31, 118, 74, 105, 24, 22, 87, 23, 85, 83, 25, 88, 21, 89, 86, 8, 19, 90, 92, 26, 28, 27, 102, 98, 91, 94, 38, 37, 101, 36, 30, 35, 96, 34, 100, 32, 99], [50, 48, 64, 57, 0, 117, 56, 59, 116, 120, 126, 113, 62, 122, 123, 55, 53, 46, 63, 127, 1, 58, 110, 97, 65, 61, 49, 121, 108, 109, 47, 119, 124, 29, 112, 40, 51, 114, 54, 60, 2, 107, 3, 125, 4, 68, 43, 70, 104, 45, 66, 67, 42, 52, 6, 106, 115, 95, 41, 33, 93, 5, 77, 13, 16, 20, 7, 79, 82, 12, 76, 71, 69, 9, 84, 111, 81, 103, 75, 73, 78, 11, 14, 10, 39, 44, 80, 17, 72, 118, 31, 18, 74, 15, 87, 23, 105, 8, 24, 22, 88, 19, 21, 83, 89, 85, 25, 102, 90, 86, 92, 28, 26, 27, 91, 98, 38, 94, 101, 37, 100, 36, 32, 35, 30, 96, 34, 99], [50, 48, 57, 0, 64, 117, 120, 56, 126, 116, 59, 62, 122, 113, 53, 123, 46, 63, 55, 127, 110, 58, 108, 1, 109, 61, 97, 65, 40, 49, 47, 112, 124, 29, 121, 51, 107, 68, 119, 114, 43, 60, 70, 4, 54, 2, 104, 42, 125, 66, 106, 52, 45, 3, 67, 33, 93, 95, 6, 115, 5, 69, 41, 71, 77, 79, 75, 84, 39, 13, 78, 12, 20, 44, 9, 103, 111, 7, 73, 82, 17, 76, 16, 10, 14, 18, 80, 105, 74, 72, 11, 31, 8, 15, 81, 118, 24, 23, 87, 83, 88, 22, 85, 89, 102, 19, 27, 92, 25, 21, 86, 91, 90, 26, 28, 38, 101, 98, 37, 30, 96, 94, 35, 34, 36, 100, 32, 99], [50, 48, 57, 64, 0, 117, 116, 120, 56, 126, 122, 59, 113, 62, 53, 123, 46, 63, 127, 55, 49, 108, 58, 110, 97, 124, 29, 67, 47, 109, 61, 112, 3, 51, 1, 121, 114, 40, 119, 54, 65, 107, 43, 60, 70, 33, 125, 104, 45, 2, 42, 106, 52, 95, 93, 66, 68, 71, 4, 13, 84, 77, 69, 6, 20, 111, 115, 17, 75, 76, 82, 41, 9, 80, 39, 7, 16, 103, 79, 5, 78, 74, 14, 44, 11, 12, 73, 18, 10, 15, 31, 81, 105, 87, 24, 118, 23, 72, 83, 8, 22, 88, 86, 19, 89, 90, 28, 25, 85, 21, 102, 92, 26, 91, 98, 27, 37, 36, 38, 101, 94, 32, 30, 99, 35, 100, 96, 34], [50, 48, 57, 64, 0, 117, 56, 120, 116, 59, 126, 62, 113, 122, 123, 55, 53, 1, 46, 63, 110, 58, 127, 65, 49, 61, 121, 97, 108, 112, 47, 29, 119, 124, 109, 51, 54, 114, 40, 60, 2, 66, 3, 125, 45, 107, 70, 104, 52, 43, 4, 6, 5, 115, 67, 95, 42, 93, 106, 68, 71, 69, 33, 77, 13, 76, 75, 9, 12, 78, 20, 80, 15, 111, 41, 18, 82, 7, 84, 103, 11, 73, 10, 39, 14, 17, 79, 8, 44, 118, 16, 81, 74, 31, 87, 105, 23, 72, 24, 25, 22, 21, 102, 19, 88, 28, 85, 89, 86, 83, 90, 92, 26, 27, 91, 98, 37, 38, 101, 30, 94, 96, 35, 34, 32, 100, 36, 99], [50, 48, 57, 0, 117, 64, 56, 116, 120, 126, 122, 59, 113, 62, 123, 46, 63, 53, 110, 127, 55, 58, 124, 97, 108, 49, 109, 47, 61, 40, 65, 112, 114, 29, 51, 119, 54, 107, 121, 1, 60, 4, 43, 125, 104, 68, 52, 42, 6, 3, 106, 33, 45, 70, 2, 95, 66, 93, 115, 67, 20, 77, 9, 5, 7, 71, 41, 13, 11, 16, 111, 103, 80, 14, 84, 15, 82, 12, 39, 69, 75, 10, 79, 17, 78, 44, 76, 73, 8, 74, 18, 105, 81, 118, 31, 24, 87, 72, 19, 23, 25, 22, 21, 27, 88, 26, 28, 83, 91, 86, 85, 92, 89, 90, 102, 38, 37, 98, 101, 30, 94, 34, 35, 96, 32, 36, 100, 99], [50, 48, 57, 0, 64, 117, 56, 120, 126, 116, 59, 122, 113, 62, 123, 46, 53, 110, 63, 55, 58, 49, 109, 127, 124, 97, 65, 112, 3, 108, 61, 47, 40, 1, 29, 67, 114, 43, 119, 121, 54, 51, 107, 60, 125, 104, 66, 42, 6, 68, 2, 33, 106, 4, 45, 52, 93, 95, 70, 13, 69, 41, 115, 12, 71, 17, 39, 44, 20, 79, 84, 9, 77, 16, 103, 76, 14, 8, 82, 74, 7, 5, 111, 80, 75, 78, 73, 11, 18, 15, 105, 10, 118, 31, 81, 87, 24, 23, 72, 88, 22, 19, 92, 26, 102, 89, 25, 28, 83, 21, 85, 86, 90, 27, 91, 38, 98, 101, 37, 36, 35, 94, 30, 100, 96, 99, 34, 32], [50, 48, 57, 64, 117, 0, 116, 56, 120, 126, 59, 122, 113, 62, 123, 55, 53, 110, 46, 63, 108, 58, 127, 1, 29, 109, 97, 49, 124, 114, 61, 40, 119, 51, 121, 47, 112, 54, 65, 125, 43, 107, 52, 60, 6, 45, 104, 2, 66, 67, 42, 106, 93, 70, 33, 3, 115, 95, 4, 69, 68, 80, 13, 11, 76, 41, 84, 71, 7, 77, 5, 74, 75, 103, 111, 14, 8, 79, 9, 20, 16, 17, 39, 73, 15, 82, 12, 78, 18, 44, 10, 31, 105, 118, 23, 87, 81, 72, 85, 24, 83, 25, 92, 22, 88, 86, 19, 89, 26, 102, 21, 28, 90, 91, 101, 38, 37, 27, 98, 94, 35, 30, 36, 34, 99, 96, 100, 32], [50, 48, 64, 57, 0, 117, 116, 120, 56, 126, 59, 113, 62, 122, 46, 123, 53, 63, 55, 110, 127, 58, 97, 49, 108, 109, 47, 1, 65, 112, 124, 61, 121, 29, 40, 114, 119, 51, 54, 60, 43, 107, 67, 125, 66, 3, 45, 6, 104, 42, 33, 2, 106, 52, 95, 68, 93, 115, 20, 4, 70, 77, 13, 41, 16, 71, 5, 7, 79, 111, 84, 69, 44, 11, 12, 39, 17, 76, 73, 75, 80, 82, 18, 14, 9, 103, 15, 10, 78, 31, 105, 74, 81, 24, 118, 8, 23, 72, 87, 19, 86, 92, 85, 88, 89, 25, 83, 28, 26, 22, 102, 21, 90, 37, 27, 98, 91, 101, 38, 30, 94, 35, 99, 34, 36, 32, 100, 96], [50, 48, 57, 0, 117, 64, 116, 56, 126, 113, 120, 59, 62, 122, 123, 53, 46, 63, 65, 127, 110, 49, 47, 97, 108, 61, 109, 124, 55, 112, 58, 29, 67, 1, 3, 51, 40, 54, 114, 119, 121, 43, 107, 68, 4, 45, 104, 60, 125, 6, 66, 2, 42, 33, 106, 95, 52, 93, 115, 20, 111, 77, 44, 11, 12, 13, 76, 84, 39, 7, 70, 71, 69, 41, 5, 73, 17, 78, 82, 103, 79, 9, 18, 10, 80, 16, 75, 15, 14, 74, 105, 81, 31, 24, 118, 8, 87, 23, 72, 83, 22, 86, 25, 88, 21, 92, 28, 85, 19, 26, 90, 102, 89, 27, 98, 37, 91, 38, 101, 100, 35, 94, 36, 32, 30, 99, 34, 96], [50, 48, 0, 57, 64, 117, 56, 120, 116, 59, 62, 113, 126, 122, 123, 46, 53, 63, 55, 110, 1, 58, 65, 108, 127, 109, 61, 97, 47, 49, 40, 121, 29, 124, 112, 51, 107, 119, 66, 114, 43, 54, 125, 45, 104, 60, 52, 42, 70, 106, 4, 2, 6, 68, 95, 3, 33, 67, 69, 93, 115, 77, 5, 41, 20, 13, 76, 12, 71, 7, 111, 11, 73, 103, 78, 80, 39, 9, 10, 79, 16, 14, 75, 84, 74, 18, 8, 44, 15, 82, 81, 31, 17, 105, 118, 87, 23, 72, 24, 19, 22, 86, 21, 88, 92, 102, 83, 85, 89, 25, 26, 90, 27, 28, 91, 38, 98, 37, 101, 94, 35, 34, 32, 36, 99, 30, 100, 96], [50, 48, 57, 0, 64, 117, 116, 120, 56, 59, 126, 113, 122, 62, 123, 55, 46, 58, 53, 63, 127, 49, 1, 108, 110, 109, 61, 97, 65, 47, 124, 112, 29, 119, 121, 40, 3, 51, 67, 54, 114, 107, 43, 104, 60, 2, 45, 70, 125, 42, 66, 106, 33, 13, 6, 95, 52, 4, 41, 93, 7, 68, 69, 111, 115, 71, 20, 5, 76, 78, 77, 74, 73, 11, 103, 14, 9, 12, 79, 84, 16, 39, 80, 75, 18, 81, 82, 44, 105, 17, 72, 8, 10, 15, 31, 118, 87, 24, 23, 25, 88, 22, 19, 83, 102, 86, 21, 89, 26, 85, 90, 28, 92, 91, 101, 27, 37, 38, 98, 94, 36, 35, 100, 30, 34, 96, 99, 32], [50, 48, 0, 57, 117, 64, 120, 116, 56, 59, 126, 113, 122, 62, 46, 53, 123, 127, 58, 110, 1, 55, 63, 109, 29, 97, 124, 49, 108, 65, 40, 3, 61, 47, 67, 51, 114, 112, 54, 121, 43, 60, 107, 119, 68, 70, 104, 2, 125, 42, 4, 45, 66, 33, 106, 52, 115, 95, 93, 6, 71, 77, 20, 41, 73, 5, 69, 13, 44, 111, 80, 16, 78, 7, 76, 12, 103, 74, 11, 9, 79, 39, 84, 81, 75, 10, 18, 14, 105, 72, 15, 31, 17, 8, 82, 118, 24, 87, 23, 22, 88, 86, 83, 19, 25, 26, 21, 92, 89, 85, 102, 28, 90, 91, 27, 38, 101, 98, 34, 37, 36, 94, 32, 100, 30, 99, 96, 35], [50, 48, 57, 117, 64, 0, 116, 59, 126, 56, 122, 120, 62, 113, 123, 46, 53, 49, 55, 127, 58, 110, 97, 108, 63, 109, 47, 124, 29, 61, 54, 112, 119, 114, 121, 40, 65, 1, 51, 107, 104, 70, 43, 125, 4, 60, 52, 42, 45, 33, 68, 93, 95, 66, 11, 13, 106, 20, 2, 6, 15, 79, 76, 77, 115, 9, 5, 18, 84, 74, 80, 44, 14, 78, 16, 103, 81, 67, 41, 75, 69, 72, 82, 12, 3, 73, 71, 7, 39, 10, 105, 111, 17, 118, 87, 31, 23, 24, 8, 85, 22, 88, 25, 19, 83, 26, 92, 21, 89, 86, 102, 90, 27, 28, 91, 38, 98, 37, 101, 94, 36, 34, 99, 30, 100, 35, 96, 32], [50, 48, 57, 64, 117, 0, 116, 120, 56, 126, 113, 62, 122, 59, 123, 53, 46, 55, 63, 127, 110, 97, 109, 108, 58, 124, 61, 49, 112, 47, 65, 29, 40, 114, 125, 119, 1, 121, 43, 54, 51, 60, 3, 107, 67, 45, 66, 70, 104, 33, 52, 42, 95, 106, 115, 2, 6, 93, 13, 5, 16, 20, 77, 14, 7, 76, 41, 84, 9, 11, 12, 44, 4, 71, 111, 81, 79, 103, 68, 18, 80, 39, 17, 75, 15, 82, 74, 10, 105, 69, 78, 73, 72, 23, 87, 31, 118, 24, 8, 19, 85, 22, 86, 88, 83, 92, 89, 25, 28, 26, 102, 90, 21, 27, 91, 38, 98, 37, 101, 94, 36, 35, 30, 34, 100, 99, 96, 32]], "model.layers.2.self_attn.q_proj": [[104, 127, 46, 34, 49, 88, 113, 82, 21, 79, 75, 76, 24, 14, 71, 7, 5, 9, 92, 40, 107, 3, 15, 19, 52, 125, 62, 11, 119, 27, 121, 12, 78, 120, 20, 116, 66, 8, 93, 53, 50, 77, 109, 57, 47, 10, 56, 69, 2, 96, 18, 65, 84, 86, 85, 73, 60, 30, 91, 115, 72, 63, 41, 83, 54, 1, 45, 95, 17, 51, 74, 99, 29, 81, 25, 59, 28, 118, 16, 108, 55, 103, 31, 101, 64, 106, 48, 32, 97, 102, 13, 90, 87, 61, 80, 35, 112, 124, 94, 39, 43, 122, 42, 23, 67, 26, 38, 117, 4, 58, 100, 33, 111, 126, 6, 105, 22, 44, 37, 70, 36, 110, 123, 114, 89, 98, 68, 0], [104, 49, 46, 34, 127, 113, 21, 108, 24, 82, 79, 105, 40, 57, 9, 88, 75, 107, 76, 121, 56, 80, 5, 27, 120, 51, 90, 14, 30, 71, 92, 62, 26, 87, 48, 16, 22, 83, 35, 28, 52, 43, 17, 50, 125, 63, 84, 106, 94, 4, 117, 23, 111, 109, 61, 89, 59, 29, 3, 100, 45, 114, 119, 126, 42, 118, 19, 36, 47, 54, 67, 38, 110, 70, 116, 115, 10, 123, 96, 72, 98, 68, 33, 122, 11, 41, 44, 124, 93, 99, 20, 112, 102, 39, 103, 55, 95, 101, 77, 53, 37, 60, 7, 31, 18, 66, 32, 81, 58, 85, 86, 91, 97, 69, 78, 73, 65, 15, 25, 1, 13, 6, 8, 64, 12, 74, 0, 2], [104, 34, 49, 127, 46, 21, 82, 71, 88, 79, 113, 76, 9, 75, 3, 5, 40, 14, 26, 121, 7, 53, 64, 1, 125, 66, 23, 119, 8, 52, 67, 10, 107, 24, 11, 81, 73, 4, 15, 17, 57, 120, 69, 77, 50, 19, 2, 20, 85, 22, 18, 32, 94, 70, 84, 6, 74, 72, 62, 92, 33, 12, 65, 16, 93, 89, 90, 63, 87, 56, 96, 45, 30, 126, 38, 31, 27, 51, 35, 13, 78, 42, 80, 108, 44, 43, 118, 103, 47, 54, 124, 48, 100, 86, 58, 95, 115, 91, 60, 28, 36, 83, 25, 37, 29, 117, 123, 102, 116, 39, 114, 122, 101, 111, 97, 112, 99, 109, 105, 41, 55, 68, 0, 59, 61, 106, 110, 98], [104, 46, 49, 34, 127, 113, 82, 21, 107, 88, 75, 125, 14, 52, 90, 53, 9, 22, 28, 79, 26, 24, 92, 5, 38, 121, 40, 19, 105, 94, 77, 76, 120, 119, 84, 27, 48, 51, 103, 50, 111, 39, 93, 29, 62, 16, 41, 60, 30, 98, 17, 124, 71, 91, 36, 56, 57, 126, 99, 118, 3, 81, 47, 106, 33, 123, 110, 109, 117, 115, 54, 44, 112, 63, 86, 37, 80, 59, 116, 100, 42, 35, 108, 20, 87, 55, 31, 122, 102, 83, 73, 58, 101, 70, 114, 96, 32, 97, 11, 15, 78, 23, 95, 43, 69, 72, 45, 61, 10, 89, 66, 6, 1, 7, 13, 18, 85, 65, 64, 67, 8, 25, 4, 68, 0, 74, 12, 2], [104, 107, 91, 35, 23, 84, 17, 27, 15, 13, 81, 47, 20, 94, 112, 113, 82, 114, 61, 43, 55, 99, 75, 77, 40, 56, 89, 53, 86, 111, 10, 96, 79, 92, 25, 117, 119, 123, 11, 28, 125, 51, 21, 115, 97, 118, 30, 124, 9, 73, 80, 93, 12, 16, 57, 120, 29, 48, 87, 122, 46, 76, 42, 70, 6, 102, 24, 26, 72, 18, 88, 116, 7, 110, 49, 108, 85, 38, 83, 14, 98, 105, 74, 22, 50, 68, 67, 60, 101, 31, 90, 41, 59, 71, 106, 8, 37, 32, 33, 78, 39, 44, 19, 63, 58, 127, 126, 103, 34, 45, 54, 69, 121, 36, 100, 95, 4, 62, 52, 109, 2, 3, 0, 65, 66, 5, 1, 64], [104, 107, 35, 91, 27, 23, 99, 94, 84, 114, 47, 43, 113, 89, 112, 61, 9, 55, 119, 123, 53, 15, 40, 73, 111, 92, 17, 12, 82, 81, 120, 28, 98, 77, 56, 46, 105, 117, 125, 97, 25, 67, 86, 116, 70, 48, 115, 122, 7, 124, 42, 13, 36, 102, 118, 51, 32, 14, 90, 29, 57, 22, 79, 11, 49, 37, 10, 85, 63, 16, 41, 109, 68, 126, 6, 103, 44, 110, 54, 121, 72, 76, 106, 58, 75, 108, 34, 93, 45, 96, 101, 52, 59, 69, 39, 38, 24, 87, 30, 95, 100, 80, 50, 18, 60, 31, 127, 74, 26, 33, 71, 62, 88, 21, 19, 83, 65, 20, 2, 78, 4, 0, 3, 8, 66, 64, 5, 1], [107, 104, 47, 35, 99, 124, 94, 89, 96, 27, 55, 91, 43, 61, 119, 112, 113, 46, 25, 123, 85, 63, 105, 48, 114, 120, 53, 116, 125, 111, 23, 54, 40, 84, 102, 52, 28, 56, 117, 97, 37, 51, 115, 82, 121, 126, 92, 42, 109, 62, 22, 41, 32, 106, 60, 29, 118, 44, 110, 95, 49, 57, 86, 122, 98, 38, 108, 100, 58, 127, 36, 34, 50, 12, 101, 33, 103, 17, 59, 67, 83, 26, 45, 31, 93, 39, 14, 78, 90, 30, 21, 15, 73, 81, 16, 24, 77, 6, 88, 18, 7, 19, 80, 79, 9, 10, 20, 68, 66, 11, 71, 70, 3, 76, 75, 13, 87, 2, 74, 72, 65, 4, 0, 8, 64, 5, 1, 69], [104, 107, 35, 15, 75, 8, 13, 17, 91, 5, 84, 4, 23, 99, 73, 40, 43, 3, 1, 71, 72, 87, 112, 114, 69, 61, 68, 6, 113, 55, 67, 20, 2, 47, 53, 123, 7, 66, 0, 56, 65, 11, 120, 70, 115, 111, 10, 119, 89, 9, 25, 110, 77, 30, 64, 125, 57, 21, 16, 46, 124, 12, 94, 18, 51, 117, 86, 79, 74, 122, 108, 81, 76, 58, 14, 92, 49, 78, 85, 83, 82, 96, 127, 116, 121, 19, 90, 28, 88, 26, 102, 118, 31, 29, 80, 97, 105, 59, 60, 48, 32, 95, 93, 24, 22, 39, 37, 62, 63, 34, 50, 106, 33, 44, 42, 38, 101, 36, 27, 98, 100, 103, 54, 41, 109, 52, 45, 126], [110, 38, 97, 112, 46, 16, 89, 78, 10, 19, 8, 4, 12, 70, 68, 48, 25, 2, 69, 7, 6, 86, 82, 56, 1, 31, 20, 64, 50, 5, 102, 71, 22, 75, 125, 119, 11, 113, 3, 111, 13, 83, 67, 0, 127, 77, 61, 66, 47, 109, 51, 80, 14, 85, 73, 52, 118, 74, 72, 84, 94, 9, 81, 116, 117, 41, 93, 40, 120, 123, 18, 76, 87, 124, 107, 63, 15, 23, 21, 115, 65, 30, 104, 79, 36, 37, 43, 28, 34, 55, 88, 53, 91, 62, 24, 59, 17, 98, 108, 27, 60, 126, 99, 96, 45, 103, 32, 44, 57, 100, 101, 121, 58, 42, 122, 106, 29, 39, 92, 54, 35, 49, 95, 114, 26, 90, 105, 33], [110, 38, 112, 97, 46, 19, 78, 16, 12, 10, 8, 89, 4, 68, 69, 48, 71, 25, 5, 9, 70, 6, 102, 13, 24, 50, 2, 1, 88, 51, 62, 111, 66, 77, 65, 56, 109, 31, 118, 64, 113, 21, 15, 75, 61, 22, 127, 85, 73, 72, 3, 7, 67, 55, 14, 30, 74, 17, 27, 0, 34, 76, 83, 79, 124, 86, 115, 125, 20, 11, 100, 43, 94, 52, 122, 104, 107, 116, 87, 93, 99, 23, 105, 80, 84, 117, 95, 119, 47, 126, 121, 35, 82, 18, 91, 98, 44, 58, 120, 81, 123, 26, 40, 54, 42, 49, 45, 37, 101, 32, 92, 53, 39, 59, 36, 41, 106, 96, 108, 60, 29, 114, 103, 90, 28, 63, 57, 33], [38, 97, 110, 94, 112, 46, 25, 19, 83, 89, 78, 48, 93, 118, 85, 16, 50, 11, 96, 62, 39, 56, 21, 111, 127, 36, 77, 61, 27, 109, 90, 106, 80, 101, 17, 10, 88, 30, 12, 125, 60, 87, 95, 44, 86, 40, 105, 35, 121, 34, 31, 24, 104, 59, 116, 108, 115, 120, 57, 43, 54, 26, 123, 58, 81, 103, 126, 117, 20, 99, 23, 119, 13, 45, 14, 52, 124, 51, 37, 122, 29, 73, 100, 55, 42, 91, 92, 47, 107, 63, 22, 75, 32, 41, 114, 98, 53, 113, 9, 28, 33, 49, 15, 84, 6, 8, 82, 7, 18, 79, 69, 71, 76, 74, 70, 72, 5, 68, 4, 102, 2, 66, 3, 67, 65, 1, 0, 64], [110, 38, 97, 112, 46, 89, 78, 16, 10, 77, 25, 17, 19, 12, 8, 48, 6, 69, 11, 71, 102, 94, 51, 15, 50, 4, 75, 68, 2, 86, 21, 83, 62, 76, 31, 70, 52, 5, 72, 111, 81, 9, 127, 1, 22, 108, 99, 29, 121, 118, 109, 84, 124, 14, 7, 116, 82, 92, 36, 45, 37, 117, 74, 123, 90, 23, 79, 88, 80, 104, 35, 64, 105, 95, 61, 56, 66, 34, 58, 65, 103, 73, 93, 42, 44, 33, 107, 122, 28, 126, 20, 18, 55, 106, 87, 27, 113, 57, 47, 125, 53, 98, 85, 115, 41, 24, 54, 114, 26, 120, 30, 39, 101, 59, 40, 13, 96, 119, 67, 60, 100, 91, 43, 32, 49, 63, 3, 0], [40, 98, 121, 125, 53, 51, 17, 20, 82, 80, 18, 115, 27, 19, 21, 88, 104, 77, 22, 42, 119, 12, 89, 26, 95, 86, 30, 75, 92, 28, 84, 78, 25, 117, 96, 94, 29, 32, 120, 23, 93, 90, 57, 102, 116, 52, 79, 38, 127, 41, 24, 97, 50, 87, 123, 105, 56, 122, 10, 109, 113, 16, 31, 63, 101, 110, 43, 44, 48, 107, 33, 61, 49, 111, 36, 58, 62, 76, 124, 100, 91, 39, 114, 59, 45, 81, 108, 83, 46, 37, 35, 9, 103, 99, 60, 112, 47, 55, 85, 8, 126, 118, 54, 74, 14, 34, 71, 73, 15, 106, 7, 13, 11, 6, 72, 5, 70, 2, 69, 65, 68, 3, 64, 4, 67, 66, 1, 0], [40, 125, 121, 53, 51, 98, 27, 20, 82, 16, 84, 81, 12, 21, 42, 19, 115, 10, 119, 88, 8, 78, 74, 29, 50, 85, 86, 35, 72, 91, 36, 95, 5, 6, 117, 14, 94, 69, 104, 17, 18, 34, 93, 24, 76, 22, 65, 67, 97, 26, 79, 68, 75, 120, 38, 116, 15, 23, 30, 122, 44, 57, 63, 66, 107, 89, 58, 80, 71, 83, 109, 87, 7, 106, 9, 99, 108, 96, 111, 41, 33, 118, 13, 105, 103, 101, 48, 127, 114, 56, 100, 37, 124, 52, 28, 45, 90, 3, 113, 49, 92, 43, 77, 39, 32, 123, 59, 102, 110, 46, 31, 60, 126, 54, 70, 25, 61, 112, 62, 64, 47, 0, 55, 11, 4, 73, 1, 2], [40, 121, 42, 53, 125, 51, 98, 91, 123, 50, 110, 52, 111, 113, 58, 63, 105, 126, 47, 33, 115, 27, 54, 60, 118, 93, 117, 107, 97, 59, 122, 127, 48, 108, 61, 56, 15, 44, 124, 109, 57, 112, 116, 43, 45, 114, 62, 46, 120, 106, 38, 119, 49, 55, 41, 95, 23, 103, 39, 36, 101, 31, 100, 24, 86, 37, 99, 32, 35, 102, 30, 21, 12, 29, 96, 18, 85, 34, 89, 104, 94, 77, 92, 17, 81, 84, 83, 78, 79, 20, 74, 87, 9, 28, 69, 72, 13, 26, 88, 19, 75, 90, 25, 82, 22, 7, 10, 80, 2, 65, 70, 4, 76, 8, 11, 64, 3, 16, 6, 68, 1, 73, 67, 14, 5, 0, 71, 66], [53, 125, 51, 121, 42, 40, 115, 98, 119, 52, 108, 57, 123, 116, 91, 107, 50, 114, 117, 122, 127, 124, 120, 58, 111, 59, 118, 113, 63, 15, 55, 41, 60, 110, 48, 49, 9, 109, 56, 61, 100, 62, 44, 43, 46, 7, 105, 99, 112, 45, 54, 77, 126, 47, 102, 72, 39, 38, 33, 74, 12, 103, 69, 106, 93, 18, 101, 35, 30, 17, 95, 75, 78, 2, 70, 31, 11, 37, 27, 32, 80, 36, 84, 3, 24, 104, 34, 23, 96, 97, 4, 86, 68, 94, 83, 79, 64, 8, 29, 92, 10, 89, 85, 6, 65, 13, 0, 73, 76, 71, 26, 20, 87, 28, 21, 19, 88, 25, 90, 5, 1, 66, 16, 81, 67, 14, 22, 82], [102, 49, 98, 56, 116, 121, 113, 89, 20, 11, 81, 9, 77, 22, 15, 124, 52, 25, 2, 6, 73, 71, 18, 96, 75, 86, 110, 72, 88, 79, 70, 101, 19, 51, 10, 74, 100, 4, 3, 29, 67, 91, 1, 31, 83, 87, 92, 78, 17, 64, 13, 60, 76, 16, 120, 24, 84, 59, 61, 8, 30, 41, 80, 14, 21, 7, 58, 93, 47, 37, 12, 26, 28, 117, 62, 115, 90, 44, 46, 82, 32, 94, 27, 112, 63, 68, 103, 40, 105, 109, 33, 39, 42, 85, 95, 114, 97, 111, 35, 122, 48, 107, 23, 118, 53, 55, 54, 45, 127, 43, 123, 50, 104, 106, 119, 34, 126, 57, 99, 36, 108, 0, 125, 5, 69, 66, 65, 38], [102, 49, 116, 56, 98, 121, 113, 89, 81, 124, 20, 11, 15, 77, 68, 9, 71, 52, 70, 25, 29, 2, 64, 91, 19, 85, 4, 0, 105, 88, 1, 65, 66, 8, 3, 73, 101, 41, 46, 75, 12, 127, 110, 62, 60, 51, 72, 80, 93, 21, 67, 7, 27, 74, 14, 86, 96, 76, 47, 117, 13, 126, 18, 84, 44, 87, 6, 111, 115, 120, 79, 17, 22, 82, 90, 112, 10, 95, 107, 5, 61, 39, 24, 23, 16, 103, 30, 122, 92, 57, 83, 94, 32, 97, 118, 45, 78, 33, 26, 59, 36, 69, 43, 123, 109, 53, 37, 40, 100, 54, 114, 28, 50, 125, 99, 42, 108, 106, 48, 55, 35, 104, 58, 31, 63, 119, 34, 38], [102, 49, 116, 56, 98, 113, 121, 89, 77, 11, 81, 20, 9, 71, 15, 52, 68, 124, 110, 29, 19, 91, 86, 10, 101, 25, 6, 70, 2, 41, 66, 74, 47, 7, 4, 14, 46, 59, 67, 117, 82, 75, 72, 80, 64, 18, 96, 92, 73, 0, 31, 3, 95, 79, 115, 126, 12, 8, 21, 16, 93, 1, 22, 13, 87, 17, 60, 39, 88, 36, 107, 127, 50, 97, 32, 35, 120, 33, 122, 69, 53, 105, 85, 30, 61, 65, 84, 111, 62, 104, 5, 26, 42, 58, 118, 76, 94, 114, 23, 83, 112, 51, 24, 109, 54, 27, 44, 28, 40, 90, 125, 119, 100, 99, 78, 43, 57, 55, 48, 106, 123, 108, 103, 63, 45, 37, 34, 38], [102, 121, 110, 49, 98, 56, 116, 113, 101, 89, 25, 62, 93, 29, 31, 52, 91, 41, 51, 46, 107, 27, 45, 126, 34, 40, 100, 38, 115, 58, 106, 60, 112, 20, 105, 63, 96, 85, 39, 48, 103, 22, 57, 19, 61, 119, 86, 50, 47, 108, 53, 18, 123, 109, 114, 55, 127, 54, 120, 84, 104, 28, 111, 78, 94, 44, 118, 43, 83, 122, 81, 21, 42, 125, 124, 59, 92, 80, 30, 24, 16, 117, 35, 95, 37, 33, 26, 32, 88, 99, 36, 15, 97, 23, 77, 76, 14, 87, 90, 82, 10, 12, 5, 17, 71, 79, 2, 11, 69, 66, 8, 13, 9, 74, 7, 72, 70, 68, 75, 67, 4, 6, 64, 73, 65, 0, 3, 1], [126, 61, 56, 60, 54, 51, 123, 63, 124, 119, 100, 58, 117, 32, 127, 108, 112, 49, 39, 97, 114, 47, 104, 42, 125, 44, 106, 46, 40, 41, 48, 118, 99, 62, 110, 53, 120, 34, 52, 105, 121, 57, 107, 115, 116, 102, 55, 103, 43, 122, 50, 45, 113, 59, 111, 109, 38, 36, 33, 101, 96, 37, 94, 98, 30, 85, 31, 35, 8, 28, 25, 67, 70, 7, 5, 91, 29, 73, 2, 74, 27, 68, 92, 75, 77, 95, 93, 83, 10, 22, 4, 21, 65, 11, 76, 26, 71, 13, 89, 64, 12, 0, 9, 90, 19, 1, 78, 86, 23, 72, 66, 80, 88, 87, 15, 24, 6, 79, 14, 20, 69, 82, 81, 84, 3, 16, 17, 18], [55, 119, 123, 127, 115, 124, 60, 58, 56, 54, 63, 51, 61, 117, 62, 113, 126, 59, 96, 57, 52, 109, 48, 53, 121, 116, 120, 107, 114, 45, 108, 43, 50, 122, 110, 112, 42, 47, 125, 49, 118, 111, 44, 94, 46, 106, 105, 41, 32, 93, 104, 40, 103, 95, 39, 30, 23, 102, 98, 91, 34, 38, 90, 24, 92, 37, 17, 16, 101, 88, 22, 18, 100, 79, 7, 31, 35, 10, 78, 99, 75, 89, 8, 29, 84, 13, 26, 70, 36, 73, 5, 76, 19, 81, 15, 21, 20, 97, 80, 33, 9, 4, 27, 65, 0, 2, 87, 67, 85, 68, 66, 77, 25, 83, 28, 72, 14, 3, 69, 82, 12, 86, 1, 71, 74, 6, 11, 64], [119, 123, 55, 100, 124, 127, 56, 60, 58, 54, 115, 51, 63, 61, 36, 117, 126, 62, 32, 98, 59, 113, 52, 57, 121, 48, 45, 109, 120, 116, 53, 43, 107, 50, 30, 122, 114, 47, 106, 108, 110, 111, 112, 125, 118, 42, 49, 44, 46, 34, 41, 105, 39, 38, 103, 101, 104, 102, 40, 35, 37, 31, 96, 33, 99, 93, 97, 95, 28, 22, 88, 91, 94, 29, 19, 24, 68, 77, 64, 85, 90, 92, 27, 12, 26, 86, 2, 5, 83, 14, 74, 1, 13, 6, 25, 11, 72, 71, 67, 7, 8, 21, 69, 3, 73, 79, 78, 10, 9, 16, 81, 76, 20, 70, 89, 66, 80, 75, 87, 15, 65, 82, 18, 0, 4, 23, 17, 84], [126, 61, 100, 26, 54, 60, 28, 20, 63, 119, 51, 58, 123, 56, 124, 87, 55, 96, 18, 93, 24, 127, 25, 115, 86, 117, 85, 92, 30, 27, 94, 17, 97, 90, 33, 91, 62, 31, 32, 49, 112, 108, 106, 125, 114, 34, 104, 42, 39, 47, 120, 46, 44, 48, 53, 110, 121, 57, 41, 118, 40, 52, 105, 59, 43, 116, 107, 103, 50, 113, 122, 45, 109, 36, 101, 80, 102, 111, 98, 23, 81, 38, 15, 16, 89, 21, 99, 37, 83, 88, 95, 29, 35, 82, 19, 22, 14, 79, 78, 84, 9, 13, 75, 10, 71, 74, 12, 76, 8, 7, 67, 5, 70, 66, 72, 4, 77, 69, 65, 73, 11, 2, 0, 64, 1, 3, 6, 68], [33, 40, 108, 117, 57, 54, 22, 19, 118, 122, 82, 24, 16, 121, 77, 89, 127, 25, 76, 14, 52, 7, 125, 49, 44, 12, 123, 92, 109, 93, 106, 107, 3, 13, 69, 73, 11, 115, 46, 29, 43, 84, 62, 21, 53, 104, 120, 32, 119, 39, 28, 20, 41, 94, 124, 98, 61, 67, 87, 86, 88, 30, 111, 83, 34, 112, 59, 2, 15, 74, 64, 70, 65, 95, 48, 85, 71, 47, 60, 91, 105, 63, 96, 6, 17, 38, 68, 27, 81, 58, 42, 113, 116, 126, 110, 8, 23, 45, 72, 4, 114, 97, 79, 50, 90, 78, 5, 18, 10, 37, 26, 31, 1, 51, 101, 35, 103, 99, 102, 100, 55, 56, 80, 36, 66, 0, 9, 75], [108, 40, 33, 57, 54, 86, 25, 44, 118, 18, 117, 82, 80, 52, 91, 29, 104, 62, 121, 94, 115, 90, 92, 122, 89, 31, 56, 30, 126, 27, 95, 93, 48, 28, 88, 63, 34, 51, 110, 99, 124, 35, 59, 101, 112, 123, 26, 98, 113, 36, 60, 37, 43, 116, 103, 109, 41, 32, 39, 47, 58, 46, 42, 105, 119, 102, 100, 96, 45, 111, 106, 120, 49, 125, 38, 50, 107, 14, 53, 127, 55, 61, 23, 84, 87, 16, 114, 24, 85, 20, 83, 77, 21, 22, 76, 19, 78, 97, 81, 79, 17, 75, 11, 13, 66, 70, 12, 15, 74, 5, 4, 6, 7, 3, 67, 0, 9, 68, 8, 72, 65, 71, 2, 1, 10, 64, 69, 73], [33, 40, 108, 57, 54, 16, 14, 89, 118, 44, 117, 78, 82, 22, 13, 104, 52, 24, 121, 27, 71, 19, 122, 12, 75, 25, 90, 87, 28, 62, 109, 124, 93, 99, 94, 48, 115, 30, 31, 29, 123, 110, 101, 26, 126, 56, 91, 127, 112, 95, 53, 63, 119, 96, 46, 59, 37, 73, 21, 120, 51, 86, 36, 98, 92, 77, 43, 116, 102, 105, 85, 41, 103, 42, 38, 34, 106, 35, 60, 32, 125, 88, 111, 100, 47, 113, 58, 23, 39, 45, 76, 49, 9, 50, 61, 107, 114, 55, 5, 17, 20, 83, 6, 18, 81, 84, 79, 68, 15, 80, 10, 74, 7, 3, 11, 8, 67, 72, 66, 70, 2, 97, 4, 1, 69, 65, 0, 64], [40, 108, 33, 57, 54, 11, 12, 89, 82, 117, 118, 76, 44, 16, 14, 22, 19, 73, 7, 9, 24, 104, 86, 77, 10, 71, 78, 75, 121, 27, 13, 127, 52, 122, 93, 70, 85, 74, 123, 48, 109, 23, 87, 66, 124, 43, 126, 106, 90, 21, 68, 101, 115, 53, 110, 28, 112, 30, 81, 26, 80, 31, 15, 119, 79, 99, 4, 63, 2, 64, 84, 25, 46, 94, 83, 62, 51, 3, 56, 96, 114, 95, 29, 98, 111, 120, 116, 36, 37, 59, 35, 32, 91, 92, 42, 102, 1, 41, 100, 47, 113, 60, 38, 125, 65, 88, 49, 20, 34, 103, 39, 17, 105, 6, 45, 50, 58, 107, 55, 69, 61, 72, 18, 67, 0, 5, 8, 97], [110, 103, 33, 31, 46, 21, 78, 76, 80, 10, 8, 19, 114, 120, 79, 13, 112, 70, 4, 82, 17, 116, 49, 12, 6, 123, 16, 61, 54, 57, 89, 124, 58, 121, 25, 73, 125, 75, 47, 39, 22, 107, 18, 67, 117, 1, 118, 94, 27, 83, 56, 43, 85, 108, 111, 115, 69, 77, 48, 2, 14, 101, 63, 24, 32, 50, 51, 29, 106, 86, 93, 44, 74, 113, 23, 62, 41, 15, 28, 119, 71, 40, 91, 68, 36, 104, 127, 55, 30, 84, 11, 20, 59, 9, 92, 38, 42, 102, 72, 88, 35, 122, 37, 34, 99, 60, 52, 81, 26, 90, 45, 87, 7, 96, 98, 105, 100, 53, 126, 109, 97, 5, 66, 0, 95, 3, 65, 64], [103, 110, 33, 31, 46, 21, 19, 114, 49, 78, 120, 116, 23, 76, 12, 24, 57, 112, 79, 80, 72, 58, 16, 61, 118, 14, 4, 8, 6, 123, 48, 54, 50, 18, 67, 111, 47, 83, 98, 10, 85, 95, 45, 89, 17, 15, 108, 117, 121, 127, 73, 91, 109, 41, 63, 20, 1, 43, 90, 104, 37, 88, 81, 55, 102, 13, 39, 32, 119, 28, 113, 94, 2, 115, 70, 84, 44, 60, 51, 126, 42, 38, 96, 53, 105, 125, 122, 35, 34, 124, 106, 59, 97, 27, 29, 100, 74, 82, 9, 36, 107, 99, 93, 75, 56, 52, 40, 101, 71, 69, 86, 92, 30, 65, 3, 87, 22, 26, 62, 5, 25, 68, 11, 77, 0, 66, 7, 64], [110, 103, 33, 31, 46, 80, 21, 10, 8, 76, 78, 4, 114, 70, 68, 57, 120, 5, 49, 123, 58, 116, 112, 16, 17, 69, 3, 61, 1, 39, 65, 66, 12, 117, 121, 67, 74, 118, 54, 124, 9, 73, 7, 71, 127, 72, 14, 50, 32, 15, 11, 75, 107, 47, 125, 0, 51, 84, 111, 64, 45, 48, 79, 25, 77, 2, 85, 6, 91, 89, 87, 13, 93, 82, 60, 24, 63, 19, 109, 23, 81, 104, 18, 88, 122, 20, 83, 56, 92, 29, 38, 37, 115, 105, 86, 55, 119, 113, 35, 102, 36, 106, 53, 28, 44, 41, 62, 94, 42, 22, 30, 108, 95, 52, 34, 26, 96, 99, 43, 40, 101, 59, 100, 98, 90, 97, 27, 126], [110, 103, 33, 31, 46, 21, 10, 80, 78, 79, 69, 8, 76, 120, 114, 70, 82, 2, 6, 67, 49, 57, 112, 73, 4, 123, 58, 12, 61, 39, 5, 66, 1, 116, 68, 23, 71, 3, 16, 13, 0, 19, 17, 127, 56, 117, 111, 72, 74, 118, 121, 7, 107, 125, 14, 51, 50, 64, 54, 87, 11, 9, 45, 75, 124, 65, 85, 24, 48, 88, 81, 43, 20, 77, 15, 89, 84, 126, 119, 94, 59, 42, 18, 32, 47, 92, 26, 37, 115, 63, 86, 83, 25, 109, 36, 62, 28, 104, 53, 44, 106, 91, 122, 60, 55, 40, 34, 27, 41, 108, 99, 30, 113, 105, 102, 22, 52, 97, 90, 35, 98, 96, 29, 100, 95, 38, 93, 101]], "model.layers.2.self_attn.k_proj": [[40, 49, 110, 127, 46, 98, 76, 9, 21, 82, 24, 79, 71, 14, 5, 75, 67, 66, 0, 64, 65, 70, 125, 88, 53, 2, 92, 81, 50, 69, 26, 93, 121, 4, 27, 116, 77, 30, 112, 52, 105, 74, 80, 119, 118, 97, 62, 13, 51, 56, 72, 57, 25, 68, 31, 109, 23, 106, 1, 126, 54, 120, 113, 83, 28, 84, 63, 48, 107, 29, 124, 44, 43, 59, 89, 20, 99, 45, 73, 47, 86, 60, 42, 108, 87, 61, 16, 95, 8, 115, 90, 122, 117, 32, 39, 17, 19, 36, 96, 22, 114, 100, 41, 33, 103, 91, 123, 35, 58, 55, 37, 111, 10, 94, 102, 38, 101, 6, 104, 15, 3, 78, 7, 12, 85, 11, 34, 18], [43, 40, 99, 56, 48, 50, 119, 23, 117, 61, 84, 13, 113, 15, 17, 75, 123, 5, 51, 47, 0, 8, 111, 4, 57, 91, 55, 53, 46, 3, 104, 94, 65, 49, 6, 124, 73, 114, 44, 110, 125, 2, 89, 80, 10, 71, 60, 120, 64, 107, 122, 108, 121, 1, 12, 58, 118, 14, 86, 42, 85, 66, 25, 82, 70, 59, 52, 62, 28, 112, 76, 109, 127, 106, 27, 92, 126, 63, 83, 88, 97, 54, 22, 41, 19, 69, 33, 18, 39, 29, 115, 101, 38, 116, 21, 74, 78, 26, 87, 45, 105, 34, 37, 103, 7, 32, 98, 36, 90, 96, 31, 93, 102, 67, 16, 30, 24, 68, 95, 100, 72, 79, 35, 20, 81, 9, 11, 77], [46, 112, 102, 33, 110, 0, 12, 48, 16, 8, 10, 78, 25, 70, 69, 65, 66, 19, 2, 4, 64, 5, 77, 89, 50, 85, 17, 67, 47, 1, 7, 68, 22, 83, 30, 20, 95, 93, 88, 82, 125, 62, 56, 124, 116, 54, 87, 51, 9, 43, 27, 127, 55, 49, 118, 23, 24, 73, 3, 11, 114, 15, 13, 84, 106, 40, 92, 79, 32, 121, 91, 103, 41, 36, 61, 99, 126, 98, 101, 113, 109, 81, 29, 31, 107, 123, 105, 45, 57, 44, 108, 94, 104, 39, 59, 52, 96, 28, 120, 100, 119, 26, 63, 60, 35, 21, 18, 71, 37, 115, 58, 6, 90, 111, 53, 42, 86, 122, 117, 34, 74, 76, 75, 14, 80, 72, 97, 38], [104, 121, 125, 53, 51, 20, 16, 34, 78, 81, 82, 119, 19, 75, 42, 43, 10, 122, 88, 50, 61, 63, 127, 93, 70, 21, 13, 109, 118, 106, 116, 44, 113, 30, 7, 120, 8, 48, 57, 124, 114, 111, 62, 47, 60, 56, 58, 107, 54, 123, 115, 31, 76, 126, 9, 49, 23, 55, 38, 105, 110, 97, 22, 112, 39, 46, 41, 100, 45, 59, 91, 52, 102, 108, 37, 117, 36, 4, 3, 99, 86, 26, 89, 35, 95, 101, 32, 92, 28, 27, 33, 103, 94, 96, 24, 25, 2, 69, 90, 29, 14, 18, 87, 73, 17, 85, 15, 65, 5, 68, 83, 64, 80, 1, 84, 98, 0, 12, 72, 79, 74, 71, 11, 77, 40, 66, 6, 67], [113, 38, 116, 56, 34, 121, 49, 77, 11, 9, 81, 15, 71, 25, 0, 93, 68, 70, 20, 46, 65, 86, 67, 18, 66, 91, 16, 30, 124, 112, 102, 115, 31, 83, 61, 22, 19, 87, 72, 5, 118, 88, 76, 27, 85, 29, 1, 45, 78, 8, 82, 89, 23, 7, 6, 111, 48, 117, 36, 44, 74, 41, 10, 58, 105, 59, 99, 47, 40, 108, 51, 106, 119, 42, 127, 101, 62, 43, 92, 50, 123, 104, 114, 103, 120, 57, 84, 37, 90, 95, 97, 125, 122, 24, 32, 55, 63, 69, 35, 109, 96, 53, 126, 60, 54, 3, 28, 33, 94, 39, 12, 107, 26, 100, 79, 13, 80, 52, 4, 73, 14, 110, 21, 75, 17, 2, 64, 98], [126, 36, 124, 60, 61, 54, 63, 58, 119, 56, 115, 51, 123, 55, 127, 32, 117, 125, 118, 49, 122, 110, 112, 114, 46, 30, 50, 53, 52, 120, 121, 116, 48, 59, 111, 62, 113, 47, 57, 34, 108, 44, 45, 109, 106, 95, 42, 93, 40, 41, 107, 104, 105, 43, 98, 20, 24, 23, 39, 103, 96, 33, 102, 97, 90, 94, 38, 82, 81, 92, 101, 26, 99, 27, 91, 89, 37, 35, 100, 22, 29, 14, 80, 87, 15, 31, 85, 16, 25, 86, 28, 79, 12, 9, 19, 13, 11, 83, 88, 7, 4, 74, 66, 72, 5, 70, 75, 10, 69, 0, 3, 21, 18, 1, 65, 6, 71, 76, 8, 77, 17, 67, 78, 64, 84, 73, 68, 2], [44, 104, 54, 57, 97, 117, 82, 16, 22, 14, 77, 73, 25, 11, 76, 9, 127, 53, 108, 89, 7, 29, 122, 40, 92, 123, 12, 27, 19, 4, 24, 52, 91, 112, 93, 95, 90, 26, 31, 30, 94, 32, 88, 115, 110, 120, 119, 124, 21, 87, 113, 96, 28, 118, 45, 62, 46, 69, 121, 126, 47, 10, 125, 116, 98, 39, 34, 58, 63, 67, 37, 111, 107, 49, 41, 43, 99, 35, 51, 36, 60, 42, 103, 105, 102, 106, 59, 70, 101, 5, 100, 23, 50, 1, 38, 109, 55, 61, 2, 20, 56, 85, 48, 17, 8, 15, 114, 75, 78, 0, 84, 72, 81, 80, 74, 71, 66, 79, 83, 18, 64, 6, 86, 13, 3, 33, 68, 65], [46, 39, 110, 97, 95, 64, 78, 76, 80, 10, 8, 21, 70, 68, 79, 66, 73, 3, 65, 19, 120, 58, 50, 57, 123, 9, 5, 48, 1, 2, 82, 7, 61, 23, 69, 113, 116, 118, 114, 67, 125, 51, 0, 124, 24, 121, 127, 77, 117, 112, 53, 49, 81, 47, 84, 52, 59, 56, 109, 72, 6, 17, 60, 87, 119, 4, 91, 75, 122, 107, 20, 22, 106, 45, 89, 96, 54, 40, 90, 29, 11, 126, 25, 63, 43, 42, 62, 92, 86, 30, 101, 98, 105, 44, 14, 115, 37, 104, 26, 27, 102, 108, 93, 36, 38, 55, 111, 12, 32, 41, 99, 100, 15, 18, 34, 83, 35, 85, 88, 28, 94, 74, 13, 71, 16, 33, 31, 103]], "model.layers.2.self_attn.qk_proj": [[46, 110, 49, 121, 112, 54, 40, 51, 127, 104, 57, 56, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 108, 48, 89, 39, 33, 25, 16, 14, 80, 115, 78, 63, 99, 107, 12, 76, 55, 18, 50, 85, 47, 98, 120, 21, 114, 58, 75, 97, 88, 86, 74, 8, 118, 82, 13, 11, 81, 95, 83, 19, 34, 20, 27, 84, 79, 15, 91, 77, 122, 17, 9, 87, 42, 62, 10, 94, 73, 29, 24, 31, 22, 38, 7, 52, 23, 36, 45, 41, 59, 92, 71, 6, 72, 111, 4, 69, 5, 30, 68, 105, 101, 35, 93, 32, 109, 70, 96, 28, 66, 2, 0, 64, 103, 90, 106, 1, 100, 67, 3, 65, 37, 26], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 57, 53, 125, 127, 113, 116, 43, 44, 126, 123, 61, 119, 124, 102, 60, 117, 48, 39, 108, 89, 25, 16, 107, 58, 120, 80, 33, 14, 115, 63, 12, 99, 47, 78, 76, 8, 118, 55, 98, 50, 85, 82, 27, 21, 114, 18, 20, 95, 75, 62, 83, 91, 15, 97, 19, 122, 77, 81, 74, 34, 13, 79, 17, 11, 94, 86, 84, 10, 9, 87, 42, 24, 31, 73, 70, 88, 23, 36, 111, 22, 4, 29, 38, 5, 7, 71, 68, 109, 52, 69, 0, 96, 28, 106, 64, 45, 103, 32, 1, 92, 105, 93, 66, 59, 30, 35, 67, 2, 72, 6, 101, 3, 65, 90, 26, 100, 41, 37], [46, 110, 49, 121, 112, 54, 56, 40, 104, 53, 51, 57, 127, 113, 125, 116, 43, 44, 61, 126, 124, 102, 119, 123, 117, 60, 108, 63, 33, 107, 55, 48, 89, 16, 25, 39, 120, 47, 58, 80, 76, 14, 8, 115, 12, 99, 118, 78, 97, 85, 50, 21, 70, 18, 98, 19, 82, 27, 114, 74, 11, 20, 77, 84, 15, 79, 94, 9, 17, 75, 42, 95, 91, 34, 13, 86, 52, 87, 10, 73, 81, 5, 38, 23, 24, 36, 7, 71, 68, 69, 122, 83, 4, 29, 62, 59, 88, 31, 111, 32, 109, 22, 66, 30, 106, 45, 64, 96, 105, 2, 1, 28, 92, 93, 100, 0, 103, 67, 41, 72, 3, 26, 65, 35, 101, 90, 6, 37], [46, 110, 49, 121, 112, 54, 127, 51, 57, 40, 56, 104, 113, 125, 53, 116, 43, 126, 44, 61, 124, 123, 102, 117, 60, 58, 119, 48, 33, 25, 108, 63, 16, 39, 89, 80, 55, 14, 8, 107, 99, 76, 118, 12, 78, 98, 21, 19, 120, 47, 18, 50, 91, 114, 115, 70, 97, 84, 11, 82, 27, 95, 52, 85, 77, 10, 74, 34, 23, 79, 20, 75, 42, 15, 86, 94, 83, 87, 62, 81, 31, 13, 17, 5, 9, 71, 73, 88, 36, 4, 69, 38, 24, 122, 68, 109, 106, 29, 30, 22, 7, 92, 41, 66, 111, 67, 59, 28, 93, 32, 2, 103, 65, 101, 0, 45, 1, 96, 105, 3, 64, 100, 26, 90, 35, 72, 6, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 125, 104, 113, 53, 116, 43, 61, 44, 126, 119, 102, 124, 123, 117, 58, 108, 25, 63, 33, 55, 16, 76, 48, 120, 8, 107, 12, 99, 118, 14, 60, 89, 80, 78, 39, 47, 115, 21, 82, 114, 98, 50, 18, 91, 95, 84, 97, 11, 20, 70, 9, 79, 94, 27, 74, 85, 17, 52, 75, 73, 15, 19, 69, 13, 77, 31, 24, 10, 88, 7, 87, 81, 68, 86, 42, 23, 62, 36, 5, 34, 38, 83, 71, 59, 4, 111, 22, 2, 67, 3, 122, 66, 28, 64, 65, 41, 0, 92, 29, 109, 30, 32, 35, 96, 103, 100, 101, 106, 93, 1, 72, 6, 105, 45, 90, 26, 37], [46, 110, 49, 121, 112, 51, 54, 56, 40, 104, 57, 127, 113, 125, 53, 116, 43, 44, 61, 126, 102, 124, 123, 119, 48, 60, 58, 25, 115, 108, 117, 33, 89, 39, 16, 107, 55, 80, 14, 12, 120, 8, 78, 76, 47, 99, 98, 18, 21, 50, 114, 82, 63, 85, 75, 52, 20, 19, 95, 27, 9, 31, 15, 11, 118, 111, 74, 84, 79, 73, 97, 34, 91, 122, 70, 88, 17, 42, 86, 13, 62, 94, 87, 24, 77, 23, 10, 83, 7, 4, 22, 36, 81, 68, 69, 38, 5, 71, 28, 32, 106, 6, 29, 93, 103, 72, 67, 96, 105, 30, 66, 41, 92, 35, 1, 59, 0, 65, 45, 64, 109, 101, 2, 3, 100, 90, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 127, 57, 51, 113, 125, 116, 53, 43, 44, 61, 126, 119, 124, 123, 102, 89, 60, 48, 117, 108, 115, 16, 107, 39, 14, 58, 33, 25, 12, 47, 55, 76, 80, 78, 99, 8, 120, 63, 85, 11, 18, 50, 9, 21, 83, 82, 118, 79, 52, 84, 75, 98, 15, 42, 17, 20, 95, 77, 91, 19, 111, 27, 13, 114, 97, 73, 74, 81, 24, 94, 88, 31, 86, 36, 122, 10, 34, 38, 87, 71, 23, 109, 22, 4, 6, 7, 29, 68, 62, 69, 5, 32, 106, 45, 72, 105, 70, 96, 30, 35, 93, 59, 103, 92, 101, 41, 28, 66, 3, 0, 1, 64, 2, 26, 67, 65, 100, 37, 90], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 119, 126, 102, 124, 123, 60, 89, 108, 117, 25, 33, 63, 55, 48, 39, 78, 107, 80, 14, 16, 12, 85, 99, 76, 115, 47, 118, 58, 18, 21, 82, 120, 50, 11, 15, 97, 74, 9, 83, 91, 31, 84, 98, 52, 20, 75, 79, 27, 19, 24, 94, 114, 17, 77, 81, 6, 10, 86, 29, 13, 73, 87, 8, 38, 111, 95, 34, 122, 42, 62, 23, 88, 72, 36, 7, 68, 69, 22, 59, 71, 30, 4, 5, 32, 103, 105, 106, 45, 28, 96, 41, 0, 3, 2, 35, 93, 64, 109, 67, 26, 66, 92, 65, 70, 101, 100, 1, 90, 37], [46, 110, 49, 121, 112, 40, 51, 56, 54, 104, 57, 127, 53, 125, 113, 116, 43, 44, 126, 61, 102, 119, 117, 124, 60, 123, 25, 39, 108, 89, 48, 16, 115, 58, 63, 55, 21, 14, 80, 78, 33, 107, 99, 12, 76, 120, 85, 62, 98, 82, 47, 18, 15, 19, 31, 97, 27, 50, 42, 72, 20, 84, 95, 6, 11, 91, 75, 114, 94, 81, 23, 9, 79, 83, 77, 74, 86, 17, 29, 10, 52, 34, 13, 118, 111, 24, 88, 122, 73, 87, 36, 38, 45, 22, 68, 41, 4, 7, 93, 105, 71, 30, 96, 106, 69, 8, 5, 28, 66, 109, 32, 59, 100, 64, 90, 0, 103, 3, 92, 35, 67, 65, 26, 2, 1, 101, 70, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 127, 125, 53, 51, 113, 116, 43, 44, 126, 61, 119, 102, 124, 60, 117, 123, 25, 108, 16, 89, 107, 39, 115, 48, 58, 80, 78, 14, 120, 33, 63, 12, 99, 76, 21, 98, 82, 27, 55, 97, 19, 47, 42, 84, 18, 72, 91, 11, 94, 6, 85, 15, 20, 83, 9, 75, 13, 17, 77, 79, 52, 24, 114, 95, 62, 10, 81, 50, 34, 118, 74, 87, 31, 88, 86, 23, 73, 111, 22, 36, 7, 71, 109, 69, 4, 68, 29, 105, 38, 5, 122, 106, 59, 96, 30, 41, 92, 8, 103, 32, 66, 28, 35, 45, 93, 2, 90, 26, 3, 70, 100, 101, 0, 67, 64, 1, 65, 37], [46, 110, 49, 121, 112, 40, 51, 54, 57, 56, 127, 104, 53, 113, 125, 116, 43, 126, 44, 61, 124, 102, 117, 63, 60, 119, 123, 108, 16, 89, 25, 33, 48, 39, 58, 80, 72, 78, 55, 12, 14, 98, 99, 107, 120, 115, 76, 50, 114, 95, 85, 82, 118, 42, 91, 21, 97, 18, 27, 47, 20, 15, 11, 19, 84, 79, 87, 94, 77, 74, 75, 10, 24, 17, 6, 73, 66, 86, 13, 83, 52, 62, 31, 69, 5, 9, 23, 111, 38, 88, 81, 4, 36, 68, 34, 29, 22, 106, 7, 71, 122, 2, 30, 64, 32, 0, 92, 70, 1, 59, 67, 3, 105, 65, 93, 96, 103, 109, 41, 28, 35, 45, 100, 90, 101, 8, 26, 37], [46, 110, 49, 121, 112, 54, 40, 127, 51, 57, 56, 104, 125, 53, 113, 116, 43, 44, 61, 126, 123, 119, 124, 102, 117, 63, 108, 33, 48, 89, 16, 72, 107, 115, 99, 58, 25, 60, 98, 39, 14, 55, 47, 114, 12, 78, 80, 76, 50, 21, 18, 82, 120, 27, 85, 118, 97, 91, 15, 84, 20, 95, 10, 111, 19, 11, 52, 87, 73, 75, 24, 42, 86, 36, 94, 31, 17, 34, 79, 83, 23, 74, 88, 81, 62, 77, 69, 9, 4, 70, 59, 5, 7, 13, 68, 22, 38, 71, 32, 30, 122, 29, 1, 3, 0, 6, 66, 105, 96, 92, 93, 106, 28, 65, 2, 109, 41, 45, 67, 35, 101, 64, 100, 103, 26, 90, 8, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 57, 51, 125, 127, 53, 113, 116, 43, 44, 119, 61, 126, 123, 102, 124, 117, 89, 25, 108, 107, 48, 58, 63, 60, 39, 33, 16, 115, 99, 78, 80, 14, 12, 47, 21, 72, 76, 120, 18, 82, 77, 20, 85, 114, 79, 97, 55, 94, 27, 15, 111, 75, 11, 52, 118, 91, 19, 42, 84, 81, 17, 31, 73, 34, 10, 50, 83, 86, 87, 98, 24, 88, 95, 9, 70, 74, 13, 23, 38, 36, 29, 62, 7, 68, 22, 30, 122, 96, 5, 59, 41, 93, 105, 109, 69, 4, 71, 92, 28, 32, 106, 35, 45, 101, 100, 67, 66, 6, 26, 3, 8, 103, 37, 64, 1, 2, 90, 0, 65], [46, 110, 49, 121, 112, 56, 54, 51, 127, 40, 57, 104, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 119, 60, 48, 123, 25, 108, 89, 117, 16, 39, 47, 33, 58, 21, 120, 99, 80, 107, 55, 12, 72, 115, 78, 14, 63, 76, 18, 97, 19, 82, 27, 85, 50, 70, 15, 42, 114, 11, 20, 84, 98, 75, 95, 118, 79, 77, 34, 13, 91, 86, 122, 17, 94, 10, 74, 73, 81, 83, 31, 59, 9, 62, 24, 88, 87, 52, 22, 4, 38, 5, 41, 23, 111, 29, 69, 71, 36, 68, 7, 105, 45, 92, 93, 64, 30, 35, 8, 28, 96, 101, 1, 100, 103, 32, 66, 2, 109, 3, 106, 67, 65, 26, 0, 6, 37, 90], [46, 110, 49, 112, 121, 54, 40, 104, 56, 57, 51, 127, 113, 125, 53, 116, 43, 44, 61, 126, 124, 123, 102, 119, 117, 89, 33, 60, 48, 25, 39, 63, 16, 108, 120, 78, 80, 47, 118, 99, 55, 107, 12, 14, 21, 76, 70, 58, 72, 115, 82, 18, 114, 98, 95, 85, 50, 42, 97, 91, 11, 27, 94, 20, 73, 79, 52, 74, 75, 31, 17, 15, 84, 88, 19, 86, 13, 83, 34, 81, 24, 62, 77, 23, 10, 22, 9, 71, 5, 68, 38, 4, 69, 122, 36, 8, 87, 29, 7, 32, 109, 105, 111, 93, 30, 59, 41, 67, 2, 92, 96, 66, 65, 45, 106, 1, 103, 0, 28, 101, 3, 100, 90, 64, 35, 26, 37, 6], [46, 110, 49, 112, 121, 54, 51, 40, 104, 127, 125, 56, 57, 113, 53, 116, 43, 44, 126, 61, 124, 117, 102, 123, 119, 63, 108, 107, 33, 60, 25, 48, 55, 115, 16, 39, 89, 120, 80, 58, 12, 78, 114, 47, 99, 118, 21, 52, 14, 50, 76, 18, 98, 97, 27, 42, 82, 95, 84, 85, 24, 72, 74, 11, 91, 20, 19, 88, 94, 70, 75, 36, 87, 73, 15, 8, 79, 23, 17, 86, 38, 31, 10, 111, 34, 13, 22, 4, 9, 81, 83, 77, 5, 29, 68, 122, 71, 69, 62, 41, 105, 7, 109, 28, 92, 65, 30, 59, 2, 32, 106, 96, 0, 3, 103, 67, 45, 93, 66, 64, 6, 35, 1, 26, 101, 90, 100, 37], [46, 110, 49, 121, 112, 54, 40, 104, 56, 51, 57, 125, 127, 113, 53, 116, 43, 44, 61, 126, 119, 102, 124, 107, 123, 117, 60, 25, 108, 89, 48, 58, 33, 63, 16, 115, 80, 76, 39, 78, 12, 14, 118, 99, 47, 55, 21, 120, 18, 98, 82, 15, 75, 84, 11, 9, 85, 19, 79, 91, 42, 52, 114, 20, 27, 74, 94, 77, 8, 31, 97, 73, 81, 86, 13, 17, 24, 50, 88, 10, 87, 83, 111, 70, 95, 36, 38, 72, 62, 23, 4, 34, 29, 71, 22, 68, 7, 122, 5, 69, 41, 109, 45, 28, 106, 6, 35, 59, 32, 92, 105, 93, 30, 96, 101, 103, 67, 100, 64, 2, 66, 3, 90, 26, 1, 0, 65, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 57, 51, 53, 127, 113, 125, 116, 43, 44, 126, 61, 124, 102, 60, 119, 123, 117, 25, 108, 39, 16, 33, 89, 107, 48, 58, 118, 55, 21, 80, 78, 76, 14, 12, 115, 8, 99, 63, 18, 97, 82, 120, 91, 85, 98, 79, 11, 47, 19, 81, 20, 84, 50, 86, 77, 114, 15, 42, 95, 10, 122, 52, 75, 27, 88, 17, 73, 13, 34, 94, 62, 6, 74, 9, 83, 22, 31, 87, 24, 111, 5, 23, 68, 38, 45, 36, 71, 29, 41, 93, 7, 59, 4, 105, 69, 32, 30, 28, 106, 103, 72, 70, 66, 92, 96, 35, 0, 2, 64, 100, 109, 65, 101, 1, 67, 90, 26, 3, 37], [46, 110, 49, 121, 112, 127, 54, 40, 56, 104, 51, 57, 125, 113, 116, 53, 43, 126, 44, 61, 102, 124, 119, 117, 123, 60, 48, 108, 33, 25, 80, 39, 63, 58, 89, 16, 14, 8, 21, 12, 107, 78, 76, 120, 55, 115, 118, 99, 18, 19, 91, 97, 82, 47, 114, 50, 85, 20, 6, 62, 98, 42, 27, 11, 84, 75, 86, 79, 10, 81, 95, 77, 15, 13, 88, 9, 34, 23, 52, 24, 74, 73, 94, 17, 87, 122, 83, 5, 31, 22, 36, 4, 109, 29, 105, 71, 68, 111, 38, 45, 69, 7, 32, 0, 59, 92, 30, 106, 28, 2, 41, 67, 93, 66, 65, 64, 96, 3, 103, 101, 35, 1, 90, 100, 72, 70, 26, 37], [46, 110, 49, 121, 112, 54, 40, 104, 51, 56, 127, 57, 113, 53, 116, 125, 43, 44, 126, 61, 124, 123, 119, 102, 25, 108, 63, 117, 60, 89, 48, 8, 80, 16, 39, 14, 115, 107, 78, 33, 55, 120, 12, 76, 58, 47, 6, 99, 98, 82, 18, 50, 21, 95, 20, 75, 97, 114, 85, 11, 9, 91, 13, 27, 84, 10, 15, 73, 19, 79, 118, 17, 74, 42, 86, 24, 62, 31, 94, 52, 5, 111, 81, 77, 68, 59, 87, 23, 36, 7, 83, 4, 88, 34, 69, 45, 71, 22, 106, 122, 38, 29, 96, 105, 92, 35, 30, 66, 3, 32, 100, 67, 109, 93, 28, 103, 41, 64, 26, 1, 101, 65, 2, 0, 90, 72, 70, 37], [46, 110, 49, 121, 112, 54, 40, 51, 56, 104, 57, 127, 113, 53, 116, 125, 43, 44, 124, 61, 126, 117, 102, 119, 48, 123, 33, 25, 58, 107, 108, 47, 16, 39, 60, 63, 80, 89, 55, 8, 12, 99, 14, 98, 78, 76, 115, 114, 97, 82, 120, 18, 21, 6, 50, 91, 95, 75, 79, 27, 85, 10, 20, 42, 94, 84, 74, 81, 11, 17, 83, 15, 19, 9, 31, 87, 52, 36, 73, 38, 34, 86, 88, 77, 4, 13, 71, 24, 22, 118, 111, 62, 23, 29, 5, 122, 69, 7, 105, 68, 106, 59, 30, 32, 28, 93, 66, 45, 96, 103, 92, 109, 64, 0, 100, 41, 2, 101, 35, 72, 3, 67, 90, 70, 1, 26, 65, 37], [46, 110, 49, 121, 112, 54, 56, 40, 57, 104, 127, 51, 113, 53, 116, 125, 43, 61, 44, 126, 123, 102, 119, 117, 124, 60, 39, 48, 25, 107, 108, 63, 33, 89, 80, 115, 114, 78, 16, 58, 55, 14, 99, 47, 12, 98, 8, 76, 118, 21, 50, 120, 18, 82, 97, 84, 27, 95, 85, 83, 75, 11, 79, 52, 9, 20, 34, 19, 74, 15, 81, 91, 13, 87, 6, 42, 73, 10, 36, 77, 94, 17, 69, 86, 31, 38, 29, 71, 4, 88, 111, 5, 24, 23, 59, 22, 68, 7, 122, 28, 64, 109, 30, 45, 41, 62, 66, 96, 106, 70, 35, 93, 100, 103, 72, 92, 67, 1, 105, 32, 2, 3, 0, 101, 65, 26, 90, 37], [46, 110, 49, 121, 112, 51, 54, 40, 104, 57, 127, 113, 56, 125, 53, 116, 43, 44, 126, 61, 119, 117, 124, 102, 123, 25, 33, 58, 55, 89, 63, 107, 16, 108, 47, 48, 39, 60, 115, 14, 80, 78, 12, 76, 99, 21, 85, 82, 8, 18, 97, 114, 98, 120, 75, 27, 20, 84, 17, 91, 50, 118, 13, 19, 79, 10, 94, 95, 15, 74, 31, 87, 11, 77, 81, 9, 24, 34, 88, 23, 83, 73, 86, 68, 22, 29, 62, 52, 72, 4, 38, 36, 70, 42, 71, 111, 5, 109, 69, 45, 59, 30, 7, 28, 122, 41, 106, 32, 96, 6, 93, 92, 105, 35, 67, 100, 2, 1, 103, 3, 64, 0, 101, 90, 66, 26, 65, 37], [46, 110, 49, 121, 112, 51, 40, 57, 56, 54, 104, 127, 113, 125, 53, 116, 43, 44, 126, 61, 124, 102, 119, 123, 60, 117, 25, 108, 58, 107, 89, 48, 33, 80, 39, 16, 63, 78, 115, 47, 12, 76, 99, 55, 14, 98, 18, 120, 27, 19, 85, 97, 94, 114, 79, 21, 20, 83, 95, 82, 42, 91, 15, 75, 8, 52, 11, 84, 17, 74, 70, 50, 118, 24, 77, 73, 13, 86, 10, 72, 87, 22, 9, 36, 81, 34, 88, 31, 23, 38, 111, 4, 62, 122, 69, 68, 29, 7, 30, 71, 5, 103, 96, 45, 28, 41, 92, 106, 93, 67, 2, 109, 32, 105, 66, 101, 35, 59, 100, 0, 64, 1, 6, 26, 65, 90, 3, 37], [46, 110, 49, 121, 112, 40, 54, 104, 56, 57, 127, 51, 125, 113, 53, 116, 43, 44, 61, 126, 119, 123, 124, 117, 102, 33, 108, 39, 48, 63, 60, 89, 107, 25, 58, 16, 78, 115, 55, 99, 80, 14, 47, 76, 12, 18, 70, 120, 97, 118, 114, 85, 98, 21, 27, 20, 75, 84, 72, 11, 52, 73, 9, 82, 17, 15, 91, 42, 79, 50, 94, 87, 13, 62, 81, 74, 10, 19, 95, 24, 34, 83, 88, 77, 86, 5, 111, 31, 36, 7, 8, 122, 71, 4, 68, 22, 69, 23, 38, 109, 30, 29, 106, 32, 105, 45, 35, 2, 103, 92, 67, 96, 28, 101, 3, 93, 64, 66, 59, 41, 100, 65, 26, 1, 0, 90, 6, 37], [46, 110, 49, 121, 112, 54, 40, 51, 127, 56, 104, 57, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 123, 119, 117, 89, 39, 107, 60, 48, 25, 55, 33, 108, 58, 47, 16, 78, 99, 80, 115, 114, 76, 63, 12, 14, 118, 85, 120, 18, 72, 27, 50, 70, 97, 98, 79, 91, 21, 19, 20, 82, 95, 11, 84, 83, 75, 15, 87, 74, 13, 52, 34, 9, 17, 122, 77, 81, 94, 10, 31, 42, 73, 22, 38, 24, 62, 86, 111, 36, 7, 5, 69, 88, 29, 23, 68, 30, 64, 4, 71, 32, 103, 106, 35, 41, 45, 28, 105, 109, 59, 92, 8, 67, 93, 1, 96, 66, 2, 26, 100, 0, 101, 3, 65, 90, 37, 6], [46, 110, 49, 121, 112, 40, 57, 104, 54, 56, 51, 127, 125, 53, 113, 116, 43, 44, 126, 61, 124, 117, 102, 119, 123, 60, 39, 25, 89, 108, 48, 63, 107, 33, 80, 16, 99, 115, 58, 78, 12, 55, 14, 76, 21, 47, 72, 50, 97, 82, 91, 20, 85, 98, 27, 18, 94, 83, 114, 79, 120, 118, 38, 24, 73, 19, 11, 95, 75, 77, 34, 86, 13, 74, 52, 15, 42, 88, 17, 81, 84, 62, 70, 87, 31, 10, 36, 45, 9, 111, 23, 5, 122, 4, 22, 71, 29, 68, 7, 69, 92, 30, 106, 93, 41, 32, 96, 103, 28, 59, 2, 109, 6, 101, 35, 105, 8, 90, 3, 1, 26, 67, 66, 0, 100, 65, 64, 37], [46, 110, 49, 121, 112, 40, 54, 56, 104, 51, 57, 113, 53, 125, 127, 116, 43, 44, 61, 126, 119, 102, 60, 124, 39, 123, 117, 108, 89, 72, 58, 25, 16, 33, 48, 80, 107, 78, 63, 76, 120, 12, 99, 14, 98, 55, 115, 47, 50, 118, 11, 18, 79, 85, 82, 73, 42, 114, 95, 21, 97, 91, 9, 75, 84, 10, 27, 20, 77, 17, 19, 13, 15, 83, 34, 38, 74, 52, 122, 94, 69, 24, 4, 31, 81, 23, 68, 22, 70, 88, 7, 71, 62, 36, 29, 86, 87, 6, 5, 111, 45, 109, 32, 3, 28, 59, 2, 30, 0, 1, 90, 103, 105, 67, 65, 96, 66, 35, 93, 64, 92, 101, 106, 100, 41, 26, 8, 37], [46, 110, 49, 121, 112, 54, 40, 57, 56, 104, 127, 51, 113, 125, 53, 116, 43, 126, 44, 61, 123, 124, 102, 119, 60, 117, 108, 58, 115, 33, 89, 48, 25, 72, 16, 107, 39, 63, 80, 76, 78, 99, 14, 120, 47, 55, 118, 12, 98, 50, 114, 82, 27, 21, 18, 52, 85, 97, 20, 9, 83, 11, 91, 84, 74, 75, 79, 15, 10, 95, 86, 87, 19, 122, 42, 13, 34, 81, 94, 24, 17, 73, 6, 77, 36, 5, 23, 88, 38, 69, 7, 111, 22, 45, 31, 68, 4, 71, 109, 30, 62, 29, 59, 28, 106, 92, 105, 3, 103, 32, 2, 70, 96, 65, 1, 64, 100, 0, 66, 90, 41, 93, 67, 8, 35, 101, 37, 26], [46, 110, 49, 121, 112, 54, 40, 51, 56, 57, 104, 127, 113, 125, 116, 53, 43, 44, 61, 126, 124, 102, 123, 119, 117, 108, 39, 107, 60, 25, 58, 48, 16, 89, 72, 33, 63, 80, 12, 76, 55, 99, 78, 14, 115, 98, 47, 50, 18, 82, 21, 6, 85, 95, 9, 114, 120, 11, 27, 97, 20, 79, 91, 118, 75, 84, 83, 73, 10, 42, 15, 34, 36, 86, 13, 77, 19, 24, 74, 88, 31, 87, 62, 94, 52, 17, 7, 4, 81, 68, 5, 111, 38, 71, 69, 103, 45, 23, 29, 122, 22, 93, 106, 32, 105, 109, 59, 30, 41, 35, 8, 28, 3, 92, 1, 64, 2, 67, 66, 70, 0, 65, 96, 100, 90, 101, 26, 37], [46, 110, 49, 121, 112, 54, 40, 56, 104, 51, 127, 57, 125, 113, 116, 53, 43, 44, 61, 126, 119, 102, 124, 123, 117, 60, 108, 25, 89, 39, 107, 48, 80, 115, 16, 12, 58, 99, 76, 78, 14, 33, 47, 18, 55, 72, 19, 21, 27, 6, 63, 75, 120, 94, 20, 82, 9, 50, 98, 83, 91, 73, 11, 15, 34, 42, 84, 13, 88, 85, 79, 81, 52, 97, 77, 74, 10, 95, 86, 114, 118, 17, 68, 87, 24, 31, 111, 36, 71, 106, 22, 62, 23, 5, 7, 38, 29, 109, 93, 4, 30, 8, 103, 122, 35, 69, 45, 92, 41, 59, 105, 32, 66, 67, 100, 28, 64, 0, 65, 3, 96, 2, 101, 90, 26, 70, 1, 37], [46, 110, 49, 112, 121, 54, 51, 40, 57, 56, 104, 127, 125, 113, 53, 116, 43, 44, 126, 61, 124, 102, 60, 123, 119, 117, 89, 58, 108, 48, 55, 39, 63, 33, 25, 80, 107, 99, 50, 16, 14, 47, 78, 76, 12, 18, 114, 115, 120, 42, 21, 85, 118, 83, 19, 98, 95, 27, 34, 97, 52, 82, 79, 91, 20, 84, 75, 6, 77, 94, 15, 11, 73, 17, 10, 13, 31, 81, 38, 72, 9, 88, 86, 62, 87, 74, 24, 8, 22, 59, 106, 36, 23, 122, 29, 5, 68, 7, 71, 4, 111, 93, 41, 30, 92, 103, 69, 109, 45, 32, 96, 100, 66, 35, 65, 90, 1, 64, 0, 28, 105, 101, 3, 67, 26, 37, 70, 2]], "model.layers.3.self_attn.q_proj": [[40, 60, 98, 56, 52, 57, 119, 62, 32, 58, 49, 37, 113, 118, 53, 59, 95, 123, 48, 105, 89, 50, 24, 45, 122, 63, 120, 25, 101, 26, 30, 93, 39, 86, 51, 61, 85, 42, 34, 17, 38, 109, 111, 102, 35, 107, 36, 121, 114, 43, 116, 21, 126, 103, 54, 83, 84, 125, 29, 112, 108, 55, 46, 97, 96, 115, 92, 127, 110, 124, 31, 99, 47, 44, 33, 100, 41, 91, 88, 94, 15, 106, 20, 18, 117, 90, 27, 28, 82, 87, 23, 22, 81, 104, 80, 11, 78, 75, 76, 79, 74, 77, 10, 19, 8, 72, 14, 12, 68, 13, 65, 71, 5, 73, 16, 4, 2, 67, 69, 6, 9, 7, 70, 3, 1, 66, 0, 64], [119, 62, 40, 98, 121, 52, 123, 58, 53, 48, 50, 118, 38, 25, 49, 30, 29, 54, 39, 63, 60, 57, 84, 56, 122, 59, 42, 115, 86, 102, 112, 33, 27, 37, 93, 113, 116, 28, 32, 111, 107, 89, 83, 15, 46, 108, 114, 120, 103, 110, 125, 101, 80, 18, 88, 36, 45, 41, 34, 47, 21, 43, 127, 24, 55, 51, 117, 44, 23, 109, 14, 124, 61, 100, 105, 97, 92, 106, 126, 73, 35, 26, 104, 12, 72, 19, 99, 31, 94, 17, 22, 95, 85, 90, 91, 96, 74, 20, 67, 70, 76, 79, 87, 81, 11, 10, 75, 71, 2, 13, 9, 5, 4, 65, 82, 68, 78, 7, 8, 0, 77, 3, 1, 66, 6, 64, 16, 69], [40, 98, 56, 60, 62, 119, 86, 29, 52, 58, 83, 14, 80, 57, 12, 54, 59, 10, 48, 93, 17, 15, 50, 63, 24, 25, 125, 4, 123, 118, 121, 101, 9, 120, 74, 113, 72, 26, 5, 85, 114, 51, 106, 33, 99, 53, 70, 45, 122, 95, 73, 71, 78, 76, 11, 16, 30, 81, 19, 22, 89, 104, 32, 103, 108, 21, 20, 88, 66, 67, 90, 84, 7, 2, 41, 49, 18, 111, 13, 1, 39, 23, 38, 87, 110, 75, 43, 36, 27, 6, 0, 82, 77, 79, 69, 37, 47, 8, 3, 97, 100, 102, 94, 92, 126, 96, 127, 34, 91, 42, 61, 28, 109, 115, 117, 35, 31, 116, 64, 105, 112, 68, 46, 55, 107, 65, 44, 124], [58, 40, 98, 119, 62, 60, 52, 29, 57, 86, 56, 48, 12, 80, 83, 59, 14, 24, 50, 51, 10, 9, 54, 72, 95, 73, 4, 19, 11, 71, 114, 116, 93, 66, 100, 76, 69, 63, 123, 32, 2, 49, 70, 106, 0, 125, 113, 16, 118, 68, 91, 5, 26, 7, 67, 75, 82, 53, 39, 90, 127, 79, 28, 120, 99, 25, 23, 103, 109, 27, 94, 22, 55, 111, 81, 13, 77, 108, 89, 18, 61, 117, 47, 46, 85, 87, 8, 121, 78, 88, 38, 36, 21, 31, 107, 3, 17, 37, 20, 30, 42, 15, 35, 122, 124, 96, 112, 84, 92, 45, 102, 43, 105, 101, 97, 33, 44, 64, 104, 126, 74, 6, 41, 115, 1, 110, 34, 65], [38, 97, 111, 120, 123, 122, 47, 84, 81, 10, 14, 71, 12, 8, 2, 67, 69, 58, 1, 0, 87, 88, 76, 19, 65, 83, 93, 106, 17, 86, 57, 34, 53, 11, 42, 32, 25, 118, 26, 20, 85, 99, 96, 56, 80, 49, 66, 31, 15, 78, 3, 105, 95, 89, 112, 109, 94, 52, 124, 101, 98, 110, 30, 126, 90, 92, 37, 22, 64, 55, 115, 82, 91, 103, 79, 44, 48, 74, 73, 119, 113, 60, 27, 45, 36, 4, 39, 29, 54, 62, 41, 61, 100, 21, 121, 108, 35, 63, 28, 51, 72, 46, 7, 18, 114, 127, 107, 125, 75, 77, 117, 16, 59, 23, 50, 43, 116, 104, 40, 13, 70, 5, 9, 24, 6, 68, 33, 102], [38, 97, 111, 122, 123, 120, 47, 81, 23, 84, 14, 71, 12, 10, 8, 69, 67, 75, 19, 73, 25, 7, 3, 1, 56, 76, 21, 105, 13, 78, 58, 5, 65, 2, 49, 79, 64, 4, 94, 70, 66, 30, 113, 0, 31, 98, 6, 112, 72, 16, 126, 86, 61, 114, 87, 52, 15, 88, 43, 35, 74, 42, 124, 62, 119, 63, 68, 37, 11, 104, 121, 45, 118, 17, 99, 20, 110, 57, 93, 82, 24, 26, 90, 77, 80, 32, 28, 22, 103, 85, 117, 46, 36, 9, 51, 106, 125, 18, 108, 92, 59, 83, 55, 115, 89, 60, 53, 41, 29, 44, 107, 33, 100, 95, 39, 127, 48, 54, 27, 101, 116, 50, 40, 91, 109, 96, 102, 34], [38, 111, 97, 123, 120, 122, 47, 81, 14, 84, 23, 12, 10, 8, 69, 71, 59, 75, 58, 3, 1, 86, 67, 16, 11, 40, 56, 127, 88, 25, 24, 105, 50, 31, 32, 87, 51, 9, 90, 45, 116, 76, 43, 73, 110, 74, 5, 125, 22, 21, 106, 4, 2, 62, 82, 19, 91, 78, 13, 98, 72, 113, 26, 93, 27, 28, 48, 100, 20, 83, 99, 66, 94, 15, 53, 117, 39, 17, 95, 42, 80, 49, 104, 108, 63, 65, 0, 118, 96, 126, 55, 60, 33, 92, 61, 77, 37, 85, 114, 30, 107, 35, 52, 119, 46, 44, 79, 103, 6, 41, 34, 109, 7, 29, 57, 18, 36, 115, 112, 64, 54, 101, 121, 89, 70, 102, 124, 68], [38, 97, 111, 123, 122, 120, 47, 84, 81, 14, 23, 12, 10, 71, 24, 8, 69, 67, 3, 1, 55, 105, 93, 76, 44, 59, 78, 86, 7, 56, 58, 70, 73, 16, 61, 107, 25, 30, 2, 21, 11, 57, 60, 126, 6, 51, 68, 4, 89, 80, 88, 79, 17, 104, 72, 32, 35, 20, 19, 74, 66, 95, 46, 41, 42, 26, 96, 43, 94, 75, 0, 98, 116, 106, 40, 15, 22, 48, 5, 50, 54, 9, 29, 63, 87, 124, 109, 114, 118, 110, 82, 37, 53, 112, 113, 27, 125, 45, 92, 121, 91, 85, 77, 34, 117, 62, 39, 49, 101, 65, 103, 119, 31, 90, 83, 28, 100, 115, 99, 36, 64, 52, 127, 18, 13, 108, 33, 102], [41, 34, 18, 20, 52, 76, 89, 79, 22, 16, 14, 120, 114, 62, 8, 10, 105, 46, 47, 63, 81, 126, 116, 69, 13, 125, 117, 7, 9, 127, 3, 59, 123, 72, 71, 5, 80, 23, 28, 73, 53, 70, 112, 12, 74, 50, 119, 24, 78, 19, 113, 6, 49, 66, 58, 15, 2, 110, 25, 75, 82, 54, 17, 109, 107, 95, 85, 86, 67, 11, 83, 94, 111, 118, 104, 60, 29, 37, 55, 61, 32, 93, 122, 27, 88, 68, 1, 39, 56, 115, 84, 64, 38, 42, 31, 43, 33, 101, 102, 87, 21, 26, 44, 103, 100, 0, 97, 35, 96, 121, 57, 48, 91, 92, 77, 45, 108, 106, 124, 4, 99, 51, 30, 90, 40, 65, 36, 98], [41, 34, 89, 52, 20, 22, 120, 18, 47, 46, 63, 114, 126, 80, 116, 76, 62, 105, 72, 14, 28, 78, 125, 117, 11, 79, 123, 82, 127, 13, 85, 7, 59, 74, 77, 10, 12, 19, 84, 58, 21, 15, 49, 9, 107, 119, 53, 54, 90, 24, 86, 81, 113, 5, 109, 73, 25, 95, 112, 29, 8, 23, 92, 103, 27, 104, 30, 37, 50, 61, 102, 39, 16, 88, 69, 96, 35, 32, 94, 3, 31, 71, 26, 110, 111, 101, 55, 33, 60, 124, 51, 93, 36, 83, 17, 38, 66, 57, 118, 43, 87, 42, 115, 98, 91, 75, 40, 4, 56, 99, 1, 108, 44, 122, 97, 100, 121, 6, 67, 68, 65, 45, 48, 106, 70, 2, 64, 0], [41, 34, 52, 81, 22, 18, 114, 20, 126, 76, 79, 120, 14, 10, 62, 69, 47, 116, 8, 3, 105, 63, 5, 46, 117, 125, 127, 72, 7, 9, 89, 54, 66, 123, 24, 1, 70, 6, 59, 74, 53, 71, 16, 113, 49, 28, 2, 64, 58, 77, 86, 110, 32, 73, 0, 119, 27, 68, 85, 50, 107, 112, 78, 12, 67, 17, 87, 83, 15, 13, 80, 84, 26, 104, 109, 61, 25, 92, 90, 60, 96, 118, 56, 38, 65, 57, 75, 95, 19, 45, 43, 82, 11, 111, 91, 99, 88, 35, 93, 33, 101, 31, 97, 122, 21, 29, 30, 37, 102, 39, 40, 100, 44, 42, 23, 106, 94, 55, 36, 124, 121, 115, 48, 103, 51, 4, 108, 98], [41, 34, 20, 18, 22, 52, 10, 76, 14, 6, 67, 62, 46, 120, 79, 126, 125, 114, 3, 116, 47, 105, 7, 8, 63, 9, 69, 127, 54, 2, 28, 117, 59, 58, 25, 0, 13, 123, 70, 110, 72, 80, 77, 112, 12, 49, 24, 90, 65, 75, 44, 113, 11, 119, 53, 71, 74, 23, 73, 91, 104, 35, 32, 78, 85, 15, 64, 87, 86, 89, 84, 88, 45, 66, 61, 21, 26, 5, 17, 68, 95, 60, 1, 42, 121, 83, 51, 122, 92, 100, 107, 56, 103, 109, 36, 106, 111, 94, 81, 48, 82, 38, 50, 55, 43, 16, 37, 19, 31, 102, 27, 29, 108, 124, 57, 97, 39, 93, 101, 99, 40, 115, 33, 96, 118, 30, 4, 98], [103, 117, 125, 109, 46, 56, 122, 97, 58, 52, 63, 57, 47, 28, 126, 59, 115, 89, 39, 21, 124, 119, 51, 49, 116, 62, 110, 60, 113, 55, 114, 111, 54, 86, 50, 87, 112, 123, 127, 61, 121, 53, 81, 33, 82, 48, 118, 108, 45, 120, 24, 107, 20, 44, 41, 31, 42, 19, 105, 106, 79, 43, 13, 96, 37, 30, 40, 100, 102, 104, 101, 99, 38, 36, 94, 34, 35, 95, 98, 32, 73, 80, 92, 93, 90, 17, 22, 91, 29, 76, 12, 26, 78, 88, 25, 71, 70, 27, 16, 83, 7, 9, 10, 85, 77, 4, 23, 68, 18, 84, 74, 5, 6, 8, 15, 3, 14, 67, 66, 0, 11, 65, 75, 69, 1, 64, 72, 2], [103, 117, 52, 109, 125, 56, 97, 122, 46, 119, 57, 28, 63, 58, 89, 21, 82, 115, 9, 47, 12, 78, 87, 126, 80, 20, 124, 33, 49, 116, 13, 62, 19, 110, 31, 83, 114, 59, 86, 54, 17, 111, 92, 61, 123, 127, 50, 60, 51, 94, 30, 55, 25, 6, 112, 74, 107, 81, 79, 71, 118, 77, 90, 11, 108, 121, 7, 24, 113, 53, 39, 48, 16, 101, 15, 18, 44, 32, 88, 95, 73, 4, 67, 22, 36, 45, 29, 84, 76, 69, 43, 41, 8, 68, 105, 37, 23, 26, 91, 85, 70, 5, 104, 93, 99, 14, 34, 75, 10, 100, 96, 120, 42, 106, 35, 3, 65, 38, 102, 66, 0, 27, 40, 98, 1, 2, 72, 64], [103, 117, 52, 125, 109, 97, 119, 122, 21, 89, 56, 28, 46, 59, 60, 20, 78, 82, 79, 87, 58, 47, 80, 55, 9, 124, 113, 50, 126, 121, 57, 12, 114, 123, 111, 62, 7, 74, 11, 63, 25, 118, 51, 53, 61, 92, 15, 54, 4, 22, 90, 116, 31, 112, 19, 110, 30, 115, 127, 17, 107, 33, 83, 70, 49, 95, 8, 13, 91, 98, 93, 6, 69, 24, 81, 44, 94, 77, 32, 71, 48, 67, 88, 73, 45, 38, 23, 99, 43, 42, 86, 29, 26, 66, 104, 84, 37, 5, 76, 101, 96, 102, 120, 39, 105, 41, 106, 34, 108, 100, 40, 1, 27, 16, 36, 10, 85, 68, 35, 18, 14, 0, 75, 2, 65, 64, 72, 3], [103, 117, 52, 46, 97, 125, 109, 119, 56, 122, 21, 63, 82, 78, 80, 87, 58, 9, 83, 89, 20, 12, 28, 92, 57, 8, 11, 6, 126, 15, 59, 75, 25, 79, 67, 124, 54, 74, 16, 13, 81, 114, 49, 62, 76, 115, 71, 123, 31, 53, 110, 68, 69, 60, 85, 90, 18, 19, 77, 95, 30, 14, 23, 127, 66, 111, 50, 72, 48, 33, 24, 84, 113, 10, 64, 70, 73, 51, 5, 3, 47, 112, 61, 4, 26, 22, 86, 55, 17, 93, 1, 2, 88, 27, 29, 0, 94, 118, 107, 91, 32, 96, 44, 99, 34, 65, 121, 108, 116, 101, 7, 41, 98, 100, 36, 35, 102, 106, 104, 120, 37, 43, 42, 38, 40, 105, 45, 39], [104, 33, 108, 20, 18, 23, 16, 13, 10, 44, 72, 68, 70, 48, 56, 107, 63, 55, 40, 117, 54, 84, 116, 57, 90, 21, 50, 12, 66, 14, 110, 31, 115, 51, 11, 9, 58, 73, 78, 61, 109, 114, 87, 111, 96, 121, 123, 25, 52, 120, 79, 1, 5, 82, 26, 100, 86, 43, 74, 125, 127, 59, 94, 119, 83, 118, 3, 76, 15, 126, 2, 38, 92, 45, 77, 34, 6, 8, 22, 102, 41, 62, 24, 88, 95, 71, 112, 36, 7, 60, 105, 106, 93, 80, 89, 42, 99, 85, 113, 47, 67, 101, 27, 39, 17, 98, 81, 46, 29, 103, 69, 53, 28, 30, 19, 35, 4, 91, 49, 124, 75, 122, 37, 32, 64, 65, 97, 0], [104, 108, 33, 65, 10, 13, 68, 18, 70, 1, 16, 64, 7, 44, 23, 67, 66, 87, 40, 0, 72, 20, 4, 56, 110, 69, 111, 116, 120, 3, 55, 5, 50, 12, 115, 6, 57, 61, 11, 123, 71, 54, 58, 84, 113, 77, 112, 48, 8, 2, 19, 62, 125, 114, 117, 51, 100, 43, 124, 52, 118, 78, 38, 14, 26, 9, 99, 80, 97, 63, 31, 96, 45, 24, 83, 49, 79, 21, 59, 107, 127, 29, 119, 74, 30, 60, 88, 109, 39, 122, 121, 89, 15, 85, 92, 36, 91, 41, 86, 32, 101, 25, 106, 94, 37, 93, 126, 34, 42, 90, 75, 82, 105, 53, 17, 73, 98, 103, 76, 47, 22, 27, 46, 28, 35, 81, 102, 95], [104, 108, 33, 64, 0, 44, 70, 13, 68, 10, 7, 66, 67, 16, 20, 18, 87, 1, 40, 23, 65, 72, 111, 56, 50, 110, 57, 55, 5, 120, 48, 116, 4, 84, 113, 115, 11, 123, 12, 61, 71, 51, 114, 100, 3, 117, 58, 21, 6, 118, 43, 125, 2, 80, 62, 45, 77, 52, 96, 75, 54, 69, 109, 86, 8, 126, 41, 38, 124, 121, 39, 47, 76, 101, 74, 27, 59, 89, 25, 99, 31, 35, 28, 79, 24, 81, 107, 83, 82, 34, 78, 92, 90, 60, 95, 37, 19, 94, 14, 9, 98, 127, 49, 17, 91, 29, 53, 30, 88, 106, 63, 85, 26, 112, 42, 36, 102, 32, 105, 46, 122, 103, 22, 93, 15, 119, 73, 97], [104, 108, 33, 23, 16, 18, 10, 44, 20, 13, 7, 72, 12, 116, 55, 87, 40, 110, 61, 67, 68, 107, 115, 117, 56, 70, 48, 114, 69, 50, 57, 125, 109, 1, 66, 118, 123, 120, 60, 54, 124, 111, 21, 81, 83, 25, 58, 126, 78, 64, 11, 59, 29, 63, 9, 88, 28, 90, 43, 32, 52, 46, 42, 62, 2, 8, 85, 47, 101, 73, 27, 89, 15, 113, 84, 76, 49, 103, 91, 99, 31, 24, 14, 3, 119, 86, 22, 98, 105, 122, 121, 36, 75, 79, 41, 106, 45, 30, 94, 127, 92, 80, 82, 93, 19, 96, 112, 77, 53, 17, 38, 6, 26, 95, 102, 39, 34, 100, 37, 4, 51, 71, 35, 5, 74, 97, 65, 0], [41, 46, 53, 127, 40, 97, 60, 105, 91, 31, 119, 39, 122, 47, 27, 112, 38, 24, 116, 99, 36, 115, 118, 42, 93, 57, 117, 33, 43, 125, 51, 126, 35, 110, 85, 55, 18, 114, 90, 87, 102, 88, 92, 96, 111, 100, 29, 124, 20, 109, 59, 49, 52, 95, 123, 63, 50, 56, 81, 44, 23, 30, 106, 48, 104, 79, 120, 108, 98, 25, 80, 107, 62, 121, 34, 37, 22, 94, 77, 82, 19, 113, 14, 26, 9, 58, 61, 28, 45, 54, 101, 75, 21, 83, 103, 13, 16, 86, 32, 84, 89, 74, 11, 76, 17, 5, 72, 15, 6, 66, 4, 73, 71, 2, 69, 10, 67, 70, 1, 68, 78, 64, 12, 0, 7, 8, 65, 3], [40, 97, 53, 127, 46, 24, 85, 31, 110, 14, 81, 75, 88, 60, 19, 90, 93, 41, 76, 47, 115, 71, 80, 74, 9, 4, 122, 82, 83, 119, 21, 91, 43, 38, 6, 104, 96, 55, 105, 42, 112, 27, 3, 72, 73, 33, 68, 101, 100, 37, 22, 26, 63, 124, 108, 13, 29, 54, 62, 126, 69, 30, 25, 125, 98, 114, 84, 15, 66, 117, 23, 45, 10, 77, 70, 118, 51, 16, 35, 106, 79, 87, 113, 17, 92, 34, 116, 1, 102, 89, 52, 44, 103, 12, 28, 48, 99, 56, 2, 20, 64, 111, 78, 36, 18, 123, 7, 61, 11, 94, 39, 65, 32, 50, 95, 86, 49, 8, 5, 59, 109, 57, 67, 121, 0, 107, 58, 120], [41, 110, 40, 53, 97, 127, 60, 43, 46, 38, 31, 47, 112, 91, 92, 35, 116, 52, 19, 122, 113, 33, 124, 119, 117, 26, 93, 59, 39, 27, 24, 125, 108, 63, 86, 56, 123, 95, 118, 57, 115, 49, 29, 126, 48, 45, 58, 88, 28, 76, 100, 96, 55, 42, 114, 18, 80, 85, 121, 99, 13, 87, 103, 37, 36, 22, 106, 51, 50, 111, 61, 90, 98, 109, 94, 12, 62, 105, 54, 14, 25, 107, 21, 20, 30, 44, 83, 82, 77, 101, 104, 70, 89, 6, 34, 102, 120, 84, 32, 81, 79, 23, 74, 16, 75, 15, 72, 9, 67, 3, 10, 78, 5, 71, 2, 0, 66, 73, 4, 65, 11, 8, 1, 69, 7, 68, 17, 64], [40, 46, 127, 97, 53, 43, 31, 38, 67, 41, 60, 75, 5, 14, 85, 71, 81, 65, 110, 24, 0, 9, 39, 90, 1, 19, 104, 47, 115, 112, 35, 91, 77, 21, 27, 88, 26, 17, 72, 7, 59, 102, 121, 124, 80, 93, 23, 79, 10, 76, 125, 48, 44, 2, 70, 8, 105, 100, 118, 54, 3, 56, 52, 103, 92, 64, 69, 116, 4, 12, 82, 57, 96, 87, 78, 49, 99, 36, 119, 42, 68, 74, 98, 11, 15, 117, 114, 18, 25, 86, 126, 37, 63, 28, 106, 122, 66, 34, 108, 120, 83, 84, 32, 62, 50, 94, 73, 107, 95, 22, 111, 30, 51, 113, 55, 16, 58, 61, 123, 109, 101, 20, 13, 45, 33, 89, 6, 29], [39, 124, 34, 117, 119, 47, 24, 62, 109, 20, 15, 13, 48, 26, 17, 11, 53, 70, 9, 72, 55, 19, 1, 65, 73, 28, 10, 78, 69, 123, 5, 21, 118, 68, 86, 83, 106, 81, 71, 49, 95, 112, 122, 25, 87, 31, 79, 116, 12, 92, 85, 18, 88, 27, 35, 115, 30, 90, 107, 14, 75, 8, 89, 7, 77, 66, 74, 22, 23, 94, 84, 45, 80, 91, 33, 3, 61, 6, 67, 76, 110, 52, 16, 82, 121, 2, 64, 4, 50, 93, 54, 29, 32, 43, 0, 97, 99, 41, 51, 111, 100, 63, 46, 37, 101, 36, 38, 56, 96, 59, 44, 127, 60, 40, 120, 114, 102, 42, 98, 113, 57, 105, 58, 108, 126, 104, 103, 125], [109, 119, 39, 62, 117, 34, 48, 124, 45, 61, 20, 30, 26, 94, 86, 110, 92, 24, 115, 17, 112, 116, 28, 122, 103, 19, 118, 114, 98, 15, 57, 123, 78, 51, 58, 125, 42, 63, 27, 53, 91, 55, 22, 47, 49, 126, 87, 95, 83, 54, 43, 127, 11, 59, 9, 44, 113, 111, 33, 121, 14, 38, 60, 50, 56, 52, 46, 108, 16, 68, 76, 41, 29, 32, 99, 89, 40, 13, 107, 105, 71, 106, 104, 102, 73, 90, 25, 21, 96, 37, 23, 100, 120, 72, 93, 35, 36, 101, 97, 31, 82, 80, 10, 85, 18, 84, 4, 70, 88, 69, 81, 7, 67, 75, 64, 0, 77, 74, 79, 2, 12, 8, 1, 3, 66, 5, 65, 6], [48, 39, 62, 34, 119, 109, 124, 117, 24, 47, 20, 11, 15, 13, 17, 94, 70, 73, 85, 10, 9, 68, 7, 86, 53, 80, 115, 19, 61, 118, 123, 45, 28, 22, 72, 121, 25, 55, 1, 3, 110, 93, 54, 92, 29, 89, 106, 30, 31, 69, 75, 112, 87, 95, 88, 66, 50, 12, 79, 26, 27, 81, 77, 16, 71, 82, 14, 60, 91, 64, 4, 43, 122, 84, 21, 40, 114, 90, 8, 23, 96, 0, 67, 18, 116, 76, 107, 57, 46, 83, 74, 97, 2, 32, 33, 102, 6, 100, 52, 111, 108, 37, 99, 105, 63, 78, 35, 44, 56, 49, 41, 36, 38, 65, 126, 127, 101, 42, 120, 51, 104, 125, 113, 58, 59, 5, 98, 103], [48, 39, 62, 124, 119, 109, 117, 34, 47, 53, 26, 30, 49, 86, 116, 45, 42, 112, 123, 20, 115, 63, 61, 55, 50, 125, 24, 111, 92, 110, 58, 33, 43, 122, 107, 118, 80, 54, 29, 114, 44, 41, 94, 46, 93, 96, 57, 40, 106, 52, 25, 120, 56, 60, 37, 113, 127, 73, 121, 51, 108, 36, 126, 104, 35, 22, 28, 98, 100, 105, 87, 82, 17, 83, 32, 102, 38, 99, 31, 59, 101, 91, 97, 95, 19, 23, 15, 27, 103, 89, 90, 85, 16, 76, 13, 21, 71, 78, 84, 74, 81, 9, 6, 14, 79, 18, 75, 72, 88, 11, 12, 10, 5, 77, 67, 68, 4, 8, 7, 70, 3, 2, 1, 69, 0, 65, 66, 64], [40, 97, 119, 31, 81, 23, 19, 52, 76, 85, 79, 111, 112, 93, 53, 110, 7, 73, 59, 61, 57, 104, 113, 117, 5, 126, 60, 58, 27, 3, 44, 54, 107, 48, 86, 13, 43, 24, 91, 114, 1, 10, 55, 67, 28, 115, 127, 78, 123, 50, 30, 90, 71, 15, 116, 56, 109, 82, 121, 21, 83, 94, 88, 108, 101, 12, 80, 87, 74, 103, 75, 29, 72, 77, 96, 47, 42, 17, 35, 16, 39, 84, 25, 11, 37, 100, 45, 26, 33, 122, 6, 22, 118, 124, 0, 34, 92, 63, 46, 8, 70, 65, 125, 36, 68, 106, 120, 18, 98, 69, 9, 99, 4, 89, 62, 32, 102, 105, 51, 38, 64, 20, 49, 14, 41, 95, 66, 2], [40, 97, 119, 31, 23, 81, 52, 13, 111, 53, 19, 79, 58, 112, 85, 25, 110, 60, 61, 117, 55, 123, 54, 24, 76, 57, 51, 126, 84, 20, 93, 113, 59, 73, 114, 104, 121, 88, 27, 91, 33, 108, 29, 47, 82, 18, 43, 26, 7, 28, 41, 86, 90, 72, 99, 103, 44, 109, 63, 107, 45, 77, 94, 122, 75, 30, 87, 70, 34, 105, 5, 21, 100, 42, 78, 74, 22, 127, 120, 50, 35, 101, 116, 32, 69, 96, 83, 56, 48, 14, 1, 115, 8, 15, 89, 38, 46, 124, 125, 17, 80, 37, 68, 49, 98, 12, 92, 16, 11, 36, 6, 118, 39, 9, 62, 102, 106, 2, 67, 95, 71, 0, 3, 66, 10, 4, 64, 65], [40, 97, 119, 31, 81, 85, 52, 23, 19, 111, 24, 79, 76, 73, 126, 57, 112, 54, 5, 93, 58, 53, 44, 104, 110, 117, 13, 48, 7, 59, 108, 107, 60, 15, 70, 123, 61, 113, 120, 34, 55, 114, 43, 25, 29, 3, 89, 1, 50, 109, 12, 30, 28, 6, 88, 82, 83, 116, 22, 39, 78, 91, 42, 56, 87, 2, 80, 121, 74, 69, 75, 127, 26, 51, 27, 115, 21, 68, 37, 103, 32, 46, 101, 94, 45, 47, 118, 99, 66, 62, 38, 122, 100, 124, 35, 20, 102, 77, 86, 64, 14, 41, 11, 96, 90, 92, 17, 33, 106, 105, 98, 16, 9, 63, 36, 10, 65, 8, 84, 125, 0, 49, 4, 71, 72, 18, 95, 67], [40, 97, 119, 31, 26, 81, 52, 19, 85, 79, 23, 112, 126, 111, 73, 110, 13, 76, 53, 57, 25, 58, 93, 55, 104, 114, 54, 78, 5, 117, 96, 51, 60, 123, 59, 7, 113, 99, 61, 42, 20, 27, 88, 84, 82, 45, 77, 43, 1, 125, 91, 87, 48, 44, 116, 24, 107, 109, 3, 70, 80, 102, 16, 94, 15, 69, 106, 21, 2, 9, 90, 56, 127, 121, 115, 83, 75, 12, 89, 33, 47, 122, 28, 101, 120, 92, 36, 34, 22, 17, 29, 6, 74, 41, 103, 108, 49, 32, 30, 63, 62, 14, 11, 86, 100, 105, 64, 0, 10, 67, 46, 35, 68, 118, 50, 39, 98, 37, 124, 66, 72, 38, 71, 4, 65, 18, 8, 95]], "model.layers.3.self_attn.k_proj": [[104, 34, 93, 56, 62, 119, 60, 58, 80, 83, 24, 14, 86, 12, 10, 0, 26, 70, 5, 2, 67, 125, 72, 54, 116, 114, 77, 25, 85, 73, 22, 9, 6, 71, 118, 115, 18, 13, 42, 112, 96, 32, 37, 68, 95, 117, 8, 92, 29, 49, 41, 51, 65, 84, 59, 7, 113, 61, 111, 100, 43, 105, 17, 103, 45, 127, 35, 121, 108, 55, 99, 110, 57, 109, 124, 30, 89, 75, 106, 122, 48, 107, 33, 63, 88, 50, 91, 28, 21, 120, 31, 76, 87, 101, 46, 102, 44, 47, 38, 82, 1, 15, 20, 78, 53, 27, 126, 97, 79, 4, 64, 123, 39, 36, 23, 52, 69, 94, 11, 90, 74, 81, 66, 3, 16, 19, 98, 40], [47, 122, 102, 123, 120, 64, 65, 69, 33, 8, 10, 12, 71, 81, 84, 67, 14, 23, 68, 66, 0, 2, 1, 75, 5, 3, 70, 24, 86, 113, 25, 13, 6, 19, 73, 43, 72, 88, 124, 87, 9, 42, 21, 40, 118, 59, 119, 111, 41, 38, 61, 103, 105, 39, 83, 79, 117, 49, 100, 114, 27, 15, 18, 16, 116, 60, 104, 82, 125, 121, 45, 50, 110, 53, 11, 108, 52, 37, 85, 55, 4, 109, 106, 127, 48, 115, 92, 96, 98, 58, 63, 28, 30, 89, 26, 107, 90, 31, 54, 80, 36, 77, 46, 95, 99, 56, 62, 51, 57, 112, 91, 29, 34, 35, 126, 94, 44, 22, 101, 20, 93, 74, 7, 17, 78, 32, 76, 97], [105, 98, 52, 79, 18, 9, 0, 20, 22, 76, 7, 111, 69, 110, 14, 120, 10, 2, 62, 50, 63, 67, 1, 4, 126, 53, 6, 127, 41, 59, 114, 125, 24, 113, 65, 43, 11, 8, 46, 48, 89, 123, 19, 77, 117, 119, 54, 58, 81, 90, 70, 17, 47, 60, 68, 95, 28, 64, 3, 40, 122, 112, 72, 91, 71, 108, 75, 85, 61, 44, 83, 51, 121, 124, 73, 115, 32, 93, 116, 66, 30, 74, 97, 29, 45, 87, 57, 33, 94, 38, 16, 55, 99, 39, 23, 21, 100, 31, 26, 88, 109, 118, 96, 56, 27, 101, 42, 103, 49, 104, 102, 78, 106, 35, 92, 37, 107, 36, 80, 25, 86, 82, 12, 15, 5, 13, 84, 34], [39, 52, 33, 45, 125, 110, 28, 74, 119, 87, 117, 46, 78, 89, 4, 21, 12, 80, 0, 66, 11, 86, 8, 17, 71, 82, 20, 1, 7, 58, 62, 65, 79, 24, 6, 53, 67, 57, 95, 3, 90, 94, 64, 72, 127, 5, 56, 50, 54, 9, 116, 126, 63, 48, 108, 55, 123, 19, 75, 43, 109, 61, 92, 69, 68, 15, 118, 49, 73, 59, 13, 60, 106, 122, 51, 77, 88, 41, 120, 113, 124, 115, 101, 112, 107, 44, 84, 32, 42, 105, 37, 121, 96, 114, 83, 102, 104, 30, 14, 111, 10, 16, 100, 40, 38, 29, 99, 35, 34, 47, 36, 2, 98, 31, 91, 27, 81, 76, 93, 26, 18, 103, 22, 70, 97, 23, 85, 25], [44, 40, 64, 97, 108, 13, 1, 112, 16, 10, 56, 23, 46, 50, 67, 18, 70, 66, 47, 7, 68, 69, 123, 2, 104, 61, 72, 20, 52, 55, 57, 120, 51, 12, 3, 125, 116, 118, 58, 119, 117, 54, 49, 109, 4, 124, 14, 73, 11, 39, 53, 6, 43, 115, 114, 86, 75, 21, 5, 85, 45, 60, 126, 62, 78, 22, 65, 107, 41, 90, 83, 42, 81, 36, 15, 76, 27, 122, 38, 19, 127, 99, 71, 95, 25, 106, 91, 89, 111, 110, 88, 79, 121, 103, 92, 17, 63, 29, 98, 31, 24, 96, 105, 26, 93, 30, 28, 59, 113, 94, 32, 84, 102, 101, 37, 77, 35, 87, 48, 9, 34, 100, 74, 82, 0, 80, 8, 33], [104, 127, 53, 110, 95, 33, 90, 85, 122, 35, 83, 64, 88, 111, 81, 19, 115, 91, 14, 87, 102, 106, 75, 70, 76, 9, 29, 71, 66, 80, 93, 24, 18, 61, 99, 48, 45, 63, 20, 92, 126, 54, 107, 103, 125, 13, 4, 97, 105, 1, 60, 108, 72, 121, 77, 2, 59, 57, 123, 6, 3, 62, 117, 49, 114, 56, 67, 5, 44, 42, 36, 50, 43, 51, 109, 28, 58, 47, 113, 116, 37, 74, 98, 55, 112, 101, 27, 124, 89, 118, 34, 46, 120, 119, 79, 32, 52, 25, 94, 69, 78, 86, 96, 30, 10, 15, 39, 82, 100, 16, 23, 84, 0, 8, 22, 41, 26, 68, 65, 21, 73, 31, 38, 11, 17, 12, 40, 7], [103, 98, 124, 119, 62, 48, 11, 24, 17, 117, 20, 15, 13, 72, 112, 26, 0, 70, 68, 1, 30, 69, 53, 2, 3, 45, 9, 47, 111, 109, 78, 7, 54, 43, 10, 67, 19, 22, 76, 52, 86, 115, 64, 87, 49, 125, 57, 46, 126, 51, 56, 122, 66, 61, 118, 63, 127, 60, 71, 44, 120, 113, 80, 107, 55, 92, 121, 101, 106, 40, 105, 110, 38, 74, 108, 95, 73, 100, 42, 35, 58, 114, 32, 29, 104, 41, 50, 28, 116, 21, 37, 33, 123, 102, 4, 31, 93, 94, 14, 25, 82, 36, 97, 59, 96, 27, 23, 83, 85, 99, 39, 5, 8, 84, 89, 91, 34, 90, 18, 6, 16, 75, 88, 12, 77, 65, 81, 79], [104, 33, 119, 23, 19, 79, 95, 47, 85, 76, 112, 81, 55, 52, 53, 73, 113, 7, 65, 75, 59, 58, 126, 5, 46, 60, 57, 29, 61, 108, 13, 64, 49, 3, 123, 107, 54, 71, 14, 44, 117, 127, 10, 69, 80, 109, 43, 68, 114, 67, 37, 86, 25, 48, 77, 24, 28, 91, 2, 56, 11, 27, 18, 111, 4, 118, 124, 72, 30, 9, 115, 66, 40, 121, 62, 42, 16, 106, 103, 22, 38, 125, 70, 63, 74, 120, 93, 50, 39, 20, 6, 105, 35, 98, 90, 17, 101, 34, 32, 94, 96, 88, 92, 26, 110, 100, 99, 102, 89, 116, 87, 45, 51, 82, 41, 122, 36, 78, 21, 84, 0, 8, 1, 12, 15, 97, 83, 31]], "model.layers.3.self_attn.qk_proj": [[119, 104, 47, 62, 120, 52, 44, 123, 122, 53, 117, 127, 105, 40, 108, 125, 97, 110, 124, 48, 46, 56, 84, 20, 58, 60, 12, 87, 41, 109, 23, 74, 17, 76, 81, 78, 111, 112, 82, 10, 7, 14, 126, 64, 39, 15, 0, 79, 71, 103, 77, 57, 18, 13, 16, 33, 54, 114, 63, 24, 72, 88, 38, 83, 22, 86, 70, 55, 3, 19, 80, 21, 67, 9, 85, 29, 8, 45, 50, 59, 65, 5, 73, 69, 34, 89, 11, 92, 66, 90, 4, 116, 1, 25, 61, 2, 95, 113, 93, 68, 98, 75, 31, 43, 118, 26, 49, 6, 27, 115, 51, 121, 94, 28, 102, 35, 42, 30, 107, 32, 37, 106, 99, 91, 36, 96, 101, 100], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 48, 46, 60, 58, 20, 84, 41, 76, 87, 109, 23, 17, 12, 74, 81, 10, 0, 14, 111, 112, 78, 64, 82, 15, 7, 103, 71, 70, 18, 86, 33, 39, 114, 63, 22, 19, 24, 80, 72, 77, 79, 57, 38, 83, 88, 13, 54, 3, 1, 65, 59, 16, 50, 8, 5, 21, 126, 69, 85, 29, 55, 92, 73, 9, 67, 2, 90, 66, 4, 68, 31, 45, 34, 11, 113, 98, 25, 116, 61, 95, 43, 89, 93, 75, 26, 51, 27, 118, 49, 6, 115, 102, 28, 94, 107, 121, 35, 30, 37, 42, 32, 96, 91, 99, 106, 101, 36, 100], [119, 104, 47, 62, 52, 120, 44, 123, 122, 53, 117, 127, 40, 105, 125, 108, 97, 56, 110, 124, 48, 58, 46, 20, 60, 84, 87, 23, 41, 12, 76, 109, 17, 74, 81, 64, 111, 82, 78, 10, 0, 14, 7, 112, 39, 18, 33, 71, 15, 38, 79, 86, 70, 67, 13, 3, 22, 24, 103, 57, 77, 88, 83, 5, 80, 16, 21, 19, 54, 126, 72, 69, 8, 1, 59, 65, 9, 55, 73, 114, 66, 4, 63, 50, 34, 29, 85, 2, 68, 92, 11, 45, 113, 93, 25, 89, 98, 90, 31, 116, 95, 61, 43, 75, 6, 26, 28, 49, 27, 51, 94, 115, 102, 118, 107, 121, 35, 42, 32, 30, 37, 91, 99, 106, 36, 96, 100, 101], [119, 104, 47, 62, 52, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 108, 97, 48, 110, 124, 56, 46, 58, 60, 84, 20, 87, 41, 12, 23, 74, 112, 17, 109, 76, 81, 111, 82, 0, 78, 10, 14, 18, 64, 79, 86, 7, 38, 71, 57, 39, 67, 24, 15, 103, 33, 19, 54, 13, 5, 8, 22, 65, 80, 114, 70, 55, 63, 88, 3, 83, 69, 126, 72, 1, 16, 9, 29, 50, 77, 85, 59, 45, 4, 92, 66, 31, 73, 2, 93, 68, 116, 90, 21, 113, 98, 89, 25, 6, 95, 34, 61, 75, 11, 43, 26, 27, 115, 118, 51, 102, 49, 28, 94, 107, 30, 121, 42, 32, 106, 91, 37, 35, 99, 96, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 40, 105, 125, 108, 97, 48, 124, 110, 56, 60, 46, 58, 84, 12, 20, 87, 17, 76, 41, 112, 10, 23, 81, 78, 74, 109, 111, 0, 7, 18, 64, 71, 79, 14, 82, 24, 86, 33, 8, 77, 5, 19, 63, 22, 114, 39, 103, 57, 126, 67, 69, 72, 15, 38, 13, 9, 54, 59, 16, 88, 55, 80, 68, 3, 1, 65, 83, 6, 50, 29, 113, 21, 85, 73, 4, 45, 66, 92, 70, 90, 2, 93, 98, 89, 11, 95, 116, 75, 31, 61, 43, 34, 25, 26, 94, 115, 27, 51, 118, 49, 121, 28, 102, 107, 42, 32, 106, 30, 35, 91, 37, 96, 36, 100, 99, 101], [119, 104, 47, 52, 62, 120, 123, 44, 53, 122, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 46, 48, 84, 87, 58, 20, 60, 12, 76, 41, 17, 74, 23, 78, 109, 81, 112, 10, 111, 7, 0, 64, 14, 71, 18, 79, 82, 24, 86, 19, 39, 69, 15, 13, 54, 8, 114, 77, 67, 3, 9, 57, 63, 83, 103, 88, 5, 6, 33, 126, 55, 16, 38, 72, 22, 80, 85, 59, 65, 29, 21, 50, 73, 2, 66, 113, 68, 1, 116, 45, 11, 4, 92, 95, 90, 61, 31, 75, 89, 98, 34, 93, 25, 70, 43, 115, 26, 118, 27, 51, 49, 42, 94, 102, 121, 28, 107, 30, 37, 91, 106, 32, 99, 35, 101, 96, 36, 100], [119, 104, 47, 52, 120, 62, 123, 122, 44, 53, 105, 117, 40, 127, 108, 125, 97, 110, 56, 124, 58, 87, 84, 46, 41, 20, 60, 48, 76, 12, 74, 23, 17, 81, 112, 109, 82, 71, 14, 10, 78, 103, 18, 79, 39, 7, 64, 0, 111, 15, 6, 22, 13, 16, 24, 54, 19, 77, 88, 86, 126, 8, 69, 57, 85, 59, 83, 33, 5, 63, 9, 3, 72, 38, 21, 80, 67, 29, 61, 113, 114, 1, 50, 73, 55, 66, 34, 11, 68, 65, 116, 75, 2, 25, 93, 95, 90, 4, 92, 89, 45, 98, 31, 43, 115, 27, 118, 26, 94, 51, 49, 70, 121, 42, 102, 30, 107, 35, 28, 99, 32, 37, 91, 106, 101, 36, 96, 100], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 56, 124, 58, 48, 87, 84, 20, 46, 41, 76, 12, 60, 17, 23, 81, 78, 74, 10, 109, 14, 79, 18, 0, 6, 111, 112, 7, 82, 64, 13, 15, 19, 86, 71, 39, 16, 103, 22, 77, 80, 88, 33, 9, 24, 8, 38, 54, 83, 63, 5, 57, 85, 21, 126, 73, 72, 59, 65, 3, 69, 55, 114, 1, 2, 29, 66, 50, 61, 34, 45, 93, 75, 68, 67, 90, 89, 95, 4, 11, 92, 98, 113, 25, 116, 31, 26, 118, 49, 70, 51, 115, 28, 27, 43, 42, 121, 94, 35, 102, 107, 32, 37, 30, 101, 91, 106, 36, 99, 96, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 124, 110, 56, 58, 48, 46, 20, 60, 84, 87, 41, 76, 12, 81, 23, 17, 109, 78, 111, 10, 74, 14, 112, 79, 64, 7, 39, 82, 18, 0, 71, 63, 86, 13, 16, 15, 103, 19, 33, 6, 8, 24, 80, 67, 114, 22, 83, 38, 77, 88, 21, 3, 5, 50, 126, 1, 57, 59, 54, 72, 55, 29, 9, 85, 69, 65, 45, 2, 4, 89, 66, 116, 31, 73, 98, 34, 90, 92, 68, 25, 11, 75, 113, 93, 95, 61, 26, 70, 43, 49, 28, 27, 115, 121, 51, 118, 94, 102, 107, 35, 42, 30, 37, 101, 99, 32, 91, 106, 96, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 105, 117, 40, 127, 108, 125, 97, 124, 56, 110, 58, 46, 48, 84, 87, 20, 12, 41, 23, 60, 81, 17, 109, 74, 76, 18, 10, 14, 78, 82, 15, 33, 111, 112, 7, 79, 86, 64, 22, 0, 103, 39, 80, 19, 83, 24, 71, 38, 13, 16, 77, 88, 57, 21, 63, 59, 55, 85, 8, 54, 29, 5, 114, 45, 69, 6, 9, 72, 3, 73, 92, 126, 67, 50, 2, 113, 66, 98, 34, 93, 89, 116, 1, 65, 61, 90, 68, 43, 95, 4, 75, 25, 26, 31, 70, 11, 49, 27, 115, 118, 28, 30, 102, 51, 94, 35, 42, 121, 107, 32, 37, 99, 106, 96, 91, 101, 36, 100], [119, 104, 47, 62, 120, 52, 123, 122, 44, 53, 127, 40, 117, 105, 125, 108, 97, 48, 124, 56, 110, 46, 60, 20, 84, 41, 58, 12, 87, 76, 17, 111, 109, 10, 23, 112, 81, 0, 14, 18, 74, 78, 64, 7, 71, 39, 79, 15, 103, 82, 63, 33, 1, 8, 24, 19, 38, 86, 80, 69, 77, 13, 57, 55, 114, 50, 85, 22, 83, 3, 59, 88, 72, 16, 54, 5, 73, 29, 65, 126, 4, 21, 45, 67, 70, 116, 66, 31, 2, 9, 95, 90, 93, 68, 75, 61, 26, 92, 98, 113, 6, 34, 25, 89, 11, 118, 49, 94, 43, 115, 51, 27, 102, 28, 121, 107, 42, 91, 106, 30, 35, 37, 32, 99, 100, 101, 96, 36], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 40, 117, 125, 105, 108, 97, 124, 56, 110, 48, 46, 20, 60, 58, 84, 87, 41, 76, 112, 12, 81, 17, 14, 23, 10, 109, 111, 74, 7, 0, 3, 39, 78, 82, 64, 71, 24, 67, 103, 18, 79, 86, 114, 15, 88, 70, 33, 5, 57, 69, 8, 38, 72, 54, 22, 19, 63, 77, 80, 1, 13, 126, 55, 83, 16, 65, 85, 59, 21, 50, 29, 73, 9, 61, 68, 66, 2, 90, 92, 4, 116, 75, 95, 34, 45, 113, 11, 25, 98, 89, 93, 31, 26, 6, 27, 118, 51, 94, 115, 43, 49, 102, 28, 121, 42, 30, 107, 32, 35, 37, 106, 99, 91, 100, 36, 101, 96], [119, 104, 52, 47, 62, 120, 123, 44, 122, 117, 53, 105, 40, 127, 108, 125, 97, 56, 110, 124, 58, 20, 84, 87, 48, 41, 46, 76, 23, 60, 12, 17, 81, 10, 14, 74, 109, 78, 71, 82, 18, 15, 79, 112, 7, 80, 86, 33, 83, 70, 39, 103, 19, 24, 22, 64, 13, 77, 111, 38, 0, 88, 63, 5, 21, 16, 57, 69, 72, 85, 8, 9, 54, 55, 29, 67, 126, 92, 116, 93, 50, 59, 113, 3, 1, 2, 114, 25, 61, 73, 66, 75, 34, 89, 4, 65, 11, 45, 90, 95, 68, 98, 31, 27, 118, 26, 43, 49, 94, 28, 115, 51, 30, 107, 102, 6, 35, 42, 32, 121, 96, 37, 91, 99, 106, 101, 100, 36], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 127, 117, 105, 40, 125, 108, 97, 110, 124, 56, 20, 48, 58, 46, 84, 60, 23, 41, 76, 17, 12, 109, 87, 74, 111, 81, 10, 14, 71, 112, 64, 82, 15, 18, 78, 103, 7, 70, 0, 39, 79, 86, 22, 13, 33, 19, 69, 24, 57, 83, 16, 38, 80, 114, 72, 54, 59, 8, 63, 77, 126, 88, 1, 73, 5, 85, 50, 55, 66, 29, 9, 21, 67, 2, 3, 45, 25, 65, 68, 90, 92, 113, 4, 34, 116, 11, 61, 93, 95, 98, 75, 31, 89, 43, 49, 26, 115, 51, 118, 6, 27, 28, 94, 121, 102, 30, 107, 42, 32, 99, 91, 37, 35, 106, 36, 101, 100, 96], [119, 104, 47, 62, 120, 52, 123, 44, 122, 53, 117, 127, 105, 40, 125, 108, 97, 110, 124, 58, 56, 48, 20, 84, 60, 87, 41, 46, 23, 12, 76, 81, 111, 14, 74, 17, 71, 10, 112, 0, 109, 64, 7, 79, 39, 18, 67, 72, 103, 78, 15, 69, 3, 86, 82, 126, 1, 33, 24, 13, 55, 5, 38, 70, 22, 80, 54, 19, 16, 57, 83, 114, 88, 9, 73, 85, 77, 68, 113, 8, 63, 65, 59, 2, 21, 29, 50, 66, 90, 31, 98, 116, 6, 4, 75, 92, 61, 93, 25, 45, 89, 34, 11, 95, 26, 49, 118, 51, 27, 115, 28, 102, 43, 42, 94, 121, 106, 35, 30, 99, 107, 32, 91, 101, 37, 96, 36, 100], [119, 104, 47, 62, 52, 120, 123, 122, 44, 53, 127, 117, 105, 40, 125, 108, 97, 48, 110, 56, 124, 46, 20, 58, 84, 41, 60, 87, 76, 12, 17, 81, 23, 111, 112, 109, 71, 10, 74, 14, 78, 39, 7, 15, 0, 82, 72, 18, 103, 86, 64, 63, 24, 33, 57, 13, 55, 38, 79, 69, 114, 67, 83, 126, 22, 88, 3, 21, 80, 16, 59, 50, 29, 54, 19, 77, 9, 5, 85, 65, 73, 8, 1, 116, 92, 90, 113, 70, 68, 4, 45, 6, 98, 95, 93, 2, 31, 61, 66, 26, 11, 34, 43, 75, 25, 89, 118, 51, 49, 115, 94, 102, 27, 121, 28, 42, 30, 106, 107, 35, 91, 32, 99, 37, 100, 101, 96, 36], [119, 104, 52, 47, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 110, 124, 56, 20, 48, 46, 58, 84, 87, 23, 41, 76, 12, 60, 10, 17, 81, 74, 78, 18, 109, 14, 7, 111, 71, 15, 112, 82, 79, 22, 80, 13, 86, 64, 69, 19, 83, 0, 57, 33, 39, 24, 77, 9, 38, 103, 88, 6, 5, 21, 72, 29, 63, 85, 54, 55, 16, 73, 8, 126, 50, 3, 114, 75, 92, 67, 59, 68, 65, 66, 2, 89, 1, 113, 116, 95, 4, 34, 93, 45, 11, 25, 90, 98, 61, 115, 31, 70, 26, 49, 27, 43, 51, 28, 118, 102, 94, 30, 42, 121, 107, 37, 91, 35, 32, 99, 106, 100, 96, 101, 36], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 117, 105, 127, 40, 108, 125, 97, 56, 124, 110, 20, 58, 84, 48, 41, 23, 76, 60, 87, 46, 17, 12, 10, 109, 74, 81, 82, 14, 78, 112, 6, 64, 15, 0, 71, 7, 111, 18, 22, 79, 69, 39, 16, 57, 86, 33, 19, 24, 103, 3, 13, 83, 77, 80, 38, 72, 67, 1, 114, 59, 63, 54, 21, 55, 88, 8, 73, 85, 29, 9, 2, 126, 93, 66, 113, 45, 4, 116, 75, 65, 50, 5, 92, 95, 98, 90, 61, 25, 31, 68, 34, 89, 11, 26, 70, 28, 43, 51, 49, 27, 118, 115, 102, 30, 42, 94, 37, 121, 107, 32, 91, 35, 106, 99, 101, 96, 36, 100], [119, 104, 47, 52, 62, 120, 123, 122, 53, 44, 127, 117, 105, 40, 125, 97, 108, 110, 124, 56, 48, 58, 60, 46, 20, 84, 76, 87, 23, 41, 12, 81, 17, 10, 74, 14, 112, 78, 111, 109, 64, 82, 71, 7, 15, 79, 39, 18, 0, 6, 86, 33, 103, 57, 126, 38, 16, 114, 69, 24, 1, 77, 19, 8, 67, 22, 13, 83, 80, 88, 54, 85, 63, 72, 55, 73, 59, 3, 29, 5, 9, 116, 50, 2, 75, 93, 4, 21, 66, 61, 92, 65, 98, 34, 90, 45, 68, 31, 95, 113, 89, 25, 11, 26, 70, 49, 43, 51, 118, 115, 27, 94, 28, 102, 42, 107, 121, 32, 30, 35, 37, 91, 99, 106, 101, 36, 96, 100], [119, 104, 47, 62, 52, 120, 122, 123, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 60, 46, 48, 84, 87, 20, 41, 12, 76, 17, 74, 81, 10, 109, 111, 23, 14, 112, 71, 7, 78, 79, 18, 82, 86, 69, 64, 15, 19, 33, 0, 24, 57, 39, 6, 13, 83, 22, 103, 63, 77, 38, 73, 80, 114, 72, 1, 88, 8, 55, 16, 67, 9, 5, 126, 54, 29, 21, 2, 3, 50, 85, 116, 59, 113, 45, 92, 68, 75, 4, 90, 93, 98, 31, 34, 65, 95, 66, 61, 89, 43, 26, 70, 11, 25, 49, 51, 102, 28, 115, 118, 27, 94, 42, 30, 107, 121, 35, 37, 32, 91, 101, 99, 96, 106, 100, 36], [119, 104, 62, 52, 47, 120, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 48, 124, 56, 46, 20, 84, 60, 58, 87, 41, 109, 17, 12, 23, 112, 76, 111, 81, 74, 10, 82, 78, 14, 71, 39, 7, 0, 79, 57, 18, 38, 64, 24, 3, 22, 33, 15, 16, 126, 67, 54, 19, 55, 80, 114, 77, 86, 63, 59, 72, 9, 88, 69, 103, 83, 13, 8, 29, 5, 50, 1, 85, 21, 116, 4, 95, 68, 89, 92, 113, 90, 6, 45, 73, 70, 65, 98, 31, 66, 93, 75, 2, 11, 61, 34, 25, 26, 51, 115, 49, 94, 118, 42, 43, 121, 28, 27, 107, 30, 102, 37, 35, 91, 99, 106, 100, 32, 101, 36, 96], [119, 104, 47, 52, 120, 62, 123, 122, 53, 44, 117, 127, 105, 40, 125, 108, 97, 110, 58, 124, 56, 84, 46, 87, 48, 20, 60, 41, 12, 17, 23, 76, 112, 111, 81, 109, 10, 0, 64, 74, 7, 14, 18, 39, 71, 57, 79, 33, 82, 15, 78, 86, 24, 54, 70, 22, 126, 38, 77, 103, 69, 63, 1, 80, 8, 16, 114, 5, 55, 67, 2, 19, 9, 72, 83, 29, 50, 68, 3, 88, 13, 66, 73, 85, 21, 116, 4, 34, 65, 59, 31, 113, 93, 45, 61, 95, 89, 90, 92, 98, 11, 26, 75, 6, 43, 25, 118, 115, 49, 27, 51, 102, 28, 42, 94, 107, 121, 30, 35, 32, 91, 99, 106, 101, 37, 96, 36, 100], [119, 104, 52, 47, 62, 120, 123, 122, 44, 53, 117, 127, 40, 105, 125, 97, 108, 110, 48, 124, 56, 46, 84, 58, 60, 20, 87, 23, 41, 12, 76, 17, 74, 81, 109, 14, 10, 111, 112, 18, 7, 78, 71, 82, 79, 22, 15, 33, 86, 70, 0, 64, 24, 39, 19, 8, 38, 83, 57, 16, 103, 29, 88, 13, 5, 63, 80, 85, 77, 9, 59, 67, 54, 69, 114, 72, 55, 21, 65, 126, 50, 68, 116, 73, 4, 1, 113, 61, 3, 45, 92, 89, 98, 2, 93, 34, 31, 90, 75, 11, 66, 95, 25, 26, 118, 6, 43, 27, 94, 28, 49, 51, 102, 115, 42, 107, 30, 121, 32, 91, 35, 106, 96, 37, 99, 101, 36, 100], [119, 104, 47, 52, 62, 120, 123, 53, 44, 122, 127, 117, 105, 40, 125, 97, 108, 110, 48, 124, 56, 46, 20, 84, 58, 60, 87, 41, 12, 76, 74, 23, 17, 112, 81, 111, 78, 109, 10, 82, 18, 14, 15, 7, 86, 39, 71, 103, 57, 0, 24, 8, 70, 16, 19, 126, 64, 38, 67, 33, 13, 22, 83, 79, 63, 77, 114, 88, 55, 80, 50, 3, 59, 9, 5, 92, 54, 116, 29, 85, 1, 73, 113, 69, 21, 34, 68, 72, 89, 45, 98, 61, 95, 4, 65, 2, 11, 90, 31, 75, 66, 93, 43, 25, 26, 51, 115, 118, 49, 27, 94, 6, 28, 102, 42, 107, 30, 121, 37, 106, 91, 32, 35, 99, 101, 36, 100, 96], [119, 104, 47, 52, 62, 120, 123, 122, 44, 53, 117, 127, 105, 40, 125, 108, 97, 110, 56, 124, 58, 48, 60, 20, 84, 46, 87, 41, 12, 76, 23, 17, 109, 81, 10, 78, 74, 112, 7, 111, 82, 39, 15, 14, 57, 18, 79, 126, 71, 64, 0, 103, 33, 86, 24, 88, 16, 54, 83, 22, 5, 8, 13, 77, 85, 80, 38, 69, 19, 70, 114, 59, 21, 72, 29, 63, 3, 1, 67, 61, 9, 55, 92, 50, 73, 95, 31, 93, 90, 4, 2, 66, 65, 25, 113, 34, 45, 75, 11, 89, 116, 98, 68, 118, 26, 6, 51, 49, 43, 27, 94, 115, 42, 102, 28, 30, 35, 121, 32, 107, 37, 106, 99, 91, 96, 36, 101, 100], [119, 104, 52, 47, 62, 120, 123, 44, 122, 53, 127, 117, 40, 105, 125, 108, 97, 56, 110, 48, 124, 20, 46, 58, 87, 60, 84, 41, 12, 76, 23, 81, 10, 112, 17, 74, 109, 111, 7, 39, 14, 18, 0, 78, 82, 64, 15, 71, 103, 79, 57, 33, 19, 22, 83, 16, 13, 86, 126, 77, 38, 69, 80, 8, 88, 3, 24, 5, 50, 67, 54, 116, 85, 70, 72, 9, 63, 1, 114, 21, 55, 59, 92, 73, 2, 29, 34, 65, 25, 68, 61, 113, 89, 45, 4, 66, 90, 95, 31, 6, 11, 98, 118, 93, 51, 75, 28, 26, 115, 27, 49, 43, 30, 107, 94, 42, 102, 121, 106, 37, 35, 91, 32, 36, 101, 100, 99, 96], [119, 104, 47, 52, 62, 120, 123, 44, 122, 53, 117, 127, 105, 40, 108, 125, 97, 56, 110, 124, 20, 58, 48, 84, 41, 46, 60, 23, 12, 87, 109, 76, 74, 17, 81, 111, 10, 112, 18, 78, 14, 7, 33, 71, 0, 15, 39, 64, 83, 82, 57, 79, 77, 86, 38, 22, 103, 88, 16, 19, 24, 3, 80, 126, 67, 59, 8, 72, 54, 13, 55, 21, 63, 65, 85, 50, 69, 45, 5, 29, 92, 68, 6, 73, 1, 25, 89, 9, 114, 34, 2, 95, 113, 4, 93, 98, 11, 90, 66, 116, 75, 49, 70, 31, 61, 26, 27, 121, 43, 115, 118, 102, 51, 28, 94, 35, 42, 107, 30, 32, 91, 37, 106, 100, 99, 96, 36, 101], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 127, 40, 125, 108, 97, 110, 124, 56, 48, 58, 84, 46, 76, 20, 87, 60, 41, 12, 74, 81, 23, 10, 17, 112, 0, 7, 71, 109, 14, 64, 78, 111, 18, 15, 57, 82, 79, 77, 126, 6, 39, 72, 69, 103, 86, 33, 8, 114, 59, 88, 1, 38, 80, 5, 16, 83, 13, 73, 22, 24, 63, 54, 67, 19, 3, 55, 9, 2, 85, 50, 21, 29, 65, 68, 66, 113, 93, 92, 34, 45, 75, 4, 98, 116, 31, 90, 61, 11, 95, 49, 25, 26, 89, 115, 27, 43, 118, 51, 70, 102, 94, 42, 28, 37, 121, 30, 107, 35, 99, 91, 32, 100, 106, 96, 36, 101], [119, 104, 47, 120, 62, 52, 123, 53, 122, 44, 127, 117, 40, 105, 125, 108, 97, 110, 124, 56, 20, 48, 60, 58, 46, 87, 84, 41, 76, 12, 74, 17, 23, 112, 111, 109, 10, 81, 78, 14, 7, 39, 71, 18, 82, 0, 79, 33, 86, 64, 114, 15, 103, 24, 77, 6, 83, 38, 8, 1, 72, 19, 22, 13, 57, 3, 126, 16, 54, 88, 55, 59, 5, 9, 63, 80, 50, 73, 67, 21, 69, 29, 65, 92, 116, 85, 113, 31, 95, 93, 68, 98, 4, 90, 66, 2, 45, 11, 34, 61, 75, 43, 25, 89, 118, 26, 115, 49, 70, 102, 27, 51, 121, 94, 28, 107, 30, 42, 37, 32, 35, 91, 106, 99, 100, 36, 101, 96], [119, 104, 47, 62, 52, 120, 123, 44, 122, 53, 127, 40, 105, 117, 125, 97, 108, 124, 48, 56, 46, 110, 20, 84, 60, 12, 58, 41, 87, 76, 23, 17, 74, 111, 112, 81, 109, 7, 10, 14, 78, 0, 39, 82, 71, 18, 15, 64, 79, 67, 6, 77, 103, 83, 72, 5, 22, 57, 33, 54, 59, 38, 126, 24, 86, 88, 13, 69, 16, 3, 63, 19, 8, 114, 29, 80, 9, 73, 55, 85, 65, 50, 68, 21, 1, 116, 2, 92, 11, 113, 4, 45, 89, 95, 31, 66, 61, 90, 34, 98, 43, 93, 25, 70, 118, 27, 75, 26, 115, 121, 51, 49, 102, 30, 94, 42, 91, 107, 28, 106, 37, 32, 35, 99, 101, 96, 100, 36], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 117, 105, 40, 127, 108, 125, 97, 124, 56, 110, 58, 20, 87, 84, 46, 41, 76, 60, 23, 48, 12, 74, 17, 10, 81, 111, 0, 78, 14, 109, 18, 64, 82, 79, 71, 86, 7, 112, 103, 15, 33, 39, 83, 5, 22, 57, 72, 77, 38, 19, 13, 59, 24, 54, 16, 80, 6, 88, 9, 55, 126, 21, 69, 8, 3, 63, 73, 65, 29, 2, 85, 34, 4, 50, 113, 67, 114, 92, 66, 1, 89, 45, 75, 70, 95, 25, 68, 11, 116, 61, 98, 93, 31, 90, 43, 26, 115, 118, 27, 49, 102, 28, 51, 30, 94, 121, 35, 32, 42, 99, 37, 107, 91, 106, 96, 101, 36, 100], [119, 104, 47, 52, 120, 62, 123, 44, 122, 53, 127, 40, 105, 117, 125, 108, 97, 56, 46, 110, 124, 48, 60, 20, 87, 41, 84, 58, 76, 12, 17, 109, 23, 81, 10, 74, 111, 112, 14, 78, 82, 103, 7, 71, 64, 15, 18, 39, 79, 72, 0, 13, 22, 80, 33, 63, 86, 83, 5, 38, 24, 126, 3, 57, 88, 114, 16, 55, 77, 67, 19, 59, 85, 116, 69, 9, 8, 54, 29, 65, 21, 2, 50, 73, 1, 34, 70, 113, 66, 4, 89, 92, 98, 6, 61, 25, 45, 95, 11, 31, 90, 75, 68, 93, 118, 115, 43, 26, 51, 27, 49, 28, 102, 121, 42, 30, 94, 107, 32, 37, 35, 106, 101, 36, 99, 91, 100, 96]], "model.layers.4.self_attn.q_proj": [[42, 122, 117, 56, 53, 123, 101, 124, 102, 121, 33, 61, 27, 107, 104, 119, 59, 48, 110, 39, 116, 83, 55, 109, 127, 115, 37, 114, 113, 126, 92, 57, 60, 43, 58, 17, 50, 26, 38, 120, 47, 125, 44, 45, 51, 54, 89, 62, 105, 111, 94, 21, 52, 46, 87, 41, 36, 108, 32, 106, 91, 49, 63, 112, 86, 118, 40, 23, 31, 30, 76, 79, 99, 22, 34, 103, 84, 96, 100, 72, 97, 98, 28, 78, 82, 25, 24, 95, 93, 16, 75, 0, 29, 35, 15, 13, 81, 10, 66, 6, 5, 73, 2, 11, 14, 3, 85, 19, 64, 8, 12, 74, 4, 68, 90, 69, 20, 65, 70, 88, 67, 7, 80, 71, 77, 9, 1, 18], [42, 101, 117, 114, 123, 56, 122, 102, 33, 53, 48, 124, 107, 59, 111, 39, 89, 94, 91, 27, 86, 121, 54, 57, 21, 37, 104, 28, 125, 112, 49, 79, 116, 40, 109, 36, 113, 43, 44, 41, 60, 118, 99, 26, 47, 38, 84, 82, 58, 46, 87, 25, 32, 17, 115, 52, 110, 61, 95, 92, 108, 62, 119, 24, 127, 45, 50, 120, 55, 51, 85, 29, 63, 90, 126, 30, 15, 93, 105, 106, 81, 34, 83, 22, 12, 96, 76, 16, 31, 98, 78, 75, 19, 35, 23, 74, 97, 13, 88, 8, 73, 20, 14, 100, 103, 77, 68, 72, 71, 6, 7, 69, 5, 11, 9, 66, 10, 4, 1, 65, 70, 3, 64, 67, 2, 0, 18, 80], [101, 42, 117, 33, 123, 122, 56, 53, 114, 87, 84, 82, 124, 91, 94, 39, 61, 79, 78, 75, 16, 73, 13, 6, 48, 86, 27, 72, 46, 111, 107, 2, 57, 89, 32, 95, 30, 21, 68, 22, 104, 20, 19, 74, 54, 71, 58, 102, 23, 109, 67, 44, 14, 85, 25, 8, 11, 88, 63, 55, 90, 10, 24, 106, 65, 41, 110, 36, 98, 64, 120, 35, 121, 92, 105, 7, 81, 99, 17, 49, 113, 28, 15, 60, 3, 77, 118, 108, 40, 116, 93, 76, 80, 18, 29, 52, 12, 38, 83, 26, 96, 4, 69, 119, 34, 70, 43, 31, 1, 9, 62, 47, 103, 115, 125, 51, 112, 127, 126, 59, 50, 97, 37, 5, 100, 0, 45, 66], [101, 117, 56, 122, 123, 33, 53, 42, 87, 82, 91, 84, 124, 54, 73, 15, 16, 103, 76, 78, 75, 24, 71, 27, 121, 13, 88, 68, 6, 37, 72, 114, 107, 86, 85, 104, 58, 59, 67, 66, 10, 3, 48, 65, 1, 126, 116, 25, 105, 95, 12, 109, 61, 118, 44, 89, 64, 45, 14, 92, 7, 120, 79, 115, 9, 127, 18, 29, 80, 43, 26, 98, 90, 77, 4, 94, 102, 70, 49, 74, 93, 0, 110, 11, 28, 23, 55, 19, 8, 81, 34, 99, 35, 20, 36, 100, 41, 125, 111, 52, 22, 32, 31, 17, 113, 46, 47, 83, 39, 51, 96, 21, 69, 5, 2, 112, 30, 62, 50, 63, 40, 60, 57, 38, 106, 119, 108, 97], [39, 52, 118, 33, 31, 79, 13, 87, 8, 11, 53, 70, 20, 117, 54, 82, 81, 9, 68, 63, 50, 66, 113, 26, 120, 121, 111, 125, 90, 10, 61, 114, 49, 119, 46, 23, 74, 3, 51, 112, 25, 47, 116, 84, 55, 21, 6, 29, 95, 38, 67, 69, 12, 73, 18, 5, 19, 60, 77, 17, 107, 0, 28, 93, 85, 97, 57, 44, 106, 62, 15, 59, 22, 100, 72, 2, 24, 83, 92, 91, 7, 94, 86, 80, 98, 36, 14, 115, 71, 102, 78, 75, 105, 34, 1, 16, 123, 65, 58, 110, 30, 42, 76, 89, 43, 32, 4, 27, 45, 124, 104, 40, 88, 48, 96, 56, 99, 41, 109, 127, 37, 101, 64, 108, 35, 122, 126, 103], [39, 118, 52, 33, 31, 102, 20, 87, 63, 25, 116, 11, 54, 47, 82, 81, 90, 117, 13, 85, 79, 49, 80, 123, 111, 121, 56, 86, 76, 104, 26, 50, 28, 53, 23, 91, 9, 7, 105, 120, 41, 112, 51, 71, 61, 44, 70, 83, 55, 114, 27, 57, 8, 60, 125, 107, 46, 110, 115, 106, 101, 124, 40, 97, 42, 10, 14, 59, 127, 29, 43, 78, 126, 113, 35, 45, 16, 69, 122, 74, 62, 119, 65, 22, 98, 108, 58, 36, 48, 3, 93, 17, 84, 37, 30, 12, 68, 99, 109, 88, 92, 64, 18, 67, 38, 100, 34, 66, 96, 19, 89, 21, 24, 32, 75, 72, 77, 94, 5, 95, 15, 73, 1, 6, 4, 103, 0, 2], [53, 39, 33, 52, 118, 31, 90, 50, 102, 81, 87, 113, 59, 63, 54, 125, 20, 114, 98, 61, 55, 97, 60, 117, 108, 47, 107, 26, 124, 121, 92, 49, 24, 44, 86, 112, 42, 79, 126, 110, 28, 116, 103, 57, 111, 82, 14, 43, 83, 120, 45, 119, 10, 127, 46, 29, 122, 85, 106, 19, 12, 51, 56, 41, 37, 32, 115, 96, 48, 62, 109, 22, 100, 105, 34, 88, 13, 40, 89, 58, 123, 101, 11, 94, 91, 25, 104, 93, 36, 5, 78, 27, 38, 99, 35, 17, 71, 21, 80, 23, 76, 74, 84, 30, 16, 95, 77, 75, 8, 18, 15, 72, 67, 6, 9, 7, 73, 69, 3, 4, 65, 1, 66, 70, 0, 68, 64, 2], [39, 118, 52, 33, 53, 31, 8, 20, 82, 87, 13, 79, 11, 54, 70, 63, 117, 102, 81, 74, 9, 50, 125, 68, 121, 46, 90, 86, 23, 61, 16, 49, 66, 10, 55, 59, 60, 15, 47, 42, 51, 69, 115, 67, 95, 12, 26, 111, 119, 97, 22, 72, 112, 114, 77, 120, 17, 80, 124, 21, 18, 94, 89, 116, 3, 25, 2, 45, 56, 92, 75, 84, 78, 85, 28, 0, 19, 14, 38, 29, 57, 27, 83, 30, 71, 24, 113, 96, 107, 76, 40, 7, 123, 99, 62, 122, 88, 44, 48, 36, 6, 100, 41, 5, 73, 98, 106, 43, 34, 127, 91, 64, 1, 93, 32, 105, 37, 126, 4, 108, 110, 101, 104, 35, 58, 109, 65, 103], [41, 39, 99, 116, 124, 118, 52, 95, 54, 28, 86, 92, 62, 75, 105, 81, 26, 31, 24, 21, 122, 121, 58, 17, 83, 55, 87, 127, 22, 61, 79, 56, 123, 117, 93, 50, 12, 53, 14, 115, 35, 88, 110, 106, 13, 82, 112, 113, 94, 47, 18, 34, 125, 57, 42, 120, 101, 108, 30, 48, 20, 126, 98, 32, 111, 46, 89, 43, 51, 78, 114, 96, 49, 104, 90, 15, 72, 40, 25, 109, 9, 84, 59, 38, 37, 107, 100, 63, 60, 44, 45, 33, 23, 16, 11, 119, 29, 85, 102, 36, 97, 27, 91, 74, 19, 8, 65, 68, 1, 69, 73, 76, 5, 4, 70, 77, 71, 6, 3, 10, 80, 7, 66, 64, 2, 0, 67, 103], [39, 99, 28, 116, 124, 118, 16, 52, 41, 83, 74, 13, 86, 95, 72, 54, 70, 49, 75, 71, 79, 92, 4, 26, 22, 3, 21, 35, 24, 12, 81, 1, 62, 2, 122, 23, 61, 31, 18, 55, 32, 85, 123, 87, 30, 50, 11, 73, 77, 127, 58, 89, 7, 112, 19, 48, 27, 117, 78, 126, 64, 111, 25, 88, 80, 42, 47, 66, 82, 53, 14, 121, 8, 59, 67, 100, 10, 101, 51, 5, 17, 6, 106, 94, 110, 113, 76, 0, 15, 90, 105, 34, 65, 107, 91, 104, 60, 63, 119, 69, 20, 46, 93, 120, 102, 125, 84, 43, 33, 97, 37, 96, 29, 36, 115, 40, 108, 68, 98, 103, 56, 45, 9, 57, 109, 38, 44, 114], [39, 116, 99, 124, 118, 52, 41, 95, 92, 86, 26, 31, 28, 54, 49, 83, 98, 87, 81, 112, 24, 62, 79, 75, 105, 22, 30, 35, 61, 84, 16, 34, 93, 13, 117, 88, 47, 121, 120, 58, 106, 21, 123, 114, 44, 110, 46, 104, 45, 60, 122, 14, 42, 108, 53, 96, 89, 125, 113, 25, 33, 27, 50, 127, 90, 48, 40, 109, 59, 20, 38, 111, 63, 37, 97, 126, 17, 55, 51, 91, 43, 107, 32, 119, 102, 101, 36, 100, 12, 57, 94, 72, 29, 56, 115, 74, 23, 78, 85, 103, 82, 11, 18, 15, 19, 4, 70, 1, 68, 76, 71, 73, 9, 5, 69, 8, 6, 65, 2, 64, 80, 77, 66, 7, 0, 10, 3, 67], [39, 99, 118, 124, 52, 116, 28, 95, 72, 13, 83, 4, 79, 86, 16, 22, 70, 75, 74, 66, 54, 105, 2, 68, 65, 64, 81, 35, 84, 87, 92, 61, 1, 21, 0, 26, 24, 3, 17, 115, 106, 77, 98, 55, 11, 89, 31, 30, 8, 14, 19, 71, 67, 88, 62, 110, 7, 123, 23, 73, 58, 82, 6, 94, 42, 33, 15, 117, 10, 41, 109, 12, 5, 103, 49, 122, 121, 76, 80, 127, 25, 57, 125, 56, 51, 40, 93, 48, 53, 34, 113, 78, 85, 18, 96, 69, 47, 59, 114, 46, 63, 27, 20, 90, 45, 44, 100, 29, 32, 60, 120, 111, 126, 43, 101, 91, 36, 119, 50, 37, 97, 108, 38, 112, 9, 102, 107, 104], [45, 102, 0, 97, 64, 31, 109, 25, 4, 66, 1, 65, 6, 76, 10, 124, 79, 69, 9, 68, 22, 17, 2, 11, 48, 38, 105, 126, 95, 125, 81, 51, 82, 59, 20, 56, 52, 5, 78, 104, 90, 18, 127, 84, 67, 118, 61, 7, 41, 63, 60, 122, 53, 71, 113, 74, 70, 73, 57, 75, 100, 15, 50, 120, 42, 14, 77, 13, 72, 28, 110, 12, 47, 114, 24, 49, 87, 107, 93, 19, 88, 121, 23, 33, 26, 16, 83, 36, 3, 46, 80, 112, 55, 39, 40, 85, 92, 29, 58, 115, 8, 30, 43, 101, 86, 96, 108, 106, 34, 111, 37, 91, 35, 99, 89, 21, 98, 32, 117, 119, 27, 123, 94, 103, 54, 62, 116, 44], [102, 45, 97, 31, 124, 79, 10, 17, 11, 5, 109, 25, 76, 89, 22, 70, 9, 4, 69, 6, 84, 48, 52, 51, 66, 82, 1, 78, 115, 75, 56, 65, 59, 104, 126, 53, 19, 90, 61, 72, 2, 105, 14, 87, 125, 20, 41, 77, 81, 42, 57, 71, 122, 38, 114, 15, 86, 28, 16, 120, 127, 118, 3, 60, 73, 63, 23, 74, 7, 83, 67, 0, 18, 13, 50, 123, 92, 12, 121, 95, 99, 24, 35, 68, 111, 88, 43, 39, 80, 113, 21, 34, 106, 30, 27, 29, 46, 93, 107, 117, 85, 101, 8, 110, 100, 94, 108, 26, 47, 96, 116, 98, 91, 36, 119, 58, 37, 62, 33, 112, 49, 32, 40, 44, 54, 55, 103, 64], [45, 102, 97, 31, 65, 25, 124, 109, 6, 11, 4, 1, 10, 76, 9, 79, 17, 0, 67, 69, 3, 48, 81, 66, 84, 51, 20, 105, 5, 52, 71, 126, 61, 90, 56, 15, 122, 125, 70, 72, 78, 118, 22, 95, 59, 120, 41, 23, 53, 57, 63, 38, 8, 7, 16, 82, 60, 113, 18, 73, 12, 64, 24, 30, 74, 127, 2, 14, 75, 47, 83, 80, 85, 68, 111, 86, 28, 19, 39, 21, 46, 87, 50, 49, 13, 29, 77, 104, 121, 26, 100, 88, 91, 93, 42, 27, 110, 58, 107, 117, 94, 101, 119, 114, 40, 108, 115, 96, 36, 103, 89, 116, 123, 37, 62, 92, 44, 34, 43, 55, 106, 112, 35, 98, 99, 54, 32, 33], [124, 102, 97, 45, 31, 90, 22, 52, 86, 120, 84, 109, 89, 78, 111, 104, 41, 53, 121, 13, 122, 18, 36, 33, 38, 115, 51, 114, 59, 113, 107, 63, 25, 34, 117, 57, 79, 127, 106, 46, 42, 26, 48, 44, 62, 17, 123, 126, 61, 100, 39, 54, 96, 56, 116, 21, 60, 49, 108, 103, 58, 105, 29, 43, 112, 110, 28, 99, 32, 101, 27, 24, 118, 35, 119, 50, 37, 91, 55, 88, 23, 76, 98, 47, 30, 40, 94, 125, 93, 82, 20, 11, 92, 14, 19, 83, 85, 77, 80, 16, 87, 10, 72, 9, 8, 12, 81, 15, 73, 74, 95, 5, 75, 7, 71, 70, 69, 67, 68, 1, 3, 6, 2, 65, 4, 66, 64, 0], [63, 59, 52, 104, 119, 125, 56, 122, 60, 113, 62, 123, 114, 124, 50, 61, 127, 53, 54, 57, 58, 46, 109, 118, 111, 49, 48, 126, 116, 51, 47, 55, 117, 82, 120, 44, 115, 108, 45, 121, 112, 43, 110, 79, 88, 107, 84, 106, 42, 39, 41, 80, 87, 102, 28, 90, 105, 91, 76, 85, 86, 96, 103, 19, 30, 93, 9, 38, 100, 12, 81, 14, 101, 16, 13, 37, 36, 7, 89, 73, 4, 5, 33, 40, 74, 32, 99, 83, 35, 21, 97, 8, 94, 11, 25, 10, 15, 27, 65, 77, 72, 70, 92, 69, 34, 22, 2, 78, 23, 75, 17, 98, 31, 95, 1, 67, 24, 18, 29, 71, 0, 26, 20, 3, 6, 66, 68, 64], [104, 34, 52, 63, 59, 77, 17, 90, 10, 7, 78, 87, 68, 75, 27, 20, 31, 21, 13, 118, 4, 119, 66, 79, 89, 93, 124, 71, 46, 70, 125, 14, 100, 26, 23, 82, 6, 83, 73, 84, 22, 19, 122, 86, 95, 64, 74, 85, 11, 1, 76, 113, 8, 81, 88, 15, 18, 80, 25, 111, 29, 56, 28, 60, 2, 65, 94, 67, 108, 123, 50, 72, 120, 36, 0, 69, 12, 5, 3, 91, 40, 102, 41, 30, 32, 61, 116, 39, 92, 127, 48, 9, 98, 16, 115, 62, 24, 126, 47, 99, 121, 38, 42, 110, 45, 109, 37, 49, 57, 101, 96, 97, 117, 33, 54, 103, 106, 105, 58, 35, 55, 51, 44, 43, 112, 107, 53, 114], [104, 34, 52, 63, 59, 7, 92, 71, 87, 17, 78, 119, 28, 68, 10, 91, 31, 75, 83, 125, 90, 77, 85, 94, 60, 1, 22, 3, 122, 100, 18, 4, 16, 8, 124, 26, 65, 113, 118, 23, 84, 46, 74, 56, 89, 93, 72, 123, 14, 25, 5, 13, 76, 79, 9, 81, 19, 12, 21, 6, 82, 62, 30, 116, 11, 88, 86, 70, 27, 66, 98, 102, 80, 24, 67, 120, 36, 111, 69, 40, 108, 20, 2, 64, 61, 48, 39, 73, 15, 96, 95, 117, 43, 101, 0, 38, 29, 37, 33, 50, 121, 32, 55, 35, 97, 109, 47, 103, 45, 99, 105, 106, 127, 54, 110, 57, 44, 41, 42, 49, 115, 51, 107, 112, 58, 126, 114, 53], [104, 52, 63, 59, 119, 125, 124, 60, 46, 62, 113, 118, 27, 123, 48, 39, 116, 50, 56, 40, 122, 88, 34, 32, 108, 57, 111, 109, 106, 61, 115, 47, 110, 30, 28, 51, 54, 120, 55, 43, 35, 117, 49, 114, 107, 121, 42, 41, 98, 58, 103, 105, 36, 102, 100, 79, 33, 45, 53, 37, 97, 76, 44, 38, 126, 80, 86, 101, 24, 82, 112, 127, 85, 26, 99, 94, 95, 9, 96, 84, 19, 73, 20, 91, 22, 31, 7, 25, 93, 81, 14, 13, 5, 74, 92, 4, 87, 90, 8, 89, 11, 16, 1, 12, 70, 17, 69, 23, 18, 72, 10, 2, 29, 21, 83, 65, 77, 15, 78, 75, 71, 67, 3, 0, 64, 6, 66, 68], [105, 112, 39, 34, 101, 48, 50, 111, 118, 85, 124, 53, 23, 59, 21, 125, 90, 31, 82, 119, 30, 108, 56, 54, 117, 25, 93, 87, 60, 18, 96, 63, 127, 41, 19, 22, 52, 61, 122, 28, 123, 92, 88, 51, 103, 94, 89, 47, 95, 26, 37, 86, 16, 80, 57, 110, 121, 81, 38, 102, 84, 55, 29, 49, 91, 17, 97, 32, 45, 33, 11, 106, 44, 40, 107, 27, 99, 109, 62, 58, 114, 20, 78, 42, 120, 79, 14, 100, 104, 13, 116, 126, 46, 15, 24, 115, 36, 83, 12, 76, 35, 43, 75, 113, 71, 77, 9, 6, 5, 74, 98, 10, 73, 72, 8, 69, 7, 70, 2, 66, 68, 3, 65, 67, 0, 1, 64, 4], [50, 48, 105, 124, 34, 111, 38, 112, 117, 108, 119, 90, 57, 54, 39, 106, 116, 94, 114, 120, 122, 125, 118, 109, 127, 42, 63, 110, 103, 126, 52, 51, 93, 49, 92, 45, 85, 35, 82, 23, 59, 36, 31, 60, 107, 113, 55, 46, 58, 29, 104, 87, 9, 101, 47, 61, 83, 62, 40, 121, 89, 56, 53, 99, 97, 123, 44, 80, 96, 43, 115, 28, 91, 76, 37, 100, 24, 19, 88, 102, 78, 71, 16, 14, 33, 32, 86, 77, 22, 30, 95, 26, 41, 27, 84, 25, 74, 98, 20, 79, 72, 17, 13, 21, 15, 18, 7, 68, 73, 75, 11, 6, 10, 2, 67, 8, 12, 4, 81, 69, 70, 66, 64, 3, 5, 0, 1, 65], [105, 101, 34, 112, 48, 50, 89, 31, 39, 85, 83, 124, 23, 111, 87, 80, 54, 14, 82, 76, 118, 123, 59, 25, 41, 93, 21, 119, 16, 29, 63, 125, 108, 53, 19, 95, 56, 9, 117, 103, 94, 49, 116, 84, 90, 22, 20, 60, 15, 75, 52, 122, 47, 71, 26, 24, 109, 46, 74, 127, 17, 28, 32, 35, 11, 33, 57, 12, 38, 96, 62, 102, 42, 120, 92, 91, 113, 30, 114, 77, 13, 79, 107, 10, 88, 73, 104, 70, 121, 43, 78, 97, 100, 61, 81, 18, 99, 44, 27, 72, 51, 6, 106, 58, 8, 110, 55, 5, 115, 126, 7, 86, 40, 36, 69, 98, 45, 37, 68, 67, 3, 66, 4, 2, 65, 1, 0, 64], [50, 101, 38, 105, 34, 112, 118, 59, 119, 83, 31, 85, 89, 37, 23, 111, 90, 78, 49, 61, 103, 124, 21, 20, 74, 71, 125, 52, 88, 113, 53, 57, 19, 122, 104, 92, 11, 44, 106, 82, 54, 14, 109, 48, 63, 29, 22, 123, 40, 51, 28, 96, 76, 79, 108, 8, 41, 17, 9, 87, 60, 18, 68, 102, 16, 127, 56, 107, 126, 25, 114, 42, 72, 121, 24, 84, 30, 110, 93, 13, 27, 10, 117, 100, 62, 46, 39, 116, 4, 47, 35, 15, 81, 58, 91, 2, 115, 67, 45, 97, 120, 32, 94, 69, 33, 43, 7, 86, 26, 99, 55, 80, 77, 6, 1, 64, 36, 95, 70, 66, 5, 65, 0, 3, 75, 98, 12, 73], [110, 39, 33, 31, 46, 85, 70, 16, 53, 10, 14, 72, 94, 82, 12, 75, 68, 3, 0, 2, 1, 9, 65, 66, 6, 50, 67, 8, 104, 21, 86, 13, 120, 124, 17, 22, 4, 79, 18, 83, 57, 54, 118, 63, 90, 11, 69, 58, 123, 73, 119, 51, 126, 103, 78, 7, 127, 41, 24, 20, 99, 38, 115, 87, 80, 88, 29, 121, 56, 49, 101, 71, 102, 114, 35, 76, 98, 28, 74, 81, 36, 23, 84, 108, 125, 89, 47, 19, 96, 77, 113, 107, 45, 111, 100, 64, 5, 40, 61, 48, 117, 95, 34, 92, 26, 25, 91, 42, 52, 55, 37, 122, 93, 116, 112, 27, 105, 15, 43, 62, 106, 60, 59, 30, 109, 44, 32, 97], [110, 39, 33, 31, 46, 12, 94, 10, 6, 16, 82, 4, 72, 14, 85, 50, 74, 2, 79, 53, 70, 17, 86, 67, 115, 124, 51, 69, 21, 23, 28, 24, 76, 80, 120, 22, 57, 78, 8, 63, 117, 75, 18, 123, 66, 73, 127, 65, 126, 104, 49, 84, 118, 103, 88, 56, 58, 96, 64, 48, 77, 90, 13, 54, 20, 32, 9, 68, 121, 0, 38, 122, 108, 41, 62, 7, 11, 101, 81, 83, 59, 113, 45, 43, 93, 87, 44, 55, 47, 15, 89, 112, 27, 109, 92, 119, 114, 107, 25, 116, 5, 98, 125, 19, 95, 105, 99, 60, 91, 40, 34, 36, 26, 61, 37, 1, 111, 29, 52, 71, 35, 100, 102, 106, 42, 3, 30, 97], [39, 33, 110, 53, 31, 23, 85, 46, 82, 79, 94, 16, 12, 14, 88, 19, 104, 10, 126, 57, 17, 112, 50, 70, 75, 51, 5, 63, 72, 124, 66, 115, 68, 120, 106, 64, 65, 71, 86, 81, 60, 48, 84, 55, 8, 123, 9, 125, 47, 11, 108, 56, 21, 40, 42, 3, 121, 41, 59, 24, 116, 95, 25, 49, 91, 119, 58, 38, 101, 96, 54, 22, 111, 117, 99, 122, 87, 113, 35, 18, 26, 105, 32, 34, 73, 118, 90, 127, 78, 29, 37, 100, 107, 114, 83, 27, 15, 44, 13, 45, 61, 102, 28, 109, 52, 89, 7, 80, 98, 67, 20, 93, 43, 36, 62, 74, 103, 92, 4, 2, 6, 76, 30, 0, 69, 77, 1, 97], [110, 33, 39, 31, 85, 46, 94, 24, 74, 82, 16, 14, 72, 10, 12, 86, 88, 83, 22, 53, 79, 23, 4, 13, 124, 51, 73, 90, 50, 80, 126, 5, 115, 123, 11, 120, 57, 71, 6, 78, 18, 63, 2, 70, 8, 48, 17, 75, 9, 103, 21, 67, 58, 119, 65, 118, 91, 106, 68, 56, 30, 104, 15, 69, 84, 92, 121, 87, 49, 19, 54, 40, 28, 41, 117, 77, 61, 59, 81, 125, 76, 36, 114, 20, 26, 27, 111, 99, 127, 7, 25, 62, 3, 32, 55, 60, 35, 42, 96, 112, 101, 89, 45, 43, 47, 113, 44, 93, 34, 38, 100, 102, 52, 105, 98, 108, 109, 66, 37, 116, 122, 1, 64, 95, 97, 29, 107, 0], [112, 40, 34, 53, 23, 27, 122, 29, 19, 81, 76, 21, 25, 87, 63, 31, 56, 79, 118, 119, 77, 59, 15, 93, 62, 57, 30, 16, 72, 90, 47, 9, 82, 18, 26, 11, 92, 97, 109, 125, 33, 95, 78, 101, 123, 32, 44, 96, 49, 83, 35, 105, 73, 120, 24, 51, 86, 80, 106, 121, 103, 88, 98, 55, 36, 14, 22, 52, 38, 10, 71, 46, 67, 12, 28, 89, 91, 110, 127, 111, 61, 6, 54, 37, 107, 13, 94, 60, 42, 85, 84, 17, 102, 58, 5, 41, 99, 43, 113, 126, 74, 20, 115, 124, 108, 48, 45, 100, 8, 69, 116, 7, 39, 114, 50, 70, 4, 104, 117, 68, 2, 3, 1, 75, 65, 66, 0, 64], [112, 53, 40, 38, 34, 56, 47, 119, 44, 43, 52, 63, 62, 49, 122, 117, 114, 125, 61, 116, 120, 111, 109, 45, 123, 55, 31, 23, 25, 95, 59, 60, 126, 51, 118, 29, 106, 42, 127, 57, 107, 50, 54, 121, 27, 92, 41, 58, 24, 115, 46, 16, 113, 105, 48, 30, 108, 39, 110, 124, 84, 21, 93, 82, 96, 103, 88, 101, 19, 35, 22, 100, 90, 89, 102, 37, 36, 33, 32, 18, 87, 26, 17, 97, 99, 81, 86, 28, 20, 94, 85, 80, 77, 11, 14, 98, 78, 83, 13, 15, 12, 91, 76, 3, 74, 68, 8, 9, 10, 65, 7, 6, 79, 71, 4, 72, 75, 66, 73, 70, 2, 104, 5, 0, 69, 1, 67, 64], [53, 40, 112, 34, 19, 23, 81, 27, 56, 125, 21, 77, 93, 47, 82, 57, 122, 88, 120, 58, 25, 76, 59, 11, 9, 28, 14, 118, 61, 95, 62, 67, 79, 70, 104, 30, 91, 114, 92, 31, 26, 36, 44, 80, 16, 73, 119, 107, 52, 15, 7, 72, 90, 121, 20, 29, 63, 83, 126, 6, 85, 99, 12, 116, 105, 100, 18, 86, 106, 48, 60, 38, 22, 43, 78, 110, 115, 127, 69, 24, 50, 97, 87, 46, 103, 1, 84, 35, 123, 13, 41, 17, 4, 5, 89, 94, 32, 55, 10, 54, 102, 37, 98, 96, 74, 109, 45, 42, 108, 49, 111, 124, 75, 33, 51, 39, 117, 68, 113, 101, 3, 65, 71, 8, 0, 2, 64, 66], [40, 53, 34, 112, 91, 19, 21, 29, 81, 122, 79, 73, 93, 76, 6, 56, 16, 11, 47, 125, 52, 8, 63, 70, 67, 59, 62, 65, 95, 72, 17, 119, 9, 118, 27, 126, 57, 1, 12, 38, 104, 10, 120, 74, 77, 71, 61, 7, 121, 58, 86, 85, 24, 44, 30, 75, 23, 88, 84, 4, 3, 31, 68, 15, 28, 13, 78, 32, 83, 114, 14, 80, 46, 89, 20, 82, 25, 90, 123, 92, 87, 0, 2, 94, 117, 69, 26, 66, 36, 96, 18, 22, 5, 100, 35, 99, 60, 45, 37, 54, 51, 124, 108, 97, 33, 55, 101, 105, 102, 42, 103, 41, 107, 43, 49, 127, 64, 39, 106, 109, 111, 115, 110, 50, 113, 116, 48, 98]], "model.layers.4.self_attn.k_proj": [[37, 53, 56, 123, 122, 97, 117, 106, 30, 84, 16, 27, 87, 82, 13, 26, 75, 100, 50, 6, 40, 77, 73, 83, 78, 31, 42, 72, 17, 28, 64, 86, 103, 65, 38, 108, 21, 94, 35, 120, 5, 68, 118, 60, 45, 25, 47, 3, 74, 61, 127, 101, 2, 0, 89, 126, 24, 88, 43, 29, 58, 79, 52, 96, 113, 110, 55, 51, 4, 46, 49, 125, 119, 54, 66, 33, 109, 62, 105, 104, 57, 59, 121, 115, 63, 116, 41, 91, 111, 8, 10, 12, 112, 85, 44, 107, 23, 98, 93, 71, 34, 114, 22, 48, 124, 67, 32, 99, 39, 70, 92, 76, 14, 1, 11, 7, 90, 36, 102, 18, 95, 69, 19, 81, 80, 15, 20, 9], [103, 118, 52, 87, 97, 95, 20, 0, 13, 82, 81, 79, 70, 11, 9, 10, 68, 90, 3, 8, 66, 1, 47, 28, 110, 42, 53, 14, 63, 5, 71, 121, 86, 43, 22, 127, 55, 26, 65, 113, 92, 123, 2, 120, 69, 58, 46, 74, 122, 44, 88, 109, 48, 102, 39, 56, 61, 107, 125, 115, 16, 36, 89, 67, 57, 117, 126, 54, 50, 62, 73, 64, 45, 59, 29, 80, 93, 124, 114, 106, 116, 7, 38, 78, 6, 104, 100, 25, 96, 40, 27, 60, 108, 51, 83, 77, 37, 15, 119, 112, 101, 19, 99, 85, 105, 32, 21, 91, 30, 12, 76, 72, 98, 111, 4, 94, 41, 34, 35, 18, 49, 33, 24, 23, 17, 84, 75, 31], [103, 52, 118, 124, 35, 86, 92, 83, 79, 13, 1, 74, 72, 75, 31, 0, 16, 4, 70, 81, 105, 71, 26, 3, 21, 69, 2, 117, 116, 24, 53, 28, 14, 94, 5, 39, 12, 61, 112, 123, 41, 46, 95, 121, 82, 22, 56, 113, 54, 47, 55, 106, 84, 58, 59, 99, 48, 122, 9, 89, 98, 49, 126, 115, 50, 30, 40, 96, 34, 114, 63, 45, 23, 119, 111, 60, 51, 108, 100, 125, 17, 101, 7, 42, 107, 127, 87, 36, 93, 104, 67, 62, 43, 88, 20, 102, 85, 120, 33, 109, 29, 57, 91, 37, 44, 76, 32, 97, 38, 110, 90, 27, 25, 18, 73, 77, 80, 66, 19, 78, 68, 6, 64, 15, 65, 10, 11, 8], [109, 38, 0, 95, 33, 45, 89, 124, 66, 65, 17, 6, 79, 4, 10, 76, 2, 112, 64, 9, 69, 11, 126, 60, 56, 84, 59, 115, 61, 3, 47, 50, 18, 125, 52, 7, 105, 63, 118, 68, 78, 39, 116, 111, 46, 67, 40, 113, 110, 122, 121, 57, 23, 8, 127, 58, 51, 22, 44, 86, 53, 83, 88, 13, 70, 16, 108, 104, 117, 41, 92, 5, 37, 43, 119, 99, 48, 103, 101, 42, 90, 96, 80, 54, 29, 30, 55, 71, 77, 91, 123, 120, 35, 85, 94, 32, 93, 98, 62, 100, 114, 107, 26, 75, 102, 27, 36, 106, 87, 24, 28, 73, 19, 49, 34, 82, 1, 21, 74, 72, 97, 12, 15, 14, 81, 20, 25, 31], [40, 52, 59, 63, 98, 75, 26, 95, 17, 119, 78, 116, 22, 10, 77, 20, 0, 2, 6, 4, 83, 7, 118, 29, 113, 72, 123, 124, 110, 67, 18, 60, 125, 70, 111, 3, 23, 28, 50, 122, 15, 115, 55, 56, 54, 46, 121, 57, 48, 53, 47, 49, 120, 87, 62, 51, 117, 126, 114, 58, 127, 16, 61, 89, 106, 24, 9, 109, 112, 12, 43, 42, 45, 44, 21, 5, 103, 91, 108, 65, 107, 105, 8, 64, 41, 36, 39, 11, 102, 37, 27, 92, 99, 101, 93, 84, 38, 104, 35, 33, 31, 94, 32, 14, 90, 30, 96, 86, 97, 100, 69, 85, 66, 1, 25, 19, 74, 88, 80, 82, 71, 13, 79, 34, 81, 73, 76, 68], [41, 48, 98, 50, 37, 95, 23, 89, 82, 93, 90, 118, 114, 76, 80, 85, 74, 65, 44, 83, 64, 14, 59, 6, 77, 47, 53, 19, 68, 124, 86, 16, 27, 81, 20, 8, 111, 102, 9, 78, 66, 121, 125, 17, 71, 7, 112, 67, 119, 84, 106, 39, 94, 12, 32, 54, 63, 116, 117, 88, 72, 113, 96, 26, 62, 42, 69, 110, 115, 127, 3, 87, 22, 43, 107, 104, 4, 56, 51, 122, 60, 99, 58, 52, 123, 30, 15, 49, 92, 46, 108, 11, 38, 36, 61, 79, 35, 97, 33, 120, 45, 126, 91, 18, 28, 13, 109, 103, 70, 29, 40, 5, 75, 21, 2, 100, 57, 55, 10, 24, 0, 31, 25, 73, 1, 105, 101, 34], [46, 103, 110, 97, 53, 14, 0, 85, 95, 82, 16, 75, 12, 72, 17, 10, 66, 70, 71, 68, 79, 3, 51, 1, 118, 5, 124, 63, 30, 126, 8, 114, 11, 23, 40, 9, 50, 123, 90, 120, 56, 65, 86, 117, 77, 119, 57, 81, 113, 125, 122, 58, 112, 88, 67, 18, 121, 83, 84, 19, 116, 87, 62, 47, 32, 20, 127, 29, 78, 91, 44, 89, 28, 25, 93, 6, 24, 54, 2, 69, 92, 107, 27, 37, 42, 43, 76, 105, 100, 101, 34, 15, 22, 35, 59, 45, 7, 109, 99, 102, 36, 60, 55, 98, 108, 61, 115, 26, 96, 48, 111, 49, 80, 106, 41, 33, 38, 73, 52, 13, 94, 104, 74, 64, 4, 21, 31, 39], [104, 53, 112, 98, 48, 21, 11, 81, 19, 16, 0, 27, 93, 5, 73, 77, 117, 76, 2, 79, 8, 6, 65, 59, 122, 111, 25, 62, 102, 68, 110, 69, 57, 67, 72, 30, 50, 71, 56, 3, 84, 116, 15, 52, 94, 92, 74, 125, 91, 45, 121, 119, 31, 39, 58, 63, 13, 108, 70, 44, 55, 54, 7, 46, 61, 107, 90, 51, 32, 105, 118, 126, 28, 88, 109, 66, 41, 22, 96, 10, 114, 82, 113, 106, 86, 100, 60, 115, 120, 26, 124, 99, 24, 64, 127, 97, 23, 33, 29, 38, 103, 36, 14, 78, 42, 1, 95, 47, 43, 49, 35, 9, 18, 37, 101, 123, 87, 89, 75, 12, 85, 20, 83, 17, 4, 80, 40, 34]], "model.layers.4.self_attn.qk_proj": [[52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 87, 17, 23, 11, 75, 81, 15, 38, 79, 85, 21, 28, 116, 102, 119, 40, 74, 18, 82, 42, 13, 80, 22, 10, 72, 97, 25, 64, 77, 20, 16, 6, 19, 83, 0, 12, 84, 70, 89, 91, 26, 86, 90, 78, 14, 76, 68, 125, 4, 37, 99, 61, 8, 94, 73, 9, 101, 126, 47, 65, 1, 35, 2, 98, 111, 54, 113, 66, 60, 114, 105, 51, 120, 29, 121, 92, 71, 7, 57, 108, 62, 67, 115, 127, 30, 44, 93, 5, 88, 27, 69, 3, 34, 106, 49, 24, 55, 107, 58, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 117, 122, 123, 39, 95, 103, 33, 41, 31, 87, 104, 11, 23, 17, 81, 15, 38, 116, 28, 0, 79, 40, 85, 75, 64, 21, 74, 18, 72, 10, 97, 102, 25, 89, 42, 22, 6, 70, 19, 82, 26, 20, 83, 91, 68, 77, 16, 80, 12, 13, 125, 78, 119, 1, 90, 14, 84, 37, 4, 76, 86, 98, 101, 54, 105, 94, 65, 66, 113, 2, 47, 9, 111, 121, 73, 126, 35, 8, 114, 61, 120, 108, 99, 57, 51, 62, 3, 115, 60, 29, 69, 71, 5, 67, 92, 106, 7, 43, 58, 34, 24, 93, 127, 27, 30, 49, 44, 55, 88, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 122, 117, 48, 123, 39, 103, 95, 33, 41, 87, 104, 31, 23, 11, 15, 81, 0, 38, 72, 70, 75, 85, 17, 28, 64, 18, 116, 74, 82, 10, 79, 21, 13, 80, 40, 16, 22, 6, 97, 4, 83, 25, 42, 102, 19, 77, 20, 14, 12, 91, 26, 89, 84, 65, 68, 78, 76, 1, 90, 119, 125, 54, 86, 37, 66, 105, 2, 98, 94, 99, 121, 9, 8, 101, 126, 47, 114, 73, 3, 61, 120, 35, 111, 67, 5, 113, 7, 71, 69, 93, 57, 60, 108, 92, 51, 24, 29, 62, 30, 115, 44, 34, 88, 127, 58, 55, 27, 106, 49, 107, 43, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 122, 48, 117, 39, 123, 95, 103, 33, 41, 31, 104, 87, 79, 23, 17, 11, 64, 81, 75, 72, 18, 38, 0, 70, 28, 85, 15, 21, 102, 74, 80, 97, 25, 116, 13, 40, 19, 91, 10, 83, 82, 16, 42, 6, 89, 4, 14, 22, 84, 77, 12, 86, 20, 76, 119, 78, 26, 94, 90, 98, 68, 2, 121, 65, 125, 66, 37, 101, 1, 47, 99, 105, 54, 9, 8, 114, 67, 113, 73, 3, 69, 60, 111, 120, 115, 35, 126, 61, 71, 58, 34, 29, 92, 93, 5, 88, 57, 108, 30, 44, 7, 62, 51, 24, 127, 43, 27, 106, 49, 107, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 117, 50, 122, 48, 123, 95, 103, 39, 41, 33, 104, 87, 79, 64, 11, 75, 81, 17, 31, 70, 72, 23, 0, 38, 40, 85, 6, 15, 116, 28, 18, 21, 80, 82, 10, 74, 13, 97, 16, 22, 83, 84, 4, 42, 77, 12, 19, 25, 78, 76, 102, 26, 20, 91, 2, 89, 68, 86, 8, 65, 119, 14, 66, 120, 1, 90, 98, 9, 94, 61, 125, 113, 73, 37, 126, 47, 111, 101, 99, 121, 35, 114, 7, 67, 115, 105, 51, 57, 69, 29, 5, 60, 71, 3, 93, 92, 54, 30, 34, 108, 62, 44, 58, 106, 127, 88, 27, 24, 49, 107, 55, 43, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 48, 56, 122, 117, 123, 95, 39, 103, 33, 41, 104, 81, 31, 87, 79, 11, 17, 23, 75, 0, 38, 85, 64, 15, 28, 82, 21, 40, 70, 10, 6, 72, 4, 102, 80, 18, 116, 13, 74, 97, 25, 12, 16, 77, 78, 76, 83, 20, 22, 8, 91, 84, 86, 68, 19, 89, 42, 119, 14, 90, 26, 66, 125, 47, 126, 2, 1, 73, 101, 61, 94, 65, 98, 37, 54, 99, 105, 9, 113, 35, 5, 111, 29, 121, 114, 3, 71, 120, 7, 51, 115, 34, 49, 62, 60, 67, 57, 92, 108, 69, 58, 127, 93, 107, 24, 106, 27, 30, 44, 88, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 103, 95, 33, 41, 104, 17, 31, 87, 79, 23, 81, 38, 11, 40, 85, 75, 28, 25, 82, 116, 18, 15, 21, 6, 74, 0, 42, 102, 13, 22, 84, 97, 76, 26, 78, 10, 20, 64, 14, 83, 16, 80, 70, 72, 19, 12, 89, 86, 119, 8, 68, 91, 4, 77, 90, 47, 61, 37, 101, 126, 1, 94, 125, 120, 9, 66, 2, 73, 98, 65, 121, 54, 113, 35, 29, 99, 111, 114, 57, 60, 71, 105, 7, 34, 93, 127, 51, 67, 62, 3, 58, 92, 108, 69, 115, 27, 5, 44, 30, 88, 24, 106, 55, 49, 43, 107, 96, 36, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 104, 81, 87, 23, 17, 79, 75, 31, 85, 28, 38, 6, 15, 11, 21, 40, 64, 82, 116, 74, 10, 25, 18, 97, 76, 0, 42, 16, 77, 70, 22, 19, 84, 80, 78, 8, 119, 13, 83, 89, 20, 26, 91, 102, 14, 12, 86, 68, 4, 101, 37, 125, 66, 72, 1, 90, 65, 9, 54, 111, 98, 94, 60, 35, 126, 44, 105, 73, 120, 113, 57, 2, 93, 47, 121, 61, 7, 99, 29, 51, 24, 69, 58, 92, 67, 114, 3, 27, 71, 115, 108, 34, 5, 30, 127, 62, 106, 43, 88, 49, 55, 107, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 87, 104, 23, 81, 31, 17, 11, 79, 85, 21, 75, 38, 82, 6, 15, 40, 97, 10, 64, 74, 83, 0, 28, 18, 80, 25, 8, 116, 22, 20, 16, 13, 42, 102, 70, 91, 76, 89, 78, 19, 84, 86, 77, 90, 12, 68, 54, 14, 119, 26, 125, 105, 4, 37, 98, 101, 66, 47, 72, 94, 111, 65, 1, 2, 61, 121, 99, 9, 126, 120, 73, 58, 113, 35, 34, 92, 93, 29, 57, 114, 51, 115, 7, 67, 108, 69, 62, 60, 3, 30, 71, 24, 44, 27, 106, 5, 43, 55, 88, 127, 49, 32, 107, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 103, 95, 33, 87, 41, 81, 23, 31, 104, 17, 75, 15, 21, 38, 11, 28, 85, 116, 79, 42, 18, 25, 82, 74, 40, 97, 83, 102, 0, 78, 8, 84, 16, 13, 19, 10, 90, 6, 22, 64, 12, 80, 89, 86, 26, 91, 20, 77, 70, 68, 14, 76, 119, 37, 4, 125, 101, 105, 94, 99, 54, 1, 111, 65, 47, 121, 98, 120, 72, 60, 73, 114, 126, 9, 92, 35, 66, 29, 2, 71, 61, 24, 34, 69, 7, 3, 115, 27, 113, 67, 51, 106, 57, 93, 44, 108, 5, 127, 49, 88, 58, 30, 62, 55, 107, 43, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 56, 48, 50, 122, 117, 39, 123, 95, 103, 33, 41, 104, 31, 87, 75, 11, 64, 17, 79, 8, 81, 15, 23, 6, 82, 38, 28, 10, 40, 0, 21, 116, 85, 42, 70, 80, 18, 102, 97, 13, 74, 16, 83, 68, 22, 19, 77, 12, 119, 25, 20, 84, 91, 78, 26, 4, 14, 76, 86, 89, 47, 1, 66, 2, 65, 72, 101, 99, 90, 94, 9, 73, 98, 125, 37, 126, 121, 114, 105, 35, 67, 115, 127, 61, 111, 5, 60, 120, 69, 54, 113, 71, 92, 7, 3, 57, 108, 106, 29, 58, 51, 93, 30, 62, 24, 34, 107, 43, 49, 44, 27, 88, 55, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 56, 50, 122, 48, 117, 123, 95, 39, 33, 103, 41, 104, 31, 15, 87, 23, 81, 75, 8, 17, 116, 64, 28, 70, 11, 40, 38, 85, 102, 18, 10, 6, 21, 0, 82, 16, 97, 79, 83, 42, 25, 77, 68, 13, 91, 19, 12, 20, 74, 22, 84, 80, 119, 89, 26, 125, 4, 14, 66, 76, 86, 78, 113, 2, 90, 101, 9, 61, 72, 65, 94, 47, 121, 51, 98, 73, 126, 111, 1, 37, 120, 105, 99, 54, 67, 35, 57, 7, 34, 69, 115, 60, 5, 93, 114, 92, 3, 62, 108, 55, 29, 71, 44, 30, 106, 127, 24, 58, 27, 88, 43, 49, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 50, 59, 63, 48, 56, 122, 117, 123, 39, 103, 95, 33, 41, 104, 87, 17, 81, 31, 23, 38, 40, 28, 85, 75, 15, 11, 79, 25, 116, 21, 82, 83, 97, 16, 10, 22, 0, 8, 70, 18, 102, 42, 86, 26, 74, 6, 20, 89, 12, 84, 14, 80, 13, 76, 91, 19, 119, 90, 77, 78, 64, 37, 4, 101, 94, 68, 9, 72, 125, 98, 113, 1, 65, 29, 35, 105, 111, 61, 126, 2, 47, 99, 54, 60, 66, 51, 93, 121, 120, 7, 114, 73, 34, 57, 3, 27, 92, 58, 24, 71, 44, 108, 55, 5, 115, 88, 62, 49, 69, 127, 106, 67, 30, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 117, 122, 123, 95, 39, 33, 103, 41, 87, 31, 104, 81, 17, 11, 23, 38, 75, 28, 79, 70, 15, 0, 85, 40, 64, 18, 10, 74, 8, 25, 97, 16, 22, 42, 102, 21, 82, 83, 116, 80, 86, 6, 84, 77, 78, 20, 12, 13, 14, 19, 91, 76, 90, 68, 72, 89, 119, 4, 9, 26, 1, 37, 125, 98, 47, 65, 2, 66, 101, 99, 105, 94, 51, 7, 61, 121, 54, 111, 35, 126, 73, 29, 120, 92, 113, 5, 60, 57, 108, 114, 67, 34, 115, 44, 127, 27, 71, 88, 3, 62, 93, 69, 58, 55, 24, 49, 30, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 104, 87, 31, 75, 11, 23, 17, 38, 116, 70, 15, 74, 0, 81, 79, 21, 85, 42, 64, 28, 40, 119, 82, 6, 18, 16, 97, 102, 72, 19, 8, 25, 10, 13, 83, 22, 20, 80, 89, 84, 14, 4, 68, 78, 91, 77, 12, 76, 86, 26, 2, 90, 125, 1, 94, 9, 98, 37, 66, 65, 101, 120, 47, 73, 121, 126, 54, 3, 67, 7, 111, 105, 44, 99, 35, 92, 113, 57, 5, 115, 51, 61, 60, 114, 93, 27, 69, 29, 34, 71, 30, 49, 127, 108, 62, 88, 24, 55, 58, 106, 43, 107, 36, 96, 100, 32], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 95, 39, 103, 41, 33, 104, 87, 31, 11, 75, 23, 81, 38, 79, 17, 74, 28, 15, 70, 85, 64, 40, 18, 20, 116, 97, 102, 21, 16, 0, 72, 25, 6, 13, 10, 82, 91, 19, 80, 83, 84, 89, 22, 42, 86, 12, 76, 119, 125, 77, 4, 90, 68, 78, 8, 14, 94, 101, 66, 120, 26, 54, 73, 2, 98, 37, 111, 9, 1, 35, 126, 47, 105, 65, 34, 113, 99, 61, 51, 108, 29, 57, 121, 114, 3, 115, 60, 71, 7, 92, 67, 62, 93, 69, 5, 106, 30, 127, 58, 107, 44, 24, 43, 88, 49, 55, 27, 96, 100, 36, 32], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 122, 56, 48, 117, 123, 39, 103, 95, 33, 41, 87, 23, 17, 31, 81, 15, 38, 11, 85, 79, 28, 75, 104, 72, 82, 18, 21, 74, 16, 25, 40, 14, 20, 22, 97, 6, 86, 89, 19, 10, 0, 102, 70, 84, 76, 77, 116, 91, 83, 64, 68, 13, 42, 12, 78, 80, 90, 4, 26, 125, 8, 47, 1, 9, 101, 94, 119, 105, 61, 37, 65, 98, 73, 99, 126, 113, 29, 2, 121, 66, 120, 114, 35, 111, 7, 71, 60, 34, 51, 54, 5, 58, 92, 3, 57, 24, 69, 93, 67, 88, 127, 108, 27, 115, 55, 62, 44, 30, 49, 106, 43, 96, 107, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 122, 48, 39, 123, 117, 103, 95, 33, 41, 87, 31, 17, 104, 15, 75, 81, 21, 85, 23, 72, 82, 11, 28, 38, 79, 6, 18, 16, 64, 42, 10, 40, 74, 116, 89, 83, 0, 97, 70, 20, 25, 22, 12, 19, 14, 13, 102, 125, 84, 78, 80, 76, 77, 91, 86, 68, 90, 119, 26, 4, 105, 9, 126, 37, 99, 1, 98, 94, 66, 65, 8, 101, 73, 121, 2, 51, 111, 113, 3, 54, 47, 60, 7, 120, 67, 29, 127, 114, 61, 35, 71, 93, 5, 92, 57, 115, 34, 69, 44, 49, 108, 24, 62, 27, 88, 30, 43, 58, 55, 106, 107, 32, 96, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 117, 123, 39, 95, 103, 33, 41, 31, 104, 17, 87, 75, 38, 23, 15, 0, 72, 6, 81, 79, 21, 11, 85, 116, 82, 10, 40, 97, 16, 28, 64, 89, 18, 70, 83, 19, 42, 20, 74, 102, 25, 13, 76, 14, 84, 80, 119, 22, 77, 90, 78, 68, 91, 26, 12, 4, 65, 37, 86, 101, 66, 98, 1, 8, 111, 9, 105, 73, 125, 94, 114, 2, 120, 3, 99, 35, 47, 67, 126, 121, 29, 60, 113, 51, 54, 61, 62, 34, 69, 93, 7, 30, 57, 92, 5, 71, 44, 115, 108, 88, 127, 24, 49, 27, 43, 58, 55, 106, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 17, 31, 87, 81, 23, 75, 11, 116, 104, 38, 15, 79, 28, 72, 21, 85, 74, 10, 0, 6, 40, 64, 70, 20, 18, 97, 16, 83, 19, 77, 82, 102, 90, 78, 84, 68, 25, 89, 13, 91, 76, 22, 4, 80, 42, 119, 125, 86, 12, 26, 14, 94, 65, 8, 37, 1, 98, 9, 101, 73, 66, 54, 105, 99, 120, 113, 47, 2, 121, 111, 61, 35, 71, 29, 3, 126, 7, 58, 62, 5, 69, 60, 92, 34, 51, 115, 114, 57, 93, 67, 106, 108, 24, 43, 127, 88, 44, 49, 27, 55, 30, 96, 36, 32, 107, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 56, 50, 122, 117, 48, 39, 123, 95, 33, 103, 41, 31, 87, 104, 11, 81, 79, 23, 75, 17, 38, 18, 15, 85, 6, 72, 21, 28, 97, 40, 102, 10, 83, 0, 74, 64, 119, 82, 16, 116, 91, 25, 70, 77, 89, 76, 22, 80, 4, 19, 84, 42, 13, 68, 20, 12, 121, 26, 86, 78, 90, 14, 8, 47, 99, 105, 125, 94, 65, 66, 73, 120, 9, 37, 98, 114, 61, 1, 2, 101, 35, 34, 60, 126, 113, 54, 111, 29, 51, 5, 3, 69, 57, 24, 71, 108, 115, 92, 62, 7, 106, 30, 67, 93, 127, 58, 49, 43, 44, 88, 27, 55, 107, 96, 32, 36, 100], [52, 118, 53, 124, 46, 109, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 31, 41, 104, 87, 17, 38, 79, 75, 81, 85, 23, 15, 64, 116, 28, 11, 21, 0, 70, 18, 102, 83, 97, 89, 72, 40, 19, 82, 42, 26, 10, 80, 74, 91, 16, 6, 25, 77, 14, 84, 13, 20, 78, 4, 8, 22, 76, 119, 12, 68, 90, 1, 86, 37, 65, 66, 98, 47, 125, 73, 105, 94, 101, 9, 99, 60, 113, 3, 2, 126, 54, 111, 115, 57, 114, 61, 121, 35, 92, 7, 24, 69, 29, 120, 5, 67, 62, 51, 34, 49, 27, 108, 127, 93, 43, 71, 30, 58, 44, 106, 107, 88, 55, 32, 96, 100, 36], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 122, 48, 123, 117, 39, 95, 103, 33, 41, 87, 104, 31, 81, 11, 38, 23, 15, 17, 21, 28, 40, 85, 79, 0, 116, 75, 70, 74, 10, 16, 83, 97, 18, 22, 19, 64, 89, 82, 8, 84, 25, 77, 80, 6, 20, 42, 86, 119, 12, 91, 26, 72, 76, 13, 102, 4, 90, 78, 68, 37, 14, 66, 94, 65, 98, 1, 73, 125, 105, 101, 2, 47, 9, 35, 121, 111, 61, 120, 29, 113, 99, 114, 3, 54, 126, 51, 92, 60, 69, 34, 7, 24, 93, 30, 71, 58, 115, 27, 67, 57, 106, 5, 127, 62, 49, 108, 44, 43, 88, 107, 55, 96, 32, 100, 36], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 48, 56, 122, 117, 123, 39, 95, 33, 103, 41, 31, 87, 104, 11, 81, 23, 17, 79, 38, 85, 75, 74, 40, 15, 116, 64, 28, 97, 18, 70, 8, 83, 25, 22, 21, 102, 42, 91, 80, 16, 6, 82, 90, 119, 68, 76, 13, 77, 89, 10, 19, 12, 86, 84, 26, 20, 78, 72, 14, 4, 0, 98, 37, 101, 9, 111, 65, 66, 105, 94, 54, 126, 125, 120, 99, 47, 2, 1, 73, 61, 113, 35, 121, 51, 115, 127, 29, 60, 71, 34, 57, 55, 93, 114, 69, 3, 7, 62, 92, 5, 67, 108, 106, 24, 30, 27, 88, 58, 49, 44, 107, 43, 96, 100, 36, 32], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 122, 48, 117, 123, 39, 103, 95, 33, 41, 104, 81, 87, 31, 17, 11, 23, 116, 15, 38, 75, 70, 85, 21, 18, 40, 79, 28, 82, 74, 83, 22, 84, 8, 77, 119, 42, 16, 19, 25, 20, 97, 80, 0, 6, 89, 10, 26, 76, 64, 12, 91, 102, 14, 78, 90, 86, 68, 47, 13, 4, 72, 65, 1, 125, 101, 37, 94, 113, 126, 73, 98, 54, 111, 9, 35, 2, 61, 114, 66, 71, 29, 60, 57, 120, 127, 121, 105, 93, 99, 34, 108, 51, 3, 92, 7, 24, 62, 69, 67, 55, 30, 106, 115, 88, 5, 44, 58, 27, 49, 43, 107, 96, 32, 100, 36], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 50, 59, 56, 48, 123, 117, 122, 39, 95, 103, 33, 41, 87, 31, 81, 23, 104, 75, 15, 85, 11, 79, 17, 38, 28, 8, 40, 70, 74, 116, 82, 21, 97, 25, 22, 64, 18, 83, 42, 10, 89, 16, 119, 0, 102, 19, 20, 6, 14, 91, 13, 77, 80, 84, 76, 12, 90, 78, 26, 4, 68, 101, 86, 37, 111, 125, 1, 9, 98, 61, 66, 120, 72, 73, 105, 47, 65, 2, 94, 60, 57, 35, 113, 121, 51, 99, 126, 58, 92, 54, 7, 29, 106, 34, 115, 93, 62, 127, 67, 108, 114, 24, 5, 3, 49, 69, 27, 43, 55, 44, 30, 88, 71, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 56, 48, 117, 122, 123, 39, 103, 95, 33, 41, 87, 104, 31, 11, 81, 85, 23, 15, 97, 0, 17, 28, 75, 79, 21, 8, 40, 38, 82, 102, 70, 18, 25, 119, 74, 22, 116, 16, 83, 19, 20, 80, 13, 77, 91, 10, 6, 64, 12, 14, 42, 89, 4, 26, 84, 76, 68, 90, 78, 47, 86, 99, 94, 125, 101, 37, 105, 66, 65, 72, 1, 126, 98, 73, 60, 9, 57, 35, 113, 2, 111, 61, 114, 29, 54, 121, 92, 120, 3, 67, 7, 51, 127, 69, 93, 30, 24, 71, 5, 106, 34, 27, 62, 115, 49, 44, 108, 58, 88, 107, 43, 55, 96, 36, 32, 100], [52, 118, 53, 124, 46, 109, 45, 112, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 103, 33, 41, 104, 31, 11, 17, 87, 75, 81, 0, 8, 15, 23, 79, 64, 116, 74, 40, 85, 28, 6, 38, 42, 21, 18, 20, 10, 82, 83, 97, 70, 4, 80, 76, 77, 22, 25, 102, 16, 91, 13, 14, 78, 86, 19, 12, 89, 68, 1, 119, 26, 66, 84, 113, 125, 65, 90, 47, 98, 72, 120, 94, 9, 2, 101, 37, 105, 60, 111, 73, 54, 99, 121, 35, 126, 61, 51, 67, 114, 7, 71, 34, 57, 3, 29, 69, 5, 115, 92, 127, 44, 62, 93, 58, 88, 108, 49, 55, 30, 24, 27, 106, 43, 107, 36, 96, 32, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 63, 59, 50, 56, 48, 122, 117, 123, 39, 95, 33, 103, 41, 11, 31, 104, 87, 75, 81, 116, 38, 79, 8, 23, 28, 17, 6, 15, 21, 74, 0, 85, 40, 82, 80, 64, 97, 25, 10, 42, 83, 102, 18, 19, 70, 89, 76, 91, 22, 77, 14, 119, 68, 13, 84, 16, 20, 4, 86, 12, 78, 72, 26, 90, 101, 121, 2, 125, 1, 37, 73, 9, 105, 65, 47, 66, 114, 94, 113, 98, 120, 111, 35, 99, 51, 108, 54, 34, 115, 61, 71, 3, 60, 67, 7, 5, 92, 29, 69, 126, 57, 62, 93, 58, 24, 30, 127, 106, 55, 49, 88, 44, 27, 43, 107, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 112, 45, 110, 59, 63, 50, 122, 48, 56, 117, 123, 39, 95, 33, 103, 41, 104, 31, 15, 87, 11, 75, 23, 17, 81, 6, 79, 38, 0, 28, 8, 85, 10, 18, 64, 22, 21, 82, 40, 97, 116, 16, 83, 13, 74, 25, 102, 77, 80, 70, 19, 86, 42, 76, 4, 91, 68, 78, 20, 12, 84, 90, 14, 89, 72, 26, 47, 66, 60, 1, 119, 125, 101, 73, 94, 37, 105, 2, 65, 98, 9, 126, 35, 113, 114, 61, 111, 120, 99, 121, 69, 7, 51, 62, 3, 54, 57, 29, 71, 108, 67, 5, 34, 115, 58, 93, 92, 106, 127, 30, 49, 24, 27, 44, 107, 88, 43, 55, 96, 32, 36, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 63, 50, 59, 56, 122, 48, 117, 123, 103, 39, 95, 33, 41, 87, 31, 38, 81, 17, 23, 104, 85, 75, 11, 15, 6, 28, 0, 18, 79, 40, 42, 97, 89, 21, 10, 82, 25, 64, 70, 80, 102, 86, 74, 22, 8, 14, 20, 19, 26, 90, 68, 76, 77, 83, 116, 78, 72, 16, 12, 4, 13, 91, 37, 101, 119, 65, 84, 9, 1, 125, 66, 98, 111, 113, 126, 105, 73, 94, 61, 35, 47, 120, 2, 54, 99, 121, 69, 67, 51, 7, 29, 34, 114, 115, 60, 71, 62, 57, 93, 49, 58, 92, 27, 5, 44, 3, 127, 108, 24, 30, 106, 88, 55, 43, 107, 96, 36, 32, 100], [52, 118, 53, 124, 109, 46, 45, 112, 110, 59, 63, 50, 56, 48, 122, 123, 117, 39, 95, 103, 41, 33, 104, 31, 81, 87, 23, 11, 75, 15, 17, 85, 40, 79, 38, 6, 10, 42, 82, 74, 97, 25, 72, 0, 28, 20, 89, 64, 21, 102, 70, 116, 77, 18, 16, 19, 91, 80, 119, 76, 14, 125, 101, 83, 8, 13, 78, 26, 68, 22, 12, 84, 86, 47, 4, 90, 37, 111, 98, 54, 105, 126, 9, 66, 2, 121, 1, 94, 61, 65, 73, 120, 114, 99, 60, 35, 115, 29, 71, 113, 57, 62, 51, 108, 58, 67, 3, 93, 5, 7, 34, 92, 24, 69, 44, 106, 127, 88, 30, 27, 49, 43, 96, 55, 107, 32, 36, 100]], "model.layers.5.self_attn.q_proj": [[39, 106, 98, 50, 20, 80, 42, 23, 127, 116, 14, 49, 18, 10, 124, 51, 27, 47, 74, 61, 120, 103, 54, 12, 24, 112, 87, 57, 84, 71, 25, 115, 68, 48, 63, 29, 111, 123, 16, 96, 15, 26, 76, 113, 85, 82, 83, 9, 79, 88, 125, 52, 118, 92, 114, 28, 121, 91, 122, 21, 78, 93, 41, 62, 94, 99, 8, 59, 75, 86, 58, 6, 89, 81, 126, 30, 44, 22, 90, 97, 35, 11, 95, 60, 31, 2, 45, 104, 109, 110, 119, 36, 108, 70, 117, 102, 107, 19, 33, 40, 105, 56, 38, 32, 46, 13, 53, 3, 17, 77, 37, 43, 100, 101, 55, 7, 69, 34, 73, 4, 72, 5, 1, 65, 0, 66, 67, 64], [39, 98, 106, 49, 47, 51, 127, 57, 54, 23, 48, 27, 120, 116, 44, 63, 124, 50, 111, 92, 59, 123, 126, 60, 28, 46, 121, 105, 61, 41, 42, 52, 55, 112, 115, 43, 107, 118, 58, 125, 108, 53, 36, 119, 22, 110, 62, 117, 18, 102, 45, 113, 32, 40, 82, 104, 109, 96, 100, 21, 38, 35, 114, 56, 101, 122, 99, 30, 37, 25, 80, 97, 93, 33, 95, 31, 85, 89, 29, 87, 83, 78, 94, 91, 90, 14, 26, 34, 10, 79, 84, 20, 24, 19, 88, 16, 17, 103, 86, 12, 76, 81, 4, 74, 75, 77, 73, 15, 7, 11, 67, 13, 64, 70, 69, 72, 8, 71, 9, 3, 66, 68, 5, 6, 0, 65, 1, 2], [39, 106, 98, 50, 18, 74, 20, 88, 80, 14, 12, 23, 42, 68, 103, 116, 49, 6, 47, 51, 10, 9, 54, 4, 2, 7, 91, 21, 44, 83, 61, 121, 75, 64, 124, 127, 57, 70, 120, 71, 115, 87, 72, 92, 125, 16, 73, 13, 15, 11, 48, 28, 3, 24, 76, 17, 52, 22, 78, 67, 84, 89, 111, 63, 85, 123, 82, 112, 77, 59, 126, 86, 5, 8, 58, 60, 30, 81, 79, 66, 27, 113, 25, 19, 0, 1, 26, 118, 69, 110, 33, 114, 38, 94, 105, 122, 65, 95, 32, 31, 29, 41, 97, 55, 36, 102, 109, 93, 96, 101, 119, 46, 90, 53, 99, 117, 100, 108, 43, 45, 35, 34, 37, 40, 104, 62, 107, 56], [106, 39, 50, 98, 18, 12, 80, 20, 6, 23, 14, 9, 2, 74, 42, 47, 49, 116, 103, 64, 70, 28, 127, 87, 54, 51, 57, 91, 7, 71, 124, 68, 120, 61, 75, 115, 60, 3, 10, 4, 76, 24, 78, 67, 121, 27, 48, 26, 69, 13, 1, 11, 118, 126, 63, 89, 125, 84, 19, 17, 44, 123, 82, 16, 21, 66, 111, 15, 59, 83, 112, 119, 73, 65, 22, 8, 55, 52, 5, 77, 58, 41, 0, 62, 72, 105, 46, 81, 88, 85, 110, 108, 53, 30, 107, 33, 113, 109, 40, 114, 79, 104, 101, 29, 38, 43, 35, 99, 45, 117, 96, 122, 25, 36, 100, 102, 97, 95, 93, 56, 31, 86, 94, 90, 92, 32, 37, 34], [38, 110, 126, 125, 115, 48, 72, 112, 7, 23, 5, 4, 74, 6, 76, 70, 67, 29, 20, 11, 16, 13, 46, 51, 90, 81, 75, 12, 93, 9, 78, 26, 83, 18, 73, 66, 14, 80, 68, 65, 10, 84, 2, 69, 62, 59, 117, 17, 21, 19, 79, 122, 123, 15, 121, 56, 113, 116, 120, 55, 50, 8, 105, 91, 31, 64, 63, 124, 85, 114, 95, 87, 118, 49, 61, 53, 47, 111, 32, 58, 104, 89, 109, 27, 54, 88, 119, 71, 40, 127, 41, 44, 77, 82, 96, 45, 52, 60, 35, 108, 106, 107, 43, 42, 37, 57, 94, 28, 101, 103, 24, 1, 100, 36, 22, 86, 98, 39, 34, 3, 99, 97, 33, 25, 92, 30, 102, 0], [110, 38, 126, 112, 48, 46, 115, 125, 51, 62, 17, 116, 122, 59, 123, 56, 16, 105, 50, 121, 117, 14, 124, 84, 120, 104, 85, 32, 113, 49, 26, 55, 114, 61, 13, 119, 63, 53, 102, 118, 47, 31, 111, 41, 58, 88, 109, 27, 107, 45, 29, 127, 54, 44, 60, 40, 15, 108, 93, 52, 11, 42, 74, 106, 100, 24, 43, 57, 37, 91, 96, 103, 87, 19, 36, 90, 99, 101, 39, 35, 76, 33, 98, 34, 82, 97, 28, 95, 81, 94, 9, 89, 22, 30, 21, 92, 20, 25, 23, 72, 18, 8, 86, 71, 83, 75, 80, 6, 77, 10, 79, 78, 7, 0, 68, 5, 69, 66, 3, 12, 1, 65, 73, 70, 64, 2, 4, 67], [38, 110, 48, 112, 125, 115, 126, 46, 93, 51, 78, 83, 26, 21, 11, 72, 88, 122, 32, 123, 10, 31, 7, 124, 62, 116, 79, 77, 81, 50, 95, 15, 74, 56, 5, 29, 41, 85, 75, 61, 84, 59, 121, 18, 55, 19, 86, 90, 73, 117, 17, 106, 53, 113, 27, 120, 63, 91, 40, 80, 70, 111, 127, 47, 58, 14, 6, 92, 96, 49, 109, 104, 60, 105, 114, 4, 25, 44, 23, 45, 119, 24, 118, 97, 9, 30, 107, 28, 99, 89, 37, 108, 13, 67, 35, 43, 94, 22, 39, 68, 52, 34, 100, 101, 20, 54, 87, 103, 33, 66, 98, 16, 36, 42, 76, 57, 65, 12, 82, 8, 102, 64, 71, 3, 69, 2, 0, 1], [38, 110, 112, 115, 48, 46, 125, 51, 26, 31, 27, 122, 62, 88, 50, 102, 93, 21, 116, 117, 59, 56, 123, 23, 124, 126, 120, 104, 105, 106, 32, 41, 83, 53, 40, 61, 58, 86, 101, 111, 17, 29, 100, 113, 121, 47, 60, 49, 63, 95, 78, 33, 45, 119, 35, 109, 114, 16, 22, 96, 91, 74, 54, 99, 55, 84, 36, 37, 85, 30, 52, 28, 89, 94, 43, 107, 11, 42, 108, 118, 87, 44, 34, 14, 103, 18, 72, 98, 90, 80, 127, 57, 39, 97, 92, 13, 20, 7, 6, 79, 24, 25, 9, 19, 5, 76, 75, 15, 66, 68, 10, 82, 65, 77, 12, 81, 3, 73, 1, 0, 71, 67, 70, 64, 8, 4, 2, 69], [113, 121, 86, 61, 125, 49, 57, 59, 85, 116, 52, 60, 56, 122, 22, 62, 55, 117, 53, 124, 114, 63, 127, 54, 115, 119, 51, 123, 18, 58, 118, 50, 112, 120, 110, 111, 126, 109, 48, 46, 45, 47, 44, 108, 107, 43, 106, 23, 42, 105, 41, 40, 90, 104, 39, 35, 36, 38, 94, 95, 102, 103, 87, 34, 37, 15, 96, 100, 82, 97, 98, 64, 99, 1, 30, 31, 101, 21, 91, 66, 33, 16, 28, 3, 84, 25, 93, 65, 88, 12, 92, 80, 0, 77, 14, 69, 26, 78, 13, 89, 74, 4, 75, 24, 73, 71, 29, 2, 67, 20, 83, 8, 6, 19, 17, 68, 27, 79, 9, 10, 32, 11, 5, 7, 70, 76, 81, 72], [121, 113, 61, 125, 49, 22, 57, 59, 116, 52, 56, 62, 122, 55, 60, 117, 63, 114, 53, 127, 124, 119, 115, 51, 58, 54, 123, 118, 50, 112, 120, 110, 111, 126, 25, 48, 109, 46, 90, 47, 45, 44, 64, 19, 108, 107, 106, 30, 43, 66, 3, 69, 65, 96, 105, 42, 95, 38, 41, 26, 92, 40, 39, 104, 34, 27, 4, 94, 102, 1, 100, 36, 103, 71, 35, 97, 23, 98, 9, 67, 29, 87, 99, 33, 70, 86, 32, 24, 6, 15, 2, 31, 37, 0, 101, 84, 28, 18, 93, 85, 68, 13, 8, 88, 10, 80, 91, 83, 5, 77, 76, 12, 89, 82, 11, 78, 17, 16, 79, 75, 81, 7, 20, 14, 72, 21, 73, 74], [113, 121, 61, 125, 49, 57, 86, 22, 59, 116, 52, 60, 56, 122, 117, 62, 55, 53, 63, 114, 124, 54, 119, 51, 127, 123, 58, 115, 118, 120, 21, 50, 110, 112, 111, 126, 109, 46, 48, 45, 47, 44, 83, 108, 107, 106, 87, 43, 92, 42, 105, 30, 25, 41, 23, 89, 96, 40, 39, 104, 38, 90, 20, 34, 88, 94, 36, 103, 102, 64, 35, 100, 99, 18, 24, 97, 66, 26, 19, 1, 27, 93, 95, 65, 3, 98, 69, 33, 31, 37, 28, 0, 101, 77, 4, 10, 32, 11, 29, 85, 71, 76, 67, 13, 9, 2, 81, 8, 12, 6, 68, 91, 14, 82, 80, 84, 72, 15, 70, 78, 79, 5, 75, 16, 17, 73, 74, 7], [37, 61, 125, 49, 121, 57, 113, 59, 22, 96, 62, 55, 122, 32, 52, 116, 20, 60, 127, 56, 91, 115, 88, 124, 83, 117, 30, 77, 28, 54, 119, 53, 123, 110, 51, 63, 29, 114, 58, 112, 45, 92, 94, 50, 118, 9, 46, 120, 44, 111, 47, 109, 48, 79, 126, 69, 11, 71, 89, 76, 18, 26, 108, 93, 105, 43, 66, 87, 107, 33, 41, 36, 106, 81, 64, 39, 8, 3, 82, 42, 6, 104, 34, 99, 40, 10, 65, 23, 90, 4, 27, 17, 78, 74, 24, 98, 97, 103, 102, 19, 101, 100, 84, 31, 1, 25, 38, 73, 14, 80, 35, 75, 95, 12, 68, 13, 16, 7, 15, 85, 70, 67, 21, 2, 72, 86, 0, 5], [38, 44, 115, 90, 51, 22, 84, 108, 17, 72, 79, 81, 74, 18, 29, 126, 8, 12, 127, 16, 10, 78, 33, 82, 28, 19, 89, 112, 123, 120, 36, 4, 116, 5, 124, 122, 13, 125, 71, 60, 76, 98, 103, 35, 39, 110, 49, 107, 50, 91, 62, 47, 52, 15, 67, 97, 111, 80, 42, 45, 121, 73, 118, 113, 119, 114, 105, 20, 25, 109, 63, 55, 59, 53, 96, 86, 58, 23, 41, 30, 54, 87, 61, 106, 6, 46, 40, 37, 43, 101, 117, 99, 48, 26, 31, 34, 70, 14, 100, 94, 57, 95, 32, 24, 104, 68, 21, 92, 9, 77, 65, 56, 85, 27, 88, 93, 2, 75, 11, 0, 102, 83, 7, 1, 3, 69, 66, 64], [38, 44, 115, 69, 71, 51, 79, 12, 17, 22, 18, 108, 84, 74, 81, 8, 78, 90, 2, 64, 3, 33, 5, 66, 102, 7, 68, 65, 120, 126, 1, 10, 49, 15, 67, 14, 57, 112, 70, 76, 95, 59, 125, 82, 20, 52, 23, 88, 9, 73, 19, 75, 26, 83, 56, 4, 16, 0, 13, 11, 87, 127, 119, 86, 92, 29, 98, 89, 116, 114, 124, 80, 72, 37, 6, 93, 58, 25, 61, 28, 91, 36, 110, 77, 104, 99, 41, 45, 27, 117, 35, 122, 106, 63, 21, 34, 96, 30, 85, 118, 121, 31, 62, 123, 100, 94, 24, 111, 48, 50, 60, 101, 32, 42, 39, 105, 97, 46, 40, 103, 109, 47, 55, 54, 53, 43, 113, 107], [38, 44, 115, 74, 18, 84, 79, 12, 51, 108, 69, 22, 71, 3, 8, 78, 2, 64, 70, 102, 33, 1, 17, 67, 90, 68, 76, 81, 6, 7, 15, 66, 120, 16, 28, 4, 112, 23, 82, 10, 52, 59, 126, 91, 65, 86, 88, 49, 98, 26, 73, 127, 125, 72, 92, 95, 5, 57, 56, 89, 41, 85, 83, 9, 24, 118, 114, 117, 35, 0, 13, 20, 14, 58, 46, 29, 93, 19, 25, 116, 45, 75, 123, 80, 105, 50, 21, 111, 62, 11, 31, 110, 37, 124, 40, 63, 30, 32, 42, 60, 107, 119, 104, 113, 36, 47, 94, 109, 101, 77, 48, 39, 99, 122, 61, 87, 103, 53, 121, 34, 106, 55, 100, 43, 96, 97, 54, 27], [38, 44, 115, 84, 51, 22, 18, 108, 12, 17, 8, 79, 78, 74, 16, 72, 33, 19, 70, 89, 67, 5, 68, 10, 4, 81, 24, 126, 120, 90, 71, 65, 127, 2, 92, 112, 76, 49, 82, 6, 73, 69, 125, 28, 7, 75, 13, 60, 14, 3, 80, 29, 116, 23, 25, 114, 86, 98, 35, 56, 9, 50, 57, 117, 64, 107, 20, 123, 0, 58, 52, 32, 36, 124, 103, 41, 87, 93, 88, 91, 21, 26, 15, 1, 102, 39, 96, 37, 118, 43, 11, 95, 53, 83, 30, 63, 111, 100, 110, 31, 61, 62, 101, 106, 85, 40, 104, 47, 94, 113, 55, 77, 27, 109, 45, 97, 121, 59, 46, 34, 119, 99, 48, 122, 105, 54, 42, 66], [38, 44, 34, 119, 118, 53, 56, 81, 30, 78, 23, 84, 126, 47, 11, 25, 71, 122, 5, 8, 12, 52, 79, 9, 82, 87, 93, 54, 27, 29, 61, 116, 20, 50, 19, 94, 90, 125, 86, 13, 14, 66, 55, 17, 67, 7, 75, 15, 18, 127, 28, 10, 106, 16, 51, 70, 89, 101, 4, 22, 62, 46, 77, 88, 21, 121, 26, 68, 85, 69, 102, 80, 48, 73, 3, 74, 2, 96, 49, 97, 63, 105, 1, 58, 98, 42, 0, 91, 114, 24, 83, 110, 35, 59, 72, 41, 92, 113, 76, 64, 32, 117, 33, 40, 65, 99, 36, 95, 39, 43, 123, 109, 37, 31, 103, 6, 100, 108, 120, 124, 104, 115, 60, 107, 57, 112, 111, 45], [44, 53, 38, 118, 47, 34, 27, 101, 119, 126, 116, 22, 122, 54, 58, 94, 102, 30, 79, 56, 82, 46, 114, 84, 108, 59, 41, 125, 23, 106, 57, 62, 55, 93, 103, 52, 110, 63, 26, 113, 90, 50, 109, 29, 123, 61, 115, 60, 42, 124, 51, 48, 117, 31, 40, 105, 49, 121, 43, 86, 104, 127, 112, 33, 45, 9, 120, 107, 100, 80, 98, 39, 18, 37, 24, 99, 36, 92, 111, 25, 91, 73, 96, 88, 83, 28, 3, 13, 32, 35, 76, 21, 95, 85, 97, 69, 81, 72, 71, 89, 11, 20, 14, 74, 16, 75, 19, 15, 70, 7, 12, 10, 87, 6, 77, 68, 67, 0, 65, 17, 8, 2, 4, 78, 66, 64, 5, 1], [44, 38, 119, 56, 118, 53, 34, 58, 27, 84, 30, 94, 42, 79, 101, 25, 122, 125, 21, 116, 114, 82, 41, 23, 26, 50, 9, 127, 18, 62, 117, 47, 24, 54, 93, 113, 59, 52, 92, 51, 46, 105, 99, 57, 108, 55, 86, 96, 126, 48, 61, 60, 81, 33, 29, 28, 123, 103, 39, 90, 124, 31, 110, 22, 89, 120, 88, 112, 87, 12, 106, 49, 15, 91, 102, 121, 63, 109, 13, 71, 45, 104, 40, 11, 35, 115, 83, 97, 19, 85, 100, 95, 77, 75, 36, 37, 20, 16, 111, 107, 43, 32, 80, 78, 73, 98, 3, 17, 7, 70, 10, 67, 74, 76, 0, 5, 6, 14, 2, 66, 4, 64, 69, 8, 68, 72, 65, 1], [38, 53, 118, 119, 56, 34, 12, 44, 88, 23, 25, 84, 30, 47, 27, 81, 108, 6, 8, 126, 94, 1, 18, 89, 9, 86, 87, 79, 19, 33, 67, 21, 112, 54, 15, 58, 20, 52, 105, 65, 82, 41, 42, 55, 125, 46, 16, 117, 114, 101, 102, 77, 123, 116, 11, 48, 92, 122, 31, 26, 72, 62, 80, 57, 120, 90, 39, 70, 76, 115, 83, 110, 63, 61, 50, 32, 78, 97, 37, 106, 75, 85, 29, 124, 113, 45, 95, 59, 51, 5, 103, 100, 35, 2, 104, 0, 127, 17, 43, 74, 14, 22, 28, 107, 121, 64, 60, 7, 96, 109, 40, 13, 69, 111, 49, 93, 99, 36, 68, 10, 4, 91, 24, 73, 71, 66, 3, 98], [39, 51, 50, 114, 97, 115, 113, 54, 117, 85, 124, 87, 120, 121, 58, 29, 63, 27, 24, 55, 61, 122, 126, 91, 111, 75, 92, 108, 25, 32, 112, 83, 57, 33, 26, 93, 53, 22, 56, 20, 59, 116, 79, 82, 73, 18, 17, 60, 88, 23, 49, 69, 21, 81, 14, 99, 48, 119, 77, 95, 34, 90, 80, 109, 16, 38, 40, 72, 41, 28, 30, 127, 107, 84, 7, 94, 78, 71, 52, 123, 15, 43, 76, 89, 100, 118, 110, 45, 125, 47, 44, 42, 86, 96, 104, 36, 98, 37, 106, 46, 102, 101, 31, 35, 105, 9, 12, 62, 11, 13, 74, 3, 5, 67, 70, 19, 8, 6, 66, 2, 4, 1, 0, 68, 10, 103, 64, 65], [51, 39, 114, 113, 50, 121, 97, 116, 61, 53, 59, 124, 60, 54, 122, 29, 120, 118, 87, 52, 105, 45, 125, 24, 56, 115, 112, 86, 63, 123, 103, 58, 26, 88, 27, 119, 109, 83, 25, 110, 107, 95, 62, 46, 85, 117, 40, 111, 92, 55, 33, 57, 44, 18, 82, 43, 127, 94, 126, 49, 98, 108, 47, 93, 106, 20, 42, 90, 41, 34, 35, 75, 23, 102, 101, 96, 38, 48, 15, 100, 89, 36, 14, 17, 37, 99, 104, 91, 19, 80, 73, 31, 32, 77, 28, 84, 12, 21, 22, 81, 30, 16, 72, 74, 70, 69, 78, 11, 0, 13, 68, 67, 64, 66, 79, 76, 1, 3, 71, 9, 6, 2, 5, 65, 8, 7, 10, 4], [39, 114, 51, 50, 97, 121, 122, 11, 9, 21, 87, 29, 36, 116, 83, 34, 54, 12, 30, 98, 14, 92, 6, 48, 19, 109, 110, 8, 119, 10, 62, 94, 53, 5, 113, 126, 35, 63, 15, 42, 13, 28, 55, 71, 47, 59, 67, 127, 57, 76, 111, 108, 58, 117, 70, 101, 43, 2, 16, 91, 79, 118, 93, 115, 68, 40, 125, 100, 4, 120, 52, 32, 60, 45, 106, 73, 102, 74, 112, 123, 56, 23, 105, 41, 46, 49, 95, 38, 89, 72, 96, 81, 44, 65, 90, 104, 99, 37, 22, 61, 107, 82, 0, 84, 18, 124, 86, 26, 85, 20, 75, 27, 78, 31, 17, 25, 88, 103, 33, 24, 80, 77, 1, 7, 69, 3, 64, 66], [39, 51, 114, 50, 97, 113, 54, 122, 121, 85, 24, 61, 87, 63, 92, 29, 48, 116, 124, 83, 95, 25, 117, 57, 27, 34, 26, 80, 126, 40, 82, 28, 111, 120, 20, 119, 33, 53, 110, 58, 91, 60, 77, 14, 17, 75, 108, 32, 15, 55, 47, 56, 73, 42, 21, 115, 89, 59, 23, 62, 88, 125, 107, 127, 19, 36, 98, 86, 74, 104, 49, 94, 81, 90, 52, 118, 18, 41, 38, 78, 101, 79, 105, 106, 22, 84, 72, 30, 43, 123, 35, 112, 13, 12, 44, 37, 76, 11, 102, 109, 93, 100, 70, 45, 99, 69, 16, 96, 31, 46, 67, 7, 68, 71, 9, 0, 10, 8, 1, 6, 2, 65, 3, 4, 5, 66, 64, 103], [108, 124, 102, 122, 62, 34, 107, 23, 30, 60, 50, 35, 119, 104, 25, 120, 45, 31, 123, 114, 121, 49, 117, 61, 103, 87, 40, 106, 47, 29, 55, 44, 110, 22, 20, 28, 32, 86, 118, 52, 38, 58, 59, 41, 33, 63, 53, 92, 115, 113, 48, 88, 43, 54, 126, 96, 105, 89, 46, 99, 56, 26, 57, 111, 51, 42, 91, 116, 112, 125, 127, 109, 101, 97, 94, 37, 79, 100, 77, 21, 39, 93, 84, 95, 27, 36, 90, 24, 11, 16, 17, 13, 83, 69, 81, 18, 72, 19, 75, 8, 15, 98, 78, 5, 10, 14, 74, 3, 65, 7, 64, 0, 67, 66, 68, 85, 1, 71, 4, 12, 80, 82, 2, 73, 9, 6, 76, 70], [102, 124, 34, 108, 62, 122, 107, 23, 17, 93, 25, 21, 11, 79, 104, 77, 18, 16, 120, 30, 69, 15, 114, 96, 29, 22, 32, 10, 83, 19, 85, 126, 89, 54, 5, 20, 117, 26, 49, 40, 31, 50, 92, 87, 127, 123, 110, 73, 2, 28, 90, 86, 119, 60, 1, 81, 51, 118, 103, 24, 91, 43, 109, 35, 41, 27, 47, 44, 84, 45, 38, 36, 116, 8, 57, 74, 78, 37, 125, 100, 13, 82, 112, 63, 94, 71, 68, 53, 67, 115, 97, 106, 0, 88, 75, 14, 61, 99, 55, 101, 105, 64, 111, 58, 42, 95, 33, 39, 46, 113, 121, 56, 72, 59, 52, 65, 4, 3, 7, 80, 48, 66, 12, 6, 9, 76, 98, 70], [102, 62, 122, 34, 124, 21, 93, 18, 108, 16, 40, 23, 12, 29, 83, 90, 77, 25, 73, 85, 11, 92, 61, 76, 126, 96, 100, 8, 24, 6, 26, 44, 27, 28, 116, 49, 82, 70, 107, 22, 119, 20, 32, 120, 84, 112, 78, 5, 42, 19, 57, 53, 58, 7, 46, 71, 17, 48, 36, 79, 106, 69, 80, 95, 123, 127, 67, 88, 4, 59, 114, 75, 109, 65, 103, 2, 43, 33, 63, 50, 0, 104, 13, 99, 9, 68, 86, 110, 14, 15, 45, 97, 66, 47, 41, 113, 117, 87, 105, 10, 89, 3, 55, 72, 37, 54, 98, 51, 91, 52, 111, 1, 64, 74, 30, 94, 81, 56, 31, 121, 115, 101, 118, 35, 60, 38, 39, 125], [62, 102, 122, 108, 34, 124, 26, 120, 23, 51, 35, 123, 30, 90, 40, 61, 21, 95, 24, 29, 96, 75, 110, 48, 93, 60, 16, 54, 5, 127, 37, 45, 114, 22, 20, 39, 50, 72, 31, 18, 94, 58, 79, 107, 49, 104, 8, 36, 32, 89, 118, 69, 44, 77, 117, 86, 59, 112, 125, 25, 103, 116, 11, 57, 106, 121, 111, 87, 56, 92, 52, 97, 55, 126, 53, 83, 76, 27, 119, 101, 91, 47, 46, 28, 41, 84, 113, 63, 109, 33, 100, 88, 74, 115, 82, 105, 3, 17, 15, 42, 98, 85, 13, 81, 73, 99, 10, 43, 38, 70, 14, 12, 71, 19, 80, 9, 0, 78, 2, 1, 68, 7, 4, 66, 67, 65, 64, 6], [38, 97, 53, 117, 21, 81, 80, 75, 14, 71, 4, 87, 76, 9, 6, 66, 13, 1, 83, 24, 123, 70, 74, 68, 118, 2, 85, 19, 12, 16, 90, 72, 0, 65, 10, 73, 94, 113, 11, 46, 18, 17, 88, 3, 92, 112, 79, 106, 64, 103, 124, 119, 86, 77, 84, 82, 25, 122, 78, 69, 7, 55, 93, 27, 111, 23, 5, 115, 42, 96, 91, 89, 15, 51, 28, 41, 62, 29, 35, 30, 120, 39, 109, 22, 56, 57, 98, 26, 54, 61, 99, 121, 50, 47, 67, 48, 36, 100, 31, 59, 52, 107, 45, 125, 20, 108, 40, 58, 104, 60, 110, 101, 44, 95, 116, 127, 114, 49, 105, 43, 32, 34, 37, 63, 126, 8, 33, 102], [38, 53, 97, 117, 87, 80, 21, 75, 76, 81, 6, 14, 70, 74, 4, 71, 9, 83, 68, 123, 24, 13, 118, 2, 1, 113, 19, 94, 66, 46, 72, 73, 55, 11, 103, 92, 119, 106, 17, 85, 16, 89, 93, 23, 12, 45, 0, 88, 96, 122, 79, 77, 112, 10, 78, 29, 20, 42, 51, 91, 111, 82, 69, 64, 84, 62, 52, 41, 86, 124, 35, 15, 28, 33, 8, 65, 7, 25, 40, 60, 18, 26, 3, 36, 105, 104, 50, 90, 120, 27, 48, 121, 107, 101, 109, 30, 116, 47, 56, 5, 108, 54, 110, 63, 99, 95, 57, 125, 100, 115, 39, 59, 98, 127, 58, 67, 114, 61, 34, 31, 22, 49, 44, 37, 43, 32, 126, 102], [38, 53, 97, 117, 21, 87, 81, 76, 75, 80, 123, 14, 4, 66, 70, 6, 118, 9, 74, 24, 19, 90, 12, 7, 69, 16, 46, 92, 71, 51, 65, 82, 112, 78, 64, 103, 56, 84, 30, 106, 23, 122, 25, 110, 15, 11, 98, 119, 121, 94, 17, 113, 47, 3, 85, 83, 1, 26, 31, 125, 35, 41, 39, 111, 88, 57, 91, 115, 73, 60, 49, 42, 93, 86, 52, 37, 18, 54, 27, 50, 29, 107, 34, 124, 58, 101, 22, 44, 59, 13, 10, 20, 126, 77, 61, 96, 36, 109, 120, 105, 79, 89, 40, 116, 63, 55, 99, 95, 28, 43, 127, 108, 104, 62, 48, 2, 45, 114, 100, 32, 67, 0, 5, 72, 8, 102, 68, 33], [38, 53, 97, 117, 87, 80, 6, 76, 21, 81, 14, 75, 74, 9, 1, 19, 3, 113, 68, 8, 71, 123, 7, 5, 82, 118, 2, 72, 85, 65, 46, 18, 70, 24, 23, 122, 91, 55, 41, 83, 35, 119, 78, 79, 10, 88, 67, 105, 12, 40, 112, 33, 106, 94, 111, 28, 15, 16, 66, 69, 84, 109, 27, 64, 45, 107, 36, 50, 86, 124, 61, 13, 90, 121, 103, 93, 26, 56, 98, 120, 77, 47, 30, 60, 115, 114, 116, 96, 25, 92, 59, 95, 42, 51, 73, 20, 104, 48, 58, 89, 0, 126, 29, 52, 17, 108, 57, 44, 110, 49, 125, 37, 22, 99, 34, 43, 62, 31, 54, 63, 127, 11, 101, 32, 39, 100, 102, 4]], "model.layers.5.self_attn.k_proj": [[42, 103, 50, 34, 12, 9, 80, 23, 20, 14, 18, 114, 7, 51, 66, 111, 6, 4, 74, 52, 64, 113, 79, 49, 126, 54, 8, 61, 127, 57, 67, 112, 116, 92, 27, 65, 108, 73, 0, 88, 121, 58, 1, 21, 124, 5, 77, 120, 59, 28, 125, 26, 86, 123, 69, 46, 62, 98, 106, 38, 81, 60, 94, 105, 91, 99, 55, 47, 44, 85, 93, 22, 45, 71, 24, 70, 41, 102, 122, 109, 37, 48, 117, 119, 90, 40, 63, 118, 11, 97, 53, 43, 25, 17, 95, 115, 104, 89, 56, 32, 3, 96, 110, 30, 101, 19, 100, 35, 107, 75, 16, 72, 29, 33, 83, 13, 31, 36, 10, 84, 15, 76, 82, 87, 78, 2, 68, 39], [46, 102, 115, 112, 125, 93, 126, 23, 26, 83, 9, 79, 80, 21, 76, 81, 20, 6, 68, 24, 3, 12, 122, 15, 13, 71, 124, 50, 77, 8, 41, 66, 116, 5, 106, 91, 123, 117, 113, 31, 16, 56, 55, 62, 58, 82, 69, 53, 78, 105, 63, 104, 95, 127, 74, 108, 111, 44, 45, 109, 118, 0, 47, 51, 96, 121, 107, 52, 40, 29, 59, 48, 60, 119, 114, 75, 33, 1, 120, 72, 49, 61, 43, 37, 19, 11, 54, 42, 36, 17, 10, 22, 18, 85, 39, 35, 110, 101, 34, 97, 99, 57, 32, 100, 88, 103, 98, 65, 84, 30, 94, 14, 25, 86, 27, 89, 28, 7, 92, 73, 87, 90, 2, 70, 38, 64, 67, 4], [61, 121, 125, 101, 57, 49, 113, 22, 32, 127, 62, 115, 55, 56, 122, 117, 17, 123, 52, 84, 51, 60, 58, 99, 59, 124, 63, 119, 54, 120, 116, 93, 114, 126, 50, 112, 88, 118, 53, 30, 48, 91, 110, 109, 47, 111, 46, 45, 43, 44, 92, 24, 95, 14, 12, 102, 35, 108, 79, 42, 82, 18, 107, 39, 10, 106, 83, 40, 41, 77, 100, 38, 37, 94, 21, 20, 97, 105, 16, 87, 104, 103, 11, 85, 36, 78, 72, 89, 80, 25, 98, 9, 33, 96, 29, 34, 31, 75, 86, 15, 90, 71, 13, 28, 74, 70, 19, 4, 23, 8, 26, 81, 66, 64, 27, 69, 3, 65, 73, 5, 6, 7, 1, 0, 76, 67, 68, 2], [108, 115, 102, 12, 22, 78, 84, 74, 18, 17, 79, 44, 69, 64, 71, 8, 68, 66, 90, 3, 75, 97, 120, 73, 2, 65, 48, 116, 1, 6, 16, 126, 24, 49, 70, 29, 67, 125, 113, 51, 93, 57, 59, 114, 19, 89, 88, 9, 21, 117, 11, 85, 124, 14, 23, 92, 58, 7, 91, 83, 0, 80, 25, 81, 62, 107, 39, 118, 30, 63, 77, 27, 55, 87, 95, 53, 10, 36, 127, 56, 99, 26, 32, 34, 31, 4, 119, 112, 41, 105, 28, 103, 43, 94, 60, 45, 52, 61, 13, 121, 104, 106, 40, 37, 100, 42, 54, 96, 123, 122, 82, 98, 110, 47, 35, 46, 101, 111, 109, 50, 33, 76, 86, 72, 38, 20, 15, 5], [102, 119, 118, 56, 53, 98, 94, 27, 108, 23, 84, 81, 25, 79, 64, 12, 9, 66, 117, 58, 5, 78, 57, 54, 37, 109, 125, 42, 67, 113, 8, 82, 114, 104, 55, 52, 105, 11, 39, 126, 24, 115, 85, 71, 62, 124, 123, 46, 110, 45, 50, 61, 19, 122, 48, 63, 121, 38, 1, 107, 6, 13, 22, 127, 112, 120, 41, 80, 4, 51, 43, 49, 60, 35, 59, 47, 74, 33, 111, 40, 18, 69, 65, 99, 90, 103, 31, 10, 116, 3, 29, 36, 97, 83, 21, 44, 68, 77, 100, 106, 96, 92, 70, 86, 32, 75, 93, 0, 30, 91, 7, 95, 88, 87, 16, 28, 14, 26, 89, 76, 72, 2, 101, 17, 20, 15, 34, 73], [103, 114, 51, 33, 31, 87, 85, 79, 49, 115, 83, 77, 27, 46, 17, 24, 7, 122, 76, 78, 25, 18, 108, 29, 4, 10, 84, 98, 121, 75, 57, 6, 3, 8, 74, 86, 30, 62, 82, 55, 48, 42, 65, 113, 69, 54, 119, 111, 5, 110, 99, 109, 127, 64, 101, 45, 89, 104, 63, 9, 112, 126, 56, 91, 123, 2, 38, 93, 58, 47, 92, 100, 52, 44, 13, 66, 125, 43, 118, 72, 94, 73, 41, 102, 40, 61, 35, 106, 23, 50, 96, 105, 116, 117, 26, 59, 12, 32, 81, 120, 36, 53, 67, 15, 22, 97, 60, 37, 107, 21, 95, 34, 88, 124, 28, 16, 80, 19, 14, 20, 90, 71, 70, 11, 39, 1, 0, 68], [38, 122, 62, 29, 98, 124, 44, 16, 18, 104, 78, 21, 22, 20, 25, 79, 73, 12, 85, 19, 11, 71, 23, 24, 6, 32, 108, 90, 77, 66, 14, 81, 82, 43, 64, 92, 17, 126, 26, 46, 68, 58, 30, 48, 40, 8, 27, 9, 117, 116, 49, 65, 57, 109, 114, 119, 102, 76, 34, 61, 60, 7, 74, 51, 103, 113, 41, 110, 91, 121, 55, 93, 101, 56, 39, 63, 10, 36, 33, 50, 59, 97, 28, 4, 105, 112, 45, 15, 99, 106, 13, 100, 125, 95, 87, 84, 75, 42, 52, 54, 96, 31, 86, 3, 53, 83, 118, 88, 120, 111, 5, 72, 115, 107, 94, 47, 123, 37, 127, 80, 35, 0, 67, 89, 1, 69, 2, 70], [117, 53, 102, 33, 14, 21, 80, 87, 81, 74, 76, 0, 75, 9, 68, 6, 8, 71, 2, 83, 65, 3, 5, 55, 24, 78, 7, 82, 1, 10, 73, 67, 42, 77, 38, 118, 123, 19, 113, 27, 84, 111, 66, 72, 110, 11, 69, 62, 91, 70, 105, 86, 94, 93, 90, 35, 26, 25, 89, 103, 112, 88, 79, 92, 15, 115, 46, 124, 57, 20, 119, 122, 48, 120, 54, 31, 59, 41, 106, 56, 17, 44, 60, 4, 30, 29, 116, 61, 37, 18, 43, 22, 114, 85, 49, 98, 127, 104, 126, 40, 100, 47, 125, 45, 51, 32, 109, 12, 28, 95, 52, 96, 58, 34, 99, 107, 36, 121, 16, 63, 108, 50, 13, 101, 39, 23, 64, 97]], "model.layers.5.self_attn.qk_proj": [[53, 115, 117, 108, 51, 125, 122, 124, 114, 62, 38, 61, 119, 112, 118, 44, 102, 50, 121, 42, 46, 56, 113, 126, 49, 23, 87, 57, 103, 76, 84, 85, 78, 82, 21, 17, 12, 81, 20, 18, 86, 80, 93, 14, 16, 110, 55, 91, 116, 10, 29, 22, 127, 74, 98, 97, 120, 15, 111, 79, 54, 26, 48, 106, 59, 63, 30, 52, 60, 9, 7, 73, 11, 47, 39, 34, 123, 8, 90, 71, 104, 75, 89, 45, 83, 24, 72, 27, 19, 5, 33, 28, 88, 109, 6, 58, 107, 70, 25, 13, 69, 77, 105, 68, 94, 0, 4, 32, 40, 35, 3, 43, 67, 37, 66, 31, 41, 64, 101, 2, 95, 1, 99, 92, 36, 65, 96, 100], [53, 115, 117, 108, 51, 38, 122, 125, 124, 62, 61, 114, 112, 118, 44, 102, 119, 46, 121, 50, 42, 113, 49, 56, 126, 23, 87, 103, 57, 82, 76, 48, 84, 86, 21, 85, 93, 81, 20, 12, 116, 78, 29, 18, 80, 17, 97, 127, 14, 91, 120, 16, 22, 55, 60, 15, 98, 110, 59, 106, 10, 30, 54, 47, 26, 63, 74, 34, 39, 58, 111, 79, 90, 123, 9, 8, 73, 6, 52, 75, 33, 11, 89, 7, 107, 71, 27, 109, 24, 28, 45, 83, 104, 5, 19, 88, 105, 25, 40, 64, 94, 72, 4, 3, 68, 69, 0, 37, 13, 35, 77, 95, 41, 2, 66, 32, 96, 70, 67, 1, 92, 43, 65, 99, 31, 101, 100, 36], [53, 115, 117, 108, 51, 124, 122, 125, 62, 38, 114, 61, 44, 112, 121, 102, 118, 46, 119, 42, 50, 56, 113, 49, 23, 103, 87, 126, 82, 57, 76, 21, 84, 12, 85, 93, 20, 86, 78, 81, 14, 116, 29, 18, 17, 16, 80, 22, 97, 98, 48, 8, 91, 120, 60, 15, 10, 110, 79, 74, 106, 26, 30, 39, 127, 55, 54, 6, 90, 9, 63, 71, 11, 7, 34, 47, 59, 75, 52, 109, 73, 123, 27, 89, 88, 19, 33, 28, 5, 58, 24, 107, 111, 104, 0, 83, 77, 64, 69, 40, 68, 25, 45, 67, 4, 13, 41, 94, 66, 105, 2, 3, 37, 35, 72, 32, 1, 43, 92, 95, 70, 65, 101, 31, 99, 96, 36, 100], [53, 115, 117, 108, 51, 114, 38, 122, 124, 125, 62, 61, 118, 102, 44, 46, 121, 112, 50, 119, 42, 113, 56, 49, 126, 103, 23, 87, 21, 85, 84, 86, 82, 17, 76, 57, 14, 20, 18, 93, 12, 80, 81, 78, 16, 22, 29, 98, 79, 91, 116, 97, 127, 55, 8, 30, 26, 54, 39, 48, 34, 106, 58, 63, 10, 59, 15, 110, 120, 90, 52, 123, 74, 47, 9, 6, 60, 75, 89, 71, 83, 73, 11, 7, 64, 24, 69, 28, 33, 88, 27, 104, 111, 19, 0, 77, 109, 25, 13, 107, 45, 5, 67, 4, 72, 3, 105, 94, 40, 66, 65, 37, 68, 1, 92, 2, 41, 96, 70, 32, 95, 31, 101, 35, 43, 99, 100, 36], [53, 115, 117, 108, 51, 122, 114, 124, 62, 125, 61, 38, 118, 121, 46, 102, 44, 112, 119, 50, 42, 113, 56, 49, 23, 103, 126, 87, 57, 82, 76, 12, 78, 85, 21, 20, 17, 86, 16, 84, 93, 81, 14, 18, 120, 98, 8, 80, 29, 22, 34, 52, 10, 110, 79, 106, 116, 15, 58, 39, 91, 123, 74, 63, 127, 6, 55, 48, 59, 7, 9, 71, 30, 54, 11, 75, 90, 111, 97, 64, 47, 109, 73, 33, 60, 26, 0, 5, 89, 83, 69, 104, 19, 27, 68, 88, 4, 3, 45, 77, 13, 67, 28, 24, 1, 40, 2, 107, 72, 25, 94, 41, 70, 66, 37, 35, 65, 32, 43, 31, 105, 99, 92, 95, 101, 36, 100, 96], [53, 115, 117, 51, 122, 124, 108, 125, 114, 61, 62, 38, 118, 119, 112, 121, 50, 102, 44, 46, 42, 56, 113, 49, 126, 87, 103, 57, 23, 76, 12, 18, 21, 78, 20, 84, 86, 85, 82, 17, 81, 93, 16, 14, 80, 55, 48, 110, 22, 120, 98, 29, 91, 74, 10, 15, 34, 47, 8, 54, 30, 59, 52, 127, 79, 123, 58, 97, 106, 116, 60, 63, 7, 9, 39, 11, 73, 71, 90, 75, 26, 111, 104, 6, 83, 33, 89, 5, 88, 68, 109, 24, 4, 45, 27, 19, 69, 72, 28, 94, 2, 70, 13, 25, 0, 64, 77, 40, 107, 3, 67, 105, 43, 41, 31, 1, 95, 32, 35, 37, 66, 101, 65, 96, 92, 100, 99, 36], [53, 115, 117, 51, 124, 125, 108, 122, 114, 38, 62, 119, 118, 61, 44, 112, 102, 121, 46, 50, 49, 42, 56, 113, 126, 87, 21, 23, 57, 103, 85, 76, 17, 82, 12, 78, 14, 110, 18, 20, 84, 81, 16, 93, 86, 80, 106, 54, 120, 48, 22, 29, 55, 60, 98, 79, 97, 127, 59, 11, 10, 111, 91, 63, 74, 39, 15, 30, 58, 34, 47, 116, 45, 8, 52, 9, 90, 123, 7, 26, 73, 88, 71, 75, 33, 19, 83, 89, 104, 27, 24, 70, 109, 40, 28, 94, 25, 41, 69, 43, 68, 6, 32, 5, 107, 72, 77, 4, 13, 3, 37, 35, 67, 105, 2, 95, 64, 31, 92, 0, 66, 101, 96, 65, 1, 36, 99, 100], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 118, 119, 112, 102, 44, 46, 121, 42, 50, 49, 56, 113, 23, 87, 21, 126, 57, 103, 12, 20, 84, 85, 80, 86, 76, 17, 82, 14, 78, 18, 81, 93, 16, 54, 29, 127, 52, 79, 97, 106, 91, 15, 111, 120, 22, 110, 58, 116, 98, 55, 10, 48, 74, 39, 60, 70, 30, 34, 47, 9, 11, 63, 88, 26, 104, 45, 27, 33, 90, 75, 19, 7, 123, 8, 71, 73, 83, 24, 109, 89, 59, 28, 43, 25, 107, 5, 37, 41, 40, 72, 69, 77, 32, 13, 94, 66, 0, 35, 4, 67, 64, 3, 105, 68, 2, 92, 31, 65, 101, 1, 6, 99, 95, 36, 100, 96], [53, 115, 117, 108, 51, 122, 38, 125, 124, 62, 114, 61, 44, 118, 112, 102, 50, 119, 121, 46, 42, 56, 126, 113, 49, 23, 87, 103, 21, 57, 20, 17, 84, 85, 14, 18, 81, 76, 12, 86, 82, 93, 80, 16, 29, 22, 127, 78, 54, 120, 110, 98, 116, 55, 97, 106, 91, 48, 10, 34, 30, 111, 79, 15, 26, 70, 52, 47, 74, 39, 90, 123, 58, 59, 33, 27, 11, 75, 24, 104, 25, 88, 71, 7, 89, 72, 9, 83, 73, 8, 19, 60, 5, 45, 94, 109, 105, 63, 32, 28, 35, 13, 0, 69, 4, 64, 41, 40, 66, 77, 67, 68, 37, 107, 43, 95, 3, 31, 92, 101, 99, 2, 1, 96, 6, 65, 36, 100], [53, 115, 117, 108, 51, 122, 38, 114, 125, 62, 124, 61, 44, 118, 112, 119, 121, 102, 46, 42, 50, 113, 56, 49, 87, 23, 103, 85, 82, 17, 76, 12, 126, 21, 57, 86, 93, 18, 84, 20, 81, 14, 78, 80, 16, 29, 127, 22, 48, 15, 97, 55, 98, 79, 91, 110, 59, 52, 74, 54, 26, 30, 10, 106, 39, 60, 90, 120, 34, 47, 111, 109, 70, 9, 58, 11, 72, 89, 75, 71, 33, 116, 63, 7, 73, 83, 27, 88, 24, 104, 123, 19, 77, 94, 69, 45, 25, 105, 4, 68, 28, 40, 13, 8, 5, 43, 37, 3, 66, 0, 107, 32, 2, 35, 6, 92, 95, 67, 31, 96, 36, 64, 41, 1, 101, 100, 99, 65], [53, 115, 117, 108, 122, 125, 51, 114, 62, 38, 124, 61, 102, 118, 44, 121, 50, 112, 119, 42, 46, 56, 113, 49, 126, 23, 87, 103, 17, 18, 12, 76, 85, 78, 57, 93, 20, 21, 81, 84, 82, 14, 86, 74, 80, 110, 16, 98, 29, 22, 79, 15, 72, 91, 48, 97, 10, 54, 116, 58, 52, 30, 120, 34, 111, 47, 55, 106, 39, 127, 90, 109, 9, 11, 73, 26, 7, 75, 70, 33, 60, 63, 27, 71, 83, 59, 69, 88, 89, 5, 107, 104, 64, 4, 6, 68, 77, 0, 3, 13, 19, 123, 8, 28, 2, 65, 45, 24, 31, 94, 66, 105, 37, 25, 67, 1, 40, 95, 41, 32, 101, 43, 35, 96, 92, 100, 99, 36], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 61, 121, 102, 118, 50, 44, 112, 46, 119, 42, 56, 113, 49, 126, 87, 23, 103, 57, 76, 84, 21, 17, 85, 18, 20, 12, 14, 78, 86, 82, 93, 58, 48, 80, 81, 72, 16, 29, 98, 22, 91, 74, 34, 116, 79, 54, 30, 10, 55, 52, 120, 106, 63, 59, 15, 127, 97, 9, 111, 110, 11, 39, 109, 47, 90, 7, 89, 75, 104, 6, 71, 107, 26, 123, 68, 27, 73, 45, 70, 5, 0, 33, 19, 28, 88, 69, 60, 24, 83, 8, 64, 67, 4, 37, 77, 3, 13, 25, 66, 94, 105, 40, 1, 2, 43, 65, 35, 41, 32, 31, 95, 92, 101, 99, 100, 36, 96], [53, 115, 117, 108, 51, 122, 62, 125, 114, 38, 124, 61, 112, 44, 119, 118, 102, 46, 50, 121, 42, 49, 113, 56, 126, 87, 23, 103, 57, 84, 76, 21, 20, 85, 81, 18, 17, 86, 93, 82, 12, 16, 78, 14, 80, 58, 48, 29, 22, 59, 15, 52, 127, 54, 91, 63, 97, 106, 110, 98, 120, 111, 26, 34, 10, 39, 9, 55, 116, 79, 74, 30, 72, 47, 90, 7, 109, 6, 19, 11, 33, 75, 27, 71, 24, 89, 73, 88, 83, 104, 123, 28, 107, 25, 60, 40, 68, 5, 45, 94, 77, 4, 43, 37, 13, 32, 41, 69, 3, 35, 105, 67, 101, 66, 95, 70, 2, 0, 31, 64, 92, 36, 8, 65, 99, 100, 96, 1], [53, 115, 117, 108, 51, 122, 125, 38, 114, 62, 124, 61, 112, 118, 44, 119, 102, 121, 46, 42, 50, 49, 56, 113, 126, 23, 103, 87, 21, 76, 78, 20, 18, 12, 57, 85, 84, 17, 48, 82, 81, 80, 86, 55, 93, 110, 14, 22, 16, 54, 6, 127, 72, 79, 29, 91, 10, 15, 58, 26, 98, 74, 30, 75, 116, 9, 39, 106, 34, 123, 120, 7, 97, 90, 71, 59, 111, 52, 11, 89, 88, 63, 73, 19, 47, 60, 33, 24, 104, 83, 68, 27, 40, 28, 5, 25, 77, 69, 0, 4, 94, 105, 2, 64, 37, 66, 8, 43, 109, 45, 100, 107, 13, 32, 3, 92, 41, 101, 35, 67, 95, 36, 65, 1, 31, 96, 99, 70], [53, 115, 117, 108, 51, 122, 125, 62, 38, 124, 61, 114, 102, 118, 44, 121, 112, 119, 46, 42, 50, 56, 126, 49, 87, 23, 113, 103, 84, 57, 81, 48, 85, 18, 76, 20, 21, 12, 78, 93, 82, 17, 110, 80, 14, 22, 98, 16, 86, 10, 29, 91, 79, 6, 72, 34, 63, 15, 74, 120, 39, 55, 97, 106, 54, 30, 116, 90, 127, 71, 104, 26, 58, 7, 52, 60, 89, 27, 59, 109, 11, 75, 9, 73, 33, 123, 19, 64, 88, 83, 24, 47, 111, 37, 5, 8, 69, 68, 107, 28, 94, 25, 40, 13, 66, 77, 4, 45, 2, 0, 105, 3, 32, 43, 67, 1, 92, 41, 70, 65, 35, 96, 101, 100, 31, 95, 99, 36], [53, 115, 117, 108, 51, 125, 122, 62, 124, 114, 38, 61, 44, 118, 112, 102, 121, 119, 50, 46, 42, 113, 56, 126, 49, 87, 23, 103, 57, 20, 14, 82, 85, 21, 12, 84, 76, 17, 18, 81, 86, 63, 80, 48, 78, 93, 22, 91, 16, 98, 110, 58, 34, 15, 74, 10, 29, 106, 60, 116, 127, 79, 54, 120, 6, 55, 59, 52, 39, 30, 72, 97, 11, 104, 90, 8, 7, 9, 89, 33, 47, 73, 26, 19, 111, 109, 107, 37, 75, 71, 69, 5, 83, 28, 123, 24, 43, 88, 25, 27, 64, 67, 13, 0, 68, 4, 70, 3, 77, 40, 94, 35, 45, 105, 92, 41, 65, 31, 100, 32, 95, 2, 101, 1, 66, 36, 96, 99], [53, 115, 117, 108, 51, 125, 124, 114, 122, 38, 62, 61, 112, 119, 118, 44, 121, 46, 102, 42, 50, 49, 113, 56, 126, 23, 87, 103, 84, 85, 81, 20, 14, 21, 12, 76, 18, 82, 80, 93, 57, 86, 78, 17, 15, 22, 16, 97, 10, 116, 52, 110, 29, 98, 48, 59, 54, 55, 60, 127, 91, 106, 58, 79, 30, 9, 120, 74, 7, 90, 8, 63, 26, 33, 73, 11, 71, 34, 39, 19, 24, 111, 75, 83, 89, 109, 72, 27, 28, 88, 104, 107, 47, 105, 6, 68, 123, 5, 25, 13, 40, 77, 70, 69, 4, 37, 32, 94, 43, 45, 92, 0, 64, 2, 31, 66, 35, 101, 41, 3, 67, 95, 65, 96, 100, 99, 36, 1], [53, 115, 117, 108, 51, 122, 62, 125, 38, 124, 61, 114, 44, 118, 112, 121, 102, 119, 50, 42, 46, 23, 49, 56, 113, 126, 87, 103, 84, 21, 12, 20, 82, 81, 57, 76, 14, 78, 85, 93, 18, 86, 17, 80, 16, 48, 110, 29, 97, 22, 91, 15, 106, 98, 10, 26, 8, 79, 116, 120, 74, 59, 55, 39, 30, 90, 58, 127, 54, 34, 9, 60, 47, 71, 11, 75, 52, 70, 33, 89, 7, 27, 73, 19, 28, 123, 111, 88, 5, 63, 24, 105, 83, 25, 43, 45, 104, 77, 72, 94, 69, 107, 13, 4, 40, 64, 109, 68, 32, 6, 67, 92, 66, 37, 0, 2, 95, 96, 41, 3, 101, 31, 99, 65, 35, 36, 100, 1], [53, 115, 117, 108, 51, 125, 38, 122, 62, 114, 124, 61, 118, 44, 102, 112, 121, 119, 50, 42, 46, 113, 56, 49, 23, 126, 103, 87, 20, 21, 12, 84, 14, 76, 85, 93, 82, 18, 16, 78, 110, 80, 17, 81, 22, 57, 48, 86, 29, 98, 39, 8, 55, 58, 52, 74, 34, 79, 106, 10, 70, 63, 97, 54, 91, 15, 30, 26, 90, 127, 116, 120, 109, 9, 89, 59, 60, 11, 75, 47, 71, 104, 7, 73, 33, 111, 27, 28, 19, 83, 24, 69, 0, 88, 64, 5, 25, 68, 123, 40, 94, 67, 66, 3, 65, 32, 45, 77, 13, 37, 43, 4, 41, 31, 2, 92, 107, 105, 6, 72, 95, 100, 1, 36, 35, 99, 96, 101], [53, 115, 117, 51, 108, 125, 122, 114, 38, 124, 62, 61, 121, 118, 44, 119, 112, 102, 46, 50, 113, 42, 49, 56, 87, 23, 126, 21, 103, 84, 57, 12, 76, 18, 82, 16, 20, 81, 86, 14, 78, 17, 85, 80, 93, 22, 48, 8, 29, 91, 10, 79, 98, 74, 15, 70, 47, 39, 55, 54, 109, 110, 63, 97, 9, 11, 59, 30, 116, 58, 127, 106, 34, 90, 60, 7, 73, 71, 111, 120, 89, 75, 52, 26, 83, 107, 33, 27, 104, 24, 4, 5, 19, 69, 77, 28, 68, 88, 123, 25, 94, 45, 43, 40, 13, 0, 105, 64, 72, 2, 37, 32, 67, 66, 3, 35, 99, 41, 92, 31, 65, 1, 95, 6, 36, 100, 96, 101], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 61, 124, 44, 102, 119, 121, 118, 50, 112, 46, 42, 113, 126, 56, 49, 87, 103, 23, 82, 84, 21, 76, 78, 12, 86, 57, 93, 85, 20, 17, 18, 14, 81, 16, 110, 80, 8, 91, 97, 120, 22, 29, 54, 116, 15, 98, 30, 79, 63, 60, 48, 74, 10, 106, 34, 39, 58, 55, 70, 90, 127, 47, 52, 9, 11, 71, 26, 73, 107, 7, 111, 28, 75, 33, 19, 24, 59, 89, 104, 69, 27, 88, 83, 123, 109, 25, 37, 5, 77, 105, 68, 43, 40, 94, 4, 13, 64, 3, 72, 0, 35, 45, 31, 32, 66, 65, 67, 92, 95, 101, 6, 41, 2, 99, 96, 36, 1, 100], [53, 115, 117, 51, 108, 114, 125, 62, 122, 38, 124, 61, 44, 118, 121, 102, 112, 119, 46, 42, 50, 113, 49, 126, 87, 56, 103, 57, 21, 23, 76, 82, 84, 93, 20, 12, 86, 80, 14, 78, 85, 18, 81, 17, 120, 110, 16, 22, 29, 60, 55, 79, 48, 98, 97, 91, 63, 8, 54, 106, 15, 127, 39, 59, 116, 30, 74, 34, 10, 71, 26, 90, 52, 33, 47, 11, 9, 58, 75, 104, 19, 7, 27, 111, 89, 123, 83, 24, 45, 25, 70, 107, 73, 69, 5, 88, 13, 77, 109, 28, 94, 64, 40, 0, 68, 37, 105, 72, 6, 3, 66, 43, 67, 4, 101, 92, 65, 1, 31, 2, 32, 41, 35, 99, 96, 95, 100, 36], [53, 115, 117, 108, 51, 114, 125, 62, 38, 124, 122, 61, 112, 121, 44, 118, 46, 102, 119, 42, 50, 113, 56, 49, 87, 126, 103, 23, 57, 21, 93, 76, 20, 18, 84, 82, 86, 14, 85, 81, 17, 12, 22, 16, 80, 78, 39, 106, 98, 97, 127, 29, 15, 120, 34, 79, 91, 48, 116, 10, 30, 55, 54, 111, 74, 110, 8, 52, 58, 59, 26, 9, 11, 71, 7, 90, 33, 60, 109, 104, 63, 6, 27, 73, 25, 75, 47, 83, 89, 19, 28, 5, 24, 88, 69, 77, 107, 13, 70, 45, 72, 123, 68, 94, 64, 37, 40, 0, 32, 43, 67, 4, 35, 3, 31, 65, 101, 1, 66, 99, 2, 105, 92, 41, 95, 36, 100, 96], [53, 115, 117, 51, 108, 122, 125, 62, 38, 114, 124, 61, 102, 118, 112, 44, 119, 50, 121, 46, 42, 56, 113, 87, 49, 126, 103, 23, 21, 76, 86, 82, 18, 85, 84, 20, 93, 17, 57, 12, 14, 59, 81, 78, 16, 48, 80, 29, 91, 22, 116, 97, 110, 98, 127, 15, 30, 10, 34, 26, 106, 111, 74, 55, 6, 60, 11, 39, 120, 52, 90, 54, 79, 63, 47, 75, 58, 8, 73, 72, 7, 89, 109, 9, 24, 33, 27, 19, 71, 83, 123, 104, 25, 68, 28, 88, 40, 13, 107, 69, 5, 94, 3, 37, 43, 64, 77, 105, 4, 101, 41, 95, 45, 0, 31, 70, 35, 67, 96, 32, 100, 66, 92, 36, 2, 99, 65, 1], [53, 115, 117, 51, 108, 122, 125, 114, 62, 124, 38, 118, 61, 44, 119, 112, 102, 121, 42, 46, 50, 113, 49, 56, 126, 87, 23, 103, 21, 57, 76, 20, 85, 18, 14, 12, 17, 86, 81, 84, 82, 78, 80, 93, 59, 54, 16, 110, 22, 116, 98, 91, 120, 6, 29, 15, 63, 10, 48, 97, 106, 39, 55, 60, 79, 74, 30, 75, 127, 111, 52, 71, 58, 9, 11, 90, 26, 34, 33, 19, 72, 73, 47, 109, 27, 7, 104, 89, 88, 28, 83, 8, 24, 37, 107, 69, 25, 68, 123, 5, 43, 13, 94, 105, 40, 77, 45, 32, 67, 41, 4, 70, 2, 3, 64, 0, 35, 31, 66, 101, 95, 65, 92, 1, 100, 99, 96, 36], [53, 115, 117, 51, 108, 122, 125, 38, 114, 124, 62, 61, 118, 44, 121, 119, 102, 112, 46, 42, 50, 113, 49, 126, 56, 87, 57, 103, 23, 21, 76, 20, 82, 81, 78, 84, 18, 85, 12, 86, 14, 80, 93, 17, 54, 16, 120, 97, 29, 48, 91, 59, 98, 22, 15, 116, 10, 110, 52, 79, 127, 106, 90, 26, 74, 30, 47, 72, 60, 11, 6, 39, 55, 34, 9, 63, 58, 75, 73, 33, 111, 104, 19, 24, 7, 27, 71, 88, 89, 5, 28, 37, 123, 13, 69, 25, 107, 83, 94, 41, 40, 45, 105, 0, 77, 109, 43, 4, 2, 8, 101, 70, 32, 68, 64, 31, 3, 35, 66, 96, 65, 67, 1, 95, 92, 99, 100, 36], [53, 115, 117, 108, 51, 122, 38, 125, 62, 124, 61, 114, 44, 118, 112, 102, 50, 121, 119, 42, 46, 126, 56, 113, 23, 49, 103, 87, 21, 57, 17, 85, 20, 18, 84, 12, 82, 93, 76, 81, 86, 80, 78, 14, 22, 110, 29, 97, 55, 48, 16, 98, 106, 10, 91, 72, 26, 127, 47, 79, 30, 116, 39, 15, 59, 34, 74, 58, 90, 120, 111, 54, 123, 73, 33, 60, 52, 7, 89, 75, 9, 11, 19, 63, 104, 6, 27, 25, 69, 83, 24, 88, 71, 94, 28, 109, 5, 45, 77, 4, 105, 64, 32, 68, 40, 35, 70, 13, 67, 8, 107, 31, 37, 0, 3, 66, 41, 101, 96, 95, 92, 2, 65, 43, 99, 1, 36, 100], [53, 115, 117, 108, 51, 122, 125, 124, 62, 38, 114, 61, 118, 121, 44, 119, 112, 102, 46, 42, 50, 126, 49, 56, 113, 87, 23, 103, 12, 21, 81, 14, 76, 18, 110, 84, 78, 57, 82, 17, 20, 86, 85, 93, 16, 22, 80, 72, 10, 98, 127, 48, 74, 58, 106, 29, 120, 116, 63, 91, 7, 15, 47, 9, 26, 30, 79, 75, 71, 55, 97, 34, 11, 39, 59, 73, 90, 54, 89, 52, 109, 19, 33, 70, 123, 45, 111, 24, 6, 69, 5, 60, 28, 27, 83, 68, 77, 4, 0, 64, 104, 40, 88, 3, 67, 13, 107, 66, 94, 2, 8, 25, 65, 37, 105, 35, 32, 43, 41, 1, 31, 92, 96, 95, 101, 99, 100, 36], [53, 115, 117, 51, 108, 122, 125, 114, 124, 62, 38, 61, 118, 44, 121, 119, 102, 112, 50, 42, 46, 56, 113, 126, 49, 87, 23, 103, 12, 57, 21, 18, 84, 78, 76, 20, 86, 82, 14, 110, 85, 93, 81, 48, 16, 80, 17, 22, 72, 58, 98, 60, 106, 91, 116, 54, 10, 29, 79, 55, 127, 74, 120, 97, 34, 30, 39, 15, 63, 59, 70, 52, 33, 47, 90, 73, 75, 26, 9, 11, 104, 111, 71, 7, 89, 109, 123, 5, 19, 37, 88, 69, 24, 28, 45, 27, 35, 68, 13, 83, 40, 4, 25, 0, 64, 105, 8, 6, 2, 77, 43, 94, 66, 107, 3, 67, 1, 95, 65, 32, 31, 100, 96, 41, 99, 36, 101, 92], [53, 115, 117, 51, 108, 125, 122, 114, 124, 62, 38, 61, 119, 118, 44, 112, 50, 121, 102, 46, 126, 42, 113, 49, 56, 87, 23, 57, 103, 85, 12, 21, 76, 20, 82, 18, 78, 14, 17, 86, 93, 84, 16, 59, 80, 81, 116, 54, 98, 22, 29, 55, 58, 47, 72, 70, 91, 120, 15, 110, 10, 74, 63, 79, 48, 106, 127, 34, 9, 7, 30, 97, 52, 39, 75, 11, 60, 71, 111, 123, 73, 90, 26, 33, 69, 89, 24, 28, 5, 109, 19, 4, 40, 67, 27, 83, 64, 88, 37, 104, 68, 45, 25, 94, 0, 3, 8, 105, 77, 13, 107, 66, 2, 43, 1, 32, 65, 35, 41, 6, 31, 100, 95, 101, 96, 92, 99, 36], [53, 115, 117, 108, 51, 125, 122, 124, 38, 62, 114, 119, 61, 44, 118, 112, 121, 46, 102, 50, 42, 49, 113, 126, 56, 87, 103, 23, 21, 57, 84, 85, 12, 18, 82, 76, 20, 93, 81, 78, 17, 14, 59, 86, 80, 22, 16, 98, 116, 106, 74, 79, 47, 54, 55, 127, 29, 110, 48, 120, 39, 15, 70, 63, 72, 60, 30, 26, 97, 11, 91, 10, 73, 9, 7, 34, 75, 52, 58, 19, 111, 71, 90, 33, 45, 69, 4, 109, 68, 123, 104, 27, 89, 28, 83, 24, 37, 8, 40, 13, 88, 25, 5, 66, 0, 107, 105, 67, 3, 64, 94, 77, 6, 2, 43, 41, 32, 31, 1, 35, 65, 95, 92, 101, 96, 36, 100, 99], [53, 115, 117, 108, 51, 122, 125, 124, 38, 114, 62, 61, 119, 44, 118, 50, 112, 121, 102, 46, 42, 49, 113, 56, 126, 87, 23, 103, 21, 76, 57, 54, 14, 85, 20, 81, 78, 18, 84, 82, 127, 86, 93, 12, 17, 80, 55, 60, 59, 16, 22, 34, 63, 116, 74, 110, 29, 10, 26, 120, 48, 91, 98, 70, 97, 79, 30, 15, 123, 73, 47, 11, 111, 39, 9, 75, 33, 52, 72, 106, 109, 7, 71, 24, 19, 89, 90, 88, 8, 105, 5, 104, 58, 27, 43, 69, 83, 28, 45, 37, 4, 25, 68, 77, 64, 6, 94, 40, 107, 13, 0, 32, 3, 66, 41, 101, 1, 92, 67, 2, 95, 31, 96, 100, 35, 65, 36, 99]], "model.layers.6.self_attn.q_proj": [[108, 55, 36, 96, 91, 23, 44, 84, 77, 81, 15, 86, 10, 16, 7, 114, 57, 75, 78, 32, 30, 28, 111, 89, 6, 25, 87, 29, 66, 70, 11, 69, 68, 22, 101, 13, 74, 72, 85, 19, 95, 17, 92, 26, 90, 51, 88, 79, 121, 14, 40, 18, 3, 9, 20, 61, 27, 76, 126, 4, 1, 54, 123, 80, 71, 107, 97, 103, 12, 64, 94, 118, 99, 102, 83, 33, 21, 39, 93, 73, 49, 58, 60, 24, 82, 125, 41, 31, 34, 98, 112, 35, 110, 48, 37, 104, 56, 5, 115, 109, 53, 38, 120, 50, 124, 2, 106, 59, 113, 47, 105, 127, 8, 46, 52, 122, 63, 42, 65, 43, 119, 116, 117, 62, 45, 67, 100, 0], [108, 55, 36, 23, 44, 84, 15, 75, 91, 81, 77, 10, 8, 96, 29, 114, 3, 72, 70, 74, 6, 64, 111, 11, 67, 86, 69, 30, 57, 2, 79, 89, 83, 27, 4, 26, 25, 65, 94, 28, 73, 18, 103, 0, 14, 5, 82, 17, 9, 87, 13, 31, 68, 85, 12, 93, 21, 1, 88, 20, 16, 80, 112, 78, 51, 7, 22, 95, 39, 32, 24, 19, 66, 97, 76, 90, 71, 33, 118, 121, 56, 123, 35, 92, 54, 98, 34, 107, 102, 48, 40, 52, 127, 41, 99, 37, 61, 59, 119, 120, 110, 49, 115, 125, 101, 104, 122, 60, 58, 100, 38, 126, 46, 109, 47, 43, 105, 124, 63, 53, 62, 106, 45, 113, 116, 42, 50, 117], [55, 108, 112, 39, 127, 36, 111, 61, 44, 54, 51, 124, 113, 118, 60, 121, 50, 59, 122, 63, 56, 58, 119, 115, 101, 116, 123, 120, 97, 62, 53, 42, 92, 26, 117, 19, 47, 43, 48, 40, 49, 94, 98, 52, 32, 90, 46, 109, 106, 110, 105, 96, 83, 45, 114, 104, 107, 41, 38, 35, 95, 88, 125, 102, 99, 33, 86, 30, 34, 23, 37, 103, 126, 29, 91, 100, 57, 78, 25, 89, 16, 93, 27, 28, 24, 31, 73, 21, 85, 81, 22, 18, 7, 14, 82, 15, 84, 75, 87, 65, 2, 77, 9, 12, 66, 11, 17, 10, 6, 4, 68, 67, 80, 76, 0, 64, 71, 5, 70, 8, 72, 79, 13, 1, 20, 69, 74, 3], [108, 55, 39, 44, 36, 94, 103, 126, 111, 90, 102, 114, 96, 57, 34, 121, 127, 29, 32, 83, 51, 54, 123, 91, 86, 60, 58, 63, 0, 19, 61, 59, 2, 110, 115, 49, 23, 125, 112, 52, 113, 98, 104, 56, 50, 38, 37, 107, 31, 42, 124, 47, 101, 106, 40, 78, 82, 117, 97, 45, 120, 109, 16, 89, 122, 92, 105, 119, 116, 33, 48, 53, 15, 10, 77, 118, 62, 46, 35, 84, 28, 43, 81, 70, 7, 41, 67, 88, 4, 68, 99, 65, 26, 95, 93, 100, 30, 5, 71, 64, 85, 25, 76, 21, 11, 75, 24, 72, 8, 12, 18, 66, 9, 73, 27, 80, 1, 22, 14, 6, 87, 69, 17, 79, 13, 3, 74, 20], [42, 102, 46, 32, 103, 89, 56, 20, 106, 76, 22, 17, 14, 19, 71, 127, 10, 110, 28, 93, 8, 120, 26, 51, 80, 60, 116, 29, 66, 50, 11, 27, 96, 38, 82, 25, 47, 123, 69, 36, 109, 63, 3, 59, 34, 118, 13, 113, 105, 79, 6, 88, 54, 33, 21, 41, 67, 52, 83, 30, 75, 81, 119, 15, 35, 77, 12, 7, 91, 86, 2, 114, 58, 112, 78, 61, 94, 84, 90, 5, 23, 16, 125, 85, 98, 101, 18, 62, 87, 70, 95, 97, 92, 74, 122, 49, 111, 4, 126, 24, 31, 9, 64, 99, 73, 72, 55, 43, 115, 57, 107, 108, 121, 44, 45, 37, 40, 124, 68, 0, 117, 53, 100, 104, 48, 39, 65, 1], [42, 41, 46, 103, 23, 32, 106, 102, 80, 92, 56, 8, 20, 26, 24, 89, 123, 29, 17, 110, 105, 96, 127, 19, 22, 116, 120, 25, 93, 28, 51, 52, 14, 109, 121, 60, 76, 27, 83, 54, 10, 69, 107, 47, 44, 49, 119, 86, 100, 61, 111, 35, 34, 58, 97, 5, 85, 91, 112, 122, 50, 63, 18, 104, 31, 39, 43, 59, 4, 9, 95, 66, 65, 117, 115, 15, 72, 124, 90, 68, 3, 126, 11, 62, 118, 48, 55, 101, 30, 98, 33, 21, 73, 108, 87, 114, 57, 53, 36, 45, 38, 94, 125, 99, 79, 37, 2, 40, 1, 77, 113, 82, 88, 71, 16, 78, 75, 81, 0, 70, 84, 6, 64, 13, 67, 12, 74, 7], [102, 46, 42, 56, 51, 116, 127, 110, 26, 123, 41, 47, 60, 50, 32, 59, 122, 120, 103, 63, 22, 111, 61, 90, 52, 19, 109, 33, 112, 27, 121, 115, 55, 49, 96, 34, 54, 118, 93, 119, 117, 43, 113, 53, 89, 57, 62, 48, 124, 16, 58, 39, 107, 126, 125, 114, 44, 29, 105, 31, 108, 101, 97, 45, 85, 79, 104, 36, 35, 98, 99, 38, 40, 24, 11, 25, 100, 37, 95, 94, 92, 106, 87, 28, 88, 20, 23, 91, 30, 83, 13, 80, 72, 86, 18, 15, 84, 17, 21, 14, 75, 73, 12, 8, 82, 69, 77, 9, 66, 5, 76, 78, 2, 81, 6, 70, 4, 74, 1, 68, 71, 10, 7, 65, 3, 0, 64, 67], [42, 46, 102, 32, 20, 17, 89, 76, 103, 22, 106, 10, 14, 71, 96, 29, 110, 69, 127, 56, 26, 51, 25, 66, 38, 11, 120, 93, 19, 5, 60, 24, 116, 79, 41, 61, 63, 119, 81, 78, 6, 109, 8, 118, 87, 82, 12, 58, 13, 75, 83, 2, 80, 23, 3, 86, 55, 15, 52, 30, 97, 54, 27, 113, 88, 59, 21, 16, 47, 28, 84, 77, 68, 85, 91, 90, 122, 35, 125, 0, 18, 72, 105, 111, 115, 50, 95, 74, 101, 70, 114, 7, 65, 112, 92, 73, 126, 99, 94, 67, 36, 43, 34, 49, 33, 98, 31, 107, 4, 37, 64, 1, 9, 121, 48, 100, 45, 123, 108, 104, 62, 44, 40, 117, 124, 39, 53, 57], [109, 103, 34, 45, 4, 20, 74, 71, 1, 18, 9, 14, 12, 0, 80, 88, 3, 66, 112, 70, 68, 79, 72, 39, 75, 67, 85, 10, 21, 7, 19, 5, 69, 13, 124, 77, 6, 16, 65, 23, 84, 11, 64, 76, 60, 8, 24, 73, 15, 2, 98, 87, 51, 22, 48, 82, 78, 83, 52, 81, 25, 86, 17, 108, 59, 57, 27, 63, 49, 55, 113, 89, 91, 121, 126, 123, 96, 110, 120, 92, 61, 114, 111, 30, 62, 90, 94, 119, 97, 28, 127, 54, 29, 41, 115, 95, 125, 93, 116, 44, 31, 58, 33, 38, 42, 26, 105, 106, 46, 107, 32, 40, 99, 122, 36, 35, 43, 56, 104, 50, 101, 100, 102, 117, 37, 118, 47, 53], [103, 109, 34, 113, 45, 88, 84, 24, 16, 76, 92, 11, 124, 8, 15, 82, 83, 73, 10, 60, 22, 127, 112, 7, 78, 18, 119, 63, 48, 57, 51, 52, 49, 62, 19, 123, 91, 114, 89, 56, 6, 115, 108, 53, 90, 120, 104, 94, 28, 9, 95, 110, 86, 26, 29, 111, 93, 97, 35, 126, 59, 122, 30, 54, 121, 117, 27, 50, 58, 96, 125, 118, 100, 31, 61, 101, 116, 32, 55, 20, 105, 33, 44, 46, 43, 42, 37, 38, 40, 99, 36, 41, 106, 25, 12, 107, 102, 47, 87, 77, 21, 13, 23, 69, 79, 75, 81, 17, 74, 72, 14, 85, 68, 98, 80, 71, 70, 66, 4, 5, 39, 67, 2, 65, 3, 1, 0, 64], [109, 103, 34, 45, 20, 74, 22, 18, 19, 4, 12, 14, 39, 67, 71, 79, 10, 72, 80, 68, 73, 1, 5, 3, 9, 13, 112, 88, 7, 66, 2, 77, 75, 0, 23, 85, 64, 70, 76, 21, 16, 15, 27, 69, 60, 92, 82, 8, 78, 65, 84, 11, 83, 81, 48, 6, 120, 123, 24, 124, 52, 63, 58, 91, 113, 51, 86, 55, 87, 17, 114, 41, 57, 25, 98, 89, 119, 90, 31, 49, 59, 126, 97, 127, 108, 94, 42, 29, 62, 28, 30, 56, 115, 96, 36, 61, 53, 105, 33, 106, 95, 32, 40, 46, 125, 93, 111, 122, 110, 47, 99, 118, 26, 107, 121, 102, 38, 100, 35, 104, 101, 54, 37, 50, 117, 44, 43, 116], [109, 103, 34, 45, 22, 80, 14, 20, 4, 18, 24, 74, 70, 75, 88, 19, 72, 112, 13, 12, 124, 89, 68, 10, 9, 60, 71, 63, 51, 73, 15, 66, 39, 76, 2, 8, 48, 52, 82, 78, 113, 77, 16, 83, 84, 49, 87, 6, 62, 127, 23, 57, 81, 11, 121, 5, 59, 79, 1, 17, 91, 0, 64, 25, 85, 92, 86, 94, 106, 7, 97, 21, 123, 27, 31, 90, 38, 28, 61, 110, 67, 69, 119, 36, 104, 126, 114, 99, 96, 3, 101, 111, 108, 95, 32, 41, 116, 40, 102, 55, 26, 120, 118, 98, 56, 50, 54, 100, 53, 30, 58, 35, 47, 93, 29, 115, 33, 125, 44, 107, 37, 117, 43, 46, 122, 65, 105, 42], [38, 124, 49, 30, 56, 86, 113, 15, 82, 84, 81, 26, 76, 77, 8, 5, 74, 71, 66, 122, 94, 67, 24, 11, 25, 1, 60, 22, 102, 72, 32, 12, 39, 97, 2, 120, 75, 21, 9, 17, 64, 10, 79, 14, 16, 89, 78, 91, 20, 6, 23, 80, 83, 87, 29, 4, 51, 73, 13, 68, 90, 27, 70, 112, 69, 33, 92, 93, 59, 85, 18, 96, 99, 117, 125, 31, 7, 114, 0, 19, 3, 107, 28, 105, 88, 118, 111, 110, 109, 44, 63, 104, 50, 108, 123, 95, 106, 52, 55, 36, 126, 101, 34, 65, 58, 116, 119, 47, 103, 62, 40, 45, 100, 42, 37, 121, 57, 54, 61, 48, 35, 41, 127, 115, 98, 43, 46, 53], [38, 49, 56, 124, 113, 30, 7, 84, 86, 82, 3, 81, 77, 15, 10, 13, 65, 89, 5, 76, 1, 67, 69, 8, 2, 64, 68, 94, 74, 73, 26, 72, 91, 122, 120, 66, 0, 27, 22, 97, 20, 79, 6, 18, 12, 9, 11, 14, 25, 87, 17, 23, 32, 24, 29, 51, 90, 75, 80, 60, 16, 99, 92, 78, 102, 83, 71, 59, 4, 88, 19, 39, 70, 33, 21, 31, 28, 103, 43, 114, 45, 125, 85, 112, 98, 93, 46, 111, 107, 118, 40, 95, 58, 37, 101, 126, 109, 104, 108, 44, 55, 105, 63, 41, 57, 61, 127, 96, 42, 115, 53, 116, 35, 54, 48, 100, 36, 47, 121, 62, 50, 117, 106, 119, 110, 34, 52, 123], [38, 49, 56, 124, 30, 86, 94, 89, 113, 81, 84, 91, 102, 122, 39, 15, 25, 82, 18, 51, 22, 26, 23, 76, 59, 97, 32, 120, 17, 33, 87, 105, 125, 29, 114, 99, 126, 72, 60, 103, 90, 62, 24, 31, 116, 77, 58, 20, 109, 118, 110, 52, 79, 45, 127, 112, 121, 63, 41, 117, 104, 96, 13, 55, 57, 50, 43, 44, 119, 111, 12, 115, 42, 48, 93, 95, 54, 101, 78, 40, 123, 75, 108, 37, 27, 53, 47, 46, 35, 100, 92, 61, 106, 36, 34, 107, 83, 28, 98, 21, 10, 19, 88, 9, 85, 80, 69, 14, 16, 73, 8, 7, 74, 11, 6, 68, 70, 3, 5, 71, 66, 4, 2, 65, 67, 1, 64, 0], [38, 49, 30, 124, 56, 113, 84, 86, 15, 77, 82, 74, 81, 5, 73, 76, 89, 8, 67, 71, 1, 20, 22, 72, 10, 122, 78, 66, 14, 64, 17, 9, 75, 79, 26, 94, 120, 16, 87, 11, 2, 7, 12, 6, 65, 99, 13, 91, 80, 69, 25, 27, 4, 3, 24, 39, 90, 97, 18, 32, 70, 93, 51, 85, 23, 92, 68, 29, 19, 21, 118, 31, 83, 60, 28, 88, 0, 114, 107, 33, 45, 46, 42, 43, 95, 103, 44, 117, 98, 125, 101, 100, 109, 63, 104, 62, 126, 54, 58, 111, 112, 40, 115, 57, 37, 59, 116, 50, 96, 55, 61, 52, 53, 34, 105, 41, 48, 35, 36, 119, 106, 121, 47, 110, 127, 123, 108, 102], [40, 98, 116, 32, 20, 78, 9, 81, 11, 89, 86, 7, 68, 69, 16, 47, 104, 4, 3, 53, 2, 49, 73, 51, 13, 117, 111, 126, 58, 121, 64, 8, 46, 56, 0, 67, 60, 94, 52, 88, 10, 127, 34, 42, 90, 71, 62, 72, 80, 74, 84, 113, 92, 50, 45, 79, 70, 63, 77, 75, 5, 27, 120, 6, 57, 44, 65, 1, 87, 14, 19, 106, 54, 17, 41, 101, 24, 125, 100, 21, 114, 99, 29, 122, 85, 105, 108, 33, 66, 15, 43, 124, 28, 119, 23, 115, 30, 12, 91, 18, 110, 25, 38, 39, 26, 22, 107, 118, 96, 61, 59, 103, 36, 83, 35, 97, 112, 123, 48, 37, 109, 102, 93, 95, 82, 31, 55, 76], [40, 98, 116, 32, 86, 20, 94, 88, 89, 28, 81, 62, 50, 121, 58, 78, 117, 52, 11, 47, 115, 45, 21, 104, 46, 126, 53, 111, 49, 87, 13, 30, 101, 41, 51, 96, 42, 16, 22, 92, 119, 90, 12, 122, 113, 24, 31, 91, 63, 26, 18, 29, 44, 37, 23, 109, 83, 93, 95, 107, 85, 38, 36, 125, 120, 110, 99, 103, 76, 79, 43, 114, 105, 102, 108, 8, 59, 25, 84, 124, 19, 9, 15, 7, 112, 54, 48, 60, 123, 82, 100, 127, 56, 27, 97, 106, 118, 35, 55, 80, 61, 57, 33, 17, 39, 14, 74, 34, 77, 75, 69, 10, 72, 2, 71, 3, 6, 68, 70, 73, 5, 67, 0, 66, 64, 4, 65, 1], [40, 98, 32, 86, 104, 116, 81, 20, 13, 47, 2, 66, 7, 8, 78, 49, 58, 51, 11, 9, 121, 60, 94, 5, 111, 45, 53, 52, 62, 68, 73, 46, 4, 16, 122, 92, 126, 70, 88, 89, 50, 0, 69, 56, 115, 34, 113, 10, 87, 64, 117, 125, 79, 3, 103, 82, 18, 28, 119, 77, 127, 63, 37, 57, 1, 108, 100, 27, 72, 25, 43, 114, 106, 44, 75, 120, 71, 22, 84, 65, 17, 30, 23, 42, 93, 14, 41, 19, 59, 54, 35, 48, 21, 31, 101, 76, 110, 12, 118, 29, 99, 90, 112, 80, 6, 85, 109, 15, 61, 124, 83, 36, 26, 91, 102, 55, 107, 24, 97, 39, 123, 95, 33, 38, 105, 74, 67, 96], [40, 98, 116, 86, 81, 32, 89, 78, 20, 88, 11, 7, 104, 47, 117, 111, 13, 49, 16, 28, 53, 74, 10, 51, 9, 121, 94, 58, 52, 126, 62, 46, 21, 45, 75, 60, 69, 25, 92, 96, 17, 8, 115, 79, 30, 50, 63, 77, 113, 90, 73, 71, 80, 56, 14, 67, 127, 95, 38, 2, 23, 42, 120, 122, 87, 72, 84, 99, 18, 44, 41, 29, 101, 105, 114, 125, 3, 22, 61, 19, 83, 91, 100, 6, 5, 54, 43, 97, 24, 27, 37, 119, 112, 108, 118, 15, 68, 85, 70, 102, 57, 107, 59, 34, 124, 55, 33, 12, 48, 110, 76, 66, 106, 26, 123, 103, 109, 31, 39, 93, 35, 65, 36, 82, 0, 1, 4, 64], [119, 38, 118, 42, 33, 25, 84, 31, 88, 52, 94, 103, 114, 99, 57, 55, 102, 28, 83, 78, 96, 54, 50, 62, 22, 106, 60, 45, 87, 93, 112, 29, 23, 53, 97, 27, 71, 61, 34, 74, 32, 85, 109, 111, 17, 24, 120, 46, 122, 110, 41, 80, 104, 21, 40, 18, 116, 125, 35, 121, 39, 105, 10, 123, 79, 56, 36, 92, 98, 47, 113, 107, 89, 127, 115, 126, 26, 20, 101, 16, 30, 43, 12, 49, 48, 81, 58, 91, 117, 37, 14, 13, 90, 63, 82, 100, 73, 11, 4, 7, 75, 44, 59, 51, 68, 124, 15, 108, 77, 95, 19, 72, 86, 1, 76, 8, 5, 6, 3, 0, 70, 69, 9, 65, 2, 67, 66, 64], [119, 118, 38, 33, 103, 42, 25, 94, 90, 54, 88, 109, 84, 31, 112, 120, 52, 102, 62, 111, 22, 45, 93, 53, 50, 57, 55, 83, 125, 47, 114, 87, 110, 106, 122, 74, 35, 18, 60, 101, 48, 58, 116, 85, 127, 61, 23, 78, 28, 59, 91, 100, 124, 121, 46, 63, 99, 49, 79, 126, 56, 123, 51, 113, 108, 30, 115, 105, 107, 117, 29, 43, 98, 26, 41, 39, 96, 0, 44, 24, 71, 34, 97, 40, 104, 27, 72, 89, 8, 11, 21, 75, 16, 68, 82, 12, 5, 37, 13, 69, 36, 64, 1, 4, 32, 95, 80, 67, 7, 2, 92, 65, 17, 14, 66, 73, 9, 3, 15, 70, 77, 10, 20, 81, 76, 6, 86, 19], [118, 38, 119, 42, 33, 25, 31, 109, 83, 55, 22, 94, 114, 84, 78, 90, 88, 106, 99, 52, 53, 54, 12, 74, 103, 120, 48, 111, 121, 102, 125, 17, 110, 50, 93, 62, 87, 107, 127, 34, 112, 60, 43, 97, 89, 115, 113, 122, 61, 57, 105, 58, 47, 91, 116, 73, 104, 35, 63, 46, 123, 30, 96, 28, 29, 72, 41, 59, 124, 126, 23, 44, 70, 37, 56, 117, 49, 101, 100, 27, 40, 85, 51, 79, 71, 95, 7, 45, 108, 80, 92, 68, 19, 39, 98, 81, 86, 26, 24, 75, 10, 36, 32, 67, 15, 76, 16, 21, 2, 18, 14, 11, 77, 82, 3, 66, 8, 1, 6, 20, 13, 0, 69, 4, 64, 5, 65, 9], [38, 118, 119, 33, 25, 83, 31, 79, 84, 89, 94, 12, 81, 78, 90, 88, 28, 85, 73, 22, 29, 54, 42, 24, 19, 112, 45, 26, 87, 93, 70, 17, 6, 102, 97, 48, 9, 55, 27, 116, 106, 35, 59, 127, 121, 114, 74, 110, 32, 125, 2, 103, 82, 20, 124, 52, 98, 96, 34, 21, 71, 50, 15, 80, 67, 18, 40, 16, 95, 109, 92, 0, 75, 91, 76, 62, 117, 72, 8, 39, 120, 69, 77, 65, 43, 30, 108, 36, 100, 99, 5, 58, 115, 57, 101, 14, 13, 23, 51, 60, 3, 61, 64, 107, 37, 86, 113, 104, 4, 66, 11, 126, 68, 41, 47, 122, 46, 111, 53, 1, 10, 123, 105, 49, 63, 7, 56, 44], [39, 57, 58, 52, 56, 116, 117, 55, 118, 51, 37, 59, 120, 86, 97, 31, 50, 126, 28, 98, 110, 99, 53, 54, 63, 112, 40, 122, 41, 20, 29, 47, 43, 92, 111, 109, 127, 124, 38, 125, 114, 95, 115, 108, 49, 123, 119, 105, 88, 89, 19, 61, 106, 62, 30, 60, 107, 113, 45, 34, 44, 104, 100, 121, 48, 36, 42, 46, 102, 33, 25, 32, 23, 6, 81, 101, 35, 80, 93, 10, 91, 24, 73, 96, 18, 87, 15, 78, 69, 27, 13, 14, 94, 22, 26, 90, 74, 66, 84, 67, 76, 71, 16, 12, 7, 68, 103, 64, 8, 21, 1, 2, 17, 72, 75, 5, 85, 83, 82, 11, 77, 0, 9, 3, 65, 79, 4, 70], [56, 39, 58, 52, 19, 59, 120, 34, 28, 90, 26, 31, 95, 96, 55, 37, 79, 53, 12, 21, 97, 88, 98, 86, 50, 116, 127, 118, 57, 24, 92, 8, 99, 25, 122, 49, 123, 111, 89, 14, 121, 18, 83, 27, 108, 29, 74, 63, 62, 51, 110, 44, 43, 87, 105, 109, 115, 91, 104, 38, 70, 126, 114, 1, 42, 10, 80, 41, 119, 68, 46, 33, 102, 72, 35, 60, 32, 45, 124, 125, 15, 93, 100, 106, 4, 48, 107, 85, 103, 112, 94, 47, 6, 78, 76, 23, 113, 81, 61, 75, 40, 101, 54, 11, 117, 0, 84, 16, 82, 36, 9, 2, 30, 13, 17, 22, 67, 71, 20, 64, 3, 69, 65, 73, 66, 77, 5, 7], [56, 39, 52, 117, 58, 57, 116, 55, 37, 118, 51, 53, 86, 54, 120, 10, 31, 73, 127, 126, 110, 111, 63, 97, 123, 122, 6, 119, 38, 76, 99, 121, 16, 29, 49, 43, 109, 13, 105, 112, 80, 42, 124, 125, 44, 78, 83, 106, 61, 7, 108, 93, 89, 28, 115, 50, 72, 48, 40, 62, 107, 45, 60, 46, 114, 77, 47, 113, 68, 88, 81, 11, 98, 41, 66, 104, 87, 82, 15, 67, 102, 69, 92, 36, 79, 30, 101, 34, 27, 59, 19, 96, 75, 22, 100, 33, 95, 20, 35, 1, 103, 71, 17, 32, 5, 84, 24, 94, 64, 26, 23, 85, 14, 4, 91, 8, 25, 0, 65, 9, 2, 12, 74, 21, 90, 70, 18, 3], [39, 52, 56, 34, 117, 57, 58, 51, 88, 116, 120, 50, 118, 95, 18, 98, 80, 20, 86, 55, 122, 21, 92, 30, 22, 90, 28, 53, 114, 97, 13, 74, 115, 19, 29, 32, 109, 60, 126, 62, 77, 61, 125, 99, 26, 89, 27, 25, 121, 37, 68, 11, 78, 49, 82, 14, 8, 54, 23, 48, 79, 24, 31, 71, 96, 112, 40, 105, 123, 127, 35, 46, 36, 94, 33, 124, 63, 12, 108, 119, 100, 110, 45, 10, 43, 44, 111, 47, 9, 107, 93, 85, 42, 16, 41, 38, 106, 15, 59, 113, 104, 102, 6, 101, 75, 91, 70, 81, 83, 2, 69, 84, 17, 1, 87, 76, 73, 0, 4, 3, 67, 5, 66, 72, 103, 7, 65, 64], [57, 48, 50, 38, 58, 54, 118, 56, 61, 100, 53, 63, 124, 115, 41, 116, 117, 119, 52, 102, 59, 84, 120, 46, 51, 121, 47, 126, 122, 125, 62, 113, 114, 111, 127, 97, 20, 44, 123, 49, 109, 60, 55, 45, 110, 112, 108, 43, 103, 106, 40, 107, 74, 42, 89, 105, 30, 27, 34, 31, 7, 33, 13, 101, 39, 24, 95, 78, 104, 32, 36, 98, 35, 21, 37, 88, 25, 99, 92, 80, 77, 87, 91, 96, 29, 23, 85, 93, 16, 71, 9, 69, 94, 28, 10, 66, 12, 90, 18, 73, 4, 26, 2, 5, 6, 65, 64, 3, 14, 19, 68, 72, 79, 83, 11, 70, 17, 0, 75, 82, 86, 1, 76, 15, 67, 22, 81, 8], [50, 38, 57, 48, 97, 112, 114, 7, 54, 115, 20, 88, 111, 77, 124, 90, 81, 100, 29, 95, 18, 79, 11, 28, 4, 27, 58, 24, 53, 68, 35, 23, 52, 59, 107, 94, 86, 119, 1, 101, 25, 63, 10, 56, 120, 99, 61, 42, 72, 104, 41, 121, 16, 103, 40, 125, 26, 43, 118, 117, 39, 108, 32, 123, 109, 110, 51, 37, 8, 46, 102, 116, 47, 105, 87, 106, 14, 93, 44, 6, 45, 85, 71, 49, 62, 126, 113, 55, 3, 76, 83, 65, 96, 75, 60, 34, 127, 89, 91, 30, 21, 122, 36, 31, 98, 78, 15, 84, 73, 12, 82, 17, 66, 5, 19, 80, 2, 74, 33, 0, 64, 92, 22, 9, 69, 13, 67, 70], [38, 57, 50, 48, 97, 23, 79, 72, 81, 76, 100, 18, 114, 28, 112, 6, 10, 3, 68, 14, 21, 65, 8, 70, 86, 77, 11, 82, 80, 87, 0, 67, 17, 54, 12, 75, 90, 88, 24, 56, 15, 2, 20, 95, 59, 73, 78, 5, 83, 9, 13, 85, 53, 84, 16, 29, 69, 58, 93, 102, 19, 92, 115, 91, 26, 25, 74, 27, 52, 66, 22, 117, 71, 94, 30, 121, 111, 61, 89, 32, 107, 7, 64, 124, 101, 36, 1, 55, 31, 35, 96, 63, 99, 4, 98, 119, 46, 34, 41, 60, 118, 125, 43, 110, 120, 106, 37, 126, 62, 123, 104, 40, 47, 42, 39, 103, 44, 127, 51, 116, 49, 105, 33, 108, 122, 109, 45, 113], [48, 50, 57, 38, 122, 115, 126, 51, 46, 54, 118, 56, 47, 62, 58, 109, 114, 124, 116, 49, 125, 55, 52, 112, 63, 113, 119, 61, 127, 60, 110, 59, 53, 45, 106, 43, 117, 44, 123, 121, 120, 108, 41, 107, 105, 40, 42, 103, 34, 102, 111, 101, 39, 89, 104, 20, 97, 37, 95, 32, 29, 90, 86, 100, 36, 31, 30, 35, 93, 99, 88, 77, 92, 96, 94, 98, 16, 22, 83, 33, 81, 27, 25, 91, 23, 80, 84, 14, 28, 24, 19, 21, 73, 26, 17, 82, 13, 18, 79, 78, 87, 85, 76, 11, 66, 10, 6, 12, 74, 9, 15, 0, 75, 2, 71, 69, 7, 1, 72, 8, 64, 68, 3, 4, 70, 5, 65, 67]], "model.layers.6.self_attn.k_proj": [[44, 55, 100, 86, 32, 84, 81, 57, 15, 77, 91, 108, 23, 12, 10, 64, 5, 75, 67, 8, 7, 65, 94, 114, 90, 16, 111, 29, 71, 76, 50, 82, 89, 103, 68, 72, 2, 83, 25, 39, 123, 78, 51, 66, 93, 121, 63, 120, 6, 58, 46, 48, 118, 125, 19, 124, 31, 126, 106, 61, 127, 104, 21, 70, 40, 1, 45, 18, 95, 88, 47, 0, 122, 69, 113, 53, 119, 34, 117, 62, 92, 54, 56, 42, 37, 109, 43, 73, 35, 41, 33, 11, 38, 116, 80, 30, 97, 49, 52, 98, 85, 28, 74, 107, 112, 59, 115, 110, 102, 4, 24, 60, 87, 99, 105, 101, 14, 27, 22, 26, 20, 79, 96, 13, 9, 17, 3, 36], [106, 110, 46, 96, 22, 93, 17, 89, 39, 10, 14, 19, 76, 20, 56, 71, 38, 120, 0, 11, 24, 45, 111, 42, 1, 123, 3, 52, 69, 66, 26, 28, 13, 103, 119, 58, 16, 5, 8, 49, 79, 37, 92, 18, 121, 116, 48, 107, 54, 43, 99, 114, 70, 105, 57, 127, 68, 122, 61, 85, 113, 60, 50, 90, 47, 73, 63, 55, 40, 36, 126, 117, 62, 109, 35, 53, 125, 124, 100, 112, 118, 30, 65, 102, 27, 115, 108, 4, 44, 104, 67, 23, 80, 82, 6, 101, 31, 51, 98, 34, 59, 97, 29, 9, 84, 64, 91, 33, 41, 94, 86, 7, 72, 21, 95, 74, 88, 2, 87, 77, 78, 12, 75, 15, 83, 25, 32, 81], [45, 39, 98, 109, 0, 20, 80, 18, 72, 12, 14, 24, 79, 75, 71, 66, 74, 5, 4, 70, 65, 9, 3, 112, 1, 67, 77, 69, 22, 92, 19, 64, 113, 123, 60, 120, 85, 91, 59, 86, 89, 23, 58, 48, 88, 17, 73, 6, 34, 78, 114, 25, 10, 90, 21, 27, 55, 63, 108, 13, 30, 122, 28, 47, 96, 126, 29, 119, 104, 33, 61, 105, 93, 83, 95, 35, 115, 31, 57, 11, 100, 51, 8, 54, 50, 38, 2, 121, 94, 111, 110, 49, 97, 26, 125, 81, 32, 87, 124, 7, 40, 46, 56, 52, 68, 101, 127, 42, 44, 117, 106, 116, 107, 118, 36, 43, 37, 41, 62, 102, 99, 53, 16, 76, 82, 84, 15, 103], [113, 124, 102, 56, 49, 94, 84, 82, 15, 76, 86, 81, 74, 77, 64, 5, 71, 8, 89, 73, 1, 66, 69, 39, 67, 122, 26, 91, 4, 75, 2, 51, 68, 70, 25, 65, 3, 72, 12, 24, 13, 38, 30, 6, 22, 23, 20, 59, 33, 79, 14, 83, 78, 90, 32, 80, 87, 7, 27, 9, 29, 88, 96, 10, 60, 99, 19, 16, 28, 21, 85, 18, 0, 11, 105, 97, 93, 17, 31, 95, 92, 103, 114, 35, 58, 125, 54, 47, 117, 43, 46, 104, 45, 40, 107, 118, 112, 36, 108, 109, 116, 115, 110, 126, 42, 62, 48, 111, 63, 98, 101, 55, 50, 53, 57, 44, 61, 127, 52, 106, 41, 34, 37, 121, 119, 120, 123, 100], [104, 116, 34, 111, 64, 20, 11, 53, 81, 86, 78, 51, 8, 16, 9, 7, 13, 121, 96, 3, 113, 2, 58, 62, 68, 69, 109, 60, 110, 122, 42, 50, 117, 56, 41, 89, 46, 127, 30, 49, 1, 126, 90, 108, 21, 98, 67, 39, 6, 105, 40, 63, 35, 37, 88, 83, 82, 32, 27, 124, 125, 65, 92, 44, 15, 57, 76, 107, 118, 61, 120, 54, 36, 74, 29, 0, 87, 70, 72, 28, 24, 31, 23, 25, 99, 93, 59, 106, 119, 47, 85, 91, 5, 43, 103, 77, 94, 33, 112, 97, 18, 55, 26, 45, 22, 38, 95, 115, 114, 19, 101, 123, 12, 52, 84, 100, 48, 71, 102, 66, 17, 79, 10, 75, 14, 4, 73, 80], [102, 119, 118, 22, 97, 25, 28, 83, 84, 78, 95, 12, 64, 74, 94, 71, 16, 18, 52, 79, 88, 30, 68, 125, 106, 111, 73, 38, 124, 46, 13, 91, 54, 45, 127, 66, 109, 6, 116, 17, 48, 104, 62, 43, 51, 58, 40, 53, 61, 57, 126, 90, 59, 123, 117, 67, 65, 1, 120, 121, 113, 44, 93, 3, 60, 50, 11, 56, 49, 69, 103, 108, 21, 47, 37, 112, 81, 19, 99, 42, 115, 122, 23, 63, 35, 82, 114, 41, 75, 2, 105, 20, 39, 26, 100, 70, 110, 55, 36, 87, 34, 32, 72, 15, 107, 101, 4, 80, 0, 98, 7, 5, 31, 77, 85, 96, 8, 27, 29, 92, 76, 86, 9, 89, 10, 24, 14, 33], [103, 57, 52, 56, 31, 58, 21, 88, 18, 90, 117, 59, 79, 98, 28, 14, 111, 72, 89, 75, 4, 120, 118, 25, 60, 12, 116, 110, 126, 74, 64, 85, 115, 55, 50, 62, 63, 124, 121, 66, 70, 46, 32, 112, 125, 49, 123, 65, 109, 39, 108, 105, 51, 53, 114, 48, 43, 73, 54, 34, 44, 92, 127, 27, 35, 113, 119, 81, 106, 19, 107, 102, 45, 47, 86, 84, 101, 20, 61, 122, 96, 7, 97, 104, 87, 77, 76, 91, 29, 41, 100, 13, 40, 82, 42, 71, 11, 94, 26, 16, 33, 6, 38, 95, 67, 3, 99, 36, 69, 37, 24, 80, 78, 30, 93, 1, 9, 68, 5, 8, 17, 15, 10, 23, 22, 83, 0, 2], [102, 50, 57, 86, 48, 33, 92, 18, 79, 76, 81, 23, 72, 6, 88, 83, 112, 14, 90, 11, 0, 93, 100, 30, 3, 10, 111, 31, 53, 58, 77, 124, 68, 73, 119, 61, 94, 117, 114, 2, 16, 63, 21, 118, 52, 36, 29, 54, 125, 38, 35, 99, 56, 115, 120, 9, 59, 65, 121, 108, 123, 105, 41, 98, 47, 113, 126, 127, 69, 55, 25, 45, 62, 51, 46, 24, 109, 60, 116, 20, 107, 110, 49, 85, 43, 32, 44, 66, 5, 96, 34, 122, 89, 106, 75, 42, 104, 1, 103, 91, 27, 39, 40, 101, 26, 37, 64, 4, 71, 95, 82, 84, 12, 78, 67, 80, 22, 15, 87, 19, 7, 8, 74, 28, 17, 13, 70, 97]], "model.layers.6.self_attn.qk_proj": [[57, 56, 118, 119, 45, 50, 55, 124, 113, 109, 48, 49, 38, 46, 44, 106, 22, 104, 52, 116, 102, 39, 108, 86, 20, 84, 58, 17, 110, 42, 30, 81, 25, 111, 34, 79, 15, 78, 76, 82, 98, 18, 88, 94, 89, 24, 10, 14, 32, 12, 112, 7, 13, 74, 0, 103, 40, 77, 117, 114, 51, 29, 72, 120, 8, 53, 75, 64, 16, 71, 11, 96, 90, 19, 28, 80, 67, 62, 31, 60, 87, 26, 121, 33, 3, 92, 63, 69, 70, 123, 127, 23, 83, 73, 97, 5, 66, 2, 9, 59, 126, 122, 68, 27, 61, 41, 47, 4, 93, 100, 21, 43, 95, 36, 115, 125, 54, 107, 1, 91, 65, 99, 85, 105, 35, 6, 101, 37], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 22, 116, 52, 102, 39, 84, 86, 108, 20, 58, 110, 17, 42, 111, 81, 30, 25, 103, 34, 89, 94, 79, 82, 78, 98, 32, 15, 18, 76, 112, 14, 117, 24, 88, 12, 51, 10, 40, 74, 29, 7, 53, 75, 77, 120, 16, 28, 71, 72, 60, 11, 8, 64, 13, 114, 90, 80, 0, 96, 19, 121, 26, 127, 59, 97, 83, 87, 66, 31, 69, 2, 23, 62, 92, 4, 5, 33, 9, 122, 73, 123, 126, 27, 68, 67, 63, 1, 70, 100, 54, 61, 93, 125, 95, 3, 47, 43, 65, 21, 41, 91, 107, 6, 115, 99, 85, 36, 105, 35, 37, 101], [57, 56, 119, 118, 45, 113, 50, 55, 109, 48, 124, 49, 38, 46, 44, 102, 22, 106, 104, 39, 52, 116, 86, 84, 108, 20, 58, 110, 42, 17, 30, 25, 81, 89, 111, 98, 79, 94, 15, 34, 18, 82, 88, 14, 103, 32, 78, 76, 74, 0, 120, 13, 77, 75, 40, 112, 10, 24, 51, 71, 117, 12, 8, 7, 53, 60, 64, 114, 96, 11, 28, 72, 16, 90, 5, 29, 67, 31, 80, 26, 19, 83, 92, 69, 66, 2, 62, 3, 123, 97, 23, 68, 87, 73, 121, 9, 59, 127, 4, 122, 27, 93, 100, 63, 33, 54, 126, 6, 61, 1, 115, 70, 95, 43, 107, 21, 85, 41, 65, 91, 47, 125, 99, 105, 35, 36, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 49, 38, 46, 44, 106, 104, 102, 52, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 30, 81, 89, 18, 79, 34, 103, 25, 98, 94, 15, 111, 14, 78, 32, 82, 64, 112, 13, 117, 88, 76, 10, 51, 40, 24, 12, 8, 74, 77, 0, 3, 71, 120, 11, 60, 7, 28, 114, 53, 67, 96, 127, 90, 66, 123, 75, 121, 80, 16, 29, 31, 5, 62, 83, 72, 26, 19, 2, 69, 92, 73, 97, 33, 122, 23, 63, 59, 87, 4, 6, 27, 126, 100, 9, 54, 68, 93, 115, 1, 61, 95, 65, 21, 41, 107, 43, 70, 47, 91, 85, 125, 37, 105, 36, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 113, 48, 124, 109, 38, 49, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 20, 84, 42, 58, 17, 110, 81, 34, 78, 30, 89, 111, 79, 15, 14, 94, 76, 18, 32, 12, 98, 112, 74, 10, 25, 8, 71, 82, 103, 88, 64, 51, 7, 11, 40, 114, 77, 13, 120, 0, 72, 24, 96, 53, 16, 28, 75, 29, 62, 80, 69, 117, 83, 66, 19, 90, 3, 92, 127, 5, 4, 60, 2, 6, 123, 26, 63, 73, 9, 67, 68, 31, 121, 59, 100, 65, 33, 23, 97, 87, 122, 126, 93, 115, 27, 41, 54, 95, 1, 61, 105, 43, 91, 47, 36, 107, 125, 85, 37, 70, 35, 99, 21, 101], [57, 56, 119, 118, 45, 50, 55, 113, 124, 109, 48, 49, 46, 38, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 108, 20, 58, 110, 17, 42, 81, 34, 79, 82, 111, 30, 25, 14, 98, 12, 78, 15, 76, 8, 74, 89, 10, 94, 103, 18, 13, 112, 88, 11, 7, 40, 51, 16, 32, 77, 117, 24, 71, 114, 0, 72, 64, 3, 53, 80, 120, 75, 29, 123, 60, 28, 127, 69, 5, 90, 96, 19, 26, 83, 97, 23, 31, 4, 68, 6, 73, 67, 66, 122, 59, 92, 62, 121, 9, 100, 87, 126, 63, 2, 33, 93, 27, 47, 41, 54, 107, 115, 95, 1, 61, 65, 43, 36, 21, 125, 85, 105, 99, 91, 70, 37, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 104, 106, 52, 102, 22, 116, 39, 86, 20, 84, 108, 17, 58, 81, 30, 110, 42, 82, 79, 89, 25, 40, 10, 78, 15, 18, 13, 12, 77, 98, 76, 34, 74, 51, 14, 94, 111, 8, 103, 16, 88, 11, 32, 112, 24, 72, 7, 96, 117, 75, 80, 120, 19, 90, 114, 71, 53, 29, 60, 59, 28, 83, 26, 127, 92, 73, 61, 63, 97, 62, 64, 123, 122, 121, 0, 33, 100, 9, 87, 31, 69, 23, 67, 5, 27, 126, 2, 4, 68, 93, 6, 3, 66, 47, 125, 107, 21, 41, 54, 91, 43, 85, 105, 95, 1, 99, 115, 65, 36, 37, 101, 70, 35], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 46, 38, 106, 44, 104, 52, 39, 116, 102, 22, 86, 84, 20, 108, 17, 58, 81, 89, 42, 110, 25, 78, 30, 34, 82, 14, 79, 15, 12, 18, 111, 74, 98, 77, 88, 103, 76, 10, 40, 13, 51, 8, 11, 16, 80, 94, 75, 117, 32, 24, 71, 112, 120, 19, 72, 114, 7, 28, 83, 53, 64, 96, 92, 121, 59, 73, 69, 31, 29, 5, 0, 87, 62, 90, 60, 97, 66, 9, 26, 100, 126, 27, 23, 122, 2, 123, 33, 63, 68, 4, 47, 3, 67, 127, 54, 21, 6, 91, 61, 93, 85, 125, 43, 95, 65, 70, 107, 41, 35, 1, 115, 105, 36, 99, 37, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 86, 84, 20, 108, 58, 81, 17, 42, 25, 110, 89, 78, 34, 82, 30, 14, 111, 79, 18, 32, 103, 98, 15, 94, 40, 10, 12, 88, 77, 80, 13, 74, 76, 24, 8, 112, 117, 29, 11, 75, 83, 53, 19, 114, 28, 51, 96, 72, 120, 90, 7, 71, 121, 16, 62, 92, 31, 87, 64, 26, 59, 127, 23, 97, 100, 66, 9, 67, 122, 60, 5, 123, 61, 33, 27, 73, 63, 3, 4, 68, 0, 126, 69, 2, 54, 93, 21, 125, 91, 95, 6, 47, 85, 65, 70, 41, 99, 35, 1, 43, 37, 115, 105, 107, 36, 101], [57, 56, 118, 119, 50, 45, 55, 113, 109, 124, 48, 49, 38, 46, 44, 116, 106, 104, 102, 52, 22, 39, 84, 86, 20, 108, 58, 81, 42, 17, 110, 30, 25, 82, 79, 111, 14, 15, 78, 76, 18, 89, 98, 34, 94, 103, 51, 77, 40, 32, 88, 10, 127, 12, 29, 24, 7, 74, 11, 112, 80, 117, 28, 8, 90, 13, 96, 72, 75, 53, 87, 16, 83, 60, 71, 19, 114, 120, 26, 31, 23, 0, 5, 123, 66, 92, 59, 64, 97, 62, 69, 73, 67, 100, 121, 68, 9, 33, 61, 27, 4, 126, 122, 63, 2, 3, 70, 54, 93, 47, 41, 125, 43, 91, 21, 85, 95, 107, 1, 105, 65, 115, 6, 35, 99, 37, 101, 36], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 106, 46, 52, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 110, 17, 81, 34, 111, 30, 15, 14, 78, 25, 76, 98, 79, 103, 82, 18, 89, 94, 10, 0, 32, 88, 12, 64, 112, 7, 72, 71, 11, 13, 24, 74, 40, 114, 51, 117, 96, 53, 77, 8, 19, 28, 2, 70, 75, 5, 80, 66, 69, 31, 120, 127, 16, 62, 26, 123, 3, 92, 63, 90, 29, 83, 67, 60, 73, 68, 23, 9, 97, 121, 41, 4, 87, 122, 59, 93, 100, 61, 65, 47, 54, 33, 126, 27, 1, 95, 115, 107, 85, 125, 43, 21, 91, 105, 6, 36, 35, 99, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 44, 46, 106, 104, 52, 39, 116, 102, 22, 84, 86, 108, 20, 58, 42, 110, 17, 81, 111, 98, 34, 82, 30, 79, 25, 78, 15, 14, 89, 94, 32, 18, 103, 13, 88, 12, 76, 51, 40, 74, 72, 10, 7, 71, 24, 112, 80, 77, 53, 67, 75, 64, 0, 123, 3, 11, 121, 8, 5, 16, 29, 19, 117, 62, 127, 28, 92, 68, 120, 96, 114, 69, 83, 9, 31, 90, 122, 4, 59, 66, 60, 2, 70, 26, 87, 63, 97, 73, 100, 41, 27, 33, 23, 1, 65, 95, 93, 54, 47, 126, 61, 91, 107, 125, 115, 21, 85, 36, 105, 6, 99, 37, 43, 35, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 46, 38, 44, 104, 106, 116, 52, 102, 22, 39, 84, 86, 20, 108, 17, 58, 81, 110, 30, 42, 25, 89, 82, 111, 15, 79, 18, 78, 34, 14, 94, 12, 74, 10, 98, 32, 76, 103, 13, 72, 40, 88, 77, 24, 11, 29, 16, 7, 51, 80, 117, 8, 83, 75, 112, 71, 114, 53, 96, 120, 92, 122, 28, 97, 121, 5, 100, 90, 59, 60, 19, 127, 26, 31, 9, 87, 123, 27, 126, 69, 73, 23, 66, 4, 41, 68, 62, 33, 2, 64, 3, 0, 93, 61, 54, 63, 47, 91, 125, 115, 43, 107, 85, 67, 21, 70, 95, 65, 6, 35, 37, 99, 1, 36, 105, 101], [57, 56, 119, 118, 45, 55, 50, 124, 113, 109, 48, 49, 46, 38, 44, 106, 52, 104, 116, 22, 102, 39, 86, 84, 108, 20, 58, 17, 110, 81, 42, 34, 82, 78, 25, 18, 12, 30, 72, 79, 15, 103, 111, 74, 98, 14, 10, 0, 32, 89, 76, 88, 24, 112, 71, 77, 7, 51, 8, 94, 64, 40, 13, 16, 75, 117, 11, 123, 80, 5, 29, 83, 53, 19, 114, 62, 69, 60, 26, 96, 127, 28, 31, 92, 9, 126, 90, 59, 120, 122, 2, 66, 73, 23, 121, 33, 54, 63, 68, 4, 87, 97, 3, 61, 67, 100, 70, 27, 21, 6, 47, 93, 115, 43, 95, 91, 41, 125, 1, 65, 85, 105, 107, 36, 35, 99, 37, 101], [57, 56, 118, 119, 45, 113, 50, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 39, 102, 86, 22, 84, 108, 20, 42, 58, 17, 81, 110, 30, 18, 82, 79, 89, 25, 78, 34, 14, 98, 15, 94, 111, 12, 103, 72, 32, 10, 74, 76, 71, 40, 77, 112, 88, 7, 24, 11, 51, 16, 8, 96, 117, 26, 5, 28, 29, 62, 13, 3, 75, 53, 83, 64, 31, 114, 0, 90, 19, 69, 120, 80, 127, 73, 66, 60, 121, 59, 92, 67, 2, 123, 100, 9, 97, 23, 68, 126, 87, 6, 63, 93, 54, 4, 122, 27, 33, 41, 115, 43, 65, 21, 95, 47, 1, 85, 61, 36, 35, 107, 70, 91, 125, 37, 105, 99, 101], [57, 56, 119, 118, 45, 50, 55, 124, 113, 48, 109, 38, 49, 44, 46, 106, 52, 104, 116, 102, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 34, 30, 111, 78, 25, 98, 15, 94, 14, 79, 82, 10, 74, 89, 76, 18, 12, 103, 117, 24, 77, 72, 88, 32, 112, 40, 7, 71, 11, 16, 8, 120, 26, 96, 51, 28, 75, 80, 29, 13, 64, 83, 121, 19, 114, 0, 92, 127, 69, 59, 53, 90, 3, 123, 31, 97, 47, 67, 62, 60, 73, 5, 122, 93, 126, 4, 87, 23, 100, 9, 68, 2, 66, 6, 41, 33, 63, 27, 54, 115, 43, 21, 95, 61, 125, 65, 1, 107, 35, 99, 105, 91, 70, 85, 36, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 38, 46, 49, 44, 116, 22, 106, 104, 52, 102, 39, 86, 84, 108, 20, 17, 42, 58, 110, 81, 30, 25, 18, 89, 15, 111, 77, 98, 78, 79, 34, 74, 82, 14, 10, 12, 94, 76, 11, 88, 40, 24, 7, 13, 16, 72, 51, 120, 117, 32, 103, 8, 75, 80, 53, 28, 71, 114, 112, 122, 123, 19, 26, 96, 29, 60, 92, 31, 0, 87, 127, 83, 5, 69, 126, 97, 23, 90, 62, 73, 66, 63, 59, 121, 9, 4, 33, 61, 64, 100, 68, 21, 27, 6, 47, 43, 54, 115, 107, 3, 2, 67, 93, 95, 41, 105, 125, 99, 35, 91, 1, 65, 36, 85, 37, 70, 101], [57, 56, 118, 119, 45, 50, 113, 124, 55, 109, 48, 46, 49, 38, 44, 106, 52, 104, 116, 22, 39, 102, 86, 84, 20, 108, 58, 42, 110, 17, 81, 89, 30, 82, 25, 18, 14, 78, 15, 98, 34, 74, 12, 10, 79, 111, 94, 72, 88, 40, 117, 8, 32, 13, 75, 76, 103, 24, 77, 7, 53, 28, 51, 112, 80, 71, 11, 16, 60, 31, 83, 64, 96, 19, 121, 5, 90, 114, 120, 126, 87, 122, 69, 26, 62, 66, 29, 92, 9, 3, 73, 23, 97, 123, 0, 127, 100, 59, 63, 67, 68, 33, 2, 61, 6, 27, 4, 47, 115, 54, 43, 21, 93, 91, 95, 41, 1, 85, 107, 65, 125, 70, 105, 99, 35, 37, 36, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 38, 49, 46, 44, 106, 102, 52, 104, 116, 39, 22, 86, 84, 108, 20, 58, 42, 17, 110, 30, 81, 89, 94, 15, 34, 18, 82, 98, 14, 111, 12, 76, 88, 78, 79, 25, 103, 40, 10, 74, 8, 0, 51, 24, 112, 71, 7, 77, 72, 11, 32, 64, 96, 117, 31, 13, 75, 28, 26, 100, 60, 90, 5, 16, 80, 114, 29, 69, 121, 122, 66, 120, 3, 92, 53, 83, 19, 2, 127, 97, 23, 62, 87, 123, 33, 9, 73, 59, 4, 68, 126, 27, 67, 63, 41, 61, 6, 93, 95, 105, 54, 115, 70, 65, 1, 47, 91, 21, 37, 85, 99, 35, 43, 107, 36, 125, 101], [57, 56, 118, 119, 50, 45, 55, 124, 113, 48, 109, 49, 38, 46, 44, 106, 52, 116, 102, 104, 22, 39, 86, 84, 108, 20, 58, 17, 110, 42, 30, 81, 14, 34, 89, 98, 82, 76, 78, 18, 10, 74, 25, 94, 15, 8, 117, 88, 12, 111, 103, 32, 79, 112, 51, 71, 40, 11, 75, 13, 24, 72, 120, 77, 19, 123, 7, 29, 26, 96, 16, 126, 80, 114, 83, 28, 90, 127, 0, 60, 53, 5, 69, 121, 66, 73, 31, 4, 64, 122, 61, 97, 92, 68, 59, 100, 3, 62, 87, 2, 23, 33, 9, 63, 47, 70, 27, 93, 67, 43, 95, 21, 54, 41, 65, 1, 115, 6, 107, 85, 105, 91, 35, 125, 36, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 48, 109, 49, 44, 38, 46, 106, 52, 104, 116, 22, 102, 39, 86, 108, 84, 20, 58, 17, 42, 110, 81, 30, 14, 98, 89, 18, 111, 79, 25, 88, 34, 78, 8, 15, 32, 82, 12, 94, 76, 10, 74, 103, 77, 112, 24, 40, 71, 117, 13, 120, 67, 16, 75, 114, 64, 51, 80, 11, 7, 28, 62, 3, 96, 72, 123, 60, 19, 92, 90, 31, 83, 29, 87, 26, 53, 127, 126, 5, 0, 121, 69, 97, 73, 66, 61, 63, 4, 47, 70, 115, 122, 33, 41, 9, 59, 93, 100, 27, 68, 107, 2, 23, 95, 54, 91, 125, 43, 21, 36, 85, 1, 6, 105, 65, 99, 37, 35, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 49, 38, 44, 46, 52, 106, 104, 102, 116, 39, 22, 86, 108, 84, 20, 58, 42, 17, 110, 81, 30, 34, 15, 98, 25, 94, 79, 18, 89, 14, 111, 78, 82, 12, 103, 88, 74, 40, 24, 8, 112, 10, 117, 76, 77, 51, 32, 120, 71, 7, 16, 75, 64, 13, 11, 96, 0, 28, 60, 80, 26, 66, 90, 114, 72, 69, 53, 59, 5, 83, 19, 29, 92, 3, 126, 87, 97, 121, 62, 2, 31, 122, 123, 127, 9, 23, 63, 27, 73, 33, 100, 70, 67, 4, 43, 68, 61, 47, 93, 41, 95, 115, 107, 105, 1, 54, 91, 125, 21, 85, 36, 37, 65, 6, 99, 35, 101], [57, 56, 118, 119, 45, 50, 55, 124, 113, 48, 109, 49, 38, 46, 44, 52, 106, 116, 104, 102, 22, 39, 20, 108, 86, 84, 58, 42, 17, 110, 30, 81, 89, 25, 34, 18, 88, 79, 32, 78, 94, 14, 15, 12, 82, 98, 74, 111, 8, 10, 112, 13, 51, 103, 76, 117, 77, 24, 71, 11, 114, 7, 120, 75, 80, 40, 16, 19, 72, 28, 87, 83, 59, 29, 26, 0, 126, 62, 96, 92, 97, 69, 90, 127, 31, 121, 5, 64, 60, 3, 122, 53, 9, 73, 66, 123, 33, 4, 68, 100, 23, 2, 27, 70, 93, 67, 61, 63, 47, 43, 41, 1, 115, 95, 54, 85, 91, 125, 35, 37, 107, 105, 21, 6, 36, 65, 99, 101], [57, 56, 118, 119, 45, 50, 124, 55, 113, 109, 48, 38, 49, 46, 44, 106, 52, 116, 104, 102, 39, 22, 84, 108, 86, 20, 110, 58, 42, 17, 30, 81, 111, 98, 89, 78, 34, 25, 94, 10, 79, 88, 103, 8, 18, 74, 76, 117, 14, 12, 82, 32, 15, 51, 75, 112, 77, 24, 40, 71, 28, 7, 114, 120, 72, 26, 80, 13, 29, 96, 16, 127, 11, 3, 123, 19, 53, 83, 97, 90, 60, 87, 92, 69, 126, 121, 31, 64, 122, 23, 9, 62, 68, 5, 73, 67, 100, 33, 4, 59, 63, 27, 0, 66, 41, 47, 115, 54, 2, 70, 93, 61, 107, 105, 125, 36, 65, 95, 43, 21, 85, 91, 35, 1, 6, 99, 37, 101], [57, 56, 118, 119, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 52, 104, 102, 106, 116, 22, 39, 108, 20, 84, 86, 58, 17, 42, 81, 110, 30, 10, 89, 34, 79, 76, 18, 111, 25, 15, 98, 14, 78, 94, 82, 11, 88, 12, 74, 40, 51, 7, 103, 13, 77, 117, 112, 8, 71, 24, 32, 120, 16, 96, 72, 114, 75, 29, 80, 60, 26, 28, 90, 31, 19, 83, 53, 122, 5, 69, 92, 73, 123, 64, 9, 0, 62, 126, 100, 97, 121, 23, 59, 127, 63, 33, 87, 68, 2, 4, 67, 3, 41, 47, 66, 93, 115, 27, 61, 6, 125, 95, 43, 107, 70, 105, 21, 65, 1, 85, 36, 91, 54, 99, 35, 37, 101], [57, 56, 118, 119, 50, 45, 113, 124, 55, 109, 48, 49, 38, 46, 44, 106, 104, 52, 116, 102, 22, 39, 84, 86, 20, 108, 58, 17, 81, 42, 110, 111, 25, 98, 30, 89, 34, 14, 82, 76, 74, 79, 10, 15, 78, 117, 94, 18, 103, 88, 24, 72, 40, 12, 71, 120, 8, 13, 7, 77, 32, 80, 11, 75, 28, 51, 16, 112, 92, 126, 114, 64, 0, 26, 19, 96, 83, 53, 5, 29, 123, 9, 62, 23, 60, 121, 66, 122, 4, 73, 127, 90, 67, 69, 97, 100, 87, 31, 93, 59, 33, 2, 47, 6, 63, 61, 3, 91, 27, 115, 125, 95, 41, 54, 68, 21, 85, 1, 107, 105, 65, 43, 36, 70, 99, 35, 101, 37], [57, 56, 119, 118, 45, 50, 124, 113, 55, 109, 48, 49, 38, 46, 44, 104, 106, 52, 116, 22, 102, 39, 86, 84, 20, 108, 58, 17, 30, 81, 25, 42, 110, 82, 34, 98, 89, 79, 32, 15, 111, 103, 88, 76, 94, 14, 78, 24, 12, 74, 10, 40, 18, 77, 51, 75, 72, 112, 13, 120, 7, 29, 19, 90, 71, 16, 92, 114, 117, 96, 80, 28, 64, 8, 0, 83, 11, 67, 3, 62, 97, 59, 53, 121, 87, 63, 73, 60, 26, 31, 126, 5, 122, 33, 66, 4, 6, 23, 2, 69, 47, 61, 68, 100, 123, 127, 41, 9, 93, 125, 27, 65, 91, 43, 95, 107, 85, 21, 54, 1, 105, 99, 115, 36, 70, 35, 37, 101], [57, 56, 119, 118, 45, 50, 124, 55, 113, 109, 48, 49, 46, 38, 44, 52, 106, 116, 104, 39, 22, 102, 84, 108, 20, 86, 58, 17, 42, 110, 81, 34, 30, 15, 111, 79, 89, 10, 25, 98, 82, 78, 12, 18, 14, 72, 76, 74, 103, 112, 8, 7, 51, 24, 94, 88, 13, 75, 120, 40, 117, 64, 71, 77, 11, 0, 80, 32, 16, 53, 60, 114, 5, 62, 2, 19, 28, 29, 90, 69, 123, 83, 73, 67, 121, 4, 26, 97, 68, 126, 92, 66, 96, 23, 122, 9, 31, 63, 127, 6, 33, 61, 47, 54, 3, 59, 93, 65, 87, 100, 1, 27, 95, 43, 41, 115, 125, 107, 91, 105, 85, 21, 70, 36, 37, 99, 101, 35], [57, 56, 119, 118, 45, 50, 113, 55, 124, 48, 109, 38, 49, 46, 44, 106, 104, 102, 52, 116, 22, 39, 108, 86, 20, 84, 42, 58, 110, 111, 17, 30, 81, 34, 94, 14, 98, 15, 112, 25, 72, 103, 79, 7, 18, 76, 89, 88, 10, 12, 32, 120, 78, 24, 74, 82, 114, 40, 51, 117, 123, 28, 71, 80, 11, 96, 75, 8, 77, 0, 64, 13, 26, 5, 53, 29, 16, 60, 69, 83, 2, 31, 67, 92, 90, 62, 126, 122, 121, 127, 9, 97, 47, 66, 19, 87, 33, 4, 73, 3, 63, 100, 59, 93, 41, 6, 23, 27, 54, 68, 107, 65, 115, 61, 36, 95, 105, 1, 21, 91, 70, 99, 125, 35, 85, 43, 101, 37], [57, 56, 119, 118, 45, 55, 124, 50, 113, 48, 109, 49, 38, 46, 44, 106, 104, 52, 116, 102, 39, 22, 108, 86, 84, 20, 110, 58, 42, 17, 81, 30, 111, 14, 72, 34, 98, 25, 76, 79, 15, 18, 10, 89, 94, 103, 112, 114, 78, 74, 88, 82, 12, 13, 7, 75, 32, 117, 24, 71, 80, 77, 40, 16, 3, 8, 53, 120, 11, 51, 96, 64, 0, 28, 63, 19, 29, 83, 5, 31, 123, 60, 4, 126, 69, 67, 73, 62, 26, 127, 66, 92, 122, 97, 90, 115, 9, 121, 23, 2, 59, 33, 107, 54, 47, 87, 27, 41, 68, 93, 100, 6, 70, 1, 95, 125, 65, 61, 105, 21, 85, 43, 99, 91, 36, 35, 37, 101], [57, 56, 119, 118, 45, 50, 113, 55, 124, 109, 48, 49, 38, 46, 44, 106, 104, 116, 52, 39, 102, 22, 84, 108, 86, 20, 110, 58, 81, 17, 42, 34, 30, 25, 15, 98, 111, 18, 82, 103, 72, 14, 79, 94, 78, 12, 76, 74, 51, 89, 88, 13, 71, 112, 10, 7, 24, 120, 40, 32, 11, 77, 80, 60, 0, 117, 114, 64, 75, 66, 16, 83, 123, 5, 8, 69, 96, 26, 68, 53, 63, 62, 29, 28, 87, 19, 90, 31, 127, 122, 9, 2, 4, 92, 97, 73, 67, 59, 47, 23, 33, 100, 61, 115, 70, 121, 3, 27, 41, 126, 93, 21, 107, 54, 6, 65, 125, 95, 91, 36, 1, 85, 43, 105, 37, 99, 35, 101], [57, 56, 118, 119, 45, 50, 113, 55, 124, 109, 49, 48, 38, 46, 44, 106, 104, 52, 116, 39, 102, 22, 20, 86, 108, 84, 58, 81, 17, 42, 110, 30, 25, 34, 111, 82, 14, 18, 79, 98, 12, 72, 76, 94, 10, 15, 89, 88, 117, 78, 51, 74, 32, 40, 103, 112, 75, 7, 8, 71, 80, 16, 24, 13, 114, 77, 60, 11, 120, 83, 28, 19, 53, 31, 96, 90, 69, 9, 0, 26, 5, 87, 62, 59, 73, 29, 97, 92, 23, 123, 66, 126, 121, 64, 3, 127, 70, 122, 68, 4, 27, 33, 2, 115, 47, 67, 61, 63, 100, 41, 93, 54, 65, 43, 125, 21, 91, 95, 1, 105, 6, 85, 107, 36, 99, 37, 101, 35]], "model.layers.7.self_attn.q_proj": [[38, 109, 43, 89, 45, 18, 93, 60, 107, 78, 12, 22, 10, 8, 17, 5, 19, 77, 80, 79, 56, 24, 91, 71, 68, 72, 3, 92, 26, 115, 25, 51, 20, 23, 57, 69, 15, 50, 81, 4, 58, 65, 102, 14, 127, 86, 97, 125, 27, 70, 34, 126, 28, 90, 88, 32, 67, 108, 30, 76, 9, 122, 29, 111, 123, 1, 6, 13, 124, 116, 74, 110, 82, 16, 63, 11, 96, 33, 75, 117, 55, 113, 52, 83, 87, 62, 35, 112, 94, 95, 7, 59, 100, 73, 119, 106, 40, 85, 54, 21, 0, 48, 44, 61, 114, 39, 105, 47, 31, 49, 120, 103, 53, 99, 36, 118, 84, 121, 41, 37, 66, 2, 98, 46, 101, 64, 104, 42], [38, 109, 43, 60, 10, 12, 19, 77, 5, 80, 93, 45, 18, 68, 78, 71, 107, 25, 22, 89, 8, 1, 73, 69, 79, 17, 3, 70, 29, 125, 65, 51, 88, 72, 87, 23, 0, 16, 14, 66, 122, 20, 4, 56, 74, 94, 26, 91, 84, 15, 32, 28, 86, 85, 7, 115, 21, 34, 24, 11, 76, 82, 6, 30, 90, 83, 92, 81, 58, 75, 116, 102, 97, 2, 13, 120, 123, 27, 95, 64, 126, 9, 100, 35, 33, 67, 98, 31, 96, 52, 46, 36, 111, 108, 47, 39, 127, 105, 99, 118, 62, 117, 112, 110, 59, 49, 55, 37, 106, 50, 54, 63, 104, 61, 103, 101, 57, 40, 114, 113, 119, 41, 48, 124, 53, 42, 44, 121], [38, 109, 43, 10, 67, 19, 93, 77, 45, 107, 80, 78, 12, 65, 6, 79, 23, 4, 5, 89, 7, 71, 3, 72, 0, 69, 25, 68, 8, 74, 1, 22, 60, 20, 14, 17, 18, 76, 16, 21, 51, 28, 75, 13, 115, 64, 88, 9, 11, 70, 73, 90, 84, 102, 29, 57, 126, 2, 82, 15, 86, 83, 87, 125, 24, 66, 34, 30, 91, 26, 35, 85, 120, 92, 94, 32, 81, 56, 55, 122, 58, 116, 98, 95, 52, 62, 31, 111, 97, 106, 50, 27, 124, 96, 118, 123, 48, 110, 40, 47, 36, 127, 99, 33, 108, 112, 113, 49, 54, 37, 117, 39, 63, 104, 46, 100, 105, 42, 103, 59, 121, 44, 53, 101, 61, 114, 41, 119], [109, 43, 60, 45, 107, 123, 34, 50, 58, 62, 115, 56, 35, 113, 110, 57, 122, 48, 63, 108, 98, 126, 47, 127, 112, 124, 119, 55, 111, 117, 49, 121, 59, 53, 114, 52, 118, 54, 39, 125, 61, 116, 44, 51, 37, 120, 106, 46, 92, 40, 42, 36, 105, 41, 103, 104, 33, 94, 38, 101, 100, 31, 97, 99, 86, 82, 95, 29, 87, 96, 16, 22, 30, 81, 78, 89, 32, 91, 84, 28, 27, 10, 93, 76, 77, 20, 85, 13, 8, 83, 24, 11, 80, 88, 23, 19, 21, 102, 25, 90, 79, 14, 17, 71, 74, 26, 18, 5, 68, 15, 9, 12, 70, 73, 66, 1, 2, 3, 72, 7, 4, 69, 6, 75, 0, 67, 65, 64], [105, 97, 118, 23, 90, 84, 79, 82, 117, 13, 93, 33, 47, 9, 54, 111, 29, 41, 10, 11, 71, 85, 20, 22, 25, 15, 17, 26, 18, 74, 56, 83, 89, 77, 75, 87, 121, 80, 21, 91, 19, 81, 86, 116, 7, 6, 92, 76, 32, 38, 30, 95, 88, 27, 3, 78, 5, 49, 48, 53, 16, 12, 69, 52, 94, 24, 31, 96, 112, 8, 108, 73, 61, 57, 46, 43, 102, 124, 4, 126, 58, 35, 100, 113, 99, 14, 28, 68, 42, 110, 107, 101, 123, 115, 55, 63, 37, 106, 119, 39, 62, 51, 103, 98, 109, 34, 67, 114, 120, 127, 60, 72, 40, 70, 125, 59, 122, 44, 45, 104, 50, 36, 1, 65, 66, 2, 0, 64], [105, 118, 64, 0, 2, 97, 1, 79, 65, 13, 66, 67, 41, 71, 4, 84, 29, 11, 26, 10, 9, 82, 69, 68, 23, 117, 70, 111, 6, 54, 87, 101, 7, 3, 76, 74, 78, 20, 8, 5, 73, 18, 12, 90, 121, 89, 75, 86, 77, 16, 93, 15, 14, 25, 38, 115, 47, 55, 17, 45, 80, 83, 37, 72, 58, 59, 48, 92, 22, 51, 39, 114, 85, 56, 49, 53, 34, 21, 81, 91, 88, 27, 120, 19, 24, 62, 46, 98, 100, 110, 124, 44, 28, 116, 96, 119, 106, 42, 94, 57, 102, 40, 61, 60, 126, 107, 127, 52, 31, 30, 103, 109, 113, 108, 122, 125, 63, 36, 43, 104, 95, 112, 123, 99, 35, 50, 32, 33], [105, 47, 118, 93, 117, 111, 90, 89, 97, 101, 116, 56, 85, 54, 23, 41, 25, 80, 33, 52, 121, 32, 96, 61, 37, 55, 24, 17, 49, 84, 31, 29, 53, 123, 21, 99, 91, 58, 98, 88, 95, 75, 35, 92, 60, 113, 34, 77, 126, 100, 12, 18, 127, 48, 30, 36, 45, 115, 108, 39, 73, 59, 112, 102, 15, 28, 43, 107, 26, 38, 27, 110, 81, 46, 114, 82, 16, 40, 103, 106, 87, 63, 94, 120, 104, 20, 62, 8, 44, 42, 122, 69, 109, 124, 7, 119, 74, 125, 51, 57, 50, 78, 11, 86, 19, 79, 83, 70, 22, 14, 67, 76, 72, 68, 13, 9, 10, 71, 5, 4, 6, 3, 66, 65, 2, 1, 0, 64], [105, 47, 118, 93, 117, 101, 90, 97, 116, 111, 54, 56, 52, 123, 89, 24, 85, 33, 55, 61, 23, 53, 58, 21, 49, 60, 121, 17, 37, 59, 75, 96, 31, 113, 115, 25, 112, 120, 106, 126, 12, 80, 114, 57, 127, 107, 48, 29, 41, 43, 77, 46, 110, 108, 45, 63, 62, 27, 124, 99, 73, 109, 40, 44, 51, 119, 42, 98, 38, 39, 87, 103, 102, 84, 50, 122, 125, 104, 32, 91, 34, 81, 26, 36, 20, 95, 16, 35, 18, 100, 30, 15, 69, 8, 7, 92, 86, 94, 28, 22, 88, 78, 74, 72, 82, 14, 70, 83, 79, 11, 10, 71, 67, 76, 19, 13, 9, 4, 68, 5, 6, 3, 66, 2, 65, 1, 64, 0], [103, 42, 34, 30, 23, 85, 106, 83, 81, 127, 70, 78, 50, 16, 39, 94, 53, 10, 41, 82, 12, 15, 4, 112, 13, 74, 20, 54, 25, 5, 8, 66, 38, 48, 19, 117, 63, 76, 93, 121, 72, 90, 88, 123, 111, 124, 24, 32, 107, 14, 118, 40, 17, 73, 55, 1, 87, 115, 75, 105, 80, 125, 21, 26, 67, 60, 29, 47, 52, 22, 101, 113, 58, 95, 36, 84, 126, 120, 11, 97, 6, 49, 100, 104, 62, 79, 59, 86, 46, 43, 45, 122, 7, 69, 102, 44, 109, 89, 33, 3, 99, 57, 77, 37, 108, 98, 27, 114, 28, 110, 18, 68, 96, 71, 119, 56, 116, 51, 35, 31, 61, 91, 65, 9, 64, 92, 0, 2], [103, 42, 34, 30, 85, 106, 23, 81, 83, 53, 50, 94, 78, 41, 127, 16, 39, 62, 15, 121, 54, 38, 44, 112, 125, 25, 12, 124, 74, 93, 63, 22, 47, 87, 13, 48, 82, 118, 111, 60, 11, 24, 5, 33, 113, 59, 109, 40, 117, 8, 55, 80, 70, 72, 107, 122, 115, 114, 123, 51, 49, 95, 32, 56, 4, 76, 110, 46, 21, 101, 45, 119, 28, 36, 58, 116, 90, 10, 57, 120, 126, 79, 29, 61, 104, 91, 108, 99, 26, 97, 92, 19, 86, 37, 20, 96, 84, 52, 102, 89, 100, 105, 43, 17, 1, 35, 75, 3, 31, 77, 14, 27, 88, 7, 98, 66, 18, 67, 6, 71, 73, 9, 64, 68, 69, 65, 0, 2], [103, 42, 34, 30, 106, 50, 85, 81, 54, 41, 23, 16, 83, 53, 112, 48, 121, 38, 44, 95, 4, 70, 94, 124, 13, 127, 20, 26, 63, 123, 125, 113, 93, 52, 60, 10, 62, 109, 59, 29, 115, 47, 78, 15, 122, 19, 76, 111, 31, 87, 8, 46, 28, 25, 45, 118, 40, 92, 82, 32, 90, 66, 58, 61, 55, 97, 110, 88, 114, 43, 105, 36, 117, 107, 74, 51, 100, 101, 119, 22, 49, 116, 37, 126, 33, 9, 80, 89, 86, 104, 99, 120, 73, 56, 57, 5, 24, 84, 21, 27, 35, 17, 96, 102, 69, 108, 67, 91, 12, 1, 79, 14, 11, 65, 75, 39, 68, 72, 18, 7, 77, 98, 71, 6, 2, 0, 3, 64], [103, 42, 34, 30, 83, 81, 85, 106, 12, 23, 50, 74, 72, 15, 78, 127, 16, 54, 69, 5, 13, 4, 113, 48, 60, 124, 8, 39, 76, 41, 1, 2, 24, 53, 80, 87, 11, 55, 17, 65, 21, 121, 94, 6, 63, 68, 3, 45, 19, 25, 62, 32, 38, 75, 126, 70, 52, 82, 93, 9, 84, 20, 125, 67, 123, 90, 14, 115, 26, 79, 44, 22, 10, 112, 104, 77, 66, 96, 95, 120, 91, 97, 117, 86, 100, 28, 98, 0, 73, 88, 71, 36, 59, 111, 89, 108, 64, 18, 29, 114, 105, 61, 7, 47, 101, 27, 99, 43, 107, 122, 31, 35, 92, 118, 51, 119, 33, 102, 58, 116, 49, 46, 109, 110, 40, 37, 57, 56], [104, 121, 100, 33, 17, 13, 21, 23, 79, 83, 90, 122, 72, 11, 8, 26, 28, 67, 112, 37, 30, 60, 59, 6, 61, 126, 45, 80, 53, 22, 115, 125, 81, 48, 109, 20, 4, 10, 51, 42, 18, 14, 46, 86, 19, 15, 40, 77, 96, 78, 34, 97, 75, 69, 99, 85, 27, 5, 1, 41, 29, 74, 39, 0, 16, 87, 94, 50, 25, 118, 111, 24, 68, 82, 12, 92, 84, 101, 73, 52, 103, 32, 117, 98, 66, 93, 95, 110, 113, 123, 47, 62, 31, 7, 107, 71, 76, 9, 70, 89, 2, 58, 108, 102, 124, 106, 55, 54, 63, 91, 88, 116, 3, 105, 64, 127, 120, 35, 49, 44, 119, 43, 57, 38, 65, 56, 114, 36], [104, 100, 121, 33, 28, 90, 21, 23, 17, 13, 109, 112, 42, 83, 115, 92, 48, 122, 30, 79, 26, 116, 18, 86, 118, 37, 61, 123, 45, 95, 117, 51, 44, 108, 39, 101, 58, 11, 120, 52, 34, 35, 107, 60, 110, 29, 124, 96, 41, 98, 25, 22, 32, 31, 82, 126, 80, 53, 16, 24, 62, 59, 27, 54, 103, 125, 43, 111, 57, 87, 119, 46, 6, 114, 36, 56, 105, 84, 106, 63, 102, 50, 85, 49, 19, 97, 55, 127, 20, 113, 99, 91, 14, 88, 93, 81, 38, 94, 10, 47, 77, 75, 72, 89, 74, 78, 12, 76, 70, 40, 67, 3, 8, 15, 69, 7, 9, 68, 5, 73, 71, 65, 2, 1, 66, 64, 4, 0], [104, 121, 33, 100, 93, 37, 79, 23, 83, 17, 21, 109, 112, 13, 8, 11, 90, 6, 67, 26, 35, 122, 59, 86, 40, 46, 3, 1, 125, 126, 71, 30, 82, 61, 45, 81, 48, 28, 42, 51, 85, 115, 60, 29, 9, 18, 4, 92, 43, 15, 52, 77, 74, 101, 25, 80, 22, 34, 65, 105, 75, 107, 19, 111, 50, 108, 10, 94, 69, 20, 78, 58, 16, 120, 39, 62, 47, 7, 119, 123, 88, 70, 96, 73, 99, 98, 31, 87, 57, 14, 32, 66, 106, 49, 102, 5, 103, 116, 110, 53, 38, 114, 76, 68, 63, 55, 124, 64, 56, 27, 113, 127, 12, 118, 84, 41, 117, 24, 91, 54, 95, 89, 44, 2, 72, 36, 97, 0], [121, 104, 100, 33, 28, 112, 122, 90, 21, 23, 83, 37, 93, 13, 116, 126, 30, 45, 92, 91, 125, 101, 79, 52, 17, 86, 115, 61, 60, 59, 46, 26, 51, 109, 35, 82, 85, 57, 48, 18, 123, 42, 118, 105, 19, 15, 97, 43, 39, 11, 54, 29, 107, 111, 103, 22, 32, 84, 31, 53, 14, 117, 95, 47, 96, 38, 120, 58, 119, 41, 108, 49, 106, 113, 110, 87, 27, 127, 99, 25, 70, 55, 34, 44, 124, 94, 6, 114, 89, 62, 36, 102, 24, 63, 75, 9, 50, 77, 98, 81, 72, 78, 20, 74, 56, 88, 10, 12, 16, 3, 80, 67, 71, 7, 76, 73, 2, 68, 8, 40, 69, 5, 66, 64, 1, 65, 4, 0], [37, 40, 105, 93, 55, 30, 124, 14, 18, 87, 63, 20, 94, 126, 48, 29, 88, 49, 104, 41, 60, 16, 117, 43, 58, 76, 89, 50, 62, 51, 57, 116, 53, 52, 39, 61, 111, 72, 119, 102, 38, 109, 100, 42, 44, 120, 59, 112, 56, 46, 122, 121, 110, 54, 114, 45, 113, 127, 123, 103, 115, 125, 27, 47, 108, 106, 118, 96, 107, 95, 86, 21, 98, 82, 26, 99, 36, 32, 33, 31, 35, 34, 23, 97, 24, 101, 69, 77, 81, 90, 25, 85, 10, 83, 92, 91, 78, 15, 28, 12, 84, 13, 74, 22, 17, 9, 80, 5, 6, 8, 79, 11, 75, 19, 1, 73, 3, 65, 71, 70, 66, 64, 2, 7, 67, 68, 4, 0], [105, 37, 40, 93, 87, 41, 18, 16, 76, 14, 20, 124, 29, 55, 69, 63, 72, 10, 57, 94, 88, 60, 49, 1, 61, 126, 15, 48, 23, 77, 117, 90, 114, 58, 27, 96, 74, 116, 89, 52, 78, 25, 12, 62, 122, 24, 81, 26, 13, 8, 30, 79, 82, 50, 102, 6, 84, 3, 68, 119, 112, 95, 86, 51, 100, 80, 104, 43, 83, 46, 17, 53, 21, 44, 91, 56, 59, 120, 71, 22, 39, 92, 19, 107, 97, 85, 45, 28, 11, 123, 4, 70, 101, 111, 38, 32, 75, 54, 5, 67, 108, 31, 121, 9, 34, 36, 65, 103, 33, 73, 35, 0, 2, 127, 7, 110, 113, 125, 98, 109, 42, 64, 115, 118, 99, 47, 66, 106], [105, 37, 40, 93, 104, 29, 55, 18, 87, 14, 16, 20, 63, 124, 76, 72, 49, 30, 60, 126, 27, 57, 94, 48, 117, 61, 88, 10, 114, 69, 77, 116, 89, 62, 50, 56, 100, 24, 90, 51, 96, 102, 119, 112, 86, 52, 15, 54, 58, 23, 127, 122, 12, 95, 8, 84, 53, 38, 79, 43, 21, 26, 92, 111, 82, 34, 108, 107, 113, 13, 32, 120, 80, 91, 25, 81, 98, 110, 28, 22, 103, 121, 44, 123, 19, 31, 39, 59, 45, 1, 115, 33, 101, 125, 47, 36, 35, 106, 46, 85, 99, 118, 42, 78, 17, 97, 109, 83, 41, 74, 6, 71, 70, 11, 3, 75, 68, 9, 5, 2, 4, 65, 66, 73, 7, 0, 64, 67], [40, 37, 105, 63, 93, 14, 16, 87, 10, 20, 18, 76, 55, 69, 29, 72, 57, 104, 30, 124, 3, 60, 41, 94, 74, 58, 61, 1, 26, 114, 88, 49, 15, 13, 23, 84, 24, 12, 126, 90, 17, 67, 95, 112, 70, 11, 71, 82, 78, 62, 116, 8, 117, 48, 52, 122, 81, 6, 22, 68, 0, 102, 7, 75, 83, 50, 77, 79, 46, 56, 98, 123, 80, 96, 4, 53, 27, 65, 89, 43, 21, 103, 86, 5, 39, 85, 119, 101, 100, 113, 51, 44, 47, 45, 25, 38, 28, 59, 92, 32, 19, 127, 111, 107, 91, 9, 120, 73, 110, 34, 64, 36, 115, 54, 108, 31, 109, 99, 35, 2, 121, 97, 118, 42, 66, 125, 33, 106], [105, 99, 89, 96, 48, 54, 83, 92, 84, 16, 14, 25, 11, 112, 22, 19, 75, 78, 72, 27, 77, 18, 80, 115, 41, 69, 81, 32, 24, 10, 36, 108, 30, 5, 44, 85, 93, 26, 13, 88, 104, 63, 119, 117, 29, 82, 15, 17, 73, 118, 20, 124, 94, 6, 9, 66, 62, 67, 49, 79, 12, 28, 91, 70, 126, 33, 23, 56, 90, 59, 21, 120, 58, 87, 76, 31, 71, 7, 74, 8, 65, 37, 4, 0, 1, 38, 98, 97, 43, 127, 46, 95, 2, 68, 60, 64, 86, 3, 51, 116, 107, 52, 47, 101, 110, 123, 50, 35, 100, 102, 109, 40, 53, 34, 122, 103, 125, 114, 121, 106, 61, 39, 42, 113, 45, 111, 57, 55], [44, 36, 104, 54, 92, 100, 95, 48, 105, 108, 28, 47, 96, 53, 32, 98, 37, 26, 27, 123, 40, 24, 97, 60, 127, 51, 112, 91, 58, 55, 85, 88, 52, 78, 93, 31, 33, 38, 21, 90, 113, 107, 49, 99, 124, 10, 110, 120, 115, 18, 89, 63, 30, 45, 34, 83, 62, 82, 56, 111, 94, 29, 122, 102, 84, 106, 118, 74, 103, 116, 117, 50, 46, 43, 22, 81, 101, 121, 119, 61, 114, 17, 39, 23, 57, 109, 59, 16, 86, 79, 126, 14, 87, 35, 70, 42, 41, 80, 5, 125, 9, 15, 75, 2, 20, 3, 66, 64, 12, 7, 73, 1, 77, 19, 76, 6, 4, 68, 0, 13, 25, 11, 67, 69, 72, 71, 8, 65], [48, 54, 37, 105, 108, 104, 36, 107, 47, 115, 63, 119, 95, 50, 44, 56, 112, 114, 118, 94, 127, 30, 117, 62, 110, 52, 55, 123, 116, 121, 53, 38, 58, 51, 124, 40, 59, 111, 113, 92, 34, 101, 120, 60, 125, 32, 46, 122, 126, 109, 57, 61, 100, 49, 24, 102, 33, 45, 98, 106, 43, 103, 28, 21, 99, 85, 42, 39, 96, 31, 41, 15, 97, 25, 29, 87, 19, 18, 81, 11, 23, 88, 12, 90, 73, 77, 17, 84, 26, 75, 91, 8, 93, 79, 22, 78, 7, 20, 5, 82, 80, 4, 35, 2, 0, 1, 64, 76, 6, 3, 14, 86, 74, 72, 70, 83, 71, 9, 89, 69, 27, 10, 16, 65, 68, 66, 67, 13], [54, 105, 48, 36, 123, 41, 25, 44, 127, 98, 99, 112, 34, 96, 115, 50, 47, 52, 60, 121, 118, 57, 117, 122, 64, 107, 63, 114, 108, 74, 119, 45, 120, 56, 22, 30, 113, 55, 8, 126, 111, 51, 58, 53, 62, 106, 110, 116, 125, 1, 124, 104, 59, 37, 61, 94, 80, 100, 68, 43, 75, 97, 109, 84, 67, 49, 81, 14, 38, 77, 6, 46, 42, 73, 103, 40, 5, 39, 79, 2, 24, 18, 4, 16, 3, 92, 89, 71, 21, 69, 12, 102, 28, 29, 33, 15, 20, 101, 88, 26, 85, 86, 7, 17, 31, 27, 95, 82, 91, 35, 32, 87, 13, 93, 76, 70, 19, 23, 90, 9, 0, 83, 11, 65, 78, 10, 66, 72], [37, 114, 49, 50, 54, 113, 94, 91, 101, 28, 118, 84, 61, 119, 25, 62, 45, 58, 11, 32, 112, 56, 87, 60, 29, 22, 124, 15, 31, 14, 90, 79, 116, 96, 81, 48, 33, 97, 82, 55, 24, 47, 110, 83, 36, 35, 117, 46, 95, 93, 109, 53, 98, 51, 99, 27, 18, 126, 127, 34, 30, 39, 115, 107, 102, 38, 40, 43, 63, 100, 52, 104, 44, 21, 105, 57, 125, 122, 123, 103, 92, 121, 41, 59, 120, 42, 8, 16, 108, 89, 106, 20, 88, 85, 26, 111, 76, 23, 86, 9, 75, 78, 5, 13, 19, 12, 6, 17, 80, 4, 10, 73, 7, 77, 2, 74, 3, 1, 71, 66, 72, 0, 70, 68, 69, 64, 65, 67], [114, 54, 49, 113, 50, 124, 119, 118, 109, 62, 123, 61, 56, 48, 47, 55, 112, 53, 58, 36, 127, 46, 37, 120, 63, 45, 126, 52, 60, 107, 125, 44, 115, 51, 111, 121, 117, 110, 122, 116, 59, 43, 57, 108, 17, 42, 105, 39, 106, 41, 101, 104, 20, 102, 38, 40, 82, 80, 30, 96, 19, 75, 79, 103, 86, 77, 78, 76, 95, 85, 24, 93, 100, 27, 97, 98, 99, 35, 34, 33, 74, 18, 32, 89, 72, 28, 71, 9, 23, 29, 14, 90, 15, 31, 92, 94, 87, 12, 88, 69, 70, 66, 13, 64, 65, 91, 84, 21, 81, 67, 26, 73, 83, 11, 25, 68, 22, 16, 6, 10, 4, 5, 1, 3, 8, 7, 0, 2], [37, 49, 114, 50, 54, 113, 101, 94, 62, 25, 91, 11, 84, 83, 119, 87, 81, 118, 32, 79, 8, 56, 5, 15, 76, 9, 28, 22, 21, 45, 6, 60, 4, 58, 61, 7, 124, 13, 110, 16, 33, 75, 112, 85, 14, 10, 18, 30, 116, 3, 82, 90, 46, 24, 1, 73, 53, 23, 96, 77, 89, 109, 78, 117, 2, 63, 74, 125, 47, 12, 80, 52, 31, 126, 20, 17, 71, 48, 93, 57, 127, 66, 19, 115, 98, 105, 55, 27, 36, 44, 86, 29, 88, 40, 35, 95, 26, 43, 97, 122, 92, 72, 111, 68, 59, 38, 123, 41, 0, 51, 107, 34, 103, 39, 121, 42, 120, 102, 99, 106, 108, 100, 104, 70, 69, 65, 64, 67], [49, 37, 114, 54, 50, 113, 5, 9, 76, 6, 8, 7, 94, 4, 77, 79, 75, 12, 87, 16, 3, 25, 30, 22, 73, 11, 74, 83, 78, 80, 82, 17, 124, 2, 81, 71, 72, 91, 10, 45, 19, 56, 60, 62, 119, 61, 112, 1, 32, 118, 13, 15, 58, 84, 85, 63, 53, 26, 101, 21, 20, 117, 110, 109, 55, 23, 90, 93, 92, 66, 14, 89, 116, 31, 88, 46, 29, 48, 18, 107, 28, 24, 27, 125, 102, 127, 115, 57, 0, 126, 36, 52, 51, 47, 86, 123, 44, 70, 96, 121, 68, 43, 33, 42, 59, 39, 122, 104, 111, 120, 98, 67, 40, 69, 41, 100, 105, 35, 34, 38, 108, 97, 95, 103, 106, 65, 99, 64], [41, 101, 25, 23, 85, 110, 18, 105, 15, 34, 13, 20, 11, 91, 124, 30, 113, 9, 14, 83, 43, 8, 109, 24, 126, 94, 82, 68, 6, 51, 84, 48, 107, 118, 92, 29, 27, 59, 112, 111, 57, 4, 45, 77, 114, 44, 87, 88, 122, 89, 36, 53, 62, 17, 26, 98, 79, 31, 32, 16, 76, 80, 58, 117, 78, 103, 90, 96, 115, 66, 50, 75, 72, 21, 67, 38, 56, 2, 99, 55, 10, 47, 95, 37, 121, 0, 61, 52, 69, 5, 102, 120, 86, 108, 19, 28, 63, 60, 46, 22, 116, 106, 39, 123, 35, 127, 54, 65, 104, 71, 70, 1, 7, 97, 73, 119, 93, 12, 3, 33, 74, 40, 49, 42, 81, 100, 125, 64], [41, 101, 25, 85, 18, 23, 20, 105, 15, 11, 34, 13, 91, 124, 14, 9, 6, 51, 113, 2, 118, 8, 82, 112, 114, 29, 27, 53, 83, 98, 84, 68, 81, 109, 75, 48, 78, 32, 43, 107, 89, 30, 66, 87, 92, 111, 94, 126, 70, 36, 90, 45, 79, 110, 17, 44, 88, 122, 62, 57, 59, 117, 21, 7, 99, 103, 60, 127, 47, 76, 19, 38, 26, 42, 31, 24, 120, 52, 58, 16, 46, 69, 108, 77, 0, 35, 119, 72, 61, 115, 56, 80, 37, 22, 123, 102, 55, 63, 121, 10, 86, 95, 28, 67, 96, 97, 125, 100, 106, 39, 71, 54, 116, 50, 73, 93, 5, 74, 12, 40, 33, 65, 3, 49, 4, 1, 104, 64], [41, 101, 109, 110, 91, 25, 20, 18, 34, 85, 105, 23, 14, 124, 11, 15, 9, 13, 43, 53, 24, 113, 82, 84, 29, 8, 107, 45, 27, 126, 83, 6, 111, 90, 127, 59, 68, 98, 112, 51, 28, 44, 120, 78, 56, 47, 121, 118, 117, 114, 116, 62, 50, 106, 122, 55, 57, 92, 58, 95, 93, 94, 81, 36, 52, 97, 16, 87, 32, 30, 60, 102, 108, 42, 99, 38, 66, 75, 26, 40, 96, 119, 88, 63, 67, 33, 31, 89, 35, 37, 48, 80, 123, 115, 70, 61, 49, 46, 39, 104, 71, 125, 2, 22, 103, 76, 77, 86, 12, 54, 17, 79, 0, 4, 19, 21, 72, 7, 10, 5, 69, 100, 73, 65, 74, 1, 3, 64], [41, 101, 25, 85, 91, 105, 23, 15, 18, 20, 34, 13, 98, 113, 124, 81, 110, 29, 30, 11, 48, 95, 126, 43, 92, 114, 27, 9, 53, 14, 109, 84, 36, 44, 94, 24, 118, 45, 35, 26, 83, 100, 111, 89, 51, 107, 46, 57, 82, 120, 76, 123, 112, 61, 119, 38, 40, 103, 122, 121, 31, 56, 87, 116, 8, 62, 60, 50, 28, 106, 99, 47, 127, 125, 39, 80, 97, 117, 78, 42, 33, 6, 32, 58, 54, 59, 86, 102, 16, 88, 73, 21, 55, 63, 77, 22, 17, 96, 115, 52, 49, 79, 108, 4, 93, 12, 90, 104, 19, 68, 37, 66, 75, 10, 70, 74, 72, 5, 67, 7, 71, 1, 2, 69, 64, 3, 0, 65]], "model.layers.7.self_attn.k_proj": [[45, 107, 102, 0, 109, 60, 1, 12, 3, 77, 70, 19, 71, 43, 10, 8, 78, 115, 80, 29, 68, 22, 25, 5, 18, 17, 11, 125, 2, 73, 57, 126, 56, 48, 122, 111, 124, 79, 55, 110, 113, 63, 20, 66, 58, 67, 52, 47, 127, 62, 117, 49, 59, 114, 112, 96, 121, 51, 44, 54, 23, 118, 6, 119, 61, 92, 123, 53, 27, 95, 46, 116, 98, 50, 120, 106, 69, 89, 100, 64, 65, 21, 4, 108, 75, 9, 7, 41, 42, 105, 40, 39, 87, 30, 35, 24, 104, 94, 103, 36, 97, 31, 26, 101, 84, 37, 32, 33, 99, 90, 28, 85, 88, 81, 72, 93, 34, 15, 74, 76, 91, 82, 14, 16, 83, 13, 86, 38], [41, 118, 64, 111, 117, 33, 79, 82, 84, 1, 13, 23, 90, 93, 56, 71, 2, 11, 9, 4, 89, 85, 10, 105, 17, 65, 5, 37, 47, 121, 6, 30, 76, 67, 14, 116, 32, 80, 66, 72, 101, 22, 24, 115, 0, 48, 19, 28, 59, 29, 127, 44, 126, 49, 95, 110, 3, 26, 55, 112, 88, 83, 16, 38, 43, 35, 94, 52, 98, 27, 18, 46, 81, 40, 120, 107, 12, 100, 21, 102, 62, 124, 119, 61, 106, 113, 99, 69, 34, 53, 91, 123, 97, 103, 96, 42, 63, 57, 39, 51, 109, 114, 60, 36, 31, 58, 73, 104, 108, 54, 8, 122, 125, 68, 50, 25, 45, 78, 75, 92, 20, 77, 87, 86, 15, 74, 7, 70], [106, 39, 94, 98, 85, 23, 83, 127, 74, 12, 81, 78, 72, 16, 15, 50, 112, 65, 68, 42, 6, 41, 53, 69, 118, 2, 114, 49, 13, 0, 124, 88, 77, 18, 52, 111, 55, 8, 79, 38, 126, 125, 115, 34, 29, 36, 102, 109, 108, 120, 54, 7, 64, 107, 31, 40, 63, 110, 9, 47, 26, 101, 75, 60, 48, 3, 71, 61, 35, 122, 67, 117, 104, 25, 22, 58, 100, 43, 119, 62, 80, 113, 66, 56, 5, 11, 121, 14, 73, 105, 90, 57, 33, 123, 44, 92, 103, 46, 89, 24, 86, 116, 93, 37, 82, 27, 59, 99, 96, 19, 91, 45, 32, 95, 84, 28, 20, 97, 30, 76, 10, 17, 4, 51, 1, 87, 70, 21], [40, 121, 97, 90, 23, 21, 83, 36, 79, 48, 17, 82, 11, 13, 86, 8, 122, 73, 0, 30, 78, 50, 51, 107, 6, 109, 116, 69, 60, 59, 10, 65, 115, 58, 110, 101, 45, 29, 9, 4, 16, 68, 53, 123, 28, 84, 93, 66, 67, 39, 100, 126, 61, 37, 27, 42, 12, 89, 55, 124, 112, 3, 56, 96, 32, 47, 57, 70, 120, 125, 117, 103, 119, 2, 114, 95, 94, 35, 98, 14, 74, 54, 92, 106, 46, 43, 75, 88, 99, 41, 111, 105, 62, 52, 118, 49, 108, 34, 63, 38, 127, 91, 113, 44, 5, 31, 102, 15, 19, 76, 1, 71, 7, 18, 22, 24, 64, 25, 20, 72, 80, 77, 85, 26, 81, 87, 104, 33], [104, 41, 101, 29, 87, 55, 20, 63, 18, 16, 76, 10, 14, 60, 37, 57, 7, 0, 65, 72, 77, 67, 52, 69, 112, 58, 124, 113, 114, 71, 61, 66, 5, 86, 30, 17, 9, 126, 24, 15, 27, 75, 8, 93, 123, 4, 12, 102, 89, 62, 43, 64, 94, 21, 53, 2, 90, 70, 83, 13, 85, 39, 79, 19, 96, 88, 56, 100, 117, 3, 107, 116, 108, 45, 127, 82, 6, 91, 46, 111, 119, 47, 11, 121, 81, 23, 51, 54, 25, 49, 120, 73, 95, 32, 68, 42, 59, 26, 78, 34, 36, 28, 122, 80, 22, 31, 98, 92, 110, 84, 109, 103, 97, 106, 118, 44, 35, 99, 115, 74, 38, 48, 50, 125, 33, 40, 1, 105], [41, 54, 48, 35, 32, 22, 93, 16, 44, 127, 40, 10, 112, 30, 90, 89, 78, 84, 124, 83, 3, 70, 77, 27, 115, 123, 113, 51, 18, 117, 25, 52, 55, 101, 120, 111, 24, 7, 59, 121, 76, 119, 110, 63, 5, 122, 60, 50, 62, 126, 114, 58, 57, 43, 11, 118, 108, 31, 116, 81, 125, 1, 61, 56, 53, 109, 47, 19, 45, 100, 72, 79, 91, 46, 103, 102, 42, 39, 106, 9, 49, 2, 23, 107, 4, 80, 64, 73, 0, 13, 74, 66, 20, 12, 8, 33, 28, 21, 86, 104, 65, 38, 14, 26, 71, 87, 98, 34, 92, 15, 17, 85, 95, 68, 97, 29, 88, 36, 105, 96, 37, 94, 82, 75, 6, 99, 69, 67], [50, 101, 113, 49, 54, 94, 114, 22, 91, 87, 25, 21, 83, 81, 74, 84, 13, 18, 16, 86, 14, 72, 56, 71, 80, 110, 69, 77, 15, 32, 112, 58, 70, 60, 67, 66, 53, 117, 124, 26, 88, 92, 12, 45, 119, 68, 52, 55, 78, 62, 63, 125, 109, 9, 127, 73, 61, 64, 57, 4, 37, 51, 43, 126, 107, 11, 118, 115, 48, 19, 100, 123, 98, 116, 31, 47, 65, 121, 59, 75, 111, 120, 42, 29, 104, 102, 95, 6, 105, 44, 41, 103, 89, 76, 46, 40, 108, 122, 39, 1, 23, 97, 17, 38, 106, 3, 27, 10, 34, 35, 36, 99, 33, 85, 24, 30, 28, 96, 93, 79, 90, 20, 5, 8, 82, 7, 2, 0], [105, 37, 23, 18, 25, 15, 20, 11, 14, 85, 101, 13, 124, 91, 113, 34, 9, 46, 49, 8, 48, 126, 6, 1, 41, 118, 43, 68, 45, 115, 2, 111, 74, 110, 94, 16, 0, 63, 119, 114, 51, 83, 24, 62, 98, 64, 70, 59, 35, 53, 109, 92, 117, 58, 81, 102, 12, 65, 90, 55, 31, 10, 30, 57, 67, 3, 44, 29, 36, 61, 120, 7, 4, 32, 112, 103, 122, 108, 27, 116, 99, 96, 77, 50, 26, 86, 72, 107, 106, 121, 19, 95, 104, 93, 88, 56, 75, 71, 42, 125, 40, 54, 22, 38, 100, 123, 52, 97, 39, 5, 17, 79, 33, 69, 127, 47, 80, 28, 73, 60, 66, 21, 76, 78, 89, 84, 87, 82]], "model.layers.7.self_attn.qk_proj": [[121, 41, 54, 118, 105, 50, 37, 45, 106, 48, 101, 49, 109, 114, 107, 104, 113, 93, 43, 23, 87, 60, 30, 42, 89, 77, 40, 85, 25, 82, 83, 13, 14, 18, 21, 26, 78, 117, 19, 16, 112, 10, 111, 124, 79, 15, 20, 34, 63, 55, 81, 84, 74, 80, 103, 72, 17, 56, 119, 115, 94, 76, 29, 12, 39, 100, 86, 33, 57, 110, 75, 122, 126, 27, 47, 127, 52, 44, 125, 0, 11, 22, 53, 102, 90, 8, 58, 123, 32, 97, 64, 61, 7, 67, 69, 5, 62, 9, 59, 71, 51, 65, 73, 70, 98, 3, 4, 91, 120, 116, 38, 96, 6, 35, 24, 36, 46, 1, 88, 68, 31, 92, 108, 28, 2, 95, 99, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 48, 49, 101, 109, 107, 114, 104, 43, 113, 93, 23, 30, 87, 40, 60, 42, 25, 21, 18, 85, 77, 19, 13, 14, 82, 89, 83, 26, 112, 111, 84, 103, 79, 78, 16, 10, 63, 80, 117, 15, 34, 55, 124, 94, 17, 20, 74, 12, 81, 72, 115, 119, 100, 56, 86, 39, 76, 122, 53, 29, 11, 125, 33, 47, 110, 127, 27, 61, 57, 22, 102, 75, 0, 52, 90, 126, 8, 97, 62, 64, 123, 32, 51, 44, 58, 98, 69, 7, 9, 4, 5, 6, 1, 65, 71, 67, 36, 91, 120, 35, 116, 73, 59, 68, 108, 70, 38, 3, 24, 96, 88, 2, 99, 28, 66, 46, 92, 31, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 114, 109, 49, 107, 104, 113, 93, 23, 43, 30, 87, 42, 60, 40, 25, 89, 85, 21, 77, 26, 18, 19, 82, 13, 117, 55, 103, 14, 94, 83, 112, 84, 34, 111, 78, 81, 124, 20, 79, 100, 16, 10, 74, 15, 29, 80, 33, 39, 86, 63, 17, 110, 76, 97, 47, 12, 122, 90, 56, 53, 115, 72, 57, 8, 27, 119, 22, 127, 102, 52, 32, 75, 0, 64, 11, 58, 62, 69, 126, 125, 6, 98, 7, 5, 1, 44, 65, 61, 68, 123, 51, 71, 91, 3, 108, 9, 96, 4, 59, 36, 35, 67, 46, 116, 73, 24, 99, 38, 31, 88, 120, 70, 2, 66, 28, 95, 92], [121, 118, 41, 105, 54, 50, 45, 37, 106, 49, 48, 101, 114, 109, 107, 104, 93, 113, 43, 87, 23, 30, 89, 40, 42, 60, 25, 82, 26, 85, 117, 77, 112, 21, 18, 103, 55, 78, 127, 83, 13, 19, 79, 119, 20, 94, 84, 34, 124, 15, 63, 14, 111, 53, 100, 81, 10, 16, 47, 80, 17, 74, 29, 39, 27, 86, 12, 33, 102, 110, 64, 125, 11, 44, 115, 76, 126, 32, 52, 56, 58, 90, 97, 122, 22, 8, 72, 0, 51, 57, 69, 71, 62, 75, 5, 6, 67, 65, 116, 3, 73, 61, 91, 59, 7, 98, 120, 68, 1, 36, 35, 9, 123, 96, 4, 2, 46, 31, 38, 70, 88, 28, 66, 108, 92, 24, 95, 99], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 87, 23, 42, 40, 89, 30, 60, 77, 26, 18, 85, 82, 21, 117, 25, 55, 13, 34, 111, 112, 14, 103, 83, 78, 10, 84, 94, 16, 63, 47, 74, 15, 19, 20, 79, 124, 76, 81, 8, 80, 100, 39, 29, 17, 12, 119, 102, 115, 11, 33, 57, 56, 86, 64, 53, 0, 126, 110, 75, 32, 65, 22, 127, 90, 97, 122, 68, 69, 72, 62, 27, 58, 61, 44, 52, 5, 98, 4, 125, 7, 73, 3, 6, 9, 51, 67, 38, 91, 71, 1, 116, 59, 66, 24, 70, 123, 108, 35, 2, 36, 88, 120, 96, 46, 92, 28, 99, 31, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 101, 48, 49, 114, 107, 109, 104, 43, 113, 93, 87, 23, 60, 42, 77, 85, 82, 89, 40, 30, 25, 18, 21, 26, 14, 78, 13, 19, 112, 79, 83, 111, 16, 103, 84, 117, 74, 20, 15, 63, 34, 17, 10, 11, 8, 81, 80, 124, 12, 76, 100, 115, 94, 47, 55, 86, 126, 53, 39, 125, 29, 57, 127, 110, 72, 61, 52, 75, 0, 119, 33, 102, 58, 62, 27, 22, 44, 64, 122, 32, 56, 7, 9, 120, 97, 3, 90, 91, 123, 68, 6, 73, 5, 69, 51, 4, 71, 70, 65, 59, 1, 67, 98, 24, 35, 66, 116, 36, 38, 96, 108, 2, 92, 88, 28, 46, 31, 99, 95], [121, 41, 54, 105, 118, 50, 37, 45, 101, 106, 49, 48, 114, 109, 107, 23, 104, 113, 93, 87, 43, 30, 82, 42, 60, 77, 18, 21, 25, 14, 89, 85, 13, 40, 78, 26, 16, 19, 15, 84, 117, 111, 83, 79, 20, 103, 63, 17, 34, 74, 8, 10, 81, 112, 80, 12, 76, 47, 94, 124, 29, 33, 110, 27, 11, 115, 86, 39, 57, 55, 52, 100, 22, 125, 126, 75, 119, 122, 127, 56, 58, 44, 90, 61, 91, 32, 102, 72, 53, 0, 59, 62, 97, 73, 69, 123, 98, 7, 96, 5, 64, 9, 71, 51, 38, 68, 6, 3, 116, 92, 35, 36, 24, 1, 88, 67, 70, 4, 120, 65, 46, 2, 31, 28, 108, 66, 95, 99], [121, 41, 105, 118, 54, 50, 45, 106, 37, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 23, 42, 87, 82, 40, 18, 21, 30, 13, 85, 19, 77, 60, 89, 26, 14, 78, 25, 79, 16, 84, 83, 15, 103, 80, 17, 20, 74, 12, 81, 8, 76, 34, 117, 10, 124, 94, 111, 29, 86, 11, 63, 112, 55, 39, 119, 27, 75, 100, 57, 127, 33, 115, 22, 61, 44, 102, 0, 47, 64, 90, 1, 125, 52, 5, 91, 110, 72, 69, 58, 32, 123, 122, 70, 56, 7, 24, 53, 97, 65, 62, 126, 59, 71, 98, 73, 4, 88, 67, 9, 96, 38, 51, 36, 116, 68, 92, 3, 2, 6, 46, 120, 31, 35, 95, 66, 108, 28, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 101, 49, 109, 107, 114, 104, 43, 93, 113, 23, 87, 40, 42, 30, 89, 21, 19, 18, 82, 25, 85, 60, 26, 84, 13, 14, 111, 77, 16, 83, 94, 63, 20, 79, 103, 117, 15, 17, 78, 112, 55, 34, 80, 12, 74, 124, 81, 10, 57, 8, 29, 53, 76, 33, 127, 47, 110, 52, 27, 39, 86, 119, 11, 126, 90, 125, 58, 100, 22, 102, 97, 44, 51, 32, 62, 91, 115, 75, 56, 0, 64, 98, 7, 61, 123, 70, 122, 4, 72, 5, 36, 65, 71, 88, 116, 59, 69, 73, 68, 67, 38, 9, 24, 1, 96, 120, 35, 92, 3, 6, 108, 28, 66, 31, 95, 99, 46, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 104, 113, 93, 43, 87, 23, 30, 60, 42, 25, 82, 21, 85, 40, 89, 13, 18, 111, 19, 16, 14, 77, 117, 83, 26, 84, 15, 20, 103, 79, 78, 34, 112, 17, 55, 80, 10, 81, 94, 74, 76, 8, 12, 115, 124, 102, 119, 11, 29, 126, 63, 27, 39, 33, 100, 86, 125, 47, 127, 57, 44, 56, 53, 110, 90, 122, 61, 51, 32, 64, 58, 91, 22, 0, 97, 75, 69, 72, 70, 62, 52, 71, 123, 73, 3, 98, 5, 65, 9, 116, 36, 67, 1, 7, 4, 92, 88, 68, 96, 59, 6, 120, 28, 24, 35, 66, 38, 46, 31, 99, 108, 95, 2], [121, 41, 118, 54, 105, 50, 37, 45, 106, 48, 49, 101, 114, 109, 107, 104, 113, 43, 93, 23, 42, 87, 30, 40, 13, 82, 25, 85, 89, 60, 77, 21, 18, 111, 55, 83, 117, 15, 16, 26, 79, 14, 63, 20, 78, 34, 119, 112, 19, 10, 103, 84, 74, 115, 47, 94, 80, 53, 127, 81, 17, 12, 52, 8, 39, 29, 110, 76, 86, 11, 33, 100, 72, 64, 102, 126, 122, 57, 124, 56, 44, 69, 62, 61, 51, 75, 125, 0, 97, 32, 1, 7, 58, 90, 5, 22, 123, 27, 70, 71, 9, 65, 73, 91, 67, 6, 98, 4, 68, 3, 38, 35, 36, 120, 96, 59, 116, 2, 66, 28, 92, 24, 31, 46, 108, 88, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 114, 101, 107, 109, 104, 113, 93, 87, 43, 23, 42, 30, 82, 85, 25, 40, 21, 18, 89, 13, 77, 60, 83, 26, 14, 15, 79, 111, 19, 84, 103, 16, 34, 20, 112, 78, 117, 81, 80, 10, 74, 94, 63, 124, 55, 76, 17, 86, 47, 39, 127, 12, 8, 53, 119, 29, 72, 33, 115, 100, 52, 27, 11, 110, 64, 56, 69, 90, 102, 22, 126, 75, 62, 44, 123, 91, 73, 32, 122, 67, 71, 57, 97, 125, 5, 61, 0, 4, 70, 68, 3, 9, 51, 7, 58, 1, 6, 98, 120, 65, 96, 38, 36, 66, 24, 116, 2, 59, 108, 88, 46, 35, 92, 28, 31, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 101, 106, 48, 49, 109, 114, 107, 104, 113, 23, 93, 43, 87, 60, 30, 18, 42, 40, 25, 89, 21, 82, 13, 85, 19, 16, 26, 14, 83, 77, 78, 124, 84, 79, 103, 117, 111, 94, 15, 34, 81, 80, 20, 72, 74, 17, 112, 63, 10, 12, 115, 76, 29, 55, 119, 86, 100, 33, 27, 47, 90, 11, 126, 125, 39, 127, 110, 53, 75, 44, 102, 8, 56, 57, 52, 62, 97, 91, 22, 32, 61, 58, 122, 59, 64, 6, 4, 123, 73, 69, 0, 71, 68, 7, 65, 9, 3, 24, 36, 5, 51, 96, 98, 116, 70, 67, 38, 46, 92, 88, 120, 35, 31, 108, 1, 2, 66, 28, 95, 99], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 49, 48, 109, 114, 107, 104, 113, 93, 43, 23, 60, 13, 40, 42, 87, 30, 18, 85, 25, 82, 89, 103, 21, 83, 26, 72, 78, 16, 14, 17, 117, 79, 19, 20, 15, 34, 77, 63, 74, 84, 81, 111, 124, 55, 10, 112, 80, 76, 12, 119, 47, 126, 29, 11, 75, 86, 0, 64, 27, 94, 115, 100, 127, 8, 125, 44, 39, 90, 61, 102, 33, 110, 22, 53, 6, 57, 52, 56, 69, 5, 58, 7, 65, 51, 71, 1, 9, 32, 68, 62, 97, 73, 116, 122, 91, 4, 3, 59, 96, 67, 70, 98, 36, 123, 88, 35, 24, 38, 66, 92, 31, 120, 46, 2, 99, 28, 95, 108], [121, 41, 118, 105, 54, 50, 45, 37, 49, 106, 101, 48, 109, 114, 107, 104, 43, 113, 93, 23, 87, 30, 42, 60, 25, 40, 89, 18, 21, 82, 85, 26, 13, 83, 111, 15, 16, 78, 72, 79, 19, 20, 112, 117, 77, 84, 63, 14, 124, 74, 103, 17, 34, 10, 81, 80, 55, 29, 47, 86, 94, 115, 12, 119, 76, 39, 127, 27, 57, 90, 33, 11, 110, 100, 53, 75, 126, 52, 61, 32, 8, 102, 62, 22, 69, 58, 6, 56, 0, 125, 44, 91, 64, 71, 73, 5, 9, 7, 122, 123, 51, 65, 4, 3, 1, 67, 36, 88, 97, 70, 68, 96, 24, 38, 120, 46, 116, 59, 92, 28, 98, 35, 2, 31, 66, 108, 99, 95], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 48, 101, 114, 107, 109, 104, 113, 93, 43, 23, 30, 60, 42, 25, 87, 85, 40, 89, 82, 18, 13, 26, 83, 21, 112, 77, 14, 111, 117, 16, 94, 78, 19, 15, 79, 34, 20, 84, 63, 74, 72, 29, 124, 55, 80, 115, 10, 17, 53, 57, 126, 47, 86, 76, 39, 103, 81, 33, 100, 119, 110, 62, 125, 127, 12, 56, 11, 90, 22, 27, 52, 6, 64, 75, 102, 61, 32, 8, 122, 44, 97, 91, 7, 51, 123, 58, 9, 73, 69, 0, 38, 68, 4, 71, 5, 3, 36, 59, 67, 46, 1, 120, 116, 98, 96, 88, 24, 70, 35, 65, 92, 108, 66, 28, 31, 99, 2, 95], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 114, 109, 48, 107, 104, 113, 23, 93, 87, 60, 43, 42, 30, 25, 89, 18, 82, 85, 13, 83, 40, 21, 26, 16, 117, 19, 78, 14, 77, 84, 81, 79, 34, 15, 74, 112, 72, 20, 124, 80, 10, 111, 94, 17, 100, 12, 115, 39, 55, 76, 119, 103, 63, 110, 33, 11, 29, 90, 57, 86, 47, 127, 123, 27, 53, 102, 58, 22, 52, 56, 8, 125, 44, 32, 75, 122, 0, 91, 126, 9, 7, 71, 97, 62, 61, 5, 69, 68, 51, 6, 98, 64, 120, 1, 70, 36, 59, 4, 108, 38, 116, 73, 35, 65, 88, 24, 46, 96, 92, 67, 2, 31, 3, 99, 95, 28, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 109, 48, 114, 107, 104, 113, 23, 93, 43, 42, 87, 30, 60, 85, 83, 40, 25, 13, 89, 82, 18, 21, 26, 19, 111, 78, 77, 16, 117, 84, 14, 79, 15, 20, 103, 80, 72, 34, 124, 10, 112, 17, 81, 29, 74, 94, 55, 76, 119, 12, 33, 63, 57, 75, 90, 8, 127, 100, 47, 110, 126, 11, 39, 86, 22, 115, 61, 52, 32, 27, 53, 62, 102, 125, 56, 64, 58, 7, 51, 0, 91, 122, 44, 71, 5, 69, 70, 97, 9, 59, 1, 67, 116, 73, 98, 6, 123, 36, 38, 3, 4, 46, 65, 120, 96, 24, 68, 99, 28, 92, 31, 108, 35, 88, 95, 2, 66], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 109, 48, 114, 107, 104, 43, 113, 23, 30, 87, 93, 89, 42, 60, 40, 25, 18, 117, 85, 13, 63, 82, 21, 83, 112, 26, 94, 16, 111, 77, 78, 119, 103, 20, 34, 55, 15, 84, 19, 80, 79, 124, 14, 33, 57, 74, 81, 29, 102, 10, 17, 52, 39, 72, 127, 90, 8, 12, 115, 53, 100, 97, 47, 76, 86, 110, 126, 32, 27, 44, 11, 64, 58, 51, 0, 62, 91, 56, 75, 22, 71, 69, 61, 7, 122, 70, 98, 5, 1, 67, 4, 3, 68, 35, 9, 36, 65, 125, 123, 59, 120, 73, 92, 24, 95, 38, 31, 6, 116, 99, 46, 28, 108, 88, 96, 2, 66], [121, 41, 118, 54, 105, 50, 37, 45, 106, 101, 49, 48, 109, 107, 114, 113, 104, 93, 23, 43, 30, 25, 87, 60, 42, 18, 77, 40, 26, 21, 89, 13, 85, 83, 19, 82, 117, 80, 111, 112, 15, 84, 14, 124, 10, 78, 34, 79, 16, 103, 74, 63, 76, 94, 55, 100, 72, 81, 17, 8, 33, 115, 39, 29, 12, 20, 126, 86, 27, 102, 11, 110, 62, 53, 75, 57, 56, 44, 97, 61, 127, 22, 70, 119, 47, 52, 71, 51, 58, 90, 69, 0, 122, 64, 5, 125, 123, 91, 32, 73, 7, 4, 9, 65, 1, 36, 98, 35, 120, 68, 59, 24, 46, 96, 38, 3, 116, 6, 67, 31, 88, 92, 108, 99, 2, 28, 95, 66], [121, 41, 118, 54, 105, 50, 45, 37, 106, 49, 101, 48, 109, 114, 107, 104, 113, 93, 43, 42, 23, 30, 60, 87, 89, 40, 25, 18, 13, 77, 85, 82, 21, 26, 103, 117, 14, 111, 74, 83, 19, 63, 79, 112, 119, 16, 34, 15, 81, 78, 84, 80, 94, 29, 33, 55, 124, 20, 10, 57, 17, 8, 76, 62, 110, 86, 127, 39, 53, 47, 12, 11, 72, 75, 126, 100, 22, 90, 102, 58, 115, 56, 9, 71, 123, 70, 27, 44, 52, 69, 5, 97, 125, 32, 51, 64, 120, 91, 7, 122, 3, 0, 61, 4, 59, 38, 73, 67, 68, 98, 46, 96, 36, 1, 6, 88, 65, 99, 35, 31, 116, 108, 28, 2, 92, 66, 24, 95], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 48, 101, 109, 114, 107, 104, 113, 87, 93, 23, 43, 42, 30, 60, 89, 40, 25, 82, 85, 26, 13, 18, 21, 77, 19, 117, 83, 14, 79, 103, 124, 34, 16, 112, 78, 111, 63, 15, 74, 84, 119, 29, 94, 33, 55, 20, 17, 110, 80, 81, 10, 76, 86, 8, 127, 12, 100, 39, 90, 115, 57, 75, 102, 47, 126, 52, 32, 53, 91, 62, 27, 56, 22, 44, 58, 11, 64, 97, 72, 61, 51, 123, 0, 71, 5, 59, 125, 69, 9, 70, 1, 98, 122, 116, 73, 38, 7, 88, 120, 46, 96, 67, 36, 4, 3, 35, 65, 68, 28, 92, 31, 99, 6, 24, 95, 2, 108, 66], [121, 41, 118, 54, 105, 50, 37, 45, 101, 49, 48, 106, 114, 109, 107, 113, 104, 43, 93, 23, 42, 30, 87, 60, 25, 40, 89, 18, 82, 13, 26, 85, 117, 77, 21, 78, 34, 94, 19, 124, 103, 83, 14, 55, 84, 15, 79, 16, 74, 111, 80, 20, 17, 8, 81, 29, 10, 112, 119, 76, 47, 115, 86, 90, 39, 33, 11, 12, 56, 63, 126, 100, 102, 75, 44, 127, 22, 110, 53, 57, 0, 32, 91, 61, 27, 58, 97, 52, 72, 64, 73, 4, 62, 51, 69, 5, 1, 122, 125, 9, 123, 68, 38, 71, 6, 59, 7, 35, 65, 24, 96, 98, 46, 70, 88, 36, 3, 67, 116, 31, 92, 66, 28, 120, 95, 99, 2, 108], [121, 41, 118, 105, 54, 50, 37, 45, 106, 101, 48, 49, 109, 114, 107, 104, 113, 43, 93, 60, 23, 42, 30, 25, 87, 40, 85, 13, 117, 21, 89, 82, 18, 26, 19, 77, 78, 103, 111, 74, 124, 55, 94, 16, 79, 8, 14, 63, 84, 83, 112, 34, 20, 127, 17, 10, 15, 119, 80, 29, 33, 126, 47, 12, 56, 53, 81, 100, 86, 39, 75, 115, 110, 76, 62, 102, 11, 51, 90, 27, 97, 125, 61, 22, 44, 58, 32, 72, 7, 57, 64, 52, 73, 6, 122, 69, 9, 4, 123, 5, 91, 68, 71, 0, 96, 3, 98, 35, 116, 38, 120, 36, 67, 59, 65, 46, 108, 1, 88, 70, 28, 31, 66, 99, 24, 95, 92, 2], [121, 41, 118, 105, 54, 50, 37, 45, 106, 49, 101, 48, 109, 114, 104, 107, 113, 43, 93, 23, 60, 30, 42, 87, 25, 13, 40, 82, 89, 21, 85, 18, 78, 19, 117, 26, 34, 77, 74, 15, 79, 124, 111, 83, 112, 14, 16, 20, 80, 84, 103, 10, 63, 17, 8, 39, 94, 115, 33, 100, 55, 29, 12, 76, 81, 110, 11, 86, 57, 56, 126, 22, 44, 90, 127, 53, 75, 119, 27, 52, 47, 58, 72, 102, 62, 123, 122, 32, 64, 97, 125, 73, 6, 71, 38, 5, 91, 69, 61, 0, 36, 9, 59, 120, 7, 51, 4, 67, 31, 1, 98, 68, 3, 88, 116, 70, 65, 46, 96, 35, 28, 108, 99, 92, 24, 2, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 49, 48, 109, 114, 107, 104, 113, 43, 93, 23, 60, 42, 87, 40, 25, 82, 13, 85, 30, 89, 18, 21, 26, 77, 117, 14, 19, 74, 83, 34, 15, 78, 79, 63, 112, 80, 84, 17, 103, 8, 111, 94, 10, 12, 20, 16, 124, 76, 29, 110, 55, 39, 100, 81, 33, 86, 119, 127, 115, 52, 75, 47, 53, 56, 27, 57, 11, 126, 102, 22, 72, 90, 97, 125, 58, 0, 122, 6, 44, 64, 7, 123, 32, 71, 51, 62, 69, 91, 59, 73, 9, 5, 61, 98, 1, 120, 65, 96, 116, 88, 68, 4, 3, 38, 24, 92, 36, 67, 46, 35, 108, 28, 31, 70, 2, 99, 95, 66], [121, 41, 118, 105, 54, 50, 45, 37, 106, 101, 48, 49, 114, 109, 107, 104, 93, 113, 43, 23, 42, 30, 87, 60, 25, 40, 85, 82, 26, 89, 21, 77, 13, 83, 18, 103, 14, 19, 111, 20, 34, 17, 117, 55, 84, 124, 16, 78, 79, 94, 112, 29, 15, 63, 127, 86, 80, 76, 126, 119, 10, 74, 110, 33, 39, 53, 58, 123, 56, 27, 90, 8, 122, 12, 115, 81, 47, 57, 11, 52, 22, 102, 44, 72, 97, 0, 100, 32, 125, 62, 38, 61, 75, 9, 51, 64, 59, 7, 4, 116, 98, 5, 69, 6, 36, 68, 67, 3, 96, 65, 91, 24, 71, 1, 120, 73, 70, 92, 35, 88, 31, 28, 46, 95, 66, 99, 2, 108], [121, 41, 118, 105, 54, 50, 45, 37, 106, 48, 49, 101, 109, 107, 114, 104, 113, 43, 93, 42, 87, 23, 60, 30, 40, 13, 82, 77, 85, 21, 25, 18, 19, 89, 78, 14, 34, 15, 117, 83, 74, 26, 103, 111, 112, 10, 20, 72, 124, 16, 79, 80, 17, 8, 12, 81, 84, 76, 47, 110, 64, 75, 56, 127, 115, 100, 55, 86, 29, 63, 94, 119, 11, 126, 39, 0, 53, 22, 122, 52, 102, 7, 61, 5, 57, 33, 58, 90, 68, 69, 9, 1, 62, 27, 97, 44, 65, 32, 73, 123, 59, 125, 98, 3, 71, 91, 4, 6, 67, 70, 51, 116, 66, 38, 36, 120, 88, 96, 35, 2, 24, 46, 108, 31, 92, 28, 99, 95], [41, 121, 118, 105, 54, 50, 45, 37, 106, 49, 48, 101, 109, 107, 114, 104, 113, 43, 93, 42, 23, 30, 87, 60, 40, 89, 25, 85, 55, 19, 77, 13, 112, 18, 21, 34, 117, 26, 103, 82, 78, 83, 111, 124, 94, 63, 15, 20, 10, 84, 16, 14, 74, 115, 79, 80, 29, 56, 100, 72, 39, 76, 33, 17, 81, 127, 47, 110, 53, 8, 126, 12, 102, 52, 86, 75, 119, 0, 11, 90, 123, 22, 57, 7, 27, 32, 44, 58, 97, 5, 125, 122, 62, 51, 70, 64, 61, 9, 91, 71, 69, 68, 38, 120, 65, 1, 67, 4, 96, 73, 92, 6, 59, 88, 98, 36, 116, 35, 3, 28, 108, 24, 66, 2, 46, 31, 95, 99], [121, 41, 118, 54, 105, 50, 45, 37, 106, 48, 49, 101, 114, 107, 109, 104, 113, 43, 93, 23, 42, 60, 87, 40, 25, 30, 89, 18, 77, 85, 21, 13, 82, 14, 78, 19, 83, 63, 26, 111, 117, 15, 79, 34, 10, 112, 115, 74, 103, 20, 119, 80, 72, 124, 16, 84, 17, 81, 11, 39, 94, 55, 12, 86, 76, 56, 100, 47, 126, 33, 110, 29, 102, 127, 44, 8, 0, 57, 75, 27, 122, 52, 7, 62, 64, 22, 90, 123, 53, 61, 73, 125, 70, 3, 68, 5, 67, 97, 69, 65, 51, 71, 9, 58, 32, 1, 91, 120, 4, 59, 98, 96, 2, 6, 36, 108, 35, 38, 88, 46, 92, 28, 99, 24, 116, 66, 95, 31], [41, 121, 118, 105, 54, 50, 45, 106, 37, 101, 48, 49, 114, 109, 107, 104, 113, 43, 60, 23, 87, 93, 42, 40, 30, 89, 25, 18, 13, 85, 21, 77, 82, 78, 26, 34, 19, 83, 14, 10, 80, 111, 119, 63, 79, 72, 15, 112, 103, 117, 55, 74, 20, 124, 16, 100, 115, 17, 84, 81, 29, 94, 12, 47, 76, 11, 52, 39, 102, 110, 57, 53, 75, 33, 22, 27, 62, 44, 86, 126, 0, 123, 5, 127, 73, 90, 122, 8, 7, 125, 59, 97, 64, 70, 4, 32, 61, 38, 56, 116, 58, 51, 69, 68, 65, 3, 91, 9, 71, 1, 88, 67, 98, 96, 120, 35, 6, 36, 2, 92, 99, 46, 28, 31, 108, 95, 66, 24], [121, 41, 118, 105, 54, 50, 45, 106, 37, 49, 48, 101, 109, 107, 104, 114, 113, 43, 93, 23, 60, 42, 87, 40, 13, 30, 25, 89, 85, 18, 19, 77, 82, 21, 14, 26, 111, 78, 72, 80, 10, 83, 34, 79, 103, 15, 63, 16, 117, 20, 74, 119, 124, 112, 84, 81, 12, 47, 11, 17, 76, 55, 127, 94, 39, 27, 75, 100, 126, 29, 52, 115, 110, 33, 86, 44, 53, 56, 22, 102, 58, 61, 125, 90, 70, 57, 51, 8, 62, 123, 32, 7, 122, 0, 69, 9, 73, 64, 59, 5, 71, 91, 4, 1, 97, 67, 3, 65, 68, 120, 98, 24, 6, 96, 116, 38, 35, 92, 36, 88, 46, 2, 31, 99, 66, 28, 95, 108]], "model.layers.8.self_attn.q_proj": [[60, 102, 116, 51, 123, 117, 91, 34, 52, 33, 118, 124, 115, 62, 119, 18, 126, 63, 48, 97, 58, 57, 50, 79, 54, 55, 53, 56, 61, 85, 49, 121, 59, 125, 94, 111, 122, 35, 101, 127, 45, 112, 120, 77, 44, 105, 43, 113, 89, 114, 73, 47, 26, 110, 46, 42, 13, 90, 21, 19, 109, 24, 28, 108, 107, 22, 12, 82, 14, 83, 96, 70, 37, 20, 87, 106, 39, 16, 40, 81, 72, 74, 84, 41, 103, 80, 7, 99, 30, 27, 32, 88, 31, 11, 15, 25, 36, 17, 95, 29, 104, 93, 23, 86, 92, 10, 100, 98, 76, 1, 4, 5, 3, 9, 75, 78, 38, 69, 64, 71, 67, 68, 66, 8, 6, 2, 65, 0], [117, 60, 51, 123, 116, 52, 122, 63, 126, 53, 55, 49, 48, 125, 57, 121, 120, 56, 110, 113, 111, 61, 47, 112, 114, 127, 54, 118, 119, 45, 46, 124, 62, 115, 109, 50, 58, 59, 44, 108, 43, 105, 107, 36, 42, 106, 41, 104, 40, 101, 39, 103, 102, 37, 35, 100, 24, 86, 98, 38, 95, 34, 99, 33, 87, 22, 96, 90, 66, 32, 0, 94, 31, 81, 5, 30, 97, 8, 84, 65, 19, 27, 93, 23, 85, 69, 29, 82, 75, 72, 91, 28, 21, 17, 15, 20, 89, 67, 88, 78, 92, 83, 2, 76, 13, 14, 64, 18, 25, 79, 80, 6, 70, 9, 26, 73, 68, 12, 77, 11, 1, 4, 74, 3, 16, 7, 71, 10], [102, 116, 60, 51, 33, 123, 117, 91, 52, 115, 85, 84, 15, 124, 70, 118, 79, 18, 7, 62, 34, 73, 12, 38, 119, 27, 26, 74, 89, 88, 11, 87, 59, 24, 94, 90, 58, 50, 20, 21, 93, 69, 30, 22, 3, 57, 54, 72, 126, 77, 4, 14, 82, 121, 35, 9, 98, 81, 61, 42, 37, 1, 13, 63, 56, 48, 92, 53, 10, 32, 23, 76, 25, 2, 28, 16, 64, 68, 55, 80, 31, 41, 105, 5, 17, 44, 95, 36, 101, 75, 49, 39, 67, 71, 103, 108, 66, 43, 29, 99, 120, 106, 96, 19, 127, 107, 46, 45, 125, 83, 78, 114, 97, 113, 40, 111, 100, 47, 104, 109, 6, 110, 8, 112, 86, 122, 65, 0], [116, 102, 60, 51, 123, 117, 115, 52, 62, 124, 119, 118, 34, 38, 39, 19, 58, 79, 57, 126, 85, 54, 50, 37, 43, 61, 48, 89, 53, 56, 121, 55, 94, 44, 104, 46, 42, 31, 107, 73, 108, 110, 125, 41, 40, 70, 72, 33, 111, 95, 36, 106, 109, 87, 63, 45, 49, 101, 59, 127, 103, 23, 105, 113, 120, 47, 32, 35, 28, 90, 114, 91, 13, 122, 96, 21, 112, 88, 100, 25, 11, 99, 74, 30, 27, 5, 93, 98, 92, 77, 29, 26, 1, 12, 17, 7, 24, 20, 14, 66, 80, 64, 10, 4, 3, 15, 76, 82, 16, 67, 78, 97, 18, 68, 22, 75, 71, 83, 69, 81, 84, 65, 2, 9, 86, 8, 0, 6], [38, 34, 119, 60, 18, 92, 78, 77, 9, 24, 88, 72, 85, 8, 75, 124, 68, 3, 21, 55, 59, 70, 80, 123, 69, 115, 28, 48, 71, 19, 22, 29, 37, 44, 47, 26, 61, 58, 117, 76, 4, 35, 73, 10, 95, 83, 125, 64, 0, 118, 14, 16, 79, 66, 12, 2, 62, 13, 31, 87, 93, 82, 46, 65, 57, 104, 49, 89, 42, 98, 103, 27, 15, 81, 84, 56, 63, 107, 67, 111, 30, 94, 110, 36, 6, 100, 7, 50, 25, 116, 17, 106, 105, 32, 101, 39, 53, 23, 120, 74, 86, 97, 113, 112, 43, 40, 20, 99, 41, 96, 126, 90, 1, 52, 11, 33, 121, 54, 108, 127, 114, 91, 5, 122, 45, 109, 51, 102], [38, 34, 59, 60, 119, 29, 18, 92, 88, 24, 115, 47, 77, 79, 9, 78, 68, 85, 1, 80, 55, 71, 124, 75, 22, 83, 70, 61, 65, 110, 44, 72, 64, 117, 35, 37, 69, 19, 123, 76, 125, 98, 3, 12, 58, 5, 16, 8, 73, 52, 21, 28, 95, 13, 7, 10, 100, 103, 27, 67, 0, 6, 48, 66, 93, 86, 106, 118, 26, 17, 4, 14, 112, 30, 20, 90, 46, 81, 63, 91, 32, 82, 107, 15, 2, 104, 102, 23, 94, 42, 74, 36, 11, 25, 99, 89, 39, 45, 54, 62, 49, 116, 120, 97, 50, 41, 96, 126, 31, 87, 113, 33, 43, 56, 127, 114, 122, 53, 105, 40, 84, 121, 101, 111, 51, 109, 108, 57], [38, 34, 60, 119, 29, 18, 85, 77, 88, 59, 78, 24, 37, 80, 9, 92, 124, 75, 71, 55, 69, 44, 21, 47, 8, 125, 3, 61, 93, 72, 16, 68, 19, 117, 35, 118, 79, 123, 22, 81, 95, 106, 11, 76, 115, 13, 32, 48, 58, 31, 10, 1, 83, 120, 65, 27, 63, 98, 122, 7, 116, 52, 84, 97, 89, 14, 26, 82, 99, 112, 107, 36, 94, 66, 64, 50, 42, 25, 30, 53, 28, 33, 100, 70, 40, 103, 87, 126, 110, 62, 51, 96, 104, 12, 41, 101, 39, 20, 54, 91, 86, 43, 49, 109, 45, 127, 23, 46, 114, 111, 57, 56, 108, 74, 113, 6, 17, 90, 15, 67, 121, 4, 0, 105, 73, 5, 2, 102], [38, 60, 119, 34, 29, 28, 88, 115, 85, 102, 93, 59, 19, 125, 80, 95, 27, 78, 92, 47, 87, 52, 98, 55, 18, 33, 44, 37, 35, 123, 118, 117, 24, 26, 83, 110, 22, 62, 58, 54, 46, 12, 89, 25, 61, 42, 48, 104, 103, 84, 30, 113, 40, 45, 97, 100, 101, 106, 91, 109, 31, 112, 116, 10, 81, 94, 114, 63, 21, 99, 107, 51, 49, 50, 105, 126, 96, 16, 111, 108, 127, 121, 124, 32, 57, 122, 43, 36, 56, 17, 120, 53, 41, 20, 86, 79, 39, 76, 23, 90, 8, 14, 74, 77, 71, 15, 75, 72, 11, 13, 82, 9, 7, 66, 2, 67, 70, 5, 69, 3, 4, 6, 73, 0, 68, 1, 64, 65], [59, 60, 116, 80, 55, 50, 73, 72, 117, 12, 62, 77, 10, 84, 120, 26, 54, 125, 103, 69, 19, 13, 24, 127, 68, 78, 15, 75, 93, 7, 21, 82, 17, 51, 90, 124, 6, 57, 49, 113, 3, 98, 56, 122, 121, 1, 66, 114, 16, 18, 112, 46, 58, 76, 70, 119, 36, 126, 123, 52, 28, 34, 118, 53, 88, 110, 61, 63, 0, 47, 29, 71, 109, 43, 27, 115, 96, 32, 104, 44, 107, 45, 108, 100, 92, 48, 102, 38, 42, 97, 106, 94, 105, 79, 111, 41, 101, 87, 89, 40, 37, 31, 33, 99, 95, 11, 91, 85, 35, 30, 86, 74, 9, 20, 81, 65, 83, 39, 23, 14, 4, 25, 8, 2, 22, 64, 67, 5], [59, 116, 60, 103, 55, 73, 117, 15, 87, 62, 80, 120, 125, 82, 54, 84, 10, 90, 127, 72, 13, 19, 69, 68, 76, 7, 6, 3, 78, 77, 51, 50, 124, 57, 66, 0, 75, 17, 113, 1, 49, 12, 21, 37, 56, 70, 121, 122, 11, 114, 119, 46, 123, 52, 36, 94, 74, 93, 32, 38, 88, 110, 58, 61, 118, 39, 43, 31, 112, 126, 28, 96, 47, 26, 104, 99, 53, 71, 42, 89, 65, 106, 101, 107, 115, 40, 100, 109, 111, 86, 83, 97, 102, 63, 108, 44, 105, 41, 81, 24, 33, 35, 30, 45, 95, 48, 8, 22, 92, 4, 98, 27, 20, 5, 16, 18, 79, 25, 91, 14, 34, 2, 29, 64, 23, 9, 85, 67], [103, 59, 116, 60, 20, 26, 16, 96, 84, 98, 24, 29, 89, 114, 93, 14, 10, 55, 125, 94, 91, 99, 88, 62, 19, 117, 70, 1, 87, 33, 18, 50, 31, 28, 80, 90, 68, 17, 57, 92, 52, 127, 69, 12, 95, 3, 25, 54, 27, 49, 23, 124, 72, 97, 120, 123, 66, 30, 74, 110, 71, 35, 22, 100, 36, 38, 32, 34, 113, 21, 0, 67, 64, 104, 102, 53, 82, 39, 78, 37, 76, 101, 40, 42, 51, 6, 119, 7, 108, 4, 122, 105, 83, 43, 81, 58, 75, 41, 118, 106, 44, 109, 48, 107, 45, 121, 47, 46, 9, 112, 56, 63, 73, 61, 111, 126, 115, 15, 5, 79, 85, 11, 8, 13, 65, 86, 77, 2], [59, 50, 60, 116, 19, 21, 15, 82, 80, 89, 76, 84, 13, 73, 88, 78, 87, 120, 62, 117, 17, 125, 54, 55, 7, 51, 90, 72, 127, 77, 6, 10, 3, 124, 61, 56, 121, 122, 58, 0, 63, 123, 112, 126, 118, 119, 11, 69, 57, 113, 53, 115, 47, 68, 75, 49, 66, 48, 111, 74, 46, 92, 86, 52, 110, 45, 109, 114, 108, 44, 1, 107, 93, 43, 106, 42, 105, 8, 41, 23, 104, 40, 27, 29, 65, 12, 100, 81, 103, 4, 28, 30, 38, 37, 102, 36, 39, 5, 70, 99, 20, 101, 14, 2, 83, 32, 35, 98, 97, 96, 22, 33, 95, 31, 67, 64, 26, 94, 71, 34, 24, 25, 85, 91, 16, 79, 9, 18], [104, 118, 34, 95, 50, 25, 70, 73, 51, 56, 45, 23, 55, 114, 49, 62, 5, 84, 3, 7, 40, 109, 117, 18, 121, 52, 74, 63, 54, 110, 124, 82, 58, 68, 14, 86, 126, 76, 15, 60, 8, 119, 112, 65, 17, 31, 123, 53, 80, 108, 97, 59, 127, 16, 27, 71, 125, 20, 35, 115, 46, 107, 89, 47, 32, 93, 120, 77, 44, 39, 111, 42, 29, 101, 61, 48, 90, 38, 41, 2, 113, 122, 100, 88, 30, 106, 92, 12, 102, 22, 37, 33, 103, 36, 105, 19, 13, 57, 94, 43, 116, 96, 28, 9, 91, 78, 85, 87, 81, 75, 99, 79, 21, 24, 83, 98, 10, 11, 26, 67, 0, 72, 6, 69, 4, 1, 66, 64], [104, 118, 34, 95, 84, 25, 73, 12, 40, 23, 18, 51, 16, 9, 49, 5, 65, 68, 70, 56, 1, 69, 66, 54, 8, 89, 80, 67, 63, 6, 13, 117, 15, 64, 62, 87, 60, 78, 45, 11, 50, 55, 3, 27, 58, 74, 14, 7, 20, 124, 72, 79, 2, 110, 24, 10, 71, 109, 76, 77, 82, 81, 127, 0, 28, 52, 4, 123, 22, 114, 31, 93, 83, 125, 97, 42, 91, 75, 126, 85, 112, 94, 86, 90, 21, 35, 46, 88, 33, 47, 19, 59, 121, 100, 38, 48, 39, 61, 119, 122, 17, 41, 26, 92, 29, 44, 101, 96, 108, 107, 111, 53, 113, 106, 36, 103, 32, 98, 120, 37, 99, 30, 102, 105, 43, 116, 115, 57], [118, 104, 34, 95, 50, 25, 23, 45, 52, 62, 84, 56, 117, 58, 49, 51, 18, 54, 16, 63, 114, 109, 123, 60, 55, 124, 106, 112, 12, 59, 27, 125, 57, 91, 121, 15, 126, 61, 110, 42, 32, 41, 31, 127, 119, 48, 43, 53, 33, 22, 39, 47, 115, 108, 111, 37, 113, 122, 120, 35, 28, 46, 116, 88, 107, 44, 99, 105, 93, 73, 103, 36, 97, 102, 38, 101, 92, 30, 82, 100, 86, 5, 87, 79, 96, 90, 13, 94, 85, 24, 89, 17, 78, 29, 9, 8, 70, 83, 26, 14, 74, 98, 21, 76, 20, 81, 80, 19, 7, 75, 3, 10, 77, 40, 72, 11, 69, 68, 65, 2, 71, 67, 6, 64, 4, 66, 1, 0], [104, 118, 34, 95, 25, 16, 12, 56, 23, 84, 18, 45, 15, 55, 69, 51, 78, 50, 117, 72, 58, 9, 40, 63, 8, 5, 49, 109, 66, 114, 67, 124, 54, 79, 71, 6, 60, 76, 4, 13, 110, 10, 80, 87, 86, 73, 31, 81, 119, 27, 62, 91, 108, 89, 75, 14, 1, 20, 70, 85, 52, 53, 112, 64, 29, 123, 11, 7, 19, 82, 42, 44, 59, 93, 121, 83, 61, 127, 39, 126, 90, 96, 94, 97, 32, 125, 74, 77, 103, 2, 21, 98, 107, 24, 41, 88, 111, 17, 22, 35, 106, 26, 57, 47, 115, 101, 92, 48, 43, 33, 30, 37, 68, 100, 122, 46, 28, 3, 36, 116, 105, 38, 102, 113, 120, 0, 99, 65], [41, 54, 59, 99, 79, 88, 101, 33, 13, 22, 90, 91, 31, 27, 73, 82, 39, 11, 123, 21, 84, 26, 71, 0, 66, 16, 40, 109, 94, 6, 74, 108, 2, 83, 28, 12, 38, 107, 68, 29, 126, 105, 51, 62, 7, 18, 95, 60, 76, 87, 34, 52, 20, 1, 98, 23, 9, 15, 32, 80, 78, 122, 67, 96, 125, 8, 110, 42, 55, 65, 4, 75, 57, 70, 36, 30, 106, 85, 49, 89, 45, 50, 46, 3, 124, 103, 56, 64, 77, 19, 127, 43, 17, 44, 58, 92, 100, 47, 104, 121, 97, 113, 118, 86, 112, 14, 115, 93, 114, 119, 10, 25, 120, 63, 102, 61, 5, 24, 81, 53, 72, 48, 69, 116, 111, 117, 37, 35], [41, 54, 99, 52, 91, 33, 126, 123, 27, 105, 113, 88, 108, 59, 61, 43, 36, 40, 101, 13, 125, 109, 22, 84, 107, 49, 16, 21, 57, 44, 118, 56, 45, 34, 115, 51, 63, 62, 17, 106, 50, 98, 31, 112, 127, 42, 25, 47, 120, 90, 39, 55, 100, 46, 124, 38, 121, 110, 114, 116, 58, 103, 29, 111, 96, 122, 94, 60, 95, 48, 53, 77, 71, 119, 117, 23, 104, 32, 102, 28, 93, 37, 82, 19, 4, 92, 85, 30, 81, 75, 20, 87, 89, 35, 26, 83, 97, 76, 68, 74, 10, 80, 73, 70, 18, 11, 24, 14, 72, 69, 12, 79, 78, 67, 6, 0, 2, 7, 8, 86, 15, 64, 65, 9, 5, 66, 1, 3], [41, 54, 99, 126, 33, 52, 91, 105, 27, 40, 101, 88, 21, 61, 89, 56, 125, 13, 45, 110, 123, 44, 34, 113, 82, 115, 36, 109, 85, 38, 59, 43, 107, 84, 22, 31, 42, 112, 81, 124, 94, 108, 114, 111, 120, 39, 116, 127, 57, 103, 16, 96, 58, 50, 63, 18, 118, 62, 60, 74, 90, 53, 100, 122, 17, 55, 37, 106, 23, 98, 104, 48, 87, 32, 49, 46, 121, 93, 28, 77, 51, 117, 19, 47, 119, 102, 25, 26, 97, 80, 29, 95, 68, 30, 83, 92, 35, 71, 24, 79, 75, 20, 11, 76, 4, 10, 78, 14, 12, 72, 86, 15, 70, 73, 7, 6, 1, 8, 9, 65, 64, 69, 2, 67, 5, 66, 0, 3], [41, 99, 59, 54, 88, 82, 31, 123, 27, 22, 91, 105, 79, 84, 13, 52, 21, 108, 87, 101, 12, 46, 33, 8, 18, 74, 83, 26, 39, 19, 81, 9, 94, 95, 118, 126, 10, 89, 77, 38, 17, 6, 92, 127, 20, 25, 55, 28, 15, 56, 40, 14, 69, 90, 70, 115, 44, 85, 68, 24, 16, 30, 97, 5, 86, 100, 23, 61, 98, 43, 32, 109, 110, 96, 104, 120, 113, 124, 76, 111, 29, 34, 122, 42, 75, 125, 80, 93, 107, 119, 57, 102, 58, 106, 36, 103, 47, 1, 63, 112, 114, 45, 72, 121, 4, 78, 50, 60, 117, 11, 3, 116, 53, 65, 49, 48, 71, 73, 37, 51, 62, 35, 67, 66, 0, 7, 64, 2], [103, 33, 59, 31, 82, 20, 23, 120, 81, 90, 114, 11, 76, 13, 14, 87, 22, 78, 7, 18, 71, 10, 21, 106, 83, 112, 16, 12, 15, 73, 124, 19, 119, 52, 77, 67, 96, 123, 38, 91, 25, 125, 41, 48, 40, 43, 107, 37, 94, 104, 98, 97, 88, 118, 27, 75, 86, 42, 51, 121, 84, 53, 117, 54, 55, 35, 58, 32, 3, 24, 30, 72, 80, 69, 111, 28, 34, 109, 126, 101, 47, 56, 100, 46, 8, 44, 85, 29, 93, 105, 115, 26, 79, 92, 108, 9, 1, 99, 62, 60, 36, 122, 127, 45, 61, 68, 113, 89, 4, 17, 57, 2, 39, 74, 63, 49, 5, 65, 102, 66, 110, 6, 95, 116, 50, 70, 64, 0], [103, 33, 59, 20, 31, 23, 120, 90, 83, 114, 82, 29, 16, 78, 19, 13, 84, 27, 88, 37, 94, 24, 32, 118, 81, 11, 72, 30, 17, 97, 7, 126, 48, 38, 92, 127, 96, 117, 21, 8, 77, 124, 56, 36, 87, 106, 25, 76, 22, 40, 9, 51, 12, 91, 119, 62, 105, 57, 41, 115, 10, 43, 45, 113, 52, 26, 110, 123, 14, 102, 79, 107, 15, 98, 54, 28, 93, 55, 34, 89, 61, 109, 18, 66, 44, 74, 80, 100, 95, 42, 99, 125, 85, 101, 47, 121, 63, 49, 35, 60, 73, 104, 68, 111, 58, 39, 108, 71, 4, 122, 53, 86, 75, 46, 112, 65, 116, 70, 50, 67, 2, 3, 6, 69, 5, 64, 0, 1], [103, 33, 59, 120, 31, 25, 13, 20, 11, 82, 78, 52, 16, 87, 10, 73, 23, 81, 7, 3, 65, 2, 66, 70, 90, 0, 5, 68, 114, 76, 69, 6, 19, 83, 18, 80, 106, 107, 67, 71, 27, 8, 75, 119, 9, 72, 96, 51, 77, 117, 48, 39, 28, 74, 105, 1, 43, 126, 4, 118, 22, 24, 37, 41, 123, 100, 46, 17, 53, 36, 127, 94, 56, 109, 115, 101, 44, 85, 40, 26, 92, 30, 54, 32, 104, 21, 84, 113, 62, 49, 47, 15, 12, 34, 14, 45, 121, 60, 93, 98, 91, 42, 88, 79, 50, 124, 29, 57, 111, 58, 110, 122, 125, 61, 116, 63, 99, 108, 35, 86, 102, 38, 89, 55, 112, 95, 64, 97], [103, 33, 114, 59, 31, 81, 120, 25, 76, 20, 78, 64, 15, 23, 82, 7, 21, 90, 16, 9, 3, 1, 83, 11, 5, 48, 106, 13, 65, 24, 66, 67, 27, 75, 115, 37, 47, 126, 77, 19, 74, 52, 44, 84, 55, 4, 10, 28, 53, 107, 93, 87, 22, 97, 71, 92, 29, 98, 39, 12, 43, 0, 109, 40, 96, 124, 46, 127, 51, 119, 17, 121, 88, 32, 105, 38, 2, 26, 100, 118, 112, 99, 111, 69, 8, 79, 104, 41, 86, 116, 58, 113, 73, 62, 45, 42, 110, 91, 18, 56, 94, 70, 14, 125, 6, 89, 54, 35, 57, 80, 108, 72, 63, 101, 123, 68, 122, 30, 50, 60, 61, 49, 34, 117, 36, 95, 102, 85], [44, 102, 50, 63, 98, 93, 108, 40, 123, 88, 80, 107, 43, 13, 31, 120, 119, 90, 7, 53, 115, 125, 57, 22, 29, 10, 114, 126, 19, 51, 84, 26, 20, 116, 92, 112, 109, 39, 55, 28, 12, 124, 127, 49, 121, 21, 45, 48, 36, 17, 18, 8, 101, 61, 52, 14, 46, 56, 85, 54, 47, 78, 118, 122, 99, 16, 113, 94, 59, 110, 74, 83, 60, 6, 58, 117, 70, 3, 97, 111, 68, 105, 81, 62, 25, 23, 4, 104, 42, 87, 32, 41, 91, 103, 30, 76, 79, 35, 106, 33, 15, 37, 82, 11, 27, 9, 96, 24, 89, 75, 66, 0, 95, 71, 73, 100, 86, 72, 67, 77, 65, 1, 2, 64, 69, 5, 38, 34], [44, 63, 102, 50, 98, 93, 40, 123, 107, 26, 108, 80, 31, 57, 115, 120, 43, 84, 90, 10, 127, 125, 53, 13, 22, 88, 116, 21, 126, 45, 19, 109, 121, 23, 51, 119, 112, 61, 124, 114, 104, 29, 18, 55, 47, 68, 56, 101, 54, 49, 36, 14, 62, 46, 52, 48, 122, 92, 17, 99, 58, 7, 60, 72, 105, 118, 12, 28, 8, 39, 97, 30, 78, 91, 59, 87, 110, 111, 117, 20, 42, 113, 77, 15, 103, 27, 81, 41, 106, 32, 95, 25, 85, 89, 94, 79, 83, 16, 24, 9, 66, 96, 2, 35, 37, 4, 76, 100, 75, 82, 11, 33, 5, 3, 73, 70, 1, 0, 74, 86, 6, 69, 65, 38, 34, 71, 67, 64], [44, 63, 102, 50, 108, 98, 120, 107, 40, 57, 53, 119, 51, 123, 88, 93, 26, 109, 125, 112, 58, 31, 127, 114, 43, 126, 61, 121, 116, 101, 118, 45, 48, 49, 54, 115, 36, 85, 47, 59, 56, 124, 21, 111, 122, 113, 55, 23, 95, 117, 80, 60, 39, 84, 52, 25, 62, 78, 104, 8, 46, 87, 41, 42, 100, 14, 19, 13, 18, 105, 90, 29, 89, 106, 110, 92, 35, 28, 32, 94, 99, 22, 103, 97, 37, 12, 91, 30, 17, 96, 33, 34, 81, 38, 66, 72, 82, 68, 24, 71, 83, 27, 20, 77, 7, 10, 74, 79, 2, 65, 16, 75, 11, 70, 15, 3, 73, 76, 9, 86, 6, 67, 69, 0, 4, 64, 5, 1], [44, 63, 102, 50, 108, 98, 40, 43, 115, 123, 120, 88, 125, 116, 31, 57, 26, 107, 112, 93, 114, 58, 53, 28, 127, 121, 84, 51, 19, 119, 101, 95, 52, 126, 56, 55, 49, 54, 36, 47, 109, 61, 39, 45, 59, 21, 104, 124, 46, 97, 62, 111, 8, 118, 48, 29, 90, 122, 23, 78, 14, 60, 113, 30, 117, 42, 110, 80, 22, 103, 12, 87, 92, 105, 99, 20, 37, 94, 33, 100, 106, 18, 32, 13, 17, 41, 27, 35, 38, 91, 81, 89, 72, 96, 24, 83, 15, 2, 77, 85, 66, 79, 25, 76, 82, 34, 7, 71, 73, 10, 68, 75, 9, 65, 16, 11, 6, 4, 70, 67, 0, 74, 69, 86, 3, 5, 1, 64], [54, 102, 122, 127, 120, 33, 28, 20, 125, 87, 92, 25, 116, 15, 12, 30, 97, 114, 47, 89, 63, 17, 96, 21, 22, 38, 32, 82, 19, 49, 94, 99, 7, 52, 48, 93, 123, 50, 16, 80, 13, 62, 115, 27, 91, 95, 121, 83, 74, 75, 72, 14, 88, 84, 31, 60, 85, 81, 55, 26, 51, 29, 10, 11, 59, 108, 35, 34, 113, 100, 111, 77, 90, 36, 98, 78, 23, 45, 118, 79, 37, 103, 24, 126, 53, 101, 57, 109, 58, 110, 112, 39, 117, 18, 105, 124, 56, 107, 41, 104, 43, 69, 42, 119, 40, 44, 46, 106, 76, 9, 2, 73, 61, 8, 71, 4, 86, 67, 6, 0, 70, 5, 68, 3, 1, 65, 66, 64], [54, 102, 122, 125, 127, 116, 25, 20, 114, 38, 33, 63, 30, 79, 47, 87, 120, 62, 123, 50, 52, 74, 48, 49, 53, 60, 55, 118, 126, 115, 28, 22, 58, 15, 121, 59, 7, 31, 57, 46, 51, 17, 94, 117, 92, 124, 113, 84, 76, 111, 70, 41, 44, 103, 89, 110, 35, 104, 32, 43, 12, 36, 107, 112, 39, 93, 34, 61, 119, 98, 96, 109, 99, 67, 40, 37, 56, 88, 105, 108, 95, 45, 2, 91, 106, 29, 77, 101, 83, 42, 97, 13, 69, 27, 100, 81, 19, 80, 72, 11, 90, 86, 21, 85, 82, 26, 73, 78, 23, 14, 75, 24, 71, 68, 18, 4, 0, 8, 16, 65, 10, 1, 9, 6, 5, 66, 64, 3], [122, 102, 54, 127, 125, 116, 120, 114, 87, 25, 33, 38, 92, 30, 12, 63, 47, 28, 20, 48, 22, 123, 62, 53, 17, 50, 59, 55, 118, 52, 60, 80, 49, 72, 82, 126, 57, 89, 24, 124, 51, 115, 76, 15, 58, 121, 91, 29, 81, 67, 94, 113, 97, 70, 109, 7, 0, 111, 44, 46, 21, 112, 2, 77, 103, 61, 56, 74, 119, 117, 69, 88, 110, 93, 108, 43, 79, 39, 65, 84, 107, 8, 36, 14, 1, 45, 73, 78, 83, 68, 32, 23, 90, 31, 27, 105, 40, 95, 96, 104, 6, 9, 37, 106, 16, 42, 3, 86, 85, 18, 100, 26, 10, 11, 41, 35, 34, 98, 101, 71, 19, 75, 13, 64, 4, 66, 5, 99], [122, 54, 102, 127, 116, 125, 114, 120, 38, 70, 25, 55, 63, 74, 53, 47, 59, 48, 62, 123, 118, 49, 103, 50, 60, 20, 67, 44, 52, 51, 124, 115, 126, 39, 57, 56, 58, 121, 113, 2, 45, 43, 33, 117, 76, 111, 7, 112, 110, 13, 46, 87, 119, 107, 109, 61, 93, 42, 108, 106, 36, 0, 105, 40, 101, 28, 104, 79, 92, 22, 41, 89, 37, 30, 35, 100, 19, 94, 26, 96, 90, 81, 88, 32, 34, 31, 29, 97, 71, 78, 23, 77, 99, 80, 69, 98, 27, 95, 84, 72, 10, 65, 4, 24, 1, 8, 12, 91, 68, 15, 73, 86, 17, 83, 11, 85, 9, 6, 14, 18, 82, 64, 21, 16, 5, 66, 75, 3]], "model.layers.8.self_attn.k_proj": [[116, 60, 38, 22, 51, 97, 94, 123, 117, 28, 89, 90, 84, 124, 18, 62, 16, 119, 81, 91, 36, 77, 52, 118, 87, 58, 15, 74, 35, 7, 54, 86, 19, 59, 50, 121, 61, 57, 21, 80, 42, 56, 99, 78, 126, 53, 100, 10, 73, 13, 48, 70, 41, 46, 75, 108, 125, 44, 127, 76, 40, 14, 114, 107, 103, 115, 104, 47, 98, 37, 63, 55, 106, 109, 105, 110, 24, 111, 26, 120, 43, 101, 45, 112, 49, 113, 39, 31, 122, 9, 95, 68, 67, 32, 85, 96, 82, 29, 1, 88, 30, 34, 64, 17, 102, 4, 66, 5, 23, 6, 92, 27, 20, 93, 71, 12, 83, 72, 3, 33, 65, 25, 11, 0, 2, 79, 8, 69], [60, 119, 98, 102, 88, 18, 85, 9, 80, 28, 78, 77, 64, 75, 71, 65, 59, 99, 3, 19, 93, 108, 68, 12, 69, 23, 115, 125, 2, 70, 61, 95, 111, 43, 51, 104, 29, 11, 38, 91, 5, 124, 123, 22, 117, 47, 37, 52, 20, 58, 46, 42, 24, 30, 10, 31, 49, 33, 66, 103, 36, 118, 53, 67, 40, 90, 113, 4, 15, 17, 122, 86, 81, 7, 114, 109, 6, 107, 83, 41, 48, 8, 92, 25, 63, 62, 54, 127, 32, 96, 13, 110, 94, 34, 0, 89, 84, 44, 106, 105, 100, 112, 120, 126, 26, 79, 97, 76, 121, 57, 55, 39, 82, 35, 116, 87, 16, 50, 45, 73, 101, 72, 56, 27, 74, 21, 1, 14], [59, 22, 60, 116, 39, 34, 114, 120, 54, 86, 124, 127, 51, 50, 35, 62, 55, 117, 113, 94, 57, 121, 56, 125, 122, 123, 49, 119, 52, 46, 29, 99, 61, 118, 43, 126, 58, 53, 110, 115, 112, 63, 103, 111, 47, 106, 108, 109, 76, 37, 18, 48, 28, 45, 44, 107, 42, 40, 81, 104, 96, 41, 14, 102, 38, 101, 105, 11, 85, 91, 95, 98, 15, 97, 100, 13, 74, 27, 26, 36, 73, 20, 25, 16, 82, 7, 32, 24, 30, 23, 78, 33, 92, 72, 6, 19, 8, 79, 80, 31, 89, 69, 88, 93, 90, 83, 66, 3, 68, 87, 17, 75, 0, 4, 1, 9, 5, 10, 67, 21, 12, 65, 84, 77, 2, 70, 71, 64], [40, 118, 98, 31, 23, 18, 25, 84, 16, 12, 56, 51, 78, 72, 10, 15, 9, 117, 55, 69, 71, 49, 63, 0, 54, 67, 62, 6, 1, 109, 4, 110, 60, 58, 45, 50, 127, 27, 124, 13, 2, 64, 66, 113, 74, 46, 8, 17, 47, 106, 42, 83, 123, 112, 44, 34, 19, 53, 91, 75, 126, 26, 24, 125, 52, 114, 61, 122, 29, 97, 68, 77, 30, 48, 11, 94, 108, 111, 59, 38, 119, 93, 107, 99, 120, 86, 103, 88, 39, 28, 121, 7, 21, 101, 33, 90, 100, 43, 105, 115, 41, 32, 96, 35, 92, 36, 102, 14, 37, 116, 57, 65, 89, 82, 85, 20, 70, 87, 22, 76, 79, 3, 80, 95, 5, 73, 81, 104], [105, 54, 59, 35, 95, 22, 91, 88, 21, 64, 110, 123, 68, 18, 97, 13, 16, 92, 74, 1, 6, 79, 37, 45, 71, 62, 52, 98, 44, 82, 41, 55, 107, 66, 83, 108, 126, 77, 57, 11, 50, 99, 48, 120, 127, 104, 58, 112, 53, 8, 90, 117, 56, 94, 122, 5, 40, 125, 60, 63, 51, 81, 121, 67, 103, 61, 106, 124, 114, 34, 12, 111, 115, 65, 3, 26, 113, 102, 116, 46, 42, 119, 30, 96, 47, 93, 73, 100, 15, 25, 109, 118, 36, 78, 17, 87, 43, 20, 14, 39, 2, 75, 38, 32, 0, 101, 76, 89, 84, 49, 29, 23, 19, 69, 72, 9, 33, 80, 31, 28, 24, 70, 7, 27, 10, 86, 4, 85], [39, 59, 120, 97, 23, 95, 83, 16, 11, 114, 20, 0, 78, 13, 82, 7, 76, 8, 73, 2, 42, 65, 48, 15, 50, 126, 69, 68, 3, 104, 107, 52, 25, 32, 27, 43, 117, 105, 119, 45, 62, 51, 4, 115, 53, 112, 118, 40, 9, 89, 81, 90, 88, 86, 58, 44, 98, 55, 1, 85, 109, 21, 10, 113, 79, 127, 66, 5, 54, 28, 30, 37, 35, 63, 122, 75, 47, 124, 46, 24, 102, 29, 61, 34, 106, 38, 94, 108, 121, 67, 91, 70, 57, 125, 49, 26, 110, 101, 22, 36, 96, 123, 72, 56, 60, 14, 18, 93, 99, 111, 100, 41, 116, 6, 84, 92, 19, 64, 17, 74, 12, 87, 77, 31, 80, 71, 103, 33], [108, 38, 22, 63, 34, 95, 50, 29, 23, 26, 18, 88, 44, 92, 104, 17, 19, 120, 20, 66, 115, 12, 114, 57, 125, 80, 13, 49, 107, 37, 123, 69, 53, 98, 112, 111, 8, 127, 45, 47, 121, 126, 51, 78, 113, 10, 55, 61, 35, 56, 52, 14, 54, 62, 91, 59, 119, 117, 65, 79, 85, 116, 109, 122, 7, 60, 64, 103, 118, 83, 124, 89, 46, 110, 105, 27, 70, 93, 58, 15, 9, 39, 100, 106, 48, 11, 40, 5, 97, 72, 42, 3, 28, 33, 94, 25, 41, 99, 96, 36, 77, 32, 30, 87, 84, 67, 43, 82, 24, 75, 76, 0, 21, 81, 31, 73, 101, 6, 4, 86, 1, 16, 90, 71, 2, 74, 68, 102], [122, 54, 38, 22, 97, 127, 63, 30, 114, 120, 125, 53, 47, 92, 50, 62, 126, 116, 82, 123, 51, 118, 59, 52, 124, 25, 57, 115, 46, 87, 55, 48, 58, 113, 44, 49, 111, 61, 56, 60, 121, 119, 117, 109, 12, 108, 98, 64, 43, 110, 112, 71, 15, 100, 17, 45, 40, 107, 20, 103, 106, 80, 32, 104, 105, 101, 41, 75, 39, 78, 42, 66, 36, 19, 67, 10, 35, 37, 102, 34, 94, 68, 5, 91, 99, 31, 77, 14, 3, 83, 8, 96, 18, 81, 73, 93, 24, 21, 26, 65, 95, 9, 11, 79, 88, 69, 29, 90, 33, 85, 70, 4, 16, 23, 27, 28, 89, 13, 6, 86, 1, 74, 72, 0, 2, 7, 84, 76]], "model.layers.8.self_attn.qk_proj": [[59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 44, 125, 105, 117, 123, 40, 41, 102, 82, 86, 127, 47, 95, 34, 53, 87, 98, 18, 55, 62, 58, 39, 57, 52, 49, 124, 24, 103, 126, 88, 23, 115, 80, 97, 16, 78, 61, 84, 27, 20, 22, 89, 29, 13, 14, 104, 110, 83, 77, 19, 107, 113, 12, 92, 85, 31, 99, 76, 93, 46, 21, 11, 43, 56, 48, 15, 75, 111, 112, 109, 25, 33, 42, 9, 90, 121, 73, 79, 28, 71, 7, 106, 8, 26, 35, 45, 30, 81, 94, 10, 74, 64, 17, 36, 91, 5, 37, 3, 4, 0, 70, 72, 32, 67, 68, 69, 101, 100, 66, 2, 6, 65, 96, 1], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 117, 44, 40, 105, 125, 123, 53, 102, 86, 41, 18, 103, 98, 127, 62, 82, 115, 87, 52, 126, 39, 88, 95, 24, 124, 34, 57, 84, 58, 97, 80, 23, 104, 55, 47, 20, 16, 22, 107, 78, 27, 77, 89, 49, 56, 14, 92, 19, 25, 85, 31, 13, 61, 83, 93, 12, 76, 21, 79, 42, 29, 99, 110, 48, 113, 11, 43, 75, 28, 9, 106, 26, 109, 8, 46, 73, 71, 45, 112, 15, 35, 37, 90, 121, 33, 111, 7, 94, 30, 36, 91, 17, 64, 10, 5, 81, 74, 0, 65, 4, 32, 67, 2, 1, 3, 70, 66, 101, 68, 69, 6, 100, 96, 72], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 38, 50, 114, 123, 44, 40, 117, 105, 53, 125, 102, 86, 34, 98, 39, 41, 95, 18, 24, 97, 87, 62, 82, 57, 127, 88, 23, 52, 115, 55, 84, 124, 22, 103, 20, 126, 49, 27, 104, 58, 89, 31, 47, 113, 16, 80, 93, 78, 29, 92, 77, 25, 56, 61, 48, 99, 33, 109, 85, 13, 75, 26, 45, 19, 21, 112, 76, 107, 43, 110, 14, 90, 35, 11, 94, 79, 12, 37, 83, 28, 46, 91, 111, 30, 42, 106, 15, 9, 7, 121, 8, 71, 81, 17, 36, 73, 32, 74, 10, 69, 100, 101, 64, 65, 0, 3, 72, 4, 66, 2, 68, 5, 6, 1, 67, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 123, 44, 40, 117, 125, 105, 127, 86, 95, 41, 98, 53, 87, 102, 24, 115, 18, 62, 39, 34, 126, 82, 49, 103, 124, 88, 97, 23, 58, 47, 57, 84, 113, 52, 20, 22, 93, 55, 29, 80, 31, 16, 27, 14, 89, 25, 104, 83, 99, 77, 92, 110, 19, 48, 13, 78, 43, 109, 107, 61, 45, 11, 56, 85, 33, 21, 28, 46, 42, 15, 76, 37, 90, 26, 12, 75, 35, 73, 30, 94, 7, 36, 106, 79, 121, 112, 8, 111, 64, 9, 32, 71, 17, 91, 101, 81, 74, 10, 67, 3, 69, 4, 1, 2, 66, 0, 5, 65, 72, 96, 68, 70, 100, 6], [59, 60, 118, 54, 119, 122, 116, 120, 51, 63, 114, 108, 38, 50, 44, 117, 123, 40, 105, 125, 102, 98, 127, 57, 53, 95, 39, 24, 18, 82, 62, 41, 124, 87, 103, 34, 86, 23, 49, 113, 126, 115, 55, 80, 104, 88, 27, 97, 78, 84, 20, 58, 47, 52, 22, 77, 29, 16, 89, 93, 19, 46, 31, 110, 13, 14, 112, 45, 11, 12, 76, 92, 56, 107, 85, 28, 99, 83, 61, 48, 79, 109, 43, 21, 25, 73, 90, 35, 42, 37, 71, 0, 33, 9, 75, 111, 15, 26, 121, 7, 64, 36, 30, 106, 94, 17, 74, 72, 1, 3, 5, 32, 8, 4, 68, 66, 67, 10, 2, 81, 69, 91, 65, 101, 6, 96, 70, 100], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 108, 50, 38, 44, 123, 125, 117, 40, 105, 86, 102, 53, 127, 41, 95, 98, 82, 18, 24, 62, 124, 39, 87, 88, 80, 103, 16, 115, 34, 84, 57, 55, 52, 20, 78, 22, 47, 58, 23, 14, 126, 77, 49, 31, 83, 97, 104, 29, 27, 13, 107, 113, 76, 19, 89, 85, 28, 11, 12, 110, 92, 93, 99, 48, 42, 56, 109, 79, 75, 15, 25, 43, 61, 45, 46, 21, 30, 73, 112, 33, 9, 71, 90, 35, 37, 7, 72, 111, 121, 26, 106, 17, 81, 5, 94, 8, 36, 74, 10, 32, 64, 2, 0, 68, 91, 3, 69, 6, 67, 4, 65, 66, 101, 100, 1, 96, 70], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 38, 123, 117, 44, 40, 125, 105, 127, 86, 53, 95, 18, 87, 98, 82, 41, 102, 115, 24, 52, 88, 62, 84, 34, 126, 124, 22, 47, 16, 58, 20, 23, 39, 80, 97, 55, 57, 103, 104, 78, 29, 83, 13, 14, 113, 89, 27, 112, 49, 92, 107, 110, 19, 77, 76, 99, 48, 56, 31, 93, 11, 111, 21, 85, 12, 75, 61, 15, 28, 109, 9, 73, 79, 121, 43, 90, 33, 42, 94, 25, 81, 30, 46, 17, 7, 35, 26, 72, 71, 45, 106, 37, 74, 10, 91, 101, 32, 5, 8, 36, 68, 6, 2, 64, 100, 3, 69, 67, 4, 70, 65, 66, 96, 0, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 108, 51, 114, 38, 50, 44, 117, 40, 105, 123, 86, 125, 87, 82, 18, 102, 127, 115, 95, 98, 88, 124, 41, 24, 34, 39, 22, 52, 58, 16, 20, 53, 80, 84, 107, 57, 23, 14, 47, 103, 104, 13, 27, 55, 56, 97, 76, 78, 62, 12, 126, 89, 83, 48, 77, 92, 85, 121, 29, 49, 93, 21, 31, 19, 75, 110, 113, 79, 99, 28, 11, 106, 15, 112, 73, 46, 9, 111, 42, 25, 30, 94, 90, 61, 109, 37, 43, 91, 71, 35, 45, 72, 33, 81, 26, 7, 10, 74, 17, 0, 64, 66, 32, 69, 36, 101, 5, 4, 65, 68, 6, 2, 67, 100, 8, 3, 1, 70, 96], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 108, 50, 38, 114, 117, 123, 44, 105, 40, 125, 102, 127, 18, 98, 95, 62, 82, 24, 87, 115, 34, 39, 41, 86, 84, 88, 52, 16, 20, 103, 57, 22, 107, 126, 58, 89, 124, 80, 55, 97, 104, 27, 47, 23, 48, 13, 29, 78, 14, 92, 53, 46, 49, 77, 28, 21, 93, 19, 31, 110, 99, 25, 76, 83, 43, 56, 11, 113, 37, 85, 111, 94, 9, 109, 75, 12, 42, 15, 79, 112, 45, 33, 91, 61, 106, 35, 72, 30, 71, 73, 26, 90, 121, 7, 17, 101, 74, 81, 32, 64, 67, 36, 10, 1, 0, 5, 4, 68, 3, 69, 2, 66, 70, 65, 6, 8, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 108, 114, 50, 117, 38, 123, 125, 44, 105, 40, 18, 102, 53, 41, 95, 82, 98, 62, 86, 87, 115, 103, 24, 88, 127, 39, 16, 49, 124, 52, 34, 80, 23, 126, 22, 84, 20, 47, 78, 58, 55, 13, 31, 97, 92, 19, 27, 89, 29, 77, 113, 107, 104, 14, 21, 75, 76, 42, 83, 57, 93, 61, 25, 12, 85, 110, 43, 99, 46, 33, 109, 26, 28, 79, 56, 112, 48, 111, 9, 45, 11, 7, 71, 72, 73, 121, 94, 15, 90, 37, 30, 35, 106, 91, 17, 36, 74, 81, 32, 10, 69, 3, 101, 0, 1, 64, 67, 68, 66, 100, 70, 5, 4, 6, 2, 65, 96, 8], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 44, 123, 125, 40, 117, 105, 102, 95, 127, 58, 55, 41, 98, 18, 47, 87, 39, 86, 82, 24, 57, 53, 52, 126, 124, 16, 103, 88, 115, 34, 49, 80, 23, 78, 22, 13, 20, 62, 84, 27, 14, 104, 107, 97, 113, 19, 61, 29, 77, 83, 31, 89, 93, 109, 28, 110, 33, 121, 75, 46, 42, 43, 11, 12, 25, 85, 99, 76, 56, 21, 92, 71, 48, 7, 15, 79, 111, 45, 73, 26, 9, 112, 35, 72, 30, 106, 37, 94, 90, 10, 0, 64, 5, 81, 101, 67, 74, 36, 17, 91, 4, 66, 32, 68, 65, 2, 69, 1, 70, 3, 96, 8, 100, 6], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 125, 44, 40, 123, 105, 127, 18, 55, 102, 86, 98, 87, 82, 95, 24, 53, 41, 39, 22, 16, 88, 78, 47, 62, 126, 124, 57, 80, 34, 84, 23, 52, 20, 29, 49, 103, 27, 58, 113, 13, 97, 115, 104, 46, 31, 83, 14, 121, 89, 110, 77, 28, 93, 43, 76, 107, 11, 75, 19, 12, 112, 92, 61, 21, 73, 99, 48, 15, 25, 90, 79, 85, 9, 109, 45, 42, 106, 33, 72, 7, 71, 56, 81, 111, 35, 30, 0, 26, 37, 36, 64, 69, 91, 10, 17, 67, 74, 3, 4, 94, 65, 8, 68, 5, 70, 66, 2, 1, 6, 101, 32, 100, 96], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 114, 50, 108, 38, 117, 44, 125, 123, 105, 40, 18, 102, 41, 127, 98, 52, 87, 34, 82, 86, 95, 88, 115, 22, 57, 23, 55, 124, 62, 24, 126, 39, 80, 84, 20, 16, 97, 58, 53, 78, 103, 110, 104, 47, 27, 89, 107, 61, 92, 49, 113, 29, 43, 13, 31, 14, 19, 28, 93, 85, 83, 77, 99, 21, 76, 25, 75, 11, 12, 56, 15, 73, 112, 109, 42, 45, 121, 46, 33, 111, 90, 9, 48, 79, 35, 26, 37, 106, 91, 7, 94, 30, 71, 17, 36, 8, 72, 81, 74, 10, 32, 69, 101, 100, 66, 68, 3, 70, 5, 4, 0, 67, 65, 64, 1, 6, 2, 96], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 114, 50, 108, 125, 38, 44, 117, 123, 105, 40, 53, 102, 41, 55, 18, 86, 95, 82, 34, 62, 127, 87, 98, 52, 124, 16, 88, 80, 103, 22, 115, 23, 24, 47, 84, 20, 58, 57, 39, 13, 104, 14, 78, 61, 49, 85, 107, 43, 110, 12, 27, 92, 31, 89, 97, 113, 19, 93, 126, 83, 21, 45, 76, 11, 77, 28, 75, 79, 29, 99, 121, 15, 112, 48, 56, 25, 42, 9, 73, 46, 7, 71, 90, 33, 8, 106, 30, 109, 26, 94, 0, 64, 111, 35, 10, 72, 91, 17, 81, 37, 74, 69, 65, 66, 4, 67, 32, 5, 68, 70, 2, 36, 101, 1, 6, 3, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 50, 114, 38, 40, 44, 117, 123, 105, 102, 125, 86, 95, 98, 18, 82, 53, 124, 41, 127, 87, 55, 84, 34, 88, 39, 62, 24, 52, 104, 57, 23, 58, 47, 49, 103, 97, 20, 16, 80, 115, 126, 14, 22, 110, 31, 78, 43, 56, 46, 113, 27, 61, 13, 29, 99, 89, 12, 107, 45, 106, 85, 77, 93, 19, 48, 83, 28, 92, 109, 75, 15, 112, 76, 25, 37, 79, 33, 21, 7, 94, 11, 73, 90, 35, 42, 71, 111, 8, 121, 91, 9, 30, 36, 26, 17, 81, 10, 0, 101, 72, 32, 67, 3, 69, 64, 68, 74, 5, 65, 100, 2, 4, 6, 66, 1, 70, 96], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 125, 44, 40, 123, 105, 102, 117, 86, 52, 124, 18, 127, 98, 115, 82, 57, 41, 88, 95, 103, 39, 87, 78, 62, 24, 34, 61, 107, 16, 20, 126, 23, 47, 53, 110, 97, 104, 22, 55, 80, 84, 49, 29, 31, 27, 89, 109, 19, 83, 113, 45, 85, 99, 58, 46, 93, 48, 14, 13, 21, 25, 43, 77, 28, 76, 92, 121, 11, 56, 37, 12, 42, 112, 75, 8, 106, 9, 79, 33, 30, 15, 7, 35, 73, 36, 90, 26, 71, 111, 17, 91, 94, 32, 67, 10, 81, 74, 68, 5, 64, 0, 65, 4, 2, 3, 101, 69, 1, 70, 96, 72, 66, 100, 6], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 108, 114, 50, 38, 117, 44, 40, 123, 105, 125, 86, 18, 102, 82, 41, 98, 52, 87, 95, 22, 34, 124, 88, 24, 115, 39, 127, 16, 23, 62, 20, 80, 55, 104, 49, 103, 84, 126, 110, 58, 48, 56, 97, 78, 57, 27, 53, 93, 89, 83, 92, 29, 14, 47, 19, 45, 31, 107, 13, 77, 61, 85, 28, 113, 21, 76, 11, 12, 33, 42, 9, 99, 112, 75, 25, 46, 79, 43, 121, 8, 15, 90, 37, 73, 35, 71, 7, 30, 109, 106, 94, 26, 17, 10, 111, 91, 5, 81, 74, 36, 101, 32, 4, 64, 6, 0, 72, 68, 100, 66, 69, 70, 65, 2, 96, 1, 67, 3], [59, 60, 54, 118, 116, 119, 122, 120, 63, 51, 108, 50, 38, 114, 44, 40, 105, 117, 86, 125, 123, 102, 127, 41, 82, 98, 87, 18, 124, 34, 22, 24, 95, 39, 88, 62, 97, 115, 23, 52, 20, 16, 55, 103, 80, 84, 126, 110, 27, 56, 49, 57, 93, 58, 104, 19, 29, 31, 78, 48, 89, 83, 92, 47, 13, 113, 85, 14, 99, 25, 28, 12, 107, 76, 53, 61, 21, 77, 112, 26, 75, 42, 45, 46, 15, 111, 7, 90, 11, 33, 109, 79, 43, 30, 9, 35, 94, 8, 73, 37, 106, 71, 121, 17, 91, 32, 36, 5, 81, 101, 6, 74, 10, 100, 0, 64, 67, 72, 4, 68, 96, 66, 3, 70, 2, 69, 65, 1], [59, 60, 54, 118, 116, 119, 122, 120, 51, 63, 108, 114, 50, 38, 117, 40, 44, 123, 105, 125, 98, 86, 102, 87, 62, 34, 95, 82, 97, 39, 127, 124, 57, 41, 18, 52, 115, 24, 49, 88, 22, 55, 103, 23, 58, 53, 80, 126, 104, 29, 84, 16, 110, 56, 45, 20, 89, 61, 99, 25, 27, 31, 14, 26, 48, 46, 47, 83, 113, 93, 109, 92, 19, 33, 107, 28, 43, 42, 13, 78, 112, 12, 85, 77, 37, 35, 75, 15, 21, 73, 121, 76, 8, 9, 106, 36, 30, 94, 90, 11, 7, 79, 32, 91, 17, 71, 111, 101, 69, 3, 81, 74, 64, 10, 67, 0, 65, 4, 96, 5, 72, 6, 68, 66, 2, 1, 100, 70], [59, 60, 54, 118, 116, 122, 119, 63, 120, 51, 50, 108, 114, 38, 117, 44, 125, 40, 105, 102, 123, 86, 127, 98, 95, 41, 52, 82, 34, 18, 103, 39, 87, 115, 24, 57, 88, 55, 124, 62, 22, 23, 104, 80, 20, 16, 107, 126, 97, 27, 89, 110, 84, 56, 61, 58, 14, 83, 47, 53, 49, 12, 77, 31, 113, 25, 78, 93, 19, 99, 13, 28, 109, 48, 92, 29, 21, 46, 45, 30, 75, 76, 85, 112, 33, 43, 106, 42, 11, 73, 37, 26, 71, 35, 79, 121, 15, 9, 90, 7, 36, 17, 8, 91, 94, 111, 101, 74, 32, 10, 72, 69, 64, 81, 67, 65, 0, 2, 68, 96, 4, 5, 6, 100, 70, 66, 3, 1], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 105, 44, 40, 86, 117, 125, 123, 102, 127, 18, 41, 124, 87, 126, 39, 52, 82, 88, 98, 55, 95, 22, 103, 49, 24, 115, 57, 34, 16, 23, 80, 20, 78, 62, 84, 53, 47, 56, 83, 104, 27, 97, 93, 13, 77, 29, 89, 14, 19, 110, 111, 61, 107, 113, 85, 75, 58, 31, 12, 48, 99, 76, 28, 21, 25, 109, 9, 71, 45, 92, 106, 79, 43, 11, 90, 37, 33, 26, 46, 42, 30, 121, 112, 73, 35, 7, 15, 36, 81, 94, 8, 32, 72, 67, 74, 17, 91, 64, 10, 68, 69, 3, 2, 0, 5, 101, 6, 1, 70, 65, 4, 66, 96, 100], [59, 60, 118, 54, 119, 122, 116, 120, 63, 51, 108, 50, 38, 114, 117, 44, 123, 105, 40, 86, 125, 87, 18, 102, 41, 98, 82, 55, 62, 124, 115, 39, 52, 34, 24, 95, 22, 57, 88, 103, 80, 127, 126, 23, 97, 47, 16, 84, 29, 48, 20, 78, 53, 89, 110, 27, 49, 61, 93, 56, 43, 92, 19, 58, 31, 42, 25, 83, 107, 104, 77, 113, 14, 75, 13, 99, 76, 85, 28, 112, 121, 12, 21, 46, 90, 106, 109, 79, 26, 11, 33, 30, 9, 15, 35, 73, 94, 111, 37, 45, 36, 71, 72, 8, 17, 7, 91, 81, 74, 5, 0, 101, 32, 64, 10, 3, 100, 68, 2, 67, 65, 69, 70, 96, 4, 1, 66, 6], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 38, 50, 117, 44, 123, 105, 125, 40, 124, 41, 86, 98, 34, 102, 18, 39, 87, 127, 95, 62, 115, 52, 82, 24, 22, 55, 23, 57, 103, 88, 47, 16, 84, 53, 49, 97, 126, 80, 61, 20, 48, 58, 27, 104, 78, 56, 89, 13, 45, 19, 93, 113, 14, 92, 77, 31, 76, 110, 29, 83, 99, 107, 11, 85, 42, 12, 112, 28, 25, 75, 21, 26, 37, 121, 73, 43, 9, 46, 106, 111, 109, 33, 90, 79, 30, 15, 94, 7, 35, 72, 71, 36, 91, 17, 101, 32, 74, 0, 81, 64, 69, 100, 10, 65, 3, 8, 5, 68, 2, 96, 66, 67, 6, 4, 70, 1], [59, 60, 118, 54, 119, 116, 122, 63, 120, 51, 108, 114, 50, 38, 117, 125, 44, 40, 105, 123, 86, 102, 98, 127, 62, 95, 124, 87, 52, 41, 88, 82, 39, 18, 103, 115, 23, 34, 24, 57, 53, 16, 22, 126, 49, 84, 92, 110, 55, 58, 80, 97, 27, 47, 25, 20, 107, 56, 89, 31, 78, 83, 99, 42, 14, 104, 19, 121, 93, 77, 29, 48, 21, 13, 113, 12, 43, 45, 76, 28, 75, 85, 46, 73, 11, 72, 61, 15, 33, 109, 106, 30, 35, 112, 90, 37, 79, 26, 7, 111, 36, 91, 71, 9, 32, 94, 68, 17, 81, 0, 74, 69, 65, 70, 64, 100, 5, 67, 101, 3, 2, 10, 1, 66, 4, 6, 8, 96], [59, 60, 54, 118, 116, 119, 122, 63, 120, 51, 108, 50, 114, 38, 44, 117, 40, 125, 123, 105, 86, 102, 127, 98, 52, 124, 18, 41, 82, 39, 95, 87, 55, 57, 24, 34, 62, 88, 126, 22, 16, 97, 115, 53, 84, 49, 110, 47, 103, 58, 23, 80, 20, 83, 14, 29, 99, 104, 46, 48, 43, 56, 77, 89, 13, 61, 113, 78, 27, 93, 31, 25, 28, 73, 45, 76, 92, 85, 21, 107, 111, 19, 106, 121, 12, 75, 11, 112, 109, 72, 33, 15, 9, 42, 90, 35, 37, 79, 26, 71, 36, 94, 7, 81, 91, 74, 32, 5, 17, 30, 101, 10, 0, 64, 100, 68, 4, 70, 67, 2, 3, 69, 1, 96, 6, 8, 65, 66], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 38, 50, 114, 44, 117, 40, 125, 105, 123, 102, 86, 52, 98, 18, 127, 124, 82, 87, 95, 22, 39, 41, 88, 126, 115, 34, 55, 62, 24, 103, 97, 16, 23, 57, 84, 80, 20, 110, 47, 83, 58, 89, 27, 14, 53, 45, 56, 78, 49, 12, 19, 92, 29, 93, 61, 13, 107, 77, 104, 46, 21, 42, 43, 121, 99, 85, 28, 31, 76, 75, 9, 48, 15, 11, 79, 26, 112, 35, 73, 25, 106, 71, 90, 33, 72, 109, 113, 30, 37, 17, 36, 74, 94, 7, 81, 91, 32, 111, 64, 0, 101, 10, 100, 69, 4, 5, 66, 65, 8, 70, 3, 96, 6, 67, 2, 1, 68], [59, 60, 118, 54, 116, 119, 122, 120, 63, 51, 50, 108, 114, 38, 40, 123, 105, 44, 117, 125, 62, 102, 124, 98, 87, 41, 86, 55, 57, 95, 82, 126, 34, 22, 52, 47, 127, 24, 39, 18, 88, 103, 97, 115, 84, 27, 89, 104, 23, 20, 53, 16, 49, 29, 93, 80, 78, 99, 48, 31, 61, 58, 83, 92, 42, 121, 21, 45, 110, 13, 107, 25, 77, 14, 85, 46, 19, 111, 12, 43, 90, 33, 11, 75, 26, 112, 28, 37, 9, 56, 15, 94, 30, 109, 76, 113, 35, 91, 7, 73, 106, 72, 79, 71, 17, 32, 101, 100, 67, 74, 69, 36, 0, 10, 4, 2, 68, 1, 66, 64, 65, 5, 81, 3, 70, 96, 6, 8], [59, 60, 118, 54, 116, 119, 122, 63, 120, 51, 108, 114, 50, 38, 44, 40, 105, 123, 117, 125, 86, 82, 87, 41, 127, 124, 102, 95, 98, 52, 39, 126, 18, 24, 34, 62, 88, 57, 22, 55, 23, 47, 16, 80, 115, 58, 20, 97, 84, 78, 83, 103, 104, 27, 56, 14, 93, 89, 29, 12, 92, 110, 19, 13, 107, 77, 76, 75, 49, 53, 21, 121, 45, 48, 61, 85, 11, 99, 46, 31, 43, 42, 109, 79, 15, 28, 9, 7, 73, 71, 111, 112, 113, 106, 25, 30, 0, 26, 94, 64, 72, 37, 33, 90, 35, 69, 17, 65, 74, 3, 8, 6, 4, 68, 66, 91, 1, 67, 10, 101, 2, 81, 36, 32, 5, 70, 96, 100], [59, 60, 118, 54, 116, 119, 122, 120, 51, 63, 114, 108, 50, 38, 40, 117, 105, 44, 123, 102, 98, 127, 87, 125, 57, 41, 124, 18, 82, 95, 39, 52, 126, 86, 34, 62, 88, 24, 22, 103, 55, 97, 115, 58, 23, 84, 49, 20, 27, 16, 80, 104, 53, 31, 56, 89, 99, 47, 93, 78, 113, 29, 107, 83, 13, 121, 14, 110, 12, 77, 85, 25, 48, 109, 61, 43, 92, 45, 19, 28, 26, 106, 46, 21, 33, 90, 11, 94, 37, 75, 30, 76, 9, 111, 42, 35, 71, 15, 73, 112, 79, 36, 72, 91, 7, 8, 32, 64, 69, 74, 10, 17, 0, 66, 67, 1, 68, 6, 65, 101, 81, 3, 4, 5, 2, 96, 70, 100], [59, 60, 118, 54, 122, 116, 119, 120, 51, 63, 114, 108, 50, 38, 44, 125, 40, 105, 123, 117, 102, 86, 98, 52, 87, 41, 18, 127, 34, 82, 95, 124, 126, 88, 53, 57, 39, 115, 58, 55, 24, 80, 20, 47, 16, 103, 78, 97, 84, 22, 14, 23, 19, 31, 56, 62, 93, 13, 113, 49, 77, 27, 76, 42, 48, 83, 75, 43, 121, 89, 110, 85, 21, 12, 107, 29, 99, 92, 61, 104, 45, 25, 11, 46, 73, 109, 35, 15, 79, 9, 28, 106, 26, 30, 8, 90, 71, 7, 33, 112, 37, 94, 111, 17, 64, 72, 0, 10, 74, 66, 69, 65, 4, 81, 100, 3, 91, 36, 5, 1, 6, 68, 32, 67, 70, 101, 2, 96], [59, 60, 118, 54, 116, 122, 119, 120, 51, 63, 114, 108, 50, 38, 44, 117, 40, 123, 125, 105, 55, 41, 102, 98, 52, 86, 95, 127, 126, 34, 124, 18, 53, 82, 115, 49, 87, 58, 39, 57, 84, 24, 23, 88, 80, 103, 62, 97, 47, 22, 20, 16, 113, 14, 31, 104, 89, 27, 110, 48, 99, 45, 19, 29, 78, 61, 107, 77, 85, 92, 12, 56, 121, 93, 21, 13, 76, 83, 11, 43, 109, 112, 42, 106, 73, 28, 75, 25, 33, 15, 8, 46, 9, 111, 26, 71, 79, 90, 35, 94, 37, 30, 7, 64, 69, 17, 74, 10, 81, 68, 32, 5, 1, 0, 2, 72, 4, 36, 91, 67, 101, 6, 70, 66, 3, 65, 100, 96], [59, 60, 54, 118, 116, 122, 119, 120, 63, 51, 108, 114, 50, 38, 123, 44, 125, 117, 40, 105, 86, 41, 52, 127, 98, 102, 55, 87, 82, 34, 95, 58, 115, 18, 126, 62, 22, 24, 53, 124, 23, 88, 80, 47, 39, 103, 16, 20, 57, 84, 97, 49, 27, 14, 12, 77, 104, 19, 92, 93, 89, 21, 13, 15, 48, 83, 110, 76, 31, 78, 61, 45, 85, 11, 107, 113, 43, 42, 29, 73, 33, 9, 99, 121, 28, 75, 109, 25, 56, 46, 26, 8, 79, 71, 90, 35, 7, 37, 106, 30, 94, 112, 111, 81, 74, 64, 17, 91, 5, 10, 0, 32, 68, 36, 1, 65, 67, 69, 72, 4, 6, 101, 70, 2, 100, 66, 3, 96]], "model.layers.9.self_attn.q_proj": [[110, 101, 46, 124, 63, 59, 28, 32, 113, 25, 89, 61, 19, 115, 27, 87, 16, 60, 57, 49, 122, 83, 85, 26, 78, 22, 47, 114, 55, 58, 56, 53, 96, 99, 37, 92, 108, 41, 23, 54, 48, 125, 30, 79, 51, 11, 119, 67, 84, 1, 24, 118, 69, 35, 62, 91, 120, 123, 116, 70, 43, 88, 97, 117, 121, 68, 106, 112, 90, 126, 109, 29, 40, 2, 77, 127, 45, 111, 8, 17, 86, 0, 74, 50, 82, 107, 15, 5, 42, 9, 66, 21, 18, 10, 52, 14, 6, 104, 105, 94, 36, 93, 100, 81, 65, 102, 44, 34, 33, 76, 20, 80, 31, 103, 3, 64, 7, 38, 4, 71, 95, 75, 73, 12, 13, 39, 98, 72], [110, 124, 101, 63, 46, 113, 25, 59, 80, 28, 24, 61, 89, 115, 57, 122, 60, 56, 83, 114, 32, 53, 58, 47, 125, 27, 22, 51, 41, 54, 119, 49, 78, 55, 40, 11, 112, 118, 62, 48, 123, 43, 96, 120, 106, 69, 23, 121, 37, 108, 52, 117, 127, 50, 42, 109, 19, 104, 45, 100, 111, 116, 35, 126, 105, 16, 107, 85, 91, 87, 102, 30, 38, 39, 31, 44, 33, 92, 34, 103, 84, 36, 86, 20, 17, 94, 88, 26, 97, 73, 98, 29, 99, 93, 8, 90, 95, 77, 72, 14, 18, 1, 21, 12, 67, 81, 5, 79, 76, 15, 82, 70, 4, 75, 2, 0, 10, 9, 7, 13, 6, 74, 66, 3, 68, 71, 64, 65], [110, 124, 46, 101, 63, 59, 89, 61, 11, 37, 113, 57, 25, 122, 60, 115, 69, 56, 53, 19, 54, 58, 47, 55, 32, 114, 120, 48, 16, 51, 49, 41, 117, 118, 116, 28, 112, 50, 62, 27, 109, 24, 119, 22, 121, 111, 125, 92, 127, 99, 126, 123, 43, 87, 45, 52, 82, 30, 107, 108, 39, 100, 72, 35, 103, 42, 105, 106, 1, 96, 104, 83, 40, 0, 4, 44, 23, 78, 2, 86, 77, 80, 26, 102, 15, 38, 29, 34, 85, 97, 90, 84, 67, 73, 36, 31, 33, 14, 88, 94, 5, 6, 98, 95, 18, 17, 9, 93, 91, 7, 21, 8, 20, 68, 79, 12, 81, 75, 66, 10, 13, 76, 71, 70, 74, 64, 3, 65], [124, 110, 101, 59, 46, 89, 63, 61, 19, 28, 115, 25, 83, 60, 113, 56, 27, 57, 23, 114, 122, 24, 118, 54, 78, 53, 55, 22, 125, 119, 47, 49, 58, 120, 48, 51, 37, 26, 107, 93, 32, 50, 81, 96, 117, 121, 99, 112, 11, 106, 43, 62, 123, 92, 127, 41, 35, 109, 108, 17, 116, 111, 103, 45, 42, 36, 104, 52, 44, 82, 126, 40, 100, 105, 30, 39, 38, 85, 88, 95, 14, 12, 34, 86, 87, 8, 98, 97, 31, 33, 102, 29, 16, 91, 94, 77, 69, 84, 18, 21, 73, 20, 15, 90, 80, 75, 76, 67, 13, 5, 79, 72, 9, 6, 74, 7, 1, 10, 70, 4, 3, 71, 2, 0, 65, 68, 66, 64], [117, 41, 59, 61, 97, 126, 101, 39, 31, 87, 55, 43, 42, 44, 28, 100, 124, 112, 110, 116, 35, 92, 102, 113, 96, 122, 57, 86, 36, 121, 127, 89, 54, 58, 114, 63, 38, 49, 52, 115, 93, 125, 47, 94, 51, 60, 53, 105, 99, 120, 16, 80, 48, 111, 32, 45, 50, 84, 46, 98, 29, 104, 19, 56, 118, 109, 123, 108, 40, 62, 27, 78, 33, 24, 107, 23, 74, 18, 26, 17, 25, 83, 37, 85, 75, 30, 91, 22, 20, 119, 21, 34, 14, 106, 12, 88, 9, 68, 81, 13, 103, 15, 77, 10, 3, 7, 76, 73, 95, 11, 71, 79, 4, 82, 66, 8, 90, 2, 70, 5, 6, 0, 72, 69, 64, 67, 65, 1], [41, 64, 107, 101, 117, 31, 67, 63, 1, 56, 69, 55, 26, 97, 42, 126, 112, 87, 105, 2, 72, 10, 39, 18, 84, 79, 76, 98, 71, 124, 65, 43, 106, 28, 78, 4, 35, 9, 111, 6, 44, 100, 53, 70, 29, 93, 23, 114, 80, 12, 121, 3, 74, 58, 86, 66, 20, 92, 94, 57, 5, 81, 11, 89, 59, 127, 33, 110, 102, 99, 83, 38, 75, 7, 48, 125, 109, 8, 61, 19, 15, 17, 77, 27, 21, 0, 46, 113, 88, 62, 118, 68, 50, 85, 45, 116, 32, 54, 104, 47, 24, 25, 120, 52, 122, 96, 40, 13, 51, 115, 103, 60, 90, 30, 34, 14, 73, 119, 123, 49, 82, 36, 16, 108, 91, 22, 95, 37], [41, 117, 26, 87, 31, 42, 101, 97, 56, 98, 18, 124, 35, 84, 105, 79, 15, 23, 12, 76, 55, 39, 57, 78, 43, 20, 80, 7, 59, 112, 93, 106, 118, 44, 74, 90, 61, 102, 10, 89, 100, 63, 28, 83, 114, 126, 107, 82, 58, 9, 53, 25, 104, 92, 69, 81, 14, 48, 8, 96, 36, 21, 127, 95, 86, 70, 24, 49, 121, 110, 99, 11, 29, 60, 111, 22, 46, 27, 17, 62, 85, 51, 122, 109, 16, 4, 52, 120, 125, 32, 5, 19, 77, 33, 13, 123, 40, 54, 45, 91, 2, 30, 67, 94, 71, 47, 3, 73, 115, 38, 113, 116, 72, 88, 64, 108, 50, 37, 103, 66, 65, 75, 68, 34, 0, 119, 6, 1], [117, 56, 41, 61, 107, 43, 97, 39, 31, 126, 35, 44, 118, 100, 42, 57, 110, 89, 121, 102, 36, 105, 46, 87, 63, 48, 101, 94, 52, 124, 92, 28, 59, 53, 113, 50, 86, 55, 38, 98, 54, 122, 49, 51, 29, 112, 111, 25, 45, 115, 108, 83, 62, 123, 125, 127, 80, 40, 47, 96, 104, 19, 99, 114, 27, 58, 116, 120, 60, 33, 109, 34, 88, 26, 18, 32, 93, 85, 9, 23, 106, 22, 64, 21, 24, 16, 119, 84, 30, 17, 103, 13, 91, 37, 77, 14, 90, 1, 2, 81, 69, 78, 67, 76, 95, 11, 79, 4, 82, 66, 73, 71, 68, 7, 70, 10, 75, 72, 20, 5, 12, 74, 6, 8, 65, 15, 3, 0], [105, 98, 86, 18, 84, 53, 79, 81, 88, 13, 111, 41, 76, 119, 74, 51, 72, 55, 29, 10, 117, 52, 7, 61, 2, 63, 123, 114, 68, 110, 107, 71, 6, 126, 31, 59, 70, 26, 125, 116, 5, 15, 4, 17, 48, 28, 120, 50, 118, 8, 127, 108, 77, 75, 90, 12, 27, 1, 3, 82, 24, 85, 92, 11, 109, 21, 122, 42, 124, 20, 38, 106, 9, 62, 40, 57, 93, 78, 25, 113, 33, 49, 22, 66, 80, 56, 16, 67, 58, 39, 30, 32, 60, 19, 23, 102, 91, 37, 54, 46, 87, 89, 64, 100, 96, 43, 115, 36, 83, 101, 14, 112, 47, 94, 95, 121, 103, 44, 35, 97, 99, 104, 0, 45, 34, 69, 73, 65], [105, 98, 84, 79, 88, 53, 18, 72, 41, 86, 111, 5, 81, 51, 2, 13, 63, 15, 76, 10, 29, 55, 52, 6, 68, 114, 119, 126, 74, 116, 110, 12, 123, 48, 92, 7, 117, 61, 28, 26, 107, 71, 9, 59, 27, 11, 118, 4, 87, 90, 31, 3, 93, 8, 85, 127, 125, 24, 122, 70, 109, 78, 1, 58, 20, 108, 50, 120, 124, 77, 91, 38, 66, 42, 49, 17, 22, 40, 39, 60, 30, 102, 69, 95, 67, 57, 54, 34, 106, 83, 23, 37, 62, 89, 32, 101, 16, 82, 21, 75, 96, 36, 19, 56, 33, 35, 121, 43, 112, 103, 113, 80, 46, 73, 94, 25, 44, 99, 14, 115, 104, 100, 0, 97, 45, 64, 47, 65], [105, 98, 84, 41, 72, 86, 53, 6, 18, 2, 79, 12, 4, 1, 76, 68, 71, 111, 74, 51, 13, 0, 3, 67, 81, 88, 66, 5, 114, 126, 63, 61, 123, 119, 8, 117, 26, 52, 116, 55, 48, 118, 125, 107, 127, 29, 65, 110, 59, 27, 10, 92, 93, 7, 85, 90, 20, 17, 109, 34, 120, 24, 9, 87, 50, 28, 122, 124, 22, 58, 62, 78, 70, 80, 49, 42, 32, 57, 108, 31, 30, 106, 83, 101, 35, 25, 56, 54, 21, 102, 77, 91, 14, 82, 113, 95, 64, 89, 15, 112, 38, 75, 97, 39, 36, 43, 115, 60, 40, 33, 121, 23, 19, 103, 37, 94, 11, 100, 46, 104, 45, 99, 69, 16, 96, 47, 44, 73], [105, 98, 74, 3, 70, 0, 86, 53, 111, 4, 1, 81, 18, 79, 76, 61, 5, 41, 67, 66, 119, 13, 117, 71, 8, 84, 2, 51, 110, 123, 68, 114, 55, 12, 52, 72, 126, 50, 125, 65, 120, 116, 107, 63, 17, 6, 9, 20, 59, 57, 64, 118, 88, 29, 109, 48, 10, 127, 49, 56, 38, 62, 73, 124, 34, 26, 7, 22, 40, 106, 78, 87, 23, 30, 15, 16, 28, 11, 24, 69, 93, 92, 19, 77, 89, 80, 83, 14, 60, 31, 94, 113, 75, 42, 39, 32, 82, 85, 108, 122, 54, 25, 43, 58, 121, 27, 90, 21, 96, 102, 115, 44, 101, 99, 112, 91, 36, 46, 95, 35, 33, 103, 104, 45, 37, 100, 47, 97], [125, 62, 104, 48, 63, 119, 122, 120, 59, 108, 55, 57, 118, 54, 121, 116, 58, 60, 27, 124, 113, 114, 115, 117, 84, 123, 111, 61, 97, 56, 53, 50, 43, 51, 47, 30, 44, 87, 52, 49, 126, 127, 107, 45, 109, 25, 110, 89, 12, 46, 21, 105, 28, 22, 72, 99, 112, 103, 41, 36, 38, 94, 106, 34, 5, 98, 90, 39, 102, 101, 20, 100, 1, 37, 92, 14, 9, 66, 96, 68, 42, 19, 16, 83, 35, 32, 79, 17, 29, 33, 95, 2, 85, 69, 31, 81, 0, 23, 40, 3, 91, 77, 86, 88, 18, 4, 93, 6, 15, 64, 82, 71, 76, 13, 11, 26, 24, 10, 73, 80, 78, 75, 74, 7, 8, 70, 65, 67], [62, 125, 48, 104, 63, 119, 122, 120, 59, 108, 97, 115, 55, 58, 54, 57, 60, 117, 114, 116, 121, 52, 124, 113, 50, 118, 47, 123, 61, 56, 111, 84, 36, 53, 127, 51, 89, 49, 109, 126, 44, 87, 5, 43, 72, 45, 14, 96, 69, 103, 46, 112, 107, 12, 30, 41, 110, 1, 106, 102, 105, 68, 2, 27, 32, 101, 66, 37, 28, 39, 38, 42, 24, 88, 98, 21, 25, 20, 22, 19, 35, 95, 99, 92, 100, 31, 64, 34, 23, 40, 83, 3, 93, 4, 15, 90, 91, 71, 76, 29, 85, 75, 79, 0, 6, 18, 94, 78, 9, 8, 17, 73, 77, 81, 13, 16, 7, 26, 10, 80, 33, 86, 11, 70, 74, 67, 82, 65], [104, 125, 62, 48, 63, 97, 119, 30, 21, 57, 122, 60, 59, 28, 127, 115, 118, 120, 116, 108, 55, 85, 84, 27, 54, 121, 113, 126, 123, 53, 114, 49, 56, 25, 111, 58, 117, 61, 22, 89, 124, 87, 52, 47, 9, 112, 109, 90, 44, 107, 51, 50, 45, 43, 40, 35, 5, 79, 46, 36, 105, 103, 94, 66, 1, 18, 68, 24, 41, 78, 92, 16, 12, 34, 15, 2, 98, 99, 102, 38, 95, 110, 72, 39, 19, 32, 3, 14, 96, 69, 11, 106, 101, 64, 31, 42, 37, 0, 6, 93, 91, 73, 100, 88, 83, 17, 23, 75, 20, 8, 29, 81, 4, 77, 26, 33, 13, 70, 71, 74, 7, 10, 80, 86, 76, 67, 65, 82], [104, 62, 125, 90, 30, 97, 63, 122, 84, 18, 19, 115, 23, 60, 80, 22, 16, 119, 27, 74, 79, 38, 57, 49, 26, 36, 78, 55, 20, 54, 59, 39, 82, 13, 124, 99, 120, 108, 94, 116, 123, 112, 45, 87, 21, 121, 111, 118, 114, 28, 83, 24, 113, 61, 126, 33, 88, 50, 58, 44, 127, 56, 17, 75, 48, 107, 10, 53, 102, 92, 34, 81, 100, 91, 109, 117, 96, 43, 51, 101, 47, 77, 103, 52, 14, 46, 25, 93, 15, 105, 37, 41, 31, 106, 6, 42, 110, 73, 98, 85, 70, 40, 5, 89, 12, 35, 32, 76, 95, 86, 68, 8, 29, 4, 7, 11, 71, 3, 9, 64, 1, 66, 65, 72, 67, 69, 0, 2], [103, 124, 55, 56, 21, 33, 15, 57, 29, 113, 81, 83, 91, 88, 27, 122, 123, 125, 93, 41, 10, 25, 35, 49, 98, 114, 24, 12, 78, 61, 105, 60, 48, 30, 109, 86, 115, 22, 52, 120, 112, 85, 53, 51, 110, 58, 31, 116, 43, 62, 45, 17, 94, 108, 44, 84, 100, 42, 107, 102, 70, 119, 111, 90, 126, 23, 18, 76, 118, 121, 117, 47, 36, 46, 4, 106, 127, 26, 89, 54, 63, 32, 50, 59, 20, 19, 11, 99, 8, 28, 40, 79, 101, 96, 13, 104, 37, 80, 38, 82, 95, 92, 16, 77, 34, 87, 72, 6, 65, 14, 2, 75, 97, 9, 5, 68, 39, 73, 74, 71, 7, 69, 0, 66, 67, 1, 64, 3], [103, 124, 55, 57, 88, 33, 56, 21, 105, 81, 113, 93, 123, 29, 15, 83, 23, 41, 122, 91, 125, 19, 49, 70, 78, 27, 114, 10, 12, 44, 86, 24, 126, 117, 45, 90, 98, 109, 2, 22, 58, 76, 52, 74, 4, 108, 120, 30, 60, 118, 107, 115, 48, 111, 106, 84, 110, 43, 65, 5, 85, 51, 35, 53, 119, 32, 18, 38, 39, 102, 17, 112, 99, 95, 34, 100, 104, 79, 47, 68, 54, 116, 92, 101, 42, 121, 20, 97, 61, 62, 31, 26, 46, 28, 96, 127, 87, 36, 11, 6, 50, 82, 94, 73, 37, 14, 89, 75, 59, 77, 8, 25, 16, 67, 40, 7, 63, 80, 0, 13, 9, 69, 72, 71, 1, 66, 64, 3], [103, 56, 124, 55, 57, 33, 88, 29, 113, 21, 15, 93, 105, 23, 86, 10, 77, 81, 114, 123, 91, 122, 64, 67, 125, 120, 78, 109, 0, 45, 85, 83, 60, 49, 61, 51, 65, 41, 52, 115, 53, 58, 119, 13, 110, 116, 112, 118, 27, 34, 2, 108, 48, 73, 24, 84, 71, 62, 42, 63, 89, 30, 50, 90, 14, 8, 6, 46, 126, 117, 107, 43, 70, 35, 26, 11, 17, 44, 121, 47, 111, 80, 3, 127, 20, 54, 59, 102, 7, 104, 100, 87, 37, 38, 82, 79, 106, 40, 22, 31, 9, 16, 97, 99, 36, 1, 18, 28, 101, 98, 66, 76, 12, 32, 25, 94, 68, 96, 4, 75, 92, 95, 5, 74, 19, 72, 69, 39], [124, 103, 55, 57, 41, 123, 88, 56, 125, 70, 114, 74, 122, 33, 19, 113, 21, 93, 110, 115, 23, 45, 49, 109, 120, 118, 52, 58, 116, 61, 15, 53, 108, 2, 112, 48, 83, 51, 60, 43, 35, 117, 81, 62, 29, 119, 126, 105, 73, 44, 50, 91, 121, 84, 127, 65, 111, 47, 46, 12, 107, 86, 42, 54, 63, 98, 30, 68, 106, 39, 14, 22, 59, 78, 100, 79, 27, 40, 102, 101, 10, 90, 5, 4, 38, 104, 67, 96, 99, 92, 24, 13, 36, 37, 34, 75, 26, 69, 20, 32, 8, 97, 85, 87, 17, 7, 28, 80, 25, 82, 18, 31, 76, 11, 94, 0, 71, 95, 16, 6, 77, 89, 9, 72, 3, 64, 1, 66], [57, 126, 39, 34, 61, 45, 95, 111, 120, 125, 60, 21, 51, 63, 58, 87, 52, 90, 54, 113, 114, 119, 117, 123, 109, 55, 127, 48, 50, 62, 118, 112, 122, 19, 106, 53, 56, 49, 46, 72, 59, 43, 115, 124, 110, 5, 82, 116, 108, 100, 41, 22, 36, 121, 47, 105, 17, 104, 28, 44, 26, 92, 107, 102, 88, 35, 73, 9, 42, 79, 14, 24, 16, 40, 38, 37, 77, 76, 78, 89, 99, 101, 27, 18, 30, 97, 75, 10, 32, 71, 93, 83, 98, 1, 33, 94, 13, 81, 66, 96, 29, 86, 25, 91, 103, 85, 8, 67, 11, 4, 2, 0, 20, 12, 74, 31, 70, 23, 15, 84, 69, 65, 3, 6, 7, 80, 68, 64], [39, 126, 57, 34, 95, 90, 19, 87, 60, 120, 21, 26, 61, 17, 92, 11, 111, 48, 79, 28, 31, 5, 71, 72, 22, 10, 45, 63, 117, 84, 27, 9, 58, 70, 102, 42, 4, 67, 100, 125, 127, 105, 40, 36, 108, 51, 83, 113, 99, 16, 101, 13, 38, 73, 41, 54, 96, 75, 46, 35, 53, 77, 124, 122, 52, 106, 66, 50, 44, 37, 49, 81, 59, 47, 112, 118, 62, 23, 104, 109, 119, 33, 43, 116, 85, 115, 114, 121, 110, 30, 32, 55, 123, 29, 97, 107, 25, 56, 0, 24, 20, 89, 65, 93, 15, 7, 80, 94, 18, 98, 91, 74, 1, 88, 76, 78, 6, 68, 12, 14, 82, 103, 69, 86, 8, 2, 3, 64], [39, 57, 126, 34, 95, 87, 60, 21, 19, 90, 61, 16, 120, 73, 28, 26, 22, 79, 75, 17, 77, 48, 111, 125, 5, 58, 51, 83, 13, 63, 9, 92, 45, 104, 74, 33, 82, 113, 118, 27, 112, 122, 109, 127, 18, 123, 7, 71, 78, 20, 10, 89, 117, 62, 23, 119, 31, 30, 49, 50, 36, 115, 53, 56, 81, 94, 102, 54, 24, 84, 76, 52, 70, 25, 67, 47, 96, 68, 55, 110, 114, 91, 44, 88, 85, 116, 108, 41, 40, 8, 15, 6, 2, 46, 38, 121, 99, 80, 1, 105, 100, 37, 106, 14, 29, 97, 124, 93, 42, 98, 35, 12, 32, 72, 101, 107, 59, 0, 3, 43, 86, 11, 69, 103, 64, 4, 65, 66], [39, 57, 126, 34, 120, 87, 95, 90, 19, 60, 26, 92, 11, 61, 21, 111, 36, 17, 72, 22, 5, 48, 117, 63, 125, 71, 9, 4, 127, 67, 58, 83, 16, 79, 75, 70, 66, 73, 52, 51, 27, 113, 13, 122, 91, 119, 53, 50, 115, 45, 46, 31, 10, 28, 124, 65, 123, 23, 55, 118, 109, 54, 56, 114, 40, 62, 20, 121, 15, 18, 32, 49, 0, 77, 7, 102, 81, 44, 47, 82, 37, 85, 112, 101, 38, 106, 43, 110, 100, 35, 24, 116, 80, 107, 105, 78, 99, 33, 76, 41, 74, 94, 59, 1, 14, 84, 108, 97, 30, 104, 86, 42, 88, 96, 68, 93, 89, 6, 25, 98, 8, 12, 3, 2, 69, 29, 64, 103], [59, 61, 53, 101, 27, 88, 76, 78, 18, 16, 25, 20, 22, 85, 29, 69, 7, 96, 28, 75, 9, 30, 119, 62, 100, 56, 87, 66, 39, 14, 126, 83, 82, 117, 90, 37, 2, 71, 43, 67, 3, 12, 91, 72, 21, 79, 80, 74, 77, 73, 84, 0, 33, 24, 26, 93, 81, 65, 15, 23, 19, 6, 17, 1, 31, 54, 94, 8, 92, 36, 50, 10, 89, 5, 97, 105, 111, 41, 95, 13, 35, 106, 108, 116, 107, 42, 70, 103, 68, 113, 86, 32, 124, 48, 49, 4, 11, 99, 44, 60, 98, 64, 34, 55, 102, 63, 115, 120, 46, 52, 125, 104, 121, 123, 122, 110, 45, 57, 58, 127, 40, 47, 109, 118, 38, 114, 112, 51], [53, 59, 61, 100, 119, 56, 107, 50, 124, 52, 126, 113, 123, 116, 60, 54, 62, 115, 117, 120, 122, 58, 23, 125, 63, 110, 121, 57, 55, 111, 77, 45, 108, 39, 49, 114, 37, 43, 127, 48, 109, 112, 47, 118, 90, 51, 42, 46, 103, 104, 44, 38, 92, 41, 40, 101, 106, 32, 98, 105, 36, 84, 33, 99, 28, 31, 102, 35, 34, 17, 73, 26, 97, 95, 3, 94, 15, 64, 2, 11, 30, 12, 5, 74, 14, 96, 82, 70, 13, 29, 93, 83, 24, 85, 27, 7, 1, 25, 69, 19, 68, 87, 72, 66, 21, 71, 91, 79, 80, 76, 9, 8, 4, 20, 75, 16, 81, 67, 89, 18, 78, 6, 88, 22, 86, 65, 0, 10], [59, 53, 61, 101, 88, 71, 75, 18, 78, 27, 16, 76, 2, 22, 69, 25, 20, 9, 68, 117, 56, 0, 119, 79, 5, 74, 81, 66, 85, 91, 93, 14, 7, 96, 72, 11, 83, 10, 87, 24, 82, 90, 28, 30, 15, 6, 13, 80, 39, 3, 4, 89, 43, 29, 21, 32, 84, 12, 116, 23, 111, 33, 26, 70, 19, 35, 64, 41, 92, 94, 1, 8, 31, 100, 77, 49, 73, 67, 62, 17, 54, 37, 65, 99, 95, 103, 55, 126, 86, 124, 108, 38, 98, 50, 113, 34, 36, 97, 115, 60, 122, 105, 42, 44, 104, 123, 110, 47, 107, 102, 48, 121, 63, 52, 120, 106, 46, 58, 40, 45, 57, 125, 109, 127, 51, 112, 118, 114], [61, 53, 119, 124, 113, 63, 54, 47, 52, 57, 115, 117, 114, 120, 56, 50, 58, 111, 55, 118, 127, 125, 59, 62, 123, 121, 51, 110, 45, 49, 122, 116, 48, 112, 109, 126, 44, 43, 60, 108, 46, 42, 106, 107, 105, 32, 103, 36, 40, 101, 41, 100, 37, 33, 39, 104, 99, 86, 22, 102, 35, 23, 38, 34, 29, 95, 92, 98, 28, 97, 31, 96, 90, 83, 17, 94, 26, 19, 81, 20, 93, 30, 27, 25, 82, 16, 88, 87, 21, 18, 24, 80, 89, 14, 84, 85, 15, 91, 13, 78, 77, 79, 76, 67, 72, 74, 75, 70, 12, 10, 9, 71, 11, 1, 68, 0, 73, 69, 8, 6, 3, 2, 4, 5, 66, 64, 7, 65], [127, 124, 102, 33, 62, 51, 94, 56, 22, 18, 24, 121, 60, 116, 59, 37, 119, 39, 113, 122, 125, 114, 27, 91, 53, 93, 99, 30, 110, 38, 55, 126, 32, 25, 61, 58, 57, 49, 104, 118, 95, 112, 54, 28, 123, 48, 117, 47, 63, 50, 97, 46, 2, 111, 115, 108, 7, 120, 84, 29, 52, 45, 109, 31, 44, 21, 101, 71, 75, 107, 34, 106, 89, 3, 0, 90, 82, 85, 70, 35, 42, 98, 92, 103, 40, 78, 6, 105, 36, 41, 67, 65, 73, 83, 15, 43, 23, 20, 26, 14, 16, 87, 69, 88, 80, 68, 72, 96, 19, 100, 1, 76, 4, 64, 12, 77, 8, 66, 17, 79, 11, 10, 81, 74, 13, 5, 86, 9], [124, 102, 127, 47, 24, 33, 62, 94, 18, 51, 91, 84, 22, 75, 25, 36, 28, 121, 114, 56, 15, 37, 26, 60, 40, 110, 112, 116, 32, 53, 59, 122, 126, 119, 38, 27, 58, 79, 89, 20, 109, 107, 13, 125, 113, 11, 57, 87, 30, 123, 111, 34, 63, 55, 103, 35, 117, 48, 100, 49, 93, 46, 29, 54, 41, 45, 108, 9, 105, 61, 7, 42, 118, 31, 39, 85, 101, 88, 115, 106, 50, 80, 98, 96, 92, 99, 52, 120, 82, 90, 23, 104, 19, 3, 67, 97, 95, 44, 83, 17, 69, 21, 0, 16, 10, 76, 5, 71, 12, 74, 81, 2, 43, 14, 78, 86, 72, 68, 77, 73, 8, 4, 65, 66, 64, 6, 1, 70], [102, 127, 124, 94, 24, 33, 84, 13, 30, 93, 19, 22, 9, 25, 62, 18, 28, 17, 15, 92, 82, 88, 97, 16, 83, 73, 91, 38, 11, 23, 56, 51, 20, 79, 42, 85, 47, 112, 36, 89, 77, 69, 27, 46, 81, 31, 87, 95, 90, 106, 29, 67, 110, 119, 34, 2, 32, 80, 35, 86, 96, 8, 5, 0, 76, 26, 14, 12, 39, 122, 7, 74, 21, 78, 37, 10, 72, 108, 99, 71, 120, 3, 60, 59, 100, 114, 70, 123, 65, 107, 75, 63, 98, 116, 40, 68, 6, 103, 104, 105, 126, 101, 111, 4, 52, 55, 1, 57, 58, 54, 41, 66, 109, 45, 118, 50, 125, 64, 48, 117, 121, 53, 113, 115, 44, 43, 49, 61], [124, 127, 102, 47, 51, 62, 121, 56, 85, 59, 116, 119, 114, 60, 110, 113, 125, 58, 126, 53, 91, 122, 55, 123, 61, 112, 54, 48, 49, 63, 57, 50, 117, 118, 18, 109, 115, 108, 26, 41, 120, 111, 107, 101, 46, 105, 40, 45, 52, 21, 33, 38, 44, 25, 104, 42, 43, 36, 106, 39, 86, 22, 73, 99, 35, 37, 82, 76, 24, 34, 100, 30, 103, 79, 97, 94, 92, 3, 15, 98, 32, 29, 95, 96, 31, 27, 77, 90, 28, 89, 23, 75, 93, 65, 69, 7, 68, 84, 88, 17, 83, 12, 20, 81, 87, 0, 1, 72, 10, 66, 67, 19, 16, 80, 11, 64, 70, 6, 9, 5, 78, 71, 13, 14, 4, 74, 2, 8]], "model.layers.9.self_attn.k_proj": [[46, 37, 110, 96, 22, 63, 124, 99, 59, 113, 28, 49, 56, 122, 55, 114, 115, 54, 58, 57, 51, 61, 60, 119, 123, 53, 120, 48, 62, 25, 47, 85, 50, 121, 125, 112, 118, 127, 19, 117, 126, 116, 78, 43, 52, 8, 42, 108, 111, 109, 105, 107, 27, 40, 45, 65, 44, 30, 41, 106, 77, 26, 104, 17, 16, 103, 23, 39, 94, 102, 36, 7, 38, 4, 5, 3, 97, 95, 31, 29, 90, 81, 33, 35, 32, 82, 67, 74, 100, 93, 2, 21, 64, 18, 98, 101, 34, 76, 84, 24, 20, 14, 1, 87, 9, 73, 11, 10, 92, 91, 66, 13, 88, 12, 79, 15, 0, 80, 69, 75, 72, 6, 68, 70, 86, 71, 83, 89], [105, 117, 95, 56, 37, 58, 43, 106, 112, 34, 92, 23, 33, 108, 63, 114, 55, 29, 110, 103, 38, 111, 99, 22, 89, 26, 9, 78, 124, 46, 107, 119, 84, 50, 122, 87, 18, 82, 127, 2, 126, 0, 60, 51, 83, 4, 20, 14, 121, 98, 54, 65, 88, 12, 57, 120, 59, 69, 67, 16, 90, 115, 52, 40, 44, 109, 118, 62, 31, 36, 10, 42, 48, 70, 79, 24, 47, 76, 80, 41, 8, 30, 25, 100, 113, 96, 32, 27, 53, 101, 6, 125, 45, 17, 123, 64, 11, 91, 102, 15, 21, 86, 49, 116, 85, 13, 7, 104, 94, 61, 3, 97, 81, 72, 71, 1, 75, 5, 74, 66, 93, 35, 77, 73, 68, 19, 28, 39], [41, 0, 34, 47, 18, 74, 13, 81, 86, 117, 76, 8, 70, 53, 65, 55, 66, 79, 105, 71, 116, 84, 3, 4, 50, 63, 51, 112, 126, 59, 119, 111, 123, 43, 61, 127, 69, 44, 88, 106, 114, 46, 125, 93, 118, 67, 49, 87, 48, 98, 92, 85, 45, 115, 26, 90, 68, 122, 104, 52, 2, 27, 54, 42, 6, 124, 7, 73, 121, 120, 60, 30, 58, 29, 1, 31, 38, 83, 57, 110, 11, 95, 24, 37, 62, 10, 25, 77, 64, 75, 32, 12, 103, 5, 14, 19, 107, 9, 36, 72, 35, 80, 102, 39, 15, 91, 16, 28, 33, 109, 40, 17, 101, 97, 23, 89, 100, 56, 78, 20, 94, 113, 108, 96, 99, 82, 21, 22], [40, 125, 62, 22, 33, 94, 48, 63, 100, 122, 90, 18, 119, 117, 92, 57, 127, 16, 114, 60, 29, 52, 49, 47, 86, 19, 54, 78, 59, 87, 45, 58, 115, 109, 110, 120, 10, 84, 56, 123, 46, 106, 113, 108, 112, 89, 70, 61, 43, 99, 24, 116, 80, 126, 53, 124, 50, 51, 96, 107, 111, 118, 28, 77, 4, 121, 39, 2, 21, 38, 79, 55, 8, 44, 105, 42, 35, 41, 0, 74, 37, 23, 101, 88, 17, 102, 32, 91, 103, 31, 98, 36, 97, 81, 82, 34, 27, 75, 93, 95, 1, 30, 7, 11, 26, 3, 15, 73, 20, 13, 83, 76, 69, 25, 12, 65, 14, 72, 71, 67, 6, 104, 85, 64, 9, 5, 68, 66], [124, 39, 55, 56, 97, 93, 113, 88, 21, 83, 57, 91, 15, 122, 81, 114, 123, 125, 78, 105, 44, 117, 6, 11, 52, 23, 18, 10, 66, 4, 45, 64, 9, 43, 109, 69, 126, 76, 106, 22, 49, 12, 110, 47, 116, 53, 112, 120, 51, 80, 107, 61, 46, 58, 118, 62, 42, 32, 54, 1, 14, 41, 82, 119, 3, 50, 121, 127, 115, 27, 60, 111, 34, 89, 108, 33, 48, 101, 100, 38, 20, 63, 72, 102, 36, 84, 99, 59, 90, 25, 98, 35, 13, 37, 26, 96, 92, 19, 104, 94, 30, 95, 86, 40, 16, 73, 28, 87, 77, 7, 29, 85, 31, 5, 8, 71, 67, 75, 24, 103, 65, 68, 79, 0, 17, 2, 70, 74], [126, 57, 103, 31, 98, 61, 87, 90, 22, 60, 16, 92, 21, 120, 19, 17, 111, 48, 77, 79, 75, 125, 51, 7, 63, 58, 113, 74, 118, 84, 122, 53, 3, 127, 73, 115, 117, 6, 45, 55, 123, 65, 64, 100, 56, 119, 82, 62, 52, 116, 66, 49, 112, 50, 114, 99, 110, 46, 69, 44, 121, 88, 54, 38, 47, 76, 105, 124, 40, 109, 107, 68, 4, 28, 59, 15, 13, 36, 32, 91, 104, 41, 97, 43, 106, 101, 35, 20, 29, 30, 1, 102, 18, 42, 86, 10, 27, 108, 33, 5, 93, 80, 14, 25, 24, 37, 96, 94, 85, 23, 70, 34, 67, 89, 12, 81, 78, 83, 8, 2, 71, 11, 72, 9, 26, 39, 95, 0], [59, 61, 53, 37, 22, 16, 27, 18, 78, 32, 20, 119, 75, 88, 29, 25, 56, 76, 71, 85, 69, 79, 60, 116, 10, 8, 34, 2, 9, 81, 126, 98, 50, 105, 19, 113, 117, 15, 1, 108, 36, 96, 123, 122, 107, 103, 124, 87, 26, 115, 74, 39, 62, 120, 3, 0, 55, 52, 43, 13, 90, 111, 63, 70, 28, 54, 35, 6, 21, 38, 125, 109, 46, 106, 112, 30, 121, 49, 4, 47, 57, 65, 41, 17, 94, 58, 23, 104, 33, 110, 127, 77, 45, 68, 118, 48, 51, 114, 83, 97, 44, 99, 42, 93, 102, 5, 86, 72, 92, 95, 40, 89, 31, 80, 100, 73, 24, 67, 12, 64, 14, 82, 84, 7, 66, 91, 11, 101], [127, 124, 38, 22, 97, 51, 62, 30, 121, 24, 15, 114, 59, 111, 110, 18, 56, 60, 116, 112, 9, 119, 63, 91, 13, 122, 125, 93, 64, 48, 113, 17, 84, 11, 53, 5, 57, 126, 47, 106, 55, 54, 58, 102, 118, 94, 49, 117, 61, 123, 50, 66, 44, 16, 100, 19, 45, 115, 43, 101, 109, 120, 76, 25, 99, 88, 104, 108, 46, 28, 29, 10, 52, 23, 42, 103, 40, 105, 98, 41, 107, 37, 68, 26, 83, 1, 35, 14, 89, 95, 6, 87, 8, 79, 36, 71, 21, 81, 7, 96, 65, 34, 78, 74, 31, 33, 12, 39, 80, 85, 90, 86, 72, 32, 20, 3, 67, 75, 77, 92, 4, 27, 82, 69, 73, 2, 70, 0]], "model.layers.9.self_attn.qk_proj": [[124, 61, 127, 53, 125, 126, 62, 41, 57, 59, 110, 105, 117, 46, 56, 55, 51, 119, 114, 86, 63, 22, 113, 48, 98, 102, 82, 111, 112, 122, 116, 60, 52, 18, 123, 49, 58, 118, 26, 84, 50, 43, 39, 121, 28, 20, 17, 88, 47, 42, 81, 24, 101, 15, 38, 23, 29, 31, 79, 76, 103, 37, 13, 87, 97, 77, 34, 93, 33, 115, 45, 120, 74, 21, 10, 40, 0, 94, 64, 12, 54, 27, 66, 108, 107, 109, 89, 106, 16, 2, 7, 71, 44, 67, 91, 30, 19, 70, 90, 83, 78, 3, 95, 80, 92, 25, 32, 104, 85, 96, 72, 35, 8, 68, 6, 14, 99, 69, 75, 11, 9, 1, 36, 4, 100, 73, 5, 65], [124, 61, 127, 53, 126, 62, 125, 41, 59, 57, 110, 105, 117, 46, 56, 55, 63, 51, 86, 119, 114, 22, 48, 113, 111, 60, 112, 122, 98, 116, 49, 43, 82, 88, 118, 28, 24, 18, 102, 84, 123, 50, 37, 103, 26, 47, 52, 58, 39, 42, 15, 101, 87, 23, 77, 17, 20, 76, 115, 34, 45, 81, 31, 38, 79, 10, 0, 107, 13, 121, 33, 40, 97, 29, 120, 27, 54, 21, 64, 74, 94, 93, 30, 44, 12, 109, 83, 70, 108, 7, 89, 16, 90, 91, 25, 68, 106, 92, 96, 104, 100, 71, 80, 78, 85, 35, 95, 36, 66, 19, 8, 32, 4, 14, 11, 65, 99, 2, 5, 9, 75, 72, 69, 1, 3, 6, 73, 67], [124, 61, 127, 53, 126, 62, 41, 125, 59, 57, 105, 117, 110, 46, 56, 55, 63, 51, 119, 114, 86, 22, 113, 111, 48, 102, 98, 116, 60, 103, 112, 37, 122, 123, 47, 88, 39, 43, 115, 82, 18, 84, 28, 121, 118, 52, 38, 26, 34, 24, 49, 50, 20, 45, 120, 101, 87, 27, 23, 40, 17, 97, 31, 76, 42, 93, 109, 15, 81, 58, 94, 108, 54, 33, 79, 29, 106, 64, 70, 91, 77, 32, 74, 12, 44, 95, 0, 89, 10, 90, 30, 66, 107, 21, 100, 13, 83, 16, 4, 96, 7, 78, 99, 71, 8, 80, 85, 25, 92, 36, 104, 2, 35, 19, 11, 3, 68, 6, 9, 72, 67, 1, 14, 5, 75, 69, 65, 73], [124, 127, 53, 61, 125, 126, 62, 41, 57, 59, 110, 105, 46, 117, 56, 55, 63, 51, 114, 48, 86, 119, 22, 122, 111, 112, 118, 98, 60, 113, 116, 50, 49, 121, 102, 45, 18, 82, 47, 52, 43, 28, 88, 123, 38, 37, 58, 84, 115, 39, 103, 26, 17, 34, 87, 101, 42, 24, 31, 20, 54, 29, 120, 81, 93, 107, 33, 97, 23, 40, 27, 15, 94, 79, 108, 76, 0, 70, 44, 77, 92, 10, 106, 89, 13, 109, 96, 64, 12, 71, 74, 25, 66, 30, 80, 91, 35, 32, 95, 85, 21, 7, 8, 90, 100, 14, 104, 2, 83, 16, 19, 67, 69, 99, 36, 3, 78, 68, 11, 65, 72, 73, 6, 1, 4, 75, 9, 5], [124, 127, 53, 61, 125, 126, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 119, 114, 116, 86, 22, 48, 98, 113, 82, 60, 18, 112, 39, 45, 102, 111, 123, 50, 122, 49, 84, 42, 24, 47, 118, 26, 38, 28, 121, 101, 52, 37, 88, 120, 34, 20, 103, 17, 115, 43, 58, 33, 87, 81, 64, 31, 40, 77, 29, 76, 23, 15, 97, 54, 93, 107, 2, 79, 44, 10, 108, 0, 12, 89, 7, 66, 13, 74, 27, 92, 109, 21, 104, 91, 96, 90, 80, 70, 94, 71, 95, 30, 106, 83, 35, 8, 78, 4, 68, 16, 85, 19, 6, 25, 65, 14, 9, 5, 75, 99, 72, 32, 11, 67, 1, 69, 100, 3, 73, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 86, 114, 48, 22, 111, 113, 82, 98, 116, 58, 112, 49, 18, 122, 60, 26, 52, 102, 123, 88, 118, 28, 84, 103, 50, 47, 24, 39, 77, 45, 101, 13, 42, 43, 120, 15, 17, 20, 38, 87, 81, 79, 33, 31, 121, 34, 23, 76, 37, 54, 10, 29, 40, 97, 74, 12, 44, 7, 115, 107, 0, 83, 104, 21, 27, 2, 94, 89, 80, 106, 108, 96, 8, 71, 109, 30, 92, 64, 85, 19, 14, 16, 91, 93, 25, 95, 6, 4, 11, 78, 90, 66, 3, 75, 35, 70, 67, 9, 68, 32, 72, 69, 99, 100, 5, 65, 36, 73, 1], [124, 127, 61, 53, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 51, 86, 22, 63, 119, 114, 48, 112, 116, 82, 111, 18, 52, 26, 49, 122, 118, 60, 84, 98, 20, 102, 120, 123, 28, 58, 88, 50, 39, 24, 101, 79, 81, 29, 15, 76, 17, 103, 121, 43, 113, 38, 13, 47, 31, 42, 37, 12, 34, 45, 106, 77, 10, 87, 21, 33, 27, 115, 8, 107, 23, 44, 40, 54, 94, 19, 97, 80, 93, 83, 74, 89, 85, 91, 25, 14, 109, 108, 92, 96, 7, 3, 30, 90, 0, 67, 71, 95, 6, 72, 16, 99, 78, 64, 11, 66, 36, 104, 32, 35, 2, 69, 9, 100, 5, 68, 75, 70, 1, 4, 65, 73], [124, 127, 61, 53, 126, 125, 41, 62, 59, 57, 110, 105, 117, 56, 46, 55, 51, 63, 22, 86, 119, 114, 48, 98, 112, 18, 82, 88, 122, 116, 84, 28, 26, 24, 111, 20, 60, 113, 118, 79, 15, 103, 102, 58, 50, 123, 38, 52, 43, 49, 34, 101, 17, 37, 87, 31, 42, 76, 29, 45, 13, 83, 77, 81, 39, 10, 120, 12, 21, 121, 47, 23, 97, 40, 94, 27, 74, 33, 0, 104, 64, 19, 54, 107, 115, 108, 7, 30, 80, 93, 25, 106, 6, 96, 89, 85, 16, 14, 66, 90, 92, 44, 8, 91, 109, 78, 100, 11, 2, 99, 35, 95, 68, 75, 71, 65, 72, 32, 36, 70, 4, 1, 69, 9, 5, 3, 73, 67], [124, 127, 53, 61, 126, 125, 41, 62, 59, 57, 110, 117, 105, 56, 46, 51, 55, 63, 114, 119, 22, 86, 122, 48, 98, 111, 88, 18, 60, 112, 49, 26, 84, 118, 82, 113, 116, 50, 20, 24, 39, 28, 47, 37, 38, 58, 103, 43, 102, 123, 52, 29, 94, 15, 121, 17, 79, 87, 101, 34, 27, 77, 31, 33, 120, 81, 13, 107, 97, 45, 12, 76, 23, 42, 0, 115, 40, 85, 30, 21, 74, 54, 10, 6, 19, 96, 83, 93, 25, 89, 78, 64, 92, 108, 44, 90, 16, 104, 80, 91, 68, 7, 106, 67, 109, 2, 32, 66, 95, 3, 100, 71, 4, 36, 1, 72, 70, 99, 65, 14, 35, 11, 69, 75, 8, 5, 9, 73], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 56, 46, 55, 63, 51, 114, 22, 86, 119, 48, 112, 116, 122, 111, 18, 98, 118, 113, 88, 60, 102, 50, 58, 82, 84, 28, 52, 79, 24, 26, 49, 38, 103, 81, 47, 15, 87, 20, 39, 34, 17, 123, 43, 29, 37, 23, 120, 107, 121, 97, 74, 76, 101, 45, 42, 31, 33, 13, 77, 27, 12, 54, 115, 21, 93, 44, 94, 0, 10, 40, 78, 64, 6, 90, 92, 83, 104, 66, 96, 7, 19, 80, 85, 25, 109, 108, 30, 91, 106, 16, 68, 71, 89, 70, 2, 72, 4, 95, 67, 32, 14, 99, 100, 5, 35, 69, 8, 3, 65, 36, 11, 73, 75, 1, 9], [124, 53, 127, 61, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 51, 55, 63, 119, 114, 22, 86, 48, 113, 111, 116, 122, 112, 98, 102, 18, 60, 82, 118, 49, 52, 50, 39, 47, 58, 20, 123, 88, 42, 31, 26, 84, 87, 15, 24, 37, 28, 103, 79, 121, 64, 81, 17, 107, 43, 34, 120, 101, 54, 77, 38, 93, 29, 12, 76, 97, 13, 66, 33, 45, 23, 27, 7, 115, 10, 40, 0, 74, 71, 2, 78, 16, 44, 21, 94, 90, 25, 6, 92, 70, 30, 72, 95, 104, 109, 85, 108, 19, 106, 35, 96, 83, 5, 68, 89, 91, 32, 75, 69, 80, 65, 4, 9, 14, 11, 3, 67, 8, 100, 99, 73, 1, 36], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 51, 55, 63, 119, 114, 86, 22, 48, 122, 111, 82, 60, 18, 116, 52, 113, 98, 39, 112, 102, 49, 118, 26, 58, 50, 88, 47, 123, 20, 15, 84, 28, 24, 120, 101, 17, 13, 77, 81, 42, 121, 29, 37, 33, 12, 43, 79, 40, 87, 31, 103, 107, 54, 38, 16, 27, 76, 0, 64, 21, 44, 34, 74, 23, 19, 10, 45, 97, 78, 93, 115, 71, 94, 66, 30, 72, 89, 108, 7, 104, 92, 2, 5, 70, 96, 90, 3, 83, 25, 106, 91, 67, 85, 109, 68, 1, 95, 35, 32, 80, 69, 14, 8, 11, 4, 100, 6, 75, 9, 73, 99, 65, 36], [124, 127, 61, 53, 126, 125, 62, 41, 57, 59, 110, 105, 117, 56, 46, 55, 51, 63, 119, 22, 86, 114, 60, 112, 116, 18, 48, 113, 122, 88, 111, 49, 118, 98, 82, 24, 102, 50, 28, 52, 123, 20, 43, 39, 26, 84, 101, 58, 103, 15, 47, 38, 37, 29, 81, 13, 87, 120, 17, 33, 121, 97, 45, 79, 34, 107, 27, 31, 12, 42, 115, 21, 77, 74, 104, 93, 23, 40, 16, 76, 94, 10, 54, 91, 19, 44, 83, 89, 30, 25, 96, 90, 85, 108, 64, 70, 78, 72, 92, 100, 106, 32, 95, 71, 80, 109, 0, 75, 14, 36, 35, 7, 68, 66, 2, 99, 69, 4, 3, 67, 11, 5, 8, 6, 65, 9, 1, 73], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 117, 105, 56, 46, 55, 63, 51, 86, 119, 22, 114, 48, 112, 111, 98, 113, 60, 18, 58, 116, 122, 82, 88, 49, 28, 15, 118, 26, 47, 123, 13, 120, 24, 43, 102, 50, 84, 52, 103, 79, 20, 34, 64, 101, 87, 81, 39, 21, 42, 45, 29, 23, 94, 17, 38, 40, 121, 74, 77, 10, 83, 104, 70, 12, 0, 33, 31, 37, 93, 107, 54, 27, 97, 89, 19, 7, 71, 76, 16, 25, 14, 90, 108, 44, 2, 92, 80, 91, 72, 115, 78, 85, 95, 30, 106, 66, 75, 4, 73, 11, 32, 96, 5, 68, 35, 6, 69, 9, 8, 65, 36, 109, 1, 100, 99, 3, 67], [124, 127, 53, 61, 126, 125, 62, 41, 59, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 114, 48, 22, 60, 111, 112, 118, 113, 52, 123, 49, 82, 98, 47, 18, 101, 116, 102, 120, 122, 88, 39, 58, 26, 28, 24, 50, 103, 13, 20, 38, 84, 37, 34, 45, 43, 33, 79, 15, 87, 23, 42, 17, 121, 97, 31, 81, 40, 107, 27, 77, 29, 108, 7, 12, 21, 0, 54, 16, 19, 44, 74, 10, 64, 94, 70, 93, 104, 92, 76, 89, 106, 25, 71, 91, 72, 83, 90, 66, 115, 2, 80, 95, 14, 85, 78, 96, 32, 30, 3, 67, 5, 4, 9, 100, 99, 35, 109, 1, 68, 6, 11, 36, 75, 69, 73, 65, 8], [124, 127, 53, 61, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 63, 51, 86, 119, 22, 114, 113, 48, 112, 60, 111, 18, 102, 98, 52, 122, 118, 49, 39, 116, 43, 82, 123, 84, 101, 28, 50, 47, 58, 120, 88, 103, 26, 20, 42, 24, 29, 33, 13, 17, 15, 81, 97, 38, 45, 34, 27, 37, 79, 12, 121, 31, 40, 23, 107, 87, 77, 94, 16, 108, 104, 10, 54, 64, 30, 21, 0, 91, 25, 7, 76, 115, 74, 90, 83, 89, 44, 35, 93, 92, 71, 70, 69, 96, 85, 19, 32, 66, 106, 67, 2, 78, 72, 95, 80, 100, 109, 99, 6, 68, 3, 75, 14, 4, 8, 5, 1, 36, 9, 11, 65, 73], [124, 127, 126, 53, 61, 125, 41, 57, 62, 59, 110, 105, 56, 46, 117, 55, 51, 63, 22, 86, 114, 48, 119, 112, 52, 18, 111, 116, 102, 123, 60, 113, 98, 28, 84, 49, 122, 82, 88, 118, 50, 101, 103, 20, 26, 39, 79, 24, 15, 47, 58, 17, 120, 37, 13, 38, 81, 107, 104, 31, 45, 33, 27, 121, 29, 43, 34, 10, 40, 21, 12, 74, 23, 94, 42, 19, 91, 87, 77, 16, 97, 32, 90, 115, 54, 25, 108, 76, 106, 93, 78, 35, 71, 83, 44, 64, 89, 85, 80, 96, 99, 0, 30, 92, 14, 6, 95, 8, 7, 68, 72, 109, 69, 4, 66, 70, 100, 75, 9, 5, 2, 65, 36, 1, 73, 67, 11, 3], [124, 127, 53, 126, 61, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 51, 22, 86, 63, 119, 114, 111, 48, 112, 122, 123, 82, 98, 52, 28, 18, 116, 49, 101, 102, 24, 84, 88, 50, 47, 26, 58, 38, 60, 20, 113, 118, 17, 103, 37, 81, 42, 15, 39, 79, 34, 31, 29, 43, 33, 21, 23, 13, 87, 97, 120, 40, 107, 45, 12, 121, 27, 74, 10, 89, 93, 77, 19, 90, 94, 83, 16, 104, 0, 25, 7, 91, 115, 54, 6, 78, 80, 92, 108, 76, 30, 44, 96, 106, 64, 85, 99, 2, 95, 71, 66, 14, 35, 75, 32, 8, 100, 67, 9, 3, 109, 36, 68, 70, 4, 73, 1, 72, 11, 69, 5, 65], [124, 127, 53, 61, 126, 41, 125, 59, 62, 57, 105, 110, 117, 46, 56, 55, 51, 86, 114, 63, 119, 22, 48, 118, 52, 111, 98, 122, 116, 60, 39, 112, 37, 123, 18, 47, 102, 49, 113, 103, 101, 82, 43, 58, 88, 84, 38, 115, 24, 34, 26, 81, 28, 31, 42, 50, 120, 107, 17, 121, 33, 20, 79, 13, 87, 40, 91, 15, 45, 54, 23, 106, 29, 108, 27, 97, 94, 0, 99, 64, 12, 93, 32, 89, 44, 10, 6, 21, 76, 19, 83, 95, 92, 85, 90, 74, 16, 100, 2, 35, 109, 96, 77, 30, 80, 25, 14, 66, 7, 104, 68, 36, 8, 4, 71, 67, 70, 1, 3, 75, 78, 9, 69, 5, 65, 73, 72, 11], [124, 127, 53, 61, 126, 125, 57, 41, 62, 59, 110, 105, 117, 46, 56, 55, 51, 86, 63, 119, 22, 114, 48, 112, 111, 113, 116, 60, 123, 49, 118, 98, 102, 103, 18, 39, 122, 88, 52, 47, 101, 58, 28, 26, 82, 50, 33, 84, 43, 24, 37, 121, 31, 38, 79, 15, 34, 87, 20, 97, 13, 42, 29, 17, 81, 21, 120, 107, 27, 45, 23, 115, 54, 74, 12, 94, 25, 10, 6, 93, 104, 40, 89, 19, 90, 92, 30, 77, 32, 91, 64, 76, 0, 108, 35, 83, 80, 7, 44, 95, 85, 96, 78, 106, 16, 8, 4, 109, 71, 68, 14, 2, 75, 66, 9, 70, 99, 100, 36, 5, 11, 72, 69, 1, 3, 65, 67, 73], [124, 61, 53, 127, 126, 125, 41, 57, 62, 59, 110, 105, 117, 56, 46, 55, 51, 63, 86, 119, 22, 114, 48, 112, 111, 116, 98, 18, 60, 102, 84, 52, 123, 122, 58, 118, 50, 113, 88, 82, 49, 101, 17, 42, 28, 26, 43, 13, 47, 121, 39, 20, 15, 33, 27, 81, 38, 79, 24, 29, 103, 120, 10, 87, 107, 31, 45, 77, 40, 23, 37, 115, 12, 54, 94, 34, 74, 95, 6, 30, 90, 25, 7, 93, 16, 64, 21, 104, 71, 66, 76, 78, 0, 2, 83, 89, 97, 91, 44, 5, 80, 85, 19, 92, 108, 106, 8, 3, 96, 75, 4, 9, 14, 69, 67, 70, 100, 109, 35, 32, 72, 36, 99, 68, 11, 73, 65, 1], [124, 127, 53, 61, 125, 126, 59, 41, 62, 57, 110, 117, 105, 56, 46, 55, 51, 63, 119, 86, 22, 114, 48, 112, 98, 111, 118, 58, 52, 18, 116, 102, 49, 50, 37, 82, 88, 101, 42, 122, 20, 60, 113, 115, 123, 38, 39, 84, 24, 28, 81, 47, 103, 29, 26, 43, 15, 17, 34, 54, 33, 120, 45, 87, 12, 121, 13, 31, 79, 64, 23, 40, 27, 93, 107, 106, 94, 97, 21, 44, 92, 10, 108, 77, 89, 16, 74, 30, 90, 76, 91, 83, 95, 25, 71, 109, 7, 19, 0, 96, 35, 78, 85, 8, 100, 66, 104, 80, 99, 70, 4, 2, 32, 14, 75, 6, 5, 36, 3, 11, 67, 69, 1, 72, 68, 9, 73, 65], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 117, 56, 46, 63, 55, 51, 119, 86, 22, 114, 48, 98, 112, 111, 18, 37, 118, 38, 60, 24, 116, 84, 102, 82, 52, 58, 113, 20, 39, 123, 101, 88, 50, 115, 120, 28, 103, 45, 15, 81, 49, 26, 43, 42, 23, 31, 47, 122, 87, 29, 27, 17, 34, 33, 94, 40, 97, 12, 13, 10, 0, 54, 91, 107, 79, 77, 121, 44, 109, 93, 21, 104, 108, 74, 64, 83, 90, 76, 106, 95, 70, 19, 80, 71, 25, 96, 66, 30, 7, 92, 16, 78, 2, 85, 89, 32, 14, 8, 100, 68, 35, 99, 4, 75, 72, 73, 5, 69, 36, 3, 1, 11, 65, 9, 6, 67], [124, 127, 53, 61, 126, 125, 62, 41, 57, 59, 110, 105, 56, 117, 46, 63, 55, 51, 86, 119, 114, 22, 48, 58, 111, 98, 18, 116, 112, 60, 113, 118, 52, 82, 49, 88, 45, 50, 103, 123, 39, 84, 38, 102, 122, 37, 43, 42, 47, 28, 26, 20, 101, 24, 81, 27, 54, 79, 15, 87, 13, 120, 23, 31, 10, 40, 77, 33, 17, 107, 94, 29, 121, 34, 115, 97, 83, 85, 30, 76, 64, 12, 70, 71, 104, 91, 21, 44, 93, 108, 95, 90, 25, 74, 89, 7, 19, 92, 16, 80, 3, 66, 0, 106, 14, 35, 78, 2, 96, 68, 69, 8, 4, 32, 109, 9, 75, 100, 11, 67, 5, 99, 73, 36, 72, 1, 6, 65], [124, 127, 61, 53, 126, 125, 62, 57, 41, 59, 110, 117, 105, 46, 56, 55, 51, 63, 86, 119, 22, 114, 48, 60, 111, 113, 118, 98, 52, 82, 122, 47, 123, 49, 116, 18, 102, 112, 39, 58, 88, 101, 84, 26, 37, 50, 20, 28, 43, 17, 24, 45, 38, 121, 81, 31, 103, 27, 42, 23, 34, 79, 29, 97, 15, 33, 54, 87, 76, 115, 120, 12, 108, 13, 21, 40, 94, 77, 70, 91, 10, 107, 106, 74, 93, 0, 44, 30, 83, 25, 89, 2, 7, 96, 16, 19, 32, 85, 71, 92, 64, 90, 95, 80, 14, 109, 100, 73, 78, 8, 35, 99, 66, 72, 104, 4, 3, 75, 69, 36, 5, 67, 6, 11, 65, 1, 68, 9], [124, 61, 127, 53, 126, 125, 59, 57, 62, 41, 110, 105, 117, 56, 46, 55, 63, 86, 51, 119, 22, 114, 48, 60, 113, 112, 58, 49, 18, 111, 98, 122, 88, 116, 118, 52, 82, 50, 102, 101, 84, 26, 123, 39, 24, 121, 81, 38, 15, 37, 20, 34, 45, 103, 28, 17, 79, 115, 76, 42, 120, 29, 33, 47, 13, 31, 43, 107, 27, 87, 40, 77, 74, 21, 23, 10, 25, 12, 97, 93, 94, 54, 90, 30, 83, 89, 64, 91, 80, 44, 7, 70, 85, 96, 92, 19, 78, 104, 108, 100, 0, 99, 71, 106, 35, 109, 16, 95, 36, 32, 72, 8, 73, 14, 75, 6, 68, 66, 69, 11, 2, 4, 5, 9, 1, 67, 3, 65], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 56, 46, 55, 63, 51, 119, 86, 22, 114, 122, 48, 58, 60, 112, 98, 88, 82, 116, 52, 111, 49, 120, 26, 43, 34, 18, 50, 102, 113, 121, 24, 101, 45, 28, 27, 123, 37, 40, 84, 42, 118, 38, 103, 31, 20, 29, 15, 47, 115, 39, 17, 94, 81, 23, 87, 21, 33, 79, 97, 91, 77, 107, 12, 109, 13, 93, 96, 76, 74, 0, 54, 104, 95, 108, 44, 80, 89, 25, 10, 90, 106, 64, 85, 30, 19, 3, 99, 78, 100, 70, 83, 7, 92, 68, 71, 2, 36, 72, 66, 67, 16, 35, 5, 6, 32, 4, 8, 73, 14, 65, 69, 1, 11, 9, 75], [124, 127, 53, 61, 126, 125, 59, 57, 41, 62, 110, 117, 105, 56, 46, 55, 63, 51, 119, 86, 22, 114, 111, 48, 98, 58, 82, 112, 113, 116, 49, 60, 18, 122, 102, 26, 84, 52, 118, 28, 79, 88, 120, 50, 121, 103, 123, 47, 20, 43, 101, 34, 45, 33, 29, 24, 39, 76, 81, 0, 42, 15, 17, 38, 37, 77, 87, 31, 94, 64, 54, 27, 10, 40, 23, 74, 12, 2, 97, 93, 7, 13, 107, 83, 21, 71, 115, 66, 6, 19, 80, 85, 44, 4, 109, 72, 106, 95, 89, 104, 96, 91, 30, 1, 92, 78, 16, 99, 25, 5, 108, 90, 14, 35, 32, 67, 9, 75, 68, 11, 100, 36, 73, 65, 70, 69, 3, 8], [124, 127, 61, 53, 125, 126, 41, 57, 62, 59, 105, 110, 117, 46, 56, 63, 55, 51, 86, 119, 22, 114, 113, 48, 111, 98, 112, 52, 122, 102, 116, 18, 47, 37, 49, 82, 123, 60, 118, 58, 50, 20, 26, 39, 101, 103, 24, 88, 34, 81, 42, 87, 38, 45, 33, 31, 120, 29, 28, 43, 15, 0, 76, 40, 121, 84, 97, 23, 17, 27, 21, 77, 79, 93, 107, 94, 7, 44, 74, 64, 2, 10, 92, 13, 106, 6, 115, 19, 89, 91, 12, 96, 90, 54, 95, 109, 80, 30, 108, 100, 16, 71, 104, 72, 66, 25, 32, 5, 78, 14, 69, 35, 85, 36, 83, 11, 9, 99, 4, 67, 75, 1, 3, 65, 68, 8, 73, 70], [124, 127, 61, 53, 126, 125, 41, 62, 57, 59, 110, 105, 56, 117, 46, 55, 51, 63, 86, 119, 114, 113, 22, 111, 48, 52, 58, 112, 60, 116, 98, 18, 102, 47, 49, 82, 50, 103, 26, 84, 88, 123, 37, 24, 28, 15, 122, 118, 38, 20, 39, 87, 101, 31, 81, 43, 42, 34, 121, 77, 97, 33, 17, 45, 40, 13, 29, 76, 10, 27, 79, 0, 23, 54, 74, 115, 108, 94, 12, 107, 6, 21, 89, 64, 99, 83, 2, 104, 72, 80, 90, 30, 120, 93, 44, 25, 71, 16, 109, 67, 96, 7, 91, 66, 19, 3, 106, 14, 92, 85, 95, 4, 78, 35, 75, 32, 36, 11, 5, 65, 100, 1, 73, 69, 8, 68, 70, 9], [124, 127, 61, 53, 126, 125, 41, 59, 62, 57, 110, 105, 117, 46, 56, 55, 63, 51, 119, 114, 22, 86, 60, 113, 48, 98, 111, 112, 52, 116, 18, 58, 122, 102, 82, 84, 118, 24, 47, 50, 88, 28, 103, 123, 43, 15, 26, 121, 49, 42, 39, 101, 20, 38, 45, 31, 17, 87, 97, 120, 81, 76, 23, 34, 29, 77, 37, 13, 40, 54, 33, 0, 27, 21, 79, 7, 44, 115, 12, 94, 74, 10, 89, 107, 106, 96, 93, 6, 83, 108, 72, 25, 2, 30, 64, 95, 71, 80, 68, 92, 104, 16, 35, 91, 4, 14, 85, 90, 19, 66, 75, 36, 100, 11, 78, 99, 32, 109, 9, 73, 70, 5, 69, 65, 1, 8, 67, 3], [124, 127, 61, 126, 53, 125, 62, 57, 59, 41, 110, 105, 56, 117, 46, 55, 51, 63, 119, 114, 86, 22, 48, 113, 60, 98, 18, 111, 116, 112, 24, 49, 82, 103, 123, 84, 88, 118, 122, 52, 58, 28, 26, 102, 50, 15, 20, 17, 76, 81, 121, 79, 13, 45, 39, 38, 43, 34, 87, 101, 23, 77, 31, 94, 21, 115, 47, 97, 29, 74, 37, 12, 10, 33, 42, 120, 40, 27, 54, 107, 44, 7, 83, 30, 0, 93, 89, 64, 71, 25, 91, 78, 11, 92, 6, 16, 90, 104, 96, 108, 72, 80, 66, 19, 85, 14, 5, 95, 70, 2, 75, 67, 4, 32, 3, 69, 36, 8, 106, 109, 99, 35, 100, 65, 68, 73, 9, 1]], "model.layers.10.self_attn.q_proj": [[121, 122, 26, 65, 36, 83, 50, 79, 21, 24, 81, 63, 89, 29, 94, 9, 0, 64, 68, 16, 6, 19, 99, 1, 91, 78, 8, 17, 23, 2, 58, 77, 75, 111, 69, 124, 86, 12, 114, 71, 95, 82, 85, 44, 52, 14, 120, 76, 30, 37, 67, 51, 96, 5, 118, 57, 31, 54, 70, 11, 123, 74, 66, 93, 20, 84, 47, 27, 60, 127, 61, 40, 4, 13, 10, 3, 18, 125, 73, 80, 102, 87, 53, 49, 117, 115, 119, 62, 113, 116, 100, 72, 112, 55, 45, 7, 15, 126, 59, 90, 48, 105, 104, 110, 92, 41, 88, 33, 56, 46, 109, 35, 42, 25, 103, 22, 39, 107, 97, 38, 106, 108, 43, 34, 32, 28, 101, 98], [121, 122, 63, 87, 50, 36, 75, 52, 118, 28, 12, 124, 17, 120, 94, 123, 60, 61, 54, 125, 51, 33, 73, 29, 57, 62, 127, 119, 40, 117, 34, 44, 115, 53, 113, 102, 15, 49, 47, 58, 111, 19, 112, 55, 23, 70, 126, 110, 114, 59, 14, 109, 46, 105, 24, 56, 97, 116, 45, 104, 99, 38, 31, 32, 21, 71, 16, 86, 13, 39, 107, 48, 76, 106, 80, 37, 35, 85, 103, 25, 101, 20, 88, 91, 100, 92, 96, 98, 108, 93, 41, 5, 18, 84, 66, 95, 67, 26, 22, 89, 90, 42, 64, 27, 83, 30, 43, 79, 11, 3, 74, 82, 4, 1, 8, 77, 72, 81, 7, 65, 10, 9, 0, 6, 2, 69, 78, 68], [50, 121, 122, 114, 63, 56, 62, 115, 120, 60, 58, 51, 49, 116, 84, 87, 59, 119, 53, 11, 113, 52, 15, 127, 44, 17, 117, 109, 45, 55, 124, 61, 126, 54, 47, 57, 123, 48, 107, 14, 111, 46, 72, 118, 108, 125, 110, 43, 112, 7, 24, 2, 27, 78, 106, 25, 99, 101, 75, 41, 4, 85, 42, 16, 77, 80, 19, 82, 10, 105, 1, 81, 3, 74, 104, 18, 13, 12, 9, 28, 83, 94, 40, 37, 97, 6, 71, 90, 73, 102, 103, 21, 39, 22, 29, 38, 23, 76, 30, 89, 93, 33, 5, 36, 0, 35, 32, 96, 34, 98, 70, 91, 95, 69, 31, 26, 92, 100, 86, 8, 65, 88, 79, 20, 66, 68, 67, 64], [121, 122, 50, 63, 14, 118, 52, 84, 124, 11, 123, 87, 120, 61, 97, 125, 15, 114, 54, 57, 62, 101, 60, 82, 47, 16, 17, 53, 119, 85, 28, 111, 115, 51, 112, 127, 117, 102, 58, 110, 40, 104, 55, 44, 109, 45, 75, 32, 59, 105, 46, 77, 113, 107, 49, 27, 106, 39, 37, 103, 126, 25, 33, 38, 81, 48, 116, 12, 34, 19, 41, 74, 80, 90, 108, 71, 72, 83, 24, 9, 13, 99, 91, 4, 2, 7, 56, 89, 21, 42, 78, 35, 6, 94, 36, 10, 96, 3, 76, 0, 73, 43, 92, 79, 93, 5, 30, 100, 23, 31, 98, 1, 22, 95, 29, 86, 26, 70, 88, 20, 66, 69, 18, 8, 67, 68, 65, 64], [103, 124, 113, 24, 82, 49, 21, 91, 31, 15, 32, 11, 62, 13, 83, 19, 16, 28, 50, 80, 72, 99, 27, 81, 63, 73, 93, 4, 76, 17, 88, 7, 75, 9, 14, 18, 74, 8, 85, 79, 12, 78, 26, 98, 77, 87, 86, 84, 64, 89, 100, 68, 1, 20, 25, 29, 45, 23, 35, 53, 92, 46, 70, 66, 90, 22, 71, 94, 10, 69, 33, 30, 34, 104, 95, 40, 6, 123, 101, 5, 67, 0, 60, 61, 122, 3, 110, 96, 59, 39, 109, 51, 111, 126, 2, 65, 117, 97, 112, 127, 116, 43, 55, 58, 54, 120, 125, 115, 105, 102, 44, 36, 37, 57, 48, 38, 56, 108, 41, 114, 121, 106, 107, 118, 52, 42, 47, 119], [62, 103, 124, 113, 49, 122, 61, 115, 57, 45, 40, 87, 26, 50, 119, 112, 53, 32, 48, 58, 104, 109, 29, 54, 121, 120, 126, 63, 59, 117, 118, 60, 90, 95, 108, 110, 93, 55, 127, 46, 116, 52, 125, 51, 94, 123, 111, 56, 114, 99, 47, 96, 44, 105, 42, 107, 31, 106, 41, 27, 43, 14, 33, 19, 17, 91, 38, 97, 28, 23, 102, 98, 21, 30, 20, 101, 74, 100, 81, 35, 36, 37, 84, 34, 24, 39, 92, 86, 78, 25, 22, 89, 70, 6, 13, 66, 88, 68, 15, 69, 83, 7, 18, 75, 85, 67, 9, 16, 79, 64, 77, 10, 5, 8, 72, 3, 12, 4, 82, 11, 71, 80, 65, 2, 0, 1, 73, 76], [113, 124, 103, 49, 62, 50, 53, 46, 26, 122, 45, 61, 32, 40, 120, 51, 59, 123, 63, 87, 126, 112, 109, 5, 58, 117, 57, 30, 47, 100, 35, 127, 115, 111, 90, 8, 108, 17, 54, 104, 121, 105, 56, 114, 68, 125, 33, 95, 48, 60, 1, 99, 52, 116, 83, 118, 97, 106, 7, 94, 31, 107, 101, 41, 25, 64, 110, 119, 43, 34, 80, 91, 75, 79, 6, 29, 93, 38, 44, 3, 36, 9, 42, 77, 92, 55, 102, 12, 98, 28, 37, 27, 0, 66, 23, 14, 96, 76, 78, 84, 20, 21, 2, 82, 86, 88, 24, 15, 19, 39, 74, 73, 89, 10, 85, 81, 67, 11, 22, 70, 71, 16, 65, 69, 4, 72, 13, 18], [113, 124, 62, 40, 103, 50, 49, 46, 122, 83, 59, 112, 126, 87, 5, 53, 8, 127, 109, 45, 77, 120, 61, 117, 1, 78, 26, 123, 6, 68, 80, 55, 63, 54, 75, 35, 58, 125, 42, 51, 57, 7, 114, 9, 115, 3, 60, 17, 111, 118, 76, 52, 56, 47, 15, 116, 119, 108, 48, 110, 121, 44, 41, 82, 30, 43, 107, 104, 32, 106, 0, 73, 105, 12, 101, 64, 33, 100, 38, 66, 102, 36, 37, 79, 11, 89, 85, 34, 2, 23, 71, 90, 84, 31, 99, 94, 24, 92, 95, 67, 29, 81, 98, 96, 97, 20, 21, 28, 25, 93, 14, 10, 19, 39, 74, 86, 65, 88, 91, 27, 22, 70, 69, 13, 4, 16, 18, 72], [102, 56, 49, 113, 23, 89, 55, 91, 19, 58, 85, 47, 37, 31, 20, 52, 18, 33, 14, 127, 95, 92, 122, 80, 124, 115, 117, 29, 125, 59, 120, 42, 112, 107, 123, 75, 61, 57, 46, 108, 28, 116, 114, 82, 50, 45, 121, 73, 48, 84, 109, 44, 77, 88, 41, 63, 51, 99, 39, 40, 53, 38, 103, 126, 34, 43, 97, 118, 32, 16, 13, 106, 11, 87, 100, 54, 104, 98, 94, 15, 86, 27, 105, 83, 60, 119, 111, 26, 35, 110, 78, 79, 62, 8, 93, 21, 30, 101, 5, 22, 25, 36, 90, 96, 74, 72, 6, 81, 10, 24, 71, 76, 17, 4, 7, 9, 12, 67, 66, 69, 65, 70, 0, 3, 1, 68, 2, 64], [102, 56, 113, 49, 85, 23, 91, 19, 80, 77, 9, 31, 11, 89, 58, 4, 13, 55, 6, 70, 68, 72, 47, 2, 73, 95, 75, 16, 21, 83, 52, 65, 8, 82, 64, 87, 39, 48, 27, 42, 7, 53, 15, 78, 37, 25, 109, 110, 124, 22, 26, 69, 125, 90, 112, 46, 5, 33, 81, 84, 79, 71, 50, 99, 88, 32, 120, 63, 12, 106, 17, 119, 35, 44, 127, 10, 61, 92, 30, 96, 107, 123, 29, 14, 0, 101, 76, 24, 28, 66, 118, 54, 40, 116, 57, 94, 115, 59, 114, 98, 45, 126, 60, 122, 41, 86, 67, 121, 18, 100, 20, 105, 36, 104, 43, 117, 111, 34, 93, 74, 97, 51, 62, 103, 108, 1, 3, 38], [102, 47, 113, 56, 49, 89, 91, 23, 85, 19, 31, 55, 58, 9, 80, 82, 77, 11, 14, 0, 120, 46, 68, 121, 72, 59, 51, 33, 66, 95, 83, 127, 37, 124, 114, 70, 52, 53, 88, 110, 78, 87, 84, 3, 48, 99, 10, 15, 112, 81, 57, 126, 60, 73, 62, 16, 35, 42, 1, 21, 109, 34, 20, 125, 32, 116, 108, 27, 79, 118, 5, 61, 12, 100, 96, 29, 7, 97, 30, 2, 93, 25, 44, 106, 40, 92, 71, 36, 104, 17, 43, 24, 39, 122, 123, 63, 22, 119, 67, 103, 69, 18, 26, 38, 13, 115, 4, 65, 117, 90, 50, 8, 54, 111, 98, 45, 105, 94, 64, 86, 101, 41, 74, 76, 28, 107, 75, 6], [102, 113, 56, 49, 23, 80, 19, 89, 9, 85, 77, 11, 70, 68, 31, 2, 55, 64, 83, 14, 1, 16, 72, 21, 91, 37, 87, 6, 58, 75, 25, 67, 82, 3, 66, 125, 13, 120, 47, 33, 27, 48, 52, 127, 7, 73, 42, 112, 78, 12, 110, 65, 8, 29, 53, 59, 5, 17, 57, 46, 24, 32, 44, 88, 22, 10, 81, 76, 116, 51, 50, 92, 124, 30, 114, 121, 40, 122, 61, 95, 109, 28, 39, 4, 84, 74, 93, 123, 90, 119, 99, 26, 43, 35, 86, 115, 18, 63, 79, 15, 62, 45, 106, 100, 118, 69, 126, 105, 103, 20, 107, 41, 104, 60, 71, 117, 94, 98, 101, 97, 54, 36, 108, 34, 96, 111, 0, 38], [52, 62, 122, 58, 102, 125, 61, 53, 60, 127, 123, 112, 121, 119, 63, 115, 114, 50, 57, 48, 117, 126, 124, 118, 56, 49, 46, 54, 55, 113, 120, 47, 116, 44, 111, 36, 110, 51, 107, 45, 59, 109, 108, 35, 42, 30, 105, 43, 106, 104, 89, 25, 40, 41, 101, 103, 100, 23, 39, 33, 38, 37, 96, 32, 92, 99, 98, 34, 95, 97, 94, 31, 87, 28, 27, 91, 75, 29, 26, 15, 93, 24, 90, 17, 84, 21, 9, 20, 82, 85, 88, 70, 2, 6, 11, 19, 81, 80, 77, 12, 0, 79, 16, 14, 74, 86, 7, 65, 10, 67, 73, 69, 3, 66, 13, 5, 4, 64, 83, 8, 18, 76, 71, 1, 72, 68, 22, 78], [58, 62, 52, 122, 60, 121, 63, 118, 125, 55, 112, 127, 115, 51, 114, 117, 126, 123, 50, 110, 46, 59, 61, 113, 119, 111, 43, 120, 56, 42, 48, 124, 47, 116, 49, 57, 45, 109, 104, 96, 53, 40, 41, 105, 54, 108, 106, 107, 37, 101, 39, 102, 32, 103, 44, 38, 100, 29, 25, 23, 35, 98, 36, 34, 99, 84, 33, 27, 95, 30, 19, 97, 86, 24, 94, 20, 90, 22, 31, 92, 28, 89, 93, 81, 91, 80, 17, 16, 88, 83, 13, 79, 77, 26, 87, 85, 15, 21, 73, 78, 76, 11, 18, 82, 14, 70, 75, 9, 72, 12, 6, 10, 0, 74, 66, 68, 7, 2, 71, 8, 3, 67, 69, 1, 65, 5, 4, 64], [62, 58, 52, 101, 121, 122, 60, 127, 125, 112, 55, 115, 118, 117, 123, 63, 51, 53, 57, 119, 114, 50, 113, 61, 126, 102, 111, 120, 124, 48, 46, 59, 110, 36, 116, 56, 42, 54, 109, 47, 45, 44, 106, 49, 108, 93, 105, 43, 107, 40, 41, 100, 96, 104, 90, 30, 39, 103, 38, 99, 94, 35, 34, 23, 29, 25, 28, 27, 97, 80, 95, 86, 98, 33, 19, 16, 85, 92, 89, 31, 91, 21, 32, 73, 75, 24, 87, 37, 83, 26, 9, 88, 81, 11, 3, 77, 13, 22, 20, 82, 68, 70, 84, 15, 71, 5, 69, 10, 74, 1, 78, 14, 17, 7, 18, 76, 8, 79, 6, 67, 64, 66, 4, 12, 2, 65, 72, 0], [58, 52, 62, 101, 24, 90, 34, 82, 93, 14, 12, 85, 19, 80, 121, 32, 60, 21, 86, 30, 37, 15, 20, 76, 78, 96, 9, 122, 18, 79, 69, 83, 16, 125, 67, 74, 10, 88, 94, 72, 55, 5, 8, 127, 100, 84, 29, 77, 87, 7, 17, 71, 1, 81, 112, 102, 26, 13, 25, 23, 118, 75, 89, 91, 113, 27, 68, 117, 0, 2, 114, 64, 98, 70, 120, 124, 11, 4, 61, 66, 115, 126, 73, 104, 119, 110, 92, 51, 63, 42, 28, 107, 123, 111, 31, 3, 116, 46, 50, 59, 45, 65, 57, 56, 53, 48, 95, 6, 47, 106, 97, 41, 33, 43, 99, 103, 109, 38, 40, 35, 105, 22, 108, 39, 36, 54, 49, 44], [118, 127, 26, 101, 17, 10, 86, 15, 77, 70, 98, 88, 82, 20, 74, 93, 6, 29, 68, 89, 13, 104, 30, 22, 72, 87, 2, 76, 85, 79, 81, 27, 92, 84, 90, 54, 24, 67, 37, 28, 34, 71, 18, 11, 7, 52, 19, 78, 3, 9, 80, 1, 12, 40, 91, 109, 59, 65, 126, 75, 95, 73, 97, 0, 66, 16, 42, 63, 64, 23, 5, 21, 83, 4, 94, 105, 62, 31, 14, 47, 113, 69, 121, 32, 33, 96, 25, 39, 43, 35, 36, 8, 38, 107, 50, 110, 99, 44, 41, 53, 103, 108, 100, 48, 56, 115, 122, 112, 46, 116, 124, 58, 102, 117, 45, 111, 55, 114, 49, 120, 61, 119, 106, 123, 125, 60, 51, 57], [118, 127, 101, 98, 37, 126, 88, 86, 62, 39, 30, 29, 50, 20, 63, 91, 104, 52, 26, 113, 72, 48, 121, 54, 112, 56, 119, 93, 82, 38, 97, 116, 43, 125, 59, 122, 44, 110, 60, 92, 66, 83, 117, 0, 35, 51, 25, 40, 109, 58, 77, 123, 53, 46, 102, 45, 55, 15, 69, 115, 41, 120, 108, 28, 31, 57, 61, 49, 1, 47, 90, 114, 99, 24, 11, 84, 111, 89, 106, 75, 36, 124, 94, 33, 103, 8, 6, 4, 79, 23, 68, 76, 80, 96, 18, 107, 73, 105, 42, 74, 3, 64, 13, 17, 100, 22, 32, 81, 78, 2, 95, 85, 16, 87, 19, 9, 27, 12, 14, 34, 21, 5, 71, 7, 65, 70, 10, 67], [127, 118, 62, 101, 126, 56, 104, 113, 63, 54, 119, 48, 112, 52, 82, 40, 60, 121, 44, 50, 122, 116, 98, 43, 51, 123, 59, 117, 88, 58, 97, 125, 120, 91, 37, 106, 110, 55, 57, 47, 109, 46, 53, 45, 114, 115, 61, 39, 86, 42, 111, 29, 49, 107, 23, 41, 8, 124, 108, 105, 100, 35, 102, 93, 36, 38, 103, 83, 96, 72, 31, 92, 26, 99, 33, 66, 30, 87, 28, 4, 95, 89, 32, 34, 80, 90, 25, 94, 22, 27, 16, 18, 68, 24, 20, 15, 19, 0, 77, 85, 12, 84, 2, 69, 78, 13, 14, 73, 21, 75, 11, 74, 71, 5, 76, 79, 81, 7, 17, 1, 64, 6, 9, 65, 3, 70, 10, 67], [127, 118, 101, 126, 56, 97, 88, 82, 62, 29, 63, 54, 52, 40, 113, 48, 44, 28, 23, 119, 109, 60, 43, 98, 112, 36, 122, 117, 50, 41, 121, 106, 45, 116, 59, 8, 86, 51, 37, 123, 115, 104, 47, 125, 91, 32, 26, 114, 58, 110, 120, 105, 89, 46, 39, 61, 102, 111, 55, 107, 93, 30, 57, 53, 108, 72, 42, 100, 38, 16, 20, 4, 92, 49, 12, 99, 124, 103, 35, 18, 34, 31, 33, 94, 77, 83, 13, 27, 15, 71, 79, 96, 95, 87, 81, 22, 14, 25, 80, 24, 75, 84, 21, 11, 85, 76, 69, 74, 68, 0, 6, 66, 90, 73, 1, 17, 19, 7, 3, 78, 9, 5, 2, 70, 10, 64, 65, 67], [42, 96, 106, 120, 45, 125, 26, 23, 20, 118, 62, 109, 78, 52, 98, 80, 67, 63, 124, 90, 122, 32, 82, 71, 126, 7, 34, 101, 94, 58, 27, 56, 87, 14, 73, 55, 77, 116, 12, 86, 97, 123, 13, 81, 48, 85, 91, 115, 35, 57, 103, 104, 114, 43, 50, 16, 49, 18, 99, 110, 111, 39, 76, 33, 22, 64, 65, 117, 24, 119, 37, 10, 1, 53, 92, 59, 31, 47, 60, 28, 112, 84, 17, 19, 46, 121, 83, 44, 127, 21, 29, 38, 89, 113, 54, 30, 74, 5, 107, 70, 61, 93, 41, 51, 95, 105, 8, 4, 102, 88, 15, 108, 100, 40, 25, 36, 3, 6, 68, 79, 9, 75, 66, 69, 72, 11, 2, 0], [42, 96, 106, 125, 45, 23, 26, 62, 20, 109, 52, 98, 80, 124, 118, 126, 120, 34, 63, 78, 90, 82, 55, 27, 94, 122, 101, 77, 56, 43, 116, 46, 111, 58, 103, 81, 44, 35, 119, 114, 97, 104, 53, 7, 123, 87, 48, 127, 115, 50, 99, 49, 107, 60, 39, 38, 41, 91, 113, 117, 110, 59, 47, 30, 100, 54, 121, 102, 40, 85, 37, 57, 67, 61, 108, 105, 36, 112, 31, 95, 18, 93, 16, 51, 28, 32, 33, 29, 73, 13, 92, 21, 14, 86, 84, 71, 24, 25, 17, 19, 12, 5, 22, 89, 10, 88, 83, 74, 9, 11, 1, 65, 69, 79, 75, 15, 76, 64, 3, 8, 72, 4, 6, 68, 70, 66, 0, 2], [42, 96, 125, 106, 52, 45, 23, 20, 26, 82, 80, 78, 120, 98, 118, 109, 5, 62, 73, 7, 77, 12, 124, 90, 101, 86, 126, 103, 67, 63, 122, 32, 94, 35, 27, 87, 9, 56, 16, 1, 55, 58, 21, 69, 85, 91, 83, 64, 84, 76, 114, 123, 75, 3, 50, 34, 13, 18, 116, 115, 22, 11, 29, 70, 17, 40, 19, 43, 15, 48, 10, 39, 97, 81, 14, 74, 30, 110, 4, 79, 37, 31, 89, 44, 108, 104, 24, 49, 25, 71, 47, 66, 59, 92, 99, 60, 113, 28, 95, 112, 107, 88, 51, 8, 111, 100, 117, 72, 121, 57, 93, 6, 119, 105, 38, 41, 33, 36, 65, 54, 127, 68, 61, 0, 2, 102, 46, 53], [42, 96, 45, 106, 125, 23, 124, 26, 20, 52, 109, 98, 120, 62, 118, 126, 80, 122, 82, 115, 94, 56, 103, 116, 55, 78, 34, 90, 63, 58, 43, 48, 87, 114, 97, 101, 50, 60, 77, 110, 39, 41, 35, 104, 13, 123, 31, 47, 108, 53, 107, 99, 119, 113, 30, 19, 121, 27, 117, 33, 57, 49, 51, 40, 7, 61, 38, 102, 54, 92, 44, 85, 100, 67, 46, 32, 111, 93, 95, 36, 59, 37, 105, 81, 91, 112, 18, 28, 16, 29, 84, 127, 71, 86, 24, 14, 25, 73, 21, 89, 88, 79, 76, 12, 22, 74, 72, 69, 83, 17, 11, 15, 10, 9, 5, 68, 75, 65, 1, 8, 4, 66, 6, 64, 70, 0, 3, 2], [102, 62, 126, 98, 87, 26, 110, 85, 51, 41, 30, 19, 38, 45, 50, 22, 15, 79, 103, 94, 90, 84, 12, 93, 31, 17, 16, 113, 21, 109, 69, 105, 112, 20, 125, 127, 23, 8, 53, 65, 83, 25, 32, 117, 119, 14, 88, 63, 6, 55, 99, 58, 1, 116, 122, 75, 46, 57, 60, 96, 100, 108, 34, 121, 124, 52, 80, 120, 115, 74, 59, 106, 24, 89, 114, 49, 97, 111, 10, 76, 104, 118, 0, 2, 95, 64, 27, 86, 47, 40, 73, 123, 48, 81, 36, 92, 70, 54, 67, 82, 28, 42, 66, 5, 35, 29, 91, 78, 18, 77, 9, 13, 56, 44, 72, 39, 11, 101, 71, 68, 107, 61, 33, 37, 4, 43, 7, 3], [102, 126, 62, 98, 87, 17, 85, 110, 90, 26, 22, 19, 50, 8, 14, 75, 45, 103, 41, 15, 94, 21, 93, 89, 6, 51, 24, 113, 109, 2, 30, 84, 88, 83, 68, 92, 12, 38, 27, 58, 20, 31, 72, 79, 78, 112, 122, 23, 25, 74, 80, 32, 4, 71, 16, 117, 119, 53, 49, 127, 77, 29, 47, 7, 35, 81, 76, 60, 105, 10, 111, 28, 55, 120, 116, 52, 82, 0, 124, 46, 48, 5, 91, 13, 99, 125, 34, 67, 39, 36, 33, 121, 96, 97, 57, 95, 86, 63, 37, 118, 65, 115, 18, 44, 100, 66, 104, 11, 73, 108, 59, 42, 114, 43, 101, 9, 61, 69, 123, 54, 56, 3, 106, 64, 107, 70, 1, 40], [102, 126, 98, 62, 26, 41, 45, 87, 85, 19, 17, 93, 38, 103, 22, 15, 94, 51, 21, 50, 12, 113, 92, 110, 32, 75, 14, 30, 8, 100, 90, 31, 79, 83, 84, 88, 89, 20, 86, 35, 36, 54, 55, 25, 117, 127, 125, 119, 76, 13, 77, 81, 123, 46, 82, 23, 6, 2, 16, 53, 109, 39, 18, 27, 121, 59, 116, 70, 74, 97, 69, 80, 114, 47, 111, 52, 57, 105, 29, 118, 78, 58, 24, 49, 11, 63, 95, 99, 108, 122, 96, 115, 112, 10, 48, 91, 104, 124, 42, 66, 33, 72, 120, 40, 60, 101, 34, 37, 7, 43, 73, 65, 28, 1, 4, 107, 67, 5, 71, 44, 106, 61, 68, 9, 56, 3, 64, 0], [62, 102, 126, 98, 87, 41, 110, 26, 85, 38, 50, 45, 22, 105, 103, 109, 30, 113, 94, 63, 112, 93, 58, 125, 84, 121, 53, 127, 119, 15, 122, 31, 51, 116, 59, 57, 19, 52, 114, 55, 47, 97, 117, 118, 90, 44, 46, 43, 60, 49, 111, 108, 56, 42, 106, 54, 120, 124, 48, 115, 101, 36, 107, 32, 40, 123, 104, 100, 99, 61, 35, 37, 25, 39, 20, 96, 12, 17, 83, 95, 79, 92, 88, 27, 28, 33, 91, 29, 16, 8, 89, 23, 74, 21, 73, 34, 6, 75, 24, 10, 18, 82, 14, 69, 78, 86, 80, 71, 2, 77, 76, 5, 13, 81, 9, 7, 1, 65, 68, 11, 72, 70, 67, 0, 64, 66, 3, 4], [120, 122, 54, 37, 53, 127, 19, 101, 87, 17, 113, 35, 91, 58, 93, 60, 123, 124, 13, 115, 50, 117, 55, 56, 89, 63, 51, 118, 116, 32, 75, 45, 74, 114, 15, 119, 57, 14, 125, 112, 80, 49, 33, 36, 62, 52, 48, 100, 98, 12, 61, 97, 103, 47, 34, 110, 96, 90, 44, 73, 106, 59, 43, 126, 111, 41, 7, 26, 38, 18, 39, 121, 102, 85, 46, 108, 99, 104, 109, 107, 105, 29, 95, 42, 31, 27, 21, 20, 83, 81, 40, 72, 23, 94, 28, 92, 25, 30, 88, 24, 86, 5, 70, 68, 82, 67, 65, 2, 22, 0, 78, 77, 79, 84, 11, 9, 76, 16, 71, 8, 3, 10, 4, 69, 6, 1, 64, 66], [120, 122, 37, 13, 19, 17, 75, 14, 87, 15, 80, 74, 32, 85, 20, 53, 12, 18, 54, 73, 113, 127, 93, 58, 123, 26, 88, 115, 7, 30, 60, 50, 35, 124, 51, 55, 91, 45, 117, 72, 101, 116, 56, 63, 118, 94, 119, 57, 114, 90, 98, 62, 125, 112, 52, 25, 92, 49, 48, 86, 24, 61, 23, 36, 126, 33, 31, 96, 59, 47, 46, 34, 44, 82, 111, 110, 43, 70, 79, 41, 121, 21, 109, 99, 84, 29, 81, 108, 106, 100, 5, 42, 103, 27, 105, 104, 107, 16, 39, 97, 40, 95, 38, 102, 78, 77, 28, 22, 89, 68, 11, 83, 76, 67, 9, 65, 10, 2, 8, 71, 3, 0, 4, 6, 69, 64, 1, 66], [122, 120, 37, 84, 7, 16, 67, 74, 66, 69, 19, 75, 1, 68, 72, 24, 5, 70, 73, 12, 94, 78, 71, 11, 8, 53, 6, 64, 23, 13, 4, 3, 82, 88, 15, 54, 14, 80, 20, 90, 32, 30, 58, 92, 87, 93, 18, 113, 26, 115, 79, 77, 10, 123, 127, 50, 51, 60, 76, 124, 17, 117, 55, 56, 85, 21, 116, 96, 83, 63, 45, 118, 35, 9, 25, 65, 98, 0, 114, 119, 57, 28, 112, 101, 62, 49, 52, 29, 99, 48, 86, 125, 31, 61, 126, 59, 36, 81, 46, 91, 41, 44, 111, 47, 43, 22, 121, 110, 27, 95, 100, 109, 108, 2, 97, 33, 34, 106, 89, 105, 42, 102, 104, 39, 107, 103, 40, 38], [120, 122, 54, 87, 19, 53, 127, 123, 113, 60, 58, 124, 125, 115, 117, 55, 51, 116, 50, 17, 62, 86, 56, 27, 57, 119, 118, 45, 94, 49, 25, 52, 85, 114, 61, 96, 48, 28, 63, 23, 126, 112, 59, 44, 47, 111, 121, 24, 46, 109, 81, 43, 110, 90, 41, 106, 108, 84, 42, 107, 39, 83, 105, 82, 103, 35, 104, 40, 21, 37, 98, 38, 88, 36, 102, 89, 91, 79, 20, 65, 100, 101, 77, 16, 80, 97, 34, 99, 22, 78, 0, 33, 13, 10, 30, 29, 26, 76, 95, 31, 64, 93, 11, 18, 2, 9, 14, 15, 32, 3, 92, 71, 68, 4, 8, 7, 12, 75, 5, 6, 67, 69, 74, 1, 73, 70, 72, 66]], "model.layers.10.self_attn.k_proj": [[122, 121, 100, 118, 86, 63, 114, 123, 54, 61, 124, 119, 62, 52, 125, 115, 57, 108, 111, 117, 120, 60, 112, 31, 53, 126, 51, 127, 116, 59, 47, 58, 110, 55, 102, 46, 48, 83, 0, 45, 109, 49, 113, 56, 98, 29, 105, 104, 77, 9, 40, 44, 76, 103, 39, 38, 91, 106, 80, 81, 24, 6, 107, 35, 101, 34, 41, 37, 43, 21, 42, 79, 97, 26, 23, 1, 30, 22, 28, 36, 99, 32, 89, 88, 84, 3, 69, 50, 10, 95, 96, 92, 78, 19, 93, 33, 4, 20, 90, 82, 11, 85, 87, 7, 27, 25, 94, 14, 2, 18, 72, 8, 74, 5, 16, 71, 13, 66, 15, 67, 73, 65, 68, 70, 12, 75, 64, 17], [113, 124, 39, 50, 53, 96, 93, 21, 123, 120, 49, 15, 62, 126, 125, 63, 51, 112, 11, 87, 24, 82, 122, 58, 55, 114, 48, 117, 13, 91, 104, 116, 46, 61, 73, 30, 54, 115, 57, 71, 52, 95, 84, 26, 81, 118, 76, 121, 59, 127, 47, 80, 60, 111, 108, 19, 110, 86, 8, 99, 109, 56, 40, 45, 119, 100, 92, 89, 3, 10, 44, 35, 14, 1, 2, 98, 43, 68, 107, 79, 6, 106, 41, 27, 88, 0, 105, 42, 18, 38, 5, 102, 101, 36, 37, 33, 28, 23, 97, 77, 16, 17, 32, 94, 69, 34, 78, 66, 70, 7, 25, 31, 22, 72, 64, 4, 74, 29, 9, 83, 65, 90, 85, 20, 67, 12, 75, 103], [113, 56, 38, 80, 19, 77, 85, 23, 89, 11, 91, 9, 95, 58, 70, 72, 68, 55, 64, 14, 76, 1, 66, 20, 46, 123, 78, 111, 127, 44, 59, 106, 116, 112, 47, 17, 57, 53, 121, 69, 114, 117, 115, 71, 125, 51, 101, 109, 86, 118, 126, 42, 79, 124, 97, 52, 74, 48, 108, 45, 18, 96, 120, 24, 28, 82, 15, 37, 50, 33, 98, 39, 60, 122, 29, 119, 31, 93, 92, 32, 43, 3, 62, 103, 90, 61, 99, 34, 67, 5, 63, 12, 40, 49, 26, 88, 110, 35, 7, 30, 94, 100, 54, 104, 36, 81, 41, 105, 22, 8, 107, 10, 75, 25, 27, 87, 6, 84, 21, 0, 13, 83, 2, 73, 102, 16, 65, 4], [58, 62, 86, 37, 52, 96, 121, 29, 90, 14, 82, 17, 19, 12, 80, 60, 127, 77, 10, 72, 118, 125, 117, 24, 119, 55, 51, 20, 115, 63, 114, 126, 57, 120, 123, 79, 7, 100, 124, 50, 56, 44, 53, 48, 116, 98, 34, 59, 113, 46, 111, 110, 43, 45, 61, 25, 54, 93, 76, 112, 99, 109, 106, 70, 47, 9, 40, 49, 36, 15, 41, 122, 108, 105, 107, 11, 69, 104, 38, 42, 39, 102, 2, 23, 8, 97, 3, 85, 91, 21, 27, 35, 4, 95, 103, 31, 84, 83, 101, 6, 28, 33, 18, 74, 94, 30, 81, 65, 0, 92, 13, 22, 1, 16, 71, 87, 88, 68, 64, 32, 78, 5, 26, 89, 75, 73, 67, 66], [118, 127, 37, 126, 86, 63, 62, 10, 52, 34, 17, 48, 113, 30, 26, 119, 56, 121, 122, 15, 54, 70, 77, 112, 60, 43, 29, 50, 104, 59, 88, 51, 20, 120, 117, 116, 41, 110, 96, 115, 61, 123, 0, 3, 109, 57, 58, 55, 125, 47, 45, 53, 114, 108, 111, 49, 124, 106, 76, 103, 107, 46, 44, 82, 42, 71, 100, 2, 83, 38, 9, 65, 40, 68, 1, 72, 99, 32, 105, 92, 85, 39, 75, 35, 33, 95, 97, 36, 11, 91, 102, 14, 23, 31, 4, 80, 94, 67, 7, 64, 5, 73, 78, 69, 19, 25, 21, 27, 101, 93, 98, 81, 24, 90, 28, 89, 16, 12, 66, 79, 84, 87, 22, 8, 18, 74, 13, 6], [106, 32, 52, 26, 23, 20, 82, 125, 120, 42, 118, 45, 80, 78, 109, 126, 73, 58, 124, 77, 62, 112, 34, 12, 123, 114, 115, 50, 56, 55, 59, 54, 7, 5, 63, 85, 116, 47, 81, 122, 3, 51, 37, 119, 61, 39, 43, 103, 30, 104, 6, 1, 110, 127, 117, 101, 0, 107, 108, 11, 27, 48, 86, 65, 91, 46, 66, 13, 44, 57, 75, 74, 49, 35, 70, 111, 40, 60, 2, 22, 76, 64, 105, 98, 53, 79, 121, 102, 113, 33, 94, 29, 72, 71, 92, 36, 41, 83, 68, 8, 19, 25, 88, 14, 24, 100, 21, 4, 69, 38, 31, 97, 89, 10, 99, 95, 15, 28, 93, 96, 84, 90, 9, 17, 18, 87, 16, 67], [126, 62, 38, 34, 26, 94, 45, 85, 19, 41, 87, 22, 17, 110, 51, 84, 15, 50, 113, 12, 103, 75, 8, 14, 1, 127, 53, 93, 57, 112, 119, 117, 55, 116, 120, 39, 124, 125, 111, 122, 109, 59, 114, 118, 69, 63, 74, 52, 121, 102, 6, 29, 60, 115, 66, 28, 106, 5, 95, 73, 24, 108, 49, 32, 48, 58, 77, 123, 97, 46, 43, 33, 68, 105, 104, 54, 56, 16, 36, 47, 27, 35, 31, 40, 86, 61, 100, 44, 91, 64, 107, 4, 71, 67, 25, 78, 18, 42, 80, 9, 101, 99, 72, 23, 7, 37, 90, 3, 98, 82, 89, 92, 96, 70, 88, 0, 30, 21, 76, 13, 81, 20, 10, 83, 79, 11, 2, 65], [122, 120, 101, 24, 30, 82, 54, 127, 32, 79, 28, 84, 117, 26, 21, 113, 115, 124, 123, 55, 16, 27, 53, 56, 78, 60, 116, 50, 11, 77, 62, 114, 118, 9, 76, 51, 89, 63, 119, 57, 112, 52, 49, 125, 48, 93, 45, 72, 58, 126, 47, 59, 46, 44, 2, 70, 23, 61, 108, 81, 109, 10, 111, 110, 99, 121, 12, 65, 43, 5, 41, 106, 107, 29, 22, 35, 42, 39, 71, 104, 8, 105, 3, 103, 40, 14, 96, 97, 100, 4, 102, 38, 95, 98, 75, 83, 36, 80, 18, 69, 34, 68, 73, 33, 86, 6, 37, 31, 20, 92, 15, 7, 64, 87, 0, 91, 13, 90, 94, 25, 67, 88, 74, 17, 19, 85, 1, 66]], "model.layers.10.self_attn.qk_proj": [[122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 124, 106, 42, 54, 50, 90, 45, 110, 102, 55, 63, 117, 37, 125, 51, 116, 123, 53, 49, 60, 114, 87, 115, 85, 61, 38, 93, 57, 46, 23, 101, 59, 80, 83, 47, 32, 21, 22, 41, 112, 96, 34, 119, 103, 19, 16, 26, 25, 44, 13, 109, 20, 48, 104, 14, 77, 82, 86, 39, 43, 88, 27, 18, 84, 24, 100, 12, 11, 94, 98, 75, 81, 78, 30, 89, 108, 15, 29, 17, 111, 9, 91, 79, 73, 76, 31, 40, 107, 36, 35, 95, 97, 72, 6, 74, 105, 92, 10, 70, 99, 28, 71, 8, 68, 33, 64, 69, 65, 2, 1, 7, 3, 67, 5, 4, 0, 66], [122, 113, 62, 120, 118, 56, 121, 127, 126, 58, 52, 124, 106, 42, 54, 50, 117, 90, 116, 45, 37, 55, 110, 102, 60, 87, 114, 63, 123, 125, 85, 49, 51, 115, 23, 61, 38, 53, 101, 41, 93, 46, 103, 21, 80, 83, 96, 22, 34, 112, 104, 32, 19, 109, 59, 20, 26, 111, 27, 48, 43, 13, 119, 57, 25, 47, 98, 24, 84, 16, 86, 77, 88, 18, 14, 89, 39, 100, 30, 81, 108, 79, 12, 82, 31, 94, 78, 11, 75, 17, 29, 44, 9, 40, 91, 73, 95, 76, 15, 70, 92, 72, 97, 33, 74, 36, 105, 10, 35, 99, 6, 107, 69, 4, 8, 0, 68, 28, 65, 71, 64, 7, 5, 3, 67, 2, 66, 1], [122, 113, 62, 120, 118, 127, 121, 56, 126, 58, 52, 106, 124, 42, 54, 50, 116, 102, 55, 110, 37, 125, 51, 45, 90, 38, 60, 53, 114, 49, 63, 123, 87, 101, 117, 23, 93, 57, 115, 34, 96, 32, 119, 26, 85, 27, 43, 41, 103, 83, 22, 104, 19, 109, 21, 48, 112, 24, 25, 46, 61, 111, 16, 80, 59, 30, 94, 20, 84, 47, 86, 100, 88, 13, 77, 95, 18, 89, 39, 31, 98, 91, 29, 81, 14, 82, 35, 108, 78, 75, 12, 79, 92, 44, 11, 99, 15, 17, 40, 9, 70, 36, 73, 33, 76, 72, 107, 105, 28, 74, 97, 71, 4, 10, 1, 6, 7, 68, 64, 69, 8, 5, 65, 0, 66, 67, 3, 2], [122, 113, 62, 120, 118, 121, 127, 126, 56, 58, 52, 106, 124, 42, 54, 55, 117, 50, 90, 102, 45, 125, 60, 51, 114, 123, 37, 110, 116, 87, 63, 49, 53, 112, 23, 93, 115, 46, 119, 101, 38, 85, 96, 19, 57, 27, 47, 41, 61, 43, 109, 34, 22, 16, 104, 25, 83, 32, 59, 21, 80, 24, 44, 100, 26, 77, 13, 48, 111, 39, 88, 103, 30, 81, 18, 86, 94, 82, 84, 89, 98, 78, 20, 108, 17, 91, 12, 14, 40, 75, 95, 92, 11, 31, 35, 15, 105, 79, 9, 29, 107, 72, 70, 36, 73, 33, 74, 76, 28, 99, 97, 10, 71, 4, 6, 8, 0, 69, 7, 65, 68, 1, 5, 64, 67, 66, 3, 2], [122, 113, 62, 118, 120, 121, 126, 56, 127, 58, 52, 124, 106, 42, 55, 110, 54, 50, 90, 123, 45, 125, 102, 51, 37, 60, 116, 53, 49, 114, 117, 63, 87, 38, 115, 47, 61, 93, 46, 57, 119, 109, 96, 112, 23, 85, 59, 26, 101, 34, 103, 83, 21, 111, 80, 19, 16, 48, 22, 41, 77, 32, 13, 43, 100, 104, 18, 25, 39, 82, 14, 89, 81, 86, 24, 30, 27, 44, 94, 108, 20, 84, 88, 78, 11, 98, 36, 91, 17, 29, 9, 79, 75, 15, 40, 35, 73, 31, 12, 76, 95, 72, 70, 74, 105, 107, 97, 33, 4, 92, 10, 68, 8, 7, 28, 6, 71, 99, 65, 64, 0, 1, 67, 69, 2, 5, 66, 3], [122, 113, 62, 118, 120, 127, 121, 56, 126, 58, 52, 124, 106, 42, 54, 55, 90, 45, 110, 125, 116, 114, 60, 50, 117, 87, 63, 102, 123, 37, 53, 51, 46, 85, 93, 61, 49, 38, 96, 119, 23, 19, 115, 80, 109, 22, 103, 21, 41, 57, 34, 83, 77, 112, 101, 47, 16, 13, 111, 48, 20, 26, 32, 59, 25, 18, 104, 14, 100, 82, 98, 27, 88, 12, 15, 86, 11, 78, 39, 108, 24, 17, 81, 30, 75, 79, 89, 84, 43, 9, 31, 76, 105, 94, 73, 35, 29, 44, 91, 40, 107, 36, 10, 72, 92, 74, 70, 95, 33, 6, 97, 8, 28, 99, 68, 4, 69, 5, 71, 67, 64, 0, 1, 66, 7, 2, 65, 3], [122, 113, 62, 120, 118, 56, 127, 126, 121, 58, 52, 106, 124, 42, 90, 45, 125, 102, 55, 116, 37, 110, 54, 114, 87, 60, 50, 53, 85, 101, 51, 63, 23, 115, 61, 38, 46, 123, 49, 119, 93, 117, 22, 112, 80, 59, 19, 83, 16, 103, 21, 20, 48, 32, 96, 77, 34, 109, 41, 47, 57, 26, 111, 13, 100, 18, 25, 86, 82, 24, 27, 81, 78, 104, 84, 44, 88, 11, 75, 15, 14, 39, 12, 30, 98, 89, 94, 43, 9, 105, 108, 17, 31, 91, 76, 79, 29, 73, 40, 35, 36, 95, 107, 92, 74, 72, 99, 10, 6, 33, 70, 97, 8, 71, 28, 68, 7, 69, 66, 4, 5, 1, 65, 0, 3, 64, 2, 67], [113, 122, 62, 120, 118, 126, 56, 127, 121, 58, 106, 52, 124, 42, 90, 54, 45, 50, 37, 125, 117, 87, 102, 55, 23, 49, 46, 110, 60, 85, 123, 116, 83, 53, 61, 38, 119, 93, 114, 112, 21, 101, 22, 51, 115, 19, 80, 16, 77, 47, 63, 109, 111, 96, 86, 26, 32, 20, 25, 57, 82, 103, 41, 59, 13, 18, 84, 24, 78, 34, 27, 81, 98, 48, 39, 30, 100, 44, 17, 108, 15, 14, 79, 88, 75, 11, 104, 12, 9, 76, 89, 91, 105, 73, 107, 31, 43, 29, 40, 94, 92, 95, 36, 35, 6, 97, 72, 74, 99, 33, 10, 71, 70, 8, 68, 7, 28, 69, 65, 5, 0, 4, 64, 3, 1, 2, 66, 67], [122, 113, 62, 118, 120, 127, 56, 121, 126, 58, 52, 106, 124, 42, 90, 54, 45, 50, 37, 110, 123, 102, 55, 125, 87, 117, 116, 23, 60, 49, 38, 101, 53, 21, 93, 85, 83, 115, 114, 46, 47, 112, 63, 109, 57, 51, 59, 32, 16, 26, 22, 61, 19, 34, 96, 41, 111, 20, 25, 98, 119, 80, 77, 39, 27, 104, 24, 84, 86, 13, 82, 100, 78, 30, 108, 88, 43, 44, 18, 81, 48, 94, 91, 40, 17, 15, 29, 103, 11, 31, 75, 79, 105, 92, 89, 6, 14, 76, 95, 36, 9, 12, 73, 107, 97, 33, 35, 74, 99, 10, 28, 8, 72, 71, 68, 70, 1, 64, 4, 65, 0, 7, 69, 66, 2, 5, 67, 3], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 124, 106, 42, 117, 90, 116, 125, 123, 50, 55, 102, 45, 37, 87, 60, 23, 51, 63, 54, 110, 101, 115, 114, 38, 83, 53, 112, 85, 49, 109, 21, 93, 59, 47, 61, 80, 46, 111, 96, 16, 26, 77, 22, 82, 57, 119, 41, 19, 32, 103, 20, 13, 24, 34, 86, 27, 100, 108, 30, 25, 98, 104, 84, 14, 48, 18, 78, 81, 91, 39, 43, 88, 17, 79, 94, 12, 75, 44, 89, 73, 29, 11, 76, 15, 92, 107, 9, 40, 6, 36, 31, 97, 105, 95, 8, 33, 10, 99, 74, 35, 72, 68, 70, 28, 71, 69, 4, 7, 67, 0, 5, 65, 3, 1, 2, 64, 66], [122, 113, 62, 118, 126, 120, 121, 56, 127, 58, 52, 124, 106, 42, 123, 54, 125, 55, 110, 45, 102, 51, 90, 60, 50, 116, 117, 37, 63, 49, 87, 114, 53, 112, 119, 23, 115, 101, 85, 96, 93, 109, 57, 59, 38, 80, 111, 21, 83, 61, 19, 46, 47, 22, 77, 34, 16, 25, 41, 26, 20, 86, 13, 44, 18, 32, 82, 98, 48, 103, 100, 39, 30, 14, 81, 108, 43, 91, 24, 27, 15, 29, 78, 17, 11, 84, 12, 94, 89, 104, 73, 9, 8, 36, 79, 75, 76, 40, 10, 88, 6, 31, 74, 92, 107, 70, 105, 97, 95, 35, 68, 33, 28, 71, 99, 4, 65, 72, 1, 69, 0, 7, 64, 3, 66, 5, 2, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 124, 106, 42, 123, 90, 51, 54, 55, 102, 110, 49, 45, 116, 125, 117, 50, 60, 87, 37, 114, 63, 112, 23, 53, 93, 38, 85, 101, 80, 109, 115, 83, 111, 59, 57, 96, 19, 46, 22, 47, 34, 20, 61, 21, 48, 25, 16, 119, 103, 77, 13, 41, 26, 32, 18, 39, 86, 14, 43, 98, 81, 82, 100, 11, 108, 75, 94, 24, 78, 27, 84, 17, 79, 15, 88, 76, 44, 9, 104, 30, 40, 12, 29, 36, 89, 73, 91, 10, 8, 107, 70, 74, 31, 6, 95, 35, 72, 97, 4, 92, 68, 7, 33, 99, 0, 69, 71, 1, 28, 65, 66, 105, 64, 5, 2, 3, 67], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 54, 50, 90, 45, 102, 37, 110, 117, 55, 51, 116, 87, 23, 125, 85, 123, 38, 60, 57, 53, 101, 59, 63, 19, 46, 21, 119, 115, 93, 80, 26, 83, 32, 114, 22, 96, 47, 49, 109, 77, 86, 16, 103, 41, 20, 111, 112, 39, 98, 48, 61, 34, 25, 27, 24, 18, 13, 81, 82, 100, 91, 78, 17, 84, 15, 14, 89, 30, 88, 12, 43, 94, 76, 79, 11, 75, 104, 44, 29, 95, 9, 40, 108, 31, 36, 107, 8, 73, 92, 35, 33, 70, 97, 105, 74, 99, 72, 10, 71, 4, 28, 6, 7, 69, 68, 0, 65, 5, 3, 1, 66, 67, 64, 2], [122, 113, 62, 118, 126, 120, 127, 56, 121, 58, 52, 106, 124, 42, 54, 117, 125, 55, 90, 45, 50, 123, 87, 102, 110, 60, 116, 37, 63, 46, 114, 49, 51, 23, 85, 119, 57, 93, 38, 115, 83, 77, 19, 101, 112, 47, 53, 61, 21, 59, 109, 16, 96, 80, 22, 111, 86, 13, 20, 25, 34, 81, 26, 18, 41, 82, 27, 48, 103, 78, 14, 98, 39, 32, 76, 24, 100, 30, 17, 43, 75, 12, 15, 84, 11, 9, 73, 44, 79, 8, 104, 70, 91, 108, 29, 94, 40, 88, 31, 89, 72, 74, 92, 97, 35, 107, 36, 6, 95, 10, 7, 64, 71, 69, 33, 105, 4, 0, 65, 68, 99, 5, 66, 1, 2, 28, 67, 3], [122, 113, 62, 120, 118, 127, 126, 56, 58, 121, 52, 124, 106, 42, 54, 110, 55, 102, 125, 123, 50, 117, 45, 37, 90, 60, 51, 116, 87, 49, 46, 114, 61, 38, 23, 57, 63, 101, 112, 96, 109, 85, 115, 93, 59, 19, 41, 32, 83, 21, 26, 111, 22, 119, 47, 80, 53, 77, 34, 103, 20, 48, 25, 27, 98, 16, 86, 24, 100, 43, 39, 82, 13, 18, 44, 78, 81, 17, 94, 84, 15, 79, 89, 30, 107, 14, 40, 108, 29, 70, 75, 91, 104, 73, 9, 76, 11, 12, 88, 36, 31, 8, 72, 95, 35, 74, 105, 92, 97, 10, 33, 6, 28, 71, 99, 64, 1, 4, 69, 68, 0, 7, 65, 5, 67, 2, 66, 3], [122, 113, 62, 118, 120, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 54, 45, 110, 90, 102, 125, 123, 117, 60, 37, 114, 51, 50, 87, 63, 46, 109, 61, 116, 85, 93, 101, 115, 49, 103, 53, 23, 38, 112, 47, 96, 57, 22, 83, 80, 59, 19, 111, 21, 26, 20, 32, 86, 77, 16, 41, 82, 34, 98, 13, 25, 100, 119, 39, 27, 24, 44, 78, 81, 18, 43, 48, 14, 30, 84, 89, 104, 75, 17, 15, 11, 76, 79, 29, 73, 12, 40, 108, 91, 94, 70, 9, 31, 88, 105, 74, 33, 10, 72, 36, 92, 107, 35, 95, 97, 99, 8, 71, 6, 68, 28, 4, 7, 0, 1, 66, 69, 64, 67, 5, 3, 2, 65], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 52, 106, 124, 42, 117, 37, 102, 54, 90, 125, 87, 45, 116, 110, 50, 55, 123, 60, 23, 85, 46, 63, 112, 51, 114, 101, 93, 80, 38, 115, 49, 96, 59, 19, 47, 83, 21, 53, 103, 32, 111, 109, 27, 57, 119, 22, 26, 86, 16, 20, 77, 61, 24, 34, 25, 98, 81, 82, 48, 100, 18, 39, 41, 78, 44, 14, 13, 11, 79, 30, 17, 91, 84, 15, 88, 43, 9, 94, 40, 89, 12, 75, 104, 108, 29, 76, 31, 95, 73, 72, 36, 10, 35, 74, 92, 70, 28, 33, 107, 8, 6, 99, 7, 105, 97, 71, 68, 4, 69, 66, 5, 0, 64, 67, 1, 3, 65, 2], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 102, 37, 110, 90, 45, 87, 54, 125, 50, 60, 55, 112, 116, 85, 51, 23, 38, 46, 49, 123, 114, 101, 63, 53, 93, 115, 21, 47, 109, 26, 19, 83, 96, 80, 103, 59, 32, 61, 111, 57, 119, 16, 27, 22, 18, 41, 44, 24, 39, 34, 25, 86, 98, 77, 20, 84, 82, 100, 108, 17, 15, 43, 81, 88, 78, 30, 13, 91, 11, 79, 14, 104, 75, 40, 76, 89, 48, 31, 94, 9, 95, 72, 29, 73, 36, 107, 92, 12, 10, 35, 33, 6, 99, 74, 28, 97, 8, 70, 105, 7, 71, 5, 4, 68, 64, 0, 1, 65, 69, 67, 2, 3, 66], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 124, 106, 42, 117, 55, 102, 37, 125, 45, 60, 116, 110, 54, 90, 51, 123, 50, 87, 46, 38, 101, 114, 49, 115, 23, 53, 63, 32, 112, 93, 47, 96, 26, 59, 119, 85, 21, 22, 109, 44, 57, 61, 34, 19, 83, 103, 43, 41, 16, 100, 111, 88, 24, 27, 80, 20, 91, 39, 98, 86, 25, 94, 18, 108, 82, 77, 89, 78, 107, 84, 48, 104, 81, 30, 40, 14, 17, 13, 95, 29, 79, 31, 15, 11, 75, 6, 9, 36, 35, 99, 72, 28, 33, 76, 92, 12, 73, 105, 74, 97, 10, 70, 7, 69, 68, 71, 8, 0, 1, 3, 4, 5, 64, 65, 67, 66, 2], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 117, 54, 60, 37, 55, 102, 45, 90, 51, 125, 87, 110, 50, 116, 123, 114, 101, 46, 61, 85, 63, 112, 23, 96, 93, 59, 38, 49, 22, 103, 47, 109, 111, 119, 53, 32, 34, 115, 41, 19, 21, 26, 27, 16, 83, 57, 20, 24, 77, 18, 25, 44, 86, 80, 39, 48, 98, 100, 30, 88, 43, 13, 89, 82, 14, 108, 78, 75, 81, 84, 31, 79, 94, 104, 29, 11, 6, 17, 73, 15, 95, 9, 35, 72, 105, 92, 76, 33, 91, 40, 12, 36, 107, 74, 99, 28, 8, 97, 10, 68, 7, 5, 71, 70, 4, 64, 66, 1, 65, 69, 0, 67, 3, 2], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 55, 60, 125, 110, 54, 90, 51, 102, 114, 50, 45, 123, 112, 87, 37, 115, 53, 85, 116, 93, 23, 119, 61, 109, 46, 63, 38, 22, 47, 49, 103, 101, 59, 83, 111, 19, 21, 96, 80, 57, 77, 26, 34, 86, 44, 27, 98, 16, 39, 32, 13, 82, 24, 14, 25, 20, 48, 100, 81, 18, 108, 17, 41, 84, 30, 88, 11, 9, 89, 75, 72, 79, 91, 43, 6, 78, 104, 29, 94, 15, 76, 73, 36, 40, 31, 107, 12, 92, 74, 10, 35, 8, 33, 99, 95, 70, 68, 28, 97, 105, 4, 5, 7, 71, 66, 0, 1, 64, 67, 69, 65, 3, 2], [122, 113, 62, 118, 120, 126, 127, 121, 56, 58, 52, 106, 124, 42, 117, 55, 110, 45, 102, 54, 90, 51, 125, 37, 46, 87, 50, 114, 115, 112, 60, 63, 61, 123, 38, 116, 53, 101, 49, 85, 23, 119, 47, 109, 22, 93, 83, 57, 26, 21, 59, 19, 96, 41, 111, 80, 44, 103, 34, 18, 32, 16, 24, 27, 77, 39, 43, 48, 100, 13, 25, 88, 20, 84, 17, 94, 98, 82, 30, 91, 86, 78, 89, 11, 14, 76, 104, 75, 29, 79, 81, 36, 107, 31, 15, 9, 95, 73, 99, 35, 40, 92, 108, 12, 6, 33, 72, 74, 10, 97, 28, 105, 68, 70, 8, 7, 64, 71, 66, 0, 67, 65, 4, 5, 69, 2, 1, 3], [122, 113, 62, 118, 126, 120, 56, 127, 121, 58, 52, 106, 124, 42, 54, 125, 117, 110, 37, 55, 50, 90, 102, 53, 60, 45, 123, 114, 51, 115, 87, 63, 38, 61, 112, 101, 49, 23, 109, 116, 57, 85, 59, 119, 111, 46, 26, 21, 96, 19, 93, 22, 34, 47, 83, 41, 48, 32, 103, 80, 16, 13, 98, 44, 27, 25, 77, 91, 24, 84, 86, 82, 30, 94, 29, 18, 20, 40, 39, 100, 107, 17, 81, 108, 78, 43, 31, 88, 11, 76, 14, 89, 75, 9, 79, 36, 35, 104, 95, 15, 73, 99, 12, 92, 10, 70, 8, 28, 97, 72, 74, 68, 33, 105, 7, 6, 64, 0, 71, 1, 65, 4, 5, 67, 69, 2, 66, 3], [122, 113, 62, 118, 120, 126, 56, 127, 121, 58, 106, 52, 124, 42, 117, 60, 54, 125, 90, 45, 87, 50, 37, 55, 110, 102, 51, 123, 114, 49, 61, 46, 63, 116, 38, 119, 93, 23, 115, 85, 22, 101, 96, 21, 59, 83, 109, 53, 112, 103, 26, 111, 80, 34, 57, 16, 25, 47, 19, 27, 32, 41, 13, 98, 77, 24, 18, 30, 20, 48, 108, 43, 44, 39, 104, 82, 17, 86, 88, 75, 100, 81, 14, 84, 29, 78, 31, 91, 11, 94, 12, 73, 15, 9, 107, 40, 70, 92, 79, 76, 89, 36, 35, 33, 95, 99, 74, 8, 72, 10, 105, 97, 68, 6, 28, 4, 71, 69, 7, 0, 5, 64, 1, 67, 65, 66, 2, 3], [122, 113, 62, 120, 118, 127, 126, 56, 121, 58, 52, 106, 124, 42, 55, 125, 117, 123, 102, 50, 116, 37, 60, 90, 45, 110, 87, 54, 51, 61, 63, 46, 53, 38, 115, 101, 23, 114, 112, 93, 57, 85, 119, 22, 96, 83, 49, 47, 26, 109, 103, 48, 21, 80, 59, 111, 32, 27, 19, 16, 25, 20, 44, 24, 18, 34, 88, 41, 86, 43, 13, 98, 82, 77, 91, 104, 100, 94, 17, 84, 39, 30, 11, 14, 81, 40, 9, 89, 75, 79, 15, 78, 70, 29, 12, 107, 76, 73, 35, 33, 31, 36, 95, 99, 74, 92, 97, 108, 10, 8, 72, 71, 28, 7, 4, 6, 5, 69, 68, 0, 105, 65, 64, 66, 3, 67, 1, 2], [122, 113, 62, 120, 118, 126, 127, 56, 121, 58, 106, 52, 124, 42, 117, 90, 50, 125, 102, 45, 60, 54, 55, 123, 87, 37, 23, 49, 63, 110, 46, 112, 116, 51, 47, 114, 38, 85, 109, 93, 101, 83, 119, 96, 115, 103, 59, 86, 22, 57, 16, 41, 32, 26, 111, 53, 21, 27, 20, 19, 61, 80, 34, 98, 13, 48, 82, 77, 100, 24, 25, 44, 14, 30, 75, 17, 18, 84, 11, 79, 9, 81, 43, 88, 12, 78, 39, 91, 94, 40, 89, 73, 31, 108, 15, 104, 70, 92, 8, 76, 36, 29, 10, 33, 107, 95, 35, 97, 74, 72, 99, 105, 6, 71, 7, 28, 68, 4, 1, 64, 5, 69, 2, 0, 65, 67, 66, 3], [122, 113, 62, 120, 118, 126, 127, 121, 56, 58, 52, 106, 124, 42, 125, 50, 54, 123, 90, 55, 60, 45, 110, 102, 37, 117, 114, 112, 87, 116, 53, 115, 38, 49, 23, 51, 101, 109, 57, 63, 93, 119, 19, 96, 85, 32, 47, 21, 59, 46, 41, 61, 22, 34, 83, 80, 26, 27, 94, 86, 111, 44, 103, 25, 24, 16, 43, 20, 91, 77, 39, 84, 13, 108, 18, 88, 40, 100, 30, 98, 31, 17, 82, 48, 104, 14, 29, 89, 81, 11, 95, 107, 15, 12, 36, 79, 73, 75, 78, 35, 9, 8, 76, 92, 99, 70, 10, 33, 28, 105, 97, 74, 7, 4, 6, 72, 71, 68, 65, 1, 0, 69, 5, 64, 67, 2, 66, 3], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 90, 60, 123, 117, 45, 37, 50, 125, 102, 55, 49, 114, 87, 110, 116, 112, 115, 93, 57, 23, 38, 53, 51, 109, 85, 83, 63, 119, 101, 46, 61, 21, 96, 19, 59, 80, 41, 16, 22, 20, 47, 98, 27, 34, 77, 82, 13, 32, 25, 26, 100, 103, 111, 44, 30, 24, 86, 81, 39, 18, 14, 48, 108, 43, 78, 73, 11, 94, 84, 75, 9, 17, 79, 91, 104, 12, 29, 15, 89, 88, 76, 8, 31, 36, 40, 92, 10, 74, 107, 70, 6, 71, 33, 35, 7, 0, 4, 97, 72, 68, 65, 64, 28, 105, 1, 99, 95, 2, 5, 69, 67, 3, 66], [122, 113, 62, 118, 120, 127, 126, 121, 56, 58, 52, 106, 124, 42, 55, 54, 102, 51, 123, 90, 37, 50, 125, 110, 45, 53, 87, 114, 117, 116, 63, 60, 38, 49, 61, 57, 115, 101, 85, 23, 112, 93, 103, 83, 96, 32, 47, 46, 109, 22, 111, 41, 34, 59, 26, 19, 24, 27, 77, 21, 80, 16, 20, 82, 98, 17, 94, 86, 39, 43, 88, 30, 48, 13, 44, 100, 18, 14, 119, 29, 25, 84, 91, 78, 89, 31, 75, 79, 11, 108, 9, 73, 104, 35, 81, 6, 12, 36, 40, 107, 95, 15, 92, 8, 76, 33, 97, 10, 99, 74, 70, 105, 71, 4, 68, 72, 28, 7, 1, 64, 2, 5, 69, 65, 0, 66, 67, 3], [122, 113, 62, 120, 118, 127, 56, 126, 121, 58, 52, 124, 106, 42, 54, 90, 102, 50, 37, 125, 55, 60, 117, 45, 110, 87, 51, 116, 123, 46, 114, 53, 49, 101, 59, 57, 38, 61, 85, 93, 112, 63, 23, 109, 83, 22, 96, 115, 47, 32, 103, 21, 80, 119, 26, 19, 77, 34, 13, 41, 44, 16, 25, 20, 14, 18, 98, 81, 27, 100, 24, 11, 30, 78, 48, 75, 86, 82, 111, 17, 108, 39, 43, 9, 79, 12, 91, 94, 84, 76, 6, 73, 8, 89, 15, 40, 104, 31, 29, 88, 36, 107, 10, 72, 33, 92, 99, 74, 35, 97, 95, 4, 71, 70, 28, 105, 68, 64, 7, 2, 65, 5, 1, 0, 3, 69, 66, 67], [122, 113, 62, 118, 120, 126, 127, 56, 121, 58, 52, 106, 124, 42, 54, 55, 90, 102, 45, 110, 37, 125, 117, 123, 87, 50, 51, 116, 57, 60, 63, 46, 53, 101, 49, 114, 23, 85, 38, 61, 93, 115, 83, 59, 112, 103, 109, 47, 32, 21, 22, 16, 96, 19, 20, 13, 41, 80, 24, 77, 34, 48, 25, 100, 82, 27, 111, 26, 86, 104, 88, 14, 17, 44, 43, 94, 119, 78, 18, 39, 11, 12, 30, 81, 98, 73, 84, 31, 108, 15, 89, 76, 75, 29, 79, 35, 6, 91, 40, 9, 92, 107, 36, 95, 8, 10, 99, 97, 68, 72, 74, 33, 71, 105, 7, 4, 5, 70, 0, 1, 64, 28, 65, 67, 66, 69, 3, 2], [122, 113, 62, 120, 118, 126, 56, 121, 127, 58, 52, 106, 124, 42, 54, 117, 45, 90, 50, 55, 125, 102, 63, 87, 60, 37, 116, 123, 51, 110, 114, 115, 85, 38, 53, 101, 93, 49, 23, 57, 61, 46, 83, 21, 59, 47, 109, 80, 119, 20, 13, 96, 22, 34, 77, 103, 111, 48, 19, 112, 25, 41, 82, 32, 26, 86, 27, 24, 16, 44, 43, 14, 30, 75, 100, 78, 89, 18, 11, 104, 98, 84, 88, 81, 17, 12, 9, 76, 73, 15, 39, 79, 94, 29, 6, 31, 40, 8, 91, 108, 105, 92, 107, 10, 72, 35, 71, 99, 95, 36, 74, 97, 70, 4, 33, 68, 69, 28, 7, 5, 64, 0, 1, 65, 66, 3, 2, 67]], "model.layers.11.self_attn.q_proj": [[45, 36, 96, 109, 92, 23, 86, 62, 48, 28, 14, 20, 10, 16, 61, 81, 71, 54, 56, 25, 89, 73, 18, 111, 22, 32, 70, 66, 4, 58, 47, 67, 125, 76, 17, 83, 88, 94, 42, 37, 38, 11, 12, 84, 29, 107, 30, 82, 78, 27, 80, 87, 95, 74, 21, 24, 19, 75, 13, 97, 64, 6, 1, 50, 9, 5, 102, 91, 15, 46, 77, 55, 85, 40, 116, 53, 41, 79, 43, 90, 49, 113, 115, 39, 122, 60, 51, 100, 3, 34, 98, 101, 69, 126, 112, 26, 123, 59, 93, 33, 114, 7, 108, 120, 105, 72, 119, 31, 63, 118, 103, 106, 99, 124, 35, 110, 44, 52, 8, 57, 127, 121, 2, 104, 117, 65, 0, 68], [45, 36, 109, 96, 62, 92, 28, 25, 54, 86, 48, 4, 89, 81, 7, 3, 47, 102, 23, 100, 40, 20, 32, 43, 14, 22, 1, 111, 8, 38, 58, 10, 101, 66, 46, 12, 116, 41, 50, 125, 107, 55, 64, 18, 114, 56, 126, 122, 51, 49, 74, 120, 42, 5, 57, 97, 112, 59, 118, 127, 123, 19, 39, 108, 113, 37, 115, 105, 99, 53, 63, 31, 104, 0, 52, 121, 35, 29, 61, 119, 124, 34, 106, 44, 110, 60, 33, 117, 30, 11, 98, 103, 95, 17, 9, 68, 91, 93, 90, 70, 84, 65, 75, 27, 24, 80, 94, 26, 15, 85, 72, 88, 78, 21, 82, 2, 6, 73, 16, 69, 79, 76, 87, 83, 71, 77, 67, 13], [45, 36, 109, 48, 100, 25, 86, 96, 92, 125, 111, 58, 54, 56, 22, 50, 107, 46, 43, 113, 55, 47, 41, 122, 23, 116, 115, 114, 120, 101, 32, 119, 112, 53, 118, 57, 28, 62, 60, 117, 38, 127, 121, 40, 59, 97, 49, 51, 20, 63, 42, 39, 110, 126, 102, 52, 124, 44, 123, 108, 10, 81, 103, 105, 14, 106, 4, 7, 61, 88, 104, 89, 15, 37, 24, 83, 31, 99, 98, 33, 35, 34, 91, 78, 30, 12, 19, 93, 29, 3, 95, 74, 84, 85, 21, 94, 90, 66, 17, 80, 18, 16, 87, 1, 8, 9, 27, 75, 26, 82, 11, 76, 70, 79, 65, 64, 0, 73, 68, 72, 5, 13, 77, 71, 6, 69, 2, 67], [45, 36, 96, 109, 25, 28, 62, 92, 86, 48, 23, 20, 54, 89, 14, 81, 111, 61, 56, 58, 10, 18, 97, 107, 47, 32, 41, 22, 46, 125, 12, 102, 80, 100, 43, 30, 16, 4, 50, 42, 113, 53, 40, 38, 39, 9, 115, 101, 37, 66, 1, 55, 64, 71, 31, 122, 3, 17, 44, 114, 126, 105, 91, 116, 21, 108, 5, 87, 70, 51, 93, 112, 94, 8, 34, 29, 60, 49, 57, 110, 90, 83, 95, 7, 85, 121, 19, 118, 98, 120, 84, 78, 127, 24, 52, 123, 99, 88, 106, 35, 15, 76, 124, 26, 27, 104, 119, 33, 59, 79, 103, 11, 117, 63, 82, 75, 72, 74, 73, 0, 77, 68, 65, 69, 6, 13, 2, 67], [58, 123, 63, 37, 117, 26, 87, 33, 88, 95, 86, 76, 17, 20, 78, 80, 83, 122, 81, 91, 126, 118, 30, 92, 0, 85, 12, 124, 67, 5, 9, 97, 82, 69, 79, 24, 90, 93, 1, 60, 70, 22, 18, 29, 14, 2, 23, 3, 25, 96, 27, 21, 61, 71, 16, 45, 120, 84, 50, 73, 75, 125, 77, 94, 72, 15, 32, 28, 109, 31, 19, 115, 89, 62, 11, 52, 56, 13, 54, 8, 113, 55, 127, 10, 59, 49, 112, 7, 57, 74, 47, 66, 100, 46, 68, 116, 111, 121, 6, 48, 103, 53, 64, 34, 106, 114, 38, 110, 40, 35, 98, 101, 65, 36, 119, 107, 105, 42, 102, 39, 51, 43, 104, 108, 41, 44, 99, 4], [123, 63, 58, 37, 117, 122, 126, 118, 60, 62, 33, 112, 115, 124, 61, 50, 52, 125, 54, 59, 30, 56, 45, 49, 101, 48, 47, 114, 119, 116, 57, 113, 55, 127, 39, 111, 99, 53, 110, 109, 40, 105, 121, 100, 106, 108, 43, 51, 46, 44, 104, 42, 120, 41, 10, 103, 102, 16, 88, 107, 34, 38, 86, 35, 21, 96, 36, 92, 32, 98, 97, 84, 95, 26, 18, 77, 93, 19, 28, 8, 29, 91, 94, 83, 14, 24, 22, 31, 65, 13, 68, 89, 15, 7, 27, 12, 17, 25, 23, 87, 82, 64, 70, 67, 90, 85, 75, 80, 79, 5, 4, 73, 78, 66, 71, 20, 72, 9, 69, 11, 81, 1, 76, 74, 3, 2, 0, 6], [63, 123, 58, 117, 60, 126, 118, 124, 122, 10, 37, 50, 62, 115, 54, 61, 68, 125, 52, 59, 65, 45, 112, 116, 56, 113, 8, 49, 127, 13, 16, 121, 48, 114, 55, 47, 7, 64, 57, 111, 53, 46, 40, 67, 21, 15, 14, 119, 17, 51, 120, 84, 12, 109, 75, 78, 110, 43, 99, 77, 100, 70, 73, 108, 5, 104, 44, 18, 105, 42, 106, 33, 39, 107, 102, 4, 41, 92, 103, 19, 101, 38, 1, 30, 66, 96, 24, 86, 9, 93, 95, 36, 88, 0, 28, 97, 34, 35, 87, 22, 80, 98, 83, 29, 79, 32, 31, 69, 91, 26, 25, 71, 76, 81, 27, 20, 23, 89, 72, 94, 74, 85, 90, 11, 82, 3, 2, 6], [58, 123, 63, 37, 122, 117, 126, 118, 60, 88, 33, 10, 112, 115, 61, 54, 62, 124, 50, 125, 52, 101, 30, 49, 59, 45, 16, 56, 55, 116, 48, 39, 111, 40, 113, 127, 114, 57, 93, 47, 8, 106, 53, 110, 100, 119, 120, 28, 92, 18, 51, 77, 19, 84, 86, 109, 108, 121, 46, 65, 21, 96, 26, 14, 68, 43, 104, 12, 44, 102, 105, 99, 42, 13, 91, 15, 41, 103, 24, 38, 95, 35, 32, 107, 97, 34, 83, 29, 36, 7, 5, 17, 98, 70, 31, 64, 75, 67, 25, 73, 22, 94, 89, 78, 66, 27, 82, 87, 90, 20, 80, 79, 71, 9, 4, 76, 23, 85, 1, 81, 72, 2, 11, 74, 0, 69, 3, 6], [102, 46, 110, 19, 91, 88, 125, 86, 114, 113, 95, 43, 111, 122, 27, 12, 10, 50, 63, 16, 18, 60, 52, 31, 107, 56, 17, 42, 54, 45, 123, 93, 71, 126, 119, 55, 69, 120, 118, 76, 83, 99, 44, 116, 41, 48, 121, 59, 51, 108, 58, 112, 47, 124, 105, 61, 57, 109, 36, 70, 53, 100, 39, 24, 104, 49, 34, 117, 4, 103, 127, 115, 67, 30, 106, 40, 62, 97, 23, 94, 101, 33, 22, 35, 9, 32, 80, 72, 82, 96, 37, 29, 89, 79, 98, 28, 14, 81, 20, 25, 87, 26, 90, 21, 11, 92, 74, 85, 65, 84, 78, 15, 38, 66, 13, 75, 77, 2, 73, 8, 0, 6, 5, 68, 7, 3, 1, 64], [102, 46, 110, 91, 86, 88, 125, 19, 16, 95, 113, 27, 69, 76, 122, 111, 10, 12, 71, 70, 72, 67, 43, 4, 85, 18, 93, 82, 114, 66, 9, 31, 24, 58, 90, 13, 17, 14, 117, 83, 23, 100, 22, 49, 121, 79, 65, 116, 74, 99, 63, 28, 124, 45, 29, 120, 80, 30, 41, 42, 54, 44, 81, 123, 56, 39, 78, 57, 60, 35, 126, 8, 52, 92, 2, 11, 112, 25, 53, 94, 103, 75, 47, 40, 59, 119, 61, 38, 36, 104, 107, 51, 127, 21, 55, 84, 115, 118, 32, 98, 0, 96, 101, 62, 108, 97, 87, 89, 33, 15, 26, 48, 50, 105, 37, 34, 106, 20, 109, 5, 6, 73, 77, 7, 68, 3, 64, 1], [102, 46, 110, 19, 27, 86, 88, 122, 95, 1, 78, 3, 7, 74, 76, 67, 91, 71, 24, 64, 65, 16, 5, 93, 4, 73, 125, 22, 18, 114, 9, 14, 68, 69, 111, 0, 82, 12, 11, 8, 31, 72, 113, 2, 70, 79, 99, 10, 66, 83, 43, 6, 75, 77, 45, 80, 17, 124, 121, 112, 40, 21, 20, 58, 116, 120, 42, 56, 63, 119, 50, 13, 123, 25, 57, 23, 47, 89, 15, 33, 81, 44, 104, 54, 90, 32, 85, 107, 103, 87, 34, 29, 118, 28, 105, 51, 36, 39, 97, 96, 38, 100, 101, 126, 94, 49, 84, 26, 92, 52, 117, 106, 62, 59, 108, 41, 30, 37, 35, 61, 115, 109, 60, 48, 98, 53, 127, 55], [102, 46, 110, 91, 88, 86, 19, 16, 95, 27, 18, 125, 122, 76, 10, 93, 31, 113, 114, 24, 111, 74, 71, 12, 69, 78, 70, 22, 72, 14, 83, 43, 4, 17, 82, 30, 67, 79, 80, 94, 60, 99, 29, 15, 58, 9, 75, 120, 23, 100, 39, 21, 66, 6, 13, 89, 90, 49, 38, 11, 116, 33, 42, 52, 126, 32, 81, 123, 36, 85, 107, 97, 65, 28, 59, 61, 41, 103, 92, 45, 73, 112, 54, 84, 50, 63, 87, 25, 53, 51, 56, 35, 44, 26, 101, 119, 48, 77, 37, 0, 121, 55, 2, 34, 106, 124, 127, 117, 109, 98, 57, 104, 115, 20, 40, 105, 62, 96, 47, 108, 118, 5, 8, 7, 68, 3, 64, 1], [63, 59, 106, 36, 14, 42, 114, 121, 60, 15, 126, 117, 47, 12, 97, 48, 52, 124, 123, 100, 120, 11, 82, 125, 122, 109, 113, 84, 115, 49, 56, 116, 58, 16, 53, 119, 62, 110, 33, 54, 44, 111, 51, 85, 61, 24, 57, 45, 55, 46, 118, 83, 127, 108, 112, 107, 43, 104, 50, 105, 72, 0, 41, 92, 17, 103, 40, 39, 65, 102, 37, 69, 38, 99, 81, 10, 89, 101, 67, 91, 34, 98, 6, 9, 7, 21, 35, 18, 73, 88, 93, 22, 68, 31, 86, 13, 29, 2, 96, 32, 94, 64, 95, 19, 27, 66, 25, 30, 28, 23, 1, 26, 4, 90, 8, 20, 75, 87, 76, 70, 77, 80, 78, 79, 5, 3, 71, 74], [59, 63, 100, 97, 42, 36, 60, 121, 93, 87, 79, 88, 73, 27, 76, 20, 117, 33, 47, 52, 82, 15, 106, 114, 92, 78, 75, 12, 124, 68, 86, 6, 116, 125, 14, 25, 85, 8, 67, 105, 126, 81, 18, 83, 104, 80, 11, 28, 48, 95, 29, 69, 56, 26, 31, 110, 7, 19, 65, 2, 90, 72, 91, 22, 10, 84, 24, 17, 123, 45, 120, 34, 115, 38, 98, 96, 89, 113, 37, 99, 16, 109, 111, 23, 53, 107, 103, 49, 102, 35, 46, 74, 21, 127, 30, 94, 62, 122, 108, 77, 39, 13, 32, 50, 101, 112, 119, 41, 58, 54, 44, 70, 40, 43, 64, 71, 51, 57, 9, 55, 5, 61, 118, 0, 3, 66, 4, 1], [59, 63, 100, 97, 14, 68, 36, 6, 76, 42, 60, 73, 7, 12, 10, 67, 69, 72, 8, 11, 87, 88, 121, 2, 93, 82, 27, 65, 25, 52, 33, 20, 13, 17, 117, 81, 16, 15, 47, 78, 106, 79, 70, 74, 29, 83, 21, 80, 86, 22, 95, 77, 116, 126, 85, 48, 71, 105, 114, 124, 123, 91, 89, 26, 9, 64, 5, 84, 125, 23, 115, 110, 19, 90, 56, 75, 104, 31, 92, 24, 49, 3, 113, 122, 120, 109, 18, 127, 111, 119, 45, 46, 107, 98, 53, 28, 58, 38, 35, 54, 37, 44, 51, 55, 62, 94, 103, 41, 50, 96, 34, 61, 39, 40, 57, 43, 108, 0, 112, 118, 102, 4, 32, 99, 30, 101, 66, 1], [59, 63, 100, 97, 36, 42, 121, 106, 60, 114, 93, 27, 73, 20, 117, 33, 78, 76, 52, 12, 125, 123, 88, 126, 86, 48, 124, 116, 120, 87, 6, 47, 113, 110, 122, 99, 67, 82, 58, 68, 111, 51, 115, 79, 25, 8, 109, 49, 53, 54, 62, 37, 119, 118, 55, 104, 56, 112, 46, 45, 69, 81, 98, 14, 92, 18, 57, 105, 44, 29, 61, 7, 127, 108, 10, 102, 2, 50, 15, 31, 107, 80, 22, 65, 43, 84, 35, 85, 39, 90, 41, 34, 83, 91, 75, 17, 103, 38, 95, 96, 94, 32, 28, 101, 24, 26, 40, 30, 89, 23, 11, 19, 5, 72, 74, 13, 9, 16, 64, 21, 0, 71, 77, 70, 66, 3, 4, 1], [101, 121, 116, 94, 57, 17, 71, 19, 2, 52, 78, 21, 3, 5, 75, 24, 64, 88, 9, 66, 76, 69, 124, 30, 1, 7, 80, 6, 118, 18, 115, 127, 117, 0, 67, 53, 11, 63, 70, 37, 120, 50, 125, 14, 33, 87, 72, 73, 29, 60, 83, 105, 97, 12, 82, 104, 54, 61, 79, 91, 59, 81, 16, 74, 122, 20, 4, 96, 43, 93, 46, 51, 113, 22, 106, 112, 77, 110, 65, 27, 40, 86, 15, 85, 111, 45, 13, 58, 126, 25, 26, 23, 55, 41, 84, 39, 48, 42, 89, 107, 102, 28, 31, 68, 10, 90, 62, 8, 92, 103, 123, 114, 49, 108, 99, 44, 56, 35, 100, 109, 34, 47, 38, 95, 32, 98, 119, 36], [101, 121, 116, 94, 57, 24, 21, 52, 19, 17, 78, 76, 80, 30, 88, 9, 75, 5, 37, 7, 15, 71, 11, 18, 69, 97, 29, 67, 33, 124, 63, 118, 66, 82, 117, 53, 127, 3, 73, 16, 115, 105, 22, 0, 120, 50, 60, 46, 86, 6, 81, 14, 2, 72, 1, 28, 26, 54, 83, 106, 20, 45, 51, 59, 43, 79, 122, 12, 85, 89, 91, 74, 77, 90, 39, 58, 104, 87, 125, 42, 13, 114, 41, 70, 110, 113, 112, 23, 61, 25, 96, 10, 109, 92, 93, 56, 95, 108, 31, 84, 49, 123, 32, 40, 102, 107, 62, 48, 126, 98, 4, 8, 99, 27, 103, 65, 111, 100, 35, 47, 38, 34, 55, 64, 119, 44, 36, 68], [101, 116, 121, 94, 52, 37, 30, 21, 57, 24, 19, 85, 80, 33, 117, 115, 105, 16, 50, 127, 124, 114, 62, 113, 26, 104, 59, 53, 125, 58, 118, 110, 60, 43, 51, 39, 49, 95, 63, 97, 107, 123, 54, 120, 55, 41, 45, 61, 91, 76, 17, 102, 46, 48, 42, 112, 56, 111, 28, 126, 122, 109, 38, 108, 103, 89, 119, 78, 36, 32, 73, 40, 22, 93, 99, 47, 44, 100, 106, 20, 35, 23, 27, 96, 15, 12, 18, 98, 87, 34, 92, 31, 86, 29, 88, 83, 79, 90, 25, 81, 82, 7, 9, 84, 14, 74, 5, 11, 66, 70, 72, 8, 75, 10, 13, 67, 77, 4, 6, 69, 2, 71, 3, 64, 68, 65, 0, 1], [101, 116, 121, 94, 57, 21, 24, 19, 76, 52, 17, 80, 78, 30, 6, 75, 9, 3, 33, 88, 71, 37, 124, 2, 127, 67, 29, 117, 97, 60, 115, 69, 83, 118, 46, 7, 18, 53, 64, 63, 50, 1, 23, 0, 16, 54, 90, 22, 106, 73, 70, 82, 14, 68, 12, 77, 26, 81, 122, 72, 112, 85, 61, 43, 13, 93, 58, 59, 104, 86, 98, 27, 79, 120, 42, 105, 20, 11, 74, 96, 84, 28, 39, 35, 125, 40, 65, 91, 25, 110, 123, 15, 107, 95, 41, 10, 92, 111, 32, 51, 99, 89, 87, 109, 103, 31, 48, 45, 62, 36, 126, 34, 113, 38, 44, 4, 55, 114, 49, 47, 56, 8, 100, 119, 102, 108, 5, 66], [40, 126, 97, 25, 89, 95, 87, 86, 28, 92, 121, 84, 15, 83, 115, 63, 82, 122, 55, 17, 49, 44, 53, 37, 119, 62, 29, 50, 38, 104, 45, 111, 91, 107, 61, 101, 46, 43, 41, 35, 9, 114, 123, 120, 102, 118, 23, 52, 117, 30, 54, 58, 109, 60, 12, 124, 48, 75, 57, 13, 42, 47, 116, 36, 103, 32, 98, 20, 34, 56, 105, 18, 85, 39, 51, 127, 110, 125, 90, 106, 16, 67, 112, 21, 59, 99, 1, 31, 96, 108, 113, 5, 80, 27, 74, 93, 79, 22, 100, 94, 71, 88, 11, 24, 0, 19, 26, 68, 77, 33, 73, 81, 6, 72, 70, 64, 14, 4, 66, 78, 7, 2, 69, 76, 65, 10, 8, 3], [40, 126, 97, 95, 89, 92, 86, 25, 120, 61, 15, 122, 28, 87, 82, 52, 83, 62, 17, 53, 84, 9, 60, 29, 121, 45, 63, 59, 12, 58, 118, 16, 107, 54, 21, 123, 114, 116, 85, 49, 47, 13, 77, 23, 117, 30, 8, 55, 33, 20, 115, 46, 127, 125, 50, 18, 79, 119, 36, 27, 56, 22, 57, 111, 80, 112, 90, 124, 44, 102, 110, 113, 106, 43, 93, 101, 91, 31, 41, 108, 109, 48, 38, 72, 24, 51, 35, 81, 68, 104, 4, 70, 19, 103, 71, 75, 100, 88, 96, 105, 37, 76, 6, 42, 2, 98, 39, 11, 78, 34, 26, 73, 14, 32, 99, 69, 66, 10, 94, 74, 1, 5, 0, 67, 64, 7, 3, 65], [126, 40, 97, 95, 86, 121, 101, 28, 82, 122, 62, 58, 120, 25, 53, 115, 45, 61, 102, 111, 84, 52, 54, 44, 114, 127, 60, 49, 92, 59, 118, 56, 63, 125, 50, 83, 46, 37, 112, 47, 113, 87, 89, 15, 43, 123, 57, 117, 51, 119, 55, 100, 29, 18, 12, 9, 116, 33, 108, 68, 30, 21, 109, 35, 48, 107, 110, 27, 73, 71, 91, 66, 38, 69, 80, 36, 93, 78, 124, 106, 19, 72, 10, 103, 99, 41, 42, 76, 14, 79, 65, 24, 90, 77, 23, 85, 16, 17, 31, 2, 13, 20, 105, 104, 5, 7, 34, 26, 3, 67, 64, 39, 98, 11, 1, 32, 94, 22, 75, 96, 88, 70, 74, 0, 8, 4, 81, 6], [126, 40, 97, 121, 61, 122, 71, 67, 49, 62, 89, 1, 54, 101, 60, 25, 53, 52, 59, 5, 45, 43, 95, 127, 63, 58, 114, 125, 120, 92, 28, 47, 56, 68, 50, 12, 46, 0, 86, 118, 84, 102, 111, 55, 57, 112, 82, 115, 117, 116, 123, 113, 66, 88, 108, 64, 81, 15, 73, 44, 51, 11, 109, 107, 77, 105, 119, 110, 106, 100, 30, 87, 24, 48, 42, 38, 19, 9, 41, 76, 14, 124, 17, 69, 2, 34, 36, 18, 27, 6, 4, 20, 78, 94, 35, 13, 72, 32, 39, 23, 103, 37, 99, 98, 16, 91, 85, 80, 3, 7, 74, 29, 83, 70, 104, 75, 21, 22, 93, 65, 10, 96, 31, 90, 26, 33, 79, 8], [104, 111, 30, 119, 26, 19, 127, 24, 94, 21, 79, 62, 9, 113, 61, 123, 12, 40, 81, 32, 86, 122, 47, 87, 118, 63, 78, 114, 54, 83, 68, 36, 49, 110, 89, 88, 126, 107, 90, 85, 100, 105, 115, 74, 56, 96, 71, 43, 4, 18, 70, 15, 57, 46, 17, 76, 125, 55, 121, 51, 120, 106, 42, 23, 37, 80, 117, 73, 3, 14, 97, 20, 112, 1, 103, 53, 92, 69, 45, 50, 5, 95, 101, 59, 66, 48, 38, 98, 39, 41, 22, 124, 102, 93, 28, 91, 108, 116, 10, 75, 52, 82, 58, 33, 99, 67, 77, 64, 35, 84, 8, 34, 31, 16, 60, 29, 44, 13, 2, 109, 27, 72, 11, 7, 25, 0, 65, 6], [104, 111, 119, 30, 62, 127, 24, 19, 21, 26, 94, 79, 12, 113, 47, 9, 123, 81, 122, 40, 32, 54, 70, 1, 61, 96, 86, 87, 49, 90, 118, 4, 91, 107, 67, 110, 15, 63, 20, 57, 120, 82, 105, 23, 64, 68, 43, 126, 85, 124, 103, 22, 56, 114, 45, 121, 36, 53, 46, 42, 55, 78, 58, 52, 112, 100, 109, 117, 18, 88, 89, 108, 71, 59, 98, 101, 2, 17, 115, 28, 83, 106, 74, 13, 41, 37, 34, 80, 27, 48, 66, 5, 50, 116, 125, 51, 75, 95, 60, 38, 84, 76, 72, 93, 8, 39, 99, 44, 73, 35, 102, 33, 3, 29, 16, 6, 10, 7, 0, 14, 31, 25, 97, 92, 11, 77, 69, 65], [104, 111, 119, 30, 127, 24, 62, 26, 94, 19, 21, 79, 113, 9, 61, 32, 122, 107, 12, 81, 86, 87, 54, 123, 40, 63, 47, 36, 105, 49, 70, 103, 117, 43, 114, 110, 56, 118, 100, 57, 126, 41, 83, 108, 85, 89, 96, 120, 90, 46, 37, 53, 28, 45, 95, 68, 88, 48, 42, 74, 115, 51, 112, 121, 39, 101, 38, 55, 1, 18, 125, 15, 4, 20, 78, 58, 76, 23, 17, 35, 91, 99, 59, 13, 98, 116, 52, 106, 44, 124, 93, 109, 50, 34, 102, 29, 97, 71, 67, 73, 64, 60, 82, 31, 33, 92, 72, 22, 14, 77, 80, 27, 66, 25, 3, 75, 84, 10, 5, 16, 2, 11, 8, 69, 0, 6, 7, 65], [104, 111, 30, 119, 24, 26, 19, 94, 79, 127, 81, 12, 61, 9, 21, 62, 32, 113, 78, 40, 118, 87, 90, 47, 96, 91, 54, 22, 123, 88, 100, 36, 4, 23, 85, 86, 18, 124, 15, 89, 107, 74, 110, 80, 63, 95, 72, 17, 53, 43, 122, 57, 83, 126, 98, 70, 120, 114, 45, 50, 49, 56, 20, 42, 37, 58, 125, 101, 34, 102, 105, 46, 112, 55, 14, 103, 35, 48, 16, 84, 7, 29, 76, 115, 97, 93, 25, 31, 99, 5, 82, 27, 51, 75, 41, 117, 109, 39, 59, 121, 38, 28, 44, 33, 92, 71, 2, 106, 52, 116, 108, 11, 60, 10, 6, 13, 68, 67, 1, 8, 77, 73, 64, 66, 69, 3, 0, 65], [59, 119, 46, 104, 51, 113, 61, 48, 56, 115, 98, 120, 116, 109, 57, 52, 49, 55, 53, 124, 60, 126, 63, 42, 112, 117, 122, 110, 50, 54, 118, 58, 111, 45, 47, 107, 121, 62, 106, 114, 123, 37, 125, 105, 41, 43, 127, 44, 39, 108, 36, 101, 38, 30, 24, 103, 33, 93, 86, 96, 89, 40, 35, 92, 102, 26, 88, 32, 95, 90, 97, 99, 100, 29, 27, 34, 80, 28, 91, 25, 20, 94, 31, 22, 87, 84, 16, 17, 18, 82, 85, 75, 13, 77, 23, 19, 21, 11, 68, 5, 83, 8, 76, 3, 14, 78, 9, 7, 81, 6, 79, 73, 2, 74, 0, 4, 1, 10, 66, 64, 15, 72, 12, 69, 65, 67, 70, 71], [46, 59, 104, 51, 53, 98, 117, 93, 91, 113, 48, 119, 49, 36, 37, 26, 114, 121, 55, 89, 61, 127, 110, 41, 30, 33, 120, 23, 57, 112, 109, 97, 54, 90, 115, 38, 107, 50, 63, 44, 83, 122, 56, 111, 94, 78, 84, 95, 106, 82, 52, 125, 118, 108, 45, 60, 99, 123, 116, 39, 102, 62, 43, 58, 27, 101, 100, 103, 105, 47, 32, 34, 35, 124, 25, 126, 96, 92, 21, 28, 18, 87, 42, 88, 80, 24, 75, 85, 22, 31, 40, 86, 81, 29, 15, 19, 13, 17, 20, 16, 77, 11, 14, 72, 79, 66, 9, 76, 74, 68, 8, 4, 73, 2, 64, 0, 70, 6, 3, 12, 5, 65, 1, 69, 71, 10, 67, 7], [46, 59, 48, 104, 51, 119, 61, 117, 113, 123, 98, 53, 55, 57, 56, 109, 126, 50, 114, 120, 37, 116, 63, 52, 93, 115, 60, 112, 49, 124, 122, 127, 42, 54, 110, 121, 118, 106, 44, 58, 45, 47, 41, 105, 111, 125, 43, 38, 62, 32, 29, 85, 89, 108, 36, 103, 107, 35, 31, 92, 39, 90, 95, 33, 84, 102, 91, 86, 97, 101, 96, 80, 26, 100, 99, 21, 28, 82, 30, 40, 25, 88, 34, 23, 20, 27, 24, 78, 68, 18, 22, 94, 75, 87, 16, 17, 11, 3, 77, 2, 83, 8, 14, 13, 0, 72, 5, 4, 65, 6, 9, 19, 73, 81, 74, 1, 79, 66, 70, 15, 10, 7, 76, 69, 67, 64, 71, 12], [104, 59, 98, 46, 83, 79, 23, 21, 81, 37, 76, 74, 48, 91, 8, 5, 69, 12, 7, 72, 78, 119, 71, 2, 3, 64, 66, 110, 67, 15, 84, 40, 19, 26, 25, 62, 10, 85, 14, 16, 27, 17, 18, 89, 87, 93, 13, 75, 31, 90, 20, 1, 51, 30, 94, 0, 73, 88, 77, 22, 86, 11, 29, 24, 70, 65, 92, 82, 9, 80, 4, 53, 6, 28, 32, 68, 96, 33, 95, 52, 61, 100, 108, 34, 99, 116, 115, 112, 39, 35, 102, 106, 56, 120, 54, 97, 36, 55, 101, 103, 113, 45, 111, 117, 50, 44, 38, 118, 124, 127, 122, 123, 109, 126, 58, 49, 121, 105, 60, 41, 42, 43, 47, 114, 57, 63, 107, 125]], "model.layers.11.self_attn.k_proj": [[109, 45, 32, 100, 54, 48, 92, 25, 56, 20, 23, 81, 47, 14, 62, 58, 107, 22, 16, 61, 112, 76, 85, 18, 69, 0, 68, 65, 120, 125, 86, 30, 55, 10, 73, 31, 122, 42, 59, 12, 7, 50, 71, 123, 72, 111, 118, 51, 2, 57, 43, 46, 126, 116, 38, 19, 3, 53, 113, 63, 40, 37, 13, 114, 102, 41, 124, 60, 108, 95, 119, 115, 93, 104, 127, 117, 44, 110, 39, 121, 49, 80, 52, 94, 11, 33, 101, 106, 97, 75, 99, 90, 91, 87, 96, 98, 17, 35, 105, 15, 28, 74, 77, 6, 103, 34, 29, 67, 27, 26, 24, 84, 5, 66, 21, 83, 88, 79, 36, 9, 82, 8, 64, 89, 78, 1, 70, 4], [58, 123, 63, 101, 126, 118, 117, 31, 61, 60, 50, 124, 86, 125, 52, 91, 62, 115, 20, 17, 121, 97, 55, 47, 56, 113, 49, 120, 111, 127, 112, 122, 59, 57, 48, 54, 53, 116, 45, 26, 119, 114, 78, 46, 76, 109, 51, 110, 108, 88, 73, 43, 75, 107, 44, 106, 32, 40, 103, 66, 42, 69, 37, 104, 41, 80, 105, 82, 39, 102, 99, 79, 36, 38, 89, 100, 70, 98, 93, 87, 96, 30, 34, 35, 33, 92, 19, 72, 27, 94, 25, 21, 74, 83, 29, 67, 13, 28, 64, 65, 5, 81, 95, 6, 90, 23, 71, 24, 11, 7, 16, 4, 9, 18, 85, 12, 8, 15, 10, 22, 84, 77, 1, 68, 0, 3, 14, 2], [110, 38, 46, 86, 91, 78, 88, 125, 18, 122, 19, 16, 31, 76, 111, 7, 114, 73, 3, 74, 1, 43, 5, 113, 64, 45, 121, 58, 49, 47, 120, 93, 56, 124, 6, 9, 42, 68, 17, 40, 15, 116, 119, 57, 63, 72, 62, 50, 13, 44, 123, 112, 99, 54, 103, 67, 35, 27, 118, 61, 59, 75, 11, 105, 41, 36, 2, 126, 108, 101, 107, 96, 90, 30, 70, 51, 55, 20, 106, 115, 82, 104, 8, 127, 21, 48, 60, 109, 100, 52, 117, 81, 87, 29, 34, 26, 94, 97, 37, 89, 79, 98, 53, 32, 66, 33, 24, 85, 39, 92, 95, 10, 28, 22, 80, 65, 25, 23, 84, 69, 14, 12, 71, 83, 0, 77, 4, 102], [59, 63, 36, 33, 121, 87, 117, 124, 93, 19, 106, 25, 60, 126, 95, 114, 81, 27, 26, 56, 125, 20, 116, 113, 47, 48, 42, 21, 53, 119, 52, 110, 86, 49, 80, 54, 123, 46, 16, 79, 58, 77, 111, 10, 98, 104, 100, 115, 122, 18, 45, 120, 55, 88, 78, 39, 51, 50, 7, 101, 40, 13, 118, 76, 41, 112, 108, 127, 105, 61, 35, 62, 109, 22, 102, 44, 34, 9, 43, 57, 38, 4, 107, 103, 69, 72, 28, 99, 37, 70, 14, 32, 74, 85, 92, 75, 3, 11, 0, 94, 96, 91, 30, 31, 73, 15, 97, 66, 17, 67, 89, 82, 71, 90, 24, 83, 6, 29, 65, 1, 23, 68, 12, 84, 2, 8, 64, 5], [121, 116, 37, 21, 24, 80, 78, 9, 75, 19, 17, 30, 57, 76, 64, 71, 69, 6, 2, 3, 97, 46, 53, 94, 124, 66, 127, 118, 65, 60, 74, 1, 59, 120, 115, 5, 63, 70, 43, 4, 122, 125, 79, 41, 93, 82, 73, 40, 50, 13, 91, 117, 42, 8, 104, 113, 88, 61, 7, 101, 0, 90, 67, 89, 33, 112, 15, 58, 92, 12, 28, 110, 107, 45, 87, 114, 34, 27, 86, 32, 108, 111, 22, 96, 99, 26, 109, 126, 55, 54, 23, 39, 31, 68, 51, 11, 10, 48, 105, 49, 84, 47, 123, 18, 72, 62, 106, 35, 29, 38, 25, 100, 20, 95, 102, 119, 44, 56, 98, 103, 14, 77, 36, 52, 16, 83, 85, 81], [126, 104, 33, 86, 28, 122, 121, 31, 62, 54, 118, 25, 60, 58, 17, 61, 53, 115, 87, 52, 127, 120, 50, 15, 63, 56, 114, 47, 49, 125, 108, 84, 113, 112, 3, 59, 100, 57, 64, 45, 46, 65, 37, 9, 82, 111, 42, 12, 109, 110, 117, 43, 75, 92, 123, 55, 116, 7, 70, 69, 107, 48, 51, 99, 106, 13, 29, 41, 83, 16, 124, 85, 105, 44, 0, 38, 102, 74, 119, 1, 39, 91, 94, 36, 8, 103, 66, 68, 19, 78, 89, 2, 4, 26, 34, 98, 101, 88, 32, 27, 96, 93, 11, 35, 81, 71, 95, 21, 90, 10, 24, 30, 6, 80, 14, 18, 22, 97, 72, 77, 23, 67, 79, 76, 20, 73, 5, 40], [40, 111, 47, 119, 94, 24, 127, 113, 26, 21, 62, 19, 110, 81, 122, 78, 114, 49, 123, 79, 118, 56, 63, 32, 12, 55, 120, 121, 105, 57, 61, 58, 126, 115, 43, 108, 54, 59, 42, 107, 87, 106, 9, 46, 18, 45, 125, 124, 104, 112, 48, 69, 116, 117, 51, 53, 37, 7, 39, 11, 41, 100, 60, 52, 109, 50, 38, 16, 36, 80, 44, 86, 73, 96, 28, 103, 99, 95, 102, 3, 101, 33, 91, 10, 30, 93, 89, 0, 97, 65, 71, 4, 35, 90, 13, 75, 76, 98, 23, 20, 27, 34, 82, 92, 31, 29, 15, 68, 22, 88, 25, 8, 84, 17, 74, 2, 14, 77, 66, 83, 85, 6, 72, 64, 70, 5, 1, 67], [40, 59, 46, 34, 110, 74, 76, 79, 7, 83, 81, 6, 21, 8, 5, 78, 2, 23, 93, 3, 0, 48, 1, 119, 9, 62, 27, 91, 117, 52, 115, 116, 77, 30, 45, 4, 89, 57, 49, 90, 64, 84, 56, 113, 126, 58, 122, 55, 120, 51, 47, 123, 118, 65, 108, 54, 109, 60, 42, 121, 124, 127, 112, 53, 61, 92, 63, 43, 125, 50, 44, 68, 37, 101, 38, 80, 111, 107, 15, 97, 72, 70, 88, 75, 114, 82, 87, 73, 105, 31, 41, 36, 102, 22, 18, 33, 95, 66, 86, 28, 103, 99, 106, 12, 39, 96, 104, 85, 11, 13, 94, 35, 67, 98, 32, 25, 29, 100, 24, 71, 20, 17, 19, 16, 26, 14, 10, 69]], "model.layers.11.self_attn.qk_proj": [[59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 57, 40, 94, 56, 37, 125, 122, 62, 61, 101, 113, 114, 36, 88, 48, 24, 118, 127, 119, 27, 47, 22, 124, 83, 21, 49, 30, 19, 78, 53, 115, 76, 17, 14, 55, 120, 100, 51, 89, 81, 12, 85, 33, 52, 102, 95, 80, 50, 86, 32, 43, 16, 28, 97, 108, 87, 91, 9, 90, 23, 112, 42, 7, 92, 71, 38, 29, 31, 44, 93, 25, 41, 73, 107, 15, 10, 82, 96, 75, 69, 74, 34, 20, 79, 3, 11, 106, 5, 18, 67, 2, 84, 105, 0, 26, 6, 66, 98, 64, 99, 39, 70, 68, 72, 103, 13, 77, 4, 1, 65, 35, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 60, 104, 57, 94, 40, 56, 113, 122, 101, 127, 37, 36, 119, 114, 88, 118, 125, 24, 61, 62, 22, 49, 27, 53, 30, 48, 76, 52, 21, 83, 115, 124, 47, 55, 78, 120, 100, 19, 81, 51, 95, 43, 14, 12, 42, 86, 108, 17, 85, 50, 33, 97, 80, 91, 90, 16, 89, 28, 9, 23, 112, 102, 92, 44, 38, 25, 107, 7, 31, 20, 79, 32, 87, 93, 71, 34, 73, 29, 105, 15, 96, 10, 106, 18, 69, 41, 74, 6, 82, 26, 5, 75, 84, 11, 0, 98, 2, 64, 66, 99, 67, 72, 39, 103, 3, 1, 70, 65, 68, 77, 4, 35, 8, 13], [59, 126, 121, 46, 63, 111, 110, 116, 58, 45, 123, 109, 54, 117, 104, 40, 94, 122, 61, 37, 101, 36, 56, 125, 57, 60, 113, 114, 24, 88, 62, 119, 118, 48, 127, 124, 27, 53, 30, 47, 21, 100, 52, 51, 22, 83, 19, 55, 115, 50, 49, 33, 102, 95, 43, 42, 76, 28, 78, 120, 108, 17, 86, 91, 89, 85, 38, 112, 106, 81, 32, 12, 23, 97, 14, 92, 93, 31, 90, 80, 87, 44, 107, 16, 29, 96, 25, 9, 73, 105, 34, 41, 71, 20, 7, 79, 84, 98, 10, 99, 26, 69, 11, 82, 18, 74, 15, 75, 6, 5, 66, 0, 103, 64, 2, 3, 67, 39, 70, 1, 65, 68, 35, 72, 4, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 60, 104, 40, 57, 94, 36, 37, 122, 61, 56, 62, 114, 127, 101, 113, 24, 49, 119, 88, 118, 124, 48, 30, 27, 52, 47, 115, 125, 120, 95, 19, 22, 55, 21, 53, 50, 100, 83, 42, 28, 89, 14, 76, 108, 78, 102, 43, 81, 90, 17, 51, 12, 97, 92, 33, 85, 91, 32, 16, 31, 86, 80, 44, 38, 93, 87, 105, 107, 23, 64, 25, 73, 71, 96, 75, 29, 106, 112, 26, 9, 41, 7, 69, 2, 67, 74, 20, 79, 34, 0, 66, 11, 18, 98, 99, 82, 10, 3, 39, 5, 84, 6, 15, 103, 70, 68, 72, 35, 65, 4, 1, 8, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 104, 40, 122, 61, 57, 60, 56, 94, 127, 36, 37, 118, 62, 113, 101, 119, 24, 125, 114, 88, 48, 49, 52, 124, 83, 21, 30, 120, 19, 76, 22, 115, 47, 55, 27, 78, 53, 12, 95, 85, 80, 28, 100, 42, 43, 50, 17, 51, 16, 102, 81, 89, 108, 33, 14, 112, 23, 97, 91, 92, 90, 32, 107, 25, 86, 7, 31, 71, 105, 93, 29, 44, 73, 9, 87, 41, 96, 0, 38, 15, 75, 106, 64, 82, 79, 20, 74, 66, 2, 10, 98, 67, 18, 84, 69, 26, 99, 34, 70, 11, 65, 3, 5, 103, 6, 39, 8, 1, 72, 68, 13, 77, 4, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 54, 117, 104, 60, 40, 57, 127, 122, 56, 113, 94, 114, 61, 36, 118, 37, 48, 101, 62, 88, 24, 119, 49, 22, 125, 83, 30, 52, 21, 27, 78, 19, 124, 12, 120, 76, 53, 81, 85, 95, 47, 14, 97, 17, 80, 115, 55, 43, 42, 16, 89, 28, 50, 51, 108, 100, 92, 102, 33, 86, 91, 25, 90, 107, 32, 73, 23, 9, 31, 96, 87, 79, 71, 7, 105, 44, 29, 20, 15, 93, 74, 82, 10, 112, 75, 69, 38, 99, 41, 26, 67, 18, 34, 11, 106, 70, 103, 84, 66, 5, 98, 3, 0, 64, 68, 6, 8, 2, 65, 39, 4, 1, 35, 77, 13, 72], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 54, 60, 104, 94, 40, 57, 36, 37, 113, 101, 24, 114, 127, 122, 88, 48, 21, 61, 118, 119, 56, 83, 19, 125, 27, 22, 62, 78, 30, 53, 49, 14, 17, 52, 95, 85, 76, 81, 12, 120, 124, 89, 86, 16, 100, 80, 55, 50, 28, 32, 115, 42, 43, 47, 112, 33, 92, 91, 87, 9, 102, 23, 97, 107, 51, 108, 38, 90, 71, 20, 96, 93, 31, 7, 11, 82, 75, 73, 25, 44, 29, 5, 41, 74, 18, 15, 34, 99, 10, 79, 105, 26, 67, 106, 98, 84, 69, 3, 39, 70, 64, 103, 2, 66, 8, 1, 6, 13, 0, 68, 4, 35, 72, 65, 77], [59, 126, 46, 121, 63, 116, 111, 110, 58, 45, 123, 109, 104, 40, 54, 94, 117, 60, 113, 24, 101, 37, 122, 36, 27, 88, 83, 57, 114, 127, 61, 22, 21, 47, 30, 52, 19, 118, 76, 56, 119, 125, 12, 62, 53, 48, 124, 78, 86, 81, 85, 49, 95, 17, 100, 14, 89, 115, 80, 91, 33, 55, 16, 28, 120, 50, 102, 92, 23, 42, 87, 43, 51, 7, 90, 32, 44, 9, 112, 108, 20, 38, 25, 82, 11, 97, 93, 75, 31, 29, 71, 15, 79, 73, 107, 106, 18, 10, 26, 5, 84, 64, 34, 41, 69, 74, 2, 70, 96, 0, 105, 3, 98, 8, 66, 99, 103, 67, 39, 1, 65, 4, 72, 6, 13, 68, 77, 35], [59, 126, 121, 46, 63, 111, 116, 110, 58, 45, 123, 109, 117, 104, 40, 94, 54, 60, 56, 36, 57, 122, 88, 101, 62, 37, 24, 114, 61, 119, 27, 49, 125, 118, 83, 113, 22, 19, 30, 21, 52, 124, 48, 100, 47, 127, 78, 55, 51, 89, 120, 85, 115, 17, 14, 76, 81, 12, 43, 86, 53, 80, 33, 95, 102, 38, 91, 28, 50, 42, 90, 108, 16, 23, 32, 107, 112, 92, 106, 93, 97, 44, 87, 31, 73, 9, 34, 7, 71, 15, 25, 18, 29, 41, 20, 10, 96, 11, 79, 105, 5, 82, 69, 98, 75, 26, 103, 84, 2, 99, 64, 0, 74, 66, 70, 67, 65, 3, 8, 6, 1, 4, 39, 72, 68, 77, 13, 35], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 104, 54, 56, 60, 117, 40, 122, 57, 113, 94, 101, 37, 61, 62, 88, 36, 24, 114, 125, 127, 27, 119, 48, 83, 118, 19, 49, 47, 115, 21, 52, 22, 30, 120, 124, 76, 78, 85, 100, 51, 43, 86, 33, 12, 17, 14, 89, 80, 53, 55, 42, 81, 95, 108, 91, 92, 32, 16, 28, 97, 23, 50, 73, 31, 87, 38, 9, 90, 7, 102, 93, 15, 25, 105, 71, 112, 96, 79, 44, 107, 41, 69, 10, 18, 29, 67, 75, 84, 74, 11, 34, 3, 82, 20, 6, 106, 103, 26, 66, 64, 98, 0, 2, 5, 70, 8, 99, 1, 13, 68, 4, 39, 77, 72, 65, 35], [59, 126, 63, 121, 46, 111, 110, 116, 58, 123, 45, 109, 104, 117, 122, 54, 40, 113, 56, 61, 60, 125, 94, 62, 57, 37, 48, 114, 101, 24, 127, 88, 118, 36, 124, 119, 49, 21, 115, 47, 19, 52, 78, 83, 120, 30, 33, 22, 27, 76, 95, 12, 100, 50, 80, 28, 17, 85, 53, 14, 102, 43, 42, 16, 51, 81, 97, 86, 89, 73, 55, 91, 23, 25, 92, 108, 41, 32, 90, 7, 106, 38, 74, 9, 71, 29, 107, 66, 96, 79, 15, 10, 18, 6, 31, 87, 11, 34, 69, 93, 82, 20, 2, 0, 5, 75, 105, 64, 98, 26, 112, 84, 70, 67, 8, 3, 44, 13, 99, 4, 68, 1, 65, 72, 103, 77, 39, 35], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 109, 45, 117, 104, 54, 40, 122, 60, 118, 56, 61, 113, 88, 57, 62, 114, 94, 48, 36, 101, 127, 37, 124, 24, 49, 19, 125, 21, 78, 83, 119, 30, 12, 22, 52, 115, 27, 76, 80, 17, 14, 47, 81, 16, 95, 120, 85, 86, 28, 100, 91, 33, 89, 50, 43, 102, 55, 108, 97, 51, 71, 53, 73, 32, 23, 90, 42, 7, 92, 10, 9, 11, 15, 38, 25, 41, 107, 82, 112, 105, 20, 87, 31, 74, 29, 18, 79, 96, 106, 69, 75, 5, 6, 0, 93, 84, 34, 3, 99, 67, 64, 1, 44, 98, 2, 26, 66, 65, 8, 72, 4, 70, 103, 13, 77, 39, 68, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 54, 117, 104, 40, 60, 122, 94, 24, 57, 88, 113, 48, 56, 49, 36, 101, 37, 114, 62, 61, 27, 124, 119, 125, 127, 19, 118, 83, 22, 78, 21, 12, 30, 55, 17, 115, 14, 80, 47, 50, 81, 76, 85, 52, 86, 120, 100, 108, 89, 95, 28, 53, 43, 91, 16, 33, 51, 42, 92, 97, 23, 25, 31, 38, 79, 32, 90, 44, 105, 15, 73, 112, 82, 9, 102, 29, 71, 87, 93, 74, 41, 7, 10, 96, 84, 69, 18, 34, 11, 107, 20, 75, 106, 26, 103, 3, 6, 99, 98, 67, 39, 5, 72, 0, 8, 2, 66, 65, 1, 70, 64, 13, 77, 35, 68, 4], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 60, 40, 117, 114, 94, 122, 57, 48, 127, 37, 36, 61, 113, 49, 47, 56, 62, 101, 50, 21, 24, 88, 12, 27, 119, 125, 124, 22, 118, 78, 19, 14, 76, 83, 30, 52, 80, 97, 53, 95, 17, 28, 81, 55, 16, 120, 86, 115, 89, 100, 51, 73, 85, 9, 108, 112, 92, 43, 7, 25, 90, 71, 33, 91, 102, 10, 15, 87, 44, 79, 23, 107, 32, 31, 69, 29, 82, 42, 38, 18, 20, 64, 96, 6, 74, 11, 0, 106, 34, 93, 84, 75, 26, 41, 5, 66, 99, 67, 105, 2, 70, 1, 3, 98, 72, 39, 103, 65, 68, 4, 35, 13, 8, 77], [59, 126, 46, 121, 63, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 113, 94, 122, 61, 56, 57, 36, 49, 37, 88, 24, 101, 48, 125, 127, 118, 62, 119, 27, 19, 21, 124, 30, 78, 22, 12, 83, 17, 47, 55, 50, 52, 14, 16, 115, 76, 89, 80, 100, 51, 95, 85, 86, 53, 81, 97, 91, 28, 120, 108, 102, 33, 92, 43, 23, 107, 31, 9, 38, 42, 25, 32, 7, 90, 44, 41, 71, 112, 73, 93, 29, 15, 11, 87, 105, 74, 82, 10, 106, 18, 20, 79, 84, 34, 96, 69, 6, 98, 5, 99, 75, 0, 3, 26, 72, 70, 66, 39, 67, 64, 103, 2, 35, 65, 68, 4, 8, 1, 13, 77], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 54, 60, 104, 40, 127, 56, 57, 61, 122, 94, 48, 37, 36, 114, 113, 118, 88, 124, 101, 49, 24, 62, 83, 30, 27, 19, 78, 52, 115, 119, 21, 12, 47, 125, 22, 81, 55, 100, 120, 17, 16, 95, 97, 50, 85, 80, 89, 14, 86, 43, 53, 76, 51, 102, 108, 91, 33, 107, 112, 90, 42, 28, 41, 32, 73, 23, 7, 9, 71, 31, 25, 10, 74, 29, 92, 96, 105, 34, 15, 44, 11, 106, 87, 18, 75, 20, 82, 98, 79, 69, 99, 93, 38, 67, 0, 84, 64, 3, 26, 5, 6, 70, 103, 66, 72, 2, 68, 35, 4, 65, 1, 8, 13, 39, 77], [59, 126, 46, 63, 121, 116, 111, 110, 58, 45, 123, 109, 104, 117, 60, 40, 54, 122, 49, 56, 24, 101, 57, 114, 94, 37, 88, 61, 27, 125, 48, 36, 127, 83, 30, 19, 113, 22, 62, 47, 85, 78, 21, 119, 118, 120, 12, 55, 52, 81, 16, 100, 14, 124, 86, 43, 17, 95, 76, 80, 89, 115, 97, 50, 33, 42, 9, 108, 51, 91, 28, 53, 112, 90, 31, 32, 23, 92, 73, 25, 41, 71, 44, 15, 107, 87, 74, 102, 106, 82, 20, 79, 38, 75, 7, 29, 93, 105, 96, 10, 34, 98, 18, 11, 84, 26, 72, 69, 99, 70, 5, 0, 3, 103, 64, 35, 67, 2, 4, 68, 39, 65, 66, 1, 77, 13, 6, 8], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 60, 54, 40, 94, 56, 57, 127, 114, 101, 24, 122, 113, 36, 37, 118, 88, 119, 49, 27, 19, 48, 62, 125, 83, 21, 22, 61, 78, 12, 30, 55, 124, 47, 52, 86, 14, 95, 50, 120, 100, 115, 17, 81, 80, 85, 16, 97, 42, 76, 89, 33, 28, 91, 43, 44, 23, 53, 112, 31, 102, 51, 108, 9, 90, 32, 38, 25, 92, 20, 41, 107, 73, 82, 71, 106, 87, 18, 93, 105, 7, 79, 15, 84, 29, 74, 11, 96, 34, 26, 10, 98, 64, 69, 5, 67, 75, 103, 70, 2, 0, 66, 72, 99, 3, 35, 77, 1, 39, 6, 8, 65, 4, 68, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 49, 104, 56, 36, 122, 57, 94, 118, 61, 54, 119, 114, 40, 60, 101, 127, 88, 125, 113, 37, 48, 24, 83, 27, 52, 62, 124, 30, 120, 19, 85, 115, 43, 55, 47, 50, 22, 17, 21, 14, 102, 51, 53, 81, 95, 108, 12, 100, 78, 33, 76, 38, 80, 112, 16, 86, 42, 97, 89, 32, 41, 91, 28, 107, 31, 105, 93, 25, 92, 106, 23, 90, 87, 82, 29, 73, 7, 96, 71, 99, 18, 44, 69, 20, 34, 0, 84, 9, 79, 11, 26, 10, 103, 70, 67, 74, 66, 98, 5, 64, 3, 35, 15, 75, 2, 39, 1, 68, 6, 8, 65, 77, 72, 4, 13], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 114, 56, 94, 57, 49, 127, 101, 37, 36, 122, 24, 118, 113, 61, 88, 27, 22, 48, 125, 119, 124, 115, 30, 19, 83, 62, 52, 21, 14, 95, 47, 120, 81, 100, 53, 89, 12, 51, 17, 97, 50, 78, 55, 108, 33, 76, 86, 28, 85, 102, 16, 43, 91, 112, 80, 90, 25, 42, 31, 73, 92, 23, 93, 107, 106, 105, 41, 38, 44, 32, 9, 87, 96, 29, 20, 7, 82, 18, 34, 79, 10, 11, 5, 71, 98, 69, 15, 74, 84, 26, 70, 99, 103, 75, 64, 2, 0, 66, 72, 67, 6, 35, 3, 39, 8, 77, 65, 68, 4, 1, 13], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 60, 54, 40, 56, 113, 114, 48, 94, 122, 118, 57, 49, 125, 36, 127, 124, 24, 88, 37, 61, 101, 62, 19, 52, 22, 27, 83, 119, 47, 21, 30, 78, 55, 12, 95, 81, 115, 76, 16, 14, 120, 100, 17, 89, 86, 108, 80, 97, 85, 50, 33, 51, 28, 42, 91, 53, 102, 9, 112, 90, 43, 23, 105, 41, 29, 32, 20, 7, 107, 25, 73, 92, 31, 106, 71, 38, 93, 74, 87, 18, 79, 10, 5, 44, 96, 82, 69, 75, 11, 98, 70, 64, 3, 34, 84, 66, 15, 26, 6, 67, 0, 2, 68, 99, 103, 39, 72, 77, 8, 4, 13, 65, 1, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 113, 94, 60, 40, 54, 114, 56, 118, 48, 125, 122, 57, 88, 49, 101, 37, 36, 24, 119, 61, 52, 124, 127, 62, 21, 27, 83, 30, 19, 22, 100, 33, 47, 43, 115, 55, 78, 50, 81, 17, 12, 95, 108, 85, 80, 120, 89, 53, 14, 102, 16, 97, 86, 91, 76, 28, 106, 90, 32, 51, 42, 112, 38, 31, 92, 107, 87, 23, 105, 9, 93, 7, 71, 96, 25, 20, 74, 11, 82, 29, 18, 73, 79, 34, 10, 41, 15, 99, 75, 26, 67, 6, 103, 0, 2, 84, 98, 69, 64, 44, 3, 5, 66, 35, 8, 72, 39, 70, 1, 77, 65, 68, 13, 4], [59, 126, 46, 63, 121, 116, 110, 111, 58, 45, 123, 109, 104, 40, 54, 117, 60, 56, 94, 113, 125, 122, 61, 37, 118, 57, 48, 24, 101, 119, 36, 88, 62, 49, 114, 22, 30, 127, 124, 83, 19, 100, 52, 27, 85, 78, 21, 33, 47, 115, 12, 43, 17, 91, 95, 120, 108, 50, 16, 97, 14, 76, 80, 28, 89, 53, 55, 81, 102, 86, 51, 32, 42, 106, 112, 38, 92, 23, 31, 105, 87, 107, 9, 96, 93, 90, 41, 73, 25, 71, 7, 79, 20, 34, 26, 74, 18, 29, 44, 10, 82, 99, 84, 15, 11, 75, 6, 5, 64, 0, 103, 69, 98, 67, 2, 1, 39, 66, 8, 72, 4, 3, 70, 13, 65, 77, 35, 68], [59, 126, 63, 46, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 56, 54, 60, 40, 113, 94, 36, 49, 57, 48, 122, 118, 125, 24, 114, 62, 37, 127, 101, 119, 88, 61, 22, 83, 124, 76, 47, 30, 27, 120, 52, 19, 97, 21, 95, 78, 100, 33, 85, 115, 80, 28, 14, 81, 55, 91, 16, 17, 50, 53, 86, 43, 12, 102, 108, 89, 73, 42, 23, 51, 107, 92, 31, 25, 90, 7, 32, 9, 112, 44, 93, 105, 41, 71, 79, 87, 38, 10, 29, 96, 82, 20, 74, 18, 26, 5, 6, 34, 84, 99, 15, 69, 67, 75, 103, 106, 11, 3, 66, 0, 98, 8, 2, 64, 70, 39, 68, 4, 65, 72, 1, 13, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 54, 117, 40, 94, 56, 62, 122, 57, 114, 60, 125, 101, 36, 48, 118, 24, 113, 37, 61, 88, 49, 127, 119, 21, 124, 83, 27, 19, 22, 86, 30, 52, 100, 120, 81, 115, 47, 78, 14, 17, 76, 85, 50, 91, 33, 55, 42, 32, 89, 102, 16, 53, 43, 95, 12, 51, 80, 97, 108, 41, 107, 92, 23, 28, 112, 90, 87, 25, 31, 73, 7, 38, 71, 29, 93, 20, 82, 74, 9, 106, 96, 105, 44, 75, 26, 18, 99, 34, 79, 10, 84, 15, 5, 69, 3, 98, 6, 11, 8, 64, 39, 70, 103, 66, 67, 0, 2, 35, 68, 1, 13, 65, 77, 4, 72], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 104, 117, 40, 54, 94, 56, 57, 125, 60, 122, 113, 36, 24, 101, 62, 37, 88, 114, 49, 48, 119, 19, 118, 127, 22, 27, 83, 21, 86, 47, 100, 30, 124, 61, 95, 52, 115, 33, 17, 43, 78, 76, 55, 120, 85, 14, 81, 80, 12, 91, 97, 16, 108, 89, 42, 53, 102, 28, 51, 92, 41, 50, 90, 23, 20, 31, 9, 25, 32, 38, 87, 107, 7, 29, 105, 112, 82, 71, 11, 106, 79, 44, 73, 93, 96, 74, 18, 10, 69, 15, 84, 98, 5, 34, 75, 26, 2, 99, 0, 70, 64, 103, 6, 39, 67, 8, 66, 13, 3, 35, 72, 77, 68, 1, 65, 4], [59, 126, 46, 121, 63, 111, 116, 110, 58, 45, 123, 109, 104, 54, 40, 125, 117, 113, 122, 94, 56, 57, 62, 101, 37, 119, 60, 36, 88, 24, 61, 114, 118, 49, 48, 22, 127, 19, 21, 47, 30, 27, 124, 83, 52, 55, 115, 43, 17, 33, 50, 100, 86, 78, 12, 108, 95, 28, 76, 14, 80, 51, 85, 97, 89, 53, 91, 81, 120, 42, 16, 112, 102, 90, 23, 38, 31, 106, 92, 87, 32, 71, 93, 9, 41, 29, 7, 73, 79, 105, 25, 44, 74, 84, 18, 75, 96, 34, 20, 82, 5, 26, 15, 67, 10, 64, 2, 0, 66, 69, 107, 11, 103, 98, 3, 70, 99, 39, 6, 8, 35, 68, 4, 1, 65, 72, 13, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 104, 117, 54, 40, 57, 94, 122, 36, 118, 37, 113, 56, 60, 114, 61, 125, 62, 48, 101, 88, 24, 49, 47, 119, 124, 83, 30, 19, 27, 22, 50, 52, 21, 76, 17, 127, 78, 95, 120, 85, 12, 28, 55, 100, 81, 115, 89, 14, 43, 33, 86, 80, 53, 97, 16, 108, 9, 91, 51, 102, 73, 32, 23, 7, 71, 92, 90, 25, 87, 42, 75, 107, 41, 106, 93, 29, 64, 38, 5, 18, 74, 112, 96, 10, 15, 31, 70, 2, 105, 82, 66, 44, 11, 3, 79, 20, 0, 84, 69, 34, 67, 26, 99, 65, 6, 1, 39, 8, 98, 103, 72, 68, 77, 4, 13, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 54, 104, 40, 61, 60, 57, 56, 113, 118, 122, 101, 37, 48, 36, 94, 62, 125, 114, 127, 88, 24, 47, 119, 21, 49, 83, 124, 22, 19, 30, 120, 27, 52, 76, 89, 53, 95, 78, 100, 115, 51, 33, 43, 55, 12, 50, 17, 102, 28, 81, 80, 108, 42, 86, 85, 31, 90, 97, 14, 32, 16, 23, 91, 87, 38, 93, 73, 112, 7, 92, 96, 41, 9, 71, 106, 25, 44, 20, 15, 107, 105, 79, 5, 26, 34, 18, 70, 99, 10, 29, 75, 84, 82, 0, 69, 3, 74, 64, 11, 66, 67, 2, 98, 1, 65, 103, 8, 39, 68, 6, 35, 72, 4, 77, 13], [59, 126, 46, 63, 121, 116, 111, 110, 58, 123, 45, 109, 117, 104, 40, 54, 60, 36, 56, 57, 113, 118, 61, 88, 94, 37, 101, 24, 122, 47, 62, 83, 125, 49, 114, 127, 27, 22, 48, 119, 21, 124, 30, 76, 19, 78, 95, 17, 52, 80, 43, 12, 85, 14, 115, 53, 108, 50, 55, 100, 16, 81, 33, 28, 120, 42, 89, 73, 51, 23, 32, 31, 9, 102, 86, 87, 90, 91, 97, 44, 71, 38, 92, 112, 25, 7, 106, 74, 93, 107, 34, 79, 29, 96, 41, 10, 67, 3, 64, 20, 5, 70, 18, 15, 11, 69, 75, 82, 84, 26, 103, 105, 0, 66, 65, 6, 99, 98, 72, 2, 4, 1, 39, 8, 13, 68, 77, 35], [59, 126, 46, 63, 121, 111, 116, 110, 58, 123, 45, 109, 117, 104, 54, 40, 60, 122, 57, 118, 113, 127, 94, 61, 56, 37, 101, 48, 36, 88, 125, 114, 24, 27, 119, 62, 124, 83, 21, 47, 49, 19, 22, 50, 76, 89, 30, 55, 14, 100, 78, 86, 120, 12, 81, 85, 42, 52, 115, 95, 53, 80, 33, 17, 16, 108, 51, 87, 112, 32, 102, 28, 97, 90, 43, 31, 23, 9, 107, 91, 92, 38, 71, 73, 41, 7, 20, 79, 18, 105, 15, 82, 74, 44, 25, 96, 11, 75, 10, 26, 34, 5, 29, 93, 99, 64, 106, 3, 98, 70, 69, 103, 84, 67, 2, 0, 66, 6, 8, 1, 72, 65, 39, 4, 13, 68, 35, 77], [59, 126, 46, 63, 121, 111, 116, 110, 58, 45, 123, 109, 117, 104, 54, 60, 40, 94, 122, 113, 127, 57, 88, 56, 36, 61, 37, 125, 48, 27, 24, 118, 101, 76, 62, 114, 22, 120, 83, 124, 119, 19, 49, 80, 30, 47, 85, 115, 14, 12, 50, 86, 55, 21, 52, 81, 17, 100, 78, 89, 95, 16, 28, 33, 108, 97, 91, 51, 42, 9, 73, 53, 90, 32, 43, 102, 87, 23, 112, 7, 71, 31, 92, 79, 107, 82, 74, 38, 44, 15, 41, 29, 10, 25, 20, 93, 69, 75, 18, 84, 3, 26, 11, 96, 105, 106, 5, 99, 34, 6, 70, 98, 0, 39, 67, 72, 64, 2, 66, 65, 1, 4, 103, 13, 8, 35, 68, 77]], "model.layers.12.self_attn.q_proj": [[102, 127, 49, 95, 21, 24, 16, 83, 80, 82, 88, 77, 28, 6, 51, 99, 44, 37, 38, 72, 2, 31, 74, 27, 106, 22, 75, 86, 13, 53, 42, 108, 26, 98, 112, 10, 93, 96, 18, 43, 62, 20, 94, 103, 91, 79, 4, 117, 90, 61, 85, 78, 29, 101, 104, 14, 40, 19, 25, 81, 11, 32, 92, 64, 89, 60, 17, 30, 36, 56, 12, 109, 125, 9, 113, 48, 111, 52, 116, 70, 87, 41, 115, 105, 84, 71, 76, 33, 15, 55, 114, 65, 8, 73, 118, 100, 45, 97, 54, 107, 34, 68, 39, 69, 0, 123, 35, 5, 66, 7, 110, 67, 58, 122, 126, 1, 47, 121, 59, 23, 120, 50, 57, 46, 119, 63, 124, 3], [127, 102, 49, 95, 51, 108, 24, 31, 26, 38, 98, 19, 83, 21, 96, 61, 106, 44, 42, 116, 97, 111, 46, 43, 109, 28, 125, 105, 29, 55, 115, 114, 93, 62, 112, 53, 40, 37, 52, 41, 126, 101, 22, 27, 54, 60, 121, 58, 45, 103, 107, 110, 117, 82, 123, 120, 104, 56, 113, 36, 124, 48, 119, 57, 50, 59, 32, 118, 94, 122, 20, 100, 88, 47, 92, 91, 13, 33, 99, 39, 84, 34, 86, 90, 85, 63, 15, 25, 77, 17, 30, 35, 79, 18, 80, 87, 89, 23, 81, 16, 14, 6, 78, 72, 12, 70, 76, 74, 64, 8, 4, 75, 11, 10, 73, 71, 9, 65, 2, 68, 66, 67, 1, 7, 0, 69, 3, 5], [102, 49, 127, 95, 21, 108, 82, 88, 77, 83, 24, 51, 96, 44, 86, 98, 99, 80, 113, 37, 28, 27, 6, 16, 31, 74, 117, 26, 42, 90, 72, 2, 62, 19, 46, 41, 53, 105, 116, 79, 8, 94, 22, 119, 25, 18, 121, 112, 85, 20, 61, 14, 38, 101, 109, 65, 29, 93, 68, 92, 12, 104, 36, 76, 43, 81, 115, 64, 10, 55, 125, 111, 56, 23, 87, 45, 91, 40, 126, 58, 107, 120, 13, 100, 59, 50, 52, 60, 97, 122, 39, 30, 4, 32, 110, 54, 71, 103, 78, 123, 34, 118, 17, 57, 33, 114, 106, 124, 11, 47, 35, 84, 63, 69, 89, 3, 48, 70, 15, 75, 7, 5, 73, 9, 0, 67, 1, 66], [102, 49, 127, 95, 21, 24, 83, 28, 82, 80, 77, 11, 99, 38, 88, 31, 96, 20, 74, 90, 42, 53, 51, 26, 113, 35, 86, 60, 105, 14, 70, 6, 68, 108, 44, 15, 22, 18, 16, 75, 98, 101, 97, 56, 79, 58, 27, 37, 62, 2, 114, 29, 85, 25, 103, 81, 19, 8, 43, 106, 110, 112, 87, 92, 9, 116, 61, 117, 5, 12, 67, 89, 10, 17, 34, 109, 30, 120, 111, 55, 13, 78, 73, 76, 36, 3, 94, 48, 46, 0, 121, 124, 119, 23, 115, 93, 39, 91, 107, 50, 122, 71, 100, 41, 123, 72, 33, 126, 65, 59, 1, 57, 125, 47, 52, 69, 104, 45, 118, 54, 32, 40, 7, 84, 64, 4, 63, 66], [45, 101, 109, 26, 28, 21, 24, 83, 88, 62, 31, 78, 95, 97, 72, 82, 124, 75, 87, 113, 4, 122, 30, 49, 44, 121, 53, 93, 94, 27, 98, 51, 16, 116, 85, 80, 110, 77, 66, 57, 111, 69, 102, 0, 18, 25, 86, 105, 106, 14, 63, 92, 68, 36, 115, 119, 100, 107, 118, 19, 114, 23, 76, 35, 34, 79, 61, 81, 15, 59, 41, 33, 47, 103, 9, 52, 91, 60, 112, 46, 120, 1, 8, 117, 39, 50, 13, 67, 104, 125, 48, 127, 10, 89, 40, 58, 32, 70, 71, 22, 56, 90, 20, 38, 11, 123, 54, 55, 42, 17, 29, 43, 108, 12, 37, 126, 99, 84, 96, 74, 6, 2, 73, 5, 7, 65, 3, 64], [45, 101, 109, 28, 62, 21, 26, 24, 31, 88, 83, 87, 97, 78, 122, 124, 75, 95, 16, 107, 53, 72, 113, 82, 98, 49, 116, 94, 17, 25, 91, 85, 47, 93, 121, 76, 102, 30, 77, 86, 34, 57, 106, 51, 59, 6, 103, 112, 22, 108, 44, 117, 36, 84, 29, 33, 10, 61, 52, 74, 63, 39, 118, 60, 27, 43, 12, 13, 18, 71, 123, 96, 48, 114, 23, 110, 105, 79, 41, 67, 92, 115, 42, 81, 111, 56, 66, 80, 69, 8, 58, 55, 50, 127, 14, 125, 0, 126, 120, 100, 35, 20, 119, 40, 11, 104, 54, 89, 68, 4, 46, 19, 32, 99, 38, 70, 37, 5, 7, 90, 73, 9, 1, 15, 2, 65, 3, 64], [45, 101, 109, 28, 26, 122, 21, 24, 83, 87, 88, 31, 97, 95, 78, 121, 62, 124, 113, 30, 75, 51, 53, 82, 116, 93, 86, 16, 110, 102, 98, 25, 36, 63, 106, 72, 59, 34, 79, 49, 112, 57, 35, 107, 47, 111, 38, 44, 52, 58, 105, 29, 39, 100, 27, 118, 61, 125, 123, 114, 115, 55, 41, 40, 48, 37, 119, 15, 92, 94, 50, 85, 6, 13, 46, 60, 103, 108, 127, 33, 99, 126, 74, 42, 12, 32, 104, 54, 20, 0, 84, 22, 14, 117, 56, 43, 120, 80, 18, 19, 90, 91, 23, 96, 8, 11, 68, 17, 89, 81, 69, 77, 76, 10, 67, 66, 9, 5, 70, 71, 73, 3, 1, 7, 4, 65, 2, 64], [45, 101, 109, 26, 28, 24, 88, 83, 21, 82, 93, 87, 121, 62, 95, 75, 78, 31, 124, 122, 16, 97, 98, 116, 30, 113, 34, 51, 53, 57, 77, 110, 49, 63, 107, 61, 125, 115, 18, 112, 41, 105, 22, 106, 94, 103, 71, 36, 111, 92, 86, 11, 118, 29, 37, 85, 40, 59, 9, 27, 67, 47, 50, 4, 127, 33, 100, 108, 44, 19, 117, 123, 32, 35, 126, 1, 60, 0, 48, 80, 69, 46, 23, 42, 84, 120, 55, 6, 79, 52, 91, 20, 81, 14, 72, 58, 68, 54, 5, 56, 8, 114, 13, 43, 66, 74, 102, 39, 90, 99, 15, 12, 119, 104, 76, 96, 17, 65, 25, 38, 89, 2, 73, 10, 70, 7, 3, 64], [43, 97, 107, 24, 91, 26, 31, 22, 83, 81, 100, 85, 103, 75, 33, 87, 30, 20, 50, 15, 88, 110, 13, 7, 114, 90, 49, 127, 95, 66, 93, 86, 84, 12, 94, 19, 102, 92, 98, 61, 113, 48, 70, 18, 35, 0, 115, 40, 118, 25, 38, 54, 1, 63, 36, 16, 80, 28, 117, 56, 58, 27, 101, 77, 42, 51, 99, 82, 32, 34, 120, 29, 55, 125, 71, 96, 4, 89, 21, 79, 17, 6, 39, 47, 112, 41, 111, 116, 8, 119, 121, 123, 23, 106, 45, 62, 52, 3, 122, 53, 46, 11, 57, 37, 105, 73, 67, 10, 9, 109, 74, 126, 108, 78, 44, 104, 14, 76, 60, 59, 124, 72, 69, 68, 5, 2, 64, 65], [43, 31, 97, 107, 22, 13, 24, 26, 15, 91, 83, 20, 85, 33, 9, 115, 100, 110, 75, 73, 18, 81, 86, 84, 114, 50, 90, 70, 68, 54, 36, 95, 72, 25, 17, 79, 27, 77, 88, 63, 121, 19, 71, 42, 111, 61, 12, 127, 76, 23, 82, 93, 74, 87, 16, 66, 49, 65, 8, 38, 122, 46, 48, 10, 7, 103, 112, 14, 101, 47, 78, 117, 69, 34, 113, 67, 55, 118, 80, 29, 21, 116, 126, 123, 92, 124, 56, 89, 28, 96, 106, 6, 1, 108, 120, 40, 102, 94, 30, 98, 11, 105, 45, 35, 125, 51, 39, 32, 5, 62, 58, 41, 4, 37, 57, 60, 64, 59, 52, 99, 44, 109, 119, 104, 0, 53, 3, 2], [43, 107, 31, 97, 26, 106, 42, 20, 113, 61, 115, 125, 50, 33, 24, 48, 127, 117, 11, 38, 114, 121, 36, 100, 49, 126, 95, 54, 63, 116, 110, 111, 56, 57, 118, 55, 123, 46, 47, 60, 58, 91, 103, 77, 108, 39, 119, 109, 101, 6, 51, 124, 45, 44, 59, 122, 7, 120, 62, 35, 85, 112, 40, 53, 41, 52, 30, 73, 90, 92, 105, 22, 37, 17, 81, 104, 16, 10, 99, 96, 3, 5, 84, 75, 1, 32, 21, 98, 79, 34, 19, 102, 93, 29, 87, 94, 4, 72, 28, 23, 25, 88, 89, 27, 0, 83, 82, 86, 18, 78, 80, 2, 71, 15, 9, 14, 12, 76, 13, 66, 8, 64, 70, 68, 65, 67, 69, 74], [43, 97, 31, 107, 24, 26, 91, 22, 54, 81, 42, 20, 115, 83, 75, 33, 13, 50, 100, 15, 18, 85, 48, 61, 84, 120, 66, 114, 110, 93, 90, 6, 57, 70, 127, 30, 113, 12, 49, 63, 103, 118, 95, 80, 19, 125, 121, 88, 86, 56, 9, 116, 77, 47, 0, 46, 1, 117, 71, 60, 36, 28, 41, 111, 45, 82, 112, 55, 123, 108, 106, 124, 89, 126, 92, 58, 14, 8, 38, 51, 29, 40, 76, 11, 109, 87, 23, 119, 122, 94, 34, 17, 73, 27, 79, 64, 25, 101, 99, 67, 44, 3, 39, 35, 68, 21, 32, 98, 52, 37, 96, 105, 53, 65, 7, 16, 69, 62, 78, 104, 74, 102, 59, 10, 72, 5, 4, 2], [38, 64, 113, 97, 93, 49, 57, 1, 6, 78, 67, 16, 70, 3, 82, 73, 65, 11, 4, 77, 23, 12, 90, 2, 31, 14, 72, 19, 8, 94, 121, 126, 66, 75, 89, 62, 71, 59, 20, 13, 84, 9, 122, 86, 18, 103, 22, 87, 37, 83, 125, 53, 69, 7, 21, 80, 118, 27, 42, 39, 54, 10, 101, 44, 76, 33, 5, 91, 58, 88, 74, 26, 17, 29, 40, 0, 30, 68, 32, 119, 79, 25, 50, 63, 96, 85, 45, 35, 15, 60, 99, 36, 95, 108, 123, 92, 81, 124, 48, 109, 100, 116, 120, 61, 111, 114, 51, 24, 115, 106, 46, 56, 117, 127, 52, 47, 102, 34, 104, 98, 28, 43, 55, 41, 105, 110, 112, 107], [38, 57, 113, 97, 49, 93, 23, 82, 78, 16, 73, 88, 11, 21, 12, 20, 25, 102, 6, 29, 5, 39, 94, 77, 72, 9, 26, 83, 37, 31, 32, 68, 27, 13, 125, 28, 90, 30, 92, 63, 71, 121, 106, 85, 19, 34, 101, 44, 22, 91, 62, 8, 65, 15, 80, 4, 74, 75, 81, 84, 59, 79, 87, 14, 17, 76, 18, 24, 122, 66, 7, 89, 33, 105, 10, 2, 95, 109, 119, 3, 108, 60, 36, 96, 42, 86, 45, 35, 98, 41, 70, 54, 100, 126, 52, 40, 99, 55, 50, 47, 51, 58, 56, 69, 53, 103, 118, 120, 61, 107, 115, 123, 112, 43, 46, 116, 104, 114, 48, 110, 127, 64, 111, 124, 67, 1, 117, 0], [113, 101, 57, 38, 62, 49, 97, 123, 126, 121, 119, 53, 52, 56, 102, 59, 60, 114, 127, 118, 125, 26, 116, 117, 51, 106, 58, 46, 100, 103, 124, 63, 44, 61, 31, 48, 111, 45, 50, 112, 120, 47, 90, 115, 109, 54, 122, 55, 110, 107, 108, 105, 40, 22, 41, 39, 32, 21, 43, 84, 104, 42, 28, 93, 94, 37, 99, 25, 36, 81, 95, 34, 30, 89, 86, 35, 85, 17, 98, 23, 19, 27, 88, 83, 96, 91, 29, 20, 76, 24, 33, 92, 79, 87, 4, 74, 75, 8, 14, 82, 9, 5, 72, 80, 13, 70, 15, 12, 18, 10, 71, 2, 73, 77, 66, 7, 16, 6, 67, 69, 65, 64, 68, 1, 11, 78, 3, 0], [38, 57, 97, 23, 93, 49, 113, 82, 29, 78, 16, 25, 11, 102, 59, 31, 62, 90, 85, 33, 87, 26, 21, 39, 83, 126, 77, 19, 27, 71, 88, 80, 18, 94, 13, 125, 101, 95, 89, 28, 15, 20, 22, 121, 79, 96, 30, 100, 75, 32, 92, 91, 119, 61, 37, 84, 68, 14, 86, 53, 98, 127, 81, 24, 35, 58, 47, 17, 44, 36, 8, 108, 34, 76, 63, 10, 99, 50, 43, 115, 40, 118, 48, 110, 12, 103, 74, 55, 106, 7, 72, 111, 114, 60, 109, 122, 123, 51, 46, 54, 42, 4, 41, 116, 56, 104, 66, 112, 117, 52, 105, 5, 2, 120, 124, 70, 69, 6, 45, 107, 67, 73, 9, 1, 0, 3, 65, 64], [39, 48, 62, 112, 29, 21, 14, 80, 61, 81, 76, 11, 67, 87, 73, 7, 56, 84, 5, 93, 24, 27, 69, 71, 75, 1, 109, 12, 0, 25, 41, 2, 96, 118, 6, 89, 116, 17, 9, 79, 117, 85, 66, 52, 4, 13, 16, 86, 54, 94, 32, 82, 101, 18, 78, 50, 95, 113, 119, 15, 19, 120, 34, 126, 83, 57, 23, 77, 26, 49, 106, 31, 45, 127, 55, 42, 107, 122, 36, 111, 28, 88, 44, 3, 22, 104, 40, 63, 97, 115, 121, 102, 35, 59, 33, 72, 108, 30, 114, 125, 60, 92, 124, 105, 8, 123, 58, 99, 110, 74, 53, 43, 51, 37, 100, 90, 38, 46, 98, 20, 47, 65, 91, 70, 10, 68, 103, 64], [39, 62, 48, 112, 87, 61, 29, 21, 81, 14, 73, 80, 11, 76, 69, 56, 24, 93, 41, 71, 77, 79, 118, 3, 86, 27, 111, 18, 9, 96, 32, 94, 109, 127, 125, 50, 101, 126, 2, 92, 37, 70, 95, 33, 6, 54, 117, 82, 85, 121, 5, 36, 28, 1, 66, 113, 22, 78, 0, 123, 74, 20, 17, 97, 91, 49, 116, 59, 120, 31, 122, 89, 99, 107, 72, 23, 119, 52, 16, 45, 84, 115, 108, 51, 43, 55, 38, 53, 40, 106, 60, 124, 12, 57, 13, 104, 19, 114, 83, 34, 102, 30, 110, 75, 35, 15, 88, 42, 58, 44, 67, 10, 100, 105, 46, 63, 26, 98, 25, 47, 90, 8, 65, 7, 68, 103, 64, 4], [39, 62, 48, 112, 87, 29, 61, 11, 25, 80, 81, 21, 14, 73, 69, 76, 5, 93, 66, 2, 71, 0, 77, 18, 67, 118, 27, 3, 1, 24, 94, 33, 117, 72, 109, 75, 125, 111, 86, 56, 92, 32, 96, 31, 41, 82, 74, 106, 50, 7, 116, 95, 37, 79, 113, 85, 28, 30, 19, 97, 70, 89, 83, 36, 122, 63, 78, 17, 123, 121, 126, 34, 120, 43, 88, 16, 38, 35, 54, 6, 12, 40, 59, 65, 26, 45, 91, 10, 9, 90, 23, 99, 127, 84, 51, 104, 52, 46, 114, 58, 98, 119, 49, 102, 20, 103, 100, 44, 124, 110, 108, 57, 101, 13, 115, 8, 4, 68, 55, 53, 15, 42, 105, 107, 60, 47, 22, 64], [39, 62, 48, 112, 71, 29, 14, 80, 73, 81, 21, 76, 11, 3, 69, 2, 61, 65, 1, 67, 56, 109, 118, 0, 72, 64, 7, 4, 13, 24, 12, 85, 8, 37, 79, 117, 17, 96, 111, 121, 87, 77, 66, 78, 93, 95, 15, 84, 5, 16, 54, 126, 75, 68, 50, 19, 83, 99, 33, 120, 53, 123, 94, 10, 116, 124, 88, 9, 103, 18, 101, 127, 106, 41, 92, 86, 28, 89, 82, 74, 38, 35, 40, 31, 57, 27, 30, 51, 20, 63, 26, 52, 70, 45, 115, 43, 36, 22, 97, 91, 107, 119, 32, 125, 34, 102, 6, 104, 105, 23, 25, 60, 110, 113, 46, 122, 100, 90, 44, 59, 98, 49, 42, 55, 108, 114, 47, 58], [105, 58, 34, 109, 26, 41, 86, 104, 88, 18, 84, 126, 24, 30, 55, 78, 92, 16, 73, 77, 36, 69, 19, 32, 49, 87, 71, 95, 91, 75, 119, 67, 121, 79, 111, 28, 45, 48, 1, 44, 2, 120, 51, 5, 94, 15, 29, 98, 114, 31, 12, 25, 40, 50, 57, 82, 112, 60, 33, 113, 64, 124, 107, 14, 125, 74, 10, 8, 123, 63, 108, 13, 37, 127, 101, 66, 62, 0, 22, 115, 81, 6, 39, 38, 118, 122, 65, 61, 72, 100, 3, 59, 47, 93, 7, 52, 90, 4, 23, 102, 46, 11, 56, 97, 42, 106, 21, 43, 76, 103, 70, 17, 99, 83, 68, 116, 9, 110, 20, 80, 54, 117, 27, 35, 53, 85, 89, 96], [105, 34, 26, 41, 58, 86, 36, 121, 84, 32, 104, 16, 126, 92, 62, 18, 51, 88, 118, 24, 78, 47, 57, 25, 52, 50, 49, 119, 116, 48, 109, 28, 102, 30, 90, 100, 114, 107, 95, 120, 23, 39, 46, 122, 55, 21, 61, 75, 113, 31, 38, 103, 127, 42, 37, 43, 83, 63, 11, 60, 80, 20, 76, 97, 112, 117, 14, 77, 108, 87, 125, 56, 45, 111, 7, 101, 22, 93, 96, 94, 19, 82, 123, 106, 115, 9, 59, 40, 33, 29, 91, 8, 124, 110, 98, 27, 44, 74, 72, 35, 85, 99, 79, 53, 81, 6, 89, 12, 54, 73, 71, 17, 15, 68, 13, 5, 10, 66, 4, 3, 70, 65, 67, 1, 64, 69, 2, 0], [105, 34, 58, 84, 26, 41, 86, 126, 77, 109, 16, 104, 73, 121, 32, 88, 18, 78, 75, 71, 30, 36, 49, 70, 108, 98, 48, 3, 29, 4, 51, 81, 24, 55, 120, 118, 11, 114, 50, 62, 111, 122, 100, 39, 2, 60, 94, 45, 102, 82, 80, 85, 52, 69, 115, 101, 37, 79, 116, 57, 65, 119, 44, 28, 125, 90, 0, 23, 19, 113, 92, 43, 38, 20, 112, 61, 83, 63, 13, 21, 124, 14, 22, 12, 95, 17, 68, 127, 117, 6, 123, 87, 46, 31, 103, 9, 99, 59, 47, 96, 76, 15, 27, 40, 67, 74, 106, 97, 25, 72, 42, 1, 35, 56, 93, 110, 53, 33, 54, 10, 89, 107, 91, 5, 8, 7, 64, 66], [105, 34, 92, 26, 41, 84, 18, 86, 109, 104, 16, 24, 126, 121, 78, 75, 77, 57, 71, 101, 58, 60, 73, 51, 36, 52, 120, 90, 118, 122, 107, 55, 62, 48, 49, 66, 45, 32, 95, 44, 30, 119, 108, 116, 14, 111, 102, 28, 29, 37, 63, 88, 50, 125, 46, 47, 7, 124, 114, 113, 106, 20, 127, 112, 40, 94, 5, 100, 87, 25, 123, 11, 31, 4, 42, 98, 82, 56, 38, 43, 17, 39, 19, 115, 103, 54, 21, 59, 91, 33, 8, 110, 35, 6, 97, 85, 83, 61, 80, 81, 117, 76, 27, 22, 12, 93, 53, 23, 96, 15, 79, 89, 99, 72, 2, 65, 64, 0, 68, 13, 69, 9, 3, 74, 1, 10, 70, 67], [104, 34, 20, 22, 124, 89, 92, 79, 95, 49, 18, 75, 119, 25, 77, 62, 48, 17, 114, 40, 120, 107, 6, 118, 46, 121, 117, 58, 54, 111, 70, 96, 93, 50, 45, 61, 53, 47, 9, 100, 84, 59, 43, 109, 68, 94, 15, 41, 110, 12, 82, 86, 19, 11, 90, 72, 55, 81, 66, 122, 98, 52, 1, 60, 123, 33, 99, 23, 126, 8, 76, 4, 56, 102, 64, 103, 3, 2, 112, 88, 37, 16, 113, 97, 73, 27, 80, 38, 78, 21, 32, 13, 24, 30, 28, 87, 91, 108, 29, 106, 105, 101, 44, 26, 35, 74, 0, 83, 85, 31, 51, 67, 10, 63, 57, 127, 42, 36, 39, 125, 115, 116, 7, 14, 5, 71, 69, 65], [104, 34, 89, 22, 49, 20, 124, 95, 18, 79, 48, 62, 119, 77, 25, 40, 118, 92, 75, 107, 31, 121, 46, 120, 114, 54, 9, 41, 6, 17, 111, 16, 82, 117, 58, 12, 15, 97, 84, 60, 38, 56, 93, 45, 115, 81, 53, 112, 101, 28, 76, 2, 86, 94, 68, 103, 110, 63, 98, 36, 55, 47, 70, 4, 96, 72, 61, 90, 85, 52, 88, 100, 127, 102, 73, 21, 113, 1, 123, 78, 50, 109, 59, 126, 24, 10, 0, 64, 91, 26, 23, 13, 87, 11, 19, 32, 99, 105, 5, 125, 108, 3, 66, 80, 30, 106, 57, 37, 67, 35, 33, 39, 43, 29, 69, 51, 83, 122, 8, 7, 27, 116, 74, 42, 14, 44, 71, 65], [104, 34, 22, 124, 89, 18, 92, 20, 48, 79, 62, 49, 119, 77, 95, 118, 25, 75, 40, 114, 17, 96, 46, 12, 9, 82, 121, 50, 111, 54, 58, 61, 117, 47, 84, 41, 120, 100, 4, 56, 107, 6, 110, 13, 59, 45, 98, 38, 15, 86, 81, 94, 52, 37, 55, 53, 108, 43, 35, 68, 63, 1, 122, 78, 112, 90, 101, 97, 126, 109, 33, 93, 87, 71, 70, 76, 19, 28, 60, 7, 72, 88, 123, 125, 39, 51, 23, 73, 31, 115, 127, 74, 26, 113, 105, 103, 2, 8, 21, 80, 91, 99, 14, 11, 3, 30, 85, 42, 116, 106, 16, 24, 10, 29, 102, 36, 44, 27, 32, 66, 57, 65, 67, 83, 64, 0, 5, 69], [104, 34, 119, 89, 22, 92, 18, 49, 20, 124, 95, 62, 48, 79, 77, 46, 25, 40, 114, 75, 121, 118, 120, 109, 41, 17, 54, 97, 12, 50, 111, 93, 9, 58, 16, 98, 82, 31, 112, 84, 4, 59, 47, 100, 117, 107, 80, 45, 61, 33, 53, 103, 108, 56, 38, 94, 21, 60, 32, 43, 24, 86, 90, 123, 55, 52, 127, 81, 126, 122, 113, 76, 96, 51, 110, 19, 35, 15, 63, 99, 105, 115, 28, 102, 85, 13, 87, 39, 44, 88, 125, 91, 37, 83, 27, 6, 101, 26, 106, 29, 78, 42, 23, 30, 1, 0, 2, 73, 72, 116, 36, 68, 69, 74, 57, 10, 70, 5, 7, 14, 71, 11, 8, 67, 65, 3, 66, 64], [41, 34, 53, 88, 93, 105, 20, 17, 79, 22, 35, 77, 72, 29, 75, 48, 25, 84, 115, 6, 66, 91, 18, 27, 82, 125, 24, 51, 55, 58, 62, 127, 126, 50, 8, 89, 63, 61, 108, 68, 2, 81, 15, 124, 13, 38, 0, 4, 16, 117, 80, 43, 114, 70, 52, 11, 19, 57, 103, 44, 60, 78, 100, 47, 90, 59, 76, 73, 98, 42, 109, 46, 26, 49, 83, 23, 86, 85, 40, 31, 112, 92, 33, 96, 28, 12, 10, 113, 14, 116, 87, 45, 21, 107, 39, 118, 101, 37, 122, 121, 64, 102, 97, 9, 36, 94, 104, 106, 119, 30, 123, 110, 7, 32, 74, 95, 99, 56, 69, 120, 71, 54, 5, 1, 111, 3, 65, 67], [41, 34, 53, 22, 105, 20, 88, 93, 48, 17, 29, 79, 50, 77, 44, 126, 62, 127, 27, 108, 35, 115, 58, 84, 80, 60, 96, 117, 43, 38, 125, 72, 112, 114, 75, 31, 49, 57, 124, 86, 24, 101, 21, 15, 118, 55, 61, 54, 123, 52, 63, 99, 26, 121, 90, 91, 8, 51, 37, 82, 103, 30, 39, 56, 28, 106, 122, 87, 6, 92, 18, 25, 47, 119, 81, 113, 40, 36, 120, 111, 70, 32, 102, 83, 110, 46, 16, 95, 33, 107, 85, 45, 23, 100, 89, 94, 97, 66, 104, 42, 71, 116, 12, 19, 73, 59, 109, 13, 78, 0, 14, 98, 10, 11, 7, 76, 9, 74, 67, 5, 68, 4, 2, 3, 69, 64, 65, 1], [41, 34, 53, 105, 22, 93, 88, 20, 27, 66, 91, 0, 79, 77, 17, 68, 127, 75, 19, 82, 72, 58, 112, 2, 67, 43, 124, 115, 48, 126, 44, 64, 35, 62, 70, 18, 65, 61, 29, 1, 60, 59, 6, 49, 38, 71, 51, 57, 84, 80, 50, 125, 3, 63, 52, 9, 96, 16, 69, 108, 109, 45, 55, 118, 8, 7, 31, 47, 15, 114, 110, 23, 5, 119, 101, 120, 99, 113, 117, 76, 46, 24, 73, 40, 12, 11, 37, 103, 4, 122, 25, 123, 74, 107, 10, 100, 116, 39, 32, 106, 87, 83, 26, 78, 13, 90, 85, 42, 56, 33, 102, 86, 81, 121, 54, 14, 97, 36, 28, 98, 104, 21, 111, 92, 89, 95, 94, 30], [41, 34, 53, 22, 88, 105, 17, 20, 58, 91, 93, 27, 29, 127, 115, 77, 126, 43, 75, 79, 61, 51, 48, 63, 35, 124, 108, 62, 112, 114, 55, 38, 57, 31, 81, 84, 44, 50, 113, 70, 123, 49, 82, 15, 25, 122, 59, 66, 52, 45, 40, 86, 39, 60, 118, 32, 106, 100, 119, 107, 116, 72, 80, 109, 101, 24, 0, 110, 125, 42, 117, 47, 37, 18, 96, 103, 46, 120, 11, 56, 19, 26, 102, 16, 54, 104, 121, 13, 99, 111, 23, 33, 83, 90, 98, 36, 92, 6, 78, 28, 21, 67, 68, 73, 95, 30, 97, 94, 87, 10, 76, 89, 12, 14, 85, 9, 71, 74, 4, 5, 8, 1, 69, 3, 2, 7, 65, 64]], "model.layers.12.self_attn.k_proj": [[38, 49, 127, 31, 113, 24, 21, 83, 80, 28, 82, 77, 35, 72, 108, 74, 62, 106, 105, 66, 96, 64, 1, 90, 112, 30, 40, 25, 102, 16, 107, 6, 51, 37, 34, 57, 116, 75, 86, 60, 125, 48, 45, 109, 76, 58, 91, 55, 44, 27, 117, 126, 29, 120, 119, 5, 46, 124, 59, 42, 67, 4, 87, 111, 85, 122, 47, 94, 23, 114, 26, 73, 84, 56, 43, 17, 63, 118, 115, 14, 79, 0, 104, 71, 98, 54, 41, 61, 69, 50, 78, 39, 12, 52, 110, 36, 92, 100, 53, 32, 123, 70, 103, 11, 121, 20, 15, 93, 97, 68, 89, 18, 101, 88, 99, 22, 81, 33, 19, 65, 10, 9, 7, 3, 8, 13, 2, 95], [109, 37, 45, 28, 24, 21, 31, 26, 82, 83, 78, 16, 62, 115, 75, 116, 124, 117, 49, 113, 121, 53, 110, 57, 30, 63, 8, 41, 105, 23, 64, 61, 127, 51, 60, 55, 126, 2, 103, 120, 33, 40, 90, 15, 93, 112, 42, 100, 118, 125, 107, 44, 111, 77, 108, 54, 34, 59, 17, 50, 18, 46, 43, 39, 56, 47, 22, 58, 5, 114, 85, 73, 65, 86, 48, 87, 38, 123, 97, 99, 95, 52, 27, 19, 102, 81, 104, 9, 106, 94, 68, 92, 13, 119, 96, 29, 98, 3, 80, 4, 20, 76, 32, 36, 12, 25, 122, 71, 35, 79, 89, 91, 84, 72, 10, 70, 66, 11, 74, 1, 7, 69, 14, 6, 67, 88, 0, 101], [107, 33, 43, 95, 22, 91, 24, 15, 26, 50, 115, 83, 13, 85, 118, 81, 20, 48, 61, 18, 49, 75, 63, 100, 117, 47, 126, 110, 9, 36, 46, 70, 54, 42, 112, 127, 124, 64, 39, 93, 116, 58, 108, 2, 121, 51, 101, 113, 111, 114, 68, 72, 123, 71, 57, 119, 120, 60, 12, 102, 125, 55, 65, 56, 99, 38, 44, 59, 40, 109, 19, 122, 45, 104, 52, 62, 41, 98, 103, 96, 53, 106, 14, 37, 105, 87, 11, 82, 92, 94, 3, 32, 30, 34, 29, 88, 74, 27, 35, 23, 76, 28, 21, 5, 78, 80, 25, 89, 16, 77, 79, 90, 31, 10, 97, 8, 86, 4, 1, 84, 17, 6, 67, 73, 7, 66, 69, 0], [113, 57, 102, 33, 82, 29, 23, 16, 11, 0, 3, 78, 65, 6, 77, 22, 26, 38, 59, 2, 71, 125, 108, 126, 73, 95, 83, 64, 63, 53, 84, 62, 93, 58, 72, 85, 12, 118, 114, 28, 79, 49, 89, 40, 56, 5, 60, 45, 61, 36, 124, 39, 68, 88, 123, 15, 96, 30, 119, 127, 117, 112, 106, 74, 116, 115, 52, 81, 94, 122, 111, 120, 27, 86, 121, 47, 100, 44, 31, 46, 4, 32, 34, 55, 109, 43, 91, 50, 103, 24, 51, 13, 37, 54, 35, 99, 41, 92, 19, 76, 48, 110, 105, 97, 98, 21, 17, 80, 20, 9, 42, 104, 107, 87, 1, 8, 69, 10, 25, 67, 14, 101, 75, 70, 90, 18, 7, 66], [103, 62, 48, 21, 81, 93, 14, 76, 11, 80, 61, 73, 71, 69, 3, 87, 0, 2, 65, 56, 45, 24, 8, 41, 70, 27, 64, 7, 66, 112, 6, 18, 117, 125, 32, 116, 54, 95, 119, 10, 94, 121, 13, 37, 33, 25, 47, 63, 74, 118, 1, 35, 51, 5, 72, 83, 4, 108, 97, 111, 99, 55, 67, 122, 50, 9, 105, 19, 75, 115, 68, 59, 15, 86, 120, 114, 31, 12, 113, 44, 43, 100, 17, 42, 107, 57, 126, 30, 28, 78, 104, 101, 22, 88, 92, 96, 84, 38, 26, 109, 16, 49, 106, 102, 60, 77, 20, 52, 79, 98, 85, 29, 123, 58, 53, 110, 127, 36, 89, 124, 23, 46, 40, 90, 34, 91, 82, 39], [41, 98, 26, 88, 18, 45, 84, 58, 16, 86, 112, 44, 111, 119, 109, 78, 40, 60, 75, 105, 77, 63, 50, 55, 96, 71, 121, 92, 73, 0, 104, 30, 2, 124, 122, 43, 4, 57, 56, 36, 49, 65, 39, 115, 125, 123, 29, 107, 101, 53, 31, 126, 110, 51, 62, 120, 85, 34, 113, 127, 69, 17, 74, 76, 47, 48, 68, 52, 42, 114, 79, 102, 100, 70, 61, 87, 38, 118, 59, 19, 89, 91, 28, 106, 94, 37, 6, 15, 103, 27, 117, 54, 9, 93, 35, 83, 108, 97, 32, 46, 11, 72, 20, 95, 116, 67, 33, 22, 23, 99, 25, 8, 81, 13, 90, 80, 10, 21, 5, 24, 12, 14, 64, 82, 66, 1, 3, 7], [40, 98, 89, 18, 22, 43, 119, 20, 79, 124, 48, 92, 49, 77, 112, 50, 17, 75, 31, 62, 61, 110, 9, 12, 6, 109, 60, 72, 117, 53, 113, 118, 54, 45, 68, 111, 107, 59, 41, 46, 84, 34, 58, 121, 64, 120, 14, 114, 47, 63, 35, 115, 56, 127, 65, 74, 44, 96, 83, 1, 100, 97, 66, 13, 70, 123, 28, 94, 104, 93, 55, 85, 33, 16, 36, 26, 2, 38, 3, 23, 67, 42, 102, 8, 103, 105, 11, 126, 29, 88, 24, 90, 81, 52, 39, 91, 122, 125, 101, 30, 51, 32, 57, 76, 116, 106, 7, 108, 37, 10, 27, 86, 99, 80, 69, 78, 73, 71, 15, 21, 0, 5, 4, 25, 19, 95, 87, 82], [105, 98, 53, 20, 88, 29, 17, 79, 22, 77, 75, 124, 61, 41, 64, 51, 126, 44, 6, 72, 119, 2, 80, 82, 59, 91, 112, 60, 118, 38, 63, 4, 108, 117, 127, 55, 58, 43, 45, 50, 62, 57, 35, 115, 125, 73, 101, 122, 12, 111, 110, 49, 46, 28, 123, 107, 42, 48, 32, 104, 19, 26, 47, 10, 40, 109, 66, 113, 34, 52, 78, 103, 114, 120, 39, 87, 56, 96, 90, 21, 86, 9, 102, 95, 116, 5, 31, 37, 36, 65, 97, 25, 106, 54, 121, 71, 67, 81, 30, 100, 33, 74, 99, 89, 8, 27, 92, 76, 1, 94, 69, 70, 85, 23, 11, 93, 7, 68, 24, 16, 15, 18, 3, 83, 14, 13, 84, 0]], "model.layers.12.self_attn.qk_proj": [[62, 49, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 88, 40, 93, 53, 24, 18, 34, 82, 86, 90, 21, 104, 58, 124, 98, 75, 97, 22, 85, 80, 20, 16, 11, 84, 95, 92, 112, 29, 77, 31, 78, 61, 13, 14, 81, 17, 119, 103, 28, 87, 26, 25, 27, 19, 73, 15, 79, 126, 33, 83, 121, 37, 89, 12, 23, 9, 50, 60, 118, 76, 6, 116, 44, 55, 115, 96, 0, 51, 108, 56, 101, 111, 125, 7, 39, 110, 64, 91, 59, 63, 71, 35, 47, 122, 117, 72, 8, 3, 1, 2, 65, 69, 66, 52, 100, 114, 36, 70, 46, 120, 5, 123, 94, 30, 67, 32, 10, 54, 42, 99, 4, 68, 106, 74], [49, 62, 105, 41, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 88, 53, 24, 86, 124, 90, 21, 58, 104, 18, 82, 11, 75, 97, 112, 98, 29, 31, 92, 22, 20, 95, 80, 84, 77, 85, 16, 13, 78, 61, 119, 14, 28, 87, 26, 17, 25, 27, 103, 81, 73, 19, 15, 9, 23, 6, 60, 126, 111, 83, 89, 33, 79, 118, 50, 55, 108, 115, 125, 37, 121, 39, 12, 44, 0, 76, 101, 110, 72, 59, 51, 47, 63, 116, 7, 35, 117, 96, 64, 56, 71, 1, 123, 36, 122, 2, 91, 94, 65, 66, 32, 100, 3, 5, 30, 46, 4, 114, 8, 120, 106, 42, 10, 54, 69, 67, 70, 52, 99, 68, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 93, 40, 53, 88, 34, 24, 104, 58, 90, 86, 124, 112, 95, 21, 98, 20, 22, 18, 82, 31, 29, 97, 92, 61, 84, 85, 75, 80, 11, 119, 77, 16, 78, 103, 26, 28, 27, 13, 87, 126, 14, 33, 37, 17, 25, 23, 81, 115, 121, 44, 110, 118, 51, 50, 73, 59, 35, 60, 19, 125, 6, 116, 55, 15, 108, 117, 83, 89, 79, 9, 39, 101, 76, 12, 111, 30, 63, 0, 36, 47, 56, 96, 72, 7, 91, 65, 122, 114, 66, 100, 3, 46, 71, 70, 64, 120, 123, 52, 1, 94, 32, 5, 106, 4, 69, 2, 42, 54, 67, 10, 68, 99, 8, 74], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 24, 93, 88, 34, 90, 58, 86, 124, 112, 82, 104, 18, 21, 20, 98, 80, 31, 95, 29, 22, 77, 97, 92, 75, 16, 85, 11, 84, 119, 14, 61, 103, 78, 25, 13, 28, 17, 27, 121, 79, 23, 87, 126, 44, 26, 81, 33, 50, 60, 51, 111, 118, 9, 83, 108, 15, 64, 89, 73, 101, 12, 115, 19, 39, 59, 63, 116, 37, 35, 55, 47, 70, 72, 125, 56, 110, 6, 91, 117, 76, 114, 0, 65, 30, 100, 7, 71, 123, 96, 2, 42, 66, 94, 120, 36, 67, 32, 3, 1, 122, 4, 54, 46, 52, 68, 69, 106, 5, 99, 8, 74, 10], [62, 49, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 88, 24, 58, 90, 93, 82, 124, 18, 104, 86, 80, 20, 75, 112, 98, 21, 31, 11, 16, 77, 84, 85, 61, 22, 13, 29, 92, 78, 97, 14, 95, 119, 103, 81, 26, 17, 25, 87, 28, 9, 23, 15, 79, 70, 83, 126, 121, 73, 27, 33, 50, 118, 111, 39, 19, 12, 0, 55, 89, 37, 7, 64, 72, 110, 76, 108, 59, 35, 2, 51, 44, 125, 116, 60, 47, 101, 91, 115, 6, 117, 1, 30, 56, 71, 120, 63, 66, 114, 123, 67, 122, 3, 42, 36, 65, 5, 96, 68, 69, 100, 52, 54, 46, 4, 94, 32, 106, 8, 99, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 40, 102, 53, 24, 34, 88, 90, 18, 86, 58, 75, 93, 21, 82, 20, 104, 124, 80, 77, 11, 98, 84, 31, 16, 85, 13, 78, 22, 14, 112, 92, 97, 61, 29, 119, 95, 17, 25, 81, 103, 28, 26, 87, 73, 15, 27, 9, 19, 121, 79, 83, 111, 12, 76, 70, 23, 89, 47, 50, 126, 37, 116, 55, 39, 72, 108, 118, 60, 56, 33, 115, 125, 117, 0, 44, 59, 35, 7, 64, 110, 51, 91, 65, 114, 94, 123, 63, 46, 66, 101, 2, 71, 96, 1, 6, 54, 52, 68, 120, 5, 122, 30, 99, 42, 36, 69, 67, 3, 100, 4, 106, 8, 74, 32, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 88, 24, 53, 90, 18, 34, 86, 21, 82, 93, 20, 80, 104, 75, 58, 124, 84, 77, 22, 98, 16, 85, 92, 11, 13, 61, 29, 78, 31, 112, 14, 97, 17, 119, 81, 95, 23, 87, 26, 79, 28, 103, 25, 76, 15, 83, 27, 33, 121, 73, 9, 44, 50, 89, 12, 70, 19, 39, 126, 111, 37, 118, 55, 110, 116, 108, 60, 72, 51, 115, 47, 56, 117, 7, 59, 64, 122, 35, 101, 96, 63, 52, 94, 36, 91, 125, 67, 120, 0, 42, 1, 123, 71, 100, 66, 65, 2, 30, 46, 54, 3, 69, 114, 8, 5, 6, 4, 32, 99, 74, 106, 68, 10], [49, 62, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 88, 40, 102, 53, 34, 93, 24, 90, 18, 21, 86, 82, 104, 20, 85, 80, 22, 75, 84, 92, 77, 11, 16, 124, 98, 31, 58, 78, 97, 13, 29, 14, 61, 95, 119, 112, 17, 87, 81, 15, 28, 19, 23, 26, 79, 25, 33, 44, 70, 121, 103, 115, 83, 89, 12, 9, 27, 73, 118, 50, 111, 37, 126, 76, 116, 117, 110, 55, 7, 60, 0, 51, 108, 47, 59, 64, 63, 35, 8, 72, 125, 122, 39, 71, 91, 3, 120, 2, 65, 101, 36, 94, 100, 66, 114, 46, 54, 5, 96, 30, 56, 1, 123, 32, 6, 67, 52, 68, 42, 99, 69, 106, 10, 4, 74], [49, 62, 105, 41, 57, 113, 48, 127, 109, 45, 43, 107, 38, 40, 102, 93, 53, 88, 24, 34, 90, 18, 20, 86, 104, 85, 82, 21, 98, 22, 29, 92, 58, 80, 16, 31, 61, 124, 119, 112, 84, 75, 95, 78, 11, 77, 97, 14, 25, 23, 17, 26, 81, 28, 13, 87, 19, 121, 79, 44, 103, 15, 115, 111, 126, 37, 89, 33, 27, 9, 73, 110, 108, 50, 60, 39, 83, 47, 76, 96, 51, 55, 125, 70, 94, 12, 116, 63, 118, 64, 117, 59, 7, 35, 122, 30, 0, 91, 2, 36, 6, 56, 8, 42, 67, 66, 54, 1, 71, 114, 101, 65, 52, 46, 100, 120, 106, 3, 123, 72, 4, 32, 68, 99, 5, 69, 74, 10], [49, 62, 41, 57, 105, 113, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 88, 24, 93, 34, 18, 90, 58, 104, 86, 85, 82, 22, 20, 75, 21, 16, 124, 92, 31, 77, 61, 80, 29, 78, 84, 112, 119, 11, 13, 95, 14, 97, 98, 81, 25, 23, 28, 26, 19, 15, 17, 87, 103, 27, 9, 121, 79, 126, 33, 83, 44, 89, 73, 50, 111, 60, 51, 47, 115, 108, 37, 8, 118, 6, 70, 76, 55, 12, 110, 125, 63, 7, 39, 71, 64, 0, 96, 67, 120, 36, 101, 30, 66, 59, 94, 4, 91, 114, 56, 117, 1, 116, 65, 123, 35, 32, 2, 3, 42, 54, 68, 5, 100, 122, 46, 106, 69, 74, 99, 72, 52, 10], [49, 62, 105, 41, 57, 113, 48, 45, 127, 43, 109, 107, 38, 102, 53, 40, 34, 24, 93, 88, 18, 82, 86, 58, 21, 90, 104, 75, 80, 16, 31, 77, 124, 61, 85, 20, 78, 22, 98, 112, 29, 92, 97, 11, 95, 84, 14, 119, 13, 17, 81, 25, 26, 6, 103, 9, 87, 28, 126, 23, 19, 27, 15, 79, 76, 83, 64, 8, 50, 73, 118, 89, 33, 115, 121, 111, 12, 37, 47, 7, 2, 116, 60, 71, 0, 51, 63, 120, 66, 91, 125, 1, 110, 44, 70, 114, 35, 67, 108, 55, 56, 5, 65, 101, 4, 30, 39, 59, 96, 123, 3, 122, 117, 36, 68, 69, 72, 42, 54, 94, 32, 52, 46, 74, 100, 99, 106, 10], [49, 62, 41, 105, 113, 57, 48, 109, 127, 45, 43, 107, 38, 40, 102, 53, 88, 34, 24, 93, 86, 18, 90, 58, 21, 82, 80, 75, 104, 77, 22, 16, 20, 98, 84, 92, 85, 78, 11, 31, 29, 112, 124, 61, 13, 14, 97, 95, 17, 81, 119, 26, 28, 103, 6, 15, 25, 87, 9, 73, 19, 76, 23, 12, 83, 89, 79, 126, 37, 121, 50, 111, 27, 8, 33, 64, 60, 44, 47, 0, 39, 71, 51, 115, 118, 2, 120, 55, 7, 66, 56, 108, 116, 125, 59, 35, 91, 3, 94, 65, 122, 63, 70, 123, 5, 117, 110, 72, 96, 1, 114, 36, 30, 101, 42, 69, 46, 100, 4, 67, 74, 52, 54, 68, 99, 32, 106, 10], [49, 62, 41, 105, 113, 57, 48, 127, 109, 45, 43, 107, 38, 102, 40, 53, 24, 93, 88, 104, 18, 34, 86, 21, 82, 90, 20, 22, 58, 85, 124, 80, 84, 92, 77, 31, 16, 98, 29, 11, 78, 75, 61, 95, 112, 97, 17, 119, 13, 14, 81, 26, 15, 25, 28, 121, 19, 89, 87, 23, 83, 37, 103, 79, 33, 126, 50, 27, 44, 9, 12, 6, 60, 111, 110, 108, 39, 73, 55, 115, 35, 118, 76, 51, 63, 8, 47, 91, 125, 101, 30, 116, 56, 123, 59, 117, 7, 122, 96, 94, 36, 114, 100, 64, 42, 2, 120, 0, 65, 54, 71, 52, 3, 1, 67, 66, 68, 70, 32, 99, 5, 4, 106, 72, 46, 69, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 34, 102, 24, 88, 104, 18, 82, 93, 21, 90, 75, 20, 77, 11, 86, 80, 58, 124, 98, 78, 22, 13, 84, 85, 31, 61, 112, 14, 16, 97, 92, 29, 95, 17, 119, 81, 25, 6, 15, 9, 19, 126, 83, 79, 87, 27, 89, 121, 26, 28, 73, 23, 103, 8, 50, 60, 118, 111, 0, 108, 33, 44, 7, 115, 64, 76, 12, 110, 125, 66, 51, 55, 47, 71, 2, 116, 70, 37, 56, 39, 91, 1, 117, 101, 30, 123, 63, 36, 65, 68, 35, 122, 96, 5, 114, 120, 59, 67, 4, 54, 72, 52, 94, 69, 42, 100, 3, 46, 32, 74, 99, 10, 106], [49, 62, 41, 105, 113, 57, 48, 127, 45, 109, 43, 107, 38, 53, 102, 40, 34, 93, 88, 24, 104, 18, 82, 90, 58, 86, 21, 20, 75, 31, 98, 124, 80, 77, 16, 11, 78, 84, 29, 92, 95, 112, 85, 61, 22, 97, 13, 14, 81, 17, 119, 9, 103, 28, 25, 121, 126, 26, 23, 15, 83, 33, 73, 27, 79, 44, 87, 89, 19, 37, 50, 110, 108, 60, 6, 118, 12, 111, 116, 51, 8, 115, 47, 0, 55, 39, 56, 70, 76, 35, 71, 30, 63, 7, 101, 91, 125, 96, 123, 66, 120, 59, 64, 117, 65, 54, 94, 114, 100, 72, 36, 32, 46, 122, 5, 52, 69, 106, 2, 1, 74, 67, 4, 3, 42, 99, 68, 10], [49, 62, 105, 41, 113, 57, 48, 109, 43, 127, 45, 107, 38, 102, 40, 53, 34, 93, 58, 24, 18, 88, 82, 104, 21, 86, 90, 20, 75, 98, 124, 61, 16, 80, 22, 85, 78, 84, 11, 31, 77, 92, 17, 95, 29, 112, 13, 97, 14, 81, 121, 119, 103, 15, 26, 87, 9, 126, 73, 27, 23, 44, 25, 79, 111, 83, 19, 12, 60, 76, 70, 28, 50, 115, 89, 33, 37, 59, 108, 64, 118, 110, 7, 0, 39, 116, 51, 123, 47, 125, 66, 55, 67, 35, 8, 101, 6, 120, 71, 117, 63, 2, 91, 56, 1, 30, 54, 96, 72, 65, 52, 114, 3, 36, 4, 122, 94, 42, 69, 5, 100, 32, 46, 68, 74, 10, 99, 106], [49, 62, 41, 105, 57, 113, 45, 48, 43, 127, 109, 107, 38, 40, 53, 102, 34, 88, 93, 18, 24, 90, 82, 104, 86, 22, 21, 20, 29, 75, 61, 80, 98, 85, 58, 16, 84, 95, 92, 77, 31, 11, 97, 112, 124, 78, 17, 14, 81, 13, 28, 119, 79, 15, 87, 23, 25, 26, 73, 103, 126, 83, 19, 50, 121, 108, 9, 33, 70, 89, 27, 111, 37, 51, 60, 118, 39, 76, 44, 12, 115, 117, 116, 91, 101, 110, 55, 30, 125, 63, 120, 47, 72, 7, 0, 114, 123, 54, 71, 56, 122, 59, 2, 1, 35, 42, 36, 5, 100, 65, 96, 64, 4, 32, 46, 94, 66, 8, 3, 68, 6, 52, 67, 99, 10, 69, 74, 106], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 34, 93, 88, 24, 18, 82, 21, 104, 86, 90, 20, 98, 11, 58, 75, 22, 80, 16, 31, 95, 92, 97, 84, 124, 112, 29, 85, 77, 61, 13, 14, 81, 17, 119, 78, 26, 103, 87, 28, 79, 15, 121, 83, 126, 50, 25, 27, 70, 23, 33, 73, 44, 9, 89, 19, 108, 37, 60, 118, 39, 111, 110, 116, 76, 51, 47, 35, 30, 101, 71, 55, 12, 117, 64, 125, 72, 91, 115, 63, 0, 7, 59, 67, 100, 96, 114, 36, 65, 66, 56, 123, 6, 2, 122, 52, 46, 120, 32, 1, 54, 94, 106, 42, 8, 69, 68, 99, 5, 10, 4, 3, 74], [49, 41, 62, 105, 57, 113, 127, 48, 45, 109, 43, 107, 38, 40, 102, 93, 53, 88, 34, 104, 58, 24, 18, 82, 90, 20, 86, 112, 29, 98, 80, 85, 31, 97, 61, 21, 124, 84, 16, 119, 75, 95, 22, 92, 11, 77, 14, 103, 26, 81, 17, 126, 78, 87, 28, 121, 13, 25, 108, 39, 110, 118, 44, 9, 33, 27, 15, 50, 79, 89, 23, 37, 73, 70, 51, 111, 60, 63, 83, 19, 59, 125, 101, 116, 47, 55, 117, 56, 96, 94, 30, 36, 72, 35, 12, 115, 71, 91, 100, 64, 42, 76, 32, 0, 7, 123, 66, 120, 65, 122, 106, 46, 6, 67, 114, 54, 52, 1, 99, 2, 69, 3, 68, 4, 8, 10, 74, 5], [49, 62, 105, 41, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 93, 34, 88, 24, 86, 90, 18, 21, 58, 104, 82, 98, 20, 31, 80, 75, 124, 11, 29, 61, 95, 97, 22, 85, 92, 112, 84, 77, 16, 78, 13, 17, 119, 14, 28, 25, 81, 103, 121, 87, 15, 33, 27, 9, 26, 50, 19, 126, 83, 37, 111, 89, 79, 101, 73, 60, 116, 47, 118, 23, 63, 39, 72, 59, 12, 108, 76, 55, 51, 44, 70, 117, 96, 115, 110, 35, 30, 125, 0, 120, 64, 71, 7, 123, 56, 91, 114, 1, 36, 42, 6, 2, 54, 4, 94, 106, 100, 32, 52, 67, 122, 46, 65, 68, 99, 5, 69, 66, 3, 74, 8, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 43, 109, 107, 38, 40, 53, 34, 102, 93, 24, 104, 88, 86, 18, 58, 82, 90, 21, 98, 80, 20, 11, 75, 124, 97, 22, 77, 31, 92, 16, 85, 13, 29, 84, 95, 112, 78, 61, 14, 81, 17, 119, 103, 121, 73, 15, 25, 9, 27, 126, 79, 83, 87, 26, 19, 23, 110, 28, 50, 89, 76, 116, 72, 12, 111, 47, 44, 33, 118, 115, 60, 51, 37, 108, 55, 6, 39, 101, 59, 70, 71, 64, 117, 0, 123, 125, 63, 7, 35, 120, 30, 52, 36, 2, 46, 1, 54, 66, 96, 69, 91, 65, 56, 114, 3, 122, 42, 32, 67, 68, 4, 99, 10, 100, 94, 74, 8, 106, 5], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 93, 34, 104, 88, 58, 24, 86, 90, 21, 18, 98, 82, 31, 97, 84, 20, 124, 95, 11, 85, 61, 22, 16, 80, 13, 29, 112, 92, 78, 75, 77, 14, 81, 28, 26, 17, 119, 121, 103, 15, 23, 126, 87, 27, 25, 110, 44, 19, 79, 9, 89, 73, 33, 111, 59, 115, 50, 83, 6, 37, 108, 12, 55, 76, 118, 47, 116, 91, 101, 35, 60, 96, 51, 63, 117, 39, 72, 64, 0, 71, 125, 123, 7, 52, 36, 42, 32, 114, 56, 122, 94, 120, 3, 30, 65, 46, 2, 70, 66, 1, 4, 100, 106, 68, 10, 8, 54, 99, 67, 69, 5, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 102, 40, 88, 93, 34, 104, 24, 18, 86, 21, 90, 82, 58, 98, 80, 61, 20, 31, 124, 22, 85, 11, 92, 84, 29, 112, 95, 75, 13, 97, 16, 77, 119, 78, 14, 26, 17, 81, 28, 126, 15, 103, 23, 87, 83, 9, 33, 110, 6, 27, 25, 121, 111, 51, 89, 44, 118, 50, 115, 73, 79, 60, 19, 12, 37, 39, 108, 55, 76, 47, 59, 63, 0, 116, 123, 35, 101, 91, 32, 64, 72, 36, 71, 96, 7, 120, 125, 65, 117, 100, 30, 114, 2, 56, 1, 122, 66, 42, 54, 3, 94, 8, 5, 46, 4, 67, 52, 99, 106, 68, 69, 70, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 109, 127, 43, 107, 38, 53, 40, 102, 34, 24, 88, 93, 104, 58, 18, 82, 86, 98, 90, 11, 31, 80, 21, 20, 85, 124, 75, 92, 77, 112, 97, 22, 16, 84, 95, 13, 29, 61, 78, 81, 14, 28, 119, 103, 26, 17, 25, 9, 6, 126, 87, 73, 111, 79, 23, 121, 83, 15, 19, 33, 27, 89, 50, 115, 37, 118, 47, 110, 116, 55, 108, 60, 76, 123, 12, 44, 59, 39, 63, 117, 7, 72, 35, 101, 51, 56, 64, 125, 96, 91, 71, 114, 8, 36, 120, 0, 30, 65, 2, 68, 32, 46, 66, 3, 99, 54, 4, 52, 70, 100, 94, 122, 1, 69, 5, 42, 106, 67, 74, 10], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 40, 102, 34, 93, 53, 88, 24, 18, 86, 104, 21, 90, 58, 82, 22, 20, 98, 16, 124, 61, 29, 31, 11, 97, 80, 112, 84, 75, 85, 92, 95, 77, 13, 28, 78, 119, 14, 103, 81, 26, 17, 25, 121, 73, 87, 23, 89, 9, 83, 27, 79, 12, 33, 108, 6, 126, 15, 50, 44, 19, 39, 111, 37, 118, 60, 76, 51, 110, 116, 117, 35, 55, 59, 101, 47, 7, 56, 63, 0, 123, 125, 115, 8, 122, 120, 64, 1, 96, 30, 70, 72, 2, 114, 71, 52, 91, 54, 46, 3, 36, 100, 32, 66, 42, 5, 67, 94, 106, 65, 99, 68, 69, 74, 4, 10], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 40, 34, 102, 88, 53, 93, 104, 24, 90, 18, 86, 22, 82, 58, 20, 21, 11, 31, 98, 92, 85, 112, 97, 75, 80, 16, 84, 124, 29, 61, 77, 78, 95, 13, 119, 28, 81, 14, 17, 26, 15, 25, 103, 19, 73, 23, 27, 83, 9, 89, 87, 79, 33, 126, 111, 50, 121, 60, 12, 51, 118, 37, 125, 110, 44, 70, 76, 55, 108, 6, 8, 115, 7, 39, 116, 35, 0, 101, 47, 63, 64, 59, 123, 56, 114, 117, 122, 120, 42, 71, 1, 30, 91, 96, 65, 100, 54, 52, 69, 2, 32, 67, 36, 106, 5, 94, 66, 3, 72, 46, 68, 10, 74, 4, 99], [49, 62, 41, 105, 57, 113, 48, 109, 127, 45, 43, 107, 38, 53, 102, 40, 88, 104, 24, 34, 93, 90, 18, 86, 82, 58, 98, 22, 124, 20, 29, 112, 21, 61, 85, 80, 92, 75, 31, 11, 16, 84, 77, 95, 78, 81, 97, 17, 126, 119, 13, 14, 25, 103, 28, 26, 87, 23, 83, 33, 27, 9, 73, 110, 121, 115, 79, 125, 15, 51, 89, 44, 12, 50, 108, 37, 55, 39, 60, 8, 111, 70, 47, 76, 35, 36, 19, 118, 116, 59, 117, 7, 101, 63, 6, 91, 0, 114, 100, 56, 65, 64, 42, 122, 66, 2, 32, 71, 94, 96, 67, 120, 123, 30, 54, 68, 1, 52, 69, 3, 46, 5, 106, 4, 10, 99, 72, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 53, 40, 102, 88, 24, 34, 18, 104, 93, 82, 90, 86, 75, 58, 11, 80, 21, 98, 124, 20, 85, 16, 61, 31, 22, 77, 92, 78, 97, 84, 13, 112, 29, 95, 81, 17, 14, 119, 70, 9, 103, 28, 126, 83, 25, 26, 73, 64, 87, 79, 121, 23, 115, 15, 27, 19, 33, 8, 89, 76, 12, 7, 118, 44, 111, 51, 55, 0, 116, 50, 2, 110, 66, 101, 60, 59, 125, 108, 117, 47, 68, 67, 65, 37, 56, 6, 39, 71, 123, 63, 3, 1, 91, 5, 52, 35, 96, 122, 36, 114, 100, 30, 46, 4, 69, 54, 120, 94, 72, 10, 106, 32, 42, 74, 99], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 34, 93, 88, 58, 24, 104, 21, 124, 11, 86, 90, 18, 98, 75, 84, 82, 29, 61, 22, 31, 95, 92, 112, 80, 20, 85, 78, 97, 77, 16, 13, 26, 17, 14, 103, 119, 81, 23, 121, 25, 70, 15, 73, 28, 27, 9, 126, 8, 87, 89, 37, 19, 50, 12, 44, 47, 33, 51, 125, 83, 116, 115, 7, 110, 118, 60, 108, 79, 111, 59, 0, 39, 35, 55, 101, 76, 63, 64, 56, 91, 114, 2, 96, 1, 117, 30, 67, 100, 36, 71, 66, 52, 46, 122, 3, 123, 120, 4, 32, 69, 65, 54, 6, 5, 42, 94, 68, 99, 106, 72, 10, 74], [49, 62, 41, 105, 57, 113, 48, 45, 127, 109, 43, 107, 38, 102, 40, 24, 53, 88, 93, 34, 18, 90, 82, 86, 11, 58, 75, 21, 20, 104, 98, 22, 16, 80, 124, 85, 31, 92, 29, 77, 13, 84, 97, 112, 78, 61, 95, 14, 17, 81, 19, 119, 25, 103, 9, 121, 26, 87, 73, 15, 28, 70, 126, 23, 79, 89, 37, 50, 33, 44, 8, 27, 12, 116, 111, 83, 76, 101, 60, 115, 0, 47, 118, 125, 55, 7, 59, 56, 117, 64, 51, 108, 66, 96, 39, 110, 65, 63, 2, 123, 35, 71, 94, 5, 114, 1, 91, 67, 30, 6, 3, 54, 46, 120, 122, 68, 69, 74, 72, 4, 100, 42, 32, 36, 99, 52, 106, 10], [49, 62, 41, 105, 113, 57, 48, 45, 127, 109, 43, 107, 38, 40, 102, 53, 88, 93, 24, 58, 34, 82, 90, 18, 104, 124, 21, 20, 112, 11, 86, 85, 16, 22, 77, 97, 75, 98, 80, 92, 84, 95, 13, 61, 31, 14, 103, 29, 78, 17, 81, 119, 15, 28, 19, 23, 121, 115, 25, 9, 83, 126, 27, 73, 79, 70, 87, 26, 116, 37, 12, 44, 89, 33, 50, 59, 51, 39, 60, 108, 35, 111, 101, 118, 125, 76, 55, 7, 8, 110, 64, 6, 47, 117, 63, 114, 0, 56, 123, 91, 2, 96, 122, 46, 65, 69, 54, 71, 30, 72, 100, 1, 52, 66, 68, 36, 67, 3, 4, 42, 120, 94, 5, 32, 99, 106, 10, 74], [62, 49, 41, 105, 57, 113, 48, 127, 45, 109, 43, 107, 38, 102, 40, 53, 88, 34, 24, 93, 82, 18, 90, 86, 75, 104, 21, 77, 11, 20, 58, 22, 85, 124, 16, 98, 31, 84, 13, 97, 95, 92, 112, 80, 61, 29, 14, 78, 17, 81, 28, 26, 119, 79, 87, 9, 15, 19, 115, 103, 73, 83, 126, 25, 12, 37, 23, 27, 33, 121, 50, 108, 89, 70, 76, 7, 118, 111, 8, 6, 44, 51, 60, 63, 116, 59, 47, 72, 39, 110, 71, 96, 0, 101, 123, 91, 55, 117, 35, 64, 68, 56, 125, 114, 2, 1, 100, 66, 46, 42, 36, 5, 122, 65, 67, 94, 10, 120, 30, 54, 4, 74, 52, 32, 69, 99, 3, 106]], "model.layers.13.self_attn.q_proj": [[115, 103, 62, 124, 126, 50, 93, 54, 90, 20, 89, 13, 122, 74, 83, 23, 127, 33, 7, 82, 61, 39, 48, 112, 80, 56, 22, 25, 2, 91, 92, 29, 31, 0, 53, 52, 67, 117, 116, 119, 51, 120, 77, 4, 65, 121, 6, 5, 87, 69, 125, 21, 15, 12, 100, 114, 64, 118, 60, 59, 66, 49, 3, 16, 95, 34, 55, 37, 28, 19, 111, 17, 18, 68, 58, 99, 85, 72, 63, 32, 57, 81, 44, 9, 70, 30, 84, 36, 88, 101, 96, 24, 113, 45, 11, 109, 35, 94, 102, 98, 26, 123, 38, 27, 14, 76, 78, 73, 8, 47, 107, 86, 43, 46, 41, 40, 42, 105, 110, 108, 79, 71, 75, 104, 97, 1, 106, 10], [115, 62, 103, 124, 50, 54, 126, 23, 74, 67, 61, 90, 122, 127, 56, 112, 7, 53, 93, 82, 6, 39, 77, 116, 48, 2, 68, 117, 52, 0, 89, 33, 121, 114, 51, 120, 85, 15, 31, 25, 119, 49, 125, 60, 11, 111, 22, 118, 55, 63, 59, 37, 69, 58, 113, 57, 95, 44, 109, 45, 9, 110, 46, 107, 38, 35, 43, 83, 13, 65, 123, 108, 102, 47, 20, 41, 34, 42, 104, 105, 106, 100, 92, 64, 5, 98, 73, 28, 40, 96, 101, 66, 32, 99, 79, 36, 87, 18, 72, 14, 30, 29, 88, 24, 94, 21, 10, 80, 26, 16, 19, 86, 4, 70, 84, 12, 3, 91, 81, 97, 27, 76, 1, 78, 17, 75, 8, 71], [62, 115, 103, 39, 124, 127, 126, 50, 90, 61, 54, 112, 122, 23, 11, 7, 15, 20, 82, 52, 74, 53, 89, 116, 51, 119, 6, 56, 14, 48, 0, 117, 68, 67, 79, 120, 84, 121, 125, 114, 59, 80, 55, 111, 100, 118, 49, 95, 60, 72, 22, 66, 57, 33, 63, 102, 58, 113, 2, 107, 73, 43, 109, 110, 93, 34, 37, 40, 101, 31, 46, 92, 45, 35, 13, 18, 44, 28, 106, 25, 98, 81, 38, 108, 96, 123, 47, 32, 69, 41, 42, 105, 104, 99, 87, 88, 16, 24, 94, 85, 77, 83, 30, 29, 91, 26, 8, 27, 9, 86, 75, 36, 70, 12, 76, 78, 65, 97, 5, 19, 64, 10, 17, 21, 71, 4, 1, 3], [115, 62, 103, 50, 126, 124, 54, 90, 61, 122, 127, 74, 112, 67, 82, 48, 56, 7, 39, 117, 77, 52, 83, 116, 114, 93, 2, 53, 120, 125, 51, 119, 121, 18, 92, 13, 6, 111, 0, 89, 49, 59, 60, 88, 109, 55, 63, 58, 118, 33, 45, 57, 23, 11, 108, 113, 47, 21, 43, 25, 102, 107, 40, 46, 65, 44, 69, 123, 66, 64, 87, 110, 37, 15, 42, 20, 105, 35, 104, 41, 106, 5, 100, 19, 101, 68, 98, 17, 73, 38, 91, 3, 95, 96, 29, 34, 9, 85, 36, 99, 10, 31, 32, 70, 4, 28, 22, 30, 12, 16, 94, 72, 26, 27, 81, 24, 1, 97, 84, 76, 86, 80, 14, 78, 71, 79, 8, 75], [123, 127, 60, 62, 38, 121, 53, 56, 61, 122, 110, 120, 59, 57, 54, 52, 44, 114, 119, 51, 125, 118, 116, 91, 117, 124, 126, 55, 115, 48, 58, 47, 113, 49, 90, 50, 111, 63, 102, 82, 46, 108, 109, 32, 112, 14, 104, 45, 29, 100, 84, 34, 103, 99, 107, 37, 86, 43, 97, 105, 39, 22, 42, 106, 36, 24, 40, 33, 89, 98, 41, 94, 101, 27, 30, 23, 75, 20, 88, 35, 96, 18, 25, 26, 95, 93, 80, 31, 78, 19, 28, 8, 9, 21, 87, 11, 81, 83, 64, 68, 85, 92, 13, 66, 17, 1, 12, 76, 77, 16, 79, 70, 71, 5, 15, 72, 3, 7, 10, 65, 6, 67, 73, 4, 0, 69, 2, 74], [127, 123, 38, 121, 62, 60, 37, 53, 56, 61, 122, 57, 120, 54, 116, 51, 59, 114, 52, 58, 44, 32, 119, 125, 36, 118, 124, 117, 49, 126, 115, 90, 110, 55, 108, 47, 50, 111, 113, 63, 89, 107, 82, 109, 91, 112, 106, 100, 48, 87, 14, 46, 84, 102, 42, 96, 45, 103, 34, 27, 25, 20, 22, 104, 40, 78, 8, 97, 94, 39, 31, 43, 95, 98, 29, 105, 101, 41, 30, 33, 99, 86, 1, 35, 75, 9, 81, 18, 26, 19, 93, 92, 24, 28, 5, 12, 68, 3, 71, 76, 64, 0, 79, 13, 88, 70, 21, 72, 23, 17, 85, 10, 15, 66, 16, 67, 83, 73, 7, 6, 80, 11, 77, 4, 2, 69, 65, 74], [123, 127, 62, 60, 121, 1, 53, 61, 56, 122, 57, 115, 120, 54, 52, 63, 51, 58, 59, 32, 114, 116, 124, 113, 125, 118, 119, 117, 49, 110, 126, 107, 55, 87, 37, 0, 111, 47, 50, 48, 112, 20, 109, 106, 104, 91, 108, 64, 46, 38, 44, 45, 67, 75, 66, 41, 42, 39, 43, 100, 40, 105, 78, 82, 103, 3, 36, 26, 71, 101, 34, 22, 68, 99, 18, 95, 72, 14, 5, 98, 88, 13, 33, 28, 79, 30, 94, 35, 102, 27, 96, 16, 97, 92, 9, 8, 93, 10, 76, 21, 17, 11, 6, 83, 2, 70, 15, 29, 23, 85, 84, 86, 31, 12, 24, 90, 25, 19, 73, 65, 7, 81, 89, 4, 69, 80, 77, 74], [127, 123, 38, 24, 83, 17, 28, 79, 90, 26, 10, 97, 13, 60, 31, 86, 104, 121, 84, 76, 4, 42, 7, 29, 62, 74, 77, 85, 40, 88, 19, 2, 53, 46, 70, 92, 15, 49, 81, 89, 57, 25, 20, 91, 9, 6, 5, 71, 64, 58, 101, 21, 116, 12, 56, 33, 18, 112, 78, 80, 99, 27, 82, 16, 87, 54, 23, 1, 65, 0, 95, 102, 14, 75, 93, 32, 51, 109, 69, 67, 115, 73, 52, 61, 94, 11, 108, 96, 30, 36, 22, 68, 59, 125, 48, 122, 72, 8, 37, 34, 55, 126, 114, 124, 66, 63, 50, 120, 43, 45, 117, 119, 103, 100, 118, 44, 3, 110, 113, 47, 98, 106, 35, 41, 107, 111, 105, 39], [121, 60, 39, 123, 120, 89, 61, 54, 49, 119, 65, 51, 122, 116, 62, 78, 127, 71, 126, 59, 112, 58, 117, 92, 110, 48, 118, 11, 50, 57, 124, 80, 53, 4, 55, 18, 27, 56, 44, 47, 68, 115, 125, 23, 63, 43, 111, 96, 69, 105, 46, 90, 99, 103, 52, 113, 106, 45, 109, 9, 114, 70, 107, 0, 101, 28, 84, 104, 91, 3, 42, 108, 73, 66, 38, 40, 82, 41, 102, 22, 95, 2, 93, 94, 36, 100, 25, 37, 10, 30, 34, 97, 98, 86, 79, 16, 33, 64, 8, 5, 29, 87, 19, 14, 20, 74, 31, 7, 35, 83, 15, 12, 72, 26, 81, 88, 17, 21, 32, 75, 77, 85, 24, 1, 67, 13, 76, 6], [60, 121, 39, 89, 123, 120, 54, 61, 116, 62, 49, 119, 23, 96, 18, 126, 84, 51, 80, 122, 117, 16, 55, 27, 127, 57, 124, 48, 9, 25, 95, 86, 92, 59, 58, 118, 28, 22, 112, 53, 50, 56, 47, 15, 65, 13, 99, 78, 69, 110, 63, 115, 52, 106, 7, 34, 94, 30, 103, 19, 111, 75, 82, 125, 105, 11, 97, 32, 44, 43, 114, 10, 46, 45, 108, 113, 21, 104, 71, 20, 77, 109, 107, 42, 38, 40, 85, 35, 66, 41, 37, 73, 93, 87, 81, 0, 91, 100, 36, 90, 31, 98, 12, 102, 33, 26, 29, 101, 70, 2, 68, 74, 83, 24, 79, 3, 17, 88, 14, 4, 67, 76, 72, 64, 5, 8, 6, 1], [121, 39, 60, 123, 120, 89, 54, 116, 23, 61, 84, 27, 28, 11, 92, 96, 51, 49, 18, 80, 99, 16, 122, 48, 62, 75, 126, 119, 86, 124, 127, 118, 58, 59, 117, 50, 55, 112, 78, 57, 22, 56, 30, 25, 47, 110, 53, 94, 95, 82, 115, 106, 111, 29, 7, 9, 44, 114, 45, 91, 15, 109, 90, 103, 46, 63, 113, 52, 33, 105, 2, 107, 4, 20, 42, 65, 102, 81, 71, 40, 104, 101, 87, 70, 43, 19, 108, 93, 125, 77, 85, 36, 37, 98, 35, 41, 79, 3, 97, 21, 6, 32, 31, 100, 83, 0, 38, 34, 68, 69, 26, 17, 14, 73, 88, 12, 64, 24, 67, 76, 13, 10, 8, 74, 72, 5, 66, 1], [60, 121, 39, 123, 54, 84, 120, 116, 89, 61, 49, 51, 18, 16, 92, 117, 99, 28, 122, 27, 126, 82, 112, 119, 62, 127, 103, 23, 124, 48, 96, 118, 58, 59, 57, 94, 55, 53, 86, 50, 9, 88, 47, 95, 56, 52, 110, 11, 25, 115, 22, 65, 30, 71, 69, 15, 42, 114, 111, 75, 45, 46, 125, 33, 106, 102, 63, 113, 43, 78, 91, 107, 90, 101, 104, 34, 41, 44, 77, 109, 40, 38, 87, 105, 0, 13, 36, 29, 85, 31, 108, 26, 20, 7, 98, 68, 14, 37, 93, 97, 100, 24, 19, 66, 32, 35, 81, 21, 80, 17, 83, 70, 2, 79, 76, 12, 74, 73, 8, 64, 5, 72, 67, 3, 4, 10, 6, 1], [51, 118, 102, 48, 90, 24, 97, 53, 126, 61, 58, 117, 18, 121, 29, 62, 63, 54, 123, 86, 50, 19, 9, 80, 124, 26, 47, 127, 111, 93, 120, 112, 125, 38, 115, 23, 59, 116, 119, 27, 60, 91, 73, 17, 46, 55, 78, 108, 20, 113, 12, 104, 103, 114, 52, 101, 57, 71, 45, 56, 39, 89, 110, 82, 11, 87, 88, 30, 107, 36, 5, 77, 109, 66, 49, 42, 13, 44, 99, 16, 14, 33, 43, 41, 98, 83, 100, 28, 74, 122, 94, 105, 31, 34, 40, 25, 106, 37, 75, 32, 35, 96, 21, 81, 95, 85, 22, 84, 92, 15, 72, 3, 7, 68, 70, 79, 76, 64, 65, 69, 2, 1, 0, 8, 10, 4, 6, 67], [118, 51, 102, 48, 61, 126, 58, 123, 53, 124, 127, 120, 62, 117, 63, 90, 54, 60, 19, 125, 121, 119, 97, 116, 59, 41, 113, 50, 3, 56, 52, 55, 29, 112, 47, 73, 57, 0, 86, 115, 38, 93, 78, 24, 26, 111, 92, 81, 122, 46, 9, 5, 1, 109, 114, 43, 69, 87, 22, 89, 103, 94, 30, 45, 110, 108, 49, 7, 44, 66, 105, 39, 98, 107, 99, 104, 106, 91, 42, 83, 28, 2, 75, 14, 18, 4, 40, 25, 76, 37, 100, 6, 84, 65, 12, 32, 35, 96, 17, 16, 33, 82, 101, 67, 21, 13, 36, 34, 15, 8, 85, 77, 11, 64, 31, 20, 88, 80, 79, 95, 71, 23, 10, 74, 27, 70, 72, 68], [51, 102, 118, 48, 24, 97, 53, 90, 19, 58, 126, 47, 63, 61, 86, 81, 38, 18, 121, 117, 93, 123, 29, 62, 60, 124, 80, 9, 50, 99, 89, 59, 112, 54, 127, 120, 45, 94, 125, 108, 119, 13, 75, 100, 11, 39, 57, 111, 113, 114, 91, 104, 88, 77, 115, 26, 116, 3, 5, 109, 73, 36, 56, 37, 98, 49, 52, 110, 103, 78, 105, 107, 42, 55, 28, 66, 41, 122, 46, 30, 71, 44, 40, 83, 35, 43, 31, 34, 27, 95, 82, 22, 87, 17, 106, 85, 92, 20, 32, 101, 23, 21, 84, 69, 15, 96, 25, 16, 7, 74, 33, 14, 72, 0, 70, 67, 10, 1, 79, 64, 12, 8, 76, 65, 68, 6, 4, 2], [51, 102, 118, 48, 24, 97, 61, 86, 53, 29, 126, 19, 90, 63, 124, 115, 78, 75, 18, 58, 80, 81, 50, 123, 93, 66, 87, 62, 89, 30, 77, 39, 117, 16, 34, 99, 54, 9, 88, 32, 47, 116, 91, 45, 120, 37, 36, 125, 96, 83, 28, 111, 31, 44, 127, 3, 59, 100, 26, 121, 11, 95, 60, 103, 38, 52, 119, 23, 109, 71, 35, 112, 92, 17, 113, 5, 94, 20, 105, 27, 110, 57, 98, 101, 106, 56, 74, 114, 41, 25, 107, 70, 108, 104, 69, 46, 33, 0, 82, 55, 21, 1, 14, 72, 49, 40, 10, 85, 43, 64, 13, 73, 122, 42, 22, 7, 15, 84, 6, 68, 12, 79, 4, 76, 8, 65, 67, 2], [47, 125, 111, 56, 14, 124, 60, 24, 120, 127, 117, 54, 118, 49, 123, 5, 57, 91, 61, 75, 2, 21, 96, 59, 122, 1, 62, 94, 55, 7, 126, 63, 26, 58, 50, 121, 48, 103, 39, 53, 9, 3, 119, 64, 116, 104, 101, 98, 40, 113, 23, 4, 20, 100, 83, 42, 114, 52, 115, 112, 37, 36, 13, 82, 18, 97, 10, 17, 85, 72, 108, 44, 22, 110, 51, 89, 73, 46, 25, 27, 6, 43, 35, 19, 78, 29, 15, 107, 41, 71, 31, 45, 81, 32, 16, 79, 106, 105, 8, 28, 34, 99, 93, 109, 87, 74, 33, 88, 102, 70, 38, 92, 95, 76, 86, 80, 11, 77, 90, 30, 65, 67, 84, 0, 12, 69, 66, 68], [47, 111, 125, 56, 98, 26, 124, 2, 5, 60, 117, 103, 85, 64, 39, 36, 61, 54, 120, 127, 92, 37, 123, 49, 4, 87, 91, 97, 126, 122, 118, 1, 59, 57, 62, 20, 14, 6, 89, 48, 7, 3, 55, 24, 9, 50, 63, 96, 121, 113, 53, 65, 29, 58, 75, 13, 27, 114, 51, 101, 10, 41, 52, 81, 83, 84, 116, 94, 35, 42, 71, 72, 46, 119, 38, 112, 44, 8, 22, 115, 106, 28, 90, 100, 16, 31, 105, 99, 107, 43, 34, 109, 0, 23, 45, 95, 110, 11, 104, 108, 68, 76, 73, 86, 102, 25, 40, 32, 79, 15, 17, 66, 21, 33, 74, 80, 30, 19, 82, 93, 67, 70, 69, 77, 18, 88, 78, 12], [125, 47, 111, 56, 60, 124, 14, 39, 117, 26, 127, 118, 120, 54, 61, 49, 123, 5, 57, 103, 126, 62, 94, 59, 122, 81, 55, 63, 98, 50, 48, 23, 121, 19, 53, 20, 46, 58, 113, 116, 76, 114, 2, 8, 1, 119, 97, 112, 64, 4, 10, 75, 115, 107, 51, 52, 38, 44, 21, 72, 87, 65, 73, 16, 104, 100, 41, 3, 90, 42, 43, 45, 71, 96, 108, 110, 106, 105, 37, 99, 86, 6, 109, 17, 24, 36, 40, 22, 79, 80, 91, 28, 9, 7, 83, 101, 85, 13, 74, 34, 102, 33, 92, 18, 35, 82, 27, 95, 31, 29, 11, 78, 84, 88, 68, 30, 0, 25, 89, 77, 12, 32, 69, 93, 15, 70, 66, 67], [47, 125, 111, 36, 56, 24, 96, 124, 91, 75, 25, 120, 100, 7, 60, 49, 83, 29, 16, 99, 84, 21, 2, 54, 94, 1, 14, 127, 117, 5, 61, 57, 80, 26, 103, 40, 118, 123, 22, 97, 64, 33, 4, 46, 35, 44, 48, 59, 110, 85, 126, 62, 104, 13, 122, 9, 27, 28, 58, 89, 3, 20, 6, 31, 116, 90, 77, 53, 108, 15, 43, 93, 121, 101, 42, 39, 119, 87, 50, 55, 17, 79, 63, 88, 98, 67, 112, 95, 82, 106, 18, 92, 23, 113, 41, 34, 30, 52, 37, 38, 102, 114, 19, 71, 32, 45, 115, 105, 74, 109, 51, 107, 11, 70, 76, 78, 8, 10, 68, 12, 65, 72, 73, 81, 86, 0, 69, 66], [123, 63, 39, 114, 57, 115, 61, 127, 121, 124, 59, 62, 120, 54, 49, 93, 116, 125, 110, 113, 97, 50, 25, 53, 58, 112, 56, 55, 118, 51, 117, 92, 52, 80, 126, 60, 42, 122, 45, 48, 22, 111, 44, 28, 108, 119, 102, 46, 107, 43, 95, 41, 40, 47, 26, 103, 100, 88, 106, 19, 37, 104, 105, 34, 109, 35, 96, 17, 86, 31, 87, 36, 99, 32, 101, 94, 38, 24, 9, 29, 6, 91, 16, 3, 98, 76, 12, 72, 30, 90, 78, 33, 83, 73, 27, 68, 0, 67, 23, 18, 81, 20, 89, 85, 77, 15, 65, 2, 1, 5, 84, 79, 21, 69, 82, 11, 8, 14, 75, 70, 13, 4, 74, 66, 71, 10, 64, 7], [63, 123, 114, 39, 57, 115, 61, 59, 127, 124, 121, 62, 54, 125, 120, 116, 113, 49, 60, 117, 58, 53, 122, 88, 56, 50, 52, 48, 19, 55, 126, 51, 28, 118, 112, 119, 43, 42, 93, 45, 97, 38, 110, 46, 6, 0, 41, 85, 104, 95, 86, 102, 107, 111, 92, 47, 87, 105, 16, 106, 103, 109, 44, 76, 25, 37, 108, 81, 40, 34, 96, 35, 73, 80, 15, 101, 22, 99, 100, 36, 26, 67, 3, 83, 94, 32, 29, 90, 98, 23, 69, 24, 33, 17, 27, 30, 72, 31, 91, 5, 2, 12, 78, 65, 84, 70, 79, 9, 89, 75, 1, 20, 4, 68, 21, 14, 18, 77, 82, 74, 11, 8, 64, 10, 13, 66, 71, 7], [39, 63, 123, 25, 82, 97, 87, 13, 115, 22, 19, 114, 74, 80, 70, 57, 6, 77, 3, 66, 83, 7, 18, 64, 84, 91, 31, 4, 12, 94, 29, 27, 10, 9, 79, 93, 92, 16, 95, 24, 75, 30, 21, 90, 86, 5, 59, 85, 81, 76, 58, 104, 78, 23, 15, 20, 89, 17, 1, 127, 71, 120, 14, 73, 61, 54, 8, 88, 124, 62, 113, 116, 43, 72, 68, 109, 49, 96, 118, 99, 32, 2, 65, 28, 11, 55, 100, 34, 35, 98, 121, 48, 107, 37, 69, 45, 110, 40, 26, 41, 50, 51, 53, 122, 67, 102, 42, 126, 0, 60, 125, 46, 56, 105, 52, 36, 38, 119, 111, 112, 117, 33, 101, 108, 44, 103, 106, 47], [63, 123, 39, 115, 57, 28, 114, 80, 97, 19, 25, 22, 127, 61, 124, 87, 9, 62, 59, 96, 88, 120, 92, 58, 54, 113, 116, 104, 121, 49, 111, 3, 12, 125, 117, 53, 51, 24, 107, 95, 20, 60, 50, 31, 126, 52, 48, 15, 56, 72, 21, 77, 112, 55, 34, 122, 41, 46, 82, 118, 40, 119, 13, 42, 94, 102, 43, 37, 30, 45, 110, 99, 23, 16, 83, 93, 84, 70, 85, 29, 90, 100, 27, 91, 44, 105, 109, 78, 38, 108, 65, 68, 106, 98, 26, 35, 47, 36, 86, 76, 81, 17, 33, 8, 101, 5, 14, 79, 32, 74, 2, 89, 73, 64, 1, 7, 75, 18, 4, 103, 6, 10, 0, 69, 67, 11, 71, 66], [38, 47, 62, 111, 93, 83, 16, 26, 33, 76, 24, 9, 23, 86, 78, 92, 81, 118, 90, 68, 1, 70, 4, 34, 73, 74, 80, 29, 84, 20, 12, 87, 14, 19, 101, 2, 77, 75, 21, 71, 51, 64, 22, 85, 15, 63, 5, 10, 91, 66, 112, 95, 17, 11, 28, 82, 67, 88, 98, 36, 27, 13, 79, 7, 69, 65, 0, 53, 32, 30, 119, 18, 89, 56, 31, 52, 6, 3, 100, 8, 25, 72, 120, 104, 61, 122, 123, 114, 102, 96, 127, 110, 117, 49, 124, 55, 94, 57, 48, 115, 60, 109, 116, 50, 99, 35, 121, 106, 59, 58, 126, 46, 42, 40, 97, 125, 54, 105, 103, 44, 37, 39, 108, 113, 45, 41, 43, 107], [38, 47, 62, 111, 33, 24, 83, 93, 79, 81, 76, 100, 86, 118, 78, 29, 67, 26, 28, 16, 74, 6, 112, 32, 87, 90, 84, 119, 53, 92, 77, 34, 63, 18, 61, 120, 70, 85, 122, 56, 71, 114, 52, 13, 25, 98, 82, 23, 3, 88, 27, 19, 75, 51, 2, 20, 127, 95, 123, 22, 9, 60, 55, 121, 99, 117, 10, 17, 5, 49, 80, 115, 48, 94, 124, 0, 1, 50, 58, 113, 91, 4, 57, 89, 30, 8, 21, 15, 31, 12, 125, 68, 101, 116, 110, 14, 104, 106, 107, 59, 11, 96, 126, 73, 105, 72, 54, 37, 46, 35, 40, 7, 36, 108, 102, 45, 43, 109, 44, 66, 39, 41, 42, 103, 97, 65, 69, 64], [62, 111, 47, 38, 118, 120, 51, 63, 56, 53, 119, 100, 117, 58, 48, 122, 55, 124, 60, 123, 113, 52, 108, 116, 115, 127, 57, 112, 114, 121, 61, 49, 50, 126, 54, 18, 104, 59, 91, 125, 106, 109, 107, 110, 46, 27, 101, 28, 41, 84, 45, 44, 95, 35, 37, 105, 40, 43, 42, 103, 96, 92, 99, 36, 39, 24, 98, 33, 32, 87, 23, 21, 34, 31, 30, 93, 86, 94, 29, 89, 26, 97, 102, 82, 25, 20, 22, 90, 88, 81, 85, 79, 83, 15, 13, 77, 17, 8, 74, 67, 10, 72, 78, 64, 16, 3, 75, 19, 7, 76, 70, 69, 71, 11, 80, 14, 6, 5, 66, 0, 12, 4, 2, 65, 1, 9, 68, 73], [62, 111, 47, 118, 120, 100, 38, 56, 119, 52, 53, 112, 58, 63, 48, 57, 123, 113, 122, 55, 125, 117, 60, 115, 54, 51, 121, 114, 50, 124, 127, 61, 49, 126, 59, 116, 109, 110, 107, 36, 46, 104, 105, 106, 44, 108, 28, 45, 43, 27, 42, 41, 35, 103, 39, 91, 40, 82, 101, 84, 95, 33, 34, 92, 23, 37, 99, 102, 31, 18, 98, 93, 87, 32, 21, 30, 29, 96, 94, 86, 24, 97, 26, 22, 25, 89, 90, 67, 81, 20, 79, 6, 85, 10, 15, 12, 88, 8, 64, 72, 78, 70, 83, 19, 14, 17, 13, 66, 3, 7, 69, 77, 74, 11, 80, 16, 65, 75, 76, 0, 73, 71, 5, 2, 4, 68, 9, 1], [42, 43, 33, 72, 38, 1, 5, 91, 113, 106, 0, 21, 114, 12, 79, 24, 30, 67, 81, 19, 119, 61, 46, 124, 82, 49, 2, 107, 74, 53, 121, 122, 69, 66, 75, 71, 13, 25, 47, 112, 28, 126, 86, 77, 94, 87, 125, 45, 4, 117, 23, 3, 123, 70, 88, 50, 20, 80, 48, 65, 68, 89, 76, 63, 101, 100, 116, 8, 15, 103, 59, 27, 17, 110, 99, 7, 60, 14, 118, 11, 62, 104, 41, 64, 58, 31, 6, 115, 102, 57, 52, 92, 37, 127, 109, 51, 9, 78, 32, 22, 111, 90, 83, 56, 54, 16, 73, 26, 35, 39, 96, 55, 105, 95, 85, 84, 120, 10, 97, 98, 18, 44, 29, 36, 40, 108, 93, 34], [42, 38, 91, 33, 117, 43, 21, 82, 79, 74, 24, 107, 12, 20, 78, 10, 70, 14, 121, 89, 112, 30, 119, 81, 106, 27, 86, 19, 18, 84, 8, 88, 92, 28, 114, 6, 3, 113, 16, 94, 61, 67, 15, 87, 64, 90, 122, 80, 96, 5, 22, 23, 123, 57, 69, 83, 9, 76, 85, 25, 37, 102, 56, 40, 72, 75, 36, 93, 73, 51, 45, 13, 29, 116, 39, 32, 65, 63, 17, 77, 4, 46, 41, 100, 71, 26, 47, 35, 31, 95, 53, 124, 101, 11, 62, 105, 127, 125, 49, 7, 110, 34, 68, 99, 52, 98, 50, 126, 103, 66, 109, 59, 104, 60, 2, 48, 58, 118, 115, 44, 111, 54, 120, 55, 1, 108, 97, 0], [117, 43, 38, 42, 113, 33, 91, 107, 30, 114, 47, 123, 41, 46, 125, 62, 24, 26, 112, 39, 106, 59, 111, 60, 53, 116, 18, 115, 40, 50, 63, 87, 56, 118, 121, 119, 80, 108, 57, 89, 48, 124, 110, 51, 86, 61, 32, 109, 45, 127, 82, 52, 54, 126, 120, 55, 31, 58, 49, 28, 34, 44, 36, 81, 101, 104, 78, 95, 122, 94, 35, 96, 98, 88, 99, 19, 14, 21, 105, 37, 84, 93, 100, 29, 90, 103, 25, 20, 79, 11, 97, 23, 22, 16, 77, 27, 92, 13, 73, 83, 102, 85, 74, 17, 9, 70, 12, 76, 75, 2, 66, 8, 67, 6, 4, 71, 72, 7, 5, 0, 68, 15, 3, 10, 1, 64, 65, 69], [43, 38, 107, 33, 114, 91, 46, 47, 26, 24, 42, 117, 62, 30, 53, 56, 123, 119, 125, 39, 116, 111, 41, 59, 82, 115, 52, 90, 113, 48, 84, 118, 49, 55, 126, 50, 124, 51, 112, 122, 60, 57, 106, 121, 105, 63, 109, 120, 61, 110, 93, 44, 45, 127, 40, 54, 77, 108, 101, 34, 58, 35, 27, 88, 18, 100, 20, 37, 32, 104, 99, 98, 103, 29, 36, 31, 25, 81, 94, 86, 96, 102, 95, 28, 89, 87, 78, 22, 92, 14, 23, 76, 80, 17, 97, 75, 83, 19, 21, 16, 13, 12, 8, 79, 73, 11, 9, 6, 70, 4, 71, 67, 74, 5, 66, 72, 68, 15, 7, 0, 3, 64, 1, 69, 10, 2, 85, 65]], "model.layers.13.self_attn.k_proj": [[115, 62, 39, 124, 127, 50, 36, 61, 54, 22, 122, 112, 126, 97, 116, 52, 53, 56, 117, 51, 121, 48, 120, 119, 49, 114, 125, 55, 118, 111, 100, 57, 60, 59, 29, 58, 63, 89, 113, 91, 46, 107, 92, 109, 47, 44, 123, 45, 110, 42, 23, 105, 95, 43, 33, 108, 106, 41, 40, 37, 82, 104, 99, 101, 38, 30, 102, 20, 87, 19, 80, 78, 31, 103, 90, 88, 98, 34, 94, 35, 24, 96, 21, 1, 81, 79, 28, 32, 8, 75, 93, 26, 86, 73, 27, 76, 17, 13, 69, 10, 16, 71, 83, 14, 12, 68, 25, 84, 85, 11, 67, 15, 5, 18, 6, 66, 70, 74, 77, 4, 72, 9, 0, 7, 3, 2, 64, 65], [127, 123, 102, 86, 35, 60, 121, 58, 28, 63, 31, 51, 57, 83, 48, 99, 115, 110, 53, 24, 108, 46, 56, 17, 62, 79, 61, 125, 49, 52, 33, 116, 122, 50, 117, 114, 90, 112, 84, 54, 42, 55, 126, 40, 25, 59, 124, 29, 119, 107, 47, 44, 45, 120, 113, 109, 43, 16, 97, 111, 104, 118, 106, 34, 13, 41, 105, 76, 10, 39, 103, 38, 101, 37, 98, 14, 100, 91, 36, 32, 82, 95, 72, 96, 71, 68, 21, 81, 94, 22, 93, 30, 87, 19, 9, 20, 80, 77, 23, 92, 26, 88, 67, 11, 1, 66, 85, 89, 27, 15, 73, 12, 74, 18, 70, 75, 5, 69, 78, 6, 2, 0, 7, 8, 4, 64, 3, 65], [121, 60, 103, 123, 86, 54, 49, 35, 120, 61, 116, 32, 51, 119, 62, 122, 127, 55, 126, 118, 53, 28, 57, 59, 48, 124, 112, 50, 58, 115, 56, 110, 47, 99, 117, 63, 42, 44, 111, 114, 46, 125, 45, 52, 109, 106, 22, 113, 100, 105, 107, 30, 43, 23, 77, 108, 41, 40, 25, 84, 18, 101, 80, 96, 102, 104, 37, 91, 89, 98, 94, 90, 75, 38, 31, 36, 19, 97, 33, 15, 29, 93, 5, 16, 95, 34, 26, 88, 21, 81, 12, 17, 87, 3, 92, 13, 24, 76, 78, 6, 79, 27, 39, 85, 64, 66, 73, 8, 0, 9, 70, 83, 11, 82, 72, 74, 20, 71, 10, 14, 4, 67, 2, 7, 1, 65, 68, 69], [51, 118, 38, 86, 33, 48, 61, 53, 93, 123, 54, 120, 58, 124, 125, 117, 62, 126, 127, 59, 60, 115, 63, 47, 121, 119, 116, 55, 56, 50, 52, 57, 24, 113, 41, 112, 99, 26, 46, 91, 122, 81, 39, 104, 43, 103, 18, 111, 80, 45, 42, 11, 114, 109, 40, 19, 49, 110, 44, 105, 101, 107, 106, 20, 108, 79, 77, 36, 102, 89, 30, 90, 84, 97, 34, 21, 25, 35, 78, 27, 37, 15, 28, 6, 95, 4, 100, 98, 23, 67, 32, 88, 17, 96, 8, 9, 74, 16, 82, 72, 1, 22, 92, 31, 94, 75, 85, 87, 12, 10, 14, 65, 5, 76, 13, 64, 29, 69, 71, 70, 7, 2, 83, 66, 0, 73, 68, 3], [111, 125, 100, 47, 22, 120, 127, 118, 56, 54, 123, 60, 124, 59, 62, 117, 61, 57, 55, 122, 53, 63, 48, 126, 29, 121, 50, 49, 32, 108, 107, 119, 116, 42, 58, 113, 114, 36, 43, 115, 52, 105, 112, 51, 40, 39, 106, 41, 44, 46, 91, 110, 24, 103, 83, 45, 102, 104, 109, 25, 80, 26, 38, 79, 84, 99, 97, 92, 101, 35, 72, 37, 95, 76, 81, 18, 30, 34, 98, 33, 10, 31, 90, 4, 93, 21, 78, 77, 11, 89, 88, 86, 17, 20, 28, 71, 85, 94, 0, 82, 96, 16, 66, 13, 3, 12, 23, 27, 15, 73, 87, 65, 14, 70, 74, 6, 19, 2, 69, 75, 5, 9, 64, 7, 1, 8, 67, 68], [103, 123, 63, 22, 57, 115, 33, 114, 54, 13, 124, 127, 82, 25, 61, 59, 62, 28, 116, 120, 87, 121, 107, 125, 58, 27, 53, 60, 49, 55, 117, 80, 84, 56, 126, 113, 19, 74, 51, 118, 50, 48, 45, 109, 38, 99, 71, 119, 52, 7, 111, 97, 26, 122, 47, 12, 46, 37, 9, 30, 112, 93, 101, 106, 108, 31, 110, 85, 44, 104, 14, 42, 29, 105, 41, 100, 75, 98, 43, 102, 70, 89, 95, 86, 2, 17, 34, 15, 4, 79, 35, 32, 36, 20, 68, 1, 67, 40, 96, 0, 88, 18, 81, 94, 5, 73, 91, 11, 21, 72, 65, 10, 24, 3, 90, 23, 77, 64, 66, 78, 8, 76, 83, 39, 92, 6, 69, 16], [111, 62, 102, 86, 97, 118, 26, 29, 47, 83, 16, 81, 76, 78, 112, 9, 24, 119, 75, 53, 127, 122, 123, 114, 51, 55, 120, 96, 52, 63, 56, 50, 4, 71, 60, 124, 117, 121, 23, 61, 115, 116, 58, 49, 79, 74, 54, 36, 126, 57, 41, 59, 84, 42, 106, 46, 48, 35, 70, 110, 125, 37, 66, 105, 103, 109, 108, 45, 98, 40, 34, 43, 107, 65, 113, 77, 69, 1, 64, 44, 28, 99, 104, 93, 39, 101, 32, 3, 30, 21, 31, 38, 91, 80, 72, 11, 25, 100, 94, 7, 95, 89, 92, 5, 33, 0, 73, 27, 82, 15, 14, 13, 12, 20, 85, 17, 19, 88, 90, 22, 18, 2, 87, 67, 10, 8, 6, 68], [106, 107, 97, 102, 122, 86, 94, 121, 117, 49, 21, 50, 36, 48, 114, 42, 79, 103, 119, 27, 63, 113, 74, 92, 126, 116, 56, 61, 109, 91, 64, 123, 70, 62, 67, 127, 51, 24, 52, 105, 40, 19, 111, 57, 5, 12, 82, 112, 110, 59, 47, 45, 77, 44, 65, 18, 53, 115, 124, 81, 84, 60, 118, 125, 99, 98, 32, 88, 55, 54, 104, 71, 90, 66, 58, 37, 72, 78, 26, 8, 108, 120, 35, 101, 4, 46, 100, 34, 96, 1, 89, 11, 80, 93, 31, 95, 43, 23, 39, 25, 29, 41, 85, 87, 75, 9, 13, 0, 38, 15, 10, 14, 73, 28, 83, 30, 68, 22, 17, 33, 7, 76, 16, 20, 2, 6, 69, 3]], "model.layers.13.self_attn.qk_proj": [[123, 62, 111, 127, 121, 60, 51, 118, 115, 63, 47, 125, 53, 124, 61, 120, 106, 56, 57, 117, 59, 48, 116, 114, 119, 107, 122, 54, 50, 126, 42, 52, 102, 22, 38, 49, 58, 112, 86, 103, 113, 39, 55, 43, 110, 33, 91, 97, 100, 46, 88, 93, 92, 45, 108, 90, 44, 19, 24, 109, 27, 29, 36, 89, 83, 99, 82, 105, 18, 104, 26, 21, 16, 25, 40, 30, 32, 35, 80, 28, 87, 17, 41, 84, 12, 77, 101, 15, 20, 79, 23, 74, 85, 37, 76, 13, 94, 81, 98, 96, 78, 14, 95, 31, 75, 10, 9, 34, 73, 65, 70, 4, 11, 64, 3, 6, 7, 8, 68, 71, 5, 0, 72, 67, 69, 2, 66, 1], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 61, 56, 59, 54, 117, 57, 124, 106, 107, 120, 48, 114, 116, 119, 50, 102, 42, 122, 22, 126, 38, 58, 49, 52, 112, 103, 113, 86, 39, 43, 55, 110, 44, 93, 100, 33, 97, 90, 46, 92, 91, 45, 108, 88, 27, 36, 89, 40, 24, 29, 26, 105, 19, 99, 82, 35, 83, 25, 16, 109, 30, 32, 21, 80, 104, 37, 84, 28, 87, 18, 41, 20, 12, 85, 15, 76, 96, 17, 23, 31, 94, 81, 101, 77, 74, 78, 98, 10, 34, 75, 13, 70, 14, 95, 9, 79, 11, 71, 4, 73, 3, 7, 1, 0, 69, 65, 72, 64, 8, 5, 68, 2, 6, 67, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 61, 56, 117, 124, 53, 59, 106, 57, 120, 54, 48, 114, 42, 107, 50, 126, 116, 119, 122, 38, 102, 103, 112, 49, 113, 39, 58, 52, 22, 86, 55, 43, 33, 93, 97, 110, 91, 100, 46, 108, 45, 44, 92, 88, 90, 105, 27, 36, 29, 99, 40, 24, 109, 104, 32, 26, 89, 35, 83, 25, 41, 28, 37, 85, 30, 87, 101, 21, 16, 19, 18, 84, 98, 82, 23, 80, 94, 20, 79, 12, 34, 77, 31, 17, 96, 15, 95, 70, 74, 76, 10, 81, 14, 78, 75, 13, 9, 72, 64, 65, 1, 69, 0, 11, 71, 3, 67, 73, 66, 2, 4, 5, 68, 7, 8, 6], [123, 62, 111, 127, 121, 51, 118, 63, 60, 115, 47, 125, 61, 56, 53, 54, 124, 117, 106, 120, 126, 116, 57, 119, 59, 107, 58, 122, 38, 48, 49, 42, 50, 114, 102, 103, 112, 52, 22, 55, 43, 113, 39, 86, 110, 33, 44, 46, 92, 97, 93, 91, 88, 100, 45, 90, 40, 36, 29, 27, 108, 26, 89, 24, 32, 109, 83, 99, 105, 28, 35, 82, 19, 41, 87, 25, 37, 16, 18, 85, 23, 21, 17, 104, 79, 30, 31, 101, 12, 80, 34, 96, 20, 76, 77, 84, 98, 94, 70, 74, 81, 95, 14, 10, 75, 13, 15, 9, 78, 1, 64, 71, 3, 72, 68, 0, 11, 65, 67, 66, 73, 7, 69, 4, 5, 2, 6, 8], [123, 111, 62, 127, 121, 51, 115, 118, 63, 60, 47, 125, 61, 56, 53, 106, 124, 54, 57, 126, 117, 116, 114, 48, 119, 59, 50, 122, 42, 107, 120, 58, 102, 38, 52, 112, 49, 103, 55, 22, 113, 86, 39, 110, 44, 43, 46, 33, 91, 93, 92, 97, 88, 100, 90, 108, 24, 99, 36, 45, 27, 29, 40, 83, 89, 105, 19, 87, 26, 25, 28, 35, 32, 104, 109, 82, 41, 18, 15, 77, 16, 17, 12, 21, 76, 80, 23, 85, 81, 79, 94, 101, 20, 84, 10, 78, 74, 96, 30, 70, 98, 13, 95, 31, 37, 9, 75, 34, 14, 64, 7, 11, 68, 65, 72, 71, 73, 1, 66, 69, 3, 0, 4, 67, 6, 2, 5, 8], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 56, 124, 106, 117, 54, 126, 116, 57, 119, 107, 59, 120, 122, 58, 42, 50, 55, 102, 38, 22, 48, 114, 112, 86, 103, 39, 52, 49, 113, 43, 44, 110, 46, 91, 88, 93, 92, 97, 33, 90, 100, 29, 83, 108, 45, 36, 104, 19, 27, 26, 24, 99, 82, 16, 105, 87, 89, 80, 17, 18, 109, 28, 41, 40, 25, 35, 76, 79, 21, 13, 77, 84, 23, 32, 81, 30, 15, 20, 12, 74, 10, 14, 34, 101, 78, 75, 96, 9, 98, 85, 31, 94, 37, 11, 7, 70, 95, 71, 72, 6, 73, 0, 65, 3, 67, 1, 66, 4, 69, 5, 68, 64, 8, 2], [123, 111, 62, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 56, 126, 54, 117, 106, 122, 59, 107, 120, 119, 57, 116, 58, 114, 49, 42, 102, 48, 103, 38, 50, 55, 22, 112, 86, 39, 52, 43, 113, 88, 44, 46, 93, 110, 91, 92, 90, 97, 100, 45, 19, 33, 83, 24, 108, 27, 99, 40, 89, 36, 104, 29, 105, 82, 18, 26, 21, 80, 17, 87, 109, 35, 85, 76, 28, 25, 23, 16, 79, 77, 12, 32, 84, 37, 15, 30, 41, 94, 20, 13, 81, 78, 96, 10, 98, 14, 31, 34, 74, 9, 101, 75, 95, 72, 7, 73, 11, 6, 70, 71, 69, 67, 0, 5, 65, 1, 66, 64, 3, 8, 68, 4, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 47, 118, 125, 61, 124, 53, 56, 106, 54, 117, 120, 48, 126, 114, 107, 122, 59, 57, 119, 103, 116, 42, 38, 102, 58, 22, 49, 86, 50, 39, 55, 112, 52, 113, 43, 44, 91, 93, 110, 90, 97, 92, 100, 33, 88, 46, 27, 24, 29, 83, 19, 89, 36, 108, 35, 40, 104, 45, 26, 87, 16, 28, 32, 23, 82, 105, 99, 109, 80, 21, 25, 18, 84, 85, 41, 37, 76, 17, 31, 30, 15, 13, 12, 96, 20, 101, 77, 81, 14, 78, 94, 74, 98, 79, 10, 9, 34, 95, 75, 6, 72, 11, 73, 1, 0, 65, 64, 7, 3, 67, 69, 5, 2, 70, 4, 68, 71, 8, 66], [123, 62, 111, 127, 51, 121, 63, 60, 115, 118, 47, 125, 61, 53, 54, 117, 124, 56, 106, 126, 119, 59, 107, 120, 48, 57, 114, 102, 50, 42, 122, 38, 116, 113, 103, 58, 22, 49, 39, 112, 55, 52, 86, 43, 93, 90, 91, 110, 44, 92, 33, 97, 100, 46, 26, 88, 45, 27, 108, 29, 89, 83, 36, 82, 24, 19, 35, 99, 104, 25, 109, 28, 32, 40, 80, 87, 85, 41, 105, 21, 23, 16, 37, 76, 18, 84, 30, 17, 98, 31, 81, 78, 96, 14, 12, 13, 94, 79, 15, 20, 101, 74, 77, 10, 34, 95, 73, 75, 6, 67, 11, 9, 3, 65, 1, 69, 8, 64, 7, 72, 0, 71, 70, 5, 66, 4, 2, 68], [62, 123, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 59, 106, 56, 54, 48, 126, 114, 119, 57, 107, 116, 120, 38, 58, 50, 102, 42, 103, 52, 22, 112, 49, 122, 86, 39, 55, 43, 113, 44, 33, 91, 46, 93, 97, 100, 92, 110, 90, 88, 108, 24, 83, 27, 45, 89, 19, 109, 29, 36, 26, 40, 105, 104, 28, 18, 25, 99, 82, 16, 35, 41, 32, 80, 76, 37, 13, 87, 96, 30, 31, 79, 12, 23, 17, 77, 20, 85, 81, 78, 84, 98, 15, 74, 94, 101, 21, 14, 95, 10, 34, 75, 9, 6, 11, 73, 67, 8, 7, 71, 1, 68, 5, 70, 64, 65, 0, 2, 3, 4, 69, 72, 66], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 124, 53, 61, 117, 54, 120, 106, 126, 114, 116, 56, 57, 59, 119, 48, 107, 50, 122, 42, 38, 102, 49, 22, 52, 58, 39, 55, 113, 86, 103, 43, 112, 91, 33, 46, 97, 93, 100, 92, 44, 88, 90, 110, 24, 29, 27, 40, 45, 99, 89, 19, 26, 83, 82, 36, 105, 108, 25, 109, 17, 37, 32, 18, 80, 16, 35, 87, 77, 12, 30, 94, 76, 84, 28, 104, 79, 15, 31, 74, 41, 21, 13, 96, 81, 10, 6, 20, 23, 101, 73, 98, 78, 14, 95, 75, 34, 85, 71, 8, 11, 7, 3, 1, 9, 67, 68, 70, 4, 65, 64, 0, 5, 66, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 61, 53, 54, 117, 106, 57, 126, 124, 119, 120, 116, 114, 59, 56, 122, 49, 42, 107, 50, 48, 52, 102, 22, 103, 55, 38, 58, 86, 113, 112, 39, 43, 46, 91, 97, 110, 93, 33, 90, 88, 44, 92, 100, 24, 19, 45, 27, 83, 99, 29, 36, 108, 21, 26, 89, 82, 87, 35, 109, 40, 32, 76, 80, 15, 16, 105, 77, 17, 10, 41, 18, 79, 104, 28, 13, 81, 12, 25, 85, 31, 94, 37, 96, 84, 30, 74, 20, 23, 34, 14, 78, 101, 98, 65, 11, 8, 9, 75, 73, 71, 95, 6, 70, 0, 68, 69, 67, 7, 3, 5, 64, 66, 4, 1, 72, 2], [123, 62, 111, 127, 121, 51, 115, 60, 63, 118, 47, 125, 61, 124, 53, 117, 126, 114, 54, 59, 56, 106, 120, 107, 57, 119, 50, 48, 38, 102, 116, 42, 49, 22, 103, 55, 122, 112, 52, 43, 86, 58, 113, 39, 46, 91, 110, 93, 97, 44, 92, 90, 100, 45, 33, 27, 89, 24, 29, 40, 108, 88, 104, 99, 19, 26, 25, 36, 82, 41, 109, 83, 35, 32, 28, 87, 105, 80, 18, 85, 16, 37, 76, 96, 23, 20, 17, 30, 31, 21, 84, 15, 94, 13, 79, 77, 12, 78, 81, 98, 10, 34, 95, 101, 14, 11, 75, 74, 73, 8, 70, 71, 0, 3, 65, 9, 5, 67, 66, 69, 4, 68, 6, 1, 2, 64, 7, 72], [123, 62, 111, 127, 121, 63, 60, 51, 118, 115, 47, 125, 61, 124, 53, 117, 59, 106, 119, 126, 57, 107, 56, 54, 120, 114, 50, 122, 38, 58, 55, 116, 49, 48, 22, 52, 102, 42, 86, 112, 43, 39, 103, 113, 46, 110, 44, 91, 92, 33, 97, 100, 93, 108, 88, 90, 109, 24, 45, 82, 19, 29, 83, 36, 105, 89, 80, 27, 35, 18, 104, 26, 25, 16, 13, 28, 40, 99, 17, 41, 30, 76, 87, 84, 78, 94, 21, 32, 79, 37, 23, 12, 14, 20, 96, 10, 85, 81, 98, 95, 31, 11, 74, 75, 101, 70, 77, 73, 15, 71, 9, 7, 8, 34, 67, 64, 0, 68, 65, 3, 4, 1, 6, 69, 66, 5, 72, 2], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 124, 53, 61, 106, 56, 117, 119, 57, 54, 114, 120, 126, 59, 107, 50, 116, 122, 55, 42, 48, 38, 49, 52, 58, 102, 22, 86, 112, 103, 43, 39, 113, 110, 46, 91, 44, 93, 97, 100, 33, 92, 108, 24, 27, 45, 109, 88, 90, 29, 19, 82, 83, 36, 26, 40, 99, 32, 25, 104, 28, 80, 35, 18, 21, 16, 41, 89, 105, 17, 76, 20, 31, 23, 13, 37, 87, 94, 84, 12, 30, 81, 78, 96, 15, 98, 85, 74, 77, 10, 79, 14, 95, 75, 70, 34, 101, 73, 11, 71, 8, 68, 7, 9, 65, 64, 4, 3, 1, 67, 0, 5, 69, 72, 6, 2, 66], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 124, 61, 53, 117, 106, 114, 54, 59, 120, 126, 57, 56, 119, 107, 116, 122, 48, 55, 42, 58, 49, 50, 102, 38, 22, 52, 112, 113, 86, 39, 103, 43, 46, 44, 93, 100, 110, 97, 33, 91, 45, 92, 27, 90, 24, 88, 108, 36, 109, 83, 82, 40, 26, 29, 21, 99, 19, 41, 104, 89, 18, 32, 15, 25, 12, 16, 28, 87, 10, 30, 79, 17, 31, 35, 13, 81, 76, 105, 80, 37, 96, 23, 84, 85, 98, 94, 74, 34, 77, 73, 20, 14, 78, 11, 101, 8, 75, 95, 70, 65, 69, 7, 71, 9, 64, 6, 1, 4, 0, 67, 68, 3, 72, 2, 5, 66], [123, 62, 111, 127, 121, 115, 51, 60, 63, 118, 47, 125, 61, 124, 53, 59, 57, 117, 126, 119, 54, 120, 106, 56, 107, 122, 38, 114, 52, 55, 58, 48, 49, 116, 42, 22, 103, 102, 86, 50, 39, 112, 113, 46, 43, 44, 91, 100, 33, 97, 93, 110, 92, 27, 90, 88, 45, 109, 108, 99, 24, 29, 83, 89, 26, 36, 104, 40, 19, 35, 32, 87, 80, 25, 105, 41, 82, 84, 28, 16, 18, 79, 85, 81, 13, 23, 37, 21, 30, 12, 17, 34, 101, 98, 94, 31, 76, 20, 15, 77, 96, 10, 95, 78, 73, 11, 75, 14, 74, 6, 7, 9, 71, 0, 64, 65, 8, 70, 3, 5, 1, 69, 72, 67, 66, 2, 4, 68], [123, 62, 111, 127, 121, 51, 115, 118, 63, 60, 47, 125, 53, 61, 124, 119, 126, 106, 117, 56, 57, 48, 107, 120, 54, 59, 38, 50, 122, 114, 55, 49, 42, 116, 22, 102, 58, 103, 86, 52, 113, 112, 39, 43, 46, 44, 97, 91, 33, 93, 100, 90, 110, 92, 45, 27, 108, 36, 88, 29, 109, 24, 40, 25, 105, 104, 19, 83, 87, 82, 35, 32, 26, 16, 89, 28, 99, 80, 18, 41, 13, 84, 34, 21, 37, 20, 17, 96, 30, 12, 81, 79, 23, 85, 14, 94, 76, 15, 78, 98, 101, 77, 31, 10, 74, 11, 95, 73, 9, 67, 75, 72, 6, 71, 70, 69, 68, 64, 7, 1, 0, 3, 65, 2, 4, 66, 5, 8], [123, 111, 62, 127, 121, 115, 51, 118, 63, 60, 47, 125, 53, 61, 117, 124, 56, 106, 120, 57, 119, 54, 58, 107, 59, 38, 126, 42, 116, 55, 102, 122, 114, 103, 50, 48, 113, 49, 39, 22, 112, 43, 86, 52, 46, 100, 97, 91, 33, 44, 93, 108, 92, 110, 45, 88, 104, 24, 36, 90, 29, 109, 40, 99, 27, 83, 25, 32, 19, 89, 82, 35, 105, 87, 26, 37, 18, 84, 21, 16, 101, 41, 30, 28, 98, 80, 96, 34, 23, 85, 94, 20, 31, 15, 81, 14, 17, 79, 13, 95, 12, 10, 77, 76, 6, 78, 9, 11, 67, 74, 75, 73, 4, 65, 0, 72, 71, 64, 7, 1, 5, 68, 3, 69, 66, 2, 8, 70], [123, 62, 111, 127, 121, 51, 63, 115, 60, 118, 47, 125, 53, 61, 124, 117, 106, 120, 59, 107, 56, 119, 57, 126, 54, 48, 116, 58, 114, 122, 38, 50, 22, 102, 42, 49, 39, 55, 86, 103, 112, 113, 52, 43, 44, 46, 97, 33, 100, 92, 93, 91, 90, 45, 27, 24, 110, 88, 29, 36, 82, 104, 19, 89, 35, 99, 26, 109, 108, 32, 28, 40, 41, 25, 83, 16, 18, 84, 87, 80, 21, 96, 23, 20, 105, 12, 30, 37, 98, 31, 13, 76, 94, 77, 81, 15, 101, 34, 17, 11, 79, 10, 14, 85, 74, 6, 95, 78, 9, 73, 75, 72, 71, 7, 3, 1, 4, 69, 64, 65, 70, 67, 5, 0, 66, 8, 68, 2], [123, 62, 111, 127, 121, 51, 115, 63, 60, 118, 47, 125, 53, 124, 61, 54, 117, 106, 119, 59, 126, 120, 122, 114, 107, 57, 50, 55, 116, 56, 49, 48, 22, 58, 38, 42, 86, 43, 102, 39, 52, 113, 103, 112, 46, 93, 91, 44, 33, 90, 92, 97, 27, 100, 45, 24, 89, 36, 88, 110, 29, 26, 108, 82, 109, 19, 40, 32, 35, 83, 28, 41, 104, 84, 18, 99, 21, 80, 16, 25, 23, 87, 17, 20, 12, 37, 13, 30, 96, 31, 98, 81, 79, 34, 76, 10, 15, 105, 94, 101, 77, 74, 85, 14, 78, 73, 7, 11, 75, 9, 3, 95, 1, 72, 6, 71, 0, 4, 69, 64, 68, 67, 65, 5, 70, 8, 2, 66], [123, 62, 111, 127, 121, 51, 118, 60, 63, 115, 47, 125, 53, 124, 61, 120, 106, 126, 59, 119, 117, 116, 54, 56, 122, 107, 58, 49, 38, 57, 114, 48, 55, 42, 103, 102, 39, 50, 113, 43, 22, 112, 52, 86, 46, 91, 93, 44, 90, 100, 97, 110, 33, 92, 24, 45, 108, 27, 36, 88, 29, 26, 32, 109, 89, 19, 104, 35, 99, 41, 83, 82, 40, 18, 28, 87, 25, 23, 16, 30, 85, 84, 96, 80, 31, 10, 21, 37, 15, 105, 17, 101, 13, 12, 94, 78, 20, 76, 98, 34, 77, 81, 79, 95, 14, 73, 75, 72, 74, 3, 11, 9, 71, 70, 64, 65, 1, 7, 0, 6, 68, 66, 67, 5, 69, 4, 8, 2], [123, 62, 111, 127, 121, 51, 60, 63, 115, 118, 47, 125, 53, 124, 117, 54, 61, 106, 59, 114, 120, 107, 126, 122, 56, 57, 119, 116, 58, 48, 39, 103, 102, 55, 38, 22, 42, 49, 50, 112, 52, 86, 113, 43, 93, 97, 100, 110, 46, 33, 44, 90, 91, 92, 45, 27, 36, 24, 108, 88, 89, 104, 40, 29, 26, 109, 99, 32, 83, 82, 25, 87, 35, 19, 28, 84, 30, 21, 16, 41, 80, 18, 96, 23, 105, 37, 101, 31, 78, 85, 15, 12, 20, 13, 94, 76, 10, 34, 95, 81, 14, 98, 79, 17, 77, 73, 11, 70, 74, 9, 4, 1, 65, 71, 0, 67, 75, 72, 3, 64, 69, 7, 2, 68, 66, 6, 5, 8], [123, 62, 111, 127, 121, 51, 60, 63, 118, 115, 47, 125, 53, 61, 124, 59, 106, 54, 117, 56, 114, 107, 126, 119, 58, 116, 57, 48, 120, 122, 49, 38, 22, 42, 55, 103, 50, 102, 52, 39, 86, 43, 113, 112, 46, 93, 92, 90, 33, 97, 91, 100, 27, 88, 44, 24, 45, 110, 36, 29, 26, 82, 89, 109, 25, 99, 83, 28, 18, 41, 80, 108, 19, 87, 32, 20, 104, 105, 40, 35, 96, 31, 16, 30, 79, 37, 84, 81, 21, 94, 12, 73, 14, 17, 76, 85, 13, 23, 77, 10, 34, 15, 74, 78, 101, 11, 98, 70, 7, 95, 75, 71, 9, 0, 3, 67, 72, 69, 4, 65, 5, 1, 66, 6, 68, 8, 64, 2], [123, 62, 111, 127, 121, 51, 118, 60, 115, 63, 47, 125, 61, 53, 124, 57, 117, 56, 106, 120, 122, 54, 126, 119, 59, 107, 48, 50, 116, 58, 22, 49, 114, 42, 38, 86, 103, 102, 55, 52, 113, 39, 112, 43, 46, 33, 110, 45, 97, 91, 100, 93, 90, 44, 92, 108, 27, 109, 88, 24, 83, 19, 99, 26, 29, 40, 89, 36, 18, 25, 82, 35, 41, 104, 21, 30, 16, 28, 87, 23, 32, 84, 12, 31, 80, 105, 94, 77, 101, 20, 13, 37, 79, 85, 17, 15, 81, 14, 96, 98, 10, 74, 78, 76, 73, 34, 70, 95, 7, 9, 11, 75, 64, 71, 1, 0, 72, 4, 3, 65, 69, 67, 66, 68, 5, 8, 2, 6], [123, 62, 111, 127, 121, 118, 51, 115, 60, 63, 47, 125, 61, 53, 124, 117, 54, 106, 57, 59, 56, 107, 120, 126, 58, 119, 122, 48, 50, 116, 49, 112, 114, 103, 38, 102, 22, 55, 42, 86, 39, 113, 52, 43, 46, 93, 44, 97, 45, 100, 92, 33, 110, 91, 90, 24, 27, 108, 88, 40, 19, 29, 109, 89, 26, 25, 99, 36, 83, 82, 28, 35, 87, 18, 21, 84, 41, 16, 105, 30, 31, 104, 32, 80, 85, 94, 17, 96, 15, 12, 98, 81, 20, 101, 23, 77, 37, 13, 76, 10, 79, 34, 95, 78, 74, 14, 11, 9, 70, 75, 73, 7, 8, 68, 71, 3, 1, 65, 64, 0, 6, 72, 67, 5, 69, 2, 4, 66], [123, 62, 111, 127, 121, 51, 118, 115, 60, 63, 47, 125, 61, 53, 124, 120, 117, 59, 106, 54, 56, 57, 114, 107, 122, 50, 48, 116, 126, 38, 58, 49, 119, 102, 22, 42, 55, 103, 113, 86, 52, 112, 39, 43, 46, 110, 97, 91, 93, 45, 44, 33, 108, 100, 92, 99, 24, 88, 90, 19, 36, 27, 40, 26, 89, 29, 109, 35, 28, 105, 104, 18, 25, 30, 83, 80, 31, 82, 101, 87, 16, 23, 21, 20, 32, 37, 85, 84, 41, 12, 94, 13, 79, 17, 78, 10, 76, 34, 77, 96, 98, 81, 15, 95, 74, 9, 11, 14, 73, 75, 7, 70, 8, 67, 6, 71, 3, 1, 64, 0, 69, 5, 4, 68, 65, 66, 2, 72], [123, 62, 111, 127, 121, 51, 60, 115, 118, 63, 47, 125, 61, 53, 124, 106, 54, 117, 120, 56, 50, 114, 107, 59, 57, 119, 48, 126, 22, 122, 58, 116, 42, 49, 38, 86, 113, 39, 102, 103, 52, 112, 55, 43, 97, 33, 92, 44, 91, 93, 110, 90, 45, 88, 46, 100, 108, 27, 82, 19, 29, 26, 36, 24, 89, 83, 109, 40, 18, 99, 16, 25, 28, 87, 80, 20, 32, 104, 23, 35, 105, 13, 17, 21, 30, 94, 84, 41, 37, 85, 31, 12, 10, 81, 78, 76, 79, 9, 96, 101, 77, 15, 34, 95, 74, 98, 6, 14, 11, 67, 75, 8, 7, 73, 0, 65, 71, 1, 3, 4, 70, 5, 64, 68, 66, 2, 69, 72], [123, 111, 62, 127, 51, 121, 60, 115, 118, 63, 47, 125, 61, 124, 53, 56, 106, 54, 117, 48, 116, 57, 114, 50, 126, 119, 107, 42, 59, 120, 122, 58, 113, 38, 102, 22, 49, 86, 112, 52, 103, 39, 55, 43, 110, 33, 97, 91, 46, 100, 44, 93, 92, 88, 90, 27, 24, 36, 45, 29, 99, 40, 89, 108, 109, 32, 26, 104, 18, 41, 19, 83, 25, 82, 28, 21, 35, 80, 94, 96, 12, 15, 105, 23, 30, 101, 87, 17, 37, 81, 16, 13, 85, 20, 31, 84, 98, 76, 74, 77, 10, 95, 78, 34, 9, 14, 6, 79, 64, 73, 11, 65, 7, 75, 3, 4, 1, 67, 70, 71, 68, 5, 8, 66, 69, 2, 0, 72], [123, 111, 62, 127, 121, 51, 115, 60, 118, 63, 47, 125, 53, 61, 124, 106, 56, 48, 57, 120, 126, 117, 50, 119, 59, 54, 107, 114, 42, 116, 22, 122, 58, 103, 86, 38, 102, 49, 55, 39, 112, 113, 52, 43, 44, 33, 110, 93, 46, 100, 97, 92, 88, 91, 27, 90, 108, 36, 24, 19, 45, 29, 40, 89, 26, 109, 32, 82, 99, 83, 104, 18, 28, 87, 25, 16, 35, 85, 76, 105, 12, 15, 84, 80, 21, 30, 96, 41, 37, 79, 81, 77, 17, 98, 23, 10, 74, 13, 14, 20, 94, 101, 95, 73, 78, 9, 6, 31, 11, 34, 75, 8, 71, 1, 67, 4, 66, 64, 7, 65, 3, 68, 5, 0, 70, 69, 72, 2], [123, 62, 111, 127, 121, 51, 60, 115, 63, 118, 47, 125, 53, 61, 124, 56, 106, 117, 48, 57, 126, 120, 119, 59, 114, 54, 50, 107, 116, 122, 49, 42, 38, 22, 58, 112, 86, 103, 102, 113, 55, 52, 43, 39, 33, 100, 46, 110, 91, 97, 93, 108, 90, 44, 92, 27, 88, 109, 45, 36, 29, 40, 83, 82, 26, 89, 105, 19, 32, 104, 99, 18, 16, 24, 80, 87, 41, 35, 25, 94, 30, 28, 77, 84, 37, 21, 101, 17, 15, 74, 98, 81, 79, 12, 6, 76, 96, 85, 14, 20, 31, 23, 10, 75, 13, 78, 9, 71, 11, 95, 73, 34, 7, 8, 0, 3, 68, 67, 1, 65, 4, 70, 64, 2, 69, 66, 72, 5], [123, 62, 111, 127, 121, 60, 51, 115, 118, 63, 47, 125, 61, 53, 124, 120, 106, 57, 117, 48, 59, 56, 54, 126, 116, 107, 114, 122, 119, 22, 42, 38, 103, 86, 102, 39, 58, 50, 49, 112, 52, 55, 113, 100, 43, 46, 110, 33, 97, 91, 93, 92, 44, 88, 90, 108, 27, 45, 89, 19, 24, 29, 26, 99, 36, 83, 16, 25, 105, 32, 104, 28, 109, 18, 30, 80, 87, 84, 82, 35, 40, 76, 77, 20, 12, 81, 85, 79, 21, 17, 15, 37, 23, 41, 94, 10, 14, 74, 13, 73, 101, 96, 75, 78, 31, 34, 98, 11, 6, 9, 7, 8, 71, 95, 5, 70, 67, 3, 4, 1, 72, 69, 64, 65, 0, 68, 66, 2]], "model.layers.14.self_attn.q_proj": [[60, 120, 62, 37, 51, 63, 33, 53, 90, 101, 30, 118, 117, 87, 19, 86, 57, 125, 58, 85, 74, 93, 123, 111, 109, 115, 119, 59, 54, 92, 21, 116, 121, 12, 26, 50, 52, 114, 42, 44, 29, 126, 61, 91, 127, 122, 45, 55, 56, 48, 39, 105, 124, 6, 110, 15, 49, 4, 81, 43, 97, 112, 46, 36, 113, 25, 108, 20, 104, 94, 38, 47, 17, 64, 106, 88, 107, 71, 41, 18, 13, 103, 78, 23, 99, 34, 102, 22, 16, 2, 98, 40, 32, 65, 28, 10, 89, 72, 100, 95, 31, 1, 27, 35, 14, 79, 96, 68, 83, 24, 84, 11, 82, 66, 76, 80, 70, 69, 77, 75, 7, 8, 3, 0, 73, 67, 5, 9], [60, 120, 62, 37, 33, 90, 30, 118, 19, 101, 87, 117, 51, 53, 86, 59, 115, 57, 119, 92, 58, 111, 121, 126, 85, 54, 12, 15, 63, 116, 123, 26, 127, 48, 114, 56, 122, 74, 125, 46, 105, 71, 52, 43, 81, 38, 97, 110, 61, 42, 124, 83, 41, 55, 44, 45, 93, 50, 94, 39, 88, 69, 108, 112, 102, 40, 109, 103, 106, 113, 47, 20, 6, 29, 13, 49, 107, 1, 79, 28, 2, 99, 22, 17, 25, 104, 31, 95, 34, 14, 91, 23, 8, 66, 36, 21, 24, 89, 7, 72, 64, 32, 82, 100, 76, 16, 84, 98, 35, 96, 80, 75, 4, 18, 78, 11, 10, 70, 27, 65, 67, 9, 77, 5, 73, 3, 68, 0], [60, 120, 37, 62, 90, 118, 51, 101, 33, 19, 53, 87, 117, 30, 86, 42, 74, 4, 12, 59, 93, 111, 21, 114, 58, 39, 116, 63, 46, 92, 113, 54, 57, 126, 50, 121, 115, 26, 52, 105, 2, 49, 125, 15, 123, 25, 119, 127, 45, 48, 43, 122, 94, 91, 88, 61, 17, 110, 83, 103, 112, 56, 71, 55, 124, 85, 109, 38, 41, 81, 98, 47, 106, 104, 40, 14, 108, 102, 44, 6, 8, 107, 78, 29, 34, 99, 79, 76, 28, 23, 64, 72, 31, 69, 22, 89, 18, 65, 35, 66, 84, 36, 16, 20, 97, 1, 32, 68, 100, 24, 27, 77, 96, 95, 7, 82, 73, 80, 13, 11, 10, 75, 5, 70, 0, 67, 9, 3], [120, 60, 37, 62, 118, 90, 33, 117, 30, 51, 101, 53, 111, 59, 19, 87, 63, 86, 58, 57, 85, 105, 121, 48, 115, 92, 54, 42, 12, 126, 116, 55, 122, 119, 123, 46, 125, 52, 56, 26, 93, 74, 61, 127, 110, 108, 50, 112, 124, 114, 88, 41, 69, 109, 49, 113, 47, 91, 15, 82, 4, 21, 79, 44, 107, 43, 106, 81, 45, 39, 2, 25, 97, 104, 103, 94, 28, 38, 64, 71, 22, 36, 83, 78, 29, 31, 17, 6, 13, 40, 1, 100, 66, 99, 76, 102, 35, 34, 95, 32, 24, 72, 98, 84, 23, 96, 73, 18, 16, 89, 75, 68, 10, 65, 8, 27, 20, 0, 14, 11, 80, 70, 5, 67, 9, 77, 7, 3], [52, 102, 51, 124, 125, 29, 33, 60, 116, 84, 61, 123, 26, 86, 87, 88, 56, 119, 62, 93, 55, 38, 107, 120, 118, 106, 114, 115, 81, 39, 57, 122, 113, 109, 127, 37, 45, 94, 50, 53, 110, 58, 121, 28, 20, 54, 42, 63, 101, 105, 111, 41, 103, 112, 99, 35, 49, 108, 36, 40, 104, 97, 48, 22, 44, 43, 117, 34, 90, 59, 46, 47, 85, 25, 15, 100, 126, 98, 95, 73, 14, 32, 24, 21, 96, 82, 30, 31, 23, 92, 17, 18, 91, 11, 27, 71, 89, 80, 83, 74, 19, 16, 10, 7, 13, 5, 67, 78, 79, 9, 77, 8, 76, 12, 72, 75, 69, 66, 68, 65, 6, 64, 1, 4, 2, 70, 3, 0], [52, 102, 124, 33, 29, 125, 87, 60, 61, 116, 88, 123, 56, 38, 55, 119, 51, 73, 84, 45, 101, 62, 107, 57, 86, 28, 112, 106, 115, 39, 113, 110, 50, 53, 105, 127, 63, 67, 26, 118, 120, 42, 36, 54, 121, 114, 49, 58, 108, 43, 117, 109, 64, 47, 46, 126, 48, 40, 94, 122, 111, 44, 41, 59, 15, 93, 32, 103, 104, 97, 24, 22, 83, 99, 11, 100, 91, 20, 98, 90, 37, 35, 65, 81, 95, 34, 82, 31, 3, 25, 66, 96, 69, 27, 30, 23, 92, 71, 21, 89, 13, 5, 7, 9, 17, 79, 77, 19, 1, 6, 76, 18, 85, 10, 80, 78, 75, 14, 72, 74, 4, 0, 16, 12, 70, 68, 2, 8], [52, 102, 33, 124, 86, 29, 116, 88, 125, 61, 84, 82, 60, 73, 26, 87, 15, 56, 38, 66, 24, 93, 123, 51, 81, 14, 119, 90, 105, 55, 20, 62, 28, 19, 50, 110, 9, 112, 57, 113, 18, 10, 95, 13, 120, 43, 91, 54, 5, 106, 21, 45, 79, 99, 2, 22, 53, 63, 121, 49, 77, 108, 115, 85, 31, 101, 30, 118, 127, 114, 83, 4, 107, 23, 34, 78, 39, 75, 48, 17, 11, 25, 40, 117, 64, 98, 58, 27, 122, 12, 109, 0, 71, 44, 7, 72, 37, 94, 42, 70, 92, 111, 126, 16, 59, 103, 97, 69, 35, 46, 47, 1, 76, 89, 36, 41, 104, 80, 96, 32, 3, 100, 8, 6, 74, 68, 67, 65], [52, 102, 33, 29, 125, 51, 61, 26, 86, 124, 56, 116, 88, 82, 60, 84, 81, 87, 93, 15, 71, 123, 20, 119, 85, 11, 120, 55, 38, 73, 110, 113, 97, 22, 32, 62, 17, 75, 28, 90, 103, 14, 57, 24, 13, 50, 19, 76, 5, 54, 106, 109, 115, 127, 10, 112, 67, 25, 18, 42, 63, 118, 77, 48, 122, 45, 49, 79, 121, 107, 27, 58, 23, 3, 111, 108, 16, 31, 37, 104, 46, 89, 114, 43, 39, 105, 101, 53, 40, 36, 98, 94, 30, 117, 64, 41, 7, 126, 35, 21, 47, 83, 6, 80, 9, 91, 92, 1, 44, 100, 65, 99, 59, 34, 74, 70, 96, 95, 78, 12, 72, 69, 68, 4, 8, 66, 2, 0], [103, 97, 53, 117, 23, 2, 80, 11, 9, 85, 65, 0, 69, 81, 83, 67, 51, 14, 4, 3, 12, 71, 116, 70, 1, 7, 124, 61, 127, 112, 64, 87, 118, 60, 106, 6, 122, 73, 50, 5, 89, 113, 126, 66, 74, 13, 55, 114, 25, 68, 121, 54, 120, 24, 88, 8, 39, 63, 30, 91, 10, 49, 76, 15, 78, 28, 75, 17, 59, 108, 29, 45, 119, 104, 82, 58, 42, 79, 109, 31, 43, 52, 105, 110, 47, 77, 44, 46, 20, 19, 27, 123, 90, 72, 56, 34, 40, 57, 22, 125, 26, 16, 21, 98, 32, 102, 100, 99, 86, 37, 35, 107, 101, 84, 41, 62, 95, 36, 93, 18, 38, 48, 92, 33, 111, 94, 115, 96], [103, 97, 53, 117, 106, 81, 80, 87, 14, 12, 85, 23, 74, 4, 2, 112, 51, 83, 7, 11, 127, 116, 9, 124, 118, 69, 0, 54, 60, 61, 3, 25, 15, 24, 113, 50, 55, 70, 65, 126, 88, 122, 64, 30, 45, 121, 120, 114, 63, 6, 1, 76, 18, 75, 29, 72, 73, 5, 71, 68, 28, 89, 91, 39, 49, 26, 78, 58, 16, 8, 10, 67, 59, 44, 40, 19, 109, 110, 21, 52, 105, 17, 82, 98, 92, 66, 108, 42, 86, 46, 32, 31, 79, 27, 41, 13, 77, 36, 119, 95, 90, 125, 43, 84, 47, 100, 48, 34, 57, 107, 62, 102, 38, 22, 104, 99, 37, 101, 20, 35, 115, 93, 56, 94, 96, 111, 123, 33], [103, 97, 117, 53, 85, 87, 80, 51, 58, 14, 83, 12, 106, 11, 27, 16, 10, 23, 24, 5, 127, 124, 49, 81, 113, 104, 76, 43, 116, 118, 112, 45, 54, 71, 57, 91, 25, 88, 120, 86, 108, 122, 17, 109, 60, 50, 66, 47, 78, 55, 121, 110, 92, 63, 114, 52, 6, 21, 61, 44, 99, 125, 119, 62, 40, 105, 42, 123, 38, 37, 111, 89, 126, 56, 41, 19, 101, 102, 75, 98, 72, 34, 59, 9, 29, 115, 67, 100, 46, 68, 20, 22, 4, 30, 48, 36, 107, 26, 32, 1, 94, 95, 28, 82, 35, 18, 93, 90, 77, 96, 64, 31, 39, 2, 15, 73, 69, 79, 8, 84, 13, 7, 70, 74, 33, 65, 3, 0], [103, 97, 53, 117, 81, 11, 85, 14, 87, 106, 23, 69, 80, 12, 9, 2, 88, 60, 127, 7, 5, 124, 3, 51, 74, 112, 116, 0, 65, 70, 61, 4, 28, 126, 24, 118, 16, 29, 121, 83, 15, 122, 22, 45, 54, 44, 30, 120, 6, 25, 105, 114, 21, 79, 49, 71, 78, 50, 75, 67, 10, 17, 89, 63, 76, 55, 73, 64, 113, 40, 8, 1, 19, 98, 66, 86, 77, 58, 46, 109, 27, 82, 42, 108, 72, 13, 92, 104, 91, 34, 110, 84, 59, 62, 20, 39, 31, 125, 52, 18, 123, 90, 68, 41, 26, 32, 93, 94, 96, 107, 95, 48, 57, 99, 43, 119, 101, 38, 111, 37, 36, 35, 100, 47, 102, 56, 115, 33], [56, 102, 127, 24, 17, 93, 44, 60, 78, 11, 61, 33, 86, 88, 29, 116, 113, 108, 90, 50, 35, 114, 115, 47, 19, 59, 6, 62, 92, 123, 31, 40, 26, 72, 63, 119, 54, 20, 46, 120, 118, 122, 112, 126, 49, 104, 51, 111, 53, 124, 99, 121, 117, 23, 125, 37, 48, 55, 85, 57, 110, 36, 109, 42, 106, 32, 34, 43, 94, 45, 58, 30, 103, 39, 107, 52, 38, 8, 16, 105, 41, 100, 83, 89, 87, 96, 91, 95, 81, 98, 79, 28, 101, 82, 70, 25, 14, 27, 74, 21, 4, 10, 22, 3, 84, 77, 97, 75, 71, 76, 13, 18, 12, 80, 2, 15, 66, 69, 64, 67, 9, 7, 5, 0, 73, 68, 1, 65], [56, 102, 127, 24, 61, 113, 93, 60, 116, 114, 50, 108, 115, 63, 62, 59, 47, 126, 122, 88, 123, 119, 90, 38, 120, 112, 49, 53, 125, 46, 51, 121, 55, 124, 111, 117, 54, 42, 33, 48, 44, 39, 110, 57, 52, 58, 118, 109, 17, 103, 45, 30, 105, 43, 32, 81, 104, 40, 36, 78, 29, 75, 107, 106, 37, 84, 35, 41, 96, 34, 22, 92, 86, 19, 66, 94, 99, 101, 26, 100, 87, 2, 8, 80, 83, 98, 20, 11, 95, 91, 31, 85, 28, 97, 27, 79, 14, 73, 89, 15, 16, 6, 72, 3, 23, 64, 70, 77, 25, 21, 76, 5, 68, 82, 4, 13, 74, 9, 0, 18, 1, 67, 10, 12, 7, 65, 71, 69], [56, 108, 102, 127, 93, 9, 33, 24, 17, 61, 90, 94, 113, 60, 103, 1, 116, 88, 86, 39, 115, 65, 2, 50, 30, 114, 47, 59, 120, 62, 34, 23, 85, 123, 122, 126, 119, 69, 63, 87, 44, 55, 46, 73, 111, 104, 3, 0, 121, 18, 20, 64, 112, 31, 38, 54, 53, 49, 96, 68, 91, 51, 109, 98, 124, 36, 79, 95, 5, 48, 11, 117, 26, 37, 13, 57, 32, 77, 28, 35, 125, 42, 58, 52, 21, 7, 110, 118, 8, 27, 97, 10, 74, 4, 106, 105, 100, 16, 83, 72, 99, 66, 107, 15, 25, 40, 29, 19, 92, 45, 89, 82, 70, 71, 101, 84, 78, 6, 41, 67, 80, 22, 43, 76, 12, 81, 75, 14], [56, 102, 33, 108, 127, 93, 17, 24, 88, 113, 61, 90, 11, 86, 60, 82, 116, 78, 35, 59, 29, 20, 6, 47, 114, 50, 26, 44, 38, 123, 62, 122, 115, 120, 16, 106, 37, 21, 34, 63, 95, 119, 30, 126, 112, 87, 76, 121, 55, 49, 94, 51, 27, 53, 54, 28, 36, 83, 85, 80, 9, 75, 72, 89, 117, 124, 25, 110, 48, 91, 111, 125, 81, 79, 46, 42, 22, 104, 45, 103, 118, 105, 43, 57, 58, 96, 109, 98, 39, 52, 40, 99, 18, 32, 12, 15, 97, 23, 107, 31, 92, 84, 5, 7, 19, 100, 101, 74, 41, 77, 10, 70, 13, 73, 64, 14, 4, 3, 0, 67, 71, 2, 66, 69, 8, 68, 1, 65], [123, 39, 113, 69, 3, 64, 112, 1, 77, 71, 49, 63, 29, 13, 48, 111, 91, 61, 73, 98, 122, 54, 60, 119, 120, 4, 11, 72, 124, 88, 82, 24, 53, 115, 101, 56, 78, 22, 66, 62, 95, 17, 121, 59, 57, 81, 125, 55, 65, 83, 40, 118, 43, 127, 114, 93, 51, 8, 94, 70, 68, 46, 92, 15, 116, 106, 96, 36, 89, 47, 50, 35, 76, 2, 99, 52, 110, 97, 109, 102, 108, 44, 80, 45, 117, 104, 107, 126, 38, 41, 28, 9, 105, 58, 74, 37, 32, 26, 23, 33, 10, 100, 21, 42, 0, 25, 67, 12, 87, 19, 6, 31, 30, 90, 16, 7, 18, 20, 27, 84, 103, 5, 34, 75, 85, 79, 14, 86], [113, 123, 39, 13, 73, 71, 17, 83, 115, 74, 92, 78, 76, 59, 60, 55, 126, 63, 114, 57, 120, 127, 47, 116, 52, 62, 95, 119, 118, 117, 53, 121, 54, 69, 112, 82, 51, 56, 99, 46, 122, 109, 50, 124, 3, 58, 111, 48, 6, 100, 16, 125, 49, 110, 61, 108, 45, 88, 11, 44, 75, 106, 37, 107, 42, 43, 31, 86, 28, 72, 21, 98, 94, 15, 84, 41, 105, 67, 104, 93, 5, 36, 68, 85, 65, 38, 102, 91, 101, 40, 35, 97, 30, 25, 1, 64, 90, 22, 103, 33, 96, 87, 20, 32, 23, 29, 34, 2, 80, 8, 27, 89, 18, 81, 24, 14, 4, 26, 0, 77, 12, 19, 79, 10, 70, 66, 7, 9], [123, 39, 113, 112, 49, 32, 54, 95, 60, 99, 63, 115, 120, 90, 83, 124, 122, 48, 43, 13, 121, 94, 87, 125, 119, 61, 98, 56, 53, 111, 57, 85, 40, 127, 17, 38, 59, 62, 118, 36, 55, 114, 102, 105, 37, 110, 97, 92, 101, 81, 26, 47, 104, 109, 28, 35, 51, 45, 46, 96, 108, 116, 107, 50, 30, 41, 100, 106, 74, 52, 117, 126, 31, 44, 23, 42, 58, 82, 93, 33, 73, 25, 91, 29, 22, 84, 6, 21, 78, 18, 79, 86, 89, 34, 88, 11, 27, 24, 14, 15, 19, 76, 16, 75, 103, 20, 69, 10, 71, 8, 80, 72, 66, 77, 7, 3, 4, 70, 68, 12, 67, 64, 2, 1, 65, 9, 0, 5], [113, 123, 39, 119, 52, 120, 63, 126, 121, 118, 53, 57, 115, 106, 114, 47, 56, 13, 59, 116, 55, 51, 60, 58, 122, 127, 117, 111, 46, 50, 54, 92, 48, 109, 110, 62, 45, 43, 124, 108, 107, 61, 44, 74, 42, 112, 105, 125, 73, 49, 69, 101, 82, 41, 76, 64, 83, 99, 21, 17, 102, 104, 78, 5, 38, 71, 91, 1, 6, 40, 3, 66, 70, 72, 37, 100, 90, 30, 97, 8, 25, 81, 36, 15, 4, 89, 93, 26, 75, 35, 16, 96, 80, 19, 2, 11, 88, 33, 68, 14, 9, 84, 79, 98, 27, 87, 86, 32, 31, 67, 24, 18, 103, 10, 95, 20, 94, 12, 28, 65, 29, 85, 34, 23, 22, 7, 77, 0], [62, 63, 39, 53, 127, 32, 57, 36, 54, 121, 107, 47, 116, 46, 115, 59, 52, 55, 50, 51, 41, 105, 58, 61, 123, 96, 60, 125, 91, 27, 113, 122, 120, 106, 48, 124, 44, 103, 117, 126, 114, 98, 49, 119, 56, 118, 110, 45, 85, 43, 35, 38, 89, 112, 109, 101, 104, 40, 111, 108, 81, 37, 25, 42, 99, 22, 100, 34, 102, 92, 94, 87, 19, 33, 31, 23, 95, 0, 93, 21, 86, 26, 20, 97, 78, 17, 10, 80, 12, 29, 28, 83, 90, 66, 74, 65, 77, 82, 15, 30, 64, 24, 71, 72, 18, 69, 9, 84, 67, 76, 6, 1, 3, 68, 13, 11, 5, 4, 8, 14, 16, 88, 2, 75, 79, 70, 7, 73], [39, 63, 62, 24, 84, 94, 33, 18, 53, 86, 26, 88, 92, 80, 32, 78, 90, 127, 14, 29, 75, 73, 23, 30, 16, 121, 67, 17, 57, 82, 70, 54, 9, 95, 107, 1, 98, 116, 58, 47, 115, 15, 50, 27, 59, 52, 21, 22, 74, 60, 20, 55, 46, 83, 11, 51, 93, 8, 13, 85, 61, 31, 34, 125, 96, 123, 124, 91, 28, 79, 117, 113, 120, 4, 64, 77, 41, 76, 81, 5, 89, 25, 6, 122, 42, 72, 48, 12, 2, 126, 36, 35, 65, 118, 49, 102, 38, 87, 110, 101, 103, 114, 56, 19, 71, 119, 106, 108, 45, 37, 7, 97, 104, 100, 109, 111, 44, 112, 43, 105, 99, 3, 10, 40, 69, 68, 66, 0], [63, 62, 39, 53, 127, 57, 32, 54, 116, 121, 52, 59, 55, 47, 50, 115, 125, 61, 46, 51, 58, 60, 96, 123, 120, 122, 90, 113, 91, 86, 117, 124, 41, 48, 126, 34, 119, 118, 56, 106, 114, 92, 27, 109, 107, 101, 49, 110, 26, 35, 33, 89, 45, 21, 111, 102, 105, 108, 100, 36, 85, 112, 93, 44, 42, 43, 104, 99, 40, 98, 22, 29, 81, 88, 37, 10, 94, 64, 38, 103, 95, 16, 66, 65, 31, 19, 0, 97, 14, 25, 30, 67, 83, 72, 17, 24, 76, 87, 20, 6, 23, 74, 28, 78, 68, 69, 9, 12, 8, 15, 11, 84, 1, 5, 7, 79, 77, 82, 80, 4, 71, 18, 70, 73, 2, 3, 13, 75], [39, 62, 63, 94, 84, 24, 88, 53, 86, 21, 32, 74, 14, 10, 33, 26, 83, 90, 18, 127, 8, 92, 20, 69, 30, 23, 36, 57, 16, 7, 76, 66, 68, 121, 67, 54, 96, 27, 116, 89, 73, 87, 13, 52, 47, 82, 55, 28, 72, 115, 59, 17, 50, 6, 85, 77, 60, 70, 58, 34, 22, 80, 46, 95, 5, 81, 65, 61, 107, 125, 29, 51, 98, 49, 123, 93, 91, 9, 117, 124, 101, 108, 19, 120, 79, 25, 11, 113, 78, 15, 31, 38, 35, 64, 12, 103, 105, 99, 118, 41, 110, 106, 48, 122, 56, 100, 45, 97, 126, 37, 119, 104, 114, 40, 71, 43, 109, 4, 44, 102, 42, 111, 112, 1, 75, 0, 3, 2], [113, 48, 101, 112, 26, 94, 87, 18, 123, 20, 49, 86, 122, 59, 120, 79, 57, 62, 124, 58, 55, 60, 54, 115, 76, 43, 125, 37, 53, 119, 6, 39, 45, 51, 117, 126, 52, 114, 63, 61, 56, 50, 118, 106, 30, 75, 98, 34, 88, 32, 104, 107, 111, 127, 47, 110, 41, 116, 121, 92, 82, 108, 33, 44, 29, 102, 40, 42, 15, 103, 89, 90, 96, 105, 109, 99, 46, 23, 36, 35, 24, 100, 17, 22, 13, 31, 19, 97, 28, 70, 25, 91, 16, 95, 85, 27, 84, 11, 78, 21, 38, 83, 93, 12, 81, 14, 77, 73, 67, 8, 74, 9, 72, 71, 80, 3, 10, 7, 5, 4, 65, 68, 69, 66, 1, 2, 0, 64], [48, 113, 101, 112, 26, 122, 18, 94, 59, 87, 20, 43, 123, 57, 76, 79, 30, 63, 92, 98, 49, 53, 61, 124, 51, 88, 120, 86, 58, 117, 119, 39, 60, 121, 55, 125, 62, 115, 6, 50, 52, 89, 54, 56, 126, 96, 107, 114, 45, 102, 110, 29, 118, 47, 82, 35, 127, 34, 40, 46, 106, 24, 111, 32, 42, 100, 116, 108, 105, 37, 109, 44, 75, 90, 103, 104, 93, 36, 97, 25, 41, 31, 13, 27, 22, 28, 16, 95, 99, 38, 19, 91, 83, 12, 23, 85, 81, 33, 15, 17, 84, 21, 14, 11, 9, 78, 70, 8, 72, 74, 73, 77, 80, 67, 3, 71, 10, 7, 5, 65, 68, 4, 69, 1, 66, 2, 0, 64], [48, 113, 74, 16, 101, 7, 13, 68, 64, 69, 4, 112, 2, 122, 0, 87, 5, 1, 66, 65, 94, 67, 86, 3, 20, 58, 71, 53, 54, 98, 125, 55, 107, 57, 70, 6, 60, 49, 56, 120, 119, 52, 114, 121, 32, 26, 18, 62, 117, 124, 50, 118, 10, 11, 75, 110, 127, 30, 63, 47, 89, 51, 45, 82, 105, 104, 115, 61, 80, 106, 8, 103, 28, 76, 44, 77, 73, 96, 72, 108, 12, 126, 111, 79, 59, 42, 9, 109, 123, 24, 90, 23, 46, 85, 84, 93, 81, 37, 15, 14, 22, 78, 29, 19, 116, 17, 40, 34, 102, 83, 27, 35, 43, 21, 91, 33, 88, 92, 31, 25, 97, 95, 41, 38, 39, 100, 99, 36], [48, 113, 101, 122, 94, 112, 87, 26, 18, 59, 20, 123, 86, 49, 43, 79, 98, 57, 19, 120, 124, 30, 89, 32, 8, 115, 51, 76, 107, 58, 39, 21, 97, 82, 61, 63, 60, 62, 88, 29, 13, 52, 42, 6, 125, 34, 53, 117, 56, 55, 37, 119, 111, 96, 54, 28, 106, 44, 100, 114, 16, 75, 108, 104, 102, 85, 118, 126, 50, 105, 17, 45, 36, 27, 103, 116, 127, 31, 40, 91, 110, 92, 25, 121, 35, 47, 41, 22, 23, 109, 38, 33, 83, 46, 90, 95, 99, 74, 84, 24, 93, 12, 14, 78, 81, 73, 15, 11, 9, 72, 70, 3, 67, 77, 7, 80, 71, 10, 69, 65, 4, 5, 68, 1, 66, 2, 0, 64], [56, 113, 63, 102, 117, 49, 124, 68, 53, 76, 59, 85, 55, 5, 114, 60, 116, 58, 94, 121, 61, 89, 16, 7, 2, 66, 72, 50, 62, 8, 9, 12, 111, 51, 80, 115, 123, 118, 6, 126, 122, 48, 77, 54, 120, 52, 101, 40, 125, 69, 87, 91, 46, 119, 21, 27, 1, 3, 4, 29, 30, 96, 127, 42, 112, 83, 105, 92, 43, 44, 57, 35, 39, 36, 75, 22, 45, 109, 17, 24, 107, 110, 14, 104, 108, 88, 103, 47, 18, 79, 25, 23, 78, 41, 64, 15, 28, 97, 37, 93, 106, 20, 99, 98, 95, 81, 19, 26, 33, 82, 90, 100, 10, 38, 70, 31, 11, 84, 34, 73, 32, 13, 67, 65, 74, 86, 71, 0], [113, 56, 63, 124, 53, 59, 117, 55, 9, 49, 40, 116, 94, 58, 81, 69, 121, 61, 77, 62, 50, 114, 48, 118, 54, 60, 64, 111, 125, 51, 115, 123, 122, 126, 2, 120, 1, 24, 43, 57, 127, 68, 8, 52, 12, 39, 46, 119, 21, 70, 101, 112, 3, 45, 105, 107, 104, 109, 110, 47, 106, 44, 102, 108, 96, 30, 79, 6, 42, 92, 85, 72, 41, 103, 26, 36, 33, 7, 37, 15, 98, 100, 66, 16, 35, 25, 65, 83, 74, 97, 34, 93, 73, 95, 99, 90, 4, 17, 31, 28, 88, 91, 75, 38, 89, 67, 19, 23, 10, 87, 0, 5, 29, 13, 20, 32, 71, 78, 84, 27, 18, 14, 11, 82, 80, 76, 86, 22], [113, 56, 63, 94, 124, 49, 102, 117, 24, 96, 53, 38, 59, 104, 19, 55, 92, 43, 40, 58, 26, 91, 89, 35, 9, 116, 48, 114, 118, 37, 121, 22, 62, 51, 61, 50, 15, 60, 30, 122, 97, 126, 87, 46, 111, 54, 69, 29, 123, 52, 83, 120, 1, 85, 33, 125, 127, 17, 36, 77, 98, 12, 64, 81, 68, 119, 115, 3, 73, 105, 57, 75, 90, 7, 112, 86, 45, 79, 39, 109, 2, 8, 108, 44, 99, 6, 110, 34, 95, 25, 28, 76, 100, 10, 106, 27, 16, 93, 101, 18, 82, 78, 14, 47, 42, 41, 103, 88, 70, 107, 23, 31, 13, 21, 72, 20, 4, 32, 84, 80, 11, 74, 66, 67, 71, 0, 65, 5], [113, 56, 63, 124, 49, 117, 53, 40, 59, 114, 55, 102, 58, 116, 48, 24, 50, 94, 26, 121, 51, 54, 96, 60, 62, 118, 61, 87, 89, 126, 35, 43, 120, 77, 122, 123, 81, 111, 125, 57, 109, 52, 105, 9, 46, 17, 115, 104, 29, 119, 127, 92, 97, 30, 112, 99, 44, 22, 70, 45, 91, 110, 1, 37, 42, 108, 107, 103, 100, 101, 47, 38, 36, 15, 39, 6, 41, 27, 85, 106, 3, 20, 31, 90, 98, 33, 34, 88, 14, 73, 28, 23, 95, 19, 72, 64, 68, 12, 93, 79, 18, 13, 16, 86, 32, 82, 21, 25, 8, 69, 83, 78, 74, 11, 84, 80, 76, 10, 2, 75, 7, 65, 66, 71, 5, 0, 4, 67]], "model.layers.14.self_attn.k_proj": [[60, 101, 97, 62, 86, 117, 120, 94, 51, 59, 53, 90, 118, 58, 126, 57, 48, 111, 123, 121, 115, 54, 116, 87, 61, 81, 114, 125, 119, 56, 52, 127, 124, 55, 107, 19, 110, 122, 105, 50, 63, 106, 112, 89, 37, 40, 46, 44, 109, 45, 47, 43, 49, 102, 88, 108, 113, 12, 103, 21, 92, 104, 78, 15, 25, 42, 41, 32, 38, 39, 16, 93, 1, 34, 74, 67, 100, 27, 36, 71, 70, 31, 95, 13, 91, 98, 35, 68, 0, 20, 83, 73, 24, 33, 99, 96, 14, 79, 84, 72, 22, 28, 29, 11, 85, 80, 26, 77, 30, 66, 18, 8, 23, 17, 76, 82, 10, 9, 75, 5, 69, 2, 7, 65, 64, 3, 6, 4], [52, 38, 97, 86, 60, 93, 124, 56, 119, 123, 125, 61, 50, 57, 26, 113, 110, 62, 121, 127, 55, 81, 58, 116, 63, 109, 112, 53, 45, 115, 117, 108, 42, 49, 15, 46, 54, 106, 48, 111, 122, 118, 120, 88, 47, 84, 33, 114, 126, 82, 107, 103, 43, 39, 59, 22, 40, 102, 41, 87, 37, 105, 85, 36, 44, 75, 83, 100, 96, 77, 35, 13, 101, 104, 9, 99, 0, 98, 31, 16, 27, 30, 92, 34, 25, 32, 95, 51, 80, 17, 89, 28, 94, 10, 76, 69, 65, 29, 19, 14, 21, 73, 70, 91, 2, 24, 78, 72, 12, 71, 7, 4, 18, 90, 67, 68, 23, 8, 20, 11, 79, 5, 3, 74, 1, 66, 6, 64], [53, 39, 33, 87, 117, 14, 81, 80, 7, 9, 12, 74, 0, 42, 11, 65, 85, 2, 3, 60, 4, 48, 69, 70, 114, 83, 116, 61, 51, 64, 127, 54, 66, 67, 122, 79, 126, 113, 55, 118, 63, 124, 52, 5, 72, 120, 68, 88, 91, 109, 89, 25, 115, 112, 121, 108, 6, 45, 103, 59, 40, 73, 71, 22, 49, 1, 41, 34, 105, 77, 104, 23, 26, 10, 21, 43, 106, 44, 47, 96, 13, 50, 30, 92, 16, 110, 119, 18, 93, 123, 28, 76, 20, 58, 125, 94, 75, 8, 78, 29, 102, 36, 24, 46, 101, 90, 111, 56, 97, 57, 107, 31, 95, 35, 99, 19, 37, 27, 98, 38, 84, 100, 62, 82, 17, 86, 32, 15], [56, 38, 127, 86, 97, 44, 60, 61, 119, 114, 50, 113, 122, 116, 123, 62, 110, 63, 29, 115, 126, 47, 112, 59, 120, 51, 90, 55, 121, 39, 53, 49, 57, 124, 24, 117, 58, 45, 30, 78, 48, 118, 54, 125, 6, 109, 111, 17, 11, 1, 103, 52, 46, 107, 101, 42, 0, 41, 102, 105, 106, 28, 23, 108, 89, 20, 40, 104, 16, 43, 33, 35, 100, 98, 76, 25, 96, 82, 68, 15, 84, 85, 99, 32, 77, 37, 18, 74, 95, 36, 2, 34, 91, 19, 7, 64, 21, 22, 12, 31, 80, 93, 27, 3, 92, 87, 10, 67, 13, 79, 72, 94, 9, 26, 71, 69, 5, 73, 8, 66, 81, 14, 75, 4, 65, 83, 70, 88], [123, 113, 103, 22, 34, 60, 122, 54, 124, 61, 53, 125, 111, 56, 57, 49, 62, 119, 121, 55, 18, 47, 63, 114, 127, 115, 51, 40, 59, 96, 112, 93, 117, 116, 52, 126, 120, 46, 58, 118, 50, 44, 48, 109, 45, 107, 42, 104, 27, 98, 110, 43, 75, 41, 108, 106, 14, 65, 38, 100, 7, 35, 105, 92, 68, 84, 102, 37, 101, 0, 26, 67, 80, 36, 15, 9, 32, 31, 5, 33, 24, 30, 8, 87, 97, 88, 77, 94, 99, 25, 95, 6, 79, 20, 10, 39, 28, 81, 72, 91, 23, 76, 83, 70, 2, 90, 89, 12, 29, 66, 78, 21, 82, 17, 16, 11, 85, 74, 64, 19, 71, 4, 69, 86, 3, 1, 73, 13], [63, 62, 103, 86, 127, 53, 57, 121, 54, 116, 55, 30, 59, 50, 52, 58, 115, 125, 123, 97, 120, 61, 92, 60, 46, 84, 18, 51, 113, 26, 117, 124, 96, 118, 47, 126, 48, 88, 49, 24, 110, 114, 122, 100, 56, 119, 80, 15, 108, 106, 91, 111, 45, 107, 42, 14, 112, 109, 101, 38, 41, 43, 102, 40, 33, 75, 99, 44, 9, 105, 37, 31, 77, 83, 104, 93, 35, 94, 21, 17, 16, 25, 13, 98, 85, 36, 72, 82, 23, 34, 11, 19, 79, 29, 6, 66, 78, 32, 68, 0, 12, 95, 69, 22, 27, 81, 89, 28, 10, 87, 71, 90, 20, 39, 74, 76, 65, 3, 7, 73, 70, 67, 8, 1, 2, 4, 5, 64], [112, 64, 37, 113, 48, 49, 1, 69, 16, 74, 30, 68, 86, 87, 7, 123, 13, 2, 53, 20, 122, 34, 43, 58, 57, 96, 121, 118, 59, 120, 66, 125, 62, 56, 119, 54, 60, 75, 3, 26, 18, 55, 4, 52, 6, 115, 50, 114, 124, 111, 117, 126, 63, 79, 61, 0, 51, 47, 76, 127, 67, 110, 103, 116, 41, 109, 46, 106, 108, 93, 45, 104, 89, 38, 44, 36, 105, 8, 29, 35, 39, 42, 100, 71, 81, 102, 28, 15, 33, 21, 31, 90, 40, 27, 107, 25, 94, 19, 78, 24, 84, 99, 92, 88, 70, 73, 97, 85, 22, 72, 80, 5, 9, 91, 98, 95, 14, 77, 17, 32, 23, 83, 65, 12, 10, 82, 101, 11], [56, 113, 63, 38, 99, 124, 117, 59, 53, 22, 58, 116, 114, 55, 62, 122, 60, 32, 61, 50, 126, 51, 123, 111, 48, 121, 54, 127, 115, 118, 52, 46, 120, 125, 119, 57, 108, 112, 49, 45, 110, 109, 27, 40, 47, 35, 43, 104, 44, 89, 107, 97, 41, 42, 101, 106, 103, 105, 39, 92, 30, 98, 93, 33, 36, 87, 100, 102, 37, 19, 96, 18, 31, 13, 95, 34, 24, 17, 26, 91, 86, 20, 15, 29, 94, 28, 88, 85, 65, 2, 23, 0, 70, 80, 90, 7, 83, 10, 78, 74, 82, 25, 84, 14, 73, 8, 11, 3, 67, 81, 79, 75, 68, 16, 69, 21, 5, 71, 76, 4, 9, 12, 77, 72, 66, 64, 1, 6]], "model.layers.14.self_attn.qk_proj": [[56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 120, 121, 124, 122, 57, 125, 127, 61, 103, 115, 55, 51, 59, 22, 97, 50, 119, 114, 38, 102, 49, 54, 39, 37, 118, 23, 126, 101, 47, 87, 111, 94, 86, 108, 16, 80, 33, 46, 0, 42, 110, 17, 26, 64, 109, 43, 90, 24, 106, 45, 88, 81, 93, 78, 14, 44, 107, 10, 69, 105, 29, 7, 5, 83, 74, 71, 12, 84, 11, 30, 40, 34, 19, 13, 4, 77, 2, 85, 9, 96, 73, 20, 21, 1, 66, 76, 41, 75, 67, 68, 28, 104, 99, 15, 65, 18, 82, 3, 36, 89, 35, 79, 92, 25, 98, 32, 70, 6, 100, 8, 31, 91, 95, 27, 72], [113, 56, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 124, 120, 127, 61, 125, 121, 122, 55, 103, 57, 115, 51, 59, 97, 102, 39, 22, 49, 38, 114, 54, 50, 23, 118, 37, 101, 119, 126, 94, 111, 47, 87, 109, 33, 86, 16, 64, 108, 43, 80, 0, 24, 17, 106, 90, 42, 46, 110, 45, 10, 26, 88, 14, 107, 78, 81, 93, 44, 29, 74, 4, 68, 71, 11, 85, 5, 105, 83, 69, 12, 9, 99, 75, 41, 34, 96, 19, 7, 21, 28, 77, 84, 40, 104, 65, 13, 20, 18, 1, 30, 76, 6, 92, 2, 73, 66, 67, 32, 36, 79, 25, 82, 70, 15, 98, 100, 35, 31, 89, 91, 8, 3, 27, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 58, 127, 124, 61, 120, 122, 103, 125, 121, 59, 115, 57, 39, 54, 55, 97, 51, 49, 38, 50, 102, 114, 118, 47, 101, 119, 126, 37, 111, 46, 23, 22, 87, 94, 43, 108, 86, 0, 33, 64, 16, 110, 106, 109, 107, 80, 45, 88, 44, 90, 24, 42, 26, 30, 17, 93, 14, 29, 71, 10, 41, 105, 81, 78, 85, 69, 83, 34, 5, 99, 96, 12, 74, 7, 11, 4, 40, 65, 19, 68, 13, 66, 28, 104, 1, 9, 21, 75, 77, 76, 20, 73, 6, 84, 2, 35, 98, 32, 3, 25, 18, 67, 36, 82, 15, 92, 89, 70, 91, 31, 27, 79, 72, 100, 95, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 116, 58, 125, 121, 122, 124, 61, 120, 127, 57, 49, 103, 114, 55, 39, 54, 97, 115, 59, 50, 51, 102, 111, 119, 101, 22, 38, 126, 47, 118, 23, 37, 94, 110, 46, 87, 108, 43, 0, 64, 16, 86, 80, 45, 109, 33, 90, 106, 24, 107, 17, 44, 93, 26, 42, 78, 88, 29, 81, 41, 69, 74, 2, 19, 7, 10, 104, 14, 105, 71, 30, 85, 96, 83, 13, 34, 12, 66, 76, 84, 77, 40, 65, 5, 11, 1, 9, 73, 4, 28, 3, 68, 20, 99, 75, 92, 36, 6, 32, 21, 98, 67, 18, 25, 100, 89, 15, 35, 82, 79, 70, 72, 31, 91, 95, 27, 8], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 122, 58, 121, 125, 116, 57, 61, 120, 55, 124, 103, 49, 127, 115, 114, 39, 59, 97, 118, 38, 51, 50, 22, 46, 102, 101, 47, 23, 54, 126, 94, 37, 108, 111, 110, 86, 0, 16, 119, 87, 80, 64, 33, 43, 109, 90, 106, 14, 81, 42, 44, 107, 24, 17, 26, 78, 71, 10, 30, 93, 2, 45, 66, 83, 11, 74, 69, 19, 88, 29, 105, 5, 4, 104, 68, 84, 13, 9, 12, 76, 41, 7, 40, 96, 73, 21, 1, 34, 75, 18, 85, 20, 77, 28, 65, 3, 79, 92, 82, 32, 89, 15, 99, 98, 6, 67, 35, 100, 70, 36, 25, 27, 91, 31, 72, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 121, 55, 127, 125, 58, 124, 120, 122, 103, 57, 61, 59, 49, 115, 114, 97, 22, 118, 39, 102, 51, 38, 54, 50, 119, 23, 37, 101, 109, 94, 87, 108, 126, 110, 111, 86, 47, 16, 80, 64, 17, 33, 14, 43, 90, 26, 42, 81, 0, 44, 24, 78, 46, 45, 106, 29, 11, 4, 88, 93, 10, 9, 83, 12, 74, 76, 84, 30, 75, 41, 85, 19, 7, 71, 105, 96, 18, 68, 107, 69, 66, 20, 5, 13, 73, 77, 104, 21, 32, 82, 79, 65, 2, 15, 3, 40, 34, 28, 89, 98, 92, 1, 70, 99, 36, 67, 6, 31, 25, 35, 100, 72, 91, 27, 95, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 116, 121, 120, 127, 55, 58, 115, 125, 61, 49, 122, 114, 59, 57, 103, 124, 22, 118, 97, 102, 23, 119, 38, 51, 39, 37, 101, 94, 50, 54, 87, 126, 86, 109, 111, 16, 47, 110, 80, 17, 26, 43, 108, 33, 90, 14, 46, 78, 81, 42, 24, 64, 45, 106, 85, 29, 0, 88, 44, 30, 11, 10, 105, 74, 104, 9, 75, 76, 93, 84, 71, 107, 20, 73, 7, 69, 83, 12, 15, 41, 5, 77, 96, 21, 19, 28, 40, 13, 82, 18, 32, 2, 66, 34, 67, 79, 4, 99, 36, 6, 98, 92, 68, 65, 89, 1, 70, 25, 3, 35, 31, 100, 27, 72, 91, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 58, 121, 116, 59, 115, 55, 114, 124, 57, 122, 127, 103, 49, 97, 125, 38, 51, 22, 102, 61, 119, 23, 118, 54, 37, 101, 39, 126, 94, 87, 47, 50, 86, 46, 111, 109, 110, 16, 43, 108, 33, 26, 0, 64, 90, 80, 88, 78, 45, 14, 81, 17, 85, 107, 42, 29, 24, 30, 44, 84, 106, 75, 10, 96, 83, 11, 105, 93, 21, 74, 20, 5, 34, 71, 104, 65, 76, 9, 77, 7, 15, 41, 13, 18, 82, 4, 69, 1, 68, 12, 73, 2, 66, 36, 70, 19, 32, 79, 35, 28, 92, 98, 99, 89, 25, 3, 6, 91, 40, 67, 31, 100, 72, 27, 95, 8], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 125, 58, 127, 121, 122, 49, 124, 55, 115, 57, 103, 97, 38, 54, 61, 51, 102, 114, 50, 101, 59, 39, 118, 23, 22, 119, 37, 111, 94, 87, 47, 43, 108, 86, 109, 80, 64, 0, 126, 110, 16, 33, 46, 106, 88, 78, 90, 17, 14, 26, 85, 105, 44, 45, 81, 29, 24, 10, 42, 30, 93, 4, 96, 74, 83, 84, 19, 104, 1, 71, 107, 41, 5, 34, 7, 76, 99, 21, 69, 20, 65, 75, 68, 66, 13, 28, 73, 2, 77, 11, 12, 15, 9, 32, 98, 18, 70, 25, 92, 36, 40, 35, 82, 6, 3, 79, 67, 91, 89, 100, 31, 27, 72, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 116, 57, 55, 125, 124, 121, 49, 122, 114, 127, 59, 51, 97, 103, 38, 115, 61, 39, 50, 54, 119, 23, 22, 111, 118, 102, 37, 101, 87, 94, 110, 126, 33, 64, 47, 0, 16, 80, 108, 86, 46, 17, 107, 14, 78, 45, 90, 42, 109, 24, 44, 81, 43, 26, 93, 10, 29, 71, 74, 7, 11, 88, 85, 99, 106, 30, 68, 69, 13, 12, 83, 34, 28, 84, 105, 5, 76, 96, 9, 66, 73, 4, 21, 18, 104, 75, 77, 20, 1, 70, 67, 65, 3, 2, 41, 15, 19, 82, 79, 40, 32, 89, 98, 6, 92, 100, 91, 35, 25, 36, 8, 31, 95, 27, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 116, 58, 120, 55, 121, 124, 57, 49, 122, 127, 125, 103, 115, 114, 59, 39, 61, 51, 97, 50, 22, 102, 54, 23, 126, 118, 38, 101, 37, 87, 119, 47, 110, 0, 86, 94, 64, 16, 108, 80, 111, 17, 109, 46, 33, 90, 10, 106, 14, 74, 42, 78, 107, 29, 93, 45, 81, 26, 24, 43, 30, 5, 44, 71, 69, 105, 88, 66, 7, 12, 76, 13, 19, 68, 73, 104, 83, 2, 11, 84, 15, 65, 96, 18, 85, 41, 9, 77, 70, 4, 20, 40, 75, 1, 21, 34, 79, 3, 82, 28, 99, 67, 6, 36, 32, 89, 98, 92, 35, 91, 100, 8, 25, 27, 31, 72, 95], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 116, 120, 58, 121, 55, 122, 127, 124, 103, 115, 114, 49, 57, 61, 125, 51, 22, 39, 118, 23, 59, 38, 102, 97, 119, 37, 54, 101, 50, 126, 94, 111, 110, 86, 87, 108, 16, 80, 109, 17, 64, 107, 0, 47, 14, 33, 42, 46, 81, 78, 26, 90, 10, 43, 74, 29, 44, 45, 106, 30, 24, 68, 105, 88, 93, 83, 69, 41, 84, 20, 4, 76, 5, 75, 85, 73, 7, 11, 19, 12, 13, 66, 2, 77, 9, 104, 28, 71, 1, 96, 32, 18, 82, 15, 21, 67, 34, 3, 70, 79, 6, 99, 40, 65, 35, 89, 25, 36, 98, 100, 8, 31, 91, 92, 27, 72, 95], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 58, 127, 115, 124, 103, 55, 57, 59, 121, 125, 114, 61, 38, 50, 49, 51, 122, 119, 39, 22, 102, 23, 97, 111, 54, 37, 118, 101, 94, 87, 108, 126, 47, 86, 110, 33, 17, 109, 46, 16, 80, 88, 14, 81, 26, 42, 64, 43, 0, 44, 106, 45, 90, 24, 107, 78, 29, 105, 93, 30, 10, 74, 104, 75, 41, 11, 21, 7, 83, 85, 68, 4, 69, 84, 99, 20, 96, 19, 28, 77, 34, 76, 5, 32, 82, 13, 18, 73, 92, 71, 66, 36, 98, 9, 1, 15, 12, 2, 40, 3, 65, 6, 31, 25, 100, 91, 89, 95, 79, 35, 70, 27, 67, 8, 72], [56, 113, 53, 52, 60, 123, 62, 63, 48, 112, 117, 120, 116, 127, 58, 59, 125, 115, 124, 49, 114, 55, 57, 51, 121, 103, 119, 39, 122, 61, 22, 50, 97, 111, 54, 118, 102, 37, 38, 87, 23, 64, 110, 94, 101, 86, 109, 108, 47, 126, 0, 17, 42, 46, 80, 14, 74, 33, 16, 44, 81, 45, 90, 43, 26, 78, 88, 24, 29, 10, 7, 71, 73, 105, 18, 75, 83, 93, 76, 30, 69, 11, 104, 77, 107, 19, 5, 13, 66, 41, 9, 6, 4, 96, 106, 1, 84, 65, 34, 20, 21, 99, 12, 85, 15, 28, 68, 40, 36, 2, 79, 32, 82, 70, 67, 3, 92, 25, 98, 35, 27, 8, 91, 89, 31, 100, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 121, 116, 124, 55, 127, 125, 49, 58, 114, 57, 59, 122, 103, 39, 61, 115, 50, 119, 51, 97, 22, 38, 118, 102, 23, 54, 37, 101, 94, 111, 110, 108, 126, 86, 87, 109, 16, 0, 64, 46, 47, 43, 80, 33, 42, 14, 90, 17, 74, 44, 45, 78, 106, 26, 81, 88, 107, 10, 7, 93, 24, 29, 30, 105, 69, 11, 71, 83, 75, 84, 73, 76, 104, 66, 19, 21, 18, 13, 5, 20, 34, 40, 4, 68, 28, 1, 3, 67, 12, 85, 2, 9, 77, 6, 96, 41, 15, 32, 65, 99, 82, 70, 79, 36, 92, 89, 98, 100, 25, 27, 8, 35, 31, 91, 95, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 114, 55, 121, 58, 127, 124, 122, 49, 103, 125, 115, 59, 61, 57, 39, 102, 119, 51, 118, 37, 22, 54, 97, 50, 23, 38, 126, 46, 111, 94, 108, 16, 86, 110, 87, 101, 17, 64, 45, 47, 80, 44, 109, 0, 33, 43, 106, 107, 74, 14, 90, 42, 10, 105, 81, 4, 26, 83, 78, 29, 2, 93, 11, 76, 88, 24, 68, 5, 69, 7, 75, 71, 66, 41, 19, 84, 28, 21, 13, 30, 96, 9, 20, 104, 65, 40, 12, 73, 77, 18, 6, 34, 85, 98, 70, 15, 67, 79, 92, 99, 36, 32, 3, 1, 8, 89, 31, 35, 82, 100, 25, 91, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 58, 116, 114, 121, 124, 127, 55, 115, 103, 59, 39, 125, 57, 49, 122, 50, 119, 61, 38, 118, 22, 51, 97, 54, 23, 102, 126, 37, 101, 94, 86, 111, 87, 110, 17, 46, 108, 44, 16, 45, 109, 80, 33, 0, 26, 47, 64, 107, 90, 106, 78, 43, 10, 42, 93, 29, 88, 81, 30, 14, 105, 24, 74, 11, 7, 28, 5, 75, 71, 41, 77, 21, 40, 34, 18, 83, 96, 68, 104, 4, 98, 9, 13, 85, 20, 12, 69, 84, 36, 66, 65, 76, 19, 99, 6, 32, 1, 2, 92, 73, 70, 79, 82, 35, 25, 15, 100, 3, 31, 91, 89, 8, 27, 67, 95, 72], [56, 113, 53, 52, 123, 60, 63, 62, 48, 112, 117, 120, 116, 58, 121, 61, 55, 124, 115, 103, 127, 125, 59, 49, 114, 122, 39, 38, 119, 57, 118, 22, 97, 23, 54, 50, 51, 101, 37, 102, 126, 87, 86, 94, 47, 111, 46, 64, 0, 108, 16, 109, 33, 110, 44, 45, 80, 106, 17, 42, 14, 90, 107, 88, 43, 26, 78, 81, 24, 30, 93, 105, 74, 75, 7, 28, 10, 66, 71, 96, 19, 29, 69, 40, 76, 83, 11, 104, 20, 65, 34, 99, 77, 21, 84, 85, 9, 5, 18, 4, 68, 1, 82, 70, 13, 41, 73, 12, 2, 36, 67, 15, 6, 3, 92, 79, 89, 31, 35, 25, 98, 91, 32, 27, 100, 8, 95, 72], [56, 113, 53, 123, 60, 52, 63, 62, 48, 112, 117, 58, 121, 116, 120, 103, 124, 114, 122, 125, 127, 38, 57, 55, 49, 59, 115, 51, 97, 61, 54, 39, 50, 119, 101, 118, 22, 102, 23, 94, 111, 110, 37, 87, 33, 109, 47, 46, 64, 126, 108, 0, 86, 16, 44, 80, 107, 106, 90, 45, 24, 26, 43, 78, 10, 17, 81, 30, 93, 40, 105, 42, 88, 14, 28, 34, 104, 96, 74, 65, 85, 29, 99, 7, 71, 69, 19, 20, 4, 13, 5, 83, 21, 76, 98, 66, 75, 12, 84, 41, 9, 11, 18, 68, 77, 2, 92, 1, 35, 25, 3, 73, 82, 70, 32, 36, 67, 100, 91, 89, 31, 15, 79, 6, 27, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 120, 58, 121, 124, 127, 59, 61, 55, 125, 49, 103, 114, 39, 57, 122, 115, 97, 54, 22, 102, 119, 51, 50, 38, 118, 23, 37, 126, 94, 101, 111, 87, 108, 44, 33, 109, 86, 107, 0, 16, 46, 80, 47, 17, 106, 43, 64, 45, 90, 26, 24, 110, 93, 78, 42, 88, 14, 74, 10, 81, 105, 30, 29, 75, 4, 76, 19, 96, 7, 83, 40, 12, 11, 71, 99, 34, 85, 84, 5, 68, 20, 77, 28, 65, 9, 104, 21, 82, 73, 98, 1, 69, 18, 36, 41, 13, 70, 2, 15, 79, 66, 32, 35, 6, 25, 92, 89, 100, 91, 67, 31, 3, 27, 95, 72, 8], [56, 113, 53, 123, 52, 60, 63, 62, 48, 112, 117, 120, 121, 61, 58, 55, 116, 59, 125, 127, 122, 124, 115, 49, 114, 103, 57, 119, 22, 54, 118, 97, 39, 38, 51, 23, 37, 50, 126, 102, 94, 101, 86, 108, 46, 111, 87, 107, 44, 80, 110, 16, 43, 47, 109, 90, 0, 45, 88, 33, 64, 26, 14, 10, 106, 17, 105, 78, 74, 42, 81, 93, 24, 7, 76, 12, 69, 30, 19, 83, 11, 29, 28, 71, 4, 96, 2, 104, 84, 9, 75, 5, 99, 20, 41, 77, 18, 40, 67, 85, 34, 13, 21, 73, 66, 65, 82, 98, 79, 68, 70, 36, 92, 1, 15, 3, 32, 6, 91, 89, 72, 35, 25, 31, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 58, 120, 121, 61, 59, 116, 125, 55, 127, 122, 49, 103, 115, 38, 114, 54, 57, 124, 39, 51, 119, 118, 23, 97, 126, 101, 22, 102, 37, 50, 87, 86, 47, 111, 94, 80, 64, 108, 46, 90, 16, 110, 0, 33, 109, 107, 44, 45, 43, 17, 26, 88, 106, 14, 10, 105, 93, 78, 42, 81, 24, 74, 29, 19, 7, 83, 5, 76, 30, 104, 28, 71, 65, 69, 34, 40, 20, 75, 9, 84, 85, 11, 18, 77, 96, 4, 68, 12, 13, 41, 21, 73, 2, 99, 1, 82, 70, 79, 3, 98, 36, 92, 91, 66, 15, 89, 31, 25, 35, 32, 6, 72, 67, 100, 27, 95, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 120, 59, 58, 124, 121, 103, 116, 38, 125, 127, 122, 55, 57, 61, 115, 54, 114, 97, 118, 49, 51, 22, 39, 50, 101, 23, 119, 102, 126, 37, 94, 86, 87, 109, 111, 47, 16, 108, 46, 64, 33, 44, 80, 110, 0, 90, 45, 88, 42, 106, 26, 81, 93, 105, 14, 10, 24, 43, 17, 78, 74, 107, 83, 30, 11, 104, 5, 68, 29, 19, 34, 4, 7, 96, 77, 40, 20, 99, 69, 85, 84, 71, 76, 2, 28, 65, 21, 66, 92, 13, 73, 9, 75, 12, 41, 18, 36, 82, 6, 89, 1, 15, 3, 25, 98, 79, 35, 32, 67, 100, 91, 27, 70, 95, 72, 31, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 125, 58, 120, 121, 124, 59, 55, 61, 116, 57, 127, 103, 114, 122, 38, 97, 115, 49, 22, 51, 39, 118, 54, 23, 37, 119, 50, 94, 101, 126, 87, 111, 102, 16, 86, 47, 33, 80, 90, 110, 44, 109, 64, 42, 26, 0, 17, 108, 46, 45, 78, 106, 10, 14, 29, 88, 81, 93, 74, 24, 43, 83, 30, 107, 11, 28, 105, 76, 104, 71, 75, 4, 73, 84, 68, 41, 12, 96, 6, 7, 99, 5, 85, 66, 69, 15, 40, 13, 34, 77, 9, 19, 2, 20, 21, 82, 3, 18, 36, 32, 67, 92, 1, 100, 65, 25, 70, 89, 98, 91, 79, 27, 35, 72, 31, 95, 8], [56, 113, 53, 60, 123, 52, 63, 62, 48, 112, 117, 120, 121, 58, 59, 125, 61, 55, 116, 127, 49, 103, 57, 114, 124, 22, 115, 122, 118, 39, 38, 54, 119, 23, 51, 37, 97, 50, 86, 102, 101, 87, 94, 47, 111, 80, 126, 108, 110, 109, 46, 16, 26, 43, 33, 17, 106, 45, 78, 90, 88, 0, 64, 44, 107, 81, 29, 11, 74, 42, 14, 10, 24, 105, 83, 5, 93, 85, 30, 76, 9, 71, 40, 7, 84, 104, 69, 28, 41, 20, 13, 2, 77, 82, 18, 12, 75, 15, 96, 73, 19, 34, 1, 21, 99, 65, 66, 4, 32, 98, 68, 36, 3, 6, 100, 67, 79, 25, 89, 35, 70, 91, 31, 72, 92, 27, 95, 8], [56, 113, 60, 52, 53, 123, 63, 62, 48, 112, 117, 120, 59, 58, 61, 121, 57, 127, 116, 55, 124, 125, 103, 122, 114, 115, 22, 49, 54, 38, 118, 39, 50, 119, 51, 97, 23, 102, 37, 101, 94, 126, 86, 87, 47, 109, 110, 106, 33, 80, 26, 90, 16, 42, 88, 43, 107, 81, 111, 64, 17, 44, 108, 78, 0, 10, 29, 14, 46, 93, 24, 45, 75, 40, 85, 30, 104, 83, 74, 28, 105, 84, 41, 11, 71, 21, 76, 19, 5, 20, 32, 96, 69, 12, 18, 7, 99, 9, 77, 68, 34, 4, 36, 13, 65, 6, 2, 15, 82, 25, 66, 98, 100, 79, 73, 70, 1, 92, 89, 35, 91, 31, 3, 67, 95, 72, 27, 8], [56, 113, 60, 53, 52, 123, 63, 62, 48, 112, 117, 120, 127, 124, 59, 58, 61, 57, 121, 122, 103, 115, 116, 125, 49, 50, 51, 55, 54, 114, 39, 22, 38, 97, 23, 119, 126, 101, 118, 47, 102, 86, 37, 46, 94, 87, 44, 17, 111, 80, 26, 0, 107, 109, 64, 108, 33, 16, 90, 43, 24, 42, 106, 45, 81, 88, 105, 110, 10, 14, 30, 93, 29, 74, 68, 28, 85, 76, 78, 84, 40, 71, 83, 5, 96, 11, 104, 2, 21, 69, 77, 13, 20, 4, 12, 41, 34, 19, 66, 7, 82, 9, 75, 99, 73, 65, 67, 25, 98, 1, 3, 18, 92, 6, 15, 35, 79, 32, 70, 36, 91, 89, 31, 27, 100, 95, 72, 8], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 124, 120, 121, 58, 116, 125, 122, 61, 127, 57, 55, 59, 114, 115, 103, 97, 22, 54, 39, 49, 38, 118, 51, 50, 101, 102, 23, 126, 37, 119, 94, 87, 86, 64, 0, 80, 108, 16, 47, 43, 90, 109, 111, 33, 26, 17, 81, 45, 46, 14, 110, 88, 106, 10, 93, 78, 44, 107, 42, 29, 71, 74, 11, 76, 105, 73, 24, 83, 75, 9, 84, 30, 12, 85, 69, 4, 7, 82, 28, 66, 65, 40, 77, 96, 104, 19, 1, 21, 5, 20, 34, 41, 13, 68, 18, 67, 2, 79, 99, 3, 70, 25, 6, 91, 32, 92, 89, 15, 98, 36, 31, 27, 35, 8, 100, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 125, 57, 121, 58, 122, 127, 124, 61, 55, 103, 97, 51, 115, 49, 54, 126, 39, 114, 118, 59, 22, 23, 38, 50, 102, 101, 37, 47, 119, 46, 94, 111, 0, 108, 109, 43, 16, 80, 87, 86, 110, 33, 90, 64, 45, 81, 17, 14, 26, 106, 42, 10, 93, 88, 74, 44, 105, 78, 83, 29, 30, 24, 71, 7, 13, 69, 107, 5, 85, 40, 34, 12, 76, 9, 2, 11, 19, 28, 65, 20, 104, 96, 98, 68, 73, 70, 4, 84, 99, 66, 82, 77, 75, 41, 92, 67, 1, 21, 79, 6, 36, 89, 18, 91, 32, 35, 100, 3, 31, 8, 15, 27, 25, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 58, 116, 121, 120, 125, 61, 55, 124, 122, 59, 127, 57, 103, 22, 97, 49, 115, 54, 51, 50, 119, 114, 39, 37, 38, 126, 102, 101, 23, 47, 118, 87, 94, 86, 45, 80, 43, 109, 16, 111, 46, 0, 26, 78, 17, 81, 10, 90, 108, 33, 64, 14, 110, 106, 29, 88, 2, 24, 93, 42, 44, 107, 74, 83, 85, 69, 12, 76, 105, 104, 73, 4, 96, 19, 40, 30, 5, 7, 11, 75, 20, 71, 84, 41, 9, 34, 68, 77, 99, 28, 21, 82, 70, 66, 13, 18, 3, 79, 65, 67, 1, 15, 32, 89, 36, 98, 6, 25, 35, 92, 8, 91, 100, 31, 95, 27, 72], [56, 113, 53, 60, 52, 123, 63, 62, 48, 112, 117, 116, 125, 58, 120, 61, 121, 122, 57, 124, 127, 59, 97, 115, 55, 103, 114, 22, 54, 39, 51, 50, 49, 119, 102, 37, 23, 126, 86, 38, 118, 111, 87, 94, 101, 47, 110, 16, 0, 17, 33, 64, 43, 45, 80, 78, 26, 108, 46, 109, 29, 42, 90, 44, 81, 10, 24, 88, 41, 83, 106, 107, 74, 93, 68, 71, 4, 30, 7, 5, 12, 14, 69, 75, 9, 99, 96, 105, 11, 19, 73, 76, 34, 18, 2, 13, 104, 70, 77, 66, 85, 40, 28, 21, 67, 65, 1, 84, 20, 79, 32, 89, 25, 98, 82, 15, 36, 35, 31, 6, 91, 92, 3, 8, 100, 27, 95, 72], [56, 113, 53, 52, 60, 123, 63, 62, 48, 112, 117, 120, 116, 121, 125, 58, 122, 61, 55, 59, 124, 103, 22, 127, 57, 97, 114, 51, 115, 54, 23, 38, 50, 49, 126, 102, 39, 119, 101, 37, 118, 86, 94, 87, 111, 47, 17, 78, 108, 109, 80, 26, 16, 33, 43, 42, 45, 88, 90, 46, 24, 10, 0, 11, 29, 110, 44, 107, 14, 64, 106, 74, 93, 81, 30, 83, 12, 5, 105, 75, 9, 71, 40, 13, 99, 76, 7, 73, 84, 19, 66, 85, 96, 4, 20, 18, 104, 34, 69, 70, 77, 41, 79, 15, 28, 21, 65, 25, 68, 82, 89, 32, 6, 3, 2, 92, 1, 67, 98, 36, 35, 91, 8, 100, 31, 27, 95, 72]], "model.layers.15.self_attn.q_proj": [[103, 62, 113, 33, 14, 21, 8, 11, 16, 18, 49, 89, 60, 67, 91, 19, 57, 69, 79, 26, 2, 22, 24, 59, 84, 46, 3, 95, 75, 70, 87, 65, 28, 66, 13, 6, 101, 27, 72, 80, 10, 25, 0, 73, 5, 78, 44, 88, 74, 83, 71, 56, 117, 77, 81, 76, 82, 85, 94, 20, 4, 64, 9, 92, 7, 17, 12, 119, 29, 15, 90, 23, 1, 43, 86, 116, 61, 68, 123, 110, 96, 104, 30, 40, 37, 98, 108, 112, 127, 125, 93, 106, 102, 51, 50, 122, 32, 45, 118, 120, 42, 124, 55, 111, 31, 34, 36, 54, 53, 100, 63, 99, 126, 121, 114, 58, 35, 38, 97, 47, 109, 41, 105, 48, 52, 107, 115, 39], [62, 113, 57, 60, 123, 59, 61, 58, 114, 121, 126, 56, 124, 127, 116, 120, 50, 63, 125, 122, 53, 54, 119, 55, 117, 51, 118, 48, 111, 46, 112, 52, 47, 109, 115, 49, 110, 45, 44, 42, 43, 108, 107, 35, 101, 41, 106, 105, 38, 37, 40, 104, 103, 102, 97, 36, 100, 88, 34, 99, 39, 98, 33, 96, 84, 23, 20, 30, 32, 95, 29, 31, 17, 19, 93, 90, 86, 22, 28, 94, 76, 92, 91, 26, 79, 81, 24, 21, 89, 12, 87, 14, 16, 77, 13, 83, 10, 73, 18, 27, 85, 25, 70, 82, 9, 68, 1, 4, 74, 7, 8, 2, 11, 15, 5, 67, 66, 64, 71, 80, 65, 78, 69, 6, 0, 3, 72, 75], [113, 62, 57, 60, 121, 59, 119, 126, 116, 61, 120, 123, 58, 127, 56, 114, 63, 54, 53, 125, 118, 55, 124, 122, 109, 112, 49, 51, 117, 52, 48, 50, 103, 111, 47, 115, 99, 38, 43, 106, 44, 108, 41, 110, 45, 107, 42, 46, 35, 105, 40, 104, 88, 100, 37, 101, 102, 20, 96, 36, 97, 29, 90, 39, 34, 98, 95, 17, 93, 33, 87, 32, 31, 23, 92, 24, 13, 30, 94, 28, 84, 26, 91, 22, 86, 81, 76, 77, 89, 74, 27, 79, 21, 83, 73, 18, 85, 19, 9, 10, 25, 68, 12, 15, 82, 69, 14, 16, 70, 8, 4, 5, 78, 1, 71, 6, 11, 80, 0, 64, 65, 7, 66, 2, 67, 72, 75, 3], [62, 103, 113, 57, 60, 59, 61, 58, 123, 56, 36, 126, 98, 121, 54, 116, 49, 120, 114, 117, 51, 55, 101, 125, 124, 119, 63, 53, 38, 108, 112, 122, 118, 46, 22, 48, 44, 52, 50, 32, 100, 45, 106, 34, 127, 33, 111, 99, 109, 42, 43, 47, 105, 107, 115, 35, 110, 104, 41, 89, 97, 40, 95, 93, 96, 102, 87, 31, 37, 94, 29, 17, 26, 28, 30, 86, 92, 19, 27, 23, 39, 91, 79, 24, 21, 83, 81, 88, 20, 25, 13, 18, 84, 90, 16, 15, 9, 80, 82, 85, 76, 77, 12, 10, 14, 73, 74, 5, 4, 75, 78, 11, 72, 2, 71, 6, 68, 70, 7, 8, 66, 1, 69, 65, 67, 3, 64, 0], [39, 112, 97, 45, 48, 125, 19, 21, 119, 115, 89, 14, 17, 95, 118, 93, 79, 25, 120, 12, 27, 8, 53, 22, 23, 111, 77, 40, 86, 57, 7, 127, 5, 28, 42, 55, 99, 124, 122, 10, 78, 49, 29, 85, 46, 47, 62, 9, 59, 91, 114, 116, 54, 90, 58, 108, 88, 63, 123, 24, 51, 110, 107, 117, 106, 94, 104, 113, 101, 32, 96, 82, 16, 52, 71, 100, 121, 37, 60, 84, 56, 73, 44, 38, 43, 30, 11, 34, 87, 67, 18, 126, 92, 35, 98, 41, 81, 26, 61, 102, 31, 69, 66, 50, 105, 80, 36, 20, 109, 3, 13, 83, 103, 72, 33, 76, 15, 64, 74, 1, 2, 0, 70, 6, 75, 65, 4, 68], [39, 112, 45, 97, 48, 115, 89, 19, 27, 14, 21, 118, 119, 125, 120, 53, 57, 46, 122, 99, 24, 95, 55, 108, 59, 17, 22, 28, 121, 123, 106, 58, 60, 116, 107, 124, 25, 126, 62, 63, 54, 101, 114, 88, 75, 111, 127, 34, 79, 37, 47, 113, 40, 11, 36, 30, 56, 49, 42, 50, 87, 100, 61, 71, 51, 104, 52, 109, 43, 8, 44, 110, 29, 117, 96, 85, 102, 41, 92, 35, 93, 105, 23, 91, 32, 26, 38, 98, 31, 73, 84, 12, 74, 94, 16, 78, 20, 90, 83, 80, 15, 82, 86, 81, 77, 33, 67, 103, 7, 18, 69, 5, 10, 13, 2, 76, 9, 6, 66, 72, 65, 64, 3, 70, 68, 0, 1, 4], [39, 45, 97, 48, 112, 118, 21, 19, 79, 89, 17, 9, 27, 14, 7, 12, 6, 95, 25, 115, 65, 23, 5, 122, 18, 124, 16, 127, 74, 119, 125, 8, 93, 66, 24, 3, 28, 4, 78, 22, 76, 55, 1, 15, 70, 64, 111, 51, 120, 29, 80, 10, 77, 46, 67, 117, 68, 114, 72, 109, 56, 86, 73, 30, 82, 13, 75, 57, 123, 20, 2, 121, 96, 69, 58, 116, 62, 81, 85, 71, 108, 84, 53, 43, 107, 61, 49, 42, 60, 92, 83, 100, 31, 88, 11, 40, 90, 99, 50, 98, 94, 36, 32, 44, 113, 103, 0, 38, 26, 126, 47, 41, 105, 59, 104, 37, 63, 33, 54, 35, 87, 34, 91, 52, 106, 101, 110, 102], [39, 97, 45, 112, 48, 12, 17, 19, 67, 21, 118, 25, 9, 3, 1, 7, 79, 0, 89, 95, 73, 69, 124, 5, 23, 22, 8, 122, 14, 71, 85, 66, 127, 120, 10, 28, 13, 74, 119, 114, 4, 103, 75, 93, 70, 76, 117, 51, 15, 81, 2, 24, 55, 58, 78, 29, 84, 111, 125, 72, 80, 83, 6, 115, 65, 116, 30, 18, 16, 77, 92, 82, 123, 57, 27, 11, 31, 49, 47, 108, 26, 91, 50, 99, 88, 94, 56, 41, 87, 121, 113, 53, 35, 96, 107, 46, 43, 86, 36, 90, 109, 59, 100, 62, 40, 64, 54, 37, 32, 106, 68, 63, 61, 20, 105, 102, 42, 34, 52, 60, 104, 38, 126, 101, 110, 98, 44, 33], [51, 61, 117, 36, 81, 22, 30, 27, 88, 97, 100, 10, 78, 18, 44, 96, 94, 19, 108, 2, 69, 28, 91, 80, 54, 42, 20, 68, 25, 5, 74, 104, 93, 33, 63, 11, 124, 64, 14, 115, 86, 77, 83, 71, 8, 21, 114, 12, 85, 16, 103, 118, 17, 53, 23, 119, 66, 62, 84, 120, 26, 109, 111, 24, 6, 31, 110, 76, 73, 15, 121, 70, 79, 127, 13, 34, 123, 59, 82, 38, 87, 102, 37, 126, 0, 105, 72, 50, 9, 112, 122, 1, 56, 52, 89, 107, 57, 7, 46, 43, 65, 45, 49, 29, 4, 106, 75, 55, 101, 67, 48, 39, 58, 98, 90, 95, 35, 92, 32, 113, 125, 116, 40, 47, 41, 60, 99, 3], [51, 61, 36, 117, 27, 88, 30, 96, 100, 22, 78, 19, 81, 25, 10, 94, 20, 97, 44, 91, 71, 32, 21, 62, 34, 54, 69, 53, 29, 89, 103, 80, 76, 85, 82, 33, 86, 42, 83, 11, 23, 124, 13, 119, 16, 90, 110, 5, 108, 14, 87, 98, 116, 66, 18, 93, 106, 92, 26, 28, 102, 118, 77, 111, 17, 84, 101, 45, 31, 43, 6, 40, 114, 38, 74, 99, 123, 50, 24, 35, 41, 2, 109, 95, 104, 121, 12, 122, 126, 73, 48, 39, 37, 127, 120, 79, 105, 115, 7, 9, 59, 64, 55, 63, 49, 107, 46, 113, 4, 56, 72, 57, 15, 67, 52, 75, 47, 0, 60, 70, 58, 112, 125, 8, 3, 1, 68, 65], [117, 51, 36, 100, 61, 97, 102, 91, 115, 88, 30, 18, 94, 96, 25, 44, 22, 80, 27, 93, 11, 62, 20, 33, 28, 54, 53, 78, 19, 124, 63, 81, 103, 108, 76, 119, 6, 120, 111, 42, 104, 21, 59, 3, 85, 10, 4, 32, 110, 16, 127, 82, 9, 114, 69, 15, 83, 71, 123, 48, 38, 112, 12, 65, 52, 73, 118, 26, 43, 107, 122, 35, 58, 50, 126, 56, 0, 46, 121, 40, 116, 109, 60, 14, 77, 37, 67, 57, 55, 125, 41, 47, 92, 66, 39, 98, 1, 84, 106, 45, 105, 113, 24, 75, 72, 90, 49, 86, 23, 87, 95, 79, 34, 13, 99, 89, 101, 29, 68, 2, 5, 8, 70, 31, 17, 74, 7, 64], [61, 117, 51, 36, 88, 100, 30, 108, 97, 44, 85, 115, 124, 102, 62, 27, 104, 34, 54, 120, 111, 63, 59, 94, 114, 119, 91, 127, 33, 46, 52, 123, 112, 103, 122, 126, 109, 121, 50, 78, 58, 42, 43, 56, 107, 110, 116, 80, 19, 23, 57, 55, 98, 31, 22, 25, 48, 60, 125, 113, 93, 53, 105, 118, 45, 35, 47, 21, 49, 101, 96, 106, 41, 81, 37, 40, 39, 38, 32, 99, 24, 26, 69, 10, 29, 28, 5, 90, 74, 95, 2, 83, 71, 17, 86, 92, 65, 89, 73, 7, 84, 14, 76, 3, 20, 82, 87, 72, 77, 16, 11, 0, 9, 75, 66, 64, 8, 18, 67, 15, 6, 79, 70, 1, 13, 68, 4, 12], [39, 47, 115, 34, 25, 71, 12, 10, 83, 17, 14, 4, 86, 117, 111, 79, 123, 87, 22, 18, 63, 92, 7, 89, 66, 64, 56, 9, 98, 2, 21, 5, 1, 78, 91, 24, 67, 94, 76, 62, 110, 30, 19, 68, 102, 65, 6, 15, 69, 13, 27, 51, 73, 85, 80, 57, 99, 8, 70, 88, 29, 74, 23, 84, 77, 101, 55, 96, 16, 38, 40, 116, 72, 11, 50, 26, 75, 81, 31, 93, 112, 60, 90, 3, 43, 0, 48, 82, 28, 42, 106, 95, 118, 108, 20, 37, 53, 45, 41, 36, 114, 97, 113, 124, 61, 52, 49, 103, 107, 126, 127, 109, 121, 32, 35, 46, 100, 33, 44, 105, 54, 120, 58, 125, 122, 59, 119, 104], [47, 39, 115, 34, 114, 27, 37, 43, 111, 116, 25, 117, 94, 112, 56, 83, 123, 63, 51, 79, 86, 124, 55, 30, 126, 91, 89, 16, 62, 110, 58, 60, 127, 17, 109, 15, 99, 54, 46, 118, 122, 40, 107, 57, 98, 53, 52, 59, 48, 113, 125, 29, 50, 104, 44, 10, 14, 24, 41, 108, 97, 102, 61, 23, 95, 38, 121, 88, 42, 101, 106, 120, 12, 96, 80, 21, 36, 49, 35, 119, 105, 28, 85, 22, 45, 90, 19, 33, 82, 100, 26, 93, 92, 9, 31, 32, 18, 20, 81, 8, 84, 87, 77, 75, 74, 78, 103, 13, 2, 4, 73, 11, 72, 68, 70, 76, 71, 6, 69, 67, 5, 66, 3, 0, 7, 64, 65, 1], [115, 47, 39, 43, 34, 27, 25, 60, 51, 117, 37, 112, 56, 83, 114, 116, 86, 123, 55, 63, 30, 94, 10, 127, 58, 24, 15, 16, 79, 110, 126, 89, 62, 111, 17, 118, 48, 61, 124, 57, 12, 14, 72, 122, 21, 98, 107, 54, 53, 52, 113, 125, 99, 109, 105, 120, 44, 102, 91, 41, 119, 121, 66, 38, 49, 101, 104, 46, 59, 95, 50, 4, 106, 36, 100, 68, 22, 108, 45, 96, 32, 13, 93, 19, 75, 28, 42, 40, 80, 31, 97, 26, 85, 84, 71, 18, 23, 29, 35, 33, 8, 82, 9, 103, 74, 5, 87, 92, 90, 67, 88, 78, 2, 81, 20, 70, 77, 76, 3, 6, 11, 73, 0, 64, 69, 65, 7, 1], [115, 39, 47, 114, 34, 27, 116, 55, 117, 25, 56, 123, 63, 24, 37, 112, 94, 41, 83, 58, 44, 126, 91, 89, 86, 124, 110, 62, 51, 30, 45, 60, 52, 53, 113, 15, 127, 54, 84, 118, 61, 43, 119, 111, 46, 121, 108, 125, 59, 101, 109, 107, 122, 106, 57, 42, 49, 102, 120, 104, 48, 50, 79, 29, 96, 22, 99, 36, 80, 10, 38, 105, 92, 85, 97, 20, 100, 21, 26, 17, 93, 40, 98, 16, 12, 14, 31, 33, 28, 19, 35, 103, 70, 8, 32, 95, 23, 75, 18, 88, 87, 81, 9, 82, 90, 68, 72, 13, 11, 78, 66, 74, 2, 4, 6, 77, 5, 3, 71, 67, 0, 76, 64, 69, 73, 7, 1, 65], [53, 120, 101, 127, 110, 38, 121, 25, 125, 56, 51, 100, 102, 28, 22, 40, 80, 30, 33, 97, 116, 29, 49, 104, 11, 114, 59, 113, 87, 118, 50, 32, 84, 86, 63, 35, 57, 55, 77, 61, 75, 60, 16, 119, 115, 24, 90, 79, 46, 39, 54, 124, 93, 112, 37, 98, 117, 7, 52, 96, 62, 99, 9, 48, 94, 106, 92, 108, 18, 23, 105, 45, 111, 27, 109, 34, 123, 126, 122, 103, 107, 89, 72, 58, 47, 69, 91, 83, 15, 88, 44, 26, 1, 78, 20, 42, 36, 71, 21, 31, 43, 13, 95, 41, 81, 5, 85, 19, 73, 66, 4, 14, 82, 10, 8, 17, 2, 12, 76, 67, 70, 74, 3, 68, 0, 6, 65, 64], [120, 127, 53, 51, 113, 110, 59, 104, 49, 50, 125, 56, 112, 124, 116, 114, 63, 121, 101, 44, 52, 62, 55, 61, 57, 117, 60, 118, 119, 126, 46, 115, 58, 111, 37, 123, 54, 45, 48, 122, 39, 43, 47, 27, 107, 105, 108, 109, 42, 40, 106, 38, 79, 41, 96, 103, 100, 29, 35, 33, 82, 19, 85, 87, 78, 98, 34, 99, 102, 36, 30, 88, 95, 97, 6, 31, 64, 86, 24, 66, 65, 77, 92, 91, 32, 10, 94, 9, 8, 83, 22, 75, 67, 84, 5, 20, 17, 93, 28, 4, 0, 71, 18, 90, 14, 80, 11, 25, 21, 12, 15, 68, 26, 74, 1, 81, 7, 2, 23, 69, 72, 70, 3, 76, 73, 13, 89, 16], [127, 120, 110, 53, 40, 101, 100, 121, 49, 38, 51, 25, 118, 97, 63, 56, 62, 28, 113, 117, 84, 61, 39, 125, 119, 30, 32, 124, 59, 29, 96, 52, 104, 115, 88, 105, 112, 18, 94, 50, 116, 19, 23, 99, 111, 0, 24, 87, 60, 33, 89, 37, 98, 70, 35, 108, 95, 55, 103, 48, 41, 31, 21, 122, 109, 114, 57, 126, 43, 54, 102, 14, 22, 76, 86, 44, 46, 27, 80, 47, 78, 90, 34, 10, 42, 45, 6, 93, 66, 107, 85, 17, 36, 13, 65, 12, 83, 123, 75, 26, 106, 58, 81, 3, 20, 82, 11, 74, 79, 68, 1, 91, 15, 67, 69, 72, 8, 77, 92, 9, 71, 64, 2, 5, 73, 16, 4, 7], [127, 101, 53, 120, 40, 59, 28, 84, 33, 97, 100, 30, 25, 37, 87, 51, 50, 113, 86, 63, 77, 46, 104, 112, 102, 88, 96, 29, 27, 20, 116, 35, 121, 38, 22, 83, 126, 91, 125, 56, 94, 11, 80, 98, 75, 45, 19, 34, 78, 17, 52, 103, 60, 55, 54, 62, 24, 122, 110, 61, 115, 32, 79, 81, 41, 114, 15, 93, 71, 107, 124, 48, 21, 58, 105, 99, 8, 31, 39, 7, 106, 12, 44, 10, 57, 118, 42, 76, 95, 123, 109, 111, 47, 82, 108, 119, 92, 43, 4, 26, 89, 2, 18, 49, 74, 85, 90, 36, 5, 68, 117, 72, 16, 1, 9, 14, 23, 69, 70, 66, 0, 65, 3, 6, 13, 67, 64, 73], [104, 117, 25, 95, 34, 92, 86, 52, 19, 54, 77, 97, 8, 53, 61, 51, 79, 22, 28, 17, 120, 80, 68, 59, 88, 85, 114, 45, 72, 12, 60, 121, 24, 124, 13, 40, 49, 116, 4, 122, 113, 56, 87, 55, 58, 57, 23, 73, 107, 91, 109, 46, 15, 110, 63, 127, 62, 89, 93, 65, 74, 118, 83, 48, 115, 47, 21, 31, 18, 69, 75, 90, 0, 33, 126, 100, 35, 39, 111, 125, 94, 2, 14, 43, 106, 82, 20, 112, 6, 42, 36, 3, 50, 32, 64, 44, 5, 101, 123, 105, 98, 119, 102, 76, 29, 99, 108, 84, 27, 26, 30, 9, 16, 96, 66, 103, 81, 11, 1, 38, 71, 78, 7, 70, 41, 37, 10, 67], [117, 104, 120, 25, 95, 54, 52, 34, 61, 92, 103, 86, 88, 97, 58, 51, 116, 12, 114, 124, 55, 122, 53, 60, 113, 102, 49, 121, 63, 47, 59, 85, 127, 46, 62, 106, 111, 109, 48, 56, 28, 18, 126, 118, 19, 44, 115, 8, 119, 69, 125, 45, 14, 112, 57, 91, 24, 123, 80, 43, 42, 110, 108, 50, 107, 17, 66, 20, 87, 98, 105, 22, 39, 36, 71, 33, 100, 41, 38, 32, 37, 79, 26, 35, 77, 99, 94, 93, 101, 75, 74, 40, 30, 84, 31, 82, 13, 29, 96, 16, 72, 3, 89, 78, 11, 27, 90, 23, 9, 4, 15, 65, 81, 7, 68, 21, 83, 5, 76, 2, 1, 0, 73, 6, 67, 64, 10, 70], [117, 104, 95, 25, 52, 54, 61, 34, 92, 56, 97, 86, 120, 60, 24, 12, 47, 17, 51, 116, 53, 113, 88, 20, 122, 49, 99, 124, 85, 103, 119, 80, 19, 59, 102, 77, 28, 48, 91, 63, 57, 31, 112, 106, 126, 26, 58, 18, 62, 109, 118, 111, 115, 114, 121, 127, 125, 107, 100, 8, 123, 110, 44, 45, 50, 69, 42, 55, 43, 22, 105, 36, 38, 46, 16, 37, 15, 108, 96, 78, 30, 87, 39, 75, 94, 79, 32, 33, 35, 41, 40, 82, 90, 14, 98, 84, 29, 27, 93, 83, 89, 101, 81, 21, 23, 9, 73, 76, 74, 72, 10, 11, 66, 3, 4, 7, 5, 2, 6, 13, 71, 1, 70, 0, 65, 68, 64, 67], [117, 104, 52, 25, 95, 61, 56, 86, 92, 97, 34, 54, 60, 51, 19, 53, 58, 77, 120, 17, 8, 80, 113, 114, 31, 122, 59, 66, 62, 124, 127, 116, 69, 22, 12, 49, 48, 46, 57, 88, 47, 121, 106, 55, 85, 112, 79, 115, 72, 28, 102, 14, 74, 109, 45, 63, 15, 10, 87, 110, 125, 36, 40, 111, 118, 24, 4, 43, 126, 35, 73, 94, 33, 119, 123, 27, 42, 93, 98, 30, 5, 103, 50, 100, 91, 21, 44, 3, 108, 82, 29, 32, 105, 37, 107, 26, 41, 99, 90, 78, 38, 101, 83, 20, 39, 6, 18, 96, 75, 89, 23, 11, 16, 76, 84, 71, 64, 13, 70, 81, 7, 2, 68, 9, 1, 65, 0, 67], [39, 121, 112, 33, 1, 114, 93, 23, 69, 82, 0, 21, 79, 20, 13, 14, 84, 67, 48, 2, 118, 10, 25, 90, 16, 91, 22, 5, 29, 81, 72, 17, 65, 89, 78, 124, 3, 75, 71, 66, 68, 61, 87, 80, 88, 7, 30, 56, 47, 52, 4, 6, 60, 19, 55, 120, 12, 11, 110, 119, 122, 125, 63, 116, 115, 57, 53, 42, 126, 73, 92, 49, 83, 76, 86, 9, 70, 34, 100, 64, 102, 107, 27, 59, 74, 108, 101, 98, 43, 127, 51, 111, 99, 106, 35, 85, 44, 28, 50, 8, 45, 113, 94, 24, 117, 18, 109, 58, 26, 40, 37, 95, 96, 54, 38, 41, 62, 77, 46, 36, 123, 104, 105, 32, 31, 15, 97, 103], [112, 39, 114, 93, 33, 121, 23, 20, 90, 84, 17, 110, 56, 10, 53, 61, 115, 49, 22, 91, 119, 101, 116, 108, 117, 113, 43, 50, 47, 29, 24, 55, 125, 51, 118, 16, 99, 40, 54, 82, 63, 81, 57, 59, 60, 100, 46, 42, 38, 52, 58, 124, 111, 79, 126, 45, 123, 120, 109, 6, 87, 48, 107, 13, 122, 105, 106, 62, 102, 88, 44, 3, 104, 41, 127, 95, 94, 34, 37, 98, 85, 36, 72, 83, 92, 97, 30, 27, 86, 80, 35, 32, 25, 26, 96, 11, 74, 28, 19, 103, 31, 89, 76, 21, 5, 70, 18, 15, 77, 12, 64, 71, 14, 73, 67, 78, 68, 9, 8, 7, 75, 0, 2, 4, 1, 65, 69, 66], [112, 39, 121, 114, 93, 33, 23, 61, 110, 48, 17, 120, 20, 84, 90, 47, 22, 56, 115, 91, 124, 117, 53, 45, 82, 60, 116, 49, 51, 118, 57, 119, 55, 10, 50, 29, 94, 38, 43, 24, 125, 111, 52, 30, 108, 63, 79, 126, 62, 99, 88, 59, 113, 54, 122, 109, 107, 102, 123, 44, 58, 46, 40, 104, 100, 85, 41, 6, 101, 42, 127, 106, 81, 103, 105, 13, 86, 98, 87, 25, 92, 97, 21, 74, 31, 35, 95, 76, 36, 37, 96, 26, 34, 32, 27, 15, 77, 28, 11, 8, 16, 18, 89, 3, 80, 67, 72, 83, 70, 19, 5, 12, 71, 75, 68, 78, 7, 64, 2, 73, 14, 0, 9, 1, 4, 69, 65, 66], [39, 112, 33, 93, 121, 114, 23, 120, 20, 10, 22, 17, 79, 84, 110, 96, 13, 61, 29, 72, 49, 87, 90, 21, 56, 82, 109, 6, 117, 60, 70, 47, 125, 16, 52, 26, 81, 50, 118, 92, 77, 119, 51, 57, 115, 42, 25, 53, 11, 30, 124, 34, 104, 107, 91, 116, 122, 3, 95, 98, 38, 99, 83, 100, 86, 43, 111, 41, 101, 126, 48, 74, 5, 37, 55, 88, 62, 94, 44, 27, 24, 63, 108, 113, 67, 40, 35, 59, 45, 54, 78, 123, 32, 89, 127, 14, 28, 58, 105, 46, 18, 15, 85, 106, 9, 36, 31, 12, 19, 8, 64, 80, 75, 73, 102, 4, 76, 7, 68, 97, 71, 2, 0, 1, 103, 69, 65, 66], [53, 60, 127, 124, 122, 119, 123, 120, 51, 116, 56, 55, 125, 57, 38, 126, 54, 115, 61, 63, 50, 52, 118, 49, 58, 36, 59, 114, 62, 48, 45, 121, 112, 111, 101, 39, 113, 105, 110, 47, 46, 117, 106, 42, 108, 109, 43, 104, 44, 34, 107, 40, 99, 103, 41, 91, 24, 102, 94, 95, 19, 27, 21, 97, 93, 35, 37, 31, 89, 98, 100, 29, 16, 96, 14, 23, 92, 33, 25, 90, 83, 13, 32, 85, 30, 87, 11, 88, 20, 82, 4, 80, 8, 78, 68, 72, 28, 26, 75, 77, 70, 86, 6, 17, 81, 67, 3, 79, 74, 84, 18, 15, 10, 73, 76, 9, 22, 65, 71, 1, 12, 5, 7, 64, 66, 2, 0, 69], [53, 60, 120, 124, 39, 62, 119, 36, 127, 123, 125, 56, 100, 122, 46, 104, 109, 31, 51, 37, 101, 108, 61, 110, 126, 121, 118, 117, 63, 33, 42, 115, 113, 57, 52, 54, 44, 116, 107, 106, 41, 58, 23, 43, 59, 45, 24, 50, 48, 112, 55, 47, 97, 114, 40, 95, 111, 49, 102, 105, 90, 38, 103, 92, 94, 21, 82, 32, 98, 29, 34, 86, 25, 35, 99, 85, 91, 30, 93, 14, 13, 96, 16, 20, 19, 89, 87, 88, 27, 26, 28, 22, 18, 11, 83, 78, 80, 72, 15, 84, 68, 17, 77, 81, 79, 4, 75, 8, 6, 70, 73, 10, 12, 74, 67, 9, 3, 76, 71, 7, 5, 1, 69, 2, 66, 65, 0, 64], [60, 53, 127, 56, 124, 119, 122, 51, 123, 125, 55, 52, 116, 61, 57, 114, 126, 58, 63, 54, 48, 115, 121, 59, 50, 113, 49, 118, 112, 106, 111, 62, 120, 47, 117, 38, 46, 110, 45, 109, 108, 44, 107, 43, 42, 41, 105, 40, 39, 36, 103, 100, 104, 102, 37, 97, 29, 22, 98, 101, 33, 34, 35, 99, 86, 95, 32, 90, 23, 31, 85, 96, 91, 83, 25, 87, 18, 80, 30, 94, 92, 84, 24, 75, 27, 93, 26, 89, 17, 82, 16, 21, 19, 13, 78, 81, 79, 14, 20, 68, 8, 28, 88, 76, 15, 72, 4, 11, 74, 9, 77, 6, 3, 7, 2, 70, 1, 10, 67, 12, 64, 69, 65, 0, 73, 5, 66, 71], [60, 53, 120, 20, 79, 76, 92, 36, 9, 124, 7, 82, 17, 24, 97, 2, 62, 86, 69, 71, 0, 66, 32, 125, 56, 5, 122, 100, 74, 14, 73, 12, 3, 52, 119, 34, 25, 33, 18, 11, 80, 38, 65, 94, 21, 89, 48, 28, 13, 67, 78, 15, 81, 123, 8, 84, 127, 88, 96, 19, 93, 10, 75, 99, 83, 31, 101, 16, 26, 95, 108, 23, 6, 90, 72, 77, 70, 1, 85, 87, 64, 68, 27, 43, 4, 41, 30, 110, 91, 55, 29, 44, 35, 22, 109, 59, 102, 117, 58, 112, 39, 126, 98, 104, 37, 107, 61, 103, 51, 49, 113, 47, 118, 42, 115, 105, 63, 106, 116, 45, 54, 114, 40, 121, 46, 57, 111, 50]], "model.layers.15.self_attn.k_proj": [[113, 39, 62, 22, 18, 11, 60, 14, 89, 16, 97, 8, 21, 19, 57, 59, 123, 56, 110, 92, 67, 58, 120, 70, 116, 63, 53, 127, 119, 108, 79, 10, 122, 126, 117, 61, 54, 55, 48, 109, 114, 121, 87, 51, 124, 91, 112, 107, 52, 111, 125, 101, 42, 50, 118, 30, 44, 106, 47, 36, 115, 13, 46, 76, 0, 45, 43, 41, 104, 105, 75, 37, 35, 49, 69, 38, 29, 26, 7, 9, 27, 102, 40, 71, 85, 83, 31, 100, 17, 68, 95, 96, 99, 72, 33, 2, 34, 88, 15, 73, 93, 32, 78, 80, 82, 20, 5, 90, 25, 98, 77, 23, 28, 84, 12, 65, 24, 81, 94, 66, 1, 74, 6, 86, 3, 64, 103, 4], [103, 109, 112, 33, 45, 89, 21, 118, 12, 19, 79, 17, 9, 122, 119, 14, 31, 51, 7, 125, 5, 0, 117, 3, 120, 116, 124, 55, 115, 53, 93, 27, 57, 127, 50, 123, 47, 60, 20, 114, 56, 1, 49, 40, 59, 94, 46, 61, 54, 43, 8, 58, 66, 62, 23, 63, 111, 18, 108, 25, 28, 24, 52, 35, 121, 110, 42, 48, 10, 32, 86, 30, 126, 68, 11, 70, 22, 4, 44, 87, 106, 91, 113, 105, 74, 96, 107, 104, 82, 41, 98, 100, 83, 38, 34, 99, 77, 92, 37, 102, 88, 6, 26, 81, 101, 95, 13, 36, 84, 29, 73, 65, 2, 16, 76, 78, 90, 69, 71, 39, 15, 80, 85, 75, 64, 72, 97, 67], [115, 51, 117, 61, 100, 22, 30, 33, 53, 27, 108, 19, 88, 78, 124, 81, 62, 10, 120, 54, 80, 59, 32, 114, 20, 119, 127, 118, 63, 111, 64, 44, 71, 106, 110, 4, 123, 116, 48, 112, 58, 66, 56, 125, 122, 52, 126, 55, 76, 60, 50, 109, 1, 25, 43, 39, 121, 107, 46, 17, 102, 113, 11, 38, 13, 104, 57, 47, 69, 85, 49, 24, 40, 45, 99, 15, 41, 84, 82, 36, 29, 77, 35, 92, 72, 90, 42, 105, 37, 73, 93, 28, 98, 67, 8, 86, 101, 103, 87, 89, 95, 96, 6, 26, 65, 34, 31, 94, 97, 75, 16, 18, 21, 3, 23, 70, 9, 79, 74, 91, 12, 83, 7, 0, 5, 68, 2, 14], [103, 115, 47, 111, 86, 98, 51, 25, 17, 12, 117, 83, 14, 27, 10, 30, 110, 71, 116, 56, 1, 79, 112, 126, 123, 4, 55, 63, 62, 6, 64, 9, 58, 114, 66, 127, 121, 52, 61, 113, 122, 69, 59, 43, 124, 48, 109, 54, 106, 101, 32, 125, 21, 45, 60, 41, 53, 75, 118, 34, 108, 5, 120, 2, 49, 50, 46, 13, 57, 88, 87, 105, 67, 119, 102, 29, 92, 85, 72, 40, 96, 36, 107, 44, 104, 31, 28, 37, 24, 8, 20, 82, 81, 76, 26, 100, 42, 95, 38, 99, 91, 70, 90, 18, 97, 23, 35, 15, 0, 33, 93, 78, 94, 16, 19, 84, 77, 73, 80, 3, 7, 68, 89, 11, 22, 65, 74, 39], [127, 37, 120, 53, 104, 22, 97, 102, 32, 56, 46, 110, 94, 61, 121, 125, 25, 51, 60, 28, 93, 35, 80, 89, 113, 92, 55, 116, 118, 54, 122, 119, 124, 63, 108, 117, 52, 20, 96, 57, 99, 126, 59, 112, 47, 45, 48, 115, 91, 58, 114, 41, 30, 111, 106, 50, 98, 77, 123, 33, 44, 9, 109, 49, 88, 87, 62, 107, 101, 42, 19, 39, 43, 18, 95, 4, 36, 103, 66, 86, 105, 84, 31, 65, 40, 71, 75, 17, 64, 34, 78, 12, 90, 8, 79, 10, 29, 15, 0, 85, 82, 21, 6, 16, 24, 27, 26, 13, 67, 3, 72, 69, 5, 11, 100, 23, 14, 83, 38, 81, 76, 7, 68, 73, 74, 1, 70, 2], [117, 40, 31, 98, 86, 28, 52, 61, 25, 62, 120, 60, 114, 48, 56, 122, 113, 116, 51, 124, 59, 49, 127, 115, 109, 57, 46, 125, 55, 47, 54, 58, 33, 19, 123, 121, 85, 110, 111, 12, 126, 112, 18, 119, 80, 45, 63, 118, 43, 106, 53, 24, 8, 79, 50, 108, 42, 35, 44, 39, 105, 15, 41, 36, 107, 2, 64, 87, 102, 38, 34, 101, 29, 81, 68, 100, 16, 92, 37, 77, 17, 23, 99, 74, 27, 5, 103, 104, 20, 32, 88, 96, 94, 75, 21, 91, 89, 65, 1, 30, 97, 13, 9, 67, 93, 10, 14, 11, 71, 90, 6, 84, 82, 26, 83, 78, 95, 70, 76, 73, 7, 22, 66, 72, 69, 0, 3, 4], [103, 121, 112, 97, 48, 29, 114, 22, 57, 120, 23, 84, 118, 50, 119, 61, 116, 17, 52, 56, 111, 115, 90, 49, 53, 79, 60, 10, 43, 42, 82, 64, 3, 59, 122, 47, 46, 55, 58, 72, 124, 125, 36, 123, 102, 62, 104, 117, 63, 126, 45, 13, 110, 51, 37, 106, 91, 113, 28, 127, 41, 44, 6, 38, 54, 109, 21, 68, 107, 24, 2, 25, 76, 93, 73, 98, 105, 32, 108, 89, 78, 86, 71, 101, 40, 99, 83, 35, 80, 33, 16, 26, 34, 15, 31, 11, 27, 85, 94, 77, 95, 19, 92, 96, 18, 100, 75, 14, 12, 30, 69, 39, 7, 65, 5, 88, 9, 1, 8, 20, 87, 66, 4, 0, 70, 81, 67, 74], [60, 53, 86, 100, 120, 94, 76, 125, 79, 7, 124, 20, 101, 9, 82, 92, 74, 69, 96, 17, 25, 122, 44, 62, 24, 61, 119, 52, 118, 2, 106, 56, 127, 123, 117, 57, 51, 63, 121, 19, 58, 112, 126, 48, 0, 109, 107, 54, 114, 55, 59, 16, 113, 49, 47, 46, 50, 14, 90, 1, 115, 116, 111, 104, 72, 110, 41, 34, 13, 45, 5, 6, 43, 108, 81, 97, 75, 39, 42, 3, 10, 27, 37, 40, 11, 105, 102, 103, 35, 98, 12, 15, 23, 99, 33, 21, 22, 31, 30, 71, 78, 38, 73, 26, 93, 29, 85, 28, 95, 32, 8, 84, 67, 91, 70, 77, 18, 87, 83, 65, 89, 4, 68, 66, 64, 88, 36, 80]], "model.layers.15.self_attn.qk_proj": [[117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 48, 86, 39, 100, 57, 56, 125, 118, 25, 116, 103, 89, 122, 22, 110, 111, 91, 109, 46, 97, 40, 63, 59, 50, 81, 30, 54, 119, 33, 123, 104, 93, 19, 49, 108, 55, 52, 83, 17, 58, 21, 79, 85, 126, 14, 101, 94, 36, 92, 98, 15, 12, 44, 107, 76, 78, 88, 20, 28, 106, 24, 37, 43, 84, 34, 74, 42, 31, 102, 95, 87, 27, 10, 9, 29, 32, 23, 7, 71, 96, 41, 82, 80, 16, 18, 5, 73, 38, 35, 105, 72, 26, 11, 69, 77, 75, 67, 3, 64, 0, 13, 66, 99, 8, 90, 2, 68, 70, 4, 1, 6, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 48, 39, 86, 124, 56, 103, 100, 25, 118, 125, 22, 89, 46, 109, 116, 122, 54, 63, 111, 119, 91, 97, 110, 50, 40, 55, 49, 59, 52, 30, 108, 81, 104, 33, 123, 93, 126, 101, 83, 94, 19, 17, 15, 36, 58, 21, 85, 107, 98, 14, 12, 78, 92, 79, 76, 88, 34, 24, 28, 37, 10, 20, 106, 43, 44, 102, 74, 27, 84, 7, 31, 9, 42, 95, 23, 87, 71, 18, 82, 41, 96, 80, 16, 29, 32, 26, 35, 38, 99, 72, 0, 8, 69, 73, 5, 105, 2, 66, 75, 77, 11, 64, 67, 3, 13, 90, 4, 65, 6, 70, 68, 1], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 48, 124, 57, 118, 86, 100, 89, 103, 122, 111, 46, 25, 116, 109, 59, 125, 22, 91, 123, 97, 104, 110, 63, 40, 54, 119, 33, 30, 108, 52, 55, 93, 36, 81, 101, 50, 94, 58, 49, 19, 83, 126, 17, 21, 15, 28, 37, 85, 12, 24, 98, 92, 107, 44, 88, 76, 78, 34, 14, 102, 43, 79, 27, 95, 42, 106, 84, 9, 10, 74, 31, 20, 87, 23, 29, 7, 71, 18, 41, 32, 38, 64, 26, 82, 16, 96, 35, 2, 5, 0, 8, 80, 66, 99, 73, 69, 105, 67, 90, 11, 75, 13, 3, 72, 77, 6, 68, 65, 1, 4, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 47, 127, 121, 45, 124, 114, 48, 39, 57, 56, 118, 100, 125, 116, 89, 103, 86, 109, 111, 25, 22, 122, 97, 91, 110, 54, 63, 59, 40, 30, 55, 81, 119, 46, 104, 58, 123, 52, 50, 108, 33, 126, 93, 101, 49, 19, 94, 21, 17, 37, 107, 79, 98, 14, 85, 12, 83, 76, 28, 88, 36, 92, 24, 78, 34, 15, 102, 44, 42, 20, 95, 27, 43, 16, 106, 84, 23, 80, 96, 9, 29, 82, 74, 87, 31, 10, 38, 18, 7, 32, 0, 71, 5, 41, 69, 35, 26, 73, 99, 66, 8, 11, 67, 3, 64, 90, 77, 2, 105, 1, 75, 72, 65, 13, 6, 4, 68, 70], [117, 112, 53, 115, 60, 62, 113, 51, 120, 61, 127, 47, 121, 45, 124, 114, 118, 48, 57, 56, 39, 125, 86, 100, 103, 116, 122, 59, 63, 22, 89, 110, 25, 109, 55, 97, 111, 54, 40, 91, 46, 119, 104, 30, 108, 123, 81, 33, 52, 49, 58, 50, 17, 19, 83, 93, 101, 12, 85, 15, 14, 98, 37, 43, 78, 79, 126, 94, 76, 92, 21, 107, 24, 42, 28, 74, 106, 44, 88, 84, 34, 102, 20, 36, 87, 10, 38, 96, 27, 9, 8, 7, 95, 32, 71, 41, 69, 35, 80, 23, 82, 18, 16, 5, 31, 105, 64, 29, 73, 26, 3, 66, 2, 75, 67, 77, 11, 13, 0, 99, 72, 1, 65, 90, 4, 68, 6, 70], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 118, 124, 57, 56, 86, 39, 48, 89, 100, 25, 116, 22, 125, 63, 111, 110, 103, 59, 91, 97, 54, 30, 109, 52, 40, 122, 119, 50, 81, 55, 49, 108, 46, 33, 123, 104, 93, 17, 58, 85, 83, 14, 19, 126, 76, 21, 12, 101, 15, 79, 94, 107, 92, 78, 44, 28, 88, 98, 34, 36, 42, 84, 24, 43, 37, 20, 10, 95, 87, 106, 9, 23, 18, 82, 16, 31, 80, 74, 7, 102, 27, 96, 38, 71, 32, 41, 8, 29, 35, 13, 75, 105, 73, 5, 69, 77, 67, 11, 99, 26, 64, 2, 66, 90, 0, 3, 68, 72, 6, 4, 65, 1, 70], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 118, 114, 57, 48, 124, 86, 100, 89, 39, 22, 56, 25, 59, 125, 103, 63, 111, 110, 91, 97, 122, 54, 116, 30, 81, 40, 123, 55, 119, 52, 109, 93, 46, 19, 126, 104, 50, 83, 108, 33, 101, 21, 49, 17, 92, 85, 76, 14, 15, 12, 58, 78, 79, 107, 94, 44, 28, 98, 84, 34, 88, 36, 24, 42, 106, 27, 37, 20, 87, 18, 10, 95, 23, 74, 96, 9, 38, 102, 16, 82, 105, 32, 31, 43, 80, 41, 71, 7, 29, 8, 13, 35, 73, 69, 11, 75, 26, 5, 77, 3, 0, 2, 99, 67, 66, 64, 72, 90, 68, 1, 6, 70, 4, 65], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 57, 118, 48, 39, 89, 56, 86, 103, 100, 25, 22, 124, 122, 125, 59, 97, 116, 91, 63, 111, 46, 30, 54, 110, 119, 109, 55, 40, 123, 104, 33, 83, 19, 108, 81, 49, 93, 50, 52, 101, 126, 17, 21, 92, 15, 36, 12, 14, 79, 28, 58, 85, 98, 78, 94, 76, 107, 24, 44, 42, 87, 88, 20, 84, 37, 106, 34, 43, 27, 96, 102, 23, 38, 31, 74, 18, 10, 41, 16, 9, 82, 95, 29, 32, 7, 105, 71, 80, 26, 35, 8, 0, 73, 11, 75, 5, 13, 66, 69, 99, 2, 64, 67, 3, 77, 72, 65, 70, 90, 1, 68, 4, 6], [117, 112, 53, 60, 115, 113, 51, 62, 120, 127, 61, 121, 47, 45, 57, 48, 118, 114, 56, 124, 39, 86, 89, 100, 125, 25, 103, 22, 122, 97, 63, 109, 30, 116, 111, 91, 55, 110, 123, 50, 54, 46, 59, 104, 119, 40, 101, 33, 49, 81, 19, 83, 93, 108, 98, 17, 58, 52, 21, 126, 36, 14, 107, 12, 79, 76, 94, 92, 85, 15, 24, 78, 42, 37, 88, 44, 28, 34, 20, 95, 106, 102, 27, 43, 16, 84, 31, 41, 87, 18, 32, 23, 10, 29, 74, 96, 80, 71, 9, 38, 82, 105, 35, 7, 73, 75, 99, 8, 13, 77, 11, 5, 0, 69, 26, 2, 72, 64, 67, 1, 66, 3, 90, 70, 65, 4, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 61, 127, 120, 47, 121, 45, 114, 48, 124, 39, 57, 56, 118, 89, 125, 86, 100, 103, 22, 122, 25, 111, 123, 97, 55, 110, 109, 63, 91, 116, 54, 52, 59, 30, 119, 58, 46, 81, 83, 40, 104, 49, 50, 108, 126, 33, 93, 17, 19, 101, 14, 15, 94, 98, 85, 21, 12, 79, 107, 76, 44, 102, 28, 92, 36, 34, 78, 74, 24, 37, 42, 20, 43, 10, 84, 27, 88, 71, 7, 87, 23, 106, 41, 95, 18, 38, 9, 31, 16, 73, 32, 80, 96, 29, 5, 69, 82, 105, 0, 8, 35, 64, 3, 99, 66, 2, 67, 72, 13, 77, 26, 75, 11, 90, 70, 4, 68, 65, 1, 6], [117, 112, 53, 60, 115, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 124, 39, 48, 118, 57, 56, 89, 86, 100, 122, 22, 25, 116, 125, 103, 110, 119, 97, 109, 46, 30, 59, 111, 54, 91, 123, 63, 40, 81, 55, 104, 52, 108, 33, 101, 93, 83, 50, 58, 49, 12, 19, 17, 14, 76, 79, 94, 21, 126, 15, 36, 78, 88, 85, 24, 28, 98, 107, 92, 10, 43, 44, 84, 34, 7, 102, 20, 87, 37, 71, 74, 18, 9, 106, 80, 5, 16, 23, 42, 29, 32, 38, 41, 95, 69, 73, 72, 75, 82, 64, 96, 27, 66, 0, 31, 35, 8, 2, 67, 105, 3, 13, 77, 11, 99, 1, 65, 90, 68, 26, 6, 4, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 39, 124, 118, 48, 86, 89, 57, 25, 22, 110, 100, 116, 125, 103, 109, 56, 63, 122, 55, 54, 59, 97, 91, 46, 30, 104, 40, 119, 111, 123, 81, 50, 52, 19, 49, 93, 108, 17, 101, 21, 12, 33, 83, 76, 15, 14, 126, 79, 98, 58, 92, 44, 94, 43, 78, 107, 28, 85, 88, 20, 87, 36, 106, 34, 24, 84, 42, 10, 18, 102, 9, 37, 74, 95, 7, 23, 16, 82, 41, 71, 27, 73, 32, 75, 80, 31, 96, 29, 35, 72, 38, 5, 69, 77, 0, 13, 11, 3, 105, 8, 99, 64, 2, 1, 26, 66, 67, 70, 6, 4, 68, 90, 65], [117, 112, 53, 115, 60, 62, 51, 113, 120, 61, 127, 47, 121, 45, 114, 118, 57, 48, 39, 56, 86, 124, 89, 100, 22, 25, 103, 111, 125, 55, 97, 63, 109, 59, 91, 54, 116, 110, 119, 122, 30, 93, 46, 40, 104, 81, 52, 108, 123, 126, 33, 36, 101, 50, 83, 19, 49, 94, 107, 76, 98, 21, 79, 17, 92, 15, 58, 14, 85, 28, 44, 20, 88, 12, 37, 42, 43, 78, 34, 87, 24, 27, 41, 106, 95, 84, 74, 102, 31, 96, 18, 23, 82, 35, 29, 38, 10, 105, 80, 7, 32, 73, 16, 72, 71, 99, 26, 5, 75, 9, 13, 11, 67, 77, 90, 69, 3, 66, 1, 2, 8, 0, 68, 64, 6, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 127, 61, 47, 120, 121, 45, 48, 114, 39, 118, 57, 56, 124, 86, 125, 89, 116, 100, 22, 54, 103, 25, 122, 55, 109, 111, 91, 119, 123, 59, 63, 40, 97, 30, 46, 104, 110, 93, 81, 33, 50, 108, 52, 94, 49, 19, 58, 83, 76, 15, 107, 101, 79, 21, 14, 126, 78, 17, 98, 36, 85, 44, 12, 74, 24, 28, 92, 20, 10, 42, 88, 71, 43, 34, 73, 37, 7, 23, 29, 16, 84, 102, 87, 32, 18, 72, 95, 38, 64, 82, 31, 80, 35, 27, 41, 96, 9, 5, 69, 0, 106, 75, 2, 99, 105, 3, 26, 66, 13, 11, 67, 77, 8, 6, 1, 65, 68, 90, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 127, 120, 61, 47, 121, 45, 114, 48, 57, 56, 118, 39, 125, 124, 86, 100, 89, 55, 103, 25, 63, 110, 22, 119, 116, 109, 111, 54, 122, 97, 59, 91, 30, 46, 40, 108, 104, 94, 50, 33, 123, 93, 19, 81, 52, 101, 43, 79, 76, 83, 36, 21, 49, 14, 126, 98, 44, 15, 42, 17, 12, 78, 85, 92, 58, 28, 10, 24, 88, 34, 84, 74, 107, 20, 95, 102, 23, 7, 73, 18, 87, 106, 27, 71, 37, 9, 41, 72, 16, 38, 32, 75, 5, 80, 31, 35, 82, 29, 96, 105, 69, 0, 64, 13, 2, 26, 66, 11, 8, 67, 77, 99, 3, 90, 68, 6, 1, 4, 70, 65], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 57, 124, 118, 48, 63, 86, 100, 116, 56, 103, 22, 89, 25, 110, 119, 125, 55, 97, 122, 54, 109, 91, 111, 40, 30, 59, 52, 108, 104, 46, 50, 81, 123, 101, 126, 93, 94, 83, 79, 19, 12, 49, 76, 17, 33, 36, 21, 14, 98, 44, 15, 85, 92, 78, 74, 107, 28, 58, 88, 42, 20, 34, 43, 24, 95, 87, 73, 37, 23, 82, 18, 106, 80, 84, 102, 16, 38, 96, 27, 9, 71, 32, 7, 41, 29, 72, 10, 99, 31, 105, 69, 5, 77, 75, 35, 67, 11, 26, 0, 13, 66, 8, 3, 64, 90, 2, 1, 6, 68, 4, 65, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 39, 57, 48, 89, 100, 116, 124, 86, 118, 125, 22, 25, 103, 56, 122, 54, 63, 97, 111, 109, 55, 110, 104, 91, 59, 52, 30, 119, 123, 46, 93, 49, 40, 19, 126, 50, 81, 83, 108, 101, 17, 33, 36, 21, 94, 107, 98, 85, 28, 44, 34, 15, 14, 76, 58, 79, 92, 12, 78, 24, 43, 102, 20, 87, 88, 96, 27, 74, 106, 84, 42, 37, 41, 18, 23, 29, 95, 38, 80, 10, 16, 31, 71, 32, 99, 105, 9, 82, 7, 73, 35, 26, 72, 5, 77, 69, 90, 64, 75, 66, 67, 13, 0, 11, 65, 8, 3, 2, 4, 70, 68, 1, 6], [117, 112, 53, 60, 115, 62, 113, 51, 127, 61, 47, 120, 121, 45, 114, 48, 56, 57, 39, 103, 124, 100, 86, 118, 116, 89, 25, 22, 125, 54, 59, 91, 122, 63, 109, 119, 123, 104, 97, 55, 30, 111, 40, 110, 46, 93, 49, 33, 52, 81, 126, 50, 108, 19, 101, 36, 17, 94, 83, 76, 14, 79, 15, 92, 98, 107, 28, 21, 44, 78, 85, 74, 58, 12, 37, 24, 34, 42, 88, 20, 102, 87, 106, 10, 43, 84, 27, 41, 38, 23, 32, 95, 71, 96, 7, 80, 82, 16, 18, 73, 9, 29, 105, 31, 5, 35, 0, 69, 8, 2, 26, 3, 72, 66, 99, 64, 75, 77, 13, 90, 67, 11, 1, 4, 68, 70, 65, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 56, 48, 118, 100, 103, 63, 86, 25, 125, 22, 116, 109, 89, 122, 104, 111, 97, 110, 54, 119, 59, 123, 91, 108, 40, 30, 55, 126, 49, 33, 46, 94, 93, 50, 83, 19, 101, 17, 81, 98, 52, 76, 106, 85, 79, 44, 42, 107, 58, 78, 34, 14, 36, 21, 102, 37, 28, 92, 24, 12, 15, 10, 88, 38, 43, 74, 96, 84, 27, 32, 41, 95, 23, 87, 7, 20, 16, 35, 18, 80, 31, 9, 71, 5, 29, 82, 73, 0, 99, 69, 64, 26, 13, 67, 66, 8, 105, 72, 2, 75, 11, 3, 90, 77, 1, 70, 4, 65, 68, 6], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 114, 39, 124, 57, 48, 89, 118, 86, 56, 100, 116, 103, 22, 54, 122, 110, 109, 97, 25, 63, 125, 30, 91, 46, 126, 59, 111, 52, 119, 55, 50, 104, 40, 33, 108, 123, 93, 81, 36, 49, 101, 19, 83, 94, 58, 12, 79, 98, 28, 21, 44, 85, 15, 34, 76, 78, 14, 42, 92, 74, 107, 17, 102, 24, 106, 37, 87, 20, 88, 95, 84, 41, 27, 31, 23, 38, 18, 96, 43, 29, 16, 82, 7, 80, 9, 71, 10, 32, 35, 8, 99, 105, 73, 26, 75, 66, 69, 13, 5, 90, 0, 11, 64, 77, 2, 67, 72, 3, 68, 1, 70, 4, 65, 6], [117, 112, 115, 53, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 56, 57, 118, 86, 39, 89, 116, 100, 103, 22, 125, 63, 109, 122, 110, 54, 59, 97, 91, 30, 25, 46, 55, 111, 40, 104, 126, 52, 119, 50, 123, 93, 81, 108, 33, 17, 21, 19, 15, 83, 101, 58, 76, 14, 79, 92, 85, 44, 12, 28, 49, 24, 34, 36, 98, 88, 74, 94, 107, 43, 78, 42, 37, 87, 20, 106, 84, 23, 102, 9, 16, 31, 18, 71, 96, 29, 35, 7, 80, 27, 32, 38, 95, 41, 10, 82, 26, 105, 69, 8, 5, 77, 75, 73, 66, 11, 3, 13, 64, 0, 2, 90, 72, 99, 67, 4, 70, 65, 68, 1, 6], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 56, 124, 57, 48, 118, 89, 100, 103, 86, 39, 116, 125, 122, 22, 63, 54, 40, 97, 91, 109, 25, 111, 59, 119, 104, 126, 30, 110, 55, 123, 46, 50, 81, 17, 93, 33, 83, 52, 94, 19, 58, 108, 49, 15, 101, 21, 98, 36, 28, 79, 76, 12, 37, 92, 88, 24, 14, 85, 44, 34, 43, 78, 20, 87, 27, 74, 107, 42, 106, 84, 31, 102, 96, 18, 38, 23, 80, 95, 7, 32, 71, 29, 35, 41, 69, 16, 10, 9, 0, 82, 8, 73, 64, 105, 26, 66, 5, 67, 99, 90, 77, 11, 3, 75, 13, 2, 1, 4, 72, 65, 6, 70, 68], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 48, 124, 118, 56, 100, 86, 125, 89, 103, 57, 116, 22, 25, 97, 122, 54, 104, 63, 91, 59, 111, 109, 55, 119, 46, 110, 30, 40, 123, 108, 52, 126, 33, 81, 19, 93, 58, 50, 83, 101, 36, 15, 94, 85, 92, 17, 79, 76, 98, 28, 21, 49, 12, 14, 78, 34, 37, 43, 24, 44, 88, 87, 42, 38, 84, 102, 107, 96, 27, 20, 106, 10, 23, 32, 95, 18, 74, 31, 7, 29, 82, 71, 41, 80, 8, 35, 9, 105, 16, 99, 11, 5, 75, 69, 26, 64, 73, 66, 77, 2, 0, 67, 90, 13, 3, 72, 4, 65, 1, 6, 68, 70], [117, 112, 53, 115, 60, 113, 62, 51, 61, 120, 127, 47, 121, 45, 114, 39, 48, 56, 57, 116, 124, 100, 89, 103, 86, 118, 125, 22, 59, 91, 97, 110, 25, 122, 52, 30, 54, 109, 63, 111, 104, 46, 119, 40, 55, 126, 33, 123, 81, 108, 93, 19, 50, 49, 58, 83, 79, 17, 94, 85, 15, 101, 14, 12, 28, 44, 21, 92, 34, 76, 98, 74, 107, 24, 78, 88, 36, 37, 43, 102, 96, 42, 20, 84, 87, 106, 8, 27, 23, 10, 7, 95, 32, 31, 29, 38, 71, 35, 16, 73, 41, 82, 80, 18, 105, 9, 26, 69, 75, 5, 67, 11, 13, 99, 2, 90, 77, 64, 3, 0, 66, 72, 6, 4, 68, 1, 70, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 127, 61, 47, 121, 45, 114, 86, 57, 118, 124, 48, 56, 125, 39, 22, 100, 103, 116, 122, 89, 59, 63, 25, 97, 109, 110, 91, 54, 40, 119, 30, 123, 111, 104, 81, 108, 55, 46, 126, 52, 50, 33, 83, 17, 14, 93, 19, 49, 76, 101, 85, 58, 21, 36, 92, 15, 44, 94, 12, 79, 107, 78, 43, 74, 28, 88, 42, 34, 98, 84, 24, 87, 106, 20, 102, 71, 23, 37, 96, 16, 27, 10, 18, 7, 95, 38, 32, 105, 82, 69, 80, 29, 41, 9, 8, 31, 73, 35, 11, 26, 77, 75, 5, 66, 3, 67, 0, 2, 72, 64, 13, 99, 90, 68, 6, 65, 1, 4, 70], [117, 112, 53, 115, 60, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 57, 56, 118, 124, 86, 48, 125, 103, 89, 22, 100, 122, 39, 25, 63, 54, 119, 59, 116, 97, 46, 91, 30, 110, 109, 126, 55, 111, 104, 40, 123, 52, 108, 49, 81, 33, 19, 93, 92, 21, 17, 50, 94, 85, 44, 83, 58, 14, 101, 36, 15, 12, 76, 79, 98, 43, 28, 37, 88, 107, 24, 34, 106, 78, 87, 84, 96, 10, 95, 102, 23, 20, 42, 74, 27, 82, 7, 32, 41, 31, 29, 38, 80, 18, 73, 16, 35, 105, 71, 9, 8, 26, 99, 69, 11, 75, 77, 5, 72, 90, 66, 13, 0, 2, 64, 3, 67, 6, 68, 65, 4, 70, 1], [117, 112, 53, 115, 60, 113, 62, 51, 61, 127, 120, 47, 121, 45, 114, 124, 56, 118, 57, 48, 86, 100, 39, 103, 89, 125, 22, 25, 122, 59, 116, 123, 40, 63, 119, 91, 109, 54, 111, 55, 30, 46, 104, 97, 110, 93, 81, 108, 52, 50, 19, 33, 101, 126, 58, 17, 92, 44, 49, 36, 83, 94, 79, 15, 21, 85, 12, 14, 28, 76, 98, 102, 24, 78, 43, 34, 88, 37, 106, 107, 96, 87, 42, 20, 16, 84, 32, 23, 82, 10, 29, 95, 38, 27, 7, 31, 71, 73, 80, 74, 18, 41, 35, 99, 9, 26, 69, 72, 105, 75, 11, 5, 64, 2, 8, 67, 0, 66, 77, 13, 65, 3, 6, 90, 68, 1, 70, 4], [117, 112, 53, 60, 115, 62, 113, 51, 61, 120, 127, 47, 121, 45, 114, 124, 48, 39, 57, 118, 56, 86, 103, 89, 100, 116, 125, 22, 25, 119, 110, 63, 97, 30, 54, 40, 91, 122, 46, 52, 59, 109, 111, 55, 104, 123, 81, 58, 50, 126, 33, 19, 17, 93, 83, 49, 108, 94, 21, 85, 15, 24, 101, 36, 76, 79, 92, 12, 14, 78, 44, 37, 43, 98, 28, 84, 74, 88, 34, 10, 107, 20, 42, 106, 87, 102, 7, 16, 96, 71, 31, 23, 82, 41, 29, 27, 72, 9, 32, 18, 73, 80, 95, 105, 35, 5, 38, 26, 0, 69, 2, 77, 67, 64, 11, 66, 3, 99, 75, 13, 8, 90, 1, 68, 4, 70, 65, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 124, 48, 57, 118, 86, 100, 125, 116, 110, 89, 56, 22, 103, 25, 63, 119, 46, 109, 59, 40, 97, 91, 104, 52, 123, 122, 54, 30, 111, 108, 58, 101, 55, 93, 33, 81, 36, 126, 21, 50, 17, 19, 83, 94, 92, 12, 15, 98, 107, 44, 14, 43, 78, 76, 79, 85, 34, 49, 106, 42, 28, 87, 24, 88, 102, 10, 37, 84, 7, 18, 74, 41, 71, 20, 82, 32, 96, 9, 95, 31, 5, 72, 16, 38, 27, 23, 29, 69, 26, 2, 64, 11, 35, 73, 80, 0, 13, 105, 90, 66, 77, 75, 3, 99, 67, 4, 70, 8, 1, 65, 68, 6], [117, 112, 53, 60, 115, 62, 113, 51, 120, 61, 127, 47, 121, 45, 114, 39, 57, 124, 56, 86, 118, 125, 116, 100, 103, 22, 48, 89, 110, 25, 59, 54, 63, 97, 91, 122, 30, 40, 52, 109, 111, 126, 46, 123, 104, 58, 55, 119, 50, 108, 33, 17, 81, 93, 19, 83, 107, 85, 12, 14, 49, 21, 101, 15, 94, 92, 98, 36, 44, 79, 76, 78, 106, 34, 24, 42, 88, 28, 37, 43, 10, 84, 96, 20, 74, 102, 87, 27, 7, 9, 16, 23, 31, 38, 72, 95, 71, 41, 18, 32, 80, 73, 29, 82, 26, 105, 5, 77, 11, 69, 35, 66, 99, 3, 13, 67, 75, 64, 2, 90, 0, 65, 8, 68, 70, 4, 6, 1], [117, 112, 53, 115, 60, 62, 113, 51, 120, 127, 61, 47, 121, 45, 124, 39, 114, 57, 125, 56, 118, 103, 48, 116, 86, 100, 89, 22, 122, 91, 46, 40, 110, 63, 109, 25, 59, 111, 55, 123, 54, 97, 52, 50, 58, 126, 108, 104, 119, 30, 33, 81, 93, 101, 83, 94, 49, 17, 21, 36, 107, 79, 12, 19, 76, 44, 14, 78, 15, 92, 42, 28, 85, 24, 34, 98, 106, 20, 10, 37, 43, 71, 88, 84, 74, 7, 96, 87, 72, 27, 102, 32, 31, 9, 73, 95, 38, 80, 23, 16, 5, 69, 29, 82, 18, 11, 41, 66, 0, 105, 26, 64, 35, 99, 77, 13, 67, 2, 3, 70, 4, 75, 90, 8, 68, 1, 6, 65], [117, 112, 53, 115, 60, 113, 62, 51, 120, 61, 127, 47, 121, 45, 114, 39, 56, 124, 57, 86, 100, 118, 125, 89, 116, 48, 22, 103, 91, 25, 59, 111, 122, 97, 40, 109, 110, 119, 54, 123, 63, 30, 55, 104, 81, 46, 93, 126, 33, 19, 50, 52, 17, 49, 79, 83, 85, 101, 14, 94, 15, 92, 12, 21, 78, 108, 76, 58, 44, 98, 107, 36, 10, 28, 88, 74, 20, 106, 24, 43, 84, 87, 34, 37, 102, 42, 31, 72, 96, 71, 27, 95, 9, 73, 23, 7, 38, 18, 16, 32, 82, 80, 29, 35, 26, 69, 99, 5, 13, 11, 41, 75, 77, 105, 90, 8, 2, 66, 3, 70, 67, 0, 64, 6, 65, 4, 68, 1]], "model.layers.16.self_attn.q_proj": [[61, 125, 115, 121, 36, 101, 43, 62, 127, 63, 55, 92, 100, 97, 57, 40, 124, 60, 119, 54, 109, 49, 123, 126, 53, 59, 58, 113, 47, 117, 114, 118, 112, 50, 103, 51, 116, 52, 122, 45, 120, 102, 56, 41, 48, 89, 46, 110, 111, 37, 42, 107, 44, 108, 104, 32, 33, 39, 38, 105, 106, 99, 84, 35, 34, 98, 76, 10, 13, 24, 29, 72, 70, 95, 5, 16, 88, 21, 96, 2, 25, 71, 9, 4, 94, 86, 65, 0, 26, 80, 31, 23, 11, 93, 15, 30, 82, 77, 20, 3, 28, 78, 22, 67, 73, 18, 27, 90, 91, 87, 85, 6, 14, 75, 83, 79, 69, 8, 74, 19, 12, 81, 17, 68, 66, 1, 7, 64], [125, 61, 121, 115, 62, 55, 124, 101, 57, 126, 49, 52, 119, 127, 60, 112, 63, 50, 123, 120, 59, 53, 47, 117, 54, 113, 114, 118, 43, 56, 116, 122, 51, 107, 58, 48, 109, 45, 46, 111, 110, 40, 108, 44, 106, 92, 42, 41, 35, 105, 104, 39, 36, 38, 103, 37, 100, 102, 98, 99, 22, 25, 34, 97, 33, 96, 19, 32, 93, 89, 95, 18, 31, 94, 86, 15, 30, 29, 76, 78, 4, 26, 24, 23, 11, 72, 28, 16, 70, 9, 13, 77, 27, 21, 87, 83, 81, 90, 85, 17, 91, 88, 10, 71, 20, 2, 73, 84, 0, 82, 65, 74, 3, 79, 5, 80, 14, 67, 6, 12, 8, 1, 75, 64, 68, 66, 69, 7], [61, 125, 27, 121, 36, 94, 87, 100, 25, 62, 115, 35, 21, 63, 101, 20, 43, 127, 122, 60, 55, 32, 57, 49, 102, 40, 96, 74, 84, 95, 54, 126, 123, 52, 79, 53, 37, 113, 124, 118, 47, 119, 93, 114, 50, 117, 44, 120, 10, 112, 56, 51, 109, 59, 16, 48, 116, 83, 58, 45, 99, 46, 42, 19, 41, 104, 30, 31, 90, 82, 110, 29, 111, 92, 6, 107, 86, 12, 89, 105, 23, 13, 80, 15, 97, 98, 39, 91, 18, 28, 34, 108, 38, 77, 78, 33, 106, 17, 4, 24, 26, 103, 88, 73, 85, 5, 66, 68, 1, 9, 2, 76, 0, 67, 14, 81, 8, 22, 71, 7, 69, 64, 70, 72, 3, 75, 11, 65], [61, 125, 100, 96, 26, 30, 88, 122, 27, 121, 87, 77, 92, 98, 115, 32, 81, 75, 20, 43, 17, 93, 8, 94, 36, 62, 33, 29, 90, 44, 86, 99, 23, 41, 63, 34, 127, 40, 85, 49, 68, 101, 83, 57, 73, 102, 69, 19, 55, 110, 53, 56, 118, 60, 113, 48, 116, 2, 103, 16, 84, 13, 24, 12, 11, 50, 52, 47, 54, 31, 112, 51, 91, 79, 14, 109, 18, 6, 117, 126, 95, 78, 123, 119, 25, 97, 89, 21, 120, 124, 35, 111, 1, 72, 46, 42, 58, 76, 15, 114, 104, 45, 64, 59, 80, 38, 7, 28, 105, 67, 39, 107, 108, 106, 74, 82, 70, 71, 37, 66, 10, 4, 22, 9, 5, 3, 0, 65], [42, 98, 30, 124, 24, 85, 116, 106, 15, 19, 12, 17, 8, 88, 102, 78, 95, 52, 27, 10, 68, 66, 70, 94, 81, 38, 7, 119, 60, 90, 125, 22, 47, 83, 64, 120, 63, 55, 11, 75, 21, 59, 37, 92, 41, 53, 1, 26, 86, 76, 9, 111, 109, 114, 28, 44, 49, 103, 126, 23, 80, 117, 93, 121, 57, 36, 91, 108, 34, 25, 62, 123, 29, 6, 14, 48, 87, 35, 58, 3, 105, 79, 32, 45, 20, 33, 99, 118, 67, 113, 100, 18, 51, 69, 43, 115, 110, 31, 61, 46, 127, 5, 13, 122, 40, 89, 39, 72, 104, 56, 73, 54, 65, 2, 97, 82, 101, 50, 77, 107, 74, 0, 112, 16, 84, 96, 71, 4], [42, 124, 98, 63, 116, 30, 106, 38, 27, 19, 85, 88, 24, 62, 15, 35, 90, 17, 28, 10, 87, 57, 102, 99, 53, 122, 114, 47, 59, 61, 78, 109, 48, 108, 56, 60, 94, 123, 120, 125, 117, 113, 40, 12, 119, 26, 110, 43, 8, 45, 50, 126, 44, 118, 115, 31, 58, 52, 95, 46, 111, 80, 121, 93, 127, 112, 51, 54, 105, 36, 107, 86, 70, 81, 20, 49, 100, 68, 55, 37, 103, 39, 22, 96, 104, 33, 11, 41, 32, 18, 83, 21, 97, 23, 5, 84, 29, 101, 91, 34, 0, 92, 3, 6, 66, 74, 25, 89, 1, 82, 77, 71, 9, 16, 4, 13, 79, 7, 14, 75, 64, 69, 67, 76, 73, 65, 2, 72], [42, 98, 30, 124, 106, 85, 19, 63, 24, 27, 17, 59, 116, 15, 88, 12, 10, 78, 68, 8, 53, 70, 60, 94, 81, 102, 95, 28, 66, 122, 117, 52, 58, 45, 93, 125, 96, 76, 90, 64, 72, 89, 119, 38, 55, 120, 41, 103, 62, 100, 49, 47, 80, 82, 126, 114, 108, 21, 127, 83, 123, 44, 87, 79, 111, 113, 32, 37, 36, 35, 40, 14, 26, 20, 50, 43, 110, 46, 91, 121, 105, 48, 75, 25, 109, 115, 56, 92, 104, 1, 4, 71, 51, 61, 65, 101, 16, 6, 57, 11, 74, 31, 54, 29, 67, 22, 7, 3, 84, 23, 99, 33, 118, 2, 97, 39, 107, 13, 112, 86, 18, 73, 9, 77, 34, 5, 0, 69], [42, 98, 124, 52, 106, 30, 85, 15, 24, 27, 19, 88, 63, 60, 17, 45, 10, 116, 53, 95, 12, 108, 117, 59, 100, 121, 8, 113, 102, 94, 114, 119, 122, 48, 93, 47, 44, 58, 40, 62, 36, 37, 123, 78, 105, 35, 111, 49, 28, 125, 81, 90, 38, 57, 43, 104, 33, 56, 109, 46, 70, 103, 50, 41, 118, 68, 115, 23, 61, 51, 32, 54, 55, 127, 20, 126, 87, 6, 112, 39, 110, 80, 101, 120, 67, 91, 99, 25, 97, 96, 83, 21, 86, 82, 66, 92, 26, 107, 89, 31, 1, 3, 13, 71, 84, 74, 29, 22, 75, 79, 18, 11, 77, 34, 16, 64, 14, 72, 76, 69, 73, 2, 5, 9, 7, 65, 4, 0], [120, 39, 51, 48, 119, 123, 25, 112, 98, 89, 82, 53, 13, 31, 54, 44, 52, 113, 127, 111, 122, 121, 59, 58, 124, 60, 126, 117, 57, 50, 118, 116, 62, 61, 63, 125, 91, 107, 47, 114, 46, 115, 41, 49, 109, 56, 55, 20, 108, 28, 106, 45, 110, 42, 104, 43, 92, 103, 102, 1, 4, 11, 87, 105, 96, 65, 35, 36, 86, 101, 37, 72, 38, 100, 40, 33, 18, 99, 32, 66, 93, 90, 29, 95, 23, 7, 79, 97, 30, 69, 94, 8, 34, 80, 21, 77, 16, 27, 84, 15, 26, 85, 75, 14, 0, 9, 68, 19, 76, 88, 64, 22, 3, 24, 83, 78, 17, 5, 81, 10, 71, 67, 12, 2, 73, 6, 74, 70], [51, 48, 39, 120, 25, 98, 82, 123, 119, 87, 127, 91, 50, 121, 124, 54, 45, 122, 89, 106, 13, 31, 118, 53, 111, 86, 28, 52, 58, 112, 61, 113, 49, 46, 92, 126, 47, 110, 60, 57, 115, 62, 40, 117, 63, 44, 56, 108, 43, 55, 15, 41, 79, 116, 104, 59, 125, 77, 73, 36, 84, 109, 4, 38, 114, 65, 20, 23, 42, 103, 96, 7, 107, 99, 105, 27, 18, 100, 11, 102, 3, 37, 35, 68, 95, 8, 93, 101, 72, 88, 76, 34, 66, 94, 64, 22, 24, 30, 69, 85, 83, 90, 32, 33, 80, 78, 26, 75, 97, 10, 81, 17, 16, 29, 19, 74, 1, 2, 12, 70, 21, 14, 67, 9, 5, 71, 6, 0], [120, 39, 51, 25, 48, 98, 31, 89, 54, 127, 53, 123, 82, 20, 91, 11, 13, 56, 58, 118, 86, 49, 41, 57, 113, 116, 119, 112, 124, 15, 43, 4, 111, 47, 121, 60, 122, 62, 115, 28, 108, 63, 52, 107, 23, 114, 92, 50, 117, 126, 61, 59, 42, 45, 125, 7, 72, 32, 46, 55, 36, 102, 79, 88, 1, 104, 8, 109, 44, 19, 101, 93, 96, 27, 103, 87, 37, 77, 40, 110, 76, 99, 9, 33, 94, 106, 95, 17, 26, 66, 105, 69, 35, 100, 81, 30, 29, 97, 90, 85, 38, 75, 0, 84, 22, 21, 16, 80, 83, 24, 65, 10, 18, 68, 14, 34, 78, 6, 64, 12, 73, 2, 74, 70, 71, 5, 67, 3], [39, 51, 56, 120, 48, 123, 98, 119, 25, 20, 31, 89, 82, 28, 13, 50, 52, 54, 122, 43, 60, 87, 86, 41, 59, 79, 53, 91, 116, 88, 115, 4, 113, 124, 9, 92, 118, 7, 42, 58, 127, 126, 73, 112, 11, 57, 77, 114, 76, 125, 101, 109, 121, 61, 63, 1, 117, 81, 17, 111, 62, 102, 49, 10, 55, 19, 90, 108, 46, 8, 104, 45, 21, 85, 47, 93, 44, 22, 27, 74, 6, 38, 97, 23, 70, 75, 107, 14, 95, 32, 69, 15, 78, 40, 105, 103, 106, 84, 99, 33, 18, 96, 26, 16, 94, 0, 66, 36, 35, 110, 72, 67, 30, 80, 29, 12, 37, 24, 3, 5, 65, 100, 64, 71, 83, 34, 68, 2], [42, 100, 106, 90, 79, 78, 18, 20, 31, 85, 94, 75, 77, 111, 16, 86, 70, 73, 26, 4, 76, 46, 72, 126, 125, 124, 58, 3, 64, 67, 66, 56, 103, 23, 55, 117, 40, 28, 82, 21, 52, 2, 45, 65, 114, 80, 0, 11, 33, 127, 24, 10, 12, 119, 44, 71, 83, 81, 95, 88, 62, 15, 7, 97, 84, 8, 74, 5, 14, 53, 25, 93, 39, 122, 22, 109, 30, 59, 50, 61, 121, 69, 29, 6, 57, 89, 123, 9, 115, 13, 32, 113, 104, 47, 17, 91, 36, 54, 27, 41, 120, 99, 37, 51, 110, 105, 112, 34, 92, 116, 19, 108, 87, 48, 107, 43, 98, 60, 96, 35, 102, 49, 38, 68, 101, 118, 63, 1], [42, 100, 20, 106, 18, 77, 85, 90, 88, 111, 79, 75, 95, 73, 31, 70, 58, 40, 62, 23, 94, 26, 125, 52, 46, 16, 127, 68, 82, 78, 2, 89, 124, 104, 126, 45, 11, 109, 117, 72, 80, 61, 84, 56, 57, 67, 64, 1, 9, 99, 47, 114, 60, 33, 48, 96, 122, 21, 86, 103, 55, 12, 36, 14, 13, 97, 119, 3, 54, 25, 91, 15, 50, 115, 112, 101, 76, 17, 81, 4, 29, 65, 69, 39, 44, 102, 51, 32, 27, 92, 38, 8, 93, 53, 116, 105, 37, 24, 107, 22, 0, 123, 30, 49, 63, 41, 87, 108, 7, 110, 28, 5, 71, 83, 113, 74, 43, 59, 10, 118, 6, 34, 121, 120, 35, 19, 98, 66], [42, 100, 106, 20, 78, 90, 18, 85, 77, 73, 88, 111, 75, 94, 16, 70, 46, 124, 58, 26, 79, 72, 31, 125, 4, 56, 2, 23, 40, 67, 82, 52, 64, 127, 62, 33, 126, 104, 9, 117, 119, 95, 3, 103, 24, 11, 13, 45, 68, 57, 65, 55, 0, 99, 30, 86, 80, 29, 114, 44, 21, 8, 47, 48, 50, 66, 112, 22, 25, 60, 12, 91, 84, 69, 115, 7, 89, 109, 1, 81, 39, 96, 83, 116, 105, 97, 51, 36, 122, 93, 15, 123, 113, 92, 54, 32, 59, 87, 14, 28, 38, 63, 120, 110, 17, 74, 49, 53, 107, 118, 102, 61, 27, 108, 71, 10, 35, 5, 37, 34, 43, 41, 19, 76, 101, 6, 121, 98], [42, 100, 94, 88, 106, 85, 31, 78, 18, 90, 16, 111, 20, 79, 95, 70, 75, 77, 46, 86, 11, 73, 56, 124, 127, 40, 80, 72, 10, 55, 58, 45, 67, 26, 71, 21, 39, 65, 125, 104, 13, 83, 23, 119, 62, 117, 33, 82, 47, 103, 126, 5, 114, 12, 57, 91, 109, 2, 69, 0, 19, 97, 6, 74, 24, 107, 60, 1, 52, 84, 17, 61, 76, 53, 30, 81, 15, 29, 7, 102, 44, 113, 28, 115, 98, 105, 110, 4, 59, 38, 89, 120, 122, 101, 116, 99, 3, 25, 50, 32, 64, 22, 49, 93, 36, 8, 54, 123, 118, 34, 37, 112, 63, 92, 27, 108, 51, 35, 96, 87, 121, 43, 48, 41, 68, 66, 14, 9], [40, 54, 63, 36, 98, 122, 90, 20, 88, 81, 22, 123, 29, 78, 15, 93, 24, 13, 82, 33, 57, 62, 115, 126, 26, 38, 91, 68, 83, 51, 27, 76, 8, 61, 125, 120, 92, 114, 17, 127, 113, 34, 45, 124, 121, 21, 74, 19, 10, 11, 59, 100, 116, 50, 117, 119, 56, 46, 55, 97, 118, 73, 70, 16, 94, 31, 84, 53, 44, 107, 52, 28, 23, 48, 111, 106, 75, 109, 87, 18, 49, 77, 72, 80, 96, 69, 30, 35, 43, 25, 79, 89, 108, 112, 85, 0, 71, 32, 95, 101, 99, 1, 47, 39, 9, 2, 65, 58, 6, 37, 42, 4, 110, 105, 41, 103, 102, 60, 86, 14, 67, 3, 5, 104, 66, 64, 12, 7], [40, 54, 98, 63, 29, 82, 20, 76, 16, 122, 22, 71, 88, 50, 36, 3, 78, 90, 26, 93, 8, 73, 7, 83, 13, 61, 123, 15, 81, 84, 87, 5, 80, 66, 17, 64, 127, 115, 21, 77, 89, 74, 113, 65, 62, 12, 0, 125, 120, 100, 119, 86, 27, 23, 104, 97, 52, 91, 116, 121, 24, 2, 59, 19, 55, 112, 10, 57, 18, 48, 14, 51, 79, 118, 126, 9, 11, 43, 69, 53, 49, 92, 124, 45, 114, 56, 106, 6, 4, 111, 25, 33, 31, 117, 95, 70, 34, 30, 94, 67, 32, 85, 72, 46, 110, 28, 75, 37, 58, 38, 35, 103, 96, 99, 108, 42, 102, 109, 105, 101, 47, 1, 68, 60, 39, 41, 107, 44], [40, 61, 63, 122, 98, 54, 26, 123, 29, 88, 20, 24, 113, 93, 57, 22, 81, 90, 127, 125, 82, 78, 45, 76, 16, 73, 91, 92, 44, 121, 99, 32, 50, 62, 96, 19, 85, 38, 107, 15, 106, 28, 126, 47, 118, 112, 41, 120, 31, 109, 105, 119, 46, 116, 102, 21, 51, 39, 23, 74, 111, 117, 14, 55, 37, 43, 33, 89, 86, 35, 94, 53, 42, 103, 52, 5, 69, 60, 114, 110, 97, 115, 49, 80, 58, 12, 27, 17, 95, 13, 124, 108, 56, 48, 71, 59, 100, 83, 36, 104, 25, 101, 84, 87, 11, 30, 3, 10, 79, 66, 0, 64, 75, 68, 2, 34, 9, 18, 65, 8, 70, 1, 77, 6, 67, 4, 72, 7], [63, 40, 54, 122, 100, 123, 125, 127, 62, 57, 98, 113, 121, 119, 118, 59, 116, 115, 61, 51, 55, 114, 120, 53, 29, 124, 50, 52, 117, 91, 48, 56, 126, 20, 111, 26, 45, 36, 93, 88, 49, 112, 46, 87, 47, 25, 110, 27, 43, 106, 22, 81, 103, 109, 23, 108, 58, 60, 107, 99, 39, 38, 42, 44, 105, 41, 102, 96, 35, 101, 34, 33, 97, 37, 89, 104, 94, 92, 83, 21, 82, 28, 32, 79, 86, 24, 18, 90, 95, 15, 31, 16, 30, 75, 84, 13, 17, 85, 11, 80, 0, 73, 19, 74, 10, 2, 1, 72, 76, 64, 3, 69, 66, 68, 5, 8, 6, 78, 14, 71, 9, 70, 67, 77, 4, 12, 65, 7], [48, 52, 63, 116, 127, 55, 123, 61, 53, 54, 124, 117, 51, 56, 119, 62, 60, 122, 115, 126, 120, 112, 118, 59, 47, 111, 121, 49, 114, 57, 125, 58, 113, 50, 45, 46, 109, 36, 110, 101, 107, 108, 42, 44, 106, 43, 105, 98, 32, 96, 41, 38, 40, 39, 103, 100, 104, 37, 92, 102, 34, 90, 26, 99, 33, 35, 89, 93, 95, 97, 91, 31, 15, 94, 28, 20, 30, 23, 27, 29, 86, 22, 84, 17, 85, 81, 79, 87, 21, 25, 82, 76, 88, 14, 18, 74, 12, 72, 8, 10, 16, 83, 78, 24, 66, 69, 67, 5, 71, 77, 7, 3, 19, 11, 64, 2, 68, 80, 6, 4, 70, 73, 13, 75, 0, 1, 65, 9], [116, 52, 48, 101, 57, 37, 100, 97, 120, 59, 123, 94, 95, 63, 112, 114, 58, 54, 126, 38, 34, 60, 92, 50, 110, 127, 15, 125, 46, 39, 47, 113, 43, 55, 106, 61, 118, 56, 124, 111, 121, 44, 45, 99, 32, 22, 49, 105, 107, 103, 108, 119, 115, 40, 62, 51, 53, 42, 36, 109, 88, 85, 117, 98, 41, 122, 102, 35, 76, 91, 31, 23, 104, 89, 27, 28, 96, 90, 86, 30, 12, 17, 18, 20, 87, 26, 33, 29, 82, 84, 93, 25, 79, 14, 72, 21, 81, 24, 74, 83, 19, 10, 8, 78, 16, 80, 71, 5, 67, 69, 7, 3, 77, 11, 13, 73, 75, 2, 70, 9, 4, 66, 68, 6, 1, 65, 0, 64], [116, 48, 97, 101, 52, 88, 83, 77, 11, 16, 73, 4, 71, 22, 14, 70, 126, 85, 18, 61, 0, 2, 68, 42, 30, 64, 26, 27, 105, 55, 75, 9, 28, 80, 13, 93, 17, 47, 120, 65, 82, 121, 6, 24, 7, 67, 19, 41, 91, 5, 78, 118, 66, 37, 74, 127, 20, 69, 34, 21, 86, 10, 89, 23, 108, 3, 29, 31, 54, 57, 81, 53, 84, 87, 79, 114, 15, 36, 103, 72, 123, 50, 12, 8, 99, 1, 76, 45, 35, 43, 51, 33, 25, 92, 96, 98, 115, 46, 110, 94, 95, 90, 49, 112, 32, 113, 109, 102, 38, 44, 63, 39, 100, 106, 104, 40, 58, 59, 125, 60, 56, 124, 119, 62, 111, 107, 117, 122], [52, 63, 116, 48, 60, 114, 55, 51, 54, 122, 124, 53, 117, 125, 59, 127, 126, 120, 119, 47, 62, 58, 115, 123, 118, 61, 121, 111, 113, 46, 49, 101, 57, 56, 50, 109, 110, 45, 36, 112, 42, 98, 107, 32, 105, 43, 44, 97, 92, 106, 15, 108, 41, 37, 94, 90, 96, 104, 40, 103, 38, 39, 99, 100, 102, 35, 85, 34, 23, 26, 95, 33, 84, 29, 82, 87, 28, 30, 20, 27, 76, 93, 31, 91, 79, 89, 25, 17, 22, 86, 18, 81, 12, 21, 74, 8, 10, 72, 88, 5, 69, 24, 83, 14, 16, 71, 67, 78, 77, 7, 2, 3, 66, 4, 73, 9, 19, 13, 68, 11, 80, 0, 65, 75, 70, 6, 64, 1], [124, 49, 37, 55, 126, 61, 87, 118, 26, 93, 96, 121, 80, 12, 86, 119, 57, 84, 101, 50, 54, 60, 78, 123, 25, 90, 32, 67, 122, 18, 58, 85, 115, 112, 72, 29, 51, 16, 102, 120, 56, 6, 127, 59, 53, 15, 83, 52, 116, 34, 63, 92, 38, 89, 114, 62, 47, 46, 76, 79, 111, 103, 110, 104, 117, 125, 48, 98, 9, 44, 109, 36, 17, 43, 39, 45, 97, 81, 33, 21, 27, 107, 30, 10, 20, 108, 23, 106, 113, 35, 28, 31, 74, 73, 105, 42, 70, 88, 41, 22, 95, 91, 94, 11, 99, 13, 3, 19, 40, 77, 100, 24, 64, 14, 82, 75, 66, 4, 1, 69, 7, 0, 8, 65, 71, 68, 2, 5], [49, 124, 37, 55, 93, 61, 96, 126, 50, 118, 54, 57, 119, 123, 102, 121, 87, 79, 127, 112, 115, 53, 60, 122, 58, 125, 51, 120, 110, 114, 59, 85, 56, 90, 101, 26, 63, 62, 43, 116, 38, 36, 5, 46, 108, 113, 42, 52, 48, 117, 111, 44, 86, 47, 107, 41, 32, 30, 45, 109, 12, 39, 40, 104, 18, 35, 80, 105, 21, 103, 106, 24, 99, 20, 69, 100, 29, 25, 84, 33, 34, 73, 8, 92, 31, 98, 71, 89, 95, 77, 2, 94, 6, 23, 97, 91, 27, 17, 67, 28, 0, 88, 19, 11, 22, 64, 72, 15, 13, 7, 83, 16, 78, 3, 65, 81, 76, 9, 4, 70, 75, 82, 10, 74, 66, 14, 1, 68], [49, 124, 37, 55, 61, 126, 101, 118, 93, 119, 26, 87, 96, 50, 57, 123, 60, 121, 54, 21, 44, 115, 122, 58, 29, 12, 84, 85, 51, 56, 63, 90, 53, 120, 125, 47, 112, 102, 25, 59, 110, 62, 48, 116, 46, 52, 113, 114, 105, 100, 104, 45, 108, 111, 127, 43, 39, 86, 117, 103, 19, 109, 99, 27, 107, 38, 42, 28, 34, 88, 81, 80, 106, 41, 36, 72, 33, 31, 6, 40, 97, 79, 32, 35, 24, 78, 95, 98, 75, 30, 92, 67, 20, 73, 94, 17, 18, 83, 23, 82, 77, 91, 22, 10, 15, 89, 76, 14, 3, 69, 11, 8, 64, 13, 16, 70, 9, 7, 71, 0, 65, 74, 4, 5, 66, 1, 68, 2], [49, 124, 37, 55, 87, 61, 126, 118, 96, 50, 57, 123, 26, 80, 93, 101, 54, 121, 90, 86, 119, 60, 12, 122, 120, 51, 127, 115, 58, 53, 78, 84, 6, 56, 63, 114, 67, 112, 29, 52, 59, 47, 104, 85, 116, 113, 46, 25, 48, 125, 62, 102, 111, 20, 83, 65, 44, 89, 70, 16, 8, 110, 108, 107, 15, 79, 23, 21, 45, 18, 33, 99, 117, 109, 64, 92, 43, 35, 94, 72, 39, 40, 106, 32, 77, 42, 22, 2, 17, 0, 97, 71, 38, 27, 36, 103, 105, 10, 28, 76, 100, 14, 81, 30, 95, 73, 88, 34, 41, 4, 19, 9, 24, 3, 11, 13, 91, 31, 98, 69, 7, 68, 74, 82, 75, 1, 5, 66], [103, 34, 45, 28, 109, 84, 24, 38, 79, 17, 85, 86, 30, 92, 31, 12, 11, 95, 9, 22, 46, 111, 49, 23, 122, 116, 88, 14, 19, 63, 27, 36, 20, 75, 57, 44, 33, 121, 53, 107, 119, 52, 83, 115, 101, 113, 104, 124, 15, 50, 32, 48, 61, 100, 37, 97, 29, 91, 25, 94, 125, 108, 26, 93, 54, 73, 126, 106, 16, 55, 47, 56, 114, 40, 43, 42, 35, 117, 99, 112, 51, 120, 60, 21, 89, 58, 41, 123, 81, 102, 82, 62, 59, 110, 72, 105, 96, 127, 13, 90, 118, 18, 10, 87, 39, 8, 5, 98, 7, 68, 76, 78, 77, 6, 4, 80, 65, 69, 74, 67, 3, 66, 70, 1, 71, 2, 64, 0], [103, 45, 34, 28, 14, 109, 17, 12, 7, 84, 71, 66, 79, 9, 3, 5, 24, 85, 69, 44, 64, 65, 67, 31, 86, 10, 20, 96, 127, 98, 105, 81, 2, 61, 25, 76, 35, 38, 39, 51, 0, 63, 8, 124, 52, 49, 55, 72, 125, 58, 90, 23, 30, 68, 78, 19, 4, 92, 73, 75, 122, 115, 118, 57, 16, 88, 95, 111, 100, 70, 11, 77, 6, 74, 21, 46, 119, 102, 60, 89, 18, 113, 94, 13, 59, 97, 120, 83, 93, 121, 107, 126, 36, 80, 43, 53, 1, 40, 15, 116, 29, 110, 112, 54, 91, 22, 56, 123, 37, 50, 82, 48, 117, 101, 47, 41, 87, 114, 42, 99, 27, 62, 104, 33, 106, 108, 26, 32], [103, 45, 34, 38, 109, 0, 28, 79, 7, 3, 24, 84, 12, 9, 14, 2, 17, 66, 5, 31, 86, 85, 1, 67, 10, 125, 69, 122, 92, 57, 100, 25, 23, 20, 112, 64, 98, 35, 61, 58, 44, 121, 51, 26, 55, 22, 68, 21, 72, 118, 59, 49, 96, 81, 65, 105, 102, 75, 48, 82, 32, 30, 123, 70, 63, 52, 74, 90, 54, 46, 93, 13, 76, 36, 60, 111, 116, 88, 56, 124, 42, 119, 104, 6, 126, 127, 8, 120, 50, 89, 40, 110, 73, 71, 115, 18, 80, 4, 114, 94, 95, 83, 19, 91, 43, 108, 16, 62, 117, 47, 39, 29, 78, 99, 77, 15, 113, 53, 27, 107, 87, 37, 106, 97, 101, 41, 11, 33], [103, 34, 45, 28, 109, 84, 79, 24, 31, 92, 17, 86, 22, 12, 108, 63, 9, 29, 88, 30, 102, 55, 25, 89, 98, 122, 112, 58, 14, 105, 104, 53, 100, 27, 49, 20, 61, 11, 50, 57, 114, 72, 113, 33, 42, 121, 127, 81, 95, 52, 116, 38, 23, 35, 90, 83, 36, 85, 111, 124, 41, 93, 125, 18, 126, 46, 77, 39, 74, 94, 115, 123, 80, 5, 101, 59, 26, 82, 62, 119, 16, 73, 19, 13, 43, 21, 47, 118, 96, 76, 56, 32, 15, 91, 48, 51, 87, 99, 110, 106, 78, 44, 117, 97, 75, 7, 37, 8, 10, 107, 70, 54, 60, 4, 6, 120, 68, 40, 69, 3, 71, 66, 67, 65, 1, 64, 0, 2]], "model.layers.16.self_attn.k_proj": [[125, 61, 86, 36, 96, 127, 17, 115, 121, 49, 30, 62, 63, 55, 57, 113, 60, 54, 26, 122, 53, 39, 51, 118, 48, 56, 126, 107, 116, 47, 112, 59, 52, 29, 58, 108, 124, 87, 40, 114, 120, 109, 44, 119, 123, 117, 79, 50, 27, 83, 11, 20, 110, 46, 78, 18, 43, 45, 111, 13, 25, 85, 98, 105, 106, 97, 41, 24, 42, 104, 72, 16, 82, 33, 71, 38, 101, 103, 9, 34, 28, 76, 10, 37, 91, 102, 100, 89, 31, 88, 35, 4, 99, 22, 70, 80, 95, 5, 94, 92, 21, 15, 14, 2, 32, 90, 19, 93, 77, 12, 65, 84, 23, 81, 0, 7, 3, 67, 73, 75, 6, 8, 74, 1, 69, 68, 66, 64], [106, 94, 34, 124, 116, 24, 63, 19, 52, 59, 85, 17, 15, 42, 55, 60, 121, 10, 70, 78, 12, 120, 27, 49, 8, 117, 111, 58, 47, 108, 114, 53, 109, 64, 68, 18, 1, 66, 4, 126, 123, 50, 122, 22, 44, 102, 45, 91, 51, 125, 46, 127, 90, 0, 119, 62, 112, 54, 56, 11, 84, 105, 104, 48, 61, 100, 28, 57, 41, 110, 40, 113, 115, 95, 118, 14, 80, 92, 103, 37, 77, 107, 101, 76, 73, 69, 2, 16, 26, 89, 32, 99, 43, 31, 13, 35, 9, 36, 23, 33, 97, 87, 20, 25, 7, 39, 93, 65, 96, 38, 75, 71, 67, 3, 86, 29, 74, 79, 5, 88, 82, 21, 83, 30, 72, 6, 98, 81], [103, 51, 120, 34, 86, 56, 48, 95, 123, 115, 112, 119, 53, 28, 54, 122, 58, 89, 60, 126, 52, 88, 40, 82, 61, 15, 124, 118, 111, 50, 121, 63, 46, 57, 113, 47, 62, 125, 117, 55, 107, 116, 20, 26, 114, 98, 91, 108, 110, 8, 59, 45, 49, 44, 81, 11, 43, 127, 109, 13, 42, 4, 38, 105, 93, 65, 41, 73, 106, 6, 64, 30, 16, 68, 37, 24, 27, 104, 72, 23, 102, 35, 33, 36, 85, 29, 80, 100, 76, 94, 101, 99, 12, 9, 87, 97, 14, 67, 1, 21, 79, 19, 66, 83, 71, 78, 17, 32, 90, 96, 10, 7, 2, 18, 92, 0, 70, 74, 77, 25, 5, 3, 84, 39, 69, 31, 75, 22], [106, 36, 90, 18, 85, 47, 20, 42, 79, 77, 110, 16, 58, 72, 75, 124, 126, 73, 78, 70, 56, 127, 109, 117, 119, 114, 30, 95, 120, 40, 86, 4, 125, 65, 108, 76, 112, 62, 45, 52, 43, 67, 60, 99, 88, 61, 96, 51, 39, 53, 111, 0, 66, 54, 93, 97, 69, 68, 8, 100, 19, 116, 25, 55, 38, 74, 13, 103, 32, 91, 29, 115, 48, 6, 23, 57, 89, 41, 107, 64, 123, 11, 2, 31, 7, 5, 63, 101, 81, 1, 44, 94, 50, 122, 104, 92, 87, 27, 28, 17, 49, 46, 37, 105, 34, 3, 118, 12, 113, 33, 71, 83, 35, 121, 22, 80, 59, 102, 14, 9, 26, 98, 15, 82, 24, 84, 10, 21], [104, 54, 63, 34, 22, 93, 122, 120, 90, 123, 62, 119, 125, 121, 126, 127, 61, 50, 88, 59, 113, 51, 118, 81, 52, 20, 16, 116, 115, 76, 55, 64, 46, 82, 57, 53, 48, 114, 117, 56, 71, 60, 73, 91, 124, 45, 69, 111, 78, 49, 39, 109, 13, 106, 110, 43, 112, 2, 58, 44, 15, 108, 1, 87, 98, 47, 74, 11, 102, 67, 42, 80, 79, 41, 29, 100, 103, 37, 105, 96, 107, 35, 14, 92, 97, 21, 28, 25, 26, 10, 85, 95, 83, 99, 32, 19, 38, 9, 101, 8, 18, 33, 30, 70, 94, 36, 12, 72, 3, 31, 65, 68, 23, 75, 24, 40, 77, 27, 6, 89, 4, 86, 66, 84, 17, 0, 7, 5], [52, 116, 37, 48, 83, 16, 77, 33, 22, 88, 11, 112, 73, 70, 4, 71, 14, 126, 61, 127, 63, 0, 2, 50, 123, 106, 1, 121, 29, 44, 51, 55, 58, 114, 60, 111, 120, 57, 125, 93, 115, 30, 118, 56, 108, 110, 91, 122, 119, 99, 53, 49, 113, 124, 59, 54, 85, 62, 105, 117, 46, 47, 109, 107, 41, 43, 40, 102, 104, 42, 90, 103, 35, 81, 74, 101, 45, 95, 38, 75, 18, 39, 98, 8, 92, 5, 36, 9, 67, 12, 25, 32, 96, 20, 97, 34, 100, 3, 65, 94, 31, 21, 78, 89, 13, 23, 69, 24, 82, 17, 84, 87, 19, 76, 15, 64, 7, 79, 10, 28, 6, 27, 80, 72, 68, 26, 66, 86], [124, 49, 101, 113, 86, 55, 32, 126, 57, 118, 29, 54, 61, 119, 121, 50, 123, 120, 26, 58, 122, 53, 115, 51, 114, 52, 63, 47, 60, 56, 125, 116, 62, 59, 111, 112, 48, 45, 108, 80, 46, 109, 44, 103, 99, 18, 127, 87, 43, 117, 78, 110, 107, 105, 37, 42, 106, 40, 102, 98, 72, 64, 104, 12, 39, 97, 73, 3, 20, 38, 84, 41, 21, 19, 36, 100, 17, 79, 94, 92, 33, 34, 25, 30, 23, 35, 85, 89, 27, 68, 81, 6, 31, 28, 24, 10, 77, 95, 14, 91, 4, 70, 93, 83, 88, 66, 1, 16, 96, 11, 5, 71, 90, 75, 0, 22, 7, 67, 13, 65, 74, 82, 15, 9, 8, 2, 76, 69], [109, 39, 98, 84, 92, 12, 17, 64, 7, 79, 14, 9, 5, 45, 3, 66, 61, 121, 41, 49, 122, 52, 86, 95, 127, 58, 21, 119, 40, 111, 124, 55, 65, 57, 118, 63, 51, 24, 23, 126, 48, 116, 25, 102, 28, 72, 30, 60, 34, 1, 22, 44, 125, 47, 0, 106, 59, 100, 18, 120, 90, 80, 82, 117, 112, 32, 97, 46, 89, 36, 29, 104, 6, 11, 19, 4, 70, 53, 26, 74, 123, 93, 77, 35, 108, 99, 85, 68, 62, 27, 50, 16, 56, 8, 107, 31, 115, 88, 114, 78, 83, 15, 91, 38, 20, 105, 94, 37, 42, 76, 13, 54, 43, 96, 110, 10, 87, 33, 73, 101, 113, 2, 71, 81, 67, 69, 75, 103]], "model.layers.16.self_attn.qk_proj": [[106, 124, 125, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 121, 45, 53, 123, 55, 119, 127, 56, 58, 98, 126, 28, 22, 39, 34, 122, 26, 84, 20, 88, 86, 79, 57, 15, 36, 59, 112, 101, 37, 40, 100, 62, 113, 118, 103, 50, 24, 30, 47, 117, 81, 85, 78, 111, 21, 115, 104, 12, 60, 29, 82, 14, 17, 76, 93, 90, 9, 92, 114, 80, 18, 16, 73, 94, 77, 110, 11, 31, 44, 83, 95, 27, 13, 46, 19, 72, 96, 108, 71, 75, 70, 25, 107, 102, 89, 64, 91, 32, 43, 97, 3, 7, 87, 23, 8, 68, 0, 41, 4, 67, 38, 6, 69, 5, 74, 105, 33, 2, 66, 10, 35, 99, 1, 65], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 121, 56, 123, 119, 127, 98, 58, 55, 53, 28, 126, 34, 39, 20, 86, 22, 118, 36, 26, 88, 84, 37, 79, 122, 47, 15, 59, 57, 111, 113, 24, 101, 112, 40, 115, 21, 100, 62, 30, 103, 85, 117, 60, 81, 29, 78, 76, 114, 14, 50, 104, 12, 92, 17, 90, 82, 93, 73, 9, 44, 77, 94, 16, 31, 18, 108, 80, 11, 72, 27, 46, 83, 13, 95, 96, 70, 107, 75, 110, 89, 19, 25, 7, 64, 102, 0, 87, 91, 38, 67, 8, 32, 71, 97, 43, 4, 68, 23, 5, 69, 41, 3, 105, 2, 74, 66, 35, 33, 1, 99, 6, 65, 10], [106, 125, 124, 116, 61, 52, 109, 42, 49, 63, 51, 54, 120, 48, 45, 121, 119, 123, 55, 98, 126, 56, 127, 58, 53, 28, 34, 39, 88, 36, 101, 122, 86, 84, 26, 100, 22, 20, 47, 113, 118, 57, 62, 79, 24, 59, 112, 85, 15, 40, 37, 103, 81, 50, 111, 104, 21, 60, 117, 29, 115, 30, 114, 93, 14, 90, 78, 12, 94, 17, 18, 92, 9, 27, 73, 44, 82, 76, 64, 31, 102, 110, 13, 95, 80, 83, 16, 96, 91, 11, 0, 77, 46, 25, 108, 72, 70, 19, 32, 107, 38, 75, 7, 71, 87, 4, 68, 3, 89, 8, 97, 41, 23, 1, 66, 69, 105, 67, 2, 65, 33, 35, 43, 5, 74, 99, 6, 10], [106, 125, 124, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 123, 119, 56, 127, 58, 98, 55, 126, 28, 39, 59, 62, 57, 22, 20, 79, 101, 118, 34, 36, 26, 84, 100, 47, 113, 122, 86, 37, 24, 115, 112, 111, 40, 117, 15, 50, 30, 88, 103, 85, 104, 81, 29, 21, 14, 78, 60, 90, 92, 114, 93, 17, 94, 76, 27, 9, 73, 18, 0, 12, 44, 82, 80, 110, 16, 83, 13, 96, 77, 46, 108, 102, 70, 31, 19, 11, 107, 95, 72, 75, 7, 25, 97, 91, 38, 67, 87, 89, 71, 68, 8, 4, 41, 43, 64, 3, 2, 32, 66, 69, 23, 1, 5, 105, 33, 65, 35, 10, 99, 74, 6], [106, 124, 125, 116, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 121, 56, 123, 119, 53, 127, 126, 58, 55, 98, 57, 20, 22, 34, 39, 86, 47, 122, 59, 101, 118, 28, 112, 79, 62, 50, 84, 15, 40, 100, 88, 26, 36, 103, 111, 24, 117, 114, 81, 37, 30, 115, 85, 104, 14, 17, 113, 18, 78, 12, 93, 29, 60, 44, 76, 27, 73, 9, 90, 21, 94, 11, 80, 92, 13, 83, 68, 82, 107, 110, 77, 16, 46, 108, 64, 19, 31, 102, 72, 95, 0, 96, 25, 7, 8, 71, 4, 70, 67, 75, 41, 3, 91, 38, 69, 87, 23, 89, 2, 43, 32, 97, 1, 5, 33, 66, 6, 105, 35, 65, 99, 74, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 119, 123, 58, 127, 56, 126, 53, 98, 55, 20, 26, 22, 34, 47, 79, 57, 39, 84, 86, 62, 118, 59, 15, 103, 101, 88, 28, 36, 100, 111, 81, 24, 112, 122, 37, 50, 85, 30, 40, 60, 78, 117, 114, 14, 17, 29, 76, 104, 93, 27, 12, 21, 80, 82, 18, 73, 90, 16, 11, 94, 113, 13, 9, 92, 77, 44, 83, 115, 31, 19, 46, 110, 108, 107, 75, 96, 89, 25, 72, 8, 95, 7, 64, 23, 71, 6, 32, 91, 102, 0, 41, 97, 43, 69, 68, 87, 4, 67, 3, 70, 105, 5, 2, 35, 38, 66, 65, 33, 74, 99, 1, 10], [106, 125, 124, 116, 61, 52, 63, 109, 42, 49, 51, 120, 54, 48, 45, 119, 121, 127, 53, 56, 123, 126, 58, 98, 55, 20, 26, 57, 122, 84, 86, 22, 88, 34, 79, 118, 36, 47, 62, 28, 39, 81, 15, 24, 112, 101, 103, 37, 50, 59, 40, 30, 114, 100, 111, 85, 14, 78, 115, 90, 18, 117, 12, 17, 104, 29, 93, 27, 80, 21, 60, 113, 44, 76, 73, 94, 9, 82, 16, 83, 11, 92, 13, 77, 110, 107, 95, 19, 25, 31, 96, 108, 46, 41, 43, 75, 71, 89, 23, 102, 97, 6, 91, 32, 8, 7, 0, 72, 38, 3, 33, 67, 87, 64, 2, 69, 4, 35, 68, 74, 105, 5, 99, 66, 70, 10, 65, 1], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 51, 54, 120, 48, 45, 119, 121, 127, 53, 56, 123, 55, 126, 98, 58, 22, 20, 26, 28, 84, 39, 86, 122, 101, 34, 57, 88, 36, 24, 100, 79, 15, 30, 81, 103, 37, 40, 112, 111, 62, 59, 85, 115, 47, 50, 118, 14, 113, 60, 117, 90, 17, 78, 93, 21, 114, 29, 104, 92, 80, 18, 12, 27, 16, 82, 73, 9, 13, 25, 76, 83, 95, 96, 44, 94, 77, 46, 107, 32, 11, 43, 91, 110, 102, 6, 19, 8, 31, 89, 0, 108, 75, 71, 64, 23, 41, 7, 97, 68, 105, 87, 33, 38, 72, 67, 3, 66, 4, 99, 35, 69, 2, 74, 5, 65, 1, 70, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 51, 49, 54, 120, 48, 45, 127, 119, 121, 53, 98, 56, 55, 58, 34, 28, 123, 22, 20, 39, 126, 84, 101, 26, 88, 103, 86, 24, 100, 122, 36, 37, 79, 85, 112, 50, 111, 15, 30, 47, 57, 115, 113, 59, 60, 40, 62, 118, 17, 81, 78, 114, 14, 12, 21, 93, 90, 104, 117, 29, 18, 27, 76, 82, 80, 44, 108, 92, 16, 9, 73, 95, 13, 25, 77, 94, 83, 46, 96, 102, 19, 11, 8, 31, 107, 43, 75, 6, 91, 89, 0, 71, 110, 32, 23, 87, 64, 7, 97, 33, 41, 2, 38, 4, 72, 105, 67, 68, 66, 3, 35, 5, 69, 99, 1, 10, 74, 65, 70], [106, 124, 125, 61, 116, 52, 109, 42, 63, 49, 51, 54, 48, 120, 45, 121, 127, 119, 53, 56, 123, 55, 98, 58, 126, 34, 28, 101, 20, 57, 59, 84, 22, 26, 47, 115, 39, 103, 118, 36, 37, 122, 88, 50, 86, 100, 79, 111, 15, 24, 117, 113, 30, 40, 112, 17, 85, 62, 21, 12, 81, 78, 104, 60, 90, 44, 14, 9, 93, 29, 114, 18, 76, 92, 82, 16, 8, 94, 110, 27, 80, 73, 13, 83, 108, 46, 95, 96, 11, 75, 77, 6, 102, 7, 0, 64, 25, 19, 31, 32, 107, 71, 89, 3, 72, 38, 43, 4, 97, 23, 70, 91, 5, 33, 41, 68, 67, 2, 87, 66, 69, 99, 35, 74, 105, 1, 65, 10], [106, 124, 116, 125, 61, 52, 109, 42, 63, 49, 54, 51, 120, 48, 121, 45, 127, 53, 123, 119, 56, 58, 98, 55, 126, 122, 118, 57, 28, 34, 20, 15, 84, 39, 79, 22, 59, 86, 101, 26, 100, 50, 81, 88, 36, 113, 85, 37, 47, 111, 24, 40, 30, 115, 62, 60, 17, 112, 9, 12, 14, 78, 103, 117, 82, 21, 76, 73, 29, 104, 8, 18, 92, 90, 114, 93, 16, 27, 44, 94, 11, 0, 13, 80, 77, 83, 31, 64, 19, 68, 110, 7, 108, 75, 71, 70, 96, 107, 72, 25, 46, 4, 91, 67, 102, 95, 32, 89, 3, 87, 6, 2, 66, 1, 69, 5, 23, 65, 38, 97, 41, 33, 35, 105, 43, 74, 10, 99], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 127, 53, 119, 58, 56, 126, 98, 123, 122, 86, 84, 22, 34, 20, 26, 57, 15, 55, 118, 28, 39, 79, 62, 47, 100, 101, 30, 50, 40, 88, 36, 37, 85, 103, 81, 112, 111, 59, 24, 17, 76, 115, 113, 14, 21, 60, 78, 117, 90, 114, 29, 82, 104, 12, 18, 9, 73, 16, 93, 27, 92, 94, 31, 80, 83, 44, 13, 77, 0, 110, 11, 95, 8, 19, 71, 46, 70, 102, 96, 108, 75, 91, 7, 72, 89, 68, 87, 4, 32, 64, 107, 25, 38, 67, 41, 66, 3, 43, 23, 97, 5, 69, 33, 2, 99, 6, 105, 10, 65, 35, 74, 1], [106, 124, 116, 125, 61, 52, 63, 109, 49, 42, 51, 120, 54, 48, 121, 45, 119, 53, 127, 56, 58, 123, 98, 126, 20, 26, 101, 39, 34, 22, 28, 118, 86, 37, 47, 57, 122, 88, 84, 114, 112, 79, 40, 55, 103, 62, 30, 59, 15, 36, 24, 100, 85, 50, 60, 113, 21, 117, 111, 115, 90, 81, 29, 93, 78, 104, 17, 82, 94, 92, 14, 18, 76, 95, 27, 44, 46, 12, 16, 9, 25, 108, 110, 31, 96, 19, 77, 13, 73, 83, 102, 70, 80, 43, 11, 107, 75, 91, 71, 87, 32, 89, 7, 41, 38, 72, 23, 97, 8, 99, 35, 0, 33, 67, 105, 5, 3, 68, 4, 64, 69, 66, 2, 74, 1, 10, 6, 65], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 119, 53, 58, 123, 127, 126, 98, 56, 55, 20, 22, 28, 39, 57, 34, 101, 112, 118, 47, 122, 26, 79, 100, 37, 88, 59, 84, 86, 50, 36, 15, 62, 85, 30, 24, 115, 40, 114, 103, 81, 117, 104, 111, 60, 78, 113, 14, 12, 93, 76, 17, 82, 21, 29, 16, 73, 18, 90, 94, 9, 92, 11, 27, 70, 80, 44, 77, 64, 46, 83, 19, 72, 110, 71, 96, 31, 108, 75, 89, 0, 8, 13, 95, 25, 7, 107, 91, 97, 102, 2, 32, 43, 41, 67, 4, 87, 38, 68, 66, 23, 35, 3, 69, 65, 99, 5, 33, 10, 1, 105, 74, 6], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 54, 49, 120, 48, 45, 121, 119, 53, 58, 127, 98, 55, 123, 126, 56, 20, 39, 22, 28, 34, 101, 15, 84, 118, 122, 47, 88, 86, 26, 79, 57, 37, 100, 30, 59, 36, 62, 24, 40, 112, 111, 115, 85, 81, 117, 50, 17, 103, 9, 78, 104, 29, 76, 93, 114, 14, 12, 113, 60, 90, 92, 94, 21, 82, 16, 73, 11, 18, 31, 80, 77, 27, 64, 46, 108, 75, 72, 19, 110, 83, 71, 44, 96, 25, 70, 102, 91, 8, 13, 41, 7, 107, 95, 4, 43, 87, 89, 67, 0, 68, 32, 3, 2, 97, 65, 38, 5, 33, 66, 69, 74, 6, 23, 35, 99, 10, 105, 1], [106, 125, 116, 124, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 119, 53, 127, 56, 123, 55, 58, 98, 126, 39, 20, 118, 86, 122, 22, 28, 84, 101, 57, 26, 15, 37, 30, 79, 34, 112, 36, 47, 59, 62, 88, 50, 111, 100, 103, 81, 85, 60, 113, 24, 78, 117, 115, 40, 76, 114, 14, 12, 9, 29, 104, 93, 73, 90, 17, 18, 77, 82, 21, 94, 44, 108, 16, 31, 92, 96, 80, 110, 11, 83, 19, 46, 75, 13, 27, 72, 0, 107, 71, 7, 102, 8, 91, 70, 87, 43, 41, 64, 4, 25, 68, 23, 95, 3, 67, 97, 89, 32, 38, 6, 69, 2, 33, 5, 66, 74, 65, 105, 35, 1, 99, 10], [106, 124, 125, 116, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 119, 127, 98, 58, 126, 123, 56, 20, 26, 57, 122, 28, 101, 39, 22, 55, 36, 86, 84, 37, 34, 50, 112, 59, 15, 79, 100, 30, 114, 40, 88, 47, 115, 24, 62, 81, 118, 60, 21, 103, 85, 90, 14, 93, 104, 29, 111, 17, 117, 78, 94, 82, 16, 110, 76, 44, 113, 92, 108, 18, 9, 12, 27, 96, 77, 80, 19, 102, 25, 46, 83, 95, 73, 31, 75, 43, 13, 107, 91, 11, 72, 38, 41, 89, 32, 97, 6, 87, 64, 71, 7, 23, 35, 33, 0, 99, 66, 8, 5, 105, 67, 3, 68, 4, 69, 70, 74, 2, 65, 1, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 51, 49, 54, 120, 48, 45, 121, 127, 119, 53, 98, 123, 58, 56, 126, 28, 39, 101, 55, 122, 26, 20, 22, 100, 86, 57, 36, 34, 59, 118, 84, 40, 15, 88, 37, 30, 117, 103, 79, 50, 24, 113, 111, 112, 62, 114, 14, 17, 60, 85, 29, 115, 81, 21, 47, 93, 44, 76, 104, 90, 95, 16, 82, 9, 27, 94, 78, 92, 73, 18, 12, 80, 108, 75, 77, 96, 31, 6, 110, 83, 72, 13, 0, 107, 46, 102, 11, 19, 91, 7, 43, 25, 87, 32, 64, 38, 97, 89, 41, 71, 23, 33, 8, 105, 2, 4, 67, 69, 3, 68, 35, 66, 74, 99, 65, 5, 1, 70, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 54, 51, 120, 48, 45, 53, 121, 119, 127, 56, 98, 58, 123, 101, 126, 34, 36, 55, 39, 118, 40, 59, 28, 20, 50, 103, 86, 115, 37, 111, 26, 100, 57, 22, 84, 24, 122, 15, 47, 88, 113, 30, 114, 62, 112, 79, 117, 104, 44, 60, 21, 81, 29, 14, 108, 78, 90, 82, 94, 76, 17, 85, 93, 92, 73, 16, 110, 18, 9, 12, 27, 102, 31, 46, 41, 77, 83, 80, 107, 75, 6, 19, 72, 95, 96, 89, 87, 8, 13, 43, 7, 11, 38, 97, 64, 0, 105, 25, 91, 23, 32, 71, 4, 33, 3, 35, 99, 69, 67, 68, 5, 66, 2, 74, 1, 65, 10, 70], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 127, 98, 58, 56, 55, 20, 39, 126, 22, 28, 57, 59, 37, 86, 34, 101, 84, 36, 118, 15, 122, 100, 30, 79, 26, 47, 103, 111, 40, 115, 24, 88, 117, 112, 113, 50, 90, 85, 81, 114, 29, 21, 14, 94, 92, 62, 93, 60, 12, 104, 78, 76, 82, 44, 27, 110, 17, 73, 80, 108, 9, 31, 77, 18, 75, 96, 83, 16, 13, 6, 95, 25, 72, 91, 11, 46, 19, 89, 102, 7, 107, 8, 0, 41, 23, 87, 32, 64, 71, 43, 105, 38, 35, 97, 33, 67, 69, 3, 68, 4, 5, 1, 99, 65, 74, 2, 66, 10, 70], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 121, 53, 119, 123, 127, 58, 56, 98, 57, 126, 22, 86, 59, 122, 34, 28, 20, 15, 100, 39, 84, 101, 26, 55, 103, 36, 40, 118, 88, 113, 37, 79, 47, 62, 114, 24, 78, 30, 117, 111, 115, 112, 50, 85, 81, 90, 76, 29, 93, 60, 17, 27, 14, 92, 9, 94, 104, 12, 44, 18, 73, 21, 82, 80, 108, 110, 83, 31, 16, 75, 13, 95, 46, 77, 96, 91, 19, 107, 89, 72, 11, 0, 25, 43, 102, 6, 7, 8, 41, 64, 87, 71, 32, 23, 97, 38, 3, 68, 33, 5, 67, 35, 4, 66, 105, 1, 74, 70, 69, 2, 65, 99, 10], [106, 124, 125, 116, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 45, 121, 123, 53, 119, 127, 98, 58, 56, 126, 59, 28, 57, 39, 55, 34, 101, 26, 22, 20, 122, 84, 15, 36, 103, 86, 47, 88, 100, 115, 113, 118, 24, 79, 112, 62, 30, 37, 114, 111, 40, 117, 85, 21, 104, 90, 78, 76, 81, 29, 93, 14, 17, 110, 108, 50, 60, 18, 27, 94, 44, 92, 82, 9, 16, 73, 80, 83, 13, 12, 25, 0, 31, 77, 46, 102, 107, 64, 95, 19, 7, 8, 96, 75, 11, 91, 97, 38, 43, 72, 71, 87, 70, 4, 89, 67, 32, 3, 23, 6, 68, 2, 33, 66, 41, 5, 65, 1, 35, 69, 105, 99, 74, 10], [106, 124, 116, 125, 61, 52, 63, 49, 109, 42, 51, 54, 120, 48, 45, 53, 119, 121, 123, 58, 127, 98, 56, 126, 57, 122, 34, 86, 84, 101, 26, 22, 36, 28, 20, 39, 59, 55, 15, 100, 114, 103, 115, 88, 47, 37, 30, 62, 111, 113, 24, 85, 40, 79, 90, 112, 118, 29, 81, 50, 60, 117, 104, 76, 14, 78, 93, 17, 21, 94, 92, 82, 18, 44, 27, 80, 9, 110, 73, 77, 31, 16, 95, 108, 12, 11, 83, 13, 25, 46, 107, 19, 102, 64, 70, 96, 91, 8, 43, 87, 41, 7, 75, 4, 71, 32, 38, 97, 72, 68, 33, 89, 0, 35, 23, 3, 2, 1, 65, 105, 5, 69, 67, 66, 74, 99, 10, 6], [106, 124, 116, 125, 61, 52, 63, 109, 42, 49, 51, 54, 120, 48, 45, 53, 121, 119, 123, 58, 98, 127, 56, 126, 57, 59, 55, 28, 34, 26, 22, 103, 101, 86, 62, 84, 39, 118, 36, 122, 79, 88, 20, 30, 15, 117, 100, 37, 112, 40, 81, 111, 113, 114, 24, 47, 29, 115, 93, 50, 85, 90, 21, 76, 78, 14, 104, 94, 12, 92, 17, 27, 44, 82, 73, 60, 18, 9, 31, 80, 108, 70, 95, 46, 110, 16, 19, 75, 77, 8, 13, 11, 83, 96, 91, 25, 89, 102, 32, 72, 107, 7, 97, 87, 43, 38, 23, 35, 71, 41, 3, 0, 105, 33, 4, 68, 64, 74, 69, 67, 5, 66, 2, 10, 65, 1, 99, 6], [106, 125, 116, 124, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 119, 53, 58, 98, 127, 56, 123, 126, 122, 26, 34, 59, 20, 55, 28, 84, 86, 22, 39, 100, 88, 15, 101, 62, 57, 118, 36, 103, 47, 79, 50, 37, 85, 40, 30, 112, 81, 117, 24, 114, 111, 14, 115, 113, 60, 76, 104, 110, 17, 93, 18, 90, 44, 78, 21, 80, 108, 82, 73, 94, 29, 12, 31, 9, 92, 27, 70, 46, 16, 77, 95, 83, 13, 19, 11, 107, 75, 8, 96, 25, 102, 71, 7, 89, 41, 23, 0, 91, 64, 87, 32, 38, 72, 43, 33, 67, 97, 68, 3, 66, 4, 5, 105, 69, 35, 74, 1, 2, 10, 65, 99, 6], [106, 116, 124, 125, 61, 52, 63, 109, 49, 42, 51, 54, 120, 48, 45, 121, 53, 98, 119, 56, 127, 123, 122, 58, 55, 59, 26, 20, 126, 28, 86, 22, 101, 100, 88, 84, 34, 39, 15, 115, 36, 57, 40, 79, 62, 37, 30, 85, 113, 118, 50, 44, 117, 24, 111, 112, 103, 60, 47, 81, 114, 93, 29, 108, 21, 94, 78, 104, 76, 17, 90, 14, 110, 12, 92, 9, 18, 46, 73, 82, 80, 77, 102, 31, 95, 27, 16, 83, 11, 25, 8, 96, 13, 19, 107, 75, 91, 70, 41, 105, 71, 89, 23, 0, 7, 43, 38, 72, 87, 67, 32, 97, 64, 35, 33, 68, 4, 99, 2, 74, 5, 66, 69, 3, 1, 6, 65, 10], [106, 124, 116, 125, 61, 52, 109, 63, 42, 49, 51, 54, 120, 48, 121, 45, 123, 127, 55, 53, 98, 56, 122, 119, 126, 40, 58, 26, 28, 34, 86, 60, 57, 39, 20, 88, 101, 115, 118, 36, 22, 37, 100, 59, 103, 84, 15, 47, 50, 79, 62, 117, 85, 30, 104, 111, 24, 90, 112, 114, 93, 81, 44, 78, 113, 29, 12, 76, 17, 9, 21, 18, 14, 82, 31, 102, 27, 95, 108, 16, 96, 110, 73, 92, 80, 46, 94, 77, 83, 8, 19, 75, 41, 11, 13, 25, 64, 89, 91, 23, 7, 71, 32, 38, 97, 0, 107, 43, 67, 68, 33, 72, 6, 3, 87, 105, 4, 70, 66, 2, 69, 35, 74, 65, 5, 99, 1, 10], [106, 124, 125, 116, 61, 52, 63, 109, 42, 49, 54, 51, 120, 48, 45, 121, 123, 55, 56, 53, 119, 58, 98, 127, 122, 28, 126, 34, 39, 20, 57, 101, 59, 84, 118, 79, 26, 22, 88, 100, 15, 103, 86, 36, 47, 117, 37, 111, 62, 115, 112, 40, 81, 85, 113, 30, 78, 24, 17, 50, 76, 60, 93, 104, 18, 29, 9, 12, 82, 21, 14, 73, 90, 114, 110, 27, 44, 92, 94, 16, 8, 77, 19, 75, 13, 80, 0, 108, 6, 83, 11, 46, 95, 31, 7, 64, 91, 102, 71, 96, 72, 3, 25, 23, 97, 41, 107, 89, 32, 5, 4, 87, 67, 66, 38, 35, 2, 105, 65, 69, 43, 68, 33, 74, 70, 99, 1, 10], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 54, 51, 120, 48, 45, 121, 53, 119, 123, 55, 56, 58, 98, 127, 39, 34, 126, 57, 101, 28, 118, 59, 22, 26, 115, 84, 20, 100, 37, 86, 36, 122, 88, 15, 103, 111, 47, 85, 79, 40, 117, 113, 30, 50, 62, 60, 24, 78, 112, 104, 44, 93, 29, 90, 12, 81, 110, 92, 76, 9, 17, 14, 18, 73, 21, 27, 114, 82, 94, 8, 80, 31, 19, 46, 6, 13, 64, 77, 108, 95, 83, 11, 16, 7, 75, 96, 102, 25, 72, 0, 71, 32, 89, 4, 91, 38, 41, 23, 107, 87, 2, 105, 3, 69, 1, 5, 67, 66, 97, 33, 68, 43, 65, 74, 99, 35, 10, 70], [106, 116, 125, 124, 61, 52, 109, 63, 49, 42, 51, 54, 120, 48, 121, 45, 53, 119, 55, 58, 98, 56, 123, 126, 39, 59, 20, 26, 127, 22, 34, 84, 15, 86, 28, 57, 118, 88, 101, 36, 122, 37, 79, 100, 62, 115, 117, 111, 81, 103, 50, 78, 60, 30, 47, 85, 40, 113, 12, 24, 14, 17, 104, 76, 9, 93, 73, 29, 21, 112, 44, 18, 27, 90, 80, 82, 92, 16, 6, 94, 114, 13, 46, 83, 19, 75, 11, 77, 110, 108, 8, 96, 64, 31, 95, 7, 71, 3, 87, 25, 91, 32, 72, 107, 4, 68, 102, 67, 89, 41, 5, 66, 105, 97, 23, 43, 38, 69, 33, 10, 0, 2, 74, 35, 99, 1, 65, 70], [106, 125, 124, 116, 61, 52, 49, 109, 63, 42, 51, 54, 120, 48, 53, 121, 45, 123, 58, 127, 119, 56, 98, 126, 59, 122, 39, 55, 26, 118, 28, 34, 100, 62, 57, 15, 101, 22, 20, 37, 47, 36, 86, 84, 88, 40, 79, 60, 50, 112, 115, 117, 111, 103, 81, 30, 24, 85, 113, 104, 14, 73, 78, 29, 93, 21, 12, 76, 9, 80, 44, 90, 17, 18, 94, 82, 27, 92, 114, 11, 6, 96, 46, 13, 31, 16, 72, 110, 83, 0, 19, 77, 7, 75, 8, 64, 43, 71, 108, 95, 91, 68, 89, 25, 32, 67, 4, 38, 5, 102, 3, 97, 66, 87, 23, 33, 70, 107, 2, 69, 105, 99, 41, 1, 35, 65, 10, 74], [106, 124, 125, 116, 61, 52, 42, 109, 63, 49, 51, 120, 54, 48, 121, 45, 53, 123, 126, 98, 58, 119, 86, 59, 127, 56, 57, 26, 55, 22, 84, 79, 28, 39, 20, 15, 115, 34, 101, 40, 88, 36, 62, 37, 111, 122, 118, 117, 60, 112, 24, 30, 100, 103, 47, 81, 85, 50, 14, 12, 78, 113, 29, 104, 21, 76, 93, 90, 73, 9, 17, 18, 114, 82, 92, 11, 16, 13, 94, 80, 27, 95, 77, 46, 44, 31, 19, 72, 83, 75, 110, 108, 107, 96, 89, 7, 25, 6, 43, 71, 91, 102, 32, 8, 41, 97, 70, 23, 105, 35, 68, 87, 67, 38, 3, 5, 64, 4, 0, 69, 10, 74, 33, 99, 2, 66, 1, 65]], "model.layers.17.self_attn.q_proj": [[60, 106, 114, 100, 29, 89, 115, 123, 93, 86, 27, 83, 120, 47, 80, 42, 22, 112, 11, 57, 21, 113, 122, 53, 62, 127, 50, 16, 46, 61, 118, 59, 32, 111, 126, 45, 125, 56, 14, 99, 110, 63, 119, 91, 116, 81, 69, 55, 108, 96, 54, 26, 48, 117, 44, 51, 52, 121, 124, 58, 37, 97, 49, 109, 43, 105, 24, 76, 38, 41, 88, 71, 75, 95, 5, 107, 102, 103, 39, 104, 87, 31, 20, 40, 36, 15, 9, 101, 79, 35, 34, 98, 84, 17, 28, 10, 2, 94, 19, 90, 85, 8, 3, 23, 30, 33, 0, 7, 12, 1, 25, 4, 82, 6, 64, 18, 66, 77, 13, 78, 92, 74, 65, 67, 73, 72, 70, 68], [114, 106, 100, 60, 29, 89, 93, 50, 42, 86, 120, 59, 44, 61, 83, 22, 80, 115, 87, 99, 118, 27, 32, 105, 119, 81, 47, 31, 95, 46, 43, 21, 121, 97, 17, 25, 63, 91, 48, 54, 56, 116, 113, 51, 57, 14, 24, 11, 111, 103, 117, 96, 45, 53, 107, 125, 62, 110, 76, 58, 122, 41, 19, 38, 126, 112, 101, 37, 127, 40, 94, 55, 52, 16, 124, 88, 20, 102, 104, 98, 109, 33, 85, 108, 123, 71, 8, 23, 49, 35, 39, 90, 92, 30, 82, 79, 34, 9, 26, 15, 5, 10, 28, 1, 13, 36, 65, 84, 69, 77, 18, 12, 74, 75, 0, 3, 66, 6, 67, 72, 4, 70, 64, 7, 78, 73, 68, 2], [106, 60, 114, 100, 50, 29, 89, 86, 42, 57, 62, 107, 59, 93, 22, 80, 83, 56, 116, 37, 11, 14, 63, 47, 124, 99, 115, 91, 111, 46, 51, 71, 123, 52, 69, 48, 97, 110, 44, 125, 27, 55, 61, 49, 9, 16, 24, 113, 32, 76, 53, 58, 105, 81, 104, 120, 126, 119, 54, 31, 117, 127, 118, 122, 41, 38, 40, 45, 102, 121, 20, 108, 79, 103, 43, 109, 112, 19, 101, 94, 98, 39, 28, 35, 26, 33, 95, 96, 5, 88, 23, 25, 2, 30, 34, 12, 90, 82, 92, 21, 75, 85, 67, 1, 0, 4, 36, 17, 6, 18, 84, 74, 87, 13, 77, 8, 15, 10, 7, 72, 70, 3, 66, 78, 65, 64, 73, 68], [106, 60, 100, 115, 114, 89, 29, 83, 14, 86, 93, 42, 11, 71, 80, 27, 9, 50, 22, 25, 69, 110, 2, 67, 76, 1, 91, 4, 24, 62, 0, 16, 6, 53, 61, 116, 57, 3, 111, 37, 85, 21, 26, 97, 122, 113, 125, 7, 63, 118, 51, 127, 90, 77, 119, 19, 55, 35, 47, 17, 105, 123, 117, 75, 112, 52, 81, 95, 54, 64, 43, 108, 44, 120, 48, 78, 49, 18, 87, 46, 13, 88, 59, 98, 101, 96, 38, 92, 20, 45, 34, 40, 58, 121, 104, 68, 84, 124, 23, 103, 28, 94, 79, 102, 10, 107, 31, 73, 65, 70, 32, 41, 126, 56, 39, 33, 12, 99, 72, 109, 36, 74, 30, 15, 8, 82, 66, 5], [103, 126, 29, 82, 99, 85, 15, 76, 73, 71, 112, 90, 3, 5, 24, 105, 35, 93, 96, 97, 57, 23, 120, 87, 21, 61, 54, 18, 81, 119, 84, 17, 33, 26, 80, 49, 67, 115, 106, 64, 116, 83, 88, 50, 79, 16, 111, 78, 28, 101, 19, 12, 9, 121, 13, 127, 1, 69, 114, 25, 65, 34, 91, 7, 122, 89, 118, 74, 8, 32, 53, 110, 62, 20, 86, 14, 104, 11, 92, 4, 98, 117, 63, 40, 59, 51, 102, 10, 22, 27, 55, 31, 66, 41, 38, 77, 36, 43, 56, 123, 94, 58, 125, 100, 52, 70, 47, 107, 39, 42, 95, 46, 60, 6, 44, 75, 37, 30, 124, 109, 72, 68, 48, 113, 108, 45, 2, 0], [103, 126, 29, 99, 15, 85, 82, 76, 73, 71, 112, 3, 5, 90, 96, 24, 26, 1, 16, 27, 21, 9, 114, 119, 67, 57, 64, 84, 74, 11, 19, 93, 78, 79, 81, 23, 120, 88, 65, 14, 97, 101, 61, 91, 49, 33, 18, 98, 104, 105, 116, 6, 50, 43, 87, 8, 69, 92, 55, 32, 106, 7, 2, 30, 121, 94, 117, 13, 34, 125, 12, 54, 25, 56, 28, 80, 70, 75, 41, 68, 66, 127, 83, 36, 20, 115, 110, 10, 22, 17, 102, 100, 63, 122, 4, 51, 118, 39, 86, 58, 59, 113, 0, 108, 111, 35, 62, 77, 124, 60, 37, 72, 46, 44, 45, 89, 107, 31, 95, 53, 109, 48, 42, 123, 40, 38, 47, 52], [103, 126, 35, 49, 112, 29, 90, 114, 99, 56, 26, 85, 88, 24, 120, 93, 82, 116, 84, 119, 121, 96, 115, 46, 86, 63, 55, 117, 41, 57, 15, 60, 54, 110, 43, 32, 111, 122, 106, 62, 74, 59, 127, 101, 76, 16, 125, 52, 61, 104, 50, 118, 22, 124, 102, 123, 58, 107, 105, 108, 47, 51, 53, 97, 23, 73, 71, 40, 34, 113, 109, 77, 2, 48, 27, 42, 36, 11, 83, 0, 21, 37, 19, 44, 33, 18, 45, 98, 20, 38, 14, 100, 5, 8, 3, 95, 4, 78, 25, 66, 79, 91, 75, 39, 94, 68, 92, 80, 31, 28, 89, 72, 67, 87, 30, 1, 13, 81, 10, 64, 17, 69, 7, 65, 70, 12, 9, 6], [103, 126, 29, 85, 24, 82, 76, 90, 15, 112, 93, 99, 120, 26, 73, 71, 74, 35, 88, 127, 119, 41, 98, 21, 23, 3, 115, 5, 13, 111, 116, 121, 78, 57, 54, 96, 92, 16, 62, 49, 34, 63, 61, 91, 36, 14, 114, 55, 60, 104, 84, 117, 25, 30, 56, 32, 18, 33, 118, 106, 27, 50, 83, 28, 53, 51, 31, 79, 110, 86, 67, 52, 43, 87, 97, 17, 122, 19, 105, 101, 8, 47, 125, 12, 42, 102, 81, 46, 58, 20, 22, 77, 107, 108, 7, 123, 89, 48, 1, 95, 40, 94, 59, 38, 113, 124, 11, 100, 80, 44, 37, 45, 68, 72, 9, 109, 64, 10, 65, 66, 75, 39, 4, 6, 69, 70, 0, 2], [45, 109, 55, 58, 111, 59, 57, 63, 124, 52, 47, 116, 46, 125, 115, 119, 61, 53, 114, 48, 62, 51, 113, 38, 123, 60, 126, 50, 54, 117, 121, 112, 102, 122, 106, 49, 118, 120, 101, 127, 107, 36, 110, 42, 56, 43, 108, 33, 44, 28, 105, 29, 104, 41, 40, 103, 26, 39, 92, 99, 97, 37, 96, 98, 87, 34, 31, 100, 21, 35, 86, 32, 91, 30, 23, 95, 24, 18, 94, 89, 90, 27, 12, 84, 93, 25, 85, 78, 88, 22, 20, 82, 81, 17, 80, 7, 15, 11, 76, 9, 14, 79, 73, 16, 19, 83, 71, 75, 13, 69, 4, 68, 70, 74, 5, 2, 77, 65, 66, 8, 3, 10, 6, 1, 0, 64, 67, 72], [58, 45, 38, 47, 94, 55, 59, 89, 95, 60, 124, 112, 109, 111, 33, 57, 52, 115, 28, 35, 127, 87, 46, 113, 61, 92, 123, 51, 117, 122, 49, 118, 116, 63, 53, 54, 125, 101, 96, 50, 119, 121, 62, 105, 126, 110, 48, 41, 114, 120, 107, 86, 106, 43, 42, 44, 40, 108, 104, 56, 85, 36, 16, 37, 19, 98, 82, 103, 23, 20, 39, 12, 34, 84, 18, 91, 100, 15, 93, 81, 32, 22, 27, 99, 97, 102, 9, 11, 31, 29, 24, 78, 17, 25, 90, 7, 26, 88, 21, 71, 30, 79, 75, 14, 73, 76, 4, 77, 69, 5, 2, 13, 83, 68, 74, 1, 80, 66, 65, 10, 64, 0, 6, 8, 70, 67, 3, 72], [45, 109, 47, 52, 38, 101, 53, 60, 58, 94, 61, 115, 92, 59, 62, 46, 126, 23, 34, 116, 28, 111, 51, 119, 118, 108, 87, 33, 54, 57, 50, 112, 110, 63, 49, 85, 127, 123, 113, 117, 48, 30, 42, 44, 114, 89, 21, 41, 122, 103, 36, 40, 102, 55, 121, 120, 43, 35, 39, 107, 125, 37, 106, 104, 29, 105, 124, 99, 95, 100, 56, 93, 27, 96, 26, 97, 31, 78, 12, 32, 25, 24, 98, 16, 80, 18, 9, 88, 91, 86, 73, 82, 90, 81, 7, 20, 22, 84, 17, 11, 76, 83, 79, 19, 14, 75, 15, 69, 77, 71, 6, 74, 66, 10, 68, 4, 5, 2, 70, 67, 13, 0, 3, 72, 65, 64, 1, 8], [45, 38, 58, 19, 89, 94, 13, 59, 16, 74, 109, 35, 70, 87, 8, 3, 47, 6, 33, 67, 95, 30, 52, 111, 0, 55, 64, 72, 124, 1, 85, 77, 92, 84, 17, 65, 88, 69, 15, 10, 5, 81, 23, 21, 83, 82, 80, 28, 22, 20, 27, 78, 115, 25, 4, 31, 66, 76, 9, 11, 90, 32, 101, 2, 18, 75, 79, 37, 14, 96, 57, 126, 71, 63, 73, 12, 26, 68, 7, 29, 86, 98, 34, 24, 39, 91, 93, 108, 50, 125, 122, 61, 62, 105, 44, 104, 103, 116, 60, 100, 118, 121, 36, 97, 43, 99, 112, 120, 117, 51, 54, 40, 49, 56, 107, 123, 42, 106, 41, 113, 48, 53, 110, 119, 114, 127, 46, 102], [40, 50, 109, 98, 126, 29, 51, 84, 23, 80, 13, 18, 71, 73, 53, 75, 26, 56, 47, 90, 104, 2, 87, 66, 4, 111, 72, 93, 69, 68, 112, 39, 86, 94, 70, 1, 20, 0, 120, 11, 22, 19, 65, 74, 30, 103, 101, 64, 5, 25, 96, 59, 43, 119, 60, 16, 67, 89, 34, 76, 31, 77, 8, 36, 95, 38, 78, 117, 57, 113, 108, 100, 32, 6, 62, 121, 27, 45, 105, 124, 127, 7, 102, 21, 88, 33, 55, 97, 48, 15, 85, 99, 41, 118, 44, 28, 110, 83, 91, 35, 92, 58, 63, 10, 52, 12, 61, 46, 42, 17, 24, 115, 116, 82, 3, 123, 114, 81, 37, 107, 14, 9, 54, 122, 125, 49, 106, 79], [40, 126, 29, 98, 109, 87, 47, 39, 50, 23, 93, 84, 20, 75, 36, 90, 125, 58, 52, 124, 26, 59, 13, 121, 119, 56, 54, 115, 117, 53, 81, 18, 57, 108, 118, 34, 17, 103, 31, 122, 88, 120, 104, 63, 60, 127, 61, 107, 111, 105, 97, 110, 72, 44, 21, 51, 42, 46, 48, 24, 15, 32, 116, 123, 41, 45, 94, 70, 30, 43, 114, 49, 112, 99, 55, 106, 113, 80, 37, 38, 62, 85, 35, 100, 25, 92, 22, 101, 95, 79, 96, 102, 33, 14, 77, 27, 19, 83, 11, 4, 89, 28, 12, 91, 16, 86, 6, 82, 8, 76, 78, 10, 74, 73, 66, 9, 1, 68, 2, 65, 7, 3, 71, 5, 64, 69, 0, 67], [40, 126, 98, 109, 29, 60, 84, 90, 51, 87, 93, 124, 47, 23, 80, 50, 26, 18, 75, 59, 54, 34, 117, 36, 39, 88, 20, 119, 118, 53, 121, 13, 63, 114, 58, 103, 22, 123, 125, 127, 37, 55, 44, 122, 52, 45, 46, 111, 56, 115, 105, 21, 48, 104, 110, 107, 62, 120, 94, 92, 102, 61, 96, 108, 57, 112, 30, 38, 91, 49, 113, 43, 72, 70, 24, 12, 116, 35, 41, 42, 106, 97, 89, 86, 32, 31, 99, 100, 101, 16, 17, 85, 28, 79, 19, 33, 25, 73, 81, 82, 76, 15, 27, 95, 83, 14, 78, 4, 11, 10, 9, 6, 74, 8, 3, 69, 68, 67, 7, 77, 5, 71, 65, 1, 66, 2, 0, 64], [40, 51, 98, 29, 126, 13, 80, 18, 87, 84, 72, 23, 90, 4, 75, 93, 96, 26, 70, 60, 10, 73, 88, 124, 52, 19, 94, 121, 56, 125, 71, 54, 86, 30, 119, 81, 65, 117, 20, 109, 47, 59, 122, 77, 28, 50, 53, 111, 46, 16, 3, 61, 104, 105, 127, 112, 63, 74, 108, 78, 64, 15, 101, 68, 32, 39, 7, 91, 5, 89, 95, 97, 79, 67, 82, 11, 83, 58, 8, 118, 41, 22, 123, 106, 14, 2, 115, 120, 33, 76, 6, 62, 31, 34, 48, 69, 49, 113, 55, 116, 27, 12, 38, 9, 21, 100, 36, 44, 37, 110, 25, 57, 43, 92, 45, 17, 1, 103, 99, 24, 35, 66, 85, 102, 114, 42, 0, 107], [113, 115, 105, 57, 49, 63, 124, 53, 119, 34, 30, 39, 112, 48, 56, 127, 123, 61, 58, 45, 59, 117, 60, 116, 52, 120, 122, 62, 47, 27, 38, 118, 110, 107, 37, 108, 54, 121, 55, 94, 51, 125, 46, 111, 106, 85, 126, 44, 88, 114, 99, 104, 109, 87, 101, 90, 42, 43, 89, 36, 25, 24, 19, 103, 78, 18, 28, 40, 50, 92, 23, 41, 100, 35, 80, 83, 91, 22, 98, 33, 102, 31, 93, 21, 32, 14, 97, 26, 74, 95, 96, 6, 17, 16, 82, 65, 29, 12, 76, 20, 70, 81, 66, 64, 67, 72, 8, 10, 3, 0, 69, 86, 15, 11, 1, 2, 5, 13, 7, 68, 9, 75, 77, 4, 79, 73, 71, 84], [115, 61, 105, 113, 34, 30, 50, 124, 121, 120, 37, 38, 48, 63, 57, 126, 122, 53, 55, 112, 119, 27, 123, 49, 56, 59, 92, 60, 101, 62, 58, 127, 52, 46, 47, 45, 39, 118, 90, 110, 107, 44, 85, 111, 54, 125, 94, 116, 117, 108, 109, 103, 99, 87, 25, 114, 51, 40, 106, 41, 42, 33, 43, 89, 88, 104, 93, 96, 80, 32, 23, 91, 35, 36, 19, 31, 95, 102, 100, 98, 82, 78, 29, 28, 83, 21, 26, 18, 16, 97, 24, 22, 70, 17, 74, 6, 14, 64, 76, 0, 65, 20, 66, 1, 3, 69, 12, 86, 8, 81, 67, 2, 5, 10, 13, 72, 11, 15, 7, 9, 71, 4, 79, 68, 84, 73, 77, 75], [105, 113, 61, 57, 34, 115, 22, 20, 27, 30, 55, 41, 24, 17, 63, 53, 80, 13, 37, 59, 74, 38, 82, 18, 15, 11, 124, 123, 21, 12, 19, 7, 56, 51, 97, 121, 122, 70, 88, 26, 62, 47, 49, 103, 75, 99, 117, 9, 90, 116, 106, 31, 95, 36, 52, 127, 119, 60, 23, 58, 71, 10, 118, 39, 120, 89, 100, 46, 72, 112, 33, 45, 83, 111, 2, 6, 91, 50, 44, 3, 28, 48, 29, 14, 114, 126, 109, 101, 77, 93, 40, 85, 107, 54, 104, 84, 64, 110, 102, 16, 73, 81, 96, 76, 66, 42, 32, 25, 108, 87, 86, 92, 79, 68, 78, 94, 5, 69, 43, 1, 98, 4, 125, 35, 65, 8, 0, 67], [105, 57, 115, 113, 53, 34, 27, 22, 20, 15, 30, 41, 63, 9, 17, 24, 13, 49, 127, 74, 120, 4, 94, 55, 39, 116, 61, 37, 85, 1, 68, 3, 47, 36, 11, 87, 76, 73, 29, 72, 78, 77, 28, 31, 6, 46, 118, 123, 48, 79, 88, 95, 58, 119, 93, 103, 60, 26, 8, 18, 107, 84, 54, 38, 92, 104, 124, 35, 90, 112, 52, 43, 83, 110, 56, 40, 100, 121, 45, 109, 126, 23, 42, 59, 106, 122, 10, 96, 21, 50, 44, 62, 117, 125, 108, 81, 64, 111, 32, 114, 33, 86, 89, 12, 70, 25, 80, 51, 97, 99, 5, 16, 19, 14, 75, 66, 91, 98, 69, 65, 101, 67, 82, 102, 0, 7, 71, 2], [108, 37, 126, 88, 19, 90, 78, 32, 75, 81, 44, 8, 5, 36, 93, 86, 85, 67, 21, 1, 24, 34, 61, 22, 0, 16, 15, 83, 18, 12, 11, 72, 20, 69, 119, 58, 80, 65, 17, 25, 28, 14, 89, 13, 84, 87, 103, 23, 73, 10, 2, 98, 77, 74, 79, 35, 91, 27, 29, 30, 3, 76, 9, 94, 71, 82, 46, 26, 100, 118, 127, 68, 70, 95, 97, 125, 31, 53, 6, 7, 51, 92, 106, 64, 45, 52, 40, 66, 54, 4, 120, 122, 33, 113, 107, 117, 60, 115, 63, 99, 57, 123, 59, 105, 39, 42, 96, 111, 62, 104, 110, 43, 50, 114, 116, 102, 124, 109, 55, 47, 121, 101, 38, 48, 41, 49, 56, 112], [126, 108, 37, 44, 23, 58, 120, 57, 113, 51, 48, 61, 122, 63, 114, 118, 56, 62, 115, 116, 102, 104, 59, 32, 52, 119, 93, 124, 55, 53, 46, 54, 60, 125, 121, 101, 49, 100, 117, 110, 50, 103, 47, 45, 111, 123, 41, 109, 112, 40, 127, 106, 16, 107, 27, 42, 43, 90, 105, 39, 38, 91, 36, 35, 92, 84, 98, 99, 97, 95, 87, 33, 94, 34, 30, 25, 86, 88, 96, 29, 31, 85, 81, 28, 17, 20, 80, 89, 22, 74, 10, 2, 19, 24, 12, 26, 83, 82, 15, 0, 6, 65, 4, 21, 76, 66, 18, 70, 71, 13, 79, 78, 11, 67, 64, 7, 5, 9, 77, 69, 68, 3, 1, 73, 14, 75, 8, 72], [108, 61, 126, 44, 36, 119, 90, 85, 32, 122, 82, 93, 35, 24, 88, 37, 104, 58, 7, 19, 12, 15, 81, 13, 62, 91, 29, 101, 59, 95, 28, 63, 86, 46, 51, 20, 23, 79, 27, 120, 48, 78, 53, 57, 60, 96, 109, 87, 98, 116, 97, 115, 74, 89, 45, 71, 4, 18, 118, 114, 70, 66, 99, 52, 49, 33, 124, 64, 54, 125, 25, 127, 113, 56, 107, 117, 100, 34, 55, 9, 75, 41, 121, 26, 42, 92, 47, 31, 110, 30, 94, 123, 38, 16, 39, 112, 50, 76, 40, 102, 84, 6, 111, 106, 105, 65, 43, 103, 80, 21, 77, 2, 68, 22, 8, 83, 3, 11, 17, 0, 10, 5, 1, 73, 67, 69, 14, 72], [108, 44, 37, 126, 93, 23, 90, 58, 81, 24, 32, 61, 36, 85, 120, 19, 123, 41, 56, 86, 31, 29, 96, 118, 114, 110, 16, 78, 125, 112, 54, 105, 113, 88, 33, 63, 38, 103, 122, 48, 106, 87, 51, 111, 42, 109, 62, 115, 52, 57, 30, 47, 124, 119, 50, 102, 53, 127, 97, 43, 116, 27, 60, 34, 55, 121, 35, 45, 46, 99, 107, 98, 94, 74, 117, 49, 40, 84, 59, 104, 91, 7, 28, 25, 80, 39, 95, 100, 92, 26, 75, 82, 76, 89, 101, 13, 17, 9, 22, 10, 77, 15, 18, 0, 79, 71, 2, 83, 8, 67, 21, 64, 66, 5, 20, 12, 11, 70, 4, 68, 6, 73, 65, 14, 1, 69, 3, 72], [56, 126, 101, 62, 60, 122, 37, 125, 116, 123, 88, 53, 119, 58, 55, 51, 118, 120, 115, 117, 63, 48, 50, 89, 46, 127, 114, 84, 54, 124, 121, 33, 61, 59, 57, 103, 49, 14, 52, 11, 34, 113, 47, 92, 110, 39, 95, 107, 104, 44, 112, 98, 109, 108, 45, 111, 100, 106, 41, 40, 43, 93, 42, 35, 26, 16, 20, 38, 91, 105, 36, 15, 94, 99, 102, 18, 12, 22, 31, 82, 96, 90, 28, 25, 72, 86, 80, 32, 30, 68, 24, 87, 5, 23, 29, 17, 74, 97, 2, 7, 21, 70, 27, 79, 77, 19, 83, 85, 76, 81, 6, 73, 78, 64, 65, 9, 13, 75, 3, 0, 8, 66, 10, 67, 71, 69, 4, 1], [56, 126, 101, 116, 39, 62, 37, 20, 91, 53, 84, 26, 123, 127, 14, 125, 60, 119, 114, 33, 46, 100, 103, 120, 48, 54, 122, 118, 59, 51, 124, 55, 88, 58, 121, 45, 11, 104, 115, 40, 110, 50, 41, 36, 25, 106, 107, 63, 89, 61, 47, 117, 35, 34, 108, 44, 57, 99, 111, 52, 105, 42, 43, 112, 109, 95, 113, 29, 38, 16, 98, 49, 82, 31, 102, 94, 32, 12, 87, 93, 96, 83, 90, 92, 72, 15, 27, 30, 24, 28, 80, 97, 18, 77, 74, 86, 73, 22, 5, 6, 7, 68, 81, 70, 19, 21, 23, 65, 78, 79, 66, 75, 85, 9, 76, 17, 64, 2, 13, 8, 71, 67, 0, 10, 4, 3, 69, 1], [56, 126, 101, 60, 84, 116, 26, 40, 11, 21, 93, 62, 89, 28, 2, 125, 109, 68, 122, 127, 92, 33, 53, 55, 31, 118, 37, 79, 51, 29, 88, 24, 90, 114, 119, 124, 63, 58, 47, 123, 57, 61, 80, 48, 59, 110, 117, 50, 0, 103, 75, 120, 23, 121, 91, 52, 15, 16, 41, 54, 49, 108, 18, 106, 32, 34, 8, 115, 46, 100, 113, 112, 96, 67, 111, 35, 44, 17, 20, 30, 85, 70, 95, 105, 81, 19, 94, 77, 7, 45, 43, 22, 42, 82, 38, 9, 39, 107, 36, 99, 13, 104, 74, 76, 98, 3, 69, 102, 83, 5, 6, 73, 14, 87, 1, 65, 25, 27, 12, 72, 86, 10, 78, 4, 71, 97, 66, 64], [126, 56, 101, 62, 37, 88, 122, 53, 123, 15, 103, 125, 116, 24, 60, 91, 119, 120, 118, 23, 124, 92, 114, 48, 58, 127, 51, 117, 55, 63, 89, 54, 12, 33, 46, 121, 94, 50, 25, 61, 84, 59, 14, 6, 39, 49, 115, 57, 113, 52, 44, 47, 100, 11, 90, 45, 110, 18, 34, 112, 111, 20, 9, 99, 107, 17, 36, 105, 87, 7, 109, 40, 108, 95, 81, 98, 72, 22, 96, 104, 42, 102, 26, 35, 3, 28, 38, 106, 31, 32, 5, 86, 93, 68, 41, 79, 30, 82, 43, 80, 27, 16, 70, 97, 29, 74, 19, 64, 65, 76, 21, 66, 85, 1, 67, 8, 78, 2, 77, 73, 10, 4, 83, 75, 0, 71, 13, 69], [120, 109, 104, 124, 34, 123, 127, 59, 27, 89, 58, 60, 30, 52, 56, 125, 114, 86, 91, 54, 126, 118, 78, 121, 61, 113, 53, 62, 29, 45, 51, 50, 88, 119, 63, 40, 35, 110, 122, 84, 116, 94, 117, 19, 25, 49, 42, 55, 57, 112, 48, 43, 46, 107, 111, 47, 39, 115, 37, 106, 100, 108, 105, 41, 36, 10, 101, 103, 96, 44, 102, 99, 33, 21, 38, 18, 97, 26, 32, 31, 92, 28, 98, 87, 90, 95, 15, 22, 93, 17, 20, 16, 82, 14, 23, 85, 83, 9, 69, 80, 79, 24, 12, 76, 74, 13, 81, 73, 64, 5, 65, 77, 1, 6, 11, 7, 75, 4, 71, 68, 70, 72, 2, 67, 0, 8, 66, 3], [120, 109, 104, 123, 52, 58, 30, 127, 121, 59, 54, 124, 60, 122, 126, 61, 114, 112, 53, 56, 125, 117, 118, 45, 62, 51, 116, 113, 63, 110, 89, 107, 50, 49, 119, 57, 48, 78, 55, 115, 111, 46, 106, 47, 43, 42, 105, 34, 38, 108, 102, 25, 44, 18, 87, 41, 40, 101, 39, 100, 99, 103, 37, 94, 36, 98, 91, 35, 97, 27, 96, 88, 10, 33, 86, 32, 93, 28, 31, 95, 19, 85, 29, 82, 92, 90, 16, 84, 22, 26, 23, 74, 14, 21, 64, 20, 65, 83, 69, 1, 79, 15, 80, 76, 6, 17, 0, 5, 9, 24, 12, 81, 4, 3, 70, 66, 2, 67, 13, 11, 7, 71, 68, 73, 8, 75, 72, 77], [104, 120, 34, 88, 109, 17, 13, 75, 15, 7, 9, 86, 84, 92, 66, 71, 91, 2, 124, 11, 85, 19, 30, 69, 5, 4, 27, 40, 45, 22, 74, 0, 98, 68, 20, 64, 81, 82, 77, 24, 83, 79, 18, 80, 87, 29, 73, 26, 21, 76, 70, 107, 1, 14, 23, 94, 3, 72, 93, 90, 35, 8, 25, 123, 16, 47, 12, 6, 89, 56, 65, 116, 67, 78, 10, 59, 127, 46, 63, 31, 96, 54, 28, 95, 32, 97, 113, 106, 58, 52, 42, 99, 100, 101, 126, 55, 118, 43, 33, 37, 36, 112, 60, 105, 122, 125, 121, 114, 62, 117, 115, 49, 38, 110, 108, 61, 50, 39, 57, 53, 48, 102, 51, 103, 44, 119, 41, 111], [109, 104, 120, 34, 123, 78, 27, 59, 88, 116, 30, 112, 54, 86, 84, 52, 127, 125, 10, 121, 89, 40, 29, 18, 58, 126, 124, 60, 91, 114, 55, 56, 46, 113, 50, 61, 87, 122, 53, 62, 25, 110, 115, 26, 98, 45, 118, 63, 117, 119, 49, 19, 43, 48, 15, 51, 39, 42, 57, 100, 47, 36, 97, 90, 35, 106, 108, 38, 32, 28, 96, 107, 99, 101, 14, 31, 44, 102, 33, 111, 94, 17, 16, 9, 37, 105, 41, 92, 95, 23, 93, 103, 83, 82, 13, 22, 21, 76, 85, 20, 24, 80, 4, 6, 74, 79, 69, 12, 0, 2, 75, 73, 72, 11, 7, 81, 5, 71, 64, 1, 77, 70, 68, 65, 67, 8, 66, 3]], "model.layers.17.self_attn.k_proj": [[42, 36, 114, 60, 86, 93, 91, 80, 83, 89, 11, 63, 14, 19, 124, 119, 0, 53, 125, 46, 9, 69, 113, 111, 59, 25, 56, 54, 120, 126, 76, 50, 47, 44, 57, 115, 123, 118, 127, 116, 117, 32, 45, 43, 15, 2, 33, 71, 62, 51, 48, 122, 101, 121, 110, 49, 4, 55, 108, 41, 52, 24, 103, 104, 66, 106, 40, 105, 112, 58, 102, 90, 74, 98, 107, 1, 109, 39, 97, 18, 30, 77, 61, 5, 35, 88, 12, 34, 31, 100, 87, 95, 38, 81, 37, 20, 23, 17, 99, 28, 21, 6, 96, 72, 82, 70, 8, 92, 27, 26, 84, 94, 10, 79, 85, 65, 78, 7, 67, 13, 68, 29, 3, 64, 22, 16, 73, 75], [126, 39, 93, 85, 15, 82, 24, 1, 76, 5, 73, 71, 64, 48, 112, 74, 26, 50, 3, 120, 116, 96, 35, 121, 119, 117, 77, 118, 57, 115, 110, 105, 41, 16, 113, 114, 63, 60, 97, 54, 55, 42, 125, 46, 56, 52, 43, 49, 6, 29, 84, 34, 4, 40, 59, 122, 66, 61, 32, 27, 123, 62, 53, 90, 47, 75, 124, 99, 19, 51, 107, 127, 72, 87, 78, 38, 86, 109, 44, 111, 58, 37, 70, 100, 108, 23, 45, 104, 101, 25, 14, 83, 22, 11, 36, 106, 89, 13, 68, 95, 28, 69, 33, 81, 94, 2, 17, 102, 98, 31, 91, 92, 0, 30, 80, 20, 10, 8, 88, 9, 65, 67, 18, 21, 7, 103, 12, 79], [109, 58, 102, 30, 59, 8, 45, 13, 74, 19, 25, 86, 16, 70, 3, 97, 89, 65, 99, 28, 23, 0, 46, 32, 49, 47, 69, 36, 127, 121, 12, 126, 94, 115, 18, 15, 62, 91, 51, 84, 118, 53, 61, 120, 111, 107, 63, 116, 35, 77, 117, 48, 125, 11, 114, 83, 43, 101, 20, 87, 9, 110, 24, 27, 57, 106, 50, 52, 54, 68, 78, 4, 98, 123, 112, 113, 124, 122, 56, 29, 105, 17, 44, 37, 55, 119, 85, 31, 42, 60, 34, 7, 108, 104, 71, 64, 41, 39, 103, 40, 2, 100, 80, 90, 81, 6, 66, 26, 93, 82, 21, 5, 10, 96, 14, 88, 95, 75, 33, 79, 72, 92, 67, 73, 22, 76, 1, 38], [104, 126, 34, 50, 51, 115, 93, 23, 84, 26, 114, 13, 80, 111, 18, 45, 75, 56, 124, 59, 53, 70, 105, 72, 48, 127, 65, 119, 4, 58, 81, 73, 122, 64, 55, 57, 125, 103, 118, 94, 123, 3, 49, 60, 121, 99, 63, 2, 30, 117, 61, 10, 109, 110, 106, 32, 120, 22, 46, 14, 29, 24, 108, 113, 7, 38, 62, 36, 12, 107, 87, 116, 92, 112, 95, 42, 52, 102, 41, 28, 44, 54, 90, 25, 39, 101, 83, 19, 31, 71, 100, 47, 86, 43, 91, 37, 21, 96, 35, 15, 27, 89, 76, 33, 98, 5, 97, 85, 78, 88, 17, 79, 9, 67, 69, 82, 1, 6, 0, 74, 77, 16, 8, 20, 68, 11, 66, 40], [41, 98, 57, 115, 113, 94, 24, 51, 11, 20, 15, 91, 13, 22, 17, 61, 27, 55, 49, 9, 63, 68, 62, 59, 85, 26, 25, 72, 127, 56, 53, 123, 19, 116, 6, 114, 101, 35, 99, 107, 7, 18, 102, 48, 80, 108, 28, 122, 109, 65, 104, 54, 87, 125, 16, 60, 118, 8, 84, 117, 66, 74, 47, 103, 82, 97, 43, 121, 111, 93, 120, 78, 45, 33, 119, 42, 52, 58, 44, 105, 110, 124, 112, 96, 36, 50, 40, 46, 100, 76, 126, 3, 32, 73, 95, 5, 106, 29, 77, 31, 64, 39, 92, 90, 0, 67, 37, 79, 75, 89, 21, 23, 69, 70, 12, 83, 88, 71, 38, 81, 34, 2, 86, 10, 30, 14, 4, 1], [44, 126, 86, 101, 108, 100, 96, 61, 29, 58, 78, 90, 57, 19, 8, 51, 63, 119, 75, 120, 62, 81, 55, 125, 5, 124, 114, 47, 117, 116, 59, 122, 118, 123, 53, 16, 67, 60, 46, 74, 111, 56, 52, 112, 115, 110, 54, 0, 50, 121, 49, 88, 106, 127, 48, 109, 113, 23, 34, 65, 104, 42, 85, 99, 105, 39, 45, 98, 9, 107, 43, 41, 27, 66, 83, 40, 103, 82, 25, 38, 4, 70, 35, 13, 36, 102, 1, 94, 92, 28, 30, 73, 31, 26, 20, 93, 33, 95, 77, 97, 2, 68, 89, 15, 24, 7, 3, 17, 91, 12, 21, 79, 76, 84, 64, 18, 6, 72, 71, 32, 22, 80, 37, 11, 10, 87, 14, 69], [126, 37, 56, 22, 97, 62, 51, 125, 122, 123, 59, 119, 61, 60, 53, 116, 127, 58, 118, 50, 63, 48, 121, 120, 117, 124, 52, 49, 57, 112, 114, 55, 113, 54, 92, 47, 104, 115, 46, 94, 111, 27, 109, 110, 44, 42, 108, 45, 107, 103, 38, 93, 33, 43, 106, 39, 105, 41, 89, 82, 40, 35, 102, 101, 80, 84, 36, 98, 96, 99, 88, 86, 28, 34, 19, 77, 79, 100, 91, 32, 95, 30, 74, 90, 25, 3, 29, 9, 31, 7, 5, 85, 87, 26, 17, 23, 15, 81, 64, 16, 83, 21, 76, 24, 14, 65, 75, 72, 12, 6, 18, 78, 13, 2, 68, 4, 73, 71, 20, 11, 8, 70, 10, 66, 67, 1, 69, 0], [120, 40, 45, 86, 98, 123, 54, 27, 109, 60, 17, 59, 127, 113, 94, 58, 7, 88, 61, 13, 126, 52, 56, 117, 114, 62, 63, 9, 15, 53, 121, 118, 119, 122, 48, 75, 34, 116, 110, 66, 57, 55, 49, 124, 50, 51, 115, 112, 111, 46, 43, 47, 125, 84, 19, 103, 42, 78, 44, 26, 10, 107, 18, 5, 39, 106, 89, 102, 65, 108, 64, 36, 31, 105, 41, 93, 16, 101, 38, 87, 21, 95, 69, 8, 6, 99, 67, 4, 35, 30, 33, 37, 72, 90, 28, 97, 96, 24, 100, 76, 20, 32, 11, 12, 29, 104, 85, 92, 23, 68, 80, 83, 3, 71, 70, 81, 91, 0, 25, 82, 77, 79, 14, 1, 74, 73, 22, 2]], "model.layers.17.self_attn.qk_proj": [[126, 120, 109, 58, 44, 114, 56, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 50, 59, 41, 40, 106, 55, 119, 22, 118, 123, 49, 34, 105, 124, 125, 116, 93, 87, 112, 86, 127, 62, 117, 90, 111, 24, 94, 98, 53, 63, 27, 80, 37, 16, 52, 47, 30, 89, 23, 100, 25, 26, 54, 121, 39, 88, 18, 91, 83, 13, 85, 82, 75, 101, 122, 19, 48, 21, 103, 77, 110, 35, 84, 11, 20, 15, 36, 73, 79, 17, 99, 102, 32, 96, 43, 107, 9, 46, 78, 81, 14, 92, 76, 72, 38, 12, 97, 69, 7, 8, 5, 74, 71, 3, 28, 6, 64, 67, 33, 10, 65, 1, 70, 0, 4, 95, 68, 66, 31, 2], [126, 120, 109, 56, 44, 58, 114, 60, 115, 45, 113, 108, 29, 57, 104, 42, 61, 59, 50, 51, 40, 106, 123, 41, 22, 119, 55, 118, 49, 105, 124, 125, 34, 86, 93, 53, 112, 47, 98, 87, 116, 37, 127, 111, 63, 90, 24, 94, 54, 52, 27, 101, 39, 62, 30, 26, 16, 25, 121, 80, 48, 23, 91, 18, 83, 103, 110, 100, 82, 35, 89, 13, 117, 21, 88, 85, 79, 19, 122, 20, 77, 11, 75, 36, 32, 84, 46, 102, 96, 99, 73, 78, 15, 9, 107, 76, 17, 92, 81, 97, 14, 43, 12, 72, 69, 5, 38, 74, 65, 71, 7, 10, 67, 8, 28, 0, 70, 33, 64, 1, 68, 3, 6, 31, 66, 4, 95, 2], [126, 120, 109, 56, 58, 44, 114, 60, 115, 45, 113, 108, 29, 104, 61, 42, 57, 51, 40, 59, 50, 106, 41, 55, 22, 123, 105, 124, 118, 119, 34, 86, 93, 90, 49, 125, 112, 98, 116, 127, 62, 39, 52, 26, 87, 27, 16, 121, 63, 47, 24, 100, 37, 101, 25, 94, 117, 30, 111, 53, 80, 91, 18, 122, 103, 48, 89, 54, 88, 82, 23, 32, 19, 83, 99, 85, 36, 15, 102, 35, 20, 21, 13, 77, 84, 79, 11, 73, 75, 110, 43, 96, 92, 46, 65, 9, 1, 17, 64, 5, 3, 78, 107, 70, 76, 14, 81, 97, 10, 8, 67, 74, 0, 33, 7, 38, 69, 12, 72, 71, 28, 4, 2, 6, 31, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 104, 29, 61, 57, 42, 51, 50, 40, 59, 41, 119, 106, 55, 49, 123, 105, 125, 118, 22, 34, 116, 124, 62, 52, 90, 93, 63, 53, 98, 39, 122, 86, 127, 94, 47, 87, 112, 16, 117, 121, 100, 27, 30, 37, 111, 85, 101, 25, 48, 80, 24, 88, 26, 54, 91, 18, 23, 15, 103, 89, 36, 19, 21, 110, 82, 35, 46, 102, 77, 20, 96, 84, 75, 79, 83, 13, 32, 99, 73, 43, 11, 67, 92, 17, 10, 65, 76, 70, 5, 78, 81, 14, 74, 9, 3, 1, 12, 64, 107, 97, 8, 0, 69, 38, 33, 7, 72, 6, 68, 28, 71, 4, 66, 2, 31, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 57, 104, 42, 50, 59, 51, 29, 41, 40, 106, 123, 119, 105, 55, 124, 116, 62, 22, 49, 118, 34, 90, 47, 125, 86, 93, 98, 127, 112, 111, 87, 122, 39, 94, 63, 27, 16, 80, 117, 24, 100, 53, 88, 37, 89, 18, 121, 91, 52, 23, 48, 19, 83, 26, 82, 30, 25, 15, 54, 85, 77, 75, 20, 13, 35, 103, 110, 101, 102, 46, 84, 79, 21, 11, 36, 43, 9, 73, 3, 96, 78, 14, 17, 76, 81, 70, 32, 99, 8, 1, 10, 67, 92, 107, 7, 5, 69, 12, 97, 74, 0, 65, 6, 72, 64, 71, 28, 33, 68, 4, 38, 66, 31, 2, 95], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 104, 29, 61, 57, 50, 42, 51, 59, 41, 40, 55, 22, 106, 123, 105, 49, 119, 124, 34, 116, 125, 118, 62, 86, 93, 87, 127, 63, 16, 98, 90, 27, 94, 24, 37, 80, 117, 47, 18, 112, 39, 111, 53, 88, 91, 85, 30, 13, 89, 19, 82, 26, 48, 122, 83, 121, 23, 54, 25, 84, 35, 15, 100, 79, 110, 75, 103, 77, 20, 21, 52, 101, 11, 46, 17, 102, 99, 96, 36, 9, 81, 97, 14, 12, 78, 73, 92, 32, 8, 10, 74, 107, 76, 43, 70, 5, 69, 7, 72, 71, 33, 65, 67, 28, 38, 1, 3, 64, 0, 6, 31, 95, 4, 68, 2, 66], [126, 120, 109, 58, 56, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 104, 51, 61, 50, 106, 40, 59, 41, 123, 22, 55, 119, 124, 49, 105, 118, 34, 127, 125, 86, 87, 93, 24, 27, 53, 94, 47, 116, 90, 80, 37, 88, 62, 89, 111, 16, 98, 25, 63, 83, 121, 52, 19, 18, 84, 112, 117, 91, 122, 15, 54, 30, 26, 77, 39, 100, 23, 48, 75, 21, 82, 85, 110, 101, 13, 46, 35, 20, 36, 103, 102, 11, 99, 79, 96, 17, 14, 81, 78, 9, 12, 32, 73, 92, 10, 107, 76, 97, 43, 8, 33, 7, 28, 38, 69, 74, 72, 5, 3, 71, 1, 0, 31, 70, 67, 65, 68, 6, 64, 4, 95, 66, 2], [126, 120, 109, 58, 44, 56, 60, 114, 115, 113, 108, 45, 57, 29, 42, 61, 51, 104, 59, 50, 22, 123, 106, 41, 40, 55, 118, 119, 125, 53, 49, 86, 124, 90, 105, 93, 34, 62, 87, 27, 127, 94, 24, 25, 89, 88, 98, 116, 37, 112, 80, 47, 63, 101, 16, 52, 39, 111, 18, 91, 26, 100, 83, 23, 30, 117, 121, 19, 54, 122, 99, 15, 103, 85, 21, 13, 82, 84, 35, 77, 102, 20, 46, 48, 75, 36, 32, 110, 97, 79, 96, 81, 92, 73, 17, 11, 33, 14, 78, 9, 43, 28, 12, 107, 8, 69, 5, 71, 6, 10, 76, 7, 65, 72, 31, 0, 3, 74, 64, 67, 1, 38, 4, 95, 68, 66, 70, 2], [126, 120, 109, 58, 56, 44, 60, 114, 113, 115, 45, 108, 57, 29, 104, 42, 61, 51, 50, 59, 40, 106, 123, 41, 55, 22, 119, 86, 49, 34, 93, 105, 118, 127, 37, 47, 124, 122, 90, 53, 116, 94, 62, 25, 89, 87, 26, 111, 27, 63, 125, 24, 30, 98, 52, 121, 16, 88, 80, 18, 101, 112, 83, 48, 54, 23, 35, 46, 100, 91, 82, 39, 85, 15, 117, 103, 84, 21, 13, 20, 110, 36, 19, 102, 32, 11, 99, 77, 43, 75, 79, 78, 96, 17, 81, 97, 73, 92, 107, 14, 74, 9, 12, 10, 33, 76, 28, 6, 69, 71, 8, 38, 7, 5, 72, 64, 65, 1, 31, 0, 68, 67, 3, 95, 4, 66, 2, 70], [126, 120, 109, 58, 56, 44, 114, 60, 113, 115, 45, 108, 57, 29, 61, 104, 42, 51, 50, 40, 123, 41, 106, 62, 59, 119, 49, 22, 55, 86, 93, 127, 34, 52, 118, 124, 105, 37, 125, 121, 47, 116, 53, 94, 90, 87, 63, 54, 98, 30, 122, 111, 89, 101, 16, 80, 25, 39, 85, 112, 24, 88, 27, 117, 83, 100, 23, 26, 46, 82, 91, 13, 18, 11, 35, 77, 36, 48, 75, 79, 110, 21, 103, 20, 19, 84, 102, 99, 76, 81, 15, 96, 9, 73, 17, 78, 32, 43, 92, 107, 5, 8, 7, 97, 71, 69, 6, 72, 12, 14, 74, 28, 3, 65, 1, 10, 38, 0, 33, 67, 64, 4, 70, 95, 68, 66, 2, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 45, 113, 108, 104, 29, 57, 42, 61, 51, 123, 41, 40, 59, 50, 55, 106, 125, 105, 34, 49, 22, 119, 118, 62, 124, 86, 53, 93, 80, 116, 112, 98, 90, 16, 47, 52, 127, 94, 37, 122, 111, 87, 89, 27, 63, 13, 24, 30, 18, 117, 39, 85, 54, 23, 103, 25, 88, 26, 15, 100, 91, 84, 19, 11, 75, 77, 83, 82, 121, 79, 20, 48, 101, 73, 21, 96, 110, 36, 9, 35, 46, 17, 5, 67, 6, 14, 81, 72, 92, 78, 32, 43, 102, 99, 1, 69, 71, 76, 7, 10, 12, 97, 74, 65, 3, 8, 0, 64, 70, 68, 38, 33, 107, 4, 28, 2, 66, 95, 31], [126, 120, 109, 58, 56, 44, 114, 60, 115, 108, 113, 45, 104, 29, 42, 57, 61, 50, 51, 40, 59, 106, 41, 123, 22, 55, 49, 34, 86, 125, 105, 118, 124, 119, 47, 62, 127, 52, 98, 80, 90, 53, 24, 93, 27, 87, 16, 116, 122, 117, 37, 89, 94, 26, 39, 121, 112, 88, 91, 13, 63, 18, 30, 111, 83, 103, 48, 19, 82, 85, 54, 11, 23, 101, 84, 100, 25, 79, 75, 21, 20, 15, 77, 35, 46, 36, 9, 110, 78, 14, 73, 81, 99, 102, 96, 12, 32, 43, 17, 71, 97, 92, 72, 33, 107, 10, 69, 7, 5, 76, 6, 74, 3, 67, 1, 8, 0, 38, 65, 70, 64, 28, 31, 4, 68, 2, 66, 95], [126, 120, 109, 56, 114, 58, 60, 44, 115, 113, 108, 45, 29, 57, 61, 42, 123, 51, 104, 50, 40, 106, 41, 59, 124, 125, 55, 119, 118, 22, 86, 105, 49, 93, 62, 122, 52, 37, 34, 90, 53, 94, 116, 87, 47, 127, 27, 24, 88, 98, 101, 89, 111, 30, 121, 112, 103, 54, 25, 82, 91, 117, 16, 80, 48, 46, 26, 100, 83, 18, 35, 110, 21, 85, 39, 19, 63, 96, 102, 99, 23, 13, 20, 36, 84, 32, 79, 97, 11, 81, 15, 77, 43, 75, 78, 92, 107, 12, 17, 73, 14, 33, 9, 10, 71, 28, 72, 5, 38, 74, 76, 70, 69, 8, 7, 31, 68, 67, 6, 95, 1, 65, 3, 0, 4, 66, 64, 2], [126, 120, 109, 56, 114, 58, 44, 60, 115, 108, 113, 45, 57, 29, 104, 51, 42, 61, 123, 50, 106, 40, 59, 119, 41, 118, 55, 22, 124, 86, 105, 125, 49, 34, 62, 127, 90, 116, 37, 93, 47, 53, 63, 54, 98, 52, 87, 30, 16, 111, 24, 117, 27, 94, 122, 80, 101, 100, 26, 25, 89, 121, 91, 112, 103, 88, 83, 23, 35, 39, 84, 13, 85, 48, 46, 19, 21, 18, 82, 11, 75, 79, 81, 99, 107, 9, 36, 102, 20, 32, 15, 77, 110, 43, 97, 78, 96, 73, 14, 71, 17, 5, 76, 70, 92, 10, 72, 64, 33, 12, 7, 65, 69, 0, 1, 74, 8, 3, 38, 67, 28, 4, 95, 31, 66, 68, 2, 6], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 108, 45, 57, 29, 104, 42, 61, 51, 50, 123, 59, 106, 40, 41, 22, 55, 119, 49, 93, 105, 34, 118, 124, 98, 86, 87, 62, 125, 37, 80, 90, 47, 16, 52, 89, 53, 94, 111, 127, 122, 27, 39, 116, 24, 30, 54, 101, 46, 117, 112, 18, 100, 26, 21, 19, 91, 13, 82, 83, 11, 85, 103, 25, 75, 23, 84, 15, 63, 36, 81, 77, 20, 88, 107, 110, 73, 79, 121, 9, 35, 102, 17, 48, 78, 96, 99, 14, 12, 76, 92, 7, 32, 70, 10, 5, 97, 72, 71, 67, 43, 69, 8, 3, 64, 74, 33, 1, 0, 65, 38, 28, 4, 6, 68, 31, 2, 95, 66], [126, 120, 109, 56, 58, 44, 60, 114, 115, 113, 45, 108, 29, 57, 42, 61, 104, 50, 51, 59, 40, 123, 41, 55, 49, 119, 106, 105, 22, 34, 90, 124, 47, 62, 98, 116, 118, 125, 86, 117, 53, 111, 122, 37, 52, 127, 103, 30, 112, 24, 93, 94, 87, 26, 46, 54, 80, 39, 101, 16, 27, 63, 82, 25, 85, 13, 88, 19, 110, 91, 83, 89, 100, 11, 121, 18, 15, 75, 73, 21, 84, 36, 35, 23, 48, 77, 20, 79, 9, 102, 99, 96, 107, 81, 92, 78, 10, 12, 32, 14, 72, 76, 70, 17, 33, 69, 97, 43, 7, 67, 3, 8, 71, 65, 74, 38, 0, 1, 5, 64, 6, 28, 68, 4, 66, 95, 31, 2], [126, 120, 109, 114, 56, 44, 58, 60, 115, 108, 113, 45, 29, 57, 61, 42, 104, 51, 50, 40, 106, 123, 41, 59, 55, 49, 119, 118, 22, 86, 34, 105, 124, 62, 94, 52, 93, 116, 98, 125, 111, 90, 37, 47, 30, 53, 27, 117, 87, 127, 101, 39, 24, 122, 54, 63, 80, 112, 88, 25, 26, 91, 100, 89, 19, 82, 46, 48, 16, 21, 13, 85, 83, 103, 84, 35, 23, 121, 18, 96, 79, 102, 15, 36, 11, 110, 107, 99, 92, 20, 32, 75, 17, 12, 77, 81, 33, 43, 73, 78, 97, 9, 14, 10, 64, 69, 28, 71, 38, 76, 72, 74, 8, 5, 7, 65, 70, 67, 1, 0, 3, 95, 31, 4, 6, 2, 68, 66], [126, 120, 109, 58, 114, 44, 56, 60, 115, 113, 108, 45, 29, 57, 104, 42, 61, 51, 50, 106, 40, 123, 41, 59, 119, 55, 22, 49, 34, 105, 118, 93, 86, 62, 124, 125, 47, 111, 98, 90, 94, 52, 122, 26, 37, 116, 117, 87, 24, 63, 27, 30, 39, 53, 80, 112, 88, 85, 16, 89, 101, 127, 25, 91, 121, 100, 82, 18, 54, 83, 13, 110, 103, 46, 19, 23, 36, 11, 20, 102, 21, 35, 15, 32, 48, 99, 96, 12, 75, 79, 84, 77, 9, 92, 73, 17, 107, 81, 97, 78, 14, 43, 71, 33, 7, 69, 5, 74, 8, 72, 28, 38, 64, 67, 10, 1, 3, 0, 76, 65, 6, 70, 31, 68, 4, 66, 2, 95], [126, 120, 109, 58, 44, 60, 56, 114, 115, 113, 45, 108, 57, 104, 29, 61, 42, 51, 50, 40, 59, 123, 41, 124, 106, 119, 62, 22, 105, 49, 34, 118, 55, 37, 93, 47, 116, 52, 94, 98, 111, 90, 125, 87, 53, 122, 39, 100, 86, 127, 30, 112, 54, 121, 27, 80, 25, 26, 101, 117, 24, 18, 103, 35, 63, 16, 88, 110, 36, 89, 46, 23, 21, 83, 82, 91, 102, 85, 96, 19, 15, 75, 11, 48, 84, 13, 20, 99, 107, 79, 81, 73, 77, 32, 38, 92, 17, 97, 9, 12, 43, 76, 14, 78, 8, 1, 74, 71, 67, 6, 10, 5, 64, 7, 3, 69, 72, 28, 33, 65, 0, 70, 95, 68, 66, 31, 4, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 57, 61, 104, 42, 51, 50, 40, 59, 41, 123, 22, 119, 106, 124, 49, 34, 55, 105, 118, 62, 125, 93, 98, 116, 52, 86, 94, 90, 37, 111, 112, 63, 47, 53, 16, 101, 87, 121, 127, 30, 25, 24, 27, 54, 26, 91, 80, 100, 103, 23, 122, 82, 89, 110, 18, 39, 48, 11, 85, 88, 83, 21, 19, 96, 46, 13, 117, 35, 84, 102, 15, 75, 107, 77, 99, 20, 36, 79, 92, 73, 81, 32, 14, 17, 9, 78, 97, 74, 76, 33, 10, 12, 8, 6, 5, 71, 43, 1, 7, 72, 38, 67, 28, 69, 65, 64, 3, 0, 95, 4, 70, 68, 31, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 115, 113, 45, 108, 29, 104, 61, 42, 57, 51, 40, 50, 59, 123, 106, 49, 41, 55, 22, 62, 119, 124, 34, 118, 86, 105, 52, 125, 90, 93, 94, 116, 37, 63, 98, 53, 16, 47, 24, 87, 80, 117, 112, 111, 39, 30, 54, 26, 27, 91, 89, 88, 101, 82, 25, 23, 121, 48, 103, 13, 21, 85, 83, 127, 18, 100, 84, 20, 19, 110, 75, 11, 77, 36, 96, 79, 122, 102, 107, 46, 32, 15, 99, 35, 9, 17, 81, 92, 73, 12, 33, 43, 78, 14, 76, 10, 6, 71, 8, 97, 74, 7, 5, 72, 3, 28, 69, 0, 65, 1, 67, 38, 64, 31, 95, 70, 4, 68, 66, 2], [126, 120, 109, 58, 56, 114, 44, 60, 113, 115, 45, 108, 104, 29, 51, 61, 57, 42, 40, 106, 59, 50, 41, 62, 123, 119, 105, 124, 55, 118, 34, 49, 125, 86, 52, 22, 37, 87, 98, 93, 116, 47, 90, 121, 53, 63, 112, 111, 94, 117, 89, 16, 39, 24, 27, 127, 26, 88, 30, 110, 54, 122, 18, 80, 91, 101, 23, 83, 85, 48, 102, 82, 25, 21, 100, 20, 96, 19, 35, 13, 77, 46, 11, 36, 84, 107, 99, 17, 15, 79, 75, 92, 103, 12, 32, 33, 73, 81, 43, 0, 9, 78, 14, 67, 65, 64, 1, 5, 8, 10, 69, 72, 97, 6, 74, 76, 38, 71, 7, 28, 3, 4, 70, 31, 2, 68, 66, 95], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 61, 29, 104, 57, 51, 42, 40, 50, 59, 123, 106, 41, 118, 62, 55, 22, 124, 119, 105, 49, 116, 86, 34, 93, 37, 98, 111, 94, 87, 90, 125, 53, 30, 117, 47, 112, 16, 121, 110, 52, 39, 54, 127, 27, 100, 103, 122, 91, 88, 63, 26, 101, 18, 89, 80, 24, 48, 19, 83, 20, 25, 82, 23, 21, 35, 36, 85, 107, 46, 77, 13, 99, 84, 102, 11, 32, 79, 96, 92, 75, 15, 17, 9, 12, 73, 33, 78, 81, 14, 43, 69, 3, 8, 97, 76, 7, 74, 10, 28, 38, 1, 65, 5, 6, 64, 68, 70, 71, 72, 0, 67, 31, 66, 95, 4, 2], [126, 120, 109, 56, 58, 114, 44, 60, 115, 113, 108, 45, 29, 104, 61, 42, 57, 51, 123, 50, 40, 106, 59, 41, 62, 22, 49, 118, 119, 55, 105, 124, 34, 116, 93, 47, 125, 86, 87, 98, 37, 63, 90, 52, 94, 111, 53, 30, 80, 91, 117, 24, 121, 88, 39, 16, 26, 18, 122, 127, 112, 100, 48, 27, 89, 19, 54, 101, 46, 85, 25, 110, 82, 20, 11, 83, 103, 15, 75, 23, 21, 36, 102, 35, 84, 77, 107, 96, 13, 79, 32, 73, 17, 92, 14, 76, 9, 99, 7, 12, 97, 81, 78, 8, 69, 43, 10, 38, 70, 33, 74, 5, 28, 71, 72, 67, 1, 0, 3, 65, 64, 68, 6, 95, 4, 31, 66, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 57, 29, 104, 42, 61, 51, 40, 123, 50, 106, 59, 41, 22, 55, 119, 105, 49, 118, 62, 34, 124, 93, 125, 86, 87, 121, 52, 47, 98, 39, 90, 116, 94, 37, 26, 111, 24, 63, 89, 127, 112, 117, 16, 54, 27, 18, 80, 30, 91, 82, 53, 88, 83, 122, 101, 100, 48, 23, 19, 21, 85, 77, 15, 20, 110, 75, 11, 13, 25, 84, 103, 35, 96, 36, 73, 17, 46, 79, 102, 9, 81, 76, 107, 92, 78, 32, 99, 74, 14, 12, 70, 43, 71, 7, 5, 72, 8, 69, 97, 10, 3, 38, 28, 67, 65, 33, 1, 64, 0, 4, 6, 31, 68, 95, 2, 66], [126, 120, 109, 56, 58, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 123, 51, 50, 59, 40, 106, 119, 41, 49, 22, 118, 124, 105, 55, 34, 63, 62, 125, 121, 86, 116, 52, 90, 98, 93, 53, 87, 47, 24, 94, 54, 37, 39, 117, 101, 27, 26, 30, 127, 89, 111, 122, 112, 18, 48, 80, 16, 88, 91, 100, 25, 83, 35, 84, 23, 82, 19, 96, 36, 46, 110, 13, 21, 75, 85, 9, 102, 107, 103, 15, 11, 99, 92, 20, 77, 79, 73, 43, 76, 32, 97, 14, 17, 33, 78, 81, 74, 28, 38, 7, 10, 71, 5, 72, 70, 69, 12, 8, 67, 1, 65, 64, 0, 31, 95, 3, 68, 6, 66, 4, 2], [126, 120, 109, 58, 44, 56, 114, 60, 115, 108, 45, 113, 29, 104, 42, 57, 61, 51, 123, 50, 59, 40, 106, 119, 41, 124, 55, 118, 62, 125, 34, 22, 105, 90, 86, 53, 49, 116, 93, 94, 127, 52, 122, 37, 98, 47, 63, 117, 39, 112, 87, 111, 27, 54, 103, 30, 121, 101, 100, 48, 25, 88, 89, 18, 23, 24, 16, 26, 80, 83, 110, 91, 36, 84, 35, 99, 75, 82, 20, 21, 19, 46, 107, 13, 96, 43, 77, 102, 15, 85, 9, 11, 32, 17, 7, 79, 97, 14, 92, 78, 81, 73, 76, 33, 38, 72, 67, 74, 12, 69, 5, 70, 28, 3, 71, 31, 8, 64, 65, 10, 0, 1, 6, 68, 4, 95, 2, 66], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 104, 57, 42, 61, 51, 50, 40, 59, 106, 123, 41, 62, 22, 124, 118, 119, 49, 105, 55, 116, 87, 86, 47, 93, 34, 125, 122, 94, 121, 111, 98, 117, 63, 52, 37, 112, 90, 127, 30, 39, 80, 18, 24, 53, 16, 27, 89, 54, 26, 25, 101, 91, 83, 100, 13, 75, 84, 23, 19, 88, 82, 21, 48, 96, 110, 11, 35, 15, 79, 20, 36, 85, 77, 102, 107, 73, 99, 32, 9, 17, 81, 103, 46, 12, 14, 76, 78, 74, 10, 7, 5, 92, 97, 72, 1, 43, 69, 67, 71, 8, 64, 33, 6, 3, 65, 38, 70, 0, 4, 28, 68, 66, 95, 31, 2], [126, 120, 109, 58, 56, 44, 114, 60, 115, 113, 45, 108, 29, 61, 104, 57, 42, 40, 50, 51, 59, 123, 41, 106, 22, 49, 124, 105, 62, 55, 119, 118, 34, 86, 125, 63, 98, 47, 93, 116, 94, 111, 87, 30, 37, 90, 122, 16, 127, 112, 39, 27, 52, 100, 24, 117, 54, 80, 96, 91, 82, 101, 25, 26, 121, 89, 88, 18, 53, 110, 75, 46, 103, 83, 13, 85, 79, 48, 23, 19, 21, 35, 84, 36, 77, 11, 15, 102, 20, 73, 9, 43, 107, 14, 99, 81, 72, 76, 5, 92, 17, 32, 1, 74, 78, 97, 64, 65, 12, 10, 8, 0, 7, 3, 69, 6, 71, 38, 67, 33, 70, 28, 4, 66, 68, 95, 31, 2], [126, 120, 109, 56, 58, 44, 114, 115, 60, 113, 45, 108, 29, 104, 61, 51, 42, 57, 59, 50, 123, 40, 41, 106, 22, 118, 105, 49, 124, 119, 55, 34, 125, 86, 116, 93, 98, 87, 94, 47, 63, 53, 90, 127, 52, 37, 30, 117, 80, 24, 111, 112, 39, 62, 16, 54, 27, 121, 26, 82, 88, 25, 75, 83, 91, 13, 21, 100, 23, 89, 101, 18, 35, 48, 79, 96, 85, 77, 110, 122, 84, 46, 20, 11, 15, 19, 73, 103, 99, 36, 92, 107, 76, 14, 9, 78, 81, 43, 102, 97, 12, 17, 32, 72, 5, 6, 7, 10, 33, 74, 28, 69, 71, 8, 67, 3, 38, 1, 64, 65, 0, 70, 31, 4, 68, 2, 95, 66], [126, 120, 109, 56, 114, 58, 44, 115, 60, 113, 45, 108, 104, 29, 61, 42, 50, 57, 51, 123, 59, 40, 41, 106, 49, 118, 119, 55, 124, 105, 116, 22, 34, 125, 86, 52, 63, 112, 98, 62, 47, 30, 94, 87, 111, 127, 93, 39, 90, 37, 54, 101, 121, 27, 24, 26, 16, 53, 80, 100, 117, 122, 103, 91, 82, 23, 89, 48, 18, 96, 88, 85, 25, 75, 35, 13, 84, 15, 110, 19, 21, 9, 36, 73, 77, 83, 20, 79, 7, 11, 107, 99, 46, 14, 5, 6, 76, 102, 81, 32, 92, 78, 43, 12, 69, 17, 10, 97, 8, 74, 67, 72, 64, 0, 33, 65, 1, 71, 38, 3, 28, 4, 95, 68, 70, 31, 66, 2], [126, 120, 109, 44, 56, 114, 58, 60, 115, 108, 113, 45, 29, 104, 57, 61, 42, 51, 50, 59, 40, 41, 106, 22, 123, 55, 118, 116, 119, 125, 124, 34, 86, 105, 49, 62, 87, 93, 47, 98, 94, 37, 127, 111, 90, 27, 39, 53, 30, 80, 100, 24, 16, 52, 63, 117, 121, 112, 122, 26, 25, 19, 101, 91, 88, 54, 13, 23, 82, 75, 18, 48, 35, 89, 83, 85, 21, 15, 110, 84, 77, 79, 96, 20, 11, 103, 9, 73, 14, 46, 76, 36, 102, 107, 97, 43, 32, 99, 81, 92, 78, 7, 17, 6, 69, 12, 72, 8, 74, 28, 33, 38, 71, 5, 10, 67, 31, 1, 95, 68, 3, 4, 65, 64, 0, 70, 66, 2]], "model.layers.18.self_attn.q_proj": [[39, 52, 112, 113, 32, 26, 87, 85, 121, 49, 29, 90, 74, 79, 116, 70, 77, 20, 53, 93, 82, 80, 46, 44, 23, 125, 119, 0, 75, 57, 98, 124, 18, 55, 61, 68, 111, 3, 48, 60, 54, 123, 115, 110, 62, 106, 65, 126, 19, 7, 117, 107, 84, 99, 118, 38, 56, 96, 120, 42, 47, 43, 9, 50, 109, 8, 2, 40, 51, 58, 11, 105, 21, 17, 25, 97, 41, 101, 59, 102, 122, 35, 24, 45, 6, 127, 63, 89, 104, 94, 33, 37, 30, 114, 100, 28, 108, 66, 22, 34, 14, 86, 36, 31, 88, 27, 92, 83, 103, 15, 69, 91, 10, 95, 81, 5, 16, 76, 71, 72, 1, 78, 12, 73, 67, 13, 4, 64], [39, 113, 52, 110, 112, 49, 29, 46, 87, 85, 32, 116, 90, 53, 26, 80, 93, 60, 83, 92, 77, 125, 115, 108, 23, 44, 123, 79, 21, 31, 121, 98, 119, 56, 111, 88, 58, 8, 48, 124, 107, 61, 105, 122, 42, 101, 18, 74, 118, 126, 62, 100, 25, 57, 55, 33, 63, 104, 106, 97, 45, 43, 24, 70, 117, 54, 99, 35, 38, 41, 82, 89, 47, 114, 120, 59, 50, 40, 109, 51, 127, 102, 36, 86, 68, 12, 11, 28, 17, 78, 91, 96, 94, 37, 15, 22, 34, 84, 9, 95, 30, 20, 7, 76, 19, 13, 27, 72, 75, 6, 16, 81, 73, 14, 2, 4, 64, 71, 65, 3, 69, 10, 0, 103, 1, 67, 66, 5], [39, 113, 52, 49, 29, 85, 32, 87, 116, 90, 26, 121, 79, 60, 125, 110, 46, 93, 112, 119, 53, 44, 21, 77, 124, 111, 122, 59, 61, 96, 109, 118, 123, 80, 98, 84, 56, 115, 54, 82, 47, 55, 57, 18, 23, 8, 74, 108, 83, 101, 120, 50, 92, 43, 22, 88, 89, 37, 31, 48, 97, 127, 42, 41, 51, 11, 105, 38, 35, 91, 106, 45, 62, 58, 75, 107, 36, 104, 78, 70, 117, 33, 63, 28, 114, 126, 94, 15, 19, 86, 95, 102, 40, 30, 27, 34, 9, 99, 17, 100, 24, 16, 76, 68, 25, 20, 71, 7, 81, 65, 3, 14, 12, 103, 2, 13, 1, 0, 72, 73, 4, 67, 64, 10, 6, 5, 69, 66], [39, 113, 52, 112, 110, 49, 29, 32, 85, 116, 77, 122, 87, 26, 31, 98, 46, 93, 80, 48, 8, 119, 50, 83, 70, 79, 53, 18, 123, 120, 74, 44, 125, 111, 114, 60, 42, 58, 124, 88, 57, 100, 92, 21, 117, 118, 90, 56, 115, 23, 121, 61, 35, 7, 108, 45, 27, 109, 63, 101, 25, 37, 62, 43, 0, 59, 82, 51, 54, 28, 6, 86, 68, 99, 102, 36, 104, 107, 41, 97, 38, 47, 55, 106, 105, 126, 127, 69, 40, 33, 34, 12, 3, 94, 95, 13, 91, 11, 30, 78, 19, 24, 84, 22, 15, 96, 81, 89, 20, 65, 67, 66, 9, 2, 73, 75, 4, 71, 76, 72, 16, 14, 17, 10, 1, 64, 5, 103], [124, 63, 56, 100, 115, 36, 39, 110, 102, 122, 116, 111, 59, 54, 105, 58, 103, 57, 121, 120, 60, 49, 62, 127, 50, 45, 25, 117, 125, 47, 112, 123, 55, 48, 114, 53, 109, 93, 126, 35, 52, 42, 40, 113, 51, 101, 41, 34, 119, 46, 87, 118, 107, 108, 61, 32, 44, 99, 104, 43, 38, 106, 37, 80, 90, 96, 94, 20, 30, 98, 23, 24, 97, 33, 84, 27, 31, 95, 29, 73, 85, 91, 89, 19, 28, 22, 21, 92, 78, 26, 88, 11, 75, 16, 86, 18, 69, 83, 77, 82, 13, 7, 9, 14, 1, 71, 5, 6, 66, 17, 3, 2, 79, 81, 67, 4, 15, 70, 12, 72, 74, 10, 65, 76, 64, 0, 8, 68], [63, 124, 30, 39, 115, 100, 19, 36, 25, 96, 22, 12, 18, 17, 15, 99, 27, 89, 32, 13, 72, 24, 5, 103, 35, 74, 92, 6, 123, 122, 56, 45, 2, 54, 4, 1, 90, 120, 126, 28, 33, 71, 21, 121, 42, 85, 14, 113, 26, 98, 83, 125, 73, 84, 75, 11, 94, 108, 86, 117, 23, 69, 52, 9, 10, 0, 81, 20, 97, 7, 67, 101, 38, 112, 87, 118, 93, 60, 43, 29, 16, 57, 34, 47, 37, 78, 91, 107, 80, 82, 88, 3, 58, 105, 79, 44, 65, 127, 119, 48, 70, 59, 62, 95, 53, 40, 109, 51, 66, 76, 31, 102, 111, 104, 49, 77, 106, 110, 114, 50, 68, 64, 61, 55, 116, 41, 46, 8], [124, 63, 100, 36, 120, 98, 103, 38, 25, 96, 32, 84, 19, 87, 97, 45, 94, 101, 126, 39, 22, 56, 51, 89, 117, 59, 121, 58, 113, 27, 21, 17, 62, 34, 54, 31, 23, 107, 40, 86, 52, 33, 13, 47, 111, 48, 46, 112, 102, 90, 122, 127, 99, 15, 60, 24, 57, 43, 91, 125, 83, 116, 29, 118, 44, 26, 28, 119, 110, 114, 30, 88, 109, 106, 49, 61, 93, 123, 105, 115, 35, 55, 53, 108, 12, 80, 85, 37, 78, 42, 20, 11, 104, 41, 81, 18, 95, 92, 50, 7, 14, 16, 82, 5, 77, 6, 74, 79, 75, 73, 72, 69, 76, 4, 2, 71, 9, 64, 68, 70, 8, 10, 66, 65, 1, 67, 3, 0], [124, 63, 24, 100, 21, 30, 18, 17, 12, 22, 15, 39, 115, 8, 74, 72, 27, 4, 90, 96, 79, 84, 13, 92, 76, 36, 1, 25, 94, 122, 68, 64, 75, 26, 77, 51, 121, 0, 6, 7, 3, 83, 2, 33, 85, 69, 81, 31, 10, 28, 70, 19, 86, 89, 46, 14, 5, 78, 67, 11, 82, 66, 20, 16, 71, 23, 80, 102, 88, 65, 73, 87, 93, 29, 43, 111, 103, 95, 32, 9, 127, 97, 35, 91, 98, 114, 126, 42, 125, 44, 37, 99, 56, 40, 61, 45, 120, 52, 58, 34, 104, 105, 119, 50, 101, 118, 38, 53, 59, 108, 107, 123, 55, 106, 112, 41, 49, 116, 60, 57, 109, 110, 113, 48, 54, 117, 47, 62], [55, 102, 118, 123, 46, 97, 61, 62, 53, 116, 106, 21, 120, 27, 57, 51, 115, 112, 126, 58, 122, 119, 48, 49, 63, 114, 91, 124, 50, 45, 31, 109, 52, 117, 24, 99, 110, 127, 60, 95, 41, 125, 121, 54, 113, 56, 103, 111, 59, 14, 47, 108, 104, 44, 38, 87, 88, 40, 18, 28, 19, 78, 70, 6, 42, 39, 26, 105, 107, 92, 86, 43, 101, 36, 37, 100, 93, 98, 96, 35, 34, 32, 65, 33, 82, 23, 94, 25, 30, 29, 0, 89, 16, 84, 90, 85, 22, 76, 80, 64, 17, 1, 3, 68, 20, 4, 83, 67, 11, 12, 8, 15, 66, 81, 2, 71, 72, 5, 9, 75, 7, 69, 74, 79, 77, 73, 13, 10], [118, 102, 55, 123, 48, 97, 53, 62, 61, 27, 46, 24, 57, 91, 51, 21, 116, 120, 31, 38, 122, 126, 115, 50, 49, 119, 124, 112, 58, 114, 63, 110, 40, 117, 95, 106, 109, 52, 54, 113, 99, 121, 18, 127, 56, 86, 88, 125, 45, 47, 111, 41, 60, 59, 104, 14, 44, 39, 43, 103, 108, 100, 107, 42, 92, 33, 105, 26, 101, 94, 98, 37, 36, 32, 78, 19, 35, 70, 34, 96, 29, 93, 30, 25, 28, 22, 17, 23, 84, 87, 82, 90, 6, 16, 85, 89, 80, 83, 20, 76, 12, 65, 15, 81, 3, 0, 1, 8, 75, 9, 71, 73, 11, 68, 79, 67, 64, 5, 4, 66, 77, 7, 2, 69, 74, 72, 10, 13], [102, 55, 118, 97, 123, 86, 24, 84, 46, 15, 27, 82, 17, 88, 77, 90, 74, 18, 48, 11, 72, 76, 30, 38, 120, 91, 54, 8, 95, 69, 92, 106, 93, 62, 71, 104, 22, 5, 16, 3, 20, 9, 81, 29, 14, 119, 75, 10, 31, 7, 116, 13, 19, 79, 21, 89, 45, 25, 85, 26, 126, 83, 28, 80, 23, 12, 67, 122, 101, 94, 40, 1, 87, 33, 108, 98, 70, 124, 49, 78, 109, 61, 96, 73, 36, 64, 65, 53, 68, 6, 114, 32, 103, 34, 0, 66, 57, 4, 37, 35, 60, 2, 107, 99, 100, 59, 110, 44, 125, 58, 41, 113, 105, 42, 47, 39, 112, 43, 50, 111, 51, 117, 56, 63, 52, 127, 121, 115], [102, 118, 55, 97, 123, 84, 24, 86, 17, 46, 27, 15, 31, 62, 91, 14, 76, 90, 18, 38, 88, 73, 95, 22, 79, 101, 82, 120, 81, 53, 61, 48, 11, 54, 12, 104, 75, 80, 19, 74, 49, 106, 28, 83, 20, 71, 29, 119, 122, 6, 116, 23, 85, 51, 45, 30, 77, 10, 25, 126, 69, 96, 87, 21, 9, 89, 112, 16, 26, 2, 108, 92, 93, 7, 32, 57, 5, 13, 94, 109, 124, 67, 65, 35, 58, 98, 115, 125, 37, 114, 78, 72, 103, 8, 34, 63, 50, 59, 52, 110, 44, 99, 100, 111, 40, 60, 113, 127, 121, 47, 3, 56, 39, 66, 117, 36, 0, 42, 68, 41, 105, 70, 107, 43, 64, 1, 33, 4], [43, 36, 54, 96, 63, 60, 89, 20, 107, 82, 126, 50, 87, 115, 80, 62, 25, 21, 9, 52, 16, 61, 32, 79, 13, 48, 101, 112, 47, 75, 53, 39, 57, 29, 85, 41, 69, 71, 103, 122, 119, 111, 19, 106, 105, 35, 104, 45, 18, 84, 55, 86, 23, 40, 127, 94, 46, 59, 125, 99, 88, 44, 93, 121, 28, 26, 90, 38, 100, 58, 92, 97, 51, 78, 6, 98, 110, 108, 120, 33, 10, 37, 114, 15, 74, 27, 123, 124, 7, 116, 56, 17, 102, 83, 113, 118, 24, 49, 67, 77, 95, 12, 8, 11, 109, 34, 31, 14, 22, 72, 30, 91, 117, 81, 76, 42, 66, 73, 4, 1, 5, 2, 70, 64, 3, 0, 65, 68], [43, 60, 96, 36, 63, 107, 89, 48, 123, 20, 126, 82, 21, 119, 46, 111, 25, 19, 85, 44, 80, 26, 42, 55, 112, 23, 13, 51, 122, 79, 9, 45, 61, 54, 18, 75, 41, 53, 59, 87, 16, 62, 50, 39, 105, 69, 52, 57, 103, 38, 40, 106, 116, 27, 121, 113, 101, 71, 114, 92, 109, 84, 49, 32, 58, 115, 35, 104, 95, 117, 124, 108, 125, 56, 86, 33, 11, 90, 127, 37, 81, 110, 88, 98, 29, 99, 83, 118, 34, 100, 30, 102, 24, 7, 120, 93, 97, 17, 91, 47, 94, 77, 1, 31, 22, 28, 8, 15, 0, 14, 72, 3, 73, 78, 67, 12, 10, 6, 2, 74, 5, 68, 76, 70, 66, 4, 64, 65], [43, 96, 54, 36, 63, 47, 20, 89, 107, 25, 60, 119, 82, 9, 80, 16, 87, 21, 69, 79, 18, 111, 28, 71, 13, 48, 55, 112, 75, 120, 53, 38, 32, 126, 19, 61, 123, 41, 62, 50, 115, 51, 84, 85, 57, 110, 104, 40, 103, 39, 45, 29, 67, 106, 122, 125, 1, 118, 59, 95, 26, 37, 116, 44, 64, 121, 56, 127, 76, 58, 100, 113, 108, 49, 7, 114, 102, 65, 105, 52, 83, 46, 94, 109, 124, 117, 91, 86, 90, 101, 99, 34, 92, 23, 30, 35, 22, 42, 73, 98, 27, 93, 4, 97, 33, 6, 14, 66, 0, 72, 81, 74, 78, 15, 88, 31, 11, 5, 24, 10, 3, 68, 2, 17, 8, 77, 12, 70], [43, 54, 96, 36, 60, 89, 107, 82, 20, 87, 75, 63, 25, 80, 79, 71, 111, 13, 9, 47, 29, 69, 85, 16, 21, 77, 126, 97, 40, 32, 41, 38, 92, 106, 18, 119, 115, 28, 34, 117, 53, 121, 67, 19, 44, 27, 46, 57, 109, 58, 50, 103, 14, 48, 101, 10, 84, 35, 55, 122, 45, 93, 65, 95, 1, 114, 91, 23, 118, 51, 12, 37, 2, 83, 112, 15, 104, 88, 94, 31, 78, 123, 62, 30, 66, 59, 116, 99, 90, 11, 100, 64, 26, 61, 42, 127, 22, 56, 113, 33, 49, 7, 120, 24, 74, 81, 76, 73, 39, 110, 125, 98, 8, 124, 52, 17, 108, 5, 86, 4, 102, 105, 6, 70, 72, 68, 0, 3], [120, 104, 59, 113, 109, 51, 62, 110, 22, 56, 88, 57, 119, 112, 90, 52, 83, 41, 21, 114, 36, 98, 55, 116, 107, 50, 103, 29, 115, 123, 93, 54, 95, 43, 38, 42, 124, 49, 60, 122, 99, 45, 117, 105, 126, 102, 61, 48, 106, 58, 33, 30, 127, 108, 39, 118, 47, 125, 44, 46, 111, 53, 27, 121, 63, 101, 37, 97, 35, 28, 100, 92, 31, 96, 80, 10, 32, 34, 79, 86, 40, 74, 17, 25, 91, 94, 81, 19, 76, 23, 85, 77, 84, 26, 13, 24, 89, 14, 15, 20, 71, 87, 72, 82, 16, 18, 7, 12, 11, 67, 69, 3, 9, 78, 5, 6, 75, 8, 1, 70, 65, 68, 73, 0, 66, 4, 2, 64], [104, 120, 59, 113, 98, 88, 18, 80, 65, 22, 6, 76, 93, 9, 110, 15, 14, 62, 68, 40, 112, 119, 72, 57, 83, 84, 97, 32, 51, 28, 36, 100, 26, 123, 29, 1, 67, 82, 74, 52, 11, 5, 8, 24, 70, 81, 99, 61, 4, 27, 86, 21, 91, 90, 46, 19, 107, 75, 66, 117, 34, 55, 116, 33, 58, 77, 16, 124, 54, 37, 49, 114, 118, 50, 31, 23, 25, 60, 13, 42, 109, 127, 115, 53, 78, 38, 3, 87, 106, 17, 126, 89, 122, 30, 69, 101, 12, 20, 92, 108, 64, 85, 105, 71, 41, 56, 45, 95, 96, 35, 0, 7, 102, 94, 43, 125, 48, 44, 103, 10, 111, 47, 2, 63, 79, 73, 121, 39], [113, 104, 120, 59, 88, 98, 22, 93, 52, 51, 83, 57, 29, 21, 112, 80, 90, 119, 62, 110, 124, 114, 76, 28, 116, 81, 55, 123, 50, 11, 95, 18, 122, 117, 118, 15, 108, 54, 49, 42, 96, 115, 58, 61, 41, 85, 100, 39, 14, 56, 53, 47, 84, 46, 109, 60, 127, 44, 125, 91, 27, 92, 31, 36, 86, 48, 43, 103, 35, 26, 101, 126, 105, 97, 45, 40, 121, 106, 38, 82, 6, 99, 72, 111, 19, 17, 102, 65, 63, 13, 30, 79, 25, 74, 24, 107, 23, 33, 37, 94, 87, 10, 77, 32, 89, 9, 34, 12, 20, 5, 75, 68, 3, 8, 71, 66, 16, 69, 7, 67, 78, 0, 2, 1, 4, 64, 70, 73], [59, 104, 120, 113, 112, 57, 62, 51, 83, 36, 43, 55, 105, 89, 119, 52, 50, 98, 95, 60, 110, 124, 49, 126, 115, 90, 35, 121, 117, 122, 88, 45, 97, 92, 56, 109, 123, 21, 61, 118, 102, 54, 116, 58, 42, 46, 103, 127, 17, 107, 114, 47, 48, 44, 15, 125, 23, 108, 41, 106, 63, 74, 34, 53, 100, 39, 101, 32, 99, 37, 111, 38, 19, 10, 26, 29, 30, 96, 13, 93, 28, 24, 33, 25, 31, 81, 94, 84, 79, 91, 5, 67, 40, 85, 87, 3, 71, 20, 0, 77, 22, 65, 69, 27, 14, 64, 18, 7, 76, 11, 75, 66, 86, 1, 68, 9, 72, 6, 82, 80, 2, 8, 4, 12, 78, 73, 70, 16], [127, 54, 122, 125, 100, 62, 126, 119, 120, 41, 40, 36, 42, 28, 117, 106, 118, 56, 83, 115, 92, 112, 58, 124, 59, 113, 39, 55, 49, 46, 57, 111, 53, 123, 63, 86, 52, 116, 99, 61, 114, 121, 48, 104, 47, 60, 51, 50, 105, 107, 37, 103, 108, 82, 110, 43, 45, 44, 38, 109, 35, 15, 74, 94, 101, 33, 25, 102, 98, 29, 34, 31, 80, 20, 96, 89, 90, 71, 95, 77, 24, 79, 97, 32, 88, 67, 26, 22, 75, 30, 91, 93, 65, 19, 27, 18, 23, 21, 6, 14, 85, 87, 10, 7, 68, 66, 84, 0, 13, 76, 73, 17, 78, 4, 1, 64, 8, 81, 16, 2, 12, 11, 70, 9, 72, 3, 69, 5], [122, 54, 127, 40, 100, 125, 24, 90, 33, 31, 62, 120, 105, 99, 19, 36, 86, 26, 126, 28, 106, 119, 53, 25, 43, 81, 102, 41, 98, 20, 88, 87, 49, 69, 35, 94, 115, 85, 59, 76, 66, 73, 61, 32, 7, 104, 56, 91, 27, 52, 34, 15, 110, 23, 8, 57, 38, 83, 72, 22, 75, 114, 29, 58, 63, 12, 78, 5, 101, 3, 109, 113, 92, 117, 48, 123, 42, 60, 111, 51, 17, 124, 16, 116, 37, 70, 103, 74, 112, 93, 55, 121, 96, 47, 82, 10, 21, 13, 50, 11, 9, 14, 30, 95, 46, 118, 39, 1, 89, 4, 80, 108, 107, 45, 77, 6, 44, 67, 18, 79, 84, 2, 71, 64, 65, 68, 0, 97], [54, 122, 127, 100, 41, 40, 92, 33, 106, 120, 125, 126, 36, 24, 103, 28, 83, 86, 119, 117, 94, 32, 26, 99, 115, 15, 42, 62, 49, 31, 59, 90, 29, 21, 105, 57, 96, 88, 53, 104, 82, 124, 113, 56, 39, 58, 95, 63, 74, 112, 35, 38, 34, 16, 123, 114, 55, 118, 48, 80, 61, 50, 98, 52, 37, 22, 51, 121, 46, 89, 101, 27, 102, 23, 47, 97, 85, 19, 77, 116, 111, 60, 107, 20, 43, 30, 110, 25, 87, 91, 13, 79, 109, 81, 44, 93, 75, 108, 76, 18, 12, 45, 71, 78, 14, 11, 17, 84, 8, 6, 10, 73, 9, 66, 4, 67, 72, 65, 7, 0, 69, 68, 70, 1, 64, 3, 5, 2], [127, 122, 54, 100, 41, 40, 126, 86, 26, 33, 28, 120, 15, 36, 42, 43, 117, 119, 96, 125, 83, 106, 49, 53, 57, 59, 21, 92, 118, 39, 124, 38, 115, 88, 103, 62, 107, 56, 24, 58, 35, 19, 114, 46, 108, 95, 112, 61, 37, 48, 105, 110, 81, 52, 123, 47, 63, 55, 113, 102, 17, 104, 50, 60, 111, 97, 116, 31, 99, 34, 98, 121, 51, 101, 44, 30, 22, 45, 13, 84, 109, 82, 32, 74, 77, 10, 94, 29, 11, 18, 93, 90, 89, 91, 87, 78, 27, 23, 14, 71, 20, 7, 85, 25, 79, 80, 75, 16, 67, 76, 72, 73, 8, 6, 4, 66, 1, 65, 12, 68, 9, 0, 64, 3, 70, 69, 2, 5], [105, 98, 117, 86, 96, 28, 53, 20, 52, 116, 25, 41, 94, 38, 80, 126, 109, 56, 63, 119, 123, 45, 111, 82, 19, 14, 44, 83, 62, 118, 71, 75, 120, 89, 77, 87, 115, 54, 78, 8, 49, 21, 112, 79, 23, 48, 51, 10, 50, 58, 57, 102, 61, 30, 46, 55, 59, 37, 39, 124, 27, 114, 125, 6, 107, 100, 127, 16, 43, 40, 93, 31, 35, 101, 104, 29, 60, 103, 121, 26, 42, 97, 110, 2, 85, 122, 4, 95, 113, 106, 17, 33, 36, 99, 47, 108, 9, 90, 34, 24, 22, 91, 88, 84, 18, 7, 32, 64, 81, 1, 92, 76, 12, 11, 15, 67, 68, 13, 73, 70, 0, 74, 66, 3, 65, 72, 5, 69], [105, 98, 96, 86, 20, 117, 82, 53, 75, 25, 41, 123, 79, 80, 52, 77, 63, 8, 116, 28, 94, 6, 126, 10, 16, 118, 89, 2, 119, 112, 4, 59, 62, 56, 45, 115, 44, 91, 111, 3, 109, 11, 57, 120, 13, 30, 88, 84, 26, 23, 54, 48, 97, 7, 51, 38, 17, 93, 50, 22, 74, 127, 67, 29, 114, 83, 18, 0, 58, 14, 31, 95, 102, 21, 61, 85, 78, 70, 39, 76, 104, 81, 19, 40, 107, 55, 92, 36, 15, 5, 24, 60, 64, 101, 125, 35, 121, 124, 106, 27, 113, 68, 87, 32, 1, 37, 90, 33, 72, 43, 49, 47, 110, 73, 34, 69, 99, 66, 65, 9, 103, 46, 122, 12, 42, 100, 108, 71], [105, 98, 86, 96, 117, 82, 8, 53, 79, 25, 63, 20, 75, 80, 4, 77, 41, 94, 2, 64, 123, 62, 6, 89, 48, 116, 118, 0, 115, 52, 72, 28, 66, 1, 29, 68, 13, 126, 16, 71, 69, 70, 87, 44, 85, 10, 112, 30, 65, 51, 109, 45, 11, 59, 119, 56, 57, 50, 88, 5, 18, 7, 24, 26, 93, 19, 111, 120, 35, 22, 78, 17, 15, 127, 21, 84, 3, 74, 76, 58, 39, 114, 12, 9, 100, 92, 60, 106, 23, 73, 14, 83, 102, 99, 67, 27, 40, 38, 121, 55, 31, 95, 91, 97, 61, 49, 101, 113, 122, 36, 124, 34, 81, 47, 33, 37, 90, 46, 54, 110, 32, 125, 107, 42, 103, 104, 43, 108], [105, 98, 20, 28, 96, 25, 86, 82, 41, 117, 53, 123, 80, 94, 63, 116, 77, 75, 52, 118, 79, 126, 115, 16, 59, 56, 44, 8, 89, 111, 62, 119, 10, 50, 57, 17, 120, 48, 109, 112, 51, 71, 45, 127, 84, 18, 12, 13, 30, 38, 22, 23, 6, 91, 95, 31, 55, 83, 78, 81, 76, 88, 54, 27, 61, 60, 19, 39, 74, 93, 46, 33, 124, 24, 104, 106, 7, 49, 34, 37, 26, 114, 58, 4, 102, 43, 107, 122, 11, 99, 15, 9, 2, 125, 87, 97, 14, 85, 92, 36, 90, 29, 21, 110, 101, 72, 100, 32, 35, 121, 40, 42, 68, 103, 70, 113, 69, 108, 47, 3, 1, 0, 64, 66, 5, 73, 65, 67], [112, 105, 113, 125, 110, 41, 114, 89, 126, 104, 117, 54, 61, 27, 84, 127, 57, 59, 58, 56, 107, 123, 30, 91, 99, 48, 52, 51, 46, 111, 49, 108, 47, 55, 120, 92, 42, 115, 101, 119, 53, 94, 122, 43, 109, 23, 118, 44, 116, 62, 86, 25, 121, 124, 106, 50, 63, 60, 34, 102, 36, 45, 39, 37, 100, 40, 98, 33, 38, 35, 103, 90, 21, 97, 32, 15, 96, 28, 82, 95, 88, 31, 79, 18, 80, 29, 93, 81, 26, 87, 85, 17, 20, 22, 24, 83, 13, 12, 19, 78, 3, 14, 76, 69, 74, 75, 65, 77, 16, 11, 10, 9, 8, 71, 72, 73, 6, 5, 70, 7, 67, 68, 66, 1, 2, 4, 64, 0], [105, 112, 86, 30, 41, 84, 91, 80, 82, 78, 27, 76, 98, 99, 89, 94, 126, 113, 9, 10, 7, 110, 61, 114, 74, 34, 25, 52, 71, 35, 127, 14, 58, 125, 20, 49, 23, 109, 88, 67, 54, 69, 57, 59, 3, 123, 56, 117, 19, 47, 13, 46, 62, 5, 22, 111, 18, 108, 81, 48, 51, 118, 55, 77, 116, 17, 21, 85, 16, 107, 43, 120, 115, 119, 37, 79, 40, 32, 73, 50, 122, 92, 11, 60, 93, 24, 102, 83, 53, 124, 65, 63, 28, 121, 15, 106, 8, 4, 104, 97, 36, 44, 87, 64, 42, 45, 90, 12, 95, 38, 39, 29, 26, 103, 33, 75, 31, 101, 96, 72, 100, 2, 70, 66, 0, 1, 6, 68], [112, 105, 86, 91, 41, 30, 84, 25, 27, 58, 126, 89, 15, 62, 80, 113, 106, 82, 33, 111, 114, 110, 17, 127, 116, 104, 52, 124, 78, 94, 83, 117, 76, 49, 61, 46, 48, 39, 125, 59, 92, 57, 11, 56, 107, 10, 85, 47, 98, 36, 99, 54, 13, 51, 35, 123, 45, 7, 87, 60, 101, 23, 32, 119, 63, 108, 18, 22, 29, 6, 121, 120, 115, 44, 12, 109, 71, 55, 93, 21, 97, 40, 43, 96, 118, 20, 95, 72, 34, 16, 37, 31, 53, 122, 28, 42, 88, 50, 26, 38, 103, 90, 69, 2, 9, 79, 24, 102, 81, 100, 64, 14, 19, 67, 68, 75, 74, 5, 77, 8, 73, 4, 66, 1, 70, 65, 0, 3], [105, 112, 30, 84, 86, 89, 41, 98, 126, 82, 48, 113, 118, 123, 110, 25, 80, 35, 85, 45, 31, 91, 99, 52, 54, 94, 88, 61, 125, 78, 27, 58, 115, 9, 59, 127, 92, 116, 109, 117, 39, 114, 33, 17, 57, 12, 60, 32, 49, 121, 95, 76, 108, 6, 40, 73, 56, 111, 47, 74, 120, 18, 103, 51, 62, 26, 97, 79, 55, 53, 122, 75, 20, 106, 63, 19, 22, 107, 104, 93, 90, 119, 13, 42, 28, 124, 102, 46, 21, 3, 100, 101, 43, 50, 44, 83, 23, 37, 4, 36, 10, 87, 96, 72, 38, 24, 15, 14, 29, 34, 11, 16, 81, 77, 68, 1, 71, 69, 70, 67, 5, 8, 7, 65, 0, 66, 2, 64]], "model.layers.18.self_attn.k_proj": [[113, 52, 103, 110, 96, 93, 90, 87, 85, 18, 80, 112, 77, 49, 55, 44, 124, 74, 48, 119, 54, 79, 118, 53, 20, 70, 121, 57, 46, 62, 105, 61, 86, 56, 125, 51, 43, 34, 111, 116, 8, 126, 127, 109, 120, 115, 64, 123, 50, 47, 108, 117, 41, 58, 0, 68, 65, 59, 107, 102, 63, 45, 122, 42, 114, 9, 60, 40, 106, 104, 38, 19, 7, 83, 99, 37, 101, 2, 3, 30, 28, 76, 36, 98, 95, 1, 4, 92, 78, 81, 88, 97, 35, 33, 75, 25, 5, 11, 94, 100, 12, 24, 91, 89, 27, 73, 31, 66, 17, 69, 14, 72, 84, 16, 22, 15, 29, 67, 13, 71, 23, 21, 26, 82, 32, 6, 39, 10], [63, 124, 36, 22, 120, 15, 30, 17, 74, 103, 12, 56, 72, 18, 13, 54, 4, 32, 27, 24, 117, 62, 60, 51, 122, 64, 107, 46, 121, 2, 44, 58, 126, 21, 47, 118, 59, 109, 28, 61, 69, 57, 114, 45, 10, 39, 38, 119, 113, 79, 50, 53, 49, 55, 43, 48, 116, 40, 19, 125, 115, 108, 42, 123, 127, 106, 105, 110, 104, 8, 112, 52, 111, 41, 102, 1, 25, 100, 37, 97, 67, 26, 101, 34, 7, 76, 29, 33, 35, 96, 95, 98, 90, 99, 71, 6, 85, 93, 91, 78, 94, 73, 31, 75, 82, 92, 23, 84, 88, 80, 3, 20, 87, 11, 5, 81, 14, 9, 83, 86, 16, 0, 89, 66, 77, 65, 68, 70], [55, 118, 38, 123, 33, 86, 91, 24, 61, 62, 48, 15, 116, 53, 18, 95, 119, 126, 120, 122, 17, 112, 102, 115, 109, 77, 49, 51, 57, 84, 124, 117, 63, 114, 110, 21, 11, 45, 52, 90, 31, 46, 50, 74, 104, 19, 44, 113, 54, 58, 121, 76, 125, 111, 127, 60, 37, 59, 56, 105, 108, 7, 3, 16, 103, 30, 5, 107, 42, 23, 78, 35, 47, 8, 106, 43, 41, 1, 39, 94, 27, 40, 9, 100, 99, 36, 32, 6, 14, 93, 98, 64, 20, 89, 101, 29, 72, 82, 34, 87, 12, 28, 96, 88, 25, 75, 26, 92, 85, 79, 66, 71, 83, 13, 80, 22, 10, 69, 4, 73, 67, 81, 68, 97, 2, 70, 65, 0], [107, 60, 100, 32, 89, 54, 47, 87, 20, 80, 63, 82, 75, 21, 9, 43, 53, 79, 126, 13, 69, 71, 1, 117, 111, 118, 29, 51, 64, 121, 61, 2, 57, 67, 52, 0, 50, 62, 114, 102, 115, 127, 124, 92, 48, 122, 125, 49, 56, 34, 103, 7, 41, 40, 77, 105, 119, 46, 108, 58, 45, 110, 120, 116, 94, 123, 42, 112, 37, 86, 113, 39, 44, 30, 3, 27, 109, 59, 38, 101, 81, 55, 104, 91, 19, 28, 31, 90, 106, 11, 22, 26, 78, 99, 85, 33, 35, 98, 72, 83, 95, 93, 4, 14, 70, 68, 8, 74, 97, 17, 6, 76, 15, 10, 66, 88, 25, 24, 12, 5, 18, 16, 23, 73, 65, 36, 96, 84], [40, 120, 59, 34, 22, 113, 29, 49, 90, 14, 80, 119, 31, 51, 9, 88, 56, 112, 57, 18, 76, 52, 6, 83, 118, 123, 110, 117, 68, 124, 109, 61, 116, 50, 58, 62, 43, 60, 98, 115, 54, 122, 55, 114, 102, 72, 53, 46, 127, 95, 125, 45, 65, 101, 0, 108, 126, 121, 63, 47, 28, 105, 111, 48, 99, 107, 82, 91, 81, 44, 16, 39, 106, 42, 79, 41, 103, 36, 89, 30, 2, 21, 66, 23, 38, 37, 32, 27, 11, 96, 75, 13, 17, 97, 35, 86, 100, 67, 71, 33, 92, 94, 74, 24, 85, 69, 3, 25, 73, 84, 5, 20, 78, 15, 104, 64, 87, 26, 10, 93, 77, 7, 70, 12, 1, 8, 4, 19], [122, 54, 127, 36, 97, 120, 125, 124, 86, 53, 62, 119, 126, 57, 118, 104, 121, 49, 58, 56, 117, 113, 48, 112, 50, 51, 52, 123, 59, 42, 26, 60, 114, 115, 55, 43, 116, 61, 63, 39, 47, 105, 106, 41, 111, 46, 94, 81, 110, 93, 28, 109, 38, 29, 107, 40, 19, 45, 100, 108, 24, 44, 30, 15, 83, 73, 103, 69, 27, 33, 67, 21, 82, 99, 102, 75, 87, 101, 76, 4, 25, 6, 8, 23, 88, 65, 34, 37, 91, 85, 98, 32, 78, 16, 64, 31, 35, 90, 12, 92, 20, 13, 0, 17, 95, 96, 10, 89, 66, 71, 84, 14, 22, 80, 79, 77, 18, 7, 2, 11, 72, 74, 1, 5, 70, 9, 68, 3], [41, 53, 34, 25, 63, 30, 52, 79, 86, 80, 117, 126, 20, 75, 32, 8, 48, 10, 77, 51, 82, 108, 116, 47, 45, 59, 123, 109, 118, 6, 4, 2, 56, 50, 64, 120, 119, 62, 1, 105, 113, 9, 40, 57, 71, 55, 111, 106, 61, 83, 28, 87, 18, 54, 14, 29, 114, 31, 67, 69, 70, 110, 127, 103, 23, 92, 44, 46, 58, 35, 36, 124, 74, 21, 0, 94, 122, 102, 37, 66, 115, 90, 121, 3, 33, 81, 107, 88, 24, 93, 39, 49, 112, 101, 97, 100, 125, 78, 60, 104, 91, 12, 38, 13, 5, 85, 42, 22, 43, 17, 73, 27, 26, 76, 19, 98, 15, 11, 84, 95, 96, 72, 99, 65, 7, 68, 16, 89], [41, 112, 48, 94, 86, 113, 35, 27, 82, 110, 80, 89, 126, 84, 78, 49, 105, 61, 52, 125, 58, 59, 76, 9, 114, 115, 111, 123, 54, 50, 34, 57, 127, 117, 51, 91, 109, 60, 108, 62, 98, 10, 45, 7, 116, 47, 118, 56, 55, 5, 63, 119, 120, 53, 96, 46, 121, 124, 107, 1, 81, 37, 43, 40, 36, 44, 88, 122, 102, 101, 16, 39, 106, 79, 12, 100, 30, 67, 97, 104, 21, 83, 42, 38, 31, 103, 32, 33, 13, 93, 29, 95, 8, 87, 99, 24, 26, 28, 17, 69, 23, 75, 15, 11, 14, 64, 90, 92, 74, 19, 20, 6, 18, 85, 71, 72, 25, 77, 68, 66, 22, 70, 73, 4, 2, 3, 65, 0]], "model.layers.18.self_attn.qk_proj": [[113, 112, 63, 124, 55, 118, 41, 54, 122, 52, 120, 59, 60, 105, 107, 127, 53, 43, 123, 36, 86, 48, 22, 102, 110, 117, 126, 25, 62, 49, 30, 84, 100, 89, 29, 51, 119, 82, 40, 18, 20, 16, 80, 96, 38, 57, 116, 15, 61, 115, 56, 46, 79, 109, 77, 34, 125, 94, 47, 23, 58, 50, 98, 27, 26, 114, 24, 45, 104, 39, 103, 121, 111, 85, 32, 21, 92, 88, 87, 93, 13, 42, 91, 8, 10, 75, 11, 90, 81, 72, 76, 74, 19, 31, 44, 97, 9, 17, 33, 95, 37, 14, 35, 108, 12, 83, 106, 28, 78, 73, 7, 71, 6, 101, 0, 99, 64, 69, 5, 2, 70, 65, 67, 68, 1, 3, 4, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 86, 102, 36, 22, 117, 110, 48, 49, 25, 126, 100, 89, 30, 29, 84, 62, 18, 116, 82, 16, 15, 40, 119, 94, 46, 20, 57, 96, 56, 51, 79, 47, 61, 38, 80, 77, 104, 58, 125, 103, 50, 39, 34, 26, 27, 23, 24, 115, 87, 109, 98, 114, 85, 21, 88, 92, 32, 13, 42, 8, 10, 93, 45, 121, 90, 111, 81, 97, 74, 75, 11, 91, 35, 28, 83, 44, 9, 12, 76, 19, 95, 31, 106, 108, 72, 17, 6, 69, 37, 78, 14, 33, 99, 0, 101, 71, 64, 7, 67, 73, 65, 66, 68, 1, 4, 70, 2, 5, 3], [113, 112, 63, 124, 55, 118, 54, 41, 122, 52, 120, 59, 105, 60, 107, 127, 43, 53, 123, 22, 86, 36, 48, 102, 126, 110, 25, 117, 49, 30, 29, 100, 89, 84, 51, 94, 62, 56, 82, 40, 104, 116, 47, 18, 20, 119, 79, 109, 38, 96, 58, 39, 57, 16, 27, 125, 15, 80, 26, 61, 103, 114, 23, 98, 115, 46, 50, 34, 45, 24, 85, 87, 77, 13, 88, 121, 90, 91, 8, 32, 21, 42, 92, 111, 75, 93, 11, 10, 19, 6, 33, 81, 106, 44, 64, 28, 31, 97, 108, 83, 35, 12, 95, 76, 72, 9, 14, 74, 0, 37, 2, 101, 7, 69, 65, 17, 78, 67, 73, 3, 1, 66, 5, 4, 68, 71, 70, 99], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 36, 117, 48, 22, 102, 126, 110, 49, 86, 25, 62, 51, 116, 89, 47, 82, 58, 100, 56, 30, 125, 119, 96, 46, 29, 20, 61, 18, 15, 94, 79, 57, 84, 39, 16, 115, 40, 103, 80, 38, 114, 50, 104, 109, 34, 27, 26, 24, 13, 23, 98, 75, 111, 77, 45, 121, 87, 85, 21, 93, 32, 97, 106, 8, 91, 90, 42, 92, 10, 88, 44, 76, 74, 81, 35, 11, 0, 6, 108, 83, 37, 72, 95, 17, 64, 31, 28, 3, 12, 14, 66, 2, 78, 1, 73, 19, 33, 9, 7, 70, 71, 4, 69, 101, 5, 68, 65, 99, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 122, 120, 60, 59, 105, 107, 127, 53, 43, 123, 48, 22, 110, 117, 36, 86, 126, 25, 62, 49, 116, 102, 125, 82, 51, 89, 30, 20, 56, 47, 57, 84, 16, 79, 100, 18, 119, 109, 29, 58, 61, 15, 40, 94, 115, 38, 96, 104, 98, 111, 46, 80, 26, 103, 50, 23, 114, 34, 13, 39, 77, 45, 121, 21, 8, 27, 24, 88, 85, 106, 10, 93, 75, 90, 32, 87, 44, 42, 81, 92, 12, 97, 91, 11, 108, 19, 28, 74, 76, 78, 72, 14, 33, 31, 17, 83, 95, 73, 35, 37, 6, 101, 70, 9, 0, 7, 66, 64, 3, 69, 65, 2, 4, 71, 68, 5, 1, 99, 67], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 86, 110, 117, 36, 126, 102, 25, 51, 49, 15, 89, 30, 82, 18, 62, 116, 79, 20, 100, 57, 96, 40, 80, 125, 16, 84, 109, 119, 47, 29, 58, 50, 39, 115, 56, 34, 94, 98, 103, 61, 104, 46, 23, 114, 77, 38, 13, 24, 21, 26, 27, 111, 85, 88, 8, 32, 87, 42, 93, 75, 10, 11, 97, 90, 45, 92, 81, 78, 12, 83, 74, 31, 17, 121, 28, 76, 106, 91, 44, 19, 72, 95, 14, 9, 35, 33, 108, 71, 37, 73, 101, 99, 70, 64, 0, 6, 69, 7, 2, 1, 66, 65, 5, 67, 4, 68, 3], [113, 63, 112, 124, 55, 118, 41, 54, 122, 52, 120, 60, 59, 105, 107, 43, 127, 53, 123, 22, 86, 48, 36, 126, 49, 25, 110, 117, 30, 102, 82, 18, 62, 89, 96, 84, 15, 29, 51, 47, 16, 100, 80, 20, 79, 40, 58, 56, 57, 119, 94, 77, 104, 23, 98, 34, 38, 115, 27, 61, 116, 125, 26, 32, 109, 24, 88, 50, 46, 114, 103, 13, 85, 91, 39, 21, 75, 93, 111, 87, 92, 121, 8, 10, 74, 97, 11, 90, 78, 31, 12, 81, 35, 33, 95, 76, 42, 44, 19, 17, 14, 101, 45, 106, 83, 9, 108, 73, 72, 28, 99, 70, 71, 37, 64, 7, 5, 65, 69, 2, 66, 0, 1, 68, 67, 6, 4, 3], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 22, 53, 123, 36, 86, 102, 126, 48, 117, 25, 30, 110, 49, 119, 62, 29, 84, 89, 82, 16, 20, 56, 18, 96, 100, 40, 47, 80, 51, 94, 57, 79, 15, 98, 116, 38, 125, 46, 23, 26, 61, 27, 104, 50, 115, 24, 58, 39, 85, 88, 34, 91, 13, 32, 114, 109, 77, 21, 87, 103, 121, 93, 90, 111, 92, 33, 75, 19, 31, 17, 97, 10, 42, 95, 11, 81, 8, 45, 44, 14, 108, 78, 9, 83, 76, 106, 12, 70, 35, 74, 72, 28, 99, 37, 73, 64, 101, 5, 71, 7, 1, 0, 68, 2, 69, 3, 4, 66, 65, 67, 6], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 22, 123, 36, 102, 110, 86, 126, 48, 117, 25, 89, 49, 30, 84, 57, 119, 96, 29, 20, 18, 79, 82, 40, 38, 62, 94, 116, 16, 100, 56, 47, 46, 125, 80, 15, 51, 115, 27, 58, 61, 104, 24, 98, 39, 34, 88, 26, 32, 87, 13, 114, 21, 23, 103, 77, 85, 93, 50, 92, 90, 97, 11, 109, 111, 75, 28, 106, 108, 81, 91, 72, 10, 19, 121, 95, 17, 31, 12, 42, 35, 74, 33, 76, 45, 8, 44, 83, 78, 70, 14, 101, 37, 99, 73, 9, 7, 5, 71, 64, 68, 0, 69, 65, 3, 4, 67, 1, 2, 66, 6], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 102, 117, 36, 22, 86, 126, 25, 110, 62, 49, 57, 100, 116, 125, 18, 51, 82, 30, 47, 89, 40, 119, 20, 56, 29, 58, 84, 79, 61, 80, 38, 16, 96, 15, 46, 27, 104, 26, 94, 34, 115, 39, 98, 50, 24, 85, 103, 77, 21, 32, 88, 72, 13, 109, 75, 23, 10, 93, 45, 87, 106, 111, 121, 92, 114, 97, 31, 74, 90, 76, 81, 108, 35, 95, 19, 91, 44, 28, 11, 8, 42, 83, 78, 12, 14, 17, 9, 70, 71, 37, 73, 7, 101, 64, 33, 5, 99, 67, 68, 3, 0, 65, 4, 69, 6, 2, 1, 66], [113, 112, 63, 124, 55, 54, 41, 118, 120, 122, 52, 60, 59, 105, 107, 127, 53, 43, 123, 117, 48, 22, 126, 36, 102, 86, 110, 116, 62, 25, 51, 18, 49, 56, 82, 89, 20, 100, 61, 79, 96, 47, 57, 30, 80, 125, 15, 84, 40, 119, 16, 46, 58, 38, 29, 104, 77, 50, 98, 115, 39, 34, 72, 94, 13, 23, 109, 24, 114, 103, 27, 26, 45, 75, 85, 87, 32, 111, 93, 121, 21, 10, 92, 11, 88, 90, 8, 28, 81, 106, 76, 12, 74, 17, 97, 14, 83, 78, 6, 35, 91, 42, 73, 44, 64, 71, 37, 31, 2, 108, 19, 9, 69, 0, 95, 7, 33, 101, 68, 70, 1, 65, 5, 4, 67, 66, 3, 99], [113, 63, 112, 124, 55, 41, 118, 54, 52, 120, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 86, 48, 102, 36, 117, 126, 25, 89, 84, 18, 110, 82, 30, 100, 49, 96, 80, 15, 20, 79, 29, 62, 51, 40, 56, 16, 116, 98, 47, 46, 94, 57, 115, 119, 77, 85, 23, 38, 58, 125, 104, 61, 34, 13, 24, 26, 72, 27, 75, 87, 103, 39, 109, 21, 32, 50, 93, 11, 88, 111, 74, 10, 90, 114, 97, 92, 19, 81, 45, 121, 31, 91, 17, 78, 14, 95, 12, 106, 76, 6, 44, 33, 108, 8, 9, 35, 83, 42, 73, 28, 99, 101, 71, 7, 37, 64, 68, 5, 2, 70, 1, 66, 3, 69, 0, 4, 67, 65], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 123, 53, 36, 48, 102, 86, 22, 117, 25, 110, 126, 30, 100, 62, 29, 96, 49, 89, 38, 18, 82, 40, 51, 20, 94, 84, 47, 80, 79, 58, 61, 16, 15, 56, 125, 116, 115, 57, 119, 98, 46, 23, 24, 39, 27, 88, 26, 104, 21, 85, 34, 32, 103, 50, 87, 90, 97, 109, 121, 13, 93, 77, 111, 91, 31, 72, 92, 75, 19, 45, 114, 10, 95, 11, 106, 35, 17, 83, 44, 28, 42, 81, 33, 74, 78, 14, 76, 12, 37, 73, 108, 6, 101, 99, 8, 9, 7, 71, 69, 64, 66, 4, 1, 65, 5, 0, 67, 2, 68, 70, 3], [113, 63, 112, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 107, 105, 127, 43, 53, 123, 22, 86, 48, 62, 117, 36, 102, 110, 126, 49, 25, 89, 100, 30, 82, 61, 51, 15, 18, 125, 96, 47, 80, 29, 20, 119, 58, 40, 115, 98, 50, 94, 79, 57, 38, 84, 116, 46, 56, 16, 39, 24, 34, 104, 121, 27, 21, 23, 85, 72, 13, 103, 77, 114, 88, 26, 75, 10, 32, 109, 111, 87, 93, 74, 97, 92, 11, 12, 35, 90, 106, 45, 33, 83, 91, 28, 31, 81, 19, 8, 95, 6, 42, 17, 0, 73, 76, 14, 7, 108, 44, 78, 65, 71, 101, 69, 64, 9, 67, 99, 37, 5, 4, 3, 66, 1, 68, 2, 70], [113, 63, 112, 124, 55, 41, 54, 118, 52, 120, 122, 59, 60, 105, 107, 43, 127, 53, 123, 22, 48, 117, 102, 86, 36, 110, 126, 25, 100, 62, 89, 51, 82, 15, 30, 40, 80, 20, 96, 18, 61, 79, 49, 115, 84, 125, 29, 47, 56, 57, 94, 46, 23, 16, 116, 98, 50, 34, 58, 38, 85, 72, 39, 13, 27, 119, 77, 26, 21, 104, 103, 45, 88, 111, 93, 32, 24, 10, 11, 75, 109, 74, 121, 87, 8, 114, 83, 12, 91, 81, 97, 90, 28, 17, 31, 106, 76, 44, 92, 33, 78, 95, 19, 35, 42, 6, 73, 14, 7, 101, 64, 69, 9, 108, 37, 4, 71, 0, 66, 3, 1, 68, 67, 65, 70, 2, 99, 5], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 22, 117, 48, 86, 36, 102, 110, 62, 126, 100, 25, 89, 82, 116, 96, 30, 15, 49, 20, 56, 51, 46, 84, 18, 40, 79, 80, 16, 47, 61, 57, 125, 94, 29, 115, 98, 77, 119, 23, 50, 58, 38, 39, 85, 34, 24, 13, 111, 72, 103, 121, 104, 114, 75, 27, 10, 26, 11, 88, 109, 32, 87, 74, 42, 45, 97, 106, 8, 21, 81, 93, 92, 12, 76, 90, 19, 95, 83, 91, 44, 108, 17, 31, 14, 9, 78, 6, 35, 33, 28, 101, 73, 7, 37, 4, 70, 64, 0, 5, 2, 71, 1, 65, 99, 67, 69, 68, 66, 3], [113, 63, 112, 124, 55, 118, 41, 52, 54, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 86, 36, 22, 126, 102, 117, 25, 30, 82, 110, 62, 49, 89, 18, 96, 80, 29, 20, 40, 47, 94, 84, 100, 16, 15, 104, 46, 57, 125, 61, 38, 79, 51, 98, 115, 39, 119, 56, 116, 23, 26, 58, 24, 27, 103, 34, 50, 21, 111, 85, 13, 109, 32, 88, 77, 97, 114, 87, 121, 91, 93, 75, 92, 42, 90, 74, 8, 19, 11, 31, 106, 10, 95, 76, 28, 108, 44, 17, 33, 72, 81, 35, 12, 45, 83, 78, 37, 14, 70, 101, 73, 69, 9, 64, 71, 65, 2, 99, 7, 6, 0, 4, 68, 1, 66, 67, 3, 5], [113, 63, 112, 124, 55, 54, 118, 41, 52, 120, 122, 60, 105, 59, 107, 43, 123, 127, 53, 48, 36, 86, 102, 22, 117, 126, 25, 89, 49, 110, 30, 96, 29, 94, 62, 40, 18, 82, 20, 84, 116, 56, 100, 47, 80, 38, 15, 79, 104, 16, 119, 46, 58, 57, 23, 115, 27, 77, 125, 98, 39, 34, 24, 61, 26, 51, 85, 21, 13, 88, 32, 111, 103, 50, 114, 11, 87, 74, 90, 92, 121, 109, 8, 97, 93, 91, 75, 10, 95, 19, 81, 106, 72, 12, 70, 31, 14, 42, 28, 35, 33, 108, 83, 76, 78, 44, 17, 71, 0, 45, 9, 1, 69, 3, 73, 64, 37, 68, 7, 66, 101, 65, 99, 6, 4, 2, 5, 67], [113, 112, 63, 124, 55, 54, 118, 41, 52, 120, 122, 59, 60, 105, 107, 53, 127, 43, 123, 36, 48, 117, 22, 102, 86, 126, 62, 116, 56, 110, 25, 57, 47, 100, 125, 30, 49, 46, 115, 89, 20, 18, 96, 40, 38, 82, 94, 61, 51, 84, 80, 29, 119, 34, 15, 104, 79, 16, 114, 58, 121, 39, 98, 32, 23, 27, 77, 50, 103, 111, 8, 88, 21, 24, 109, 74, 87, 26, 106, 85, 13, 45, 91, 11, 92, 90, 97, 95, 93, 10, 75, 35, 31, 42, 17, 76, 37, 19, 81, 72, 108, 28, 44, 83, 70, 12, 33, 14, 9, 99, 71, 0, 64, 101, 73, 69, 1, 78, 3, 4, 7, 65, 2, 66, 67, 68, 5, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 53, 43, 48, 123, 117, 102, 36, 22, 86, 126, 110, 56, 62, 100, 25, 30, 49, 57, 116, 47, 15, 96, 119, 51, 125, 89, 61, 18, 94, 29, 84, 46, 115, 39, 20, 40, 82, 79, 16, 38, 80, 23, 98, 34, 27, 13, 103, 21, 58, 104, 26, 50, 8, 24, 109, 121, 88, 87, 114, 32, 77, 45, 111, 97, 85, 93, 92, 74, 11, 10, 95, 90, 91, 31, 108, 28, 76, 75, 106, 33, 83, 42, 12, 35, 19, 17, 81, 78, 72, 44, 37, 14, 70, 9, 64, 73, 65, 71, 5, 101, 0, 4, 7, 99, 2, 69, 1, 67, 68, 3, 66, 6], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 36, 48, 22, 102, 86, 126, 25, 62, 89, 96, 110, 51, 56, 57, 30, 119, 100, 116, 47, 49, 84, 94, 38, 20, 79, 61, 15, 29, 98, 18, 115, 80, 40, 82, 125, 46, 23, 34, 58, 16, 121, 26, 104, 111, 50, 77, 27, 24, 39, 88, 103, 13, 85, 32, 8, 97, 21, 114, 109, 75, 87, 93, 92, 91, 90, 19, 10, 106, 11, 95, 78, 81, 76, 31, 74, 17, 108, 12, 35, 83, 42, 28, 33, 72, 44, 45, 14, 37, 70, 71, 73, 9, 101, 99, 64, 65, 7, 0, 1, 3, 67, 4, 6, 5, 68, 66, 69, 2], [113, 112, 63, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 117, 48, 36, 126, 86, 62, 22, 25, 102, 116, 110, 56, 89, 30, 61, 51, 57, 100, 125, 47, 49, 119, 94, 38, 84, 96, 20, 82, 40, 80, 79, 115, 15, 29, 18, 98, 46, 58, 16, 104, 34, 50, 103, 121, 27, 23, 26, 77, 111, 88, 32, 24, 85, 114, 39, 21, 13, 90, 91, 8, 93, 87, 11, 92, 109, 106, 75, 97, 74, 19, 95, 28, 10, 35, 31, 17, 83, 42, 108, 44, 12, 72, 76, 45, 0, 81, 14, 64, 33, 6, 78, 9, 71, 70, 37, 2, 7, 73, 99, 1, 101, 65, 69, 68, 67, 5, 3, 66, 4], [113, 63, 112, 124, 55, 54, 118, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 117, 36, 102, 22, 86, 25, 126, 110, 62, 57, 30, 116, 51, 119, 100, 40, 96, 47, 61, 84, 89, 38, 56, 29, 18, 94, 20, 82, 15, 16, 49, 46, 125, 79, 98, 23, 104, 50, 34, 80, 26, 24, 115, 58, 103, 111, 39, 27, 121, 87, 77, 32, 85, 88, 21, 109, 93, 114, 97, 35, 13, 31, 28, 45, 90, 106, 92, 91, 8, 42, 83, 75, 74, 108, 95, 19, 44, 11, 81, 76, 10, 6, 33, 12, 72, 17, 78, 9, 14, 101, 37, 0, 64, 7, 65, 71, 73, 1, 99, 5, 4, 69, 66, 2, 67, 3, 70, 68], [113, 63, 112, 124, 55, 118, 54, 52, 41, 120, 122, 59, 60, 105, 107, 127, 43, 53, 123, 48, 22, 36, 117, 86, 102, 126, 116, 25, 62, 110, 96, 51, 30, 100, 57, 18, 119, 16, 89, 61, 49, 47, 15, 94, 84, 82, 29, 20, 79, 40, 38, 125, 34, 98, 39, 46, 23, 104, 26, 50, 56, 80, 103, 77, 115, 58, 24, 27, 121, 13, 32, 21, 85, 109, 8, 74, 88, 92, 114, 90, 75, 111, 93, 97, 87, 72, 10, 91, 95, 81, 11, 19, 45, 106, 76, 83, 28, 42, 12, 6, 17, 31, 35, 14, 78, 108, 44, 7, 73, 37, 9, 33, 71, 99, 101, 0, 4, 5, 68, 67, 69, 1, 3, 64, 66, 65, 70, 2], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 59, 60, 105, 107, 127, 43, 123, 53, 22, 117, 86, 48, 36, 102, 25, 62, 126, 89, 116, 30, 110, 20, 16, 51, 84, 18, 115, 82, 61, 100, 96, 40, 119, 79, 57, 47, 15, 49, 38, 80, 29, 94, 23, 98, 46, 109, 58, 125, 34, 56, 50, 104, 26, 88, 103, 27, 114, 77, 13, 111, 85, 32, 75, 21, 121, 39, 24, 10, 87, 90, 42, 72, 11, 93, 31, 8, 92, 97, 74, 28, 91, 76, 45, 81, 19, 83, 95, 6, 106, 35, 17, 78, 12, 108, 33, 44, 14, 73, 101, 9, 64, 7, 99, 71, 4, 65, 0, 37, 69, 66, 2, 5, 3, 1, 68, 67, 70], [113, 63, 112, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 43, 123, 53, 48, 86, 36, 22, 117, 102, 62, 126, 25, 30, 110, 100, 89, 116, 29, 51, 96, 94, 20, 119, 40, 61, 56, 57, 18, 16, 82, 47, 84, 15, 38, 49, 125, 46, 109, 115, 23, 98, 58, 80, 50, 104, 24, 79, 77, 26, 34, 121, 85, 103, 27, 21, 13, 88, 92, 39, 87, 11, 93, 111, 32, 114, 72, 91, 74, 97, 42, 90, 75, 10, 31, 95, 81, 19, 45, 106, 33, 8, 12, 28, 76, 108, 17, 83, 44, 78, 35, 73, 14, 6, 9, 37, 0, 101, 7, 99, 65, 69, 2, 3, 71, 64, 67, 5, 68, 70, 66, 4, 1], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 43, 123, 53, 36, 117, 48, 102, 62, 22, 110, 126, 86, 25, 61, 116, 89, 49, 30, 18, 100, 51, 56, 96, 40, 119, 57, 84, 47, 29, 125, 20, 121, 23, 58, 39, 16, 38, 98, 104, 94, 82, 46, 50, 115, 79, 34, 15, 80, 109, 114, 26, 111, 77, 103, 27, 24, 13, 88, 32, 85, 97, 21, 42, 72, 87, 93, 92, 90, 75, 74, 91, 106, 35, 11, 45, 108, 10, 44, 19, 17, 12, 31, 81, 83, 76, 28, 95, 37, 33, 101, 8, 78, 14, 73, 99, 6, 9, 0, 7, 70, 64, 69, 5, 71, 4, 68, 67, 66, 3, 2, 1, 65], [113, 112, 63, 124, 55, 54, 118, 41, 120, 52, 122, 60, 59, 105, 107, 127, 43, 53, 123, 22, 48, 36, 117, 126, 102, 86, 110, 62, 25, 61, 20, 89, 18, 47, 100, 82, 30, 51, 125, 96, 49, 56, 116, 79, 40, 80, 15, 84, 16, 119, 29, 46, 57, 23, 58, 38, 94, 50, 104, 121, 88, 77, 34, 115, 27, 26, 103, 72, 13, 39, 85, 111, 24, 114, 32, 21, 93, 92, 98, 11, 109, 74, 75, 91, 90, 45, 97, 87, 10, 19, 12, 28, 106, 17, 76, 81, 42, 8, 78, 31, 35, 95, 70, 83, 7, 9, 44, 0, 73, 14, 108, 69, 99, 3, 71, 65, 37, 64, 101, 4, 33, 5, 2, 1, 68, 6, 67, 66], [113, 63, 112, 124, 55, 118, 41, 54, 120, 52, 122, 60, 59, 105, 107, 127, 123, 53, 43, 48, 117, 36, 62, 22, 86, 110, 51, 102, 126, 119, 49, 30, 25, 125, 100, 96, 89, 47, 61, 40, 18, 115, 57, 56, 20, 15, 29, 84, 46, 116, 16, 79, 82, 94, 39, 34, 23, 104, 58, 38, 50, 103, 77, 121, 80, 26, 98, 72, 109, 45, 85, 32, 27, 92, 13, 93, 111, 21, 114, 106, 74, 10, 42, 90, 11, 24, 75, 88, 97, 87, 81, 108, 95, 35, 76, 70, 91, 28, 78, 31, 44, 12, 8, 19, 17, 33, 9, 5, 0, 64, 83, 14, 37, 67, 7, 65, 1, 73, 71, 101, 66, 69, 3, 4, 2, 99, 68, 6], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 60, 59, 105, 107, 127, 53, 43, 123, 48, 117, 22, 36, 86, 110, 102, 126, 62, 51, 119, 25, 49, 89, 18, 30, 100, 82, 79, 56, 46, 125, 20, 84, 116, 115, 15, 40, 29, 61, 96, 94, 16, 47, 80, 23, 109, 77, 98, 57, 39, 38, 34, 103, 24, 58, 13, 114, 50, 72, 121, 32, 42, 21, 88, 85, 104, 45, 111, 92, 27, 93, 87, 11, 76, 10, 97, 75, 74, 26, 106, 17, 90, 108, 91, 81, 14, 19, 8, 44, 95, 70, 83, 31, 28, 35, 12, 78, 9, 33, 7, 73, 71, 5, 101, 64, 67, 37, 69, 4, 99, 65, 3, 2, 0, 1, 68, 6, 66], [113, 112, 63, 124, 55, 118, 54, 41, 52, 120, 122, 59, 60, 105, 107, 127, 53, 43, 123, 48, 117, 102, 86, 36, 62, 22, 51, 110, 126, 25, 116, 30, 89, 100, 49, 125, 82, 96, 29, 61, 47, 16, 40, 20, 115, 119, 94, 39, 79, 18, 15, 84, 23, 58, 121, 56, 57, 109, 77, 50, 26, 34, 38, 104, 80, 98, 46, 114, 103, 27, 72, 85, 74, 88, 10, 24, 21, 87, 42, 32, 92, 111, 45, 97, 93, 75, 13, 76, 91, 8, 11, 90, 70, 14, 81, 31, 7, 19, 83, 12, 33, 5, 78, 95, 35, 17, 28, 106, 73, 108, 71, 44, 9, 0, 64, 69, 3, 4, 1, 65, 68, 67, 101, 66, 2, 37, 6, 99], [113, 112, 63, 124, 55, 118, 54, 41, 52, 122, 120, 60, 59, 105, 107, 127, 43, 53, 123, 86, 36, 102, 48, 22, 117, 25, 126, 62, 110, 100, 82, 29, 30, 51, 15, 49, 84, 89, 18, 16, 96, 61, 79, 116, 20, 47, 40, 119, 80, 94, 125, 38, 34, 56, 77, 57, 26, 98, 115, 23, 39, 27, 109, 46, 50, 88, 13, 24, 114, 58, 85, 21, 103, 92, 121, 32, 10, 72, 104, 74, 75, 42, 111, 45, 91, 97, 76, 87, 93, 11, 8, 95, 90, 106, 81, 83, 14, 19, 12, 78, 31, 33, 44, 28, 17, 73, 35, 108, 70, 9, 7, 71, 101, 37, 69, 99, 5, 67, 64, 6, 68, 4, 65, 66, 0, 1, 3, 2]], "model.layers.19.self_attn.q_proj": [[124, 110, 63, 115, 58, 119, 101, 120, 57, 46, 112, 52, 60, 123, 127, 109, 53, 59, 56, 116, 54, 88, 126, 50, 108, 61, 122, 55, 94, 111, 121, 114, 104, 51, 37, 47, 62, 97, 48, 40, 44, 49, 113, 117, 125, 45, 27, 19, 24, 118, 90, 34, 41, 105, 42, 81, 98, 106, 43, 107, 96, 31, 22, 39, 89, 102, 26, 16, 92, 9, 100, 38, 103, 36, 35, 78, 91, 99, 93, 25, 18, 30, 80, 29, 32, 73, 17, 20, 95, 23, 67, 85, 15, 28, 21, 69, 77, 84, 33, 12, 6, 14, 87, 83, 86, 10, 70, 79, 75, 13, 4, 1, 82, 68, 72, 0, 74, 5, 11, 64, 3, 65, 66, 8, 7, 76, 71, 2], [63, 57, 46, 101, 123, 112, 59, 110, 124, 120, 47, 97, 53, 121, 119, 81, 109, 114, 90, 24, 94, 111, 34, 105, 60, 19, 88, 87, 122, 126, 31, 85, 55, 104, 27, 91, 117, 58, 45, 52, 61, 25, 56, 106, 20, 39, 22, 116, 32, 28, 78, 42, 16, 38, 10, 107, 62, 44, 41, 54, 83, 127, 113, 50, 108, 14, 102, 100, 17, 80, 37, 49, 35, 51, 118, 40, 26, 12, 103, 36, 95, 96, 125, 43, 115, 89, 21, 48, 84, 98, 99, 93, 74, 30, 23, 29, 86, 92, 15, 11, 76, 77, 75, 6, 67, 7, 72, 18, 70, 13, 9, 73, 82, 68, 33, 79, 8, 69, 66, 1, 71, 4, 0, 2, 3, 64, 65, 5], [63, 124, 110, 57, 115, 101, 58, 46, 50, 122, 97, 88, 52, 60, 27, 120, 49, 55, 116, 81, 56, 59, 112, 123, 94, 119, 42, 121, 109, 111, 54, 126, 24, 125, 105, 127, 62, 117, 114, 44, 51, 93, 113, 37, 53, 84, 61, 90, 47, 31, 104, 20, 85, 108, 45, 6, 103, 118, 75, 73, 9, 41, 40, 67, 106, 48, 65, 43, 36, 38, 16, 4, 39, 72, 22, 14, 107, 0, 17, 100, 68, 12, 83, 80, 69, 30, 102, 87, 71, 91, 95, 7, 26, 15, 99, 19, 96, 11, 2, 35, 10, 74, 25, 3, 29, 34, 66, 77, 64, 32, 28, 92, 98, 13, 23, 21, 82, 70, 1, 8, 89, 18, 5, 79, 86, 76, 78, 33], [124, 63, 110, 101, 54, 46, 122, 119, 58, 59, 28, 112, 61, 55, 62, 120, 94, 56, 114, 111, 57, 90, 50, 53, 113, 116, 126, 123, 127, 121, 60, 125, 118, 97, 82, 22, 117, 48, 37, 47, 52, 115, 18, 49, 108, 109, 51, 41, 45, 44, 43, 39, 107, 32, 19, 106, 86, 102, 100, 42, 27, 92, 36, 30, 104, 40, 105, 25, 38, 103, 99, 34, 14, 35, 98, 96, 78, 20, 15, 33, 83, 88, 31, 77, 12, 26, 10, 29, 93, 24, 95, 89, 87, 23, 84, 74, 91, 13, 85, 76, 79, 21, 81, 17, 8, 75, 7, 72, 11, 80, 71, 9, 16, 0, 70, 68, 73, 67, 3, 2, 6, 66, 4, 5, 65, 69, 64, 1], [59, 118, 39, 46, 105, 127, 63, 34, 113, 112, 60, 119, 53, 54, 111, 50, 48, 126, 61, 122, 30, 125, 109, 49, 123, 89, 58, 97, 57, 121, 115, 56, 110, 43, 52, 25, 47, 104, 116, 55, 51, 107, 31, 108, 45, 62, 114, 36, 44, 41, 42, 120, 117, 102, 23, 33, 106, 28, 40, 82, 37, 38, 22, 94, 90, 84, 27, 98, 91, 18, 80, 101, 78, 87, 86, 99, 100, 26, 35, 32, 93, 103, 95, 96, 29, 14, 92, 21, 24, 19, 75, 20, 8, 124, 0, 76, 16, 12, 7, 88, 67, 66, 9, 2, 68, 70, 71, 73, 79, 6, 85, 65, 81, 3, 69, 4, 64, 83, 11, 1, 72, 17, 13, 5, 15, 10, 77, 74], [39, 59, 118, 34, 124, 90, 46, 84, 43, 79, 75, 54, 60, 122, 98, 30, 112, 49, 109, 127, 21, 19, 88, 111, 25, 94, 115, 76, 119, 23, 63, 89, 105, 22, 113, 9, 38, 87, 56, 8, 41, 106, 57, 31, 44, 81, 110, 33, 50, 70, 92, 48, 82, 116, 125, 100, 114, 53, 27, 62, 120, 99, 107, 42, 40, 73, 108, 58, 45, 123, 93, 104, 55, 95, 126, 121, 35, 91, 20, 18, 37, 52, 51, 61, 14, 72, 97, 28, 47, 6, 16, 86, 13, 24, 36, 96, 3, 102, 29, 11, 12, 101, 32, 26, 117, 71, 80, 66, 85, 78, 15, 7, 83, 2, 67, 68, 17, 10, 5, 69, 77, 4, 65, 103, 64, 0, 1, 74], [118, 39, 59, 34, 124, 127, 46, 105, 54, 63, 43, 55, 89, 112, 90, 84, 50, 120, 47, 57, 49, 113, 30, 115, 122, 53, 51, 125, 107, 60, 119, 80, 35, 75, 126, 45, 21, 19, 116, 123, 110, 25, 8, 111, 109, 22, 82, 88, 48, 52, 104, 27, 94, 33, 56, 121, 44, 61, 108, 62, 98, 92, 117, 58, 106, 37, 78, 79, 31, 38, 23, 40, 101, 103, 86, 99, 91, 42, 95, 24, 114, 97, 93, 12, 87, 102, 81, 32, 26, 100, 41, 20, 29, 18, 76, 14, 7, 36, 73, 28, 71, 96, 16, 13, 3, 67, 11, 70, 2, 69, 66, 68, 6, 10, 9, 0, 72, 4, 64, 77, 65, 85, 1, 83, 15, 5, 17, 74], [39, 59, 118, 34, 81, 90, 21, 13, 19, 79, 10, 46, 69, 54, 100, 65, 8, 68, 74, 115, 77, 33, 3, 11, 75, 95, 49, 55, 38, 120, 122, 71, 85, 30, 94, 0, 32, 88, 87, 18, 17, 107, 56, 112, 23, 44, 57, 63, 25, 126, 86, 67, 24, 83, 58, 119, 113, 5, 76, 72, 15, 41, 127, 89, 7, 111, 4, 12, 16, 26, 20, 43, 29, 80, 91, 14, 64, 110, 99, 22, 78, 84, 60, 123, 66, 92, 70, 37, 6, 105, 31, 1, 27, 36, 82, 96, 73, 98, 50, 35, 124, 93, 9, 47, 28, 45, 2, 53, 101, 125, 52, 114, 102, 121, 103, 109, 48, 62, 61, 104, 116, 97, 51, 40, 42, 117, 108, 106], [105, 112, 34, 84, 18, 12, 15, 27, 22, 5, 71, 48, 9, 41, 96, 2, 64, 89, 53, 82, 91, 58, 13, 4, 122, 30, 83, 1, 20, 25, 79, 63, 86, 118, 76, 87, 7, 74, 31, 78, 80, 50, 19, 85, 69, 119, 120, 70, 75, 94, 127, 16, 73, 11, 14, 81, 10, 92, 77, 46, 98, 17, 68, 6, 103, 60, 90, 28, 21, 24, 37, 72, 93, 23, 65, 107, 113, 61, 3, 26, 67, 59, 8, 56, 52, 114, 32, 115, 66, 57, 88, 110, 29, 95, 0, 123, 33, 62, 49, 55, 35, 125, 44, 36, 126, 101, 42, 99, 116, 51, 97, 47, 54, 104, 100, 102, 124, 109, 111, 43, 39, 117, 45, 38, 106, 40, 108, 121], [112, 105, 53, 25, 127, 58, 118, 89, 56, 34, 114, 54, 101, 83, 125, 96, 32, 49, 48, 120, 124, 116, 55, 52, 57, 93, 59, 33, 41, 113, 119, 62, 117, 115, 126, 61, 60, 107, 50, 106, 121, 122, 104, 45, 111, 110, 98, 51, 63, 44, 109, 47, 42, 46, 94, 123, 43, 108, 102, 37, 103, 100, 27, 38, 36, 39, 23, 99, 35, 40, 81, 87, 18, 95, 30, 22, 97, 77, 29, 31, 13, 6, 28, 8, 92, 10, 90, 68, 24, 91, 84, 16, 65, 26, 88, 86, 70, 15, 4, 19, 14, 85, 17, 80, 82, 11, 67, 21, 1, 74, 72, 75, 64, 3, 7, 78, 12, 0, 20, 66, 73, 71, 79, 5, 9, 2, 76, 69], [105, 112, 34, 15, 22, 12, 84, 18, 9, 27, 41, 89, 48, 71, 53, 5, 96, 4, 91, 58, 25, 13, 85, 2, 78, 81, 75, 64, 122, 19, 50, 67, 30, 1, 80, 31, 86, 20, 77, 82, 63, 79, 119, 72, 59, 83, 95, 88, 69, 73, 118, 21, 37, 76, 74, 7, 46, 11, 57, 3, 60, 26, 14, 6, 65, 23, 92, 28, 17, 127, 70, 62, 93, 87, 8, 24, 115, 98, 56, 61, 52, 68, 107, 125, 90, 33, 16, 120, 114, 110, 36, 10, 66, 47, 94, 35, 29, 126, 55, 116, 51, 113, 0, 97, 32, 42, 103, 123, 49, 100, 124, 102, 43, 54, 109, 117, 39, 101, 38, 99, 44, 111, 45, 104, 121, 106, 108, 40], [105, 53, 112, 34, 41, 48, 88, 25, 89, 94, 27, 33, 98, 13, 22, 84, 58, 56, 114, 118, 101, 96, 125, 120, 59, 19, 18, 78, 57, 80, 91, 124, 116, 117, 55, 62, 127, 107, 119, 49, 93, 68, 113, 6, 75, 115, 60, 50, 122, 70, 100, 15, 110, 52, 63, 54, 126, 83, 103, 47, 61, 51, 111, 4, 37, 29, 121, 92, 12, 104, 45, 97, 109, 30, 31, 42, 71, 38, 46, 123, 65, 36, 23, 44, 102, 43, 67, 74, 21, 99, 108, 10, 26, 106, 1, 24, 35, 0, 95, 39, 11, 3, 86, 81, 77, 16, 32, 40, 82, 14, 90, 9, 28, 72, 85, 17, 8, 87, 2, 20, 66, 64, 5, 7, 79, 73, 69, 76], [120, 121, 53, 57, 118, 126, 63, 113, 124, 51, 59, 60, 58, 123, 125, 115, 49, 116, 122, 119, 117, 62, 127, 56, 112, 54, 47, 55, 99, 50, 61, 114, 48, 111, 83, 44, 108, 110, 52, 13, 45, 103, 109, 46, 43, 41, 107, 40, 31, 32, 90, 106, 42, 105, 39, 38, 102, 100, 35, 9, 69, 34, 3, 97, 104, 101, 21, 82, 71, 96, 37, 28, 2, 36, 70, 98, 18, 76, 85, 68, 77, 23, 33, 30, 66, 11, 81, 92, 25, 26, 72, 0, 10, 93, 15, 29, 94, 84, 24, 86, 88, 22, 20, 16, 1, 89, 91, 75, 4, 79, 27, 78, 12, 5, 95, 65, 80, 17, 14, 19, 7, 64, 87, 73, 8, 74, 67, 6], [121, 120, 53, 57, 113, 63, 58, 118, 60, 125, 124, 126, 59, 56, 117, 119, 51, 116, 123, 55, 115, 48, 114, 99, 49, 62, 50, 61, 112, 122, 54, 52, 47, 43, 44, 110, 45, 108, 127, 111, 109, 46, 39, 41, 40, 42, 102, 106, 107, 105, 89, 90, 100, 83, 103, 35, 26, 104, 38, 32, 3, 101, 96, 37, 31, 2, 71, 72, 36, 82, 70, 34, 98, 33, 0, 13, 4, 69, 16, 76, 11, 91, 9, 92, 21, 27, 29, 22, 97, 95, 1, 94, 93, 77, 85, 8, 87, 30, 23, 15, 18, 86, 28, 88, 10, 68, 14, 25, 64, 19, 80, 12, 20, 78, 65, 84, 75, 17, 74, 79, 24, 66, 6, 7, 81, 67, 5, 73], [121, 120, 126, 40, 53, 99, 36, 63, 58, 124, 39, 51, 42, 106, 117, 88, 49, 113, 89, 59, 57, 102, 26, 122, 96, 31, 60, 123, 56, 28, 50, 46, 118, 119, 24, 105, 16, 116, 87, 112, 110, 55, 125, 32, 90, 98, 34, 82, 52, 103, 127, 35, 10, 115, 30, 54, 48, 95, 47, 100, 25, 45, 37, 38, 111, 104, 44, 41, 109, 61, 62, 33, 114, 22, 83, 43, 80, 21, 107, 27, 91, 97, 101, 29, 108, 69, 72, 92, 93, 19, 94, 15, 9, 70, 13, 71, 23, 11, 77, 84, 3, 86, 14, 85, 76, 20, 81, 79, 18, 17, 68, 74, 2, 0, 1, 12, 7, 65, 75, 78, 8, 4, 6, 73, 67, 64, 66, 5], [121, 120, 118, 39, 63, 117, 126, 124, 53, 116, 57, 51, 102, 125, 56, 60, 54, 119, 123, 58, 113, 49, 35, 99, 62, 83, 105, 112, 15, 40, 36, 44, 127, 114, 61, 48, 52, 59, 122, 42, 45, 111, 50, 47, 55, 38, 108, 107, 115, 101, 32, 106, 103, 110, 21, 46, 24, 13, 96, 88, 41, 109, 90, 26, 81, 16, 31, 43, 100, 37, 75, 34, 104, 91, 33, 25, 20, 28, 84, 69, 92, 97, 79, 98, 9, 27, 71, 89, 85, 74, 86, 87, 29, 94, 23, 22, 30, 77, 19, 93, 12, 76, 10, 11, 14, 17, 80, 5, 7, 3, 73, 95, 82, 18, 68, 78, 70, 8, 67, 4, 0, 72, 2, 65, 66, 6, 64, 1], [55, 105, 101, 120, 116, 49, 32, 112, 25, 87, 46, 57, 110, 81, 91, 109, 27, 48, 47, 119, 28, 60, 96, 94, 39, 113, 126, 44, 54, 59, 115, 63, 89, 107, 118, 84, 127, 76, 78, 85, 83, 11, 50, 106, 37, 20, 14, 122, 104, 31, 123, 58, 52, 111, 125, 61, 124, 108, 71, 88, 99, 51, 121, 19, 92, 114, 56, 45, 42, 103, 38, 43, 117, 26, 97, 62, 30, 35, 95, 34, 98, 21, 7, 86, 53, 22, 29, 73, 100, 18, 24, 102, 93, 90, 79, 15, 17, 33, 40, 5, 16, 12, 80, 70, 36, 82, 68, 41, 10, 75, 3, 23, 8, 9, 74, 2, 4, 77, 13, 72, 6, 69, 67, 65, 66, 1, 64, 0], [105, 55, 109, 116, 101, 120, 32, 112, 57, 46, 87, 25, 113, 28, 60, 110, 81, 119, 91, 104, 49, 35, 89, 84, 48, 121, 85, 39, 88, 111, 92, 42, 108, 37, 78, 27, 58, 114, 11, 107, 97, 117, 53, 31, 96, 95, 127, 125, 118, 56, 61, 94, 124, 126, 63, 62, 38, 44, 115, 99, 59, 93, 30, 54, 14, 45, 26, 50, 52, 29, 51, 98, 122, 76, 100, 106, 20, 41, 123, 33, 17, 73, 36, 79, 47, 103, 40, 12, 23, 43, 102, 86, 83, 16, 21, 24, 22, 34, 7, 19, 15, 71, 80, 75, 82, 18, 90, 10, 74, 5, 70, 6, 72, 77, 9, 3, 8, 4, 13, 2, 67, 68, 69, 65, 1, 0, 64, 66], [55, 105, 101, 120, 46, 112, 116, 57, 32, 87, 25, 39, 81, 28, 126, 47, 119, 48, 94, 84, 27, 89, 50, 56, 113, 34, 92, 118, 14, 122, 91, 60, 124, 125, 127, 38, 11, 111, 104, 78, 115, 45, 83, 110, 58, 36, 114, 108, 96, 54, 121, 109, 52, 117, 20, 31, 103, 106, 95, 42, 44, 49, 30, 107, 63, 123, 33, 37, 73, 29, 59, 79, 61, 70, 43, 51, 85, 62, 72, 99, 53, 19, 100, 102, 22, 97, 86, 21, 93, 35, 98, 17, 8, 41, 76, 40, 24, 12, 82, 90, 26, 88, 16, 75, 80, 18, 10, 15, 5, 7, 3, 6, 23, 4, 68, 9, 71, 77, 69, 66, 74, 2, 67, 13, 65, 64, 0, 1], [105, 57, 120, 116, 46, 112, 32, 55, 101, 119, 39, 113, 127, 87, 92, 94, 28, 53, 62, 104, 50, 115, 25, 126, 60, 114, 124, 38, 108, 58, 52, 27, 121, 56, 54, 111, 123, 84, 107, 98, 110, 61, 48, 122, 117, 44, 36, 106, 96, 59, 42, 51, 100, 118, 47, 63, 43, 125, 45, 109, 89, 91, 83, 79, 41, 49, 85, 33, 99, 88, 81, 35, 86, 29, 37, 95, 22, 20, 77, 30, 34, 97, 21, 18, 26, 31, 103, 40, 93, 102, 14, 78, 74, 82, 24, 90, 73, 23, 13, 75, 80, 70, 11, 76, 16, 5, 15, 64, 19, 17, 12, 7, 66, 0, 8, 2, 67, 3, 65, 72, 1, 10, 69, 68, 71, 4, 6, 9], [102, 62, 56, 55, 97, 22, 29, 19, 89, 35, 15, 88, 93, 38, 31, 76, 45, 84, 27, 60, 17, 90, 25, 33, 21, 12, 86, 46, 96, 30, 81, 95, 43, 14, 98, 78, 50, 100, 7, 26, 9, 24, 44, 61, 101, 91, 40, 87, 79, 94, 57, 122, 116, 23, 67, 71, 83, 59, 10, 34, 20, 39, 127, 85, 32, 92, 53, 28, 121, 54, 103, 73, 36, 99, 82, 18, 115, 105, 114, 117, 63, 113, 13, 104, 111, 11, 49, 124, 16, 112, 70, 41, 58, 80, 37, 125, 48, 109, 42, 8, 119, 106, 68, 51, 110, 2, 107, 123, 52, 108, 120, 75, 47, 1, 69, 126, 118, 74, 77, 0, 3, 5, 65, 72, 4, 64, 66, 6], [56, 102, 97, 55, 62, 29, 15, 45, 19, 89, 22, 60, 76, 88, 38, 93, 7, 69, 17, 82, 10, 74, 2, 59, 32, 49, 31, 81, 3, 54, 33, 40, 46, 86, 58, 25, 91, 9, 98, 68, 112, 21, 104, 43, 50, 122, 28, 27, 64, 26, 11, 13, 110, 120, 108, 90, 44, 78, 84, 73, 57, 14, 23, 117, 121, 53, 124, 61, 80, 100, 116, 101, 83, 1, 37, 115, 126, 51, 6, 12, 48, 20, 47, 114, 52, 96, 36, 87, 30, 107, 63, 119, 41, 35, 127, 118, 125, 92, 123, 75, 109, 85, 113, 67, 99, 39, 42, 106, 103, 4, 8, 94, 71, 16, 24, 77, 18, 111, 70, 105, 34, 79, 95, 72, 5, 0, 66, 65], [55, 102, 62, 56, 97, 29, 22, 38, 93, 19, 60, 15, 76, 74, 88, 12, 69, 27, 28, 13, 45, 17, 90, 81, 84, 25, 118, 32, 104, 10, 23, 82, 73, 89, 21, 31, 49, 37, 79, 42, 11, 35, 58, 16, 30, 53, 50, 101, 61, 92, 46, 109, 98, 94, 86, 7, 71, 107, 117, 26, 91, 96, 44, 83, 59, 57, 106, 114, 121, 68, 24, 43, 2, 100, 110, 124, 126, 87, 48, 47, 54, 20, 39, 36, 112, 52, 85, 125, 113, 63, 33, 95, 99, 108, 116, 119, 78, 123, 75, 9, 3, 105, 14, 127, 115, 41, 122, 51, 80, 111, 120, 34, 40, 5, 103, 77, 18, 67, 72, 64, 6, 4, 8, 70, 66, 0, 1, 65], [102, 62, 56, 97, 55, 50, 29, 43, 40, 22, 61, 45, 25, 38, 89, 19, 126, 27, 17, 42, 127, 116, 18, 88, 46, 91, 57, 15, 59, 24, 53, 13, 114, 123, 93, 8, 54, 39, 109, 60, 113, 76, 125, 81, 117, 121, 111, 104, 124, 52, 115, 119, 47, 58, 122, 120, 63, 107, 112, 110, 6, 10, 87, 48, 77, 51, 49, 106, 31, 100, 20, 44, 86, 108, 103, 26, 118, 101, 65, 84, 30, 33, 105, 78, 35, 94, 11, 95, 41, 9, 37, 23, 72, 5, 99, 36, 14, 98, 69, 28, 96, 7, 74, 73, 90, 3, 75, 83, 68, 34, 66, 82, 32, 92, 85, 80, 21, 70, 16, 2, 1, 4, 67, 12, 79, 64, 71, 0], [41, 120, 47, 34, 57, 105, 22, 93, 84, 80, 109, 60, 78, 76, 88, 117, 45, 26, 103, 61, 118, 127, 125, 74, 126, 110, 108, 8, 90, 52, 113, 86, 114, 59, 124, 58, 119, 75, 35, 49, 89, 48, 116, 99, 111, 20, 1, 123, 101, 53, 81, 51, 62, 54, 104, 24, 28, 3, 43, 29, 7, 56, 19, 31, 112, 87, 2, 69, 92, 70, 107, 121, 17, 4, 25, 95, 18, 63, 50, 16, 55, 73, 115, 122, 27, 44, 102, 13, 21, 42, 94, 32, 91, 100, 106, 96, 11, 40, 46, 33, 71, 85, 36, 79, 39, 14, 23, 37, 10, 83, 6, 97, 30, 82, 77, 15, 38, 12, 98, 5, 9, 72, 65, 66, 68, 67, 0, 64], [41, 120, 57, 34, 84, 22, 88, 109, 93, 60, 110, 116, 29, 108, 90, 80, 49, 95, 42, 105, 76, 52, 123, 24, 26, 35, 74, 39, 127, 126, 113, 104, 78, 81, 25, 44, 82, 61, 40, 53, 124, 101, 112, 19, 43, 54, 119, 111, 48, 45, 122, 125, 7, 59, 121, 86, 55, 107, 58, 98, 63, 97, 115, 32, 62, 38, 106, 56, 37, 51, 96, 27, 18, 28, 118, 103, 79, 85, 36, 117, 114, 92, 83, 20, 102, 47, 16, 91, 50, 17, 71, 94, 46, 15, 89, 21, 23, 31, 100, 30, 99, 70, 33, 14, 87, 77, 13, 75, 8, 4, 69, 66, 68, 11, 73, 1, 12, 9, 65, 10, 5, 2, 72, 3, 6, 67, 64, 0], [41, 120, 34, 47, 70, 57, 64, 8, 76, 78, 93, 67, 66, 80, 87, 105, 7, 22, 84, 74, 1, 124, 0, 60, 3, 19, 11, 88, 13, 65, 18, 86, 117, 101, 126, 61, 20, 83, 69, 59, 26, 5, 72, 110, 73, 90, 82, 68, 121, 108, 16, 89, 31, 37, 25, 9, 2, 109, 79, 10, 75, 17, 111, 14, 4, 122, 24, 12, 114, 92, 32, 27, 116, 112, 6, 23, 77, 58, 85, 71, 81, 15, 95, 127, 100, 91, 102, 21, 115, 94, 62, 113, 43, 119, 28, 30, 107, 125, 103, 53, 48, 104, 99, 55, 33, 97, 50, 45, 96, 35, 36, 51, 63, 49, 118, 38, 39, 44, 46, 106, 123, 56, 40, 42, 54, 52, 29, 98], [120, 41, 34, 109, 22, 88, 48, 84, 47, 60, 113, 93, 117, 26, 110, 49, 119, 51, 124, 43, 61, 111, 29, 58, 24, 125, 50, 55, 80, 115, 81, 62, 63, 127, 116, 57, 114, 123, 108, 112, 54, 42, 126, 56, 78, 105, 90, 118, 36, 53, 40, 121, 102, 74, 44, 46, 35, 52, 28, 59, 107, 37, 122, 76, 106, 7, 95, 86, 39, 19, 45, 101, 38, 87, 99, 103, 18, 104, 100, 97, 27, 32, 83, 20, 31, 25, 33, 30, 94, 92, 96, 91, 17, 89, 70, 98, 11, 79, 73, 68, 8, 21, 1, 82, 23, 75, 15, 16, 77, 66, 9, 85, 13, 14, 71, 10, 12, 67, 5, 69, 65, 4, 6, 2, 64, 72, 3, 0], [38, 48, 112, 24, 125, 84, 17, 109, 27, 22, 91, 97, 78, 47, 18, 10, 107, 35, 95, 81, 121, 43, 11, 98, 118, 77, 115, 46, 52, 49, 119, 86, 102, 20, 68, 13, 50, 26, 120, 90, 54, 59, 92, 40, 34, 29, 88, 53, 7, 14, 15, 56, 108, 89, 82, 55, 30, 9, 37, 96, 103, 42, 117, 51, 39, 60, 64, 106, 110, 28, 87, 94, 32, 69, 126, 61, 113, 83, 123, 105, 41, 45, 25, 63, 122, 111, 85, 19, 93, 44, 33, 116, 114, 127, 8, 80, 72, 124, 23, 104, 31, 58, 21, 100, 73, 57, 79, 16, 76, 36, 99, 6, 101, 2, 12, 75, 74, 62, 5, 71, 70, 67, 65, 4, 1, 3, 66, 0], [48, 38, 112, 24, 22, 93, 107, 84, 123, 17, 52, 50, 27, 91, 95, 118, 53, 30, 20, 98, 94, 115, 78, 125, 111, 99, 113, 49, 117, 55, 126, 45, 97, 81, 19, 119, 104, 77, 58, 110, 92, 41, 109, 28, 47, 102, 10, 36, 13, 116, 86, 105, 101, 32, 88, 35, 57, 18, 54, 80, 90, 122, 61, 121, 62, 120, 108, 34, 106, 39, 33, 40, 44, 56, 59, 43, 42, 31, 100, 124, 63, 60, 103, 127, 51, 37, 72, 96, 114, 8, 26, 46, 25, 29, 65, 23, 87, 11, 16, 68, 9, 71, 85, 89, 21, 14, 12, 75, 82, 83, 15, 7, 6, 76, 64, 79, 74, 5, 2, 73, 67, 69, 70, 4, 66, 3, 1, 0], [38, 48, 112, 24, 22, 125, 84, 107, 17, 93, 19, 80, 95, 10, 20, 78, 109, 50, 62, 13, 77, 33, 18, 27, 47, 98, 97, 68, 94, 111, 28, 92, 72, 81, 34, 35, 75, 56, 119, 6, 52, 88, 96, 8, 86, 90, 26, 9, 30, 91, 82, 49, 116, 14, 32, 31, 69, 99, 64, 43, 67, 12, 54, 53, 41, 117, 55, 29, 126, 89, 16, 15, 45, 120, 21, 36, 83, 61, 85, 114, 0, 79, 73, 39, 74, 1, 121, 57, 23, 7, 118, 59, 51, 87, 63, 25, 46, 100, 44, 110, 76, 40, 124, 58, 123, 2, 106, 5, 42, 66, 104, 37, 115, 101, 103, 65, 102, 113, 127, 11, 108, 71, 70, 105, 60, 122, 4, 3], [38, 48, 112, 24, 22, 107, 84, 93, 17, 62, 123, 97, 78, 10, 121, 30, 20, 110, 125, 18, 98, 49, 27, 96, 95, 115, 94, 50, 33, 35, 127, 116, 19, 34, 13, 45, 124, 91, 120, 72, 43, 77, 59, 86, 126, 101, 105, 51, 14, 53, 92, 111, 119, 102, 113, 41, 52, 28, 118, 80, 57, 7, 68, 109, 103, 106, 32, 55, 81, 88, 89, 40, 36, 47, 54, 117, 8, 104, 114, 42, 60, 5, 21, 99, 63, 58, 122, 26, 61, 44, 56, 39, 74, 12, 31, 100, 9, 108, 46, 25, 37, 29, 87, 82, 64, 3, 11, 65, 90, 75, 15, 16, 83, 2, 71, 66, 67, 79, 76, 85, 23, 70, 6, 69, 73, 4, 1, 0]], "model.layers.19.self_attn.k_proj": [[63, 124, 110, 37, 33, 22, 53, 55, 112, 113, 59, 122, 114, 121, 120, 125, 54, 116, 61, 126, 62, 127, 119, 50, 49, 117, 56, 111, 47, 57, 123, 118, 58, 60, 30, 48, 109, 92, 51, 105, 43, 52, 14, 108, 44, 90, 45, 15, 104, 36, 69, 35, 12, 81, 71, 101, 107, 115, 3, 9, 46, 85, 88, 99, 74, 106, 40, 39, 42, 96, 41, 102, 38, 86, 103, 91, 20, 1, 77, 82, 64, 34, 19, 72, 100, 24, 11, 25, 98, 78, 32, 75, 70, 29, 18, 87, 66, 31, 26, 93, 95, 68, 89, 94, 76, 23, 16, 28, 27, 21, 80, 83, 84, 97, 79, 0, 13, 4, 10, 2, 8, 7, 73, 5, 67, 17, 65, 6], [118, 59, 103, 98, 94, 10, 79, 81, 13, 21, 69, 19, 75, 110, 25, 8, 97, 90, 80, 78, 0, 122, 44, 43, 57, 49, 23, 48, 108, 84, 120, 87, 77, 22, 71, 82, 26, 115, 35, 74, 76, 42, 2, 106, 54, 107, 119, 95, 1, 123, 52, 102, 68, 91, 41, 113, 9, 7, 121, 112, 55, 53, 63, 50, 88, 127, 14, 27, 67, 24, 65, 101, 93, 58, 100, 62, 99, 56, 104, 60, 117, 126, 92, 105, 111, 51, 114, 38, 116, 125, 28, 61, 47, 109, 36, 17, 96, 5, 32, 124, 73, 40, 37, 4, 83, 18, 70, 11, 12, 31, 46, 33, 45, 86, 72, 66, 6, 89, 29, 85, 3, 16, 30, 39, 20, 15, 34, 64], [41, 112, 98, 22, 48, 9, 27, 64, 15, 12, 71, 84, 2, 63, 5, 25, 114, 18, 53, 62, 122, 125, 56, 113, 94, 117, 59, 124, 1, 60, 120, 58, 119, 57, 83, 118, 116, 13, 51, 50, 126, 55, 49, 4, 46, 47, 105, 115, 54, 45, 32, 111, 74, 97, 29, 123, 110, 108, 61, 70, 52, 43, 127, 121, 78, 75, 107, 80, 67, 42, 37, 85, 88, 44, 87, 109, 103, 106, 104, 102, 101, 100, 40, 19, 93, 28, 21, 39, 8, 66, 95, 35, 81, 38, 30, 36, 72, 6, 11, 99, 10, 14, 16, 90, 92, 31, 26, 33, 17, 96, 91, 0, 24, 73, 76, 65, 34, 69, 3, 89, 77, 23, 86, 68, 79, 7, 20, 82], [121, 120, 35, 22, 104, 126, 124, 95, 57, 49, 60, 117, 59, 46, 112, 63, 50, 38, 58, 51, 119, 92, 127, 106, 111, 48, 123, 118, 125, 56, 122, 113, 61, 115, 116, 91, 41, 52, 54, 103, 45, 36, 47, 28, 109, 110, 62, 18, 42, 108, 55, 32, 114, 102, 43, 12, 44, 20, 101, 53, 14, 40, 81, 39, 107, 105, 26, 94, 37, 34, 79, 29, 80, 88, 74, 97, 99, 100, 27, 96, 33, 70, 13, 23, 98, 9, 8, 71, 31, 24, 30, 21, 89, 93, 83, 87, 25, 90, 3, 4, 10, 11, 86, 84, 16, 15, 17, 1, 69, 75, 85, 2, 19, 73, 7, 72, 82, 78, 5, 76, 68, 64, 77, 6, 67, 0, 66, 65], [55, 37, 110, 116, 120, 96, 112, 41, 30, 91, 89, 28, 57, 81, 45, 124, 87, 23, 78, 103, 56, 109, 83, 102, 84, 52, 49, 36, 60, 11, 123, 31, 40, 25, 34, 76, 85, 94, 121, 117, 62, 99, 8, 71, 73, 111, 39, 97, 54, 126, 58, 14, 44, 12, 59, 125, 115, 114, 50, 122, 22, 17, 61, 24, 119, 35, 118, 51, 127, 107, 43, 93, 79, 33, 18, 48, 95, 9, 47, 63, 21, 108, 42, 106, 113, 68, 6, 5, 2, 32, 70, 27, 26, 88, 15, 90, 92, 98, 104, 3, 29, 80, 100, 86, 53, 19, 16, 7, 38, 46, 82, 67, 20, 74, 75, 13, 10, 72, 1, 65, 105, 64, 0, 77, 69, 4, 101, 66], [62, 56, 38, 55, 33, 22, 93, 15, 19, 89, 17, 76, 81, 59, 7, 109, 29, 120, 102, 78, 110, 57, 90, 53, 121, 88, 74, 27, 12, 60, 119, 10, 69, 117, 124, 103, 63, 2, 113, 125, 104, 54, 45, 9, 31, 61, 77, 58, 116, 47, 0, 49, 107, 126, 118, 114, 122, 21, 112, 52, 39, 44, 127, 115, 34, 111, 46, 48, 13, 87, 123, 84, 108, 41, 3, 37, 99, 51, 23, 18, 106, 80, 96, 105, 71, 73, 36, 24, 79, 30, 95, 94, 50, 4, 92, 16, 14, 91, 85, 83, 32, 40, 42, 28, 67, 65, 6, 26, 75, 100, 20, 68, 101, 11, 8, 64, 98, 82, 43, 35, 5, 25, 72, 86, 70, 97, 66, 1], [105, 120, 98, 47, 29, 22, 57, 64, 84, 8, 80, 76, 111, 78, 67, 88, 74, 56, 70, 48, 66, 69, 60, 50, 26, 19, 119, 46, 117, 126, 124, 61, 82, 114, 44, 115, 53, 116, 125, 63, 62, 122, 43, 45, 113, 95, 59, 7, 23, 118, 41, 99, 112, 51, 55, 65, 58, 127, 81, 107, 110, 52, 49, 18, 33, 54, 89, 71, 123, 106, 27, 109, 42, 38, 121, 4, 101, 3, 104, 87, 15, 37, 83, 39, 77, 79, 40, 13, 103, 108, 28, 102, 96, 25, 30, 31, 2, 1, 91, 94, 32, 34, 16, 35, 85, 92, 36, 100, 12, 97, 21, 73, 11, 75, 5, 9, 24, 90, 20, 68, 17, 6, 93, 72, 0, 10, 14, 86], [112, 48, 102, 22, 84, 24, 78, 107, 18, 17, 10, 13, 33, 68, 93, 72, 31, 92, 9, 109, 125, 94, 65, 19, 64, 12, 98, 6, 27, 30, 62, 80, 71, 2, 67, 77, 52, 91, 85, 8, 119, 5, 32, 35, 121, 113, 123, 11, 7, 105, 34, 36, 28, 118, 43, 61, 26, 111, 87, 106, 56, 75, 69, 120, 50, 115, 122, 83, 47, 89, 126, 23, 63, 79, 58, 49, 97, 40, 16, 90, 51, 53, 99, 45, 29, 46, 104, 76, 103, 117, 41, 25, 44, 101, 54, 100, 21, 60, 70, 124, 15, 57, 42, 20, 37, 55, 95, 127, 108, 96, 110, 116, 73, 74, 114, 59, 39, 38, 82, 14, 88, 3, 86, 81, 4, 66, 1, 0]], "model.layers.19.self_attn.qk_proj": [[112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 98, 22, 53, 125, 50, 20, 111, 46, 58, 29, 60, 122, 93, 119, 84, 126, 113, 37, 25, 127, 17, 91, 51, 24, 89, 27, 109, 79, 61, 88, 12, 49, 76, 81, 83, 19, 90, 115, 97, 78, 54, 101, 14, 117, 114, 94, 123, 34, 82, 15, 52, 33, 35, 39, 32, 26, 45, 107, 87, 10, 30, 13, 18, 43, 103, 74, 92, 31, 44, 16, 77, 106, 72, 23, 108, 21, 95, 28, 96, 42, 9, 73, 11, 104, 71, 64, 80, 5, 40, 99, 75, 85, 36, 6, 7, 69, 0, 8, 100, 2, 70, 66, 67, 3, 4, 68, 65, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 98, 58, 22, 125, 119, 50, 84, 91, 37, 111, 113, 20, 89, 126, 127, 93, 29, 46, 60, 25, 76, 24, 83, 12, 17, 49, 51, 109, 88, 79, 78, 114, 117, 81, 123, 34, 27, 61, 122, 90, 35, 19, 107, 33, 94, 97, 54, 115, 15, 18, 32, 39, 43, 103, 14, 92, 52, 26, 101, 30, 87, 10, 82, 74, 13, 45, 104, 96, 23, 44, 16, 77, 108, 31, 95, 40, 7, 9, 42, 21, 106, 28, 80, 73, 69, 99, 5, 0, 75, 71, 85, 11, 8, 64, 100, 6, 72, 70, 36, 2, 68, 66, 67, 4, 1, 65, 3], [112, 120, 48, 55, 59, 62, 56, 121, 63, 118, 41, 124, 105, 110, 57, 116, 86, 38, 102, 47, 22, 53, 58, 98, 29, 125, 93, 126, 20, 84, 46, 119, 49, 113, 127, 60, 122, 91, 89, 25, 50, 24, 81, 37, 111, 109, 88, 27, 19, 90, 34, 54, 94, 79, 51, 61, 12, 15, 76, 17, 35, 33, 97, 18, 117, 115, 107, 78, 26, 14, 83, 32, 39, 103, 101, 123, 52, 30, 114, 43, 23, 108, 45, 87, 8, 92, 13, 10, 82, 77, 96, 74, 95, 16, 42, 64, 31, 0, 11, 104, 44, 99, 9, 7, 21, 69, 5, 71, 80, 70, 28, 106, 36, 73, 100, 75, 66, 2, 3, 85, 40, 68, 4, 1, 67, 72, 6, 65], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 124, 41, 110, 105, 57, 116, 38, 86, 102, 47, 126, 58, 53, 22, 29, 98, 84, 93, 89, 50, 46, 27, 91, 25, 20, 113, 123, 119, 49, 127, 51, 122, 125, 34, 81, 94, 15, 61, 37, 19, 76, 117, 24, 12, 115, 109, 78, 17, 88, 54, 79, 14, 60, 90, 107, 39, 83, 114, 97, 32, 111, 101, 35, 30, 33, 52, 18, 82, 45, 43, 23, 92, 87, 103, 8, 108, 104, 10, 26, 96, 95, 74, 13, 44, 106, 77, 0, 70, 69, 16, 64, 31, 71, 99, 5, 73, 42, 9, 80, 11, 7, 21, 75, 2, 40, 85, 66, 36, 3, 100, 68, 28, 6, 67, 4, 72, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 124, 41, 110, 105, 57, 116, 38, 86, 47, 22, 102, 53, 98, 126, 20, 58, 29, 125, 84, 61, 25, 60, 51, 27, 46, 93, 122, 127, 119, 78, 17, 81, 19, 76, 49, 111, 37, 117, 12, 91, 89, 88, 50, 113, 79, 34, 24, 94, 15, 115, 90, 123, 14, 54, 97, 103, 18, 83, 101, 109, 30, 45, 33, 8, 43, 82, 106, 52, 32, 39, 13, 107, 74, 35, 23, 10, 114, 77, 73, 104, 96, 87, 108, 44, 31, 16, 11, 26, 92, 28, 71, 5, 64, 95, 7, 99, 9, 69, 70, 100, 85, 0, 42, 36, 21, 75, 40, 68, 66, 80, 2, 67, 3, 4, 72, 6, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 57, 105, 116, 86, 38, 102, 47, 22, 98, 84, 25, 29, 20, 126, 119, 53, 58, 27, 17, 113, 46, 49, 37, 81, 89, 76, 12, 88, 24, 78, 60, 93, 15, 83, 19, 122, 125, 127, 52, 50, 54, 14, 117, 123, 79, 111, 51, 91, 61, 90, 94, 115, 109, 97, 82, 34, 33, 13, 18, 39, 32, 43, 74, 101, 107, 10, 30, 8, 114, 103, 35, 96, 92, 44, 23, 45, 87, 16, 106, 104, 21, 108, 73, 11, 31, 28, 5, 80, 71, 77, 85, 26, 0, 42, 9, 75, 95, 69, 7, 99, 70, 64, 2, 66, 40, 6, 36, 100, 67, 3, 68, 72, 4, 1, 65], [112, 120, 55, 48, 62, 59, 121, 56, 63, 118, 41, 124, 57, 110, 105, 116, 86, 102, 38, 47, 22, 29, 20, 84, 98, 58, 17, 125, 53, 119, 24, 60, 25, 93, 111, 126, 27, 109, 12, 46, 19, 50, 49, 89, 76, 90, 14, 37, 81, 52, 91, 122, 78, 79, 88, 39, 117, 113, 83, 127, 15, 94, 61, 34, 97, 123, 82, 114, 32, 51, 18, 45, 33, 101, 54, 115, 92, 30, 8, 13, 96, 23, 10, 43, 103, 74, 104, 26, 35, 77, 87, 16, 31, 80, 107, 21, 85, 28, 9, 11, 44, 42, 108, 71, 106, 5, 7, 95, 99, 73, 100, 69, 6, 75, 64, 70, 40, 36, 0, 2, 68, 3, 67, 66, 1, 4, 72, 65], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 105, 57, 110, 116, 86, 102, 38, 47, 53, 22, 29, 84, 98, 93, 24, 58, 126, 119, 20, 60, 111, 109, 27, 46, 37, 17, 89, 122, 25, 91, 50, 117, 81, 76, 88, 125, 12, 49, 54, 19, 90, 15, 107, 14, 83, 79, 94, 114, 78, 39, 113, 34, 123, 32, 115, 61, 97, 33, 101, 127, 51, 18, 35, 82, 92, 52, 26, 43, 103, 96, 23, 45, 77, 10, 30, 42, 104, 87, 8, 16, 74, 31, 44, 28, 13, 80, 21, 99, 7, 9, 71, 108, 5, 11, 36, 40, 85, 6, 106, 95, 64, 73, 0, 69, 75, 100, 2, 66, 72, 3, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 118, 121, 41, 124, 105, 110, 57, 116, 86, 102, 47, 38, 53, 22, 29, 126, 84, 98, 119, 37, 93, 60, 122, 25, 58, 27, 91, 17, 20, 88, 24, 125, 109, 81, 114, 54, 94, 46, 50, 111, 113, 89, 12, 15, 49, 19, 127, 83, 33, 76, 123, 14, 117, 79, 107, 39, 61, 90, 97, 51, 34, 32, 35, 115, 52, 101, 18, 78, 103, 43, 13, 30, 82, 77, 92, 104, 26, 42, 87, 74, 23, 45, 10, 28, 31, 108, 44, 96, 16, 7, 99, 21, 6, 80, 36, 9, 8, 40, 73, 95, 11, 106, 5, 71, 85, 75, 64, 69, 100, 72, 0, 2, 70, 3, 67, 4, 66, 68, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 63, 121, 41, 124, 57, 105, 110, 116, 102, 38, 86, 47, 22, 53, 98, 84, 58, 29, 125, 60, 126, 25, 93, 20, 37, 49, 12, 46, 119, 27, 88, 89, 122, 113, 50, 91, 81, 117, 61, 127, 109, 114, 19, 123, 76, 54, 94, 111, 24, 17, 97, 83, 79, 34, 107, 15, 115, 39, 32, 51, 14, 30, 10, 78, 103, 33, 101, 90, 92, 35, 104, 31, 82, 45, 74, 43, 77, 18, 13, 44, 42, 52, 23, 108, 26, 96, 9, 87, 106, 6, 16, 7, 99, 80, 95, 5, 0, 71, 72, 73, 69, 11, 28, 8, 64, 36, 40, 75, 21, 100, 85, 68, 67, 70, 66, 2, 4, 3, 1, 65], [112, 120, 55, 48, 62, 59, 118, 121, 56, 63, 124, 41, 110, 57, 105, 116, 38, 86, 102, 22, 53, 47, 29, 46, 20, 98, 25, 60, 84, 50, 126, 125, 122, 27, 58, 93, 119, 81, 117, 12, 61, 97, 89, 76, 88, 14, 101, 37, 78, 17, 113, 19, 15, 51, 91, 49, 83, 39, 24, 79, 127, 111, 10, 18, 114, 115, 54, 34, 94, 103, 30, 90, 109, 106, 123, 33, 107, 32, 72, 74, 82, 35, 43, 104, 77, 11, 5, 9, 92, 6, 96, 45, 13, 42, 31, 23, 52, 87, 16, 108, 26, 64, 44, 69, 73, 80, 7, 71, 0, 28, 2, 66, 21, 75, 95, 85, 36, 99, 8, 70, 68, 40, 67, 3, 4, 100, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 124, 41, 105, 110, 57, 116, 86, 38, 47, 22, 102, 53, 29, 20, 84, 60, 98, 119, 27, 81, 93, 126, 50, 25, 17, 125, 46, 122, 58, 88, 37, 117, 12, 19, 89, 76, 24, 78, 34, 113, 109, 79, 83, 51, 97, 49, 15, 94, 61, 39, 14, 18, 111, 91, 107, 90, 32, 33, 115, 114, 74, 54, 101, 10, 103, 127, 106, 82, 52, 30, 123, 35, 87, 77, 104, 72, 26, 42, 31, 11, 13, 16, 9, 108, 96, 23, 80, 28, 45, 92, 43, 73, 44, 21, 99, 71, 95, 5, 85, 7, 0, 64, 69, 75, 6, 66, 36, 70, 3, 2, 100, 40, 68, 4, 8, 67, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 63, 118, 41, 124, 110, 105, 57, 116, 86, 102, 47, 38, 53, 22, 119, 29, 84, 98, 58, 60, 20, 125, 93, 24, 27, 50, 126, 37, 122, 109, 89, 88, 117, 17, 49, 25, 91, 81, 113, 114, 76, 61, 19, 111, 12, 33, 51, 39, 46, 54, 15, 78, 34, 123, 83, 115, 90, 127, 107, 79, 52, 94, 32, 103, 35, 18, 92, 101, 26, 82, 74, 97, 14, 30, 96, 28, 72, 10, 77, 87, 44, 23, 13, 45, 106, 31, 80, 108, 104, 42, 85, 16, 21, 71, 43, 9, 99, 95, 100, 36, 11, 7, 69, 40, 5, 73, 70, 75, 0, 64, 6, 66, 67, 2, 68, 4, 65, 3, 8, 1], [112, 120, 55, 48, 59, 62, 56, 121, 63, 118, 124, 41, 105, 57, 110, 116, 86, 102, 38, 47, 22, 60, 53, 125, 58, 98, 84, 93, 119, 50, 29, 126, 27, 46, 25, 20, 61, 122, 113, 109, 12, 91, 49, 37, 117, 81, 17, 88, 89, 111, 115, 19, 76, 35, 24, 127, 34, 94, 101, 15, 54, 52, 97, 51, 79, 14, 114, 78, 33, 30, 107, 123, 72, 32, 18, 39, 26, 83, 90, 10, 82, 103, 74, 13, 104, 31, 23, 87, 96, 80, 77, 73, 92, 7, 16, 71, 106, 0, 42, 69, 45, 21, 95, 43, 108, 44, 70, 9, 99, 11, 85, 5, 28, 64, 40, 75, 100, 36, 66, 2, 4, 67, 6, 68, 3, 65, 1, 8], [112, 120, 55, 48, 62, 59, 56, 118, 121, 63, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 98, 125, 84, 22, 29, 119, 53, 25, 20, 93, 60, 27, 12, 17, 58, 24, 126, 81, 50, 37, 117, 76, 113, 94, 46, 34, 49, 91, 78, 61, 19, 15, 88, 89, 14, 39, 122, 109, 111, 101, 97, 79, 35, 52, 83, 30, 18, 51, 72, 33, 54, 115, 10, 127, 123, 103, 82, 74, 90, 107, 13, 32, 26, 31, 23, 114, 73, 87, 106, 45, 92, 96, 9, 80, 71, 16, 77, 43, 69, 44, 64, 70, 104, 108, 11, 85, 5, 7, 0, 36, 99, 28, 95, 66, 40, 42, 21, 75, 6, 67, 68, 3, 2, 8, 100, 65, 4, 1], [112, 120, 48, 55, 62, 59, 56, 63, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 22, 53, 29, 58, 98, 84, 20, 27, 125, 17, 117, 46, 76, 119, 126, 25, 60, 37, 81, 93, 113, 49, 50, 61, 24, 12, 34, 122, 89, 15, 91, 88, 19, 14, 94, 101, 78, 107, 79, 123, 51, 109, 52, 97, 115, 111, 83, 127, 35, 10, 54, 74, 39, 33, 18, 30, 82, 13, 90, 31, 32, 106, 72, 114, 103, 9, 26, 43, 77, 11, 73, 80, 44, 87, 45, 104, 92, 96, 42, 69, 71, 23, 95, 7, 108, 16, 70, 40, 85, 64, 5, 21, 75, 28, 99, 36, 0, 8, 2, 66, 67, 6, 100, 3, 68, 4, 1, 65], [112, 120, 48, 55, 62, 59, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 86, 38, 102, 22, 53, 47, 98, 126, 29, 60, 58, 119, 20, 84, 125, 27, 46, 88, 50, 34, 49, 93, 37, 91, 89, 17, 81, 24, 25, 117, 122, 19, 113, 76, 109, 127, 12, 51, 33, 90, 83, 61, 32, 111, 94, 52, 79, 15, 97, 54, 78, 82, 39, 107, 92, 101, 18, 14, 114, 123, 30, 26, 74, 115, 43, 103, 35, 87, 96, 31, 10, 106, 13, 104, 77, 44, 23, 80, 16, 85, 28, 95, 42, 21, 45, 108, 40, 99, 36, 72, 8, 73, 5, 71, 9, 64, 70, 0, 11, 7, 75, 69, 100, 66, 6, 3, 2, 68, 4, 67, 1, 65], [112, 120, 48, 55, 62, 59, 56, 118, 121, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 98, 20, 29, 93, 113, 60, 46, 119, 84, 58, 125, 37, 89, 27, 91, 126, 25, 122, 17, 88, 50, 12, 49, 24, 76, 19, 117, 81, 109, 94, 15, 14, 79, 90, 127, 123, 83, 78, 61, 32, 34, 33, 30, 82, 74, 114, 107, 111, 39, 97, 115, 101, 103, 52, 54, 35, 51, 26, 10, 92, 18, 43, 13, 104, 87, 77, 31, 96, 23, 80, 0, 42, 73, 71, 16, 45, 28, 85, 106, 8, 21, 36, 7, 9, 5, 69, 108, 44, 95, 6, 64, 99, 75, 72, 11, 40, 70, 66, 2, 100, 67, 3, 4, 68, 65, 1], [112, 120, 48, 55, 62, 59, 63, 118, 56, 121, 41, 124, 110, 105, 57, 116, 38, 86, 47, 102, 53, 98, 22, 46, 125, 126, 29, 50, 93, 117, 27, 20, 25, 58, 84, 122, 89, 17, 127, 113, 37, 119, 114, 115, 91, 60, 81, 49, 19, 61, 12, 24, 94, 34, 33, 14, 109, 76, 52, 88, 123, 79, 83, 97, 15, 30, 101, 107, 51, 32, 90, 39, 54, 92, 104, 35, 82, 78, 74, 103, 8, 18, 31, 111, 43, 10, 87, 26, 45, 13, 96, 108, 77, 106, 44, 73, 23, 36, 6, 40, 16, 0, 80, 9, 71, 64, 69, 7, 42, 21, 28, 5, 11, 75, 95, 99, 85, 100, 3, 2, 70, 68, 66, 72, 4, 67, 65, 1], [112, 120, 48, 55, 62, 59, 63, 56, 118, 121, 41, 124, 110, 105, 57, 116, 86, 102, 38, 47, 53, 98, 22, 29, 125, 46, 84, 58, 25, 126, 27, 117, 89, 37, 20, 93, 119, 91, 17, 61, 24, 12, 50, 34, 60, 76, 49, 113, 109, 83, 14, 127, 111, 88, 81, 51, 115, 122, 33, 94, 114, 79, 32, 19, 123, 15, 78, 10, 54, 8, 97, 74, 107, 90, 104, 103, 82, 101, 39, 43, 30, 52, 92, 31, 77, 35, 106, 18, 13, 96, 26, 87, 80, 108, 44, 45, 28, 23, 6, 21, 73, 16, 9, 11, 95, 71, 69, 99, 75, 42, 5, 7, 64, 85, 0, 40, 36, 100, 66, 2, 70, 68, 72, 4, 1, 67, 3, 65], [112, 120, 48, 55, 62, 56, 59, 118, 63, 121, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 53, 22, 98, 119, 25, 126, 125, 29, 20, 93, 27, 58, 60, 117, 61, 46, 84, 114, 89, 127, 113, 50, 91, 76, 37, 24, 122, 49, 109, 17, 19, 83, 81, 115, 78, 88, 123, 12, 94, 79, 14, 97, 30, 34, 52, 51, 15, 54, 107, 33, 18, 103, 32, 10, 43, 82, 111, 39, 101, 90, 8, 28, 106, 77, 13, 26, 35, 9, 104, 74, 44, 42, 92, 45, 87, 31, 80, 21, 23, 96, 16, 75, 95, 69, 6, 71, 73, 85, 11, 99, 7, 108, 5, 0, 36, 40, 64, 2, 70, 4, 67, 66, 3, 100, 68, 72, 65, 1], [112, 120, 48, 55, 62, 59, 56, 121, 63, 118, 41, 124, 110, 105, 57, 116, 86, 38, 102, 47, 58, 22, 98, 53, 126, 29, 119, 93, 20, 117, 49, 50, 125, 61, 25, 27, 51, 60, 46, 89, 84, 37, 115, 17, 91, 113, 123, 83, 114, 109, 88, 122, 76, 12, 127, 24, 81, 54, 52, 94, 14, 79, 19, 97, 18, 15, 103, 78, 32, 33, 34, 8, 30, 39, 43, 107, 92, 26, 82, 45, 111, 74, 90, 10, 13, 77, 87, 35, 31, 101, 96, 104, 21, 28, 80, 23, 44, 95, 9, 64, 0, 108, 69, 40, 106, 42, 7, 99, 5, 16, 71, 11, 73, 85, 75, 6, 36, 70, 100, 66, 4, 2, 67, 65, 3, 68, 1, 72], [112, 120, 48, 55, 59, 62, 56, 121, 118, 63, 41, 124, 105, 110, 57, 116, 38, 86, 102, 47, 22, 53, 58, 98, 119, 126, 29, 50, 60, 49, 20, 117, 113, 27, 91, 93, 61, 51, 25, 46, 89, 24, 17, 37, 125, 123, 84, 122, 76, 127, 114, 111, 88, 81, 12, 54, 15, 115, 94, 52, 19, 79, 33, 109, 83, 30, 78, 34, 107, 14, 90, 35, 97, 32, 103, 92, 26, 101, 43, 45, 77, 82, 18, 8, 10, 106, 104, 31, 39, 44, 74, 13, 87, 99, 21, 23, 108, 36, 80, 28, 16, 95, 96, 9, 7, 42, 0, 70, 73, 85, 71, 5, 11, 75, 40, 100, 69, 6, 64, 2, 66, 4, 67, 68, 72, 3, 1, 65], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 58, 29, 126, 98, 53, 119, 25, 20, 84, 27, 89, 125, 93, 113, 50, 24, 61, 91, 117, 60, 46, 15, 115, 81, 37, 88, 17, 94, 109, 123, 76, 122, 12, 127, 49, 51, 111, 97, 32, 34, 114, 78, 19, 14, 83, 33, 79, 30, 10, 107, 54, 39, 82, 18, 35, 74, 104, 92, 13, 101, 103, 90, 96, 8, 73, 52, 31, 77, 43, 87, 21, 23, 26, 106, 44, 45, 9, 16, 71, 28, 7, 42, 69, 75, 95, 80, 40, 70, 11, 5, 108, 85, 100, 0, 99, 2, 6, 64, 68, 36, 72, 67, 66, 4, 3, 65, 1], [112, 120, 48, 55, 62, 59, 121, 56, 118, 63, 41, 124, 110, 105, 57, 116, 86, 38, 22, 47, 102, 58, 98, 60, 20, 53, 29, 84, 25, 113, 126, 89, 119, 125, 17, 88, 81, 24, 50, 27, 61, 12, 93, 76, 49, 78, 97, 37, 19, 117, 123, 46, 91, 127, 109, 34, 122, 15, 111, 14, 83, 107, 54, 51, 103, 79, 39, 10, 33, 101, 115, 32, 94, 82, 114, 18, 90, 26, 30, 74, 13, 77, 52, 35, 43, 92, 31, 106, 87, 96, 28, 23, 9, 16, 45, 80, 104, 7, 21, 42, 44, 73, 70, 75, 69, 108, 8, 71, 5, 72, 11, 85, 0, 64, 40, 36, 100, 99, 95, 66, 2, 3, 68, 67, 6, 4, 65, 1], [112, 120, 48, 55, 62, 56, 59, 121, 118, 63, 124, 41, 105, 110, 57, 116, 86, 102, 38, 47, 53, 22, 125, 58, 98, 119, 29, 60, 84, 20, 113, 126, 49, 27, 91, 88, 46, 37, 89, 117, 122, 123, 93, 25, 61, 50, 76, 107, 12, 24, 34, 17, 127, 109, 111, 81, 32, 19, 78, 114, 51, 83, 54, 33, 115, 94, 97, 79, 103, 15, 30, 26, 35, 90, 14, 74, 39, 18, 82, 101, 43, 92, 87, 10, 96, 42, 44, 23, 31, 77, 28, 13, 45, 52, 21, 40, 16, 7, 95, 80, 104, 108, 99, 106, 72, 36, 70, 9, 85, 75, 69, 73, 71, 8, 5, 0, 100, 11, 64, 2, 66, 67, 6, 68, 1, 3, 4, 65], [112, 120, 48, 55, 56, 62, 59, 63, 121, 118, 124, 41, 105, 110, 57, 116, 47, 38, 86, 102, 53, 22, 98, 119, 126, 50, 58, 29, 20, 60, 125, 122, 113, 91, 93, 27, 61, 37, 25, 88, 111, 49, 84, 114, 46, 89, 107, 117, 51, 34, 24, 115, 109, 81, 123, 17, 76, 127, 15, 30, 12, 90, 39, 54, 83, 97, 33, 79, 14, 32, 101, 35, 103, 19, 94, 43, 92, 42, 104, 18, 78, 23, 10, 82, 52, 74, 44, 45, 26, 87, 77, 72, 96, 13, 16, 106, 99, 95, 108, 21, 28, 36, 75, 31, 80, 5, 71, 73, 9, 70, 69, 40, 100, 7, 64, 11, 0, 2, 85, 67, 66, 4, 8, 6, 3, 68, 65, 1], [112, 120, 48, 55, 56, 62, 59, 121, 63, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 22, 53, 98, 126, 119, 84, 58, 29, 125, 122, 50, 20, 46, 54, 60, 89, 93, 113, 17, 24, 27, 127, 25, 76, 12, 49, 51, 37, 78, 115, 81, 61, 91, 15, 117, 19, 109, 79, 123, 111, 83, 88, 97, 14, 114, 34, 39, 18, 94, 30, 32, 33, 10, 35, 90, 103, 101, 72, 107, 52, 74, 45, 13, 82, 77, 43, 92, 26, 87, 9, 16, 23, 71, 73, 64, 31, 44, 7, 5, 28, 96, 42, 108, 104, 75, 80, 21, 69, 0, 95, 99, 106, 40, 85, 70, 11, 36, 66, 6, 2, 8, 100, 3, 68, 67, 4, 65, 1], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 38, 102, 47, 98, 22, 58, 53, 60, 125, 119, 20, 122, 46, 113, 25, 126, 29, 27, 84, 51, 50, 89, 49, 76, 83, 93, 81, 127, 114, 78, 37, 12, 17, 91, 111, 97, 61, 79, 19, 24, 117, 15, 115, 88, 34, 109, 94, 54, 33, 101, 123, 45, 35, 14, 82, 43, 30, 92, 10, 90, 18, 52, 72, 107, 32, 44, 103, 104, 31, 77, 106, 9, 74, 23, 39, 13, 108, 87, 0, 69, 64, 21, 75, 5, 96, 71, 26, 73, 95, 6, 36, 16, 28, 7, 80, 85, 66, 42, 11, 99, 40, 67, 4, 100, 68, 2, 3, 1, 70, 8, 65], [112, 120, 55, 48, 62, 59, 56, 121, 118, 63, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 22, 58, 98, 53, 20, 119, 46, 60, 29, 122, 125, 84, 49, 113, 76, 93, 25, 24, 123, 91, 27, 51, 37, 12, 81, 17, 126, 54, 89, 19, 88, 15, 14, 50, 83, 78, 115, 127, 79, 111, 34, 97, 114, 117, 101, 82, 109, 94, 61, 90, 72, 10, 33, 39, 43, 32, 30, 35, 18, 107, 87, 26, 103, 52, 92, 74, 44, 45, 106, 13, 96, 9, 23, 77, 28, 16, 6, 108, 5, 21, 104, 85, 95, 73, 71, 42, 80, 31, 7, 36, 75, 40, 69, 11, 64, 99, 2, 0, 100, 66, 3, 70, 67, 4, 8, 68, 1, 65], [112, 120, 55, 48, 59, 62, 121, 56, 118, 63, 124, 41, 110, 105, 57, 116, 38, 102, 86, 47, 98, 22, 58, 53, 60, 122, 29, 46, 20, 113, 119, 125, 126, 84, 89, 49, 24, 12, 93, 76, 51, 50, 81, 91, 127, 115, 27, 25, 123, 17, 88, 37, 97, 15, 61, 78, 14, 111, 54, 83, 101, 19, 30, 34, 72, 79, 103, 117, 94, 39, 33, 35, 109, 107, 92, 114, 32, 10, 45, 18, 82, 90, 52, 26, 74, 9, 108, 106, 6, 23, 71, 13, 44, 5, 16, 69, 95, 73, 87, 7, 96, 80, 21, 99, 31, 77, 64, 43, 85, 104, 28, 11, 75, 40, 0, 42, 2, 3, 100, 66, 36, 8, 4, 68, 70, 67, 1, 65], [112, 120, 55, 48, 62, 59, 56, 63, 121, 118, 124, 41, 110, 105, 57, 116, 86, 102, 38, 47, 58, 22, 53, 98, 50, 20, 93, 29, 60, 113, 24, 126, 84, 125, 89, 25, 49, 119, 37, 81, 12, 27, 46, 111, 91, 127, 17, 76, 61, 122, 51, 78, 15, 123, 88, 19, 79, 14, 97, 109, 94, 115, 32, 54, 90, 33, 39, 117, 34, 83, 35, 101, 52, 10, 45, 74, 92, 114, 72, 103, 107, 82, 18, 30, 26, 96, 87, 77, 44, 43, 106, 13, 108, 73, 23, 80, 16, 71, 11, 36, 6, 21, 95, 5, 9, 28, 31, 7, 75, 104, 99, 40, 42, 69, 85, 64, 100, 4, 3, 8, 70, 0, 67, 2, 68, 66, 65, 1]], "model.layers.20.self_attn.q_proj": [[114, 126, 101, 110, 46, 98, 19, 86, 29, 92, 25, 120, 24, 67, 124, 82, 12, 89, 9, 87, 28, 37, 119, 7, 49, 90, 16, 60, 23, 51, 13, 74, 115, 79, 31, 122, 58, 65, 127, 62, 77, 112, 121, 100, 116, 54, 4, 0, 97, 57, 88, 43, 118, 53, 125, 39, 61, 111, 108, 96, 34, 55, 50, 15, 14, 63, 32, 113, 47, 48, 40, 94, 30, 5, 93, 6, 56, 59, 64, 20, 109, 123, 42, 44, 117, 66, 1, 35, 72, 107, 68, 95, 45, 52, 2, 17, 27, 36, 83, 41, 26, 3, 70, 102, 38, 80, 104, 33, 105, 84, 106, 91, 81, 103, 18, 21, 99, 69, 8, 78, 73, 10, 85, 75, 71, 22, 76, 11], [46, 126, 114, 110, 101, 120, 98, 49, 124, 92, 55, 115, 58, 122, 19, 60, 119, 28, 37, 40, 54, 127, 86, 121, 116, 52, 53, 125, 118, 112, 57, 50, 51, 47, 61, 48, 62, 56, 108, 111, 90, 31, 42, 63, 113, 59, 24, 123, 43, 89, 109, 117, 105, 91, 45, 103, 44, 29, 106, 88, 15, 104, 82, 102, 39, 107, 41, 25, 95, 36, 16, 94, 35, 100, 23, 99, 85, 7, 38, 22, 33, 97, 13, 93, 96, 87, 34, 14, 79, 9, 32, 26, 21, 30, 27, 83, 73, 12, 17, 20, 84, 80, 8, 77, 81, 72, 11, 18, 78, 75, 66, 74, 10, 68, 6, 5, 4, 71, 76, 3, 1, 69, 64, 70, 2, 65, 67, 0], [126, 101, 110, 114, 46, 98, 28, 92, 120, 19, 86, 115, 124, 14, 24, 60, 58, 121, 82, 123, 23, 29, 89, 119, 116, 125, 37, 42, 39, 122, 36, 49, 31, 55, 118, 112, 103, 54, 109, 48, 25, 105, 127, 57, 62, 99, 47, 53, 51, 16, 111, 50, 59, 61, 56, 117, 26, 100, 91, 63, 33, 104, 95, 38, 32, 45, 30, 113, 83, 93, 52, 35, 94, 40, 41, 44, 85, 97, 87, 90, 96, 12, 107, 27, 108, 43, 84, 7, 102, 74, 34, 106, 21, 13, 81, 22, 15, 17, 88, 79, 80, 5, 8, 20, 18, 76, 78, 75, 71, 77, 3, 72, 9, 11, 73, 10, 66, 70, 69, 6, 1, 68, 67, 4, 2, 65, 0, 64], [114, 126, 46, 110, 101, 92, 98, 120, 119, 12, 124, 28, 19, 82, 60, 5, 51, 25, 86, 37, 127, 49, 115, 0, 62, 58, 16, 122, 50, 57, 74, 112, 117, 121, 7, 123, 61, 53, 38, 55, 113, 54, 68, 125, 118, 14, 31, 116, 47, 44, 48, 56, 109, 111, 63, 89, 87, 104, 43, 108, 2, 100, 36, 91, 45, 107, 33, 105, 94, 29, 52, 23, 59, 8, 41, 39, 6, 85, 77, 40, 70, 79, 3, 102, 42, 15, 35, 66, 18, 95, 32, 88, 69, 83, 99, 106, 96, 73, 75, 30, 9, 1, 71, 93, 64, 17, 27, 13, 103, 97, 24, 4, 20, 65, 78, 11, 67, 76, 84, 81, 21, 80, 22, 26, 72, 90, 10, 34], [43, 98, 93, 103, 107, 116, 46, 117, 78, 58, 21, 24, 81, 19, 127, 12, 63, 8, 51, 95, 121, 62, 92, 90, 120, 29, 37, 70, 26, 109, 41, 59, 101, 104, 119, 105, 100, 52, 48, 50, 36, 57, 87, 74, 86, 16, 27, 44, 7, 110, 73, 2, 72, 42, 54, 33, 67, 13, 32, 39, 56, 126, 115, 75, 53, 79, 114, 68, 83, 84, 3, 10, 125, 23, 64, 118, 9, 22, 1, 112, 61, 113, 80, 40, 25, 17, 47, 108, 85, 66, 106, 45, 30, 14, 97, 99, 38, 76, 123, 94, 35, 122, 96, 11, 82, 18, 60, 28, 49, 111, 31, 6, 15, 0, 55, 91, 71, 20, 77, 69, 89, 34, 102, 88, 124, 5, 4, 65], [43, 121, 107, 98, 93, 46, 63, 62, 52, 58, 118, 109, 38, 24, 116, 39, 127, 103, 54, 119, 37, 104, 19, 53, 51, 55, 26, 29, 44, 49, 56, 105, 21, 57, 110, 115, 48, 117, 41, 95, 50, 90, 61, 33, 47, 59, 32, 125, 112, 92, 120, 81, 123, 42, 106, 114, 108, 60, 23, 78, 111, 25, 126, 124, 122, 96, 86, 45, 113, 87, 34, 75, 12, 40, 100, 36, 101, 70, 9, 91, 83, 97, 73, 102, 31, 35, 84, 28, 22, 30, 80, 88, 99, 18, 77, 10, 82, 68, 94, 11, 27, 20, 13, 79, 17, 89, 4, 15, 8, 16, 67, 1, 85, 7, 64, 6, 71, 74, 76, 3, 65, 69, 2, 5, 14, 66, 0, 72], [43, 113, 121, 116, 93, 98, 107, 59, 52, 24, 103, 63, 53, 125, 49, 29, 127, 105, 58, 51, 110, 95, 26, 48, 117, 54, 119, 112, 46, 124, 37, 92, 120, 47, 44, 106, 61, 21, 123, 118, 109, 114, 56, 62, 45, 111, 104, 60, 90, 115, 91, 41, 122, 126, 39, 108, 57, 100, 81, 42, 97, 50, 40, 32, 33, 87, 38, 23, 101, 55, 102, 36, 22, 19, 35, 96, 88, 34, 83, 86, 84, 99, 94, 31, 18, 30, 80, 27, 77, 28, 25, 82, 89, 75, 85, 20, 16, 15, 78, 12, 17, 9, 76, 13, 79, 73, 11, 74, 70, 14, 10, 6, 72, 7, 5, 4, 66, 68, 8, 0, 69, 65, 71, 67, 2, 3, 1, 64], [43, 98, 93, 107, 103, 46, 116, 121, 24, 81, 78, 51, 21, 12, 109, 62, 8, 63, 19, 120, 26, 7, 117, 52, 29, 69, 74, 127, 68, 58, 38, 70, 92, 119, 73, 65, 90, 3, 2, 105, 37, 16, 66, 125, 44, 95, 50, 86, 48, 10, 39, 75, 106, 54, 0, 33, 64, 47, 59, 87, 56, 4, 96, 5, 97, 49, 82, 53, 71, 41, 101, 45, 84, 67, 18, 113, 114, 123, 110, 115, 104, 83, 13, 77, 79, 118, 6, 126, 11, 111, 36, 60, 17, 55, 124, 61, 1, 108, 76, 57, 23, 28, 25, 42, 35, 20, 112, 27, 80, 30, 72, 22, 40, 99, 85, 100, 122, 31, 9, 91, 15, 32, 14, 89, 102, 94, 34, 88], [39, 109, 46, 97, 93, 110, 78, 17, 10, 71, 19, 45, 5, 85, 76, 86, 16, 65, 26, 3, 22, 116, 83, 9, 89, 66, 2, 121, 35, 90, 87, 88, 107, 81, 24, 64, 79, 21, 91, 80, 119, 103, 75, 11, 28, 12, 74, 29, 15, 82, 54, 8, 92, 72, 127, 7, 18, 111, 59, 13, 23, 77, 14, 94, 1, 84, 4, 123, 48, 69, 68, 67, 20, 32, 73, 6, 70, 0, 27, 118, 30, 25, 115, 34, 99, 33, 114, 96, 102, 113, 31, 100, 38, 43, 95, 101, 58, 44, 41, 51, 125, 120, 122, 57, 98, 53, 50, 55, 126, 40, 60, 36, 104, 52, 47, 106, 62, 37, 42, 61, 105, 63, 117, 124, 112, 49, 56, 108], [109, 39, 45, 93, 97, 85, 107, 123, 110, 16, 99, 116, 54, 26, 76, 59, 53, 119, 57, 90, 23, 121, 56, 60, 118, 58, 115, 24, 55, 125, 51, 112, 40, 126, 63, 111, 43, 114, 29, 48, 102, 19, 122, 113, 127, 52, 86, 44, 124, 9, 105, 47, 62, 49, 117, 92, 46, 108, 120, 37, 61, 50, 41, 30, 106, 21, 42, 83, 6, 31, 104, 36, 94, 98, 100, 17, 64, 28, 5, 103, 35, 101, 34, 88, 96, 38, 4, 32, 25, 95, 0, 18, 87, 12, 27, 80, 10, 89, 22, 78, 1, 33, 3, 65, 68, 66, 84, 70, 73, 20, 91, 71, 82, 67, 81, 79, 77, 11, 72, 15, 75, 13, 2, 69, 8, 7, 14, 74], [109, 39, 45, 97, 93, 107, 123, 85, 110, 46, 26, 16, 86, 76, 23, 19, 121, 24, 83, 90, 29, 119, 59, 116, 51, 102, 111, 127, 57, 118, 99, 54, 17, 22, 10, 125, 53, 78, 48, 122, 58, 114, 55, 112, 60, 113, 63, 21, 124, 56, 126, 115, 27, 30, 104, 120, 50, 49, 62, 52, 12, 92, 44, 105, 61, 108, 47, 96, 35, 103, 42, 41, 89, 40, 117, 43, 15, 36, 88, 101, 37, 5, 79, 100, 32, 38, 71, 64, 33, 106, 94, 80, 95, 34, 9, 11, 6, 98, 18, 68, 82, 28, 87, 3, 77, 31, 81, 91, 25, 84, 75, 13, 20, 66, 70, 1, 0, 65, 67, 14, 4, 8, 74, 72, 73, 69, 7, 2], [39, 109, 110, 46, 97, 45, 93, 102, 86, 26, 85, 118, 117, 96, 19, 24, 123, 17, 111, 92, 57, 35, 30, 15, 60, 125, 49, 59, 116, 53, 58, 29, 121, 119, 113, 54, 99, 127, 78, 44, 63, 48, 27, 89, 122, 52, 107, 55, 51, 124, 47, 11, 112, 36, 56, 98, 126, 95, 41, 37, 62, 61, 83, 115, 40, 108, 42, 114, 106, 101, 50, 43, 105, 100, 10, 120, 32, 87, 31, 34, 28, 104, 88, 22, 76, 75, 84, 90, 16, 9, 91, 23, 25, 94, 18, 5, 79, 38, 20, 0, 21, 4, 82, 6, 13, 80, 33, 77, 70, 12, 1, 2, 8, 71, 103, 81, 65, 14, 64, 69, 72, 67, 68, 73, 3, 74, 7, 66], [51, 101, 120, 119, 97, 127, 25, 115, 82, 37, 113, 110, 118, 46, 104, 22, 28, 62, 89, 31, 93, 78, 53, 109, 21, 94, 116, 42, 122, 121, 98, 75, 30, 27, 123, 117, 126, 23, 87, 20, 59, 52, 96, 54, 47, 69, 44, 114, 48, 60, 9, 124, 81, 61, 63, 57, 56, 83, 58, 111, 55, 16, 50, 80, 125, 86, 91, 49, 95, 77, 112, 88, 24, 29, 41, 10, 90, 92, 14, 105, 45, 107, 108, 73, 3, 19, 84, 99, 71, 85, 39, 34, 43, 103, 106, 102, 100, 35, 70, 18, 38, 64, 32, 26, 5, 40, 15, 17, 8, 36, 12, 68, 67, 65, 66, 7, 74, 72, 33, 76, 13, 2, 1, 4, 11, 79, 0, 6], [127, 120, 51, 119, 101, 109, 97, 56, 113, 107, 82, 118, 102, 54, 106, 62, 59, 117, 25, 93, 124, 123, 116, 52, 126, 122, 57, 37, 38, 121, 114, 104, 55, 105, 48, 50, 125, 49, 98, 115, 112, 58, 47, 91, 44, 60, 89, 63, 35, 45, 61, 111, 110, 40, 17, 30, 46, 92, 28, 53, 23, 85, 90, 94, 42, 108, 18, 34, 36, 43, 31, 39, 96, 84, 27, 103, 41, 100, 99, 24, 88, 95, 79, 29, 26, 15, 12, 19, 32, 81, 73, 33, 83, 87, 21, 11, 78, 20, 22, 16, 77, 75, 13, 86, 1, 66, 76, 7, 0, 80, 14, 72, 69, 3, 70, 6, 9, 4, 68, 74, 10, 5, 2, 8, 67, 64, 65, 71], [120, 51, 119, 101, 25, 89, 97, 37, 47, 118, 127, 115, 62, 117, 113, 123, 84, 122, 54, 121, 52, 60, 126, 53, 55, 116, 125, 46, 56, 16, 57, 49, 50, 59, 124, 61, 63, 109, 114, 58, 48, 110, 93, 111, 28, 44, 30, 22, 42, 108, 112, 45, 12, 29, 107, 40, 92, 41, 43, 80, 103, 39, 38, 104, 105, 86, 106, 87, 73, 20, 27, 31, 95, 102, 18, 35, 98, 33, 69, 7, 36, 3, 66, 100, 34, 14, 99, 88, 77, 72, 94, 82, 76, 17, 32, 10, 1, 96, 13, 91, 70, 85, 15, 8, 74, 75, 4, 19, 83, 24, 78, 26, 11, 0, 5, 68, 21, 9, 23, 71, 79, 81, 90, 6, 67, 64, 2, 65], [119, 101, 51, 127, 120, 84, 97, 113, 115, 54, 59, 123, 62, 118, 82, 92, 30, 50, 37, 46, 27, 124, 126, 89, 25, 28, 43, 47, 110, 125, 61, 52, 45, 121, 40, 109, 53, 100, 122, 42, 102, 18, 107, 116, 58, 60, 49, 104, 48, 35, 15, 57, 114, 44, 63, 117, 95, 56, 105, 91, 96, 36, 22, 111, 108, 77, 106, 112, 73, 55, 90, 34, 41, 32, 99, 20, 38, 33, 39, 103, 98, 12, 79, 19, 26, 94, 86, 17, 93, 31, 81, 88, 29, 24, 87, 11, 13, 16, 21, 23, 74, 83, 76, 80, 10, 85, 14, 75, 72, 9, 71, 69, 6, 78, 70, 7, 68, 8, 0, 5, 67, 2, 4, 1, 3, 66, 65, 64], [110, 50, 37, 46, 26, 38, 88, 114, 41, 58, 98, 30, 36, 82, 53, 94, 90, 85, 97, 28, 34, 84, 117, 109, 100, 122, 20, 96, 123, 86, 57, 87, 104, 39, 51, 21, 25, 111, 81, 126, 120, 102, 99, 62, 22, 61, 113, 52, 45, 76, 112, 48, 116, 115, 27, 95, 29, 43, 78, 44, 79, 42, 103, 59, 47, 101, 108, 17, 105, 107, 83, 18, 121, 127, 106, 23, 31, 56, 60, 119, 54, 40, 125, 49, 124, 63, 10, 33, 91, 35, 32, 89, 55, 24, 92, 16, 4, 93, 118, 64, 75, 6, 66, 13, 72, 65, 15, 80, 77, 7, 74, 11, 73, 14, 19, 5, 12, 3, 67, 70, 68, 71, 8, 1, 9, 69, 2, 0], [110, 50, 46, 37, 58, 26, 97, 88, 41, 84, 96, 62, 114, 90, 106, 109, 30, 78, 126, 20, 92, 85, 86, 81, 34, 28, 87, 53, 123, 94, 99, 83, 104, 91, 23, 122, 38, 117, 57, 31, 93, 39, 10, 124, 63, 98, 112, 11, 21, 42, 51, 56, 48, 127, 61, 125, 17, 108, 111, 107, 116, 29, 16, 105, 115, 3, 24, 52, 118, 49, 121, 60, 7, 55, 6, 113, 70, 43, 54, 103, 47, 45, 36, 68, 102, 59, 80, 4, 32, 33, 120, 25, 95, 119, 100, 64, 44, 76, 40, 89, 77, 27, 19, 14, 82, 15, 18, 73, 35, 0, 75, 5, 79, 66, 74, 71, 72, 101, 22, 8, 9, 12, 65, 67, 13, 1, 69, 2], [110, 50, 37, 58, 46, 26, 41, 98, 97, 30, 109, 87, 122, 90, 84, 88, 94, 31, 85, 62, 60, 81, 28, 125, 78, 29, 63, 35, 20, 91, 36, 43, 126, 48, 17, 86, 121, 124, 61, 83, 51, 32, 102, 100, 54, 15, 118, 117, 114, 123, 52, 93, 92, 47, 45, 127, 56, 11, 112, 111, 113, 40, 107, 120, 116, 4, 103, 22, 27, 108, 106, 55, 104, 53, 24, 42, 101, 76, 49, 99, 89, 21, 115, 34, 96, 16, 38, 57, 33, 44, 39, 66, 6, 18, 59, 75, 119, 7, 95, 19, 79, 25, 105, 23, 8, 80, 72, 77, 10, 73, 14, 64, 74, 82, 70, 12, 5, 3, 65, 13, 67, 69, 71, 9, 2, 68, 1, 0], [110, 50, 37, 46, 58, 26, 97, 84, 30, 41, 109, 88, 28, 78, 90, 96, 117, 94, 126, 17, 47, 20, 91, 87, 85, 53, 106, 108, 81, 114, 29, 122, 105, 93, 86, 107, 11, 31, 111, 48, 34, 23, 40, 95, 45, 57, 43, 124, 16, 92, 119, 123, 49, 59, 83, 51, 15, 21, 118, 62, 63, 112, 104, 55, 98, 36, 24, 56, 116, 4, 61, 10, 127, 113, 44, 65, 54, 60, 121, 18, 103, 5, 120, 125, 39, 77, 52, 38, 100, 72, 115, 35, 89, 102, 76, 6, 99, 33, 42, 32, 74, 79, 75, 7, 70, 3, 27, 80, 73, 66, 14, 22, 25, 82, 12, 13, 101, 71, 0, 19, 8, 68, 9, 64, 67, 2, 1, 69], [125, 48, 112, 37, 53, 124, 46, 126, 61, 117, 100, 113, 121, 54, 51, 116, 57, 56, 59, 58, 114, 34, 120, 127, 118, 52, 60, 122, 123, 50, 49, 63, 62, 55, 110, 115, 97, 111, 119, 47, 109, 45, 108, 42, 44, 102, 43, 105, 103, 107, 41, 99, 104, 26, 39, 40, 106, 32, 35, 84, 38, 101, 86, 90, 96, 94, 95, 82, 88, 36, 28, 33, 30, 98, 31, 29, 87, 22, 93, 73, 14, 76, 20, 23, 16, 92, 18, 24, 78, 25, 80, 27, 7, 89, 5, 12, 91, 3, 65, 69, 9, 85, 1, 71, 83, 0, 67, 17, 21, 64, 2, 4, 15, 66, 19, 70, 10, 11, 13, 75, 77, 68, 6, 8, 74, 81, 72, 79], [48, 100, 46, 112, 25, 85, 125, 17, 34, 87, 14, 15, 83, 27, 86, 80, 96, 12, 10, 40, 29, 84, 97, 98, 103, 13, 94, 95, 37, 93, 16, 9, 22, 75, 24, 99, 8, 78, 42, 124, 73, 28, 32, 110, 69, 6, 3, 36, 77, 118, 20, 117, 11, 81, 68, 26, 21, 5, 92, 71, 74, 76, 18, 30, 82, 90, 126, 88, 50, 121, 89, 23, 79, 47, 38, 19, 0, 91, 39, 105, 1, 31, 70, 7, 115, 35, 2, 41, 116, 67, 45, 54, 114, 65, 101, 120, 53, 4, 102, 63, 104, 61, 109, 43, 64, 72, 58, 113, 33, 57, 127, 59, 56, 119, 62, 123, 52, 49, 66, 108, 106, 111, 122, 51, 55, 107, 44, 60], [48, 112, 27, 100, 46, 85, 83, 34, 25, 97, 17, 13, 15, 125, 75, 8, 28, 87, 86, 32, 37, 30, 29, 4, 12, 10, 18, 72, 124, 68, 66, 6, 96, 16, 2, 24, 77, 40, 47, 91, 33, 20, 76, 93, 26, 70, 71, 7, 31, 84, 95, 14, 90, 22, 88, 9, 36, 0, 23, 82, 39, 121, 89, 126, 81, 21, 11, 79, 38, 92, 94, 74, 118, 73, 1, 99, 54, 98, 43, 103, 19, 78, 69, 110, 114, 117, 35, 80, 115, 5, 65, 102, 67, 53, 113, 3, 45, 59, 57, 101, 56, 64, 116, 50, 105, 44, 42, 123, 111, 41, 58, 61, 52, 49, 63, 62, 104, 120, 107, 60, 119, 51, 109, 108, 127, 55, 122, 106], [46, 48, 112, 125, 121, 37, 124, 54, 126, 57, 56, 117, 62, 53, 113, 59, 120, 58, 116, 61, 114, 127, 122, 49, 123, 51, 52, 63, 118, 115, 60, 110, 55, 50, 111, 119, 45, 47, 109, 43, 44, 100, 108, 102, 104, 106, 103, 107, 41, 39, 42, 105, 40, 38, 96, 34, 101, 98, 99, 97, 33, 32, 36, 95, 86, 35, 90, 88, 28, 22, 94, 84, 82, 26, 31, 30, 92, 29, 93, 25, 89, 65, 91, 87, 20, 78, 18, 7, 16, 24, 80, 27, 76, 14, 23, 0, 3, 12, 85, 67, 73, 2, 69, 71, 5, 83, 9, 1, 21, 17, 70, 4, 81, 19, 10, 64, 13, 15, 66, 8, 75, 79, 68, 74, 72, 77, 11, 6], [116, 122, 103, 57, 126, 55, 119, 118, 40, 127, 54, 51, 59, 100, 120, 114, 125, 124, 121, 61, 115, 112, 111, 50, 123, 85, 63, 53, 62, 113, 108, 56, 47, 117, 60, 30, 48, 58, 52, 46, 104, 49, 109, 42, 33, 110, 44, 102, 45, 106, 41, 43, 107, 29, 105, 38, 36, 24, 37, 27, 34, 99, 101, 35, 25, 32, 96, 98, 19, 31, 94, 95, 97, 39, 93, 23, 78, 28, 91, 87, 83, 21, 14, 22, 92, 89, 17, 86, 90, 26, 10, 1, 72, 0, 88, 8, 5, 3, 4, 16, 81, 80, 74, 75, 68, 13, 71, 64, 67, 66, 6, 18, 84, 69, 12, 2, 70, 7, 65, 77, 82, 9, 76, 79, 15, 11, 73, 20], [122, 116, 103, 55, 54, 120, 50, 40, 114, 118, 33, 53, 124, 24, 126, 100, 115, 59, 127, 19, 113, 117, 31, 125, 63, 121, 61, 57, 119, 112, 111, 60, 123, 56, 62, 58, 46, 110, 52, 51, 30, 102, 47, 42, 49, 48, 108, 27, 85, 74, 106, 109, 104, 22, 94, 80, 25, 44, 35, 107, 36, 45, 93, 43, 91, 41, 105, 32, 29, 38, 37, 84, 101, 28, 34, 26, 20, 98, 89, 82, 86, 23, 99, 97, 87, 96, 90, 21, 95, 83, 15, 72, 92, 88, 39, 5, 12, 0, 14, 18, 16, 78, 77, 3, 17, 73, 71, 6, 79, 76, 81, 68, 13, 10, 70, 9, 4, 2, 8, 66, 69, 64, 67, 75, 1, 65, 11, 7], [116, 122, 103, 55, 118, 54, 126, 50, 127, 61, 114, 57, 120, 112, 119, 121, 125, 115, 124, 62, 117, 40, 123, 33, 52, 53, 59, 60, 56, 113, 58, 104, 63, 111, 51, 100, 34, 91, 47, 49, 26, 36, 48, 93, 22, 110, 19, 46, 42, 44, 45, 25, 109, 102, 108, 24, 27, 107, 21, 106, 41, 31, 43, 80, 101, 94, 37, 35, 99, 38, 105, 29, 85, 28, 98, 86, 32, 39, 97, 96, 95, 89, 30, 92, 23, 78, 90, 74, 16, 14, 83, 82, 17, 88, 84, 87, 3, 18, 77, 12, 15, 10, 11, 1, 9, 68, 4, 81, 66, 0, 8, 20, 65, 13, 6, 79, 67, 70, 64, 71, 75, 69, 5, 2, 72, 73, 76, 7], [103, 116, 122, 93, 100, 84, 82, 55, 81, 33, 88, 15, 22, 75, 91, 87, 77, 52, 24, 11, 118, 79, 17, 71, 50, 26, 31, 119, 102, 126, 27, 40, 120, 13, 57, 32, 54, 89, 29, 23, 20, 34, 25, 90, 12, 21, 28, 72, 83, 125, 115, 121, 14, 18, 86, 59, 68, 94, 114, 61, 112, 85, 117, 124, 113, 127, 53, 96, 92, 123, 63, 8, 30, 111, 80, 58, 76, 62, 16, 110, 46, 47, 51, 7, 56, 104, 78, 69, 9, 10, 19, 60, 73, 97, 67, 95, 5, 37, 44, 49, 4, 66, 3, 108, 48, 99, 6, 74, 43, 109, 42, 70, 45, 98, 38, 36, 0, 107, 35, 106, 1, 64, 2, 105, 41, 101, 65, 39], [102, 113, 56, 97, 23, 49, 79, 29, 77, 81, 20, 75, 10, 72, 6, 3, 22, 16, 18, 32, 67, 38, 28, 15, 34, 5, 9, 88, 48, 17, 112, 65, 115, 92, 25, 70, 26, 74, 53, 78, 0, 39, 69, 86, 84, 63, 87, 64, 11, 31, 12, 105, 83, 19, 89, 66, 58, 117, 57, 80, 30, 85, 21, 13, 91, 93, 73, 8, 55, 82, 90, 52, 100, 41, 27, 1, 44, 7, 95, 96, 103, 36, 76, 94, 59, 99, 24, 119, 121, 14, 51, 50, 116, 35, 118, 33, 47, 60, 106, 101, 110, 46, 122, 2, 98, 123, 45, 62, 40, 109, 4, 42, 71, 37, 126, 104, 54, 124, 68, 61, 43, 114, 120, 125, 107, 108, 127, 111], [102, 56, 113, 97, 29, 49, 23, 20, 79, 77, 75, 81, 10, 3, 6, 72, 22, 69, 0, 30, 18, 65, 88, 25, 53, 84, 74, 13, 70, 17, 87, 86, 59, 26, 93, 71, 1, 89, 64, 66, 15, 76, 52, 80, 92, 11, 119, 73, 24, 4, 95, 96, 8, 31, 34, 90, 91, 14, 33, 19, 32, 28, 38, 62, 58, 82, 9, 41, 36, 112, 94, 110, 12, 83, 78, 16, 5, 121, 27, 57, 21, 103, 7, 48, 115, 50, 63, 117, 85, 35, 99, 2, 101, 43, 98, 39, 100, 67, 125, 60, 68, 45, 123, 122, 105, 51, 126, 44, 54, 127, 47, 61, 37, 124, 114, 106, 55, 108, 111, 40, 120, 46, 42, 107, 109, 104, 118, 116], [113, 102, 56, 121, 48, 49, 97, 39, 29, 22, 44, 126, 16, 115, 52, 50, 60, 59, 124, 118, 51, 117, 123, 38, 57, 119, 63, 54, 47, 112, 28, 114, 127, 58, 46, 122, 90, 62, 120, 110, 53, 125, 43, 55, 40, 109, 104, 61, 116, 108, 24, 41, 37, 111, 105, 103, 107, 32, 101, 106, 42, 45, 91, 23, 20, 26, 36, 100, 86, 25, 34, 98, 81, 99, 88, 80, 96, 93, 31, 94, 35, 85, 14, 30, 27, 95, 92, 83, 76, 33, 18, 78, 82, 9, 89, 19, 21, 12, 73, 77, 75, 69, 84, 10, 71, 79, 7, 4, 2, 68, 87, 17, 66, 5, 72, 64, 65, 0, 3, 1, 6, 15, 11, 13, 67, 8, 70, 74], [56, 102, 113, 48, 97, 49, 29, 23, 41, 22, 117, 52, 28, 115, 16, 46, 36, 20, 114, 103, 81, 86, 121, 44, 31, 50, 53, 63, 105, 58, 25, 77, 39, 119, 51, 106, 126, 91, 54, 90, 76, 78, 27, 60, 89, 55, 59, 123, 62, 92, 79, 118, 10, 38, 107, 104, 42, 120, 122, 109, 108, 124, 72, 69, 112, 14, 12, 127, 47, 116, 57, 110, 33, 43, 19, 111, 93, 32, 83, 24, 40, 101, 45, 125, 30, 61, 80, 75, 37, 35, 99, 82, 84, 26, 98, 88, 18, 34, 85, 96, 100, 9, 94, 21, 3, 64, 95, 66, 68, 71, 5, 7, 74, 1, 0, 17, 73, 6, 4, 65, 2, 87, 13, 70, 15, 67, 11, 8]], "model.layers.20.self_attn.k_proj": [[126, 114, 46, 37, 86, 34, 50, 64, 124, 28, 12, 16, 25, 51, 19, 66, 119, 57, 120, 7, 60, 58, 115, 82, 53, 127, 123, 62, 116, 61, 74, 122, 14, 113, 112, 9, 117, 63, 56, 125, 54, 121, 55, 47, 118, 48, 44, 111, 43, 79, 59, 49, 13, 45, 109, 29, 39, 101, 95, 6, 52, 107, 24, 3, 41, 36, 42, 108, 102, 104, 87, 105, 106, 40, 94, 103, 21, 38, 98, 4, 33, 90, 27, 32, 23, 8, 35, 72, 96, 5, 1, 99, 15, 100, 91, 80, 30, 110, 77, 97, 89, 84, 20, 93, 70, 88, 17, 2, 31, 11, 18, 65, 68, 26, 81, 75, 92, 69, 76, 85, 78, 73, 67, 0, 10, 83, 71, 22], [107, 34, 43, 29, 116, 51, 90, 78, 81, 8, 127, 19, 31, 70, 12, 56, 0, 110, 7, 120, 87, 32, 68, 61, 63, 3, 74, 62, 75, 54, 2, 39, 121, 88, 21, 66, 100, 24, 92, 73, 85, 58, 46, 95, 28, 86, 109, 45, 52, 59, 69, 77, 126, 117, 37, 1, 48, 113, 57, 36, 33, 47, 42, 114, 53, 27, 64, 4, 11, 84, 99, 18, 15, 5, 50, 119, 108, 97, 112, 65, 82, 80, 124, 44, 125, 16, 30, 71, 102, 22, 25, 41, 122, 40, 94, 60, 118, 105, 55, 101, 104, 10, 79, 13, 106, 38, 89, 115, 20, 26, 49, 9, 123, 111, 23, 35, 91, 72, 76, 96, 17, 14, 67, 93, 83, 6, 103, 98], [45, 103, 46, 33, 29, 109, 86, 19, 85, 123, 90, 119, 5, 10, 16, 64, 78, 57, 116, 76, 71, 52, 17, 112, 54, 126, 127, 121, 65, 3, 53, 115, 125, 51, 122, 107, 55, 48, 113, 58, 56, 114, 118, 59, 120, 60, 24, 124, 9, 2, 47, 44, 61, 63, 34, 111, 89, 62, 108, 106, 67, 13, 41, 11, 49, 32, 117, 100, 43, 99, 50, 72, 35, 38, 101, 104, 6, 31, 69, 105, 23, 42, 37, 15, 18, 88, 30, 95, 82, 87, 36, 91, 94, 102, 25, 84, 92, 75, 0, 40, 110, 27, 93, 98, 14, 1, 77, 96, 28, 20, 8, 4, 73, 79, 66, 81, 26, 83, 68, 22, 7, 70, 74, 39, 12, 97, 80, 21], [51, 37, 119, 120, 127, 33, 22, 46, 122, 123, 121, 50, 118, 56, 45, 124, 113, 52, 126, 61, 116, 53, 62, 117, 48, 125, 106, 115, 54, 55, 60, 57, 30, 59, 58, 63, 40, 89, 91, 49, 108, 114, 101, 47, 110, 104, 111, 112, 41, 109, 80, 44, 102, 103, 43, 42, 107, 38, 105, 39, 16, 92, 93, 95, 23, 36, 32, 82, 35, 34, 31, 21, 84, 29, 25, 86, 99, 96, 100, 28, 77, 76, 26, 98, 87, 88, 19, 72, 12, 27, 90, 94, 24, 83, 70, 78, 15, 85, 10, 75, 97, 7, 9, 74, 81, 14, 17, 13, 68, 3, 79, 11, 5, 2, 20, 18, 73, 65, 6, 4, 71, 64, 67, 8, 69, 0, 1, 66], [46, 50, 110, 101, 26, 30, 58, 20, 114, 85, 78, 33, 17, 4, 66, 88, 11, 76, 79, 41, 62, 7, 72, 126, 64, 75, 5, 53, 122, 70, 6, 3, 19, 34, 87, 73, 18, 23, 91, 86, 15, 68, 83, 16, 32, 77, 60, 84, 81, 49, 43, 10, 65, 117, 100, 118, 105, 127, 24, 112, 109, 31, 51, 92, 45, 67, 61, 120, 48, 35, 57, 63, 56, 29, 40, 107, 96, 25, 89, 22, 27, 113, 119, 125, 82, 52, 80, 124, 55, 116, 21, 93, 0, 1, 115, 121, 103, 108, 44, 54, 106, 99, 28, 74, 38, 123, 95, 13, 104, 39, 47, 59, 98, 36, 102, 111, 42, 8, 97, 14, 9, 37, 12, 2, 90, 94, 71, 69], [112, 48, 36, 86, 15, 17, 125, 110, 83, 75, 25, 10, 27, 33, 13, 29, 85, 70, 32, 30, 8, 124, 117, 46, 56, 126, 121, 54, 84, 82, 115, 61, 28, 57, 53, 2, 116, 98, 0, 47, 4, 62, 90, 120, 59, 51, 80, 63, 102, 60, 114, 87, 123, 35, 127, 44, 113, 50, 109, 52, 122, 118, 49, 58, 92, 105, 111, 55, 41, 119, 77, 45, 11, 104, 26, 107, 42, 73, 43, 38, 6, 103, 101, 106, 37, 31, 14, 108, 40, 74, 79, 65, 99, 7, 88, 16, 34, 23, 72, 39, 12, 76, 96, 95, 67, 21, 93, 78, 3, 24, 20, 89, 91, 9, 69, 94, 97, 100, 19, 18, 68, 71, 81, 5, 66, 22, 1, 64], [122, 116, 39, 22, 55, 97, 54, 61, 126, 118, 52, 117, 112, 124, 57, 121, 50, 123, 120, 115, 62, 113, 127, 27, 58, 56, 114, 53, 59, 63, 51, 95, 119, 98, 60, 26, 34, 49, 125, 111, 47, 110, 104, 45, 42, 44, 46, 82, 109, 48, 24, 107, 108, 80, 32, 38, 35, 41, 43, 84, 102, 37, 106, 16, 105, 101, 12, 36, 96, 40, 14, 99, 85, 92, 23, 15, 29, 9, 33, 94, 90, 17, 77, 19, 30, 31, 93, 100, 28, 10, 71, 75, 103, 81, 87, 25, 91, 89, 20, 79, 78, 76, 70, 67, 83, 18, 11, 8, 72, 1, 88, 13, 69, 86, 74, 5, 3, 21, 73, 0, 6, 66, 7, 4, 64, 2, 68, 65], [113, 56, 38, 33, 22, 93, 81, 23, 79, 72, 77, 20, 6, 10, 75, 112, 0, 3, 120, 69, 117, 27, 57, 65, 16, 49, 52, 115, 103, 54, 59, 110, 50, 126, 46, 122, 51, 28, 53, 60, 124, 26, 58, 55, 61, 119, 125, 102, 118, 105, 62, 123, 5, 121, 48, 95, 116, 2, 94, 47, 63, 45, 127, 83, 108, 30, 114, 99, 109, 106, 9, 107, 66, 100, 89, 67, 32, 104, 44, 43, 35, 21, 14, 76, 40, 42, 18, 96, 78, 85, 88, 24, 101, 39, 111, 34, 73, 68, 41, 25, 4, 36, 31, 37, 29, 90, 98, 12, 92, 1, 80, 19, 7, 71, 74, 82, 64, 91, 87, 84, 8, 17, 70, 11, 13, 15, 86, 97]], "model.layers.20.self_attn.qk_proj": [[46, 113, 56, 116, 122, 112, 126, 48, 110, 51, 114, 45, 107, 43, 119, 120, 50, 109, 127, 37, 93, 125, 121, 54, 49, 22, 52, 123, 86, 101, 58, 39, 102, 118, 55, 29, 115, 53, 57, 90, 63, 124, 60, 62, 103, 26, 33, 92, 61, 97, 59, 17, 81, 34, 23, 83, 38, 19, 84, 89, 47, 117, 91, 87, 85, 24, 14, 111, 44, 32, 108, 41, 78, 98, 21, 16, 74, 30, 80, 88, 25, 20, 105, 36, 79, 100, 75, 77, 10, 15, 106, 95, 11, 12, 13, 28, 104, 27, 76, 31, 94, 72, 6, 82, 40, 18, 8, 96, 35, 7, 42, 67, 71, 3, 66, 9, 64, 0, 99, 73, 5, 2, 70, 4, 65, 1, 69, 68], [46, 113, 56, 116, 126, 112, 122, 48, 110, 45, 51, 114, 120, 107, 119, 43, 50, 109, 127, 93, 37, 125, 121, 101, 49, 52, 58, 123, 86, 22, 54, 102, 39, 29, 53, 103, 115, 55, 60, 97, 61, 90, 118, 117, 63, 34, 57, 59, 33, 124, 26, 62, 38, 81, 92, 23, 85, 87, 83, 24, 47, 89, 19, 17, 30, 98, 91, 84, 14, 41, 78, 16, 88, 32, 111, 21, 80, 36, 75, 95, 20, 76, 100, 44, 25, 108, 10, 77, 15, 12, 11, 74, 104, 79, 105, 106, 13, 42, 28, 94, 96, 6, 27, 72, 82, 31, 40, 9, 8, 64, 18, 35, 3, 71, 66, 2, 7, 0, 99, 70, 5, 68, 67, 1, 73, 69, 65, 4], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 114, 45, 119, 107, 43, 120, 50, 109, 127, 93, 125, 37, 121, 22, 86, 101, 49, 54, 52, 58, 55, 39, 29, 63, 102, 123, 103, 115, 33, 118, 60, 90, 53, 124, 61, 97, 62, 92, 26, 34, 57, 59, 30, 81, 17, 23, 24, 83, 38, 117, 47, 14, 85, 87, 44, 88, 19, 21, 41, 84, 111, 108, 98, 91, 32, 95, 78, 100, 89, 80, 25, 105, 16, 28, 36, 75, 104, 74, 79, 31, 20, 15, 77, 106, 12, 27, 76, 13, 94, 10, 82, 72, 18, 11, 64, 35, 96, 67, 0, 42, 8, 6, 9, 40, 70, 99, 3, 66, 7, 69, 2, 5, 71, 68, 73, 4, 65, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 22, 101, 121, 54, 58, 55, 53, 39, 102, 115, 86, 63, 29, 60, 103, 52, 61, 97, 90, 57, 118, 62, 34, 124, 33, 117, 26, 59, 92, 47, 17, 81, 123, 19, 14, 87, 30, 83, 38, 75, 23, 24, 78, 88, 16, 80, 85, 95, 20, 98, 106, 89, 91, 84, 21, 44, 111, 31, 79, 15, 105, 25, 12, 100, 32, 28, 74, 42, 41, 104, 13, 27, 76, 36, 94, 10, 11, 77, 82, 96, 108, 67, 72, 40, 18, 3, 8, 2, 70, 7, 66, 64, 0, 5, 71, 6, 35, 73, 69, 9, 99, 68, 4, 65, 1], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 127, 125, 93, 49, 37, 54, 121, 22, 86, 39, 115, 58, 101, 63, 123, 55, 29, 61, 103, 53, 102, 57, 52, 60, 59, 62, 26, 124, 97, 81, 90, 118, 34, 33, 47, 117, 19, 17, 98, 38, 23, 87, 78, 83, 84, 24, 92, 30, 85, 20, 80, 21, 75, 16, 111, 89, 79, 91, 25, 95, 14, 88, 105, 74, 44, 15, 12, 104, 40, 36, 10, 108, 31, 28, 41, 77, 100, 106, 76, 32, 27, 13, 11, 82, 42, 72, 70, 73, 18, 94, 71, 8, 7, 35, 0, 3, 96, 66, 64, 99, 67, 9, 69, 2, 6, 5, 68, 65, 4, 1], [46, 113, 116, 56, 122, 126, 112, 48, 110, 51, 45, 114, 119, 120, 107, 43, 50, 109, 127, 93, 125, 49, 37, 22, 86, 121, 58, 54, 101, 39, 60, 115, 29, 123, 102, 52, 97, 90, 63, 124, 53, 117, 26, 57, 81, 103, 118, 34, 19, 59, 55, 61, 84, 16, 87, 80, 85, 62, 33, 17, 23, 47, 79, 78, 98, 38, 92, 24, 89, 14, 20, 111, 30, 12, 83, 44, 75, 31, 88, 95, 104, 91, 15, 25, 21, 10, 74, 76, 32, 28, 11, 77, 100, 36, 105, 82, 13, 108, 106, 41, 27, 72, 96, 8, 70, 40, 42, 94, 18, 73, 9, 3, 67, 7, 0, 64, 71, 35, 99, 66, 6, 5, 2, 69, 4, 65, 1, 68], [46, 113, 56, 116, 48, 126, 112, 110, 122, 114, 45, 51, 120, 119, 109, 107, 43, 50, 127, 125, 93, 37, 22, 86, 49, 39, 102, 101, 121, 58, 29, 123, 115, 57, 54, 60, 103, 90, 34, 118, 26, 117, 55, 52, 63, 53, 33, 61, 124, 87, 19, 97, 17, 23, 16, 84, 24, 47, 81, 62, 85, 59, 38, 92, 83, 91, 88, 30, 98, 80, 20, 79, 78, 14, 44, 21, 89, 25, 15, 76, 104, 32, 36, 75, 100, 108, 40, 10, 12, 28, 31, 95, 27, 41, 74, 77, 13, 111, 11, 42, 105, 106, 82, 18, 94, 35, 9, 96, 8, 72, 70, 99, 7, 73, 64, 71, 67, 3, 0, 6, 66, 2, 69, 5, 4, 68, 1, 65], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 114, 120, 119, 107, 43, 109, 50, 127, 125, 93, 22, 37, 86, 101, 121, 123, 102, 49, 58, 39, 29, 53, 57, 117, 54, 55, 60, 90, 62, 115, 118, 103, 124, 52, 33, 26, 87, 34, 63, 91, 92, 59, 24, 81, 23, 61, 19, 97, 83, 85, 38, 30, 17, 47, 98, 89, 21, 111, 80, 108, 41, 88, 32, 36, 84, 25, 15, 14, 20, 79, 104, 27, 16, 10, 44, 31, 78, 28, 40, 100, 95, 77, 94, 75, 12, 42, 13, 76, 18, 74, 11, 105, 106, 82, 99, 35, 70, 8, 96, 7, 72, 64, 3, 9, 0, 71, 66, 73, 69, 2, 6, 67, 5, 68, 1, 4, 65], [46, 113, 56, 126, 116, 48, 112, 122, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 93, 125, 49, 37, 121, 101, 22, 123, 58, 102, 29, 86, 55, 117, 60, 39, 26, 53, 63, 54, 61, 57, 62, 103, 90, 52, 118, 115, 34, 33, 124, 92, 17, 81, 38, 97, 47, 85, 87, 83, 23, 78, 59, 24, 30, 21, 91, 36, 88, 32, 84, 19, 89, 14, 16, 20, 104, 41, 111, 15, 98, 25, 44, 80, 79, 31, 100, 95, 27, 28, 11, 10, 105, 77, 94, 76, 12, 40, 75, 108, 13, 106, 74, 18, 96, 42, 8, 35, 7, 82, 99, 72, 3, 70, 0, 2, 73, 6, 67, 64, 9, 71, 69, 66, 68, 1, 4, 5, 65], [46, 113, 116, 56, 126, 48, 112, 122, 110, 51, 45, 114, 120, 107, 43, 119, 50, 109, 127, 93, 37, 125, 121, 49, 58, 22, 54, 52, 123, 101, 117, 86, 102, 53, 55, 124, 29, 57, 115, 39, 63, 118, 61, 62, 103, 90, 34, 97, 60, 38, 26, 17, 33, 92, 81, 47, 78, 87, 85, 59, 23, 19, 83, 88, 24, 89, 25, 30, 32, 98, 10, 16, 80, 21, 41, 84, 14, 111, 11, 95, 100, 15, 36, 44, 91, 77, 79, 104, 12, 75, 28, 40, 31, 20, 27, 8, 74, 76, 106, 96, 13, 105, 42, 108, 94, 7, 6, 71, 0, 72, 3, 18, 35, 2, 64, 9, 82, 70, 73, 69, 67, 66, 99, 68, 5, 4, 1, 65], [46, 113, 56, 116, 122, 126, 48, 110, 112, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 37, 121, 49, 115, 22, 58, 101, 54, 86, 118, 123, 39, 63, 52, 124, 53, 62, 102, 97, 29, 103, 55, 26, 57, 61, 60, 90, 81, 17, 19, 117, 33, 111, 87, 84, 92, 34, 38, 83, 85, 78, 16, 15, 47, 25, 79, 24, 14, 80, 95, 30, 11, 59, 23, 21, 44, 75, 20, 88, 10, 91, 98, 12, 104, 77, 32, 28, 106, 6, 76, 8, 74, 36, 100, 82, 72, 89, 105, 27, 31, 41, 108, 13, 18, 40, 94, 42, 7, 64, 96, 2, 73, 71, 0, 3, 9, 35, 67, 69, 66, 99, 5, 70, 4, 1, 65, 68], [46, 113, 56, 116, 48, 126, 110, 122, 112, 51, 45, 114, 120, 107, 119, 43, 50, 109, 125, 127, 93, 37, 22, 86, 49, 101, 58, 102, 121, 39, 29, 115, 53, 54, 123, 63, 117, 34, 55, 52, 90, 62, 26, 103, 61, 17, 19, 124, 60, 97, 33, 118, 81, 57, 83, 38, 85, 87, 59, 84, 92, 23, 78, 47, 15, 98, 16, 80, 24, 14, 10, 30, 88, 91, 20, 100, 79, 12, 21, 44, 111, 36, 25, 32, 11, 95, 31, 89, 104, 75, 28, 41, 77, 76, 27, 74, 106, 82, 8, 108, 94, 96, 40, 18, 105, 6, 13, 72, 71, 73, 9, 42, 67, 35, 66, 3, 0, 64, 7, 99, 69, 2, 70, 5, 1, 65, 4, 68], [46, 113, 56, 116, 126, 48, 112, 110, 122, 51, 45, 120, 114, 107, 50, 109, 43, 119, 93, 127, 125, 37, 49, 22, 102, 86, 58, 123, 39, 121, 54, 101, 29, 55, 117, 115, 124, 62, 63, 52, 33, 53, 97, 60, 103, 34, 118, 90, 61, 47, 38, 57, 26, 92, 30, 19, 85, 59, 87, 81, 83, 89, 88, 98, 17, 32, 91, 23, 84, 36, 25, 44, 24, 16, 111, 80, 78, 21, 100, 20, 108, 15, 95, 11, 31, 105, 77, 28, 10, 79, 27, 12, 14, 41, 74, 82, 76, 94, 40, 13, 104, 75, 106, 96, 35, 71, 8, 42, 18, 6, 72, 0, 66, 64, 73, 7, 9, 3, 99, 2, 67, 70, 5, 69, 1, 4, 68, 65], [46, 113, 56, 122, 126, 116, 48, 112, 110, 51, 45, 114, 120, 107, 119, 50, 43, 109, 127, 125, 93, 37, 22, 49, 58, 54, 101, 121, 86, 39, 60, 29, 102, 123, 57, 52, 53, 62, 103, 115, 55, 63, 117, 90, 61, 124, 118, 34, 26, 59, 17, 81, 47, 33, 92, 97, 83, 111, 38, 87, 19, 32, 85, 78, 98, 23, 30, 80, 21, 91, 24, 89, 84, 11, 36, 20, 14, 95, 15, 44, 10, 28, 104, 25, 105, 108, 16, 79, 41, 106, 31, 88, 76, 96, 74, 12, 77, 100, 82, 75, 27, 72, 94, 42, 8, 18, 64, 0, 13, 71, 73, 6, 35, 9, 70, 7, 99, 66, 67, 40, 5, 1, 2, 3, 69, 4, 65, 68], [46, 113, 116, 56, 48, 126, 112, 122, 110, 51, 114, 45, 107, 120, 43, 119, 50, 109, 127, 93, 125, 37, 22, 86, 121, 49, 102, 101, 58, 52, 123, 39, 55, 29, 118, 54, 117, 103, 63, 61, 57, 33, 62, 90, 26, 17, 53, 81, 19, 60, 124, 59, 34, 115, 87, 97, 24, 92, 78, 85, 83, 74, 11, 91, 79, 16, 21, 10, 89, 104, 23, 80, 32, 38, 98, 20, 84, 36, 111, 95, 47, 12, 27, 30, 14, 28, 88, 76, 15, 25, 100, 75, 41, 108, 106, 77, 96, 94, 44, 18, 31, 73, 105, 40, 8, 82, 70, 35, 13, 71, 72, 0, 64, 67, 7, 2, 3, 9, 6, 66, 42, 69, 5, 68, 1, 65, 99, 4], [46, 113, 56, 116, 126, 48, 122, 110, 112, 51, 114, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 49, 22, 101, 121, 86, 54, 58, 102, 57, 123, 52, 63, 60, 117, 115, 29, 53, 55, 62, 118, 103, 39, 81, 59, 26, 124, 97, 61, 90, 83, 87, 33, 17, 34, 38, 19, 74, 47, 23, 78, 85, 32, 89, 11, 91, 24, 84, 95, 92, 98, 20, 15, 79, 104, 111, 36, 16, 30, 77, 14, 25, 10, 88, 75, 27, 12, 21, 80, 28, 40, 44, 41, 106, 105, 13, 70, 31, 100, 108, 76, 94, 8, 72, 18, 42, 96, 82, 71, 73, 67, 7, 35, 9, 3, 0, 66, 2, 64, 99, 6, 69, 5, 65, 1, 4, 68], [46, 113, 56, 116, 126, 122, 48, 112, 110, 51, 45, 120, 114, 107, 119, 43, 109, 50, 127, 125, 93, 37, 121, 49, 22, 86, 101, 102, 58, 29, 54, 103, 123, 60, 52, 39, 115, 118, 124, 117, 57, 55, 26, 34, 53, 62, 97, 90, 63, 33, 47, 92, 59, 83, 61, 81, 19, 17, 23, 24, 111, 38, 87, 30, 20, 91, 89, 85, 84, 32, 98, 16, 21, 88, 36, 78, 80, 25, 100, 40, 104, 44, 79, 74, 108, 95, 28, 14, 27, 11, 31, 15, 12, 77, 94, 41, 76, 10, 96, 82, 13, 72, 18, 70, 75, 35, 105, 106, 42, 64, 71, 9, 0, 7, 67, 2, 8, 5, 73, 99, 66, 3, 6, 68, 69, 65, 1, 4], [46, 113, 56, 116, 126, 48, 110, 112, 122, 45, 51, 114, 107, 120, 43, 119, 109, 50, 127, 93, 125, 37, 121, 22, 86, 49, 101, 117, 58, 55, 29, 102, 53, 39, 118, 54, 103, 63, 124, 33, 115, 90, 123, 92, 60, 26, 34, 57, 62, 52, 97, 17, 61, 19, 81, 24, 111, 14, 83, 91, 85, 98, 23, 30, 16, 38, 87, 36, 89, 44, 11, 21, 59, 20, 47, 88, 74, 84, 28, 94, 95, 32, 15, 76, 79, 41, 80, 108, 10, 40, 78, 27, 31, 25, 13, 12, 100, 104, 75, 77, 42, 70, 72, 18, 96, 82, 106, 0, 105, 9, 35, 64, 67, 71, 99, 5, 3, 2, 8, 69, 7, 66, 73, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 112, 48, 122, 110, 51, 45, 114, 120, 107, 43, 119, 109, 50, 127, 93, 125, 37, 49, 121, 117, 58, 63, 54, 22, 102, 39, 52, 101, 86, 62, 61, 29, 55, 53, 118, 103, 123, 115, 97, 57, 124, 33, 60, 90, 34, 26, 59, 23, 38, 83, 111, 24, 92, 17, 81, 14, 36, 30, 47, 21, 95, 87, 88, 85, 19, 20, 89, 78, 40, 16, 80, 98, 91, 44, 104, 84, 32, 28, 41, 11, 106, 76, 100, 25, 10, 74, 75, 15, 27, 31, 94, 105, 77, 42, 96, 79, 72, 108, 12, 82, 13, 70, 0, 66, 71, 35, 8, 67, 18, 6, 64, 7, 9, 4, 3, 5, 2, 99, 73, 1, 68, 69, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 45, 114, 107, 120, 43, 109, 50, 119, 127, 125, 93, 37, 121, 49, 22, 58, 101, 86, 117, 102, 52, 54, 97, 118, 63, 123, 29, 115, 39, 62, 55, 60, 124, 61, 103, 53, 34, 26, 38, 90, 57, 33, 87, 19, 95, 59, 81, 85, 92, 47, 24, 23, 83, 32, 98, 80, 11, 111, 84, 17, 16, 30, 89, 20, 78, 88, 91, 36, 14, 25, 21, 74, 104, 106, 15, 79, 100, 108, 40, 76, 28, 94, 12, 31, 10, 13, 72, 96, 44, 41, 27, 75, 77, 42, 8, 35, 6, 18, 71, 9, 0, 7, 82, 66, 73, 70, 105, 67, 64, 99, 3, 2, 4, 68, 69, 5, 1, 65], [46, 113, 116, 56, 126, 48, 110, 122, 112, 51, 114, 45, 107, 43, 120, 119, 109, 50, 127, 125, 93, 49, 37, 121, 22, 58, 86, 117, 54, 53, 123, 115, 63, 101, 124, 52, 118, 29, 39, 102, 55, 60, 26, 57, 103, 34, 90, 62, 97, 33, 81, 92, 78, 61, 87, 16, 38, 59, 83, 19, 111, 47, 17, 95, 32, 30, 91, 98, 11, 20, 84, 21, 23, 89, 88, 14, 36, 24, 25, 15, 44, 74, 28, 85, 108, 79, 76, 106, 12, 31, 94, 105, 13, 80, 104, 42, 75, 27, 41, 100, 82, 40, 77, 6, 10, 72, 8, 67, 18, 7, 35, 9, 96, 73, 99, 5, 0, 71, 3, 70, 66, 69, 2, 1, 68, 64, 4, 65], [46, 113, 116, 56, 48, 126, 112, 110, 122, 51, 114, 45, 120, 107, 43, 119, 50, 109, 127, 125, 93, 49, 37, 121, 54, 58, 117, 22, 53, 118, 39, 63, 86, 124, 29, 115, 55, 52, 101, 102, 123, 97, 103, 62, 61, 34, 57, 90, 26, 60, 33, 59, 92, 38, 17, 95, 81, 91, 83, 19, 87, 24, 47, 84, 16, 21, 88, 85, 111, 78, 108, 23, 89, 14, 30, 11, 32, 25, 44, 104, 36, 100, 28, 98, 74, 20, 80, 42, 79, 94, 41, 15, 106, 31, 76, 40, 75, 27, 82, 96, 77, 8, 64, 10, 13, 6, 105, 12, 71, 0, 18, 7, 3, 72, 9, 73, 67, 35, 2, 70, 99, 66, 5, 69, 68, 65, 1, 4], [46, 113, 116, 56, 126, 112, 122, 48, 110, 51, 114, 45, 120, 107, 119, 43, 109, 50, 125, 127, 93, 49, 121, 37, 123, 54, 22, 86, 117, 101, 58, 115, 52, 39, 102, 124, 118, 53, 29, 57, 60, 63, 26, 103, 61, 55, 34, 97, 62, 38, 90, 33, 92, 83, 47, 59, 17, 111, 88, 87, 81, 44, 19, 21, 91, 89, 24, 23, 32, 80, 36, 85, 78, 84, 98, 95, 30, 108, 14, 20, 74, 16, 94, 25, 27, 41, 31, 28, 106, 100, 79, 104, 12, 40, 15, 96, 10, 76, 11, 75, 77, 105, 18, 82, 13, 6, 35, 8, 73, 42, 7, 99, 64, 0, 72, 3, 71, 67, 66, 2, 9, 5, 69, 70, 4, 68, 65, 1], [46, 113, 116, 122, 56, 126, 48, 112, 110, 51, 114, 45, 107, 120, 119, 43, 109, 50, 127, 125, 93, 37, 49, 54, 22, 58, 86, 121, 101, 115, 117, 123, 29, 118, 39, 57, 52, 53, 124, 102, 97, 63, 26, 55, 90, 103, 60, 111, 61, 34, 59, 33, 92, 87, 47, 81, 62, 17, 19, 83, 85, 84, 23, 16, 98, 32, 36, 30, 91, 80, 78, 89, 38, 24, 95, 75, 14, 20, 88, 21, 31, 25, 44, 76, 74, 28, 79, 11, 12, 106, 15, 27, 13, 41, 10, 100, 104, 72, 96, 77, 105, 94, 108, 8, 82, 18, 42, 6, 40, 71, 9, 35, 73, 67, 7, 3, 70, 0, 64, 66, 5, 2, 69, 99, 1, 4, 68, 65], [46, 113, 116, 56, 122, 112, 48, 126, 110, 114, 51, 45, 120, 107, 43, 119, 109, 50, 125, 127, 93, 37, 22, 49, 86, 121, 115, 58, 54, 124, 63, 29, 102, 118, 53, 117, 39, 52, 123, 57, 101, 90, 55, 61, 26, 81, 34, 83, 19, 103, 97, 87, 60, 33, 59, 62, 23, 111, 24, 92, 80, 47, 84, 38, 17, 85, 89, 78, 21, 20, 88, 44, 91, 30, 16, 75, 12, 15, 25, 36, 95, 14, 74, 98, 10, 32, 31, 28, 76, 100, 104, 27, 13, 79, 41, 11, 94, 8, 77, 42, 40, 106, 96, 108, 18, 7, 9, 99, 72, 105, 35, 0, 82, 6, 71, 73, 70, 67, 5, 2, 66, 64, 3, 69, 1, 4, 68, 65], [46, 113, 116, 56, 112, 126, 48, 122, 110, 51, 45, 114, 120, 107, 119, 43, 109, 50, 127, 125, 93, 37, 124, 121, 22, 49, 58, 86, 115, 54, 118, 53, 101, 117, 102, 123, 29, 60, 39, 63, 55, 52, 61, 90, 34, 97, 62, 57, 103, 26, 33, 92, 38, 24, 83, 36, 59, 47, 111, 98, 19, 21, 32, 30, 88, 91, 17, 85, 95, 84, 87, 20, 81, 89, 23, 74, 14, 16, 104, 25, 100, 78, 75, 27, 80, 108, 44, 28, 41, 94, 79, 15, 31, 12, 96, 40, 105, 10, 11, 77, 35, 76, 8, 82, 13, 42, 72, 73, 106, 18, 70, 99, 64, 67, 71, 9, 66, 6, 7, 3, 69, 0, 5, 2, 1, 4, 65, 68], [46, 113, 122, 56, 116, 126, 112, 48, 110, 114, 45, 51, 107, 120, 119, 109, 43, 50, 127, 125, 93, 121, 123, 115, 49, 37, 101, 54, 58, 102, 118, 22, 124, 117, 86, 55, 53, 39, 61, 57, 103, 29, 60, 63, 52, 97, 90, 34, 62, 26, 92, 47, 38, 111, 33, 19, 81, 83, 17, 23, 44, 98, 59, 36, 87, 30, 95, 84, 32, 21, 85, 24, 91, 14, 108, 100, 78, 88, 89, 25, 75, 28, 94, 41, 79, 20, 80, 104, 10, 31, 16, 76, 74, 105, 15, 77, 13, 96, 40, 12, 27, 18, 106, 70, 82, 11, 42, 99, 8, 35, 2, 3, 9, 64, 72, 71, 69, 7, 73, 67, 5, 0, 66, 6, 65, 4, 68, 1], [46, 113, 116, 56, 126, 122, 112, 48, 110, 51, 114, 45, 107, 50, 120, 43, 119, 109, 127, 93, 125, 37, 49, 121, 58, 22, 55, 54, 118, 101, 115, 86, 124, 102, 39, 123, 29, 63, 53, 117, 57, 97, 90, 61, 52, 103, 62, 33, 59, 26, 60, 92, 19, 81, 17, 87, 34, 83, 47, 23, 85, 14, 75, 106, 16, 20, 38, 78, 88, 21, 111, 25, 84, 30, 24, 80, 95, 32, 91, 89, 44, 74, 10, 15, 8, 100, 31, 76, 12, 79, 11, 70, 41, 13, 98, 77, 94, 28, 104, 40, 96, 36, 27, 105, 108, 7, 42, 82, 72, 35, 9, 67, 0, 18, 64, 71, 69, 66, 73, 6, 99, 2, 3, 68, 5, 1, 4, 65], [46, 113, 116, 56, 48, 126, 122, 112, 110, 51, 114, 45, 119, 107, 43, 120, 109, 50, 127, 37, 93, 125, 49, 121, 54, 22, 123, 86, 118, 102, 57, 39, 58, 52, 55, 101, 62, 115, 53, 29, 124, 103, 34, 117, 63, 60, 26, 97, 33, 90, 61, 47, 59, 81, 78, 19, 17, 87, 95, 38, 85, 83, 84, 100, 75, 111, 21, 92, 89, 91, 16, 23, 14, 98, 32, 74, 44, 80, 104, 24, 30, 10, 41, 12, 79, 20, 88, 15, 36, 105, 31, 76, 40, 108, 25, 27, 28, 77, 64, 8, 11, 70, 3, 72, 96, 82, 35, 106, 18, 94, 67, 13, 2, 9, 0, 7, 71, 42, 5, 66, 73, 69, 6, 4, 1, 68, 99, 65], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 114, 45, 119, 120, 107, 43, 50, 109, 127, 125, 93, 37, 121, 54, 49, 22, 118, 86, 101, 123, 63, 57, 102, 58, 117, 29, 53, 124, 52, 59, 103, 26, 115, 39, 97, 61, 62, 34, 60, 90, 33, 47, 55, 81, 92, 111, 87, 19, 23, 16, 17, 83, 38, 14, 80, 30, 84, 78, 24, 21, 44, 91, 98, 75, 15, 20, 88, 89, 12, 10, 85, 104, 25, 95, 79, 36, 32, 100, 31, 74, 40, 105, 13, 28, 76, 41, 77, 11, 27, 8, 106, 96, 108, 72, 18, 9, 94, 70, 82, 6, 71, 42, 3, 7, 67, 35, 66, 73, 64, 5, 0, 99, 69, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 48, 112, 126, 110, 51, 45, 114, 107, 119, 120, 43, 50, 109, 127, 125, 93, 54, 37, 49, 121, 86, 22, 58, 101, 52, 102, 124, 118, 123, 63, 39, 57, 115, 53, 60, 29, 117, 103, 55, 90, 62, 97, 61, 34, 47, 33, 59, 26, 92, 83, 111, 17, 81, 91, 87, 44, 23, 38, 19, 98, 14, 32, 75, 85, 78, 88, 80, 24, 104, 21, 30, 89, 10, 100, 36, 16, 25, 79, 72, 74, 12, 20, 76, 95, 84, 28, 41, 15, 31, 40, 13, 108, 11, 105, 77, 35, 96, 71, 27, 8, 18, 94, 73, 6, 64, 82, 106, 66, 7, 70, 67, 0, 42, 9, 69, 99, 5, 3, 2, 68, 65, 1, 4], [46, 113, 116, 56, 122, 126, 48, 112, 110, 51, 45, 114, 120, 119, 107, 50, 43, 109, 93, 127, 125, 37, 22, 121, 54, 86, 101, 49, 52, 102, 58, 124, 39, 118, 123, 53, 29, 60, 103, 57, 90, 55, 61, 97, 34, 115, 63, 62, 117, 33, 92, 47, 26, 81, 83, 38, 59, 85, 111, 24, 19, 91, 98, 23, 87, 17, 89, 88, 14, 30, 78, 32, 21, 12, 84, 75, 44, 20, 95, 36, 16, 100, 80, 10, 15, 79, 25, 41, 76, 104, 27, 74, 77, 105, 13, 31, 72, 94, 40, 28, 18, 106, 108, 11, 96, 42, 6, 8, 82, 70, 9, 7, 35, 73, 71, 3, 64, 5, 99, 0, 69, 2, 66, 65, 67, 4, 68, 1]], "model.layers.21.self_attn.q_proj": [[103, 49, 112, 48, 97, 82, 15, 84, 29, 86, 12, 113, 99, 62, 8, 26, 54, 78, 13, 57, 4, 70, 96, 52, 90, 33, 17, 72, 23, 37, 80, 56, 89, 79, 77, 88, 73, 120, 98, 93, 24, 21, 20, 92, 18, 9, 27, 22, 59, 87, 83, 16, 5, 76, 94, 39, 19, 107, 85, 123, 14, 1, 124, 91, 46, 50, 28, 67, 31, 81, 108, 66, 25, 10, 6, 40, 51, 74, 95, 30, 122, 106, 102, 2, 116, 127, 44, 68, 115, 38, 60, 7, 125, 100, 75, 119, 109, 55, 63, 58, 34, 101, 53, 118, 61, 36, 121, 69, 32, 64, 117, 114, 47, 104, 35, 11, 126, 41, 110, 43, 45, 42, 71, 65, 111, 105, 3, 0], [49, 103, 112, 48, 97, 29, 113, 62, 56, 84, 87, 35, 38, 26, 123, 52, 82, 63, 95, 115, 120, 59, 57, 51, 124, 61, 116, 104, 86, 89, 53, 106, 119, 78, 45, 12, 122, 125, 60, 118, 126, 47, 40, 54, 117, 127, 50, 46, 88, 110, 108, 111, 55, 96, 30, 58, 114, 43, 37, 105, 34, 107, 44, 121, 102, 42, 15, 11, 19, 41, 109, 8, 100, 16, 99, 36, 92, 101, 31, 94, 32, 98, 39, 67, 23, 14, 25, 33, 85, 91, 3, 28, 17, 73, 70, 10, 4, 13, 18, 7, 22, 27, 90, 93, 24, 21, 68, 80, 71, 66, 5, 20, 83, 75, 76, 77, 9, 79, 69, 2, 65, 72, 81, 64, 6, 1, 74, 0], [112, 62, 49, 103, 48, 56, 52, 123, 122, 59, 120, 113, 50, 97, 115, 121, 51, 57, 54, 109, 46, 119, 63, 125, 35, 114, 118, 60, 105, 104, 117, 58, 108, 124, 29, 61, 127, 126, 42, 41, 110, 116, 45, 47, 88, 55, 53, 99, 87, 106, 43, 111, 107, 74, 3, 71, 37, 44, 82, 40, 38, 2, 102, 68, 6, 100, 95, 0, 84, 86, 101, 36, 30, 85, 34, 5, 26, 31, 98, 32, 14, 33, 27, 72, 94, 92, 96, 65, 89, 81, 9, 21, 90, 76, 18, 39, 91, 7, 28, 1, 19, 23, 83, 22, 11, 12, 25, 75, 93, 24, 67, 15, 17, 80, 16, 4, 73, 70, 77, 13, 20, 79, 10, 69, 78, 64, 8, 66], [112, 103, 49, 48, 97, 113, 29, 0, 65, 50, 26, 11, 95, 84, 78, 17, 54, 82, 31, 86, 62, 5, 56, 57, 123, 2, 66, 122, 88, 114, 70, 52, 42, 46, 118, 120, 59, 121, 4, 64, 8, 104, 115, 67, 125, 63, 116, 60, 108, 1, 106, 44, 35, 10, 127, 38, 119, 15, 75, 51, 12, 109, 13, 58, 117, 61, 93, 7, 3, 25, 124, 69, 126, 90, 110, 111, 45, 53, 47, 81, 99, 43, 34, 55, 96, 9, 94, 107, 37, 41, 6, 105, 100, 83, 72, 98, 74, 71, 102, 77, 85, 73, 22, 33, 40, 24, 92, 19, 87, 101, 32, 30, 89, 27, 68, 91, 14, 28, 23, 16, 20, 21, 36, 80, 39, 18, 76, 79], [102, 120, 118, 54, 53, 84, 56, 11, 15, 9, 18, 1, 67, 76, 6, 5, 87, 31, 71, 64, 66, 0, 93, 68, 77, 78, 89, 72, 13, 80, 25, 65, 86, 74, 46, 21, 109, 63, 48, 92, 69, 114, 52, 22, 32, 8, 127, 40, 126, 75, 16, 7, 10, 117, 107, 70, 115, 82, 4, 73, 57, 12, 116, 121, 113, 3, 94, 60, 38, 14, 24, 23, 79, 41, 45, 44, 110, 20, 91, 90, 98, 2, 85, 125, 103, 36, 124, 95, 17, 59, 30, 100, 47, 119, 50, 101, 62, 105, 29, 43, 34, 19, 42, 61, 99, 35, 33, 112, 108, 123, 97, 27, 51, 28, 39, 49, 96, 122, 106, 26, 37, 81, 104, 55, 83, 111, 88, 58], [102, 54, 118, 120, 84, 15, 18, 56, 72, 76, 53, 11, 25, 6, 93, 9, 78, 5, 80, 71, 31, 67, 89, 66, 74, 16, 22, 64, 1, 69, 3, 77, 46, 87, 48, 29, 92, 24, 85, 8, 117, 10, 13, 90, 65, 124, 40, 110, 75, 20, 107, 23, 109, 57, 12, 70, 83, 115, 125, 114, 79, 41, 103, 0, 42, 113, 99, 82, 63, 45, 86, 36, 14, 127, 73, 68, 32, 19, 44, 37, 39, 47, 116, 26, 61, 55, 88, 121, 51, 105, 7, 30, 81, 96, 112, 59, 123, 33, 95, 2, 38, 60, 49, 119, 4, 52, 126, 91, 17, 100, 34, 122, 97, 35, 50, 27, 104, 21, 111, 62, 98, 101, 43, 108, 28, 94, 106, 58], [102, 120, 54, 118, 56, 80, 18, 84, 76, 25, 9, 78, 93, 11, 89, 87, 31, 53, 15, 6, 71, 72, 5, 67, 77, 66, 13, 1, 64, 0, 22, 86, 74, 113, 46, 68, 65, 116, 115, 90, 32, 63, 26, 4, 81, 16, 7, 107, 8, 52, 82, 69, 40, 17, 125, 117, 73, 48, 92, 99, 36, 23, 70, 75, 12, 98, 43, 88, 51, 14, 126, 124, 19, 29, 55, 79, 37, 39, 20, 61, 114, 85, 3, 10, 122, 112, 49, 57, 109, 38, 127, 59, 30, 2, 27, 119, 24, 21, 33, 121, 35, 104, 83, 60, 100, 103, 105, 45, 34, 91, 42, 96, 94, 110, 28, 108, 62, 106, 41, 111, 101, 123, 50, 58, 47, 95, 97, 44], [102, 120, 54, 118, 56, 78, 53, 84, 72, 31, 18, 5, 15, 1, 93, 76, 71, 67, 66, 11, 9, 89, 25, 87, 13, 64, 65, 0, 117, 6, 77, 46, 68, 80, 73, 70, 107, 2, 26, 75, 22, 88, 8, 3, 39, 113, 32, 48, 69, 12, 4, 63, 82, 85, 60, 19, 109, 127, 74, 16, 116, 90, 125, 27, 28, 52, 79, 14, 110, 86, 29, 41, 126, 98, 114, 81, 124, 7, 34, 55, 92, 62, 105, 91, 96, 44, 101, 51, 115, 57, 40, 17, 42, 20, 104, 111, 36, 61, 119, 24, 94, 83, 43, 45, 50, 58, 121, 33, 38, 122, 59, 108, 100, 23, 10, 99, 106, 112, 103, 97, 95, 123, 30, 47, 37, 21, 35, 49], [59, 106, 60, 36, 99, 62, 28, 119, 42, 87, 123, 55, 61, 85, 121, 18, 53, 127, 31, 114, 32, 98, 89, 122, 124, 86, 47, 56, 92, 54, 117, 104, 112, 111, 126, 116, 125, 58, 115, 13, 118, 30, 41, 57, 113, 25, 90, 107, 100, 49, 110, 97, 109, 120, 46, 43, 50, 39, 38, 51, 52, 105, 45, 44, 101, 103, 48, 91, 63, 108, 35, 93, 102, 21, 80, 94, 29, 37, 34, 33, 40, 14, 75, 19, 79, 20, 96, 27, 4, 26, 15, 65, 22, 74, 23, 95, 82, 7, 17, 71, 83, 24, 6, 88, 77, 76, 73, 9, 68, 12, 8, 1, 84, 64, 81, 2, 69, 67, 3, 66, 16, 0, 5, 11, 10, 78, 70, 72], [106, 99, 59, 80, 28, 13, 20, 74, 8, 14, 86, 6, 60, 42, 75, 4, 18, 65, 58, 55, 30, 32, 85, 89, 64, 2, 67, 114, 31, 111, 27, 24, 91, 10, 119, 25, 62, 92, 21, 95, 22, 90, 19, 35, 107, 84, 81, 72, 82, 36, 96, 123, 16, 7, 70, 102, 112, 97, 78, 125, 17, 101, 57, 77, 79, 11, 88, 29, 87, 56, 12, 23, 69, 9, 5, 68, 73, 109, 46, 53, 113, 34, 71, 93, 0, 83, 61, 15, 127, 122, 94, 51, 45, 54, 37, 26, 66, 41, 116, 76, 117, 3, 1, 108, 52, 121, 104, 98, 115, 33, 43, 100, 120, 39, 110, 118, 105, 38, 103, 40, 124, 49, 47, 48, 50, 63, 44, 126], [106, 99, 59, 20, 14, 80, 86, 75, 74, 32, 6, 67, 8, 28, 64, 92, 60, 42, 18, 111, 25, 58, 55, 4, 2, 3, 91, 114, 69, 29, 87, 97, 65, 123, 102, 66, 127, 119, 57, 30, 1, 62, 107, 90, 36, 95, 31, 7, 88, 113, 112, 35, 12, 22, 70, 10, 122, 96, 21, 68, 89, 24, 73, 82, 11, 109, 17, 56, 125, 13, 45, 16, 72, 100, 61, 77, 78, 53, 79, 27, 15, 5, 116, 81, 19, 9, 71, 34, 84, 104, 0, 26, 108, 51, 23, 101, 85, 94, 54, 76, 98, 83, 121, 43, 48, 37, 52, 39, 115, 93, 41, 33, 46, 105, 103, 110, 38, 117, 120, 47, 118, 40, 50, 44, 49, 124, 126, 63], [106, 60, 36, 62, 42, 119, 58, 123, 55, 112, 61, 114, 121, 59, 87, 124, 111, 53, 125, 41, 115, 47, 110, 118, 63, 103, 122, 101, 127, 113, 85, 99, 18, 126, 54, 104, 56, 50, 120, 49, 107, 25, 51, 46, 108, 52, 57, 48, 28, 116, 44, 109, 43, 45, 100, 117, 105, 40, 32, 92, 89, 39, 35, 38, 102, 90, 27, 31, 37, 15, 34, 98, 33, 91, 88, 19, 30, 97, 86, 26, 93, 96, 95, 21, 94, 29, 13, 83, 22, 24, 17, 23, 76, 79, 82, 20, 81, 77, 71, 14, 80, 73, 12, 7, 9, 84, 65, 75, 4, 74, 68, 78, 1, 16, 6, 69, 11, 67, 2, 5, 8, 64, 10, 70, 3, 0, 66, 72], [102, 57, 58, 59, 114, 90, 87, 127, 26, 96, 60, 29, 38, 93, 85, 74, 82, 15, 61, 18, 110, 21, 46, 35, 48, 71, 111, 121, 62, 12, 44, 51, 84, 113, 32, 22, 49, 117, 79, 16, 50, 112, 13, 41, 119, 109, 86, 63, 43, 124, 72, 100, 4, 23, 123, 53, 52, 105, 115, 56, 118, 122, 107, 40, 106, 54, 120, 116, 98, 45, 126, 125, 55, 2, 39, 108, 33, 104, 101, 17, 94, 27, 37, 30, 103, 47, 31, 95, 20, 42, 34, 5, 91, 25, 88, 8, 28, 97, 0, 92, 89, 36, 99, 11, 83, 81, 3, 80, 19, 24, 78, 68, 1, 75, 9, 77, 14, 70, 66, 73, 65, 76, 7, 6, 64, 10, 69, 67], [102, 58, 57, 59, 114, 90, 87, 127, 121, 96, 26, 29, 38, 15, 93, 85, 49, 74, 41, 110, 82, 32, 60, 84, 63, 43, 109, 113, 50, 116, 117, 79, 62, 21, 124, 35, 52, 54, 105, 22, 48, 53, 46, 56, 13, 55, 18, 20, 125, 44, 71, 51, 12, 86, 126, 123, 61, 119, 111, 112, 108, 118, 122, 115, 120, 103, 45, 95, 23, 104, 106, 36, 47, 101, 42, 39, 31, 16, 100, 107, 33, 40, 89, 80, 28, 98, 37, 78, 2, 97, 88, 94, 34, 30, 4, 91, 99, 27, 25, 0, 72, 92, 77, 24, 83, 11, 19, 17, 73, 5, 81, 14, 68, 3, 75, 64, 70, 1, 8, 7, 66, 6, 9, 76, 10, 65, 67, 69], [102, 59, 58, 114, 57, 90, 87, 26, 15, 29, 85, 96, 74, 93, 121, 71, 18, 82, 21, 38, 109, 13, 79, 47, 44, 84, 48, 2, 116, 117, 12, 115, 49, 32, 86, 60, 54, 0, 4, 119, 55, 127, 105, 124, 35, 45, 112, 5, 43, 46, 113, 110, 17, 118, 63, 53, 52, 123, 41, 111, 37, 72, 122, 65, 61, 23, 120, 51, 50, 66, 25, 108, 1, 126, 62, 81, 56, 106, 89, 3, 22, 20, 104, 125, 16, 100, 24, 101, 36, 107, 83, 98, 8, 91, 28, 40, 94, 67, 19, 103, 31, 39, 42, 7, 68, 77, 10, 99, 95, 64, 27, 33, 75, 6, 88, 92, 73, 78, 30, 9, 34, 70, 80, 97, 11, 76, 69, 14], [102, 59, 58, 57, 127, 114, 90, 60, 26, 87, 96, 15, 85, 71, 38, 29, 74, 121, 18, 93, 105, 35, 12, 84, 49, 108, 82, 79, 41, 47, 4, 120, 110, 32, 22, 54, 21, 52, 119, 43, 118, 46, 122, 117, 124, 62, 20, 123, 116, 109, 16, 111, 104, 44, 55, 99, 112, 113, 13, 115, 61, 86, 63, 89, 40, 5, 2, 48, 107, 50, 45, 30, 103, 23, 98, 51, 0, 39, 106, 94, 125, 36, 126, 37, 33, 56, 34, 53, 42, 101, 100, 24, 27, 97, 8, 83, 17, 66, 28, 73, 31, 25, 95, 68, 88, 92, 19, 91, 1, 78, 3, 77, 72, 81, 7, 65, 64, 11, 70, 80, 9, 76, 14, 75, 10, 69, 6, 67], [126, 104, 99, 87, 28, 32, 83, 92, 85, 81, 127, 110, 79, 47, 42, 76, 115, 111, 119, 72, 11, 15, 30, 74, 51, 113, 31, 108, 21, 95, 116, 78, 37, 50, 24, 57, 122, 45, 6, 48, 25, 12, 90, 96, 5, 121, 56, 68, 26, 53, 23, 107, 55, 106, 120, 62, 59, 101, 70, 123, 89, 17, 49, 43, 46, 109, 105, 41, 97, 13, 19, 2, 33, 112, 93, 39, 52, 124, 82, 40, 65, 114, 117, 34, 0, 125, 102, 61, 29, 118, 3, 44, 80, 60, 20, 84, 58, 54, 100, 88, 27, 22, 94, 36, 98, 103, 64, 18, 86, 91, 75, 63, 38, 66, 16, 10, 67, 14, 9, 77, 1, 73, 4, 71, 35, 7, 69, 8], [126, 104, 99, 87, 28, 83, 85, 32, 127, 92, 47, 115, 81, 79, 119, 51, 106, 76, 50, 118, 72, 48, 74, 111, 113, 42, 108, 15, 116, 45, 120, 31, 122, 121, 6, 58, 5, 61, 110, 57, 11, 41, 12, 70, 68, 37, 53, 91, 54, 21, 75, 43, 9, 97, 89, 49, 23, 38, 46, 30, 105, 13, 55, 59, 56, 40, 35, 65, 0, 62, 123, 17, 117, 109, 107, 34, 78, 33, 114, 60, 95, 103, 98, 112, 25, 67, 52, 39, 3, 93, 101, 36, 82, 19, 63, 102, 125, 18, 44, 26, 90, 29, 94, 2, 24, 66, 124, 20, 96, 86, 100, 84, 27, 73, 88, 80, 1, 69, 14, 22, 10, 16, 7, 71, 77, 4, 8, 64], [126, 104, 99, 87, 28, 32, 85, 81, 92, 83, 115, 47, 127, 79, 110, 42, 76, 45, 121, 116, 58, 95, 108, 68, 31, 15, 78, 72, 41, 113, 60, 86, 106, 5, 54, 122, 117, 11, 119, 96, 50, 9, 24, 26, 6, 7, 52, 91, 22, 0, 101, 21, 57, 8, 12, 90, 17, 111, 66, 13, 44, 18, 23, 93, 94, 30, 55, 105, 114, 70, 118, 37, 49, 51, 25, 123, 48, 97, 88, 46, 74, 63, 38, 56, 39, 43, 59, 27, 120, 82, 34, 53, 62, 89, 102, 33, 103, 19, 80, 36, 61, 98, 124, 125, 14, 112, 109, 29, 100, 16, 107, 73, 84, 2, 20, 10, 35, 4, 77, 40, 71, 3, 64, 67, 1, 75, 65, 69], [104, 126, 99, 28, 87, 85, 32, 83, 81, 79, 92, 127, 116, 89, 111, 76, 47, 48, 25, 50, 72, 122, 15, 74, 54, 6, 11, 106, 51, 12, 119, 2, 52, 68, 105, 43, 58, 57, 42, 118, 3, 45, 108, 1, 13, 14, 31, 70, 23, 94, 5, 0, 115, 17, 29, 110, 41, 107, 113, 64, 93, 21, 46, 63, 120, 30, 123, 26, 61, 82, 39, 8, 125, 62, 102, 59, 112, 75, 90, 124, 53, 37, 60, 9, 19, 86, 65, 101, 84, 78, 73, 117, 69, 44, 40, 109, 33, 114, 77, 18, 24, 121, 34, 56, 38, 103, 66, 10, 95, 36, 100, 49, 97, 55, 98, 27, 7, 22, 67, 4, 35, 91, 88, 96, 20, 16, 80, 71], [62, 126, 48, 39, 116, 53, 30, 27, 122, 88, 20, 35, 117, 76, 121, 15, 91, 106, 112, 83, 55, 57, 79, 81, 94, 58, 86, 32, 123, 124, 52, 46, 49, 72, 19, 84, 33, 60, 98, 63, 89, 115, 100, 127, 50, 29, 24, 34, 56, 107, 114, 5, 41, 105, 108, 109, 51, 61, 44, 113, 118, 110, 26, 125, 47, 42, 23, 119, 87, 103, 18, 28, 43, 120, 36, 111, 17, 54, 22, 37, 59, 45, 31, 75, 95, 104, 93, 10, 97, 13, 102, 12, 85, 99, 40, 25, 38, 21, 74, 101, 90, 92, 78, 2, 14, 8, 82, 80, 96, 9, 16, 77, 3, 1, 11, 71, 73, 69, 6, 7, 70, 67, 66, 68, 4, 0, 65, 64], [116, 126, 39, 62, 48, 55, 57, 117, 123, 46, 53, 125, 111, 49, 121, 30, 50, 60, 41, 20, 122, 52, 127, 51, 120, 63, 114, 124, 89, 56, 115, 35, 59, 58, 113, 118, 43, 109, 32, 86, 54, 112, 47, 61, 88, 42, 106, 119, 108, 44, 45, 110, 107, 37, 34, 91, 40, 103, 105, 27, 104, 81, 23, 92, 102, 80, 38, 98, 29, 94, 101, 33, 28, 99, 36, 22, 87, 93, 9, 79, 100, 90, 73, 95, 96, 13, 97, 25, 31, 76, 26, 14, 82, 85, 84, 68, 83, 17, 1, 77, 21, 15, 75, 7, 78, 18, 2, 24, 16, 3, 71, 70, 11, 72, 4, 67, 0, 74, 12, 10, 5, 66, 65, 6, 19, 64, 8, 69], [126, 62, 116, 39, 48, 117, 55, 57, 53, 111, 123, 112, 49, 122, 114, 58, 120, 118, 124, 50, 20, 35, 127, 121, 42, 51, 30, 109, 88, 46, 56, 32, 52, 60, 28, 41, 115, 119, 43, 113, 125, 59, 47, 63, 106, 108, 103, 94, 44, 61, 110, 54, 101, 45, 89, 104, 107, 40, 86, 34, 38, 102, 100, 105, 81, 99, 37, 95, 87, 29, 93, 23, 92, 14, 36, 98, 80, 33, 97, 79, 21, 26, 31, 90, 27, 85, 91, 18, 22, 96, 25, 13, 83, 15, 77, 17, 84, 24, 76, 9, 82, 12, 69, 73, 75, 70, 78, 2, 19, 11, 16, 72, 10, 64, 67, 66, 3, 4, 0, 6, 71, 65, 68, 74, 1, 8, 5, 7], [126, 116, 39, 48, 55, 57, 27, 123, 30, 117, 83, 86, 122, 52, 37, 20, 53, 26, 62, 118, 110, 89, 112, 91, 72, 17, 19, 35, 32, 121, 16, 41, 94, 10, 79, 4, 74, 76, 88, 15, 125, 124, 114, 60, 25, 120, 115, 49, 29, 107, 56, 63, 22, 13, 106, 97, 50, 58, 51, 93, 42, 46, 34, 113, 127, 80, 111, 109, 81, 61, 108, 28, 5, 105, 66, 95, 100, 59, 31, 47, 75, 119, 38, 24, 23, 87, 33, 92, 44, 98, 9, 54, 21, 78, 45, 14, 36, 18, 77, 102, 84, 67, 7, 104, 43, 101, 68, 103, 8, 73, 6, 12, 90, 40, 11, 99, 82, 69, 85, 0, 71, 65, 64, 96, 70, 2, 3, 1], [55, 103, 62, 52, 117, 98, 25, 57, 89, 92, 115, 28, 119, 54, 20, 83, 22, 78, 120, 126, 48, 121, 73, 19, 58, 50, 76, 27, 51, 82, 45, 125, 60, 111, 49, 32, 109, 12, 123, 112, 107, 94, 81, 124, 53, 40, 33, 113, 110, 38, 61, 118, 114, 43, 42, 116, 86, 17, 100, 84, 59, 14, 63, 47, 46, 108, 127, 97, 79, 95, 101, 18, 56, 96, 122, 106, 88, 44, 5, 104, 105, 41, 6, 37, 77, 99, 36, 15, 35, 8, 75, 39, 70, 93, 80, 30, 102, 91, 26, 72, 31, 16, 90, 3, 23, 85, 29, 66, 24, 64, 71, 34, 68, 69, 21, 1, 11, 9, 87, 0, 74, 67, 13, 65, 7, 10, 2, 4], [55, 103, 98, 20, 25, 89, 120, 92, 112, 51, 117, 28, 126, 78, 54, 84, 48, 62, 57, 73, 58, 52, 50, 121, 32, 118, 125, 99, 95, 127, 22, 108, 56, 119, 122, 59, 124, 110, 116, 83, 104, 109, 47, 61, 15, 111, 106, 113, 46, 53, 115, 49, 101, 82, 43, 79, 63, 114, 100, 107, 40, 44, 37, 67, 123, 60, 24, 97, 45, 35, 102, 27, 38, 42, 31, 11, 76, 74, 88, 105, 87, 36, 68, 91, 81, 23, 71, 94, 29, 19, 18, 26, 93, 33, 85, 30, 41, 90, 21, 39, 17, 9, 96, 86, 12, 10, 77, 3, 14, 80, 69, 34, 1, 8, 16, 2, 70, 13, 65, 6, 75, 72, 4, 7, 5, 66, 64, 0], [55, 103, 57, 62, 25, 98, 89, 83, 92, 78, 117, 20, 52, 54, 28, 120, 112, 22, 126, 125, 115, 48, 32, 76, 84, 61, 73, 51, 118, 50, 58, 19, 119, 41, 116, 60, 53, 64, 8, 108, 5, 56, 113, 105, 109, 27, 59, 49, 46, 127, 121, 95, 111, 114, 47, 12, 82, 124, 63, 123, 110, 66, 43, 65, 106, 44, 107, 30, 17, 122, 6, 38, 104, 3, 39, 79, 99, 45, 16, 72, 77, 81, 102, 86, 40, 13, 37, 94, 100, 36, 91, 14, 42, 75, 93, 2, 35, 23, 69, 101, 29, 97, 33, 90, 85, 34, 26, 70, 68, 31, 88, 24, 74, 10, 80, 96, 1, 15, 87, 18, 0, 21, 9, 67, 11, 7, 4, 71], [55, 103, 62, 52, 98, 20, 82, 92, 89, 22, 25, 117, 63, 51, 108, 112, 84, 57, 58, 54, 73, 56, 125, 50, 28, 44, 71, 119, 124, 48, 32, 79, 11, 15, 109, 87, 120, 74, 46, 53, 49, 43, 18, 97, 42, 126, 121, 110, 116, 77, 59, 113, 101, 45, 122, 61, 115, 21, 39, 107, 114, 106, 37, 30, 118, 26, 88, 123, 75, 111, 2, 104, 100, 31, 41, 127, 68, 14, 38, 60, 93, 99, 81, 102, 33, 94, 29, 17, 40, 105, 95, 85, 1, 47, 83, 23, 35, 7, 10, 27, 12, 8, 24, 90, 91, 0, 80, 19, 86, 4, 70, 96, 13, 36, 78, 16, 67, 69, 34, 6, 3, 76, 9, 5, 66, 72, 65, 64], [121, 127, 56, 55, 39, 118, 63, 98, 31, 51, 122, 89, 25, 57, 124, 120, 119, 84, 112, 125, 18, 105, 92, 53, 62, 59, 58, 52, 115, 15, 50, 49, 110, 28, 23, 47, 76, 54, 108, 60, 117, 116, 61, 114, 111, 113, 106, 126, 123, 10, 46, 30, 16, 107, 40, 24, 104, 48, 32, 96, 102, 43, 101, 6, 44, 90, 45, 41, 80, 87, 109, 99, 88, 42, 94, 79, 22, 36, 100, 14, 83, 95, 38, 37, 85, 69, 82, 93, 19, 75, 20, 86, 27, 77, 9, 81, 35, 97, 8, 34, 74, 71, 64, 13, 78, 2, 26, 103, 33, 12, 67, 72, 29, 65, 21, 11, 5, 91, 70, 68, 7, 17, 3, 1, 4, 0, 73, 66], [55, 127, 56, 39, 121, 118, 63, 122, 51, 89, 31, 84, 57, 120, 125, 18, 124, 119, 105, 98, 116, 62, 112, 117, 52, 53, 25, 115, 59, 50, 92, 95, 28, 76, 60, 15, 61, 49, 106, 47, 96, 111, 58, 54, 48, 123, 114, 126, 40, 113, 23, 108, 90, 85, 110, 46, 45, 107, 43, 44, 30, 109, 36, 102, 16, 101, 41, 38, 42, 79, 104, 24, 10, 103, 99, 20, 100, 22, 37, 35, 14, 87, 32, 33, 34, 97, 93, 88, 81, 80, 83, 9, 86, 82, 77, 27, 94, 29, 91, 78, 19, 75, 8, 74, 13, 21, 72, 71, 4, 26, 12, 17, 70, 73, 2, 6, 11, 5, 68, 65, 0, 69, 1, 64, 7, 3, 67, 66], [127, 39, 56, 63, 118, 55, 31, 98, 89, 51, 122, 57, 84, 92, 53, 25, 120, 125, 62, 105, 47, 115, 54, 96, 30, 126, 119, 52, 124, 18, 112, 114, 87, 59, 110, 116, 40, 44, 50, 58, 49, 90, 48, 113, 103, 85, 117, 60, 107, 23, 45, 16, 123, 111, 61, 121, 95, 106, 43, 76, 108, 102, 21, 35, 46, 104, 42, 79, 109, 26, 22, 34, 99, 41, 101, 83, 93, 15, 36, 32, 28, 38, 91, 37, 100, 24, 20, 27, 33, 19, 82, 97, 81, 94, 10, 80, 86, 74, 88, 29, 14, 77, 72, 17, 75, 8, 12, 13, 78, 9, 5, 69, 71, 6, 4, 11, 2, 70, 66, 68, 64, 73, 65, 67, 3, 7, 1, 0], [56, 39, 127, 55, 121, 118, 51, 31, 63, 122, 57, 84, 92, 25, 105, 120, 124, 62, 59, 115, 119, 125, 53, 50, 112, 52, 98, 47, 58, 116, 60, 49, 95, 89, 61, 113, 126, 110, 123, 114, 117, 54, 18, 44, 111, 41, 79, 48, 43, 108, 45, 90, 23, 76, 109, 101, 106, 104, 107, 42, 46, 96, 40, 102, 88, 100, 38, 10, 28, 15, 37, 36, 99, 35, 22, 29, 94, 97, 32, 103, 30, 82, 33, 83, 16, 86, 93, 77, 34, 8, 21, 27, 75, 87, 20, 85, 24, 80, 9, 72, 5, 91, 26, 81, 12, 13, 74, 17, 19, 71, 78, 14, 2, 4, 70, 73, 6, 69, 64, 3, 11, 67, 66, 0, 68, 7, 1, 65]], "model.layers.21.self_attn.k_proj": [[112, 49, 39, 86, 93, 57, 84, 15, 33, 26, 54, 123, 35, 50, 82, 122, 12, 8, 121, 52, 59, 64, 119, 60, 113, 116, 127, 118, 13, 125, 109, 58, 51, 73, 126, 117, 70, 111, 114, 53, 120, 55, 108, 29, 63, 48, 115, 78, 66, 10, 110, 46, 32, 47, 56, 43, 61, 44, 45, 40, 107, 88, 124, 62, 41, 68, 87, 42, 101, 106, 105, 16, 102, 38, 36, 65, 17, 95, 104, 100, 34, 94, 98, 89, 67, 27, 5, 92, 24, 4, 91, 37, 9, 30, 6, 7, 83, 96, 19, 80, 85, 28, 81, 69, 31, 21, 99, 22, 3, 90, 75, 71, 25, 2, 97, 11, 79, 74, 23, 18, 1, 77, 20, 14, 76, 72, 0, 103], [120, 54, 64, 118, 9, 84, 78, 76, 15, 5, 18, 53, 71, 11, 72, 65, 56, 38, 6, 67, 25, 2, 3, 0, 66, 13, 87, 29, 80, 4, 95, 69, 1, 89, 93, 22, 85, 102, 92, 31, 10, 70, 7, 116, 68, 43, 86, 73, 40, 24, 88, 17, 90, 19, 8, 50, 110, 32, 39, 114, 26, 112, 55, 63, 57, 115, 46, 27, 83, 21, 124, 49, 48, 99, 36, 52, 45, 113, 51, 16, 111, 33, 74, 109, 91, 47, 28, 103, 125, 96, 81, 30, 59, 41, 105, 23, 98, 60, 108, 97, 121, 119, 100, 117, 77, 75, 34, 62, 94, 12, 127, 58, 61, 82, 35, 126, 122, 123, 44, 42, 104, 14, 37, 107, 20, 101, 106, 79], [42, 59, 35, 86, 55, 64, 8, 119, 58, 47, 80, 6, 14, 20, 92, 74, 114, 75, 25, 2, 62, 60, 18, 53, 127, 106, 13, 116, 112, 123, 57, 61, 56, 122, 4, 50, 125, 118, 95, 54, 87, 65, 67, 124, 96, 117, 51, 111, 126, 109, 48, 99, 110, 121, 49, 113, 103, 52, 69, 45, 120, 108, 30, 115, 85, 46, 36, 38, 90, 44, 76, 63, 40, 28, 32, 26, 91, 5, 100, 43, 83, 107, 94, 0, 98, 37, 3, 41, 101, 33, 105, 34, 15, 71, 97, 17, 102, 93, 79, 1, 104, 39, 27, 68, 73, 9, 88, 19, 31, 23, 29, 66, 21, 7, 24, 89, 72, 81, 12, 84, 16, 22, 77, 11, 78, 82, 10, 70], [38, 57, 59, 58, 93, 32, 26, 85, 87, 84, 15, 18, 74, 71, 0, 13, 123, 45, 52, 12, 111, 121, 22, 126, 63, 116, 110, 1, 55, 117, 42, 127, 16, 107, 125, 108, 62, 51, 114, 54, 61, 109, 56, 53, 60, 124, 99, 122, 25, 106, 3, 112, 46, 115, 48, 118, 41, 119, 49, 5, 43, 50, 89, 44, 105, 102, 4, 120, 2, 24, 104, 75, 47, 113, 68, 35, 101, 95, 14, 103, 39, 17, 40, 66, 19, 69, 100, 83, 78, 36, 70, 27, 81, 9, 34, 98, 88, 37, 33, 92, 30, 28, 97, 72, 82, 31, 94, 91, 11, 73, 8, 80, 7, 20, 29, 86, 6, 77, 65, 23, 21, 67, 76, 90, 64, 79, 96, 10], [40, 126, 83, 81, 92, 35, 115, 85, 87, 15, 74, 111, 96, 89, 72, 76, 46, 42, 79, 52, 122, 12, 68, 58, 0, 28, 70, 106, 127, 114, 48, 6, 66, 5, 14, 54, 44, 57, 116, 11, 119, 31, 2, 56, 41, 1, 3, 43, 123, 110, 65, 125, 32, 118, 60, 75, 107, 9, 109, 29, 50, 82, 112, 120, 78, 49, 101, 93, 7, 13, 25, 121, 84, 59, 77, 88, 90, 86, 24, 34, 64, 8, 67, 108, 94, 30, 53, 20, 61, 47, 27, 39, 55, 117, 113, 69, 100, 26, 36, 22, 33, 105, 97, 103, 99, 102, 91, 51, 124, 95, 73, 98, 38, 23, 37, 16, 62, 45, 4, 63, 80, 10, 18, 71, 21, 19, 17, 104], [126, 103, 116, 86, 99, 55, 96, 52, 94, 62, 48, 26, 53, 112, 57, 109, 42, 91, 88, 63, 28, 58, 113, 56, 123, 20, 29, 110, 121, 102, 117, 50, 114, 47, 92, 124, 83, 35, 13, 115, 60, 81, 119, 127, 120, 59, 105, 125, 51, 37, 97, 43, 98, 49, 122, 61, 54, 111, 107, 95, 89, 46, 30, 33, 44, 79, 45, 118, 106, 10, 70, 40, 101, 108, 15, 36, 31, 104, 34, 7, 76, 27, 41, 85, 65, 38, 100, 72, 73, 87, 23, 66, 21, 14, 93, 0, 18, 5, 3, 90, 32, 17, 80, 25, 82, 6, 39, 71, 68, 16, 4, 8, 74, 24, 22, 84, 75, 77, 78, 12, 19, 9, 2, 11, 67, 69, 1, 64], [55, 39, 34, 22, 28, 89, 62, 125, 78, 117, 20, 53, 58, 50, 51, 54, 83, 109, 96, 112, 61, 59, 44, 113, 49, 114, 48, 121, 118, 119, 111, 60, 116, 63, 124, 126, 46, 127, 8, 47, 52, 64, 5, 56, 123, 82, 73, 43, 120, 91, 42, 122, 115, 12, 106, 1, 108, 110, 45, 30, 40, 18, 104, 79, 105, 107, 95, 57, 37, 93, 100, 81, 98, 23, 76, 41, 70, 77, 36, 3, 38, 66, 102, 27, 26, 90, 101, 35, 33, 31, 2, 32, 4, 11, 85, 99, 92, 10, 75, 24, 88, 97, 0, 80, 6, 65, 15, 17, 29, 21, 71, 94, 69, 103, 16, 25, 74, 87, 84, 13, 19, 7, 67, 68, 14, 9, 72, 86], [103, 127, 56, 34, 22, 55, 121, 95, 28, 118, 63, 122, 89, 116, 51, 125, 115, 62, 119, 57, 59, 58, 60, 53, 43, 52, 38, 124, 49, 48, 47, 117, 114, 61, 84, 92, 120, 123, 46, 106, 111, 44, 112, 113, 54, 126, 107, 50, 108, 71, 45, 23, 105, 29, 80, 109, 42, 75, 110, 40, 104, 35, 16, 41, 78, 101, 99, 37, 27, 98, 96, 102, 100, 18, 36, 10, 12, 64, 25, 33, 81, 86, 97, 30, 94, 87, 68, 79, 91, 39, 93, 88, 13, 90, 32, 66, 85, 83, 82, 17, 24, 26, 69, 21, 31, 1, 9, 20, 6, 8, 73, 14, 77, 19, 4, 5, 7, 76, 11, 65, 15, 74, 67, 2, 3, 72, 0, 70]], "model.layers.21.self_attn.qk_proj": [[126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 48, 103, 84, 20, 79, 119, 114, 25, 38, 15, 89, 29, 113, 86, 82, 122, 92, 40, 60, 22, 52, 23, 87, 99, 90, 35, 115, 18, 32, 12, 14, 111, 46, 78, 76, 6, 85, 72, 50, 117, 93, 51, 64, 75, 61, 21, 96, 125, 19, 10, 43, 26, 63, 123, 30, 108, 31, 34, 0, 83, 110, 124, 9, 45, 17, 44, 47, 8, 16, 11, 74, 80, 104, 73, 105, 5, 7, 69, 66, 95, 109, 67, 81, 107, 2, 97, 65, 13, 1, 71, 98, 68, 27, 77, 37, 33, 3, 41, 70, 36, 24, 88, 101, 94, 91, 4, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 116, 42, 102, 118, 62, 53, 106, 121, 39, 28, 103, 48, 79, 20, 114, 84, 25, 29, 38, 89, 113, 119, 92, 22, 23, 60, 15, 99, 90, 52, 122, 87, 82, 12, 115, 86, 32, 35, 18, 78, 14, 51, 111, 40, 6, 76, 72, 117, 123, 93, 85, 75, 50, 125, 46, 0, 43, 44, 63, 108, 19, 26, 17, 21, 61, 64, 47, 30, 69, 31, 66, 124, 8, 74, 73, 11, 9, 96, 10, 109, 105, 104, 34, 110, 45, 2, 5, 83, 7, 95, 71, 13, 67, 27, 80, 16, 3, 88, 81, 33, 98, 1, 65, 107, 36, 97, 77, 41, 37, 68, 70, 4, 101, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 62, 106, 118, 53, 121, 28, 39, 103, 84, 48, 29, 20, 25, 15, 99, 22, 92, 23, 122, 119, 38, 79, 114, 86, 89, 115, 90, 113, 60, 87, 35, 117, 52, 78, 40, 82, 18, 76, 14, 12, 111, 32, 6, 51, 64, 85, 72, 46, 0, 96, 124, 44, 125, 75, 63, 93, 31, 21, 69, 43, 30, 34, 108, 2, 50, 19, 123, 10, 110, 5, 74, 8, 17, 65, 66, 11, 47, 109, 83, 9, 104, 61, 26, 98, 73, 70, 16, 1, 95, 7, 80, 67, 13, 3, 81, 77, 41, 105, 45, 36, 88, 27, 71, 107, 4, 68, 33, 24, 97, 37, 91, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 48, 103, 84, 20, 89, 79, 15, 29, 92, 119, 22, 25, 99, 113, 23, 60, 38, 52, 115, 40, 14, 82, 122, 90, 35, 18, 114, 12, 86, 78, 111, 76, 32, 87, 117, 46, 85, 6, 0, 17, 51, 72, 125, 96, 50, 30, 5, 11, 75, 93, 63, 123, 43, 110, 69, 47, 44, 34, 70, 19, 64, 2, 8, 10, 104, 83, 108, 124, 31, 21, 26, 61, 16, 66, 109, 65, 73, 74, 1, 9, 77, 98, 27, 81, 95, 45, 105, 67, 80, 7, 107, 71, 3, 88, 41, 101, 33, 36, 4, 13, 68, 91, 24, 97, 37, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 56, 58, 116, 42, 118, 102, 62, 106, 121, 53, 39, 103, 113, 28, 20, 84, 48, 60, 79, 38, 15, 23, 89, 119, 114, 29, 115, 82, 25, 92, 22, 86, 52, 90, 76, 122, 99, 40, 18, 78, 12, 32, 117, 35, 111, 14, 51, 50, 75, 46, 87, 85, 72, 96, 63, 125, 8, 70, 64, 0, 21, 93, 26, 43, 109, 44, 17, 6, 31, 69, 10, 30, 47, 2, 19, 5, 110, 61, 11, 7, 74, 16, 83, 34, 73, 123, 104, 66, 81, 98, 9, 108, 95, 65, 1, 105, 124, 27, 80, 77, 4, 67, 45, 71, 33, 88, 3, 107, 97, 13, 37, 36, 68, 101, 41, 91, 94, 24, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 103, 28, 39, 20, 84, 79, 29, 48, 114, 89, 86, 15, 115, 82, 119, 23, 113, 60, 76, 22, 25, 40, 38, 92, 35, 99, 78, 122, 18, 32, 14, 90, 12, 87, 52, 111, 85, 50, 75, 63, 72, 70, 43, 51, 46, 125, 117, 8, 17, 31, 64, 21, 96, 93, 34, 0, 124, 26, 16, 83, 30, 44, 69, 19, 11, 123, 10, 108, 109, 9, 81, 47, 104, 74, 95, 66, 5, 2, 61, 73, 97, 7, 27, 110, 13, 65, 80, 45, 67, 1, 105, 3, 98, 107, 6, 4, 71, 77, 33, 88, 24, 41, 68, 37, 94, 91, 36, 101, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 42, 116, 118, 102, 62, 106, 121, 53, 39, 28, 20, 84, 79, 23, 25, 29, 103, 38, 60, 15, 89, 86, 90, 22, 99, 82, 119, 18, 76, 40, 122, 12, 92, 113, 48, 35, 114, 87, 32, 78, 115, 111, 70, 85, 14, 51, 75, 46, 52, 117, 50, 93, 64, 19, 8, 13, 125, 72, 16, 17, 11, 10, 30, 21, 63, 9, 0, 77, 26, 31, 83, 44, 96, 123, 43, 34, 1, 66, 47, 74, 104, 80, 81, 45, 124, 105, 69, 110, 61, 7, 73, 2, 71, 65, 108, 109, 95, 97, 5, 41, 27, 4, 3, 107, 98, 36, 37, 88, 6, 67, 33, 24, 91, 101, 68, 94, 100], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 116, 42, 102, 118, 106, 62, 121, 53, 28, 39, 20, 25, 122, 38, 103, 23, 84, 22, 29, 90, 48, 60, 119, 15, 79, 32, 86, 111, 89, 82, 99, 52, 113, 18, 92, 12, 40, 70, 117, 85, 115, 87, 35, 78, 14, 76, 114, 51, 46, 125, 0, 93, 21, 75, 123, 30, 64, 19, 96, 11, 63, 8, 31, 83, 43, 26, 50, 104, 34, 44, 16, 47, 17, 110, 61, 13, 10, 72, 108, 74, 69, 124, 9, 80, 73, 2, 65, 45, 95, 77, 81, 66, 109, 67, 7, 41, 5, 3, 36, 68, 1, 98, 6, 105, 33, 71, 107, 37, 4, 27, 88, 24, 97, 91, 94, 101, 100], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 42, 116, 102, 118, 62, 106, 121, 53, 28, 39, 48, 84, 38, 20, 29, 15, 103, 89, 122, 25, 119, 111, 23, 79, 60, 22, 92, 90, 87, 51, 52, 99, 32, 86, 78, 82, 35, 18, 76, 115, 40, 114, 12, 113, 14, 85, 117, 46, 70, 93, 123, 125, 47, 17, 0, 21, 8, 96, 44, 26, 31, 19, 109, 110, 50, 30, 63, 83, 11, 75, 34, 64, 43, 45, 61, 72, 9, 10, 81, 104, 13, 73, 5, 74, 108, 16, 2, 65, 66, 6, 69, 77, 67, 124, 95, 80, 41, 105, 7, 68, 27, 3, 98, 71, 107, 33, 1, 88, 97, 101, 4, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 53, 106, 39, 103, 28, 48, 84, 79, 38, 20, 119, 114, 92, 89, 15, 29, 52, 25, 60, 86, 51, 23, 99, 90, 115, 122, 22, 113, 35, 40, 78, 76, 87, 111, 46, 50, 32, 12, 18, 14, 125, 82, 123, 8, 64, 85, 43, 117, 108, 110, 0, 63, 96, 21, 6, 74, 109, 75, 31, 70, 83, 11, 17, 124, 5, 30, 10, 72, 44, 93, 66, 47, 34, 45, 26, 2, 19, 73, 104, 9, 105, 81, 61, 69, 95, 98, 3, 67, 16, 7, 1, 27, 41, 71, 65, 80, 33, 77, 107, 13, 97, 36, 37, 88, 4, 91, 24, 68, 101, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 56, 58, 116, 42, 102, 118, 62, 53, 121, 106, 39, 103, 28, 20, 48, 79, 84, 113, 119, 89, 15, 38, 115, 99, 86, 25, 92, 23, 22, 29, 78, 40, 51, 35, 52, 60, 76, 82, 114, 8, 122, 90, 18, 14, 12, 0, 87, 32, 85, 6, 5, 50, 46, 64, 111, 125, 96, 66, 123, 72, 47, 117, 30, 34, 75, 74, 83, 43, 110, 93, 17, 11, 70, 124, 19, 9, 31, 73, 81, 44, 21, 65, 98, 61, 108, 2, 45, 67, 63, 69, 80, 3, 26, 71, 109, 1, 10, 105, 27, 7, 95, 16, 77, 104, 68, 33, 97, 13, 107, 88, 41, 4, 91, 94, 24, 101, 100, 36, 37], [126, 55, 120, 54, 59, 112, 57, 127, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 89, 20, 25, 84, 48, 86, 23, 22, 60, 15, 79, 103, 29, 38, 115, 99, 76, 40, 114, 35, 12, 90, 122, 78, 32, 113, 51, 92, 6, 18, 82, 119, 85, 111, 87, 8, 14, 96, 52, 93, 75, 19, 47, 17, 74, 11, 72, 0, 46, 50, 64, 83, 13, 34, 31, 43, 21, 123, 9, 73, 117, 30, 110, 125, 44, 81, 10, 16, 80, 26, 63, 5, 65, 77, 69, 109, 71, 108, 66, 104, 3, 1, 97, 67, 98, 45, 95, 61, 2, 105, 124, 27, 4, 7, 107, 70, 88, 41, 68, 36, 101, 91, 33, 94, 24, 37, 100], [126, 55, 54, 120, 112, 59, 49, 57, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 103, 39, 28, 25, 48, 20, 84, 60, 29, 23, 38, 15, 122, 90, 86, 89, 115, 52, 79, 92, 119, 32, 22, 111, 113, 51, 87, 35, 114, 18, 40, 82, 12, 99, 6, 78, 85, 76, 123, 50, 96, 43, 14, 46, 93, 125, 21, 47, 117, 108, 31, 72, 34, 64, 83, 124, 19, 63, 110, 11, 61, 104, 74, 73, 75, 17, 8, 26, 109, 10, 105, 77, 41, 30, 45, 44, 95, 0, 16, 5, 81, 2, 97, 9, 13, 69, 67, 80, 36, 66, 27, 98, 71, 3, 33, 1, 65, 7, 88, 107, 68, 37, 91, 101, 94, 4, 70, 24, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 39, 103, 119, 48, 84, 79, 22, 20, 86, 29, 15, 99, 89, 25, 60, 38, 23, 113, 114, 122, 51, 111, 87, 52, 12, 115, 40, 50, 35, 18, 92, 90, 76, 32, 14, 82, 6, 64, 78, 125, 0, 85, 72, 46, 123, 117, 8, 96, 63, 17, 75, 43, 45, 11, 21, 110, 61, 104, 10, 19, 1, 73, 83, 30, 31, 26, 74, 47, 2, 34, 5, 124, 93, 80, 66, 81, 9, 70, 108, 3, 69, 109, 13, 65, 105, 71, 41, 44, 16, 95, 67, 7, 27, 77, 107, 98, 97, 4, 33, 68, 101, 88, 36, 24, 37, 94, 91, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 39, 28, 103, 22, 79, 84, 20, 29, 38, 23, 89, 48, 15, 25, 86, 40, 60, 113, 119, 92, 122, 114, 99, 12, 18, 76, 90, 82, 52, 32, 35, 115, 51, 78, 111, 0, 14, 46, 85, 72, 87, 8, 6, 123, 64, 96, 75, 21, 50, 5, 125, 117, 17, 83, 30, 74, 10, 26, 34, 70, 31, 11, 47, 45, 73, 81, 9, 93, 105, 66, 98, 44, 7, 43, 110, 2, 19, 61, 108, 63, 69, 1, 67, 16, 71, 80, 124, 95, 109, 3, 65, 27, 13, 41, 4, 77, 68, 91, 97, 33, 104, 37, 24, 88, 36, 107, 101, 94, 100], [126, 55, 120, 54, 112, 59, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 103, 39, 22, 84, 79, 48, 89, 119, 29, 20, 115, 23, 99, 60, 25, 122, 15, 32, 86, 114, 12, 38, 113, 92, 40, 76, 18, 78, 90, 35, 52, 46, 111, 82, 85, 72, 87, 14, 51, 125, 8, 93, 0, 123, 75, 43, 50, 11, 96, 70, 30, 6, 10, 5, 21, 61, 64, 110, 17, 63, 34, 26, 2, 19, 83, 74, 73, 69, 105, 31, 47, 95, 7, 44, 109, 117, 108, 81, 9, 65, 98, 66, 13, 16, 124, 1, 80, 68, 45, 104, 71, 27, 4, 41, 97, 67, 77, 3, 107, 24, 91, 88, 33, 36, 37, 101, 100, 94], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 42, 116, 102, 118, 62, 121, 106, 28, 39, 53, 103, 25, 84, 89, 29, 20, 86, 38, 48, 60, 23, 119, 114, 79, 15, 22, 92, 90, 87, 111, 76, 115, 18, 40, 113, 52, 35, 82, 99, 32, 85, 122, 12, 51, 46, 125, 78, 70, 50, 14, 93, 123, 31, 117, 72, 21, 96, 19, 61, 10, 11, 63, 124, 44, 26, 30, 0, 64, 47, 9, 75, 34, 13, 83, 108, 43, 5, 16, 81, 17, 110, 8, 45, 80, 95, 41, 66, 1, 77, 105, 73, 104, 69, 65, 2, 74, 36, 67, 27, 97, 98, 109, 71, 7, 107, 37, 3, 6, 4, 24, 88, 101, 33, 91, 94, 68, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 106, 121, 28, 53, 39, 48, 84, 20, 103, 29, 89, 79, 38, 22, 60, 86, 15, 99, 114, 23, 113, 25, 12, 90, 92, 87, 76, 119, 122, 46, 111, 70, 82, 18, 14, 40, 35, 78, 32, 85, 115, 117, 0, 52, 123, 72, 64, 51, 44, 31, 26, 93, 63, 47, 96, 11, 30, 43, 75, 21, 125, 17, 10, 50, 83, 19, 69, 74, 104, 61, 66, 9, 5, 1, 34, 8, 110, 73, 80, 16, 2, 124, 108, 3, 95, 65, 13, 81, 27, 71, 77, 7, 45, 4, 67, 41, 36, 68, 109, 107, 6, 98, 97, 101, 88, 105, 37, 33, 91, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 53, 106, 39, 48, 28, 103, 38, 20, 84, 119, 79, 89, 46, 122, 15, 29, 60, 111, 114, 92, 23, 113, 52, 86, 25, 99, 12, 40, 90, 22, 76, 82, 32, 35, 18, 123, 70, 87, 115, 14, 125, 78, 117, 51, 72, 93, 44, 63, 47, 11, 85, 64, 31, 96, 0, 75, 30, 110, 43, 10, 34, 61, 17, 26, 2, 5, 21, 83, 50, 109, 9, 8, 104, 45, 108, 74, 19, 81, 73, 105, 124, 98, 16, 1, 7, 97, 66, 107, 95, 6, 69, 80, 41, 71, 27, 67, 13, 3, 68, 33, 88, 37, 101, 36, 65, 77, 24, 91, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 28, 39, 103, 48, 20, 119, 84, 29, 25, 38, 79, 60, 113, 99, 52, 90, 40, 114, 89, 23, 12, 15, 86, 92, 35, 115, 32, 22, 123, 76, 14, 46, 125, 122, 78, 87, 18, 82, 85, 51, 111, 70, 72, 117, 96, 93, 43, 0, 11, 30, 50, 108, 47, 31, 10, 34, 44, 124, 61, 63, 75, 66, 5, 64, 19, 8, 109, 74, 26, 9, 95, 6, 17, 104, 73, 21, 83, 110, 2, 81, 105, 98, 69, 7, 71, 13, 45, 107, 1, 3, 16, 33, 67, 80, 68, 37, 97, 77, 88, 27, 65, 41, 4, 36, 91, 101, 100, 94, 24], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 121, 62, 106, 53, 28, 48, 103, 39, 119, 20, 113, 29, 84, 79, 60, 38, 22, 86, 23, 89, 25, 15, 35, 40, 122, 92, 90, 46, 99, 114, 12, 115, 52, 76, 32, 87, 18, 78, 111, 82, 125, 123, 51, 14, 117, 43, 11, 85, 72, 96, 50, 34, 6, 47, 31, 26, 104, 30, 0, 17, 93, 44, 19, 63, 75, 64, 10, 61, 45, 108, 21, 83, 9, 8, 110, 70, 74, 80, 95, 66, 2, 5, 81, 124, 109, 73, 71, 67, 16, 69, 1, 13, 98, 3, 7, 27, 107, 88, 36, 91, 97, 33, 77, 65, 68, 24, 41, 101, 105, 37, 4, 94, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 118, 102, 121, 62, 106, 53, 28, 39, 20, 48, 84, 29, 60, 38, 103, 15, 25, 22, 113, 79, 122, 92, 119, 99, 23, 89, 111, 46, 86, 114, 52, 35, 90, 12, 125, 82, 14, 40, 78, 76, 115, 32, 64, 6, 87, 123, 0, 18, 51, 43, 85, 117, 47, 93, 72, 50, 8, 63, 21, 61, 2, 30, 31, 10, 17, 11, 5, 75, 66, 19, 44, 110, 124, 96, 26, 34, 108, 104, 80, 1, 83, 95, 71, 74, 69, 9, 73, 45, 65, 81, 109, 70, 107, 67, 16, 77, 7, 41, 97, 13, 27, 98, 36, 3, 105, 4, 33, 68, 88, 91, 101, 37, 24, 94, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 118, 102, 62, 121, 106, 53, 28, 39, 103, 20, 25, 60, 38, 114, 84, 48, 92, 119, 29, 122, 22, 86, 90, 23, 117, 111, 113, 46, 89, 79, 15, 32, 125, 40, 52, 35, 115, 87, 14, 99, 51, 82, 18, 12, 61, 6, 78, 76, 64, 43, 96, 30, 123, 44, 108, 124, 21, 31, 11, 10, 50, 93, 47, 85, 8, 34, 17, 19, 72, 0, 63, 110, 104, 83, 5, 75, 109, 26, 66, 74, 9, 45, 95, 107, 3, 73, 2, 69, 71, 81, 16, 80, 1, 97, 7, 98, 36, 13, 67, 88, 33, 77, 41, 27, 70, 65, 91, 37, 24, 68, 105, 4, 101, 94, 100], [126, 55, 120, 54, 112, 59, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 39, 48, 103, 114, 29, 20, 84, 38, 79, 113, 89, 25, 119, 22, 15, 60, 40, 23, 92, 52, 115, 86, 82, 90, 12, 14, 99, 35, 122, 76, 87, 46, 18, 32, 111, 6, 78, 8, 123, 96, 43, 85, 0, 104, 51, 72, 10, 50, 125, 63, 11, 21, 30, 61, 26, 34, 83, 44, 17, 31, 9, 93, 19, 108, 117, 110, 47, 2, 5, 74, 75, 64, 71, 7, 45, 69, 73, 95, 81, 13, 109, 27, 1, 105, 67, 66, 16, 98, 80, 70, 65, 124, 3, 97, 107, 33, 4, 77, 88, 91, 101, 41, 24, 37, 68, 36, 94, 100], [126, 55, 120, 54, 59, 112, 57, 49, 127, 58, 56, 116, 42, 102, 118, 62, 121, 106, 39, 28, 53, 20, 84, 25, 23, 38, 103, 60, 89, 79, 15, 119, 86, 48, 22, 29, 113, 90, 12, 114, 92, 40, 18, 122, 99, 115, 82, 14, 46, 52, 32, 35, 51, 76, 78, 123, 111, 125, 87, 6, 85, 8, 21, 11, 83, 0, 61, 10, 108, 64, 34, 72, 96, 75, 93, 19, 17, 45, 26, 63, 74, 47, 43, 50, 71, 9, 104, 30, 124, 16, 31, 109, 73, 80, 117, 13, 81, 70, 5, 66, 77, 44, 2, 95, 110, 65, 69, 1, 67, 97, 7, 105, 98, 27, 107, 4, 91, 3, 33, 37, 68, 88, 36, 24, 101, 94, 41, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 121, 28, 53, 106, 39, 48, 103, 25, 89, 23, 38, 20, 119, 22, 60, 29, 84, 86, 79, 122, 113, 90, 40, 123, 15, 32, 35, 12, 92, 82, 46, 52, 99, 61, 114, 14, 18, 51, 111, 115, 85, 76, 87, 78, 93, 43, 21, 8, 125, 50, 30, 108, 0, 96, 10, 31, 11, 117, 34, 44, 26, 70, 47, 64, 19, 83, 75, 6, 110, 9, 17, 72, 80, 63, 104, 124, 77, 66, 95, 109, 69, 73, 45, 74, 5, 98, 81, 16, 13, 97, 41, 71, 7, 105, 2, 65, 3, 27, 107, 1, 67, 33, 36, 68, 94, 37, 88, 101, 4, 24, 91, 100], [126, 55, 120, 54, 59, 112, 49, 127, 57, 58, 56, 42, 116, 102, 118, 121, 53, 62, 106, 28, 39, 103, 119, 48, 25, 20, 23, 84, 38, 89, 60, 29, 15, 79, 22, 115, 46, 52, 86, 114, 99, 35, 122, 51, 113, 123, 90, 82, 111, 12, 92, 32, 76, 40, 87, 14, 117, 61, 18, 125, 70, 78, 8, 11, 85, 50, 44, 63, 96, 34, 110, 45, 64, 83, 108, 0, 30, 93, 21, 104, 109, 47, 31, 72, 19, 73, 17, 75, 43, 5, 124, 69, 3, 10, 13, 105, 74, 80, 26, 98, 9, 65, 2, 81, 16, 1, 41, 66, 97, 7, 77, 6, 68, 95, 107, 67, 27, 36, 71, 33, 4, 24, 101, 94, 88, 91, 37, 100], [126, 55, 120, 54, 59, 112, 127, 57, 49, 58, 56, 116, 42, 102, 118, 62, 121, 106, 53, 39, 28, 20, 84, 79, 103, 89, 48, 23, 15, 38, 25, 52, 119, 29, 12, 82, 113, 22, 60, 114, 90, 76, 14, 92, 122, 32, 78, 18, 99, 40, 86, 70, 35, 46, 50, 123, 8, 115, 87, 85, 117, 125, 111, 64, 21, 72, 83, 96, 43, 5, 11, 19, 0, 93, 51, 74, 47, 73, 9, 66, 61, 31, 63, 75, 26, 81, 10, 30, 2, 17, 104, 44, 34, 108, 45, 69, 110, 80, 1, 7, 124, 67, 95, 109, 77, 105, 6, 98, 71, 16, 13, 68, 41, 3, 65, 27, 4, 97, 33, 24, 107, 36, 101, 88, 37, 94, 91, 100], [126, 55, 120, 54, 59, 112, 127, 49, 57, 58, 56, 116, 42, 102, 118, 53, 62, 121, 106, 103, 28, 114, 39, 84, 38, 20, 48, 52, 79, 25, 89, 15, 22, 29, 92, 40, 23, 113, 86, 119, 60, 32, 12, 99, 50, 35, 76, 70, 90, 18, 115, 82, 78, 14, 122, 64, 8, 111, 46, 31, 85, 51, 0, 123, 125, 63, 87, 43, 45, 5, 61, 11, 72, 34, 108, 75, 96, 66, 69, 74, 117, 10, 93, 124, 17, 2, 30, 19, 21, 83, 1, 110, 73, 104, 44, 109, 7, 47, 95, 65, 81, 71, 9, 26, 16, 33, 98, 3, 107, 27, 77, 67, 105, 13, 97, 4, 80, 6, 37, 101, 36, 24, 88, 41, 68, 100, 94, 91], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 121, 62, 53, 106, 28, 39, 103, 84, 20, 38, 89, 15, 29, 60, 79, 52, 25, 48, 86, 119, 22, 113, 23, 32, 92, 114, 82, 18, 99, 76, 14, 40, 12, 35, 78, 115, 90, 46, 70, 87, 50, 51, 122, 125, 111, 8, 63, 123, 93, 117, 96, 19, 21, 85, 72, 31, 11, 43, 75, 124, 17, 73, 30, 83, 10, 74, 34, 108, 69, 61, 26, 110, 0, 45, 104, 80, 44, 13, 7, 64, 9, 109, 5, 47, 81, 27, 67, 66, 95, 16, 1, 2, 65, 105, 77, 71, 98, 3, 6, 68, 107, 97, 37, 33, 91, 4, 41, 101, 24, 88, 36, 94, 100], [126, 55, 120, 54, 112, 59, 127, 49, 57, 58, 56, 116, 42, 102, 118, 62, 106, 121, 53, 28, 103, 39, 114, 84, 20, 25, 113, 48, 15, 115, 119, 29, 23, 52, 79, 22, 89, 90, 122, 60, 40, 86, 38, 12, 32, 99, 92, 14, 51, 76, 78, 18, 46, 111, 35, 125, 87, 82, 50, 63, 85, 0, 72, 123, 124, 70, 43, 74, 8, 96, 64, 31, 117, 6, 19, 75, 110, 93, 104, 34, 9, 73, 7, 30, 5, 11, 2, 47, 10, 83, 69, 21, 17, 66, 44, 13, 108, 45, 65, 61, 16, 1, 26, 81, 80, 98, 95, 109, 97, 67, 68, 77, 71, 3, 41, 27, 105, 4, 107, 37, 36, 101, 33, 24, 88, 91, 100, 94], [126, 55, 120, 54, 59, 112, 49, 57, 127, 58, 56, 42, 116, 102, 118, 62, 121, 106, 103, 39, 53, 28, 48, 20, 84, 25, 23, 15, 29, 79, 38, 89, 114, 86, 119, 60, 22, 90, 92, 115, 18, 99, 40, 35, 12, 32, 14, 52, 122, 76, 113, 111, 51, 82, 125, 123, 46, 63, 87, 50, 85, 78, 75, 72, 43, 61, 93, 8, 74, 104, 96, 6, 11, 117, 108, 19, 110, 21, 31, 17, 30, 124, 73, 70, 44, 9, 34, 10, 83, 45, 64, 109, 47, 5, 26, 80, 105, 13, 81, 7, 0, 66, 41, 98, 77, 27, 95, 69, 71, 65, 16, 2, 107, 33, 67, 3, 88, 36, 68, 97, 1, 37, 4, 94, 24, 91, 101, 100]], "model.layers.22.self_attn.q_proj": [[57, 40, 50, 121, 54, 118, 51, 125, 52, 119, 123, 124, 63, 60, 117, 58, 61, 116, 33, 47, 62, 45, 114, 127, 59, 48, 56, 111, 126, 112, 49, 38, 55, 44, 53, 113, 122, 115, 120, 36, 102, 92, 108, 42, 109, 46, 107, 28, 17, 43, 110, 106, 94, 77, 41, 26, 25, 85, 105, 103, 83, 86, 89, 99, 39, 32, 87, 95, 19, 35, 100, 30, 37, 104, 27, 101, 31, 11, 22, 97, 21, 96, 34, 24, 98, 81, 13, 90, 29, 93, 69, 15, 2, 88, 84, 71, 0, 91, 79, 20, 16, 3, 75, 82, 1, 5, 9, 23, 67, 78, 65, 64, 73, 18, 66, 7, 80, 4, 14, 10, 74, 68, 72, 12, 8, 6, 70, 76], [50, 40, 121, 57, 51, 118, 54, 33, 114, 125, 52, 119, 123, 45, 44, 63, 58, 60, 116, 117, 127, 124, 112, 62, 49, 47, 59, 126, 61, 36, 113, 56, 48, 53, 94, 120, 122, 111, 55, 46, 115, 28, 17, 107, 26, 109, 110, 92, 25, 41, 108, 38, 43, 77, 99, 30, 42, 89, 102, 106, 85, 105, 86, 83, 19, 95, 87, 100, 24, 103, 21, 27, 31, 35, 39, 104, 101, 22, 37, 32, 98, 97, 11, 90, 84, 34, 2, 81, 96, 16, 13, 15, 69, 64, 93, 88, 82, 78, 0, 29, 79, 5, 66, 73, 23, 1, 65, 91, 67, 3, 75, 20, 7, 72, 9, 12, 18, 4, 14, 71, 68, 74, 10, 80, 6, 8, 70, 76], [121, 40, 50, 114, 125, 51, 54, 118, 57, 33, 63, 45, 52, 123, 119, 60, 117, 127, 58, 44, 116, 124, 56, 112, 59, 126, 62, 108, 61, 48, 113, 28, 46, 53, 120, 49, 122, 47, 55, 111, 115, 110, 17, 102, 26, 92, 42, 107, 109, 36, 94, 25, 38, 99, 106, 85, 43, 41, 105, 89, 95, 100, 30, 77, 87, 86, 19, 83, 103, 21, 35, 39, 104, 24, 37, 101, 96, 32, 22, 27, 97, 98, 34, 31, 81, 11, 13, 84, 2, 82, 88, 90, 64, 73, 29, 93, 16, 0, 15, 79, 9, 78, 1, 91, 69, 65, 23, 14, 5, 75, 66, 3, 4, 7, 67, 20, 71, 12, 18, 72, 10, 74, 68, 6, 70, 8, 80, 76], [40, 121, 50, 57, 82, 114, 78, 12, 33, 27, 16, 25, 99, 118, 23, 38, 29, 74, 19, 86, 6, 10, 93, 18, 36, 89, 72, 94, 51, 14, 70, 76, 104, 8, 28, 20, 84, 63, 102, 88, 90, 30, 95, 68, 54, 21, 80, 58, 119, 4, 55, 87, 91, 22, 125, 9, 15, 85, 49, 31, 1, 81, 96, 26, 62, 47, 42, 73, 24, 7, 127, 66, 116, 83, 92, 3, 52, 123, 39, 11, 32, 13, 115, 2, 124, 60, 64, 44, 98, 34, 117, 17, 105, 61, 79, 35, 126, 53, 101, 112, 48, 56, 5, 122, 45, 75, 77, 109, 71, 107, 67, 120, 113, 106, 37, 108, 43, 110, 59, 41, 46, 111, 100, 103, 69, 97, 0, 65], [42, 54, 32, 35, 123, 99, 106, 50, 89, 85, 27, 87, 16, 23, 58, 29, 96, 18, 78, 36, 57, 40, 98, 83, 103, 116, 107, 117, 7, 84, 62, 31, 17, 108, 61, 48, 60, 82, 44, 12, 47, 37, 105, 95, 100, 77, 26, 21, 125, 74, 76, 118, 46, 55, 45, 41, 90, 9, 114, 10, 20, 115, 94, 113, 101, 119, 92, 127, 110, 19, 124, 112, 81, 97, 122, 109, 126, 102, 49, 28, 24, 121, 59, 75, 104, 56, 22, 70, 88, 79, 43, 120, 34, 68, 33, 51, 25, 53, 63, 93, 15, 8, 91, 30, 52, 86, 39, 111, 80, 38, 4, 11, 14, 0, 72, 66, 13, 3, 69, 71, 5, 73, 64, 67, 6, 65, 1, 2], [54, 42, 32, 89, 117, 99, 123, 85, 87, 23, 29, 58, 35, 106, 96, 28, 50, 121, 55, 44, 36, 60, 15, 111, 33, 101, 57, 40, 46, 82, 122, 52, 126, 51, 112, 53, 31, 47, 10, 16, 100, 78, 103, 104, 41, 18, 125, 119, 45, 110, 109, 38, 83, 114, 95, 81, 17, 105, 115, 39, 118, 21, 12, 25, 113, 56, 76, 127, 19, 124, 48, 116, 34, 107, 37, 14, 27, 8, 98, 90, 108, 91, 49, 77, 63, 62, 79, 59, 102, 43, 120, 97, 30, 61, 69, 93, 84, 80, 92, 24, 72, 26, 94, 86, 74, 22, 88, 6, 67, 20, 75, 66, 71, 13, 11, 7, 5, 3, 70, 65, 73, 64, 68, 2, 4, 9, 1, 0], [54, 42, 32, 99, 35, 29, 89, 87, 96, 115, 23, 85, 36, 106, 95, 27, 110, 62, 105, 101, 18, 61, 123, 119, 94, 44, 45, 50, 83, 82, 117, 118, 63, 59, 48, 41, 86, 113, 28, 40, 121, 112, 55, 58, 31, 93, 120, 47, 114, 92, 103, 109, 125, 107, 37, 127, 122, 104, 57, 53, 108, 34, 60, 100, 126, 38, 52, 49, 16, 51, 20, 43, 116, 15, 124, 102, 90, 46, 33, 56, 22, 39, 26, 97, 12, 77, 98, 78, 84, 10, 25, 111, 30, 91, 81, 17, 24, 13, 75, 88, 14, 19, 76, 21, 4, 6, 8, 72, 74, 67, 7, 69, 79, 73, 1, 68, 5, 80, 70, 9, 11, 0, 66, 71, 64, 3, 65, 2], [42, 54, 32, 85, 35, 123, 106, 99, 89, 23, 16, 87, 76, 78, 18, 12, 50, 48, 27, 84, 29, 125, 36, 11, 14, 6, 72, 117, 10, 81, 4, 115, 31, 21, 80, 51, 25, 17, 8, 116, 28, 26, 101, 111, 79, 104, 83, 77, 3, 74, 109, 55, 92, 82, 2, 19, 9, 73, 60, 88, 57, 62, 1, 98, 69, 86, 64, 124, 97, 103, 119, 33, 66, 37, 44, 90, 93, 49, 102, 40, 105, 108, 30, 95, 114, 7, 110, 96, 67, 100, 41, 94, 75, 46, 56, 15, 127, 24, 126, 34, 38, 107, 53, 70, 63, 20, 118, 91, 43, 5, 112, 22, 39, 47, 52, 13, 45, 58, 71, 120, 122, 61, 59, 113, 121, 68, 65, 0], [117, 38, 120, 97, 25, 58, 63, 93, 89, 83, 116, 52, 53, 121, 55, 49, 102, 39, 86, 112, 115, 75, 122, 126, 127, 113, 119, 57, 125, 123, 50, 109, 47, 21, 48, 60, 114, 118, 46, 62, 20, 84, 41, 88, 105, 61, 59, 111, 110, 51, 44, 73, 56, 29, 124, 17, 100, 54, 104, 45, 80, 106, 42, 107, 108, 103, 40, 43, 22, 87, 16, 101, 37, 24, 26, 98, 99, 34, 36, 35, 15, 95, 18, 96, 31, 77, 94, 19, 23, 71, 32, 30, 33, 92, 4, 28, 79, 91, 14, 81, 90, 13, 11, 85, 27, 68, 1, 7, 82, 72, 3, 74, 70, 65, 64, 10, 76, 69, 66, 78, 9, 5, 0, 2, 8, 12, 67, 6], [38, 120, 117, 63, 97, 93, 17, 83, 25, 89, 86, 20, 29, 88, 102, 14, 58, 76, 74, 121, 65, 72, 15, 53, 116, 77, 71, 73, 49, 64, 41, 126, 122, 60, 100, 57, 4, 91, 46, 55, 59, 22, 11, 119, 123, 98, 94, 50, 81, 127, 115, 34, 10, 26, 39, 66, 104, 7, 47, 111, 109, 125, 16, 113, 52, 42, 118, 19, 56, 48, 51, 112, 90, 54, 114, 61, 44, 45, 84, 79, 62, 96, 24, 28, 35, 37, 78, 82, 2, 103, 87, 70, 108, 31, 75, 101, 124, 99, 106, 67, 40, 105, 32, 95, 21, 30, 36, 80, 5, 43, 27, 85, 110, 92, 12, 9, 13, 69, 1, 23, 33, 68, 107, 8, 3, 18, 6, 0], [38, 120, 117, 97, 58, 25, 83, 86, 93, 63, 73, 89, 17, 102, 29, 121, 15, 53, 88, 3, 14, 49, 75, 22, 126, 57, 20, 71, 19, 109, 41, 76, 77, 113, 116, 55, 127, 84, 46, 111, 79, 72, 52, 119, 11, 4, 47, 122, 65, 118, 112, 39, 125, 87, 115, 51, 60, 62, 50, 59, 123, 74, 70, 45, 21, 80, 56, 81, 9, 44, 114, 61, 16, 48, 42, 104, 10, 82, 100, 24, 108, 67, 124, 78, 34, 105, 103, 2, 85, 110, 33, 90, 23, 54, 7, 107, 40, 8, 37, 98, 35, 91, 28, 101, 43, 36, 0, 66, 99, 13, 69, 26, 12, 94, 18, 96, 1, 106, 68, 92, 31, 95, 6, 27, 30, 32, 5, 64], [38, 117, 120, 97, 63, 83, 70, 93, 25, 15, 73, 89, 3, 86, 58, 53, 41, 14, 17, 88, 80, 74, 29, 109, 78, 77, 75, 121, 57, 84, 65, 22, 55, 111, 115, 46, 49, 19, 79, 105, 0, 102, 118, 98, 122, 76, 42, 126, 106, 116, 119, 52, 125, 72, 4, 20, 107, 113, 127, 50, 56, 21, 54, 104, 62, 61, 47, 60, 9, 6, 123, 45, 59, 110, 11, 10, 44, 39, 51, 26, 114, 48, 12, 103, 2, 112, 81, 124, 82, 85, 8, 66, 108, 67, 43, 87, 71, 24, 40, 16, 36, 69, 34, 101, 94, 1, 23, 31, 7, 37, 95, 30, 90, 99, 28, 91, 100, 5, 32, 18, 35, 27, 68, 92, 64, 96, 13, 33], [122, 58, 53, 38, 125, 124, 121, 56, 87, 90, 92, 55, 59, 34, 111, 127, 76, 117, 63, 126, 30, 81, 123, 113, 62, 50, 120, 54, 52, 110, 49, 57, 61, 60, 114, 119, 48, 118, 51, 116, 115, 47, 100, 102, 19, 112, 8, 40, 107, 109, 91, 3, 22, 44, 89, 46, 83, 45, 21, 108, 41, 23, 2, 32, 80, 31, 103, 84, 105, 42, 43, 104, 106, 72, 36, 94, 39, 5, 37, 29, 35, 85, 28, 96, 33, 16, 24, 93, 25, 101, 70, 65, 95, 17, 14, 27, 26, 99, 10, 86, 0, 97, 13, 6, 78, 64, 12, 73, 9, 88, 98, 77, 1, 18, 15, 82, 4, 11, 75, 79, 66, 71, 20, 7, 68, 69, 67, 74], [53, 38, 125, 122, 58, 87, 121, 56, 30, 127, 34, 92, 90, 60, 50, 113, 63, 124, 55, 59, 123, 48, 120, 126, 32, 51, 41, 57, 114, 61, 62, 52, 118, 54, 119, 116, 47, 111, 115, 49, 21, 117, 76, 102, 110, 19, 108, 46, 85, 112, 40, 45, 107, 83, 106, 22, 109, 81, 44, 94, 37, 43, 80, 42, 100, 27, 28, 26, 84, 104, 105, 39, 103, 33, 91, 101, 23, 29, 36, 15, 31, 72, 97, 99, 16, 89, 77, 35, 86, 13, 78, 96, 98, 17, 93, 3, 0, 95, 65, 8, 5, 14, 18, 88, 2, 25, 69, 20, 73, 10, 75, 74, 24, 11, 9, 82, 12, 79, 71, 7, 6, 68, 70, 66, 4, 67, 64, 1], [122, 53, 38, 90, 121, 58, 56, 59, 127, 34, 87, 60, 55, 124, 92, 126, 123, 63, 117, 52, 30, 120, 48, 57, 111, 113, 61, 49, 110, 114, 125, 115, 62, 118, 119, 54, 50, 51, 47, 116, 102, 83, 22, 112, 46, 81, 19, 94, 32, 89, 109, 85, 23, 104, 108, 41, 14, 43, 84, 103, 107, 76, 31, 44, 105, 17, 106, 45, 36, 40, 42, 101, 25, 86, 28, 78, 100, 13, 82, 80, 37, 27, 39, 35, 98, 33, 26, 21, 29, 15, 97, 99, 88, 95, 8, 24, 91, 96, 16, 65, 93, 72, 18, 10, 12, 79, 69, 20, 3, 11, 6, 9, 77, 75, 7, 68, 74, 73, 71, 70, 67, 5, 0, 2, 66, 64, 4, 1], [125, 122, 53, 38, 58, 90, 121, 56, 51, 59, 127, 113, 120, 110, 81, 87, 55, 50, 63, 30, 61, 48, 126, 124, 123, 102, 49, 108, 57, 52, 117, 60, 114, 119, 34, 118, 62, 47, 111, 46, 109, 54, 115, 116, 19, 44, 106, 112, 92, 39, 42, 107, 43, 76, 103, 83, 22, 101, 32, 100, 94, 37, 105, 41, 21, 40, 45, 104, 36, 99, 14, 31, 35, 33, 8, 85, 91, 17, 26, 93, 24, 96, 29, 23, 97, 27, 84, 78, 95, 88, 28, 25, 98, 86, 80, 72, 10, 89, 74, 12, 15, 79, 13, 16, 18, 82, 11, 77, 3, 9, 73, 20, 70, 69, 75, 6, 5, 68, 7, 4, 71, 66, 0, 64, 65, 1, 67, 2], [102, 110, 97, 126, 46, 73, 21, 77, 82, 68, 11, 15, 26, 66, 90, 70, 53, 0, 7, 25, 87, 71, 85, 121, 31, 18, 1, 23, 29, 79, 100, 93, 4, 54, 118, 81, 88, 124, 10, 117, 55, 9, 57, 106, 78, 28, 64, 13, 80, 22, 20, 127, 2, 75, 32, 5, 108, 76, 83, 65, 69, 17, 12, 58, 6, 105, 92, 3, 89, 63, 16, 24, 116, 115, 86, 125, 84, 27, 72, 39, 40, 104, 123, 98, 91, 48, 59, 30, 62, 96, 49, 74, 52, 122, 37, 101, 67, 60, 14, 95, 112, 34, 19, 8, 51, 45, 99, 35, 94, 103, 109, 50, 61, 120, 56, 47, 36, 43, 107, 42, 44, 119, 111, 113, 41, 114, 33, 38], [102, 110, 126, 97, 21, 26, 82, 46, 15, 77, 73, 11, 87, 90, 93, 68, 7, 23, 12, 25, 45, 115, 40, 53, 78, 70, 8, 18, 44, 76, 85, 118, 84, 55, 66, 120, 54, 116, 109, 127, 0, 96, 101, 36, 95, 31, 37, 91, 13, 112, 79, 72, 125, 24, 83, 1, 98, 121, 124, 27, 41, 48, 19, 71, 32, 114, 17, 103, 52, 56, 75, 122, 58, 69, 14, 9, 123, 39, 89, 49, 42, 106, 59, 88, 63, 22, 81, 105, 6, 20, 51, 107, 62, 94, 80, 119, 117, 28, 34, 29, 3, 86, 60, 38, 47, 99, 61, 57, 108, 43, 92, 104, 30, 111, 10, 16, 100, 50, 35, 113, 74, 4, 33, 2, 5, 67, 65, 64], [102, 110, 126, 97, 118, 46, 26, 21, 15, 82, 87, 11, 31, 63, 95, 90, 93, 68, 49, 73, 77, 44, 41, 120, 53, 116, 88, 127, 106, 122, 123, 25, 54, 112, 22, 7, 101, 38, 84, 42, 56, 37, 12, 58, 52, 17, 51, 80, 29, 59, 109, 70, 23, 43, 92, 48, 119, 55, 57, 124, 66, 5, 113, 94, 45, 125, 111, 115, 0, 65, 104, 121, 117, 83, 10, 50, 108, 60, 105, 1, 62, 85, 100, 47, 96, 91, 69, 61, 36, 107, 27, 78, 98, 99, 28, 114, 40, 30, 39, 19, 8, 86, 76, 3, 32, 103, 79, 74, 16, 34, 71, 64, 67, 24, 72, 35, 89, 20, 18, 4, 81, 75, 33, 14, 2, 6, 13, 9], [102, 110, 97, 126, 21, 46, 26, 77, 82, 11, 15, 90, 73, 116, 87, 70, 92, 91, 68, 7, 25, 66, 37, 54, 32, 45, 85, 99, 62, 72, 49, 30, 83, 108, 41, 51, 19, 124, 18, 44, 111, 52, 71, 0, 113, 53, 122, 112, 127, 105, 104, 88, 24, 118, 48, 58, 100, 89, 40, 106, 50, 17, 123, 8, 75, 101, 16, 95, 36, 121, 96, 78, 63, 42, 93, 31, 94, 22, 38, 98, 3, 76, 115, 79, 28, 60, 55, 5, 107, 64, 33, 34, 114, 39, 35, 125, 109, 61, 119, 20, 57, 10, 47, 84, 6, 120, 29, 103, 59, 13, 43, 1, 86, 56, 81, 2, 117, 14, 9, 23, 27, 74, 80, 12, 67, 69, 4, 65], [111, 55, 101, 47, 25, 87, 113, 95, 21, 28, 48, 109, 58, 119, 82, 56, 59, 31, 62, 97, 91, 118, 94, 44, 54, 45, 122, 89, 33, 127, 126, 42, 116, 51, 115, 50, 120, 107, 61, 102, 80, 106, 23, 15, 76, 85, 123, 38, 53, 74, 121, 112, 32, 39, 92, 18, 57, 40, 110, 52, 105, 19, 104, 125, 117, 72, 124, 22, 41, 77, 63, 37, 49, 46, 30, 108, 16, 98, 29, 103, 99, 35, 60, 34, 93, 114, 26, 100, 14, 90, 84, 71, 96, 17, 36, 83, 69, 43, 86, 88, 27, 24, 75, 81, 20, 11, 13, 73, 79, 65, 9, 7, 78, 70, 3, 12, 67, 1, 5, 4, 8, 66, 10, 2, 68, 6, 0, 64], [111, 55, 101, 47, 25, 95, 21, 87, 28, 82, 33, 94, 91, 15, 74, 53, 76, 59, 62, 97, 22, 89, 72, 29, 114, 60, 56, 121, 49, 116, 107, 37, 92, 45, 93, 123, 38, 32, 11, 70, 43, 18, 110, 109, 67, 122, 98, 54, 113, 48, 103, 57, 23, 99, 115, 78, 102, 4, 69, 31, 80, 44, 51, 40, 83, 85, 105, 124, 36, 119, 50, 34, 86, 46, 66, 41, 13, 117, 77, 104, 120, 58, 26, 52, 108, 118, 42, 27, 71, 17, 61, 127, 63, 112, 96, 39, 125, 106, 84, 19, 90, 126, 79, 10, 81, 100, 30, 8, 7, 24, 12, 35, 73, 88, 20, 9, 75, 16, 64, 68, 5, 3, 1, 14, 65, 2, 6, 0], [111, 101, 55, 47, 25, 87, 21, 95, 15, 74, 82, 70, 4, 89, 76, 94, 64, 97, 67, 77, 28, 80, 113, 2, 114, 72, 84, 50, 73, 18, 0, 91, 58, 31, 126, 29, 79, 66, 11, 85, 6, 122, 23, 92, 40, 83, 16, 10, 71, 45, 3, 116, 107, 75, 41, 65, 5, 7, 86, 35, 27, 88, 13, 98, 93, 9, 14, 12, 121, 33, 8, 78, 19, 123, 17, 68, 96, 81, 69, 20, 24, 90, 26, 105, 34, 22, 117, 44, 30, 32, 51, 115, 46, 38, 99, 54, 37, 1, 62, 124, 60, 109, 127, 103, 100, 59, 61, 48, 39, 110, 108, 102, 53, 120, 125, 42, 36, 112, 52, 104, 56, 63, 106, 57, 119, 49, 118, 43], [111, 55, 101, 47, 25, 120, 87, 21, 121, 28, 95, 56, 116, 31, 82, 110, 53, 33, 123, 37, 126, 38, 46, 58, 50, 124, 104, 122, 115, 89, 60, 44, 62, 51, 97, 112, 59, 29, 118, 61, 57, 127, 76, 23, 63, 48, 94, 52, 45, 39, 107, 103, 113, 91, 49, 117, 114, 85, 80, 42, 109, 125, 105, 119, 32, 40, 108, 41, 98, 102, 99, 106, 18, 15, 90, 54, 30, 26, 84, 43, 34, 72, 77, 92, 35, 96, 86, 100, 93, 27, 36, 16, 74, 78, 20, 67, 17, 11, 83, 22, 79, 88, 24, 81, 13, 14, 69, 19, 71, 70, 9, 12, 65, 75, 7, 73, 8, 4, 64, 3, 66, 5, 10, 2, 0, 1, 68, 6], [115, 100, 51, 23, 91, 32, 96, 84, 86, 9, 77, 15, 81, 39, 71, 5, 11, 27, 20, 108, 49, 10, 94, 36, 24, 83, 22, 44, 68, 21, 67, 118, 17, 0, 64, 65, 57, 82, 126, 127, 121, 117, 114, 16, 54, 45, 110, 2, 87, 79, 28, 35, 95, 107, 33, 13, 12, 80, 73, 25, 101, 74, 3, 69, 19, 76, 40, 85, 116, 37, 18, 42, 75, 30, 112, 8, 90, 89, 104, 92, 72, 124, 70, 88, 122, 14, 41, 78, 48, 120, 52, 34, 99, 50, 62, 4, 93, 97, 102, 123, 47, 105, 29, 98, 63, 119, 125, 109, 60, 53, 113, 31, 66, 38, 6, 46, 103, 56, 26, 7, 59, 1, 43, 106, 55, 111, 61, 58], [115, 100, 51, 121, 36, 91, 32, 23, 86, 84, 39, 82, 77, 81, 127, 48, 96, 113, 15, 118, 33, 20, 29, 44, 63, 104, 55, 71, 116, 120, 45, 119, 123, 49, 37, 47, 108, 22, 102, 114, 41, 126, 107, 50, 42, 106, 46, 54, 35, 110, 124, 53, 125, 105, 27, 10, 43, 38, 52, 111, 112, 16, 59, 56, 60, 58, 57, 30, 61, 122, 40, 95, 11, 19, 18, 62, 99, 109, 97, 117, 66, 34, 101, 98, 64, 31, 103, 83, 17, 26, 92, 24, 94, 5, 68, 85, 87, 90, 12, 21, 93, 89, 28, 76, 73, 6, 88, 8, 25, 13, 79, 78, 2, 75, 9, 80, 14, 67, 74, 70, 72, 1, 69, 4, 0, 65, 7, 3], [115, 100, 51, 121, 36, 91, 23, 32, 84, 86, 103, 63, 82, 20, 114, 104, 44, 22, 77, 15, 48, 39, 117, 46, 107, 99, 55, 43, 127, 49, 116, 29, 34, 45, 81, 110, 31, 30, 40, 119, 61, 118, 37, 33, 57, 108, 38, 41, 112, 42, 120, 113, 54, 125, 101, 35, 111, 60, 87, 106, 96, 58, 109, 53, 59, 56, 52, 95, 123, 47, 98, 126, 92, 105, 62, 24, 90, 97, 102, 27, 50, 94, 124, 122, 28, 9, 93, 71, 26, 17, 16, 85, 74, 25, 88, 76, 18, 19, 14, 89, 79, 13, 21, 11, 83, 80, 78, 10, 6, 12, 66, 64, 7, 73, 1, 4, 0, 75, 5, 72, 8, 68, 70, 69, 3, 2, 67, 65], [115, 100, 51, 36, 23, 91, 121, 96, 84, 86, 81, 11, 32, 15, 118, 94, 48, 117, 39, 9, 50, 108, 116, 107, 102, 5, 20, 2, 119, 25, 105, 45, 77, 44, 0, 37, 113, 27, 63, 82, 58, 62, 68, 19, 57, 46, 47, 110, 120, 49, 114, 126, 122, 43, 112, 123, 99, 125, 127, 53, 59, 22, 54, 41, 61, 55, 24, 124, 109, 52, 104, 42, 95, 17, 60, 65, 40, 56, 33, 38, 106, 35, 111, 70, 101, 71, 103, 87, 88, 93, 64, 16, 26, 98, 30, 79, 34, 29, 73, 83, 10, 31, 97, 90, 72, 75, 69, 8, 28, 14, 89, 12, 67, 6, 92, 13, 21, 18, 66, 78, 85, 4, 1, 80, 76, 3, 74, 7], [115, 40, 122, 53, 93, 124, 49, 55, 58, 51, 52, 48, 120, 33, 59, 60, 56, 62, 127, 111, 125, 26, 44, 61, 54, 47, 50, 123, 63, 113, 43, 126, 114, 118, 121, 116, 107, 57, 119, 45, 46, 112, 117, 22, 19, 109, 42, 110, 90, 29, 108, 88, 84, 41, 101, 77, 99, 38, 104, 106, 105, 32, 94, 92, 95, 103, 39, 83, 36, 102, 37, 34, 81, 100, 89, 97, 35, 86, 16, 98, 96, 25, 82, 28, 87, 71, 27, 10, 31, 13, 21, 23, 91, 79, 30, 80, 17, 14, 7, 3, 24, 74, 15, 12, 78, 85, 2, 20, 64, 66, 0, 18, 9, 67, 5, 72, 11, 69, 65, 6, 1, 75, 73, 4, 68, 76, 8, 70], [53, 115, 40, 122, 124, 93, 49, 52, 55, 59, 51, 33, 60, 56, 62, 48, 111, 126, 61, 58, 113, 117, 120, 125, 43, 114, 50, 127, 54, 26, 121, 63, 47, 123, 109, 119, 57, 116, 118, 46, 45, 112, 90, 22, 108, 81, 107, 44, 110, 35, 29, 88, 37, 42, 41, 39, 105, 101, 106, 103, 92, 77, 99, 36, 104, 32, 34, 102, 95, 84, 100, 83, 38, 94, 19, 89, 97, 86, 98, 17, 25, 96, 31, 27, 16, 82, 13, 28, 20, 30, 15, 79, 24, 10, 91, 85, 14, 87, 21, 71, 80, 74, 78, 23, 72, 5, 7, 12, 66, 64, 11, 8, 0, 2, 1, 3, 75, 4, 67, 18, 69, 65, 68, 9, 73, 6, 70, 76], [40, 115, 53, 122, 88, 14, 33, 19, 82, 12, 22, 90, 80, 9, 92, 49, 28, 111, 91, 26, 30, 1, 55, 73, 93, 6, 31, 50, 124, 29, 25, 75, 20, 72, 27, 4, 95, 23, 17, 89, 69, 86, 78, 15, 96, 85, 32, 58, 10, 38, 117, 59, 76, 18, 51, 24, 34, 45, 98, 83, 70, 112, 21, 100, 104, 35, 127, 52, 16, 99, 105, 46, 56, 11, 123, 94, 79, 84, 42, 81, 106, 108, 61, 113, 121, 13, 8, 68, 109, 62, 114, 3, 120, 54, 110, 87, 116, 103, 101, 126, 37, 63, 125, 119, 39, 102, 71, 77, 48, 107, 36, 47, 43, 60, 118, 41, 57, 67, 7, 74, 44, 97, 5, 0, 66, 65, 2, 64], [122, 40, 53, 115, 33, 124, 93, 55, 52, 60, 49, 59, 26, 48, 62, 111, 61, 56, 120, 58, 125, 113, 101, 47, 127, 51, 54, 116, 126, 121, 119, 114, 50, 117, 123, 57, 63, 118, 109, 45, 84, 95, 39, 88, 112, 90, 43, 29, 46, 108, 22, 41, 107, 110, 44, 77, 34, 32, 19, 25, 42, 89, 94, 92, 106, 99, 100, 105, 83, 103, 37, 21, 81, 97, 102, 104, 35, 38, 24, 36, 98, 96, 86, 31, 28, 23, 17, 87, 27, 16, 91, 30, 2, 80, 3, 82, 69, 64, 13, 74, 1, 65, 14, 5, 0, 79, 66, 10, 7, 71, 67, 85, 15, 20, 78, 12, 11, 68, 9, 6, 72, 75, 4, 73, 8, 18, 70, 76]], "model.layers.22.self_attn.k_proj": [[104, 121, 50, 86, 57, 97, 92, 100, 54, 94, 51, 118, 114, 119, 25, 31, 124, 63, 125, 58, 17, 60, 117, 123, 102, 62, 126, 116, 52, 127, 48, 85, 55, 83, 47, 53, 49, 56, 112, 122, 44, 120, 59, 113, 111, 115, 27, 105, 61, 46, 107, 108, 35, 110, 106, 42, 19, 109, 43, 77, 39, 41, 45, 16, 80, 103, 12, 15, 37, 78, 93, 33, 99, 34, 38, 87, 26, 75, 79, 98, 101, 91, 32, 76, 84, 11, 23, 72, 96, 82, 90, 7, 95, 36, 5, 28, 10, 20, 24, 13, 30, 29, 40, 74, 6, 88, 73, 71, 18, 89, 21, 9, 4, 3, 22, 8, 14, 81, 70, 1, 68, 0, 67, 69, 2, 64, 66, 65], [106, 54, 89, 96, 123, 50, 23, 85, 16, 17, 99, 42, 78, 82, 76, 48, 124, 27, 114, 30, 29, 21, 57, 72, 19, 34, 112, 74, 12, 93, 14, 108, 10, 18, 70, 55, 77, 100, 60, 47, 68, 127, 125, 3, 35, 118, 7, 11, 53, 115, 46, 8, 79, 104, 110, 95, 59, 20, 49, 15, 111, 109, 119, 41, 98, 38, 28, 105, 44, 65, 13, 83, 101, 107, 5, 126, 9, 81, 113, 33, 84, 37, 117, 120, 43, 116, 52, 103, 92, 56, 66, 31, 61, 63, 90, 122, 102, 40, 45, 62, 39, 121, 86, 97, 24, 51, 94, 87, 36, 75, 64, 91, 58, 88, 26, 22, 25, 6, 69, 80, 32, 71, 73, 67, 4, 2, 1, 0], [120, 117, 102, 33, 86, 29, 25, 58, 17, 83, 49, 121, 0, 50, 88, 73, 118, 14, 126, 65, 119, 53, 3, 122, 15, 116, 57, 46, 47, 125, 103, 55, 123, 52, 115, 113, 111, 38, 4, 62, 75, 112, 44, 48, 51, 114, 127, 61, 105, 45, 7, 59, 56, 70, 107, 36, 106, 41, 124, 60, 77, 74, 108, 109, 84, 63, 80, 110, 42, 21, 104, 94, 16, 40, 20, 18, 72, 54, 43, 98, 39, 90, 13, 5, 89, 37, 87, 34, 76, 91, 100, 26, 99, 2, 68, 31, 35, 92, 28, 101, 95, 64, 71, 1, 10, 67, 23, 96, 69, 32, 8, 82, 27, 85, 30, 12, 93, 79, 11, 66, 78, 24, 81, 19, 6, 9, 97, 22], [122, 53, 102, 22, 98, 57, 121, 117, 62, 56, 55, 110, 50, 116, 28, 59, 123, 115, 63, 124, 120, 126, 52, 48, 60, 58, 118, 114, 113, 127, 119, 54, 51, 61, 112, 49, 111, 33, 47, 109, 44, 94, 104, 108, 96, 46, 107, 90, 45, 40, 105, 34, 43, 125, 41, 80, 37, 42, 106, 78, 103, 36, 92, 101, 39, 16, 82, 100, 85, 13, 95, 19, 86, 83, 97, 99, 35, 17, 73, 87, 24, 25, 93, 31, 11, 29, 27, 81, 76, 32, 38, 30, 68, 84, 23, 91, 21, 15, 26, 88, 10, 71, 5, 14, 18, 8, 79, 75, 7, 67, 89, 72, 66, 20, 77, 70, 9, 69, 74, 1, 12, 0, 2, 65, 6, 3, 64, 4], [46, 126, 38, 33, 110, 82, 21, 26, 15, 77, 73, 0, 11, 70, 66, 7, 68, 64, 87, 25, 53, 3, 93, 65, 121, 19, 17, 95, 54, 2, 84, 125, 58, 10, 80, 12, 118, 88, 59, 115, 63, 48, 22, 1, 124, 45, 94, 55, 14, 105, 71, 16, 49, 112, 74, 117, 40, 8, 60, 104, 44, 92, 101, 116, 123, 51, 52, 67, 62, 6, 113, 42, 120, 96, 127, 69, 41, 43, 56, 39, 5, 119, 109, 34, 83, 114, 23, 103, 111, 122, 107, 108, 57, 35, 37, 32, 50, 36, 72, 99, 28, 61, 47, 102, 29, 106, 24, 100, 81, 89, 20, 86, 98, 30, 27, 78, 31, 91, 4, 9, 76, 79, 97, 75, 13, 85, 90, 18], [47, 111, 37, 55, 87, 21, 72, 82, 15, 25, 76, 31, 74, 66, 70, 107, 77, 69, 64, 4, 94, 80, 65, 119, 97, 102, 28, 116, 78, 115, 56, 91, 40, 44, 58, 123, 118, 53, 38, 43, 110, 121, 39, 120, 127, 62, 7, 92, 83, 122, 42, 60, 113, 52, 1, 105, 114, 45, 124, 126, 117, 104, 108, 112, 59, 106, 125, 22, 34, 63, 51, 48, 73, 17, 61, 93, 46, 54, 57, 50, 90, 98, 49, 103, 32, 41, 109, 36, 71, 20, 33, 86, 35, 29, 99, 19, 81, 100, 11, 30, 26, 13, 84, 96, 85, 24, 2, 14, 27, 68, 101, 88, 16, 23, 75, 10, 3, 9, 5, 12, 67, 8, 6, 89, 79, 0, 95, 18], [51, 115, 36, 86, 91, 96, 84, 23, 81, 77, 15, 71, 64, 121, 94, 82, 11, 117, 1, 49, 44, 113, 119, 68, 57, 9, 48, 66, 112, 39, 47, 46, 106, 45, 108, 67, 120, 122, 127, 50, 53, 125, 5, 101, 63, 24, 104, 58, 59, 41, 114, 109, 61, 107, 126, 118, 62, 56, 55, 3, 60, 103, 123, 70, 105, 54, 10, 116, 43, 124, 88, 52, 26, 0, 110, 2, 19, 97, 38, 76, 42, 99, 111, 100, 31, 25, 37, 74, 80, 40, 85, 89, 78, 33, 93, 16, 65, 30, 98, 6, 69, 34, 12, 102, 72, 92, 14, 21, 29, 95, 35, 8, 90, 28, 83, 4, 18, 73, 32, 7, 27, 87, 13, 79, 75, 22, 20, 17], [104, 115, 53, 22, 122, 97, 51, 29, 124, 111, 59, 56, 52, 26, 28, 55, 49, 62, 126, 50, 54, 118, 117, 113, 61, 60, 127, 116, 121, 119, 47, 58, 82, 63, 12, 19, 83, 120, 114, 48, 125, 110, 123, 35, 91, 112, 16, 109, 81, 88, 44, 57, 45, 6, 107, 105, 77, 43, 15, 93, 46, 32, 89, 94, 108, 17, 42, 41, 84, 31, 36, 14, 80, 37, 33, 78, 101, 103, 96, 24, 102, 79, 30, 106, 39, 27, 95, 23, 38, 9, 99, 98, 34, 90, 13, 87, 100, 69, 75, 40, 74, 25, 85, 21, 11, 18, 92, 72, 66, 20, 7, 68, 71, 73, 70, 0, 64, 76, 10, 65, 86, 4, 67, 1, 8, 2, 3, 5]], "model.layers.22.self_attn.qk_proj": [[115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 58, 22, 123, 118, 125, 25, 87, 97, 85, 49, 40, 27, 56, 36, 21, 90, 86, 23, 93, 18, 62, 63, 127, 79, 60, 101, 59, 96, 15, 33, 82, 77, 114, 124, 99, 13, 116, 81, 48, 17, 26, 112, 52, 61, 19, 100, 29, 44, 83, 119, 11, 37, 113, 28, 84, 32, 95, 92, 45, 9, 30, 94, 73, 91, 31, 75, 7, 108, 20, 16, 109, 41, 14, 80, 78, 76, 70, 12, 39, 35, 4, 0, 71, 34, 74, 10, 105, 107, 8, 98, 64, 43, 68, 88, 103, 67, 3, 2, 66, 6, 24, 65, 1, 72, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 123, 58, 89, 22, 104, 125, 87, 97, 118, 25, 36, 62, 85, 56, 49, 23, 93, 27, 40, 86, 21, 90, 114, 101, 79, 59, 116, 82, 99, 113, 52, 18, 13, 63, 100, 96, 127, 77, 15, 33, 29, 60, 19, 81, 124, 17, 112, 73, 61, 26, 119, 44, 48, 92, 28, 91, 11, 109, 32, 94, 107, 45, 20, 105, 70, 37, 83, 41, 75, 9, 84, 76, 108, 0, 16, 14, 12, 78, 95, 4, 7, 30, 39, 31, 35, 98, 80, 74, 43, 103, 10, 71, 64, 34, 2, 68, 24, 88, 8, 1, 67, 3, 66, 72, 65, 6, 5, 69], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 57, 110, 55, 106, 42, 102, 38, 58, 104, 22, 25, 118, 123, 89, 97, 93, 27, 56, 23, 87, 59, 36, 21, 40, 62, 85, 86, 125, 49, 63, 116, 15, 90, 114, 33, 60, 18, 82, 101, 77, 79, 100, 99, 96, 13, 52, 48, 32, 127, 124, 113, 81, 17, 119, 45, 92, 112, 29, 44, 107, 26, 73, 19, 28, 37, 11, 20, 0, 108, 94, 41, 91, 30, 61, 83, 84, 70, 109, 71, 9, 75, 95, 34, 7, 98, 35, 80, 105, 76, 12, 78, 31, 4, 39, 16, 14, 43, 64, 10, 103, 74, 68, 88, 66, 72, 2, 67, 3, 65, 1, 24, 69, 6, 8, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 57, 110, 106, 55, 42, 38, 102, 58, 123, 104, 25, 22, 89, 125, 56, 62, 21, 87, 63, 59, 27, 97, 93, 86, 118, 85, 23, 15, 114, 49, 82, 18, 77, 40, 90, 36, 100, 124, 101, 127, 13, 52, 33, 81, 79, 45, 116, 48, 99, 96, 119, 113, 112, 32, 60, 17, 107, 11, 37, 29, 84, 61, 83, 70, 26, 73, 12, 44, 19, 9, 92, 95, 28, 64, 41, 105, 109, 91, 7, 71, 98, 14, 94, 75, 78, 43, 16, 80, 30, 76, 31, 4, 108, 20, 0, 74, 68, 34, 66, 35, 72, 10, 103, 67, 39, 3, 24, 65, 88, 2, 6, 1, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 22, 89, 97, 104, 118, 49, 63, 25, 21, 86, 87, 85, 62, 27, 125, 23, 93, 40, 59, 82, 36, 114, 56, 15, 79, 60, 18, 124, 48, 77, 33, 81, 45, 90, 116, 100, 99, 13, 119, 52, 127, 96, 44, 101, 19, 26, 112, 61, 17, 92, 11, 29, 73, 91, 9, 113, 75, 37, 71, 20, 32, 31, 95, 83, 28, 64, 16, 94, 14, 78, 7, 80, 41, 12, 84, 76, 0, 109, 98, 108, 39, 10, 43, 70, 34, 105, 107, 72, 74, 4, 68, 35, 30, 6, 2, 67, 24, 66, 103, 1, 3, 88, 5, 69, 65, 8], [115, 53, 122, 117, 51, 120, 47, 111, 121, 126, 50, 54, 46, 57, 110, 106, 55, 42, 102, 38, 123, 22, 58, 86, 27, 89, 85, 125, 97, 21, 25, 87, 104, 62, 93, 40, 23, 56, 49, 63, 82, 15, 77, 33, 59, 36, 60, 118, 18, 81, 13, 119, 90, 79, 114, 124, 99, 96, 100, 127, 116, 11, 48, 17, 37, 92, 84, 113, 32, 29, 19, 83, 26, 52, 41, 101, 16, 112, 31, 73, 28, 75, 94, 91, 9, 44, 20, 61, 95, 14, 12, 98, 45, 80, 76, 7, 109, 108, 78, 74, 0, 64, 10, 105, 71, 68, 107, 30, 72, 39, 6, 35, 43, 4, 34, 24, 2, 103, 65, 70, 3, 67, 88, 1, 66, 5, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 54, 121, 50, 46, 57, 110, 106, 55, 42, 102, 38, 22, 123, 87, 89, 86, 25, 85, 104, 23, 27, 21, 58, 40, 97, 93, 60, 63, 36, 15, 62, 90, 49, 82, 96, 79, 125, 33, 18, 118, 77, 114, 13, 99, 116, 32, 17, 59, 48, 81, 113, 119, 52, 44, 100, 19, 101, 56, 84, 127, 29, 112, 94, 11, 124, 26, 92, 91, 45, 83, 28, 107, 37, 76, 73, 109, 20, 12, 61, 95, 16, 9, 78, 80, 75, 14, 41, 31, 43, 39, 105, 34, 72, 108, 7, 35, 71, 30, 10, 6, 64, 74, 98, 0, 24, 88, 103, 68, 4, 1, 66, 3, 2, 67, 70, 69, 65, 5, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 57, 110, 55, 106, 42, 38, 102, 22, 89, 87, 97, 25, 63, 85, 21, 58, 36, 60, 23, 27, 104, 118, 93, 90, 86, 40, 123, 18, 49, 62, 59, 96, 33, 15, 125, 82, 112, 116, 119, 48, 79, 77, 100, 113, 101, 17, 99, 52, 114, 61, 56, 32, 13, 124, 81, 44, 26, 91, 29, 20, 28, 127, 19, 37, 84, 108, 80, 35, 12, 73, 83, 11, 109, 92, 94, 107, 39, 34, 95, 43, 45, 14, 6, 9, 16, 88, 76, 78, 75, 41, 105, 30, 7, 31, 64, 72, 4, 74, 71, 10, 98, 0, 24, 103, 68, 66, 65, 2, 3, 67, 69, 1, 5, 70, 8], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 89, 58, 22, 125, 25, 62, 36, 27, 93, 87, 21, 23, 97, 123, 118, 90, 104, 85, 63, 49, 59, 114, 82, 86, 112, 127, 40, 15, 77, 96, 116, 56, 33, 79, 101, 124, 60, 61, 52, 18, 48, 32, 100, 99, 119, 19, 107, 13, 28, 113, 81, 9, 17, 109, 45, 29, 26, 37, 44, 11, 20, 41, 83, 91, 92, 84, 35, 94, 108, 12, 78, 105, 39, 80, 14, 34, 31, 76, 73, 95, 30, 6, 98, 75, 16, 7, 103, 43, 74, 72, 68, 64, 0, 88, 71, 4, 10, 24, 66, 65, 2, 1, 3, 67, 5, 70, 69, 8], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 102, 38, 104, 58, 123, 89, 22, 125, 62, 25, 87, 118, 27, 63, 97, 23, 119, 85, 36, 93, 86, 49, 21, 90, 114, 52, 56, 59, 40, 61, 96, 99, 124, 101, 79, 116, 127, 18, 60, 77, 82, 13, 48, 15, 100, 33, 45, 107, 11, 32, 91, 109, 41, 9, 94, 17, 28, 81, 112, 19, 37, 113, 92, 44, 26, 29, 20, 83, 108, 84, 73, 35, 75, 105, 31, 14, 78, 7, 12, 39, 43, 16, 6, 34, 30, 64, 95, 0, 76, 71, 98, 80, 103, 74, 68, 4, 10, 70, 2, 72, 3, 66, 1, 8, 67, 24, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 104, 125, 89, 22, 25, 63, 97, 27, 123, 86, 87, 62, 21, 49, 118, 85, 23, 119, 100, 93, 82, 59, 114, 101, 77, 18, 52, 90, 15, 79, 116, 40, 33, 99, 56, 36, 127, 61, 13, 96, 48, 113, 17, 124, 83, 112, 81, 29, 9, 60, 41, 94, 28, 37, 32, 26, 109, 92, 16, 64, 44, 91, 20, 11, 12, 84, 7, 45, 75, 107, 70, 19, 74, 73, 95, 76, 31, 71, 78, 80, 14, 39, 98, 68, 10, 4, 35, 0, 108, 103, 105, 43, 8, 34, 6, 66, 30, 2, 3, 24, 72, 67, 1, 88, 65, 69, 5], [115, 53, 122, 51, 117, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 22, 89, 123, 86, 25, 21, 104, 58, 40, 87, 27, 97, 36, 85, 118, 23, 90, 82, 93, 79, 62, 125, 49, 124, 63, 77, 18, 116, 96, 59, 15, 33, 119, 61, 114, 100, 99, 127, 17, 13, 112, 32, 113, 81, 101, 19, 26, 83, 60, 94, 28, 44, 84, 92, 20, 48, 11, 37, 29, 9, 73, 56, 52, 12, 75, 91, 41, 95, 14, 70, 31, 76, 80, 78, 10, 16, 7, 45, 107, 64, 98, 0, 74, 35, 34, 105, 71, 4, 109, 108, 68, 39, 43, 30, 88, 24, 8, 103, 66, 3, 1, 6, 2, 67, 72, 69, 65, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 102, 42, 38, 89, 22, 123, 87, 25, 118, 36, 23, 93, 97, 21, 86, 27, 58, 40, 85, 104, 90, 82, 125, 96, 116, 63, 49, 33, 59, 100, 99, 56, 114, 112, 124, 18, 61, 119, 60, 15, 113, 29, 101, 13, 62, 32, 79, 77, 26, 81, 28, 94, 19, 44, 17, 91, 92, 20, 83, 127, 48, 37, 107, 41, 109, 84, 9, 75, 105, 95, 39, 11, 31, 52, 14, 16, 73, 35, 108, 45, 12, 70, 30, 78, 80, 34, 64, 76, 10, 43, 71, 98, 7, 8, 68, 24, 88, 0, 103, 4, 74, 65, 66, 2, 3, 1, 67, 69, 5, 72, 6], [115, 122, 53, 117, 51, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 123, 22, 104, 25, 86, 125, 89, 21, 118, 59, 23, 27, 85, 119, 40, 87, 90, 93, 36, 82, 124, 116, 77, 60, 63, 49, 112, 56, 114, 62, 97, 100, 79, 127, 33, 96, 15, 113, 52, 61, 18, 81, 9, 37, 101, 99, 32, 29, 17, 11, 13, 44, 48, 70, 75, 31, 109, 73, 94, 0, 41, 20, 19, 84, 45, 91, 83, 95, 92, 26, 28, 30, 107, 98, 14, 12, 71, 7, 35, 108, 80, 16, 64, 105, 78, 8, 76, 39, 68, 43, 10, 4, 74, 103, 34, 2, 1, 24, 66, 67, 65, 88, 3, 5, 72, 6, 69], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 106, 57, 55, 42, 38, 102, 58, 22, 104, 97, 86, 25, 89, 87, 27, 123, 21, 125, 40, 36, 62, 23, 118, 82, 79, 93, 119, 63, 85, 77, 90, 49, 112, 60, 114, 99, 15, 56, 18, 81, 127, 33, 124, 96, 13, 100, 61, 59, 101, 113, 17, 75, 83, 32, 48, 116, 109, 9, 91, 45, 29, 52, 37, 26, 92, 19, 28, 71, 76, 84, 14, 11, 31, 44, 95, 73, 12, 107, 20, 39, 80, 7, 70, 41, 78, 94, 35, 64, 30, 0, 43, 105, 8, 74, 108, 34, 10, 16, 98, 4, 103, 68, 2, 24, 6, 3, 67, 88, 66, 1, 65, 72, 5, 69], [115, 122, 53, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 106, 55, 57, 42, 38, 102, 123, 22, 104, 86, 21, 25, 40, 125, 58, 89, 85, 87, 97, 27, 23, 63, 93, 36, 82, 90, 77, 79, 49, 18, 56, 100, 60, 118, 59, 15, 116, 33, 81, 62, 61, 127, 114, 113, 52, 119, 48, 96, 13, 101, 112, 124, 99, 29, 41, 17, 26, 12, 32, 45, 75, 73, 109, 83, 84, 76, 94, 19, 91, 92, 44, 9, 64, 28, 31, 7, 16, 37, 20, 78, 11, 34, 14, 10, 8, 95, 35, 39, 68, 70, 74, 108, 103, 105, 107, 30, 43, 71, 98, 80, 0, 24, 4, 6, 66, 88, 65, 67, 1, 2, 3, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 126, 111, 121, 54, 50, 46, 110, 55, 57, 106, 42, 102, 38, 104, 89, 22, 58, 86, 123, 87, 40, 25, 85, 23, 90, 36, 114, 93, 125, 97, 21, 27, 63, 96, 118, 82, 79, 100, 33, 59, 49, 60, 56, 18, 77, 116, 15, 81, 113, 29, 124, 61, 94, 99, 17, 84, 19, 26, 44, 101, 127, 48, 28, 119, 62, 112, 32, 83, 13, 41, 31, 52, 75, 20, 91, 73, 37, 92, 30, 11, 45, 16, 35, 39, 78, 76, 109, 98, 12, 80, 108, 107, 9, 95, 6, 34, 43, 103, 71, 0, 10, 105, 7, 14, 24, 8, 88, 68, 64, 4, 74, 1, 2, 66, 70, 65, 3, 67, 5, 72, 69], [115, 53, 122, 51, 117, 120, 47, 126, 111, 54, 121, 50, 46, 110, 57, 55, 106, 42, 38, 102, 25, 58, 22, 86, 104, 87, 89, 40, 123, 93, 27, 114, 36, 90, 21, 97, 125, 118, 85, 79, 63, 23, 59, 96, 60, 18, 99, 82, 77, 62, 112, 15, 49, 33, 116, 100, 26, 124, 81, 48, 56, 101, 29, 13, 61, 17, 127, 52, 32, 75, 19, 37, 113, 44, 107, 109, 45, 73, 92, 20, 91, 119, 9, 28, 84, 94, 6, 83, 78, 16, 12, 95, 41, 30, 31, 108, 11, 39, 0, 105, 7, 35, 34, 80, 71, 76, 14, 98, 10, 64, 68, 4, 103, 43, 74, 88, 66, 67, 8, 2, 65, 24, 3, 1, 72, 69, 70, 5], [115, 53, 122, 117, 51, 120, 47, 126, 111, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 63, 104, 89, 125, 22, 25, 114, 97, 123, 58, 21, 40, 86, 36, 87, 62, 48, 27, 82, 23, 85, 90, 93, 127, 112, 99, 49, 118, 101, 79, 124, 60, 96, 77, 59, 52, 116, 100, 13, 61, 15, 18, 33, 56, 26, 29, 17, 41, 44, 32, 75, 28, 37, 84, 45, 109, 119, 91, 73, 81, 113, 94, 83, 107, 19, 6, 105, 9, 95, 12, 11, 39, 92, 78, 7, 108, 34, 31, 30, 35, 98, 43, 76, 20, 0, 64, 14, 103, 10, 71, 16, 4, 80, 74, 68, 67, 24, 88, 2, 72, 65, 8, 66, 1, 3, 70, 5, 69], [115, 122, 53, 117, 51, 120, 47, 126, 111, 121, 50, 54, 46, 110, 57, 106, 55, 42, 102, 38, 104, 89, 22, 123, 97, 58, 125, 93, 85, 87, 40, 21, 25, 86, 63, 118, 36, 23, 27, 62, 60, 56, 90, 114, 116, 124, 49, 96, 77, 101, 100, 18, 15, 33, 13, 79, 82, 99, 127, 113, 52, 59, 92, 81, 48, 26, 41, 112, 119, 29, 94, 44, 75, 28, 83, 17, 9, 45, 61, 91, 73, 32, 11, 84, 37, 107, 20, 19, 109, 30, 31, 78, 12, 6, 35, 95, 105, 76, 64, 39, 14, 0, 10, 80, 71, 16, 7, 34, 43, 98, 103, 68, 108, 66, 74, 72, 4, 67, 88, 1, 24, 2, 3, 65, 8, 70, 69, 5], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 89, 22, 104, 118, 25, 40, 123, 97, 63, 86, 87, 85, 27, 93, 23, 116, 21, 60, 36, 49, 90, 125, 56, 79, 82, 59, 77, 124, 96, 33, 62, 18, 99, 112, 113, 48, 114, 100, 15, 119, 52, 13, 127, 101, 81, 92, 44, 31, 26, 32, 29, 17, 19, 107, 61, 94, 75, 41, 73, 109, 28, 37, 20, 83, 11, 91, 78, 7, 39, 30, 95, 45, 84, 12, 10, 108, 9, 71, 64, 76, 98, 14, 34, 68, 6, 16, 80, 103, 0, 74, 35, 72, 105, 4, 88, 43, 70, 67, 24, 1, 2, 66, 3, 5, 65, 69, 8], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 38, 102, 118, 89, 58, 123, 125, 104, 22, 25, 87, 63, 85, 62, 97, 56, 124, 93, 23, 86, 90, 114, 27, 52, 48, 21, 40, 59, 36, 49, 96, 79, 82, 119, 116, 60, 18, 127, 15, 112, 113, 13, 29, 33, 101, 77, 99, 100, 26, 44, 91, 61, 32, 109, 81, 17, 92, 19, 45, 107, 31, 20, 41, 75, 37, 28, 64, 83, 73, 39, 11, 0, 7, 71, 94, 9, 105, 10, 95, 78, 98, 68, 80, 103, 72, 12, 84, 16, 35, 70, 108, 76, 30, 14, 43, 34, 4, 1, 88, 6, 74, 67, 66, 65, 24, 3, 2, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 89, 123, 22, 58, 104, 97, 27, 25, 52, 85, 59, 87, 93, 125, 60, 23, 36, 86, 49, 63, 116, 40, 124, 21, 96, 90, 61, 119, 48, 82, 79, 18, 33, 56, 112, 113, 13, 101, 114, 62, 44, 100, 26, 99, 77, 29, 15, 92, 109, 17, 127, 28, 107, 81, 94, 32, 20, 83, 91, 19, 37, 45, 31, 30, 41, 84, 75, 73, 7, 70, 11, 39, 9, 64, 12, 72, 95, 0, 14, 76, 105, 35, 78, 43, 71, 34, 80, 98, 10, 16, 108, 68, 74, 88, 4, 2, 67, 103, 1, 24, 66, 6, 65, 3, 5, 69, 8], [115, 122, 53, 117, 51, 120, 47, 111, 126, 121, 54, 50, 46, 110, 57, 55, 106, 42, 102, 38, 118, 22, 89, 86, 104, 25, 27, 97, 58, 49, 87, 85, 123, 125, 93, 40, 36, 90, 21, 124, 56, 23, 18, 63, 62, 96, 15, 79, 119, 59, 82, 116, 114, 77, 127, 52, 60, 13, 81, 61, 33, 99, 113, 48, 100, 112, 92, 17, 26, 101, 37, 29, 83, 9, 28, 75, 11, 19, 44, 70, 31, 84, 107, 41, 73, 76, 94, 32, 20, 91, 95, 109, 10, 108, 105, 7, 45, 12, 30, 98, 14, 72, 78, 16, 80, 71, 0, 103, 68, 64, 43, 74, 4, 35, 39, 34, 88, 65, 24, 67, 3, 66, 2, 1, 69, 5, 8, 6], [115, 53, 122, 117, 51, 120, 47, 111, 126, 121, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 22, 89, 104, 25, 123, 87, 86, 118, 85, 58, 23, 93, 21, 40, 97, 27, 36, 125, 90, 52, 49, 62, 79, 59, 18, 56, 63, 48, 77, 119, 33, 124, 82, 96, 127, 15, 114, 116, 61, 99, 13, 112, 100, 17, 81, 101, 32, 60, 19, 92, 29, 84, 26, 44, 11, 83, 9, 113, 20, 28, 75, 91, 37, 73, 109, 94, 41, 70, 7, 16, 31, 76, 12, 45, 95, 10, 35, 107, 105, 64, 71, 78, 39, 14, 108, 30, 34, 80, 98, 0, 43, 74, 68, 72, 24, 88, 4, 103, 66, 1, 67, 65, 3, 2, 69, 5, 6, 8], [115, 53, 122, 117, 51, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 123, 22, 104, 25, 89, 118, 36, 87, 125, 23, 86, 85, 90, 124, 40, 27, 93, 97, 21, 96, 63, 52, 59, 49, 82, 116, 33, 60, 18, 79, 119, 114, 99, 127, 56, 62, 13, 100, 77, 26, 48, 101, 29, 44, 15, 41, 61, 112, 109, 19, 113, 17, 94, 81, 92, 32, 28, 37, 91, 84, 73, 83, 20, 9, 39, 30, 11, 107, 76, 35, 7, 75, 70, 105, 45, 0, 78, 98, 80, 10, 43, 31, 12, 108, 16, 103, 95, 34, 14, 71, 24, 68, 74, 64, 72, 88, 4, 66, 1, 3, 6, 2, 8, 67, 65, 5, 69], [115, 53, 122, 51, 117, 120, 47, 121, 126, 111, 50, 54, 46, 110, 57, 55, 106, 42, 102, 38, 58, 118, 125, 89, 104, 123, 22, 87, 25, 27, 63, 21, 59, 36, 62, 86, 116, 49, 124, 40, 85, 96, 52, 93, 56, 23, 90, 97, 60, 114, 119, 99, 127, 82, 100, 112, 33, 48, 18, 61, 101, 13, 79, 15, 107, 77, 81, 113, 32, 44, 26, 37, 19, 92, 109, 29, 17, 45, 83, 94, 28, 11, 20, 9, 30, 12, 73, 91, 34, 105, 98, 108, 41, 76, 14, 95, 31, 84, 71, 43, 35, 16, 103, 0, 80, 39, 78, 75, 70, 10, 74, 7, 68, 6, 4, 24, 64, 88, 8, 66, 2, 1, 72, 67, 3, 69, 5, 65], [115, 53, 122, 51, 117, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 25, 58, 123, 118, 104, 89, 62, 125, 22, 87, 27, 97, 85, 21, 90, 93, 86, 36, 23, 56, 59, 18, 124, 13, 79, 40, 63, 48, 82, 49, 60, 114, 96, 52, 116, 33, 15, 113, 77, 99, 101, 100, 29, 112, 119, 44, 127, 17, 11, 81, 107, 28, 32, 26, 92, 19, 73, 61, 45, 84, 75, 9, 83, 41, 94, 37, 6, 71, 78, 16, 76, 64, 91, 105, 109, 95, 20, 30, 31, 14, 98, 43, 0, 12, 108, 7, 10, 39, 80, 74, 35, 68, 4, 34, 66, 103, 8, 65, 3, 88, 67, 70, 2, 24, 1, 72, 5, 69], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 102, 38, 58, 123, 104, 89, 118, 97, 22, 25, 86, 27, 21, 125, 87, 40, 62, 56, 60, 93, 36, 85, 63, 59, 23, 116, 79, 49, 124, 33, 52, 82, 13, 18, 99, 90, 127, 44, 48, 96, 77, 100, 119, 114, 113, 61, 15, 101, 112, 81, 32, 26, 11, 73, 92, 28, 6, 31, 29, 17, 0, 20, 45, 83, 41, 91, 84, 94, 37, 19, 71, 9, 39, 109, 105, 34, 108, 95, 98, 76, 75, 107, 78, 12, 14, 64, 30, 8, 80, 4, 7, 43, 68, 74, 35, 16, 10, 103, 67, 88, 1, 66, 3, 2, 65, 24, 69, 5, 72, 70], [115, 53, 122, 117, 51, 120, 47, 121, 111, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 123, 22, 58, 118, 89, 87, 125, 104, 21, 25, 86, 97, 27, 85, 23, 124, 63, 93, 60, 36, 127, 90, 62, 116, 59, 82, 79, 40, 18, 100, 13, 56, 49, 99, 15, 119, 96, 81, 77, 52, 33, 48, 114, 61, 113, 19, 101, 11, 29, 17, 32, 26, 28, 83, 9, 92, 84, 73, 91, 94, 109, 6, 41, 44, 31, 20, 37, 112, 12, 80, 76, 108, 78, 98, 75, 43, 74, 95, 107, 8, 71, 14, 105, 68, 7, 45, 39, 0, 30, 35, 34, 16, 103, 10, 4, 64, 88, 24, 66, 67, 1, 69, 3, 2, 65, 70, 5, 72], [115, 53, 122, 51, 117, 120, 47, 111, 121, 126, 54, 50, 46, 110, 57, 106, 55, 42, 38, 102, 58, 89, 104, 123, 25, 22, 118, 97, 87, 56, 27, 93, 86, 23, 40, 21, 36, 85, 90, 59, 124, 100, 49, 63, 125, 33, 127, 52, 60, 82, 79, 116, 18, 13, 96, 77, 99, 15, 101, 29, 62, 114, 113, 61, 119, 11, 81, 17, 48, 19, 44, 84, 32, 94, 9, 31, 28, 7, 73, 6, 30, 75, 92, 83, 112, 45, 91, 71, 26, 78, 74, 109, 37, 64, 0, 20, 41, 12, 95, 8, 16, 76, 4, 35, 98, 108, 80, 34, 14, 68, 107, 10, 105, 43, 39, 66, 103, 2, 67, 24, 65, 3, 70, 88, 1, 5, 69, 72], [115, 53, 122, 51, 117, 120, 47, 111, 126, 54, 121, 50, 46, 110, 55, 106, 57, 42, 102, 38, 22, 123, 58, 104, 89, 25, 87, 86, 27, 97, 85, 118, 56, 21, 125, 63, 40, 93, 36, 23, 90, 18, 82, 49, 79, 124, 96, 116, 60, 77, 15, 62, 119, 99, 101, 13, 114, 33, 59, 100, 61, 11, 48, 29, 81, 17, 9, 94, 26, 112, 127, 44, 19, 52, 113, 83, 32, 84, 28, 37, 92, 75, 73, 91, 76, 41, 30, 107, 8, 80, 35, 16, 78, 12, 109, 39, 31, 20, 45, 7, 14, 95, 108, 71, 10, 105, 6, 103, 74, 64, 98, 43, 4, 34, 68, 24, 0, 70, 88, 66, 3, 2, 65, 67, 1, 5, 69, 72]], "model.layers.23.self_attn.q_proj": [[111, 100, 107, 15, 88, 90, 21, 93, 81, 47, 20, 75, 11, 31, 52, 82, 73, 39, 19, 83, 85, 22, 41, 104, 12, 110, 54, 28, 13, 32, 95, 120, 14, 77, 27, 72, 76, 45, 119, 37, 87, 36, 40, 106, 71, 94, 24, 50, 6, 118, 26, 92, 101, 51, 33, 44, 38, 25, 16, 122, 114, 59, 10, 80, 112, 34, 58, 124, 5, 56, 108, 46, 123, 116, 109, 84, 79, 62, 68, 23, 70, 125, 96, 43, 103, 63, 127, 17, 57, 61, 105, 113, 78, 97, 55, 53, 126, 86, 89, 117, 74, 60, 98, 30, 18, 9, 8, 48, 49, 91, 42, 115, 29, 69, 99, 35, 121, 7, 102, 2, 67, 4, 0, 3, 66, 1, 65, 64], [111, 100, 47, 107, 32, 88, 90, 27, 94, 21, 83, 85, 24, 78, 80, 84, 81, 40, 95, 109, 30, 22, 75, 86, 17, 26, 44, 15, 10, 89, 7, 96, 77, 110, 28, 118, 119, 36, 31, 123, 50, 51, 39, 68, 0, 23, 46, 2, 63, 43, 116, 126, 127, 60, 120, 5, 73, 56, 121, 93, 41, 105, 124, 48, 42, 49, 52, 37, 33, 106, 57, 72, 101, 45, 114, 125, 98, 91, 122, 99, 103, 29, 61, 35, 20, 55, 115, 113, 112, 53, 11, 92, 54, 71, 59, 62, 34, 102, 38, 58, 108, 104, 97, 117, 6, 74, 87, 64, 25, 79, 82, 66, 19, 3, 16, 67, 18, 12, 4, 69, 14, 76, 8, 13, 65, 9, 1, 70], [111, 100, 107, 47, 90, 83, 86, 27, 88, 31, 93, 18, 80, 24, 78, 79, 91, 41, 17, 95, 85, 23, 32, 74, 72, 101, 39, 22, 96, 110, 15, 36, 94, 29, 76, 77, 50, 21, 120, 92, 34, 109, 81, 62, 13, 89, 40, 9, 118, 123, 82, 122, 119, 7, 30, 11, 84, 75, 121, 12, 127, 16, 106, 10, 56, 59, 114, 60, 20, 117, 116, 61, 5, 57, 104, 87, 58, 124, 43, 33, 52, 25, 115, 26, 46, 68, 37, 35, 28, 19, 54, 108, 97, 51, 126, 8, 102, 14, 112, 48, 73, 53, 38, 98, 105, 42, 103, 2, 0, 45, 63, 55, 71, 125, 44, 99, 49, 113, 4, 3, 6, 69, 70, 64, 1, 66, 67, 65], [111, 100, 107, 47, 32, 88, 121, 27, 85, 24, 63, 90, 51, 119, 21, 95, 81, 49, 36, 94, 40, 80, 91, 52, 50, 109, 84, 83, 93, 45, 42, 96, 61, 78, 38, 106, 34, 58, 124, 108, 110, 10, 118, 56, 46, 44, 122, 117, 125, 30, 57, 15, 115, 62, 28, 97, 104, 55, 39, 120, 59, 60, 54, 126, 112, 114, 103, 53, 23, 22, 92, 105, 75, 99, 116, 31, 41, 98, 113, 48, 35, 86, 43, 127, 123, 102, 37, 33, 77, 29, 101, 26, 5, 20, 89, 68, 17, 0, 16, 18, 72, 7, 2, 71, 25, 87, 82, 11, 73, 67, 19, 79, 14, 76, 12, 6, 74, 13, 3, 9, 8, 66, 64, 70, 69, 1, 4, 65], [56, 51, 103, 46, 110, 19, 13, 97, 15, 90, 28, 17, 99, 23, 116, 60, 94, 21, 10, 115, 47, 81, 87, 24, 57, 79, 100, 75, 72, 124, 91, 9, 53, 93, 86, 109, 80, 31, 61, 84, 89, 54, 14, 85, 74, 58, 92, 126, 3, 7, 77, 76, 6, 52, 18, 48, 113, 25, 11, 5, 123, 114, 41, 30, 127, 59, 119, 122, 27, 55, 45, 112, 12, 83, 63, 16, 71, 68, 49, 121, 29, 62, 22, 67, 43, 26, 78, 70, 88, 8, 98, 102, 82, 95, 20, 32, 108, 125, 0, 34, 36, 118, 117, 104, 105, 1, 69, 120, 101, 96, 38, 37, 106, 111, 35, 50, 107, 40, 44, 42, 64, 73, 4, 2, 33, 65, 66, 39], [46, 56, 103, 110, 53, 57, 60, 115, 61, 116, 47, 122, 124, 20, 52, 89, 127, 97, 126, 112, 51, 59, 55, 108, 114, 121, 58, 94, 54, 118, 63, 49, 62, 123, 48, 113, 109, 119, 50, 125, 111, 84, 88, 30, 106, 28, 99, 117, 25, 86, 44, 45, 80, 105, 43, 120, 107, 37, 36, 41, 24, 92, 42, 104, 27, 78, 14, 40, 38, 39, 22, 101, 102, 98, 82, 16, 33, 100, 35, 91, 19, 87, 95, 34, 31, 96, 32, 11, 9, 4, 17, 71, 64, 66, 69, 93, 0, 26, 90, 68, 73, 2, 29, 18, 1, 13, 75, 10, 21, 65, 85, 5, 83, 12, 15, 7, 67, 72, 3, 81, 23, 79, 6, 70, 76, 77, 8, 74], [56, 103, 46, 53, 57, 110, 60, 61, 124, 116, 47, 126, 51, 122, 112, 52, 97, 127, 115, 20, 89, 55, 59, 54, 58, 94, 123, 118, 28, 121, 114, 49, 113, 125, 62, 37, 119, 109, 108, 63, 99, 48, 120, 111, 25, 117, 45, 50, 80, 44, 30, 36, 105, 86, 84, 43, 92, 106, 41, 107, 88, 24, 104, 14, 42, 40, 19, 23, 39, 82, 102, 101, 38, 22, 98, 100, 27, 34, 33, 35, 78, 16, 91, 13, 96, 31, 21, 71, 87, 95, 9, 81, 12, 90, 15, 69, 32, 18, 68, 11, 72, 0, 93, 3, 74, 5, 29, 26, 85, 10, 64, 66, 1, 2, 4, 83, 17, 75, 73, 67, 6, 65, 76, 7, 79, 8, 77, 70], [46, 51, 103, 56, 19, 110, 87, 17, 15, 97, 28, 13, 92, 116, 10, 81, 61, 12, 47, 90, 53, 94, 23, 86, 112, 60, 57, 126, 72, 58, 52, 59, 96, 121, 124, 43, 89, 122, 115, 25, 114, 127, 106, 49, 55, 62, 21, 48, 63, 54, 108, 45, 4, 123, 109, 125, 119, 120, 42, 111, 118, 113, 24, 117, 50, 83, 30, 44, 105, 79, 85, 93, 107, 75, 67, 99, 9, 82, 11, 31, 20, 71, 8, 98, 104, 70, 6, 76, 26, 101, 84, 77, 40, 32, 41, 38, 7, 34, 37, 29, 22, 36, 102, 100, 95, 5, 3, 91, 80, 18, 14, 68, 27, 73, 69, 0, 1, 65, 35, 88, 74, 66, 16, 64, 2, 78, 39, 33], [102, 110, 33, 49, 46, 92, 113, 111, 86, 28, 122, 19, 57, 81, 24, 55, 78, 54, 70, 74, 85, 13, 108, 90, 61, 8, 117, 94, 30, 23, 66, 105, 80, 109, 114, 26, 76, 87, 44, 14, 60, 75, 9, 68, 79, 127, 67, 17, 21, 47, 77, 107, 31, 119, 65, 51, 0, 106, 18, 63, 116, 56, 62, 15, 7, 120, 58, 53, 126, 83, 52, 112, 115, 40, 11, 50, 39, 103, 43, 121, 104, 36, 124, 2, 123, 12, 20, 99, 125, 3, 118, 45, 48, 41, 59, 37, 22, 32, 29, 98, 71, 42, 89, 72, 97, 25, 35, 101, 69, 88, 16, 100, 27, 93, 95, 34, 91, 82, 96, 5, 4, 10, 1, 84, 38, 64, 73, 6], [110, 102, 33, 46, 49, 57, 92, 113, 111, 86, 28, 19, 24, 81, 85, 61, 116, 119, 56, 122, 115, 87, 26, 79, 108, 90, 107, 78, 94, 63, 109, 114, 9, 13, 52, 23, 121, 127, 30, 74, 47, 104, 123, 70, 21, 17, 99, 126, 59, 48, 62, 51, 31, 14, 60, 98, 124, 58, 125, 80, 55, 106, 54, 43, 112, 53, 120, 50, 100, 44, 118, 29, 42, 76, 41, 105, 117, 103, 75, 77, 8, 45, 37, 36, 83, 15, 101, 25, 20, 66, 68, 35, 22, 95, 34, 40, 12, 18, 39, 93, 11, 32, 96, 0, 97, 38, 27, 89, 88, 69, 91, 3, 65, 84, 1, 72, 4, 10, 71, 73, 7, 16, 82, 6, 64, 5, 67, 2], [110, 102, 46, 113, 49, 33, 92, 111, 86, 19, 122, 28, 24, 81, 79, 55, 60, 85, 78, 39, 26, 127, 87, 74, 115, 68, 90, 93, 12, 118, 119, 114, 77, 70, 9, 125, 56, 8, 13, 50, 84, 108, 61, 29, 107, 98, 14, 59, 37, 105, 51, 100, 22, 112, 73, 71, 35, 25, 47, 94, 58, 88, 44, 117, 124, 53, 120, 42, 106, 21, 95, 57, 40, 23, 20, 17, 83, 82, 121, 2, 96, 62, 54, 52, 32, 30, 41, 65, 31, 116, 43, 109, 76, 104, 126, 75, 36, 64, 63, 38, 99, 45, 48, 91, 101, 34, 11, 5, 80, 103, 123, 66, 15, 89, 27, 18, 10, 97, 16, 4, 6, 72, 0, 7, 3, 69, 67, 1], [110, 102, 46, 49, 33, 113, 92, 111, 86, 122, 19, 28, 54, 81, 85, 119, 24, 90, 26, 109, 94, 126, 79, 70, 13, 120, 78, 9, 51, 57, 112, 74, 125, 21, 87, 108, 123, 61, 50, 59, 105, 107, 121, 52, 14, 58, 8, 53, 47, 56, 68, 17, 60, 43, 45, 100, 93, 127, 114, 63, 12, 118, 106, 80, 44, 116, 55, 20, 48, 29, 15, 83, 30, 124, 23, 117, 103, 77, 31, 39, 65, 115, 42, 62, 75, 22, 104, 91, 40, 76, 99, 36, 71, 66, 34, 97, 101, 41, 98, 37, 32, 95, 88, 35, 82, 10, 27, 89, 18, 96, 25, 73, 84, 5, 11, 64, 16, 38, 72, 3, 0, 2, 4, 69, 7, 6, 1, 67], [48, 39, 119, 117, 56, 33, 60, 112, 121, 127, 58, 120, 47, 116, 125, 63, 122, 49, 90, 29, 114, 118, 55, 51, 123, 115, 124, 52, 59, 126, 113, 61, 62, 53, 107, 24, 44, 111, 54, 50, 110, 45, 91, 108, 93, 46, 106, 57, 88, 95, 109, 26, 86, 42, 20, 40, 43, 41, 105, 102, 17, 101, 104, 97, 36, 103, 38, 27, 85, 87, 37, 92, 100, 34, 22, 96, 94, 99, 98, 28, 89, 35, 21, 81, 16, 32, 31, 30, 84, 83, 78, 76, 14, 25, 12, 15, 82, 80, 10, 23, 18, 74, 19, 72, 65, 1, 67, 3, 0, 13, 64, 8, 69, 5, 68, 66, 2, 79, 4, 11, 7, 6, 77, 71, 9, 75, 73, 70], [119, 39, 48, 117, 60, 33, 56, 121, 47, 127, 120, 55, 63, 125, 58, 116, 51, 122, 49, 123, 114, 118, 115, 61, 52, 112, 90, 53, 59, 113, 106, 44, 62, 126, 124, 110, 29, 57, 54, 24, 107, 50, 111, 45, 108, 46, 41, 91, 93, 26, 95, 88, 105, 20, 86, 109, 43, 42, 102, 40, 104, 17, 103, 92, 101, 100, 38, 27, 97, 21, 34, 36, 94, 85, 28, 37, 87, 35, 99, 96, 89, 31, 22, 98, 16, 32, 76, 30, 81, 84, 25, 83, 15, 80, 78, 23, 14, 12, 82, 18, 10, 74, 1, 0, 65, 67, 64, 8, 19, 69, 72, 5, 4, 3, 13, 79, 11, 68, 66, 2, 7, 6, 71, 75, 70, 9, 77, 73], [117, 39, 119, 48, 56, 90, 60, 33, 55, 62, 124, 52, 121, 120, 95, 127, 47, 46, 105, 93, 58, 29, 116, 106, 61, 115, 122, 114, 63, 83, 26, 125, 51, 10, 118, 102, 126, 113, 49, 24, 57, 107, 50, 53, 123, 89, 59, 45, 104, 109, 108, 96, 54, 16, 111, 86, 44, 110, 112, 20, 42, 27, 43, 18, 41, 81, 85, 78, 38, 30, 40, 32, 94, 31, 19, 36, 25, 17, 34, 101, 82, 92, 100, 98, 14, 88, 91, 23, 75, 35, 97, 84, 87, 28, 71, 37, 13, 99, 22, 103, 12, 8, 21, 76, 3, 15, 11, 80, 4, 9, 68, 2, 72, 7, 74, 79, 5, 0, 6, 69, 73, 70, 67, 77, 1, 66, 65, 64], [39, 117, 119, 48, 83, 33, 15, 13, 11, 9, 29, 86, 6, 89, 66, 7, 74, 55, 88, 27, 87, 73, 81, 68, 80, 53, 90, 72, 22, 84, 91, 75, 82, 85, 8, 65, 30, 23, 18, 62, 26, 25, 17, 56, 16, 34, 0, 77, 79, 12, 21, 67, 76, 19, 92, 14, 31, 78, 70, 71, 28, 94, 4, 20, 5, 10, 32, 58, 50, 96, 35, 3, 64, 43, 106, 45, 24, 95, 40, 2, 69, 47, 102, 104, 93, 112, 38, 105, 99, 101, 54, 121, 108, 61, 124, 120, 57, 122, 1, 103, 110, 41, 98, 127, 49, 100, 115, 123, 36, 60, 111, 44, 63, 125, 118, 42, 52, 51, 107, 37, 116, 59, 97, 114, 113, 126, 109, 46], [44, 37, 108, 76, 18, 21, 14, 97, 24, 57, 80, 28, 71, 0, 5, 73, 125, 90, 69, 67, 7, 2, 51, 121, 58, 3, 11, 72, 75, 113, 8, 10, 19, 52, 23, 12, 89, 118, 55, 33, 91, 68, 16, 64, 65, 9, 88, 101, 34, 92, 70, 96, 77, 85, 13, 1, 84, 119, 99, 124, 20, 109, 25, 123, 66, 86, 26, 6, 82, 46, 93, 83, 78, 127, 114, 104, 79, 74, 15, 81, 35, 4, 100, 95, 47, 111, 29, 30, 27, 50, 56, 120, 98, 60, 126, 17, 117, 22, 31, 32, 61, 63, 110, 94, 36, 87, 106, 62, 103, 45, 41, 39, 102, 38, 105, 112, 43, 40, 107, 54, 116, 115, 122, 48, 42, 49, 53, 59], [44, 37, 108, 28, 24, 21, 97, 80, 18, 14, 19, 76, 73, 57, 87, 51, 90, 58, 71, 125, 15, 121, 17, 113, 117, 47, 5, 55, 11, 7, 106, 95, 69, 23, 52, 50, 98, 2, 93, 30, 114, 99, 85, 83, 45, 86, 22, 127, 27, 92, 10, 66, 35, 109, 31, 84, 20, 46, 3, 81, 82, 100, 29, 6, 75, 91, 79, 78, 124, 70, 16, 34, 96, 118, 105, 12, 33, 123, 61, 13, 9, 48, 89, 94, 88, 26, 36, 8, 111, 104, 25, 112, 32, 40, 103, 77, 38, 42, 53, 120, 126, 102, 41, 119, 107, 49, 43, 74, 110, 72, 68, 115, 0, 67, 122, 116, 39, 56, 54, 63, 59, 60, 101, 62, 4, 64, 65, 1], [57, 44, 37, 125, 108, 90, 97, 19, 28, 58, 21, 114, 24, 126, 113, 109, 117, 17, 86, 116, 121, 61, 51, 40, 110, 50, 49, 122, 55, 118, 41, 46, 59, 47, 11, 48, 43, 111, 106, 56, 124, 102, 63, 53, 123, 52, 62, 115, 112, 120, 119, 91, 54, 60, 127, 42, 107, 80, 104, 45, 105, 18, 87, 93, 39, 95, 14, 72, 35, 38, 84, 73, 77, 103, 23, 13, 101, 75, 27, 36, 20, 74, 66, 34, 89, 83, 25, 15, 26, 6, 30, 98, 8, 33, 100, 4, 29, 22, 81, 99, 94, 5, 79, 31, 68, 96, 32, 70, 76, 69, 2, 10, 1, 65, 12, 85, 92, 0, 64, 88, 78, 7, 9, 82, 16, 3, 71, 67], [44, 37, 125, 24, 108, 21, 28, 18, 97, 14, 90, 19, 58, 80, 57, 51, 11, 121, 114, 73, 109, 17, 13, 95, 118, 117, 52, 87, 75, 113, 69, 35, 50, 93, 61, 81, 46, 123, 76, 9, 124, 34, 119, 102, 71, 111, 115, 59, 36, 104, 100, 32, 96, 4, 55, 103, 20, 84, 99, 30, 85, 45, 105, 60, 106, 62, 29, 86, 83, 39, 25, 43, 40, 48, 66, 94, 38, 42, 120, 107, 47, 91, 110, 23, 112, 82, 15, 127, 78, 116, 98, 92, 16, 31, 122, 54, 89, 126, 49, 33, 72, 88, 41, 56, 22, 53, 101, 26, 10, 63, 74, 79, 67, 6, 1, 27, 8, 77, 5, 64, 12, 2, 3, 70, 68, 65, 7, 0], [54, 37, 127, 62, 63, 24, 116, 33, 101, 119, 123, 18, 55, 59, 51, 120, 58, 112, 46, 53, 60, 125, 114, 117, 113, 39, 126, 44, 15, 57, 61, 56, 111, 121, 122, 118, 47, 52, 49, 27, 110, 50, 48, 91, 115, 45, 124, 94, 109, 85, 107, 108, 105, 106, 43, 92, 86, 20, 21, 104, 42, 73, 103, 41, 102, 88, 100, 30, 40, 38, 25, 29, 82, 23, 95, 36, 13, 28, 97, 35, 31, 22, 9, 77, 79, 99, 84, 90, 98, 34, 16, 93, 87, 26, 17, 89, 12, 14, 1, 80, 75, 71, 65, 32, 4, 19, 6, 96, 70, 2, 66, 10, 7, 11, 68, 67, 72, 78, 3, 76, 5, 69, 0, 83, 8, 81, 64, 74], [127, 54, 37, 25, 62, 17, 94, 101, 33, 86, 15, 123, 20, 6, 63, 24, 14, 12, 27, 77, 19, 10, 90, 30, 109, 23, 116, 84, 88, 13, 61, 93, 72, 113, 9, 51, 107, 95, 91, 59, 120, 38, 119, 92, 125, 117, 112, 22, 7, 41, 124, 70, 32, 58, 85, 35, 67, 36, 75, 118, 48, 42, 78, 31, 18, 126, 122, 21, 79, 28, 102, 55, 34, 80, 29, 115, 114, 68, 82, 5, 11, 53, 50, 16, 87, 57, 83, 103, 46, 98, 106, 81, 26, 56, 60, 44, 4, 111, 39, 74, 89, 8, 108, 76, 52, 110, 121, 1, 45, 49, 73, 40, 96, 100, 97, 66, 105, 104, 99, 43, 47, 0, 71, 3, 2, 69, 65, 64], [127, 63, 37, 62, 54, 24, 33, 51, 112, 123, 116, 101, 117, 55, 120, 59, 58, 119, 125, 53, 19, 114, 46, 111, 44, 60, 113, 122, 61, 47, 126, 57, 121, 91, 118, 52, 56, 49, 48, 27, 50, 45, 94, 110, 115, 85, 124, 39, 108, 109, 18, 102, 86, 106, 88, 30, 107, 29, 105, 43, 41, 104, 103, 42, 96, 21, 14, 40, 38, 95, 20, 97, 32, 26, 100, 22, 16, 31, 75, 83, 36, 25, 98, 12, 92, 35, 99, 78, 34, 15, 90, 82, 13, 93, 80, 17, 6, 11, 73, 77, 9, 65, 1, 84, 89, 28, 87, 23, 68, 4, 66, 2, 7, 70, 10, 71, 72, 5, 8, 76, 79, 0, 81, 67, 64, 3, 69, 74], [127, 54, 37, 33, 17, 62, 86, 10, 15, 24, 14, 123, 12, 5, 72, 90, 27, 65, 7, 116, 25, 0, 101, 63, 77, 91, 64, 20, 44, 23, 88, 73, 4, 19, 67, 87, 89, 113, 22, 71, 112, 2, 93, 8, 107, 83, 59, 81, 111, 80, 76, 68, 79, 74, 82, 118, 21, 13, 6, 120, 104, 46, 114, 75, 126, 28, 119, 70, 122, 18, 3, 97, 11, 30, 102, 1, 16, 84, 69, 56, 31, 106, 58, 26, 9, 66, 78, 32, 85, 48, 95, 40, 117, 61, 94, 96, 45, 125, 99, 35, 29, 98, 57, 92, 60, 39, 53, 124, 34, 36, 100, 51, 38, 52, 103, 41, 108, 55, 109, 110, 105, 47, 43, 121, 49, 50, 115, 42], [43, 121, 45, 107, 100, 97, 109, 27, 91, 102, 123, 24, 103, 15, 22, 54, 81, 59, 88, 127, 120, 118, 114, 117, 110, 20, 44, 124, 85, 61, 111, 31, 79, 119, 51, 26, 75, 57, 7, 41, 11, 33, 58, 98, 49, 92, 46, 23, 112, 39, 52, 99, 42, 13, 113, 55, 21, 62, 29, 17, 122, 28, 47, 84, 63, 125, 25, 48, 18, 93, 50, 108, 126, 56, 82, 115, 116, 86, 101, 104, 37, 105, 53, 60, 32, 30, 96, 38, 40, 35, 34, 89, 80, 106, 90, 19, 94, 77, 83, 95, 16, 73, 87, 71, 74, 70, 76, 14, 78, 36, 3, 6, 9, 12, 5, 69, 8, 10, 72, 66, 67, 65, 2, 1, 68, 64, 0, 4], [43, 107, 123, 117, 100, 27, 97, 91, 88, 51, 103, 21, 49, 124, 80, 59, 54, 24, 109, 55, 48, 85, 115, 127, 57, 14, 50, 121, 61, 114, 113, 126, 58, 118, 22, 45, 119, 60, 62, 52, 41, 31, 63, 125, 112, 47, 111, 74, 122, 116, 53, 29, 19, 76, 110, 102, 46, 44, 56, 108, 39, 93, 37, 120, 5, 20, 42, 89, 8, 30, 90, 6, 2, 99, 105, 3, 18, 40, 106, 68, 35, 101, 98, 87, 104, 84, 38, 78, 32, 33, 16, 1, 86, 12, 17, 96, 0, 28, 34, 25, 94, 26, 81, 10, 72, 11, 95, 23, 15, 36, 92, 4, 69, 83, 82, 66, 73, 65, 64, 70, 13, 77, 67, 71, 7, 75, 9, 79], [43, 121, 45, 107, 100, 27, 97, 91, 116, 63, 24, 22, 61, 106, 125, 54, 114, 37, 15, 49, 88, 118, 58, 124, 29, 33, 85, 76, 120, 57, 31, 17, 19, 32, 81, 21, 41, 127, 47, 109, 123, 80, 50, 18, 52, 60, 51, 55, 98, 122, 74, 86, 53, 101, 113, 34, 105, 103, 12, 28, 119, 14, 111, 62, 94, 93, 59, 8, 115, 46, 38, 44, 112, 42, 7, 110, 90, 40, 126, 68, 48, 20, 117, 26, 104, 102, 36, 99, 108, 35, 2, 39, 96, 30, 92, 84, 13, 11, 56, 79, 82, 25, 95, 89, 75, 73, 83, 77, 1, 78, 5, 3, 10, 87, 71, 69, 0, 72, 9, 23, 67, 6, 4, 66, 65, 64, 70, 16], [121, 43, 45, 100, 91, 97, 27, 107, 88, 85, 21, 123, 80, 51, 54, 24, 115, 49, 61, 117, 127, 126, 48, 59, 57, 114, 109, 46, 52, 116, 118, 63, 55, 74, 53, 50, 22, 14, 76, 60, 105, 31, 62, 119, 124, 120, 111, 19, 122, 125, 113, 58, 106, 47, 44, 56, 102, 8, 112, 90, 103, 110, 12, 108, 39, 93, 37, 20, 32, 29, 42, 41, 99, 89, 81, 33, 5, 87, 2, 104, 17, 40, 38, 6, 34, 16, 82, 26, 101, 78, 28, 86, 96, 92, 30, 3, 35, 1, 98, 36, 68, 94, 72, 23, 95, 18, 10, 84, 0, 69, 25, 83, 15, 70, 4, 73, 65, 67, 11, 13, 66, 64, 77, 79, 71, 9, 7, 75], [103, 115, 51, 85, 83, 80, 62, 10, 90, 13, 124, 118, 26, 63, 72, 48, 56, 70, 60, 98, 68, 52, 55, 88, 106, 47, 59, 41, 53, 50, 93, 110, 30, 91, 100, 107, 76, 71, 86, 58, 111, 19, 45, 84, 15, 66, 82, 122, 61, 42, 123, 57, 125, 38, 44, 96, 89, 77, 119, 65, 81, 40, 21, 28, 16, 14, 87, 11, 3, 23, 25, 94, 101, 104, 43, 33, 102, 17, 74, 27, 105, 121, 49, 95, 112, 31, 92, 117, 116, 24, 108, 20, 120, 36, 7, 9, 109, 5, 114, 46, 97, 34, 69, 126, 127, 73, 35, 113, 79, 8, 99, 22, 29, 18, 32, 78, 54, 12, 37, 64, 2, 39, 6, 1, 75, 4, 0, 67], [103, 51, 115, 85, 83, 10, 80, 13, 72, 70, 66, 68, 26, 62, 63, 90, 118, 60, 81, 124, 107, 64, 2, 91, 53, 59, 55, 44, 52, 119, 89, 56, 65, 93, 106, 47, 111, 50, 0, 123, 98, 19, 15, 100, 40, 76, 1, 110, 61, 5, 48, 114, 69, 77, 16, 41, 25, 6, 99, 74, 8, 4, 104, 3, 32, 126, 79, 11, 94, 117, 105, 22, 121, 87, 20, 88, 29, 58, 116, 46, 43, 73, 112, 14, 75, 21, 27, 23, 127, 24, 120, 35, 71, 38, 86, 109, 57, 49, 82, 92, 125, 45, 113, 122, 67, 54, 102, 78, 96, 84, 9, 108, 18, 95, 12, 101, 42, 37, 30, 33, 17, 28, 34, 39, 7, 31, 36, 97], [103, 51, 115, 85, 80, 13, 10, 68, 83, 70, 72, 26, 66, 0, 62, 124, 55, 65, 90, 118, 63, 53, 60, 64, 47, 76, 100, 98, 89, 107, 46, 59, 88, 50, 56, 48, 61, 2, 91, 93, 82, 1, 58, 23, 111, 4, 3, 19, 81, 73, 99, 121, 25, 44, 5, 106, 123, 6, 69, 75, 74, 11, 41, 87, 119, 15, 20, 127, 71, 79, 21, 95, 77, 110, 8, 52, 40, 126, 16, 114, 67, 27, 17, 112, 9, 122, 78, 94, 57, 22, 117, 86, 49, 12, 14, 32, 39, 113, 30, 37, 38, 92, 120, 18, 29, 105, 84, 96, 31, 36, 7, 33, 108, 42, 125, 35, 116, 97, 101, 24, 109, 45, 104, 54, 43, 102, 28, 34], [103, 115, 51, 80, 85, 10, 13, 83, 72, 70, 68, 62, 26, 124, 118, 60, 59, 55, 66, 107, 90, 82, 64, 93, 119, 47, 52, 25, 15, 63, 48, 53, 98, 40, 88, 56, 111, 87, 73, 5, 1, 89, 76, 65, 69, 23, 29, 2, 61, 71, 0, 46, 44, 81, 100, 50, 117, 77, 58, 19, 22, 96, 84, 32, 112, 110, 24, 74, 8, 7, 3, 16, 114, 12, 17, 122, 126, 121, 125, 91, 27, 11, 123, 35, 38, 28, 99, 4, 104, 106, 116, 20, 101, 45, 6, 9, 79, 31, 78, 95, 41, 18, 30, 21, 86, 67, 92, 49, 108, 113, 43, 94, 37, 33, 57, 42, 127, 36, 105, 109, 54, 34, 102, 14, 120, 97, 75, 39]], "model.layers.23.self_attn.k_proj": [[47, 111, 36, 107, 88, 96, 30, 90, 80, 21, 0, 78, 81, 2, 27, 104, 68, 84, 83, 7, 15, 75, 74, 93, 95, 5, 103, 106, 10, 77, 11, 86, 124, 110, 22, 6, 120, 91, 59, 71, 73, 85, 43, 122, 46, 127, 13, 51, 67, 94, 119, 114, 48, 32, 50, 105, 55, 92, 101, 23, 66, 118, 42, 52, 33, 97, 56, 63, 60, 102, 125, 35, 39, 123, 72, 29, 31, 99, 116, 64, 19, 14, 45, 41, 53, 61, 112, 28, 12, 117, 49, 58, 113, 16, 40, 34, 57, 17, 25, 38, 108, 62, 3, 79, 98, 126, 37, 54, 109, 121, 18, 4, 44, 115, 24, 20, 26, 82, 89, 87, 65, 8, 76, 9, 100, 1, 70, 69], [39, 56, 46, 51, 86, 33, 110, 116, 61, 53, 30, 20, 60, 47, 57, 124, 55, 59, 120, 80, 89, 58, 114, 119, 52, 122, 127, 112, 49, 91, 126, 123, 63, 54, 118, 48, 115, 121, 44, 113, 62, 34, 125, 45, 106, 109, 117, 92, 50, 111, 101, 96, 100, 93, 14, 94, 108, 105, 107, 88, 98, 43, 42, 35, 40, 24, 37, 78, 104, 38, 41, 36, 11, 102, 97, 95, 27, 16, 90, 82, 29, 65, 72, 13, 31, 19, 84, 71, 99, 64, 5, 73, 26, 87, 85, 3, 32, 25, 10, 28, 12, 17, 21, 15, 69, 9, 18, 7, 23, 70, 6, 0, 76, 1, 79, 77, 103, 75, 83, 4, 67, 22, 68, 66, 81, 8, 2, 74], [46, 113, 38, 110, 97, 28, 19, 86, 79, 26, 94, 24, 81, 21, 87, 77, 122, 14, 70, 74, 47, 17, 78, 60, 68, 9, 59, 52, 126, 65, 57, 112, 111, 119, 62, 44, 8, 55, 114, 20, 105, 115, 56, 127, 108, 117, 106, 85, 58, 76, 104, 0, 125, 50, 11, 42, 64, 93, 118, 45, 41, 109, 13, 2, 43, 103, 80, 124, 92, 48, 123, 63, 51, 116, 39, 16, 121, 120, 36, 61, 66, 100, 53, 99, 75, 54, 40, 18, 89, 101, 49, 37, 107, 90, 15, 73, 95, 5, 29, 72, 32, 71, 23, 3, 35, 25, 31, 34, 96, 84, 98, 12, 91, 27, 30, 22, 69, 82, 10, 33, 88, 7, 6, 83, 4, 67, 1, 102], [117, 103, 119, 86, 48, 97, 112, 13, 93, 11, 83, 15, 56, 26, 60, 120, 124, 47, 121, 6, 89, 113, 53, 122, 116, 63, 58, 127, 55, 125, 115, 24, 9, 114, 123, 7, 52, 49, 118, 51, 61, 62, 91, 59, 54, 126, 18, 107, 81, 111, 45, 42, 50, 46, 44, 57, 109, 16, 110, 20, 43, 108, 74, 41, 73, 98, 104, 29, 92, 30, 23, 105, 40, 25, 106, 37, 88, 35, 38, 21, 68, 17, 64, 100, 101, 78, 102, 31, 14, 27, 36, 99, 2, 95, 28, 8, 32, 96, 94, 34, 69, 85, 10, 82, 4, 19, 87, 84, 66, 90, 80, 12, 76, 79, 75, 65, 33, 70, 71, 5, 77, 72, 67, 0, 39, 1, 3, 22], [108, 101, 44, 57, 21, 80, 28, 14, 24, 11, 71, 33, 18, 76, 73, 90, 58, 125, 19, 67, 50, 5, 64, 17, 1, 52, 55, 15, 118, 117, 121, 51, 115, 122, 53, 119, 4, 47, 48, 69, 75, 124, 123, 120, 95, 45, 35, 31, 111, 2, 59, 13, 87, 126, 12, 104, 61, 60, 56, 72, 110, 42, 127, 41, 114, 62, 43, 112, 8, 84, 7, 63, 109, 107, 68, 116, 54, 23, 40, 98, 106, 88, 91, 39, 6, 46, 49, 27, 105, 10, 70, 89, 113, 26, 30, 86, 36, 29, 34, 103, 32, 100, 92, 38, 81, 102, 94, 22, 78, 93, 74, 3, 66, 16, 0, 97, 77, 99, 25, 79, 9, 96, 82, 20, 85, 83, 37, 65], [127, 54, 101, 97, 86, 24, 27, 123, 118, 18, 94, 77, 113, 63, 116, 120, 48, 17, 10, 64, 58, 85, 62, 59, 117, 57, 15, 38, 114, 12, 111, 67, 42, 121, 122, 14, 108, 29, 55, 61, 7, 125, 53, 126, 60, 49, 103, 1, 51, 119, 19, 56, 16, 52, 112, 124, 43, 109, 50, 5, 105, 45, 110, 46, 115, 47, 31, 72, 75, 71, 37, 90, 6, 25, 66, 107, 40, 44, 20, 39, 89, 87, 95, 104, 106, 83, 36, 100, 41, 34, 102, 93, 98, 68, 96, 11, 99, 0, 2, 35, 3, 21, 26, 92, 9, 79, 23, 30, 28, 80, 32, 78, 91, 81, 84, 76, 8, 88, 73, 69, 74, 13, 82, 33, 70, 22, 4, 65], [107, 33, 121, 36, 22, 91, 43, 124, 88, 45, 61, 80, 21, 51, 54, 60, 57, 49, 74, 55, 119, 123, 58, 115, 117, 29, 116, 62, 63, 122, 127, 50, 26, 125, 53, 52, 111, 42, 118, 59, 112, 113, 109, 120, 126, 46, 110, 56, 48, 38, 47, 76, 95, 108, 101, 19, 114, 39, 14, 65, 17, 106, 66, 44, 69, 4, 78, 20, 104, 12, 41, 0, 105, 98, 8, 35, 37, 15, 30, 67, 103, 70, 64, 83, 102, 6, 23, 40, 99, 31, 32, 34, 94, 92, 5, 81, 10, 87, 73, 96, 28, 25, 100, 82, 9, 97, 18, 11, 93, 90, 89, 79, 71, 27, 72, 86, 85, 77, 84, 13, 1, 7, 3, 75, 68, 2, 24, 16], [115, 39, 13, 72, 51, 80, 64, 83, 10, 70, 85, 68, 66, 90, 62, 2, 124, 0, 65, 76, 43, 111, 26, 60, 91, 112, 118, 75, 53, 55, 61, 56, 67, 5, 81, 88, 63, 4, 119, 89, 46, 42, 40, 34, 59, 52, 50, 108, 73, 29, 123, 1, 93, 82, 114, 15, 121, 105, 23, 35, 36, 71, 126, 30, 31, 22, 96, 7, 122, 125, 84, 69, 41, 110, 58, 3, 102, 79, 25, 109, 87, 86, 24, 120, 94, 57, 127, 38, 14, 98, 28, 100, 104, 99, 6, 106, 117, 44, 33, 107, 54, 45, 8, 48, 101, 95, 78, 97, 49, 116, 17, 12, 47, 18, 92, 27, 32, 20, 9, 113, 37, 11, 74, 21, 19, 77, 16, 103]], "model.layers.23.self_attn.qk_proj": [[46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 117, 107, 51, 44, 121, 48, 43, 113, 22, 24, 26, 21, 55, 83, 85, 57, 62, 19, 92, 27, 124, 101, 80, 88, 118, 90, 63, 39, 37, 16, 59, 86, 45, 60, 123, 103, 61, 33, 13, 116, 58, 125, 49, 122, 81, 77, 74, 97, 14, 10, 120, 78, 50, 72, 79, 53, 52, 17, 36, 28, 93, 15, 91, 94, 29, 70, 106, 112, 11, 126, 75, 68, 109, 8, 4, 64, 82, 114, 0, 89, 71, 18, 100, 2, 25, 9, 76, 66, 102, 38, 12, 30, 105, 23, 73, 7, 42, 5, 96, 32, 20, 84, 104, 40, 95, 87, 69, 6, 41, 98, 1, 31, 34, 67, 99, 35, 65, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 57, 21, 22, 26, 83, 124, 27, 118, 39, 24, 92, 62, 85, 63, 55, 101, 19, 88, 16, 60, 86, 90, 80, 58, 37, 59, 123, 116, 33, 125, 74, 61, 103, 49, 77, 45, 78, 13, 10, 81, 120, 112, 52, 70, 91, 122, 53, 14, 72, 97, 50, 17, 15, 93, 114, 94, 28, 4, 0, 11, 29, 36, 106, 126, 2, 66, 79, 109, 75, 8, 82, 68, 102, 30, 38, 25, 105, 100, 64, 76, 73, 12, 104, 23, 18, 42, 89, 7, 20, 9, 71, 5, 96, 31, 69, 32, 98, 84, 95, 6, 40, 41, 87, 3, 34, 35, 65, 99, 1, 67], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 124, 26, 62, 85, 21, 24, 118, 83, 27, 55, 39, 92, 37, 63, 19, 101, 60, 90, 80, 88, 33, 58, 16, 86, 103, 59, 97, 61, 123, 125, 74, 53, 77, 122, 116, 45, 14, 13, 78, 91, 120, 10, 52, 72, 81, 49, 112, 70, 36, 66, 4, 28, 50, 15, 17, 109, 93, 64, 79, 11, 29, 126, 2, 114, 38, 94, 0, 68, 106, 25, 12, 82, 75, 104, 100, 23, 76, 89, 18, 42, 71, 7, 30, 84, 8, 96, 105, 41, 20, 5, 9, 32, 98, 73, 102, 40, 69, 87, 35, 31, 95, 6, 34, 99, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 47, 110, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 57, 22, 21, 83, 63, 124, 26, 62, 118, 60, 19, 24, 85, 92, 39, 88, 58, 27, 55, 80, 101, 37, 16, 122, 53, 123, 86, 59, 90, 77, 61, 33, 125, 10, 116, 17, 81, 13, 14, 78, 120, 103, 74, 45, 72, 112, 49, 97, 15, 70, 52, 4, 50, 91, 11, 66, 114, 106, 126, 2, 109, 36, 100, 68, 29, 28, 93, 94, 0, 18, 75, 76, 79, 12, 71, 38, 89, 8, 102, 7, 69, 64, 42, 73, 82, 25, 9, 104, 30, 84, 20, 5, 96, 105, 23, 6, 31, 41, 32, 98, 87, 95, 35, 65, 67, 40, 99, 34, 3, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 117, 107, 51, 44, 48, 121, 43, 113, 57, 62, 83, 124, 63, 22, 21, 19, 26, 85, 55, 24, 60, 27, 101, 39, 118, 86, 16, 37, 92, 58, 90, 122, 123, 88, 80, 61, 33, 103, 97, 72, 59, 14, 13, 10, 77, 45, 74, 49, 125, 52, 4, 81, 78, 68, 112, 17, 50, 116, 75, 53, 120, 91, 66, 79, 15, 28, 11, 109, 0, 126, 106, 93, 114, 29, 64, 2, 36, 12, 94, 70, 6, 7, 18, 20, 76, 25, 100, 82, 71, 38, 102, 9, 23, 73, 69, 89, 8, 42, 30, 5, 105, 84, 40, 104, 41, 32, 98, 96, 35, 34, 87, 31, 99, 95, 67, 3, 1, 65], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 21, 26, 19, 27, 55, 24, 83, 62, 85, 101, 63, 39, 16, 80, 124, 88, 86, 58, 92, 60, 118, 37, 122, 72, 90, 123, 10, 33, 77, 45, 74, 13, 61, 81, 14, 97, 59, 125, 103, 17, 112, 49, 15, 78, 116, 75, 120, 53, 52, 0, 50, 91, 11, 68, 126, 28, 79, 64, 94, 66, 29, 82, 106, 114, 20, 36, 100, 6, 7, 4, 12, 76, 93, 25, 18, 23, 109, 38, 71, 2, 69, 102, 89, 30, 8, 105, 9, 70, 84, 73, 96, 42, 87, 32, 34, 95, 41, 5, 104, 65, 98, 40, 1, 35, 67, 31, 99, 3], [46, 115, 111, 127, 54, 56, 110, 47, 108, 119, 107, 51, 121, 44, 48, 117, 43, 113, 26, 22, 57, 21, 27, 85, 24, 83, 19, 88, 124, 92, 62, 80, 16, 101, 90, 86, 63, 39, 58, 60, 55, 77, 37, 122, 123, 33, 14, 103, 125, 74, 118, 10, 13, 97, 72, 81, 120, 45, 61, 59, 17, 78, 112, 116, 28, 49, 91, 106, 93, 50, 15, 79, 36, 53, 6, 126, 64, 114, 75, 82, 11, 52, 94, 109, 29, 68, 12, 89, 20, 38, 18, 25, 100, 4, 8, 66, 76, 23, 84, 102, 7, 0, 9, 71, 2, 73, 87, 42, 96, 105, 30, 32, 98, 31, 69, 70, 34, 5, 1, 41, 95, 104, 40, 35, 65, 99, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 119, 107, 51, 117, 44, 121, 48, 43, 113, 27, 21, 22, 24, 83, 57, 85, 26, 60, 92, 62, 19, 124, 55, 16, 80, 88, 90, 39, 86, 101, 123, 63, 33, 37, 61, 118, 58, 125, 59, 49, 77, 122, 116, 13, 97, 74, 103, 14, 45, 10, 81, 78, 112, 79, 17, 120, 28, 114, 36, 6, 91, 50, 52, 126, 72, 53, 94, 93, 11, 82, 106, 75, 4, 100, 109, 68, 29, 15, 18, 8, 0, 38, 23, 64, 89, 2, 25, 12, 105, 102, 66, 32, 73, 76, 30, 71, 20, 42, 96, 84, 9, 104, 87, 7, 70, 31, 98, 95, 5, 69, 65, 40, 41, 35, 34, 99, 1, 67, 3], [46, 115, 111, 127, 56, 54, 110, 47, 108, 107, 117, 119, 51, 121, 44, 48, 43, 113, 57, 83, 85, 24, 26, 21, 124, 22, 92, 27, 19, 88, 60, 55, 16, 62, 90, 123, 39, 101, 63, 86, 118, 58, 80, 33, 13, 116, 61, 37, 125, 97, 77, 74, 49, 45, 120, 103, 112, 81, 59, 17, 28, 122, 53, 10, 14, 52, 91, 78, 93, 79, 8, 36, 6, 29, 126, 11, 106, 75, 50, 100, 72, 114, 15, 12, 82, 109, 68, 94, 102, 23, 38, 4, 18, 76, 42, 73, 0, 2, 66, 105, 32, 104, 7, 89, 30, 41, 96, 31, 9, 64, 20, 25, 87, 98, 84, 5, 71, 95, 70, 34, 35, 40, 99, 69, 65, 3, 1, 67], [46, 115, 111, 127, 56, 54, 47, 110, 108, 117, 107, 51, 119, 44, 121, 48, 43, 113, 57, 62, 21, 26, 123, 85, 19, 124, 27, 83, 24, 22, 92, 118, 55, 39, 60, 16, 88, 86, 37, 58, 90, 63, 101, 33, 10, 80, 116, 61, 77, 45, 125, 8, 13, 103, 122, 59, 112, 74, 49, 97, 120, 17, 78, 52, 53, 14, 114, 2, 0, 81, 91, 28, 68, 4, 94, 36, 15, 93, 75, 29, 79, 50, 64, 109, 72, 6, 12, 106, 66, 126, 38, 71, 11, 76, 42, 89, 70, 82, 100, 18, 7, 73, 96, 25, 5, 9, 20, 105, 102, 23, 87, 104, 32, 30, 31, 69, 98, 41, 40, 84, 99, 1, 67, 34, 65, 95, 35, 3], [46, 115, 111, 127, 54, 47, 56, 110, 108, 117, 51, 119, 107, 44, 121, 43, 48, 113, 57, 124, 62, 22, 21, 85, 39, 19, 26, 80, 83, 123, 86, 92, 24, 16, 118, 60, 63, 88, 27, 58, 77, 37, 101, 55, 125, 61, 10, 13, 116, 90, 45, 103, 8, 120, 74, 97, 33, 59, 28, 52, 4, 14, 81, 75, 78, 112, 49, 17, 53, 50, 66, 68, 15, 91, 122, 11, 94, 79, 114, 29, 70, 64, 36, 71, 82, 2, 109, 106, 126, 0, 6, 100, 7, 12, 76, 42, 93, 20, 38, 25, 32, 72, 23, 9, 84, 73, 5, 18, 105, 89, 102, 69, 30, 104, 87, 96, 31, 95, 34, 98, 40, 99, 67, 3, 41, 1, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 107, 119, 51, 117, 44, 121, 48, 43, 113, 22, 21, 57, 85, 26, 83, 19, 24, 92, 124, 80, 88, 27, 86, 101, 39, 62, 63, 16, 90, 60, 58, 55, 33, 13, 77, 10, 8, 125, 118, 14, 37, 123, 74, 61, 97, 116, 45, 81, 52, 28, 49, 103, 91, 78, 59, 53, 120, 17, 79, 15, 70, 122, 75, 4, 112, 68, 11, 50, 36, 64, 94, 29, 93, 109, 12, 23, 126, 100, 106, 18, 114, 2, 38, 82, 25, 0, 9, 7, 76, 66, 71, 30, 84, 73, 20, 89, 102, 105, 32, 72, 6, 5, 96, 34, 98, 95, 69, 104, 87, 31, 40, 41, 42, 99, 1, 67, 35, 65, 3], [46, 115, 111, 127, 56, 110, 54, 47, 108, 107, 119, 117, 51, 44, 121, 48, 43, 113, 21, 57, 27, 24, 22, 85, 26, 60, 92, 19, 88, 80, 90, 55, 101, 83, 61, 16, 39, 86, 62, 124, 33, 58, 59, 63, 123, 49, 125, 122, 116, 91, 13, 103, 37, 74, 8, 77, 118, 97, 10, 52, 120, 81, 14, 45, 29, 53, 100, 17, 94, 28, 112, 70, 50, 36, 78, 15, 126, 93, 106, 79, 114, 64, 109, 38, 25, 82, 4, 18, 75, 23, 2, 102, 89, 12, 68, 105, 30, 0, 20, 11, 76, 71, 9, 72, 87, 66, 42, 96, 31, 7, 104, 98, 73, 41, 99, 65, 34, 84, 5, 32, 69, 95, 6, 35, 40, 1, 3, 67], [46, 115, 111, 127, 54, 110, 56, 47, 108, 117, 119, 51, 107, 44, 121, 48, 43, 113, 57, 22, 60, 21, 85, 19, 26, 55, 63, 80, 62, 27, 24, 123, 124, 92, 88, 101, 58, 86, 83, 39, 118, 61, 16, 59, 13, 10, 122, 37, 90, 45, 8, 74, 120, 81, 33, 49, 125, 52, 53, 77, 70, 112, 17, 116, 103, 14, 91, 28, 97, 78, 126, 64, 114, 93, 2, 68, 15, 94, 79, 11, 100, 50, 36, 4, 29, 0, 109, 106, 66, 75, 89, 12, 38, 76, 82, 18, 9, 42, 20, 30, 72, 23, 104, 7, 71, 105, 32, 102, 96, 69, 95, 41, 73, 25, 5, 84, 87, 6, 31, 98, 34, 1, 99, 67, 40, 35, 3, 65], [46, 115, 111, 127, 54, 56, 47, 110, 108, 51, 117, 107, 119, 44, 48, 121, 43, 113, 22, 57, 21, 19, 85, 26, 83, 24, 124, 62, 123, 55, 80, 27, 63, 92, 39, 88, 37, 16, 90, 60, 101, 61, 13, 58, 97, 86, 74, 8, 125, 103, 33, 118, 45, 116, 10, 77, 120, 122, 81, 52, 49, 78, 70, 94, 14, 91, 15, 59, 93, 36, 50, 68, 112, 17, 28, 66, 79, 0, 64, 53, 75, 106, 4, 2, 29, 11, 72, 102, 100, 109, 114, 71, 12, 126, 30, 7, 6, 76, 20, 82, 18, 105, 73, 9, 32, 25, 96, 42, 89, 38, 104, 69, 23, 41, 87, 5, 40, 35, 84, 98, 99, 34, 31, 95, 65, 67, 1, 3], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 117, 51, 119, 44, 121, 48, 43, 113, 22, 57, 21, 19, 26, 124, 85, 62, 80, 123, 39, 83, 55, 88, 24, 16, 92, 60, 61, 86, 27, 13, 37, 90, 33, 77, 63, 101, 125, 118, 103, 10, 58, 74, 45, 49, 8, 59, 14, 78, 97, 122, 116, 81, 91, 17, 50, 120, 75, 28, 53, 93, 112, 52, 79, 11, 29, 68, 15, 94, 36, 70, 72, 100, 109, 4, 102, 64, 114, 126, 18, 76, 106, 6, 2, 73, 82, 71, 7, 12, 66, 9, 20, 32, 23, 89, 30, 0, 105, 25, 84, 38, 87, 69, 96, 41, 104, 42, 95, 40, 5, 98, 31, 3, 1, 34, 99, 35, 67, 65], [46, 115, 127, 111, 54, 56, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 22, 21, 85, 57, 88, 26, 27, 19, 80, 24, 83, 92, 62, 86, 90, 124, 39, 55, 101, 61, 60, 33, 123, 16, 125, 103, 63, 116, 13, 97, 91, 58, 122, 37, 118, 10, 45, 77, 17, 59, 49, 74, 81, 14, 78, 94, 36, 112, 28, 93, 120, 52, 114, 29, 6, 72, 79, 53, 15, 102, 0, 8, 106, 100, 68, 18, 25, 126, 64, 82, 38, 50, 11, 12, 2, 109, 30, 75, 4, 23, 89, 66, 105, 71, 76, 41, 7, 32, 87, 96, 20, 84, 70, 73, 9, 98, 69, 5, 40, 95, 42, 65, 31, 104, 1, 99, 34, 35, 67, 3], [46, 115, 127, 111, 56, 54, 110, 47, 108, 107, 119, 51, 117, 44, 48, 121, 43, 113, 57, 22, 85, 83, 21, 19, 26, 124, 92, 86, 27, 62, 24, 60, 80, 63, 39, 123, 88, 101, 61, 103, 125, 13, 55, 90, 16, 37, 58, 10, 118, 77, 74, 120, 14, 91, 59, 33, 122, 97, 116, 53, 72, 94, 6, 112, 17, 45, 81, 36, 52, 49, 28, 114, 78, 15, 29, 79, 126, 4, 109, 75, 100, 50, 66, 2, 0, 11, 93, 82, 64, 106, 68, 12, 38, 102, 89, 18, 76, 8, 25, 84, 71, 30, 7, 87, 73, 104, 105, 32, 20, 9, 42, 96, 23, 41, 40, 98, 69, 31, 5, 65, 70, 34, 99, 95, 35, 1, 3, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 57, 22, 62, 85, 83, 26, 21, 123, 124, 19, 24, 63, 80, 92, 101, 39, 90, 58, 103, 60, 86, 88, 55, 45, 120, 118, 61, 125, 37, 74, 33, 27, 97, 16, 10, 53, 122, 116, 13, 77, 50, 72, 59, 91, 49, 52, 28, 78, 14, 114, 126, 112, 2, 6, 81, 93, 94, 79, 29, 11, 4, 17, 15, 0, 68, 106, 75, 36, 64, 102, 12, 18, 32, 71, 109, 66, 38, 82, 89, 105, 76, 8, 41, 73, 100, 7, 9, 25, 42, 31, 104, 20, 84, 98, 23, 69, 40, 30, 96, 5, 99, 87, 95, 70, 35, 65, 34, 3, 67, 1], [46, 115, 127, 111, 56, 54, 47, 110, 108, 119, 107, 117, 44, 51, 48, 121, 43, 113, 57, 62, 21, 22, 19, 39, 124, 24, 26, 101, 63, 83, 85, 90, 92, 123, 88, 27, 58, 16, 55, 86, 61, 80, 33, 103, 60, 10, 37, 13, 125, 120, 118, 116, 45, 97, 59, 77, 49, 112, 72, 74, 78, 122, 53, 52, 91, 29, 126, 81, 6, 75, 28, 94, 68, 15, 106, 114, 14, 93, 79, 0, 4, 36, 17, 109, 11, 64, 66, 102, 82, 2, 100, 50, 12, 32, 84, 30, 7, 76, 38, 89, 71, 9, 41, 18, 105, 25, 23, 8, 73, 42, 96, 5, 69, 87, 20, 104, 40, 70, 34, 98, 95, 31, 35, 67, 65, 99, 1, 3], [46, 115, 111, 127, 47, 54, 56, 110, 108, 119, 107, 117, 51, 44, 121, 48, 43, 113, 22, 57, 62, 124, 63, 85, 26, 19, 21, 83, 24, 39, 92, 27, 55, 61, 37, 88, 101, 123, 125, 49, 80, 58, 86, 16, 118, 60, 90, 59, 45, 72, 116, 53, 33, 52, 13, 77, 97, 120, 10, 74, 103, 91, 112, 78, 81, 14, 68, 122, 94, 2, 15, 28, 79, 93, 36, 75, 29, 126, 17, 6, 100, 64, 114, 109, 11, 82, 106, 25, 18, 0, 4, 50, 89, 32, 76, 73, 30, 7, 38, 66, 20, 71, 70, 12, 41, 84, 102, 40, 105, 9, 23, 87, 104, 5, 69, 42, 34, 98, 8, 96, 95, 1, 31, 67, 35, 65, 99, 3], [46, 115, 127, 111, 56, 54, 47, 110, 119, 108, 107, 51, 117, 48, 44, 121, 43, 113, 57, 62, 63, 85, 124, 22, 24, 60, 19, 21, 123, 83, 26, 92, 90, 27, 39, 53, 101, 80, 88, 61, 49, 86, 59, 37, 16, 58, 116, 74, 118, 120, 103, 125, 72, 13, 122, 10, 55, 45, 91, 112, 33, 52, 77, 14, 78, 97, 64, 114, 81, 17, 68, 29, 126, 93, 79, 70, 28, 36, 11, 15, 89, 100, 2, 106, 94, 50, 66, 0, 4, 75, 18, 38, 20, 109, 76, 82, 32, 7, 42, 30, 25, 12, 6, 84, 41, 71, 105, 8, 9, 73, 69, 31, 87, 96, 102, 104, 5, 95, 23, 65, 35, 99, 98, 34, 3, 1, 40, 67], [46, 115, 127, 111, 47, 56, 54, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 62, 124, 22, 85, 21, 19, 27, 24, 92, 101, 90, 59, 63, 26, 60, 123, 83, 61, 39, 86, 55, 80, 125, 33, 118, 16, 88, 37, 49, 58, 116, 103, 120, 13, 77, 45, 74, 52, 97, 122, 10, 53, 112, 91, 14, 70, 72, 81, 50, 29, 17, 68, 15, 79, 78, 114, 28, 36, 94, 64, 100, 109, 93, 4, 126, 18, 38, 106, 75, 76, 11, 8, 66, 2, 89, 42, 0, 82, 105, 25, 7, 9, 12, 20, 73, 96, 30, 102, 71, 23, 31, 32, 84, 87, 41, 69, 5, 104, 99, 6, 98, 95, 40, 34, 35, 67, 3, 65, 1], [46, 115, 127, 111, 54, 47, 56, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 22, 57, 21, 19, 24, 85, 62, 124, 27, 101, 83, 26, 92, 39, 88, 60, 63, 16, 86, 55, 123, 90, 80, 13, 37, 33, 58, 74, 45, 61, 77, 103, 118, 125, 97, 59, 120, 14, 49, 10, 81, 70, 72, 52, 91, 122, 114, 17, 116, 53, 36, 29, 78, 94, 15, 11, 8, 93, 75, 50, 28, 112, 109, 79, 106, 126, 68, 12, 0, 76, 82, 38, 4, 30, 32, 18, 66, 100, 25, 89, 42, 7, 71, 64, 20, 23, 73, 105, 9, 2, 102, 84, 41, 40, 96, 95, 87, 5, 98, 104, 6, 69, 31, 34, 1, 3, 99, 67, 65, 35], [46, 115, 111, 127, 54, 56, 47, 110, 108, 119, 107, 51, 117, 44, 48, 121, 43, 113, 22, 85, 26, 21, 124, 24, 19, 83, 57, 62, 88, 55, 63, 27, 39, 90, 60, 16, 92, 101, 103, 86, 80, 123, 59, 74, 37, 58, 33, 125, 45, 10, 77, 118, 97, 13, 49, 91, 120, 14, 61, 116, 53, 52, 17, 112, 81, 70, 78, 122, 93, 79, 94, 114, 15, 11, 8, 72, 28, 64, 126, 36, 0, 75, 18, 29, 106, 109, 82, 68, 50, 2, 12, 4, 25, 7, 66, 89, 20, 71, 100, 38, 76, 32, 84, 30, 105, 9, 23, 5, 42, 73, 102, 96, 6, 69, 98, 87, 104, 41, 40, 31, 95, 34, 1, 65, 35, 3, 99, 67], [46, 115, 127, 111, 54, 56, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 21, 22, 85, 124, 27, 60, 26, 19, 83, 92, 62, 24, 123, 63, 88, 86, 61, 90, 59, 55, 101, 39, 80, 16, 33, 125, 118, 49, 58, 53, 103, 97, 120, 74, 91, 37, 116, 13, 112, 77, 52, 45, 122, 10, 94, 36, 78, 93, 8, 81, 106, 14, 79, 50, 29, 70, 126, 114, 100, 28, 17, 4, 109, 75, 66, 64, 12, 15, 18, 68, 11, 82, 25, 30, 38, 89, 0, 41, 42, 105, 76, 72, 20, 102, 23, 2, 7, 104, 71, 32, 9, 95, 40, 96, 73, 87, 6, 84, 99, 5, 34, 69, 31, 98, 65, 67, 1, 3, 35], [46, 115, 127, 111, 54, 47, 56, 110, 108, 107, 117, 119, 51, 44, 48, 121, 43, 113, 57, 21, 85, 60, 124, 26, 125, 83, 22, 19, 62, 92, 55, 27, 61, 39, 88, 123, 24, 63, 90, 118, 80, 101, 59, 120, 13, 86, 37, 58, 16, 33, 8, 45, 52, 53, 97, 77, 103, 49, 74, 114, 81, 112, 10, 126, 122, 28, 116, 14, 50, 109, 91, 17, 15, 78, 29, 79, 36, 94, 75, 11, 82, 4, 106, 68, 100, 93, 6, 2, 38, 104, 76, 66, 12, 64, 42, 0, 32, 30, 25, 41, 70, 20, 102, 89, 71, 105, 18, 9, 96, 87, 7, 23, 73, 84, 40, 95, 98, 31, 34, 69, 99, 72, 35, 5, 1, 67, 3, 65], [46, 115, 111, 127, 56, 47, 54, 110, 108, 107, 117, 119, 51, 44, 121, 48, 43, 113, 57, 21, 62, 22, 124, 19, 24, 85, 26, 83, 63, 88, 92, 80, 27, 118, 55, 58, 39, 123, 120, 60, 101, 90, 16, 53, 13, 86, 8, 74, 103, 61, 45, 125, 37, 49, 33, 77, 59, 10, 52, 122, 97, 78, 14, 112, 6, 116, 81, 91, 79, 94, 0, 114, 17, 15, 68, 11, 29, 28, 66, 75, 126, 4, 109, 100, 50, 12, 2, 36, 106, 38, 93, 76, 89, 82, 102, 7, 18, 71, 64, 42, 32, 5, 84, 73, 20, 25, 23, 70, 30, 105, 72, 9, 87, 69, 96, 104, 98, 95, 67, 31, 41, 40, 34, 65, 1, 3, 35, 99], [46, 115, 111, 127, 54, 47, 56, 110, 108, 119, 51, 107, 117, 44, 121, 48, 43, 113, 57, 62, 22, 26, 55, 21, 83, 124, 24, 19, 85, 39, 45, 118, 27, 63, 92, 88, 37, 60, 101, 90, 123, 61, 8, 80, 86, 59, 74, 97, 33, 16, 125, 120, 49, 13, 103, 52, 53, 58, 77, 122, 114, 6, 78, 10, 68, 14, 50, 0, 81, 11, 17, 29, 91, 116, 66, 106, 112, 75, 15, 4, 79, 94, 28, 64, 126, 93, 109, 100, 71, 36, 76, 102, 18, 7, 2, 73, 25, 32, 12, 84, 38, 89, 82, 105, 9, 98, 5, 72, 30, 69, 96, 41, 35, 42, 23, 20, 34, 95, 87, 70, 40, 99, 104, 65, 31, 67, 3, 1], [46, 115, 111, 127, 56, 54, 47, 110, 108, 119, 107, 117, 51, 44, 48, 121, 43, 113, 57, 22, 26, 19, 62, 55, 85, 21, 24, 39, 27, 124, 83, 92, 86, 88, 60, 101, 16, 45, 80, 123, 58, 118, 90, 59, 37, 61, 63, 13, 33, 8, 103, 125, 77, 74, 97, 81, 122, 120, 53, 10, 14, 49, 78, 6, 112, 116, 17, 68, 52, 11, 91, 114, 29, 50, 94, 15, 36, 75, 28, 79, 93, 18, 4, 109, 100, 66, 106, 126, 25, 82, 0, 102, 76, 12, 38, 71, 105, 20, 72, 30, 7, 9, 2, 89, 73, 32, 64, 23, 5, 104, 87, 95, 84, 69, 70, 98, 40, 42, 96, 41, 34, 31, 35, 67, 1, 3, 65, 99], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 51, 44, 48, 121, 43, 113, 26, 22, 57, 21, 62, 27, 124, 88, 19, 24, 55, 85, 63, 60, 39, 92, 59, 83, 16, 86, 118, 80, 101, 58, 45, 37, 103, 125, 90, 53, 123, 74, 61, 97, 8, 33, 10, 13, 116, 120, 14, 49, 122, 77, 91, 78, 112, 81, 6, 52, 0, 15, 68, 36, 93, 17, 114, 94, 102, 29, 50, 11, 2, 75, 4, 28, 126, 109, 64, 7, 79, 76, 18, 71, 100, 66, 12, 89, 25, 82, 70, 72, 38, 106, 9, 20, 5, 30, 69, 32, 73, 98, 105, 84, 104, 23, 87, 96, 1, 3, 95, 65, 42, 41, 40, 34, 31, 35, 99, 67], [46, 115, 127, 111, 56, 54, 47, 110, 108, 107, 119, 117, 44, 51, 121, 48, 43, 113, 57, 26, 22, 62, 19, 27, 80, 21, 24, 92, 63, 39, 124, 55, 85, 88, 83, 86, 60, 90, 16, 123, 101, 118, 37, 58, 61, 77, 59, 33, 97, 45, 120, 10, 125, 116, 49, 74, 13, 53, 103, 112, 122, 8, 81, 78, 14, 17, 93, 91, 36, 50, 29, 79, 28, 11, 52, 15, 94, 114, 75, 106, 76, 72, 126, 18, 68, 25, 12, 100, 82, 6, 109, 102, 70, 64, 7, 38, 66, 84, 73, 9, 71, 89, 4, 42, 105, 20, 30, 96, 23, 2, 32, 0, 69, 41, 98, 104, 87, 95, 31, 5, 34, 99, 40, 65, 35, 1, 3, 67]], "model.layers.24.self_attn.q_proj": [[112, 37, 48, 54, 90, 41, 94, 86, 30, 101, 56, 25, 59, 81, 62, 19, 20, 106, 58, 44, 63, 121, 123, 89, 105, 114, 12, 84, 9, 22, 95, 35, 74, 113, 46, 77, 52, 97, 104, 70, 2, 16, 24, 107, 117, 124, 60, 39, 18, 43, 119, 31, 88, 14, 125, 79, 122, 26, 10, 126, 109, 49, 4, 108, 68, 67, 80, 50, 45, 92, 8, 47, 120, 111, 29, 57, 0, 32, 115, 98, 51, 61, 118, 53, 99, 27, 71, 28, 116, 42, 110, 23, 103, 93, 102, 36, 127, 33, 91, 38, 40, 55, 100, 3, 21, 87, 17, 15, 78, 34, 85, 75, 1, 5, 96, 82, 13, 83, 65, 7, 69, 73, 76, 11, 66, 64, 72, 6], [112, 37, 48, 54, 122, 90, 111, 94, 30, 86, 125, 25, 101, 117, 41, 81, 24, 58, 19, 89, 88, 22, 35, 121, 59, 84, 45, 57, 126, 114, 109, 53, 20, 63, 106, 42, 105, 77, 12, 31, 119, 102, 120, 55, 104, 46, 10, 82, 60, 124, 3, 113, 14, 50, 18, 123, 70, 62, 100, 97, 49, 56, 47, 8, 52, 110, 44, 61, 74, 21, 26, 16, 107, 118, 87, 36, 9, 43, 103, 29, 40, 116, 65, 115, 95, 64, 93, 4, 27, 79, 32, 34, 51, 108, 28, 23, 17, 92, 91, 85, 127, 15, 96, 33, 2, 38, 39, 99, 80, 78, 98, 72, 73, 5, 83, 71, 11, 6, 75, 13, 7, 66, 68, 67, 0, 69, 76, 1], [112, 37, 48, 89, 25, 90, 54, 86, 94, 30, 56, 24, 18, 19, 35, 41, 22, 45, 105, 43, 20, 59, 50, 101, 116, 127, 97, 42, 81, 117, 125, 63, 16, 77, 60, 123, 88, 49, 95, 85, 38, 47, 9, 121, 106, 122, 109, 12, 114, 84, 115, 119, 100, 67, 0, 108, 55, 40, 26, 31, 110, 58, 70, 87, 51, 103, 104, 36, 102, 27, 92, 124, 111, 79, 2, 62, 46, 61, 13, 107, 91, 113, 53, 23, 126, 57, 74, 32, 34, 71, 39, 52, 33, 120, 29, 14, 82, 93, 99, 98, 44, 83, 96, 21, 15, 118, 69, 66, 8, 28, 78, 17, 10, 80, 7, 1, 75, 68, 76, 3, 11, 4, 72, 73, 6, 65, 64, 5], [112, 37, 48, 54, 90, 41, 94, 84, 81, 86, 117, 56, 62, 88, 30, 14, 20, 105, 19, 111, 24, 25, 51, 125, 101, 109, 49, 114, 10, 120, 63, 45, 58, 47, 31, 8, 60, 70, 126, 59, 43, 64, 77, 55, 121, 22, 52, 92, 102, 42, 119, 18, 57, 118, 53, 66, 3, 124, 4, 122, 113, 127, 65, 97, 106, 108, 89, 35, 123, 44, 107, 50, 16, 34, 39, 78, 103, 74, 110, 61, 26, 46, 32, 98, 104, 99, 38, 116, 40, 28, 12, 69, 17, 72, 100, 80, 21, 115, 93, 87, 1, 95, 33, 83, 29, 36, 85, 73, 96, 67, 5, 79, 91, 11, 2, 27, 9, 23, 71, 68, 75, 15, 0, 82, 6, 13, 76, 7], [108, 36, 44, 123, 51, 90, 26, 114, 85, 92, 96, 52, 121, 120, 18, 82, 31, 77, 28, 111, 94, 24, 15, 125, 56, 53, 95, 127, 60, 126, 20, 62, 107, 63, 113, 122, 23, 76, 46, 73, 110, 55, 32, 124, 39, 105, 102, 101, 45, 84, 87, 58, 50, 43, 74, 30, 37, 118, 41, 109, 116, 16, 6, 8, 97, 29, 27, 49, 21, 54, 33, 61, 115, 38, 9, 48, 7, 57, 112, 106, 40, 98, 14, 103, 117, 91, 89, 119, 47, 22, 99, 34, 25, 59, 104, 100, 35, 3, 42, 86, 78, 64, 88, 93, 4, 19, 80, 2, 83, 81, 11, 12, 17, 65, 10, 79, 67, 71, 5, 13, 69, 75, 66, 0, 68, 70, 1, 72], [108, 36, 44, 123, 51, 63, 127, 52, 90, 28, 96, 56, 114, 120, 92, 23, 105, 95, 119, 118, 46, 58, 31, 85, 87, 111, 30, 102, 106, 112, 57, 54, 124, 40, 103, 122, 55, 42, 77, 26, 107, 125, 60, 113, 99, 104, 116, 110, 15, 43, 109, 39, 32, 94, 61, 41, 100, 45, 62, 47, 38, 49, 11, 121, 82, 18, 115, 117, 126, 48, 53, 17, 37, 50, 10, 20, 59, 101, 93, 84, 8, 35, 97, 98, 33, 76, 34, 27, 6, 91, 88, 24, 16, 9, 89, 29, 83, 25, 19, 80, 73, 5, 21, 86, 22, 74, 3, 79, 81, 13, 7, 68, 12, 72, 75, 78, 14, 69, 66, 70, 64, 1, 4, 71, 65, 67, 2, 0], [108, 36, 44, 51, 52, 92, 96, 56, 85, 121, 90, 126, 26, 28, 31, 60, 114, 23, 120, 45, 117, 87, 18, 82, 15, 105, 49, 94, 77, 124, 102, 24, 112, 32, 48, 88, 109, 123, 55, 104, 110, 46, 103, 93, 41, 27, 73, 76, 100, 107, 125, 115, 34, 116, 61, 29, 63, 118, 113, 122, 43, 50, 53, 84, 119, 89, 111, 57, 127, 95, 101, 16, 22, 38, 47, 42, 39, 33, 106, 54, 25, 62, 4, 2, 40, 8, 37, 59, 98, 21, 97, 58, 64, 6, 3, 74, 99, 86, 30, 91, 9, 35, 80, 20, 65, 83, 78, 19, 7, 69, 71, 79, 81, 14, 5, 12, 17, 70, 10, 0, 13, 11, 75, 68, 72, 67, 66, 1], [108, 36, 123, 52, 90, 94, 44, 92, 28, 121, 124, 111, 63, 62, 93, 59, 109, 86, 105, 82, 87, 23, 116, 114, 50, 60, 26, 20, 100, 49, 96, 42, 91, 46, 31, 127, 119, 17, 103, 51, 78, 19, 41, 84, 57, 110, 15, 27, 53, 98, 48, 102, 37, 122, 117, 112, 58, 55, 33, 101, 99, 30, 104, 47, 45, 40, 25, 56, 120, 43, 61, 35, 54, 11, 106, 38, 118, 39, 113, 115, 97, 126, 107, 125, 22, 29, 18, 34, 8, 14, 85, 89, 80, 95, 88, 32, 21, 73, 77, 76, 24, 9, 83, 6, 16, 10, 81, 13, 79, 3, 7, 74, 12, 71, 75, 64, 2, 70, 72, 4, 5, 69, 65, 68, 66, 1, 67, 0], [45, 52, 109, 32, 89, 85, 18, 123, 12, 29, 87, 10, 9, 79, 69, 81, 96, 83, 71, 91, 101, 58, 4, 30, 117, 0, 68, 2, 82, 8, 19, 1, 16, 105, 59, 84, 35, 108, 37, 7, 64, 23, 6, 50, 54, 77, 76, 78, 127, 42, 25, 124, 97, 33, 17, 46, 102, 24, 31, 120, 114, 21, 28, 72, 15, 20, 94, 80, 73, 26, 88, 3, 126, 62, 104, 92, 70, 74, 110, 75, 66, 14, 103, 13, 122, 61, 44, 116, 27, 95, 67, 36, 60, 112, 39, 40, 86, 100, 22, 90, 34, 5, 119, 38, 113, 43, 98, 49, 93, 125, 65, 115, 56, 107, 11, 118, 48, 51, 53, 111, 99, 121, 63, 106, 57, 41, 47, 55], [45, 52, 109, 87, 123, 91, 79, 85, 69, 89, 12, 10, 32, 18, 96, 101, 4, 71, 1, 29, 9, 0, 81, 66, 83, 92, 30, 2, 36, 35, 56, 82, 65, 122, 42, 120, 117, 60, 8, 105, 22, 70, 58, 39, 110, 97, 64, 104, 88, 127, 106, 7, 84, 72, 53, 90, 14, 61, 95, 63, 74, 3, 126, 5, 21, 77, 16, 48, 54, 73, 111, 80, 27, 23, 68, 24, 76, 67, 94, 119, 17, 25, 13, 50, 33, 75, 62, 26, 107, 15, 116, 51, 44, 20, 49, 93, 31, 19, 46, 11, 6, 102, 98, 37, 108, 114, 103, 78, 28, 41, 38, 59, 121, 86, 125, 34, 118, 113, 99, 40, 57, 112, 100, 55, 124, 115, 43, 47], [45, 52, 109, 91, 32, 123, 85, 89, 87, 29, 18, 12, 9, 79, 46, 81, 112, 10, 4, 114, 69, 71, 83, 126, 96, 50, 54, 94, 61, 58, 47, 119, 108, 60, 82, 127, 92, 25, 63, 72, 22, 70, 62, 77, 30, 84, 107, 59, 110, 101, 35, 105, 64, 15, 57, 106, 16, 13, 111, 40, 49, 27, 93, 68, 116, 117, 42, 21, 1, 56, 55, 41, 39, 2, 115, 17, 33, 20, 23, 118, 24, 80, 0, 75, 19, 37, 113, 120, 103, 99, 95, 36, 26, 7, 122, 88, 124, 31, 34, 98, 90, 125, 28, 104, 100, 51, 8, 121, 102, 44, 73, 76, 14, 97, 5, 6, 38, 11, 3, 86, 43, 74, 53, 48, 78, 67, 65, 66], [45, 52, 109, 89, 32, 87, 91, 29, 85, 123, 79, 96, 18, 12, 10, 71, 83, 69, 50, 114, 9, 60, 35, 119, 49, 92, 84, 81, 106, 4, 77, 101, 95, 57, 107, 93, 63, 16, 1, 30, 36, 46, 82, 108, 100, 61, 117, 94, 39, 22, 112, 120, 118, 23, 58, 127, 53, 26, 21, 113, 110, 116, 43, 40, 44, 59, 28, 105, 25, 103, 88, 99, 42, 122, 55, 31, 121, 111, 124, 24, 126, 13, 54, 98, 62, 125, 37, 56, 27, 38, 115, 97, 15, 80, 51, 19, 20, 48, 102, 86, 68, 14, 90, 6, 74, 34, 33, 75, 78, 47, 17, 64, 8, 41, 76, 72, 104, 7, 73, 11, 5, 0, 70, 65, 67, 2, 3, 66], [104, 127, 98, 92, 87, 84, 81, 109, 31, 15, 17, 22, 54, 41, 28, 58, 82, 51, 37, 76, 124, 14, 21, 63, 122, 116, 95, 121, 89, 30, 20, 43, 60, 38, 57, 90, 53, 13, 56, 25, 73, 18, 94, 85, 102, 29, 24, 11, 88, 126, 111, 117, 125, 70, 27, 46, 55, 107, 23, 48, 62, 12, 42, 59, 78, 67, 61, 4, 40, 45, 93, 110, 80, 105, 118, 106, 32, 9, 114, 83, 79, 19, 35, 97, 99, 120, 96, 36, 47, 6, 39, 101, 33, 68, 86, 52, 75, 10, 50, 49, 77, 100, 119, 74, 113, 123, 115, 44, 103, 26, 5, 108, 91, 16, 72, 69, 8, 112, 34, 71, 7, 66, 3, 1, 65, 64, 0, 2], [104, 127, 92, 98, 84, 87, 14, 81, 109, 31, 73, 76, 37, 6, 63, 125, 122, 82, 57, 105, 75, 89, 124, 18, 41, 4, 107, 94, 99, 78, 25, 17, 97, 116, 95, 83, 61, 10, 64, 58, 28, 30, 68, 59, 108, 66, 43, 54, 20, 65, 60, 51, 47, 115, 72, 15, 100, 106, 3, 50, 46, 53, 45, 56, 49, 85, 48, 23, 13, 16, 22, 114, 96, 24, 9, 62, 77, 12, 74, 35, 26, 80, 70, 119, 19, 36, 33, 38, 91, 79, 90, 40, 117, 44, 55, 29, 111, 121, 101, 39, 88, 102, 103, 2, 21, 34, 93, 8, 123, 69, 120, 126, 7, 32, 113, 112, 5, 27, 11, 42, 110, 52, 118, 71, 0, 86, 1, 67], [104, 127, 98, 101, 22, 43, 92, 18, 105, 53, 31, 59, 125, 84, 124, 87, 63, 61, 89, 54, 81, 41, 39, 122, 57, 106, 50, 95, 45, 19, 46, 28, 99, 35, 121, 109, 71, 79, 118, 38, 26, 62, 25, 76, 58, 37, 116, 44, 8, 96, 15, 56, 0, 30, 102, 27, 111, 117, 14, 115, 13, 120, 126, 107, 108, 60, 94, 93, 49, 73, 74, 51, 55, 110, 52, 11, 85, 123, 113, 103, 82, 119, 32, 4, 97, 42, 75, 36, 77, 112, 100, 90, 48, 33, 47, 86, 114, 68, 65, 6, 5, 29, 24, 34, 10, 80, 88, 23, 91, 83, 40, 7, 67, 17, 20, 72, 69, 2, 21, 1, 16, 70, 66, 3, 64, 9, 12, 78], [104, 127, 98, 73, 87, 31, 66, 14, 92, 81, 124, 84, 76, 4, 6, 116, 37, 25, 64, 89, 2, 63, 71, 13, 5, 60, 41, 80, 65, 75, 19, 0, 18, 106, 61, 114, 22, 79, 67, 46, 10, 1, 105, 53, 47, 58, 57, 125, 8, 122, 95, 40, 85, 54, 82, 101, 12, 59, 102, 51, 77, 74, 117, 121, 62, 70, 3, 111, 11, 90, 55, 24, 69, 72, 29, 109, 50, 23, 49, 15, 9, 100, 16, 86, 21, 91, 42, 33, 56, 88, 35, 99, 26, 7, 119, 123, 107, 17, 112, 43, 68, 36, 32, 38, 44, 118, 48, 94, 103, 96, 27, 126, 113, 52, 97, 108, 83, 28, 115, 34, 110, 39, 120, 78, 45, 30, 93, 20], [61, 102, 121, 114, 58, 116, 119, 62, 115, 54, 50, 33, 111, 126, 60, 125, 59, 122, 120, 124, 123, 117, 112, 110, 63, 52, 38, 24, 56, 53, 49, 45, 113, 42, 57, 29, 127, 55, 48, 26, 51, 118, 46, 107, 15, 27, 108, 47, 109, 91, 88, 105, 41, 85, 21, 84, 106, 44, 43, 86, 93, 37, 90, 18, 101, 28, 103, 100, 40, 104, 39, 22, 30, 97, 36, 32, 83, 17, 79, 34, 98, 35, 82, 96, 12, 19, 73, 95, 99, 31, 92, 81, 9, 94, 23, 80, 77, 64, 7, 20, 76, 25, 71, 66, 78, 89, 67, 87, 0, 2, 1, 65, 3, 5, 4, 69, 16, 13, 68, 11, 6, 14, 75, 10, 8, 72, 74, 70], [121, 102, 114, 61, 116, 58, 119, 62, 115, 54, 33, 60, 126, 59, 122, 111, 50, 123, 117, 120, 124, 110, 125, 52, 112, 53, 57, 24, 29, 63, 56, 45, 38, 49, 48, 26, 127, 55, 42, 113, 46, 118, 51, 88, 15, 47, 109, 108, 27, 91, 105, 41, 107, 106, 21, 85, 43, 84, 44, 86, 93, 28, 37, 90, 18, 103, 100, 101, 104, 39, 30, 97, 22, 40, 83, 17, 36, 79, 32, 12, 99, 95, 35, 34, 96, 98, 73, 19, 20, 31, 81, 87, 92, 82, 94, 80, 76, 23, 9, 25, 71, 67, 7, 77, 64, 66, 0, 3, 65, 1, 78, 89, 4, 2, 69, 5, 68, 13, 16, 11, 14, 75, 6, 74, 10, 72, 8, 70], [114, 121, 102, 61, 58, 116, 119, 115, 62, 122, 60, 124, 126, 50, 59, 123, 54, 33, 111, 112, 125, 120, 110, 117, 24, 52, 63, 56, 53, 42, 106, 57, 127, 48, 49, 107, 55, 26, 118, 113, 38, 15, 46, 51, 45, 29, 47, 108, 27, 105, 84, 109, 91, 85, 21, 44, 41, 88, 93, 28, 43, 86, 37, 90, 18, 103, 40, 104, 32, 101, 36, 100, 39, 30, 17, 22, 82, 79, 97, 98, 35, 9, 34, 96, 12, 99, 83, 92, 31, 95, 73, 81, 94, 7, 19, 76, 67, 69, 20, 77, 25, 64, 1, 0, 78, 71, 66, 65, 3, 4, 2, 80, 23, 5, 89, 68, 87, 14, 13, 11, 16, 6, 74, 75, 8, 72, 70, 10], [61, 121, 114, 102, 23, 80, 83, 50, 58, 74, 77, 30, 25, 45, 11, 8, 33, 37, 29, 90, 91, 78, 32, 92, 20, 70, 81, 119, 100, 69, 116, 88, 86, 62, 75, 68, 72, 120, 89, 59, 42, 105, 26, 21, 14, 125, 18, 6, 87, 16, 10, 66, 28, 54, 24, 76, 19, 27, 127, 85, 96, 126, 103, 56, 107, 1, 111, 106, 93, 63, 64, 51, 12, 67, 22, 13, 48, 84, 34, 4, 82, 17, 60, 31, 47, 110, 43, 115, 122, 98, 124, 52, 101, 7, 112, 117, 94, 35, 123, 57, 79, 108, 49, 0, 104, 41, 95, 2, 46, 71, 55, 5, 9, 36, 118, 3, 113, 99, 44, 53, 109, 15, 38, 39, 40, 65, 73, 97], [53, 42, 58, 120, 122, 100, 125, 116, 61, 45, 63, 123, 127, 48, 50, 52, 121, 49, 30, 54, 113, 89, 126, 51, 23, 108, 56, 59, 32, 44, 112, 111, 115, 46, 60, 106, 114, 62, 18, 47, 117, 57, 55, 21, 124, 119, 110, 109, 43, 118, 90, 91, 107, 40, 86, 92, 35, 15, 25, 104, 103, 28, 94, 24, 85, 41, 105, 12, 31, 37, 27, 38, 39, 83, 99, 101, 96, 36, 22, 102, 82, 34, 80, 98, 88, 93, 81, 95, 87, 78, 33, 1, 20, 76, 79, 97, 17, 5, 14, 29, 11, 26, 7, 77, 74, 16, 67, 68, 65, 0, 3, 72, 84, 19, 73, 64, 2, 70, 4, 71, 66, 69, 10, 9, 8, 13, 75, 6], [53, 42, 58, 120, 83, 80, 100, 89, 32, 11, 77, 73, 86, 27, 106, 45, 122, 70, 28, 24, 1, 92, 99, 26, 85, 81, 17, 67, 25, 74, 79, 72, 14, 5, 103, 20, 75, 111, 116, 23, 40, 113, 76, 125, 63, 123, 57, 48, 84, 119, 96, 61, 66, 6, 37, 93, 78, 82, 19, 50, 8, 91, 127, 34, 88, 15, 30, 126, 43, 94, 31, 9, 87, 69, 10, 49, 22, 105, 29, 4, 21, 0, 16, 95, 112, 71, 13, 39, 90, 56, 33, 54, 52, 124, 121, 101, 18, 51, 104, 118, 59, 109, 108, 7, 68, 114, 62, 12, 3, 35, 97, 117, 60, 115, 46, 44, 55, 98, 38, 2, 65, 102, 107, 64, 47, 110, 41, 36], [58, 42, 53, 120, 122, 125, 116, 45, 100, 61, 30, 123, 63, 48, 121, 50, 127, 49, 111, 54, 52, 89, 56, 59, 51, 112, 126, 23, 106, 108, 32, 46, 113, 115, 57, 114, 44, 62, 47, 60, 117, 55, 119, 21, 124, 110, 18, 109, 24, 118, 90, 94, 43, 107, 104, 91, 40, 25, 41, 105, 103, 92, 31, 35, 85, 86, 39, 37, 99, 12, 101, 38, 28, 27, 15, 96, 102, 36, 22, 82, 95, 93, 34, 98, 17, 81, 88, 97, 79, 33, 83, 76, 20, 80, 14, 87, 29, 26, 7, 16, 78, 2, 74, 8, 19, 64, 65, 72, 66, 77, 68, 0, 5, 71, 1, 67, 3, 84, 10, 4, 11, 69, 6, 70, 13, 75, 9, 73], [120, 42, 58, 53, 122, 125, 45, 61, 116, 123, 106, 50, 89, 52, 23, 49, 100, 48, 59, 56, 63, 112, 111, 44, 121, 54, 127, 117, 126, 57, 62, 110, 113, 51, 115, 30, 55, 60, 46, 109, 118, 114, 47, 90, 119, 124, 18, 108, 107, 28, 43, 21, 105, 24, 104, 91, 32, 83, 95, 29, 40, 103, 38, 39, 35, 101, 34, 41, 99, 102, 94, 86, 37, 97, 85, 92, 27, 31, 33, 25, 15, 98, 96, 77, 36, 26, 12, 82, 93, 20, 13, 19, 22, 75, 17, 10, 14, 87, 7, 11, 76, 84, 74, 79, 88, 16, 80, 71, 67, 6, 68, 66, 72, 1, 8, 2, 5, 81, 70, 3, 64, 0, 4, 69, 65, 78, 9, 73], [51, 118, 55, 39, 63, 54, 121, 25, 23, 122, 124, 120, 126, 59, 53, 61, 113, 123, 62, 58, 117, 95, 50, 56, 57, 89, 52, 125, 60, 82, 49, 116, 115, 119, 34, 47, 48, 114, 127, 106, 111, 112, 42, 46, 44, 109, 104, 110, 45, 108, 29, 87, 80, 14, 27, 43, 107, 84, 36, 76, 35, 37, 99, 41, 96, 40, 102, 103, 20, 101, 100, 105, 28, 97, 38, 31, 18, 12, 85, 91, 26, 24, 16, 74, 32, 93, 92, 98, 78, 86, 30, 71, 69, 33, 94, 8, 21, 88, 7, 2, 22, 1, 68, 90, 83, 10, 70, 72, 13, 81, 3, 73, 6, 19, 11, 0, 5, 9, 79, 65, 4, 64, 66, 75, 67, 17, 77, 15], [55, 39, 118, 122, 25, 23, 120, 121, 63, 89, 95, 115, 124, 51, 123, 113, 126, 61, 58, 60, 34, 82, 59, 112, 48, 53, 119, 106, 52, 57, 62, 117, 111, 42, 49, 50, 56, 47, 114, 125, 44, 127, 116, 43, 54, 80, 14, 29, 108, 45, 110, 46, 109, 84, 76, 41, 104, 107, 40, 27, 87, 20, 31, 105, 18, 102, 100, 16, 28, 96, 78, 37, 36, 101, 38, 103, 12, 93, 35, 8, 97, 32, 24, 94, 99, 83, 88, 74, 33, 86, 10, 71, 92, 85, 72, 7, 98, 90, 70, 73, 69, 68, 13, 30, 6, 21, 3, 2, 5, 26, 9, 22, 0, 65, 66, 19, 1, 4, 67, 64, 81, 11, 91, 79, 17, 77, 75, 15], [51, 55, 39, 120, 121, 118, 54, 122, 63, 23, 89, 25, 59, 123, 46, 113, 62, 115, 124, 60, 61, 126, 44, 112, 125, 58, 50, 116, 33, 53, 52, 49, 82, 95, 57, 34, 119, 29, 108, 56, 45, 110, 114, 111, 117, 87, 47, 42, 127, 48, 43, 76, 14, 109, 100, 84, 68, 71, 31, 36, 99, 98, 3, 106, 26, 72, 11, 80, 38, 107, 37, 40, 88, 32, 18, 102, 105, 27, 0, 83, 104, 20, 41, 28, 96, 97, 1, 12, 101, 81, 30, 92, 93, 90, 94, 78, 103, 5, 35, 2, 64, 74, 24, 69, 65, 70, 9, 4, 85, 19, 21, 8, 91, 13, 16, 6, 86, 7, 75, 67, 10, 73, 79, 66, 17, 22, 77, 15], [51, 55, 120, 39, 63, 121, 27, 124, 126, 59, 53, 61, 123, 116, 122, 57, 58, 60, 62, 56, 45, 118, 119, 54, 25, 117, 125, 114, 113, 50, 47, 49, 127, 111, 34, 115, 108, 95, 52, 48, 46, 84, 107, 112, 36, 86, 110, 82, 43, 20, 109, 42, 76, 89, 44, 29, 40, 23, 14, 92, 41, 106, 24, 94, 80, 102, 101, 105, 91, 93, 22, 79, 99, 31, 103, 28, 17, 81, 90, 35, 104, 38, 15, 74, 98, 100, 12, 69, 85, 7, 37, 8, 88, 32, 13, 16, 9, 96, 21, 70, 71, 97, 26, 18, 33, 87, 68, 1, 77, 83, 6, 30, 2, 11, 78, 73, 10, 19, 0, 67, 72, 5, 75, 66, 4, 65, 3, 64], [38, 115, 114, 50, 51, 113, 89, 75, 19, 82, 16, 23, 77, 7, 73, 78, 69, 10, 39, 87, 25, 6, 94, 67, 8, 85, 57, 104, 13, 74, 125, 52, 27, 3, 26, 80, 118, 21, 14, 68, 65, 5, 106, 92, 124, 12, 105, 11, 123, 30, 64, 79, 98, 122, 37, 95, 81, 43, 44, 63, 62, 56, 41, 18, 93, 70, 102, 54, 101, 2, 112, 121, 107, 83, 42, 76, 100, 59, 34, 48, 111, 49, 45, 40, 119, 117, 33, 28, 126, 60, 84, 47, 108, 120, 36, 96, 109, 103, 4, 61, 88, 86, 29, 32, 17, 99, 127, 35, 58, 72, 46, 9, 116, 97, 31, 55, 22, 15, 90, 110, 53, 91, 71, 24, 20, 1, 66, 0], [38, 114, 115, 50, 51, 89, 23, 82, 19, 77, 16, 69, 75, 7, 73, 113, 64, 2, 85, 14, 93, 17, 67, 66, 78, 1, 21, 72, 3, 87, 79, 12, 94, 8, 13, 32, 71, 126, 106, 25, 15, 68, 95, 102, 10, 22, 41, 57, 74, 6, 127, 24, 44, 34, 112, 107, 56, 39, 76, 27, 92, 26, 90, 98, 0, 54, 18, 101, 80, 65, 40, 121, 105, 49, 81, 30, 122, 84, 47, 9, 104, 108, 83, 109, 117, 4, 125, 11, 20, 100, 37, 36, 48, 42, 116, 43, 86, 103, 97, 59, 63, 120, 28, 29, 61, 46, 45, 111, 99, 55, 110, 119, 53, 35, 70, 58, 60, 91, 33, 124, 118, 31, 88, 96, 62, 52, 123, 5], [38, 114, 115, 50, 51, 89, 7, 19, 73, 16, 75, 113, 82, 2, 77, 64, 69, 23, 3, 67, 14, 25, 85, 87, 93, 6, 13, 1, 17, 48, 66, 107, 8, 125, 9, 124, 41, 56, 71, 104, 0, 12, 79, 20, 52, 122, 45, 126, 5, 94, 81, 18, 22, 65, 74, 95, 127, 91, 57, 49, 59, 83, 15, 11, 80, 46, 24, 60, 92, 39, 118, 21, 86, 90, 105, 68, 10, 117, 121, 4, 44, 110, 29, 78, 72, 62, 106, 111, 88, 32, 108, 96, 58, 27, 40, 31, 123, 34, 109, 116, 120, 76, 119, 97, 101, 43, 63, 37, 100, 112, 98, 36, 28, 30, 99, 33, 61, 54, 103, 53, 26, 42, 55, 47, 84, 35, 102, 70], [38, 50, 114, 115, 51, 113, 23, 16, 82, 89, 39, 19, 77, 85, 126, 75, 73, 78, 7, 30, 24, 37, 22, 57, 116, 14, 52, 12, 35, 92, 34, 27, 125, 112, 36, 79, 83, 69, 13, 81, 26, 107, 49, 56, 88, 63, 80, 104, 17, 106, 67, 21, 94, 25, 20, 120, 44, 90, 46, 87, 60, 121, 45, 117, 42, 40, 9, 108, 47, 41, 48, 8, 109, 93, 76, 6, 122, 32, 72, 84, 97, 102, 53, 62, 10, 59, 119, 61, 99, 103, 98, 2, 118, 31, 111, 11, 54, 105, 74, 28, 18, 86, 127, 124, 33, 110, 15, 95, 29, 123, 43, 100, 96, 101, 58, 55, 91, 68, 3, 70, 5, 64, 71, 65, 1, 4, 66, 0]], "model.layers.24.self_attn.k_proj": [[48, 112, 101, 30, 86, 90, 19, 88, 89, 25, 81, 84, 12, 70, 14, 107, 77, 63, 8, 54, 47, 120, 60, 17, 52, 125, 41, 0, 49, 117, 115, 119, 93, 114, 2, 124, 74, 10, 33, 79, 64, 113, 43, 46, 102, 1, 57, 62, 80, 59, 42, 18, 20, 100, 118, 110, 35, 9, 109, 56, 29, 123, 44, 55, 58, 51, 68, 106, 116, 45, 28, 38, 103, 122, 108, 61, 50, 111, 4, 3, 40, 104, 87, 53, 127, 126, 39, 121, 67, 98, 99, 16, 97, 96, 15, 23, 32, 36, 31, 27, 85, 91, 5, 105, 92, 83, 34, 71, 94, 95, 73, 24, 82, 75, 21, 78, 7, 26, 11, 13, 72, 69, 76, 65, 22, 6, 66, 37], [44, 108, 52, 121, 82, 77, 15, 85, 100, 26, 32, 87, 60, 123, 6, 8, 76, 73, 55, 28, 3, 51, 74, 127, 120, 24, 48, 92, 61, 0, 9, 64, 65, 29, 124, 4, 109, 2, 89, 16, 5, 111, 94, 20, 110, 63, 18, 114, 105, 116, 78, 56, 35, 118, 7, 83, 47, 126, 59, 22, 88, 91, 50, 95, 62, 33, 122, 46, 84, 119, 115, 25, 31, 80, 54, 23, 27, 113, 86, 97, 98, 45, 81, 99, 103, 39, 102, 69, 40, 12, 70, 101, 107, 37, 57, 43, 117, 34, 112, 49, 11, 93, 53, 41, 38, 36, 42, 106, 96, 125, 104, 58, 19, 90, 30, 21, 14, 17, 71, 10, 13, 75, 79, 68, 72, 66, 1, 67], [109, 52, 45, 0, 123, 85, 32, 12, 69, 10, 87, 79, 116, 89, 18, 9, 4, 91, 81, 68, 2, 29, 83, 60, 71, 84, 8, 77, 78, 3, 1, 70, 16, 119, 120, 7, 93, 126, 58, 117, 94, 41, 30, 64, 66, 115, 124, 72, 118, 112, 114, 113, 61, 46, 54, 47, 44, 37, 92, 49, 111, 127, 65, 50, 53, 125, 95, 105, 106, 51, 110, 63, 35, 40, 43, 107, 48, 56, 102, 55, 39, 34, 19, 108, 104, 22, 121, 57, 97, 6, 59, 42, 62, 75, 103, 36, 100, 122, 82, 38, 67, 25, 28, 101, 99, 98, 13, 33, 27, 14, 20, 11, 88, 31, 21, 23, 90, 96, 86, 26, 73, 15, 80, 24, 5, 76, 17, 74], [127, 40, 34, 84, 87, 76, 95, 81, 73, 14, 17, 89, 6, 5, 28, 66, 125, 124, 64, 4, 116, 60, 122, 110, 0, 111, 13, 79, 58, 74, 19, 75, 92, 117, 51, 18, 80, 50, 24, 57, 105, 43, 63, 22, 45, 41, 101, 107, 26, 106, 54, 16, 121, 70, 10, 86, 71, 53, 20, 39, 78, 59, 8, 65, 52, 56, 30, 37, 62, 49, 94, 3, 42, 25, 123, 114, 61, 27, 83, 90, 7, 85, 96, 15, 109, 119, 103, 67, 21, 35, 44, 118, 33, 47, 99, 113, 55, 77, 108, 120, 126, 32, 1, 102, 69, 112, 115, 93, 68, 31, 98, 38, 36, 88, 46, 29, 48, 97, 82, 91, 12, 72, 100, 23, 9, 11, 2, 104], [38, 61, 114, 121, 86, 97, 119, 93, 18, 54, 30, 27, 50, 116, 15, 59, 41, 62, 56, 120, 126, 58, 115, 125, 124, 122, 55, 63, 90, 60, 127, 57, 49, 117, 48, 85, 123, 47, 112, 113, 118, 108, 52, 110, 101, 24, 51, 45, 36, 109, 46, 96, 53, 43, 94, 29, 106, 111, 95, 17, 104, 44, 25, 42, 107, 39, 99, 91, 105, 79, 33, 98, 12, 20, 28, 81, 80, 103, 92, 13, 40, 73, 77, 88, 100, 83, 7, 14, 16, 34, 10, 21, 31, 37, 84, 102, 23, 35, 74, 19, 26, 32, 78, 11, 89, 87, 8, 82, 70, 22, 75, 6, 68, 3, 9, 1, 76, 72, 2, 69, 66, 65, 5, 71, 0, 4, 67, 64], [106, 36, 120, 86, 53, 58, 96, 116, 94, 28, 49, 61, 89, 63, 27, 121, 123, 42, 122, 50, 45, 125, 48, 56, 62, 111, 54, 124, 59, 52, 18, 117, 55, 80, 127, 57, 119, 115, 112, 35, 46, 114, 51, 83, 60, 108, 34, 126, 107, 118, 47, 113, 109, 21, 98, 23, 12, 41, 43, 73, 15, 81, 110, 9, 105, 11, 104, 77, 29, 44, 40, 102, 14, 16, 39, 38, 7, 37, 103, 101, 17, 90, 0, 95, 87, 97, 33, 92, 24, 91, 20, 31, 67, 26, 100, 13, 99, 32, 71, 93, 72, 5, 88, 30, 68, 84, 19, 66, 78, 70, 74, 82, 85, 25, 2, 65, 64, 76, 69, 10, 75, 6, 79, 8, 22, 3, 4, 1], [103, 51, 55, 86, 120, 98, 123, 63, 93, 126, 124, 61, 31, 114, 53, 45, 59, 58, 49, 57, 121, 89, 122, 52, 125, 62, 112, 56, 47, 117, 116, 50, 60, 46, 119, 127, 115, 48, 110, 113, 108, 111, 107, 38, 109, 40, 42, 105, 44, 54, 43, 36, 80, 41, 106, 22, 18, 33, 118, 34, 79, 82, 16, 104, 99, 88, 90, 81, 84, 26, 96, 97, 102, 92, 91, 11, 100, 13, 37, 27, 21, 101, 32, 35, 83, 12, 23, 25, 29, 94, 72, 78, 85, 20, 10, 6, 30, 28, 17, 39, 24, 14, 66, 87, 19, 95, 9, 5, 73, 69, 4, 15, 8, 77, 75, 67, 76, 65, 71, 68, 0, 74, 7, 3, 1, 70, 2, 64], [50, 115, 102, 77, 89, 16, 19, 82, 75, 73, 114, 7, 23, 64, 69, 113, 2, 3, 67, 66, 1, 85, 10, 93, 17, 8, 70, 14, 65, 78, 21, 57, 30, 38, 112, 49, 94, 42, 43, 126, 34, 76, 71, 39, 45, 108, 92, 25, 90, 48, 6, 32, 44, 117, 123, 122, 13, 121, 87, 56, 106, 103, 125, 22, 79, 28, 74, 20, 27, 120, 63, 124, 116, 68, 33, 127, 61, 54, 15, 60, 104, 4, 41, 52, 119, 105, 100, 95, 12, 91, 72, 9, 88, 26, 53, 36, 31, 101, 29, 98, 62, 35, 118, 47, 59, 84, 46, 109, 51, 40, 96, 99, 0, 110, 55, 107, 111, 24, 37, 83, 58, 80, 97, 11, 5, 81, 86, 18]], "model.layers.24.self_attn.qk_proj": [[127, 50, 48, 115, 112, 114, 109, 51, 45, 108, 55, 61, 121, 44, 53, 120, 58, 52, 38, 25, 23, 124, 123, 63, 87, 18, 42, 89, 92, 60, 22, 113, 106, 40, 125, 82, 126, 116, 29, 122, 83, 19, 59, 17, 77, 32, 101, 54, 20, 26, 13, 94, 84, 73, 16, 9, 30, 81, 12, 56, 102, 85, 80, 90, 21, 57, 117, 49, 37, 76, 46, 86, 79, 111, 75, 98, 118, 110, 36, 7, 62, 15, 78, 14, 41, 11, 47, 119, 71, 27, 91, 103, 64, 43, 95, 34, 39, 5, 69, 74, 10, 105, 0, 96, 24, 31, 100, 104, 6, 88, 93, 28, 3, 68, 107, 67, 2, 66, 33, 70, 8, 4, 97, 99, 72, 35, 1, 65], [127, 50, 115, 114, 48, 109, 112, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 38, 25, 123, 92, 23, 87, 60, 89, 113, 18, 42, 124, 22, 106, 116, 40, 122, 63, 54, 125, 82, 29, 126, 73, 59, 83, 77, 101, 56, 32, 13, 20, 85, 19, 94, 16, 17, 26, 30, 81, 102, 9, 12, 49, 117, 86, 76, 84, 79, 21, 119, 111, 118, 57, 71, 36, 98, 80, 110, 41, 27, 103, 39, 75, 37, 90, 62, 47, 104, 78, 14, 46, 91, 11, 15, 43, 74, 34, 64, 105, 5, 69, 24, 100, 31, 95, 107, 7, 0, 96, 4, 93, 10, 28, 6, 88, 2, 68, 8, 66, 3, 67, 33, 97, 70, 72, 99, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 55, 61, 121, 120, 44, 53, 58, 52, 25, 38, 123, 113, 23, 87, 106, 92, 42, 18, 89, 60, 40, 22, 126, 125, 122, 82, 63, 32, 56, 116, 124, 54, 29, 59, 17, 26, 101, 13, 19, 73, 30, 102, 49, 83, 85, 16, 119, 20, 77, 94, 81, 76, 86, 12, 90, 9, 117, 21, 47, 57, 111, 118, 98, 110, 84, 104, 41, 80, 103, 78, 34, 79, 36, 31, 39, 7, 37, 62, 71, 75, 64, 15, 11, 14, 27, 5, 105, 69, 96, 91, 46, 95, 74, 100, 43, 10, 107, 0, 28, 93, 24, 66, 6, 33, 67, 2, 88, 3, 8, 4, 68, 97, 70, 99, 35, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 51, 61, 45, 108, 55, 121, 44, 53, 120, 58, 52, 123, 38, 25, 106, 113, 23, 42, 87, 89, 18, 126, 82, 92, 59, 22, 124, 63, 125, 56, 116, 119, 29, 60, 94, 40, 76, 13, 122, 17, 32, 30, 73, 86, 19, 77, 80, 9, 101, 85, 16, 54, 49, 83, 81, 26, 79, 102, 12, 84, 111, 20, 98, 118, 110, 75, 47, 117, 5, 15, 41, 21, 104, 62, 71, 39, 36, 37, 69, 57, 14, 90, 7, 27, 34, 78, 46, 31, 0, 43, 11, 74, 103, 10, 95, 100, 96, 2, 28, 24, 88, 66, 67, 8, 70, 64, 93, 91, 4, 105, 6, 3, 107, 33, 35, 68, 97, 99, 65, 1, 72], [127, 48, 50, 115, 114, 112, 109, 51, 108, 61, 55, 45, 44, 121, 120, 53, 58, 52, 25, 123, 38, 23, 42, 113, 60, 63, 82, 18, 89, 87, 106, 59, 92, 56, 126, 124, 125, 22, 122, 40, 54, 13, 17, 73, 19, 86, 49, 29, 116, 9, 94, 80, 77, 30, 57, 32, 84, 83, 81, 26, 20, 76, 12, 101, 16, 75, 85, 90, 110, 118, 119, 14, 79, 111, 102, 21, 117, 37, 15, 104, 62, 98, 36, 5, 103, 47, 41, 34, 71, 78, 27, 95, 7, 11, 10, 46, 69, 91, 64, 70, 74, 96, 105, 31, 100, 39, 0, 24, 43, 107, 33, 28, 88, 3, 66, 8, 2, 67, 4, 93, 97, 68, 6, 35, 1, 99, 72, 65], [127, 50, 48, 115, 114, 112, 109, 61, 51, 108, 55, 45, 44, 121, 53, 120, 58, 52, 25, 38, 42, 23, 22, 123, 89, 113, 87, 18, 82, 106, 92, 60, 126, 40, 124, 77, 84, 122, 73, 63, 59, 76, 83, 81, 17, 94, 80, 19, 116, 125, 86, 29, 12, 13, 101, 85, 54, 30, 16, 32, 9, 26, 102, 37, 79, 56, 15, 98, 21, 41, 110, 57, 90, 36, 34, 118, 49, 20, 47, 14, 103, 5, 11, 95, 75, 27, 111, 78, 71, 46, 70, 10, 7, 104, 0, 117, 64, 43, 105, 62, 119, 39, 91, 8, 74, 24, 69, 100, 96, 93, 107, 88, 31, 2, 33, 66, 97, 28, 3, 67, 68, 6, 35, 99, 4, 65, 1, 72], [127, 50, 48, 115, 114, 112, 109, 55, 108, 51, 45, 61, 44, 121, 53, 120, 58, 52, 25, 23, 42, 87, 38, 82, 89, 123, 92, 18, 106, 124, 22, 60, 113, 125, 122, 40, 63, 19, 17, 54, 126, 32, 94, 84, 86, 26, 73, 30, 80, 13, 77, 29, 83, 101, 81, 56, 116, 12, 16, 59, 76, 85, 20, 9, 57, 21, 119, 98, 79, 102, 15, 90, 49, 111, 78, 41, 75, 118, 47, 104, 11, 110, 27, 14, 46, 71, 117, 5, 36, 37, 91, 7, 103, 95, 34, 64, 10, 62, 105, 100, 39, 74, 0, 24, 43, 31, 69, 70, 107, 96, 88, 8, 28, 93, 33, 35, 2, 4, 3, 97, 68, 67, 99, 66, 6, 72, 1, 65], [127, 50, 48, 115, 114, 109, 112, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 25, 23, 123, 38, 124, 89, 92, 18, 87, 42, 60, 82, 22, 113, 63, 125, 106, 40, 122, 116, 83, 59, 29, 126, 119, 32, 30, 56, 16, 94, 54, 73, 86, 26, 101, 20, 80, 19, 49, 13, 84, 17, 21, 90, 81, 102, 12, 9, 85, 77, 57, 76, 37, 15, 111, 41, 79, 78, 62, 11, 46, 71, 36, 110, 39, 117, 118, 34, 91, 27, 5, 105, 104, 43, 28, 75, 98, 31, 0, 47, 24, 103, 64, 10, 14, 95, 7, 69, 100, 70, 88, 74, 93, 96, 68, 107, 66, 2, 35, 3, 33, 4, 8, 97, 67, 72, 6, 99, 1, 65], [127, 50, 48, 115, 112, 109, 114, 55, 45, 51, 108, 121, 61, 44, 53, 120, 58, 52, 25, 38, 23, 92, 89, 123, 124, 87, 42, 125, 113, 63, 82, 106, 60, 59, 18, 126, 116, 122, 83, 22, 32, 77, 40, 56, 101, 80, 94, 81, 26, 84, 29, 54, 86, 30, 19, 117, 13, 119, 73, 102, 20, 85, 17, 49, 90, 76, 16, 9, 21, 111, 12, 37, 98, 118, 43, 15, 41, 47, 79, 57, 14, 34, 91, 110, 103, 27, 96, 75, 105, 11, 71, 36, 46, 69, 39, 100, 62, 104, 31, 78, 28, 7, 5, 95, 24, 74, 107, 10, 88, 93, 64, 0, 70, 33, 2, 4, 35, 68, 66, 6, 97, 3, 72, 99, 67, 8, 1, 65], [127, 50, 48, 114, 115, 109, 112, 45, 51, 108, 55, 121, 61, 44, 53, 120, 58, 52, 25, 38, 123, 124, 23, 87, 89, 42, 125, 113, 63, 106, 92, 82, 59, 18, 60, 116, 40, 126, 22, 29, 122, 30, 101, 83, 9, 73, 13, 94, 32, 54, 17, 119, 19, 77, 76, 56, 86, 84, 26, 85, 117, 16, 81, 80, 102, 57, 36, 111, 49, 12, 21, 37, 41, 118, 79, 103, 69, 20, 14, 110, 7, 11, 34, 90, 75, 104, 71, 98, 15, 5, 27, 47, 62, 78, 91, 95, 0, 10, 105, 6, 64, 31, 96, 24, 43, 39, 74, 28, 46, 100, 4, 66, 3, 88, 72, 2, 68, 93, 33, 70, 107, 67, 35, 97, 99, 8, 1, 65], [127, 50, 48, 115, 109, 112, 114, 51, 108, 45, 44, 55, 61, 121, 53, 120, 58, 52, 25, 42, 38, 123, 124, 87, 23, 106, 92, 113, 18, 82, 89, 22, 63, 122, 126, 125, 40, 73, 13, 29, 60, 116, 83, 9, 59, 56, 54, 77, 80, 86, 16, 84, 85, 19, 32, 81, 94, 17, 30, 26, 12, 76, 119, 102, 101, 20, 118, 36, 15, 62, 79, 71, 11, 117, 5, 49, 111, 90, 0, 75, 21, 69, 37, 96, 34, 27, 98, 104, 103, 7, 6, 57, 14, 39, 95, 78, 10, 64, 43, 74, 110, 46, 105, 47, 91, 2, 31, 41, 88, 24, 72, 67, 100, 28, 66, 3, 68, 93, 107, 4, 70, 97, 33, 8, 65, 35, 99, 1], [127, 50, 48, 115, 114, 112, 109, 51, 45, 108, 61, 44, 55, 121, 53, 120, 58, 52, 25, 42, 23, 87, 92, 38, 123, 22, 18, 106, 124, 89, 113, 82, 63, 40, 77, 13, 76, 125, 19, 126, 86, 30, 80, 73, 29, 17, 60, 85, 20, 84, 83, 9, 54, 94, 116, 81, 122, 101, 32, 16, 49, 26, 12, 56, 118, 90, 98, 15, 21, 57, 59, 111, 79, 11, 37, 102, 36, 7, 34, 95, 75, 71, 0, 117, 104, 27, 62, 47, 14, 100, 10, 91, 6, 119, 39, 96, 41, 78, 5, 69, 103, 74, 43, 24, 105, 46, 64, 88, 107, 28, 93, 31, 110, 72, 33, 97, 67, 66, 68, 2, 4, 3, 99, 70, 35, 8, 1, 65], [127, 50, 48, 115, 112, 109, 114, 51, 55, 45, 108, 61, 44, 121, 53, 120, 58, 52, 25, 38, 92, 124, 123, 23, 89, 42, 22, 87, 125, 113, 82, 63, 126, 18, 106, 40, 60, 32, 116, 94, 101, 30, 59, 54, 122, 29, 84, 26, 85, 77, 17, 13, 9, 86, 49, 117, 83, 56, 81, 19, 57, 20, 80, 73, 16, 111, 21, 119, 90, 37, 76, 12, 98, 41, 79, 36, 103, 118, 91, 15, 34, 27, 95, 11, 105, 100, 102, 24, 110, 75, 14, 71, 39, 7, 62, 47, 78, 46, 0, 5, 104, 28, 69, 31, 74, 10, 88, 43, 96, 72, 6, 68, 64, 93, 33, 97, 107, 2, 99, 66, 67, 4, 35, 70, 3, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 108, 45, 55, 61, 44, 120, 121, 53, 58, 52, 25, 124, 42, 38, 123, 23, 126, 89, 113, 106, 87, 22, 125, 59, 92, 40, 63, 18, 82, 122, 32, 56, 102, 77, 60, 101, 73, 116, 83, 84, 9, 119, 13, 30, 94, 54, 19, 81, 118, 16, 17, 86, 49, 26, 57, 111, 85, 29, 80, 117, 12, 37, 21, 76, 79, 90, 36, 20, 7, 41, 11, 98, 62, 27, 47, 15, 78, 103, 39, 69, 34, 75, 105, 71, 43, 0, 100, 64, 104, 10, 88, 74, 31, 14, 5, 95, 110, 96, 24, 28, 72, 6, 66, 91, 46, 33, 107, 93, 4, 2, 3, 67, 70, 97, 35, 68, 99, 65, 8, 1], [127, 114, 48, 50, 115, 109, 112, 51, 108, 45, 44, 55, 61, 121, 120, 53, 58, 52, 42, 25, 38, 23, 106, 89, 123, 22, 124, 113, 87, 63, 82, 92, 40, 18, 125, 126, 116, 59, 29, 77, 32, 9, 81, 60, 73, 83, 122, 16, 19, 30, 94, 54, 84, 13, 85, 26, 12, 101, 111, 80, 17, 20, 118, 49, 21, 76, 36, 119, 102, 56, 57, 37, 11, 98, 86, 79, 7, 15, 117, 41, 14, 62, 5, 103, 43, 90, 34, 64, 78, 110, 69, 39, 95, 10, 71, 24, 47, 31, 105, 91, 100, 27, 75, 0, 104, 74, 46, 96, 72, 33, 70, 66, 28, 97, 107, 88, 2, 67, 6, 68, 3, 4, 93, 35, 99, 8, 65, 1], [127, 50, 48, 115, 114, 109, 112, 51, 45, 55, 108, 61, 121, 44, 120, 53, 58, 52, 38, 25, 42, 23, 123, 92, 82, 87, 106, 89, 124, 22, 63, 113, 18, 59, 125, 126, 77, 122, 116, 84, 40, 73, 83, 54, 13, 80, 81, 94, 30, 9, 16, 19, 32, 60, 12, 29, 86, 17, 76, 56, 20, 26, 85, 21, 111, 37, 101, 15, 118, 102, 90, 119, 62, 27, 57, 98, 110, 11, 71, 7, 75, 79, 39, 36, 10, 43, 14, 5, 46, 34, 31, 49, 117, 95, 78, 24, 41, 0, 69, 103, 104, 74, 105, 70, 47, 96, 91, 88, 107, 68, 100, 64, 72, 28, 66, 2, 67, 93, 97, 33, 6, 99, 4, 3, 8, 35, 65, 1], [127, 50, 48, 115, 114, 112, 109, 51, 55, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 38, 123, 42, 92, 23, 87, 124, 106, 89, 126, 18, 82, 22, 63, 125, 40, 59, 113, 30, 32, 116, 19, 29, 86, 101, 77, 60, 84, 26, 94, 81, 122, 16, 13, 56, 20, 17, 21, 54, 73, 9, 83, 111, 57, 76, 85, 102, 27, 110, 80, 118, 12, 90, 79, 7, 98, 103, 47, 41, 117, 36, 75, 119, 49, 24, 37, 95, 91, 15, 11, 62, 14, 104, 31, 10, 34, 28, 43, 105, 46, 96, 5, 78, 0, 39, 71, 107, 74, 64, 100, 88, 93, 70, 69, 68, 66, 4, 33, 97, 8, 67, 2, 72, 35, 99, 3, 6, 65, 1], [127, 50, 48, 115, 114, 109, 112, 55, 45, 51, 108, 61, 120, 121, 44, 53, 58, 52, 25, 38, 92, 123, 89, 87, 23, 126, 42, 106, 125, 113, 124, 82, 18, 116, 63, 22, 59, 86, 29, 40, 54, 32, 17, 19, 77, 94, 60, 101, 83, 26, 56, 16, 122, 9, 73, 13, 30, 57, 102, 119, 84, 81, 80, 76, 20, 85, 49, 12, 90, 39, 98, 111, 118, 21, 36, 37, 79, 7, 15, 41, 5, 27, 78, 47, 11, 69, 110, 91, 75, 64, 31, 104, 96, 103, 71, 14, 24, 117, 34, 95, 28, 105, 0, 62, 10, 43, 74, 70, 46, 100, 88, 2, 93, 107, 68, 4, 33, 8, 67, 66, 3, 35, 97, 6, 65, 72, 99, 1], [127, 50, 48, 114, 115, 109, 112, 51, 45, 108, 61, 55, 44, 121, 53, 120, 58, 52, 38, 123, 124, 89, 42, 25, 23, 113, 126, 125, 106, 18, 92, 63, 87, 59, 82, 116, 22, 40, 77, 73, 29, 32, 30, 94, 101, 9, 76, 60, 84, 119, 81, 19, 54, 17, 86, 16, 26, 13, 21, 122, 83, 103, 57, 56, 111, 85, 49, 98, 37, 41, 36, 80, 117, 20, 11, 12, 102, 75, 27, 47, 118, 79, 110, 5, 90, 7, 69, 62, 34, 64, 104, 39, 105, 31, 15, 78, 43, 46, 74, 0, 14, 24, 71, 91, 95, 107, 8, 100, 88, 66, 96, 28, 70, 68, 10, 4, 67, 93, 3, 33, 2, 97, 6, 35, 99, 65, 72, 1], [127, 48, 50, 114, 115, 112, 109, 108, 51, 55, 45, 61, 44, 121, 53, 120, 58, 52, 38, 25, 123, 113, 124, 42, 23, 89, 18, 106, 125, 126, 22, 92, 87, 82, 40, 59, 63, 116, 29, 60, 32, 77, 30, 81, 73, 13, 9, 101, 83, 94, 122, 86, 76, 80, 84, 17, 26, 85, 37, 110, 57, 98, 111, 12, 21, 56, 19, 54, 36, 20, 102, 16, 104, 119, 117, 75, 15, 7, 41, 24, 103, 10, 27, 79, 34, 69, 11, 91, 118, 105, 64, 5, 49, 78, 62, 39, 95, 71, 90, 43, 88, 74, 8, 46, 107, 0, 31, 96, 28, 47, 14, 100, 93, 97, 70, 2, 6, 68, 4, 33, 66, 67, 3, 99, 35, 65, 72, 1], [127, 48, 50, 114, 112, 115, 109, 51, 108, 55, 45, 120, 44, 61, 121, 53, 58, 52, 25, 124, 123, 23, 38, 92, 42, 113, 125, 126, 106, 89, 116, 59, 22, 87, 60, 18, 82, 40, 122, 32, 63, 13, 77, 83, 80, 73, 86, 26, 29, 19, 101, 119, 94, 17, 9, 81, 12, 56, 84, 57, 76, 85, 16, 30, 102, 104, 54, 20, 21, 110, 37, 49, 90, 111, 98, 117, 75, 15, 27, 11, 79, 69, 118, 41, 34, 5, 31, 36, 46, 7, 14, 71, 96, 47, 24, 10, 91, 39, 78, 95, 74, 62, 6, 43, 100, 103, 107, 0, 28, 105, 2, 93, 64, 8, 88, 4, 66, 68, 70, 67, 33, 3, 97, 35, 99, 65, 1, 72], [127, 50, 48, 114, 115, 112, 109, 45, 55, 51, 108, 120, 121, 61, 44, 53, 58, 52, 38, 25, 123, 113, 125, 59, 126, 89, 42, 23, 106, 92, 124, 18, 116, 63, 82, 87, 22, 32, 40, 122, 54, 60, 56, 101, 119, 94, 30, 29, 77, 9, 86, 81, 73, 57, 13, 12, 17, 83, 110, 76, 26, 85, 20, 80, 21, 16, 19, 102, 84, 111, 71, 36, 46, 79, 11, 117, 90, 5, 34, 7, 75, 103, 69, 15, 98, 37, 62, 104, 0, 27, 31, 41, 118, 49, 39, 47, 78, 24, 6, 91, 96, 14, 64, 10, 4, 95, 88, 100, 74, 43, 67, 107, 93, 66, 68, 105, 2, 8, 28, 3, 33, 70, 97, 99, 65, 35, 1, 72], [127, 48, 50, 115, 112, 114, 109, 55, 51, 45, 108, 121, 44, 61, 120, 53, 58, 52, 38, 25, 113, 125, 23, 124, 89, 123, 92, 42, 59, 18, 87, 106, 126, 82, 63, 40, 122, 22, 30, 60, 77, 54, 116, 29, 86, 32, 73, 20, 56, 94, 57, 83, 17, 26, 101, 19, 9, 13, 111, 85, 81, 102, 119, 80, 12, 16, 37, 21, 84, 62, 98, 110, 76, 117, 15, 5, 75, 118, 47, 90, 104, 79, 91, 14, 49, 7, 36, 103, 11, 71, 34, 27, 46, 69, 24, 6, 100, 41, 95, 31, 39, 28, 0, 78, 88, 43, 74, 107, 93, 68, 10, 96, 105, 4, 8, 2, 66, 64, 97, 67, 33, 3, 70, 99, 35, 72, 65, 1], [127, 48, 50, 115, 114, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 25, 38, 42, 124, 23, 89, 123, 87, 18, 106, 113, 92, 63, 22, 82, 125, 122, 60, 126, 40, 77, 54, 59, 9, 29, 86, 73, 116, 32, 17, 101, 13, 94, 83, 30, 12, 19, 56, 26, 84, 20, 80, 16, 76, 57, 21, 85, 102, 119, 81, 98, 37, 104, 36, 27, 90, 7, 15, 111, 79, 75, 110, 103, 41, 78, 10, 34, 69, 62, 47, 11, 118, 95, 74, 5, 71, 31, 14, 91, 46, 28, 117, 88, 43, 49, 0, 96, 39, 100, 107, 6, 64, 8, 24, 105, 2, 67, 66, 33, 70, 68, 4, 97, 3, 93, 99, 72, 35, 65, 1], [127, 50, 48, 114, 115, 112, 109, 51, 45, 108, 55, 61, 44, 121, 53, 120, 58, 52, 25, 38, 123, 23, 22, 124, 82, 42, 87, 18, 106, 113, 89, 63, 92, 126, 122, 125, 84, 40, 60, 13, 54, 116, 83, 12, 32, 73, 59, 101, 94, 29, 86, 17, 9, 85, 57, 30, 16, 77, 21, 19, 26, 111, 56, 81, 20, 110, 80, 98, 76, 75, 117, 90, 79, 102, 91, 37, 71, 15, 119, 36, 103, 34, 78, 5, 46, 27, 95, 39, 11, 62, 7, 104, 14, 31, 47, 41, 24, 69, 64, 74, 105, 49, 10, 118, 0, 96, 43, 28, 88, 93, 6, 100, 107, 3, 2, 67, 4, 70, 68, 33, 66, 72, 8, 97, 99, 1, 35, 65], [127, 50, 48, 114, 115, 109, 112, 51, 45, 55, 61, 108, 121, 53, 44, 120, 58, 52, 25, 38, 123, 23, 124, 92, 126, 113, 42, 63, 87, 89, 106, 116, 22, 122, 82, 125, 18, 40, 32, 59, 54, 30, 86, 101, 29, 94, 84, 77, 73, 17, 57, 9, 83, 21, 85, 13, 60, 19, 119, 12, 118, 26, 110, 98, 20, 16, 111, 81, 41, 56, 103, 79, 90, 80, 37, 76, 102, 117, 15, 75, 27, 49, 34, 71, 36, 43, 78, 47, 62, 46, 91, 74, 107, 104, 69, 11, 100, 96, 24, 14, 28, 7, 64, 31, 39, 0, 95, 5, 105, 10, 72, 93, 68, 88, 97, 2, 4, 33, 70, 3, 6, 35, 99, 66, 67, 65, 8, 1], [127, 50, 48, 115, 109, 114, 112, 55, 51, 45, 61, 108, 121, 44, 120, 53, 52, 58, 25, 124, 23, 38, 123, 92, 63, 87, 106, 89, 42, 59, 113, 60, 122, 54, 125, 18, 116, 82, 126, 32, 40, 22, 118, 83, 111, 119, 19, 57, 77, 56, 9, 29, 20, 26, 101, 73, 102, 12, 94, 86, 30, 85, 81, 84, 13, 17, 49, 117, 80, 16, 36, 76, 98, 75, 21, 27, 103, 41, 110, 47, 104, 90, 71, 37, 62, 78, 79, 91, 95, 15, 34, 105, 46, 43, 11, 31, 39, 69, 14, 74, 5, 7, 88, 64, 96, 100, 107, 10, 28, 70, 93, 24, 33, 0, 2, 68, 4, 66, 97, 72, 3, 99, 6, 35, 67, 65, 1, 8], [127, 50, 48, 114, 115, 112, 109, 51, 55, 108, 61, 45, 121, 44, 53, 120, 58, 52, 123, 38, 25, 124, 23, 89, 87, 116, 113, 125, 106, 18, 60, 82, 63, 92, 42, 126, 22, 40, 122, 32, 9, 73, 59, 101, 77, 13, 54, 12, 83, 30, 84, 94, 81, 110, 85, 19, 29, 57, 17, 119, 86, 26, 80, 111, 16, 76, 36, 56, 21, 49, 20, 117, 41, 15, 102, 79, 75, 71, 98, 46, 37, 34, 103, 69, 74, 27, 39, 91, 5, 0, 7, 11, 90, 78, 104, 118, 14, 96, 62, 24, 10, 47, 95, 31, 70, 105, 28, 64, 100, 72, 2, 43, 4, 66, 67, 107, 88, 93, 97, 3, 33, 68, 35, 6, 99, 65, 1, 8], [127, 50, 114, 115, 48, 112, 109, 51, 108, 45, 61, 44, 55, 121, 53, 120, 58, 52, 124, 38, 23, 25, 63, 123, 113, 89, 42, 92, 18, 22, 122, 87, 82, 60, 116, 106, 40, 125, 59, 126, 73, 77, 32, 54, 12, 13, 83, 94, 9, 101, 85, 81, 110, 29, 30, 17, 16, 19, 111, 86, 20, 76, 80, 119, 75, 26, 37, 56, 36, 117, 62, 49, 98, 102, 41, 84, 90, 57, 71, 118, 27, 79, 104, 103, 34, 69, 21, 15, 70, 11, 46, 64, 14, 7, 5, 10, 47, 105, 95, 74, 78, 72, 43, 39, 2, 91, 88, 96, 31, 28, 100, 0, 66, 107, 24, 4, 33, 3, 93, 67, 97, 99, 68, 35, 6, 65, 1, 8], [127, 50, 48, 115, 114, 112, 109, 51, 55, 108, 61, 45, 44, 53, 121, 120, 58, 52, 25, 124, 38, 123, 23, 42, 89, 113, 87, 92, 22, 63, 82, 18, 106, 60, 122, 126, 116, 59, 40, 125, 77, 17, 32, 13, 73, 56, 94, 9, 86, 83, 81, 29, 12, 19, 80, 101, 54, 16, 30, 84, 20, 85, 21, 102, 26, 76, 111, 57, 75, 117, 98, 36, 62, 79, 47, 118, 41, 110, 15, 71, 90, 46, 7, 103, 78, 34, 14, 49, 27, 74, 37, 119, 11, 105, 39, 69, 91, 104, 70, 31, 5, 64, 100, 10, 95, 96, 88, 43, 72, 24, 33, 107, 68, 0, 2, 93, 28, 4, 66, 6, 67, 3, 97, 99, 35, 8, 65, 1], [127, 50, 48, 115, 114, 51, 109, 112, 45, 108, 61, 55, 121, 44, 53, 120, 58, 52, 25, 123, 23, 124, 38, 42, 87, 106, 89, 116, 60, 113, 126, 18, 125, 92, 122, 82, 40, 63, 22, 77, 94, 29, 30, 73, 59, 9, 32, 101, 17, 86, 13, 12, 83, 81, 80, 20, 56, 54, 85, 26, 76, 111, 118, 84, 57, 21, 16, 98, 19, 41, 102, 62, 110, 117, 36, 71, 7, 37, 34, 49, 75, 47, 79, 15, 69, 27, 119, 90, 78, 10, 104, 64, 0, 24, 91, 11, 39, 14, 95, 96, 74, 5, 93, 46, 31, 103, 107, 43, 6, 33, 105, 28, 88, 67, 72, 100, 2, 66, 3, 68, 70, 4, 35, 8, 97, 99, 1, 65], [127, 50, 48, 115, 114, 109, 112, 51, 55, 108, 45, 61, 121, 44, 53, 120, 52, 58, 25, 38, 123, 23, 124, 42, 87, 89, 92, 113, 22, 82, 60, 106, 18, 63, 40, 116, 125, 13, 126, 29, 59, 122, 32, 9, 54, 83, 57, 12, 94, 73, 81, 77, 26, 101, 20, 16, 86, 19, 30, 17, 85, 111, 56, 75, 84, 80, 21, 102, 15, 76, 36, 110, 117, 41, 98, 90, 78, 14, 62, 118, 79, 37, 10, 11, 49, 7, 91, 119, 47, 71, 27, 24, 103, 31, 43, 46, 74, 96, 34, 95, 100, 69, 5, 105, 104, 39, 6, 64, 28, 88, 93, 107, 0, 67, 72, 3, 4, 68, 33, 2, 70, 97, 66, 8, 35, 99, 1, 65]], "model.layers.25.self_attn.q_proj": [[62, 102, 124, 121, 56, 60, 97, 47, 50, 63, 38, 93, 114, 24, 127, 117, 95, 86, 105, 53, 89, 41, 35, 29, 43, 106, 115, 51, 52, 22, 37, 49, 103, 122, 20, 25, 123, 59, 87, 55, 33, 83, 91, 116, 57, 58, 100, 36, 99, 79, 27, 26, 46, 113, 111, 92, 108, 112, 109, 94, 61, 48, 125, 82, 19, 110, 54, 34, 42, 17, 39, 44, 12, 119, 126, 18, 118, 80, 120, 101, 104, 40, 45, 77, 98, 32, 16, 96, 28, 30, 107, 31, 90, 88, 21, 23, 81, 85, 70, 75, 84, 74, 72, 9, 4, 78, 15, 2, 14, 8, 76, 13, 67, 11, 0, 5, 66, 71, 10, 7, 73, 6, 68, 65, 3, 1, 64, 69], [102, 62, 93, 121, 124, 24, 97, 83, 56, 35, 60, 127, 85, 81, 13, 21, 17, 63, 86, 20, 22, 98, 36, 25, 123, 114, 92, 19, 120, 77, 84, 46, 74, 78, 48, 116, 111, 76, 52, 49, 11, 95, 33, 30, 32, 79, 115, 1, 38, 9, 71, 28, 122, 88, 34, 69, 31, 12, 104, 75, 101, 7, 66, 80, 108, 0, 43, 99, 117, 67, 29, 4, 72, 107, 8, 90, 113, 70, 58, 96, 23, 119, 103, 27, 26, 37, 18, 59, 54, 87, 14, 15, 126, 89, 16, 91, 94, 105, 53, 125, 10, 40, 106, 2, 51, 6, 118, 39, 82, 68, 42, 100, 73, 57, 50, 44, 3, 61, 65, 47, 41, 110, 55, 64, 45, 109, 112, 5], [62, 124, 102, 60, 97, 122, 123, 116, 93, 38, 44, 117, 43, 127, 59, 121, 87, 48, 114, 53, 118, 24, 89, 54, 39, 29, 58, 46, 94, 56, 40, 25, 82, 119, 57, 95, 106, 50, 125, 108, 99, 51, 49, 20, 109, 55, 47, 86, 126, 52, 111, 112, 63, 113, 45, 41, 91, 98, 104, 42, 110, 22, 79, 61, 115, 120, 103, 100, 31, 27, 92, 107, 17, 23, 105, 37, 35, 76, 85, 36, 34, 90, 101, 80, 32, 33, 96, 26, 28, 84, 12, 30, 13, 21, 18, 83, 19, 77, 15, 81, 16, 8, 9, 88, 74, 78, 7, 6, 73, 11, 75, 14, 70, 71, 10, 4, 5, 72, 68, 69, 2, 3, 67, 1, 66, 65, 64, 0], [62, 121, 102, 117, 124, 114, 60, 49, 50, 53, 116, 97, 123, 43, 38, 108, 122, 93, 45, 63, 127, 94, 59, 113, 104, 46, 58, 25, 44, 87, 125, 106, 61, 52, 54, 109, 17, 31, 118, 111, 56, 48, 47, 103, 33, 42, 119, 51, 120, 112, 34, 126, 29, 110, 107, 115, 57, 41, 40, 55, 89, 28, 105, 39, 22, 99, 37, 27, 98, 95, 82, 85, 90, 86, 100, 35, 20, 30, 26, 91, 36, 101, 24, 79, 96, 76, 80, 32, 19, 18, 23, 12, 92, 21, 8, 11, 84, 75, 77, 64, 1, 78, 10, 6, 16, 14, 0, 83, 4, 66, 74, 68, 69, 81, 67, 72, 7, 2, 70, 5, 71, 9, 65, 13, 73, 88, 3, 15], [58, 126, 117, 39, 111, 124, 59, 48, 51, 112, 25, 125, 54, 127, 47, 118, 123, 35, 62, 21, 55, 57, 89, 113, 96, 49, 63, 50, 60, 53, 61, 119, 116, 56, 29, 91, 45, 120, 110, 87, 52, 23, 114, 82, 42, 121, 122, 115, 46, 85, 109, 44, 108, 80, 27, 76, 107, 20, 105, 14, 93, 106, 83, 99, 41, 40, 43, 104, 38, 102, 9, 103, 28, 18, 31, 32, 16, 101, 36, 22, 95, 100, 79, 24, 8, 88, 81, 12, 13, 4, 84, 78, 92, 33, 71, 90, 37, 86, 97, 10, 69, 6, 94, 34, 3, 11, 30, 74, 75, 98, 73, 19, 77, 26, 66, 7, 70, 72, 17, 15, 2, 5, 0, 68, 64, 67, 1, 65], [111, 117, 58, 126, 39, 124, 59, 51, 48, 118, 25, 21, 50, 57, 54, 127, 55, 91, 125, 62, 112, 49, 113, 47, 63, 53, 110, 96, 61, 52, 119, 123, 116, 35, 56, 60, 120, 29, 87, 89, 23, 42, 121, 115, 45, 114, 46, 85, 82, 44, 108, 122, 109, 80, 76, 27, 107, 14, 106, 93, 41, 37, 105, 9, 102, 38, 43, 83, 104, 88, 40, 8, 20, 99, 100, 18, 24, 12, 28, 31, 10, 13, 81, 79, 32, 71, 101, 92, 103, 36, 69, 97, 30, 78, 94, 4, 33, 6, 16, 70, 22, 3, 90, 98, 95, 34, 86, 84, 26, 73, 19, 75, 74, 7, 11, 66, 5, 72, 77, 17, 65, 64, 15, 2, 67, 1, 68, 0], [117, 126, 39, 58, 111, 124, 59, 51, 48, 112, 21, 25, 57, 54, 127, 118, 62, 29, 55, 110, 50, 125, 49, 53, 35, 96, 91, 123, 63, 52, 61, 119, 113, 56, 116, 89, 120, 60, 45, 47, 87, 115, 121, 114, 82, 23, 80, 46, 44, 108, 85, 122, 42, 109, 107, 27, 76, 83, 14, 105, 102, 40, 38, 43, 9, 103, 28, 41, 106, 104, 20, 93, 8, 99, 32, 10, 13, 31, 37, 24, 36, 18, 81, 79, 12, 88, 70, 100, 16, 71, 92, 78, 101, 34, 22, 84, 33, 90, 94, 3, 7, 6, 97, 86, 69, 95, 73, 11, 30, 4, 75, 98, 19, 74, 72, 77, 66, 26, 15, 17, 2, 5, 64, 1, 0, 68, 65, 67], [126, 117, 39, 58, 111, 124, 51, 59, 48, 21, 25, 112, 54, 57, 62, 55, 127, 89, 125, 96, 63, 118, 29, 123, 50, 53, 113, 35, 110, 49, 61, 52, 60, 119, 120, 56, 116, 87, 91, 47, 23, 42, 45, 121, 44, 114, 80, 115, 122, 85, 46, 82, 108, 109, 83, 14, 76, 107, 27, 20, 40, 105, 102, 9, 8, 103, 106, 43, 38, 104, 41, 99, 93, 32, 18, 31, 4, 28, 79, 37, 95, 36, 24, 16, 84, 10, 100, 13, 22, 101, 81, 12, 92, 33, 90, 71, 78, 97, 30, 69, 70, 86, 88, 34, 94, 6, 11, 19, 98, 73, 3, 75, 74, 26, 7, 72, 77, 15, 17, 0, 66, 5, 64, 68, 65, 1, 2, 67], [40, 59, 34, 90, 83, 12, 53, 91, 63, 88, 17, 95, 45, 112, 85, 31, 79, 9, 124, 123, 13, 6, 108, 119, 113, 5, 109, 74, 126, 72, 99, 93, 96, 15, 78, 114, 62, 58, 47, 28, 29, 61, 55, 8, 73, 107, 18, 97, 52, 35, 41, 20, 86, 48, 14, 33, 87, 76, 94, 127, 122, 92, 16, 50, 121, 11, 98, 82, 116, 24, 101, 57, 110, 77, 120, 103, 10, 51, 71, 111, 49, 38, 60, 125, 54, 115, 42, 56, 105, 118, 106, 80, 23, 44, 46, 69, 100, 26, 117, 32, 39, 30, 102, 43, 36, 81, 89, 7, 37, 27, 22, 21, 84, 19, 25, 75, 70, 3, 2, 67, 4, 68, 64, 1, 66, 104, 0, 65], [40, 59, 34, 90, 124, 88, 83, 13, 63, 126, 79, 17, 85, 6, 95, 50, 121, 118, 53, 48, 46, 127, 12, 45, 10, 91, 106, 103, 22, 29, 4, 72, 116, 68, 74, 96, 60, 98, 55, 99, 112, 11, 58, 9, 125, 35, 69, 41, 24, 115, 120, 43, 31, 37, 61, 122, 62, 0, 39, 33, 113, 123, 105, 110, 78, 21, 77, 84, 107, 51, 67, 20, 7, 87, 54, 119, 28, 38, 47, 14, 73, 114, 82, 93, 26, 75, 117, 32, 94, 66, 18, 42, 56, 108, 111, 52, 44, 36, 1, 109, 2, 8, 102, 15, 30, 70, 23, 19, 71, 57, 86, 92, 49, 76, 81, 100, 101, 80, 27, 89, 97, 5, 16, 25, 104, 3, 65, 64], [40, 59, 34, 90, 85, 83, 127, 124, 63, 53, 17, 48, 88, 110, 116, 118, 79, 95, 13, 113, 29, 58, 96, 91, 31, 42, 111, 121, 12, 24, 11, 50, 94, 115, 98, 38, 45, 60, 10, 112, 47, 14, 114, 103, 125, 22, 55, 108, 25, 35, 44, 119, 109, 100, 36, 9, 57, 56, 86, 117, 102, 49, 26, 39, 72, 28, 6, 71, 61, 101, 23, 89, 52, 43, 27, 122, 41, 21, 51, 33, 123, 62, 30, 126, 54, 106, 105, 87, 37, 99, 46, 97, 92, 107, 84, 93, 120, 20, 0, 18, 104, 81, 16, 4, 82, 76, 78, 32, 15, 19, 80, 1, 7, 68, 74, 69, 75, 2, 8, 77, 73, 70, 66, 3, 5, 67, 65, 64], [40, 59, 34, 90, 88, 83, 124, 79, 17, 13, 95, 53, 12, 118, 9, 58, 126, 72, 85, 24, 63, 81, 4, 48, 114, 86, 2, 71, 11, 6, 45, 44, 112, 121, 75, 103, 116, 1, 50, 10, 105, 14, 0, 110, 3, 31, 69, 60, 78, 42, 46, 106, 39, 120, 74, 67, 115, 54, 32, 29, 43, 26, 98, 62, 8, 51, 113, 68, 52, 7, 20, 22, 21, 80, 91, 65, 92, 107, 57, 35, 37, 47, 56, 127, 84, 18, 89, 125, 16, 77, 15, 19, 94, 41, 96, 82, 108, 70, 55, 25, 109, 104, 76, 28, 49, 27, 61, 64, 93, 97, 23, 36, 100, 33, 99, 122, 111, 101, 66, 30, 119, 38, 73, 123, 102, 117, 87, 5], [58, 61, 127, 56, 38, 120, 125, 53, 89, 52, 115, 112, 107, 50, 83, 122, 57, 106, 124, 59, 116, 85, 60, 119, 23, 114, 118, 121, 123, 117, 55, 126, 87, 48, 99, 46, 96, 102, 49, 63, 110, 54, 62, 51, 111, 47, 76, 109, 113, 29, 45, 94, 41, 18, 44, 91, 42, 16, 28, 43, 36, 25, 104, 108, 103, 92, 78, 95, 40, 90, 82, 105, 80, 100, 35, 84, 39, 14, 17, 26, 34, 73, 70, 69, 12, 71, 32, 21, 13, 22, 37, 3, 74, 7, 27, 101, 20, 97, 98, 24, 31, 11, 33, 86, 79, 9, 30, 93, 8, 15, 88, 68, 72, 75, 5, 19, 10, 77, 81, 66, 4, 2, 6, 1, 64, 67, 65, 0], [56, 127, 61, 58, 38, 120, 89, 125, 53, 52, 50, 115, 59, 57, 124, 83, 114, 107, 119, 116, 112, 85, 106, 122, 118, 60, 55, 121, 123, 117, 126, 102, 23, 99, 63, 51, 54, 49, 110, 48, 46, 62, 96, 45, 29, 111, 47, 87, 94, 109, 113, 18, 25, 43, 76, 16, 41, 95, 44, 42, 90, 36, 91, 108, 104, 103, 92, 28, 40, 35, 80, 105, 73, 78, 84, 100, 39, 21, 82, 17, 70, 32, 71, 37, 13, 31, 98, 14, 8, 74, 26, 97, 22, 12, 101, 79, 24, 34, 27, 86, 11, 3, 15, 7, 20, 30, 33, 69, 77, 93, 68, 9, 72, 10, 19, 88, 81, 5, 0, 2, 6, 75, 4, 67, 1, 64, 65, 66], [61, 127, 58, 38, 56, 125, 120, 53, 89, 52, 115, 107, 50, 124, 83, 57, 59, 119, 116, 114, 85, 112, 121, 118, 126, 60, 122, 55, 123, 46, 23, 117, 48, 49, 106, 110, 62, 99, 63, 51, 102, 96, 54, 94, 87, 111, 47, 42, 29, 45, 113, 109, 16, 18, 25, 76, 43, 28, 41, 44, 108, 103, 40, 78, 95, 36, 90, 105, 104, 91, 80, 92, 39, 84, 82, 100, 73, 34, 35, 17, 21, 37, 14, 32, 71, 13, 12, 26, 22, 33, 70, 101, 97, 20, 74, 86, 11, 27, 98, 79, 68, 8, 15, 10, 24, 30, 93, 31, 9, 72, 77, 3, 69, 7, 88, 19, 75, 6, 81, 5, 4, 0, 65, 2, 64, 66, 1, 67], [127, 61, 56, 38, 58, 120, 89, 125, 53, 52, 57, 50, 115, 119, 59, 124, 114, 83, 116, 55, 107, 112, 23, 85, 123, 122, 121, 63, 118, 126, 99, 60, 62, 110, 117, 29, 48, 49, 106, 96, 94, 102, 87, 51, 54, 46, 45, 111, 47, 113, 109, 16, 18, 25, 43, 42, 44, 76, 90, 41, 91, 28, 108, 36, 92, 103, 104, 84, 40, 82, 14, 105, 26, 37, 35, 80, 100, 34, 95, 32, 39, 17, 12, 22, 69, 21, 3, 78, 13, 97, 70, 71, 101, 73, 74, 24, 98, 11, 20, 8, 86, 27, 33, 31, 30, 79, 93, 68, 72, 7, 10, 9, 15, 19, 81, 88, 77, 5, 75, 2, 4, 1, 6, 65, 66, 0, 64, 67], [42, 98, 106, 112, 26, 36, 21, 49, 118, 16, 94, 18, 54, 22, 48, 31, 60, 14, 13, 95, 101, 77, 93, 9, 123, 116, 88, 90, 114, 125, 58, 38, 113, 127, 53, 92, 15, 19, 8, 63, 47, 20, 52, 17, 99, 61, 124, 46, 12, 55, 111, 23, 73, 29, 27, 120, 115, 117, 84, 28, 122, 87, 10, 105, 51, 85, 33, 72, 24, 109, 104, 44, 34, 97, 74, 126, 45, 102, 43, 40, 30, 75, 59, 62, 107, 50, 4, 39, 100, 81, 108, 119, 121, 57, 41, 76, 2, 83, 35, 110, 71, 89, 64, 103, 37, 56, 67, 80, 7, 32, 91, 11, 82, 79, 25, 86, 96, 6, 0, 3, 78, 5, 1, 69, 65, 66, 70, 68], [42, 118, 98, 36, 106, 26, 18, 21, 124, 116, 63, 49, 111, 127, 54, 112, 123, 52, 88, 94, 24, 122, 126, 48, 31, 22, 99, 117, 53, 110, 55, 121, 58, 16, 38, 101, 20, 92, 47, 62, 33, 59, 61, 90, 95, 12, 51, 119, 60, 108, 46, 109, 8, 57, 29, 30, 120, 114, 28, 50, 44, 93, 11, 14, 113, 86, 125, 115, 107, 100, 39, 43, 77, 105, 56, 4, 34, 79, 97, 103, 40, 41, 27, 104, 45, 15, 37, 82, 102, 35, 23, 2, 10, 13, 96, 85, 74, 32, 75, 91, 25, 89, 0, 7, 87, 83, 84, 17, 19, 1, 73, 80, 71, 9, 81, 76, 6, 64, 67, 68, 5, 65, 78, 72, 66, 69, 70, 3], [42, 98, 118, 126, 106, 36, 26, 18, 112, 127, 24, 21, 48, 88, 46, 31, 41, 94, 63, 116, 20, 113, 125, 62, 53, 11, 54, 7, 8, 16, 52, 38, 49, 12, 124, 121, 123, 99, 111, 122, 39, 60, 4, 2, 50, 58, 109, 114, 120, 86, 117, 95, 90, 47, 30, 119, 57, 61, 1, 55, 33, 59, 79, 14, 56, 77, 37, 44, 51, 115, 103, 101, 92, 91, 107, 43, 75, 29, 6, 64, 102, 40, 110, 97, 105, 22, 108, 9, 80, 23, 45, 89, 19, 96, 93, 27, 82, 13, 67, 0, 104, 35, 83, 34, 17, 69, 85, 100, 32, 10, 74, 73, 25, 5, 76, 28, 84, 15, 81, 66, 87, 72, 65, 68, 71, 78, 3, 70], [42, 98, 106, 26, 31, 20, 16, 21, 18, 36, 112, 54, 75, 14, 77, 113, 9, 73, 49, 94, 79, 76, 15, 93, 116, 123, 118, 90, 101, 70, 10, 7, 19, 8, 1, 60, 53, 48, 6, 69, 58, 84, 74, 3, 34, 127, 78, 107, 114, 71, 99, 88, 0, 17, 2, 41, 105, 125, 67, 87, 109, 81, 25, 68, 119, 11, 28, 23, 22, 13, 27, 80, 40, 12, 64, 91, 82, 37, 72, 52, 4, 61, 47, 83, 62, 85, 38, 92, 24, 33, 32, 122, 44, 66, 115, 111, 46, 39, 86, 65, 104, 110, 102, 108, 120, 45, 63, 50, 96, 89, 5, 29, 55, 56, 95, 126, 57, 117, 59, 97, 100, 35, 121, 51, 124, 43, 30, 103], [39, 63, 64, 0, 66, 1, 12, 121, 3, 4, 2, 69, 79, 77, 9, 23, 87, 74, 70, 122, 43, 49, 7, 108, 127, 42, 83, 85, 53, 46, 67, 118, 48, 61, 18, 115, 31, 58, 93, 103, 113, 68, 45, 59, 105, 16, 99, 40, 47, 124, 52, 60, 75, 17, 20, 78, 89, 97, 19, 71, 114, 100, 112, 111, 33, 65, 55, 11, 84, 126, 25, 50, 57, 5, 88, 51, 116, 81, 6, 90, 102, 120, 98, 117, 14, 86, 21, 30, 109, 34, 32, 41, 92, 94, 80, 13, 10, 27, 28, 106, 73, 38, 56, 26, 123, 76, 96, 82, 101, 35, 29, 54, 15, 119, 36, 107, 22, 37, 8, 110, 91, 72, 104, 95, 125, 44, 24, 62], [39, 63, 87, 122, 79, 77, 121, 74, 12, 118, 9, 31, 46, 69, 4, 66, 83, 127, 23, 115, 108, 3, 17, 70, 64, 1, 93, 49, 100, 18, 43, 75, 20, 41, 52, 15, 72, 90, 85, 58, 42, 89, 7, 82, 16, 48, 78, 59, 13, 60, 113, 27, 124, 109, 50, 10, 94, 8, 112, 80, 92, 95, 125, 99, 106, 35, 67, 96, 117, 53, 114, 97, 81, 29, 19, 24, 91, 123, 55, 86, 11, 104, 56, 126, 111, 36, 21, 105, 101, 57, 71, 6, 65, 116, 62, 119, 54, 61, 45, 51, 26, 22, 33, 84, 47, 40, 120, 110, 88, 37, 38, 28, 14, 73, 34, 107, 25, 102, 44, 30, 98, 76, 68, 0, 5, 32, 2, 103], [39, 63, 64, 0, 1, 121, 12, 74, 4, 3, 23, 9, 69, 79, 77, 66, 87, 70, 2, 122, 43, 108, 49, 42, 127, 83, 46, 53, 93, 7, 68, 99, 48, 52, 61, 67, 5, 118, 40, 103, 31, 18, 85, 59, 113, 45, 58, 112, 114, 6, 65, 81, 60, 55, 120, 105, 51, 100, 116, 88, 71, 115, 25, 90, 126, 80, 34, 91, 22, 86, 14, 33, 11, 124, 17, 16, 78, 47, 72, 73, 20, 101, 94, 56, 123, 57, 37, 19, 89, 75, 8, 111, 35, 106, 109, 82, 29, 10, 36, 26, 97, 62, 27, 98, 96, 54, 28, 24, 92, 84, 119, 125, 117, 38, 104, 30, 76, 32, 95, 41, 102, 50, 107, 44, 13, 15, 110, 21], [39, 63, 74, 79, 9, 87, 77, 12, 4, 121, 3, 66, 64, 23, 1, 70, 69, 43, 108, 18, 72, 127, 93, 49, 42, 2, 122, 46, 67, 20, 31, 115, 40, 83, 5, 113, 48, 7, 53, 118, 61, 27, 68, 60, 45, 6, 19, 125, 100, 97, 52, 24, 114, 16, 124, 111, 71, 112, 58, 81, 47, 89, 13, 85, 17, 34, 10, 99, 86, 8, 14, 78, 76, 73, 59, 26, 82, 90, 51, 92, 33, 84, 120, 11, 75, 0, 65, 119, 29, 22, 126, 57, 62, 32, 80, 28, 95, 30, 117, 56, 88, 25, 116, 15, 94, 37, 105, 36, 104, 91, 109, 21, 102, 98, 41, 35, 50, 123, 55, 106, 101, 107, 44, 96, 110, 38, 54, 103], [106, 36, 85, 42, 63, 83, 77, 113, 80, 15, 6, 73, 122, 124, 91, 74, 119, 3, 71, 48, 27, 115, 68, 58, 127, 0, 2, 76, 25, 98, 50, 112, 95, 66, 32, 57, 118, 67, 93, 69, 94, 75, 12, 61, 9, 87, 65, 56, 5, 14, 1, 10, 19, 29, 22, 121, 79, 126, 4, 16, 11, 81, 13, 84, 125, 8, 54, 37, 64, 109, 7, 110, 88, 21, 116, 114, 70, 39, 46, 30, 120, 105, 33, 41, 18, 99, 55, 20, 23, 123, 86, 78, 59, 72, 102, 90, 52, 51, 34, 49, 82, 104, 117, 107, 26, 89, 96, 24, 53, 111, 45, 28, 40, 31, 17, 97, 44, 60, 35, 108, 62, 103, 43, 101, 47, 92, 38, 100], [106, 36, 63, 42, 122, 85, 91, 58, 83, 124, 48, 80, 113, 77, 50, 27, 61, 112, 119, 15, 93, 116, 29, 25, 87, 98, 115, 120, 95, 73, 59, 32, 74, 114, 46, 123, 51, 57, 35, 71, 45, 109, 117, 126, 44, 2, 121, 54, 94, 30, 96, 118, 127, 52, 47, 111, 65, 56, 125, 108, 60, 68, 105, 49, 110, 6, 41, 62, 90, 55, 26, 21, 39, 99, 86, 17, 53, 10, 81, 8, 38, 107, 11, 101, 43, 19, 78, 104, 40, 103, 75, 33, 82, 66, 102, 18, 20, 37, 88, 5, 14, 0, 23, 79, 97, 92, 24, 31, 16, 76, 28, 34, 89, 22, 3, 12, 84, 64, 72, 7, 13, 100, 69, 1, 9, 4, 70, 67], [106, 36, 63, 42, 122, 113, 85, 91, 50, 83, 27, 80, 117, 61, 48, 124, 25, 112, 29, 118, 98, 87, 93, 52, 123, 77, 95, 60, 15, 119, 32, 59, 58, 127, 115, 57, 96, 90, 53, 126, 51, 120, 17, 73, 54, 35, 49, 40, 94, 114, 41, 109, 45, 46, 44, 121, 55, 111, 116, 21, 110, 62, 47, 105, 74, 71, 56, 30, 125, 108, 99, 39, 107, 38, 88, 103, 24, 43, 6, 104, 11, 86, 23, 101, 78, 102, 37, 34, 20, 19, 97, 5, 79, 31, 75, 28, 8, 33, 14, 81, 22, 92, 68, 89, 100, 84, 10, 16, 66, 26, 12, 2, 82, 0, 13, 76, 69, 72, 18, 65, 3, 7, 4, 9, 67, 70, 1, 64], [106, 36, 122, 42, 85, 124, 113, 48, 91, 50, 83, 127, 93, 58, 63, 59, 27, 98, 80, 118, 32, 126, 77, 47, 15, 87, 29, 25, 73, 95, 35, 115, 57, 109, 119, 96, 39, 90, 54, 114, 17, 112, 55, 123, 99, 56, 94, 21, 71, 41, 117, 30, 74, 45, 11, 51, 61, 38, 6, 75, 52, 121, 49, 103, 5, 68, 46, 62, 120, 34, 111, 2, 116, 40, 60, 125, 110, 53, 108, 105, 86, 18, 24, 33, 44, 43, 107, 97, 81, 102, 104, 26, 19, 12, 88, 37, 101, 79, 10, 72, 89, 78, 8, 20, 31, 82, 92, 16, 28, 14, 23, 84, 0, 76, 3, 22, 65, 66, 13, 69, 100, 7, 9, 4, 70, 64, 67, 1], [108, 111, 93, 44, 24, 14, 125, 116, 15, 72, 85, 74, 82, 19, 121, 98, 76, 6, 47, 124, 81, 36, 63, 20, 35, 37, 119, 95, 1, 25, 11, 87, 3, 33, 67, 68, 126, 38, 97, 57, 64, 27, 34, 100, 29, 55, 77, 60, 96, 13, 51, 65, 40, 4, 62, 49, 70, 105, 123, 75, 52, 5, 30, 0, 79, 43, 73, 86, 41, 92, 71, 9, 56, 46, 113, 88, 32, 117, 112, 80, 31, 50, 7, 109, 103, 127, 84, 53, 107, 23, 10, 48, 104, 26, 106, 114, 45, 12, 115, 122, 83, 120, 90, 78, 2, 61, 89, 21, 17, 18, 99, 42, 28, 101, 110, 102, 59, 118, 54, 8, 94, 66, 22, 16, 58, 69, 39, 91], [108, 44, 111, 93, 74, 14, 24, 116, 72, 124, 125, 82, 76, 85, 19, 68, 15, 6, 36, 1, 98, 64, 13, 47, 87, 81, 37, 63, 4, 20, 126, 121, 100, 65, 29, 53, 3, 67, 35, 119, 46, 7, 51, 60, 43, 49, 11, 34, 123, 57, 95, 2, 50, 80, 41, 113, 0, 55, 62, 86, 101, 56, 127, 69, 45, 88, 70, 40, 73, 99, 75, 104, 97, 38, 96, 90, 92, 117, 105, 9, 5, 31, 27, 58, 10, 114, 61, 18, 23, 21, 12, 89, 25, 71, 33, 42, 83, 66, 102, 103, 77, 120, 107, 30, 112, 79, 54, 8, 122, 59, 78, 118, 28, 115, 48, 84, 16, 110, 106, 91, 26, 22, 32, 39, 52, 94, 17, 109], [108, 111, 44, 125, 93, 24, 82, 15, 14, 76, 13, 74, 50, 68, 36, 116, 124, 72, 19, 98, 1, 6, 121, 113, 0, 87, 20, 86, 89, 67, 9, 64, 47, 37, 126, 95, 34, 53, 60, 35, 11, 71, 63, 4, 33, 32, 119, 92, 29, 100, 127, 41, 22, 3, 62, 84, 73, 83, 80, 55, 59, 49, 46, 123, 85, 70, 16, 57, 94, 23, 43, 7, 56, 88, 27, 65, 52, 115, 51, 8, 120, 90, 96, 105, 30, 26, 31, 102, 103, 69, 110, 40, 97, 61, 81, 91, 42, 17, 28, 114, 38, 18, 106, 104, 10, 117, 45, 21, 58, 118, 12, 107, 78, 101, 112, 109, 99, 75, 39, 122, 77, 25, 48, 5, 79, 66, 54, 2], [108, 111, 93, 44, 82, 24, 125, 14, 68, 36, 76, 6, 72, 116, 22, 13, 74, 64, 19, 67, 50, 1, 37, 0, 124, 87, 15, 35, 98, 75, 119, 85, 103, 113, 47, 121, 18, 57, 51, 11, 55, 49, 29, 70, 62, 63, 34, 3, 53, 95, 88, 9, 127, 112, 86, 25, 65, 10, 60, 46, 126, 59, 20, 100, 89, 5, 97, 52, 2, 81, 40, 90, 109, 115, 120, 38, 92, 61, 117, 94, 104, 83, 30, 12, 96, 4, 123, 31, 84, 78, 56, 26, 43, 66, 33, 110, 48, 23, 41, 122, 54, 21, 28, 42, 106, 102, 58, 91, 39, 99, 107, 32, 7, 118, 73, 16, 105, 45, 114, 101, 27, 69, 80, 17, 8, 79, 71, 77]], "model.layers.25.self_attn.k_proj": [[62, 38, 121, 33, 124, 56, 127, 29, 20, 60, 72, 13, 26, 79, 83, 63, 74, 17, 91, 16, 78, 50, 24, 86, 80, 4, 77, 52, 71, 49, 22, 32, 25, 19, 102, 100, 123, 53, 47, 6, 9, 28, 120, 5, 18, 93, 117, 113, 12, 75, 99, 115, 3, 46, 110, 2, 54, 126, 119, 36, 111, 82, 44, 42, 95, 70, 116, 40, 34, 23, 57, 30, 87, 61, 118, 59, 43, 81, 8, 48, 92, 125, 58, 45, 106, 94, 11, 51, 41, 122, 88, 108, 0, 112, 107, 105, 55, 103, 65, 109, 37, 114, 21, 101, 31, 27, 89, 67, 10, 96, 104, 97, 85, 98, 35, 39, 68, 90, 1, 15, 84, 73, 14, 64, 7, 76, 69, 66], [103, 22, 99, 117, 126, 58, 32, 111, 59, 124, 51, 119, 60, 48, 62, 54, 93, 92, 114, 127, 61, 57, 123, 116, 56, 49, 120, 52, 55, 121, 63, 110, 46, 112, 50, 125, 122, 118, 113, 30, 53, 89, 43, 109, 44, 86, 115, 45, 41, 108, 42, 100, 104, 47, 40, 37, 27, 106, 107, 38, 35, 91, 105, 82, 34, 84, 97, 101, 80, 81, 14, 29, 79, 102, 21, 16, 36, 33, 90, 26, 94, 83, 78, 20, 18, 87, 76, 98, 31, 28, 19, 74, 24, 11, 85, 12, 95, 23, 25, 6, 7, 39, 77, 17, 96, 5, 8, 13, 75, 72, 15, 88, 68, 10, 67, 73, 2, 0, 1, 4, 9, 71, 69, 3, 66, 70, 65, 64], [59, 104, 98, 90, 83, 17, 13, 79, 72, 112, 63, 31, 12, 88, 24, 75, 118, 85, 0, 53, 50, 9, 124, 1, 4, 71, 121, 126, 14, 6, 86, 68, 81, 10, 67, 2, 125, 116, 111, 66, 123, 42, 55, 109, 41, 45, 11, 25, 57, 47, 52, 107, 46, 60, 7, 26, 28, 115, 20, 114, 49, 51, 95, 69, 48, 108, 58, 56, 18, 103, 84, 29, 74, 44, 91, 62, 94, 106, 22, 33, 54, 92, 122, 39, 21, 30, 119, 110, 82, 113, 117, 99, 16, 100, 105, 93, 89, 38, 8, 87, 23, 27, 43, 96, 80, 78, 102, 35, 120, 76, 97, 32, 3, 77, 101, 36, 37, 127, 70, 65, 61, 19, 15, 73, 5, 34, 64, 40], [102, 22, 61, 127, 35, 56, 58, 32, 125, 50, 52, 120, 60, 53, 30, 59, 98, 116, 111, 86, 49, 117, 55, 126, 115, 119, 124, 118, 110, 123, 54, 92, 121, 33, 63, 112, 122, 57, 48, 113, 16, 109, 62, 114, 51, 28, 43, 26, 46, 44, 47, 108, 42, 45, 100, 106, 36, 107, 105, 14, 18, 85, 27, 40, 99, 41, 37, 83, 104, 103, 101, 25, 11, 39, 89, 24, 34, 20, 77, 19, 29, 38, 23, 90, 91, 87, 82, 12, 17, 97, 94, 31, 93, 84, 74, 95, 72, 79, 96, 13, 5, 88, 73, 78, 15, 70, 8, 4, 10, 81, 67, 9, 80, 1, 6, 21, 76, 66, 7, 64, 75, 71, 2, 68, 0, 69, 3, 65], [106, 34, 30, 26, 49, 48, 118, 14, 64, 21, 88, 16, 42, 116, 18, 54, 123, 8, 6, 119, 20, 121, 60, 117, 53, 127, 114, 124, 55, 120, 31, 63, 4, 126, 105, 104, 87, 58, 67, 107, 10, 12, 2, 3, 28, 122, 99, 110, 74, 56, 47, 57, 125, 83, 111, 59, 1, 46, 109, 108, 62, 9, 52, 86, 43, 61, 51, 115, 112, 40, 44, 45, 98, 37, 113, 50, 79, 23, 17, 84, 65, 13, 101, 102, 39, 35, 103, 89, 22, 25, 41, 38, 77, 27, 73, 95, 100, 29, 75, 33, 11, 93, 19, 96, 68, 70, 32, 7, 36, 81, 91, 66, 97, 71, 15, 78, 69, 82, 94, 92, 5, 76, 0, 85, 72, 80, 24, 90], [63, 103, 64, 121, 9, 65, 1, 79, 12, 66, 4, 77, 69, 3, 87, 74, 70, 122, 44, 113, 107, 118, 110, 106, 67, 23, 127, 48, 52, 115, 18, 58, 68, 7, 20, 5, 85, 59, 81, 6, 40, 105, 50, 83, 31, 93, 61, 78, 53, 117, 47, 89, 39, 2, 33, 60, 27, 19, 124, 46, 43, 125, 75, 95, 109, 16, 17, 80, 84, 35, 10, 36, 29, 49, 126, 25, 72, 42, 90, 112, 55, 116, 11, 54, 92, 99, 98, 71, 56, 97, 73, 0, 21, 86, 111, 45, 15, 41, 37, 38, 82, 30, 51, 101, 8, 96, 62, 114, 24, 104, 91, 26, 14, 88, 119, 28, 120, 123, 100, 57, 94, 108, 34, 22, 32, 102, 76, 13], [42, 63, 122, 83, 85, 100, 112, 0, 119, 80, 15, 77, 91, 73, 124, 29, 115, 50, 57, 127, 66, 106, 25, 113, 96, 71, 6, 61, 87, 109, 34, 126, 121, 74, 65, 68, 3, 58, 75, 1, 49, 45, 11, 46, 2, 111, 56, 90, 51, 30, 44, 125, 120, 95, 8, 17, 81, 5, 14, 60, 4, 98, 31, 24, 108, 55, 23, 93, 41, 89, 59, 116, 27, 36, 110, 52, 12, 118, 102, 62, 86, 35, 10, 47, 94, 114, 104, 54, 48, 101, 105, 82, 43, 33, 18, 103, 78, 107, 40, 20, 38, 22, 99, 117, 53, 123, 28, 92, 7, 39, 84, 37, 26, 97, 76, 88, 32, 69, 13, 72, 64, 67, 16, 19, 70, 21, 79, 9], [44, 111, 108, 6, 0, 93, 47, 24, 14, 72, 76, 68, 65, 74, 64, 82, 2, 15, 53, 66, 125, 124, 50, 67, 116, 9, 85, 19, 87, 20, 81, 121, 119, 52, 13, 3, 57, 70, 75, 34, 101, 7, 11, 113, 73, 54, 86, 100, 62, 51, 127, 55, 80, 126, 120, 112, 10, 90, 16, 123, 104, 63, 60, 110, 49, 41, 61, 97, 45, 32, 107, 4, 96, 105, 89, 115, 95, 58, 59, 114, 22, 83, 88, 122, 79, 37, 36, 31, 5, 1, 109, 33, 56, 43, 117, 98, 46, 103, 118, 99, 94, 27, 39, 28, 23, 21, 42, 91, 35, 102, 29, 25, 40, 69, 38, 48, 106, 84, 30, 71, 26, 92, 18, 77, 12, 17, 78, 8]], "model.layers.25.self_attn.qk_proj": [[63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 61, 108, 121, 126, 117, 56, 122, 15, 93, 64, 50, 0, 79, 102, 125, 60, 77, 19, 118, 53, 13, 85, 90, 104, 10, 12, 23, 83, 115, 21, 36, 38, 76, 24, 113, 87, 98, 26, 103, 123, 73, 112, 119, 9, 74, 88, 116, 49, 52, 27, 68, 48, 47, 120, 6, 82, 34, 65, 46, 18, 4, 2, 39, 55, 57, 114, 67, 80, 51, 16, 1, 78, 17, 66, 54, 70, 29, 14, 99, 3, 40, 8, 31, 81, 86, 72, 110, 95, 97, 89, 33, 20, 84, 43, 22, 35, 109, 41, 25, 107, 32, 94, 69, 91, 100, 96, 7, 5, 71, 11, 105, 37, 28, 30, 75, 45, 101, 92], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 124, 121, 117, 108, 126, 56, 122, 0, 93, 102, 15, 50, 79, 64, 125, 77, 60, 19, 23, 118, 10, 21, 12, 112, 53, 103, 38, 76, 83, 13, 6, 90, 98, 115, 74, 119, 88, 113, 4, 24, 36, 9, 87, 116, 85, 104, 26, 49, 39, 48, 68, 73, 123, 120, 52, 47, 34, 65, 40, 2, 29, 82, 27, 57, 46, 1, 16, 66, 18, 55, 54, 72, 78, 14, 80, 17, 51, 3, 70, 99, 110, 114, 81, 89, 31, 22, 67, 97, 20, 109, 94, 8, 95, 43, 33, 25, 86, 107, 84, 41, 100, 5, 11, 75, 71, 96, 45, 28, 91, 69, 35, 105, 7, 30, 37, 92, 32, 101], [63, 59, 42, 62, 106, 44, 111, 127, 58, 61, 126, 108, 121, 124, 117, 56, 64, 93, 122, 125, 50, 79, 60, 0, 77, 118, 15, 83, 13, 112, 19, 119, 23, 90, 102, 36, 76, 21, 38, 98, 10, 74, 116, 115, 87, 24, 12, 120, 85, 73, 103, 9, 104, 113, 53, 49, 88, 4, 34, 48, 39, 6, 68, 82, 29, 26, 52, 66, 47, 27, 2, 1, 65, 72, 123, 3, 16, 18, 57, 70, 67, 80, 54, 51, 55, 46, 40, 78, 114, 14, 99, 17, 31, 89, 81, 110, 86, 35, 94, 22, 20, 109, 33, 84, 100, 25, 95, 107, 91, 96, 43, 69, 71, 105, 5, 8, 97, 92, 41, 28, 75, 32, 7, 11, 45, 30, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 126, 117, 56, 124, 122, 125, 0, 93, 64, 50, 79, 53, 102, 60, 77, 15, 83, 115, 13, 112, 113, 19, 21, 10, 85, 118, 38, 12, 119, 74, 76, 87, 73, 23, 26, 98, 104, 34, 9, 36, 24, 88, 39, 6, 120, 52, 49, 116, 55, 4, 103, 68, 65, 66, 67, 72, 40, 90, 80, 48, 3, 47, 16, 46, 123, 114, 18, 54, 1, 78, 82, 27, 110, 57, 29, 70, 14, 99, 2, 17, 81, 51, 31, 89, 86, 94, 33, 20, 43, 22, 109, 45, 97, 69, 35, 84, 25, 100, 41, 5, 71, 8, 7, 95, 91, 96, 105, 107, 28, 75, 32, 92, 11, 30, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 126, 61, 108, 56, 117, 124, 125, 122, 50, 93, 79, 60, 15, 0, 102, 64, 13, 77, 118, 119, 23, 83, 53, 113, 98, 38, 74, 85, 116, 104, 112, 76, 73, 19, 115, 12, 36, 90, 10, 21, 103, 123, 34, 4, 9, 49, 88, 55, 24, 52, 87, 26, 72, 46, 68, 120, 70, 48, 27, 39, 65, 18, 2, 57, 82, 1, 47, 16, 14, 99, 66, 114, 51, 81, 80, 6, 17, 54, 29, 40, 67, 78, 31, 3, 22, 110, 86, 25, 33, 94, 35, 95, 20, 107, 89, 84, 43, 109, 41, 97, 75, 69, 105, 45, 7, 101, 71, 5, 92, 96, 28, 30, 100, 91, 8, 32, 11, 37], [63, 59, 106, 62, 42, 44, 58, 111, 127, 121, 61, 124, 126, 108, 56, 117, 122, 93, 125, 79, 50, 64, 0, 15, 13, 102, 77, 83, 118, 115, 119, 36, 23, 85, 90, 21, 98, 87, 52, 12, 88, 19, 112, 38, 60, 113, 34, 10, 49, 104, 76, 73, 53, 103, 26, 4, 9, 116, 74, 24, 27, 72, 68, 70, 39, 17, 1, 14, 51, 16, 46, 81, 66, 57, 55, 47, 48, 6, 82, 2, 3, 40, 123, 18, 120, 80, 65, 99, 54, 114, 67, 22, 78, 86, 110, 97, 29, 84, 35, 31, 89, 94, 25, 41, 20, 32, 33, 109, 69, 107, 43, 100, 5, 75, 95, 7, 8, 105, 71, 96, 28, 11, 30, 45, 92, 91, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 108, 124, 126, 117, 56, 122, 79, 93, 102, 15, 0, 77, 64, 125, 19, 13, 85, 26, 118, 113, 60, 98, 38, 119, 21, 112, 83, 76, 116, 88, 74, 90, 104, 50, 12, 23, 103, 36, 1, 9, 49, 10, 53, 24, 52, 73, 57, 48, 39, 70, 34, 87, 18, 47, 46, 27, 16, 82, 115, 72, 2, 51, 81, 123, 80, 65, 68, 99, 14, 120, 67, 66, 17, 78, 55, 4, 54, 6, 29, 40, 3, 25, 22, 86, 31, 35, 114, 94, 41, 33, 110, 109, 89, 8, 100, 107, 84, 5, 20, 75, 97, 32, 95, 11, 28, 91, 7, 45, 30, 69, 71, 43, 105, 101, 37, 96, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 108, 61, 117, 126, 56, 122, 93, 125, 0, 64, 79, 60, 15, 50, 118, 13, 77, 116, 112, 38, 21, 90, 102, 52, 83, 19, 23, 103, 26, 85, 119, 10, 48, 12, 36, 98, 113, 24, 53, 88, 76, 87, 74, 70, 57, 39, 27, 73, 65, 9, 49, 47, 104, 1, 51, 29, 4, 68, 34, 46, 18, 123, 115, 2, 55, 66, 82, 78, 16, 120, 54, 72, 81, 99, 80, 6, 110, 22, 95, 86, 14, 17, 40, 3, 67, 31, 33, 114, 89, 25, 94, 35, 8, 84, 97, 100, 107, 41, 43, 28, 20, 109, 45, 91, 69, 5, 96, 7, 30, 75, 32, 37, 11, 105, 71, 92, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 117, 126, 56, 122, 93, 125, 0, 64, 15, 50, 79, 85, 13, 102, 77, 38, 90, 19, 118, 112, 83, 116, 21, 23, 26, 103, 60, 119, 113, 24, 88, 53, 76, 12, 10, 48, 52, 74, 68, 87, 73, 34, 46, 27, 39, 36, 9, 98, 47, 115, 4, 104, 51, 29, 49, 70, 1, 120, 65, 82, 123, 14, 80, 114, 55, 6, 17, 66, 16, 3, 54, 57, 67, 78, 81, 2, 18, 99, 110, 31, 22, 8, 86, 40, 25, 72, 89, 35, 100, 97, 95, 94, 20, 43, 33, 109, 91, 84, 107, 41, 69, 5, 45, 30, 96, 75, 28, 71, 32, 11, 7, 37, 92, 105, 101], [63, 59, 106, 42, 62, 44, 111, 58, 127, 121, 124, 108, 61, 117, 126, 56, 122, 0, 64, 93, 125, 119, 50, 15, 60, 79, 102, 53, 83, 116, 118, 77, 74, 13, 38, 90, 12, 120, 23, 112, 98, 76, 85, 10, 73, 103, 21, 19, 36, 113, 9, 88, 104, 115, 49, 68, 24, 46, 48, 6, 4, 87, 26, 70, 34, 47, 39, 1, 52, 65, 57, 66, 18, 54, 2, 8, 51, 3, 123, 80, 27, 29, 67, 55, 14, 17, 40, 81, 114, 82, 86, 16, 110, 72, 78, 97, 99, 31, 25, 22, 94, 33, 84, 43, 89, 20, 35, 7, 95, 69, 75, 100, 11, 28, 30, 32, 5, 71, 96, 107, 105, 45, 109, 41, 91, 37, 101, 92], [63, 59, 62, 42, 106, 44, 58, 111, 127, 124, 108, 121, 61, 117, 126, 56, 122, 93, 0, 64, 79, 15, 13, 125, 119, 50, 77, 102, 38, 116, 118, 83, 53, 60, 21, 10, 74, 85, 112, 12, 19, 23, 76, 98, 73, 49, 113, 104, 34, 36, 6, 90, 9, 115, 87, 88, 68, 103, 4, 48, 24, 120, 2, 39, 26, 8, 55, 65, 80, 1, 47, 52, 46, 40, 66, 54, 51, 123, 14, 70, 16, 99, 81, 67, 3, 82, 86, 29, 27, 78, 18, 17, 114, 110, 94, 22, 57, 35, 84, 31, 33, 25, 20, 97, 72, 89, 69, 96, 95, 107, 43, 7, 75, 71, 5, 105, 11, 28, 91, 100, 30, 92, 32, 41, 45, 101, 37, 109], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 108, 126, 124, 61, 56, 117, 122, 0, 93, 15, 79, 64, 77, 13, 85, 19, 83, 50, 102, 118, 21, 116, 9, 23, 90, 53, 38, 98, 104, 26, 10, 76, 125, 112, 60, 88, 74, 36, 119, 24, 12, 103, 48, 113, 73, 68, 6, 87, 4, 49, 40, 82, 65, 115, 47, 39, 46, 27, 34, 55, 52, 8, 123, 120, 14, 80, 17, 57, 16, 54, 2, 99, 1, 110, 29, 67, 114, 81, 18, 66, 22, 78, 3, 86, 31, 70, 97, 35, 94, 69, 51, 84, 25, 107, 33, 72, 89, 30, 28, 20, 75, 41, 5, 100, 45, 43, 95, 96, 91, 7, 11, 37, 71, 105, 32, 109, 101, 92], [63, 59, 42, 106, 62, 44, 111, 58, 127, 124, 121, 61, 108, 126, 117, 56, 122, 93, 64, 116, 118, 102, 50, 0, 15, 79, 83, 112, 90, 125, 77, 13, 19, 48, 113, 60, 23, 119, 85, 24, 26, 103, 104, 38, 36, 53, 21, 12, 87, 10, 9, 76, 88, 98, 74, 73, 54, 39, 6, 52, 34, 8, 68, 49, 4, 115, 1, 65, 27, 123, 47, 18, 46, 82, 51, 120, 66, 14, 2, 55, 29, 57, 99, 16, 22, 17, 80, 40, 3, 81, 70, 89, 110, 25, 86, 114, 107, 94, 78, 67, 31, 33, 97, 20, 35, 84, 91, 28, 43, 95, 41, 32, 30, 105, 11, 109, 69, 100, 75, 5, 72, 7, 96, 45, 92, 71, 37, 101], [63, 59, 42, 106, 62, 44, 111, 127, 58, 61, 124, 108, 121, 117, 126, 56, 64, 122, 93, 50, 0, 60, 125, 77, 79, 102, 90, 15, 118, 116, 13, 119, 48, 83, 85, 112, 120, 36, 19, 53, 38, 52, 9, 76, 113, 10, 74, 87, 21, 27, 103, 23, 73, 6, 104, 12, 26, 24, 8, 34, 88, 98, 1, 51, 49, 39, 65, 115, 68, 47, 66, 4, 80, 54, 81, 67, 123, 16, 82, 18, 57, 110, 2, 29, 17, 70, 55, 99, 14, 31, 114, 86, 46, 3, 78, 22, 89, 40, 11, 43, 95, 20, 97, 100, 33, 41, 35, 72, 32, 94, 5, 69, 84, 45, 25, 37, 107, 7, 75, 109, 91, 28, 105, 30, 92, 71, 96, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 108, 121, 124, 56, 117, 126, 122, 0, 93, 64, 79, 102, 77, 15, 60, 50, 83, 125, 85, 13, 116, 76, 118, 74, 112, 38, 104, 36, 53, 19, 98, 119, 26, 10, 34, 9, 23, 48, 12, 24, 21, 113, 52, 90, 103, 73, 68, 27, 87, 115, 88, 39, 8, 55, 1, 6, 70, 49, 4, 67, 2, 120, 66, 80, 82, 65, 29, 18, 51, 3, 47, 16, 81, 123, 14, 46, 54, 99, 86, 31, 57, 22, 78, 17, 97, 110, 114, 40, 43, 100, 72, 20, 33, 89, 94, 35, 41, 84, 25, 107, 11, 7, 96, 95, 69, 5, 30, 75, 105, 28, 32, 45, 71, 91, 101, 92, 37, 109], [63, 59, 62, 42, 106, 44, 127, 111, 58, 124, 121, 126, 108, 61, 56, 117, 122, 93, 79, 15, 64, 77, 50, 85, 0, 125, 116, 102, 83, 112, 26, 76, 118, 13, 38, 23, 73, 90, 24, 60, 19, 113, 87, 74, 98, 10, 12, 9, 103, 119, 21, 88, 53, 4, 55, 34, 27, 47, 36, 104, 48, 39, 68, 120, 52, 115, 80, 1, 40, 123, 16, 49, 14, 8, 70, 78, 29, 46, 114, 17, 18, 81, 57, 2, 51, 82, 3, 66, 65, 6, 110, 54, 67, 99, 33, 86, 97, 31, 22, 20, 72, 89, 69, 95, 84, 94, 41, 35, 5, 107, 25, 43, 11, 100, 105, 7, 28, 75, 91, 71, 45, 109, 30, 96, 101, 32, 37, 92], [63, 59, 42, 106, 62, 44, 58, 111, 127, 121, 124, 61, 126, 108, 117, 56, 122, 93, 64, 0, 15, 102, 50, 79, 23, 118, 125, 112, 90, 19, 116, 13, 119, 83, 77, 103, 113, 85, 88, 87, 38, 21, 53, 39, 26, 104, 120, 98, 12, 36, 76, 60, 24, 74, 27, 73, 48, 10, 34, 9, 29, 70, 47, 52, 65, 66, 123, 82, 49, 68, 1, 4, 80, 54, 46, 40, 16, 51, 114, 110, 18, 2, 57, 3, 78, 115, 99, 14, 81, 72, 95, 33, 17, 8, 67, 89, 86, 97, 6, 22, 31, 84, 25, 41, 94, 55, 20, 43, 35, 5, 107, 11, 28, 45, 75, 96, 100, 30, 69, 32, 91, 105, 71, 109, 7, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 127, 111, 121, 61, 117, 124, 108, 126, 56, 93, 122, 0, 64, 50, 118, 79, 116, 15, 77, 112, 13, 19, 102, 90, 125, 23, 76, 60, 38, 85, 87, 48, 21, 119, 83, 24, 74, 53, 98, 103, 88, 39, 70, 12, 104, 26, 1, 120, 113, 36, 73, 54, 10, 9, 115, 29, 34, 27, 4, 47, 52, 49, 66, 3, 114, 18, 82, 16, 65, 123, 110, 68, 67, 72, 81, 57, 80, 78, 2, 14, 51, 6, 99, 40, 55, 86, 89, 31, 8, 46, 22, 43, 17, 94, 95, 20, 33, 69, 25, 84, 28, 109, 107, 41, 35, 11, 100, 97, 91, 5, 96, 32, 7, 75, 71, 30, 45, 105, 92, 37, 101], [63, 59, 42, 62, 106, 44, 58, 111, 127, 121, 61, 108, 124, 117, 126, 56, 122, 64, 50, 102, 93, 0, 60, 79, 15, 13, 125, 116, 103, 77, 118, 98, 83, 53, 112, 104, 76, 23, 113, 38, 12, 119, 34, 19, 74, 36, 24, 10, 85, 70, 87, 21, 52, 9, 4, 90, 73, 88, 48, 123, 1, 120, 29, 47, 72, 46, 68, 26, 49, 39, 82, 114, 54, 115, 16, 66, 51, 67, 18, 55, 27, 40, 2, 80, 3, 110, 6, 99, 81, 57, 65, 78, 43, 14, 86, 33, 31, 94, 17, 20, 89, 35, 109, 25, 97, 71, 107, 105, 69, 22, 100, 96, 45, 32, 7, 91, 84, 5, 8, 11, 30, 95, 41, 101, 28, 75, 92, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 121, 108, 61, 117, 124, 126, 56, 122, 64, 93, 0, 102, 79, 15, 125, 50, 116, 23, 13, 60, 112, 77, 83, 118, 103, 19, 76, 12, 38, 119, 36, 87, 85, 74, 98, 120, 104, 34, 21, 26, 115, 90, 113, 48, 88, 73, 4, 53, 1, 10, 68, 72, 70, 24, 9, 52, 47, 49, 54, 123, 39, 18, 46, 2, 29, 40, 55, 6, 65, 27, 114, 16, 67, 82, 66, 99, 80, 81, 78, 57, 3, 51, 110, 17, 97, 14, 22, 94, 89, 86, 25, 107, 96, 105, 33, 41, 95, 100, 20, 71, 31, 35, 11, 69, 109, 84, 7, 43, 75, 8, 5, 91, 37, 32, 28, 45, 30, 101, 92], [63, 59, 106, 42, 62, 44, 127, 111, 58, 124, 121, 117, 61, 108, 126, 56, 122, 93, 0, 50, 15, 13, 125, 64, 116, 112, 79, 77, 104, 23, 118, 113, 60, 98, 74, 85, 87, 53, 83, 76, 120, 36, 12, 119, 38, 90, 24, 48, 49, 54, 102, 103, 9, 73, 19, 21, 10, 47, 72, 34, 88, 26, 70, 115, 46, 4, 27, 67, 114, 1, 55, 39, 66, 52, 29, 17, 40, 65, 6, 123, 51, 16, 18, 80, 2, 68, 82, 86, 78, 14, 57, 3, 110, 99, 89, 43, 81, 31, 94, 20, 25, 95, 33, 22, 5, 35, 97, 84, 107, 91, 100, 109, 71, 7, 96, 41, 32, 75, 8, 69, 37, 11, 105, 30, 28, 92, 45, 101], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 117, 126, 124, 56, 64, 122, 0, 93, 125, 15, 50, 79, 112, 77, 60, 13, 118, 53, 116, 23, 102, 119, 74, 24, 83, 12, 113, 120, 38, 85, 87, 10, 36, 115, 76, 104, 98, 19, 73, 103, 21, 1, 54, 9, 48, 26, 6, 72, 88, 90, 47, 49, 34, 66, 68, 51, 4, 82, 52, 123, 39, 114, 3, 46, 70, 65, 55, 18, 2, 29, 27, 40, 57, 67, 99, 16, 14, 110, 80, 31, 78, 17, 43, 33, 86, 35, 89, 94, 109, 84, 95, 81, 8, 22, 25, 20, 97, 5, 45, 11, 107, 7, 69, 91, 100, 28, 32, 71, 105, 30, 75, 92, 41, 96, 101, 37], [63, 59, 42, 106, 62, 44, 58, 111, 127, 61, 121, 108, 124, 117, 126, 56, 125, 93, 122, 0, 79, 23, 102, 118, 77, 50, 64, 116, 15, 60, 123, 76, 103, 13, 53, 90, 120, 83, 104, 115, 9, 24, 38, 21, 113, 74, 36, 98, 85, 87, 26, 12, 112, 48, 6, 19, 51, 73, 4, 10, 88, 34, 49, 68, 119, 52, 29, 1, 39, 66, 46, 55, 82, 47, 72, 114, 27, 18, 2, 54, 3, 16, 57, 80, 40, 95, 67, 99, 14, 33, 65, 81, 94, 31, 70, 78, 22, 110, 25, 35, 89, 86, 17, 20, 8, 97, 43, 91, 84, 105, 96, 71, 28, 45, 5, 11, 109, 32, 100, 69, 7, 30, 41, 101, 75, 107, 37, 92], [63, 59, 42, 62, 106, 44, 58, 111, 127, 61, 124, 121, 108, 126, 117, 56, 125, 122, 93, 15, 0, 64, 77, 23, 102, 79, 118, 83, 50, 13, 112, 116, 19, 104, 21, 38, 113, 53, 98, 49, 115, 60, 103, 12, 85, 74, 6, 76, 119, 9, 10, 26, 48, 90, 34, 120, 36, 87, 88, 24, 73, 4, 68, 123, 54, 52, 46, 72, 16, 51, 39, 65, 40, 47, 27, 82, 18, 55, 3, 66, 2, 80, 29, 114, 22, 110, 1, 81, 78, 94, 99, 14, 17, 67, 70, 33, 57, 45, 84, 8, 97, 31, 20, 25, 86, 89, 5, 32, 41, 35, 75, 71, 95, 96, 100, 43, 28, 11, 107, 69, 109, 7, 91, 105, 92, 37, 30, 101], [63, 59, 42, 106, 62, 44, 111, 58, 127, 121, 61, 108, 124, 126, 117, 56, 122, 93, 64, 15, 0, 102, 79, 125, 13, 50, 118, 83, 77, 90, 53, 98, 12, 85, 23, 21, 112, 19, 103, 119, 60, 113, 10, 24, 104, 76, 87, 36, 49, 9, 74, 116, 48, 26, 120, 73, 88, 38, 6, 115, 47, 123, 18, 39, 34, 82, 27, 52, 16, 65, 1, 2, 54, 40, 70, 14, 68, 55, 4, 80, 29, 66, 72, 86, 46, 57, 81, 110, 17, 67, 8, 51, 99, 31, 89, 114, 3, 25, 78, 20, 33, 22, 94, 97, 95, 84, 43, 107, 5, 32, 11, 71, 75, 7, 105, 100, 109, 35, 91, 96, 41, 28, 92, 30, 45, 69, 101, 37], [63, 59, 62, 42, 106, 44, 58, 111, 127, 121, 124, 61, 108, 126, 117, 56, 93, 122, 64, 50, 0, 15, 118, 125, 79, 116, 102, 13, 24, 90, 87, 48, 26, 23, 112, 120, 77, 60, 38, 119, 19, 12, 21, 103, 113, 85, 76, 49, 104, 83, 10, 73, 36, 54, 74, 98, 53, 47, 52, 123, 88, 39, 9, 27, 29, 34, 4, 6, 68, 51, 82, 115, 1, 70, 18, 2, 40, 16, 46, 65, 114, 80, 66, 67, 8, 99, 14, 3, 57, 89, 110, 31, 81, 22, 55, 33, 78, 25, 17, 43, 95, 86, 94, 97, 20, 72, 35, 84, 5, 91, 109, 7, 30, 41, 69, 11, 100, 71, 107, 45, 105, 28, 32, 75, 96, 92, 101, 37], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 121, 117, 108, 126, 56, 122, 93, 125, 50, 0, 64, 102, 118, 15, 112, 79, 48, 23, 90, 19, 13, 83, 116, 120, 103, 38, 77, 115, 39, 12, 76, 47, 113, 87, 49, 85, 53, 88, 74, 36, 73, 119, 60, 98, 24, 26, 21, 51, 52, 10, 123, 9, 4, 104, 68, 70, 40, 34, 55, 1, 110, 54, 57, 29, 27, 16, 46, 81, 78, 65, 8, 82, 67, 80, 2, 18, 66, 14, 17, 99, 6, 43, 94, 114, 22, 31, 35, 84, 3, 89, 95, 97, 33, 20, 91, 86, 25, 69, 5, 100, 41, 45, 28, 105, 96, 107, 72, 109, 11, 71, 37, 32, 7, 101, 92, 75, 30], [63, 59, 42, 106, 62, 44, 111, 127, 58, 121, 61, 124, 108, 117, 126, 56, 122, 64, 93, 79, 0, 15, 125, 50, 102, 13, 118, 60, 77, 116, 53, 12, 112, 23, 21, 38, 83, 74, 115, 10, 36, 119, 73, 85, 76, 103, 19, 70, 24, 87, 52, 98, 49, 26, 88, 114, 9, 113, 68, 8, 104, 51, 90, 34, 54, 47, 1, 120, 123, 4, 39, 80, 18, 65, 48, 2, 81, 16, 27, 29, 82, 66, 55, 57, 110, 78, 46, 14, 40, 94, 99, 3, 17, 6, 67, 43, 97, 22, 31, 89, 33, 20, 86, 25, 69, 84, 100, 45, 72, 75, 95, 35, 96, 11, 7, 107, 71, 109, 5, 32, 91, 37, 92, 30, 28, 105, 41, 101], [63, 59, 42, 62, 106, 44, 111, 58, 127, 124, 121, 108, 61, 117, 56, 126, 64, 122, 125, 15, 0, 93, 79, 102, 53, 115, 118, 23, 13, 77, 50, 10, 119, 104, 12, 73, 60, 98, 83, 19, 113, 103, 112, 85, 9, 24, 38, 70, 21, 90, 52, 36, 116, 8, 74, 26, 87, 49, 76, 120, 1, 48, 88, 39, 54, 46, 68, 51, 34, 123, 27, 47, 2, 65, 18, 57, 4, 66, 114, 16, 17, 67, 55, 81, 82, 80, 99, 31, 110, 3, 78, 29, 40, 97, 6, 14, 22, 86, 84, 89, 94, 33, 20, 25, 35, 69, 43, 100, 71, 7, 45, 109, 107, 72, 75, 32, 105, 5, 96, 95, 101, 30, 91, 41, 11, 28, 37, 92], [63, 59, 62, 42, 106, 44, 111, 127, 58, 124, 61, 117, 108, 121, 56, 126, 122, 93, 125, 15, 79, 50, 0, 118, 102, 13, 85, 12, 23, 116, 64, 77, 112, 113, 119, 19, 38, 83, 60, 76, 90, 26, 21, 52, 24, 49, 104, 87, 10, 73, 120, 4, 115, 88, 9, 98, 53, 103, 39, 70, 48, 123, 74, 34, 68, 8, 47, 27, 36, 81, 55, 54, 57, 40, 80, 51, 46, 1, 3, 18, 78, 16, 2, 67, 110, 114, 22, 65, 82, 29, 17, 66, 99, 14, 6, 20, 94, 31, 97, 25, 89, 33, 86, 84, 69, 43, 100, 5, 35, 95, 107, 75, 72, 41, 109, 32, 105, 71, 11, 28, 7, 30, 45, 92, 96, 91, 101, 37], [63, 59, 42, 62, 106, 44, 111, 127, 58, 124, 61, 121, 108, 117, 126, 56, 0, 122, 64, 93, 125, 50, 15, 118, 79, 102, 19, 13, 60, 98, 77, 112, 119, 36, 85, 53, 23, 116, 12, 113, 10, 120, 76, 26, 74, 83, 104, 73, 90, 103, 49, 9, 88, 24, 38, 52, 4, 21, 87, 115, 47, 48, 34, 123, 68, 70, 65, 8, 39, 1, 54, 27, 66, 18, 80, 29, 2, 51, 6, 40, 46, 55, 3, 82, 110, 81, 78, 114, 14, 57, 99, 31, 16, 89, 94, 67, 33, 17, 43, 97, 25, 22, 72, 86, 20, 35, 95, 100, 5, 84, 109, 32, 69, 28, 11, 75, 105, 45, 91, 71, 107, 92, 41, 7, 30, 96, 37, 101], [63, 59, 62, 106, 42, 44, 111, 58, 127, 124, 61, 121, 108, 117, 126, 56, 15, 122, 93, 50, 125, 79, 64, 13, 102, 19, 60, 112, 118, 12, 0, 77, 23, 74, 52, 116, 76, 98, 83, 87, 104, 10, 21, 53, 90, 103, 115, 119, 73, 85, 38, 9, 24, 26, 36, 113, 48, 123, 49, 120, 34, 54, 88, 39, 47, 8, 6, 29, 18, 27, 55, 70, 4, 82, 46, 68, 16, 65, 78, 1, 81, 3, 66, 80, 51, 57, 14, 110, 17, 22, 31, 94, 2, 99, 67, 40, 114, 89, 20, 25, 97, 72, 35, 95, 91, 84, 33, 86, 11, 32, 69, 28, 100, 43, 109, 107, 5, 7, 71, 75, 45, 105, 96, 30, 41, 92, 101, 37]], "model.layers.26.self_attn.q_proj": [[107, 43, 51, 99, 36, 100, 127, 53, 57, 52, 105, 116, 54, 58, 126, 60, 114, 48, 122, 62, 123, 118, 119, 32, 115, 55, 112, 124, 46, 120, 49, 125, 28, 44, 117, 106, 40, 113, 59, 56, 30, 35, 110, 63, 111, 42, 109, 103, 61, 39, 24, 47, 37, 108, 50, 41, 95, 121, 102, 104, 45, 96, 38, 93, 98, 23, 81, 92, 33, 76, 34, 29, 101, 84, 97, 85, 86, 88, 21, 31, 67, 91, 82, 25, 19, 22, 5, 94, 27, 89, 20, 26, 9, 90, 73, 77, 66, 83, 70, 18, 72, 87, 80, 79, 8, 78, 12, 17, 14, 15, 16, 3, 74, 75, 11, 6, 13, 0, 4, 69, 65, 7, 71, 1, 68, 10, 2, 64], [107, 43, 35, 65, 13, 74, 80, 7, 68, 28, 22, 32, 18, 126, 99, 127, 69, 3, 2, 57, 0, 119, 100, 52, 64, 118, 12, 122, 86, 72, 11, 114, 124, 54, 115, 53, 46, 9, 56, 62, 112, 67, 58, 1, 123, 117, 51, 108, 113, 34, 63, 49, 121, 116, 20, 6, 111, 47, 120, 84, 125, 110, 92, 61, 50, 10, 4, 83, 55, 78, 59, 70, 60, 71, 21, 16, 109, 5, 44, 31, 19, 66, 30, 17, 90, 45, 77, 76, 87, 48, 8, 85, 105, 79, 15, 41, 93, 73, 82, 81, 25, 23, 91, 75, 14, 26, 89, 101, 95, 42, 38, 104, 103, 27, 102, 24, 40, 106, 33, 37, 36, 97, 88, 29, 98, 94, 96, 39], [107, 43, 105, 32, 122, 100, 116, 37, 53, 103, 28, 124, 35, 92, 127, 57, 56, 21, 51, 39, 30, 52, 126, 49, 115, 117, 22, 58, 118, 112, 80, 62, 104, 108, 24, 55, 123, 41, 44, 109, 29, 91, 20, 54, 45, 18, 89, 23, 61, 17, 119, 75, 99, 46, 111, 34, 113, 114, 120, 27, 125, 63, 110, 121, 47, 42, 60, 19, 93, 48, 59, 72, 101, 50, 106, 38, 26, 25, 98, 95, 96, 40, 31, 84, 78, 83, 94, 33, 12, 90, 87, 79, 97, 85, 16, 13, 36, 11, 76, 82, 9, 70, 102, 15, 88, 77, 66, 81, 14, 73, 71, 86, 69, 8, 10, 4, 0, 6, 67, 7, 5, 74, 68, 3, 65, 1, 2, 64], [107, 126, 43, 127, 114, 52, 57, 37, 119, 103, 110, 99, 39, 122, 123, 46, 105, 54, 111, 118, 62, 28, 115, 55, 113, 124, 58, 84, 35, 44, 117, 120, 49, 19, 100, 112, 53, 63, 47, 9, 125, 116, 60, 48, 56, 51, 109, 61, 108, 59, 32, 76, 121, 104, 33, 50, 41, 45, 101, 36, 23, 80, 30, 93, 20, 89, 38, 22, 83, 42, 106, 40, 21, 91, 102, 67, 34, 87, 17, 97, 98, 29, 95, 11, 26, 78, 96, 25, 94, 92, 18, 85, 2, 15, 88, 86, 0, 13, 31, 65, 7, 4, 24, 6, 14, 10, 12, 90, 16, 79, 27, 74, 70, 77, 81, 5, 73, 82, 72, 8, 66, 1, 69, 68, 75, 71, 3, 64], [110, 55, 36, 46, 115, 95, 121, 26, 60, 50, 123, 61, 54, 53, 124, 100, 22, 119, 45, 58, 88, 31, 30, 116, 42, 127, 51, 40, 114, 106, 59, 92, 24, 47, 113, 44, 34, 117, 112, 105, 56, 28, 37, 41, 109, 63, 107, 99, 19, 122, 48, 52, 125, 126, 111, 39, 118, 49, 57, 43, 38, 33, 85, 93, 104, 120, 32, 17, 27, 108, 35, 25, 62, 87, 103, 101, 86, 21, 18, 102, 91, 29, 97, 98, 96, 16, 15, 94, 20, 23, 77, 90, 89, 14, 84, 75, 81, 76, 12, 80, 13, 83, 82, 74, 79, 9, 71, 78, 10, 5, 72, 7, 67, 6, 70, 73, 3, 68, 4, 2, 1, 11, 66, 8, 65, 64, 0, 69], [110, 46, 36, 26, 95, 22, 55, 19, 115, 4, 54, 30, 72, 31, 14, 100, 17, 58, 41, 62, 119, 2, 77, 74, 65, 121, 5, 63, 106, 37, 28, 125, 61, 97, 111, 91, 60, 10, 90, 107, 44, 94, 18, 73, 86, 52, 93, 32, 88, 104, 21, 84, 24, 118, 7, 33, 13, 34, 71, 45, 92, 39, 123, 11, 98, 75, 23, 6, 29, 101, 112, 113, 27, 116, 35, 117, 103, 127, 59, 42, 96, 25, 51, 89, 1, 15, 50, 81, 108, 85, 105, 79, 82, 80, 49, 69, 126, 76, 114, 12, 102, 20, 120, 87, 53, 99, 48, 9, 64, 56, 38, 57, 124, 47, 122, 67, 40, 68, 43, 109, 83, 3, 78, 70, 66, 8, 16, 0], [110, 36, 115, 62, 46, 26, 95, 30, 55, 88, 119, 22, 58, 100, 60, 31, 54, 51, 19, 124, 50, 96, 63, 125, 117, 34, 40, 86, 42, 123, 97, 41, 24, 122, 45, 44, 48, 87, 38, 126, 17, 116, 121, 108, 21, 16, 14, 59, 28, 61, 83, 49, 57, 120, 103, 52, 118, 33, 53, 127, 27, 106, 35, 111, 43, 114, 113, 107, 81, 56, 39, 89, 12, 93, 90, 91, 102, 47, 105, 94, 104, 109, 29, 112, 15, 98, 99, 32, 92, 101, 18, 79, 37, 80, 10, 82, 74, 25, 85, 75, 77, 20, 23, 7, 76, 8, 84, 72, 11, 78, 73, 13, 4, 9, 70, 5, 6, 69, 2, 67, 71, 66, 1, 0, 64, 65, 3, 68], [110, 36, 46, 54, 58, 121, 123, 50, 95, 26, 108, 62, 60, 119, 55, 30, 88, 31, 48, 100, 22, 49, 47, 24, 59, 33, 57, 114, 115, 109, 125, 113, 63, 56, 28, 17, 19, 112, 52, 106, 41, 117, 92, 122, 45, 51, 18, 21, 116, 15, 37, 42, 104, 40, 44, 61, 103, 111, 127, 126, 53, 107, 120, 86, 77, 118, 105, 14, 32, 124, 85, 39, 93, 99, 25, 101, 102, 43, 98, 38, 75, 13, 35, 20, 29, 16, 97, 90, 12, 34, 27, 83, 96, 89, 72, 94, 91, 87, 74, 84, 81, 23, 80, 7, 6, 73, 76, 10, 67, 5, 79, 1, 9, 4, 82, 2, 71, 64, 78, 8, 11, 65, 69, 66, 0, 70, 68, 3], [123, 57, 101, 33, 50, 37, 121, 88, 51, 44, 63, 92, 118, 24, 115, 62, 28, 60, 108, 23, 21, 54, 85, 116, 103, 117, 127, 97, 31, 105, 41, 61, 52, 56, 26, 114, 91, 109, 47, 82, 45, 110, 124, 49, 104, 126, 58, 125, 59, 34, 119, 112, 90, 55, 93, 48, 107, 111, 113, 18, 96, 122, 46, 106, 43, 87, 53, 42, 38, 79, 32, 40, 98, 20, 95, 83, 19, 75, 36, 100, 39, 15, 102, 30, 77, 120, 35, 89, 27, 94, 29, 99, 76, 16, 10, 17, 71, 86, 65, 25, 22, 84, 80, 67, 70, 14, 73, 69, 78, 81, 5, 68, 72, 8, 13, 12, 66, 0, 4, 1, 74, 2, 3, 9, 6, 64, 7, 11], [57, 101, 123, 33, 50, 37, 121, 24, 51, 124, 85, 103, 15, 88, 97, 76, 90, 20, 115, 82, 54, 105, 60, 10, 118, 29, 21, 34, 93, 104, 19, 96, 18, 91, 119, 95, 26, 87, 44, 117, 46, 67, 89, 49, 77, 59, 25, 79, 65, 28, 40, 62, 112, 73, 31, 99, 75, 30, 72, 12, 92, 74, 13, 9, 107, 43, 8, 7, 5, 83, 35, 102, 63, 127, 38, 94, 108, 114, 69, 4, 32, 2, 27, 125, 1, 52, 70, 22, 81, 71, 61, 109, 106, 53, 17, 68, 47, 126, 3, 6, 36, 23, 111, 39, 98, 116, 48, 16, 64, 100, 84, 120, 14, 110, 45, 41, 42, 58, 55, 122, 0, 113, 66, 78, 56, 80, 11, 86], [101, 123, 57, 50, 124, 33, 37, 121, 88, 28, 119, 21, 92, 61, 51, 90, 24, 44, 54, 85, 118, 126, 19, 58, 56, 105, 62, 114, 31, 117, 18, 16, 60, 115, 49, 103, 26, 82, 46, 52, 106, 97, 45, 116, 75, 38, 59, 14, 77, 42, 73, 47, 122, 86, 112, 91, 48, 109, 35, 107, 43, 63, 53, 76, 55, 39, 125, 94, 30, 110, 41, 113, 111, 89, 36, 79, 120, 80, 83, 127, 34, 93, 22, 95, 104, 99, 102, 17, 96, 29, 98, 108, 40, 71, 10, 100, 32, 20, 84, 27, 72, 81, 78, 5, 87, 23, 67, 25, 70, 65, 68, 13, 66, 15, 64, 4, 69, 11, 74, 9, 12, 2, 8, 7, 3, 1, 6, 0], [101, 123, 57, 50, 33, 121, 37, 51, 88, 24, 19, 115, 61, 54, 92, 28, 85, 21, 124, 75, 118, 105, 73, 117, 119, 77, 114, 60, 59, 44, 113, 18, 100, 46, 97, 62, 14, 83, 31, 90, 17, 98, 93, 58, 16, 95, 79, 43, 107, 78, 126, 104, 52, 45, 111, 48, 38, 127, 110, 125, 82, 41, 109, 47, 35, 116, 122, 103, 71, 29, 84, 112, 86, 89, 55, 30, 56, 42, 53, 76, 26, 36, 34, 120, 106, 5, 49, 87, 99, 63, 96, 40, 108, 102, 94, 32, 20, 4, 22, 39, 27, 11, 23, 91, 15, 80, 72, 10, 70, 66, 25, 67, 12, 13, 81, 68, 65, 6, 74, 7, 3, 1, 0, 9, 8, 69, 64, 2], [61, 59, 101, 63, 44, 125, 116, 51, 21, 62, 33, 58, 93, 118, 113, 55, 85, 88, 119, 127, 114, 110, 49, 56, 42, 54, 30, 123, 126, 124, 115, 52, 45, 50, 53, 40, 120, 112, 117, 57, 27, 48, 103, 78, 121, 122, 109, 60, 83, 24, 92, 105, 111, 37, 47, 80, 43, 90, 107, 91, 46, 26, 106, 89, 108, 95, 104, 41, 22, 82, 81, 25, 39, 74, 20, 11, 99, 75, 7, 13, 29, 12, 84, 77, 102, 32, 38, 9, 36, 6, 19, 10, 73, 17, 96, 100, 98, 34, 94, 8, 79, 23, 14, 69, 3, 15, 35, 16, 70, 28, 86, 31, 87, 18, 68, 72, 5, 76, 97, 67, 65, 71, 4, 0, 66, 1, 2, 64], [125, 63, 61, 101, 59, 116, 44, 51, 55, 58, 21, 50, 49, 127, 85, 62, 119, 114, 56, 110, 33, 113, 53, 126, 118, 52, 54, 124, 123, 57, 115, 48, 117, 42, 88, 45, 40, 120, 93, 37, 121, 109, 60, 83, 24, 112, 122, 30, 111, 27, 91, 47, 90, 46, 43, 92, 78, 105, 26, 107, 106, 41, 108, 103, 104, 89, 80, 39, 29, 22, 34, 38, 99, 96, 25, 102, 100, 82, 95, 17, 77, 81, 94, 7, 98, 36, 73, 32, 20, 75, 35, 28, 87, 31, 97, 11, 10, 13, 19, 79, 86, 6, 8, 74, 84, 12, 9, 23, 70, 14, 16, 67, 69, 72, 0, 4, 68, 18, 76, 15, 66, 3, 71, 5, 64, 1, 65, 2], [63, 101, 59, 61, 44, 21, 51, 33, 115, 85, 118, 127, 62, 113, 58, 93, 114, 119, 88, 49, 55, 126, 56, 116, 30, 54, 123, 110, 45, 117, 42, 53, 124, 50, 125, 103, 52, 40, 92, 120, 37, 57, 91, 48, 27, 112, 121, 83, 80, 109, 60, 78, 122, 111, 105, 47, 24, 43, 108, 46, 90, 26, 106, 107, 41, 104, 29, 7, 74, 82, 22, 95, 39, 89, 12, 20, 77, 25, 99, 81, 75, 19, 32, 13, 84, 14, 102, 17, 11, 38, 10, 73, 36, 15, 79, 16, 8, 100, 23, 34, 9, 94, 96, 98, 4, 18, 86, 35, 28, 69, 76, 72, 6, 87, 68, 5, 31, 97, 71, 70, 0, 1, 66, 67, 3, 64, 2, 65], [125, 59, 61, 63, 101, 62, 115, 54, 55, 116, 56, 33, 58, 44, 120, 126, 119, 114, 50, 110, 51, 124, 127, 118, 88, 53, 123, 37, 48, 49, 57, 117, 113, 112, 40, 52, 121, 45, 111, 60, 122, 90, 109, 93, 83, 42, 47, 103, 46, 43, 24, 108, 106, 81, 107, 41, 104, 78, 85, 105, 22, 7, 32, 39, 25, 89, 19, 80, 30, 74, 73, 21, 34, 6, 17, 27, 29, 91, 11, 100, 102, 26, 69, 36, 38, 77, 79, 75, 15, 13, 95, 82, 86, 12, 16, 14, 28, 97, 35, 98, 10, 9, 99, 87, 84, 8, 96, 92, 31, 94, 20, 4, 76, 68, 70, 5, 2, 23, 1, 0, 18, 67, 71, 66, 72, 65, 3, 64], [41, 117, 55, 34, 52, 31, 23, 115, 119, 21, 62, 39, 80, 58, 83, 45, 27, 105, 60, 99, 59, 44, 63, 13, 113, 30, 61, 72, 103, 57, 11, 109, 42, 29, 74, 32, 93, 94, 126, 111, 33, 88, 89, 8, 108, 112, 104, 121, 92, 116, 56, 118, 81, 90, 100, 54, 75, 26, 43, 17, 47, 37, 107, 25, 87, 20, 85, 24, 101, 51, 18, 91, 48, 15, 36, 22, 95, 124, 76, 127, 106, 110, 123, 71, 82, 50, 97, 49, 35, 38, 122, 114, 86, 84, 12, 19, 79, 40, 69, 96, 14, 46, 102, 28, 120, 53, 125, 78, 16, 67, 5, 10, 77, 64, 9, 66, 6, 7, 68, 98, 73, 4, 3, 70, 2, 65, 0, 1], [41, 55, 34, 52, 31, 13, 21, 23, 83, 80, 117, 27, 11, 66, 115, 105, 6, 96, 109, 28, 74, 19, 8, 49, 108, 44, 60, 73, 26, 64, 63, 67, 37, 30, 72, 119, 62, 4, 91, 45, 33, 85, 18, 22, 15, 7, 89, 43, 25, 17, 56, 87, 94, 9, 78, 84, 113, 122, 99, 77, 59, 93, 5, 61, 88, 57, 118, 81, 32, 48, 97, 70, 102, 10, 46, 42, 16, 29, 120, 92, 35, 71, 24, 82, 36, 103, 65, 69, 116, 2, 101, 20, 106, 40, 12, 14, 76, 90, 39, 68, 54, 127, 1, 53, 124, 123, 79, 75, 100, 110, 104, 111, 125, 86, 121, 47, 58, 3, 112, 38, 126, 50, 95, 0, 51, 114, 107, 98], [55, 41, 59, 34, 117, 45, 31, 23, 112, 54, 105, 119, 120, 110, 44, 111, 56, 46, 61, 92, 47, 116, 115, 58, 108, 42, 28, 118, 48, 124, 63, 121, 122, 123, 51, 114, 39, 127, 126, 109, 60, 25, 43, 125, 53, 57, 107, 94, 113, 21, 62, 50, 36, 106, 40, 20, 30, 49, 83, 97, 26, 103, 88, 102, 104, 100, 38, 33, 79, 52, 37, 99, 93, 29, 101, 80, 15, 35, 11, 91, 95, 86, 66, 96, 32, 71, 24, 27, 98, 69, 10, 7, 81, 87, 3, 74, 85, 17, 84, 14, 12, 89, 90, 22, 5, 1, 76, 70, 65, 82, 13, 64, 67, 0, 9, 8, 78, 19, 18, 68, 2, 4, 6, 73, 72, 75, 16, 77], [41, 55, 34, 52, 23, 31, 21, 13, 83, 80, 82, 74, 40, 117, 119, 125, 4, 103, 49, 28, 27, 105, 59, 60, 17, 115, 64, 61, 43, 44, 6, 101, 113, 66, 26, 96, 47, 118, 57, 56, 45, 94, 42, 62, 109, 3, 25, 116, 39, 70, 100, 65, 111, 112, 11, 53, 97, 72, 54, 108, 36, 126, 0, 85, 33, 104, 99, 29, 63, 123, 110, 18, 91, 16, 50, 38, 127, 32, 124, 68, 93, 87, 88, 92, 106, 122, 48, 102, 46, 22, 37, 84, 95, 58, 9, 89, 107, 69, 114, 35, 81, 78, 121, 30, 51, 10, 120, 79, 73, 19, 14, 15, 24, 7, 8, 90, 1, 86, 71, 98, 5, 12, 75, 20, 77, 76, 2, 67], [45, 103, 109, 38, 29, 54, 126, 87, 119, 57, 107, 83, 84, 114, 115, 117, 93, 118, 111, 47, 80, 122, 50, 95, 31, 46, 22, 121, 113, 127, 78, 61, 13, 108, 63, 85, 44, 10, 124, 53, 26, 52, 112, 98, 17, 59, 62, 49, 56, 125, 41, 116, 123, 51, 32, 71, 55, 58, 120, 110, 104, 30, 60, 96, 86, 36, 21, 42, 12, 48, 102, 37, 24, 34, 40, 100, 28, 106, 2, 88, 33, 35, 43, 89, 94, 70, 72, 101, 9, 14, 91, 92, 97, 90, 27, 69, 23, 76, 81, 105, 3, 6, 25, 75, 99, 19, 8, 0, 11, 67, 82, 68, 18, 4, 79, 20, 15, 73, 66, 7, 39, 65, 5, 1, 16, 64, 77, 74], [103, 45, 29, 109, 87, 80, 84, 13, 54, 10, 71, 69, 113, 32, 23, 115, 65, 47, 67, 126, 93, 64, 118, 3, 12, 127, 60, 77, 125, 38, 94, 17, 30, 62, 31, 72, 95, 57, 49, 59, 88, 20, 39, 66, 89, 68, 14, 119, 16, 83, 27, 24, 120, 74, 21, 102, 101, 79, 111, 34, 22, 18, 96, 7, 82, 15, 98, 85, 58, 90, 100, 121, 107, 51, 81, 76, 36, 5, 37, 42, 78, 25, 11, 48, 56, 63, 97, 46, 117, 91, 33, 112, 116, 4, 55, 104, 19, 99, 92, 73, 86, 41, 28, 50, 53, 6, 110, 105, 108, 52, 124, 35, 40, 61, 44, 122, 8, 70, 43, 123, 9, 106, 26, 114, 2, 0, 75, 1], [103, 45, 29, 109, 84, 87, 38, 13, 80, 54, 10, 69, 93, 71, 12, 83, 113, 3, 126, 2, 47, 32, 78, 127, 82, 79, 118, 65, 1, 115, 23, 22, 119, 88, 59, 64, 6, 95, 0, 46, 60, 89, 77, 123, 111, 31, 37, 49, 8, 62, 30, 66, 57, 122, 24, 5, 70, 97, 72, 33, 81, 125, 42, 15, 16, 58, 68, 43, 52, 7, 25, 124, 14, 74, 67, 96, 50, 112, 121, 20, 106, 85, 98, 107, 41, 92, 9, 34, 117, 17, 53, 51, 61, 120, 63, 48, 26, 114, 40, 91, 18, 110, 36, 55, 11, 101, 76, 19, 104, 99, 116, 108, 4, 35, 27, 94, 56, 100, 105, 86, 90, 44, 73, 21, 102, 28, 39, 75], [103, 45, 29, 109, 87, 84, 54, 80, 13, 10, 38, 126, 69, 71, 113, 93, 115, 32, 65, 47, 82, 67, 23, 118, 127, 30, 64, 95, 88, 40, 77, 56, 60, 125, 21, 102, 24, 94, 72, 79, 31, 46, 41, 108, 49, 83, 58, 20, 11, 101, 121, 78, 57, 25, 76, 66, 22, 26, 53, 52, 75, 117, 50, 17, 3, 119, 114, 44, 74, 42, 0, 16, 34, 9, 91, 97, 100, 48, 14, 4, 63, 62, 18, 59, 111, 124, 7, 15, 19, 5, 61, 96, 89, 122, 36, 37, 43, 112, 106, 8, 28, 55, 33, 90, 85, 104, 51, 105, 99, 81, 35, 107, 68, 110, 98, 86, 123, 12, 116, 27, 2, 39, 70, 73, 92, 120, 6, 1], [104, 59, 99, 92, 20, 114, 119, 54, 62, 82, 117, 55, 95, 98, 51, 107, 34, 63, 47, 120, 25, 125, 124, 49, 113, 115, 60, 86, 45, 13, 30, 112, 16, 123, 28, 103, 33, 58, 108, 121, 50, 48, 42, 56, 18, 44, 89, 61, 31, 39, 110, 87, 127, 52, 106, 116, 46, 122, 126, 88, 53, 57, 111, 105, 37, 118, 109, 41, 40, 43, 78, 29, 101, 38, 97, 93, 11, 94, 17, 22, 24, 76, 10, 80, 67, 15, 84, 36, 100, 71, 90, 21, 102, 79, 77, 26, 73, 96, 68, 85, 75, 23, 8, 91, 27, 19, 1, 81, 32, 35, 74, 14, 70, 7, 5, 83, 9, 66, 12, 0, 4, 65, 6, 69, 64, 72, 3, 2], [104, 59, 99, 92, 114, 82, 20, 51, 98, 62, 115, 95, 63, 105, 54, 47, 13, 16, 61, 117, 30, 34, 25, 119, 58, 122, 124, 45, 60, 109, 116, 86, 67, 55, 89, 87, 102, 66, 28, 11, 56, 80, 127, 43, 50, 88, 113, 15, 65, 123, 46, 57, 10, 18, 64, 53, 70, 42, 101, 7, 49, 48, 9, 31, 17, 97, 103, 8, 106, 44, 112, 78, 73, 40, 111, 68, 69, 22, 12, 94, 0, 5, 37, 121, 120, 76, 41, 125, 74, 33, 39, 84, 93, 29, 79, 38, 21, 118, 126, 24, 108, 75, 91, 32, 110, 107, 26, 85, 52, 71, 19, 90, 36, 83, 6, 23, 77, 81, 1, 100, 27, 14, 35, 96, 72, 4, 3, 2], [104, 59, 99, 92, 114, 20, 82, 16, 51, 116, 13, 25, 95, 87, 89, 54, 10, 115, 124, 34, 17, 8, 56, 50, 45, 53, 21, 100, 78, 76, 94, 86, 31, 63, 68, 62, 61, 18, 66, 70, 67, 65, 71, 123, 73, 80, 125, 98, 74, 85, 41, 52, 44, 12, 101, 97, 47, 39, 28, 40, 88, 91, 55, 29, 9, 127, 103, 15, 107, 118, 119, 108, 96, 11, 69, 27, 122, 37, 49, 90, 33, 84, 19, 7, 30, 5, 24, 106, 42, 121, 126, 32, 22, 58, 113, 110, 48, 105, 77, 112, 36, 6, 111, 79, 93, 23, 120, 43, 64, 109, 38, 75, 102, 26, 0, 60, 117, 46, 57, 14, 1, 83, 81, 35, 2, 4, 72, 3], [104, 59, 122, 45, 92, 20, 95, 114, 25, 99, 86, 115, 54, 62, 82, 63, 61, 34, 116, 51, 98, 49, 11, 56, 43, 0, 127, 124, 123, 33, 4, 119, 53, 1, 28, 110, 2, 46, 41, 58, 113, 31, 125, 55, 89, 120, 52, 70, 48, 111, 126, 76, 73, 50, 108, 118, 39, 121, 71, 112, 88, 78, 5, 13, 42, 106, 117, 109, 60, 83, 57, 44, 107, 105, 30, 22, 19, 87, 90, 47, 26, 102, 103, 84, 101, 16, 14, 38, 36, 21, 93, 3, 24, 100, 32, 37, 94, 6, 91, 81, 97, 35, 96, 15, 18, 8, 17, 29, 27, 65, 40, 12, 85, 10, 23, 69, 7, 66, 74, 75, 80, 64, 79, 9, 67, 68, 77, 72], [107, 56, 126, 43, 99, 87, 82, 110, 91, 78, 20, 25, 81, 40, 75, 104, 79, 27, 12, 112, 7, 72, 52, 31, 86, 32, 89, 117, 118, 49, 116, 92, 10, 125, 62, 48, 55, 16, 88, 127, 109, 95, 120, 26, 28, 85, 121, 69, 17, 93, 23, 44, 113, 74, 37, 77, 6, 57, 14, 53, 122, 84, 97, 58, 9, 80, 115, 61, 54, 11, 50, 101, 90, 102, 29, 76, 15, 108, 30, 51, 46, 68, 124, 18, 21, 83, 60, 114, 119, 105, 19, 96, 47, 33, 45, 24, 123, 38, 98, 59, 41, 100, 5, 103, 111, 94, 36, 63, 22, 2, 34, 106, 67, 8, 73, 71, 42, 39, 35, 70, 13, 4, 64, 3, 1, 65, 66, 0], [107, 43, 112, 56, 53, 127, 48, 115, 50, 116, 55, 125, 117, 25, 99, 19, 52, 120, 126, 61, 37, 118, 46, 121, 57, 62, 93, 113, 63, 58, 124, 29, 51, 114, 109, 122, 59, 54, 60, 49, 110, 91, 45, 119, 89, 31, 123, 36, 105, 47, 104, 111, 33, 44, 40, 86, 108, 24, 106, 42, 27, 28, 41, 95, 88, 38, 80, 39, 35, 101, 100, 83, 34, 102, 22, 20, 16, 103, 13, 98, 21, 77, 87, 97, 92, 90, 30, 17, 26, 79, 82, 32, 23, 70, 14, 96, 94, 81, 4, 10, 65, 85, 7, 11, 68, 6, 3, 1, 67, 84, 71, 15, 78, 18, 74, 5, 69, 75, 2, 72, 64, 66, 0, 8, 12, 9, 76, 73], [107, 126, 56, 99, 43, 117, 109, 25, 79, 86, 82, 31, 93, 91, 84, 89, 118, 116, 18, 33, 4, 110, 0, 73, 20, 77, 53, 28, 66, 78, 9, 55, 96, 41, 105, 1, 75, 98, 39, 112, 37, 81, 120, 121, 125, 72, 127, 36, 32, 17, 26, 30, 52, 95, 113, 54, 64, 80, 29, 3, 97, 92, 42, 6, 27, 76, 60, 50, 101, 58, 61, 94, 48, 40, 22, 114, 49, 63, 62, 57, 51, 44, 46, 122, 19, 23, 5, 14, 104, 38, 24, 15, 87, 90, 123, 69, 16, 67, 2, 34, 100, 59, 45, 70, 12, 10, 103, 21, 88, 102, 13, 7, 74, 119, 108, 124, 47, 83, 85, 111, 11, 68, 8, 35, 71, 65, 106, 115], [56, 107, 112, 99, 125, 19, 50, 126, 116, 43, 127, 25, 52, 117, 62, 121, 120, 48, 53, 45, 31, 124, 93, 57, 113, 40, 61, 109, 55, 46, 60, 51, 122, 49, 118, 63, 114, 54, 59, 29, 58, 123, 115, 119, 105, 47, 111, 36, 104, 89, 108, 110, 33, 35, 44, 91, 24, 86, 106, 27, 41, 42, 39, 38, 98, 88, 101, 37, 100, 83, 95, 103, 102, 28, 21, 80, 22, 92, 34, 90, 13, 74, 77, 17, 23, 30, 97, 87, 32, 26, 94, 81, 16, 96, 75, 10, 7, 71, 14, 20, 78, 65, 82, 70, 6, 1, 3, 79, 69, 4, 67, 72, 85, 18, 68, 11, 2, 5, 8, 66, 84, 15, 0, 64, 12, 76, 9, 73]], "model.layers.26.self_attn.k_proj": [[43, 22, 107, 99, 92, 126, 127, 96, 64, 47, 46, 56, 48, 80, 119, 124, 68, 7, 122, 13, 116, 57, 49, 109, 54, 113, 18, 2, 117, 112, 63, 118, 44, 62, 115, 55, 52, 58, 123, 50, 125, 53, 120, 51, 59, 121, 60, 110, 108, 114, 61, 101, 74, 98, 111, 11, 106, 42, 45, 76, 41, 102, 3, 37, 78, 36, 6, 40, 105, 103, 34, 104, 1, 97, 69, 39, 38, 10, 70, 8, 21, 89, 72, 15, 5, 94, 31, 90, 84, 33, 95, 87, 30, 9, 17, 100, 23, 24, 82, 28, 29, 26, 88, 91, 32, 81, 12, 25, 93, 27, 73, 85, 75, 67, 19, 20, 79, 71, 35, 83, 14, 16, 66, 77, 4, 65, 0, 86], [46, 110, 100, 31, 26, 22, 19, 86, 17, 14, 94, 12, 74, 28, 58, 121, 60, 24, 77, 15, 72, 62, 104, 123, 54, 29, 118, 43, 10, 106, 122, 52, 112, 32, 37, 61, 108, 68, 83, 49, 67, 7, 16, 40, 63, 125, 5, 44, 119, 127, 116, 97, 98, 33, 38, 113, 50, 27, 59, 89, 57, 41, 42, 111, 115, 109, 102, 126, 93, 91, 81, 56, 34, 101, 53, 69, 55, 70, 124, 103, 105, 84, 99, 48, 18, 117, 107, 71, 39, 51, 1, 95, 35, 23, 114, 47, 76, 120, 96, 87, 36, 65, 45, 0, 21, 64, 30, 88, 92, 25, 20, 85, 9, 13, 82, 8, 79, 75, 90, 6, 66, 80, 4, 3, 11, 73, 78, 2], [123, 57, 97, 37, 24, 95, 92, 85, 82, 90, 80, 108, 115, 124, 73, 75, 19, 50, 114, 104, 76, 40, 78, 77, 83, 101, 53, 59, 121, 39, 35, 70, 119, 10, 0, 88, 105, 14, 55, 111, 71, 5, 48, 45, 126, 61, 4, 103, 30, 93, 109, 52, 21, 43, 1, 113, 116, 3, 112, 49, 42, 72, 54, 62, 96, 44, 117, 107, 41, 51, 58, 106, 60, 81, 118, 66, 46, 56, 127, 47, 110, 15, 63, 122, 120, 17, 20, 125, 36, 86, 89, 102, 100, 91, 32, 99, 34, 33, 84, 98, 27, 22, 94, 38, 29, 87, 2, 13, 25, 23, 79, 18, 16, 67, 31, 8, 7, 74, 26, 28, 64, 11, 6, 12, 69, 65, 68, 9], [125, 37, 63, 61, 22, 97, 59, 114, 126, 119, 51, 55, 124, 117, 112, 58, 57, 127, 53, 60, 121, 110, 123, 108, 122, 56, 52, 49, 111, 107, 44, 118, 113, 45, 31, 50, 62, 47, 29, 115, 120, 48, 109, 104, 54, 46, 36, 106, 43, 101, 90, 105, 79, 42, 34, 27, 17, 102, 116, 41, 40, 83, 38, 35, 39, 98, 85, 88, 92, 86, 30, 81, 89, 103, 87, 91, 99, 20, 24, 25, 93, 100, 13, 15, 78, 95, 82, 28, 18, 32, 96, 73, 21, 75, 94, 84, 26, 19, 33, 12, 69, 23, 11, 6, 72, 4, 8, 80, 10, 77, 76, 7, 14, 16, 67, 2, 74, 65, 71, 64, 9, 70, 1, 68, 66, 5, 0, 3], [105, 55, 98, 52, 95, 23, 21, 80, 83, 64, 13, 61, 11, 116, 74, 6, 28, 117, 51, 60, 27, 46, 69, 8, 49, 109, 2, 90, 71, 59, 123, 65, 42, 57, 12, 111, 4, 115, 127, 67, 113, 72, 103, 53, 126, 50, 79, 54, 58, 93, 82, 124, 43, 118, 26, 100, 62, 119, 108, 56, 110, 25, 122, 44, 41, 112, 63, 78, 47, 34, 97, 45, 114, 99, 1, 104, 106, 29, 125, 37, 121, 107, 35, 30, 101, 32, 38, 39, 3, 120, 96, 18, 40, 102, 81, 33, 24, 48, 94, 15, 36, 20, 88, 17, 86, 91, 76, 9, 14, 77, 22, 89, 92, 75, 66, 5, 84, 31, 73, 10, 7, 87, 70, 19, 0, 85, 16, 68], [109, 39, 80, 84, 10, 93, 13, 87, 45, 71, 54, 69, 64, 3, 111, 118, 57, 12, 119, 29, 126, 117, 65, 121, 49, 78, 68, 106, 88, 31, 1, 9, 122, 127, 55, 123, 22, 51, 115, 113, 17, 104, 102, 96, 53, 82, 46, 76, 58, 125, 41, 14, 61, 63, 83, 2, 52, 59, 62, 112, 72, 15, 44, 60, 124, 101, 100, 32, 86, 36, 26, 50, 94, 95, 43, 30, 48, 105, 6, 47, 114, 108, 85, 107, 110, 42, 34, 27, 56, 98, 120, 116, 66, 21, 37, 75, 0, 40, 25, 91, 35, 28, 90, 79, 4, 67, 24, 89, 33, 99, 92, 38, 70, 18, 5, 19, 8, 23, 97, 73, 103, 7, 11, 81, 74, 77, 20, 16], [59, 40, 31, 20, 28, 86, 114, 98, 13, 113, 89, 88, 54, 115, 82, 56, 116, 111, 50, 99, 124, 21, 66, 0, 67, 35, 127, 41, 87, 62, 16, 53, 49, 17, 121, 27, 15, 68, 80, 126, 18, 11, 78, 44, 48, 55, 1, 109, 8, 70, 123, 47, 92, 10, 105, 58, 57, 112, 25, 60, 46, 118, 33, 122, 106, 63, 23, 51, 76, 29, 43, 125, 83, 30, 34, 36, 7, 61, 117, 119, 45, 52, 120, 73, 93, 65, 103, 39, 108, 26, 107, 42, 69, 9, 100, 96, 74, 97, 5, 101, 110, 102, 38, 85, 94, 79, 24, 81, 71, 64, 37, 90, 14, 32, 19, 6, 12, 72, 91, 4, 2, 77, 22, 75, 95, 3, 84, 104], [43, 35, 86, 56, 107, 29, 25, 19, 118, 126, 117, 95, 127, 116, 121, 57, 79, 109, 119, 55, 101, 92, 112, 125, 58, 63, 113, 41, 97, 52, 59, 51, 53, 54, 122, 114, 115, 61, 60, 62, 49, 124, 123, 45, 50, 111, 105, 82, 120, 48, 47, 64, 42, 44, 110, 20, 27, 34, 94, 46, 108, 80, 103, 38, 77, 37, 106, 104, 102, 75, 70, 100, 9, 65, 98, 39, 4, 40, 2, 36, 85, 84, 24, 7, 32, 67, 99, 90, 93, 69, 28, 16, 71, 26, 73, 96, 33, 11, 30, 87, 78, 91, 31, 76, 17, 81, 14, 21, 12, 18, 68, 23, 74, 8, 88, 10, 72, 13, 5, 15, 0, 83, 89, 1, 6, 3, 22, 66]], "model.layers.26.self_attn.qk_proj": [[59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 126, 29, 41, 50, 117, 99, 40, 101, 28, 115, 114, 23, 62, 119, 22, 58, 95, 92, 127, 121, 80, 31, 124, 87, 16, 53, 116, 13, 88, 86, 103, 60, 39, 77, 51, 20, 112, 111, 44, 21, 19, 104, 84, 97, 90, 48, 24, 47, 49, 85, 82, 93, 113, 118, 83, 122, 120, 10, 34, 18, 26, 74, 33, 106, 100, 98, 42, 108, 7, 35, 25, 89, 36, 91, 71, 64, 30, 72, 75, 11, 102, 32, 96, 81, 79, 15, 14, 0, 78, 17, 76, 38, 67, 12, 3, 69, 27, 94, 5, 66, 68, 1, 73, 9, 70, 65, 8, 2, 6, 4], [43, 59, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 29, 117, 101, 41, 115, 50, 114, 62, 40, 99, 127, 58, 28, 31, 121, 23, 92, 103, 51, 22, 124, 119, 112, 116, 77, 80, 95, 104, 39, 16, 87, 86, 24, 60, 49, 53, 13, 88, 85, 48, 21, 111, 93, 97, 44, 47, 120, 122, 20, 113, 19, 118, 83, 90, 34, 10, 84, 108, 82, 74, 100, 18, 33, 26, 35, 25, 89, 98, 42, 91, 71, 106, 15, 30, 36, 0, 75, 64, 72, 96, 11, 32, 7, 79, 67, 12, 14, 102, 5, 76, 81, 78, 3, 38, 94, 69, 27, 17, 9, 66, 73, 2, 6, 4, 1, 68, 70, 8, 65], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 125, 56, 45, 105, 54, 52, 37, 126, 29, 117, 41, 101, 114, 62, 99, 50, 40, 115, 124, 28, 104, 127, 121, 119, 22, 51, 112, 103, 86, 53, 80, 95, 92, 116, 31, 23, 87, 48, 77, 44, 16, 49, 88, 39, 58, 113, 97, 13, 93, 34, 47, 24, 21, 60, 20, 122, 118, 85, 83, 111, 90, 74, 19, 84, 120, 98, 108, 82, 10, 18, 33, 36, 26, 100, 35, 64, 106, 42, 91, 71, 89, 25, 0, 7, 15, 78, 38, 81, 17, 30, 11, 76, 79, 75, 14, 67, 96, 102, 32, 69, 72, 94, 3, 27, 5, 12, 1, 6, 68, 2, 4, 73, 65, 8, 66, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 117, 29, 114, 101, 50, 49, 41, 121, 40, 115, 62, 124, 99, 53, 44, 112, 104, 80, 119, 116, 23, 28, 22, 16, 92, 95, 86, 51, 77, 60, 113, 31, 13, 103, 58, 127, 111, 88, 87, 93, 120, 19, 48, 20, 39, 21, 118, 24, 85, 10, 90, 84, 97, 122, 108, 47, 74, 34, 83, 82, 33, 98, 26, 18, 0, 100, 106, 71, 7, 36, 89, 35, 81, 79, 42, 15, 102, 64, 91, 25, 17, 75, 78, 30, 14, 76, 5, 67, 11, 3, 38, 69, 72, 8, 96, 27, 6, 4, 12, 94, 32, 65, 68, 2, 1, 66, 73, 9, 70], [43, 59, 57, 123, 109, 110, 107, 46, 63, 61, 125, 56, 55, 45, 105, 52, 126, 37, 117, 54, 114, 50, 41, 29, 40, 121, 99, 119, 44, 101, 49, 124, 23, 22, 62, 127, 28, 51, 104, 53, 80, 16, 13, 116, 88, 115, 60, 86, 95, 118, 112, 92, 87, 77, 120, 31, 47, 58, 20, 84, 93, 103, 111, 74, 39, 34, 19, 10, 21, 97, 122, 83, 24, 48, 85, 113, 90, 82, 108, 18, 26, 98, 33, 35, 7, 100, 106, 64, 89, 71, 36, 75, 96, 0, 15, 81, 91, 25, 79, 5, 67, 76, 30, 102, 8, 78, 3, 11, 17, 32, 69, 14, 42, 4, 94, 12, 1, 6, 2, 38, 65, 27, 72, 68, 9, 66, 73, 70], [43, 59, 57, 123, 107, 110, 109, 46, 63, 61, 125, 55, 56, 45, 105, 52, 37, 54, 117, 29, 50, 126, 41, 99, 40, 23, 80, 115, 22, 114, 101, 86, 44, 116, 124, 16, 95, 28, 121, 104, 62, 13, 60, 92, 88, 49, 119, 118, 19, 103, 93, 51, 127, 77, 31, 39, 58, 85, 87, 112, 20, 21, 53, 111, 120, 84, 47, 97, 10, 74, 24, 48, 122, 34, 113, 83, 26, 82, 33, 108, 90, 100, 106, 36, 98, 15, 18, 35, 25, 7, 64, 0, 91, 11, 89, 81, 71, 30, 14, 12, 75, 102, 79, 32, 96, 42, 17, 5, 78, 76, 38, 8, 3, 67, 94, 72, 69, 66, 70, 4, 6, 27, 9, 73, 65, 68, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 63, 61, 55, 125, 56, 45, 105, 52, 37, 54, 29, 126, 41, 50, 117, 28, 121, 114, 124, 40, 99, 101, 22, 80, 119, 95, 87, 23, 86, 49, 115, 16, 92, 31, 44, 48, 88, 77, 13, 51, 116, 104, 62, 20, 122, 60, 58, 19, 84, 97, 127, 85, 118, 39, 103, 47, 34, 21, 120, 93, 90, 24, 112, 83, 74, 100, 111, 10, 108, 18, 26, 53, 82, 113, 98, 35, 33, 25, 30, 36, 106, 91, 89, 15, 7, 42, 11, 78, 71, 102, 14, 79, 75, 17, 64, 32, 81, 0, 96, 5, 38, 12, 8, 76, 94, 69, 3, 67, 27, 6, 4, 73, 9, 68, 1, 2, 66, 65, 70, 72], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 29, 126, 54, 117, 41, 101, 115, 114, 28, 121, 99, 50, 22, 119, 23, 116, 31, 86, 40, 124, 95, 92, 87, 51, 62, 16, 49, 103, 80, 88, 112, 39, 127, 104, 53, 113, 13, 44, 48, 60, 58, 97, 20, 19, 122, 90, 77, 21, 93, 85, 24, 100, 84, 118, 120, 34, 83, 82, 42, 74, 108, 91, 47, 18, 33, 26, 10, 36, 25, 106, 35, 111, 30, 89, 98, 7, 14, 78, 79, 17, 0, 32, 75, 81, 64, 15, 12, 71, 8, 27, 94, 102, 5, 38, 11, 3, 96, 67, 69, 66, 76, 65, 2, 4, 73, 9, 1, 68, 70, 6, 72], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 52, 29, 54, 41, 117, 126, 101, 50, 115, 58, 99, 124, 114, 28, 40, 121, 44, 23, 119, 92, 22, 62, 116, 95, 31, 16, 127, 49, 13, 104, 97, 87, 122, 60, 86, 53, 112, 77, 103, 88, 80, 113, 85, 51, 48, 93, 21, 118, 34, 20, 24, 39, 84, 111, 90, 47, 19, 82, 100, 120, 74, 36, 108, 83, 33, 18, 91, 10, 26, 25, 106, 35, 98, 89, 30, 42, 38, 15, 71, 14, 32, 11, 78, 79, 96, 7, 27, 8, 17, 75, 94, 64, 102, 12, 0, 76, 81, 67, 5, 3, 69, 70, 9, 4, 1, 66, 68, 73, 2, 65, 72, 6], [59, 43, 57, 123, 109, 46, 110, 107, 63, 61, 55, 125, 56, 45, 105, 37, 54, 52, 126, 29, 117, 115, 50, 41, 40, 101, 62, 28, 114, 112, 99, 121, 119, 58, 124, 53, 116, 44, 23, 95, 103, 51, 86, 92, 22, 127, 104, 60, 31, 13, 16, 48, 80, 87, 49, 77, 122, 88, 21, 39, 118, 108, 113, 120, 111, 93, 20, 97, 24, 84, 19, 85, 90, 18, 34, 74, 47, 10, 100, 83, 33, 36, 82, 7, 71, 35, 98, 0, 91, 42, 25, 75, 11, 26, 8, 64, 79, 30, 96, 89, 14, 15, 106, 17, 5, 67, 76, 3, 78, 81, 38, 70, 102, 69, 32, 12, 94, 1, 9, 65, 2, 4, 73, 27, 66, 68, 72, 6], [59, 43, 57, 123, 109, 110, 46, 107, 61, 63, 56, 55, 125, 45, 105, 52, 37, 126, 54, 29, 115, 41, 121, 117, 101, 99, 50, 114, 119, 80, 28, 53, 40, 127, 16, 51, 60, 22, 13, 62, 23, 124, 58, 44, 87, 116, 86, 95, 112, 77, 104, 49, 31, 88, 103, 20, 111, 92, 113, 39, 74, 19, 84, 21, 97, 118, 122, 10, 90, 24, 85, 120, 83, 93, 26, 33, 108, 100, 7, 18, 47, 34, 48, 98, 82, 91, 35, 42, 71, 36, 25, 0, 11, 64, 81, 79, 15, 67, 75, 14, 12, 106, 3, 38, 89, 76, 78, 96, 8, 17, 30, 69, 102, 5, 32, 72, 73, 4, 1, 70, 2, 94, 27, 65, 68, 66, 6, 9], [43, 59, 57, 123, 109, 107, 46, 110, 63, 61, 55, 125, 56, 45, 105, 52, 37, 126, 29, 54, 41, 114, 101, 117, 22, 121, 40, 115, 99, 28, 86, 80, 23, 50, 119, 13, 104, 95, 31, 16, 124, 127, 51, 92, 87, 44, 58, 116, 77, 88, 97, 39, 60, 20, 62, 93, 53, 103, 84, 118, 74, 112, 113, 21, 85, 111, 48, 120, 90, 19, 49, 10, 33, 82, 83, 122, 34, 24, 18, 25, 100, 108, 26, 35, 91, 47, 89, 71, 98, 79, 7, 15, 30, 96, 36, 106, 11, 12, 78, 0, 102, 14, 64, 75, 17, 81, 42, 32, 69, 67, 27, 73, 38, 76, 3, 8, 65, 94, 5, 2, 4, 1, 72, 6, 9, 70, 68, 66], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 54, 29, 126, 121, 114, 50, 101, 41, 117, 115, 99, 22, 95, 119, 40, 28, 86, 87, 116, 80, 58, 118, 92, 124, 62, 103, 104, 31, 88, 23, 97, 127, 93, 122, 51, 16, 44, 39, 112, 49, 113, 77, 60, 33, 84, 48, 34, 85, 19, 53, 13, 108, 111, 120, 90, 24, 21, 10, 20, 83, 26, 82, 18, 36, 30, 74, 89, 98, 91, 100, 25, 35, 47, 42, 106, 96, 7, 64, 79, 15, 32, 75, 78, 71, 12, 81, 38, 94, 11, 69, 14, 102, 76, 17, 67, 5, 27, 72, 8, 0, 1, 6, 3, 2, 73, 4, 65, 66, 9, 68, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 125, 45, 56, 105, 37, 52, 126, 29, 54, 101, 41, 116, 50, 117, 114, 121, 124, 115, 119, 23, 40, 99, 62, 120, 51, 86, 13, 28, 103, 122, 95, 104, 87, 31, 16, 22, 80, 127, 92, 118, 44, 53, 49, 113, 39, 111, 58, 88, 19, 60, 112, 77, 24, 97, 84, 10, 85, 21, 20, 90, 48, 93, 34, 108, 106, 100, 47, 18, 83, 82, 74, 26, 33, 98, 42, 0, 30, 35, 36, 7, 89, 71, 91, 64, 25, 72, 15, 81, 79, 75, 38, 69, 12, 67, 3, 11, 32, 14, 94, 96, 5, 78, 17, 76, 102, 65, 8, 4, 6, 1, 2, 73, 27, 9, 70, 66, 68], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 41, 114, 101, 117, 40, 50, 99, 115, 23, 124, 31, 60, 44, 86, 121, 58, 22, 119, 95, 62, 16, 13, 28, 104, 116, 87, 80, 39, 88, 112, 103, 53, 113, 127, 77, 92, 51, 49, 48, 118, 93, 85, 120, 111, 84, 74, 97, 20, 19, 24, 21, 10, 90, 122, 82, 100, 47, 34, 83, 18, 33, 106, 108, 91, 26, 71, 0, 35, 42, 98, 7, 64, 75, 25, 89, 79, 36, 30, 38, 81, 15, 5, 78, 72, 3, 11, 102, 67, 76, 96, 17, 14, 32, 94, 73, 69, 6, 4, 68, 1, 66, 8, 65, 12, 2, 27, 9, 70], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 52, 37, 126, 54, 101, 29, 117, 50, 41, 115, 114, 62, 44, 80, 99, 23, 28, 95, 22, 86, 121, 40, 124, 116, 16, 31, 92, 60, 13, 119, 87, 127, 112, 51, 104, 113, 58, 88, 103, 111, 118, 97, 77, 84, 53, 39, 48, 85, 19, 20, 10, 74, 49, 120, 122, 21, 108, 90, 93, 100, 24, 26, 34, 47, 18, 83, 33, 82, 106, 89, 25, 98, 7, 35, 71, 30, 42, 79, 0, 96, 91, 36, 15, 75, 78, 17, 12, 102, 76, 11, 64, 14, 27, 81, 72, 67, 5, 38, 3, 94, 1, 32, 69, 65, 73, 66, 9, 68, 70, 6, 2, 4, 8], [59, 43, 57, 123, 107, 109, 110, 46, 61, 63, 125, 55, 56, 45, 105, 52, 37, 54, 126, 115, 101, 29, 40, 114, 50, 28, 117, 121, 41, 62, 95, 116, 31, 99, 92, 22, 127, 80, 23, 124, 86, 39, 87, 16, 58, 60, 13, 49, 44, 51, 88, 119, 122, 112, 53, 97, 84, 118, 19, 120, 85, 104, 93, 103, 24, 20, 21, 48, 108, 34, 90, 33, 82, 77, 111, 100, 26, 74, 83, 113, 35, 98, 18, 36, 89, 47, 25, 10, 42, 91, 30, 106, 71, 96, 7, 79, 64, 0, 81, 78, 15, 32, 11, 75, 102, 72, 12, 38, 76, 17, 27, 3, 14, 94, 67, 69, 5, 73, 65, 6, 70, 1, 9, 68, 4, 66, 8, 2], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 37, 52, 126, 41, 29, 54, 50, 115, 101, 117, 40, 114, 80, 22, 28, 49, 121, 23, 104, 31, 99, 95, 86, 92, 103, 119, 60, 124, 116, 58, 13, 87, 62, 16, 51, 39, 44, 127, 77, 88, 93, 97, 113, 24, 120, 122, 21, 118, 20, 19, 112, 84, 48, 74, 53, 34, 85, 111, 18, 108, 26, 90, 100, 82, 47, 33, 10, 83, 42, 89, 35, 98, 25, 71, 91, 79, 0, 30, 106, 81, 36, 72, 7, 17, 96, 64, 11, 15, 78, 5, 67, 14, 69, 94, 32, 102, 75, 3, 12, 76, 65, 38, 66, 68, 27, 73, 70, 9, 4, 1, 6, 2, 8], [59, 43, 57, 123, 109, 110, 46, 107, 63, 61, 55, 56, 125, 45, 105, 37, 126, 52, 54, 117, 50, 29, 114, 41, 40, 44, 101, 115, 121, 62, 124, 104, 58, 28, 60, 99, 80, 119, 116, 103, 113, 23, 53, 95, 118, 22, 51, 31, 122, 92, 13, 86, 120, 49, 39, 112, 16, 87, 88, 77, 111, 97, 100, 24, 74, 93, 108, 84, 21, 20, 127, 48, 85, 47, 34, 19, 90, 82, 10, 18, 106, 33, 83, 35, 98, 42, 26, 71, 36, 7, 30, 0, 91, 89, 102, 64, 79, 11, 81, 38, 25, 75, 3, 72, 32, 94, 96, 78, 76, 15, 12, 17, 69, 67, 27, 5, 14, 70, 73, 65, 66, 9, 8, 4, 1, 68, 2, 6], [43, 59, 57, 123, 109, 107, 110, 46, 63, 61, 56, 55, 125, 45, 105, 37, 52, 126, 54, 29, 117, 50, 41, 121, 101, 62, 40, 114, 115, 44, 23, 99, 95, 104, 28, 16, 58, 116, 124, 31, 60, 103, 13, 51, 49, 92, 22, 80, 39, 122, 87, 127, 86, 119, 88, 93, 118, 77, 120, 112, 113, 97, 85, 111, 108, 47, 24, 10, 53, 20, 84, 74, 21, 34, 100, 90, 19, 33, 18, 83, 98, 48, 89, 26, 82, 91, 42, 35, 30, 11, 71, 7, 64, 36, 75, 106, 32, 3, 25, 12, 79, 69, 15, 0, 102, 96, 94, 17, 67, 76, 78, 38, 72, 14, 8, 5, 81, 27, 70, 65, 68, 66, 9, 73, 1, 2, 4, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 126, 54, 37, 52, 29, 41, 121, 40, 101, 50, 117, 115, 99, 116, 114, 95, 127, 28, 44, 119, 62, 104, 49, 86, 122, 22, 124, 39, 80, 23, 53, 118, 16, 31, 113, 120, 13, 103, 87, 60, 92, 112, 51, 93, 88, 58, 111, 77, 84, 48, 34, 97, 20, 74, 85, 33, 21, 19, 47, 98, 90, 10, 24, 108, 83, 18, 26, 100, 35, 91, 25, 89, 71, 82, 36, 7, 42, 15, 11, 79, 30, 75, 17, 106, 96, 76, 81, 102, 78, 0, 72, 14, 64, 32, 69, 67, 3, 94, 38, 12, 8, 5, 70, 1, 68, 73, 4, 65, 66, 2, 27, 9, 6], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 56, 125, 55, 45, 105, 126, 37, 52, 54, 29, 117, 41, 50, 101, 99, 40, 121, 115, 114, 116, 49, 104, 28, 44, 113, 119, 122, 124, 62, 23, 80, 103, 92, 22, 58, 16, 95, 87, 60, 51, 86, 39, 112, 118, 31, 88, 53, 48, 127, 120, 97, 13, 24, 77, 34, 20, 84, 111, 93, 21, 74, 85, 108, 90, 47, 19, 33, 83, 100, 26, 82, 64, 10, 98, 18, 71, 36, 106, 35, 0, 30, 42, 91, 89, 25, 7, 15, 11, 67, 17, 75, 8, 81, 79, 78, 3, 14, 38, 5, 102, 69, 32, 12, 94, 76, 72, 65, 1, 96, 70, 68, 27, 4, 66, 6, 2, 73, 9], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 55, 56, 125, 45, 105, 37, 126, 54, 52, 117, 29, 50, 101, 41, 115, 40, 99, 114, 28, 119, 116, 121, 104, 124, 23, 62, 53, 95, 22, 92, 127, 51, 58, 49, 86, 103, 80, 122, 112, 31, 118, 16, 44, 39, 97, 113, 13, 77, 93, 20, 87, 88, 21, 90, 111, 24, 60, 120, 19, 85, 48, 34, 83, 84, 47, 91, 30, 108, 74, 33, 100, 26, 18, 35, 10, 82, 42, 98, 25, 89, 36, 7, 15, 71, 102, 11, 106, 32, 8, 0, 75, 79, 81, 94, 27, 14, 12, 17, 67, 64, 3, 69, 96, 38, 78, 5, 66, 65, 76, 68, 73, 72, 1, 70, 6, 9, 2, 4], [59, 43, 57, 123, 107, 109, 110, 46, 63, 61, 125, 56, 55, 45, 105, 37, 52, 29, 54, 126, 41, 114, 121, 116, 40, 50, 117, 28, 62, 101, 115, 95, 104, 99, 44, 23, 119, 80, 22, 103, 16, 60, 92, 118, 13, 124, 58, 39, 86, 88, 77, 111, 31, 87, 112, 127, 20, 93, 51, 53, 49, 21, 84, 122, 85, 113, 83, 10, 97, 74, 90, 19, 24, 33, 120, 82, 25, 98, 26, 18, 47, 34, 108, 48, 7, 100, 35, 36, 89, 91, 71, 79, 8, 75, 11, 106, 30, 12, 76, 32, 42, 15, 0, 78, 102, 14, 69, 38, 17, 96, 81, 3, 73, 67, 64, 5, 94, 6, 9, 65, 27, 70, 66, 4, 72, 68, 2, 1], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 125, 55, 56, 45, 105, 37, 52, 126, 29, 41, 54, 114, 117, 40, 50, 99, 28, 127, 101, 121, 23, 22, 95, 86, 115, 16, 80, 31, 60, 111, 62, 116, 44, 104, 58, 77, 53, 92, 88, 49, 113, 13, 103, 124, 119, 87, 39, 21, 51, 20, 118, 19, 97, 122, 84, 112, 93, 18, 47, 10, 74, 90, 108, 24, 34, 48, 85, 83, 26, 120, 82, 100, 35, 98, 25, 33, 7, 30, 8, 89, 36, 71, 11, 106, 14, 42, 91, 79, 15, 102, 75, 76, 0, 78, 32, 96, 17, 81, 64, 12, 94, 69, 67, 3, 6, 38, 5, 73, 65, 68, 9, 70, 1, 4, 27, 66, 2, 72], [59, 43, 57, 123, 107, 109, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 54, 126, 52, 50, 29, 117, 114, 101, 41, 121, 115, 28, 99, 40, 31, 112, 49, 116, 124, 62, 92, 22, 23, 95, 127, 113, 58, 86, 44, 88, 97, 60, 39, 87, 103, 104, 118, 122, 80, 16, 111, 21, 13, 53, 77, 24, 84, 119, 51, 85, 34, 20, 120, 100, 19, 48, 90, 93, 26, 47, 74, 83, 42, 33, 106, 108, 36, 18, 30, 89, 10, 98, 82, 35, 7, 25, 91, 11, 0, 64, 96, 15, 75, 32, 102, 79, 78, 8, 14, 71, 94, 81, 27, 76, 69, 12, 38, 3, 17, 67, 1, 5, 68, 73, 65, 6, 66, 2, 4, 9, 70, 72], [59, 43, 57, 123, 109, 107, 46, 110, 61, 63, 55, 125, 56, 45, 105, 37, 52, 126, 54, 29, 101, 50, 114, 41, 121, 40, 115, 117, 99, 62, 124, 28, 53, 127, 23, 95, 116, 58, 119, 31, 92, 80, 49, 44, 104, 118, 22, 86, 39, 16, 13, 112, 51, 111, 122, 88, 103, 60, 87, 97, 48, 77, 120, 20, 19, 34, 21, 113, 24, 85, 93, 84, 26, 47, 100, 74, 90, 10, 108, 33, 98, 83, 35, 106, 18, 82, 36, 42, 91, 7, 25, 30, 89, 38, 102, 79, 96, 8, 71, 81, 12, 14, 11, 78, 15, 94, 17, 75, 5, 76, 64, 32, 27, 0, 3, 69, 67, 4, 6, 72, 73, 65, 9, 68, 66, 2, 1, 70], [59, 43, 57, 123, 109, 107, 110, 46, 63, 61, 125, 55, 56, 45, 105, 37, 52, 126, 29, 54, 114, 121, 41, 50, 101, 117, 99, 40, 116, 28, 115, 124, 119, 80, 118, 104, 44, 95, 86, 49, 16, 23, 103, 13, 127, 62, 92, 22, 77, 58, 31, 51, 87, 112, 60, 39, 21, 88, 53, 113, 24, 111, 85, 93, 48, 84, 19, 10, 20, 122, 34, 74, 83, 120, 108, 97, 90, 18, 82, 26, 7, 106, 47, 33, 36, 100, 98, 42, 35, 71, 25, 0, 89, 64, 11, 79, 30, 69, 75, 3, 91, 78, 102, 14, 17, 12, 8, 67, 32, 15, 76, 96, 81, 5, 38, 94, 65, 6, 70, 72, 27, 68, 73, 4, 9, 66, 1, 2], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 56, 125, 55, 45, 105, 52, 37, 54, 126, 29, 41, 50, 117, 53, 114, 101, 121, 40, 99, 60, 44, 104, 28, 95, 116, 58, 127, 124, 115, 80, 119, 86, 23, 111, 22, 92, 16, 112, 118, 103, 49, 77, 31, 88, 87, 39, 13, 62, 113, 20, 21, 10, 48, 93, 74, 122, 24, 51, 84, 97, 108, 33, 34, 19, 64, 106, 90, 83, 35, 26, 7, 85, 82, 120, 47, 36, 18, 98, 71, 100, 91, 11, 79, 0, 75, 42, 96, 102, 30, 15, 76, 78, 69, 17, 25, 89, 3, 67, 12, 32, 38, 14, 72, 65, 66, 81, 8, 5, 70, 4, 68, 94, 9, 1, 6, 73, 2, 27], [43, 59, 57, 123, 107, 109, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 54, 126, 29, 50, 41, 101, 114, 99, 116, 117, 115, 40, 112, 127, 28, 22, 23, 95, 60, 119, 16, 124, 104, 80, 86, 13, 118, 88, 92, 58, 44, 121, 103, 77, 62, 53, 31, 87, 49, 97, 113, 21, 85, 39, 93, 111, 20, 84, 10, 83, 19, 51, 122, 34, 90, 108, 24, 26, 48, 47, 120, 74, 33, 100, 106, 82, 18, 36, 98, 7, 91, 75, 25, 71, 35, 30, 15, 78, 64, 32, 17, 89, 12, 42, 14, 96, 81, 79, 11, 0, 102, 38, 3, 72, 5, 69, 76, 70, 67, 9, 65, 4, 8, 68, 2, 94, 66, 6, 73, 1, 27], [59, 43, 57, 123, 109, 110, 107, 46, 61, 63, 55, 125, 56, 45, 105, 52, 37, 126, 54, 29, 101, 50, 41, 28, 117, 115, 40, 116, 121, 99, 114, 112, 92, 104, 119, 124, 62, 127, 23, 16, 95, 58, 103, 80, 22, 31, 86, 39, 13, 88, 60, 53, 44, 113, 87, 118, 97, 111, 77, 34, 47, 85, 49, 21, 24, 122, 84, 108, 120, 20, 51, 48, 19, 93, 74, 90, 10, 33, 83, 18, 82, 100, 26, 0, 98, 7, 106, 35, 91, 71, 36, 64, 89, 11, 25, 42, 79, 67, 75, 102, 72, 96, 30, 15, 69, 14, 32, 12, 3, 5, 76, 68, 38, 81, 17, 70, 78, 94, 9, 2, 73, 4, 65, 8, 6, 27, 1, 66], [59, 43, 57, 123, 109, 107, 110, 46, 61, 63, 55, 56, 125, 45, 105, 52, 37, 126, 54, 29, 114, 41, 101, 50, 40, 115, 116, 28, 99, 117, 112, 95, 121, 127, 22, 23, 80, 86, 119, 92, 62, 103, 16, 87, 31, 39, 53, 13, 124, 60, 118, 44, 58, 88, 104, 113, 77, 51, 49, 21, 97, 19, 48, 93, 84, 111, 24, 10, 34, 33, 122, 85, 47, 20, 120, 74, 90, 82, 108, 83, 18, 26, 35, 36, 91, 100, 7, 75, 30, 25, 98, 14, 72, 106, 71, 79, 42, 89, 11, 76, 102, 96, 32, 17, 12, 64, 78, 15, 94, 5, 81, 27, 38, 73, 69, 0, 67, 9, 3, 70, 2, 68, 8, 66, 4, 1, 65, 6]], "model.layers.27.self_attn.q_proj": [[109, 45, 94, 90, 33, 23, 83, 81, 21, 60, 54, 117, 79, 125, 76, 28, 78, 123, 111, 112, 39, 58, 73, 62, 56, 61, 51, 5, 32, 59, 38, 114, 57, 115, 6, 52, 48, 105, 11, 35, 97, 43, 55, 100, 9, 122, 113, 106, 46, 7, 37, 53, 121, 10, 116, 98, 118, 63, 24, 120, 3, 126, 19, 0, 110, 49, 87, 17, 44, 22, 85, 50, 119, 75, 104, 124, 47, 103, 42, 127, 95, 101, 31, 29, 88, 108, 92, 25, 13, 4, 26, 14, 36, 30, 40, 80, 18, 107, 41, 20, 84, 86, 27, 93, 102, 99, 96, 15, 77, 89, 91, 82, 34, 74, 16, 66, 70, 72, 65, 8, 12, 64, 1, 2, 68, 69, 71, 67], [109, 45, 33, 94, 90, 21, 83, 23, 81, 79, 76, 105, 54, 28, 112, 73, 97, 123, 99, 7, 117, 75, 121, 6, 5, 127, 60, 39, 4, 38, 11, 9, 47, 125, 78, 106, 3, 0, 37, 48, 32, 56, 115, 111, 62, 63, 26, 71, 14, 74, 2, 46, 120, 18, 58, 119, 35, 113, 114, 17, 51, 110, 87, 16, 19, 49, 85, 1, 66, 107, 36, 70, 77, 59, 98, 126, 53, 89, 57, 44, 24, 50, 31, 124, 43, 22, 102, 122, 88, 55, 52, 42, 103, 108, 10, 92, 118, 82, 86, 104, 96, 80, 34, 30, 65, 116, 20, 84, 95, 41, 13, 29, 8, 61, 101, 25, 91, 72, 40, 100, 67, 69, 93, 27, 64, 15, 12, 68], [109, 45, 94, 90, 33, 83, 23, 21, 76, 79, 81, 123, 125, 28, 54, 39, 105, 99, 6, 73, 112, 111, 106, 5, 127, 60, 62, 58, 56, 9, 78, 38, 70, 32, 50, 3, 115, 44, 42, 72, 75, 66, 37, 40, 17, 48, 7, 97, 11, 63, 124, 52, 18, 84, 89, 51, 120, 10, 113, 101, 26, 87, 25, 98, 47, 41, 88, 80, 57, 59, 14, 49, 85, 0, 108, 19, 117, 126, 24, 31, 30, 95, 77, 116, 104, 103, 119, 107, 12, 74, 121, 110, 22, 1, 35, 102, 13, 4, 2, 55, 46, 34, 53, 118, 92, 96, 43, 91, 61, 114, 29, 122, 8, 100, 86, 16, 36, 82, 20, 93, 65, 15, 27, 71, 69, 67, 68, 64], [109, 45, 94, 90, 33, 125, 83, 23, 117, 81, 21, 76, 115, 32, 124, 57, 119, 113, 112, 79, 54, 121, 114, 9, 28, 123, 58, 39, 47, 51, 52, 60, 91, 118, 122, 53, 59, 29, 48, 49, 106, 61, 24, 14, 30, 101, 13, 110, 37, 38, 43, 100, 126, 116, 44, 17, 85, 80, 87, 50, 127, 5, 108, 95, 92, 88, 102, 103, 25, 63, 111, 99, 73, 96, 97, 55, 19, 107, 56, 40, 120, 89, 41, 75, 46, 6, 36, 84, 78, 22, 62, 42, 82, 20, 18, 35, 31, 16, 27, 26, 104, 98, 105, 93, 34, 86, 15, 12, 77, 65, 10, 8, 7, 74, 72, 11, 71, 0, 1, 68, 3, 69, 70, 4, 67, 66, 2, 64], [118, 53, 120, 63, 101, 60, 127, 121, 84, 57, 126, 119, 88, 56, 50, 112, 61, 124, 92, 52, 17, 54, 125, 117, 123, 24, 115, 113, 33, 55, 116, 49, 58, 59, 62, 51, 94, 110, 122, 93, 48, 47, 111, 45, 114, 43, 30, 37, 108, 39, 46, 44, 7, 14, 107, 90, 41, 109, 11, 104, 26, 9, 25, 74, 78, 20, 91, 77, 42, 103, 106, 5, 105, 29, 31, 75, 40, 22, 32, 80, 38, 36, 100, 21, 18, 6, 13, 72, 95, 99, 96, 73, 79, 4, 34, 16, 86, 35, 98, 83, 12, 3, 19, 81, 66, 10, 102, 85, 28, 97, 69, 23, 27, 87, 2, 89, 64, 1, 65, 70, 68, 71, 82, 0, 15, 67, 8, 76], [53, 120, 118, 101, 63, 60, 127, 50, 56, 57, 61, 124, 126, 84, 123, 121, 112, 119, 54, 115, 62, 52, 49, 117, 88, 93, 113, 116, 58, 110, 125, 55, 51, 122, 59, 108, 39, 111, 48, 47, 92, 114, 33, 45, 17, 46, 107, 37, 44, 14, 24, 77, 30, 90, 7, 109, 41, 94, 104, 26, 43, 105, 20, 103, 74, 106, 91, 9, 42, 83, 4, 19, 100, 25, 96, 3, 5, 11, 22, 29, 23, 40, 75, 38, 34, 80, 12, 6, 16, 73, 21, 32, 18, 36, 78, 98, 102, 95, 31, 89, 99, 86, 35, 27, 85, 72, 97, 79, 28, 81, 10, 87, 64, 65, 13, 70, 66, 1, 2, 71, 15, 82, 69, 76, 68, 8, 0, 67], [120, 118, 53, 101, 63, 60, 126, 121, 127, 56, 57, 84, 50, 119, 61, 124, 54, 112, 52, 62, 123, 115, 117, 125, 58, 116, 55, 110, 88, 93, 39, 113, 122, 49, 59, 47, 17, 51, 48, 111, 92, 108, 114, 33, 46, 14, 45, 43, 94, 109, 37, 90, 30, 11, 26, 41, 7, 25, 44, 107, 104, 24, 106, 9, 103, 42, 77, 22, 20, 100, 105, 74, 29, 91, 40, 78, 6, 34, 80, 5, 38, 31, 102, 18, 98, 95, 36, 32, 96, 73, 3, 85, 83, 16, 4, 86, 12, 79, 10, 28, 19, 75, 99, 27, 23, 21, 35, 13, 97, 89, 72, 81, 64, 66, 65, 1, 87, 70, 15, 71, 82, 67, 69, 0, 76, 68, 2, 8], [63, 53, 120, 118, 101, 60, 57, 50, 84, 127, 62, 61, 54, 112, 56, 124, 121, 88, 123, 115, 119, 117, 52, 49, 55, 126, 116, 39, 113, 58, 110, 122, 30, 59, 51, 125, 93, 111, 44, 48, 108, 114, 47, 14, 24, 45, 90, 43, 17, 46, 94, 77, 37, 33, 109, 92, 103, 74, 107, 11, 5, 7, 104, 73, 106, 98, 26, 9, 91, 41, 42, 105, 100, 36, 6, 83, 3, 18, 4, 40, 80, 32, 22, 38, 102, 85, 28, 25, 96, 20, 27, 75, 35, 34, 31, 95, 21, 23, 16, 78, 19, 99, 79, 72, 97, 86, 89, 10, 66, 29, 81, 12, 87, 13, 82, 65, 68, 15, 0, 64, 71, 2, 70, 1, 76, 69, 67, 8], [40, 98, 63, 23, 31, 85, 80, 26, 121, 60, 13, 19, 125, 54, 82, 122, 79, 8, 49, 74, 6, 1, 117, 55, 113, 105, 59, 57, 9, 12, 106, 127, 111, 52, 46, 66, 50, 119, 56, 58, 53, 112, 43, 103, 11, 27, 120, 108, 39, 126, 4, 124, 37, 24, 62, 123, 109, 28, 115, 38, 67, 100, 107, 64, 0, 90, 51, 15, 73, 75, 47, 61, 76, 42, 116, 97, 78, 21, 99, 35, 18, 30, 25, 104, 36, 118, 41, 87, 48, 88, 44, 102, 32, 93, 45, 94, 84, 101, 114, 3, 95, 33, 68, 5, 83, 110, 34, 96, 77, 29, 20, 89, 16, 91, 86, 69, 92, 7, 14, 81, 17, 72, 70, 22, 65, 71, 10, 2], [40, 63, 98, 31, 80, 23, 60, 85, 13, 74, 6, 4, 19, 8, 22, 64, 52, 66, 121, 46, 108, 28, 124, 56, 122, 106, 0, 79, 125, 104, 54, 65, 11, 55, 107, 113, 75, 59, 18, 90, 48, 82, 7, 1, 58, 73, 126, 119, 12, 68, 117, 84, 127, 83, 105, 34, 57, 49, 21, 120, 35, 77, 109, 20, 101, 111, 94, 47, 14, 15, 39, 81, 30, 97, 24, 100, 96, 2, 3, 16, 62, 114, 10, 53, 33, 32, 70, 9, 25, 112, 95, 17, 123, 116, 38, 89, 5, 91, 87, 88, 72, 86, 76, 51, 71, 78, 29, 67, 69, 43, 27, 36, 103, 44, 115, 50, 93, 41, 110, 102, 99, 26, 92, 42, 45, 37, 61, 118], [40, 63, 98, 31, 60, 8, 80, 85, 19, 13, 23, 6, 74, 1, 66, 121, 64, 122, 117, 124, 4, 26, 46, 106, 15, 52, 28, 54, 57, 109, 55, 125, 0, 89, 104, 87, 12, 24, 120, 56, 79, 119, 105, 68, 69, 108, 72, 27, 7, 113, 11, 71, 59, 126, 42, 21, 48, 18, 76, 83, 33, 78, 84, 103, 96, 75, 32, 16, 82, 65, 115, 20, 41, 90, 81, 30, 77, 50, 2, 62, 3, 51, 25, 116, 92, 123, 5, 114, 127, 67, 37, 93, 88, 61, 10, 47, 29, 118, 58, 36, 38, 107, 39, 94, 17, 70, 111, 97, 14, 49, 86, 110, 45, 91, 73, 100, 101, 35, 44, 53, 112, 99, 102, 22, 43, 9, 95, 34], [40, 63, 98, 23, 85, 60, 26, 80, 31, 13, 8, 74, 19, 66, 82, 6, 121, 59, 52, 106, 4, 58, 41, 122, 113, 46, 119, 28, 49, 48, 95, 57, 84, 5, 120, 111, 105, 125, 65, 38, 67, 79, 54, 55, 108, 109, 117, 2, 73, 102, 90, 7, 123, 47, 61, 76, 64, 0, 18, 107, 50, 83, 45, 69, 56, 53, 97, 32, 115, 77, 81, 87, 71, 124, 15, 42, 89, 103, 91, 11, 30, 44, 114, 126, 21, 118, 78, 100, 43, 24, 39, 96, 36, 62, 17, 68, 35, 127, 93, 20, 29, 51, 116, 16, 3, 22, 94, 27, 70, 75, 9, 112, 88, 33, 110, 92, 37, 10, 101, 25, 14, 12, 72, 86, 1, 99, 104, 34], [104, 120, 98, 46, 95, 126, 115, 52, 44, 91, 108, 59, 56, 60, 54, 27, 84, 58, 88, 96, 118, 82, 24, 122, 51, 36, 50, 21, 45, 42, 48, 112, 101, 57, 62, 124, 114, 55, 63, 113, 92, 119, 117, 116, 61, 76, 125, 86, 85, 47, 127, 23, 53, 123, 111, 121, 49, 43, 38, 13, 107, 110, 109, 22, 41, 106, 74, 99, 103, 105, 80, 89, 33, 97, 31, 29, 32, 19, 6, 78, 37, 102, 25, 39, 69, 94, 28, 100, 35, 64, 90, 18, 93, 8, 30, 15, 26, 83, 65, 87, 34, 14, 12, 0, 40, 2, 79, 20, 66, 81, 72, 7, 1, 17, 3, 73, 16, 67, 11, 4, 75, 10, 5, 68, 77, 71, 9, 70], [120, 104, 98, 95, 44, 108, 126, 46, 113, 125, 54, 27, 88, 91, 55, 58, 127, 122, 61, 59, 41, 60, 115, 50, 110, 106, 56, 84, 36, 118, 116, 109, 123, 22, 38, 103, 62, 124, 111, 57, 96, 63, 114, 49, 112, 43, 28, 121, 40, 117, 53, 48, 35, 42, 24, 52, 45, 51, 101, 107, 119, 37, 47, 39, 105, 82, 94, 19, 85, 86, 83, 97, 102, 21, 12, 79, 93, 73, 26, 25, 81, 71, 100, 99, 31, 29, 92, 9, 30, 4, 33, 32, 89, 34, 23, 76, 74, 78, 90, 80, 15, 20, 18, 87, 8, 14, 10, 17, 70, 69, 72, 66, 7, 6, 2, 68, 13, 75, 3, 67, 5, 16, 77, 11, 65, 0, 64, 1], [104, 120, 98, 95, 126, 91, 80, 84, 13, 82, 22, 86, 115, 27, 74, 44, 31, 93, 116, 36, 16, 97, 69, 88, 20, 77, 60, 100, 52, 59, 58, 90, 56, 6, 108, 125, 72, 94, 45, 2, 19, 122, 32, 25, 3, 62, 63, 18, 54, 114, 92, 96, 42, 64, 41, 43, 75, 15, 29, 113, 68, 55, 87, 30, 46, 106, 11, 118, 102, 33, 79, 48, 81, 76, 24, 99, 78, 103, 117, 70, 89, 83, 57, 85, 50, 65, 37, 8, 23, 21, 35, 26, 67, 17, 61, 73, 47, 107, 9, 4, 7, 1, 111, 38, 49, 71, 10, 12, 28, 14, 110, 53, 5, 127, 0, 34, 39, 66, 105, 101, 124, 121, 119, 112, 109, 123, 51, 40], [104, 120, 98, 46, 95, 82, 84, 96, 91, 80, 115, 14, 126, 44, 66, 48, 125, 27, 13, 67, 60, 29, 42, 116, 68, 74, 93, 8, 28, 85, 63, 102, 113, 36, 0, 23, 55, 117, 101, 105, 38, 6, 76, 65, 1, 21, 35, 24, 111, 54, 39, 45, 56, 99, 88, 26, 32, 119, 103, 86, 37, 50, 17, 108, 107, 64, 9, 97, 109, 52, 79, 40, 90, 16, 22, 31, 100, 92, 124, 41, 94, 122, 25, 58, 61, 62, 59, 47, 121, 72, 57, 112, 34, 123, 81, 15, 43, 110, 5, 89, 70, 20, 19, 106, 30, 114, 127, 83, 51, 118, 49, 53, 12, 87, 33, 11, 73, 78, 18, 69, 75, 10, 77, 3, 4, 7, 71, 2], [111, 100, 47, 56, 24, 53, 95, 121, 120, 58, 31, 54, 77, 36, 82, 103, 86, 57, 1, 124, 59, 119, 60, 84, 127, 16, 52, 83, 104, 97, 94, 10, 63, 45, 105, 49, 64, 116, 110, 28, 122, 62, 51, 126, 108, 92, 42, 112, 67, 113, 39, 27, 55, 125, 43, 102, 61, 46, 19, 118, 26, 40, 50, 8, 68, 123, 4, 38, 44, 114, 85, 71, 2, 109, 115, 22, 73, 6, 15, 41, 117, 99, 48, 66, 69, 106, 98, 37, 107, 101, 20, 35, 90, 0, 96, 81, 70, 30, 32, 89, 33, 17, 80, 78, 9, 88, 65, 23, 29, 91, 34, 87, 93, 5, 25, 11, 12, 18, 79, 13, 72, 21, 3, 75, 7, 74, 14, 76], [111, 47, 58, 100, 24, 31, 122, 95, 61, 106, 51, 125, 82, 54, 84, 28, 113, 26, 85, 56, 53, 20, 126, 110, 117, 112, 30, 77, 127, 50, 13, 46, 118, 98, 83, 36, 104, 22, 55, 6, 12, 79, 120, 39, 45, 37, 49, 15, 123, 86, 116, 41, 94, 52, 119, 78, 59, 124, 14, 121, 17, 57, 114, 92, 43, 108, 105, 60, 109, 101, 63, 25, 107, 99, 88, 23, 103, 115, 44, 68, 96, 40, 27, 90, 80, 35, 62, 0, 29, 97, 32, 73, 48, 74, 33, 89, 87, 38, 102, 3, 42, 21, 81, 10, 34, 72, 66, 76, 18, 16, 93, 19, 91, 11, 65, 7, 8, 69, 75, 9, 4, 2, 5, 71, 67, 70, 64, 1], [111, 47, 100, 58, 56, 24, 95, 31, 114, 115, 52, 94, 106, 83, 121, 36, 82, 119, 53, 86, 54, 120, 112, 110, 28, 85, 113, 55, 103, 51, 49, 92, 104, 59, 109, 118, 17, 84, 126, 63, 127, 20, 48, 50, 60, 62, 45, 117, 30, 122, 116, 105, 38, 57, 124, 61, 77, 96, 46, 125, 16, 107, 43, 41, 123, 39, 108, 88, 42, 102, 22, 80, 90, 26, 40, 79, 6, 78, 44, 98, 101, 10, 89, 37, 97, 35, 12, 81, 68, 99, 34, 15, 11, 33, 0, 19, 23, 13, 29, 73, 9, 21, 32, 27, 91, 93, 71, 72, 25, 18, 74, 87, 66, 3, 14, 8, 76, 7, 4, 75, 64, 67, 5, 69, 65, 70, 1, 2], [111, 47, 58, 100, 56, 24, 95, 31, 127, 121, 122, 106, 115, 36, 94, 120, 44, 62, 82, 28, 83, 108, 116, 126, 59, 53, 52, 112, 85, 77, 54, 92, 118, 84, 105, 51, 113, 104, 60, 49, 17, 107, 110, 124, 45, 109, 20, 63, 57, 61, 103, 86, 55, 43, 114, 46, 48, 123, 125, 78, 6, 80, 119, 102, 96, 42, 30, 117, 38, 26, 39, 37, 33, 16, 50, 22, 41, 90, 34, 19, 12, 10, 40, 79, 32, 15, 88, 101, 74, 98, 91, 27, 29, 93, 81, 89, 35, 99, 97, 66, 11, 23, 68, 13, 65, 21, 25, 87, 71, 72, 8, 7, 3, 18, 1, 73, 76, 14, 0, 9, 4, 75, 5, 70, 69, 2, 67, 64], [48, 41, 51, 62, 125, 55, 121, 112, 24, 88, 30, 57, 52, 54, 60, 53, 123, 114, 49, 119, 115, 118, 117, 113, 56, 124, 110, 126, 61, 97, 50, 127, 94, 116, 47, 44, 27, 120, 122, 106, 58, 90, 103, 82, 111, 91, 63, 59, 36, 109, 107, 108, 89, 45, 37, 43, 46, 79, 21, 42, 19, 22, 104, 28, 39, 101, 102, 105, 93, 15, 83, 38, 35, 80, 33, 40, 98, 32, 99, 34, 18, 73, 86, 100, 95, 76, 96, 71, 84, 13, 16, 29, 92, 77, 85, 87, 25, 9, 31, 20, 26, 65, 17, 23, 81, 7, 3, 66, 68, 5, 12, 6, 1, 64, 0, 2, 67, 4, 70, 14, 11, 69, 78, 74, 10, 75, 8, 72], [41, 48, 51, 119, 112, 62, 20, 89, 105, 14, 81, 115, 80, 11, 97, 10, 12, 30, 8, 125, 54, 27, 60, 5, 70, 78, 25, 121, 22, 49, 52, 84, 56, 117, 0, 28, 118, 72, 57, 87, 126, 61, 29, 127, 104, 17, 37, 110, 96, 124, 55, 93, 50, 94, 67, 3, 111, 88, 123, 114, 58, 16, 122, 23, 90, 53, 36, 45, 120, 2, 74, 100, 44, 92, 113, 76, 32, 46, 31, 116, 83, 24, 102, 43, 47, 19, 26, 75, 59, 108, 103, 107, 21, 106, 63, 109, 6, 71, 101, 95, 91, 98, 34, 42, 39, 77, 18, 35, 33, 99, 7, 68, 38, 40, 79, 86, 65, 69, 4, 15, 82, 85, 13, 9, 66, 1, 64, 73], [51, 48, 41, 119, 125, 24, 30, 121, 52, 62, 57, 124, 115, 113, 54, 88, 55, 60, 53, 123, 49, 117, 110, 27, 118, 127, 56, 126, 97, 61, 112, 50, 114, 103, 120, 94, 122, 58, 36, 91, 116, 44, 111, 47, 106, 101, 59, 82, 90, 107, 109, 46, 63, 108, 43, 45, 42, 37, 39, 22, 104, 102, 15, 35, 93, 77, 40, 79, 89, 21, 18, 33, 38, 19, 28, 105, 9, 83, 26, 32, 84, 17, 100, 99, 98, 86, 13, 76, 5, 95, 92, 71, 73, 12, 65, 87, 34, 31, 29, 16, 96, 70, 7, 3, 66, 80, 23, 4, 11, 85, 1, 25, 81, 68, 6, 0, 2, 20, 14, 10, 64, 67, 75, 69, 74, 78, 8, 72], [119, 48, 41, 51, 125, 24, 121, 57, 62, 52, 55, 60, 30, 97, 54, 88, 113, 123, 53, 115, 56, 124, 49, 110, 117, 126, 127, 118, 114, 50, 61, 112, 116, 94, 120, 103, 58, 91, 122, 27, 36, 111, 44, 82, 47, 63, 109, 101, 59, 90, 107, 106, 46, 108, 37, 45, 89, 104, 43, 22, 39, 19, 79, 42, 102, 99, 98, 73, 28, 105, 38, 80, 35, 21, 34, 18, 93, 83, 40, 96, 100, 32, 15, 33, 86, 71, 77, 95, 65, 85, 13, 31, 26, 92, 29, 76, 16, 3, 84, 9, 23, 17, 87, 68, 66, 5, 81, 6, 7, 4, 75, 64, 0, 1, 12, 70, 2, 67, 25, 20, 11, 69, 10, 74, 14, 78, 8, 72], [106, 35, 42, 110, 91, 85, 50, 89, 52, 96, 83, 54, 113, 31, 51, 56, 122, 123, 114, 55, 14, 17, 32, 126, 57, 46, 117, 59, 53, 120, 62, 60, 27, 107, 75, 22, 9, 58, 29, 87, 119, 49, 70, 125, 15, 115, 16, 112, 109, 21, 61, 13, 48, 44, 63, 127, 39, 28, 68, 86, 23, 84, 41, 69, 19, 71, 45, 33, 92, 97, 25, 116, 111, 5, 20, 26, 34, 81, 121, 37, 79, 36, 38, 40, 124, 47, 94, 90, 11, 76, 82, 80, 18, 93, 105, 88, 43, 100, 104, 30, 108, 118, 103, 77, 24, 98, 102, 99, 95, 101, 78, 73, 8, 12, 72, 74, 10, 7, 65, 3, 6, 4, 66, 2, 0, 67, 1, 64], [106, 35, 42, 85, 31, 17, 83, 96, 89, 50, 52, 55, 51, 14, 9, 27, 113, 56, 69, 75, 62, 57, 110, 53, 93, 91, 122, 6, 117, 3, 66, 1, 61, 65, 127, 64, 87, 123, 32, 126, 7, 15, 59, 68, 13, 70, 114, 25, 23, 76, 84, 120, 0, 21, 63, 5, 22, 74, 10, 109, 54, 119, 71, 29, 11, 19, 16, 82, 72, 20, 8, 18, 4, 67, 78, 81, 77, 118, 41, 37, 73, 79, 88, 90, 38, 30, 121, 86, 115, 12, 48, 94, 98, 45, 43, 44, 26, 24, 33, 80, 39, 108, 100, 105, 99, 47, 36, 92, 28, 125, 107, 111, 102, 34, 49, 124, 116, 46, 2, 97, 60, 101, 112, 103, 104, 95, 58, 40], [106, 35, 42, 110, 91, 85, 50, 122, 83, 126, 56, 57, 17, 89, 117, 96, 113, 14, 54, 55, 31, 32, 59, 52, 114, 62, 127, 63, 61, 53, 22, 9, 116, 112, 75, 51, 94, 29, 27, 123, 48, 37, 65, 45, 99, 47, 25, 76, 7, 23, 58, 3, 21, 107, 69, 124, 46, 33, 66, 19, 115, 64, 93, 109, 41, 71, 120, 39, 118, 36, 84, 70, 119, 79, 97, 13, 105, 60, 92, 100, 87, 28, 81, 11, 5, 98, 12, 88, 111, 6, 44, 125, 86, 104, 102, 43, 108, 68, 49, 121, 26, 18, 103, 80, 20, 24, 90, 38, 82, 16, 30, 4, 95, 15, 40, 78, 101, 8, 73, 34, 74, 10, 77, 72, 67, 1, 0, 2], [106, 35, 42, 110, 56, 91, 85, 52, 53, 83, 122, 96, 57, 17, 114, 50, 113, 51, 32, 62, 31, 120, 89, 14, 55, 59, 117, 29, 112, 27, 22, 54, 46, 126, 49, 119, 75, 45, 99, 109, 94, 87, 123, 124, 115, 116, 61, 9, 118, 21, 92, 11, 43, 86, 60, 25, 37, 47, 39, 125, 127, 28, 58, 38, 34, 19, 23, 121, 108, 44, 33, 84, 107, 103, 41, 97, 111, 63, 88, 48, 40, 100, 30, 16, 102, 93, 104, 26, 101, 105, 81, 24, 90, 82, 98, 36, 15, 78, 20, 76, 18, 79, 95, 71, 4, 12, 13, 80, 74, 77, 68, 70, 73, 7, 3, 10, 8, 65, 69, 6, 72, 66, 67, 64, 0, 5, 1, 2], [61, 122, 118, 102, 49, 54, 109, 45, 59, 50, 116, 90, 103, 119, 63, 125, 101, 112, 108, 94, 55, 127, 56, 57, 113, 62, 111, 37, 124, 60, 38, 22, 110, 44, 121, 114, 123, 115, 47, 40, 117, 41, 42, 51, 48, 46, 96, 43, 120, 58, 53, 104, 93, 52, 107, 105, 106, 126, 36, 100, 39, 92, 31, 23, 11, 34, 86, 99, 97, 98, 32, 79, 30, 35, 33, 85, 81, 4, 83, 20, 88, 14, 18, 2, 26, 66, 95, 25, 76, 16, 15, 91, 73, 28, 29, 13, 3, 5, 27, 72, 10, 6, 82, 21, 24, 68, 89, 78, 80, 0, 70, 8, 71, 19, 84, 65, 75, 17, 74, 12, 87, 69, 67, 7, 1, 77, 9, 64], [102, 118, 54, 61, 116, 77, 9, 29, 7, 122, 23, 1, 64, 81, 69, 20, 4, 68, 49, 3, 15, 67, 82, 65, 26, 74, 45, 33, 66, 70, 90, 11, 113, 53, 0, 22, 75, 86, 19, 5, 112, 83, 73, 124, 71, 25, 87, 10, 96, 72, 18, 107, 119, 14, 36, 85, 24, 30, 78, 89, 6, 79, 91, 80, 13, 108, 59, 12, 94, 31, 17, 88, 27, 125, 93, 8, 16, 21, 120, 2, 84, 99, 63, 32, 76, 95, 110, 28, 55, 92, 56, 127, 98, 100, 101, 35, 57, 109, 46, 50, 39, 97, 47, 37, 34, 126, 42, 40, 104, 103, 38, 43, 111, 44, 60, 117, 123, 106, 58, 41, 114, 115, 48, 105, 121, 62, 52, 51], [102, 118, 54, 116, 122, 61, 45, 90, 49, 23, 112, 53, 33, 93, 113, 127, 15, 81, 38, 43, 107, 114, 57, 31, 44, 96, 125, 101, 20, 50, 119, 30, 24, 106, 83, 28, 26, 56, 59, 29, 124, 48, 55, 21, 39, 77, 11, 100, 60, 117, 6, 76, 22, 42, 67, 14, 51, 52, 126, 82, 37, 94, 111, 66, 92, 97, 47, 99, 108, 64, 58, 40, 5, 41, 19, 74, 46, 110, 63, 34, 98, 103, 4, 123, 62, 75, 36, 71, 105, 7, 86, 109, 32, 78, 120, 85, 89, 35, 121, 9, 12, 69, 80, 8, 84, 115, 1, 70, 18, 72, 95, 91, 17, 104, 25, 79, 88, 2, 27, 16, 3, 10, 73, 87, 68, 0, 13, 65], [102, 118, 116, 61, 122, 54, 49, 90, 23, 20, 45, 9, 92, 81, 74, 77, 29, 15, 22, 93, 33, 113, 107, 7, 124, 53, 11, 64, 26, 38, 112, 55, 6, 94, 82, 31, 21, 70, 79, 66, 100, 50, 28, 119, 72, 57, 4, 83, 44, 37, 41, 127, 86, 59, 101, 36, 89, 5, 3, 85, 58, 110, 121, 67, 126, 125, 106, 35, 43, 115, 75, 1, 103, 63, 108, 60, 117, 123, 68, 56, 114, 98, 39, 109, 69, 47, 19, 34, 104, 42, 51, 62, 48, 24, 97, 111, 96, 27, 120, 65, 105, 95, 30, 52, 84, 14, 2, 32, 88, 46, 99, 10, 16, 80, 78, 76, 87, 18, 12, 0, 8, 25, 40, 91, 17, 71, 73, 13]], "model.layers.27.self_attn.k_proj": [[45, 109, 83, 23, 21, 90, 94, 81, 76, 79, 33, 74, 30, 54, 97, 28, 125, 60, 6, 123, 48, 127, 52, 22, 51, 8, 0, 113, 49, 112, 32, 101, 47, 126, 73, 63, 106, 31, 122, 46, 118, 42, 103, 117, 124, 91, 24, 121, 96, 44, 18, 11, 35, 102, 50, 7, 37, 95, 108, 114, 13, 39, 4, 119, 43, 65, 105, 116, 120, 115, 16, 36, 57, 86, 89, 61, 104, 111, 53, 40, 56, 110, 34, 82, 55, 15, 29, 38, 107, 25, 26, 59, 9, 5, 62, 41, 92, 14, 58, 88, 80, 75, 99, 100, 27, 98, 78, 69, 93, 10, 84, 3, 20, 77, 87, 85, 17, 71, 72, 19, 66, 70, 68, 12, 1, 67, 2, 64], [53, 37, 22, 120, 63, 118, 97, 86, 58, 61, 60, 57, 116, 121, 125, 112, 113, 56, 55, 114, 26, 51, 119, 127, 62, 124, 52, 122, 50, 108, 110, 59, 117, 123, 48, 126, 95, 45, 115, 54, 49, 44, 93, 111, 109, 35, 47, 46, 42, 107, 96, 43, 41, 83, 28, 105, 40, 106, 29, 104, 15, 89, 38, 103, 102, 79, 100, 23, 101, 81, 32, 99, 82, 36, 39, 31, 12, 13, 34, 33, 18, 98, 72, 17, 91, 88, 92, 30, 78, 87, 27, 11, 25, 85, 94, 16, 20, 10, 9, 90, 6, 77, 7, 24, 84, 14, 68, 80, 21, 75, 5, 76, 19, 2, 69, 74, 3, 67, 71, 65, 73, 70, 0, 1, 4, 8, 64, 66], [104, 63, 34, 80, 13, 74, 23, 8, 85, 60, 19, 4, 64, 6, 95, 26, 66, 52, 121, 114, 106, 122, 105, 117, 49, 79, 54, 119, 65, 57, 110, 28, 7, 44, 127, 82, 124, 120, 43, 55, 56, 125, 1, 112, 107, 59, 45, 11, 98, 3, 116, 103, 46, 70, 41, 2, 9, 58, 126, 69, 39, 53, 5, 47, 84, 51, 18, 24, 71, 115, 0, 123, 42, 50, 108, 67, 48, 31, 14, 96, 12, 20, 62, 61, 100, 35, 27, 75, 68, 97, 22, 118, 30, 99, 102, 36, 29, 109, 111, 10, 25, 83, 37, 113, 76, 78, 17, 32, 101, 94, 72, 89, 38, 33, 88, 93, 90, 15, 92, 81, 86, 91, 73, 87, 77, 21, 16, 40], [40, 120, 34, 27, 31, 110, 126, 56, 84, 82, 88, 74, 46, 80, 52, 13, 125, 64, 48, 93, 67, 108, 6, 16, 58, 85, 49, 76, 65, 72, 114, 23, 44, 30, 106, 77, 55, 100, 45, 115, 66, 117, 68, 111, 60, 86, 107, 5, 20, 22, 69, 79, 8, 78, 75, 94, 54, 59, 53, 21, 91, 61, 24, 57, 63, 109, 102, 124, 96, 73, 62, 112, 42, 87, 90, 15, 81, 83, 28, 2, 116, 101, 122, 123, 105, 103, 12, 121, 113, 127, 70, 38, 7, 41, 92, 118, 3, 119, 29, 33, 43, 39, 97, 51, 50, 36, 26, 89, 32, 11, 25, 99, 17, 37, 18, 47, 35, 0, 14, 98, 95, 4, 19, 71, 10, 9, 1, 104], [47, 111, 58, 36, 31, 86, 24, 92, 56, 82, 0, 26, 17, 15, 6, 16, 121, 3, 83, 12, 54, 11, 105, 91, 113, 84, 116, 20, 66, 85, 45, 55, 19, 77, 63, 46, 51, 126, 57, 78, 43, 124, 25, 61, 119, 68, 49, 42, 38, 117, 48, 44, 104, 59, 118, 60, 89, 50, 114, 52, 112, 53, 103, 127, 123, 7, 115, 97, 122, 120, 65, 110, 62, 99, 9, 41, 1, 8, 5, 39, 40, 34, 29, 109, 107, 37, 10, 108, 94, 93, 125, 35, 106, 71, 32, 74, 101, 21, 102, 72, 76, 4, 33, 67, 90, 22, 87, 64, 79, 98, 81, 100, 96, 30, 23, 69, 27, 18, 80, 2, 13, 75, 28, 73, 88, 70, 14, 95], [105, 48, 22, 119, 33, 51, 62, 112, 54, 99, 49, 61, 94, 118, 125, 60, 55, 122, 50, 121, 127, 35, 45, 56, 124, 106, 117, 123, 126, 108, 40, 52, 120, 114, 53, 82, 59, 100, 111, 47, 63, 98, 58, 39, 24, 110, 57, 44, 28, 32, 113, 92, 46, 115, 43, 79, 109, 91, 107, 41, 38, 34, 101, 42, 26, 116, 19, 18, 29, 102, 103, 36, 9, 15, 104, 73, 85, 88, 96, 37, 97, 21, 80, 77, 13, 90, 23, 31, 89, 30, 93, 95, 87, 4, 81, 27, 25, 83, 20, 14, 11, 71, 76, 1, 86, 12, 16, 74, 66, 17, 7, 8, 78, 10, 5, 75, 68, 6, 72, 84, 70, 64, 67, 3, 69, 2, 65, 0], [42, 85, 83, 89, 17, 114, 126, 120, 55, 35, 46, 14, 9, 56, 61, 127, 32, 117, 99, 123, 109, 57, 49, 62, 3, 75, 106, 69, 95, 7, 45, 91, 122, 48, 115, 15, 119, 53, 50, 51, 31, 93, 110, 59, 52, 0, 1, 87, 116, 13, 64, 58, 6, 54, 29, 44, 60, 23, 84, 111, 43, 66, 121, 41, 40, 38, 22, 113, 112, 74, 97, 107, 5, 10, 125, 68, 30, 108, 8, 118, 124, 20, 104, 80, 71, 98, 101, 76, 100, 102, 47, 33, 63, 11, 25, 39, 88, 26, 92, 103, 82, 12, 36, 105, 34, 94, 90, 37, 96, 16, 28, 86, 27, 72, 24, 18, 77, 79, 70, 21, 65, 4, 78, 81, 19, 73, 2, 67], [118, 38, 61, 54, 116, 113, 64, 122, 65, 0, 109, 69, 7, 23, 77, 74, 90, 93, 3, 15, 94, 108, 81, 97, 9, 59, 2, 48, 6, 68, 52, 124, 53, 117, 112, 43, 82, 75, 57, 119, 103, 20, 30, 62, 50, 19, 111, 123, 21, 83, 106, 115, 110, 127, 56, 67, 107, 120, 60, 125, 11, 63, 47, 46, 105, 96, 102, 66, 49, 51, 121, 55, 104, 22, 114, 126, 100, 44, 41, 40, 58, 4, 84, 39, 36, 45, 35, 98, 76, 33, 89, 34, 1, 8, 42, 14, 37, 92, 101, 95, 27, 88, 5, 80, 85, 16, 99, 72, 28, 70, 31, 79, 12, 24, 25, 91, 32, 86, 10, 18, 71, 78, 87, 13, 73, 29, 17, 26]], "model.layers.27.self_attn.qk_proj": [[120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 119, 51, 54, 58, 104, 60, 122, 56, 126, 40, 116, 31, 85, 94, 62, 52, 35, 55, 26, 83, 21, 91, 19, 125, 87, 121, 102, 110, 127, 95, 57, 117, 90, 23, 105, 46, 113, 49, 112, 108, 33, 80, 41, 77, 6, 16, 81, 34, 13, 98, 84, 88, 27, 50, 17, 10, 115, 86, 44, 38, 79, 59, 114, 24, 15, 20, 22, 37, 124, 36, 74, 123, 30, 64, 97, 100, 28, 0, 25, 18, 107, 93, 12, 76, 82, 101, 72, 43, 39, 11, 96, 92, 29, 89, 32, 9, 103, 69, 8, 99, 4, 65, 70, 73, 67, 14, 68, 3, 75, 66, 78, 2, 5, 71, 1, 7], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 51, 119, 106, 61, 54, 122, 58, 104, 60, 126, 125, 56, 31, 116, 62, 40, 121, 35, 55, 52, 85, 102, 19, 94, 95, 21, 46, 26, 23, 127, 105, 110, 57, 113, 112, 90, 33, 6, 117, 98, 49, 91, 87, 77, 108, 41, 13, 124, 50, 83, 80, 44, 59, 17, 114, 86, 115, 22, 123, 24, 38, 27, 16, 36, 88, 84, 15, 10, 74, 81, 79, 37, 34, 64, 93, 28, 20, 100, 43, 30, 0, 82, 101, 97, 107, 8, 96, 12, 11, 92, 68, 76, 25, 18, 9, 32, 39, 103, 1, 72, 69, 65, 99, 29, 3, 73, 66, 2, 75, 78, 4, 89, 5, 67, 7, 71, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 60, 56, 122, 104, 40, 116, 126, 62, 121, 125, 85, 31, 19, 105, 127, 94, 21, 46, 26, 83, 49, 110, 112, 52, 55, 35, 6, 117, 113, 57, 91, 102, 87, 95, 98, 23, 90, 80, 33, 17, 16, 108, 41, 123, 13, 27, 77, 43, 88, 24, 107, 115, 22, 38, 79, 37, 36, 34, 10, 59, 50, 44, 81, 84, 86, 64, 114, 93, 97, 30, 15, 124, 100, 0, 20, 8, 18, 74, 28, 92, 39, 101, 12, 82, 96, 11, 9, 68, 32, 4, 29, 76, 78, 99, 25, 65, 103, 1, 67, 73, 69, 14, 75, 89, 3, 66, 5, 72, 7, 2, 71, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 119, 54, 58, 60, 104, 122, 62, 116, 56, 40, 126, 125, 127, 85, 121, 31, 55, 46, 19, 26, 110, 113, 52, 112, 35, 87, 105, 102, 83, 91, 21, 49, 6, 94, 57, 16, 95, 124, 41, 117, 90, 77, 59, 98, 123, 43, 86, 17, 23, 115, 108, 79, 33, 27, 13, 10, 37, 88, 22, 44, 80, 107, 24, 81, 84, 8, 34, 0, 114, 30, 93, 64, 74, 15, 97, 50, 36, 38, 101, 100, 18, 28, 69, 68, 82, 76, 92, 20, 12, 73, 65, 11, 67, 29, 9, 1, 103, 4, 3, 99, 2, 96, 39, 71, 32, 70, 5, 25, 78, 66, 14, 89, 75, 72, 7], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 122, 60, 40, 126, 56, 116, 85, 125, 52, 112, 21, 35, 62, 46, 87, 83, 121, 127, 31, 105, 113, 90, 26, 110, 91, 49, 55, 57, 117, 19, 108, 98, 27, 16, 102, 13, 94, 23, 41, 95, 123, 6, 59, 17, 77, 86, 34, 33, 50, 124, 80, 74, 8, 79, 88, 44, 64, 22, 115, 84, 81, 15, 37, 93, 114, 107, 70, 20, 36, 24, 0, 10, 97, 18, 28, 82, 38, 43, 30, 73, 92, 12, 69, 11, 29, 25, 101, 100, 76, 68, 103, 96, 2, 39, 99, 9, 65, 67, 3, 1, 78, 5, 14, 72, 89, 4, 71, 7, 75, 32, 66], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 119, 54, 61, 58, 104, 60, 122, 116, 56, 126, 40, 127, 21, 87, 125, 85, 62, 52, 83, 110, 113, 35, 90, 112, 94, 46, 49, 105, 33, 13, 102, 41, 31, 26, 91, 77, 117, 57, 98, 23, 108, 17, 19, 16, 124, 15, 121, 86, 59, 80, 34, 95, 79, 22, 55, 8, 44, 27, 70, 81, 115, 20, 88, 10, 84, 74, 64, 43, 37, 24, 0, 38, 123, 18, 114, 50, 30, 107, 6, 39, 28, 12, 97, 100, 36, 93, 82, 92, 96, 73, 3, 75, 4, 25, 9, 101, 5, 99, 76, 65, 29, 11, 66, 69, 103, 78, 68, 2, 1, 67, 71, 7, 14, 32, 72, 89], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 51, 61, 119, 54, 58, 104, 122, 116, 56, 40, 94, 126, 60, 90, 87, 46, 83, 35, 52, 21, 125, 127, 85, 113, 110, 23, 112, 105, 62, 31, 57, 49, 102, 19, 55, 91, 121, 98, 117, 41, 27, 80, 77, 22, 33, 16, 95, 26, 124, 13, 70, 88, 59, 15, 24, 34, 84, 115, 108, 17, 86, 81, 8, 10, 79, 20, 18, 114, 44, 50, 38, 74, 30, 36, 123, 107, 37, 82, 28, 97, 64, 25, 93, 0, 43, 100, 39, 12, 96, 11, 76, 101, 99, 92, 66, 73, 14, 103, 69, 9, 32, 4, 5, 75, 71, 29, 89, 78, 3, 1, 68, 72, 65, 7, 6, 2, 67], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 106, 54, 119, 58, 122, 56, 60, 104, 126, 94, 40, 116, 35, 125, 52, 21, 105, 121, 85, 87, 31, 112, 110, 55, 83, 19, 113, 46, 102, 90, 62, 33, 95, 23, 98, 117, 57, 127, 49, 91, 27, 26, 70, 108, 80, 41, 22, 88, 115, 77, 13, 38, 16, 84, 36, 81, 79, 10, 24, 86, 59, 34, 114, 124, 28, 15, 17, 107, 74, 97, 44, 82, 20, 123, 30, 25, 0, 100, 50, 64, 43, 37, 93, 101, 8, 32, 12, 18, 99, 92, 76, 103, 96, 29, 72, 39, 66, 11, 75, 89, 9, 5, 4, 68, 14, 78, 73, 65, 3, 1, 67, 69, 7, 71, 2, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 51, 106, 119, 54, 58, 122, 104, 60, 56, 126, 116, 21, 94, 52, 121, 125, 40, 31, 117, 19, 85, 105, 110, 95, 35, 57, 62, 90, 87, 26, 113, 127, 112, 33, 23, 102, 46, 91, 83, 49, 16, 27, 59, 13, 55, 98, 80, 44, 114, 41, 77, 70, 108, 115, 22, 88, 50, 124, 17, 38, 24, 123, 34, 37, 10, 79, 84, 92, 86, 81, 43, 15, 36, 18, 97, 20, 74, 107, 30, 100, 28, 82, 64, 103, 93, 96, 0, 32, 101, 9, 76, 25, 89, 39, 12, 72, 99, 8, 75, 4, 73, 14, 29, 11, 68, 78, 5, 3, 2, 67, 7, 6, 69, 1, 65, 71, 66], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 51, 54, 106, 119, 58, 60, 122, 104, 125, 116, 56, 55, 40, 52, 46, 31, 121, 126, 85, 62, 35, 94, 90, 112, 83, 113, 127, 21, 110, 33, 95, 105, 117, 87, 26, 102, 49, 59, 124, 70, 19, 115, 23, 98, 41, 50, 91, 77, 80, 57, 108, 27, 0, 10, 74, 16, 114, 86, 123, 13, 43, 15, 34, 22, 17, 38, 30, 81, 44, 24, 64, 84, 100, 28, 36, 79, 20, 37, 107, 88, 72, 82, 92, 93, 18, 9, 97, 6, 32, 8, 65, 25, 39, 68, 76, 73, 5, 1, 67, 75, 96, 101, 69, 4, 2, 29, 103, 3, 12, 7, 66, 99, 11, 14, 78, 89, 71], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 61, 51, 119, 106, 54, 58, 104, 60, 122, 126, 40, 56, 116, 85, 125, 31, 87, 113, 117, 21, 94, 62, 55, 46, 105, 57, 127, 83, 26, 91, 35, 52, 90, 23, 112, 33, 80, 27, 95, 124, 19, 115, 110, 102, 13, 81, 121, 59, 77, 16, 74, 86, 49, 34, 6, 72, 79, 15, 98, 108, 114, 0, 22, 17, 41, 38, 24, 50, 10, 20, 43, 70, 44, 64, 88, 84, 30, 37, 18, 82, 36, 93, 28, 107, 123, 75, 100, 73, 12, 97, 68, 76, 92, 9, 3, 1, 101, 5, 29, 14, 99, 4, 67, 78, 8, 66, 11, 96, 32, 69, 2, 39, 89, 7, 103, 65, 71, 25], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 119, 51, 54, 104, 58, 56, 60, 122, 116, 126, 40, 31, 87, 21, 105, 85, 83, 113, 49, 19, 127, 35, 46, 23, 94, 55, 102, 117, 90, 52, 121, 125, 110, 6, 33, 91, 115, 112, 95, 77, 41, 26, 62, 80, 57, 34, 86, 124, 13, 98, 16, 79, 27, 72, 17, 59, 88, 50, 44, 108, 15, 114, 81, 22, 84, 74, 24, 123, 38, 10, 20, 0, 28, 43, 97, 93, 64, 30, 18, 75, 37, 76, 107, 99, 25, 73, 82, 92, 14, 100, 36, 12, 67, 68, 96, 9, 29, 101, 39, 65, 11, 78, 103, 2, 32, 3, 7, 70, 8, 89, 69, 66, 4, 5, 71, 1], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 61, 106, 51, 119, 54, 58, 56, 104, 60, 122, 116, 40, 105, 126, 121, 35, 52, 49, 125, 94, 112, 102, 110, 87, 23, 21, 57, 85, 90, 83, 113, 46, 31, 33, 19, 62, 41, 117, 26, 91, 95, 98, 115, 80, 55, 108, 27, 86, 127, 6, 124, 24, 15, 77, 22, 34, 79, 59, 38, 13, 50, 74, 88, 72, 123, 44, 114, 84, 16, 30, 17, 10, 100, 37, 81, 18, 20, 36, 28, 93, 43, 92, 64, 82, 97, 25, 0, 39, 75, 101, 107, 89, 12, 32, 96, 9, 14, 29, 76, 103, 99, 73, 4, 5, 1, 2, 68, 69, 66, 3, 78, 11, 8, 7, 67, 71, 65, 70], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 119, 106, 61, 54, 51, 58, 56, 60, 104, 122, 40, 126, 116, 62, 94, 125, 52, 46, 49, 31, 121, 117, 127, 21, 105, 114, 57, 85, 110, 87, 83, 113, 90, 6, 91, 98, 35, 102, 112, 23, 13, 19, 27, 26, 55, 95, 124, 33, 41, 80, 86, 59, 44, 16, 72, 24, 115, 74, 50, 84, 34, 123, 17, 108, 10, 77, 38, 36, 79, 0, 22, 43, 15, 88, 64, 81, 107, 37, 93, 82, 20, 30, 100, 18, 28, 4, 75, 92, 97, 76, 101, 32, 9, 25, 96, 2, 12, 29, 78, 5, 3, 39, 1, 68, 73, 14, 69, 67, 99, 65, 89, 103, 7, 11, 71, 66, 8, 70], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 104, 60, 122, 116, 56, 126, 40, 94, 87, 52, 125, 21, 46, 31, 35, 62, 85, 83, 19, 105, 112, 121, 90, 113, 102, 127, 117, 26, 95, 57, 91, 55, 110, 27, 33, 124, 80, 49, 59, 114, 13, 81, 6, 41, 86, 15, 10, 108, 23, 50, 72, 34, 17, 64, 88, 16, 84, 98, 115, 44, 43, 36, 74, 77, 79, 123, 20, 100, 30, 37, 0, 38, 24, 73, 97, 75, 93, 12, 18, 22, 82, 32, 107, 28, 76, 39, 101, 92, 89, 69, 65, 68, 3, 5, 70, 4, 25, 9, 67, 1, 96, 2, 103, 29, 66, 78, 11, 14, 71, 99, 8, 7], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 119, 54, 58, 104, 56, 60, 122, 116, 126, 85, 87, 83, 62, 105, 40, 35, 21, 52, 102, 31, 125, 46, 94, 95, 113, 91, 19, 33, 55, 110, 57, 13, 23, 26, 121, 59, 112, 90, 127, 41, 80, 117, 108, 115, 34, 81, 124, 49, 16, 77, 27, 38, 79, 17, 15, 44, 123, 10, 86, 37, 22, 114, 50, 6, 43, 88, 84, 74, 98, 24, 72, 0, 36, 93, 20, 12, 75, 76, 92, 101, 82, 100, 28, 70, 30, 73, 18, 64, 9, 4, 97, 99, 107, 8, 1, 5, 96, 14, 68, 11, 39, 25, 3, 2, 32, 89, 69, 29, 78, 71, 66, 7, 103, 67, 65], [120, 118, 63, 45, 109, 47, 53, 42, 111, 48, 106, 51, 61, 54, 119, 58, 104, 122, 56, 60, 52, 116, 121, 46, 94, 126, 35, 21, 40, 95, 83, 102, 31, 49, 85, 23, 90, 110, 125, 62, 113, 41, 33, 55, 112, 57, 87, 19, 105, 26, 91, 98, 13, 80, 127, 86, 108, 115, 59, 124, 34, 27, 15, 16, 88, 117, 123, 24, 92, 44, 28, 10, 22, 17, 84, 74, 114, 70, 50, 36, 77, 81, 79, 30, 97, 20, 38, 0, 100, 93, 37, 82, 18, 64, 43, 101, 107, 72, 8, 12, 103, 9, 25, 99, 96, 32, 39, 29, 76, 69, 1, 4, 89, 66, 73, 6, 68, 78, 75, 14, 67, 11, 3, 7, 65, 2, 71, 5], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 61, 119, 106, 54, 58, 122, 104, 60, 56, 116, 94, 121, 40, 52, 62, 126, 85, 31, 35, 90, 21, 105, 19, 125, 49, 87, 110, 95, 102, 46, 83, 55, 23, 41, 112, 91, 59, 117, 26, 108, 113, 98, 57, 33, 27, 70, 24, 13, 127, 81, 114, 17, 44, 74, 77, 22, 16, 86, 15, 36, 79, 80, 43, 34, 124, 88, 50, 37, 38, 115, 100, 10, 84, 92, 123, 97, 28, 30, 0, 64, 20, 8, 12, 32, 101, 18, 107, 82, 96, 72, 9, 75, 1, 103, 25, 69, 76, 29, 93, 4, 78, 89, 99, 68, 67, 39, 11, 66, 7, 73, 14, 2, 5, 3, 6, 71, 65], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 119, 51, 61, 54, 58, 104, 60, 122, 116, 40, 121, 52, 56, 125, 62, 126, 46, 35, 85, 19, 49, 110, 21, 105, 102, 90, 94, 31, 108, 95, 70, 87, 55, 117, 127, 124, 112, 59, 26, 113, 91, 33, 23, 57, 83, 41, 37, 123, 114, 98, 13, 115, 79, 50, 44, 38, 80, 16, 8, 81, 27, 10, 36, 17, 15, 34, 24, 88, 43, 84, 64, 0, 77, 22, 30, 86, 107, 74, 1, 20, 97, 100, 82, 92, 101, 75, 93, 28, 9, 99, 18, 32, 69, 12, 103, 96, 25, 2, 73, 4, 78, 76, 5, 39, 72, 89, 29, 67, 68, 11, 3, 65, 66, 7, 71, 6, 14], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 60, 122, 116, 40, 52, 62, 35, 56, 113, 126, 85, 55, 46, 125, 94, 110, 102, 90, 21, 26, 121, 105, 31, 23, 70, 57, 83, 127, 41, 117, 33, 19, 95, 87, 59, 108, 112, 49, 124, 13, 91, 86, 16, 27, 123, 50, 8, 98, 15, 37, 77, 114, 80, 74, 44, 10, 17, 38, 24, 34, 115, 81, 88, 43, 36, 79, 64, 22, 30, 107, 84, 100, 97, 82, 0, 93, 73, 28, 20, 92, 32, 101, 12, 1, 68, 18, 66, 96, 4, 75, 76, 69, 103, 39, 3, 5, 9, 25, 67, 99, 14, 78, 29, 65, 71, 2, 72, 89, 7, 11, 6], [120, 63, 118, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 58, 119, 104, 60, 116, 122, 40, 56, 126, 85, 55, 125, 105, 52, 113, 94, 62, 46, 121, 21, 35, 83, 110, 90, 87, 127, 117, 31, 19, 57, 102, 95, 41, 49, 115, 26, 124, 16, 91, 23, 112, 77, 33, 108, 98, 13, 34, 27, 59, 81, 86, 15, 70, 123, 38, 74, 44, 8, 22, 80, 50, 43, 88, 10, 79, 17, 36, 84, 30, 114, 20, 92, 24, 100, 28, 37, 0, 75, 107, 99, 82, 25, 97, 39, 18, 101, 93, 12, 6, 64, 73, 32, 9, 76, 14, 68, 4, 5, 67, 1, 78, 66, 103, 3, 69, 96, 72, 11, 65, 71, 7, 89, 29, 2], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 106, 54, 119, 58, 122, 60, 104, 116, 121, 40, 126, 125, 56, 62, 110, 52, 46, 85, 113, 55, 21, 94, 127, 35, 105, 102, 112, 19, 117, 31, 124, 59, 87, 83, 26, 49, 90, 57, 95, 41, 43, 108, 44, 114, 23, 98, 91, 13, 38, 33, 123, 0, 27, 88, 77, 16, 80, 6, 34, 8, 10, 64, 86, 37, 15, 115, 36, 24, 74, 81, 100, 79, 50, 17, 107, 22, 28, 30, 93, 97, 1, 84, 32, 82, 20, 92, 101, 5, 70, 39, 18, 73, 68, 4, 66, 12, 9, 69, 76, 67, 2, 65, 103, 75, 11, 14, 96, 29, 99, 25, 71, 3, 89, 78, 7, 72], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 61, 106, 54, 51, 119, 58, 122, 56, 104, 60, 116, 40, 52, 117, 85, 126, 125, 105, 46, 35, 62, 121, 31, 94, 95, 21, 83, 87, 55, 102, 59, 49, 127, 57, 90, 113, 26, 110, 124, 112, 19, 6, 41, 114, 91, 27, 23, 33, 108, 98, 34, 44, 123, 16, 80, 77, 115, 88, 13, 15, 38, 10, 22, 36, 24, 28, 81, 84, 79, 8, 74, 50, 100, 37, 97, 86, 107, 30, 43, 0, 17, 93, 18, 92, 39, 82, 20, 101, 25, 64, 9, 4, 68, 103, 96, 76, 75, 1, 89, 99, 32, 12, 11, 69, 73, 78, 2, 3, 5, 67, 29, 71, 72, 66, 7, 65, 14, 70], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 106, 61, 119, 54, 58, 104, 122, 60, 56, 116, 40, 126, 52, 21, 105, 94, 31, 85, 125, 113, 121, 35, 46, 87, 90, 83, 95, 110, 49, 57, 6, 55, 59, 62, 26, 102, 127, 41, 112, 33, 117, 124, 19, 23, 44, 108, 91, 114, 27, 86, 80, 77, 98, 22, 34, 16, 13, 88, 10, 81, 107, 15, 123, 17, 115, 74, 43, 8, 79, 37, 38, 84, 50, 30, 36, 20, 97, 18, 92, 100, 0, 93, 24, 82, 28, 64, 4, 76, 101, 12, 9, 73, 75, 96, 25, 5, 39, 11, 78, 68, 3, 14, 2, 103, 69, 66, 29, 32, 67, 72, 1, 65, 99, 7, 89, 71, 70], [120, 118, 63, 45, 109, 111, 47, 42, 53, 48, 51, 106, 54, 61, 119, 122, 104, 58, 60, 116, 40, 126, 56, 125, 52, 21, 83, 87, 90, 85, 94, 62, 105, 121, 35, 46, 124, 55, 6, 117, 27, 33, 113, 31, 49, 26, 102, 127, 95, 91, 110, 115, 19, 77, 112, 23, 80, 13, 86, 108, 41, 17, 59, 123, 57, 98, 16, 22, 88, 34, 79, 10, 15, 38, 84, 24, 44, 43, 74, 30, 81, 114, 50, 20, 8, 36, 64, 107, 18, 37, 28, 82, 93, 97, 0, 100, 25, 9, 92, 72, 101, 12, 99, 73, 75, 103, 78, 96, 32, 1, 76, 5, 66, 68, 69, 4, 11, 29, 3, 7, 14, 89, 65, 2, 39, 67, 71, 70], [120, 118, 63, 45, 109, 47, 111, 53, 42, 48, 51, 61, 106, 54, 119, 60, 122, 58, 104, 56, 116, 125, 21, 35, 40, 121, 52, 126, 87, 31, 62, 83, 94, 95, 46, 105, 55, 57, 85, 33, 113, 102, 117, 127, 26, 115, 49, 91, 23, 59, 110, 19, 90, 108, 123, 41, 124, 77, 114, 27, 13, 98, 22, 50, 44, 36, 112, 80, 38, 34, 88, 86, 24, 6, 16, 10, 15, 81, 84, 43, 30, 79, 37, 28, 17, 93, 0, 92, 74, 97, 100, 107, 18, 72, 32, 82, 101, 64, 20, 103, 76, 96, 25, 1, 29, 73, 8, 12, 9, 39, 68, 78, 75, 89, 11, 99, 14, 3, 69, 66, 4, 5, 67, 7, 70, 65, 71, 2], [120, 118, 63, 45, 47, 109, 42, 111, 53, 48, 54, 106, 61, 119, 51, 104, 122, 56, 58, 60, 126, 116, 31, 40, 94, 125, 35, 62, 117, 52, 21, 87, 121, 105, 127, 55, 83, 46, 85, 95, 90, 91, 115, 33, 102, 19, 113, 23, 41, 112, 57, 27, 26, 114, 110, 108, 13, 44, 77, 49, 124, 22, 59, 98, 80, 34, 16, 36, 81, 88, 123, 17, 72, 24, 38, 74, 37, 15, 86, 107, 100, 50, 10, 18, 79, 84, 43, 97, 92, 20, 25, 82, 30, 28, 101, 93, 64, 70, 6, 103, 96, 39, 12, 68, 0, 73, 29, 32, 4, 11, 99, 76, 69, 9, 1, 75, 3, 14, 67, 66, 78, 8, 89, 65, 7, 2, 71, 5], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 51, 61, 54, 106, 119, 58, 104, 60, 122, 116, 126, 56, 40, 52, 62, 21, 94, 125, 35, 46, 112, 127, 31, 85, 90, 87, 121, 102, 19, 105, 95, 108, 110, 55, 41, 83, 26, 23, 49, 117, 113, 33, 59, 70, 91, 123, 27, 13, 57, 72, 16, 98, 77, 124, 115, 44, 43, 79, 17, 34, 88, 80, 37, 50, 15, 86, 36, 22, 114, 74, 81, 10, 64, 84, 100, 0, 38, 30, 24, 20, 18, 39, 107, 92, 82, 28, 4, 93, 5, 66, 101, 97, 73, 76, 9, 1, 11, 32, 12, 25, 6, 69, 89, 65, 68, 75, 29, 67, 3, 7, 2, 78, 96, 14, 103, 99, 8, 71], [120, 118, 63, 45, 109, 47, 111, 42, 53, 48, 106, 61, 51, 54, 119, 58, 122, 104, 60, 56, 116, 125, 40, 126, 46, 85, 55, 52, 62, 121, 35, 70, 49, 90, 31, 87, 117, 19, 105, 26, 83, 21, 57, 94, 112, 113, 110, 102, 95, 124, 91, 27, 108, 41, 17, 115, 50, 77, 33, 13, 64, 72, 34, 127, 23, 59, 16, 81, 44, 80, 86, 10, 43, 123, 38, 36, 84, 98, 0, 79, 15, 100, 74, 88, 22, 1, 37, 20, 28, 24, 107, 92, 9, 30, 93, 114, 82, 76, 101, 97, 11, 2, 5, 73, 25, 12, 18, 29, 96, 3, 67, 66, 39, 4, 103, 75, 68, 99, 69, 78, 32, 89, 14, 7, 65, 71, 8, 6], [120, 63, 118, 45, 109, 47, 42, 111, 53, 48, 51, 106, 61, 54, 119, 58, 60, 122, 104, 56, 125, 116, 126, 52, 94, 40, 85, 62, 90, 87, 21, 55, 46, 35, 117, 83, 31, 105, 49, 102, 70, 110, 113, 23, 57, 26, 123, 19, 121, 127, 91, 59, 77, 95, 33, 114, 112, 13, 41, 108, 27, 16, 34, 80, 124, 22, 81, 86, 44, 15, 10, 72, 98, 17, 84, 43, 74, 115, 38, 37, 36, 79, 88, 24, 50, 20, 100, 18, 107, 64, 97, 92, 82, 28, 101, 93, 76, 96, 0, 73, 30, 11, 25, 9, 4, 68, 12, 103, 39, 32, 5, 69, 99, 14, 89, 65, 3, 1, 78, 75, 67, 66, 29, 7, 71, 8, 2, 6], [120, 63, 118, 45, 109, 47, 42, 53, 111, 48, 51, 61, 106, 54, 119, 58, 60, 104, 56, 122, 125, 116, 126, 40, 94, 52, 62, 35, 121, 57, 85, 87, 105, 83, 46, 21, 110, 19, 26, 90, 31, 55, 113, 98, 49, 70, 102, 117, 91, 127, 95, 41, 112, 23, 33, 59, 124, 115, 123, 77, 86, 16, 72, 34, 27, 88, 79, 15, 44, 10, 81, 80, 74, 13, 108, 64, 114, 84, 17, 43, 22, 24, 0, 28, 38, 50, 36, 30, 20, 97, 92, 100, 82, 37, 93, 68, 107, 101, 4, 25, 18, 5, 76, 9, 39, 69, 73, 66, 67, 11, 78, 1, 32, 65, 96, 3, 14, 12, 71, 2, 8, 103, 75, 29, 99, 7, 89, 6], [120, 118, 63, 45, 109, 47, 42, 111, 53, 48, 106, 61, 51, 54, 119, 58, 122, 60, 104, 56, 126, 40, 125, 116, 52, 85, 31, 35, 94, 121, 87, 83, 19, 90, 21, 117, 55, 62, 105, 57, 108, 26, 127, 95, 46, 110, 77, 102, 33, 112, 86, 113, 49, 23, 41, 91, 98, 27, 124, 13, 80, 16, 79, 38, 22, 44, 114, 115, 34, 15, 17, 74, 50, 10, 88, 59, 70, 81, 72, 36, 100, 24, 30, 25, 84, 20, 82, 123, 76, 97, 37, 92, 43, 93, 28, 18, 64, 11, 0, 12, 73, 107, 99, 96, 6, 101, 39, 32, 78, 4, 14, 9, 67, 69, 103, 66, 29, 89, 68, 8, 5, 65, 1, 75, 3, 71, 7, 2]], "model.layers.28.self_attn.q_proj": [[40, 120, 48, 34, 27, 51, 23, 85, 116, 126, 89, 53, 54, 16, 17, 112, 121, 3, 62, 31, 125, 124, 61, 93, 123, 50, 52, 109, 58, 114, 30, 49, 122, 19, 45, 110, 42, 7, 78, 29, 57, 82, 119, 60, 8, 115, 113, 59, 47, 55, 111, 63, 107, 117, 5, 118, 12, 56, 108, 46, 43, 79, 127, 74, 14, 44, 105, 88, 71, 100, 21, 20, 41, 65, 37, 64, 106, 11, 39, 1, 101, 38, 36, 87, 35, 91, 102, 0, 103, 22, 97, 66, 95, 6, 96, 10, 99, 98, 104, 84, 26, 32, 94, 2, 4, 90, 68, 73, 24, 77, 15, 13, 33, 9, 25, 28, 69, 18, 67, 83, 92, 80, 86, 72, 81, 75, 70, 76], [40, 120, 34, 48, 23, 27, 85, 17, 16, 31, 51, 11, 25, 13, 97, 61, 29, 62, 100, 55, 4, 78, 126, 19, 87, 56, 58, 73, 70, 59, 93, 28, 36, 35, 21, 113, 79, 46, 68, 22, 53, 18, 32, 96, 66, 109, 92, 10, 123, 95, 84, 8, 60, 63, 127, 0, 106, 49, 104, 37, 12, 112, 114, 116, 94, 26, 119, 99, 124, 7, 101, 30, 117, 83, 76, 103, 2, 118, 20, 44, 89, 52, 54, 125, 108, 81, 38, 33, 77, 110, 107, 15, 39, 102, 121, 111, 43, 122, 80, 86, 91, 47, 14, 57, 24, 90, 41, 82, 105, 50, 98, 75, 45, 74, 115, 5, 6, 88, 42, 67, 9, 72, 69, 71, 1, 65, 3, 64], [120, 40, 34, 48, 62, 23, 51, 27, 126, 85, 31, 93, 117, 109, 112, 58, 56, 113, 107, 16, 61, 50, 55, 44, 89, 123, 17, 19, 49, 53, 47, 8, 0, 30, 116, 106, 21, 95, 42, 52, 46, 25, 79, 43, 29, 108, 100, 121, 111, 119, 68, 110, 11, 57, 73, 38, 98, 54, 122, 59, 96, 36, 124, 118, 99, 63, 5, 103, 12, 105, 41, 78, 127, 125, 115, 94, 114, 97, 45, 35, 87, 37, 33, 60, 32, 39, 102, 22, 66, 101, 14, 24, 2, 70, 13, 10, 26, 86, 28, 91, 88, 92, 65, 18, 82, 67, 90, 6, 83, 84, 77, 4, 20, 64, 71, 75, 81, 104, 7, 1, 80, 69, 3, 15, 76, 9, 72, 74], [40, 120, 48, 34, 23, 63, 51, 31, 27, 62, 85, 49, 13, 123, 61, 108, 56, 79, 1, 110, 53, 55, 67, 93, 10, 64, 107, 16, 74, 113, 30, 19, 2, 17, 117, 70, 124, 18, 100, 59, 115, 127, 45, 68, 38, 26, 103, 102, 105, 111, 58, 46, 122, 119, 109, 52, 37, 116, 96, 36, 54, 24, 78, 106, 121, 112, 60, 50, 6, 84, 11, 125, 29, 114, 21, 118, 126, 0, 101, 57, 94, 43, 12, 73, 89, 88, 97, 47, 77, 39, 22, 35, 98, 95, 41, 5, 99, 33, 65, 44, 28, 32, 42, 86, 87, 92, 90, 8, 14, 72, 83, 25, 9, 82, 20, 69, 91, 3, 76, 7, 75, 80, 81, 15, 4, 71, 104, 66], [40, 125, 87, 17, 34, 21, 82, 70, 12, 74, 14, 4, 44, 0, 8, 124, 113, 111, 66, 54, 79, 58, 127, 68, 52, 109, 73, 110, 53, 19, 28, 75, 16, 33, 35, 123, 22, 59, 104, 27, 1, 95, 115, 119, 69, 86, 25, 114, 42, 50, 89, 18, 93, 108, 11, 26, 71, 96, 13, 90, 77, 56, 116, 67, 84, 63, 55, 5, 64, 62, 81, 78, 6, 92, 85, 106, 51, 7, 45, 39, 101, 23, 57, 103, 122, 98, 49, 94, 91, 88, 99, 32, 107, 48, 9, 2, 100, 29, 83, 105, 112, 65, 60, 24, 15, 76, 97, 3, 61, 47, 72, 38, 80, 36, 37, 121, 126, 46, 30, 20, 10, 31, 102, 118, 120, 41, 117, 43], [125, 40, 3, 87, 34, 12, 17, 14, 70, 0, 74, 1, 67, 66, 82, 21, 58, 64, 65, 8, 4, 69, 124, 54, 113, 25, 44, 111, 106, 52, 71, 49, 127, 2, 28, 75, 104, 109, 110, 50, 11, 42, 114, 22, 5, 79, 43, 26, 23, 121, 37, 68, 123, 101, 53, 86, 96, 31, 98, 112, 90, 27, 95, 77, 78, 20, 59, 13, 35, 122, 102, 81, 16, 29, 119, 6, 19, 91, 45, 39, 80, 47, 24, 32, 73, 117, 9, 76, 118, 103, 7, 88, 93, 84, 46, 72, 18, 33, 116, 55, 41, 30, 126, 105, 36, 115, 89, 62, 15, 120, 60, 63, 10, 57, 85, 48, 100, 38, 92, 107, 108, 94, 56, 83, 51, 97, 61, 99], [125, 40, 87, 34, 82, 21, 124, 12, 17, 14, 74, 109, 103, 50, 19, 4, 44, 8, 71, 95, 89, 92, 54, 70, 123, 127, 90, 9, 111, 68, 93, 108, 5, 39, 28, 115, 43, 41, 25, 77, 29, 110, 42, 105, 27, 61, 59, 100, 119, 58, 116, 64, 56, 66, 22, 79, 114, 88, 113, 47, 73, 121, 45, 91, 20, 60, 62, 106, 18, 75, 37, 49, 15, 10, 57, 52, 122, 31, 63, 51, 23, 48, 38, 33, 26, 96, 94, 6, 53, 32, 117, 86, 120, 112, 102, 36, 80, 97, 101, 72, 118, 83, 35, 69, 126, 99, 107, 46, 30, 85, 24, 55, 81, 13, 84, 78, 2, 98, 16, 7, 76, 11, 0, 1, 3, 67, 65, 104], [40, 125, 87, 0, 17, 34, 21, 12, 82, 1, 14, 66, 124, 4, 70, 50, 74, 44, 75, 58, 26, 127, 54, 69, 110, 113, 28, 111, 8, 95, 52, 13, 88, 33, 45, 79, 93, 42, 114, 64, 22, 103, 32, 98, 109, 67, 20, 123, 73, 119, 35, 11, 92, 19, 49, 104, 15, 39, 30, 27, 18, 90, 25, 59, 68, 96, 71, 102, 107, 53, 72, 101, 112, 62, 2, 65, 91, 60, 16, 77, 61, 80, 106, 99, 121, 115, 43, 7, 23, 105, 81, 84, 78, 55, 97, 3, 118, 76, 51, 122, 94, 83, 29, 89, 48, 117, 41, 86, 63, 9, 56, 5, 46, 108, 120, 31, 126, 24, 6, 116, 57, 85, 47, 100, 38, 10, 36, 37], [42, 36, 30, 43, 56, 106, 20, 118, 87, 77, 35, 82, 91, 123, 94, 46, 52, 80, 117, 49, 53, 89, 104, 120, 27, 99, 112, 17, 39, 47, 23, 122, 51, 78, 85, 54, 15, 11, 60, 19, 114, 103, 124, 71, 105, 125, 34, 58, 57, 95, 107, 108, 126, 110, 61, 62, 22, 74, 50, 32, 84, 109, 9, 121, 44, 88, 29, 13, 116, 101, 18, 55, 31, 59, 92, 115, 86, 45, 113, 98, 90, 111, 102, 41, 63, 48, 26, 12, 21, 25, 97, 119, 8, 24, 38, 127, 16, 37, 81, 83, 75, 14, 100, 33, 7, 96, 40, 28, 93, 72, 70, 69, 5, 76, 73, 79, 3, 67, 66, 2, 10, 68, 6, 1, 4, 65, 0, 64], [42, 36, 114, 118, 30, 15, 74, 35, 20, 17, 68, 87, 106, 77, 82, 70, 4, 91, 56, 76, 71, 0, 66, 123, 95, 64, 50, 31, 94, 43, 52, 21, 27, 23, 62, 7, 53, 11, 10, 122, 98, 9, 12, 29, 5, 67, 102, 84, 19, 18, 28, 111, 13, 65, 88, 1, 37, 85, 6, 8, 120, 2, 107, 32, 49, 112, 57, 103, 81, 83, 93, 14, 79, 69, 59, 96, 124, 46, 24, 44, 97, 26, 54, 126, 47, 109, 25, 16, 45, 73, 3, 60, 38, 72, 104, 86, 80, 78, 34, 108, 115, 41, 101, 58, 39, 22, 75, 61, 105, 92, 89, 48, 90, 51, 117, 33, 121, 119, 125, 110, 116, 63, 100, 113, 55, 127, 40, 99], [42, 36, 114, 118, 30, 87, 15, 123, 17, 35, 89, 74, 106, 20, 82, 94, 70, 56, 77, 50, 19, 66, 104, 52, 27, 68, 91, 8, 43, 6, 54, 72, 29, 44, 120, 58, 95, 75, 11, 47, 1, 38, 23, 51, 59, 122, 112, 113, 57, 49, 124, 61, 32, 86, 53, 62, 125, 99, 93, 111, 3, 116, 92, 102, 98, 12, 24, 103, 34, 71, 48, 28, 78, 64, 63, 76, 40, 127, 107, 108, 97, 46, 83, 119, 126, 96, 60, 110, 84, 10, 115, 81, 45, 25, 117, 37, 101, 0, 105, 21, 121, 2, 33, 31, 14, 109, 80, 55, 18, 26, 41, 9, 39, 85, 13, 16, 79, 5, 22, 90, 67, 88, 69, 100, 73, 7, 4, 65], [42, 114, 36, 35, 30, 106, 91, 103, 20, 87, 50, 27, 15, 17, 43, 74, 82, 94, 89, 68, 71, 123, 56, 104, 77, 95, 70, 120, 113, 85, 105, 65, 29, 62, 46, 3, 57, 127, 118, 61, 80, 119, 2, 125, 76, 122, 48, 117, 90, 1, 126, 47, 45, 111, 52, 44, 11, 51, 55, 109, 12, 49, 54, 121, 38, 75, 124, 63, 8, 102, 112, 67, 78, 107, 58, 66, 31, 59, 110, 115, 108, 116, 7, 72, 69, 81, 101, 41, 24, 86, 53, 33, 0, 73, 5, 26, 39, 28, 37, 19, 60, 13, 23, 21, 98, 84, 34, 93, 99, 14, 64, 92, 16, 9, 83, 96, 40, 25, 97, 88, 32, 22, 10, 4, 18, 79, 6, 100], [126, 41, 98, 56, 115, 89, 84, 13, 111, 80, 18, 112, 28, 45, 86, 75, 74, 31, 50, 21, 96, 82, 54, 58, 79, 5, 73, 8, 29, 81, 55, 11, 14, 70, 63, 120, 68, 4, 3, 25, 106, 119, 127, 117, 77, 40, 60, 71, 122, 92, 48, 10, 95, 88, 66, 46, 57, 59, 6, 76, 78, 83, 108, 34, 2, 93, 43, 51, 72, 16, 22, 0, 19, 20, 35, 124, 23, 44, 102, 121, 125, 62, 87, 107, 1, 113, 100, 61, 52, 24, 27, 85, 12, 90, 26, 17, 123, 49, 47, 105, 9, 114, 118, 32, 116, 103, 36, 7, 53, 99, 91, 110, 67, 109, 104, 33, 38, 94, 30, 97, 37, 39, 101, 65, 42, 64, 15, 69], [56, 41, 58, 98, 87, 63, 31, 32, 126, 28, 54, 122, 115, 44, 57, 17, 53, 60, 121, 46, 112, 43, 103, 55, 124, 116, 108, 127, 50, 120, 51, 26, 47, 117, 111, 86, 48, 113, 62, 52, 110, 61, 118, 119, 49, 45, 92, 114, 123, 109, 59, 125, 89, 101, 105, 40, 95, 107, 42, 23, 88, 22, 39, 91, 24, 100, 106, 84, 25, 94, 34, 30, 104, 37, 82, 15, 102, 38, 97, 99, 36, 35, 90, 21, 93, 80, 96, 33, 83, 29, 18, 20, 81, 79, 12, 27, 78, 19, 10, 85, 77, 1, 76, 14, 13, 7, 72, 66, 0, 5, 67, 3, 65, 16, 2, 64, 69, 11, 75, 71, 4, 8, 73, 70, 9, 74, 6, 68], [56, 41, 126, 98, 89, 79, 84, 18, 115, 80, 15, 31, 21, 28, 72, 13, 58, 10, 11, 63, 86, 82, 46, 4, 57, 40, 14, 73, 112, 70, 92, 77, 111, 100, 25, 23, 32, 122, 3, 121, 113, 60, 43, 127, 119, 75, 81, 16, 45, 87, 125, 95, 53, 66, 29, 62, 55, 109, 96, 51, 19, 124, 69, 5, 50, 52, 120, 9, 49, 71, 123, 1, 0, 48, 59, 22, 20, 47, 33, 76, 116, 8, 85, 6, 44, 27, 117, 103, 93, 17, 30, 61, 78, 37, 110, 64, 114, 88, 118, 24, 35, 2, 12, 91, 94, 83, 39, 90, 104, 54, 105, 99, 26, 102, 108, 68, 101, 7, 65, 97, 106, 36, 107, 42, 38, 74, 34, 67], [41, 126, 56, 63, 105, 60, 31, 127, 122, 87, 51, 52, 115, 98, 47, 49, 62, 121, 57, 123, 112, 124, 58, 46, 28, 117, 103, 114, 59, 116, 118, 45, 109, 86, 119, 54, 61, 113, 48, 55, 111, 120, 92, 50, 32, 53, 125, 44, 110, 108, 89, 84, 40, 43, 42, 107, 18, 39, 106, 104, 26, 100, 101, 37, 38, 102, 23, 7, 36, 35, 94, 96, 34, 91, 95, 99, 97, 17, 33, 80, 30, 25, 21, 85, 82, 88, 20, 22, 93, 24, 29, 27, 90, 11, 19, 83, 78, 77, 5, 73, 14, 81, 12, 3, 70, 0, 8, 71, 76, 79, 1, 16, 65, 10, 13, 67, 66, 64, 2, 74, 69, 9, 4, 72, 68, 75, 6, 15], [123, 103, 55, 47, 34, 125, 27, 95, 88, 32, 110, 122, 21, 15, 76, 19, 127, 24, 53, 97, 91, 94, 18, 36, 71, 51, 59, 43, 104, 38, 40, 101, 44, 118, 74, 106, 42, 26, 58, 102, 57, 52, 90, 117, 9, 92, 115, 111, 62, 2, 124, 96, 10, 87, 29, 48, 116, 33, 16, 126, 22, 78, 120, 112, 105, 85, 60, 63, 46, 107, 100, 114, 41, 113, 25, 1, 45, 54, 121, 37, 99, 23, 14, 31, 98, 49, 108, 56, 93, 39, 89, 50, 61, 109, 5, 79, 70, 119, 68, 13, 35, 28, 84, 82, 83, 86, 81, 30, 20, 80, 66, 64, 17, 11, 77, 12, 8, 73, 75, 3, 69, 4, 72, 7, 6, 65, 0, 67], [123, 103, 55, 34, 116, 27, 76, 88, 70, 0, 3, 18, 15, 1, 19, 68, 122, 66, 21, 95, 99, 71, 94, 91, 22, 59, 51, 9, 62, 125, 74, 56, 30, 73, 77, 67, 33, 126, 10, 47, 31, 37, 96, 109, 36, 57, 6, 79, 8, 107, 48, 80, 110, 29, 49, 35, 45, 118, 115, 58, 50, 63, 44, 121, 65, 86, 24, 26, 60, 17, 82, 85, 92, 23, 119, 40, 20, 52, 104, 13, 98, 112, 5, 120, 28, 113, 117, 114, 69, 72, 11, 102, 124, 81, 78, 14, 38, 53, 46, 43, 93, 61, 39, 87, 16, 108, 105, 75, 111, 42, 54, 106, 41, 25, 84, 32, 90, 97, 83, 101, 127, 4, 89, 100, 7, 12, 2, 64], [123, 103, 58, 122, 27, 88, 21, 91, 47, 18, 15, 59, 38, 34, 62, 9, 5, 95, 48, 120, 55, 49, 125, 60, 110, 44, 53, 121, 50, 56, 117, 119, 30, 105, 57, 52, 63, 24, 124, 67, 40, 113, 126, 94, 109, 112, 96, 51, 54, 118, 97, 46, 45, 116, 107, 111, 76, 108, 114, 85, 42, 115, 100, 43, 106, 104, 61, 36, 127, 101, 99, 41, 75, 29, 0, 92, 71, 86, 33, 102, 2, 64, 4, 37, 14, 39, 35, 17, 81, 3, 98, 73, 93, 82, 69, 77, 7, 11, 32, 87, 79, 68, 26, 84, 72, 89, 31, 8, 25, 65, 20, 83, 80, 90, 23, 22, 13, 28, 6, 78, 16, 19, 74, 1, 10, 12, 70, 66], [123, 103, 47, 34, 71, 76, 62, 122, 27, 55, 21, 22, 59, 19, 88, 125, 68, 15, 18, 101, 91, 109, 97, 45, 58, 24, 2, 56, 48, 64, 95, 117, 63, 36, 32, 38, 85, 111, 94, 66, 92, 77, 107, 10, 84, 9, 110, 30, 44, 119, 127, 5, 115, 50, 87, 79, 82, 73, 20, 99, 40, 124, 83, 49, 51, 42, 75, 61, 116, 57, 41, 46, 89, 60, 126, 120, 52, 54, 70, 118, 104, 113, 29, 112, 114, 53, 35, 121, 26, 105, 1, 93, 0, 100, 37, 108, 106, 80, 33, 65, 98, 12, 17, 43, 16, 72, 86, 74, 90, 23, 25, 96, 14, 31, 11, 28, 81, 4, 102, 78, 7, 13, 8, 6, 3, 69, 67, 39], [103, 61, 95, 60, 23, 104, 54, 91, 18, 116, 114, 27, 127, 80, 20, 108, 51, 40, 98, 62, 7, 46, 56, 78, 123, 107, 121, 122, 55, 113, 57, 12, 97, 53, 117, 109, 59, 58, 42, 75, 125, 124, 112, 86, 74, 126, 47, 41, 45, 63, 22, 115, 25, 65, 120, 49, 52, 69, 119, 50, 101, 28, 31, 44, 43, 118, 48, 99, 110, 105, 81, 39, 87, 38, 32, 24, 111, 102, 106, 93, 37, 85, 36, 79, 88, 100, 19, 34, 35, 21, 33, 16, 29, 84, 9, 83, 30, 90, 13, 82, 5, 96, 92, 8, 66, 94, 15, 89, 6, 70, 2, 72, 67, 26, 77, 76, 3, 17, 73, 1, 68, 71, 14, 4, 11, 0, 10, 64], [103, 61, 95, 113, 23, 104, 27, 91, 49, 42, 108, 60, 78, 114, 80, 18, 12, 74, 109, 7, 54, 127, 57, 21, 20, 34, 52, 58, 53, 45, 105, 112, 101, 2, 6, 56, 47, 86, 59, 69, 121, 117, 1, 70, 51, 39, 107, 126, 46, 43, 3, 64, 31, 119, 85, 116, 19, 62, 125, 32, 88, 72, 122, 67, 97, 13, 40, 92, 102, 65, 123, 8, 41, 120, 111, 48, 106, 124, 55, 118, 63, 44, 4, 115, 99, 25, 96, 66, 0, 22, 28, 9, 11, 38, 83, 50, 17, 110, 30, 100, 84, 35, 89, 68, 73, 90, 36, 81, 98, 33, 77, 15, 71, 37, 93, 24, 94, 29, 26, 79, 87, 75, 5, 14, 16, 10, 82, 76], [103, 61, 95, 74, 18, 86, 69, 27, 12, 78, 23, 48, 60, 64, 7, 3, 20, 54, 66, 113, 65, 22, 80, 123, 0, 127, 6, 21, 98, 33, 47, 72, 114, 38, 82, 68, 121, 73, 85, 35, 49, 75, 29, 81, 43, 94, 62, 16, 31, 88, 84, 19, 104, 53, 106, 34, 26, 32, 55, 56, 90, 119, 87, 44, 124, 14, 118, 101, 83, 24, 15, 46, 13, 2, 97, 17, 10, 93, 28, 25, 77, 92, 108, 51, 96, 100, 58, 111, 5, 99, 50, 89, 30, 11, 36, 116, 110, 40, 115, 57, 102, 67, 79, 42, 105, 45, 52, 107, 76, 126, 70, 122, 63, 120, 1, 59, 117, 8, 125, 71, 112, 41, 37, 109, 91, 9, 4, 39], [103, 61, 60, 55, 23, 95, 54, 91, 27, 18, 56, 78, 20, 113, 7, 53, 51, 114, 98, 123, 121, 12, 116, 107, 104, 69, 41, 58, 46, 63, 108, 59, 40, 109, 126, 62, 80, 49, 74, 65, 97, 124, 22, 117, 52, 45, 112, 86, 57, 75, 127, 115, 47, 105, 42, 122, 38, 25, 125, 44, 101, 37, 110, 119, 34, 118, 43, 50, 106, 6, 66, 32, 102, 48, 39, 87, 111, 31, 71, 93, 83, 120, 33, 35, 28, 29, 4, 79, 36, 21, 17, 96, 100, 68, 72, 99, 30, 16, 67, 84, 19, 15, 76, 92, 88, 89, 3, 13, 24, 14, 90, 5, 85, 82, 73, 77, 81, 9, 0, 1, 94, 8, 26, 70, 10, 11, 2, 64], [121, 114, 39, 56, 120, 33, 93, 116, 19, 101, 29, 57, 113, 126, 63, 77, 49, 103, 109, 51, 110, 127, 54, 122, 111, 53, 119, 58, 50, 66, 105, 23, 59, 125, 117, 115, 43, 55, 124, 52, 46, 48, 61, 118, 41, 112, 123, 22, 108, 62, 45, 60, 107, 8, 65, 47, 25, 89, 42, 106, 24, 74, 44, 1, 104, 98, 37, 15, 69, 102, 40, 38, 32, 72, 26, 99, 3, 35, 90, 97, 67, 28, 96, 100, 91, 36, 88, 34, 79, 94, 7, 17, 2, 16, 83, 92, 18, 86, 95, 81, 10, 30, 31, 6, 84, 13, 85, 71, 87, 27, 20, 5, 21, 76, 9, 11, 68, 78, 64, 70, 80, 82, 0, 75, 4, 73, 14, 12], [56, 39, 114, 33, 88, 85, 16, 78, 18, 26, 120, 76, 71, 29, 91, 50, 11, 31, 20, 121, 116, 100, 115, 94, 119, 58, 84, 80, 97, 6, 9, 54, 24, 57, 37, 19, 51, 22, 15, 124, 110, 113, 75, 81, 79, 126, 99, 95, 83, 87, 111, 108, 13, 90, 92, 25, 27, 61, 34, 44, 7, 122, 63, 125, 52, 28, 96, 89, 127, 53, 30, 72, 60, 77, 82, 93, 41, 74, 21, 48, 86, 123, 55, 117, 59, 49, 10, 40, 98, 14, 17, 43, 105, 101, 23, 62, 68, 36, 32, 12, 109, 102, 35, 47, 107, 118, 45, 42, 106, 112, 104, 38, 46, 69, 1, 73, 70, 8, 65, 3, 66, 67, 103, 5, 4, 2, 64, 0], [39, 56, 114, 0, 64, 9, 121, 120, 68, 6, 11, 67, 78, 16, 33, 65, 88, 66, 71, 4, 19, 29, 3, 85, 1, 76, 77, 2, 18, 22, 73, 12, 70, 86, 75, 20, 5, 87, 83, 8, 81, 7, 10, 72, 90, 99, 110, 31, 27, 116, 13, 14, 91, 69, 108, 17, 21, 41, 23, 80, 98, 58, 26, 25, 113, 79, 96, 74, 61, 35, 84, 52, 54, 95, 119, 82, 94, 107, 89, 103, 125, 15, 28, 102, 30, 57, 92, 24, 118, 34, 59, 32, 117, 49, 100, 44, 53, 115, 93, 63, 36, 42, 127, 97, 105, 123, 101, 104, 62, 40, 38, 50, 124, 37, 51, 43, 47, 106, 60, 48, 46, 109, 55, 111, 122, 45, 126, 112], [114, 39, 56, 120, 116, 29, 57, 117, 126, 93, 54, 119, 19, 113, 51, 111, 122, 49, 110, 58, 63, 50, 127, 101, 53, 23, 125, 109, 33, 55, 60, 59, 62, 118, 123, 61, 115, 52, 121, 46, 48, 47, 124, 43, 112, 45, 103, 107, 108, 22, 44, 42, 105, 41, 106, 37, 104, 27, 97, 38, 40, 86, 98, 83, 91, 26, 102, 89, 31, 88, 99, 36, 35, 25, 77, 15, 100, 34, 20, 87, 95, 84, 24, 32, 96, 90, 17, 30, 85, 18, 81, 94, 79, 28, 13, 78, 92, 16, 71, 6, 69, 11, 21, 10, 80, 74, 7, 82, 5, 65, 72, 9, 14, 1, 76, 8, 12, 75, 68, 67, 66, 70, 73, 2, 3, 4, 64, 0], [46, 102, 49, 110, 26, 84, 16, 53, 29, 123, 60, 126, 98, 10, 119, 90, 25, 91, 44, 56, 124, 113, 62, 41, 112, 86, 63, 127, 42, 92, 59, 57, 33, 47, 118, 107, 31, 54, 125, 6, 87, 115, 80, 114, 66, 104, 95, 105, 20, 99, 111, 120, 55, 61, 43, 52, 24, 58, 116, 36, 108, 117, 45, 18, 7, 122, 93, 39, 51, 48, 109, 35, 97, 121, 37, 50, 30, 32, 106, 100, 64, 103, 27, 2, 13, 74, 40, 1, 22, 101, 81, 88, 72, 28, 94, 21, 78, 70, 96, 34, 69, 0, 83, 38, 85, 15, 17, 89, 11, 12, 3, 23, 68, 73, 19, 75, 67, 76, 5, 14, 71, 79, 65, 4, 9, 8, 82, 77], [46, 102, 110, 29, 26, 60, 84, 16, 86, 51, 69, 18, 100, 123, 109, 67, 121, 36, 112, 49, 113, 1, 63, 97, 90, 11, 13, 50, 31, 53, 33, 108, 7, 61, 115, 30, 47, 107, 28, 12, 105, 119, 37, 59, 106, 114, 42, 120, 99, 45, 52, 98, 87, 124, 104, 89, 125, 92, 41, 116, 55, 111, 39, 103, 10, 57, 58, 127, 117, 62, 56, 35, 44, 54, 122, 40, 126, 64, 6, 38, 101, 85, 76, 93, 27, 81, 14, 32, 24, 25, 4, 43, 95, 48, 34, 73, 118, 96, 70, 3, 66, 91, 9, 17, 19, 83, 72, 88, 94, 78, 22, 79, 20, 80, 2, 74, 68, 21, 71, 15, 23, 5, 75, 82, 8, 65, 77, 0], [46, 102, 110, 26, 84, 18, 29, 13, 72, 16, 67, 116, 10, 109, 68, 107, 121, 113, 59, 0, 47, 6, 90, 112, 60, 98, 115, 48, 24, 87, 22, 123, 120, 8, 12, 11, 49, 127, 30, 73, 23, 106, 126, 41, 119, 34, 61, 40, 37, 35, 66, 97, 124, 108, 100, 114, 42, 93, 25, 85, 103, 53, 33, 81, 80, 89, 63, 55, 31, 104, 105, 74, 50, 44, 117, 95, 45, 3, 111, 21, 57, 36, 43, 62, 92, 58, 125, 88, 86, 122, 20, 54, 83, 96, 70, 39, 7, 75, 91, 28, 52, 118, 14, 27, 101, 99, 51, 56, 32, 19, 9, 2, 69, 17, 94, 82, 64, 79, 5, 77, 65, 1, 76, 4, 71, 15, 78, 38], [46, 102, 18, 110, 13, 29, 84, 26, 16, 10, 72, 6, 68, 93, 25, 87, 1, 73, 90, 60, 0, 2, 22, 91, 78, 7, 14, 49, 9, 88, 15, 82, 79, 36, 59, 33, 112, 20, 17, 30, 123, 109, 27, 31, 19, 116, 67, 65, 8, 71, 92, 96, 85, 3, 80, 61, 76, 81, 24, 11, 77, 98, 113, 83, 50, 5, 107, 28, 106, 23, 118, 54, 101, 86, 37, 75, 34, 12, 66, 89, 63, 74, 41, 56, 48, 4, 108, 32, 35, 115, 99, 94, 21, 100, 119, 62, 57, 114, 70, 95, 52, 58, 124, 69, 39, 117, 120, 44, 121, 127, 105, 43, 55, 64, 126, 104, 40, 53, 103, 47, 42, 122, 125, 97, 51, 45, 111, 38]], "model.layers.28.self_attn.k_proj": [[120, 104, 98, 23, 85, 0, 29, 11, 17, 16, 27, 61, 58, 30, 19, 95, 110, 73, 78, 125, 66, 114, 91, 108, 42, 50, 115, 51, 62, 52, 68, 121, 127, 57, 47, 124, 60, 45, 59, 53, 122, 55, 49, 70, 43, 67, 112, 119, 109, 65, 117, 54, 123, 116, 48, 102, 113, 56, 44, 8, 118, 111, 63, 38, 4, 46, 79, 105, 107, 89, 106, 5, 12, 126, 41, 25, 77, 2, 39, 36, 86, 88, 13, 7, 82, 31, 92, 35, 103, 101, 100, 37, 10, 90, 33, 28, 71, 34, 84, 26, 24, 1, 32, 22, 94, 20, 99, 96, 9, 72, 6, 15, 97, 83, 18, 81, 3, 93, 80, 14, 21, 74, 75, 69, 76, 64, 87, 40], [125, 104, 64, 87, 98, 14, 70, 74, 17, 12, 82, 0, 21, 65, 8, 68, 66, 124, 108, 58, 69, 3, 1, 90, 4, 114, 47, 79, 28, 49, 2, 54, 19, 127, 73, 9, 75, 46, 50, 123, 61, 93, 52, 25, 16, 77, 103, 111, 31, 67, 41, 113, 53, 106, 92, 20, 109, 89, 45, 122, 105, 27, 97, 107, 71, 94, 60, 33, 13, 55, 59, 121, 120, 86, 26, 11, 63, 99, 29, 95, 91, 51, 101, 119, 5, 62, 118, 88, 83, 112, 7, 102, 24, 80, 22, 115, 56, 38, 42, 44, 48, 37, 32, 116, 110, 57, 15, 36, 84, 100, 23, 117, 43, 39, 76, 30, 35, 126, 85, 96, 6, 72, 34, 18, 10, 78, 81, 40], [106, 114, 100, 94, 118, 87, 68, 0, 99, 71, 77, 74, 20, 82, 15, 17, 50, 70, 52, 119, 42, 27, 47, 44, 115, 62, 75, 89, 56, 66, 123, 1, 122, 109, 102, 120, 76, 2, 110, 53, 57, 45, 48, 9, 65, 41, 43, 107, 121, 39, 35, 127, 95, 73, 29, 54, 59, 91, 31, 58, 124, 61, 72, 69, 51, 22, 113, 49, 28, 12, 111, 93, 85, 63, 97, 60, 3, 125, 46, 40, 116, 96, 30, 98, 126, 108, 117, 33, 55, 34, 32, 37, 5, 112, 101, 11, 92, 64, 19, 80, 104, 78, 103, 90, 14, 7, 83, 21, 86, 24, 88, 8, 16, 67, 105, 38, 26, 4, 81, 25, 79, 18, 6, 10, 84, 36, 23, 13], [105, 126, 56, 34, 95, 86, 92, 89, 115, 84, 82, 79, 57, 119, 51, 62, 110, 63, 87, 54, 59, 80, 120, 112, 58, 121, 113, 127, 117, 55, 124, 116, 118, 109, 48, 60, 123, 61, 122, 46, 49, 114, 52, 53, 47, 125, 111, 45, 32, 104, 44, 107, 99, 50, 108, 21, 17, 91, 39, 101, 43, 26, 106, 42, 103, 96, 11, 36, 77, 27, 28, 102, 30, 13, 38, 40, 97, 33, 37, 73, 29, 8, 22, 41, 76, 35, 10, 19, 93, 75, 100, 98, 14, 94, 70, 85, 18, 88, 24, 12, 20, 31, 16, 23, 90, 25, 83, 4, 5, 3, 78, 81, 0, 72, 9, 65, 74, 7, 71, 68, 1, 6, 66, 15, 69, 67, 64, 2], [123, 39, 88, 21, 27, 64, 76, 15, 18, 30, 1, 68, 98, 71, 122, 95, 70, 125, 9, 47, 63, 121, 66, 46, 19, 57, 60, 86, 116, 110, 2, 49, 119, 120, 118, 33, 113, 3, 45, 52, 117, 44, 112, 48, 51, 124, 40, 10, 50, 54, 59, 114, 20, 69, 111, 31, 102, 37, 126, 8, 108, 61, 56, 106, 109, 115, 55, 34, 43, 87, 53, 62, 13, 42, 14, 41, 127, 58, 104, 29, 107, 105, 32, 101, 100, 11, 36, 75, 93, 92, 23, 90, 28, 35, 99, 22, 26, 74, 89, 97, 25, 96, 84, 65, 77, 17, 80, 81, 0, 16, 94, 38, 82, 78, 79, 103, 72, 4, 5, 6, 7, 83, 67, 73, 91, 24, 12, 85], [61, 39, 31, 23, 91, 18, 64, 74, 80, 78, 86, 62, 69, 49, 56, 12, 54, 3, 51, 112, 66, 7, 55, 113, 110, 124, 121, 109, 50, 20, 52, 117, 114, 58, 0, 34, 122, 59, 60, 123, 115, 42, 111, 126, 118, 107, 63, 53, 127, 116, 44, 41, 108, 45, 79, 47, 57, 35, 25, 46, 98, 65, 106, 6, 48, 88, 119, 75, 1, 43, 40, 72, 120, 33, 125, 21, 32, 105, 67, 68, 19, 37, 100, 36, 38, 92, 70, 73, 17, 2, 102, 96, 97, 81, 99, 27, 90, 94, 101, 103, 4, 11, 84, 104, 77, 93, 29, 9, 28, 15, 24, 13, 22, 89, 26, 83, 30, 85, 16, 5, 8, 10, 76, 14, 71, 95, 82, 87], [56, 103, 64, 114, 93, 121, 97, 1, 50, 19, 9, 68, 6, 22, 120, 67, 116, 11, 23, 113, 78, 66, 77, 16, 46, 88, 57, 58, 110, 54, 3, 69, 76, 71, 51, 122, 105, 118, 126, 43, 95, 8, 62, 61, 117, 24, 55, 59, 47, 49, 53, 123, 125, 42, 119, 63, 60, 44, 48, 115, 85, 124, 12, 26, 37, 52, 109, 18, 112, 2, 45, 74, 127, 111, 108, 41, 89, 106, 21, 107, 40, 81, 38, 65, 101, 91, 14, 15, 35, 36, 84, 104, 0, 99, 102, 92, 4, 34, 100, 94, 30, 28, 96, 25, 32, 17, 20, 31, 98, 72, 80, 10, 75, 90, 70, 82, 27, 79, 13, 5, 87, 73, 7, 86, 29, 39, 33, 83], [110, 38, 46, 29, 84, 26, 13, 16, 18, 6, 2, 0, 112, 10, 25, 121, 123, 68, 11, 113, 73, 86, 1, 47, 7, 97, 72, 63, 60, 109, 115, 34, 119, 53, 114, 30, 59, 55, 61, 124, 32, 41, 103, 107, 101, 88, 49, 50, 116, 42, 87, 51, 104, 62, 37, 57, 111, 108, 58, 127, 120, 45, 56, 122, 118, 40, 44, 105, 39, 125, 117, 54, 52, 64, 92, 43, 106, 95, 15, 76, 48, 14, 69, 36, 96, 91, 4, 35, 99, 31, 100, 126, 98, 33, 75, 22, 67, 23, 5, 94, 81, 19, 24, 85, 17, 78, 12, 27, 83, 8, 28, 3, 89, 21, 65, 9, 79, 71, 93, 82, 77, 74, 80, 66, 70, 90, 20, 102]], "model.layers.28.self_attn.qk_proj": [[123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 118, 103, 82, 18, 91, 64, 0, 29, 95, 58, 39, 40, 50, 85, 41, 27, 10, 70, 20, 76, 74, 105, 84, 30, 12, 55, 62, 98, 93, 21, 81, 51, 68, 115, 80, 79, 13, 77, 122, 124, 44, 90, 6, 60, 15, 4, 48, 112, 59, 113, 54, 31, 116, 78, 49, 57, 47, 53, 109, 66, 65, 17, 127, 16, 2, 22, 14, 63, 28, 102, 24, 11, 34, 1, 83, 9, 86, 43, 73, 52, 19, 38, 25, 7, 88, 108, 36, 8, 119, 94, 89, 26, 33, 107, 3, 71, 117, 75, 5, 45, 67, 111, 99, 97, 35, 101, 100, 72, 69, 32, 37, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 91, 0, 82, 64, 18, 40, 70, 95, 50, 58, 29, 30, 47, 10, 62, 39, 27, 20, 85, 105, 4, 41, 76, 12, 93, 51, 98, 81, 55, 74, 21, 60, 84, 13, 49, 77, 44, 115, 63, 53, 80, 65, 124, 66, 116, 34, 112, 113, 22, 54, 79, 102, 17, 16, 2, 6, 68, 11, 14, 78, 90, 15, 122, 28, 1, 9, 109, 59, 127, 48, 8, 108, 52, 31, 38, 57, 43, 71, 24, 86, 88, 83, 73, 33, 117, 7, 107, 111, 119, 19, 94, 45, 5, 36, 100, 3, 89, 75, 69, 26, 67, 35, 25, 37, 97, 99, 72, 101, 92, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 0, 118, 64, 18, 103, 82, 91, 70, 39, 95, 27, 58, 41, 40, 98, 105, 29, 12, 21, 84, 76, 10, 85, 112, 30, 50, 74, 93, 55, 20, 115, 44, 53, 4, 62, 63, 80, 34, 81, 77, 124, 60, 15, 68, 79, 113, 22, 13, 66, 2, 90, 57, 51, 49, 78, 47, 17, 54, 102, 65, 122, 16, 14, 1, 116, 8, 48, 59, 28, 31, 119, 127, 86, 109, 19, 6, 43, 71, 36, 88, 9, 107, 38, 108, 52, 7, 25, 11, 117, 111, 24, 94, 67, 89, 26, 75, 3, 73, 83, 33, 5, 69, 99, 101, 100, 97, 37, 45, 35, 92, 96, 32, 72], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 118, 64, 0, 18, 91, 82, 70, 40, 39, 41, 74, 76, 58, 95, 53, 105, 12, 27, 50, 29, 10, 98, 21, 84, 112, 122, 62, 4, 63, 81, 30, 124, 55, 2, 77, 14, 78, 85, 34, 93, 80, 1, 20, 44, 68, 66, 113, 16, 17, 115, 60, 22, 13, 15, 54, 6, 79, 8, 90, 49, 57, 51, 59, 65, 47, 25, 116, 31, 7, 86, 28, 11, 48, 36, 33, 19, 67, 9, 83, 119, 71, 102, 52, 38, 88, 94, 73, 107, 109, 3, 89, 69, 101, 127, 117, 108, 24, 75, 43, 45, 5, 26, 97, 111, 35, 99, 100, 37, 92, 72, 32, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 103, 118, 18, 0, 82, 64, 40, 58, 41, 70, 91, 50, 53, 39, 29, 105, 55, 12, 76, 95, 98, 112, 21, 27, 80, 74, 10, 84, 20, 85, 30, 6, 78, 77, 113, 4, 81, 62, 115, 15, 122, 60, 124, 68, 66, 54, 48, 14, 2, 57, 44, 59, 90, 86, 116, 16, 47, 79, 65, 17, 8, 28, 63, 93, 49, 13, 51, 127, 1, 25, 31, 11, 34, 19, 22, 7, 107, 117, 9, 109, 24, 52, 108, 102, 83, 36, 73, 119, 67, 88, 3, 38, 71, 94, 111, 75, 5, 26, 69, 89, 33, 101, 43, 45, 97, 100, 35, 99, 92, 72, 37, 96, 32], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 103, 87, 18, 82, 118, 64, 0, 50, 29, 58, 40, 91, 95, 21, 41, 74, 27, 44, 112, 62, 20, 30, 105, 98, 85, 81, 10, 55, 53, 68, 113, 84, 12, 76, 6, 115, 70, 34, 54, 39, 49, 48, 4, 15, 77, 31, 80, 124, 2, 65, 66, 93, 47, 14, 13, 78, 60, 122, 79, 1, 8, 22, 63, 17, 16, 116, 19, 86, 28, 57, 127, 109, 24, 90, 119, 25, 108, 5, 7, 59, 51, 71, 11, 94, 83, 43, 117, 52, 75, 89, 33, 9, 102, 3, 88, 97, 73, 38, 100, 35, 111, 67, 69, 99, 45, 101, 107, 26, 37, 36, 96, 92, 32, 72], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 41, 118, 82, 40, 50, 64, 58, 18, 91, 0, 29, 98, 105, 95, 27, 6, 39, 76, 44, 30, 21, 113, 85, 53, 112, 55, 20, 12, 81, 90, 10, 115, 15, 84, 4, 93, 13, 80, 74, 16, 109, 34, 47, 77, 49, 14, 68, 17, 124, 60, 54, 79, 48, 127, 51, 63, 66, 62, 59, 70, 78, 1, 57, 22, 122, 86, 25, 31, 116, 2, 24, 19, 36, 65, 8, 28, 7, 89, 111, 119, 9, 43, 83, 108, 73, 94, 88, 52, 117, 71, 11, 38, 67, 45, 97, 102, 26, 69, 3, 101, 99, 100, 75, 107, 37, 35, 72, 33, 5, 32, 92, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 103, 118, 18, 82, 91, 0, 29, 64, 39, 41, 58, 40, 105, 6, 95, 44, 27, 98, 21, 50, 112, 55, 30, 76, 34, 54, 12, 85, 47, 20, 74, 122, 4, 53, 81, 84, 77, 79, 115, 51, 10, 113, 48, 93, 90, 31, 80, 16, 49, 14, 15, 102, 17, 116, 68, 65, 124, 59, 2, 22, 60, 13, 1, 57, 66, 43, 127, 63, 109, 62, 119, 89, 78, 38, 25, 70, 28, 108, 24, 88, 9, 86, 73, 83, 45, 52, 107, 36, 11, 19, 94, 71, 67, 111, 7, 117, 33, 26, 100, 37, 5, 72, 97, 99, 69, 75, 3, 8, 101, 92, 32, 35, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 87, 23, 104, 121, 103, 18, 82, 29, 118, 64, 91, 27, 0, 44, 58, 95, 6, 40, 41, 39, 21, 50, 76, 20, 112, 30, 74, 113, 55, 47, 10, 105, 84, 13, 63, 85, 98, 68, 124, 77, 122, 80, 12, 93, 51, 16, 34, 78, 109, 54, 48, 90, 115, 4, 116, 79, 62, 81, 17, 53, 15, 66, 49, 14, 102, 65, 43, 119, 22, 31, 28, 60, 59, 57, 127, 83, 88, 2, 1, 24, 86, 89, 72, 70, 52, 73, 19, 7, 108, 25, 107, 9, 117, 38, 94, 67, 71, 33, 111, 36, 45, 75, 99, 26, 5, 100, 11, 3, 101, 35, 97, 8, 32, 69, 92, 37, 96], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 23, 64, 58, 118, 103, 0, 82, 18, 91, 6, 29, 41, 40, 30, 27, 76, 48, 4, 39, 50, 98, 85, 63, 55, 12, 74, 113, 10, 122, 44, 20, 95, 112, 93, 21, 51, 124, 105, 47, 68, 2, 62, 34, 22, 53, 84, 54, 70, 66, 16, 115, 81, 13, 108, 49, 60, 57, 43, 79, 77, 90, 14, 80, 17, 116, 65, 15, 102, 78, 1, 59, 86, 109, 127, 119, 72, 24, 28, 31, 11, 73, 7, 9, 38, 25, 71, 111, 19, 33, 83, 94, 3, 107, 36, 88, 75, 52, 89, 26, 5, 67, 8, 45, 117, 69, 101, 100, 97, 35, 99, 32, 37, 92, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 118, 82, 18, 103, 0, 64, 91, 10, 40, 58, 70, 27, 74, 29, 95, 50, 76, 98, 6, 85, 39, 41, 30, 21, 4, 20, 55, 77, 84, 66, 53, 13, 16, 12, 90, 79, 81, 54, 105, 68, 78, 72, 93, 15, 1, 80, 124, 122, 2, 62, 48, 112, 57, 17, 47, 60, 51, 115, 34, 113, 71, 59, 94, 63, 49, 7, 31, 65, 28, 14, 24, 19, 86, 22, 25, 73, 88, 109, 83, 11, 44, 89, 116, 3, 102, 52, 9, 127, 108, 119, 38, 5, 67, 33, 43, 107, 75, 36, 26, 117, 45, 111, 69, 35, 97, 100, 101, 99, 92, 8, 37, 32, 96], [123, 56, 125, 61, 120, 114, 110, 46, 126, 106, 104, 42, 87, 23, 121, 82, 18, 103, 118, 64, 29, 40, 91, 0, 98, 95, 70, 76, 27, 50, 84, 21, 39, 10, 20, 74, 44, 58, 30, 4, 79, 41, 12, 85, 55, 105, 34, 13, 16, 93, 78, 77, 80, 122, 68, 66, 90, 59, 113, 54, 17, 51, 47, 15, 62, 115, 112, 6, 31, 81, 2, 49, 14, 22, 60, 53, 65, 72, 57, 1, 25, 102, 24, 124, 86, 116, 28, 48, 75, 83, 63, 73, 109, 127, 7, 71, 36, 9, 3, 19, 119, 94, 89, 67, 52, 43, 11, 107, 88, 33, 45, 38, 69, 108, 111, 35, 26, 5, 101, 99, 117, 100, 32, 37, 97, 8, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 87, 42, 103, 121, 23, 118, 58, 18, 91, 40, 82, 64, 0, 50, 29, 39, 95, 44, 70, 51, 47, 27, 41, 20, 98, 12, 105, 54, 21, 113, 30, 115, 55, 85, 93, 60, 116, 63, 112, 84, 102, 76, 74, 34, 57, 68, 59, 62, 53, 28, 17, 4, 31, 48, 81, 90, 72, 49, 109, 124, 122, 65, 86, 80, 16, 14, 77, 78, 22, 13, 10, 79, 108, 2, 66, 24, 15, 127, 6, 43, 1, 9, 88, 119, 75, 73, 45, 52, 19, 89, 38, 7, 111, 11, 36, 71, 94, 107, 83, 25, 26, 117, 3, 97, 35, 5, 67, 99, 37, 69, 100, 101, 33, 92, 32, 96, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 87, 23, 64, 103, 0, 118, 91, 18, 82, 70, 58, 39, 54, 95, 76, 29, 40, 21, 41, 74, 50, 12, 27, 93, 30, 122, 68, 10, 20, 124, 85, 63, 57, 55, 105, 115, 84, 4, 98, 2, 44, 22, 34, 13, 81, 77, 116, 113, 1, 60, 112, 16, 78, 90, 79, 109, 49, 51, 65, 66, 47, 15, 14, 62, 17, 6, 48, 59, 80, 53, 43, 102, 119, 31, 127, 72, 24, 108, 111, 28, 88, 86, 71, 11, 38, 19, 9, 107, 52, 7, 36, 75, 73, 83, 25, 33, 94, 97, 99, 117, 5, 89, 3, 35, 67, 45, 69, 100, 101, 26, 37, 8, 92, 32, 96], [123, 61, 56, 125, 120, 110, 114, 46, 126, 106, 42, 104, 23, 121, 87, 18, 64, 0, 91, 103, 118, 82, 40, 70, 41, 74, 58, 76, 29, 98, 85, 95, 12, 50, 10, 105, 84, 21, 39, 30, 4, 2, 81, 20, 68, 27, 63, 90, 16, 55, 66, 124, 6, 115, 13, 15, 54, 77, 112, 53, 47, 44, 79, 22, 113, 93, 14, 122, 34, 78, 80, 17, 62, 116, 109, 65, 51, 57, 48, 59, 60, 1, 31, 9, 7, 86, 119, 75, 72, 127, 49, 28, 33, 108, 11, 88, 52, 24, 94, 67, 38, 89, 3, 25, 19, 5, 111, 73, 71, 36, 83, 107, 69, 43, 97, 35, 117, 45, 100, 26, 99, 102, 8, 101, 37, 96, 92, 32], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 18, 118, 91, 82, 64, 29, 50, 58, 0, 95, 10, 70, 40, 27, 74, 105, 55, 93, 6, 30, 76, 84, 39, 85, 13, 12, 20, 98, 68, 21, 41, 81, 90, 122, 115, 77, 47, 16, 112, 80, 78, 15, 4, 44, 34, 66, 54, 1, 62, 113, 109, 14, 2, 49, 60, 22, 79, 17, 59, 65, 119, 31, 63, 19, 28, 53, 48, 124, 86, 7, 75, 71, 9, 57, 24, 116, 73, 127, 102, 25, 3, 88, 33, 89, 43, 108, 83, 52, 72, 11, 51, 67, 8, 94, 111, 107, 38, 100, 45, 69, 5, 97, 26, 37, 35, 117, 36, 32, 99, 101, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 64, 18, 118, 0, 82, 58, 95, 29, 40, 91, 39, 30, 41, 27, 50, 98, 20, 93, 21, 74, 85, 112, 12, 105, 6, 84, 115, 90, 55, 10, 16, 76, 44, 81, 54, 68, 108, 122, 4, 14, 60, 1, 53, 34, 80, 62, 22, 13, 113, 59, 77, 17, 66, 109, 31, 86, 28, 47, 57, 48, 116, 49, 78, 15, 2, 70, 63, 51, 127, 102, 79, 124, 9, 88, 38, 43, 119, 19, 24, 71, 45, 52, 65, 73, 75, 25, 89, 107, 7, 26, 83, 36, 8, 94, 33, 35, 11, 99, 111, 100, 67, 3, 97, 72, 117, 5, 101, 69, 92, 37, 32, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 64, 103, 82, 18, 118, 0, 91, 6, 39, 29, 40, 27, 95, 74, 76, 30, 50, 85, 58, 10, 112, 1, 12, 34, 20, 41, 54, 44, 55, 21, 84, 13, 93, 105, 81, 98, 2, 68, 14, 47, 113, 80, 122, 63, 16, 53, 4, 77, 90, 51, 115, 119, 60, 78, 62, 49, 102, 15, 17, 79, 22, 124, 66, 48, 70, 31, 43, 116, 86, 59, 65, 9, 109, 71, 57, 75, 19, 108, 8, 33, 94, 88, 28, 25, 26, 127, 24, 38, 36, 89, 3, 52, 67, 83, 111, 107, 7, 69, 99, 117, 73, 101, 97, 11, 5, 100, 35, 45, 37, 32, 72, 96, 92], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 103, 87, 64, 0, 118, 91, 40, 82, 6, 18, 58, 41, 29, 27, 112, 76, 39, 50, 74, 98, 49, 10, 95, 93, 85, 12, 2, 122, 113, 20, 60, 84, 4, 30, 105, 124, 44, 63, 21, 34, 81, 62, 48, 115, 14, 68, 13, 77, 55, 80, 109, 116, 119, 53, 47, 54, 17, 51, 1, 8, 15, 59, 127, 43, 16, 66, 78, 70, 22, 102, 108, 31, 90, 79, 75, 57, 33, 65, 86, 36, 19, 38, 28, 45, 88, 52, 24, 9, 89, 71, 94, 111, 73, 3, 83, 11, 25, 99, 7, 107, 69, 67, 101, 97, 35, 100, 26, 5, 117, 37, 96, 92, 32, 72], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 103, 121, 23, 87, 118, 0, 91, 82, 40, 6, 64, 18, 58, 29, 95, 10, 27, 93, 30, 41, 74, 39, 4, 55, 50, 98, 122, 81, 124, 12, 76, 66, 105, 85, 20, 112, 21, 62, 34, 54, 8, 68, 84, 48, 44, 16, 53, 115, 116, 90, 77, 113, 17, 14, 70, 63, 13, 78, 2, 1, 60, 80, 79, 15, 47, 28, 59, 109, 49, 102, 57, 75, 108, 119, 22, 127, 83, 71, 51, 31, 86, 73, 65, 33, 52, 9, 45, 7, 38, 35, 3, 19, 25, 88, 43, 111, 94, 69, 36, 97, 89, 26, 107, 24, 117, 11, 100, 99, 67, 5, 101, 92, 72, 37, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 118, 103, 18, 82, 40, 29, 64, 0, 91, 27, 44, 39, 58, 95, 21, 105, 50, 55, 30, 41, 6, 112, 74, 12, 10, 76, 113, 85, 122, 115, 20, 51, 98, 116, 47, 93, 54, 62, 34, 63, 70, 84, 1, 68, 109, 60, 17, 13, 8, 16, 80, 79, 78, 4, 90, 119, 124, 81, 2, 49, 108, 127, 102, 48, 15, 77, 53, 31, 86, 22, 43, 66, 59, 107, 52, 57, 14, 65, 28, 25, 75, 88, 7, 24, 19, 9, 94, 3, 71, 38, 73, 33, 67, 36, 83, 100, 5, 89, 97, 26, 11, 117, 111, 101, 69, 35, 37, 32, 99, 96, 45, 92, 72], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 42, 104, 121, 87, 64, 23, 0, 118, 103, 91, 18, 40, 82, 70, 58, 41, 27, 29, 39, 50, 12, 76, 112, 105, 95, 68, 20, 21, 10, 34, 30, 124, 54, 44, 85, 122, 74, 62, 78, 93, 55, 115, 2, 6, 81, 98, 113, 63, 84, 116, 13, 1, 51, 53, 60, 15, 119, 65, 66, 43, 17, 49, 57, 16, 4, 86, 79, 102, 77, 80, 47, 22, 8, 48, 59, 14, 52, 109, 31, 127, 90, 38, 75, 7, 108, 28, 71, 19, 88, 67, 25, 107, 33, 94, 9, 24, 5, 36, 89, 73, 97, 11, 26, 3, 45, 83, 92, 69, 100, 117, 37, 111, 101, 99, 35, 72, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 121, 104, 23, 87, 103, 64, 118, 18, 0, 91, 82, 39, 58, 70, 40, 95, 98, 29, 27, 50, 62, 30, 105, 93, 112, 116, 12, 44, 41, 55, 21, 10, 115, 122, 81, 74, 20, 63, 85, 68, 113, 76, 57, 78, 84, 54, 34, 2, 16, 80, 102, 13, 4, 6, 31, 66, 15, 79, 124, 109, 86, 1, 51, 90, 17, 60, 119, 59, 53, 48, 49, 47, 14, 22, 28, 65, 88, 108, 77, 127, 38, 7, 36, 73, 9, 75, 107, 25, 3, 8, 52, 24, 43, 71, 89, 26, 33, 67, 11, 83, 111, 94, 97, 45, 69, 19, 117, 5, 100, 72, 99, 35, 37, 92, 101, 32, 96], [123, 56, 61, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 18, 103, 91, 118, 82, 58, 0, 64, 95, 70, 29, 74, 40, 122, 105, 21, 55, 27, 85, 39, 10, 30, 12, 50, 115, 34, 98, 20, 93, 41, 76, 13, 4, 54, 81, 80, 68, 15, 84, 62, 78, 113, 90, 63, 44, 51, 59, 16, 112, 57, 14, 124, 28, 48, 116, 22, 86, 79, 1, 77, 17, 47, 6, 66, 108, 60, 102, 119, 49, 2, 71, 25, 109, 31, 94, 53, 127, 24, 9, 19, 75, 111, 11, 43, 52, 33, 7, 83, 8, 36, 65, 88, 38, 107, 72, 97, 73, 3, 5, 89, 100, 69, 35, 26, 67, 99, 45, 37, 117, 32, 101, 96, 92], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 23, 104, 87, 121, 118, 103, 18, 82, 91, 64, 70, 58, 0, 29, 40, 122, 98, 39, 41, 50, 10, 30, 21, 55, 12, 27, 105, 20, 54, 124, 74, 113, 81, 95, 57, 85, 76, 116, 112, 109, 84, 47, 62, 93, 44, 15, 79, 127, 22, 13, 115, 78, 4, 17, 16, 34, 68, 49, 65, 80, 90, 77, 2, 48, 86, 119, 59, 63, 6, 51, 66, 60, 14, 53, 24, 31, 28, 1, 108, 52, 43, 89, 9, 94, 102, 19, 71, 73, 111, 88, 25, 75, 83, 72, 38, 7, 107, 67, 11, 36, 101, 117, 26, 33, 45, 3, 35, 5, 100, 97, 69, 37, 8, 99, 96, 32, 92], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 87, 104, 23, 121, 118, 91, 103, 0, 18, 64, 82, 58, 29, 39, 95, 30, 44, 51, 50, 27, 41, 40, 70, 116, 54, 122, 10, 47, 98, 34, 20, 12, 113, 84, 119, 112, 105, 21, 124, 93, 76, 115, 81, 109, 102, 60, 74, 85, 68, 53, 49, 63, 62, 57, 16, 4, 59, 31, 14, 66, 48, 77, 55, 86, 90, 13, 43, 79, 6, 1, 65, 127, 15, 78, 80, 17, 108, 33, 22, 28, 36, 2, 19, 24, 38, 9, 52, 72, 107, 73, 88, 71, 26, 45, 11, 89, 3, 97, 25, 94, 111, 67, 7, 100, 75, 117, 69, 83, 35, 92, 99, 101, 37, 32, 5, 8, 96], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 104, 23, 121, 87, 103, 118, 64, 0, 18, 40, 82, 41, 50, 29, 58, 91, 47, 95, 55, 39, 30, 27, 98, 60, 122, 124, 62, 57, 21, 20, 105, 93, 10, 76, 34, 49, 54, 12, 51, 115, 74, 85, 116, 81, 90, 6, 112, 53, 63, 84, 113, 1, 70, 4, 48, 13, 17, 59, 43, 16, 14, 77, 80, 15, 31, 2, 68, 78, 127, 102, 44, 28, 109, 72, 86, 79, 22, 119, 65, 25, 52, 108, 24, 36, 107, 7, 66, 88, 19, 38, 89, 99, 33, 11, 83, 9, 73, 71, 94, 75, 111, 101, 67, 35, 97, 3, 117, 100, 26, 37, 45, 69, 5, 96, 8, 32, 92], [123, 125, 56, 61, 120, 114, 110, 46, 126, 106, 42, 23, 104, 121, 87, 118, 64, 82, 0, 103, 18, 91, 58, 40, 6, 41, 10, 29, 74, 122, 12, 55, 62, 50, 95, 27, 85, 124, 17, 76, 39, 13, 105, 93, 30, 21, 20, 34, 68, 81, 98, 4, 57, 115, 54, 60, 78, 59, 90, 70, 80, 63, 48, 84, 77, 2, 79, 66, 112, 15, 116, 16, 72, 1, 19, 44, 49, 47, 119, 127, 113, 22, 11, 14, 51, 86, 28, 31, 102, 65, 7, 52, 53, 36, 73, 109, 33, 71, 43, 67, 38, 89, 24, 108, 75, 9, 111, 83, 25, 88, 101, 3, 69, 94, 26, 107, 117, 35, 5, 99, 45, 97, 100, 8, 37, 92, 32, 96], [123, 61, 56, 125, 120, 114, 110, 46, 126, 106, 42, 104, 121, 23, 87, 0, 103, 64, 118, 18, 91, 82, 6, 40, 58, 115, 41, 98, 62, 116, 29, 12, 50, 76, 39, 30, 95, 10, 55, 93, 85, 74, 105, 21, 4, 122, 47, 68, 84, 124, 78, 60, 49, 27, 51, 13, 66, 20, 44, 34, 48, 59, 77, 2, 112, 72, 63, 1, 17, 53, 70, 113, 57, 79, 65, 81, 14, 54, 86, 80, 16, 22, 90, 15, 31, 7, 52, 119, 11, 127, 28, 109, 111, 108, 25, 38, 19, 33, 83, 3, 9, 89, 94, 73, 102, 67, 36, 71, 100, 43, 24, 117, 97, 88, 75, 107, 45, 69, 5, 35, 26, 101, 37, 99, 96, 32, 92, 8], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 23, 87, 121, 103, 18, 82, 58, 118, 6, 91, 29, 64, 40, 50, 0, 95, 44, 85, 39, 10, 12, 30, 62, 74, 21, 47, 27, 122, 51, 20, 41, 84, 124, 115, 105, 63, 76, 116, 98, 54, 60, 55, 112, 81, 68, 78, 17, 77, 80, 93, 4, 14, 13, 34, 16, 48, 113, 65, 15, 53, 119, 59, 66, 57, 31, 109, 90, 2, 49, 70, 79, 83, 72, 86, 1, 11, 7, 28, 22, 52, 127, 71, 24, 9, 111, 19, 73, 108, 25, 38, 89, 88, 75, 3, 67, 45, 43, 94, 33, 102, 97, 117, 69, 107, 26, 36, 37, 100, 5, 101, 99, 8, 35, 32, 92, 96], [123, 125, 61, 56, 120, 114, 110, 46, 126, 106, 104, 42, 23, 121, 87, 0, 103, 64, 118, 18, 91, 6, 29, 82, 58, 40, 95, 10, 105, 50, 39, 41, 27, 74, 68, 12, 62, 55, 76, 81, 63, 30, 122, 85, 112, 84, 20, 124, 21, 115, 34, 4, 70, 93, 98, 54, 60, 66, 59, 80, 65, 2, 90, 78, 31, 51, 15, 44, 49, 14, 16, 116, 77, 13, 22, 57, 17, 113, 86, 79, 28, 53, 47, 48, 72, 71, 109, 7, 127, 119, 1, 108, 24, 25, 11, 111, 9, 36, 73, 88, 83, 67, 75, 117, 52, 102, 43, 19, 99, 69, 94, 38, 33, 89, 45, 5, 3, 97, 100, 35, 26, 107, 8, 37, 101, 92, 96, 32], [123, 61, 125, 56, 120, 114, 110, 46, 126, 106, 42, 104, 87, 23, 121, 103, 118, 18, 82, 29, 50, 91, 53, 95, 12, 58, 64, 30, 85, 41, 40, 51, 6, 27, 39, 21, 10, 124, 0, 34, 76, 62, 105, 54, 112, 93, 63, 98, 55, 48, 74, 60, 47, 15, 115, 90, 20, 84, 17, 81, 16, 44, 122, 113, 57, 14, 68, 22, 4, 80, 59, 79, 13, 70, 77, 116, 31, 127, 78, 28, 108, 86, 102, 49, 66, 109, 83, 9, 1, 2, 71, 11, 43, 65, 119, 24, 7, 25, 33, 73, 88, 52, 111, 89, 72, 107, 19, 94, 38, 45, 8, 35, 100, 37, 36, 99, 26, 75, 67, 3, 117, 5, 97, 69, 101, 92, 32, 96]], "model.layers.29.self_attn.q_proj": [[45, 35, 109, 93, 22, 25, 80, 18, 83, 11, 14, 91, 67, 31, 1, 115, 124, 10, 88, 127, 43, 55, 13, 119, 8, 64, 81, 90, 6, 21, 7, 102, 48, 95, 5, 9, 69, 27, 112, 70, 40, 113, 78, 114, 126, 4, 120, 53, 50, 111, 61, 121, 2, 101, 24, 12, 103, 54, 37, 63, 57, 108, 38, 122, 92, 16, 44, 59, 97, 33, 52, 123, 28, 3, 96, 26, 116, 79, 104, 47, 49, 86, 100, 15, 17, 32, 60, 117, 62, 106, 36, 34, 118, 41, 58, 84, 51, 39, 89, 94, 46, 110, 19, 107, 29, 42, 77, 72, 56, 105, 125, 76, 30, 85, 0, 87, 23, 98, 20, 99, 82, 66, 73, 75, 65, 71, 68, 74], [45, 109, 35, 112, 93, 25, 22, 115, 18, 83, 14, 61, 13, 10, 80, 119, 31, 91, 55, 11, 121, 124, 54, 126, 7, 114, 50, 103, 60, 90, 102, 101, 21, 113, 62, 116, 123, 1, 8, 84, 15, 127, 110, 33, 88, 9, 20, 79, 48, 108, 27, 47, 122, 92, 125, 23, 95, 49, 28, 17, 69, 32, 118, 100, 75, 117, 85, 120, 52, 24, 51, 97, 107, 111, 37, 98, 63, 41, 46, 34, 59, 81, 4, 56, 43, 39, 104, 40, 106, 26, 53, 87, 57, 58, 42, 86, 94, 82, 38, 6, 36, 19, 105, 44, 76, 89, 30, 5, 29, 12, 64, 73, 67, 96, 16, 77, 71, 68, 74, 78, 72, 99, 70, 2, 66, 65, 3, 0], [45, 109, 55, 35, 118, 93, 22, 112, 111, 121, 113, 115, 43, 48, 25, 31, 91, 53, 62, 126, 40, 122, 52, 80, 119, 123, 50, 61, 49, 117, 51, 56, 27, 92, 58, 103, 104, 102, 18, 83, 60, 39, 124, 46, 108, 59, 44, 94, 54, 41, 127, 106, 90, 114, 76, 47, 57, 107, 99, 42, 20, 21, 116, 101, 120, 110, 32, 125, 88, 84, 63, 14, 16, 23, 38, 33, 28, 74, 64, 105, 95, 37, 97, 36, 7, 15, 98, 96, 1, 100, 2, 30, 79, 19, 89, 11, 86, 34, 85, 69, 10, 0, 87, 66, 29, 8, 72, 67, 12, 68, 65, 26, 24, 17, 70, 75, 78, 4, 81, 5, 6, 82, 3, 71, 13, 9, 77, 73], [45, 35, 109, 93, 22, 18, 25, 55, 83, 13, 124, 31, 115, 14, 88, 80, 61, 112, 91, 81, 10, 118, 21, 50, 11, 15, 121, 28, 90, 63, 77, 9, 73, 48, 64, 56, 33, 7, 103, 85, 23, 44, 126, 120, 127, 27, 41, 24, 43, 92, 119, 111, 114, 113, 29, 26, 12, 62, 99, 30, 58, 67, 1, 8, 122, 52, 117, 76, 97, 95, 57, 123, 60, 69, 116, 108, 84, 89, 102, 71, 20, 51, 39, 70, 74, 40, 53, 2, 17, 37, 86, 59, 107, 6, 104, 106, 79, 5, 38, 19, 105, 54, 100, 110, 82, 49, 42, 36, 101, 34, 47, 32, 98, 46, 96, 125, 72, 87, 16, 94, 3, 66, 78, 75, 0, 68, 4, 65], [41, 98, 57, 117, 25, 51, 86, 127, 95, 114, 125, 120, 121, 18, 93, 16, 78, 50, 115, 61, 110, 105, 59, 53, 48, 55, 119, 46, 15, 49, 31, 58, 38, 87, 40, 12, 47, 39, 28, 19, 88, 118, 62, 108, 63, 45, 74, 85, 11, 116, 72, 75, 54, 67, 123, 68, 9, 76, 56, 106, 124, 35, 122, 10, 126, 80, 101, 90, 7, 69, 107, 13, 65, 111, 91, 66, 60, 43, 42, 109, 64, 92, 1, 113, 71, 112, 100, 33, 70, 23, 103, 32, 44, 82, 21, 89, 52, 20, 27, 22, 36, 104, 17, 29, 102, 99, 81, 30, 37, 24, 26, 79, 77, 14, 2, 96, 84, 97, 94, 4, 83, 8, 73, 0, 3, 34, 5, 6], [41, 98, 117, 57, 127, 95, 25, 125, 86, 114, 51, 93, 53, 54, 50, 18, 48, 120, 60, 61, 58, 105, 119, 55, 111, 38, 78, 115, 31, 16, 87, 59, 62, 121, 40, 108, 88, 21, 106, 43, 126, 56, 35, 63, 19, 15, 116, 85, 110, 12, 28, 10, 45, 52, 122, 47, 92, 7, 90, 101, 124, 123, 104, 109, 36, 113, 49, 72, 29, 80, 100, 33, 81, 76, 112, 103, 44, 30, 74, 107, 32, 46, 91, 118, 9, 77, 42, 23, 24, 82, 97, 22, 1, 75, 2, 39, 89, 67, 37, 20, 94, 27, 83, 13, 5, 102, 11, 26, 99, 70, 96, 79, 17, 3, 4, 84, 14, 68, 71, 69, 73, 64, 34, 6, 8, 0, 66, 65], [41, 98, 57, 117, 25, 86, 95, 51, 61, 127, 114, 18, 78, 59, 125, 119, 121, 48, 16, 93, 53, 110, 50, 105, 120, 116, 62, 58, 55, 38, 49, 31, 7, 76, 67, 65, 39, 10, 15, 87, 115, 12, 28, 108, 9, 74, 88, 45, 71, 56, 75, 21, 118, 109, 44, 72, 47, 19, 46, 60, 113, 124, 111, 68, 99, 40, 122, 106, 54, 103, 92, 35, 64, 81, 85, 70, 100, 89, 90, 107, 29, 43, 23, 63, 42, 69, 66, 112, 82, 1, 52, 91, 20, 32, 126, 123, 104, 11, 36, 80, 27, 13, 102, 14, 2, 33, 37, 26, 84, 22, 4, 8, 5, 83, 30, 101, 77, 17, 79, 97, 73, 96, 24, 94, 0, 3, 34, 6], [41, 98, 57, 117, 95, 120, 86, 25, 114, 51, 127, 125, 18, 110, 93, 121, 78, 16, 105, 59, 48, 108, 53, 58, 62, 119, 50, 45, 61, 38, 47, 49, 15, 124, 115, 39, 87, 44, 72, 85, 31, 46, 55, 111, 118, 12, 76, 21, 101, 54, 109, 7, 65, 116, 77, 67, 92, 35, 107, 43, 40, 69, 32, 90, 9, 11, 10, 112, 19, 88, 52, 75, 80, 97, 106, 28, 42, 82, 68, 113, 123, 102, 83, 81, 66, 20, 74, 70, 84, 24, 56, 64, 60, 22, 89, 99, 29, 63, 94, 126, 104, 36, 14, 13, 122, 91, 26, 103, 27, 73, 33, 5, 30, 100, 4, 96, 17, 37, 79, 0, 23, 8, 71, 1, 2, 34, 6, 3], [39, 63, 52, 97, 18, 113, 13, 87, 22, 28, 100, 120, 10, 4, 8, 45, 92, 11, 66, 0, 89, 126, 6, 81, 94, 27, 29, 68, 69, 42, 72, 65, 15, 12, 1, 14, 30, 59, 127, 108, 25, 70, 86, 74, 82, 110, 80, 119, 20, 79, 77, 44, 111, 90, 21, 7, 78, 62, 50, 85, 43, 16, 17, 121, 5, 19, 102, 23, 101, 40, 35, 106, 116, 9, 64, 24, 67, 96, 122, 75, 26, 84, 55, 83, 47, 38, 118, 125, 95, 54, 105, 91, 99, 76, 58, 88, 37, 104, 93, 3, 98, 32, 61, 34, 57, 117, 49, 73, 123, 115, 56, 36, 109, 103, 46, 41, 51, 31, 60, 48, 53, 124, 112, 71, 107, 114, 2, 33], [63, 52, 39, 113, 120, 37, 122, 125, 126, 45, 106, 119, 58, 61, 127, 111, 55, 110, 57, 47, 116, 51, 62, 103, 123, 56, 48, 115, 117, 59, 49, 54, 118, 108, 60, 97, 109, 53, 50, 121, 44, 114, 46, 124, 112, 100, 105, 107, 43, 36, 22, 42, 28, 41, 40, 38, 104, 21, 87, 20, 101, 102, 27, 35, 96, 2, 91, 94, 98, 11, 32, 34, 19, 99, 18, 5, 25, 92, 24, 33, 90, 29, 26, 31, 85, 8, 89, 93, 30, 95, 88, 84, 78, 81, 14, 86, 15, 13, 73, 6, 83, 23, 16, 64, 75, 79, 17, 69, 66, 9, 10, 80, 82, 12, 76, 77, 3, 67, 72, 0, 65, 70, 71, 4, 74, 7, 1, 68], [63, 39, 52, 97, 24, 87, 50, 66, 54, 110, 64, 83, 18, 69, 11, 7, 92, 58, 120, 79, 3, 36, 73, 13, 20, 114, 28, 0, 57, 2, 67, 6, 9, 22, 119, 106, 80, 44, 45, 10, 113, 8, 26, 1, 68, 108, 48, 37, 111, 4, 30, 65, 122, 85, 29, 62, 47, 72, 71, 107, 126, 25, 88, 125, 49, 61, 89, 76, 103, 70, 51, 96, 34, 12, 123, 40, 16, 104, 35, 81, 5, 38, 78, 116, 124, 117, 115, 31, 90, 84, 127, 32, 46, 56, 59, 55, 101, 98, 42, 60, 100, 118, 14, 75, 121, 94, 91, 105, 53, 112, 43, 27, 15, 17, 23, 109, 19, 95, 99, 93, 77, 102, 86, 41, 21, 82, 74, 33], [63, 39, 113, 52, 97, 120, 55, 28, 122, 87, 126, 119, 62, 92, 50, 114, 59, 22, 111, 58, 109, 36, 118, 106, 51, 47, 18, 110, 121, 61, 125, 54, 56, 116, 105, 115, 48, 57, 42, 45, 60, 49, 15, 102, 107, 21, 101, 127, 123, 124, 46, 100, 41, 37, 103, 117, 40, 44, 112, 13, 19, 35, 24, 53, 108, 43, 89, 81, 38, 84, 93, 99, 20, 69, 91, 104, 33, 8, 96, 74, 85, 34, 94, 30, 11, 98, 27, 31, 32, 86, 25, 23, 29, 26, 95, 90, 10, 17, 9, 88, 79, 83, 82, 76, 12, 4, 14, 5, 78, 6, 80, 64, 16, 71, 72, 73, 66, 3, 1, 77, 70, 2, 65, 68, 7, 75, 0, 67], [111, 47, 95, 28, 25, 84, 97, 18, 86, 33, 39, 122, 16, 123, 74, 29, 69, 15, 53, 117, 78, 120, 107, 49, 58, 7, 4, 106, 124, 35, 75, 125, 64, 17, 2, 110, 62, 55, 61, 113, 63, 91, 13, 32, 46, 42, 126, 99, 23, 12, 54, 121, 3, 108, 112, 72, 115, 1, 37, 57, 38, 96, 73, 59, 85, 44, 36, 43, 41, 60, 45, 50, 119, 77, 19, 114, 27, 48, 109, 101, 67, 116, 118, 104, 79, 52, 87, 127, 56, 102, 22, 76, 26, 103, 70, 90, 40, 24, 20, 83, 31, 89, 51, 6, 82, 92, 105, 30, 88, 94, 0, 65, 93, 100, 98, 9, 34, 21, 11, 80, 71, 81, 5, 14, 8, 10, 66, 68], [111, 47, 28, 95, 25, 84, 39, 97, 86, 33, 18, 16, 60, 59, 115, 112, 29, 54, 15, 74, 7, 106, 78, 52, 110, 118, 123, 1, 114, 69, 64, 17, 46, 62, 120, 57, 109, 2, 13, 4, 75, 12, 122, 101, 116, 107, 126, 49, 99, 27, 119, 9, 113, 121, 50, 45, 48, 41, 124, 42, 70, 63, 72, 58, 81, 36, 55, 85, 90, 56, 23, 73, 87, 117, 125, 76, 94, 61, 26, 67, 20, 35, 3, 96, 30, 32, 38, 43, 100, 53, 44, 103, 77, 24, 51, 40, 127, 89, 91, 37, 108, 31, 88, 22, 79, 19, 82, 83, 102, 6, 34, 105, 65, 98, 104, 92, 93, 80, 21, 66, 5, 0, 14, 68, 71, 11, 10, 8], [111, 47, 28, 95, 25, 84, 126, 33, 18, 97, 57, 86, 39, 120, 16, 60, 110, 124, 61, 29, 63, 116, 74, 54, 78, 56, 17, 64, 7, 46, 15, 125, 99, 50, 4, 12, 107, 62, 127, 75, 48, 53, 27, 35, 96, 106, 112, 109, 92, 59, 38, 113, 67, 2, 49, 55, 101, 72, 45, 42, 36, 69, 1, 41, 119, 20, 31, 94, 108, 52, 58, 123, 13, 89, 85, 122, 121, 87, 23, 22, 40, 114, 117, 73, 37, 19, 44, 32, 115, 70, 76, 105, 100, 90, 118, 9, 24, 51, 103, 43, 104, 91, 102, 34, 26, 88, 82, 81, 30, 98, 6, 3, 83, 0, 68, 21, 14, 80, 77, 93, 71, 10, 79, 8, 5, 65, 11, 66], [111, 47, 28, 25, 95, 84, 57, 86, 39, 110, 62, 33, 18, 16, 74, 116, 78, 29, 97, 2, 49, 41, 17, 118, 96, 15, 115, 1, 7, 64, 69, 4, 126, 13, 59, 120, 3, 127, 113, 63, 51, 65, 58, 56, 125, 36, 44, 50, 85, 123, 35, 60, 121, 75, 99, 43, 54, 37, 42, 76, 117, 119, 98, 114, 106, 46, 6, 55, 67, 70, 108, 45, 79, 124, 40, 89, 109, 52, 48, 38, 32, 53, 0, 26, 24, 105, 101, 72, 73, 27, 9, 20, 82, 112, 103, 22, 5, 122, 30, 81, 94, 61, 12, 102, 104, 107, 83, 90, 34, 77, 23, 11, 80, 92, 87, 88, 93, 21, 10, 91, 31, 14, 19, 100, 71, 66, 68, 8], [49, 117, 102, 113, 123, 52, 116, 53, 38, 23, 41, 91, 33, 25, 121, 85, 118, 119, 54, 122, 115, 109, 59, 93, 104, 120, 30, 110, 62, 61, 51, 108, 60, 124, 82, 112, 45, 111, 98, 114, 57, 48, 50, 96, 42, 63, 36, 94, 56, 47, 126, 46, 106, 37, 125, 21, 35, 43, 107, 127, 58, 44, 77, 79, 27, 55, 100, 103, 99, 95, 89, 40, 105, 39, 97, 24, 84, 18, 34, 101, 32, 20, 16, 17, 22, 31, 19, 29, 76, 87, 11, 78, 90, 83, 28, 88, 86, 26, 72, 92, 15, 5, 12, 4, 14, 70, 71, 13, 74, 80, 73, 9, 75, 3, 7, 81, 66, 1, 10, 8, 6, 0, 65, 68, 69, 67, 2, 64], [49, 102, 113, 117, 33, 91, 25, 38, 116, 123, 54, 30, 82, 23, 110, 55, 85, 35, 53, 121, 109, 39, 115, 95, 58, 62, 122, 100, 43, 93, 98, 51, 89, 52, 59, 60, 27, 21, 41, 125, 78, 48, 119, 57, 120, 42, 77, 40, 37, 92, 50, 118, 108, 80, 90, 96, 63, 28, 114, 112, 24, 111, 61, 103, 106, 124, 104, 71, 56, 34, 47, 101, 5, 14, 11, 127, 99, 22, 20, 36, 46, 94, 29, 12, 126, 32, 9, 65, 19, 88, 68, 86, 31, 97, 44, 15, 107, 105, 26, 45, 87, 2, 18, 17, 83, 67, 70, 72, 6, 64, 84, 8, 10, 76, 1, 3, 0, 16, 13, 75, 81, 74, 79, 7, 73, 69, 66, 4], [49, 102, 113, 117, 53, 33, 30, 25, 38, 91, 85, 120, 23, 125, 122, 93, 60, 82, 119, 54, 61, 41, 48, 118, 96, 107, 59, 27, 94, 109, 50, 108, 121, 110, 114, 106, 52, 35, 98, 42, 103, 101, 20, 40, 95, 43, 45, 123, 58, 57, 105, 99, 14, 116, 62, 89, 46, 36, 44, 56, 100, 124, 39, 47, 80, 77, 26, 127, 51, 111, 55, 97, 78, 72, 126, 112, 92, 3, 63, 115, 31, 34, 104, 22, 11, 24, 90, 29, 84, 32, 28, 75, 37, 10, 88, 73, 71, 64, 83, 4, 87, 18, 21, 15, 17, 12, 19, 2, 81, 65, 74, 9, 5, 86, 70, 16, 79, 76, 7, 13, 6, 68, 1, 0, 69, 8, 66, 67], [49, 102, 113, 117, 123, 33, 52, 25, 53, 91, 122, 61, 23, 38, 63, 20, 41, 48, 82, 85, 100, 118, 121, 93, 112, 27, 89, 116, 15, 96, 114, 42, 108, 11, 35, 30, 97, 50, 39, 101, 62, 9, 120, 99, 51, 87, 46, 45, 6, 115, 47, 60, 57, 59, 44, 124, 68, 126, 111, 127, 105, 109, 58, 43, 94, 54, 90, 34, 110, 80, 32, 103, 40, 55, 56, 28, 29, 26, 107, 119, 77, 125, 106, 104, 14, 98, 4, 1, 37, 95, 84, 22, 73, 36, 81, 21, 18, 16, 92, 83, 88, 31, 8, 86, 66, 79, 70, 19, 24, 10, 12, 0, 75, 2, 17, 76, 69, 5, 71, 78, 13, 7, 67, 3, 74, 64, 65, 72], [117, 120, 60, 38, 58, 127, 122, 46, 52, 62, 102, 126, 55, 125, 118, 43, 110, 116, 123, 121, 124, 115, 113, 63, 61, 119, 51, 57, 53, 114, 112, 56, 25, 54, 48, 50, 45, 59, 47, 108, 111, 39, 107, 44, 97, 104, 35, 37, 49, 26, 109, 94, 106, 41, 105, 86, 42, 98, 83, 103, 17, 24, 101, 23, 28, 36, 40, 85, 88, 92, 100, 91, 78, 19, 22, 21, 75, 32, 30, 34, 99, 11, 96, 16, 95, 27, 77, 79, 29, 82, 84, 31, 33, 14, 90, 93, 20, 81, 89, 80, 72, 18, 9, 3, 12, 87, 7, 67, 76, 15, 73, 71, 69, 70, 8, 2, 6, 1, 5, 0, 13, 65, 64, 74, 66, 68, 10, 4], [117, 60, 58, 127, 38, 110, 122, 46, 52, 116, 62, 115, 118, 126, 120, 125, 123, 43, 102, 51, 61, 124, 63, 55, 113, 109, 53, 94, 56, 121, 119, 57, 112, 59, 97, 54, 50, 114, 48, 45, 107, 47, 108, 111, 39, 104, 28, 44, 106, 25, 103, 23, 49, 86, 26, 105, 42, 35, 83, 41, 37, 36, 98, 101, 17, 91, 40, 74, 30, 100, 88, 84, 33, 24, 32, 99, 19, 22, 95, 90, 85, 81, 34, 96, 92, 75, 93, 31, 11, 69, 79, 27, 77, 78, 29, 66, 15, 80, 72, 16, 0, 14, 3, 64, 5, 89, 12, 87, 71, 20, 67, 18, 2, 82, 21, 65, 1, 8, 68, 6, 13, 10, 70, 76, 7, 9, 73, 4], [117, 60, 46, 120, 38, 110, 127, 122, 58, 52, 115, 62, 57, 125, 116, 118, 43, 126, 119, 48, 63, 123, 61, 56, 35, 49, 25, 109, 94, 121, 55, 124, 47, 51, 111, 50, 112, 54, 114, 83, 53, 113, 24, 26, 108, 45, 59, 105, 40, 107, 104, 102, 44, 86, 41, 97, 106, 28, 103, 37, 42, 88, 30, 23, 36, 39, 101, 100, 22, 75, 34, 85, 99, 11, 84, 17, 31, 19, 32, 16, 98, 79, 27, 81, 91, 69, 92, 33, 78, 93, 96, 95, 3, 14, 72, 82, 90, 29, 8, 89, 77, 20, 5, 13, 18, 66, 80, 74, 15, 87, 2, 64, 0, 9, 21, 67, 70, 65, 71, 6, 1, 76, 7, 12, 10, 73, 68, 4], [46, 117, 38, 120, 110, 85, 94, 25, 97, 16, 9, 90, 12, 126, 122, 81, 23, 116, 58, 100, 15, 62, 118, 98, 52, 68, 14, 76, 28, 89, 7, 125, 87, 82, 61, 71, 121, 37, 18, 123, 48, 86, 1, 42, 60, 30, 21, 99, 57, 96, 31, 91, 44, 107, 32, 119, 112, 36, 53, 47, 56, 127, 51, 101, 45, 55, 115, 10, 113, 43, 70, 114, 124, 88, 104, 29, 26, 73, 92, 54, 79, 109, 111, 75, 49, 50, 63, 24, 19, 13, 20, 93, 59, 83, 4, 108, 95, 74, 80, 34, 84, 35, 17, 106, 72, 27, 6, 11, 103, 2, 77, 22, 105, 78, 41, 8, 40, 39, 5, 65, 33, 69, 64, 3, 66, 0, 67, 102], [41, 99, 67, 74, 80, 0, 86, 44, 4, 13, 71, 73, 19, 115, 118, 69, 124, 105, 52, 119, 113, 114, 51, 65, 122, 54, 12, 116, 112, 111, 35, 117, 110, 126, 57, 2, 62, 63, 64, 60, 48, 107, 61, 22, 6, 89, 121, 76, 84, 123, 109, 1, 29, 90, 3, 45, 30, 88, 14, 106, 83, 79, 10, 33, 92, 9, 108, 120, 20, 55, 23, 68, 39, 15, 40, 101, 102, 103, 98, 104, 66, 75, 100, 56, 27, 94, 11, 38, 53, 17, 58, 24, 31, 32, 8, 77, 70, 7, 5, 26, 43, 16, 85, 95, 81, 46, 34, 37, 49, 125, 96, 21, 28, 36, 72, 50, 127, 91, 93, 59, 97, 42, 82, 78, 87, 18, 47, 25], [41, 99, 80, 74, 13, 4, 86, 71, 19, 67, 73, 113, 52, 69, 61, 51, 12, 44, 118, 115, 112, 122, 64, 60, 114, 105, 111, 66, 2, 120, 57, 116, 103, 63, 6, 54, 119, 62, 124, 35, 75, 117, 43, 1, 55, 20, 15, 88, 48, 108, 126, 22, 101, 27, 125, 107, 21, 82, 29, 85, 31, 102, 76, 17, 50, 78, 34, 3, 90, 106, 37, 79, 26, 24, 8, 109, 83, 127, 59, 39, 23, 91, 45, 46, 87, 49, 98, 123, 100, 81, 96, 92, 110, 32, 30, 68, 93, 89, 38, 56, 97, 42, 18, 53, 84, 36, 33, 5, 58, 94, 72, 25, 28, 104, 14, 95, 121, 10, 65, 47, 70, 40, 11, 7, 16, 77, 9, 0], [41, 74, 0, 99, 65, 86, 2, 44, 80, 13, 67, 69, 4, 118, 119, 116, 115, 113, 71, 112, 54, 19, 111, 110, 114, 105, 73, 124, 12, 117, 121, 52, 122, 51, 126, 57, 35, 22, 107, 62, 48, 63, 29, 61, 15, 94, 3, 59, 68, 89, 83, 60, 6, 123, 10, 27, 90, 33, 45, 20, 108, 5, 24, 40, 88, 16, 106, 1, 49, 92, 64, 9, 76, 23, 53, 8, 32, 38, 75, 120, 100, 66, 7, 56, 21, 97, 79, 39, 104, 85, 26, 91, 18, 109, 93, 11, 82, 70, 55, 101, 87, 30, 58, 125, 98, 17, 78, 127, 31, 81, 72, 96, 84, 14, 77, 42, 28, 95, 34, 43, 25, 103, 102, 37, 47, 46, 36, 50], [41, 99, 0, 2, 44, 67, 86, 80, 13, 4, 65, 115, 118, 69, 71, 119, 116, 113, 73, 114, 111, 112, 122, 54, 19, 52, 51, 105, 124, 110, 126, 22, 121, 117, 57, 35, 74, 62, 29, 12, 107, 3, 48, 68, 61, 60, 63, 89, 33, 123, 64, 15, 6, 108, 90, 40, 5, 97, 94, 88, 16, 70, 27, 106, 20, 83, 100, 98, 17, 75, 79, 38, 49, 81, 45, 84, 11, 1, 39, 24, 76, 92, 56, 9, 91, 120, 104, 55, 96, 28, 18, 10, 26, 8, 14, 78, 82, 125, 109, 42, 25, 30, 85, 53, 59, 93, 7, 23, 37, 95, 101, 36, 58, 72, 46, 21, 127, 87, 31, 47, 34, 32, 102, 43, 50, 77, 66, 103], [106, 99, 59, 26, 118, 124, 94, 49, 42, 21, 62, 19, 115, 86, 17, 113, 60, 121, 40, 109, 55, 30, 78, 32, 80, 56, 117, 107, 73, 122, 28, 123, 47, 12, 24, 96, 98, 29, 120, 74, 23, 43, 50, 53, 11, 69, 51, 37, 125, 61, 63, 114, 52, 27, 79, 108, 116, 103, 57, 126, 110, 58, 127, 112, 119, 48, 84, 89, 105, 92, 39, 54, 46, 111, 36, 104, 4, 22, 8, 71, 101, 45, 3, 72, 67, 7, 90, 44, 41, 34, 38, 15, 77, 33, 102, 93, 97, 31, 100, 70, 14, 66, 64, 95, 88, 85, 76, 0, 1, 82, 91, 25, 81, 16, 87, 18, 5, 83, 20, 13, 6, 35, 65, 9, 68, 10, 75, 2], [106, 61, 99, 118, 115, 59, 121, 26, 94, 42, 107, 62, 113, 21, 19, 86, 125, 29, 11, 109, 28, 79, 50, 22, 55, 60, 25, 117, 40, 17, 126, 47, 90, 56, 123, 23, 119, 124, 24, 58, 52, 93, 80, 57, 78, 114, 39, 98, 13, 74, 66, 32, 16, 73, 116, 101, 85, 34, 65, 104, 53, 63, 103, 49, 12, 70, 122, 35, 36, 82, 120, 108, 45, 46, 4, 51, 71, 95, 30, 44, 20, 91, 37, 38, 105, 87, 72, 48, 112, 43, 27, 100, 77, 6, 7, 111, 97, 31, 41, 3, 89, 64, 110, 54, 127, 88, 96, 102, 92, 18, 84, 83, 33, 5, 69, 81, 76, 15, 14, 0, 67, 10, 75, 8, 9, 2, 1, 68], [106, 59, 115, 99, 118, 62, 50, 112, 26, 55, 94, 125, 86, 19, 117, 121, 124, 42, 24, 116, 21, 53, 80, 17, 29, 52, 122, 107, 44, 78, 49, 123, 30, 32, 40, 28, 45, 11, 57, 74, 61, 98, 63, 58, 120, 101, 113, 23, 46, 12, 110, 60, 104, 27, 48, 96, 51, 108, 109, 114, 47, 72, 126, 119, 69, 4, 127, 111, 54, 97, 73, 56, 71, 43, 36, 39, 95, 37, 103, 38, 102, 105, 41, 84, 33, 100, 7, 22, 34, 91, 93, 90, 35, 31, 79, 8, 66, 92, 87, 67, 88, 82, 20, 16, 25, 18, 68, 85, 76, 70, 15, 81, 64, 89, 3, 83, 1, 0, 14, 77, 6, 5, 9, 10, 65, 13, 75, 2], [106, 99, 107, 115, 59, 26, 94, 113, 86, 17, 19, 42, 21, 12, 121, 80, 30, 62, 40, 119, 117, 55, 96, 49, 78, 74, 60, 69, 61, 125, 112, 24, 72, 11, 118, 53, 114, 56, 124, 70, 120, 4, 32, 123, 116, 109, 108, 58, 5, 44, 104, 73, 110, 68, 52, 126, 93, 54, 89, 122, 67, 48, 45, 57, 1, 101, 63, 51, 37, 29, 47, 28, 71, 36, 127, 16, 43, 41, 111, 25, 105, 46, 22, 102, 13, 88, 9, 98, 83, 50, 27, 0, 84, 39, 8, 20, 95, 90, 14, 87, 7, 38, 77, 33, 64, 100, 6, 82, 92, 34, 79, 23, 76, 35, 3, 103, 31, 65, 18, 91, 85, 66, 97, 81, 10, 15, 2, 75]], "model.layers.29.self_attn.k_proj": [[109, 45, 99, 93, 22, 64, 31, 83, 80, 14, 25, 18, 11, 91, 67, 8, 2, 7, 126, 119, 1, 124, 6, 127, 70, 122, 112, 77, 13, 115, 60, 16, 21, 55, 9, 43, 114, 23, 10, 51, 4, 29, 90, 12, 52, 110, 63, 116, 41, 42, 36, 111, 57, 69, 118, 28, 59, 49, 120, 84, 103, 72, 123, 56, 61, 39, 121, 88, 113, 62, 54, 105, 44, 47, 53, 27, 40, 82, 58, 50, 117, 85, 102, 96, 48, 38, 92, 33, 100, 5, 24, 87, 108, 107, 106, 94, 101, 125, 98, 32, 46, 26, 34, 104, 15, 37, 30, 76, 97, 89, 79, 20, 95, 78, 75, 17, 35, 73, 81, 74, 66, 19, 86, 68, 71, 3, 0, 65], [105, 34, 57, 18, 53, 46, 51, 125, 78, 86, 117, 25, 58, 120, 16, 45, 64, 31, 127, 67, 95, 109, 98, 50, 12, 119, 29, 69, 59, 111, 108, 61, 66, 10, 112, 9, 19, 74, 87, 7, 70, 113, 41, 60, 92, 80, 13, 49, 15, 55, 91, 52, 114, 88, 21, 121, 62, 76, 2, 107, 65, 75, 116, 103, 38, 83, 63, 68, 100, 93, 104, 39, 0, 40, 72, 56, 20, 35, 106, 90, 110, 115, 73, 54, 118, 27, 36, 48, 6, 42, 44, 126, 81, 33, 122, 124, 47, 102, 43, 26, 123, 84, 77, 37, 1, 101, 28, 30, 96, 94, 23, 24, 99, 79, 85, 97, 32, 17, 71, 89, 11, 8, 22, 82, 5, 4, 14, 3], [63, 103, 22, 33, 92, 87, 13, 65, 18, 120, 10, 50, 4, 119, 126, 111, 54, 0, 61, 109, 49, 6, 62, 58, 81, 51, 117, 48, 127, 47, 125, 123, 122, 118, 59, 124, 46, 8, 45, 60, 56, 121, 115, 44, 30, 110, 113, 53, 116, 114, 112, 29, 2, 57, 71, 40, 55, 42, 43, 102, 52, 108, 37, 107, 25, 105, 73, 41, 98, 36, 20, 106, 104, 38, 11, 3, 34, 5, 39, 101, 91, 14, 100, 12, 7, 16, 78, 89, 67, 99, 83, 79, 31, 80, 24, 90, 76, 15, 32, 70, 95, 27, 93, 96, 35, 88, 21, 74, 94, 23, 17, 9, 85, 84, 26, 28, 86, 69, 75, 82, 19, 64, 1, 68, 97, 72, 66, 77], [47, 111, 64, 86, 95, 16, 84, 28, 25, 18, 74, 4, 1, 2, 72, 7, 97, 78, 67, 65, 6, 69, 15, 17, 29, 123, 12, 3, 124, 23, 76, 103, 75, 54, 31, 14, 0, 39, 77, 120, 55, 53, 27, 113, 85, 79, 61, 110, 70, 35, 81, 56, 13, 63, 122, 36, 105, 50, 126, 9, 19, 33, 101, 87, 91, 121, 57, 71, 125, 114, 32, 117, 62, 116, 24, 73, 48, 99, 109, 38, 127, 100, 108, 106, 118, 43, 115, 46, 52, 37, 44, 41, 119, 21, 42, 51, 26, 102, 60, 104, 30, 45, 107, 59, 34, 58, 90, 40, 98, 80, 94, 88, 49, 92, 82, 112, 89, 8, 96, 93, 20, 83, 11, 22, 10, 5, 66, 68], [49, 38, 97, 91, 25, 93, 117, 23, 80, 20, 92, 82, 15, 9, 64, 77, 11, 85, 2, 30, 68, 65, 87, 3, 45, 14, 108, 21, 5, 22, 17, 95, 96, 94, 50, 71, 120, 112, 36, 35, 18, 61, 37, 119, 8, 6, 118, 53, 70, 48, 46, 126, 106, 109, 55, 104, 59, 57, 34, 58, 122, 63, 111, 107, 33, 54, 123, 40, 62, 103, 52, 78, 105, 116, 124, 121, 39, 110, 99, 47, 28, 44, 24, 31, 101, 115, 42, 56, 41, 43, 89, 51, 127, 114, 125, 10, 60, 90, 83, 32, 19, 26, 98, 29, 1, 13, 67, 7, 100, 72, 113, 88, 84, 86, 0, 102, 12, 75, 81, 79, 16, 27, 76, 69, 74, 66, 73, 4], [110, 117, 102, 86, 33, 30, 52, 92, 25, 99, 58, 62, 119, 22, 125, 123, 54, 122, 114, 126, 53, 112, 55, 14, 81, 49, 111, 116, 51, 124, 47, 121, 48, 59, 61, 44, 103, 56, 63, 120, 75, 41, 108, 113, 115, 57, 118, 50, 83, 109, 16, 127, 39, 38, 107, 105, 35, 43, 31, 104, 46, 106, 28, 42, 45, 94, 91, 34, 32, 85, 23, 40, 37, 72, 100, 24, 101, 93, 26, 36, 18, 12, 29, 90, 5, 97, 9, 27, 98, 60, 84, 67, 0, 96, 95, 21, 87, 6, 20, 19, 82, 89, 80, 8, 78, 15, 2, 1, 7, 10, 88, 76, 68, 13, 77, 11, 79, 3, 17, 74, 65, 71, 64, 70, 69, 66, 73, 4], [105, 64, 35, 1, 118, 0, 116, 108, 67, 51, 119, 71, 19, 13, 86, 69, 73, 66, 4, 80, 65, 49, 2, 124, 48, 57, 12, 61, 122, 113, 47, 46, 114, 115, 43, 111, 44, 54, 3, 63, 99, 117, 126, 50, 74, 53, 58, 62, 121, 123, 52, 107, 60, 5, 56, 70, 106, 89, 6, 93, 55, 15, 103, 110, 59, 29, 94, 20, 88, 26, 112, 104, 17, 7, 75, 85, 72, 41, 33, 109, 76, 79, 97, 68, 45, 120, 90, 92, 9, 18, 101, 11, 125, 102, 84, 91, 30, 96, 32, 8, 77, 82, 87, 27, 23, 100, 38, 34, 78, 24, 14, 40, 21, 31, 37, 39, 127, 25, 98, 95, 42, 28, 81, 10, 83, 36, 22, 16], [42, 30, 99, 118, 35, 86, 26, 51, 21, 19, 78, 113, 61, 17, 64, 45, 11, 121, 66, 63, 111, 117, 106, 125, 28, 73, 124, 114, 123, 102, 71, 126, 53, 80, 108, 49, 122, 62, 12, 105, 41, 4, 32, 43, 74, 116, 16, 7, 38, 23, 56, 40, 119, 44, 55, 52, 3, 46, 104, 54, 65, 101, 115, 24, 47, 127, 110, 57, 109, 79, 69, 95, 70, 89, 36, 103, 58, 120, 48, 39, 29, 112, 33, 107, 98, 100, 25, 1, 59, 60, 50, 97, 37, 34, 94, 87, 15, 27, 91, 92, 20, 6, 72, 82, 31, 96, 85, 88, 93, 8, 10, 84, 67, 77, 13, 68, 18, 5, 75, 76, 90, 0, 22, 83, 81, 9, 2, 14]], "model.layers.29.self_attn.qk_proj": [[49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 99, 22, 57, 106, 118, 41, 113, 86, 64, 0, 89, 51, 35, 119, 38, 16, 80, 53, 93, 61, 116, 95, 52, 28, 115, 82, 120, 124, 121, 94, 114, 62, 122, 25, 27, 46, 58, 125, 19, 44, 77, 102, 98, 18, 103, 83, 13, 54, 126, 108, 65, 50, 55, 33, 1, 39, 48, 10, 127, 3, 92, 71, 68, 78, 43, 112, 123, 7, 4, 97, 59, 14, 67, 107, 31, 56, 2, 66, 9, 90, 87, 5, 69, 74, 84, 73, 6, 21, 60, 26, 85, 29, 30, 11, 12, 23, 20, 75, 76, 79, 104, 81, 15, 8, 34, 72, 37, 17, 40, 101, 91, 36, 32, 24, 96, 88, 100, 70], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 41, 106, 99, 22, 0, 118, 64, 113, 86, 51, 89, 52, 35, 38, 119, 61, 16, 115, 95, 80, 121, 120, 124, 28, 116, 94, 62, 93, 114, 55, 53, 122, 59, 65, 82, 25, 18, 98, 127, 46, 44, 58, 77, 50, 27, 123, 13, 126, 83, 102, 54, 2, 43, 68, 1, 108, 33, 125, 103, 19, 48, 39, 4, 7, 3, 74, 71, 87, 67, 14, 97, 92, 112, 5, 78, 10, 31, 73, 66, 56, 60, 9, 90, 107, 21, 11, 6, 29, 76, 84, 69, 75, 12, 23, 20, 30, 104, 17, 85, 34, 40, 8, 72, 79, 81, 15, 26, 91, 36, 101, 88, 37, 24, 70, 32, 96, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 22, 113, 106, 64, 41, 86, 0, 118, 89, 35, 51, 119, 38, 115, 52, 124, 116, 16, 61, 80, 93, 120, 95, 62, 94, 122, 18, 28, 98, 102, 123, 25, 59, 53, 82, 65, 127, 121, 19, 50, 27, 67, 48, 46, 54, 83, 126, 125, 114, 77, 1, 44, 7, 58, 33, 43, 56, 108, 103, 2, 3, 13, 68, 112, 14, 39, 66, 10, 92, 97, 4, 55, 78, 90, 71, 9, 74, 84, 69, 87, 60, 73, 5, 21, 75, 31, 29, 107, 15, 20, 30, 12, 23, 11, 76, 85, 104, 6, 26, 70, 79, 81, 34, 40, 37, 101, 72, 17, 8, 91, 36, 32, 96, 88, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 106, 41, 99, 22, 113, 86, 64, 0, 118, 35, 89, 122, 51, 119, 115, 52, 16, 124, 62, 116, 80, 38, 120, 61, 25, 94, 93, 95, 53, 28, 123, 46, 18, 121, 114, 82, 55, 54, 126, 102, 19, 1, 44, 50, 77, 39, 66, 127, 13, 3, 98, 125, 43, 33, 56, 48, 59, 58, 4, 27, 78, 83, 71, 7, 65, 68, 67, 14, 103, 97, 74, 69, 60, 108, 112, 11, 9, 31, 73, 2, 107, 87, 92, 70, 21, 10, 5, 76, 90, 85, 26, 29, 20, 79, 104, 84, 23, 12, 75, 15, 37, 81, 32, 72, 30, 36, 40, 101, 8, 6, 91, 34, 17, 88, 100, 24, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 86, 22, 64, 118, 0, 113, 89, 51, 119, 124, 61, 35, 122, 16, 115, 52, 80, 62, 121, 120, 38, 54, 28, 46, 116, 25, 93, 18, 82, 95, 94, 53, 125, 114, 48, 44, 58, 126, 98, 1, 102, 83, 19, 27, 13, 59, 123, 50, 77, 43, 55, 78, 127, 7, 112, 39, 56, 103, 4, 68, 108, 67, 71, 73, 33, 65, 74, 66, 87, 14, 70, 2, 107, 10, 92, 69, 31, 97, 84, 3, 9, 60, 90, 30, 5, 21, 20, 11, 26, 76, 81, 23, 29, 15, 12, 85, 79, 91, 75, 104, 40, 8, 34, 37, 101, 100, 32, 17, 72, 88, 96, 24, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 22, 86, 118, 0, 99, 89, 64, 51, 113, 119, 35, 80, 52, 16, 116, 38, 115, 122, 124, 95, 61, 18, 44, 62, 28, 82, 94, 46, 53, 121, 93, 25, 125, 120, 27, 13, 126, 58, 19, 54, 114, 98, 50, 55, 48, 77, 65, 83, 71, 112, 102, 4, 123, 108, 107, 33, 59, 39, 68, 1, 56, 127, 103, 66, 78, 97, 92, 3, 87, 67, 74, 70, 14, 73, 10, 2, 43, 7, 60, 11, 90, 5, 31, 84, 21, 20, 9, 30, 69, 76, 23, 15, 34, 79, 29, 37, 75, 12, 8, 85, 40, 17, 26, 91, 81, 72, 104, 36, 32, 88, 101, 6, 24, 100, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 106, 22, 57, 99, 110, 86, 118, 64, 89, 0, 51, 35, 113, 38, 80, 119, 16, 95, 52, 115, 61, 116, 120, 94, 28, 46, 124, 122, 121, 126, 18, 125, 82, 53, 77, 93, 62, 25, 65, 13, 50, 27, 83, 1, 114, 112, 44, 127, 54, 123, 59, 43, 58, 48, 19, 103, 2, 102, 3, 67, 55, 98, 14, 108, 31, 71, 39, 4, 68, 74, 78, 10, 87, 90, 92, 5, 107, 7, 33, 97, 66, 29, 56, 60, 21, 9, 30, 84, 23, 70, 11, 73, 76, 20, 26, 85, 69, 12, 15, 34, 37, 81, 72, 75, 79, 17, 40, 91, 104, 101, 8, 24, 88, 100, 96, 32, 36, 6], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 99, 22, 106, 86, 57, 0, 113, 118, 64, 51, 38, 35, 89, 119, 52, 95, 28, 61, 115, 120, 93, 124, 80, 122, 62, 116, 16, 94, 25, 53, 121, 125, 82, 58, 123, 77, 102, 27, 103, 126, 44, 98, 54, 1, 83, 18, 114, 65, 112, 13, 127, 39, 48, 67, 59, 50, 19, 46, 56, 71, 108, 33, 68, 2, 66, 3, 97, 43, 92, 7, 55, 31, 14, 87, 90, 74, 10, 78, 4, 5, 23, 9, 107, 73, 20, 29, 85, 60, 21, 30, 69, 11, 84, 70, 26, 12, 40, 75, 79, 76, 15, 81, 17, 34, 37, 72, 101, 36, 91, 88, 104, 32, 6, 96, 24, 100, 8], [49, 63, 117, 105, 111, 47, 45, 109, 42, 99, 41, 110, 57, 22, 106, 118, 86, 64, 113, 89, 51, 0, 35, 38, 95, 119, 120, 52, 93, 16, 61, 116, 115, 123, 28, 80, 62, 124, 121, 94, 127, 126, 44, 59, 114, 82, 122, 58, 83, 48, 1, 98, 18, 25, 102, 103, 125, 65, 108, 13, 53, 19, 56, 67, 46, 112, 39, 54, 55, 3, 68, 33, 77, 27, 4, 43, 71, 66, 10, 31, 60, 50, 92, 97, 107, 90, 7, 74, 78, 30, 87, 69, 29, 14, 2, 85, 20, 23, 5, 26, 73, 21, 84, 76, 11, 75, 9, 12, 17, 79, 37, 81, 40, 15, 91, 104, 36, 101, 6, 70, 72, 34, 88, 96, 8, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 57, 99, 118, 106, 41, 113, 22, 119, 64, 0, 86, 51, 35, 89, 38, 61, 115, 62, 52, 124, 116, 16, 55, 28, 80, 121, 120, 123, 25, 58, 46, 95, 65, 122, 126, 127, 82, 94, 53, 93, 18, 39, 83, 13, 98, 112, 114, 59, 44, 67, 125, 102, 19, 33, 77, 48, 56, 50, 92, 27, 4, 54, 31, 7, 68, 9, 2, 60, 43, 1, 71, 78, 3, 103, 108, 66, 74, 69, 97, 87, 14, 84, 10, 90, 11, 5, 107, 73, 29, 30, 76, 6, 20, 21, 23, 79, 26, 104, 40, 75, 85, 12, 91, 15, 34, 17, 81, 101, 8, 72, 36, 37, 70, 88, 100, 32, 96, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 99, 64, 22, 118, 0, 106, 57, 86, 51, 113, 89, 119, 35, 52, 38, 16, 122, 80, 61, 116, 25, 82, 28, 95, 121, 120, 93, 115, 46, 124, 18, 94, 126, 62, 125, 127, 53, 13, 19, 48, 65, 50, 1, 44, 83, 123, 59, 71, 4, 27, 3, 78, 114, 54, 77, 2, 55, 39, 108, 7, 58, 9, 102, 67, 14, 97, 10, 66, 103, 74, 68, 6, 73, 5, 33, 60, 56, 92, 98, 90, 69, 20, 23, 11, 87, 43, 112, 29, 84, 21, 30, 75, 31, 15, 85, 107, 12, 76, 79, 26, 81, 72, 104, 91, 17, 34, 32, 8, 37, 40, 101, 70, 36, 96, 24, 88, 100], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 110, 22, 57, 106, 99, 86, 64, 118, 0, 89, 113, 35, 122, 52, 51, 38, 119, 16, 80, 61, 120, 28, 116, 95, 115, 93, 62, 82, 25, 124, 94, 18, 121, 19, 126, 53, 13, 83, 125, 103, 58, 1, 59, 3, 77, 48, 44, 55, 46, 4, 27, 123, 54, 102, 33, 114, 50, 2, 71, 43, 65, 78, 98, 68, 74, 67, 127, 108, 39, 14, 112, 60, 10, 66, 92, 97, 7, 107, 31, 56, 20, 90, 75, 9, 21, 6, 85, 23, 69, 87, 5, 29, 12, 26, 73, 76, 11, 81, 84, 30, 40, 17, 79, 34, 91, 15, 8, 37, 72, 36, 88, 24, 104, 100, 32, 101, 70, 96], [49, 63, 105, 117, 111, 47, 45, 109, 42, 110, 57, 99, 106, 41, 118, 113, 22, 86, 51, 64, 89, 120, 119, 0, 35, 52, 38, 61, 115, 116, 80, 95, 94, 124, 25, 122, 28, 16, 62, 46, 125, 121, 93, 43, 44, 53, 59, 58, 126, 123, 98, 102, 83, 18, 114, 33, 39, 82, 55, 65, 27, 48, 103, 77, 127, 19, 56, 54, 50, 13, 108, 112, 3, 67, 92, 1, 97, 71, 4, 90, 31, 60, 23, 2, 66, 68, 10, 78, 87, 73, 74, 107, 7, 26, 21, 40, 5, 29, 69, 85, 30, 20, 84, 14, 76, 9, 75, 6, 91, 17, 15, 12, 79, 34, 81, 11, 104, 101, 88, 100, 8, 37, 24, 96, 72, 36, 70, 32], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 99, 41, 106, 22, 57, 64, 118, 0, 86, 113, 51, 89, 35, 119, 80, 52, 61, 120, 38, 121, 95, 115, 16, 124, 122, 28, 116, 62, 25, 93, 53, 94, 59, 58, 123, 1, 48, 46, 65, 125, 50, 83, 13, 127, 114, 18, 92, 19, 98, 82, 71, 33, 4, 103, 66, 3, 126, 67, 77, 108, 54, 102, 27, 55, 60, 39, 14, 112, 2, 44, 43, 73, 10, 7, 107, 78, 56, 74, 68, 97, 31, 69, 84, 5, 87, 90, 20, 29, 9, 23, 76, 12, 75, 11, 26, 21, 30, 85, 40, 6, 34, 70, 79, 15, 81, 17, 37, 72, 104, 8, 100, 91, 101, 24, 88, 36, 32, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 118, 22, 99, 0, 64, 86, 89, 51, 113, 119, 38, 35, 52, 80, 95, 120, 16, 122, 116, 61, 28, 121, 126, 25, 94, 124, 82, 46, 53, 93, 123, 115, 13, 48, 114, 125, 65, 55, 83, 19, 62, 18, 39, 102, 3, 2, 1, 127, 71, 9, 98, 108, 54, 44, 50, 59, 73, 67, 4, 78, 43, 77, 7, 68, 92, 27, 14, 84, 60, 58, 74, 103, 66, 70, 10, 112, 33, 97, 21, 5, 20, 69, 75, 31, 107, 56, 90, 11, 23, 29, 76, 87, 30, 15, 26, 12, 79, 34, 91, 17, 85, 8, 40, 81, 104, 6, 101, 72, 37, 88, 100, 32, 24, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 57, 22, 99, 118, 106, 0, 86, 64, 113, 89, 51, 38, 35, 116, 119, 16, 80, 95, 52, 121, 124, 122, 46, 61, 123, 28, 120, 62, 25, 93, 126, 115, 18, 94, 82, 53, 83, 114, 13, 125, 102, 50, 33, 127, 1, 44, 48, 19, 4, 98, 108, 59, 103, 3, 68, 7, 65, 27, 78, 67, 77, 74, 39, 56, 58, 54, 55, 2, 71, 66, 73, 60, 97, 14, 43, 9, 70, 21, 90, 92, 69, 10, 87, 107, 5, 75, 85, 29, 84, 31, 30, 76, 20, 23, 12, 112, 104, 11, 79, 26, 81, 8, 37, 17, 15, 40, 34, 91, 72, 101, 32, 36, 100, 88, 96, 6, 24], [49, 63, 105, 117, 111, 47, 45, 109, 42, 41, 22, 110, 106, 99, 57, 118, 86, 51, 113, 64, 0, 35, 89, 38, 52, 80, 95, 16, 120, 28, 119, 61, 94, 115, 93, 25, 124, 116, 125, 123, 53, 121, 122, 46, 82, 102, 18, 114, 62, 83, 39, 126, 19, 27, 58, 13, 55, 1, 59, 48, 92, 33, 103, 127, 50, 98, 43, 44, 77, 87, 108, 54, 56, 4, 14, 65, 74, 90, 67, 31, 78, 66, 97, 3, 71, 73, 7, 10, 85, 68, 112, 2, 60, 29, 21, 23, 69, 9, 30, 76, 11, 107, 84, 26, 20, 5, 70, 81, 75, 12, 34, 104, 17, 91, 37, 15, 40, 32, 36, 24, 101, 79, 8, 72, 100, 96, 88, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 86, 41, 106, 99, 22, 113, 118, 64, 0, 51, 89, 35, 95, 16, 115, 38, 119, 80, 122, 94, 28, 120, 116, 18, 62, 121, 93, 52, 124, 61, 123, 25, 53, 98, 83, 102, 125, 19, 82, 1, 27, 114, 3, 13, 48, 65, 50, 126, 67, 103, 39, 58, 59, 55, 127, 92, 54, 66, 68, 33, 77, 2, 71, 7, 74, 56, 46, 44, 14, 112, 78, 4, 108, 73, 10, 97, 60, 43, 5, 29, 20, 31, 9, 69, 90, 85, 30, 70, 87, 84, 23, 76, 107, 21, 26, 12, 75, 11, 15, 34, 81, 79, 91, 40, 17, 104, 101, 88, 8, 36, 72, 37, 32, 96, 6, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 41, 106, 57, 118, 99, 22, 113, 64, 86, 0, 51, 89, 122, 38, 123, 119, 115, 120, 35, 121, 116, 61, 16, 28, 94, 80, 58, 55, 95, 25, 62, 124, 102, 125, 52, 127, 93, 53, 82, 19, 46, 39, 114, 13, 126, 43, 59, 18, 44, 83, 1, 54, 65, 66, 98, 33, 50, 92, 56, 7, 77, 48, 3, 27, 112, 108, 71, 14, 67, 78, 9, 73, 4, 103, 10, 31, 2, 23, 68, 29, 74, 60, 97, 107, 104, 5, 21, 69, 84, 30, 90, 87, 85, 12, 76, 20, 75, 11, 15, 26, 70, 91, 34, 72, 81, 6, 79, 36, 17, 40, 37, 8, 96, 88, 101, 32, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 110, 106, 57, 99, 22, 118, 86, 64, 113, 51, 0, 89, 35, 119, 38, 61, 52, 16, 28, 94, 120, 124, 116, 122, 80, 121, 62, 115, 82, 48, 125, 55, 25, 95, 93, 102, 127, 123, 54, 53, 39, 58, 46, 44, 83, 98, 114, 27, 59, 19, 18, 65, 1, 13, 112, 126, 33, 14, 2, 50, 77, 74, 3, 108, 68, 103, 73, 92, 43, 97, 71, 4, 7, 66, 78, 31, 60, 9, 10, 21, 56, 67, 6, 5, 90, 75, 23, 87, 76, 84, 69, 11, 29, 107, 20, 30, 85, 12, 79, 104, 26, 17, 72, 81, 36, 34, 37, 15, 101, 8, 70, 88, 40, 91, 96, 24, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 106, 22, 41, 99, 86, 118, 113, 0, 51, 89, 64, 35, 124, 61, 120, 62, 38, 119, 116, 16, 125, 93, 28, 121, 115, 94, 52, 80, 95, 122, 53, 25, 102, 18, 50, 44, 48, 83, 123, 82, 27, 98, 67, 114, 59, 1, 55, 54, 66, 13, 33, 77, 46, 108, 19, 103, 58, 4, 126, 112, 2, 71, 43, 14, 39, 3, 127, 7, 92, 65, 10, 97, 74, 9, 78, 69, 73, 60, 31, 68, 107, 56, 20, 87, 75, 23, 6, 84, 21, 90, 30, 29, 11, 5, 15, 76, 40, 26, 37, 104, 79, 12, 91, 34, 72, 81, 17, 85, 36, 24, 8, 32, 101, 96, 70, 100, 88], [49, 63, 117, 111, 105, 47, 45, 109, 42, 57, 41, 64, 110, 106, 22, 0, 99, 113, 118, 86, 51, 119, 124, 89, 62, 38, 80, 116, 35, 52, 121, 122, 95, 120, 16, 93, 115, 61, 28, 123, 46, 25, 102, 94, 48, 98, 127, 50, 1, 18, 58, 125, 83, 65, 53, 33, 2, 13, 59, 19, 3, 114, 77, 82, 54, 66, 4, 71, 27, 44, 112, 55, 103, 39, 67, 108, 7, 97, 56, 78, 126, 60, 92, 31, 6, 9, 68, 43, 14, 73, 10, 74, 107, 5, 29, 87, 30, 90, 69, 84, 75, 23, 72, 21, 20, 11, 85, 76, 104, 37, 26, 91, 79, 40, 12, 15, 17, 34, 81, 101, 36, 96, 8, 32, 88, 24, 70, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 99, 22, 118, 113, 64, 86, 51, 0, 52, 89, 35, 119, 121, 62, 120, 16, 38, 80, 95, 61, 28, 124, 93, 58, 25, 122, 94, 83, 115, 125, 116, 98, 48, 82, 123, 102, 39, 46, 55, 27, 53, 103, 18, 126, 127, 108, 114, 44, 13, 19, 77, 43, 54, 50, 33, 1, 59, 7, 65, 3, 71, 56, 4, 92, 31, 66, 78, 97, 10, 112, 29, 2, 67, 68, 9, 74, 69, 73, 87, 90, 14, 107, 5, 30, 76, 60, 84, 6, 20, 26, 23, 21, 11, 79, 75, 85, 40, 12, 104, 34, 91, 37, 72, 15, 17, 81, 101, 8, 70, 88, 32, 36, 96, 100, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 106, 57, 22, 99, 118, 0, 86, 51, 89, 113, 52, 35, 64, 16, 119, 62, 38, 28, 121, 95, 116, 115, 80, 122, 124, 82, 25, 120, 94, 61, 93, 125, 53, 58, 27, 46, 18, 98, 55, 126, 48, 50, 102, 83, 114, 19, 33, 39, 3, 77, 123, 71, 13, 108, 127, 54, 1, 103, 68, 44, 78, 74, 65, 59, 7, 97, 112, 67, 43, 66, 14, 4, 92, 60, 56, 10, 73, 69, 9, 21, 84, 76, 31, 2, 30, 11, 87, 107, 29, 5, 6, 85, 23, 90, 75, 20, 26, 12, 34, 72, 37, 79, 15, 40, 101, 8, 36, 104, 81, 17, 91, 70, 88, 32, 96, 24, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 106, 57, 118, 22, 110, 99, 86, 113, 64, 0, 89, 119, 51, 35, 124, 62, 52, 80, 28, 38, 61, 16, 121, 116, 120, 115, 95, 93, 122, 82, 53, 25, 94, 83, 125, 18, 54, 114, 48, 27, 98, 77, 46, 55, 127, 108, 102, 59, 58, 39, 50, 65, 126, 13, 123, 44, 66, 19, 1, 43, 74, 92, 33, 78, 2, 3, 67, 103, 97, 71, 68, 7, 4, 112, 73, 14, 31, 84, 21, 10, 87, 60, 20, 9, 29, 23, 11, 90, 5, 75, 69, 107, 56, 30, 76, 26, 40, 12, 34, 70, 85, 6, 104, 91, 15, 37, 101, 81, 79, 8, 72, 17, 32, 96, 88, 24, 100, 36], [49, 63, 117, 105, 111, 47, 45, 109, 42, 41, 99, 110, 57, 118, 22, 106, 86, 0, 120, 113, 64, 89, 51, 35, 119, 124, 62, 61, 52, 95, 16, 38, 80, 28, 115, 94, 116, 93, 122, 126, 121, 25, 102, 123, 58, 48, 83, 18, 53, 125, 114, 82, 55, 27, 127, 54, 98, 46, 33, 65, 59, 44, 77, 13, 103, 108, 2, 39, 19, 97, 1, 60, 92, 10, 43, 7, 50, 112, 67, 3, 71, 74, 78, 87, 31, 14, 4, 73, 68, 90, 66, 69, 56, 29, 9, 107, 11, 21, 30, 20, 23, 5, 85, 70, 76, 40, 84, 12, 26, 37, 75, 91, 17, 104, 15, 81, 36, 34, 8, 101, 100, 72, 79, 88, 32, 96, 6, 24], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 118, 41, 22, 99, 57, 106, 86, 64, 0, 89, 51, 113, 35, 119, 38, 52, 124, 61, 120, 116, 80, 115, 16, 95, 126, 25, 102, 121, 93, 28, 122, 82, 114, 53, 62, 94, 54, 125, 83, 108, 58, 1, 123, 18, 39, 103, 19, 127, 27, 44, 50, 3, 55, 33, 66, 46, 67, 7, 92, 59, 13, 77, 65, 48, 43, 98, 4, 74, 112, 107, 97, 56, 68, 87, 78, 71, 9, 60, 90, 31, 10, 5, 14, 73, 2, 29, 26, 84, 85, 69, 20, 21, 11, 30, 70, 23, 12, 37, 17, 76, 75, 15, 34, 81, 104, 91, 40, 36, 79, 72, 101, 32, 8, 88, 96, 100, 24, 6], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 118, 57, 41, 106, 22, 99, 64, 86, 113, 0, 51, 89, 119, 124, 52, 61, 120, 38, 62, 115, 122, 28, 35, 95, 16, 116, 82, 94, 80, 53, 25, 121, 93, 58, 98, 18, 114, 46, 19, 44, 126, 83, 125, 33, 123, 77, 1, 13, 127, 27, 59, 54, 55, 102, 9, 108, 39, 50, 3, 60, 7, 65, 78, 68, 71, 43, 4, 92, 66, 48, 10, 67, 73, 2, 14, 103, 31, 56, 97, 74, 5, 84, 70, 112, 87, 29, 23, 21, 69, 90, 107, 11, 15, 30, 85, 20, 76, 75, 79, 26, 37, 8, 12, 104, 17, 40, 101, 91, 34, 72, 36, 81, 32, 88, 6, 24, 96, 100], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 106, 99, 118, 22, 64, 0, 113, 86, 51, 119, 35, 89, 61, 38, 16, 120, 116, 46, 52, 80, 124, 53, 115, 25, 95, 94, 122, 28, 121, 62, 18, 125, 55, 126, 93, 114, 82, 44, 58, 65, 33, 77, 13, 108, 54, 98, 4, 48, 102, 27, 59, 1, 19, 83, 127, 60, 39, 123, 2, 78, 103, 43, 9, 56, 50, 92, 73, 67, 14, 71, 68, 70, 3, 66, 74, 7, 10, 112, 97, 29, 11, 87, 31, 5, 69, 84, 90, 20, 21, 107, 30, 75, 26, 76, 85, 79, 23, 8, 40, 34, 17, 91, 104, 15, 12, 81, 6, 36, 72, 101, 37, 100, 32, 88, 96, 24], [49, 63, 117, 111, 105, 47, 45, 109, 42, 41, 57, 110, 22, 99, 106, 86, 118, 51, 64, 113, 89, 0, 80, 35, 16, 120, 38, 119, 61, 95, 122, 52, 115, 124, 94, 25, 93, 121, 116, 62, 46, 28, 125, 82, 18, 126, 53, 27, 102, 58, 83, 44, 114, 103, 65, 13, 55, 54, 77, 108, 98, 127, 19, 123, 50, 39, 33, 3, 48, 59, 60, 7, 92, 10, 71, 67, 68, 4, 1, 56, 90, 9, 43, 78, 74, 14, 66, 87, 69, 97, 31, 112, 21, 29, 73, 84, 2, 23, 85, 107, 5, 11, 20, 70, 30, 26, 79, 34, 12, 37, 6, 15, 40, 76, 81, 75, 104, 91, 72, 17, 8, 101, 88, 36, 24, 96, 32, 100], [49, 63, 117, 111, 105, 47, 45, 109, 42, 110, 57, 99, 41, 22, 0, 118, 106, 86, 64, 51, 113, 89, 35, 80, 119, 62, 52, 38, 124, 28, 16, 120, 95, 116, 121, 115, 61, 93, 122, 53, 18, 123, 94, 46, 25, 83, 98, 125, 77, 102, 48, 58, 54, 108, 82, 4, 55, 126, 1, 44, 27, 7, 59, 114, 127, 68, 39, 66, 103, 65, 50, 92, 71, 43, 10, 13, 33, 73, 2, 3, 74, 112, 90, 9, 67, 19, 60, 14, 78, 11, 87, 20, 29, 31, 97, 12, 21, 56, 6, 107, 5, 104, 69, 84, 40, 30, 75, 23, 85, 34, 76, 79, 26, 91, 70, 8, 37, 101, 81, 72, 17, 24, 88, 15, 32, 100, 36, 96], [49, 63, 117, 105, 111, 47, 45, 109, 42, 110, 41, 57, 22, 118, 106, 99, 86, 113, 51, 35, 119, 89, 64, 0, 61, 115, 38, 16, 52, 80, 62, 28, 53, 120, 95, 124, 121, 116, 122, 94, 25, 125, 93, 77, 114, 18, 55, 82, 102, 44, 27, 59, 98, 83, 48, 50, 126, 33, 19, 54, 123, 13, 7, 58, 65, 108, 10, 46, 3, 39, 127, 97, 71, 92, 103, 112, 1, 68, 67, 43, 78, 74, 14, 31, 73, 2, 87, 66, 4, 5, 90, 60, 11, 21, 85, 56, 23, 84, 107, 6, 26, 20, 12, 9, 69, 75, 29, 76, 34, 30, 104, 40, 79, 81, 15, 91, 17, 72, 8, 101, 37, 36, 32, 24, 96, 88, 100, 70]], "model.layers.30.self_attn.q_proj": [[61, 38, 48, 50, 97, 49, 111, 123, 83, 64, 89, 1, 127, 108, 79, 41, 86, 28, 94, 114, 14, 116, 102, 106, 17, 119, 67, 57, 54, 4, 40, 13, 59, 65, 85, 73, 109, 39, 52, 37, 58, 126, 12, 68, 60, 92, 84, 45, 120, 11, 125, 76, 8, 72, 44, 55, 87, 105, 80, 121, 43, 10, 2, 62, 110, 63, 74, 46, 118, 51, 81, 42, 101, 23, 3, 24, 9, 56, 71, 6, 112, 124, 5, 31, 15, 53, 35, 107, 0, 34, 117, 115, 21, 18, 88, 36, 113, 100, 7, 27, 47, 104, 16, 69, 93, 26, 90, 19, 95, 77, 103, 122, 20, 82, 32, 96, 66, 91, 99, 30, 29, 75, 22, 98, 33, 70, 78, 25], [61, 38, 50, 48, 97, 89, 112, 83, 55, 108, 17, 11, 60, 114, 72, 57, 85, 70, 28, 14, 111, 15, 62, 101, 3, 92, 123, 66, 25, 127, 34, 6, 126, 113, 36, 106, 86, 59, 52, 87, 69, 124, 39, 102, 45, 58, 0, 120, 94, 125, 63, 37, 29, 118, 104, 35, 43, 109, 47, 115, 10, 64, 119, 121, 56, 122, 51, 49, 40, 22, 27, 95, 53, 4, 46, 116, 90, 103, 21, 24, 32, 107, 81, 110, 23, 117, 26, 44, 19, 99, 16, 76, 42, 105, 41, 98, 88, 77, 30, 67, 20, 54, 73, 31, 93, 100, 65, 96, 18, 8, 84, 82, 12, 5, 80, 1, 78, 2, 91, 75, 7, 33, 71, 79, 13, 68, 74, 9], [50, 61, 38, 111, 48, 55, 97, 89, 106, 45, 86, 127, 60, 41, 125, 123, 124, 120, 17, 85, 102, 108, 70, 121, 62, 112, 92, 59, 52, 58, 51, 43, 119, 114, 49, 126, 66, 87, 11, 57, 83, 14, 53, 46, 39, 28, 40, 101, 115, 113, 104, 118, 117, 47, 56, 116, 122, 105, 63, 54, 44, 110, 103, 91, 3, 25, 72, 107, 109, 32, 36, 42, 0, 64, 34, 15, 6, 100, 30, 35, 94, 9, 33, 22, 37, 81, 90, 10, 31, 21, 95, 27, 96, 13, 12, 24, 99, 88, 26, 78, 84, 93, 19, 69, 67, 4, 98, 8, 16, 23, 18, 76, 29, 82, 77, 65, 73, 20, 80, 2, 7, 5, 79, 75, 1, 71, 68, 74], [61, 50, 111, 38, 112, 55, 60, 114, 124, 48, 110, 125, 97, 123, 109, 92, 53, 127, 89, 58, 52, 51, 62, 121, 44, 28, 118, 59, 41, 108, 119, 120, 45, 102, 46, 63, 126, 122, 116, 47, 104, 113, 56, 117, 54, 43, 101, 105, 57, 107, 37, 42, 29, 115, 88, 86, 106, 103, 40, 80, 100, 39, 34, 91, 49, 35, 30, 84, 36, 85, 83, 93, 25, 98, 32, 95, 99, 24, 16, 94, 17, 31, 23, 18, 22, 15, 69, 96, 21, 76, 82, 11, 90, 33, 87, 77, 20, 27, 81, 26, 4, 13, 19, 12, 68, 7, 73, 2, 71, 9, 5, 79, 75, 14, 10, 72, 74, 66, 70, 6, 0, 78, 1, 64, 67, 8, 65, 3], [46, 110, 34, 92, 119, 86, 24, 90, 83, 113, 61, 49, 54, 95, 82, 94, 31, 38, 70, 85, 74, 0, 121, 13, 7, 120, 127, 114, 12, 8, 101, 9, 59, 29, 79, 17, 47, 112, 23, 1, 48, 98, 53, 80, 51, 52, 4, 63, 2, 109, 67, 97, 55, 26, 42, 125, 58, 116, 36, 117, 60, 40, 41, 126, 108, 18, 62, 56, 124, 111, 122, 43, 115, 123, 73, 57, 105, 87, 118, 91, 81, 45, 25, 102, 28, 11, 19, 39, 37, 106, 64, 107, 100, 44, 104, 66, 93, 16, 27, 103, 20, 33, 88, 50, 96, 32, 68, 89, 35, 22, 84, 14, 5, 99, 21, 69, 30, 65, 78, 3, 10, 15, 6, 77, 71, 75, 76, 72], [46, 110, 34, 90, 86, 92, 24, 79, 18, 113, 31, 95, 83, 17, 119, 9, 70, 54, 4, 85, 42, 74, 55, 56, 61, 12, 120, 59, 58, 107, 98, 117, 7, 63, 67, 115, 13, 44, 50, 49, 116, 88, 114, 38, 16, 36, 40, 96, 29, 73, 8, 94, 91, 20, 48, 111, 52, 23, 97, 118, 37, 81, 2, 0, 109, 100, 57, 122, 125, 43, 45, 1, 51, 39, 127, 103, 93, 104, 47, 41, 27, 106, 5, 123, 33, 126, 101, 112, 124, 80, 26, 62, 28, 14, 102, 15, 32, 60, 87, 77, 108, 11, 78, 75, 105, 99, 84, 53, 76, 25, 69, 121, 35, 89, 30, 82, 21, 19, 72, 10, 68, 64, 6, 22, 66, 3, 65, 71], [46, 34, 110, 113, 120, 92, 24, 83, 90, 42, 31, 86, 17, 94, 8, 79, 14, 54, 118, 112, 117, 15, 52, 9, 66, 100, 45, 85, 36, 55, 38, 70, 13, 76, 84, 58, 11, 60, 10, 51, 98, 3, 56, 4, 63, 0, 29, 95, 49, 65, 32, 12, 48, 127, 119, 124, 122, 39, 59, 116, 47, 104, 41, 23, 43, 53, 126, 19, 44, 107, 61, 75, 7, 62, 1, 111, 67, 78, 106, 22, 114, 68, 109, 105, 35, 108, 18, 80, 82, 37, 123, 115, 96, 91, 102, 88, 97, 28, 93, 26, 6, 103, 50, 33, 40, 27, 121, 69, 87, 5, 20, 25, 57, 71, 101, 81, 30, 2, 89, 21, 125, 16, 77, 74, 73, 99, 72, 64], [46, 110, 34, 92, 90, 24, 86, 49, 83, 69, 13, 113, 114, 95, 61, 119, 31, 23, 85, 56, 94, 53, 80, 100, 17, 98, 54, 108, 9, 125, 57, 52, 103, 79, 27, 115, 74, 63, 50, 11, 18, 126, 111, 7, 41, 99, 122, 25, 28, 120, 1, 67, 55, 88, 29, 4, 26, 87, 62, 101, 45, 39, 33, 117, 104, 36, 43, 60, 102, 58, 38, 116, 118, 127, 59, 40, 51, 96, 44, 124, 123, 109, 73, 42, 121, 15, 70, 107, 5, 97, 106, 30, 112, 91, 78, 48, 47, 37, 89, 35, 32, 76, 105, 12, 0, 20, 21, 81, 75, 93, 16, 71, 19, 14, 8, 84, 64, 68, 10, 82, 72, 2, 65, 3, 22, 77, 6, 66], [42, 63, 117, 97, 27, 86, 106, 89, 16, 78, 31, 84, 115, 52, 11, 99, 81, 76, 53, 119, 96, 56, 49, 13, 10, 88, 100, 26, 73, 92, 121, 55, 118, 7, 24, 51, 50, 61, 114, 57, 8, 102, 127, 90, 20, 122, 41, 23, 46, 19, 116, 60, 28, 95, 82, 38, 6, 25, 36, 58, 94, 33, 120, 91, 62, 22, 35, 21, 54, 110, 83, 48, 9, 59, 15, 18, 66, 5, 125, 111, 126, 109, 17, 124, 72, 47, 39, 103, 112, 107, 108, 85, 113, 44, 123, 87, 40, 93, 14, 69, 80, 29, 45, 4, 77, 70, 30, 105, 43, 32, 74, 34, 101, 104, 98, 3, 12, 79, 37, 75, 68, 0, 71, 65, 67, 2, 1, 64], [42, 63, 117, 97, 35, 16, 56, 84, 27, 83, 86, 102, 101, 94, 92, 18, 19, 57, 11, 64, 106, 121, 49, 116, 93, 52, 100, 4, 37, 103, 24, 29, 98, 81, 72, 85, 65, 77, 99, 127, 9, 50, 76, 88, 70, 54, 55, 122, 61, 31, 73, 32, 96, 51, 119, 118, 58, 120, 68, 115, 78, 79, 95, 114, 82, 26, 74, 90, 109, 111, 28, 20, 125, 124, 23, 25, 41, 60, 7, 59, 48, 34, 3, 10, 80, 47, 46, 43, 36, 113, 67, 53, 5, 108, 126, 66, 1, 22, 44, 110, 105, 89, 15, 39, 21, 112, 38, 12, 30, 91, 8, 45, 62, 40, 104, 33, 123, 14, 6, 69, 13, 0, 107, 2, 17, 87, 71, 75], [42, 63, 11, 81, 78, 7, 97, 117, 84, 1, 86, 106, 27, 102, 94, 4, 55, 0, 65, 114, 6, 116, 98, 16, 9, 21, 13, 2, 100, 66, 58, 50, 92, 119, 89, 14, 30, 19, 101, 115, 93, 8, 64, 67, 52, 35, 18, 77, 76, 71, 82, 73, 75, 88, 31, 108, 12, 38, 70, 122, 103, 68, 72, 36, 20, 83, 10, 3, 91, 17, 23, 74, 80, 24, 79, 15, 32, 26, 5, 22, 29, 28, 90, 85, 120, 46, 96, 53, 39, 61, 69, 25, 34, 99, 49, 95, 118, 59, 87, 104, 121, 123, 57, 111, 47, 109, 126, 56, 37, 54, 124, 62, 45, 110, 125, 127, 48, 33, 60, 51, 105, 44, 41, 40, 43, 113, 107, 112], [63, 42, 55, 57, 49, 56, 51, 121, 112, 52, 122, 54, 116, 118, 127, 61, 120, 50, 48, 111, 59, 125, 47, 124, 108, 113, 115, 126, 60, 117, 114, 110, 97, 62, 44, 53, 85, 58, 123, 94, 46, 109, 38, 45, 24, 43, 86, 107, 41, 119, 27, 33, 102, 29, 106, 40, 39, 100, 28, 92, 104, 105, 18, 36, 26, 103, 22, 31, 90, 99, 101, 35, 88, 91, 21, 81, 37, 96, 84, 77, 30, 87, 93, 32, 15, 98, 95, 34, 20, 17, 82, 83, 19, 79, 25, 23, 80, 13, 16, 89, 12, 73, 75, 76, 11, 70, 78, 68, 74, 1, 10, 14, 0, 4, 8, 5, 67, 72, 9, 7, 6, 69, 71, 65, 66, 64, 2, 3], [111, 44, 61, 125, 116, 84, 108, 20, 124, 97, 52, 75, 100, 60, 11, 120, 117, 48, 115, 46, 47, 106, 56, 50, 49, 123, 121, 119, 28, 112, 87, 70, 63, 62, 55, 53, 59, 122, 127, 118, 110, 57, 113, 42, 90, 126, 114, 39, 51, 92, 45, 88, 54, 58, 93, 107, 109, 105, 32, 41, 98, 104, 91, 40, 79, 102, 25, 43, 15, 17, 86, 30, 37, 103, 27, 38, 31, 26, 101, 19, 35, 81, 33, 34, 36, 94, 99, 74, 23, 95, 83, 96, 22, 29, 89, 10, 2, 82, 21, 6, 16, 13, 85, 66, 24, 72, 64, 67, 18, 0, 76, 14, 65, 77, 68, 80, 5, 71, 9, 78, 4, 1, 12, 8, 73, 3, 7, 69], [44, 111, 116, 88, 18, 97, 61, 16, 78, 9, 115, 90, 51, 108, 89, 30, 109, 100, 71, 47, 4, 125, 102, 85, 26, 76, 2, 20, 120, 83, 10, 86, 79, 80, 73, 56, 68, 92, 23, 82, 94, 93, 112, 27, 55, 95, 11, 58, 66, 6, 40, 64, 118, 14, 87, 70, 52, 63, 53, 42, 127, 122, 126, 65, 7, 124, 67, 41, 50, 49, 12, 19, 24, 119, 91, 113, 28, 29, 74, 117, 81, 101, 59, 25, 1, 114, 103, 13, 38, 43, 21, 31, 17, 32, 107, 57, 123, 48, 96, 98, 0, 121, 72, 62, 34, 60, 84, 105, 106, 77, 46, 22, 8, 35, 54, 104, 5, 3, 45, 39, 36, 99, 33, 110, 37, 15, 75, 69], [44, 61, 108, 125, 97, 106, 115, 85, 88, 100, 116, 47, 51, 49, 120, 117, 126, 46, 124, 52, 87, 28, 93, 112, 59, 111, 50, 48, 119, 55, 17, 122, 92, 76, 57, 127, 62, 90, 123, 113, 118, 114, 56, 63, 60, 121, 21, 26, 53, 32, 54, 107, 58, 105, 45, 91, 40, 30, 42, 12, 104, 109, 86, 41, 39, 79, 27, 74, 110, 94, 103, 5, 34, 102, 43, 19, 23, 69, 38, 98, 83, 36, 37, 33, 31, 15, 101, 81, 95, 82, 99, 22, 35, 67, 72, 25, 89, 84, 96, 13, 2, 10, 29, 64, 18, 24, 16, 71, 20, 7, 77, 80, 3, 0, 66, 8, 78, 14, 65, 70, 4, 9, 11, 1, 68, 6, 73, 75], [44, 125, 111, 108, 61, 116, 115, 97, 51, 117, 87, 52, 122, 88, 106, 112, 49, 55, 126, 60, 100, 120, 93, 47, 59, 124, 58, 50, 113, 57, 48, 119, 118, 62, 127, 28, 46, 45, 92, 56, 63, 121, 114, 123, 90, 42, 17, 53, 74, 39, 41, 107, 104, 109, 19, 54, 86, 32, 79, 30, 91, 102, 40, 105, 13, 98, 110, 103, 26, 94, 15, 43, 95, 38, 36, 83, 35, 25, 101, 27, 37, 34, 89, 81, 96, 22, 23, 31, 33, 99, 29, 77, 67, 21, 85, 72, 84, 82, 10, 16, 2, 64, 24, 0, 9, 18, 70, 20, 80, 65, 8, 76, 5, 71, 78, 3, 14, 66, 12, 7, 11, 69, 75, 1, 73, 4, 68, 6], [113, 103, 122, 124, 52, 33, 55, 93, 119, 107, 47, 24, 42, 26, 48, 31, 112, 43, 54, 104, 90, 117, 123, 56, 45, 85, 41, 57, 108, 118, 116, 53, 87, 83, 115, 121, 37, 62, 120, 63, 35, 61, 99, 22, 29, 125, 114, 91, 59, 23, 39, 40, 21, 28, 60, 105, 49, 44, 102, 111, 101, 98, 51, 92, 127, 126, 106, 110, 58, 32, 36, 50, 38, 88, 46, 34, 97, 95, 100, 27, 11, 109, 96, 84, 30, 89, 18, 94, 82, 8, 10, 16, 72, 19, 14, 20, 80, 13, 71, 25, 75, 86, 77, 74, 81, 15, 78, 70, 5, 7, 68, 65, 2, 76, 6, 69, 0, 66, 64, 79, 12, 67, 17, 4, 3, 73, 1, 9], [122, 103, 33, 52, 93, 31, 117, 119, 91, 83, 22, 24, 54, 112, 111, 124, 90, 85, 26, 60, 105, 43, 27, 125, 42, 107, 45, 39, 97, 127, 29, 113, 20, 114, 96, 123, 73, 92, 118, 53, 78, 46, 61, 120, 110, 44, 58, 86, 116, 40, 115, 102, 48, 51, 21, 49, 47, 38, 62, 50, 55, 109, 101, 108, 25, 121, 126, 95, 35, 81, 56, 99, 37, 36, 63, 19, 30, 41, 28, 15, 57, 13, 34, 104, 89, 84, 87, 94, 32, 100, 106, 9, 82, 80, 23, 75, 98, 18, 14, 16, 59, 72, 88, 77, 5, 6, 64, 70, 67, 8, 1, 11, 0, 69, 10, 65, 68, 2, 17, 71, 4, 3, 79, 66, 7, 76, 12, 74], [57, 122, 103, 52, 113, 110, 105, 112, 22, 33, 54, 38, 101, 59, 116, 125, 43, 111, 60, 26, 109, 118, 50, 48, 97, 55, 3, 34, 39, 47, 126, 92, 78, 45, 127, 24, 40, 124, 31, 114, 61, 64, 120, 90, 75, 65, 119, 49, 123, 108, 83, 115, 18, 67, 20, 68, 71, 99, 36, 107, 106, 104, 44, 62, 63, 46, 100, 53, 96, 102, 25, 41, 42, 117, 19, 95, 51, 58, 74, 84, 121, 93, 56, 37, 21, 89, 87, 27, 94, 30, 32, 13, 35, 73, 86, 88, 82, 85, 98, 72, 28, 91, 10, 14, 23, 4, 77, 29, 9, 6, 8, 2, 1, 7, 0, 69, 66, 80, 70, 11, 16, 12, 15, 5, 79, 81, 17, 76], [103, 113, 93, 122, 24, 33, 15, 81, 76, 29, 83, 17, 57, 85, 10, 70, 104, 88, 21, 36, 34, 48, 19, 7, 22, 84, 97, 26, 66, 107, 5, 35, 12, 27, 124, 87, 42, 54, 30, 38, 55, 90, 23, 43, 16, 82, 74, 102, 75, 8, 20, 121, 63, 86, 49, 119, 94, 77, 45, 91, 68, 127, 14, 71, 62, 117, 73, 0, 79, 61, 13, 25, 32, 28, 2, 106, 80, 111, 11, 69, 116, 18, 47, 98, 59, 52, 31, 58, 89, 3, 72, 67, 92, 4, 78, 37, 96, 40, 99, 123, 100, 95, 118, 110, 39, 120, 112, 6, 46, 101, 65, 9, 64, 105, 60, 53, 126, 108, 44, 125, 114, 1, 41, 50, 115, 51, 56, 109], [56, 39, 60, 53, 117, 122, 94, 127, 109, 123, 116, 92, 58, 50, 124, 61, 103, 105, 59, 51, 55, 52, 113, 33, 111, 90, 114, 119, 115, 62, 97, 110, 54, 42, 121, 63, 47, 96, 57, 98, 34, 43, 95, 25, 36, 44, 45, 26, 108, 46, 112, 19, 126, 28, 49, 91, 85, 125, 30, 107, 118, 40, 38, 120, 102, 37, 106, 104, 88, 48, 41, 100, 23, 35, 101, 99, 27, 31, 29, 81, 20, 32, 83, 86, 87, 84, 89, 22, 93, 24, 76, 21, 78, 14, 17, 15, 75, 16, 66, 18, 79, 71, 74, 9, 0, 5, 4, 12, 67, 11, 80, 65, 82, 73, 7, 69, 70, 1, 2, 72, 77, 8, 68, 64, 3, 10, 6, 13], [53, 39, 60, 123, 61, 124, 55, 56, 50, 109, 51, 116, 54, 127, 120, 57, 122, 63, 97, 115, 47, 52, 59, 121, 90, 112, 119, 26, 58, 113, 117, 62, 114, 108, 126, 28, 125, 44, 49, 118, 107, 43, 94, 48, 110, 46, 19, 45, 105, 106, 111, 23, 38, 42, 41, 103, 101, 29, 100, 40, 85, 33, 104, 87, 102, 34, 25, 78, 35, 99, 22, 92, 95, 37, 36, 86, 30, 96, 98, 31, 91, 32, 83, 93, 21, 24, 17, 18, 27, 81, 20, 15, 14, 84, 75, 88, 89, 11, 79, 72, 76, 82, 16, 80, 12, 5, 77, 9, 74, 70, 66, 13, 67, 7, 6, 71, 8, 2, 69, 3, 10, 4, 65, 73, 0, 68, 1, 64], [39, 53, 56, 25, 18, 77, 16, 23, 74, 71, 4, 60, 97, 70, 64, 9, 0, 91, 52, 29, 31, 76, 34, 92, 67, 65, 122, 1, 100, 117, 30, 36, 15, 84, 13, 124, 40, 68, 2, 12, 82, 125, 99, 20, 10, 5, 17, 69, 98, 8, 94, 55, 6, 104, 79, 19, 81, 73, 14, 85, 42, 61, 88, 32, 7, 50, 86, 11, 90, 26, 3, 72, 24, 66, 21, 83, 75, 27, 80, 37, 107, 93, 38, 118, 87, 22, 114, 89, 113, 35, 78, 28, 41, 95, 63, 46, 126, 109, 96, 127, 51, 101, 58, 119, 33, 116, 123, 120, 59, 105, 45, 108, 102, 47, 112, 106, 54, 115, 62, 43, 57, 121, 44, 111, 110, 49, 48, 103], [39, 56, 53, 25, 18, 16, 23, 34, 97, 30, 83, 77, 52, 60, 122, 76, 29, 100, 74, 86, 93, 124, 61, 11, 94, 104, 10, 9, 71, 15, 22, 21, 13, 36, 55, 127, 90, 20, 26, 108, 91, 27, 119, 19, 80, 89, 98, 35, 40, 117, 79, 88, 92, 82, 59, 70, 107, 31, 12, 65, 51, 85, 84, 28, 41, 17, 4, 78, 102, 14, 7, 64, 87, 67, 75, 114, 72, 99, 8, 69, 81, 0, 37, 46, 32, 57, 24, 1, 112, 2, 125, 96, 73, 95, 54, 58, 115, 33, 126, 50, 118, 3, 101, 113, 38, 116, 42, 105, 6, 111, 63, 45, 120, 62, 44, 123, 49, 106, 68, 66, 103, 48, 110, 109, 5, 47, 43, 121], [123, 41, 34, 120, 115, 107, 58, 60, 12, 48, 51, 90, 127, 125, 82, 70, 23, 109, 73, 116, 32, 47, 77, 121, 62, 29, 106, 84, 126, 14, 59, 55, 86, 24, 108, 80, 46, 26, 45, 57, 63, 52, 54, 122, 37, 110, 4, 36, 92, 38, 50, 61, 105, 56, 117, 79, 113, 111, 112, 119, 104, 103, 118, 42, 69, 124, 15, 43, 94, 49, 53, 67, 66, 101, 85, 44, 93, 99, 114, 7, 89, 33, 88, 39, 97, 1, 95, 31, 28, 30, 91, 100, 35, 40, 64, 102, 21, 22, 74, 10, 13, 19, 83, 78, 8, 98, 9, 20, 76, 6, 16, 18, 87, 17, 25, 96, 27, 2, 81, 0, 71, 65, 11, 75, 5, 3, 72, 68], [41, 123, 34, 107, 65, 66, 0, 29, 90, 121, 14, 37, 1, 74, 2, 11, 59, 93, 70, 80, 124, 72, 81, 69, 68, 105, 26, 47, 4, 10, 60, 75, 58, 12, 84, 3, 24, 86, 32, 109, 17, 82, 103, 7, 115, 77, 111, 127, 125, 35, 13, 21, 5, 15, 120, 73, 46, 110, 51, 71, 45, 8, 67, 94, 52, 97, 126, 118, 55, 87, 48, 56, 106, 116, 42, 62, 61, 57, 108, 101, 63, 112, 38, 33, 98, 28, 50, 122, 31, 44, 117, 113, 43, 53, 6, 18, 64, 79, 100, 119, 88, 92, 104, 54, 114, 49, 16, 99, 30, 27, 85, 20, 39, 83, 78, 91, 36, 76, 40, 95, 19, 25, 89, 9, 96, 23, 102, 22], [123, 41, 109, 34, 59, 29, 115, 90, 58, 23, 127, 61, 120, 51, 125, 43, 126, 111, 103, 108, 46, 54, 113, 122, 47, 121, 57, 110, 116, 56, 86, 107, 48, 50, 124, 52, 118, 63, 38, 117, 12, 55, 112, 80, 53, 119, 84, 42, 89, 93, 60, 21, 62, 49, 45, 114, 102, 106, 104, 99, 70, 44, 101, 26, 77, 6, 7, 4, 82, 25, 14, 36, 39, 32, 40, 37, 31, 94, 88, 35, 65, 33, 100, 97, 73, 92, 10, 105, 22, 74, 3, 27, 83, 96, 87, 0, 67, 20, 30, 19, 17, 91, 15, 64, 85, 98, 95, 28, 24, 72, 66, 79, 68, 81, 13, 18, 75, 11, 69, 9, 71, 78, 16, 76, 1, 8, 2, 5], [123, 41, 109, 34, 107, 59, 58, 121, 23, 47, 60, 127, 29, 115, 84, 55, 90, 120, 125, 106, 56, 104, 43, 112, 12, 80, 126, 88, 61, 46, 50, 62, 103, 53, 122, 48, 116, 124, 57, 51, 32, 54, 45, 52, 111, 110, 63, 93, 113, 26, 38, 73, 114, 117, 42, 118, 82, 99, 108, 36, 101, 119, 86, 14, 37, 35, 89, 49, 24, 102, 105, 70, 100, 74, 17, 77, 65, 25, 44, 39, 21, 85, 97, 83, 31, 4, 94, 40, 67, 95, 7, 3, 92, 30, 76, 91, 79, 33, 0, 16, 9, 19, 28, 98, 20, 27, 96, 18, 22, 6, 87, 11, 78, 72, 15, 69, 81, 64, 13, 75, 66, 1, 71, 10, 68, 8, 5, 2], [127, 126, 63, 38, 120, 82, 20, 97, 91, 80, 9, 11, 13, 21, 56, 25, 69, 14, 30, 15, 29, 24, 90, 26, 75, 8, 94, 86, 102, 99, 124, 2, 88, 37, 93, 39, 95, 76, 103, 84, 78, 3, 7, 74, 43, 98, 71, 46, 68, 114, 17, 54, 121, 18, 47, 67, 125, 108, 19, 105, 0, 101, 85, 119, 58, 77, 44, 5, 49, 10, 55, 111, 42, 34, 51, 89, 96, 83, 31, 27, 62, 92, 107, 123, 52, 57, 35, 53, 48, 28, 110, 61, 87, 6, 72, 12, 60, 116, 16, 33, 64, 32, 59, 22, 118, 73, 70, 23, 66, 115, 79, 122, 100, 117, 112, 50, 41, 109, 36, 113, 104, 65, 81, 4, 40, 45, 106, 1], [63, 38, 126, 123, 119, 54, 56, 120, 62, 110, 47, 124, 61, 59, 125, 118, 60, 122, 51, 121, 115, 117, 111, 58, 48, 53, 50, 46, 49, 55, 23, 127, 116, 102, 109, 112, 97, 113, 114, 57, 52, 44, 87, 29, 45, 81, 93, 43, 26, 105, 92, 86, 108, 94, 88, 107, 41, 106, 17, 42, 104, 99, 103, 22, 40, 33, 21, 90, 39, 84, 101, 37, 95, 15, 83, 18, 35, 98, 79, 30, 74, 36, 31, 96, 100, 89, 70, 34, 27, 25, 28, 76, 0, 32, 80, 20, 6, 85, 19, 91, 65, 10, 24, 82, 78, 16, 12, 13, 1, 75, 14, 8, 67, 66, 64, 4, 69, 73, 68, 3, 9, 11, 7, 71, 2, 5, 77, 72], [126, 127, 63, 38, 123, 54, 119, 56, 62, 110, 125, 120, 61, 47, 60, 124, 59, 118, 46, 122, 49, 121, 51, 111, 115, 53, 55, 48, 58, 50, 116, 117, 113, 112, 97, 52, 44, 23, 114, 102, 29, 57, 43, 109, 45, 105, 93, 81, 86, 94, 26, 108, 104, 92, 88, 87, 107, 41, 42, 40, 99, 90, 103, 17, 22, 106, 33, 39, 31, 37, 74, 98, 101, 30, 15, 21, 95, 83, 36, 100, 35, 70, 96, 6, 34, 84, 27, 32, 20, 28, 79, 18, 25, 89, 0, 65, 16, 76, 19, 85, 80, 14, 91, 12, 82, 10, 24, 67, 78, 1, 4, 13, 64, 8, 75, 66, 68, 9, 3, 7, 73, 77, 69, 72, 5, 71, 2, 11], [126, 63, 38, 127, 16, 120, 20, 88, 78, 97, 82, 91, 75, 56, 30, 47, 73, 99, 21, 29, 54, 32, 13, 125, 39, 7, 15, 86, 119, 95, 62, 5, 90, 124, 25, 10, 8, 66, 80, 51, 69, 76, 0, 123, 9, 67, 84, 102, 111, 24, 77, 49, 58, 98, 53, 103, 109, 43, 114, 28, 26, 61, 44, 27, 79, 60, 50, 34, 18, 104, 14, 117, 118, 71, 59, 107, 19, 115, 70, 122, 57, 65, 121, 48, 17, 12, 85, 87, 89, 31, 113, 55, 83, 101, 112, 92, 100, 72, 52, 116, 23, 4, 110, 94, 22, 74, 35, 11, 46, 6, 45, 96, 33, 37, 105, 41, 108, 93, 68, 106, 40, 36, 81, 3, 1, 42, 2, 64]], "model.layers.30.self_attn.k_proj": [[61, 102, 50, 33, 48, 89, 86, 112, 92, 85, 14, 83, 60, 0, 58, 52, 113, 2, 108, 94, 11, 17, 114, 40, 120, 121, 47, 62, 65, 59, 55, 45, 123, 126, 72, 57, 53, 43, 116, 127, 56, 119, 124, 125, 63, 118, 51, 69, 98, 42, 38, 122, 117, 67, 46, 115, 44, 77, 49, 109, 41, 106, 80, 110, 54, 68, 107, 103, 100, 104, 29, 37, 105, 101, 9, 39, 10, 87, 91, 7, 6, 35, 64, 15, 31, 99, 111, 12, 36, 71, 4, 5, 84, 93, 70, 30, 90, 96, 32, 88, 27, 95, 23, 26, 34, 73, 82, 18, 28, 13, 66, 22, 76, 24, 16, 20, 21, 3, 74, 75, 1, 78, 25, 8, 19, 79, 97, 81], [110, 46, 98, 90, 0, 92, 24, 13, 31, 67, 18, 86, 2, 70, 113, 74, 1, 15, 83, 52, 17, 8, 4, 12, 16, 80, 51, 105, 9, 49, 84, 59, 21, 79, 41, 123, 116, 11, 64, 68, 94, 85, 7, 23, 54, 127, 119, 109, 29, 96, 124, 58, 115, 22, 122, 34, 100, 97, 101, 30, 38, 48, 14, 53, 45, 60, 126, 106, 118, 104, 39, 40, 35, 117, 56, 108, 120, 62, 63, 107, 47, 93, 37, 114, 5, 44, 87, 50, 102, 33, 43, 78, 111, 91, 27, 125, 42, 10, 89, 103, 112, 55, 36, 99, 32, 73, 69, 121, 61, 76, 26, 57, 95, 66, 71, 25, 28, 65, 75, 19, 20, 77, 81, 72, 6, 3, 88, 82], [106, 63, 86, 33, 117, 27, 11, 77, 78, 9, 4, 81, 7, 0, 84, 1, 55, 18, 49, 57, 31, 122, 16, 121, 95, 30, 72, 42, 116, 85, 56, 50, 59, 114, 119, 52, 51, 87, 58, 110, 54, 53, 43, 120, 34, 111, 44, 48, 118, 61, 124, 112, 115, 90, 113, 125, 103, 46, 36, 47, 62, 24, 29, 66, 41, 107, 105, 127, 37, 64, 45, 65, 109, 123, 3, 28, 126, 60, 104, 6, 67, 102, 10, 100, 108, 68, 32, 38, 99, 39, 35, 92, 40, 80, 94, 12, 5, 101, 73, 26, 23, 69, 93, 25, 76, 15, 8, 88, 20, 89, 70, 98, 91, 96, 83, 82, 14, 19, 74, 79, 13, 71, 2, 21, 17, 22, 75, 97], [108, 36, 44, 33, 86, 115, 47, 94, 111, 127, 17, 61, 52, 59, 26, 28, 41, 57, 96, 114, 126, 119, 49, 112, 104, 125, 117, 122, 118, 37, 48, 56, 120, 42, 62, 107, 60, 13, 55, 63, 50, 21, 123, 58, 124, 121, 88, 46, 43, 113, 39, 51, 106, 53, 102, 45, 40, 110, 109, 116, 54, 79, 101, 100, 105, 90, 31, 103, 72, 38, 5, 95, 98, 85, 93, 99, 76, 19, 14, 34, 24, 35, 27, 84, 87, 30, 91, 16, 12, 92, 89, 80, 32, 67, 97, 11, 3, 78, 74, 25, 65, 29, 23, 9, 75, 22, 81, 82, 18, 0, 20, 77, 2, 70, 83, 8, 73, 71, 6, 15, 7, 66, 1, 10, 68, 64, 4, 69], [39, 122, 113, 97, 49, 29, 57, 112, 26, 22, 88, 27, 73, 124, 24, 19, 93, 78, 48, 102, 59, 30, 127, 81, 123, 21, 10, 17, 47, 105, 118, 117, 85, 1, 83, 40, 54, 15, 111, 61, 107, 119, 63, 76, 43, 115, 95, 100, 60, 58, 109, 114, 52, 106, 126, 31, 12, 62, 55, 5, 53, 87, 41, 110, 37, 0, 121, 66, 120, 46, 7, 108, 4, 92, 104, 116, 51, 44, 103, 36, 125, 89, 79, 84, 101, 99, 96, 50, 42, 34, 25, 3, 32, 45, 13, 94, 56, 98, 14, 35, 38, 82, 77, 90, 75, 91, 80, 8, 69, 28, 20, 9, 18, 23, 16, 11, 33, 72, 71, 68, 86, 67, 6, 64, 70, 65, 2, 74], [103, 56, 53, 25, 33, 18, 16, 94, 77, 74, 60, 90, 23, 4, 70, 71, 87, 76, 22, 119, 0, 65, 109, 124, 55, 89, 122, 50, 120, 63, 52, 75, 123, 51, 86, 85, 67, 91, 93, 113, 44, 54, 35, 125, 127, 117, 114, 100, 47, 61, 116, 29, 9, 118, 36, 28, 19, 101, 59, 107, 106, 57, 98, 43, 62, 32, 72, 111, 49, 121, 42, 48, 58, 110, 8, 115, 45, 78, 108, 126, 11, 81, 41, 104, 46, 96, 105, 112, 79, 102, 2, 34, 5, 95, 92, 17, 31, 7, 37, 40, 38, 84, 66, 99, 30, 24, 12, 20, 6, 73, 27, 68, 88, 13, 14, 15, 39, 3, 80, 83, 1, 82, 64, 21, 97, 69, 26, 10], [105, 123, 98, 86, 59, 93, 64, 127, 50, 125, 60, 90, 67, 116, 121, 46, 80, 117, 124, 126, 51, 119, 73, 96, 56, 122, 101, 58, 57, 62, 48, 63, 14, 115, 45, 61, 11, 53, 112, 118, 52, 120, 82, 110, 55, 42, 43, 111, 113, 47, 106, 12, 114, 54, 44, 69, 1, 40, 109, 84, 49, 70, 108, 66, 39, 102, 24, 81, 37, 68, 3, 23, 100, 95, 19, 104, 36, 91, 71, 28, 8, 15, 77, 32, 21, 94, 35, 74, 17, 72, 103, 97, 85, 10, 5, 89, 99, 30, 38, 29, 25, 107, 33, 20, 87, 27, 83, 31, 92, 79, 41, 34, 6, 7, 88, 75, 65, 22, 26, 2, 13, 76, 18, 78, 4, 0, 9, 16], [126, 102, 63, 33, 86, 93, 127, 56, 95, 26, 120, 47, 124, 111, 112, 121, 54, 114, 46, 119, 53, 23, 61, 117, 51, 118, 57, 107, 48, 15, 122, 60, 58, 45, 55, 44, 59, 105, 52, 17, 104, 62, 20, 115, 49, 82, 94, 103, 13, 123, 50, 106, 108, 40, 125, 113, 116, 30, 41, 43, 42, 109, 110, 88, 39, 91, 79, 31, 28, 37, 38, 27, 87, 101, 18, 77, 92, 81, 89, 99, 29, 36, 96, 100, 34, 35, 83, 21, 11, 74, 32, 7, 10, 98, 75, 19, 25, 90, 84, 71, 65, 9, 6, 85, 16, 8, 14, 12, 76, 5, 97, 22, 78, 80, 24, 4, 73, 67, 1, 72, 66, 0, 69, 68, 70, 3, 2, 64]], "model.layers.30.self_attn.qk_proj": [[63, 61, 123, 56, 110, 126, 53, 46, 122, 113, 44, 50, 106, 127, 111, 117, 108, 60, 48, 22, 42, 57, 102, 59, 103, 52, 124, 125, 39, 105, 33, 49, 120, 119, 116, 38, 51, 90, 112, 86, 115, 55, 54, 26, 93, 29, 58, 97, 118, 114, 41, 109, 121, 47, 34, 24, 88, 30, 94, 92, 43, 62, 82, 25, 31, 45, 98, 28, 81, 89, 36, 23, 18, 107, 77, 83, 91, 101, 87, 13, 40, 80, 14, 17, 21, 85, 100, 78, 11, 16, 19, 27, 37, 79, 9, 84, 104, 75, 95, 15, 73, 76, 64, 0, 20, 96, 12, 71, 10, 7, 8, 35, 74, 67, 6, 4, 70, 68, 5, 99, 32, 65, 1, 72, 3, 69, 2, 66], [63, 61, 123, 126, 56, 110, 46, 53, 122, 113, 50, 44, 111, 106, 42, 117, 127, 108, 60, 48, 22, 57, 102, 120, 125, 103, 105, 124, 59, 52, 38, 112, 39, 86, 51, 33, 29, 115, 58, 54, 49, 26, 47, 55, 118, 97, 41, 109, 93, 116, 90, 119, 114, 30, 121, 45, 24, 94, 62, 88, 28, 25, 107, 92, 43, 31, 34, 98, 82, 91, 23, 89, 13, 17, 100, 18, 40, 104, 36, 81, 77, 14, 11, 19, 16, 78, 83, 80, 85, 87, 21, 37, 95, 9, 79, 99, 101, 76, 27, 0, 12, 75, 15, 70, 73, 68, 74, 64, 10, 20, 71, 35, 32, 84, 8, 7, 4, 3, 96, 1, 65, 67, 69, 6, 5, 72, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 127, 117, 42, 48, 108, 22, 57, 102, 60, 103, 124, 105, 120, 59, 52, 51, 86, 33, 38, 39, 115, 54, 116, 49, 125, 55, 112, 58, 93, 119, 26, 47, 118, 121, 114, 29, 90, 97, 107, 24, 94, 109, 30, 62, 28, 88, 41, 34, 92, 25, 43, 40, 89, 45, 98, 18, 83, 31, 36, 81, 23, 101, 82, 100, 13, 16, 77, 80, 14, 91, 21, 19, 0, 11, 70, 17, 87, 85, 104, 95, 78, 20, 9, 37, 99, 75, 79, 15, 27, 74, 76, 71, 10, 84, 12, 32, 73, 64, 7, 4, 35, 67, 1, 96, 72, 68, 65, 3, 8, 5, 69, 2, 66, 6], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 117, 108, 42, 127, 48, 52, 57, 22, 120, 124, 103, 102, 60, 58, 125, 39, 105, 38, 116, 121, 47, 49, 112, 51, 54, 86, 26, 33, 59, 93, 118, 114, 97, 119, 55, 43, 29, 109, 90, 115, 30, 24, 62, 41, 94, 45, 107, 34, 28, 92, 40, 25, 31, 88, 82, 104, 13, 98, 81, 89, 14, 80, 23, 36, 83, 78, 19, 18, 101, 100, 21, 91, 17, 77, 64, 11, 37, 87, 16, 76, 85, 95, 70, 99, 75, 79, 84, 15, 9, 73, 0, 27, 10, 71, 12, 20, 68, 32, 7, 3, 35, 1, 74, 96, 72, 65, 4, 8, 67, 69, 5, 66, 2, 6], [63, 61, 123, 56, 126, 53, 110, 46, 122, 113, 44, 50, 111, 127, 106, 108, 117, 42, 22, 48, 102, 52, 60, 57, 124, 120, 103, 125, 59, 51, 86, 116, 105, 39, 119, 58, 54, 47, 49, 55, 33, 112, 90, 26, 115, 118, 38, 93, 114, 29, 97, 41, 121, 109, 24, 62, 43, 107, 34, 31, 83, 92, 25, 89, 88, 94, 36, 30, 82, 81, 23, 45, 28, 18, 80, 21, 13, 98, 77, 14, 11, 17, 78, 40, 87, 101, 85, 27, 16, 100, 0, 19, 95, 79, 91, 104, 76, 15, 73, 12, 9, 75, 70, 64, 20, 84, 74, 37, 10, 1, 7, 72, 3, 71, 4, 65, 68, 35, 67, 96, 32, 8, 6, 5, 99, 69, 66, 2], [63, 61, 126, 56, 123, 110, 53, 46, 122, 113, 44, 50, 108, 111, 106, 42, 127, 22, 57, 48, 117, 105, 103, 102, 120, 86, 60, 52, 59, 38, 47, 112, 125, 124, 39, 54, 58, 116, 118, 51, 93, 33, 114, 29, 26, 45, 49, 90, 97, 55, 119, 41, 24, 109, 121, 115, 92, 25, 28, 43, 62, 88, 94, 81, 82, 23, 30, 21, 98, 36, 34, 14, 95, 31, 18, 13, 83, 107, 87, 89, 91, 80, 78, 17, 77, 27, 85, 40, 11, 19, 15, 104, 16, 101, 100, 12, 64, 9, 76, 79, 75, 37, 73, 70, 0, 4, 72, 74, 84, 20, 10, 7, 35, 6, 68, 65, 71, 32, 96, 3, 99, 67, 66, 69, 5, 8, 1, 2], [63, 61, 110, 123, 126, 56, 53, 122, 46, 44, 113, 50, 42, 111, 106, 108, 48, 22, 127, 102, 117, 59, 105, 125, 103, 124, 60, 57, 52, 120, 86, 112, 39, 47, 90, 116, 26, 38, 29, 93, 119, 115, 41, 109, 45, 55, 33, 121, 97, 114, 58, 51, 118, 43, 92, 54, 88, 62, 25, 24, 28, 49, 94, 23, 31, 81, 18, 107, 21, 82, 87, 85, 30, 80, 36, 89, 83, 98, 91, 17, 27, 77, 40, 95, 100, 11, 34, 14, 13, 78, 19, 16, 20, 15, 101, 12, 76, 6, 79, 73, 9, 75, 74, 104, 84, 10, 35, 37, 64, 0, 32, 7, 67, 71, 4, 72, 99, 65, 68, 70, 8, 1, 96, 3, 2, 5, 69, 66], [63, 61, 123, 56, 110, 126, 53, 122, 46, 113, 44, 50, 111, 106, 108, 42, 127, 48, 22, 117, 57, 102, 59, 60, 103, 124, 120, 52, 125, 115, 55, 105, 119, 112, 33, 90, 93, 38, 58, 116, 39, 49, 86, 51, 26, 47, 29, 97, 41, 121, 109, 114, 118, 92, 45, 54, 88, 25, 62, 24, 43, 30, 31, 89, 28, 107, 91, 94, 34, 18, 21, 36, 81, 23, 82, 17, 98, 40, 83, 100, 87, 13, 27, 14, 101, 16, 80, 78, 11, 85, 77, 20, 37, 95, 64, 75, 79, 0, 76, 19, 104, 15, 6, 9, 73, 84, 35, 10, 12, 96, 67, 7, 1, 99, 74, 72, 32, 65, 4, 71, 68, 3, 8, 69, 2, 66, 5, 70], [63, 61, 56, 110, 123, 53, 126, 46, 122, 113, 44, 50, 106, 111, 127, 42, 48, 108, 52, 60, 102, 57, 124, 117, 103, 22, 120, 58, 59, 116, 86, 125, 105, 39, 38, 51, 93, 119, 55, 112, 109, 97, 33, 29, 90, 26, 115, 49, 45, 47, 118, 43, 25, 41, 94, 92, 114, 88, 24, 54, 31, 107, 121, 30, 62, 23, 28, 18, 89, 34, 98, 91, 13, 40, 82, 27, 87, 83, 80, 21, 78, 100, 17, 36, 19, 77, 81, 95, 85, 11, 104, 20, 16, 14, 76, 84, 101, 79, 75, 37, 73, 6, 15, 12, 68, 99, 0, 32, 7, 4, 72, 9, 10, 74, 64, 71, 35, 1, 3, 96, 65, 5, 70, 8, 67, 2, 69, 66], [63, 61, 123, 126, 56, 110, 53, 122, 46, 113, 44, 50, 111, 106, 108, 127, 48, 60, 117, 124, 125, 120, 52, 102, 57, 22, 42, 103, 55, 38, 59, 49, 51, 58, 105, 33, 39, 47, 119, 121, 97, 86, 116, 115, 90, 93, 26, 43, 109, 54, 112, 29, 114, 34, 62, 118, 92, 24, 94, 88, 45, 30, 98, 107, 31, 41, 13, 40, 23, 28, 25, 83, 100, 82, 18, 91, 81, 37, 89, 104, 16, 77, 17, 85, 87, 80, 11, 21, 19, 64, 78, 6, 14, 36, 101, 75, 95, 27, 99, 74, 84, 73, 7, 68, 10, 4, 0, 20, 9, 71, 79, 12, 15, 76, 35, 72, 3, 65, 67, 96, 32, 1, 8, 70, 5, 66, 69, 2], [63, 61, 123, 126, 56, 110, 46, 122, 53, 113, 44, 50, 111, 106, 127, 108, 117, 48, 42, 22, 60, 52, 125, 57, 102, 103, 59, 124, 105, 112, 33, 39, 93, 49, 55, 120, 119, 86, 114, 121, 51, 116, 47, 38, 26, 118, 90, 54, 97, 29, 62, 115, 41, 109, 58, 24, 88, 34, 43, 92, 94, 30, 25, 31, 13, 89, 82, 14, 107, 17, 45, 28, 40, 98, 85, 36, 18, 77, 78, 19, 23, 91, 101, 81, 80, 83, 21, 104, 16, 11, 95, 73, 74, 9, 79, 87, 76, 12, 15, 0, 75, 64, 100, 27, 20, 6, 84, 7, 71, 4, 10, 72, 35, 8, 65, 67, 70, 3, 37, 68, 99, 32, 69, 96, 1, 66, 5, 2], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 44, 50, 106, 108, 127, 111, 42, 22, 102, 48, 60, 124, 125, 52, 105, 103, 86, 117, 57, 112, 120, 59, 116, 33, 39, 29, 58, 93, 55, 119, 26, 41, 38, 115, 97, 47, 90, 114, 49, 51, 109, 118, 25, 121, 92, 24, 43, 54, 31, 62, 88, 30, 82, 23, 17, 94, 13, 81, 18, 98, 40, 28, 45, 14, 91, 77, 78, 101, 85, 16, 107, 34, 83, 89, 19, 21, 87, 80, 36, 27, 11, 95, 73, 100, 79, 75, 76, 9, 12, 84, 104, 37, 20, 71, 64, 35, 15, 4, 74, 0, 10, 68, 70, 69, 32, 72, 7, 8, 3, 96, 67, 99, 1, 6, 65, 5, 66, 2], [63, 61, 56, 110, 123, 126, 53, 122, 46, 113, 44, 50, 108, 127, 111, 106, 22, 42, 48, 124, 60, 102, 103, 38, 57, 120, 125, 55, 52, 58, 86, 119, 59, 33, 49, 105, 117, 47, 90, 29, 26, 97, 112, 39, 115, 116, 93, 109, 92, 118, 51, 54, 45, 114, 41, 24, 25, 43, 94, 88, 23, 121, 28, 91, 31, 62, 30, 18, 34, 89, 107, 17, 100, 82, 85, 83, 27, 87, 98, 95, 81, 19, 40, 13, 80, 78, 36, 21, 77, 37, 75, 14, 15, 16, 101, 104, 11, 0, 79, 70, 35, 76, 84, 73, 20, 99, 10, 9, 64, 12, 4, 96, 32, 68, 7, 74, 71, 72, 3, 1, 67, 8, 65, 66, 2, 69, 6, 5], [63, 61, 123, 56, 110, 126, 46, 53, 122, 113, 44, 50, 108, 111, 42, 106, 48, 127, 22, 124, 120, 117, 60, 57, 55, 102, 105, 103, 59, 39, 52, 58, 125, 38, 119, 33, 49, 51, 121, 47, 86, 116, 93, 112, 90, 26, 115, 41, 118, 109, 114, 29, 97, 24, 54, 62, 94, 43, 45, 92, 31, 25, 28, 88, 98, 107, 23, 82, 30, 21, 91, 89, 18, 34, 40, 83, 87, 17, 13, 78, 80, 36, 14, 81, 85, 75, 77, 104, 101, 16, 100, 64, 0, 15, 27, 70, 11, 95, 19, 37, 76, 12, 73, 79, 84, 7, 68, 74, 35, 8, 9, 71, 32, 67, 20, 99, 10, 4, 65, 69, 1, 3, 5, 72, 96, 66, 6, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 44, 113, 50, 111, 106, 108, 42, 22, 127, 48, 117, 105, 57, 124, 102, 52, 103, 125, 55, 60, 120, 38, 115, 49, 39, 33, 121, 26, 97, 59, 116, 119, 86, 93, 54, 41, 47, 109, 90, 43, 51, 29, 114, 112, 34, 118, 58, 24, 45, 98, 62, 88, 92, 94, 25, 30, 31, 82, 28, 13, 23, 81, 89, 107, 40, 14, 78, 0, 80, 21, 87, 77, 18, 91, 83, 16, 17, 19, 100, 101, 36, 85, 76, 95, 75, 11, 70, 64, 104, 84, 9, 74, 73, 79, 15, 20, 12, 8, 37, 27, 35, 71, 10, 99, 7, 1, 68, 67, 4, 69, 65, 32, 3, 72, 96, 66, 2, 5, 6], [63, 61, 126, 123, 56, 110, 53, 46, 122, 113, 44, 50, 111, 106, 127, 108, 42, 22, 48, 117, 102, 52, 124, 57, 103, 105, 38, 60, 33, 86, 125, 116, 58, 59, 120, 118, 115, 114, 49, 39, 55, 112, 119, 93, 29, 97, 51, 26, 47, 41, 30, 94, 90, 54, 43, 121, 24, 109, 92, 25, 31, 89, 45, 34, 88, 40, 28, 107, 82, 98, 14, 91, 23, 81, 77, 62, 18, 80, 78, 100, 16, 85, 83, 87, 95, 13, 17, 21, 19, 101, 11, 64, 36, 76, 27, 75, 104, 79, 9, 73, 20, 12, 37, 84, 8, 3, 10, 70, 74, 0, 68, 4, 15, 7, 71, 1, 35, 96, 67, 99, 65, 5, 32, 69, 72, 2, 6, 66], [63, 61, 110, 126, 53, 56, 123, 46, 122, 44, 113, 50, 111, 106, 42, 108, 127, 22, 124, 57, 102, 48, 117, 60, 52, 105, 86, 103, 38, 116, 59, 125, 58, 33, 55, 47, 39, 112, 26, 29, 119, 120, 118, 115, 97, 90, 93, 49, 51, 121, 94, 109, 114, 43, 41, 62, 92, 31, 88, 45, 24, 25, 30, 34, 54, 23, 28, 40, 81, 83, 18, 89, 82, 17, 91, 98, 13, 100, 77, 16, 107, 87, 36, 21, 101, 95, 78, 14, 80, 27, 19, 84, 11, 85, 37, 20, 75, 104, 79, 15, 10, 12, 99, 9, 64, 76, 0, 4, 73, 96, 35, 7, 74, 8, 67, 68, 6, 3, 65, 71, 70, 32, 1, 5, 72, 66, 69, 2], [63, 61, 110, 56, 126, 123, 53, 46, 122, 113, 44, 50, 111, 106, 127, 42, 108, 48, 117, 57, 22, 102, 52, 105, 103, 60, 124, 59, 86, 116, 115, 33, 55, 38, 120, 39, 47, 125, 109, 112, 49, 97, 26, 51, 93, 62, 41, 29, 90, 119, 114, 118, 121, 58, 54, 94, 24, 92, 30, 107, 34, 88, 28, 45, 89, 43, 25, 18, 98, 31, 91, 23, 81, 83, 40, 13, 36, 16, 82, 19, 17, 78, 75, 77, 37, 14, 87, 100, 21, 85, 27, 101, 80, 104, 95, 9, 0, 64, 79, 20, 6, 15, 76, 73, 84, 35, 74, 11, 99, 12, 7, 96, 8, 32, 10, 1, 67, 71, 3, 4, 69, 72, 70, 66, 68, 65, 5, 2], [63, 61, 123, 126, 110, 56, 46, 53, 122, 113, 50, 44, 111, 106, 42, 127, 52, 108, 48, 57, 102, 117, 124, 120, 22, 103, 60, 51, 105, 39, 125, 33, 38, 47, 116, 59, 55, 86, 115, 49, 121, 118, 26, 112, 29, 58, 119, 97, 93, 109, 45, 43, 90, 54, 62, 41, 114, 30, 31, 107, 94, 24, 34, 40, 92, 98, 88, 23, 28, 36, 25, 77, 100, 81, 101, 89, 82, 17, 18, 91, 13, 21, 75, 78, 83, 95, 19, 14, 80, 16, 87, 37, 104, 9, 85, 11, 99, 6, 27, 12, 35, 76, 7, 84, 73, 71, 20, 64, 15, 79, 74, 10, 0, 68, 1, 32, 96, 8, 69, 3, 4, 65, 2, 72, 67, 5, 70, 66], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 48, 127, 117, 22, 38, 124, 57, 52, 102, 105, 103, 120, 60, 33, 125, 59, 47, 39, 86, 26, 112, 97, 119, 51, 49, 58, 114, 116, 118, 29, 55, 115, 93, 90, 109, 121, 41, 54, 92, 30, 24, 45, 94, 34, 88, 31, 28, 62, 43, 107, 25, 23, 91, 98, 77, 36, 18, 81, 104, 40, 89, 87, 82, 17, 13, 19, 78, 101, 21, 85, 80, 83, 14, 79, 95, 16, 9, 10, 75, 6, 100, 76, 20, 11, 0, 15, 74, 37, 12, 71, 73, 68, 84, 64, 8, 27, 7, 32, 99, 4, 3, 72, 65, 35, 1, 96, 67, 66, 5, 69, 2, 70], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 106, 42, 108, 117, 57, 48, 127, 22, 60, 124, 52, 105, 102, 103, 59, 109, 33, 120, 125, 115, 114, 112, 116, 55, 86, 49, 38, 51, 118, 47, 39, 62, 90, 58, 93, 119, 26, 29, 97, 41, 121, 54, 107, 94, 45, 24, 28, 88, 92, 43, 98, 34, 30, 89, 36, 25, 31, 82, 95, 91, 83, 13, 80, 40, 77, 18, 104, 21, 87, 17, 81, 101, 23, 14, 78, 16, 75, 19, 11, 85, 64, 27, 37, 100, 15, 9, 76, 79, 6, 73, 0, 10, 35, 20, 99, 84, 4, 74, 65, 67, 32, 7, 71, 68, 12, 8, 1, 72, 69, 3, 96, 70, 2, 5, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 117, 42, 106, 108, 127, 124, 52, 48, 102, 57, 60, 120, 103, 125, 22, 105, 112, 55, 58, 39, 59, 114, 38, 33, 115, 121, 119, 51, 118, 47, 116, 86, 29, 49, 26, 90, 109, 93, 54, 41, 107, 97, 94, 88, 62, 45, 30, 43, 31, 28, 23, 34, 92, 24, 91, 25, 83, 87, 36, 98, 82, 100, 17, 81, 89, 18, 77, 104, 64, 40, 37, 14, 19, 13, 85, 21, 95, 27, 101, 75, 80, 16, 78, 0, 76, 15, 20, 79, 11, 4, 9, 3, 73, 74, 99, 35, 10, 12, 96, 84, 6, 1, 70, 71, 7, 67, 8, 32, 65, 72, 69, 5, 68, 2, 66], [63, 61, 123, 110, 126, 56, 53, 46, 122, 113, 44, 50, 111, 127, 106, 108, 42, 124, 117, 48, 102, 22, 120, 60, 52, 125, 58, 38, 116, 33, 57, 103, 59, 119, 105, 115, 86, 118, 112, 49, 39, 55, 26, 47, 29, 93, 90, 121, 51, 97, 114, 41, 54, 94, 109, 88, 43, 28, 34, 45, 24, 62, 25, 30, 31, 92, 107, 89, 36, 98, 91, 82, 18, 23, 83, 87, 81, 100, 77, 17, 13, 40, 21, 101, 16, 14, 95, 85, 80, 78, 19, 0, 11, 27, 75, 104, 64, 9, 15, 84, 76, 71, 20, 73, 4, 74, 37, 79, 72, 65, 70, 12, 35, 99, 3, 1, 96, 7, 10, 68, 67, 2, 32, 8, 5, 66, 6, 69], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 106, 111, 108, 42, 48, 22, 127, 120, 117, 124, 105, 52, 102, 57, 103, 38, 125, 33, 60, 59, 116, 118, 55, 112, 86, 97, 26, 93, 49, 90, 39, 121, 47, 119, 29, 51, 58, 114, 41, 115, 54, 109, 24, 92, 43, 62, 94, 28, 30, 88, 34, 25, 45, 36, 89, 107, 40, 82, 17, 98, 31, 23, 78, 91, 18, 81, 13, 104, 77, 83, 80, 19, 14, 95, 85, 87, 21, 16, 11, 75, 15, 79, 27, 76, 100, 70, 10, 9, 101, 37, 12, 73, 35, 4, 99, 20, 64, 72, 84, 32, 74, 71, 68, 7, 0, 67, 5, 1, 65, 3, 8, 96, 69, 6, 66, 2], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 117, 57, 124, 52, 127, 60, 105, 120, 102, 59, 103, 116, 112, 55, 114, 115, 38, 125, 119, 86, 33, 121, 49, 39, 26, 109, 90, 47, 51, 118, 93, 29, 54, 43, 97, 41, 107, 58, 94, 28, 92, 25, 24, 30, 62, 23, 45, 88, 31, 36, 17, 34, 77, 85, 83, 98, 89, 40, 81, 87, 82, 101, 21, 18, 78, 16, 19, 91, 100, 27, 95, 13, 11, 104, 80, 75, 15, 70, 79, 9, 14, 73, 64, 84, 76, 12, 37, 7, 35, 74, 20, 0, 10, 68, 71, 32, 72, 67, 3, 99, 65, 8, 1, 96, 4, 5, 66, 69, 6, 2], [63, 61, 110, 126, 123, 56, 53, 46, 122, 113, 44, 50, 111, 108, 106, 42, 22, 48, 127, 124, 120, 60, 52, 102, 117, 57, 103, 105, 55, 125, 116, 33, 59, 115, 38, 119, 86, 49, 29, 39, 112, 118, 47, 90, 26, 58, 93, 121, 109, 97, 114, 94, 54, 51, 41, 24, 28, 62, 30, 92, 34, 25, 88, 43, 107, 31, 89, 98, 23, 91, 45, 82, 18, 100, 40, 36, 37, 83, 77, 19, 87, 17, 81, 85, 101, 27, 80, 95, 16, 13, 104, 21, 11, 75, 14, 64, 78, 35, 12, 79, 84, 9, 20, 15, 73, 99, 76, 10, 70, 0, 71, 74, 96, 68, 65, 32, 72, 3, 1, 7, 67, 5, 4, 69, 8, 2, 66, 6], [63, 61, 110, 123, 126, 56, 53, 46, 122, 113, 50, 44, 106, 111, 42, 108, 48, 117, 127, 22, 57, 52, 60, 102, 103, 105, 124, 59, 116, 38, 112, 125, 120, 109, 115, 121, 39, 55, 33, 119, 49, 86, 51, 93, 97, 118, 114, 54, 58, 26, 90, 41, 29, 62, 47, 88, 24, 94, 30, 91, 89, 34, 43, 31, 92, 25, 107, 28, 45, 101, 36, 98, 40, 18, 81, 82, 87, 83, 17, 85, 23, 19, 80, 11, 21, 13, 78, 100, 16, 27, 14, 77, 95, 104, 84, 37, 79, 35, 75, 12, 73, 0, 99, 76, 15, 9, 8, 64, 20, 74, 70, 32, 68, 10, 71, 7, 65, 67, 96, 4, 72, 3, 69, 1, 5, 6, 2, 66], [63, 61, 123, 126, 56, 110, 53, 46, 122, 113, 50, 44, 111, 106, 108, 117, 48, 127, 42, 22, 124, 120, 57, 52, 102, 103, 60, 125, 38, 105, 116, 33, 59, 39, 121, 49, 55, 112, 51, 47, 119, 26, 114, 115, 86, 93, 41, 43, 54, 90, 97, 58, 29, 109, 118, 62, 45, 34, 88, 92, 24, 94, 31, 107, 25, 89, 28, 30, 36, 91, 82, 81, 23, 98, 18, 77, 21, 11, 83, 14, 17, 40, 80, 0, 87, 78, 37, 16, 13, 19, 85, 100, 79, 9, 104, 84, 73, 95, 75, 12, 10, 64, 27, 15, 101, 76, 35, 71, 74, 99, 20, 6, 67, 68, 7, 1, 4, 65, 96, 70, 3, 32, 2, 72, 5, 8, 69, 66], [63, 61, 123, 126, 110, 53, 56, 46, 122, 113, 50, 44, 111, 106, 127, 117, 108, 42, 48, 125, 52, 124, 59, 22, 102, 57, 103, 38, 112, 60, 105, 33, 120, 116, 51, 119, 39, 115, 55, 97, 54, 43, 86, 62, 93, 49, 29, 26, 114, 41, 47, 58, 90, 118, 121, 34, 94, 24, 92, 109, 45, 89, 30, 88, 28, 31, 40, 95, 18, 101, 98, 25, 23, 14, 91, 107, 80, 82, 17, 16, 11, 85, 36, 77, 13, 78, 19, 100, 81, 104, 0, 87, 15, 83, 64, 73, 71, 6, 21, 79, 9, 75, 37, 76, 12, 10, 74, 27, 35, 20, 84, 4, 65, 68, 7, 8, 3, 67, 1, 96, 99, 70, 32, 72, 69, 66, 5, 2], [63, 61, 126, 123, 56, 110, 46, 53, 122, 113, 50, 44, 111, 42, 106, 57, 108, 117, 124, 22, 127, 48, 59, 52, 102, 125, 60, 120, 103, 105, 116, 38, 86, 39, 55, 112, 119, 58, 33, 26, 49, 114, 97, 29, 93, 51, 47, 41, 118, 90, 43, 115, 54, 45, 24, 109, 62, 88, 28, 92, 25, 94, 121, 18, 30, 34, 91, 31, 23, 100, 82, 81, 98, 89, 77, 21, 36, 85, 78, 87, 17, 19, 40, 14, 107, 80, 101, 13, 83, 11, 16, 104, 95, 73, 27, 15, 6, 9, 76, 79, 84, 75, 12, 37, 0, 4, 35, 74, 20, 68, 10, 71, 99, 64, 8, 7, 3, 67, 32, 1, 72, 69, 65, 5, 70, 96, 66, 2], [63, 61, 126, 123, 110, 56, 53, 46, 122, 113, 50, 44, 111, 124, 108, 117, 106, 42, 57, 60, 127, 22, 48, 103, 52, 102, 38, 116, 120, 59, 105, 49, 86, 55, 125, 39, 33, 47, 121, 114, 112, 51, 90, 115, 97, 29, 119, 26, 54, 118, 41, 93, 109, 58, 43, 25, 28, 62, 94, 30, 34, 24, 92, 31, 23, 88, 45, 36, 95, 78, 81, 82, 98, 19, 18, 83, 107, 91, 89, 13, 77, 21, 87, 6, 100, 14, 80, 85, 11, 40, 64, 73, 17, 104, 16, 15, 101, 9, 0, 71, 37, 4, 79, 7, 27, 75, 10, 74, 12, 84, 68, 67, 76, 8, 35, 3, 20, 1, 99, 65, 96, 66, 32, 72, 2, 69, 5, 70], [63, 61, 123, 110, 56, 126, 53, 122, 46, 113, 44, 50, 111, 106, 42, 108, 127, 22, 48, 60, 117, 102, 103, 124, 57, 125, 120, 52, 59, 38, 33, 105, 86, 55, 39, 29, 97, 49, 51, 121, 119, 26, 47, 90, 112, 115, 116, 93, 54, 114, 30, 109, 118, 41, 62, 43, 94, 25, 92, 31, 24, 88, 58, 34, 28, 91, 18, 81, 45, 100, 98, 82, 23, 107, 85, 11, 14, 13, 89, 37, 17, 36, 21, 80, 19, 83, 16, 40, 87, 77, 95, 27, 79, 78, 104, 73, 101, 6, 15, 9, 75, 35, 12, 10, 76, 99, 74, 8, 20, 71, 0, 4, 84, 96, 7, 32, 68, 72, 67, 69, 1, 3, 70, 65, 5, 64, 2, 66]], "model.layers.31.self_attn.q_proj": [[40, 121, 98, 46, 18, 25, 111, 9, 30, 21, 13, 51, 15, 11, 48, 105, 49, 67, 23, 60, 55, 53, 4, 6, 58, 116, 114, 124, 72, 107, 31, 88, 115, 95, 41, 109, 22, 108, 61, 17, 37, 59, 92, 50, 45, 19, 87, 43, 127, 20, 5, 97, 1, 14, 123, 27, 126, 69, 75, 78, 81, 90, 16, 120, 102, 101, 8, 24, 112, 93, 85, 52, 26, 63, 118, 0, 7, 33, 117, 62, 122, 96, 76, 94, 74, 113, 35, 36, 54, 28, 12, 77, 79, 57, 39, 56, 82, 70, 73, 42, 89, 84, 3, 80, 32, 106, 110, 103, 83, 99, 100, 2, 125, 65, 119, 68, 38, 29, 86, 71, 44, 47, 10, 104, 91, 64, 66, 34], [40, 121, 98, 46, 9, 13, 111, 21, 25, 18, 51, 67, 15, 49, 4, 95, 11, 108, 60, 50, 48, 117, 45, 68, 107, 88, 61, 116, 37, 64, 122, 41, 30, 58, 6, 92, 1, 72, 2, 105, 43, 123, 65, 109, 55, 0, 28, 87, 42, 3, 53, 5, 126, 23, 27, 39, 127, 54, 74, 19, 101, 114, 73, 85, 14, 62, 70, 113, 22, 115, 66, 31, 63, 34, 20, 93, 16, 78, 120, 33, 118, 69, 110, 124, 36, 90, 7, 75, 52, 104, 82, 57, 80, 77, 94, 32, 29, 24, 81, 10, 103, 12, 83, 17, 79, 76, 89, 26, 59, 8, 35, 112, 38, 44, 99, 119, 84, 125, 56, 71, 86, 100, 96, 102, 106, 91, 97, 47], [40, 111, 46, 121, 98, 37, 49, 21, 25, 117, 54, 114, 18, 95, 23, 60, 11, 15, 107, 51, 61, 53, 13, 116, 105, 4, 78, 62, 16, 48, 67, 30, 9, 113, 124, 80, 24, 72, 39, 41, 123, 17, 19, 6, 1, 120, 108, 88, 42, 90, 92, 70, 83, 52, 63, 8, 109, 0, 50, 58, 20, 85, 27, 87, 55, 122, 56, 84, 26, 65, 127, 115, 45, 38, 22, 103, 59, 112, 43, 96, 125, 29, 71, 126, 75, 118, 91, 57, 100, 7, 119, 36, 102, 101, 10, 68, 33, 79, 76, 2, 14, 34, 35, 81, 44, 74, 86, 94, 12, 97, 82, 110, 32, 47, 89, 73, 104, 5, 106, 93, 31, 69, 28, 99, 77, 66, 64, 3], [40, 121, 98, 46, 117, 111, 13, 21, 18, 61, 37, 9, 25, 15, 4, 51, 48, 30, 105, 11, 122, 107, 72, 49, 58, 87, 108, 24, 6, 23, 123, 67, 116, 63, 1, 110, 90, 88, 60, 0, 124, 14, 31, 127, 101, 39, 126, 45, 32, 95, 43, 41, 27, 109, 85, 52, 26, 92, 93, 64, 80, 50, 75, 5, 97, 22, 36, 78, 55, 100, 74, 16, 57, 70, 53, 103, 89, 19, 82, 73, 102, 28, 115, 113, 120, 83, 59, 35, 38, 42, 62, 12, 118, 17, 106, 20, 2, 114, 119, 79, 76, 71, 33, 125, 81, 66, 54, 112, 8, 68, 44, 91, 7, 77, 56, 96, 86, 94, 69, 65, 99, 29, 3, 84, 10, 104, 47, 34], [117, 50, 105, 108, 123, 49, 99, 44, 97, 25, 31, 87, 120, 21, 112, 16, 93, 82, 58, 78, 34, 29, 62, 118, 48, 114, 51, 38, 30, 43, 109, 32, 90, 42, 55, 7, 106, 10, 59, 103, 36, 53, 107, 52, 12, 61, 96, 45, 33, 124, 47, 46, 60, 26, 56, 23, 35, 63, 88, 39, 95, 113, 17, 125, 77, 28, 115, 121, 54, 19, 83, 57, 24, 122, 98, 94, 84, 68, 119, 116, 104, 15, 110, 92, 11, 69, 111, 40, 102, 126, 3, 100, 20, 127, 14, 22, 91, 86, 79, 27, 101, 8, 75, 37, 71, 89, 65, 74, 81, 85, 2, 72, 18, 64, 6, 76, 66, 73, 9, 80, 70, 67, 4, 13, 1, 5, 41, 0], [105, 123, 49, 117, 31, 99, 118, 2, 114, 109, 10, 16, 7, 82, 25, 78, 64, 69, 3, 68, 65, 50, 12, 8, 21, 77, 44, 97, 87, 67, 89, 115, 58, 41, 0, 66, 4, 71, 1, 111, 23, 98, 59, 61, 93, 104, 120, 127, 72, 63, 5, 39, 56, 28, 51, 70, 42, 91, 108, 112, 24, 57, 95, 122, 20, 107, 74, 15, 36, 75, 121, 106, 14, 34, 83, 52, 86, 29, 9, 54, 6, 37, 79, 92, 84, 88, 30, 18, 27, 101, 22, 13, 73, 110, 103, 46, 80, 11, 17, 96, 62, 26, 19, 90, 119, 33, 76, 113, 43, 85, 60, 102, 45, 38, 48, 53, 81, 116, 47, 40, 55, 125, 32, 124, 94, 35, 126, 100], [105, 49, 123, 108, 31, 58, 118, 10, 25, 16, 117, 82, 99, 3, 69, 21, 8, 87, 12, 65, 78, 77, 68, 109, 94, 7, 29, 122, 114, 1, 120, 42, 5, 55, 97, 64, 4, 44, 50, 111, 43, 6, 36, 98, 115, 23, 73, 95, 33, 13, 125, 38, 2, 93, 11, 79, 48, 76, 63, 62, 66, 88, 61, 72, 89, 30, 67, 91, 47, 54, 46, 51, 27, 18, 56, 39, 32, 9, 92, 110, 90, 101, 100, 45, 112, 75, 86, 70, 113, 103, 116, 0, 127, 17, 34, 106, 74, 80, 71, 81, 14, 52, 40, 20, 104, 22, 121, 53, 60, 41, 84, 19, 83, 59, 96, 24, 107, 26, 37, 119, 15, 28, 57, 126, 85, 102, 124, 35], [118, 49, 108, 105, 58, 44, 42, 99, 21, 117, 87, 97, 94, 31, 12, 39, 25, 93, 29, 82, 104, 46, 51, 56, 52, 33, 38, 95, 43, 103, 113, 119, 107, 26, 109, 45, 73, 50, 16, 59, 115, 91, 125, 123, 111, 47, 60, 57, 112, 40, 127, 114, 20, 121, 54, 110, 77, 53, 101, 63, 37, 122, 126, 23, 61, 48, 1, 120, 124, 116, 32, 19, 102, 36, 55, 100, 96, 81, 78, 75, 4, 98, 62, 8, 85, 106, 92, 79, 34, 5, 17, 30, 70, 72, 10, 89, 9, 7, 90, 69, 35, 6, 22, 66, 76, 68, 24, 65, 28, 27, 18, 15, 3, 88, 83, 84, 13, 86, 2, 0, 41, 14, 67, 11, 80, 74, 71, 64], [40, 109, 121, 34, 59, 89, 53, 120, 17, 45, 55, 15, 86, 83, 114, 12, 29, 113, 71, 95, 28, 93, 4, 73, 94, 31, 69, 125, 67, 46, 122, 56, 106, 42, 33, 11, 92, 2, 77, 107, 61, 44, 88, 65, 108, 23, 38, 6, 58, 115, 8, 30, 50, 110, 26, 116, 0, 118, 126, 16, 99, 90, 21, 119, 87, 80, 62, 117, 20, 48, 101, 37, 49, 3, 68, 32, 85, 24, 41, 81, 84, 13, 52, 76, 19, 36, 82, 18, 1, 43, 91, 100, 112, 97, 74, 103, 78, 14, 96, 79, 35, 5, 64, 22, 75, 27, 39, 54, 57, 25, 47, 123, 63, 10, 124, 7, 127, 70, 104, 111, 51, 105, 9, 72, 102, 60, 66, 98], [40, 109, 121, 34, 15, 12, 86, 17, 73, 83, 53, 4, 89, 55, 71, 67, 59, 120, 45, 42, 92, 114, 2, 106, 95, 69, 122, 113, 0, 77, 6, 46, 29, 68, 28, 103, 56, 118, 11, 8, 33, 116, 72, 31, 18, 125, 65, 111, 37, 38, 88, 1, 61, 100, 85, 23, 108, 30, 101, 115, 64, 107, 98, 119, 20, 112, 82, 48, 3, 74, 126, 66, 70, 16, 13, 14, 81, 5, 93, 21, 84, 110, 19, 27, 94, 79, 80, 87, 96, 22, 102, 76, 58, 35, 7, 97, 25, 10, 91, 24, 36, 26, 62, 104, 75, 63, 117, 57, 47, 39, 78, 49, 41, 32, 90, 44, 9, 51, 43, 52, 127, 60, 99, 123, 105, 50, 124, 54], [40, 109, 121, 34, 89, 17, 86, 15, 55, 83, 12, 53, 45, 95, 59, 67, 122, 28, 92, 2, 29, 0, 65, 73, 69, 71, 114, 56, 4, 21, 42, 116, 6, 62, 120, 61, 125, 119, 31, 77, 46, 49, 10, 11, 93, 16, 1, 19, 26, 96, 88, 98, 36, 126, 106, 113, 118, 64, 30, 23, 57, 108, 44, 48, 101, 13, 103, 18, 123, 58, 22, 66, 37, 14, 27, 72, 76, 20, 87, 70, 80, 112, 127, 8, 115, 78, 94, 54, 68, 33, 84, 32, 81, 82, 24, 25, 7, 91, 39, 79, 90, 107, 74, 3, 85, 104, 52, 35, 5, 60, 51, 110, 75, 99, 9, 43, 100, 38, 47, 111, 97, 124, 102, 63, 117, 41, 50, 105], [121, 109, 40, 59, 34, 53, 111, 44, 114, 39, 124, 118, 116, 115, 88, 20, 61, 58, 55, 93, 117, 60, 126, 56, 62, 49, 108, 50, 72, 122, 48, 54, 63, 43, 46, 41, 57, 51, 119, 92, 127, 123, 47, 86, 125, 42, 31, 110, 105, 120, 98, 38, 52, 107, 113, 112, 45, 106, 83, 97, 89, 103, 104, 27, 36, 102, 28, 96, 26, 100, 14, 101, 32, 35, 37, 99, 10, 17, 95, 29, 33, 30, 94, 18, 85, 91, 87, 68, 23, 78, 25, 90, 13, 84, 73, 15, 4, 77, 70, 21, 74, 82, 8, 24, 6, 16, 75, 12, 80, 2, 71, 11, 19, 22, 66, 1, 64, 9, 67, 3, 81, 7, 69, 5, 76, 79, 0, 65], [107, 100, 58, 32, 21, 112, 78, 19, 61, 127, 87, 93, 115, 12, 43, 9, 114, 120, 116, 119, 16, 56, 82, 55, 42, 96, 111, 113, 25, 72, 38, 125, 54, 92, 79, 48, 29, 5, 66, 47, 88, 97, 110, 105, 57, 123, 4, 124, 35, 50, 49, 2, 18, 27, 0, 24, 63, 36, 62, 28, 53, 71, 91, 39, 77, 75, 68, 1, 86, 11, 3, 26, 69, 46, 20, 7, 15, 104, 126, 14, 95, 117, 106, 83, 73, 74, 98, 13, 23, 6, 70, 64, 41, 84, 45, 89, 67, 80, 30, 44, 99, 90, 85, 81, 10, 108, 40, 76, 59, 51, 94, 37, 31, 34, 103, 22, 65, 101, 102, 122, 8, 60, 52, 17, 109, 33, 121, 118], [107, 58, 100, 63, 127, 32, 112, 25, 56, 61, 47, 39, 115, 114, 116, 79, 21, 105, 113, 54, 108, 104, 96, 19, 119, 87, 42, 55, 120, 126, 93, 45, 43, 50, 41, 110, 125, 35, 106, 17, 122, 82, 111, 57, 97, 20, 44, 118, 60, 52, 48, 88, 123, 30, 124, 117, 49, 59, 16, 53, 38, 27, 46, 109, 62, 36, 74, 91, 78, 33, 51, 5, 121, 102, 101, 28, 24, 103, 40, 71, 75, 92, 99, 89, 31, 11, 37, 86, 94, 90, 72, 95, 12, 26, 98, 34, 15, 22, 9, 23, 84, 10, 14, 13, 77, 81, 18, 65, 29, 85, 70, 6, 3, 67, 66, 7, 83, 80, 69, 0, 68, 1, 8, 76, 4, 64, 2, 73], [43, 93, 58, 2, 68, 65, 107, 67, 0, 39, 87, 12, 70, 9, 69, 78, 64, 1, 21, 25, 63, 72, 96, 10, 16, 66, 110, 19, 112, 103, 100, 8, 61, 114, 97, 56, 7, 32, 37, 73, 29, 79, 5, 75, 55, 82, 120, 116, 14, 38, 20, 17, 49, 44, 91, 42, 3, 80, 57, 115, 113, 51, 109, 119, 101, 45, 74, 4, 104, 23, 40, 85, 28, 125, 98, 33, 127, 124, 47, 46, 31, 6, 53, 105, 59, 99, 48, 71, 35, 83, 11, 24, 121, 15, 26, 84, 106, 81, 60, 77, 86, 22, 89, 108, 123, 34, 95, 18, 30, 13, 76, 27, 92, 126, 118, 88, 50, 90, 41, 54, 94, 122, 36, 52, 111, 102, 62, 117], [107, 100, 58, 127, 112, 32, 12, 19, 78, 61, 25, 16, 9, 82, 119, 114, 21, 87, 43, 93, 113, 115, 54, 120, 66, 56, 0, 55, 125, 39, 49, 4, 38, 42, 105, 29, 116, 72, 97, 111, 68, 27, 63, 5, 81, 110, 96, 47, 69, 48, 124, 62, 51, 6, 71, 57, 35, 10, 92, 123, 3, 83, 28, 91, 44, 84, 126, 95, 79, 76, 7, 89, 23, 98, 75, 20, 18, 64, 88, 80, 90, 37, 101, 106, 22, 8, 52, 15, 85, 30, 73, 67, 50, 103, 59, 117, 11, 74, 70, 53, 118, 2, 13, 31, 14, 34, 1, 45, 109, 24, 46, 65, 77, 94, 33, 86, 108, 26, 104, 99, 41, 40, 60, 17, 122, 121, 102, 36], [55, 58, 115, 50, 52, 39, 122, 62, 63, 59, 121, 106, 34, 54, 103, 53, 47, 48, 95, 120, 119, 118, 117, 124, 56, 126, 113, 107, 127, 123, 49, 105, 96, 111, 116, 51, 112, 93, 42, 61, 84, 46, 109, 60, 114, 57, 44, 125, 43, 110, 45, 102, 91, 27, 22, 108, 38, 29, 99, 86, 41, 104, 76, 97, 24, 89, 30, 35, 40, 37, 82, 33, 32, 101, 100, 25, 36, 92, 16, 18, 10, 8, 20, 88, 31, 98, 94, 90, 28, 65, 21, 85, 23, 81, 15, 74, 14, 87, 80, 26, 4, 12, 78, 3, 17, 11, 1, 72, 5, 19, 2, 77, 68, 71, 79, 83, 6, 69, 73, 70, 64, 67, 66, 0, 75, 13, 9, 7], [50, 58, 39, 62, 52, 34, 122, 55, 45, 120, 118, 115, 106, 24, 108, 105, 81, 47, 64, 51, 126, 63, 71, 124, 93, 84, 103, 56, 109, 95, 113, 59, 96, 44, 53, 125, 117, 57, 27, 119, 54, 25, 48, 5, 112, 116, 111, 38, 42, 99, 127, 60, 91, 49, 22, 121, 114, 3, 86, 30, 2, 32, 61, 0, 20, 82, 123, 104, 35, 4, 46, 73, 98, 107, 43, 41, 110, 77, 66, 65, 101, 6, 15, 31, 76, 102, 14, 97, 28, 40, 88, 29, 67, 10, 37, 92, 21, 83, 18, 36, 26, 87, 8, 80, 89, 33, 100, 23, 78, 16, 85, 11, 94, 90, 12, 7, 69, 70, 79, 68, 74, 17, 72, 13, 9, 1, 19, 75], [58, 50, 115, 39, 34, 83, 77, 15, 2, 73, 11, 3, 71, 109, 24, 27, 6, 0, 81, 64, 122, 93, 79, 1, 51, 52, 85, 44, 68, 4, 22, 20, 67, 45, 113, 82, 9, 54, 99, 5, 25, 26, 66, 72, 8, 80, 106, 111, 65, 56, 19, 88, 114, 7, 12, 95, 116, 13, 78, 62, 104, 59, 63, 107, 21, 35, 89, 75, 10, 17, 119, 105, 70, 98, 74, 69, 28, 30, 32, 18, 61, 29, 76, 33, 14, 120, 91, 117, 118, 86, 41, 57, 92, 46, 127, 87, 16, 96, 126, 108, 31, 43, 23, 97, 94, 90, 38, 36, 103, 100, 102, 101, 37, 53, 110, 84, 124, 121, 47, 49, 125, 55, 40, 48, 123, 60, 112, 42], [58, 115, 62, 39, 55, 52, 122, 106, 34, 108, 107, 53, 120, 61, 63, 103, 105, 95, 118, 47, 54, 124, 59, 109, 49, 91, 117, 60, 57, 119, 48, 114, 112, 111, 93, 96, 99, 121, 84, 126, 46, 45, 116, 127, 123, 125, 41, 104, 51, 110, 42, 43, 19, 56, 22, 113, 44, 24, 33, 86, 7, 40, 38, 71, 32, 102, 101, 76, 25, 82, 97, 87, 37, 27, 16, 29, 35, 10, 100, 36, 83, 94, 50, 31, 98, 21, 30, 18, 90, 92, 20, 81, 11, 23, 78, 28, 89, 26, 75, 88, 74, 85, 15, 5, 14, 64, 8, 4, 80, 67, 0, 3, 69, 1, 12, 65, 66, 17, 68, 2, 79, 73, 77, 72, 6, 13, 9, 70], [122, 104, 105, 126, 53, 112, 51, 110, 118, 123, 55, 119, 57, 111, 124, 120, 114, 117, 59, 121, 125, 63, 116, 115, 61, 49, 34, 54, 113, 106, 60, 89, 32, 86, 50, 31, 62, 48, 46, 58, 56, 127, 44, 52, 45, 109, 91, 33, 98, 47, 107, 43, 25, 108, 95, 73, 41, 40, 20, 82, 93, 42, 75, 22, 83, 81, 103, 23, 19, 88, 39, 96, 101, 37, 87, 102, 92, 99, 36, 11, 90, 38, 29, 100, 27, 26, 35, 21, 72, 97, 84, 17, 94, 13, 30, 18, 14, 24, 16, 28, 78, 76, 85, 6, 80, 8, 77, 9, 68, 4, 10, 70, 79, 5, 12, 71, 67, 3, 74, 69, 15, 7, 65, 2, 64, 0, 66, 1], [55, 122, 104, 105, 127, 119, 121, 114, 62, 60, 118, 117, 112, 110, 57, 51, 63, 111, 49, 124, 34, 56, 116, 123, 61, 31, 125, 113, 59, 44, 86, 47, 120, 48, 115, 108, 54, 58, 33, 89, 106, 45, 46, 50, 52, 53, 126, 109, 98, 107, 91, 41, 43, 81, 42, 95, 88, 40, 87, 23, 103, 83, 27, 101, 32, 37, 25, 29, 92, 19, 93, 90, 39, 20, 30, 96, 22, 99, 38, 102, 36, 18, 82, 100, 94, 80, 26, 35, 97, 75, 15, 17, 28, 24, 85, 69, 12, 77, 76, 78, 84, 21, 13, 6, 14, 3, 10, 16, 71, 79, 8, 5, 70, 4, 72, 11, 9, 65, 67, 74, 68, 73, 2, 7, 64, 66, 0, 1], [55, 122, 104, 105, 126, 41, 88, 18, 20, 80, 79, 10, 31, 60, 98, 15, 85, 78, 114, 76, 121, 29, 86, 75, 34, 69, 12, 33, 93, 91, 118, 8, 47, 53, 123, 89, 27, 62, 110, 13, 120, 71, 59, 119, 63, 3, 116, 108, 57, 113, 51, 124, 56, 28, 44, 96, 97, 127, 74, 49, 43, 112, 87, 16, 106, 83, 115, 111, 54, 50, 117, 5, 26, 61, 25, 58, 94, 24, 125, 95, 46, 21, 48, 52, 72, 45, 42, 90, 82, 84, 109, 107, 14, 92, 23, 9, 39, 30, 100, 19, 32, 77, 101, 22, 73, 102, 36, 37, 81, 99, 35, 11, 17, 38, 70, 103, 7, 2, 6, 4, 40, 68, 67, 66, 65, 64, 1, 0], [122, 55, 104, 64, 71, 2, 126, 79, 4, 10, 1, 3, 73, 105, 78, 18, 8, 76, 20, 80, 77, 75, 9, 31, 69, 86, 7, 93, 51, 98, 68, 81, 88, 6, 27, 85, 121, 5, 66, 65, 107, 119, 70, 16, 23, 67, 114, 0, 84, 37, 74, 63, 11, 59, 97, 82, 106, 17, 15, 33, 120, 91, 118, 127, 21, 14, 57, 52, 41, 49, 12, 53, 87, 34, 95, 39, 72, 13, 110, 100, 89, 24, 47, 83, 29, 101, 46, 32, 25, 117, 112, 19, 56, 44, 28, 60, 111, 103, 99, 90, 26, 22, 58, 50, 124, 45, 125, 30, 96, 102, 92, 123, 94, 109, 36, 43, 113, 35, 40, 108, 38, 42, 48, 115, 116, 54, 61, 62], [103, 126, 98, 9, 14, 4, 21, 17, 71, 95, 64, 75, 67, 115, 27, 24, 88, 1, 109, 63, 48, 44, 62, 61, 16, 70, 118, 104, 114, 116, 31, 124, 45, 90, 122, 106, 3, 19, 5, 99, 105, 127, 6, 65, 56, 12, 46, 32, 37, 91, 53, 50, 113, 13, 66, 47, 100, 93, 22, 57, 81, 23, 20, 78, 15, 87, 0, 74, 92, 51, 69, 123, 72, 7, 41, 76, 73, 34, 10, 83, 11, 85, 112, 68, 49, 2, 79, 94, 125, 55, 58, 40, 89, 8, 77, 117, 26, 35, 121, 33, 80, 107, 119, 25, 82, 38, 52, 86, 42, 84, 54, 101, 97, 30, 36, 29, 28, 18, 110, 59, 39, 60, 96, 43, 102, 108, 111, 120], [103, 126, 98, 17, 65, 75, 14, 9, 71, 21, 4, 3, 61, 48, 88, 95, 109, 69, 0, 115, 44, 41, 5, 24, 27, 106, 124, 63, 118, 70, 62, 105, 114, 112, 12, 16, 122, 1, 31, 64, 45, 13, 50, 127, 91, 113, 19, 76, 56, 99, 46, 47, 90, 68, 81, 66, 84, 85, 86, 57, 116, 110, 78, 123, 55, 104, 23, 80, 77, 117, 96, 92, 18, 15, 100, 37, 20, 29, 42, 73, 67, 6, 82, 87, 72, 33, 53, 74, 107, 120, 34, 7, 119, 43, 54, 11, 32, 58, 26, 121, 89, 30, 83, 94, 93, 49, 51, 36, 102, 97, 38, 25, 79, 40, 35, 2, 22, 125, 10, 8, 28, 108, 52, 101, 111, 59, 60, 39], [103, 126, 98, 61, 95, 21, 88, 17, 27, 124, 44, 14, 109, 63, 91, 48, 113, 75, 115, 9, 104, 31, 122, 105, 77, 71, 90, 39, 50, 24, 83, 100, 127, 12, 4, 47, 37, 41, 114, 116, 53, 45, 19, 99, 36, 42, 57, 107, 87, 56, 33, 121, 93, 62, 94, 74, 51, 72, 13, 32, 70, 35, 29, 106, 119, 38, 112, 10, 123, 67, 1, 108, 40, 118, 84, 96, 97, 79, 20, 52, 110, 85, 26, 28, 5, 58, 54, 80, 76, 86, 46, 25, 55, 111, 73, 15, 69, 18, 22, 59, 125, 102, 64, 43, 117, 16, 101, 30, 81, 78, 23, 82, 8, 49, 89, 120, 92, 60, 6, 34, 11, 2, 3, 7, 68, 0, 66, 65], [126, 103, 41, 98, 61, 95, 118, 51, 112, 127, 119, 109, 54, 110, 91, 124, 27, 116, 62, 57, 88, 21, 53, 48, 29, 44, 39, 106, 122, 113, 105, 104, 46, 31, 114, 55, 63, 37, 42, 45, 47, 58, 50, 56, 92, 108, 22, 17, 115, 121, 14, 16, 100, 59, 60, 111, 117, 90, 123, 19, 43, 49, 120, 125, 52, 107, 94, 9, 84, 15, 75, 36, 102, 33, 38, 99, 30, 101, 40, 89, 35, 93, 24, 18, 32, 8, 83, 82, 96, 71, 85, 3, 23, 97, 5, 72, 4, 86, 13, 77, 20, 87, 25, 76, 28, 10, 26, 12, 80, 2, 65, 34, 74, 79, 69, 78, 66, 70, 1, 6, 67, 81, 0, 64, 68, 11, 73, 7], [58, 104, 56, 34, 80, 52, 84, 26, 42, 109, 12, 9, 82, 31, 28, 24, 35, 64, 70, 37, 103, 96, 11, 6, 94, 36, 66, 95, 49, 60, 114, 2, 87, 126, 76, 92, 7, 20, 73, 47, 115, 23, 14, 86, 113, 21, 51, 16, 97, 53, 22, 85, 10, 48, 41, 107, 38, 127, 27, 91, 93, 4, 106, 67, 111, 77, 30, 74, 0, 122, 54, 68, 121, 108, 59, 17, 18, 19, 45, 15, 13, 29, 75, 90, 124, 125, 102, 120, 83, 110, 118, 8, 88, 63, 116, 40, 46, 55, 50, 123, 43, 61, 101, 3, 65, 39, 69, 98, 89, 105, 79, 72, 32, 71, 119, 81, 57, 99, 33, 112, 62, 1, 44, 78, 100, 25, 5, 117], [56, 52, 104, 58, 107, 37, 34, 31, 109, 48, 21, 85, 55, 120, 95, 87, 112, 114, 126, 63, 40, 26, 111, 41, 113, 121, 47, 36, 54, 116, 60, 125, 28, 42, 124, 59, 50, 110, 92, 123, 61, 118, 62, 83, 108, 91, 49, 14, 119, 24, 115, 43, 45, 44, 53, 127, 74, 51, 46, 84, 94, 106, 105, 66, 7, 80, 0, 38, 39, 99, 67, 11, 103, 57, 64, 23, 35, 101, 25, 19, 102, 122, 88, 17, 6, 98, 81, 96, 27, 79, 32, 86, 13, 10, 15, 117, 30, 90, 100, 8, 93, 33, 78, 4, 97, 82, 29, 3, 72, 12, 68, 89, 2, 5, 1, 22, 77, 75, 65, 70, 9, 16, 69, 18, 71, 20, 73, 76], [58, 104, 37, 109, 107, 53, 127, 34, 126, 57, 50, 31, 120, 60, 121, 41, 63, 59, 114, 115, 118, 54, 48, 123, 113, 116, 108, 111, 110, 52, 49, 21, 61, 119, 51, 112, 95, 45, 55, 47, 44, 103, 46, 106, 40, 43, 124, 105, 87, 26, 39, 36, 62, 125, 92, 42, 117, 77, 17, 96, 28, 122, 56, 82, 97, 102, 38, 24, 18, 35, 15, 30, 85, 32, 83, 90, 86, 99, 69, 94, 33, 101, 25, 100, 65, 98, 27, 79, 74, 89, 84, 23, 13, 29, 80, 88, 93, 4, 72, 7, 67, 14, 91, 1, 5, 19, 81, 6, 64, 66, 12, 3, 10, 22, 11, 8, 71, 9, 20, 78, 68, 75, 70, 0, 2, 16, 76, 73], [104, 58, 34, 56, 31, 126, 26, 28, 47, 52, 23, 127, 84, 92, 41, 82, 83, 122, 38, 87, 103, 80, 115, 94, 35, 114, 48, 109, 14, 53, 54, 95, 90, 12, 20, 50, 125, 36, 71, 42, 86, 27, 101, 113, 7, 44, 74, 32, 102, 118, 119, 37, 96, 121, 60, 97, 30, 88, 111, 106, 62, 55, 24, 59, 29, 1, 124, 18, 57, 78, 99, 112, 45, 22, 105, 93, 9, 63, 117, 116, 120, 25, 33, 51, 123, 6, 49, 3, 110, 89, 19, 100, 85, 43, 21, 17, 81, 98, 108, 39, 16, 5, 46, 61, 91, 72, 76, 11, 77, 10, 69, 8, 15, 40, 107, 13, 68, 79, 73, 65, 4, 67, 0, 75, 66, 70, 2, 64]], "model.layers.31.self_attn.k_proj": [[104, 121, 34, 25, 47, 1, 72, 6, 21, 110, 51, 0, 15, 11, 13, 18, 112, 113, 4, 9, 64, 60, 44, 41, 111, 31, 87, 54, 30, 116, 66, 45, 107, 114, 67, 120, 123, 19, 10, 105, 109, 117, 53, 80, 58, 71, 92, 61, 98, 57, 17, 62, 23, 124, 55, 27, 42, 14, 22, 103, 63, 56, 3, 7, 65, 102, 69, 5, 84, 115, 16, 78, 50, 106, 49, 46, 43, 90, 122, 28, 100, 101, 119, 94, 99, 91, 125, 24, 79, 93, 52, 59, 26, 36, 33, 118, 81, 76, 97, 108, 96, 88, 68, 74, 83, 75, 12, 39, 95, 126, 8, 32, 89, 38, 70, 48, 29, 127, 37, 40, 35, 86, 20, 82, 2, 73, 77, 85], [41, 49, 123, 117, 64, 50, 113, 25, 87, 95, 44, 53, 93, 7, 65, 33, 35, 82, 69, 16, 21, 3, 45, 108, 10, 118, 8, 77, 54, 78, 12, 121, 68, 66, 127, 63, 115, 43, 120, 58, 47, 83, 106, 91, 42, 57, 111, 2, 0, 99, 110, 36, 124, 11, 55, 109, 94, 97, 125, 112, 102, 116, 61, 107, 101, 59, 37, 31, 32, 51, 119, 19, 126, 62, 6, 60, 52, 100, 84, 40, 34, 46, 39, 114, 98, 26, 9, 15, 122, 24, 38, 105, 103, 70, 28, 88, 22, 56, 73, 48, 27, 29, 5, 89, 86, 104, 75, 17, 30, 13, 96, 79, 90, 85, 76, 92, 20, 1, 81, 14, 4, 67, 80, 23, 18, 72, 71, 74], [104, 121, 45, 98, 109, 0, 86, 65, 83, 89, 15, 67, 73, 56, 6, 17, 69, 50, 12, 53, 120, 93, 114, 55, 92, 61, 125, 110, 59, 31, 49, 71, 118, 2, 62, 58, 116, 126, 124, 11, 26, 57, 52, 88, 37, 123, 119, 47, 122, 48, 42, 108, 106, 30, 111, 54, 100, 51, 43, 115, 113, 77, 105, 102, 4, 127, 63, 60, 112, 97, 44, 29, 117, 103, 66, 28, 85, 7, 107, 64, 46, 20, 27, 33, 16, 90, 23, 41, 13, 101, 99, 96, 39, 38, 21, 95, 32, 18, 36, 74, 94, 91, 76, 68, 10, 35, 75, 70, 25, 82, 87, 72, 3, 80, 81, 24, 84, 14, 22, 19, 78, 79, 5, 9, 1, 8, 34, 40], [43, 93, 96, 48, 87, 16, 12, 127, 61, 78, 9, 36, 49, 19, 21, 66, 51, 0, 25, 119, 68, 58, 82, 65, 39, 5, 114, 54, 72, 50, 111, 10, 79, 3, 47, 56, 8, 106, 110, 120, 75, 116, 70, 105, 53, 107, 4, 33, 6, 63, 20, 55, 115, 108, 125, 44, 71, 32, 35, 91, 46, 17, 62, 27, 76, 126, 37, 123, 31, 102, 24, 97, 42, 57, 118, 121, 117, 98, 40, 7, 26, 18, 52, 113, 1, 59, 73, 11, 99, 88, 109, 122, 34, 41, 92, 60, 64, 74, 112, 124, 2, 45, 101, 95, 67, 103, 38, 94, 104, 13, 29, 89, 30, 77, 80, 86, 22, 83, 90, 28, 81, 84, 23, 15, 14, 100, 85, 69], [103, 58, 98, 115, 50, 45, 77, 83, 73, 81, 91, 114, 6, 11, 15, 24, 108, 29, 25, 99, 27, 4, 64, 106, 2, 31, 65, 30, 71, 127, 3, 95, 110, 28, 82, 54, 117, 57, 113, 35, 63, 107, 93, 96, 105, 49, 42, 90, 88, 84, 125, 22, 111, 119, 120, 43, 48, 126, 53, 59, 60, 118, 124, 102, 51, 46, 47, 39, 21, 68, 123, 26, 112, 122, 61, 14, 109, 5, 44, 41, 40, 52, 97, 80, 23, 69, 87, 86, 85, 37, 56, 33, 100, 116, 38, 8, 36, 101, 34, 92, 104, 94, 89, 62, 79, 55, 74, 121, 12, 16, 32, 70, 17, 10, 76, 66, 19, 20, 78, 18, 72, 67, 1, 13, 75, 7, 9, 0], [40, 126, 122, 95, 55, 34, 91, 86, 20, 29, 65, 88, 64, 33, 23, 2, 119, 41, 82, 89, 107, 114, 113, 98, 63, 127, 115, 4, 110, 121, 1, 61, 71, 118, 106, 75, 18, 78, 42, 73, 0, 49, 125, 80, 79, 103, 51, 117, 116, 59, 62, 124, 97, 47, 56, 45, 36, 85, 123, 52, 43, 48, 108, 46, 50, 68, 60, 76, 54, 111, 10, 57, 101, 109, 22, 44, 58, 81, 112, 25, 120, 7, 93, 13, 77, 8, 92, 53, 37, 90, 83, 102, 30, 39, 32, 38, 67, 17, 96, 19, 74, 3, 21, 100, 14, 99, 69, 105, 35, 28, 94, 72, 5, 24, 6, 27, 66, 26, 12, 70, 15, 87, 16, 84, 9, 31, 11, 104], [126, 34, 39, 64, 17, 88, 21, 75, 9, 71, 14, 31, 4, 112, 3, 63, 108, 45, 114, 1, 27, 124, 70, 105, 115, 65, 103, 12, 109, 111, 49, 61, 5, 106, 2, 36, 51, 91, 58, 16, 53, 46, 18, 116, 118, 122, 95, 127, 69, 24, 94, 104, 55, 41, 54, 76, 42, 50, 113, 35, 90, 107, 13, 19, 96, 62, 22, 32, 20, 40, 48, 79, 125, 57, 110, 83, 87, 59, 99, 102, 60, 119, 89, 52, 92, 26, 100, 123, 117, 47, 67, 77, 29, 93, 86, 25, 23, 38, 97, 30, 10, 28, 101, 56, 72, 66, 84, 43, 15, 37, 120, 82, 33, 121, 8, 80, 44, 98, 74, 68, 0, 78, 7, 81, 11, 73, 6, 85], [40, 58, 56, 98, 95, 87, 92, 52, 90, 26, 30, 101, 84, 32, 6, 66, 45, 80, 11, 64, 21, 41, 106, 18, 51, 28, 9, 54, 82, 60, 48, 59, 126, 112, 118, 100, 61, 12, 25, 79, 116, 88, 125, 122, 63, 110, 111, 113, 78, 39, 3, 8, 62, 127, 20, 43, 74, 119, 49, 114, 13, 72, 29, 47, 24, 50, 121, 108, 46, 97, 16, 76, 17, 124, 53, 123, 44, 104, 120, 109, 89, 115, 55, 102, 105, 27, 96, 57, 14, 1, 38, 4, 91, 73, 7, 117, 85, 65, 10, 94, 77, 22, 33, 83, 15, 42, 0, 93, 36, 103, 35, 19, 99, 37, 75, 5, 86, 81, 23, 107, 67, 70, 69, 68, 71, 2, 31, 34]], "model.layers.31.self_attn.qk_proj": [[121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 103, 56, 104, 117, 45, 93, 89, 40, 61, 98, 105, 85, 21, 107, 25, 127, 23, 34, 41, 51, 114, 64, 118, 39, 95, 111, 0, 31, 53, 73, 9, 48, 108, 52, 4, 29, 87, 78, 27, 82, 12, 14, 44, 120, 24, 113, 28, 54, 16, 65, 79, 15, 76, 18, 81, 68, 66, 88, 80, 112, 63, 116, 17, 124, 83, 71, 2, 75, 7, 67, 119, 3, 19, 1, 110, 11, 60, 22, 5, 106, 62, 59, 46, 47, 6, 96, 13, 8, 99, 32, 72, 77, 125, 69, 42, 70, 91, 57, 26, 74, 86, 30, 36, 100, 20, 10, 37, 84, 97, 33, 90, 35, 101, 102, 94, 38, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 103, 123, 117, 104, 45, 93, 89, 61, 98, 85, 107, 21, 105, 40, 118, 25, 127, 51, 23, 34, 41, 114, 111, 64, 0, 31, 39, 9, 29, 95, 52, 73, 27, 68, 108, 87, 76, 124, 48, 112, 12, 54, 53, 4, 113, 28, 78, 80, 88, 15, 18, 82, 81, 116, 1, 65, 66, 120, 24, 16, 2, 14, 67, 7, 83, 6, 71, 79, 11, 110, 119, 75, 44, 63, 17, 19, 72, 106, 3, 47, 46, 60, 22, 99, 59, 13, 5, 77, 62, 32, 69, 125, 96, 42, 57, 8, 74, 91, 86, 84, 70, 26, 37, 30, 20, 97, 10, 36, 100, 90, 94, 33, 35, 38, 102, 101, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 98, 21, 89, 93, 105, 107, 40, 61, 85, 51, 114, 25, 34, 41, 0, 64, 127, 118, 108, 111, 23, 95, 39, 31, 53, 9, 68, 87, 73, 52, 12, 27, 48, 82, 63, 15, 29, 14, 28, 124, 78, 4, 116, 24, 88, 120, 112, 44, 54, 76, 113, 18, 2, 80, 16, 1, 81, 66, 6, 65, 7, 79, 110, 72, 17, 83, 67, 3, 60, 19, 75, 119, 11, 71, 46, 59, 69, 106, 47, 99, 22, 5, 62, 77, 32, 13, 96, 125, 42, 91, 86, 100, 57, 20, 84, 37, 70, 26, 33, 10, 36, 97, 30, 74, 8, 35, 94, 90, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 105, 89, 21, 107, 98, 93, 61, 40, 118, 85, 51, 41, 34, 25, 23, 127, 31, 0, 64, 52, 114, 108, 73, 39, 111, 95, 68, 29, 9, 113, 53, 116, 16, 54, 27, 124, 28, 78, 87, 12, 4, 76, 14, 24, 80, 120, 66, 79, 15, 88, 1, 112, 81, 72, 82, 65, 44, 48, 18, 63, 71, 83, 7, 2, 67, 6, 46, 3, 110, 106, 59, 17, 60, 19, 75, 11, 42, 69, 13, 32, 47, 99, 5, 125, 77, 26, 22, 57, 119, 62, 84, 74, 91, 96, 33, 20, 86, 10, 100, 94, 90, 37, 70, 36, 8, 30, 97, 102, 101, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 21, 93, 98, 105, 40, 107, 25, 89, 41, 61, 23, 118, 85, 127, 51, 34, 0, 73, 111, 114, 31, 9, 64, 39, 53, 108, 52, 95, 68, 12, 78, 24, 120, 15, 4, 113, 65, 82, 16, 79, 116, 124, 14, 48, 29, 27, 54, 87, 80, 44, 81, 72, 67, 2, 3, 76, 63, 83, 28, 71, 1, 7, 66, 18, 112, 47, 75, 17, 11, 46, 88, 119, 106, 19, 5, 60, 42, 13, 6, 69, 110, 77, 26, 59, 99, 22, 62, 57, 70, 125, 96, 32, 33, 86, 84, 10, 30, 20, 100, 91, 37, 8, 74, 97, 36, 102, 94, 90, 92, 38, 35, 101], [121, 126, 58, 43, 49, 122, 109, 50, 55, 115, 56, 123, 104, 103, 45, 117, 98, 93, 21, 61, 41, 89, 105, 40, 107, 23, 25, 0, 85, 127, 64, 114, 31, 39, 51, 118, 34, 73, 111, 9, 108, 95, 27, 53, 15, 16, 79, 80, 4, 87, 113, 24, 52, 120, 82, 78, 76, 68, 1, 12, 14, 119, 83, 48, 54, 66, 18, 65, 116, 17, 7, 112, 29, 75, 88, 44, 2, 28, 71, 72, 81, 67, 63, 11, 106, 5, 99, 3, 59, 124, 46, 19, 60, 110, 13, 47, 69, 77, 22, 70, 42, 26, 96, 125, 6, 32, 62, 20, 84, 86, 57, 10, 91, 33, 30, 37, 36, 100, 74, 8, 101, 97, 94, 102, 90, 35, 38, 92], [121, 126, 58, 43, 122, 49, 55, 109, 50, 115, 123, 56, 93, 104, 103, 117, 45, 21, 89, 98, 61, 107, 105, 40, 25, 41, 85, 23, 127, 114, 34, 108, 118, 39, 51, 0, 9, 31, 64, 73, 53, 95, 27, 111, 113, 79, 14, 29, 4, 24, 15, 82, 78, 52, 12, 76, 16, 44, 88, 68, 48, 18, 54, 87, 17, 80, 116, 81, 11, 65, 75, 63, 112, 120, 19, 28, 1, 119, 7, 66, 124, 71, 110, 83, 3, 72, 70, 106, 67, 2, 13, 96, 22, 125, 42, 59, 77, 30, 5, 47, 62, 99, 57, 69, 60, 26, 46, 32, 86, 20, 33, 8, 84, 97, 6, 37, 10, 36, 100, 91, 74, 90, 101, 38, 94, 102, 35, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 93, 98, 21, 89, 61, 40, 85, 105, 41, 107, 25, 114, 108, 23, 34, 118, 51, 127, 0, 64, 39, 111, 9, 95, 53, 31, 73, 29, 27, 24, 113, 124, 82, 4, 14, 79, 44, 48, 76, 68, 52, 18, 112, 12, 16, 28, 87, 116, 120, 66, 15, 54, 80, 63, 78, 81, 1, 7, 119, 75, 65, 3, 83, 71, 88, 70, 17, 2, 19, 11, 60, 67, 59, 106, 47, 110, 22, 13, 72, 77, 99, 125, 46, 5, 96, 57, 42, 32, 26, 30, 69, 6, 91, 8, 36, 86, 62, 37, 84, 10, 74, 97, 20, 33, 101, 90, 100, 94, 38, 102, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 93, 104, 40, 85, 21, 89, 98, 105, 41, 61, 107, 25, 23, 114, 34, 51, 108, 111, 39, 127, 9, 95, 29, 31, 118, 73, 27, 113, 53, 52, 4, 64, 87, 0, 12, 24, 124, 44, 16, 48, 82, 76, 63, 78, 116, 18, 17, 88, 112, 120, 28, 68, 119, 60, 54, 15, 79, 14, 80, 81, 83, 65, 110, 19, 11, 7, 1, 46, 59, 75, 70, 106, 71, 3, 32, 42, 2, 66, 67, 125, 77, 47, 26, 13, 96, 99, 57, 91, 72, 22, 8, 62, 5, 86, 30, 97, 6, 74, 20, 69, 84, 37, 10, 36, 33, 94, 90, 100, 101, 35, 102, 92, 38], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 93, 104, 98, 89, 105, 61, 40, 21, 107, 41, 25, 118, 64, 0, 34, 51, 114, 85, 95, 39, 127, 53, 23, 9, 108, 31, 73, 27, 111, 112, 87, 68, 29, 113, 52, 120, 15, 78, 12, 24, 4, 44, 80, 48, 88, 16, 2, 14, 110, 82, 3, 67, 7, 116, 71, 63, 1, 65, 76, 28, 54, 66, 81, 18, 11, 106, 79, 83, 60, 75, 124, 17, 119, 19, 70, 46, 59, 8, 99, 13, 32, 42, 6, 47, 5, 77, 69, 22, 62, 125, 57, 72, 30, 20, 26, 91, 74, 86, 96, 84, 10, 97, 37, 36, 94, 90, 100, 33, 101, 35, 38, 102, 92], [121, 58, 126, 43, 122, 55, 49, 109, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 21, 105, 61, 41, 89, 0, 40, 107, 127, 23, 118, 64, 34, 25, 51, 114, 9, 73, 85, 39, 31, 108, 52, 95, 111, 4, 16, 78, 15, 12, 113, 87, 14, 76, 80, 24, 124, 68, 27, 44, 53, 79, 1, 82, 120, 29, 65, 67, 83, 8, 116, 54, 88, 48, 18, 71, 7, 63, 11, 28, 2, 66, 3, 75, 60, 81, 5, 17, 110, 46, 19, 106, 112, 13, 70, 119, 59, 42, 6, 32, 77, 99, 69, 62, 22, 57, 86, 26, 47, 91, 125, 84, 74, 10, 20, 97, 96, 33, 72, 37, 100, 30, 36, 94, 90, 35, 102, 101, 92, 38], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 89, 21, 98, 85, 105, 61, 107, 40, 41, 25, 51, 118, 23, 114, 127, 34, 95, 0, 73, 64, 39, 31, 9, 111, 108, 53, 113, 27, 29, 12, 52, 28, 24, 44, 78, 124, 14, 87, 16, 4, 82, 15, 79, 80, 68, 65, 76, 54, 48, 1, 11, 81, 18, 120, 88, 116, 8, 63, 2, 17, 71, 75, 7, 83, 110, 6, 112, 119, 66, 46, 67, 13, 19, 60, 3, 42, 77, 57, 59, 32, 47, 99, 106, 5, 26, 69, 22, 125, 91, 86, 20, 96, 62, 70, 100, 84, 97, 74, 37, 30, 72, 33, 10, 36, 102, 90, 101, 94, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 89, 98, 40, 21, 107, 25, 61, 105, 85, 95, 51, 41, 127, 114, 118, 39, 34, 108, 29, 23, 111, 27, 31, 53, 64, 112, 73, 113, 0, 87, 9, 44, 119, 76, 28, 4, 78, 82, 54, 24, 79, 120, 18, 80, 14, 71, 16, 52, 68, 15, 12, 48, 17, 124, 2, 88, 8, 1, 65, 63, 19, 6, 11, 7, 81, 110, 67, 83, 75, 116, 3, 60, 59, 106, 66, 47, 46, 32, 22, 125, 13, 42, 77, 57, 99, 30, 69, 96, 5, 26, 84, 97, 70, 91, 10, 86, 74, 36, 62, 20, 100, 37, 33, 90, 72, 94, 38, 35, 101, 102, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 104, 103, 117, 45, 93, 21, 89, 98, 61, 64, 40, 105, 107, 51, 127, 34, 0, 25, 41, 118, 85, 31, 73, 114, 95, 23, 39, 9, 52, 111, 4, 53, 108, 29, 113, 76, 87, 27, 78, 16, 1, 28, 82, 44, 71, 48, 8, 12, 24, 68, 18, 67, 112, 110, 81, 3, 120, 88, 65, 14, 116, 80, 15, 2, 11, 6, 60, 79, 66, 83, 54, 63, 19, 7, 106, 124, 75, 17, 125, 119, 47, 13, 46, 99, 5, 69, 59, 77, 42, 91, 62, 32, 96, 86, 10, 57, 97, 22, 26, 74, 84, 37, 20, 70, 72, 36, 30, 100, 33, 101, 94, 90, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 123, 56, 117, 103, 104, 45, 21, 93, 98, 0, 105, 64, 40, 107, 25, 34, 41, 89, 51, 73, 23, 61, 114, 118, 53, 85, 9, 127, 31, 27, 95, 111, 76, 68, 113, 4, 39, 108, 52, 29, 16, 78, 82, 67, 24, 65, 18, 66, 80, 54, 7, 44, 87, 15, 79, 14, 71, 116, 1, 63, 6, 83, 28, 12, 8, 48, 81, 11, 112, 88, 2, 120, 17, 3, 106, 75, 119, 124, 5, 60, 19, 22, 110, 47, 99, 46, 59, 13, 69, 77, 42, 125, 84, 26, 32, 70, 57, 10, 91, 62, 74, 96, 37, 97, 20, 72, 30, 86, 100, 90, 36, 33, 94, 38, 102, 101, 92, 35], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 56, 103, 123, 45, 117, 104, 93, 98, 21, 89, 85, 118, 107, 114, 61, 25, 23, 34, 105, 41, 40, 127, 73, 51, 31, 108, 95, 9, 0, 111, 64, 39, 29, 52, 4, 53, 87, 27, 76, 14, 79, 44, 24, 1, 16, 78, 120, 68, 28, 82, 112, 12, 15, 124, 48, 116, 65, 113, 80, 81, 71, 63, 18, 54, 7, 88, 83, 67, 11, 66, 106, 119, 110, 60, 75, 19, 17, 8, 2, 47, 32, 22, 59, 57, 13, 3, 26, 77, 6, 99, 70, 46, 125, 5, 42, 69, 62, 72, 91, 96, 33, 20, 84, 74, 10, 30, 86, 97, 37, 100, 36, 101, 90, 102, 35, 38, 94, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 117, 103, 104, 45, 93, 89, 85, 21, 40, 107, 25, 105, 23, 98, 61, 31, 34, 127, 39, 51, 114, 95, 118, 64, 41, 111, 0, 108, 29, 9, 27, 73, 87, 53, 28, 82, 52, 112, 68, 88, 76, 113, 24, 15, 48, 1, 81, 14, 120, 44, 18, 79, 4, 71, 78, 16, 106, 12, 80, 54, 2, 110, 66, 67, 17, 63, 19, 65, 116, 11, 119, 83, 75, 7, 60, 70, 32, 125, 13, 124, 3, 59, 47, 77, 91, 42, 96, 8, 57, 62, 99, 97, 72, 26, 46, 22, 69, 37, 5, 20, 74, 30, 100, 6, 86, 10, 33, 84, 90, 35, 36, 38, 101, 94, 102, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 98, 89, 40, 21, 107, 61, 85, 51, 0, 34, 64, 41, 118, 114, 25, 95, 23, 39, 73, 111, 31, 9, 127, 108, 53, 76, 52, 28, 27, 29, 87, 4, 116, 82, 44, 112, 78, 15, 113, 120, 48, 80, 24, 54, 106, 16, 124, 110, 18, 1, 2, 63, 68, 71, 88, 14, 12, 11, 83, 19, 17, 81, 66, 119, 65, 79, 67, 7, 46, 60, 3, 70, 77, 72, 47, 59, 99, 75, 32, 5, 22, 69, 91, 125, 96, 62, 42, 13, 57, 6, 84, 86, 26, 8, 30, 10, 36, 37, 74, 97, 33, 20, 94, 90, 100, 102, 35, 38, 101, 92], [121, 126, 58, 43, 49, 55, 122, 50, 109, 115, 123, 56, 103, 117, 104, 45, 105, 98, 93, 40, 21, 25, 107, 89, 61, 51, 41, 85, 0, 95, 111, 23, 34, 31, 39, 9, 64, 114, 118, 73, 29, 127, 53, 27, 4, 87, 113, 52, 44, 14, 78, 71, 116, 1, 76, 28, 24, 108, 16, 80, 48, 54, 66, 124, 82, 112, 68, 15, 18, 88, 3, 12, 2, 83, 81, 46, 79, 120, 17, 70, 47, 67, 11, 106, 7, 119, 63, 65, 72, 110, 59, 75, 42, 60, 125, 19, 32, 13, 77, 99, 26, 5, 62, 57, 91, 69, 20, 22, 84, 96, 86, 74, 6, 8, 37, 10, 97, 100, 36, 30, 90, 33, 101, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 117, 103, 45, 93, 98, 21, 105, 25, 89, 40, 61, 107, 85, 41, 23, 34, 64, 51, 127, 31, 118, 0, 9, 39, 114, 95, 73, 111, 27, 68, 16, 79, 108, 4, 53, 82, 29, 52, 76, 87, 48, 44, 24, 12, 65, 78, 88, 28, 116, 14, 66, 46, 71, 18, 80, 113, 3, 11, 15, 112, 7, 2, 72, 1, 54, 124, 70, 106, 119, 81, 17, 120, 83, 67, 75, 63, 47, 60, 19, 99, 110, 5, 32, 77, 13, 22, 69, 59, 42, 26, 125, 57, 62, 84, 74, 10, 96, 91, 86, 30, 6, 33, 20, 37, 97, 8, 90, 100, 36, 38, 102, 94, 35, 101, 92], [121, 126, 58, 43, 109, 49, 122, 50, 55, 115, 56, 123, 103, 117, 45, 104, 93, 105, 61, 98, 40, 21, 89, 41, 107, 114, 51, 25, 127, 85, 118, 34, 23, 0, 95, 64, 31, 111, 53, 52, 9, 39, 108, 44, 73, 48, 27, 87, 76, 78, 82, 79, 16, 24, 29, 113, 4, 14, 1, 68, 17, 120, 28, 12, 110, 81, 88, 15, 71, 72, 124, 2, 54, 80, 11, 112, 66, 60, 63, 116, 65, 18, 46, 7, 106, 3, 83, 67, 47, 99, 62, 19, 59, 119, 70, 22, 75, 5, 125, 13, 69, 77, 100, 91, 6, 57, 96, 42, 84, 32, 26, 37, 20, 86, 33, 74, 36, 30, 10, 8, 102, 97, 94, 90, 101, 35, 38, 92], [121, 126, 58, 43, 122, 55, 49, 109, 50, 115, 56, 123, 117, 45, 103, 104, 93, 105, 98, 61, 64, 51, 89, 95, 21, 40, 41, 107, 85, 0, 25, 34, 118, 23, 31, 29, 114, 53, 108, 39, 9, 111, 52, 73, 127, 44, 87, 4, 27, 124, 16, 113, 116, 14, 68, 79, 76, 18, 67, 82, 120, 66, 78, 2, 28, 112, 63, 12, 48, 1, 106, 80, 72, 15, 24, 17, 88, 54, 65, 7, 47, 83, 110, 81, 71, 119, 3, 19, 11, 75, 6, 46, 125, 60, 57, 69, 99, 59, 5, 13, 77, 32, 70, 42, 26, 22, 62, 84, 96, 86, 91, 10, 97, 20, 8, 74, 33, 37, 30, 90, 101, 100, 36, 94, 102, 38, 35, 92], [121, 58, 126, 43, 122, 109, 49, 50, 55, 115, 56, 123, 117, 103, 45, 104, 93, 105, 98, 21, 40, 89, 61, 25, 107, 51, 41, 34, 85, 114, 118, 127, 95, 23, 31, 39, 111, 0, 108, 73, 64, 53, 29, 9, 27, 52, 87, 4, 113, 44, 28, 112, 76, 78, 14, 48, 79, 82, 88, 3, 80, 54, 71, 116, 16, 15, 110, 17, 18, 120, 1, 24, 7, 12, 124, 47, 63, 68, 67, 83, 2, 65, 81, 59, 60, 119, 19, 106, 11, 72, 6, 75, 66, 5, 125, 46, 77, 13, 69, 32, 99, 57, 22, 26, 96, 42, 62, 91, 20, 86, 74, 97, 84, 8, 10, 30, 70, 37, 33, 36, 100, 90, 38, 35, 94, 92, 101, 102], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 123, 56, 103, 117, 104, 45, 93, 21, 98, 61, 40, 105, 23, 89, 25, 41, 107, 118, 34, 85, 127, 31, 51, 0, 111, 108, 114, 64, 95, 73, 39, 9, 52, 27, 53, 24, 113, 4, 76, 18, 15, 12, 54, 80, 79, 87, 120, 44, 16, 68, 78, 82, 48, 29, 28, 112, 6, 1, 14, 81, 71, 63, 88, 2, 65, 7, 17, 75, 66, 11, 124, 83, 116, 119, 72, 106, 47, 19, 46, 3, 67, 110, 69, 22, 59, 60, 99, 42, 125, 77, 32, 13, 26, 62, 5, 91, 74, 96, 8, 57, 86, 84, 100, 20, 33, 97, 10, 36, 30, 37, 70, 90, 94, 101, 102, 35, 38, 92], [121, 58, 126, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 117, 104, 45, 93, 98, 40, 89, 21, 105, 41, 61, 85, 107, 51, 25, 127, 23, 34, 95, 118, 114, 0, 39, 9, 73, 31, 64, 53, 108, 111, 27, 52, 4, 113, 79, 87, 44, 48, 78, 12, 80, 28, 68, 24, 14, 76, 54, 3, 120, 29, 82, 15, 81, 110, 66, 65, 116, 18, 75, 63, 1, 112, 124, 7, 6, 88, 16, 11, 71, 17, 2, 83, 119, 106, 67, 19, 42, 60, 46, 22, 99, 47, 77, 62, 57, 96, 72, 8, 13, 59, 32, 5, 69, 125, 91, 70, 30, 20, 26, 97, 74, 86, 84, 37, 10, 33, 36, 100, 90, 101, 38, 102, 35, 94, 92], [121, 126, 58, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 45, 117, 104, 89, 93, 98, 40, 105, 21, 85, 34, 107, 25, 114, 41, 23, 118, 95, 127, 51, 64, 31, 61, 39, 111, 29, 108, 0, 27, 112, 53, 73, 28, 87, 44, 9, 12, 113, 52, 68, 18, 78, 88, 24, 15, 82, 124, 116, 120, 54, 48, 14, 76, 60, 79, 16, 66, 1, 7, 75, 81, 65, 80, 4, 17, 63, 110, 11, 19, 2, 71, 106, 6, 119, 47, 59, 83, 99, 67, 3, 13, 46, 77, 42, 8, 125, 96, 22, 32, 57, 91, 5, 62, 70, 69, 20, 86, 37, 30, 74, 72, 84, 26, 36, 97, 100, 94, 38, 10, 33, 101, 90, 35, 102, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 40, 98, 21, 89, 105, 127, 61, 25, 107, 23, 85, 34, 41, 111, 114, 51, 108, 73, 118, 39, 31, 95, 0, 29, 9, 64, 48, 28, 52, 68, 53, 27, 54, 112, 87, 24, 82, 12, 120, 113, 4, 78, 76, 44, 88, 16, 14, 79, 17, 80, 1, 15, 18, 116, 110, 81, 83, 63, 65, 2, 75, 7, 47, 59, 46, 66, 19, 71, 60, 8, 67, 119, 11, 106, 3, 13, 6, 42, 32, 99, 62, 125, 124, 5, 77, 26, 70, 69, 57, 86, 96, 37, 30, 91, 33, 72, 22, 97, 20, 84, 74, 100, 10, 36, 102, 94, 35, 90, 38, 101, 92], [121, 58, 126, 43, 122, 49, 109, 50, 55, 115, 56, 123, 103, 117, 104, 45, 93, 98, 21, 61, 89, 40, 105, 41, 85, 107, 23, 0, 25, 51, 34, 114, 64, 95, 31, 127, 111, 73, 118, 39, 9, 113, 108, 29, 24, 53, 52, 27, 68, 4, 120, 76, 79, 16, 18, 82, 28, 87, 12, 78, 44, 54, 116, 1, 124, 14, 15, 80, 7, 71, 48, 46, 3, 8, 66, 2, 75, 88, 67, 83, 112, 81, 110, 65, 63, 11, 17, 60, 70, 106, 99, 125, 47, 19, 22, 5, 119, 13, 59, 42, 77, 57, 32, 69, 62, 26, 74, 6, 37, 10, 97, 91, 96, 84, 100, 72, 20, 86, 33, 30, 36, 90, 94, 101, 102, 35, 38, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 103, 104, 117, 45, 93, 98, 105, 89, 21, 64, 40, 61, 85, 0, 107, 51, 25, 114, 118, 23, 41, 127, 9, 53, 34, 111, 95, 31, 73, 39, 108, 48, 113, 1, 68, 12, 52, 27, 44, 54, 87, 4, 29, 120, 14, 80, 18, 78, 70, 24, 66, 7, 8, 79, 28, 82, 15, 76, 63, 16, 112, 67, 71, 81, 124, 75, 46, 65, 88, 2, 3, 17, 110, 116, 69, 83, 11, 47, 106, 119, 19, 125, 59, 60, 77, 57, 32, 13, 99, 42, 62, 5, 26, 96, 10, 22, 33, 84, 86, 74, 97, 100, 20, 37, 91, 30, 36, 6, 72, 90, 102, 101, 94, 35, 92, 38], [121, 58, 126, 43, 122, 49, 50, 55, 109, 115, 56, 123, 103, 104, 117, 45, 93, 21, 40, 105, 98, 89, 61, 85, 127, 23, 41, 25, 107, 118, 114, 51, 34, 31, 39, 95, 73, 111, 9, 108, 64, 53, 0, 27, 29, 48, 12, 44, 79, 52, 82, 68, 87, 24, 54, 15, 18, 14, 113, 80, 78, 28, 16, 76, 120, 81, 70, 65, 112, 17, 116, 4, 88, 124, 75, 8, 83, 7, 110, 3, 1, 71, 11, 60, 19, 66, 63, 59, 106, 125, 47, 119, 2, 67, 13, 57, 69, 46, 77, 99, 5, 32, 96, 22, 42, 62, 20, 30, 26, 86, 10, 74, 91, 84, 97, 100, 33, 37, 36, 72, 94, 6, 90, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 109, 55, 50, 115, 56, 123, 104, 103, 117, 45, 98, 93, 89, 105, 61, 21, 40, 107, 41, 23, 25, 51, 127, 64, 34, 85, 0, 114, 39, 118, 31, 108, 73, 111, 9, 95, 4, 12, 53, 15, 87, 44, 29, 27, 113, 28, 79, 68, 52, 1, 78, 48, 14, 112, 80, 24, 16, 120, 116, 63, 76, 18, 65, 70, 2, 7, 54, 71, 67, 66, 81, 8, 82, 110, 11, 83, 124, 17, 3, 19, 47, 88, 75, 106, 46, 119, 125, 13, 60, 69, 59, 32, 5, 77, 99, 57, 62, 22, 96, 91, 6, 42, 86, 26, 30, 97, 72, 37, 20, 74, 84, 10, 90, 36, 94, 33, 100, 101, 102, 38, 35, 92], [121, 126, 58, 43, 122, 49, 50, 109, 55, 115, 123, 56, 103, 117, 104, 45, 93, 40, 89, 85, 105, 107, 98, 61, 21, 127, 23, 25, 114, 34, 118, 41, 95, 111, 51, 39, 9, 108, 29, 73, 31, 52, 53, 27, 12, 0, 24, 64, 18, 16, 120, 14, 113, 78, 87, 79, 54, 44, 112, 48, 4, 76, 15, 82, 80, 88, 68, 63, 28, 17, 1, 75, 81, 83, 119, 116, 8, 71, 11, 19, 7, 65, 106, 124, 2, 67, 3, 110, 66, 60, 70, 59, 5, 77, 13, 47, 96, 22, 62, 32, 99, 69, 46, 6, 42, 125, 57, 91, 86, 20, 26, 30, 72, 84, 74, 97, 10, 37, 36, 33, 94, 90, 100, 101, 35, 38, 102, 92]]} diff --git a/test/srt/entrypoints/http_server/test_abort_request.py b/test/srt/entrypoints/http_server/test_abort_request.py new file mode 100644 index 000000000..7a366f244 --- /dev/null +++ b/test/srt/entrypoints/http_server/test_abort_request.py @@ -0,0 +1,206 @@ +""" +Integration test for abort_request functionality with a SGLang server. + +Run with: + python -m unittest sglang.test.srt.entrypoints.http_server.test_abort_request -v +""" + +import threading +import time +import unittest +from typing import Optional + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAbortRequest(CustomTestCase): + """Integration test class for abort request functionality.""" + + model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + + @classmethod + def setUpClass(cls): + """Launch the server.""" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-cuda-graph"], + ) + + cls.completion_url = f"{cls.base_url}/generate" + cls.abort_url = f"{cls.base_url}/abort_request" + cls.health_url = f"{cls.base_url}/health" + + print(f"Server started at {cls.base_url}") + + @classmethod + def tearDownClass(cls): + """Clean up the server.""" + kill_process_tree(cls.process.pid) + + def _send_completion_request( + self, + text: str, + request_id: str, + max_tokens: int = 50, + temperature: float = 0.8, + stream: bool = True, + ) -> requests.Response: + """Send a completion request to the server.""" + payload = { + "text": text, + "sampling_params": { + "max_new_tokens": max_tokens, + "temperature": temperature, + }, + "stream": stream, + "rid": request_id, + } + + response = requests.post( + self.completion_url, + json=payload, + headers={"Content-Type": "application/json"}, + timeout=30, + stream=stream, + ) + + return response + + def _send_abort_request(self, request_id: str) -> requests.Response: + """Send an abort request.""" + payload = {"rid": request_id} + return requests.post(self.abort_url, json=payload, timeout=10) + + def _check_server_health(self) -> bool: + """Check if server is healthy.""" + try: + response = requests.get(self.health_url, timeout=5) + return response.status_code == 200 + except: + return False + + def test_abort_during_non_streaming_generation(self): + """Test aborting a non-streaming request during generation.""" + self.assertTrue(self._check_server_health(), "Server should be healthy") + + request_id = "test_abort_non_streaming" + completion_result = {} + + def run_completion(): + response = self._send_completion_request( + "Write a detailed essay about artificial intelligence", + max_tokens=500, + temperature=1, + request_id=request_id, + stream=False, + ) + + if response.status_code == 200: + result = response.json() + completion_result["text"] = result.get("text", "") + completion_result["finish_reason"] = result.get("meta_info", {}).get( + "finish_reason" + ) + + completion_thread = threading.Thread(target=run_completion) + completion_thread.start() + time.sleep(0.1) + + abort_response = self._send_abort_request(request_id) + completion_thread.join() + + self.assertEqual(abort_response.status_code, 200) + self.assertIsNotNone(completion_result, "Should have completion result") + if completion_result: + finish_reason_obj = completion_result.get("finish_reason") + self.assertIsNotNone(finish_reason_obj, "Should have finish_reason") + if finish_reason_obj: + self.assertEqual( + finish_reason_obj.get("type"), "abort", "Should be aborted" + ) + + def test_batch_requests_with_selective_abort(self): + """Test multiple concurrent requests with selective abort of one request.""" + self.assertTrue(self._check_server_health(), "Server should be healthy") + + request_ids = ["batch_test_0", "batch_test_1", "batch_test_2"] + abort_target_id = "batch_test_1" + completion_results = {} + threads = [] + + def run_completion(req_id, prompt): + response = self._send_completion_request( + f"Write a story about {prompt}", + max_tokens=100, + temperature=0.8, + request_id=req_id, + stream=False, + ) + + if response.status_code == 200: + result = response.json() + completion_results[req_id] = { + "text": result.get("text", ""), + "finish_reason": result.get("meta_info", {}).get("finish_reason"), + } + + # Start all requests + prompts = ["a knight's adventure", "a space discovery", "a chef's restaurant"] + for i, req_id in enumerate(request_ids): + thread = threading.Thread(target=run_completion, args=(req_id, prompts[i])) + threads.append(thread) + thread.start() + + # Abort one request + time.sleep(0.1) + abort_response = self._send_abort_request(abort_target_id) + + # Wait for completion + for thread in threads: + thread.join(timeout=30) + + # Verify results + self.assertEqual(abort_response.status_code, 200) + + # Check aborted request + aborted_result = completion_results.get(abort_target_id) + self.assertIsNotNone( + aborted_result, f"Aborted request {abort_target_id} should have result" + ) + if aborted_result: + aborted_finish_reason = aborted_result.get("finish_reason") + self.assertIsNotNone( + aborted_finish_reason, "Aborted request should have finish_reason" + ) + if aborted_finish_reason: + self.assertEqual(aborted_finish_reason.get("type"), "abort") + + # Check other requests completed normally + normal_completions = 0 + for req_id in request_ids: + if req_id != abort_target_id and req_id in completion_results: + result = completion_results[req_id] + if result: + finish_reason = result.get("finish_reason") + if finish_reason and finish_reason.get("type") == "length": + normal_completions += 1 + + self.assertEqual( + normal_completions, 2, "Other 2 requests should complete normally" + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2, warnings="ignore") diff --git a/test/srt/ep/test_deepep_internode.py b/test/srt/ep/test_deepep_internode.py new file mode 100644 index 000000000..1e4239606 --- /dev/null +++ b/test/srt/ep/test_deepep_internode.py @@ -0,0 +1,445 @@ +# Copy from deepseek-ai/DeepEP/tests/test_internode.py + +import os +import time + +# noinspection PyUnresolvedReferences +import deep_ep + +# Test compatibility with low latency functions +import test_deepep_low_latency +import torch +import torch.distributed as dist + +from sglang.test.test_deepep_utils import ( + bench, + calc_diff, + create_grouped_scores, + init_dist, + inplace_unique, + per_token_cast_back, + per_token_cast_to_fp8, +) + + +def test_main( + num_sms: int, + local_rank: int, + num_local_ranks: int, + num_ranks: int, + num_nodes: int, + rank: int, + buffer: deep_ep.Buffer, + group: dist.ProcessGroup, +): + # Settings + num_tokens, hidden, num_topk_groups, num_topk, num_experts = ( + 4096, + 7168, + min(num_nodes, 4), + 8, + (256 // num_ranks) * num_ranks, + ) + assert num_experts % num_ranks == 0 and num_local_ranks == 8 + if local_rank == 0: + print( + f"[config] num_tokens={num_tokens}, hidden={hidden}, num_topk_groups={num_topk_groups}, num_topk={num_topk}", + flush=True, + ) + + # Random data + x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * rank + x_pure_rand = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") + x_e4m3 = per_token_cast_to_fp8(x) + scores = ( + torch.randn((num_tokens, num_experts), dtype=torch.float32, device="cuda").abs() + + 1 + ) + group_scores = scores.view(num_tokens, num_nodes, -1).amax(dim=-1) + group_idx = torch.topk( + group_scores, k=num_topk_groups, dim=-1, sorted=False + ).indices + masked_scores = create_grouped_scores(scores, group_idx, num_nodes) + topk_idx = torch.topk(masked_scores, num_topk, dim=-1, largest=True, sorted=False)[ + 1 + ] + topk_weights = ( + torch.ones((num_tokens, num_topk), dtype=torch.float32, device="cuda") * rank + ) + topk_weights_pure_rand = torch.randn( + (num_tokens, num_topk), dtype=torch.float32, device="cuda" + ) + rank_idx = topk_idx // (num_experts // num_ranks) + rank_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rank_idx, num_ranks) + rdma_rank_idx = rank_idx // num_local_ranks + rdma_rank_idx.masked_fill_(rank_idx == -1, -1) + inplace_unique(rdma_rank_idx, num_nodes) + + # RDMA dispatch counts + rdma_idx = topk_idx // (num_experts // num_nodes) + rdma_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rdma_idx, num_nodes) + num_rdma_token_sent = rdma_idx.ne(-1).sum().item() + + # Expert meta + num_tokens_per_expert = torch.zeros((num_experts,), dtype=torch.int, device="cuda") + for i in range(num_experts): + num_tokens_per_expert[i] = (topk_idx == i).sum() + gbl_num_tokens_per_expert = num_tokens_per_expert.clone() + dist.all_reduce(gbl_num_tokens_per_expert, group=group) + + # Rank layout meta + num_tokens_per_rank = torch.empty((num_ranks,), dtype=torch.int, device="cuda") + num_tokens_per_rdma_rank = torch.empty((num_nodes,), dtype=torch.int, device="cuda") + token_idx_in_rank = torch.full( + (num_ranks, num_tokens), -1, dtype=torch.long, device="cuda" + ) + for i in range(num_ranks): + num_tokens_per_rank[i] = (rank_idx == i).sum() + token_sel = (rank_idx == i).max(dim=-1)[0] + count = token_sel.sum().item() + tokens = torch.sort(token_sel.to(torch.int), descending=True)[1] + tokens[:count] = torch.sort(tokens[:count])[0] + token_idx_in_rank[i][tokens[:count]] = torch.arange( + count, dtype=torch.long, device="cuda" + ) + for i in range(num_nodes): + num_tokens_per_rdma_rank[i] = (rdma_rank_idx == i).sum() + token_idx_in_rank = token_idx_in_rank.T.contiguous().to(torch.int) + is_token_in_rank = token_idx_in_rank >= 0 + gbl_num_tokens_per_rank = num_tokens_per_rank.clone() + dist.all_reduce(gbl_num_tokens_per_rank, group=group) + + ( + ref_num_tokens_per_rank, + ref_num_tokens_per_rdma_rank, + ref_num_tokens_per_expert, + ref_is_token_in_rank, + _, + ) = buffer.get_dispatch_layout(topk_idx, num_experts) + assert torch.allclose(ref_num_tokens_per_rank, num_tokens_per_rank) + assert torch.allclose(ref_num_tokens_per_rdma_rank, num_tokens_per_rdma_rank) + assert torch.allclose(ref_num_tokens_per_expert, num_tokens_per_expert) + assert torch.allclose(ref_is_token_in_rank, is_token_in_rank) + t = bench(lambda: buffer.get_dispatch_layout(topk_idx, num_experts))[0] + if local_rank == 0: + print(f"[layout] Kernel performance: {t * 1000:.3f} ms", flush=True) + print("", flush=True) + group.barrier() + time.sleep(1) + + # Config + rdma_buffer_size, nvl_buffer_size = 128, (720 if num_ranks in (144, 160) else 512) + config = deep_ep.Config(num_sms, 8, nvl_buffer_size, 16, rdma_buffer_size) + + # Test dispatch + # noinspection PyShadowingNames + def check_data(check_x, recv_gbl_rank_prefix_sum): + assert torch.allclose(check_x.amin(dim=1), check_x.amax(dim=1)) + check_start = 0 + for i in range(num_ranks): + check_end = recv_gbl_rank_prefix_sum[i].item() + assert (check_x[check_start:check_end, :].int() - i).sum().item() == 0 + check_start = check_end + + for previous_mode in (False, True): + for async_mode in (False, True): + for current_x in (x_pure_rand, x, x_e4m3): + for with_topk in (False, True): + if local_rank == 0: + print( + f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...', + flush=True, + end="", + ) + dispatch_args = { + "x": current_x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": config, + "async_finish": async_mode, + } + if with_topk: + dispatch_args.update( + { + "topk_idx": topk_idx, + "topk_weights": ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ), + } + ) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + ( + recv_x, + recv_topk_idx, + recv_topk_weights, + recv_num_tokens_per_expert_list, + handle, + event, + ) = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + + # Checks + recv_gbl_rank_prefix_sum = handle[-4] + assert gbl_num_tokens_per_rank[rank].item() == recv_x.size( + 0 + ), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}" + assert ( + gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist() + == recv_num_tokens_per_expert_list + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + if with_topk: + # Check `topk_idx` + assert ( + recv_topk_idx.eq(-1) + | ( + (recv_topk_idx >= 0) + & (recv_topk_idx < (num_experts // num_ranks)) + ) + ).sum().item() == recv_topk_idx.numel() + for i, count in enumerate(recv_num_tokens_per_expert_list): + assert recv_topk_idx.eq(i).sum().item() == count + + # Check `topk_weights` + if current_x is not x_pure_rand: + recv_topk_weights[recv_topk_idx.eq(-1)] = ( + recv_topk_weights.amax(dim=1, keepdim=True).expand_as( + recv_topk_weights + )[recv_topk_idx.eq(-1)] + ) + check_data(recv_topk_weights, recv_gbl_rank_prefix_sum) + + # Test cached dispatch (must without top-k staffs) + if not with_topk: + dispatch_args = { + "x": current_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + recv_x, _, _, _, _, event = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + if current_x is not x_pure_rand: + check_data(recv_x, recv_gbl_rank_prefix_sum) + + # Test combine + combine_args = { + "x": recv_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if with_topk: + combine_args.update({"topk_weights": recv_topk_weights}) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + combined_x, combined_topk_weights, event = buffer.combine( + **combine_args + ) + event.current_stream_wait() if async_mode else () + check_x = combined_x.float() / is_token_in_rank.sum( + dim=1 + ).unsqueeze(1) + ref_x = x_pure_rand if current_x is x_pure_rand else x + assert calc_diff(check_x, ref_x) < 5e-6 + if with_topk: + check_topk_weights = ( + combined_topk_weights + if (current_x is x_pure_rand) + else ( + combined_topk_weights + / is_token_in_rank.sum(dim=1).unsqueeze(1) + ) + ) + ref_topk_weights = ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ) + assert calc_diff(check_topk_weights, ref_topk_weights) < 1e-9 + + # For later tuning + dispatch_bf16_rdma_send_bytes = num_rdma_token_sent * hidden * 2 + dispatch_bf16_nvl_recv_bytes = recv_x.numel() * 2 + combine_bf16_nvl_send_bytes = dispatch_bf16_nvl_recv_bytes + combine_bf16_rdma_recv_bytes = dispatch_bf16_rdma_send_bytes + + if local_rank == 0: + print(" passed", flush=True) + if local_rank == 0: + print("", flush=True) + + # Tune dispatch performance + best_dispatch_results = None + fp8_factor = (1 + 4 / 128) / 2 + for current_x in (x_e4m3, x): + best_time, best_results = 1e10, None + rdma_send_bytes = ( + (dispatch_bf16_rdma_send_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_rdma_send_bytes + ) + nvl_recv_bytes = ( + (dispatch_bf16_nvl_recv_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_nvl_recv_bytes + ) + for nvl_chunk_size in range(4, 33, 4): + for rdma_chunk_size in range(4, 33, 4): + config = deep_ep.Config( + num_sms, + nvl_chunk_size, + nvl_buffer_size, + rdma_chunk_size, + rdma_buffer_size, + ) + tune_args = {"x": current_x, "handle": handle, "config": config} + t = bench(lambda: buffer.dispatch(**tune_args))[0] + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + ) + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {rdma_send_bytes / 1e9 / t:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if local_rank == 0: + print( + f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {rdma_send_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)', + flush=True, + ) + print("", flush=True) + + if isinstance(current_x, tuple): + # Gather FP8 the best config from rank 0 + best_dispatch_results = torch.tensor( + [best_results[0], best_results[1], best_results[2]], + dtype=torch.int32, + device="cuda", + ) + all_best_fp8_results_list = [ + torch.zeros_like(best_dispatch_results) + for _ in range(torch.distributed.get_world_size()) + ] + dist.all_gather( + all_best_fp8_results_list, best_dispatch_results, group=group + ) + best_dispatch_results = all_best_fp8_results_list[0].tolist() + dispatch_config = deep_ep.Config( + best_dispatch_results[0], + best_dispatch_results[1], + nvl_buffer_size, + best_dispatch_results[2], + rdma_buffer_size, + ) + + dispatch_args = { + "x": x, + "num_tokens_per_rank": num_tokens_per_rank, + "num_tokens_per_rdma_rank": num_tokens_per_rdma_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": dispatch_config if dispatch_config is not None else config, + } + recv_x, _, _, _, handle, _ = buffer.dispatch(**dispatch_args) + + # Tune combine performance + best_time, best_results = 1e10, None + for nvl_chunk_size in range(1, 5, 1): + for rdma_chunk_size in range(8, 33, 4): + config = deep_ep.Config( + num_sms, + nvl_chunk_size, + nvl_buffer_size, + rdma_chunk_size, + rdma_buffer_size, + ) + tune_args = {"x": recv_x, "handle": handle, "config": config} + t = bench(lambda: buffer.combine(**tune_args))[0] + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}, RDMA chunk {rdma_chunk_size}: {combine_bf16_rdma_recv_bytes / 1e9 / t:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if t < best_time: + best_time, best_results = t, ( + num_sms, + nvl_chunk_size, + rdma_chunk_size, + ) + + if local_rank == 0: + print( + f"[tuning] Best combine: SMs {best_results[0]}, NVL chunk {best_results[1]}, RDMA chunk {best_results[2]}: {combine_bf16_rdma_recv_bytes / 1e9 / best_time:.2f} GB/s (RDMA), {combine_bf16_nvl_send_bytes / 1e9 / best_time:.2f} GB/s (NVL)", + flush=True, + ) + print("", flush=True) + + +# noinspection PyUnboundLocalVariable +def test_loop(local_rank: int, num_local_ranks: int): + num_nodes = int(os.getenv("WORLD_SIZE", 1)) + rank, num_ranks, group = init_dist(local_rank, num_local_ranks) + test_ll_compatibility = False + if test_ll_compatibility: + ll_num_tokens, ll_hidden, ll_num_experts, ll_num_topk = 16, 5120, 256, 9 + + buffer = deep_ep.Buffer( + group, + int(1e9), + int(1e9), + low_latency_mode=test_ll_compatibility, + num_qps_per_rank=(ll_num_experts // num_ranks if test_ll_compatibility else 1), + ) + assert num_local_ranks == 8 and num_ranks > 8 + torch.manual_seed(rank) + + for i in (24,): + test_main( + i, local_rank, num_local_ranks, num_ranks, num_nodes, rank, buffer, group + ) + if local_rank == 0: + print("", flush=True) + + # Test compatibility with low latency functions + if test_ll_compatibility: + buffer.clean_low_latency_buffer(ll_num_tokens, ll_hidden, ll_num_experts) + test_deepep_low_latency.test_main( + ll_num_tokens, + ll_hidden, + ll_num_experts, + ll_num_topk, + rank, + num_ranks, + group, + buffer, + seed=1, + ) + + +if __name__ == "__main__": + num_processes = 8 + torch.multiprocessing.spawn(test_loop, args=(num_processes,), nprocs=num_processes) diff --git a/test/srt/ep/test_deepep_intranode.py b/test/srt/ep/test_deepep_intranode.py new file mode 100644 index 000000000..97acd000c --- /dev/null +++ b/test/srt/ep/test_deepep_intranode.py @@ -0,0 +1,379 @@ +# Copy from deepseek-ai/DeepEP/tests/test_intranode.py + +import os +import time + +# noinspection PyUnresolvedReferences +import deep_ep + +# Test compatibility with low latency functions +import test_deepep_low_latency +import torch +import torch.distributed as dist + +from sglang.test.test_deepep_utils import ( + bench, + calc_diff, + init_dist, + inplace_unique, + per_token_cast_back, + per_token_cast_to_fp8, +) + + +def test_main( + num_sms: int, + local_rank: int, + num_ranks: int, + rank: int, + buffer: deep_ep.Buffer, + group: dist.ProcessGroup, +): + # Settings + num_tokens, hidden, num_topk, num_experts = ( + 4096, + 7168, + 8, + (256 // num_ranks) * num_ranks, + ) + assert num_experts % num_ranks == 0 + if local_rank == 0: + print( + f"[config] num_tokens={num_tokens}, hidden={hidden}, num_topk={num_topk}", + flush=True, + ) + + # Random data + x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * rank + x_pure_rand = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") + x_e4m3 = per_token_cast_to_fp8(x) + scores = ( + torch.randn((num_tokens, num_experts), dtype=torch.float32, device="cuda").abs() + + 1 + ) + topk_idx = torch.topk(scores, num_topk, dim=-1, largest=True, sorted=False)[1] + topk_weights = ( + torch.ones((num_tokens, num_topk), dtype=torch.float32, device="cuda") * rank + ) + topk_weights_pure_rand = torch.randn( + (num_tokens, num_topk), dtype=torch.float32, device="cuda" + ) + rank_idx = topk_idx // (num_experts // num_ranks) + rank_idx.masked_fill_(topk_idx == -1, -1) + inplace_unique(rank_idx, num_ranks) + + # Expert meta + num_tokens_per_expert = torch.zeros((num_experts,), dtype=torch.int, device="cuda") + for i in range(num_experts): + num_tokens_per_expert[i] = (topk_idx == i).sum() + gbl_num_tokens_per_expert = num_tokens_per_expert.clone() + dist.all_reduce(gbl_num_tokens_per_expert, group=group) + + # Rank layout meta + num_tokens_per_rank = torch.empty((num_ranks,), dtype=torch.int, device="cuda") + token_idx_in_rank = torch.full( + (num_ranks, num_tokens), -1, dtype=torch.long, device="cuda" + ) + for i in range(num_ranks): + num_tokens_per_rank[i] = (rank_idx == i).sum() + token_sel = (rank_idx == i).max(dim=-1)[0] + count = token_sel.sum().item() + tokens = torch.sort(token_sel.to(torch.int), descending=True)[1] + tokens[:count] = torch.sort(tokens[:count])[0] + token_idx_in_rank[i][tokens[:count]] = torch.arange( + count, dtype=torch.long, device="cuda" + ) + token_idx_in_rank = token_idx_in_rank.T.contiguous().to(torch.int) + is_token_in_rank = token_idx_in_rank >= 0 + gbl_num_tokens_per_rank = num_tokens_per_rank.clone() + dist.all_reduce(gbl_num_tokens_per_rank, group=group) + + ref_num_tokens_per_rank, _, ref_num_tokens_per_expert, ref_is_token_in_rank, _ = ( + buffer.get_dispatch_layout(topk_idx, num_experts) + ) + assert torch.allclose(ref_num_tokens_per_rank, num_tokens_per_rank) + assert torch.allclose(ref_num_tokens_per_expert, num_tokens_per_expert) + assert torch.allclose(ref_is_token_in_rank, is_token_in_rank) + t = bench(lambda: buffer.get_dispatch_layout(topk_idx, num_experts))[0] + if local_rank == 0: + print(f"[layout] Kernel performance: {t * 1000:.3f} ms", flush=True) + print("", flush=True) + group.barrier() + time.sleep(1) + + # Config + nvl_buffer_size = 256 + config = deep_ep.Config(num_sms, 8, nvl_buffer_size) + + # Test dispatch + # noinspection PyShadowingNames + def check_data(check_x, rank_prefix_matrix): + assert torch.allclose(check_x.amin(dim=1), check_x.amax(dim=1)) + check_start = 0 + for i in range(num_ranks): + check_end = rank_prefix_matrix[i][rank].item() + assert (check_x[check_start:check_end, :].int() - i).sum().item() == 0 + check_start = check_end + + for previous_mode in (False, True): + for async_mode in (False, True): + for current_x in (x_pure_rand, x, x_e4m3): + for with_topk in (False, True): + if local_rank == 0: + print( + f'[testing] Running with {"FP8" if isinstance(current_x, tuple) else "BF16"}, {"with" if with_topk else "without"} top-k (async={async_mode}, previous={previous_mode}) ...', + flush=True, + end="", + ) + dispatch_args = { + "x": current_x, + "num_tokens_per_rank": num_tokens_per_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": config, + "async_finish": async_mode, + } + if with_topk: + dispatch_args.update( + { + "topk_idx": topk_idx, + "topk_weights": ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ), + } + ) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + ( + recv_x, + recv_topk_idx, + recv_topk_weights, + recv_num_tokens_per_expert_list, + handle, + event, + ) = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + + # Checks + rank_prefix_matrix = handle[0] + assert gbl_num_tokens_per_rank[rank].item() == recv_x.size( + 0 + ), f"{gbl_num_tokens_per_rank[rank].item()} != {recv_x.size(0)}" + assert ( + gbl_num_tokens_per_expert.view(num_ranks, -1)[rank].tolist() + == recv_num_tokens_per_expert_list + ) + if current_x is not x_pure_rand: + check_data(recv_x, rank_prefix_matrix) + if with_topk: + # Check `topk_idx` + assert ( + recv_topk_idx.eq(-1) + | ( + (recv_topk_idx >= 0) + & (recv_topk_idx < (num_experts // num_ranks)) + ) + ).sum().item() == recv_topk_idx.numel() + for i, count in enumerate(recv_num_tokens_per_expert_list): + assert recv_topk_idx.eq(i).sum().item() == count + + # Check `topk_weights` + if current_x is not x_pure_rand: + recv_topk_weights[recv_topk_idx.eq(-1)] = ( + recv_topk_weights.amax(dim=1, keepdim=True).expand_as( + recv_topk_weights + )[recv_topk_idx.eq(-1)] + ) + check_data(recv_topk_weights, rank_prefix_matrix) + + # Test cached dispatch (must without top-k staffs) + if not with_topk: + dispatch_args = { + "x": current_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + recv_x, _, _, _, _, event = buffer.dispatch(**dispatch_args) + event.current_stream_wait() if async_mode else () + recv_x = ( + per_token_cast_back(*recv_x) + if isinstance(recv_x, tuple) + else recv_x + ) + if current_x is not x_pure_rand: + check_data(recv_x, rank_prefix_matrix) + + # Test combine + combine_args = { + "x": recv_x, + "handle": handle, + "config": config, + "async_finish": async_mode, + } + if with_topk: + combine_args.update({"topk_weights": recv_topk_weights}) + if previous_mode: + dispatch_args.update({"previous_event": buffer.capture()}) + combined_x, combined_topk_weights, event = buffer.combine( + **combine_args + ) + event.current_stream_wait() if async_mode else () + check_x = combined_x.float() / is_token_in_rank.sum( + dim=1 + ).unsqueeze(1) + ref_x = x_pure_rand if current_x is x_pure_rand else x + assert calc_diff(check_x, ref_x) < 5e-6 + if with_topk: + check_topk_weights = ( + combined_topk_weights + if (current_x is x_pure_rand) + else ( + combined_topk_weights + / is_token_in_rank.sum(dim=1).unsqueeze(1) + ) + ) + ref_topk_weights = ( + topk_weights_pure_rand + if current_x is x_pure_rand + else topk_weights + ) + assert calc_diff(check_topk_weights, ref_topk_weights) < 1e-9 + + # For later tuning + dispatch_bf16_nvl_recv_bytes = recv_x.numel() * 2 + combine_bf16_nvl_send_bytes = dispatch_bf16_nvl_recv_bytes + + if local_rank == 0: + print(" passed", flush=True) + if local_rank == 0: + print("", flush=True) + + # Tune dispatch performance + best_dispatch_results = None + fp8_factor = (1 + 4 / 128) / 2 + for current_x in (x_e4m3, x): + best_time, best_results = 1e10, None + nvl_recv_bytes = ( + (dispatch_bf16_nvl_recv_bytes * fp8_factor) + if isinstance(current_x, tuple) + else dispatch_bf16_nvl_recv_bytes + ) + for nvl_chunk_size in range(4, 33, 4): + config = deep_ep.Config(num_sms, nvl_chunk_size, nvl_buffer_size) + tune_args = {"x": current_x, "handle": handle, "config": config} + t = bench(lambda: buffer.dispatch(**tune_args))[0] + if t < best_time: + best_time, best_results = t, (num_sms, nvl_chunk_size) + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}: {nvl_recv_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if local_rank == 0: + print( + f'[tuning] Best dispatch ({"FP8" if isinstance(current_x, tuple) else "BF16"}): SMs {best_results[0]}, NVL chunk {best_results[1]}, {nvl_recv_bytes / 1e9 / best_time:.2f} GB/s (NVL)', + flush=True, + ) + print("", flush=True) + + if isinstance(current_x, tuple): + # Gather FP8 the best config from rank 0 + best_dispatch_results = torch.tensor( + [best_results[0], best_results[1]], dtype=torch.int32, device="cuda" + ) + all_best_fp8_results_list = [ + torch.zeros_like(best_dispatch_results) + for _ in range(torch.distributed.get_world_size()) + ] + dist.all_gather( + all_best_fp8_results_list, best_dispatch_results, group=group + ) + best_dispatch_results = all_best_fp8_results_list[0].tolist() + dispatch_config = deep_ep.Config( + best_dispatch_results[0], best_dispatch_results[1], nvl_buffer_size + ) + + dispatch_args = { + "x": x, + "num_tokens_per_rank": num_tokens_per_rank, + "is_token_in_rank": is_token_in_rank, + "num_tokens_per_expert": num_tokens_per_expert, + "config": dispatch_config if dispatch_config is not None else config, + } + recv_x, _, _, _, handle, _ = buffer.dispatch(**dispatch_args) + + # Tune combine performance + best_time, best_results = 1e10, None + for nvl_chunk_size in range(1, 7, 1): + config = deep_ep.Config(num_sms, nvl_chunk_size, nvl_buffer_size) + tune_args = {"x": recv_x, "handle": handle, "config": config} + t = bench(lambda: buffer.combine(**tune_args))[0] + if local_rank == 0: + print( + f"[tuning] SMs {num_sms}, NVL chunk {nvl_chunk_size}: {combine_bf16_nvl_send_bytes / 1e9 / t:.2f} GB/s (NVL) ", + flush=True, + ) + if t < best_time: + best_time, best_results = t, (num_sms, nvl_chunk_size) + + if local_rank == 0: + print( + f"[tuning] Best combine: SMs {best_results[0]}, NVL chunk {best_results[1]}: {combine_bf16_nvl_send_bytes / 1e9 / best_time:.2f} GB/s (NVL)", + flush=True, + ) + print("", flush=True) + + +# noinspection PyUnboundLocalVariable +def test_loop(local_rank: int, num_local_ranks: int): + rank, num_ranks, group = init_dist(local_rank, num_local_ranks) + test_ll_compatibility, num_rdma_bytes = False, 0 + if test_ll_compatibility: + ll_num_tokens, ll_hidden, ll_num_experts, ll_num_topk = 16, 5120, 256, 9 + num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint( + ll_num_tokens, ll_hidden, num_ranks, ll_num_experts + ) + + buffer = deep_ep.Buffer( + group, + int(1e9), + num_rdma_bytes, + low_latency_mode=test_ll_compatibility, + num_qps_per_rank=(ll_num_experts // num_ranks if test_ll_compatibility else 1), + ) + torch.manual_seed(rank) + + for i in (24,): + test_main(i, local_rank, num_ranks, rank, buffer, group) + if local_rank == 0: + print("", flush=True) + + # Test compatibility with low latency functions + if test_ll_compatibility: + buffer.clean_low_latency_buffer(ll_num_tokens, ll_hidden, ll_num_experts) + test_deepep_low_latency.test_main( + ll_num_tokens, + ll_hidden, + ll_num_experts, + ll_num_topk, + rank, + num_ranks, + group, + buffer, + seed=1, + ) + + +if __name__ == "__main__": + num_processes = 8 + torch.multiprocessing.spawn(test_loop, args=(num_processes,), nprocs=num_processes) diff --git a/test/srt/ep/test_deepep_large.py b/test/srt/ep/test_deepep_large.py new file mode 100644 index 000000000..94fff566c --- /dev/null +++ b/test/srt/ep/test_deepep_large.py @@ -0,0 +1,149 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDeepseek(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--ep-num-redundant-experts", + "32", + "--ep-dispatch-algorithm", + "dynamic", + "--eplb-algorithm", + "deepseek", + "--cuda-graph-bs", + "256", + "--max-running-requests", + "2048", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +class TestDeepseekMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--ep-num-redundant-experts", + "32", + "--ep-dispatch-algorithm", + "dynamic", + "--eplb-algorithm", + "deepseek", + "--cuda-graph-bs", + "64", # TODO: increase it to 128 when TBO is supported in draft_extend + "--max-running-requests", + "512", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k:\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 1.85) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_deepep_low_latency.py b/test/srt/ep/test_deepep_low_latency.py new file mode 100644 index 000000000..1b3f34da9 --- /dev/null +++ b/test/srt/ep/test_deepep_low_latency.py @@ -0,0 +1,325 @@ +# Copy from deepseek-ai/DeepEP/tests/test_low_latency.py + +import random +from functools import partial + +import deep_ep +import torch +import torch.distributed as dist + +from sglang.test.test_deepep_utils import ( + bench, + bench_kineto, + calc_diff, + hash_tensor, + init_dist, + per_token_cast_back, +) + + +def test_main( + num_tokens: int, + hidden: int, + num_experts: int, + num_topk: int, + rank: int, + num_ranks: int, + group: dist.ProcessGroup, + buffer: deep_ep.Buffer, + seed: int = 0, +): + torch.manual_seed(seed + rank) + random.seed(seed + rank) + + assert num_experts % num_ranks == 0 + num_local_experts = num_experts // num_ranks + + # NOTES: the integers greater than 256 exceeds the BF16 precision limit + rank_offset = 128 + assert ( + num_ranks - rank_offset < 257 + ), "Too many ranks (exceeding test precision limit)" + + x = torch.ones((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") * ( + rank - rank_offset + ) + x[:, -128:] = torch.arange(num_tokens, device="cuda").to(torch.bfloat16).view(-1, 1) + scores = ( + torch.randn((num_tokens, num_experts), dtype=torch.float32, device="cuda").abs() + + 1 + ) + topk_idx = torch.topk(scores, num_topk, dim=-1, largest=True, sorted=True)[1] + topk_weights = torch.randn( + (num_tokens, num_topk), dtype=torch.float32, device="cuda" + ).abs() + + # Randomly mask some positions + for i in range(10): + topk_idx[random.randint(0, num_tokens - 1), random.randint(0, num_topk - 1)] = ( + -1 + ) + + # Check dispatch correctness + do_check = True + hash_value, num_times = 0, 0 + for return_recv_hook in (False, True): + for dispatch_use_fp8 in (False, True): + num_times += 1 + for i in range((num_times % 2) + 1): + packed_recv_x, packed_recv_count, handle, event, hook = ( + buffer.low_latency_dispatch( + x, + topk_idx, + num_tokens, + num_experts, + use_fp8=dispatch_use_fp8, + async_finish=not return_recv_hook, + return_recv_hook=return_recv_hook, + ) + ) + hook() if return_recv_hook else event.current_stream_wait() + packed_recv_x = ( + (packed_recv_x[0], packed_recv_x[1].contiguous()) + if dispatch_use_fp8 + else packed_recv_x + ) + simulated_gemm_x = ( + per_token_cast_back( + packed_recv_x[0].view(-1, hidden), + packed_recv_x[1].view(-1, hidden // 128), + ).view(packed_recv_x[0].shape) + if dispatch_use_fp8 + else packed_recv_x.clone() + ) + all_topk_idx = torch.empty( + (num_ranks, num_tokens, num_topk), dtype=topk_idx.dtype, device="cuda" + ) + dist.all_gather_into_tensor(all_topk_idx, topk_idx, group=group) + for i in range(num_local_experts if do_check else 0): + expert_id = rank * num_local_experts + i + recv_x = ( + per_token_cast_back(packed_recv_x[0][i], packed_recv_x[1][i]) + if dispatch_use_fp8 + else packed_recv_x[i] + ) + recv_count, recv_src_info, recv_layout_range = ( + packed_recv_count[i], + handle[0][i], + handle[1][i], + ) + + # Check expert indices + int_mask = (2**32) - 1 + num_valid_tokens = recv_count.item() + assert ( + num_valid_tokens == (recv_layout_range & int_mask).sum().item() + ), f"{num_valid_tokens} != {recv_layout_range & int_mask}.sum().item()" + assert ( + num_valid_tokens == (all_topk_idx == expert_id).sum().item() + ), f"{num_valid_tokens} != {(all_topk_idx == expert_id).sum().item()}" + + # Check received data + recv_x = recv_x[:num_valid_tokens] + recv_x_amin = recv_x[:, :-128].amin(dim=-1) + recv_src_info = recv_src_info[:num_valid_tokens] + assert torch.equal(recv_x_amin, recv_x[:, :-128].amax(dim=-1)) + assert ( + recv_x[:, -128:] - recv_src_info.view(-1, 1) % num_tokens + ).sum().item() == 0 + for j in range(num_ranks): + begin_idx, count = (recv_layout_range[j] >> 32).item(), ( + recv_layout_range[j] & int_mask + ).item() + assert (recv_x_amin == j - rank_offset).sum().item() == ( + all_topk_idx[j] == expert_id + ).sum().item() + assert ( + recv_x[begin_idx : begin_idx + count][:-128] - j + ).sum().item() == 0 + if dispatch_use_fp8: + hash_value ^= hash_tensor(packed_recv_x[0][i, :num_valid_tokens]) + hash_value ^= hash_tensor(packed_recv_x[1][i, :num_valid_tokens]) + else: + hash_value ^= hash_tensor(packed_recv_x[i, :num_valid_tokens]) + + # Check combine correctness + for zero_copy in (False, True): + if zero_copy: + buffer.get_next_low_latency_combine_buffer(handle)[ + :, :, : + ] = simulated_gemm_x + out = torch.empty( + (num_tokens, hidden), dtype=torch.bfloat16, device="cuda" + ) + combined_x, event, hook = buffer.low_latency_combine( + simulated_gemm_x, + topk_idx, + topk_weights, + handle, + async_finish=not return_recv_hook, + zero_copy=zero_copy, + return_recv_hook=return_recv_hook, + out=out, + ) + hook() if return_recv_hook else event.current_stream_wait() + if do_check: + diff = calc_diff( + x + * topk_weights.masked_fill(topk_idx == -1, 0) + .sum(dim=1) + .view(-1, 1), + combined_x, + ) + assert torch.isnan(combined_x).sum().item() == 0 + assert diff < 1e-5, f"Error: {diff=}, {zero_copy=}" + hash_value ^= hash_tensor(combined_x) + + def create_test_cast_with_outliers(num_outliers): + tmp = torch.randn((num_tokens, hidden), dtype=torch.bfloat16, device="cuda") + tmp /= tmp.abs().amax(dim=1).view(-1, 1) + assert tmp.abs().amax().item() <= 1 + + # Create some amax outliers + for i in range(num_outliers): + tmp[random.randint(0, num_tokens - 1)] *= 1e3 + return tmp + + # noinspection PyShadowingNames + def large_gemm_with_hook(hook): + mat_0 = torch.randn((8192, 8192), dtype=torch.float) + mat_1 = torch.randn((8192, 8192), dtype=torch.float) + mat_0 @ mat_1 + hook() + + # noinspection PyShadowingNames + def test_func(zero_copy: bool, return_recv_hook: bool): + recv_x, recv_count, handle, event, hook = buffer.low_latency_dispatch( + x, + topk_idx, + num_tokens, + num_experts, + async_finish=False, + return_recv_hook=return_recv_hook, + ) + large_gemm_with_hook(hook) if return_recv_hook else None + if zero_copy: + buffer.get_next_low_latency_combine_buffer(handle)[ + :, :, : + ] = simulated_gemm_x + combined_x, event, hook = buffer.low_latency_combine( + simulated_gemm_x, + topk_idx, + topk_weights, + handle, + zero_copy=zero_copy, + return_recv_hook=return_recv_hook, + ) + large_gemm_with_hook(hook) if return_recv_hook else None + + # Calculate bandwidth + num_fp8_bytes, num_bf16_bytes = (hidden + hidden / 128 * 4 + 16), hidden * 2 + num_dispatch_comm_bytes, num_combine_comm_bytes = 0, 0 + for i in range(num_tokens): + num_selections = (topk_idx[i] != -1).sum().item() + num_dispatch_comm_bytes += num_fp8_bytes * num_selections + num_combine_comm_bytes += num_bf16_bytes * num_selections + + # Dispatch + combine testing + avg_t, min_t, max_t = bench( + partial(test_func, zero_copy=False, return_recv_hook=False) + ) + print( + f"[rank {rank}] Dispatch + combine bandwidth: {(num_dispatch_comm_bytes + num_combine_comm_bytes) / 1e9 / avg_t:.2f} GB/s, " + f"avg_t={avg_t * 1e6:.2f} us, min_t={min_t * 1e6:.2f} us, max_t={max_t * 1e6:.2f} us", + flush=True, + ) + + # Separate profiling + for return_recv_hook in (False, True): + group.barrier() + dispatch_t, combine_t = bench_kineto( + partial(test_func, zero_copy=True, return_recv_hook=return_recv_hook), + kernel_names=("dispatch", "combine"), + barrier_comm_profiling=True, + suppress_kineto_output=True, + ) + if not return_recv_hook: + print( + f"[rank {rank}] Dispatch bandwidth: {num_dispatch_comm_bytes / 1e9 / dispatch_t:.2f} GB/s, avg_t={dispatch_t * 1e6:.2f} us | " + f"Combine bandwidth: {num_combine_comm_bytes / 1e9 / combine_t:.2f} GB/s, avg_t={combine_t * 1e6:.2f} us", + flush=True, + ) + else: + print( + f"[rank {rank}] Dispatch send/recv time: {dispatch_t * 2 * 1e6:.2f} us | " + f"Combine send/recv time: {combine_t * 2 * 1e6:.2f} us", + flush=True, + ) + + return hash_value + + +# noinspection PyUnboundLocalVariable +def test_loop(local_rank: int, num_local_ranks: int): + rank, num_ranks, group = init_dist(local_rank, num_local_ranks) + num_tokens, hidden, num_topk, num_experts = 128, 7168, 8, 288 + + num_rdma_bytes = deep_ep.Buffer.get_low_latency_rdma_size_hint( + num_tokens, hidden, num_ranks, num_experts + ) + if local_rank == 0: + print(f"Allocating buffer size: {num_rdma_bytes / 1e6} MB ...", flush=True) + buffer = deep_ep.Buffer( + group, + num_rdma_bytes=num_rdma_bytes, + low_latency_mode=True, + num_qps_per_rank=num_experts // num_ranks, + ) + test_main( + num_tokens, + hidden, + num_experts, + num_topk, + rank, + num_ranks, + group, + buffer, + seed=1, + ) + + do_pressure_test = False + for seed in range(int(1e9) if do_pressure_test else 0): + if local_rank == 0: + print(f"Testing with seed {seed} ...", flush=True) + ref_hash = test_main( + num_tokens, + hidden, + num_experts, + num_topk, + rank, + num_ranks, + group, + buffer, + seed=seed, + ) + for i in range(20): + assert ( + test_main( + num_tokens, + hidden, + num_experts, + num_topk, + rank, + num_ranks, + group, + buffer, + seed=seed, + ) + == ref_hash + ), f"Error: seed={seed}" + + +if __name__ == "__main__": + # TODO: you may modify NUMA binding for less CPU overhead + num_processes = 8 + torch.multiprocessing.spawn(test_loop, args=(num_processes,), nprocs=num_processes) diff --git a/test/srt/ep/test_deepep_small.py b/test/srt/ep/test_deepep_small.py new file mode 100644 index 000000000..05aefe79a --- /dev/null +++ b/test/srt/ep/test_deepep_small.py @@ -0,0 +1,389 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPureDP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "512", + "--mem-fraction-static", + "0.5", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestHybridDPTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "2", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "256", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +@unittest.skip("covered in test_deepep_large.py") +class TestNoGatherdBuffer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "512", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestTBO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--enable-two-batch-overlap", + "--cuda-graph-max-bs", + "128", + "--max-running-requests", + "512", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +@unittest.skip("covered in TestMTPWithTBO") +class TestMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "4", + "--enable-dp-attention", + "--dp", + "2", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "3", + "--speculative-num-draft-tokens", + "3", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "64", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.1) + + +class TestMTPWithTBO(CustomTestCase): + @classmethod + def setUpClass(cls): + import os + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--enable-dp-attention", + "--dp-size", + "4", + "--enable-two-batch-overlap", + "--moe-a2a-backend", + "deepep", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "3", + "--speculative-num-draft-tokens", + "3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--chunked-prefill-size", + "256", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp + tbo):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_eplb.py b/test/srt/ep/test_eplb.py new file mode 100755 index 000000000..c2acc07bb --- /dev/null +++ b/test/srt/ep/test_eplb.py @@ -0,0 +1,155 @@ +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace + +import sglang as sgl +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class _BaseTestDynamicEPLB(CustomTestCase): + extra_args = [] + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--dp", + "2", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-cuda-graph", + "--enable-eplb", + "--ep-num-redundant-experts", + "4", + "--eplb-rebalance-num-iterations", + "50", + "--expert-distribution-recorder-buffer-size", + "50", + # TODO pr-chain: enable later + # "--enable-expert-distribution-metrics", + # TODO auto determine these flags + "--expert-distribution-recorder-mode", + "stat", + "--ep-dispatch-algorithm", + "static", + *cls.extra_args, + ], + env={ + "SGL_ENABLE_JIT_DEEPGEMM": "0", + "SGLANG_EXPERT_LOCATION_UPDATER_CANARY": "1", + **os.environ, + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +class TestDynamicEPLBSimple(_BaseTestDynamicEPLB): + pass + + +class TestDynamicEPLBMultiChunk(_BaseTestDynamicEPLB): + extra_args = ["--eplb-rebalance-layers-per-chunk", "1"] + + +class TestStaticEPLB(CustomTestCase): + def test_save_expert_distribution_and_init_expert_location(self): + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "0" + + with tempfile.TemporaryDirectory() as tmp_dir: + engine_kwargs = dict( + model_path=DEFAULT_MLA_MODEL_NAME_FOR_TEST, + trust_remote_code=True, + ep_num_redundant_experts=4, + enable_dp_attention=True, + moe_a2a_backend="deepep", + disable_cuda_graph=True, + expert_distribution_recorder_mode="stat", + tp_size=2, + dp_size=2, + log_level="info", + # TODO pr-chain: enable later + # enable_expert_distribution_metrics=True, + ) + + print(f"Action: start engine") + os.environ["SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR"] = tmp_dir + engine = sgl.Engine( + **engine_kwargs, + disable_overlap_schedule=True, + ) + engine.start_expert_distribution_record() + self._assert_engine_generate_correct(engine) + + print(f"Action: dump_expert_distribution_record") + engine.dump_expert_distribution_record() + snapshot_path = list(Path(tmp_dir).glob("*.pt"))[0] + assert snapshot_path is not None + print(f"{snapshot_path=}") + + print(f"Action: shutdown engine") + engine.shutdown() + del engine + + print(f"Action: start engine with init_expert_location") + engine = sgl.Engine( + **engine_kwargs, + init_expert_location=str(snapshot_path), + port=21000, + # TODO auto determine these flags + ep_dispatch_algorithm="static", + ) + self._assert_engine_generate_correct(engine) + print(f"Action: shutdown engine") + engine.shutdown() + del engine + + def _assert_engine_generate_correct(self, engine: sgl.Engine): + output = engine.generate( + prompt=["1+1=2, 2+2=4", "One plus one is two, two plus two is four"], + sampling_params=dict(max_new_tokens=8, temperature=0.0), + ) + print(f"engine.generate {output=}") + self.assertEqual( + [x["text"] for x in output], + [", 4+4=8,", ", four plus four is eight, eight"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_hybrid_dp_ep_tp_mtp.py b/test/srt/ep/test_hybrid_dp_ep_tp_mtp.py new file mode 100644 index 000000000..65fbad428 --- /dev/null +++ b/test/srt/ep/test_hybrid_dp_ep_tp_mtp.py @@ -0,0 +1,2728 @@ +# Comprehensive test for hybrid parallelism (DP/TP attention, DP/TP Dense FFN, TP/EP Sparse FFN, DP/VP LM head) plus speculative decoding. +# These tests are not run by default but can be launched on demand. + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST, + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class Test00(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test01(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test02(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test03(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test04(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test05(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test06(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test07(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test08(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test09(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test10(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test11(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test12(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test13(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test14(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test15(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test16(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test17(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test18(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test19(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test20(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test21(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test22(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test23(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test24(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test25(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test26(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test27(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test28(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test29(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--ep", + "8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test30(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test31(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test32(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test33(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test34(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test35(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test36(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test37(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test38(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test39(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test40(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test41(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test42(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test43(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test44(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test45(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test46(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test47(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test48(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test49(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "32", + "--max-running-requests", + "32", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test50(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test51(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test52(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test53(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test54(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test55(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test56(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--enable-dp-lm-head", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test57(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--enable-dp-lm-head", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test58(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "4", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +class Test59(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--enable-dp-attention", + "--dp", + "8", + "--moe-dense-tp-size", + "1", + "--enable-dp-lm-head", + "--ep", + "8", + "--speculative-algo", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/DeepSeek-V3-0324-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.48) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_moe_deepep.py b/test/srt/ep/test_moe_deepep.py new file mode 100644 index 000000000..aa9d7a1f8 --- /dev/null +++ b/test/srt/ep/test_moe_deepep.py @@ -0,0 +1,119 @@ +import json +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPureTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--moe-a2a-backend", + "deepep", + "--disable-cuda-graph", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +class TestDPAttn(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--dp", + "2", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-cuda-graph", + # Test custom config + "--deepep-config", + json.dumps( + { + "normal_dispatch": { + "num_sms": 20, + "num_max_nvl_chunked_send_tokens": 16, + "num_max_nvl_chunked_recv_tokens": 256, + "num_max_rdma_chunked_send_tokens": 6, + "num_max_rdma_chunked_recv_tokens": 128, + }, + "normal_combine": { + "num_sms": 20, + "num_max_nvl_chunked_send_tokens": 6, + "num_max_nvl_chunked_recv_tokens": 256, + "num_max_rdma_chunked_send_tokens": 6, + "num_max_rdma_chunked_recv_tokens": 128, + }, + } + ), + ], + env={ + "SGL_ENABLE_JIT_DEEPGEMM": "0", + **os.environ, + }, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_moe_deepep_eval_accuracy_large.py b/test/srt/ep/test_moe_deepep_eval_accuracy_large.py new file mode 100644 index 000000000..66797ffa1 --- /dev/null +++ b/test/srt/ep/test_moe_deepep_eval_accuracy_large.py @@ -0,0 +1,75 @@ +""" +Usage: +python -m unittest test_moe_deepep_eval_accuracy_large.TestMoEDeepEPEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMoEDeepEPEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_DEEPPEP_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "8", + "--moe-a2a-backend", + "deepep", + "--cuda-graph-max-bs", + "128", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=200, + parallel=64, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.93) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"Eval accuracy of MMLU: {metrics=}") + self.assertGreater(metrics["score"], 0.87) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/ep/test_moe_ep.py b/test/srt/ep/test_moe_ep.py new file mode 100644 index 000000000..7456c9329 --- /dev/null +++ b/test/srt/ep/test_moe_ep.py @@ -0,0 +1,112 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEpMoE(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.8) + + +class TestEpMoEFP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--ep-size", + "2", + "--quantization", + "fp8", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/experiment_runner.py b/test/srt/experiment_runner.py new file mode 100644 index 000000000..f32f61d3b --- /dev/null +++ b/test/srt/experiment_runner.py @@ -0,0 +1,365 @@ +import argparse +import logging +import os +import queue +import re +import subprocess +import threading +import time +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import psutil +import requests +import yaml + + +@dataclass +class ServerConfig: + command: str + process_names: List[str] + default_port: int + + +@dataclass +class TaskConfig: + server_cmd: str + client_cmd: str + name: Optional[str] = None + server_type: Optional[str] = None + + +@dataclass +class TaskResult: + name: str + success: bool + output: str + runtime: float + timestamp: str + + +SERVER_DEFAULTS = { + "sglang": ServerConfig( + command="sglang.launch_server", + process_names=["sglang.launch_server"], + default_port=30000, + ), + "vllm": ServerConfig( + command="vllm.entrypoints.openai.api_server", + process_names=["vllm.entrypoints.openai.api_server"], + default_port=8000, + ), +} + + +def parse_key_info(output: str) -> str: + """Extract and format key information from the output""" + key_info = [] + + # Extract Args namespace + args_match = re.search(r"Namespace\(.*?\)", output, re.DOTALL) + if args_match: + key_info.append(args_match.group(0)) + + # Extract input/output token counts + token_matches = re.findall(r"#(Input|Output) tokens: \d+", output) + key_info.extend(token_matches) + + # Extract benchmark result section + result_match = re.search( + r"============ Serving Benchmark Result ============.*?={50,}", + output, + re.DOTALL, + ) + if result_match: + key_info.append(result_match.group(0)) + + return "\n\n".join(key_info) + + +def extract_port_from_command(cmd: str, server_type: str) -> int: + port_match = re.search(r"--port[= ](\d+)", cmd) + if port_match: + return int(port_match.group(1)) + return SERVER_DEFAULTS.get(server_type, ServerConfig("", [], 8000)).default_port + + +def detect_server_type(cmd: str) -> str: + for server_type, config in SERVER_DEFAULTS.items(): + if config.command in cmd: + return server_type + return "unknown" + + +def stream_output( + process: subprocess.Popen, prefix: str, logger: logging.Logger +) -> queue.Queue: + output_queue = queue.Queue() + + def stream_pipe(pipe, prefix): + for line in iter(pipe.readline, ""): + if prefix == "CLIENT": + output_queue.put(line.rstrip()) + logger.debug(f"{prefix} | {line.rstrip()}") + + stdout_thread = threading.Thread( + target=stream_pipe, args=(process.stdout, prefix), daemon=True + ) + stderr_thread = threading.Thread( + target=stream_pipe, args=(process.stderr, prefix), daemon=True + ) + + stdout_thread.start() + stderr_thread.start() + return output_queue, (stdout_thread, stderr_thread) + + +class ProcessManager: + def __init__(self): + self.server_process: Optional[subprocess.Popen] = None + self.client_process: Optional[subprocess.Popen] = None + self.logger = logging.getLogger(__name__) + + def start_process( + self, command: str, prefix: str + ) -> Tuple[subprocess.Popen, queue.Queue]: + process = subprocess.Popen( + command, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + output_queue, threads = stream_output(process, prefix, self.logger) + return process, output_queue, threads + + def kill_process_tree(self, process: subprocess.Popen): + try: + parent = psutil.Process(process.pid) + children = parent.children(recursive=True) + + for child in children: + try: + child.kill() + except psutil.NoSuchProcess: + pass + + parent.kill() + gone, alive = psutil.wait_procs(children + [parent], timeout=3) + + for p in alive: + try: + p.kill() + except psutil.NoSuchProcess: + pass + + except psutil.NoSuchProcess: + pass + + def cleanup(self, process_names: List[str]): + if self.client_process: + self.kill_process_tree(self.client_process) + self.client_process = None + + if self.server_process: + self.kill_process_tree(self.server_process) + self.server_process = None + + for proc in psutil.process_iter(["pid", "name", "cmdline"]): + try: + cmdline = " ".join(proc.cmdline()) + if any(name in cmdline for name in process_names): + proc.kill() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + + +class ExperimentRunner: + def __init__(self): + self.process_manager = ProcessManager() + self.logger = logging.getLogger(__name__) + + def wait_for_server(self, port: int, timeout: int = 300) -> bool: + start_time = time.perf_counter() + + while time.perf_counter() - start_time < timeout: + try: + response = requests.get(f"http://localhost:{port}/health") + if response.status_code == 200: + self.logger.debug(f"Server ready on port {port}") + return True + except requests.RequestException: + time.sleep(2) + return False + + def run_task(self, config: TaskConfig) -> TaskResult: + start_time = time.perf_counter() + client_output = [] + + try: + if not config.server_type: + config.server_type = detect_server_type(config.server_cmd) + + server_config = SERVER_DEFAULTS.get(config.server_type) + if not server_config: + raise ValueError(f"Unknown server type: {config.server_type}") + + port = extract_port_from_command(config.server_cmd, config.server_type) + + self.process_manager.cleanup(server_config.process_names) + + self.logger.debug(f"Starting server: {config.name}") + self.process_manager.server_process, _, server_threads = ( + self.process_manager.start_process(config.server_cmd, "SERVER") + ) + + if not self.wait_for_server(port): + raise TimeoutError("Server startup timeout") + + time.sleep(10) + + self.logger.debug("Starting client") + self.process_manager.client_process, output_queue, client_threads = ( + self.process_manager.start_process(config.client_cmd, "CLIENT") + ) + + returncode = self.process_manager.client_process.wait() + + while True: + try: + line = output_queue.get_nowait() + client_output.append(line) + except queue.Empty: + break + + if returncode != 0: + raise RuntimeError(f"Client failed with code {returncode}") + + # Parse and format the output + full_output = "\n".join(client_output) + formatted_output = parse_key_info(full_output) + + return TaskResult( + name=config.name, + success=True, + output=formatted_output, + runtime=time.perf_counter() - start_time, + timestamp=datetime.now().isoformat(), + ) + + except Exception as e: + return TaskResult( + name=config.name, + success=False, + output=str(e), + runtime=time.perf_counter() - start_time, + timestamp=datetime.now().isoformat(), + ) + + finally: + if config.server_type in SERVER_DEFAULTS: + self.process_manager.cleanup( + SERVER_DEFAULTS[config.server_type].process_names + ) + time.sleep(10) + + +def load_config(config_path: str) -> List[TaskConfig]: + with open(config_path, "r") as f: + config_data = yaml.safe_load(f) + + configs = [] + for idx, entry in enumerate(config_data.get("tasks", [])): + if not isinstance(entry, dict): + raise ValueError(f"Invalid entry at index {idx}") + + config = TaskConfig( + server_cmd=entry.get("server_cmd"), + client_cmd=entry.get("client_cmd"), + name=entry.get("name", f"task-{idx+1}"), + server_type=entry.get("server_type"), + ) + + if not config.server_cmd or not config.client_cmd: + raise ValueError(f"Missing commands in {config.name}") + + configs.append(config) + + return configs + + +def setup_logging(debug: bool = False): + level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s - %(levelname)s - %(message)s", + handlers=[logging.StreamHandler(), logging.FileHandler("experiment.log")], + ) + + +def format_results(results: List[TaskResult]) -> str: + """Format experiment results in Markdown for GitHub step summary.""" + output = ["# Experiment Results\n"] + + for result in results: + output.append(f"## {result.name}") + output.append(f"**Status**: {'✅ Success' if result.success else '❌ Failed'}") + output.append(f"**Runtime**: {result.runtime:.2f} seconds") + output.append(f"**Timestamp**: {result.timestamp}") + output.append("\n**Output**:\n```") + output.append(result.output) + output.append("```\n") + + return "\n".join(output) + + +def get_bool_env_var(name: str, default: str = "false") -> bool: + value = os.getenv(name, default) + return value.lower() in ("true", "1") + + +def write_in_github_step_summary(results: List[TaskResult]): + """Write formatted results to GitHub step summary.""" + if not os.environ.get("GITHUB_STEP_SUMMARY"): + logging.warning("GITHUB_STEP_SUMMARY environment variable not set") + return + + formatted_content = format_results(results) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(formatted_content) + + +def main(): + parser = argparse.ArgumentParser(description="Experiment Runner") + parser.add_argument( + "--config", type=str, required=True, help="Path to YAML config file" + ) + parser.add_argument("--debug", action="store_true", help="Enable debug output") + args = parser.parse_args() + + setup_logging(args.debug) + logger = logging.getLogger(__name__) + results = [] + + try: + configs = load_config(args.config) + runner = ExperimentRunner() + + for config in configs: + logger.info(f"Running {config.name}") + result = runner.run_task(config) + results.append(result) + + if get_bool_env_var("SGLANG_IS_IN_CI"): + write_in_github_step_summary(results) + except Exception as e: + logger.error(f"Error: {e}") + raise + + +if __name__ == "__main__": + main() diff --git a/test/srt/hicache/test_hicache.py b/test/srt/hicache/test_hicache.py new file mode 100644 index 000000000..f7616d098 --- /dev/null +++ b/test/srt/hicache/test_hicache.py @@ -0,0 +1,53 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() + + +class TestHiCache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-hierarchical-cache", + "--mem-fraction-static", + 0.7, + "--hicache-size", + 100 if not _is_hip else 200, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/hicache/test_hicache_mla.py b/test/srt/hicache/test_hicache_mla.py new file mode 100644 index 000000000..c5db0f74a --- /dev/null +++ b/test/srt/hicache/test_hicache_mla.py @@ -0,0 +1,67 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() +if _is_hip: + hicache_args = ["--hicache-size", 200] +else: + hicache_args = ["--hicache-ratio", 2] + + +class TestHierarchicalMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--enable-hierarchical-cache", + ] + + hicache_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/hicache/test_hicache_page.py b/test/srt/hicache/test_hicache_page.py new file mode 100644 index 000000000..b1e1459c2 --- /dev/null +++ b/test/srt/hicache/test_hicache_page.py @@ -0,0 +1,51 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestHiCachePage(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-hierarchical-cache", + "--page-size", + 32, + "--hicache-write-policy", + "write_back", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/hicache/test_hicache_storage.py b/test/srt/hicache/test_hicache_storage.py new file mode 100644 index 000000000..7bc947b8c --- /dev/null +++ b/test/srt/hicache/test_hicache_storage.py @@ -0,0 +1,57 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +_is_hip = is_hip() + + +class TestHiCache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-hierarchical-cache", + "--mem-fraction-static", + 0.7, + "--hicache-size", + 100 if not _is_hip else 200, + "--page-size", + "64", + "--hicache-storage-backend", + "file", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/hicache/test_hicache_storage_3fs_backend.py b/test/srt/hicache/test_hicache_storage_3fs_backend.py new file mode 100644 index 000000000..d0f519075 --- /dev/null +++ b/test/srt/hicache/test_hicache_storage_3fs_backend.py @@ -0,0 +1,135 @@ +""" +Benchmark tests for HiCache Storage with 3FS backend. +Usage: + python3 -m pytest test/srt/hicache/test_hicache_storage_3fs_backend.py -v +""" + +import json +import os +import time +import unittest +from types import SimpleNamespace + +from test_hicache_storage_file_backend import HiCacheStorageBaseMixin + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import CustomTestCase + + +class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin): + """Base mixin class with common setup and utilities""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + # Create a temporary JSON config file for HF3FS + hf3fs_config = { + "file_path_prefix": os.path.join(cls.temp_dir, "hicache"), + "file_size": 1024 * 1024 * 1024 * 2, + "numjobs": 2, + "entries": 8, + "use_mock_hf3fs_client": True, + } + + # Write config to temporary file + config_file = os.path.join(cls.temp_dir, "hf3fs_config.json") + with open(config_file, "w") as f: + json.dump(hf3fs_config, f, indent=2) + + server_args = { + "--tp-size": 1, + "--hicache-ratio": 1.2, + "--hicache-storage-backend": "hf3fs", + "--hicache-storage-backend-extra-config": json.dumps(hf3fs_config), + } + + # Set the environment variable to point to our config file + env_vars = { + "SGLANG_HICACHE_HF3FS_CONFIG_PATH": config_file, + } + + return server_args, env_vars + + +class TestHf3fsBackendLayerFirstLayout( + HiCacheStorage3FSBackendBaseMixin, CustomTestCase +): + """Layer first layout tests for HiCache-Hf3fs backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "layer_first" + server_args["--hicache-io-backend"] = "direct" + return server_args, env_vars + + +class TestHf3fsBackendPageFirstLayout( + HiCacheStorage3FSBackendBaseMixin, CustomTestCase +): + """Page first layout tests for HiCache-Hf3fs backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-mem-layout"] = "page_first" + return server_args, env_vars + + +class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase): + """Accuracy tests for HiCache-Hf3fs backend""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args, env_vars = super()._get_additional_server_args_and_env() + server_args["--hicache-ratio"] = 1.5 + server_args["--tp-size"] = 2 + return server_args, env_vars + + def test_eval_accuracy(self): + """Test eval accuracy with cache persistence across cache flushes""" + print("\n=== Testing Eval Accuracy with Cache Persistence ===") + + # First evaluation - populate cache + print("Phase 1: Running initial GSM8K evaluation to populate cache...") + args_initial = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=50, + max_new_tokens=512, + parallel=10, + host=f"http://{self.base_host}", + port=int(self.base_port), + ) + metrics_initial = run_eval_few_shot_gsm8k(args_initial) + + # Flush cache to force remote storage access + print("Phase 2: Flushing device cache...") + self.assertTrue(self.flush_cache(), "Cache flush should succeed") + time.sleep(2) + + # Second evaluation - should use remote cache + print("Phase 3: Running second GSM8K evaluation using remote cache...") + metrics_cached = run_eval_few_shot_gsm8k(args_initial) + + # Verify accuracy consistency + accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"]) + print(f"Accuracy difference: {accuracy_diff:.4f}") + + # Assertions + self.assertGreater( + metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable" + ) + self.assertGreater( + metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable" + ) + self.assertLess( + accuracy_diff, 0.05, "Accuracy should be consistent between cache states" + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/hicache/test_hicache_storage_file_backend.py b/test/srt/hicache/test_hicache_storage_file_backend.py new file mode 100644 index 000000000..fc8a0e25d --- /dev/null +++ b/test/srt/hicache/test_hicache_storage_file_backend.py @@ -0,0 +1,334 @@ +""" +E2E tests for HiCache Storage functionality. +Usage: + python3 -m pytest test/srt/hicache/test_hicache_storage_e2e.py -v +""" + +import os +import random +import tempfile +import time +import unittest +from types import SimpleNamespace +from typing import Dict +from urllib.parse import urlparse + +import requests + +from sglang.bench_serving import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class HiCacheStorageBaseMixin: + """Base mixin class with common setup and utilities""" + + @classmethod + def setUpClass(cls): + """Set up test environment and launch server once for all tests""" + cls.temp_dir = tempfile.mkdtemp() + cls.model = cls._get_model_name() + cls.base_url = DEFAULT_URL_FOR_TEST + + parsed_url = urlparse(cls.base_url) + cls.base_host = parsed_url.hostname + cls.base_port = str(parsed_url.port) + + # Prepare tokenizer for prompt generation + cls.tokenizer = get_tokenizer(cls.model) + + # Launch server with HiCache enabled and cache report + cls.process = cls._launch_server_with_hicache() + cls._wait_for_server_ready() + + print(f"Test server launched successfully at {cls.base_url}") + print(f"Cache directory: {cls.temp_dir}") + + @classmethod + def tearDownClass(cls): + """Clean up test environment""" + kill_process_tree(cls.process.pid) + + import shutil + + shutil.rmtree(cls.temp_dir, ignore_errors=True) + + @classmethod + def _get_model_name(cls): + """Get model name for the test configuration - override in subclasses""" + return DEFAULT_MODEL_NAME_FOR_TEST + + @classmethod + def _get_base_server_args(cls): + """Get base server arguments - can be extended in subclasses""" + return { + "--enable-hierarchical-cache": True, + "--mem-fraction-static": 0.6, + "--hicache-ratio": 1.2, + "--page-size": 64, + "--enable-cache-report": True, + "--hicache-storage-prefetch-policy": "wait_complete", + "--hicache-storage-backend": "file", + } + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + return {}, {"SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR": cls.temp_dir} + + @classmethod + def _launch_server_with_hicache(cls): + """Launch server with HiCache enabled""" + + additional_server_args, env_vars = cls._get_additional_server_args_and_env() + server_args = cls._get_base_server_args() + if additional_server_args: + server_args.update(additional_server_args) + + final_server_args = [] + for k, v in server_args.items(): + if isinstance(v, bool): + final_server_args.append(str(k)) + else: + final_server_args.append(str(k)) + final_server_args.append(str(v)) + + print(f"final_server_args: {final_server_args}") + + env_vars = { + **os.environ, + **env_vars, + } + + return popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=final_server_args, + env=env_vars, + ) + + @classmethod + def _wait_for_server_ready(cls, timeout: int = 60) -> bool: + """Wait for server to be ready""" + start_time = time.time() + while time.time() - start_time < timeout: + try: + response = requests.get(f"{cls.base_url}/health", timeout=5) + if response.status_code == 200: + return True + except requests.RequestException: + pass + time.sleep(2) + raise TimeoutError("Server failed to start within timeout") + + def send_request( + self, prompt: str, max_tokens: int = 100, temperature: float = 0.0 + ) -> Dict: + """Send a generate request and return response""" + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_tokens, + "ignore_eos": True, + }, + }, + timeout=60, + ) + + self.assertEqual( + response.status_code, + 200, + f"Request failed: {response.status_code} - {response.text}", + ) + return response.json() + + def get_cached_tokens(self, response_json: Dict) -> int: + """Extract cached tokens count from /generate response""" + meta = response_json.get("meta_info", {}) + return int(meta.get("cached_tokens", 0)) + + def flush_cache(self) -> bool: + """Flush device cache to force remote storage access""" + try: + response = requests.post(f"{self.base_url}/flush_cache", timeout=10) + return response.status_code == 200 + except requests.RequestException: + return False + + def gen_prompt(self, token_num: int) -> str: + """Generate a random prompt of specified token length using tokenizer vocabulary.""" + all_available_tokens = list(self.tokenizer.get_vocab().values()) + selected_tokens = random.choices(all_available_tokens, k=token_num) + return self.tokenizer.decode(selected_tokens) + + def trigger_offloading_and_flush(self): + """Helper method to trigger offloading and flush cache""" + # Trigger offloading + self.send_request(self.gen_prompt(1), max_tokens=150) + + # Flush device cache to force remote storage access + time.sleep(2) + self.assertTrue(self.flush_cache(), "Cache flush should succeed") + + def test_basic_backup_and_prefetch(self): + """Test storage and retrieval of large context through remote cache""" + print("\n=== Testing Large Context Cache Storage & Retrieval ===") + + # Generate substantial context that will be cached + base_prompt = self.gen_prompt(768) + + # First request - populate cache + print("Step 1: Populating cache with large context...") + response1 = self.send_request(base_prompt, max_tokens=150) + self.assertIsNotNone(response1) + + # Flush device cache to force remote storage access + self.trigger_offloading_and_flush() + + # Second request with extended prompt - should hit remote cache + print("Step 2: Testing cache hit from remote storage...") + + start_time = time.time() + response2 = self.send_request(base_prompt, max_tokens=150) + retrieval_time = time.time() - start_time + + cached_tokens = self.get_cached_tokens(response2) + print( + f"Remote cache retrieval time: {retrieval_time:.3f}s, cached_tokens={cached_tokens}" + ) + + # Assert cached tokens indicate a remote hit + self.assertGreater( + cached_tokens, 700, "Expected significant cached tokens for remote hit" + ) + + +class TestHiCacheStorageTP(HiCacheStorageBaseMixin, CustomTestCase): + """Multi-TP tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--tp-size": 2} + return server_args, {} + + +class TestHiCacheStorageLayerFirstDirectIO(HiCacheStorageBaseMixin, CustomTestCase): + """Layer first direct tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = { + "--hicache-mem-layout": "layer_first", + "--hicache-io-backend": "direct", + } + return server_args, {} + + +class TestHiCacheStoragePageFirstLayout(HiCacheStorageBaseMixin, CustomTestCase): + """Page first layout tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--hicache-mem-layout": "page_first"} + return server_args, {} + + +class TestHiCacheStorageMLA(HiCacheStorageBaseMixin, CustomTestCase): + """MLA Model tests for HiCache Storage functionality""" + + @classmethod + def _get_model_name(cls): + """Use MLA model for testing""" + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--tp-size": 2} + return server_args, {} + + +class TestHiCacheStorageAccuracy(HiCacheStorageBaseMixin, CustomTestCase): + """Accuracy tests for HiCache Storage functionality""" + + @classmethod + def _get_additional_server_args_and_env(cls): + """Get additional server arguments specific to configuration - override in subclasses""" + server_args = {"--tp-size": 2, "--hicache-ratio": 1.5} + return server_args, {} + + def test_eval_accuracy(self): + """Test eval accuracy with cache persistence across cache flushes""" + print("\n=== Testing Eval Accuracy with Cache Persistence ===") + + # First evaluation - populate cache + print("Phase 1: Running initial GSM8K evaluation to populate cache...") + args_initial = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=50, + max_new_tokens=512, + parallel=10, + host=f"http://{self.base_host}", + port=int(self.base_port), + ) + metrics_initial = run_eval_few_shot_gsm8k(args_initial) + + # Flush cache to force remote storage access + print("Phase 2: Flushing device cache...") + self.assertTrue(self.flush_cache(), "Cache flush should succeed") + time.sleep(2) + + # Second evaluation - should use remote cache + print("Phase 3: Running second GSM8K evaluation using remote cache...") + metrics_cached = run_eval_few_shot_gsm8k(args_initial) + + # Verify accuracy consistency + accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"]) + print(f"Accuracy difference: {accuracy_diff:.4f}") + + # Assertions + self.assertGreater( + metrics_initial["accuracy"], 0.6, "Initial accuracy should be reasonable" + ) + self.assertGreater( + metrics_cached["accuracy"], 0.6, "Cached accuracy should be reasonable" + ) + self.assertLess( + accuracy_diff, 0.05, "Accuracy should be consistent between cache states" + ) + + +# TODO: Add other backends tests（3fs/mooncake） +# class TestHiCacheStorageMooncakeBackend(HiCacheStorageBaseTest): +# """Mooncake backend tests for HiCache Storage functionality""" + +# @classmethod +# def _get_additional_server_args_and_env(cls): +# """Get additional server arguments specific to configuration - override in subclasses""" +# server_args = ["--hicache-storage-backend", "mooncake"] +# env = { +# "MOONCAKE_TE_META_DATA_SERVER": "http://127.0.0.1:8080/metadata", +# "MOONCAKE_MASTER": "127.0.0.1:50051" +# xxxxx +# } +# return server_args, {} + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/kv_cache_scales_llama3_1_8b.json b/test/srt/kv_cache_scales_llama3_1_8b.json new file mode 100644 index 000000000..3e890e50e --- /dev/null +++ b/test/srt/kv_cache_scales_llama3_1_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 1, + "1": 1, + "2": 1, + "3": 1, + "4": 1, + "5": 1, + "6": 1, + "7": 1, + "8": 1, + "9": 1, + "10": 1, + "11": 1, + "12": 1, + "13": 1, + "14": 1, + "15": 1, + "16": 1, + "17": 1, + "18": 1, + "19": 1, + "20": 1, + "21": 1, + "22": 1, + "23": 1, + "24": 1, + "25": 1, + "26": 1, + "27": 1, + "28": 1, + "29": 1, + "30": 1, + "31": 1 + } + } + } +} diff --git a/test/srt/kv_cache_scales_llama3_8b.json b/test/srt/kv_cache_scales_llama3_8b.json new file mode 100644 index 000000000..466b0d01a --- /dev/null +++ b/test/srt/kv_cache_scales_llama3_8b.json @@ -0,0 +1,42 @@ +{ + "model_type": "llama", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.0408, + "1": 0.0503, + "2": 0.0667, + "3": 0.0909, + "4": 0.1135, + "5": 0.127, + "6": 0.1768, + "7": 0.1488, + "8": 0.1135, + "9": 0.1203, + "10": 0.1013, + "11": 0.0842, + "12": 0.1231, + "13": 0.1096, + "14": 0.1221, + "15": 0.1013, + "16": 0.1067, + "17": 0.0952, + "18": 0.0899, + "19": 0.097, + "20": 0.087, + "21": 0.0994, + "22": 0.0904, + "23": 0.1013, + "24": 0.1019, + "25": 0.1053, + "26": 0.1, + "27": 0.0894, + "28": 0.1013, + "29": 0.1488, + "30": 0.0766, + "31": 0.0821 + } + } + } +} diff --git a/test/srt/kv_cache_scales_qwen2_1_5b.json b/test/srt/kv_cache_scales_qwen2_1_5b.json new file mode 100644 index 000000000..984747509 --- /dev/null +++ b/test/srt/kv_cache_scales_qwen2_1_5b.json @@ -0,0 +1,38 @@ +{ + "model_type": "qwen", + "kv_cache": { + "dtype": "float8_e4m3fn", + "scaling_factor": { + "0": { + "0": 0.9846, + "1": 0.0645, + "2": 0.0731, + "3": 0.0800, + "4": 0.0748, + "5": 0.0780, + "6": 0.0702, + "7": 0.0894, + "8": 0.0410, + "9": 0.0758, + "10": 0.0556, + "11": 0.0731, + "12": 0.0899, + "13": 0.0780, + "14": 0.1441, + "15": 0.0914, + "16": 0.5614, + "17": 0.1067, + "18": 0.0537, + "19": 0.0658, + "20": 0.0523, + "21": 0.0533, + "22": 0.0699, + "23": 0.0635, + "24": 0.0588, + "25": 0.0884, + "26": 0.0947, + "27": 0.1032 + } + } + } +} diff --git a/test/srt/lora/test_lora.py b/test/srt/lora/test_lora.py new file mode 100644 index 000000000..ab1c630fc --- /dev/null +++ b/test/srt/lora/test_lora.py @@ -0,0 +1,172 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import random +import unittest +from typing import List + +import torch +from utils import ( + ALL_OTHER_MULTI_LORA_MODELS, + CI_MULTI_LORA_MODELS, + TORCH_DTYPES, + LoRAModelCase, + ensure_reproducibility, +) + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci + +TEST_MULTIPLE_BATCH_PROMPTS = [ + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, + """ + ### Instruction: + Write a poem about the transformers Python library. + Mention the word "large language models" in that poem. + ### Response: + The Transformers are large language models, + They're used to make predictions on text. + """, + "AI is a field of computer science focused on", + "Computer science is the study of", + "Write a short story.", + "What are the main components of a computer?", +] + + +class TestLoRA(CustomTestCase): + def _create_test_samples( + self, lora_adapter_paths: List[str], repeated_trials: int = 3 + ): + random.seed(42) # Ensure reproducibility + + patterns = [ + [None, lora_adapter_paths[0], lora_adapter_paths[1]], + [lora_adapter_paths[0], None, lora_adapter_paths[1]], + [lora_adapter_paths[0], lora_adapter_paths[1], None], + [None, lora_adapter_paths[1], None], + [None, None, None], + ] + + batches = [ + [random.choice(pattern) for _ in range(3)] + for pattern in patterns + for _ in range(repeated_trials) + ] + + return batches + + def _run_lora_multiple_batch_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + for torch_dtype in TORCH_DTYPES: + max_new_tokens = 32 + backend = "triton" + base_path = model_case.base + lora_adapter_paths = [a.name for a in model_case.adaptors] + assert len(lora_adapter_paths) >= 2 + + print( + f"\n========== Testing multiple batches on base '{base_path}' with backend={backend}, dtype={torch_dtype} ---" + ) + + # Initialize runners + srt_runner = SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=[lora_adapter_paths[0], lora_adapter_paths[1]], + max_loras_per_batch=len(lora_adapter_paths) + 1, + lora_backend=backend, + sleep_on_idle=True, # Eliminate non-determinism by forcing all requests to be processed in one batch. + attention_backend="torch_native", + ) + hf_runner = HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) + + batches = self._create_test_samples(lora_adapter_paths) + with srt_runner, hf_runner: + for i, lora_paths in enumerate(batches, start=1): + prompts = [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS) for _ in range(3) + ] + print( + f"\n--- Running Batch {i} --- prompts: {prompts}, lora_paths: {lora_paths}" + ) + + ensure_reproducibility() + srt_outputs = srt_runner.batch_forward( + prompts, + max_new_tokens=max_new_tokens, + lora_paths=lora_paths, + ) + + ensure_reproducibility() + hf_outputs = hf_runner.forward( + prompts, + max_new_tokens=max_new_tokens, + lora_paths=lora_paths, + ) + + print("SRT outputs:", [s for s in srt_outputs.output_strs]) + print("HF outputs:", [s for s in hf_outputs.output_strs]) + + for srt_out, hf_out in zip( + srt_outputs.output_strs, hf_outputs.output_strs + ): + srt_str = srt_out.strip() + hf_str = hf_out.strip() + rouge_tol = model_case.rouge_l_tolerance + rouge_score = calculate_rouge_l([srt_str], [hf_str])[0] + if rouge_score < rouge_tol: + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{lora_paths}', backend '{backend}', prompt: '{prompts}...'" + ) + + print(f"--- Batch {i} Comparison Passed --- ") + + def test_ci_lora_models(self): + self._run_lora_multiple_batch_on_model_cases(CI_MULTI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + filtered_models = [] + for model_case in ALL_OTHER_MULTI_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_lora_multiple_batch_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_backend.py b/test/srt/lora/test_lora_backend.py new file mode 100644 index 000000000..2217e8dd6 --- /dev/null +++ b/test/srt/lora/test_lora_backend.py @@ -0,0 +1,76 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +from utils import ( + ALL_OTHER_LORA_MODELS, + BACKENDS, + CI_LORA_MODELS, + DEFAULT_PROMPTS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +class TestLoRABackend(CustomTestCase): + + def _run_backend_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + DEFAULT_PROMPTS + if not model_case.skip_long_prompt + else [p for p in DEFAULT_PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + for backend in BACKENDS: + run_lora_test_one_by_one( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend=backend, + ) + + def test_ci_lora_models(self): + self._run_backend_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_backend_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_cuda_graph.py b/test/srt/lora/test_lora_cuda_graph.py new file mode 100644 index 000000000..ba68df59a --- /dev/null +++ b/test/srt/lora/test_lora_cuda_graph.py @@ -0,0 +1,110 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +from utils import ( + ALL_OTHER_LORA_MODELS, + CI_LORA_MODELS, + DEFAULT_PROMPTS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_by_batch, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + +TEST_CUDA_GRAPH_PADDING_PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, + "Computer science is the study of", +] + + +class TestLoRACudaGraph(CustomTestCase): + + def _run_without_cuda_graph_on_model_cases(self, model_cases: List[LoRAModelCase]): + # Since we have already enabled CUDA graph by default in other lora tests, + # we only need to run lora tests without CUDA graph here. + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + DEFAULT_PROMPTS + if not model_case.skip_long_prompt + else [p for p in DEFAULT_PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + run_lora_test_one_by_one( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend="triton", + disable_cuda_graph=True, + test_tag="without_cuda_graph", + ) + + def _run_cuda_graph_padding_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # Run a batch size of 3, which will not be captured by CUDA graph and need padding + prompts = TEST_CUDA_GRAPH_PADDING_PROMPTS + for torch_dtype in TORCH_DTYPES: + run_lora_test_by_batch( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend="triton", + disable_cuda_graph=False, + test_tag="cuda_graph_padding", + ) + + def test_ci_lora_models(self): + self._run_without_cuda_graph_on_model_cases(CI_LORA_MODELS) + self._run_cuda_graph_padding_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_without_cuda_graph_on_model_cases(filtered_models) + self._run_cuda_graph_padding_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_eviction.py b/test/srt/lora/test_lora_eviction.py new file mode 100644 index 000000000..d27b11906 --- /dev/null +++ b/test/srt/lora/test_lora_eviction.py @@ -0,0 +1,146 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import contextlib +import multiprocessing as mp +import unittest +from typing import Dict, List, Tuple + +import torch + +from sglang.test.runners import SRTRunner +from sglang.test.test_utils import CustomTestCase + +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Compose a SQL query that uses the following table: users, and returns the user_id and name of all users whose name that does not have a duplicate in the table. + ### Response: + SELECT user_id, name FROM users WHERE name LIKE 'A%'; + """, +] + +ADAPTERS = [ + "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", # target_modules = q, v + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # target_modules = q, k, v, o, gate, up, down +] + +BASE_MODEL = "meta-llama/Meta-Llama-3.1-8B-Instruct" + + +@contextlib.contextmanager +def dynamically_loaded_adapter(runner, lora_path: str, lora_name: str): + """A context manager to load and automatically unload a LoRA adapter.""" + try: + runner.load_lora_adapter(lora_name=lora_name, lora_path=lora_path) + yield + finally: + runner.unload_lora_adapter(lora_name=lora_name) + + +class TestLoRAEviction(CustomTestCase): + def test_lora_eviction_with_different_target_modules(self): + """ + Test LoRA eviction with different target modules. + + This test runs inference against two LoRA adapters in different orders to force eviction behavior, and ensures + that the outputs of the same (adapter, prompt) pair are consistent across runs. + """ + output_history = {} + self._run_test(ADAPTERS, output_history, reverse=False) + self._run_test(ADAPTERS, output_history, reverse=True) + + def test_lora_eviction_with_reused_lora_name(self): + """ + Test LoRA eviction with reused LoRA names. + + This test runs inference against two LoRA adapters with the same name to ensure that the eviction behavior + works correctly when reusing LoRA names. + """ + output_history = {} + self._run_test(ADAPTERS, output_history, reuse_lora_name=True, repeat=1) + self._run_test(ADAPTERS, output_history, reuse_lora_name=False, repeat=1) + + def _run_test( + self, + lora_paths: List[str], + output_history: Dict[Tuple[str, str], str], + reverse: bool = False, + repeat: int = 2, + reuse_lora_name: bool = False, + ): + REUSED_LORA_NAME = "lora" + max_new_tokens = 256 + backend = "triton" + torch_dtype = torch.float16 + base_path = BASE_MODEL + assert len(lora_paths) >= 2 + + initial_lora_paths = lora_paths if not reuse_lora_name else None + # Initialize runners + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=initial_lora_paths, + max_loras_per_batch=1, + lora_backend=backend, + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + ) as srt_runner: + adapter_sequence = lora_paths if not reverse else lora_paths[::-1] + + for i in range(repeat): + for j, lora_path in enumerate(adapter_sequence): + print( + f"\n========== Testing LoRA eviction with adapter '{lora_path}' (#{j + 1}/{len(adapter_sequence)}), reuse_lora_name: {reuse_lora_name}, reversed: {reverse}, repeat: {i + 1}/{repeat} ---" + ) + + lora_name = REUSED_LORA_NAME if reuse_lora_name else lora_path + context = ( + dynamically_loaded_adapter(srt_runner, lora_path, lora_name) + if reuse_lora_name + else contextlib.nullcontext() + ) + with context: + for prompt in PROMPTS: + print("\nprompt:\n", prompt) + srt_outputs = srt_runner.forward( + [prompt], + max_new_tokens=max_new_tokens, + lora_paths=[lora_name], + ) + output = srt_outputs.output_strs[0].strip() + print("\noutput:\n", output) + + prev_output = output_history.get((lora_path, prompt)) + if prev_output is not None: + self.assertEqual( + prev_output, + output, + f"Output mismatch for adapter {lora_path} and prompt '{prompt}' on repeat {j + 1}, previous: '{prev_output}', current: '{output}'.", + ) + else: + output_history[(lora_path, prompt)] = output + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_llama4.py b/test/srt/lora/test_lora_llama4.py new file mode 100644 index 000000000..65a4b766f --- /dev/null +++ b/test/srt/lora/test_lora_llama4.py @@ -0,0 +1,61 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +MODELS = [ + SimpleNamespace( + model="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + tp_size=8, + ), +] + + +class TestLlama4LoRA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_bringup(self): + for model in MODELS: + try: + process = popen_launch_server( + model.model, + self.base_url, + timeout=3 * DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-lora", + "--max-lora-rank", + "64", + "--lora-target-modules", + "all", + "--tp-size", + str(model.tp_size), + "--context-length", + "1048576", + "--attention-backend", + "fa3", + ], + ) + except Exception as e: + print(f"Error testing {model.model}: {e}") + self.fail(f"Test failed for {model.model}: {e}") + + finally: + # Ensure process cleanup happens regardless of success/failure + if process is not None and process.poll() is None: + print(f"Cleaning up process {process.pid}") + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process: {e}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/lora/test_lora_qwen3.py b/test/srt/lora/test_lora_qwen3.py new file mode 100644 index 000000000..f77156707 --- /dev/null +++ b/test/srt/lora/test_lora_qwen3.py @@ -0,0 +1,212 @@ +# Copyright 2023-2025 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import random +import unittest +from typing import List + +from utils import TORCH_DTYPES, LoRAAdaptor, LoRAModelCase, ensure_reproducibility + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, calculate_rouge_l, is_in_ci + +LORA_MODELS_QWEN3 = [ + LoRAModelCase( + base="Qwen/Qwen3-4B", + adaptors=[ + LoRAAdaptor( + name="nissenj/Qwen3-4B-lora-v2", + prefill_tolerance=3e-1, + ), + LoRAAdaptor( + name="y9760210/Qwen3-4B-lora_model", + prefill_tolerance=3e-1, + ), + ], + max_loras_per_batch=2, + ), +] + + +TEST_MULTIPLE_BATCH_PROMPTS = [ + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, + """ + ### Instruction: + Write a poem about the transformers Python library. + Mention the word "large language models" in that poem. + ### Response: + The Transformers are large language models, + They're used to make predictions on text. + """, + "AI is a field of computer science focused on", + "Computer science is the study of", + "Write a short story.", + "What are the main components of a computer?", +] + + +class TestLoRAQwen3(CustomTestCase): + def _run_lora_multiple_batch_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + for torch_dtype in TORCH_DTYPES: + max_new_tokens = 32 + backend = "triton" + base_path = model_case.base + lora_adapter_paths = [a.name for a in model_case.adaptors] + assert len(lora_adapter_paths) >= 2 + + batches = [ + ( + [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + ], + [ + None, + lora_adapter_paths[0], + lora_adapter_paths[1], + ], + ), + ( + [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + ], + [ + lora_adapter_paths[0], + None, + lora_adapter_paths[1], + ], + ), + ( + [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + ], + [lora_adapter_paths[0], lora_adapter_paths[1], None], + ), + ( + [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + ], + [None, lora_adapter_paths[1], None], + ), + ( + [ + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + random.choice(TEST_MULTIPLE_BATCH_PROMPTS), + ], + [None, None, None], + ), + ] + + print( + f"\n========== Testing multiple batches on base '{base_path}' with backend={backend}, dtype={torch_dtype} ---" + ) + + # Initialize runners + ensure_reproducibility() + srt_runner = SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + lora_paths=[lora_adapter_paths[0], lora_adapter_paths[1]], + max_loras_per_batch=len(lora_adapter_paths) + 1, + lora_backend=backend, + sleep_on_idle=True, # Eliminate non-determinism by forcing all requests to be processed in one batch. + attention_backend="torch_native", + ) + + ensure_reproducibility() + hf_runner = HFRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + patch_model_do_sample_false=True, + ) + + with srt_runner, hf_runner: + for i, (prompts, lora_paths) in enumerate(batches): + print( + f"\n--- Running Batch {i+1} --- prompts: {prompts}, lora_paths: {lora_paths}" + ) + + srt_outputs = srt_runner.batch_forward( + prompts, + max_new_tokens=max_new_tokens, + lora_paths=lora_paths, + ) + + hf_outputs = hf_runner.forward( + prompts, + max_new_tokens=max_new_tokens, + lora_paths=lora_paths, + ) + + print("SRT outputs:", [s for s in srt_outputs.output_strs]) + print("HF outputs:", [s for s in hf_outputs.output_strs]) + + for srt_out, hf_out in zip( + srt_outputs.output_strs, hf_outputs.output_strs + ): + srt_str = srt_out.strip() + hf_str = hf_out.strip() + rouge_tol = model_case.rouge_l_tolerance + rouge_score = calculate_rouge_l([srt_str], [hf_str])[0] + if rouge_score < rouge_tol: + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{lora_paths}', backend '{backend}', prompt: '{prompts}...'" + ) + + print(f"--- Batch {i+1} Comparison Passed --- ") + + def test_ci_lora_models(self): + self._run_lora_multiple_batch_on_model_cases(LORA_MODELS_QWEN3) + + def test_all_lora_models(self): + if is_in_ci(): + return + qwen_filtered_models = [] + for model_case in LORA_MODELS_QWEN3: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + qwen_filtered_models.append(model_case) + + self._run_lora_multiple_batch_on_model_cases(qwen_filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_radix_cache.py b/test/srt/lora/test_lora_radix_cache.py new file mode 100644 index 000000000..d3ecb219c --- /dev/null +++ b/test/srt/lora/test_lora_radix_cache.py @@ -0,0 +1,83 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import random +import unittest + +import torch +from utils import CI_MULTI_LORA_MODELS, DEFAULT_PROMPTS, run_lora_test_one_by_one + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase + +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids. + ### Question: + What do you know about llamas? + ### Answer: + """, +] + + +class TestLoRARadixCache(CustomTestCase): + + def test_lora_radix_cache(self): + # Here we need a model case with multiple adaptors for testing correctness of radix cache + model_case = CI_MULTI_LORA_MODELS[0] + + torch_dtype = torch.float16 + max_new_tokens = 32 + backend = "triton" + batch_prompts = ( + PROMPTS + if not model_case.skip_long_prompt + else [p for p in PROMPTS if len(p) < 1000] + ) + + # Test lora with radix cache + run_lora_test_one_by_one( + batch_prompts, + model_case, + torch_dtype, + max_new_tokens=max_new_tokens, + backend=backend, + disable_radix_cache=False, + test_tag="lora-with-radix-cache", + ) + + # Test lora without radix cache + run_lora_test_one_by_one( + batch_prompts, + model_case, + torch_dtype, + max_new_tokens=max_new_tokens, + backend=backend, + disable_radix_cache=True, + test_tag="lora-without-radix-cache", + ) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_tp.py b/test/srt/lora/test_lora_tp.py new file mode 100644 index 000000000..51a552d78 --- /dev/null +++ b/test/srt/lora/test_lora_tp.py @@ -0,0 +1,78 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +from utils import ( + ALL_OTHER_LORA_MODELS, + CI_LORA_MODELS, + DEFAULT_PROMPTS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +class TestLoRATP(CustomTestCase): + + def _run_tp_on_model_cases(self, model_cases: List[LoRAModelCase]): + tp_list = [2] # Define TP sizes to iterate over + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters + prompts = ( + DEFAULT_PROMPTS + if not model_case.skip_long_prompt + else [p for p in DEFAULT_PROMPTS if len(p) < 1000] + ) + for tp_size in tp_list: + model_case.tp_size = tp_size + for torch_dtype in TORCH_DTYPES: + run_lora_test_one_by_one( + prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend="triton", + test_tag=f"tp={tp_size}", + ) + + def test_ci_lora_models(self): + self._run_tp_on_model_cases(CI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_tp_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_lora_update.py b/test/srt/lora/test_lora_update.py new file mode 100644 index 000000000..f2058f466 --- /dev/null +++ b/test/srt/lora/test_lora_update.py @@ -0,0 +1,1306 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import json +import multiprocessing as mp +import unittest +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterable, List, Optional, Union + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.runners import SRTRunner +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +PROMPTS = [ + "SGL is a", + "AI is a field of computer science focused on", + "Computer science is the study of", + "Write a short story.", + "What are the main components of a computer?", +] + +MEM_FRACTION_STATIC = 0.8 + + +class OperationType(Enum): + LOAD = "load" + UNLOAD = "unload" + FORWARD = "forward" + + +@dataclass +class Operation: + # Operation type, can be LOAD, UNLOAD, FORWARD + type: OperationType + # Data associated with the operation. Exact type varies depending on the operation + data: Optional[Any] + # If the operation is expected to fail, this is the error message to expect + expected_error: Optional[str] = None + + +@dataclass +class TestCase: + description: str + base: str + max_loras_per_batch: int + all_adapters: List[str] + op_sequence: List[Operation] + initial_adapters: Optional[List[str]] = None + enable_lora: Optional[bool] = None + max_lora_rank: Optional[int] = None + lora_target_modules: Optional[List] = None + max_new_tokens: int = 32 + max_loaded_loras: Optional[int] = None + + +def create_batch_data(adapters: Union[str, list]) -> List[tuple[str, str]]: + if not isinstance(adapters, list): + adapters = [adapters] + return [(prompt, adapter) for prompt in PROMPTS for adapter in adapters] + + +BASIC_TESTS = [ + TestCase( + description="dynamic lora update with initial lora_paths", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + initial_adapters=[ + # Testing 3 supported lora-path formats. + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + { + "lora_name": "pbevan11/llama-3.1-8b-ocr-correction", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": False, + }, + ], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + None, + ), + ), + ], + ), + TestCase( + description="dynamic lora update without initial lora_paths", + base="meta-llama/Llama-3.1-8B-Instruct", + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + max_loras_per_batch=4, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + None, + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.UNLOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data(None), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + ], + ), +] +TARGET_MODULE_TESTS = [ + TestCase( + description="Test explicitly specified lora-target-modules.", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + lora_target_modules=[ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + ], + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="Test inferred lora-target-modules - start with larger adapter", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="Test inferred lora-target-modules - start with smaller adapter", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=64, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # target_modules = q, k, v, o, gate, up, down + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", # target_modules = q, k, v, o, gate + ], + initial_adapters=["algoprog/fact-generation-llama-3.1-8b-instruct-lora"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "algoprog/fact-generation-llama-3.1-8b-instruct-lora" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "algoprog/fact-generation-llama-3.1-8b-instruct-lora", + None, + ] + ), + ), + ], + ), +] +MAX_LORA_RANK_TESTS = [ + TestCase( + description="Test explicitly specified max-lora-rank.", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + max_lora_rank=32, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 + ], + initial_adapters=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16" + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ), + expected_error="not loaded", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + None, + ] + ), + ), + ], + ), + TestCase( + description="test implicitly inferred max-lora-rank", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=3, + all_adapters=[ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", # r = 4 + "pbevan11/llama-3.1-8b-ocr-correction", # r = 32 + "philschmid/code-llama-3-1-8b-text-to-sql-lora", # r = 256 + ], + initial_adapters=["pbevan11/llama-3.1-8b-ocr-correction"], + op_sequence=[ + Operation( + type=OperationType.FORWARD, + data=create_batch_data("pbevan11/llama-3.1-8b-ocr-correction"), + ), + Operation( + type=OperationType.LOAD, + data="philschmid/code-llama-3-1-8b-text-to-sql-lora", + expected_error="incompatible", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data("philschmid/code-llama-3-1-8b-text-to-sql-lora"), + expected_error="not loaded", + ), + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + None, + ] + ), + ), + ], + ), +] +MAX_LOADED_LORAS_TESTS = [ + TestCase( + description="Test max_loaded_loras limit", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=2, + max_loaded_loras=2, + all_adapters=[ + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pbevan11/llama-3.1-8b-ocr-correction", + ], + initial_adapters=["philschmid/code-llama-3-1-8b-text-to-sql-lora"], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + expected_error="Maximum number of loaded LoRA adapters", + ), + Operation( + type=OperationType.UNLOAD, + data="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + ), + Operation( + type=OperationType.LOAD, + data="pbevan11/llama-3.1-8b-ocr-correction", + ), + ], + ), +] +EVICTION_TESTS = [ + TestCase( + description="dynamic lora update with evictions", + base="meta-llama/Llama-3.1-8B-Instruct", + max_loras_per_batch=2, + all_adapters=[ + "lora1=philschmid/code-llama-3-1-8b-text-to-sql-lora", + "lora2=Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "lora3=pbevan11/llama-3.1-8b-ocr-correction", + ], + enable_lora=True, + max_lora_rank=256, + lora_target_modules=["all"], + op_sequence=[ + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": True, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora2", + "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pinned": True, + }, + expected_error="starvation", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora2", + "lora_path": "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "pinned": False, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora3", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": False, + }, + ), + Operation( + type=OperationType.UNLOAD, + data="lora1", + ), + Operation( + type=OperationType.UNLOAD, + data="lora3", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora3", + "lora_path": "pbevan11/llama-3.1-8b-ocr-correction", + "pinned": True, + }, + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": True, + }, + expected_error="starvation", + ), + Operation( + type=OperationType.LOAD, + data={ + "lora_name": "lora1", + "lora_path": "philschmid/code-llama-3-1-8b-text-to-sql-lora", + "pinned": False, + }, + ), + # pinned: lora3 + # unpinned: lora1, lora2 + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora3", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + ] + ), + ), + Operation( + type=OperationType.FORWARD, + data=create_batch_data( + [ + "lora1", + "lora2", + None, + ] + ), + ), + ], + ), +] + +ALL_TESTS = ( + BASIC_TESTS + + TARGET_MODULE_TESTS + + MAX_LORA_RANK_TESTS + + MAX_LOADED_LORAS_TESTS + + EVICTION_TESTS +) + + +class LoRAUpdateTestSessionMode(Enum): + ENGINE = "engine" + SERVER = "server" + + +class LoRAUpdateTestSessionBase: + """ + Base context manager for testing LoRA adapters. + """ + + def __init__( + self, + *, + testcase: Optional[TestCase], + model_path: str, + lora_paths: List[Union[str, dict]], + max_loras_per_batch: int, + max_loaded_loras: Optional[int] = None, + max_lora_rank: Optional[int], + enable_lora: Optional[bool] = None, + lora_target_modules: Optional[List[str]] = None, + lora_backend: str = "triton", + disable_cuda_graph: bool = False, + cuda_graph_max_bs: int = 4, + ): + self.testcase = testcase + self.model_path = model_path + self.lora_paths = lora_paths + self.max_lora_rank = max_lora_rank + self.lora_target_modules = lora_target_modules + self.max_loras_per_batch = max_loras_per_batch + self.max_loaded_loras = max_loaded_loras + self.lora_backend = lora_backend + self.disable_cuda_graph = disable_cuda_graph + self.cuda_graph_max_bs = cuda_graph_max_bs + self.enable_lora = enable_lora + + self.expected_adapters = set() + if self.lora_paths: + for adapter in self.lora_paths: + if isinstance(adapter, dict): + lora_name = adapter["lora_name"] + elif "=" in adapter: + lora_name = adapter.split("=")[0] + else: + lora_name = adapter + self.expected_adapters.add(lora_name) + + self.handle = None # Will be set in __enter__ + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + # Don't suppress exceptions by default + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + ): + """ + Load a LoRA adapter by name and path. + """ + raise NotImplementedError("Subclasses must implement load_lora_adapter") + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + raise NotImplementedError("Subclasses must implement unload_lora_adapter") + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + raise NotImplementedError("Subclasses must implement forward") + + +class LoRAUpdateEngineTestSession(LoRAUpdateTestSessionBase): + """ + Context manager for testing LoRA adapters with in-process engine. + """ + + def __enter__(self): + # in-process runner + self.handle = SRTRunner( + model_path=self.model_path, + model_type="generation", + lora_paths=self.lora_paths, + max_lora_rank=self.max_lora_rank, + lora_target_modules=self.lora_target_modules, + lora_backend=self.lora_backend, + torch_dtype=torch.float16, + mem_fraction_static=MEM_FRACTION_STATIC, + max_loras_per_batch=self.max_loras_per_batch, + max_loaded_loras=self.max_loaded_loras, + disable_cuda_graph=self.disable_cuda_graph, + cuda_graph_max_bs=self.cuda_graph_max_bs, + enable_lora=self.enable_lora, + disable_radix_cache=True, + ) + self.handle.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.handle is not None: + # delegate cleanup to SRTRunner + return self.handle.__exit__(exc_type, exc_val, exc_tb) + # don't suppress exceptions + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + pinned: bool = False, + ): + """ + Load a LoRA adapter by name and path. + """ + if lora_path is None: + lora_path = lora_name + + response = self.handle.load_lora_adapter( + lora_name=lora_name, + lora_path=lora_path, + pinned=pinned, + ) + if expected_error: + self.testcase.assertFalse( + response.success, f"Expected failure for {lora_name}, but got success." + ) + self.testcase.assertIn( + expected_error, + response.error_message, + f"Expected error message to contain '{expected_error}', but got '{response.error_message}'", + ) + print(f"Received error as expected: {response.error_message}") + else: + self.expected_adapters.add(lora_name) + self.testcase.assertTrue( + response.success, + f"Failed to load LoRA adapter {lora_name}: {response.error_message}", + ) + loaded_adapters = set(response.loaded_adapters) + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + self.expected_adapters.remove(lora_name) + + response = self.handle.unload_lora_adapter( + lora_name=lora_name, + ) + self.testcase.assertTrue( + response.success, + f"Failed to unload LoRA adapter {lora_name}: {response.error_message}", + ) + loaded_adapters = set(response.loaded_adapters) + + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + try: + response = self.handle.batch_forward( + prompts=prompts, + lora_paths=lora_paths, + max_new_tokens=max_new_tokens, + ) + except ValueError as e: + if expected_error: + error_message = str(e) + self.testcase.assertIn( + expected_error, + error_message, + f"Expected error message to contain '{expected_error}', but got '{error_message}'", + ) + print(f"Received error as expected: {error_message}") + return error_message + + raise e + + self.testcase.assertEqual( + len(response.output_strs), + len(prompts), + f"Expected {len(prompts)} outputs, but got {len(response.output_strs)}", + ) + output = response.output_strs + print(f"output_strs: {output}") + + return output + + +class LoRAUpdateServerTestSession(LoRAUpdateTestSessionBase): + """ + Context manager for testing LoRA adapters with standalone server. + """ + + def __enter__(self): + other_args = [ + "--cuda-graph-max-bs", + str(self.cuda_graph_max_bs), + "--max-loras-per-batch", + str(self.max_loras_per_batch), + "--lora-backend", + self.lora_backend, + "--random-seed", + "42", + "--max-running-request", + "1", + "--mem-fraction-static", + str(MEM_FRACTION_STATIC), + "--disable-radix-cache", + ] + if self.enable_lora: + other_args.append("--enable-lora") + if self.lora_paths: + other_args.append("--lora-paths") + for lora_path in self.lora_paths: + if isinstance(lora_path, dict): + lora_path = json.dumps(lora_path) + other_args.append(lora_path) + if self.disable_cuda_graph: + other_args.append("--disable-cuda-graph") + if self.max_lora_rank is not None: + other_args.extend(["--max-lora-rank", str(self.max_lora_rank)]) + if self.lora_target_modules is not None: + other_args.extend(["--lora-target-modules"] + self.lora_target_modules) + if self.max_loaded_loras is not None: + other_args.extend(["--max-loaded-loras", str(self.max_loaded_loras)]) + + # launch external server + self.handle = popen_launch_server( + self.model_path, + DEFAULT_URL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.handle is not None: + kill_process_tree(self.handle.pid) + # don't suppress exceptions + return False + + def load_lora_adapter( + self, + lora_name: str, + lora_path: Optional[str] = None, + expected_error: Optional[str] = None, + pinned: bool = False, + ): + """ + Load a LoRA adapter by name and path. + """ + if lora_path is None: + lora_path = lora_name + + response = requests.post( + DEFAULT_URL_FOR_TEST + "/load_lora_adapter", + json={"lora_name": lora_name, "lora_path": lora_path, "pinned": pinned}, + ) + if expected_error: + self.testcase.assertEqual( + response.status_code, + 400, + f"Expected error for {lora_name}, but got success.", + ) + self.testcase.assertIn( + expected_error, + response.text, + f"Expected error message to contain '{expected_error}', but got '{response.text}'", + ) + print(f"Received error as expected: {response.text}") + else: + self.expected_adapters.add(lora_name) + self.testcase.assertTrue( + response.ok, f"Failed to load LoRA adapter {lora_name}: {response.text}" + ) + loaded_adapters = set(response.json()["loaded_adapters"]) + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def unload_lora_adapter(self, lora_name: str): + """ + Unload a LoRA adapter by name. + """ + self.expected_adapters.remove(lora_name) + + response = requests.post( + DEFAULT_URL_FOR_TEST + "/unload_lora_adapter", + json={"lora_name": lora_name}, + ) + self.testcase.assertTrue( + response.ok, f"Failed to unload LoRA adapter {lora_name}: {response.text}" + ) + loaded_adapters = set(response.json()["loaded_adapters"]) + + print(f"loaded_adapters: {loaded_adapters}") + self.testcase.assertEqual( + loaded_adapters, + self.expected_adapters, + f"Expected loaded adapters to be {self.expected_adapters}, but got {loaded_adapters}", + ) + + def forward( + self, + prompts: List[str], + lora_paths: List[str], + max_new_tokens: int = 32, + expected_error: Optional[str] = None, + ): + """ + Perform a batch forward pass with the current set of loaded LoRA adapters. + """ + response = requests.post( + DEFAULT_URL_FOR_TEST + "/generate", + json={ + "text": prompts, + "lora_path": lora_paths, + "sampling_params": { + "temperature": 0, + "top_k": 1, + "max_new_tokens": max_new_tokens, + }, + }, + ) + if expected_error: + self.testcase.assertEqual( + response.status_code, + 400, + f"Expected error for forward pass, but got success: {response.text}", + ) + self.testcase.assertIn( + expected_error, + response.text, + f"Expected error message to contain '{expected_error}', but got '{response.text}'", + ) + output = response.text + print(f"Received error as expected: {response.text}") + return output + else: + self.testcase.assertTrue( + response.ok, f"Failed to generate text: {response.text}" + ) + output = [r["text"] for r in response.json()] + self.testcase.assertEqual( + len(output), + len(prompts), + f"Expected {len(prompts)} outputs, but got {len(output)}", + ) + print(f"output_strs: {output}") + return output + + +# Factory function to create the appropriate LoRA test session based on mode +def LoRAUpdateTestSession( + testcase: Optional[TestCase], + mode: LoRAUpdateTestSessionMode, + **kwargs: Any, +): + if mode == LoRAUpdateTestSessionMode.ENGINE: + return LoRAUpdateEngineTestSession(testcase=testcase, **kwargs) + elif mode == LoRAUpdateTestSessionMode.SERVER: + return LoRAUpdateServerTestSession(testcase=testcase, **kwargs) + else: + raise ValueError(f"Unrecognized mode: {mode!r}") + + +class TestLoRADynamicUpdate(CustomTestCase): + """ + This test case verifies that the SRT runner can dynamically load and unload LoRA adapters + during a sequence of operations, and that the outputs of forward passes with dynamically loaded + adapters match the outputs of forward passes with statically loaded adapters. + """ + + def _repeat_each(lst, n): + return [x for x in lst for _ in range(n)] + + def _run_operation_sequence( + self, + mode: LoRAUpdateTestSessionMode, + base: str, + initial_adapters: List[Union[str, dict]], + op_sequence: List[Operation], + max_loras_per_batch: int, + max_loaded_loras: Optional[int] = None, + enable_lora: Optional[bool] = None, + max_lora_rank: Optional[int] = None, + lora_target_modules: Optional[List[str]] = None, + max_new_tokens: int = 32, + ) -> List[tuple]: + """ + Runs a sequence of operations on the SRT runner, including loading and unloading LoRA adapters, + and performing forward passes with the current set of loaded adapters. + """ + + forward_outputs = [] + with LoRAUpdateTestSession( + testcase=self, + mode=mode, + model_path=base, + lora_paths=initial_adapters, + max_loras_per_batch=max_loras_per_batch, + max_loaded_loras=max_loaded_loras, + max_lora_rank=max_lora_rank, + lora_target_modules=lora_target_modules, + enable_lora=enable_lora, + ) as session: + for op in op_sequence: + op_type = op.type + data = op.data + expected_error = op.expected_error + print("-" * 100) + print( + f"Running operation: {op_type} --- data: {data} --- mode: {mode} ---" + ) + if op_type == OperationType.LOAD: + if isinstance(data, str): + adapter_info = { + "lora_name": data, + "lora_path": data, + "pinned": False, + } + else: + adapter_info = data + + result = session.load_lora_adapter( + expected_error=expected_error, + **adapter_info, + ) + elif op_type == OperationType.UNLOAD: + result = session.unload_lora_adapter( + lora_name=data, + ) + elif op_type == OperationType.FORWARD: + prompts, adapters = zip(*data) + result = session.forward( + prompts=list(prompts), + lora_paths=list(adapters), + max_new_tokens=max_new_tokens, + expected_error=expected_error, + ) + if not expected_error: + forward_outputs.append(result) + + return forward_outputs + + def _run_dynamic_adapter_updates( + self, mode: LoRAUpdateTestSessionMode, test_cases: Iterable[TestCase] + ): + for case_idx, test_case in enumerate(test_cases, start=1): + print("=" * 100) + print( + f"Starting test case {case_idx} in {mode.value} mode. Test description: {test_case.description}" + ) + print("=" * 100) + + print( + f"--- Running dynamic update pass with {len(test_case.op_sequence)} operations ---" + ) + # Test dynamic loading of adapters + dynamic_output = self._run_operation_sequence( + mode=mode, + initial_adapters=test_case.initial_adapters, + enable_lora=test_case.enable_lora, + base=test_case.base, + max_loras_per_batch=test_case.max_loras_per_batch, + max_loaded_loras=test_case.max_loaded_loras, + op_sequence=test_case.op_sequence, + max_new_tokens=test_case.max_new_tokens, + max_lora_rank=test_case.max_lora_rank, + lora_target_modules=test_case.lora_target_modules, + ) + + # static loading + forward_ops = [ + x + for x in test_case.op_sequence + if x.type == OperationType.FORWARD and x.expected_error is None + ] + + if not forward_ops: + print( + f"No forward operations found in test case {case_idx}. Skipping static pass." + ) + continue + + print("=" * 100) + print(f"\n--- Running static pass with {len(forward_ops)} operations ---") + static_output = self._run_operation_sequence( + mode=mode, + initial_adapters=test_case.all_adapters, + enable_lora=test_case.enable_lora, + base=test_case.base, + max_loras_per_batch=test_case.max_loras_per_batch, + op_sequence=forward_ops, + max_new_tokens=test_case.max_new_tokens, + ) + + print(f"Dynamic output: {dynamic_output}") + print(f"Static output: {static_output}") + print("=" * 100) + self.assertEqual( + len(dynamic_output), + len(static_output), + f"Dynamic output length {len(dynamic_output)} does not match static output length {len(static_output)}", + ) + for i, (dynamic, static) in enumerate( + zip(dynamic_output, static_output), start=1 + ): + self.assertEqual( + len(dynamic), + len(static), + f"Output length mismatch at batch {i}:\n- Dynamic={len(dynamic)}\n- Static={len(static)}", + ) + for j, (d_out, s_out) in enumerate(zip(dynamic, static), start=1): + d_out = d_out.strip() + s_out = s_out.strip() + self.assertEqual( + d_out, + s_out, + f"Output mismatch at batch {i}, prompt {j}:\n- Dynamic: '{d_out}'\n- Static: '{s_out}'", + ) + + def test_dynamic_lora_update_engine(self): + """ + Test dynamic LoRA updates in engine mode. + """ + test_cases = BASIC_TESTS if is_in_ci() else ALL_TESTS + self._run_dynamic_adapter_updates( + mode=LoRAUpdateTestSessionMode.ENGINE, + test_cases=test_cases, + ) + + def test_dynamic_lora_update_server(self): + """ + Test dynamic LoRA updates in server mode. + """ + test_cases = BASIC_TESTS if is_in_ci() else ALL_TESTS + self._run_dynamic_adapter_updates( + mode=LoRAUpdateTestSessionMode.SERVER, test_cases=test_cases + ) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/test_multi_lora_backend.py b/test/srt/lora/test_multi_lora_backend.py new file mode 100644 index 000000000..310ce72e4 --- /dev/null +++ b/test/srt/lora/test_multi_lora_backend.py @@ -0,0 +1,90 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import os +import unittest +from typing import List + +from utils import ( + ALL_OTHER_MULTI_LORA_MODELS, + BACKENDS, + CI_MULTI_LORA_MODELS, + TORCH_DTYPES, + LoRAModelCase, + run_lora_test_one_by_one, +) + +from sglang.test.test_utils import CustomTestCase, is_in_ci + +# All prompts are used at once in a batch. +PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids. + ### Question: + What do you know about llamas? + ### Answer: + """, +] + + +class TestMultiLoRABackend(CustomTestCase): + + def _run_multi_lora_test_on_model_cases(self, model_cases: List[LoRAModelCase]): + for model_case in model_cases: + # If skip_long_prompt is True, filter out prompts longer than 1000 characters. + batch_prompts = ( + PROMPTS + if not model_case.skip_long_prompt + else [p for p in PROMPTS if len(p) < 1000] + ) + for torch_dtype in TORCH_DTYPES: + for backend in BACKENDS: + run_lora_test_one_by_one( + batch_prompts, + model_case, + torch_dtype, + max_new_tokens=32, + backend=backend, + test_tag="multi-lora-backend", + ) + + def test_ci_lora_models(self): + self._run_multi_lora_test_on_model_cases(CI_MULTI_LORA_MODELS) + + def test_all_lora_models(self): + if is_in_ci(): + return + + # Retain ONLY_RUN check here + filtered_models = [] + for model_case in ALL_OTHER_MULTI_LORA_MODELS: + if "ONLY_RUN" in os.environ and os.environ["ONLY_RUN"] != model_case.base: + continue + filtered_models.append(model_case) + + self._run_multi_lora_test_on_model_cases(filtered_models) + + +if __name__ == "__main__": + try: + mp.set_start_method("spawn") + except RuntimeError: + pass + + unittest.main(warnings="ignore") diff --git a/test/srt/lora/utils.py b/test/srt/lora/utils.py new file mode 100644 index 000000000..94ce8ab60 --- /dev/null +++ b/test/srt/lora/utils.py @@ -0,0 +1,397 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import dataclasses +import random +from typing import List + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import calculate_rouge_l + + +@dataclasses.dataclass +class LoRAAdaptor: + name: str + prefill_tolerance: float = None + decode_tolerance: float = None + rouge_l_tolerance: float = None + + +@dataclasses.dataclass +class LoRAModelCase: + base: str + adaptors: List[LoRAAdaptor] + tp_size: int = 1 + prefill_tolerance: float = 1e-1 + decode_tolerance: float = 1e-1 + rouge_l_tolerance: float = 1.0 + max_loras_per_batch: int = 1 + skip_long_prompt: bool = False + + def __post_init__(self): + if len(self.adaptors) > self.max_loras_per_batch: + raise ValueError( + f"For base '{self.base}', number of adaptors ({len(self.adaptors)}) " + f"must be <= max_loras_per_batch ({self.max_loras_per_batch})" + ) + + +TORCH_DTYPES = [torch.float16] +BACKENDS = ["triton"] +DEFAULT_PROMPTS = [ + "AI is a field of computer science focused on", + """ + ### Instruction: + Tell me about llamas and alpacas + ### Response: + Llamas are large, long-necked animals with a woolly coat. They have two toes on each foot instead of three like other camelids (camels, dromedaries). Llamas live in the Andean mountains of South America where they graze on grasses and shrubs. Alpaca is another name for domesticated llama. The word "alpaca" comes from an Incan language meaning "golden fleece." Alpacas look very similar to llamas but are smaller than their wild relatives. Both species were used by ancient people as pack animals and for meat. Today both llamas and alpacas are raised primarily for their fiber which can be spun into yarn or knitted into clothing. + ### Question 2: + What do you know about llamas? + ### Answer: + """, +] + +CI_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + ), + ], + max_loras_per_batch=1, + ), +] + +ALL_OTHER_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + prefill_tolerance=1e-1, + ), + ], + max_loras_per_batch=1, + ), + LoRAModelCase( + base="meta-llama/Llama-2-7b-hf", + adaptors=[LoRAAdaptor(name="winddude/wizardLM-LlaMA-LoRA-7B")], + max_loras_per_batch=2, + ), +] + +CI_MULTI_LORA_MODELS = [ + # multi-rank case + LoRAModelCase( + base="meta-llama/Llama-2-7b-hf", + adaptors=[ + LoRAAdaptor( + name="winddude/wizardLM-LlaMA-LoRA-7B", + prefill_tolerance=1e-1, + ), + LoRAAdaptor( + name="RuterNorway/Llama-2-7b-chat-norwegian-LoRa", + prefill_tolerance=3e-1, + ), + ], + max_loras_per_batch=2, + ), +] + +ALL_OTHER_MULTI_LORA_MODELS = [ + LoRAModelCase( + base="meta-llama/Llama-3.1-8B-Instruct", + adaptors=[ + LoRAAdaptor( + name="algoprog/fact-generation-llama-3.1-8b-instruct-lora", + prefill_tolerance=1e-1, + ), + LoRAAdaptor( + name="Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + prefill_tolerance=1e-1, + ), + ], + max_loras_per_batch=2, + ), +] + + +def run_lora_test_one_by_one( + prompts: List[str], + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + backend: str, + disable_cuda_graph: bool = False, + disable_radix_cache: bool = False, + mem_fraction_static: float = 0.88, + test_tag: str = "", +): + """ + Input a batch of prompts, and run lora tests one by one with several generate requests + (each request will have bs=1). + For prompt0, prompt1, ..., promptN, + we will use adaptor0, adaptor1, ..., adaptorN included in model case, + We will then compare the outputs of HF and SRT with and without LoRA. + If number of prompts is larger than number of adaptors, + the prompt i will use adaptor i % (number of adaptors). + + Args: + prompts (List[str]): The batch of prompts to test. + model_case (LoRAModelCase): The model case to test. + torch_dtype (torch.dtype): The torch dtype to use. + max_new_tokens (int): The maximum number of new tokens to generate. + backend (str): The lora backend to use. + disable_cuda_graph (bool, optional): Whether to disable CUDA graph. Defaults to False. + disable_radix_cache (bool, optional): Whether to disable radix cache. Defaults to False. + mem_fraction_static (float, optional): The fraction of memory to use. Defaults to 0.88. + test_tag (str, optional): The tag to use for the test. Defaults to "". + """ + base_path = model_case.base + + # Create used adaptors for each prompt in batch + i, adaptors = 0, [] + for _ in range(len(prompts)): + adaptors.append(model_case.adaptors[i]) + i = (i + 1) % len(model_case.adaptors) + adaptor_names = [adaptor.name for adaptor in adaptors] + + print( + f"\n========== Testing {test_tag} on base '{model_case.base}' with backend={backend}, dtype={torch_dtype} --- " + f"Using prompts {[p[:50] for p in prompts]} with adaptors: {adaptor_names} ---" + ) + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + lora_paths=[ + adaptor.name for adaptor in model_case.adaptors if adaptor.name is not None + ], + max_loras_per_batch=model_case.max_loras_per_batch, + lora_backend=backend, + disable_cuda_graph=disable_cuda_graph, + disable_radix_cache=disable_radix_cache, + mem_fraction_static=mem_fraction_static, + ) as srt_runner: + srt_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + mem_fraction_static=mem_fraction_static, + ) as srt_runner: + srt_no_lora_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + hf_no_lora_outputs = hf_runner.forward(prompts, max_new_tokens=max_new_tokens) + + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + for i in range(len(prompts)): + adaptor = adaptors[i] + # Use individual adaptor tolerances if set, otherwise use model defaults + prefill_tol = ( + adaptor.prefill_tolerance + if adaptor.prefill_tolerance is not None + else model_case.prefill_tolerance + ) + decode_tol = ( + adaptor.decode_tolerance + if adaptor.decode_tolerance is not None + else model_case.decode_tolerance + ) + rouge_tol = ( + adaptor.rouge_l_tolerance + if adaptor.rouge_l_tolerance is not None + else model_case.rouge_l_tolerance + ) + # Compare prefill stage logprobs (HF vs SRTRunner with LoRA) + hf_prefill = torch.tensor(hf_outputs.top_input_logprobs[i]) + srt_prefill = torch.tensor(srt_outputs.top_input_logprobs[i]) + max_prefill_diff = torch.max(torch.abs(hf_prefill - srt_prefill)) + print("Max prefill diff (HF vs SRT):", max_prefill_diff) + + # Compare decode stage logprobs + hf_decode = torch.tensor(hf_outputs.top_output_logprobs[i]) + srt_decode = torch.tensor(srt_outputs.top_output_logprobs[i]) + max_decode_diff = torch.max(torch.abs(hf_decode - srt_decode)) + print("Max decode diff (HF vs SRT):", max_decode_diff) + + srt_output_str = srt_outputs.output_strs[i].strip() + hf_output_str = hf_outputs.output_strs[i].strip() + rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0] + print("ROUGE-L score:", rouge_score) + print("SRT output:", srt_output_str) + print("HF output:", hf_output_str) + + # Additional: compare prefill outputs between base model (no LoRA) and LoRA model for reference + hf_no_lora_prefill = torch.tensor(hf_no_lora_outputs.top_input_logprobs[i]) + srt_no_lora_prefill = torch.tensor(srt_no_lora_outputs.top_input_logprobs[i]) + print( + "Max diff (SRT base vs SRT LoRA prefill):", + torch.max(torch.abs(srt_no_lora_prefill - srt_prefill)), + ) + print( + "Max diff (HF base vs HF LoRA prefill):", + torch.max(torch.abs(hf_no_lora_prefill - hf_prefill)), + ) + + if hf_prefill.shape[0] <= 100: + assert torch.all(torch.abs(hf_prefill - srt_prefill) < prefill_tol), ( + f"Prefill logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', " + f"backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + if hf_decode.shape[0] <= 100: + assert torch.all(torch.abs(hf_decode - srt_decode) < decode_tol), ( + f"Decode logprobs mismatch for base '{base_path}', adaptor '{adaptor_names}', " + f"backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + if rouge_score < rouge_tol: + raise AssertionError( + f"ROUGE-L score {rouge_score} below tolerance {rouge_tol} " + f"for base '{base_path}', adaptor '{adaptor_names}', backend '{backend}', prompt: '{prompts[0][:50]}...'" + ) + + +def run_lora_test_by_batch( + prompts: List[str], + model_case: LoRAModelCase, + torch_dtype: torch.dtype, + max_new_tokens: int, + backend: str, + disable_cuda_graph: bool = False, + disable_radix_cache: bool = False, + mem_fraction_static: float = 0.88, + test_tag: str = "", +): + """ + Run lora tests as a batch. + For prompt0, prompt1, ..., promptN, + we will use adaptor0, adaptor1, ..., adaptorN included in model case, + We will then compare the outputs of HF and SRT with LoRA. + If number of prompts is larger than number of adaptors, + the prompt i will use adaptor i % (number of adaptors). + + Args: + prompts (List[str]): The batch of prompts to test. + model_case (LoRAModelCase): The model case to test. + torch_dtype (torch.dtype): The torch dtype to use. + max_new_tokens (int): The maximum number of new tokens to generate. + backend (str): The lora backend to use. + disable_cuda_graph (bool, optional): Whether to disable CUDA graph. Defaults to False. + disable_radix_cache (bool, optional): Whether to disable radix cache. Defaults to False. + mem_fraction_static (float, optional): The fraction of memory to use. Defaults to 0.88. + test_tag (str, optional): The tag to use for the test. Defaults to "". + """ + base_path = model_case.base + + # Create used adaptors for each prompt in batch + i, adaptors = 0, [] + for _ in range(len(prompts)): + adaptors.append(model_case.adaptors[i]) + i = (i + 1) % len(model_case.adaptors) + adaptor_names = [adaptor.name for adaptor in adaptors] + + print( + f"\n========== Testing {test_tag} on base '{model_case.base}' with backend={backend}, dtype={torch_dtype} --- " + f"Using prompts {[p[:50] for p in prompts]} with adaptors: {adaptor_names} ---" + ) + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + lora_paths=[ + adaptor.name for adaptor in model_case.adaptors if adaptor.name is not None + ], + max_loras_per_batch=model_case.max_loras_per_batch, + lora_backend=backend, + disable_cuda_graph=disable_cuda_graph, + disable_radix_cache=disable_radix_cache, + mem_fraction_static=mem_fraction_static, + ) as srt_runner: + srt_outputs = srt_runner.batch_forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + + with SRTRunner( + base_path, + torch_dtype=torch_dtype, + model_type="generation", + tp_size=model_case.tp_size, + mem_fraction_static=mem_fraction_static, + ) as srt_runner: + srt_no_lora_outputs = srt_runner.batch_forward( + prompts, max_new_tokens=max_new_tokens + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_outputs = hf_runner.forward( + prompts, max_new_tokens=max_new_tokens, lora_paths=adaptor_names + ) + + with HFRunner( + base_path, torch_dtype=torch_dtype, model_type="generation" + ) as hf_runner: + hf_no_lora_outputs = hf_runner.forward( + prompts, + max_new_tokens=max_new_tokens, + ) + + for i in range(len(prompts)): + + srt_output_str = srt_outputs.output_strs[i].strip() + hf_output_str = hf_outputs.output_strs[i].strip() + rouge_score = calculate_rouge_l([srt_output_str], [hf_output_str])[0] + print("ROUGE-L score:", rouge_score) + print("SRT output:", srt_output_str) + print("HF output:", hf_output_str) + print("SRT no lora output:", srt_no_lora_outputs.output_strs[i].strip()) + print("HF no lora output:", hf_no_lora_outputs.output_strs[i].strip()) + assert srt_outputs.output_strs[i].strip(" ") == hf_outputs.output_strs[i].strip( + " " + ), ( + srt_outputs.output_strs[i].strip(" "), + hf_outputs.output_strs[i].strip(" "), + ) + assert srt_no_lora_outputs.output_strs[i].strip( + " " + ) == hf_no_lora_outputs.output_strs[i].strip(" "), ( + srt_no_lora_outputs.output_strs[i].strip(" "), + hf_no_lora_outputs.output_strs[i].strip(" "), + ) + + +def ensure_reproducibility(): + seed = 42 + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.use_deterministic_algorithms(True) diff --git a/test/srt/models/compare.py b/test/srt/models/compare.py new file mode 100644 index 000000000..2fe35357c --- /dev/null +++ b/test/srt/models/compare.py @@ -0,0 +1,52 @@ +""" +used for debug using tensor comparison +dump {name: tensor} into "log_hf.jsonl" and "log_srt.jsonl" +use the same name for two tensors that supposed to be close +recommend name like: "layer 2 after mlp" +""" + +import json +import sys + +import torch + +if len(sys.argv) > 1: + assert sys.argv[1] == "base" + hf_log = "base_log_hf.jsonl" + srt_log = "base_log_srt.jsonl" +else: + hf_log = "log_hf.jsonl" + srt_log = "log_srt.jsonl" + + +def load_data(filepath): + tensors = {} + with open(filepath, "r") as f: + lines = f.readlines() + for line in lines: + data = json.loads(line) + for k, v in data.items(): + tensors[k] = torch.tensor(v) + return tensors + + +hf_tensors = load_data(hf_log) +srt_tensors = load_data(srt_log) + + +def get_diff(t1, t2): + t1 = t1.reshape(t2.shape) + max_diff = torch.max(abs(t1.reshape(t2.shape) - t2)) + l2_dis = torch.dist(t1, t2, p=2) + return l2_dis, max_diff + + +for k, _ in srt_tensors.items(): + l2_dis, max_diff = get_diff(hf_tensors[k], srt_tensors[k]) + print(f"{k} {l2_dis=} {max_diff=}") + if k == "layer 1 attn": + print(hf_tensors[k]) + print(srt_tensors[k]) + if k == "layer 0 prefill k": + print(srt_tensors[k].shape) + print(hf_tensors[k].shape) diff --git a/test/srt/models/test_clip_models.py b/test/srt/models/test_clip_models.py new file mode 100644 index 000000000..8a79656d0 --- /dev/null +++ b/test/srt/models/test_clip_models.py @@ -0,0 +1,80 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch +from transformers import AutoProcessor + +from sglang.srt.utils import load_image +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import get_similarities + +TEXTS = "two Subway Series sandwiches with meats, cheese, lettuce, tomatoes, and onions on a black background, accompanied by the Subway Series logo, highlighting a new sandwich series." +IMAGES = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" +MODELS = [ + ("openai/clip-vit-large-patch14-336", 1e-5), +] +TORCH_DTYPES = [torch.float16] + + +class TestClipModels(unittest.TestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_embeddings(self, model, prefill_tolerance, torch_dtype): + + with HFRunner( + model, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_text_embeds = hf_runner.forward(prompts=TEXTS) + hf_image_embeds = hf_runner.forward(image_data=IMAGES) + + with SRTRunner( + model, + tp_size=1, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + text_embeds = srt_runner.forward(prompts=TEXTS) + image_embeds = srt_runner.forward(prompts="padding", image_data=IMAGES) + + text_similarity = get_similarities( + text_embeds.embed_logits[0], hf_text_embeds.embed_logits[0] + ) + image_similarity = get_similarities( + image_embeds.embed_logits[0], hf_image_embeds.embed_logits[0] + ) + print("text similarity diff", abs(text_similarity - 1)) + print("image similarity diff", abs(image_similarity - 1)) + assert torch.all( + abs(text_similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + assert torch.all( + abs(image_similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_accuracy(self): + for model, prefill_tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_embeddings(model, prefill_tolerance, torch_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_compressed_tensors_models.py b/test/srt/models/test_compressed_tensors_models.py new file mode 100644 index 000000000..34f699de4 --- /dev/null +++ b/test/srt/models/test_compressed_tensors_models.py @@ -0,0 +1,46 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestCompressedTensorsLlama3FP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "RedHatAI/Meta-Llama-3.1-8B-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["accuracy"], 0.45) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_cross_encoder_models.py b/test/srt/models/test_cross_encoder_models.py new file mode 100644 index 000000000..93edc3fa1 --- /dev/null +++ b/test/srt/models/test_cross_encoder_models.py @@ -0,0 +1,91 @@ +import multiprocessing as mp +import random +import unittest + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import TEST_RERANK_QUERY_DOCS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, is_in_ci + +MODELS = [ + ("cross-encoder/ms-marco-MiniLM-L6-v2", 1, 1e-2), + ("BAAI/bge-reranker-v2-m3", 1, 1e-2), +] +ATTENTION_BACKEND = ["torch_native", "triton"] + +TORCH_DTYPES = [torch.float32] + + +class TestCrossEncoderModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + score_tolerance, + attention_backend, + ) -> None: + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="cross_encoder", + ) as hf_runner: + hf_scores = hf_runner.forward(prompts).scores + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="cross_encoder", + attention_backend=attention_backend, + chunked_prefill_size=-1, + disable_radix_cache=True, + ) as srt_runner: + srt_scores = srt_runner.forward(prompts).scores + + for i in range(len(srt_scores)): + score_difference = abs(hf_scores[i] - srt_scores[i]) + + assert ( + score_difference < score_tolerance + ), "cross encoder scores are not all close" + + def preprocess_prompts(self, prompt): + processed_prompts = [] + query = prompt["query"] + documents = prompt["documents"] + for document in documents: + processed_prompts.append([query, document]) + + return processed_prompts + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for attention_backend in ATTENTION_BACKEND: + for queryDocs in TEST_RERANK_QUERY_DOCS: + prompts = self.preprocess_prompts(queryDocs) + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + prompts, + model, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_dummy_grok_models.py b/test/srt/models/test_dummy_grok_models.py new file mode 100644 index 000000000..bebf9949c --- /dev/null +++ b/test/srt/models/test_dummy_grok_models.py @@ -0,0 +1,34 @@ +import unittest + +from sglang.test.test_utils import CustomTestCase, is_in_ci, run_bench_one_batch + + +class TestDummyGrok1(CustomTestCase): + + def test_dummy_grok_1(self): + _, output_throughput, _ = run_bench_one_batch( + None, + [ + "--model", + "/dummy-grok", + "--tokenizer-path", + "Xenova/grok-1-tokenizer", + "--batch-size", + "2", + "--tp", + "2", + "--quantization", + "fp8", + "--load-format", + "dummy", + "--json-model-override-args", + '{"num_hidden_layers": 2}', + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_embedding_models.py b/test/srt/models/test_embedding_models.py new file mode 100644 index 000000000..b56e952d7 --- /dev/null +++ b/test/srt/models/test_embedding_models.py @@ -0,0 +1,111 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import random +import unittest + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci + +MODELS = [ + ("Alibaba-NLP/gte-Qwen2-1.5B-instruct", 1, 1e-5), + ("intfloat/e5-mistral-7b-instruct", 1, 1e-5), + ("marco/mcdse-2b-v1", 1, 1e-5), + ("Qwen/Qwen3-Embedding-8B", 1, 1e-5), + # Temporarily disable before this model is fixed + # ("jason9693/Qwen2.5-1.5B-apeach", 1, 1e-5), +] +TORCH_DTYPES = [torch.float16] + + +class TestEmbeddingModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", 2048) + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + ) -> None: + truncated_prompts = self._truncate_prompts(prompts, model_path) + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_outputs = hf_runner.forward(truncated_prompts) + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + srt_outputs = srt_runner.forward(truncated_prompts) + + for i in range(len(prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + print("similarity diff", abs(similarity - 1)) + + if len(prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for torch_dtype in TORCH_DTYPES: + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, model, tp_size, torch_dtype, prefill_tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_encoder_embedding_models.py b/test/srt/models/test_encoder_embedding_models.py new file mode 100644 index 000000000..dafaa72db --- /dev/null +++ b/test/srt/models/test_encoder_embedding_models.py @@ -0,0 +1,162 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# python -m unittest test_encoder_embedding_models.TestEncoderEmbeddingModels.test_prefill_logits + +import multiprocessing as mp +import random +import time +import unittest + +import torch +from transformers import AutoConfig, AutoTokenizer + +from sglang.test.runners import DEFAULT_PROMPTS, HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities, is_in_ci + +MODELS = [("BAAI/bge-small-en", 1, 1e-5), ("BAAI/bge-m3", 1, 1e-5)] + +ATTENTION_BACKEND = ["torch_native", "triton", "flashinfer"] +BATCH_SIZE = [1, 2] +TORCH_DTYPES = [torch.float32, torch.float16] +sgl_to_st_ratio = [] + + +class TestEncoderEmbeddingModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def _truncate_prompts(self, prompts, model_path): + config = AutoConfig.from_pretrained(model_path) + max_length = getattr(config, "max_position_embeddings", 512) - 20 + + tokenizer = AutoTokenizer.from_pretrained(model_path) + + truncated_prompts = [] + for prompt in prompts: + tokens = tokenizer(prompt, return_tensors="pt", truncation=False) + if len(tokens.input_ids[0]) > max_length: + truncated_text = tokenizer.decode( + tokens.input_ids[0][: max_length - 1], skip_special_tokens=True + ) + truncated_prompts.append(truncated_text) + else: + truncated_prompts.append(prompt) + + return truncated_prompts + + def assert_close_prefill_logits( + self, + prompts, + model_path, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + batch_size, + ) -> None: + truncated_prompts = self._truncate_prompts(prompts, model_path) + truncated_prompts = truncated_prompts * batch_size + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + # warm up + hf_outputs = hf_runner.forward(truncated_prompts) + + st_start_time = time.perf_counter() + hf_outputs = hf_runner.forward(truncated_prompts) + st_end_time = time.perf_counter() + + with SRTRunner( + model_path, + tp_size=tp_size, + torch_dtype=torch_dtype, + model_type="embedding", + attention_backend=attention_backend, + chunked_prefill_size=-1, + disable_radix_cache=True, + ) as srt_runner: + # warm up + srt_outputs = srt_runner.forward(truncated_prompts) + + sgl_start_time = time.perf_counter() + srt_outputs = srt_runner.forward(truncated_prompts) + sgl_end_time = time.perf_counter() + + transformer_time = st_end_time - st_start_time + sgl_time = sgl_end_time - sgl_start_time + sgl_to_st_ratio.append(sgl_time / transformer_time) + + for i in range(len(truncated_prompts)): + hf_logits = torch.Tensor(hf_outputs.embed_logits[i]) + srt_logits = torch.Tensor(srt_outputs.embed_logits[i]) + + similarity = torch.tensor(get_similarities(hf_logits, srt_logits)) + # If something is wrong, uncomment this to observe similarity. + # print("similarity diff", abs(similarity - 1)) + + if len(truncated_prompts[i]) <= 1000: + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_prefill_logits(self): + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model, tp_size, prefill_tolerance in models_to_test: + for attention_backend in ATTENTION_BACKEND: + for batch_size in BATCH_SIZE: + for torch_dtype in TORCH_DTYPES: + # NOTE: FlashInfer currently has limitations with head_dim = 32 or + # other dimensions. + # The FlashInfer head_dim limitation itself is tracked here: + # https://github.com/flashinfer-ai/flashinfer/issues/1048 + # + # Flashinfer does not support torch.float32 for dtype_q, so skip it + if attention_backend == "flashinfer": + if ( + model == "BAAI/bge-small-en" + or torch_dtype == torch.float32 + ): + continue + + self.assert_close_prefill_logits( + DEFAULT_PROMPTS, + model, + tp_size, + torch_dtype, + prefill_tolerance, + attention_backend, + batch_size, + ) + + for i in range(len(BATCH_SIZE)): + print( + "bacth size: ", + BATCH_SIZE[i] * 5, + "sgl_time/st_time", + round(sgl_to_st_ratio[i], 3), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_generation_models.py b/test/srt/models/test_generation_models.py new file mode 100644 index 000000000..a9d8fe0df --- /dev/null +++ b/test/srt/models/test_generation_models.py @@ -0,0 +1,181 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Usage: + +To test a specific model locally: +1. Add it to ALL_MODELS, for example, `ModelCase("Qwen/Qwen2-1.5B")` +2. Run `ONLY_RUN=Qwen/Qwen2-1.5B python3 -m unittest test_generation_models.TestGenerationModels` +""" + +import dataclasses +import multiprocessing as mp +import os +import random +import unittest +from typing import List + +import torch + +from sglang.test.runners import ( + DEFAULT_PROMPTS, + HFRunner, + SRTRunner, + check_close_model_outputs, +) +from sglang.test.test_utils import CustomTestCase, is_in_ci + + +@dataclasses.dataclass +class ModelCase: + model_path: str + tp_size: int = 1 + prefill_tolerance: float = 5e-2 + decode_tolerance: float = 6e-2 # Increased to fix numerical error in issue #8614. + rouge_l_tolerance: float = 1 + skip_long_prompt: bool = False + trust_remote_code: bool = False + + +# Popular models that run on the CI +CI_MODELS = [ + ModelCase("meta-llama/Llama-3.1-8B-Instruct"), + ModelCase("google/gemma-2-2b"), +] + +# the complete set of models to test sglang's generation model +ALL_MODELS = [ + *CI_MODELS, + ModelCase("Qwen/Qwen2-1.5B"), + ModelCase("Qwen/Qwen2.5-14B-Instruct"), + ModelCase("HuggingFaceTB/SmolLM-135M-Instruct", skip_long_prompt=True), + ModelCase("allenai/OLMo-1B-0724-hf", decode_tolerance=8e-2, skip_long_prompt=True), + ModelCase( + "THUDM/glm-4-9b-chat", tp_size=2, trust_remote_code=True, skip_long_prompt=True + ), + ModelCase("openai-community/gpt2"), + ModelCase("microsoft/phi-1_5", trust_remote_code=True), + ModelCase("adept/persimmon-8b-chat"), + ModelCase("inclusionAI/Ling-lite", trust_remote_code=True), + ModelCase("microsoft/Phi-3-small-8k-instruct", trust_remote_code=True), + ModelCase("allenai/OLMo-2-1124-7B-Instruct", skip_long_prompt=True), + ModelCase("ibm-granite/granite-3.0-2b-instruct", skip_long_prompt=True), + ModelCase( + "microsoft/Phi-3.5-MoE-instruct", + tp_size=2, + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase("facebook/opt-125m", skip_long_prompt=True), + ModelCase( + "nvidia/Llama-3_3-Nemotron-Super-49B-v1_5", + tp_size=2, + trust_remote_code=True, + skip_long_prompt=True, + ), + ModelCase( + "nvidia/Llama-3_1-Nemotron-Ultra-253B-v1", + tp_size=8, + trust_remote_code=True, + skip_long_prompt=True, + ), +] + +TORCH_DTYPES = [torch.float16] + + +class TestGenerationModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_logits_and_output_strs( + self, + prompts: List[str], + model_case: ModelCase, + torch_dtype: torch.dtype, + ) -> None: + model_path = model_case.model_path + prefill_tolerance, decode_tolerance, rouge_l_tolerance = ( + model_case.prefill_tolerance, + model_case.decode_tolerance, + model_case.rouge_l_tolerance, + ) + max_new_tokens = 32 + + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as hf_runner: + hf_outputs = hf_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + ) as srt_runner: + srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=model_case.prefill_tolerance, + decode_tolerance=model_case.decode_tolerance, + rouge_l_tolerance=model_case.rouge_l_tolerance, + debug_text=f"model_path={model_path} prompts={prompts}", + ) + + @unittest.skipIf(not is_in_ci(), "Local test should run all models") + def test_ci_models(self): + for model_case in CI_MODELS: + for torch_dtype in TORCH_DTYPES: + prompts = DEFAULT_PROMPTS + + # Skip long prompts for models that do not have a long context + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs( + prompts, model_case, torch_dtype + ) + + @unittest.skipIf(is_in_ci(), "CI only runs selected models for simplicity") + def test_all_models(self): + for model_case in ALL_MODELS: + for torch_dtype in TORCH_DTYPES: + if ( + "ONLY_RUN" in os.environ + and os.environ["ONLY_RUN"] != model_case.model_path + ): + continue + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs( + prompts, model_case, torch_dtype + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_gme_qwen_models.py b/test/srt/models/test_gme_qwen_models.py new file mode 100644 index 000000000..265bf8a39 --- /dev/null +++ b/test/srt/models/test_gme_qwen_models.py @@ -0,0 +1,85 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase, get_similarities + +TEXTS = "two Subway Series sandwiches with meats, cheese, lettuce, tomatoes, and onions on a black background, accompanied by the Subway Series logo, highlighting a new sandwich series." +IMAGES = "https://huggingface.co/datasets/liuhaotian/llava-bench-in-the-wild/resolve/main/images/023.jpg" + + +MODELS = [ + ("Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", 1e-3), +] +TORCH_DTYPES = [torch.float16] + + +class TestQmeQwenModels(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_embeddings(self, model, prefill_tolerance, torch_dtype): + + prompts_no_image = f"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\n{TEXTS}<|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + prompts_with_image = f"<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|><|im_end|>\n<|im_start|>assistant\n<|endoftext|>" + with HFRunner( + model, + torch_dtype=torch_dtype, + model_type="embedding", + ) as hf_runner: + hf_text_embeddings = hf_runner.forward(prompts=[prompts_no_image]) + hf_image_embeddings = hf_runner.forward( + prompts=[prompts_with_image], image_data=[IMAGES] + ) + with SRTRunner( + model, + tp_size=1, + torch_dtype=torch_dtype, + model_type="embedding", + ) as srt_runner: + srt_text_embeddings = srt_runner.forward(prompts=prompts_no_image) + srt_image_embeddings = srt_runner.forward( + prompts=prompts_with_image, image_data=IMAGES + ) + + similarity = get_similarities( + hf_text_embeddings.embed_logits[0], srt_text_embeddings.embed_logits[0] + ) + print("texts similarity diff", abs(similarity - 1)) + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + similarity = get_similarities( + hf_image_embeddings.embed_logits[0], srt_image_embeddings.embed_logits[0] + ) + print("images similarity diff", abs(similarity - 1)) + assert torch.all( + abs(similarity - 1) < prefill_tolerance + ), "embeddings are not all close" + + def test_accuracy(self): + for model, prefill_tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_embeddings(model, prefill_tolerance, torch_dtype) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_grok_models.py b/test/srt/models/test_grok_models.py new file mode 100644 index 000000000..625fa1a65 --- /dev/null +++ b/test/srt/models/test_grok_models.py @@ -0,0 +1,53 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestGrok(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmzheng/grok-1" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "dummy", + "--json-model-override-args", + '{"num_hidden_layers": 2}', + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=64, + max_new_tokens=256, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + + # It is dummy weights so we only assert the output throughput instead of accuracy. + self.assertGreater(metrics["output_throughput"], 1000) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_llama4_models.py b/test/srt/models/test_llama4_models.py new file mode 100644 index 000000000..3835ca8d7 --- /dev/null +++ b/test/srt/models/test_llama4_models.py @@ -0,0 +1,74 @@ +import random +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +MODELS = [ + SimpleNamespace( + model="meta-llama/Llama-4-Scout-17B-16E-Instruct", + accuracy=0.9, + tp_size=4, + ), +] + + +class TestLlama4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_gsm8k(self): + + for model in MODELS: + try: + process = popen_launch_server( + model.model, + self.base_url, + timeout=3 * DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chat-template", + "llama-4", + "--tp-size", + str(model.tp_size), + "--mem-fraction-static", + "0.8", + "--context-length", + "8192", + ], + ) + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["accuracy"], model.accuracy) + except Exception as e: + print(f"Error testing {model.model}: {e}") + self.fail(f"Test failed for {model.model}: {e}") + + finally: + # Ensure process cleanup happens regardless of success/failure + if process is not None and process.poll() is None: + print(f"Cleaning up process {process.pid}") + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process: {e}") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_mtp_models.py b/test/srt/models/test_mtp_models.py new file mode 100644 index 000000000..49b53c1e4 --- /dev/null +++ b/test/srt/models/test_mtp_models.py @@ -0,0 +1,58 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMiMoMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "XiaomiMiMo/MiMo-7B-RL" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--mem-fraction-static", + "0.5", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.7) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_qwen_models.py b/test/srt/models/test_qwen_models.py new file mode 100644 index 000000000..567e19f08 --- /dev/null +++ b/test/srt/models/test_qwen_models.py @@ -0,0 +1,77 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestQwen2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +class TestQwen2FP8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "neuralmagic/Qwen2-7B-Instruct-FP8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_reward_models.py b/test/srt/models/test_reward_models.py new file mode 100644 index 000000000..5592ce223 --- /dev/null +++ b/test/srt/models/test_reward_models.py @@ -0,0 +1,92 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import multiprocessing as mp +import unittest + +import torch + +from sglang.test.runners import HFRunner, SRTRunner +from sglang.test.test_utils import CustomTestCase + +MODELS = [ + ("LxzGordon/URM-LLaMa-3.1-8B", 1, 4e-2), + ("Skywork/Skywork-Reward-Llama-3.1-8B-v0.2", 1, 4e-2), +] +TORCH_DTYPES = [torch.float16] + +# PROMPT = "Jane has 12 apples. She gives 4 apples to her friend Mark, then buys 1 more apple, and finally splits all her apples equally among herself and her 2 siblings. How many apples does each person get?" +# RESPONSE1 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among herself and her 2 siblings (3 people in total). 9 ÷ 3 = 3 apples each. Each person gets 3 apples." +# RESPONSE2 = "1. Jane starts with 12 apples and gives 4 to Mark. 12 - 4 = 8. Jane now has 8 apples.\n2. Jane buys 1 more apple. 8 + 1 = 9. Jane now has 9 apples.\n3. Jane splits the 9 apples equally among her 2 siblings (2 people in total). 9 ÷ 2 = 4.5 apples each. Each person gets 4 apples." + +PROMPT = ( + "What is the range of the numeric output of a sigmoid node in a neural network?" +) +RESPONSE1 = "The output of a sigmoid node is bounded between -1 and 1." +RESPONSE2 = "The output of a sigmoid node is bounded between 0 and 1." + +CONVS = [ + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE1}], + [{"role": "user", "content": PROMPT}, {"role": "assistant", "content": RESPONSE2}], +] + + +class TestRewardModels(CustomTestCase): + + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_reward_scores( + self, + convs, + model_path, + tp_size, + torch_dtype, + tolerance, + ) -> None: + with HFRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as hf_runner: + hf_outputs = hf_runner.forward(convs) + + with SRTRunner( + model_path, + torch_dtype=torch_dtype, + model_type="reward", + ) as srt_runner: + prompts = srt_runner.tokenizer.apply_chat_template(convs, tokenize=False) + srt_outputs = srt_runner.forward(prompts) + + hf_scores = torch.tensor(hf_outputs.scores) + srt_scores = torch.tensor(srt_outputs.scores) + print(f"{hf_scores=}") + print(f"{srt_scores=}") + + assert torch.all( + abs(hf_scores - srt_scores) < tolerance + ), "reward scores are not all close" + + def test_reward_scores(self): + for model, tp_size, tolerance in MODELS: + for torch_dtype in TORCH_DTYPES: + self.assert_close_reward_scores( + CONVS, model, tp_size, torch_dtype, tolerance + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_transformers_models.py b/test/srt/models/test_transformers_models.py new file mode 100644 index 000000000..95592453f --- /dev/null +++ b/test/srt/models/test_transformers_models.py @@ -0,0 +1,181 @@ +import dataclasses +import multiprocessing as mp +import unittest +from types import SimpleNamespace +from typing import List + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestTransformersFallbackEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--model-impl", "transformers"], + ) + cls.mmlu_lower_bound = 0.65 + cls.gsm8k_lower_bound = 0.65 + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + from sglang.test.run_eval import run_eval + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + from sglang.test.few_shot_gsm8k import run_eval + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], self.gsm8k_lower_bound) + + +class TestTransformersFallbackTorchAO(TestTransformersFallbackEndpoint): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--model-impl", + "transformers", + "--torchao-config", + "int4wo-128", + ], + ) + cls.mmlu_lower_bound = 0.65 + cls.gsm8k_lower_bound = 0.65 + + +@dataclasses.dataclass +class ModelCase: + model_path: str + tp_size: int = 1 + prefill_tolerance: float = 5e-2 + decode_tolerance: float = 5e-2 + rouge_l_tolerance: float = 1 + skip_long_prompt: bool = False + trust_remote_code: bool = False + torchao_config: str = None + torch_dtype: torch.dtype = torch.float16 + + +# Popular models that run on the CI +CI_MODELS = [ + ModelCase(DEFAULT_MODEL_NAME_FOR_TEST), +] + +ALL_OTHER_MODELS = [ + ModelCase(DEFAULT_MODEL_NAME_FOR_TEST, tp_size=2), +] + + +class TestTransformersFallbackEngine(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def assert_close_logits_and_output_strs( + self, + prompts: List[str], + model_case: ModelCase, + ) -> None: + model_path = model_case.model_path + max_new_tokens = 32 + # force to use transformers impl + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=model_case.torch_dtype, + model_type="generation", + model_impl="transformers", + trust_remote_code=model_case.trust_remote_code, + torchao_config=model_case.torchao_config, + ) as srt_runner: + srt_outputs = srt_runner.forward(prompts, max_new_tokens=max_new_tokens) + + with SRTRunner( + model_path, + tp_size=model_case.tp_size, + torch_dtype=model_case.torch_dtype, + model_type="generation", + trust_remote_code=model_case.trust_remote_code, + torchao_config=model_case.torchao_config, + ) as srt_runner: + srt_transformers_outputs = srt_runner.forward( + prompts, max_new_tokens=max_new_tokens + ) + + check_close_model_outputs( + hf_outputs=srt_transformers_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=model_case.prefill_tolerance, + decode_tolerance=model_case.decode_tolerance, + rouge_l_tolerance=model_case.rouge_l_tolerance, + debug_text=f"model_path={model_path} prompts={prompts}", + ) + + def test_ci_models(self): + for model_case in CI_MODELS: + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs(prompts, model_case) + + def test_others(self): + if is_in_ci(): + return + + # Skip long prompts for models that do not have a long context + prompts = DEFAULT_PROMPTS + for model_case in ALL_OTHER_MODELS: + if model_case.skip_long_prompt: + prompts = [p for p in DEFAULT_PROMPTS if len(p) < 1000] + + # Assert the logits and output strs are close + self.assert_close_logits_and_output_strs(prompts, model_case) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_unsloth_models.py b/test/srt/models/test_unsloth_models.py new file mode 100644 index 000000000..24660ea34 --- /dev/null +++ b/test/srt/models/test_unsloth_models.py @@ -0,0 +1,213 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestUnslothPhi4(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/phi-4" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.78) + + +class TestUnslothPhi4Bnb4bit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/phi-4-bnb-4bit" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "bitsandbytes", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.75) + + +class TestUnslothPhi4UnslothBnb4bit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/phi-4-unsloth-bnb-4bit" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "bitsandbytes", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.75) + + +class TestUnslothPhi4MiniInstruct(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/Phi-4-mini-instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.65) + + +class TestUnslothPhi4MiniBnb4bit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/Phi-4-mini-instruct-bnb-4bit" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "bitsandbytes", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.6) + + +class TestUnslothPhi4MiniUnslothBnb4bit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/Phi-4-mini-instruct-unsloth-bnb-4bit" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--load-format", + "bitsandbytes", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.6) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/models/test_vlm_models.py b/test/srt/models/test_vlm_models.py new file mode 100644 index 000000000..0748f1ee0 --- /dev/null +++ b/test/srt/models/test_vlm_models.py @@ -0,0 +1,315 @@ +import argparse +import glob +import json +import os +import random +import subprocess +import sys +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +# VLM models for testing +MODELS = [ + SimpleNamespace(model="google/gemma-3-27b-it", mmmu_accuracy=0.45), + SimpleNamespace( + model="Qwen/Qwen2.5-VL-3B-Instruct", + mmmu_accuracy=0.4, + ), + SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), +] + + +class TestVLMModels(CustomTestCase): + parsed_args = None # Class variable to store args + + @classmethod + def setUpClass(cls): + # Removed argument parsing from here + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.time_out = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + + # Set OpenAI API key and base URL environment variables. Needed for lmm-evals to work. + os.environ["OPENAI_API_KEY"] = cls.api_key + os.environ["OPENAI_API_BASE"] = f"{cls.base_url}/v1" + + def _detect_eviction_in_logs(self, log_output): + """Detect if eviction events occurred in the log output.""" + eviction_keywords = ["Cache eviction: evicted"] + + eviction_detected = False + eviction_count = 0 + + for line in log_output.split("\n"): + if any(keyword in line for keyword in eviction_keywords): + eviction_detected = True + eviction_count += 1 + print(f"Eviction detected: {line.strip()}") + + return eviction_detected, eviction_count + + def run_mmmu_eval( + self, + model_version: str, + output_path: str, + *, + env: dict | None = None, + ): + """ + Evaluate a VLM on the MMMU validation set with lmms‑eval. + Only `model_version` (checkpoint) and `chat_template` vary; + We are focusing only on the validation set due to resource constraints. + """ + # -------- fixed settings -------- + model = "openai_compatible" + tp = 1 + tasks = "mmmu_val" + batch_size = 2 + log_suffix = "openai_compatible" + os.makedirs(output_path, exist_ok=True) + + # -------- compose --model_args -------- + model_args = f'model_version="{model_version}",' f"tp={tp}" + + # -------- build command list -------- + cmd = [ + "python3", + "-m", + "lmms_eval", + "--model", + model, + "--model_args", + model_args, + "--tasks", + tasks, + "--batch_size", + str(batch_size), + "--log_samples", + "--log_samples_suffix", + log_suffix, + "--output_path", + str(output_path), + ] + + subprocess.run( + cmd, + check=True, + timeout=3600, + ) + + def _run_vlm_mmmu_test( + self, + model, + output_path, + test_name="", + custom_env=None, + log_level="info", + capture_output=False, + ): + """ + Common method to run VLM MMMU benchmark test. + + Args: + model: Model to test + output_path: Path for output logs + test_name: Optional test name for logging + custom_env: Optional custom environment variables + log_level: Log level for server (default: "info") + capture_output: Whether to capture server stdout/stderr + """ + print(f"\nTesting model: {model.model}{test_name}") + + process = None + mmmu_accuracy = 0 # Initialize to handle potential exceptions + server_output = "" + + try: + # Prepare environment variables + process_env = os.environ.copy() + if custom_env: + process_env.update(custom_env) + + # Prepare stdout/stderr redirection if needed + stdout_file = None + stderr_file = None + if capture_output: + stdout_file = open("/tmp/server_stdout.log", "w") + stderr_file = open("/tmp/server_stderr.log", "w") + + # Launch server for testing + process = popen_launch_server( + model.model, + base_url=self.base_url, + timeout=self.time_out, + api_key=self.api_key, + other_args=[ + "--trust-remote-code", + "--cuda-graph-max-bs", + "32", + "--enable-multimodal", + "--mem-fraction-static", + str(self.parsed_args.mem_fraction_static), # Use class variable + "--log-level", + log_level, + ], + env=process_env, + return_stdout_stderr=( + (stdout_file, stderr_file) if capture_output else None + ), + ) + + # Run evaluation + self.run_mmmu_eval(model.model, output_path) + + # Get the result file + result_file_path = glob.glob(f"{output_path}/*.json")[0] + + with open(result_file_path, "r") as f: + result = json.load(f) + print(f"Result{test_name}\n: {result}") + + # Process the result + mmmu_accuracy = result["results"]["mmmu_val"]["mmmu_acc,none"] + print( + f"Model {model.model} achieved accuracy{test_name}: {mmmu_accuracy:.4f}" + ) + + # Capture server output if requested + if capture_output and process: + server_output = self._read_output_from_files() + + # Assert performance meets expected threshold + self.assertGreaterEqual( + mmmu_accuracy, + model.mmmu_accuracy, + f"Model {model.model} accuracy ({mmmu_accuracy:.4f}) below expected threshold ({model.mmmu_accuracy:.4f}){test_name}", + ) + + return server_output + + except Exception as e: + print(f"Error testing {model.model}{test_name}: {e}") + self.fail(f"Test failed for {model.model}{test_name}: {e}") + + finally: + # Ensure process cleanup happens regardless of success/failure + if process is not None and process.poll() is None: + print(f"Cleaning up process {process.pid}") + try: + kill_process_tree(process.pid) + except Exception as e: + print(f"Error killing process: {e}") + + # clean up temporary files + if capture_output: + if stdout_file: + stdout_file.close() + if stderr_file: + stderr_file.close() + for filename in ["/tmp/server_stdout.log", "/tmp/server_stderr.log"]: + try: + if os.path.exists(filename): + os.remove(filename) + except Exception as e: + print(f"Error removing {filename}: {e}") + + def _read_output_from_files(self): + output_lines = [] + + log_files = [ + ("/tmp/server_stdout.log", "[STDOUT]"), + ("/tmp/server_stderr.log", "[STDERR]"), + ] + for filename, tag in log_files: + try: + if os.path.exists(filename): + with open(filename, "r") as f: + for line in f: + output_lines.append(f"{tag} {line.rstrip()}") + except Exception as e: + print(f"Error reading {tag.lower()} file: {e}") + + return "\n".join(output_lines) + + def test_vlm_mmmu_benchmark(self): + """Test VLM models against MMMU benchmark.""" + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model in models_to_test: + self._run_vlm_mmmu_test(model, "./logs") + + def test_vlm_mmmu_benchmark_with_small_cache(self): + """Test VLM models against MMMU benchmark with a small embedding cache to force eviction.""" + models_to_test = MODELS + + if is_in_ci(): + models_to_test = [random.choice(MODELS)] + + for model in models_to_test: + custom_env = {"SGLANG_VLM_CACHE_SIZE_MB": "5"} + + # Run the test with output capture + server_output = self._run_vlm_mmmu_test( + model, + "./logs_small_cache", + test_name=" with small embedding cache (evict test)", + custom_env=custom_env, + log_level="debug", # Enable debug logging for eviction detection + capture_output=True, # Capture server output + ) + + # Print server output for debugging + print("Server output:\n", server_output) + + # Analyze server output for eviction events + eviction_detected, eviction_count = self._detect_eviction_in_logs( + server_output + ) + + # Assert that eviction was detected (since we're using small cache) + self.assertTrue( + eviction_detected, + f"Expected eviction events to be detected with small cache (5MB), but none found. " + f"Cache size may be too large for the workload or eviction logic may not be working. " + f"Total log content length: {len(server_output)} characters", + ) + + print( + f"Eviction detection summary: {eviction_count} eviction events detected" + ) + + # Additional assertion: if eviction was detected, the test passed + if eviction_detected: + print("✅ Eviction logic successfully triggered and detected!") + + +if __name__ == "__main__": + # Define and parse arguments here, before unittest.main + parser = argparse.ArgumentParser(description="Test VLM models") + parser.add_argument( + "--mem-fraction-static", + type=float, + help="Static memory fraction for the model", + default=0.8, + ) + + # Parse args intended for unittest + args = parser.parse_args() + + # Store the parsed args object on the class + TestVLMModels.parsed_args = args + + # Pass args to unittest + unittest.main(argv=[sys.argv[0]]) diff --git a/test/srt/openai_server/__init__.py b/test/srt/openai_server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/srt/openai_server/basic/__init__.py b/test/srt/openai_server/basic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/srt/openai_server/basic/test_openai_embedding.py b/test/srt/openai_server/basic/test_openai_embedding.py new file mode 100644 index 000000000..60eb8f764 --- /dev/null +++ b/test/srt/openai_server/basic/test_openai_embedding.py @@ -0,0 +1,97 @@ +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIEmbedding(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Configure embedding-specific args + other_args = ["--is-embedding", "--enable-metrics"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_embedding_single(self): + """Test single embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create(model=self.model, input="Hello world") + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_embedding_batch(self): + """Test batch embedding request""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, input=["Hello world", "Test text"] + ) + self.assertEqual(len(response.data), 2) + self.assertTrue(len(response.data[0].embedding) > 0) + self.assertTrue(len(response.data[1].embedding) > 0) + + def test_embedding_single_batch_str(self): + """Test embedding with a List[str] and length equals to 1""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create(model=self.model, input=["Hello world"]) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_embedding_single_int_list(self): + """Test embedding with a List[int] or List[List[int]]]""" + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, + input=[[15339, 314, 703, 284, 612, 262, 10658, 10188, 286, 2061]], + ) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.embeddings.create( + model=self.model, + input=[15339, 314, 703, 284, 612, 262, 10658, 10188, 286, 2061], + ) + self.assertEqual(len(response.data), 1) + self.assertTrue(len(response.data[0].embedding) > 0) + + def test_empty_string_embedding(self): + """Test embedding an empty string.""" + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Text embedding example with empty string + text = "" + # Expect a BadRequestError for empty input + with self.assertRaises(openai.BadRequestError) as cm: + client.embeddings.create( + model=self.model, + input=text, + ) + # check the status code + self.assertEqual(cm.exception.status_code, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/basic/test_openai_server.py b/test/srt/openai_server/basic/test_openai_server.py new file mode 100644 index 000000000..f42039bff --- /dev/null +++ b/test/srt/openai_server/basic/test_openai_server.py @@ -0,0 +1,669 @@ +""" +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_completion +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_completion_stream +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_chat_completion +python3 -m unittest openai_server.basic.test_openai_server.TestOpenAIServer.test_chat_completion_stream +""" + +import json +import re +import unittest + +import numpy as np +import openai +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.runners import TEST_RERANK_QUERY_DOCS +from sglang.test.test_utils import ( + DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_completion( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + response = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + n=parallel_sample_num, + ) + + assert len(response.choices) == num_choices * parallel_sample_num + + if echo: + text = response.choices[0].text + assert text.startswith(prompt) + + if logprobs: + assert response.choices[0].logprobs + assert isinstance(response.choices[0].logprobs.tokens[0], str) + assert isinstance(response.choices[0].logprobs.top_logprobs[1], dict) + ret_num_top_logprobs = len(response.choices[0].logprobs.top_logprobs[1]) + + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0 + + # when echo=True and request.logprobs>0, logprob_start_len is 0, so the first token's logprob would be None. + if not echo: + assert response.choices[0].logprobs.token_logprobs[0] + + assert response.id + assert response.created + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_completion_stream( + self, echo, logprobs, use_list_input, parallel_sample_num, token_input + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + if token_input: + prompt_input = self.tokenizer.encode(prompt) + num_prompt_tokens = len(prompt_input) + else: + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + generator = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + echo=echo, + logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" + assert usage.completion_tokens > 0, f"usage.completion_tokens was zero" + assert usage.total_tokens > 0, f"usage.total_tokens was zero" + continue + + index = response.choices[0].index + is_first = is_firsts.get(index, True) + + if logprobs: + assert response.choices[0].logprobs, f"no logprobs in response" + assert isinstance( + response.choices[0].logprobs.tokens[0], str + ), f"{response.choices[0].logprobs.tokens[0]} is not a string" + if not (is_first and echo): + assert isinstance( + response.choices[0].logprobs.top_logprobs[0], dict + ), f"top_logprobs was not a dictionary" + ret_num_top_logprobs = len( + response.choices[0].logprobs.top_logprobs[0] + ) + # FIXME: Sometimes, some top_logprobs are missing in the return value. The reason is that some output id maps to the same output token and duplicate in the map + # assert ret_num_top_logprobs == logprobs, f"{ret_num_top_logprobs} vs {logprobs}" + assert ret_num_top_logprobs > 0, f"ret_num_top_logprobs was 0" + + if is_first: + if echo: + assert response.choices[0].text.startswith( + prompt + ), f"{response.choices[0].text} and all args {echo} {logprobs} {token_input} {is_first}" + is_firsts[index] = False + assert response.id, f"no id in response" + assert response.created, f"no created in response" + + for index in [i for i in range(parallel_sample_num * num_choices)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def run_chat_completion(self, logprobs, parallel_sample_num): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + n=parallel_sample_num, + ) + + if logprobs: + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ) + + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response.choices) == parallel_sample_num + assert response.choices[0].message.role == "assistant" + assert isinstance(response.choices[0].message.content, str) + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + logprobs=logprobs is not None and logprobs > 0, + top_logprobs=logprobs, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + ) + + is_firsts = {} + is_finished = {} + finish_reason_counts = {} + for response in generator: + usage = response.usage + if usage is not None: + assert usage.prompt_tokens > 0, f"usage.prompt_tokens was zero" + assert usage.completion_tokens > 0, f"usage.completion_tokens was zero" + assert usage.total_tokens > 0, f"usage.total_tokens was zero" + continue + + index = response.choices[0].index + finish_reason = response.choices[0].finish_reason + if finish_reason is not None: + is_finished[index] = True + finish_reason_counts[index] = finish_reason_counts.get(index, 0) + 1 + + data = response.choices[0].delta + + if is_firsts.get(index, True): + assert ( + data.role == "assistant" + ), f"data.role was not 'assistant' for first chunk" + is_firsts[index] = False + continue + + if logprobs and not is_finished.get(index, False): + assert response.choices[0].logprobs, f"logprobs was not returned" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs[0].token, str + ), f"top_logprobs token was not a string" + assert isinstance( + response.choices[0].logprobs.content[0].top_logprobs, list + ), f"top_logprobs was not a list" + ret_num_top_logprobs = len( + response.choices[0].logprobs.content[0].top_logprobs + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert ( + isinstance(data.content, str) + or isinstance(data.reasoning_content, str) + or (isinstance(data.tool_calls, list) and len(data.tool_calls) > 0) + or response.choices[0].finish_reason + ) + assert response.id + assert response.created + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + # Verify that each choice gets exactly one finish_reason chunk + for index in range(parallel_sample_num): + assert ( + index in finish_reason_counts + ), f"No finish_reason found for index {index}" + assert ( + finish_reason_counts[index] == 1 + ), f"Expected 1 finish_reason chunk for index {index}, got {finish_reason_counts[index]}" + + def test_completion(self): + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_completion_stream(self): + # parallel sampling and list input are not supported in streaming mode + for echo in [False, True]: + for logprobs in [None, 5]: + for use_list_input in [True, False]: + for parallel_sample_num in [1, 2]: + for token_input in [False, True]: + self.run_completion_stream( + echo, + logprobs, + use_list_input, + parallel_sample_num, + token_input, + ) + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + def test_regex(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + regex = ( + r"""\{\n""" + + r""" "name": "[\w]+",\n""" + + r""" "population": [\d]+\n""" + + r"""\}""" + ) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=128, + extra_body={"regex": regex}, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + assert isinstance(js_obj["name"], str) + assert isinstance(js_obj["population"], int) + + def test_penalty(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "Introduce the capital of France."}, + ], + temperature=0, + max_tokens=32, + frequency_penalty=1.0, + ) + text = response.choices[0].message.content + assert isinstance(text, str) + + def test_response_prefill(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": """ +Extract the name, size, price, and color from this product description as a JSON object: + + +The SmartHome Mini is a compact smart home assistant available in black or white for only $49.99. At just 5 inches wide, it lets you control lights, thermostats, and other connected devices via voice or app—no matter where you place it in your home. This affordable little hub brings convenient hands-free control to your smart devices. + +""", + }, + { + "role": "assistant", + "content": "{\n", + }, + ], + temperature=0, + extra_body={"continue_final_message": True}, + ) + + assert ( + response.choices[0] + .message.content.strip() + .startswith('"name": "SmartHome Mini",') + ) + + def test_model_list(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + models = list(client.models.list()) + assert len(models) == 1 + assert isinstance(getattr(models[0], "max_model_len", None), int) + + def test_retrieve_model(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + # Test retrieving an existing model + retrieved_model = client.models.retrieve(self.model) + self.assertEqual(retrieved_model.id, self.model) + self.assertEqual(retrieved_model.root, self.model) + + # Test retrieving a non-existent model + with self.assertRaises(openai.NotFoundError): + client.models.retrieve("non-existent-model") + + +class TestOpenAIV1Rerank(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_CROSS_ENCODER_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.score_tolerance = 1e-2 + + # Configure embedding-specific args + other_args = [ + "--is-embedding", + "--enable-metrics", + "--disable-radix-cache", + "--chunked-prefill-size", + "-1", + "--attention-backend", + "torch_native", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1/rerank" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_rerank(self, query, docs): + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={"query": query, "documents": docs}, + ) + + return response.json() + + def test_rerank_single(self): + """Test single rerank request""" + query = TEST_RERANK_QUERY_DOCS[0]["query"] + docs = TEST_RERANK_QUERY_DOCS[0]["documents"] + + response = self.run_rerank(query, docs) + + self.assertEqual(len(response), 1) + self.assertTrue(isinstance(response[0]["score"], float)) + self.assertTrue(isinstance(response[0]["document"], str)) + self.assertTrue(isinstance(response[0]["index"], int)) + + def test_rerank_batch(self): + """Test batch rerank request""" + query = TEST_RERANK_QUERY_DOCS[1]["query"] + docs = TEST_RERANK_QUERY_DOCS[1]["documents"] + + response = self.run_rerank(query, docs) + + self.assertEqual(len(response), 2) + self.assertTrue(isinstance(response[0]["score"], float)) + self.assertTrue(isinstance(response[1]["score"], float)) + self.assertTrue(isinstance(response[0]["document"], str)) + self.assertTrue(isinstance(response[1]["document"], str)) + self.assertTrue(isinstance(response[0]["index"], int)) + self.assertTrue(isinstance(response[1]["index"], int)) + + +class TestOpenAIV1Score(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1/score" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_score( + self, query, items, label_token_ids, apply_softmax=False, item_first=False + ): + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": self.model, + "query": query, + "items": items, + "label_token_ids": label_token_ids, + "apply_softmax": apply_softmax, + "item_first": item_first, + }, + ) + return response.json() + + def test_score_text_input(self): + """Test scoring with text input""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Get valid token IDs from the tokenizer + label_token_ids = [] + for item in items: + token_ids = self.tokenizer.encode(item, add_special_tokens=False) + if not token_ids: + self.fail(f"Failed to encode item: {item}") + label_token_ids.append(token_ids[0]) + + response = self.run_score(query, items, label_token_ids, apply_softmax=True) + + # Handle error responses + if response.get("type") == "BadRequestError": + self.fail(f"Score request failed with error: {response['message']}") + + # Verify response structure + self.assertIn("scores", response, "Response should have a 'scores' field") + self.assertIsInstance(response["scores"], list, "scores should be a list") + self.assertEqual( + len(response["scores"]), + len(items), + "Number of scores should match number of items", + ) + + # Each score should be a list of floats in the order of label_token_ids + for i, score_list in enumerate(response["scores"]): + self.assertIsInstance(score_list, list, f"Score {i} should be a list") + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score {i} length should match label_token_ids", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + f"Score {i} values should be floats", + ) + self.assertAlmostEqual( + sum(score_list), + 1.0, + places=6, + msg=f"Score {i} probabilities should sum to 1", + ) + + def test_score_token_input(self): + """Test scoring with token IDs input""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Get valid token IDs + query_ids = self.tokenizer.encode(query, add_special_tokens=False) + item_ids = [ + self.tokenizer.encode(item, add_special_tokens=False) for item in items + ] + label_token_ids = [ + ids[0] for ids in item_ids if ids + ] # Get first token ID of each item + + response = self.run_score( + query_ids, item_ids, label_token_ids, apply_softmax=True + ) + + # Handle error responses + if response.get("type") == "BadRequestError": + self.fail(f"Score request failed with error: {response['message']}") + + # Verify response structure + self.assertIn("scores", response, "Response should have a 'scores' field") + self.assertIsInstance(response["scores"], list, "scores should be a list") + self.assertEqual( + len(response["scores"]), + len(items), + "Number of scores should match number of items", + ) + + # Each score should be a list of floats in the order of label_token_ids + for i, score_list in enumerate(response["scores"]): + self.assertIsInstance(score_list, list, f"Score {i} should be a list") + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score {i} length should match label_token_ids", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + f"Score {i} values should be floats", + ) + self.assertAlmostEqual( + sum(score_list), + 1.0, + places=6, + msg=f"Score {i} probabilities should sum to 1", + ) + + def test_score_error_handling(self): + """Test error handling for invalid inputs""" + query = "The capital of France is" + items = ["Paris", "London", "Berlin"] + + # Test with invalid token ID + response = requests.post( + self.base_url, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + json={ + "model": self.model, + "query": query, + "items": items, + "label_token_ids": [999999], # Invalid token ID + "apply_softmax": True, + }, + ) + self.assertEqual(response.status_code, 400) + error_response = response.json() + self.assertEqual(error_response["type"], "BadRequestError") + self.assertIn("Token ID 999999 is out of vocabulary", error_response["message"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/basic/test_protocol.py b/test/srt/openai_server/basic/test_protocol.py new file mode 100644 index 000000000..fcaa9770b --- /dev/null +++ b/test/srt/openai_server/basic/test_protocol.py @@ -0,0 +1,368 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for OpenAI API protocol models""" + +import json +import time +import unittest +from typing import Dict, List, Optional + +from pydantic import BaseModel, Field, ValidationError + +from sglang.srt.entrypoints.openai.protocol import ( + BatchRequest, + BatchResponse, + ChatCompletionMessageContentImagePart, + ChatCompletionMessageContentTextPart, + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + ChatCompletionTokenLogprob, + ChatMessage, + ChoiceLogprobs, + CompletionRequest, + CompletionResponse, + CompletionResponseChoice, + DeltaMessage, + EmbeddingObject, + EmbeddingRequest, + EmbeddingResponse, + ErrorResponse, + FileDeleteResponse, + FileRequest, + FileResponse, + Function, + FunctionResponse, + JsonSchemaResponseFormat, + LogProbs, + ModelCard, + ModelList, + MultimodalEmbeddingInput, + ResponseFormat, + ScoringRequest, + ScoringResponse, + StreamOptions, + StructuralTagResponseFormat, + Tool, + ToolCall, + ToolChoice, + TopLogprob, + UsageInfo, +) + + +class TestModelCard(unittest.TestCase): + """Test ModelCard protocol model""" + + def test_model_card_serialization(self): + """Test model card JSON serialization""" + card = ModelCard(id="test-model", max_model_len=4096) + data = card.model_dump() + self.assertEqual(data["id"], "test-model") + self.assertEqual(data["object"], "model") + self.assertEqual(data["max_model_len"], 4096) + + +class TestModelList(unittest.TestCase): + """Test ModelList protocol model""" + + def test_empty_model_list(self): + """Test empty model list creation""" + model_list = ModelList() + self.assertEqual(model_list.object, "list") + self.assertEqual(len(model_list.data), 0) + + def test_model_list_with_cards(self): + """Test model list with model cards""" + cards = [ + ModelCard(id="model-1"), + ModelCard(id="model-2", max_model_len=2048), + ] + model_list = ModelList(data=cards) + self.assertEqual(len(model_list.data), 2) + self.assertEqual(model_list.data[0].id, "model-1") + self.assertEqual(model_list.data[1].id, "model-2") + + +class TestCompletionRequest(unittest.TestCase): + """Test CompletionRequest protocol model""" + + def test_basic_completion_request(self): + """Test basic completion request""" + request = CompletionRequest(model="test-model", prompt="Hello world") + self.assertEqual(request.model, "test-model") + self.assertEqual(request.prompt, "Hello world") + self.assertEqual(request.max_tokens, 16) # default + self.assertEqual(request.temperature, 1.0) # default + self.assertEqual(request.n, 1) # default + self.assertFalse(request.stream) # default + self.assertFalse(request.echo) # default + + def test_completion_request_sglang_extensions(self): + """Test completion request with SGLang-specific extensions""" + request = CompletionRequest( + model="test-model", + prompt="Hello", + top_k=50, + min_p=0.1, + repetition_penalty=1.1, + regex=r"\d+", + json_schema='{"type": "object"}', + lora_path="/path/to/lora", + ) + self.assertEqual(request.top_k, 50) + self.assertEqual(request.min_p, 0.1) + self.assertEqual(request.repetition_penalty, 1.1) + self.assertEqual(request.regex, r"\d+") + self.assertEqual(request.json_schema, '{"type": "object"}') + self.assertEqual(request.lora_path, "/path/to/lora") + + def test_completion_request_validation_errors(self): + """Test completion request validation errors""" + with self.assertRaises(ValidationError): + CompletionRequest() # missing required fields + + with self.assertRaises(ValidationError): + CompletionRequest(model="test-model") # missing prompt + + +class TestChatCompletionRequest(unittest.TestCase): + """Test ChatCompletionRequest protocol model""" + + def test_basic_chat_completion_request(self): + """Test basic chat completion request""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest(model="test-model", messages=messages) + self.assertEqual(request.model, "test-model") + self.assertEqual(len(request.messages), 1) + self.assertEqual(request.messages[0].role, "user") + self.assertEqual(request.messages[0].content, "Hello") + self.assertEqual(request.temperature, 0.7) # default + self.assertFalse(request.stream) # default + self.assertEqual(request.tool_choice, "none") # default when no tools + + def test_chat_completion_tool_choice_validation(self): + """Test tool choice validation logic""" + messages = [{"role": "user", "content": "Hello"}] + + # No tools, tool_choice should default to "none" + request1 = ChatCompletionRequest(model="test-model", messages=messages) + self.assertEqual(request1.tool_choice, "none") + + # With tools, tool_choice should default to "auto" + tools = [ + { + "type": "function", + "function": {"name": "test_func", "description": "Test function"}, + } + ] + request2 = ChatCompletionRequest( + model="test-model", messages=messages, tools=tools + ) + self.assertEqual(request2.tool_choice, "auto") + + def test_chat_completion_sglang_extensions(self): + """Test chat completion with SGLang extensions""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + ) + self.assertEqual(request.top_k, 40) + self.assertEqual(request.min_p, 0.05) + self.assertFalse(request.separate_reasoning) + self.assertFalse(request.stream_reasoning) + self.assertEqual(request.chat_template_kwargs, {"custom_param": "value"}) + + def test_chat_completion_reasoning_effort(self): + """Test chat completion with reasoning effort""" + messages = [{"role": "user", "content": "Hello"}] + request = ChatCompletionRequest( + model="test-model", + messages=messages, + reasoning={ + "enabled": True, + "reasoning_effort": "high", + }, + ) + self.assertEqual(request.reasoning_effort, "high") + self.assertEqual(request.chat_template_kwargs, {"thinking": True}) + + def test_chat_completion_json_format(self): + """Test chat completion json format""" + transcript = "Good morning! It's 7:00 AM, and I'm just waking up. Today is going to be a busy day, " + "so let's get started. First, I need to make a quick breakfast. I think I'll have some " + "scrambled eggs and toast with a cup of coffee. While I'm cooking, I'll also check my " + "emails to see if there's anything urgent." + + messages = [ + { + "role": "system", + "content": "The following is a voice message transcript. Only answer in JSON.", + }, + { + "role": "user", + "content": transcript, + }, + ] + + class VoiceNote(BaseModel): + title: str = Field(description="A title for the voice note") + summary: str = Field( + description="A short one sentence summary of the voice note." + ) + strict: Optional[bool] = True + actionItems: List[str] = Field( + description="A list of action items from the voice note" + ) + + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + response_format={ + "type": "json_schema", + "schema": VoiceNote.model_json_schema(), + }, + ) + res_format = request.response_format + json_format = res_format.json_schema + name = json_format.name + schema = json_format.schema_ + strict = json_format.strict + self.assertEqual(name, "VoiceNote") + self.assertEqual(strict, True) + self.assertNotIn("strict", schema["properties"]) + + request = ChatCompletionRequest( + model="test-model", + messages=messages, + top_k=40, + min_p=0.05, + separate_reasoning=False, + stream_reasoning=False, + chat_template_kwargs={"custom_param": "value"}, + response_format={ + "type": "json_schema", + "json_schema": { + "name": "VoiceNote", + "schema": VoiceNote.model_json_schema(), + "strict": True, + }, + }, + ) + res_format = request.response_format + json_format = res_format.json_schema + name = json_format.name + schema = json_format.schema_ + strict = json_format.strict + self.assertEqual(name, "VoiceNote") + self.assertEqual(strict, True) + + +class TestModelSerialization(unittest.TestCase): + """Test model serialization with hidden states""" + + def test_hidden_states_excluded_when_none(self): + """Test that None hidden_states are excluded with exclude_none=True""" + choice = ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", content="Hello"), + finish_reason="stop", + hidden_states=None, + ) + + response = ChatCompletionResponse( + id="test-id", + model="test-model", + choices=[choice], + usage=UsageInfo(prompt_tokens=5, completion_tokens=1, total_tokens=6), + ) + + # Test exclude_none serialization (should exclude None hidden_states) + data = response.model_dump(exclude_none=True) + self.assertNotIn("hidden_states", data["choices"][0]) + + def test_hidden_states_included_when_not_none(self): + """Test that non-None hidden_states are included""" + choice = ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", content="Hello"), + finish_reason="stop", + hidden_states=[0.1, 0.2, 0.3], + ) + + response = ChatCompletionResponse( + id="test-id", + model="test-model", + choices=[choice], + usage=UsageInfo(prompt_tokens=5, completion_tokens=1, total_tokens=6), + ) + + # Test exclude_none serialization (should include non-None hidden_states) + data = response.model_dump(exclude_none=True) + self.assertIn("hidden_states", data["choices"][0]) + self.assertEqual(data["choices"][0]["hidden_states"], [0.1, 0.2, 0.3]) + + +class TestValidationEdgeCases(unittest.TestCase): + """Test edge cases and validation scenarios""" + + def test_invalid_tool_choice_type(self): + """Test invalid tool choice type""" + messages = [{"role": "user", "content": "Hello"}] + with self.assertRaises(ValidationError): + ChatCompletionRequest( + model="test-model", messages=messages, tool_choice=123 + ) + + def test_negative_token_limits(self): + """Test negative token limits""" + with self.assertRaises(ValidationError): + CompletionRequest(model="test-model", prompt="Hello", max_tokens=-1) + + def test_model_serialization_roundtrip(self): + """Test that models can be serialized and deserialized""" + original_request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Hello"}], + temperature=0.7, + max_tokens=100, + ) + + # Serialize to dict + data = original_request.model_dump() + + # Deserialize back + restored_request = ChatCompletionRequest(**data) + + self.assertEqual(restored_request.model, original_request.model) + self.assertEqual(restored_request.temperature, original_request.temperature) + self.assertEqual(restored_request.max_tokens, original_request.max_tokens) + self.assertEqual(len(restored_request.messages), len(original_request.messages)) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/openai_server/basic/test_serving_chat.py b/test/srt/openai_server/basic/test_serving_chat.py new file mode 100644 index 000000000..9f0d48004 --- /dev/null +++ b/test/srt/openai_server/basic/test_serving_chat.py @@ -0,0 +1,425 @@ +""" +Unit-tests for OpenAIServingChat — rewritten to use only the std-lib 'unittest'. +Run with either: + python tests/test_serving_chat_unit.py -v +or + python -m unittest discover -s tests -p "test_*unit.py" -v +""" + +import asyncio +import json +import unittest +import uuid +from typing import Optional +from unittest.mock import Mock, patch + +from fastapi import Request + +from sglang.srt.entrypoints.openai.protocol import ( + ChatCompletionRequest, + MessageProcessingResult, +) +from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat +from sglang.srt.managers.io_struct import GenerateReqInput + + +class _MockTokenizerManager: + """Minimal mock that satisfies OpenAIServingChat.""" + + def __init__(self): + self.model_config = Mock(is_multimodal=False) + self.server_args = Mock( + enable_cache_report=False, + tool_call_parser="hermes", + reasoning_parser=None, + ) + self.chat_template_name: Optional[str] = "llama-3" + + # tokenizer stub + self.tokenizer = Mock() + self.tokenizer.encode.return_value = [1, 2, 3, 4, 5] + self.tokenizer.decode.return_value = "Test response" + self.tokenizer.chat_template = None + self.tokenizer.bos_token_id = 1 + + # async generator stub for generate_request + async def _mock_generate(): + yield { + "text": "Test response", + "meta_info": { + "id": f"chatcmpl-{uuid.uuid4()}", + "prompt_tokens": 10, + "completion_tokens": 5, + "cached_tokens": 0, + "finish_reason": {"type": "stop", "matched": None}, + "output_token_logprobs": [(0.1, 1, "Test"), (0.2, 2, "response")], + "output_top_logprobs": None, + }, + "index": 0, + } + + self.generate_request = Mock(return_value=_mock_generate()) + self.create_abort_task = Mock() + + +class _MockTemplateManager: + """Minimal mock for TemplateManager.""" + + def __init__(self): + self.chat_template_name: Optional[str] = "llama-3" + self.jinja_template_content_format: Optional[str] = None + self.completion_template_name: Optional[str] = None + + +class ServingChatTestCase(unittest.TestCase): + # ------------- common fixtures ------------- + def setUp(self): + self.tm = _MockTokenizerManager() + self.template_manager = _MockTemplateManager() + self.chat = OpenAIServingChat(self.tm, self.template_manager) + + # frequently reused requests + self.basic_req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + temperature=0.7, + max_tokens=100, + stream=False, + ) + self.stream_req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + temperature=0.7, + max_tokens=100, + stream=True, + ) + + self.fastapi_request = Mock(spec=Request) + self.fastapi_request.headers = {} + + # ------------- conversion tests ------------- + def test_convert_to_internal_request_single(self): + with patch( + "sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv" + ) as conv_mock, patch.object(self.chat, "_process_messages") as proc_mock: + conv_ins = Mock() + conv_ins.get_prompt.return_value = "Test prompt" + conv_ins.image_data = conv_ins.audio_data = None + conv_ins.modalities = [] + conv_ins.stop_str = [""] + conv_mock.return_value = conv_ins + + proc_mock.return_value = MessageProcessingResult( + "Test prompt", + [1, 2, 3], + None, + None, + [], + [""], + None, + ) + + adapted, processed = self.chat._convert_to_internal_request(self.basic_req) + self.assertIsInstance(adapted, GenerateReqInput) + self.assertFalse(adapted.stream) + self.assertEqual(processed, self.basic_req) + + def test_stop_str_isolation_between_requests(self): + """Test that stop strings from one request don't affect subsequent requests. + + This tests the fix for the bug where conv.stop_str was being mutated globally, + causing stop strings from one request to persist in subsequent requests. + """ + # Mock conversation template with initial stop_str + initial_stop_str = ["\n"] + + with patch( + "sglang.srt.entrypoints.openai.serving_chat.generate_chat_conv" + ) as conv_mock: + # Create a mock conversation object that will be returned by generate_chat_conv + conv_ins = Mock() + conv_ins.get_prompt.return_value = "Test prompt" + conv_ins.image_data = None + conv_ins.audio_data = None + conv_ins.modalities = [] + conv_ins.stop_str = ( + initial_stop_str.copy() + ) # Template's default stop strings + conv_mock.return_value = conv_ins + + # First request with additional stop string + req1 = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "First request"}], + stop=["CUSTOM_STOP"], + ) + + # Call the actual _apply_conversation_template method (not mocked) + result1 = self.chat._apply_conversation_template(req1, is_multimodal=False) + + # Verify first request has both stop strings + expected_stop1 = initial_stop_str + ["CUSTOM_STOP"] + self.assertEqual(result1.stop, expected_stop1) + + # Verify the original template's stop_str wasn't mutated after first request + self.assertEqual(conv_ins.stop_str, initial_stop_str) + + # Second request without additional stop string + req2 = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Second request"}], + # No custom stop strings + ) + result2 = self.chat._apply_conversation_template(req2, is_multimodal=False) + + # Verify second request only has original stop strings (no CUSTOM_STOP from req1) + self.assertEqual(result2.stop, initial_stop_str) + self.assertNotIn("CUSTOM_STOP", result2.stop) + self.assertEqual(conv_ins.stop_str, initial_stop_str) + + # ------------- sampling-params ------------- + def test_sampling_param_build(self): + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi"}], + temperature=0.8, + max_tokens=150, + min_tokens=5, + top_p=0.9, + stop=[""], + ) + with patch.object( + self.chat, + "_process_messages", + return_value=("Prompt", [1], None, None, [], [""], None), + ): + params = self.chat._build_sampling_params(req, [""], None) + self.assertEqual(params["temperature"], 0.8) + self.assertEqual(params["max_new_tokens"], 150) + self.assertEqual(params["min_new_tokens"], 5) + self.assertEqual(params["stop"], [""]) + + async def test_unstreamed_tool_args_completion(self): + """Test that remaining tool call arguments are sent when generation finishes.""" + + # Mock FunctionCallParser with detector that has partial tool call data + mock_parser = Mock() + mock_detector = Mock() + + # Simulate a tool call that was partially streamed + mock_detector.prev_tool_call_arr = [ + { + "name": "get_weather", + "arguments": {"location": "San Francisco", "unit": "celsius"}, + } + ] + mock_detector.streamed_args_for_tool = [ + '{"location": "San Francisco"' # Partial arguments streamed so far + ] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + finish_reason_type="stop", + index=0, + ) + + # Should return a chunk with remaining arguments + self.assertIsNotNone(result, "Should return chunk with remaining arguments") + self.assertIn('"arguments":', result, "Should contain arguments field") + self.assertIn( + ', "unit": "celsius"}', result, "Should contain remaining arguments" + ) + self.assertIn( + '"finish_reason":null', + result, + "Should not include finish_reason in completion chunk", + ) + + async def test_unstreamed_tool_args_no_completion_needed(self): + """Test that no completion chunk is sent when all arguments were already streamed.""" + + # Mock FunctionCallParser with detector that has complete tool call data + mock_parser = Mock() + mock_detector = Mock() + + # Simulate a tool call that was completely streamed + mock_detector.prev_tool_call_arr = [ + {"name": "get_weather", "arguments": {"location": "San Francisco"}} + ] + mock_detector.streamed_args_for_tool = [ + '{"location": "San Francisco"}' # All arguments already streamed + ] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + finish_reason_type="stop", + index=0, + ) + + # Should return None since no completion is needed + self.assertIsNone(result, "Should return None when no completion is needed") + + async def test_unstreamed_tool_args_no_parser_data(self): + """Test that no completion chunk is sent when parser has no tool call data.""" + + # Mock FunctionCallParser with empty detector + mock_parser = Mock() + mock_detector = Mock() + mock_detector.prev_tool_call_arr = [] + mock_detector.streamed_args_for_tool = [] + mock_parser.detector = mock_detector + + content = { + "meta_info": { + "id": "chatcmpl-test123", + } + } + + request = ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "What's the weather?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + ) + + # Test the completion method + result = self.chat._check_for_unstreamed_tool_args( + parser=mock_parser, + content=content, + request=request, + finish_reason_type="stop", + index=0, + ) + + # Should return None since there's no parser data + self.assertIsNone( + result, "Should return None when parser has no tool call data" + ) + + # ------------- kimi_k2 tool_call_id formatting ------------- + def test_kimi_k2_non_streaming_tool_call_id_format(self): + """Ensure non-streaming tool_call.id matches functions.{name}:{index} for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Mock FunctionCallParser.parse_non_stream to return one tool call + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # Build a mock ToolCallItem-like object + call_info = Mock() + call_info.name = "get_weather" + call_info.parameters = '{"city":"Paris"}' + call_info.tool_index = 0 + + parser_instance.has_tool_call.return_value = True + parser_instance.parse_non_stream.return_value = ("", [call_info]) + + finish_reason = {"type": "stop", "matched": None} + tools = [ + {"type": "function", "function": {"name": "get_weather"}}, + ] + + tool_calls, remaining_text, _ = self.chat._process_tool_calls( + text="<|tool_calls_section_begin|>...", + tools=tools, + finish_reason=finish_reason, + ) + + self.assertIsNotNone(tool_calls) + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0].id, "functions.get_weather:0") + self.assertEqual(tool_calls[0].function.name, "get_weather") + + def test_kimi_k2_streaming_tool_call_id_format(self): + """Ensure streaming first chunk tool_call.id matches functions.{name}:{index} for kimi_k2 parser.""" + + # Force kimi_k2 parser + self.chat.tool_call_parser = "kimi_k2" + + # Prepare request with tools + req = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hi?"}], + tools=[{"type": "function", "function": {"name": "get_weather"}}], + stream=True, + ) + + # Patch FunctionCallParser used inside _process_tool_call_stream + with patch( + "sglang.srt.entrypoints.openai.serving_chat.FunctionCallParser" + ) as ParserMock: + parser_instance = ParserMock.return_value + + # First call returns one ToolCallItem-like chunk (with name) + first_chunk_call = Mock() + first_chunk_call.tool_index = 0 + first_chunk_call.name = "get_weather" + first_chunk_call.parameters = "" + parser_instance.parse_stream_chunk.side_effect = [ + ("", [first_chunk_call]), + ("", []), + ] + + async def collect_first_tool_chunk(): + gen = self.chat._process_tool_call_stream( + index=0, + delta="irrelevant", + parser_dict={}, + content={"meta_info": {"id": "chatcmpl-test"}}, + request=req, + has_tool_calls={}, + ) + # Get first yielded SSE line + line = None + async for emitted in gen: + line = emitted + break + return line + + loop = asyncio.get_event_loop() + line = loop.run_until_complete(collect_first_tool_chunk()) + self.assertIsNotNone(line) + self.assertTrue(line.startswith("data: ")) + + payload = json.loads(line[len("data: ") :]) + tool_calls = payload["choices"][0]["delta"]["tool_calls"] + self.assertEqual(tool_calls[0]["id"], "functions.get_weather:0") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/openai_server/basic/test_serving_completions.py b/test/srt/openai_server/basic/test_serving_completions.py new file mode 100644 index 000000000..022ba9ad1 --- /dev/null +++ b/test/srt/openai_server/basic/test_serving_completions.py @@ -0,0 +1,157 @@ +""" +Unit-tests for the refactored completions-serving handler (no pytest). +Run with: + python -m unittest tests.test_serving_completions_unit -v +""" + +import unittest +from typing import Optional +from unittest.mock import AsyncMock, Mock, patch + +from sglang.srt.entrypoints.openai.protocol import CompletionRequest +from sglang.srt.entrypoints.openai.serving_completions import OpenAIServingCompletion +from sglang.srt.managers.tokenizer_manager import TokenizerManager + + +class _MockTemplateManager: + """Minimal mock for TemplateManager.""" + + def __init__(self): + self.chat_template_name: Optional[str] = None + self.jinja_template_content_format: Optional[str] = None + self.completion_template_name: Optional[str] = ( + None # Set to None to avoid template processing + ) + + +class ServingCompletionTestCase(unittest.TestCase): + """Bundle all prompt/echo tests in one TestCase.""" + + # ---------- shared test fixtures ---------- + def setUp(self): + # build the mock TokenizerManager once for every test + tm = Mock(spec=TokenizerManager) + + tm.tokenizer = Mock() + tm.tokenizer.encode.return_value = [1, 2, 3, 4] + tm.tokenizer.decode.return_value = "decoded text" + tm.tokenizer.bos_token_id = 1 + + tm.model_config = Mock(is_multimodal=False) + tm.server_args = Mock(enable_cache_report=False) + + tm.generate_request = AsyncMock() + tm.create_abort_task = Mock() + + self.template_manager = _MockTemplateManager() + self.sc = OpenAIServingCompletion(tm, self.template_manager) + + # ---------- prompt-handling ---------- + def test_single_string_prompt(self): + req = CompletionRequest(model="x", prompt="Hello world", max_tokens=100) + internal, _ = self.sc._convert_to_internal_request(req) + self.assertEqual(internal.text, "Hello world") + + def test_single_token_ids_prompt(self): + req = CompletionRequest(model="x", prompt=[1, 2, 3, 4], max_tokens=100) + internal, _ = self.sc._convert_to_internal_request(req) + self.assertEqual(internal.input_ids, [1, 2, 3, 4]) + + # ---------- echo-handling ---------- + def test_echo_with_string_prompt_streaming(self): + req = CompletionRequest(model="x", prompt="Hello", max_tokens=1, echo=True) + self.assertEqual(self.sc._get_echo_text(req, 0), "Hello") + + def test_echo_with_list_of_strings_streaming(self): + req = CompletionRequest( + model="x", prompt=["A", "B"], max_tokens=1, echo=True, n=1 + ) + self.assertEqual(self.sc._get_echo_text(req, 0), "A") + self.assertEqual(self.sc._get_echo_text(req, 1), "B") + + def test_echo_with_token_ids_streaming(self): + req = CompletionRequest(model="x", prompt=[1, 2, 3], max_tokens=1, echo=True) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded_prompt" + self.assertEqual(self.sc._get_echo_text(req, 0), "decoded_prompt") + + def test_echo_with_multiple_token_ids_streaming(self): + req = CompletionRequest( + model="x", prompt=[[1, 2], [3, 4]], max_tokens=1, echo=True, n=1 + ) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded" + self.assertEqual(self.sc._get_echo_text(req, 0), "decoded") + + def test_prepare_echo_prompts_non_streaming(self): + # single string + req = CompletionRequest(model="x", prompt="Hi", echo=True) + self.assertEqual(self.sc._prepare_echo_prompts(req), ["Hi"]) + + # list of strings + req = CompletionRequest(model="x", prompt=["Hi", "Yo"], echo=True) + self.assertEqual(self.sc._prepare_echo_prompts(req), ["Hi", "Yo"]) + + # token IDs + req = CompletionRequest(model="x", prompt=[1, 2, 3], echo=True) + self.sc.tokenizer_manager.tokenizer.decode.return_value = "decoded" + self.assertEqual(self.sc._prepare_echo_prompts(req), ["decoded"]) + + # ---------- response_format handling ---------- + def test_response_format_json_object(self): + """Test that response_format json_object is correctly processed in sampling params.""" + req = CompletionRequest( + model="x", + prompt="Generate a JSON object:", + max_tokens=100, + response_format={"type": "json_object"}, + ) + sampling_params = self.sc._build_sampling_params(req) + self.assertEqual(sampling_params["json_schema"], '{"type": "object"}') + + def test_response_format_json_schema(self): + """Test that response_format json_schema is correctly processed in sampling params.""" + schema = { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + } + req = CompletionRequest( + model="x", + prompt="Generate a JSON object:", + max_tokens=100, + response_format={ + "type": "json_schema", + "json_schema": {"name": "person", "schema": schema}, + }, + ) + sampling_params = self.sc._build_sampling_params(req) + # The schema should be converted to string by convert_json_schema_to_str + self.assertIn("json_schema", sampling_params) + self.assertIsInstance(sampling_params["json_schema"], str) + + def test_response_format_structural_tag(self): + """Test that response_format structural_tag is correctly processed in sampling params.""" + req = CompletionRequest( + model="x", + prompt="Generate structured output:", + max_tokens=100, + response_format={ + "type": "structural_tag", + "structures": [{"begin": "", "end": ""}], + "triggers": [""], + }, + ) + sampling_params = self.sc._build_sampling_params(req) + # The structural_tag should be processed + self.assertIn("structural_tag", sampling_params) + self.assertIsInstance(sampling_params["structural_tag"], str) + + def test_response_format_none(self): + """Test that no response_format doesn't add extra constraints.""" + req = CompletionRequest(model="x", prompt="Generate text:", max_tokens=100) + sampling_params = self.sc._build_sampling_params(req) + # Should not have json_schema or structural_tag from response_format + # (but might have json_schema from the legacy json_schema field) + self.assertIsNone(sampling_params.get("structural_tag")) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/openai_server/basic/test_serving_embedding.py b/test/srt/openai_server/basic/test_serving_embedding.py new file mode 100644 index 000000000..47f8330b7 --- /dev/null +++ b/test/srt/openai_server/basic/test_serving_embedding.py @@ -0,0 +1,145 @@ +""" +Unit tests for the OpenAIServingEmbedding class from serving_embedding.py. +""" + +import unittest +import uuid +from unittest.mock import Mock + +from fastapi import Request + +from sglang.srt.entrypoints.openai.protocol import ( + EmbeddingRequest, + EmbeddingResponse, + MultimodalEmbeddingInput, +) +from sglang.srt.entrypoints.openai.serving_embedding import OpenAIServingEmbedding +from sglang.srt.managers.io_struct import EmbeddingReqInput + + +# Mock TokenizerManager for embedding tests +class _MockTokenizerManager: + def __init__(self): + self.model_config = Mock() + self.model_config.is_multimodal = False + self.server_args = Mock() + self.server_args.enable_cache_report = False + self.model_path = "test-model" + + # Mock tokenizer + self.tokenizer = Mock() + self.tokenizer.encode = Mock(return_value=[1, 2, 3, 4, 5]) + self.tokenizer.decode = Mock(return_value="Test embedding input") + self.tokenizer.chat_template = None + self.tokenizer.bos_token_id = 1 + + # Mock generate_request method for embeddings + async def mock_generate_embedding(): + yield { + "embedding": [0.1, 0.2, 0.3, 0.4, 0.5] * 20, # 100-dim embedding + "meta_info": { + "id": f"embd-{uuid.uuid4()}", + "prompt_tokens": 5, + }, + } + + self.generate_request = Mock(return_value=mock_generate_embedding()) + + +# Mock TemplateManager for embedding tests +class _MockTemplateManager: + def __init__(self): + self.chat_template_name = None # None for embeddings usually + self.jinja_template_content_format = None + self.completion_template_name = None + + +class ServingEmbeddingTestCase(unittest.TestCase): + def setUp(self): + """Set up test fixtures.""" + self.tokenizer_manager = _MockTokenizerManager() + self.template_manager = _MockTemplateManager() + self.serving_embedding = OpenAIServingEmbedding( + self.tokenizer_manager, self.template_manager + ) + + self.request = Mock(spec=Request) + self.request.headers = {} + + self.basic_req = EmbeddingRequest( + model="test-model", + input="Hello, how are you?", + encoding_format="float", + ) + self.list_req = EmbeddingRequest( + model="test-model", + input=["Hello, how are you?", "I am fine, thank you!"], + encoding_format="float", + ) + self.multimodal_req = EmbeddingRequest( + model="test-model", + input=[ + MultimodalEmbeddingInput(text="Hello", image="base64_image_data"), + MultimodalEmbeddingInput(text="World", image=None), + ], + encoding_format="float", + ) + self.token_ids_req = EmbeddingRequest( + model="test-model", + input=[1, 2, 3, 4, 5], + encoding_format="float", + ) + + def test_convert_single_string_request(self): + """Test converting single string request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.basic_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual(adapted_request.text, "Hello, how are you?") + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.basic_req) + + def test_convert_list_string_request(self): + """Test converting list of strings request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.list_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual( + adapted_request.text, ["Hello, how are you?", "I am fine, thank you!"] + ) + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.list_req) + + def test_convert_token_ids_request(self): + """Test converting token IDs request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.token_ids_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + self.assertEqual(adapted_request.input_ids, [1, 2, 3, 4, 5]) + # self.assertEqual(adapted_request.rid, "test-id") + self.assertEqual(processed_request, self.token_ids_req) + + def test_convert_multimodal_request(self): + """Test converting multimodal request to internal format.""" + adapted_request, processed_request = ( + self.serving_embedding._convert_to_internal_request(self.multimodal_req) + ) + + self.assertIsInstance(adapted_request, EmbeddingReqInput) + # Should extract text and images separately + self.assertEqual(len(adapted_request.text), 2) + self.assertIn("Hello", adapted_request.text) + self.assertIn("World", adapted_request.text) + self.assertEqual(adapted_request.image_data[0], "base64_image_data") + self.assertIsNone(adapted_request.image_data[1]) + # self.assertEqual(adapted_request.rid, "test-id") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/openai_server/features/__init__.py b/test/srt/openai_server/features/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/srt/openai_server/features/test_cache_report.py b/test/srt/openai_server/features/test_cache_report.py new file mode 100644 index 000000000..999111a2e --- /dev/null +++ b/test/srt/openai_server/features/test_cache_report.py @@ -0,0 +1,212 @@ +import asyncio +import unittest + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestCacheReport(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.min_cached = 5 + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=300, + other_args=[ + "--chunked-prefill-size=40", + "--enable-cache-report", + ], + ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + cls.aclient = openai.AsyncClient(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + + usage = cls.run_openai(cls, "1").usage + # we can assume that our request is of size 1, plus the total template size + # ideally we would like to know the begin size / end size of the template to be more precise + total_template_size = usage.prompt_tokens - 1 + print(f"template size: {total_template_size}") + usage2 = cls.run_openai(cls, "2").usage + assert usage2.prompt_tokens_details.cached_tokens <= total_template_size + cls.min_cached = max( + usage2.prompt_tokens_details.cached_tokens, + total_template_size - usage2.prompt_tokens_details.cached_tokens, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, return_logprob=False, top_logprobs_num=0, n=1): + response = requests.post( + self.base_url + "/generate", + # we use an uncommon start to minimise the chance that the cache is hit by chance + json={ + "text": "_ The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "stop_token_ids": [119690], + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + return response + + def run_openai(self, message): + response = self.client.chat.completions.create( + model=self.model, + messages=[ + # {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": message}, + ], + temperature=0, + max_tokens=100, + ) + return response + + async def run_openai_async(self, message): + response = await self.aclient.chat.completions.create( + model=self.model, + messages=[ + {"role": "user", "content": message}, + ], + temperature=0, + max_tokens=100, + ) + return response + + def cache_report_openai(self, message): + response = self.run_openai(message) + print( + f"openai first request cached_tokens: {int(response.usage.prompt_tokens_details.cached_tokens)}" + ) + first_cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + # assert int(response.usage.cached_tokens) == 0 + assert first_cached_tokens <= self.min_cached + response = self.run_openai(message) + cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + print(f"openai second request cached_tokens: {cached_tokens}") + assert cached_tokens > 0 + assert cached_tokens == int(response.usage.prompt_tokens) - 1 + return first_cached_tokens + + async def cache_report_openai_async(self, message): + response = await self.run_openai_async(message) + cached_tokens = int(response.usage.prompt_tokens_details.cached_tokens) + prompt_tokens = int(response.usage.prompt_tokens) + return cached_tokens, prompt_tokens + + def test_generate(self): + print("=" * 100) + response = self.run_decode() + # print(response.json()) + cached_tokens = int(response.json()["meta_info"]["cached_tokens"]) + print(f"sglang first request cached_tokens: {cached_tokens}") + print( + f"sglang first request prompt_tokens: {int(response.json()['meta_info']['prompt_tokens'])}" + ) + # can't assure to be 0: depends on the initialisation request / if a template is used with the model + assert cached_tokens < self.min_cached + response = self.run_decode() + cached_tokens = int(response.json()["meta_info"]["cached_tokens"]) + print(f"sglang second request cached_tokens: {cached_tokens}") + print( + f"sglang second request prompt_tokens: {int(response.json()['meta_info']['prompt_tokens'])}" + ) + assert cached_tokens == int(response.json()["meta_info"]["prompt_tokens"]) - 1 + + def test_cache_split_prefill_openai(self): + print("=" * 100) + self.cache_report_openai( + "€ This is a very long and unique text that should not be already cached, the twist is" + " that it should be longer than the chunked-prefill-size, so it should be split among" + " several prefill requests. Still, it shouldn't be cached" + ) + + def test_cache_report_openai(self): + print("=" * 100) + # warm up the cache, for the template + self.run_openai("Introduce the capital of France.") + + first_cached_tokens_1 = self.run_openai( + "How many sparrow do you need to lift a coconut?" + ).usage.prompt_tokens_details.cached_tokens + + usage_2 = self.run_openai("* sing something about cats").usage + first_cached_tokens_2 = usage_2.prompt_tokens_details.cached_tokens + # first request may not have 0 cached tokens, but if they only have the template in common they + # should be the same once the cache is warmed up + assert first_cached_tokens_1 == first_cached_tokens_2 + + resp = self.run_openai("* sing something about cats and dogs") + print(resp.usage) + + resp = self.run_openai("* sing something about cats, please") + print(resp.usage) + assert ( + resp.usage.prompt_tokens_details.cached_tokens + >= usage_2.prompt_tokens - self.min_cached + ) + + # TODO: flaky test + # def test_cache_report_openai_async(self): + # print("=" * 100) + + # async def run_test(): + # task0 = asyncio.create_task( + # self.cache_report_openai_async( + # "first request, to start the inference and let the next two request be started in the same batch" + # ) + # ) + # await asyncio.sleep(1) # to force the first request to be started first + # task1 = asyncio.create_task( + # self.cache_report_openai_async( + # "> can the same batch parallel request use the cache?" + # ) + # ) + # task2 = asyncio.create_task( + # self.cache_report_openai_async( + # "> can the same batch parallel request use the cache?" + # ) + # ) + # result0, result1, result2 = await asyncio.gather(task0, task1, task2) + + # cached_tokens0, prompt_tokens0 = result0 + # cached_tokens1, prompt_tokens1 = result1 + # cached_tokens2, prompt_tokens2 = result2 + + # print( + # f"Async request 0 - Cached tokens: {cached_tokens0}, Prompt tokens: {prompt_tokens0}" + # ) + # print( + # f"Async request 1 - Cached tokens: {cached_tokens1}, Prompt tokens: {prompt_tokens1}" + # ) + # print( + # f"Async request 2 - Cached tokens: {cached_tokens2}, Prompt tokens: {prompt_tokens2}" + # ) + + # # Assert that no requests used the cache (because first is alone, and the next two are in the same batch) + # # If a new optimisation limiting starting request with same prefix at the same time was added + # # to maximise the cache hit, this would not be true + # assert cached_tokens1 == cached_tokens2 == cached_tokens0 + + # asyncio.run(run_test()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_enable_thinking.py b/test/srt/openai_server/features/test_enable_thinking.py new file mode 100644 index 000000000..00ba4fc94 --- /dev/null +++ b/test/srt/openai_server/features/test_enable_thinking.py @@ -0,0 +1,243 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_chat_completion_without_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_with_reasoning +python3 -m unittest openai_server.features.test_enable_thinking.TestEnableThinking.test_stream_chat_completion_without_reasoning +""" + +import asyncio +import json +import os +import sys +import time +import unittest + +import openai +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEnableThinking(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--reasoning-parser", + "qwen3", + ], + ) + cls.additional_chat_kwargs = {} + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_chat_completion_with_reasoning(self): + # Test non-streaming with "enable_thinking": True, reasoning_content should not be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + self.assertIn("reasoning_content", data["choices"][0]["message"]) + self.assertIsNotNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_chat_completion_without_reasoning(self): + # Test non-streaming with "enable_thinking": False, reasoning_content should be empty + client = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + ) + + self.assertEqual(client.status_code, 200, f"Failed with: {client.text}") + data = client.json() + + self.assertIn("choices", data) + self.assertTrue(len(data["choices"]) > 0) + self.assertIn("message", data["choices"][0]) + + if "reasoning_content" in data["choices"][0]["message"]: + self.assertIsNone(data["choices"][0]["message"]["reasoning_content"]) + + def test_stream_chat_completion_with_reasoning(self): + # Test streaming with "enable_thinking": True, reasoning_content should not be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": True}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + print("\n=== Stream With Reasoning ===") + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertTrue( + has_reasoning, + "The reasoning content is not included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + def test_stream_chat_completion_without_reasoning(self): + # Test streaming with "enable_thinking": False, reasoning_content should be empty + response = requests.post( + f"{self.base_url}/v1/chat/completions", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0, + "separate_reasoning": True, + "stream": True, + "chat_template_kwargs": {"enable_thinking": False}, + **self.additional_chat_kwargs, + }, + stream=True, + ) + + self.assertEqual(response.status_code, 200, f"Failed with: {response.text}") + + has_reasoning = False + has_content = False + + print("\n=== Stream Without Reasoning ===") + for line in response.iter_lines(): + if line: + line = line.decode("utf-8") + if line.startswith("data:") and not line.startswith("data: [DONE]"): + data = json.loads(line[6:]) + if "choices" in data and len(data["choices"]) > 0: + delta = data["choices"][0].get("delta", {}) + + if "reasoning_content" in delta and delta["reasoning_content"]: + has_reasoning = True + + if "content" in delta and delta["content"]: + has_content = True + + self.assertFalse( + has_reasoning, + "The reasoning content should not be included in the stream response", + ) + self.assertTrue( + has_content, "The stream response does not contain normal content" + ) + + +# Skip for ci test +# class TestGLM45EnableThinking(TestEnableThinking): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-1234" +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) + +# # Validate whether enable-thinking conflict with tool_calls +# cls.additional_chat_kwargs = { +# "tools": [ +# { +# "type": "function", +# "function": { +# "name": "add", +# "description": "Compute the sum of two numbers", +# "parameters": { +# "type": "object", +# "properties": { +# "a": { +# "type": "int", +# "description": "A number", +# }, +# "b": { +# "type": "int", +# "description": "A number", +# }, +# }, +# "required": ["a", "b"], +# }, +# }, +# } +# ] +# } + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_json_constrained.py b/test/srt/openai_server/features/test_json_constrained.py new file mode 100644 index 000000000..e4fdeecb5 --- /dev/null +++ b/test/srt/openai_server/features/test_json_constrained.py @@ -0,0 +1,153 @@ +""" +python3 -m unittest openai_server.features.test_json_constrained.TestJSONConstrainedOutlinesBackend.test_json_generate +python3 -m unittest openai_server.features.test_json_constrained.TestJSONConstrainedXGrammarBackend.test_json_generate +python3 -m unittest openai_server.features.test_json_constrained.TestJSONConstrainedLLGuidanceBackend.test_json_generate +""" + +import json +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + "additionalProperties": False, + } + ) + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestJSONConstrainedOutlinesBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="outlines") + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, json_schema, return_logprob=False, top_logprobs_num=0, n=1): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "stop_token_ids": [119690], + "json_schema": json_schema, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + ret = response.json() + print(json.dumps(ret)) + print("=" * 100) + + if not json_schema or json_schema == "INVALID": + return + + # Make sure the json output is valid + try: + js_obj = json.loads(ret["text"]) + except (TypeError, json.decoder.JSONDecodeError): + raise + + self.assertIsInstance(js_obj["name"], str) + self.assertIsInstance(js_obj["population"], int) + + def test_json_generate(self): + self.run_decode(json_schema=self.json_schema) + + def test_json_invalid(self): + self.run_decode(json_schema="INVALID") + + def test_json_openai(self): + client = openai.Client(api_key="EMPTY", base_url=f"{self.base_url}/v1") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Introduce the capital of France. Return in a JSON format.", + }, + ], + temperature=0, + max_tokens=128, + response_format={ + "type": "json_schema", + "json_schema": {"name": "foo", "schema": json.loads(self.json_schema)}, + }, + ) + text = response.choices[0].message.content + + try: + js_obj = json.loads(text) + except (TypeError, json.decoder.JSONDecodeError): + print("JSONDecodeError", text) + raise + + self.assertIsInstance(js_obj["name"], str) + self.assertIsInstance(js_obj["population"], int) + + def test_mix_json_and_other(self): + json_schemas = [None, None, self.json_schema, self.json_schema] * 10 + + with ThreadPoolExecutor(len(json_schemas)) as executor: + list(executor.map(self.run_decode, json_schemas)) + + +class TestJSONConstrainedXGrammarBackend(TestJSONConstrainedOutlinesBackend): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="xgrammar") + + +class TestJSONConstrainedLLGuidanceBackend(TestJSONConstrainedOutlinesBackend): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="llguidance") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_json_mode.py b/test/srt/openai_server/features/test_json_mode.py new file mode 100644 index 000000000..d74eccf0f --- /dev/null +++ b/test/srt/openai_server/features/test_json_mode.py @@ -0,0 +1,137 @@ +""" +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeOutlines.test_json_mode_response +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeOutlines.test_json_mode_with_streaming + +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeXGrammar.test_json_mode_response +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeXGrammar.test_json_mode_with_streaming + +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeLLGuidance.test_json_mode_response +python3 -m unittest openai_server.features.test_json_mode.TestJSONModeLLGuidance.test_json_mode_with_streaming +""" + +import json +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + + +def setup_class(cls, backend): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + cls.client = openai.Client(api_key="EMPTY", base_url=f"{cls.base_url}/v1") + + +class TestJSONModeOutlines(unittest.TestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, "outlines") + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_json_mode_response(self): + """Test that response_format json_object (also known as "json mode") produces valid JSON, even without a system prompt that mentions JSON.""" + response = self.client.chat.completions.create( + model=self.model, + messages=[ + # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + ) + text = response.choices[0].message.content + + print(f"Response ({len(text)} characters): {text}") + + # Verify the response is valid JSON + try: + js_obj = json.loads(text) + except json.JSONDecodeError as e: + self.fail(f"Response is not valid JSON. Error: {e}. Response: {text}") + + # Verify it's actually an object (dict) + self.assertIsInstance(js_obj, dict, f"Response is not a JSON object: {text}") + + def test_json_mode_with_streaming(self): + """Test that streaming with json_object response (also known as "json mode") format works correctly, even without a system prompt that mentions JSON.""" + stream = self.client.chat.completions.create( + model=self.model, + messages=[ + # We are deliberately omitting "That produces JSON" or similar phrases from the assistant prompt so that we don't have misleading test results + { + "role": "system", + "content": "You are a helpful AI assistant that gives a short answer.", + }, + {"role": "user", "content": "What is the capital of Bulgaria?"}, + ], + temperature=0, + max_tokens=128, + response_format={"type": "json_object"}, + stream=True, + ) + + # Collect all chunks + chunks = [] + for chunk in stream: + if chunk.choices[0].delta.content is not None: + chunks.append(chunk.choices[0].delta.content) + full_response = "".join(chunks) + + print( + f"Concatenated Response ({len(full_response)} characters): {full_response}" + ) + + # Verify the combined response is valid JSON + try: + js_obj = json.loads(full_response) + except json.JSONDecodeError as e: + self.fail( + f"Streamed response is not valid JSON. Error: {e}. Response: {full_response}" + ) + + self.assertIsInstance(js_obj, dict) + + +class TestJSONModeXGrammar(TestJSONModeOutlines): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="xgrammar") + + +class TestJSONModeLLGuidance(TestJSONModeOutlines): + @classmethod + def setUpClass(cls): + setup_class(cls, backend="llguidance") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_openai_server_ebnf.py b/test/srt/openai_server/features/test_openai_server_ebnf.py new file mode 100644 index 000000000..126556ed7 --- /dev/null +++ b/test/srt/openai_server/features/test_openai_server_ebnf.py @@ -0,0 +1,98 @@ +import re + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +# ------------------------------------------------------------------------- +# EBNF Test Class: TestOpenAIServerEBNF +# Launches the server with xgrammar, has only EBNF tests +# ------------------------------------------------------------------------- +class TestOpenAIServerEBNF(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # passing xgrammar specifically + other_args = ["--grammar-backend", "xgrammar"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_ebnf(self): + """ + Ensure we can pass `ebnf` to the local openai server + and that it enforces the grammar. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "Hello" | "Hi" | "Hey" + """ + pattern = re.compile(r"^(Hello|Hi|Hey)[.!?]*\s*$") + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful EBNF test bot."}, + {"role": "user", "content": "Say a greeting (Hello, Hi, or Hey)."}, + ], + temperature=0, + max_tokens=32, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF generation") + self.assertRegex(text, pattern, f"Text '{text}' doesn't match EBNF choices") + + def test_ebnf_strict_json(self): + """ + A stricter EBNF that produces exactly {"name":"Alice"} format + with no trailing punctuation or extra fields. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + ebnf_grammar = r""" + root ::= "{" pair "}" + pair ::= "\"name\"" ":" string + string ::= "\"" [A-Za-z]+ "\"" + """ + pattern = re.compile(r'^\{"name":"[A-Za-z]+"\}$') + + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "EBNF mini-JSON generator."}, + { + "role": "user", + "content": "Generate single key JSON with only letters.", + }, + ], + temperature=0, + max_tokens=64, + extra_body={"ebnf": ebnf_grammar}, + ) + text = response.choices[0].message.content.strip() + self.assertTrue(len(text) > 0, "Got empty text from EBNF strict JSON test") + self.assertRegex( + text, pattern, f"Text '{text}' not matching the EBNF strict JSON shape" + ) diff --git a/test/srt/openai_server/features/test_openai_server_hidden_states.py b/test/srt/openai_server/features/test_openai_server_hidden_states.py new file mode 100644 index 000000000..34e5ddde7 --- /dev/null +++ b/test/srt/openai_server/features/test_openai_server_hidden_states.py @@ -0,0 +1,356 @@ +import json +import re +import time +import unittest +from abc import ABC + +import numpy as np +import openai +import torch + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class BaseTestOpenAIServerWithHiddenStates(ABC): + + @classmethod + def setUpClass(cls): + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1, 2] + + def test_completion(self): + for return_hidden_states in self.return_hidden_states: + for use_list_input in self.use_list_input: + for parallel_sample_num in self.parallel_sample_nums: + self.run_completion( + use_list_input, + parallel_sample_num, + return_hidden_states, + ) + + def test_completion_stream(self): + # parallel sampling and list input are not supported in streaming mode + for return_hidden_states in self.return_hidden_states: + for use_list_input in self.use_list_input: + for parallel_sample_num in self.parallel_sample_nums: + self.run_completion_stream( + use_list_input, + parallel_sample_num, + return_hidden_states, + ) + + def test_chat_completion(self): + for return_hidden_states in self.return_hidden_states: + for ( + parallel_sample_num + ) in ( + self.parallel_sample_nums + ): # parallel sample num 2 breaks in the adapter with a 400 for EAGLE + self.run_chat_completion(parallel_sample_num, return_hidden_states) + + def test_chat_completion_stream(self): + for return_hidden_states in self.return_hidden_states: + for ( + parallel_sample_num + ) in ( + self.parallel_sample_nums + ): # parallel sample num > 1 breaks in the adapter with a 400 for EAGLE + self.run_chat_completion_stream( + parallel_sample_num, return_hidden_states + ) + + def run_completion( + self, + use_list_input, + parallel_sample_num, + return_hidden_states, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + prompt_input = prompt + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + else: + prompt_arg = prompt_input + num_choices = 1 + + response = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + for choice in response.choices: + assert hasattr(choice, "hidden_states") == return_hidden_states + if return_hidden_states: + assert choice.hidden_states is not None, "hidden_states was None" + + def run_completion_stream( + self, + use_list_input, + parallel_sample_num, + return_hidden_states, + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "The capital of France is" + prompt_input = prompt + num_prompt_tokens = len(self.tokenizer.encode(prompt)) + + if use_list_input: + prompt_arg = [prompt_input, prompt_input] + num_choices = len(prompt_arg) + num_prompt_tokens *= 2 + else: + prompt_arg = prompt_input + num_choices = 1 + + generator = client.completions.create( + model=self.model, + prompt=prompt_arg, + temperature=0, + max_tokens=32, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + hidden_states_list = [] + for response in generator: + usage = response.usage + for choice in response.choices: + if hasattr(choice, "hidden_states"): + assert return_hidden_states + assert choice.hidden_states is not None + hidden_states_list.append(choice.hidden_states) + + if return_hidden_states: + assert ( + len(hidden_states_list) == parallel_sample_num * num_choices + ), f"Expected {parallel_sample_num * num_choices} hidden states, got {len(hidden_states_list)}" + else: + assert ( + hidden_states_list == [] + ), "hidden_states were returned and should not have been" + + def run_chat_completion(self, parallel_sample_num, return_hidden_states): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + temperature=0, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + for choice in response.choices: + assert hasattr(choice, "hidden_states") == return_hidden_states + if return_hidden_states: + assert choice.hidden_states is not None, "hidden_states was None" + + def run_chat_completion_stream( + self, parallel_sample_num=1, return_hidden_states=False + ): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + generator = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": "What is the capital of France?"}, + ], + temperature=0, + stream=True, + stream_options={"include_usage": True}, + n=parallel_sample_num, + extra_body=dict(return_hidden_states=return_hidden_states), + ) + + is_firsts = {} + hidden_states_list = [] + + for response in generator: + for choice in response.choices: + if hasattr(choice.delta, "hidden_states"): + assert return_hidden_states + assert choice.delta.hidden_states is not None + hidden_states_list.append(choice.delta.hidden_states) + + if return_hidden_states: + assert ( + len(hidden_states_list) == parallel_sample_num + ), f"Expected {parallel_sample_num} hidden states, got {len(hidden_states_list)}" + else: + assert ( + hidden_states_list == [] + ), "hidden_states were returned and should not have been" + + +class TestOpenAIServerWithHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=["--enable-return-hidden-states"], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1, 2] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithHiddenStatesEnabledAndCUDAGraphDisabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=["--enable-return-hidden-states", "--disable-cuda-graph"], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithEAGLEAndHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.speculative_draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + cls.speculative_algorithm = "EAGLE" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--enable-return-hidden-states", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +class TestOpenAIServerWithEAGLE3AndHiddenStatesEnabled( + CustomTestCase, BaseTestOpenAIServerWithHiddenStates +): + @classmethod + def setUpClass(cls): + cls.model = "meta-llama/Llama-3.1-8B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.speculative_algorithm = "EAGLE3" + cls.speculative_draft_model = "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + cls.speculative_algorithm, + "--speculative-draft-model-path", + cls.speculative_draft_model, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 16, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--dtype", + "float16", + "--enable-return-hidden-states", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + cls.return_hidden_states = [False, True] + cls.use_list_input = [True, False] + cls.parallel_sample_nums = [1] + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/features/test_reasoning_content.py b/test/srt/openai_server/features/test_reasoning_content.py new file mode 100644 index 000000000..04e5160a7 --- /dev/null +++ b/test/srt/openai_server/features/test_reasoning_content.py @@ -0,0 +1,343 @@ +""" +Usage: +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_streaming_separate_reasoning_true_stream_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_false +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentAPI.test_nonstreaming_separate_reasoning_true +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_nonstreaming +python3 -m unittest openai_server.features.test_reasoning_content.TestReasoningContentStartup.test_streaming +""" + +import json +import unittest + +import openai +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_REASONING_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestReasoningContentAPI(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--reasoning-parser", + "deepseek-r1", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert len(reasoning_content) > 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert len(response.choices[0].message.reasoning_content) > 0 + assert len(response.choices[0].message.content) > 0 + + +class TestReasoningContentWithoutParser(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_REASONING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[], # No reasoning parser + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_streaming_separate_reasoning_false(self): + # Test streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + for chunk in response: + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + elif chunk.choices[0].delta.reasoning_content: + reasoning_content += chunk.choices[0].delta.reasoning_content + + assert len(reasoning_content) == 0 + assert len(content) > 0 + + def test_streaming_separate_reasoning_true_stream_reasoning_false(self): + # Test streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "stream": True, + "extra_body": {"separate_reasoning": True, "stream_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + reasoning_content = "" + content = "" + first_chunk = False + for chunk in response: + if chunk.choices[0].delta.reasoning_content: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if chunk.choices[0].delta.content: + content += chunk.choices[0].delta.content + if not first_chunk: + reasoning_content = chunk.choices[0].delta.reasoning_content + first_chunk = True + if not first_chunk: + assert ( + not chunk.choices[0].delta.reasoning_content + or len(chunk.choices[0].delta.reasoning_content) == 0 + ) + assert not reasoning_content or len(reasoning_content) == 0 + assert len(content) > 0 + + def test_nonstreaming_separate_reasoning_false(self): + # Test non-streaming with separate_reasoning=False, reasoning_content should be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": False}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + def test_nonstreaming_separate_reasoning_true(self): + # Test non-streaming with separate_reasoning=True, reasoning_content should not be empty + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + payload = { + "model": self.model, + "messages": [ + { + "role": "user", + "content": "What is 1+3?", + } + ], + "max_tokens": 100, + "extra_body": {"separate_reasoning": True}, + } + response = client.chat.completions.create(**payload) + + assert ( + not response.choices[0].message.reasoning_content + or len(response.choices[0].message.reasoning_content) == 0 + ) + assert len(response.choices[0].message.content) > 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/function_call/__init__.py b/test/srt/openai_server/function_call/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/srt/openai_server/function_call/test_openai_function_calling.py b/test/srt/openai_server/function_call/test_openai_function_calling.py new file mode 100644 index 000000000..291ef98b7 --- /dev/null +++ b/test/srt/openai_server/function_call/test_openai_function_calling.py @@ -0,0 +1,953 @@ +import json +import time +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServerFunctionCalling(CustomTestCase): + # NOTE: this system_message is for Llama3.2 system prompt. Without this, + # sometimes Llama3.2 gives a different tool call format such as: + # '<|python_tag|>{"type": "function", "function": "add", "parameters": {"a": "3", "b": "5"}}' + SYSTEM_MESSAGE = ( + "You are a helpful assistant with tool calling capabilities. " + "Only reply with a tool call if the function exists in the library provided by the user. " + "If it doesn't exist, just reply directly in natural language. " + "When you receive a tool call response, use the output to format an answer to the original user question. " + "You have access to the following functions. " + "To call a function, please respond with JSON for a function call. " + 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' + "Do not use variables.\n\n" + ) + + @classmethod + def setUpClass(cls): + # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + # If your server needs extra parameters to test function calling, please add them here. + "--tool-call-parser", + "llama3", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_function_calling_format(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "int", + "description": "A number", + }, + "b": { + "type": "int", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Compute (3+5)"}, + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + + assert ( + isinstance(tool_calls, list) and len(tool_calls) > 0 + ), "tool_calls should be a non-empty list" + + function_name = tool_calls[0].function.name + assert function_name == "add", "Function name should be 'add'" + + # This unit test is too difficult for default model. Mark it as optional unit tests so it won't trigger unless specified. + def _test_function_calling_multiturn(self): + """ + Test: Whether the function call format returned by the AI is correct. + When returning a tool call, message.content should be None, and tool_calls should be a list. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two numbers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "int", + "description": "A number", + }, + "b": { + "type": "int", + "description": "A number", + }, + }, + "required": ["a", "b"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Compute (3+5)"}] + + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_call = response.choices[0].message.tool_calls[0] + function_name = tool_call.function.name + assert function_name == "add", "Function name should be 'add'" + function_arguments = tool_call.function.arguments + function_arguments = json.loads(tool_call.function.arguments) + assert function_arguments in [ + {"a": 3, "b": 5}, + {"a": "3", "b": "5"}, + ], f"Unexpected function arguments: {function_arguments}" + + messages.append(response.choices[0].message) + messages.append( + { + "role": "tool", + "tool_call_id": tool_call.id, + "content": "8", + "name": function_name, + } + ) + + final_response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + assert ( + "8" in final_response.choices[0].message.content + ), "tool_call response should have the sum 8 in the content" + + def test_function_calling_streaming_simple(self): + """ + Test: Whether the function name can be correctly recognized in streaming mode. + - Expect a function call to be found, and the function name to be correct. + - Verify that streaming mode returns at least multiple chunks. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + { + "role": "user", + "content": "What is the temperature in Paris in celsius??", + }, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_function_name = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + if tool_call.function.name: + self.assertEqual( + tool_call.function.name, + "get_current_weather", + "Function name should be 'get_current_weather'", + ) + found_function_name = True + break + + self.assertTrue( + found_function_name, + "Target function name 'get_current_weather' was not found in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + def test_function_calling_streaming_args_parsing(self): + """ + Test: Whether the function call arguments returned in streaming mode can be correctly concatenated into valid JSON. + - The user request requires multiple parameters. + - AI may return the arguments in chunks that need to be concatenated. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "add", + "description": "Compute the sum of two integers", + "parameters": { + "type": "object", + "properties": { + "a": { + "type": "integer", + "description": "First integer", + }, + "b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["a", "b"], + }, + "strict": True, # Llama-3.2-1B is flaky in tool call. It won't always respond with parameters unless we set strict. + }, + } + ] + + messages = [ + {"role": "system", "content": self.SYSTEM_MESSAGE}, + {"role": "user", "content": "Please sum 5 and 7, just call the function."}, + ] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.9, + top_p=0.9, + stream=True, + tools=tools, + ) + + argument_fragments = [] + chunks = list(response_stream) + function_name = None + for chunk in chunks: + choice = chunk.choices[0] + if choice.delta.tool_calls: + tool_call = choice.delta.tool_calls[0] + # Record the function name on first occurrence + function_name = tool_call.function.name or function_name + # In case of multiple chunks, JSON fragments may need to be concatenated + if tool_call.function.arguments is not None: + argument_fragments.append(tool_call.function.arguments) + + self.assertEqual(function_name, "add", "Function name should be 'add'") + joined_args = "".join(argument_fragments) + self.assertTrue( + len(joined_args) > 0, + "No parameter fragments were returned in the function call", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "tool_calls", + "Final response of function calling should have finish_reason 'tool_calls'", + ) + + # Check whether the concatenated JSON is valid + try: + args_obj = json.loads(joined_args) + except json.JSONDecodeError: + self.fail( + "The concatenated tool call arguments are not valid JSON, parsing failed" + ) + + self.assertIn("a", args_obj, "Missing parameter 'a'") + self.assertIn("b", args_obj, "Missing parameter 'b'") + self.assertEqual(str(args_obj["a"]), "5", "Parameter a should be 5") + self.assertEqual(str(args_obj["b"]), "7", "Parameter b should be 7") + + def test_function_call_strict(self): + """ + Test: Whether the strict mode of function calling works as expected. + - When strict mode is enabled, the AI should not return a function call if the function name is not recognized. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + } + ] + + messages = [ + {"role": "user", "content": "Please compute 5 - 7, using your tool."} + ] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + ) + + tool_calls = response.choices[0].message.tool_calls + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual(function_name, "sub", "Function name should be 'sub'") + self.assertEqual(str(args_obj["int_a"]), "5", "Parameter int_a should be 5") + self.assertEqual(str(args_obj["int_b"]), "7", "Parameter int_b should be 7") + + def test_function_call_required(self): + """ + Test: Whether tool_choice: "required" works as expected + - When tool_choice == "required", the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice="required", + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, + "get_weather", + f"Function name should be 'get_weather', got: {function_name}", + ) + self.assertIn( + "city", args_obj, f"Function arguments should have 'city', got: {args_obj}" + ) + + # Make the test more robust by checking type and accepting valid responses + city_value = args_obj["city"] + self.assertIsInstance( + city_value, + str, + f"Parameter city should be a string, got: {type(city_value)}", + ) + self.assertTrue( + "Paris" in city_value or "France" in city_value, + f"Parameter city should contain either 'Paris' or 'France', got: {city_value}", + ) + + def test_function_call_specific(self): + """ + Test: Whether tool_choice: ToolChoice works as expected + - When tool_choice is a specific ToolChoice, the model should return one or more tool_calls. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "sub", + "description": "Compute the difference of two integers", + "parameters": { + "type": "object", + "properties": { + "int_a": { + "type": "integer", + "description": "First integer", + }, + "int_b": { + "type": "integer", + "description": "Second integer", + }, + }, + "required": ["int_a", "int_b"], + }, + "strict": True, + }, + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + } + }, + "required": ["city"], + }, + }, + }, + ] + + messages = [{"role": "user", "content": "What is the capital of France?"}] + response = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=False, + tools=tools, + tool_choice={"type": "function", "function": {"name": "get_weather"}}, + ) + + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls, "No tool_calls in the response") + function_name = tool_calls[0].function.name + arguments = tool_calls[0].function.arguments + args_obj = json.loads(arguments) + + self.assertEqual( + function_name, "get_weather", "Function name should be 'get_weather'" + ) + self.assertIn("city", args_obj, "Function arguments should have 'city'") + + def test_streaming_multiple_choices_finish_reason(self): + """ + Test: Verify that each choice gets its own finish_reason chunk in streaming mode with n > 1. + This tests the fix for the bug where only the last index got a finish_reason chunk. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + } + ] + + messages = [ + {"role": "user", "content": "What is the weather like in Los Angeles?"} + ] + + # Request with n=2 to get multiple choices + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + max_tokens=2048, + temperature=0.8, + stream=True, + tools=tools, + tool_choice="required", # Force tool calls + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "tool_calls" since we forced tool calls + for index, reasons in finish_reason_chunks.items(): + self.assertEqual( + reasons[-1], # Last finish_reason for this index + "tool_calls", + f"Expected finish_reason 'tool_calls' for index {index}, got {reasons[-1]}", + ) + + def test_function_calling_streaming_no_tool_call(self): + """ + Test: Whether the finish_reason is stop in streaming mode when no tool call is given. + - Expect no function call to be found. + - Verify that finish_reason is stop + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The city to find the weather for", + }, + "unit": { + "type": "string", + "description": "Weather unit (celsius or fahrenheit)", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city", "unit"], + }, + }, + } + ] + + messages = [{"role": "user", "content": "Who are you?"}] + + response_stream = client.chat.completions.create( + model=self.model, + max_tokens=2048, + messages=messages, + temperature=0.8, + top_p=0.8, + stream=True, + tools=tools, + tool_choice="none", + ) + + chunks = list(response_stream) + self.assertTrue(len(chunks) > 0, "Streaming should return at least one chunk") + + found_tool_call = False + for chunk in chunks: + choice = chunk.choices[0] + # Check whether the current chunk contains tool_calls + found_tool_call = choice.delta.tool_calls is not None + + self.assertFalse( + found_tool_call, + "Shouldn't have any tool_call in the streaming chunks", + ) + + finish_reason = chunks[-1].choices[0].finish_reason + self.assertEqual( + finish_reason, + "stop", + "Final response of no function calling should have finish_reason 'stop'", + ) + + def test_streaming_multiple_choices_without_tools(self): + """ + Test: Verify that each choice gets its own finish_reason chunk without tool calls. + This tests the fix for regular content streaming with multiple choices. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = [{"role": "user", "content": "Say hello in one word."}] + + # Request with n=2 to get multiple choices, no tools + response_stream = client.chat.completions.create( + model=self.model, + messages=messages, + temperature=0.8, + stream=True, + max_tokens=10, # Keep it short + n=2, # Multiple choices + ) + + chunks = list(response_stream) + + # Track finish_reason chunks for each index + finish_reason_chunks = {} + for chunk in chunks: + if chunk.choices: + for choice in chunk.choices: + if choice.finish_reason is not None: + index = choice.index + if index not in finish_reason_chunks: + finish_reason_chunks[index] = [] + finish_reason_chunks[index].append(choice.finish_reason) + + # Verify we got finish_reason chunks for both indices + self.assertEqual( + len(finish_reason_chunks), + 2, + f"Expected finish_reason chunks for 2 indices, got {len(finish_reason_chunks)}", + ) + + # Verify both index 0 and 1 have finish_reason + self.assertIn( + 0, finish_reason_chunks, "Missing finish_reason chunk for index 0" + ) + self.assertIn( + 1, finish_reason_chunks, "Missing finish_reason chunk for index 1" + ) + + # Verify the finish_reason is "stop" (regular completion) + for index, reasons in finish_reason_chunks.items(): + self.assertIn( + reasons[-1], + ["stop", "length"], # Could be either depending on how model responds + f"Expected finish_reason 'stop' or 'length' for index {index}, got {reasons[-1]}", + ) + + +class TestOpenAIPythonicFunctionCalling(CustomTestCase): + PYTHONIC_TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + } + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + PYTHONIC_MESSAGES = [ + { + "role": "system", + "content": ( + "You are a travel assistant. " + "When asked to call functions, ALWAYS respond ONLY with a python list of function calls, " + "using this format: [func_name1(param1=value1, param2=value2), func_name2(param=value)]. " + "Do NOT use JSON, do NOT use variables, do NOT use any other format. " + "Here is an example:\n" + '[get_weather(location="Paris"), get_tourist_attractions(city="Paris")]' + ), + }, + { + "role": "user", + "content": ( + "I'm planning a trip to Tokyo next week. What's the weather like and what are some top tourist attractions? " + "Propose parallel tool calls at once, using the python list of function calls format as shown above." + ), + }, + ] + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "pythonic", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_pythonic_tool_call_prompt(self): + """ + Test: Explicit prompt for pythonic tool call format without chat template. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=False, + ) + tool_calls = response.choices[0].message.tool_calls + self.assertIsInstance(tool_calls, list, "No tool_calls found") + self.assertGreaterEqual(len(tool_calls), 1) + names = [tc.function.name for tc in tool_calls] + self.assertTrue( + "get_weather" in names or "get_tourist_attractions" in names, + f"Function name '{names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + def test_pythonic_tool_call_streaming(self): + """ + Test: Streaming pythonic tool call format; assert tool_call index is present. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response_stream = client.chat.completions.create( + model=self.model, + messages=self.PYTHONIC_MESSAGES, + tools=self.PYTHONIC_TOOLS, + temperature=0.1, + stream=True, + ) + found_tool_calls = False + found_index = False + found_names = set() + for chunk in response_stream: + choice = chunk.choices[0] + if getattr(choice.delta, "tool_calls", None): + found_tool_calls = True + tool_call = choice.delta.tool_calls[0] + if hasattr(tool_call, "index") or ( + isinstance(tool_call, dict) and "index" in tool_call + ): + found_index = True + found_names.add(str(tool_call.function.name)) + + self.assertTrue(found_tool_calls, "No tool_calls found in streaming response") + self.assertTrue(found_index, "No index field found in any streamed tool_call") + self.assertTrue( + "get_weather" in found_names or "get_tourist_attractions" in found_names, + f"Function name '{found_names}' should container either 'get_weather' or 'get_tourist_attractions'", + ) + + +# Skip for ci test +# class TestGLM45ServerFunctionCalling(TestOpenAIServerFunctionCalling): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" + +# # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# # If your server needs extra parameters to test function calling, please add them here. +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) +# cls.base_url += "/v1" +# cls.tokenizer = get_tokenizer(cls.model) + +# # This test is too difficult for GLM4-moe. Skip it from the UT +# def test_function_call_required(self): +# pass + +# def test_function_calling_multiturn(self): +# self._test_function_calling_multiturn() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/function_call/test_tool_choice.py b/test/srt/openai_server/function_call/test_tool_choice.py new file mode 100644 index 000000000..d8094e930 --- /dev/null +++ b/test/srt/openai_server/function_call/test_tool_choice.py @@ -0,0 +1,550 @@ +""" +Test script for tool_choice functionality in SGLang +Tests: required, auto, and specific function choices in both streaming and non-streaming modes + +# To run the tests, use the following command: +# +# python3 -m unittest openai_server.function_call.test_tool_choice +""" + +import json +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestToolChoiceLlama32(CustomTestCase): + + @classmethod + def setUpClass(cls): + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + # Use a model that supports function calling + cls.model = "meta-llama/Llama-3.2-1B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start the local OpenAI Server with tool calling support + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "llama3", # Default parser for the test model + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def setUp(self): + self.client = openai.Client(base_url=self.base_url, api_key=self.api_key) + self.model_name = self.client.models.list().data[0].id + + def _is_flaky_test(self): + """Check if the current test is marked as flaky for this class""" + return ( + hasattr(self.__class__, "flaky_tests") + and self._testMethodName in self.__class__.flaky_tests + ) + + def get_test_tools(self): + """Get the test tools for function calling""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "use this to get latest weather information for a city given its name", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "name of the city to get weather for", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["city"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_pokemon_info", + "description": "get detailed information about a pokemon given its name", + "parameters": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "name of the pokemon to get info for", + } + }, + "required": ["name"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "make_next_step_decision", + "description": "You will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools. \n You should never call the same tool with the same input twice in a row.\n If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again.\nOBSERVATION: the result of the tool call, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information,\n or you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\n If the previous conversation history already contains the answer, respond with the answer right away.\n\n If no tools are configured, naturally mention this limitation while still being helpful. Briefly note that adding tools in the agent configuration would expand capabilities.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.", + "parameters": { + "type": "object", + "properties": { + "decision": { + "type": "string", + "description": 'The next step to take, it must be either "TOOL" or "ANSWER". If the previous conversation history already contains the information that can be retrieved from the tool, you should not call the tool again. If there are no defined tools, you should not return "TOOL" in your response.', + }, + "content": { + "type": "string", + "description": 'The content of the next step. If the decision is "TOOL", this should be a short and concise reasoning of why you chose the tool, MUST include the tool name. If the decision is "ANSWER", this should be the answer to the question. If no tools are available, integrate this limitation conversationally without sounding scripted.', + }, + }, + "required": ["decision", "content"], + }, + }, + }, + ] + + def get_test_messages(self): + """Get test messages that should trigger tool usage""" + return [ + { + "role": "user", + "content": "Answer the following questions as best you can:\n\nYou will be given a trace of thinking process in the following format.\n\nQuestion: the input question you must answer\nTOOL: think about what to do, and choose a tool to use ONLY IF there are defined tools\nOBSERVATION: the result of the tool call or the observation of the current task, NEVER include this in your response, this information will be provided\n... (this TOOL/OBSERVATION can repeat N times)\nANSWER: If you know the answer to the original question, require for more information, \nif the previous conversation history already contains the answer, \nor you don't know the answer and there are no defined tools or all available tools are not helpful, respond with the answer without mentioning anything else.\nYou may use light Markdown formatting to improve clarity (e.g. lists, **bold**, *italics*), but keep it minimal and unobtrusive.\n\nYour task is to respond with the next step to take, based on the traces, \nor answer the question if you have enough information.\n\nQuestion: what is the weather in top 5 populated cities in the US in celsius?\n\nTraces:\n\n\nThese are some additional instructions that you should follow:", + } + ] + + def get_travel_tools(self): + """Get tools for travel assistant scenario that should trigger multiple tool calls""" + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a given location.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The name of the city or location.", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + }, + }, + { + "type": "function", + "function": { + "name": "get_tourist_attractions", + "description": "Get a list of top tourist attractions for a given city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "The name of the city to find attractions for.", + } + }, + "required": ["city"], + }, + }, + }, + ] + + def get_travel_messages(self): + """Get travel assistant messages that should trigger multiple tool calls""" + return [ + { + "content": "You are a travel assistant providing real-time weather updates and top tourist attractions.", + "role": "system", + }, + { + "content": "I'm planning a trip to Tokyo next week. What's the weather like? What are the most amazing sights?", + "role": "user", + }, + ] + + def test_tool_choice_auto_non_streaming(self): + """Test tool_choice='auto' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=False, + ) + + self.assertIsNotNone(response.choices[0].message) + # With auto, tool calls are optional + + def test_tool_choice_auto_streaming(self): + """Test tool_choice='auto' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="auto", + stream=True, + ) + + # Collect streaming response + content_chunks = [] + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.content: + content_chunks.append(chunk.choices[0].delta.content) + elif chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should complete without errors + self.assertIsInstance(content_chunks, list) + self.assertIsInstance(tool_call_chunks, list) + + def test_tool_choice_required_non_streaming(self): + """Test tool_choice='required' in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get tool calls + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + def test_tool_choice_required_streaming(self): + """Test tool_choice='required' in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice="required", + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # With required, we should get tool call chunks + self.assertGreater(len(tool_call_chunks), 0) + + def test_tool_choice_specific_function_non_streaming(self): + """Test tool_choice with specific function in non-streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + # Should call the specific function + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + # Our messages ask the top 5 populated cities in the US, so the model could get 5 tool calls + self.assertGreaterEqual(len(tool_calls), 1) + for tool_call in tool_calls: + self.assertEqual(tool_call.function.name, "get_weather") + + def test_tool_choice_specific_function_streaming(self): + """Test tool_choice with specific function in streaming mode""" + tools = self.get_test_tools() + messages = self.get_test_messages() + + tool_choice = {"type": "function", "function": {"name": "get_weather"}} + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=True, + ) + + # Collect streaming response + tool_call_chunks = [] + + for chunk in response: + if chunk.choices[0].delta.tool_calls: + tool_call_chunks.extend(chunk.choices[0].delta.tool_calls) + + # Should get tool call chunks for the specific function + self.assertGreater(len(tool_call_chunks), 0) + + # Find function name in chunks + found_name = None + for chunk in tool_call_chunks: + if chunk.function and chunk.function.name: + found_name = chunk.function.name + break + + self.assertEqual(found_name, "get_weather") + + def test_multi_tool_scenario_auto(self): + """Test multi-tool scenario with tool_choice='auto'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="auto", + stream=False, + ) + + # Should complete without errors + self.assertIsNotNone(response.choices[0].message) + + tool_calls = response.choices[0].message.tool_calls + expected_functions = {"get_weather", "get_tourist_attractions"} + + if self._is_flaky_test(): + # For flaky tests, just verify all called functions are available tools + if tool_calls: + available_names = [tool["function"]["name"] for tool in tools] + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertIsNotNone(tool_calls, "Expected tool calls but got none") + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_multi_tool_scenario_required(self): + """Test multi-tool scenario with tool_choice='required'""" + tools = self.get_travel_tools() + messages = self.get_travel_messages() + + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + temperature=0.2, + tools=tools, + tool_choice="required", + stream=False, + ) + + # With required, we should get at least one tool call + tool_calls = response.choices[0].message.tool_calls + self.assertIsNotNone(tool_calls) + self.assertGreater(len(tool_calls), 0) + + # Verify all called functions are available tools + available_names = [tool["function"]["name"] for tool in tools] + expected_functions = {"get_weather", "get_tourist_attractions"} + + if self._is_flaky_test(): + # For flaky tests, just ensure basic functionality works + self.assertGreater( + len(tool_calls), + 0, + f"Expected at least 1 tool call, got {len(tool_calls)}", + ) + for call in tool_calls: + self.assertIn(call.function.name, available_names) + else: + # For non-flaky tests, enforce strict requirements + self.assertEqual( + len(tool_calls), 2, f"Expected 2 tool calls, got {len(tool_calls)}" + ) + + called_functions = {call.function.name for call in tool_calls} + self.assertEqual( + called_functions, + expected_functions, + f"Expected functions {expected_functions}, got {called_functions}", + ) + + def test_error_handling_invalid_tool_choice(self): + """Test error handling for invalid tool_choice""" + import logging + from unittest.mock import patch + + tools = self.get_test_tools() + messages = self.get_test_messages() + + # Test with invalid function name + tool_choice = {"type": "function", "function": {"name": "nonexistent_function"}} + + # The behavior could be either: + # 1. Log a warning and continue (if fallback is implemented) + # 2. Raise an exception (if strict validation is implemented) + + # First try to capture any logging that might happen + with patch("logging.warning") as mock_warning: + response = self.client.chat.completions.create( + model=self.model_name, + messages=messages, + max_tokens=2048, + tools=tools, + tool_choice=tool_choice, + stream=False, + ) + + self.assertIsNotNone(response.choices[0].message) + + if mock_warning.called: + warning_message = mock_warning.call_args[0][0] + self.assertIn("nonexistent_function", warning_message) + + +class TestToolChoiceQwen25(TestToolChoiceLlama32): + """Test tool_choice functionality with Qwen2.5 model""" + + @classmethod + def setUpClass(cls): + cls.flaky_tests = {} + + cls.model = "Qwen/Qwen2.5-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "qwen25", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + +class TestToolChoiceMistral(TestToolChoiceLlama32): + """Test tool_choice functionality with Mistral model""" + + @classmethod + def setUpClass(cls): + # Mark flaky tests for this model + cls.flaky_tests = { + "test_multi_tool_scenario_auto", + "test_multi_tool_scenario_required", + } + + cls.model = "mistralai/Mistral-7B-Instruct-v0.3" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--tool-call-parser", + "mistral", + ], + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + +# Skip for ci test +# class TestToolChoiceGLM45(TestToolChoiceLlama32): +# @classmethod +# def setUpClass(cls): +# # Replace with the model name needed for testing; if not required, reuse DEFAULT_SMALL_MODEL_NAME_FOR_TEST +# cls.model = "THUDM/GLM-4.5" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" + +# # Start the local OpenAI Server. If necessary, you can add other parameters such as --enable-tools. +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# other_args=[ +# # If your server needs extra parameters to test function calling, please add them here. +# "--tool-call-parser", +# "glm45", +# "--reasoning-parser", +# "glm45", +# "--tp-size", +# "8" +# ], +# ) +# cls.base_url += "/v1" +# cls.tokenizer = get_tokenizer(cls.model) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/validation/__init__.py b/test/srt/openai_server/validation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/srt/openai_server/validation/test_large_max_new_tokens.py b/test/srt/openai_server/validation/test_large_max_new_tokens.py new file mode 100644 index 000000000..49601a784 --- /dev/null +++ b/test/srt/openai_server/validation/test_large_max_new_tokens.py @@ -0,0 +1,103 @@ +""" +python3 -m unittest openai_server.validation.test_large_max_new_tokens.TestLargeMaxNewTokens.test_chat_completion +""" + +import os +import time +import unittest +from concurrent.futures import ThreadPoolExecutor + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, +) + + +class TestLargeMaxNewTokens(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=( + "--max-total-token", + "1536", + "--context-len", + "8192", + "--decode-log-interval", + "2", + ), + env={"SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION": "256", **os.environ}, + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def run_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "Please repeat the world 'hello' for 10000 times.", + }, + ], + temperature=0, + ) + return response + + def test_chat_completion(self): + num_requests = 4 + + futures = [] + with ThreadPoolExecutor(num_requests) as executor: + # Send multiple requests + for i in range(num_requests): + futures.append(executor.submit(self.run_chat_completion)) + + # Ensure that they are running concurrently + pt = 0 + while pt >= 0: + time.sleep(5) + lines = open(STDERR_FILENAME).readlines() + for line in lines[pt:]: + print(line, end="", flush=True) + if f"#running-req: {num_requests}" in line: + all_requests_running = True + pt = -1 + break + pt += 1 + + assert all_requests_running + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/validation/test_matched_stop.py b/test/srt/openai_server/validation/test_matched_stop.py new file mode 100644 index 000000000..357b07f31 --- /dev/null +++ b/test/srt/openai_server/validation/test_matched_stop.py @@ -0,0 +1,140 @@ +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +MANY_NEW_TOKENS_PROMPT = """ +Please write an extremely detailed and vivid fantasy story, set in a world full of intricate magic systems, political intrigue, and complex characters. +Ensure that you thoroughly describe every scene, character's motivations, and the environment. Include long, engaging dialogues and elaborate on the inner thoughts of the characters. +Each section should be as comprehensive as possible to create a rich and immersive experience for the reader. +The story should span multiple events, challenges, and character developments over time. Aim to make the story at least 3,000 words long. +""" + + +class TestMatchedStop(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=300, + other_args=["--max-running-requests", "10"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_completions_generation( + self, + prompt=MANY_NEW_TOKENS_PROMPT, + max_tokens=1, + stop=None, + finish_reason=None, + matched_stop=None, + ): + payload = { + "prompt": prompt, + "model": self.model, + "temperature": 0, + "top_p": 1, + "max_tokens": max_tokens, + } + + if stop is not None: + payload["stop"] = stop + + response_completions = requests.post( + self.base_url + "/v1/completions", + json=payload, + ) + print(json.dumps(response_completions.json())) + print("=" * 100) + + assert ( + response_completions.json()["choices"][0]["finish_reason"] == finish_reason + ) + assert response_completions.json()["choices"][0]["matched_stop"] == matched_stop + + def run_chat_completions_generation( + self, + prompt=MANY_NEW_TOKENS_PROMPT, + max_tokens=1, + stop=None, + finish_reason=None, + matched_stop=None, + ): + chat_payload = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + {"role": "user", "content": prompt}, + ], + "temperature": 0, + "top_p": 1, + "max_tokens": max_tokens, + } + + if stop is not None: + chat_payload["stop"] = stop + + response_chat = requests.post( + self.base_url + "/v1/chat/completions", + json=chat_payload, + ) + print(json.dumps(response_chat.json())) + print("=" * 100) + + assert response_chat.json()["choices"][0]["finish_reason"] == finish_reason + assert response_chat.json()["choices"][0]["matched_stop"] == matched_stop + + def test_finish_stop_str(self): + self.run_completions_generation( + max_tokens=1000, stop="\n", finish_reason="stop", matched_stop="\n" + ) + self.run_chat_completions_generation( + max_tokens=1000, stop="\n", finish_reason="stop", matched_stop="\n" + ) + + def test_finish_stop_eos(self): + llama_format_prompt = """ + <|begin_of_text|><|start_header_id|>system<|end_header_id|> + You are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|> + + What is 2 + 2?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + """ + eos_token_id = 128009 + self.run_completions_generation( + prompt=llama_format_prompt, + max_tokens=1000, + finish_reason="stop", + matched_stop=eos_token_id, + ) + self.run_chat_completions_generation( + prompt="What is 2 + 2?", + max_tokens=1000, + finish_reason="stop", + matched_stop=eos_token_id, + ) + + def test_finish_length(self): + self.run_completions_generation( + max_tokens=5, finish_reason="length", matched_stop=None + ) + self.run_chat_completions_generation( + max_tokens=5, finish_reason="length", matched_stop=None + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/openai_server/validation/test_openai_server_ignore_eos.py b/test/srt/openai_server/validation/test_openai_server_ignore_eos.py new file mode 100644 index 000000000..a3594dfd0 --- /dev/null +++ b/test/srt/openai_server/validation/test_openai_server_ignore_eos.py @@ -0,0 +1,84 @@ +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestOpenAIServerIgnoreEOS(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_ignore_eos(self): + """ + Test that ignore_eos=True allows generation to continue beyond EOS token + and reach the max_tokens limit. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + max_tokens = 200 + + response_default = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": False}, + ) + + response_ignore_eos = client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Count from 1 to 20."}, + ], + temperature=0, + max_tokens=max_tokens, + extra_body={"ignore_eos": True}, + ) + + default_tokens = len( + self.tokenizer.encode(response_default.choices[0].message.content) + ) + ignore_eos_tokens = len( + self.tokenizer.encode(response_ignore_eos.choices[0].message.content) + ) + + # Check if ignore_eos resulted in more tokens or exactly max_tokens + # The ignore_eos response should either: + # 1. Have more tokens than the default response (if default stopped at EOS before max_tokens) + # 2. Have exactly max_tokens (if it reached the max_tokens limit) + self.assertTrue( + ignore_eos_tokens > default_tokens or ignore_eos_tokens >= max_tokens, + f"ignore_eos did not generate more tokens: {ignore_eos_tokens} vs {default_tokens}", + ) + + self.assertEqual( + response_ignore_eos.choices[0].finish_reason, + "length", + f"Expected finish_reason='length' for ignore_eos=True, got {response_ignore_eos.choices[0].finish_reason}", + ) diff --git a/test/srt/openai_server/validation/test_request_length_validation.py b/test/srt/openai_server/validation/test_request_length_validation.py new file mode 100644 index 000000000..7276906d4 --- /dev/null +++ b/test/srt/openai_server/validation/test_request_length_validation.py @@ -0,0 +1,88 @@ +import unittest + +import openai + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestRequestLengthValidation(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + # Start server with auto truncate disabled + cls.process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=("--max-total-tokens", "1000", "--context-length", "1000"), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_input_length_longer_than_context_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 1200 # Will tokenize to more than context length + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_input_length_longer_than_maximum_allowed_length(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " * 999 # the maximum allowed length is 994 tokens + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + ) + + self.assertIn("is longer than the model's context length", str(cm.exception)) + + def test_max_tokens_validation(self): + client = openai.Client(api_key=self.api_key, base_url=f"{self.base_url}/v1") + + long_text = "hello " + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + messages=[ + {"role": "user", "content": long_text}, + ], + temperature=0, + max_tokens=1200, + ) + + self.assertIn( + "max_completion_tokens is too large", + str(cm.exception), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/parse_results.py b/test/srt/parse_results.py new file mode 100644 index 000000000..f552739f5 --- /dev/null +++ b/test/srt/parse_results.py @@ -0,0 +1,57 @@ +import argparse +import json +import os + +import pandas as pd +from tabulate import tabulate + +# Parse command-line arguments +parser = argparse.ArgumentParser(description="Parse JSONL benchmark and summarize.") +parser.add_argument("input_file", type=str, help="Path to input JSONL file") +parser.add_argument( + "--md", + action="store_true", + help="If set, print the summary table in Markdown format (GitHub style)", +) +args = parser.parse_args() + +input_file = args.input_file +base_name = os.path.splitext(os.path.basename(input_file))[0] +output_file = f"{base_name}_summary.csv" + +fields = [ + "max_concurrency", + "input_throughput", + "output_throughput", + "mean_ttft_ms", + "median_ttft_ms", + "p99_ttft_ms", + "mean_tpot_ms", + "median_tpot_ms", + "p99_tpot_ms", +] + +# Read JSONL and parse +results = [] +with open(input_file, "r") as f: + for line in f: + data = json.loads(line) + row = {field: data.get(field, None) for field in fields} + max_conc = data.get("max_concurrency") + out_tp = data.get("output_throughput") + row["per_user_throughput"] = out_tp / max_conc if max_conc else None + results.append(row) + +# Convert to DataFrame +df = pd.DataFrame(results) + +# Save to CSV +df.to_csv(output_file, index=False) +print(f"\nSaved summary to: {output_file}\n") + +if args.md: + # Print Markdown table + print(tabulate(df, headers="keys", tablefmt="github", floatfmt=".3f")) +else: + # Print ASCII table + print(tabulate(df, headers="keys", tablefmt="grid", floatfmt=".3f")) diff --git a/test/srt/quant/test_awq.py b/test/srt/quant/test_awq.py new file mode 100644 index 000000000..99ca3f842 --- /dev/null +++ b/test/srt/quant/test_awq.py @@ -0,0 +1,45 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestAWQ(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_AWQ_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--trust-remote-code"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.64) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/quant/test_awq_dequant.py b/test/srt/quant/test_awq_dequant.py new file mode 100644 index 000000000..ec1f2b16a --- /dev/null +++ b/test/srt/quant/test_awq_dequant.py @@ -0,0 +1,175 @@ +# Adapted from https://github.com/vllm-project/vllm/blob/main/tests/kernels/quantization/test_awq_triton.py +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +unittest version of the AWQ Triton kernel tests. + +Run with: + python -m unittest test_awq_dequant.py +""" +import unittest + +import torch + +from sglang.srt.layers.quantization.awq_triton import ( + AWQ_TRITON_SUPPORTED_GROUP_SIZES, + awq_dequantize_triton, + awq_gemm_triton, +) +from sglang.test.test_utils import CustomTestCase + +device = "cuda" + + +def reverse_awq_order(t: torch.Tensor) -> torch.Tensor: + bits = 4 + AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7] + idx = torch.arange(t.shape[-1], dtype=torch.int32, device=t.device) + idx = idx.view(-1, 32 // bits)[:, AWQ_REVERSE_ORDER].view(-1) + return (t[:, idx] & 0xF).contiguous() + + +def awq_dequantize_torch( + qweight: torch.Tensor, + scales: torch.Tensor, + qzeros: torch.Tensor, + group_size: int, +) -> torch.Tensor: + if group_size == -1: + group_size = qweight.shape[0] + + bits = 4 + shifts = torch.arange(0, 32, bits, device=qzeros.device) + + iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + iweights = reverse_awq_order(iweights.view(iweights.shape[0], -1)) + + zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to( + torch.int8 + ) + zeros = reverse_awq_order(zeros.view(qzeros.shape[0], -1)) + + iweights = torch.bitwise_and(iweights, (2**bits) - 1) + zeros = torch.bitwise_and(zeros, (2**bits) - 1) + + scales = scales.repeat_interleave(group_size, dim=0) + zeros = zeros.repeat_interleave(group_size, dim=0) + return (iweights - zeros) * scales + + +class TestAWQTriton(CustomTestCase): + def test_dequantize(self): + rows_list = [3584, 18944, 128, 256, 512, 1024] + cols_list = [448, 576, 4736, 16, 32, 64, 128] + + for qweight_rows in rows_list: + for qweight_cols in cols_list: + for group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES: + with self.subTest( + rows=qweight_rows, cols=qweight_cols, g=group_size + ): + self._run_dequant_case( + qweight_rows=qweight_rows, + qweight_cols=qweight_cols, + group_size=group_size, + ) + + def _run_dequant_case(self, qweight_rows, qweight_cols, group_size): + if group_size == -1: + group_size = qweight_rows + + torch.manual_seed(0) + + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_rows, qweight_cols), + dtype=torch.int32, + device=device, + ) + scales = torch.rand( + qweight_rows // group_size, + qweight_cols * 8, + dtype=torch.float16, + device=device, + ) + zeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (qweight_rows // group_size, qweight_cols), + dtype=torch.int32, + device=device, + ) + + ref = awq_dequantize_torch(qweight, scales, zeros, group_size) + tri = awq_dequantize_triton(qweight, scales, zeros) + + # sanity + self.assertFalse(torch.any(torch.isinf(tri)) or torch.any(torch.isnan(tri))) + torch.testing.assert_close(ref, tri) + + # GEMM + def test_gemm(self): + N_list = [1, 2, 4, 8, 14, 17, 23, 32] + K_list = [128] + M_list = [16, 24, 32] + splitK_list = [1, 8] + + for N in N_list: + for K in K_list: + for M in M_list: + for group_size in AWQ_TRITON_SUPPORTED_GROUP_SIZES: + for splitK in splitK_list: + with self.subTest(N=N, K=K, M=M, g=group_size, sk=splitK): + self._run_gemm_case( + N=N, + K=K, + M=M, + group_size=group_size, + splitK=splitK, + ) + + def _run_gemm_case(self, N, K, M, group_size, splitK): + if group_size == -1: + group_size = K + + torch.manual_seed(0) + + x = torch.rand((N, K), dtype=torch.float32, device=device) + qweight = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (K, M // 8), + dtype=torch.int32, + device=device, + ) + qzeros = torch.randint( + 0, + torch.iinfo(torch.int32).max, + (K // group_size, M // 8), + dtype=torch.int32, + device=device, + ) + scales = torch.rand((K // group_size, M), dtype=torch.float32, device=device) + + tri_out = awq_gemm_triton(x, qweight, scales, qzeros, splitK) + + self.assertFalse( + torch.any(torch.isinf(tri_out)) or torch.any(torch.isnan(tri_out)) + ) + + # dequantize & compare + w_deq = awq_dequantize_triton(qweight, scales, qzeros) + ref_out = torch.matmul(x, w_deq) + + self.assertFalse( + torch.any(torch.isinf(ref_out)) or torch.any(torch.isnan(ref_out)) + ) + + torch.testing.assert_close(tri_out.cpu(), ref_out.cpu(), atol=1e-1, rtol=1e-1) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/quant/test_block_int8.py b/test/srt/quant/test_block_int8.py new file mode 100644 index 000000000..f6ceb03d0 --- /dev/null +++ b/test/srt/quant/test_block_int8.py @@ -0,0 +1,227 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.test.test_utils import CustomTestCase + + +# For test +def native_per_token_group_quant_int8(x, group_size, eps=1e-10, dtype=torch.int8): + """Function to perform per-token-group quantization on an input tensor `x` using native torch. + + It converts the tensor values into float8 values and returns the + quantized tensor along with the scaling factor used for quantization. + Note that only `torch.float8_e4m3fn` is supported for now. + """ + assert ( + x.shape[-1] % group_size == 0 + ), "the last dimension of `x` cannot be divisible by `group_size`" + assert x.is_contiguous(), "`x` is not contiguous" + + iinfo = torch.iinfo(dtype) + int8_min = iinfo.min + int8_max = iinfo.max + + x_ = x.reshape(x.numel() // group_size, group_size) + amax = x_.abs().max(dim=-1, keepdim=True)[0].clamp(min=eps).to(torch.float32) + x_s = amax / int8_max + x_q = (x_ / x_s).clamp(min=int8_min, max=int8_max).to(dtype) + x_q = x_q.reshape(x.shape) + x_s = x_s.reshape(x.shape[:-1] + (x.shape[-1] // group_size,)) + + return x_q, x_s + + +# For test +def native_w8a8_block_int8_matmul(A, B, As, Bs, block_size, output_dtype=torch.float16): + """This function performs matrix multiplication with block-wise quantization using native torch. + + It takes two input tensors `A` and `B` with scales `As` and `Bs`. + The output is returned in the specified `output_dtype`. + """ + + A = A.to(torch.float32) + B = B.to(torch.float32) + assert A.shape[-1] == B.shape[-1] + assert B.ndim == 2 and B.is_contiguous() and Bs.ndim == 2 + assert len(block_size) == 2 + block_n, block_k = block_size[0], block_size[1] + assert (A.shape[-1] + block_k - 1) // block_k == As.shape[-1] + assert A.shape[:-1] == As.shape[:-1] + + M = A.numel() // A.shape[-1] + N, K = B.shape + origin_C_shape = A.shape[:-1] + (N,) + A = A.reshape(M, A.shape[-1]) + As = As.reshape(M, As.shape[-1]) + n_tiles = (N + block_n - 1) // block_n + k_tiles = (K + block_k - 1) // block_k + assert n_tiles == Bs.shape[0] + assert k_tiles == Bs.shape[1] + + C_shape = (M, N) + C = torch.zeros(C_shape, dtype=torch.float32, device=A.device) + + A_tiles = [A[:, i * block_k : min((i + 1) * block_k, K)] for i in range(k_tiles)] + B_tiles = [ + [ + B[ + j * block_n : min((j + 1) * block_n, N), + i * block_k : min((i + 1) * block_k, K), + ] + for i in range(k_tiles) + ] + for j in range(n_tiles) + ] + C_tiles = [C[:, j * block_n : min((j + 1) * block_n, N)] for j in range(n_tiles)] + As_tiles = [As[:, i : i + 1] for i in range(k_tiles)] + + for i in range(k_tiles): + for j in range(n_tiles): + a = A_tiles[i] + b = B_tiles[j][i] + c = C_tiles[j] + s = As_tiles[i] * Bs[j][i] + c[:, :] += torch.matmul(a, b.t()) * s + + C = C.reshape(origin_C_shape).to(output_dtype) + return C + + +# For test +def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape): + """This function performs fused moe with block-wise quantization using native torch.""" + + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + _, block_k = block_shape[0], block_shape[1] + a_q, a_s = native_per_token_group_quant_int8(a, block_k) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + inter_out = native_w8a8_block_int8_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], block_shape, output_dtype=a.dtype + ) + act_out = SiluAndMul().forward_native(inter_out) + act_out_q, act_out_s = native_per_token_group_quant_int8(act_out, block_k) + act_out = act_out.to(torch.float32) + out[mask] = native_w8a8_block_int8_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], block_shape, output_dtype=a.dtype + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8BlockINT8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33, 64, 222] + N = [128, 1024] + K = [256, 4096] + E = [8, 24] + TOP_KS = [2, 6] + # BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_block_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # NOTE(HandH1998): to avoid overflow when out_dtype = torch.half + factor_for_scale = 1e-2 + int8_info = torch.iinfo(torch.int8) + int8_max, int8_min = int8_info.max, int8_info.min + + a = torch.randn((M, K), dtype=dtype) / 10 + + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 * int8_max + w1 = w1_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 * int8_max + w2 = w2_fp32.clamp(min=int8_min, max=int8_max).to(torch.int8) + + block_n, block_k = block_size[0], block_size[1] + n_tiles_w1 = (2 * N + block_n - 1) // block_n + n_tiles_w2 = (K + block_n - 1) // block_n + k_tiles_w1 = (K + block_k - 1) // block_k + k_tiles_w2 = (N + block_k - 1) // block_k + + w1_s = ( + torch.rand((E, n_tiles_w1, k_tiles_w1), dtype=torch.float32) + * factor_for_scale + ) + w2_s = ( + torch.rand((E, n_tiles_w2, k_tiles_w2), dtype=torch.float32) + * factor_for_scale + ) + + score = torch.randn((M, E), dtype=dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + with torch.inference_mode(): + ref_out = torch_w8a8_block_int8_moe( + a, w1, w2, w1_s, w2_s, score, topk, block_size + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_int8_w8a8=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=block_size, + ) + + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.02 + ) + + def test_w8a8_block_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_block_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/quant/test_fp8_kernel.py b/test/srt/quant/test_fp8_kernel.py new file mode 100644 index 000000000..42502277b --- /dev/null +++ b/test/srt/quant/test_fp8_kernel.py @@ -0,0 +1,126 @@ +import unittest + +import torch + +from sglang.srt.layers.quantization.fp8_kernel import ( + per_token_group_quant_fp8, + w8a8_block_fp8_matmul, +) +from sglang.test.test_utils import CustomTestCase + + +class TestFP8Base(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.M = 256 + # test non-aligned + cls.N = 1024 + 64 + cls.K = 512 + cls.group_size = 128 + cls.quant_type = torch.float8_e4m3fn + cls.output_type = torch.bfloat16 + + @staticmethod + def _make_A(M, K, group_size, out_dtype): + quant_A = torch.rand( + M, K // group_size, group_size, dtype=torch.float32, device="cuda" + ) + # -1 ~ 1 + quant_A = quant_A * 2 - 1 + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_A.abs().amax(-1, keepdim=True) + quant_A *= scaling + quant_A = quant_A.to(out_dtype).to(torch.float32) + + # create scale and A + scale = torch.rand(M, K // group_size, dtype=torch.float32, device="cuda") + scale /= fmax + A = quant_A * scale[..., None] + + A = A.reshape(M, K) + quant_A = quant_A.reshape(M, K).to(out_dtype) + return A, quant_A, scale + + @staticmethod + def _make_B(K, N, group_size, out_dtype): + def _aligned_size(a, b): + return (a + b - 1) // b * b + + K_aligned = _aligned_size(K, group_size) + N_aligned = _aligned_size(N, group_size) + + quant_B = torch.rand( + K_aligned // group_size, + group_size, + N_aligned // group_size, + group_size, + dtype=torch.float32, + device="cuda", + ) + quant_B = quant_B * 2 - 1 + + # scaling abs max to fmax + finfo = torch.finfo(out_dtype) + fmax = finfo.max + scaling = fmax / quant_B.abs().amax((1, 3), keepdim=True) + quant_B *= scaling + quant_B = quant_B.to(out_dtype).to(torch.float32) + + scale = torch.rand( + K_aligned // group_size, + 1, + N_aligned // group_size, + 1, + dtype=torch.float32, + device="cuda", + ) + scale /= fmax + + B = quant_B * scale + + B = B.reshape(K_aligned, N_aligned)[:K, :N] + quant_B = quant_B.reshape(K_aligned, N_aligned).to(out_dtype)[:K, :N] + scale = scale.reshape(K_aligned // group_size, N_aligned // group_size) + return B, quant_B, scale + + +class TestPerTokenGroupQuantFP8(TestFP8Base): + def test_per_token_group_quant_fp8(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + A_quant, scale = per_token_group_quant_fp8(x=A, group_size=self.group_size) + torch.testing.assert_close(scale, scale_gt) + diff = (A_quant.to(torch.float16) - A_quant_gt.to(torch.float16)).abs() + diff_count = (diff > 1e-5).count_nonzero() + assert diff_count / diff.numel() < 1e-4 + + +class TestW8A8BlockFP8Matmul(TestFP8Base): + def test_w8a8_block_fp8_matmul(self): + if torch.cuda.get_device_capability()[0] < 9: + return + A, A_quant_gt, A_scale_gt = self._make_A( + M=self.M, K=self.K, group_size=self.group_size, out_dtype=self.quant_type + ) + B, B_quant_gt, B_scale_gt = self._make_B( + K=self.K, N=self.N, group_size=self.group_size, out_dtype=self.quant_type + ) + C_gt = A.to(self.output_type) @ B.to(self.output_type) + C = w8a8_block_fp8_matmul( + A=A_quant_gt, + B=B_quant_gt.T.contiguous(), + As=A_scale_gt, + Bs=B_scale_gt.T.contiguous(), + block_size=[128, 128], + output_dtype=self.output_type, + ) + torch.testing.assert_close(C, C_gt, atol=0.5, rtol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/quant/test_fp8_kvcache.py b/test/srt/quant/test_fp8_kvcache.py new file mode 100644 index 000000000..669926f5d --- /dev/null +++ b/test/srt/quant/test_fp8_kvcache.py @@ -0,0 +1,114 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFp8KvcacheBase(CustomTestCase): + model_config = None + + @classmethod + def setUpClass(cls): + if cls.model_config is None: + raise NotImplementedError("model_config must be specified in subclass") + + cls.model = cls.model_config["model_name"] + cls.base_url = DEFAULT_URL_FOR_TEST + dirpath = os.path.dirname(__file__) + config_file = os.path.join(dirpath, cls.model_config["config_filename"]) + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--kv-cache-dtype", + "fp8_e4m3", + "--quantization-param-path", + config_file, + ], + ) + + +class TestFp8KvcacheLlama(TestFp8KvcacheBase): + model_config = { + "model_name": DEFAULT_MODEL_NAME_FOR_TEST, + "config_filename": "kv_cache_scales_llama3_8b.json", + } + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.80) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestFp8KvcacheQwen(TestFp8KvcacheBase): + model_config = { + "model_name": DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN, + "config_filename": "kv_cache_scales_qwen2_1_5b.json", + } + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.01) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/quant/test_int8_kernel.py b/test/srt/quant/test_int8_kernel.py new file mode 100644 index 000000000..dd75d06af --- /dev/null +++ b/test/srt/quant/test_int8_kernel.py @@ -0,0 +1,169 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.int8_kernel import per_token_quant_int8 +from sglang.test.test_utils import CustomTestCase + + +def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + return C.reshape(origin_C_shape).to(output_dtype) + + +def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = per_token_quant_int8(a) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + # Calculate routing + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + a_q[mask], w1[i], a_s[mask], w1_s[i], output_dtype=a.dtype + ) + # Activation function + act_out = SiluAndMul().forward_native(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = per_token_quant_int8(act_out) + + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype + ) + # Apply routing weights and sum + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8Int8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33] + N = [128, 1024] + K = [256, 4096] + E = [8] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_int8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # Initialize int8 quantization parameters + factor_for_scale = 1e-2 + int8_max = 127 + int8_min = -128 + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / 10 + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=False, # Not using fp8 + use_int8_w8a16=False, # Not using int8-w8a16 + use_int8_w8a8=True, # Using int8-w8a8 + per_channel_quant=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=None, # Not using block quantization + ) + + # Check results + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.05 + ) + + def test_w8a8_int8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_int8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/quant/test_triton_scaled_mm.py b/test/srt/quant/test_triton_scaled_mm.py new file mode 100644 index 000000000..dafde83be --- /dev/null +++ b/test/srt/quant/test_triton_scaled_mm.py @@ -0,0 +1,94 @@ +import itertools +import unittest +from typing import Optional + +import torch +import torch.testing + +from sglang.srt.layers.quantization.fp8_kernel import triton_scaled_mm +from sglang.test.test_utils import CustomTestCase + + +def torch_scaled_mm( + a: torch.Tensor, + b: torch.Tensor, + scale_a: torch.Tensor, + scale_b: torch.Tensor, + out_dtype: torch.dtype, + bias: Optional[torch.Tensor] = None, +) -> torch.Tensor: + """Reference implementation using float32 for stability""" + out = torch.mm(a.to(torch.float32), b.to(torch.float32)) + out = scale_a.to(torch.float32) * out * scale_b.to(torch.float32).T + if bias is not None: + out = out + bias.to(torch.float32) + return out.to(out_dtype) + + +class TestScaledMM(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("This test requires a CUDA device.") + torch.set_default_device("cuda") + + def _make_inputs(self, M, K, N, in_dtype): + if in_dtype == torch.int8: + a = torch.randint(-8, 8, (M, K), dtype=in_dtype, device="cuda") + b = torch.randint(-8, 8, (K, N), dtype=in_dtype, device="cuda") + else: # fp8 + a = torch.clamp( + 0.1 * torch.randn((M, K), dtype=torch.float16, device="cuda"), -0.3, 0.3 + ).to(in_dtype) + b = torch.clamp( + 0.1 * torch.randn((K, N), dtype=torch.float16, device="cuda"), -0.3, 0.3 + ).to(in_dtype) + return a, b + + def test_basic_cases(self): + """Test core functionality with reduced precision requirements""" + test_configs = [ + (32, 32, 32, torch.int8, torch.float16, False), + (64, 64, 64, torch.int8, torch.float16, True), + ] + + try: + torch.tensor([1.0], dtype=torch.float8_e4m3fn, device="cuda") + test_configs.append((32, 32, 32, torch.float8_e4m3fn, torch.float16, False)) + except: + print("FP8 not supported, skipping") + + for M, K, N, in_dtype, out_dtype, with_bias in test_configs: + with self.subTest(M=M, K=K, N=N, dtype=in_dtype, bias=with_bias): + print(f"Currently testing with in_dtype: {in_dtype}") + torch.manual_seed(42) + + input, weight = self._make_inputs(M, K, N, in_dtype) + scale_a = 0.1 + 0.05 * torch.rand( + (M, 1), dtype=torch.float32, device="cuda" + ) + scale_b = 0.1 + 0.05 * torch.rand( + (N, 1), dtype=torch.float32, device="cuda" + ) + bias = ( + 0.01 * torch.randn((M, N), dtype=out_dtype, device="cuda") + if with_bias + else None + ) + + triton_out = triton_scaled_mm( + input, weight, scale_a, scale_b, out_dtype, bias + ) + ref_out = torch_scaled_mm( + input, weight, scale_a, scale_b, out_dtype, bias + ) + + # Use relaxed tolerances + rtol = 0.15 if in_dtype == torch.int8 else 0.25 + atol = 0.1 if in_dtype == torch.int8 else 0.15 + + torch.testing.assert_close(triton_out, ref_out, rtol=rtol, atol=atol) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/quant/test_w4a8_deepseek_v3.py b/test/srt/quant/test_w4a8_deepseek_v3.py new file mode 100644 index 000000000..eb813bd70 --- /dev/null +++ b/test/srt/quant/test_w4a8_deepseek_v3.py @@ -0,0 +1,122 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + popen_launch_server, + try_cached_model, + write_github_step_summary, +) + + +class TestDeepseekV3W4afp8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--tp", "8", "--ep-size", "8"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=1200, + parallel=1200, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Eval accuracy of GSM8K: {metrics=}") + + self.assertGreater(metrics["accuracy"], 0.92) + + +class TestDeepseekV3W4Afp8Mtp(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = try_cached_model(DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--ep-size", + "8", + "--cuda-graph-bs", + "256", + "--disable-radix-cache", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k( + self, + ): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(avg_spec_accept_length, 2.9) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/quant/test_w8a8_quantization.py b/test/srt/quant/test_w8a8_quantization.py new file mode 100644 index 000000000..acb7f5c7d --- /dev/null +++ b/test/srt/quant/test_w8a8_quantization.py @@ -0,0 +1,75 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestW8A8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "neuralmagic/Meta-Llama-3-8B-Instruct-quantized.w8a8" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--quantization", "w8a8_int8"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.69) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 140 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/rl/test_update_weights_from_disk.py b/test/srt/rl/test_update_weights_from_disk.py new file mode 100644 index 000000000..02b283efd --- /dev/null +++ b/test/srt/rl/test_update_weights_from_disk.py @@ -0,0 +1,371 @@ +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests + +import sglang as sgl +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +############################################################################### +# Engine Mode Tests (Single-configuration) +############################################################################### +class TestEngineUpdateWeightsFromDisk(CustomTestCase): + def setUp(self): + self.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + # Initialize the engine in offline (direct) mode. + self.engine = sgl.Engine(model_path=self.model) + + def tearDown(self): + self.engine.shutdown() + + def run_decode(self): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = self.engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Engine Mode] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(self, model_path): + ret = self.engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + def test_update_weights(self): + origin_response = self.run_decode() + # Update weights: use new model (remove "-Instruct") + new_model_path = self.model.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret[0]) # ret is a tuple; index 0 holds the success flag + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + # Revert back to original weights + ret = self.run_update_weights(self.model) + self.assertTrue(ret[0]) + reverted_response = self.run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + + def test_update_weights_unexist_model(self): + origin_response = self.run_decode() + new_model_path = self.model.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret[0]) + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +############################################################################### +# HTTP Server Mode Tests (Single-configuration) +############################################################################### +class TestServerUpdateWeightsFromDisk(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Server Mode] Generated text: {response.json()['text']}") + return response.json()["text"] + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(self, model_path): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": model_path}, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + def test_update_weights(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + + ret = self.run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = self.get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + def test_update_weights_unexist_model(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + origin_response = self.run_decode() + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "wrong") + ret = self.run_update_weights(new_model_path) + self.assertFalse(ret["success"]) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, origin_model_path) + + updated_response = self.run_decode() + self.assertEqual(origin_response[:32], updated_response[:32]) + + +class TestServerUpdateWeightsFromDiskAbortAllRequests(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--max-running-requests", 8], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens=32): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def get_model_info(self): + response = requests.get(self.base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(self, model_path, abort_all_requests=False): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={ + "model_path": model_path, + "abort_all_requests": abort_all_requests, + }, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + def test_update_weights_abort_all_requests(self): + origin_model_path = self.get_model_info() + print(f"[Server Mode] origin_model_path: {origin_model_path}") + + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [ + executor.submit(self.run_decode, 16000) for _ in range(num_requests) + ] + + # ensure the decode has been started + time.sleep(2) + + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = self.run_update_weights(new_model_path, abort_all_requests=True) + self.assertTrue(ret["success"]) + + for future in as_completed(futures): + self.assertEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + updated_model_path = self.get_model_info() + print(f"[Server Mode] updated_model_path: {updated_model_path}") + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + + +############################################################################### +# Parameterized Tests for update_weights_from_disk +# Test coverage is determined based on the value of is_in_ci: +# - In a CI environment: randomly select one mode (Engine or Server) and test only with tp=1, dp=1. +# - In a non-CI environment: test both Engine and Server modes, and enumerate all combinations +# with tp and dp ranging from 1 to 2. +############################################################################### +class TestUpdateWeightsFromDiskParameterized(CustomTestCase): + def run_common_test(self, mode, tp, dp): + """ + Common test procedure for update_weights_from_disk. + For Engine mode, we instantiate the engine with tp_size=tp. + For Server mode, we launch the server with additional arguments for tp (dp is not used in server launch here). + """ + if mode == "Engine": + # Instantiate engine with additional parameter tp_size. + print(f"[Parameterized Engine] Testing with tp={tp}, dp={dp}") + engine = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + random_seed=42, + tp_size=tp, + # dp parameter is not explicitly used in this API. + ) + try: + origin_response = self._engine_update_weights_test(engine) + finally: + engine.shutdown() + elif mode == "Server": + print(f"[Parameterized Server] Testing with tp={tp}, dp={dp}") + # Pass additional arguments to launch the server. + base_args = ["--tp-size", str(tp)] + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=base_args, + ) + try: + origin_response = self._server_update_weights_test(DEFAULT_URL_FOR_TEST) + finally: + kill_process_tree(process.pid) + else: + raise ValueError(f"Unknown mode: {mode}") + + def _engine_update_weights_test(self, engine): + # Run the update weights test on the given engine instance. + def run_decode(): + prompts = ["The capital of France is"] + sampling_params = {"temperature": 0, "max_new_tokens": 32} + outputs = engine.generate(prompts, sampling_params) + print("=" * 100) + print( + f"[Parameterized Engine] Prompt: {prompts[0]}\nGenerated text: {outputs[0]['text']}" + ) + return outputs[0]["text"] + + def run_update_weights(model_path): + ret = engine.update_weights_from_disk(model_path) + print(json.dumps(ret)) + return ret + + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret[0]) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + self.assertTrue(ret[0]) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def _server_update_weights_test(self, base_url): + def run_decode(): + response = requests.post( + base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": {"temperature": 0, "max_new_tokens": 32}, + }, + ) + print("=" * 100) + print(f"[Parameterized Server] Generated text: {response.json()['text']}") + return response.json()["text"] + + def get_model_info(): + response = requests.get(base_url + "/get_model_info") + model_path = response.json()["model_path"] + print(json.dumps(response.json())) + return model_path + + def run_update_weights(model_path): + response = requests.post( + base_url + "/update_weights_from_disk", + json={"model_path": model_path}, + ) + ret = response.json() + print(json.dumps(ret)) + return ret + + origin_model_path = get_model_info() + origin_response = run_decode() + new_model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST.replace("-Instruct", "") + ret = run_update_weights(new_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, new_model_path) + self.assertNotEqual(updated_model_path, origin_model_path) + updated_response = run_decode() + self.assertNotEqual(origin_response[:32], updated_response[:32]) + ret = run_update_weights(origin_model_path) + self.assertTrue(ret["success"]) + updated_model_path = get_model_info() + self.assertEqual(updated_model_path, origin_model_path) + reverted_response = run_decode() + self.assertEqual(origin_response[:32], reverted_response[:32]) + return origin_response + + def test_parameterized_update_weights(self): + if is_in_ci(): + # In CI, choose one random mode (Engine or Server) with tp=1, dp=1. + mode = random.choice(["Engine", "Server"]) + test_suits = [(1, 1, mode)] + else: + # Otherwise, test both modes and enumerate tp,dp combinations from 1 to 2. + test_suits = [] + for mode in ["Engine", "Server"]: + for tp in [1, 2]: + for dp in [1, 2]: + test_suits.append((tp, dp, mode)) + for tp, dp, mode in test_suits: + with self.subTest(mode=mode, tp=tp, dp=dp): + self.run_common_test(mode, tp, dp) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/rl/test_update_weights_from_distributed.py b/test/srt/rl/test_update_weights_from_distributed.py new file mode 100644 index 000000000..a3b938c38 --- /dev/null +++ b/test/srt/rl/test_update_weights_from_distributed.py @@ -0,0 +1,615 @@ +"""Test distributed weight updates. + +This test suite simulates a distributed training environment to ensure +correct weight synchronization. On rank 0, the instruct model represents +pre-training weights, and the base model represents post-training weights. +The base model's weights are broadcasted to other ranks using the online +weight update API. + +On other ranks, an engine is initialized with the instruct model, and its +parameters are verified against the Hugging Face model. After updating +weights from the distributed system, post-training weights are loaded +and verified again to ensure consistency and accuracy across the +distributed setup. +""" + +import gc +import os +import random +import time +import unittest + +import numpy as np +import requests +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.srt.utils import init_custom_process_group +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + +mp.set_start_method("spawn", force=True) + + +def verify_params_close(params1, params2, error_msg): + """Verify if two parameter arrays are close enough.""" + try: + assert np.allclose(np.array(params1), np.array(params2)), error_msg + except Exception as e: + print(f"Parameters not close for {error_msg}") + print("Params1:", np.array(params1)) + print("Params2:", np.array(params2)) + raise e + + +def verify_params_not_close(params1, params2, error_msg): + """Verify if two parameter arrays are different enough.""" + assert not np.allclose(np.array(params1), np.array(params2)), error_msg + + +def init_process( + rank, + world_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, +): + torch.cuda.set_device(rank) + + if rank == 0: + init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + ) + elif rank in [1, 2]: + init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, + ) + + +def init_process_hf( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, +): + # These two environment variables are very important + # to avoid unexpected behaviors of CUDA and NCCL. + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["NCCL_NVLS_ENABLE"] = "0" + + # Load model and get parameters + hf_instruct_model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + base_model_name = model_name.replace("-Instruct", "") + hf_base_model = AutoModelForCausalLM.from_pretrained( + base_model_name, + torch_dtype="bfloat16", + tie_word_embeddings=tie_word_embeddings, + ).to("cuda:0") + + hf_instruct_params = [] + hf_base_params = [] + + print("[hf] get parameter in hf instruct model and base model") + for parameter_name in checking_parameters: + hf_instruct_params.append( + hf_instruct_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + hf_base_params.append( + hf_base_model.get_parameter(parameter_name)[:truncate_size] + .cpu() + .detach() + .float() + .numpy() + .tolist() + ) + + param_queue.put(("hf_instruct_params", hf_instruct_params)) + param_queue.put(("hf_base_params", hf_base_params)) + + # Init weight update group for rank 0 (the training engine in RLHF). + port = 60000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + init_method = f"tcp://localhost:{port}" + print(f"[hf] {rank=} {world_size=} init custom process group. {init_method=}") + group = init_custom_process_group( + backend="nccl", + init_method=init_method, + world_size=world_size, + rank=rank, + group_name="test_parameter_update_group", + ) + torch.cuda.synchronize() + time_begin_broadcast = time.perf_counter() + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to broadcast embed_tokens.weight once. + broadcast_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + broadcast_parameters.remove("lm_head.weight") + + # Broadcast all the weights from the training + # engine to other ranks (inference engine). + for parameter_name in broadcast_parameters: + torch.distributed.broadcast( + hf_base_model.get_parameter(parameter_name), + src=0, + group=group, + ) + torch.cuda.synchronize() + time_end_broadcast = time.perf_counter() + + # Measure the latency of broadcasting/weights update. + broadcast_time = time_end_broadcast - time_begin_broadcast + print(f"[hf] {rank=} {broadcast_time=:.3f}s") + param_queue.put(("broadcast_time", broadcast_time)) + + # Delete the huggingface models to free up memory. + del hf_instruct_model + del hf_base_model + gc.collect() + torch.cuda.empty_cache() + + +def init_process_sgl( + rank, + world_size, + param_queue, + truncate_size, + model_name, + checking_parameters, + tie_word_embeddings, + state_dict_key_to_shape, + backend, + tp_size, +): + torch.cuda.set_device(rank) + torch.cuda.synchronize() + base_gpu_id = 1 if rank == 1 else 1 + tp_size + if backend == "Engine": + print(f"[sgl] rank {rank} init engine") + engine = sgl.Engine( + model_path=model_name, + base_gpu_id=base_gpu_id, + tp_size=tp_size, + cuda_graph_max_bs=2, + ) + else: + if rank == 1: + url = DEFAULT_URL_FOR_TEST + else: + host, _, port = DEFAULT_URL_FOR_TEST.rpartition(":") + url = ":".join([host, str(int(port) + 10000)]) + + print(f"[sgl] rank {rank} init server on url: {url}") + process = popen_launch_server( + model_name, + url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--base-gpu-id", + str(base_gpu_id), + "--tp-size", + str(tp_size), + "--cuda-graph-max-bs", + 2, + ), + ) + torch.cuda.synchronize() + + # Get weights of instruct model, i.e. pre-training weights. + instruct_params = [] + for parameter_name in checking_parameters: + instruct_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + if backend == "Engine" + else requests.get( + f"{url}/get_weights_by_name", + json={"name": parameter_name, "truncate_size": truncate_size}, + ).json() + ) + + param_queue.put((f"sgl_dp_{rank}_instruct_params", instruct_params)) + + port = 60000 + int(os.environ.get("CUDA_VISIBLE_DEVICES", "0")[0]) * 100 + + # Init weight update group with the training engine. + if backend == "Engine": + engine.init_weights_update_group( + master_address="localhost", + master_port=str(port), + rank_offset=base_gpu_id, + world_size=world_size, + group_name="test_parameter_update_group", + backend="nccl", + ) + else: + requests.post( + f"{url}/init_weights_update_group", + json={ + "master_address": "localhost", + "master_port": str(port), + "rank_offset": base_gpu_id, + "world_size": world_size, + "group_name": "test_parameter_update_group", + "backend": "nccl", + }, + ) + + torch.cuda.synchronize() + time_begin_update = time.perf_counter() + + # The last parameter is lm_head.weight, which is tied + # with embed_tokens.weight. Actually, we only need + # to update embed_tokens.weight once. + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + update_parameters = list(state_dict_key_to_shape.keys()) + if tie_word_embeddings: + update_parameters.remove("lm_head.weight") + + # Get weights from the training engine and update the inference engine. + names = [parameter_name for parameter_name in update_parameters] + dtypes = [torch.bfloat16 if backend == "Engine" else "bfloat16"] * len(names) + shapes = [state_dict_key_to_shape[parameter_name] for parameter_name in names] + + if backend == "Engine": + engine.update_weights_from_distributed( + names, + dtypes=dtypes, + shapes=shapes, + group_name="test_parameter_update_group", + ) + else: + requests.post( + f"{url}/update_weights_from_distributed", + json={ + "names": names, + "dtypes": dtypes, + "shapes": shapes, + "group_name": "test_parameter_update_group", + }, + ) + torch.cuda.synchronize() + time_end_update = time.perf_counter() + + # Measure the latency of broadcast/weights update. + update_time = time_end_update - time_begin_update + print( + f"[sgl] fully update model_name {model_name} rank {rank} parameter from distributed time: {update_time:.3f}s" + ) + param_queue.put((f"update_sgl_dp_{rank}_time", update_time)) + + # Get the weights of post-training model after weights update for correctness check. + base_params = [] + for parameter_name in checking_parameters: + if backend == "Engine": + base_params.append( + engine.get_weights_by_name(parameter_name, truncate_size) + ) + else: + base_params.append( + requests.get( + f"{url}/get_weights_by_name", + json={ + "name": parameter_name, + "truncate_size": truncate_size, + }, + ).json() + ) + param_queue.put((f"sgl_dp_{rank}_base_params", base_params)) + + # Shutdown the engine or terminate the server process. + if backend == "Engine": + engine.shutdown() + else: + terminate_process(process) + + +def assert_tied_weights(params_list, message, should_be_tied): + for params in params_list: + if should_be_tied: + assert np.allclose(params[0], params[-1]), message + else: + assert not np.allclose(params[0], params[-1]), message + + +def test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + state_dict_key_to_shape, + truncate_size, + checking_parameters, +): + tie_word_embeddings = ( + True if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else False + ) + + print( + f"Testing model: {model_name} tp_size: {tp_size}, dp_size: {dp_size} backend: {backend}" + ) + param_queue = mp.Queue() + results = {} + + context = mp.spawn( + init_process, + args=( + 1 + tp_size * dp_size, + param_queue, + truncate_size, + state_dict_key_to_shape, + tp_size, + model_name, + backend, + checking_parameters, + tie_word_embeddings, + ), + nprocs=1 + dp_size, + join=False, + ) + + while len(results) < 3 * (1 + dp_size): + try: + key, value = param_queue.get(timeout=5) + results[key] = value + except Exception as e: + if all(not p.is_alive() for p in context.processes): + break + + context.join() + + if len(results) != 3 * (1 + dp_size): + raise RuntimeError( + f"Expected {3 * (1 + dp_size)} parameters but got {len(results)}" + ) + + params = { + "hf_instruct": results.get("hf_instruct_params"), + "hf_base": results.get("hf_base_params"), + "sgl_dp_1_instruct": results.get("sgl_dp_1_instruct_params"), + "sgl_dp_1_base": results.get("sgl_dp_1_base_params"), + "broadcast_time": results.get("broadcast_time"), + "update_sgl_dp_1_time": results.get("update_sgl_dp_1_time"), + } + + if dp_size == 2: + dp2_params = { + "sgl_dp_2_instruct": results.get("sgl_dp_2_instruct_params"), + "sgl_dp_2_base": results.get("sgl_dp_2_base_params"), + "update_sgl_dp_2_time": results.get("update_sgl_dp_2_time"), + } + assert all(v is not None for v in dp2_params.values()) + params.update(dp2_params) + + # Check the correctness of weights update by verifying + # the weights of instruct model and base model. + for i in range(len(params["hf_instruct"])): + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_1_instruct"][i], + f"sgl_dp_1_instruct_params rank {i}", + ) + + verify_params_close( + params["hf_base"][i], + params["sgl_dp_1_base"][i], + f"sgl_dp_1_base_params rank {i}", + ) + + verify_params_not_close( + params["hf_instruct"][i], + params["hf_base"][i], + f"hf_instruct_params rank {i}", + ) + + if dp_size == 2: + verify_params_close( + params["hf_base"][i], + params["sgl_dp_2_base"][i], + f"sgl_dp_2_base_params rank {i}", + ) + verify_params_close( + params["hf_instruct"][i], + params["sgl_dp_2_instruct"][i], + f"sgl_dp_2_instruct_params rank {i}", + ) + + assert len(params["hf_instruct"]) == len( + params["hf_base"] + ), "hf_instruct_params and hf_base_params have different lengths" + + # Check if the weights of lm_head are tied with embed_tokens. + params_to_check = [ + ( + params["hf_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["hf_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_1_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + + if dp_size == 2: + params_to_check.extend( + [ + ( + params["sgl_dp_2_instruct"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ( + params["sgl_dp_2_base"], + "lm_head.weight is not tied with embed_tokens.weight", + ), + ] + ) + + assert_tied_weights( + [params for params, _ in params_to_check], + ( + "lm_head.weight is not tied with embed_tokens.weight" + if tie_word_embeddings + else "lm_head.weight is tied with embed_tokens.weight" + ), + tie_word_embeddings, + ) + + # Time limit for broadcast and update on CI is 3 / 6 + # On local H100, it's 1 / 2 + time_limit = 3 if model_name == DEFAULT_SMALL_MODEL_NAME_FOR_TEST else 6 + + assert ( + params["broadcast_time"] < time_limit + ), f"broadcast_time exceeds time limit {time_limit}s" + + assert ( + params["update_sgl_dp_1_time"] < time_limit + ), f"update_sgl_dp_one_time exceeds time limit {time_limit}s" + + if dp_size == 2: + assert ( + params["update_sgl_dp_2_time"] < time_limit + ), f"update_sgl_dp_two_time exceeds time limit {time_limit}s" + + # Delete the context and close the parameter queue. + del context + param_queue.close() + param_queue.join_thread() + gc.collect() + torch.cuda.empty_cache() + + +class TestUpdateWeightsFromDistributed(CustomTestCase): + + def test_update_weights_from_distributed(self): + + assert torch.cuda.device_count() >= 2, "At least 2 GPUs are required" + # test_suits : tp, dp, model_name, backend + if is_in_ci(): + mode = random.choice(["Engine", "Server"]) + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, mode), + ] + else: + test_suits = [ + (1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (1, 1, DEFAULT_MODEL_NAME_FOR_TEST, "Sever"), + ] + + if torch.cuda.device_count() >= 4: + test_suits.extend( + [ + (2, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (1, 2, DEFAULT_MODEL_NAME_FOR_TEST, "Server"), + ] + ) + + if torch.cuda.device_count() >= 5: + test_suits.extend( + [ + (2, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST, "Engine"), + (2, 2, DEFAULT_MODEL_NAME_FOR_TEST, "Server"), + ] + ) + + model_state_dict_shapes = {} + test_models = [test_suit[2] for test_suit in test_suits] + + for model_name in test_models: + model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16" + ).to("cuda:0") + state_dict = model.state_dict() + state_dict_keys = list(state_dict.keys()) + model_state_dict_shapes[model_name] = { + key: state_dict[key].shape for key in state_dict_keys + } + del model + gc.collect() + torch.cuda.empty_cache() + + truncate_size = 10 + checking_parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + "lm_head.weight", + ] + + for tp_size, dp_size, model_name, backend in test_suits: + test_update_weights_from_distributed( + tp_size, + dp_size, + model_name, + backend, + model_state_dict_shapes[model_name], + truncate_size, + checking_parameters, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/rl/test_update_weights_from_tensor.py b/test/srt/rl/test_update_weights_from_tensor.py new file mode 100644 index 000000000..0dc947b60 --- /dev/null +++ b/test/srt/rl/test_update_weights_from_tensor.py @@ -0,0 +1,178 @@ +import gc +import time +import unittest + +import torch + +import sglang as sgl +from sglang.srt.weight_sync.tensor_bucket import FlattenedTensorBucket +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +def test_update_weights_from_tensor(tp_size): + assert torch.cuda.device_count() >= tp_size, f"At least {tp_size} GPUs are required" + torch.cuda.empty_cache() + + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, tp_size=tp_size) + + param_names = [f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 16)] + + _check_param(engine, param_names[0], [0.0087, -0.0214, -0.0004, 0.0039, 0.0110]) + + memory_before = torch.cuda.memory_allocated() + new_tensor = torch.full((16384, 2048), 1.5, device="cuda") + + time_start = time.perf_counter() + engine.update_weights_from_tensor([(x, new_tensor) for x in param_names]) + print(f"Time delta: {time.perf_counter() - time_start:.03f}") + + for param_name in param_names[:3]: + _check_param(engine, param_name, [1.5] * 5) + + engine.shutdown() + + del new_tensor + gc.collect() + torch.cuda.ipc_collect() + torch.cuda.empty_cache() + memory_after = torch.cuda.memory_allocated() + assert ( + memory_after <= memory_before + 1024 + ), f"Memory leak detected: {memory_after - memory_before} bytes" + + +class TestUpdateWeightsFromTensor(CustomTestCase): + def test_update_weights_from_tensor(self): + tp_sizes = [1, 2] + for tp_size in tp_sizes: + if torch.cuda.device_count() < tp_size: + continue + + with self.subTest(tp_size=tp_size): + test_update_weights_from_tensor(tp_size) + + def test_update_weights_from_tensor_load_format_direct(self): + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + write_param_names = [ + f"model.layers.{i}.self_attn.qkv_proj.weight" for i in range(6, 16) + ] + read_param_names = [ + f"model.layers.{i}.self_attn.k_proj.weight" for i in range(6, 16) + ] + + _check_param( + engine, read_param_names[0], [-0.0198, 0.0227, 0.0168, 0.0232, -0.0178] + ) + + new_tensor = torch.full((3072, 2048), 1.5) + engine.update_weights_from_tensor( + [ + (write_param_name, new_tensor.clone()) + for write_param_name in write_param_names + ], + load_format="direct", + ) + + for read_param_name in read_param_names[:3]: + _check_param(engine, read_param_name, [1.5] * 5) + + engine.shutdown() + + def test_update_weights_from_tensor_load_format_custom(self): + custom_loader_name = ( + "sglang.srt.model_executor.model_runner._model_load_weights_direct" + ) + engine = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + custom_weight_loader=[custom_loader_name], + ) + + write_param_names = [ + f"model.layers.{i}.self_attn.qkv_proj.weight" for i in range(6, 16) + ] + read_param_names = [ + f"model.layers.{i}.self_attn.k_proj.weight" for i in range(6, 16) + ] + + _check_param( + engine, read_param_names[0], [-0.0198, 0.0227, 0.0168, 0.0232, -0.0178] + ) + + new_tensor = torch.full((3072, 2048), 1.5) + engine.update_weights_from_tensor( + [ + (write_param_name, new_tensor.clone()) + for write_param_name in write_param_names + ], + load_format=custom_loader_name, + ) + + for read_param_name in read_param_names[:3]: + _check_param(engine, read_param_name, [1.5] * 5) + + engine.shutdown() + + def test_update_weights_from_tensor_load_format_flattened_bucket(self): + """Test updating weights using flattened_bucket format""" + engine = sgl.Engine(model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + # Create a small set of parameters for testing + param_names = [f"model.layers.{i}.mlp.up_proj.weight" for i in range(6, 10)] + + # Check original values + _check_param(engine, param_names[0], [0.0087, -0.0214, -0.0004, 0.0039, 0.0110]) + + # Create new tensors with different values + new_tensors = [] + for _, name in enumerate(param_names): + # Create tensors with different values for each parameter + value = 2.0 # Different value for each parameter + new_tensor = torch.full((16384, 2048), value, device="cuda") + new_tensors.append((name, new_tensor)) + + # Create a flattened bucket + flattened_bucket = FlattenedTensorBucket(named_tensors=new_tensors) + + # Extract the flattened tensor and metadata in the format expected by model_runner + flattened_tensor = flattened_bucket.get_flattened_tensor() + metadata = flattened_bucket.get_metadata() + + # Create the dict format expected by _update_weights_from_flattened_bucket + bucket_dict = {"flattened_tensor": flattened_tensor, "metadata": metadata} + + # Serialize the bucket data + from sglang.srt.utils import MultiprocessingSerializer + + serialized_bucket = MultiprocessingSerializer.serialize( + bucket_dict, output_str=True + ) + + # Create a list where each rank contains the same serialized data + # This simulates the distributed environment where each rank has the same data + serialized_bucket_list = [serialized_bucket] + + # Update weights using flattened_bucket format + time_start = time.perf_counter() + engine.update_weights_from_tensor( + named_tensors=serialized_bucket_list, load_format="flattened_bucket" + ) + update_time = time.perf_counter() - time_start + print(f"Flattened bucket update time: {update_time:.03f}") + + # Verify the weights were updated correctly + for i, param_name in enumerate(param_names): + _check_param(engine, param_name, [2.0] * 5) + + engine.shutdown() + + +def _check_param(engine, param_name, expect_values): + actual_values = torch.tensor(engine.get_weights_by_name(param_name))[0, :5] + assert torch.allclose( + actual_values, torch.tensor(expect_values), atol=0.002 + ), f"{actual_values=}" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/rl/test_verl_engine_2_gpu.py b/test/srt/rl/test_verl_engine_2_gpu.py new file mode 100644 index 000000000..40321ee3f --- /dev/null +++ b/test/srt/rl/test_verl_engine_2_gpu.py @@ -0,0 +1,277 @@ +import multiprocessing +import multiprocessing as mp +import os +import random +import traceback +import unittest +from multiprocessing import Process + +import torch +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.fsdp import CPUOffload +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision +from torch.distributed.fsdp.api import ( + ShardedStateDictConfig, + ShardingStrategy, + StateDictType, +) +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.verl_engine import VerlEngine +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import is_port_available +from sglang.test.runners import ( + HFRunner, + SRTRunner, + check_close_model_outputs, + get_dtype_str, +) +from sglang.test.test_utils import CustomTestCase, find_available_port, is_in_ci + +_MAX_NEW_TOKENS = 8 +_PROMPTS = ["1+1=2, 1+2=3, 1+3=4, 1+4=5, 1+5=", "1*1=1, 1*2=2, 1*3=3, 1*4=4, 1*5="] +_TORCH_DTYPE = torch.float16 + +# Set to false to temporarily debug issues unrelated to weight update +_ENABLE_UPDATE_WEIGHTS = True +# _ENABLE_UPDATE_WEIGHTS = False + +# TODO maybe we should add more other models? should we keep it in sync with test_generation_models.py? +ALL_MODELS = [ + dict(model_path="meta-llama/Llama-3.2-1B-Instruct"), + dict(model_path="Qwen/Qwen2-1.5B"), + dict(model_path="allenai/OLMo-1B-0724-hf"), + dict(model_path="allenai/OLMo-2-1124-7B-Instruct"), + dict( + model_path="ibm-granite/granite-3.0-2b-instruct", + prefill_tolerance=0.22, + decode_tolerance=0.22, + ), +] + + +class TestVerlEngine(CustomTestCase): + @classmethod + def setUpClass(cls): + multiprocessing.set_start_method("spawn") + + def assert_fragment_e2e_execution( + self, + index: int, + model_path: str, + mem_fraction_static: float = 0.4, + dp_size: int = 1, + tp_size: int = 2, + tight_memory: bool = False, + prefill_tolerance: float = 0.1, + decode_tolerance: float = 0.1, + ): + master_port = find_available_port(23456) + + print(f"assert_fragment_e2e_execution START {index=} {model_path=}") + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + world_size = dp_size * tp_size + for rank in range(world_size): + p = Process( + target=_run_subprocess, + kwargs=dict( + rank=rank, + dp_size=dp_size, + tp_size=tp_size, + master_port=master_port, + output_writer=output_writer, + model_path=model_path, + mem_fraction_static=mem_fraction_static, + tight_memory=tight_memory, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + ), + ) + p.start() + processes.append(p) + + for _ in range(tp_size): + self.assertTrue( + output_reader.recv(), + f"Subprocess has error, please see logs above. ({index=} {model_path=})", + ) + + for p in processes: + p.join() + + def test_ci_models(self): + ci_models = [random.choice(ALL_MODELS)] + for index, model_info in enumerate(ci_models): + self.assert_fragment_e2e_execution(index=index, **model_info) + + def test_others(self): + if is_in_ci(): + return + + for index, model_info in enumerate(ALL_MODELS): + self.assert_fragment_e2e_execution(index=index, **model_info) + + # def test_adhoc(self): + # self.assert_fragment_e2e_execution(index=0, model_path="meta-llama/Llama-3.2-1B-Instruct") + + +def _run_subprocess( + rank: int, + dp_size: int, + tp_size: int, + master_port: int, + output_writer, + model_path: str, + mem_fraction_static: float, + tight_memory: bool, + prefill_tolerance: float, + decode_tolerance: float, +): + try: + print(f"subprocess[{rank=}] Start {os.environ.get('CUDA_VISIBLE_DEVICES')=}") + + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + torch.distributed.init_process_group(rank=rank, world_size=dp_size * tp_size) + torch.cuda.set_device(rank) + + base_gpu_id = rank // tp_size * tp_size + + mesh_kwargs = dict( + mesh_shape=(dp_size, tp_size, 1), mesh_dim_names=["dp", "tp", "pp"] + ) + inference_device_mesh_device = init_device_mesh("cuda", **mesh_kwargs) + inference_device_mesh_cpu = init_device_mesh("cpu", **mesh_kwargs) + print( + f"subprocess[{rank=},{base_gpu_id=}] {inference_device_mesh_device=} {inference_device_mesh_cpu=}" + ) + + # hf model is used for comparison + hf_model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=_TORCH_DTYPE, trust_remote_code=True + ).cuda() + hf_tokenizer = get_tokenizer(model_path, trust_remote_code=True) + + hf_outputs = HFRunner.forward_generation_raw( + base_model=hf_model, + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + tokenizer=hf_tokenizer, + lora_paths=None, + torch_dtype=_TORCH_DTYPE, + output_str_only=False, + ) + print( + f"subprocess[{rank=}] call hf.forward {hf_outputs=}", + flush=True, + ) + + if _ENABLE_UPDATE_WEIGHTS: + if tight_memory: + hf_model.cpu() + torch.cuda.empty_cache() + + # test update weights + print(f"subprocess[{rank=}] get_fsdp_state_dict", flush=True) + fsdp_state_dict = _get_fsdp_state_dict( + hf_model=hf_model, world_size=dp_size * tp_size + ) + + engine = VerlEngine( + model_path=model_path, + load_format="dummy" if _ENABLE_UPDATE_WEIGHTS else "auto", + mem_fraction_static=mem_fraction_static, + random_seed=42, + base_gpu_id=base_gpu_id, + trust_remote_code=True, + dtype=get_dtype_str(_TORCH_DTYPE), + device_mesh_cpu=inference_device_mesh_cpu["tp"], + ) + print(f"subprocess[{rank=}] {engine=}", flush=True) + + if _ENABLE_UPDATE_WEIGHTS: + print(f"subprocess[{rank=}] call update_weights_from_tensor", flush=True) + engine.update_weights_from_tensor( + [(k, v) for k, v in fsdp_state_dict.items()] + ) + + for enable_batch in [False, True]: + if enable_batch: + fn = SRTRunner.batch_forward_generation_raw + else: + fn = SRTRunner.forward_generation_raw + + srt_outputs = fn( + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + lora_paths=None, + engine=engine, + ) + print( + f"subprocess[{rank=}] call srt.forward {enable_batch=} {srt_outputs=}", + flush=True, + ) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + rouge_l_tolerance=1, + check_logprobs=not enable_batch, + debug_text=f"{enable_batch=} {rank=}", + ) + + execution_ok = True + + except Exception as e: + print(f"subprocess[{rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + if "engine" in locals() and engine is not None: + engine.shutdown() + print(f"subprocess[{rank=}] end", flush=True) + + +# Adapted from https://github.com/volcengine/verl/blob/main/tests/rollout/run_fsdp_vllm.py +def _get_fsdp_state_dict(hf_model, world_size: int): + device_mesh = init_device_mesh( + "cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"] + ) + + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + buffer_dtype=torch.float32, + ) + fsdp_model = FSDP( + hf_model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + print(f"{fsdp_model=}") + + FSDP.set_state_dict_type( + fsdp_model, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + return fsdp_model.state_dict() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/rl/test_verl_engine_4_gpu.py b/test/srt/rl/test_verl_engine_4_gpu.py new file mode 100644 index 000000000..014f17daf --- /dev/null +++ b/test/srt/rl/test_verl_engine_4_gpu.py @@ -0,0 +1,291 @@ +import multiprocessing +import multiprocessing as mp +import os +import random +import traceback +import unittest +from multiprocessing import Process + +import torch +from torch.distributed.device_mesh import init_device_mesh +from torch.distributed.fsdp import CPUOffload +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp import MixedPrecision +from torch.distributed.fsdp.api import ( + ShardedStateDictConfig, + ShardingStrategy, + StateDictType, +) +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.verl_engine import VerlEngine +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import is_port_available +from sglang.test.runners import ( + HFRunner, + SRTRunner, + check_close_model_outputs, + get_dtype_str, +) +from sglang.test.test_utils import CustomTestCase, find_available_port, is_in_ci + +_MAX_NEW_TOKENS = 8 +_PROMPTS = ["1+1=2, 1+2=3, 1+3=4, 1+4=5, 1+5=", "1*1=1, 1*2=2, 1*3=3, 1*4=4, 1*5="] +_TORCH_DTYPE = torch.float16 + +# Set to false to temporarily debug issues unrelated to weight update +_ENABLE_UPDATE_WEIGHTS = True +# _ENABLE_UPDATE_WEIGHTS = False + +# TODO maybe we should add more other models? should we keep it in sync with test_generation_models.py? +ALL_MODELS = [ + dict( + model_path="Qwen/Qwen2.5-0.5B", + dp_size=2, + tp_size=2, # default to 2 + ), + dict( + model_path="Qwen/Qwen2.5-14B-Instruct", + mem_fraction_static=0.7, + dp_size=2, + tp_size=2, + tight_memory=True, + decode_tolerance=1.3, + ), # test_generation_models.py same config (qwen + tp=8) gives 1.22 decode error + dict( + model_path="THUDM/glm-4-9b-chat", + mem_fraction_static=0.5, + dp_size=2, + tp_size=2, + tight_memory=True, + ), + # Fail to run these models in test_generation_models.py, need to fix that first + # dict(model_path="openai-community/gpt2"), + # dict(model_path="microsoft/Phi-3-small-8k-instruct"), +] + + +class TestVerlEngine(CustomTestCase): + @classmethod + def setUpClass(cls): + multiprocessing.set_start_method("spawn") + + def assert_fragment_e2e_execution( + self, + index: int, + model_path: str, + mem_fraction_static: float = 0.4, + dp_size: int = 1, + tp_size: int = 2, + tight_memory: bool = False, + prefill_tolerance: float = 0.1, + decode_tolerance: float = 0.1, + ): + master_port = find_available_port(23456) + + print(f"assert_fragment_e2e_execution START {index=} {model_path=}") + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + world_size = dp_size * tp_size + for rank in range(world_size): + p = Process( + target=_run_subprocess, + kwargs=dict( + rank=rank, + dp_size=dp_size, + tp_size=tp_size, + master_port=master_port, + output_writer=output_writer, + model_path=model_path, + mem_fraction_static=mem_fraction_static, + tight_memory=tight_memory, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + ), + ) + p.start() + processes.append(p) + + for _ in range(tp_size): + self.assertTrue( + output_reader.recv(), + f"Subprocess has error, please see logs above. ({index=} {model_path=})", + ) + + for p in processes: + p.join() + + def test_ci_models(self): + ci_models = [random.choice(ALL_MODELS)] + for index, model_info in enumerate(ci_models): + self.assert_fragment_e2e_execution(index=index, **model_info) + + def test_others(self): + if is_in_ci(): + return + + for index, model_info in enumerate(ALL_OTHER_MODELS): + self.assert_fragment_e2e_execution(index=index, **model_info) + + # def test_adhoc(self): + # self.assert_fragment_e2e_execution(index=0, model_path="meta-llama/Llama-3.2-1B-Instruct") + + +def _run_subprocess( + rank: int, + dp_size: int, + tp_size: int, + master_port: int, + output_writer, + model_path: str, + mem_fraction_static: float, + tight_memory: bool, + prefill_tolerance: float, + decode_tolerance: float, +): + try: + print(f"subprocess[{rank=}] Start {os.environ.get('CUDA_VISIBLE_DEVICES')=}") + + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + torch.distributed.init_process_group(rank=rank, world_size=dp_size * tp_size) + torch.cuda.set_device(rank) + + base_gpu_id = rank // tp_size * tp_size + + mesh_kwargs = dict( + mesh_shape=(dp_size, tp_size, 1), mesh_dim_names=["dp", "tp", "pp"] + ) + inference_device_mesh_device = init_device_mesh("cuda", **mesh_kwargs) + inference_device_mesh_cpu = init_device_mesh("cpu", **mesh_kwargs) + print( + f"subprocess[{rank=},{base_gpu_id=}] {inference_device_mesh_device=} {inference_device_mesh_cpu=}" + ) + + # hf model is used for comparison + hf_model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=_TORCH_DTYPE, trust_remote_code=True + ).cuda() + hf_tokenizer = get_tokenizer(model_path, trust_remote_code=True) + + hf_outputs = HFRunner.forward_generation_raw( + base_model=hf_model, + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + tokenizer=hf_tokenizer, + lora_paths=None, + torch_dtype=_TORCH_DTYPE, + output_str_only=False, + ) + print( + f"subprocess[{rank=}] call hf.forward {hf_outputs=}", + flush=True, + ) + + if _ENABLE_UPDATE_WEIGHTS: + if tight_memory: + hf_model.cpu() + torch.cuda.empty_cache() + + # test update weights + print(f"subprocess[{rank=}] get_fsdp_state_dict", flush=True) + fsdp_state_dict = _get_fsdp_state_dict( + hf_model=hf_model, world_size=dp_size * tp_size + ) + + engine = VerlEngine( + model_path=model_path, + load_format="dummy" if _ENABLE_UPDATE_WEIGHTS else "auto", + mem_fraction_static=mem_fraction_static, + random_seed=42, + base_gpu_id=base_gpu_id, + trust_remote_code=True, + dtype=get_dtype_str(_TORCH_DTYPE), + device_mesh_cpu=inference_device_mesh_cpu["tp"], + ) + print(f"subprocess[{rank=}] {engine=}", flush=True) + + if _ENABLE_UPDATE_WEIGHTS: + print(f"subprocess[{rank=}] call update_weights_from_tensor", flush=True) + engine.update_weights_from_tensor( + [(k, v) for k, v in fsdp_state_dict.items()] + ) + + for enable_batch in [False, True]: + if enable_batch: + fn = SRTRunner.batch_forward_generation_raw + else: + fn = SRTRunner.forward_generation_raw + + srt_outputs = fn( + prompts=_PROMPTS, + max_new_tokens=_MAX_NEW_TOKENS, + lora_paths=None, + engine=engine, + ) + print( + f"subprocess[{rank=}] call srt.forward {enable_batch=} {srt_outputs=}", + flush=True, + ) + + check_close_model_outputs( + hf_outputs=hf_outputs, + srt_outputs=srt_outputs, + prefill_tolerance=prefill_tolerance, + decode_tolerance=decode_tolerance, + rouge_l_tolerance=1, + check_logprobs=not enable_batch, + debug_text=f"{enable_batch=} {rank=}", + ) + + execution_ok = True + + except Exception as e: + print(f"subprocess[{rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + if "engine" in locals() and engine is not None: + engine.shutdown() + print(f"subprocess[{rank=}] end", flush=True) + + +# Adapted from https://github.com/volcengine/verl/blob/main/tests/rollout/run_fsdp_vllm.py +def _get_fsdp_state_dict(hf_model, world_size: int): + device_mesh = init_device_mesh( + "cuda", mesh_shape=(world_size,), mesh_dim_names=["fsdp"] + ) + + mixed_precision = MixedPrecision( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + buffer_dtype=torch.float32, + ) + fsdp_model = FSDP( + hf_model, + use_orig_params=True, + auto_wrap_policy=None, + device_id=torch.cuda.current_device(), + sharding_strategy=ShardingStrategy.FULL_SHARD, + mixed_precision=mixed_precision, + cpu_offload=CPUOffload(offload_params=False), + sync_module_states=False, + device_mesh=device_mesh, + ) + print(f"{fsdp_model=}") + + FSDP.set_state_dict_type( + fsdp_model, + state_dict_type=StateDictType.SHARDED_STATE_DICT, + state_dict_config=ShardedStateDictConfig(), + ) + + return fsdp_model.state_dict() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py new file mode 100644 index 000000000..593920d9d --- /dev/null +++ b/test/srt/run_suite.py @@ -0,0 +1,410 @@ +import argparse +import glob +from dataclasses import dataclass + +from sglang.test.test_utils import run_unittest_files + + +@dataclass +class TestFile: + name: str + estimated_time: float = 60 + + +suites = { + "per-commit": [ + TestFile("hicache/test_hicache.py", 116), + TestFile("hicache/test_hicache_mla.py", 127), + TestFile("hicache/test_hicache_storage.py", 127), + TestFile("lora/test_lora.py", 200), + TestFile("lora/test_lora_eviction.py", 200), + TestFile("lora/test_lora_backend.py", 99), + TestFile("lora/test_multi_lora_backend.py", 60), + TestFile("lora/test_lora_cuda_graph.py", 250), + TestFile("lora/test_lora_update.py", 400), + TestFile("lora/test_lora_qwen3.py", 97), + TestFile("lora/test_lora_radix_cache.py", 100), + TestFile("models/test_embedding_models.py", 73), + # TestFile("models/test_clip_models.py", 52), + TestFile("models/test_encoder_embedding_models.py", 100), + TestFile("models/test_cross_encoder_models.py", 100), + TestFile("models/test_compressed_tensors_models.py", 42), + TestFile("models/test_generation_models.py", 103), + # TestFile("models/test_gme_qwen_models.py", 45), + # TestFile("models/test_grok_models.py", 60), # Disabled due to illegal memory access + TestFile("models/test_qwen_models.py", 82), + TestFile("models/test_reward_models.py", 132), + TestFile("models/test_vlm_models.py", 437), + TestFile("models/test_transformers_models.py", 320), + TestFile("openai_server/basic/test_protocol.py", 10), + TestFile("openai_server/basic/test_serving_chat.py", 10), + TestFile("openai_server/basic/test_serving_completions.py", 10), + TestFile("openai_server/basic/test_serving_embedding.py", 10), + TestFile("openai_server/basic/test_openai_embedding.py", 141), + TestFile("openai_server/basic/test_openai_server.py", 149), + TestFile("openai_server/features/test_enable_thinking.py", 70), + TestFile("openai_server/features/test_json_constrained.py", 98), + TestFile("openai_server/features/test_json_mode.py", 90), + TestFile("openai_server/features/test_openai_server_ebnf.py", 95), + TestFile("openai_server/features/test_openai_server_hidden_states.py", 240), + TestFile("openai_server/features/test_reasoning_content.py", 89), + TestFile("openai_server/function_call/test_openai_function_calling.py", 60), + TestFile("openai_server/function_call/test_tool_choice.py", 226), + TestFile("openai_server/validation/test_large_max_new_tokens.py", 41), + TestFile("openai_server/validation/test_matched_stop.py", 60), + TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85), + TestFile("openai_server/validation/test_request_length_validation.py", 31), + TestFile("quant/test_block_int8.py", 22), + TestFile("quant/test_fp8_kernel.py", 8), + TestFile("quant/test_int8_kernel.py", 8), + TestFile("quant/test_triton_scaled_mm.py", 8), + TestFile("quant/test_w8a8_quantization.py", 46), + TestFile("rl/test_update_weights_from_disk.py", 114), + TestFile("rl/test_update_weights_from_tensor.py", 48), + TestFile("test_abort.py", 51), + TestFile("test_create_kvindices.py", 2), + TestFile("test_chunked_prefill.py", 313), + TestFile("test_eagle_infer_a.py", 370), + TestFile("test_eagle_infer_b.py", 700), + TestFile("test_ebnf_constrained.py", 108), + TestFile("test_eval_fp8_accuracy.py", 303), + TestFile("test_fa3.py", 376), + # TestFile("test_flashmla.py", 352), + TestFile("test_function_call_parser.py", 10), + TestFile("test_fused_moe.py", 30), + TestFile("test_gpt_oss_1gpu.py", 600), + TestFile("test_harmony_parser.py", 20), + TestFile("test_hidden_states.py", 55), + TestFile("test_hybrid_attn_backend.py", 100), + TestFile("test_standalone_speculative_decoding.py", 250), + TestFile("test_input_embeddings.py", 38), + TestFile("test_io_struct.py", 8), + TestFile("test_jinja_template_utils.py", 1), + TestFile("test_metrics.py", 32), + TestFile("test_metrics_utils.py", 1), + TestFile("test_mla.py", 167), + TestFile("test_mla_deepseek_v3.py", 700), + TestFile("test_mla_int8_deepseek_v3.py", 429), + TestFile("test_mla_flashinfer.py", 302), + TestFile("test_mla_fp8.py", 93), + TestFile("test_multi_tokenizer.py", 230), + TestFile("test_no_chunked_prefill.py", 108), + TestFile("test_no_overlap_scheduler.py", 234), + TestFile("test_original_logprobs.py", 200), + TestFile("test_penalty.py", 41), + TestFile("test_page_size.py", 60), + TestFile("test_pytorch_sampling_backend.py", 66), + TestFile("test_radix_attention.py", 105), + TestFile("test_regex_constrained.py", 64), + TestFile("test_reasoning_parser.py", 5), + TestFile("test_retract_decode.py", 54), + TestFile("test_request_queue_validation.py", 30), + TestFile("test_server_args.py", 1), + TestFile("test_skip_tokenizer_init.py", 117), + TestFile("test_srt_engine.py", 261), + TestFile("test_srt_endpoint.py", 130), + TestFile("test_start_profile.py", 60), + TestFile("test_torch_compile.py", 76), + TestFile("test_torch_compile_moe.py", 172), + TestFile("test_torch_native_attention_backend.py", 123), + TestFile("test_torchao.py", 70), + TestFile("test_triton_attention_kernels.py", 4), + TestFile("test_triton_attention_backend.py", 150), + TestFile("test_triton_moe_channel_fp8_kernel.py", 25), + TestFile("test_triton_sliding_window.py", 250), + TestFile("test_utils_update_weights.py", 48), + TestFile("test_vision_chunked_prefill.py", 175), + TestFile("test_vlm_input_format.py", 300), + TestFile("test_vision_openai_server_a.py", 403), + TestFile("test_vision_openai_server_b.py", 446), + ], + "per-commit-2-gpu": [ + TestFile("lora/test_lora_tp.py", 116), + TestFile("rl/test_update_weights_from_distributed.py", 103), + TestFile("test_data_parallelism.py", 73), + TestFile("test_dp_attention.py", 277), + TestFile("test_patch_torch.py", 19), + TestFile("test_release_memory_occupation.py", 127), + TestFile("hicache/test_hicache_storage_file_backend.py", 400), + TestFile("hicache/test_hicache_storage_3fs_backend.py", 400), + ], + "per-commit-4-gpu": [ + TestFile("test_gpt_oss_4gpu.py", 600), + TestFile("test_local_attn.py", 250), + TestFile("test_pp_single_node.py", 372), + TestFile("test_multi_instance_release_memory_occupation.py", 64), + ], + "per-commit-8-gpu": [ + # Disabled because it hangs on the CI. + # TestFile("ep/test_moe_ep.py", 181), + TestFile("lora/test_lora_llama4.py", 600), + TestFile("test_disaggregation.py", 499), + TestFile("test_disaggregation_different_tp.py", 155), + TestFile("test_disaggregation_pp.py", 60), + TestFile("test_full_deepseek_v3.py", 333), + ], + "per-commit-8-gpu-b200": [ + # add more here + TestFile("test_gpt_oss_4gpu.py", 600), + ], + "per-commit-4-gpu-deepep": [ + TestFile("ep/test_deepep_small.py", 531), + ], + "per-commit-8-gpu-deepep": [ + TestFile("ep/test_deepep_large.py", 338), + ], + "per-commit-8-gpu-h20": [ + TestFile("quant/test_w4a8_deepseek_v3.py", 371), + ], + "nightly": [ + TestFile("test_nightly_gsm8k_eval.py"), + ], + "vllm_dependency_test": [ + TestFile("quant/test_awq.py", 163), + TestFile("test_bnb.py", 5), + TestFile("test_gptqmodel_dynamic.py", 102), + TestFile("test_vllm_dependency.py", 185), + # TestFile("test_gguf.py", 96), + ], +} + +# Add AMD tests +suite_amd = { + "per-commit-amd": [ + TestFile("hicache/test_hicache.py", 116), + TestFile("hicache/test_hicache_mla.py", 127), + TestFile("hicache/test_hicache_storage.py", 127), + TestFile("lora/test_lora.py", 200), + TestFile("lora/test_lora_eviction.py", 200), + TestFile("lora/test_lora_backend.py", 99), + TestFile("lora/test_multi_lora_backend.py", 60), + TestFile("lora/test_lora_cuda_graph.py", 250), + TestFile("lora/test_lora_qwen3.py", 97), + TestFile("models/test_embedding_models.py", 73), + TestFile("models/test_compressed_tensors_models.py", 42), + TestFile("models/test_qwen_models.py", 82), + TestFile("models/test_reward_models.py", 132), + TestFile("models/test_transformers_models.py", 320), + TestFile("openai_server/basic/test_protocol.py", 10), + TestFile("openai_server/basic/test_serving_chat.py", 10), + TestFile("openai_server/basic/test_serving_completions.py", 10), + TestFile("openai_server/basic/test_serving_embedding.py", 10), + TestFile("openai_server/basic/test_openai_embedding.py", 141), + TestFile("openai_server/basic/test_openai_server.py", 149), + TestFile("openai_server/features/test_enable_thinking.py", 70), + TestFile("openai_server/features/test_json_constrained.py", 98), + TestFile("openai_server/features/test_json_mode.py", 90), + TestFile("openai_server/features/test_openai_server_ebnf.py", 95), + # TestFile("openai_server/features/test_openai_server_hidden_states.py", 240), + TestFile("openai_server/features/test_reasoning_content.py", 89), + TestFile("openai_server/function_call/test_openai_function_calling.py", 60), + TestFile("openai_server/function_call/test_tool_choice.py", 226), + TestFile("openai_server/validation/test_large_max_new_tokens.py", 41), + TestFile("openai_server/validation/test_matched_stop.py", 60), + TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85), + TestFile("openai_server/validation/test_request_length_validation.py", 31), + TestFile("quant/test_block_int8.py", 22), + TestFile("quant/test_awq_dequant.py", 2), + TestFile("rl/test_update_weights_from_disk.py", 114), + # TestFile("rl/test_update_weights_from_tensor.py", 48), + TestFile("test_abort.py", 51), + TestFile("test_create_kvindices.py", 2), + TestFile("test_chunked_prefill.py", 313), + TestFile("test_ebnf_constrained.py", 108), + TestFile("test_eval_fp8_accuracy.py", 303), + TestFile("test_function_call_parser.py", 10), + TestFile("test_fused_moe.py", 30), + TestFile("test_input_embeddings.py", 38), + TestFile("test_io_struct.py", 8), + TestFile("test_jinja_template_utils.py", 1), + TestFile("test_metrics.py", 32), + TestFile("test_metrics_utils.py", 1), + TestFile("test_mla.py", 242), + TestFile("test_mla_deepseek_v3.py", 221), + TestFile("test_no_chunked_prefill.py", 108), + # TestFile("test_no_overlap_scheduler.py", 234), # Disabled temporarily and track in #7703 + TestFile("test_penalty.py", 41), + TestFile("test_page_size.py", 60), + TestFile("test_pytorch_sampling_backend.py", 66), + TestFile("test_radix_attention.py", 105), + TestFile("test_regex_constrained.py", 64), + TestFile("test_retract_decode.py", 54), + TestFile("test_reasoning_parser.py", 5), + TestFile("test_rope_rocm.py", 3), + TestFile("test_server_args.py", 1), + TestFile("test_skip_tokenizer_init.py", 117), + TestFile("test_srt_engine.py", 261), + TestFile("test_srt_endpoint.py", 130), + TestFile("test_torch_compile.py", 76), + TestFile("test_torch_compile_moe.py", 172), + TestFile("test_torch_native_attention_backend.py", 123), + TestFile("test_triton_attention_backend.py", 150), + # TestFile("test_vision_chunked_prefill.py", 175), # Disabled temporarily and track in #7701 + TestFile("test_wave_attention_kernels.py", 2), + TestFile("test_wave_attention_backend.py", 150), + ], + "per-commit-amd-mi35x": [ + TestFile("test_mla.py", 242), + TestFile("test_gpt_oss_1gpu.py", 600), + ], + "per-commit-2-gpu-amd": [ + TestFile("lora/test_lora_tp.py", 116), + TestFile("rl/test_update_weights_from_distributed.py", 103), + TestFile("test_data_parallelism.py", 73), + TestFile("test_patch_torch.py", 19), + ], + "per-commit-4-gpu-amd": [ + TestFile("test_pp_single_node.py", 150), + ], + "per-commit-8-gpu-amd": [ + TestFile("test_full_deepseek_v3.py", 250), + ], + "nightly-amd": [ + TestFile("test_nightly_gsm8k_eval_amd.py"), + ], +} + +# Add Intel Xeon tests +suite_xeon = { + "per-commit-cpu": [ + TestFile("cpu/test_activation.py"), + TestFile("cpu/test_binding.py"), + TestFile("cpu/test_decode.py"), + TestFile("cpu/test_extend.py"), + TestFile("cpu/test_gemm.py"), + TestFile("cpu/test_mla.py"), + TestFile("cpu/test_moe.py"), + TestFile("cpu/test_norm.py"), + TestFile("cpu/test_qkv_proj_with_rope.py"), + TestFile("cpu/test_rope.py"), + TestFile("cpu/test_shared_expert.py"), + TestFile("cpu/test_topk.py"), + TestFile("test_intel_amx_attention_backend.py"), + TestFile("test_cpu_graph.py"), + ], +} + +# Add Ascend NPU tests +suite_ascend = { + "per-commit-1-ascend-npu": [ + TestFile("ascend/test_ascend_tp1_bf16.py", 400), + TestFile("ascend/test_ascend_graph_tp1_bf16.py", 400), + ], + "per-commit-2-ascend-npu": [ + TestFile("ascend/test_ascend_tp2_bf16.py", 400), + TestFile("ascend/test_ascend_graph_tp2_bf16.py", 400), + TestFile("ascend/test_ascend_tp2_fia_bf16.py", 400), + TestFile("ascend/test_ascend_mla_fia_w8a8int8.py", 400), + ], + "per-commit-4-ascend-npu": [ + TestFile("ascend/test_ascend_mla_w8a8int8.py", 400), + TestFile("ascend/test_ascend_tp4_bf16.py", 400), + ], + "per-commit-16-ascend-a3": [ + TestFile("ascend/test_ascend_deepep.py", 400), + ], +} + +suites.update(suite_amd) +suites.update(suite_xeon) +suites.update(suite_ascend) + + +def auto_partition(files, rank, size): + """ + Partition files into size sublists with approximately equal sums of estimated times + using stable sorting, and return the partition for the specified rank. + + Args: + files (list): List of file objects with estimated_time attribute + rank (int): Index of the partition to return (0 to size-1) + size (int): Number of partitions + + Returns: + list: List of file objects in the specified rank's partition + """ + weights = [f.estimated_time for f in files] + + if not weights or size <= 0 or size > len(weights): + return [] + + # Create list of (weight, original_index) tuples + # Using negative index as secondary key to maintain original order for equal weights + indexed_weights = [(w, -i) for i, w in enumerate(weights)] + # Stable sort in descending order by weight + # If weights are equal, larger (negative) index comes first (i.e., earlier original position) + indexed_weights = sorted(indexed_weights, reverse=True) + + # Extract original indices (negate back to positive) + indexed_weights = [(w, -i) for w, i in indexed_weights] + + # Initialize partitions and their sums + partitions = [[] for _ in range(size)] + sums = [0.0] * size + + # Greedy approach: assign each weight to partition with smallest current sum + for weight, idx in indexed_weights: + # Find partition with minimum sum + min_sum_idx = sums.index(min(sums)) + partitions[min_sum_idx].append(idx) + sums[min_sum_idx] += weight + + # Return the files corresponding to the indices in the specified rank's partition + indices = partitions[rank] + return [files[i] for i in indices] + + +if __name__ == "__main__": + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + "--timeout-per-file", + type=int, + default=1800, + help="The time limit for running one file in seconds.", + ) + arg_parser.add_argument( + "--suite", + type=str, + default=list(suites.keys())[0], + choices=list(suites.keys()) + ["all"], + help="The suite to run", + ) + arg_parser.add_argument( + "--range-begin", + type=int, + default=0, + help="The begin index of the range of the files to run.", + ) + arg_parser.add_argument( + "--range-end", + type=int, + default=None, + help="The end index of the range of the files to run.", + ) + arg_parser.add_argument( + "--auto-partition-id", + type=int, + help="Use auto load balancing. The part id.", + ) + arg_parser.add_argument( + "--auto-partition-size", + type=int, + help="Use auto load balancing. The number of parts.", + ) + args = arg_parser.parse_args() + print(f"{args=}") + + if args.suite == "all": + files = glob.glob("**/test_*.py", recursive=True) + else: + files = suites[args.suite] + + if args.auto_partition_size: + files = auto_partition(files, args.auto_partition_id, args.auto_partition_size) + else: + files = files[args.range_begin : args.range_end] + + print("The running tests are ", [f.name for f in files]) + + exit_code = run_unittest_files(files, args.timeout_per_file) + exit(exit_code) diff --git a/test/srt/test_abort.py b/test/srt/test_abort.py new file mode 100644 index 000000000..591c21674 --- /dev/null +++ b/test/srt/test_abort.py @@ -0,0 +1,114 @@ +import json +import multiprocessing +import time +import unittest +from concurrent.futures import ThreadPoolExecutor, as_completed + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_and_check_memory_leak, +) + + +class TestAbort(CustomTestCase): + def workload_func(self, base_url, model): + def process_func(): + def run_one(_): + prompt = """ + System: You are a helpful assistant. + User: What is the capital of France? + Assistant: The capital of France is + """ + + response = requests.post( + f"{base_url}/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 2048, + }, + }, + ) + ret = response.json() + + with ThreadPoolExecutor(16) as executor: + list(executor.map(run_one, list(range(16)))) + + p = multiprocessing.Process(target=process_func) + p.start() + time.sleep(0.5) + p.terminate() + time.sleep(10) + + def test_memory_leak(self): + run_and_check_memory_leak( + self.workload_func, + disable_radix_cache=False, + enable_mixed_chunk=False, + disable_overlap=False, + chunked_prefill_size=8192, + assert_has_abort=True, + ) + + +class TestAbortAll(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--max-running-requests", 8], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _run_decode(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16000, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_abort_all(self): + num_requests = 32 + with ThreadPoolExecutor(num_requests) as executor: + futures = [executor.submit(self._run_decode) for _ in range(num_requests)] + + # ensure the decode has been started + time.sleep(2) + + requests.post( + self.base_url + "/abort_request", + json={ + "abort_all": True, + }, + ) + + for future in as_completed(futures): + self.assertEqual( + future.result()["meta_info"]["finish_reason"]["type"], "abort" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_bench_one_batch.py b/test/srt/test_bench_one_batch.py new file mode 100644 index 000000000..7ec33a559 --- /dev/null +++ b/test/srt/test_bench_one_batch.py @@ -0,0 +1,73 @@ +import os +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + run_bench_offline_throughput, + run_bench_one_batch, + write_github_step_summary, +) + +# We use `run_bench_offline_throughput`` instead of `run_bench_one_batch` for most cases +# because `run_bench_offline_throughput`` has overlap scheduler. + + +class TestBenchOneBatch(CustomTestCase): + + def test_bs1_small(self): + _, output_throughput, _ = run_bench_one_batch( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + ) + self.assertGreater(output_throughput, 50) + + def test_bs1_default(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, ["--cuda-graph-max-bs", "2"] + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_bs1_default (llama-3.1-8b)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + self.assertGreater(output_throughput, 135) + + def test_moe_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, ["--tp", "2", "--cuda-graph-max-bs", "2"] + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_tp2_bs1 (Mixtral-8x7B)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(output_throughput, 85) + else: + self.assertGreater(output_throughput, 125) + + def test_torch_compile_tp2_bs1(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_torch_compile_tp2_bs1 (Mixtral-8x7B)\n" + f"output_throughput: {output_throughput:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(output_throughput, 200) + else: + self.assertGreater(output_throughput, 220) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_bench_serving.py b/test/srt/test_bench_serving.py new file mode 100644 index 000000000..608595b95 --- /dev/null +++ b/test/srt/test_bench_serving.py @@ -0,0 +1,445 @@ +import asyncio +import itertools +import unittest + +import requests + +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_FP8, + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + run_bench_serving, + write_github_step_summary, +) + + +class TestBenchServing(CustomTestCase): + def test_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3050) + else: + self.assertGreater(res["output_throughput"], 3800) + + def test_offline_throughput_non_stream_small_batch_size(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=["--max-running-requests", "10"], + dataset_name="sharegpt", + random_input_len=None, + random_output_len=None, + disable_stream=True, + need_warmup=True, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_non_stream_small_batch_size\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 1000) + else: + self.assertGreater(res["output_throughput"], 1050) + + def test_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_radix_cache\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3050) + else: + self.assertGreater(res["output_throughput"], 3800) + + def test_offline_throughput_without_chunked_prefill(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=["--chunked-prefill-size", "-1"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_without_chunked_prefill\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + self.assertGreater(res["output_throughput"], 2600) + + def test_offline_throughput_with_triton_attention_backend(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[ + "--attention-backend", + "triton", + "--context-length", + "8192", + ], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_with_triton_attention_backend\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3500) + else: + self.assertGreater(res["output_throughput"], 3700) + + def test_offline_throughput_default_fp8(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST_FP8, + num_prompts=500, + request_rate=float("inf"), + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_offline_throughput_default_fp8\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 3500) + else: + self.assertGreater(res["output_throughput"], 4300) + + def test_online_latency_default(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=100, + request_rate=1, + other_server_args=[], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_default\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 11000) + if is_in_amd_ci(): + self.assertLess(res["median_ttft_ms"], 115) + else: + self.assertLess(res["median_ttft_ms"], 86) + self.assertLess(res["median_itl_ms"], 10) + + def test_vlm_offline_throughput(self): + res = run_bench_serving( + model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + num_prompts=200, + request_rate=float("inf"), + other_server_args=[ + "--mem-fraction-static", + "0.7", + ], + dataset_name="mmmu", + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_vlm_offline_throughput\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2000) + # TODO: not set yet, need AMD machine + else: + self.assertGreater(res["output_throughput"], 2500) + + def test_vlm_online_latency(self): + res = run_bench_serving( + model=DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + num_prompts=250, + request_rate=1, + other_server_args=[ + "--mem-fraction-static", + "0.7", + ], + dataset_name="mmmu", + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_vlm_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 16500) + if is_in_amd_ci(): + self.assertLess(res["median_ttft_ms"], 150) + # TODO: not set yet, need AMD machine + else: + self.assertLess(res["median_ttft_ms"], 100) + self.assertLess(res["median_itl_ms"], 8) + + def test_lora_online_latency(self): + # TODO (lifuhuang): verify LoRA support in AMD. + if is_in_amd_ci(): + pass + + res = self._run_lora_latency_test(enable_background_task=False) + + if is_in_ci(): + write_github_step_summary( + f"### test_lora_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 2400) + self.assertLess(res["median_ttft_ms"], 58) + + def test_lora_online_latency_with_concurrent_adapter_updates(self): + # TODO (lifuhuang): verify LoRA support in AMD. + if is_in_amd_ci(): + pass + + res = self._run_lora_latency_test(enable_background_task=True) + + if is_in_ci(): + write_github_step_summary( + f"### test_lora_online_latency\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"median_ttft_ms: {res['median_ttft_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 4000) + self.assertLess(res["median_ttft_ms"], 80) + + def _run_lora_latency_test(self, enable_background_task: bool): + """ + Run a latency test for LoRA with the specified background task setting. + """ + + async def lora_loader_unloader_task( + base_url: str, + start_event: asyncio.Event, + stop_event: asyncio.Event, + ): + """ + A background task that repeatedly loads and unloads a LoRA adapter. + """ + await start_event.wait() + + path_cycler = itertools.cycle( + [ + "pbevan11/llama-3.1-8b-ocr-correction", + "faridlazuarda/valadapt-llama-3.1-8B-it-chinese", + "philschmid/code-llama-3-1-8b-text-to-sql-lora", + ] + ) + load_url = f"{base_url}/load_lora_adapter" + unload_url = f"{base_url}/unload_lora_adapter" + num_updates = 0 + + while not stop_event.is_set(): + # 1. Load the LoRA adapter + lora_path = next(path_cycler) + response = await asyncio.to_thread( + requests.post, + load_url, + json={"lora_name": lora_path, "lora_path": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to load LoRA adapter: {response.text}" + ) + num_updates += 1 + + if stop_event.is_set(): + break + + # Yield control to allow other tasks to run. + await asyncio.sleep(1) + + # 2. Unload the LoRA adapter + response = await asyncio.to_thread( + requests.post, + unload_url, + json={"lora_name": lora_path}, + ) + self.assertTrue( + response.ok, f"Failed to unload LoRA adapter: {response.text}" + ) + num_updates += 1 + + # Yield control to allow other tasks to run. + await asyncio.sleep(1) + + background_task = lora_loader_unloader_task if enable_background_task else None + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=400, + request_rate=8, + other_server_args=[ + "--enable-lora", + "--max-loras-per-batch", + "1", + "--disable-radix-cache", + "--random-seed", + "42", + "--mem-fraction-static", + "0.8", + "--lora-paths", + "Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16", + "--max-lora-rank", + "256", + ], + dataset_name="random", + random_input_len=256, + random_output_len=256, + lora_name=["Nutanix/Meta-Llama-3.1-8B-Instruct_lora_4_alpha_16"], + background_task=background_task, + ) + + return res + + def test_online_latency_eagle(self): + res = run_bench_serving( + model=DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + num_prompts=300, + request_rate=8, + sharegpt_context_len=3072, + disable_ignore_eos=True, + dataset_name="sharegpt", + other_server_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--mem-fraction-static", + "0.7", + ], + need_warmup=True, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_online_latency_eagle\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + f"accept_length: {res['accept_length']:.2f} \n" + ) + if is_in_amd_ci(): + self.assertLess(res["median_e2e_latency_ms"], 1800) + else: + self.assertLess(res["median_e2e_latency_ms"], 900) + self.assertGreater(res["accept_length"], 3.0) + + def test_moe_offline_throughput_default(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_default\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2100) + else: + self.assertGreater(res["output_throughput"], 2200) + + def test_moe_offline_throughput_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=300, + request_rate=float("inf"), + other_server_args=["--tp", "2", "--disable-radix-cache"], + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_moe_offline_throughput_without_radix_cache\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(res["output_throughput"], 2100) + else: + self.assertGreater(res["output_throughput"], 2200) + + def test_pp_offline_throughput_default_decode(self): + res = run_bench_serving( + model=DEFAULT_MOE_MODEL_NAME_FOR_TEST, + num_prompts=1000, + request_rate=float("inf"), + random_input_len=1, + random_output_len=1024, + other_server_args=["--pp", "2"], + need_warmup=True, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_pp_offline_throughput_default_decode\n" + f"Output throughput: {res['output_throughput']:.2f} token/s\n" + ) + self.assertGreater(res["output_throughput"], 6700) + + def test_pp_long_context_prefill(self): + res = run_bench_serving( + model="meta-llama/Llama-3.3-70B-Instruct", + num_prompts=4, + request_rate=float("inf"), + random_input_len=128000, + random_output_len=1, + dataset_name="random", + other_server_args=[ + "--quantization", + "fp8", + "--pp", + 2, + ], + need_warmup=False, + seed=42, + ) + + if is_in_ci(): + write_github_step_summary( + f"### test_pp_long_context_latency_prefill\n" + f"input_throughput: {res['input_throughput']:.2f} ms\n" + ) + self.assertGreater(res["input_throughput"], 4000) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_bnb.py b/test/srt/test_bnb.py new file mode 100644 index 000000000..1d9f0201d --- /dev/null +++ b/test/srt/test_bnb.py @@ -0,0 +1,314 @@ +""" +Usage: +python3 -m unittest test_bnb.TestVisionModel.test_vlm +python3 -m unittest test_bnb.TestLanguageModel.test_mmlu +""" + +import base64 +import io +import json +import multiprocessing as mp +import os +import unittest +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace + +import numpy as np +import openai +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +VISION_MODELS = [ + "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", + "unsloth/Qwen2-VL-7B-Instruct-bnb-4bit", + "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit", + "unsloth/Llama-3.2-11B-Vision-bnb-4bit", + "unsloth/gemma-3-4b-it-bnb-4bit", + "unsloth/gemma-3-4b-it-unsloth-bnb-4bit", +] +LANGUAGE_MODELS = [ + "unsloth/Qwen2.5-7B-Instruct-bnb-4bit", + "unsloth/Qwen2-7B-Instruct-bnb-4bit", + "unsloth/Llama-3.2-3B-Instruct-bnb-4bit", + "unsloth/gemma-3-1b-it-bnb-4bit", +] + +# image +IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" +IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png" + +# video +VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4" + +# audio +AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3" +AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3" + + +def popen_launch_server_wrapper(base_url, model, other_args): + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +class TestVisionModel(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.base_url += "/v1" + cls.api_key = "sk-123456" + + def _run_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + # `driver` is for gemma-3-it + assert "man" in text or "person" or "driver" in text, text + assert "cab" in text or "taxi" in text or "SUV" in text, text + # MiniCPMO fails to recognize `iron`, but `hanging` + assert "iron" in text or "hang" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def _run_multi_turn_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a very short sentence.", + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "There is a man at the back of a yellow cab ironing his clothes.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Repeat your previous answer."} + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + assert "man" in text or "cab" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def _run_multi_images_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have two very different images. They are not related at all. " + "Please describe the first image in one sentence, and then describe the second image in another sentence.", + }, + ], + }, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Multi images response:\n{text}") + print("-" * 30) + assert "man" in text or "cab" in text or "SUV" in text or "taxi" in text, text + assert "logo" in text or '"S"' in text or "SG" in text, text + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def run_decode_with_image(self, image_id): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + content = [] + if image_id == 0: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + } + ) + elif image_id == 1: + content.append( + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + } + ) + else: + pass + + content.append( + { + "type": "text", + "text": "Describe this image in a very short sentence.", + } + ) + + response = client.chat.completions.create( + model="default", + messages=[ + {"role": "user", "content": content}, + ], + temperature=0, + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + + def _run_test_mixed_batch(self): + image_ids = [0, 1, 2] * 4 + with ThreadPoolExecutor(4) as executor: + list(executor.map(self.run_decode_with_image, image_ids)) + + def test_vlm(self): + models_to_test = VISION_MODELS + + if is_in_ci(): + models_to_test = [random.choice(VISION_MODELS)] + + for model in models_to_test: + with self.subTest(model=model): + other_args = [ + "--mem-fraction-static", + "0.6", + "--load-format", + "bitsandbytes", + "--enable-multimodal", + ] + try: + process = popen_launch_server_wrapper( + DEFAULT_URL_FOR_TEST, model, other_args + ) + self._run_test_mixed_batch() + self._run_multi_images_chat_completion() + self._run_multi_turn_chat_completion() + self._run_single_image_chat_completion() + finally: + kill_process_tree(process.pid) + + +class TestLanguageModel(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + cls.base_url = DEFAULT_URL_FOR_TEST + # cls.base_url += "/v1" + cls.api_key = "sk-123456" + + def test_mmlu(self): + models_to_test = LANGUAGE_MODELS + + if is_in_ci(): + models_to_test = [random.choice(LANGUAGE_MODELS)] + + for model in models_to_test: + with self.subTest(model=model): + other_args = [ + "--mem-fraction-static", + "0.6", + "--load-format", + "bitsandbytes", + ] + try: + process = popen_launch_server_wrapper( + DEFAULT_URL_FOR_TEST, model, other_args + ) + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mmlu", + num_examples=32, + num_threads=16, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.3) + finally: + kill_process_tree(process.pid) diff --git a/test/srt/test_chunked_prefill.py b/test/srt/test_chunked_prefill.py new file mode 100644 index 000000000..fbdc87c55 --- /dev/null +++ b/test/srt/test_chunked_prefill.py @@ -0,0 +1,31 @@ +""" +python3 -m unittest test_chunked_prefill.TestChunkedPrefill.test_mixed_chunked_prefill_without_radix_cache +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test, run_mulit_request_test + + +class TestChunkedPrefill(CustomTestCase): + def test_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill(self): + run_mmlu_test(disable_radix_cache=False, enable_mixed_chunk=True) + + def test_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=False) + + def test_mixed_chunked_prefill_without_radix_cache(self): + run_mmlu_test(disable_radix_cache=True, enable_mixed_chunk=True) + + def test_mixed_chunked_prefill_multi_requests(self): + run_mulit_request_test( + enable_mixed_chunk=True, + chunked_prefill_size=2048, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_cpp_radix_cache.py b/test/srt/test_cpp_radix_cache.py new file mode 100644 index 000000000..cb2822b88 --- /dev/null +++ b/test/srt/test_cpp_radix_cache.py @@ -0,0 +1,47 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestCppRadixCache(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_EXPERIMENTAL_CPP_RADIX_TREE"] = "1" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + print(metrics) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_cpu_graph.py b/test/srt/test_cpu_graph.py new file mode 100644 index 000000000..4e3c40539 --- /dev/null +++ b/test/srt/test_cpu_graph.py @@ -0,0 +1,87 @@ +""" +Usage: +python3 -m unittest test_cpu_graph.TestCPUGraph.test_mmlu_torch_compile_cpu +""" + +import copy +import os +import unittest +from types import SimpleNamespace + +from test_intel_amx_attention_backend import intel_amx_benchmark + +from sglang.srt.utils import get_cpu_ids_by_node, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestCPUGraph(CustomTestCase): + + @intel_amx_benchmark( + extra_args=[ + "--batch-size", + "1", + "--mem-fraction-static", + "0.05", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + ], + min_throughput=10, + ) + def test_latency_torch_compile_cpu(self): + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + def test_mmlu_torch_compile_cpu(self): + model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + cpu_ids_by_node = get_cpu_ids_by_node() + n_numa_node = len(cpu_ids_by_node) + env = copy.deepcopy(os.environ) + env["SGLANG_CPU_OMP_THREADS_BIND"] = "all" + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + "intel_amx", + "--mem-fraction-static", + "0.05", + "--disable-radix", + "--trust-remote-code", + "--disable-overlap-schedule", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--tp", + f"{n_numa_node}", + ], + env=env, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + if is_in_ci(): + self.assertGreater(metrics["score"], 0.45) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_create_kvindices.py b/test/srt/test_create_kvindices.py new file mode 100644 index 000000000..4196eb290 --- /dev/null +++ b/test/srt/test_create_kvindices.py @@ -0,0 +1,75 @@ +import itertools +import unittest + +import numpy as np +import torch + +from sglang.srt.layers.attention.utils import create_flashinfer_kv_indices_triton +from sglang.test.test_utils import CustomTestCase + + +class TestCreateKvIndices(CustomTestCase): + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _run_test(self, batch, max_batch, max_context_len): + req_to_token = torch.arange( + max_batch * max_context_len, dtype=torch.int32, device="cuda" + ).reshape((max_batch, max_context_len)) + req_pool_indices = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_batch), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + paged_kernel_lens = torch.tensor( + torch.from_numpy( + np.random.choice(range(max_context_len), size=batch, replace=False) + ), + dtype=torch.int32, + device="cuda", + ) + + kv_indptr = torch.zeros((batch + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1:] = torch.cumsum(paged_kernel_lens, dim=0) + + # ref + req_pool_indices_cpu = req_pool_indices.cpu().numpy() + paged_kernel_lens_cpu = paged_kernel_lens.cpu().numpy() + kv_indices_ref = torch.cat( + [ + req_to_token[req_pool_indices_cpu[i], : paged_kernel_lens_cpu[i]] + for i in range(batch) + ], + dim=0, + ).contiguous() + + # triton + kv_indices_triton = torch.empty(kv_indptr[-1], dtype=torch.int32, device="cuda") + create_flashinfer_kv_indices_triton[(batch,)]( + req_to_token, + req_pool_indices, + paged_kernel_lens, + kv_indptr, + None, + kv_indices_triton, + req_to_token.size(1), + ) + + # Check + self.assertTrue(torch.equal(kv_indices_ref, kv_indices_triton)) + + def test_create_kvindices(self): + BATCH = [1, 37, 1786] + MAX_BATCH = 4096 + MAX_CONTEXT_LEN = 4096 + for batch in BATCH: + self._run_test(batch, MAX_BATCH, MAX_CONTEXT_LEN) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_custom_allreduce.py b/test/srt/test_custom_allreduce.py new file mode 100644 index 000000000..462ac578e --- /dev/null +++ b/test/srt/test_custom_allreduce.py @@ -0,0 +1,174 @@ +import os +import random +import socket +import unittest +from typing import Any + +import ray +import torch +import torch.distributed as dist + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.communication_op import ( # noqa + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, +) +from sglang.test.test_utils import CustomTestCase + + +def get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, + cls: Any, + test_target: Any, +) -> None: + + # Using ray helps debugging the error when it failed + # as compared to multiprocessing. + # NOTE: We need to set working_dir for distributed tests, + # otherwise we may get import errors on ray workers + + ray.init(log_to_driver=True) + + distributed_init_port = get_open_port() + refs = [] + for rank in range(world_size): + refs.append(test_target.remote(cls, world_size, rank, distributed_init_port)) + ray.get(refs) + + ray.shutdown() + + +class TestCustomAllReduce(CustomTestCase): + TEST_SIZES = [ + 512, + 4096, + 32768, + 262144, + 2097152, + 16777216, + 33554432, + ] # 512B...32MB + WORLD_SIZES = [2, 4, 6, 8] + TEST_LOOP = 10 + + @classmethod + def setUpClass(cls): + random.seed(42) # keep the deterministic seed + + def test_graph_allreduce(self): + for world_size in self.WORLD_SIZES: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.graph_allreduce) + + def test_eager_allreduce(self): + for world_size in self.WORLD_SIZES: + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.eager_allreduce) + + @ray.remote(num_gpus=1, max_calls=1) + def graph_allreduce(self, world_size, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + # A small all_reduce for warmup. + # this is needed because device communicators might be created lazily + # (e.g. NCCL). This will ensure that the communicator is initialized + # before any communication happens, so that this group can be used for + # graph capture immediately. + data = torch.zeros(1) + data = data.to(device=device) + torch.distributed.all_reduce(data, group=group) + torch.cuda.synchronize() + del data + + for sz in self.TEST_SIZES: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.TEST_LOOP): + with graph_capture() as graph_capture_context: + # use integers so result matches NCCL exactly + inp1 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + inp2 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph( + graph, stream=graph_capture_context.stream + ): + out1 = tensor_model_parallel_all_reduce(inp1) + # the input buffer is immediately modified to test + # synchronization + dist.all_reduce(inp1, group=group) + out2 = tensor_model_parallel_all_reduce(inp2) + dist.all_reduce(inp2, group=group) + graph.replay() + torch.testing.assert_close(out1, inp1) + torch.testing.assert_close(out2, inp2) + + @ray.remote(num_gpus=1, max_calls=1) + def eager_allreduce(self, world_size, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + for sz in self.TEST_SIZES: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.TEST_LOOP): + inp1 = torch.randint( + 1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device() + ) + out1 = tensor_model_parallel_all_reduce(inp1) + dist.all_reduce(inp1, group=group) + torch.testing.assert_close(out1, inp1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_data_parallelism.py b/test/srt/test_data_parallelism.py new file mode 100644 index 000000000..e54e3a5cf --- /dev/null +++ b/test/srt/test_data_parallelism.py @@ -0,0 +1,78 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDataParallelism(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dp", 2], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_update_weight(self): + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + # pause a few seconds then send again + time.sleep(1) + + response = requests.post( + self.base_url + "/update_weights_from_disk", + json={"model_path": DEFAULT_MODEL_NAME_FOR_TEST}, + ) + + # check if the response is 200 + assert response.status_code == 200 + + def test_get_memory_pool_size(self): + # use `get_server_info` instead since `get_memory_pool_size` is merged into `get_server_info` + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + time.sleep(1) + + response = requests.get(self.base_url + "/get_server_info") + assert response.status_code == 200 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_disaggregation.py b/test/srt/test_disaggregation.py new file mode 100644 index 000000000..827bfc3b8 --- /dev/null +++ b/test/srt/test_disaggregation.py @@ -0,0 +1,434 @@ +import json +import os +import time +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +import requests + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_pd_server, +) + + +class TestDisaggregationAccuracy(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + "--disaggregation-ib-device", + "mlx5_roce0", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + "--disaggregation-ib-device", + "mlx5_roce1", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.62) + + def test_logprob(self): + prompt = "The capital of france is " + response = requests.post( + self.lb_url + "/generate", + json={ + "text": prompt, + "sampling_params": {"temperature": 0}, + "return_logprob": True, + "return_input_logprob": True, + "logprob_start_len": 0, + }, + ) + + j = response.json() + completion_tokens = j["meta_info"]["completion_tokens"] + input_logprobs = j["meta_info"]["input_token_logprobs"] + output_logprobs = j["meta_info"]["output_token_logprobs"] + + assert ( + len(output_logprobs) == completion_tokens + ), f"output_logprobs and completion_tokens should have the same length, but got {len(output_logprobs)} and {completion_tokens}" + assert ( + len(input_logprobs) > 0 + ), f"input_logprobs should have at least one token, but got {len(input_logprobs)}" + + def test_structured_output(self): + json_schema = json.dumps( + { + "type": "object", + "properties": { + "name": {"type": "string", "pattern": "^[\\w]+$"}, + "population": {"type": "integer"}, + }, + "required": ["name", "population"], + } + ) + + # JSON + response = requests.post( + f"{self.lb_url}/generate", + json={ + "text": "Here is the information of the capital of France in the JSON format.\n", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 64, + "json_schema": json_schema, + }, + }, + ) + output = response.json()["text"] + # ensure the output is a valid JSON + json.loads(output) + + +class TestDisaggregationMooncakeFailure(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + # set DISAGGREGATION_TEST_FAILURE_PROB to simulate failure + os.environ["DISAGGREGATION_TEST_FAILURE_PROB"] = "0.05" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def tearDownClass(cls): + os.environ.pop("DISAGGREGATION_TEST_FAILURE_PROB") + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + "--disaggregation-ib-device", + "mlx5_roce0", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + "--disaggregation-ib-device", + "mlx5_roce1", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + + # Expect lots of failure but the server cannot crash + try: + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + except Exception as e: + print(f"Test encountered expected errors: {e}") + # Check if servers are still healthy + try: + response = requests.get(self.prefill_url + "/health_generate") + assert response.status_code == 200 + response = requests.get(self.decode_url + "/health_generate") + assert response.status_code == 200 + except Exception as health_check_error: + # If health check fails, re-raise the original exception + raise e from health_check_error + + +class TestDisaggregationMooncakeSpec(TestDisaggregationBase): + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + cls.draft_model = DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + cls.spec_args = [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + cls.draft_model, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "16", + "--cuda-graph-max-bs", + "8", + ] + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "2", + "--disaggregation-ib-device", + "mlx5_roce0,mlx5_roce1", + ] + cls.spec_args + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "2", + "--base-gpu-id", + "2", + "--disaggregation-ib-device", + "mlx5_roce2,mlx5_roce3", + ] + cls.spec_args + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=2, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.20) + + +class TestDisaggregationSimulatedRetract(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_TEST_RETRACT"] = "true" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def tearDownClass(cls): + os.environ.pop("SGLANG_TEST_RETRACT") + super().tearDownClass() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + "--disaggregation-ib-device", + "mlx5_roce0", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "1", + "--disaggregation-ib-device", + "mlx5_roce1", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.62) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_disaggregation_different_tp.py b/test/srt/test_disaggregation_different_tp.py new file mode 100644 index 000000000..67a3afcbe --- /dev/null +++ b/test/srt/test_disaggregation_different_tp.py @@ -0,0 +1,184 @@ +import os +import time +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_pd_server, +) + + +class TestDisaggregationMooncakePrefillLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + # Temporarily disable JIT DeepGEMM + cls.original_jit_deepgemm = os.environ.get("SGL_ENABLE_JIT_DEEPGEMM") + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "false" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "2", + "--disaggregation-ib-device", + "mlx5_roce0,mlx5_roce1", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "2", + "--disaggregation-ib-device", + "mlx5_roce2", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestDisaggregationMooncakeDecodeLargerTP(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + # Temporarily disable JIT DeepGEMM + cls.original_jit_deepgemm = os.environ.get("SGL_ENABLE_JIT_DEEPGEMM") + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "false" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp", + "1", + "--disaggregation-ib-device", + "mlx5_roce0", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "2", + "--base-gpu-id", + "1", + "--disaggregation-ib-device", + "mlx5_roce1,mlx5_roce2", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"Evaluation metrics: {metrics}") + + self.assertGreater(metrics["accuracy"], 0.60) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_disaggregation_pp.py b/test/srt/test_disaggregation_pp.py new file mode 100644 index 000000000..a8bab8f81 --- /dev/null +++ b/test/srt/test_disaggregation_pp.py @@ -0,0 +1,101 @@ +import time +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_disaggregation_utils import TestDisaggregationBase +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_pd_server, +) + + +class TestDisaggregationPPAccuracy(TestDisaggregationBase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + parsed_url = urlparse(DEFAULT_URL_FOR_TEST) + cls.base_host = parsed_url.hostname + base_port = str(parsed_url.port) + cls.lb_port = base_port + cls.prefill_port = f"{int(base_port) + 100}" + cls.decode_port = f"{int(base_port) + 200}" + cls.prefill_url = f"http://{cls.base_host}:{cls.prefill_port}" + cls.decode_url = f"http://{cls.base_host}:{cls.decode_port}" + cls.lb_url = f"http://{cls.base_host}:{cls.lb_port}" + print(f"{cls.base_host=} {cls.lb_port=} {cls.prefill_port=} {cls.decode_port=}") + + # Non blocking start servers + cls.start_prefill() + cls.start_decode() + + # Block until both + cls.wait_server_ready(cls.prefill_url + "/health") + cls.wait_server_ready(cls.decode_url + "/health") + + cls.launch_lb() + + @classmethod + def start_prefill(cls): + prefill_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "prefill", + "--tp-size", + "1", + "--pp-size", + "2", + "--disaggregation-ib-device", + "mlx5_roce0,mlx5_roce1", + "--disable-overlap-schedule", + ] + cls.process_prefill = popen_launch_pd_server( + cls.model, + cls.prefill_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=prefill_args, + ) + + @classmethod + def start_decode(cls): + decode_args = [ + "--trust-remote-code", + "--disaggregation-mode", + "decode", + "--tp", + "1", + "--base-gpu-id", + "2", + "--disaggregation-ib-device", + "mlx5_roce2", + ] + cls.process_decode = popen_launch_pd_server( + cls.model, + cls.decode_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=decode_args, + ) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host=f"http://{self.base_host}", + port=int(self.lb_port), + ) + metrics = run_eval(args) + print(f"{metrics=}") + + self.assertGreater(metrics["accuracy"], 0.24) + # Wait a little bit so that the memory check happens. + time.sleep(5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_double_sparsity.py b/test/srt/test_double_sparsity.py new file mode 100644 index 000000000..c936e79bb --- /dev/null +++ b/test/srt/test_double_sparsity.py @@ -0,0 +1,65 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDoubleSparsity(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + dirpath = os.path.dirname(__file__) + config_file = os.path.join( + dirpath, "double-sparsity-config-Llama-3.1-8B-Instruct.json" + ) + # NOTE: Generate the config file by running https://github.com/andy-yang-1/DoubleSparse/blob/main/evaluation/group_channel_config.py + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enable-double-sparsity", + "--ds-channel-config-path", + config_file, + "--ds-heavy-channel-num", + "32", + "--ds-heavy-channel-type", + "k", + "--ds-heavy-token-num", + "512", + "--ds-sparse-decode-threshold", + "0", + "--max-total-tokens", + "200000", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_dp_attention.py b/test/srt/test_dp_attention.py new file mode 100644 index 000000000..4486dc16e --- /dev/null +++ b/test/srt/test_dp_attention.py @@ -0,0 +1,170 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +class TestDPAttentionDP2TP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--enable-dp-attention", + "--dp", + "2", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +class TestDPAttentionDP2TP2DeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--disable-radix", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--tp-size", + "2", + "--enable-dp-attention", + "--dp-size", + "2", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp):\n" + f"accuracy={metrics['accuracy']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.5) + + +class TestDPAttentionMinimumTokenLoadBalance(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--enable-dp-attention", + "--dp", + "2", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + "--load-balance-method", + "minimum_tokens", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eagle_infer_a.py b/test/srt/test_eagle_infer_a.py new file mode 100644 index 000000000..c19f0c22f --- /dev/null +++ b/test/srt/test_eagle_infer_a.py @@ -0,0 +1,323 @@ +import unittest + +import requests +import torch + +import sglang as sgl +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +torch_dtype = torch.float16 +prefill_tolerance = 5e-2 +decode_tolerance: float = 5e-2 + + +class TestEAGLEEngine(CustomTestCase): + BASE_CONFIG = { + "model_path": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + "speculative_draft_model_path": DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + } + NUM_CONFIGS = 2 + + def setUp(self): + self.prompt = "Today is a sunny day and I like" + self.sampling_params = {"temperature": 0, "max_new_tokens": 8} + + ref_engine = sgl.Engine( + model_path=self.BASE_CONFIG["model_path"], cuda_graph_max_bs=1 + ) + self.ref_output = ref_engine.generate(self.prompt, self.sampling_params)["text"] + ref_engine.shutdown() + + def test_correctness(self): + configs = [ + # Basic config + self.BASE_CONFIG, + # Chunked prefill + {**self.BASE_CONFIG, "chunked_prefill_size": 4}, + ] + + for i, config in enumerate(configs[: self.NUM_CONFIGS]): + with self.subTest(i=i): + print(f"{config=}") + engine = sgl.Engine(**config, log_level="info", decode_log_interval=10) + try: + self._test_single_generation(engine) + self._test_batch_generation(engine) + self._test_eos_token(engine) + self._test_acc_length(engine) + finally: + engine.shutdown() + print("=" * 100) + + def _test_single_generation(self, engine): + output = engine.generate(self.prompt, self.sampling_params)["text"] + print(f"{output=}, {self.ref_output=}") + self.assertEqual(output, self.ref_output) + + def _test_batch_generation(self, engine): + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + params = {"temperature": 0, "max_new_tokens": 50} + + outputs = engine.generate(prompts, params) + for prompt, output in zip(prompts, outputs): + print(f"Prompt: {prompt}") + print(f"Generated: {output['text']}") + print("-" * 40) + + print(f"{engine.get_server_info()=}") + + avg_spec_accept_length = engine.get_server_info()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 1.9) + + def _test_eos_token(self, engine): + prompt = "[INST] <>\nYou are a helpful assistant.\n<>\nToday is a sunny day and I like [/INST]" + params = { + "temperature": 0.1, + "max_new_tokens": 1024, + "skip_special_tokens": False, + } + + tokenizer = get_tokenizer(DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST) + output = engine.generate(prompt, params)["text"] + print(f"{output=}") + + tokens = tokenizer.encode(output, truncation=False) + self.assertNotIn(tokenizer.eos_token_id, tokens) + + def _test_acc_length(self, engine): + prompt = [ + "Human: Give me a fully functional FastAPI server. Show the python code.\n\nAssistant:", + ] * 5 # test batched generation + sampling_params = {"temperature": 0, "max_new_tokens": 512} + output = engine.generate(prompt, sampling_params) + output = output[0] + + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + speed = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["e2e_latency"] + ) + print(f"{acc_length=:.4f}, {speed=}") + + if engine.server_args.model_path == DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST: + self.assertGreater(acc_length, 3.6) + else: + self.assertGreater(acc_length, 2.5) + + +class TestEAGLEEngineTokenMap(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": "meta-llama/Meta-Llama-3-8B-Instruct", + "speculative_draft_model_path": "lmsys/sglang-EAGLE-LLaMA3-Instruct-8B", + "speculative_algorithm": "EAGLE", + "speculative_num_steps": 5, + "speculative_eagle_topk": 4, + "speculative_num_draft_tokens": 8, + "speculative_token_map": "thunlp/LLaMA3-Instruct-8B-FR-Spec/freq_32768.pt", + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + "dtype": "float16", + } + NUM_CONFIGS = 1 + + +class TestEAGLE3Engine(TestEAGLEEngine): + BASE_CONFIG = { + "model_path": "meta-llama/Llama-3.1-8B-Instruct", + "speculative_draft_model_path": "jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B", + "speculative_algorithm": "EAGLE3", + "speculative_num_steps": 5, + "speculative_eagle_topk": 16, + "speculative_num_draft_tokens": 64, + "mem_fraction_static": 0.7, + "cuda_graph_max_bs": 5, + "dtype": "float16", + } + NUM_CONFIGS = 1 + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "fa3", + ], + ) + cls.accept_len_threshold = 1.50 + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_one_batch_accept_length(self): + resp = requests.get(self.base_url + "/flush_cache") + self.assertEqual(resp.status_code, 200) + + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + url = self.base_url + "/generate" + data = { + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 512, + }, + } + response = requests.post(url, json=data) + self.assertEqual(response.status_code, 200) + outputs = response.json() + for i in range(len(prompts)): + output = outputs[i] + if "spec_verify_ct" in output["meta_info"]: + acc_length = ( + output["meta_info"]["completion_tokens"] + / output["meta_info"]["spec_verify_ct"] + ) + else: + acc_length = 1.0 + + print(f"{acc_length=}") + self.assertGreater(acc_length, self.accept_len_threshold) + + +class TestEAGLEDraftExtendFlashinfer(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "flashinfer", + ], + ) + cls.accept_len_threshold = 1.50 + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtendTriton(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "triton", + ], + ) + cls.accept_len_threshold = 1.50 + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestEAGLEDraftExtendFlashinferMLA(TestEAGLEDraftExtend): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + 1, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 2, + "--max-running-requests", + 4, + "--attention-backend", + "flashinfer", + ], + ) + cls.accept_len_threshold = 1.85 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eagle_infer_b.py b/test/srt/test_eagle_infer_b.py new file mode 100644 index 000000000..5b9df1630 --- /dev/null +++ b/test/srt/test_eagle_infer_b.py @@ -0,0 +1,511 @@ +import json +import os +import random +import threading +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from types import SimpleNamespace + +import numpy as np +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + + +class TestEAGLEServer(CustomTestCase): + PROMPTS = [ + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nToday is a sunny day and I like[/INST]" + '[INST] <>\\nYou are a helpful assistant.\\n<>\\nWhat are the mental triggers in Jeff Walker\'s Product Launch Formula and "Launch" book?[/INST]', + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nSummarize Russell Brunson's Perfect Webinar Script...[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwho are you?[/INST]", + "[INST] <>\\nYou are a helpful assistant.\\n<>\\nwhere are you from?[/INST]", + ] + + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def send_request(self): + time.sleep(random.uniform(0, 2)) + for prompt in self.PROMPTS: + url = self.base_url + "/generate" + data = { + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + response = requests.post(url, json=data) + assert response.status_code == 200 + + def send_requests_abort(self): + for prompt in self.PROMPTS: + try: + time.sleep(random.uniform(0, 2)) + url = self.base_url + "/generate" + data = { + "model": "base", + "text": prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 1024, + }, + } + # set timeout = 1s, mock disconnected + requests.post(url, json=data, timeout=1) + except Exception as e: + print(e) + pass + + def test_request_abort(self): + concurrency = 4 + threads = [ + threading.Thread(target=self.send_request) for _ in range(concurrency) + ] + [ + threading.Thread(target=self.send_requests_abort) + for _ in range(concurrency) + ] + for worker in threads: + worker.start() + for p in threads: + p.join() + + def test_max_token_one(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=1, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + # Just run and check it does not hang + metrics = run_eval(args) + self.assertGreater(metrics["output_throughput"], 50) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["accuracy"], 0.20) + + server_info = requests.get(self.base_url + "/get_server_info").json() + avg_spec_accept_length = server_info["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + speculative_eagle_topk = server_info["speculative_eagle_topk"] + + if speculative_eagle_topk == 1: + self.assertGreater(avg_spec_accept_length, 2.5) + else: + self.assertGreater(avg_spec_accept_length, 3.5) + + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for res in response_json: + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, + return_logprob=False, + max_new_tokens=512, + logprob_start_len=-1, + temperature=1.0, + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": temperature, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + "temp_scaled_logprobs": True, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + for temperature in [1.0]: + gen = run_generate( + prompt, + return_logprob=True, + logprob_start_len=0, + temperature=temperature, + ) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, + return_logprob=True, + logprob_start_len=0, + max_new_tokens=0, + temperature=temperature, + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][ + num_prompts_tokens: + ] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.255) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + # Llama 2 context length seems to be only 2k, so we can only test small length. + for input_len in [200, 500, 1000, 2000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 100, 300, 800, 1998]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def run_decode(self, sampling_params): + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "Human: Write a travel blog post to Hawaii.\n\nAssistant:", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + "temperature": 0.7, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + def test_constrained_decoding(self): + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Give me a json"}, + ] + + response = requests.post( + self.base_url + "/v1/chat/completions", + json={ + "model": DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + "messages": messages, + "temperature": 0, + "response_format": {"type": "json_object"}, + }, + ) + self.assertEqual(response.status_code, 200) + res = response.json() + + # Validate response structure + self.assertIn("choices", res) + self.assertEqual(len(res["choices"]), 1) + self.assertIn("message", res["choices"][0]) + self.assertIn("content", res["choices"][0]["message"]) + + # Validate JSON content + content_json = res["choices"][0]["message"]["content"] + is_valid_json = True + try: + content = json.loads(content_json) + self.assertIsInstance(content, dict) + except Exception: + print(f"parse JSON failed: {content_json}") + is_valid_json = False + self.assertTrue(is_valid_json) + + +class TestEAGLERetract(TestEAGLEServer): + @classmethod + def setUpClass(cls): + # These config helps find a leak. + os.environ["SGLANG_CI_SMALL_KV_SIZE"] = "4500" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 64, + ], + ) + + +class TestEAGLEServerTriton(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--attention-backend", + "triton", + "--max-running-requests", + 8, + ], + ) + + +class TestEAGLEServerPageSize(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 1, + "--speculative-num-draft-tokens", + 6, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--page-size", + 4, + "--attention-backend", + "flashinfer", + ], + ) + + +class TestEAGLEServerPageSizeTopk(TestEAGLEServer): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + 5, + "--speculative-eagle-topk", + 8, + "--speculative-num-draft-tokens", + 64, + "--mem-fraction-static", + 0.7, + "--chunked-prefill-size", + 128, + "--max-running-requests", + 8, + "--page-size", + 4, + "--attention-backend", + "flashinfer", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_ebnf_constrained.py b/test/srt/test_ebnf_constrained.py new file mode 100644 index 000000000..c029f7743 --- /dev/null +++ b/test/srt/test_ebnf_constrained.py @@ -0,0 +1,282 @@ +""" +python3 -m unittest test_ebnf_constrained.TestEBNFConstrained.test_ebnf_generate_email +python3 -m unittest test_ebnf_constrained.TestEBNFConstrained.test_ebnf_generate_greeting +python3 -m unittest test_ebnf_constrained.TestEBNFConstrained.test_ebnf_generate_all_optional_function_params +python3 -m unittest test_ebnf_constrained.TestEBNFConstrainedLLGuidance.test_ebnf_generate_email +python3 -m unittest test_ebnf_constrained.TestEBNFConstrainedLLGuidance.test_ebnf_generate_greeting +python3 -m unittest test_ebnf_constrained.TestEBNFConstrainedLLGuidance.test_ebnf_generate_all_optional_function_params +""" + +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str, disable_overlap: bool): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.ebnf_grammar = 'root ::= "test"' # Default grammar + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestEBNFConstrained(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, "xgrammar", disable_overlap=False) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + ebnf, + expected_patterns, + prompt, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "ebnf": ebnf, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + + ret = response.json() + print(json.dumps(ret, indent=2)) + print("=" * 100) + + if not isinstance(ret, list): + self.fail(f"Expected response to be a list, but got {type(ret)}") + + for item in ret: + text = item.get("text", "").strip() + if not text: + self.fail("Generated text is empty.") + + match = False + for pattern in expected_patterns: + if self.regex_match(text, pattern): + match = True + break + if not match: + self.fail(f"Text '{text}' does not match any of the allowed patterns.") + + def regex_match(self, text, pattern): + import re + + return re.match(pattern, text) is not None + + def test_ebnf_generate_email(self): + self.__class__.ebnf_grammar = 'root ::= "user@example.com"' + allowed_patterns = [r"^user@example\.com$"] + prompt = "Generate an email address:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_greeting(self): + self.__class__.ebnf_grammar = 'root ::= "Hello" | "Hi" | "Hey"' + allowed_patterns = [r"^(Hello|Hi|Hey)$"] + prompt = "Generate a greeting:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_number(self): + self.__class__.ebnf_grammar = """ + root ::= digit digit digit + digit ::= [0-9] + """ + allowed_patterns = [r"^\d{3}$"] + prompt = "Generate a three-digit number:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_phone(self): + self.__class__.ebnf_grammar = """ + root ::= "(" area ")" " " prefix "-" line + area ::= [0-9] [0-9] [0-9] + prefix ::= [0-9] [0-9] [0-9] + line ::= [0-9] [0-9] [0-9] [0-9] + """ + allowed_patterns = [r"^\(\d{3}\) \d{3}-\d{4}$"] + prompt = "Generate a phone number:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_date(self): + self.__class__.ebnf_grammar = """ + root ::= year "-" month "-" day + year ::= "2024" + month ::= "01" | "02" | "03" | "04" | "05" | "06" | "07" | "08" | "09" | "10" | "11" | "12" + day ::= "01" | "02" | "03" | "04" | "05" | "06" | "07" | "08" | "09" | "10" | + "11" | "12" | "13" | "14" | "15" | "16" | "17" | "18" | "19" | "20" | + "21" | "22" | "23" | "24" | "25" | "26" | "27" | "28" | "29" | "30" | "31" + """ + allowed_patterns = [r"^2024-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$"] + prompt = "Generate a date in YYYY-MM-DD format:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_hex_color(self): + self.__class__.ebnf_grammar = """ + root ::= "#" hex hex hex hex hex hex + hex ::= [0-9] | [A-F] + """ + allowed_patterns = [r"^#[0-9A-F]{6}$"] + prompt = "Generate a hex color code:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_complex_json(self): + self.__class__.ebnf_grammar = """ + root ::= object + object ::= "{" ws pair (ws "," ws pair)* ws "}" + pair ::= "\\"name\\"" ws ":" ws value | + "\\"age\\"" ws ":" ws number | + "\\"city\\"" ws ":" ws string + value ::= string | number + string ::= "\\"" [a-zA-Z0-9 ]+ "\\"" + number ::= [1-9] [0-9]* + ws ::= [ ]* + """ + allowed_patterns = [ + r'^{\s*"name"\s*:\s*"[a-zA-Z0-9 ]+"\s*,\s*"age"\s*:\s*[1-9][0-9]*\s*,\s*"city"\s*:\s*"[a-zA-Z0-9 ]+"\s*}$', + ] + prompt = "Generate a simple JSON with name, age, and city:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_custom_log_format(self): + self.__class__.ebnf_grammar = """ + root ::= logentry + logentry ::= "[" datetime "] " level ": System.process - " message + datetime ::= "2024-01-01T12:00:00Z" + level ::= "INFO" + message ::= "Operation " [a-z]+ " successfully" + """ + allowed_patterns = [ + r"^\[2024-01-01T12:00:00Z\] INFO: System\.process - Operation [a-z]+ successfully$" + ] + prompt = "Generate a log entry:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=3, + ) + + def test_ebnf_generate_all_optional_function_params(self): + """Test function call with all optional parameters - verifies flexible ordering.""" + self.__class__.ebnf_grammar = """ + root ::= function_call + function_call ::= call_config_service + call_config_service ::= "{" "\\"name\\"" ":" "\\"config_service\\"" ", " "\\"arguments\\"" ":" arguments_config_service "}" + arguments_config_service ::= "{" ( "\\"theme\\"" ":" ("\\"light\\"" | "\\"dark\\"") ( "," "\\"language\\"" ":" ("\\"en\\"" | "\\"es\\"" | "\\"fr\\"") )? ( "," "\\"notifications\\"" ":" ("true" | "false") )? | "\\"language\\"" ":" ("\\"en\\"" | "\\"es\\"" | "\\"fr\\"") ( "," "\\"notifications\\"" ":" ("true" | "false") )? | "\\"notifications\\"" ":" ("true" | "false") )? "}" + """ + # Test patterns that should match - flexible ordering of optional parameters + allowed_patterns = [ + # Empty arguments + r'^\{"name":"config_service",\s*"arguments":\{\}\}$', + # Single optional parameters (any can appear first) + r'^\{"name":"config_service",\s*"arguments":\{"theme":"(light|dark)"\}\}$', + r'^\{"name":"config_service",\s*"arguments":\{"language":"(en|es|fr)"\}\}$', + r'^\{"name":"config_service",\s*"arguments":\{"notifications":(true|false)\}\}$', + # Two optional parameters (in any order) + r'^\{"name":"config_service",\s*"arguments":\{"theme":"(light|dark)",\s*"language":"(en|es|fr)"\}\}$', + r'^\{"name":"config_service",\s*"arguments":\{"theme":"(light|dark)",\s*"notifications":(true|false)\}\}$', + r'^\{"name":"config_service",\s*"arguments":\{"language":"(en|es|fr)",\s*"notifications":(true|false)\}\}$', + # All three optional parameters + r'^\{"name":"config_service",\s*"arguments":\{"theme":"(light|dark)",\s*"language":"(en|es|fr)",\s*"notifications":(true|false)\}\}$', + ] + prompt = "Configure the service with optional settings:" + + self.run_decode( + ebnf=self.__class__.ebnf_grammar, + expected_patterns=allowed_patterns, + prompt=prompt, + n=5, + ) + + +class TestEBNFConstrainedLLGuidance(TestEBNFConstrained): + @classmethod + def setUpClass(cls): + setup_class(cls, "llguidance", disable_overlap=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eval_accuracy_large.py b/test/srt/test_eval_accuracy_large.py new file mode 100644 index 000000000..99304dfde --- /dev/null +++ b/test/srt/test_eval_accuracy_large.py @@ -0,0 +1,94 @@ +""" +Usage: +python -m unittest test_eval_accuracy_large.TestEvalAccuracyLarge.test_mmlu +""" + +import os +import time +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--log-level-http", "warning"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + self.assertGreater(metrics["score"], 0.70) + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.64) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + self.assertGreater(metrics["score"], 0.835) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_eval_fp8_accuracy.py b/test/srt/test_eval_fp8_accuracy.py new file mode 100644 index 000000000..329e2dad8 --- /dev/null +++ b/test/srt/test_eval_fp8_accuracy.py @@ -0,0 +1,114 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8, + DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEvalFP8Accuracy(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_ACCURACY_TEST_FP8 + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + if is_hip(): + # Another threshold for AMD because fp8 dtype is difference + self.assertGreaterEqual(metrics["score"], 0.60) + else: + self.assertGreaterEqual(metrics["score"], 0.60) + + +class TestEvalFP8DynamicQuantAccuracy(CustomTestCase): + + def _run_test(self, model, other_args, expected_score): + base_url = DEFAULT_URL_FOR_TEST + other_args = other_args or [] + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], expected_score) + finally: + kill_process_tree(process.pid) + + def test_mmlu_offline_only(self): + """Test with offline quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + other_args=[], + expected_score=0.64, + ) + + def test_mmlu_offline_and_online_override(self): + """Test with both offline and online quantization.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_DYNAMIC_QUANT_ACCURACY_TEST_FP8, + other_args=["--quantization", "w8a8_fp8"], + # inference will use sgl kernel w/ online quant override + # we observed that the accuracy is higher then offline only + expected_score=0.64, + ) + + def test_mmlu_online_only(self): + """Test with online quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + # inference will use sgl kernel w/ online quantization only + # we observed that the accuracy is higher then offline only + other_args=["--quantization", "w8a8_fp8"], + expected_score=0.64, + ) + + def test_mmlu_fp16_baseline(self): + """Test with unquantized fp16 baseline.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_TEST, + other_args=[], + expected_score=0.64, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_expert_distribution.py b/test/srt/test_expert_distribution.py new file mode 100755 index 000000000..5d4add72f --- /dev/null +++ b/test/srt/test_expert_distribution.py @@ -0,0 +1,102 @@ +import os +import tempfile +import unittest +from pathlib import Path + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestExpertDistribution(CustomTestCase): + def test_expert_distribution_record(self): + # TODO: Add tests for DeepEP gatherer (currently our CI cannot run that) + for info in [ + dict(model_path="deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct"), + dict(model_path="Qwen/Qwen1.5-MoE-A2.7B"), + dict(model_path="Qwen/Qwen1.5-MoE-A2.7B", tp_size=2), + dict(model_path="Qwen/Qwen1.5-MoE-A2.7B", mode="per_pass"), + dict(model_path="Qwen/Qwen1.5-MoE-A2.7B", mode="per_token"), + ]: + with self.subTest(info=info): + self._execute_core(**info) + + def _execute_core(self, model_path: str, mode: str = "stat", tp_size: int = 1): + """Test expert distribution record endpoints""" + with tempfile.TemporaryDirectory() as tmp_dir: + os.environ["SGLANG_EXPERT_DISTRIBUTION_RECORDER_DIR"] = tmp_dir + + process = popen_launch_server( + model_path, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp-size", + str(tp_size), + "--expert-distribution-recorder-mode", + mode, + "--disable-cuda-graph", + "--disable-overlap-schedule", + ], + ) + + try: + # Start recording + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/start_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Make some requests to generate expert distribution data + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + # Stop recording + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/stop_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Dump the recorded data + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/dump_expert_distribution_record" + ) + self.assertEqual(response.status_code, 200) + + # Check data rows + data = torch.load( + list(Path(tmp_dir).glob("*.pt"))[0], weights_only=True + ) + print(f"{data=}") + + if mode in ["per_pass", "per_token"]: + self.assertGreater(len(data), 0, "Should contain data rows") + else: + logical_count = data["logical_count"] + print(f"{logical_count.sum()=} {logical_count=}") + self.assertTrue(logical_count.sum() > 0) + + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_expert_location_updater.py b/test/srt/test_expert_location_updater.py new file mode 100644 index 000000000..094540294 --- /dev/null +++ b/test/srt/test_expert_location_updater.py @@ -0,0 +1,255 @@ +import os +import traceback +import unittest +from dataclasses import dataclass +from typing import List + +import torch +import torch.distributed +import torch.multiprocessing as mp +from torch.multiprocessing import Process + +from sglang.srt.eplb import expert_location_updater +from sglang.test.test_utils import CustomTestCase, find_available_port +from sglang.utils import is_in_ci + + +@dataclass +class _TestInfo: + nnodes: int + num_logical_experts: int + num_physical_experts: int + num_repeat: int = 5000 + + +class TestExpertLocationUpdater(CustomTestCase): + @classmethod + def setUpClass(cls): + mp.set_start_method("spawn", force=True) + + def test_cpu(self): + self._test_common(device="cpu") + self._test_core( + num_gpus=32, + device="cpu", + infos=[ + _TestInfo( + nnodes=4, + num_logical_experts=256, + num_physical_experts=288, + num_repeat=10000, + ) + ], + ) + + def test_cpu_slow(self): + if is_in_ci(): + return + self._test_core( + num_gpus=144, + device="cpu", + infos=[ + _TestInfo( + nnodes=18, + num_logical_experts=256, + num_physical_experts=288, + num_repeat=10000, + ) + ], + ) + + def test_gpu(self): + if is_in_ci(): + return + self._test_common(device="cuda") + + def _test_common(self, device): + infos = [] + + for nnodes in [1, 2, 4]: + for num_logical_experts in [2, 5, 20, 256]: + for num_physical_experts in [8, 16, 256, 288]: + if num_logical_experts > num_physical_experts: + continue + infos.append( + _TestInfo( + nnodes=nnodes, + num_logical_experts=num_logical_experts, + num_physical_experts=num_physical_experts, + ) + ) + + self._test_core(num_gpus=8, device=device, infos=infos) + + def _test_core( + self, + num_gpus: int, + device: str, + infos: List[_TestInfo], + ): + master_port = find_available_port(23456) + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + for rank in range(num_gpus): + p = Process( + target=_run_subprocess, + kwargs=dict( + rank=rank, + num_gpus=num_gpus, + output_writer=output_writer, + master_port=master_port, + device=device, + infos=infos, + ), + ) + p.start() + processes.append(p) + + for _ in range(num_gpus): + self.assertTrue( + output_reader.recv(), f"Subprocess has error, please see logs above." + ) + + for p in processes: + p.join() + + +def _run_subprocess( + rank: int, + num_gpus: int, + master_port: int, + device: str, + infos: List[_TestInfo], + output_writer, +): + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + + torch.random.manual_seed(42) + torch.distributed.init_process_group( + rank=rank, + world_size=num_gpus, + backend={"cpu": "gloo", "cuda": None}[device], + ) + if device == "cuda": + torch.cuda.set_device(f"cuda:{rank}") + + for info in infos: + _execute_test(info, rank=rank, num_gpus=num_gpus, device=device) + + execution_ok = True + except Exception as e: + print(f"subprocess[{rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + +def _execute_test(info: _TestInfo, rank: int, num_gpus: int, device: str): + if rank == 0: + print(f"Test: {num_gpus=} {info=}", flush=True) + + assert info.num_physical_experts % num_gpus == 0 + num_local_physical_experts = info.num_physical_experts // num_gpus + assert num_gpus % info.nnodes == 0 + num_gpu_per_node = num_gpus // info.nnodes + + def _create_routed_experts_weights(physical_to_logical_map): + local_logical_expert_ids = physical_to_logical_map[ + rank * num_local_physical_experts : (rank + 1) * num_local_physical_experts + ].cpu() + return [ + local_logical_expert_ids.to(device).clone(), + torch.tensor( + [ + [local_logical_expert_id * 10, local_logical_expert_id * 100] + for local_logical_expert_id in local_logical_expert_ids.tolist() + ], + device=device, + ), + ] + + def _create_physical_to_logical_map(): + if rank == 0: + ans = torch.concat( + [ + torch.arange(0, info.num_logical_experts), + torch.randint( + 0, + info.num_logical_experts, + (info.num_physical_experts - info.num_logical_experts,), + ), + ] + ) + ans = ans[torch.randperm(ans.shape[0])] + else: + ans = torch.empty((info.num_physical_experts,), dtype=torch.int64) + + assert ans.dtype == torch.int64 and ans.shape == (info.num_physical_experts,) + ans = ans.to(device) + torch.distributed.broadcast(ans, src=0) + + return ans.cpu() + + physical_to_logical_map = _create_physical_to_logical_map() + routed_experts_weights = _create_routed_experts_weights(physical_to_logical_map) + + for i in range(info.num_repeat): + if rank == 0 and ((i % 500 == 0) or (i == info.num_repeat - 1)): + print(f"Step {i}/{info.num_repeat}", flush=True) + + new_physical_to_logical_map = _create_physical_to_logical_map() + expect_new_weights = _create_routed_experts_weights(new_physical_to_logical_map) + + output_logs = expert_location_updater.update_expert_weights_single_layer( + routed_experts_weights=routed_experts_weights, + temp_buffers=expert_location_updater.create_temp_buffers( + routed_experts_weights + ), + old_physical_to_logical_map=physical_to_logical_map.tolist(), + new_physical_to_logical_map=new_physical_to_logical_map.tolist(), + num_local_physical_experts=num_local_physical_experts, + num_gpu_per_node=num_gpu_per_node, + rank=rank, + debug=True, + ) + + local_has_error = not all( + torch.all(x == y) + for x, y in zip(routed_experts_weights, expect_new_weights, strict=True) + ) + global_has_error = torch.tensor(local_has_error, device=device) + torch.distributed.all_reduce( + global_has_error, op=torch.distributed.ReduceOp.MAX + ) + + if global_has_error.cpu().item(): + output_logs_str = "\n".join(output_logs) + local_message = ( + f"===================== rank {rank} ============================\n" + f"{num_gpus=} {info=}\n" + f"{routed_experts_weights[0].tolist()=}\n" + f"{expect_new_weights[0].tolist()=}\n" + f"{physical_to_logical_map.tolist()=}\n" + f"{new_physical_to_logical_map.tolist()=}\n" + f"===logs===\n" + f"{output_logs_str}\n" + f"==============================================================\n" + ) + + global_messages = ([None] * num_gpus) if rank == 0 else None + torch.distributed.gather_object(local_message, global_messages, dst=0) + + if rank == 0: + print("\n\n".join(global_messages), flush=True) + raise AssertionError(f"Error happens, see logs above") + + physical_to_logical_map = new_physical_to_logical_map + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fa3.py b/test/srt/test_fa3.py new file mode 100644 index 000000000..c9f286fca --- /dev/null +++ b/test/srt/test_fa3.py @@ -0,0 +1,261 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + +# In case of some machine lack internet connection, we can set OFFLINE_MODE to True. +OFFLINE_MODE = False + +# Change the path below when OFFLINE_MODE is True. +OFFLINE_PATH_DICT = { + DEFAULT_MODEL_NAME_FOR_TEST: "/shared/public/elr-models/meta-llama/Meta-Llama-3.1-8B-Instruct", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3: "/shared/public/elr-models/jamesliu1/sglang-EAGLE3-Llama-3.1-Instruct-8B", + DEFAULT_MODEL_NAME_FOR_TEST_MLA: "/shared/public/sharing/deepseek/dsv3-test/snapshots/", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN: "/shared/public/sharing/deepseek/dsv3-test-NextN/snapshots/", + GSM_DATASET_PATH: "/shared/public/data/gsm8k/test.jsonl", +} + + +if OFFLINE_MODE: + DEFAULT_MODEL_NAME_FOR_TEST = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST] + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 = OFFLINE_PATH_DICT[ + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3 + ] + DEFAULT_MODEL_NAME_FOR_TEST_MLA = OFFLINE_PATH_DICT[DEFAULT_MODEL_NAME_FOR_TEST_MLA] + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN = OFFLINE_PATH_DICT[ + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN + ] + GSM_DATASET_PATH = OFFLINE_PATH_DICT[GSM_DATASET_PATH] + + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--attention-backend", + "fa3", +] + +""" +Integration test for python/sglang/srt/layers/attention/flashattention_backend.py +""" + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class BaseFlashAttentionTest(CustomTestCase): + """Base class for testing FlashAttention3.""" + + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.65 # derived tests need to override this + speculative_decode = False + spec_decode_threshold = 1.0 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + os.environ["SGL_JIT_DEEPGEMM_PRECOMPILE"] = "false" + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "false" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + if self.speculative_decode: + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestFlashAttention3MLA(BaseFlashAttentionTest): + """Test FlashAttention3 with MLA, e.g. deepseek v3 test model""" + + accuracy_threshold = 0.60 + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + + +class TestFlashAttention3SpeculativeDecode(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with Llama 3.1 8B and its eagle3 model""" + + model = DEFAULT_MODEL_NAME_FOR_TEST + accuracy_threshold = 0.65 + speculative_decode = True + spec_decode_threshold = 1.5 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--dtype", + "float16", + ] + ) + return args + + +class TestFlashAttention3SpeculativeDecodeTopk(BaseFlashAttentionTest): + """Tests FlashAttention3 with enhanced speculative decoding using Llama 3.1 8B and EAGLE3. + This test will be using top-k value > 1 which would verify the other branches of the FA3 code + """ + + model = DEFAULT_MODEL_NAME_FOR_TEST + accuracy_threshold = 0.65 + speculative_decode = True + spec_decode_threshold = 1.6 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE3", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_EAGLE3, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + "--dtype", + "float16", + ] + ) + return args + + +class TestFlashAttention3MLASpeculativeDecode(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with deepseek v3 test model and its nextN model""" + + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + accuracy_threshold = 0.60 + speculative_decode = True + spec_decode_threshold = 2.5 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + ) + return args + + +class TestFlashAttention3MLASpeculativeDecodeTopk(BaseFlashAttentionTest): + """Test FlashAttention3 with speculative decode enabled with deepseek v3 test model and its nextN model + This test will be using top-k value > 1 which would verify the other branches of the FA3 code + """ + + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + accuracy_threshold = 0.60 + speculative_decode = True + spec_decode_threshold = 2.95 + + @classmethod + def get_server_args(cls): + args = DEFAULT_SERVER_ARGS + args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--speculative-num-steps", + "5", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "8", + ] + ) + return args + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fim_completion.py b/test/srt/test_fim_completion.py new file mode 100644 index 000000000..09db1d4bc --- /dev/null +++ b/test/srt/test_fim_completion.py @@ -0,0 +1,72 @@ +import unittest + +import openai + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFimCompletion(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/deepseek-coder-1.3b-base" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + other_args = ["--completion-template", "deepseek_coder"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=other_args, + ) + cls.base_url += "/v1" + cls.tokenizer = get_tokenizer(cls.model) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_fim_completion(self, number_of_completion): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + prompt = "function sum(a: number, b: number): number{\n" + suffix = "}" + + prompt_input = self.tokenizer.encode(prompt) + self.tokenizer.encode(suffix) + num_prompt_tokens = len(prompt_input) + 2 + + response = client.completions.create( + model=self.model, + prompt=prompt, + suffix=suffix, + temperature=0.3, + max_tokens=32, + stream=False, + n=number_of_completion, + ) + + print(response) + print(len(response.choices)) + assert len(response.choices) == number_of_completion + assert response.id + assert response.created + assert response.object == "text_completion" + assert ( + response.usage.prompt_tokens == num_prompt_tokens + ), f"{response.usage.prompt_tokens} vs {num_prompt_tokens}" + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_fim_completion(self): + for number_of_completion in [1, 3]: + self.run_fim_completion(number_of_completion) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_flashmla.py b/test/srt/test_flashmla.py new file mode 100644 index 000000000..681c9b8eb --- /dev/null +++ b/test/srt/test_flashmla.py @@ -0,0 +1,153 @@ +""" +Usage: +python3 test/srt/test_flashmla.py +""" + +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch, +) + + +class TestFlashMLAAttnBackend(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--cuda-graph-max-bs", + "2", + "--attention-backend", + "flashmla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + +class TestFlashMLAAttnLatency(unittest.TestCase): + def test_latency(self): + _, output_throughput, _ = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + [ + "--attention-backend", + "flashmla", + "--enable-torch-compile", + "--cuda-graph-max-bs", + "16", + "--trust-remote-code", + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 100) + + +class TestFlashMLAMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + "lmsys/sglang-ci-dsv3-test-NextN", + "--speculative-num-steps", + "1", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "2", + "--attention-backend", + "flashmla", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 1.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_forward_split_prefill.py b/test/srt/test_forward_split_prefill.py new file mode 100644 index 000000000..bbd247583 --- /dev/null +++ b/test/srt/test_forward_split_prefill.py @@ -0,0 +1,299 @@ +""" +Test forward_split_prefill functionality. + +Usage: +python3 -m unittest test_forward_split_prefill.TestForwardSplitPrefill +or +python3 test_forward_split_prefill.py +""" + +import time +import unittest + +import numpy as np +import torch + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestForwardSplitPrefill(CustomTestCase): + """Test cases for forward_split_prefill functionality.""" + + @classmethod + def setUpClass(cls): + """Set up the test environment once for all tests.""" + cls.model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.tp_size = 1 + cls.device = "cuda" + + # Initialize server args + cls.server_args = ServerArgs( + model_path=cls.model_path, + tokenizer_path=cls.model_path, + host="127.0.0.1", + disable_cuda_graph=True, # Disable CUDA graph for testing split prefill + disable_hybrid_swa_memory=True, + port=30000, + tp_size=cls.tp_size, + mem_fraction_static=0.8, + trust_remote_code=True, + ) + + cls.port_args = PortArgs.init_new(cls.server_args) + + # Load model and tokenizer + cls.model_config = ModelConfig.from_server_args(cls.server_args) + cls.model_runner = ModelRunner( + model_config=cls.model_config, + mem_fraction_static=cls.server_args.mem_fraction_static, + gpu_id=0, + tp_rank=0, + tp_size=cls.tp_size, + pp_rank=0, + pp_size=1, + nccl_port=cls.port_args.nccl_port, + server_args=cls.server_args, + ) + + cls.tokenizer = get_tokenizer( + cls.server_args.tokenizer_path, + tokenizer_mode=cls.server_args.tokenizer_mode, + trust_remote_code=cls.server_args.trust_remote_code, + ) + + print( + f"Test with model: {cls.model_path}, num_hidden_layers: {cls.model_config.num_hidden_layers}" + ) + + def prepare_test_batch(self, batch_size=2, input_len=128, is_split_prefill=True): + """Prepare a test batch for split prefill testing.""" + # Create synthetic input + input_ids = np.random.randint(10, 1000, (batch_size, input_len), dtype=np.int32) + + sampling_params = SamplingParams( + temperature=0.0, + max_new_tokens=8, + ) + + reqs = [] + for i in range(batch_size): + req = Req( + rid=i, + origin_input_text="", + origin_input_ids=list(input_ids[i]), + sampling_params=sampling_params, + ) + req.prefix_indices = [] + req.fill_ids = req.origin_input_ids + req.extend_input_len = len(req.fill_ids) - len(req.prefix_indices) + req.logprob_start_len = len(req.origin_input_ids) - 1 + reqs.append(req) + + batch = ScheduleBatch.init_new( + reqs=reqs, + req_to_token_pool=self.model_runner.req_to_token_pool, + token_to_kv_pool_allocator=self.model_runner.token_to_kv_pool_allocator, + tree_cache=None, + model_config=self.model_config, + enable_overlap=False, + spec_algorithm=SpeculativeAlgorithm.NONE, + enable_custom_logit_processor=False, + ) + if is_split_prefill: + batch.prepare_for_split_prefill() + else: + batch.prepare_for_extend() + + # Create forward batch + model_worker_batch = batch.get_model_worker_batch() + forward_batch = ForwardBatch.init_new(model_worker_batch, self.model_runner) + + return forward_batch + + def test_split_prefill_functionality(self): + """Test that split prefill can complete successfully.""" + print("\n=== Testing split prefill functionality ===") + + forward_batch = self.prepare_test_batch(batch_size=2, input_len=64) + + # Reset split index + forward_batch.split_index = 0 + + # Test split prefill in chunks + num_layers = self.model_config.num_hidden_layers + chunk_size = max(1, num_layers // 4) # Split into 4 chunks + + results = [] + split_count = 0 + + while forward_batch.split_index < num_layers: + print( + f"Processing split {split_count}, split_index: {forward_batch.split_index}" + ) + + result = self.model_runner.forward_split_prefill( + forward_batch=forward_batch, + reinit_attn_backend=(split_count == 0), + forward_count=chunk_size, + ) + + results.append(result) + split_count += 1 + + # Verify split_index is updated correctly + expected_next_index = min(split_count * chunk_size, num_layers) + self.assertEqual(forward_batch.split_index, expected_next_index) + + # The last result should contain logits + self.assertIsNotNone(results[-1], "Final split should return logits") + print(f"Split prefill completed in {split_count} splits") + + def test_split_prefill_vs_normal_prefill(self): + """Test that split prefill produces the same results as normal prefill.""" + print("\n=== Testing split prefill vs normal prefill consistency ===") + + forward_batch_normal = self.prepare_test_batch( + batch_size=2, input_len=128, is_split_prefill=False + ) + forward_batch_split = self.prepare_test_batch( + batch_size=2, input_len=128, is_split_prefill=True + ) + + # Ensure same input + forward_batch_split.input_ids = forward_batch_normal.input_ids.clone() + forward_batch_split.positions = forward_batch_normal.positions.clone() + + # Method 1: Normal extend (prefill) + print("Running normal extend (prefill)...") + normal_result = self.model_runner.forward_extend(forward_batch_normal) + + # Method 2: Split prefill + print("Running split prefill...") + num_layers = self.model_config.num_hidden_layers + chunk_size = max(1, num_layers // 3) # Split into 3 chunks + + split_result = None + + while forward_batch_split.split_index < num_layers: + result = self.model_runner.forward_split_prefill( + forward_batch=forward_batch_split, + forward_count=chunk_size, + ) + if result is not None: + split_result = result + + # Compare results + self.assertIsNotNone(normal_result, "Normal prefill should return result") + self.assertIsNotNone(split_result, "Split prefill should return result") + + # Compare logits shapes + self.assertEqual( + normal_result.next_token_logits.shape, + split_result.next_token_logits.shape, + "Logits shapes should match", + ) + + # Compare logits values (should be very close due to same computation) + # Use a larger tolerance for numerical differences in split computation + torch.testing.assert_close( + normal_result.next_token_logits, + split_result.next_token_logits, + rtol=1e-3, + atol=1e-3, + msg="Split prefill and normal prefill should produce similar logits", + ) + + print("✓ Split prefill and normal prefill produce consistent results") + + def test_split_prefill_different_chunk_sizes(self): + """Test split prefill with different chunk sizes.""" + print("\n=== Testing split prefill with different chunk sizes ===") + + num_layers = self.model_config.num_hidden_layers + chunk_sizes = [1, 2, max(1, num_layers // 2), num_layers] + + # Prepare identical batches for each test + base_batch = self.prepare_test_batch(batch_size=1, input_len=16) + base_input_ids = base_batch.input_ids.clone() + base_positions = base_batch.positions.clone() + + results = [] + + for chunk_size in chunk_sizes: + if chunk_size > num_layers: + continue + + print(f"Testing chunk size: {chunk_size}") + + # Prepare fresh batch + forward_batch = self.prepare_test_batch(batch_size=1, input_len=16) + forward_batch.input_ids = base_input_ids.clone() + forward_batch.positions = base_positions.clone() + forward_batch.split_index = 0 + + # Run split prefill + split_result = None + + while forward_batch.split_index < num_layers: + result = self.model_runner.forward_split_prefill( + forward_batch=forward_batch, + forward_count=chunk_size, + ) + if result is not None: + split_result = result + + self.assertIsNotNone( + split_result, + f"Split prefill should succeed with chunk_size={chunk_size}", + ) + results.append(split_result) + + # Compare all results should be identical (same input, same computation) + if len(results) > 1: + for i, result in enumerate(results[1:], 1): + torch.testing.assert_close( + results[0].next_token_logits, + result.next_token_logits, + rtol=1e-3, + atol=1e-3, + msg=f"Results with different chunk sizes should be identical (chunk_size {chunk_sizes[i]})", + ) + + print("✓ All chunk sizes produce consistent results") + + def test_split_prefill_edge_cases(self): + """Test edge cases for split prefill.""" + print("\n=== Testing split prefill edge cases ===") + + # Test with single layer chunks + forward_batch = self.prepare_test_batch(batch_size=1, input_len=8) + + # Process one layer at a time + num_layers = self.model_config.num_hidden_layers + for layer_idx in range(num_layers): + result = self.model_runner.forward_split_prefill( + forward_batch=forward_batch, + reinit_attn_backend=(layer_idx == 0), + forward_count=1, # One layer at a time + ) + + if layer_idx == num_layers - 1: + # Last layer should return result + self.assertIsNotNone(result, "Last layer should return logits") + else: + # Intermediate layers should return None + self.assertIsNone(result, f"Layer {layer_idx} should return None") + + print("✓ Single layer processing works correctly") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_full_deepseek_v3.py b/test/srt/test_full_deepseek_v3.py new file mode 100644 index 000000000..f6a58536a --- /dev/null +++ b/test/srt/test_full_deepseek_v3.py @@ -0,0 +1,162 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" + + +class TestDeepseekV3(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--tp", "8"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + args = SimpleNamespace( + num_shots=8, + data_path=None, + num_questions=1400, + parallel=1400, + max_new_tokens=512, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3)\n" f'{metrics["accuracy"]=:.3f}\n' + ) + self.assertGreater(metrics["accuracy"], 0.935) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3)\n" f"{speed=:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(speed, 12) + else: + self.assertGreater(speed, 75) + + +class TestDeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = FULL_DEEPSEEK_V3_MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_a_gsm8k( + self, + ): # Append an "a" to make this test run first (alphabetically) to warm up the server + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + + if is_in_ci(): + write_github_step_summary( + f"### test_gsm8k (deepseek-v3 mtp)\n" + f'{metrics["accuracy"]=:.3f}\n' + f"{avg_spec_accept_length=:.2f}\n" + ) + self.assertGreater(metrics["accuracy"], 0.935) + self.assertGreater(avg_spec_accept_length, 2.9) + + def test_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + if is_in_amd_ci(): + self.assertGreater(acc_length, 2.8) + else: + self.assertGreater(acc_length, 2.9) + if is_in_amd_ci(): + self.assertGreater(speed, 15) + else: + self.assertGreater(speed, 130) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_function_call_parser.py b/test/srt/test_function_call_parser.py new file mode 100644 index 000000000..10003a4db --- /dev/null +++ b/test/srt/test_function_call_parser.py @@ -0,0 +1,2194 @@ +import json +import unittest + +from xgrammar import GrammarCompiler, TokenizerInfo + +from sglang.srt.entrypoints.openai.protocol import Function, Tool +from sglang.srt.function_call.base_format_detector import BaseFormatDetector +from sglang.srt.function_call.deepseekv3_detector import DeepSeekV3Detector +from sglang.srt.function_call.glm4_moe_detector import Glm4MoeDetector +from sglang.srt.function_call.kimik2_detector import KimiK2Detector +from sglang.srt.function_call.llama32_detector import Llama32Detector +from sglang.srt.function_call.mistral_detector import MistralDetector +from sglang.srt.function_call.pythonic_detector import PythonicDetector +from sglang.srt.function_call.qwen3_coder_detector import Qwen3CoderDetector +from sglang.srt.function_call.qwen25_detector import Qwen25Detector +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class TestPythonicDetector(unittest.TestCase): + def setUp(self): + # Create sample tools for testing + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "properties": { + "location": { + "type": "string", + "description": "Location to get weather for", + }, + "unit": { + "type": "string", + "description": "Temperature unit", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="search", + description="Search for information", + parameters={ + "properties": { + "query": { + "type": "string", + "description": "Search query", + }, + }, + "required": ["query"], + }, + ), + ), + ] + self.detector = PythonicDetector() + + def test_parse_streaming_no_brackets(self): + """Test parsing text with no brackets (no tool calls).""" + text = "This is just normal text without any tool calls." + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, text) + self.assertEqual(result.calls, []) + self.assertEqual(self.detector._buffer, "") # Buffer should be cleared + + def test_parse_streaming_complete_tool_call(self): + """Test parsing a complete tool call.""" + text = "Here's a tool call: [get_weather(location='New York', unit='celsius')]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "Here's a tool call: ") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + self.detector._buffer, "" + ) # Buffer should be cleared after processing + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["location"], "New York") + self.assertEqual(params["unit"], "celsius") + + def test_parse_streaming_text_before_tool_call(self): + """Test parsing text that appears before a tool call.""" + text = "This is some text before [get_weather(location='London')]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "This is some text before ") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["location"], "London") + + def test_parse_streaming_partial_tool_call(self): + """Test parsing a partial tool call that spans multiple chunks.""" + # First chunk with opening bracket but no closing bracket + text1 = "Let me check the weather: [get_weather(location=" + result1 = self.detector.parse_streaming_increment(text1, self.tools) + + self.assertEqual(result1.normal_text, "Let me check the weather: ") + self.assertEqual(result1.calls, []) + self.assertEqual( + self.detector._buffer, "[get_weather(location=" + ) # Partial tool call remains in buffer + + # Second chunk completing the tool call + text2 = "'Paris')]" + result2 = self.detector.parse_streaming_increment(text2, self.tools) + + self.assertEqual(result2.normal_text, "") + self.assertEqual(len(result2.calls), 1) + self.assertEqual(result2.calls[0].name, "get_weather") + + # Check the parameters + params = json.loads(result2.calls[0].parameters) + self.assertEqual(params["location"], "Paris") + self.assertEqual( + self.detector._buffer, "" + ) # Buffer should be cleared after processing + + def test_parse_streaming_bracket_without_text_before(self): + """Test parsing a tool call that starts at the beginning of the text.""" + text = "[search(query='python programming')]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "search") + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["query"], "python programming") + + def test_parse_streaming_text_after_tool_call(self): + """Test parsing text that appears after a tool call.""" + # First chunk with complete tool call and some text after + text = "[get_weather(location='Tokyo')] Here's the forecast:" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + self.detector._buffer, " Here's the forecast:" + ) # Text after tool call remains in buffer + + # Process the remaining text in buffer + result2 = self.detector.parse_streaming_increment("", self.tools) + self.assertEqual(result2.normal_text, " Here's the forecast:") + self.assertEqual(result2.calls, []) + self.assertEqual(self.detector._buffer, "") # Buffer should be cleared + + def test_parse_streaming_multiple_tool_calls(self): + """Test parsing multiple tool calls in sequence.""" + text = "[get_weather(location='Berlin')] and [search(query='restaurants')]" + + # First tool call + result1 = self.detector.parse_streaming_increment(text, self.tools) + self.assertEqual(len(result1.calls), 1) + self.assertEqual(result1.calls[0].name, "get_weather") + self.assertEqual(self.detector._buffer, " and [search(query='restaurants')]") + + # Second tool call + result2 = self.detector.parse_streaming_increment("", self.tools) + self.assertEqual(result2.normal_text, " and ") + self.assertEqual(len(result2.calls), 1) + self.assertEqual(result2.calls[0].name, "search") + self.assertEqual(self.detector._buffer, "") + + def test_parse_streaming_opening_bracket_only(self): + """Test parsing text with only an opening bracket but no closing bracket.""" + text = "Let's try this: [" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "Let's try this: ") + self.assertEqual(result.calls, []) + self.assertEqual( + self.detector._buffer, "[" + ) # Opening bracket remains in buffer + + def test_parse_streaming_nested_brackets(self): + """Test parsing tool calls with nested brackets in arguments.""" + # Test with list argument containing nested brackets + text = "[get_weather(location='New York', unit='celsius', data=[1, 2, 3])]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual(self.detector._buffer, "") + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["location"], "New York") + self.assertEqual(params["unit"], "celsius") + self.assertEqual(params["data"], [1, 2, 3]) + + def test_parse_streaming_nested_brackets_dict(self): + """Test parsing tool calls with nested dictionaries and lists.""" + # Test with nested dict and list arguments + text = "[search(query='test', config={'options': [1, 2], 'nested': {'key': 'value'}})]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "search") + self.assertEqual(self.detector._buffer, "") + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["query"], "test") + self.assertEqual(params["config"]["options"], [1, 2]) + self.assertEqual(params["config"]["nested"]["key"], "value") + + def test_parse_streaming_multiple_tools_with_nested_brackets(self): + """Test parsing multiple tool calls with nested brackets.""" + text = "[get_weather(location='Paris', data=[10, 20]), search(query='test', filters=['a', 'b'])]" + result = self.detector.parse_streaming_increment(text, self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 2) + self.assertEqual(self.detector._buffer, "") + + # Check first tool call + params1 = json.loads(result.calls[0].parameters) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual(params1["location"], "Paris") + self.assertEqual(params1["data"], [10, 20]) + + # Check second tool call + params2 = json.loads(result.calls[1].parameters) + self.assertEqual(result.calls[1].name, "search") + self.assertEqual(params2["query"], "test") + self.assertEqual(params2["filters"], ["a", "b"]) + + def test_parse_streaming_partial_nested_brackets(self): + """Test parsing partial tool calls with nested brackets across chunks.""" + # First chunk with nested brackets but incomplete + text1 = "Here's a call: [get_weather(location='Tokyo', data=[1, 2" + result1 = self.detector.parse_streaming_increment(text1, self.tools) + + self.assertEqual(result1.normal_text, "Here's a call: ") + self.assertEqual(result1.calls, []) + self.assertEqual( + self.detector._buffer, "[get_weather(location='Tokyo', data=[1, 2" + ) + + # Second chunk completing the nested brackets + text2 = ", 3])]" + result2 = self.detector.parse_streaming_increment(text2, self.tools) + + self.assertEqual(result2.normal_text, "") + self.assertEqual(len(result2.calls), 1) + self.assertEqual(result2.calls[0].name, "get_weather") + self.assertEqual(self.detector._buffer, "") + + # Check the parameters + params = json.loads(result2.calls[0].parameters) + self.assertEqual(params["location"], "Tokyo") + self.assertEqual(params["data"], [1, 2, 3]) + + def test_parse_streaming_with_python_start_and_end_token(self): + """Test parsing a message that starts with <|python_start|> and <|python_end|> across chunks.""" + chunks = [ + "Here's a call: ", + "<|python_", + "start|>[get_weather(location=", + "'Tokyo', data=[1, 2", + ", 3])]<|python_end|>", + ] + + normal_text = "" + call_name = "" + parameters = "" + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + if result.normal_text: + normal_text += result.normal_text + if result.calls: + call_name += result.calls[0].name + parameters += result.calls[0].parameters + + self.assertEqual(normal_text, "Here's a call: ") + self.assertEqual(call_name, "get_weather") + self.assertEqual(self.detector._buffer, "") + self.assertEqual( + result.normal_text, "", "Final result should have no normal text" + ) + + # Check the parameters + params = json.loads(parameters) + self.assertEqual(params["location"], "Tokyo") + self.assertEqual(params["data"], [1, 2, 3]) + + chunks = [ + "Here's a call: <|python_start|>[get_weather(location='Tokyo', data=[1, 2, 3])]<|python_end|>" + ] + + normal_text = "" + call_name = "" + parameters = "" + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + if result.normal_text: + normal_text += result.normal_text + if result.calls: + call_name += result.calls[0].name + parameters += result.calls[0].parameters + + self.assertEqual(normal_text, "Here's a call: ") + self.assertEqual(call_name, "get_weather") + self.assertEqual(self.detector._buffer, "") + + # Check the parameters + params = json.loads(parameters) + self.assertEqual(params["location"], "Tokyo") + self.assertEqual(params["data"], [1, 2, 3]) + + def test_detect_and_parse_with_python_start_and_end_token(self): + """Test parsing a message that starts with <|python_start|> and contains a valid tool call.""" + text = "User wants to get the weather in Mars. <|python_start|>[get_weather(location='Mars', unit='celsius')]<|python_end|> In this way we will get the weather in Mars." + result = self.detector.detect_and_parse(text, self.tools) + + self.assertEqual( + result.normal_text, + "User wants to get the weather in Mars. In this way we will get the weather in Mars.", + ) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual(self.detector._buffer, "") + + # Check the parameters + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["location"], "Mars") + self.assertEqual(params["unit"], "celsius") + + +class TestMistralDetector(unittest.TestCase): + def setUp(self): + """Set up test tools and detector for Mistral format testing.""" + self.tools = [ + Tool( + type="function", + function=Function( + name="make_next_step_decision", + description="Test function for decision making", + parameters={ + "type": "object", + "properties": { + "decision": { + "type": "string", + "description": "The next step to take", + }, + "content": { + "type": "string", + "description": "The content of the next step", + }, + }, + "required": ["decision", "content"], + }, + ), + ), + ] + self.detector = MistralDetector() + + def test_detect_and_parse_with_nested_brackets_in_content(self): + """Test parsing Mistral format with nested brackets in JSON content. + + This test case specifically addresses the issue where the regex pattern + was incorrectly truncating JSON when it contained nested brackets like [City Name]. + """ + # This is the exact problematic text from the original test failure + test_text = '[TOOL_CALLS] [{"name":"make_next_step_decision", "arguments":{"decision":"","content":"```\\nTOOL: Access a weather API or service\\nOBSERVATION: Retrieve the current weather data for the top 5 populated cities in the US\\nANSWER: The weather in the top 5 populated cities in the US is as follows: [City Name] - [Weather Conditions] - [Temperature]\\n```"}}]' + + result = self.detector.detect_and_parse(test_text, self.tools) + + # Verify that the parsing was successful + self.assertEqual(len(result.calls), 1, "Should detect exactly one tool call") + + call = result.calls[0] + self.assertEqual( + call.name, + "make_next_step_decision", + "Should detect the correct function name", + ) + + # Verify that the parameters are valid JSON and contain the expected content + params = json.loads(call.parameters) + self.assertEqual( + params["decision"], "", "Decision parameter should be empty string" + ) + + # The content should contain the full text including the nested brackets [City Name] + expected_content = "```\nTOOL: Access a weather API or service\nOBSERVATION: Retrieve the current weather data for the top 5 populated cities in the US\nANSWER: The weather in the top 5 populated cities in the US is as follows: [City Name] - [Weather Conditions] - [Temperature]\n```" + self.assertEqual( + params["content"], + expected_content, + "Content should include nested brackets without truncation", + ) + + # Verify that normal text is empty (since the entire input is a tool call) + self.assertEqual( + result.normal_text, "", "Normal text should be empty for pure tool call" + ) + + def test_detect_and_parse_simple_case(self): + """Test parsing a simple Mistral format tool call without nested brackets.""" + test_text = '[TOOL_CALLS] [{"name":"make_next_step_decision", "arguments":{"decision":"TOOL", "content":"Use weather API"}}]' + + result = self.detector.detect_and_parse(test_text, self.tools) + + self.assertEqual(len(result.calls), 1) + call = result.calls[0] + self.assertEqual(call.name, "make_next_step_decision") + + params = json.loads(call.parameters) + self.assertEqual(params["decision"], "TOOL") + self.assertEqual(params["content"], "Use weather API") + + def test_detect_and_parse_no_tool_calls(self): + """Test parsing text without any tool calls.""" + test_text = "This is just normal text without any tool calls." + + result = self.detector.detect_and_parse(test_text, self.tools) + + self.assertEqual(len(result.calls), 0, "Should detect no tool calls") + self.assertEqual( + result.normal_text, + test_text, + "Should return the original text as normal text", + ) + + def test_detect_and_parse_with_text_before_tool_call(self): + """Test parsing text that has content before the tool call.""" + test_text = 'Here is some text before the tool call: [TOOL_CALLS] [{"name":"make_next_step_decision", "arguments":{"decision":"ANSWER", "content":"The answer is 42"}}]' + + result = self.detector.detect_and_parse(test_text, self.tools) + + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.normal_text, "Here is some text before the tool call:") + + call = result.calls[0] + self.assertEqual(call.name, "make_next_step_decision") + + params = json.loads(call.parameters) + self.assertEqual(params["decision"], "ANSWER") + self.assertEqual(params["content"], "The answer is 42") + + +class TestEBNFGeneration(unittest.TestCase): + def setUp(self): + # Create sample tools for testing + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "properties": { + "location": { + "type": "string", + "description": "Location to get weather for", + }, + "unit": { + "type": "string", + "description": "Temperature unit", + "enum": ["celsius", "fahrenheit"], + }, + }, + "required": ["location"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="search", + description="Search for information", + parameters={ + "properties": { + "query": { + "type": "string", + "description": "Search query", + }, + }, + "required": ["query"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="empty_param_func", + description="Function with empty parameters", + parameters={ + "properties": {}, + "required": [], + }, + ), + ), + ] + + self.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + tokenizer_info = TokenizerInfo.from_huggingface(self.tokenizer) + self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info) + + # Initialize all detectors + self.pythonic_detector = PythonicDetector() + self.deepseekv3_detector = DeepSeekV3Detector() + self.llama32_detector = Llama32Detector() + self.mistral_detector = MistralDetector() + self.qwen25_detector = Qwen25Detector() + self.qwen3_coder_detector = Qwen3CoderDetector() + self.kimik2_detector = KimiK2Detector() + self.glm45_detector = Glm4MoeDetector() + + def test_pythonic_detector_ebnf(self): + """Test that the PythonicDetector generates valid EBNF.""" + ebnf = self.pythonic_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns + self.assertIn('call_get_weather ::= "get_weather" "(" ', ebnf) + self.assertIn('"location" "=" basic_string', ebnf) + self.assertIn('( "unit" "=" ("\\"celsius\\"" | "\\"fahrenheit\\"") )', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_deepseekv3_detector_ebnf(self): + """Test that the DeepSeekV3Detector generates valid EBNF.""" + ebnf = self.deepseekv3_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns + self.assertIn("<｜tool▁calls▁begin｜>", ebnf) + self.assertIn("<｜tool▁call▁begin｜>function<｜tool▁sep｜>get_weather", ebnf) + self.assertIn('\\"location\\"" ws ":" ws basic_string ', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_kimik2_detector_ebnf(self): + """Test that the KimiK2Detector generates valid EBNF.""" + ebnf = self.kimik2_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns for KimiK2 format + self.assertIn("<|tool_calls_section_begin|>", ebnf) + self.assertIn("<|tool_calls_section_end|>", ebnf) + + # Check for KimiK2-specific function call structure + self.assertIn("<|tool_call_begin|>functions.get_weather:", ebnf) + self.assertIn("<|tool_call_begin|>functions.search:", ebnf) + self.assertIn("<|tool_call_argument_begin|>", ebnf) + self.assertIn("<|tool_call_end|>", ebnf) + + # Check that it uses the correct namespace.function format with numeric index pattern + self.assertIn("functions.get_weather:", ebnf) + self.assertIn("functions.search:", ebnf) + self.assertIn("[0-9]+", ebnf) # Numeric index pattern + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_llama32_detector_ebnf(self): + """Test that the Llama32Detector generates valid EBNF.""" + ebnf = self.llama32_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns + self.assertIn('\\"name\\"" ws ":" ws "\\"get_weather\\"', ebnf) + self.assertIn('"\\"arguments\\"" ws ":"', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_mistral_detector_ebnf(self): + """Test that the MistralDetector generates valid EBNF.""" + ebnf = self.mistral_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns + self.assertIn('"[TOOL_CALLS] ["', ebnf) + self.assertIn("call_get_weather | call_search", ebnf) + self.assertIn('"\\"arguments\\"" ws ":"', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_qwen25_detector_ebnf(self): + """Test that the Qwen25Detector generates valid EBNF.""" + ebnf = self.qwen25_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + + # Check that the EBNF contains expected patterns + self.assertIn("", ebnf) + self.assertIn('\\"name\\"" ws ":" ws "\\"get_weather\\"', ebnf) + self.assertIn('"\\"arguments\\"" ws ":"', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_glm45_detector_ebnf(self): + """Test that the Glm4MoeDetector generates valid EBNF.""" + ebnf = self.glm45_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + # Check that the EBNF contains expected patterns for XML format + self.assertIn('"" function_call ""', ebnf) + self.assertIn('"get_weather" "\\n" ( arguments_get_weather "\\n" )?', ebnf) + self.assertIn( + '"location" "\\n" "" xml_text "" ( "\\n" ( "unit" "\\n" "" ("celsius" | "fahrenheit") "" ) )?', + ebnf, + ) + self.assertIn('"search" "\\n" ( arguments_search "\\n" )?', ebnf) + self.assertIn( + '"query" "\\n" "" xml_text ""', + ebnf, + ) + self.assertIn( + '"empty_param_func" "\\n" ( arguments_empty_param_func "\\n" )?', ebnf + ) + self.assertIn('arguments_empty_param_func ::= ""', ebnf) + + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_qwen3_coder_detector_ebnf(self): + """Test that the Qwen3CoderDetector generates valid EBNF.""" + ebnf = self.qwen3_coder_detector.build_ebnf(self.tools) + self.assertIsNotNone(ebnf) + # Check that the EBNF contains expected patterns for XML format + self.assertIn("", ebnf) + self.assertIn("", ebnf) + self.assertIn('"\\n"', ebnf) + self.assertIn('"\\n"', ebnf) + self.assertIn('"\\n"', ebnf) + self.assertIn('"\\n"', ebnf) + # Check that it uses xml_text for string parameters + self.assertIn("xml_text", ebnf) + # Validate that the EBNF can be compiled by GrammarCompiler + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone(ctx, "EBNF should be valid and compile successfully") + except RuntimeError as e: + self.fail(f"Failed to compile EBNF: {e}") + + def test_weather_function_optional_parameter_handling(self): + """Test that weather function with optional unit parameter generates correct EBNF without trailing commas.""" + # Create a weather tool with required location and optional unit + weather_tool = Tool( + type="function", + function=Function( + name="get_current_weather", + description="Get the current weather in a given location", + parameters={ + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + ), + ) + + # Test all detectors with the weather tool + detectors = { + "pythonic": self.pythonic_detector, + "deepseekv3": self.deepseekv3_detector, + "llama32": self.llama32_detector, + "mistral": self.mistral_detector, + "qwen25": self.qwen25_detector, + } + + for name, detector in detectors.items(): + with self.subTest(detector=name): + ebnf = detector.build_ebnf([weather_tool]) + self.assertIsNotNone(ebnf, f"{name} detector should generate EBNF") + + # Check that the EBNF properly handles optional parameters + if name == "pythonic": + # Pythonic format: location="Paris" ( , ( unit=("celsius" | "fahrenheit") )? + self.assertIn('"location" "=" basic_string', ebnf) + # The comma should be inside the optional brackets for unit + self.assertIn('( ws "," ws ( "unit" "=" ', ebnf) + else: + # JSON format: "location": "Paris" ( , ( "unit": ("celsius" | "fahrenheit") )? + self.assertIn('"location\\"" ws ":" ws basic_string', ebnf) + # The comma should be part of the optional group + # This pattern ensures no trailing comma when unit is omitted + self.assertIn('( ws "," ws ( "\\"unit\\"" ws ":"', ebnf) + + # Validate that the EBNF can be compiled + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone( + ctx, f"{name} EBNF should compile successfully" + ) + except RuntimeError as e: + self.fail(f"Failed to compile {name} EBNF: {e}") + + def test_multiple_optional_parameters_flexible_ordering(self): + """Test that multiple optional parameters allow flexible ordering using llama.cpp approach.""" + # Create a tool with one required and multiple optional parameters + test_tool = Tool( + type="function", + function=Function( + name="test_func", + description="Test function with multiple optional parameters", + parameters={ + "type": "object", + "properties": { + "required_field": {"type": "string"}, + "opt1": {"type": "number"}, + "opt2": {"type": "boolean"}, + "opt3": {"type": "string"}, + }, + "required": ["required_field"], + }, + ), + ) + + # Test JSON-based detectors (not pythonic) + json_detectors = { + "deepseekv3": self.deepseekv3_detector, + "llama32": self.llama32_detector, + "mistral": self.mistral_detector, + "qwen25": self.qwen25_detector, + } + + for name, detector in json_detectors.items(): + with self.subTest(detector=name): + ebnf = detector.build_ebnf([test_tool]) + self.assertIsNotNone(ebnf, f"{name} detector should generate EBNF") + + # Print the arguments rule for debugging + lines = ebnf.split("\n") + args_rule = None + for line in lines: + if line.startswith("arguments_test_func ::="): + args_rule = line + break + + self.assertIsNotNone( + args_rule, f"{name} should have arguments_test_func rule" + ) + + # Check required field + self.assertIn('"required_field\\"" ws ":" ws basic_string', ebnf) + + # Check the structure for optional parameters + # The pattern should be: required_field ( "," ( opt1 ... | opt2 ... | opt3 ... ) )? + # This allows flexible ordering where any optional can be first + + # Check that optional parameters are in a group with comma + if args_rule: # Only check if args_rule was found + self.assertIn( + '( ws "," ws (', + args_rule, + f"{name} should have comma grouped with optional parameters", + ) + + # Check for the alternation pattern that allows flexible ordering + # Should contain patterns like: opt1 ... | opt2 ... | opt3 + self.assertIn('"opt1\\"" ws ":" ws basic_number', args_rule) + self.assertIn('"opt2\\"" ws ":" ws basic_boolean', args_rule) + self.assertIn('"opt3\\"" ws ":" ws basic_string', args_rule) + + # Check for alternation (|) which allows skipping optional parameters + self.assertIn( + "|", + args_rule, + f"{name} should use alternation for flexible optional ordering", + ) + + # Check that the pattern ends properly with closing braces + self.assertTrue( + args_rule.endswith('"}"'), + f"{name} arguments rule should end with closing brace", + ) + + # Validate compilation + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone( + ctx, f"{name} EBNF should compile successfully" + ) + except RuntimeError as e: + self.fail(f"Failed to compile {name} EBNF: {e}") + + def test_all_optional_parameters_ordering(self): + """Test the behavior when ALL parameters are optional - verifies ordering constraints.""" + # Create a tool with only optional parameters + all_optional_tool = Tool( + type="function", + function=Function( + name="optional_func", + description="Function with all optional parameters", + parameters={ + "type": "object", + "properties": { + "opt1": {"type": "string"}, + "opt2": {"type": "number"}, + "opt3": {"type": "boolean"}, + }, + "required": [], # No required parameters + }, + ), + ) + + # Test JSON-based detectors + json_detectors = { + "deepseekv3": self.deepseekv3_detector, + "llama32": self.llama32_detector, + "mistral": self.mistral_detector, + "qwen25": self.qwen25_detector, + } + + for name, detector in json_detectors.items(): + with self.subTest(detector=name): + ebnf = detector.build_ebnf([all_optional_tool]) + self.assertIsNotNone(ebnf, f"{name} detector should generate EBNF") + + # Extract the arguments rule + lines = ebnf.split("\n") + args_rule = None + for line in lines: + if line.startswith("arguments_optional_func ::="): + args_rule = line + break + + self.assertIsNotNone( + args_rule, f"{name} should have arguments_optional_func rule" + ) + + if args_rule: + # When all parameters are optional, the pattern now uses alternation: + # "{" ( opt1 ... | opt2 ... | opt3 ... )? "}" + # This allows flexible ordering where any optional can appear first + + # Check the structure + self.assertIn('"opt1\\"" ws ":" ws basic_string', args_rule) + self.assertIn('"opt2\\"" ws ":" ws basic_number', args_rule) + self.assertIn('"opt3\\"" ws ":" ws basic_boolean', args_rule) + + # The pattern SHOULD have alternation (|) for flexible ordering + self.assertIn( + "|", + args_rule, + f"{name} should use alternation for flexible ordering even when all properties are optional", + ) + + # Validate compilation + try: + ctx = self.grammar_compiler.compile_grammar(ebnf) + self.assertIsNotNone( + ctx, f"{name} EBNF should compile successfully" + ) + except RuntimeError as e: + self.fail(f"Failed to compile {name} EBNF: {e}") + + +class TestBaseFormatDetector(unittest.TestCase): + """Test buffer management and sequential tool index assignment in BaseFormatDetector.""" + + def setUp(self): + """Set up test detector and tools.""" + + # Create a concrete implementation of BaseFormatDetector for testing + class TestFormatDetector(BaseFormatDetector): + def __init__(self): + super().__init__() + self.bot_token = "" + self.eot_token = "" + + def detect_and_parse(self, text, tools): + # Not used in streaming tests + pass + + def has_tool_call(self, text): + return "" in text + + def structure_info(self): + # Not used in streaming tests + pass + + def build_ebnf(self, tools): + # Not used in streaming tests + pass + + self.detector = TestFormatDetector() + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_tourist_attractions", + description="Get tourist attractions", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + ] + + def test_sequential_tool_index_assignment(self): + """Test that multiple tool calls get sequential tool_index values (0, 1, 2, ...).""" + # Simulate streaming chunks for two consecutive tool calls + chunks = [ + "", + '{"name": "get_weather", ', + '"arguments": {"city": "Paris"}}', + ", ", + '{"name": "get_tourist_attractions", ', + '"arguments": {"city": "London"}}', + "", + ] + + tool_indices_seen = [] + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + if result.calls: + for call in result.calls: + if call.tool_index is not None: + tool_indices_seen.append(call.tool_index) + + # Verify we got sequential tool indices + unique_indices = sorted(set(tool_indices_seen)) + self.assertEqual( + unique_indices, + [0, 1], + f"Expected sequential tool indices [0, 1], got {unique_indices}", + ) + + def test_buffer_content_preservation(self): + """Test that buffer correctly preserves unprocessed content when tool completes.""" + # Test simpler scenario: tool completion followed by new tool start + chunks = [ + "", + '{"name": "get_weather", ', + '"arguments": {"city": "Paris"}}', + ", ", + '{"name": "get_tourist_attractions", ', + '"arguments": {"city": "London"}} ', + ] + + tool_calls_seen = [] + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + if result.calls: + for call in result.calls: + if ( + call.name + ): # Only count calls with names (not just parameter updates) + tool_calls_seen.append(call.name) + + # Should see both tool names + self.assertIn("get_weather", tool_calls_seen, "Should process first tool") + self.assertIn( + "get_tourist_attractions", tool_calls_seen, "Should process second tool" + ) + + def test_current_tool_id_increment_on_completion(self): + """Test that current_tool_id increments when a tool completes.""" + # Initial state + self.assertEqual( + self.detector.current_tool_id, -1, "Should start with current_tool_id=-1" + ) + + # Process first tool completely + chunks = [ + "", + '{"name": "get_weather", ', + ] + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + self.assertEqual( + self.detector.current_tool_id, 0, "current_tool_id should be 0" + ) + self.assertEqual( + result.calls[0].name, "get_weather", "The first tool should be get_weather" + ) + self.assertEqual( + result.calls[0].tool_index, 0, "The first tool index should be 0" + ) + + # Complete second tool name - this should show that current_tool_id is now 1 + result = self.detector.parse_streaming_increment( + '"arguments": {"city": "Paris"}}, {"name": "get_', self.tools + ) + self.assertEqual(result.calls[0].parameters, '{"city": "Paris"}') + + self.assertEqual( + self.detector.current_tool_id, + 1, + "current_tool_id should be 1 after first tool completes and second tool starts", + ) + + result = self.detector.parse_streaming_increment( + 'tourist_attractions", ', self.tools + ) + + # Second tool should have tool_index=1 + tourist_calls = [ + call for call in result.calls if call.name == "get_tourist_attractions" + ] + self.assertEqual( + tourist_calls[0].tool_index, 1, "Second tool should have tool_index=1" + ) + + def test_tool_name_streaming_with_correct_index(self): + """Test that tool names are streamed with correct tool_index values.""" + # Process first tool + self.detector.parse_streaming_increment("", self.tools) + result1 = self.detector.parse_streaming_increment( + '{"name": "get_weather", ', self.tools + ) + + # First tool name should have tool_index=0 + weather_calls = [call for call in result1.calls if call.name == "get_weather"] + self.assertEqual(len(weather_calls), 1, "Should have one weather call") + self.assertEqual( + weather_calls[0].tool_index, 0, "First tool should have tool_index=0" + ) + + # Complete first tool + self.detector.parse_streaming_increment( + '"arguments": {"city": "Paris"}}', self.tools + ) + + # Start second tool + self.detector.parse_streaming_increment(", ", self.tools) + result2 = self.detector.parse_streaming_increment( + '{"name": "get_tourist_attractions", ', self.tools + ) + + # Second tool name should have tool_index=1 + tourist_calls = [ + call for call in result2.calls if call.name == "get_tourist_attractions" + ] + self.assertEqual( + len(tourist_calls), 1, "Should have one tourist attractions call" + ) + self.assertEqual( + tourist_calls[0].tool_index, 1, "Second tool should have tool_index=1" + ) + + def test_buffer_reset_on_invalid_tool(self): + """Test that buffer and state are reset when an invalid tool name is encountered.""" + # Start fresh with an invalid tool name from the beginning + result = self.detector.parse_streaming_increment( + '{"name": "invalid_tool", ', self.tools + ) + + # Should return empty result and reset state + self.assertEqual(result.calls, [], "Should return no calls for invalid tool") + self.assertEqual( + self.detector.current_tool_id, + -1, + "current_tool_id should remain -1 for invalid tool", + ) + self.assertEqual( + self.detector._buffer, "", "Buffer should be cleared for invalid tool" + ) + + +class TestLlama32Detector(unittest.TestCase): + def setUp(self): + """Set up test tools and detector for Mistral format testing.""" + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_tourist_attractions", + description="Get tourist attractions", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + ] + self.detector = Llama32Detector() + + def test_single_json(self): + text = '{"name": "get_weather", "parameters": {"city": "Paris"}}' + result = self.detector.detect_and_parse(text, self.tools) + assert len(result.calls) == 1 + assert result.calls[0].name == "get_weather" + assert result.normal_text == "" + + def test_multiple_json_with_separator(self): + text = ( + '<|python_tag|>{"name": "get_weather", "parameters": {"city": "Paris"}};' + '{"name": "get_tourist_attractions", "parameters": {"city": "Paris"}}' + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 2) + self.assertEqual(result.calls[1].name, "get_tourist_attractions") + self.assertEqual(result.normal_text, "") + + def test_multiple_json_with_separator_customized(self): + text = ( + '<|python_tag|>{"name": "get_weather", "parameters": {}}' + '<|python_tag|>{"name": "get_tourist_attractions", "parameters": {}}' + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 2) + self.assertEqual(result.calls[1].name, "get_tourist_attractions") + self.assertEqual(result.normal_text, "") + + def test_json_with_trailing_text(self): + text = '{"name": "get_weather", "parameters": {}} Some follow-up text' + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertIn("follow-up", result.normal_text) + + def test_invalid_then_valid_json(self): + text = ( + '{"name": "get_weather", "parameters": {' # malformed + '{"name": "get_weather", "parameters": {}}' + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + + def test_plain_text_only(self): + text = "This is just plain explanation text." + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(result.calls, []) + self.assertEqual(result.normal_text, text) + + def test_with_python_tag_prefix(self): + text = 'Some intro. <|python_tag|>{"name": "get_weather", "parameters": {}}' + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertTrue(result.normal_text.strip().startswith("Some intro.")) + + +class TestKimiK2Detector(unittest.TestCase): + + def setUp(self): + """Set up test tools and detector.""" + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_tourist_attractions", + description="Get tourist attractions", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + ] + self.detector = KimiK2Detector() + + def test_single_tool_call(self): + """Test parsing a single tool call in a complete text.""" + text = '<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"city": "Paris"}<|tool_call_end|><|tool_calls_section_end|>' + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual(result.calls[0].parameters, '{"city": "Paris"}') + self.assertEqual(result.normal_text, "") + + def test_multiple_tool_calls(self): + """Test parsing multiple tool calls in a complete text.""" + text = '<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"city": "Paris"}<|tool_call_end|><|tool_call_begin|>functions.get_tourist_attractions:1<|tool_call_argument_begin|>{"city": "London"}<|tool_call_end|><|tool_calls_section_end|>' + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 2) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual(result.calls[0].parameters, '{"city": "Paris"}') + self.assertEqual(result.calls[1].name, "get_tourist_attractions") + self.assertEqual(result.calls[1].parameters, '{"city": "London"}') + self.assertEqual(result.normal_text, "") + + def test_streaming_tool_call(self): + """Test streaming incremental parsing of a tool call.""" + chunks = [ + "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{", + '"city": "Paris"', + "}", + "<|tool_call_end|><|tool_calls_section_end|>", + ] + + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if tool_call_chunk.tool_index is not None: + + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + + tc = tool_calls[tool_call_chunk.tool_index] + + if tool_call_chunk.name: + tc["name"] += tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual(tool_calls[0]["parameters"], '{"city": "Paris"}') + + def test_streaming_multiple_tool_calls(self): + """Test streaming incremental parsing of multiple tool calls.""" + chunks = [ + "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{", + '"city": "Paris"', + "}<|tool_call_end|>", + "<|tool_call_begin|>functions.get_tourist_attractions:1<|tool_call_argument_begin|>{", + '"city": "London"', + "}<|tool_call_end|>", + "<|tool_calls_section_end|>", + ] + + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if tool_call_chunk.tool_index is not None: + + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + + tc = tool_calls[tool_call_chunk.tool_index] + + if tool_call_chunk.name: + tc["name"] += tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual(tool_calls[0]["parameters"], '{"city": "Paris"}') + self.assertEqual(tool_calls[1]["name"], "get_tourist_attractions") + self.assertEqual(tool_calls[1]["parameters"], '{"city": "London"}') + + def test_tool_call_completion(self): + """Test that the buffer and state are reset after a tool call is completed.""" + chunks = [ + "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{", + '"city": "Paris"', + "}", + "<|tool_call_end|>", + "<|tool_calls_section_end|>", + ] + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + + # After processing all chunks, the buffer should be empty and current_tool_id should be reset + self.assertEqual(self.detector._buffer, "") + self.assertEqual(self.detector.current_tool_id, 1) + + def test_tool_name_streaming(self): + """Test that tool names are streamed correctly with the right index.""" + chunks = [ + "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{", + '"city": "Paris"', + "}", + "<|tool_call_end|>", + "<|tool_call_begin|>functions.get_tourist_attractions:1<|tool_call_argument_begin|>{", + ] + + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if tool_call_chunk.tool_index is not None: + + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + + tc = tool_calls[tool_call_chunk.tool_index] + + if tool_call_chunk.name: + tc["name"] += tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual(tool_calls[0]["parameters"], '{"city": "Paris"}') + self.assertEqual(tool_calls[1]["name"], "get_tourist_attractions") + + def test_invalid_tool_call(self): + """Test that invalid tool calls are handled correctly.""" + text = 'invalid_tool:0<|tool_call_argument_begin|>{"city": "Paris"}<|tool_call_end|><|tool_calls_section_end|>' + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 0) + self.assertEqual(result.normal_text, text) + + def test_partial_tool_call(self): + """Test that partial tool calls are handled correctly in streaming mode.""" + chunks = [ + "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{", + '"city": "Paris"', + ] + + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if tool_call_chunk.tool_index is not None: + + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + + tc = tool_calls[tool_call_chunk.tool_index] + + if tool_call_chunk.name: + tc["name"] += tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual(tool_calls[0]["parameters"], '{"city": "Paris"') + + +class TestDeepSeekV3Detector(unittest.TestCase): + def setUp(self): + """Set up test tools and detector for DeepSeekV3 format testing.""" + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="get_tourist_attractions", + description="Get tourist attractions", + parameters={ + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name", + } + }, + "required": ["city"], + }, + ), + ), + ] + self.detector = DeepSeekV3Detector() + + def test_parse_streaming_multiple_tool_calls_with_multi_token_chunk(self): + """Test parsing multiple tool calls when streaming chunks contains multi-tokens (e.g. DeepSeekV3 enable MTP)""" + # Simulate streaming chunks with multi-tokens for two consecutive tool calls + chunks = [ + "<｜tool▁calls▁begin｜>", + "<｜tool▁call▁begin｜>function", + "<｜tool▁sep｜>get", + "_weather\n", + "```json\n", + '{"city":', + '"Shanghai', + '"}\n```<｜tool▁call▁end｜>', + "\n<｜tool▁call▁begin｜>", + "function<｜tool▁sep｜>", + "get_tour", + "ist_att", + "ractions\n```" 'json\n{"', + 'city": "', + 'Beijing"}\n', + "```<｜tool▁call▁end｜>", + "<｜tool▁calls▁end｜>", + ] + + tool_calls_seen = [] + tool_calls_parameters = [] + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + if result.calls: + for call in result.calls: + if call.name: + tool_calls_seen.append(call.name) + if call.parameters: + tool_calls_parameters.append(call.parameters) + + # Should see both tool names + self.assertIn("get_weather", tool_calls_seen, "Should process first tool") + self.assertIn( + "get_tourist_attractions", tool_calls_seen, "Should process second tool" + ) + + # Verify that the parameters are valid JSON and contain the expected content + params1 = json.loads(tool_calls_parameters[0]) + params2 = json.loads(tool_calls_parameters[1]) + self.assertEqual(params1["city"], "Shanghai") + self.assertEqual(params2["city"], "Beijing") + + +class TestQwen3CoderDetector(unittest.TestCase): + def setUp(self): + # Create sample tools for testing + self.tools = [ + Tool( + type="function", + function=Function( + name="get_current_weather", + description="Get the current weather", + parameters={ + "properties": { + "city": {"type": "string", "description": "The city name"}, + "state": { + "type": "string", + "description": "The state code", + }, + "unit": { + "type": "string", + "enum": ["fahrenheit", "celsius"], + }, + }, + "required": ["city", "state"], + }, + ), + ), + Tool( + type="function", + function=Function( + name="calculate_area", + description="Calculate area of a shape", + parameters={ + "properties": { + "shape": {"type": "string"}, + "dimensions": {"type": "object"}, + "precision": {"type": "integer"}, + } + }, + ), + ), + ] + self.detector = Qwen3CoderDetector() + + def test_has_tool_call(self): + """Test detection of tool call markers.""" + self.assertTrue(self.detector.has_tool_call("test")) + self.assertFalse(self.detector.has_tool_call("No tool call here")) + + def test_detect_and_parse_no_tools(self): + """Test parsing text without tool calls.""" + model_output = "This is a test response without any tool calls" + result = self.detector.detect_and_parse(model_output, tools=[]) + self.assertEqual(result.normal_text, model_output) + self.assertEqual(result.calls, []) + + def test_detect_and_parse_single_tool(self): + """Test parsing a single tool call.""" + model_output = """ + + +Dallas + + +TX + + +fahrenheit + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["city"], "Dallas") + self.assertEqual(params["state"], "TX") + self.assertEqual(params["unit"], "fahrenheit") + + def test_detect_and_parse_with_content(self): + """Test parsing tool call with surrounding text.""" + model_output = """Sure! Let me check the weather for you. + + +Dallas + + +TX + + +fahrenheit + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "Sure! Let me check the weather for you.") + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + + def test_detect_and_parse_multiline_param(self): + """Test parsing tool call with multiline parameter values.""" + model_output = """ + + +rectangle + + +{"width": 10, + "height": 20} + + +2 + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "calculate_area") + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["shape"], "rectangle") + self.assertEqual(params["dimensions"], {"width": 10, "height": 20}) + self.assertEqual(params["precision"], 2) + + def test_detect_and_parse_parallel_tools(self): + """Test parsing multiple tool calls.""" + model_output = """ + + +Dallas + + +TX + + +fahrenheit + + + + + + +Orlando + + +FL + + +fahrenheit + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertEqual(result.normal_text, "\n") + self.assertEqual(len(result.calls), 2) + + # First call + self.assertEqual(result.calls[0].name, "get_current_weather") + params1 = json.loads(result.calls[0].parameters) + self.assertEqual(params1["city"], "Dallas") + self.assertEqual(params1["state"], "TX") + + # Second call + self.assertEqual(result.calls[1].name, "get_current_weather") + params2 = json.loads(result.calls[1].parameters) + self.assertEqual(params2["city"], "Orlando") + self.assertEqual(params2["state"], "FL") + + def test_parse_streaming_simple(self): + """Test basic streaming parsing.""" + chunks = [ + "Sure! ", + "Let me check ", + "the weather.", + "", + "\n", + "\n", + "\nDallas", + "\n", + "\n", + "\nTX", + "\n", + "\n", + "\n", + ] + + accumulated_text = "" + accumulated_calls = [] + tool_calls_by_index = {} + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, tools=self.tools) + accumulated_text += result.normal_text + + # Track calls by tool_index to handle streaming properly + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + self.assertEqual(accumulated_text, "Sure! Let me check the weather.") + self.assertEqual(len(tool_calls_by_index), 1) + + # Get the complete tool call + tool_call = tool_calls_by_index[0] + self.assertEqual(tool_call["name"], "get_current_weather") + + # Parse the accumulated parameters + params = json.loads(tool_call["parameters"]) + self.assertEqual(params["city"], "Dallas") + self.assertEqual(params["state"], "TX") + + def test_parse_streaming_incomplete(self): + """Test streaming with incomplete tool call.""" + # Send incomplete tool call + chunks = [ + "", + "\n", + "\n", + "\nDallas", + "\n", + "\n", + "\nTX", + # Missing , , + ] + + tool_calls_by_index = {} + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, tools=self.tools) + + # Track calls by tool_index to handle streaming properly + for call in result.calls: + if call.tool_index is not None: + if call.tool_index not in tool_calls_by_index: + tool_calls_by_index[call.tool_index] = { + "name": "", + "parameters": "", + } + + if call.name: + tool_calls_by_index[call.tool_index]["name"] = call.name + if call.parameters: + tool_calls_by_index[call.tool_index][ + "parameters" + ] += call.parameters + + # Should have partial tool call with name but incomplete parameters + self.assertGreater(len(tool_calls_by_index), 0) + self.assertEqual(tool_calls_by_index[0]["name"], "get_current_weather") + + # Parameters should be incomplete (no closing brace) + params_str = tool_calls_by_index[0]["parameters"] + self.assertTrue(params_str.startswith('{"city": "Dallas"')) + self.assertFalse(params_str.endswith("}")) + + # Now complete it + result = self.detector.parse_streaming_increment( + "\n\n\n", tools=self.tools + ) + + # Update the accumulated parameters + for call in result.calls: + if call.tool_index is not None and call.parameters: + tool_calls_by_index[call.tool_index]["parameters"] += call.parameters + + # Now should have complete parameters + final_params = json.loads(tool_calls_by_index[0]["parameters"]) + self.assertEqual(final_params["city"], "Dallas") + self.assertEqual(final_params["state"], "TX") + + def test_edge_case_no_parameters(self): + """Test tool call without parameters.""" + model_output = """ + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_current_weather") + self.assertEqual(json.loads(result.calls[0].parameters), {}) + + def test_edge_case_special_chars_in_value(self): + """Test parameter with special characters in value.""" + model_output = """ + + +Dallas->TX + + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + self.assertEqual(len(result.calls), 1) + + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["city"], "Dallas->TX") + + def test_extract_tool_calls_fallback_no_tags(self): + """Test fallback parsing when XML tags are missing (just function without tool_call wrapper).""" + model_output = """ + +Dallas + + +TX + +""" + + result = self.detector.detect_and_parse(model_output, tools=self.tools) + + self.assertIsNotNone(result) + + def test_extract_tool_calls_type_conversion(self): + """Test parameter type conversion based on tool schema.""" + test_tool = Tool( + type="function", + function=Function( + name="test_types", + parameters={ + "type": "object", + "properties": { + "int_param": {"type": "integer"}, + "float_param": {"type": "float"}, + "bool_param": {"type": "boolean"}, + "str_param": {"type": "string"}, + "obj_param": {"type": "object"}, + }, + }, + ), + ) + + model_output = """ + + +42 + + +3.14 + + +true + + +hello world + + +{"key": "value"} + + +""" + + result = self.detector.detect_and_parse(model_output, tools=[test_tool]) + + self.assertEqual(len(result.calls), 1) + params = json.loads(result.calls[0].parameters) + self.assertEqual(params["int_param"], 42) + self.assertEqual(params["float_param"], 3.14) + self.assertEqual(params["bool_param"], True) + self.assertEqual(params["str_param"], "hello world") + self.assertEqual(params["obj_param"], {"key": "value"}) + + def test_parse_streaming_incremental(self): + """Test that streaming is truly incremental with very small chunks.""" + model_output = """I'll check the weather. + + + Dallas + + + TX + + + """ + + # Simulate more realistic token-based chunks where is a single token + chunks = [ + "I'll check the weather.", + "", + "\n\n", + "\n", + "Dallas\n", + "\n", + "\n", + "TX\n", + "\n", + "\n", + "", + ] + + accumulated_text = "" + tool_calls = [] + chunks_count = 0 + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + chunks_count += 1 + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertGreater(chunks_count, 3) + + # Verify the accumulated results + self.assertIn("I'll check the weather.", accumulated_text) + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_current_weather") + + params = json.loads(tool_calls[0]["parameters"]) + self.assertEqual(params, {"city": "Dallas", "state": "TX"}) + + def test_parse_streaming_multiple_tools(self): + """Test streaming with multiple tool calls.""" + model_output = """ + + + Dallas + + + TX + + + + Some text in between. + + + + circle + + + {"radius": 5} + + + """ + + # Simulate streaming by chunks + chunk_size = 20 + chunks = [ + model_output[i : i + chunk_size] + for i in range(0, len(model_output), chunk_size) + ] + + accumulated_text = "" + tool_calls = [] + chunks_count = 0 + + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + accumulated_text += result.normal_text + chunks_count += 1 + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": ""}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] += tool_call_chunk.parameters + + self.assertIn("Some text in between.", accumulated_text) + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_current_weather") + self.assertEqual(tool_calls[1]["name"], "calculate_area") + + # Verify parameters + params1 = json.loads(tool_calls[0]["parameters"]) + self.assertEqual(params1, {"city": "Dallas", "state": "TX"}) + + params2 = json.loads(tool_calls[1]["parameters"]) + self.assertEqual(params2, {"shape": "circle", "dimensions": {"radius": 5}}) + + +class TestGlm4MoeDetector(unittest.TestCase): + def setUp(self): + self.tools = [ + Tool( + type="function", + function=Function( + name="get_weather", + description="Get weather information", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "date": {"type": "string", "description": "Date"}, + }, + "required": ["city", "date"], + }, + ), + ), + ] + self.detector = Glm4MoeDetector() + + def test_single_tool_call(self): + text = ( + "get_weather\n" + "city\nBeijing\n" + "date\n2024-06-27\n" + "" + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 1) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + result.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(result.normal_text, "") + + def test_multiple_tool_calls(self): + text = ( + "get_weather\n" + "city\nBeijing\n" + "date\n2024-06-27\n" + "" + "get_weather\n" + "city\nShanghai\n" + "date\n2024-06-28\n" + "" + ) + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 2) + self.assertEqual(result.calls[0].name, "get_weather") + self.assertEqual( + result.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(result.calls[1].name, "get_weather") + self.assertEqual( + result.calls[1].parameters, '{"city": "Shanghai", "date": "2024-06-28"}' + ) + self.assertEqual(result.normal_text, "") + + def test_streaming_tool_call(self): + """Test streaming incremental parsing of a tool call.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "", + ] + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": {}}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] = tool_call_chunk.parameters + self.assertEqual(len(tool_calls), 1) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual( + tool_calls[0]["parameters"], '{"city": "Beijing", "date": "2024-06-27"}' + ) + + def test_streaming_multiple_tool_calls(self): + """Test streaming incremental parsing of multiple tool calls.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "get_weather\n", + "city\nShanghai\n", + "date\n2024-06-28\n", + "", + ] + tool_calls = [] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + for tool_call_chunk in result.calls: + if ( + hasattr(tool_call_chunk, "tool_index") + and tool_call_chunk.tool_index is not None + ): + while len(tool_calls) <= tool_call_chunk.tool_index: + tool_calls.append({"name": "", "parameters": {}}) + tc = tool_calls[tool_call_chunk.tool_index] + if tool_call_chunk.name: + tc["name"] = tool_call_chunk.name + if tool_call_chunk.parameters: + tc["parameters"] = tool_call_chunk.parameters + self.assertEqual(len(tool_calls), 2) + self.assertEqual(tool_calls[0]["name"], "get_weather") + self.assertEqual( + tool_calls[0]["parameters"], '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(tool_calls[1]["name"], "get_weather") + self.assertEqual( + tool_calls[1]["parameters"], '{"city": "Shanghai", "date": "2024-06-28"}' + ) + + def test_tool_call_id(self): + """Test that the buffer and state are reset after a tool call is completed.""" + chunks = [ + "get_weather\n", + "city\nBeijing\n", + "date\n2024-06-27\n", + "", + ] + for chunk in chunks: + result = self.detector.parse_streaming_increment(chunk, self.tools) + self.assertEqual(self.detector.current_tool_id, 1) + + def test_invalid_tool_call(self): + """Test that invalid tool calls are handled correctly.""" + text = "invalid_func\ncity\nBeijing\n" + result = self.detector.detect_and_parse(text, self.tools) + self.assertEqual(len(result.calls), 0) + + def test_partial_tool_call(self): + """Test parsing a partial tool call that spans multiple chunks.""" + text1 = "get_weather\ncity\n" + result1 = self.detector.parse_streaming_increment(text1, self.tools) + self.assertEqual(result1.normal_text, "") + self.assertEqual(result1.calls, []) + self.assertEqual(self.detector._buffer, text1) + text2 = "Beijing\ndate\n2024-06-27\n" + result2 = self.detector.parse_streaming_increment(text2, self.tools) + self.assertEqual(len(result2.calls), 1) + self.assertEqual(result2.calls[0].name, "get_weather") + self.assertEqual( + result2.calls[0].parameters, '{"city": "Beijing", "date": "2024-06-27"}' + ) + self.assertEqual(self.detector._buffer, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_fused_moe.py b/test/srt/test_fused_moe.py new file mode 100644 index 000000000..9f2cc31b1 --- /dev/null +++ b/test/srt/test_fused_moe.py @@ -0,0 +1,236 @@ +import unittest + +import torch +import torch.nn.functional as F +from tqdm import tqdm + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import is_fp8_fnuz +from sglang.srt.layers.quantization.fp8_utils import normalize_e4m3fn_to_e4m3fnuz +from sglang.srt.utils import is_hip +from sglang.test.test_utils import CustomTestCase + +_is_hip = is_hip() +_is_fp8_fnuz = is_fp8_fnuz() + + +class TestFusedMOE(CustomTestCase): + NUM_EXPERTS = [8, 64] + TOP_KS = [2, 6] + + @staticmethod + def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): + """Create a random CUDA tensor + + Args: + shape: Tensor shape + dtype: Data type + mean: Mean value + std: Standard deviation + + Returns: + torch.Tensor: Randomly initialized CUDA tensor + """ + return torch.empty(shape, dtype=dtype, device="cuda").normal_(mean, std) + + def get_tolerance(self, dtype): + """Get tolerance values for different data types + + Args: + dtype: Data type + + Returns: + tuple: (relative tolerance, absolute tolerance) + """ + if dtype == torch.float32: + return 1e-3, 1e-5 + elif dtype in [torch.float16, torch.bfloat16]: + return 1e-1, 1e-2 + else: + return 1e-2, 1e-2 # Default values for other types + + def torch_naive_moe( + self, + a, + w1, + w2, + score, + topk, + w1_scale=None, + w2_scale=None, + a1_scale=None, + a2_scale=None, + ): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + if w1.dtype in [torch.float8_e4m3fn, torch.float8_e4m3fnuz]: + w1_compute = w1.to(a.dtype) + w2_compute = w2.to(a.dtype) + + if w1_scale is not None: + w1_compute = (w1_compute * w1_scale.view(-1, 1, 1)).to(a.dtype) + if w2_scale is not None: + w2_compute = (w2_compute * w2_scale.view(-1, 1, 1)).to(a.dtype) + if a1_scale is not None: + a = (a * a1_scale).to(a.dtype) + if a2_scale is not None: + a = (a * a2_scale).to(a.dtype) + else: + w1_compute = w1 + w2_compute = w2 + + for i in range(w1_compute.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()( + a[mask] @ w1_compute[i].transpose(0, 1) + ) @ w2_compute[i].transpose(0, 1) + + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + def _test_case(self, m, n, k, e, topk, dtype, use_fp8_w8a8=False): + rtol, atol = self.get_tolerance(dtype) + + if use_fp8_w8a8: + # AssertionError: fp8e4nv data type is not supported on CUDA arch < 89 + capability = torch.cuda.get_device_capability() + if not _is_hip and not (capability[0] >= 9 or capability == (8, 9)): + return + + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + w1 = w1.to(torch.float8_e4m3fn) + w2 = w2.to(torch.float8_e4m3fn) + score = self.create_random_cuda_tensor((m, e), dtype) + w1_scale = self.create_random_cuda_tensor(e, torch.float32) + w2_scale = self.create_random_cuda_tensor(e, torch.float32) + a1_scale = self.create_random_cuda_tensor(1, torch.float32) + a2_scale = self.create_random_cuda_tensor(1, torch.float32) + + # Handle HIP case: normalize float8 weights so fused kernel doesn't break + # on ROCm. + if _is_fp8_fnuz: + # Normalize to e4m3fnuz on HIP + w1, w1_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=w1, + weight_scale=w1_scale, + input_scale=a1_scale, + ) + w2, w2_scale, _ = normalize_e4m3fn_to_e4m3fnuz( + weight=w2, + weight_scale=w2_scale, + input_scale=a2_scale, + ) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + torch_output = self.torch_naive_moe( + a, + w1, + w2, + score, + topk, + w1_scale, + w2_scale, + a1_scale, + a2_scale, + ) + + sglang_output = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, + w1_scale=w1_scale, + w2_scale=w2_scale, + a1_scale=a1_scale, + a2_scale=a2_scale, + ) + torch.testing.assert_close( + sglang_output, torch_output, rtol=rtol, atol=atol + ) + else: + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + score = self.create_random_cuda_tensor((m, e), dtype) + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + + triton_output = fused_moe(a, w1, w2, topk_output) + torch_output = self.torch_naive_moe(a, w1, w2, score, topk) + torch.testing.assert_close( + triton_output, torch_output, rtol=rtol, atol=atol + ) + + def test_various_configurations(self): + m_values = [1, 33, 64, 222] + n_values = [128, 1024] + k_values = [128, 511, 1024] + dtypes = [torch.float16, torch.bfloat16] + fp8_modes = [False, True] + + # Calculate total number of tests + total_tests = ( + len(m_values) + * len(n_values) + * len(k_values) + * len(self.NUM_EXPERTS) + * len(self.TOP_KS) + * len(dtypes) + * len(fp8_modes) + ) + + # Create progress bar + with tqdm(total=total_tests, desc="Running MoE tests") as pbar: + for m in m_values: + for n in n_values: + for k in k_values: + for e in self.NUM_EXPERTS: + for topk in self.TOP_KS: + for dtype in dtypes: + for use_fp8_w8a8 in fp8_modes: + with self.subTest( + m=m, + n=n, + k=k, + e=e, + topk=topk, + dtype=dtype, + fp8=use_fp8_w8a8, + ): + self._test_case( + m, + n, + k, + e, + topk, + dtype, + use_fp8_w8a8=use_fp8_w8a8, + ) + torch.cuda.empty_cache() + pbar.update(1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_get_weights_by_name.py b/test/srt/test_get_weights_by_name.py new file mode 100644 index 000000000..3d404df10 --- /dev/null +++ b/test/srt/test_get_weights_by_name.py @@ -0,0 +1,183 @@ +import gc +import unittest + +import numpy as np +import requests +import torch +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) +from sglang.utils import terminate_process + + +def _process_return(ret): + if isinstance(ret, list) and len(ret) == 2: + print(f"running assert_allclose on data parallel") + np.testing.assert_allclose(ret[0], ret[1]) + return np.array(ret[0]) + return np.array(ret) + + +class TestGetWeightsByName(CustomTestCase): + + def init_hf_model(self, model_name, tie_word_embeddings): + self.hf_model = AutoModelForCausalLM.from_pretrained( + model_name, torch_dtype="bfloat16", tie_word_embeddings=tie_word_embeddings + ).to("cuda:0") + + def init_backend(self, backend, dp, tp, model_name): + self.backend = backend + self.dp = dp + self.tp = tp + if backend == "Engine": + self.engine = sgl.Engine( + model_path=model_name, + random_seed=42, + tp_size=tp, + dp_size=dp, + ) + else: + self.process = popen_launch_server( + model_name, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--tp-size", + str(tp), + "--dp-size", + str(dp), + ), + ) + + def clean_up(self): + del self.hf_model + gc.collect() + torch.cuda.empty_cache() + if self.backend == "Engine": + self.engine.shutdown() + else: + terminate_process(self.process) + + def assert_tie_word_embeddings(self, truncate_size): + print("assert_tie_word_embeddings") + if self.backend == "Engine": + backend_ret = _process_return( + self.engine.get_weights_by_name("lm_head.weight", truncate_size) + ) + else: + backend_ret = _process_return( + requests.get( + f"{DEFAULT_URL_FOR_TEST}/get_weights_by_name", + json={"name": "lm_head.weight", "truncate_size": truncate_size}, + ).json() + ) + print("assert_tie_word_embeddings of hf and backend") + assert np.allclose( + self.hf_model.get_parameter("model.embed_tokens.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + backend_ret, + ) + assert np.allclose( + self.hf_model.get_parameter("lm_head.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + self.hf_model.get_parameter("model.embed_tokens.weight") + .cpu() + .detach() + .float() + .numpy()[:truncate_size], + ) + + def assert_weights_all_close(self, param_name, truncate_size): + print( + f"param_name: {param_name}, backend: {self.backend}, dp: {self.dp}, tp: {self.tp}" + ) + param = self.hf_model.get_parameter(param_name)[:truncate_size] + param_np = param.cpu().detach().float().numpy() + + if self.backend == "Engine": + engine_ret = self.engine.get_weights_by_name(param_name, truncate_size) + engine_ret = _process_return(engine_ret) + np.testing.assert_allclose(engine_ret, param_np, rtol=1e-5, atol=1e-5) + + if self.backend == "Runtime": + runtime_ret = requests.get( + f"{DEFAULT_URL_FOR_TEST}/get_weights_by_name", + json={"name": param_name, "truncate_size": truncate_size}, + ).json() + runtime_ret = _process_return(runtime_ret) + np.testing.assert_allclose(runtime_ret, param_np, rtol=1e-5, atol=1e-5) + + def test_get_weights_by_name(self): + if is_in_ci(): + test_suits = [ + ("Engine", 1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ] + else: + test_suits = [ + ("Runtime", 1, 1, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ("Engine", 1, 1, DEFAULT_MODEL_NAME_FOR_TEST), + ] + if torch.cuda.device_count() >= 2: + test_suits.append(("Engine", 1, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST)) + test_suits.append(("Runtime", 2, 1, DEFAULT_MODEL_NAME_FOR_TEST)) + + if torch.cuda.device_count() >= 4: + test_suits.extend( + [ + ("Engine", 2, 2, DEFAULT_SMALL_MODEL_NAME_FOR_TEST), + ("Runtime", 2, 2, DEFAULT_MODEL_NAME_FOR_TEST), + ] + ) + + parameters = [ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "model.layers.1.self_attn.q_proj.weight", + "model.layers.2.self_attn.k_proj.weight", + "model.layers.3.self_attn.v_proj.weight", + "model.layers.4.self_attn.o_proj.weight", + "model.layers.5.mlp.gate_proj.weight", + "model.layers.6.mlp.up_proj.weight", + "model.layers.7.mlp.down_proj.weight", + "model.layers.8.post_attention_layernorm.weight", + "model.norm.weight", + "lm_head.weight", + ] + + truncate_size = 100 + + for test_suit in test_suits: + if test_suit[-1] == DEFAULT_MODEL_NAME_FOR_TEST: + tie_word_embeddings = False + else: + tie_word_embeddings = True + + self.init_hf_model(test_suit[-1], tie_word_embeddings) + self.init_backend(*test_suit) + + for param_name in parameters: + self.assert_weights_all_close(param_name, truncate_size) + + if tie_word_embeddings: + self.assert_tie_word_embeddings(truncate_size) + + self.clean_up() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gguf.py b/test/srt/test_gguf.py new file mode 100644 index 000000000..e9776067c --- /dev/null +++ b/test/srt/test_gguf.py @@ -0,0 +1,27 @@ +import unittest + +from huggingface_hub import hf_hub_download + +import sglang as sgl +from sglang.test.test_utils import CustomTestCase + + +class TestGGUF(CustomTestCase): + def test_models(self): + prompt = "Today is a sunny day and I like" + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + model_path = hf_hub_download( + "Qwen/Qwen2-1.5B-Instruct-GGUF", + filename="qwen2-1_5b-instruct-q4_k_m.gguf", + ) + + engine = sgl.Engine(model_path=model_path, random_seed=42, cuda_graph_max_bs=2) + outputs = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + self.assertEqual(outputs, " it. I have a lot of work") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gpt_oss_1gpu.py b/test/srt/test_gpt_oss_1gpu.py new file mode 100644 index 000000000..d3fc06931 --- /dev/null +++ b/test/srt/test_gpt_oss_1gpu.py @@ -0,0 +1,31 @@ +import unittest + +from test_gpt_oss_common import BaseTestGptOss + + +class TestGptOss1Gpu(BaseTestGptOss): + def test_mxfp4_20b(self): + self.run_test( + model_variant="20b", + quantization="mxfp4", + expected_score_of_reasoning_effort={ + "low": 0.34, + "medium": 0.34, + "high": 0.27, # TODO investigate + }, + ) + + def test_bf16_20b(self): + self.run_test( + model_variant="20b", + quantization="bf16", + expected_score_of_reasoning_effort={ + "low": 0.34, + "medium": 0.34, + "high": 0.27, # TODO investigate + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gpt_oss_4gpu.py b/test/srt/test_gpt_oss_4gpu.py new file mode 100644 index 000000000..da787c6fb --- /dev/null +++ b/test/srt/test_gpt_oss_4gpu.py @@ -0,0 +1,36 @@ +import unittest + +from test_gpt_oss_common import BaseTestGptOss + + +class TestGptOss4Gpu(BaseTestGptOss): + def test_bf16_120b(self): + self.run_test( + model_variant="120b", + quantization="bf16", + expected_score_of_reasoning_effort={ + "low": 0.60, + }, + other_args=["--tp", "4", "--cuda-graph-max-bs", "200"], + ) + + def test_mxfp4_120b(self): + self.run_test( + model_variant="120b", + quantization="mxfp4", + expected_score_of_reasoning_effort={ + "low": 0.60, + }, + other_args=[ + "--tp", + "4", + "--cuda-graph-max-bs", + "200", + "--mem-fraction-static", + "0.93", + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_gpt_oss_common.py b/test/srt/test_gpt_oss_common.py new file mode 100644 index 000000000..6be739277 --- /dev/null +++ b/test/srt/test_gpt_oss_common.py @@ -0,0 +1,111 @@ +import os +from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace +from typing import Dict, List, Literal, Optional + +from sglang.srt.utils import is_hip, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +_base_url = DEFAULT_URL_FOR_TEST +_is_hip = is_hip() + + +class BaseTestGptOss(CustomTestCase): + def run_test( + self, + model_variant: Literal["20b", "120b"], + quantization: Literal["mxfp4", "bf16"], + expected_score_of_reasoning_effort: Dict[str, float], + other_args: Optional[List[str]] = None, + ): + if other_args is None: + other_args = [] + + model = { + ("20b", "bf16"): "lmsys/gpt-oss-20b-bf16", + ("120b", "bf16"): "lmsys/gpt-oss-120b-bf16", + ("20b", "mxfp4"): "openai/gpt-oss-20b", + ("120b", "mxfp4"): "openai/gpt-oss-120b", + }[(model_variant, quantization)] + + if model_variant == "20b": + other_args += ["--cuda-graph-max-bs", "600"] + if _is_hip: + os.environ["SGLANG_USE_AITER"] = "0" + self._run_test_raw( + model=model, + expected_score_of_reasoning_effort=expected_score_of_reasoning_effort, + other_args=other_args, + ) + + def _run_test_raw( + self, + model: str, + expected_score_of_reasoning_effort: Dict[str, float], + other_args: List[str], + ): + process = popen_launch_server( + model, + _base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + # run multiple tests in parallel since we are mostly bound by the longest generate sequence + # instead of the number of questions + with ThreadPoolExecutor(max_workers=4) as executor: + list( + executor.map( + lambda d: self._run_one_eval(**d), + [ + dict( + model=model, + reasoning_effort=reasoning_effort, + expected_score=expected_score, + ) + for reasoning_effort, expected_score in expected_score_of_reasoning_effort.items() + ], + ) + ) + finally: + kill_process_tree(process.pid) + + def _run_one_eval(self, model, reasoning_effort, expected_score): + args = SimpleNamespace( + base_url=_base_url, + model=model, + eval_name="gpqa", + num_examples=198, + # use enough threads to allow parallelism + num_threads=198, + # TODO 4k is still not enough, we need e.g. 64k token, but that is super slow + # otherwise a lot of questions are not answered + max_tokens=4096, + # simple-evals by default use 0.5 and is better than 0.0 temperature + # but here for reproducibility, we use 0.1 + temperature=0.1, + reasoning_effort=reasoning_effort, + ) + + setup = f"model={model} reasoning_effort={reasoning_effort} expected_score={expected_score}" + + print(f"Evaluation start: {setup}") + metrics = run_eval(args) + print(f"Evaluation end: {setup} {metrics=}") + self.assertGreaterEqual(metrics["score"], expected_score) + + if is_in_ci(): + write_github_step_summary( + f"### test_gpt_oss_common\n" + f"Setup: {setup}\n" + f"Score: {metrics['score']:.2f}\n" + ) diff --git a/test/srt/test_gptqmodel_dynamic.py b/test/srt/test_gptqmodel_dynamic.py new file mode 100644 index 000000000..9be711d12 --- /dev/null +++ b/test/srt/test_gptqmodel_dynamic.py @@ -0,0 +1,204 @@ +import time +import unittest + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def check_quant_method(model_path: str, use_marlin_kernel: bool): + from sglang.srt.configs.device_config import DeviceConfig + from sglang.srt.configs.load_config import LoadConfig + from sglang.srt.configs.model_config import AttentionArch, ModelConfig + from sglang.srt.distributed import ( + get_tp_group, + init_distributed_environment, + initialize_model_parallel, + set_custom_all_reduce, + ) + from sglang.srt.distributed.parallel_state import monkey_patch_vllm_parallel_state + from sglang.srt.layers.quantization.utils import get_dynamic_override + from sglang.srt.model_loader import get_model + from sglang.srt.server_args import PortArgs, ServerArgs + + try: + init_distributed_environment( + backend="nccl", + world_size=1, + rank=0, + local_rank=0, + distributed_init_method="tcp://127.0.0.1:2646", + ) + initialize_model_parallel(tensor_model_parallel_size=1) + monkey_patch_vllm_parallel_state() + except AssertionError: + # ignore this error: tensor model parallel group is already initialized + pass + + server_args = ServerArgs(model_path=model_path, dtype=torch.float16) + model_config = ModelConfig.from_server_args(server_args) + + load_config = LoadConfig() + device_config = DeviceConfig("cuda") + model = get_model( + model_config=model_config, load_config=load_config, device_config=device_config + ) + + from sglang.srt.layers.linear import UnquantizedLinearMethod + from sglang.srt.layers.quantization.gptq import ( + GPTQLinearMethod, + GPTQMarlinLinearMethod, + ) + + linear_method_cls = ( + GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod) + ) + + for name, submodule in model.named_modules(): + if name == "lm_head": + assert isinstance(submodule.quant_method, linear_method_cls) + elif name == "model.layers.0.self_attn.qkv_proj": + # The first layer is quantized using bits=4, group_size=128 + # desc_act=True + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert config.weight_bits == 4 + assert config.group_size == 128 + assert config.desc_act + elif name == "model.layers.1.self_attn.qkv_proj": + # The second layer is quantized using bits=8, group_size=32 + # desc_act=False + assert isinstance(submodule.quant_method, linear_method_cls) + config = submodule.quant_method.quant_config + assert get_dynamic_override(config, layer_name=name, key="bits") == 8 + assert get_dynamic_override(config, layer_name=name, key="group_size") == 32 + assert not get_dynamic_override(config, layer_name=name, key="desc_act") + elif ( + name == "model.layers.2.self_attn.qkv_proj" + or name == "model.layers.2.mlp.gate_up_proj" + ): + # All other layers (layer index >= 2) are not quantized + assert isinstance(submodule.quant_method, UnquantizedLinearMethod) + + del model + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test GPTQ fallback kernel that is not Marlin +class TestGPTQModelDynamic(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "float16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + result = self.run_decode(max_tokens) + tok = time.perf_counter() + + print(f"result = `{result}`") + + self.assertIn("paris", result["text"].lower()) + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + self.assertGreaterEqual(throughput, 140) + + def test_gptq_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=False) + + +# GPTQ with Dynamic Per/Module Quantization Control +# Leverages GPTQModel (pypi) to produce the `dynamic` models +# Test Marlin kernel +class TestGPTQModelDynamicWithMarlin(CustomTestCase): + MODEL_PATH = ( + "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue" + ) + + @classmethod + def setUpClass(cls): + cls.model = cls.MODEL_PATH + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "bfloat16"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "max_new_tokens": max_new_tokens, + "temperature": 0.001, + }, + }, + ) + return response.json() + + def test_throughput(self): + max_tokens = 256 + + tic = time.perf_counter() + result = self.run_decode(max_tokens) + tok = time.perf_counter() + + print(f"result = `{result}`") + + assert "paris" in result["text"].lower() + + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 140 + + def test_gptq_marlin_module(self): + check_quant_method(self.MODEL_PATH, use_marlin_kernel=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_harmony_parser.py b/test/srt/test_harmony_parser.py new file mode 100644 index 000000000..20cc02e5c --- /dev/null +++ b/test/srt/test_harmony_parser.py @@ -0,0 +1,876 @@ +import unittest + +from sglang.srt.parser.harmony_parser import ( + CanonicalStrategy, + Event, + HarmonyParser, + TextStrategy, + Token, + iter_tokens, + prefix_hold, +) +from sglang.test.test_utils import CustomTestCase + + +class TestEvent(CustomTestCase): + def test_init(self): + """Test Event dataclass initialization.""" + event = Event("reasoning", "content") + self.assertEqual(event.event_type, "reasoning") + self.assertEqual(event.content, "content") + + +class TestToken(CustomTestCase): + def test_init(self): + """Test Token dataclass initialization.""" + token = Token("START", 0, 7) + self.assertEqual(token.type, "START") + self.assertEqual(token.start, 0) + self.assertEqual(token.end, 7) + + +class TestPrefixHold(CustomTestCase): + def test_empty_text(self): + """Test prefix_hold with empty text.""" + emit, hold = prefix_hold("", ["<|start|>"]) + self.assertEqual(emit, "") + self.assertEqual(hold, "") + + def test_no_matching_prefixes(self): + """Test prefix_hold with no matching prefixes.""" + emit, hold = prefix_hold("hello world", ["<|start|>", "<|end|>"]) + self.assertEqual(emit, "hello world") + self.assertEqual(hold, "") + + def test_partial_token_suffix(self): + """Test prefix_hold with partial token at end.""" + emit, hold = prefix_hold("hello <|ret", ["<|return|>"]) + self.assertEqual(emit, "hello ") + self.assertEqual(hold, "<|ret") + + def test_multiple_potential_matches(self): + """Test prefix_hold with multiple potential matches.""" + emit, hold = prefix_hold("text <|", ["<|start|>", "<|end|>"]) + self.assertEqual(emit, "text ") + self.assertEqual(hold, "<|") + + def test_exact_token_match(self): + """Test prefix_hold with exact token match.""" + emit, hold = prefix_hold("text <|start|>", ["<|start|>"]) + self.assertEqual(emit, "text <|start|>") + self.assertEqual(hold, "") + + +class TestIterTokens(CustomTestCase): + def test_empty_text(self): + """Test iter_tokens with empty text.""" + tokens = list(iter_tokens("")) + self.assertEqual(tokens, []) + + def test_plain_text(self): + """Test iter_tokens with plain text.""" + tokens = list(iter_tokens("hello world")) + self.assertEqual(len(tokens), 1) + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 11) + + def test_single_token(self): + """Test iter_tokens with single structural token.""" + tokens = list(iter_tokens("<|start|>")) + self.assertEqual(len(tokens), 1) + self.assertEqual(tokens[0].type, "START") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 9) + + def test_mixed_content(self): + """Test iter_tokens with mixed text and tokens.""" + tokens = list(iter_tokens("text<|start|>more text")) + self.assertEqual(len(tokens), 3) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 4) + + self.assertEqual(tokens[1].type, "START") + self.assertEqual(tokens[1].start, 4) + self.assertEqual(tokens[1].end, 13) + + self.assertEqual(tokens[2].type, "TEXT") + self.assertEqual(tokens[2].start, 13) + self.assertEqual(tokens[2].end, 22) + + def test_unknown_token_partial_suffix(self): + """Test iter_tokens with unknown token that could be partial.""" + tokens = list(iter_tokens("text <|ret")) + self.assertEqual(len(tokens), 2) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[0].start, 0) + self.assertEqual(tokens[0].end, 5) + + self.assertEqual(tokens[1].type, "TEXT") + self.assertEqual(tokens[1].start, 5) + self.assertEqual(tokens[1].end, 10) + + def test_unknown_token_middle(self): + """Test iter_tokens with unknown token in middle.""" + tokens = list(iter_tokens("text <|weird|> more <|start|>")) + self.assertEqual(len(tokens), 5) + + self.assertEqual(tokens[0].type, "TEXT") + self.assertEqual(tokens[1].type, "TEXT") # "<|" + self.assertEqual(tokens[2].type, "TEXT") # "weird|> more " + self.assertEqual(tokens[3].type, "START") + # No trailing text token since it ends with a known token + + def test_all_structural_tokens(self): + """Test iter_tokens recognizes all structural tokens.""" + text = "<|start|><|channel|><|message|><|constrain|><|end|><|call|><|return|>" + tokens = list(iter_tokens(text)) + + expected_types = [ + "START", + "CHANNEL", + "MESSAGE", + "CONSTRAIN", + "END", + "CALL", + "RETURN", + ] + self.assertEqual(len(tokens), len(expected_types)) + + for token, expected_type in zip(tokens, expected_types): + self.assertEqual(token.type, expected_type) + + +class TestCanonicalStrategy(CustomTestCase): + def setUp(self): + self.strategy = CanonicalStrategy() + + def test_init(self): + """Test CanonicalStrategy initialization.""" + self.assertIn("<|start|>", self.strategy.guard_tokens) + self.assertIn("<|constrain|>", self.strategy.guard_tokens) + + def test_extract_channel_type(self): + """Test _extract_channel_type method.""" + self.assertEqual(self.strategy._extract_channel_type("analysis"), "analysis") + self.assertEqual( + self.strategy._extract_channel_type("commentary to=functions.tool"), + "commentary", + ) + self.assertEqual(self.strategy._extract_channel_type("final to=user"), "final") + self.assertEqual(self.strategy._extract_channel_type("ANALYSIS"), "analysis") + self.assertIsNone(self.strategy._extract_channel_type("unknown")) + + def test_parse_single_analysis_block(self): + """Test parsing single analysis block.""" + text = "<|channel|>analysis<|message|>Let me think about this<|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Let me think about this") + self.assertEqual(remaining, "") + + def test_parse_single_commentary_block(self): + """Test parsing single commentary block.""" + text = "<|channel|>commentary<|message|>User-visible message<|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "User-visible message") + self.assertEqual(remaining, "") + + def test_parse_single_final_block(self): + """Test parsing single final block.""" + text = "<|start|>assistant<|channel|>final<|message|>The answer is 42<|return|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "The answer is 42") + self.assertEqual(remaining, "") + + def test_parse_tool_call_commentary(self): + """Test parsing tool call on commentary channel.""" + text = '<|channel|>commentary to=functions.get_weather<|message|>{"location": "SF"}<|call|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"location": "SF"}') + self.assertEqual(remaining, "") + + def test_parse_tool_call_analysis(self): + """Test parsing built-in tool call on analysis channel.""" + text = '<|channel|>analysis to=browser.search<|message|>{"query": "SGLang"}<|call|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"query": "SGLang"}') + self.assertEqual(remaining, "") + + def test_parse_complex_sequence(self): + """Test parsing complex sequence with multiple blocks.""" + text = ( + "<|channel|>analysis<|message|>Need to use function get_weather.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_weather<|message|>" + '{"location":"San Francisco"}<|call|>' + ) + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Need to use function get_weather.") + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"San Francisco"}') + self.assertEqual(remaining, "") + + def test_parse_with_interspersed_text(self): + """Test parsing with plain text between blocks.""" + text = ( + "Some text " + "<|channel|>analysis<|message|>reasoning<|end|>" + " more text " + "<|start|>assistant<|channel|>final<|message|>answer<|return|>" + " trailing text" + ) + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 4) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "Some text ") + self.assertEqual(events[1].event_type, "reasoning") + self.assertEqual(events[1].content, "reasoning") + self.assertEqual(events[2].event_type, "normal") + self.assertEqual(events[2].content, " more text ") + self.assertEqual(events[3].event_type, "normal") + self.assertEqual(events[3].content, "answer trailing text") + self.assertEqual(remaining, "") + + def test_parse_incomplete_block(self): + """Test parsing incomplete block (streaming scenario).""" + text = "<|channel|>analysis<|message|>partial content" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "partial content") + self.assertEqual(remaining, "<|channel|>analysis<|message|>") + + def test_parse_partial_token_suffix(self): + """Test parsing with partial token at end.""" + text = "complete text <|ret" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "complete text ") + self.assertEqual(remaining, "<|ret") + + def test_parse_tool_response_message(self): + """Test parsing tool response message (no channel).""" + text = '<|start|>functions.get_weather to=assistant<|message|>{"sunny": true}<|end|>' + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, '{"sunny": true}') + self.assertEqual(remaining, "") + + def test_parse_empty_content_blocks(self): + """Test parsing blocks with empty content.""" + text = "<|channel|>analysis<|message|><|end|>" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "") + self.assertEqual(remaining, "") + + def test_parse_commentary_filler_between_blocks(self): + """Test that 'commentary' filler between <|call|> and <|channel|> is filtered out.""" + # This pattern occurs when the model generates malformed output + text = ( + '<|channel|>commentary to=functions.get_weather<|message|>{"location":"SF"}<|call|>' + "commentary" # This should be filtered out + '<|channel|>commentary to=functions.get_temp<|message|>{"location":"NYC"}<|call|>' + ) + events, remaining = self.strategy.parse(text) + + # Should have 2 tool calls, no "commentary" normal text + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"location":"SF"}') + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"NYC"}') + self.assertEqual(remaining, "") + + # Verify no "commentary" text was emitted as normal content + normal_events = [e for e in events if e.event_type == "normal"] + commentary_events = [ + e for e in normal_events if "commentary" in e.content.lower() + ] + self.assertEqual( + len(commentary_events), 0, "Commentary filler should be filtered out" + ) + + +class TestTextStrategy(CustomTestCase): + def setUp(self): + self.strategy = TextStrategy() + + def test_init(self): + """Test TextStrategy initialization.""" + self.assertIn("analysis_then_final", self.strategy.patterns) + + def test_parse_analysis_then_final(self): + """Test parsing analysis then final format.""" + text = "analysis I need to think about this. assistantfinal The answer is 42." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "I need to think about this.") + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "The answer is 42.") + self.assertEqual(remaining, "") + + def test_parse_commentary_then_final(self): + """Test parsing commentary then final format.""" + text = "commentary User-visible preamble. assistantfinal The answer is 42." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "User-visible preamble.") + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "The answer is 42.") + self.assertEqual(remaining, "") + + def test_parse_final_only(self): + """Test parsing final-only format.""" + text = "assistantfinal The direct answer." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "The direct answer.") + self.assertEqual(remaining, "") + + def test_parse_analysis_only(self): + """Test parsing analysis-only format.""" + text = "analysis This is reasoning content." + events, remaining = self.strategy.parse(text) + + # For analysis-only, streaming parse should keep header and emit with leading space + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, " This is reasoning content.") + self.assertEqual(remaining, "analysis") + + def test_parse_incomplete_assistantfinal(self): + """Test parsing with incomplete assistantfinal.""" + text = "analysis reasoning content assistantfin" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 0) + self.assertEqual(remaining, text) # Hold entire buffer + + def test_parse_partial_analysis_streaming(self): + """Test streaming partial analysis content.""" + text = "analysis partial content" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, " partial content") # Space preserved + self.assertEqual(remaining, "analysis") # Hold header + + def test_parse_case_insensitive(self): + """Test case insensitive parsing.""" + text = "ANALYSIS reasoning ASSISTANTFINAL answer" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "normal") + + def test_parse_plain_text_fallback(self): + """Test parsing plain text without harmony markers.""" + text = "Just plain text without any markers." + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, "Just plain text without any markers.") + self.assertEqual(remaining, "") + + def test_parse_analysis_no_space_after_header(self): + """Test parsing analysis format without space after header (real gpt-oss output).""" + text = "analysisThe user typed random strings. We should respond politely.assistantfinalIt looks like you're testing. How can I help?" + events, remaining = self.strategy.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual( + events[0].content, + "The user typed random strings. We should respond politely.", + ) + self.assertEqual(events[1].event_type, "normal") + self.assertEqual( + events[1].content, "It looks like you're testing. How can I help?" + ) + + +class TestHarmonyParser(CustomTestCase): + def setUp(self): + self.parser = HarmonyParser() + + def test_init(self): + """Test HarmonyParser initialization.""" + self.assertIsNone(self.parser.strategy) + self.assertEqual(self.parser._buffer, "") + + def test_strategy_selection_canonical(self): + """Test automatic strategy selection for canonical format.""" + events = self.parser.parse("<|channel|>analysis<|message|>test<|end|>") + + self.assertIsInstance(self.parser.strategy, CanonicalStrategy) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + + def test_strategy_selection_text(self): + """Test automatic strategy selection for text format.""" + events = self.parser.parse("analysis test content") + + self.assertIsInstance(self.parser.strategy, TextStrategy) + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "reasoning") + + def test_strategy_selection_delayed(self): + """Test strategy selection with insufficient initial content.""" + # First chunk doesn't have enough info + events1 = self.parser.parse("some") + self.assertEqual(len(events1), 0) + self.assertIsNone(self.parser.strategy) + + # Second chunk triggers strategy selection + events2 = self.parser.parse(" analysis content") + self.assertIsInstance(self.parser.strategy, TextStrategy) + self.assertEqual(len(events2), 1) + + def test_streaming_canonical_format(self): + """Test streaming with canonical format.""" + chunks = [ + "<|channel|>analysis<|message|>", + "reasoning content", + "<|end|>", + "<|start|>assistant<|channel|>final<|message|>", + "final answer", + "<|return|>", + ] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + self.assertEqual(len(all_events), 5) + + # Verify we get reasoning events + reasoning_events = [e for e in all_events if e.event_type == "reasoning"] + self.assertTrue(len(reasoning_events) > 0) + + # Verify we get normal events + normal_events = [e for e in all_events if e.event_type == "normal"] + self.assertTrue(len(normal_events) > 0) + + # Verify content is eventually parsed correctly + combined_reasoning = "".join(e.content for e in reasoning_events) + combined_normal = "".join( + e.content + for e in normal_events + if e.content and "<|return|>" not in e.content + ) + + self.assertIn("reasoning content", combined_reasoning) + self.assertIn("final answer", combined_normal) + + def test_streaming_text_format(self): + """Test streaming with text format.""" + chunks = ["analysis reasoning", " content assistantfinal", " the answer"] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + # Should have reasoning and normal events + reasoning_events = [e for e in all_events if e.event_type == "reasoning"] + normal_events = [e for e in all_events if e.event_type == "normal"] + + self.assertGreater(len(reasoning_events), 0) + self.assertGreater(len(normal_events), 0) + + def test_streaming_commentary_filler(self): + """Test that 'commentary' filler is filtered in streaming case.""" + # Test when commentary arrives as a separate chunk after <|call|> + chunks = [ + "<|channel|>commentary to=functions.get_weather", + "<|message|>", + '{"location":"SF"}', + "<|call|>", + "comment", # This arrives as separate chunk - should be filtered + "ary", # Continuation of the filler - should be filtered + "<|channel|>commentary to=functions.get_temp", + "<|message|>", + '{"location":"NYC"}', + "<|call|>", + "comment", # Another separate chunk - should be filtered + "ary", # Continuation of the filler - should be filtered + "<|start|>assistant<|channel|>final", + "<|message|>Done<|return|>", + ] + + all_events = [] + for chunk in chunks: + events = self.parser.parse(chunk) + all_events.extend(events) + + # Count event types + tool_events = [e for e in all_events if e.event_type == "tool_call"] + normal_events = [e for e in all_events if e.event_type == "normal"] + + # Should have 2 tool calls and 1 final message + self.assertEqual(len(tool_events), 2, "Should have 2 tool calls") + self.assertEqual( + len(normal_events), 1, "Should have 1 normal event (final message)" + ) + + # Verify no "commentary" in normal events + for event in normal_events: + self.assertNotEqual( + event.content.strip().lower(), + "commentary", + "Commentary filler should not appear as normal content in streaming", + ) + + # Verify content + self.assertEqual(tool_events[0].content, '{"location":"SF"}') + self.assertEqual(tool_events[1].content, '{"location":"NYC"}') + self.assertEqual(normal_events[0].content, "Done") + + def test_repetitive_tool_calls_with_commentary_filler(self): + """Test handling of repetitive tool calls with 'commentary' filler text.""" + # This simulates malformed output with repeated tool calls and commentary filler + text = ( + "<|channel|>analysis<|message|>Need to get weather<|end|>" + '<|start|>assistant<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "commentary" # Filler that should be filtered + '<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "commentary" # Another filler + '<|channel|>commentary to=functions.get_weather<|message|>{"city":"Boston"}<|call|>' + "<|channel|>analysis<|message|>Tool not responding<|end|>" + "<|start|>assistant<|channel|>final<|message|>Unable to fetch weather data<|return|>" + ) + + events = self.parser.parse(text) + + # Count event types + reasoning_events = [e for e in events if e.event_type == "reasoning"] + tool_events = [e for e in events if e.event_type == "tool_call"] + normal_events = [e for e in events if e.event_type == "normal"] + + # Verify correct number of each type + self.assertEqual(len(reasoning_events), 2, "Should have 2 reasoning events") + self.assertEqual(len(tool_events), 3, "Should have 3 tool calls") + self.assertEqual( + len(normal_events), 1, "Should have 1 normal event (final message)" + ) + + # Verify no "commentary" filler in normal events + for event in normal_events: + self.assertNotEqual( + event.content.strip().lower(), + "commentary", + "Commentary filler should not appear as normal content", + ) + + # Verify content is correct + self.assertEqual(reasoning_events[0].content, "Need to get weather") + self.assertEqual(reasoning_events[1].content, "Tool not responding") + self.assertEqual(normal_events[0].content, "Unable to fetch weather data") + + +class TestIntegrationScenarios(CustomTestCase): + """Integration tests for realistic Harmony parsing scenarios.""" + + def test_complete_reasoning_flow(self): + """Test complete reasoning flow from HARMONY_DOCS.md examples.""" + parser = HarmonyParser() + + text = ( + '<|channel|>analysis<|message|>User asks: "What is 2 + 2?" Simple arithmetic. Provide answer.<|end|>' + "<|start|>assistant<|channel|>final<|message|>2 + 2 = 4.<|return|>" + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertIn("Simple arithmetic", events[0].content) + self.assertEqual(events[1].event_type, "normal") + self.assertEqual(events[1].content, "2 + 2 = 4.") + + def test_tool_call_sequence(self): + """Test tool call sequence from HARMONY_DOCS.md examples.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>Need to use function get_weather.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>" + '{"location":"San Francisco"}<|call|>' + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[0].content, "Need to use function get_weather.") + self.assertEqual(events[1].event_type, "tool_call") + self.assertEqual(events[1].content, '{"location":"San Francisco"}') + + def test_preamble_sequence(self): + """Test preamble sequence with multiple commentary blocks.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>Long chain of thought<|end|>" + "<|start|>assistant<|channel|>commentary<|message|>**Action plan**: 1. Generate file 2. Start server<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.generate_file<|message|>" + '{"template": "basic_html"}<|call|>' + ) + + events = parser.parse(text) + + self.assertEqual(len(events), 3) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "normal") + self.assertIn("Action plan", events[1].content) + self.assertEqual(events[2].event_type, "tool_call") + + def test_built_in_tool_call(self): + """Test built-in tool call on analysis channel.""" + parser = HarmonyParser() + + text = '<|channel|>analysis to=browser.search<|message|>{"query": "SGLang"}<|call|>' + + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "tool_call") + self.assertEqual(events[0].content, '{"query": "SGLang"}') + + def test_tool_response_handling(self): + """Test tool response message handling.""" + parser = HarmonyParser() + + text = '<|start|>functions.get_weather to=assistant<|channel|>commentary<|message|>{"sunny": true, "temperature": 20}<|end|>' + + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].event_type, "normal") + self.assertEqual(events[0].content, '{"sunny": true, "temperature": 20}') + + def test_text_fallback_formats(self): + """Test various text fallback formats.""" + parser = HarmonyParser() + + # Test analysis then final + events1 = parser.parse("analysis thinking assistantfinal answer") + self.assertEqual(len([e for e in events1 if e.event_type == "reasoning"]), 1) + self.assertEqual(len([e for e in events1 if e.event_type == "normal"]), 1) + + # Reset parser for next test + parser = HarmonyParser() + + # Test final only + events2 = parser.parse("assistantfinal direct answer") + self.assertEqual(len(events2), 1) + self.assertEqual(events2[0].event_type, "normal") + + def test_streaming_property_canonical(self): + """Test streaming property: chunked parsing produces same semantic content as one-shot parsing.""" + full_text = ( + "<|channel|>analysis<|message|>reasoning content<|end|>" + "<|start|>assistant<|channel|>final<|message|>final content" + ) + + # One-shot parsing + parser1 = HarmonyParser() + events_oneshot = parser1.parse(full_text) + events_oneshot += parser1.parse("") + + # Chunked parsing + parser2 = HarmonyParser() + chunks = [ + "<|channel|>", + "analysis", + "<|message|>", + "reasoning content", + "<|end|>", + "<|start|>assistant", + "<|channel|>final", + "<|message|>", + "final ", + "content", + ] + events_chunked = [] + for chunk in chunks: + events_chunked.extend(parser2.parse(chunk)) + + # Compare semantic content rather than exact event structure + reasoning_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "reasoning" + ) + normal_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "normal" + ) + + reasoning_chunked = "".join( + e.content for e in events_chunked if e.event_type == "reasoning" + ) + normal_chunked = "".join( + e.content for e in events_chunked if e.event_type == "normal" + ) + + self.assertEqual(reasoning_chunked, reasoning_oneshot) + self.assertEqual(normal_chunked, normal_oneshot) + + def test_streaming_property_text(self): + """Test streaming property for text format.""" + full_text = "analysis reasoning content assistantfinal final answer" + + # One-shot parsing + parser1 = HarmonyParser() + events_oneshot = parser1.parse(full_text) + + # Chunked parsing + parser2 = HarmonyParser() + chunks = ["analysis reason", "ing content assistant", "final final answer"] + events_chunked = [] + for chunk in chunks: + events_chunked.extend(parser2.parse(chunk)) + + # Combine content by type for comparison + reasoning_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "reasoning" + ) + normal_oneshot = "".join( + e.content for e in events_oneshot if e.event_type == "normal" + ) + + reasoning_chunked = "".join( + e.content for e in events_chunked if e.event_type == "reasoning" + ) + normal_chunked = "".join( + e.content for e in events_chunked if e.event_type == "normal" + ) + + # Account for whitespace differences due to streaming - compare trimmed content + self.assertEqual(reasoning_oneshot.strip(), reasoning_chunked.strip()) + self.assertEqual(normal_oneshot.strip(), normal_chunked.strip()) + + +class TestEdgeCases(CustomTestCase): + """Test edge cases and error conditions.""" + + def test_malformed_channel_headers(self): + """Test handling of malformed channel headers.""" + parser = HarmonyParser() + + # Unknown channel type + text = "<|channel|>unknown<|message|>content<|end|>" + events = parser.parse(text) + + # Should be held as incomplete since channel is unknown + self.assertEqual(len(events), 0) + + def test_mixed_unknown_tokens(self): + """Test handling of mixed unknown tokens.""" + parser = HarmonyParser() + + text = "text <|weird|> more text <|channel|>analysis<|message|>content<|end|>" + events = parser.parse(text) + + # Should parse the valid parts + reasoning_events = [e for e in events if e.event_type == "reasoning"] + normal_events = [e for e in events if e.event_type == "normal"] + + self.assertEqual(len(reasoning_events), 1) + self.assertGreater(len(normal_events), 0) + + def test_empty_input(self): + """Test handling of empty input.""" + parser = HarmonyParser() + events = parser.parse("") + self.assertEqual(len(events), 0) + + def test_whitespace_preservation(self): + """Test that whitespace is preserved correctly.""" + parser = HarmonyParser() + + text = "<|channel|>analysis<|message|> content with spaces <|end|>" + events = parser.parse(text) + + self.assertEqual(len(events), 1) + self.assertEqual(events[0].content, " content with spaces ") + + def test_streaming_whitespace_preservation(self): + """Test that streaming preserves whitespace between chunks.""" + parser = HarmonyParser() + + # Simulate streaming where space is at chunk boundary + chunks = ["analysis The user typed ", '"wapppa". Not a question.'] + + all_events = [] + for chunk in chunks: + events = parser.parse(chunk) + all_events.extend(events) + + # Combine all reasoning content + reasoning_content = "".join( + e.content for e in all_events if e.event_type == "reasoning" + ) + + # Should preserve the space before the quote + self.assertIn('typed "wapppa"', reasoning_content) + self.assertNotIn( + 'typed"wapppa"', reasoning_content + ) # Should not be mashed together + + def test_consecutive_blocks_same_type(self): + """Test consecutive blocks of the same type.""" + parser = HarmonyParser() + + text = ( + "<|channel|>analysis<|message|>first reasoning<|end|>" + "<|channel|>analysis<|message|>second reasoning<|end|>" + ) + events = parser.parse(text) + + self.assertEqual(len(events), 2) + self.assertEqual(events[0].event_type, "reasoning") + self.assertEqual(events[1].event_type, "reasoning") + self.assertEqual(events[0].content, "first reasoning") + self.assertEqual(events[1].content, "second reasoning") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_health_check.py b/test/srt/test_health_check.py new file mode 100644 index 000000000..1f101e43b --- /dev/null +++ b/test/srt/test_health_check.py @@ -0,0 +1,28 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestHealthCheck(CustomTestCase): + def test_health_check(self): + """Test that metrics endpoint returns data when enabled""" + with self.assertRaises(TimeoutError): + popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=60, + other_args=[ + "--disable-cuda-graph", + "--json-model-override-args", + '{"architectures": ["LlamaForCausalLMForHealthTest"]}', + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_hidden_states.py b/test/srt/test_hidden_states.py new file mode 100644 index 000000000..2046ce529 --- /dev/null +++ b/test/srt/test_hidden_states.py @@ -0,0 +1,139 @@ +import unittest + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestHiddenState(CustomTestCase): + def test_return_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + enable_return_hidden_states=True, + ) + outputs = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for output in outputs: + self.assertEqual(len(output["meta_info"]["hidden_states"]), 8) + for i in range(len(output["meta_info"]["hidden_states"])): + assert isinstance(output["meta_info"]["hidden_states"][i], list) + output["meta_info"]["hidden_states"][i] = torch.tensor( + output["meta_info"]["hidden_states"][i], dtype=torch.bfloat16 + ) + # Checks that splicing of the batch was done correctly + self.assertGreater( + outputs[1]["meta_info"]["hidden_states"][0].shape[0], + outputs[0]["meta_info"]["hidden_states"][0].shape[0], + ) + + model = AutoModelForCausalLM.from_pretrained( + model_path, torch_dtype=torch.bfloat16, device_map="cuda" + ) + + for input_id, output in zip(input_ids, outputs): + with torch.inference_mode(): + hf_out = model( + torch.tensor( + [input_id + output["output_ids"][:-1]], device=model.device + ), + output_hidden_states=True, + ) + print("=== HF Hiddens ===") + print(hf_out["hidden_states"][-1][0]) + sg_hidden_states = torch.cat( + [ + i.unsqueeze(0) if len(i.shape) == 1 else i + for i in output["meta_info"]["hidden_states"] + ] + ).to("cuda") + print("=== SRT Hiddens ===") + print(sg_hidden_states) + + print( + f"Max diff: {torch.max(torch.abs(hf_out['hidden_states'][-1][0] - sg_hidden_states))}" + ) + + atol = 0.8 + self.assertTrue( + torch.allclose( + hf_out["hidden_states"][-1][0], + sg_hidden_states, + atol=atol, + rtol=0, + ) + ) + + def test_repeatedly_changes_hidden_states(self): + prompts = ["Today is", "Today is a sunny day and I like"] + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + tokenizer = AutoTokenizer.from_pretrained(model_path) + input_ids = tokenizer(prompts).input_ids + + sampling_params = { + "temperature": 0, + "max_new_tokens": 8, + } + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + skip_tokenizer_init=True, + enable_return_hidden_states=True, + ) + outputs_completion_first_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + outputs_hidden_state = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=False, + ) + + outputs_completion_last_round = engine.generate( + input_ids=input_ids, + sampling_params=sampling_params, + return_hidden_states=True, + ) + engine.shutdown() + + for ( + output_completion_first_round, + output_hidden_state, + output_completion_last_round, + ) in zip( + outputs_completion_first_round, + outputs_hidden_state, + outputs_completion_last_round, + ): + self.assertEqual( + len(output_completion_first_round["meta_info"]["hidden_states"]), 8 + ) + self.assertNotIn("hidden_states", output_hidden_state["meta_info"]) + self.assertEqual( + len(output_completion_last_round["meta_info"]["hidden_states"]), 8 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_hybrid_attn_backend.py b/test/srt/test_hybrid_attn_backend.py new file mode 100644 index 000000000..1574ff873 --- /dev/null +++ b/test/srt/test_hybrid_attn_backend.py @@ -0,0 +1,161 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--prefill-attention-backend", + "fa3", + "--decode-attention-backend", + "flashinfer", +] + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class TestHybridAttnBackendBase(CustomTestCase): + + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.65 # derived tests need to override this + speculative_decode = False + spec_decode_threshold = 2.2 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + os.environ["SGL_JIT_DEEPGEMM_PRECOMPILE"] = "false" + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "false" + if cls.speculative_decode: + model = DEFAULT_EAGLE_TARGET_MODEL_FOR_TEST + else: + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + if self.speculative_decode: + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestHybridAttnBackendMLA(TestHybridAttnBackendBase): + accuracy_threshold = 0.60 + model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + + +class TestHybridAttnBackendTorchCompile(TestHybridAttnBackendBase): + accuracy_threshold = 0.65 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--enable-torch-compile"] + + +class TestHybridAttnBackendSpeculativeDecodingPrefillBackend(TestHybridAttnBackendBase): + speculative_decode = True + # This eagle test uses a very small model, so the accuracy is low. + accuracy_threshold = 0.2 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "prefill", + ] + + +class TestHybridAttnBackendSpeculativeDecodingDecodeBackend(TestHybridAttnBackendBase): + speculative_decode = True + # This eagle test uses a very small model, so the accuracy is low. + accuracy_threshold = 0.2 + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + [ + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + DEFAULT_EAGLE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "4", + "--speculative-attention-mode", + "decode", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_input_embeddings.py b/test/srt/test_input_embeddings.py new file mode 100644 index 000000000..7efd66565 --- /dev/null +++ b/test/srt/test_input_embeddings.py @@ -0,0 +1,151 @@ +import json +import os +import tempfile +import unittest + +import requests +from transformers import AutoModelForCausalLM, AutoTokenizer + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestInputEmbeds(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model) + cls.ref_model = AutoModelForCausalLM.from_pretrained(cls.model) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-radix", "--cuda-graph-max-bs", 4], + ) + cls.texts = [ + "The capital of France is", + "What is the best time of year to visit Japan for cherry blossoms?", + ] + + def generate_input_embeddings(self, text): + """Generate input embeddings for a given text.""" + input_ids = self.tokenizer(text, return_tensors="pt")["input_ids"] + embeddings = self.ref_model.get_input_embeddings()(input_ids) + return embeddings.squeeze().tolist() # Convert tensor to a list for API use + + def send_request(self, payload): + """Send a POST request to the /generate endpoint and return the response.""" + response = requests.post( + self.base_url + "/generate", + json=payload, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def send_file_request(self, file_path): + """Send a POST request to the /generate_from_file endpoint with a file.""" + with open(file_path, "rb") as f: + response = requests.post( + self.base_url + "/generate_from_file", + files={"file": f}, + timeout=30, # Set a reasonable timeout for the API request + ) + if response.status_code == 200: + return response.json() + return { + "error": f"Request failed with status {response.status_code}: {response.text}" + } + + def test_text_based_response(self): + """Test and print API responses using text-based input.""" + for text in self.texts: + payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Text Input: {text}\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_embedding_based_response(self): + """Test and print API responses using input embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + response = self.send_request(payload) + print( + f"Embeddings Input (for text '{text}'):\nResponse: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + + def test_compare_text_vs_embedding(self): + """Test and compare responses for text-based and embedding-based inputs.""" + for text in self.texts: + # Text-based payload + text_payload = { + "model": self.model, + "text": text, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Embedding-based payload + embeddings = self.generate_input_embeddings(text) + embed_payload = { + "model": self.model, + "input_embeds": embeddings, + "sampling_params": {"temperature": 0, "max_new_tokens": 50}, + } + # Get responses + text_response = self.send_request(text_payload) + embed_response = self.send_request(embed_payload) + # Print responses + print( + f"Text Input: {text}\nText-Based Response: {json.dumps(text_response, indent=2)}\n" + ) + print( + f"Embeddings Input (for text '{text}'):\nEmbedding-Based Response: {json.dumps(embed_response, indent=2)}\n{'-' * 80}" + ) + # This is flaky, so we skip this temporarily + # self.assertEqual(text_response["text"], embed_response["text"]) + + def test_generate_from_file(self): + """Test the /generate_from_file endpoint using tokenized embeddings.""" + for text in self.texts: + embeddings = self.generate_input_embeddings(text) + with tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False + ) as tmp_file: + json.dump(embeddings, tmp_file) + tmp_file_path = tmp_file.name + + try: + response = self.send_file_request(tmp_file_path) + print( + f"Text Input: {text}\nResponse from /generate_from_file: {json.dumps(response, indent=2)}\n{'-' * 80}" + ) + finally: + # Ensure the temporary file is deleted + os.remove(tmp_file_path) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_intel_amx_attention_backend.py b/test/srt/test_intel_amx_attention_backend.py new file mode 100644 index 000000000..5534c57f9 --- /dev/null +++ b/test/srt/test_intel_amx_attention_backend.py @@ -0,0 +1,135 @@ +""" +Usage: +python3 -m unittest test_intel_amx_attention_backend.TestIntelAMXAttnBackend.test_mmlu +""" + +import unittest +from functools import wraps +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE, + DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8, + DEFAULT_MODEL_NAME_FOR_TEST_W8A8, + DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch, +) + + +def intel_amx_benchmark(extra_args=None, min_throughput=None): + def decorator(test_func): + @wraps(test_func) + def wrapper(self): + common_args = [ + "--attention-backend", + "intel_amx", + "--disable-radix", + "--trust-remote-code", + ] + full_args = common_args + (extra_args or []) + + model = test_func(self) + prefill_latency, decode_throughput, decode_latency = run_bench_one_batch( + model, full_args + ) + + print(f"{model=}") + print(f"{prefill_latency=}") + print(f"{decode_throughput=}") + print(f"{decode_latency=}") + + if is_in_ci() and min_throughput is not None: + self.assertGreater(decode_throughput, min_throughput) + + return wrapper + + return decorator + + +class TestIntelAMXAttnBackend(CustomTestCase): + + @intel_amx_benchmark(extra_args=["--batch-size", "4"], min_throughput=10) + def test_latency_mla_model(self): + return DEFAULT_MLA_MODEL_NAME_FOR_TEST + + @intel_amx_benchmark(extra_args=["--batch-size", "4"], min_throughput=40) + def test_latency_default_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST + + @intel_amx_benchmark(extra_args=["--batch-size", "4"], min_throughput=150) + def test_latency_fp8_qwen(self): + return DEFAULT_MODEL_NAME_FOR_TEST_QWEN_FP8 + + @intel_amx_benchmark(extra_args=["--batch-size", "4"], min_throughput=50) + def test_latency_fp8_moe_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_FP8_WITH_MOE + + @intel_amx_benchmark( + extra_args=["--batch-size", "4", "--quantization", "w8a8_int8"], + min_throughput=100, + ) + def test_latency_w8a8_default_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_W8A8 + + @intel_amx_benchmark( + extra_args=[ + "--batch-size", + "4", + "--quantization", + "w8a8_int8", + "--mem-fraction-static", + "0.9", + "--max-total-tokens", + "65536", + "--tp", + "6", + ], + min_throughput=100, + ) + def test_latency_w8a8_moe_model(self): + return DEFAULT_MODEL_NAME_FOR_TEST_W8A8_WITH_MOE + + def test_mmlu(self): + model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + "intel_amx", + "--mem-fraction-static", + "0.3", + "--disable-radix", + "--trust-remote-code", + "--disable-overlap-schedule", + ], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + metrics = run_eval(args) + if is_in_ci(): + self.assertGreater(metrics["score"], 0.45) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_io_struct.py b/test/srt/test_io_struct.py new file mode 100644 index 000000000..a8efebb23 --- /dev/null +++ b/test/srt/test_io_struct.py @@ -0,0 +1,577 @@ +import copy +import unittest + +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + CustomTestCase, +) + + +class TestGenerateReqInputNormalization(CustomTestCase): + """Test the normalization of GenerateReqInput for batch processing and different input formats.""" + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + def setUp(self): + # Common setup for all tests + self.base_req = GenerateReqInput( + text=["Hello", "World"], + sampling_params=[{}, {}], + rid=["id1", "id2"], + ) + + def test_single_image_to_list_of_lists(self): + """Test that a single image is converted to a list of single-image lists.""" + req = copy.deepcopy(self.base_req) + req.image_data = "single_image.jpg" # A single image (non-list) + + req.normalize_batch_and_arguments() + + # Should be converted to [[image], [image]] + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + self.assertEqual(req.image_data[0][0], "single_image.jpg") + self.assertEqual(req.image_data[1][0], "single_image.jpg") + + # Check modalities + self.assertEqual(req.modalities, ["image", "image"]) + + def test_list_of_images_to_list_of_lists(self): + """Test that a list of images is converted to a list of single-image lists.""" + req = copy.deepcopy(self.base_req) + req.image_data = ["image1.jpg", "image2.jpg"] # List of images + + req.normalize_batch_and_arguments() + + # Should be converted to [[image1], [image2]] + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + self.assertEqual(req.image_data[0][0], "image1.jpg") + self.assertEqual(req.image_data[1][0], "image2.jpg") + + # Check modalities + self.assertEqual(req.modalities, ["image", "image"]) + + def test_list_of_lists_with_different_modalities(self): + """Test handling of list of lists of images with different modalities.""" + req = copy.deepcopy(self.base_req) + req.image_data = [ + ["image1.jpg"], # Single image (image modality) + ["image2.jpg", "image3.jpg"], # Multiple images (multi-images modality) + ] + + req.normalize_batch_and_arguments() + + # Structure should remain the same + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 2) + + # Check modalities + self.assertEqual(req.modalities, ["image", "multi-images"]) + + def test_list_of_lists_with_none_values(self): + """Test handling of list of lists with None values.""" + req = copy.deepcopy(self.base_req) + req.image_data = [ + [None], # None value + ["image.jpg"], # Single image + ] + + req.normalize_batch_and_arguments() + + # Structure should remain the same + self.assertEqual(len(req.image_data), 2) + self.assertEqual(len(req.image_data[0]), 1) + self.assertEqual(len(req.image_data[1]), 1) + + # Check modalities + self.assertEqual(req.modalities, [None, "image"]) + + def test_expanding_parallel_sample_correlation(self): + """Test that when expanding with parallel samples, prompts, images and modalities are properly correlated.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg", "image3.jpg"], + ] + req.sampling_params = {"n": 3} # All prompts get 3 samples + + # Define expected values before normalization + expected_text = req.text * 3 + expected_images = req.image_data * 3 + expected_modalities = ["image", "multi-images"] * 3 + + req.normalize_batch_and_arguments() + + # Should be expanded to 6 items (2 original * 3 parallel) + self.assertEqual(len(req.image_data), 6) + + # Check that images are properly expanded + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Ensure that text items are properly duplicated too + self.assertEqual(req.text, expected_text) + + def test_specific_parallel_n_per_sample(self): + """Test parallel expansion when different samples have different n values.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg", "image3.jpg"], + ] + req.sampling_params = [ + {"n": 2}, + {"n": 2}, + ] # First prompt gets 2 samples, second prompt gets 2 samples + + expected_images = req.image_data * 2 + expected_modalities = ["image", "multi-images"] * 2 + expected_text = req.text * 2 + + req.normalize_batch_and_arguments() + + # Should be expanded to 4 items (2 original * 2 parallel) + self.assertEqual(len(req.image_data), 4) + + # Check that the first 2 are copies for the first prompt + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Check text expansion + self.assertEqual(req.text, expected_text) + + def test_mixed_none_and_images_with_parallel_samples(self): + """Test that when some batch items have images and others None, parallel expansion works correctly.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2", "Prompt 3"] + req.rid = ["id1", "id2", "id3"] + req.image_data = [ + ["image1.jpg"], + None, + ["image3_1.jpg", "image3_2.jpg"], + ] + req.sampling_params = {"n": 2} # All prompts get 2 samples + + expected_images = req.image_data * 2 + expected_modalities = ["image", None, "multi-images"] * 2 + expected_text = req.text * 2 + + req.normalize_batch_and_arguments() + + # Should be expanded to 6 items (3 original * 2 parallel) + self.assertEqual(len(req.image_data), 6) + + # Check image data + self.assertEqual(req.image_data, expected_images) + + # Check modalities + self.assertEqual(req.modalities, expected_modalities) + + # Check text expansion + self.assertEqual(req.text, expected_text) + + def test_correlation_with_sampling_params(self): + """Test that sampling parameters are correctly correlated with prompts during expansion.""" + req = copy.deepcopy(self.base_req) + req.text = ["Prompt 1", "Prompt 2"] + req.image_data = [ + ["image1.jpg"], + ["image2.jpg"], + ] + req.sampling_params = [ + {"temperature": 0.7, "n": 2}, + {"temperature": 0.9, "n": 2}, + ] + + req.normalize_batch_and_arguments() + + # Check sampling params expansion + self.assertEqual(len(req.sampling_params), 4) + self.assertEqual(req.sampling_params[0]["temperature"], 0.7) + self.assertEqual(req.sampling_params[1]["temperature"], 0.9) + self.assertEqual(req.sampling_params[2]["temperature"], 0.7) + self.assertEqual(req.sampling_params[3]["temperature"], 0.9) + + # Should be expanded to 4 items (2 original * 2 parallel) + self.assertEqual(len(req.image_data), 4) + + # Check correlation with images + self.assertEqual(req.image_data[0], ["image1.jpg"]) + self.assertEqual(req.image_data[1], ["image2.jpg"]) + self.assertEqual(req.image_data[2], ["image1.jpg"]) + self.assertEqual(req.image_data[3], ["image2.jpg"]) + + def test_single_example_with_image(self): + """Test handling of single example with image.""" + req = GenerateReqInput( + text="Hello", + image_data="single_image.jpg", + ) + + req.normalize_batch_and_arguments() + + # For single examples, image_data doesn't get processed into lists + self.assertEqual(req.image_data, "single_image.jpg") + self.assertIsNone(req.modalities) # Modalities isn't set for single examples + + def test_single_to_batch_with_parallel_sampling(self): + """Test single example converted to batch with parallel sampling.""" + req = GenerateReqInput( + text="Hello", + image_data="single_image.jpg", + sampling_params={"n": 3}, # parallel_sample_num = 3 + ) + + # Define expected values before normalization + expected_text = ["Hello"] * 3 + + req.normalize_batch_and_arguments() + + # Should be converted to batch with text=["Hello"] + self.assertEqual(req.text, expected_text) + + # Image should be automatically wrapped to list of lists with length 1*3=3 + self.assertEqual(len(req.image_data), 3) + self.assertEqual(req.image_data[0][0], "single_image.jpg") + self.assertEqual(req.image_data[1][0], "single_image.jpg") + self.assertEqual(req.image_data[2][0], "single_image.jpg") + + # Modalities should be set for all 3 examples + self.assertEqual(req.modalities, ["image", "image", "image"]) + + def test_audio_data_handling(self): + """Test handling of audio_data.""" + req = copy.deepcopy(self.base_req) + req.audio_data = "audio.mp3" # Single audio + + req.normalize_batch_and_arguments() + + # Should be converted to ["audio.mp3", "audio.mp3"] + self.assertEqual(len(req.audio_data), 2) + self.assertEqual(req.audio_data[0], "audio.mp3") + self.assertEqual(req.audio_data[1], "audio.mp3") + + # Test with list + req = copy.deepcopy(self.base_req) + req.audio_data = ["audio1.mp3", "audio2.mp3"] + + req.normalize_batch_and_arguments() + + # Should remain the same + self.assertEqual(len(req.audio_data), 2) + self.assertEqual(req.audio_data[0], "audio1.mp3") + self.assertEqual(req.audio_data[1], "audio2.mp3") + + def test_input_ids_normalization(self): + """Test normalization of input_ids instead of text.""" + # Test single input_ids + req = GenerateReqInput(input_ids=[1, 2, 3]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + self.assertEqual(req.batch_size, 1) + + # Test batch input_ids + req = GenerateReqInput(input_ids=[[1, 2, 3], [4, 5, 6]]) + req.normalize_batch_and_arguments() + self.assertFalse(req.is_single) + self.assertEqual(req.batch_size, 2) + + # Test with parallel sampling + req = GenerateReqInput( + input_ids=[[1, 2, 3], [4, 5, 6]], sampling_params={"n": 2} + ) + req.normalize_batch_and_arguments() + self.assertEqual(len(req.input_ids), 4) # 2 original * 2 parallel + + def test_input_embeds_normalization(self): + """Test normalization of input_embeds.""" + # Test single input_embeds + req = GenerateReqInput(input_embeds=[[0.1, 0.2], [0.3, 0.4]]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + self.assertEqual(req.batch_size, 1) + + # Test batch input_embeds + req = GenerateReqInput(input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]]) + req.normalize_batch_and_arguments() + self.assertFalse(req.is_single) + self.assertEqual(req.batch_size, 2) + + def test_input_embeds_with_parallel_sampling(self): + """Test input_embeds normalization with parallel sampling (n > 1).""" + # Test single input_embeds with parallel sampling + req = GenerateReqInput( + input_embeds=[[0.1, 0.2]], # single embedding vector + sampling_params={"n": 2}, + ) + req.normalize_batch_and_arguments() + + # Should be converted from single to batch and then expanded + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 2) + # Both should be the same input_embeds + self.assertEqual(req.input_embeds[0], [[0.1, 0.2]]) + self.assertEqual(req.input_embeds[1], [[0.1, 0.2]]) + + # Test batch input_embeds with parallel sampling + req = GenerateReqInput( + input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]], sampling_params={"n": 3} + ) + req.normalize_batch_and_arguments() + + # Should be expanded + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 6) + + # Check that the expansion is correct + expected_embeds = [[[0.1, 0.2]], [[0.3, 0.4]]] * 3 + self.assertEqual(req.input_embeds, expected_embeds) + + # Test with different n values per sample (should raise error) + req = GenerateReqInput( + input_embeds=[[[0.1, 0.2]], [[0.3, 0.4]]], + sampling_params=[{"n": 2}, {"n": 3}], + ) + with self.assertRaises(ValueError): + req.normalize_batch_and_arguments() + + def test_input_embeds_single_to_batch_conversion(self): + """Test that single input_embeds are properly converted to batch when using parallel sampling.""" + # Test the specific case that was fixed: single input_embeds with n > 1 + req = GenerateReqInput( + input_embeds=[[0.1, 0.2, 0.3]], sampling_params={"n": 2} # Single embedding + ) + req.normalize_batch_and_arguments() + + # Should convert single to batch and then expand + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 2) + + # Both should be the same single embedding + self.assertEqual(req.input_embeds[0], [[0.1, 0.2, 0.3]]) + self.assertEqual(req.input_embeds[1], [[0.1, 0.2, 0.3]]) + + # Test with higher n value + req = GenerateReqInput(input_embeds=[[0.1, 0.2, 0.3]], sampling_params={"n": 5}) + req.normalize_batch_and_arguments() + + self.assertFalse(req.is_single) + self.assertEqual(len(req.input_embeds), 5) + + # All should be the same + for i in range(5): + self.assertEqual(req.input_embeds[i], [[0.1, 0.2, 0.3]]) + + def test_lora_path_normalization(self): + """Test normalization of lora_path.""" + # Test single lora_path with batch input + req = GenerateReqInput(text=["Hello", "World"], lora_path="path/to/lora") + + # Define expected lora_paths before normalization + expected_lora_paths = ["path/to/lora", "path/to/lora"] + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + # Test list of lora_paths + req = GenerateReqInput(text=["Hello", "World"], lora_path=["path1", "path2"]) + + # Define expected lora_paths before normalization + expected_lora_paths = ["path1", "path2"] + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + # Test with parallel sampling + req = GenerateReqInput( + text=["Hello", "World"], + lora_path=["path1", "path2"], + sampling_params={"n": 2}, + ) + + # Define expected lora_paths before normalization + expected_lora_paths = ["path1", "path2"] * 2 + + req.normalize_batch_and_arguments() + self.assertEqual(req.lora_path, expected_lora_paths) + + def test_logprob_parameters_normalization(self): + """Test normalization of logprob-related parameters.""" + # Test single example + req = GenerateReqInput( + text="Hello", + return_logprob=True, + logprob_start_len=10, + top_logprobs_num=5, + token_ids_logprob=[7, 8, 9], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, True) + self.assertEqual(req.logprob_start_len, 10) + self.assertEqual(req.top_logprobs_num, 5) + self.assertEqual(req.token_ids_logprob, [7, 8, 9]) + + # Test batch with scalar values + req = GenerateReqInput( + text=["Hello", "World"], + return_logprob=True, + logprob_start_len=10, + top_logprobs_num=5, + token_ids_logprob=[7, 8, 9], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, [True, True]) + self.assertEqual(req.logprob_start_len, [10, 10]) + self.assertEqual(req.top_logprobs_num, [5, 5]) + self.assertEqual(req.token_ids_logprob, [[7, 8, 9], [7, 8, 9]]) + + # Test batch with list values + req = GenerateReqInput( + text=["Hello", "World"], + return_logprob=[True, False], + logprob_start_len=[10, 5], + top_logprobs_num=[5, 3], + token_ids_logprob=[[7, 8, 9], [4, 5, 6]], + return_hidden_states=[False, False, True], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.return_logprob, [True, False]) + self.assertEqual(req.logprob_start_len, [10, 5]) + self.assertEqual(req.top_logprobs_num, [5, 3]) + self.assertEqual(req.token_ids_logprob, [[7, 8, 9], [4, 5, 6]]) + self.assertEqual(req.return_hidden_states, [False, False, True]) + + def test_custom_logit_processor_normalization(self): + """Test normalization of custom_logit_processor.""" + # Test single processor + req = GenerateReqInput( + text=["Hello", "World"], custom_logit_processor="serialized_processor" + ) + req.normalize_batch_and_arguments() + self.assertEqual( + req.custom_logit_processor, ["serialized_processor", "serialized_processor"] + ) + + # Test list of processors + req = GenerateReqInput( + text=["Hello", "World"], custom_logit_processor=["processor1", "processor2"] + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.custom_logit_processor, ["processor1", "processor2"]) + + def test_session_params_handling(self): + """Test handling of session_params.""" + # Test with dict + req = GenerateReqInput( + text=["Hello", "World"], session_params={"id": "session1", "offset": 10} + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.session_params, {"id": "session1", "offset": 10}) + + # Test with list of dicts + req = GenerateReqInput( + text=["Hello", "World"], + session_params=[{"id": "session1"}, {"id": "session2"}], + ) + req.normalize_batch_and_arguments() + self.assertEqual(req.session_params, [{"id": "session1"}, {"id": "session2"}]) + + def test_getitem_method(self): + """Test the __getitem__ method.""" + req = GenerateReqInput( + text=["Hello", "World"], + image_data=[["img1.jpg"], ["img2.jpg"]], + audio_data=["audio1.mp3", "audio2.mp3"], + sampling_params=[{"temp": 0.7}, {"temp": 0.8}], + rid=["id1", "id2"], + return_logprob=[True, False], + logprob_start_len=[10, 5], + top_logprobs_num=[5, 3], + token_ids_logprob=[[7, 8, 9], [4, 5, 6]], + stream=True, + log_metrics=True, + modalities=["image", "image"], + lora_path=["path1", "path2"], + custom_logit_processor=["processor1", "processor2"], + return_hidden_states=True, + ) + req.normalize_batch_and_arguments() + + # Get the first item + item0 = req[0] + self.assertEqual(item0.text, "Hello") + self.assertEqual(item0.image_data, ["img1.jpg"]) + self.assertEqual(item0.audio_data, "audio1.mp3") + self.assertEqual(item0.sampling_params, {"temp": 0.7}) + self.assertEqual(item0.rid, "id1") + self.assertEqual(item0.return_logprob, True) + self.assertEqual(item0.logprob_start_len, 10) + self.assertEqual(item0.top_logprobs_num, 5) + self.assertEqual(item0.token_ids_logprob, [7, 8, 9]) + self.assertEqual(item0.stream, True) + self.assertEqual(item0.log_metrics, True) + self.assertEqual(item0.modalities, "image") + self.assertEqual(item0.lora_path, "path1") + self.assertEqual(item0.custom_logit_processor, "processor1") + self.assertEqual(item0.return_hidden_states, True) + + def test_regenerate_rid(self): + """Test the regenerate_rid method.""" + req = GenerateReqInput(text="Hello") + req.normalize_batch_and_arguments() + + original_rid = req.rid + new_rid = req.regenerate_rid() + + self.assertNotEqual(original_rid, new_rid) + self.assertEqual(req.rid, new_rid) + + def test_error_cases(self): + """Test various error cases.""" + # Test when neither text, input_ids, nor input_embeds is provided + with self.assertRaises(ValueError): + req = GenerateReqInput() + req.normalize_batch_and_arguments() + + # Test when all of text, input_ids, and input_embeds are provided + with self.assertRaises(ValueError): + req = GenerateReqInput( + text="Hello", input_ids=[1, 2, 3], input_embeds=[[0.1, 0.2]] + ) + req.normalize_batch_and_arguments() + + def test_multiple_input_formats(self): + """Test different combinations of input formats.""" + # Test with text only + req = GenerateReqInput(text="Hello") + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + # Test with input_ids only + req = GenerateReqInput(input_ids=[1, 2, 3]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + # Test with input_embeds only + req = GenerateReqInput(input_embeds=[[0.1, 0.2]]) + req.normalize_batch_and_arguments() + self.assertTrue(req.is_single) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_jinja_template_utils.py b/test/srt/test_jinja_template_utils.py new file mode 100644 index 000000000..46e634006 --- /dev/null +++ b/test/srt/test_jinja_template_utils.py @@ -0,0 +1,310 @@ +""" +Unit tests for Jinja chat template utils. +""" + +import unittest + +from sglang.srt.parser.jinja_template_utils import ( + detect_jinja_template_content_format, + process_content_for_template_format, +) +from sglang.test.test_utils import CustomTestCase + + +class TestTemplateContentFormatDetection(CustomTestCase): + """Test template content format detection functionality.""" + + def test_detect_llama4_openai_format(self): + """Test detection of llama4-style template (should be 'openai' format).""" + llama4_pattern = """ +{%- for message in messages %} + {%- if message['content'] is string %} + {{- message['content'] }} + {%- else %} + {%- for content in message['content'] %} + {%- if content['type'] == 'image' %} + {{- '<|image|>' }} + {%- elif content['type'] == 'text' %} + {{- content['text'] | trim }} + {%- endif %} + {%- endfor %} + {%- endif %} +{%- endfor %} + """ + + result = detect_jinja_template_content_format(llama4_pattern) + self.assertEqual(result, "openai") + + def test_detect_deepseek_string_format(self): + """Test detection of deepseek-style template (should be 'string' format).""" + deepseek_pattern = """ +{%- for message in messages %} + {%- if message['role'] == 'user' %} + {{- '<|User|>' + message['content'] + '<|Assistant|>' }} + {%- endif %} +{%- endfor %} + """ + + result = detect_jinja_template_content_format(deepseek_pattern) + self.assertEqual(result, "string") + + def test_detect_invalid_template(self): + """Test handling of invalid template (should default to 'string').""" + invalid_pattern = "{{{{ invalid jinja syntax }}}}" + + result = detect_jinja_template_content_format(invalid_pattern) + self.assertEqual(result, "string") + + def test_detect_empty_template(self): + """Test handling of empty template (should default to 'string').""" + result = detect_jinja_template_content_format("") + self.assertEqual(result, "string") + + def test_detect_msg_content_pattern(self): + """Test detection of template with msg.content pattern (should be 'openai' format).""" + msg_content_pattern = """ +[gMASK] +{%- for msg in messages %} + {%- if msg.role == 'system' %} +<|system|> +{{ msg.content }} + {%- elif msg.role == 'user' %} +<|user|>{{ '\n' }} + {%- if msg.content is string %} +{{ msg.content }} + {%- else %} + {%- for item in msg.content %} + {%- if item.type == 'video' or 'video' in item %} +<|begin_of_video|><|video|><|end_of_video|> + {%- elif item.type == 'image' or 'image' in item %} +<|begin_of_image|><|image|><|end_of_image|> + {%- elif item.type == 'text' %} +{{ item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- elif msg.role == 'assistant' %} + {%- if msg.metadata %} +<|assistant|>{{ msg.metadata }} +{{ msg.content }} + {%- else %} +<|assistant|> +{{ msg.content }} + {%- endif %} + {%- endif %} +{%- endfor %} +{% if add_generation_prompt %}<|assistant|> +{% endif %} + """ + + result = detect_jinja_template_content_format(msg_content_pattern) + self.assertEqual(result, "openai") + + def test_detect_m_content_pattern(self): + """Test detection of template with m.content pattern (should be 'openai' format).""" + msg_content_pattern = """ +[gMASK] +{%- for m in messages %} + {%- if m.role == 'system' %} +<|system|> +{{ m.content }} + {%- elif m.role == 'user' %} +<|user|>{{ '\n' }} + {%- if m.content is string %} +{{ m.content }} + {%- else %} + {%- for item in m.content %} + {%- if item.type == 'video' or 'video' in item %} +<|begin_of_video|><|video|><|end_of_video|> + {%- elif item.type == 'image' or 'image' in item %} +<|begin_of_image|><|image|><|end_of_image|> + {%- elif item.type == 'text' %} +{{ item.text }} + {%- endif %} + {%- endfor %} + {%- endif %} + {%- elif m.role == 'assistant' %} + {%- if m.metadata %} +<|assistant|>{{ m.metadata }} +{{ m.content }} + {%- else %} +<|assistant|> +{{ m.content }} + {%- endif %} + {%- endif %} +{%- endfor %} +{% if add_generation_prompt %}<|assistant|> +{% endif %} + """ + + result = detect_jinja_template_content_format(msg_content_pattern) + self.assertEqual(result, "openai") + + def test_process_content_openai_format(self): + """Test content processing for openai format.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Look at this image:"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + }, + {"type": "text", "text": "What do you see?"}, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that image_data was extracted + self.assertEqual(len(image_data), 1) + self.assertEqual(image_data[0].url, "http://example.com/image.jpg") + + # Check that content was normalized + expected_content = [ + {"type": "text", "text": "Look at this image:"}, + {"type": "image"}, # normalized from image_url + {"type": "text", "text": "What do you see?"}, + ] + self.assertEqual(result["content"], expected_content) + self.assertEqual(result["role"], "user") + + def test_process_content_string_format(self): + """Test content processing for string format.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + }, + {"type": "text", "text": "world"}, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "string", image_data, video_data, audio_data, modalities + ) + + # For string format, should flatten to text only + self.assertEqual(result["content"], "Hello world") + self.assertEqual(result["role"], "user") + + # Image data should not be extracted for string format + self.assertEqual(len(image_data), 0) + + def test_process_content_with_audio(self): + """Test content processing with audio content.""" + msg_dict = { + "role": "user", + "content": [ + {"type": "text", "text": "Listen to this:"}, + { + "type": "audio_url", + "audio_url": {"url": "http://example.com/audio.mp3"}, + }, + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that audio_data was extracted + self.assertEqual(len(audio_data), 1) + self.assertEqual(audio_data[0], "http://example.com/audio.mp3") + + # Check that content was normalized + expected_content = [ + {"type": "text", "text": "Listen to this:"}, + {"type": "audio"}, # normalized from audio_url + ] + self.assertEqual(result["content"], expected_content) + + def test_process_content_already_string(self): + """Test processing content that's already a string.""" + msg_dict = {"role": "user", "content": "Hello world"} + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Should pass through unchanged + self.assertEqual(result["content"], "Hello world") + self.assertEqual(result["role"], "user") + self.assertEqual(len(image_data), 0) + + def test_process_content_with_modalities(self): + """Test content processing with modalities field.""" + msg_dict = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "http://example.com/image.jpg"}, + "modalities": ["vision"], + } + ], + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "openai", image_data, video_data, audio_data, modalities + ) + + # Check that modalities was extracted + self.assertEqual(len(modalities), 1) + self.assertEqual(modalities[0], ["vision"]) + + def test_process_content_filter_none_values(self): + """Test that None values are filtered out of processed messages.""" + msg_dict = { + "role": "user", + "content": "Hello", + "name": None, + "tool_call_id": None, + } + + image_data = [] + video_data = [] + audio_data = [] + modalities = [] + + result = process_content_for_template_format( + msg_dict, "string", image_data, video_data, audio_data, modalities + ) + + # None values should be filtered out + expected_keys = {"role", "content"} + self.assertEqual(set(result.keys()), expected_keys) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_kv_events.py b/test/srt/test_kv_events.py new file mode 100644 index 000000000..d333738c7 --- /dev/null +++ b/test/srt/test_kv_events.py @@ -0,0 +1,377 @@ +import time +import unittest + +import msgspec +import requests +import zmq +from msgspec.msgpack import Decoder + +from sglang.srt.disaggregation.kv_events import ( + AllBlocksCleared, + BlockRemoved, + BlockStored, + EventBatch, + KVCacheEvent, + KVEventBatch, +) +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestKvEvents(CustomTestCase): + def test_kv_events_enabled(self): + """Test that kv events are sent and received by subscriber data when enabled""" + + # Launch kv events subscriber + decoder = Decoder(type=KVEventBatch) + context = zmq.Context() + sub = context.socket(zmq.SUB) + sub.connect("tcp://localhost:5557") + topic = "kv-events" + sub.setsockopt_string(zmq.SUBSCRIBE, topic) + + # Launch sglang server + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--kv-events-config", + '{"publisher": "zmq", "topic": "kv-events"}', + "--max-total-tokens", + 32, + "--cuda-graph-max-bs", + 2, + "--enable-dp-attention", + "--dp-size", + 1, + ], + ) + + try: + # Make some requests to generate some metrics + response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate") + self.assertEqual(response.status_code, 200) + + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of Spain is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + + # Expected events. These may be dependent on model used (meta-llama/Llama-3.2-1B-Instruct) + expected_events = [ + # The capital city of France is + BlockStored( + block_hashes=[-6650323075460941099], + parent_block_hash=5740354900026072187, + token_ids=[128000, 791, 6864, 3363, 315, 9822, 374], + block_size=7, + lora_id=None, + ), + # Paris. The Eiffel Tower + BlockStored( + block_hashes=[-7584018293207282755], + parent_block_hash=-6650323075460941099, + token_ids=[12366, 13, 578, 469, 3168, 301, 22703], + block_size=7, + lora_id=None, + ), + BlockStored( + block_hashes=[-8753497827991233192], + parent_block_hash=5740354900026072187, + token_ids=[0], + block_size=1, + lora_id=None, + ), + BlockRemoved(block_hashes=[-6650323075460941099]), + # The capital + BlockStored( + block_hashes=[-2697055055087824455], + parent_block_hash=5740354900026072187, + token_ids=[128000, 791, 6864], + block_size=3, + lora_id=None, + ), + # city of France is + BlockStored( + block_hashes=[-7505627135785778022], + parent_block_hash=-2697055055087824455, + token_ids=[3363, 315, 9822, 374], + block_size=4, + lora_id=None, + ), + # of France is + BlockStored( + block_hashes=[-3861108700662737012], + parent_block_hash=-2697055055087824455, + token_ids=[315, 9822, 374], + block_size=3, + lora_id=None, + ), + BlockRemoved(block_hashes=[-7584018293207282755]), + BlockRemoved(block_hashes=[-8753497827991233192]), + BlockRemoved(block_hashes=[-7505627135785778022]), + # Paris. The Eiffel Tower is located in Paris. The Eiffel Tower is a famous landmark in Paris + BlockStored( + block_hashes=[-3064341286825792715], + parent_block_hash=-3861108700662737012, + token_ids=[ + 12366, + 13, + 578, + 469, + 3168, + 301, + 22703, + 374, + 7559, + 304, + 12366, + 13, + 578, + 469, + 3168, + 301, + 22703, + 374, + 264, + 11495, + 38350, + 304, + 12366, + ], + block_size=23, + lora_id=None, + ), + BlockRemoved(block_hashes=[-3861108700662737012]), + # of + BlockStored( + block_hashes=[6115672085296369592], + parent_block_hash=-2697055055087824455, + token_ids=[315], + block_size=1, + lora_id=None, + ), + # France is + BlockStored( + block_hashes=[4208810872343132234], + parent_block_hash=6115672085296369592, + token_ids=[9822, 374], + block_size=2, + lora_id=None, + ), + # Spain is + BlockStored( + block_hashes=[1675819893649989955], + parent_block_hash=6115672085296369592, + token_ids=[18157, 374], + block_size=2, + lora_id=None, + ), + BlockRemoved(block_hashes=[-3064341286825792715]), + # Madrid. The capital of France is Paris. The capital of Italy is Rome. The capital of Spain is Madrid. + BlockStored( + block_hashes=[-8505834929190027295], + parent_block_hash=1675819893649989955, + token_ids=[ + 25048, + 13, + 578, + 6864, + 315, + 9822, + 374, + 12366, + 13, + 578, + 6864, + 315, + 15704, + 374, + 22463, + 13, + 578, + 6864, + 315, + 18157, + 374, + 25048, + 13, + ], + block_size=23, + lora_id=None, + ), + ] + + # Get events + events = [] + start = time.time() + max_wait_s = 5 + while ( + len(events) < len(expected_events) + and (time.time() - start) < max_wait_s + ): + _, seq_bytes, payload = sub.recv_multipart() + event_batch = decoder.decode(payload) + for event in event_batch.events: + events.append(event) + + for expected in expected_events: + self.assertIn(expected, events) + + finally: + kill_process_tree(process.pid) + + def test_kv_events_attn_dp(self): + """Test that kv events are properly tagged with DP rank in attention DP mode""" + + # Launch multiple subscribers for different DP ranks + decoder = Decoder(type=KVEventBatch) + context = zmq.Context() + + # Subscribe to both DP rank endpoints + sub_dp0 = context.socket(zmq.SUB) + sub_dp0.connect("tcp://localhost:5557") # DP rank 0 + topic = "kv-events" + sub_dp0.setsockopt_string(zmq.SUBSCRIBE, topic) + + sub_dp1 = context.socket(zmq.SUB) + sub_dp1.connect("tcp://localhost:5558") # DP rank 1 (offset by rank) + sub_dp1.setsockopt_string(zmq.SUBSCRIBE, topic) + + # Launch sglang server with DP attention enabled + process = popen_launch_server( + "silence09/DeepSeek-R1-Small-2layers", + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--kv-events-config", + '{"publisher": "zmq", "topic": "kv-events"}', + "--max-total-tokens", + 64, + "--cuda-graph-max-bs", + 4, + "--enable-dp-attention", + "--dp-size", + 2, + "--tp-size", + 2, + ], + ) + + try: + # Make requests to generate events + response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate") + self.assertEqual(response.status_code, 200) + + # Send multiple requests to trigger events from both DP ranks + for i in range(4): + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": f"Request {i}: The capital of country {i} is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16, + }, + }, + ) + + # Collect events from both DP ranks + events_dp0 = [] + events_dp1 = [] + start = time.time() + max_wait_s = 10 + min_events_per_rank = 3 # Expect at least a few events from each rank + + while (time.time() - start) < max_wait_s and ( + len(events_dp0) < min_events_per_rank + or len(events_dp1) < min_events_per_rank + ): + # Check DP rank 0 + if sub_dp0.poll(timeout=100): # 100ms timeout + _, seq_bytes, payload = sub_dp0.recv_multipart() + event_batch = decoder.decode(payload) + print( + f"DP Rank 0 - EventBatch: ts={event_batch.ts}, attn_dp_rank={event_batch.attn_dp_rank}" + ) + self.assertEqual( + event_batch.attn_dp_rank, + 0, + "DP rank 0 events should have attn_dp_rank=0", + ) + for event in event_batch.events: + print(f" DP0 - {event}") + events_dp0.append(event) + + # Check DP rank 1 + if sub_dp1.poll(timeout=100): # 100ms timeout + _, seq_bytes, payload = sub_dp1.recv_multipart() + event_batch = decoder.decode(payload) + print( + f"DP Rank 1 - EventBatch: ts={event_batch.ts}, attn_dp_rank={event_batch.attn_dp_rank}" + ) + self.assertEqual( + event_batch.attn_dp_rank, + 1, + "DP rank 1 events should have attn_dp_rank=1", + ) + for event in event_batch.events: + print(f" DP1 - {event}") + events_dp1.append(event) + + # Verify we got events from both DP ranks + print(f"Collected {len(events_dp0)} events from DP rank 0") + print(f"Collected {len(events_dp1)} events from DP rank 1") + + self.assertGreaterEqual( + len(events_dp0), + min_events_per_rank, + f"Expected at least {min_events_per_rank} events from DP rank 0", + ) + self.assertGreaterEqual( + len(events_dp1), + min_events_per_rank, + f"Expected at least {min_events_per_rank} events from DP rank 1", + ) + + # Verify event types are as expected + for events in [events_dp0, events_dp1]: + for event in events: + self.assertIsInstance( + event, + (BlockStored, BlockRemoved, AllBlocksCleared), + f"Event should be a KV cache event, got {type(event)}", + ) + + finally: + sub_dp0.close() + sub_dp1.close() + context.term() + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_local_attn.py b/test/srt/test_local_attn.py new file mode 100644 index 000000000..923ffdd5e --- /dev/null +++ b/test/srt/test_local_attn.py @@ -0,0 +1,72 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +@unittest.skipIf(get_device_sm() < 90, "Test requires CUDA SM 90 or higher") +class TestFlashAttention3LocalAttn(CustomTestCase): + model = DEFAULT_MODEL_NAME_FOR_TEST_LOCAL_ATTENTION + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.90 + + @classmethod + def get_server_args(cls): + return [ + "--cuda-graph-max-bs", + "2", + "--attention-backend", + "fa3", + "--tp", + "4", + "--context-length", + "1000000", + ] + + @classmethod + def setUpClass(cls): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + env=os.environ, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=None, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_metrics.py b/test/srt/test_metrics.py new file mode 100644 index 000000000..ce85e0d8a --- /dev/null +++ b/test/srt/test_metrics.py @@ -0,0 +1,88 @@ +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEnableMetrics(CustomTestCase): + def test_metrics_enabled(self): + """Test that metrics endpoint returns data when enabled""" + process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_URL_FOR_TEST, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-metrics", "--cuda-graph-max-bs", 2], + ) + + try: + # Make some requests to generate some metrics + response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate") + self.assertEqual(response.status_code, 200) + + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + "stream": True, + }, + stream=True, + ) + for _ in response.iter_lines(decode_unicode=False): + pass + + # Get metrics + metrics_response = requests.get(f"{DEFAULT_URL_FOR_TEST}/metrics") + self.assertEqual(metrics_response.status_code, 200) + metrics_content = metrics_response.text + + print(f"metrics_content=\n{metrics_content}") + + # Verify essential metrics are present + essential_metrics = [ + "sglang:num_running_reqs", + "sglang:num_used_tokens", + "sglang:token_usage", + "sglang:gen_throughput", + "sglang:num_queue_reqs", + "sglang:num_grammar_queue_reqs", + "sglang:cache_hit_rate", + "sglang:spec_accept_length", + "sglang:prompt_tokens_total", + "sglang:generation_tokens_total", + "sglang:cached_tokens_total", + "sglang:num_requests_total", + "sglang:time_to_first_token_seconds", + "sglang:inter_token_latency_seconds", + "sglang:e2e_request_latency_seconds", + ] + + for metric in essential_metrics: + self.assertIn(metric, metrics_content, f"Missing metric: {metric}") + + # Verify model name label is present and correct + expected_model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + self.assertIn(f'model_name="{expected_model_name}"', metrics_content) + + # Verify metrics have values (not empty) + self.assertIn("_sum{", metrics_content) + self.assertIn("_count{", metrics_content) + self.assertIn("_bucket{", metrics_content) + + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_metrics_utils.py b/test/srt/test_metrics_utils.py new file mode 100644 index 000000000..4d33ad950 --- /dev/null +++ b/test/srt/test_metrics_utils.py @@ -0,0 +1,137 @@ +import unittest + +from sglang.srt.metrics.utils import generate_buckets, two_sides_exponential_buckets + + +class TestMetricsUtils(unittest.TestCase): + """Test cases for metrics utility functions.""" + + def test_two_sides_exponential_buckets_basic(self): + """Test basic functionality of two_sides_exponential_buckets.""" + # Test with simple parameters + count = 5 + buckets = two_sides_exponential_buckets(middle=10.0, base=2.0, count=count) + + # Should contain the middle value + self.assertIn(10.0, buckets) + + # Should be sorted + self.assertEqual(buckets, sorted(buckets)) + + # Should have unique values (no duplicates) + self.assertEqual(len(buckets), len(set(buckets))) + + # Should have reasonable number of buckets (not exactly count due to ceiling and deduplication) + self.assertGreaterEqual(len(buckets), 3) + self.assertLessEqual(len(buckets), count + 2) + + def test_two_sides_exponential_buckets_specific_values(self): + """Test specific values for two_sides_exponential_buckets.""" + buckets = two_sides_exponential_buckets(middle=100.0, base=2.0, count=4) + expected_values = [96.0, 98.0, 100.0, 102.0, 104.0] + self.assertEqual(buckets, expected_values) + + def test_two_sides_exponential_buckets_negative_values(self): + """Test two_sides_exponential_buckets with values that could go negative.""" + buckets = two_sides_exponential_buckets(middle=5.0, base=3.0, count=4) + + # Should not contain negative values (max(0, middle - distance)) + for bucket in buckets: + self.assertGreaterEqual(bucket, 0.0) + + # Should contain the middle value + self.assertIn(5.0, buckets) + + def test_two_sides_exponential_buckets_edge_cases(self): + """Test edge cases for two_sides_exponential_buckets.""" + # Count = 1 + buckets = two_sides_exponential_buckets(middle=10.0, base=2.0, count=1) + self.assertIn(10.0, buckets) + + # Very small middle value + buckets = two_sides_exponential_buckets(middle=0.1, base=2.0, count=2) + self.assertIn(0.1, buckets) + for bucket in buckets: + self.assertGreaterEqual(bucket, 0.0) + + def test_generate_buckets_default(self): + """Test generate_buckets with default rule.""" + default_buckets = [1.0, 5.0, 10.0, 50.0, 100.0] + + # Test with "default" rule + result = generate_buckets(["default"], default_buckets) + self.assertEqual(result, default_buckets) + + # Test with None (should default to "default") + result = generate_buckets(None, default_buckets) + self.assertEqual(result, default_buckets) + + # Test with empty (should default to "default") + result = generate_buckets(None, default_buckets) + self.assertEqual(result, default_buckets) + + def test_generate_buckets_tse(self): + """Test generate_buckets with tse (two sides exponential) rule.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with "tse" rule + result = generate_buckets(["tse", "10", "2.0", "4"], default_buckets) + + # Should return the same as calling two_sides_exponential_buckets directly + expected = two_sides_exponential_buckets(10.0, 2.0, 4) + self.assertEqual(result, expected) + + def test_generate_buckets_customer(self): + """Test generate_buckets with customer rule.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with "customer" rule + result = generate_buckets( + ["customer", "1.5", "3.2", "7.8", "15.6"], default_buckets + ) + expected = [1.5, 3.2, 7.8, 15.6] + self.assertEqual(result, expected) + + def test_generate_buckets_customer_with_integers(self): + """Test generate_buckets with customer rule using integer strings.""" + default_buckets = [1.0, 5.0, 10.0] + + # Test with integer strings + result = generate_buckets(["customer", "1", "5", "10", "50"], default_buckets) + expected = [1.0, 5.0, 10.0, 50.0] + self.assertEqual(result, expected) + + def test_generate_buckets_preserves_order_and_type(self): + """Test that generate_buckets preserves order and returns floats.""" + default_buckets = [1, 5, 10, 50, 100] # integers + + # Test default rule + result = generate_buckets(["default"], default_buckets) + self.assertEqual(result, default_buckets) + self.assertIsInstance(result, list) + + # Test customer rule with proper float conversion + result = generate_buckets( + ["customer", "100", "50", "10", "5", "1"], default_buckets + ) + expected = [1.0, 5.0, 10.0, 50.0, 100.0] + self.assertEqual(result, expected) + + # All values should be floats + for value in result: + self.assertIsInstance(value, float) + + def test_integration_tse_through_generate_buckets(self): + """Test integration of TSE buckets through generate_buckets function.""" + default_buckets = [1.0, 10.0, 100.0] + + # Generate buckets using both methods + direct_result = two_sides_exponential_buckets(50.0, 1.5, 6) + indirect_result = generate_buckets(["tse", "50.0", "1.5", "6"], default_buckets) + + # Results should be identical + self.assertEqual(direct_result, indirect_result) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla.py b/test/srt/test_mla.py new file mode 100644 index 000000000..af867797c --- /dev/null +++ b/test/srt/test_mla.py @@ -0,0 +1,52 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--enable-torch-compile", + "--torch-compile-max-bs", + "4", + "--chunked-prefill-size", + "256", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_deepseek_v3.py b/test/srt/test_mla_deepseek_v3.py new file mode 100644 index 000000000..634100fdb --- /dev/null +++ b/test/srt/test_mla_deepseek_v3.py @@ -0,0 +1,190 @@ +import os +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import is_cuda, is_hip, kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLADeepseekV3(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] + if is_cuda(): + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestMLADeepseekV3DisableFusedFunc(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_CI_DISABLE_MOE_FUSED_FUNC"] = "1" + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code", "--chunked-prefill-size", "256"] + if is_cuda(): + other_args.extend(["--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +@unittest.skipIf(is_hip(), "FA is not available.") +class TestMLADeepseekV3Fa3Fp8Kvcache(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--chunked-prefill-size", + "256", + "--kv-cache-dtype", + "fp8_e4m3", + ] + if is_cuda(): + other_args.extend(["--attention-backend", "fa3"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_flashinfer.py b/test/srt/test_mla_flashinfer.py new file mode 100644 index 000000000..f72aef5a5 --- /dev/null +++ b/test/srt/test_mla_flashinfer.py @@ -0,0 +1,124 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestFlashinferMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--cuda-graph-max-bs", + "4", + "--attention-backend", + "flashinfer", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.615) + + +class TestFlashinferMLAMTP(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "4", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--attention-backend", + "flashinfer", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + print(f"{server_info=}") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_fp8.py b/test/srt/test_mla_fp8.py new file mode 100644 index 000000000..a2fac9883 --- /dev/null +++ b/test/srt/test_mla_fp8.py @@ -0,0 +1,49 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_FP8_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--kv-cache-dtype", + "fp8_e5m2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.8 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_int8_deepseek_v3.py b/test/srt/test_mla_int8_deepseek_v3.py new file mode 100644 index 000000000..519cb0554 --- /dev/null +++ b/test/srt/test_mla_int8_deepseek_v3.py @@ -0,0 +1,214 @@ +import unittest +from types import SimpleNamespace + +import requests +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestMLADeepseekV3ChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreaterEqual(metrics["accuracy"], 0.61) + + +class TestDeepseekV3MTPChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-draft-model-path", + "sgl-project/sglang-ci-dsv3-channel-int8-test-NextN", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestMLADeepseekV3BlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend(["--enable-torch-compile", "--cuda-graph-max-bs", "2"]) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.62) + + +class TestDeepseekV3MTPBlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "sgl-project/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--disable-radix", + "--enable-torch-compile", + "--torch-compile-max-bs", + "1", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(metrics) + + self.assertGreater(metrics["accuracy"], 0.60) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mla_tp.py b/test/srt/test_mla_tp.py new file mode 100644 index 000000000..e957cf2de --- /dev/null +++ b/test/srt/test_mla_tp.py @@ -0,0 +1,66 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestDeepseekTP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + ["--tp", "2", "--enable-torch-compile", "--cuda-graph-max-bs", "2"] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreater(metrics["accuracy"], 0.62) + + def test_gsm8k_bs1(self): + # test torch compile accuracy for bs=1 + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=10, + max_new_tokens=512, + parallel=1, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + self.assertGreater(metrics["accuracy"], 0.62) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_modelopt.py b/test/srt/test_modelopt.py new file mode 100644 index 000000000..ef6a959ec --- /dev/null +++ b/test/srt/test_modelopt.py @@ -0,0 +1,58 @@ +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8, + DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8_REVISION, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestEvalFP8ModelOptQuantAccuracy(CustomTestCase): + + def _run_test(self, model, other_args, expected_score): + base_url = DEFAULT_URL_FOR_TEST + other_args = other_args or [] + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], expected_score) + finally: + kill_process_tree(process.pid) + + @unittest.skipIf( + torch.version.hip is not None, "modelopt quantization unsupported on ROCm" + ) + def test_mmlu_offline_only(self): + """Test with offline quantization only.""" + self._run_test( + model=DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8, + other_args=[ + "--revision", + DEFAULT_MODEL_NAME_FOR_MODELOPT_QUANT_ACCURACY_TEST_FP8_REVISION, + ], + expected_score=0.64, + ) diff --git a/test/srt/test_modelopt_fp8kvcache.py b/test/srt/test_modelopt_fp8kvcache.py new file mode 100644 index 000000000..a4704c239 --- /dev/null +++ b/test/srt/test_modelopt_fp8kvcache.py @@ -0,0 +1,30 @@ +import unittest + +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod + +from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptFp8Config, + ModelOptFp8KVCacheMethod, +) +from sglang.test.test_utils import CustomTestCase + + +class TestModelOptFp8KVCacheMethod(CustomTestCase): + def test_kv_cache_method_initialization(self): + """Test that ModelOptFp8KVCacheMethod can be instantiated and + inherits from BaseKVCacheMethod.""" + # Create a ModelOptFp8Config object + quant_config = ModelOptFp8Config(is_checkpoint_fp8_serialized=True) + + # Instantiate the KV cache method + kv_cache_method = ModelOptFp8KVCacheMethod(quant_config) + + # Check inheritance + self.assertIsInstance(kv_cache_method, BaseKVCacheMethod) + + # Check that the quant_config is stored + self.assertEqual(kv_cache_method.quant_config, quant_config) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_models_from_modelscope.py b/test/srt/test_models_from_modelscope.py new file mode 100644 index 000000000..dacaae30a --- /dev/null +++ b/test/srt/test_models_from_modelscope.py @@ -0,0 +1,40 @@ +import os +import shutil +import subprocess +import unittest +from unittest import mock + +from sglang.srt.utils import prepare_model_and_tokenizer +from sglang.test.test_utils import CustomTestCase + + +class TestDownloadFromModelScope(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = "iic/nlp_lstmcrf_word-segmentation_chinese-news" + stat, output = subprocess.getstatusoutput("pip install modelscope") + + cls.with_modelscope_environ = {k: v for k, v in os.environ.items()} + cls.with_modelscope_environ["SGLANG_USE_MODELSCOPE"] = "True" + + @classmethod + def tearDownClass(cls): + pass + + def test_prepare_model_and_tokenizer(self): + from modelscope.utils.file_utils import get_model_cache_root + + model_cache_root = get_model_cache_root() + if os.path.exists(model_cache_root): + shutil.rmtree(model_cache_root) + with mock.patch.dict(os.environ, self.with_modelscope_environ, clear=True): + model_path, tokenizer_path = prepare_model_and_tokenizer( + self.model, self.model + ) + assert os.path.exists(os.path.join(model_path, "pytorch_model.bin")) + assert os.path.exists(os.path.join(tokenizer_path, "config.json")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_moe_eval_accuracy_large.py b/test/srt/test_moe_eval_accuracy_large.py new file mode 100644 index 000000000..26bbd247e --- /dev/null +++ b/test/srt/test_moe_eval_accuracy_large.py @@ -0,0 +1,94 @@ +""" +Usage: +python -m unittest test_moe_eval_accuracy_large.TestMoEEvalAccuracyLarge.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MOE_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + + +class TestMoEEvalAccuracyLarge(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MOE_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--log-level-http", + "warning", + "--tp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=5000, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.62) + + if is_in_ci(): + write_github_step_summary(f"### test_mmlu\n" f'{metrics["score"]=:.4f}\n') + + def test_human_eval(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="humaneval", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.40) + + if is_in_ci(): + write_github_step_summary( + f"### test_human_eval\n" f'{metrics["score"]=:.4f}\n' + ) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.61) + + if is_in_ci(): + write_github_step_summary( + f"### test_mgsm_en\n" f'{metrics["score"]=:.4f}\n' + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_mscclpp.py b/test/srt/test_mscclpp.py new file mode 100644 index 000000000..894598b3d --- /dev/null +++ b/test/srt/test_mscclpp.py @@ -0,0 +1,205 @@ +"""For Now, MSCCL is only supported on TP16 and TP8 case + +if [[ $RANK -eq 0 ]]; then + ray start --block --head --port=6379 & + python3 test_mscclpp.py; +else + ray start --block --address=${MASTER_ADDR}:6379; +fi +""" + +import itertools +import os +import random +import socket +import unittest +from contextlib import contextmanager, nullcontext +from typing import Any, List, Optional, Union + +import ray +import torch +import torch.distributed as dist +from torch.distributed import ProcessGroup, ReduceOp + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.communication_op import ( # noqa + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.custom_all_reduce import ( + CustomAllreduce, +) +from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator +from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, + set_custom_all_reduce, + set_mscclpp_all_reduce, +) +from sglang.srt.distributed.utils import StatelessProcessGroup +from sglang.test.test_utils import CustomTestCase + + +def get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, + master_addr: str, + cls: Any, + test_target: Any, +) -> None: + + # Using ray helps debugging the error when it failed + # as compared to multiprocessing. + # NOTE: We need to set working_dir for distributed tests, + # otherwise we may get import errors on ray workers + + ray.init(log_to_driver=True) + + distributed_init_port = get_open_port() + refs = [] + for rank in range(world_size): + refs.append( + test_target.remote( + cls, world_size, master_addr, rank, distributed_init_port + ) + ) + ray.get(refs) + + ray.shutdown() + + +class TestMSCCLAllReduce(CustomTestCase): + @classmethod + def setUpClass(cls): + random.seed(42) + # 1KB to 1MB + cls.test_sizes = [512, 4096, 32768, 262144, 524288] + cls.world_sizes = [8] + TEST_TP16 = int(os.getenv("SGL_MSCCLPP_TEST_TP16", "0")) + if TEST_TP16: + cls.world_sizes = [16] + cls.test_loop = 10 + + def test_graph_allreduce(self): + TEST_MASTER_ADDR = os.getenv("SGL_MSCCLPP_TEST_MASTER_ADDR", "localhost") + for world_size in self.world_sizes: + if world_size not in [8, 16]: + continue + multi_process_parallel( + world_size, TEST_MASTER_ADDR, self, self.graph_allreduce + ) + + def test_eager_allreduce(self): + TEST_MASTER_ADDR = os.getenv("SGL_MSCCLPP_TEST_MASTER_ADDR", "localhost") + for world_size in self.world_sizes: + if world_size not in [8, 16]: + continue + multi_process_parallel( + world_size, TEST_MASTER_ADDR, self, self.eager_allreduce + ) + + @ray.remote(num_gpus=1, max_calls=1) + def graph_allreduce(self, world_size, master_addr, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://{master_addr}:{distributed_init_port}" + set_mscclpp_all_reduce(True) + set_custom_all_reduce(False) + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank % torch.cuda.device_count(), + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + # A small all_reduce for warmup. + # this is needed because device communicators might be created lazily + # (e.g. NCCL). This will ensure that the communicator is initialized + # before any communication happens, so that this group can be used for + # graph capture immediately. + data = torch.zeros(1) + data = data.to(device=device) + torch.distributed.all_reduce(data, group=group) + torch.cuda.synchronize() + del data + + for sz in self.test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.test_loop): + with graph_capture() as graph_capture_context: + # use integers so result matches NCCL exactly + inp1 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + inp2 = torch.randint( + 1, + 16, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph( + graph, stream=graph_capture_context.stream + ): + out1 = tensor_model_parallel_all_reduce(inp1) + # the input buffer is immediately modified to test + # synchronization + dist.all_reduce(inp1, group=group) + out2 = tensor_model_parallel_all_reduce(inp2) + dist.all_reduce(inp2, group=group) + graph.replay() + torch.testing.assert_close(out1, inp1) + torch.testing.assert_close(out2, inp2) + + @ray.remote(num_gpus=1, max_calls=1) + def eager_allreduce(self, world_size, master_addr, rank, distributed_init_port): + del os.environ["CUDA_VISIBLE_DEVICES"] + device = torch.device(f"cuda:{rank % torch.cuda.device_count()}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://{master_addr}:{distributed_init_port}" + set_mscclpp_all_reduce(True) + set_custom_all_reduce(False) + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + for sz in self.test_sizes: + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + for _ in range(self.test_loop): + inp1 = torch.randint( + 1, 16, (sz,), dtype=dtype, device=torch.cuda.current_device() + ) + out1 = tensor_model_parallel_all_reduce(inp1) + dist.all_reduce(inp1, group=group) + torch.testing.assert_close(out1, inp1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_multi_instance_release_memory_occupation.py b/test/srt/test_multi_instance_release_memory_occupation.py new file mode 100644 index 000000000..e4e8d9081 --- /dev/null +++ b/test/srt/test_multi_instance_release_memory_occupation.py @@ -0,0 +1,250 @@ +import multiprocessing +import os +import subprocess +import traceback +import unittest +from multiprocessing import Process +from typing import Iterable, Tuple + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import init_device_mesh +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.engine import Engine as SglangEngine +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + CustomTestCase, + find_available_port, +) + +TEST_SUITE = dict( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + mem_fraction_static=0.85, + dp_size=2, + tp_size=2, +) + + +class EngineWrapper: + """ + A wrapper around Sglang engine to mock multi instance cases such as RL traing. + + """ + + def __init__( + self, model_path, random_seed, mem_fraction_static, device_mesh_cpu, base_gpu_id + ): + self._device_mesh_cpu = device_mesh_cpu + self._tp_rank = device_mesh_cpu.get_local_rank() + self._rank = device_mesh_cpu.get_rank() + self._tp_size = device_mesh_cpu.size() + tp_size_per_node = self._tp_size + node_rank = self._tp_rank // tp_size_per_node + first_rank_in_node = self._tp_rank % tp_size_per_node == 0 + engine_kwargs = dict( + model_path=model_path, + random_seed=random_seed, + mem_fraction_static=mem_fraction_static, + base_gpu_id=base_gpu_id, + enable_memory_saver=True, + tp_size=self._tp_size, + node_rank=node_rank, + nnodes=1, + ) + self._engine = None + if first_rank_in_node: + os.environ["SGLANG_BLOCK_NONZERO_RANK_CHILDREN"] = "0" + self._engine = SglangEngine(**engine_kwargs) + + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def update_weights_from_tensor( + self, named_tensors: Iterable[Tuple[str, torch.Tensor]] + ): + if self._tp_rank == 0: + self._engine.update_weights_from_tensor(list(named_tensors)) + self._engine.flush_cache() + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def release_memory_occupation(self, tags): + if self._tp_rank == 0: + self._engine.release_memory_occupation(tags) + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def resume_memory_occupation(self, tags): + if self._tp_rank == 0: + self._engine.resume_memory_occupation(tags) + dist.barrier(group=self._device_mesh_cpu.get_group()) + + def shutdown(self): + if self._tp_rank == 0: + self._engine.shutdown() + dist.barrier(group=self._device_mesh_cpu.get_group()) + + +def get_gpu_memory_gb(gpu_id=0): + return torch.cuda.device_memory_used() / 1024**3 + + +class TestMultiInstanceReleaseMemoryOccupation(CustomTestCase): + @classmethod + def setUpClass(cls): + multiprocessing.set_start_method("spawn") + + def test_multi_instance_release_memory_occupation(self): + master_port = find_available_port(23456) + + dp_size = TEST_SUITE["dp_size"] + tp_size = TEST_SUITE["tp_size"] + world_size = dp_size * tp_size + processes = [] + output_reader, output_writer = multiprocessing.Pipe(duplex=False) + for rank in range(world_size): + p = Process( + target=_run_sglang_subprocess, + kwargs=dict( + rank=rank, + dp_size=dp_size, + tp_size=tp_size, + model_path=TEST_SUITE["model_path"], + master_port=master_port, + output_writer=output_writer, + mem_fraction_static=TEST_SUITE["mem_fraction_static"], + ), + ) + p.start() + processes.append(p) + + for _ in range(world_size): + self.assertTrue( + output_reader.recv(), f"Subprocess fail. Check the logs above." + ) + for p in processes: + p.join() + + +def _run_sglang_subprocess( + rank: int, + dp_size: int, + tp_size: int, + model_path: str, + master_port: int, + output_writer, + mem_fraction_static: float, +): + engine = None + try: + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = str(master_port) + dist.init_process_group( + rank=rank, + device_id=torch.device(f"cuda:{rank}"), + world_size=dp_size * tp_size, + ) + torch.cuda.set_device(rank) + + base_gpu_id = rank // tp_size * tp_size + mesh_kwargs = dict( + mesh_shape=(dp_size, tp_size, 1), mesh_dim_names=["dp", "tp", "pp"] + ) + inference_device_mesh_device = init_device_mesh("cuda", **mesh_kwargs) + inference_device_mesh_cpu = init_device_mesh("cpu", **mesh_kwargs) + print( + f"subprocess[{rank=},{base_gpu_id=},{rank=},{tp_size=}] {inference_device_mesh_device=} {inference_device_mesh_cpu=}" + ) + + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before starting Engine: {_mem_usage}") + + engine = EngineWrapper( + model_path=model_path, + random_seed=42, + mem_fraction_static=mem_fraction_static, + device_mesh_cpu=inference_device_mesh_cpu["tp"], + base_gpu_id=base_gpu_id, + ) + print(f"subprocess[{rank=}] {engine=}", flush=True) + + # 1 - release kv cache + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before releasing Sgl KV cache: {_mem_usage}") + engine.release_memory_occupation(tags=["kv_cache"]) + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing KV cache must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 2 - release sglang weights + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage before releasing Sgl weights: {_mem_usage}") + engine.release_memory_occupation(tags=["weights"]) + + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing weights must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 3 - load hf model + _mem_usage = get_gpu_memory_gb(rank) + print( + f"GPU{rank} Memory usage after releasing Sgl weights and kv cache: {_mem_usage}" + ) + hf_model = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map=f"cuda:{rank}", + trust_remote_code=True, + ).cuda() + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage > _mem_usage + ), f"Memory usage after loading hf model must be increased! before: {_mem_usage} vs after: {_curr_usage}" + + # 4 - resume sglang weights and update the weights + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after loading hf model: {_mem_usage}") + engine.resume_memory_occupation(tags=["weights"]) + engine.update_weights_from_tensor( + named_tensors=list(hf_model.named_parameters()) + ) + + # 5 - release hf model + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after resuming Sgl weights: {_mem_usage}") + del hf_model + torch.cuda.empty_cache() + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage < _mem_usage + ), f"Memory usage after releasing hf model must be reduced! before: {_mem_usage} vs after: {_curr_usage}" + + # 6 - resume slgang kv cache + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after releasing hf model: {_mem_usage}") + engine.resume_memory_occupation(tags=["kv_cache"]) + _curr_usage = get_gpu_memory_gb(rank) + assert ( + _curr_usage > _mem_usage + ), f"Memory usage after resuming kv cache must be increased! before: {_mem_usage} vs after: {_curr_usage}" + + # 7 - Final checking! + _mem_usage = get_gpu_memory_gb(rank) + print(f"GPU{rank} Memory usage after resuming Sgl KV cache: {_mem_usage}") + + execution_ok = True + except Exception as e: + print(f"subprocess[{rank=}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + if engine: + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_multi_tokenizer.py b/test/srt/test_multi_tokenizer.py new file mode 100644 index 000000000..182454e5e --- /dev/null +++ b/test/srt/test_multi_tokenizer.py @@ -0,0 +1,84 @@ +import unittest +from types import SimpleNamespace + +import sglang.srt.managers.io_struct as io_struct +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + auto_config_device, + get_benchmark_args, + is_in_ci, + popen_launch_server, + run_benchmark, + write_github_step_summary, +) + + +class TestMultiTokenizer(CustomTestCase): + # from test_hicache.py + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tokenizer-worker-num", + 8, + "--mem-fraction-static", + 0.7, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_multi_tokenizer_ttft(self): + # from test_bench_serving.py run_bench_serving + args = get_benchmark_args( + base_url=self.base_url, + dataset_name="random", + dataset_path="", + tokenizer=None, + num_prompts=100, + random_input_len=4096, + random_output_len=2048, + sharegpt_context_len=None, + request_rate=1, + disable_stream=False, + disable_ignore_eos=False, + seed=0, + device=auto_config_device(), + lora_name=None, + ) + res = run_benchmark(args) + if is_in_ci(): + write_github_step_summary( + f"### test_multi_tokenizer_ttft\n" + f"median_e2e_latency_ms: {res['median_e2e_latency_ms']:.2f} ms\n" + ) + self.assertLess(res["median_e2e_latency_ms"], 11000) + self.assertLess(res["median_ttft_ms"], 86) + self.assertLess(res["median_itl_ms"], 10) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_gsm8k_eval.py b/test/srt/test_nightly_gsm8k_eval.py new file mode 100644 index 000000000..a6b3070e4 --- /dev/null +++ b/test/srt/test_nightly_gsm8k_eval.py @@ -0,0 +1,171 @@ +import json +import os +import unittest +import warnings +from datetime import datetime +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +MODEL_SCORE_THRESHOLDS = { + "meta-llama/Llama-3.1-8B-Instruct": 0.82, + "mistralai/Mistral-7B-Instruct-v0.3": 0.58, + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, + "google/gemma-2-27b-it": 0.91, + "meta-llama/Llama-3.1-70B-Instruct": 0.95, + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64, + "Qwen/Qwen2-57B-A14B-Instruct": 0.86, + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83, + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, + "zai-org/GLM-4.5-Air-FP8": 0.75, + # The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression. + # The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green. + "neuralmagic/gemma-2-2b-it-FP8": 0.50, + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65, + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82, +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_tp2: + other_args.extend(["--tp", "2"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}") + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + process = popen_launch_server_wrapper(self.base_url, model, is_tp2) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_nightly_gsm8k_eval_amd.py b/test/srt/test_nightly_gsm8k_eval_amd.py new file mode 100644 index 000000000..d03684b99 --- /dev/null +++ b/test/srt/test_nightly_gsm8k_eval_amd.py @@ -0,0 +1,221 @@ +import json +import os +import unittest +import warnings +from datetime import datetime +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +MODEL_SCORE_THRESHOLDS = { + "meta-llama/Llama-3.1-8B-Instruct": 0.82, + "mistralai/Mistral-7B-Instruct-v0.3": 0.58, + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, + "meta-llama/Llama-3.1-70B-Instruct": 0.95, + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64, + "Qwen/Qwen2-57B-A14B-Instruct": 0.86, + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83, + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.86, + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65, + "google/gemma-2-27b-it": 0.91, + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, +} + +failing_models = { + "neuralmagic/gemma-2-2b-it-FP8", +} + + +def remove_failing_models(model_str): + models = model_str.split(",") + filtered = [m for m in models if m not in failing_models] + return ",".join(filtered) + + +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 +) +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = remove_failing_models( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 +) + +NO_MOE_PADDING_MODELS = {"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"} +DISABLE_HF_XET_MODELS = { + "Qwen/Qwen2-57B-A14B-Instruct", + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8", +} +TRITON_MOE_MODELS = { + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8", + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8", + "mistralai/Mixtral-8x7B-Instruct-v0.1", + "mistralai/Mistral-7B-Instruct-v0.3", +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_tp2: + other_args.extend(["--tp", "2"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}") + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + os.environ["SGLANG_MOE_PADDING"] = ( + "0" if model in NO_MOE_PADDING_MODELS else "1" + ) + os.environ["HF_HUB_DISABLE_XET"] = ( + "1" if model in DISABLE_HF_XET_MODELS else "0" + ) + os.environ["SGLANG_USE_AITER"] = ( + "0" if model in TRITON_MOE_MODELS else "1" + ) + + process = popen_launch_server_wrapper(self.base_url, model, is_tp2) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + # Allow retries, so flaky errors are avoided. + threshold = MODEL_SCORE_THRESHOLDS.get(model) + for attempt in range(3): + try: + metrics = run_eval(args) + score = metrics["score"] + if score >= threshold: + break + except Exception as e: + print(f"Attempt {attempt + 1} failed with error: {e}") + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_no_chunked_prefill.py b/test/srt/test_no_chunked_prefill.py new file mode 100644 index 000000000..59869ff59 --- /dev/null +++ b/test/srt/test_no_chunked_prefill.py @@ -0,0 +1,30 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + CustomTestCase, + run_bench_serving, + run_mmlu_test, +) + + +class TestNoChunkedPrefill(CustomTestCase): + + def test_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, enable_mixed_chunk=False, chunked_prefill_size=-1 + ) + + def test_no_chunked_prefill_without_radix_cache(self): + res = run_bench_serving( + model=DEFAULT_MODEL_NAME_FOR_TEST, + num_prompts=10, + request_rate=float("inf"), + other_server_args=["--disable-radix-cache", "--chunked-prefill-size", "-1"], + ) + + assert res["completed"] == 10 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_no_overlap_scheduler.py b/test/srt/test_no_overlap_scheduler.py new file mode 100644 index 000000000..d236819bc --- /dev/null +++ b/test/srt/test_no_overlap_scheduler.py @@ -0,0 +1,35 @@ +""" +Usage: +python3 -m unittest test_overlap_schedule.TestOverlapSchedule.test_radix_attention_chunked_prefill +python3 test_overlap_schedule.py +""" + +import unittest + +from sglang.test.test_utils import CustomTestCase, run_mmlu_test + + +class TestOverlapSchedule(CustomTestCase): + def test_no_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=32, disable_overlap=True + ) + + def test_no_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=True, chunked_prefill_size=-1, disable_overlap=True + ) + + def test_radix_attention_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=32, disable_overlap=True + ) + + def test_radix_attention_no_chunked_prefill(self): + run_mmlu_test( + disable_radix_cache=False, chunked_prefill_size=-1, disable_overlap=True + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_original_logprobs.py b/test/srt/test_original_logprobs.py new file mode 100644 index 000000000..ddcfe3d8e --- /dev/null +++ b/test/srt/test_original_logprobs.py @@ -0,0 +1,196 @@ +"""Test original log probability alignment between SGLang and Hugging Face. + +This test suite verifies the correctness of the `origin_logprobs` output (temperature=1) +and the `logprobs` output (temperature=0.5) in SGLang by comparing it against +raw logit-based probabilities computed directly from a reference Hugging Face model. + +The test covers the following scenarios: +- Next-token prediction: Verifies that the log probability of the next token from + SGLang matches the Hugging Face model. +- Top-k logprobs: Ensures that the top-k original logprobs returned by SGLang are + consistent with Hugging Face outputs. +- Specified token IDs: Confirms that the original logprobs for specific token IDs + match the values computed from Hugging Face logits. +""" + +import os +import random +import unittest + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import AutoModelForCausalLM, AutoTokenizer + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + +# ------------------------- Configurable via env ------------------------- # +MODEL_ID = DEFAULT_SMALL_MODEL_NAME_FOR_TEST +PROMPTS = [ + "Hello, my name is", + "The future of AI is", + "The president of the United States is", + "The capital of France is ", +] +TOP_LOGPROBS_NUM = 50 +NUM_RANDOM_TOKEN_IDS = 10 +RTOL = 0.20 +ATOL = 0.00 +# ------------------------------------------------ + +torch.manual_seed(1234) +if torch.cuda.is_available(): + torch.cuda.manual_seed_all(1234) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + +class TestOriginalLogprob(unittest.TestCase): + def setUp(self): + # ----- HF side (float32 weights) ----- + self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, padding_side="right") + self.hf_model = AutoModelForCausalLM.from_pretrained( + MODEL_ID, torch_dtype=torch.float32, device_map="auto" + ) + + # Shared sampling parameters + self.sampling_params = { + "temperature": 0.5, # SGLang uses 0.5, but original logprobs are used 1.0 + "top_p": 1.0, + "top_k": 10, + "max_new_tokens": 1, + } + + # --------------------------------------------------------------------- + # Helper: compare one SGLang block (token_logprobs / top_logprobs / ids_logprobs) + # against a reference HF log‑prob vector. + # --------------------------------------------------------------------- + def assert_logprobs_block_equal( + self, + hf_log_probs: torch.Tensor, # [V] + token_log_probs: list, + top_log_probs: list, + ids_log_probs: list, + random_token_ids: list, + tag: str = "", + ): + vals, idxs, _ = zip(*token_log_probs) + sgl_vals = torch.tensor(vals, device=self.hf_model.device, dtype=torch.float32) + sgl_idxs = torch.tensor(idxs, device=self.hf_model.device, dtype=torch.long) + hf_vals = hf_log_probs[sgl_idxs] + + self.assertTrue( + torch.allclose(hf_vals, sgl_vals, rtol=RTOL, atol=ATOL), + msg=f"[{tag}] token‑level mismatch at indices {sgl_idxs.tolist()}", + ) + + hf_topk, _ = torch.topk(hf_log_probs, k=TOP_LOGPROBS_NUM, dim=-1) + + sgl_topk = torch.tensor( + [float(t[0]) for t in top_log_probs[0] if t and t[0] is not None][ + :TOP_LOGPROBS_NUM + ], + dtype=torch.float32, + device=self.hf_model.device, + ) + + k = min(hf_topk.numel(), sgl_topk.numel()) + self.assertTrue( + torch.allclose(hf_topk[:k], sgl_topk[:k], rtol=RTOL, atol=ATOL), + msg=f"[{tag}] top‑k mismatch", + ) + + indices = torch.tensor( + random_token_ids, dtype=torch.long, device=hf_log_probs.device + ) + + hf_token_ids = hf_log_probs[indices] + + sgl_token_ids = torch.tensor( + [v for v, _, _ in ids_log_probs[0]], + device=self.hf_model.device, + dtype=torch.float32, + ) + self.assertTrue( + torch.allclose(hf_token_ids, sgl_token_ids, rtol=RTOL, atol=ATOL), + msg=f"[{tag}] token‑IDs mismatch", + ) + + # Optional: print max abs diff for quick diagnostics + max_diff = torch.max(torch.abs(hf_vals - sgl_vals)).item() + print(f"[{tag}] max|diff| token‑level = {max_diff:.4f}") + + def test_logprob_match(self): + vocab_size = self.tokenizer.vocab_size + + for env_val in ["True", "False"]: + with self.subTest(return_original_logprob=env_val): + os.environ["RETURN_ORIGINAL_LOGPROB"] = env_val + + # ----- SGLang side ----- + sgl_engine = sgl.Engine( + model_path=MODEL_ID, + skip_tokenizer_init=True, + trust_remote_code=True, + mem_fraction_static=0.60, + ) + + for prompt in PROMPTS: + random_token_ids = sorted( + random.sample(range(vocab_size), NUM_RANDOM_TOKEN_IDS) + ) + + enc = self.tokenizer(prompt, return_tensors="pt") + input_ids = enc["input_ids"].to(self.hf_model.device) + attn_mask = enc["attention_mask"].to(self.hf_model.device) + + with torch.inference_mode(): + hf_out = self.hf_model( + input_ids=input_ids, + attention_mask=attn_mask, + return_dict=True, + ) + logits = hf_out.logits[:, -1, :] # [1, V] + hf_log_probs = F.log_softmax( + logits.float() / self.sampling_params["temperature"], dim=-1 + )[0] + hf_original_log_probs = F.log_softmax(logits.float(), dim=-1)[0] + + outputs = sgl_engine.generate( + input_ids=input_ids[0].tolist(), + sampling_params=self.sampling_params, + return_logprob=True, + top_logprobs_num=TOP_LOGPROBS_NUM, + token_ids_logprob=random_token_ids, + ) + + if isinstance(outputs, list): + outputs = outputs[0] + meta = outputs["meta_info"] + + # Check original logprobs only if enabled + if env_val.lower() == "true": + self.assert_logprobs_block_equal( + hf_log_probs=hf_original_log_probs, + token_log_probs=meta["output_token_logprobs"], + top_log_probs=meta["output_top_logprobs"], + ids_log_probs=meta["output_token_ids_logprobs"], + random_token_ids=random_token_ids, + tag=f"Original logprobs SGLang vs HF: {prompt} ({env_val})", + ) + else: + # Always check regular logprobs + self.assert_logprobs_block_equal( + hf_log_probs=hf_log_probs, + token_log_probs=meta["output_token_logprobs"], + top_log_probs=meta["output_top_logprobs"], + ids_log_probs=meta["output_token_ids_logprobs"], + random_token_ids=random_token_ids, + tag=f"logprobs SGLang vs HF: {prompt} ({env_val})", + ) + sgl_engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_page_size.py b/test/srt/test_page_size.py new file mode 100644 index 000000000..f56e1ed6b --- /dev/null +++ b/test/srt/test_page_size.py @@ -0,0 +1,47 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPageSize(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_DEBUG_MEMORY_POOL"] = "1" + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--page-size", 4, "--chunked-prefill-size", 128], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_patch_torch.py b/test/srt/test_patch_torch.py new file mode 100644 index 000000000..a2c04509e --- /dev/null +++ b/test/srt/test_patch_torch.py @@ -0,0 +1,133 @@ +import os +import traceback +import unittest +from typing import Dict, List + +import torch +import torch.multiprocessing as mp + +from sglang.srt.patch_torch import monkey_patch_torch_reductions + + +class TestReleaseMemoryOccupation(unittest.TestCase): + def test_monkey_patch_torch_reductions(self): + mp.set_start_method("spawn", force=True) + + for enable_patch in [False, True]: + for params in [ + # Same visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + ), + # Different visible devices + dict( + sender_info=dict( + visible_devices=[0, 1], + tensor_device=1, + ), + receiver_info=dict( + visible_devices=[1, 0], + # If enable patch, this should be fixed, and cuda:1 becomes cuda:0 + tensor_device=0 if enable_patch else 1, + ), + ), + ]: + with self.subTest(f"{enable_patch=} {params=}"): + self._test_monkey_patch_torch_reductions_core( + enable_patch=enable_patch, **params + ) + + def _test_monkey_patch_torch_reductions_core( + self, + sender_info: Dict, + receiver_info: Dict, + enable_patch: bool, + ): + print( + f'test_monkey_patch_torch_reductions_core {os.environ.get("CUDA_VISIBLE_DEVICES")=}' + ) + cuda_visible_devices_list: List[int] = [ + int(x) + for x in os.environ.get("CUDA_VISIBLE_DEVICES", "0,1,2,3,4,5,6,7").split( + "," + ) + ] + + processes = [] + output_reader, output_writer = mp.Pipe(duplex=False) + queue = mp.Queue() + for role, info in [ + ("sender", sender_info), + ("receiver", receiver_info), + ]: + os.environ["CUDA_VISIBLE_DEVICES"] = ",".join( + str(cuda_visible_devices_list[device]) + for device in info["visible_devices"] + ) + p = mp.Process( + target=_run_subprocess, + kwargs=dict( + role=role, + queue=queue, + output_writer=output_writer, + tensor_device=info["tensor_device"], + enable_patch=enable_patch, + ), + ) + p.start() + processes.append(p) + + for _ in range(len(processes)): + self.assertTrue( + output_reader.recv(), f"Subprocess has error, please see logs above." + ) + + for p in processes: + p.join() + + +def _run_subprocess( + role: str, queue: mp.Queue, output_writer, tensor_device: int, enable_patch: bool +): + print( + f'subprocess[{role}] start {os.environ.get("CUDA_VISIBLE_DEVICES")=}', + flush=True, + ) + + if enable_patch: + print(f"subprocess[{role}] execute monkey_patch_torch_reductions", flush=True) + monkey_patch_torch_reductions() + + try: + if role == "sender": + tensor = torch.tensor([1.0, 2.0], device=f"cuda:{tensor_device}") + print(f"sender queue.put {tensor=} {tensor.device=}") + queue.put(tensor) + assert queue.get() == "done" + elif role == "receiver": + tensor = queue.get() + print(f"receiver queue.get {tensor=} {tensor.device=}") + assert str(tensor.device) == f"cuda:{tensor_device}" + queue.put("done") + else: + raise NotImplementedError + + execution_ok = True + except Exception as e: + print(f"subprocess[{role}] has error: {e}", flush=True) + traceback.print_exc() + execution_ok = False + + output_writer.send(execution_ok) + output_writer.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_penalty.py b/test/srt/test_penalty.py new file mode 100644 index 000000000..bfbd2777a --- /dev/null +++ b/test/srt/test_penalty.py @@ -0,0 +1,95 @@ +import json +import random +import unittest +from concurrent.futures import ThreadPoolExecutor + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPenalty(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode(self, sampling_params): + return_logprob = True + top_logprobs_num = 5 + return_text = True + n = 1 + + response = requests.post( + self.base_url + "/generate", + json={ + # prompt that is supposed to generate < 32 tokens + "text": "<|start_header_id|>user<|end_header_id|>\n\nWhat is the answer for 1 + 1 = ?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "sampling_params": { + "max_new_tokens": 48, + "n": n, + **sampling_params, + }, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + self.assertEqual(response.status_code, 200) + print(json.dumps(response.json())) + print("=" * 100) + + def test_default_values(self): + self.run_decode({}) + + def test_frequency_penalty(self): + self.run_decode({"frequency_penalty": 2}) + + def test_min_new_tokens(self): + self.run_decode({"min_new_tokens": 16}) + + def test_presence_penalty(self): + self.run_decode({"presence_penalty": 2}) + + def test_penalty_mixed(self): + args = [ + {}, + {}, + {}, + {"frequency_penalty": 2}, + {"presence_penalty": 1}, + {"min_new_tokens": 16}, + {"frequency_penalty": 0.2}, + {"presence_penalty": 0.4}, + {"min_new_tokens": 8}, + {"frequency_penalty": 0.4, "presence_penalty": 0.8}, + {"frequency_penalty": 0.4, "min_new_tokens": 12}, + {"presence_penalty": 0.8, "min_new_tokens": 12}, + {"presence_penalty": -0.3, "frequency_penalty": 1.3, "min_new_tokens": 32}, + {"presence_penalty": 0.3, "frequency_penalty": -1.3, "min_new_tokens": 32}, + ] + random.shuffle(args * 5) + with ThreadPoolExecutor(8) as executor: + list(executor.map(self.run_decode, args)) + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/test/srt/test_pp_single_node.py b/test/srt/test_pp_single_node.py new file mode 100644 index 000000000..e333c2d4c --- /dev/null +++ b/test/srt/test_pp_single_node.py @@ -0,0 +1,322 @@ +""" +Usage: +python3 -m unittest test_pp_single_node.TestPPAccuracy.test_gsm8k +python3 -m unittest test_pp_single_node.TestQwenPPAccuracy.test_pp_consistency +python3 -m unittest test_pp_single_node.TestFixedBugs.test_chunked_prefill_with_small_bs +""" + +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.bench_one_batch_server import BenchArgs as OneBatchBenchArgs +from sglang.srt.server_args import ServerArgs +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_one_batch_server, +) + + +class TestPPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23333" + cls.process = popen_launch_server( + DEFAULT_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + 2, + "--pp-size", + 2, + "--chunked-prefill-size", + 256, + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + self.assertGreater(metrics["accuracy"], 0.74) + # Wait a little bit so that the memory check happens. + time.sleep(4) + + def test_logprob(self): + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 16, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "logprob_start_len": 0, + }, + ) + response_json = response.json() + input_token_logprobs = response_json["meta_info"]["input_token_logprobs"] + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"] + + assert len(input_token_logprobs) == 6 + assert len(output_token_logprobs) == 16 + assert len(output_top_logprobs) == 16 + + +class TestDPAttentionDP2PP2(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--pp-size", + "2", + "--enable-dp-attention", + "--dp", + "2", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mgsm_en(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreater(metrics["score"], 0.8) + + +class TestQwenPPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23334" # different ports to avoid conflicts + cls.model_name = "Qwen/Qwen3-8B" # replace with your Qwen Model if needed + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestQwenPPTieWeightsAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23335" # different ports to avoid conflicts + cls.model_name = ( + "Qwen/Qwen3-0.6B" # qwen3 < 8B all have tie_word_embeddings = True + ) + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.38) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestQwenMoePPAccuracy(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_url = "http://127.0.0.1:23336" # different ports to avoid conflicts + cls.model_name = "Qwen/Qwen3-30B-A3B" # replace with your Qwen Model if needed + + def run_gsm8k_test(self, pp_size): + process = popen_launch_server( + self.model_name, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--pp-size", + pp_size, + "--chunked-prefill-size", + 256, + ], + ) + + try: + args = SimpleNamespace( + num_shots=5, + data_path=None, + num_questions=200, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval_few_shot_gsm8k(args) + time.sleep(5) + return metrics + finally: + kill_process_tree(process.pid) + + def test_pp_consistency(self): + baseline = self.run_gsm8k_test(pp_size=1) + pp_metrics = self.run_gsm8k_test(pp_size=2) + + print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}") + + self.assertGreaterEqual(baseline["accuracy"], 0.74) + self.assertGreaterEqual( + pp_metrics["accuracy"], + baseline["accuracy"] - 0.02, + msg=( + f"PP accuracy dropped more than 1% compared to baseline. " + f"Baseline: {baseline['accuracy']:.2%}, PP: {pp_metrics['accuracy']:.2%}" + ), + ) + + +class TestFixedBugs(unittest.TestCase): + def test_chunked_prefill_with_small_bs(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + server_args = ServerArgs(model_path=model) + bench_args = OneBatchBenchArgs( + batch_size=(1,), + input_len=(1,), + output_len=(1,), + base_url=DEFAULT_URL_FOR_TEST, + ) + other_server_args = [ + "--tp-size", + 2, + "--pp-size", + 2, + "--chunked-prefill", + 256, + "--max-running-requests", + 2, + ] + run_bench_one_batch_server( + model, + DEFAULT_URL_FOR_TEST, + server_args, + bench_args, + other_server_args, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_pytorch_sampling_backend.py b/test/srt/test_pytorch_sampling_backend.py new file mode 100644 index 000000000..8a368310e --- /dev/null +++ b/test/srt/test_pytorch_sampling_backend.py @@ -0,0 +1,90 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestPyTorchSamplingBackend(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--sampling-backend", "pytorch", "--disable-radix-cache"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + temperature=0.1, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def test_greedy(self): + + first_text = None + + # ensure the answer is identical across single response + for _ in range(5): + response_single = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of Germany is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + text = response_single["text"] + if first_text is None: + first_text = text + + self.assertEqual(text, first_text) + + first_text = None + + response_batch = requests.post( + self.base_url + "/generate", + json={ + "text": ["The capital of Germany is"] * 10, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ).json() + + # ensure the answer is identical among the batch + for i in range(10): + text = response_batch[i]["text"] + if first_text is None: + first_text = text + self.assertEqual(text, first_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_quick_allreduce.py b/test/srt/test_quick_allreduce.py new file mode 100644 index 000000000..ed081255f --- /dev/null +++ b/test/srt/test_quick_allreduce.py @@ -0,0 +1,212 @@ +import os +import random +import socket +import unittest +from typing import Any + +import ray +import torch +import torch.distributed as dist + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.communication_op import ( # noqa + tensor_model_parallel_all_reduce, +) +from sglang.srt.distributed.device_communicators.quick_all_reduce import ( + qr_rocm_arch_available, +) +from sglang.srt.distributed.parallel_state import ( + get_tensor_model_parallel_group, + graph_capture, + initialize_model_parallel, +) +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(42) +random.seed(44) # keep the deterministic seed + + +def get_open_port() -> int: + # try ipv4 + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + except OSError: + # try ipv6 + with socket.socket(socket.AF_INET6, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +def multi_process_parallel( + world_size: int, cls: Any, test_target: Any, quant_mode: str +) -> None: + + # Using ray helps debugging the error when it failed + # as compared to multiprocessing. + # NOTE: We need to set working_dir for distributed tests, + # otherwise we may get import errors on ray workers + + ray.init(log_to_driver=True) + + distributed_init_port = get_open_port() + refs = [] + for rank in range(world_size): + refs.append( + test_target.remote(cls, world_size, rank, distributed_init_port, quant_mode) + ) + ray.get(refs) + + ray.shutdown() + + +class TestQuickAllReduce(CustomTestCase): + TEST_SIZES = [ + 2 * 1024 * 1024, + 4 * 1024 * 1024, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + 32 * 1024 * 1024, + ] + TEST_LOOP = 5 + # Too many configurations can lead to a test grid that is too large + # The tp takes too long to boot,let's just choose 4 out of 12 configurations + # WORLD_SIZES = [2, 4, 8] + # QUANT_MODE = ["FP", "INT8", "INT6", "INT4"] + QUANT_MODE_WORLD_SIZE_PART = [["FP", 8], ["INT4", 4], ["INT8", 2], ["INT6", 2]] + + @unittest.skipIf( + not qr_rocm_arch_available(), + "Only test Quick AllReduce on ROCm architectures >= gfx94*", + ) + def test_graph_allreduce(self): + for quant_mode_world_size_part in self.QUANT_MODE_WORLD_SIZE_PART: + quant_mode = quant_mode_world_size_part[0] + world_size = quant_mode_world_size_part[1] + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.graph_allreduce, quant_mode) + + @unittest.skipIf( + not qr_rocm_arch_available(), + "Only test Quick AllReduce on ROCm architectures >= gfx94*", + ) + def test_eager_allreduce(self): + for quant_mode_world_size_part in self.QUANT_MODE_WORLD_SIZE_PART: + quant_mode = quant_mode_world_size_part[0] + world_size = quant_mode_world_size_part[1] + if world_size > torch.cuda.device_count(): + continue + multi_process_parallel(world_size, self, self.eager_allreduce, quant_mode) + + @ray.remote(num_gpus=1, max_calls=1) + def graph_allreduce(self, world_size, rank, distributed_init_port, quant_mode): + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + os.environ["ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_mode + os.environ["ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0" + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + # A small all_reduce for warmup. + # this is needed because device communicators might be created lazily + # (e.g. NCCL). This will ensure that the communicator is initialized + # before any communication happens, so that this group can be used for + # graph capture immediately. + data = torch.zeros(1) + data = data.to(device=device) + torch.distributed.all_reduce(data, group=group) + torch.cuda.synchronize() + del data + + for sz in self.TEST_SIZES: + for dtype in [torch.float16, torch.bfloat16]: + for _ in range(self.TEST_LOOP): + with graph_capture() as graph_capture_context: + # use integers so result matches NCCL exactly + inp1 = torch.randint( + 1, + 23, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + inp2 = torch.randint( + -23, + 1, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph( + graph, stream=graph_capture_context.stream + ): + out1 = tensor_model_parallel_all_reduce(inp1) + # the input buffer is immediately modified to test + # synchronization + dist.all_reduce(inp1, group=group) + out2 = tensor_model_parallel_all_reduce(inp2) + dist.all_reduce(inp2, group=group) + graph.replay() + atol = 1.25 * world_size + rtol = 0.5 * world_size + for inp, out in [[inp1, out1], [inp2, out2]]: + torch.testing.assert_close(out, inp, atol=atol, rtol=rtol) + # try: + # torch.testing.assert_close(out, inp, atol=atol, rtol=rtol) + # except AssertionError as e: + # print("Max abs diff:", (out - inp).abs().max()) + # print("Max rel diff:", ((out - inp).abs() / inp.abs().clamp(min=1e-5)).max()) + + @ray.remote(num_gpus=1, max_calls=1) + def eager_allreduce(self, world_size, rank, distributed_init_port, quant_mode): + os.environ.pop("CUDA_VISIBLE_DEVICES", None) + os.environ["ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_mode + os.environ["ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0" + device = torch.device(f"cuda:{rank}") + torch.cuda.set_device(device) + distributed_init_method = f"tcp://localhost:{distributed_init_port}" + init_distributed_environment( + world_size=world_size, + rank=rank, + distributed_init_method=distributed_init_method, + local_rank=rank, + ) + initialize_model_parallel(tensor_model_parallel_size=world_size) + group = get_tensor_model_parallel_group().device_group + + for sz in self.TEST_SIZES: + for dtype in [torch.float16, torch.bfloat16]: + for _ in range(self.TEST_LOOP): + inp1 = torch.randint( + 1, + 23, + (sz,), + dtype=dtype, + device=torch.cuda.current_device(), + ) + out1 = tensor_model_parallel_all_reduce(inp1) + dist.all_reduce(inp1, group=group) + atol = 1.25 * world_size + rtol = 0.5 * world_size + torch.testing.assert_close(out1, inp1, atol=atol, rtol=rtol) + # try: + # torch.testing.assert_close(out1, inp1, atol=atol, rtol=rtol) + # except AssertionError as e: + # print("Max abs diff:", (out1 - inp1).abs().max()) + # print("Max rel diff:", ((out1 - inp1).abs() / inp1.abs().clamp(min=1e-5)).max()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_radix_attention.py b/test/srt/test_radix_attention.py new file mode 100644 index 000000000..66f948621 --- /dev/null +++ b/test/srt/test_radix_attention.py @@ -0,0 +1,136 @@ +import os +import random +import unittest + +import requests + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + kill_process_tree, + popen_launch_server, +) + + +def gen_radix_tree(num_nodes=400, chunk_len=256): + num0 = num_nodes // 2 + num1 = num_nodes - num0 + nodes = [{"input_ids": [37] * 117, "decode_len": 217}] + for _ in range(num0): + parent = random.choice(nodes) + unique_len = random.randint(0, chunk_len) + decode_len = random.randint(0, chunk_len) + token_id = random.randint(0, 32000) + child = { + "input_ids": parent["input_ids"] + [token_id] * unique_len, + "decode_len": decode_len, + } + nodes.append(child) + + while num1 > 0: + num_branch = random.randint(1, min(num1, 10)) + parent = random.choice(nodes) + for _ in range(num_branch): + unique_len = random.randint(0, chunk_len) + decode_len = random.randint(0, chunk_len) + token_id = random.randint(0, 32000) + child = { + "input_ids": parent["input_ids"] + [token_id] * unique_len, + "decode_len": decode_len, + } + nodes.append(child) + + num1 -= num_branch + + random.shuffle(nodes) + return nodes + + +def run_test(base_url, nodes): + data = { + "input_ids": [node["input_ids"] for node in nodes], + "sampling_params": [ + {"max_new_tokens": node["decode_len"], "temperature": 0} for node in nodes + ], + } + + res = requests.post(base_url + "/generate", json=data) + assert res.status_code == 200 + + +class TestRadixCacheFCFS(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "fcfs", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_radix_attention(self): + nodes = gen_radix_tree() + run_test(self.base_url, nodes) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestRadixCacheLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +class TestRadixCacheNonOverlapLPM(TestRadixCacheFCFS): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--disable-overlap-schedule", + "--chunked-prefill-size", + "128", + "--max-total-tokens", + "20000", + "--schedule-policy", + "lpm", + ], + ) + + +if __name__ == "__main__": + os.environ["SGLANG_TEST_RETRACT"] = "true" + unittest.main() diff --git a/test/srt/test_reasoning_parser.py b/test/srt/test_reasoning_parser.py new file mode 100644 index 000000000..7d3f2a139 --- /dev/null +++ b/test/srt/test_reasoning_parser.py @@ -0,0 +1,603 @@ +import unittest + +from sglang.srt.parser.reasoning_parser import ( + BaseReasoningFormatDetector, + DeepSeekR1Detector, + KimiDetector, + Qwen3Detector, + ReasoningParser, + StreamingParseResult, +) +from sglang.test.test_utils import CustomTestCase + + +class TestStreamingParseResult(CustomTestCase): + def test_init_default(self): + """Test default initialization of StreamingParseResult.""" + result = StreamingParseResult() + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + + def test_init_with_values(self): + """Test initialization with specific values.""" + result = StreamingParseResult("normal", "reasoning") + self.assertEqual(result.normal_text, "normal") + self.assertEqual(result.reasoning_text, "reasoning") + + +class TestBaseReasoningFormatDetector(CustomTestCase): + def setUp(self): + self.detector = BaseReasoningFormatDetector( + think_start_token="", + think_end_token="", + force_reasoning=False, + stream_reasoning=True, + ) + + def test_init(self): + """Test initialization of BaseReasoningFormatDetector.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertFalse(self.detector._in_reasoning) + self.assertTrue(self.detector.stream_reasoning) + self.assertEqual(self.detector._buffer, "") + self.assertFalse(self.detector.stripped_think_start) + + def test_detect_and_parse_normal_text(self): + """Test parsing normal text without reasoning.""" + text = "This is normal text" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + def test_detect_and_parse_with_start_token(self): + """Test parsing text starting with think token.""" + text = "This is reasoning" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This is reasoning") + self.assertEqual(result.normal_text, "") + + def test_detect_and_parse_complete_reasoning(self): + """Test parsing complete reasoning block.""" + text = "This is reasoningThis is normal" + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This is reasoning") + self.assertEqual(result.normal_text, "This is normal") + + def test_detect_and_parse_force_reasoning(self): + """Test forced reasoning mode.""" + detector = BaseReasoningFormatDetector( + "", "", force_reasoning=True + ) + text = "This should be reasoning" + result = detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "This should be reasoning") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_normal(self): + """Test streaming parse of normal text.""" + result = self.detector.parse_streaming_increment("Hello world") + self.assertEqual(result.normal_text, "Hello world") + self.assertEqual(result.reasoning_text, "") + + def test_parse_streaming_increment_partial_token(self): + """Test streaming parse with partial token.""" + # Test partial start token + result = self.detector.parse_streaming_increment("", "") + detector._in_reasoning = True + result = detector.parse_streaming_increment("") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + self.assertTrue(self.detector._in_reasoning) + self.assertTrue(self.detector.stripped_think_start) + + def test_parse_streaming_increment_reasoning_content(self): + """Test streaming parse of reasoning content.""" + # First add start token + self.detector.parse_streaming_increment("") + + # Then add reasoning content + result = self.detector.parse_streaming_increment("reasoning content") + self.assertEqual(result.reasoning_text, "reasoning content") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_end_token(self): + """Test streaming parse with end token.""" + # Start reasoning mode + self.detector.parse_streaming_increment("") + self.detector.parse_streaming_increment("reasoning") + + # End reasoning - the reasoning content accumulated in previous calls is cleared when end token is found + result = self.detector.parse_streaming_increment("normal text") + self.assertEqual(result.reasoning_text, "") # Buffer cleared, returns empty + self.assertEqual(result.normal_text, "normal text") + self.assertFalse(self.detector._in_reasoning) + + def test_parse_streaming_increment_no_stream_reasoning(self): + """Test streaming parse without streaming reasoning.""" + detector = BaseReasoningFormatDetector( + "", "", stream_reasoning=False + ) + + # Start reasoning mode + detector.parse_streaming_increment("") + + # Add reasoning content - should not return content + result = detector.parse_streaming_increment("reasoning content") + self.assertEqual(result.reasoning_text, "") + self.assertEqual(result.normal_text, "") + + def test_parse_streaming_increment_mixed_content(self): + """Test streaming parse with mixed content in one chunk.""" + result = self.detector.parse_streaming_increment( + "reasoningnormal" + ) + self.assertEqual(result.reasoning_text, "reasoning") + self.assertEqual(result.normal_text, "normal") + + +class TestDeepSeekR1Detector(CustomTestCase): + def setUp(self): + self.detector = DeepSeekR1Detector() + + def test_init(self): + """Test DeepSeekR1Detector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertTrue(self.detector._in_reasoning) # force_reasoning=True + self.assertTrue(self.detector.stream_reasoning) + + def test_init_no_stream_reasoning(self): + """Test DeepSeekR1Detector with stream_reasoning=False.""" + detector = DeepSeekR1Detector(stream_reasoning=False) + self.assertFalse(detector.stream_reasoning) + + def test_detect_and_parse_r1_format(self): + """Test parsing DeepSeek-R1 format.""" + text = "I need to think about this. The answer is 42." + result = self.detector.detect_and_parse(text) + # Should be treated as reasoning because force_reasoning=True + self.assertEqual( + result.reasoning_text, "I need to think about this. The answer is 42." + ) + self.assertEqual(result.normal_text, "") + + def test_detect_and_parse_with_end_token(self): + """Test parsing with end token.""" + text = "I think this is the answerThe final answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "I think this is the answer") + self.assertEqual(result.normal_text, "The final answer is 42.") + + def test_detect_and_parse_with_start_token(self): + """Test parsing deepseek-ai/DeepSeek-R1-0528 format, which generates the token.""" + text = "I need to think about this.The answer is 42." + result = self.detector.detect_and_parse(text) + # Should be treated as reasoning because force_reasoning=True + self.assertEqual(result.reasoning_text, "I need to think about this.") + self.assertEqual(result.normal_text, "The answer is 42.") + + +class TestQwen3Detector(CustomTestCase): + def setUp(self): + self.detector = Qwen3Detector() + + def test_init(self): + """Test Qwen3Detector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertFalse(self.detector._in_reasoning) # force_reasoning=False + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_qwen3_format(self): + """Test parsing Qwen3 format.""" + text = "Let me think about this problemThe answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Let me think about this problem") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_without_thinking(self): + """Test parsing without thinking (enable_thinking=False case).""" + text = "Direct answer without thinking." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + +class TestQwen3ForcedReasoningDetector(CustomTestCase): + def setUp(self): + self.detector = Qwen3Detector(force_reasoning=True) + + def test_init(self): + """Test Qwen3ForcedReasoningDetector initialization.""" + self.assertEqual(self.detector.think_start_token, "") + self.assertEqual(self.detector.think_end_token, "") + self.assertTrue(self.detector._in_reasoning) # force_reasoning=True + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_qwen3_forced_reasoning_format(self): + """Test parsing Qwen3-ForcedReasoning format (no start tag).""" + text = "I need to think about this step by step.The answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual( + result.reasoning_text, "I need to think about this step by step." + ) + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_with_start_token(self): + """Test parsing Qwen3-ForcedReasoning with optional start tag.""" + text = "I need to think about this.The answer is 42." + result = self.detector.detect_and_parse(text) + # Should work because base class logic handles both force_reasoning=True OR start token + self.assertEqual(result.reasoning_text, "I need to think about this.") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_streaming_qwen3_forced_reasoning_format(self): + """Test streaming parse of Qwen3-ForcedReasoning format.""" + # First chunk without start + result = self.detector.parse_streaming_increment("I need to") + self.assertEqual(result.reasoning_text, "I need to") + self.assertEqual(result.normal_text, "") + + # More reasoning content + result = self.detector.parse_streaming_increment(" think about this.") + self.assertEqual(result.reasoning_text, " think about this.") + self.assertEqual(result.normal_text, "") + + # End token with normal text + result = self.detector.parse_streaming_increment("The answer is 42.") + self.assertEqual(result.reasoning_text, "") # Buffer cleared + self.assertEqual(result.normal_text, "The answer is 42.") + + +class TestKimiDetector(CustomTestCase): + def setUp(self): + self.detector = KimiDetector() + + def test_init(self): + """Test KimiDetector initialization.""" + self.assertEqual(self.detector.think_start_token, "◁think▷") + self.assertEqual(self.detector.think_end_token, "◁/think▷") + self.assertFalse(self.detector._in_reasoning) + self.assertTrue(self.detector.stream_reasoning) + + def test_detect_and_parse_kimi_format(self): + """Test parsing Kimi format.""" + text = "◁think▷Let me consider this carefully◁/think▷The answer is 42." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.reasoning_text, "Let me consider this carefully") + self.assertEqual(result.normal_text, "The answer is 42.") + + def test_detect_and_parse_kimi_no_thinking(self): + """Test parsing Kimi format without thinking.""" + text = "Direct answer without thinking tokens." + result = self.detector.detect_and_parse(text) + self.assertEqual(result.normal_text, text) + self.assertEqual(result.reasoning_text, "") + + def test_streaming_kimi_format(self): + """Test streaming parse of Kimi format.""" + # Test partial token + result = self.detector.parse_streaming_increment("◁thi") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + + # Complete start token + result = self.detector.parse_streaming_increment("nk▷Start") + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "Start") + self.assertTrue(self.detector._in_reasoning) + + # Add reasoning content + result = self.detector.parse_streaming_increment("thinking...") + self.assertEqual(result.reasoning_text, "thinking...") + self.assertEqual(result.normal_text, "") + + # End token - reasoning content is cleared when end token is processed + result = self.detector.parse_streaming_increment("◁/think▷answer") + self.assertEqual(result.reasoning_text, "") # Buffer cleared + self.assertEqual(result.normal_text, "answer") + + +class TestReasoningParser(CustomTestCase): + def test_init_valid_model(self): + """Test initialization with valid model types.""" + parser = ReasoningParser("deepseek-r1") + self.assertIsInstance(parser.detector, DeepSeekR1Detector) + + parser = ReasoningParser("qwen3") + self.assertIsInstance(parser.detector, Qwen3Detector) + + parser = ReasoningParser("kimi") + self.assertIsInstance(parser.detector, KimiDetector) + + def test_init_invalid_model(self): + """Test initialization with invalid model type.""" + with self.assertRaises(ValueError) as context: + ReasoningParser("invalid-model") + self.assertIn("Unsupported model type", str(context.exception)) + + def test_init_no_model(self): + """Test initialization without model type.""" + with self.assertRaises(ValueError) as context: + ReasoningParser(None) + self.assertEqual(str(context.exception), "Model type must be specified") + + def test_parse_non_stream(self): + """Test non-streaming parsing.""" + parser = ReasoningParser("qwen3") + reasoning, normal = parser.parse_non_stream( + "Let me thinkThe answer is 42." + ) + self.assertEqual(reasoning, "Let me think") + self.assertEqual(normal, "The answer is 42.") + + def test_parse_stream_chunk(self): + """Test streaming chunk parsing.""" + parser = ReasoningParser("qwen3") + + # First chunk with start token + reasoning, normal = parser.parse_stream_chunk("") + self.assertEqual(reasoning, "") + self.assertEqual(normal, "") + + # Second chunk with reasoning content + reasoning, normal = parser.parse_stream_chunk("thinking...") + self.assertEqual(reasoning, "thinking...") + self.assertEqual(normal, "") + + # Third chunk with end token and normal text + reasoning, normal = parser.parse_stream_chunk("answer") + self.assertEqual(reasoning, "") # Buffer cleared when end token processed + self.assertEqual(normal, "answer") + + def test_case_insensitive_model_type(self): + """Test case insensitive model type matching.""" + parser1 = ReasoningParser("DeepSeek-R1") + parser2 = ReasoningParser("QWEN3") + parser3 = ReasoningParser("Kimi") + + self.assertIsInstance(parser1.detector, DeepSeekR1Detector) + self.assertIsInstance(parser2.detector, Qwen3Detector) + self.assertIsInstance(parser3.detector, KimiDetector) + + def test_stream_reasoning_parameter(self): + """Test stream_reasoning parameter is passed correctly.""" + parser = ReasoningParser("qwen3", stream_reasoning=False) + self.assertFalse(parser.detector.stream_reasoning) + + parser = ReasoningParser("qwen3", stream_reasoning=True) + self.assertTrue(parser.detector.stream_reasoning) + + +class TestIntegrationScenarios(CustomTestCase): + """Integration tests for realistic usage scenarios.""" + + def test_deepseek_r1_complete_response(self): + """Test complete DeepSeek-R1 response parsing.""" + parser = ReasoningParser("deepseek-r1") + text = "I need to solve this step by step. First, I'll analyze the problem. The given equation is x + 2 = 5. To solve for x, I subtract 2 from both sides: x = 5 - 2 = 3.The answer is x = 3." + + reasoning, normal = parser.parse_non_stream(text) + self.assertIn("step by step", reasoning) + self.assertIn( + "= 3", reasoning + ) # The reasoning contains "x = 5 - 2 = 3" which has "= 3" + self.assertEqual(normal, "The answer is x = 3.") + + def test_qwen3_streaming_scenario(self): + """Test Qwen3 streaming scenario.""" + parser = ReasoningParser("qwen3") + + chunks = [ + "", + "Let me analyze this problem.", + " I need to consider multiple factors.", + "", + "Based on my analysis, the solution is to use a different approach.", + ] + + all_reasoning = "" + all_normal = "" + + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("multiple factors", all_reasoning) + self.assertIn("different approach", all_normal) + + def test_kimi_streaming_scenario(self): + """Test Kimi streaming scenario.""" + parser = ReasoningParser("kimi") + chunks = [ + "◁thi", + "nk▷", + "Let me analyze this problem.", + " I need to consider multiple factors.", + "◁/th", + "ink▷", + "The answer is 42.", + ] + all_reasoning = "" + all_normal = "" + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("multiple factors", all_reasoning) + self.assertIn("42", all_normal) + + def test_empty_reasoning_blocks(self): + """Test handling of empty reasoning blocks.""" + parser = ReasoningParser("qwen3") + text = "Just the answer." + + reasoning, normal = parser.parse_non_stream(text) + self.assertEqual(reasoning, "") + self.assertEqual(normal, "Just the answer.") + + def test_qwen3_forced_reasoning_complete_response(self): + """Test complete Qwen3-ForcedReasoning response parsing.""" + parser = ReasoningParser("qwen3", force_reasoning=True) + text = "Let me solve this step by step. The equation is x + 2 = 5. Subtracting 2 from both sides gives x = 3.The solution is x = 3." + + reasoning, normal = parser.parse_non_stream(text) + self.assertIn("step by step", reasoning) + self.assertIn("x = 3", reasoning) + self.assertEqual(normal, "The solution is x = 3.") + + def test_qwen3_forced_reasoning_streaming_scenario(self): + """Test Qwen3-ForcedReasoning streaming scenario.""" + parser = ReasoningParser("qwen3", force_reasoning=True) + + chunks = [ + "I need to analyze", + " this problem carefully.", + " Let me break it down.", + "", + "The final answer is 42.", + ] + + all_reasoning = "" + all_normal = "" + + for chunk in chunks: + reasoning, normal = parser.parse_stream_chunk(chunk) + all_reasoning += reasoning + all_normal += normal + + self.assertIn("analyze", all_reasoning) + self.assertIn("break it down", all_reasoning) + self.assertIn("final answer", all_normal) + + +class TestBufferLossBugFix(CustomTestCase): + """Test cases for the buffer loss bug fix in parse_streaming_increment.""" + + def test_partial_end_tag_buffer_loss_bug(self): + """ + Test the bug where partial end tag fragments are lost when followed by normal text. + + Bug scenario: + 1. _in_reasoning is False + 2. new_text is "", "") + + # Step 1: Send partial end tag when not in reasoning mode + # This should be buffered since it could be start of "" + result1 = detector.parse_streaming_increment("", "") + + # Send partial start tag + result1 = detector.parse_streaming_increment("", "") + + # Enter reasoning mode + detector.parse_streaming_increment("") + detector.parse_streaming_increment("some reasoning") + + # Send partial end tag + result1 = detector.parse_streaming_increment("normal text") + self.assertEqual(result2.normal_text, "normal text") + # The reasoning text should be empty since buffer was cleared when end tag was processed + self.assertEqual(result2.reasoning_text, "") + + def test_multiple_partial_fragments(self): + """ + Test handling of multiple partial fragments that don't match any tokens. + """ + detector = BaseReasoningFormatDetector("", "") + + # Send multiple partial fragments + result1 = detector.parse_streaming_increment("<") + self.assertEqual(result1.normal_text, "") + self.assertEqual(result1.reasoning_text, "") + + result2 = detector.parse_streaming_increment("/") + self.assertEqual(result2.normal_text, "") + self.assertEqual(result2.reasoning_text, "") + + result3 = detector.parse_streaming_increment("random>") + self.assertEqual(result3.normal_text, "") + self.assertEqual(result3.reasoning_text, "") + + def test_edge_case_exact_token_match(self): + """ + Test edge case where buffer content exactly matches a token. + """ + detector = BaseReasoningFormatDetector("", "") + + # Build up the exact start token character by character + detector.parse_streaming_increment("<") + detector.parse_streaming_increment("t") + detector.parse_streaming_increment("h") + detector.parse_streaming_increment("i") + detector.parse_streaming_increment("n") + result = detector.parse_streaming_increment("k>") + + # Should enter reasoning mode + self.assertEqual(result.normal_text, "") + self.assertEqual(result.reasoning_text, "") + self.assertTrue(detector._in_reasoning) + self.assertTrue(detector.stripped_think_start) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_regex_constrained.py b/test/srt/test_regex_constrained.py new file mode 100644 index 000000000..b0fb49796 --- /dev/null +++ b/test/srt/test_regex_constrained.py @@ -0,0 +1,187 @@ +""" +python3 -m unittest test_regex_constrained.TestRegexConstrained.test_regex_generate_email +python3 -m unittest test_regex_constrained.TestRegexConstrained.test_regex_generate_greeting +python3 -m unittest test_regex_constrained.TestRegexConstrainedLLGuidance.test_regex_generate_email +python3 -m unittest test_regex_constrained.TestRegexConstrainedLLGuidance.test_regex_generate_greeting +""" + +import json +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def setup_class(cls, backend: str, disable_overlap: bool): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + other_args = [ + "--max-running-requests", + "10", + "--grammar-backend", + backend, + ] + + if disable_overlap: + other_args += ["--disable-overlap-schedule"] + + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + +class TestRegexConstrained(CustomTestCase): + @classmethod + def setUpClass(cls): + setup_class(cls, "xgrammar", disable_overlap=False) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + regex, + prompt, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompt, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 128, + "n": n, + "regex": regex, + }, + "stream": False, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + }, + ) + + ret = response.json() + print(json.dumps(ret, indent=2)) + print("=" * 100) + + if not isinstance(ret, list): + self.fail(f"Expected response to be a list, but got {type(ret)}") + + for item in ret: + text = item.get("text", "").strip() + if not text: + self.fail("Generated text is empty.") + + if not self.regex_match(text, regex): + self.fail(f"Text '{text}' does not match regex pattern.") + + def regex_match(self, text, pattern): + import re + + return re.match(pattern, text) is not None + + def test_regex_generate_email(self): + pattern = r"^user@example\.com$" + prompt = "Generate an email address:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_greeting(self): + pattern = r"^(Hello|Hi|Hey)$" + prompt = "Generate a greeting:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_number(self): + pattern = r"^\d{3}$" + prompt = "Generate a three-digit number:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_phone(self): + pattern = r"^\(\d{3}\) \d{3}-\d{4}$" + prompt = "Generate a phone number:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_date(self): + pattern = r"^2024-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$" + prompt = "Generate a date in YYYY-MM-DD format:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_hex_color(self): + pattern = r"^#[0-9A-F]{6}$" + prompt = "Generate a hex color code:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_complex_json(self): + pattern = r'^\{\s*"name"\s*:\s*"[a-zA-Z0-9 ]+"\s*,\s*"age"\s*:\s*[1-9][0-9]*\s*,\s*"city"\s*:\s*"[a-zA-Z0-9 ]+"\s*\}$' + prompt = "Generate a simple JSON with name, age, and city:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + def test_regex_generate_custom_log_format(self): + pattern = r"^\[2024-01-01T12:00:00Z\] INFO: System\.process - Operation [a-z]+ successfully$" + prompt = "Generate a log entry:" + + self.run_decode( + regex=pattern, + prompt=prompt, + n=3, + ) + + +class TestRegexConstrainedLLGuidance(TestRegexConstrained): + @classmethod + def setUpClass(cls): + setup_class(cls, "llguidance", disable_overlap=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_release_memory_occupation.py b/test/srt/test_release_memory_occupation.py new file mode 100644 index 000000000..35be029df --- /dev/null +++ b/test/srt/test_release_memory_occupation.py @@ -0,0 +1,328 @@ +"""Test memory release and resume operations for SGLang engine in hybrid RL training. + +This test suite evaluates the SGLang engine's memory management capabilities, focusing +on releasing and resuming memory occupation for KV cache and model weights. It simulates +an RL workflow where the SGLang engine acts as a rollout engine for experience collection. +The process involves initializing the engine, sending a small number of requests to simulate +rollout, releasing memory to mimic offloading during RL training, resuming memory occupation, +updating weights with a trained HuggingFace model, and verifying the updated weights. + +Detailed in our proposal (https://github.com/sgl-project/sglang/pull/7099), two test cases +are included: + +1. Basic Release and Resume: Uses a lower mem_fraction_static (0.6) to control memory allocation +and avoid OOM errors carefully. This test simulates a scenario without multi-stage memory management, +ensuring the engine can release and resume memory occupation while maintaining functionality after +weight updates. + +2. Multi-Stage Release and Resume: Employs a higher mem_fraction_static (0.85) to simulate higher +memory pressure, leveraging multi-stage memory management. It sequentially releases and resumes +KV cache and model weights, verifying memory deallocation and reallocation at each stage, and +ensuring correct weight updates and text generation. + +3. Tensor Parallel Tests: Tests memory release and resume operations with different tensor parallel +configurations (tp=1, tp=2) to ensure proper memory management in distributed settings. For different +data parallel size, we test it in verl. +""" + +import gc +import os +import time +import unittest + +import torch +from transformers import AutoModelForCausalLM + +import sglang as sgl +from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE, GPU_MEMORY_TYPE_WEIGHTS +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT, + CustomTestCase, +) + +# (temporarily) set to true to observe memory usage in nvidia-smi more clearly +_DEBUG_EXTRA = False + + +def get_gpu_memory_gb(): + return torch.cuda.device_memory_used() / 1024**3 + + +class TestReleaseMemoryOccupation(CustomTestCase): + def _setup_engine(self, model_name, mem_fraction_static=0.8, tp_size=1, ep_size=1): + """Common setup for engine and HF model.""" + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=mem_fraction_static, + tp_size=tp_size, + ep_size=ep_size, + # disable_cuda_graph=True, # for debugging only + ) + + return engine + + def _common_test_params(self): + """Common test parameters.""" + return { + "prompt": "Today is a sunny day and I like", + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + "expect_output_before_update_weights": " to spend it outdoors. I decided to", + "expect_output_after_update_weights": " to go for a walk. I like", + "prompt_moe": "The weather is nice today, and I want to", + "sampling_params_moe": {"temperature": 0, "max_new_tokens": 16}, + "expect_output_before_update_weights_moe": " go to the park. I have a picnic basket, a book, and a", + "expect_output_after_update_weights_moe": " go to the park. I have a lot of things to do, but I", + } + + def _test_initial_generation( + self, engine, prompt, sampling_params, expect_output_before_update_weights + ): + """Test initial generation and memory allocation.""" + print("generate (#1)") + outputs = engine.generate(prompt, sampling_params)["text"] + self.assertEqual(outputs, expect_output_before_update_weights) + + if _DEBUG_EXTRA: + time.sleep(3) + + def test_release_and_resume_occupation(self): + # Without multi-stage release and resume, we need to carefully control the memory fraction to avoid OOM + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + assert ( + torch.cuda.device_count() >= 2 + ), "Need at least 2 GPUs for tensor parallel tests" + + for tp_size in [1, 2]: + + print(f"Testing tp_size={tp_size} for test_release_and_resume_occupation") + engine = self._setup_engine( + model_name=model_name, mem_fraction_static=0.6, tp_size=tp_size + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt"], + params["sampling_params"], + params["expect_output_before_update_weights"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation() + gpu_memory_usage_after_release = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release, + gpu_memory_usage_before_release, + ) + + print( + f"Release took {time.perf_counter() - t:.2f}s, memory: {gpu_memory_usage_before_release:.1f} GB → {gpu_memory_usage_after_release:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + engine.resume_memory_occupation() + print( + f"Resume took {time.perf_counter() - t:.2f}s, memory: {get_gpu_memory_gb():.1f} GB" + ) + + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + + print("generate (#2)") + outputs = engine.generate(params["prompt"], params["sampling_params"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights"]) + engine.shutdown() + + def test_multi_stage_release_and_resume(self): + # With multi-stage release and resume, we can set the memory fraction to 0.85 without concern of OOM + model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + for tp_size in [1, 2]: + if tp_size == 2 and torch.cuda.device_count() < 2: + continue + + print(f"Testing tp_size={tp_size} for test_multi_stage_release_and_resume") + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=0.85, # Higher memory pressure + tp_size=tp_size, + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt"], + params["sampling_params"], + params["expect_output_before_update_weights"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release_kv_cache = get_gpu_memory_gb() + engine.release_memory_occupation(tags=[GPU_MEMORY_TYPE_KV_CACHE]) + + gpu_memory_usage_after_release_kv_cache = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release_kv_cache, + gpu_memory_usage_before_release_kv_cache, + ) + engine.release_memory_occupation(tags=[GPU_MEMORY_TYPE_WEIGHTS]) + + gpu_memory_usage_after_release_weights = get_gpu_memory_gb() + + self.assertLess( + gpu_memory_usage_after_release_weights, + gpu_memory_usage_after_release_kv_cache, + ) + + print(f"Release took {time.perf_counter() - t:.2f}s") + print( + f"Memory: {gpu_memory_usage_before_release_kv_cache:.1f} → {gpu_memory_usage_after_release_kv_cache:.1f} → {gpu_memory_usage_after_release_weights:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + gpu_memory_usage_before_resume_weights = get_gpu_memory_gb() + + # gpu_memory_usage_after_release_weights and gpu_memory_usage_before_resume_weights should be close + + self.assertAlmostEqual( + gpu_memory_usage_after_release_weights, + gpu_memory_usage_before_resume_weights, + delta=3.0, + ) + print(f"Resume weights took {time.perf_counter() - t:.2f}s") + + engine.resume_memory_occupation(tags=[GPU_MEMORY_TYPE_WEIGHTS]) + gpu_memory_usage_after_resume_weights = get_gpu_memory_gb() + + self.assertGreater( + gpu_memory_usage_after_resume_weights, + gpu_memory_usage_before_resume_weights, + ) + + # Update weights from a trained model to serving engine, and then destroy the trained model + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + gpu_memory_usage_after_loaded_hf_model = get_gpu_memory_gb() + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + engine.resume_memory_occupation(tags=[GPU_MEMORY_TYPE_KV_CACHE]) + + gpu_memory_usage_after_resume_kv_cache = get_gpu_memory_gb() + self.assertGreater( + gpu_memory_usage_after_resume_kv_cache, + gpu_memory_usage_after_resume_weights, + ) + + print(f"Resume + update took {time.perf_counter() - t:.2f}s") + print( + f"Memory: {gpu_memory_usage_before_resume_weights:.1f} → {gpu_memory_usage_after_resume_weights:.1f} → {gpu_memory_usage_after_loaded_hf_model:.1f} → {gpu_memory_usage_after_resume_kv_cache:.1f} GB" + ) + + print("generate (#2)") + outputs = engine.generate(params["prompt"], params["sampling_params"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights"]) + engine.shutdown() + + def test_moe_model_release_and_resume(self): + # Test with MoE model + model_name = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_CHAT + + tp_size = ep_size = 2 + + print( + f"Testing tp_size={tp_size} and ep_size={ep_size} for test_moe_model_release_and_resume" + ) + engine = sgl.Engine( + model_path=model_name, + random_seed=42, + enable_memory_saver=True, + mem_fraction_static=0.5, + tp_size=tp_size, + ep_size=ep_size, + ) + params = self._common_test_params() + + self._test_initial_generation( + engine, + params["prompt_moe"], + params["sampling_params_moe"], + params["expect_output_before_update_weights_moe"], + ) + + t = time.perf_counter() + gpu_memory_usage_before_release = get_gpu_memory_gb() + engine.release_memory_occupation() + gpu_memory_usage_after_release = get_gpu_memory_gb() + self.assertLess( + gpu_memory_usage_after_release, + gpu_memory_usage_before_release, + ) + + print( + f"Release took {time.perf_counter() - t:.2f}s, memory: {gpu_memory_usage_before_release:.1f} GB → {gpu_memory_usage_after_release:.1f} GB" + ) + + if _DEBUG_EXTRA: + time.sleep(3) + + t = time.perf_counter() + engine.resume_memory_occupation() + print( + f"Resume took {time.perf_counter() - t:.2f}s, memory: {get_gpu_memory_gb():.1f} GB" + ) + + hf_model_new = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + torch_dtype="bfloat16", + device_map="cuda", + ) + engine.update_weights_from_tensor(list(hf_model_new.named_parameters())) + + # destroy the hf model + del hf_model_new + torch.cuda.empty_cache() + + print("generate (#2)") + outputs = engine.generate(params["prompt_moe"], params["sampling_params_moe"])[ + "text" + ] + self.assertEqual(outputs, params["expect_output_after_update_weights_moe"]) + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_request_queue_validation.py b/test/srt/test_request_queue_validation.py new file mode 100644 index 000000000..2a9739a1c --- /dev/null +++ b/test/srt/test_request_queue_validation.py @@ -0,0 +1,87 @@ +import asyncio +import os +import re +import unittest +from concurrent.futures import ThreadPoolExecutor + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + STDERR_FILENAME, + STDOUT_FILENAME, + CustomTestCase, + popen_launch_server, + send_concurrent_generate_requests, + send_generate_requests, +) + + +class TestMaxQueuedRequests(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.stdout = open(STDOUT_FILENAME, "w") + cls.stderr = open(STDERR_FILENAME, "w") + + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--max-running-requests", # Enforce max request concurrency is 1 + "1", + "--max-queued-requests", # Enforce max queued request number is 1 + "1", + ), + return_stdout_stderr=(cls.stdout, cls.stderr), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + cls.stdout.close() + cls.stderr.close() + os.remove(STDOUT_FILENAME) + os.remove(STDERR_FILENAME) + + def test_max_queued_requests_validation_with_serial_requests(self): + """Verify request is not throttled when the max concurrency is 1.""" + status_codes = send_generate_requests( + self.base_url, + num_requests=10, + ) + + for status_code in status_codes: + assert status_code == 200 # request shouldn't be throttled + + def test_max_queued_requests_validation_with_concurrent_requests(self): + """Verify request throttling with concurrent requests.""" + status_codes = asyncio.run( + send_concurrent_generate_requests(self.base_url, num_requests=10) + ) + + assert 200 in status_codes + assert 503 in status_codes + assert all(status_code in [200, 503] for status_code in status_codes) + + def test_max_running_requests_and_max_queued_request_validation(self): + """Verify running request and queued request numbers based on server logs.""" + rr_pattern = re.compile(r"#running-req:\s*(\d+)") + qr_pattern = re.compile(r"#queue-req:\s*(\d+)") + + with open(STDERR_FILENAME) as lines: + for line in lines: + rr_match, qr_match = rr_pattern.search(line), qr_pattern.search(line) + if rr_match: + assert int(rr_match.group(1)) <= 1 + if qr_match: + assert int(qr_match.group(1)) <= 1 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_retract_decode.py b/test/srt/test_retract_decode.py new file mode 100644 index 000000000..92f5ab915 --- /dev/null +++ b/test/srt/test_retract_decode.py @@ -0,0 +1,60 @@ +import os +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestRetractDecode(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_TEST_RETRACT"] = "1" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, cls.base_url, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + +class TestRetractDecodeChunkCache(CustomTestCase): + @classmethod + def setUpClass(cls): + os.environ["SGLANG_TEST_RETRACT"] = "1" + + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--disable-radix-cache", "--chunked-prefill-size", 128], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_rope_rocm.py b/test/srt/test_rope_rocm.py new file mode 100644 index 000000000..5850e7061 --- /dev/null +++ b/test/srt/test_rope_rocm.py @@ -0,0 +1,116 @@ +import unittest + +import torch + +from sglang.srt.layers.rotary_embedding import RotaryEmbedding +from sglang.srt.utils import get_bool_env_var, is_hip +from sglang.test.test_utils import CustomTestCase + +torch.manual_seed(0) + +_is_hip = is_hip() +_use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip + + +_CASES = [ + (64, 64, 32, 8000, True, torch.bfloat16, "cuda", 32, 32, 1, 1), + (256, 128, 4096, 10000, True, torch.bfloat16, "cuda", 2, 512, 4, 2), + (512, 128, 311, 10000, True, torch.bfloat16, "cuda", 3, 39, 4, 2), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 32, 8), + (128, 128, 2048, 10000, False, torch.bfloat16, "cuda", 2, 512, 16, 4), + (512, 128, 311, 10000, False, torch.bfloat16, "cuda", 3, 39, 4, 2), +] + + +@unittest.skipIf(_use_aiter, reason="SGLANG_USE_AITER=1 will not use vllm path.") +class TestRotaryEmbeddingNative(CustomTestCase): + # Compare RotaryEmbedding.forward_hip() to forward_native(). + def _run_case( + self, + head_size: int, + rotary_dim: int, + max_pos: int, + base: int, + is_neox: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q: int, + num_kv: int, + ) -> None: + rope_ref = RotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + rope_hip = RotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv * head_size, dtype=dtype, device=device + ) + + q_ref, k_ref = rope_ref.forward_native(pos_ids, query.clone(), key.clone()) + q_hip, k_hip = rope_hip.forward_hip(pos_ids, query.clone(), key.clone()) + + torch.testing.assert_close(q_ref, q_hip, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(k_ref, k_hip, atol=1e-2, rtol=1e-2) + + def test_all_cases(self) -> None: + """Drive over the full parameter matrix using subTest().""" + for case in _CASES: + with self.subTest(case=case): + self._run_case(*case) + + +@unittest.skipIf(not _use_aiter, reason="Requires AMD GPU plus SGLANG_USE_AITER=1") +class TestRotaryEmbeddingAITer(CustomTestCase): + @staticmethod + def _run_case_aiter( + head_size: int, + rotary_dim: int, + max_pos: int, + base: int, + is_neox: bool, + dtype: torch.dtype, + device: str, + batch_size: int, + seq_len: int, + num_q: int, + num_kv: int, + ) -> None: + from aiter.rotary_embedding import RotaryEmbedding as AiterRotaryEmbedding + + rope_ref = AiterRotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + rope_hip = AiterRotaryEmbedding( + head_size, rotary_dim, max_pos, base, is_neox, dtype + ).to(device) + + pos_ids = torch.arange(seq_len, device=device).repeat(batch_size) + query = torch.randn( + batch_size * seq_len, num_q * head_size, dtype=dtype, device=device + ) + key = torch.randn( + batch_size * seq_len, num_kv * head_size, dtype=dtype, device=device + ) + + q_ref, k_ref = rope_ref.forward_native(pos_ids, query.clone(), key.clone()) + q_hip, k_hip = rope_hip.forward_hip(pos_ids, query.clone(), key.clone()) + + torch.testing.assert_close(q_ref, q_hip, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(k_ref, k_hip, atol=1e-2, rtol=1e-2) + + def test_all_cases(self) -> None: + for case in _CASES: + with self.subTest(case=case): + self._run_case_aiter(*case) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_sagemaker_server.py b/test/srt/test_sagemaker_server.py new file mode 100644 index 000000000..68688c112 --- /dev/null +++ b/test/srt/test_sagemaker_server.py @@ -0,0 +1,179 @@ +""" +python3 -m unittest test_sagemaker_server.TestSageMakerServer.test_chat_completion +""" + +import json +import unittest + +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestSageMakerServer(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.tokenizer = get_tokenizer(DEFAULT_SMALL_MODEL_NAME_FOR_TEST) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_chat_completion(self, logprobs, parallel_sample_num): + data = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + "temperature": 0, + "logprobs": logprobs is not None and logprobs > 0, + "top_logprobs": logprobs, + "n": parallel_sample_num, + } + + headers = {"Authorization": f"Bearer {self.api_key}"} + + response = requests.post( + f"{self.base_url}/invocations", json=data, headers=headers + ).json() + + if logprobs: + assert isinstance( + response["choices"][0]["logprobs"]["content"][0]["top_logprobs"][0][ + "token" + ], + str, + ) + + ret_num_top_logprobs = len( + response["choices"][0]["logprobs"]["content"][0]["top_logprobs"] + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert len(response["choices"]) == parallel_sample_num + assert response["choices"][0]["message"]["role"] == "assistant" + assert isinstance(response["choices"][0]["message"]["content"], str) + assert response["id"] + assert response["created"] + assert response["usage"]["prompt_tokens"] > 0 + assert response["usage"]["completion_tokens"] > 0 + assert response["usage"]["total_tokens"] > 0 + + def run_chat_completion_stream(self, logprobs, parallel_sample_num=1): + data = { + "model": self.model, + "messages": [ + {"role": "system", "content": "You are a helpful AI assistant"}, + { + "role": "user", + "content": "What is the capital of France? Answer in a few words.", + }, + ], + "temperature": 0, + "logprobs": logprobs is not None and logprobs > 0, + "top_logprobs": logprobs, + "stream": True, + "stream_options": {"include_usage": True}, + "n": parallel_sample_num, + } + + headers = {"Authorization": f"Bearer {self.api_key}"} + + response = requests.post( + f"{self.base_url}/invocations", json=data, stream=True, headers=headers + ) + + is_firsts = {} + for line in response.iter_lines(): + line = line.decode("utf-8").replace("data: ", "") + if len(line) < 1 or line == "[DONE]": + continue + print(f"value: {line}") + line = json.loads(line) + usage = line.get("usage") + if usage is not None: + assert usage["prompt_tokens"] > 0 + assert usage["completion_tokens"] > 0 + assert usage["total_tokens"] > 0 + continue + + index = line.get("choices")[0].get("index") + data = line.get("choices")[0].get("delta") + + if is_firsts.get(index, True): + assert data["role"] == "assistant" + is_firsts[index] = False + continue + + if logprobs: + assert line.get("choices")[0].get("logprobs") + assert isinstance( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs")[0] + .get("token"), + str, + ) + assert isinstance( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs"), + list, + ) + ret_num_top_logprobs = len( + line.get("choices")[0] + .get("logprobs") + .get("content")[0] + .get("top_logprobs") + ) + assert ( + ret_num_top_logprobs == logprobs + ), f"{ret_num_top_logprobs} vs {logprobs}" + + assert isinstance(data["content"], str) + assert line["id"] + assert line["created"] + + for index in [i for i in range(parallel_sample_num)]: + assert not is_firsts.get( + index, True + ), f"index {index} is not found in the response" + + def test_chat_completion(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion(logprobs, parallel_sample_num) + + def test_chat_completion_stream(self): + for logprobs in [None, 5]: + for parallel_sample_num in [1, 2]: + self.run_chat_completion_stream(logprobs, parallel_sample_num) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_schedule_policy.py b/test/srt/test_schedule_policy.py new file mode 100644 index 000000000..4a4f57b35 --- /dev/null +++ b/test/srt/test_schedule_policy.py @@ -0,0 +1,65 @@ +import unittest + +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.schedule_policy import ( + CacheAgnosticPolicy, + CacheAwarePolicy, + SchedulePolicy, +) +from sglang.srt.mem_cache.radix_cache import RadixCache, TreeNode +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.test.test_utils import CustomTestCase + + +class TestSchedulePolicy(CustomTestCase): + + def setUp(self): + self.tree_cache = RadixCache(None, None, False) + + def test_init_with_cache_aware_policy(self): + policy = SchedulePolicy( + policy="lpm", tree_cache=self.tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAwarePolicy.LPM) + + def test_init_with_cache_agnostic_policy(self): + policy = SchedulePolicy( + policy="fcfs", tree_cache=self.tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAgnosticPolicy.FCFS) + + def test_init_with_unknown_policy(self): + with self.assertRaises(ValueError): + SchedulePolicy( + policy="invalid", + tree_cache=self.tree_cache, + enable_hierarchical_cache=True, + ) + + def test_init_with_disabled_cache(self): + disabled_tree_cache = RadixCache(None, None, disable=True, page_size=1) + policy = SchedulePolicy( + policy="lpm", tree_cache=disabled_tree_cache, enable_hierarchical_cache=True + ) + self.assertEqual(policy.policy, CacheAgnosticPolicy.FCFS) + + def test_calc_priority_fcfs(self): + tree_cache = RadixCache(None, None, False) + waiting_queue = [ + Req(1, "a b", [1, 2], SamplingParams()), + Req(3, "a b c", [1, 2, 3], SamplingParams()), + Req(2, "a", [1], SamplingParams()), + ] + + policy = SchedulePolicy( + policy="fcfs", tree_cache=tree_cache, enable_hierarchical_cache=True + ) + policy.calc_priority(waiting_queue) + # Check if FCFS keeps the original order + self.assertEqual(waiting_queue[0].rid, 1) + self.assertEqual(waiting_queue[1].rid, 3) + self.assertEqual(waiting_queue[2].rid, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_score_api.py b/test/srt/test_score_api.py new file mode 100644 index 000000000..d08ae9df7 --- /dev/null +++ b/test/srt/test_score_api.py @@ -0,0 +1,300 @@ +import unittest + +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +from sglang.srt.entrypoints.engine import Engine +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + +TEST_MODEL_NAME = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class TestScoreAPI(CustomTestCase): + """Test the scoring API functionality.""" + + def setUp(self): + """Set up each test case.""" + self.engine = Engine(model_path=TEST_MODEL_NAME) + + def tearDown(self): + """Clean up after each test case.""" + if self.engine is not None: + self.engine.shutdown() + torch.cuda.empty_cache() + + def compute_hf_scores( + self, query, items, label_token_ids, apply_softmax=False, item_first=False + ): + """Compute scores using direct HuggingFace model inference. + Returns probabilities for each token ID, optionally normalized with softmax. + + Args: + query: The query text + items: List of item texts + label_token_ids: List of token IDs to compute probabilities for + apply_softmax: Whether to normalize probabilities using softmax + item_first: If True, prepend items to query. Otherwise append items to query. + """ + # Initialize HF model and tokenizer + tokenizer = AutoTokenizer.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + model = AutoModelForCausalLM.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + + try: + scores = [] + for item in items: + # Construct full text based on item_first parameter + full_text = f"{item}{query}" if item_first else f"{query}{item}" + inputs = tokenizer(full_text, return_tensors="pt").to(model.device) + + # Get logits for the last token + with torch.no_grad(): + outputs = model(**inputs) + last_token_logits = outputs.logits[0, -1] + + # Get logits for just our target tokens + target_logits = last_token_logits[label_token_ids] + + # Apply softmax over just the target tokens + target_probs = torch.softmax(target_logits, dim=-1) + + # Convert to list of probabilities in order of label_token_ids + probs = [target_probs[i].item() for i in range(len(label_token_ids))] + + scores.append(probs) + + return scores + finally: + # Clean up HF resources + model.cpu() + del model + del tokenizer + torch.cuda.empty_cache() + + def _get_token_ids(self, tokens): + """Helper method to get token IDs for a list of tokens.""" + tokenizer = AutoTokenizer.from_pretrained( + TEST_MODEL_NAME, trust_remote_code=True + ) + try: + label_token_ids = [] + for token in tokens: + encoding = tokenizer.encode_plus(token, add_special_tokens=False) + token_ids = encoding["input_ids"] + label_token_ids.append(token_ids[0]) + return label_token_ids + finally: + del tokenizer + + def _compare_scores(self, hf_scores, sglang_scores, label_token_ids, case_name=""): + """Helper method to compare scores between HF and SGLang using relative tolerance.""" + self.assertEqual( + len(hf_scores), + len(sglang_scores), + f"Score lengths don't match for {case_name}", + ) + + # Use a relative tolerance of 1% + TOLERANCE = 0.01 + + for hf_score_list, sglang_score_list in zip(hf_scores, sglang_scores): + self.assertEqual( + len(hf_score_list), + len(sglang_score_list), + f"Score list lengths don't match for {case_name}", + ) + + for hf_score, sglang_score in zip(hf_score_list, sglang_score_list): + diff = abs(hf_score - sglang_score) + self.assertLessEqual( + diff, + TOLERANCE, + msg=f"Scores differ by {diff:.2%} ({case_name}): " + f"HF={hf_score:.6f}, SGLang={sglang_score:.6f}", + ) + + self.assertGreaterEqual( + sglang_score, 0, f"SGLang score {sglang_score:.6f} not in [0,1]" + ) + self.assertLessEqual( + sglang_score, 1, f"SGLang score {sglang_score:.6f} not in [0,1]" + ) + + self.assertAlmostEqual( + sum(sglang_score_list), + 1.0, + places=6, + msg=f"SGLang scores don't sum to 1 ({case_name}): {sum(sglang_score_list):.6f}", + ) + + def test_score_consistency(self): + """Test that SGLang scoring matches direct HuggingFace model scoring.""" + # Define test cases + test_cases = [ + { + "name": "default case", + "query": "I pledge allegiance", + "items": ["", " to"], + "item_first": False, + }, + { + "name": "item_first case", + "query": " is a city", + "items": ["Tokyo", "Japan"], + "item_first": True, + }, + ] + + # Common tokens to test for all cases + tokens = [" to", " the"] + label_token_ids = self._get_token_ids(tokens) + + # Run each test case + for case in test_cases: + # Get scores from SGLang + sglang_scores = self.engine.score( + query=case["query"], + items=case["items"], + label_token_ids=label_token_ids, + apply_softmax=True, + item_first=case["item_first"], + ) + + # Get scores from HuggingFace using the same parameters + hf_scores = self.compute_hf_scores( + query=case["query"], + items=case["items"], + label_token_ids=label_token_ids, + apply_softmax=True, + item_first=case["item_first"], + ) + + # Compare scores + self._compare_scores( + hf_scores, sglang_scores, label_token_ids, case["name"] + ) + + def test_score_batch_handling(self): + """Test that batch scoring works correctly.""" + # Test with different batch sizes + batch_sizes = [1, 2, 4, 8] + label_token_ids = [1, 2, 3] + + for batch_size in batch_sizes: + texts = [f"test {i}" for i in range(batch_size)] + scores = self.engine.score( + query="The test was", + items=texts, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + self.assertEqual( + len(scores), + batch_size, + f"Expected {batch_size} scores, got {len(scores)}", + ) + + # Verify each score list has the correct length + for score_list in scores: + self.assertEqual( + len(score_list), + len(label_token_ids), + f"Score list length {len(score_list)} doesn't match label_token_ids length {len(label_token_ids)}", + ) + self.assertTrue( + all(isinstance(v, float) for v in score_list), + "All scores should be floats", + ) + self.assertAlmostEqual( + 1.0, sum(score_list), 6, "Scores should sum to 1" + ) + + def test_score_request_construction(self): + """Test that scoring requests are constructed to avoid decode phase.""" + from unittest.mock import patch + + # Capture the internal request to verify optimization + captured_requests = [] + original_gen = self.engine.tokenizer_manager.generate_request + + async def mock_generate_request(req, request=None): + captured_requests.append(req) + async for result in original_gen(req, request): + yield result + + # Patch the generate_request method + with patch.object( + self.engine.tokenizer_manager, + "generate_request", + side_effect=mock_generate_request, + ): + # Run a scoring request + query = "What is the capital of" + items = ["France", "Germany"] + label_token_ids = [1, 2, 3] + + scores = self.engine.score( + query=query, + items=items, + label_token_ids=label_token_ids, + apply_softmax=True, + ) + + # Verify we got results + self.assertEqual(len(scores), len(items)) + + # Verify the captured request has decode-avoiding properties + self.assertEqual(len(captured_requests), 1) + request = captured_requests[0] + + # Key assertions for decode phase avoidance: + # 1. max_new_tokens should be 0 (prevents token generation) + # Handle both single and batch request cases + if isinstance(request.sampling_params, dict): + max_new_tokens = request.sampling_params.get("max_new_tokens", 0) + elif isinstance(request.sampling_params, list): + # For batch requests, check the first item + max_new_tokens = request.sampling_params[0].get("max_new_tokens", 0) + else: + max_new_tokens = getattr(request.sampling_params, "max_new_tokens", 0) + + self.assertEqual( + max_new_tokens, 0, "max_new_tokens should be 0 to avoid decode phase" + ) + + # 2. Should have token_ids_logprob for scoring + # Handle both single and batch request cases + if ( + isinstance(request.token_ids_logprob, list) + and len(request.token_ids_logprob) > 0 + and isinstance(request.token_ids_logprob[0], list) + ): + # Batch case: token_ids_logprob is a list of lists + # Each item in the batch should have the same label_token_ids + for item_token_ids in request.token_ids_logprob: + self.assertEqual( + item_token_ids, + label_token_ids, + "Each batch item should have label_token_ids for scoring", + ) + else: + # Single request case + self.assertEqual( + request.token_ids_logprob, + label_token_ids, + "Should have label_token_ids for scoring", + ) + + # 3. Should request logprobs but not stream + self.assertTrue( + request.return_logprob, "Should request logprobs for scoring" + ) + self.assertFalse(request.stream, "Scoring requests should not stream") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_server_args.py b/test/srt/test_server_args.py new file mode 100644 index 000000000..6096bc13b --- /dev/null +++ b/test/srt/test_server_args.py @@ -0,0 +1,273 @@ +import json +import unittest +from unittest.mock import MagicMock, patch + +from sglang.srt.server_args import PortArgs, prepare_server_args +from sglang.test.test_utils import CustomTestCase + + +class TestPrepareServerArgs(CustomTestCase): + def test_prepare_server_args(self): + server_args = prepare_server_args( + [ + "--model-path", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "--json-model-override-args", + '{"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}', + ] + ) + self.assertEqual( + server_args.model_path, "meta-llama/Meta-Llama-3.1-8B-Instruct" + ) + self.assertEqual( + json.loads(server_args.json_model_override_args), + {"rope_scaling": {"factor": 2.0, "rope_type": "linear"}}, + ) + + +class TestPortArgs(unittest.TestCase): + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.tempfile.NamedTemporaryFile") + def test_init_new_standard_case(self, mock_temp_file, mock_is_port_available): + mock_is_port_available.return_value = True + mock_temp_file.return_value.name = "temp_file" + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = False + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.scheduler_input_ipc_name.startswith("ipc://")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("ipc://")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_single_node_dp_attention(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = None + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://127.0.0.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://127.0.0.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_dp_rank(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + server_args.enable_dp_attention = True + server_args.nnodes = 1 + server_args.dist_init_addr = "192.168.1.1:25000" + + port_args = PortArgs.init_new(server_args, dp_rank=2) + + self.assertTrue(port_args.scheduler_input_ipc_name.endswith(":25008")) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_ipv4_address(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://192.168.1.1:") + ) + self.assertTrue(port_args.detokenizer_ipc_name.startswith("tcp://192.168.1.1:")) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address(self, mock_is_port_available): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1" + + with self.assertRaises(AssertionError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "please provide --dist-init-addr as host:port", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv4_address_invalid_port( + self, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "192.168.1.1:abc" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:25000" + + port_args = PortArgs.init_new(server_args) + + self.assertTrue(port_args.tokenizer_ipc_name.startswith("tcp://[2001:db8::1]:")) + self.assertTrue( + port_args.scheduler_input_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertTrue( + port_args.detokenizer_ipc_name.startswith("tcp://[2001:db8::1]:") + ) + self.assertIsInstance(port_args.nccl_port, int) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=False) + def test_init_new_with_invalid_ipv6_address( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[invalid-ipv6]:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + def test_init_new_with_malformed_ipv6_address_missing_bracket( + self, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1:25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid IPv6 address format", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_missing_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn( + "a port must be specified in IPv6 address", str(context.exception) + ) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_invalid_port( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]:abcde" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("invalid port in IPv6 address", str(context.exception)) + + @patch("sglang.srt.server_args.is_port_available") + @patch("sglang.srt.server_args.is_valid_ipv6_address", return_value=True) + def test_init_new_with_malformed_ipv6_address_wrong_separator( + self, mock_is_valid_ipv6, mock_is_port_available + ): + mock_is_port_available.return_value = True + + server_args = MagicMock() + server_args.port = 30000 + server_args.nccl_port = None + + server_args.enable_dp_attention = True + server_args.nnodes = 2 + server_args.dist_init_addr = "[2001:db8::1]#25000" + + with self.assertRaises(ValueError) as context: + PortArgs.init_new(server_args) + + self.assertIn("expected ':' after ']'", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_session_control.py b/test/srt/test_session_control.py new file mode 100644 index 000000000..4b0da75dc --- /dev/null +++ b/test/srt/test_session_control.py @@ -0,0 +1,786 @@ +""" +Usage: +python3 -m unittest test_session_control.TestSessionControl.test_session_control +python3 -m unittest test_session_control.TestSessionControl.test_session_control_with_branching +python3 -m unittest test_session_control.TestSessionControl.test_session_control_backtrack_with_abort +python3 -m unittest test_session_control.TestSessionControlVision.test_session_control +""" + +import asyncio +import json +import unittest + +import aiohttp +import requests + +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +def remove_prefix(text: str, prefix: str) -> str: + return text[len(prefix) :] if text.startswith(prefix) else text + + +class TestSessionControl(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--attention-backend", + "flashinfer", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_session_control(self, gen_len=12): + chunks = [ + "Let me tell you something about France.", + "The capital of France is", + "The population of the city is", + "A brief history about that city is", + ] + tokenizer = get_tokenizer(self.model) + chunks_ids = [tokenizer.encode(x) for x in chunks] + for i in range(1, len(chunks_ids)): + if chunks_ids[i][0] == tokenizer.bos_token_id: + chunks_ids[i] = chunks_ids[i][1:] + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + # open an existing session, should get session_id as None + ret = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000, "session_id": session_id}, + ) + self.assertNotEqual(ret.status_code, 200) + + first_rid = None + outputs_from_session = [] + logprobs_from_session = [] + cur_logprob_start_len = 0 + for i, chunk_ids in enumerate(chunks_ids): + max_new_tokens = gen_len if i > 0 else 1 # prefill only for the first chunk + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunk_ids, + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len - 1, + }, + ).json() + rid = response["meta_info"]["id"] + if i == 0: + first_rid = rid + if i > 0: + outputs_from_session.append(response["text"]) + logprobs_from_session.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + cur_logprob_start_len += len(chunk_ids) + max_new_tokens + + # query with a logprob_start_len longer than the request, should see error + ret = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunk_ids, + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len + len(chunk_ids), + }, + ) + self.assertNotEqual(ret.status_code, 200) + + # backtrack to the first request and regenerate + cur_logprob_start_len = 0 + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + "logprob_start_len": cur_logprob_start_len, + }, + ).json() + outputs_from_session.append(response["text"]) + logprobs_from_session.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + + # query with a non-existing rid (the last one should be disappeared because of backtrack), should see abort + ret = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ) + self.assertNotEqual(ret.status_code, 200) + + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + self.assertEqual(ret.status_code, 200) + + # send a request to a closed session, should see abort + ret = requests.post( + self.base_url + "/generate", + json={ + "input_ids": chunks_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ) + self.assertNotEqual(ret.status_code, 200) + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + input_ids_first_req = None + input_ids = [] + outputs_normal = [] + logprobs_normal = [] + for i, chunk_ids in enumerate(chunks_ids): + input_ids += chunk_ids + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 1 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + if i > 0: + output_ids = tokenizer.encode(response["text"]) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids += output_ids[:-1] + outputs_normal.append(response["text"]) + logprobs_normal.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + if i == 0: + input_ids_first_req = input_ids.copy() + + input_ids_first_req += chunks_ids[-1] + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids_first_req, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "return_logprob": True, + }, + ).json() + outputs_normal.append(response["text"]) + logprobs_normal.extend( + [ + round(sublist[0], 2) + for sublist in response["meta_info"]["output_token_logprobs"] + ] + ) + + print("outputs from chunked queries with session control:") + print(outputs_from_session) + print("outputs from normal queries:") + print(outputs_normal) + self.assertEqual(outputs_from_session, outputs_normal) + print("logprobs from chunked queries with session control:") + print(logprobs_from_session) + print("logprobs from normal queries:") + print(logprobs_normal) + assert len(logprobs_from_session) == len( + logprobs_normal + ), "logprobs must have equal length" + for a, b in zip(logprobs_from_session, logprobs_normal): + assert abs(a - b) <= 0.15, f"logprobs {a} and {b} differ by more than 0.15" + + async def async_generate(self, payload): + url = self.base_url + "/generate" + async with aiohttp.ClientSession() as session: + async with session.post(url=url, json=payload) as response: + assert response.status == 200 + async for chunk_bytes in response.content: + chunk_bytes = chunk_bytes.strip() + if not chunk_bytes: + continue + chunk = remove_prefix(chunk_bytes.decode("utf-8"), "data: ") + if chunk == "[DONE]": + yield "", None, "" + else: + data = json.loads(chunk) + finish_reason = ( + data["meta_info"]["finish_reason"]["type"] + if data["meta_info"]["finish_reason"] + else "" + ) + yield data["text"], data["meta_info"]["id"], finish_reason + + async def run_session_control_backtrack_with_abort(self, replace): + chunks = [ + "Let me tell you something about France.", + "The capital of France is", + ] + tokenizer = get_tokenizer(self.model) + chunks_ids = [tokenizer.encode(x) for x in chunks] + for i in range(1, len(chunks_ids)): + if chunks_ids[i][0] == tokenizer.bos_token_id: + chunks_ids[i] = chunks_ids[i][1:] + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + payload = { + "input_ids": chunks_ids[0], + "session_params": { + "id": session_id, + "rid": rid, + "offset": -1, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 100, + "no_stop_trim": True, + "skip_special_tokens": False, + "ignore_eos": True, + }, + "stream": True, + } + gen_so_far = "" + finish_reason = "" + second_output = "" + async for chunk, rid, finish_reason_chunk in self.async_generate(payload): + gen_so_far += chunk + if finish_reason == "": + finish_reason = finish_reason_chunk + if len(gen_so_far) > 50 and second_output == "": + payload2 = { + "input_ids": chunks_ids[1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 50, + "replace": replace, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "stream": False, + "stream_output": True, + } + response = requests.post( + url=self.base_url + "/generate", json=payload2 + ).json() + second_output = response["text"] + if replace: + assert finish_reason == "abort" + print("first request output:") + print(gen_so_far) + print("second request output:") + print(second_output) + + # close the session + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + if not replace: + assert response["meta_info"]["finish_reason"]["type"] == "abort" + else: + # 2. not using session control + requests.post(self.base_url + "/flush_cache") + output_ids = tokenizer.encode(gen_so_far) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids = chunks_ids[0] + output_ids + input_ids = input_ids[:50] + chunks_ids[1] + payload = { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "stream": False, + "stream_output": True, + } + response = requests.post( + url=self.base_url + "/generate", json=payload + ).json() + output_no_session = response["text"] + print("second request output without session:") + print(output_no_session) + assert ( + second_output == output_no_session + ), f"second_output: {second_output}, output_no_session: {output_no_session}" + + @unittest.skip("broken") + def test_session_control_backtrack_with_abort(self): + asyncio.run(self.run_session_control_backtrack_with_abort(replace=True)) + asyncio.run(self.run_session_control_backtrack_with_abort(replace=False)) + + def run_session_control_with_branching( + self, root_prompt, chunks_per_step, gen_len=16 + ): + for x in chunks_per_step: + assert len(x) == len(chunks_per_step[0]) + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + + outputs_from_session = [] + # send the root prompt + response = requests.post( + self.base_url + "/generate", + json={ + "text": root_prompt, + "session_params": { + "id": session_id, + "rid": None, + "offset": 0, + "replace": False, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid_per_branch = [response["meta_info"]["id"]] * len(chunks_per_step[0]) + outputs_from_session.append(response["text"]) + + # send the prompts in branches + for chunks_for_branches in chunks_per_step: + for j, chunk in enumerate(chunks_for_branches): + response = requests.post( + self.base_url + "/generate", + json={ + "text": chunk, + "session_params": { + "id": session_id, + "rid": rid_per_branch[j], + "offset": 0, + "replace": False, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid = response["meta_info"]["id"] + rid_per_branch[j] = rid + outputs_from_session.append(response["text"]) + + # close the session + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + assert ret.status_code == 200 + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + outputs_normal = [] + input_texts = [root_prompt] * len(chunks_per_step[0]) + # send the root prompt + response = requests.post( + self.base_url + "/generate", + json={ + "text": root_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + input_texts = [x + response["text"] for x in input_texts] + + # send the prompts in branches + for chunks_for_branches in chunks_per_step: + for j, chunk in enumerate(chunks_for_branches): + input_texts[j] += chunk + response = requests.post( + self.base_url + "/generate", + json={ + "text": input_texts[j], + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + input_texts[j] += response["text"] + + print("====== outputs from chunked queries with session control: =======") + print(outputs_from_session) + print("====== outputs from normal queries: =======") + print(outputs_normal) + assert ( + outputs_from_session == outputs_normal + ), f"outputs_from_session: {outputs_from_session}, outputs_normal: {outputs_normal}" + + def test_session_control_with_branching(self): + root_prompt = "First, let me explain in one sentence about AI" + chunks_per_step = [ + [ + "Then, briefly, the positive side of AI is", + "But, briefly, AI could be harmful to human", + ], + ["For example", "For example"], + ] + self.run_session_control_with_branching( + root_prompt=root_prompt, chunks_per_step=chunks_per_step, gen_len=8 + ) + + root_prompt = "I have three apples." + chunks_per_step = [ + ["I then give one apple to my friend", "My friend give me another apple."], + ["I still have", "I now have"], + ] + self.run_session_control_with_branching( + root_prompt=root_prompt, chunks_per_step=chunks_per_step, gen_len=8 + ) + + +@unittest.skip("broken") +class TestSessionControlVision(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmms-lab/llava-onevision-qwen2-7b-ov" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + # other_args={"--disable-radix"}, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_session_control(self): + text_chunks = [ + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n", + "<|im_start|>user\n\nDescribe this image in a very short sentence.<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\n\nIs this image same with one of the previous images?<|im_end|>\n<|im_start|>assistant\n", + "<|im_start|>user\nDescribe this image in a very short sentence.<|im_end|>\nassistant:", + ] + image_chunks = [ + "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png", + "https://raw.githubusercontent.com/sgl-project/sglang/main/test/lang/example_image.png", + "https://raw.githubusercontent.com/sgl-project/sglang/main/assets/logo.png", + ] + + self.assertEqual( + len(text_chunks), len(image_chunks) + 2 + ) # the first and the last prompt does not contain images + tokenizer = get_tokenizer(self.model) + text_input_ids = [tokenizer.encode(x) for x in text_chunks] + for i in range(1, len(text_input_ids)): + if text_input_ids[i][0] == tokenizer.bos_token_id: + text_input_ids[i] = text_input_ids[i][1:] + gen_len = 32 + + # 1. using session control + requests.post(self.base_url + "/flush_cache") + session_id = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000}, + ).json() + rid = None + + # open an existing session, should get session_id as None + ret = requests.post( + self.base_url + "/open_session", + json={"capacity_of_str_len": 1000, "session_id": session_id}, + ) + self.assertNotEqual(ret.status_code, 200) + + first_rid = None + outputs_from_session = [] + for i in range(len(text_input_ids[:-1])): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[i], + "image_data": image_chunks[i - 1] if i > 0 else None, + "modalities": ["multi-images"], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 0 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + rid = response["meta_info"]["id"] + if i == 0: + first_rid = rid + if i > 0: + outputs_from_session.append(response["text"]) + + # backtrack to the first request and regenerate + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_from_session.append(response["text"]) + + # query with a non-existing rid (the last one should be disappeared because of backtrack), should see abort + ret = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ) + self.assertNotEqual(ret.status_code, 200) + + ret = requests.post( + self.base_url + "/close_session", + json={"session_id": session_id}, + ) + self.assertEqual(ret.status_code, 200) + + # send a request to a closed session, should see abort + ret = requests.post( + self.base_url + "/generate", + json={ + "input_ids": text_input_ids[-1], + "session_params": { + "id": session_id, + "rid": first_rid, + "offset": 0, + "replace": True, + }, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ) + self.assertNotEqual(ret.status_code, 200) + + # 2. not use session control + requests.post(self.base_url + "/flush_cache") + + input_ids_first_req = None + input_ids = [] + outputs_normal = [] + for i in range(len(text_input_ids[:-1])): + input_ids += text_input_ids[i] + image_data = image_chunks[:i] if i > 0 else None + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids, + "image_data": image_data, + "modalities": ["multi-images"], + "sampling_params": { + "temperature": 0, + "max_new_tokens": ( + gen_len if i > 0 else 0 + ), # prefill only for the first chunk + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + if i > 0: + output_ids = tokenizer.encode(response["text"]) + if output_ids[0] == tokenizer.bos_token_id: + output_ids = output_ids[1:] + input_ids += output_ids + outputs_normal.append(response["text"]) + if i == 0: + input_ids_first_req = input_ids.copy() + + input_ids_first_req += text_input_ids[-1] + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": input_ids_first_req, + "sampling_params": { + "temperature": 0, + "max_new_tokens": gen_len, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + }, + ).json() + outputs_normal.append(response["text"]) + + print("outputs from chunked queries with session control:") + print(outputs_from_session) + print("outputs from normal queries:") + print(outputs_normal) + assert ( + outputs_from_session == outputs_normal + ), f"outputs_from_session: {outputs_from_session}, outputs_normal: {outputs_normal}" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_skip_tokenizer_init.py b/test/srt/test_skip_tokenizer_init.py new file mode 100644 index 000000000..02b1b40c6 --- /dev/null +++ b/test/srt/test_skip_tokenizer_init.py @@ -0,0 +1,246 @@ +""" +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.test_parallel_sample +python3 -m unittest test_skip_tokenizer_init.TestSkipTokenizerInit.run_decode_stream +""" + +import json +import unittest +from io import BytesIO + +import requests +from PIL import Image +from transformers import AutoProcessor, AutoTokenizer + +from sglang.lang.chat_template import get_chat_template_by_model_path +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_IMAGE_URL, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestSkipTokenizerInit(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init", "--stream-output"], + ) + cls.eos_token_id = [119690] + cls.tokenizer = AutoTokenizer.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, use_fast=False + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + prompt_text="The capital of France is", + max_new_tokens=32, + return_logprob=False, + top_logprobs_num=0, + n=1, + ): + input_ids = self.get_input_ids(prompt_text) + + request = self.get_request_json( + input_ids=input_ids, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + max_new_tokens=max_new_tokens, + stream=False, + n=n, + ) + response = requests.post( + self.base_url + "/generate", + json=request, + ) + ret = response.json() + print(json.dumps(ret, indent=2)) + + def assert_one_item(item): + if item["meta_info"]["finish_reason"]["type"] == "stop": + self.assertEqual( + item["meta_info"]["finish_reason"]["matched"], + self.tokenizer.eos_token_id, + ) + elif item["meta_info"]["finish_reason"]["type"] == "length": + self.assertEqual( + len(item["output_ids"]), item["meta_info"]["completion_tokens"] + ) + self.assertEqual(len(item["output_ids"]), max_new_tokens) + self.assertEqual(item["meta_info"]["prompt_tokens"], len(input_ids)) + + if return_logprob: + num_input_logprobs = len(input_ids) - request["logprob_start_len"] + if num_input_logprobs > len(input_ids): + num_input_logprobs -= len(input_ids) + self.assertEqual( + len(item["meta_info"]["input_token_logprobs"]), + num_input_logprobs, + f'{len(item["meta_info"]["input_token_logprobs"])} mismatch with {len(input_ids)}', + ) + self.assertEqual( + len(item["meta_info"]["output_token_logprobs"]), + max_new_tokens, + ) + + # Determine whether to assert a single item or multiple items based on n + if n == 1: + assert_one_item(ret) + else: + self.assertEqual(len(ret), n) + for i in range(n): + assert_one_item(ret[i]) + + print("=" * 100) + + def run_decode_stream(self, return_logprob=False, top_logprobs_num=0, n=1): + max_new_tokens = 32 + input_ids = self.get_input_ids("The capital of France is") + requests.post(self.base_url + "/flush_cache") + response = requests.post( + self.base_url + "/generate", + json=self.get_request_json( + input_ids=input_ids, + max_new_tokens=max_new_tokens, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + stream=False, + n=n, + ), + ) + ret = response.json() + print(json.dumps(ret)) + output_ids = ret["output_ids"] + print("output from non-streaming request:") + print(output_ids) + print(self.tokenizer.decode(output_ids, skip_special_tokens=True)) + + requests.post(self.base_url + "/flush_cache") + response_stream = requests.post( + self.base_url + "/generate", + json=self.get_request_json( + input_ids=input_ids, + return_logprob=return_logprob, + top_logprobs_num=top_logprobs_num, + stream=True, + n=n, + ), + ) + + response_stream_json = [] + for line in response_stream.iter_lines(): + print(line) + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_stream_json.append(json.loads(line[6:])) + out_stream_ids = [] + for x in response_stream_json: + out_stream_ids += x["output_ids"] + print("output from streaming request:") + print(out_stream_ids) + print(self.tokenizer.decode(out_stream_ids, skip_special_tokens=True)) + + assert output_ids == out_stream_ids + + def test_simple_decode(self): + self.run_decode() + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_logprob(self): + for top_logprobs_num in [0, 3]: + self.run_decode(return_logprob=True, top_logprobs_num=top_logprobs_num) + + def test_eos_behavior(self): + self.run_decode(max_new_tokens=256) + + def test_simple_decode_stream(self): + self.run_decode_stream() + + def get_input_ids(self, prompt_text) -> list[int]: + input_ids = self.tokenizer(prompt_text, return_tensors="pt")["input_ids"][ + 0 + ].tolist() + return input_ids + + def get_request_json( + self, + input_ids, + max_new_tokens=32, + return_logprob=False, + top_logprobs_num=0, + stream=False, + n=1, + ): + return { + "input_ids": input_ids, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": max_new_tokens, + "n": n, + "stop_token_ids": self.eos_token_id, + }, + "stream": stream, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "logprob_start_len": 0, + } + + +class TestSkipTokenizerInitVLM(TestSkipTokenizerInit): + @classmethod + def setUpClass(cls): + cls.image_url = DEFAULT_IMAGE_URL + response = requests.get(cls.image_url) + cls.image = Image.open(BytesIO(response.content)) + cls.model = DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model, use_fast=False) + cls.processor = AutoProcessor.from_pretrained(cls.model, trust_remote_code=True) + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--skip-tokenizer-init"], + ) + cls.eos_token_id = [cls.tokenizer.eos_token_id] + + def get_input_ids(self, _prompt_text) -> list[int]: + chat_template = get_chat_template_by_model_path(self.model) + text = f"{chat_template.image_token}What is in this picture?" + inputs = self.processor( + text=[text], + images=[self.image], + return_tensors="pt", + ) + + return inputs.input_ids[0].tolist() + + def get_request_json(self, *args, **kwargs): + ret = super().get_request_json(*args, **kwargs) + ret["image_data"] = [self.image_url] + ret["logprob_start_len"] = ( + -1 + ) # Do not try to calculate logprobs of image embeddings. + return ret + + def test_simple_decode_stream(self): + # TODO mick + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_endpoint.py b/test/srt/test_srt_endpoint.py new file mode 100644 index 000000000..089da355d --- /dev/null +++ b/test/srt/test_srt_endpoint.py @@ -0,0 +1,640 @@ +""" +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_simple_decode +python3 -m unittest test_srt_endpoint.TestSRTEndpoint.test_logprob_with_chunked_prefill +""" + +import json +import random +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from typing import Optional + +import numpy as np +import requests + +from sglang.srt.sampling.custom_logit_processor import CustomLogitProcessor +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, + run_logprob_check, +) + + +class TestSRTEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=( + "--enable-custom-logit-processor", + "--mem-fraction-static", + "0.7", + "--cuda-graph-max-bs", + "8", + ), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_decode( + self, + return_logprob=False, + top_logprobs_num=0, + return_text=False, + n=1, + stream=False, + batch=False, + ): + if batch: + text = ["The capital of France is"] + else: + text = "The capital of France is" + + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "sampling_params": { + "temperature": 0 if n == 1 else 0.5, + "max_new_tokens": 16, + "n": n, + }, + "stream": stream, + "return_logprob": return_logprob, + "top_logprobs_num": top_logprobs_num, + "return_text_in_logprobs": return_text, + "logprob_start_len": 0, + }, + ) + if not stream: + response_json = response.json() + else: + response_json = [] + for line in response.iter_lines(): + if line.startswith(b"data: ") and line[6:] != b"[DONE]": + response_json.append(json.loads(line[6:])) + + print(json.dumps(response_json, indent=2)) + print("=" * 100) + + def test_simple_decode(self): + self.run_decode() + + def test_simple_decode_batch(self): + self.run_decode(batch=True) + + def test_parallel_sample(self): + self.run_decode(n=3) + + def test_parallel_sample_stream(self): + self.run_decode(n=3, stream=True) + + def test_logprob(self): + self.run_decode( + return_logprob=True, + top_logprobs_num=5, + return_text=True, + ) + + def test_logprob_start_len(self): + logprob_start_len = 4 + new_tokens = 4 + prompts = [ + "I have a very good idea on", + "Today is a sunndy day and", + ] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "top_logprobs_num": 5, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + response_json = response.json() + print(json.dumps(response_json, indent=2)) + + for i, res in enumerate(response_json): + self.assertEqual( + res["meta_info"]["prompt_tokens"], + logprob_start_len + len(res["meta_info"]["input_token_logprobs"]), + ) + assert prompts[i].endswith( + "".join([x[-1] for x in res["meta_info"]["input_token_logprobs"]]) + ) + + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual( + res["text"], + "".join([x[-1] for x in res["meta_info"]["output_token_logprobs"]]), + ) + + def test_logprob_with_chunked_prefill(self): + """Test a long prompt that requests output logprobs will not hit OOM.""" + new_tokens = 4 + prompts = "I have a very good idea on this. " * 8000 + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 0, + "max_new_tokens": new_tokens, + }, + "return_logprob": True, + "logprob_start_len": -1, + "top_logprobs_num": 5, + }, + ) + response_json = response.json() + # print(json.dumps(response_json, indent=2)) + + res = response_json + self.assertEqual(res["meta_info"]["completion_tokens"], new_tokens) + + # Test the number of tokens are correct + self.assertEqual(len(res["meta_info"]["output_token_logprobs"]), new_tokens) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"]), new_tokens) + + # Test the top-1 tokens are the same as output tokens (because temp = 0.0) + for i in range(new_tokens): + self.assertListEqual( + res["meta_info"]["output_token_logprobs"][i], + res["meta_info"]["output_top_logprobs"][i][0], + ) + self.assertEqual(len(res["meta_info"]["output_top_logprobs"][i]), 5) + + def test_logprob_match(self): + """Test the output logprobs are close to the input logprobs if we run a prefill again.""" + + def run_generate( + prompt, return_logprob=False, max_new_tokens=512, logprob_start_len=-1 + ): + + if isinstance(prompt, str): + prompt_kwargs = {"text": prompt} + else: + prompt_kwargs = {"input_ids": prompt} + + response = requests.post( + self.base_url + "/generate", + json={ + **prompt_kwargs, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + "return_logprob": return_logprob, + "return_text_in_logprobs": True, + "logprob_start_len": logprob_start_len, + }, + ) + return response.json() + + prompt = "I have a very good idea on how to" + + gen = run_generate(prompt, return_logprob=True, logprob_start_len=0) + output_logprobs = np.array( + [x[0] for x in gen["meta_info"]["output_token_logprobs"]] + ) + num_prompts_tokens = gen["meta_info"]["prompt_tokens"] + + input_tokens = [x[1] for x in gen["meta_info"]["input_token_logprobs"]] + output_tokens = [x[1] for x in gen["meta_info"]["output_token_logprobs"]] + + new_prompt = input_tokens + output_tokens + score = run_generate( + new_prompt, return_logprob=True, logprob_start_len=0, max_new_tokens=0 + ) + output_logprobs_score = np.array( + [ + x[0] + for x in score["meta_info"]["input_token_logprobs"][num_prompts_tokens:] + ] + ) + + print(f"{output_logprobs[-10:]=}") + print(f"{output_logprobs_score[-10:]=}") + + diff = np.abs(output_logprobs - output_logprobs_score) + max_diff = np.max(diff) + self.assertLess(max_diff, 0.35) + + def test_logprob_mixed(self): + args = [] + temperature = 0 + # input_len, output_len, temperature, logprob_start_len, return_logprob, top_logprobs_num + for input_len in [1000, 5000, 10000, 50000]: + for output_len in [4, 8]: + for logprob_start_len in [0, 500, 2500, 5000, 25000]: + for return_logprob in [True, False]: + for top_logprobs_num in [0, 5]: + + if logprob_start_len >= input_len: + continue + + args.append( + ( + input_len, + output_len, + temperature, + logprob_start_len, + return_logprob, + top_logprobs_num, + ) + ) + + random.shuffle(args) + + func = partial(run_logprob_check, self) + with ThreadPoolExecutor(8) as executor: + list(executor.map(func, args)) + + def test_logprob_grammar(self): + prompts = "Question: Is Paris the Capital of France? Answer:" + allowed_tokens = [" Yes", " No"] + + response = requests.post( + self.base_url + "/generate", + json={ + "text": prompts, + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 1, + "regex": "( Yes| No)", + }, + "return_logprob": True, + "top_logprobs_num": 5, # The grammar constraint allows all prefix tokens so we need to use a larger top_k. + "return_text_in_logprobs": True, + }, + ) + response_json = response.json() + output_top_logprobs = response_json["meta_info"]["output_top_logprobs"][0] + print(f"{output_top_logprobs=}") + + # Parse results + # This is because the grammar constraint allows all prefix tokens + logprobs = [None] * 2 + for i in range(len(output_top_logprobs)): + try: + idx = allowed_tokens.index(output_top_logprobs[i][2]) + except ValueError: + # Not found + continue + logprobs[idx] = output_top_logprobs[i][0] + + self.assertTrue(all(x is not None for x in logprobs)) + + def run_custom_logit_processor(self, target_token_id: Optional[int] = None): + """Test custom logit processor with custom params. + + If target_token_id is None, the custom logit processor won't be passed in. + """ + + custom_params = {"token_id": target_token_id} + + class DeterministicLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + key = "token_id" + + for i, param_dict in enumerate(custom_param_list): + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, param_dict[key]] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if target_token_id is not None: + custom_json["custom_logit_processor"] = DeterministicLogitProcessor.to_str() + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # The logit processor should always sample the given token as the logits is deterministic. + if target_token_id is not None: + self.assertTrue( + all(x == custom_params["token_id"] for x in sampled_tokens), + # Print the detailed test case info if the test fails. + f"{target_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def run_stateful_custom_logit_processor( + self, first_token_id: int | None, delay: int = 2 + ): + """Test custom logit processor with custom params and state. + + Should sample the first `delay` tokens normally, then output first_token_id and consecutive tokens after that. + If first_token_id is None, the custom logit processor won't be passed in. + """ + custom_params = {"token_id": first_token_id, "delay": 2} + + class DeterministicStatefulLogitProcessor(CustomLogitProcessor): + """A dummy logit processor that changes the logits to always + sample the given token id. + """ + + def __call__(self, logits, custom_param_list): + assert logits.shape[0] == len(custom_param_list) + + for i, param_dict in enumerate(custom_param_list): + if param_dict["delay"] > 0: + param_dict["delay"] -= 1 + continue + if param_dict["delay"] == 0: + param_dict["delay"] -= 1 + force_token = param_dict["token_id"] + else: + output_ids = param_dict["__req__"].output_ids + force_token = output_ids[-1] + 1 + # Mask all other tokens + logits[i, :] = -float("inf") + # Assign highest probability to the specified token + logits[i, force_token] = 0.0 + return logits + + prompts = "Question: Is Paris the Capital of France? Answer:" + + # Base case json data to be posted to the server. + base_json = { + "text": prompts, + "sampling_params": {"temperature": 0.0}, + "return_logprob": True, + } + + # Custom json data with custom logit processor and params. + custom_json = base_json.copy() + # Only set the custom logit processor if target_token_id is not None. + if first_token_id is not None: + custom_json["custom_logit_processor"] = ( + DeterministicStatefulLogitProcessor().to_str() + ) + custom_json["sampling_params"]["custom_params"] = custom_params + + custom_response = requests.post( + self.base_url + "/generate", + json=custom_json, + ).json() + + output_token_logprobs = custom_response["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + # The logit processor should always sample the given token as the logits is deterministic. + if first_token_id is not None: + self.assertTrue( + all( + x == custom_params["token_id"] + k + for k, x in enumerate(sampled_tokens[custom_params["delay"] :]) + ), + # Print the detailed test case info if the test fails. + f"{first_token_id=}\n{sampled_tokens=}\n{custom_response=}", + ) + + def test_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + self.run_custom_logit_processor(target_token_id=5) + + def test_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list(executor.map(self.run_custom_logit_processor, target_token_ids)) + + @unittest.skip("Skip this test because this feature has a bug. See comments below.") + def test_stateful_custom_logit_processor(self): + """Test custom logit processor with a single request.""" + + """ + NOTE: This feature has a race condition bug. + This line https://github.com/sgl-project/sglang/blob/ef8ec07b2ce4c70c2a33ec5acda4ce529bc3cda4/test/srt/test_srt_endpoint.py#L395-L396 can be accessed by two concurrent threads at the same time. The access order is not guaranteed. + In sglang, we use two python threads to overlap the GPU computation and CPU scheduling. + Thread 1 (the CPU scheduling thread) will update the `param_dict["__req__"].output_ids`. + Thread 2 (the GPU computation thread) will call `DeterministicStatefulLogitProcessor` because sampling is considered as GPU computation. + We can fix this by moving the call of DeterministicStatefulLogitProcessor to the CPU scheduling thread. + """ + + self.run_stateful_custom_logit_processor(first_token_id=5) + + @unittest.skip("Skip this test because this feature has a bug. See comments above.") + def test_stateful_custom_logit_processor_batch_mixed(self): + """Test a batch of requests mixed of requests with and without custom logit processor.""" + target_token_ids = list(range(32)) + [None] * 16 + random.shuffle(target_token_ids) + with ThreadPoolExecutor(len(target_token_ids)) as executor: + list( + executor.map(self.run_stateful_custom_logit_processor, target_token_ids) + ) + + def test_cache_tokens(self): + for _ in range(2): + time.sleep(1) + response = requests.post(self.base_url + "/flush_cache") + assert response.status_code == 200 + + def send_and_check_cached_tokens(input_ids): + response = requests.post( + self.base_url + "/generate", + json={ + "input_ids": list(input_ids), + "sampling_params": { + "max_new_tokens": 1, + }, + }, + ) + response_json = response.json() + return response_json["meta_info"]["cached_tokens"] + + self.assertEqual(send_and_check_cached_tokens(range(0, 100)), 0) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 100) + self.assertEqual(send_and_check_cached_tokens(range(0, 10000)), 9999) + self.assertEqual(send_and_check_cached_tokens(range(0, 1000)), 999) + self.assertEqual(send_and_check_cached_tokens(range(0, 11000)), 10000) + + def test_get_server_info(self): + response = requests.get(self.base_url + "/get_server_info") + response_json = response.json() + + max_total_num_tokens = response_json["max_total_num_tokens"] + self.assertIsInstance(max_total_num_tokens, int) + + version = response_json["version"] + self.assertIsInstance(version, str) + + def test_logit_bias(self): + """Test that a very high logit bias forces sampling of a specific token.""" + # Choose a token ID to bias (using 5 as an example) + target_token_id = 60704 # Paris for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + logit_bias = {str(target_token_id): 100.0} # Very high positive bias + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, # Use high temperature to encourage exploration + "max_new_tokens": 4, + "logit_bias": logit_bias, + }, + "return_logprob": True, + }, + ) + response_json = response.json() + + # Extract the sampled token IDs from the output + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # Verify that all sampled tokens are the target token + self.assertTrue( + all(x == target_token_id for x in sampled_tokens), + f"Expected all tokens to be {target_token_id}, but got {sampled_tokens}", + ) + + def test_forbidden_token(self): + """Test that a forbidden token (very negative logit bias) doesn't appear in the output.""" + # Choose a token ID to forbid (using 10 as an example) + forbidden_token_id = 23994 # rice for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + logit_bias = { + str(forbidden_token_id): -100.0 + } # Very negative bias to forbid the token + + response = requests.post( + self.base_url + "/generate", + json={ + "text": "Only output 'rice' exactly like this, in lowercase ONLY: rice", + "sampling_params": { + "temperature": 1.0, # Use high temperature to encourage diverse output + "max_new_tokens": 50, # Generate enough tokens to likely include numbers + "logit_bias": logit_bias, + }, + "return_logprob": True, + }, + ) + response_json = response.json() + + # Extract the sampled token IDs from the output + output_token_logprobs = response_json["meta_info"]["output_token_logprobs"] + sampled_tokens = [x[1] for x in output_token_logprobs] + + # Verify that the forbidden token doesn't appear in the output + self.assertNotIn( + forbidden_token_id, + sampled_tokens, + f"Expected forbidden token {forbidden_token_id} not to be present, but it was found", + ) + + def test_logit_bias_isolation(self): + """Test that logit_bias applied to one request doesn't affect other requests in batch.""" + # Choose a token ID to bias in first request only + biased_token_id = 60704 # Paris for meta-llama/Llama-3.2-1B-Instruct, DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + # Prepare batch requests - one with logit_bias and one without + requests_data = [ + { + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 4, + "logit_bias": {str(biased_token_id): 100.0}, # Strong bias + }, + "return_logprob": True, + }, + { + "text": "The capital of France is", + "sampling_params": { + "temperature": 1.0, + "max_new_tokens": 4, + }, + "return_logprob": True, + }, + ] + + # Send both requests + responses = [] + for req in requests_data: + response = requests.post(self.base_url + "/generate", json=req) + responses.append(response.json()) + + # Extract token IDs from each response + biased_tokens = [ + x[1] for x in responses[0]["meta_info"]["output_token_logprobs"] + ] + unbiased_tokens = [ + x[1] for x in responses[1]["meta_info"]["output_token_logprobs"] + ] + + # Verify first response contains only biased tokens + self.assertTrue( + all(x == biased_token_id for x in biased_tokens), + f"Expected all tokens to be {biased_token_id} in first response, but got {biased_tokens}", + ) + + # Verify second response contains at least some different tokens + # (We can't guarantee exactly what tokens will be generated, but they shouldn't all be the biased token) + self.assertTrue( + any(x != biased_token_id for x in unbiased_tokens), + f"Expected some tokens to be different from {biased_token_id} in second response, but got {unbiased_tokens}", + ) + + def test_get_server_info_concurrent(self): + """Make sure the concurrent get_server_info doesn't crash the server.""" + tp = ThreadPoolExecutor(max_workers=30) + + def s(): + server_info = requests.get(self.base_url + "/get_server_info") + server_info.json() + + futures = [] + for _ in range(4): + futures.append(tp.submit(s)) + + for f in futures: + f.result() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_engine.py b/test/srt/test_srt_engine.py new file mode 100644 index 000000000..a50669d48 --- /dev/null +++ b/test/srt/test_srt_engine.py @@ -0,0 +1,219 @@ +""" +Usage: +python3 -m unittest test_srt_engine.TestSRTEngine.test_4_sync_async_stream_combination +""" + +import asyncio +import json +import unittest +from types import SimpleNamespace + +import torch + +import sglang as sgl +from sglang.bench_offline_throughput import BenchArgs, throughput_test +from sglang.srt.hf_transformers_utils import get_tokenizer +from sglang.srt.server_args import ServerArgs +from sglang.test.few_shot_gsm8k_engine import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST, + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, +) + + +class TestSRTEngine(CustomTestCase): + + def test_1_engine_runtime_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine(model_path=model_path, random_seed=42) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, random_seed=42) + out2 = json.loads(runtime.generate(prompt, sampling_params))["text"] + runtime.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_2_engine_runtime_encode_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + + engine = sgl.Engine(model_path=model_path, is_embedding=True, random_seed=42) + out1 = torch.tensor(engine.encode(prompt)["embedding"]) + engine.shutdown() + + runtime = sgl.Runtime(model_path=model_path, is_embedding=True, random_seed=42) + out2 = torch.tensor(json.loads(runtime.encode(prompt))["embedding"]) + runtime.shutdown() + + self.assertTrue(torch.allclose(out1, out2, atol=1e-5, rtol=1e-3)) + + def test_3_engine_token_ids_consistency(self): + # just to ensure there is no issue running multiple generate calls + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, random_seed=42, disable_radix_cache=True + ) + out1 = engine.generate(prompt, sampling_params)["text"] + + tokenizer = get_tokenizer(model_path) + token_ids = tokenizer.encode(prompt) + out2 = engine.generate(input_ids=token_ids, sampling_params=sampling_params)[ + "text" + ] + + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_4_sync_async_stream_combination(self): + prompt = "AI safety is" + sampling_params = {"temperature": 0.8, "top_p": 0.95} + + # Create an LLM. + llm = sgl.Engine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + + if True: + # 1. sync + non streaming + print("\n\n==== 1. sync + non streaming ====") + output = llm.generate(prompt, sampling_params) + print(output["text"]) + + # 2. sync + streaming + print("\n\n==== 2. sync + streaming ====") + output_generator = llm.generate(prompt, sampling_params, stream=True) + offset = 0 + for output in output_generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + if True: + loop = asyncio.get_event_loop() + # 3. async + non_streaming + print("\n\n==== 3. async + non streaming ====") + output = loop.run_until_complete( + llm.async_generate(prompt, sampling_params) + ) + print(output["text"]) + + # 4. async + streaming + async def async_streaming(engine): + generator = await engine.async_generate( + prompt, sampling_params, stream=True + ) + + offset = 0 + async for output in generator: + print(output["text"][offset:], end="", flush=True) + offset = len(output["text"]) + print() + + print("\n\n==== 4. async + streaming ====") + loop.run_until_complete(async_streaming(llm)) + + llm.shutdown() + + def test_5_gsm8k(self): + + args = SimpleNamespace( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + local_data_path=None, + num_shots=5, + num_questions=1400, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["accuracy"], 0.33) + + def test_6_engine_cpu_offload(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + ) + out1 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + engine = sgl.Engine( + model_path=model_path, + random_seed=42, + max_total_tokens=128, + cpu_offload_gb=3, + ) + out2 = engine.generate(prompt, sampling_params)["text"] + engine.shutdown() + + print("==== Answer 1 ====") + print(out1) + + print("==== Answer 2 ====") + print(out2) + self.assertEqual(out1, out2) + + def test_7_engine_offline_throughput(self): + server_args = ServerArgs( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + ) + bench_args = BenchArgs(num_prompts=10) + result = throughput_test(server_args=server_args, bench_args=bench_args) + self.assertGreater(result["total_throughput"], 3000) + + def test_8_engine_async_encode_consistency(self): + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_EMBEDDING_MODEL_NAME_FOR_TEST + + engine = sgl.Engine( + model_path=model_path, + is_embedding=True, + random_seed=42, + disable_radix_cache=True, + ) + + # Get sync and async embeddings + out1 = torch.tensor(engine.encode(prompt)["embedding"]) + loop = asyncio.get_event_loop() + out2 = torch.tensor( + loop.run_until_complete(engine.async_encode(prompt))["embedding"] + ) + + engine.shutdown() + + print("\n==== Shapes ====") + print(f"sync shape: {out1.shape}") + print(f"async shape: {out2.shape}") + + self.assertTrue( + torch.allclose(out1, out2, atol=1e-5, rtol=1e-3), + "Sync and async embeddings are not equal within tolerance", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_srt_engine_with_quant_args.py b/test/srt/test_srt_engine_with_quant_args.py new file mode 100644 index 000000000..47baf5688 --- /dev/null +++ b/test/srt/test_srt_engine_with_quant_args.py @@ -0,0 +1,60 @@ +import unittest + +import sglang as sgl +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST, CustomTestCase + + +class TestSRTEngineWithQuantArgs(CustomTestCase): + + def test_1_quantization_args(self): + + # we only test fp8 because other methods are currently dependent on vllm. We can add other methods back to test after vllm dependency is resolved. + quantization_args_list = [ + # "awq", + "fp8", + # "gptq", + # "marlin", + # "gptq_marlin", + # "awq_marlin", + # "bitsandbytes", + # "gguf", + ] + + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + for quantization_args in quantization_args_list: + engine = sgl.Engine( + model_path=model_path, random_seed=42, quantization=quantization_args + ) + engine.generate(prompt, sampling_params) + engine.shutdown() + + def test_2_torchao_args(self): + + # we don't test int8dq because currently there is conflict between int8dq and capture cuda graph + torchao_args_list = [ + # "int8dq", + "int8wo", + "fp8wo", + "fp8dq-per_tensor", + "fp8dq-per_row", + ] + [f"int4wo-{group_size}" for group_size in [32, 64, 128, 256]] + + prompt = "Today is a sunny day and I like" + model_path = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + sampling_params = {"temperature": 0, "max_new_tokens": 8} + + for torchao_config in torchao_args_list: + engine = sgl.Engine( + model_path=model_path, random_seed=42, torchao_config=torchao_config + ) + engine.generate(prompt, sampling_params) + engine.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_standalone_speculative_decoding.py b/test/srt/test_standalone_speculative_decoding.py new file mode 100644 index 000000000..e2962b716 --- /dev/null +++ b/test/srt/test_standalone_speculative_decoding.py @@ -0,0 +1,115 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k +from sglang.test.test_utils import ( + DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST, + DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +GSM_DATASET_PATH = None + + +# Default server arguments shared across all tests +DEFAULT_SERVER_ARGS = [ + "--trust-remote-code", + "--cuda-graph-max-bs", + "8", + "--speculative-algorithm", + "STANDALONE", + "--speculative-draft-model-path", + DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST, + "--speculative-num-steps", + "4", + "--speculative-eagle-topk", + "2", + "--speculative-num-draft-tokens", + "7", + "--mem-fraction-static", + 0.7, +] + + +class TestStandaloneSpeculativeDecodingBase(CustomTestCase): + + model = DEFAULT_STANDALONE_SPECULATIVE_TARGET_MODEL_FOR_TEST + draft_model = DEFAULT_STANDALONE_SPECULATIVE_DRAFT_MODEL_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + accuracy_threshold = 0.7 # derived tests need to override this + spec_decode_threshold = 3.6 # derived spec decoding tests need to override this + + @classmethod + def get_server_args(cls): + """Return the arguments for the server launch. Override in subclasses.""" + return DEFAULT_SERVER_ARGS + ["--attention-backend", "fa3"] + + @classmethod + def setUpClass(cls): + # disable deep gemm precompile to make launch server faster + # please don't do this if you want to make your inference workload faster + os.environ["SGL_JIT_DEEPGEMM_PRECOMPILE"] = "false" + os.environ["SGL_ENABLE_JIT_DEEPGEMM"] = "false" + model = cls.model + cls.process = popen_launch_server( + model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=cls.get_server_args(), + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + num_shots=4, + num_questions=100, + max_new_tokens=512, + parallel=128, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + data_path=GSM_DATASET_PATH, + ) + metrics = run_eval_few_shot_gsm8k(args) + print(f"{metrics=}") + + # Use the appropriate metric key based on the test class + metric_key = "accuracy" + self.assertGreater(metrics[metric_key], self.accuracy_threshold) + + server_info = requests.get(self.base_url + "/get_server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, self.spec_decode_threshold) + + +class TestStandaloneSpeculativeDecodingTriton(TestStandaloneSpeculativeDecodingBase): + + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "triton"] + + +class TestStandaloneSpeculativeDecodingFlashinfer( + TestStandaloneSpeculativeDecodingBase +): + @classmethod + def get_server_args(cls): + return DEFAULT_SERVER_ARGS + ["--attention-backend", "flashinfer"] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_start_profile.py b/test/srt/test_start_profile.py new file mode 100644 index 000000000..60f5f7960 --- /dev/null +++ b/test/srt/test_start_profile.py @@ -0,0 +1,115 @@ +""" +Usage: +python3 -m unittest test_srt_engine.TestSRTEngine.test_4_sync_async_stream_combination +""" + +import os +import shutil +import unittest + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +OUTPUT_DIR = "./profiler_dir" + + +class TestStartProfile(CustomTestCase): + + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def setUp(self): + self._clear_profile_dir() + + def test_start_profile_1(self): + """Test /start_profile with start_step and num_steps argument. This have to be the first test for start_step to work""" + response = self._start_profile(start_step="15", num_steps=5) + + self._post_request() + + self._check_non_empty_profile_dir() + + def test_start_profile_2(self): + """Test /start_profile with no argument""" + response = self._start_profile() + + self._post_request() + + # Before /stop_profile, the profile directory should be empty + self._check_empty_profile_dir() + + # Post /stop_profile and check the profile directory is non-empty + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/stop_profile", + ) + self._check_non_empty_profile_dir() + + def test_start_profile_3(self): + """Test /start_profile with num_steps argument""" + response = self._start_profile(num_steps=5) + + self._post_request() + + self._check_non_empty_profile_dir() + + def _start_profile(self, **kwargs): + """Start profiling with optional parameters.""" + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/start_profile", + json=kwargs if kwargs else None, + ) + self.assertEqual(response.status_code, 200) + + def _post_request(self): + response = requests.post( + f"{DEFAULT_URL_FOR_TEST}/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + }, + }, + ) + self.assertEqual(response.status_code, 200) + + def _clear_profile_dir(self): + if os.path.isdir(OUTPUT_DIR): + # Remove the directory and all its contents + shutil.rmtree(OUTPUT_DIR) + + def _check_non_empty_profile_dir(self): + self.assertTrue(os.path.isdir(OUTPUT_DIR), "Output directory does not exist.") + self.assertNotEqual( + len(os.listdir(OUTPUT_DIR)), 0, "Output directory is empty!" + ) + + def _check_empty_profile_dir(self): + if os.path.isdir(OUTPUT_DIR): + self.assertEqual( + len(os.listdir(OUTPUT_DIR)), 0, "Output directory is non-empty!" + ) + + +if __name__ == "__main__": + os.environ["SGLANG_TORCH_PROFILER_DIR"] = OUTPUT_DIR + unittest.main() diff --git a/test/srt/test_swa_unittest.py b/test/srt/test_swa_unittest.py new file mode 100644 index 000000000..128462029 --- /dev/null +++ b/test/srt/test_swa_unittest.py @@ -0,0 +1,182 @@ +import unittest + +import torch + +from sglang.srt.mem_cache.allocator import SWAKVPool, SWATokenToKVPoolAllocator +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import SWARadixCache + + +class TestSWA(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + + @classmethod + def tearDownClass(cls): + pass + + def test_swa_memory_pool(self): + size = 16 + size_swa = 16 + num_head = 8 + head_dim = 128 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [i for i in range(0, num_layers, global_interval)] + full_attention_layer_ids_set = set(full_attention_layer_ids) + swa_attention_layer_ids = [ + i for i in range(num_layers) if i not in full_attention_layer_ids_set + ] + pool = SWAKVPool( + size=size, + size_swa=size_swa, + dtype=dtype, + num_head=num_head, + head_dim=head_dim, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + device=device, + ) + alloc = SWATokenToKVPoolAllocator( + size=size, size_swa=size_swa, dtype=dtype, device=device, kvcache=pool + ) + assert alloc.available_size() == size + size_swa + index = alloc.alloc(1) + assert alloc.available_size() == size_swa + size_swa - 2 + alloc.free_swa(index) + result = alloc.translate_loc_from_full_to_swa(index) + print(result) + + def test_swa_radix_cache_1(self): + # args + req_size = 10 + max_context_len = 128 + kv_size = 128 + kv_size_swa = 64 + sliding_window_size = 4 + num_head = 8 + head_dim = 128 + num_layers = 48 + global_interval = 4 + dtype = torch.bfloat16 + device = "cuda" + full_attention_layer_ids = [i for i in range(0, num_layers, global_interval)] + full_attention_layer_ids_set = set(full_attention_layer_ids) + swa_attention_layer_ids = [ + i for i in range(num_layers) if i not in full_attention_layer_ids_set + ] + # setup req to token pool + req_to_token_pool = ReqToTokenPool( + size=req_size, + max_context_len=max_context_len, + device=device, + enable_memory_saver=False, + ) + # setup kv pool + kv_pool = SWAKVPool( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + num_head=num_head, + head_dim=head_dim, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + device=device, + ) + # setup token to kv pool allocator + allocator = SWATokenToKVPoolAllocator( + size=kv_size, + size_swa=kv_size_swa, + dtype=dtype, + device=device, + kvcache=kv_pool, + ) + # setup radix cache + tree = SWARadixCache( + req_to_token_pool=req_to_token_pool, + token_to_kv_pool_allocator=allocator, + sliding_window_size=sliding_window_size, + page_size=1, + disable=False, + ) + + # test + print( + f"[Start] allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req1_token_ids, req1_kv_indices = [1, 2, 3], allocator.alloc(3) + assert len(req1_token_ids) == len(req1_kv_indices) + print( + f"req1: inserting, req1_token_ids: {req1_token_ids}, req1_kv_indices: {req1_kv_indices}" + ) + prefix_len = tree.insert(req1_token_ids, req1_kv_indices) + print( + f"req1: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req2_token_ids, req2_kv_indices = [1, 2, 3, 4, 5, 6, 7], allocator.alloc(7) + assert len(req2_token_ids) == len(req2_kv_indices) + print( + f"req2: inserting, req2_token_ids: {req2_token_ids}, req2_kv_indices: {req2_kv_indices}" + ) + prefix_len = tree.insert(req2_token_ids, req2_kv_indices) + print( + f"req2: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req3_token_ids, req3_kv_indices = [10, 11, 12], allocator.alloc(3) + assert len(req3_token_ids) == len(req3_kv_indices) + print( + f"req3: inserting, req3_token_ids: {req3_token_ids}, req3_kv_indices: {req3_kv_indices}" + ) + prefix_len = tree.insert(req3_token_ids, req3_kv_indices) + print( + f"req3: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + req4_token_ids, req4_kv_indices = [1, 2, 3, 4, 5, 60, 70], allocator.alloc(7) + assert len(req4_token_ids) == len(req4_kv_indices) + print( + f"req4: inserting, req4_token_ids: {req4_token_ids}, req4_kv_indices: {req4_kv_indices}" + ) + prefix_len = tree.insert(req4_token_ids, req4_kv_indices) + print( + f"req4: prefix_len: {prefix_len}, allocator swa available size: {allocator.swa_available_size()}, full available size: {allocator.full_available_size()}" + ) + + tree.pretty_print() + full_num_tokens, swa_num_tokens = 1, 0 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 0, 1 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + full_num_tokens, swa_num_tokens = 1, 2 + print(f"evicting {full_num_tokens} full token and {swa_num_tokens} swa token") + tree.evict(full_num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens) + tree.pretty_print() + + req5_token_ids = [1, 2, 3, 4, 5] + kv_indices, last_node = tree.match_prefix(req5_token_ids) + print( + f"req5: token_ids: {req5_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 0 + + req6_token_ids = [1, 2, 3, 4, 5, 60, 70] + kv_indices, last_node = tree.match_prefix(req6_token_ids) + print( + f"req6: token_ids: {req6_token_ids}, matched kv_indices: {kv_indices}, last_node.key: {last_node.key}" + ) + assert len(kv_indices) == 7 + assert len(last_node.key) == 2 + assert last_node.key[0] == 60 + assert last_node.key[1] == 70 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_tokenizer_batch_encode.py b/test/srt/test_tokenizer_batch_encode.py new file mode 100644 index 000000000..f3294c049 --- /dev/null +++ b/test/srt/test_tokenizer_batch_encode.py @@ -0,0 +1,121 @@ +""" +Unit tests for enable_tokenizer_batch_encode feature. + +This tests the batch tokenization functionality which allows processing +multiple text inputs in a single batch for improved performance. + +Usage: +python3 -m unittest test_tokenizer_batch_encode.TestTokenizerBatchEncode.test_batch_validation_constraints +python3 -m unittest test_tokenizer_batch_encode.TestTokenizerBatchEncodeUnit.test_batch_tokenize_and_process_logic +python3 -m unittest test_tokenizer_batch_encode.TestTokenizerBatchEncodeLogic.test_batch_processing_path +""" + +import asyncio +import unittest +from typing import List +from unittest.mock import AsyncMock, Mock, call, patch + +from sglang.srt.managers.io_struct import GenerateReqInput, TokenizedGenerateReqInput +from sglang.srt.managers.tokenizer_manager import TokenizerManager +from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class TestTokenizerBatchEncode(unittest.TestCase): + """Test cases for tokenizer batch encoding validation and setup.""" + + def setUp(self): + """Set up test fixtures.""" + self.server_args = ServerArgs( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + enable_tokenizer_batch_encode=True, + ) + self.port_args = PortArgs.init_new(self.server_args) + + with patch("zmq.asyncio.Context"), patch( + "sglang.srt.utils.get_zmq_socket" + ), patch("sglang.srt.hf_transformers_utils.get_tokenizer") as mock_tokenizer: + + mock_tokenizer.return_value = Mock(vocab_size=32000) + self.tokenizer_manager = TokenizerManager(self.server_args, self.port_args) + + def test_batch_encode_enabled(self): + """Test that batch encoding is enabled when configured.""" + self.assertTrue(self.server_args.enable_tokenizer_batch_encode) + + def test_batch_encode_disabled(self): + """Test that batch encoding can be disabled.""" + server_args_disabled = ServerArgs( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + enable_tokenizer_batch_encode=False, + ) + self.assertFalse(server_args_disabled.enable_tokenizer_batch_encode) + + def test_multimodal_input_validation(self): + """Test that multimodal inputs are rejected in batch mode.""" + req = GenerateReqInput(text="test", image_data=["dummy"]) + req.contains_mm_input = Mock(return_value=True) + + batch_obj = Mock() + batch_obj.__getitem__ = lambda self, i: req + + self.tokenizer_manager.is_generation = True + + with self.assertRaises(ValueError) as cm: + self.tokenizer_manager._validate_batch_tokenization_constraints( + 1, batch_obj + ) + + self.assertIn("multimodal", str(cm.exception)) + + def test_pretokenized_input_validation(self): + """Test that pre-tokenized inputs are rejected in batch mode.""" + req = GenerateReqInput(input_ids=[1, 2, 3]) + + batch_obj = Mock() + batch_obj.__getitem__ = lambda self, i: req + + with self.assertRaises(ValueError) as cm: + self.tokenizer_manager._validate_batch_tokenization_constraints( + 1, batch_obj + ) + + self.assertIn("pre-tokenized", str(cm.exception)) + + def test_input_embeds_validation(self): + """Test that input embeds are rejected in batch mode.""" + req = GenerateReqInput(input_embeds=[0.1, 0.2]) + + batch_obj = Mock() + batch_obj.__getitem__ = lambda self, i: req + + with self.assertRaises(ValueError) as cm: + self.tokenizer_manager._validate_batch_tokenization_constraints( + 1, batch_obj + ) + + self.assertIn("input_embeds", str(cm.exception)) + + def test_valid_text_only_requests_pass_validation(self): + """Test that valid text-only requests pass validation.""" + # Create valid requests (text-only) + requests = [] + for i in range(3): + req = GenerateReqInput(text=f"test text {i}") + req.contains_mm_input = Mock(return_value=False) + requests.append(req) + + batch_obj = Mock() + batch_obj.__getitem__ = Mock(side_effect=lambda i: requests[i]) + + # Should not raise any exception + try: + self.tokenizer_manager._validate_batch_tokenization_constraints( + 3, batch_obj + ) + except Exception as e: + self.fail(f"Validation failed for valid text-only requests: {e}") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/test_torch_compile.py b/test/srt/test_torch_compile.py new file mode 100644 index 000000000..36815718a --- /dev/null +++ b/test/srt/test_torch_compile.py @@ -0,0 +1,80 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +class TestTorchCompile(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--cuda-graph-max-bs", "4"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + + if is_in_amd_ci(): + self.assertGreaterEqual(throughput, 145) + else: + self.assertGreaterEqual(throughput, 152) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_compile_moe.py b/test/srt/test_torch_compile_moe.py new file mode 100644 index 000000000..8bc7b45d3 --- /dev/null +++ b/test/srt/test_torch_compile_moe.py @@ -0,0 +1,77 @@ +import time +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import is_cuda, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchCompileMoe(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MOE_MODEL_NAME_FOR_TEST_BASE + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-torch-compile", "--torch-compile-max-bs", "4"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.50) + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + "ignore_eos": True, + }, + }, + ) + return response.json() + + def test_throughput(self): + # Warmup + res = self.run_decode(16) + + max_tokens = 256 + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(f"{res=}") + throughput = max_tokens / (tok - tic) + if is_cuda(): + self.assertGreaterEqual(throughput, 285) + else: + self.assertGreaterEqual(throughput, 270) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_native_attention_backend.py b/test/srt/test_torch_native_attention_backend.py new file mode 100644 index 000000000..5bf012cca --- /dev/null +++ b/test/srt/test_torch_native_attention_backend.py @@ -0,0 +1,47 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchNativeAttnBackend(CustomTestCase): + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "torch_native"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torch_tp.py b/test/srt/test_torch_tp.py new file mode 100644 index 000000000..dca2612d4 --- /dev/null +++ b/test/srt/test_torch_tp.py @@ -0,0 +1,30 @@ +import unittest + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + CustomTestCase, + is_in_ci, + run_bench_offline_throughput, +) + + +class TestTorchTP(CustomTestCase): + def test_torch_native_llama(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + [ + "--tp", + "2", + # This cannot run anymore with the new torch version. + # "--json-model-override-args", + # '{"architectures": ["TorchNativeLlamaForCausalLM"]}', + "--disable-cuda-graph", + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_torchao.py b/test/srt/test_torchao.py new file mode 100644 index 000000000..13c7b60b5 --- /dev/null +++ b/test/srt/test_torchao.py @@ -0,0 +1,74 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestTorchAO(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--torchao-config", "int4wo-128"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + assert metrics["score"] >= 0.60 + + def run_decode(self, max_new_tokens): + response = requests.post( + self.base_url + "/generate", + json={ + "text": "The capital of France is", + "sampling_params": { + "temperature": 0, + "max_new_tokens": max_new_tokens, + }, + "ignore_eos": True, + }, + ) + return response.json() + + def test_throughput(self): + import time + + max_tokens = 256 + + tic = time.perf_counter() + res = self.run_decode(max_tokens) + tok = time.perf_counter() + print(res["text"]) + throughput = max_tokens / (tok - tic) + print(f"Throughput: {throughput} tokens/s") + assert throughput >= 210 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_backend.py b/test/srt/test_triton_attention_backend.py new file mode 100644 index 000000000..05725301a --- /dev/null +++ b/test/srt/test_triton_attention_backend.py @@ -0,0 +1,66 @@ +""" +Usage: +python3 -m unittest test_triton_attention_backend.TestTritonAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, + run_bench_offline_throughput, +) + + +class TestTritonAttnBackend(CustomTestCase): + def test_latency(self): + output_throughput = run_bench_offline_throughput( + DEFAULT_MODEL_NAME_FOR_TEST, + [ + "--attention-backend", + "triton", + "--enable-torch-compile", + "--cuda-graph-max-bs", + 4, + ], + ) + + print(f"{output_throughput=}") + + if is_in_ci(): + self.assertGreater(output_throughput, 153) + + def test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "triton"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_kernels.py b/test/srt/test_triton_attention_kernels.py new file mode 100644 index 000000000..b15684f9a --- /dev/null +++ b/test/srt/test_triton_attention_kernels.py @@ -0,0 +1,576 @@ +import random +import unittest + +import torch +import torch.nn.functional as F + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd, + decode_attention_fwd_grouped, + decode_attention_fwd_normal, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.test.test_utils import CustomTestCase + + +def extend_attention_fwd_torch( + q: torch.Tensor, # [extend_tokens, H_Q, D] + k: torch.Tensor, # [extend_tokens, H_KV, D] + v: torch.Tensor, # [extend_tokens, H_KV, D] + o: torch.Tensor, # [extend_tokens, H_Q, D] + k_cache: torch.Tensor, # [total_tokens, H_KV, D] + v_cache: torch.Tensor, # [total_tokens, H_KV, D] + qo_indptr: torch.Tensor, # [B+1] + kv_indptr: torch.Tensor, # [B+1] + kv_indices: torch.Tensor, # [prefix_tokens] + sliding_window_size: int, +): + B = qo_indptr.size(0) - 1 + _, H_Q, D = q.shape + _, H_KV, _ = k.shape + + group_size = H_Q // H_KV + scale = 1.0 / D**0.5 + + for i in range(B): + q_start = int(qo_indptr[i].item()) + q_end = int(qo_indptr[i + 1].item()) + kv_start = int(kv_indptr[i].item()) + kv_end = int(kv_indptr[i + 1].item()) + + prefix_indices = kv_indices[kv_start:kv_end] + k_prefix = k_cache[prefix_indices] # [prefix_len, H_KV, D] + v_prefix = v_cache[prefix_indices] # [prefix_len, H_KV, D] + + k_extend = k[q_start:q_end] # [extend_len, H_KV, D] + v_extend = v[q_start:q_end] # [extend_len, H_KV, D] + q_extend = q[q_start:q_end] # [extend_len, H_Q, D] + + k_full = torch.cat([k_prefix, k_extend], dim=0) # [total_len, H_KV, D] + v_full = torch.cat([v_prefix, v_extend], dim=0) # [total_len, H_KV, D] + + if group_size != 1: + k_full_hq = k_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + v_full_hq = v_full.repeat_interleave( + group_size, dim=1 + ) # [total_len, H_Q, D] + else: + k_full_hq = k_full + v_full_hq = v_full + + prefix_len = k_prefix.size(0) + extend_len = k_extend.size(0) + total_len = prefix_len + extend_len + + # causal + pos_keys = torch.arange(total_len, device=q.device) + t = prefix_len + torch.arange(extend_len, device=q.device) # [extend_len] + causal_mask = pos_keys.unsqueeze(0) <= t.unsqueeze(1) + + # sliding window + if sliding_window_size is not None and sliding_window_size > 0: + start = (t - (sliding_window_size)).clamp_min(0) # [extend_len] + else: + start = torch.zeros_like(t) + window_mask = pos_keys.unsqueeze(0) >= start.unsqueeze(1) + + final_mask = causal_mask & window_mask + + attn_scores = ( + torch.einsum("qhd,khd->qhk", q_extend, k_full_hq) * scale + ) # [extend_len, H_Q, total_len] + attn_scores = attn_scores.masked_fill(~final_mask.unsqueeze(1), float("-inf")) + + attn_weights = F.softmax(attn_scores, dim=-1) + o[q_start:q_end] = torch.einsum("qhk,khd->qhd", attn_weights, v_full_hq) + + +class TestTritonAttention(CustomTestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D): + dtype = torch.bfloat16 + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32, device="cuda") + b_start_loc = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device="cuda" + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + o_extend_mask = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + o_redundant = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + custom_mask = None + mask_indptr = None + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + True, + mask_indptr, + max_len_extend, + ) + + b_seq_mask_len = b_seq_len_extend * b_seq_len + custom_mask = torch.ones( + (b_seq_mask_len.sum().item(),), dtype=torch.bool, device="cuda" + ) + mask_indptr = torch.zeros((B + 1,), dtype=torch.int64, device="cuda") + mask_indptr[1 : B + 1] = torch.cumsum(b_seq_mask_len[:B], dim=0) + for i in range(B): + causal_mask = ( + torch.tril( + torch.ones(b_seq_len_extend[i], b_seq_len_extend[i]), diagonal=0 + ) + == 1 + ) + prefix_mask = torch.ones( + b_seq_len_extend[i], b_seq_len_prefix[i], dtype=torch.bool + ) + mask_flatten = torch.cat([prefix_mask, causal_mask], dim=1).flatten() + custom_mask[mask_indptr[i] : mask_indptr[i + 1]] = mask_flatten + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend_mask, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + True, + mask_indptr, + max_len_extend, + ) + + redundant_attention( + q_extend, + o_redundant, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + + self.assertTrue(torch.allclose(o_extend, o_redundant, rtol=1e-2)) + self.assertTrue(torch.allclose(o_extend_mask, o_redundant, rtol=1e-2)) + + def test_extend_attention(self): + + # Define the varying parameter values + attention_values = [128, 96, 80, 13] + + # Loop through the values and call the method + for value in attention_values: + self._test_extend_attention_once(19, 12331, 12, 4, value) + + def _test_extend_attention_sliding_window_once( + self, B, N_CTX, H_Q, H_KV, D, WINDOW_SIZE + ): + dtype = torch.bfloat16 + + b_seq_len_prefix = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len_extend = torch.randint( + 1, N_CTX // 2, (B,), dtype=torch.int32, device="cuda" + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + + b_start_loc = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device="cuda" + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + o_extend_triton = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + o_extend_torch = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend_triton, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask=None, + is_causal=True, + mask_indptr=None, + max_len_extend=max_len_extend, + sliding_window_size=WINDOW_SIZE, + ) + + extend_attention_fwd_torch( + q_extend, + k_extend, + v_extend, + o_extend_torch, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + WINDOW_SIZE, + ) + + self.assertTrue( + torch.allclose(o_extend_triton, o_extend_torch, rtol=1e-3, atol=1e-3) + ) + + def test_extend_attention_sliding_window(self): + window_sizes = [-1, 127] + for window_size in window_sizes: + self._test_extend_attention_sliding_window_once( + 19, 12331, 64, 8, 128, window_size + ) + + def _test_context_attention_once(self, head_dim, is_causal): + # Set up a simple test case + num_heads = 4 + seq_lens = [8, 12] + max_seq_len = max(seq_lens) + + # Create random input tensors + q = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + k = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + v = torch.randn(sum(seq_lens), num_heads, head_dim, device="cuda") + o = torch.zeros(sum(seq_lens), num_heads, head_dim, device="cuda") + + # Create b_start_loc and b_seq_len tensors + b_start_loc = torch.tensor([0, seq_lens[0]], device="cuda") + b_seq_len = torch.tensor(seq_lens, device="cuda") + + context_attention_fwd( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + + cu_seq_lens = [0] * (len(seq_lens) + 1) + for i, seq_len in enumerate(seq_lens): + cu_seq_lens[i + 1] = cu_seq_lens[i] + seq_len + + for i in range(len(seq_lens)): + start, end = cu_seq_lens[i], cu_seq_lens[i + 1] + o_torch = torch.nn.functional.scaled_dot_product_attention( + q[start:end].permute(1, 0, 2), + k[start:end].permute(1, 0, 2), + v[start:end].permute(1, 0, 2), + is_causal=is_causal, + ).permute(1, 0, 2) + + cos_sim = torch.nn.functional.cosine_similarity( + o[start:end].flatten(), o_torch.flatten(), dim=0 + ) + self.assertTrue(cos_sim.item() > 1 - (1e-5)) + self.assertTrue(torch.allclose(o[start:end], o_torch, atol=1e-2)) + + def test_context_attention(self): + head_dim = [128, 96, 80, 13] + + for dim in head_dim: + for is_causal in [True, False]: + self._test_context_attention_once(dim, is_causal) + + def _test_decode_attention_once(self, B, H_Q, H_KV, D): + dtype = torch.bfloat16 + seq_len = 10 # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + def test_decode_attention(self): + # Here we just to ensure there is no error + # TODO: correctnesss test + + # Test configurations + configs = [ + (2, 4, 4, 64), # MHA + (2, 4, 2, 64), # GQA + (2, 4, 4, 80), # Non-standard head dim + (2, 4, 4, 13), # Prime number head dim + ] + + for B, H_Q, H_KV, D in configs: + self._test_decode_attention_once(B, H_Q, H_KV, D) + + def _test_grouped_decode_attention_once(self, B, S, H_Q, H_KV, D, D_V): + dtype = torch.bfloat16 + seq_len = S # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device="cuda") + + # o will have the same shape as q + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + o_grouped = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + b_seq_len = torch.full((B,), seq_len, device="cuda") + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len[:B], dim=0) + kv_indices = torch.arange(total_tokens, device="cuda") + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd_normal( + q, + k_buffer, + v_buffer, + o, + kv_indptr, + kv_indices, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + attn_logits1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + attn_lse1 = torch.empty( + (B, H_Q, max_kv_splits, D_V), + dtype=torch.float32, + device="cuda", + ) + + decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o_grouped, + kv_indptr, + kv_indices, + attn_logits1, + attn_lse1, + num_kv_splits, + max_kv_splits, + sm_scale, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_grouped.flatten(), dim=0 + ) + print(cos_sim.item()) + self.assertTrue(cos_sim.item() > 0.99) + self.assertTrue(torch.allclose(o, o_grouped, atol=3e-2)) + + def test_grouped_decode_attention(self): + seq_lens = [5, 100, 128, 500] + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 64, 64), + (2, 64, 1, 13, 13), + (2, 128, 1, 80, 80), + (2, 128, 2, 512, 512), + (2, 128, 1, 576, 512), + ] + + for S in seq_lens: + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_attention_rocm_mla.py b/test/srt/test_triton_attention_rocm_mla.py new file mode 100644 index 000000000..f3074aeeb --- /dev/null +++ b/test/srt/test_triton_attention_rocm_mla.py @@ -0,0 +1,259 @@ +import random +import unittest + +import torch + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.rocm_mla_decode_rope import ( + decode_attention_fwd_grouped_rope, +) +from sglang.srt.layers.rotary_embedding import DeepseekScalingRotaryEmbedding +from sglang.test.test_utils import CustomTestCase + + +class TestTritonAttentionMLA(CustomTestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def preprocess_kv_cache(self, kv_cache, kv_lora_rank): + latent_cache = kv_cache + v_input = latent_cache[..., :kv_lora_rank] + v_input = v_input.contiguous().unsqueeze(1) + k_input = latent_cache.unsqueeze(1) + k_input[..., :kv_lora_rank] = v_input + + return k_input, v_input + + def input_helper( + self, + B, + H, + S, + kv_lora_rank, + rotary_dim, + qk_rope_head_dim, + num_kv_splits, + dtype, + device, + rope_base=10, + rope_max_seq_len=16384, + rope_scaling=1.0, + is_neox_style=False, + ): + q = torch.randn( + B, H, kv_lora_rank + qk_rope_head_dim, device=device, dtype=dtype + ) + kv_cache = torch.randn( + B * S, kv_lora_rank + qk_rope_head_dim, dtype=dtype, device=device + ) + kv_indptr = torch.arange(B + 1, device=device) * S + kv_indices = torch.arange(B * S, device=device) + attn_logits = torch.empty( + B, H, num_kv_splits, kv_lora_rank + 1, dtype=dtype, device=device + ) + rotary_emb = DeepseekScalingRotaryEmbedding( + qk_rope_head_dim, + rotary_dim, + rope_max_seq_len, + rope_base, + is_neox_style, + rope_scaling, + q.dtype, + device="cpu", + ).cuda() + positions = torch.tensor([S], device=device).unsqueeze(0).repeat(B, 1) + + return kv_indptr, kv_indices, q, kv_cache, attn_logits, rotary_emb, positions + + def ref_compute_full_fwd( + self, + q, + k_input, + v_input, + kv_lora_rank, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + rotary_emb, + positions, + use_rope, + device="cuda", + ): + + B, H = q.shape[0], q.shape[1] + S = kv_indptr[1].item() + qk_rope_head_dim = k_input.shape[-1] - kv_lora_rank + + q_input = torch.empty(B, H, kv_lora_rank + qk_rope_head_dim, dtype=q.dtype).to( + device + ) + q_nope_out, q_pe = q.split([kv_lora_rank, qk_rope_head_dim], dim=-1) + k_pe_t = k_input.view(B, 1, S, -1)[:, :, -1:, kv_lora_rank:] + + if use_rope: + q_pe, k_pe_t = rotary_emb(positions, q_pe.unsqueeze(2), k_pe_t) + q_pe = q_pe.squeeze() + + k_input.view(B, 1, S, -1)[:, :, -1:, kv_lora_rank:] = k_pe_t + + q_input[..., :kv_lora_rank] = q_nope_out + q_input[..., kv_lora_rank:] = q_pe + + B, H = q_input.shape[0], q_input.shape[1] + kv_lora_rank = v_input.shape[-1] + device = q_input.device + + attn_logits = torch.empty( + B, H, num_kv_splits, kv_lora_rank + 1, dtype=q_input.dtype, device=device + ) + o = torch.empty(B, H, kv_lora_rank, dtype=q_input.dtype, device=device) + + decode_attention_fwd_grouped( + q_input, + k_input, + v_input, + o, + kv_indptr, + kv_indices, + attn_logits, + num_kv_splits, + sm_scale, + logit_cap, + ) + + return attn_logits, o, k_pe_t.squeeze() + + def _test_rocm_fused_mla_kernel( + self, + B, + H, + S, + kv_lora_rank, + qk_rope_head_dim, + rotary_dim, + dtype, + use_rope, + is_neox_style, + num_kv_splits=2, + sm_scale=1.0, + logit_cap=0.0, + device="cuda", + ): + kv_indptr, kv_indices, q, kv_cache, attn_logits, rotary_emb, positions = ( + self.input_helper( + B, + H, + S, + kv_lora_rank, + rotary_dim, + qk_rope_head_dim, + num_kv_splits, + dtype, + device=device, + is_neox_style=is_neox_style, + ) + ) + + k_input, v_input = self.preprocess_kv_cache(kv_cache, kv_lora_rank) + k_pe_tokens = torch.empty( + B, qk_rope_head_dim, dtype=kv_cache.dtype, device=device + ) + tri_o = torch.empty(B, H, kv_lora_rank, dtype=kv_cache.dtype, device=device) + + decode_attention_fwd_grouped_rope( + q, + k_input, + v_input, + tri_o, + kv_indptr, + kv_indices, + k_pe_tokens if use_rope else None, + kv_lora_rank, + rotary_dim if use_rope else None, + rotary_emb.cos_sin_cache if use_rope else None, + positions if use_rope else None, + attn_logits, + num_kv_splits, + sm_scale, + logit_cap, + use_rope, + is_neox_style, + ) + + tri_logits = attn_logits + + # reference + ref_logits, ref_o, ref_k_pe_tokens = self.ref_compute_full_fwd( + q, + k_input, + v_input, + kv_lora_rank, + kv_indptr, + kv_indices, + num_kv_splits, + sm_scale, + logit_cap, + rotary_emb, + positions, + use_rope, + device="cuda", + ) + + if use_rope: + torch.testing.assert_close( + ref_k_pe_tokens, k_pe_tokens.squeeze(), atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(ref_logits, tri_logits, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(ref_o, tri_o, atol=1e-2, rtol=1e-2) + + def test_grouped_rocm_fused_mla(self): + configs = [ + (1, 128, 2048, 512, 64, 64), + (1, 128, 2048, 512, 128, 64), + (1, 128, 2048, 512, 127, 64), + (1, 128, 2050, 512, 127, 64), + (1, 128, 2050, 512, 128, 64), + (8, 128, 2048, 512, 64, 64), + (8, 128, 2048, 512, 128, 64), + (8, 128, 2048, 512, 127, 64), + (8, 128, 2050, 512, 127, 64), + (8, 128, 2050, 512, 128, 64), + ] + dtypes = [torch.bfloat16, torch.float32] + use_rope_list = [True, False] + is_neox_style_list = [True, False] + + for B, H, S, kv_lora_rank, qk_rope_head_dim, rotary_dim in configs: + for dtype in dtypes: + for use_rope in use_rope_list: + for is_neox_style in is_neox_style_list: + self._test_rocm_fused_mla_kernel( + B, + H, + S, + kv_lora_rank, + qk_rope_head_dim, + rotary_dim, + dtype, + use_rope, + is_neox_style, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_fused_moe.py b/test/srt/test_triton_fused_moe.py new file mode 100644 index 000000000..88d33b5f7 --- /dev/null +++ b/test/srt/test_triton_fused_moe.py @@ -0,0 +1,162 @@ +import unittest + +import torch +import torch.nn.functional as F +from tqdm import tqdm + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.triton_kernels_moe import ( + triton_kernel_moe_forward, +) +from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig +from sglang.srt.layers.moe.topk import TopK +from sglang.test.test_utils import CustomTestCase + + +class TestFusedMOE(CustomTestCase): + NUM_EXPERTS = [8, 64] + TOP_KS = [2, 4] + + @staticmethod + def create_random_cuda_tensor(shape, dtype, mean=0, std=0.01): + """Create a random CUDA tensor + + Args: + shape: Tensor shape + dtype: Data type + mean: Mean value + std: Standard deviation + + Returns: + torch.Tensor: Randomly initialized CUDA tensor + """ + return torch.empty(shape, dtype=dtype, device="cuda").normal_(mean, std) + + def get_tolerance(self, dtype): + """Get tolerance values for different data types + + Args: + dtype: Data type + + Returns: + tuple: (relative tolerance, absolute tolerance) + """ + if dtype == torch.float32: + return 1e-5, 1e-5 + elif dtype in [torch.float16, torch.bfloat16]: + return 1e-5, 1e-5 + else: + return 1e-2, 1e-2 # Default values for other types + + def torch_naive_moe( + self, + a, + w1, + w2, + score, + topk, + ): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + + if w1.dtype == torch.float8_e4m3fn: + w1_compute = w1.to(a.dtype) + w2_compute = w2.to(a.dtype) + else: + w1_compute = w1 + w2_compute = w2 + + for i in range(w1_compute.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()( + a[mask] @ w1_compute[i].transpose(0, 1) + ) @ w2_compute[i].transpose(0, 1) + + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + def _test_case(self, m, n, k, e, topk, dtype): + rtol, atol = self.get_tolerance(dtype) + + a = self.create_random_cuda_tensor((m, k), dtype) + w1 = self.create_random_cuda_tensor((e, 2 * n, k), dtype) + w2 = self.create_random_cuda_tensor((e, k, n), dtype) + w1_tri = w1.clone() + w2_tri = w2.clone() + w1_tri = w1_tri.transpose(-2, -1).contiguous() + w2_tri = w2_tri.transpose(-2, -1).contiguous() + score = self.create_random_cuda_tensor((m, e), dtype) + + topk_op = TopK( + top_k=topk, + renormalize=False, + use_grouped_topk=False, + ) + topk_op.use_triton_kernels = True + triton_topk_output = topk_op.forward_cuda( + hidden_states=a, + router_logits=score, + ) + + moe_runner_config = MoeRunnerConfig( + inplace=False, + ) + triton_output = triton_kernel_moe_forward( + a, w1_tri, w2_tri, triton_topk_output, moe_runner_config + ) + torch_output = self.torch_naive_moe(a, w1, w2, score, topk) + torch.testing.assert_close(triton_output, torch_output, rtol=rtol, atol=atol) + + def test_various_configurations(self): + m_values = [1, 32, 64, 256] + n_values = [128, 1024] + k_values = [128, 512, 1024] + dtypes = [torch.bfloat16] + + # Calculate total number of tests + total_tests = ( + len(m_values) + * len(n_values) + * len(k_values) + * len(self.NUM_EXPERTS) + * len(self.TOP_KS) + * len(dtypes) + ) + + # Create progress bar + with tqdm(total=total_tests, desc="Running MoE tests") as pbar: + for m in m_values: + for n in n_values: + for k in k_values: + for e in self.NUM_EXPERTS: + for topk in self.TOP_KS: + for dtype in dtypes: + with self.subTest( + m=m, + n=n, + k=k, + e=e, + topk=topk, + dtype=dtype, + ): + self._test_case( + m, + n, + k, + e, + topk, + dtype, + ) + torch.cuda.empty_cache() + pbar.update(1) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_triton_moe_channel_fp8_kernel.py b/test/srt/test_triton_moe_channel_fp8_kernel.py new file mode 100644 index 000000000..bbe44308f --- /dev/null +++ b/test/srt/test_triton_moe_channel_fp8_kernel.py @@ -0,0 +1,181 @@ +import itertools +import unittest + +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts +from sglang.srt.layers.quantization.fp8_kernel import scaled_fp8_quant +from sglang.test.test_utils import CustomTestCase + + +def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): + """Matrix multiplication function that supports per-token input quantization and per-column weight quantization""" + A = A.to(torch.float32) + B = B.to(torch.float32) + + assert A.shape[-1] == B.shape[-1], "Dimension mismatch" + assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor" + + # Reshape input + M = A.numel() // A.shape[-1] + B = B.t() # Transpose weight matrix + N, K = B.shape + origin_C_shape = A.shape[:-1] + (K,) + A = A.reshape(M, N) + + # As is per-token [M, 1], Bs is per-column [1, K] + C = torch.matmul(A, B) # [M, K] + C = As * C * Bs.view(1, -1) # Broadcast per-column scale + + return C.reshape(origin_C_shape).to(output_dtype) + + +def fp8_mask(a, mask): + dtype = a.dtype + return a.view(torch.int8)[mask].view(dtype) + + +def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): + """This function performs fused moe with per-column int8 quantization using native torch.""" + + B, D = a.shape + # Perform per-token quantization + a_q, a_s = scaled_fp8_quant(a, use_per_token_if_dynamic=True) + # Repeat tokens to match topk + a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + # Also repeat the scale + a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1] + + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + + # Calculate routing + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + # Process each expert + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + # First MLP layer: note that a_s is now per-token + inter_out = native_w8a8_per_token_matmul( + fp8_mask(a_q, mask), + w1[i], + fp8_mask(a_s, mask), + w1_s[i], + output_dtype=a.dtype, + ) + # Activation function + act_out = SiluAndMul().forward_native(inter_out) + # Quantize activation output with per-token + act_out_q, act_out_s = scaled_fp8_quant( + act_out, use_per_token_if_dynamic=True + ) + + # Second MLP layer + out[mask] = native_w8a8_per_token_matmul( + act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype + ) + # Apply routing weights and sum + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +class TestW8A8FP8FusedMoE(CustomTestCase): + DTYPES = [torch.half, torch.bfloat16] + M = [1, 33] + N = [128, 1024] + K = [256, 4096] + E = [8] + TOP_KS = [2, 6] + BLOCK_SIZE = [[64, 64], [64, 128], [128, 64], [128, 128]] + BLOCK_SIZE = [[128, 128]] + SEEDS = [0] + + @classmethod + def setUpClass(cls): + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA is not available") + torch.set_default_device("cuda") + + def _w8a8_fp8_fused_moe(self, M, N, K, E, topk, block_size, dtype, seed): + torch.manual_seed(seed) + # Initialize int8 quantization parameters + factor_for_scale = 1e-2 + finfo = torch.finfo(torch.float8_e4m3fn) + fp8_max = finfo.max + fp8_min = finfo.min + + # Input tensor + # M * K + a = torch.randn((M, K), dtype=dtype) / 10 + + # Generate int8 weights + w1_fp32 = (torch.rand((E, 2 * N, K), dtype=torch.float32) - 0.5) * 2 + w1 = (w1_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32) - 0.5) * 2 + w2 = (w2_fp32 * fp8_max).clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn) + + # Generate scale for each column (per-column quantization) + w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale + w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale + score = torch.randn((M, E), dtype=dtype) + + with torch.inference_mode(): + ref_out = torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk) + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk, renormalize=False), + ) + out = fused_moe( + a, + w1, + w2, + topk_output, + use_fp8_w8a8=True, # using fp8 + use_int8_w8a16=False, + use_int8_w8a8=False, + per_channel_quant=True, + w1_scale=w1_s, + w2_scale=w2_s, + block_shape=None, # Not using block quantization + ) + + # Check results + self.assertTrue( + torch.mean(torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))) + / torch.mean(torch.abs(ref_out.to(torch.float32))) + < 0.05 + ) + + def test_w8a8_fp8_fused_moe(self): + for params in itertools.product( + self.M, + self.N, + self.K, + self.E, + self.TOP_KS, + self.BLOCK_SIZE, + self.DTYPES, + self.SEEDS, + ): + with self.subTest( + M=params[0], + N=params[1], + K=params[2], + E=params[3], + topk=params[4], + block_size=params[5], + dtype=params[6], + seed=params[7], + ): + self._w8a8_fp8_fused_moe(*params) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/test/srt/test_triton_moe_wna16.py b/test/srt/test_triton_moe_wna16.py new file mode 100644 index 000000000..b447b532f --- /dev/null +++ b/test/srt/test_triton_moe_wna16.py @@ -0,0 +1,243 @@ +from typing import Optional + +import pytest +import torch + +from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.moe.fused_moe_triton.fused_moe import fused_moe +from sglang.srt.layers.moe.topk import TopKConfig, select_experts + +NUM_EXPERTS = [8, 64] +TOP_KS = [2, 6] + + +def quantize_weights( + w: torch.Tensor, + quant_type: str, + group_size: Optional[int], + zero_points: bool = False, + ref_zero_points_after_scales: bool = False, +): + assert quant_type in ["w4a16", "w4a16b8", "w8a16", "w8a16b128"] + assert not zero_points or group_size is not None, ( + "to have group zero points, group_size must be provided " + "(-1 group_size is channelwise)" + ) + + orig_device = w.device + orig_type = w.dtype + size_k, size_n = w.shape + + assert w.is_floating_point(), "w must be float" + + if group_size == -1: + group_size = size_k + + # Reshape to [groupsize, -1] + if group_size is not None and group_size < size_k: + w = w.reshape((-1, group_size, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((group_size, -1)) + + # Compute scale for each group + max_val = torch.max(w, 0, keepdim=True).values + min_val = torch.min(w, 0, keepdim=True).values + + if quant_type == "w4a16": + max_q_val = 15 + min_q_val = 0 + elif quant_type == "w4a16b8": + max_q_val = 7 + min_q_val = -1 + elif quant_type == "w8a16": + max_q_val = 255 + min_q_val = 0 + elif quant_type == "w8a16b128": + max_q_val = 127 + min_q_val = -128 + + w_s = torch.Tensor([1.0]).to(w.device) # unscaled case + maybe_w_zp = None + if group_size is not None: + if zero_points: + w_s = (max_val - min_val).clamp(min=1e-5) / max_q_val + maybe_w_zp = ( + torch.round(torch.abs(min_val / w_s)).clamp(min_q_val, max_q_val).int() + ) + else: + # If the bias is such that there are no possible negative/positive + # values, set the max value to inf to avoid divide by 0 + w_s = torch.max( + abs(max_val / (max_q_val if max_q_val != 0 else torch.inf)), + abs(min_val / (min_q_val if min_q_val != 0 else torch.inf)), + ) + + # Quantize + w_q = torch.round(w / w_s).int() + (maybe_w_zp if zero_points else 0) + w_q = torch.clamp(w_q, min_q_val, max_q_val) + + # Compute ref (dequantized) + # For some kernels (namely Machete) the zero-points are applied after the + # scales are applied, for this case computing the reference in similar way + # allows us to use tighter error tolerances in our unit tests. + if ref_zero_points_after_scales and maybe_w_zp is not None: + w_ref = w_q.to(orig_type) * w_s - maybe_w_zp.to(orig_type) * w_s + else: + w_ref = (w_q - (maybe_w_zp if zero_points else 0)).to(orig_type) * w_s + + if quant_type == "w4a16b8": + w_q += 8 + elif quant_type == "w8a16b128": + w_q += 128 + + # Restore original shapes + if group_size is not None and group_size < size_k: + + def reshape_w(w): + w = w.reshape((group_size, -1, size_n)) + w = w.permute(1, 0, 2) + w = w.reshape((size_k, size_n)).contiguous() + return w + + w_q = reshape_w(w_q) + w_ref = reshape_w(w_ref) + w_s = w_s.reshape((-1, size_n)).contiguous() + + if maybe_w_zp is not None: + maybe_w_zp = maybe_w_zp.reshape((-1, size_n)).contiguous() + maybe_w_zp = maybe_w_zp.to(device=orig_device) + + return ( + w_ref.to(device=orig_device), + w_q.to(device=orig_device), + w_s if group_size is not None else None, + maybe_w_zp, + ) + + +def torch_moe(a, w1, w2, score, topk): + B, D = a.shape + a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D) + out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device) + score = torch.softmax(score, dim=-1, dtype=torch.float32) + topk_weight, topk_ids = torch.topk(score, topk) + topk_weight = topk_weight.view(-1) + topk_ids = topk_ids.view(-1) + for i in range(w1.shape[0]): + mask = topk_ids == i + if mask.sum(): + out[mask] = SiluAndMul()(a[mask] @ w1[i].transpose(0, 1)) @ w2[i].transpose( + 0, 1 + ) + return ( + out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype) + ).sum(dim=1) + + +# fork from https://github.com/vllm-project/vllm/blob/main/tests/kernels/test_moe.py +@pytest.mark.parametrize("m", [1, 32, 222]) +@pytest.mark.parametrize("n", [128, 1024, 2048]) +@pytest.mark.parametrize("k", [128, 1024]) +@pytest.mark.parametrize("e", NUM_EXPERTS) +@pytest.mark.parametrize("topk", TOP_KS) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("group_size", [64, 128]) +@pytest.mark.parametrize("has_zp", [True, False]) +@pytest.mark.parametrize("weight_bits", [8]) # [4, 8]) +def test_fused_moe_wn16( + m: int, + n: int, + k: int, + e: int, + topk: int, + dtype: torch.dtype, + group_size: int, + has_zp: bool, + weight_bits: int, +): + print(m, n, k, e, topk, dtype, group_size, has_zp, weight_bits) + a = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 + w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 + score = torch.randn((m, e), device="cuda", dtype=dtype) + + if weight_bits == 4: + pack_factor = 2 + quant_type = "w4a16" if has_zp else "w4a16b8" + elif weight_bits == 8: + pack_factor = 1 + quant_type = "w8a16" if has_zp else "w8a16b128" + + w1_ref = w1.clone() + w2_ref = w2.clone() + w1_qweight = torch.empty( + (e, 2 * n, k // pack_factor), device="cuda", dtype=torch.uint8 + ) + w2_qweight = torch.empty((e, k, n // pack_factor), device="cuda", dtype=torch.uint8) + w1_scales = torch.empty((e, 2 * n, k // group_size), device="cuda", dtype=dtype) + w2_scales = torch.empty((e, k, n // group_size), device="cuda", dtype=dtype) + w1_qzeros = torch.empty( + (e, 2 * n // pack_factor, k // group_size), device="cuda", dtype=torch.uint8 + ) + w2_qzeros = torch.empty( + (e, k // pack_factor, n // group_size), device="cuda", dtype=torch.uint8 + ) + + for i in range(e * 2): + expert_id = i % e + if i // e == 0: + w, w_ref, w_qweight, w_scales, w_qzeros = ( + w1, + w1_ref, + w1_qweight, + w1_scales, + w1_qzeros, + ) + else: + w, w_ref, w_qweight, w_scales, w_qzeros = ( + w2, + w2_ref, + w2_qweight, + w2_scales, + w2_qzeros, + ) + weight, qweight, scales, qzeros = quantize_weights( + w[expert_id].T, quant_type, group_size, has_zp, False + ) + weight = weight.T + qweight = qweight.T.contiguous().to(torch.uint8) + scales = scales.T + if has_zp: + qzeros = qzeros.T.contiguous().to(torch.uint8) + if weight_bits == 4: + qweight = qweight[:, 1::2] * 16 + qweight[:, ::2] + if has_zp: + qzeros = qzeros[1::2, :] * 16 + qzeros[::2, :] + + w_ref[expert_id] = weight + w_qweight[expert_id] = qweight + w_scales[expert_id] = scales + if has_zp: + w_qzeros[expert_id] = qzeros + + topk_output = select_experts( + hidden_states=a, + router_logits=score, + topk_config=TopKConfig(top_k=topk), + ) + + triton_output = fused_moe( + a, + w1_qweight, + w2_qweight, + topk_output, + use_int4_w4a16=weight_bits == 4, + use_int8_w8a16=weight_bits == 8, + w1_scale=w1_scales, + w2_scale=w2_scales, + w1_zp=w1_qzeros if has_zp else None, + w2_zp=w2_qzeros if has_zp else None, + block_shape=[0, group_size], + ) + torch_output = torch_moe(a, w1_ref, w2_ref, score, topk) + torch.testing.assert_close(triton_output, torch_output, atol=2e-2, rtol=0) diff --git a/test/srt/test_triton_sliding_window.py b/test/srt/test_triton_sliding_window.py new file mode 100644 index 000000000..f2b422c5e --- /dev/null +++ b/test/srt/test_triton_sliding_window.py @@ -0,0 +1,129 @@ +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +class TestSlidingWindowAttentionTriton(CustomTestCase): + """Test sliding window attention functionality with triton backend.""" + + @classmethod + def setUpClass(cls): + """Set up the test server with Gemma3 model and triton backend.""" + # Gemma3 model supports sliding window attention + cls.model = "google/gemma-3-4b-it" + cls.base_url = DEFAULT_URL_FOR_TEST + + cls.common_args = [ + "--trust-remote-code", + "--attention-backend", + "triton", + "--context-length", + "8192", + "--random-seed", + "42", + ] + + cls.short_context_prompt = "The capital of France is" + + # Test prompt longer than window size + cls.long_context_prompt = ( + """ + Once upon a time, there was a mountain. In the mountain, there was a temple. In the temple, there was an old monk telling a story. The story was: + """ + * 100 + ) + cls.long_context_prompt += "\nNow, summarize the story in one sentence:" + + def _test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=200, + num_threads=32, + ) + + metrics = run_eval(args) + print(f"MMLU metrics with sliding window: {metrics}") + + self.assertGreaterEqual(metrics["score"], 0.60) + + def _test_short_context_generation(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": self.short_context_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 256, + }, + }, + ) + + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertIn("paris", result["text"].lower()) + print(f"Short context generation result: {result['text']}") + + def _test_long_context_generation(self): + response = requests.post( + self.base_url + "/generate", + json={ + "text": self.long_context_prompt, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 256, + }, + }, + ) + + self.assertEqual(response.status_code, 200) + result = response.json() + self.assertGreater(len(result["text"].strip()), 0) + print(f"Long context generation result: {result['text'][:100]}...") + + @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") + def test_no_cuda_graph(self): + self.no_cuda_graph_process = popen_launch_server( + self.model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=self.common_args + ["--disable-cuda-graph"], + ) + + try: + self._test_short_context_generation() + self._test_long_context_generation() + self._test_mmlu() + finally: + kill_process_tree(self.no_cuda_graph_process.pid) + + def test_cuda_graph(self): + self.cuda_graph_process = popen_launch_server( + self.model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=self.common_args, + ) + + try: + self._test_short_context_generation() + self._test_long_context_generation() + self._test_mmlu() + finally: + kill_process_tree(self.cuda_graph_process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_two_batch_overlap.py b/test/srt/test_two_batch_overlap.py new file mode 100644 index 000000000..6aa550c46 --- /dev/null +++ b/test/srt/test_two_batch_overlap.py @@ -0,0 +1,152 @@ +import os +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.two_batch_overlap import ( + compute_split_seq_index, + compute_split_token_index, +) +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST, + DEFAULT_MLA_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + + +class TestTwoBatchOverlap(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MLA_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--dp", + "2", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-cuda-graph", # DeepEP normal does not support CUDA Graph + "--enable-two-batch-overlap", + ], + env={"SGL_ENABLE_JIT_DEEPGEMM": "0", **os.environ}, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_generate_single_prompt(self): + response = requests.post( + self.base_url + "/generate", + # we use an uncommon start to minimise the chance that the cache is hit by chance + json={ + "text": "_ 1+1=2, 1+2=3, 1+3=4, 1+4=", + "sampling_params": {"temperature": 0, "max_new_tokens": 8}, + }, + ) + print(f"{response.json()=}") + self.assertEqual(response.json()["text"], "5, 1+5=6") + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.5) + + +class TestTwoBatchOverlapUnitTest(unittest.TestCase): + def test_compute_split_seq_and_token_index(self): + for num_tokens, expect in [ + (0, 0), + (100, 50), + (99, 49), + ]: + actual = compute_split_seq_index( + forward_mode=ForwardMode.DECODE, + num_tokens=num_tokens, + extend_lens=None, + token_num_per_seq=1, + ) + self.assertEqual(actual, expect) + + for extend_lens, expect in [ + ([], (0, 0)), + ([42], (0, 21)), + ([42, 999], (1, 520)), + ([999, 42], (0, 520)), + ([498, 502], (1, 498)), + ([4096, 4096, 4096, 4096], (2, 8192)), + ([4095, 4096, 4096, 4096, 1], (2, 8191)), + ([1, 4095, 4096, 4096, 4096], (3, 8192)), + ([4097, 4096, 4096, 4095, 1], (2, 8193)), + ([1, 1, 1, 1, 99999], (4, 50001)), + ([99999, 1, 1, 1, 1], (0, 50001)), + ]: + actual_seq_idx = compute_split_seq_index( + forward_mode=ForwardMode.EXTEND, + num_tokens=None, + extend_lens=extend_lens, + token_num_per_seq=None, + ) + actual_token_idx = compute_split_token_index( + split_seq_index=actual_seq_idx, + forward_mode=ForwardMode.EXTEND, + extend_seq_lens=extend_lens, + token_num_per_seq=None, + ) + actual = (actual_seq_idx, actual_token_idx) + print(f"{extend_lens=} {expect=} {actual=}") + self.assertEqual(actual, expect) + + +class TestQwen3TwoBatchOverlap(TestTwoBatchOverlap): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_ENABLE_THINKING_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-1234" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--tp", + "2", + "--dp", + "2", + "--enable-dp-attention", + "--moe-a2a-backend", + "deepep", + "--deepep-mode", + "normal", + "--disable-cuda-graph", # DeepEP normal does not support CUDA Graph + "--enable-two-batch-overlap", + ], + env={"SGL_ENABLE_JIT_DEEPGEMM": "0", **os.environ}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_utils_update_weights.py b/test/srt/test_utils_update_weights.py new file mode 100644 index 000000000..8c138f0ab --- /dev/null +++ b/test/srt/test_utils_update_weights.py @@ -0,0 +1,170 @@ +import asyncio +import os +import unittest + +import torch +import torch.distributed as dist +from torch.distributed.device_mesh import init_device_mesh +from transformers import AutoModelForCausalLM + +from sglang.srt.entrypoints.engine import Engine +from sglang.srt.weight_sync.utils import update_weights +from sglang.test.test_utils import DEFAULT_SMALL_MODEL_NAME_FOR_TEST + + +class AsyncEngine(Engine): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + async def update_weights_from_tensor(self, update_weights_request): + return await self.tokenizer_manager.update_weights_from_tensor( + update_weights_request, None + ) + + +def is_distributed_available(): + """Check if distributed training environment is available""" + required_vars = ["RANK", "WORLD_SIZE", "MASTER_ADDR", "MASTER_PORT"] + return all(var in os.environ for var in required_vars) + + +def setup_single_process_distributed(): + """Setup distributed environment for single process testing""" + if not is_distributed_available(): + os.environ["RANK"] = "0" + os.environ["WORLD_SIZE"] = "1" + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "12356" + os.environ["LOCAL_RANK"] = "0" + + +class TestUtilsUpdateWeights(unittest.TestCase): + """Test class for utils.update_weights function""" + + @classmethod + def setUpClass(cls): + """Setup distributed environment and test fixtures for the entire test class""" + cls.setup_distributed() + cls.setup_test_engine() + cls.setup_test_model() + cls.setup_device_mesh() + + @classmethod + def tearDownClass(cls): + """Cleanup after all tests""" + if hasattr(cls, "engine") and cls.engine: + cls.engine.shutdown() + + # Cleanup distributed + if dist.is_initialized(): + dist.destroy_process_group() + + @classmethod + def setup_distributed(cls): + """Setup distributed environment for testing""" + setup_single_process_distributed() + + if not dist.is_initialized(): + try: + dist.init_process_group( + backend="nccl" if torch.cuda.is_available() else "gloo" + ) + except Exception as e: + raise unittest.SkipTest( + f"Could not initialize distributed backend: {e}" + ) + + cls.rank = dist.get_rank() + cls.world_size = dist.get_world_size() + + if torch.cuda.is_available(): + torch.cuda.set_device(cls.rank % torch.cuda.device_count()) + + # Set up environment variables + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + os.environ["NCCL_CUMEM_ENABLE"] = "0" + os.environ["CUDA_DEVICE_MAX_CONNECTIONS"] = "4" + os.environ["CUDA_MODULE_LOADING"] = "AUTO" + + @classmethod + def setup_test_engine(cls): + """Setup test engine""" + if cls.rank == 0: + cls.engine = AsyncEngine( + model_path=DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + dtype="bfloat16", + mem_fraction_static=0.3, + enable_memory_saver=True, + tp_size=cls.world_size, + disable_cuda_graph=False, + ) + else: + cls.engine = None + + @classmethod + def setup_test_model(cls): + """Load test model""" + try: + cls.model = AutoModelForCausalLM.from_pretrained( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + device_map="cpu", + trust_remote_code=True, + low_cpu_mem_usage=True, + torch_dtype=( + torch.float16 if torch.cuda.is_available() else torch.float32 + ), + ) + except Exception as e: + raise unittest.SkipTest(f"Could not load test model: {e}") + + @classmethod + def setup_device_mesh(cls): + """Create device mesh for testing""" + if not torch.cuda.is_available(): + raise unittest.SkipTest("CUDA not available for device mesh") + + cls.device_mesh_key = "tp" + cls.mesh = init_device_mesh( + "cuda", (cls.world_size,), mesh_dim_names=(cls.device_mesh_key,) + ) + + def create_test_params_batch(self, model, num_params=64): + """Create a batch of test parameters from the model""" + param_names = [] + test_tensors = [] + + # Get first few parameters from the model for testing + for i, (name, tensor) in enumerate(model.named_parameters()): + if i >= num_params: + break + param_names.append(name) + # Create test tensor with known values, matching original shape and dtype + test_tensor = torch.full_like(tensor, 1.5, dtype=tensor.dtype).cuda() + test_tensors.append(test_tensor) + + return list(zip(param_names, test_tensors)) + + def test_utils_update_weights(self): + """Test basic functionality of utils.update_weights""" + + async def async_test(): + # Create test parameters batch + params_batch = self.create_test_params_batch(self.model, num_params=2) + + # Test the utils.update_weights function + result = await update_weights( + engine=self.engine, + params_batch=params_batch, + device_mesh_key=self.device_mesh_key, + device_mesh=self.mesh, + load_format=None, + ) + + self.assertIn("Success", result) + + # Run the async test + asyncio.run(async_test()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vertex_endpoint.py b/test/srt/test_vertex_endpoint.py new file mode 100644 index 000000000..42e48cb1b --- /dev/null +++ b/test/srt/test_vertex_endpoint.py @@ -0,0 +1,64 @@ +""" +python3 -m unittest test_vertex_endpoint.TestVertexEndpoint.test_vertex_generate +""" + +import unittest +from http import HTTPStatus + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestVertexEndpoint(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--cuda-graph-max-bs", 2], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def run_generate(self, parameters): + data = { + "instances": [ + {"text": "The capital of France is"}, + {"text": "The capital of China is"}, + ], + "parameters": parameters, + } + response = requests.post(self.base_url + "/vertex_generate", json=data) + response_json = response.json() + assert len(response_json["predictions"]) == len(data["instances"]) + return response_json + + def test_vertex_generate(self): + for parameters in [None, {"sampling_params": {"max_new_tokens": 4}}]: + self.run_generate(parameters) + + def test_vertex_generate_fail(self): + data = { + "instances": [ + {"prompt": "The capital of France is"}, + ], + } + response = requests.post(self.base_url + "/vertex_generate", json=data) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vision_chunked_prefill.py b/test/srt/test_vision_chunked_prefill.py new file mode 100644 index 000000000..3876e915b --- /dev/null +++ b/test/srt/test_vision_chunked_prefill.py @@ -0,0 +1,188 @@ +""" +Usage: +python3 -m unittest test_vision_chunked_prefill.TestVisionChunkedPrefill.test_chunked_prefill +""" + +import base64 +import io +import os +import unittest +from concurrent.futures import ThreadPoolExecutor +from typing import Union + +import numpy as np +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +class TestVisionChunkedPrefill(CustomTestCase): + + def prepare_video_messages(self, video_path, max_frames_num=8): + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "video", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video briefly."} + messages[0]["content"].append(prompt) + + return messages + + def get_prompt_from_messages(self, messages): + text = ( + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + ) + image_data = [] + for content in messages[0]["content"]: + if content["type"] == "image_url": + text += "\n" + image_data.append(content["image_url"]["url"]) + text += "Please describe the video briefly.<|im_end|>\n<|im_start|>assistant\n" + return text, image_data + + def generate(self, text, image_data): + response = requests.post( + self.base_url + "/generate", + json={ + "text": text, + "image_data": image_data, + "sampling_params": { + "temperature": 0, + "max_new_tokens": 32, + "no_stop_trim": True, + "skip_special_tokens": False, + }, + "modalities": ["multi-images"], + }, + ).json() + return response["text"] + + def generate_for_video(self, batch, num_frame) -> Union[str, list[str]]: + # prepare the video input about Steven introducing ipod nano + url = "https://raw.githubusercontent.com/evolvinglmms-lab/sglang/dev/onevision_local/assets/jobs.mp4" + cache_dir = os.path.expanduser("~/.cache") + file_path = os.path.join(cache_dir, "jobs.mp4") + os.makedirs(cache_dir, exist_ok=True) + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + with open(file_path, "wb") as f: + f.write(response.content) + + if not batch: + assert isinstance(num_frame, int) + messages = self.prepare_video_messages(file_path, max_frames_num=num_frame) + text, image_data = self.get_prompt_from_messages(messages) + return self.generate(text, image_data) + else: + assert isinstance(num_frame, list) + func_args = [] + for max_frames_num in num_frame: + messages = self.prepare_video_messages( + file_path, + max_frames_num=max_frames_num, + ) + text, image_data = self.get_prompt_from_messages(messages) + func_args.append((text, image_data)) + + with ThreadPoolExecutor(max_workers=10) as executor: + responses = list(executor.map(lambda p: self.generate(*p), func_args)) + + return responses + + def launch_server(self, chunked_prefill_size) -> int: + # launch server + model = "lmms-lab/llava-onevision-qwen2-7b-ov" + # model = "meta-llama/Llama-3.2-11B-Vision-Instruct" + self.base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + self.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--chunked-prefill-size", + f"{chunked_prefill_size}", + ], + ) + return process.pid + + def _test_chunked_prefill(self, batches, num_frames): + # Chunked + chunked_server_pid = self.launch_server(chunked_prefill_size=1024) + try: + outputs_chunked = [] + for batch, num_frame in zip(batches, num_frames): + output_chunked = self.generate_for_video( + batch=batch, num_frame=num_frame + ) + outputs_chunked += [output_chunked] + finally: + kill_process_tree(chunked_server_pid) + + # None-chunked + try: + no_chunked_server_pid = self.launch_server(chunked_prefill_size=-1) + outputs_no_chunked = [] + for batch, num_frame in zip(batches, num_frames): + output_no_chunked = self.generate_for_video( + batch=batch, num_frame=num_frame + ) + outputs_no_chunked += [output_no_chunked] + + finally: + kill_process_tree(no_chunked_server_pid) + + for output_chunked, output_no_chunked in zip( + outputs_chunked, outputs_no_chunked + ): + print("output with chunked prefill:") + print(output_chunked) + print("output without chunked prefill:") + print(output_no_chunked) + self.assertEqual(output_chunked, output_no_chunked) + + def test_chunked_prefill(self): + self._test_chunked_prefill(batches=[False, True], num_frames=[1, [2, 6, 8, 10]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vision_openai_server_a.py b/test/srt/test_vision_openai_server_a.py new file mode 100644 index 000000000..e8e0d62e9 --- /dev/null +++ b/test/srt/test_vision_openai_server_a.py @@ -0,0 +1,342 @@ +""" +Usage: +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_mixed_batch +python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_multi_images_chat_completion +""" + +import unittest + +from test_vision_openai_server_common import * + +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_server, +) + + +class TestLlava(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "lmms-lab/llava-onevision-qwen2-0.5b-ov" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + ) + cls.base_url += "/v1" + + +class TestQwen2VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--mem-fraction-static", + "0.35", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestQwen2_5_VLServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2.5-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--mem-fraction-static", + "0.35", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestVLMContextLengthIssue(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-VL-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--context-length", + "300", + "--mem-fraction-static=0.75", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + with self.assertRaises(openai.BadRequestError) as cm: + client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Give a lengthy description of this picture", + }, + ], + }, + ], + temperature=0, + ) + + # context length is checked first, then max_req_input_len, which is calculated from the former + assert ( + "Multimodal prompt is too long after expanding multimodal tokens." + in str(cm.exception) + or "is longer than the model's context length" in str(cm.exception) + ) + + +# Note(Xinyuan): mllama is not stable for now, skip for CI +# class TestMllamaServer(TestOpenAIVisionServer): +# @classmethod +# def setUpClass(cls): +# cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# api_key=cls.api_key, +# ) +# cls.base_url += "/v1" + + +class TestMinicpmvServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "openbmb/MiniCPM-V-2_6" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.35", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestMinicpmv4Server(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "openbmb/MiniCPM-V-4" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.35", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestInternVL2_5Server(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "OpenGVLab/InternVL2_5-2B" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestMinicpmo2_6Server(ImageOpenAITestMixin, AudioOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "openbmb/MiniCPM-o-2_6" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.65", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestMimoVLServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "XiaomiMiMo/MiMo-VL-7B-RL" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.6", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestVILAServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "Efficient-Large-Model/NVILA-Lite-2B-hf-0626" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.revision = "6bde1de5964b40e61c802b375fff419edc867506" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + api_key=cls.api_key, + other_args=[ + "--trust-remote-code", + "--context-length=65536", + f"--revision={cls.revision}", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestPhi4MMServer(ImageOpenAITestMixin, AudioOpenAITestMixin): + @classmethod + def setUpClass(cls): + # Manually download LoRA adapter_config.json as it's not downloaded by the model loader by default. + from huggingface_hub import constants, snapshot_download + + snapshot_download( + "microsoft/Phi-4-multimodal-instruct", + allow_patterns=["**/adapter_config.json"], + ) + + cls.model = "microsoft/Phi-4-multimodal-instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + + revision = "33e62acdd07cd7d6635badd529aa0a3467bb9c6a" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.70", + "--disable-radix-cache", + "--max-loras-per-batch", + "2", + "--revision", + revision, + "--lora-paths", + f"vision={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/vision-lora", + f"speech={constants.HF_HUB_CACHE}/models--microsoft--Phi-4-multimodal-instruct/snapshots/{revision}/speech-lora", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + def get_vision_request_kwargs(self): + return { + "extra_body": { + "lora_path": "vision", + "top_k": 1, + "top_p": 1.0, + } + } + + def get_audio_request_kwargs(self): + return { + "extra_body": { + "lora_path": "speech", + "top_k": 1, + "top_p": 1.0, + } + } + + # This _test_audio_ambient_completion test is way too complicated to pass for a small LLM + def test_audio_ambient_completion(self): + pass + + +if __name__ == "__main__": + del ( + TestOpenAIOmniServerBase, + ImageOpenAITestMixin, + VideoOpenAITestMixin, + AudioOpenAITestMixin, + ) + unittest.main() diff --git a/test/srt/test_vision_openai_server_b.py b/test/srt/test_vision_openai_server_b.py new file mode 100644 index 000000000..6c2fa86d5 --- /dev/null +++ b/test/srt/test_vision_openai_server_b.py @@ -0,0 +1,250 @@ +import unittest + +from test_vision_openai_server_common import * + +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + popen_launch_server, +) + + +class TestPixtralServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "mistral-community/pixtral-12b" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.70", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestMistral3_1Server(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "unsloth/Mistral-Small-3.1-24B-Instruct-2503" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.75", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestDeepseekVL2Server(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/deepseek-vl2-small" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--context-length", + "4096", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestJanusProServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "deepseek-ai/Janus-Pro-7B" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.35", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + def test_video_images_chat_completion(self): + pass + + +## Skip for ci test +# class TestLlama4Server(TestOpenAIVisionServer): +# @classmethod +# def setUpClass(cls): +# cls.model = "meta-llama/Llama-4-Scout-17B-16E-Instruct" +# cls.base_url = DEFAULT_URL_FOR_TEST +# cls.api_key = "sk-123456" +# cls.process = popen_launch_server( +# cls.model, +# cls.base_url, +# timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, +# other_args=[ +# "--chat-template", +# "llama-4", +# "--mem-fraction-static", +# "0.8", +# "--tp-size=8", +# "--context-length=8192", +# "--mm-attention-backend", +# "fa3", +# "--cuda-graph-max-bs", +# "4", +# ], +# ) +# cls.base_url += "/v1" + + +class TestGemma3itServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "google/gemma-3-4b-it" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.70", + "--enable-multimodal", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + +class TestGemma3nServer(ImageOpenAITestMixin, AudioOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "google/gemma-3n-E4B-it" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.70", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + # This _test_audio_ambient_completion test is way too complicated to pass for a small LLM + def test_audio_ambient_completion(self): + pass + + def _test_mixed_image_audio_chat_completion(self): + self._test_mixed_image_audio_chat_completion() + + +class TestQwen2AudioServer(AudioOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "Qwen/Qwen2-Audio-7B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.70", + ], + ) + cls.base_url += "/v1" + + +class TestKimiVLServer(ImageOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "moonshotai/Kimi-VL-A3B-Instruct" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--context-length", + "4096", + "--dtype", + "bfloat16", + "--cuda-graph-max-bs", + "4", + ], + ) + cls.base_url += "/v1" + + def test_video_images_chat_completion(self): + pass + + +class TestGLM41VServer(ImageOpenAITestMixin, VideoOpenAITestMixin): + @classmethod + def setUpClass(cls): + cls.model = "zai-org/GLM-4.1V-9B-Thinking" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--trust-remote-code", + "--mem-fraction-static", + "0.68", + "--cuda-graph-max-bs", + "4", + "--reasoning-parser", + "glm45", + ], + ) + cls.base_url += "/v1" + + +if __name__ == "__main__": + del ( + TestOpenAIOmniServerBase, + ImageOpenAITestMixin, + VideoOpenAITestMixin, + AudioOpenAITestMixin, + ) + unittest.main() diff --git a/test/srt/test_vision_openai_server_common.py b/test/srt/test_vision_openai_server_common.py new file mode 100644 index 000000000..792636060 --- /dev/null +++ b/test/srt/test_vision_openai_server_common.py @@ -0,0 +1,527 @@ +import base64 +import io +import os + +import numpy as np +import openai +import requests +from PIL import Image + +from sglang.srt.utils import kill_process_tree +from sglang.test.test_utils import DEFAULT_URL_FOR_TEST, CustomTestCase + +# image +IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png" +IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png" + +# video +VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4" + +# audio +AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3" +AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3" + + +class TestOpenAIOmniServerBase(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.api_key = "sk-123456" + cls.process = None + cls.base_url += "/v1" + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def get_vision_request_kwargs(self): + return self.get_request_kwargs() + + def get_request_kwargs(self): + return {} + + def get_or_download_file(self, url: str) -> str: + cache_dir = os.path.expanduser("~/.cache") + if url is None: + raise ValueError() + file_name = url.split("/")[-1] + file_path = os.path.join(cache_dir, file_name) + os.makedirs(cache_dir, exist_ok=True) + + if not os.path.exists(file_path): + response = requests.get(url) + response.raise_for_status() + + with open(file_path, "wb") as f: + f.write(response.content) + return file_path + + +class AudioOpenAITestMixin(TestOpenAIOmniServerBase): + def prepare_audio_messages(self, prompt, audio_file_name): + messages = [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": f"{audio_file_name}"}, + }, + { + "type": "text", + "text": prompt, + }, + ], + } + ] + + return messages + + def get_audio_request_kwargs(self): + return self.get_request_kwargs() + + def get_audio_response(self, url: str, prompt, category): + audio_file_path = self.get_or_download_file(url) + client = openai.Client(api_key="sk-123456", base_url=self.base_url) + + messages = self.prepare_audio_messages(prompt, audio_file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=128, + stream=False, + **(self.get_audio_request_kwargs()), + ) + + audio_response = response.choices[0].message.content + + print("-" * 30) + print(f"audio {category} response:\n{audio_response}") + print("-" * 30) + + audio_response = audio_response.lower() + + self.assertIsNotNone(audio_response) + self.assertGreater(len(audio_response), 0) + + return audio_response.lower() + + def test_audio_speech_completion(self): + # a fragment of Trump's speech + audio_response = self.get_audio_response( + AUDIO_TRUMP_SPEECH_URL, + "Listen to this audio and write down the audio transcription in English.", + category="speech", + ) + check_list = [ + "thank you", + "it's a privilege to be here", + "leader", + "science", + "art", + ] + for check_word in check_list: + assert ( + check_word in audio_response + ), f"audio_response: ｜{audio_response}｜ should contain ｜{check_word}｜" + + def test_audio_ambient_completion(self): + # bird song + audio_response = self.get_audio_response( + AUDIO_BIRD_SONG_URL, + "Please listen to the audio snippet carefully and transcribe the content in English.", + "ambient", + ) + assert "bird" in audio_response + + +class ImageOpenAITestMixin(TestOpenAIOmniServerBase): + def test_single_image_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a sentence.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + # `driver` is for gemma-3-it + assert ( + "man" in text or "person" or "driver" in text + ), f"text: {text}, should contain man, person or driver" + assert ( + "cab" in text + or "taxi" in text + or "SUV" in text + or "vehicle" in text + or "car" in text + ), f"text: {text}, should contain cab, taxi, SUV, vehicle or car" + # MiniCPMO fails to recognize `iron`, but `hanging` + assert ( + "iron" in text + or "hang" in text + or "cloth" in text + or "coat" in text + or "holding" in text + or "outfit" in text + ), f"text: {text}, should contain iron, hang, cloth, coat or holding or outfit" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_multi_turn_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "text", + "text": "Describe this image in a sentence.", + }, + ], + }, + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": "There is a man at the back of a yellow cab ironing his clothes.", + } + ], + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Repeat your previous answer."} + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + assert ( + "man" in text or "cab" in text + ), f"text: {text}, should contain man or cab" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def test_multi_images_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + "modalities": "multi-images", + }, + { + "type": "image_url", + "image_url": {"url": IMAGE_SGL_LOGO_URL}, + "modalities": "multi-images", + }, + { + "type": "text", + "text": "I have two very different images. They are not related at all. " + "Please describe the first image in one sentence, and then describe the second image in another sentence.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Multi images response:\n{text}") + print("-" * 30) + assert ( + "man" in text + or "cab" in text + or "SUV" in text + or "taxi" in text + or "car" in text + ), f"text: {text}, should contain man, cab, SUV, taxi or car" + assert ( + "logo" in text or '"S"' in text or "SG" in text or "graphic" in text + ), f"text: {text}, should contain logo, S or SG or graphic" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def _test_mixed_image_audio_chat_completion(self): + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": IMAGE_MAN_IRONING_URL}, + }, + { + "type": "audio_url", + "audio_url": {"url": AUDIO_TRUMP_SPEECH_URL}, + }, + { + "type": "text", + "text": "Please describe the image in one sentence, and then write down the audio transcription in English.", + }, + ], + }, + ], + temperature=0, + **(self.get_vision_request_kwargs()), + ) + + assert response.choices[0].message.role == "assistant" + text = response.choices[0].message.content + assert isinstance(text, str) + print("-" * 30) + print(f"Mixed image & audio response:\n{text}") + print("-" * 30) + assert ( + "man" in text + or "cab" in text + or "SUV" in text + or "taxi" in text + or "car" in text + ), f"text: {text}, should contain man, cab, SUV, taxi or car" + check_list = [ + "thank you", + "it's a privilege to be here", + "leader", + "science", + "art", + ] + for check_word in check_list: + assert ( + check_word in text + ), f"text: ｜{text}｜ should contain ｜{check_word}｜" + assert response.id + assert response.created + assert response.usage.prompt_tokens > 0 + assert response.usage.completion_tokens > 0 + assert response.usage.total_tokens > 0 + + def prepare_video_images_messages(self, video_path): + # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa + # the size of the video embeds differs from the `modality` argument when preprocessed + + # We import decord here to avoid a strange Segmentation fault (core dumped) issue. + # The following import order will cause Segmentation fault. + # import decord + # from transformers import AutoTokenizer + from decord import VideoReader, cpu + + max_frames_num = 10 + vr = VideoReader(video_path, ctx=cpu(0)) + total_frame_num = len(vr) + uniform_sampled_frames = np.linspace( + 0, total_frame_num - 1, max_frames_num, dtype=int + ) + frame_idx = uniform_sampled_frames.tolist() + frames = vr.get_batch(frame_idx).asnumpy() + + base64_frames = [] + for frame in frames: + pil_img = Image.fromarray(frame) + buff = io.BytesIO() + pil_img.save(buff, format="JPEG") + base64_str = base64.b64encode(buff.getvalue()).decode("utf-8") + base64_frames.append(base64_str) + + messages = [{"role": "user", "content": []}] + frame_format = { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,{}"}, + "modalities": "image", + } + + for base64_frame in base64_frames: + frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format( + base64_frame + ) + messages[0]["content"].append(frame_format.copy()) + + prompt = {"type": "text", "text": "Please describe the video in detail."} + messages[0]["content"].append(prompt) + + return messages + + def test_video_images_chat_completion(self): + url = VIDEO_JOBS_URL + file_path = self.get_or_download_file(url) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = self.prepare_video_images_messages(file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=False, + ) + + video_response = response.choices[0].message.content + + print("-" * 30) + print(f"Video images response:\n{video_response}") + print("-" * 30) + + # Add assertions to validate the video response + assert ( + "iPod" in video_response + or "device" in video_response + or "microphone" in video_response + ), f""" + ====================== video_response ===================== + {video_response} + =========================================================== + should contain 'iPod' or 'device' or 'microphone' + """ + assert ( + "man" in video_response + or "person" in video_response + or "individual" in video_response + or "speaker" in video_response + or "presenter" in video_response + or "Steve" in video_response + or "hand" in video_response + ), f""" + ====================== video_response ===================== + {video_response} + =========================================================== + should contain 'man' or 'person' or 'individual' or 'speaker' or 'presenter' or 'Steve' or 'hand' + """ + assert ( + "present" in video_response + or "examine" in video_response + or "display" in video_response + or "hold" in video_response + ), f""" + ====================== video_response ===================== + {video_response} + =========================================================== + should contain 'present' or 'examine' or 'display' or 'hold' + """ + self.assertIsNotNone(video_response) + self.assertGreater(len(video_response), 0) + + +class VideoOpenAITestMixin(TestOpenAIOmniServerBase): + def prepare_video_messages(self, video_path): + messages = [ + { + "role": "user", + "content": [ + { + "type": "video_url", + "video_url": {"url": f"{video_path}"}, + }, + {"type": "text", "text": "Please describe the video in detail."}, + ], + }, + ] + return messages + + def test_video_chat_completion(self): + url = VIDEO_JOBS_URL + file_path = self.get_or_download_file(url) + + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + messages = self.prepare_video_messages(file_path) + + response = client.chat.completions.create( + model="default", + messages=messages, + temperature=0, + max_tokens=1024, + stream=False, + **(self.get_vision_request_kwargs()), + ) + + video_response = response.choices[0].message.content + + print("-" * 30) + print(f"Video response:\n{video_response}") + print("-" * 30) + + # Add assertions to validate the video response + assert ( + "iPod" in video_response + or "device" in video_response + or "microphone" in video_response + ), f"video_response: {video_response}, should contain 'iPod' or 'device'" + assert ( + "man" in video_response + or "person" in video_response + or "individual" in video_response + or "speaker" in video_response + or "presenter" in video_response + or "hand" in video_response + ), f"video_response: {video_response}, should either have 'man' in video_response, or 'person' in video_response, or 'individual' in video_response or 'speaker' in video_response or 'presenter' or 'hand' in video_response" + assert ( + "present" in video_response + or "examine" in video_response + or "display" in video_response + or "hold" in video_response + ), f"video_response: {video_response}, should contain 'present', 'examine', 'display', or 'hold'" + assert ( + "black" in video_response or "dark" in video_response + ), f"video_response: {video_response}, should contain 'black' or 'dark'" + self.assertIsNotNone(video_response) + self.assertGreater(len(video_response), 0) diff --git a/test/srt/test_vllm_dependency.py b/test/srt/test_vllm_dependency.py new file mode 100644 index 000000000..b4451f369 --- /dev/null +++ b/test/srt/test_vllm_dependency.py @@ -0,0 +1,164 @@ +import json +import os +import unittest +import warnings +from datetime import datetime +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + write_github_step_summary, +) + +MODEL_SCORE_THRESHOLDS = { + "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.825, + "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.825, + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.62, +} + + +def parse_models(model_string): + return [model.strip() for model in model_string.split(",") if model.strip()] + + +def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2): + other_args = ["--log-level-http", "warning", "--trust-remote-code"] + if is_fp8: + if "Llama-3" in model or "gemma-2" in model: + other_args.extend(["--kv-cache-dtype", "fp8_e5m2"]) + elif "Qwen2-72B-Instruct-FP8" in model: + other_args.extend(["--quantization", "fp8"]) + elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model: + other_args.extend([]) + else: + other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"]) + if is_tp2: + other_args.extend(["--tp", "2"]) + if "DeepSeek" in model: + other_args.extend(["--mem-frac", "0.85"]) + + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + return process + + +def write_results_to_json(model, metrics, mode="a"): + result = { + "timestamp": datetime.now().isoformat(), + "model": model, + "metrics": metrics, + "score": metrics["score"], + } + + existing_results = [] + if mode == "a" and os.path.exists("results.json"): + try: + with open("results.json", "r") as f: + existing_results = json.load(f) + except json.JSONDecodeError: + existing_results = [] + + if isinstance(existing_results, list): + existing_results.append(result) + else: + existing_results = [result] + + with open("results.json", "w") as f: + json.dump(existing_results, f, indent=2) + + +def check_model_scores(results): + failed_models = [] + summary = " | model | score | threshold |\n" + summary += "| ----- | ----- | --------- |\n" + + for model, score in results: + threshold = MODEL_SCORE_THRESHOLDS.get(model) + if threshold is None: + print(f"Warning: No threshold defined for model {model}") + continue + + if score < threshold: + failed_models.append( + f"\nScore Check Failed: {model}\n" + f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})" + ) + + line = f"| {model} | {score} | {threshold} |\n" + summary += line + + print(summary) + + if is_in_ci(): + write_github_step_summary( + f"### TestNightlyGsm8KEval for vLLM awq, gptq, gguf\n{summary}" + ) + + if failed_models: + raise AssertionError("\n".join(failed_models)) + + +class TestNightlyGsm8KEval(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model_groups = [ + (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1), False, False), + ] + cls.base_url = DEFAULT_URL_FOR_TEST + + def test_mgsm_en_all_models(self): + warnings.filterwarnings( + "ignore", category=ResourceWarning, message="unclosed.*socket" + ) + is_first = True + all_results = [] + + for model_group, is_fp8, is_tp2 in self.model_groups: + for model in model_group: + with self.subTest(model=model): + process = popen_launch_server_wrapper( + self.base_url, model, is_fp8, is_tp2 + ) + + args = SimpleNamespace( + base_url=self.base_url, + model=model, + eval_name="mgsm_en", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print( + f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n" + ) + + write_results_to_json(model, metrics, "w" if is_first else "a") + is_first = False + + all_results.append((model, metrics["score"])) + kill_process_tree(process.pid) + + try: + with open("results.json", "r") as f: + print("\nFinal Results from results.json:") + print(json.dumps(json.load(f), indent=2)) + except Exception as e: + print(f"Error reading results.json: {e}") + + # Check all scores after collecting all results + check_model_scores(all_results) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_vlm_accuracy.py b/test/srt/test_vlm_accuracy.py new file mode 100644 index 000000000..ef9a2ad51 --- /dev/null +++ b/test/srt/test_vlm_accuracy.py @@ -0,0 +1,324 @@ +""" +""" + +import unittest +from io import BytesIO +from typing import List, Optional + +import numpy as np +import requests +import torch +import torch.nn.functional as F +from PIL import Image +from transformers import AutoModel, AutoProcessor, AutoTokenizer + +from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.managers.mm_utils import embed_mm_inputs, init_embedding_cache +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, +) +from sglang.srt.model_executor.model_runner import ModelRunner +from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor +from sglang.srt.parser.conversation import generate_chat_conv +from sglang.srt.server_args import ServerArgs + + +# Test the logits output between HF and SGLang +class VisionLLMLogitsBase(unittest.IsolatedAsyncioTestCase): + @classmethod + def setUpClass(cls): + cls.image_url = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.model_path = "" + cls.chat_template = "" + cls.processor = "" + response = requests.get(cls.image_url) + cls.main_image = Image.open(BytesIO(response.content)) + + def compare_outputs(self, sglang_output: torch.Tensor, hf_output: torch.Tensor): + # Convert to float32 for numerical stability if needed + hf = hf_output.float() + sg = sglang_output.float() + + # Basic shape and dtype comparison + print("\n=== Basic Properties ===") + print(f"Shapes match: {hf.shape == sg.shape}") + print(f"HF shape: {hf.shape}, SGLang shape: {sg.shape}") + print(f"HF dtype: {hf.dtype}, SGLang dtype: {sg.dtype}") + + # Move tensors to CPU for numpy operations + hf_np = hf.cpu().numpy() + sg_np = sg.cpu().numpy() + + # Statistical metrics + print("\n=== Statistical Metrics ===") + print(f"Mean absolute difference: {torch.mean(torch.abs(hf - sg)).item():.6f}") + print(f"Max absolute difference: {torch.max(torch.abs(hf - sg)).item():.6f}") + print(f"Mean squared error: {torch.mean((hf - sg) ** 2).item():.6f}") + print( + f"Root mean squared error: {torch.sqrt(torch.mean((hf - sg) ** 2)).item():.6f}" + ) + + # Cosine similarity (across feature dimension) + cos_sim = F.cosine_similarity(hf, sg) + print(f"Mean cosine similarity: {torch.mean(cos_sim).item():.6f}") + print(f"Min cosine similarity: {torch.min(cos_sim).item():.6f}") + + # Find largest absolute differences + print("\n=== Largest Absolute Differences ===") + diffs = torch.abs(hf - sg) + flat_diffs = diffs.flatten() + + # Get indices of top 10 differences + top_k = 10 + top_values, top_flat_indices = torch.topk(flat_diffs, top_k) + + # Convert flat indices to multidimensional indices + top_indices = np.unravel_index(top_flat_indices.cpu().numpy(), diffs.shape) + + print(f"\nTop {top_k} largest absolute differences:") + print( + "Index".ljust(30) + + "Difference".ljust(15) + + "HF Value".ljust(15) + + "SGLang Value" + ) + print("-" * 75) + + for i in range(top_k): + # Get the index tuple for this difference + idx = tuple(dim[i] for dim in top_indices) + diff_val = top_values[i].item() + hf_val = hf[idx].item() + sg_val = sg[idx].item() + + # Format the index tuple and values + idx_str = str(idx) + print(f"{idx_str:<30}{diff_val:<15.6f}{hf_val:<15.6f}{sg_val:.6f}") + + np.testing.assert_allclose(hf_np, sg_np) + + def get_completion_request(self) -> ChatCompletionRequest: + json_str = f""" + {{ + "model": "{self.model_path}", + "messages": [ + {{ + "role": "user", + "content": [ + {{ + "type": "image_url", + "image_url": {{ + "url": "{self.image_url}" + }} + }}, + {{ + "type": "text", + "text": "What's in this picture?" + }} + ] + }} + ] +}} + """ + + return ChatCompletionRequest.model_validate_json(json_str) + + def get_processor_output(self, req: Optional[ChatCompletionRequest] = None): + if req is None: + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + + # Process inputs using processor + # FIXME: the formal arguments may differ + inputs = self.processor( + text=[text], + images=[self.main_image], + return_tensors="pt", + ).to(self.device) + + return inputs + + def get_sglang_model(self): + self.model_runner = ModelRunner( + model_config=ModelConfig(self.model_path, model_override_args="{}"), + mem_fraction_static=0.8, + gpu_id=0, + tp_rank=0, + tp_size=1, + pp_rank=0, + pp_size=1, + nccl_port=12435, + server_args=ServerArgs( + model_path=self.model_path, + disable_cuda_graph=True, + ), + ) + return self.model_runner.model + + +class TestMiniCPMV2_6Logits(VisionLLMLogitsBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model_path = "openbmb/MiniCPM-V-2_6" + cls.tokenizer = AutoTokenizer.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.chat_template = "minicpmv" + + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.hf_model = ( + AutoModel.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16, trust_remote_code=True + ) + .eval() + .to(cls.device) + ) + init_embedding_cache() + + async def test_vlm_embedding_output(self): + """ + Compares the embedding output of vlm + """ + inputs = self.get_processor_output() + + with torch.no_grad(): + # hf + model_inputs = { + "input_ids": inputs.input_ids, + "image_bound": inputs.image_bound, + "pixel_values": inputs.pixel_values, + "tgt_sizes": inputs.tgt_sizes, + } + (hf_output, _) = self.hf_model.get_vllm_embedding( + model_inputs, + ) + hf_output = hf_output.squeeze(0) + + # sglang + model = self.get_sglang_model() + input_ids = inputs["input_ids"].to(self.device).flatten() + + pixel_values = inputs["pixel_values"] + tgt_sizes = inputs["tgt_sizes"] + pixel_values_flat: List[torch.Tensor] = [] + tgt_sizes_flat: List[torch.Tensor] = [] + for pixel_b, tgt_b in zip(pixel_values, tgt_sizes): + # per image + if len(pixel_b) != len(tgt_b): + raise ValueError( + "Inconsistent N lengths, found: " + f"{len(pixel_b)} vs {len(tgt_b)}" + ) + for pixel_n, tgt_n in zip(pixel_b, tgt_b): + pixel_values_flat += [pixel_n] + tgt_sizes_flat += [tgt_n] + + im_start_id, im_end_id = ( + self.tokenizer.im_start_id, + self.tokenizer.im_end_id, + ) + slice_start_id, slice_end_id = ( + self.tokenizer.slice_start_id, + self.tokenizer.slice_end_id, + ) + + image_offsets = BaseMultimodalProcessor.get_mm_items_offset_by_pair( + input_ids=input_ids, mm_start_id=im_start_id, mm_end_id=im_end_id + ) + slice_offsets = BaseMultimodalProcessor.get_mm_items_offset_by_pair( + input_ids=input_ids, mm_start_id=slice_start_id, mm_end_id=slice_end_id + ) + image_offsets.extend(slice_offsets) + image_offsets = sorted(image_offsets) + + sglang_output = embed_mm_inputs( + mm_inputs_list=[ + MultimodalInputs( + mm_items=[ + MultimodalDataItem( + feature=pixel_values_flat, + offsets=image_offsets, + tgt_size=tgt_sizes_flat, + modality=Modality.IMAGE, + pad_value=self.processor.tokenizer.unk_token_id, + ) + ] + ), + ], + extend_prefix_lens=[0], + extend_seq_lens=[input_ids.shape[0]], + input_ids=input_ids, + input_embedding=model.get_input_embeddings(), + multimodal_model=model, + placeholder_tokens={ + Modality.IMAGE: self.processor.tokenizer.unk_token_id, + }, + ) + + self.compare_outputs(sglang_output, hf_output) + + +class TestMiniCPMV4Logits(VisionLLMLogitsBase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.model_path = "openbmb/MiniCPM-V-4" + cls.tokenizer = AutoTokenizer.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True + ) + cls.chat_template = "minicpmv" + + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + cls.hf_model = ( + AutoModel.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16, trust_remote_code=True + ) + .eval() + .to(cls.device) + ) + init_embedding_cache() + + async def test_vlm_embedding_output(self): + """ + Compares the embedding output of vlm + """ + inputs = self.get_processor_output() + + with torch.no_grad(): + # hf + model_inputs = { + "input_ids": inputs.input_ids, + "image_bound": inputs.image_bound, + "pixel_values": inputs.pixel_values, + "tgt_sizes": inputs.tgt_sizes, + } + hf_output = self.hf_model.get_input_embeddings()(inputs.input_ids) + + # sglang + model = self.get_model() + sglang_output = self.vlm_func( + model, + input_ids=inputs.input_ids.to(self.device), + pixel_values=inputs.pixel_values, + image_bound=inputs.image_bound.to(self.device), + tgt_sizes=inputs.tgt_sizes.to(self.device), + input_embedding=model.get_input_embeddings(), + multimodal_model=model, + placeholder_tokens={ + Modality.IMAGE: self.processor.tokenizer.unk_token_id, + }, + ) + + self.compare_outputs(sglang_output, hf_output) diff --git a/test/srt/test_vlm_input_format.py b/test/srt/test_vlm_input_format.py new file mode 100644 index 000000000..261700da5 --- /dev/null +++ b/test/srt/test_vlm_input_format.py @@ -0,0 +1,258 @@ +import json +import unittest +from io import BytesIO +from typing import Optional + +import requests +import torch +from PIL import Image +from transformers import ( + AutoModel, + AutoProcessor, + Gemma3ForConditionalGeneration, + Qwen2_5_VLForConditionalGeneration, +) + +from sglang import Engine +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.parser.conversation import generate_chat_conv + +TEST_IMAGE_URL = "https://github.com/sgl-project/sglang/blob/main/test/lang/example_image.png?raw=true" + + +class VLMInputTestBase: + model_path = None + chat_template = None + processor = None + visual = None # Should be a callable for precomputed embeddings + + @classmethod + def setUpClass(cls): + assert cls.model_path is not None, "Set model_path in subclass" + assert cls.chat_template is not None, "Set chat_template in subclass" + cls.image_url = TEST_IMAGE_URL + cls.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + response = requests.get(cls.image_url) + cls.main_image = Image.open(BytesIO(response.content)) + cls.processor = AutoProcessor.from_pretrained( + cls.model_path, trust_remote_code=True, use_fast=True + ) + cls._init_visual() + + @classmethod + def _init_visual(cls): + """Override in subclass to set up cls.visual as a callable for precomputed embeddings.""" + raise NotImplementedError + + def setUp(self): + self.engine = Engine( + model_path=self.model_path, + chat_template=self.chat_template, + device=self.device.type, + mem_fraction_static=0.8, + enable_multimodal=True, + disable_cuda_graph=True, + trust_remote_code=True, + ) + + def tearDown(self): + self.engine.shutdown() + + def verify_response(self, output): + out_text = output["text"].lower() + assert "taxi" in out_text or "cab" in out_text or "car" in out_text, out_text + + def get_completion_request(self) -> ChatCompletionRequest: + json_structure = { + "model": self.model_path, + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": self.image_url}}, + {"type": "text", "text": "What's in this picture?"}, + ], + } + ], + } + json_str = json.dumps(json_structure) + return ChatCompletionRequest.model_validate_json(json_str) + + def get_processor_output(self, req: Optional[ChatCompletionRequest] = None): + if req is None: + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + + # Process inputs using processor + inputs = self.processor( + text=[text], + images=[self.main_image], + return_tensors="pt", + ).to(self.device) + + return inputs + + async def test_understands_image(self): + req = self.get_completion_request() + conv = generate_chat_conv(req, template_name=self.chat_template) + text = conv.get_prompt() + output = await self.engine.async_generate( + prompt=text, + image_data=[self.main_image], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + async def test_understands_precomputed_embeddings(self): + req = self.get_completion_request() + processor_output = self.get_processor_output(req=req) + with torch.inference_mode(): + precomputed_embeddings = self.__class__.visual(processor_output) + output = await self.engine.async_generate( + input_ids=processor_output["input_ids"][0].detach().cpu().tolist(), + image_data=[ + self._precomputed_image_data(processor_output, precomputed_embeddings) + ], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + async def test_understands_pixel_values(self): + req = self.get_completion_request() + processor_output = self.get_processor_output(req=req) + output = await self.engine.async_generate( + input_ids=processor_output["input_ids"][0].detach().cpu().tolist(), + image_data=[self._pixel_values_image_data(processor_output)], + sampling_params=dict(temperature=0.0), + ) + self.verify_response(output) + + def _precomputed_image_data(self, processor_output, precomputed_embeddings): + """This should not be overridden.""" + return dict( + modality="IMAGE", + precomputed_embeddings=precomputed_embeddings, + ) + + def _pixel_values_image_data(self, processor_output): + """Override in subclass to pass the correct set of arguments.""" + raise NotImplementedError + + +class TestQwenVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "Qwen/Qwen2.5-VL-3B-Instruct" + chat_template = "qwen2-vl" + + @classmethod + def _init_visual(cls): + cls.visual_model = ( + Qwen2_5_VLForConditionalGeneration.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16 + ) + .eval() + .visual.to(cls.device) + ) + cls.visual = lambda processor_output: cls.visual_model( + processor_output["pixel_values"], processor_output["image_grid_thw"] + ) + + def _pixel_values_image_data(self, processor_output): + return dict( + modality="IMAGE", + image_grid_thw=processor_output["image_grid_thw"], + pixel_values=processor_output["pixel_values"], + ) + + +class TestGemmaUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTestCase): + model_path = "google/gemma-3-4b-it" + chat_template = "gemma-it" + + @classmethod + def _init_visual(cls): + model = Gemma3ForConditionalGeneration.from_pretrained( + cls.model_path, torch_dtype=torch.bfloat16 + ) + cls.vision_tower = model.vision_tower.eval().to(cls.device) + cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + cls.visual = lambda processor_output: cls.mm_projector( + cls.vision_tower( + pixel_values=processor_output["pixel_values"] + ).last_hidden_state + ) + + def _pixel_values_image_data(self, processor_output): + return dict( + modality="IMAGE", + pixel_values=processor_output["pixel_values"][0], + ) + + +class TestKimiVLImageUnderstandsImage( + VLMInputTestBase, unittest.IsolatedAsyncioTestCase +): + model_path = "moonshotai/Kimi-VL-A3B-Instruct" + chat_template = "kimi-vl" + + @classmethod + def _init_visual(cls): + model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True) + cls.vision_tower = model.vision_tower.eval().to(cls.device) + cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + + cls.visual = lambda tokenizer_output: cls.mm_projector( + cls.vision_tower( + pixel_values=tokenizer_output["pixel_values"], + grid_hws=tokenizer_output["image_grid_hws"], + ) + ) + + def _pixel_values_image_data(self, processor_output): + return dict( + modality="IMAGE", + pixel_values=processor_output["pixel_values"], + image_grid_hws=processor_output["image_grid_hws"], + ) + + +# not for CI: too large +# class TestLlama4ImageUnderstandsImage( +# VLMInputTestBase, unittest.IsolatedAsyncioTestCase +# ): +# model_path = "meta-llama/Llama-4-Scout-17B-16E-Instruct" +# chat_template = "llama_4_vision" + +# def setUp(self): +# self.engine = Engine( +# model_path=self.model_path, +# trust_remote_code=True, +# chat_template=self.chat_template, +# enable_multimodal=True, +# mem_fraction_static=0.8, +# tp_size=4, +# attention_backend="fa3", +# context_length=65536, +# ) + +# @classmethod +# def _init_visual(cls): +# model = AutoModel.from_pretrained(cls.model_path, trust_remote_code=True, torch_dtype="auto") +# cls.vision_tower = model.vision_model.eval().to(cls.device) +# cls.mm_projector = model.multi_modal_projector.eval().to(cls.device) + +# cls.visual = lambda tokenizer_output: cls.mm_projector( +# cls.vision_tower( +# pixel_values=tokenizer_output["pixel_values"], +# ).last_hidden_state.flatten(0, -2) +# ) + +# def _pixel_values_image_data(self, processor_output): +# return dict( +# modality="IMAGE", +# pixel_values=processor_output["pixel_values"], +# ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_wave_attention_backend.py b/test/srt/test_wave_attention_backend.py new file mode 100644 index 000000000..5feab4595 --- /dev/null +++ b/test/srt/test_wave_attention_backend.py @@ -0,0 +1,61 @@ +""" +Usage: +python3 -m unittest test_wave_attention_backend.TestWaveAttnBackend.test_mmlu +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + is_in_ci, + popen_launch_server, + run_bench_one_batch, +) + + +class TestWaveAttnBackend(unittest.TestCase): + def test_latency(self): + _, output_throughput, _ = run_bench_one_batch( + DEFAULT_MODEL_NAME_FOR_TEST, + [ + "--attention-backend", + "wave", + "--enable-torch-compile", + ], + ) + + if is_in_ci(): + self.assertGreater(output_throughput, 153) + + def _test_mmlu(self): + model = DEFAULT_MODEL_NAME_FOR_TEST + base_url = DEFAULT_URL_FOR_TEST + process = popen_launch_server( + model, + base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--attention-backend", "wave"], + ) + + try: + args = SimpleNamespace( + base_url=base_url, + model=model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreaterEqual(metrics["score"], 0.65) + finally: + kill_process_tree(process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_wave_attention_kernels.py b/test/srt/test_wave_attention_kernels.py new file mode 100644 index 000000000..d4c2ff8e5 --- /dev/null +++ b/test/srt/test_wave_attention_kernels.py @@ -0,0 +1,322 @@ +import random +import unittest + +import torch + +from sglang.srt.layers.attention.triton_ops.decode_attention import ( + decode_attention_fwd_grouped as triton_decode_attention_fwd_grouped, +) +from sglang.srt.layers.attention.triton_ops.extend_attention import ( + extend_attention_fwd, + redundant_attention, +) +from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, +) +from sglang.srt.layers.attention.wave_ops.decode_attention import ( + decode_attention_intermediate_arrays_shapes, + decode_attention_wave, +) +from sglang.srt.layers.attention.wave_ops.extend_attention import extend_attention_wave +from sglang.srt.layers.attention.wave_ops.prefill_attention import ( + prefill_attention_wave, +) + + +class TestWaveAttention(unittest.TestCase): + + def _set_all_seeds(self, seed): + """Set all random seeds for reproducibility.""" + random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + def setUp(self): + # Set seeds before each test method + self._set_all_seeds(42) + + def _test_extend_attention_once(self, B, N_CTX, H_Q, H_KV, D): + dtype = torch.float16 + extend_seq_len = 1024 + + b_seq_len_prefix = torch.full( + (B,), N_CTX // B, dtype=torch.int32, device="cuda" + ) + b_seq_len_extend = torch.full( + (B,), extend_seq_len, dtype=torch.int32, device="cuda" + ) + b_seq_len = b_seq_len_prefix + b_seq_len_extend + max_len_in_batch = torch.max(b_seq_len, 0)[0].item() + + b_req_idx = torch.arange(B, dtype=torch.int32, device="cuda") + b_start_loc = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc[1:] = torch.cumsum(b_seq_len[:-1], 0) + b_start_loc_extend = torch.zeros((B,), dtype=torch.int32, device="cuda") + b_start_loc_extend[1:] = torch.cumsum(b_seq_len_extend[:-1], 0) + + kv_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + kv_indptr[1 : B + 1] = torch.cumsum(b_seq_len_prefix[:B], dim=0) + kv_indices = torch.zeros( + (b_seq_len_prefix.sum().item(),), dtype=torch.int32, device="cuda" + ) + + for i in range(B): + kv_indices[kv_indptr[i] : kv_indptr[i + 1]] = torch.arange( + b_start_loc[i], b_start_loc[i] + b_seq_len_prefix[i] + ) + + total_token_num = torch.sum(b_seq_len).item() + extend_token_num = torch.sum(b_seq_len_extend).item() + k_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + v_buffer = torch.empty( + (total_token_num, H_KV, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + k_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + v_extend = torch.empty((extend_token_num, H_KV, D), dtype=dtype, device="cuda") + q_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + for i in range(B): + extend_start_in_buffer = b_start_loc[i] + b_seq_len_prefix[i] + extend_end_in_buffer = b_start_loc[i] + b_seq_len[i] + extend_start = b_start_loc_extend[i] + extend_end = b_start_loc_extend[i] + b_seq_len_extend[i] + k_extend[extend_start:extend_end] = k_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + v_extend[extend_start:extend_end] = v_buffer[ + extend_start_in_buffer:extend_end_in_buffer + ] + q_extend[extend_start:extend_end] = torch.empty( + (b_seq_len_extend[i], H_Q, D), dtype=dtype, device="cuda" + ).normal_(mean=0.1, std=0.2) + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + o_extend_mask = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + o_redundant = torch.empty( + (extend_token_num, H_Q, D), dtype=dtype, device="cuda" + ) + + b_seq_len_extend = b_seq_len - b_seq_len_prefix + max_len_extend = torch.max(b_seq_len_extend, 0)[0].item() + qo_indptr = torch.zeros((B + 1,), dtype=torch.int32, device="cuda") + qo_indptr[1 : B + 1] = torch.cumsum(b_seq_len_extend[:B], dim=0) + + custom_mask = None + mask_indptr = None + + redundant_attention( + q_extend, + o_redundant, + k_buffer, + v_buffer, + b_req_idx, + b_start_loc, + b_seq_len, + b_seq_len_prefix, + max_len_in_batch, + ) + + is_causal = True + + o_extend = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + extend_attention_fwd( + q_extend, + k_extend, + v_extend, + o_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + is_causal, + mask_indptr, + max_len_extend, + ) + + o_wave = torch.empty((extend_token_num, H_Q, D), dtype=dtype, device="cuda") + extend_attention_wave( + q_extend, + k_extend, + v_extend, + k_buffer, + v_buffer, + qo_indptr, + kv_indptr, + kv_indices, + custom_mask, + mask_indptr, + max_len_extend, + o_wave, + is_causal=is_causal, + ) + + self.assertTrue(torch.allclose(o_extend, o_redundant, rtol=1e-2)) + self.assertTrue(torch.allclose(o_wave, o_redundant, rtol=1e-2)) + + def test_extend_attention(self): + + # Define the varying parameter values + attention_values = [128] + + # Loop through the values and call the method + for value in attention_values: + self._test_extend_attention_once(32, 16384, 6, 1, value) + + def _test_grouped_decode_attention_once(self, B, S, H_Q, H_KV, D, D_V): + dtype = torch.float16 + seq_len = S # This represents the number of tokens already in the sequence + total_tokens = B * seq_len + sm_scale = 1.0 / (D**0.5) + max_kv_splits = 8 + num_kv_splits = torch.full((B,), 4, dtype=torch.int32, device="cuda") + + # q represents the new token being generated, one per batch + q = torch.randn(B, H_Q, D, dtype=dtype, device="cuda") + + # k_buffer and v_buffer represent all previous tokens + k_buffer = torch.randn(total_tokens, H_KV, D, dtype=dtype, device="cuda") + v_buffer = torch.randn(total_tokens, H_KV, D_V, dtype=dtype, device="cuda") + + # o will have the same shape as q + o_triton = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + o = torch.zeros(B, H_Q, D_V, dtype=dtype, device="cuda") + + req_to_token = torch.arange(total_tokens, device="cuda", dtype=torch.int32) + b_req_idx = torch.zeros(B + 1, device="cuda", dtype=torch.int32) + b_seq_len = torch.full((B,), seq_len, device="cuda", dtype=torch.int32) + b_req_idx[1 : B + 1] = torch.cumsum(b_seq_len, dim=0) + + attn_logits = torch.empty( + (B, H_Q, max_kv_splits, D_V + 1), + dtype=torch.float32, + device="cuda", + ) + attn_lse = torch.empty( + (B, H_Q, max_kv_splits), + dtype=torch.float32, + device="cuda", + ) + + logit_cap = 0.0 + triton_decode_attention_fwd_grouped( + q, + k_buffer, + v_buffer, + o_triton, + b_req_idx, + req_to_token, + attn_logits, + attn_lse, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + ) + + attn_logits_shape, attn_logits_max_shape = ( + decode_attention_intermediate_arrays_shapes(B, D_V, H_Q, max_kv_splits) + ) + + attn_logits = torch.empty( + attn_logits_shape, + dtype=torch.float32, + device="cuda", + ) + + attn_logits_max = torch.empty( + attn_logits_max_shape, + dtype=torch.float32, + device="cuda", + ) + + decode_attention_wave( + q, + k_buffer, + v_buffer, + o, + b_req_idx, + req_to_token, + attn_logits, + attn_logits_max, + num_kv_splits, + max_kv_splits, + sm_scale, + logit_cap, + ) + + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_triton.flatten(), dim=0 + ) + print(cos_sim.item()) + self.assertTrue(cos_sim.item() > 0.99) + self.assertTrue(torch.allclose(o, o_triton, atol=3e-2)) + + def test_grouped_decode_attention(self): + seq_lens = [5, 100, 128, 500] + configs = [ + (2, 16, 16, 64, 64), + (2, 16, 1, 64, 64), + (2, 128, 1, 80, 80), + (32, 128, 2, 512, 512), + (2, 128, 2, 512, 512), + (2, 128, 1, 576, 512), + ] + + for S in seq_lens: + for B, H_Q, H_KV, D, D_V in configs: + self._test_grouped_decode_attention_once(B, S, H_Q, H_KV, D, D_V) + + def _test_context_attention_once(self, head_dim, is_causal): + # Set up a simple test case + dtype = torch.float16 + num_heads = 4 + kv_heads = 1 + seq_lens = [128, 256] + max_seq_len = max(seq_lens) + + # Create random input tensors + q = torch.randn(sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda") + k = torch.randn(sum(seq_lens), kv_heads, head_dim, dtype=dtype, device="cuda") + v = torch.randn(sum(seq_lens), kv_heads, head_dim, dtype=dtype, device="cuda") + o_triton = torch.zeros( + sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda" + ) + o = torch.zeros(sum(seq_lens), num_heads, head_dim, dtype=dtype, device="cuda") + + # Create b_start_loc and b_seq_len tensors + b_start_loc = torch.tensor([0, seq_lens[0]], device="cuda") + b_seq_len = torch.tensor(seq_lens, device="cuda") + + context_attention_fwd( + q, k, v, o_triton, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + prefill_attention_wave( + q, k, v, o, b_start_loc, b_seq_len, max_seq_len, is_causal=is_causal + ) + cos_sim = torch.nn.functional.cosine_similarity( + o.flatten(), o_triton.flatten(), dim=0 + ) + + print(cos_sim.item()) + self.assertTrue(torch.allclose(o, o_triton, atol=3e-2)) + self.assertTrue(cos_sim.item() > 1 - (1e-5)) + + def test_context_attention(self): + head_dim = [128, 96] + + for dim in head_dim: + for is_causal in [False]: + self._test_context_attention_once(dim, is_causal) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/srt/test_weight_version.py b/test/srt/test_weight_version.py new file mode 100644 index 000000000..5011ee701 --- /dev/null +++ b/test/srt/test_weight_version.py @@ -0,0 +1,227 @@ +""" +Test weight version functionality. + +This test suite verifies the weight_version feature implementation including: +1. Default weight_version setting +2. /get_weight_version endpoint +3. /update_weight_version endpoint +4. /generate request meta_info contains weight_version +5. OpenAI API response metadata contains weight_version +""" + +import unittest + +import requests + +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + CustomTestCase, + popen_launch_server, +) + + +class TestWeightVersion(CustomTestCase): + @classmethod + def setUpClass(cls): + """Start server once for all tests with custom weight version.""" + cls.model = DEFAULT_SMALL_MODEL_NAME_FOR_TEST + cls.base_url = "http://127.0.0.1:30000" + cls.process = popen_launch_server( + cls.model, + base_url=cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--weight-version", + "test_version_1.0", + "--attention-backend", + "flashinfer", + ], + ) + + @classmethod + def tearDownClass(cls): + """Terminate server after all tests complete.""" + if cls.process: + cls.process.terminate() + + def test_weight_version_comprehensive(self): + """Comprehensive test for all weight_version functionality.""" + + response = requests.get(f"{self.base_url}/get_model_info") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIn("weight_version", data) + self.assertEqual(data["weight_version"], "test_version_1.0") + + response = requests.get(f"{self.base_url}/get_weight_version") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIn("weight_version", data) + self.assertEqual(data["weight_version"], "test_version_1.0") + + request_data = { + "text": "Hello, how are you?", + "sampling_params": { + "temperature": 0.0, + "max_new_tokens": 5, + }, + } + response = requests.post(f"{self.base_url}/generate", json=request_data) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIn("meta_info", data) + self.assertIn("weight_version", data["meta_info"]) + self.assertEqual(data["meta_info"]["weight_version"], "test_version_1.0") + + request_data = { + "model": self.model, + "messages": [{"role": "user", "content": "Hello"}], + "max_tokens": 5, + "temperature": 0.0, + } + response = requests.post( + f"{self.base_url}/v1/chat/completions", json=request_data + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIn("metadata", data) + self.assertIn("weight_version", data["metadata"]) + self.assertEqual(data["metadata"]["weight_version"], "test_version_1.0") + + request_data = { + "model": self.model, + "prompt": "Hello", + "max_tokens": 5, + "temperature": 0.0, + } + response = requests.post(f"{self.base_url}/v1/completions", json=request_data) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertIn("metadata", data) + self.assertIn("weight_version", data["metadata"]) + self.assertEqual(data["metadata"]["weight_version"], "test_version_1.0") + + update_data = { + "new_version": "updated_version_2.0", + "abort_all_requests": False, + } + response = requests.post( + f"{self.base_url}/update_weight_version", json=update_data + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertTrue(data["success"]) + self.assertEqual(data["new_version"], "updated_version_2.0") + + response = requests.get(f"{self.base_url}/get_weight_version") + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["weight_version"], "updated_version_2.0") + + gen_data = { + "text": "Test persistence", + "sampling_params": {"temperature": 0.0, "max_new_tokens": 3}, + } + response = requests.post(f"{self.base_url}/generate", json=gen_data) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["meta_info"]["weight_version"], "updated_version_2.0") + + chat_data = { + "model": self.model, + "messages": [{"role": "user", "content": "Test"}], + "max_tokens": 3, + "temperature": 0.0, + } + response = requests.post(f"{self.base_url}/v1/chat/completions", json=chat_data) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertEqual(data["metadata"]["weight_version"], "updated_version_2.0") + + update_data = {"new_version": "final_version_3.0", "abort_all_requests": True} + response = requests.post( + f"{self.base_url}/update_weight_version", json=update_data + ) + self.assertEqual(response.status_code, 200) + data = response.json() + self.assertTrue(data["success"]) + self.assertEqual(data["new_version"], "final_version_3.0") + + # Check /get_weight_version + response = requests.get(f"{self.base_url}/get_weight_version") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["weight_version"], "final_version_3.0") + + # Check /get_model_info + response = requests.get(f"{self.base_url}/get_model_info") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["weight_version"], "final_version_3.0") + + # Check /generate meta_info + response = requests.post( + f"{self.base_url}/generate", + json={ + "text": "Final test", + "sampling_params": {"temperature": 0.0, "max_new_tokens": 2}, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.json()["meta_info"]["weight_version"], "final_version_3.0" + ) + + # Check OpenAI chat metadata + response = requests.post( + f"{self.base_url}/v1/chat/completions", + json={ + "model": self.model, + "messages": [{"role": "user", "content": "Final"}], + "max_tokens": 2, + "temperature": 0.0, + }, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual( + response.json()["metadata"]["weight_version"], "final_version_3.0" + ) + + print("All weight_version functionality tests passed!") + + def test_update_weight_version_with_weight_updates(self): + """Test that weight_version can be updated along with weight updates using real model data.""" + print("Testing weight_version update with real weight operations...") + + # Get current model info for reference + model_info_response = requests.get(f"{self.base_url}/get_model_info") + self.assertEqual(model_info_response.status_code, 200) + current_model_path = model_info_response.json()["model_path"] + + update_data = { + "model_path": current_model_path, + "load_format": "auto", + "abort_all_requests": False, + "weight_version": "disk_update_v2.0.0", + } + + response = requests.post( + f"{self.base_url}/update_weights_from_disk", json=update_data + ) + self.assertEqual( + response.status_code, + 200, + f"update_weights_from_disk failed with status {response.status_code}", + ) + + # Verify version was updated + version_response = requests.get(f"{self.base_url}/get_weight_version") + self.assertEqual(version_response.status_code, 200) + self.assertEqual( + version_response.json()["weight_version"], "disk_update_v2.0.0" + ) + + print("Weight update with weight_version test completed!") + + +if __name__ == "__main__": + unittest.main()